History

avcodec/d3d12va_decode: enable reference-only mode

The Reference-Only feature in DirectX 12 is a memory optimization technique designed for video decoding scenarios. This feature requires that reference resources must be allocated with the D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY resource flag. Reference textures must also be separated from output textures. This feature is not supported in the current version of ffmpeg. Since AMD GPU uses this feature in Direct 12 decoder, ffmpeg does not support AMD GPU Direct 12 decoding.

Araz Iusubov committed 1 year ago

Commit ccf073e1 parent 37a319f2 files · +165 −12Browse files

Unified · Split

2 files
  1. libavcodec/d3d12va_decode.c +152 −12
  2. libavcodec/d3d12va_decode.h +13 −0
libavcodec/d3d12va_decode.c+152−12
@@ -41,6 +41,101 @@ typedef struct HelperObjects { · 40 unmodified lines
4141 uint64_t fence_value;
4242 } HelperObjects;
4343
44+typedef struct ReferenceFrame {
45+ ID3D12Resource *resource;
46+ int used;
47+ ID3D12Resource *output_resource;
48+} ReferenceFrame;
49+
50+static ID3D12Resource *get_reference_only_resource(AVCodecContext *avctx, ID3D12Resource *output_resource)
51+{
52+ D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx);
53+ AVHWFramesContext *frames_ctx = D3D12VA_FRAMES_CONTEXT(avctx);
54+ AVD3D12VADeviceContext *device_hwctx = ctx->device_ctx;
55+ int i = 0;
56+ ID3D12Resource *resource = NULL;
57+ D3D12_HEAP_PROPERTIES props = { .Type = D3D12_HEAP_TYPE_DEFAULT };
58+ D3D12_RESOURCE_DESC desc;
59+ ReferenceFrame *reference_only_map = ctx->reference_only_map;
60+ if (reference_only_map == NULL) {
61+ av_log(avctx, AV_LOG_ERROR, "Reference frames are not allocated!\n");
62+ return NULL;
63+ }
64+
65+ // find unused resource
66+ for (i = 0; i < ctx->max_num_ref; i++) {
67+ if (!reference_only_map[i].used && reference_only_map[i].resource != NULL) {
68+ reference_only_map[i].used = 1;
69+ resource = reference_only_map[i].resource;
70+ reference_only_map[i].output_resource = output_resource;
71+ return resource;
72+ }
73+ }
74+
75+ // find space to allocate
76+ for (i = 0; i < ctx->max_num_ref; i++) {
77+ if (reference_only_map[i].resource == NULL)
78+ break;
79+ }
80+
81+ if (i == ctx->max_num_ref) {
82+ av_log(avctx, AV_LOG_ERROR, "No space for new Reference frame!\n");
83+ return NULL;
84+ }
85+
86+ // allocate frame
87+ output_resource->lpVtbl->GetDesc(output_resource, &desc);
88+ desc.Flags = D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY | D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE;
89+
90+ if (FAILED(ID3D12Device_CreateCommittedResource(device_hwctx->device, &props, D3D12_HEAP_FLAG_NONE, &desc,
91+ D3D12_RESOURCE_STATE_COMMON, NULL, &IID_ID3D12Resource, (void **)&reference_only_map[i].resource))) {
92+ av_log(ctx, AV_LOG_ERROR, "Failed to create D3D12 Reference Resource!\n");
93+ return NULL;
94+ }
95+
96+ reference_only_map[i].used = 1;
97+ resource = reference_only_map[i].resource;
98+ reference_only_map[i].output_resource = output_resource;
99+
100+ return resource;
101+}
102+
103+static void free_reference_only_resources(AVCodecContext *avctx)
104+{
105+ D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx);
106+ int i;
107+ ReferenceFrame *reference_only_map = ctx->reference_only_map;
108+ if (reference_only_map != NULL) {
109+ for (i = 0; i < ctx->max_num_ref; i++) {
110+ if (reference_only_map[i].resource != NULL) {
111+ D3D12_OBJECT_RELEASE(reference_only_map[i].resource);
112+ }
113+ }
114+ av_freep(&ctx->reference_only_map);
115+ av_freep(&ctx->ref_only_resources);
116+ }
117+}
118+
119+static void prepare_reference_only_resources(AVCodecContext *avctx)
120+{
121+ D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx);
122+ int i, j;
123+ ReferenceFrame *reference_only_map = ctx->reference_only_map;
124+ if (reference_only_map == NULL)
125+ return;
126+ memset(ctx->ref_only_resources, 0, ctx->max_num_ref * sizeof(*(ctx->ref_only_resources)));
127+ for (j = 0; j < ctx->max_num_ref; j++) {
128+ for (i = 0; i < ctx->max_num_ref; i++) {
129+ if (reference_only_map[j].used && reference_only_map[j].output_resource == ctx->ref_resources[i]) {
130+ ctx->ref_only_resources[i] = reference_only_map[j].resource;
131+ break;
132+ }
133+ }
134+ if (i == ctx->max_num_ref)
135+ reference_only_map[j].used = 0;
136+ }
137+}
138+
44139 int ff_d3d12va_get_suitable_max_bitstream_size(AVCodecContext *avctx)
45140 {
46141 AVHWFramesContext *frames_ctx = D3D12VA_FRAMES_CONTEXT(avctx);
@@ -250,6 +345,18 @@ static int d3d12va_create_decoder(AVCodecContext *avctx) · 203 unmodified lines
250345 return AVERROR_PATCHWELCOME;
251346 }
252347
348+ ctx->reference_only_map = NULL;
349+ ctx->ref_only_resources = NULL;
350+ if (feature.ConfigurationFlags & D3D12_VIDEO_DECODE_CONFIGURATION_FLAG_REFERENCE_ONLY_ALLOCATIONS_REQUIRED) {
351+ av_log(avctx, AV_LOG_VERBOSE, "Reference-Only Allocations are required for this D3D12 decoder configuration.\n");
352+ ctx->reference_only_map = av_calloc(ctx->max_num_ref + 1, sizeof(ReferenceFrame));
353+ if (!ctx->reference_only_map)
354+ return AVERROR(ENOMEM);
355+ ctx->ref_only_resources = av_calloc(ctx->max_num_ref, sizeof(*ctx->ref_only_resources));
356+ if (!ctx->ref_only_resources)
357+ return AVERROR(ENOMEM);
358+ }
359+
253360 desc = (D3D12_VIDEO_DECODER_DESC) {
254361 .NodeMask = 0,
255362 .Configuration = ctx->cfg,
@@ -394,6 +501,7 @@ av_cold int ff_d3d12va_decode_uninit(AVCodecContext *avctx) · 138 unmodified lines
394501
395502 av_log(avctx, AV_LOG_VERBOSE, "Total number of command allocators reused: %d\n", num_allocator);
396503 }
504+ free_reference_only_resources(avctx);
397505
398506 av_fifo_freep2(&ctx->objects_queue);
399507
@@ -412,14 +520,15 @@ static inline int d3d12va_update_reference_frames_state(AVCodecContext *avctx, D · 12 unmodified lines
412520 ID3D12Resource *current_resource, int state_before, int state_end)
413521 {
414522 D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx);
523+ ID3D12Resource **ref_resources = ctx->ref_only_resources ? ctx->ref_only_resources : ctx->ref_resources;
415524
416525 int num_barrier = 0;
417526 for (int i = 0; i < ctx->max_num_ref; i++) {
418− if (((ctx->used_mask >> i) & 0x1) && ctx->ref_resources[i] && ctx->ref_resources[i] != current_resource) {
527+ if (((ctx->used_mask >> i) & 0x1) && ref_resources[i] && ref_resources[i] != current_resource) {
419528 barriers[num_barrier].Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
420529 barriers[num_barrier].Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE;
421− barriers[num_barrier].Transition = (D3D12_RESOURCE_TRANSITION_BARRIER){
422− .pResource = ctx->ref_resources[i],
530+ barriers[num_barrier].Transition = (D3D12_RESOURCE_TRANSITION_BARRIER) {
531+ .pResource = ref_resources[i],
423532 .Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES,
424533 .StateBefore = state_before,
425534 .StateAfter = state_end,
@@ -440,8 +549,9 @@ int ff_d3d12va_common_end_frame(AVCodecContext *avctx, AVFrame *frame, · 14 unmodified lines
440549 D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx);
441550 ID3D12Resource *buffer = NULL;
442551 ID3D12CommandAllocator *command_allocator = NULL;
443− AVD3D12VAFrame *f = (AVD3D12VAFrame *)frame->data[0];
444− ID3D12Resource *resource = (ID3D12Resource *)f->texture;
552+ AVD3D12VAFrame *f = (AVD3D12VAFrame*)frame->data[0];
553+ ID3D12Resource *output_resource = (ID3D12Resource*)f->texture;
554+ ID3D12Resource *ref_resource = NULL;
445555
446556 ID3D12VideoDecodeCommandList *cmd_list = ctx->command_list;
447557 D3D12_RESOURCE_BARRIER barriers[32] = { 0 };
@@ -466,25 +576,55 @@ int ff_d3d12va_common_end_frame(AVCodecContext *avctx, AVFrame *frame, · 18 unmodified lines
466576 D3D12_VIDEO_DECODE_OUTPUT_STREAM_ARGUMENTS output_args = {
467577 .ConversionArguments = { 0 },
468578 .OutputSubresource = 0,
469− .pOutputTexture2D = resource,
579+ .pOutputTexture2D = output_resource,
470580 };
471581
582+ memset(ctx->ref_subresources, 0, sizeof(UINT) * ctx->max_num_ref);
583+ input_args.ReferenceFrames.NumTexture2Ds = ctx->max_num_ref;
584+ input_args.ReferenceFrames.pSubresources = ctx->ref_subresources;
585+
586+ if (ctx->reference_only_map) {
587+ ref_resource = get_reference_only_resource(avctx, output_resource);
588+ if (ref_resource == NULL) {
589+ av_log(avctx, AV_LOG_ERROR, "Failed to get reference frame!\n");
590+ goto fail;
591+ }
592+ prepare_reference_only_resources(avctx);
593+
594+ output_args.ConversionArguments.Enable = 1;
595+ input_args.ReferenceFrames.ppTexture2Ds = ctx->ref_only_resources;
596+ output_args.ConversionArguments.pReferenceTexture2D = ref_resource;
597+ output_args.ConversionArguments.ReferenceSubresource = 0;
598+ } else {
599+ ref_resource = output_resource;
600+ input_args.ReferenceFrames.ppTexture2Ds = ctx->ref_resources;
601+ }
602+
472603 UINT num_barrier = 1;
473604 barriers[0] = (D3D12_RESOURCE_BARRIER) {
474605 .Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION,
475606 .Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE,
476607 .Transition = {
477− .pResource = resource,
608+ .pResource = output_resource,
478609 .Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES,
479610 .StateBefore = D3D12_RESOURCE_STATE_COMMON,
480611 .StateAfter = D3D12_RESOURCE_STATE_VIDEO_DECODE_WRITE,
481612 },
482613 };
483614
484− memset(ctx->ref_subresources, 0, sizeof(UINT) * ctx->max_num_ref);
485− input_args.ReferenceFrames.NumTexture2Ds = ctx->max_num_ref;
486− input_args.ReferenceFrames.ppTexture2Ds = ctx->ref_resources;
487− input_args.ReferenceFrames.pSubresources = ctx->ref_subresources;
615+ if (ctx->reference_only_map) {
616+ barriers[1] = (D3D12_RESOURCE_BARRIER) {
617+ .Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION,
618+ .Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE,
619+ .Transition = {
620+ .pResource = ref_resource,
621+ .Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES,
622+ .StateBefore = D3D12_RESOURCE_STATE_COMMON,
623+ .StateAfter = D3D12_RESOURCE_STATE_VIDEO_DECODE_WRITE,
624+ },
625+ };
626+ num_barrier++;
627+ }
488628
489629 ret = d3d12va_fence_completion(&f->sync_ctx);
490630 if (ret < 0)
@@ -505,7 +645,7 @@ int ff_d3d12va_common_end_frame(AVCodecContext *avctx, AVFrame *frame, · 14 unmodified lines
505645
506646 DX_CHECK(ID3D12VideoDecodeCommandList_Reset(cmd_list, command_allocator));
507647
508− num_barrier += d3d12va_update_reference_frames_state(avctx, &barriers[1], resource, D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_VIDEO_DECODE_READ);
648+ num_barrier += d3d12va_update_reference_frames_state(avctx, &barriers[num_barrier], ref_resource, D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_VIDEO_DECODE_READ);
509649
510650 ID3D12VideoDecodeCommandList_ResourceBarrier(cmd_list, num_barrier, barriers);
511651
libavcodec/d3d12va_decode.h+13−0
@@ -119,6 +119,19 @@ typedef struct D3D12VADecodeContext { · 118 unmodified lines
119119 * Private to the FFmpeg AVHWAccel implementation
120120 */
121121 unsigned report_id;
122+
123+ /**
124+ * The Reference-Only feature in DirectX 12 is a memory optimization
125+ * technique designed for video decoding/encoding scenarios.
126+ * This feature requires that reference resources must be allocated
127+ * with the `D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY` resource flag.
128+ * Reference textures must also be separated from output textures.
129+ * reference_only_map used as a storage for reference only frames
130+ * ref_only_resources used as a shadow for ref_resources
131+ */
132+ void *reference_only_map;
133+ ID3D12Resource **ref_only_resources;
134+
122135 } D3D12VADecodeContext;
123136
124137 /**

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t