History

avcodec/d3d12va_decode: enable reference-only mode

The Reference-Only feature in DirectX 12 is a memory optimization technique designed for video decoding scenarios. This feature requires that reference resources must be allocated with the D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY resource flag. Reference textures must also be separated from output textures. This feature is not supported in the current version of ffmpeg. Since AMD GPU uses this feature in Direct 12 decoder, ffmpeg does not support AMD GPU Direct 12 decoding.

Araz Iusubov committed 1 year ago

Commit ccf073e1 parent 37a319f2 files · +165 −12Browse files

Unified · Split

2 files
  1. libavcodec/d3d12va_decode.c +152 −12
  2. libavcodec/d3d12va_decode.h +13 −0
libavcodec/d3d12va_decode.c+152−12
@@ -41,6 +41,101 @@ typedef struct HelperObjects { · 40 unmodified lines
41 uint64_t fence_value;41 uint64_t fence_value;
42} HelperObjects;42} HelperObjects;
43 43
44typedef struct ReferenceFrame {
45 ID3D12Resource *resource;
46 int used;
47 ID3D12Resource *output_resource;
48} ReferenceFrame;
49
50static ID3D12Resource *get_reference_only_resource(AVCodecContext *avctx, ID3D12Resource *output_resource)
51{
52 D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx);
53 AVHWFramesContext *frames_ctx = D3D12VA_FRAMES_CONTEXT(avctx);
54 AVD3D12VADeviceContext *device_hwctx = ctx->device_ctx;
55 int i = 0;
56 ID3D12Resource *resource = NULL;
57 D3D12_HEAP_PROPERTIES props = { .Type = D3D12_HEAP_TYPE_DEFAULT };
58 D3D12_RESOURCE_DESC desc;
59 ReferenceFrame *reference_only_map = ctx->reference_only_map;
60 if (reference_only_map == NULL) {
61 av_log(avctx, AV_LOG_ERROR, "Reference frames are not allocated!\n");
62 return NULL;
63 }
64
65 // find unused resource
66 for (i = 0; i < ctx->max_num_ref; i++) {
67 if (!reference_only_map[i].used && reference_only_map[i].resource != NULL) {
68 reference_only_map[i].used = 1;
69 resource = reference_only_map[i].resource;
70 reference_only_map[i].output_resource = output_resource;
71 return resource;
72 }
73 }
74
75 // find space to allocate
76 for (i = 0; i < ctx->max_num_ref; i++) {
77 if (reference_only_map[i].resource == NULL)
78 break;
79 }
80
81 if (i == ctx->max_num_ref) {
82 av_log(avctx, AV_LOG_ERROR, "No space for new Reference frame!\n");
83 return NULL;
84 }
85
86 // allocate frame
87 output_resource->lpVtbl->GetDesc(output_resource, &desc);
88 desc.Flags = D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY | D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE;
89
90 if (FAILED(ID3D12Device_CreateCommittedResource(device_hwctx->device, &props, D3D12_HEAP_FLAG_NONE, &desc,
91 D3D12_RESOURCE_STATE_COMMON, NULL, &IID_ID3D12Resource, (void **)&reference_only_map[i].resource))) {
92 av_log(ctx, AV_LOG_ERROR, "Failed to create D3D12 Reference Resource!\n");
93 return NULL;
94 }
95
96 reference_only_map[i].used = 1;
97 resource = reference_only_map[i].resource;
98 reference_only_map[i].output_resource = output_resource;
99
100 return resource;
101}
102
103static void free_reference_only_resources(AVCodecContext *avctx)
104{
105 D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx);
106 int i;
107 ReferenceFrame *reference_only_map = ctx->reference_only_map;
108 if (reference_only_map != NULL) {
109 for (i = 0; i < ctx->max_num_ref; i++) {
110 if (reference_only_map[i].resource != NULL) {
111 D3D12_OBJECT_RELEASE(reference_only_map[i].resource);
112 }
113 }
114 av_freep(&ctx->reference_only_map);
115 av_freep(&ctx->ref_only_resources);
116 }
117}
118
119static void prepare_reference_only_resources(AVCodecContext *avctx)
120{
121 D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx);
122 int i, j;
123 ReferenceFrame *reference_only_map = ctx->reference_only_map;
124 if (reference_only_map == NULL)
125 return;
126 memset(ctx->ref_only_resources, 0, ctx->max_num_ref * sizeof(*(ctx->ref_only_resources)));
127 for (j = 0; j < ctx->max_num_ref; j++) {
128 for (i = 0; i < ctx->max_num_ref; i++) {
129 if (reference_only_map[j].used && reference_only_map[j].output_resource == ctx->ref_resources[i]) {
130 ctx->ref_only_resources[i] = reference_only_map[j].resource;
131 break;
132 }
133 }
134 if (i == ctx->max_num_ref)
135 reference_only_map[j].used = 0;
136 }
137}
138
44int ff_d3d12va_get_suitable_max_bitstream_size(AVCodecContext *avctx)139int ff_d3d12va_get_suitable_max_bitstream_size(AVCodecContext *avctx)
45{140{
46 AVHWFramesContext *frames_ctx = D3D12VA_FRAMES_CONTEXT(avctx);141 AVHWFramesContext *frames_ctx = D3D12VA_FRAMES_CONTEXT(avctx);
@@ -250,6 +345,18 @@ static int d3d12va_create_decoder(AVCodecContext *avctx) · 203 unmodified lines
250 return AVERROR_PATCHWELCOME;345 return AVERROR_PATCHWELCOME;
251 }346 }
252 347
348 ctx->reference_only_map = NULL;
349 ctx->ref_only_resources = NULL;
350 if (feature.ConfigurationFlags & D3D12_VIDEO_DECODE_CONFIGURATION_FLAG_REFERENCE_ONLY_ALLOCATIONS_REQUIRED) {
351 av_log(avctx, AV_LOG_VERBOSE, "Reference-Only Allocations are required for this D3D12 decoder configuration.\n");
352 ctx->reference_only_map = av_calloc(ctx->max_num_ref + 1, sizeof(ReferenceFrame));
353 if (!ctx->reference_only_map)
354 return AVERROR(ENOMEM);
355 ctx->ref_only_resources = av_calloc(ctx->max_num_ref, sizeof(*ctx->ref_only_resources));
356 if (!ctx->ref_only_resources)
357 return AVERROR(ENOMEM);
358 }
359
253 desc = (D3D12_VIDEO_DECODER_DESC) {360 desc = (D3D12_VIDEO_DECODER_DESC) {
254 .NodeMask = 0,361 .NodeMask = 0,
255 .Configuration = ctx->cfg,362 .Configuration = ctx->cfg,
@@ -394,6 +501,7 @@ av_cold int ff_d3d12va_decode_uninit(AVCodecContext *avctx) · 138 unmodified lines
394 501
395 av_log(avctx, AV_LOG_VERBOSE, "Total number of command allocators reused: %d\n", num_allocator);502 av_log(avctx, AV_LOG_VERBOSE, "Total number of command allocators reused: %d\n", num_allocator);
396 }503 }
504 free_reference_only_resources(avctx);
397 505
398 av_fifo_freep2(&ctx->objects_queue);506 av_fifo_freep2(&ctx->objects_queue);
399 507
@@ -412,14 +520,15 @@ static inline int d3d12va_update_reference_frames_state(AVCodecContext *avctx, D · 12 unmodified lines
412 ID3D12Resource *current_resource, int state_before, int state_end)520 ID3D12Resource *current_resource, int state_before, int state_end)
413{521{
414 D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx);522 D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx);
523 ID3D12Resource **ref_resources = ctx->ref_only_resources ? ctx->ref_only_resources : ctx->ref_resources;
415 524
416 int num_barrier = 0;525 int num_barrier = 0;
417 for (int i = 0; i < ctx->max_num_ref; i++) {526 for (int i = 0; i < ctx->max_num_ref; i++) {
418 if (((ctx->used_mask >> i) & 0x1) && ctx->ref_resources[i] && ctx->ref_resources[i] != current_resource) {527 if (((ctx->used_mask >> i) & 0x1) && ref_resources[i] && ref_resources[i] != current_resource) {
419 barriers[num_barrier].Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;528 barriers[num_barrier].Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
420 barriers[num_barrier].Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE;529 barriers[num_barrier].Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE;
421 barriers[num_barrier].Transition = (D3D12_RESOURCE_TRANSITION_BARRIER){530 barriers[num_barrier].Transition = (D3D12_RESOURCE_TRANSITION_BARRIER) {
422 .pResource = ctx->ref_resources[i],531 .pResource = ref_resources[i],
423 .Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES,532 .Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES,
424 .StateBefore = state_before,533 .StateBefore = state_before,
425 .StateAfter = state_end,534 .StateAfter = state_end,
@@ -440,8 +549,9 @@ int ff_d3d12va_common_end_frame(AVCodecContext *avctx, AVFrame *frame, · 14 unmodified lines
440 D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx);549 D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx);
441 ID3D12Resource *buffer = NULL;550 ID3D12Resource *buffer = NULL;
442 ID3D12CommandAllocator *command_allocator = NULL;551 ID3D12CommandAllocator *command_allocator = NULL;
443 AVD3D12VAFrame *f = (AVD3D12VAFrame *)frame->data[0];552 AVD3D12VAFrame *f = (AVD3D12VAFrame*)frame->data[0];
444 ID3D12Resource *resource = (ID3D12Resource *)f->texture;553 ID3D12Resource *output_resource = (ID3D12Resource*)f->texture;
554 ID3D12Resource *ref_resource = NULL;
445 555
446 ID3D12VideoDecodeCommandList *cmd_list = ctx->command_list;556 ID3D12VideoDecodeCommandList *cmd_list = ctx->command_list;
447 D3D12_RESOURCE_BARRIER barriers[32] = { 0 };557 D3D12_RESOURCE_BARRIER barriers[32] = { 0 };
@@ -466,25 +576,55 @@ int ff_d3d12va_common_end_frame(AVCodecContext *avctx, AVFrame *frame, · 18 unmodified lines
466 D3D12_VIDEO_DECODE_OUTPUT_STREAM_ARGUMENTS output_args = {576 D3D12_VIDEO_DECODE_OUTPUT_STREAM_ARGUMENTS output_args = {
467 .ConversionArguments = { 0 },577 .ConversionArguments = { 0 },
468 .OutputSubresource = 0,578 .OutputSubresource = 0,
469 .pOutputTexture2D = resource,579 .pOutputTexture2D = output_resource,
470 };580 };
471 581
582 memset(ctx->ref_subresources, 0, sizeof(UINT) * ctx->max_num_ref);
583 input_args.ReferenceFrames.NumTexture2Ds = ctx->max_num_ref;
584 input_args.ReferenceFrames.pSubresources = ctx->ref_subresources;
585
586 if (ctx->reference_only_map) {
587 ref_resource = get_reference_only_resource(avctx, output_resource);
588 if (ref_resource == NULL) {
589 av_log(avctx, AV_LOG_ERROR, "Failed to get reference frame!\n");
590 goto fail;
591 }
592 prepare_reference_only_resources(avctx);
593
594 output_args.ConversionArguments.Enable = 1;
595 input_args.ReferenceFrames.ppTexture2Ds = ctx->ref_only_resources;
596 output_args.ConversionArguments.pReferenceTexture2D = ref_resource;
597 output_args.ConversionArguments.ReferenceSubresource = 0;
598 } else {
599 ref_resource = output_resource;
600 input_args.ReferenceFrames.ppTexture2Ds = ctx->ref_resources;
601 }
602
472 UINT num_barrier = 1;603 UINT num_barrier = 1;
473 barriers[0] = (D3D12_RESOURCE_BARRIER) {604 barriers[0] = (D3D12_RESOURCE_BARRIER) {
474 .Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION,605 .Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION,
475 .Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE,606 .Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE,
476 .Transition = {607 .Transition = {
477 .pResource = resource,608 .pResource = output_resource,
478 .Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES,609 .Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES,
479 .StateBefore = D3D12_RESOURCE_STATE_COMMON,610 .StateBefore = D3D12_RESOURCE_STATE_COMMON,
480 .StateAfter = D3D12_RESOURCE_STATE_VIDEO_DECODE_WRITE,611 .StateAfter = D3D12_RESOURCE_STATE_VIDEO_DECODE_WRITE,
481 },612 },
482 };613 };
483 614
484 memset(ctx->ref_subresources, 0, sizeof(UINT) * ctx->max_num_ref);615 if (ctx->reference_only_map) {
485 input_args.ReferenceFrames.NumTexture2Ds = ctx->max_num_ref;616 barriers[1] = (D3D12_RESOURCE_BARRIER) {
486 input_args.ReferenceFrames.ppTexture2Ds = ctx->ref_resources;617 .Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION,
487 input_args.ReferenceFrames.pSubresources = ctx->ref_subresources;618 .Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE,
619 .Transition = {
620 .pResource = ref_resource,
621 .Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES,
622 .StateBefore = D3D12_RESOURCE_STATE_COMMON,
623 .StateAfter = D3D12_RESOURCE_STATE_VIDEO_DECODE_WRITE,
624 },
625 };
626 num_barrier++;
627 }
488 628
489 ret = d3d12va_fence_completion(&f->sync_ctx);629 ret = d3d12va_fence_completion(&f->sync_ctx);
490 if (ret < 0)630 if (ret < 0)
@@ -505,7 +645,7 @@ int ff_d3d12va_common_end_frame(AVCodecContext *avctx, AVFrame *frame, · 14 unmodified lines
505 645
506 DX_CHECK(ID3D12VideoDecodeCommandList_Reset(cmd_list, command_allocator));646 DX_CHECK(ID3D12VideoDecodeCommandList_Reset(cmd_list, command_allocator));
507 647
508 num_barrier += d3d12va_update_reference_frames_state(avctx, &barriers[1], resource, D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_VIDEO_DECODE_READ);648 num_barrier += d3d12va_update_reference_frames_state(avctx, &barriers[num_barrier], ref_resource, D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_VIDEO_DECODE_READ);
509 649
510 ID3D12VideoDecodeCommandList_ResourceBarrier(cmd_list, num_barrier, barriers);650 ID3D12VideoDecodeCommandList_ResourceBarrier(cmd_list, num_barrier, barriers);
511 651
libavcodec/d3d12va_decode.h+13−0
@@ -119,6 +119,19 @@ typedef struct D3D12VADecodeContext { · 118 unmodified lines
119 * Private to the FFmpeg AVHWAccel implementation119 * Private to the FFmpeg AVHWAccel implementation
120 */120 */
121 unsigned report_id;121 unsigned report_id;
122
123 /**
124 * The Reference-Only feature in DirectX 12 is a memory optimization
125 * technique designed for video decoding/encoding scenarios.
126 * This feature requires that reference resources must be allocated
127 * with the `D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY` resource flag.
128 * Reference textures must also be separated from output textures.
129 * reference_only_map used as a storage for reference only frames
130 * ref_only_resources used as a shadow for ref_resources
131 */
132 void *reference_only_map;
133 ID3D12Resource **ref_only_resources;
134
122} D3D12VADecodeContext;135} D3D12VADecodeContext;
123 136
124/**137/**

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t