History

avcodec/mfenc: add support for D3D11 input surfaces

Adds D3D11 input surface support to the MediaFoundation encoder (mfenc), allowing direct encoding of GPU frames without readback to system memory. This improves performance and compatibility when used alongside scale_d3d11.

Dash Santosh committed 1 year ago

Commit d56522c1 parent 96821212 files · +185 −29Browse files

Showing the changes to libavcodec (2 of 2 files). Show all

Unified · Split

2 files
  1. libavcodec/mf_utils.h +7 −0
  2. libavcodec/mfenc.c +178 −29
libavcodec/mf_utils.h+7−0
@@ -53,6 +53,13 @@ typedef struct MFFunctions { · 52 unmodified lines
53 IMFMediaBuffer **ppBuffer);53 IMFMediaBuffer **ppBuffer);
54 HRESULT (WINAPI *MFCreateSample) (IMFSample **ppIMFSample);54 HRESULT (WINAPI *MFCreateSample) (IMFSample **ppIMFSample);
55 HRESULT (WINAPI *MFCreateMediaType) (IMFMediaType **ppMFType);55 HRESULT (WINAPI *MFCreateMediaType) (IMFMediaType **ppMFType);
56 HRESULT (WINAPI *MFCreateDXGISurfaceBuffer) (REFIID riid,
57 IUnknown* punkSurface,
58 UINT uSubresourceIndex,
59 BOOL fBottomUpWhenLinear,
60 IMFMediaBuffer** ppBuffer);
61 HRESULT (WINAPI *MFCreateDXGIDeviceManager) (UINT* resetToken,
62 IMFDXGIDeviceManager** ppDeviceManager);
56 // MFTEnumEx is missing in Windows Vista's mfplat.dll.63 // MFTEnumEx is missing in Windows Vista's mfplat.dll.
57 HRESULT (WINAPI *MFTEnumEx)(GUID guidCategory, UINT32 Flags,64 HRESULT (WINAPI *MFTEnumEx)(GUID guidCategory, UINT32 Flags,
58 const MFT_REGISTER_TYPE_INFO *pInputType,65 const MFT_REGISTER_TYPE_INFO *pInputType,
libavcodec/mfenc.c+178−29
@@ -31,10 +31,18 @@ · 30 unmodified lines
31#include "codec_internal.h"31#include "codec_internal.h"
32#include "internal.h"32#include "internal.h"
33#include "compat/w32dlfcn.h"33#include "compat/w32dlfcn.h"
34#if CONFIG_D3D11VA
35#include "libavutil/hwcontext_d3d11va.h"
36#endif
34 37
35typedef struct MFContext {38typedef struct MFContext {
36 AVClass *av_class;39 AVClass *av_class;
37 HMODULE library;40 HMODULE library;
41 HMODULE d3d_dll;
42 ID3D11DeviceContext* d3d_context;
43 IMFDXGIDeviceManager *dxgiManager;
44 int resetToken;
45
38 MFFunctions functions;46 MFFunctions functions;
39 AVFrame *frame;47 AVFrame *frame;
40 int is_video, is_audio;48 int is_video, is_audio;
@@ -47,6 +55,7 @@ typedef struct MFContext { · 6 unmodified lines
47 int out_stream_provides_samples;55 int out_stream_provides_samples;
48 int draining, draining_done;56 int draining, draining_done;
49 int sample_sent;57 int sample_sent;
58 int stream_started;
50 int async_need_input, async_have_output, async_marker;59 int async_need_input, async_have_output, async_marker;
51 int64_t reorder_delay;60 int64_t reorder_delay;
52 ICodecAPI *codec_api;61 ICodecAPI *codec_api;
@@ -55,6 +64,7 @@ typedef struct MFContext { · 2 unmodified lines
55 int opt_enc_quality;64 int opt_enc_quality;
56 int opt_enc_scenario;65 int opt_enc_scenario;
57 int opt_enc_hw;66 int opt_enc_hw;
67 AVD3D11VADeviceContext* device_hwctx;
58} MFContext;68} MFContext;
59 69
60static int mf_choose_output_type(AVCodecContext *avctx);70static int mf_choose_output_type(AVCodecContext *avctx);
@@ -303,36 +313,118 @@ static IMFSample *mf_a_avframe_to_sample(AVCodecContext *avctx, const AVFrame *f · 242 unmodified lines
303 return sample;313 return sample;
304}314}
305 315
306static IMFSample *mf_v_avframe_to_sample(AVCodecContext *avctx, const AVFrame *frame)316static int initialize_dxgi_manager(AVCodecContext *avctx)
307{317{
308 MFContext *c = avctx->priv_data;318 MFContext *c = avctx->priv_data;
309 IMFSample *sample;319 MFFunctions *func = &c->functions;
310 IMFMediaBuffer *buffer;
311 BYTE *data;
312 HRESULT hr;320 HRESULT hr;
313 int ret;321
314 int size;322 hr = func->MFCreateDXGIDeviceManager(&c->resetToken, &c->dxgiManager);
323 if (FAILED(hr)) {
324 av_log(avctx, AV_LOG_ERROR, "Failed to create DXGI device manager: %s\n", ff_hr_str(hr));
325 return AVERROR_EXTERNAL;
326 }
327
328 hr = IMFDXGIDeviceManager_ResetDevice(c->dxgiManager, c->device_hwctx->device, c->resetToken);
329 if (FAILED(hr)) {
330 av_log(avctx, AV_LOG_ERROR, "Failed to reset device: %s\n", ff_hr_str(hr));
331 return AVERROR_EXTERNAL;
332 }
333
334 hr = IMFTransform_ProcessMessage(c->mft, MFT_MESSAGE_SET_D3D_MANAGER, (ULONG_PTR)c->dxgiManager);
335 if (FAILED(hr)) {
336 av_log(avctx, AV_LOG_ERROR, "Failed to set D3D manager: %s\n", ff_hr_str(hr));
337 return AVERROR_EXTERNAL;
338 }
339
340 return 0;
341}
342
343static int process_d3d11_frame(AVCodecContext *avctx, const AVFrame *frame, IMFSample **out_sample)
344{
345 MFContext *c = avctx->priv_data;
346 MFFunctions *func = &c->functions;
347 AVHWFramesContext *frames_ctx = NULL;
348 ID3D11Texture2D *d3d11_texture = NULL;
349 IMFSample *sample = NULL;
350 IMFMediaBuffer *buffer = NULL;
351 int subIdx = 0;
352 HRESULT hr;
353
354 frames_ctx = (AVHWFramesContext*)frame->hw_frames_ctx->data;
355 c->device_hwctx = (AVD3D11VADeviceContext*)frames_ctx->device_ctx->hwctx;
356
357 if (!c->dxgiManager) {
358 hr = initialize_dxgi_manager(avctx);
359 if (FAILED(hr)) {
360 return AVERROR_EXTERNAL;
361 }
362 }
363
364 d3d11_texture = (ID3D11Texture2D*)frame->data[0];
365 subIdx = (int)(intptr_t)frame->data[1];
366
367 if (!d3d11_texture) {
368 av_log(avctx, AV_LOG_ERROR, "D3D11 texture not found\n");
369 return AVERROR(EINVAL);
370 }
371
372 hr = func->MFCreateSample(&sample);
373 if (FAILED(hr)) {
374 av_log(avctx, AV_LOG_ERROR, "Failed to create MF sample: %s\n", ff_hr_str(hr));
375 return AVERROR_EXTERNAL;
376 }
377
378 hr = func->MFCreateDXGISurfaceBuffer(&IID_ID3D11Texture2D, d3d11_texture, subIdx, 0, &buffer);
379 if (FAILED(hr)) {
380 av_log(avctx, AV_LOG_ERROR, "Failed to create DXGI surface buffer: %s\n", ff_hr_str(hr));
381 IMFSample_Release(sample);
382 return AVERROR_EXTERNAL;
383 }
384
385 hr = IMFSample_AddBuffer(sample, buffer);
386 if (FAILED(hr)) {
387 av_log(avctx, AV_LOG_ERROR, "Failed to add buffer to sample: %s\n", ff_hr_str(hr));
388 IMFMediaBuffer_Release(buffer);
389 IMFSample_Release(sample);
390 return AVERROR_EXTERNAL;
391 }
392
393 IMFMediaBuffer_Release(buffer);
394
395 *out_sample = sample;
396 return 0;
397}
398
399static int process_software_frame(AVCodecContext *avctx, const AVFrame *frame, IMFSample **out_sample)
400{
401 MFContext *c = avctx->priv_data;
402 IMFSample *sample = NULL;
403 IMFMediaBuffer *buffer = NULL;
404 BYTE *data = NULL;
405 HRESULT hr;
406 int size, ret;
315 407
316 size = av_image_get_buffer_size(avctx->pix_fmt, avctx->width, avctx->height, 1);408 size = av_image_get_buffer_size(avctx->pix_fmt, avctx->width, avctx->height, 1);
317 if (size < 0)409 if (size < 0)
318 return NULL;410 return AVERROR_EXTERNAL;
319 411
320 sample = ff_create_memory_sample(&c->functions, NULL, size,412 sample = ff_create_memory_sample(&c->functions, NULL, size,
321 c->in_info.cbAlignment);413 c->in_info.cbAlignment);
322 if (!sample)414 if (!sample)
323 return NULL;415 return AVERROR_EXTERNAL;
324 416
325 hr = IMFSample_GetBufferByIndex(sample, 0, &buffer);417 hr = IMFSample_GetBufferByIndex(sample, 0, &buffer);
326 if (FAILED(hr)) {418 if (FAILED(hr)) {
327 IMFSample_Release(sample);419 IMFSample_Release(sample);
328 return NULL;420 return AVERROR_EXTERNAL;
329 }421 }
330 422
331 hr = IMFMediaBuffer_Lock(buffer, &data, NULL, NULL);423 hr = IMFMediaBuffer_Lock(buffer, &data, NULL, NULL);
332 if (FAILED(hr)) {424 if (FAILED(hr)) {
333 IMFMediaBuffer_Release(buffer);425 IMFMediaBuffer_Release(buffer);
334 IMFSample_Release(sample);426 IMFSample_Release(sample);
335 return NULL;427 return AVERROR_EXTERNAL;
336 }428 }
337 429
338 ret = av_image_copy_to_buffer((uint8_t *)data, size, (void *)frame->data, frame->linesize,430 ret = av_image_copy_to_buffer((uint8_t *)data, size, (void *)frame->data, frame->linesize,
@@ -342,10 +434,43 @@ static IMFSample *mf_v_avframe_to_sample(AVCodecContext *avctx, const AVFrame *f · 3 unmodified lines
342 IMFMediaBuffer_Release(buffer);434 IMFMediaBuffer_Release(buffer);
343 if (ret < 0) {435 if (ret < 0) {
344 IMFSample_Release(sample);436 IMFSample_Release(sample);
345 return NULL;437 return AVERROR_EXTERNAL;
346 }438 }
347 439
348 IMFSample_SetSampleDuration(sample, mf_to_mf_time(avctx, frame->duration));440 IMFSample_SetSampleDuration(sample, mf_to_mf_time(avctx, frame->duration));
441 *out_sample = sample;
442
443 return 0;
444}
445
446static IMFSample *mf_v_avframe_to_sample(AVCodecContext *avctx, const AVFrame *frame)
447{
448 MFContext *c = avctx->priv_data;
449 MFFunctions *func = &c->functions;
450 IMFSample *sample = NULL;
451 IMFMediaBuffer *buffer = NULL;
452 HRESULT hr;
453 int ret;
454
455 if (frame->format == AV_PIX_FMT_D3D11) {
456 // Handle D3D11 hardware frames
457 ret = process_d3d11_frame(avctx, frame, &sample);
458 if (ret < 0) {
459 return NULL;
460 }
461 } else {
462 // Handle software frames
463 ret = process_software_frame(avctx, frame, &sample);
464 if (ret < 0) {
465 return NULL;
466 }
467 }
468
469 // Set sample duration
470 hr = IMFSample_SetSampleDuration(sample, mf_to_mf_time(avctx, frame->duration));
471 if (FAILED(hr)) {
472 av_log(avctx, AV_LOG_WARNING, "Failed to set sample duration: %s\n", ff_hr_str(hr));
473 }
349 474
350 return sample;475 return sample;
351}476}
@@ -511,6 +636,23 @@ static int mf_receive_packet(AVCodecContext *avctx, AVPacket *avpkt) · 159 unmodified lines
511 }636 }
512 }637 }
513 638
639 if(!c->stream_started)
640 {
641 HRESULT hr = IMFTransform_ProcessMessage(c->mft, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0);
642 if (FAILED(hr)) {
643 av_log(avctx, AV_LOG_ERROR, "could not start streaming (%s)\n", ff_hr_str(hr));
644 return AVERROR(EBADMSG);
645 }
646
647 hr = IMFTransform_ProcessMessage(c->mft, MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0);
648 if (FAILED(hr)) {
649 av_log(avctx, AV_LOG_ERROR, "could not start stream (%s)\n", ff_hr_str(hr));
650 return AVERROR(EBADMSG);
651 }
652
653 c->stream_started = 1;
654 }
655
514 ret = mf_send_sample(avctx, sample);656 ret = mf_send_sample(avctx, sample);
515 if (sample)657 if (sample)
516 IMFSample_Release(sample);658 IMFSample_Release(sample);
@@ -727,8 +869,16 @@ static int mf_encv_output_adjust(AVCodecContext *avctx, IMFMediaType *type) · 210 unmodified lines
727static int64_t mf_encv_input_score(AVCodecContext *avctx, IMFMediaType *type)869static int64_t mf_encv_input_score(AVCodecContext *avctx, IMFMediaType *type)
728{870{
729 enum AVPixelFormat pix_fmt = ff_media_type_to_pix_fmt((IMFAttributes *)type);871 enum AVPixelFormat pix_fmt = ff_media_type_to_pix_fmt((IMFAttributes *)type);
730 if (pix_fmt != avctx->pix_fmt)872
731 return -1; // can not use873 if (avctx->pix_fmt == AV_PIX_FMT_D3D11) {
874 if (pix_fmt != AV_PIX_FMT_NV12) {
875 return -1; // can not use
876 }
877 }
878 else {
879 if (pix_fmt != avctx->pix_fmt)
880 return -1; // can not use
881 }
732 882
733 return 0;883 return 0;
734}884}
@@ -736,9 +886,16 @@ static int64_t mf_encv_input_score(AVCodecContext *avctx, IMFMediaType *type) · 1 unmodified line
736static int mf_encv_input_adjust(AVCodecContext *avctx, IMFMediaType *type)886static int mf_encv_input_adjust(AVCodecContext *avctx, IMFMediaType *type)
737{887{
738 enum AVPixelFormat pix_fmt = ff_media_type_to_pix_fmt((IMFAttributes *)type);888 enum AVPixelFormat pix_fmt = ff_media_type_to_pix_fmt((IMFAttributes *)type);
739 if (pix_fmt != avctx->pix_fmt) {889 if (avctx->pix_fmt == AV_PIX_FMT_D3D11) {
740 av_log(avctx, AV_LOG_ERROR, "unsupported input pixel format set\n");890 if (pix_fmt != AV_PIX_FMT_NV12 && pix_fmt != AV_PIX_FMT_D3D11) {
741 return AVERROR(EINVAL);891 av_log(avctx, AV_LOG_ERROR, "unsupported input pixel format set\n");
892 return AVERROR(EINVAL);
893 }
894 } else {
895 if (pix_fmt != avctx->pix_fmt) {
896 av_log(avctx, AV_LOG_ERROR, "unsupported input pixel format set\n");
897 return AVERROR(EINVAL);
898 }
742 }899 }
743 900
744 //ff_MFSetAttributeSize((IMFAttributes *)type, &MF_MT_FRAME_SIZE, avctx->width, avctx->height);901 //ff_MFSetAttributeSize((IMFAttributes *)type, &MF_MT_FRAME_SIZE, avctx->width, avctx->height);
@@ -1106,18 +1263,6 @@ static int mf_init_encoder(AVCodecContext *avctx) · 361 unmodified lines
1106 if ((ret = mf_setup_context(avctx)) < 0)1263 if ((ret = mf_setup_context(avctx)) < 0)
1107 return ret;1264 return ret;
1108 1265
1109 hr = IMFTransform_ProcessMessage(c->mft, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0);
1110 if (FAILED(hr)) {
1111 av_log(avctx, AV_LOG_ERROR, "could not start streaming (%s)\n", ff_hr_str(hr));
1112 return AVERROR_EXTERNAL;
1113 }
1114
1115 hr = IMFTransform_ProcessMessage(c->mft, MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0);
1116 if (FAILED(hr)) {
1117 av_log(avctx, AV_LOG_ERROR, "could not start stream (%s)\n", ff_hr_str(hr));
1118 return AVERROR_EXTERNAL;
1119 }
1120
1121 if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER && c->async_events &&1266 if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER && c->async_events &&
1122 c->is_video && !avctx->extradata) {1267 c->is_video && !avctx->extradata) {
1123 int sleep = 10000, total = 0;1268 int sleep = 10000, total = 0;
@@ -1175,6 +1320,7 @@ static int mf_load_library(AVCodecContext *avctx) · 51 unmodified lines
1175 1320
1176#if !HAVE_UWP1321#if !HAVE_UWP
1177 c->library = dlopen("mfplat.dll", 0);1322 c->library = dlopen("mfplat.dll", 0);
1323 c->d3d_dll = dlopen("D3D11.dll", 0);
1178 1324
1179 if (!c->library) {1325 if (!c->library) {
1180 av_log(c, AV_LOG_ERROR, "DLL mfplat.dll failed to open\n");1326 av_log(c, AV_LOG_ERROR, "DLL mfplat.dll failed to open\n");
@@ -1187,6 +1333,8 @@ static int mf_load_library(AVCodecContext *avctx) · 6 unmodified lines
1187 LOAD_MF_FUNCTION(c, MFCreateAlignedMemoryBuffer);1333 LOAD_MF_FUNCTION(c, MFCreateAlignedMemoryBuffer);
1188 LOAD_MF_FUNCTION(c, MFCreateSample);1334 LOAD_MF_FUNCTION(c, MFCreateSample);
1189 LOAD_MF_FUNCTION(c, MFCreateMediaType);1335 LOAD_MF_FUNCTION(c, MFCreateMediaType);
1336 LOAD_MF_FUNCTION(c, MFCreateDXGISurfaceBuffer);
1337 LOAD_MF_FUNCTION(c, MFCreateDXGIDeviceManager);
1190 // MFTEnumEx is missing in Windows Vista's mfplat.dll.1338 // MFTEnumEx is missing in Windows Vista's mfplat.dll.
1191 LOAD_MF_FUNCTION(c, MFTEnumEx);1339 LOAD_MF_FUNCTION(c, MFTEnumEx);
1192 1340
@@ -1208,6 +1356,7 @@ static int mf_close(AVCodecContext *avctx) · 15 unmodified lines
1208 ff_free_mf(&c->functions, &c->mft);1356 ff_free_mf(&c->functions, &c->mft);
1209 1357
1210 dlclose(c->library);1358 dlclose(c->library);
1359 dlclose(c->d3d_dll);
1211 c->library = NULL;1360 c->library = NULL;
1212#else1361#else
1213 ff_free_mf(&c->functions, &c->mft);1362 ff_free_mf(&c->functions, &c->mft);
@@ -1300,7 +1449,7 @@ static const FFCodecDefault defaults[] = { · 86 unmodified lines
1300};1449};
1301 1450
1302#define VFMTS \1451#define VFMTS \
1303 CODEC_PIXFMTS(AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P),1452 CODEC_PIXFMTS(AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P, AV_PIX_FMT_D3D11),
1304#define VCAPS \1453#define VCAPS \
1305 .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HYBRID | \1454 .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HYBRID | \
1306 AV_CODEC_CAP_DR1,1455 AV_CODEC_CAP_DR1,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t