History

vulkan_decode: support software-defined decoders

Lynne committed 1 year ago

Commit 491b65e1 parent bbb06f12 files · +154 −51Browse files

Unified · Split

2 files
  1. libavcodec/vulkan_decode.c +143 −51
  2. libavcodec/vulkan_decode.h +11 −0
libavcodec/vulkan_decode.c+143−51
@@ -24,6 +24,9 @@ · 23 unmodified lines
2424 #include "libavutil/mem.h"
2525 #include "libavutil/vulkan_loader.h"
2626
27+#define DECODER_IS_SDR(codec_id) \
28+ ((codec_id) == AV_CODEC_ID_FFV1)
29+
2730 #if CONFIG_H264_VULKAN_HWACCEL
2831 extern const FFVulkanDecodeDescriptor ff_vk_dec_h264_desc;
2932 #endif
@@ -63,7 +66,9 @@ static const VkVideoProfileInfoKHR *get_video_profile(FFVulkanDecodeShared *ctx, · 33 unmodified lines
6366 codec_id == AV_CODEC_ID_H264 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR :
6467 codec_id == AV_CODEC_ID_HEVC ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR :
6568 codec_id == AV_CODEC_ID_AV1 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR :
66− 0;
69+ VK_STRUCTURE_TYPE_MAX_ENUM;
70+ if (profile_struct_type == VK_STRUCTURE_TYPE_MAX_ENUM)
71+ return NULL;
6772
6873 profile_list = ff_vk_find_struct(ctx->s.hwfc->create_pnext,
6974 VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
@@ -119,13 +124,26 @@ static AVFrame *vk_get_dpb_pool(FFVulkanDecodeShared *ctx) · 49 unmodified lines
119124 return avf;
120125 }
121126
127+static void init_frame(FFVulkanDecodeContext *dec, FFVulkanDecodePicture *vkpic)
128+{
129+ FFVulkanDecodeShared *ctx = dec->shared_ctx;
130+ FFVulkanFunctions *vk = &ctx->s.vkfn;
131+
132+ vkpic->dpb_frame = NULL;
133+ vkpic->img_view_ref = VK_NULL_HANDLE;
134+ vkpic->img_view_out = VK_NULL_HANDLE;
135+ vkpic->img_view_dest = VK_NULL_HANDLE;
136+
137+ vkpic->destroy_image_view = vk->DestroyImageView;
138+ vkpic->wait_semaphores = vk->WaitSemaphores;
139+}
140+
122141 int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic,
123142 FFVulkanDecodePicture *vkpic, int is_current,
124143 int alloc_dpb)
125144 {
126145 int err;
127146 FFVulkanDecodeShared *ctx = dec->shared_ctx;
128− FFVulkanFunctions *vk = &ctx->s.vkfn;
129147
130148 vkpic->slices_size = 0;
131149
@@ -134,13 +152,7 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, · 2 unmodified lines
134152 if (vkpic->img_view_ref)
135153 return 0;
136154
137− vkpic->dpb_frame = NULL;
138− vkpic->img_view_ref = VK_NULL_HANDLE;
139− vkpic->img_view_out = VK_NULL_HANDLE;
140− vkpic->img_view_dest = VK_NULL_HANDLE;
141−
142− vkpic->destroy_image_view = vk->DestroyImageView;
143− vkpic->wait_semaphores = vk->WaitSemaphores;
155+ init_frame(dec, vkpic);
144156
145157 if (ctx->common.layered_dpb && alloc_dpb) {
146158 vkpic->img_view_ref = ctx->common.layered_view;
@@ -183,6 +195,53 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, · 36 unmodified lines
183195 return 0;
184196 }
185197
198+int ff_vk_decode_prepare_frame_sdr(FFVulkanDecodeContext *dec, AVFrame *pic,
199+ FFVulkanDecodePicture *vkpic, int is_current,
200+ enum FFVkShaderRepFormat rep_fmt, int alloc_dpb)
201+{
202+ int err;
203+ FFVulkanDecodeShared *ctx = dec->shared_ctx;
204+
205+ vkpic->slices_size = 0;
206+
207+ if (vkpic->img_view_ref)
208+ return 0;
209+
210+ init_frame(dec, vkpic);
211+
212+ if (ctx->common.layered_dpb && alloc_dpb) {
213+ vkpic->img_view_ref = ctx->common.layered_view;
214+ vkpic->img_aspect_ref = ctx->common.layered_aspect;
215+ } else if (alloc_dpb) {
216+ vkpic->dpb_frame = vk_get_dpb_pool(ctx);
217+ if (!vkpic->dpb_frame)
218+ return AVERROR(ENOMEM);
219+
220+ err = ff_vk_create_imageview(&ctx->s,
221+ &vkpic->img_view_ref, &vkpic->img_aspect_ref,
222+ vkpic->dpb_frame, 0, rep_fmt);
223+ if (err < 0)
224+ return err;
225+
226+ vkpic->img_view_dest = vkpic->img_view_ref;
227+ }
228+
229+ if (!alloc_dpb || is_current) {
230+ err = ff_vk_create_imageview(&ctx->s,
231+ &vkpic->img_view_out, &vkpic->img_aspect,
232+ pic, 0, rep_fmt);
233+ if (err < 0)
234+ return err;
235+
236+ if (!alloc_dpb) {
237+ vkpic->img_view_ref = vkpic->img_view_out;
238+ vkpic->img_aspect_ref = vkpic->img_aspect;
239+ }
240+ }
241+
242+ return 0;
243+}
244+
186245 int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp,
187246 const uint8_t *data, size_t size, int add_startcode,
188247 uint32_t *nb_slices, const uint32_t **offsets)
@@ -223,9 +282,14 @@ int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp, · 34 unmodified lines
223282 buf_size = 2 << av_log2(buf_size);
224283
225284 err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &new_ref,
285+ DECODER_IS_SDR(avctx->codec_id) ?
286+ (VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
287+ VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) :
226288 VK_BUFFER_USAGE_VIDEO_DECODE_SRC_BIT_KHR,
227289 ctx->s.hwfc->create_pnext, buf_size,
228− VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
290+ VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
291+ (DECODER_IS_SDR(avctx->codec_id) ?
292+ VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT : 0x0));
229293 if (err < 0)
230294 return err;
231295
@@ -276,6 +340,10 @@ void ff_vk_decode_flush(AVCodecContext *avctx) · 44 unmodified lines
276340 VkCommandBuffer cmd_buf;
277341 FFVkExecContext *exec;
278342
343+ /* Non-video queues do not need to be reset */
344+ if (!(get_codecdesc(avctx->codec_id)->decode_op))
345+ return;
346+
279347 exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool);
280348 ff_vk_exec_start(&ctx->s, exec);
281349 cmd_buf = exec->buf;
@@ -544,6 +612,9 @@ static void free_common(AVRefStructOpaque unused, void *obj) · 262 unmodified lines
544612
545613 ff_vk_video_common_uninit(s, &ctx->common);
546614
615+ if (ctx->sd_ctx_free)
616+ ctx->sd_ctx_free(ctx);
617+
547618 ff_vk_uninit(s);
548619 }
549620
@@ -551,6 +622,7 @@ static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_re · 1 unmodified line
551622 {
552623 int err;
553624 FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
625+ const FFVulkanDecodeDescriptor *vk_desc = get_codecdesc(avctx->codec_id);
554626 AVHWFramesContext *frames = (AVHWFramesContext *)frames_ref->data;
555627 AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;
556628 AVVulkanDeviceContext *hwctx = device->hwctx;
@@ -569,11 +641,13 @@ static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_re · 12 unmodified lines
569641 ctx->s.extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions,
570642 hwctx->nb_enabled_dev_extensions);
571643
572− if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {
573− av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
574− VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);
575− av_refstruct_unref(&dec->shared_ctx);
576− return AVERROR(ENOSYS);
644+ if (vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) {
645+ if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {
646+ av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
647+ VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);
648+ av_refstruct_unref(&dec->shared_ctx);
649+ return AVERROR(ENOSYS);
650+ }
577651 }
578652
579653 err = ff_vk_load_functions(device, &ctx->s.vkfn, ctx->s.extensions, 1, 1);
@@ -927,53 +1001,61 @@ static void free_profile_data(AVHWFramesContext *hwfc) · 347 unmodified lines
9271001
9281002 int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
9291003 {
930− VkFormat vkfmt;
1004+ VkFormat vkfmt = VK_FORMAT_UNDEFINED;
9311005 int err, dedicated_dpb;
9321006 AVHWFramesContext *frames_ctx = (AVHWFramesContext*)hw_frames_ctx->data;
9331007 AVVulkanFramesContext *hwfc = frames_ctx->hwctx;
9341008 FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
935− FFVulkanDecodeProfileData *prof;
936− FFVulkanDecodeShared *ctx;
937−
938− frames_ctx->sw_format = AV_PIX_FMT_NONE;
1009+ FFVulkanDecodeProfileData *prof = NULL;
9391010
9401011 err = vulkan_decode_bootstrap(avctx, hw_frames_ctx);
9411012 if (err < 0)
9421013 return err;
9431014
944− prof = av_mallocz(sizeof(FFVulkanDecodeProfileData));
945− if (!prof)
946− return AVERROR(ENOMEM);
1015+ frames_ctx->sw_format = avctx->sw_pix_fmt;
9471016
948− err = vulkan_decode_get_profile(avctx, hw_frames_ctx,
949− &frames_ctx->sw_format, &vkfmt,
950− prof, &dedicated_dpb);
951− if (err < 0) {
952− av_free(prof);
953− return err;
954− }
1017+ if (!DECODER_IS_SDR(avctx->codec_id)) {
1018+ prof = av_mallocz(sizeof(FFVulkanDecodeProfileData));
1019+ if (!prof)
1020+ return AVERROR(ENOMEM);
9551021
956− frames_ctx->user_opaque = prof;
957− frames_ctx->free = free_profile_data;
1022+ err = vulkan_decode_get_profile(avctx, hw_frames_ctx,
1023+ &frames_ctx->sw_format, &vkfmt,
1024+ prof, &dedicated_dpb);
1025+ if (err < 0) {
1026+ av_free(prof);
1027+ return err;
1028+ }
1029+
1030+ frames_ctx->user_opaque = prof;
1031+ frames_ctx->free = free_profile_data;
1032+
1033+ hwfc->create_pnext = &prof->profile_list;
1034+ }
9581035
9591036 frames_ctx->width = avctx->coded_width;
9601037 frames_ctx->height = avctx->coded_height;
9611038 frames_ctx->format = AV_PIX_FMT_VULKAN;
9621039
9631040 hwfc->format[0] = vkfmt;
964− hwfc->create_pnext = &prof->profile_list;
9651041 hwfc->tiling = VK_IMAGE_TILING_OPTIMAL;
9661042 hwfc->usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
967− VK_IMAGE_USAGE_SAMPLED_BIT |
968− VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR;
1043+ VK_IMAGE_USAGE_SAMPLED_BIT;
9691044
970− if (!dec->dedicated_dpb)
971− hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
1045+ if (prof) {
1046+ FFVulkanDecodeShared *ctx;
9721047
973− ctx = dec->shared_ctx;
974− if (ctx->s.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
975− FF_VK_EXT_VIDEO_MAINTENANCE_1))
976− hwfc->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
1048+ hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR;
1049+ if (!dec->dedicated_dpb)
1050+ hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
1051+
1052+ ctx = dec->shared_ctx;
1053+ if (ctx->s.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
1054+ FF_VK_EXT_VIDEO_MAINTENANCE_1))
1055+ hwfc->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
1056+ } else if (DECODER_IS_SDR(avctx->codec_id)) {
1057+ hwfc->usage |= VK_IMAGE_USAGE_STORAGE_BIT;
1058+ }
9771059
9781060 return err;
9791061 }
@@ -1075,8 +1157,10 @@ int ff_vk_decode_init(AVCodecContext *avctx) · 95 unmodified lines
10751157 if (err < 0)
10761158 return err;
10771159
1160+ vk_desc = get_codecdesc(avctx->codec_id);
1161+
10781162 profile = get_video_profile(ctx, avctx->codec_id);
1079− if (!profile) {
1163+ if ((vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) && !profile) {
10801164 av_log(avctx, AV_LOG_ERROR, "Video profile missing from frames context!");
10811165 return AVERROR(EINVAL);
10821166 }
@@ -1109,9 +1193,11 @@ int ff_vk_decode_init(AVCodecContext *avctx) · 26 unmodified lines
11091193 if (err < 0)
11101194 goto fail;
11111195
1112− err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create);
1113− if (err < 0)
1114− goto fail;
1196+ if (profile) {
1197+ err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create);
1198+ if (err < 0)
1199+ goto fail;
1200+ }
11151201
11161202 /* If doing an out-of-place decoding, create a DPB pool */
11171203 if (dec->dedicated_dpb || avctx->codec_id == AV_CODEC_ID_AV1) {
@@ -1163,12 +1249,18 @@ int ff_vk_decode_init(AVCodecContext *avctx) · 45 unmodified lines
11631249 }
11641250
11651251 session_params_create.videoSession = ctx->common.session;
1166− ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create,
1167− s->hwctx->alloc, &ctx->empty_session_params);
1168− if (ret != VK_SUCCESS) {
1169− av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n",
1170− ff_vk_ret2str(ret));
1171− return AVERROR_EXTERNAL;
1252+ if (profile) {
1253+ ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create,
1254+ s->hwctx->alloc, &ctx->empty_session_params);
1255+ if (ret != VK_SUCCESS) {
1256+ av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n",
1257+ ff_vk_ret2str(ret));
1258+ return AVERROR_EXTERNAL;
1259+ }
1260+ } else {
1261+ /* For SDR decoders, this alignment value will be 0. Since this will make
1262+ * add_slice() malfunction, set it to a sane default value. */
1263+ ctx->caps.minBitstreamBufferSizeAlignment = AV_INPUT_BUFFER_PADDING_SIZE;
11721264 }
11731265
11741266 driver_props = &dec->shared_ctx->s.driver_props;
libavcodec/vulkan_decode.h+11−0
@@ -56,6 +56,10 @@ typedef struct FFVulkanDecodeShared { · 55 unmodified lines
5656 VkVideoDecodeCapabilitiesKHR dec_caps;
5757
5858 VkVideoSessionParametersKHR empty_session_params;
59+
60+ /* Software-defined decoder context */
61+ void *sd_ctx;
62+ void (*sd_ctx_free)(struct FFVulkanDecodeShared *ctx);
5963 } FFVulkanDecodeShared;
6064
6165 typedef struct FFVulkanDecodeContext {
@@ -141,6 +145,13 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, · 79 unmodified lines
141145 FFVulkanDecodePicture *vkpic, int is_current,
142146 int alloc_dpb);
143147
148+/**
149+ * Software-defined decoder version of ff_vk_decode_prepare_frame.
150+ */
151+int ff_vk_decode_prepare_frame_sdr(FFVulkanDecodeContext *dec, AVFrame *pic,
152+ FFVulkanDecodePicture *vkpic, int is_current,
153+ enum FFVkShaderRepFormat rep_fmt, int alloc_dpb);
154+
144155 /**
145156 * Add slice data to frame.
146157 */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t