vulkan_decode: support software-defined decoders

Lynne committed 1 year ago

Commit 491b65e1 parent bbb06f12 files · +154 −51Browse files

Unified · Split

2 files

libavcodec/vulkan_decode.c +143 −51
libavcodec/vulkan_decode.h +11 −0

@@ -24,6 +24,9 @@ · 23 unmodified lines
24	#include "libavutil/mem.h"	24	#include "libavutil/mem.h"
25	#include "libavutil/vulkan_loader.h"	25	#include "libavutil/vulkan_loader.h"
26		26
		27	#define DECODER_IS_SDR(codec_id) \
		28	((codec_id) == AV_CODEC_ID_FFV1)
		29
27	#if CONFIG_H264_VULKAN_HWACCEL	30	#if CONFIG_H264_VULKAN_HWACCEL
28	extern const FFVulkanDecodeDescriptor ff_vk_dec_h264_desc;	31	extern const FFVulkanDecodeDescriptor ff_vk_dec_h264_desc;
29	#endif	32	#endif
@@ -63,7 +66,9 @@ static const VkVideoProfileInfoKHR get_video_profile(FFVulkanDecodeShared ctx, · 33 unmodified lines
63	codec_id == AV_CODEC_ID_H264 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR :	66	codec_id == AV_CODEC_ID_H264 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR :
64	codec_id == AV_CODEC_ID_HEVC ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR :	67	codec_id == AV_CODEC_ID_HEVC ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR :
65	codec_id == AV_CODEC_ID_AV1 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR :	68	codec_id == AV_CODEC_ID_AV1 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR :
66	0;	69	VK_STRUCTURE_TYPE_MAX_ENUM;
		70	if (profile_struct_type == VK_STRUCTURE_TYPE_MAX_ENUM)
		71	return NULL;
67		72
68	profile_list = ff_vk_find_struct(ctx->s.hwfc->create_pnext,	73	profile_list = ff_vk_find_struct(ctx->s.hwfc->create_pnext,
69	VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);	74	VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
@@ -119,13 +124,26 @@ static AVFrame vk_get_dpb_pool(FFVulkanDecodeShared ctx) · 49 unmodified lines
119	return avf;	124	return avf;
120	}	125	}
121		126
		127	static void init_frame(FFVulkanDecodeContext dec, FFVulkanDecodePicture vkpic)
		128	{
		129	FFVulkanDecodeShared *ctx = dec->shared_ctx;
		130	FFVulkanFunctions *vk = &ctx->s.vkfn;
		131
		132	vkpic->dpb_frame = NULL;
		133	vkpic->img_view_ref = VK_NULL_HANDLE;
		134	vkpic->img_view_out = VK_NULL_HANDLE;
		135	vkpic->img_view_dest = VK_NULL_HANDLE;
		136
		137	vkpic->destroy_image_view = vk->DestroyImageView;
		138	vkpic->wait_semaphores = vk->WaitSemaphores;
		139	}
		140
122	int ff_vk_decode_prepare_frame(FFVulkanDecodeContext dec, AVFrame pic,	141	int ff_vk_decode_prepare_frame(FFVulkanDecodeContext dec, AVFrame pic,
123	FFVulkanDecodePicture *vkpic, int is_current,	142	FFVulkanDecodePicture *vkpic, int is_current,
124	int alloc_dpb)	143	int alloc_dpb)
125	{	144	{
126	int err;	145	int err;
127	FFVulkanDecodeShared *ctx = dec->shared_ctx;	146	FFVulkanDecodeShared *ctx = dec->shared_ctx;
128	FFVulkanFunctions *vk = &ctx->s.vkfn;
129		147
130	vkpic->slices_size = 0;	148	vkpic->slices_size = 0;
131		149
@@ -134,13 +152,7 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext dec, AVFrame pic, · 2 unmodified lines
134	if (vkpic->img_view_ref)	152	if (vkpic->img_view_ref)
135	return 0;	153	return 0;
136		154
137	vkpic->dpb_frame = NULL;	155	init_frame(dec, vkpic);
138	vkpic->img_view_ref = VK_NULL_HANDLE;
139	vkpic->img_view_out = VK_NULL_HANDLE;
140	vkpic->img_view_dest = VK_NULL_HANDLE;
141
142	vkpic->destroy_image_view = vk->DestroyImageView;
143	vkpic->wait_semaphores = vk->WaitSemaphores;
144		156
145	if (ctx->common.layered_dpb && alloc_dpb) {	157	if (ctx->common.layered_dpb && alloc_dpb) {
146	vkpic->img_view_ref = ctx->common.layered_view;	158	vkpic->img_view_ref = ctx->common.layered_view;
@@ -183,6 +195,53 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext dec, AVFrame pic, · 36 unmodified lines
183	return 0;	195	return 0;
184	}	196	}
185		197
		198	int ff_vk_decode_prepare_frame_sdr(FFVulkanDecodeContext dec, AVFrame pic,
		199	FFVulkanDecodePicture *vkpic, int is_current,
		200	enum FFVkShaderRepFormat rep_fmt, int alloc_dpb)
		201	{
		202	int err;
		203	FFVulkanDecodeShared *ctx = dec->shared_ctx;
		204
		205	vkpic->slices_size = 0;
		206
		207	if (vkpic->img_view_ref)
		208	return 0;
		209
		210	init_frame(dec, vkpic);
		211
		212	if (ctx->common.layered_dpb && alloc_dpb) {
		213	vkpic->img_view_ref = ctx->common.layered_view;
		214	vkpic->img_aspect_ref = ctx->common.layered_aspect;
		215	} else if (alloc_dpb) {
		216	vkpic->dpb_frame = vk_get_dpb_pool(ctx);
		217	if (!vkpic->dpb_frame)
		218	return AVERROR(ENOMEM);
		219
		220	err = ff_vk_create_imageview(&ctx->s,
		221	&vkpic->img_view_ref, &vkpic->img_aspect_ref,
		222	vkpic->dpb_frame, 0, rep_fmt);
		223	if (err < 0)
		224	return err;
		225
		226	vkpic->img_view_dest = vkpic->img_view_ref;
		227	}
		228
		229	if (!alloc_dpb \|\| is_current) {
		230	err = ff_vk_create_imageview(&ctx->s,
		231	&vkpic->img_view_out, &vkpic->img_aspect,
		232	pic, 0, rep_fmt);
		233	if (err < 0)
		234	return err;
		235
		236	if (!alloc_dpb) {
		237	vkpic->img_view_ref = vkpic->img_view_out;
		238	vkpic->img_aspect_ref = vkpic->img_aspect;
		239	}
		240	}
		241
		242	return 0;
		243	}
		244
186	int ff_vk_decode_add_slice(AVCodecContext avctx, FFVulkanDecodePicture vp,	245	int ff_vk_decode_add_slice(AVCodecContext avctx, FFVulkanDecodePicture vp,
187	const uint8_t *data, size_t size, int add_startcode,	246	const uint8_t *data, size_t size, int add_startcode,
188	uint32_t nb_slices, const uint32_t *offsets)	247	uint32_t nb_slices, const uint32_t *offsets)
@@ -223,9 +282,14 @@ int ff_vk_decode_add_slice(AVCodecContext avctx, FFVulkanDecodePicture vp, · 34 unmodified lines
223	buf_size = 2 << av_log2(buf_size);	282	buf_size = 2 << av_log2(buf_size);
224		283
225	err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &new_ref,	284	err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &new_ref,
		285	DECODER_IS_SDR(avctx->codec_id) ?
		286	(VK_BUFFER_USAGE_STORAGE_BUFFER_BIT \|
		287	VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) :
226	VK_BUFFER_USAGE_VIDEO_DECODE_SRC_BIT_KHR,	288	VK_BUFFER_USAGE_VIDEO_DECODE_SRC_BIT_KHR,
227	ctx->s.hwfc->create_pnext, buf_size,	289	ctx->s.hwfc->create_pnext, buf_size,
228	VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);	290	VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT \|
		291	(DECODER_IS_SDR(avctx->codec_id) ?
		292	VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT : 0x0));
229	if (err < 0)	293	if (err < 0)
230	return err;	294	return err;
231		295
@@ -276,6 +340,10 @@ void ff_vk_decode_flush(AVCodecContext *avctx) · 44 unmodified lines
276	VkCommandBuffer cmd_buf;	340	VkCommandBuffer cmd_buf;
277	FFVkExecContext *exec;	341	FFVkExecContext *exec;
278		342
		343	/* Non-video queues do not need to be reset */
		344	if (!(get_codecdesc(avctx->codec_id)->decode_op))
		345	return;
		346
279	exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool);	347	exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool);
280	ff_vk_exec_start(&ctx->s, exec);	348	ff_vk_exec_start(&ctx->s, exec);
281	cmd_buf = exec->buf;	349	cmd_buf = exec->buf;
@@ -544,6 +612,9 @@ static void free_common(AVRefStructOpaque unused, void *obj) · 262 unmodified lines
544		612
545	ff_vk_video_common_uninit(s, &ctx->common);	613	ff_vk_video_common_uninit(s, &ctx->common);
546		614
		615	if (ctx->sd_ctx_free)
		616	ctx->sd_ctx_free(ctx);
		617
547	ff_vk_uninit(s);	618	ff_vk_uninit(s);
548	}	619	}
549		620
@@ -551,6 +622,7 @@ static int vulkan_decode_bootstrap(AVCodecContext avctx, AVBufferRef frames_re · 1 unmodified line
551	{	622	{
552	int err;	623	int err;
553	FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;	624	FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
		625	const FFVulkanDecodeDescriptor *vk_desc = get_codecdesc(avctx->codec_id);
554	AVHWFramesContext frames = (AVHWFramesContext )frames_ref->data;	626	AVHWFramesContext frames = (AVHWFramesContext )frames_ref->data;
555	AVHWDeviceContext device = (AVHWDeviceContext )frames->device_ref->data;	627	AVHWDeviceContext device = (AVHWDeviceContext )frames->device_ref->data;
556	AVVulkanDeviceContext *hwctx = device->hwctx;	628	AVVulkanDeviceContext *hwctx = device->hwctx;
@@ -569,11 +641,13 @@ static int vulkan_decode_bootstrap(AVCodecContext avctx, AVBufferRef frames_re · 12 unmodified lines
569	ctx->s.extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions,	641	ctx->s.extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions,
570	hwctx->nb_enabled_dev_extensions);	642	hwctx->nb_enabled_dev_extensions);
571		643
572	if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {	644	if (vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) {
573	av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",	645	if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {
574	VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);	646	av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
575	av_refstruct_unref(&dec->shared_ctx);	647	VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);
576	return AVERROR(ENOSYS);	648	av_refstruct_unref(&dec->shared_ctx);
		649	return AVERROR(ENOSYS);
		650	}
577	}	651	}
578		652
579	err = ff_vk_load_functions(device, &ctx->s.vkfn, ctx->s.extensions, 1, 1);	653	err = ff_vk_load_functions(device, &ctx->s.vkfn, ctx->s.extensions, 1, 1);
@@ -927,53 +1001,61 @@ static void free_profile_data(AVHWFramesContext *hwfc) · 347 unmodified lines
927		1001
928	int ff_vk_frame_params(AVCodecContext avctx, AVBufferRef hw_frames_ctx)	1002	int ff_vk_frame_params(AVCodecContext avctx, AVBufferRef hw_frames_ctx)
929	{	1003	{
930	VkFormat vkfmt;	1004	VkFormat vkfmt = VK_FORMAT_UNDEFINED;
931	int err, dedicated_dpb;	1005	int err, dedicated_dpb;
932	AVHWFramesContext frames_ctx = (AVHWFramesContext)hw_frames_ctx->data;	1006	AVHWFramesContext frames_ctx = (AVHWFramesContext)hw_frames_ctx->data;
933	AVVulkanFramesContext *hwfc = frames_ctx->hwctx;	1007	AVVulkanFramesContext *hwfc = frames_ctx->hwctx;
934	FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;	1008	FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
935	FFVulkanDecodeProfileData *prof;	1009	FFVulkanDecodeProfileData *prof = NULL;
936	FFVulkanDecodeShared *ctx;
937
938	frames_ctx->sw_format = AV_PIX_FMT_NONE;
939		1010
940	err = vulkan_decode_bootstrap(avctx, hw_frames_ctx);	1011	err = vulkan_decode_bootstrap(avctx, hw_frames_ctx);
941	if (err < 0)	1012	if (err < 0)
942	return err;	1013	return err;
943		1014
944	prof = av_mallocz(sizeof(FFVulkanDecodeProfileData));	1015	frames_ctx->sw_format = avctx->sw_pix_fmt;
945	if (!prof)
946	return AVERROR(ENOMEM);
947		1016
948	err = vulkan_decode_get_profile(avctx, hw_frames_ctx,	1017	if (!DECODER_IS_SDR(avctx->codec_id)) {
949	&frames_ctx->sw_format, &vkfmt,	1018	prof = av_mallocz(sizeof(FFVulkanDecodeProfileData));
950	prof, &dedicated_dpb);	1019	if (!prof)
951	if (err < 0) {	1020	return AVERROR(ENOMEM);
952	av_free(prof);
953	return err;
954	}
955		1021
956	frames_ctx->user_opaque = prof;	1022	err = vulkan_decode_get_profile(avctx, hw_frames_ctx,
957	frames_ctx->free = free_profile_data;	1023	&frames_ctx->sw_format, &vkfmt,
		1024	prof, &dedicated_dpb);
		1025	if (err < 0) {
		1026	av_free(prof);
		1027	return err;
		1028	}
		1029
		1030	frames_ctx->user_opaque = prof;
		1031	frames_ctx->free = free_profile_data;
		1032
		1033	hwfc->create_pnext = &prof->profile_list;
		1034	}
958		1035
959	frames_ctx->width = avctx->coded_width;	1036	frames_ctx->width = avctx->coded_width;
960	frames_ctx->height = avctx->coded_height;	1037	frames_ctx->height = avctx->coded_height;
961	frames_ctx->format = AV_PIX_FMT_VULKAN;	1038	frames_ctx->format = AV_PIX_FMT_VULKAN;
962		1039
963	hwfc->format[0] = vkfmt;	1040	hwfc->format[0] = vkfmt;
964	hwfc->create_pnext = &prof->profile_list;
965	hwfc->tiling = VK_IMAGE_TILING_OPTIMAL;	1041	hwfc->tiling = VK_IMAGE_TILING_OPTIMAL;
966	hwfc->usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT \|	1042	hwfc->usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT \|
967	VK_IMAGE_USAGE_SAMPLED_BIT \|	1043	VK_IMAGE_USAGE_SAMPLED_BIT;
968	VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR;
969		1044
970	if (!dec->dedicated_dpb)	1045	if (prof) {
971	hwfc->usage \|= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;	1046	FFVulkanDecodeShared *ctx;
972		1047
973	ctx = dec->shared_ctx;	1048	hwfc->usage \|= VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR;
974	if (ctx->s.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE \|	1049	if (!dec->dedicated_dpb)
975	FF_VK_EXT_VIDEO_MAINTENANCE_1))	1050	hwfc->usage \|= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
976	hwfc->usage \|= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;	1051
		1052	ctx = dec->shared_ctx;
		1053	if (ctx->s.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE \|
		1054	FF_VK_EXT_VIDEO_MAINTENANCE_1))
		1055	hwfc->usage \|= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
		1056	} else if (DECODER_IS_SDR(avctx->codec_id)) {
		1057	hwfc->usage \|= VK_IMAGE_USAGE_STORAGE_BIT;
		1058	}
977		1059
978	return err;	1060	return err;
979	}	1061	}
@@ -1075,8 +1157,10 @@ int ff_vk_decode_init(AVCodecContext *avctx) · 95 unmodified lines
1075	if (err < 0)	1157	if (err < 0)
1076	return err;	1158	return err;
1077		1159
		1160	vk_desc = get_codecdesc(avctx->codec_id);
		1161
1078	profile = get_video_profile(ctx, avctx->codec_id);	1162	profile = get_video_profile(ctx, avctx->codec_id);
1079	if (!profile) {	1163	if ((vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) && !profile) {
1080	av_log(avctx, AV_LOG_ERROR, "Video profile missing from frames context!");	1164	av_log(avctx, AV_LOG_ERROR, "Video profile missing from frames context!");
1081	return AVERROR(EINVAL);	1165	return AVERROR(EINVAL);
1082	}	1166	}
@@ -1109,9 +1193,11 @@ int ff_vk_decode_init(AVCodecContext *avctx) · 26 unmodified lines
1109	if (err < 0)	1193	if (err < 0)
1110	goto fail;	1194	goto fail;
1111		1195
1112	err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create);	1196	if (profile) {
1113	if (err < 0)	1197	err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create);
1114	goto fail;	1198	if (err < 0)
		1199	goto fail;
		1200	}
1115		1201
1116	/* If doing an out-of-place decoding, create a DPB pool */	1202	/* If doing an out-of-place decoding, create a DPB pool */
1117	if (dec->dedicated_dpb \|\| avctx->codec_id == AV_CODEC_ID_AV1) {	1203	if (dec->dedicated_dpb \|\| avctx->codec_id == AV_CODEC_ID_AV1) {
@@ -1163,12 +1249,18 @@ int ff_vk_decode_init(AVCodecContext *avctx) · 45 unmodified lines
1163	}	1249	}
1164		1250
1165	session_params_create.videoSession = ctx->common.session;	1251	session_params_create.videoSession = ctx->common.session;
1166	ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create,	1252	if (profile) {
1167	s->hwctx->alloc, &ctx->empty_session_params);	1253	ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create,
1168	if (ret != VK_SUCCESS) {	1254	s->hwctx->alloc, &ctx->empty_session_params);
1169	av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n",	1255	if (ret != VK_SUCCESS) {
1170	ff_vk_ret2str(ret));	1256	av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n",
1171	return AVERROR_EXTERNAL;	1257	ff_vk_ret2str(ret));
		1258	return AVERROR_EXTERNAL;
		1259	}
		1260	} else {
		1261	/* For SDR decoders, this alignment value will be 0. Since this will make
		1262	* add_slice() malfunction, set it to a sane default value. */
		1263	ctx->caps.minBitstreamBufferSizeAlignment = AV_INPUT_BUFFER_PADDING_SIZE;
1172	}	1264	}
1173		1265
1174	driver_props = &dec->shared_ctx->s.driver_props;	1266	driver_props = &dec->shared_ctx->s.driver_props;

libavcodec/vulkan_decode.h+11−0

     VkVideoDecodeCapabilitiesKHR dec_caps;
     VkVideoSessionParametersKHR empty_session_params;
+    /* Software-defined decoder context */
+    void *sd_ctx;
+    void (*sd_ctx_free)(struct FFVulkanDecodeShared *ctx);
 } FFVulkanDecodeShared;
 typedef struct FFVulkanDecodeContext {
                                FFVulkanDecodePicture *vkpic, int is_current,
                                int alloc_dpb);
+/**
+ * Software-defined decoder version of ff_vk_decode_prepare_frame.
+ */
+int ff_vk_decode_prepare_frame_sdr(FFVulkanDecodeContext *dec, AVFrame *pic,
+                                   FFVulkanDecodePicture *vkpic, int is_current,
+                                   enum FFVkShaderRepFormat rep_fmt, int alloc_dpb);
 /**
  * Add slice data to frame.
  */

vulkan_decode: support software-defined decoders

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code