History

vulkan_decode: support software-defined decoders

Lynne committed 1 year ago

Commit 491b65e1 parent bbb06f12 files · +154 −51Browse files

Showing the changes to libavcodec (2 of 2 files). Show all

Unified · Split

2 files
  1. libavcodec/vulkan_decode.c +143 −51
  2. libavcodec/vulkan_decode.h +11 −0
libavcodec/vulkan_decode.c+143−51
@@ -24,6 +24,9 @@ · 23 unmodified lines
24#include "libavutil/mem.h"24#include "libavutil/mem.h"
25#include "libavutil/vulkan_loader.h"25#include "libavutil/vulkan_loader.h"
26 26
27#define DECODER_IS_SDR(codec_id) \
28 ((codec_id) == AV_CODEC_ID_FFV1)
29
27#if CONFIG_H264_VULKAN_HWACCEL30#if CONFIG_H264_VULKAN_HWACCEL
28extern const FFVulkanDecodeDescriptor ff_vk_dec_h264_desc;31extern const FFVulkanDecodeDescriptor ff_vk_dec_h264_desc;
29#endif32#endif
@@ -63,7 +66,9 @@ static const VkVideoProfileInfoKHR *get_video_profile(FFVulkanDecodeShared *ctx, · 33 unmodified lines
63 codec_id == AV_CODEC_ID_H264 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR :66 codec_id == AV_CODEC_ID_H264 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR :
64 codec_id == AV_CODEC_ID_HEVC ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR :67 codec_id == AV_CODEC_ID_HEVC ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR :
65 codec_id == AV_CODEC_ID_AV1 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR :68 codec_id == AV_CODEC_ID_AV1 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_KHR :
66 0;69 VK_STRUCTURE_TYPE_MAX_ENUM;
70 if (profile_struct_type == VK_STRUCTURE_TYPE_MAX_ENUM)
71 return NULL;
67 72
68 profile_list = ff_vk_find_struct(ctx->s.hwfc->create_pnext,73 profile_list = ff_vk_find_struct(ctx->s.hwfc->create_pnext,
69 VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);74 VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
@@ -119,13 +124,26 @@ static AVFrame *vk_get_dpb_pool(FFVulkanDecodeShared *ctx) · 49 unmodified lines
119 return avf;124 return avf;
120}125}
121 126
127static void init_frame(FFVulkanDecodeContext *dec, FFVulkanDecodePicture *vkpic)
128{
129 FFVulkanDecodeShared *ctx = dec->shared_ctx;
130 FFVulkanFunctions *vk = &ctx->s.vkfn;
131
132 vkpic->dpb_frame = NULL;
133 vkpic->img_view_ref = VK_NULL_HANDLE;
134 vkpic->img_view_out = VK_NULL_HANDLE;
135 vkpic->img_view_dest = VK_NULL_HANDLE;
136
137 vkpic->destroy_image_view = vk->DestroyImageView;
138 vkpic->wait_semaphores = vk->WaitSemaphores;
139}
140
122int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic,141int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic,
123 FFVulkanDecodePicture *vkpic, int is_current,142 FFVulkanDecodePicture *vkpic, int is_current,
124 int alloc_dpb)143 int alloc_dpb)
125{144{
126 int err;145 int err;
127 FFVulkanDecodeShared *ctx = dec->shared_ctx;146 FFVulkanDecodeShared *ctx = dec->shared_ctx;
128 FFVulkanFunctions *vk = &ctx->s.vkfn;
129 147
130 vkpic->slices_size = 0;148 vkpic->slices_size = 0;
131 149
@@ -134,13 +152,7 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, · 2 unmodified lines
134 if (vkpic->img_view_ref)152 if (vkpic->img_view_ref)
135 return 0;153 return 0;
136 154
137 vkpic->dpb_frame = NULL;155 init_frame(dec, vkpic);
138 vkpic->img_view_ref = VK_NULL_HANDLE;
139 vkpic->img_view_out = VK_NULL_HANDLE;
140 vkpic->img_view_dest = VK_NULL_HANDLE;
141
142 vkpic->destroy_image_view = vk->DestroyImageView;
143 vkpic->wait_semaphores = vk->WaitSemaphores;
144 156
145 if (ctx->common.layered_dpb && alloc_dpb) {157 if (ctx->common.layered_dpb && alloc_dpb) {
146 vkpic->img_view_ref = ctx->common.layered_view;158 vkpic->img_view_ref = ctx->common.layered_view;
@@ -183,6 +195,53 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, · 36 unmodified lines
183 return 0;195 return 0;
184}196}
185 197
198int ff_vk_decode_prepare_frame_sdr(FFVulkanDecodeContext *dec, AVFrame *pic,
199 FFVulkanDecodePicture *vkpic, int is_current,
200 enum FFVkShaderRepFormat rep_fmt, int alloc_dpb)
201{
202 int err;
203 FFVulkanDecodeShared *ctx = dec->shared_ctx;
204
205 vkpic->slices_size = 0;
206
207 if (vkpic->img_view_ref)
208 return 0;
209
210 init_frame(dec, vkpic);
211
212 if (ctx->common.layered_dpb && alloc_dpb) {
213 vkpic->img_view_ref = ctx->common.layered_view;
214 vkpic->img_aspect_ref = ctx->common.layered_aspect;
215 } else if (alloc_dpb) {
216 vkpic->dpb_frame = vk_get_dpb_pool(ctx);
217 if (!vkpic->dpb_frame)
218 return AVERROR(ENOMEM);
219
220 err = ff_vk_create_imageview(&ctx->s,
221 &vkpic->img_view_ref, &vkpic->img_aspect_ref,
222 vkpic->dpb_frame, 0, rep_fmt);
223 if (err < 0)
224 return err;
225
226 vkpic->img_view_dest = vkpic->img_view_ref;
227 }
228
229 if (!alloc_dpb || is_current) {
230 err = ff_vk_create_imageview(&ctx->s,
231 &vkpic->img_view_out, &vkpic->img_aspect,
232 pic, 0, rep_fmt);
233 if (err < 0)
234 return err;
235
236 if (!alloc_dpb) {
237 vkpic->img_view_ref = vkpic->img_view_out;
238 vkpic->img_aspect_ref = vkpic->img_aspect;
239 }
240 }
241
242 return 0;
243}
244
186int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp,245int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp,
187 const uint8_t *data, size_t size, int add_startcode,246 const uint8_t *data, size_t size, int add_startcode,
188 uint32_t *nb_slices, const uint32_t **offsets)247 uint32_t *nb_slices, const uint32_t **offsets)
@@ -223,9 +282,14 @@ int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp, · 34 unmodified lines
223 buf_size = 2 << av_log2(buf_size);282 buf_size = 2 << av_log2(buf_size);
224 283
225 err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &new_ref,284 err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &new_ref,
285 DECODER_IS_SDR(avctx->codec_id) ?
286 (VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
287 VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) :
226 VK_BUFFER_USAGE_VIDEO_DECODE_SRC_BIT_KHR,288 VK_BUFFER_USAGE_VIDEO_DECODE_SRC_BIT_KHR,
227 ctx->s.hwfc->create_pnext, buf_size,289 ctx->s.hwfc->create_pnext, buf_size,
228 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);290 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
291 (DECODER_IS_SDR(avctx->codec_id) ?
292 VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT : 0x0));
229 if (err < 0)293 if (err < 0)
230 return err;294 return err;
231 295
@@ -276,6 +340,10 @@ void ff_vk_decode_flush(AVCodecContext *avctx) · 44 unmodified lines
276 VkCommandBuffer cmd_buf;340 VkCommandBuffer cmd_buf;
277 FFVkExecContext *exec;341 FFVkExecContext *exec;
278 342
343 /* Non-video queues do not need to be reset */
344 if (!(get_codecdesc(avctx->codec_id)->decode_op))
345 return;
346
279 exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool);347 exec = ff_vk_exec_get(&ctx->s, &ctx->exec_pool);
280 ff_vk_exec_start(&ctx->s, exec);348 ff_vk_exec_start(&ctx->s, exec);
281 cmd_buf = exec->buf;349 cmd_buf = exec->buf;
@@ -544,6 +612,9 @@ static void free_common(AVRefStructOpaque unused, void *obj) · 262 unmodified lines
544 612
545 ff_vk_video_common_uninit(s, &ctx->common);613 ff_vk_video_common_uninit(s, &ctx->common);
546 614
615 if (ctx->sd_ctx_free)
616 ctx->sd_ctx_free(ctx);
617
547 ff_vk_uninit(s);618 ff_vk_uninit(s);
548}619}
549 620
@@ -551,6 +622,7 @@ static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_re · 1 unmodified line
551{622{
552 int err;623 int err;
553 FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;624 FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
625 const FFVulkanDecodeDescriptor *vk_desc = get_codecdesc(avctx->codec_id);
554 AVHWFramesContext *frames = (AVHWFramesContext *)frames_ref->data;626 AVHWFramesContext *frames = (AVHWFramesContext *)frames_ref->data;
555 AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;627 AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;
556 AVVulkanDeviceContext *hwctx = device->hwctx;628 AVVulkanDeviceContext *hwctx = device->hwctx;
@@ -569,11 +641,13 @@ static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_re · 12 unmodified lines
569 ctx->s.extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions,641 ctx->s.extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions,
570 hwctx->nb_enabled_dev_extensions);642 hwctx->nb_enabled_dev_extensions);
571 643
572 if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {644 if (vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) {
573 av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",645 if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {
574 VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);646 av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
575 av_refstruct_unref(&dec->shared_ctx);647 VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);
576 return AVERROR(ENOSYS);648 av_refstruct_unref(&dec->shared_ctx);
649 return AVERROR(ENOSYS);
650 }
577 }651 }
578 652
579 err = ff_vk_load_functions(device, &ctx->s.vkfn, ctx->s.extensions, 1, 1);653 err = ff_vk_load_functions(device, &ctx->s.vkfn, ctx->s.extensions, 1, 1);
@@ -927,53 +1001,61 @@ static void free_profile_data(AVHWFramesContext *hwfc) · 347 unmodified lines
927 1001
928int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)1002int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
929{1003{
930 VkFormat vkfmt;1004 VkFormat vkfmt = VK_FORMAT_UNDEFINED;
931 int err, dedicated_dpb;1005 int err, dedicated_dpb;
932 AVHWFramesContext *frames_ctx = (AVHWFramesContext*)hw_frames_ctx->data;1006 AVHWFramesContext *frames_ctx = (AVHWFramesContext*)hw_frames_ctx->data;
933 AVVulkanFramesContext *hwfc = frames_ctx->hwctx;1007 AVVulkanFramesContext *hwfc = frames_ctx->hwctx;
934 FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;1008 FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
935 FFVulkanDecodeProfileData *prof;1009 FFVulkanDecodeProfileData *prof = NULL;
936 FFVulkanDecodeShared *ctx;
937
938 frames_ctx->sw_format = AV_PIX_FMT_NONE;
939 1010
940 err = vulkan_decode_bootstrap(avctx, hw_frames_ctx);1011 err = vulkan_decode_bootstrap(avctx, hw_frames_ctx);
941 if (err < 0)1012 if (err < 0)
942 return err;1013 return err;
943 1014
944 prof = av_mallocz(sizeof(FFVulkanDecodeProfileData));1015 frames_ctx->sw_format = avctx->sw_pix_fmt;
945 if (!prof)
946 return AVERROR(ENOMEM);
947 1016
948 err = vulkan_decode_get_profile(avctx, hw_frames_ctx,1017 if (!DECODER_IS_SDR(avctx->codec_id)) {
949 &frames_ctx->sw_format, &vkfmt,1018 prof = av_mallocz(sizeof(FFVulkanDecodeProfileData));
950 prof, &dedicated_dpb);1019 if (!prof)
951 if (err < 0) {1020 return AVERROR(ENOMEM);
952 av_free(prof);
953 return err;
954 }
955 1021
956 frames_ctx->user_opaque = prof;1022 err = vulkan_decode_get_profile(avctx, hw_frames_ctx,
957 frames_ctx->free = free_profile_data;1023 &frames_ctx->sw_format, &vkfmt,
1024 prof, &dedicated_dpb);
1025 if (err < 0) {
1026 av_free(prof);
1027 return err;
1028 }
1029
1030 frames_ctx->user_opaque = prof;
1031 frames_ctx->free = free_profile_data;
1032
1033 hwfc->create_pnext = &prof->profile_list;
1034 }
958 1035
959 frames_ctx->width = avctx->coded_width;1036 frames_ctx->width = avctx->coded_width;
960 frames_ctx->height = avctx->coded_height;1037 frames_ctx->height = avctx->coded_height;
961 frames_ctx->format = AV_PIX_FMT_VULKAN;1038 frames_ctx->format = AV_PIX_FMT_VULKAN;
962 1039
963 hwfc->format[0] = vkfmt;1040 hwfc->format[0] = vkfmt;
964 hwfc->create_pnext = &prof->profile_list;
965 hwfc->tiling = VK_IMAGE_TILING_OPTIMAL;1041 hwfc->tiling = VK_IMAGE_TILING_OPTIMAL;
966 hwfc->usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT |1042 hwfc->usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
967 VK_IMAGE_USAGE_SAMPLED_BIT |1043 VK_IMAGE_USAGE_SAMPLED_BIT;
968 VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR;
969 1044
970 if (!dec->dedicated_dpb)1045 if (prof) {
971 hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;1046 FFVulkanDecodeShared *ctx;
972 1047
973 ctx = dec->shared_ctx;1048 hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR;
974 if (ctx->s.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |1049 if (!dec->dedicated_dpb)
975 FF_VK_EXT_VIDEO_MAINTENANCE_1))1050 hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
976 hwfc->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;1051
1052 ctx = dec->shared_ctx;
1053 if (ctx->s.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
1054 FF_VK_EXT_VIDEO_MAINTENANCE_1))
1055 hwfc->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
1056 } else if (DECODER_IS_SDR(avctx->codec_id)) {
1057 hwfc->usage |= VK_IMAGE_USAGE_STORAGE_BIT;
1058 }
977 1059
978 return err;1060 return err;
979}1061}
@@ -1075,8 +1157,10 @@ int ff_vk_decode_init(AVCodecContext *avctx) · 95 unmodified lines
1075 if (err < 0)1157 if (err < 0)
1076 return err;1158 return err;
1077 1159
1160 vk_desc = get_codecdesc(avctx->codec_id);
1161
1078 profile = get_video_profile(ctx, avctx->codec_id);1162 profile = get_video_profile(ctx, avctx->codec_id);
1079 if (!profile) {1163 if ((vk_desc->queue_flags & VK_QUEUE_VIDEO_DECODE_BIT_KHR) && !profile) {
1080 av_log(avctx, AV_LOG_ERROR, "Video profile missing from frames context!");1164 av_log(avctx, AV_LOG_ERROR, "Video profile missing from frames context!");
1081 return AVERROR(EINVAL);1165 return AVERROR(EINVAL);
1082 }1166 }
@@ -1109,9 +1193,11 @@ int ff_vk_decode_init(AVCodecContext *avctx) · 26 unmodified lines
1109 if (err < 0)1193 if (err < 0)
1110 goto fail;1194 goto fail;
1111 1195
1112 err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create);1196 if (profile) {
1113 if (err < 0)1197 err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create);
1114 goto fail;1198 if (err < 0)
1199 goto fail;
1200 }
1115 1201
1116 /* If doing an out-of-place decoding, create a DPB pool */1202 /* If doing an out-of-place decoding, create a DPB pool */
1117 if (dec->dedicated_dpb || avctx->codec_id == AV_CODEC_ID_AV1) {1203 if (dec->dedicated_dpb || avctx->codec_id == AV_CODEC_ID_AV1) {
@@ -1163,12 +1249,18 @@ int ff_vk_decode_init(AVCodecContext *avctx) · 45 unmodified lines
1163 }1249 }
1164 1250
1165 session_params_create.videoSession = ctx->common.session;1251 session_params_create.videoSession = ctx->common.session;
1166 ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create,1252 if (profile) {
1167 s->hwctx->alloc, &ctx->empty_session_params);1253 ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create,
1168 if (ret != VK_SUCCESS) {1254 s->hwctx->alloc, &ctx->empty_session_params);
1169 av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n",1255 if (ret != VK_SUCCESS) {
1170 ff_vk_ret2str(ret));1256 av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n",
1171 return AVERROR_EXTERNAL;1257 ff_vk_ret2str(ret));
1258 return AVERROR_EXTERNAL;
1259 }
1260 } else {
1261 /* For SDR decoders, this alignment value will be 0. Since this will make
1262 * add_slice() malfunction, set it to a sane default value. */
1263 ctx->caps.minBitstreamBufferSizeAlignment = AV_INPUT_BUFFER_PADDING_SIZE;
1172 }1264 }
1173 1265
1174 driver_props = &dec->shared_ctx->s.driver_props;1266 driver_props = &dec->shared_ctx->s.driver_props;
libavcodec/vulkan_decode.h+11−0
@@ -56,6 +56,10 @@ typedef struct FFVulkanDecodeShared { · 55 unmodified lines
56 VkVideoDecodeCapabilitiesKHR dec_caps;56 VkVideoDecodeCapabilitiesKHR dec_caps;
57 57
58 VkVideoSessionParametersKHR empty_session_params;58 VkVideoSessionParametersKHR empty_session_params;
59
60 /* Software-defined decoder context */
61 void *sd_ctx;
62 void (*sd_ctx_free)(struct FFVulkanDecodeShared *ctx);
59} FFVulkanDecodeShared;63} FFVulkanDecodeShared;
60 64
61typedef struct FFVulkanDecodeContext {65typedef struct FFVulkanDecodeContext {
@@ -141,6 +145,13 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, · 79 unmodified lines
141 FFVulkanDecodePicture *vkpic, int is_current,145 FFVulkanDecodePicture *vkpic, int is_current,
142 int alloc_dpb);146 int alloc_dpb);
143 147
148/**
149 * Software-defined decoder version of ff_vk_decode_prepare_frame.
150 */
151int ff_vk_decode_prepare_frame_sdr(FFVulkanDecodeContext *dec, AVFrame *pic,
152 FFVulkanDecodePicture *vkpic, int is_current,
153 enum FFVkShaderRepFormat rep_fmt, int alloc_dpb);
154
144/**155/**
145 * Add slice data to frame.156 * Add slice data to frame.
146 */157 */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t