History

vulkan_decode: support multiple image views

Enables non-monochrome video decoding using all our existing functions in the context of an SDR decoder.

Lynne committed 1 year ago

Commit 44958021 parent 491b65e5 files · +60 −54Browse files

Unified · Split

5 files
  1. libavcodec/vulkan_av1.c +2 −2
  2. libavcodec/vulkan_decode.c +47 −43
  3. libavcodec/vulkan_decode.h +7 −5
  4. libavcodec/vulkan_h264.c +2 −2
  5. libavcodec/vulkan_hevc.c +2 −2
libavcodec/vulkan_av1.c+2−2
@@ -123,7 +123,7 @@ static int vk_av1_fill_pict(AVCodecContext *avctx, const AV1Frame **ref_src, · 122 unmodified lines
123 .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },123 .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
124 .baseArrayLayer = ((has_grain || dec->dedicated_dpb) && ctx->common.layered_dpb) ?124 .baseArrayLayer = ((has_grain || dec->dedicated_dpb) && ctx->common.layered_dpb) ?
125 hp->frame_id : 0,125 hp->frame_id : 0,
126 .imageViewBinding = vkpic->img_view_ref,126 .imageViewBinding = vkpic->view.ref[0],
127 };127 };
128 128
129 *ref_slot = (VkVideoReferenceSlotInfoKHR) {129 *ref_slot = (VkVideoReferenceSlotInfoKHR) {
@@ -346,7 +346,7 @@ static int vk_av1_start_frame(AVCodecContext *avctx, · 216 unmodified lines
346 .codedOffset = (VkOffset2D){ 0, 0 },346 .codedOffset = (VkOffset2D){ 0, 0 },
347 .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },347 .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
348 .baseArrayLayer = 0,348 .baseArrayLayer = 0,
349 .imageViewBinding = vp->img_view_out,349 .imageViewBinding = vp->view.out[0],
350 },350 },
351 };351 };
352 352
libavcodec/vulkan_decode.c+47−43
@@ -130,9 +130,11 @@ static void init_frame(FFVulkanDecodeContext *dec, FFVulkanDecodePicture *vkpic) · 129 unmodified lines
130 FFVulkanFunctions *vk = &ctx->s.vkfn;130 FFVulkanFunctions *vk = &ctx->s.vkfn;
131 131
132 vkpic->dpb_frame = NULL;132 vkpic->dpb_frame = NULL;
133 vkpic->img_view_ref = VK_NULL_HANDLE;133 for (int i = 0; i < AV_NUM_DATA_POINTERS; i++) {
134 vkpic->img_view_out = VK_NULL_HANDLE;134 vkpic->view.ref[i] = VK_NULL_HANDLE;
135 vkpic->img_view_dest = VK_NULL_HANDLE;135 vkpic->view.out[i] = VK_NULL_HANDLE;
136 vkpic->view.dst[i] = VK_NULL_HANDLE;
137 }
136 138
137 vkpic->destroy_image_view = vk->DestroyImageView;139 vkpic->destroy_image_view = vk->DestroyImageView;
138 vkpic->wait_semaphores = vk->WaitSemaphores;140 vkpic->wait_semaphores = vk->WaitSemaphores;
@@ -149,14 +151,14 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, · 10 unmodified lines
149 151
150 /* If the decoder made a blank frame to make up for a missing ref, or the152 /* If the decoder made a blank frame to make up for a missing ref, or the
151 * frame is the current frame so it's missing one, create a re-representation */153 * frame is the current frame so it's missing one, create a re-representation */
152 if (vkpic->img_view_ref)154 if (vkpic->view.ref[0])
153 return 0;155 return 0;
154 156
155 init_frame(dec, vkpic);157 init_frame(dec, vkpic);
156 158
157 if (ctx->common.layered_dpb && alloc_dpb) {159 if (ctx->common.layered_dpb && alloc_dpb) {
158 vkpic->img_view_ref = ctx->common.layered_view;160 vkpic->view.ref[0] = ctx->common.layered_view;
159 vkpic->img_aspect_ref = ctx->common.layered_aspect;161 vkpic->view.aspect_ref[0] = ctx->common.layered_aspect;
160 } else if (alloc_dpb) {162 } else if (alloc_dpb) {
161 AVHWFramesContext *dpb_frames = (AVHWFramesContext *)ctx->common.dpb_hwfc_ref->data;163 AVHWFramesContext *dpb_frames = (AVHWFramesContext *)ctx->common.dpb_hwfc_ref->data;
162 AVVulkanFramesContext *dpb_hwfc = dpb_frames->hwctx;164 AVVulkanFramesContext *dpb_hwfc = dpb_frames->hwctx;
@@ -166,13 +168,13 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, · 3 unmodified lines
166 return AVERROR(ENOMEM);168 return AVERROR(ENOMEM);
167 169
168 err = ff_vk_create_view(&ctx->s, &ctx->common,170 err = ff_vk_create_view(&ctx->s, &ctx->common,
169 &vkpic->img_view_ref, &vkpic->img_aspect_ref,171 &vkpic->view.ref[0], &vkpic->view.aspect_ref[0],
170 (AVVkFrame *)vkpic->dpb_frame->data[0],172 (AVVkFrame *)vkpic->dpb_frame->data[0],
171 dpb_hwfc->format[0], !is_current);173 dpb_hwfc->format[0], !is_current);
172 if (err < 0)174 if (err < 0)
173 return err;175 return err;
174 176
175 vkpic->img_view_dest = vkpic->img_view_ref;177 vkpic->view.dst[0] = vkpic->view.ref[0];
176 }178 }
177 179
178 if (!alloc_dpb || is_current) {180 if (!alloc_dpb || is_current) {
@@ -180,15 +182,15 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, · 1 unmodified line
180 AVVulkanFramesContext *hwfc = frames->hwctx;182 AVVulkanFramesContext *hwfc = frames->hwctx;
181 183
182 err = ff_vk_create_view(&ctx->s, &ctx->common,184 err = ff_vk_create_view(&ctx->s, &ctx->common,
183 &vkpic->img_view_out, &vkpic->img_aspect,185 &vkpic->view.out[0], &vkpic->view.aspect[0],
184 (AVVkFrame *)pic->data[0],186 (AVVkFrame *)pic->data[0],
185 hwfc->format[0], !is_current);187 hwfc->format[0], !is_current);
186 if (err < 0)188 if (err < 0)
187 return err;189 return err;
188 190
189 if (!alloc_dpb) {191 if (!alloc_dpb) {
190 vkpic->img_view_ref = vkpic->img_view_out;192 vkpic->view.ref[0] = vkpic->view.out[0];
191 vkpic->img_aspect_ref = vkpic->img_aspect;193 vkpic->view.aspect_ref[0] = vkpic->view.aspect[0];
192 }194 }
193 }195 }
194 196
@@ -201,41 +203,41 @@ int ff_vk_decode_prepare_frame_sdr(FFVulkanDecodeContext *dec, AVFrame *pic, · 6 unmodified lines
201{203{
202 int err;204 int err;
203 FFVulkanDecodeShared *ctx = dec->shared_ctx;205 FFVulkanDecodeShared *ctx = dec->shared_ctx;
206 AVHWFramesContext *frames = (AVHWFramesContext *)pic->hw_frames_ctx->data;
204 207
205 vkpic->slices_size = 0;208 vkpic->slices_size = 0;
206 209
207 if (vkpic->img_view_ref)210 if (vkpic->view.ref[0])
208 return 0;211 return 0;
209 212
210 init_frame(dec, vkpic);213 init_frame(dec, vkpic);
211 214
212 if (ctx->common.layered_dpb && alloc_dpb) {215 for (int i = 0; i < av_pix_fmt_count_planes(frames->sw_format); i++) {
213 vkpic->img_view_ref = ctx->common.layered_view;216 if (alloc_dpb) {
214 vkpic->img_aspect_ref = ctx->common.layered_aspect;217 vkpic->dpb_frame = vk_get_dpb_pool(ctx);
215 } else if (alloc_dpb) {218 if (!vkpic->dpb_frame)
216 vkpic->dpb_frame = vk_get_dpb_pool(ctx);219 return AVERROR(ENOMEM);
217 if (!vkpic->dpb_frame)
218 return AVERROR(ENOMEM);
219 220
220 err = ff_vk_create_imageview(&ctx->s,221 err = ff_vk_create_imageview(&ctx->s,
221 &vkpic->img_view_ref, &vkpic->img_aspect_ref,222 &vkpic->view.ref[i], &vkpic->view.aspect_ref[i],
222 vkpic->dpb_frame, 0, rep_fmt);223 vkpic->dpb_frame, i, rep_fmt);
223 if (err < 0)224 if (err < 0)
224 return err;225 return err;
225 226
226 vkpic->img_view_dest = vkpic->img_view_ref;227 vkpic->view.dst[i] = vkpic->view.ref[i];
227 }228 }
228 229
229 if (!alloc_dpb || is_current) {230 if (!alloc_dpb || is_current) {
230 err = ff_vk_create_imageview(&ctx->s,231 err = ff_vk_create_imageview(&ctx->s,
231 &vkpic->img_view_out, &vkpic->img_aspect,232 &vkpic->view.out[i], &vkpic->view.aspect[i],
232 pic, 0, rep_fmt);233 pic, i, rep_fmt);
233 if (err < 0)234 if (err < 0)
234 return err;235 return err;
235 236
236 if (!alloc_dpb) {237 if (!alloc_dpb) {
237 vkpic->img_view_ref = vkpic->img_view_out;238 vkpic->view.ref[i] = vkpic->view.out[i];
238 vkpic->img_aspect_ref = vkpic->img_aspect;239 vkpic->view.aspect_ref[i] = vkpic->view.aspect[i];
240 }
239 }241 }
240 }242 }
241 243
@@ -467,7 +469,7 @@ int ff_vk_decode_frame(AVCodecContext *avctx, · 225 unmodified lines
467 .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,469 .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
468 .image = vkf->img[0],470 .image = vkf->img[0],
469 .subresourceRange = (VkImageSubresourceRange) {471 .subresourceRange = (VkImageSubresourceRange) {
470 .aspectMask = vp->img_aspect,472 .aspectMask = vp->view.aspect[0],
471 .layerCount = 1,473 .layerCount = 1,
472 .levelCount = 1,474 .levelCount = 1,
473 },475 },
@@ -523,7 +525,7 @@ int ff_vk_decode_frame(AVCodecContext *avctx, · 49 unmodified lines
523 .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,525 .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
524 .image = rvkf->img[0],526 .image = rvkf->img[0],
525 .subresourceRange = (VkImageSubresourceRange) {527 .subresourceRange = (VkImageSubresourceRange) {
526 .aspectMask = rvp->img_aspect_ref,528 .aspectMask = rvp->view.aspect_ref[0],
527 .layerCount = 1,529 .layerCount = 1,
528 .levelCount = 1,530 .levelCount = 1,
529 },531 },
@@ -533,7 +535,7 @@ int ff_vk_decode_frame(AVCodecContext *avctx, · 3 unmodified lines
533 }535 }
534 }536 }
535 } else if (vp->decode_info.referenceSlotCount ||537 } else if (vp->decode_info.referenceSlotCount ||
536 vp->img_view_out != vp->img_view_ref) {538 vp->view.out[0] != vp->view.ref[0]) {
537 /* Single barrier for a single layered ref */539 /* Single barrier for a single layered ref */
538 err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ctx->common.layered_frame,540 err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ctx->common.layered_frame,
539 VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,541 VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
@@ -580,12 +582,14 @@ void ff_vk_decode_free_frame(AVHWDeviceContext *dev_ctx, FFVulkanDecodePicture * · 40 unmodified lines
580 av_buffer_unref(&vp->slices_buf);582 av_buffer_unref(&vp->slices_buf);
581 583
582 /* Destroy image view (out) */584 /* Destroy image view (out) */
583 if (vp->img_view_out && vp->img_view_out != vp->img_view_dest)585 for (int i = 0; i < AV_NUM_DATA_POINTERS; i++) {
584 vp->destroy_image_view(hwctx->act_dev, vp->img_view_out, hwctx->alloc);586 if (vp->view.out[i] && vp->view.out[i] != vp->view.dst[i])
587 vp->destroy_image_view(hwctx->act_dev, vp->view.out[i], hwctx->alloc);
585 588
586 /* Destroy image view (ref, unlayered) */589 /* Destroy image view (ref, unlayered) */
587 if (vp->img_view_dest)590 if (vp->view.dst[i])
588 vp->destroy_image_view(hwctx->act_dev, vp->img_view_dest, hwctx->alloc);591 vp->destroy_image_view(hwctx->act_dev, vp->view.dst[i], hwctx->alloc);
592 }
589 593
590 av_frame_free(&vp->dpb_frame);594 av_frame_free(&vp->dpb_frame);
591}595}
libavcodec/vulkan_decode.h+7−5
@@ -85,11 +85,13 @@ typedef struct FFVulkanDecodeContext { · 84 unmodified lines
85typedef struct FFVulkanDecodePicture {85typedef struct FFVulkanDecodePicture {
86 AVFrame *dpb_frame; /* Only used for out-of-place decoding. */86 AVFrame *dpb_frame; /* Only used for out-of-place decoding. */
87 87
88 VkImageView img_view_ref; /* Image representation view (reference) */88 struct {
89 VkImageView img_view_out; /* Image representation view (output-only) */89 VkImageView ref[AV_NUM_DATA_POINTERS]; /* Image representation view (reference) */
90 VkImageView img_view_dest; /* Set to img_view_out if no layered refs are used */90 VkImageView out[AV_NUM_DATA_POINTERS]; /* Image representation view (output-only) */
91 VkImageAspectFlags img_aspect; /* Image plane mask bits */91 VkImageView dst[AV_NUM_DATA_POINTERS]; /* Set to img_view_out if no layered refs are used */
92 VkImageAspectFlags img_aspect_ref; /* Only used for out-of-place decoding */92 VkImageAspectFlags aspect[AV_NUM_DATA_POINTERS]; /* Image plane mask bits */
93 VkImageAspectFlags aspect_ref[AV_NUM_DATA_POINTERS]; /* Only used for out-of-place decoding */
94 } view;
93 95
94 VkSemaphore sem;96 VkSemaphore sem;
95 uint64_t sem_value;97 uint64_t sem_value;
libavcodec/vulkan_h264.c+2−2
@@ -98,7 +98,7 @@ static int vk_h264_fill_pict(AVCodecContext *avctx, H264Picture **ref_src, · 97 unmodified lines
98 .codedOffset = (VkOffset2D){ 0, 0 },98 .codedOffset = (VkOffset2D){ 0, 0 },
99 .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },99 .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
100 .baseArrayLayer = ctx->common.layered_dpb ? dpb_slot_index : 0,100 .baseArrayLayer = ctx->common.layered_dpb ? dpb_slot_index : 0,
101 .imageViewBinding = vkpic->img_view_ref,101 .imageViewBinding = vkpic->view.ref[0],
102 };102 };
103 103
104 *ref_slot = (VkVideoReferenceSlotInfoKHR) {104 *ref_slot = (VkVideoReferenceSlotInfoKHR) {
@@ -471,7 +471,7 @@ static int vk_h264_start_frame(AVCodecContext *avctx, · 366 unmodified lines
471 .codedOffset = (VkOffset2D){ 0, 0 },471 .codedOffset = (VkOffset2D){ 0, 0 },
472 .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },472 .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
473 .baseArrayLayer = 0,473 .baseArrayLayer = 0,
474 .imageViewBinding = vp->img_view_out,474 .imageViewBinding = vp->view.out[0],
475 },475 },
476 };476 };
477 477
libavcodec/vulkan_hevc.c+2−2
@@ -164,7 +164,7 @@ static int vk_hevc_fill_pict(AVCodecContext *avctx, HEVCFrame **ref_src, · 163 unmodified lines
164 .codedOffset = (VkOffset2D){ 0, 0 },164 .codedOffset = (VkOffset2D){ 0, 0 },
165 .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },165 .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
166 .baseArrayLayer = ctx->common.layered_dpb ? pic_id : 0,166 .baseArrayLayer = ctx->common.layered_dpb ? pic_id : 0,
167 .imageViewBinding = vkpic->img_view_ref,167 .imageViewBinding = vkpic->view.ref[0],
168 };168 };
169 169
170 *ref_slot = (VkVideoReferenceSlotInfoKHR) {170 *ref_slot = (VkVideoReferenceSlotInfoKHR) {
@@ -823,7 +823,7 @@ static int vk_hevc_start_frame(AVCodecContext *avctx, · 652 unmodified lines
823 .codedOffset = (VkOffset2D){ 0, 0 },823 .codedOffset = (VkOffset2D){ 0, 0 },
824 .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },824 .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
825 .baseArrayLayer = 0,825 .baseArrayLayer = 0,
826 .imageViewBinding = vp->img_view_out,826 .imageViewBinding = vp->view.out[0],
827 },827 },
828 };828 };
829 829

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t