History

lavc: add Vulkan video encoding base code

This commit adds the common Vulkan video encoding framework. It makes full use of the asynchronous features of our new common hardware encoding code, and of Vulkan. The code is able to handle anything from H264 to AV1 and MJPEG.

Lynne committed 2 years ago

Commit 37243b21 parent 0fa6f334 files · +1281 −1Browse files

Unified · Split

4 files
  1. configure +2 −0
  2. libavcodec/Makefile +1 −1
  3. libavcodec/vulkan_encode.c +1025 −0
  4. libavcodec/vulkan_encode.h +253 −0
configure+2−0
@@ -2639,6 +2639,7 @@ CONFIG_EXTRA=" · 2638 unmodified lines
2639 vp3dsp2639 vp3dsp
2640 vp56dsp2640 vp56dsp
2641 vp8dsp2641 vp8dsp
2642 vulkan_encode
2642 wma_freqs2643 wma_freqs
2643 wmv2dsp2644 wmv2dsp
2644"2645"
@@ -3300,6 +3301,7 @@ qsvdec_select="qsv" · 655 unmodified lines
3300qsvenc_select="qsv"3301qsvenc_select="qsv"
3301qsvvpp_select="qsv"3302qsvvpp_select="qsv"
3302vaapi_encode_deps="vaapi"3303vaapi_encode_deps="vaapi"
3304vulkan_encode_deps="vulkan"
3303v4l2_m2m_deps="linux_videodev2_h sem_timedwait"3305v4l2_m2m_deps="linux_videodev2_h sem_timedwait"
3304 3306
3305bilateral_cuda_filter_deps="ffnvcodec"3307bilateral_cuda_filter_deps="ffnvcodec"
libavcodec/Makefile+1−1
@@ -1282,7 +1282,7 @@ SKIPHEADERS-$(CONFIG_QSVENC) += qsvenc.h · 1281 unmodified lines
1282SKIPHEADERS-$(CONFIG_VAAPI) += vaapi_decode.h vaapi_hevc.h vaapi_encode.h1282SKIPHEADERS-$(CONFIG_VAAPI) += vaapi_decode.h vaapi_hevc.h vaapi_encode.h
1283SKIPHEADERS-$(CONFIG_VDPAU) += vdpau.h vdpau_internal.h1283SKIPHEADERS-$(CONFIG_VDPAU) += vdpau.h vdpau_internal.h
1284SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX) += videotoolbox.h vt_internal.h1284SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX) += videotoolbox.h vt_internal.h
1285SKIPHEADERS-$(CONFIG_VULKAN) += vulkan.h vulkan_video.h vulkan_decode.h1285SKIPHEADERS-$(CONFIG_VULKAN) += vulkan.h vulkan_video.h vulkan_encode.h vulkan_decode.h
1286SKIPHEADERS-$(CONFIG_V4L2_M2M) += v4l2_buffers.h v4l2_context.h v4l2_m2m.h1286SKIPHEADERS-$(CONFIG_V4L2_M2M) += v4l2_buffers.h v4l2_context.h v4l2_m2m.h
1287SKIPHEADERS-$(CONFIG_ZLIB) += zlib_wrapper.h1287SKIPHEADERS-$(CONFIG_ZLIB) += zlib_wrapper.h
1288 1288
libavcodec/vulkan_encode.c+1025−0
@@ -0,0 +1,1025 @@
1/*
2 * This file is part of FFmpeg.
3 *
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19#include "libavutil/mem.h"
20#include "libavutil/avassert.h"
21#include "vulkan_encode.h"
22#include "config.h"
23
24#include "libavutil/vulkan_loader.h"
25
26const AVCodecHWConfigInternal *const ff_vulkan_encode_hw_configs[] = {
27 HW_CONFIG_ENCODER_FRAMES(VULKAN, VULKAN),
28 NULL,
29};
30
31av_cold void ff_vulkan_encode_uninit(FFVulkanEncodeContext *ctx)
32{
33 FFVulkanContext *s = &ctx->s;
34 FFVulkanFunctions *vk = &s->vkfn;
35
36 /* Wait on and free execution pool */
37 ff_vk_exec_pool_free(s, &ctx->enc_pool);
38
39 /* Destroy the session params */
40 if (ctx->session_params)
41 vk->DestroyVideoSessionParametersKHR(s->hwctx->act_dev,
42 ctx->session_params,
43 s->hwctx->alloc);
44
45 ff_hw_base_encode_close(&ctx->base);
46
47 av_buffer_pool_uninit(&ctx->buf_pool);
48
49 ff_vk_video_common_uninit(s, &ctx->common);
50
51 ff_vk_uninit(s);
52}
53
54static int vulkan_encode_init(AVCodecContext *avctx, FFHWBaseEncodePicture *pic)
55{
56 int err;
57 FFVulkanEncodeContext *ctx = avctx->priv_data;
58 FFVulkanEncodePicture *vp = pic->priv;
59
60 AVFrame *f = pic->input_image;
61 AVHWFramesContext *hwfc = (AVHWFramesContext *)f->hw_frames_ctx->data;
62 AVVulkanFramesContext *vkfc = hwfc->hwctx;
63 AVVkFrame *vkf = (AVVkFrame *)f->data[0];
64
65 if (ctx->codec->picture_priv_data_size > 0) {
66 pic->codec_priv = av_mallocz(ctx->codec->picture_priv_data_size);
67 if (!pic->codec_priv)
68 return AVERROR(ENOMEM);
69 }
70
71 /* Input image view */
72 err = ff_vk_create_view(&ctx->s, &ctx->common,
73 &vp->in.view, &vp->in.aspect,
74 vkf, vkfc->format[0], 0);
75 if (err < 0)
76 return err;
77
78 /* Reference view */
79 if (!ctx->common.layered_dpb) {
80 AVFrame *rf = pic->recon_image;
81 AVVkFrame *rvkf = (AVVkFrame *)rf->data[0];
82 err = ff_vk_create_view(&ctx->s, &ctx->common,
83 &vp->dpb.view, &vp->dpb.aspect,
84 rvkf, ctx->pic_format, 1);
85 if (err < 0)
86 return err;
87 } else {
88 vp->dpb.view = ctx->common.layered_view;
89 vp->dpb.aspect = ctx->common.layered_aspect;
90 }
91
92 return 0;
93}
94
95static int vulkan_encode_free(AVCodecContext *avctx, FFHWBaseEncodePicture *pic)
96{
97 FFVulkanEncodeContext *ctx = avctx->priv_data;
98 FFVulkanFunctions *vk = &ctx->s.vkfn;
99
100 FFVulkanEncodePicture *vp = pic->priv;
101
102 if (vp->in.view)
103 vk->DestroyImageView(ctx->s.hwctx->act_dev, vp->in.view,
104 ctx->s.hwctx->alloc);
105
106 if (!ctx->common.layered_dpb && vp->dpb.view)
107 vk->DestroyImageView(ctx->s.hwctx->act_dev, vp->dpb.view,
108 ctx->s.hwctx->alloc);
109
110 ctx->slots[vp->dpb_slot.slotIndex] = NULL;
111
112 return 0;
113}
114
115static int init_pic_rc(AVCodecContext *avctx, FFHWBaseEncodePicture *pic,
116 VkVideoEncodeRateControlInfoKHR *rc_info,
117 VkVideoEncodeRateControlLayerInfoKHR *rc_layer /* Goes in ^ */)
118{
119 FFVulkanEncodeContext *ctx = avctx->priv_data;
120
121 *rc_info = (VkVideoEncodeRateControlInfoKHR) {
122 .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_RATE_CONTROL_INFO_KHR,
123 .rateControlMode = ctx->opts.rc_mode,
124 };
125
126 if (ctx->opts.rc_mode > VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR) {
127 *rc_layer = (VkVideoEncodeRateControlLayerInfoKHR) {
128 .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_RATE_CONTROL_LAYER_INFO_KHR,
129 .averageBitrate = avctx->bit_rate,
130 .maxBitrate = avctx->rc_max_rate ? avctx->rc_max_rate : avctx->bit_rate,
131 .frameRateNumerator = avctx->framerate.num,
132 .frameRateDenominator = avctx->framerate.den,
133 };
134 rc_info->layerCount++;
135 rc_info->pLayers = rc_layer;
136 }
137
138 return ctx->codec->init_pic_rc(avctx, pic, rc_info, rc_layer);
139}
140
141static int vulkan_encode_issue(AVCodecContext *avctx,
142 FFHWBaseEncodePicture *base_pic)
143{
144 FFVulkanEncodeContext *ctx = avctx->priv_data;
145 FFVulkanFunctions *vk = &ctx->s.vkfn;
146
147 const size_t size_align = ctx->caps.minBitstreamBufferSizeAlignment;
148
149 FFVulkanEncodePicture *vp = base_pic->priv;
150 AVFrame *src = (AVFrame *)base_pic->input_image;
151 AVVkFrame *vkf = (AVVkFrame *)src->data[0];
152
153 int err, max_pkt_size;
154
155 FFVkBuffer *sd_buf;
156
157 int slot_index = -1;
158 FFVkExecContext *exec;
159 VkCommandBuffer cmd_buf;
160 VkImageMemoryBarrier2 img_bar[37];
161 int nb_img_bar = 0;
162
163 /* Coding start/end */
164 VkVideoBeginCodingInfoKHR encode_start;
165 VkVideoEndCodingInfoKHR encode_end = {
166 .sType = VK_STRUCTURE_TYPE_VIDEO_END_CODING_INFO_KHR,
167 };
168
169 VkVideoEncodeRateControlLayerInfoKHR rc_layer;
170 VkVideoEncodeRateControlInfoKHR rc_info;
171 VkVideoEncodeQualityLevelInfoKHR q_info;
172 VkVideoCodingControlInfoKHR encode_ctrl;
173
174 VkVideoReferenceSlotInfoKHR ref_slot[37];
175 VkVideoEncodeInfoKHR encode_info;
176
177 /* Create packet data buffer */
178 max_pkt_size = FFALIGN(3 * ctx->base.surface_width * ctx->base.surface_height + (1 << 16),
179 ctx->caps.minBitstreamBufferSizeAlignment);
180
181 err = ff_vk_get_pooled_buffer(&ctx->s, &ctx->buf_pool, &vp->pkt_buf,
182 VK_BUFFER_USAGE_VIDEO_ENCODE_DST_BIT_KHR,
183 &ctx->profile_list, max_pkt_size,
184 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
185 VK_MEMORY_PROPERTY_HOST_CACHED_BIT);
186 if (err < 0)
187 return err;
188
189 sd_buf = (FFVkBuffer *)vp->pkt_buf->data;
190
191 /* Setup rate control */
192 err = init_pic_rc(avctx, base_pic, &rc_info, &rc_layer);
193 if (err < 0)
194 return err;
195
196 q_info = (VkVideoEncodeQualityLevelInfoKHR) {
197 .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_QUALITY_LEVEL_INFO_KHR,
198 .pNext = &rc_info,
199 .qualityLevel = ctx->opts.quality,
200 };
201 encode_ctrl = (VkVideoCodingControlInfoKHR) {
202 .sType = VK_STRUCTURE_TYPE_VIDEO_CODING_CONTROL_INFO_KHR,
203 .pNext = &q_info,
204 .flags = VK_VIDEO_CODING_CONTROL_ENCODE_QUALITY_LEVEL_BIT_KHR |
205 VK_VIDEO_CODING_CONTROL_ENCODE_RATE_CONTROL_BIT_KHR |
206 VK_VIDEO_CODING_CONTROL_RESET_BIT_KHR,
207 };
208
209 for (int i = 0; i < ctx->caps.maxDpbSlots; i++) {
210 if (ctx->slots[i] == NULL) {
211 slot_index = i;
212 ctx->slots[i] = base_pic;
213 break;
214 }
215 }
216 av_assert0(slot_index >= 0);
217
218 /* Current picture's ref slot */
219 vp->dpb_res = (VkVideoPictureResourceInfoKHR) {
220 .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
221 .pNext = NULL,
222 .codedOffset = { 0 },
223 .codedExtent = (VkExtent2D){ ctx->base.surface_width,
224 ctx->base.surface_height },
225 .baseArrayLayer = ctx->common.layered_dpb ? slot_index : 0,
226 .imageViewBinding = vp->dpb.view,
227 };
228
229 vp->dpb_slot = (VkVideoReferenceSlotInfoKHR) {
230 .sType = VK_STRUCTURE_TYPE_VIDEO_REFERENCE_SLOT_INFO_KHR,
231 .pNext = NULL, // Set later
232 .slotIndex = slot_index,
233 .pPictureResource = &vp->dpb_res,
234 };
235
236 encode_info = (VkVideoEncodeInfoKHR) {
237 .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_INFO_KHR,
238 .pNext = NULL, // Set later
239 .flags = 0x0,
240 .srcPictureResource = (VkVideoPictureResourceInfoKHR) {
241 .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
242 .pNext = NULL,
243 .codedOffset = { 0, 0 },
244 .codedExtent = (VkExtent2D){ base_pic->input_image->width,
245 base_pic->input_image->height },
246 .baseArrayLayer = 0,
247 .imageViewBinding = vp->in.view,
248 },
249 .pSetupReferenceSlot = &vp->dpb_slot,
250 .referenceSlotCount = 0,
251 .pReferenceSlots = ref_slot,
252 .dstBuffer = sd_buf->buf,
253 .dstBufferOffset = 0,
254 .dstBufferRange = sd_buf->size,
255 .precedingExternallyEncodedBytes = 0,
256 };
257
258 for (int i = 0; i < MAX_REFERENCE_LIST_NUM; i++) {
259 for (int j = 0; j < base_pic->nb_refs[i]; j++) {
260 FFHWBaseEncodePicture *ref = base_pic->refs[i][j];
261 FFVulkanEncodePicture *rvp = ref->priv;
262 ref_slot[encode_info.referenceSlotCount++] = rvp->dpb_slot;
263 }
264 }
265
266 /* Calling vkCmdBeginVideoCodingKHR requires to declare all references
267 * being enabled upfront, including the current frame's output ref. */
268 ref_slot[encode_info.referenceSlotCount] = vp->dpb_slot;
269 ref_slot[encode_info.referenceSlotCount].slotIndex = -1;
270
271 /* Setup picture parameters */
272 err = ctx->codec->init_pic_params(avctx, base_pic,
273 &encode_info);
274 if (err < 0)
275 return err;
276
277 encode_start = (VkVideoBeginCodingInfoKHR) {
278 .sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR,
279 .pNext = !base_pic->force_idr ? &rc_info : NULL,
280 .videoSession = ctx->common.session,
281 .videoSessionParameters = ctx->session_params,
282 .referenceSlotCount = encode_info.referenceSlotCount + 1,
283 .pReferenceSlots = ref_slot,
284 };
285
286 /* Write header */
287 if (base_pic->type == FF_HW_PICTURE_TYPE_IDR) {
288 uint8_t *hdr_dst = sd_buf->mapped_mem + encode_info.dstBufferOffset;
289 size_t data_size = encode_info.dstBufferRange;
290 err = ctx->codec->write_sequence_headers(avctx, base_pic, hdr_dst, &data_size);
291 if (err < 0)
292 goto fail;
293 encode_info.dstBufferOffset += data_size;
294 encode_info.dstBufferRange -= data_size;
295 }
296
297 /* Write extra units */
298 if (ctx->codec->write_extra_headers) {
299 uint8_t *hdr_dst = sd_buf->mapped_mem + encode_info.dstBufferOffset;
300 size_t data_size = encode_info.dstBufferRange;
301 err = ctx->codec->write_extra_headers(avctx, base_pic, hdr_dst, &data_size);
302 if (err < 0)
303 goto fail;
304 encode_info.dstBufferOffset += data_size;
305 encode_info.dstBufferRange -= data_size;
306 }
307
308 /* Align buffer offset to the required value with filler units */
309 if (ctx->codec->write_filler) {
310 uint8_t *hdr_dst = sd_buf->mapped_mem + encode_info.dstBufferOffset;
311 size_t data_size = encode_info.dstBufferRange;
312
313 uint32_t offset = encode_info.dstBufferOffset;
314 size_t offset_align = ctx->caps.minBitstreamBufferOffsetAlignment;
315
316 uint32_t filler_data = FFALIGN(offset, offset_align) - offset;
317
318 if (filler_data) {
319 while (filler_data < ctx->codec->filler_header_size)
320 filler_data += offset_align;
321
322 filler_data -= ctx->codec->filler_header_size;
323
324 err = ctx->codec->write_filler(avctx, filler_data,
325 hdr_dst, &data_size);
326 if (err < 0)
327 goto fail;
328
329 encode_info.dstBufferOffset += data_size;
330 encode_info.dstBufferRange -= data_size;
331 }
332 }
333
334 vp->slices_offset = encode_info.dstBufferOffset;
335
336 /* Align buffer size to the nearest lower alignment requirement. */
337 encode_info.dstBufferRange -= size_align;
338 encode_info.dstBufferRange = FFALIGN(encode_info.dstBufferRange,
339 size_align);
340
341 /* Start command buffer recording */
342 exec = vp->exec = ff_vk_exec_get(&ctx->enc_pool);
343 ff_vk_exec_start(&ctx->s, exec);
344 cmd_buf = exec->buf;
345
346 /* Output packet buffer */
347 err = ff_vk_exec_add_dep_buf(&ctx->s, exec, &vp->pkt_buf, 1, 1);
348 if (err < 0)
349 goto fail;
350
351 /* Source image */
352 err = ff_vk_exec_add_dep_frame(&ctx->s, exec, src,
353 VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
354 VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR);
355 if (err < 0)
356 goto fail;
357
358 /* Source image layout conversion */
359 img_bar[nb_img_bar] = (VkImageMemoryBarrier2) {
360 .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
361 .pNext = NULL,
362 .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
363 .srcAccessMask = vkf->access[0],
364 .dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR,
365 .dstAccessMask = VK_ACCESS_2_VIDEO_ENCODE_READ_BIT_KHR,
366 .oldLayout = vkf->layout[0],
367 .newLayout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_SRC_KHR,
368 .srcQueueFamilyIndex = vkf->queue_family[0],
369 .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
370 .image = vkf->img[0],
371 .subresourceRange = (VkImageSubresourceRange) {
372 .aspectMask = vp->in.aspect,
373 .layerCount = 1,
374 .levelCount = 1,
375 },
376 };
377 ff_vk_exec_update_frame(&ctx->s, exec, src,
378 &img_bar[nb_img_bar], &nb_img_bar);
379
380 if (!ctx->common.layered_dpb) {
381 /* Source image's ref slot.
382 * No need to do a layout conversion, since the frames which are allocated
383 * with a DPB usage are automatically converted. */
384 err = ff_vk_exec_add_dep_frame(&ctx->s, exec, base_pic->recon_image,
385 VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
386 VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR);
387 if (err < 0)
388 return err;
389
390 /* All references */
391 for (int i = 0; i < MAX_REFERENCE_LIST_NUM; i++) {
392 for (int j = 0; j < base_pic->nb_refs[i]; j++) {
393 FFHWBaseEncodePicture *ref = base_pic->refs[i][j];
394 err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ref->recon_image,
395 VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
396 VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR);
397 if (err < 0)
398 return err;
399 }
400 }
401 } else {
402 err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ctx->common.layered_frame,
403 VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR,
404 VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR);
405 if (err < 0)
406 return err;
407 }
408
409 /* Change image layout */
410 vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
411 .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
412 .pImageMemoryBarriers = img_bar,
413 .imageMemoryBarrierCount = nb_img_bar,
414 });
415
416 /* Start, use parameters */
417 vk->CmdBeginVideoCodingKHR(cmd_buf, &encode_start);
418
419 /* Send control data */
420 if (!ctx->session_reset) {
421 vk->CmdControlVideoCodingKHR(cmd_buf, &encode_ctrl);
422 ctx->session_reset++;
423 }
424
425 /* Encode */
426 vk->CmdBeginQuery(cmd_buf, ctx->enc_pool.query_pool, exec->query_idx + 0, 0);
427 vk->CmdEncodeVideoKHR(cmd_buf, &encode_info);
428 vk->CmdEndQuery(cmd_buf, ctx->enc_pool.query_pool, exec->query_idx + 0);
429
430 /* End encoding */
431 vk->CmdEndVideoCodingKHR(cmd_buf, &encode_end);
432
433 /* End recording and submit for execution */
434 err = ff_vk_exec_submit(&ctx->s, vp->exec);
435 if (err < 0)
436 goto fail;
437
438 /* We don't need to keep the input image any longer, its already ref'd */
439 av_frame_free(&base_pic->input_image);
440
441 return 0;
442
443fail:
444 return err;
445}
446
447static void vulkan_encode_wait(AVCodecContext *avctx,
448 FFHWBaseEncodePicture *base_pic)
449{
450 FFVulkanEncodeContext *ctx = avctx->priv_data;
451 FFVulkanEncodePicture *vp = base_pic->priv;
452
453 av_assert0(base_pic->encode_issued);
454
455 if (base_pic->encode_complete)
456 return;
457
458 ff_vk_exec_wait(&ctx->s, vp->exec);
459 base_pic->encode_complete = 1;
460}
461
462static int vulkan_encode_output(AVCodecContext *avctx,
463 FFHWBaseEncodePicture *base_pic, AVPacket *pkt)
464{
465 VkResult ret;
466 FFVulkanEncodePicture *vp = base_pic->priv;
467 FFVulkanEncodeContext *ctx = avctx->priv_data;
468 FFVkBuffer *sd_buf = (FFVkBuffer *)vp->pkt_buf->data;
469 uint32_t *query_data;
470
471 vulkan_encode_wait(avctx, base_pic);
472
473 ret = ff_vk_exec_get_query(&ctx->s, vp->exec, (void **)&query_data, 0);
474 if (ret == VK_NOT_READY) {
475 av_log(avctx, AV_LOG_ERROR, "Unable to perform query: %s!\n",
476 ff_vk_ret2str(ret));
477 return AVERROR(EINVAL);
478 }
479
480 if (ret != VK_NOT_READY && ret != VK_SUCCESS) {
481 av_log(avctx, AV_LOG_ERROR, "Unable to perform query: %s!\n",
482 ff_vk_ret2str(ret));
483 return AVERROR_EXTERNAL;
484 }
485
486 if (query_data[2] != VK_QUERY_RESULT_STATUS_COMPLETE_KHR) {
487 av_log(avctx, AV_LOG_ERROR, "Unable to encode: %u\n", query_data[2]);
488 return AVERROR_EXTERNAL;
489 }
490
491 /* Invalidate buffer if needed */
492 if (!(sd_buf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) {
493 FFVulkanFunctions *vk = &ctx->s.vkfn;
494 VkMappedMemoryRange invalidate_buf;
495
496 int offs = vp->slices_offset;
497 /* If the non-coherent alignment is greater than the bitstream buffer
498 * offset's alignment, and the offs value is not aligned already,
499 * align it to the previous alignment point. */
500 if (ctx->s.props.properties.limits.nonCoherentAtomSize >
501 ctx->caps.minBitstreamBufferOffsetAlignment && offs &&
502 (FFALIGN(offs, ctx->s.props.properties.limits.nonCoherentAtomSize) != offs)) {
503 offs -= ctx->s.props.properties.limits.nonCoherentAtomSize;
504 offs = FFALIGN(FFMAX(offs, 0), ctx->s.props.properties.limits.nonCoherentAtomSize);
505 }
506
507 invalidate_buf = (VkMappedMemoryRange) {
508 .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
509 .memory = sd_buf->mem,
510 .offset = offs,
511 .size = VK_WHOLE_SIZE,
512 };
513
514 vk->FlushMappedMemoryRanges(ctx->s.hwctx->act_dev, 1, &invalidate_buf);
515 }
516
517 pkt->data = sd_buf->mapped_mem;
518 pkt->size = vp->slices_offset + /* base offset */
519 query_data[0] /* secondary offset */ +
520 query_data[1] /* size */;
521
522 /* Move reference */
523 pkt->buf = vp->pkt_buf;
524 vp->pkt_buf = NULL;
525
526 av_log(avctx, AV_LOG_DEBUG, "Frame %"PRId64"/%"PRId64 " encoded\n",
527 base_pic->display_order, base_pic->encode_order);
528
529 return ff_hw_base_encode_set_output_property(&ctx->base, avctx,
530 base_pic, pkt,
531 ctx->codec->flags & VK_ENC_FLAG_NO_DELAY);
532}
533
534static const FFHWEncodePictureOperation vulkan_base_encode_ops = {
535 .priv_size = sizeof(FFVulkanEncodePicture),
536 .init = &vulkan_encode_init,
537 .issue = &vulkan_encode_issue,
538 .output = &vulkan_encode_output,
539 .free = &vulkan_encode_free,
540};
541
542int ff_vulkan_encode_receive_packet(AVCodecContext *avctx, AVPacket *pkt)
543{
544 FFVulkanEncodeContext *ctx = avctx->priv_data;
545 return ff_hw_base_encode_receive_packet(&ctx->base, avctx, pkt);
546}
547
548static int vulkan_encode_create_dpb(AVCodecContext *avctx, FFVulkanEncodeContext *ctx)
549{
550 int err;
551 FFHWBaseEncodeContext *base_ctx = &ctx->base;
552 AVVulkanFramesContext *hwfc;
553
554 enum AVPixelFormat dpb_format;
555 err = ff_hw_base_get_recon_format(base_ctx, NULL, &dpb_format);
556 if (err < 0)
557 return err;
558
559 base_ctx->recon_frames_ref = av_hwframe_ctx_alloc(base_ctx->device_ref);
560 if (!base_ctx->recon_frames_ref)
561 return AVERROR(ENOMEM);
562
563 base_ctx->recon_frames = (AVHWFramesContext *)base_ctx->recon_frames_ref->data;
564 hwfc = (AVVulkanFramesContext *)base_ctx->recon_frames->hwctx;
565
566 base_ctx->recon_frames->format = AV_PIX_FMT_VULKAN;
567 base_ctx->recon_frames->sw_format = dpb_format;
568 base_ctx->recon_frames->width = base_ctx->surface_width;
569 base_ctx->recon_frames->height = base_ctx->surface_height;
570
571 hwfc->format[0] = ctx->pic_format;
572 hwfc->create_pnext = &ctx->profile_list;
573 hwfc->tiling = VK_IMAGE_TILING_OPTIMAL;
574 hwfc->usage = VK_IMAGE_USAGE_SAMPLED_BIT |
575 VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR;
576
577 if (ctx->common.layered_dpb)
578 hwfc->nb_layers = ctx->caps.maxDpbSlots;
579
580 err = av_hwframe_ctx_init(base_ctx->recon_frames_ref);
581 if (err < 0) {
582 av_log(avctx, AV_LOG_ERROR, "Failed to initialise DPB frame context: %s\n",
583 av_err2str(err));
584 return err;
585 }
586
587 if (ctx->common.layered_dpb) {
588 ctx->common.layered_frame = av_frame_alloc();
589 if (!ctx->common.layered_frame)
590 return AVERROR(ENOMEM);
591
592 err = av_hwframe_get_buffer(base_ctx->recon_frames_ref,
593 ctx->common.layered_frame, 0);
594 if (err < 0)
595 return AVERROR(ENOMEM);
596
597 err = ff_vk_create_view(&ctx->s, &ctx->common,
598 &ctx->common.layered_view,
599 &ctx->common.layered_aspect,
600 (AVVkFrame *)ctx->common.layered_frame->data[0],
601 hwfc->format[0], 1);
602 if (err < 0)
603 return err;
604
605 av_buffer_unref(&base_ctx->recon_frames_ref);
606 }
607
608 return 0;
609}
610
611static av_cold int init_rc(AVCodecContext *avctx, FFVulkanEncodeContext *ctx)
612{
613 if (ctx->opts.qp) {
614 ctx->explicit_qp = ctx->opts.qp;
615 } else if (avctx->global_quality > 0) {
616 if (avctx->flags & AV_CODEC_FLAG_QSCALE)
617 ctx->explicit_qp = avctx->global_quality / FF_QP2LAMBDA;
618 else
619 ctx->explicit_qp = avctx->global_quality;
620 }
621
622 if (ctx->opts.rc_mode == FF_VK_RC_MODE_AUTO) {
623 if (ctx->explicit_qp >= 0) {
624 ctx->opts.rc_mode = VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR;
625 } else if (avctx->global_quality > 0) {
626 if (avctx->flags & AV_CODEC_FLAG_QSCALE)
627 ctx->explicit_qp = avctx->global_quality / FF_QP2LAMBDA;
628 else
629 ctx->explicit_qp = avctx->global_quality;
630 ctx->opts.rc_mode = VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR;
631 } else if (avctx->bit_rate) {
632 if (ctx->enc_caps.rateControlModes & VK_VIDEO_ENCODE_RATE_CONTROL_MODE_VBR_BIT_KHR)
633 ctx->opts.rc_mode = VK_VIDEO_ENCODE_RATE_CONTROL_MODE_VBR_BIT_KHR;
634 else if (ctx->enc_caps.rateControlModes & VK_VIDEO_ENCODE_RATE_CONTROL_MODE_CBR_BIT_KHR)
635 ctx->opts.rc_mode = VK_VIDEO_ENCODE_RATE_CONTROL_MODE_CBR_BIT_KHR;
636 else
637 ctx->opts.rc_mode = VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DEFAULT_KHR;
638 } else {
639 ctx->explicit_qp = 18;
640 ctx->opts.rc_mode = VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR;
641 av_log(avctx, AV_LOG_WARNING, "No rate control settings specified, using fixed QP = %i\n",
642 ctx->explicit_qp);
643 }
644 } else if (ctx->opts.rc_mode != VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR &&
645 !avctx->bit_rate) {
646 av_log(avctx, AV_LOG_WARNING, "No bitrate specified!\n");
647 return AVERROR(EINVAL);
648 }
649
650 if (ctx->opts.rc_mode && !(ctx->enc_caps.rateControlModes & ctx->opts.rc_mode)) {
651 static const char *rc_modes[] = {
652 [VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DEFAULT_KHR] = "default",
653 [VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR] = "cqp",
654 [VK_VIDEO_ENCODE_RATE_CONTROL_MODE_CBR_BIT_KHR] = "cbr",
655 [VK_VIDEO_ENCODE_RATE_CONTROL_MODE_VBR_BIT_KHR] = "vbr",
656 };
657 av_log(avctx, AV_LOG_ERROR, "Unsupported rate control mode %s, supported are:\n",
658 rc_modes[FFMIN(FF_ARRAY_ELEMS(rc_modes), ctx->opts.rc_mode)]);
659 av_log(avctx, AV_LOG_ERROR, " %s\n", rc_modes[0]);
660 for (int i = VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR;
661 i <= VK_VIDEO_ENCODE_RATE_CONTROL_MODE_VBR_BIT_KHR; i <<= 1) {
662 if (!(ctx->enc_caps.rateControlModes & i))
663 continue;
664 av_log(avctx, AV_LOG_ERROR, " %s\n", rc_modes[i]);
665 }
666 return AVERROR(ENOTSUP);
667 }
668
669 return 0;
670}
671
672av_cold int ff_vulkan_write_global_header(AVCodecContext *avctx,
673 FFVulkanEncodeContext *ctx)
674{
675 int err;
676
677 /* Write extradata if needed */
678 if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) {
679 uint8_t data[4096];
680 size_t data_len = sizeof(data);
681
682 err = ctx->codec->write_sequence_headers(avctx, NULL, data, &data_len);
683 if (err < 0) {
684 av_log(avctx, AV_LOG_ERROR, "Failed to write sequence header "
685 "for extradata: %d.\n", err);
686 return err;
687 } else {
688 avctx->extradata_size = data_len;
689 avctx->extradata = av_mallocz(avctx->extradata_size +
690 AV_INPUT_BUFFER_PADDING_SIZE);
691 if (!avctx->extradata) {
692 err = AVERROR(ENOMEM);
693 return err;
694 }
695 memcpy(avctx->extradata, data, avctx->extradata_size);
696 }
697 }
698
699 return 0;
700}
701
702av_cold int ff_vulkan_encode_init(AVCodecContext *avctx, FFVulkanEncodeContext *ctx,
703 const FFVulkanEncodeDescriptor *vk_desc,
704 const FFVulkanCodec *codec,
705 void *codec_caps, void *quality_pnext)
706{
707 int i, err;
708 VkResult ret;
709 FFVulkanFunctions *vk = &ctx->s.vkfn;
710 FFVulkanContext *s = &ctx->s;
711 FFHWBaseEncodeContext *base_ctx = &ctx->base;
712
713 const AVPixFmtDescriptor *desc;
714
715 VkVideoFormatPropertiesKHR *ret_info;
716 uint32_t nb_out_fmts = 0;
717
718 VkPhysicalDeviceVideoEncodeQualityLevelInfoKHR quality_info;
719
720 VkQueryPoolVideoEncodeFeedbackCreateInfoKHR query_create;
721
722 VkVideoSessionCreateInfoKHR session_create = {
723 .sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_CREATE_INFO_KHR,
724 };
725 VkPhysicalDeviceVideoFormatInfoKHR fmt_info = {
726 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_FORMAT_INFO_KHR,
727 .pNext = &ctx->profile_list,
728 };
729
730 if (!avctx->hw_frames_ctx) {
731 av_log(avctx, AV_LOG_ERROR, "A hardware frames reference is "
732 "required to associate the encoding device.\n");
733 return AVERROR(EINVAL);
734 }
735
736 ctx->base.op = &vulkan_base_encode_ops;
737 ctx->codec = codec;
738
739 s->frames_ref = av_buffer_ref(avctx->hw_frames_ctx);
740 s->frames = (AVHWFramesContext *)s->frames_ref->data;
741 s->hwfc = s->frames->hwctx;
742
743 s->device = (AVHWDeviceContext *)s->frames->device_ref->data;
744 s->hwctx = s->device->hwctx;
745
746 desc = av_pix_fmt_desc_get(avctx->sw_pix_fmt);
747 if (!desc)
748 return AVERROR(EINVAL);
749
750 s->extensions = ff_vk_extensions_to_mask(s->hwctx->enabled_dev_extensions,
751 s->hwctx->nb_enabled_dev_extensions);
752
753 if (!(s->extensions & FF_VK_EXT_VIDEO_ENCODE_QUEUE)) {
754 av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
755 VK_KHR_VIDEO_ENCODE_QUEUE_EXTENSION_NAME);
756 return AVERROR(ENOSYS);
757 } else if (!(s->extensions & FF_VK_EXT_VIDEO_MAINTENANCE_1)) {
758 av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
759 VK_KHR_VIDEO_MAINTENANCE_1_EXTENSION_NAME);
760 return AVERROR(ENOSYS);
761 } else if (!(s->extensions & vk_desc->encode_extension)) {
762 av_log(avctx, AV_LOG_ERROR, "Device does not support encoding %s!\n",
763 avcodec_get_name(avctx->codec_id));
764 return AVERROR(ENOSYS);
765 }
766
767 /* Load functions */
768 err = ff_vk_load_functions(s->device, vk, s->extensions, 1, 1);
769 if (err < 0)
770 return err;
771
772 /* Create queue context */
773 err = ff_vk_video_qf_init(s, &ctx->qf_enc,
774 VK_QUEUE_VIDEO_ENCODE_BIT_KHR,
775 vk_desc->encode_op);
776 if (err < 0) {
777 av_log(avctx, AV_LOG_ERROR, "Encoding of %s is not supported by this device\n",
778 avcodec_get_name(avctx->codec_id));
779 return err;
780 }
781
782 /* Load all properties */
783 err = ff_vk_load_props(s);
784 if (err < 0)
785 return err;
786
787 /* Set tuning */
788 ctx->usage_info = (VkVideoEncodeUsageInfoKHR) {
789 .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_USAGE_INFO_KHR,
790 .videoUsageHints = ctx->opts.usage,
791 .videoContentHints = ctx->opts.content,
792 .tuningMode = ctx->opts.tune,
793 };
794
795 /* Load up the profile now, needed for caps and to create a query pool */
796 ctx->profile.sType = VK_STRUCTURE_TYPE_VIDEO_PROFILE_INFO_KHR;
797 ctx->profile.pNext = &ctx->usage_info;
798 ctx->profile.videoCodecOperation = vk_desc->encode_op;
799 ctx->profile.chromaSubsampling = ff_vk_subsampling_from_av_desc(desc);
800 ctx->profile.lumaBitDepth = ff_vk_depth_from_av_depth(desc->comp[0].depth);
801 ctx->profile.chromaBitDepth = ctx->profile.lumaBitDepth;
802
803 /* Setup a profile */
804 err = codec->init_profile(avctx, &ctx->profile, &ctx->usage_info);
805 if (err < 0)
806 return err;
807
808 ctx->profile_list.sType = VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR;
809 ctx->profile_list.profileCount = 1;
810 ctx->profile_list.pProfiles = &ctx->profile;
811
812 /* Get the capabilities of the encoder for the given profile */
813 ctx->enc_caps.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_CAPABILITIES_KHR;
814 ctx->enc_caps.pNext = codec_caps;
815 ctx->caps.sType = VK_STRUCTURE_TYPE_VIDEO_CAPABILITIES_KHR;
816 ctx->caps.pNext = &ctx->enc_caps;
817
818 ret = vk->GetPhysicalDeviceVideoCapabilitiesKHR(s->hwctx->phys_dev,
819 &ctx->profile,
820 &ctx->caps);
821 if (ret == VK_ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR) {
822 av_log(avctx, AV_LOG_ERROR, "Unable to initialize encoding: "
823 "%s profile \"%s\" not supported!\n",
824 avcodec_get_name(avctx->codec_id),
825 avcodec_profile_name(avctx->codec_id, avctx->profile));
826 return AVERROR(EINVAL);
827 } else if (ret == VK_ERROR_VIDEO_PROFILE_FORMAT_NOT_SUPPORTED_KHR) {
828 av_log(avctx, AV_LOG_ERROR, "Unable to initialize encoding: "
829 "format (%s) not supported!\n",
830 av_get_pix_fmt_name(avctx->sw_pix_fmt));
831 return AVERROR(EINVAL);
832 } else if (ret == VK_ERROR_FEATURE_NOT_PRESENT ||
833 ret == VK_ERROR_FORMAT_NOT_SUPPORTED) {
834 return AVERROR(EINVAL);
835 } else if (ret != VK_SUCCESS) {
836 return AVERROR_EXTERNAL;
837 }
838
839 err = init_rc(avctx, ctx);
840 if (err < 0)
841 return err;
842
843 /* Create command and query pool */
844 query_create = (VkQueryPoolVideoEncodeFeedbackCreateInfoKHR) {
845 .sType = VK_STRUCTURE_TYPE_QUERY_POOL_VIDEO_ENCODE_FEEDBACK_CREATE_INFO_KHR,
846 .pNext = &ctx->profile,
847 .encodeFeedbackFlags = ctx->enc_caps.supportedEncodeFeedbackFlags &
848 (~VK_VIDEO_ENCODE_FEEDBACK_BITSTREAM_HAS_OVERRIDES_BIT_KHR),
849 };
850 err = ff_vk_exec_pool_init(s, &ctx->qf_enc, &ctx->enc_pool, base_ctx->async_depth,
851 1, VK_QUERY_TYPE_VIDEO_ENCODE_FEEDBACK_KHR, 0,
852 &query_create);
853 if (err < 0)
854 return err;
855
856 if (ctx->opts.quality > ctx->enc_caps.maxQualityLevels) {
857 av_log(avctx, AV_LOG_ERROR, "Invalid quality level %i: allowed range is "
858 "0 to %i\n",
859 ctx->opts.quality, ctx->enc_caps.maxQualityLevels);
860 return AVERROR(EINVAL);
861 }
862
863 /* Get quality properties for the profile and quality level */
864 quality_info = (VkPhysicalDeviceVideoEncodeQualityLevelInfoKHR) {
865 .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_ENCODE_QUALITY_LEVEL_INFO_KHR,
866 .pVideoProfile = &ctx->profile,
867 .qualityLevel = ctx->opts.quality,
868 };
869 ctx->quality_props = (VkVideoEncodeQualityLevelPropertiesKHR) {
870 .sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_QUALITY_LEVEL_PROPERTIES_KHR,
871 .pNext = quality_pnext,
872 };
873 ret = vk->GetPhysicalDeviceVideoEncodeQualityLevelPropertiesKHR(s->hwctx->phys_dev,
874 &quality_info,
875 &ctx->quality_props);
876 if (ret != VK_SUCCESS)
877 return AVERROR_EXTERNAL;
878
879 /* Printout informative properties */
880 av_log(avctx, AV_LOG_VERBOSE, "Encoder capabilities for %s profile \"%s\":\n",
881 avcodec_get_name(avctx->codec_id),
882 avcodec_profile_name(avctx->codec_id, avctx->profile));
883 av_log(avctx, AV_LOG_VERBOSE, " Width: from %i to %i\n",
884 ctx->caps.minCodedExtent.width, ctx->caps.maxCodedExtent.width);
885 av_log(avctx, AV_LOG_VERBOSE, " Height: from %i to %i\n",
886 ctx->caps.minCodedExtent.height, ctx->caps.maxCodedExtent.height);
887 av_log(avctx, AV_LOG_VERBOSE, " Width alignment: %i\n",
888 ctx->caps.pictureAccessGranularity.width);
889 av_log(avctx, AV_LOG_VERBOSE, " Height alignment: %i\n",
890 ctx->caps.pictureAccessGranularity.height);
891 av_log(avctx, AV_LOG_VERBOSE, " Bitstream offset alignment: %"PRIu64"\n",
892 ctx->caps.minBitstreamBufferOffsetAlignment);
893 av_log(avctx, AV_LOG_VERBOSE, " Bitstream size alignment: %"PRIu64"\n",
894 ctx->caps.minBitstreamBufferSizeAlignment);
895 av_log(avctx, AV_LOG_VERBOSE, " Maximum references: %u\n",
896 ctx->caps.maxDpbSlots);
897 av_log(avctx, AV_LOG_VERBOSE, " Maximum active references: %u\n",
898 ctx->caps.maxActiveReferencePictures);
899 av_log(avctx, AV_LOG_VERBOSE, " Codec header version: %i.%i.%i (driver), %i.%i.%i (compiled)\n",
900 CODEC_VER(ctx->caps.stdHeaderVersion.specVersion),
901 CODEC_VER(vk_desc->ext_props.specVersion));
902 av_log(avctx, AV_LOG_VERBOSE, " Encoder max quality: %i\n",
903 ctx->enc_caps.maxQualityLevels);
904 av_log(avctx, AV_LOG_VERBOSE, " Encoder image width alignment: %i\n",
905 ctx->enc_caps.encodeInputPictureGranularity.width);
906 av_log(avctx, AV_LOG_VERBOSE, " Encoder image height alignment: %i\n",
907 ctx->enc_caps.encodeInputPictureGranularity.height);
908 av_log(avctx, AV_LOG_VERBOSE, " Capability flags:%s%s%s\n",
909 ctx->caps.flags ? "" :
910 " none",
911 ctx->caps.flags & VK_VIDEO_CAPABILITY_PROTECTED_CONTENT_BIT_KHR ?
912 " protected" : "",
913 ctx->caps.flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR ?
914 " separate_references" : "");
915
916 /* Setup width/height alignment */
917 base_ctx->surface_width = avctx->coded_width =
918 FFALIGN(avctx->width, ctx->caps.pictureAccessGranularity.width);
919 base_ctx->surface_height = avctx->coded_height =
920 FFALIGN(avctx->height, ctx->caps.pictureAccessGranularity.height);
921
922 /* Setup slice width/height */
923 base_ctx->slice_block_width = ctx->enc_caps.encodeInputPictureGranularity.width;
924 base_ctx->slice_block_height = ctx->enc_caps.encodeInputPictureGranularity.height;
925
926 /* Check if encoding is possible with the given parameters */
927 if (avctx->coded_width < ctx->caps.minCodedExtent.width ||
928 avctx->coded_height < ctx->caps.minCodedExtent.height ||
929 avctx->coded_width > ctx->caps.maxCodedExtent.width ||
930 avctx->coded_height > ctx->caps.maxCodedExtent.height) {
931 av_log(avctx, AV_LOG_ERROR, "Input of %ix%i too large for encoder limits: %ix%i max\n",
932 avctx->coded_width, avctx->coded_height,
933 ctx->caps.minCodedExtent.width, ctx->caps.minCodedExtent.height);
934 return AVERROR(EINVAL);
935 }
936
937 fmt_info.imageUsage = VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR |
938 VK_IMAGE_USAGE_VIDEO_ENCODE_DST_BIT_KHR;
939
940 ctx->common.layered_dpb = !(ctx->caps.flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR);
941
942 /* Get the supported image formats */
943 ret = vk->GetPhysicalDeviceVideoFormatPropertiesKHR(s->hwctx->phys_dev,
944 &fmt_info,
945 &nb_out_fmts, NULL);
946 if (ret == VK_ERROR_FORMAT_NOT_SUPPORTED ||
947 (!nb_out_fmts && ret == VK_SUCCESS)) {
948 return AVERROR(EINVAL);
949 } else if (ret != VK_SUCCESS) {
950 av_log(avctx, AV_LOG_ERROR, "Unable to get Vulkan format properties: %s!\n",
951 ff_vk_ret2str(ret));
952 return AVERROR_EXTERNAL;
953 }
954
955 ret_info = av_mallocz(sizeof(*ret_info)*nb_out_fmts);
956 if (!ret_info)
957 return AVERROR(ENOMEM);
958
959 for (int i = 0; i < nb_out_fmts; i++)
960 ret_info[i].sType = VK_STRUCTURE_TYPE_VIDEO_FORMAT_PROPERTIES_KHR;
961
962 ret = vk->GetPhysicalDeviceVideoFormatPropertiesKHR(s->hwctx->phys_dev,
963 &fmt_info,
964 &nb_out_fmts, ret_info);
965 if (ret == VK_ERROR_FORMAT_NOT_SUPPORTED ||
966 (!nb_out_fmts && ret == VK_SUCCESS)) {
967 av_free(ret_info);
968 return AVERROR(EINVAL);
969 } else if (ret != VK_SUCCESS) {
970 av_log(avctx, AV_LOG_ERROR, "Unable to get Vulkan format properties: %s!\n",
971 ff_vk_ret2str(ret));
972 av_free(ret_info);
973 return AVERROR_EXTERNAL;
974 }
975
976 av_log(avctx, AV_LOG_VERBOSE, "Supported input formats:\n");
977 for (i = 0; i < nb_out_fmts; i++)
978 av_log(avctx, AV_LOG_VERBOSE, " %i: %i\n", i, ret_info[i].format);
979
980 for (i = 0; i < nb_out_fmts; i++) {
981 if (ff_vk_pix_fmt_from_vkfmt(ret_info[i].format) == s->frames->sw_format) {
982 ctx->pic_format = ret_info[i].format;
983 break;
984 }
985 }
986
987 av_free(ret_info);
988
989 if (i == nb_out_fmts) {
990 av_log(avctx, AV_LOG_ERROR, "Pixel format %s of input frames not supported!\n",
991 av_get_pix_fmt_name(s->frames->sw_format));
992 return AVERROR(EINVAL);
993 }
994
995 /* Create session */
996 session_create.pVideoProfile = &ctx->profile;
997 session_create.flags = 0x0;
998 session_create.queueFamilyIndex = ctx->qf_enc.queue_family;
999 session_create.maxCodedExtent = ctx->caps.maxCodedExtent;
1000 session_create.maxDpbSlots = ctx->caps.maxDpbSlots;
1001 session_create.maxActiveReferencePictures = ctx->caps.maxActiveReferencePictures;
1002 session_create.pictureFormat = ctx->pic_format;
1003 session_create.referencePictureFormat = session_create.pictureFormat;
1004 session_create.pStdHeaderVersion = &vk_desc->ext_props;
1005
1006 err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create);
1007 if (err < 0)
1008 return err;
1009
1010 err = ff_hw_base_encode_init(avctx, &ctx->base);
1011 if (err < 0)
1012 return err;
1013
1014 err = vulkan_encode_create_dpb(avctx, ctx);
1015 if (err < 0)
1016 return err;
1017
1018 base_ctx->async_encode = 1;
1019 base_ctx->encode_fifo = av_fifo_alloc2(base_ctx->async_depth,
1020 sizeof(FFVulkanEncodePicture *), 0);
1021 if (!base_ctx->encode_fifo)
1022 return AVERROR(ENOMEM);
1023
1024 return 0;
1025}
libavcodec/vulkan_encode.h+253−0
@@ -0,0 +1,253 @@
1/*
2 * This file is part of FFmpeg.
3 *
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19#ifndef AVCODEC_VULKAN_ENCODE_H
20#define AVCODEC_VULKAN_ENCODE_H
21
22#include "codec_id.h"
23#include "internal.h"
24
25#include "encode.h"
26#include "hwconfig.h"
27
28#include "vulkan_video.h"
29#include "hw_base_encode.h"
30
31typedef struct FFVulkanEncodeDescriptor {
32 enum AVCodecID codec_id;
33 FFVulkanExtensions encode_extension;
34 VkVideoCodecOperationFlagBitsKHR encode_op;
35
36 VkExtensionProperties ext_props;
37} FFVulkanEncodeDescriptor;
38
39typedef struct FFVulkanEncodePicture {
40 FFHWBaseEncodePicture base;
41 VkVideoPictureResourceInfoKHR dpb_res;
42 VkVideoReferenceSlotInfoKHR dpb_slot;
43
44 struct {
45 VkImageView view;
46 VkImageAspectFlags aspect;
47 } in;
48
49 struct {
50 VkImageView view;
51 VkImageAspectFlags aspect;
52 } dpb;
53
54 void *codec_layer;
55 void *codec_rc_layer;
56
57 FFVkExecContext *exec;
58 AVBufferRef *pkt_buf;
59 int slices_offset;
60} FFVulkanEncodePicture;
61
62/**
63 * Callback for writing stream-level headers.
64 */
65typedef int (*vkenc_cb_write_stream_headers)(AVCodecContext *avctx,
66 uint8_t *data, size_t *data_len);
67
68/**
69 * Callback for initializing codec-specific picture headers.
70 */
71typedef int (*vkenc_cb_init_pic_headers)(AVCodecContext *avctx,
72 FFVulkanEncodePicture *pic);
73
74/**
75 * Callback for writing alignment data.
76 * Align is the value to align offset to.
77 */
78typedef int (*vkenc_cb_write_filler)(AVCodecContext *avctx, uint32_t filler,
79 uint8_t *data, size_t *data_len);
80
81/**
82 * Callback for writing any extra units requested. data_len must be set
83 * to the available size, and its value will be overwritten by the #bytes written
84 * to the output buffer.
85 */
86typedef int (*vkenc_cb_write_extra_headers)(AVCodecContext *avctx,
87 FFVulkanEncodePicture *pic,
88 uint8_t *data, size_t *data_len);
89
90typedef struct FFVulkanCodec {
91 /**
92 * Codec feature flags.
93 */
94 int flags;
95/* Codec output packet without timestamp delay, which means the
96 * output packet has same PTS and DTS. For AV1. */
97#define VK_ENC_FLAG_NO_DELAY 1 << 6
98
99 /**
100 * Size of the codec-specific picture struct.
101 */
102 size_t picture_priv_data_size;
103
104 /**
105 * Size of the filler header.
106 */
107 size_t filler_header_size;
108
109 /**
110 * Initialize codec-specific structs in a Vulkan profile.
111 */
112 int (*init_profile)(AVCodecContext *avctx, VkVideoProfileInfoKHR *profile,
113 void *pnext);
114
115 /**
116 * Initialize codec-specific rate control structures for a picture.
117 */
118 int (*init_pic_rc)(AVCodecContext *avctx, FFHWBaseEncodePicture *pic,
119 VkVideoEncodeRateControlInfoKHR *rc_info,
120 VkVideoEncodeRateControlLayerInfoKHR *rc_layer);
121
122 /**
123 * Initialize codec-specific picture parameters.
124 */
125 int (*init_pic_params)(AVCodecContext *avctx, FFHWBaseEncodePicture *pic,
126 VkVideoEncodeInfoKHR *encode_info);
127
128 /**
129 * Callback for writing stream headers.
130 */
131 int (*write_sequence_headers)(AVCodecContext *avctx,
132 FFHWBaseEncodePicture *base_pic,
133 uint8_t *data, size_t *data_len);
134
135 /**
136 * Callback for writing alignment data.
137 */
138 int (*write_filler)(AVCodecContext *avctx, uint32_t filler,
139 uint8_t *data, size_t *data_len);
140
141 /**
142 * Callback for writing any extra units requested. data_len must be set
143 * to the available size, and its value will be overwritten by the #bytes written
144 * to the output buffer.
145 */
146 int (*write_extra_headers)(AVCodecContext *avctx, FFHWBaseEncodePicture *pic,
147 uint8_t *data, size_t *data_len);
148} FFVulkanCodec;
149
150typedef struct FFVkEncodeCommonOptions {
151 int qp;
152 int quality;
153 int profile;
154 int level;
155 int tier;
156 int async_depth;
157 VkVideoEncodeUsageFlagBitsKHR usage;
158 VkVideoEncodeContentFlagBitsKHR content;
159 VkVideoEncodeTuningModeKHR tune;
160
161 VkVideoEncodeRateControlModeFlagBitsKHR rc_mode;
162#define FF_VK_RC_MODE_AUTO 0xFFFFFFFF
163} FFVkEncodeCommonOptions;
164
165typedef struct FFVulkanEncodeContext {
166 FFVulkanContext s;
167 FFVkVideoCommon common;
168 FFHWBaseEncodeContext base;
169 const FFVulkanCodec *codec;
170
171 int explicit_qp;
172 int session_reset;
173
174 /* Session parameters object, initialized by each codec independently
175 * and set here. */
176 VkVideoSessionParametersKHR session_params;
177
178 AVBufferPool *buf_pool;
179
180 VkFormat pic_format;
181
182 FFVkEncodeCommonOptions opts;
183
184 VkVideoProfileInfoKHR profile;
185 VkVideoProfileListInfoKHR profile_list;
186 VkVideoCapabilitiesKHR caps;
187 VkVideoEncodeQualityLevelPropertiesKHR quality_props;
188 VkVideoEncodeCapabilitiesKHR enc_caps;
189 VkVideoEncodeUsageInfoKHR usage_info;
190
191 FFVkQueueFamilyCtx qf_enc;
192 FFVkExecPool enc_pool;
193
194 FFHWBaseEncodePicture *slots[32];
195} FFVulkanEncodeContext;
196
197#define VULKAN_ENCODE_COMMON_OPTIONS \
198 { "qp", "Use an explicit constant quantizer for the whole stream", OFFSET(common.opts.qp), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, 255, FLAGS }, \
199 { "quality", "Set encode quality (trades off against speed, higher is faster)", OFFSET(common.opts.quality), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS }, \
200 { "rc_mode", "Select rate control type", OFFSET(common.opts.rc_mode), AV_OPT_TYPE_INT, { .i64 = FF_VK_RC_MODE_AUTO }, 0, FF_VK_RC_MODE_AUTO, FLAGS, "rc_mode" }, \
201 { "auto", "Choose mode automatically based on parameters", 0, AV_OPT_TYPE_CONST, { .i64 = FF_VK_RC_MODE_AUTO }, INT_MIN, INT_MAX, FLAGS, "rc_mode" }, \
202 { "driver", "Driver-specific rate control", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DEFAULT_KHR }, INT_MIN, INT_MAX, FLAGS, "rc_mode" }, \
203 { "cqp", "Constant quantizer mode", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR }, INT_MIN, INT_MAX, FLAGS, "rc_mode" }, \
204 { "cbr", "Constant bitrate mode", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_RATE_CONTROL_MODE_CBR_BIT_KHR }, INT_MIN, INT_MAX, FLAGS, "rc_mode" }, \
205 { "vbr", "Variable bitrate mode", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_RATE_CONTROL_MODE_VBR_BIT_KHR }, INT_MIN, INT_MAX, FLAGS, "rc_mode" }, \
206 { "tune", "Select tuning type", OFFSET(common.opts.tune), AV_OPT_TYPE_INT, { .i64 = VK_VIDEO_ENCODE_TUNING_MODE_DEFAULT_KHR }, 0, INT_MAX, FLAGS, "tune" }, \
207 { "default", "Default tuning", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_TUNING_MODE_DEFAULT_KHR }, INT_MIN, INT_MAX, FLAGS, "tune" }, \
208 { "hq", "High quality tuning", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_TUNING_MODE_HIGH_QUALITY_KHR }, INT_MIN, INT_MAX, FLAGS, "tune" }, \
209 { "ll", "Low-latency tuning", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_TUNING_MODE_LOW_LATENCY_KHR }, INT_MIN, INT_MAX, FLAGS, "tune" }, \
210 { "ull", "Ultra low-latency tuning", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_TUNING_MODE_ULTRA_LOW_LATENCY_KHR }, INT_MIN, INT_MAX, FLAGS, "tune" }, \
211 { "lossless", "Lossless mode tuning", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_TUNING_MODE_LOSSLESS_KHR }, INT_MIN, INT_MAX, FLAGS, "tune" }, \
212 { "usage", "Select usage type", OFFSET(common.opts.usage), AV_OPT_TYPE_FLAGS, { .i64 = VK_VIDEO_ENCODE_USAGE_DEFAULT_KHR }, 0, INT_MAX, FLAGS, "usage" }, \
213 { "default", "Default optimizations", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_USAGE_DEFAULT_KHR }, INT_MIN, INT_MAX, FLAGS, "usage" }, \
214 { "transcode", "Optimize for transcoding", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_USAGE_TRANSCODING_BIT_KHR }, INT_MIN, INT_MAX, FLAGS, "usage" }, \
215 { "stream", "Optimize for streaming", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_USAGE_STREAMING_BIT_KHR }, INT_MIN, INT_MAX, FLAGS, "usage" }, \
216 { "record", "Optimize for offline recording", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_USAGE_RECORDING_BIT_KHR }, INT_MIN, INT_MAX, FLAGS, "usage" }, \
217 { "conference", "Optimize for teleconferencing", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_USAGE_CONFERENCING_BIT_KHR }, INT_MIN, INT_MAX, FLAGS, "usage" }, \
218 { "content", "Select content type", OFFSET(common.opts.content), AV_OPT_TYPE_FLAGS, { .i64 = VK_VIDEO_ENCODE_CONTENT_DEFAULT_KHR }, 0, INT_MAX, FLAGS, "content" }, \
219 { "default", "Default content", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_CONTENT_DEFAULT_KHR }, INT_MIN, INT_MAX, FLAGS, "content" }, \
220 { "camera", "Camera footage", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_CONTENT_CAMERA_BIT_KHR }, INT_MIN, INT_MAX, FLAGS, "content" }, \
221 { "desktop", "Screen recording", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_CONTENT_DESKTOP_BIT_KHR }, INT_MIN, INT_MAX, FLAGS, "content" }, \
222 { "rendered", "Game or 3D content", 0, AV_OPT_TYPE_CONST, { .i64 = VK_VIDEO_ENCODE_CONTENT_RENDERED_BIT_KHR }, INT_MIN, INT_MAX, FLAGS, "content" }
223
224/**
225 * Initialize encoder.
226 */
227av_cold int ff_vulkan_encode_init(AVCodecContext *avctx, FFVulkanEncodeContext *ctx,
228 const FFVulkanEncodeDescriptor *vk_desc,
229 const FFVulkanCodec *codec,
230 void *codec_caps, void *quality_pnext);
231
232/**
233 * Write out the extradata in case its needed.
234 */
235av_cold int ff_vulkan_write_global_header(AVCodecContext *avctx,
236 FFVulkanEncodeContext *ctx);
237
238/**
239 * Encode.
240 */
241int ff_vulkan_encode_receive_packet(AVCodecContext *avctx, AVPacket *pkt);
242
243/**
244 * Uninitialize encoder.
245 */
246void ff_vulkan_encode_uninit(FFVulkanEncodeContext *ctx);
247
248/**
249 * Paperwork.
250 */
251extern const AVCodecHWConfigInternal *const ff_vulkan_encode_hw_configs[];
252
253#endif /* AVCODEC_VULKAN_ENCODE_H */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t