This allows the encoder to fully saturate all queues the GPU has, giving a good 10% in certain cases and resolutions. This also improves error resilience if an allocation fails, and properly cleans up after itself if it does.
Unified · Split
| @@ -19,6 +19,7 @@ · 18 unmodified lines | |||
| 19 | */ | 19 | */ |
| 20 | 20 | ||
| 21 | #include "libavutil/crc.h" | 21 | #include "libavutil/crc.h" |
| 22 | #include "libavutil/mem.h" | ||
| 22 | #include "libavutil/vulkan.h" | 23 | #include "libavutil/vulkan.h" |
| 23 | #include "libavutil/vulkan_spirv.h" | 24 | #include "libavutil/vulkan_spirv.h" |
| 24 | 25 | ||
| @@ -36,13 +37,38 @@ · 11 unmodified lines | |||
| 36 | #define LG_ALIGN_W 32 | 37 | #define LG_ALIGN_W 32 |
| 37 | #define LG_ALIGN_H 32 | 38 | #define LG_ALIGN_H 32 |
| 38 | 39 | ||
| 40 | typedef struct VulkanEncodeFFv1FrameData { | ||
| 41 | /* Output data */ | ||
| 42 | AVBufferRef *out_data_ref; | ||
| 43 | |||
| 44 | /* Results data */ | ||
| 45 | AVBufferRef *results_data_ref; | ||
| 46 | |||
| 47 | /* Copied from the source */ | ||
| 48 | int64_t pts; | ||
| 49 | int64_t duration; | ||
| 50 | void *frame_opaque; | ||
| 51 | AVBufferRef *frame_opaque_ref; | ||
| 52 | |||
| 53 | int key_frame; | ||
| 54 | } VulkanEncodeFFv1FrameData; | ||
| 55 | |||
| 39 | typedef struct VulkanEncodeFFv1Context { | 56 | typedef struct VulkanEncodeFFv1Context { |
| 40 | FFV1Context ctx; | 57 | FFV1Context ctx; |
| 58 | AVFrame *frame; | ||
| 41 | 59 | ||
| 42 | FFVulkanContext s; | 60 | FFVulkanContext s; |
| 43 | FFVkQueueFamilyCtx qf; | 61 | FFVkQueueFamilyCtx qf; |
| 44 | FFVkExecPool exec_pool; | 62 | FFVkExecPool exec_pool; |
| 45 | 63 | ||
| 64 | FFVkQueueFamilyCtx transfer_qf; | ||
| 65 | FFVkExecPool transfer_exec_pool; | ||
| 66 | |||
| 67 | VkBufferCopy *buf_regions; | ||
| 68 | VulkanEncodeFFv1FrameData *exec_ctx_info; | ||
| 69 | int in_flight; | ||
| 70 | int async_depth; | ||
| 71 | |||
| 46 | FFVulkanShader setup; | 72 | FFVulkanShader setup; |
| 47 | FFVulkanShader reset; | 73 | FFVulkanShader reset; |
| 48 | FFVulkanShader rct; | 74 | FFVulkanShader rct; |
| @@ -59,6 +85,7 @@ typedef struct VulkanEncodeFFv1Context { · 10 unmodified lines | |||
| 59 | 85 | ||
| 60 | /* Output data buffer */ | 86 | /* Output data buffer */ |
| 61 | AVBufferPool *out_data_pool; | 87 | AVBufferPool *out_data_pool; |
| 88 | AVBufferPool *pkt_data_pool; | ||
| 62 | 89 | ||
| 63 | /* Temporary data buffer */ | 90 | /* Temporary data buffer */ |
| 64 | AVBufferPool *tmp_data_pool; | 91 | AVBufferPool *tmp_data_pool; |
| @@ -271,15 +298,16 @@ fail: · 206 unmodified lines | |||
| 271 | return err; | 298 | return err; |
| 272 | } | 299 | } |
| 273 | 300 | ||
| 274 | static int vulkan_encode_ffv1_frame(AVCodecContext *avctx, AVPacket *pkt, | 301 | static int vulkan_encode_ffv1_submit_frame(AVCodecContext *avctx, |
| 275 | const AVFrame *pict, int *got_packet) | 302 | FFVkExecContext *exec, |
| 303 | const AVFrame *pict) | ||
| 276 | { | 304 | { |
| 277 | int err; | 305 | int err; |
| 278 | VulkanEncodeFFv1Context *fv = avctx->priv_data; | 306 | VulkanEncodeFFv1Context *fv = avctx->priv_data; |
| 279 | FFV1Context *f = &fv->ctx; | 307 | FFV1Context *f = &fv->ctx; |
| 280 | FFVulkanFunctions *vk = &fv->s.vkfn; | 308 | FFVulkanFunctions *vk = &fv->s.vkfn; |
| 281 | FFVkExecContext *exec; | ||
| 282 | 309 | ||
| 310 | VulkanEncodeFFv1FrameData *fd = exec->opaque; | ||
| 283 | FFv1VkParameters pd; | 311 | FFv1VkParameters pd; |
| 284 | 312 | ||
| 285 | AVFrame *intermediate_frame = NULL; | 313 | AVFrame *intermediate_frame = NULL; |
| @@ -298,14 +326,10 @@ static int vulkan_encode_ffv1_frame(AVCodecContext *avctx, AVPacket *pkt, · 12 unmodified lines | |||
| 298 | 326 | ||
| 299 | /* Output data */ | 327 | /* Output data */ |
| 300 | size_t maxsize; | 328 | size_t maxsize; |
| 301 | AVBufferRef *out_data_ref; | ||
| 302 | FFVkBuffer *out_data_buf; | 329 | FFVkBuffer *out_data_buf; |
| 303 | uint8_t *buf_p; | ||
| 304 | 330 | ||
| 305 | /* Results data */ | 331 | /* Results data */ |
| 306 | AVBufferRef *results_data_ref; | ||
| 307 | FFVkBuffer *results_data_buf; | 332 | FFVkBuffer *results_data_buf; |
| 308 | uint64_t *sc; | ||
| 309 | 333 | ||
| 310 | int has_inter = avctx->gop_size > 1; | 334 | int has_inter = avctx->gop_size > 1; |
| 311 | uint32_t context_count = f->context_count[f->context_model]; | 335 | uint32_t context_count = f->context_count[f->context_model]; |
| @@ -316,44 +340,36 @@ static int vulkan_encode_ffv1_frame(AVCodecContext *avctx, AVPacket *pkt, · 4 unmodified lines | |||
| 316 | AVFrame *enc_in = (AVFrame *)pict; | 340 | AVFrame *enc_in = (AVFrame *)pict; |
| 317 | VkImageView *enc_in_views = in_views; | 341 | VkImageView *enc_in_views = in_views; |
| 318 | 342 | ||
| 319 | VkMappedMemoryRange invalidate_data[2]; | ||
| 320 | int nb_invalidate_data = 0; | ||
| 321 | |||
| 322 | VkImageMemoryBarrier2 img_bar[37]; | 343 | VkImageMemoryBarrier2 img_bar[37]; |
| 323 | int nb_img_bar = 0; | 344 | int nb_img_bar = 0; |
| 324 | VkBufferMemoryBarrier2 buf_bar[8]; | 345 | VkBufferMemoryBarrier2 buf_bar[8]; |
| 325 | int nb_buf_bar = 0; | 346 | int nb_buf_bar = 0; |
| 326 | 347 | ||
| 327 | if (!pict) | 348 | /* Start recording */ |
| 328 | return 0; | ||
| 329 | |||
| 330 | exec = ff_vk_exec_get(&fv->s, &fv->exec_pool); | ||
| 331 | ff_vk_exec_start(&fv->s, exec); | 349 | ff_vk_exec_start(&fv->s, exec); |
| 332 | 350 | ||
| 333 | /* Frame state */ | 351 | /* Frame state */ |
| 334 | f->cur_enc_frame = pict; | 352 | f->cur_enc_frame = pict; |
| 335 | if (avctx->gop_size == 0 || f->picture_number % avctx->gop_size == 0) { | 353 | if (avctx->gop_size == 0 || f->picture_number % avctx->gop_size == 0) { |
| 336 | av_buffer_unref(&fv->keyframe_slice_data_ref); | 354 | av_buffer_unref(&fv->keyframe_slice_data_ref); |
| 337 | f->key_frame = 1; | 355 | f->key_frame = fd->key_frame = 1; |
| 338 | f->gob_count++; | 356 | f->gob_count++; |
| 339 | } else { | 357 | } else { |
| 340 | f->key_frame = 0; | 358 | f->key_frame = fd->key_frame = 0; |
| 341 | } | 359 | } |
| 342 | 360 | ||
| 343 | f->max_slice_count = f->num_h_slices * f->num_v_slices; | ||
| 344 | f->slice_count = f->max_slice_count; | 361 | f->slice_count = f->max_slice_count; |
| 345 | 362 | ||
| 346 | /* Allocate temporary data buffer */ | 363 | /* Allocate temporary data buffer */ |
| 347 | tmp_data_size = f->slice_count*CONTEXT_SIZE; | 364 | tmp_data_size = f->slice_count*CONTEXT_SIZE; |
| 348 | err = ff_vk_get_pooled_buffer(&fv->s, &fv->tmp_data_pool, | 365 | RET(ff_vk_get_pooled_buffer(&fv->s, &fv->tmp_data_pool, |
| 349 | &tmp_data_ref, | 366 | &tmp_data_ref, |
| 350 | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | | 367 | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | |
| 351 | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, | 368 | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, |
| 352 | NULL, tmp_data_size, | 369 | NULL, tmp_data_size, |
| 353 | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); | 370 | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)); |
| 354 | if (err < 0) | ||
| 355 | return err; | ||
| 356 | tmp_data_buf = (FFVkBuffer *)tmp_data_ref->data; | 371 | tmp_data_buf = (FFVkBuffer *)tmp_data_ref->data; |
| 372 | ff_vk_exec_add_dep_buf(&fv->s, exec, &tmp_data_ref, 1, 0); | ||
| 357 | 373 | ||
| 358 | /* Allocate slice buffer data */ | 374 | /* Allocate slice buffer data */ |
| 359 | if (f->ac == AC_GOLOMB_RICE) | 375 | if (f->ac == AC_GOLOMB_RICE) |
| @@ -368,35 +384,33 @@ static int vulkan_encode_ffv1_frame(AVCodecContext *avctx, AVPacket *pkt, · 8 unmodified lines | |||
| 368 | slice_state_size += slice_data_size; | 384 | slice_state_size += slice_data_size; |
| 369 | slice_state_size = FFALIGN(slice_state_size, 8); | 385 | slice_state_size = FFALIGN(slice_state_size, 8); |
| 370 | 386 | ||
| 387 | /* Allocate slice data buffer */ | ||
| 371 | slice_data_ref = fv->keyframe_slice_data_ref; | 388 | slice_data_ref = fv->keyframe_slice_data_ref; |
| 372 | if (!slice_data_ref) { | 389 | if (!slice_data_ref) { |
| 373 | /* Allocate slice data buffer */ | 390 | RET(ff_vk_get_pooled_buffer(&fv->s, &fv->slice_data_pool, |
| 374 | err = ff_vk_get_pooled_buffer(&fv->s, &fv->slice_data_pool, | 391 | &slice_data_ref, |
| 375 | &slice_data_ref, | 392 | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | |
| 376 | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | | 393 | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, |
| 377 | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, | 394 | NULL, slice_state_size*f->slice_count, |
| 378 | NULL, slice_state_size*f->slice_count, | 395 | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)); |
| 379 | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); | ||
| 380 | if (err < 0) | ||
| 381 | return err; | ||
| 382 | 396 | ||
| 383 | /* Only save it if we're going to use it again */ | 397 | /* Only save it if we're going to use it again */ |
| 384 | if (has_inter) | 398 | if (has_inter) |
| 385 | fv->keyframe_slice_data_ref = slice_data_ref; | 399 | fv->keyframe_slice_data_ref = slice_data_ref; |
| 386 | } | 400 | } |
| 387 | slice_data_buf = (FFVkBuffer *)slice_data_ref->data; | 401 | slice_data_buf = (FFVkBuffer *)slice_data_ref->data; |
| 402 | ff_vk_exec_add_dep_buf(&fv->s, exec, &slice_data_ref, 1, has_inter); | ||
| 388 | 403 | ||
| 389 | /* Allocate results buffer */ | 404 | /* Allocate results buffer */ |
| 390 | err = ff_vk_get_pooled_buffer(&fv->s, &fv->results_data_pool, | 405 | RET(ff_vk_get_pooled_buffer(&fv->s, &fv->results_data_pool, |
| 391 | &results_data_ref, | 406 | &fd->results_data_ref, |
| 392 | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | | 407 | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | |
| 393 | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, | 408 | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, |
| 394 | NULL, 2*f->slice_count*sizeof(uint64_t), | 409 | NULL, 2*f->slice_count*sizeof(uint64_t), |
| 395 | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | | 410 | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | |
| 396 | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); | 411 | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT)); |
| 397 | if (err < 0) | 412 | results_data_buf = (FFVkBuffer *)fd->results_data_ref->data; |
| 398 | return err; | 413 | ff_vk_exec_add_dep_buf(&fv->s, exec, &fd->results_data_ref, 1, 1); |
| 399 | results_data_buf = (FFVkBuffer *)results_data_ref->data; | ||
| 400 | 414 | ||
| 401 | /* Output buffer size */ | 415 | /* Output buffer size */ |
| 402 | maxsize = avctx->width*avctx->height*(1 + f->transparency); | 416 | maxsize = avctx->width*avctx->height*(1 + f->transparency); |
| @@ -414,26 +428,17 @@ static int vulkan_encode_ffv1_frame(AVCodecContext *avctx, AVPacket *pkt, · 11 unmodified lines | |||
| 414 | maxsize += FF_INPUT_BUFFER_MIN_SIZE; | 428 | maxsize += FF_INPUT_BUFFER_MIN_SIZE; |
| 415 | 429 | ||
| 416 | /* Allocate output buffer */ | 430 | /* Allocate output buffer */ |
| 417 | err = ff_vk_get_pooled_buffer(&fv->s, &fv->out_data_pool, | 431 | RET(ff_vk_get_pooled_buffer(&fv->s, &fv->out_data_pool, |
| 418 | &out_data_ref, | 432 | &fd->out_data_ref, |
| 419 | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | | 433 | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | |
| 420 | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, | 434 | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | |
| 421 | NULL, maxsize, | 435 | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, |
| 422 | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | | 436 | NULL, maxsize, |
| 423 | VK_MEMORY_PROPERTY_HOST_CACHED_BIT); | 437 | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)); |
| 424 | if (err < 0) | 438 | out_data_buf = (FFVkBuffer *)fd->out_data_ref->data; |
| 425 | return err; | 439 | ff_vk_exec_add_dep_buf(&fv->s, exec, &fd->out_data_ref, 1, 1); |
| 426 | 440 | ||
| 427 | out_data_buf = (FFVkBuffer *)out_data_ref->data; | 441 | /* Prepare input frame */ |
| 428 | pkt->data = out_data_buf->mapped_mem; | ||
| 429 | pkt->size = out_data_buf->size; | ||
| 430 | pkt->buf = out_data_ref; | ||
| 431 | |||
| 432 | /* Add dependencies */ | ||
| 433 | ff_vk_exec_add_dep_buf(&fv->s, exec, &tmp_data_ref, 1, 0); | ||
| 434 | ff_vk_exec_add_dep_buf(&fv->s, exec, &results_data_ref, 1, 0); | ||
| 435 | ff_vk_exec_add_dep_buf(&fv->s, exec, &slice_data_ref, 1, has_inter); | ||
| 436 | ff_vk_exec_add_dep_buf(&fv->s, exec, &out_data_ref, 1, 1); | ||
| 437 | RET(ff_vk_exec_add_dep_frame(&fv->s, exec, enc_in, | 442 | RET(ff_vk_exec_add_dep_frame(&fv->s, exec, enc_in, |
| 438 | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, | 443 | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, |
| 439 | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT)); | 444 | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT)); |
| @@ -645,6 +650,76 @@ static int vulkan_encode_ffv1_frame(AVCodecContext *avctx, AVPacket *pkt, · 205 unmodified lines | |||
| 645 | 0, sizeof(pd), &pd); | 650 | 0, sizeof(pd), &pd); |
| 646 | vk->CmdDispatch(exec->buf, fv->ctx.num_h_slices, fv->ctx.num_v_slices, 1); | 651 | vk->CmdDispatch(exec->buf, fv->ctx.num_h_slices, fv->ctx.num_v_slices, 1); |
| 647 | 652 | ||
| 653 | /* Submit */ | ||
| 654 | err = ff_vk_exec_submit(&fv->s, exec); | ||
| 655 | if (err < 0) | ||
| 656 | return err; | ||
| 657 | |||
| 658 | f->picture_number++; | ||
| 659 | |||
| 660 | /* This, if needed, was referenced by the execution context | ||
| 661 | * as it was declared as a dependency. */ | ||
| 662 | av_frame_free(&intermediate_frame); | ||
| 663 | return 0; | ||
| 664 | |||
| 665 | fail: | ||
| 666 | av_frame_free(&intermediate_frame); | ||
| 667 | ff_vk_exec_discard_deps(&fv->s, exec); | ||
| 668 | |||
| 669 | return err; | ||
| 670 | } | ||
| 671 | |||
| 672 | static int download_slices(AVCodecContext *avctx, | ||
| 673 | VkBufferCopy *buf_regions, int nb_regions, | ||
| 674 | VulkanEncodeFFv1FrameData *fd, | ||
| 675 | AVBufferRef *pkt_data_ref) | ||
| 676 | { | ||
| 677 | int err; | ||
| 678 | VulkanEncodeFFv1Context *fv = avctx->priv_data; | ||
| 679 | FFVulkanFunctions *vk = &fv->s.vkfn; | ||
| 680 | FFVkExecContext *exec; | ||
| 681 | |||
| 682 | FFVkBuffer *out_data_buf = (FFVkBuffer *)fd->out_data_ref->data; | ||
| 683 | FFVkBuffer *pkt_data_buf = (FFVkBuffer *)pkt_data_ref->data; | ||
| 684 | |||
| 685 | VkBufferMemoryBarrier2 buf_bar[8]; | ||
| 686 | int nb_buf_bar = 0; | ||
| 687 | |||
| 688 | /* Transfer the slices */ | ||
| 689 | exec = ff_vk_exec_get(&fv->s, &fv->transfer_exec_pool); | ||
| 690 | ff_vk_exec_start(&fv->s, exec); | ||
| 691 | |||
| 692 | ff_vk_exec_add_dep_buf(&fv->s, exec, &fd->out_data_ref, 1, 0); | ||
| 693 | fd->out_data_ref = NULL; /* Ownership passed */ | ||
| 694 | |||
| 695 | ff_vk_exec_add_dep_buf(&fv->s, exec, &pkt_data_ref, 1, 1); | ||
| 696 | |||
| 697 | /* Ensure the output buffer is finished */ | ||
| 698 | buf_bar[nb_buf_bar++] = (VkBufferMemoryBarrier2) { | ||
| 699 | .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER_2, | ||
| 700 | .srcStageMask = out_data_buf->stage, | ||
| 701 | .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, | ||
| 702 | .srcAccessMask = out_data_buf->access, | ||
| 703 | .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, | ||
| 704 | .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, | ||
| 705 | .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, | ||
| 706 | .buffer = out_data_buf->buf, | ||
| 707 | .size = VK_WHOLE_SIZE, | ||
| 708 | .offset = 0, | ||
| 709 | }; | ||
| 710 | vk->CmdPipelineBarrier2(exec->buf, &(VkDependencyInfo) { | ||
| 711 | .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, | ||
| 712 | .pBufferMemoryBarriers = buf_bar, | ||
| 713 | .bufferMemoryBarrierCount = nb_buf_bar, | ||
| 714 | }); | ||
| 715 | out_data_buf->stage = buf_bar[0].dstStageMask; | ||
| 716 | out_data_buf->access = buf_bar[0].dstAccessMask; | ||
| 717 | nb_buf_bar = 0; | ||
| 718 | |||
| 719 | vk->CmdCopyBuffer(exec->buf, | ||
| 720 | out_data_buf->buf, pkt_data_buf->buf, | ||
| 721 | nb_regions, buf_regions); | ||
| 722 | |||
| 648 | /* Submit */ | 723 | /* Submit */ |
| 649 | err = ff_vk_exec_submit(&fv->s, exec); | 724 | err = ff_vk_exec_submit(&fv->s, exec); |
| 650 | if (err < 0) | 725 | if (err < 0) |
| @@ -652,68 +727,152 @@ static int vulkan_encode_ffv1_frame(AVCodecContext *avctx, AVPacket *pkt, · 1 unmodified line | |||
| 652 | 727 | ||
| 653 | /* We need the encoded data immediately */ | 728 | /* We need the encoded data immediately */ |
| 654 | ff_vk_exec_wait(&fv->s, exec); | 729 | ff_vk_exec_wait(&fv->s, exec); |
| 655 | av_frame_free(&intermediate_frame); | ||
| 656 | 730 | ||
| 657 | /* Invalidate slice/output data if needed */ | 731 | /* Invalidate slice/output data if needed */ |
| 658 | if (!(results_data_buf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) | 732 | if (!(pkt_data_buf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) { |
| 659 | invalidate_data[nb_invalidate_data++] = (VkMappedMemoryRange) { | 733 | VkMappedMemoryRange invalidate_data = { |
| 660 | .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE, | 734 | .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE, |
| 661 | .memory = results_data_buf->mem, | 735 | .memory = pkt_data_buf->mem, |
| 662 | .offset = 0, | 736 | .offset = 0, |
| 663 | .size = VK_WHOLE_SIZE, | 737 | .size = VK_WHOLE_SIZE, |
| 664 | }; | 738 | }; |
| 665 | if (!(out_data_buf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) | 739 | vk->InvalidateMappedMemoryRanges(fv->s.hwctx->act_dev, |
| 666 | invalidate_data[nb_invalidate_data++] = (VkMappedMemoryRange) { | 740 | 1, &invalidate_data); |
| 741 | } | ||
| 742 | |||
| 743 | return 0; | ||
| 744 | } | ||
| 745 | |||
| 746 | static int get_packet(AVCodecContext *avctx, FFVkExecContext *exec, | ||
| 747 | AVPacket *pkt) | ||
| 748 | { | ||
| 749 | int err; | ||
| 750 | VulkanEncodeFFv1Context *fv = avctx->priv_data; | ||
| 751 | FFV1Context *f = &fv->ctx; | ||
| 752 | FFVulkanFunctions *vk = &fv->s.vkfn; | ||
| 753 | |||
| 754 | /* Packet data */ | ||
| 755 | AVBufferRef *pkt_data_ref; | ||
| 756 | FFVkBuffer *pkt_data_buf; | ||
| 757 | |||
| 758 | VulkanEncodeFFv1FrameData *fd = exec->opaque; | ||
| 759 | |||
| 760 | FFVkBuffer *results_data_buf = (FFVkBuffer *)fd->results_data_ref->data; | ||
| 761 | uint64_t *sc; | ||
| 762 | |||
| 763 | /* Make sure encoding's done */ | ||
| 764 | ff_vk_exec_wait(&fv->s, exec); | ||
| 765 | |||
| 766 | /* Invalidate slice/output data if needed */ | ||
| 767 | if (!(results_data_buf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) { | ||
| 768 | VkMappedMemoryRange invalidate_data = { | ||
| 667 | .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE, | 769 | .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE, |
| 668 | .memory = out_data_buf->mem, | 770 | .memory = results_data_buf->mem, |
| 669 | .offset = 0, | 771 | .offset = 0, |
| 670 | .size = VK_WHOLE_SIZE, | 772 | .size = VK_WHOLE_SIZE, |
| 671 | }; | 773 | }; |
| 672 | if (nb_invalidate_data) | ||
| 673 | vk->InvalidateMappedMemoryRanges(fv->s.hwctx->act_dev, | 774 | vk->InvalidateMappedMemoryRanges(fv->s.hwctx->act_dev, |
| 674 | nb_invalidate_data, invalidate_data); | 775 | 1, &invalidate_data); |
| 675 | 776 | } | |
| 676 | /* First slice is in-place */ | ||
| 677 | buf_p = pkt->data; | ||
| 678 | sc = &((uint64_t *)results_data_buf->mapped_mem)[0]; | ||
| 679 | av_log(avctx, AV_LOG_DEBUG, "Slice size = %"PRIu64" (max %i), src offset = %"PRIu64"\n", | ||
| 680 | sc[0], pkt->size / f->slice_count, sc[1]); | ||
| 681 | av_assert0(sc[0] < pd.slice_size_max); | ||
| 682 | av_assert0(sc[0] < (1 << 24)); | ||
| 683 | buf_p += sc[0]; | ||
| 684 | |||
| 685 | /* We have to copy the rest */ | ||
| 686 | for (int i = 1; i < f->slice_count; i++) { | ||
| 687 | uint64_t bytes; | ||
| 688 | uint8_t *bs_start; | ||
| 689 | 777 | ||
| 778 | /* Calculate final size */ | ||
| 779 | pkt->size = 0; | ||
| 780 | for (int i = 0; i < f->slice_count; i++) { | ||
| 690 | sc = &((uint64_t *)results_data_buf->mapped_mem)[i*2]; | 781 | sc = &((uint64_t *)results_data_buf->mapped_mem)[i*2]; |
| 691 | bytes = sc[0]; | 782 | av_log(avctx, AV_LOG_DEBUG, "Slice %i size = %"PRIu64", " |
| 692 | bs_start = pkt->data + sc[1]; | ||
| 693 | |||
| 694 | av_log(avctx, AV_LOG_DEBUG, "Slice %i size = %"PRIu64" (max %"PRIu64"), " | ||
| 695 | "src offset = %"PRIu64"\n", | 783 | "src offset = %"PRIu64"\n", |
| 696 | i, bytes, pd.slice_size_max, sc[1]); | 784 | i, sc[0], sc[1]); |
| 697 | av_assert0(bytes < pd.slice_size_max); | 785 | |
| 698 | av_assert0(bytes < (1 << 24)); | 786 | fv->buf_regions[i] = (VkBufferCopy) { |
| 787 | .srcOffset = sc[1], | ||
| 788 | .dstOffset = pkt->size, | ||
| 789 | .size = sc[0], | ||
| 790 | }; | ||
| 791 | pkt->size += sc[0]; | ||
| 792 | } | ||
| 793 | av_log(avctx, AV_LOG_VERBOSE, "Encoded data: %iMiB\n", pkt->size / (1024*1024)); | ||
| 794 | av_buffer_unref(&fd->results_data_ref); /* No need for this buffer anymore */ | ||
| 795 | |||
| 796 | /* Allocate packet buffer */ | ||
| 797 | err = ff_vk_get_pooled_buffer(&fv->s, &fv->pkt_data_pool, | ||
| 798 | &pkt_data_ref, | ||
| 799 | VK_BUFFER_USAGE_TRANSFER_DST_BIT, | ||
| 800 | NULL, pkt->size, | ||
| 801 | VK_MEMORY_PROPERTY_HOST_CACHED_BIT | | ||
| 802 | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); | ||
| 803 | if (err < 0) | ||
| 804 | return err; | ||
| 805 | pkt_data_buf = (FFVkBuffer *)pkt_data_ref->data; | ||
| 806 | |||
| 807 | /* Setup packet data */ | ||
| 808 | pkt->data = pkt_data_buf->mapped_mem; | ||
| 809 | pkt->buf = pkt_data_ref; | ||
| 699 | 810 | ||
| 700 | memmove(buf_p, bs_start, bytes); | 811 | pkt->pts = fd->pts; |
| 812 | pkt->dts = fd->pts; | ||
| 813 | pkt->duration = fd->duration; | ||
| 814 | pkt->flags |= AV_PKT_FLAG_KEY * fd->key_frame; | ||
| 701 | 815 | ||
| 702 | buf_p += bytes; | 816 | if (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) { |
| 817 | pkt->opaque = fd->frame_opaque; | ||
| 818 | pkt->opaque_ref = fd->frame_opaque_ref; | ||
| 819 | fd->frame_opaque_ref = NULL; | ||
| 703 | } | 820 | } |
| 704 | 821 | ||
| 705 | f->picture_number++; | 822 | return download_slices(avctx, fv->buf_regions, f->slice_count, fd, |
| 706 | pkt->size = buf_p - pkt->data; | 823 | pkt_data_ref); |
| 707 | pkt->flags |= AV_PKT_FLAG_KEY * f->key_frame; | 824 | } |
| 708 | *got_packet = 1; | ||
| 709 | 825 | ||
| 710 | av_log(avctx, AV_LOG_VERBOSE, "Total data = %i\n", | 826 | static int vulkan_encode_ffv1_receive_packet(AVCodecContext *avctx, |
| 711 | pkt->size); | 827 | AVPacket *pkt) |
| 828 | { | ||
| 829 | int err; | ||
| 830 | VulkanEncodeFFv1Context *fv = avctx->priv_data; | ||
| 831 | VulkanEncodeFFv1FrameData *fd; | ||
| 832 | FFVkExecContext *exec; | ||
| 833 | AVFrame *frame; | ||
| 712 | 834 | ||
| 713 | fail: | 835 | while (1) { |
| 714 | /* Frames added as a dep are always referenced, so we only need to | 836 | /* Roll an execution context */ |
| 715 | * clean this up. */ | 837 | exec = ff_vk_exec_get(&fv->s, &fv->exec_pool); |
| 716 | av_frame_free(&intermediate_frame); | 838 | |
| 839 | /* If it had a frame, immediately output it */ | ||
| 840 | if (exec->had_submission) { | ||
| 841 | exec->had_submission = 0; | ||
| 842 | fv->in_flight--; | ||
| 843 | return get_packet(avctx, exec, pkt); | ||
| 844 | } | ||
| 845 | |||
| 846 | /* Get next frame to encode */ | ||
| 847 | frame = fv->frame; | ||
| 848 | err = ff_encode_get_frame(avctx, frame); | ||
| 849 | if (err < 0 && err != AVERROR_EOF) { | ||
| 850 | return err; | ||
| 851 | } else if (err == AVERROR_EOF) { | ||
| 852 | if (!fv->in_flight) | ||
| 853 | return err; | ||
| 854 | continue; | ||
| 855 | } | ||
| 856 | |||
| 857 | /* Encode frame */ | ||
| 858 | fd = exec->opaque; | ||
| 859 | fd->pts = frame->pts; | ||
| 860 | fd->duration = frame->duration; | ||
| 861 | if (avctx->flags & AV_CODEC_FLAG_COPY_OPAQUE) { | ||
| 862 | fd->frame_opaque = frame->opaque; | ||
| 863 | fd->frame_opaque_ref = frame->opaque_ref; | ||
| 864 | frame->opaque_ref = NULL; | ||
| 865 | } | ||
| 866 | |||
| 867 | err = vulkan_encode_ffv1_submit_frame(avctx, exec, frame); | ||
| 868 | av_frame_unref(frame); | ||
| 869 | if (err < 0) | ||
| 870 | return err; | ||
| 871 | |||
| 872 | fv->in_flight++; | ||
| 873 | if (fv->in_flight < fv->async_depth) | ||
| 874 | return AVERROR(EAGAIN); | ||
| 875 | } | ||
| 717 | 876 | ||
| 718 | return 0; | 877 | return 0; |
| 719 | } | 878 | } |
| @@ -1441,8 +1600,23 @@ static av_cold int vulkan_encode_ffv1_init(AVCodecContext *avctx) · 721 unmodified lines | |||
| 1441 | return err; | 1600 | return err; |
| 1442 | } | 1601 | } |
| 1443 | 1602 | ||
| 1603 | if (!fv->async_depth) | ||
| 1604 | fv->async_depth = fv->qf.nb_queues; | ||
| 1605 | |||
| 1444 | err = ff_vk_exec_pool_init(&fv->s, &fv->qf, &fv->exec_pool, | 1606 | err = ff_vk_exec_pool_init(&fv->s, &fv->qf, &fv->exec_pool, |
| 1445 | 1, /* Single-threaded for now */ | 1607 | FFMIN(fv->qf.nb_queues, fv->async_depth), |
| 1608 | 0, 0, 0, NULL); | ||
| 1609 | if (err < 0) | ||
| 1610 | return err; | ||
| 1611 | |||
| 1612 | err = ff_vk_qf_init(&fv->s, &fv->transfer_qf, VK_QUEUE_TRANSFER_BIT); | ||
| 1613 | if (err < 0) { | ||
| 1614 | av_log(avctx, AV_LOG_ERROR, "Device has no transfer queues!\n"); | ||
| 1615 | return err; | ||
| 1616 | } | ||
| 1617 | |||
| 1618 | err = ff_vk_exec_pool_init(&fv->s, &fv->transfer_qf, &fv->transfer_exec_pool, | ||
| 1619 | 1, | ||
| 1446 | 0, 0, 0, NULL); | 1620 | 0, 0, 0, NULL); |
| 1447 | if (err < 0) | 1621 | if (err < 0) |
| 1448 | return err; | 1622 | return err; |
| @@ -1510,6 +1684,24 @@ static av_cold int vulkan_encode_ffv1_init(AVCodecContext *avctx) · 61 unmodified lines | |||
| 1510 | if (err < 0) | 1684 | if (err < 0) |
| 1511 | return err; | 1685 | return err; |
| 1512 | 1686 | ||
| 1687 | /* Temporary frame */ | ||
| 1688 | fv->frame = av_frame_alloc(); | ||
| 1689 | if (!fv->frame) | ||
| 1690 | return AVERROR(ENOMEM); | ||
| 1691 | |||
| 1692 | /* Async data pool */ | ||
| 1693 | fv->async_depth = fv->exec_pool.pool_size; | ||
| 1694 | fv->exec_ctx_info = av_calloc(fv->async_depth, sizeof(*fv->exec_ctx_info)); | ||
| 1695 | if (!fv->exec_ctx_info) | ||
| 1696 | return AVERROR(ENOMEM); | ||
| 1697 | for (int i = 0; i < fv->async_depth; i++) | ||
| 1698 | fv->exec_pool.contexts[i].opaque = &fv->exec_ctx_info[i]; | ||
| 1699 | |||
| 1700 | f->max_slice_count = f->num_h_slices * f->num_v_slices; | ||
| 1701 | fv->buf_regions = av_malloc_array(f->max_slice_count, sizeof(*fv->buf_regions)); | ||
| 1702 | if (!fv->buf_regions) | ||
| 1703 | return AVERROR(ENOMEM); | ||
| 1704 | |||
| 1513 | return 0; | 1705 | return 0; |
| 1514 | } | 1706 | } |
| 1515 | 1707 | ||
| @@ -1518,17 +1710,29 @@ static av_cold int vulkan_encode_ffv1_close(AVCodecContext *avctx) · 2 unmodified lines | |||
| 1518 | VulkanEncodeFFv1Context *fv = avctx->priv_data; | 1710 | VulkanEncodeFFv1Context *fv = avctx->priv_data; |
| 1519 | 1711 | ||
| 1520 | ff_vk_exec_pool_free(&fv->s, &fv->exec_pool); | 1712 | ff_vk_exec_pool_free(&fv->s, &fv->exec_pool); |
| 1713 | ff_vk_exec_pool_free(&fv->s, &fv->transfer_exec_pool); | ||
| 1521 | 1714 | ||
| 1522 | ff_vk_shader_free(&fv->s, &fv->enc); | 1715 | ff_vk_shader_free(&fv->s, &fv->enc); |
| 1523 | ff_vk_shader_free(&fv->s, &fv->rct); | 1716 | ff_vk_shader_free(&fv->s, &fv->rct); |
| 1524 | ff_vk_shader_free(&fv->s, &fv->reset); | 1717 | ff_vk_shader_free(&fv->s, &fv->reset); |
| 1525 | ff_vk_shader_free(&fv->s, &fv->setup); | 1718 | ff_vk_shader_free(&fv->s, &fv->setup); |
| 1526 | 1719 | ||
| 1720 | if (fv->exec_ctx_info) { | ||
| 1721 | for (int i = 0; i < fv->async_depth; i++) { | ||
| 1722 | VulkanEncodeFFv1FrameData *fd = &fv->exec_ctx_info[i]; | ||
| 1723 | av_buffer_unref(&fd->out_data_ref); | ||
| 1724 | av_buffer_unref(&fd->results_data_ref); | ||
| 1725 | av_buffer_unref(&fd->frame_opaque_ref); | ||
| 1726 | } | ||
| 1727 | } | ||
| 1728 | av_free(fv->exec_ctx_info); | ||
| 1729 | |||
| 1527 | av_buffer_unref(&fv->intermediate_frames_ref); | 1730 | av_buffer_unref(&fv->intermediate_frames_ref); |
| 1528 | 1731 | ||
| 1529 | av_buffer_pool_uninit(&fv->results_data_pool); | 1732 | av_buffer_pool_uninit(&fv->results_data_pool); |
| 1530 | 1733 | ||
| 1531 | av_buffer_pool_uninit(&fv->out_data_pool); | 1734 | av_buffer_pool_uninit(&fv->out_data_pool); |
| 1735 | av_buffer_pool_uninit(&fv->pkt_data_pool); | ||
| 1532 | av_buffer_pool_uninit(&fv->tmp_data_pool); | 1736 | av_buffer_pool_uninit(&fv->tmp_data_pool); |
| 1533 | 1737 | ||
| 1534 | av_buffer_unref(&fv->keyframe_slice_data_ref); | 1738 | av_buffer_unref(&fv->keyframe_slice_data_ref); |
| @@ -1538,6 +1742,8 @@ static av_cold int vulkan_encode_ffv1_close(AVCodecContext *avctx) · 3 unmodified lines | |||
| 1538 | ff_vk_free_buf(&fv->s, &fv->rangecoder_static_buf); | 1742 | ff_vk_free_buf(&fv->s, &fv->rangecoder_static_buf); |
| 1539 | ff_vk_free_buf(&fv->s, &fv->crc_tab_buf); | 1743 | ff_vk_free_buf(&fv->s, &fv->crc_tab_buf); |
| 1540 | 1744 | ||
| 1745 | av_free(fv->buf_regions); | ||
| 1746 | av_frame_free(&fv->frame); | ||
| 1541 | ff_vk_uninit(&fv->s); | 1747 | ff_vk_uninit(&fv->s); |
| 1542 | 1748 | ||
| 1543 | return 0; | 1749 | return 0; |
| @@ -1567,6 +1773,9 @@ static const AVOption vulkan_encode_ffv1_options[] = { · 23 unmodified lines | |||
| 1567 | { "force_pcm", "Code all slices with no prediction", OFFSET(force_pcm), AV_OPT_TYPE_BOOL, | 1773 | { "force_pcm", "Code all slices with no prediction", OFFSET(force_pcm), AV_OPT_TYPE_BOOL, |
| 1568 | { .i64 = 0 }, 0, 1, VE }, | 1774 | { .i64 = 0 }, 0, 1, VE }, |
| 1569 | 1775 | ||
| 1776 | { "async_depth", "Internal parallelization depth", OFFSET(async_depth), AV_OPT_TYPE_INT, | ||
| 1777 | { .i64 = 0 }, 0, INT_MAX, VE }, | ||
| 1778 | |||
| 1570 | { NULL } | 1779 | { NULL } |
| 1571 | }; | 1780 | }; |
| 1572 | 1781 | ||
| @@ -1594,7 +1803,7 @@ const FFCodec ff_ffv1_vulkan_encoder = { · 21 unmodified lines | |||
| 1594 | .p.id = AV_CODEC_ID_FFV1, | 1803 | .p.id = AV_CODEC_ID_FFV1, |
| 1595 | .priv_data_size = sizeof(VulkanEncodeFFv1Context), | 1804 | .priv_data_size = sizeof(VulkanEncodeFFv1Context), |
| 1596 | .init = &vulkan_encode_ffv1_init, | 1805 | .init = &vulkan_encode_ffv1_init, |
| 1597 | FF_CODEC_ENCODE_CB(vulkan_encode_ffv1_frame), | 1806 | FF_CODEC_RECEIVE_PACKET_CB(&vulkan_encode_ffv1_receive_packet), |
| 1598 | .close = &vulkan_encode_ffv1_close, | 1807 | .close = &vulkan_encode_ffv1_close, |
| 1599 | .p.priv_class = &vulkan_encode_ffv1_class, | 1808 | .p.priv_class = &vulkan_encode_ffv1_class, |
| 1600 | .p.capabilities = AV_CODEC_CAP_DELAY | | 1809 | .p.capabilities = AV_CODEC_CAP_DELAY | |
| @@ -146,6 +146,7 @@ typedef uint64_t FFVulkanExtensions; · 145 unmodified lines | |||
| 146 | MACRO(1, 1, FF_VK_EXT_NO_FLAG, CmdPipelineBarrier) \ | 146 | MACRO(1, 1, FF_VK_EXT_NO_FLAG, CmdPipelineBarrier) \ |
| 147 | MACRO(1, 1, FF_VK_EXT_NO_FLAG, CmdCopyBufferToImage) \ | 147 | MACRO(1, 1, FF_VK_EXT_NO_FLAG, CmdCopyBufferToImage) \ |
| 148 | MACRO(1, 1, FF_VK_EXT_NO_FLAG, CmdCopyImageToBuffer) \ | 148 | MACRO(1, 1, FF_VK_EXT_NO_FLAG, CmdCopyImageToBuffer) \ |
| 149 | MACRO(1, 1, FF_VK_EXT_NO_FLAG, CmdCopyBuffer) \ | ||
| 149 | \ | 150 | \ |
| 150 | /* Buffer */ \ | 151 | /* Buffer */ \ |
| 151 | MACRO(1, 1, FF_VK_EXT_NO_FLAG, GetBufferMemoryRequirements2) \ | 152 | MACRO(1, 1, FF_VK_EXT_NO_FLAG, GetBufferMemoryRequirements2) \ |