This commit was long overdue. The old transfer dubiously tried to merge as much code as possible, and had very little in the way of optimizations, apart from basic host-mapping. The new code uses buffer pools for any temporary bufflers, and handles falling back to buffer-based uploads if host-mapping fails. Roundtrip performance difference: ffmpeg -init_hw_device "vulkan=vk:0,debug=0,disable_multiplane=1" -f lavfi \ -i color=red:s=3840x2160 -vf hwupload,hwdownload,format=yuv420p -f null - 7900XTX: Before: 224fps After: 502fps Ada, with proprietary drivers: Before: 29fps After: 54fps Alder Lake: Before: 85fps After: 108fps With the host-mapping codepath disabled: Before: 32fps After: 51fps
Unified · Split
| @@ -129,6 +129,9 @@ typedef struct VulkanFramesPriv { · 128 unmodified lines | |||
| 129 | FFVkExecPool upload_exec; | 129 | FFVkExecPool upload_exec; |
| 130 | FFVkExecPool download_exec; | 130 | FFVkExecPool download_exec; |
| 131 | 131 | ||
| 132 | /* Temporary buffer pools */ | ||
| 133 | AVBufferPool *tmp; | ||
| 134 | |||
| 132 | /* Modifier info list to free at uninit */ | 135 | /* Modifier info list to free at uninit */ |
| 133 | VkImageDrmFormatModifierListCreateInfoEXT *modifier_info; | 136 | VkImageDrmFormatModifierListCreateInfoEXT *modifier_info; |
| 134 | } VulkanFramesPriv; | 137 | } VulkanFramesPriv; |
| @@ -2425,6 +2428,8 @@ static void vulkan_frames_uninit(AVHWFramesContext *hwfc) · 2290 unmodified lines | |||
| 2425 | ff_vk_exec_pool_free(&p->vkctx, &fp->compute_exec); | 2428 | ff_vk_exec_pool_free(&p->vkctx, &fp->compute_exec); |
| 2426 | ff_vk_exec_pool_free(&p->vkctx, &fp->upload_exec); | 2429 | ff_vk_exec_pool_free(&p->vkctx, &fp->upload_exec); |
| 2427 | ff_vk_exec_pool_free(&p->vkctx, &fp->download_exec); | 2430 | ff_vk_exec_pool_free(&p->vkctx, &fp->download_exec); |
| 2431 | |||
| 2432 | av_buffer_pool_uninit(&fp->tmp); | ||
| 2428 | } | 2433 | } |
| 2429 | 2434 | ||
| 2430 | static int vulkan_frames_init(AVHWFramesContext *hwfc) | 2435 | static int vulkan_frames_init(AVHWFramesContext *hwfc) |
| @@ -3451,128 +3456,288 @@ static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst, · 1020 unmodified lines | |||
| 3451 | return AVERROR(ENOSYS); | 3456 | return AVERROR(ENOSYS); |
| 3452 | } | 3457 | } |
| 3453 | 3458 | ||
| 3454 | static size_t get_req_buffer_size(VulkanDevicePriv *p, int *stride, int height) | 3459 | static int copy_buffer_data(AVHWFramesContext *hwfc, AVBufferRef *buf, |
| 3460 | AVFrame *swf, VkBufferImageCopy *region, | ||
| 3461 | int planes, int upload) | ||
| 3455 | { | 3462 | { |
| 3456 | size_t size; | 3463 | VkResult ret; |
| 3457 | *stride = FFALIGN(*stride, p->props.properties.limits.optimalBufferCopyRowPitchAlignment); | 3464 | VulkanDevicePriv *p = hwfc->device_ctx->hwctx; |
| 3458 | size = height*(*stride); | 3465 | FFVulkanFunctions *vk = &p->vkctx.vkfn; |
| 3459 | size = FFALIGN(size, p->props.properties.limits.minMemoryMapAlignment); | 3466 | AVVulkanDeviceContext *hwctx = &p->p; |
| 3460 | return size; | 3467 | |
| 3468 | FFVkBuffer *vkbuf = (FFVkBuffer *)buf->data; | ||
| 3469 | |||
| 3470 | const VkMappedMemoryRange flush_info = { | ||
| 3471 | .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE, | ||
| 3472 | .memory = vkbuf->mem, | ||
| 3473 | .size = VK_WHOLE_SIZE, | ||
| 3474 | }; | ||
| 3475 | |||
| 3476 | if (!(vkbuf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) && !upload) { | ||
| 3477 | ret = vk->InvalidateMappedMemoryRanges(hwctx->act_dev, 1, | ||
| 3478 | &flush_info); | ||
| 3479 | if (ret != VK_SUCCESS) { | ||
| 3480 | av_log(hwfc, AV_LOG_ERROR, "Failed to invalidate buffer data: %s\n", | ||
| 3481 | ff_vk_ret2str(ret)); | ||
| 3482 | return AVERROR_EXTERNAL; | ||
| 3483 | } | ||
| 3484 | } | ||
| 3485 | |||
| 3486 | for (int i = 0; i < planes; i++) | ||
| 3487 | av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset, | ||
| 3488 | region[i].bufferRowLength, | ||
| 3489 | swf->data[i], | ||
| 3490 | swf->linesize[i], | ||
| 3491 | swf->linesize[i], | ||
| 3492 | region[i].imageExtent.height); | ||
| 3493 | |||
| 3494 | if (!(vkbuf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) && upload) { | ||
| 3495 | ret = vk->FlushMappedMemoryRanges(hwctx->act_dev, 1, | ||
| 3496 | &flush_info); | ||
| 3497 | if (ret != VK_SUCCESS) { | ||
| 3498 | av_log(hwfc, AV_LOG_ERROR, "Failed to flush buffer data: %s\n", | ||
| 3499 | ff_vk_ret2str(ret)); | ||
| 3500 | return AVERROR_EXTERNAL; | ||
| 3501 | } | ||
| 3502 | } | ||
| 3503 | |||
| 3504 | return 0; | ||
| 3461 | } | 3505 | } |
| 3462 | 3506 | ||
| 3463 | static int transfer_image_buf(AVHWFramesContext *hwfc, AVFrame *f, | 3507 | static int get_plane_buf(AVHWFramesContext *hwfc, AVBufferRef **dst, |
| 3464 | AVBufferRef **bufs, size_t *buf_offsets, | 3508 | AVFrame *swf, VkBufferImageCopy *region, int upload) |
| 3465 | const int *buf_stride, int w, | ||
| 3466 | int h, enum AVPixelFormat pix_fmt, int to_buf) | ||
| 3467 | { | 3509 | { |
| 3468 | int err; | 3510 | int err; |
| 3469 | AVVkFrame *frame = (AVVkFrame *)f->data[0]; | ||
| 3470 | VulkanFramesPriv *fp = hwfc->hwctx; | 3511 | VulkanFramesPriv *fp = hwfc->hwctx; |
| 3471 | VulkanDevicePriv *p = hwfc->device_ctx->hwctx; | 3512 | VulkanDevicePriv *p = hwfc->device_ctx->hwctx; |
| 3472 | FFVulkanFunctions *vk = &p->vkctx.vkfn; | 3513 | const int planes = av_pix_fmt_count_planes(swf->format); |
| 3473 | VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS]; | 3514 | |
| 3474 | int nb_img_bar = 0; | 3515 | size_t buf_offset = 0; |
| 3516 | for (int i = 0; i < planes; i++) { | ||
| 3517 | size_t size; | ||
| 3518 | ptrdiff_t linesize = swf->linesize[i]; | ||
| 3475 | 3519 | ||
| 3476 | const int nb_images = ff_vk_count_images(frame); | 3520 | uint32_t p_w, p_h; |
| 3477 | int pixfmt_planes = av_pix_fmt_count_planes(pix_fmt); | 3521 | get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i); |
| 3478 | const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt); | ||
| 3479 | 3522 | ||
| 3480 | VkCommandBuffer cmd_buf; | 3523 | linesize = FFALIGN(linesize, |
| 3481 | FFVkExecContext *exec = ff_vk_exec_get(to_buf ? &fp->download_exec : | 3524 | p->props.properties.limits.optimalBufferCopyRowPitchAlignment); |
| 3482 | &fp->upload_exec); | 3525 | size = p_h*linesize; |
| 3483 | cmd_buf = exec->buf; | ||
| 3484 | ff_vk_exec_start(&p->vkctx, exec); | ||
| 3485 | 3526 | ||
| 3486 | err = ff_vk_exec_add_dep_buf(&p->vkctx, exec, bufs, pixfmt_planes, 1); | 3527 | region[i] = (VkBufferImageCopy) { |
| 3528 | .bufferOffset = buf_offset, | ||
| 3529 | .bufferRowLength = linesize, | ||
| 3530 | .bufferImageHeight = p_h, | ||
| 3531 | .imageSubresource.layerCount = 1, | ||
| 3532 | .imageExtent = (VkExtent3D){ p_w, p_h, 1 }, | ||
| 3533 | /* Rest of the fields adjusted/filled in later */ | ||
| 3534 | }; | ||
| 3535 | |||
| 3536 | buf_offset = FFALIGN(buf_offset + size, | ||
| 3537 | p->props.properties.limits.optimalBufferCopyOffsetAlignment); | ||
| 3538 | } | ||
| 3539 | |||
| 3540 | err = ff_vk_get_pooled_buffer(&p->vkctx, &fp->tmp, dst, | ||
| 3541 | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | | ||
| 3542 | VK_BUFFER_USAGE_TRANSFER_DST_BIT, | ||
| 3543 | NULL, buf_offset, | ||
| 3544 | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); | ||
| 3487 | if (err < 0) | 3545 | if (err < 0) |
| 3488 | return err; | 3546 | return err; |
| 3489 | 3547 | ||
| 3490 | err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, f, | 3548 | return 0; |
| 3491 | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, | 3549 | } |
| 3492 | VK_PIPELINE_STAGE_2_TRANSFER_BIT); | 3550 | |
| 3551 | static int create_mapped_buffer(AVHWFramesContext *hwfc, | ||
| 3552 | FFVkBuffer *vkb, VkBufferUsageFlags usage, | ||
| 3553 | size_t size, | ||
| 3554 | VkExternalMemoryBufferCreateInfo *create_desc, | ||
| 3555 | VkImportMemoryHostPointerInfoEXT *import_desc, | ||
| 3556 | VkMemoryHostPointerPropertiesEXT props) | ||
| 3557 | { | ||
| 3558 | int err; | ||
| 3559 | VkResult ret; | ||
| 3560 | VulkanDevicePriv *p = hwfc->device_ctx->hwctx; | ||
| 3561 | FFVulkanFunctions *vk = &p->vkctx.vkfn; | ||
| 3562 | AVVulkanDeviceContext *hwctx = &p->p; | ||
| 3563 | |||
| 3564 | VkBufferCreateInfo buf_spawn = { | ||
| 3565 | .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, | ||
| 3566 | .pNext = create_desc, | ||
| 3567 | .usage = usage, | ||
| 3568 | .sharingMode = VK_SHARING_MODE_EXCLUSIVE, | ||
| 3569 | .size = size, | ||
| 3570 | }; | ||
| 3571 | VkMemoryRequirements req = { | ||
| 3572 | .size = size, | ||
| 3573 | .alignment = p->hprops.minImportedHostPointerAlignment, | ||
| 3574 | .memoryTypeBits = props.memoryTypeBits, | ||
| 3575 | }; | ||
| 3576 | |||
| 3577 | err = ff_vk_alloc_mem(&p->vkctx, &req, | ||
| 3578 | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, | ||
| 3579 | import_desc, &vkb->flags, &vkb->mem); | ||
| 3493 | if (err < 0) | 3580 | if (err < 0) |
| 3494 | return err; | 3581 | return err; |
| 3495 | 3582 | ||
| 3496 | ff_vk_frame_barrier(&p->vkctx, exec, f, img_bar, &nb_img_bar, | 3583 | ret = vk->CreateBuffer(hwctx->act_dev, &buf_spawn, hwctx->alloc, &vkb->buf); |
| 3497 | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, | 3584 | if (ret != VK_SUCCESS) { |
| 3498 | VK_PIPELINE_STAGE_2_TRANSFER_BIT_KHR, | 3585 | vk->FreeMemory(hwctx->act_dev, vkb->mem, hwctx->alloc); |
| 3499 | to_buf ? VK_ACCESS_TRANSFER_READ_BIT : | 3586 | return AVERROR_EXTERNAL; |
| 3500 | VK_ACCESS_TRANSFER_WRITE_BIT, | 3587 | } |
| 3501 | to_buf ? VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL : | ||
| 3502 | VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, | ||
| 3503 | VK_QUEUE_FAMILY_IGNORED); | ||
| 3504 | 3588 | ||
| 3505 | vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) { | 3589 | ret = vk->BindBufferMemory(hwctx->act_dev, vkb->buf, vkb->mem, 0); |
| 3506 | .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, | 3590 | if (ret != VK_SUCCESS) { |
| 3507 | .pImageMemoryBarriers = img_bar, | 3591 | vk->FreeMemory(hwctx->act_dev, vkb->mem, hwctx->alloc); |
| 3508 | .imageMemoryBarrierCount = nb_img_bar, | 3592 | vk->DestroyBuffer(hwctx->act_dev, vkb->buf, hwctx->alloc); |
| 3509 | }); | 3593 | return AVERROR_EXTERNAL; |
| 3594 | } | ||
| 3595 | |||
| 3596 | return 0; | ||
| 3597 | } | ||
| 3598 | |||
| 3599 | static void destroy_avvkbuf(void *opaque, uint8_t *data) | ||
| 3600 | { | ||
| 3601 | FFVulkanContext *s = opaque; | ||
| 3602 | FFVkBuffer *buf = (FFVkBuffer *)data; | ||
| 3603 | ff_vk_free_buf(s, buf); | ||
| 3604 | av_free(buf); | ||
| 3605 | } | ||
| 3606 | |||
| 3607 | static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs, | ||
| 3608 | AVFrame *swf, VkBufferImageCopy *region, int upload) | ||
| 3609 | { | ||
| 3610 | int err; | ||
| 3611 | VkResult ret; | ||
| 3612 | VulkanDevicePriv *p = hwfc->device_ctx->hwctx; | ||
| 3613 | FFVulkanFunctions *vk = &p->vkctx.vkfn; | ||
| 3614 | AVVulkanDeviceContext *hwctx = &p->p; | ||
| 3615 | |||
| 3616 | const int planes = av_pix_fmt_count_planes(swf->format); | ||
| 3617 | |||
| 3618 | VkExternalMemoryBufferCreateInfo create_desc = { | ||
| 3619 | .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, | ||
| 3620 | .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, | ||
| 3621 | }; | ||
| 3622 | VkImportMemoryHostPointerInfoEXT import_desc = { | ||
| 3623 | .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT, | ||
| 3624 | .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, | ||
| 3625 | }; | ||
| 3626 | VkMemoryHostPointerPropertiesEXT props; | ||
| 3627 | |||
| 3628 | for (int i = 0; i < planes; i++) { | ||
| 3629 | FFVkBuffer *vkb; | ||
| 3630 | uint32_t p_w, p_h; | ||
| 3631 | size_t offs; | ||
| 3632 | size_t buffer_size; | ||
| 3633 | |||
| 3634 | /* We can't host map images with negative strides */ | ||
| 3635 | if (swf->linesize[i] < 0) { | ||
| 3636 | err = AVERROR(EINVAL); | ||
| 3637 | goto fail; | ||
| 3638 | } | ||
| 3639 | |||
| 3640 | get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i); | ||
| 3641 | |||
| 3642 | /* Get the previous point at which mapping was possible and use it */ | ||
| 3643 | offs = (uintptr_t)swf->data[i] % p->hprops.minImportedHostPointerAlignment; | ||
| 3644 | import_desc.pHostPointer = swf->data[i] - offs; | ||
| 3510 | 3645 | ||
| 3511 | /* Schedule a copy for each plane */ | 3646 | props = (VkMemoryHostPointerPropertiesEXT) { |
| 3512 | for (int i = 0; i < pixfmt_planes; i++) { | 3647 | VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT, |
| 3513 | int idx = FFMIN(i, nb_images - 1); | 3648 | }; |
| 3514 | VkImageAspectFlags plane_aspect[] = { VK_IMAGE_ASPECT_COLOR_BIT, | 3649 | ret = vk->GetMemoryHostPointerPropertiesEXT(hwctx->act_dev, |
| 3515 | VK_IMAGE_ASPECT_PLANE_0_BIT, | 3650 | import_desc.handleType, |
| 3516 | VK_IMAGE_ASPECT_PLANE_1_BIT, | 3651 | import_desc.pHostPointer, |
| 3517 | VK_IMAGE_ASPECT_PLANE_2_BIT, }; | 3652 | &props); |
| 3518 | 3653 | if (!(ret == VK_SUCCESS && props.memoryTypeBits)) { | |
| 3519 | FFVkBuffer *vkbuf = (FFVkBuffer *)bufs[i]->data; | 3654 | err = AVERROR(EINVAL); |
| 3520 | VkBufferImageCopy buf_reg = { | 3655 | goto fail; |
| 3521 | .bufferOffset = buf_offsets[i], | 3656 | } |
| 3522 | .bufferRowLength = buf_stride[i] / desc->comp[i].step, | 3657 | |
| 3658 | /* Buffer region for this plane */ | ||
| 3659 | region[i] = (VkBufferImageCopy) { | ||
| 3660 | .bufferOffset = offs, | ||
| 3661 | .bufferRowLength = swf->linesize[i], | ||
| 3662 | .bufferImageHeight = p_h, | ||
| 3523 | .imageSubresource.layerCount = 1, | 3663 | .imageSubresource.layerCount = 1, |
| 3524 | .imageSubresource.aspectMask = plane_aspect[(pixfmt_planes != nb_images) + | 3664 | .imageExtent = (VkExtent3D){ p_w, p_h, 1 }, |
| 3525 | i*(pixfmt_planes != nb_images)], | 3665 | /* Rest of the fields adjusted/filled in later */ |
| 3526 | .imageOffset = { 0, 0, 0, }, | ||
| 3527 | }; | 3666 | }; |
| 3528 | 3667 | ||
| 3529 | uint32_t p_w, p_h; | 3668 | /* Add the offset at the start, which gets ignored */ |
| 3530 | get_plane_wh(&p_w, &p_h, pix_fmt, w, h, i); | 3669 | buffer_size = offs + swf->linesize[i]*p_h; |
| 3670 | buffer_size = FFALIGN(buffer_size, p->props.properties.limits.minMemoryMapAlignment); | ||
| 3531 | 3671 | ||
| 3532 | buf_reg.bufferImageHeight = p_h; | 3672 | /* Create a buffer */ |
| 3533 | buf_reg.imageExtent = (VkExtent3D){ p_w, p_h, 1, }; | 3673 | vkb = av_mallocz(sizeof(*vkb)); |
| 3674 | if (!vkb) { | ||
| 3675 | err = AVERROR(ENOMEM); | ||
| 3676 | goto fail; | ||
| 3677 | } | ||
| 3534 | 3678 | ||
| 3535 | if (to_buf) | 3679 | err = create_mapped_buffer(hwfc, vkb, |
| 3536 | vk->CmdCopyImageToBuffer(cmd_buf, frame->img[idx], | 3680 | upload ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT : |
| 3537 | img_bar[0].newLayout, | 3681 | VK_BUFFER_USAGE_TRANSFER_DST_BIT, |
| 3538 | vkbuf->buf, | 3682 | buffer_size, &create_desc, &import_desc, |
| 3539 | 1, &buf_reg); | 3683 | props); |
| 3540 | else | 3684 | if (err < 0) { |
| 3541 | vk->CmdCopyBufferToImage(cmd_buf, vkbuf->buf, frame->img[idx], | 3685 | av_free(vkb); |
| 3542 | img_bar[0].newLayout, | 3686 | goto fail; |
| 3543 | 1, &buf_reg); | 3687 | } |
| 3544 | } | ||
| 3545 | 3688 | ||
| 3546 | err = ff_vk_exec_submit(&p->vkctx, exec); | 3689 | /* Create a ref */ |
| 3547 | if (err < 0) | 3690 | dst[*nb_bufs] = av_buffer_create((uint8_t *)vkb, sizeof(*vkb), |
| 3548 | return err; | 3691 | destroy_avvkbuf, &p->vkctx, 0); |
| 3692 | if (!dst[*nb_bufs]) { | ||
| 3693 | destroy_avvkbuf(&p->vkctx, (uint8_t *)vkb); | ||
| 3694 | err = AVERROR(ENOMEM); | ||
| 3695 | goto fail; | ||
| 3696 | } | ||
| 3549 | 3697 | ||
| 3550 | ff_vk_exec_wait(&p->vkctx, exec); | 3698 | (*nb_bufs)++; |
| 3699 | } | ||
| 3551 | 3700 | ||
| 3552 | return 0; | 3701 | return 0; |
| 3702 | |||
| 3703 | fail: | ||
| 3704 | for (int i = 0; i < (*nb_bufs); i++) | ||
| 3705 | av_buffer_unref(&dst[i]); | ||
| 3706 | return err; | ||
| 3553 | } | 3707 | } |
| 3554 | 3708 | ||
| 3555 | static int vulkan_transfer_data(AVHWFramesContext *hwfc, const AVFrame *vkf, | 3709 | static int vulkan_transfer_frame(AVHWFramesContext *hwfc, |
| 3556 | const AVFrame *swf, int from) | 3710 | AVFrame *swf, AVFrame *hwf, |
| 3711 | int upload) | ||
| 3557 | { | 3712 | { |
| 3558 | int err = 0; | 3713 | int err; |
| 3559 | VkResult ret; | 3714 | VulkanFramesPriv *fp = hwfc->hwctx; |
| 3560 | AVHWDeviceContext *dev_ctx = hwfc->device_ctx; | 3715 | VulkanDevicePriv *p = hwfc->device_ctx->hwctx; |
| 3561 | VulkanDevicePriv *p = dev_ctx->hwctx; | ||
| 3562 | AVVulkanDeviceContext *hwctx = &p->p; | ||
| 3563 | FFVulkanFunctions *vk = &p->vkctx.vkfn; | 3716 | FFVulkanFunctions *vk = &p->vkctx.vkfn; |
| 3564 | 3717 | ||
| 3565 | AVFrame tmp; | 3718 | int host_mapped = 0; |
| 3566 | FFVkBuffer *vkbufs[AV_NUM_DATA_POINTERS]; | 3719 | |
| 3567 | AVBufferRef *bufs[AV_NUM_DATA_POINTERS] = { 0 }; | 3720 | AVVkFrame *hwf_vk = (AVVkFrame *)hwf->data[0]; |
| 3568 | size_t buf_offsets[AV_NUM_DATA_POINTERS] = { 0 }; | 3721 | VkBufferImageCopy region[AV_NUM_DATA_POINTERS]; // always one per plane |
| 3569 | 3722 | ||
| 3570 | uint32_t p_w, p_h; | ||
| 3571 | const int planes = av_pix_fmt_count_planes(swf->format); | 3723 | const int planes = av_pix_fmt_count_planes(swf->format); |
| 3724 | const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swf->format); | ||
| 3725 | const int nb_images = ff_vk_count_images(hwf_vk); | ||
| 3726 | static const VkImageAspectFlags plane_aspect[] = { VK_IMAGE_ASPECT_COLOR_BIT, | ||
| 3727 | VK_IMAGE_ASPECT_PLANE_0_BIT, | ||
| 3728 | VK_IMAGE_ASPECT_PLANE_1_BIT, | ||
| 3729 | VK_IMAGE_ASPECT_PLANE_2_BIT, }; | ||
| 3730 | |||
| 3731 | VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS]; | ||
| 3732 | int nb_img_bar = 0; | ||
| 3572 | 3733 | ||
| 3573 | int host_mapped[AV_NUM_DATA_POINTERS] = { 0 }; | 3734 | AVBufferRef *bufs[AV_NUM_DATA_POINTERS]; |
| 3574 | const int map_host = !!(p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY); | 3735 | int nb_bufs = 0; |
| 3575 | 3736 | ||
| 3737 | VkCommandBuffer cmd_buf; | ||
| 3738 | FFVkExecContext *exec; | ||
| 3739 | |||
| 3740 | /* Sanity checking */ | ||
| 3576 | if ((swf->format != AV_PIX_FMT_NONE && !av_vkfmt_from_pixfmt(swf->format))) { | 3741 | if ((swf->format != AV_PIX_FMT_NONE && !av_vkfmt_from_pixfmt(swf->format))) { |
| 3577 | av_log(hwfc, AV_LOG_ERROR, "Unsupported software frame pixel format!\n"); | 3742 | av_log(hwfc, AV_LOG_ERROR, "Unsupported software frame pixel format!\n"); |
| 3578 | return AVERROR(EINVAL); | 3743 | return AVERROR(EINVAL); |
| @@ -3581,115 +3746,97 @@ static int vulkan_transfer_data(AVHWFramesContext *hwfc, const AVFrame *vkf, · 2 unmodified lines | |||
| 3581 | if (swf->width > hwfc->width || swf->height > hwfc->height) | 3746 | if (swf->width > hwfc->width || swf->height > hwfc->height) |
| 3582 | return AVERROR(EINVAL); | 3747 | return AVERROR(EINVAL); |
| 3583 | 3748 | ||
| 3584 | /* Create buffers */ | 3749 | /* Setup buffers first */ |
| 3585 | for (int i = 0; i < planes; i++) { | 3750 | if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY) { |
| 3586 | size_t req_size; | 3751 | err = host_map_frame(hwfc, bufs, &nb_bufs, swf, region, upload); |
| 3587 | 3752 | if (err >= 0) | |
| 3588 | VkExternalMemoryBufferCreateInfo create_desc = { | 3753 | host_mapped = 1; |
| 3589 | .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, | 3754 | } |
| 3590 | .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, | ||
| 3591 | }; | ||
| 3592 | |||
| 3593 | VkImportMemoryHostPointerInfoEXT import_desc = { | ||
| 3594 | .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT, | ||
| 3595 | .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, | ||
| 3596 | }; | ||
| 3597 | |||
| 3598 | VkMemoryHostPointerPropertiesEXT p_props = { | ||
| 3599 | .sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT, | ||
| 3600 | }; | ||
| 3601 | |||
| 3602 | get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i); | ||
| 3603 | |||
| 3604 | tmp.linesize[i] = FFABS(swf->linesize[i]); | ||
| 3605 | |||
| 3606 | /* Do not map images with a negative stride */ | ||
| 3607 | if (map_host && swf->linesize[i] > 0) { | ||
| 3608 | size_t offs; | ||
| 3609 | offs = (uintptr_t)swf->data[i] % p->hprops.minImportedHostPointerAlignment; | ||
| 3610 | import_desc.pHostPointer = swf->data[i] - offs; | ||
| 3611 | |||
| 3612 | /* We have to compensate for the few extra bytes of padding we | ||
| 3613 | * completely ignore at the start */ | ||
| 3614 | req_size = FFALIGN(offs + tmp.linesize[i] * p_h, | ||
| 3615 | p->hprops.minImportedHostPointerAlignment); | ||
| 3616 | |||
| 3617 | ret = vk->GetMemoryHostPointerPropertiesEXT(hwctx->act_dev, | ||
| 3618 | import_desc.handleType, | ||
| 3619 | import_desc.pHostPointer, | ||
| 3620 | &p_props); | ||
| 3621 | if (ret == VK_SUCCESS && p_props.memoryTypeBits) { | ||
| 3622 | host_mapped[i] = 1; | ||
| 3623 | buf_offsets[i] = offs; | ||
| 3624 | } | ||
| 3625 | } | ||
| 3626 | 3755 | ||
| 3627 | if (!host_mapped[i]) | 3756 | if (!host_mapped) { |
| 3628 | req_size = get_req_buffer_size(p, &tmp.linesize[i], p_h); | 3757 | err = get_plane_buf(hwfc, &bufs[0], swf, region, upload); |
| 3629 | |||
| 3630 | err = ff_vk_create_avbuf(&p->vkctx, &bufs[i], req_size, | ||
| 3631 | host_mapped[i] ? &create_desc : NULL, | ||
| 3632 | host_mapped[i] ? &import_desc : NULL, | ||
| 3633 | from ? VK_BUFFER_USAGE_TRANSFER_DST_BIT : | ||
| 3634 | VK_BUFFER_USAGE_TRANSFER_SRC_BIT, | ||
| 3635 | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | | ||
| 3636 | (host_mapped[i] ? | ||
| 3637 | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT : 0x0)); | ||
| 3638 | if (err < 0) | 3758 | if (err < 0) |
| 3639 | goto end; | 3759 | goto end; |
| 3760 | nb_bufs = 1; | ||
| 3640 | 3761 | ||
| 3641 | vkbufs[i] = (FFVkBuffer *)bufs[i]->data; | 3762 | if (upload) { |
| 3763 | err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 1); | ||
| 3764 | if (err < 0) | ||
| 3765 | goto end; | ||
| 3766 | } | ||
| 3642 | } | 3767 | } |
| 3643 | 3768 | ||
| 3644 | if (!from) { | 3769 | exec = ff_vk_exec_get(&fp->upload_exec); |
| 3645 | /* Map, copy image TO buffer (which then goes to the VkImage), unmap */ | 3770 | cmd_buf = exec->buf; |
| 3646 | if ((err = ff_vk_map_buffers(&p->vkctx, vkbufs, tmp.data, planes, 0))) | ||
| 3647 | goto end; | ||
| 3648 | |||
| 3649 | for (int i = 0; i < planes; i++) { | ||
| 3650 | if (host_mapped[i]) | ||
| 3651 | continue; | ||
| 3652 | 3771 | ||
| 3653 | get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i); | 3772 | ff_vk_exec_start(&p->vkctx, exec); |
| 3654 | 3773 | ||
| 3655 | av_image_copy_plane(tmp.data[i], tmp.linesize[i], | 3774 | /* Prep destination Vulkan frame */ |
| 3656 | (const uint8_t *)swf->data[i], swf->linesize[i], | 3775 | err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, hwf, |
| 3657 | FFMIN(tmp.linesize[i], FFABS(swf->linesize[i])), | 3776 | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, |
| 3658 | p_h); | 3777 | VK_PIPELINE_STAGE_2_TRANSFER_BIT); |
| 3659 | } | 3778 | if (err < 0) |
| 3779 | goto end; | ||
| 3660 | 3780 | ||
| 3661 | if ((err = ff_vk_unmap_buffers(&p->vkctx, vkbufs, planes, 1))) | 3781 | /* No need to declare buf deps for synchronous transfers */ |
| 3782 | if (upload) { | ||
| 3783 | err = ff_vk_exec_add_dep_buf(&p->vkctx, exec, bufs, nb_bufs, 1); | ||
| 3784 | if (err < 0) { | ||
| 3785 | ff_vk_exec_discard_deps(&p->vkctx, exec); | ||
| 3662 | goto end; | 3786 | goto end; |
| 3787 | } | ||
| 3663 | } | 3788 | } |
| 3664 | 3789 | ||
| 3665 | /* Copy buffers into/from image */ | 3790 | ff_vk_frame_barrier(&p->vkctx, exec, hwf, img_bar, &nb_img_bar, |
| 3666 | err = transfer_image_buf(hwfc, (AVFrame *)vkf, bufs, buf_offsets, | 3791 | VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, |
| 3667 | tmp.linesize, swf->width, swf->height, swf->format, | 3792 | VK_PIPELINE_STAGE_2_TRANSFER_BIT_KHR, |
| 3668 | from); | 3793 | upload ? VK_ACCESS_TRANSFER_WRITE_BIT : |
| 3669 | 3794 | VK_ACCESS_TRANSFER_READ_BIT, | |
| 3670 | if (from) { | 3795 | upload ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL : |
| 3671 | /* Map, copy buffer (which came FROM the VkImage) to the frame, unmap */ | 3796 | VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, |
| 3672 | if ((err = ff_vk_map_buffers(&p->vkctx, vkbufs, tmp.data, planes, 0))) | 3797 | VK_QUEUE_FAMILY_IGNORED); |
| 3673 | goto end; | ||
| 3674 | 3798 | ||
| 3675 | for (int i = 0; i < planes; i++) { | 3799 | vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) { |
| 3676 | if (host_mapped[i]) | 3800 | .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, |
| 3677 | continue; | 3801 | .pImageMemoryBarriers = img_bar, |
| 3802 | .imageMemoryBarrierCount = nb_img_bar, | ||
| 3803 | }); | ||
| 3678 | 3804 | ||
| 3679 | get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i); | 3805 | for (int i = 0; i < planes; i++) { |
| 3806 | int buf_idx = FFMIN(i, (nb_bufs - 1)); | ||
| 3807 | int img_idx = FFMIN(i, (nb_images - 1)); | ||
| 3808 | FFVkBuffer *vkbuf = (FFVkBuffer *)bufs[buf_idx]->data; | ||
| 3809 | |||
| 3810 | uint32_t orig_stride = region[i].bufferRowLength; | ||
| 3811 | region[i].bufferRowLength /= desc->comp[i].step; | ||
| 3812 | region[i].imageSubresource.aspectMask = plane_aspect[(planes != nb_images) + | ||
| 3813 | i*(planes != nb_images)]; | ||
| 3814 | |||
| 3815 | if (upload) | ||
| 3816 | vk->CmdCopyBufferToImage(cmd_buf, vkbuf->buf, | ||
| 3817 | hwf_vk->img[img_idx], | ||
| 3818 | img_bar[img_idx].newLayout, | ||
| 3819 | 1, ®ion[i]); | ||
| 3820 | else | ||
| 3821 | vk->CmdCopyImageToBuffer(cmd_buf, hwf_vk->img[img_idx], | ||
| 3822 | img_bar[img_idx].newLayout, | ||
| 3823 | vkbuf->buf, | ||
| 3824 | 1, ®ion[i]); | ||
| 3680 | 3825 | ||
| 3681 | av_image_copy_plane_uc_from(swf->data[i], swf->linesize[i], | 3826 | region[i].bufferRowLength = orig_stride; |
| 3682 | (const uint8_t *)tmp.data[i], tmp.linesize[i], | 3827 | } |
| 3683 | FFMIN(tmp.linesize[i], FFABS(swf->linesize[i])), | ||
| 3684 | p_h); | ||
| 3685 | } | ||
| 3686 | 3828 | ||
| 3687 | if ((err = ff_vk_unmap_buffers(&p->vkctx, vkbufs, planes, 1))) | 3829 | err = ff_vk_exec_submit(&p->vkctx, exec); |
| 3688 | goto end; | 3830 | if (err < 0) { |
| 3831 | ff_vk_exec_discard_deps(&p->vkctx, exec); | ||
| 3832 | } else if (!upload) { | ||
| 3833 | ff_vk_exec_wait(&p->vkctx, exec); | ||
| 3834 | if (!host_mapped) | ||
| 3835 | err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 0); | ||
| 3689 | } | 3836 | } |
| 3690 | 3837 | ||
| 3691 | end: | 3838 | end: |
| 3692 | for (int i = 0; i < planes; i++) | 3839 | for (int i = 0; i < nb_bufs; i++) |
| 3693 | av_buffer_unref(&bufs[i]); | 3840 | av_buffer_unref(&bufs[i]); |
| 3694 | 3841 | ||
| 3695 | return err; | 3842 | return err; |
| @@ -3716,7 +3863,7 @@ static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst, · 20 unmodified lines | |||
| 3716 | if (src->hw_frames_ctx) | 3863 | if (src->hw_frames_ctx) |
| 3717 | return AVERROR(ENOSYS); | 3864 | return AVERROR(ENOSYS); |
| 3718 | else | 3865 | else |
| 3719 | return vulkan_transfer_data(hwfc, dst, src, 0); | 3866 | return vulkan_transfer_frame(hwfc, (AVFrame *)src, dst, 1); |
| 3720 | } | 3867 | } |
| 3721 | } | 3868 | } |
| 3722 | 3869 | ||
| @@ -3833,7 +3980,7 @@ static int vulkan_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst, · 110 unmodified lines | |||
| 3833 | if (dst->hw_frames_ctx) | 3980 | if (dst->hw_frames_ctx) |
| 3834 | return AVERROR(ENOSYS); | 3981 | return AVERROR(ENOSYS); |
| 3835 | else | 3982 | else |
| 3836 | return vulkan_transfer_data(hwfc, src, dst, 1); | 3983 | return vulkan_transfer_frame(hwfc, dst, (AVFrame *)src, 0); |
| 3837 | } | 3984 | } |
| 3838 | } | 3985 | } |
| 3839 | 3986 | ||
| @@ -809,11 +809,8 @@ int ff_vk_alloc_mem(FFVulkanContext *s, VkMemoryRequirements *req, · 808 unmodified lines | |||
| 809 | 809 | ||
| 810 | ret = vk->AllocateMemory(s->hwctx->act_dev, &alloc_info, | 810 | ret = vk->AllocateMemory(s->hwctx->act_dev, &alloc_info, |
| 811 | s->hwctx->alloc, mem); | 811 | s->hwctx->alloc, mem); |
| 812 | if (ret != VK_SUCCESS) { | 812 | if (ret != VK_SUCCESS) |
| 813 | av_log(s, AV_LOG_ERROR, "Failed to allocate memory: %s\n", | ||
| 814 | ff_vk_ret2str(ret)); | ||
| 815 | return AVERROR(ENOMEM); | 813 | return AVERROR(ENOMEM); |
| 816 | } | ||
| 817 | 814 | ||
| 818 | if (mem_flags) | 815 | if (mem_flags) |
| 819 | *mem_flags |= s->mprops.memoryTypes[index].propertyFlags; | 816 | *mem_flags |= s->mprops.memoryTypes[index].propertyFlags; |