This commit was long overdue. The old transfer dubiously tried to merge as much code as possible, and had very little in the way of optimizations, apart from basic host-mapping. The new code uses buffer pools for any temporary bufflers, and handles falling back to buffer-based uploads if host-mapping fails. Roundtrip performance difference: ffmpeg -init_hw_device "vulkan=vk:0,debug=0,disable_multiplane=1" -f lavfi \ -i color=red:s=3840x2160 -vf hwupload,hwdownload,format=yuv420p -f null - 7900XTX: Before: 224fps After: 502fps Ada, with proprietary drivers: Before: 29fps After: 54fps Alder Lake: Before: 85fps After: 108fps With the host-mapping codepath disabled: Before: 32fps After: 51fps
Unified · Split
| @@ -129,6 +129,9 @@ typedef struct VulkanFramesPriv { · 128 unmodified lines | ||
| 129 | 129 | FFVkExecPool upload_exec; |
| 130 | 130 | FFVkExecPool download_exec; |
| 131 | 131 | |
| 132 | + /* Temporary buffer pools */ | |
| 133 | + AVBufferPool *tmp; | |
| 134 | + | |
| 132 | 135 | /* Modifier info list to free at uninit */ |
| 133 | 136 | VkImageDrmFormatModifierListCreateInfoEXT *modifier_info; |
| 134 | 137 | } VulkanFramesPriv; |
| @@ -2425,6 +2428,8 @@ static void vulkan_frames_uninit(AVHWFramesContext *hwfc) · 2290 unmodified lines | ||
| 2425 | 2428 | ff_vk_exec_pool_free(&p->vkctx, &fp->compute_exec); |
| 2426 | 2429 | ff_vk_exec_pool_free(&p->vkctx, &fp->upload_exec); |
| 2427 | 2430 | ff_vk_exec_pool_free(&p->vkctx, &fp->download_exec); |
| 2431 | + | |
| 2432 | + av_buffer_pool_uninit(&fp->tmp); | |
| 2428 | 2433 | } |
| 2429 | 2434 | |
| 2430 | 2435 | static int vulkan_frames_init(AVHWFramesContext *hwfc) |
| @@ -3451,128 +3456,288 @@ static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst, · 1020 unmodified lines | ||
| 3451 | 3456 | return AVERROR(ENOSYS); |
| 3452 | 3457 | } |
| 3453 | 3458 | |
| 3454 | −static size_t get_req_buffer_size(VulkanDevicePriv *p, int *stride, int height) | |
| 3459 | +static int copy_buffer_data(AVHWFramesContext *hwfc, AVBufferRef *buf, | |
| 3460 | + AVFrame *swf, VkBufferImageCopy *region, | |
| 3461 | + int planes, int upload) | |
| 3455 | 3462 | { |
| 3456 | − size_t size; | |
| 3457 | − *stride = FFALIGN(*stride, p->props.properties.limits.optimalBufferCopyRowPitchAlignment); | |
| 3458 | − size = height*(*stride); | |
| 3459 | − size = FFALIGN(size, p->props.properties.limits.minMemoryMapAlignment); | |
| 3460 | − return size; | |
| 3463 | + VkResult ret; | |
| 3464 | + VulkanDevicePriv *p = hwfc->device_ctx->hwctx; | |
| 3465 | + FFVulkanFunctions *vk = &p->vkctx.vkfn; | |
| 3466 | + AVVulkanDeviceContext *hwctx = &p->p; | |
| 3467 | + | |
| 3468 | + FFVkBuffer *vkbuf = (FFVkBuffer *)buf->data; | |
| 3469 | + | |
| 3470 | + const VkMappedMemoryRange flush_info = { | |
| 3471 | + .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE, | |
| 3472 | + .memory = vkbuf->mem, | |
| 3473 | + .size = VK_WHOLE_SIZE, | |
| 3474 | + }; | |
| 3475 | + | |
| 3476 | + if (!(vkbuf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) && !upload) { | |
| 3477 | + ret = vk->InvalidateMappedMemoryRanges(hwctx->act_dev, 1, | |
| 3478 | + &flush_info); | |
| 3479 | + if (ret != VK_SUCCESS) { | |
| 3480 | + av_log(hwfc, AV_LOG_ERROR, "Failed to invalidate buffer data: %s\n", | |
| 3481 | + ff_vk_ret2str(ret)); | |
| 3482 | + return AVERROR_EXTERNAL; | |
| 3483 | + } | |
| 3484 | + } | |
| 3485 | + | |
| 3486 | + for (int i = 0; i < planes; i++) | |
| 3487 | + av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset, | |
| 3488 | + region[i].bufferRowLength, | |
| 3489 | + swf->data[i], | |
| 3490 | + swf->linesize[i], | |
| 3491 | + swf->linesize[i], | |
| 3492 | + region[i].imageExtent.height); | |
| 3493 | + | |
| 3494 | + if (!(vkbuf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) && upload) { | |
| 3495 | + ret = vk->FlushMappedMemoryRanges(hwctx->act_dev, 1, | |
| 3496 | + &flush_info); | |
| 3497 | + if (ret != VK_SUCCESS) { | |
| 3498 | + av_log(hwfc, AV_LOG_ERROR, "Failed to flush buffer data: %s\n", | |
| 3499 | + ff_vk_ret2str(ret)); | |
| 3500 | + return AVERROR_EXTERNAL; | |
| 3501 | + } | |
| 3502 | + } | |
| 3503 | + | |
| 3504 | + return 0; | |
| 3461 | 3505 | } |
| 3462 | 3506 | |
| 3463 | −static int transfer_image_buf(AVHWFramesContext *hwfc, AVFrame *f, | |
| 3464 | − AVBufferRef **bufs, size_t *buf_offsets, | |
| 3465 | − const int *buf_stride, int w, | |
| 3466 | − int h, enum AVPixelFormat pix_fmt, int to_buf) | |
| 3507 | +static int get_plane_buf(AVHWFramesContext *hwfc, AVBufferRef **dst, | |
| 3508 | + AVFrame *swf, VkBufferImageCopy *region, int upload) | |
| 3467 | 3509 | { |
| 3468 | 3510 | int err; |
| 3469 | − AVVkFrame *frame = (AVVkFrame *)f->data[0]; | |
| 3470 | 3511 | VulkanFramesPriv *fp = hwfc->hwctx; |
| 3471 | 3512 | VulkanDevicePriv *p = hwfc->device_ctx->hwctx; |
| 3472 | − FFVulkanFunctions *vk = &p->vkctx.vkfn; | |
| 3473 | − VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS]; | |
| 3474 | − int nb_img_bar = 0; | |
| 3513 | + const int planes = av_pix_fmt_count_planes(swf->format); | |
| 3514 | + | |
| 3515 | + size_t buf_offset = 0; | |
| 3516 | + for (int i = 0; i < planes; i++) { | |
| 3517 | + size_t size; | |
| 3518 | + ptrdiff_t linesize = swf->linesize[i]; | |
| 3475 | 3519 | |
| 3476 | − const int nb_images = ff_vk_count_images(frame); | |
| 3477 | − int pixfmt_planes = av_pix_fmt_count_planes(pix_fmt); | |
| 3478 | − const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt); | |
| 3520 | + uint32_t p_w, p_h; | |
| 3521 | + get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i); | |
| 3479 | 3522 | |
| 3480 | − VkCommandBuffer cmd_buf; | |
| 3481 | − FFVkExecContext *exec = ff_vk_exec_get(to_buf ? &fp->download_exec : | |
| 3482 | − &fp->upload_exec); | |
| 3483 | − cmd_buf = exec->buf; | |
| 3484 | − ff_vk_exec_start(&p->vkctx, exec); | |
| 3523 | + linesize = FFALIGN(linesize, | |
| 3524 | + p->props.properties.limits.optimalBufferCopyRowPitchAlignment); | |
| 3525 | + size = p_h*linesize; | |
| 3485 | 3526 | |
| 3486 | − err = ff_vk_exec_add_dep_buf(&p->vkctx, exec, bufs, pixfmt_planes, 1); | |
| 3527 | + region[i] = (VkBufferImageCopy) { | |
| 3528 | + .bufferOffset = buf_offset, | |
| 3529 | + .bufferRowLength = linesize, | |
| 3530 | + .bufferImageHeight = p_h, | |
| 3531 | + .imageSubresource.layerCount = 1, | |
| 3532 | + .imageExtent = (VkExtent3D){ p_w, p_h, 1 }, | |
| 3533 | + /* Rest of the fields adjusted/filled in later */ | |
| 3534 | + }; | |
| 3535 | + | |
| 3536 | + buf_offset = FFALIGN(buf_offset + size, | |
| 3537 | + p->props.properties.limits.optimalBufferCopyOffsetAlignment); | |
| 3538 | + } | |
| 3539 | + | |
| 3540 | + err = ff_vk_get_pooled_buffer(&p->vkctx, &fp->tmp, dst, | |
| 3541 | + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | | |
| 3542 | + VK_BUFFER_USAGE_TRANSFER_DST_BIT, | |
| 3543 | + NULL, buf_offset, | |
| 3544 | + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT); | |
| 3487 | 3545 | if (err < 0) |
| 3488 | 3546 | return err; |
| 3489 | 3547 | |
| 3490 | − err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, f, | |
| 3491 | − VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, | |
| 3492 | − VK_PIPELINE_STAGE_2_TRANSFER_BIT); | |
| 3548 | + return 0; | |
| 3549 | +} | |
| 3550 | + | |
| 3551 | +static int create_mapped_buffer(AVHWFramesContext *hwfc, | |
| 3552 | + FFVkBuffer *vkb, VkBufferUsageFlags usage, | |
| 3553 | + size_t size, | |
| 3554 | + VkExternalMemoryBufferCreateInfo *create_desc, | |
| 3555 | + VkImportMemoryHostPointerInfoEXT *import_desc, | |
| 3556 | + VkMemoryHostPointerPropertiesEXT props) | |
| 3557 | +{ | |
| 3558 | + int err; | |
| 3559 | + VkResult ret; | |
| 3560 | + VulkanDevicePriv *p = hwfc->device_ctx->hwctx; | |
| 3561 | + FFVulkanFunctions *vk = &p->vkctx.vkfn; | |
| 3562 | + AVVulkanDeviceContext *hwctx = &p->p; | |
| 3563 | + | |
| 3564 | + VkBufferCreateInfo buf_spawn = { | |
| 3565 | + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, | |
| 3566 | + .pNext = create_desc, | |
| 3567 | + .usage = usage, | |
| 3568 | + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, | |
| 3569 | + .size = size, | |
| 3570 | + }; | |
| 3571 | + VkMemoryRequirements req = { | |
| 3572 | + .size = size, | |
| 3573 | + .alignment = p->hprops.minImportedHostPointerAlignment, | |
| 3574 | + .memoryTypeBits = props.memoryTypeBits, | |
| 3575 | + }; | |
| 3576 | + | |
| 3577 | + err = ff_vk_alloc_mem(&p->vkctx, &req, | |
| 3578 | + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, | |
| 3579 | + import_desc, &vkb->flags, &vkb->mem); | |
| 3493 | 3580 | if (err < 0) |
| 3494 | 3581 | return err; |
| 3495 | 3582 | |
| 3496 | − ff_vk_frame_barrier(&p->vkctx, exec, f, img_bar, &nb_img_bar, | |
| 3497 | − VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, | |
| 3498 | − VK_PIPELINE_STAGE_2_TRANSFER_BIT_KHR, | |
| 3499 | − to_buf ? VK_ACCESS_TRANSFER_READ_BIT : | |
| 3500 | − VK_ACCESS_TRANSFER_WRITE_BIT, | |
| 3501 | − to_buf ? VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL : | |
| 3502 | − VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, | |
| 3503 | − VK_QUEUE_FAMILY_IGNORED); | |
| 3583 | + ret = vk->CreateBuffer(hwctx->act_dev, &buf_spawn, hwctx->alloc, &vkb->buf); | |
| 3584 | + if (ret != VK_SUCCESS) { | |
| 3585 | + vk->FreeMemory(hwctx->act_dev, vkb->mem, hwctx->alloc); | |
| 3586 | + return AVERROR_EXTERNAL; | |
| 3587 | + } | |
| 3504 | 3588 | |
| 3505 | − vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) { | |
| 3506 | − .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, | |
| 3507 | − .pImageMemoryBarriers = img_bar, | |
| 3508 | − .imageMemoryBarrierCount = nb_img_bar, | |
| 3509 | − }); | |
| 3589 | + ret = vk->BindBufferMemory(hwctx->act_dev, vkb->buf, vkb->mem, 0); | |
| 3590 | + if (ret != VK_SUCCESS) { | |
| 3591 | + vk->FreeMemory(hwctx->act_dev, vkb->mem, hwctx->alloc); | |
| 3592 | + vk->DestroyBuffer(hwctx->act_dev, vkb->buf, hwctx->alloc); | |
| 3593 | + return AVERROR_EXTERNAL; | |
| 3594 | + } | |
| 3595 | + | |
| 3596 | + return 0; | |
| 3597 | +} | |
| 3598 | + | |
| 3599 | +static void destroy_avvkbuf(void *opaque, uint8_t *data) | |
| 3600 | +{ | |
| 3601 | + FFVulkanContext *s = opaque; | |
| 3602 | + FFVkBuffer *buf = (FFVkBuffer *)data; | |
| 3603 | + ff_vk_free_buf(s, buf); | |
| 3604 | + av_free(buf); | |
| 3605 | +} | |
| 3606 | + | |
| 3607 | +static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs, | |
| 3608 | + AVFrame *swf, VkBufferImageCopy *region, int upload) | |
| 3609 | +{ | |
| 3610 | + int err; | |
| 3611 | + VkResult ret; | |
| 3612 | + VulkanDevicePriv *p = hwfc->device_ctx->hwctx; | |
| 3613 | + FFVulkanFunctions *vk = &p->vkctx.vkfn; | |
| 3614 | + AVVulkanDeviceContext *hwctx = &p->p; | |
| 3615 | + | |
| 3616 | + const int planes = av_pix_fmt_count_planes(swf->format); | |
| 3617 | + | |
| 3618 | + VkExternalMemoryBufferCreateInfo create_desc = { | |
| 3619 | + .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, | |
| 3620 | + .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, | |
| 3621 | + }; | |
| 3622 | + VkImportMemoryHostPointerInfoEXT import_desc = { | |
| 3623 | + .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT, | |
| 3624 | + .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, | |
| 3625 | + }; | |
| 3626 | + VkMemoryHostPointerPropertiesEXT props; | |
| 3627 | + | |
| 3628 | + for (int i = 0; i < planes; i++) { | |
| 3629 | + FFVkBuffer *vkb; | |
| 3630 | + uint32_t p_w, p_h; | |
| 3631 | + size_t offs; | |
| 3632 | + size_t buffer_size; | |
| 3633 | + | |
| 3634 | + /* We can't host map images with negative strides */ | |
| 3635 | + if (swf->linesize[i] < 0) { | |
| 3636 | + err = AVERROR(EINVAL); | |
| 3637 | + goto fail; | |
| 3638 | + } | |
| 3639 | + | |
| 3640 | + get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i); | |
| 3641 | + | |
| 3642 | + /* Get the previous point at which mapping was possible and use it */ | |
| 3643 | + offs = (uintptr_t)swf->data[i] % p->hprops.minImportedHostPointerAlignment; | |
| 3644 | + import_desc.pHostPointer = swf->data[i] - offs; | |
| 3510 | 3645 | |
| 3511 | − /* Schedule a copy for each plane */ | |
| 3512 | − for (int i = 0; i < pixfmt_planes; i++) { | |
| 3513 | − int idx = FFMIN(i, nb_images - 1); | |
| 3514 | − VkImageAspectFlags plane_aspect[] = { VK_IMAGE_ASPECT_COLOR_BIT, | |
| 3515 | − VK_IMAGE_ASPECT_PLANE_0_BIT, | |
| 3516 | − VK_IMAGE_ASPECT_PLANE_1_BIT, | |
| 3517 | − VK_IMAGE_ASPECT_PLANE_2_BIT, }; | |
| 3518 | − | |
| 3519 | − FFVkBuffer *vkbuf = (FFVkBuffer *)bufs[i]->data; | |
| 3520 | − VkBufferImageCopy buf_reg = { | |
| 3521 | − .bufferOffset = buf_offsets[i], | |
| 3522 | − .bufferRowLength = buf_stride[i] / desc->comp[i].step, | |
| 3646 | + props = (VkMemoryHostPointerPropertiesEXT) { | |
| 3647 | + VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT, | |
| 3648 | + }; | |
| 3649 | + ret = vk->GetMemoryHostPointerPropertiesEXT(hwctx->act_dev, | |
| 3650 | + import_desc.handleType, | |
| 3651 | + import_desc.pHostPointer, | |
| 3652 | + &props); | |
| 3653 | + if (!(ret == VK_SUCCESS && props.memoryTypeBits)) { | |
| 3654 | + err = AVERROR(EINVAL); | |
| 3655 | + goto fail; | |
| 3656 | + } | |
| 3657 | + | |
| 3658 | + /* Buffer region for this plane */ | |
| 3659 | + region[i] = (VkBufferImageCopy) { | |
| 3660 | + .bufferOffset = offs, | |
| 3661 | + .bufferRowLength = swf->linesize[i], | |
| 3662 | + .bufferImageHeight = p_h, | |
| 3523 | 3663 | .imageSubresource.layerCount = 1, |
| 3524 | − .imageSubresource.aspectMask = plane_aspect[(pixfmt_planes != nb_images) + | |
| 3525 | − i*(pixfmt_planes != nb_images)], | |
| 3526 | − .imageOffset = { 0, 0, 0, }, | |
| 3664 | + .imageExtent = (VkExtent3D){ p_w, p_h, 1 }, | |
| 3665 | + /* Rest of the fields adjusted/filled in later */ | |
| 3527 | 3666 | }; |
| 3528 | 3667 | |
| 3529 | − uint32_t p_w, p_h; | |
| 3530 | − get_plane_wh(&p_w, &p_h, pix_fmt, w, h, i); | |
| 3668 | + /* Add the offset at the start, which gets ignored */ | |
| 3669 | + buffer_size = offs + swf->linesize[i]*p_h; | |
| 3670 | + buffer_size = FFALIGN(buffer_size, p->props.properties.limits.minMemoryMapAlignment); | |
| 3531 | 3671 | |
| 3532 | − buf_reg.bufferImageHeight = p_h; | |
| 3533 | − buf_reg.imageExtent = (VkExtent3D){ p_w, p_h, 1, }; | |
| 3672 | + /* Create a buffer */ | |
| 3673 | + vkb = av_mallocz(sizeof(*vkb)); | |
| 3674 | + if (!vkb) { | |
| 3675 | + err = AVERROR(ENOMEM); | |
| 3676 | + goto fail; | |
| 3677 | + } | |
| 3534 | 3678 | |
| 3535 | − if (to_buf) | |
| 3536 | − vk->CmdCopyImageToBuffer(cmd_buf, frame->img[idx], | |
| 3537 | − img_bar[0].newLayout, | |
| 3538 | − vkbuf->buf, | |
| 3539 | − 1, &buf_reg); | |
| 3540 | − else | |
| 3541 | − vk->CmdCopyBufferToImage(cmd_buf, vkbuf->buf, frame->img[idx], | |
| 3542 | − img_bar[0].newLayout, | |
| 3543 | − 1, &buf_reg); | |
| 3544 | − } | |
| 3679 | + err = create_mapped_buffer(hwfc, vkb, | |
| 3680 | + upload ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT : | |
| 3681 | + VK_BUFFER_USAGE_TRANSFER_DST_BIT, | |
| 3682 | + buffer_size, &create_desc, &import_desc, | |
| 3683 | + props); | |
| 3684 | + if (err < 0) { | |
| 3685 | + av_free(vkb); | |
| 3686 | + goto fail; | |
| 3687 | + } | |
| 3545 | 3688 | |
| 3546 | − err = ff_vk_exec_submit(&p->vkctx, exec); | |
| 3547 | − if (err < 0) | |
| 3548 | − return err; | |
| 3689 | + /* Create a ref */ | |
| 3690 | + dst[*nb_bufs] = av_buffer_create((uint8_t *)vkb, sizeof(*vkb), | |
| 3691 | + destroy_avvkbuf, &p->vkctx, 0); | |
| 3692 | + if (!dst[*nb_bufs]) { | |
| 3693 | + destroy_avvkbuf(&p->vkctx, (uint8_t *)vkb); | |
| 3694 | + err = AVERROR(ENOMEM); | |
| 3695 | + goto fail; | |
| 3696 | + } | |
| 3549 | 3697 | |
| 3550 | − ff_vk_exec_wait(&p->vkctx, exec); | |
| 3698 | + (*nb_bufs)++; | |
| 3699 | + } | |
| 3551 | 3700 | |
| 3552 | 3701 | return 0; |
| 3702 | + | |
| 3703 | +fail: | |
| 3704 | + for (int i = 0; i < (*nb_bufs); i++) | |
| 3705 | + av_buffer_unref(&dst[i]); | |
| 3706 | + return err; | |
| 3553 | 3707 | } |
| 3554 | 3708 | |
| 3555 | −static int vulkan_transfer_data(AVHWFramesContext *hwfc, const AVFrame *vkf, | |
| 3556 | − const AVFrame *swf, int from) | |
| 3709 | +static int vulkan_transfer_frame(AVHWFramesContext *hwfc, | |
| 3710 | + AVFrame *swf, AVFrame *hwf, | |
| 3711 | + int upload) | |
| 3557 | 3712 | { |
| 3558 | − int err = 0; | |
| 3559 | − VkResult ret; | |
| 3560 | − AVHWDeviceContext *dev_ctx = hwfc->device_ctx; | |
| 3561 | − VulkanDevicePriv *p = dev_ctx->hwctx; | |
| 3562 | − AVVulkanDeviceContext *hwctx = &p->p; | |
| 3713 | + int err; | |
| 3714 | + VulkanFramesPriv *fp = hwfc->hwctx; | |
| 3715 | + VulkanDevicePriv *p = hwfc->device_ctx->hwctx; | |
| 3563 | 3716 | FFVulkanFunctions *vk = &p->vkctx.vkfn; |
| 3564 | 3717 | |
| 3565 | − AVFrame tmp; | |
| 3566 | − FFVkBuffer *vkbufs[AV_NUM_DATA_POINTERS]; | |
| 3567 | − AVBufferRef *bufs[AV_NUM_DATA_POINTERS] = { 0 }; | |
| 3568 | − size_t buf_offsets[AV_NUM_DATA_POINTERS] = { 0 }; | |
| 3718 | + int host_mapped = 0; | |
| 3719 | + | |
| 3720 | + AVVkFrame *hwf_vk = (AVVkFrame *)hwf->data[0]; | |
| 3721 | + VkBufferImageCopy region[AV_NUM_DATA_POINTERS]; // always one per plane | |
| 3569 | 3722 | |
| 3570 | − uint32_t p_w, p_h; | |
| 3571 | 3723 | const int planes = av_pix_fmt_count_planes(swf->format); |
| 3724 | + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swf->format); | |
| 3725 | + const int nb_images = ff_vk_count_images(hwf_vk); | |
| 3726 | + static const VkImageAspectFlags plane_aspect[] = { VK_IMAGE_ASPECT_COLOR_BIT, | |
| 3727 | + VK_IMAGE_ASPECT_PLANE_0_BIT, | |
| 3728 | + VK_IMAGE_ASPECT_PLANE_1_BIT, | |
| 3729 | + VK_IMAGE_ASPECT_PLANE_2_BIT, }; | |
| 3730 | + | |
| 3731 | + VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS]; | |
| 3732 | + int nb_img_bar = 0; | |
| 3572 | 3733 | |
| 3573 | − int host_mapped[AV_NUM_DATA_POINTERS] = { 0 }; | |
| 3574 | − const int map_host = !!(p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY); | |
| 3734 | + AVBufferRef *bufs[AV_NUM_DATA_POINTERS]; | |
| 3735 | + int nb_bufs = 0; | |
| 3575 | 3736 | |
| 3737 | + VkCommandBuffer cmd_buf; | |
| 3738 | + FFVkExecContext *exec; | |
| 3739 | + | |
| 3740 | + /* Sanity checking */ | |
| 3576 | 3741 | if ((swf->format != AV_PIX_FMT_NONE && !av_vkfmt_from_pixfmt(swf->format))) { |
| 3577 | 3742 | av_log(hwfc, AV_LOG_ERROR, "Unsupported software frame pixel format!\n"); |
| 3578 | 3743 | return AVERROR(EINVAL); |
| @@ -3581,115 +3746,97 @@ static int vulkan_transfer_data(AVHWFramesContext *hwfc, const AVFrame *vkf, · 2 unmodified lines | ||
| 3581 | 3746 | if (swf->width > hwfc->width || swf->height > hwfc->height) |
| 3582 | 3747 | return AVERROR(EINVAL); |
| 3583 | 3748 | |
| 3584 | − /* Create buffers */ | |
| 3585 | − for (int i = 0; i < planes; i++) { | |
| 3586 | − size_t req_size; | |
| 3587 | − | |
| 3588 | − VkExternalMemoryBufferCreateInfo create_desc = { | |
| 3589 | − .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, | |
| 3590 | − .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, | |
| 3591 | − }; | |
| 3592 | − | |
| 3593 | − VkImportMemoryHostPointerInfoEXT import_desc = { | |
| 3594 | − .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT, | |
| 3595 | − .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, | |
| 3596 | − }; | |
| 3597 | − | |
| 3598 | − VkMemoryHostPointerPropertiesEXT p_props = { | |
| 3599 | − .sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT, | |
| 3600 | − }; | |
| 3601 | − | |
| 3602 | − get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i); | |
| 3603 | − | |
| 3604 | − tmp.linesize[i] = FFABS(swf->linesize[i]); | |
| 3605 | − | |
| 3606 | − /* Do not map images with a negative stride */ | |
| 3607 | − if (map_host && swf->linesize[i] > 0) { | |
| 3608 | − size_t offs; | |
| 3609 | − offs = (uintptr_t)swf->data[i] % p->hprops.minImportedHostPointerAlignment; | |
| 3610 | − import_desc.pHostPointer = swf->data[i] - offs; | |
| 3611 | − | |
| 3612 | − /* We have to compensate for the few extra bytes of padding we | |
| 3613 | − * completely ignore at the start */ | |
| 3614 | − req_size = FFALIGN(offs + tmp.linesize[i] * p_h, | |
| 3615 | − p->hprops.minImportedHostPointerAlignment); | |
| 3616 | − | |
| 3617 | − ret = vk->GetMemoryHostPointerPropertiesEXT(hwctx->act_dev, | |
| 3618 | − import_desc.handleType, | |
| 3619 | − import_desc.pHostPointer, | |
| 3620 | − &p_props); | |
| 3621 | − if (ret == VK_SUCCESS && p_props.memoryTypeBits) { | |
| 3622 | − host_mapped[i] = 1; | |
| 3623 | − buf_offsets[i] = offs; | |
| 3624 | − } | |
| 3625 | − } | |
| 3749 | + /* Setup buffers first */ | |
| 3750 | + if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY) { | |
| 3751 | + err = host_map_frame(hwfc, bufs, &nb_bufs, swf, region, upload); | |
| 3752 | + if (err >= 0) | |
| 3753 | + host_mapped = 1; | |
| 3754 | + } | |
| 3626 | 3755 | |
| 3627 | − if (!host_mapped[i]) | |
| 3628 | − req_size = get_req_buffer_size(p, &tmp.linesize[i], p_h); | |
| 3629 | − | |
| 3630 | − err = ff_vk_create_avbuf(&p->vkctx, &bufs[i], req_size, | |
| 3631 | − host_mapped[i] ? &create_desc : NULL, | |
| 3632 | − host_mapped[i] ? &import_desc : NULL, | |
| 3633 | − from ? VK_BUFFER_USAGE_TRANSFER_DST_BIT : | |
| 3634 | − VK_BUFFER_USAGE_TRANSFER_SRC_BIT, | |
| 3635 | − VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | | |
| 3636 | − (host_mapped[i] ? | |
| 3637 | − VK_MEMORY_PROPERTY_HOST_COHERENT_BIT : 0x0)); | |
| 3756 | + if (!host_mapped) { | |
| 3757 | + err = get_plane_buf(hwfc, &bufs[0], swf, region, upload); | |
| 3638 | 3758 | if (err < 0) |
| 3639 | 3759 | goto end; |
| 3760 | + nb_bufs = 1; | |
| 3640 | 3761 | |
| 3641 | − vkbufs[i] = (FFVkBuffer *)bufs[i]->data; | |
| 3762 | + if (upload) { | |
| 3763 | + err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 1); | |
| 3764 | + if (err < 0) | |
| 3765 | + goto end; | |
| 3766 | + } | |
| 3642 | 3767 | } |
| 3643 | 3768 | |
| 3644 | − if (!from) { | |
| 3645 | − /* Map, copy image TO buffer (which then goes to the VkImage), unmap */ | |
| 3646 | − if ((err = ff_vk_map_buffers(&p->vkctx, vkbufs, tmp.data, planes, 0))) | |
| 3647 | − goto end; | |
| 3648 | − | |
| 3649 | − for (int i = 0; i < planes; i++) { | |
| 3650 | − if (host_mapped[i]) | |
| 3651 | − continue; | |
| 3769 | + exec = ff_vk_exec_get(&fp->upload_exec); | |
| 3770 | + cmd_buf = exec->buf; | |
| 3652 | 3771 | |
| 3653 | − get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i); | |
| 3772 | + ff_vk_exec_start(&p->vkctx, exec); | |
| 3654 | 3773 | |
| 3655 | − av_image_copy_plane(tmp.data[i], tmp.linesize[i], | |
| 3656 | − (const uint8_t *)swf->data[i], swf->linesize[i], | |
| 3657 | − FFMIN(tmp.linesize[i], FFABS(swf->linesize[i])), | |
| 3658 | − p_h); | |
| 3659 | − } | |
| 3774 | + /* Prep destination Vulkan frame */ | |
| 3775 | + err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, hwf, | |
| 3776 | + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, | |
| 3777 | + VK_PIPELINE_STAGE_2_TRANSFER_BIT); | |
| 3778 | + if (err < 0) | |
| 3779 | + goto end; | |
| 3660 | 3780 | |
| 3661 | − if ((err = ff_vk_unmap_buffers(&p->vkctx, vkbufs, planes, 1))) | |
| 3781 | + /* No need to declare buf deps for synchronous transfers */ | |
| 3782 | + if (upload) { | |
| 3783 | + err = ff_vk_exec_add_dep_buf(&p->vkctx, exec, bufs, nb_bufs, 1); | |
| 3784 | + if (err < 0) { | |
| 3785 | + ff_vk_exec_discard_deps(&p->vkctx, exec); | |
| 3662 | 3786 | goto end; |
| 3787 | + } | |
| 3663 | 3788 | } |
| 3664 | 3789 | |
| 3665 | − /* Copy buffers into/from image */ | |
| 3666 | − err = transfer_image_buf(hwfc, (AVFrame *)vkf, bufs, buf_offsets, | |
| 3667 | − tmp.linesize, swf->width, swf->height, swf->format, | |
| 3668 | − from); | |
| 3669 | − | |
| 3670 | − if (from) { | |
| 3671 | − /* Map, copy buffer (which came FROM the VkImage) to the frame, unmap */ | |
| 3672 | − if ((err = ff_vk_map_buffers(&p->vkctx, vkbufs, tmp.data, planes, 0))) | |
| 3673 | − goto end; | |
| 3790 | + ff_vk_frame_barrier(&p->vkctx, exec, hwf, img_bar, &nb_img_bar, | |
| 3791 | + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, | |
| 3792 | + VK_PIPELINE_STAGE_2_TRANSFER_BIT_KHR, | |
| 3793 | + upload ? VK_ACCESS_TRANSFER_WRITE_BIT : | |
| 3794 | + VK_ACCESS_TRANSFER_READ_BIT, | |
| 3795 | + upload ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL : | |
| 3796 | + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, | |
| 3797 | + VK_QUEUE_FAMILY_IGNORED); | |
| 3674 | 3798 | |
| 3675 | − for (int i = 0; i < planes; i++) { | |
| 3676 | − if (host_mapped[i]) | |
| 3677 | − continue; | |
| 3799 | + vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) { | |
| 3800 | + .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, | |
| 3801 | + .pImageMemoryBarriers = img_bar, | |
| 3802 | + .imageMemoryBarrierCount = nb_img_bar, | |
| 3803 | + }); | |
| 3678 | 3804 | |
| 3679 | − get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i); | |
| 3805 | + for (int i = 0; i < planes; i++) { | |
| 3806 | + int buf_idx = FFMIN(i, (nb_bufs - 1)); | |
| 3807 | + int img_idx = FFMIN(i, (nb_images - 1)); | |
| 3808 | + FFVkBuffer *vkbuf = (FFVkBuffer *)bufs[buf_idx]->data; | |
| 3809 | + | |
| 3810 | + uint32_t orig_stride = region[i].bufferRowLength; | |
| 3811 | + region[i].bufferRowLength /= desc->comp[i].step; | |
| 3812 | + region[i].imageSubresource.aspectMask = plane_aspect[(planes != nb_images) + | |
| 3813 | + i*(planes != nb_images)]; | |
| 3814 | + | |
| 3815 | + if (upload) | |
| 3816 | + vk->CmdCopyBufferToImage(cmd_buf, vkbuf->buf, | |
| 3817 | + hwf_vk->img[img_idx], | |
| 3818 | + img_bar[img_idx].newLayout, | |
| 3819 | + 1, ®ion[i]); | |
| 3820 | + else | |
| 3821 | + vk->CmdCopyImageToBuffer(cmd_buf, hwf_vk->img[img_idx], | |
| 3822 | + img_bar[img_idx].newLayout, | |
| 3823 | + vkbuf->buf, | |
| 3824 | + 1, ®ion[i]); | |
| 3680 | 3825 | |
| 3681 | − av_image_copy_plane_uc_from(swf->data[i], swf->linesize[i], | |
| 3682 | − (const uint8_t *)tmp.data[i], tmp.linesize[i], | |
| 3683 | − FFMIN(tmp.linesize[i], FFABS(swf->linesize[i])), | |
| 3684 | − p_h); | |
| 3685 | − } | |
| 3826 | + region[i].bufferRowLength = orig_stride; | |
| 3827 | + } | |
| 3686 | 3828 | |
| 3687 | − if ((err = ff_vk_unmap_buffers(&p->vkctx, vkbufs, planes, 1))) | |
| 3688 | − goto end; | |
| 3829 | + err = ff_vk_exec_submit(&p->vkctx, exec); | |
| 3830 | + if (err < 0) { | |
| 3831 | + ff_vk_exec_discard_deps(&p->vkctx, exec); | |
| 3832 | + } else if (!upload) { | |
| 3833 | + ff_vk_exec_wait(&p->vkctx, exec); | |
| 3834 | + if (!host_mapped) | |
| 3835 | + err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 0); | |
| 3689 | 3836 | } |
| 3690 | 3837 | |
| 3691 | 3838 | end: |
| 3692 | − for (int i = 0; i < planes; i++) | |
| 3839 | + for (int i = 0; i < nb_bufs; i++) | |
| 3693 | 3840 | av_buffer_unref(&bufs[i]); |
| 3694 | 3841 | |
| 3695 | 3842 | return err; |
| @@ -3716,7 +3863,7 @@ static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst, · 20 unmodified lines | ||
| 3716 | 3863 | if (src->hw_frames_ctx) |
| 3717 | 3864 | return AVERROR(ENOSYS); |
| 3718 | 3865 | else |
| 3719 | − return vulkan_transfer_data(hwfc, dst, src, 0); | |
| 3866 | + return vulkan_transfer_frame(hwfc, (AVFrame *)src, dst, 1); | |
| 3720 | 3867 | } |
| 3721 | 3868 | } |
| 3722 | 3869 | |
| @@ -3833,7 +3980,7 @@ static int vulkan_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst, · 110 unmodified lines | ||
| 3833 | 3980 | if (dst->hw_frames_ctx) |
| 3834 | 3981 | return AVERROR(ENOSYS); |
| 3835 | 3982 | else |
| 3836 | − return vulkan_transfer_data(hwfc, src, dst, 1); | |
| 3983 | + return vulkan_transfer_frame(hwfc, dst, (AVFrame *)src, 0); | |
| 3837 | 3984 | } |
| 3838 | 3985 | } |
| 3839 | 3986 | |
| @@ -809,11 +809,8 @@ int ff_vk_alloc_mem(FFVulkanContext *s, VkMemoryRequirements *req, · 808 unmodified lines | ||
| 809 | 809 | |
| 810 | 810 | ret = vk->AllocateMemory(s->hwctx->act_dev, &alloc_info, |
| 811 | 811 | s->hwctx->alloc, mem); |
| 812 | − if (ret != VK_SUCCESS) { | |
| 813 | − av_log(s, AV_LOG_ERROR, "Failed to allocate memory: %s\n", | |
| 814 | − ff_vk_ret2str(ret)); | |
| 812 | + if (ret != VK_SUCCESS) | |
| 815 | 813 | return AVERROR(ENOMEM); |
| 816 | − } | |
| 817 | 814 | |
| 818 | 815 | if (mem_flags) |
| 819 | 816 | *mem_flags |= s->mprops.memoryTypes[index].propertyFlags; |