History

hwcontext_vulkan: add support for implicit DRM synchronization

More recent kernel versions allow for users to extract a sync_file handle from a DMA-BUF, which can then be imported into Vulkan as a binary semaphore. This finally allows for synchronization between Vulkan and DMA-BUF images, such as those from screen capture software, or VAAPI, avoiding any corruption artifacts. This is done fully asynchronously, where we use the kernel's given binary semaphores as a dependency to increment the image's usual VkSemaphores we allocate. The old imported binary semaphores are cleaned up after execution as usual. In the future, hwcontext_drm should receive support for explicitly synchronized images as well, which would make the synchronization more robust and portable.

Lynne committed 2 years ago

Commit a577d311 parent 23954442 files · +138 −11Browse files

Unified · Split

2 files
  1. libavutil/hwcontext_vulkan.c +137 −11
  2. libavutil/vulkan_functions.h +1 −0
libavutil/hwcontext_vulkan.c+137−11
@@ -61,6 +61,11 @@ · 60 unmodified lines
61#include "hwcontext_drm.h"61#include "hwcontext_drm.h"
62#endif62#endif
63 63
64#if HAVE_LINUX_DMA_BUF_H
65#include <sys/ioctl.h>
66#include <linux/dma-buf.h>
67#endif
68
64#if CONFIG_CUDA69#if CONFIG_CUDA
65#include "hwcontext_cuda_internal.h"70#include "hwcontext_cuda_internal.h"
66#include "cuda_check.h"71#include "cuda_check.h"
@@ -2842,7 +2847,6 @@ static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **f · 2775 unmodified lines
2842 VulkanDevicePriv *p = ctx->hwctx;2847 VulkanDevicePriv *p = ctx->hwctx;
2843 AVVulkanDeviceContext *hwctx = &p->p;2848 AVVulkanDeviceContext *hwctx = &p->p;
2844 FFVulkanFunctions *vk = &p->vkctx.vkfn;2849 FFVulkanFunctions *vk = &p->vkctx.vkfn;
2845 VulkanFramesPriv *fp = hwfc->hwctx;
2846 const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];2850 const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
2847 VkBindImageMemoryInfo bind_info[AV_DRM_MAX_PLANES];2851 VkBindImageMemoryInfo bind_info[AV_DRM_MAX_PLANES];
2848 VkBindImagePlaneMemoryInfo plane_info[AV_DRM_MAX_PLANES];2852 VkBindImagePlaneMemoryInfo plane_info[AV_DRM_MAX_PLANES];
@@ -2990,12 +2994,7 @@ static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **f · 141 unmodified lines
2990 goto fail;2994 goto fail;
2991 }2995 }
2992 2996
2993 /* We'd import a semaphore onto the one we created using2997 f->queue_family[i] = VK_QUEUE_FAMILY_EXTERNAL;
2994 * vkImportSemaphoreFdKHR but unfortunately neither DRM nor VAAPI
2995 * offer us anything we could import and sync with, so instead
2996 * just signal the semaphore we created. */
2997
2998 f->queue_family[i] = p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0];
2999 f->layout[i] = create_info.initialLayout;2998 f->layout[i] = create_info.initialLayout;
3000 f->access[i] = 0x0;2999 f->access[i] = 0x0;
3001 f->sem_value[i] = 0;3000 f->sem_value[i] = 0;
@@ -3097,10 +3096,6 @@ static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **f · 95 unmodified lines
3097 goto fail;3096 goto fail;
3098 }3097 }
3099 3098
3100 err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_IMPORT);
3101 if (err)
3102 goto fail;
3103
3104 *frame = f;3099 *frame = f;
3105 3100
3106 return 0;3101 return 0;
@@ -3111,6 +3106,133 @@ fail: · 4 unmodified lines
3111 return err;3106 return err;
3112}3107}
3113 3108
3109static int vulkan_map_from_drm_frame_sync(AVHWFramesContext *hwfc, AVFrame *dst,
3110 const AVFrame *src, int flags)
3111{
3112 int err;
3113 VkResult ret;
3114 AVHWDeviceContext *ctx = hwfc->device_ctx;
3115 VulkanDevicePriv *p = ctx->hwctx;
3116 VulkanFramesPriv *fp = hwfc->hwctx;
3117 AVVulkanDeviceContext *hwctx = &p->p;
3118 FFVulkanFunctions *vk = &p->vkctx.vkfn;
3119
3120 const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
3121
3122#ifdef DMA_BUF_IOCTL_EXPORT_SYNC_FILE
3123 if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM) {
3124 VkCommandBuffer cmd_buf;
3125 FFVkExecContext *exec;
3126 VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
3127 VkSemaphore drm_sync_sem[AV_DRM_MAX_PLANES] = { 0 };
3128 int nb_img_bar = 0;
3129
3130 for (int i = 0; i < desc->nb_objects; i++) {
3131 VkSemaphoreTypeCreateInfo sem_type_info = {
3132 .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
3133 .semaphoreType = VK_SEMAPHORE_TYPE_BINARY,
3134 };
3135 VkSemaphoreCreateInfo sem_spawn = {
3136 .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3137 .pNext = &sem_type_info,
3138 };
3139 VkImportSemaphoreFdInfoKHR import_info;
3140 struct dma_buf_export_sync_file implicit_fd_info = {
3141 .flags = DMA_BUF_SYNC_READ,
3142 .fd = -1,
3143 };
3144
3145 if (ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_EXPORT_SYNC_FILE,
3146 &implicit_fd_info)) {
3147 err = AVERROR(errno);
3148 av_log(hwctx, AV_LOG_ERROR, "Failed to retrieve implicit DRM sync file: %s\n",
3149 av_err2str(err));
3150 for (; i >= 0; i--)
3151 vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3152 return err;
3153 }
3154
3155 ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3156 hwctx->alloc, &drm_sync_sem[i]);
3157 if (ret != VK_SUCCESS) {
3158 av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3159 ff_vk_ret2str(ret));
3160 err = AVERROR_EXTERNAL;
3161 for (; i >= 0; i--)
3162 vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3163 return err;
3164 }
3165
3166 import_info = (VkImportSemaphoreFdInfoKHR) {
3167 .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR,
3168 .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
3169 .flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT,
3170 .semaphore = drm_sync_sem[i],
3171 .fd = implicit_fd_info.fd,
3172 };
3173
3174 ret = vk->ImportSemaphoreFdKHR(hwctx->act_dev, &import_info);
3175 if (ret != VK_SUCCESS) {
3176 av_log(hwctx, AV_LOG_ERROR, "Failed to import semaphore: %s\n",
3177 ff_vk_ret2str(ret));
3178 err = AVERROR_EXTERNAL;
3179 for (; i >= 0; i--)
3180 vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3181 return err;
3182 }
3183 }
3184
3185 exec = ff_vk_exec_get(&fp->compute_exec);
3186 cmd_buf = exec->buf;
3187
3188 ff_vk_exec_start(&p->vkctx, exec);
3189
3190 /* Ownership of semaphores is passed */
3191 err = ff_vk_exec_add_dep_bool_sem(&p->vkctx, exec,
3192 drm_sync_sem, desc->nb_objects,
3193 VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 1);
3194 if (err < 0)
3195 return err;
3196
3197 err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, dst,
3198 VK_PIPELINE_STAGE_2_NONE,
3199 VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
3200 if (err < 0)
3201 return err;
3202
3203 ff_vk_frame_barrier(&p->vkctx, exec, dst, img_bar, &nb_img_bar,
3204 VK_PIPELINE_STAGE_2_NONE,
3205 VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
3206 ((flags & AV_HWFRAME_MAP_READ) ?
3207 VK_ACCESS_2_SHADER_SAMPLED_READ_BIT : 0x0) |
3208 ((flags & AV_HWFRAME_MAP_WRITE) ?
3209 VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT : 0x0),
3210 VK_IMAGE_LAYOUT_GENERAL,
3211 VK_QUEUE_FAMILY_IGNORED);
3212
3213 vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
3214 .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
3215 .pImageMemoryBarriers = img_bar,
3216 .imageMemoryBarrierCount = nb_img_bar,
3217 });
3218
3219 err = ff_vk_exec_submit(&p->vkctx, exec);
3220 if (err < 0)
3221 return err;
3222 } else
3223#endif
3224 {
3225 AVVkFrame *f = (AVVkFrame *)dst->data[0];
3226 av_log(hwctx, AV_LOG_WARNING, "No support for synchronization when importing DMA-BUFs, "
3227 "image may be corrupted.\n");
3228 err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_IMPORT);
3229 if (err)
3230 return err;
3231 }
3232
3233 return 0;
3234}
3235
3114static int vulkan_map_from_drm(AVHWFramesContext *hwfc, AVFrame *dst,3236static int vulkan_map_from_drm(AVHWFramesContext *hwfc, AVFrame *dst,
3115 const AVFrame *src, int flags)3237 const AVFrame *src, int flags)
3116{3238{
@@ -3130,6 +3252,10 @@ static int vulkan_map_from_drm(AVHWFramesContext *hwfc, AVFrame *dst, · 13 unmodified lines
3130 if (err < 0)3252 if (err < 0)
3131 goto fail;3253 goto fail;
3132 3254
3255 err = vulkan_map_from_drm_frame_sync(hwfc, dst, src, flags);
3256 if (err < 0)
3257 return err;
3258
3133 av_log(hwfc, AV_LOG_DEBUG, "Mapped DRM object to Vulkan!\n");3259 av_log(hwfc, AV_LOG_DEBUG, "Mapped DRM object to Vulkan!\n");
3134 3260
3135 return 0;3261 return 0;
libavutil/vulkan_functions.h+1−0
@@ -114,6 +114,7 @@ typedef enum FFVulkanExtensions { · 113 unmodified lines
114 \114 \
115 /* Semaphores */ \115 /* Semaphores */ \
116 MACRO(1, 1, FF_VK_EXT_EXTERNAL_FD_SEM, GetSemaphoreFdKHR) \116 MACRO(1, 1, FF_VK_EXT_EXTERNAL_FD_SEM, GetSemaphoreFdKHR) \
117 MACRO(1, 1, FF_VK_EXT_EXTERNAL_FD_SEM, ImportSemaphoreFdKHR) \
117 MACRO(1, 1, FF_VK_EXT_NO_FLAG, CreateSemaphore) \118 MACRO(1, 1, FF_VK_EXT_NO_FLAG, CreateSemaphore) \
118 MACRO(1, 1, FF_VK_EXT_NO_FLAG, WaitSemaphores) \119 MACRO(1, 1, FF_VK_EXT_NO_FLAG, WaitSemaphores) \
119 MACRO(1, 1, FF_VK_EXT_NO_FLAG, DestroySemaphore) \120 MACRO(1, 1, FF_VK_EXT_NO_FLAG, DestroySemaphore) \

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t