History

hwcontext_vulkan: add support for implicit DRM synchronization

More recent kernel versions allow for users to extract a sync_file handle from a DMA-BUF, which can then be imported into Vulkan as a binary semaphore. This finally allows for synchronization between Vulkan and DMA-BUF images, such as those from screen capture software, or VAAPI, avoiding any corruption artifacts. This is done fully asynchronously, where we use the kernel's given binary semaphores as a dependency to increment the image's usual VkSemaphores we allocate. The old imported binary semaphores are cleaned up after execution as usual. In the future, hwcontext_drm should receive support for explicitly synchronized images as well, which would make the synchronization more robust and portable.

Lynne committed 2 years ago

Commit a577d311 parent 23954442 files · +138 −11Browse files

Unified · Split

2 files
  1. libavutil/hwcontext_vulkan.c +137 −11
  2. libavutil/vulkan_functions.h +1 −0
libavutil/hwcontext_vulkan.c+137−11
@@ -61,6 +61,11 @@ · 60 unmodified lines
6161 #include "hwcontext_drm.h"
6262 #endif
6363
64+#if HAVE_LINUX_DMA_BUF_H
65+#include <sys/ioctl.h>
66+#include <linux/dma-buf.h>
67+#endif
68+
6469 #if CONFIG_CUDA
6570 #include "hwcontext_cuda_internal.h"
6671 #include "cuda_check.h"
@@ -2842,7 +2847,6 @@ static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **f · 2775 unmodified lines
28422847 VulkanDevicePriv *p = ctx->hwctx;
28432848 AVVulkanDeviceContext *hwctx = &p->p;
28442849 FFVulkanFunctions *vk = &p->vkctx.vkfn;
2845− VulkanFramesPriv *fp = hwfc->hwctx;
28462850 const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
28472851 VkBindImageMemoryInfo bind_info[AV_DRM_MAX_PLANES];
28482852 VkBindImagePlaneMemoryInfo plane_info[AV_DRM_MAX_PLANES];
@@ -2990,12 +2994,7 @@ static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **f · 141 unmodified lines
29902994 goto fail;
29912995 }
29922996
2993− /* We'd import a semaphore onto the one we created using
2994− * vkImportSemaphoreFdKHR but unfortunately neither DRM nor VAAPI
2995− * offer us anything we could import and sync with, so instead
2996− * just signal the semaphore we created. */
2997−
2998− f->queue_family[i] = p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0];
2997+ f->queue_family[i] = VK_QUEUE_FAMILY_EXTERNAL;
29992998 f->layout[i] = create_info.initialLayout;
30002999 f->access[i] = 0x0;
30013000 f->sem_value[i] = 0;
@@ -3097,10 +3096,6 @@ static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **f · 95 unmodified lines
30973096 goto fail;
30983097 }
30993098
3100− err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_IMPORT);
3101− if (err)
3102− goto fail;
3103−
31043099 *frame = f;
31053100
31063101 return 0;
@@ -3111,6 +3106,133 @@ fail: · 4 unmodified lines
31113106 return err;
31123107 }
31133108
3109+static int vulkan_map_from_drm_frame_sync(AVHWFramesContext *hwfc, AVFrame *dst,
3110+ const AVFrame *src, int flags)
3111+{
3112+ int err;
3113+ VkResult ret;
3114+ AVHWDeviceContext *ctx = hwfc->device_ctx;
3115+ VulkanDevicePriv *p = ctx->hwctx;
3116+ VulkanFramesPriv *fp = hwfc->hwctx;
3117+ AVVulkanDeviceContext *hwctx = &p->p;
3118+ FFVulkanFunctions *vk = &p->vkctx.vkfn;
3119+
3120+ const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
3121+
3122+#ifdef DMA_BUF_IOCTL_EXPORT_SYNC_FILE
3123+ if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM) {
3124+ VkCommandBuffer cmd_buf;
3125+ FFVkExecContext *exec;
3126+ VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
3127+ VkSemaphore drm_sync_sem[AV_DRM_MAX_PLANES] = { 0 };
3128+ int nb_img_bar = 0;
3129+
3130+ for (int i = 0; i < desc->nb_objects; i++) {
3131+ VkSemaphoreTypeCreateInfo sem_type_info = {
3132+ .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
3133+ .semaphoreType = VK_SEMAPHORE_TYPE_BINARY,
3134+ };
3135+ VkSemaphoreCreateInfo sem_spawn = {
3136+ .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3137+ .pNext = &sem_type_info,
3138+ };
3139+ VkImportSemaphoreFdInfoKHR import_info;
3140+ struct dma_buf_export_sync_file implicit_fd_info = {
3141+ .flags = DMA_BUF_SYNC_READ,
3142+ .fd = -1,
3143+ };
3144+
3145+ if (ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_EXPORT_SYNC_FILE,
3146+ &implicit_fd_info)) {
3147+ err = AVERROR(errno);
3148+ av_log(hwctx, AV_LOG_ERROR, "Failed to retrieve implicit DRM sync file: %s\n",
3149+ av_err2str(err));
3150+ for (; i >= 0; i--)
3151+ vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3152+ return err;
3153+ }
3154+
3155+ ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3156+ hwctx->alloc, &drm_sync_sem[i]);
3157+ if (ret != VK_SUCCESS) {
3158+ av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3159+ ff_vk_ret2str(ret));
3160+ err = AVERROR_EXTERNAL;
3161+ for (; i >= 0; i--)
3162+ vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3163+ return err;
3164+ }
3165+
3166+ import_info = (VkImportSemaphoreFdInfoKHR) {
3167+ .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR,
3168+ .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
3169+ .flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT,
3170+ .semaphore = drm_sync_sem[i],
3171+ .fd = implicit_fd_info.fd,
3172+ };
3173+
3174+ ret = vk->ImportSemaphoreFdKHR(hwctx->act_dev, &import_info);
3175+ if (ret != VK_SUCCESS) {
3176+ av_log(hwctx, AV_LOG_ERROR, "Failed to import semaphore: %s\n",
3177+ ff_vk_ret2str(ret));
3178+ err = AVERROR_EXTERNAL;
3179+ for (; i >= 0; i--)
3180+ vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3181+ return err;
3182+ }
3183+ }
3184+
3185+ exec = ff_vk_exec_get(&fp->compute_exec);
3186+ cmd_buf = exec->buf;
3187+
3188+ ff_vk_exec_start(&p->vkctx, exec);
3189+
3190+ /* Ownership of semaphores is passed */
3191+ err = ff_vk_exec_add_dep_bool_sem(&p->vkctx, exec,
3192+ drm_sync_sem, desc->nb_objects,
3193+ VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 1);
3194+ if (err < 0)
3195+ return err;
3196+
3197+ err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, dst,
3198+ VK_PIPELINE_STAGE_2_NONE,
3199+ VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
3200+ if (err < 0)
3201+ return err;
3202+
3203+ ff_vk_frame_barrier(&p->vkctx, exec, dst, img_bar, &nb_img_bar,
3204+ VK_PIPELINE_STAGE_2_NONE,
3205+ VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
3206+ ((flags & AV_HWFRAME_MAP_READ) ?
3207+ VK_ACCESS_2_SHADER_SAMPLED_READ_BIT : 0x0) |
3208+ ((flags & AV_HWFRAME_MAP_WRITE) ?
3209+ VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT : 0x0),
3210+ VK_IMAGE_LAYOUT_GENERAL,
3211+ VK_QUEUE_FAMILY_IGNORED);
3212+
3213+ vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
3214+ .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
3215+ .pImageMemoryBarriers = img_bar,
3216+ .imageMemoryBarrierCount = nb_img_bar,
3217+ });
3218+
3219+ err = ff_vk_exec_submit(&p->vkctx, exec);
3220+ if (err < 0)
3221+ return err;
3222+ } else
3223+#endif
3224+ {
3225+ AVVkFrame *f = (AVVkFrame *)dst->data[0];
3226+ av_log(hwctx, AV_LOG_WARNING, "No support for synchronization when importing DMA-BUFs, "
3227+ "image may be corrupted.\n");
3228+ err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_IMPORT);
3229+ if (err)
3230+ return err;
3231+ }
3232+
3233+ return 0;
3234+}
3235+
31143236 static int vulkan_map_from_drm(AVHWFramesContext *hwfc, AVFrame *dst,
31153237 const AVFrame *src, int flags)
31163238 {
@@ -3130,6 +3252,10 @@ static int vulkan_map_from_drm(AVHWFramesContext *hwfc, AVFrame *dst, · 13 unmodified lines
31303252 if (err < 0)
31313253 goto fail;
31323254
3255+ err = vulkan_map_from_drm_frame_sync(hwfc, dst, src, flags);
3256+ if (err < 0)
3257+ return err;
3258+
31333259 av_log(hwfc, AV_LOG_DEBUG, "Mapped DRM object to Vulkan!\n");
31343260
31353261 return 0;
libavutil/vulkan_functions.h+1−0
@@ -114,6 +114,7 @@ typedef enum FFVulkanExtensions { · 113 unmodified lines
114114 \
115115 /* Semaphores */ \
116116 MACRO(1, 1, FF_VK_EXT_EXTERNAL_FD_SEM, GetSemaphoreFdKHR) \
117+ MACRO(1, 1, FF_VK_EXT_EXTERNAL_FD_SEM, ImportSemaphoreFdKHR) \
117118 MACRO(1, 1, FF_VK_EXT_NO_FLAG, CreateSemaphore) \
118119 MACRO(1, 1, FF_VK_EXT_NO_FLAG, WaitSemaphores) \
119120 MACRO(1, 1, FF_VK_EXT_NO_FLAG, DestroySemaphore) \

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t