More recent kernel versions allow for users to extract a sync_file handle from a DMA-BUF, which can then be imported into Vulkan as a binary semaphore. This finally allows for synchronization between Vulkan and DMA-BUF images, such as those from screen capture software, or VAAPI, avoiding any corruption artifacts. This is done fully asynchronously, where we use the kernel's given binary semaphores as a dependency to increment the image's usual VkSemaphores we allocate. The old imported binary semaphores are cleaned up after execution as usual. In the future, hwcontext_drm should receive support for explicitly synchronized images as well, which would make the synchronization more robust and portable.
Unified · Split
| @@ -61,6 +61,11 @@ · 60 unmodified lines | ||
| 61 | 61 | #include "hwcontext_drm.h" |
| 62 | 62 | #endif |
| 63 | 63 | |
| 64 | +#if HAVE_LINUX_DMA_BUF_H | |
| 65 | +#include <sys/ioctl.h> | |
| 66 | +#include <linux/dma-buf.h> | |
| 67 | +#endif | |
| 68 | + | |
| 64 | 69 | #if CONFIG_CUDA |
| 65 | 70 | #include "hwcontext_cuda_internal.h" |
| 66 | 71 | #include "cuda_check.h" |
| @@ -2842,7 +2847,6 @@ static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **f · 2775 unmodified lines | ||
| 2842 | 2847 | VulkanDevicePriv *p = ctx->hwctx; |
| 2843 | 2848 | AVVulkanDeviceContext *hwctx = &p->p; |
| 2844 | 2849 | FFVulkanFunctions *vk = &p->vkctx.vkfn; |
| 2845 | − VulkanFramesPriv *fp = hwfc->hwctx; | |
| 2846 | 2850 | const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0]; |
| 2847 | 2851 | VkBindImageMemoryInfo bind_info[AV_DRM_MAX_PLANES]; |
| 2848 | 2852 | VkBindImagePlaneMemoryInfo plane_info[AV_DRM_MAX_PLANES]; |
| @@ -2990,12 +2994,7 @@ static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **f · 141 unmodified lines | ||
| 2990 | 2994 | goto fail; |
| 2991 | 2995 | } |
| 2992 | 2996 | |
| 2993 | − /* We'd import a semaphore onto the one we created using | |
| 2994 | − * vkImportSemaphoreFdKHR but unfortunately neither DRM nor VAAPI | |
| 2995 | − * offer us anything we could import and sync with, so instead | |
| 2996 | − * just signal the semaphore we created. */ | |
| 2997 | − | |
| 2998 | − f->queue_family[i] = p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0]; | |
| 2997 | + f->queue_family[i] = VK_QUEUE_FAMILY_EXTERNAL; | |
| 2999 | 2998 | f->layout[i] = create_info.initialLayout; |
| 3000 | 2999 | f->access[i] = 0x0; |
| 3001 | 3000 | f->sem_value[i] = 0; |
| @@ -3097,10 +3096,6 @@ static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **f · 95 unmodified lines | ||
| 3097 | 3096 | goto fail; |
| 3098 | 3097 | } |
| 3099 | 3098 | |
| 3100 | − err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_IMPORT); | |
| 3101 | − if (err) | |
| 3102 | − goto fail; | |
| 3103 | − | |
| 3104 | 3099 | *frame = f; |
| 3105 | 3100 | |
| 3106 | 3101 | return 0; |
| @@ -3111,6 +3106,133 @@ fail: · 4 unmodified lines | ||
| 3111 | 3106 | return err; |
| 3112 | 3107 | } |
| 3113 | 3108 | |
| 3109 | +static int vulkan_map_from_drm_frame_sync(AVHWFramesContext *hwfc, AVFrame *dst, | |
| 3110 | + const AVFrame *src, int flags) | |
| 3111 | +{ | |
| 3112 | + int err; | |
| 3113 | + VkResult ret; | |
| 3114 | + AVHWDeviceContext *ctx = hwfc->device_ctx; | |
| 3115 | + VulkanDevicePriv *p = ctx->hwctx; | |
| 3116 | + VulkanFramesPriv *fp = hwfc->hwctx; | |
| 3117 | + AVVulkanDeviceContext *hwctx = &p->p; | |
| 3118 | + FFVulkanFunctions *vk = &p->vkctx.vkfn; | |
| 3119 | + | |
| 3120 | + const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0]; | |
| 3121 | + | |
| 3122 | +#ifdef DMA_BUF_IOCTL_EXPORT_SYNC_FILE | |
| 3123 | + if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM) { | |
| 3124 | + VkCommandBuffer cmd_buf; | |
| 3125 | + FFVkExecContext *exec; | |
| 3126 | + VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS]; | |
| 3127 | + VkSemaphore drm_sync_sem[AV_DRM_MAX_PLANES] = { 0 }; | |
| 3128 | + int nb_img_bar = 0; | |
| 3129 | + | |
| 3130 | + for (int i = 0; i < desc->nb_objects; i++) { | |
| 3131 | + VkSemaphoreTypeCreateInfo sem_type_info = { | |
| 3132 | + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO, | |
| 3133 | + .semaphoreType = VK_SEMAPHORE_TYPE_BINARY, | |
| 3134 | + }; | |
| 3135 | + VkSemaphoreCreateInfo sem_spawn = { | |
| 3136 | + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, | |
| 3137 | + .pNext = &sem_type_info, | |
| 3138 | + }; | |
| 3139 | + VkImportSemaphoreFdInfoKHR import_info; | |
| 3140 | + struct dma_buf_export_sync_file implicit_fd_info = { | |
| 3141 | + .flags = DMA_BUF_SYNC_READ, | |
| 3142 | + .fd = -1, | |
| 3143 | + }; | |
| 3144 | + | |
| 3145 | + if (ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_EXPORT_SYNC_FILE, | |
| 3146 | + &implicit_fd_info)) { | |
| 3147 | + err = AVERROR(errno); | |
| 3148 | + av_log(hwctx, AV_LOG_ERROR, "Failed to retrieve implicit DRM sync file: %s\n", | |
| 3149 | + av_err2str(err)); | |
| 3150 | + for (; i >= 0; i--) | |
| 3151 | + vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc); | |
| 3152 | + return err; | |
| 3153 | + } | |
| 3154 | + | |
| 3155 | + ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn, | |
| 3156 | + hwctx->alloc, &drm_sync_sem[i]); | |
| 3157 | + if (ret != VK_SUCCESS) { | |
| 3158 | + av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n", | |
| 3159 | + ff_vk_ret2str(ret)); | |
| 3160 | + err = AVERROR_EXTERNAL; | |
| 3161 | + for (; i >= 0; i--) | |
| 3162 | + vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc); | |
| 3163 | + return err; | |
| 3164 | + } | |
| 3165 | + | |
| 3166 | + import_info = (VkImportSemaphoreFdInfoKHR) { | |
| 3167 | + .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR, | |
| 3168 | + .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT, | |
| 3169 | + .flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT, | |
| 3170 | + .semaphore = drm_sync_sem[i], | |
| 3171 | + .fd = implicit_fd_info.fd, | |
| 3172 | + }; | |
| 3173 | + | |
| 3174 | + ret = vk->ImportSemaphoreFdKHR(hwctx->act_dev, &import_info); | |
| 3175 | + if (ret != VK_SUCCESS) { | |
| 3176 | + av_log(hwctx, AV_LOG_ERROR, "Failed to import semaphore: %s\n", | |
| 3177 | + ff_vk_ret2str(ret)); | |
| 3178 | + err = AVERROR_EXTERNAL; | |
| 3179 | + for (; i >= 0; i--) | |
| 3180 | + vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc); | |
| 3181 | + return err; | |
| 3182 | + } | |
| 3183 | + } | |
| 3184 | + | |
| 3185 | + exec = ff_vk_exec_get(&fp->compute_exec); | |
| 3186 | + cmd_buf = exec->buf; | |
| 3187 | + | |
| 3188 | + ff_vk_exec_start(&p->vkctx, exec); | |
| 3189 | + | |
| 3190 | + /* Ownership of semaphores is passed */ | |
| 3191 | + err = ff_vk_exec_add_dep_bool_sem(&p->vkctx, exec, | |
| 3192 | + drm_sync_sem, desc->nb_objects, | |
| 3193 | + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 1); | |
| 3194 | + if (err < 0) | |
| 3195 | + return err; | |
| 3196 | + | |
| 3197 | + err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, dst, | |
| 3198 | + VK_PIPELINE_STAGE_2_NONE, | |
| 3199 | + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT); | |
| 3200 | + if (err < 0) | |
| 3201 | + return err; | |
| 3202 | + | |
| 3203 | + ff_vk_frame_barrier(&p->vkctx, exec, dst, img_bar, &nb_img_bar, | |
| 3204 | + VK_PIPELINE_STAGE_2_NONE, | |
| 3205 | + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, | |
| 3206 | + ((flags & AV_HWFRAME_MAP_READ) ? | |
| 3207 | + VK_ACCESS_2_SHADER_SAMPLED_READ_BIT : 0x0) | | |
| 3208 | + ((flags & AV_HWFRAME_MAP_WRITE) ? | |
| 3209 | + VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT : 0x0), | |
| 3210 | + VK_IMAGE_LAYOUT_GENERAL, | |
| 3211 | + VK_QUEUE_FAMILY_IGNORED); | |
| 3212 | + | |
| 3213 | + vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) { | |
| 3214 | + .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, | |
| 3215 | + .pImageMemoryBarriers = img_bar, | |
| 3216 | + .imageMemoryBarrierCount = nb_img_bar, | |
| 3217 | + }); | |
| 3218 | + | |
| 3219 | + err = ff_vk_exec_submit(&p->vkctx, exec); | |
| 3220 | + if (err < 0) | |
| 3221 | + return err; | |
| 3222 | + } else | |
| 3223 | +#endif | |
| 3224 | + { | |
| 3225 | + AVVkFrame *f = (AVVkFrame *)dst->data[0]; | |
| 3226 | + av_log(hwctx, AV_LOG_WARNING, "No support for synchronization when importing DMA-BUFs, " | |
| 3227 | + "image may be corrupted.\n"); | |
| 3228 | + err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_IMPORT); | |
| 3229 | + if (err) | |
| 3230 | + return err; | |
| 3231 | + } | |
| 3232 | + | |
| 3233 | + return 0; | |
| 3234 | +} | |
| 3235 | + | |
| 3114 | 3236 | static int vulkan_map_from_drm(AVHWFramesContext *hwfc, AVFrame *dst, |
| 3115 | 3237 | const AVFrame *src, int flags) |
| 3116 | 3238 | { |
| @@ -3130,6 +3252,10 @@ static int vulkan_map_from_drm(AVHWFramesContext *hwfc, AVFrame *dst, · 13 unmodified lines | ||
| 3130 | 3252 | if (err < 0) |
| 3131 | 3253 | goto fail; |
| 3132 | 3254 | |
| 3255 | + err = vulkan_map_from_drm_frame_sync(hwfc, dst, src, flags); | |
| 3256 | + if (err < 0) | |
| 3257 | + return err; | |
| 3258 | + | |
| 3133 | 3259 | av_log(hwfc, AV_LOG_DEBUG, "Mapped DRM object to Vulkan!\n"); |
| 3134 | 3260 | |
| 3135 | 3261 | return 0; |
| @@ -114,6 +114,7 @@ typedef enum FFVulkanExtensions { · 113 unmodified lines | ||
| 114 | 114 | \ |
| 115 | 115 | /* Semaphores */ \ |
| 116 | 116 | MACRO(1, 1, FF_VK_EXT_EXTERNAL_FD_SEM, GetSemaphoreFdKHR) \ |
| 117 | + MACRO(1, 1, FF_VK_EXT_EXTERNAL_FD_SEM, ImportSemaphoreFdKHR) \ | |
| 117 | 118 | MACRO(1, 1, FF_VK_EXT_NO_FLAG, CreateSemaphore) \ |
| 118 | 119 | MACRO(1, 1, FF_VK_EXT_NO_FLAG, WaitSemaphores) \ |
| 119 | 120 | MACRO(1, 1, FF_VK_EXT_NO_FLAG, DestroySemaphore) \ |