| @@ -0,0 +1,438 @@ |
| 1 | +/* |
| 2 | + * Copyright (c) 2020 Yaroslav Pogrebnyak <yyyaroslav@gmail.com> |
| 3 | + * |
| 4 | + * This file is part of FFmpeg. |
| 5 | + * |
| 6 | + * FFmpeg is free software; you can redistribute it and/or |
| 7 | + * modify it under the terms of the GNU Lesser General Public |
| 8 | + * License as published by the Free Software Foundation; either |
| 9 | + * version 2.1 of the License, or (at your option) any later version. |
| 10 | + * |
| 11 | + * FFmpeg is distributed in the hope that it will be useful, |
| 12 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 13 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 14 | + * Lesser General Public License for more details. |
| 15 | + * |
| 16 | + * You should have received a copy of the GNU Lesser General Public |
| 17 | + * License along with FFmpeg; if not, write to the Free Software |
| 18 | + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA |
| 19 | + */ |
| 20 | + |
| 21 | +/** |
| 22 | + * @file |
| 23 | + * Overlay one video on top of another using cuda hardware acceleration |
| 24 | + */ |
| 25 | + |
| 26 | +#include "libavutil/log.h" |
| 27 | +#include "libavutil/mem.h" |
| 28 | +#include "libavutil/opt.h" |
| 29 | +#include "libavutil/pixdesc.h" |
| 30 | +#include "libavutil/hwcontext.h" |
| 31 | +#include "libavutil/hwcontext_cuda_internal.h" |
| 32 | +#include "libavutil/cuda_check.h" |
| 33 | + |
| 34 | +#include "avfilter.h" |
| 35 | +#include "framesync.h" |
| 36 | +#include "internal.h" |
| 37 | + |
| 38 | +#define CHECK_CU(x) FF_CUDA_CHECK_DL(ctx, ctx->hwctx->internal->cuda_dl, x) |
| 39 | +#define DIV_UP(a, b) ( ((a) + (b) - 1) / (b) ) |
| 40 | + |
| 41 | +#define BLOCK_X 32 |
| 42 | +#define BLOCK_Y 16 |
| 43 | + |
| 44 | +static const enum AVPixelFormat supported_main_formats[] = { |
| 45 | + AV_PIX_FMT_NV12, |
| 46 | + AV_PIX_FMT_YUV420P, |
| 47 | + AV_PIX_FMT_NONE, |
| 48 | +}; |
| 49 | + |
| 50 | +static const enum AVPixelFormat supported_overlay_formats[] = { |
| 51 | + AV_PIX_FMT_NV12, |
| 52 | + AV_PIX_FMT_YUV420P, |
| 53 | + AV_PIX_FMT_YUVA420P, |
| 54 | + AV_PIX_FMT_NONE, |
| 55 | +}; |
| 56 | + |
| 57 | +/** |
| 58 | + * OverlayCUDAContext |
| 59 | + */ |
| 60 | +typedef struct OverlayCUDAContext { |
| 61 | + const AVClass *class; |
| 62 | + |
| 63 | + enum AVPixelFormat in_format_overlay; |
| 64 | + enum AVPixelFormat in_format_main; |
| 65 | + |
| 66 | + AVCUDADeviceContext *hwctx; |
| 67 | + |
| 68 | + CUcontext cu_ctx; |
| 69 | + CUmodule cu_module; |
| 70 | + CUfunction cu_func; |
| 71 | + CUstream cu_stream; |
| 72 | + |
| 73 | + FFFrameSync fs; |
| 74 | + |
| 75 | + int x_position; |
| 76 | + int y_position; |
| 77 | + |
| 78 | +} OverlayCUDAContext; |
| 79 | + |
| 80 | +/** |
| 81 | + * Helper to find out if provided format is supported by filter |
| 82 | + */ |
| 83 | +static int format_is_supported(const enum AVPixelFormat formats[], enum AVPixelFormat fmt) |
| 84 | +{ |
| 85 | + for (int i = 0; formats[i] != AV_PIX_FMT_NONE; i++) |
| 86 | + if (formats[i] == fmt) |
| 87 | + return 1; |
| 88 | + return 0; |
| 89 | +} |
| 90 | + |
| 91 | +/** |
| 92 | + * Helper checks if we can process main and overlay pixel formats |
| 93 | + */ |
| 94 | +static int formats_match(const enum AVPixelFormat format_main, const enum AVPixelFormat format_overlay) { |
| 95 | + switch(format_main) { |
| 96 | + case AV_PIX_FMT_NV12: |
| 97 | + return format_overlay == AV_PIX_FMT_NV12; |
| 98 | + case AV_PIX_FMT_YUV420P: |
| 99 | + return format_overlay == AV_PIX_FMT_YUV420P || |
| 100 | + format_overlay == AV_PIX_FMT_YUVA420P; |
| 101 | + default: |
| 102 | + return 0; |
| 103 | + } |
| 104 | +} |
| 105 | + |
| 106 | +/** |
| 107 | + * Call overlay kernell for a plane |
| 108 | + */ |
| 109 | +static int overlay_cuda_call_kernel( |
| 110 | + OverlayCUDAContext *ctx, |
| 111 | + int x_position, int y_position, |
| 112 | + uint8_t* main_data, int main_linesize, |
| 113 | + int main_width, int main_height, |
| 114 | + uint8_t* overlay_data, int overlay_linesize, |
| 115 | + int overlay_width, int overlay_height, |
| 116 | + uint8_t* alpha_data, int alpha_linesize, |
| 117 | + int alpha_adj_x, int alpha_adj_y) { |
| 118 | + |
| 119 | + CudaFunctions *cu = ctx->hwctx->internal->cuda_dl; |
| 120 | + |
| 121 | + void* kernel_args[] = { |
| 122 | + &x_position, &y_position, |
| 123 | + &main_data, &main_linesize, |
| 124 | + &overlay_data, &overlay_linesize, |
| 125 | + &overlay_width, &overlay_height, |
| 126 | + &alpha_data, &alpha_linesize, |
| 127 | + &alpha_adj_x, &alpha_adj_y, |
| 128 | + }; |
| 129 | + |
| 130 | + return CHECK_CU(cu->cuLaunchKernel( |
| 131 | + ctx->cu_func, |
| 132 | + DIV_UP(main_width, BLOCK_X), DIV_UP(main_height, BLOCK_Y), 1, |
| 133 | + BLOCK_X, BLOCK_Y, 1, |
| 134 | + 0, ctx->cu_stream, kernel_args, NULL)); |
| 135 | +} |
| 136 | + |
| 137 | +/** |
| 138 | + * Perform blend overlay picture over main picture |
| 139 | + */ |
| 140 | +static int overlay_cuda_blend(FFFrameSync *fs) |
| 141 | +{ |
| 142 | + int ret; |
| 143 | + |
| 144 | + AVFilterContext *avctx = fs->parent; |
| 145 | + OverlayCUDAContext *ctx = avctx->priv; |
| 146 | + AVFilterLink *outlink = avctx->outputs[0]; |
| 147 | + |
| 148 | + CudaFunctions *cu = ctx->hwctx->internal->cuda_dl; |
| 149 | + CUcontext dummy, cuda_ctx = ctx->hwctx->cuda_ctx; |
| 150 | + |
| 151 | + AVFrame *input_main, *input_overlay; |
| 152 | + |
| 153 | + ctx->cu_ctx = cuda_ctx; |
| 154 | + |
| 155 | + // read main and overlay frames from inputs |
| 156 | + ret = ff_framesync_dualinput_get(fs, &input_main, &input_overlay); |
| 157 | + if (ret < 0) |
| 158 | + return ret; |
| 159 | + |
| 160 | + if (!input_main || !input_overlay) |
| 161 | + return AVERROR_BUG; |
| 162 | + |
| 163 | + ret = av_frame_make_writable(input_main); |
| 164 | + if (ret < 0) { |
| 165 | + av_frame_free(&input_main); |
| 166 | + return ret; |
| 167 | + } |
| 168 | + |
| 169 | + // push cuda context |
| 170 | + |
| 171 | + ret = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx)); |
| 172 | + if (ret < 0) { |
| 173 | + av_frame_free(&input_main); |
| 174 | + return ret; |
| 175 | + } |
| 176 | + |
| 177 | + // overlay first plane |
| 178 | + |
| 179 | + overlay_cuda_call_kernel(ctx, |
| 180 | + ctx->x_position, ctx->y_position, |
| 181 | + input_main->data[0], input_main->linesize[0], |
| 182 | + input_main->width, input_main->height, |
| 183 | + input_overlay->data[0], input_overlay->linesize[0], |
| 184 | + input_overlay->width, input_overlay->height, |
| 185 | + input_overlay->data[3], input_overlay->linesize[3], 1, 1); |
| 186 | + |
| 187 | + // overlay rest planes depending on pixel format |
| 188 | + |
| 189 | + switch(ctx->in_format_overlay) { |
| 190 | + case AV_PIX_FMT_NV12: |
| 191 | + overlay_cuda_call_kernel(ctx, |
| 192 | + ctx->x_position, ctx->y_position / 2, |
| 193 | + input_main->data[1], input_main->linesize[1], |
| 194 | + input_main->width, input_main->height / 2, |
| 195 | + input_overlay->data[1], input_overlay->linesize[1], |
| 196 | + input_overlay->width, input_overlay->height / 2, |
| 197 | + 0, 0, 0, 0); |
| 198 | + break; |
| 199 | + case AV_PIX_FMT_YUV420P: |
| 200 | + case AV_PIX_FMT_YUVA420P: |
| 201 | + overlay_cuda_call_kernel(ctx, |
| 202 | + ctx->x_position / 2 , ctx->y_position / 2, |
| 203 | + input_main->data[1], input_main->linesize[1], |
| 204 | + input_main->width / 2, input_main->height / 2, |
| 205 | + input_overlay->data[1], input_overlay->linesize[1], |
| 206 | + input_overlay->width / 2, input_overlay->height / 2, |
| 207 | + input_overlay->data[3], input_overlay->linesize[3], 2, 2); |
| 208 | + |
| 209 | + overlay_cuda_call_kernel(ctx, |
| 210 | + ctx->x_position / 2 , ctx->y_position / 2, |
| 211 | + input_main->data[2], input_main->linesize[2], |
| 212 | + input_main->width / 2, input_main->height / 2, |
| 213 | + input_overlay->data[2], input_overlay->linesize[2], |
| 214 | + input_overlay->width / 2, input_overlay->height / 2, |
| 215 | + input_overlay->data[3], input_overlay->linesize[3], 2, 2); |
| 216 | + break; |
| 217 | + default: |
| 218 | + av_log(ctx, AV_LOG_ERROR, "Passed unsupported overlay pixel format\n"); |
| 219 | + av_frame_free(&input_main); |
| 220 | + CHECK_CU(cu->cuCtxPopCurrent(&dummy)); |
| 221 | + return AVERROR_BUG; |
| 222 | + } |
| 223 | + |
| 224 | + CHECK_CU(cu->cuCtxPopCurrent(&dummy)); |
| 225 | + |
| 226 | + return ff_filter_frame(outlink, input_main); |
| 227 | +} |
| 228 | + |
| 229 | +/** |
| 230 | + * Initialize overlay_cuda |
| 231 | + */ |
| 232 | +static av_cold int overlay_cuda_init(AVFilterContext *avctx) |
| 233 | +{ |
| 234 | + OverlayCUDAContext* ctx = avctx->priv; |
| 235 | + ctx->fs.on_event = &overlay_cuda_blend; |
| 236 | + |
| 237 | + return 0; |
| 238 | +} |
| 239 | + |
| 240 | +/** |
| 241 | + * Uninitialize overlay_cuda |
| 242 | + */ |
| 243 | +static av_cold void overlay_cuda_uninit(AVFilterContext *avctx) |
| 244 | +{ |
| 245 | + OverlayCUDAContext* ctx = avctx->priv; |
| 246 | + |
| 247 | + ff_framesync_uninit(&ctx->fs); |
| 248 | + |
| 249 | + if (ctx->hwctx && ctx->cu_module) { |
| 250 | + CUcontext dummy; |
| 251 | + CudaFunctions *cu = ctx->hwctx->internal->cuda_dl; |
| 252 | + CHECK_CU(cu->cuCtxPushCurrent(ctx->cu_ctx)); |
| 253 | + CHECK_CU(cu->cuModuleUnload(ctx->cu_module)); |
| 254 | + CHECK_CU(cu->cuCtxPopCurrent(&dummy)); |
| 255 | + } |
| 256 | +} |
| 257 | + |
| 258 | +/** |
| 259 | + * Activate overlay_cuda |
| 260 | + */ |
| 261 | +static int overlay_cuda_activate(AVFilterContext *avctx) |
| 262 | +{ |
| 263 | + OverlayCUDAContext *ctx = avctx->priv; |
| 264 | + |
| 265 | + return ff_framesync_activate(&ctx->fs); |
| 266 | +} |
| 267 | + |
| 268 | +/** |
| 269 | + * Query formats |
| 270 | + */ |
| 271 | +static int overlay_cuda_query_formats(AVFilterContext *avctx) |
| 272 | +{ |
| 273 | + static const enum AVPixelFormat pixel_formats[] = { |
| 274 | + AV_PIX_FMT_CUDA, AV_PIX_FMT_NONE, |
| 275 | + }; |
| 276 | + |
| 277 | + AVFilterFormats *pix_fmts = ff_make_format_list(pixel_formats); |
| 278 | + |
| 279 | + return ff_set_common_formats(avctx, pix_fmts); |
| 280 | +} |
| 281 | + |
| 282 | +/** |
| 283 | + * Configure output |
| 284 | + */ |
| 285 | +static int overlay_cuda_config_output(AVFilterLink *outlink) |
| 286 | +{ |
| 287 | + |
| 288 | + extern char vf_overlay_cuda_ptx[]; |
| 289 | + |
| 290 | + int err; |
| 291 | + AVFilterContext* avctx = outlink->src; |
| 292 | + OverlayCUDAContext* ctx = avctx->priv; |
| 293 | + |
| 294 | + AVFilterLink *inlink = avctx->inputs[0]; |
| 295 | + AVHWFramesContext *frames_ctx = (AVHWFramesContext*)inlink->hw_frames_ctx->data; |
| 296 | + |
| 297 | + AVFilterLink *inlink_overlay = avctx->inputs[1]; |
| 298 | + AVHWFramesContext *frames_ctx_overlay = (AVHWFramesContext*)inlink_overlay->hw_frames_ctx->data; |
| 299 | + |
| 300 | + CUcontext dummy, cuda_ctx; |
| 301 | + CudaFunctions *cu; |
| 302 | + |
| 303 | + // check main input formats |
| 304 | + |
| 305 | + if (!frames_ctx) { |
| 306 | + av_log(ctx, AV_LOG_ERROR, "No hw context provided on main input\n"); |
| 307 | + return AVERROR(EINVAL); |
| 308 | + } |
| 309 | + |
| 310 | + ctx->in_format_main = frames_ctx->sw_format; |
| 311 | + if (!format_is_supported(supported_main_formats, ctx->in_format_main)) { |
| 312 | + av_log(ctx, AV_LOG_ERROR, "Unsupported main input format: %s\n", |
| 313 | + av_get_pix_fmt_name(ctx->in_format_main)); |
| 314 | + return AVERROR(ENOSYS); |
| 315 | + } |
| 316 | + |
| 317 | + // check overlay input formats |
| 318 | + |
| 319 | + if (!frames_ctx_overlay) { |
| 320 | + av_log(ctx, AV_LOG_ERROR, "No hw context provided on overlay input\n"); |
| 321 | + return AVERROR(EINVAL); |
| 322 | + } |
| 323 | + |
| 324 | + ctx->in_format_overlay = frames_ctx_overlay->sw_format; |
| 325 | + if (!format_is_supported(supported_overlay_formats, ctx->in_format_overlay)) { |
| 326 | + av_log(ctx, AV_LOG_ERROR, "Unsupported overlay input format: %s\n", |
| 327 | + av_get_pix_fmt_name(ctx->in_format_overlay)); |
| 328 | + return AVERROR(ENOSYS); |
| 329 | + } |
| 330 | + |
| 331 | + // check we can overlay pictures with those pixel formats |
| 332 | + |
| 333 | + if (!formats_match(ctx->in_format_main, ctx->in_format_overlay)) { |
| 334 | + av_log(ctx, AV_LOG_ERROR, "Can't overlay %s on %s \n", |
| 335 | + av_get_pix_fmt_name(ctx->in_format_overlay), av_get_pix_fmt_name(ctx->in_format_main)); |
| 336 | + return AVERROR(EINVAL); |
| 337 | + } |
| 338 | + |
| 339 | + // initialize |
| 340 | + |
| 341 | + ctx->hwctx = frames_ctx->device_ctx->hwctx; |
| 342 | + cuda_ctx = ctx->hwctx->cuda_ctx; |
| 343 | + ctx->fs.time_base = inlink->time_base; |
| 344 | + |
| 345 | + ctx->cu_stream = ctx->hwctx->stream; |
| 346 | + |
| 347 | + outlink->hw_frames_ctx = av_buffer_ref(inlink->hw_frames_ctx); |
| 348 | + |
| 349 | + // load functions |
| 350 | + |
| 351 | + cu = ctx->hwctx->internal->cuda_dl; |
| 352 | + |
| 353 | + err = CHECK_CU(cu->cuCtxPushCurrent(cuda_ctx)); |
| 354 | + if (err < 0) { |
| 355 | + return err; |
| 356 | + } |
| 357 | + |
| 358 | + err = CHECK_CU(cu->cuModuleLoadData(&ctx->cu_module, vf_overlay_cuda_ptx)); |
| 359 | + if (err < 0) { |
| 360 | + CHECK_CU(cu->cuCtxPopCurrent(&dummy)); |
| 361 | + return err; |
| 362 | + } |
| 363 | + |
| 364 | + err = CHECK_CU(cu->cuModuleGetFunction(&ctx->cu_func, ctx->cu_module, "Overlay_Cuda")); |
| 365 | + if (err < 0) { |
| 366 | + CHECK_CU(cu->cuCtxPopCurrent(&dummy)); |
| 367 | + return err; |
| 368 | + } |
| 369 | + |
| 370 | + CHECK_CU(cu->cuCtxPopCurrent(&dummy)); |
| 371 | + |
| 372 | + // init dual input |
| 373 | + |
| 374 | + err = ff_framesync_init_dualinput(&ctx->fs, avctx); |
| 375 | + if (err < 0) { |
| 376 | + return err; |
| 377 | + } |
| 378 | + |
| 379 | + return ff_framesync_configure(&ctx->fs); |
| 380 | +} |
| 381 | + |
| 382 | + |
| 383 | +#define OFFSET(x) offsetof(OverlayCUDAContext, x) |
| 384 | +#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM) |
| 385 | + |
| 386 | +static const AVOption overlay_cuda_options[] = { |
| 387 | + { "x", "Overlay x position", |
| 388 | + OFFSET(x_position), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, .flags = FLAGS }, |
| 389 | + { "y", "Overlay y position", |
| 390 | + OFFSET(y_position), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, .flags = FLAGS }, |
| 391 | + { "eof_action", "Action to take when encountering EOF from secondary input ", |
| 392 | + OFFSET(fs.opt_eof_action), AV_OPT_TYPE_INT, { .i64 = EOF_ACTION_REPEAT }, |
| 393 | + EOF_ACTION_REPEAT, EOF_ACTION_PASS, .flags = FLAGS, "eof_action" }, |
| 394 | + { "repeat", "Repeat the previous frame.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_REPEAT }, .flags = FLAGS, "eof_action" }, |
| 395 | + { "endall", "End both streams.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_ENDALL }, .flags = FLAGS, "eof_action" }, |
| 396 | + { "pass", "Pass through the main input.", 0, AV_OPT_TYPE_CONST, { .i64 = EOF_ACTION_PASS }, .flags = FLAGS, "eof_action" }, |
| 397 | + { "shortest", "force termination when the shortest input terminates", OFFSET(fs.opt_shortest), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS }, |
| 398 | + { "repeatlast", "repeat overlay of the last overlay frame", OFFSET(fs.opt_repeatlast), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, FLAGS }, |
| 399 | + { NULL }, |
| 400 | +}; |
| 401 | + |
| 402 | +FRAMESYNC_DEFINE_CLASS(overlay_cuda, OverlayCUDAContext, fs); |
| 403 | + |
| 404 | +static const AVFilterPad overlay_cuda_inputs[] = { |
| 405 | + { |
| 406 | + .name = "main", |
| 407 | + .type = AVMEDIA_TYPE_VIDEO, |
| 408 | + }, |
| 409 | + { |
| 410 | + .name = "overlay", |
| 411 | + .type = AVMEDIA_TYPE_VIDEO, |
| 412 | + }, |
| 413 | + { NULL } |
| 414 | +}; |
| 415 | + |
| 416 | +static const AVFilterPad overlay_cuda_outputs[] = { |
| 417 | + { |
| 418 | + .name = "default", |
| 419 | + .type = AVMEDIA_TYPE_VIDEO, |
| 420 | + .config_props = &overlay_cuda_config_output, |
| 421 | + }, |
| 422 | + { NULL } |
| 423 | +}; |
| 424 | + |
| 425 | +AVFilter ff_vf_overlay_cuda = { |
| 426 | + .name = "overlay_cuda", |
| 427 | + .description = NULL_IF_CONFIG_SMALL("Overlay one video on top of another using CUDA"), |
| 428 | + .priv_size = sizeof(OverlayCUDAContext), |
| 429 | + .priv_class = &overlay_cuda_class, |
| 430 | + .init = &overlay_cuda_init, |
| 431 | + .uninit = &overlay_cuda_uninit, |
| 432 | + .activate = &overlay_cuda_activate, |
| 433 | + .query_formats = &overlay_cuda_query_formats, |
| 434 | + .inputs = overlay_cuda_inputs, |
| 435 | + .outputs = overlay_cuda_outputs, |
| 436 | + .preinit = overlay_cuda_framesync_preinit, |
| 437 | + .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE, |
| 438 | +}; |