For a frame comes from AV_HWDEVICE_TYPE_VIDEOTOOLBOX, it's CVPixelBufferRef is maintained by a pool. CVPixelBufferRef returned to the pool when frame buffer reference reached to zero. However, VTCompressionSessionEncodeFrame also hold a reference to the CVPixelBufferRef. So a new frame get from av_hwframe_get_buffer may access a CVPixelBufferRef which still used by the encoder. It's only after vtenc_output_callback that we can make sure CVPixelBufferRef has been released by the encoder. The issue can be tested with sample from trac #10884. ffmpeg -hwaccel videotoolbox \ -hwaccel_output_format videotoolbox_vld \ -i input.mp4 \ -c:v hevc_videotoolbox \ -profile:v main \ -b:v 3M \ -vf scale_vt=w=iw/2:h=ih/2:color_matrix=bt709:color_primaries=bt709:color_transfer=bt709 \ -c:a copy \ -tag:v hvc1 \ output.mp4 Withtout the patch, there are some out of order images in output.mp4.
Showing the changes to libavcodec (1 of 1 file). Show all
Unified · Split
| @@ -228,6 +228,7 @@ typedef struct ExtraSEI { · 227 unmodified lines | |||
| 228 | typedef struct BufNode { | 228 | typedef struct BufNode { |
| 229 | CMSampleBufferRef cm_buffer; | 229 | CMSampleBufferRef cm_buffer; |
| 230 | ExtraSEI *sei; | 230 | ExtraSEI *sei; |
| 231 | AVBufferRef *frame_buf; | ||
| 231 | struct BufNode* next; | 232 | struct BufNode* next; |
| 232 | } BufNode; | 233 | } BufNode; |
| 233 | 234 | ||
| @@ -727,6 +728,7 @@ static void vtenc_free_buf_node(BufNode *info) · 493 unmodified lines | |||
| 727 | if (info->cm_buffer) | 728 | if (info->cm_buffer) |
| 728 | CFRelease(info->cm_buffer); | 729 | CFRelease(info->cm_buffer); |
| 729 | 730 | ||
| 731 | av_buffer_unref(&info->frame_buf); | ||
| 730 | av_free(info); | 732 | av_free(info); |
| 731 | } | 733 | } |
| 732 | 734 | ||
| @@ -741,6 +743,7 @@ static void vtenc_output_callback( · 8 unmodified lines | |||
| 741 | VTEncContext *vtctx = avctx->priv_data; | 743 | VTEncContext *vtctx = avctx->priv_data; |
| 742 | BufNode *info = sourceFrameCtx; | 744 | BufNode *info = sourceFrameCtx; |
| 743 | 745 | ||
| 746 | av_buffer_unref(&info->frame_buf); | ||
| 744 | if (vtctx->async_error) { | 747 | if (vtctx->async_error) { |
| 745 | vtenc_free_buf_node(info); | 748 | vtenc_free_buf_node(info); |
| 746 | return; | 749 | return; |
| @@ -2459,7 +2462,8 @@ static int copy_avframe_to_pixel_buffer(AVCodecContext *avctx, · 1712 unmodified lines | |||
| 2459 | 2462 | ||
| 2460 | static int create_cv_pixel_buffer(AVCodecContext *avctx, | 2463 | static int create_cv_pixel_buffer(AVCodecContext *avctx, |
| 2461 | const AVFrame *frame, | 2464 | const AVFrame *frame, |
| 2462 | CVPixelBufferRef *cv_img) | 2465 | CVPixelBufferRef *cv_img, |
| 2466 | BufNode *node) | ||
| 2463 | { | 2467 | { |
| 2464 | int plane_count; | 2468 | int plane_count; |
| 2465 | int color; | 2469 | int color; |
| @@ -2478,6 +2482,12 @@ static int create_cv_pixel_buffer(AVCodecContext *avctx, · 12 unmodified lines | |||
| 2478 | av_assert0(*cv_img); | 2482 | av_assert0(*cv_img); |
| 2479 | 2483 | ||
| 2480 | CFRetain(*cv_img); | 2484 | CFRetain(*cv_img); |
| 2485 | if (frame->buf[0]) { | ||
| 2486 | node->frame_buf = av_buffer_ref(frame->buf[0]); | ||
| 2487 | if (!node->frame_buf) | ||
| 2488 | return AVERROR(ENOMEM); | ||
| 2489 | } | ||
| 2490 | |||
| 2481 | return 0; | 2491 | return 0; |
| 2482 | } | 2492 | } |
| 2483 | 2493 | ||
| @@ -2585,7 +2595,7 @@ static int vtenc_send_frame(AVCodecContext *avctx, · 101 unmodified lines | |||
| 2585 | if (!node) | 2595 | if (!node) |
| 2586 | return AVERROR(ENOMEM); | 2596 | return AVERROR(ENOMEM); |
| 2587 | 2597 | ||
| 2588 | status = create_cv_pixel_buffer(avctx, frame, &cv_img); | 2598 | status = create_cv_pixel_buffer(avctx, frame, &cv_img, node); |
| 2589 | if (status) | 2599 | if (status) |
| 2590 | goto out; | 2600 | goto out; |
| 2591 | 2601 | ||