For a frame comes from AV_HWDEVICE_TYPE_VIDEOTOOLBOX, it's CVPixelBufferRef is maintained by a pool. CVPixelBufferRef returned to the pool when frame buffer reference reached to zero. However, VTCompressionSessionEncodeFrame also hold a reference to the CVPixelBufferRef. So a new frame get from av_hwframe_get_buffer may access a CVPixelBufferRef which still used by the encoder. It's only after vtenc_output_callback that we can make sure CVPixelBufferRef has been released by the encoder. The issue can be tested with sample from trac #10884. ffmpeg -hwaccel videotoolbox \ -hwaccel_output_format videotoolbox_vld \ -i input.mp4 \ -c:v hevc_videotoolbox \ -profile:v main \ -b:v 3M \ -vf scale_vt=w=iw/2:h=ih/2:color_matrix=bt709:color_primaries=bt709:color_transfer=bt709 \ -c:a copy \ -tag:v hvc1 \ output.mp4 Withtout the patch, there are some out of order images in output.mp4.
Showing the changes to libavcodec (1 of 1 file). Show all
Unified · Split
| @@ -228,6 +228,7 @@ typedef struct ExtraSEI { · 227 unmodified lines | ||
| 228 | 228 | typedef struct BufNode { |
| 229 | 229 | CMSampleBufferRef cm_buffer; |
| 230 | 230 | ExtraSEI *sei; |
| 231 | + AVBufferRef *frame_buf; | |
| 231 | 232 | struct BufNode* next; |
| 232 | 233 | } BufNode; |
| 233 | 234 | |
| @@ -727,6 +728,7 @@ static void vtenc_free_buf_node(BufNode *info) · 493 unmodified lines | ||
| 727 | 728 | if (info->cm_buffer) |
| 728 | 729 | CFRelease(info->cm_buffer); |
| 729 | 730 | |
| 731 | + av_buffer_unref(&info->frame_buf); | |
| 730 | 732 | av_free(info); |
| 731 | 733 | } |
| 732 | 734 | |
| @@ -741,6 +743,7 @@ static void vtenc_output_callback( · 8 unmodified lines | ||
| 741 | 743 | VTEncContext *vtctx = avctx->priv_data; |
| 742 | 744 | BufNode *info = sourceFrameCtx; |
| 743 | 745 | |
| 746 | + av_buffer_unref(&info->frame_buf); | |
| 744 | 747 | if (vtctx->async_error) { |
| 745 | 748 | vtenc_free_buf_node(info); |
| 746 | 749 | return; |
| @@ -2459,7 +2462,8 @@ static int copy_avframe_to_pixel_buffer(AVCodecContext *avctx, · 1712 unmodified lines | ||
| 2459 | 2462 | |
| 2460 | 2463 | static int create_cv_pixel_buffer(AVCodecContext *avctx, |
| 2461 | 2464 | const AVFrame *frame, |
| 2462 | − CVPixelBufferRef *cv_img) | |
| 2465 | + CVPixelBufferRef *cv_img, | |
| 2466 | + BufNode *node) | |
| 2463 | 2467 | { |
| 2464 | 2468 | int plane_count; |
| 2465 | 2469 | int color; |
| @@ -2478,6 +2482,12 @@ static int create_cv_pixel_buffer(AVCodecContext *avctx, · 12 unmodified lines | ||
| 2478 | 2482 | av_assert0(*cv_img); |
| 2479 | 2483 | |
| 2480 | 2484 | CFRetain(*cv_img); |
| 2485 | + if (frame->buf[0]) { | |
| 2486 | + node->frame_buf = av_buffer_ref(frame->buf[0]); | |
| 2487 | + if (!node->frame_buf) | |
| 2488 | + return AVERROR(ENOMEM); | |
| 2489 | + } | |
| 2490 | + | |
| 2481 | 2491 | return 0; |
| 2482 | 2492 | } |
| 2483 | 2493 | |
| @@ -2585,7 +2595,7 @@ static int vtenc_send_frame(AVCodecContext *avctx, · 101 unmodified lines | ||
| 2585 | 2595 | if (!node) |
| 2586 | 2596 | return AVERROR(ENOMEM); |
| 2587 | 2597 | |
| 2588 | − status = create_cv_pixel_buffer(avctx, frame, &cv_img); | |
| 2598 | + status = create_cv_pixel_buffer(avctx, frame, &cv_img, node); | |
| 2589 | 2599 | if (status) |
| 2590 | 2600 | goto out; |
| 2591 | 2601 | |