In vtenc_populate_extradata, the cleanup function vtenc_reset should not be used when no error occurs, otherwise some color information is lost (#11036). This patch checks the status code and conducts the correct cleanup.
Showing the changes to libavcodec (1 of 1 file). Show all
Unified · Split
| @@ -2767,7 +2767,14 @@ static int vtenc_populate_extradata(AVCodecContext *avctx, · 2766 unmodified lines | |||
| 2767 | 2767 | ||
| 2768 | pe_cleanup: | 2768 | pe_cleanup: |
| 2769 | CVPixelBufferRelease(pix_buf); | 2769 | CVPixelBufferRelease(pix_buf); |
| 2770 | vtenc_reset(vtctx); | 2770 | |
| 2771 | if (status) { | ||
| 2772 | vtenc_reset(vtctx); | ||
| 2773 | } else if (vtctx->session) { | ||
| 2774 | CFRelease(vtctx->session); | ||
| 2775 | vtctx->session = NULL; | ||
| 2776 | } | ||
| 2777 | |||
| 2771 | vtctx->frame_ct_out = 0; | 2778 | vtctx->frame_ct_out = 0; |
| 2772 | 2779 | ||
| 2773 | av_assert0(status != 0 || (avctx->extradata && avctx->extradata_size > 0)); | 2780 | av_assert0(status != 0 || (avctx->extradata && avctx->extradata_size > 0)); |