Parameter sets may be coded in the packet before an IRAP (as is the case for the hev1 ISO-BMFF brand), and they should have priority as they may override the extradata ones. As such, prepend the extradata PS NALUs to the packet PS NALUs if they are present before an IRAP, instead of prepending them to the IRAP slice. Should fix ticket #11458.
Unified · Split
| @@ -126,6 +126,7 @@ static int hevc_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *out) · 125 unmodified lines | ||
| 126 | 126 | GetByteContext gb; |
| 127 | 127 | |
| 128 | 128 | int got_irap = 0; |
| 129 | + int got_ps = 0, seen_irap_ps = 0; | |
| 129 | 130 | int i, ret = 0; |
| 130 | 131 | |
| 131 | 132 | ret = ff_bsf_get_packet(ctx, &in); |
| @@ -140,10 +141,37 @@ static int hevc_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *out) · 8 unmodified lines | ||
| 140 | 141 | |
| 141 | 142 | bytestream2_init(&gb, in->data, in->size); |
| 142 | 143 | |
| 144 | + while (!got_irap && bytestream2_get_bytes_left(&gb)) { | |
| 145 | + uint32_t nalu_size = 0; | |
| 146 | + int nalu_type; | |
| 147 | + | |
| 148 | + if (bytestream2_get_bytes_left(&gb) < s->length_size) { | |
| 149 | + ret = AVERROR_INVALIDDATA; | |
| 150 | + goto fail; | |
| 151 | + } | |
| 152 | + for (i = 0; i < s->length_size; i++) | |
| 153 | + nalu_size = (nalu_size << 8) | bytestream2_get_byte(&gb); | |
| 154 | + | |
| 155 | + if (nalu_size < 2 || nalu_size > bytestream2_get_bytes_left(&gb)) { | |
| 156 | + ret = AVERROR_INVALIDDATA; | |
| 157 | + goto fail; | |
| 158 | + } | |
| 159 | + | |
| 160 | + nalu_type = (bytestream2_peek_byte(&gb) >> 1) & 0x3f; | |
| 161 | + bytestream2_skip(&gb, nalu_size); | |
| 162 | + got_irap |= nalu_type >= HEVC_NAL_BLA_W_LP && | |
| 163 | + nalu_type <= HEVC_NAL_RSV_IRAP_VCL23; | |
| 164 | + got_ps |= nalu_type >= HEVC_NAL_VPS && nalu_type <= HEVC_NAL_PPS; | |
| 165 | + } | |
| 166 | + seen_irap_ps = got_irap && got_ps; | |
| 167 | + got_irap = got_ps = 0; | |
| 168 | + | |
| 169 | + bytestream2_init(&gb, in->data, in->size); | |
| 170 | + | |
| 143 | 171 | while (bytestream2_get_bytes_left(&gb)) { |
| 144 | 172 | uint32_t nalu_size = 0; |
| 145 | 173 | int nalu_type; |
| 146 | − int is_irap, add_extradata, extra_size, prev_size; | |
| 174 | + int is_irap, is_ps, add_extradata, extra_size, prev_size; | |
| 147 | 175 | |
| 148 | 176 | if (bytestream2_get_bytes_left(&gb) < s->length_size) { |
| 149 | 177 | ret = AVERROR_INVALIDDATA; |
| @@ -162,9 +190,11 @@ static int hevc_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *out) · 12 unmodified lines | ||
| 162 | 190 | /* prepend extradata to IRAP frames */ |
| 163 | 191 | is_irap = nalu_type >= HEVC_NAL_BLA_W_LP && |
| 164 | 192 | nalu_type <= HEVC_NAL_RSV_IRAP_VCL23; |
| 165 | − add_extradata = is_irap && !got_irap; | |
| 193 | + is_ps = nalu_type >= HEVC_NAL_VPS && nalu_type <= HEVC_NAL_PPS && seen_irap_ps; | |
| 194 | + add_extradata = (is_ps || is_irap) && !got_ps && !got_irap; | |
| 166 | 195 | extra_size = add_extradata * ctx->par_out->extradata_size; |
| 167 | 196 | got_irap |= is_irap; |
| 197 | + got_ps |= is_ps; | |
| 168 | 198 | |
| 169 | 199 | if (FFMIN(INT_MAX, SIZE_MAX) < 4ULL + nalu_size + extra_size) { |
| 170 | 200 | ret = AVERROR_INVALIDDATA; |