Fixes invalid reads while initializing the dequant tables, which uses the bit depth to determine the QP table size.
Unified · Split
| @@ -2707,11 +2707,6 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ · 2706 unmodified lines | |||
| 2707 | s->avctx->level = h->sps.level_idc; | 2707 | s->avctx->level = h->sps.level_idc; |
| 2708 | s->avctx->refs = h->sps.ref_frame_count; | 2708 | s->avctx->refs = h->sps.ref_frame_count; |
| 2709 | 2709 | ||
| 2710 | if(h == h0 && h->dequant_coeff_pps != pps_id){ | ||
| 2711 | h->dequant_coeff_pps = pps_id; | ||
| 2712 | init_dequant_tables(h); | ||
| 2713 | } | ||
| 2714 | |||
| 2715 | s->mb_width= h->sps.mb_width; | 2710 | s->mb_width= h->sps.mb_width; |
| 2716 | s->mb_height= h->sps.mb_height * (2 - h->sps.frame_mbs_only_flag); | 2711 | s->mb_height= h->sps.mb_height * (2 - h->sps.frame_mbs_only_flag); |
| 2717 | 2712 | ||
| @@ -2786,7 +2781,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ · 68 unmodified lines | |||
| 2786 | else | 2781 | else |
| 2787 | s->avctx->pix_fmt = PIX_FMT_YUV420P10; | 2782 | s->avctx->pix_fmt = PIX_FMT_YUV420P10; |
| 2788 | break; | 2783 | break; |
| 2789 | default: | 2784 | case 8: |
| 2790 | if (CHROMA444){ | 2785 | if (CHROMA444){ |
| 2791 | if (s->avctx->colorspace == AVCOL_SPC_RGB) { | 2786 | if (s->avctx->colorspace == AVCOL_SPC_RGB) { |
| 2792 | s->avctx->pix_fmt = PIX_FMT_GBRP; | 2787 | s->avctx->pix_fmt = PIX_FMT_GBRP; |
| @@ -2802,6 +2797,11 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ · 9 unmodified lines | |||
| 2802 | hwaccel_pixfmt_list_h264_jpeg_420 : | 2797 | hwaccel_pixfmt_list_h264_jpeg_420 : |
| 2803 | ff_hwaccel_pixfmt_list_420); | 2798 | ff_hwaccel_pixfmt_list_420); |
| 2804 | } | 2799 | } |
| 2800 | break; | ||
| 2801 | default: | ||
| 2802 | av_log(s->avctx, AV_LOG_ERROR, | ||
| 2803 | "Unsupported bit depth: %d\n", h->sps.bit_depth_luma); | ||
| 2804 | return AVERROR_INVALIDDATA; | ||
| 2805 | } | 2805 | } |
| 2806 | 2806 | ||
| 2807 | s->avctx->hwaccel = ff_find_hwaccel(s->avctx->codec->id, s->avctx->pix_fmt); | 2807 | s->avctx->hwaccel = ff_find_hwaccel(s->avctx->codec->id, s->avctx->pix_fmt); |
| @@ -2846,6 +2846,11 @@ static int decode_slice_header(H264Context *h, H264Context *h0){ · 38 unmodified lines | |||
| 2846 | } | 2846 | } |
| 2847 | } | 2847 | } |
| 2848 | 2848 | ||
| 2849 | if(h == h0 && h->dequant_coeff_pps != pps_id){ | ||
| 2850 | h->dequant_coeff_pps = pps_id; | ||
| 2851 | init_dequant_tables(h); | ||
| 2852 | } | ||
| 2853 | |||
| 2849 | h->frame_num= get_bits(&s->gb, h->sps.log2_max_frame_num); | 2854 | h->frame_num= get_bits(&s->gb, h->sps.log2_max_frame_num); |
| 2850 | 2855 | ||
| 2851 | h->mb_mbaff = 0; | 2856 | h->mb_mbaff = 0; |