In f7ac3512f5b5cb8eb149f37300b43461d8e93af3 the size of the dynamically allocated buffer was shrunk, but it was made too small for very small alphabet sizes. This patch restores the size to prevent an OOB read.
Unified · Split
| @@ -683,7 +683,7 @@ static int read_vlc_prefix(GetBitContext *gb, JXLEntropyDecoder *dec, JXLSymbolD · 682 unmodified lines | |||
| 683 | int repeat_count_prev = 0, repeat_count_zero = 0, prev = 8; | 683 | int repeat_count_prev = 0, repeat_count_zero = 0, prev = 8; |
| 684 | int total_code = 0, len, hskip, num_codes = 0, ret; | 684 | int total_code = 0, len, hskip, num_codes = 0, ret; |
| 685 | 685 | ||
| 686 | VLC level1_vlc; | 686 | VLC level1_vlc = { 0 }; |
| 687 | 687 | ||
| 688 | if (dist->alphabet_size == 1) { | 688 | if (dist->alphabet_size == 1) { |
| 689 | dist->vlc.bits = 0; | 689 | dist->vlc.bits = 0; |
| @@ -709,8 +709,10 @@ static int read_vlc_prefix(GetBitContext *gb, JXLEntropyDecoder *dec, JXLSymbolD · 19 unmodified lines | |||
| 709 | } | 709 | } |
| 710 | } | 710 | } |
| 711 | 711 | ||
| 712 | if (total_code != 32 && num_codes >= 2 || num_codes < 1) | 712 | if (total_code != 32 && num_codes >= 2 || num_codes < 1) { |
| 713 | return AVERROR_INVALIDDATA; | 713 | ret = AVERROR_INVALIDDATA; |
| 714 | goto end; | ||
| 715 | } | ||
| 714 | 716 | ||
| 715 | for (int i = 1; i < 19; i++) | 717 | for (int i = 1; i < 19; i++) |
| 716 | level1_codecounts[i] += level1_codecounts[i - 1]; | 718 | level1_codecounts[i] += level1_codecounts[i - 1]; |
| @@ -726,7 +728,7 @@ static int read_vlc_prefix(GetBitContext *gb, JXLEntropyDecoder *dec, JXLSymbolD · 9 unmodified lines | |||
| 726 | if (ret < 0) | 728 | if (ret < 0) |
| 727 | goto end; | 729 | goto end; |
| 728 | 730 | ||
| 729 | buf = av_mallocz(dist->alphabet_size * (2 * sizeof(int8_t) + sizeof(int16_t) + sizeof(uint32_t)) | 731 | buf = av_mallocz(MAX_PREFIX_ALPHABET_SIZE * (2 * sizeof(int8_t) + sizeof(int16_t) + sizeof(uint32_t)) |
| 730 | + sizeof(uint32_t)); | 732 | + sizeof(uint32_t)); |
| 731 | if (!buf) { | 733 | if (!buf) { |
| 732 | ret = AVERROR(ENOMEM); | 734 | ret = AVERROR(ENOMEM); |
| @@ -734,21 +736,22 @@ static int read_vlc_prefix(GetBitContext *gb, JXLEntropyDecoder *dec, JXLSymbolD · 1 unmodified line | |||
| 734 | } | 736 | } |
| 735 | 737 | ||
| 736 | level2_lens = (int8_t *)buf; | 738 | level2_lens = (int8_t *)buf; |
| 737 | level2_lens_s = (int8_t *)(buf + dist->alphabet_size * sizeof(int8_t)); | 739 | level2_lens_s = (int8_t *)(buf + MAX_PREFIX_ALPHABET_SIZE * sizeof(int8_t)); |
| 738 | level2_syms = (int16_t *)(buf + dist->alphabet_size * (2 * sizeof(int8_t))); | 740 | level2_syms = (int16_t *)(buf + MAX_PREFIX_ALPHABET_SIZE * (2 * sizeof(int8_t))); |
| 739 | level2_codecounts = (uint32_t *)(buf + dist->alphabet_size * (2 * sizeof(int8_t) + sizeof(int16_t))); | 741 | level2_codecounts = (uint32_t *)(buf + MAX_PREFIX_ALPHABET_SIZE * (2 * sizeof(int8_t) + sizeof(int16_t))); |
| 740 | 742 | ||
| 741 | total_code = 0; | 743 | total_code = 0; |
| 742 | for (int i = 0; i < dist->alphabet_size; i++) { | 744 | for (int i = 0; i < dist->alphabet_size; i++) { |
| 743 | len = get_vlc2(gb, level1_vlc.table, 5, 1); | 745 | len = get_vlc2(gb, level1_vlc.table, 5, 1); |
| 746 | if (get_bits_left(gb) < 0) { | ||
| 747 | ret = AVERROR_BUFFER_TOO_SMALL; | ||
| 748 | goto end; | ||
| 749 | } | ||
| 744 | if (len == 16) { | 750 | if (len == 16) { |
| 745 | int extra = 3 + get_bits(gb, 2); | 751 | int extra = 3 + get_bits(gb, 2); |
| 746 | if (repeat_count_prev) | 752 | if (repeat_count_prev) |
| 747 | extra = 4 * (repeat_count_prev - 2) - repeat_count_prev + extra; | 753 | extra += 4 * (repeat_count_prev - 2) - repeat_count_prev; |
| 748 | if (i + extra > dist->alphabet_size) { | 754 | extra = FFMIN(extra, dist->alphabet_size - i); |
| 749 | ret = AVERROR_INVALIDDATA; | ||
| 750 | goto end; | ||
| 751 | } | ||
| 752 | for (int j = 0; j < extra; j++) | 755 | for (int j = 0; j < extra; j++) |
| 753 | level2_lens[i + j] = prev; | 756 | level2_lens[i + j] = prev; |
| 754 | total_code += (32768 >> prev) * extra; | 757 | total_code += (32768 >> prev) * extra; |
| @@ -759,7 +762,8 @@ static int read_vlc_prefix(GetBitContext *gb, JXLEntropyDecoder *dec, JXLSymbolD · 4 unmodified lines | |||
| 759 | } else if (len == 17) { | 762 | } else if (len == 17) { |
| 760 | int extra = 3 + get_bits(gb, 3); | 763 | int extra = 3 + get_bits(gb, 3); |
| 761 | if (repeat_count_zero > 0) | 764 | if (repeat_count_zero > 0) |
| 762 | extra = 8 * (repeat_count_zero - 2) - repeat_count_zero + extra; | 765 | extra += 8 * (repeat_count_zero - 2) - repeat_count_zero; |
| 766 | extra = FFMIN(extra, dist->alphabet_size - i); | ||
| 763 | i += extra - 1; | 767 | i += extra - 1; |
| 764 | repeat_count_prev = 0; | 768 | repeat_count_prev = 0; |
| 765 | repeat_count_zero += extra; | 769 | repeat_count_zero += extra; |