History

avcodec/jpeg2000htdec: Check magp before using it in a shift

Fixes: shift exponent -1 is negative Fixes: 65378/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_JPEG2000_fuzzer-5457678193197056

Michael Niedermayer committed 2 years ago

Commit 19ad05e1 parent 63f56c91 file · +12 −4Browse files

Unified · Split

libavcodec/jpeg2000dec.c+12−4
@@ -1886,7 +1886,7 @@ static inline void roi_scale_cblk(Jpeg2000Cblk *cblk, · 1885 unmodified lines
1886 }1886 }
1887}1887}
1888 1888
1889static inline void tile_codeblocks(const Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)1889static inline int tile_codeblocks(const Jpeg2000DecoderContext *s, Jpeg2000Tile *tile)
1890{1890{
1891 Jpeg2000T1Context t1;1891 Jpeg2000T1Context t1;
1892 1892
@@ -1911,6 +1911,8 @@ static inline void tile_codeblocks(const Jpeg2000DecoderContext *s, Jpeg2000Tile · 18 unmodified lines
1911 int nb_precincts, precno;1911 int nb_precincts, precno;
1912 Jpeg2000Band *band = rlevel->band + bandno;1912 Jpeg2000Band *band = rlevel->band + bandno;
1913 int cblkno = 0, bandpos;1913 int cblkno = 0, bandpos;
1914 /* See Rec. ITU-T T.800, Equation E-2 */
1915 int magp = quantsty->expn[subbandno] + quantsty->nguardbits - 1;
1914 1916
1915 bandpos = bandno + (reslevelno > 0);1917 bandpos = bandno + (reslevelno > 0);
1916 1918
@@ -1918,6 +1920,11 @@ static inline void tile_codeblocks(const Jpeg2000DecoderContext *s, Jpeg2000Tile · 1 unmodified line
1918 band->coord[1][0] == band->coord[1][1])1920 band->coord[1][0] == band->coord[1][1])
1919 continue;1921 continue;
1920 1922
1923 if ((codsty->cblk_style & JPEG2000_CTSY_HTJ2K_F) && magp >= 31) {
1924 avpriv_request_sample(s->avctx, "JPEG2000_CTSY_HTJ2K_F and magp >= 31");
1925 return AVERROR_PATCHWELCOME;
1926 }
1927
1921 nb_precincts = rlevel->num_precincts_x * rlevel->num_precincts_y;1928 nb_precincts = rlevel->num_precincts_x * rlevel->num_precincts_y;
1922 /* Loop on precincts */1929 /* Loop on precincts */
1923 for (precno = 0; precno < nb_precincts; precno++) {1930 for (precno = 0; precno < nb_precincts; precno++) {
@@ -1928,8 +1935,6 @@ static inline void tile_codeblocks(const Jpeg2000DecoderContext *s, Jpeg2000Tile · 4 unmodified lines
1928 cblkno < prec->nb_codeblocks_width * prec->nb_codeblocks_height;1935 cblkno < prec->nb_codeblocks_width * prec->nb_codeblocks_height;
1929 cblkno++) {1936 cblkno++) {
1930 int x, y, ret;1937 int x, y, ret;
1931 /* See Rec. ITU-T T.800, Equation E-2 */
1932 int magp = quantsty->expn[subbandno] + quantsty->nguardbits - 1;
1933 1938
1934 Jpeg2000Cblk *cblk = prec->cblk + cblkno;1939 Jpeg2000Cblk *cblk = prec->cblk + cblkno;
1935 1940
@@ -1969,6 +1974,7 @@ static inline void tile_codeblocks(const Jpeg2000DecoderContext *s, Jpeg2000Tile · 33 unmodified lines
1969 ff_dwt_decode(&comp->dwt, codsty->transform == FF_DWT97 ? (void*)comp->f_data : (void*)comp->i_data);1974 ff_dwt_decode(&comp->dwt, codsty->transform == FF_DWT97 ? (void*)comp->f_data : (void*)comp->i_data);
1970 1975
1971 } /*end comp */1976 } /*end comp */
1977 return 0;
1972}1978}
1973 1979
1974#define WRITE_FRAME(D, PIXEL) \1980#define WRITE_FRAME(D, PIXEL) \
@@ -2045,7 +2051,9 @@ static int jpeg2000_decode_tile(AVCodecContext *avctx, void *td, · 70 unmodified lines
2045 AVFrame *picture = td;2051 AVFrame *picture = td;
2046 Jpeg2000Tile *tile = s->tile + jobnr;2052 Jpeg2000Tile *tile = s->tile + jobnr;
2047 2053
2048 tile_codeblocks(s, tile);2054 int ret = tile_codeblocks(s, tile);
2055 if (ret < 0)
2056 return ret;
2049 2057
2050 /* inverse MCT transformation */2058 /* inverse MCT transformation */
2051 if (tile->codsty[0].mct)2059 if (tile->codsty[0].mct)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t