History

avcodec/jpeg2000htdec: check if block decoding will exceed internal precision

Intended to replace https://patchwork.ffmpeg.org/project/ffmpeg/patch/20230802000135.26482-3-michael@niedermayer.cc/ with a more accurate block decoding magnitude bound.

Pierre-Anthony Lemieux committed 3 years ago

Commit a1384b41 parent aa1e7681 file · +36 −6Browse files

Unified · Split

libavcodec/jpeg2000htdec.c+36−6
@@ -567,11 +567,19 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s, · 566 unmodified lines
567567
568568 uint64_t c;
569569
570− uint8_t *sigma;
571− uint32_t *mu;
570+ uint8_t *sigma, *sigma_n, *E;
571+ uint32_t *mu, *mu_n;
572572
573573 const uint8_t *vlc_buf = Dcup + Pcup;
574574
575+ /*
576+ * Bound on the precision needed to process the codeblock. The number of
577+ * decoded bit planes is equal to at most cblk->zbp + 2 since S_blk = P if
578+ * there are no placeholder passes or HT Sets and P = cblk->zbp. See Rec.
579+ * ITU-T T.814, 7.6.
580+ */
581+ int maxbp = cblk->zbp + 2;
582+
575583 /* convert to raster-scan */
576584 const uint16_t is_border_x = width % 2;
577585 const uint16_t is_border_y = height % 2;
@@ -581,9 +589,13 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s, · 3 unmodified lines
581589
582590 size_t buf_size = 4 * quad_width * quad_height;
583591
584− uint8_t *sigma_n = av_calloc(buf_size, sizeof(uint8_t));
585− uint8_t *E = av_calloc(buf_size, sizeof(uint8_t));
586− uint32_t *mu_n = av_calloc(buf_size, sizeof(uint32_t));
592+ /* do we have enough precision, assuming a 32-bit decoding path */
593+ if (maxbp >= 32)
594+ return AVERROR_INVALIDDATA;
595+
596+ sigma_n = av_calloc(buf_size, sizeof(uint8_t));
597+ E = av_calloc(buf_size, sizeof(uint8_t));
598+ mu_n = av_calloc(buf_size, sizeof(uint32_t));
587599
588600 if (!sigma_n || !E || !mu_n) {
589601 ret = AVERROR(ENOMEM);
@@ -676,6 +688,10 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s, · 86 unmodified lines
676688 }
677689 U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];
678690 U[J2K_Q2] = kappa[J2K_Q2] + u[J2K_Q2];
691+ if (U[J2K_Q1] > maxbp || U[J2K_Q2] > maxbp) {
692+ ret = AVERROR_INVALIDDATA;
693+ goto free;
694+ }
679695
680696 for (int i = 0; i < 4; i++) {
681697 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);
@@ -713,6 +729,10 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s, · 31 unmodified lines
713729 }
714730
715731 U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];
732+ if (U[J2K_Q1] > maxbp) {
733+ ret = AVERROR_INVALIDDATA;
734+ goto free;
735+ }
716736
717737 for (int i = 0; i < 4; i++)
718738 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);
@@ -842,6 +862,10 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s, · 123 unmodified lines
842862
843863 U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];
844864 U[J2K_Q2] = kappa[J2K_Q2] + u[J2K_Q2];
865+ if (U[J2K_Q1] > maxbp || U[J2K_Q2] > maxbp) {
866+ ret = AVERROR_INVALIDDATA;
867+ goto free;
868+ }
845869
846870 for (int i = 0; i < 4; i++) {
847871 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);
@@ -910,6 +934,10 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s, · 62 unmodified lines
910934 kappa[J2K_Q1] = FFMAX(1, gamma[J2K_Q1] * (max_e[J2K_Q1] - 1));
911935
912936 U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];
937+ if (U[J2K_Q1] > maxbp) {
938+ ret = AVERROR_INVALIDDATA;
939+ goto free;
940+ }
913941
914942 for (int i = 0; i < 4; i++)
915943 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);
@@ -1238,8 +1266,10 @@ ff_jpeg2000_decode_htj2k(const Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *c · 322 unmodified lines
12381266 }
12391267 if ((ret = jpeg2000_decode_ht_cleanup_segment(s, cblk, t1, &mel_state, &mel, &vlc,
12401268 &mag_sgn, Dcup, Lcup, Pcup, pLSB, width,
1241− height, sample_buf, block_states)) < 0)
1269+ height, sample_buf, block_states)) < 0) {
1270+ av_log(s->avctx, AV_LOG_ERROR, "Bad HT cleanup segment\n");
12421271 goto free;
1272+ }
12431273
12441274 if (cblk->npasses > 1)
12451275 jpeg2000_decode_sigprop_segment(cblk, width, height, Dref, Lref,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t