History

avcodec/jpeg2000htdec: check if block decoding will exceed internal precision

Intended to replace https://patchwork.ffmpeg.org/project/ffmpeg/patch/20230802000135.26482-3-michael@niedermayer.cc/ with a more accurate block decoding magnitude bound.

Pierre-Anthony Lemieux committed 3 years ago

Commit a1384b41 parent aa1e7681 file · +36 −6Browse files

Unified · Split

libavcodec/jpeg2000htdec.c+36−6
@@ -567,11 +567,19 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s, · 566 unmodified lines
567 567
568 uint64_t c;568 uint64_t c;
569 569
570 uint8_t *sigma;570 uint8_t *sigma, *sigma_n, *E;
571 uint32_t *mu;571 uint32_t *mu, *mu_n;
572 572
573 const uint8_t *vlc_buf = Dcup + Pcup;573 const uint8_t *vlc_buf = Dcup + Pcup;
574 574
575 /*
576 * Bound on the precision needed to process the codeblock. The number of
577 * decoded bit planes is equal to at most cblk->zbp + 2 since S_blk = P if
578 * there are no placeholder passes or HT Sets and P = cblk->zbp. See Rec.
579 * ITU-T T.814, 7.6.
580 */
581 int maxbp = cblk->zbp + 2;
582
575 /* convert to raster-scan */583 /* convert to raster-scan */
576 const uint16_t is_border_x = width % 2;584 const uint16_t is_border_x = width % 2;
577 const uint16_t is_border_y = height % 2;585 const uint16_t is_border_y = height % 2;
@@ -581,9 +589,13 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s, · 3 unmodified lines
581 589
582 size_t buf_size = 4 * quad_width * quad_height;590 size_t buf_size = 4 * quad_width * quad_height;
583 591
584 uint8_t *sigma_n = av_calloc(buf_size, sizeof(uint8_t));592 /* do we have enough precision, assuming a 32-bit decoding path */
585 uint8_t *E = av_calloc(buf_size, sizeof(uint8_t));593 if (maxbp >= 32)
586 uint32_t *mu_n = av_calloc(buf_size, sizeof(uint32_t));594 return AVERROR_INVALIDDATA;
595
596 sigma_n = av_calloc(buf_size, sizeof(uint8_t));
597 E = av_calloc(buf_size, sizeof(uint8_t));
598 mu_n = av_calloc(buf_size, sizeof(uint32_t));
587 599
588 if (!sigma_n || !E || !mu_n) {600 if (!sigma_n || !E || !mu_n) {
589 ret = AVERROR(ENOMEM);601 ret = AVERROR(ENOMEM);
@@ -676,6 +688,10 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s, · 86 unmodified lines
676 }688 }
677 U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];689 U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];
678 U[J2K_Q2] = kappa[J2K_Q2] + u[J2K_Q2];690 U[J2K_Q2] = kappa[J2K_Q2] + u[J2K_Q2];
691 if (U[J2K_Q1] > maxbp || U[J2K_Q2] > maxbp) {
692 ret = AVERROR_INVALIDDATA;
693 goto free;
694 }
679 695
680 for (int i = 0; i < 4; i++) {696 for (int i = 0; i < 4; i++) {
681 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);697 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);
@@ -713,6 +729,10 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s, · 31 unmodified lines
713 }729 }
714 730
715 U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];731 U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];
732 if (U[J2K_Q1] > maxbp) {
733 ret = AVERROR_INVALIDDATA;
734 goto free;
735 }
716 736
717 for (int i = 0; i < 4; i++)737 for (int i = 0; i < 4; i++)
718 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);738 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);
@@ -842,6 +862,10 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s, · 123 unmodified lines
842 862
843 U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];863 U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];
844 U[J2K_Q2] = kappa[J2K_Q2] + u[J2K_Q2];864 U[J2K_Q2] = kappa[J2K_Q2] + u[J2K_Q2];
865 if (U[J2K_Q1] > maxbp || U[J2K_Q2] > maxbp) {
866 ret = AVERROR_INVALIDDATA;
867 goto free;
868 }
845 869
846 for (int i = 0; i < 4; i++) {870 for (int i = 0; i < 4; i++) {
847 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);871 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);
@@ -910,6 +934,10 @@ static int jpeg2000_decode_ht_cleanup_segment(const Jpeg2000DecoderContext *s, · 62 unmodified lines
910 kappa[J2K_Q1] = FFMAX(1, gamma[J2K_Q1] * (max_e[J2K_Q1] - 1));934 kappa[J2K_Q1] = FFMAX(1, gamma[J2K_Q1] * (max_e[J2K_Q1] - 1));
911 935
912 U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];936 U[J2K_Q1] = kappa[J2K_Q1] + u[J2K_Q1];
937 if (U[J2K_Q1] > maxbp) {
938 ret = AVERROR_INVALIDDATA;
939 goto free;
940 }
913 941
914 for (int i = 0; i < 4; i++)942 for (int i = 0; i < 4; i++)
915 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);943 m[J2K_Q1][i] = sigma_n[4 * q1 + i] * U[J2K_Q1] - ((emb_pat_k[J2K_Q1] >> i) & 1);
@@ -1238,8 +1266,10 @@ ff_jpeg2000_decode_htj2k(const Jpeg2000DecoderContext *s, Jpeg2000CodingStyle *c · 322 unmodified lines
1238 }1266 }
1239 if ((ret = jpeg2000_decode_ht_cleanup_segment(s, cblk, t1, &mel_state, &mel, &vlc,1267 if ((ret = jpeg2000_decode_ht_cleanup_segment(s, cblk, t1, &mel_state, &mel, &vlc,
1240 &mag_sgn, Dcup, Lcup, Pcup, pLSB, width,1268 &mag_sgn, Dcup, Lcup, Pcup, pLSB, width,
1241 height, sample_buf, block_states)) < 0)1269 height, sample_buf, block_states)) < 0) {
1270 av_log(s->avctx, AV_LOG_ERROR, "Bad HT cleanup segment\n");
1242 goto free;1271 goto free;
1272 }
1243 1273
1244 if (cblk->npasses > 1)1274 if (cblk->npasses > 1)
1245 jpeg2000_decode_sigprop_segment(cblk, width, height, Dref, Lref,1275 jpeg2000_decode_sigprop_segment(cblk, width, height, Dref, Lref,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t