Fixes: signed integer overflow: -895002 * 2400 cannot be represented in type 'int' Fixes: 26052/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MV30_fuzzer-5431812577558528
Unified · Split
| @@ -198,12 +198,12 @@ static void idct_add(uint8_t *dst, int stride, · 197 unmodified lines | ||
| 198 | 198 | |
| 199 | 199 | static inline void idct2_1d(int *blk, int step) |
| 200 | 200 | { |
| 201 | − const int t0 = blk[0 * step]; | |
| 202 | − const int t1 = blk[1 * step]; | |
| 203 | − const int t2 = (int)(t1 * 473U) >> 8; | |
| 204 | − const int t3 = t2 - t1; | |
| 205 | − const int t4 = ((int)(t1 * 362U) >> 8) - t3; | |
| 206 | − const int t5 = (((int)(t1 * 277U) >> 8) - t2) + t4; | |
| 201 | + const unsigned int t0 = blk[0 * step]; | |
| 202 | + const unsigned int t1 = blk[1 * step]; | |
| 203 | + const unsigned int t2 = (int)(t1 * 473U) >> 8; | |
| 204 | + const unsigned int t3 = t2 - t1; | |
| 205 | + const unsigned int t4 = ((int)(t1 * 362U) >> 8) - t3; | |
| 206 | + const unsigned int t5 = (((int)(t1 * 277U) >> 8) - t2) + t4; | |
| 207 | 207 | |
| 208 | 208 | blk[0 * step] = t1 + t0; |
| 209 | 209 | blk[1 * step] = t0 + t3; |
| @@ -305,14 +305,14 @@ static int decode_intra_block(AVCodecContext *avctx, int mode, · 95 unmodified lines | ||
| 305 | 305 | case 1: |
| 306 | 306 | fill = sign_extend(bytestream2_get_ne16(gbyte), 16); |
| 307 | 307 | pfill[0] += fill; |
| 308 | − block[0] = ((pfill[0] * qtab[0]) >> 5) + 128; | |
| 308 | + block[0] = ((int)((unsigned)pfill[0] * qtab[0]) >> 5) + 128; | |
| 309 | 309 | s->bdsp.fill_block_tab[1](dst, block[0], linesize, 8); |
| 310 | 310 | break; |
| 311 | 311 | case 2: |
| 312 | 312 | memset(block, 0, sizeof(*block) * 64); |
| 313 | 313 | fill = sign_extend(bytestream2_get_ne16(gbyte), 16); |
| 314 | 314 | pfill[0] += fill; |
| 315 | − block[0] = pfill[0] * qtab[0]; | |
| 315 | + block[0] = (unsigned)pfill[0] * qtab[0]; | |
| 316 | 316 | block[1] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[1]; |
| 317 | 317 | block[8] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[8]; |
| 318 | 318 | block[9] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[9]; |
| @@ -321,7 +321,7 @@ static int decode_intra_block(AVCodecContext *avctx, int mode, · 2 unmodified lines | ||
| 321 | 321 | case 3: |
| 322 | 322 | fill = sign_extend(bytestream2_get_ne16(gbyte), 16); |
| 323 | 323 | pfill[0] += fill; |
| 324 | − block[0] = pfill[0] * qtab[0]; | |
| 324 | + block[0] = (unsigned)pfill[0] * qtab[0]; | |
| 325 | 325 | for (int i = 1; i < 64; i++) |
| 326 | 326 | block[zigzag[i]] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[zigzag[i]]; |
| 327 | 327 | idct_put(dst, linesize, block); |
| @@ -346,14 +346,14 @@ static int decode_inter_block(AVCodecContext *avctx, int mode, · 18 unmodified lines | ||
| 346 | 346 | case 1: |
| 347 | 347 | fill = sign_extend(bytestream2_get_ne16(gbyte), 16); |
| 348 | 348 | pfill[0] += fill; |
| 349 | − block[0] = (pfill[0] * qtab[0]) >> 5; | |
| 349 | + block[0] = (int)((unsigned)pfill[0] * qtab[0]) >> 5; | |
| 350 | 350 | update_inter_block(dst, linesize, src, in_linesize, block[0]); |
| 351 | 351 | break; |
| 352 | 352 | case 2: |
| 353 | 353 | memset(block, 0, sizeof(*block) * 64); |
| 354 | 354 | fill = sign_extend(bytestream2_get_ne16(gbyte), 16); |
| 355 | 355 | pfill[0] += fill; |
| 356 | − block[0] = pfill[0] * qtab[0]; | |
| 356 | + block[0] = (unsigned)pfill[0] * qtab[0]; | |
| 357 | 357 | block[1] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[1]; |
| 358 | 358 | block[8] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[8]; |
| 359 | 359 | block[9] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[9]; |
| @@ -362,7 +362,7 @@ static int decode_inter_block(AVCodecContext *avctx, int mode, · 2 unmodified lines | ||
| 362 | 362 | case 3: |
| 363 | 363 | fill = sign_extend(bytestream2_get_ne16(gbyte), 16); |
| 364 | 364 | pfill[0] += fill; |
| 365 | − block[0] = pfill[0] * qtab[0]; | |
| 365 | + block[0] = (unsigned)pfill[0] * qtab[0]; | |
| 366 | 366 | for (int i = 1; i < 64; i++) |
| 367 | 367 | block[zigzag[i]] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[zigzag[i]]; |
| 368 | 368 | idct_add(dst, linesize, src, in_linesize, block); |