Fixes: signed integer overflow: -1403461578 + -843974775 cannot be represented in type 'int' Fixes: 60868/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MPEG1VIDEO_fuzzer-4599793035378688
Showing the changes to libavcodec (1 of 1 file). Show all
Unified · Split
| @@ -69,24 +69,24 @@ static int idct_row(short *in, const int *const tab, int rnd) · 68 unmodified lines | ||
| 69 | 69 | if (!(right | in[4])) { |
| 70 | 70 | const int k = c4 * in[0] + rnd; |
| 71 | 71 | if (left) { |
| 72 | − const int a0 = k + c2 * in[2]; | |
| 73 | − const int a1 = k + c6 * in[2]; | |
| 74 | − const int a2 = k - c6 * in[2]; | |
| 75 | − const int a3 = k - c2 * in[2]; | |
| 72 | + const unsigned a0 = k + c2 * in[2]; | |
| 73 | + const unsigned a1 = k + c6 * in[2]; | |
| 74 | + const unsigned a2 = k - c6 * in[2]; | |
| 75 | + const unsigned a3 = k - c2 * in[2]; | |
| 76 | 76 | |
| 77 | 77 | const int b0 = c1 * in[1] + c3 * in[3]; |
| 78 | 78 | const int b1 = c3 * in[1] - c7 * in[3]; |
| 79 | 79 | const int b2 = c5 * in[1] - c1 * in[3]; |
| 80 | 80 | const int b3 = c7 * in[1] - c5 * in[3]; |
| 81 | 81 | |
| 82 | − in[0] = (a0 + b0) >> ROW_SHIFT; | |
| 83 | − in[1] = (a1 + b1) >> ROW_SHIFT; | |
| 84 | − in[2] = (a2 + b2) >> ROW_SHIFT; | |
| 85 | − in[3] = (a3 + b3) >> ROW_SHIFT; | |
| 86 | − in[4] = (a3 - b3) >> ROW_SHIFT; | |
| 87 | − in[5] = (a2 - b2) >> ROW_SHIFT; | |
| 88 | − in[6] = (a1 - b1) >> ROW_SHIFT; | |
| 89 | − in[7] = (a0 - b0) >> ROW_SHIFT; | |
| 82 | + in[0] = (int)(a0 + b0) >> ROW_SHIFT; | |
| 83 | + in[1] = (int)(a1 + b1) >> ROW_SHIFT; | |
| 84 | + in[2] = (int)(a2 + b2) >> ROW_SHIFT; | |
| 85 | + in[3] = (int)(a3 + b3) >> ROW_SHIFT; | |
| 86 | + in[4] = (int)(a3 - b3) >> ROW_SHIFT; | |
| 87 | + in[5] = (int)(a2 - b2) >> ROW_SHIFT; | |
| 88 | + in[6] = (int)(a1 - b1) >> ROW_SHIFT; | |
| 89 | + in[7] = (int)(a0 - b0) >> ROW_SHIFT; | |
| 90 | 90 | } else { |
| 91 | 91 | const int a0 = k >> ROW_SHIFT; |
| 92 | 92 | if (a0) { |