Fixes: signed integer overflow: -1403461578 + -843974775 cannot be represented in type 'int' Fixes: 60868/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MPEG1VIDEO_fuzzer-4599793035378688
Showing the changes to libavcodec (1 of 1 file). Show all
Unified · Split
| @@ -69,24 +69,24 @@ static int idct_row(short *in, const int *const tab, int rnd) · 68 unmodified lines | |||
| 69 | if (!(right | in[4])) { | 69 | if (!(right | in[4])) { |
| 70 | const int k = c4 * in[0] + rnd; | 70 | const int k = c4 * in[0] + rnd; |
| 71 | if (left) { | 71 | if (left) { |
| 72 | const int a0 = k + c2 * in[2]; | 72 | const unsigned a0 = k + c2 * in[2]; |
| 73 | const int a1 = k + c6 * in[2]; | 73 | const unsigned a1 = k + c6 * in[2]; |
| 74 | const int a2 = k - c6 * in[2]; | 74 | const unsigned a2 = k - c6 * in[2]; |
| 75 | const int a3 = k - c2 * in[2]; | 75 | const unsigned a3 = k - c2 * in[2]; |
| 76 | 76 | ||
| 77 | const int b0 = c1 * in[1] + c3 * in[3]; | 77 | const int b0 = c1 * in[1] + c3 * in[3]; |
| 78 | const int b1 = c3 * in[1] - c7 * in[3]; | 78 | const int b1 = c3 * in[1] - c7 * in[3]; |
| 79 | const int b2 = c5 * in[1] - c1 * in[3]; | 79 | const int b2 = c5 * in[1] - c1 * in[3]; |
| 80 | const int b3 = c7 * in[1] - c5 * in[3]; | 80 | const int b3 = c7 * in[1] - c5 * in[3]; |
| 81 | 81 | ||
| 82 | in[0] = (a0 + b0) >> ROW_SHIFT; | 82 | in[0] = (int)(a0 + b0) >> ROW_SHIFT; |
| 83 | in[1] = (a1 + b1) >> ROW_SHIFT; | 83 | in[1] = (int)(a1 + b1) >> ROW_SHIFT; |
| 84 | in[2] = (a2 + b2) >> ROW_SHIFT; | 84 | in[2] = (int)(a2 + b2) >> ROW_SHIFT; |
| 85 | in[3] = (a3 + b3) >> ROW_SHIFT; | 85 | in[3] = (int)(a3 + b3) >> ROW_SHIFT; |
| 86 | in[4] = (a3 - b3) >> ROW_SHIFT; | 86 | in[4] = (int)(a3 - b3) >> ROW_SHIFT; |
| 87 | in[5] = (a2 - b2) >> ROW_SHIFT; | 87 | in[5] = (int)(a2 - b2) >> ROW_SHIFT; |
| 88 | in[6] = (a1 - b1) >> ROW_SHIFT; | 88 | in[6] = (int)(a1 - b1) >> ROW_SHIFT; |
| 89 | in[7] = (a0 - b0) >> ROW_SHIFT; | 89 | in[7] = (int)(a0 - b0) >> ROW_SHIFT; |
| 90 | } else { | 90 | } else { |
| 91 | const int a0 = k >> ROW_SHIFT; | 91 | const int a0 = k >> ROW_SHIFT; |
| 92 | if (a0) { | 92 | if (a0) { |