Fixes: signed integer overflow: 998938090 + 1169275991 cannot be represented in type 'int' Fixes: 23411/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_VP9_fuzzer-4644692330545152
Unified · Split
| @@ -1260,20 +1260,20 @@ static av_always_inline void iadst8_1d(const dctcoef *in, ptrdiff_t stride, · 1259 unmodified lines | ||
| 1260 | 1260 | t6 = (t2a - t6a + (1 << 13)) >> 14; |
| 1261 | 1261 | t7 = (t3a - t7a + (1 << 13)) >> 14; |
| 1262 | 1262 | |
| 1263 | − t4a = 15137 * t4 + 6270 * t5; | |
| 1264 | − t5a = 6270 * t4 - 15137 * t5; | |
| 1265 | − t6a = 15137 * t7 - 6270 * t6; | |
| 1266 | − t7a = 6270 * t7 + 15137 * t6; | |
| 1263 | + t4a = 15137U * t4 + 6270U * t5; | |
| 1264 | + t5a = 6270U * t4 - 15137U * t5; | |
| 1265 | + t6a = 15137U * t7 - 6270U * t6; | |
| 1266 | + t7a = 6270U * t7 + 15137U * t6; | |
| 1267 | 1267 | |
| 1268 | 1268 | out[0] = t0 + t2; |
| 1269 | 1269 | out[7] = -(t1 + t3); |
| 1270 | 1270 | t2 = t0 - t2; |
| 1271 | 1271 | t3 = t1 - t3; |
| 1272 | 1272 | |
| 1273 | − out[1] = -((t4a + t6a + (1 << 13)) >> 14); | |
| 1274 | − out[6] = (t5a + t7a + (1 << 13)) >> 14; | |
| 1275 | − t6 = (t4a - t6a + (1 << 13)) >> 14; | |
| 1276 | − t7 = (t5a - t7a + (1 << 13)) >> 14; | |
| 1273 | + out[1] = -((dctint)((1U << 13) + t4a + t6a) >> 14); | |
| 1274 | + out[6] = (dctint)((1U << 13) + t5a + t7a) >> 14; | |
| 1275 | + t6 = (dctint)((1U << 13) + t4a - t6a) >> 14; | |
| 1276 | + t7 = (dctint)((1U << 13) + t5a - t7a) >> 14; | |
| 1277 | 1277 | |
| 1278 | 1278 | out[3] = -(((t2 + t3) * 11585 + (1 << 13)) >> 14); |
| 1279 | 1279 | out[4] = ((t2 - t3) * 11585 + (1 << 13)) >> 14; |