Fixes: signed integer overflow: 998938090 + 1169275991 cannot be represented in type 'int' Fixes: 23411/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_VP9_fuzzer-4644692330545152
Unified · Split
| @@ -1260,20 +1260,20 @@ static av_always_inline void iadst8_1d(const dctcoef *in, ptrdiff_t stride, · 1259 unmodified lines | |||
| 1260 | t6 = (t2a - t6a + (1 << 13)) >> 14; | 1260 | t6 = (t2a - t6a + (1 << 13)) >> 14; |
| 1261 | t7 = (t3a - t7a + (1 << 13)) >> 14; | 1261 | t7 = (t3a - t7a + (1 << 13)) >> 14; |
| 1262 | 1262 | ||
| 1263 | t4a = 15137 * t4 + 6270 * t5; | 1263 | t4a = 15137U * t4 + 6270U * t5; |
| 1264 | t5a = 6270 * t4 - 15137 * t5; | 1264 | t5a = 6270U * t4 - 15137U * t5; |
| 1265 | t6a = 15137 * t7 - 6270 * t6; | 1265 | t6a = 15137U * t7 - 6270U * t6; |
| 1266 | t7a = 6270 * t7 + 15137 * t6; | 1266 | t7a = 6270U * t7 + 15137U * t6; |
| 1267 | 1267 | ||
| 1268 | out[0] = t0 + t2; | 1268 | out[0] = t0 + t2; |
| 1269 | out[7] = -(t1 + t3); | 1269 | out[7] = -(t1 + t3); |
| 1270 | t2 = t0 - t2; | 1270 | t2 = t0 - t2; |
| 1271 | t3 = t1 - t3; | 1271 | t3 = t1 - t3; |
| 1272 | 1272 | ||
| 1273 | out[1] = -((t4a + t6a + (1 << 13)) >> 14); | 1273 | out[1] = -((dctint)((1U << 13) + t4a + t6a) >> 14); |
| 1274 | out[6] = (t5a + t7a + (1 << 13)) >> 14; | 1274 | out[6] = (dctint)((1U << 13) + t5a + t7a) >> 14; |
| 1275 | t6 = (t4a - t6a + (1 << 13)) >> 14; | 1275 | t6 = (dctint)((1U << 13) + t4a - t6a) >> 14; |
| 1276 | t7 = (t5a - t7a + (1 << 13)) >> 14; | 1276 | t7 = (dctint)((1U << 13) + t5a - t7a) >> 14; |
| 1277 | 1277 | ||
| 1278 | out[3] = -(((t2 + t3) * 11585 + (1 << 13)) >> 14); | 1278 | out[3] = -(((t2 + t3) * 11585 + (1 << 13)) >> 14); |
| 1279 | out[4] = ((t2 - t3) * 11585 + (1 << 13)) >> 14; | 1279 | out[4] = ((t2 - t3) * 11585 + (1 << 13)) >> 14; |