avcodec/dca: Fix multiple runtime error: signed integer overflow

Fixes: 680/clusterfuzz-testcase-5416627266912256 Fixes: 681/clusterfuzz-testcase-5013323462475776

Michael Niedermayer committed 9 years ago

Commit 949d2171 parent 63e400a2 files · +3 −3Browse files

Unified · Split

2 files

libavcodec/dca_xll.c +2 −2
libavcodec/dcadsp.c +1 −1

libavcodec/dca_xll.c+2−2

@@ -652,7 +652,7 @@ static void chs_filter_band_data(DCAXllDecoder s, DCAXllChSet c, int band) · 651 unmodified lines
652	int64_t err = 0;	652	int64_t err = 0;
653	for (k = 0; k < order; k++)	653	for (k = 0; k < order; k++)
654	err += (int64_t)buf[j + k] * coeff[order - k - 1];	654	err += (int64_t)buf[j + k] * coeff[order - k - 1];
655	buf[j + k] -= clip23(norm16(err));	655	buf[j + k] -= (SUINT)clip23(norm16(err));
656	}	656	}
657	} else {	657	} else {
658	// Inverse fixed coefficient prediction	658	// Inverse fixed coefficient prediction
@@ -1308,7 +1308,7 @@ static int combine_residual_frame(DCAXllDecoder s, DCAXllChSet c) · 649 unmodified lines
1308	// Undo embedded core downmix pre-scaling	1308	// Undo embedded core downmix pre-scaling
1309	int scale_inv = o->dmix_scale_inv[c->hier_ofs + ch];	1309	int scale_inv = o->dmix_scale_inv[c->hier_ofs + ch];
1310	for (n = 0; n < nsamples; n++)	1310	for (n = 0; n < nsamples; n++)
1311	dst[n] += clip23((mul16(src[n], scale_inv) + round) >> shift);	1311	dst[n] += (SUINT)clip23((mul16(src[n], scale_inv) + round) >> shift);
1312	} else {	1312	} else {
1313	// No downmix scaling	1313	// No downmix scaling
1314	for (n = 0; n < nsamples; n++)	1314	for (n = 0; n < nsamples; n++)

libavcodec/dcadsp.c+1−1

@@ -300,7 +300,7 @@ static void decor_c(int32_t dst, const int32_t src, int coeff, ptrdiff_t len) · 299 unmodified lines
300	int i;	300	int i;
301		301
302	for (i = 0; i < len; i++)	302	for (i = 0; i < len; i++)
303	dst[i] += (int)(src[i] * (SUINT)coeff + (1 << 2)) >> 3;	303	dst[i] += (SUINT)((int)(src[i] * (SUINT)coeff + (1 << 2)) >> 3);
304	}	304	}
305		305
306	static void dmix_sub_xch_c(int32_t dst1, int32_t dst2,	306	static void dmix_sub_xch_c(int32_t dst1, int32_t dst2,