swscale/output: Fix integer overflow in yuv2rgb_write_full() with out of range input

Fixes: signed integer overflow: 1169365504 + 981452800 cannot be represented in type 'int' Fixes: ticket8293

Michael Niedermayer committed 6 years ago

Commit e057e831 parent 49ba1871 file · +3 −3Browse files

Unified · Split

libswscale/output.c+3−3

@@ -1850,9 +1850,9 @@ static av_always_inline void yuv2rgb_write_full(SwsContext *c, · 1849 unmodified lines
1850	Y -= c->yuv2rgb_y_offset;	1850	Y -= c->yuv2rgb_y_offset;
1851	Y *= c->yuv2rgb_y_coeff;	1851	Y *= c->yuv2rgb_y_coeff;
1852	Y += 1 << 21;	1852	Y += 1 << 21;
1853	R = Y + V*c->yuv2rgb_v2r_coeff;	1853	R = (unsigned)Y + V*c->yuv2rgb_v2r_coeff;
1854	G = Y + Vc->yuv2rgb_v2g_coeff + Uc->yuv2rgb_u2g_coeff;	1854	G = (unsigned)Y + Vc->yuv2rgb_v2g_coeff + Uc->yuv2rgb_u2g_coeff;
1855	B = Y + U*c->yuv2rgb_u2b_coeff;	1855	B = (unsigned)Y + U*c->yuv2rgb_u2b_coeff;
1856	if ((R \| G \| B) & 0xC0000000) {	1856	if ((R \| G \| B) & 0xC0000000) {
1857	R = av_clip_uintp2(R, 30);	1857	R = av_clip_uintp2(R, 30);
1858	G = av_clip_uintp2(G, 30);	1858	G = av_clip_uintp2(G, 30);