History

swscale/output: Fix undefined integer overflow in yuv2rgba64_2_c_template()

Fixes: signed integer overflow: -1082982400 + -1083218484 cannot be represented in type 'int' Fixes: 70657/clusterfuzz-testcase-minimized-ffmpeg_SWS_fuzzer-6707819712675840

Michael Niedermayer committed 2 years ago

Commit bd80c971 parent 44c56411 file · +11 −11Browse files

Unified · Split

libswscale/output.c+11−11
@@ -1150,8 +1150,8 @@ yuv2rgba64_2_c_template(SwsContext *c, const int32_t *buf[2], · 1149 unmodified lines
11501150 av_assert2(uvalpha <= 4096U);
11511151
11521152 for (i = 0; i < ((dstW + 1) >> 1); i++) {
1153− int Y1 = (buf0[i * 2] * yalpha1 + buf1[i * 2] * yalpha) >> 14;
1154− int Y2 = (buf0[i * 2 + 1] * yalpha1 + buf1[i * 2 + 1] * yalpha) >> 14;
1153+ unsigned Y1 = (buf0[i * 2] * yalpha1 + buf1[i * 2] * yalpha) >> 14;
1154+ unsigned Y2 = (buf0[i * 2 + 1] * yalpha1 + buf1[i * 2 + 1] * yalpha) >> 14;
11551155 int U = (ubuf0[i] * uvalpha1 + ubuf1[i] * uvalpha - (128 << 23)) >> 14;
11561156 int V = (vbuf0[i] * uvalpha1 + vbuf1[i] * uvalpha - (128 << 23)) >> 14;
11571157 int R, G, B;
@@ -1175,20 +1175,20 @@ yuv2rgba64_2_c_template(SwsContext *c, const int32_t *buf[2], · 17 unmodified lines
11751175 A2 += 1 << 13;
11761176 }
11771177
1178− output_pixel(&dest[0], av_clip_uintp2(((R_B + Y1) >> 14) + (1<<15), 16));
1179− output_pixel(&dest[1], av_clip_uintp2((( G + Y1) >> 14) + (1<<15), 16));
1180− output_pixel(&dest[2], av_clip_uintp2(((B_R + Y1) >> 14) + (1<<15), 16));
1178+ output_pixel(&dest[0], av_clip_uintp2(((int)(R_B + Y1) >> 14) + (1<<15), 16));
1179+ output_pixel(&dest[1], av_clip_uintp2(((int)( G + Y1) >> 14) + (1<<15), 16));
1180+ output_pixel(&dest[2], av_clip_uintp2(((int)(B_R + Y1) >> 14) + (1<<15), 16));
11811181 if (eightbytes) {
11821182 output_pixel(&dest[3], av_clip_uintp2(A1 , 30) >> 14);
1183− output_pixel(&dest[4], av_clip_uintp2(((R_B + Y2) >> 14) + (1<<15), 16));
1184− output_pixel(&dest[5], av_clip_uintp2((( G + Y2) >> 14) + (1<<15), 16));
1185− output_pixel(&dest[6], av_clip_uintp2(((B_R + Y2) >> 14) + (1<<15), 16));
1183+ output_pixel(&dest[4], av_clip_uintp2(((int)(R_B + Y2) >> 14) + (1<<15), 16));
1184+ output_pixel(&dest[5], av_clip_uintp2(((int)( G + Y2) >> 14) + (1<<15), 16));
1185+ output_pixel(&dest[6], av_clip_uintp2(((int)(B_R + Y2) >> 14) + (1<<15), 16));
11861186 output_pixel(&dest[7], av_clip_uintp2(A2 , 30) >> 14);
11871187 dest += 8;
11881188 } else {
1189− output_pixel(&dest[3], av_clip_uintp2(((R_B + Y2) >> 14) + (1<<15), 16));
1190− output_pixel(&dest[4], av_clip_uintp2((( G + Y2) >> 14) + (1<<15), 16));
1191− output_pixel(&dest[5], av_clip_uintp2(((B_R + Y2) >> 14) + (1<<15), 16));
1189+ output_pixel(&dest[3], av_clip_uintp2(((int)(R_B + Y2) >> 14) + (1<<15), 16));
1190+ output_pixel(&dest[4], av_clip_uintp2(((int)( G + Y2) >> 14) + (1<<15), 16));
1191+ output_pixel(&dest[5], av_clip_uintp2(((int)(B_R + Y2) >> 14) + (1<<15), 16));
11921192 dest += 6;
11931193 }
11941194 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t