History

swscale/output: Fix undefined integer overflow in yuv2rgba64_2_c_template()

Fixes: signed integer overflow: -1082982400 + -1083218484 cannot be represented in type 'int' Fixes: 70657/clusterfuzz-testcase-minimized-ffmpeg_SWS_fuzzer-6707819712675840

Michael Niedermayer committed 2 years ago

Commit bd80c971 parent 44c56411 file · +11 −11Browse files

Unified · Split

libswscale/output.c+11−11
@@ -1150,8 +1150,8 @@ yuv2rgba64_2_c_template(SwsContext *c, const int32_t *buf[2], · 1149 unmodified lines
1150 av_assert2(uvalpha <= 4096U);1150 av_assert2(uvalpha <= 4096U);
1151 1151
1152 for (i = 0; i < ((dstW + 1) >> 1); i++) {1152 for (i = 0; i < ((dstW + 1) >> 1); i++) {
1153 int Y1 = (buf0[i * 2] * yalpha1 + buf1[i * 2] * yalpha) >> 14;1153 unsigned Y1 = (buf0[i * 2] * yalpha1 + buf1[i * 2] * yalpha) >> 14;
1154 int Y2 = (buf0[i * 2 + 1] * yalpha1 + buf1[i * 2 + 1] * yalpha) >> 14;1154 unsigned Y2 = (buf0[i * 2 + 1] * yalpha1 + buf1[i * 2 + 1] * yalpha) >> 14;
1155 int U = (ubuf0[i] * uvalpha1 + ubuf1[i] * uvalpha - (128 << 23)) >> 14;1155 int U = (ubuf0[i] * uvalpha1 + ubuf1[i] * uvalpha - (128 << 23)) >> 14;
1156 int V = (vbuf0[i] * uvalpha1 + vbuf1[i] * uvalpha - (128 << 23)) >> 14;1156 int V = (vbuf0[i] * uvalpha1 + vbuf1[i] * uvalpha - (128 << 23)) >> 14;
1157 int R, G, B;1157 int R, G, B;
@@ -1175,20 +1175,20 @@ yuv2rgba64_2_c_template(SwsContext *c, const int32_t *buf[2], · 17 unmodified lines
1175 A2 += 1 << 13;1175 A2 += 1 << 13;
1176 }1176 }
1177 1177
1178 output_pixel(&dest[0], av_clip_uintp2(((R_B + Y1) >> 14) + (1<<15), 16));1178 output_pixel(&dest[0], av_clip_uintp2(((int)(R_B + Y1) >> 14) + (1<<15), 16));
1179 output_pixel(&dest[1], av_clip_uintp2((( G + Y1) >> 14) + (1<<15), 16));1179 output_pixel(&dest[1], av_clip_uintp2(((int)( G + Y1) >> 14) + (1<<15), 16));
1180 output_pixel(&dest[2], av_clip_uintp2(((B_R + Y1) >> 14) + (1<<15), 16));1180 output_pixel(&dest[2], av_clip_uintp2(((int)(B_R + Y1) >> 14) + (1<<15), 16));
1181 if (eightbytes) {1181 if (eightbytes) {
1182 output_pixel(&dest[3], av_clip_uintp2(A1 , 30) >> 14);1182 output_pixel(&dest[3], av_clip_uintp2(A1 , 30) >> 14);
1183 output_pixel(&dest[4], av_clip_uintp2(((R_B + Y2) >> 14) + (1<<15), 16));1183 output_pixel(&dest[4], av_clip_uintp2(((int)(R_B + Y2) >> 14) + (1<<15), 16));
1184 output_pixel(&dest[5], av_clip_uintp2((( G + Y2) >> 14) + (1<<15), 16));1184 output_pixel(&dest[5], av_clip_uintp2(((int)( G + Y2) >> 14) + (1<<15), 16));
1185 output_pixel(&dest[6], av_clip_uintp2(((B_R + Y2) >> 14) + (1<<15), 16));1185 output_pixel(&dest[6], av_clip_uintp2(((int)(B_R + Y2) >> 14) + (1<<15), 16));
1186 output_pixel(&dest[7], av_clip_uintp2(A2 , 30) >> 14);1186 output_pixel(&dest[7], av_clip_uintp2(A2 , 30) >> 14);
1187 dest += 8;1187 dest += 8;
1188 } else {1188 } else {
1189 output_pixel(&dest[3], av_clip_uintp2(((R_B + Y2) >> 14) + (1<<15), 16));1189 output_pixel(&dest[3], av_clip_uintp2(((int)(R_B + Y2) >> 14) + (1<<15), 16));
1190 output_pixel(&dest[4], av_clip_uintp2((( G + Y2) >> 14) + (1<<15), 16));1190 output_pixel(&dest[4], av_clip_uintp2(((int)( G + Y2) >> 14) + (1<<15), 16));
1191 output_pixel(&dest[5], av_clip_uintp2(((B_R + Y2) >> 14) + (1<<15), 16));1191 output_pixel(&dest[5], av_clip_uintp2(((int)(B_R + Y2) >> 14) + (1<<15), 16));
1192 dest += 6;1192 dest += 6;
1193 }1193 }
1194 }1194 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t