History

avcodec/mv30: Fix multiple integer overflows

Fixes: signed integer overflow: -895002 * 2400 cannot be represented in type 'int' Fixes: 26052/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MV30_fuzzer-5431812577558528

Michael Niedermayer committed 6 years ago

Commit 77cdc681 parent 42ded4d1 file · +12 −12Browse files

Unified · Split

libavcodec/mv30.c+12−12
@@ -198,12 +198,12 @@ static void idct_add(uint8_t *dst, int stride, · 197 unmodified lines
198198
199199 static inline void idct2_1d(int *blk, int step)
200200 {
201− const int t0 = blk[0 * step];
202− const int t1 = blk[1 * step];
203− const int t2 = (int)(t1 * 473U) >> 8;
204− const int t3 = t2 - t1;
205− const int t4 = ((int)(t1 * 362U) >> 8) - t3;
206− const int t5 = (((int)(t1 * 277U) >> 8) - t2) + t4;
201+ const unsigned int t0 = blk[0 * step];
202+ const unsigned int t1 = blk[1 * step];
203+ const unsigned int t2 = (int)(t1 * 473U) >> 8;
204+ const unsigned int t3 = t2 - t1;
205+ const unsigned int t4 = ((int)(t1 * 362U) >> 8) - t3;
206+ const unsigned int t5 = (((int)(t1 * 277U) >> 8) - t2) + t4;
207207
208208 blk[0 * step] = t1 + t0;
209209 blk[1 * step] = t0 + t3;
@@ -305,14 +305,14 @@ static int decode_intra_block(AVCodecContext *avctx, int mode, · 95 unmodified lines
305305 case 1:
306306 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
307307 pfill[0] += fill;
308− block[0] = ((pfill[0] * qtab[0]) >> 5) + 128;
308+ block[0] = ((int)((unsigned)pfill[0] * qtab[0]) >> 5) + 128;
309309 s->bdsp.fill_block_tab[1](dst, block[0], linesize, 8);
310310 break;
311311 case 2:
312312 memset(block, 0, sizeof(*block) * 64);
313313 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
314314 pfill[0] += fill;
315− block[0] = pfill[0] * qtab[0];
315+ block[0] = (unsigned)pfill[0] * qtab[0];
316316 block[1] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[1];
317317 block[8] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[8];
318318 block[9] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[9];
@@ -321,7 +321,7 @@ static int decode_intra_block(AVCodecContext *avctx, int mode, · 2 unmodified lines
321321 case 3:
322322 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
323323 pfill[0] += fill;
324− block[0] = pfill[0] * qtab[0];
324+ block[0] = (unsigned)pfill[0] * qtab[0];
325325 for (int i = 1; i < 64; i++)
326326 block[zigzag[i]] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[zigzag[i]];
327327 idct_put(dst, linesize, block);
@@ -346,14 +346,14 @@ static int decode_inter_block(AVCodecContext *avctx, int mode, · 18 unmodified lines
346346 case 1:
347347 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
348348 pfill[0] += fill;
349− block[0] = (pfill[0] * qtab[0]) >> 5;
349+ block[0] = (int)((unsigned)pfill[0] * qtab[0]) >> 5;
350350 update_inter_block(dst, linesize, src, in_linesize, block[0]);
351351 break;
352352 case 2:
353353 memset(block, 0, sizeof(*block) * 64);
354354 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
355355 pfill[0] += fill;
356− block[0] = pfill[0] * qtab[0];
356+ block[0] = (unsigned)pfill[0] * qtab[0];
357357 block[1] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[1];
358358 block[8] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[8];
359359 block[9] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[9];
@@ -362,7 +362,7 @@ static int decode_inter_block(AVCodecContext *avctx, int mode, · 2 unmodified lines
362362 case 3:
363363 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
364364 pfill[0] += fill;
365− block[0] = pfill[0] * qtab[0];
365+ block[0] = (unsigned)pfill[0] * qtab[0];
366366 for (int i = 1; i < 64; i++)
367367 block[zigzag[i]] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[zigzag[i]];
368368 idct_add(dst, linesize, src, in_linesize, block);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t