History

avcodec/mv30: Fix multiple integer overflows

Fixes: signed integer overflow: -895002 * 2400 cannot be represented in type 'int' Fixes: 26052/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MV30_fuzzer-5431812577558528

Michael Niedermayer committed 6 years ago

Commit 77cdc681 parent 42ded4d1 file · +12 −12Browse files

Unified · Split

libavcodec/mv30.c+12−12
@@ -198,12 +198,12 @@ static void idct_add(uint8_t *dst, int stride, · 197 unmodified lines
198 198
199static inline void idct2_1d(int *blk, int step)199static inline void idct2_1d(int *blk, int step)
200{200{
201 const int t0 = blk[0 * step];201 const unsigned int t0 = blk[0 * step];
202 const int t1 = blk[1 * step];202 const unsigned int t1 = blk[1 * step];
203 const int t2 = (int)(t1 * 473U) >> 8;203 const unsigned int t2 = (int)(t1 * 473U) >> 8;
204 const int t3 = t2 - t1;204 const unsigned int t3 = t2 - t1;
205 const int t4 = ((int)(t1 * 362U) >> 8) - t3;205 const unsigned int t4 = ((int)(t1 * 362U) >> 8) - t3;
206 const int t5 = (((int)(t1 * 277U) >> 8) - t2) + t4;206 const unsigned int t5 = (((int)(t1 * 277U) >> 8) - t2) + t4;
207 207
208 blk[0 * step] = t1 + t0;208 blk[0 * step] = t1 + t0;
209 blk[1 * step] = t0 + t3;209 blk[1 * step] = t0 + t3;
@@ -305,14 +305,14 @@ static int decode_intra_block(AVCodecContext *avctx, int mode, · 95 unmodified lines
305 case 1:305 case 1:
306 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);306 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
307 pfill[0] += fill;307 pfill[0] += fill;
308 block[0] = ((pfill[0] * qtab[0]) >> 5) + 128;308 block[0] = ((int)((unsigned)pfill[0] * qtab[0]) >> 5) + 128;
309 s->bdsp.fill_block_tab[1](dst, block[0], linesize, 8);309 s->bdsp.fill_block_tab[1](dst, block[0], linesize, 8);
310 break;310 break;
311 case 2:311 case 2:
312 memset(block, 0, sizeof(*block) * 64);312 memset(block, 0, sizeof(*block) * 64);
313 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);313 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
314 pfill[0] += fill;314 pfill[0] += fill;
315 block[0] = pfill[0] * qtab[0];315 block[0] = (unsigned)pfill[0] * qtab[0];
316 block[1] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[1];316 block[1] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[1];
317 block[8] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[8];317 block[8] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[8];
318 block[9] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[9];318 block[9] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[9];
@@ -321,7 +321,7 @@ static int decode_intra_block(AVCodecContext *avctx, int mode, · 2 unmodified lines
321 case 3:321 case 3:
322 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);322 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
323 pfill[0] += fill;323 pfill[0] += fill;
324 block[0] = pfill[0] * qtab[0];324 block[0] = (unsigned)pfill[0] * qtab[0];
325 for (int i = 1; i < 64; i++)325 for (int i = 1; i < 64; i++)
326 block[zigzag[i]] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[zigzag[i]];326 block[zigzag[i]] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[zigzag[i]];
327 idct_put(dst, linesize, block);327 idct_put(dst, linesize, block);
@@ -346,14 +346,14 @@ static int decode_inter_block(AVCodecContext *avctx, int mode, · 18 unmodified lines
346 case 1:346 case 1:
347 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);347 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
348 pfill[0] += fill;348 pfill[0] += fill;
349 block[0] = (pfill[0] * qtab[0]) >> 5;349 block[0] = (int)((unsigned)pfill[0] * qtab[0]) >> 5;
350 update_inter_block(dst, linesize, src, in_linesize, block[0]);350 update_inter_block(dst, linesize, src, in_linesize, block[0]);
351 break;351 break;
352 case 2:352 case 2:
353 memset(block, 0, sizeof(*block) * 64);353 memset(block, 0, sizeof(*block) * 64);
354 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);354 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
355 pfill[0] += fill;355 pfill[0] += fill;
356 block[0] = pfill[0] * qtab[0];356 block[0] = (unsigned)pfill[0] * qtab[0];
357 block[1] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[1];357 block[1] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[1];
358 block[8] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[8];358 block[8] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[8];
359 block[9] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[9];359 block[9] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[9];
@@ -362,7 +362,7 @@ static int decode_inter_block(AVCodecContext *avctx, int mode, · 2 unmodified lines
362 case 3:362 case 3:
363 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);363 fill = sign_extend(bytestream2_get_ne16(gbyte), 16);
364 pfill[0] += fill;364 pfill[0] += fill;
365 block[0] = pfill[0] * qtab[0];365 block[0] = (unsigned)pfill[0] * qtab[0];
366 for (int i = 1; i < 64; i++)366 for (int i = 1; i < 64; i++)
367 block[zigzag[i]] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[zigzag[i]];367 block[zigzag[i]] = sign_extend(bytestream2_get_ne16(gbyte), 16) * qtab[zigzag[i]];
368 idct_add(dst, linesize, src, in_linesize, block);368 idct_add(dst, linesize, src, in_linesize, block);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t