History

avcodec/xvididct: Fix integer overflow in idct_row()

Fixes: signed integer overflow: -1403461578 + -843974775 cannot be represented in type 'int' Fixes: 60868/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MPEG1VIDEO_fuzzer-4599793035378688

Michael Niedermayer committed 3 years ago

Commit 0ce322a1 parent f225f8d1 file · +12 −12Browse files

Showing the changes to libavcodec (1 of 1 file). Show all

Unified · Split

libavcodec/xvididct.c+12−12
@@ -69,24 +69,24 @@ static int idct_row(short *in, const int *const tab, int rnd) · 68 unmodified lines
6969 if (!(right | in[4])) {
7070 const int k = c4 * in[0] + rnd;
7171 if (left) {
72− const int a0 = k + c2 * in[2];
73− const int a1 = k + c6 * in[2];
74− const int a2 = k - c6 * in[2];
75− const int a3 = k - c2 * in[2];
72+ const unsigned a0 = k + c2 * in[2];
73+ const unsigned a1 = k + c6 * in[2];
74+ const unsigned a2 = k - c6 * in[2];
75+ const unsigned a3 = k - c2 * in[2];
7676
7777 const int b0 = c1 * in[1] + c3 * in[3];
7878 const int b1 = c3 * in[1] - c7 * in[3];
7979 const int b2 = c5 * in[1] - c1 * in[3];
8080 const int b3 = c7 * in[1] - c5 * in[3];
8181
82− in[0] = (a0 + b0) >> ROW_SHIFT;
83− in[1] = (a1 + b1) >> ROW_SHIFT;
84− in[2] = (a2 + b2) >> ROW_SHIFT;
85− in[3] = (a3 + b3) >> ROW_SHIFT;
86− in[4] = (a3 - b3) >> ROW_SHIFT;
87− in[5] = (a2 - b2) >> ROW_SHIFT;
88− in[6] = (a1 - b1) >> ROW_SHIFT;
89− in[7] = (a0 - b0) >> ROW_SHIFT;
82+ in[0] = (int)(a0 + b0) >> ROW_SHIFT;
83+ in[1] = (int)(a1 + b1) >> ROW_SHIFT;
84+ in[2] = (int)(a2 + b2) >> ROW_SHIFT;
85+ in[3] = (int)(a3 + b3) >> ROW_SHIFT;
86+ in[4] = (int)(a3 - b3) >> ROW_SHIFT;
87+ in[5] = (int)(a2 - b2) >> ROW_SHIFT;
88+ in[6] = (int)(a1 - b1) >> ROW_SHIFT;
89+ in[7] = (int)(a0 - b0) >> ROW_SHIFT;
9090 } else {
9191 const int a0 = k >> ROW_SHIFT;
9292 if (a0) {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t