History

avcodec/xvididct: Fix integer overflow in idct_row()

Fixes: signed integer overflow: -1403461578 + -843974775 cannot be represented in type 'int' Fixes: 60868/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MPEG1VIDEO_fuzzer-4599793035378688

Michael Niedermayer committed 3 years ago

Commit 0ce322a1 parent f225f8d1 file · +12 −12Browse files

Showing the changes to libavcodec (1 of 1 file). Show all

Unified · Split

libavcodec/xvididct.c+12−12
@@ -69,24 +69,24 @@ static int idct_row(short *in, const int *const tab, int rnd) · 68 unmodified lines
69 if (!(right | in[4])) {69 if (!(right | in[4])) {
70 const int k = c4 * in[0] + rnd;70 const int k = c4 * in[0] + rnd;
71 if (left) {71 if (left) {
72 const int a0 = k + c2 * in[2];72 const unsigned a0 = k + c2 * in[2];
73 const int a1 = k + c6 * in[2];73 const unsigned a1 = k + c6 * in[2];
74 const int a2 = k - c6 * in[2];74 const unsigned a2 = k - c6 * in[2];
75 const int a3 = k - c2 * in[2];75 const unsigned a3 = k - c2 * in[2];
76 76
77 const int b0 = c1 * in[1] + c3 * in[3];77 const int b0 = c1 * in[1] + c3 * in[3];
78 const int b1 = c3 * in[1] - c7 * in[3];78 const int b1 = c3 * in[1] - c7 * in[3];
79 const int b2 = c5 * in[1] - c1 * in[3];79 const int b2 = c5 * in[1] - c1 * in[3];
80 const int b3 = c7 * in[1] - c5 * in[3];80 const int b3 = c7 * in[1] - c5 * in[3];
81 81
82 in[0] = (a0 + b0) >> ROW_SHIFT;82 in[0] = (int)(a0 + b0) >> ROW_SHIFT;
83 in[1] = (a1 + b1) >> ROW_SHIFT;83 in[1] = (int)(a1 + b1) >> ROW_SHIFT;
84 in[2] = (a2 + b2) >> ROW_SHIFT;84 in[2] = (int)(a2 + b2) >> ROW_SHIFT;
85 in[3] = (a3 + b3) >> ROW_SHIFT;85 in[3] = (int)(a3 + b3) >> ROW_SHIFT;
86 in[4] = (a3 - b3) >> ROW_SHIFT;86 in[4] = (int)(a3 - b3) >> ROW_SHIFT;
87 in[5] = (a2 - b2) >> ROW_SHIFT;87 in[5] = (int)(a2 - b2) >> ROW_SHIFT;
88 in[6] = (a1 - b1) >> ROW_SHIFT;88 in[6] = (int)(a1 - b1) >> ROW_SHIFT;
89 in[7] = (a0 - b0) >> ROW_SHIFT;89 in[7] = (int)(a0 - b0) >> ROW_SHIFT;
90 } else {90 } else {
91 const int a0 = k >> ROW_SHIFT;91 const int a0 = k >> ROW_SHIFT;
92 if (a0) {92 if (a0) {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t