History

avfilter/vf_convolution: fix undefined behaviour

Fixes #8263

Paul B Mahol committed 7 years ago

Commit 99f8d321 parent 36508351 file · +20 −20Browse files

Showing the changes to libavfilter (1 of 1 file). Show all

Unified · Split

libavfilter/vf_convolution.c+20−20
@@ -115,10 +115,10 @@ static void filter16_prewitt(uint8_t *dstp, int width, · 114 unmodified lines
115 int x;115 int x;
116 116
117 for (x = 0; x < width; x++) {117 for (x = 0; x < width; x++) {
118 int suma = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[1][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) * -1 +118 float suma = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[1][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) * -1 +
119 AV_RN16A(&c[6][2 * x]) * 1 + AV_RN16A(&c[7][2 * x]) * 1 + AV_RN16A(&c[8][2 * x]) * 1;119 AV_RN16A(&c[6][2 * x]) * 1 + AV_RN16A(&c[7][2 * x]) * 1 + AV_RN16A(&c[8][2 * x]) * 1;
120 int sumb = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) * 1 + AV_RN16A(&c[3][2 * x]) * -1 +120 float sumb = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) * 1 + AV_RN16A(&c[3][2 * x]) * -1 +
121 AV_RN16A(&c[5][2 * x]) * 1 + AV_RN16A(&c[6][2 * x]) * -1 + AV_RN16A(&c[8][2 * x]) * 1;121 AV_RN16A(&c[5][2 * x]) * 1 + AV_RN16A(&c[6][2 * x]) * -1 + AV_RN16A(&c[8][2 * x]) * 1;
122 122
123 dst[x] = av_clip(sqrtf(suma*suma + sumb*sumb) * scale + delta, 0, peak);123 dst[x] = av_clip(sqrtf(suma*suma + sumb*sumb) * scale + delta, 0, peak);
124 }124 }
@@ -133,8 +133,8 @@ static void filter16_roberts(uint8_t *dstp, int width, · 8 unmodified lines
133 int x;133 int x;
134 134
135 for (x = 0; x < width; x++) {135 for (x = 0; x < width; x++) {
136 int suma = AV_RN16A(&c[0][2 * x]) * 1 + AV_RN16A(&c[1][2 * x]) * -1;136 float suma = AV_RN16A(&c[0][2 * x]) * 1 + AV_RN16A(&c[1][2 * x]) * -1;
137 int sumb = AV_RN16A(&c[4][2 * x]) * 1 + AV_RN16A(&c[3][2 * x]) * -1;137 float sumb = AV_RN16A(&c[4][2 * x]) * 1 + AV_RN16A(&c[3][2 * x]) * -1;
138 138
139 dst[x] = av_clip(sqrtf(suma*suma + sumb*sumb) * scale + delta, 0, peak);139 dst[x] = av_clip(sqrtf(suma*suma + sumb*sumb) * scale + delta, 0, peak);
140 }140 }
@@ -149,10 +149,10 @@ static void filter16_sobel(uint8_t *dstp, int width, · 8 unmodified lines
149 int x;149 int x;
150 150
151 for (x = 0; x < width; x++) {151 for (x = 0; x < width; x++) {
152 int suma = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[1][2 * x]) * -2 + AV_RN16A(&c[2][2 * x]) * -1 +152 float suma = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[1][2 * x]) * -2 + AV_RN16A(&c[2][2 * x]) * -1 +
153 AV_RN16A(&c[6][2 * x]) * 1 + AV_RN16A(&c[7][2 * x]) * 2 + AV_RN16A(&c[8][2 * x]) * 1;153 AV_RN16A(&c[6][2 * x]) * 1 + AV_RN16A(&c[7][2 * x]) * 2 + AV_RN16A(&c[8][2 * x]) * 1;
154 int sumb = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) * 1 + AV_RN16A(&c[3][2 * x]) * -2 +154 float sumb = AV_RN16A(&c[0][2 * x]) * -1 + AV_RN16A(&c[2][2 * x]) * 1 + AV_RN16A(&c[3][2 * x]) * -2 +
155 AV_RN16A(&c[5][2 * x]) * 2 + AV_RN16A(&c[6][2 * x]) * -1 + AV_RN16A(&c[8][2 * x]) * 1;155 AV_RN16A(&c[5][2 * x]) * 2 + AV_RN16A(&c[6][2 * x]) * -1 + AV_RN16A(&c[8][2 * x]) * 1;
156 156
157 dst[x] = av_clip(sqrtf(suma*suma + sumb*sumb) * scale + delta, 0, peak);157 dst[x] = av_clip(sqrtf(suma*suma + sumb*sumb) * scale + delta, 0, peak);
158 }158 }
@@ -169,10 +169,10 @@ static void filter_prewitt(uint8_t *dst, int width, · 10 unmodified lines
169 int x;169 int x;
170 170
171 for (x = 0; x < width; x++) {171 for (x = 0; x < width; x++) {
172 int suma = c0[x] * -1 + c1[x] * -1 + c2[x] * -1 +172 float suma = c0[x] * -1 + c1[x] * -1 + c2[x] * -1 +
173 c6[x] * 1 + c7[x] * 1 + c8[x] * 1;173 c6[x] * 1 + c7[x] * 1 + c8[x] * 1;
174 int sumb = c0[x] * -1 + c2[x] * 1 + c3[x] * -1 +174 float sumb = c0[x] * -1 + c2[x] * 1 + c3[x] * -1 +
175 c5[x] * 1 + c6[x] * -1 + c8[x] * 1;175 c5[x] * 1 + c6[x] * -1 + c8[x] * 1;
176 176
177 dst[x] = av_clip_uint8(sqrtf(suma*suma + sumb*sumb) * scale + delta);177 dst[x] = av_clip_uint8(sqrtf(suma*suma + sumb*sumb) * scale + delta);
178 }178 }
@@ -186,8 +186,8 @@ static void filter_roberts(uint8_t *dst, int width, · 7 unmodified lines
186 int x;186 int x;
187 187
188 for (x = 0; x < width; x++) {188 for (x = 0; x < width; x++) {
189 int suma = c[0][x] * 1 + c[1][x] * -1;189 float suma = c[0][x] * 1 + c[1][x] * -1;
190 int sumb = c[4][x] * 1 + c[3][x] * -1;190 float sumb = c[4][x] * 1 + c[3][x] * -1;
191 191
192 dst[x] = av_clip_uint8(sqrtf(suma*suma + sumb*sumb) * scale + delta);192 dst[x] = av_clip_uint8(sqrtf(suma*suma + sumb*sumb) * scale + delta);
193 }193 }
@@ -204,10 +204,10 @@ static void filter_sobel(uint8_t *dst, int width, · 10 unmodified lines
204 int x;204 int x;
205 205
206 for (x = 0; x < width; x++) {206 for (x = 0; x < width; x++) {
207 int suma = c0[x] * -1 + c1[x] * -2 + c2[x] * -1 +207 float suma = c0[x] * -1 + c1[x] * -2 + c2[x] * -1 +
208 c6[x] * 1 + c7[x] * 2 + c8[x] * 1;208 c6[x] * 1 + c7[x] * 2 + c8[x] * 1;
209 int sumb = c0[x] * -1 + c2[x] * 1 + c3[x] * -2 +209 float sumb = c0[x] * -1 + c2[x] * 1 + c3[x] * -2 +
210 c5[x] * 2 + c6[x] * -1 + c8[x] * 1;210 c5[x] * 2 + c6[x] * -1 + c8[x] * 1;
211 211
212 dst[x] = av_clip_uint8(sqrtf(suma*suma + sumb*sumb) * scale + delta);212 dst[x] = av_clip_uint8(sqrtf(suma*suma + sumb*sumb) * scale + delta);
213 }213 }

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t