History

postproc/postprocess_template: Fix reading uninitialized pixels in dering_C()

This issue was found through the new blocktest

Michael Niedermayer committed 1 year ago

Commit 0118f391 parent 8bb682d2 files · +8 −8Browse files

Unified · Split

2 files
  1. libpostproc/postprocess_altivec_template.c +1 −1
  2. libpostproc/postprocess_template.c +7 −7
libpostproc/postprocess_altivec_template.c+1−1
@@ -530,7 +530,7 @@ static inline void doVertDefFilter_altivec(uint8_t src[], int stride, PPContext · 529 unmodified lines
530530 STORE(5)
531531 }
532532
533−static inline void dering_altivec(uint8_t src[], int stride, PPContext *c) {
533+static inline void dering_altivec(uint8_t src[], int stride, PPContext *c, int leftborder, int rightborder) {
534534 const vector signed int vsint32_8 = vec_splat_s32(8);
535535 const vector unsigned int vuint32_4 = vec_splat_u32(4);
536536 const vector signed char neg1 = vec_splat_s8(-1);
libpostproc/postprocess_template.c+7−7
@@ -831,7 +831,7 @@ static inline void RENAME(doVertDefFilter)(uint8_t src[], int stride, PPContext · 830 unmodified lines
831831 #endif //TEMPLATE_PP_ALTIVEC
832832
833833 #if !TEMPLATE_PP_ALTIVEC
834−static inline void RENAME(dering)(uint8_t src[], int stride, PPContext *c)
834+static inline void RENAME(dering)(uint8_t src[], int stride, PPContext *c, int leftborder, int rightborder)
835835 {
836836 #if TEMPLATE_PP_MMXEXT && HAVE_7REGS
837837 DECLARE_ALIGNED(8, uint64_t, tmp)[3];
@@ -1046,7 +1046,7 @@ DERING_CORE((%0, %1, 8) ,(%%FF_REGd, %1, 4),%%mm2,%%mm4,%%mm0,%%mm3,%%mm5, · 208 unmodified lines
10461046 for(y=0; y<10; y++){
10471047 int t = 0;
10481048
1049− if(src[stride*y + 0] > avg) t+= 1;
1049+ if(!leftborder && src[stride*y + 0] > avg) t+= 1;
10501050 if(src[stride*y + 1] > avg) t+= 2;
10511051 if(src[stride*y + 2] > avg) t+= 4;
10521052 if(src[stride*y + 3] > avg) t+= 8;
@@ -1055,7 +1055,7 @@ DERING_CORE((%0, %1, 8) ,(%%FF_REGd, %1, 4),%%mm2,%%mm4,%%mm0,%%mm3,%%mm5, · 2 unmodified lines
10551055 if(src[stride*y + 6] > avg) t+= 64;
10561056 if(src[stride*y + 7] > avg) t+= 128;
10571057 if(src[stride*y + 8] > avg) t+= 256;
1058− if(src[stride*y + 9] > avg) t+= 512;
1058+ if(!rightborder && src[stride*y + 9] > avg) t+= 512;
10591059
10601060 t |= (~t)<<16;
10611061 t &= (t<<1) & (t>>1);
@@ -1072,8 +1072,8 @@ DERING_CORE((%0, %1, 8) ,(%%FF_REGd, %1, 4),%%mm2,%%mm4,%%mm0,%%mm3,%%mm5, · 10 unmodified lines
10721072 int x;
10731073 int t = s[y-1];
10741074
1075− p= src + stride*y;
1076− for(x=1; x<9; x++){
1075+ p= src + stride*y + leftborder;
1076+ for(x=1+leftborder; x<9-rightborder; x++){
10771077 p++;
10781078 if(t & (1<<x)){
10791079 int f= (*(p-stride-1)) + 2*(*(p-stride)) + (*(p-stride+1))
@@ -3210,7 +3210,7 @@ static void RENAME(postProcess)(const uint8_t src[], int srcStride, uint8_t dst[ · 2130 unmodified lines
32103210 #endif //TEMPLATE_PP_MMX
32113211 if(mode & DERING){
32123212 //FIXME filter first line
3213− if(y>0) RENAME(dering)(dstBlock - stride - 8, stride, c);
3213+ if(y>0) RENAME(dering)(dstBlock - stride - 8, stride, c, x<=8, 0);
32143214 }
32153215
32163216 if(mode & TEMP_NOISE_FILTER)
@@ -3232,7 +3232,7 @@ static void RENAME(postProcess)(const uint8_t src[], int srcStride, uint8_t dst[ · 15 unmodified lines
32323232 }
32333233
32343234 if(mode & DERING){
3235− if(y > 0) RENAME(dering)(dstBlock - dstStride - 8, dstStride, c);
3235+ if(y > 0) RENAME(dering)(dstBlock - dstStride - 8, dstStride, c, 0, 1);
32363236 }
32373237
32383238 if((mode & TEMP_NOISE_FILTER)){

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t