History

swscale/swscale_unscaled: account for semi planar formats with data in the msb

Fixes fate failures introduced by recent tests that exercise the faulty code.

James Almer committed 1 year ago

Commit 819dec61 parent 26c5d8c11 files · +48 −46Browse files

Unified · Split

11 files
  1. libswscale/swscale_unscaled.c +38 −36
  2. tests/ref/pixfmt/p410-nv24 +1 −1
  3. tests/ref/pixfmt/p410-p412be +1 −1
  4. tests/ref/pixfmt/p410-p412le +1 −1
  5. tests/ref/pixfmt/p410-p416be +1 −1
  6. tests/ref/pixfmt/p410-p416le +1 −1
  7. tests/ref/pixfmt/p412-nv24 +1 −1
  8. tests/ref/pixfmt/p412-p410be +1 −1
  9. tests/ref/pixfmt/p412-p410le +1 −1
  10. tests/ref/pixfmt/p412-p416be +1 −1
  11. tests/ref/pixfmt/p412-p416le +1 −1
libswscale/swscale_unscaled.c+38−36
@@ -2146,17 +2146,17 @@ static int packedCopyWrapper(SwsInternal *c, const uint8_t *const src[], · 2145 unmodified lines
2146 if (c->opts.dither == SWS_DITHER_NONE) {\2146 if (c->opts.dither == SWS_DITHER_NONE) {\
2147 for (i = 0; i < height; i++) {\2147 for (i = 0; i < height; i++) {\
2148 for (j = 0; j < length-7; j+=8) {\2148 for (j = 0; j < length-7; j+=8) {\
2149 tmp = (bswap(src[j+0]) + bias)>>shift; dst[j+0] = dbswap(tmp - (tmp>>dst_depth));\2149 tmp = ((bswap(src[j+0]) >> src_shift) + bias)>>shift; dst[j+0] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2150 tmp = (bswap(src[j+1]) + bias)>>shift; dst[j+1] = dbswap(tmp - (tmp>>dst_depth));\2150 tmp = ((bswap(src[j+1]) >> src_shift) + bias)>>shift; dst[j+1] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2151 tmp = (bswap(src[j+2]) + bias)>>shift; dst[j+2] = dbswap(tmp - (tmp>>dst_depth));\2151 tmp = ((bswap(src[j+2]) >> src_shift) + bias)>>shift; dst[j+2] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2152 tmp = (bswap(src[j+3]) + bias)>>shift; dst[j+3] = dbswap(tmp - (tmp>>dst_depth));\2152 tmp = ((bswap(src[j+3]) >> src_shift) + bias)>>shift; dst[j+3] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2153 tmp = (bswap(src[j+4]) + bias)>>shift; dst[j+4] = dbswap(tmp - (tmp>>dst_depth));\2153 tmp = ((bswap(src[j+4]) >> src_shift) + bias)>>shift; dst[j+4] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2154 tmp = (bswap(src[j+5]) + bias)>>shift; dst[j+5] = dbswap(tmp - (tmp>>dst_depth));\2154 tmp = ((bswap(src[j+5]) >> src_shift) + bias)>>shift; dst[j+5] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2155 tmp = (bswap(src[j+6]) + bias)>>shift; dst[j+6] = dbswap(tmp - (tmp>>dst_depth));\2155 tmp = ((bswap(src[j+6]) >> src_shift) + bias)>>shift; dst[j+6] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2156 tmp = (bswap(src[j+7]) + bias)>>shift; dst[j+7] = dbswap(tmp - (tmp>>dst_depth));\2156 tmp = ((bswap(src[j+7]) >> src_shift) + bias)>>shift; dst[j+7] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2157 }\2157 }\
2158 for (; j < length; j++) {\2158 for (; j < length; j++) {\
2159 tmp = (bswap(src[j]) + bias)>>shift; dst[j] = dbswap(tmp - (tmp>>dst_depth));\2159 tmp = (bswap(src[j]) + bias)>>shift; dst[j] = dbswap(tmp - (tmp>>dst_depth) << dst_shift);\
2160 }\2160 }\
2161 dst += dstStride;\2161 dst += dstStride;\
2162 src += srcStride;\2162 src += srcStride;\
@@ -2165,17 +2165,17 @@ static int packedCopyWrapper(SwsInternal *c, const uint8_t *const src[], · 2 unmodified lines
2165 for (i = 0; i < height; i++) {\2165 for (i = 0; i < height; i++) {\
2166 const uint8_t *dither= dithers[shift-1][i&7];\2166 const uint8_t *dither= dithers[shift-1][i&7];\
2167 for (j = 0; j < length-7; j+=8) {\2167 for (j = 0; j < length-7; j+=8) {\
2168 tmp = (bswap(src[j+0]) + dither[0])>>shift; dst[j+0] = dbswap(tmp - (tmp>>dst_depth));\2168 tmp = ((bswap(src[j+0]) >> src_shift) + dither[0])>>shift; dst[j+0] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2169 tmp = (bswap(src[j+1]) + dither[1])>>shift; dst[j+1] = dbswap(tmp - (tmp>>dst_depth));\2169 tmp = ((bswap(src[j+1]) >> src_shift) + dither[1])>>shift; dst[j+1] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2170 tmp = (bswap(src[j+2]) + dither[2])>>shift; dst[j+2] = dbswap(tmp - (tmp>>dst_depth));\2170 tmp = ((bswap(src[j+2]) >> src_shift) + dither[2])>>shift; dst[j+2] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2171 tmp = (bswap(src[j+3]) + dither[3])>>shift; dst[j+3] = dbswap(tmp - (tmp>>dst_depth));\2171 tmp = ((bswap(src[j+3]) >> src_shift) + dither[3])>>shift; dst[j+3] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2172 tmp = (bswap(src[j+4]) + dither[4])>>shift; dst[j+4] = dbswap(tmp - (tmp>>dst_depth));\2172 tmp = ((bswap(src[j+4]) >> src_shift) + dither[4])>>shift; dst[j+4] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2173 tmp = (bswap(src[j+5]) + dither[5])>>shift; dst[j+5] = dbswap(tmp - (tmp>>dst_depth));\2173 tmp = ((bswap(src[j+5]) >> src_shift) + dither[5])>>shift; dst[j+5] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2174 tmp = (bswap(src[j+6]) + dither[6])>>shift; dst[j+6] = dbswap(tmp - (tmp>>dst_depth));\2174 tmp = ((bswap(src[j+6]) >> src_shift) + dither[6])>>shift; dst[j+6] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2175 tmp = (bswap(src[j+7]) + dither[7])>>shift; dst[j+7] = dbswap(tmp - (tmp>>dst_depth));\2175 tmp = ((bswap(src[j+7]) >> src_shift) + dither[7])>>shift; dst[j+7] = dbswap((tmp - (tmp>>dst_depth)) << dst_shift);\
2176 }\2176 }\
2177 for (; j < length; j++) {\2177 for (; j < length; j++) {\
2178 tmp = (bswap(src[j]) + dither[j&7])>>shift; dst[j] = dbswap(tmp - (tmp>>dst_depth));\2178 tmp = (bswap(src[j]) + dither[j&7])>>shift; dst[j] = dbswap(tmp - (tmp>>dst_depth) << dst_shift);\
2179 }\2179 }\
2180 dst += dstStride;\2180 dst += dstStride;\
2181 src += srcStride;\2181 src += srcStride;\
@@ -2184,14 +2184,14 @@ static int packedCopyWrapper(SwsInternal *c, const uint8_t *const src[], · 2 unmodified lines
2184 for (i = 0; i < height; i++) {\2184 for (i = 0; i < height; i++) {\
2185 const uint8_t *dither= dithers[shift-1][i&7];\2185 const uint8_t *dither= dithers[shift-1][i&7];\
2186 for (j = 0; j < length-7; j+=8) {\2186 for (j = 0; j < length-7; j+=8) {\
2187 tmp = bswap(src[j+0]); dst[j+0] = dbswap((tmp - (tmp>>dst_depth) + dither[0])>>shift);\2187 tmp = bswap(src[j+0]) >> src_shift; dst[j+0] = dbswap(((tmp - (tmp>>dst_depth) + dither[0])>>shift) << dst_shift);\
2188 tmp = bswap(src[j+1]); dst[j+1] = dbswap((tmp - (tmp>>dst_depth) + dither[1])>>shift);\2188 tmp = bswap(src[j+1]) >> src_shift; dst[j+1] = dbswap(((tmp - (tmp>>dst_depth) + dither[1])>>shift) << dst_shift);\
2189 tmp = bswap(src[j+2]); dst[j+2] = dbswap((tmp - (tmp>>dst_depth) + dither[2])>>shift);\2189 tmp = bswap(src[j+2]) >> src_shift; dst[j+2] = dbswap(((tmp - (tmp>>dst_depth) + dither[2])>>shift) << dst_shift);\
2190 tmp = bswap(src[j+3]); dst[j+3] = dbswap((tmp - (tmp>>dst_depth) + dither[3])>>shift);\2190 tmp = bswap(src[j+3]) >> src_shift; dst[j+3] = dbswap(((tmp - (tmp>>dst_depth) + dither[3])>>shift) << dst_shift);\
2191 tmp = bswap(src[j+4]); dst[j+4] = dbswap((tmp - (tmp>>dst_depth) + dither[4])>>shift);\2191 tmp = bswap(src[j+4]) >> src_shift; dst[j+4] = dbswap(((tmp - (tmp>>dst_depth) + dither[4])>>shift) << dst_shift);\
2192 tmp = bswap(src[j+5]); dst[j+5] = dbswap((tmp - (tmp>>dst_depth) + dither[5])>>shift);\2192 tmp = bswap(src[j+5]) >> src_shift; dst[j+5] = dbswap(((tmp - (tmp>>dst_depth) + dither[5])>>shift) << dst_shift);\
2193 tmp = bswap(src[j+6]); dst[j+6] = dbswap((tmp - (tmp>>dst_depth) + dither[6])>>shift);\2193 tmp = bswap(src[j+6]) >> src_shift; dst[j+6] = dbswap(((tmp - (tmp>>dst_depth) + dither[6])>>shift) << dst_shift);\
2194 tmp = bswap(src[j+7]); dst[j+7] = dbswap((tmp - (tmp>>dst_depth) + dither[7])>>shift);\2194 tmp = bswap(src[j+7]) >> src_shift; dst[j+7] = dbswap(((tmp - (tmp>>dst_depth) + dither[7])>>shift) << dst_shift);\
2195 }\2195 }\
2196 for (; j < length; j++) {\2196 for (; j < length; j++) {\
2197 tmp = bswap(src[j]); dst[j] = dbswap((tmp - (tmp>>dst_depth) + dither[j&7])>>shift);\2197 tmp = bswap(src[j]); dst[j] = dbswap((tmp - (tmp>>dst_depth) + dither[j&7])>>shift);\
@@ -2235,6 +2235,8 @@ static int planarCopyWrapper(SwsInternal *c, const uint8_t *const src[], · 37 unmodified lines
2235 ) {2235 ) {
2236 const int src_depth = desc_src->comp[plane].depth;2236 const int src_depth = desc_src->comp[plane].depth;
2237 const int dst_depth = desc_dst->comp[plane].depth;2237 const int dst_depth = desc_dst->comp[plane].depth;
2238 const int src_shift = desc_src->comp[plane].shift;
2239 const int dst_shift = desc_dst->comp[plane].shift;
2238 const uint16_t *srcPtr2 = (const uint16_t *) srcPtr;2240 const uint16_t *srcPtr2 = (const uint16_t *) srcPtr;
2239 uint16_t *dstPtr2 = (uint16_t*)dstPtr;2241 uint16_t *dstPtr2 = (uint16_t*)dstPtr;
2240 2242
@@ -2250,11 +2252,11 @@ static int planarCopyWrapper(SwsInternal *c, const uint8_t *const src[], · 9 unmodified lines
2250 #define COPY816(w)\2252 #define COPY816(w)\
2251 if (shiftonly) {\2253 if (shiftonly) {\
2252 for (j = 0; j < length; j++)\2254 for (j = 0; j < length; j++)\
2253 w(&dstPtr2[j], srcPtr[j]<<(dst_depth-8));\2255 w(&dstPtr2[j], (srcPtr[j]<<(dst_depth-8)) << dst_shift);\
2254 } else {\2256 } else {\
2255 for (j = 0; j < length; j++)\2257 for (j = 0; j < length; j++)\
2256 w(&dstPtr2[j], (srcPtr[j]<<(dst_depth-8)) |\2258 w(&dstPtr2[j], ((srcPtr[j]<<(dst_depth-8)) |\
2257 (srcPtr[j]>>(2*8-dst_depth)));\2259 (srcPtr[j]>>(2*8-dst_depth))) << dst_shift);\
2258 }2260 }
2259 if(isBE(c->opts.dst_format)){2261 if(isBE(c->opts.dst_format)){
2260 COPY816(AV_WB16)2262 COPY816(AV_WB16)
@@ -2274,13 +2276,13 @@ static int planarCopyWrapper(SwsInternal *c, const uint8_t *const src[], · 13 unmodified lines
2274#if HAVE_FAST_64BIT2276#if HAVE_FAST_64BIT
2275#define FAST_COPY_UP(shift) \2277#define FAST_COPY_UP(shift) \
2276 for (; j < length - 3; j += 4) { \2278 for (; j < length - 3; j += 4) { \
2277 uint64_t v = AV_RN64A(srcPtr2 + j); \2279 uint64_t v = AV_RN64A(srcPtr2 + j) >> src_shift; \
2278 AV_WN64A(dstPtr2 + j, v << shift); \2280 AV_WN64A(dstPtr2 + j, v << shift); \
2279 }2281 }
2280#else2282#else
2281#define FAST_COPY_UP(shift) \2283#define FAST_COPY_UP(shift) \
2282 for (; j < length - 1; j += 2) { \2284 for (; j < length - 1; j += 2) { \
2283 uint32_t v = AV_RN32A(srcPtr2 + j); \2285 uint32_t v = AV_RN32A(srcPtr2 + j) >> src_shift; \
2284 AV_WN32A(dstPtr2 + j, v << shift); \2286 AV_WN32A(dstPtr2 + j, v << shift); \
2285 }2287 }
2286#endif2288#endif
@@ -2293,14 +2295,14 @@ static int planarCopyWrapper(SwsInternal *c, const uint8_t *const src[], · 6 unmodified lines
2293#define COPY_UP(r,w) \2295#define COPY_UP(r,w) \
2294 if(shiftonly){\2296 if(shiftonly){\
2295 for (; j < length; j++){ \2297 for (; j < length; j++){ \
2296 unsigned int v= r(&srcPtr2[j]);\2298 unsigned int v= r(&srcPtr2[j]) >> src_shift;\
2297 w(&dstPtr2[j], v<<(dst_depth-src_depth));\2299 w(&dstPtr2[j], (v << (dst_depth-src_depth)) << dst_shift);\
2298 }\2300 }\
2299 }else{\2301 }else{\
2300 for (; j < length; j++){ \2302 for (; j < length; j++){ \
2301 unsigned int v= r(&srcPtr2[j]);\2303 unsigned int v= r(&srcPtr2[j]) >> src_shift;\
2302 w(&dstPtr2[j], (v<<(dst_depth-src_depth)) | \2304 w(&dstPtr2[j], ((v<<(dst_depth-src_depth)) | \
2303 (v>>(2*src_depth-dst_depth)));\2305 (v>>(2*src_depth-dst_depth))) << dst_shift);\
2304 }\2306 }\
2305 }2307 }
2306 if(isBE(c->opts.src_format)){2308 if(isBE(c->opts.src_format)){
tests/ref/pixfmt/p410-nv24+1−1
@@ -1,2 +1,2 @@
165d33af5a5828eb16b42573e9720b370 *tests/data/pixfmt/p410-nv24.yuv1bd4ba14d2828d10001aa6111adda31a8 *tests/data/pixfmt/p410-nv24.yuv
215206400 tests/data/pixfmt/p410-nv24.yuv215206400 tests/data/pixfmt/p410-nv24.yuv
tests/ref/pixfmt/p410-p412be+1−1
@@ -1,2 +1,2 @@
11b2996e48f9f2f9c08b47a7af2d1ad3a *tests/data/pixfmt/p410-p412be.yuv160b454bfd6f48b64cc24de3041d0a01d *tests/data/pixfmt/p410-p412be.yuv
215206400 tests/data/pixfmt/p410-p412be.yuv215206400 tests/data/pixfmt/p410-p412be.yuv
tests/ref/pixfmt/p410-p412le+1−1
@@ -1,2 +1,2 @@
11b2996e48f9f2f9c08b47a7af2d1ad3a *tests/data/pixfmt/p410-p412le.yuv160b454bfd6f48b64cc24de3041d0a01d *tests/data/pixfmt/p410-p412le.yuv
215206400 tests/data/pixfmt/p410-p412le.yuv215206400 tests/data/pixfmt/p410-p412le.yuv
tests/ref/pixfmt/p410-p416be+1−1
@@ -1,2 +1,2 @@
10a3bffd9200664676aa69684570a0053 *tests/data/pixfmt/p410-p416be.yuv160b454bfd6f48b64cc24de3041d0a01d *tests/data/pixfmt/p410-p416be.yuv
215206400 tests/data/pixfmt/p410-p416be.yuv215206400 tests/data/pixfmt/p410-p416be.yuv
tests/ref/pixfmt/p410-p416le+1−1
@@ -1,2 +1,2 @@
1cffbc4dedbfc209aec8bb4921c19fbf2 *tests/data/pixfmt/p410-p416le.yuv160b454bfd6f48b64cc24de3041d0a01d *tests/data/pixfmt/p410-p416le.yuv
215206400 tests/data/pixfmt/p410-p416le.yuv215206400 tests/data/pixfmt/p410-p416le.yuv
tests/ref/pixfmt/p412-nv24+1−1
@@ -1,2 +1,2 @@
15400f2bfef1b3dc0fcea1fd8ef91e2c8 *tests/data/pixfmt/p412-nv24.yuv1bd4ba14d2828d10001aa6111adda31a8 *tests/data/pixfmt/p412-nv24.yuv
215206400 tests/data/pixfmt/p412-nv24.yuv215206400 tests/data/pixfmt/p412-nv24.yuv
tests/ref/pixfmt/p412-p410be+1−1
@@ -1,2 +1,2 @@
11e268eb904e8d173d1585474d12e6f10 *tests/data/pixfmt/p412-p410be.yuv151e761c143c3fce8e9921d8cceb0a433 *tests/data/pixfmt/p412-p410be.yuv
215206400 tests/data/pixfmt/p412-p410be.yuv215206400 tests/data/pixfmt/p412-p410be.yuv
tests/ref/pixfmt/p412-p410le+1−1
@@ -1,2 +1,2 @@
11e268eb904e8d173d1585474d12e6f10 *tests/data/pixfmt/p412-p410le.yuv151e761c143c3fce8e9921d8cceb0a433 *tests/data/pixfmt/p412-p410le.yuv
215206400 tests/data/pixfmt/p412-p410le.yuv215206400 tests/data/pixfmt/p412-p410le.yuv
tests/ref/pixfmt/p412-p416be+1−1
@@ -1,2 +1,2 @@
1598b43d980ff2a5e24ee4003befcb875 *tests/data/pixfmt/p412-p416be.yuv19b5fc804354cf299fcc4c3f7bd2781ad *tests/data/pixfmt/p412-p416be.yuv
215206400 tests/data/pixfmt/p412-p416be.yuv215206400 tests/data/pixfmt/p412-p416be.yuv
tests/ref/pixfmt/p412-p416le+1−1
@@ -1,2 +1,2 @@
1598b43d980ff2a5e24ee4003befcb875 *tests/data/pixfmt/p412-p416le.yuv19b5fc804354cf299fcc4c3f7bd2781ad *tests/data/pixfmt/p412-p416le.yuv
215206400 tests/data/pixfmt/p412-p416le.yuv215206400 tests/data/pixfmt/p412-p416le.yuv

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t