Assume that data can be written into the padding area following each line. This enables the use of faster routines for more cases. Align downsample buffer stride to a multiple of 32. With this all strides used should be a multiple of 16, which means that use of narrower downsample routines can be dropped altogether.
Showing the changes to codec/processing/src (2 of 2 files). Show all
Unified · Split
| @@ -32,6 +32,7 @@ · 31 unmodified lines | ||
| 32 | 32 | |
| 33 | 33 | #include "downsample.h" |
| 34 | 34 | #include "cpu.h" |
| 35 | +#include <cassert> | |
| 35 | 36 | |
| 36 | 37 | WELSVP_NAMESPACE_BEGIN |
| 37 | 38 | #define MAX_SAMPLE_WIDTH 1920 |
| @@ -75,20 +76,18 @@ void CDownsampling::FreeSampleBuffer() { · 37 unmodified lines | ||
| 75 | 76 | WelsFree (m_pSampleBuffer[i][2]); |
| 76 | 77 | } |
| 77 | 78 | } |
| 79 | + | |
| 78 | 80 | void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int32_t iCpuFlag) { |
| 79 | − sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsampler_c; | |
| 80 | − sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsampler_c; | |
| 81 | − sDownsampleFunc.pfHalfAverage[2] = DyadicBilinearDownsampler_c; | |
| 82 | − sDownsampleFunc.pfHalfAverage[3] = DyadicBilinearDownsampler_c; | |
| 81 | + sDownsampleFunc.pfHalfAverageWidthx32 = DyadicBilinearDownsampler_c; | |
| 82 | + sDownsampleFunc.pfHalfAverageWidthx16 = DyadicBilinearDownsampler_c; | |
| 83 | 83 | sDownsampleFunc.pfOneThirdDownsampler = DyadicBilinearOneThirdDownsampler_c; |
| 84 | 84 | sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_c; |
| 85 | 85 | sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsampler_c; |
| 86 | 86 | sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearFastDownsampler_c; |
| 87 | 87 | #if defined(X86_ASM) |
| 88 | 88 | if (iCpuFlag & WELS_CPU_SSE) { |
| 89 | − sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsamplerWidthx32_sse; | |
| 90 | − sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsamplerWidthx16_sse; | |
| 91 | − sDownsampleFunc.pfHalfAverage[2] = DyadicBilinearDownsamplerWidthx8_sse; | |
| 89 | + sDownsampleFunc.pfHalfAverageWidthx32 = DyadicBilinearDownsamplerWidthx32_sse; | |
| 90 | + sDownsampleFunc.pfHalfAverageWidthx16 = DyadicBilinearDownsamplerWidthx16_sse; | |
| 92 | 91 | sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_sse; |
| 93 | 92 | } |
| 94 | 93 | if (iCpuFlag & WELS_CPU_SSE2) { |
| @@ -96,15 +95,15 @@ void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int · 1 unmodified line | ||
| 96 | 95 | sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearFastDownsamplerWrap_sse2; |
| 97 | 96 | } |
| 98 | 97 | if (iCpuFlag & WELS_CPU_SSSE3) { |
| 99 | − sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsamplerWidthx32_ssse3; | |
| 100 | − sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsamplerWidthx16_ssse3; | |
| 98 | + sDownsampleFunc.pfHalfAverageWidthx32 = DyadicBilinearDownsamplerWidthx32_ssse3; | |
| 99 | + sDownsampleFunc.pfHalfAverageWidthx16 = DyadicBilinearDownsamplerWidthx16_ssse3; | |
| 101 | 100 | sDownsampleFunc.pfOneThirdDownsampler = DyadicBilinearOneThirdDownsampler_ssse3; |
| 102 | 101 | sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_ssse3; |
| 103 | 102 | sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearFastDownsamplerWrap_ssse3; |
| 104 | 103 | } |
| 105 | 104 | if (iCpuFlag & WELS_CPU_SSE41) { |
| 106 | − sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsamplerWidthx32_sse4; | |
| 107 | − sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsamplerWidthx16_sse4; | |
| 105 | + sDownsampleFunc.pfHalfAverageWidthx32 = DyadicBilinearDownsamplerWidthx32_sse4; | |
| 106 | + sDownsampleFunc.pfHalfAverageWidthx16 = DyadicBilinearDownsamplerWidthx16_sse4; | |
| 108 | 107 | sDownsampleFunc.pfOneThirdDownsampler = DyadicBilinearOneThirdDownsampler_sse4; |
| 109 | 108 | sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_sse4; |
| 110 | 109 | sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_sse41; |
| @@ -117,10 +116,8 @@ void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int · 6 unmodified lines | ||
| 117 | 116 | |
| 118 | 117 | #if defined(HAVE_NEON) |
| 119 | 118 | if (iCpuFlag & WELS_CPU_NEON) { |
| 120 | − sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsamplerWidthx32_neon; | |
| 121 | − sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsampler_neon; | |
| 122 | − sDownsampleFunc.pfHalfAverage[2] = DyadicBilinearDownsampler_neon; | |
| 123 | − sDownsampleFunc.pfHalfAverage[3] = DyadicBilinearDownsampler_neon; | |
| 119 | + sDownsampleFunc.pfHalfAverageWidthx32 = DyadicBilinearDownsamplerWidthx32_neon; | |
| 120 | + sDownsampleFunc.pfHalfAverageWidthx16 = DyadicBilinearDownsampler_neon; | |
| 124 | 121 | sDownsampleFunc.pfOneThirdDownsampler = DyadicBilinearOneThirdDownsampler_neon; |
| 125 | 122 | sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_neon; |
| 126 | 123 | sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_neon; |
| @@ -130,10 +127,8 @@ void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int · 3 unmodified lines | ||
| 130 | 127 | |
| 131 | 128 | #if defined(HAVE_NEON_AARCH64) |
| 132 | 129 | if (iCpuFlag & WELS_CPU_NEON) { |
| 133 | − sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsamplerWidthx32_AArch64_neon; | |
| 134 | − sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsampler_AArch64_neon; | |
| 135 | − sDownsampleFunc.pfHalfAverage[2] = DyadicBilinearDownsampler_AArch64_neon; | |
| 136 | − sDownsampleFunc.pfHalfAverage[3] = DyadicBilinearDownsampler_AArch64_neon; | |
| 130 | + sDownsampleFunc.pfHalfAverageWidthx32 = DyadicBilinearDownsamplerWidthx32_AArch64_neon; | |
| 131 | + sDownsampleFunc.pfHalfAverageWidthx16 = DyadicBilinearDownsampler_AArch64_neon; | |
| 137 | 132 | sDownsampleFunc.pfOneThirdDownsampler = DyadicBilinearOneThirdDownsampler_AArch64_neon; |
| 138 | 133 | sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_AArch64_neon; |
| 139 | 134 | sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_AArch64_neon; |
| @@ -159,14 +154,11 @@ EResult CDownsampling::Process (int32_t iType, SPixMap* pSrcPixMap, SPixMap* pDs · 19 unmodified lines | ||
| 159 | 154 | if (iSrcWidthY > MAX_SAMPLE_WIDTH || iSrcHeightY > MAX_SAMPLE_HEIGHT || m_bNoSampleBuffer) { |
| 160 | 155 | if ((iSrcWidthY >> 1) == iDstWidthY && (iSrcHeightY >> 1) == iDstHeightY) { |
| 161 | 156 | // use half average functions |
| 162 | − uint8_t iAlignIndex = GetAlignedIndex (iSrcWidthY); | |
| 163 | − m_pfDownsample.pfHalfAverage[iAlignIndex] ((uint8_t*)pDstPixMap->pPixel[0], pDstPixMap->iStride[0], | |
| 157 | + DownsampleHalfAverage ((uint8_t*)pDstPixMap->pPixel[0], pDstPixMap->iStride[0], | |
| 164 | 158 | (uint8_t*)pSrcPixMap->pPixel[0], pSrcPixMap->iStride[0], iSrcWidthY, iSrcHeightY); |
| 165 | − | |
| 166 | − iAlignIndex = GetAlignedIndex (iSrcWidthUV); | |
| 167 | − m_pfDownsample.pfHalfAverage[iAlignIndex] ((uint8_t*)pDstPixMap->pPixel[1], pDstPixMap->iStride[1], | |
| 159 | + DownsampleHalfAverage ((uint8_t*)pDstPixMap->pPixel[1], pDstPixMap->iStride[1], | |
| 168 | 160 | (uint8_t*)pSrcPixMap->pPixel[1], pSrcPixMap->iStride[1], iSrcWidthUV, iSrcHeightUV); |
| 169 | − m_pfDownsample.pfHalfAverage[iAlignIndex] ((uint8_t*)pDstPixMap->pPixel[2], pDstPixMap->iStride[2], | |
| 161 | + DownsampleHalfAverage ((uint8_t*)pDstPixMap->pPixel[2], pDstPixMap->iStride[2], | |
| 170 | 162 | (uint8_t*)pSrcPixMap->pPixel[2], pSrcPixMap->iStride[2], iSrcWidthUV, iSrcHeightUV); |
| 171 | 163 | } else if ((iSrcWidthY >> 2) == iDstWidthY && (iSrcHeightY >> 2) == iDstHeightY) { |
| 172 | 164 | |
| @@ -223,29 +215,23 @@ EResult CDownsampling::Process (int32_t iType, SPixMap* pSrcPixMap, SPixMap* pDs · 50 unmodified lines | ||
| 223 | 215 | do { |
| 224 | 216 | if ((iHalfSrcWidth == iDstWidthY) && (iHalfSrcHeight == iDstHeightY)) { //end |
| 225 | 217 | // use half average functions |
| 226 | − uint8_t iAlignIndex = GetAlignedIndex (iSrcWidthY); | |
| 227 | − m_pfDownsample.pfHalfAverage[iAlignIndex] ((uint8_t*)pDstPixMap->pPixel[0], pDstPixMap->iStride[0], | |
| 218 | + DownsampleHalfAverage ((uint8_t*)pDstPixMap->pPixel[0], pDstPixMap->iStride[0], | |
| 228 | 219 | (uint8_t*)pSrcY, iSrcStrideY, iSrcWidthY, iSrcHeightY); |
| 229 | − | |
| 230 | − iAlignIndex = GetAlignedIndex (iSrcWidthUV); | |
| 231 | − m_pfDownsample.pfHalfAverage[iAlignIndex] ((uint8_t*)pDstPixMap->pPixel[1], pDstPixMap->iStride[1], | |
| 220 | + DownsampleHalfAverage ((uint8_t*)pDstPixMap->pPixel[1], pDstPixMap->iStride[1], | |
| 232 | 221 | (uint8_t*)pSrcU, iSrcStrideU, iSrcWidthUV, iSrcHeightUV); |
| 233 | − m_pfDownsample.pfHalfAverage[iAlignIndex] ((uint8_t*)pDstPixMap->pPixel[2], pDstPixMap->iStride[2], | |
| 222 | + DownsampleHalfAverage ((uint8_t*)pDstPixMap->pPixel[2], pDstPixMap->iStride[2], | |
| 234 | 223 | (uint8_t*)pSrcV, iSrcStrideV, iSrcWidthUV, iSrcHeightUV); |
| 235 | 224 | break; |
| 236 | 225 | } else if (((iHalfSrcWidth >> 1) >= iDstWidthY) && ((iHalfSrcHeight >> 1) >= iDstHeightY)) { |
| 237 | 226 | // use half average functions |
| 238 | − iDstStrideY = iHalfSrcWidth; | |
| 239 | − iDstStrideU = iHalfSrcWidth >> 1; | |
| 240 | − iDstStrideV = iHalfSrcWidth >> 1; | |
| 241 | − uint8_t iAlignIndex = GetAlignedIndex (iSrcWidthY); | |
| 242 | − m_pfDownsample.pfHalfAverage[iAlignIndex] ((uint8_t*)pDstY, iDstStrideY, | |
| 227 | + iDstStrideY = WELS_ALIGN (iHalfSrcWidth, 32); | |
| 228 | + iDstStrideU = WELS_ALIGN (iHalfSrcWidth >> 1, 32); | |
| 229 | + iDstStrideV = WELS_ALIGN (iHalfSrcWidth >> 1, 32); | |
| 230 | + DownsampleHalfAverage ((uint8_t*)pDstY, iDstStrideY, | |
| 243 | 231 | (uint8_t*)pSrcY, iSrcStrideY, iSrcWidthY, iSrcHeightY); |
| 244 | − | |
| 245 | − iAlignIndex = GetAlignedIndex (iSrcWidthUV); | |
| 246 | − m_pfDownsample.pfHalfAverage[iAlignIndex] ((uint8_t*)pDstU, iDstStrideU, | |
| 232 | + DownsampleHalfAverage ((uint8_t*)pDstU, iDstStrideU, | |
| 247 | 233 | (uint8_t*)pSrcU, iSrcStrideU, iSrcWidthUV, iSrcHeightUV); |
| 248 | − m_pfDownsample.pfHalfAverage[iAlignIndex] ((uint8_t*)pDstV, iDstStrideV, | |
| 234 | + DownsampleHalfAverage ((uint8_t*)pDstV, iDstStrideV, | |
| 249 | 235 | (uint8_t*)pSrcV, iSrcStrideV, iSrcWidthUV, iSrcHeightUV); |
| 250 | 236 | |
| 251 | 237 | pSrcY = (uint8_t*)pDstY; |
| @@ -258,9 +244,9 @@ EResult CDownsampling::Process (int32_t iType, SPixMap* pSrcPixMap, SPixMap* pDs · 6 unmodified lines | ||
| 258 | 244 | iSrcHeightY = iHalfSrcHeight; |
| 259 | 245 | iSrcHeightUV = iHalfSrcHeight >> 1; |
| 260 | 246 | |
| 261 | − iSrcStrideY = iSrcWidthY; | |
| 262 | − iSrcStrideU = iSrcWidthUV; | |
| 263 | − iSrcStrideV = iSrcWidthUV; | |
| 247 | + iSrcStrideY = iDstStrideY; | |
| 248 | + iSrcStrideU = iDstStrideU; | |
| 249 | + iSrcStrideV = iDstStrideV; | |
| 264 | 250 | |
| 265 | 251 | iHalfSrcWidth >>= 1; |
| 266 | 252 | iHalfSrcHeight >>= 1; |
| @@ -286,17 +272,18 @@ EResult CDownsampling::Process (int32_t iType, SPixMap* pSrcPixMap, SPixMap* pDs · 19 unmodified lines | ||
| 286 | 272 | return RET_SUCCESS; |
| 287 | 273 | } |
| 288 | 274 | |
| 289 | −int32_t CDownsampling::GetAlignedIndex (const int32_t kiSrcWidth) { | |
| 290 | − int32_t iAlignIndex; | |
| 291 | − if ((kiSrcWidth & 0x1f) == 0) // x32 | |
| 292 | − iAlignIndex = 0; | |
| 293 | − else if ((kiSrcWidth & 0x0f) == 0) // x16 | |
| 294 | − iAlignIndex = 1; | |
| 295 | − else if ((kiSrcWidth & 0x07) == 0) // x8 | |
| 296 | − iAlignIndex = 2; | |
| 297 | − else | |
| 298 | − iAlignIndex = 3; | |
| 299 | − return iAlignIndex; | |
| 275 | +void CDownsampling::DownsampleHalfAverage (uint8_t* pDst, int32_t iDstStride, | |
| 276 | + uint8_t* pSrc, int32_t iSrcStride, int32_t iSrcWidth, int32_t iSrcHeight) { | |
| 277 | + if ((iSrcStride & 31) == 0) { | |
| 278 | + assert ((iDstStride & 15) == 0); | |
| 279 | + m_pfDownsample.pfHalfAverageWidthx32 (pDst, iDstStride, | |
| 280 | + pSrc, iSrcStride, WELS_ALIGN (iSrcWidth, 32), iSrcHeight); | |
| 281 | + } else { | |
| 282 | + assert ((iSrcStride & 15) == 0); | |
| 283 | + assert ((iDstStride & 7) == 0); | |
| 284 | + m_pfDownsample.pfHalfAverageWidthx16 (pDst, iDstStride, | |
| 285 | + pSrc, iSrcStride, WELS_ALIGN (iSrcWidth, 16), iSrcHeight); | |
| 286 | + } | |
| 300 | 287 | } |
| 301 | 288 | |
| 302 | 289 | |
| @@ -73,8 +73,8 @@ SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_c; · 72 unmodified lines | ||
| 73 | 73 | SpecificDownsampleFunc DyadicBilinearQuarterDownsampler_c; |
| 74 | 74 | |
| 75 | 75 | typedef struct { |
| 76 | − // align_index: 0 = x32; 1 = x16; 2 = x8; 3 = common case left; | |
| 77 | − PHalveDownsampleFunc pfHalfAverage[4]; | |
| 76 | + PHalveDownsampleFunc pfHalfAverageWidthx32; | |
| 77 | + PHalveDownsampleFunc pfHalfAverageWidthx16; | |
| 78 | 78 | PSpecificDownsampleFunc pfOneThirdDownsampler; |
| 79 | 79 | PSpecificDownsampleFunc pfQuarterDownsampler; |
| 80 | 80 | PGeneralDownsampleFunc pfGeneralRatioLuma; |
| @@ -185,7 +185,8 @@ class CDownsampling : public IStrategy { · 104 unmodified lines | ||
| 185 | 185 | private: |
| 186 | 186 | void InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int32_t iCpuFlag); |
| 187 | 187 | |
| 188 | − int32_t GetAlignedIndex (const int32_t kiSrcWidth); | |
| 188 | + void DownsampleHalfAverage (uint8_t* pDst, int32_t iDstStride, | |
| 189 | + uint8_t* pSrc, int32_t iSrcStride, int32_t iSrcWidth, int32_t iSrcHeight); | |
| 189 | 190 | bool AllocateSampleBuffer(); |
| 190 | 191 | void FreeSampleBuffer(); |
| 191 | 192 | private: |