History

add new c and assembly functions to optimize downsampler when downscale equal 1:3/1:4

Guangwei Wang committed 11 years ago

Commit 64657d31 parent 69a62ea9 files · +1125 −8Browse files

Showing the changes to codec/processing/src (6 of 9 files). Show all

Unified · Split

6 files
  1. codec/processing/src/arm/down_sample_neon.S +117 −0
  2. codec/processing/src/arm64/down_sample_aarch64_neon.S +107 −1
  3. codec/processing/src/downsample/downsample.cpp +33 −0
  4. codec/processing/src/downsample/downsample.h +32 −5
  5. codec/processing/src/downsample/downsamplefuncs.cpp +47 −0
  6. codec/processing/src/x86/downsample_bilinear.asm +699 −0
codec/processing/src/arm/down_sample_neon.S+117−0
@@ -338,4 +338,121 @@ _LAST_ROW_WIDTH: · 337 unmodified lines
338 ldmia sp!, {r4-r12, lr}338 ldmia sp!, {r4-r12, lr}
339WELS_ASM_FUNC_END339WELS_ASM_FUNC_END
340 340
341WELS_ASM_FUNC_BEGIN DyadicBilinearOneThirdDownsampler_neon
342 stmdb sp!, {r4-r8, lr}
343
344 //Get the width and height
345 ldr r4, [sp, #24] //src_width
346 ldr r5, [sp, #28] //src_height
347
348 //Initialize the register
349 mov r6, r2
350 mov r8, r0
351 mov lr, #0
352
353 //Save the tailer for the un-aligned size
354 mla r7, r1, r5, r0
355 vld1.32 {q15}, [r7]
356
357 add r7, r2, r3
358 //processing a colume data
359comp_ds_bilinear_onethird_loop0:
360
361 vld3.8 {d0, d1, d2}, [r2]!
362 vld3.8 {d3, d4, d5}, [r2]!
363 vld3.8 {d16, d17, d18}, [r7]!
364 vld3.8 {d19, d20, d21}, [r7]!
365
366 vaddl.u8 q11, d0, d1
367 vaddl.u8 q12, d3, d4
368 vaddl.u8 q13, d16, d17
369 vaddl.u8 q14, d19, d20
370 vrshr.u16 q11, #1
371 vrshr.u16 q12, #1
372 vrshr.u16 q13, #1
373 vrshr.u16 q14, #1
374
375 vrhadd.u16 q11, q13
376 vrhadd.u16 q12, q14
377
378 vmovn.u16 d0, q11
379 vmovn.u16 d1, q12
380 vst1.8 {q0}, [r0]!
381
382 add lr, #48
383 cmp lr, r4
384 movcs lr, #0
385 addcs r6, r3, lsl #1
386 addcs r6, r6, r3
387 movcs r2, r6
388 addcs r7, r2, r3
389 addcs r8, r1
390 movcs r0, r8
391 subscs r5, #1
392 bne comp_ds_bilinear_onethird_loop0
393
394 //restore the tailer for the un-aligned size
395 vst1.32 {q15}, [r0]
396
397 ldmia sp!, {r4-r8,lr}
398WELS_ASM_FUNC_END
399
400WELS_ASM_FUNC_BEGIN DyadicBilinearQuarterDownsampler_neon
401 stmdb sp!, {r4-r8, lr}
402
403 //Get the width and height
404 ldr r4, [sp, #24] //src_width
405 ldr r5, [sp, #28] //src_height
406
407 //Initialize the register
408 mov r6, r2
409 mov r8, r0
410 mov lr, #0
411 lsr r5, #2
412
413 //Save the tailer for the un-aligned size
414 mla r7, r1, r5, r0
415 vld1.32 {q15}, [r7]
416
417 add r7, r2, r3
418 //processing a colume data
419comp_ds_bilinear_quarter_loop0:
420
421 vld2.16 {q0, q1}, [r2]!
422 vld2.16 {q2, q3}, [r2]!
423 vld2.16 {q8, q9}, [r7]!
424 vld2.16 {q10, q11}, [r7]!
425
426 vpaddl.u8 q0, q0
427 vpaddl.u8 q2, q2
428 vpaddl.u8 q8, q8
429 vpaddl.u8 q10, q10
430 vrshr.u16 q0, #1
431 vrshr.u16 q2, #1
432 vrshr.u16 q8, #1
433 vrshr.u16 q10, #1
434
435 vrhadd.u16 q0, q8
436 vrhadd.u16 q2, q10
437 vmovn.u16 d0, q0
438 vmovn.u16 d1, q2
439 vst1.8 {q0}, [r0]!
440
441 add lr, #64
442 cmp lr, r4
443 movcs lr, #0
444 addcs r6, r3, lsl #2
445 movcs r2, r6
446 addcs r7, r2, r3
447 addcs r8, r1
448 movcs r0, r8
449 subscs r5, #1
450 bne comp_ds_bilinear_quarter_loop0
451
452 //restore the tailer for the un-aligned size
453 vst1.32 {q15}, [r0]
454
455 ldmia sp!, {r4-r8,lr}
456WELS_ASM_FUNC_END
457
341#endif458#endif
codec/processing/src/arm64/down_sample_aarch64_neon.S+107−1
@@ -84,7 +84,6 @@ comp_ds_bilinear_loop0: · 83 unmodified lines
84 84
85WELS_ASM_AARCH64_FUNC_END85WELS_ASM_AARCH64_FUNC_END
86 86
87
88WELS_ASM_AARCH64_FUNC_BEGIN DyadicBilinearDownsamplerWidthx32_AArch64_neon87WELS_ASM_AARCH64_FUNC_BEGIN DyadicBilinearDownsamplerWidthx32_AArch64_neon
89 sub w9, w3, w488 sub w9, w3, w4
90 sub w1, w1, w4, lsr #189 sub w1, w1, w4, lsr #1
@@ -123,6 +122,113 @@ comp_ds_bilinear_w_x32_loop1: · 32 unmodified lines
123 cbnz w5, comp_ds_bilinear_w_x32_loop0122 cbnz w5, comp_ds_bilinear_w_x32_loop0
124WELS_ASM_AARCH64_FUNC_END123WELS_ASM_AARCH64_FUNC_END
125 124
125WELS_ASM_AARCH64_FUNC_BEGIN DyadicBilinearOneThirdDownsampler_AArch64_neon
126
127 //Initialize the register
128 mov x6, x2
129 mov x8, x0
130 mov w9, #0
131
132 //Save the tailer for the unasigned size
133 smaddl x7, w1, w5, x0
134 ld1 {v16.16b}, [x7]
135
136 add x7, x2, w3, sxtw
137 //processing a colume data
138comp_ds_bilinear_onethird_loop0:
139
140 ld3 {v0.16b, v1.16b, v2.16b}, [x2], #48
141 ld3 {v4.16b, v5.16b, v6.16b}, [x7], #48
142
143 uaddl v2.8h, v0.8b, v1.8b
144 uaddl2 v3.8h, v0.16b, v1.16b
145 uaddl v6.8h, v4.8b, v5.8b
146 uaddl2 v7.8h, v4.16b, v5.16b
147 urshr v2.8h, v2.8h, #1
148 urshr v3.8h, v3.8h, #1
149 urshr v6.8h, v6.8h, #1
150 urshr v7.8h, v7.8h, #1
151
152 urhadd v0.8h, v2.8h, v6.8h
153 urhadd v1.8h, v3.8h, v7.8h
154 xtn v0.8b, v0.8h
155 xtn v1.8b, v1.8h
156 st1 {v0.8b,v1.8b}, [x0], #16
157
158 add w9, w9, #48
159
160 cmp w9, w4
161 b.cc comp_ds_bilinear_onethird_loop0
162
163 mov w9, #0
164 add x6, x6, w3, sxtw #1
165 add x6, x6, w3, sxtw
166 mov x2, x6
167 add x7, x2, w3, sxtw
168 add x8, x8, w1, sxtw
169 mov x0, x8
170 sub w5, w5, #1
171
172 cbnz w5, comp_ds_bilinear_onethird_loop0
173
174 //restore the tailer for the unasigned size
175 st1 {v16.16b}, [x0]
176WELS_ASM_AARCH64_FUNC_END
177
178WELS_ASM_AARCH64_FUNC_BEGIN DyadicBilinearQuarterDownsampler_AArch64_neon
179 //Initialize the register
180 mov x6, x2
181 mov x8, x0
182 mov w9, #0
183 lsr w5, w5, #2
184
185 //Save the tailer for the unasigned size
186 smaddl x7, w1, w5, x0
187 ld1 {v16.16b}, [x7]
188
189 add x7, x2, w3, sxtw
190 //processing a colume data
191comp_ds_bilinear_quarter_loop0:
192
193 ld2 {v0.8h, v1.8h}, [x2], #32
194 ld2 {v2.8h, v3.8h}, [x2], #32
195 ld2 {v4.8h, v5.8h}, [x7], #32
196 ld2 {v6.8h, v7.8h}, [x7], #32
197
198 uaddlp v0.8h, v0.16b
199 uaddlp v1.8h, v2.16b
200 uaddlp v4.8h, v4.16b
201 uaddlp v5.8h, v6.16b
202 urshr v0.8h, v0.8h, #1
203 urshr v1.8h, v1.8h, #1
204 urshr v4.8h, v4.8h, #1
205 urshr v5.8h, v5.8h, #1
206
207 urhadd v0.8h, v0.8h, v4.8h
208 urhadd v1.8h, v1.8h, v5.8h
209 xtn v0.8b, v0.8h
210 xtn v1.8b, v1.8h
211 st1 {v0.8b,v1.8b}, [x0], #16
212
213 add w9, w9, #64
214
215 cmp w9, w4
216 b.cc comp_ds_bilinear_quarter_loop0
217
218 mov w9, #0
219 add x6, x6, w3, sxtw #2
220 mov x2, x6
221 add x7, x2, w3, sxtw
222 add x8, x8, w1, sxtw
223 mov x0, x8
224 sub w5, w5, #1
225
226 cbnz w5, comp_ds_bilinear_quarter_loop0
227
228 //restore the tailer for the unasigned size
229 st1 {v16.16b}, [x0]
230WELS_ASM_AARCH64_FUNC_END
231
126WELS_ASM_AARCH64_FUNC_BEGIN GeneralBilinearAccurateDownsampler_AArch64_neon232WELS_ASM_AARCH64_FUNC_BEGIN GeneralBilinearAccurateDownsampler_AArch64_neon
127 mov w10, #32767233 mov w10, #32767
128 and w8, w6, w10234 and w8, w6, w10
codec/processing/src/downsample/downsample.cpp+33−0
@@ -53,6 +53,8 @@ void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int · 52 unmodified lines
53 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsampler_c;53 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsampler_c;
54 sDownsampleFunc.pfHalfAverage[2] = DyadicBilinearDownsampler_c;54 sDownsampleFunc.pfHalfAverage[2] = DyadicBilinearDownsampler_c;
55 sDownsampleFunc.pfHalfAverage[3] = DyadicBilinearDownsampler_c;55 sDownsampleFunc.pfHalfAverage[3] = DyadicBilinearDownsampler_c;
56 sDownsampleFunc.pfOneThirdDownsampler = DyadicBilinearOneThirdDownsampler_c;
57 sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_c;
56 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsampler_c;58 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsampler_c;
57 sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearFastDownsampler_c;59 sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearFastDownsampler_c;
58#if defined(X86_ASM)60#if defined(X86_ASM)
@@ -60,6 +62,7 @@ void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int · 1 unmodified line
60 sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsamplerWidthx32_sse;62 sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsamplerWidthx32_sse;
61 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsamplerWidthx16_sse;63 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsamplerWidthx16_sse;
62 sDownsampleFunc.pfHalfAverage[2] = DyadicBilinearDownsamplerWidthx8_sse;64 sDownsampleFunc.pfHalfAverage[2] = DyadicBilinearDownsamplerWidthx8_sse;
65 sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_sse;
63 }66 }
64 if (iCpuFlag & WELS_CPU_SSE2) {67 if (iCpuFlag & WELS_CPU_SSE2) {
65 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_sse2;68 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_sse2;
@@ -68,10 +71,14 @@ void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int · 2 unmodified lines
68 if (iCpuFlag & WELS_CPU_SSSE3) {71 if (iCpuFlag & WELS_CPU_SSSE3) {
69 sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsamplerWidthx32_ssse3;72 sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsamplerWidthx32_ssse3;
70 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsamplerWidthx16_ssse3;73 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsamplerWidthx16_ssse3;
74 sDownsampleFunc.pfOneThirdDownsampler = DyadicBilinearOneThirdDownsampler_ssse3;
75 sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_ssse3;
71 }76 }
72 if (iCpuFlag & WELS_CPU_SSE41) {77 if (iCpuFlag & WELS_CPU_SSE41) {
73 sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsamplerWidthx32_sse4;78 sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsamplerWidthx32_sse4;
74 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsamplerWidthx16_sse4;79 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsamplerWidthx16_sse4;
80 sDownsampleFunc.pfOneThirdDownsampler = DyadicBilinearOneThirdDownsampler_sse4;
81 sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_sse4;
75 }82 }
76#endif//X86_ASM83#endif//X86_ASM
77 84
@@ -81,6 +88,8 @@ void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int · 3 unmodified lines
81 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsampler_neon;88 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsampler_neon;
82 sDownsampleFunc.pfHalfAverage[2] = DyadicBilinearDownsampler_neon;89 sDownsampleFunc.pfHalfAverage[2] = DyadicBilinearDownsampler_neon;
83 sDownsampleFunc.pfHalfAverage[3] = DyadicBilinearDownsampler_neon;90 sDownsampleFunc.pfHalfAverage[3] = DyadicBilinearDownsampler_neon;
91 sDownsampleFunc.pfOneThirdDownsampler = DyadicBilinearOneThirdDownsampler_neon;
92 sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_neon;
84 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_neon;93 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_neon;
85 sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearAccurateDownsamplerWrap_neon;94 sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearAccurateDownsamplerWrap_neon;
86 }95 }
@@ -92,6 +101,8 @@ void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int · 5 unmodified lines
92 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsampler_AArch64_neon;101 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsampler_AArch64_neon;
93 sDownsampleFunc.pfHalfAverage[2] = DyadicBilinearDownsampler_AArch64_neon;102 sDownsampleFunc.pfHalfAverage[2] = DyadicBilinearDownsampler_AArch64_neon;
94 sDownsampleFunc.pfHalfAverage[3] = DyadicBilinearDownsampler_AArch64_neon;103 sDownsampleFunc.pfHalfAverage[3] = DyadicBilinearDownsampler_AArch64_neon;
104 sDownsampleFunc.pfOneThirdDownsampler = DyadicBilinearOneThirdDownsampler_AArch64_neon;
105 sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_AArch64_neon;
95 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_AArch64_neon;106 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_AArch64_neon;
96 sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearAccurateDownsamplerWrap_AArch64_neon;107 sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearAccurateDownsamplerWrap_AArch64_neon;
97 }108 }
@@ -124,6 +135,28 @@ EResult CDownsampling::Process (int32_t iType, SPixMap* pSrcPixMap, SPixMap* pDs · 26 unmodified lines
124 (uint8_t*)pSrcPixMap->pPixel[1], pSrcPixMap->iStride[1], iSrcWidthUV, iSrcHeightUV);135 (uint8_t*)pSrcPixMap->pPixel[1], pSrcPixMap->iStride[1], iSrcWidthUV, iSrcHeightUV);
125 m_pfDownsample.pfHalfAverage[iAlignIndex] ((uint8_t*)pDstPixMap->pPixel[2], pDstPixMap->iStride[2],136 m_pfDownsample.pfHalfAverage[iAlignIndex] ((uint8_t*)pDstPixMap->pPixel[2], pDstPixMap->iStride[2],
126 (uint8_t*)pSrcPixMap->pPixel[2], pSrcPixMap->iStride[2], iSrcWidthUV, iSrcHeightUV);137 (uint8_t*)pSrcPixMap->pPixel[2], pSrcPixMap->iStride[2], iSrcWidthUV, iSrcHeightUV);
138 } else if ((iSrcWidthY >> 2) == iDstWidthY && (iSrcHeightY >> 2) == iDstHeightY) {
139
140 m_pfDownsample.pfQuarterDownsampler ((uint8_t*)pDstPixMap->pPixel[0], pDstPixMap->iStride[0],
141 (uint8_t*)pSrcPixMap->pPixel[0], pSrcPixMap->iStride[0], iSrcWidthY, iSrcHeightY);
142
143 m_pfDownsample.pfQuarterDownsampler ((uint8_t*)pDstPixMap->pPixel[1], pDstPixMap->iStride[1],
144 (uint8_t*)pSrcPixMap->pPixel[1], pSrcPixMap->iStride[1], iSrcWidthUV, iSrcHeightUV);
145
146 m_pfDownsample.pfQuarterDownsampler ((uint8_t*)pDstPixMap->pPixel[2], pDstPixMap->iStride[2],
147 (uint8_t*)pSrcPixMap->pPixel[2], pSrcPixMap->iStride[2], iSrcWidthUV, iSrcHeightUV);
148
149 } else if ((iSrcWidthY / 3) == iDstWidthY && (iSrcHeightY / 3) == iDstHeightY) {
150
151 m_pfDownsample.pfOneThirdDownsampler ((uint8_t*)pDstPixMap->pPixel[0], pDstPixMap->iStride[0],
152 (uint8_t*)pSrcPixMap->pPixel[0], pSrcPixMap->iStride[0], iSrcWidthY, iDstHeightY);
153
154 m_pfDownsample.pfOneThirdDownsampler ((uint8_t*)pDstPixMap->pPixel[1], pDstPixMap->iStride[1],
155 (uint8_t*)pSrcPixMap->pPixel[1], pSrcPixMap->iStride[1], iSrcWidthUV, iDstHeightUV);
156
157 m_pfDownsample.pfOneThirdDownsampler ((uint8_t*)pDstPixMap->pPixel[2], pDstPixMap->iStride[2],
158 (uint8_t*)pSrcPixMap->pPixel[2], pSrcPixMap->iStride[2], iSrcWidthUV, iDstHeightUV);
159
127 } else {160 } else {
128 m_pfDownsample.pfGeneralRatioLuma ((uint8_t*)pDstPixMap->pPixel[0], pDstPixMap->iStride[0], iDstWidthY, iDstHeightY,161 m_pfDownsample.pfGeneralRatioLuma ((uint8_t*)pDstPixMap->pPixel[0], pDstPixMap->iStride[0], iDstWidthY, iDstHeightY,
129 (uint8_t*)pSrcPixMap->pPixel[0], pSrcPixMap->iStride[0], iSrcWidthY, iSrcHeightY);162 (uint8_t*)pSrcPixMap->pPixel[0], pSrcPixMap->iStride[0], iSrcWidthY, iSrcHeightY);
codec/processing/src/downsample/downsample.h+32−5
@@ -54,20 +54,29 @@ typedef void (HalveDownsampleFunc) (uint8_t* pDst, const int32_t kiDstStride, · 53 unmodified lines
54 uint8_t* pSrc, const int32_t kiSrcStride,54 uint8_t* pSrc, const int32_t kiSrcStride,
55 const int32_t kiSrcWidth, const int32_t kiSrcHeight);55 const int32_t kiSrcWidth, const int32_t kiSrcHeight);
56 56
57typedef void (SpecificDownsampleFunc) (uint8_t* pDst, const int32_t kiDstStride,
58 uint8_t* pSrc, const int32_t kiSrcStride,
59 const int32_t kiSrcWidth, const int32_t kiHeight);
60
57typedef void (GeneralDownsampleFunc) (uint8_t* pDst, const int32_t kiDstStride, const int32_t kiDstWidth,61typedef void (GeneralDownsampleFunc) (uint8_t* pDst, const int32_t kiDstStride, const int32_t kiDstWidth,
58 const int32_t kiDstHeight,62 const int32_t kiDstHeight,
59 uint8_t* pSrc, const int32_t kiSrcStride, const int32_t kiSrcWidth, const int32_t kiSrcHeight);63 uint8_t* pSrc, const int32_t kiSrcStride, const int32_t kiSrcWidth, const int32_t kiSrcHeight);
60 64
61typedef HalveDownsampleFunc* PHalveDownsampleFunc;65typedef HalveDownsampleFunc* PHalveDownsampleFunc;
66typedef SpecificDownsampleFunc* PSpecificDownsampleFunc;
62typedef GeneralDownsampleFunc* PGeneralDownsampleFunc;67typedef GeneralDownsampleFunc* PGeneralDownsampleFunc;
63 68
64HalveDownsampleFunc DyadicBilinearDownsampler_c;69HalveDownsampleFunc DyadicBilinearDownsampler_c;
65GeneralDownsampleFunc GeneralBilinearFastDownsampler_c;70GeneralDownsampleFunc GeneralBilinearFastDownsampler_c;
66GeneralDownsampleFunc GeneralBilinearAccurateDownsampler_c;71GeneralDownsampleFunc GeneralBilinearAccurateDownsampler_c;
72SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_c;
73SpecificDownsampleFunc DyadicBilinearQuarterDownsampler_c;
67 74
68typedef struct {75typedef struct {
69 // align_index: 0 = x32; 1 = x16; 2 = x8; 3 = common case left;76 // align_index: 0 = x32; 1 = x16; 2 = x8; 3 = common case left;
70 PHalveDownsampleFunc pfHalfAverage[4];77 PHalveDownsampleFunc pfHalfAverage[4];
78 PSpecificDownsampleFunc pfOneThirdDownsampler;
79 PSpecificDownsampleFunc pfQuarterDownsampler;
71 PGeneralDownsampleFunc pfGeneralRatioLuma;80 PGeneralDownsampleFunc pfGeneralRatioLuma;
72 PGeneralDownsampleFunc pfGeneralRatioChroma;81 PGeneralDownsampleFunc pfGeneralRatioChroma;
73} SDownsampleFuncs;82} SDownsampleFuncs;
@@ -93,10 +102,19 @@ HalveDownsampleFunc DyadicBilinearDownsamplerWidthx32_sse4; · 19 unmodified lines
93GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_sse2;102GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_sse2;
94GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse2;103GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse2;
95 104
105SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_ssse3;
106SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_sse4;
107SpecificDownsampleFunc DyadicBilinearQuarterDownsampler_sse;
108SpecificDownsampleFunc DyadicBilinearQuarterDownsampler_ssse3;
109SpecificDownsampleFunc DyadicBilinearQuarterDownsampler_sse4;
110
96void GeneralBilinearFastDownsampler_sse2 (uint8_t* pDst, const int32_t kiDstStride, const int32_t kiDstWidth,111void GeneralBilinearFastDownsampler_sse2 (uint8_t* pDst, const int32_t kiDstStride, const int32_t kiDstWidth,
97 const int32_t kiDstHeight, uint8_t* pSrc, const int32_t kiSrcStride, const uint32_t kuiScaleX, const uint32_t kuiScaleY);112 const int32_t kiDstHeight, uint8_t* pSrc, const int32_t kiSrcStride, const uint32_t kuiScaleX,
113 const uint32_t kuiScaleY);
98void GeneralBilinearAccurateDownsampler_sse2 (uint8_t* pDst, const int32_t kiDstStride, const int32_t kiDstWidth,114void GeneralBilinearAccurateDownsampler_sse2 (uint8_t* pDst, const int32_t kiDstStride, const int32_t kiDstWidth,
99 const int32_t kiDstHeight, uint8_t* pSrc, const int32_t kiSrcStride, const uint32_t kuiScaleX, const uint32_t kuiScaleY);115 const int32_t kiDstHeight, uint8_t* pSrc, const int32_t kiSrcStride, const uint32_t kuiScaleX,
116 const uint32_t kuiScaleY);
117
100WELSVP_EXTERN_C_END118WELSVP_EXTERN_C_END
101#endif119#endif
102 120
@@ -109,6 +127,10 @@ HalveDownsampleFunc DyadicBilinearDownsamplerWidthx32_neon; · 6 unmodified lines
109 127
110GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_neon;128GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_neon;
111 129
130SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_neon;
131
132SpecificDownsampleFunc DyadicBilinearQuarterDownsampler_neon;
133
112void GeneralBilinearAccurateDownsampler_neon (uint8_t* pDst, const int32_t kiDstStride, const int32_t kiDstWidth,134void GeneralBilinearAccurateDownsampler_neon (uint8_t* pDst, const int32_t kiDstStride, const int32_t kiDstWidth,
113 const int32_t kiDstHeight,135 const int32_t kiDstHeight,
114 uint8_t* pSrc, const int32_t kiSrcStride, const uint32_t kuiScaleX, const uint32_t kuiScaleY);136 uint8_t* pSrc, const int32_t kiSrcStride, const uint32_t kuiScaleX, const uint32_t kuiScaleY);
@@ -125,8 +147,13 @@ HalveDownsampleFunc DyadicBilinearDownsamplerWidthx32_AArch64_neon; · 10 unmodified lines
125 147
126GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_AArch64_neon;148GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_AArch64_neon;
127 149
128void GeneralBilinearAccurateDownsampler_AArch64_neon (uint8_t* pDst, const int32_t kiDstStride, const int32_t kiDstWidth, const int32_t kiDstHeight,150SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_AArch64_neon;
129 uint8_t* pSrc, const int32_t kiSrcStride, const uint32_t kuiScaleX, const uint32_t kuiScaleY);151
152SpecificDownsampleFunc DyadicBilinearQuarterDownsampler_AArch64_neon;
153
154void GeneralBilinearAccurateDownsampler_AArch64_neon (uint8_t* pDst, const int32_t kiDstStride,
155 const int32_t kiDstWidth, const int32_t kiDstHeight,
156 uint8_t* pSrc, const int32_t kiSrcStride, const uint32_t kuiScaleX, const uint32_t kuiScaleY);
130 157
131WELSVP_EXTERN_C_END158WELSVP_EXTERN_C_END
132#endif159#endif
codec/processing/src/downsample/downsamplefuncs.cpp+47−0
@@ -68,6 +68,53 @@ void DyadicBilinearDownsampler_c (uint8_t* pDst, const int32_t kiDstStride, · 67 unmodified lines
68 }68 }
69}69}
70 70
71void DyadicBilinearQuarterDownsampler_c (uint8_t* pDst, const int32_t kiDstStride,
72 uint8_t* pSrc, const int32_t kiSrcStride,
73 const int32_t kiSrcWidth, const int32_t kiSrcHeight)
74
75{
76 uint8_t* pDstLine = pDst;
77 uint8_t* pSrcLine = pSrc;
78 const int32_t kiSrcStridex4 = kiSrcStride << 2;
79 const int32_t kiDstWidth = kiSrcWidth >> 2;
80 const int32_t kiDstHeight = kiSrcHeight >> 2;
81
82 for (int32_t j = 0; j < kiDstHeight; j ++) {
83 for (int32_t i = 0; i < kiDstWidth; i ++) {
84 const int32_t kiSrcX = i << 2;
85 const int32_t kiTempRow1 = (pSrcLine[kiSrcX] + pSrcLine[kiSrcX + 1] + 1) >> 1;
86 const int32_t kiTempRow2 = (pSrcLine[kiSrcX + kiSrcStride] + pSrcLine[kiSrcX + kiSrcStride + 1] + 1) >> 1;
87
88 pDstLine[i] = (uint8_t) ((kiTempRow1 + kiTempRow2 + 1) >> 1);
89 }
90 pDstLine += kiDstStride;
91 pSrcLine += kiSrcStridex4;
92 }
93}
94
95void DyadicBilinearOneThirdDownsampler_c (uint8_t* pDst, const int32_t kiDstStride,
96 uint8_t* pSrc, const int32_t kiSrcStride,
97 const int32_t kiSrcWidth, const int32_t kiDstHeight)
98
99{
100 uint8_t* pDstLine = pDst;
101 uint8_t* pSrcLine = pSrc;
102 const int32_t kiSrcStridex3 = kiSrcStride * 3;
103 const int32_t kiDstWidth = kiSrcWidth / 3;
104
105 for (int32_t j = 0; j < kiDstHeight; j ++) {
106 for (int32_t i = 0; i < kiDstWidth; i ++) {
107 const int32_t kiSrcX = i * 3;
108 const int32_t kiTempRow1 = (pSrcLine[kiSrcX] + pSrcLine[kiSrcX + 1] + 1) >> 1;
109 const int32_t kiTempRow2 = (pSrcLine[kiSrcX + kiSrcStride] + pSrcLine[kiSrcX + kiSrcStride + 1] + 1) >> 1;
110
111 pDstLine[i] = (uint8_t) ((kiTempRow1 + kiTempRow2 + 1) >> 1);
112 }
113 pDstLine += kiDstStride;
114 pSrcLine += kiSrcStridex3;
115 }
116}
117
71void GeneralBilinearFastDownsampler_c (uint8_t* pDst, const int32_t kiDstStride, const int32_t kiDstWidth,118void GeneralBilinearFastDownsampler_c (uint8_t* pDst, const int32_t kiDstStride, const int32_t kiDstWidth,
72 const int32_t kiDstHeight,119 const int32_t kiDstHeight,
73 uint8_t* pSrc, const int32_t kiSrcStride, const int32_t kiSrcWidth, const int32_t kiSrcHeight) {120 uint8_t* pSrc, const int32_t kiSrcStride, const int32_t kiSrcWidth, const int32_t kiSrcHeight) {
codec/processing/src/x86/downsample_bilinear.asm+699−0
@@ -67,6 +67,22 @@ shufb_mask_high: · 66 unmodified lines
67add_extra_half:67add_extra_half:
68 dd 16384,0,0,068 dd 16384,0,0,0
69 69
70shufb_mask_quarter:
71db 00h, 04h, 08h, 0ch, 80h, 80h, 80h, 80h, 01h, 05h, 09h, 0dh, 80h, 80h, 80h, 80h
72
73shufb_mask_onethird_low_1:
74db 00h, 03h, 06h, 09h, 0ch, 0fh, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h
75shufb_mask_onethird_low_2:
76db 80h, 80h, 80h, 80h, 80h, 80h, 02h, 05h, 08h, 0bh, 0eh, 80h, 80h, 80h, 80h, 80h
77shufb_mask_onethird_low_3:
78db 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 01h, 04h, 07h, 0ah, 0dh
79
80shufb_mask_onethird_high_1:
81db 01h, 04h, 07h, 0ah, 0dh, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h
82shufb_mask_onethird_high_2:
83db 80h, 80h, 80h, 80h, 80h, 00h, 03h, 06h, 09h, 0ch, 0fh, 80h, 80h, 80h, 80h, 80h
84shufb_mask_onethird_high_3:
85db 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 80h, 02h, 05h, 08h, 0bh, 0eh
70 86
71;***********************************************************************87;***********************************************************************
72; Code88; Code
@@ -1896,3 +1912,686 @@ FAST_LAST_ROW_END: · 1823 unmodified lines
1896 pop r121912 pop r12
1897 ret1913 ret
1898%endif1914%endif
1915
1916;***********************************************************************
1917; void DyadicBilinearOneThirdDownsampler_ssse3( unsigned char* pDst, const int iDstStride,
1918; unsigned char* pSrc, const int iSrcStride,
1919; const int iSrcWidth, const int iSrcHeight );
1920;***********************************************************************
1921WELS_EXTERN DyadicBilinearOneThirdDownsampler_ssse3
1922%ifdef X86_32
1923 push r6
1924 %assign push_num 1
1925%else
1926 %assign push_num 0
1927%endif
1928 LOAD_6_PARA
1929 PUSH_XMM 8
1930 SIGN_EXTENSION r1, r1d
1931 SIGN_EXTENSION r3, r3d
1932 SIGN_EXTENSION r4, r4d
1933 SIGN_EXTENSION r5, r5d
1934
1935%ifndef X86_32
1936 push r12
1937 mov r12, r4
1938%endif
1939
1940 mov r6, r1 ;Save the tailer for the unasigned size
1941 imul r6, r5
1942 add r6, r0
1943 movdqa xmm7, [r6]
1944
1945.yloops_onethird_sse3:
1946%ifdef X86_32
1947 mov r4, arg5
1948%else
1949 mov r4, r12
1950%endif
1951
1952 mov r6, r0 ;save base address
1953 ; each loop = source bandwidth: 48 bytes
1954.xloops_onethird_sse3:
1955 ; 1st part horizonal loop: x48 bytes
1956 ; mem hi<- ->lo
1957 ;1st Line Src: xmm0: F * e E * d D * c C * b B * a A
1958 ; xmm2: k K * j J * i I * h H * g G * f
1959 ; xmm2: * p P * o O * n N * m M * l L *
1960 ;
1961 ;2nd Line Src: xmm2: F' * e' E' * d' D' * c' C' * b' B' * a' A'
1962 ; xmm1: k' K' * j' J' * i' I' * h' H' * g' G' * f'
1963 ; xmm1: * p' P' * o' O' * n' N' * m' M' * l' L' *
1964 ;=> target:
1965 ;: P O N M L K J I H G F E D C B A
1966 ;: p o n m l k j i h g f e d c b a
1967 ;: P' .. A'
1968 ;: p' .. a'
1969
1970 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
1971 ;1st line
1972 movdqa xmm0, [r2] ;F * e E * d D * c C * b B * a A
1973 movdqa xmm1, xmm0
1974 movdqa xmm5, [shufb_mask_onethird_low_1]
1975 movdqa xmm6, [shufb_mask_onethird_high_1]
1976 pshufb xmm0, xmm5 ;0 0 0 0 0 0 0 0 0 0 F E D C B A -> xmm0
1977 pshufb xmm1, xmm6 ;0 0 0 0 0 0 0 0 0 0 0 e d c b a -> xmm1
1978
1979 movdqa xmm2, [r2+16] ;k K * j J * i I * h H * g G * f
1980 movdqa xmm3, xmm2
1981 movdqa xmm5, [shufb_mask_onethird_low_2]
1982 movdqa xmm6, [shufb_mask_onethird_high_2]
1983 pshufb xmm2, xmm5 ;0 0 0 0 0 K J I H G 0 0 0 0 0 0 -> xmm2
1984 pshufb xmm3, xmm6 ;0 0 0 0 0 k j i h g f 0 0 0 0 0 -> xmm3
1985
1986 paddusb xmm0, xmm2 ;0 0 0 0 0 K J I H G F E D C B A -> xmm0
1987 paddusb xmm1, xmm3 ;0 0 0 0 0 k j i h g f e d c b a -> xmm1
1988
1989 movdqa xmm2, [r2+32] ;* p P * o O * n N * m M * l L *
1990 movdqa xmm3, xmm2
1991 movdqa xmm5, [shufb_mask_onethird_low_3]
1992 movdqa xmm6, [shufb_mask_onethird_high_3]
1993 pshufb xmm2, xmm5 ;P O N M L 0 0 0 0 0 0 0 0 0 0 0 -> xmm2
1994 pshufb xmm3, xmm6 ;p o n m l 0 0 0 0 0 0 0 0 0 0 0 -> xmm3
1995
1996 paddusb xmm0, xmm2 ;P O N M L K J I H G F E D C B A -> xmm0
1997 paddusb xmm1, xmm3 ;p o n m l k j i h g f e d c b a -> xmm1
1998 pavgb xmm0, xmm1 ;1st line average -> xmm0
1999
2000 ;2nd line
2001 movdqa xmm2, [r2+r3] ;F' * e' E' * d' D' * c' C' * b' B' * a' A'
2002 movdqa xmm3, xmm2
2003 movdqa xmm5, [shufb_mask_onethird_low_1]
2004 movdqa xmm6, [shufb_mask_onethird_high_1]
2005 pshufb xmm2, xmm5 ;0 0 0 0 0 0 0 0 0 0 F' E' D' C' B' A' -> xmm2
2006 pshufb xmm3, xmm6 ;0 0 0 0 0 0 0 0 0 0 0 e' d' c' b' a' -> xmm3
2007
2008 movdqa xmm1, [r2+r3+16] ;k' K' * j' J' * i' I' * h' H' * g' G' * f'
2009 movdqa xmm4, xmm1
2010 movdqa xmm5, [shufb_mask_onethird_low_2]
2011 movdqa xmm6, [shufb_mask_onethird_high_2]
2012 pshufb xmm1, xmm5 ;0 0 0 0 0 K' J' I' H' G' 0 0 0 0 0 0 -> xmm1
2013 pshufb xmm4, xmm6 ;0 0 0 0 0 k' j' i' h' g' f' 0 0 0 0 0 -> xmm4
2014
2015 paddusb xmm2, xmm1 ;0 0 0 0 0 K' J' I' H' G' F' E' D' C' B' A' -> xmm2
2016 paddusb xmm3, xmm4 ;0 0 0 0 0 k' j' i' h' g' f' e' d' c' b' a' -> xmm3
2017
2018 movdqa xmm1, [r2+r3+32] ; * p' P' * o' O' * n' N' * m' M' * l' L' *
2019 movdqa xmm4, xmm1
2020 movdqa xmm5, [shufb_mask_onethird_low_3]
2021 movdqa xmm6, [shufb_mask_onethird_high_3]
2022 pshufb xmm1, xmm5 ;P' O' N' M' L' 0 0 0 0 0 0 0 0 0 0 0 -> xmm1
2023 pshufb xmm4, xmm6 ;p' o' n' m' l' 0 0 0 0 0 0 0 0 0 0 0 -> xmm4
2024
2025 paddusb xmm2, xmm1 ;P' O' N' M' L' K' J' I' H' G' F' E' D' C' B' A' -> xmm2
2026 paddusb xmm3, xmm4 ;p' o' n' m' l' k' j' i' h' g' f' e' d' c' b' a' -> xmm3
2027 pavgb xmm2, xmm3 ;2nd line average -> xmm2
2028
2029 pavgb xmm0, xmm2 ; bytes-average(1st line , 2nd line )
2030
2031 ; write pDst
2032 movdqa [r0], xmm0 ;write result in dst
2033
2034 ; next SMB
2035 lea r2, [r2+48] ;current src address
2036 lea r0, [r0+16] ;current dst address
2037
2038 sub r4, 48 ;xloops counter
2039 cmp r4, 0
2040 jg near .xloops_onethird_sse3
2041
2042 sub r6, r0 ;offset = base address - current address
2043 lea r2, [r2+2*r3] ;
2044 lea r2, [r2+r3] ;
2045 lea r2, [r2+2*r6] ;current line + 3 lines
2046 lea r2, [r2+r6]
2047 lea r0, [r0+r1]
2048 lea r0, [r0+r6] ;current dst lien + 1 line
2049
2050 dec r5
2051 jg near .yloops_onethird_sse3
2052
2053 movdqa [r0], xmm7 ;restore the tailer for the unasigned size
2054
2055%ifndef X86_32
2056 pop r12
2057%endif
2058
2059 POP_XMM
2060 LOAD_6_PARA_POP
2061%ifdef X86_32
2062 pop r6
2063%endif
2064 ret
2065
2066;***********************************************************************
2067; void DyadicBilinearOneThirdDownsampler_sse4( unsigned char* pDst, const int iDstStride,
2068; unsigned char* pSrc, const int iSrcStride,
2069; const int iSrcWidth, const int iSrcHeight );
2070;***********************************************************************
2071WELS_EXTERN DyadicBilinearOneThirdDownsampler_sse4
2072%ifdef X86_32
2073 push r6
2074 %assign push_num 1
2075%else
2076 %assign push_num 0
2077%endif
2078 LOAD_6_PARA
2079 PUSH_XMM 8
2080 SIGN_EXTENSION r1, r1d
2081 SIGN_EXTENSION r3, r3d
2082 SIGN_EXTENSION r4, r4d
2083 SIGN_EXTENSION r5, r5d
2084
2085%ifndef X86_32
2086 push r12
2087 mov r12, r4
2088%endif
2089
2090 mov r6, r1 ;Save the tailer for the unasigned size
2091 imul r6, r5
2092 add r6, r0
2093 movdqa xmm7, [r6]
2094
2095.yloops_onethird_sse4:
2096%ifdef X86_32
2097 mov r4, arg5
2098%else
2099 mov r4, r12
2100%endif
2101
2102 mov r6, r0 ;save base address
2103 ; each loop = source bandwidth: 48 bytes
2104.xloops_onethird_sse4:
2105 ; 1st part horizonal loop: x48 bytes
2106 ; mem hi<- ->lo
2107 ;1st Line Src: xmm0: F * e E * d D * c C * b B * a A
2108 ; xmm2: k K * j J * i I * h H * g G * f
2109 ; xmm2: * p P * o O * n N * m M * l L *
2110 ;
2111 ;2nd Line Src: xmm2: F' * e' E' * d' D' * c' C' * b' B' * a' A'
2112 ; xmm1: k' K' * j' J' * i' I' * h' H' * g' G' * f'
2113 ; xmm1: * p' P' * o' O' * n' N' * m' M' * l' L' *
2114 ;=> target:
2115 ;: P O N M L K J I H G F E D C B A
2116 ;: p o n m l k j i h g f e d c b a
2117 ;: P' .. A'
2118 ;: p' .. a'
2119
2120 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2121 ;1st line
2122 movntdqa xmm0, [r2] ;F * e E * d D * c C * b B * a A
2123 movdqa xmm1, xmm0
2124 movdqa xmm5, [shufb_mask_onethird_low_1]
2125 movdqa xmm6, [shufb_mask_onethird_high_1]
2126 pshufb xmm0, xmm5 ;0 0 0 0 0 0 0 0 0 0 F E D C B A -> xmm0
2127 pshufb xmm1, xmm6 ;0 0 0 0 0 0 0 0 0 0 0 e d c b a -> xmm1
2128
2129 movntdqa xmm2, [r2+16] ;k K * j J * i I * h H * g G * f
2130 movdqa xmm3, xmm2
2131 movdqa xmm5, [shufb_mask_onethird_low_2]
2132 movdqa xmm6, [shufb_mask_onethird_high_2]
2133 pshufb xmm2, xmm5 ;0 0 0 0 0 K J I H G 0 0 0 0 0 0 -> xmm2
2134 pshufb xmm3, xmm6 ;0 0 0 0 0 k j i h g f 0 0 0 0 0 -> xmm3
2135
2136 paddusb xmm0, xmm2 ;0 0 0 0 0 K J I H G F E D C B A -> xmm0
2137 paddusb xmm1, xmm3 ;0 0 0 0 0 k j i h g f e d c b a -> xmm1
2138
2139 movntdqa xmm2, [r2+32] ;* p P * o O * n N * m M * l L *
2140 movdqa xmm3, xmm2
2141 movdqa xmm5, [shufb_mask_onethird_low_3]
2142 movdqa xmm6, [shufb_mask_onethird_high_3]
2143 pshufb xmm2, xmm5 ;P O N M L 0 0 0 0 0 0 0 0 0 0 0 -> xmm2
2144 pshufb xmm3, xmm6 ;p o n m l 0 0 0 0 0 0 0 0 0 0 0 -> xmm3
2145
2146 paddusb xmm0, xmm2 ;P O N M L K J I H G F E D C B A -> xmm0
2147 paddusb xmm1, xmm3 ;p o n m l k j i h g f e d c b a -> xmm1
2148 pavgb xmm0, xmm1 ;1st line average -> xmm0
2149
2150 ;2nd line
2151 movntdqa xmm2, [r2+r3] ;F' * e' E' * d' D' * c' C' * b' B' * a' A'
2152 movdqa xmm3, xmm2
2153 movdqa xmm5, [shufb_mask_onethird_low_1]
2154 movdqa xmm6, [shufb_mask_onethird_high_1]
2155 pshufb xmm2, xmm5 ;0 0 0 0 0 0 0 0 0 0 F' E' D' C' B' A' -> xmm2
2156 pshufb xmm3, xmm6 ;0 0 0 0 0 0 0 0 0 0 0 e' d' c' b' a' -> xmm3
2157
2158 movntdqa xmm1, [r2+r3+16] ;k' K' * j' J' * i' I' * h' H' * g' G' * f'
2159 movdqa xmm4, xmm1
2160 movdqa xmm5, [shufb_mask_onethird_low_2]
2161 movdqa xmm6, [shufb_mask_onethird_high_2]
2162 pshufb xmm1, xmm5 ;0 0 0 0 0 K' J' I' H' G' 0 0 0 0 0 0 -> xmm1
2163 pshufb xmm4, xmm6 ;0 0 0 0 0 k' j' i' h' g' f' 0 0 0 0 0 -> xmm4
2164
2165 paddusb xmm2, xmm1 ;0 0 0 0 0 K' J' I' H' G' F' E' D' C' B' A' -> xmm2
2166 paddusb xmm3, xmm4 ;0 0 0 0 0 k' j' i' h' g' f' e' d' c' b' a' -> xmm3
2167
2168 movntdqa xmm1, [r2+r3+32] ; * p' P' * o' O' * n' N' * m' M' * l' L' *
2169 movdqa xmm4, xmm1
2170 movdqa xmm5, [shufb_mask_onethird_low_3]
2171 movdqa xmm6, [shufb_mask_onethird_high_3]
2172 pshufb xmm1, xmm5 ;P' O' N' M' L' 0 0 0 0 0 0 0 0 0 0 0 -> xmm1
2173 pshufb xmm4, xmm6 ;p' o' n' m' l' 0 0 0 0 0 0 0 0 0 0 0 -> xmm4
2174
2175 paddusb xmm2, xmm1 ;P' O' N' M' L' K' J' I' H' G' F' E' D' C' B' A' -> xmm2
2176 paddusb xmm3, xmm4 ;p' o' n' m' l' k' j' i' h' g' f' e' d' c' b' a' -> xmm3
2177 pavgb xmm2, xmm3 ;2nd line average -> xmm2
2178
2179 pavgb xmm0, xmm2 ; bytes-average(1st line , 2nd line )
2180
2181 ; write pDst
2182 movdqa [r0], xmm0 ;write result in dst
2183
2184 ; next SMB
2185 lea r2, [r2+48] ;current src address
2186 lea r0, [r0+16] ;current dst address
2187
2188 sub r4, 48 ;xloops counter
2189 cmp r4, 0
2190 jg near .xloops_onethird_sse4
2191
2192 sub r6, r0 ;offset = base address - current address
2193 lea r2, [r2+2*r3] ;
2194 lea r2, [r2+r3] ;
2195 lea r2, [r2+2*r6] ;current line + 3 lines
2196 lea r2, [r2+r6]
2197 lea r0, [r0+r1]
2198 lea r0, [r0+r6] ;current dst lien + 1 line
2199
2200 dec r5
2201 jg near .yloops_onethird_sse4
2202
2203 movdqa [r0], xmm7 ;restore the tailer for the unasigned size
2204
2205%ifndef X86_32
2206 pop r12
2207%endif
2208
2209 POP_XMM
2210 LOAD_6_PARA_POP
2211%ifdef X86_32
2212 pop r6
2213%endif
2214 ret
2215
2216;***********************************************************************
2217; void DyadicBilinearQuarterDownsampler_sse( unsigned char* pDst, const int iDstStride,
2218; unsigned char* pSrc, const int iSrcStride,
2219; const int iSrcWidth, const int iSrcHeight );
2220;***********************************************************************
2221WELS_EXTERN DyadicBilinearQuarterDownsampler_sse
2222%ifdef X86_32
2223 push r6
2224 %assign push_num 1
2225%else
2226 %assign push_num 0
2227%endif
2228 LOAD_6_PARA
2229 SIGN_EXTENSION r1, r1d
2230 SIGN_EXTENSION r3, r3d
2231 SIGN_EXTENSION r4, r4d
2232 SIGN_EXTENSION r5, r5d
2233
2234%ifndef X86_32
2235 push r12
2236 mov r12, r4
2237%endif
2238 sar r5, $02 ; iSrcHeight >> 2
2239
2240 mov r6, r1 ;Save the tailer for the unasigned size
2241 imul r6, r5
2242 add r6, r0
2243 movq xmm7, [r6]
2244
2245.yloops_quarter_sse:
2246%ifdef X86_32
2247 mov r4, arg5
2248%else
2249 mov r4, r12
2250%endif
2251
2252 mov r6, r0 ;save base address
2253 ; each loop = source bandwidth: 32 bytes
2254.xloops_quarter_sse:
2255 ; 1st part horizonal loop: x16 bytes
2256 ; mem hi<- ->lo
2257 ;1st Line Src: mm0: d D c C b B a A mm1: h H g G f F e E
2258 ;2nd Line Src: mm2: l L k K j J i I mm3: p P o O n N m M
2259 ;
2260 ;=> target:
2261 ;: G E C A,
2262 ;:
2263 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2264 movq mm0, [r2] ; 1st pSrc line
2265 movq mm1, [r2+8] ; 1st pSrc line + 8
2266 movq mm2, [r2+r3] ; 2nd pSrc line
2267 movq mm3, [r2+r3+8] ; 2nd pSrc line + 8
2268
2269 pshufw mm0, mm0, 0d8h ; x X x X c C a A
2270 pshufw mm1, mm1, 0d8h ; x X x X g G e E
2271 pshufw mm2, mm2, 0d8h ; x X x X k K i I
2272 pshufw mm3, mm3, 0d8h ; x X x X o O m M
2273
2274 punpckldq mm0, mm1 ; g G e E c C a A
2275 punpckldq mm2, mm3 ; o O m M k K i I
2276
2277 ; to handle mm0,mm2
2278 pshufw mm4, mm0, 0d8h ;g G c C e E a A
2279 pshufw mm5, mm4, 04eh ;e E a A g G c C
2280 punpcklbw mm4, mm5 ;g e G E c a C A -> mm4
2281 pshufw mm4, mm4, 0d8h ;g e c a G E C A -> mm4
2282
2283 pshufw mm5, mm2, 0d8h ;o O k K m M i I
2284 pshufw mm6, mm5, 04eh ;m M i I o O k K
2285 punpcklbw mm5, mm6 ;o m O M k i K I
2286 pshufw mm5, mm5, 0d8h ;o m k i O M K I -> mm5
2287
2288 ; to handle mm4, mm5
2289 movq mm0, mm4
2290 punpckldq mm0, mm6 ;x x x x G E C A
2291 punpckhdq mm4, mm6 ;x x x x g e c a
2292
2293 movq mm1, mm5
2294 punpckldq mm1, mm6 ;x x x x O M K I
2295 punpckhdq mm5, mm6 ;x x x x o m k i
2296
2297 ; avg within MB horizon width (8 x 2 lines)
2298 pavgb mm0, mm4 ; (A+a+1)>>1, .., (H+h+1)>>1, temp_row1
2299 pavgb mm1, mm5 ; (I+i+1)>>1, .., (P+p+1)>>1, temp_row2
2300 pavgb mm0, mm1 ; (temp_row1+temp_row2+1)>>1, pending here and wait another horizonal part done then write memory once
2301
2302 ; 2nd part horizonal loop: x16 bytes
2303 movq mm1, [r2+16] ; 1st pSrc line + 16
2304 movq mm2, [r2+24] ; 1st pSrc line + 24
2305 movq mm3, [r2+r3+16] ; 2nd pSrc line + 16
2306 movq mm4, [r2+r3+24] ; 2nd pSrc line + 24
2307
2308 pshufw mm1, mm1, 0d8h
2309 pshufw mm2, mm2, 0d8h
2310 pshufw mm3, mm3, 0d8h
2311 pshufw mm4, mm4, 0d8h
2312
2313 punpckldq mm1, mm2
2314 punpckldq mm3, mm4
2315
2316 ; to handle mm1, mm3
2317 pshufw mm4, mm1, 0d8h
2318 pshufw mm5, mm4, 04eh
2319 punpcklbw mm4, mm5
2320 pshufw mm4, mm4, 0d8h
2321
2322 pshufw mm5, mm3, 0d8h
2323 pshufw mm6, mm5, 04eh
2324 punpcklbw mm5, mm6
2325 pshufw mm5, mm5, 0d8h
2326
2327 ; to handle mm4, mm5
2328 movq mm2, mm4
2329 punpckldq mm2, mm6
2330 punpckhdq mm4, mm6
2331
2332 movq mm3, mm5
2333 punpckldq mm3, mm6
2334 punpckhdq mm5, mm6
2335
2336 ; avg within MB horizon width (8 x 2 lines)
2337 pavgb mm2, mm4 ; (A+a+1)>>1, .., (H+h+1)>>1, temp_row1
2338 pavgb mm3, mm5 ; (I+i+1)>>1, .., (P+p+1)>>1, temp_row2
2339 pavgb mm2, mm3 ; (temp_row1+temp_row2+1)>>1, done in another 2nd horizonal part
2340
2341 movd [r0 ], mm0
2342 movd [r0+4], mm2
2343
2344 ; next SMB
2345 lea r2, [r2+32]
2346 lea r0, [r0+8]
2347
2348 sub r4, 32
2349 cmp r4, 0
2350 jg near .xloops_quarter_sse
2351
2352 sub r6, r0
2353 ; next line
2354 lea r2, [r2+4*r3] ; next 4 end of lines
2355 lea r2, [r2+4*r6] ; reset to base 0 [- 4 * iDstWidth]
2356 lea r0, [r0+r1]
2357 lea r0, [r0+r6] ; reset to base 0 [- iDstWidth]
2358
2359 dec r5
2360 jg near .yloops_quarter_sse
2361
2362 movq [r0], xmm7 ;restored the tailer for the unasigned size
2363
2364 WELSEMMS
2365%ifndef X86_32
2366 pop r12
2367%endif
2368 LOAD_6_PARA_POP
2369%ifdef X86_32
2370 pop r6
2371%endif
2372 ret
2373
2374;***********************************************************************
2375; void DyadicBilinearQuarterDownsampler_ssse3( unsigned char* pDst, const int iDstStride,
2376; unsigned char* pSrc, const int iSrcStride,
2377; const int iSrcWidth, const int iSrcHeight );
2378;***********************************************************************
2379WELS_EXTERN DyadicBilinearQuarterDownsampler_ssse3
2380 ;push ebx
2381 ;push edx
2382 ;push esi
2383 ;push edi
2384 ;push ebp
2385
2386 ;mov edi, [esp+24] ; pDst
2387 ;mov edx, [esp+28] ; iDstStride
2388 ;mov esi, [esp+32] ; pSrc
2389 ;mov ecx, [esp+36] ; iSrcStride
2390 ;mov ebp, [esp+44] ; iSrcHeight
2391%ifdef X86_32
2392 push r6
2393 %assign push_num 1
2394%else
2395 %assign push_num 0
2396%endif
2397 LOAD_6_PARA
2398 PUSH_XMM 8
2399 SIGN_EXTENSION r1, r1d
2400 SIGN_EXTENSION r3, r3d
2401 SIGN_EXTENSION r4, r4d
2402 SIGN_EXTENSION r5, r5d
2403
2404%ifndef X86_32
2405 push r12
2406 mov r12, r4
2407%endif
2408 sar r5, $02 ; iSrcHeight >> 2
2409
2410 mov r6, r1 ;Save the tailer for the unasigned size
2411 imul r6, r5
2412 add r6, r0
2413 movq xmm7, [r6]
2414
2415 movdqa xmm6, [shufb_mask_quarter]
2416.yloops_quarter_sse3:
2417 ;mov eax, [esp+40] ; iSrcWidth
2418 ;sar eax, $02 ; iSrcWidth >> 2
2419 ;mov ebx, eax ; iDstWidth restored at ebx
2420 ;sar eax, $04 ; (iSrcWidth >> 2) / 16 ; loop count = num_of_mb
2421 ;neg ebx ; - (iSrcWidth >> 2)
2422%ifdef X86_32
2423 mov r4, arg5
2424%else
2425 mov r4, r12
2426%endif
2427
2428 mov r6, r0
2429 ; each loop = source bandwidth: 32 bytes
2430.xloops_quarter_sse3:
2431 ; 1st part horizonal loop: x32 bytes
2432 ; mem hi<- ->lo
2433 ;1st Line Src: xmm0: h H g G f F e E d D c C b B a A
2434 ; xmm1: p P o O n N m M l L k K j J i I
2435 ;2nd Line Src: xmm2: h H g G f F e E d D c C b B a A
2436 ; xmm3: p P o O n N m M l L k K j J i I
2437
2438 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2439 movdqa xmm0, [r2] ; 1st_src_line
2440 movdqa xmm1, [r2+16] ; 1st_src_line + 16
2441 movdqa xmm2, [r2+r3] ; 2nd_src_line
2442 movdqa xmm3, [r2+r3+16] ; 2nd_src_line + 16
2443
2444 pshufb xmm0, xmm6 ;1st line: 0 0 0 0 g e c a 0 0 0 0 G E C A
2445 pshufb xmm1, xmm6 ;1st line: 0 0 0 0 o m k i 0 0 0 0 O M K I
2446 pshufb xmm2, xmm6 ;2nd line: 0 0 0 0 g e c a 0 0 0 0 G E C A
2447 pshufb xmm3, xmm6 ;2nd line: 0 0 0 0 o m k i 0 0 0 0 O M K I
2448
2449 movdqa xmm4, xmm0
2450 movdqa xmm5, xmm2
2451 punpckldq xmm0, xmm1 ;1st line: 0 0 0 0 0 0 0 0 O M K I G E C A -> xmm0
2452 punpckhdq xmm4, xmm1 ;1st line: 0 0 0 0 0 0 0 0 o m k i g e c a -> xmm4
2453 punpckldq xmm2, xmm3 ;2nd line: 0 0 0 0 0 0 0 0 O M K I G E C A -> xmm2
2454 punpckhdq xmm5, xmm3 ;2nd line: 0 0 0 0 0 0 0 0 o m k i g e c a -> xmm5
2455
2456 pavgb xmm0, xmm4
2457 pavgb xmm2, xmm5
2458 pavgb xmm0, xmm2 ;average
2459
2460 ; write pDst
2461 movq [r0], xmm0
2462
2463 ; next SMB
2464 lea r2, [r2+32]
2465 lea r0, [r0+8]
2466
2467 sub r4, 32
2468 cmp r4, 0
2469 jg near .xloops_quarter_sse3
2470
2471 sub r6, r0
2472 ; next line
2473 lea r2, [r2+4*r3] ; next end of lines
2474 lea r2, [r2+4*r6] ; reset to base 0 [- 4 * iDstWidth]
2475 lea r0, [r0+r1]
2476 lea r0, [r0+r6] ; reset to base 0 [- iDstWidth]
2477
2478 dec r5
2479 jg near .yloops_quarter_sse3
2480
2481 movq [r0], xmm7 ;restored the tailer for the unasigned size
2482
2483%ifndef X86_32
2484 pop r12
2485%endif
2486
2487 POP_XMM
2488 LOAD_6_PARA_POP
2489%ifdef X86_32
2490 pop r6
2491%endif
2492 ret
2493
2494;***********************************************************************
2495; void DyadicBilinearQuarterDownsampler_sse4( unsigned char* pDst, const int iDstStride,
2496; unsigned char* pSrc, const int iSrcStride,
2497; const int iSrcWidth, const int iSrcHeight );
2498;***********************************************************************
2499WELS_EXTERN DyadicBilinearQuarterDownsampler_sse4
2500%ifdef X86_32
2501 push r6
2502 %assign push_num 1
2503%else
2504 %assign push_num 0
2505%endif
2506 LOAD_6_PARA
2507 PUSH_XMM 8
2508 SIGN_EXTENSION r1, r1d
2509 SIGN_EXTENSION r3, r3d
2510 SIGN_EXTENSION r4, r4d
2511 SIGN_EXTENSION r5, r5d
2512
2513%ifndef X86_32
2514 push r12
2515 mov r12, r4
2516%endif
2517 sar r5, $02 ; iSrcHeight >> 2
2518
2519 mov r6, r1 ;Save the tailer for the unasigned size
2520 imul r6, r5
2521 add r6, r0
2522 movq xmm7, [r6]
2523
2524 movdqa xmm6, [shufb_mask_quarter] ;mask
2525
2526.yloops_quarter_sse4:
2527%ifdef X86_32
2528 mov r4, arg5
2529%else
2530 mov r4, r12
2531%endif
2532
2533 mov r6, r0
2534 ; each loop = source bandwidth: 32 bytes
2535.xloops_quarter_sse4:
2536 ; 1st part horizonal loop: x16 bytes
2537 ; mem hi<- ->lo
2538 ;1st Line Src: xmm0: h H g G f F e E d D c C b B a A
2539 ; xmm1: p P o O n N m M l L k K j J i I
2540 ;2nd Line Src: xmm2: h H g G f F e E d D c C b B a A
2541 ; xmm3: p P o O n N m M l L k K j J i I
2542
2543 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
2544 movntdqa xmm0, [r2] ; 1st_src_line
2545 movntdqa xmm1, [r2+16] ; 1st_src_line + 16
2546 movntdqa xmm2, [r2+r3] ; 2nd_src_line
2547 movntdqa xmm3, [r2+r3+16] ; 2nd_src_line + 16
2548
2549 pshufb xmm0, xmm6 ;1st line: 0 0 0 0 g e c a 0 0 0 0 G E C A
2550 pshufb xmm1, xmm6 ;1st line: 0 0 0 0 o m k i 0 0 0 0 O M K I
2551 pshufb xmm2, xmm6 ;2nd line: 0 0 0 0 g e c a 0 0 0 0 G E C A
2552 pshufb xmm3, xmm6 ;2nd line: 0 0 0 0 o m k i 0 0 0 0 O M K I
2553
2554 movdqa xmm4, xmm0
2555 movdqa xmm5, xmm2
2556 punpckldq xmm0, xmm1 ;1st line: 0 0 0 0 0 0 0 0 O M K I G E C A -> xmm0
2557 punpckhdq xmm4, xmm1 ;1st line: 0 0 0 0 0 0 0 0 o m k i g e c a -> xmm4
2558 punpckldq xmm2, xmm3 ;2nd line: 0 0 0 0 0 0 0 0 O M K I G E C A -> xmm2
2559 punpckhdq xmm5, xmm3 ;2nd line: 0 0 0 0 0 0 0 0 o m k i g e c a -> xmm5
2560
2561 pavgb xmm0, xmm4
2562 pavgb xmm2, xmm5
2563 pavgb xmm0, xmm2 ;average
2564
2565 ; write pDst
2566 movq [r0], xmm0
2567
2568 ; next SMB
2569 lea r2, [r2+32]
2570 lea r0, [r0+8]
2571
2572 sub r4, 32
2573 cmp r4, 0
2574 jg near .xloops_quarter_sse4
2575
2576 sub r6, r0
2577 lea r2, [r2+4*r3] ; next end of lines
2578 lea r2, [r2+4*r6] ; reset to base 0 [- 2 * iDstWidth]
2579 lea r0, [r0+r1]
2580 lea r0, [r0+r6] ; reset to base 0 [- iDstWidth]
2581
2582 dec r5
2583 jg near .yloops_quarter_sse4
2584
2585 movq [r0], xmm7 ;restore the tailer for the unasigned size
2586
2587%ifndef X86_32
2588 pop r12
2589%endif
2590
2591 POP_XMM
2592 LOAD_6_PARA_POP
2593%ifdef X86_32
2594 pop r6
2595%endif
2596 ret
2597

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t