History

[Processing/x86] Add an AVX2 implementation of GeneralBilinearAccurateDownsample

Keep track of relative pixel offsets and utilize pshufb to efficiently extract relevant pixels for horizontal scaling ratios <= 8. Because pshufb does not cross 128-bit lanes, the overhead of address calculations and loads is relatively greater as compared with an SSSE3/SSE4.1 implementation. Fall back to a generic approach for ratios > 8. The implementation assumes that data beyond the end of each line, before the next line begins, can be dirtied; which AFAICT is safe with the current usage of these routines. Speedup is ~8.52x/~6.89x (32-bit/64-bit) for horizontal ratios <= 2, ~7.81x/~6.13x for ratios within (2, 4], ~5.81x/~4.52x for ratios within (4, 8], and ~5.06x/~4.09x for ratios > 8 when not memory-bound on Haswell as compared with the current SSE2 implementation.

Sindre Aamås committed 10 years ago

Commit e4902151 parent b43e58a5 files · +616 −0Browse files

Showing the changes to codec/processing/src (4 of 5 files). Show all

Unified · Split

4 files
  1. codec/processing/src/downsample/downsample.cpp +1 −0
  2. codec/processing/src/downsample/downsample.h +4 −0
  3. codec/processing/src/downsample/downsamplefuncs.cpp +1 −0
  4. codec/processing/src/x86/downsample_bilinear.asm +608 −0
codec/processing/src/downsample/downsample.cpp+1−0
@@ -110,6 +110,7 @@ void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int · 109 unmodified lines
110110 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_sse41;
111111 }
112112 if (iCpuFlag & WELS_CPU_AVX2) {
113+ sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_avx2;
113114 sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearFastDownsamplerWrap_avx2;
114115 }
115116 #endif//X86_ASM
codec/processing/src/downsample/downsample.h+4−0
@@ -104,6 +104,7 @@ GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse2; · 103 unmodified lines
104104 GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_ssse3;
105105 GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse41;
106106 GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_avx2;
107+GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_avx2;
107108
108109 SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_ssse3;
109110 SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_sse4;
@@ -126,6 +127,9 @@ void GeneralBilinearAccurateDownsampler_sse41 (uint8_t* pDst, int32_t iDstStride · 16 unmodified lines
126127 void GeneralBilinearFastDownsampler_avx2 (uint8_t* pDst, int32_t iDstStride, int32_t iDstWidth,
127128 int32_t iDstHeight, uint8_t* pSrc, int32_t iSrcStride, uint32_t uiScaleX,
128129 uint32_t uiScaleY);
130+void GeneralBilinearAccurateDownsampler_avx2 (uint8_t* pDst, int32_t iDstStride, int32_t iDstWidth,
131+ int32_t iDstHeight, uint8_t* pSrc, int32_t iSrcStride, uint32_t uiScaleX,
132+ uint32_t uiScaleY);
129133
130134 WELSVP_EXTERN_C_END
131135 #endif
codec/processing/src/downsample/downsamplefuncs.cpp+1−0
@@ -285,6 +285,7 @@ DEFINE_GENERAL_BILINEAR_ACCURATE_DOWNSAMPLER_WRAP (sse2) · 284 unmodified lines
285285 DEFINE_GENERAL_BILINEAR_FAST_DOWNSAMPLER_WRAP (ssse3)
286286 DEFINE_GENERAL_BILINEAR_ACCURATE_DOWNSAMPLER_WRAP (sse41)
287287 DEFINE_GENERAL_BILINEAR_FAST_DOWNSAMPLER_WRAP (avx2)
288+DEFINE_GENERAL_BILINEAR_ACCURATE_DOWNSAMPLER_WRAP (avx2)
288289 #endif //X86_ASM
289290
290291 #ifdef HAVE_NEON
codec/processing/src/x86/downsample_bilinear.asm+608−0
@@ -3854,6 +3854,291 @@ WELS_EXTERN GeneralBilinearAccurateDownsampler_sse41 · 3853 unmodified lines
38543854 vpaddw ymm_xfrac1, ymm_xfrac1, ymm_xfrac_inc
38553855 %endmacro
38563856
3857+; xpos_int=%1 xpos_frac=%2 inc_int=%3 inc_frac=%4 7FFFh=%5 tmp=%6,%7
3858+%macro AVX2_BilinearIncXposw 7
3859+ vpaddb %1, %1, %3
3860+ vpaddw %6, %2, %4
3861+ vpcmpgtw %7, %2, %6
3862+ vpsubb %1, %1, %7 ; add carry
3863+ vpand %2, %6, %5
3864+%endmacro
3865+
3866+; res>>29=%1 data0=%2 data1=%3 frac0=%4 frac1=%5 tmp=%6
3867+%macro AVX2_LinearAccurateInterpolateVerticalDwords 6
3868+ vpshufd %1, %2, 10110001b
3869+ vpshufd %6, %3, 10110001b
3870+ vpmuludq %1, %1, %4
3871+ vpmuludq %6, %6, %5
3872+ vpaddq %1, %1, %6
3873+ vpmuludq %2, %2, %4
3874+ vpmuludq %3, %3, %5
3875+ vpaddq %2, %2, %3
3876+ vpsllq %1, %1, 3
3877+ vpsrlq %2, %2, 29
3878+ vpblendd %1, %1, %2, 01010101b
3879+%endmacro
3880+
3881+%macro AVX2_BilinearAccurateDownsample2xOrLess_16px 0
3882+ vpshufb ymm_tmp0, ymm_xpos_int, ymm_0
3883+ vpsubb ymm_xpos_int, ymm_xpos_int, ymm_tmp0
3884+ AVX2_UnpckXFrac ymm_tmp0, ymm_tmp1, ymm_xpos_frac, ymm_7fff
3885+ mov r_tmp0, i_xpos
3886+ shr r_tmp0, 16
3887+ vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3888+ vmovdqu xmm_tmp5, [p_src_row1 + r_tmp0]
3889+ lea r_tmp0, [i_xpos + 4 * i_scalex2]
3890+ lea i_xpos, [i_xpos + 8 * i_scalex2]
3891+ shr r_tmp0, 16
3892+ vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
3893+ vinserti128 ymm_tmp5, ymm_tmp5, [p_src_row1 + r_tmp0], 1
3894+ vpshufb ymm_tmp4, ymm_tmp4, ymm_xpos_int
3895+ vpshufb ymm_tmp5, ymm_tmp5, ymm_xpos_int
3896+ vpunpcklbw ymm_tmp2, ymm_tmp4, ymm_0
3897+ vpunpcklbw ymm_tmp3, ymm_tmp5, ymm_0
3898+ vpunpckhbw ymm_tmp4, ymm_tmp4, ymm_0
3899+ vpunpckhbw ymm_tmp5, ymm_tmp5, ymm_0
3900+ vpmaddwd ymm_tmp2, ymm_tmp2, ymm_tmp0
3901+ vpmaddwd ymm_tmp3, ymm_tmp3, ymm_tmp0
3902+ vpmaddwd ymm_tmp4, ymm_tmp4, ymm_tmp1
3903+ vpmaddwd ymm_tmp5, ymm_tmp5, ymm_tmp1
3904+ AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp0, ymm_tmp2, ymm_tmp3, ymm_yfrac0, ymm_yfrac1, ymm_tmp1
3905+ AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp1, ymm_tmp4, ymm_tmp5, ymm_yfrac0, ymm_yfrac1, ymm_tmp2
3906+ vpackssdw ymm_tmp0, ymm_tmp0, ymm_tmp1
3907+ vpavgw ymm_tmp0, ymm_tmp0, ymm_0
3908+ vpackuswb ymm_tmp0, ymm_tmp0, ymm_tmp0
3909+ vmovlps [p_dst], xmm_tmp0
3910+ vextracti128 [p_dst + 8], ymm_tmp0, 1
3911+ add p_dst, 16
3912+ AVX2_BilinearIncXposw ymm_xpos_int, ymm_xpos_frac, ymm_xpos_int_inc, ymm_xpos_frac_inc, ymm_7fff, ymm_tmp0, ymm_tmp1
3913+%endmacro
3914+
3915+%macro AVX2_BilinearAccurateDownsample4xOrLess_16px 0
3916+ vbroadcasti128 ymm_tmp0, [shufb_0000000088888888]
3917+ vpshufb ymm_tmp0, ymm_xpos_int, ymm_tmp0
3918+ vpsubb ymm_xpos_int, ymm_xpos_int, ymm_tmp0
3919+ AVX2_UnpckXFrac ymm_tmp0, ymm_tmp1, ymm_xpos_frac, ymm_7fff
3920+ mov r_tmp0, i_xpos
3921+ shr r_tmp0, 16
3922+ vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3923+ vmovdqu xmm_tmp2, [p_src_row1 + r_tmp0]
3924+ lea r_tmp0, [i_xpos + 4 * i_scalex2]
3925+ shr r_tmp0, 16
3926+ vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
3927+ vinserti128 ymm_tmp2, ymm_tmp2, [p_src_row1 + r_tmp0], 1
3928+ lea r_tmp0, [i_xpos + 2 * i_scalex2]
3929+ lea i_xpos, [r_tmp0 + 4 * i_scalex2]
3930+ shr r_tmp0, 16
3931+ vpunpcklbw ymm_tmp3, ymm_xpos_int, [db80h_256]
3932+ vpshufb ymm_tmp4, ymm_tmp4, ymm_tmp3
3933+ vpshufb ymm_tmp2, ymm_tmp2, ymm_tmp3
3934+ vpmaddwd ymm_tmp4, ymm_tmp4, ymm_tmp0
3935+ vpmaddwd ymm_tmp2, ymm_tmp2, ymm_tmp0
3936+ AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp0, ymm_tmp4, ymm_tmp2, ymm_yfrac0, ymm_yfrac1, ymm_tmp3
3937+ vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3938+ vmovdqu xmm_tmp2, [p_src_row1 + r_tmp0]
3939+ mov r_tmp0, i_xpos
3940+ lea i_xpos, [i_xpos + 2 * i_scalex2]
3941+ shr r_tmp0, 16
3942+ vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
3943+ vinserti128 ymm_tmp2, ymm_tmp2, [p_src_row1 + r_tmp0], 1
3944+ vpunpckhbw ymm_tmp3, ymm_xpos_int, [db80h_256]
3945+ vpshufb ymm_tmp4, ymm_tmp4, ymm_tmp3
3946+ vpshufb ymm_tmp2, ymm_tmp2, ymm_tmp3
3947+ vpmaddwd ymm_tmp4, ymm_tmp4, ymm_tmp1
3948+ vpmaddwd ymm_tmp2, ymm_tmp2, ymm_tmp1
3949+ AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp1, ymm_tmp4, ymm_tmp2, ymm_yfrac0, ymm_yfrac1, ymm_tmp3
3950+ vpackssdw ymm_tmp0, ymm_tmp0, ymm_tmp1
3951+ vpavgw ymm_tmp0, ymm_tmp0, ymm_0
3952+ vpackuswb ymm_tmp0, ymm_tmp0, ymm_tmp0
3953+ vmovlps [p_dst], xmm_tmp0
3954+ vextracti128 [p_dst + 8], ymm_tmp0, 1
3955+ add p_dst, 16
3956+ AVX2_BilinearIncXposw ymm_xpos_int, ymm_xpos_frac, ymm_xpos_int_inc, ymm_xpos_frac_inc, ymm_7fff, ymm_tmp0, ymm_tmp1
3957+%endmacro
3958+
3959+%macro AVX2_BilinearAccurateDownsample8xOrLess_16px 0
3960+ vbroadcasti128 ymm_tmp0, [shufb_000044448888CCCC]
3961+ vpshufb ymm_tmp0, ymm_xpos_int, ymm_tmp0
3962+ vpsubb ymm_xpos_int, ymm_xpos_int, ymm_tmp0
3963+ mov r_tmp0, i_xpos
3964+ shr r_tmp0, 16
3965+ vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3966+ vmovdqu xmm_tmp5, [p_src_row1 + r_tmp0]
3967+ lea r_tmp0, [i_xpos + 4 * i_scalex2]
3968+ add i_xpos, i_scalex2
3969+ shr r_tmp0, 16
3970+ vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
3971+ vinserti128 ymm_tmp5, ymm_tmp5, [p_src_row1 + r_tmp0], 1
3972+ mov r_tmp0, i_xpos
3973+ shr r_tmp0, 16
3974+ vmovdqu xmm_tmp0, [p_src_row0 + r_tmp0]
3975+ vmovdqu xmm_tmp1, [p_src_row1 + r_tmp0]
3976+ lea r_tmp0, [i_xpos + 4 * i_scalex2]
3977+ add i_xpos, i_scalex2
3978+ shr r_tmp0, 16
3979+ vinserti128 ymm_tmp0, ymm_tmp0, [p_src_row0 + r_tmp0], 1
3980+ vinserti128 ymm_tmp1, ymm_tmp1, [p_src_row1 + r_tmp0], 1
3981+ vpunpcklbw ymm_tmp3, ymm_xpos_int, [db80h_256]
3982+ vpshufb ymm_tmp4, ymm_tmp4, ymm_tmp3
3983+ vpshufb ymm_tmp5, ymm_tmp5, ymm_tmp3
3984+ vpshufb ymm_tmp0, ymm_tmp0, ymm_tmp3
3985+ vpshufb ymm_tmp1, ymm_tmp1, ymm_tmp3
3986+ vpblendd ymm_tmp4, ymm_tmp4, ymm_tmp0, 11001100b
3987+ vpblendd ymm_tmp5, ymm_tmp5, ymm_tmp1, 11001100b
3988+ AVX2_UnpckXFrac ymm_tmp0, ymm_tmp1, ymm_xpos_frac, ymm_7fff
3989+ vpmaddwd ymm_tmp4, ymm_tmp4, ymm_tmp0
3990+ vpmaddwd ymm_tmp5, ymm_tmp5, ymm_tmp0
3991+ AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp0, ymm_tmp4, ymm_tmp5, ymm_yfrac0, ymm_yfrac1, ymm_tmp3
3992+ mov r_tmp0, i_xpos
3993+ shr r_tmp0, 16
3994+ vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3995+ vmovdqu xmm_tmp5, [p_src_row1 + r_tmp0]
3996+ lea r_tmp0, [i_xpos + 4 * i_scalex2]
3997+ add i_xpos, i_scalex2
3998+ shr r_tmp0, 16
3999+ vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
4000+ vinserti128 ymm_tmp5, ymm_tmp5, [p_src_row1 + r_tmp0], 1
4001+ mov r_tmp0, i_xpos
4002+ lea i_xpos, [i_xpos + 4 * i_scalex2]
4003+ shr r_tmp0, 16
4004+ vmovdqu xmm_tmp2, [p_src_row0 + r_tmp0]
4005+ vmovdqu xmm_tmp3, [p_src_row1 + r_tmp0]
4006+ mov r_tmp0, i_xpos
4007+ add i_xpos, i_scalex2
4008+ shr r_tmp0, 16
4009+ vinserti128 ymm_tmp2, ymm_tmp2, [p_src_row0 + r_tmp0], 1
4010+ vinserti128 ymm_tmp3, ymm_tmp3, [p_src_row1 + r_tmp0], 1
4011+ vpshufb ymm_tmp4, ymm_tmp4, ymm_xpos_int
4012+ vpshufb ymm_tmp5, ymm_tmp5, ymm_xpos_int
4013+ vpshufb ymm_tmp2, ymm_tmp2, ymm_xpos_int
4014+ vpshufb ymm_tmp3, ymm_tmp3, ymm_xpos_int
4015+ vpblendd ymm_tmp4, ymm_tmp4, ymm_tmp2, 10001000b
4016+ vpblendd ymm_tmp5, ymm_tmp5, ymm_tmp3, 10001000b
4017+ vpunpckhbw ymm_tmp4, ymm_tmp4, ymm_0
4018+ vpunpckhbw ymm_tmp5, ymm_tmp5, ymm_0
4019+ vpmaddwd ymm_tmp4, ymm_tmp4, ymm_tmp1
4020+ vpmaddwd ymm_tmp5, ymm_tmp5, ymm_tmp1
4021+ AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp1, ymm_tmp4, ymm_tmp5, ymm_yfrac0, ymm_yfrac1, ymm_tmp3
4022+ vpackssdw ymm_tmp0, ymm_tmp0, ymm_tmp1
4023+ vpavgw ymm_tmp0, ymm_tmp0, ymm_0
4024+ vpackuswb ymm_tmp0, ymm_tmp0, ymm_tmp0
4025+ vmovlps [p_dst], xmm_tmp0
4026+ vextracti128 [p_dst + 8], ymm_tmp0, 1
4027+ add p_dst, 16
4028+ AVX2_BilinearIncXposw ymm_xpos_int, ymm_xpos_frac, ymm_xpos_int_inc, ymm_xpos_frac_inc, ymm_7fff, ymm_tmp0, ymm_tmp1
4029+%endmacro
4030+
4031+%macro AVX2_GeneralBilinearAccurateDownsample_16px 0
4032+ mov r_tmp0, i_xpos
4033+ shr r_tmp0, 16
4034+ vpbroadcastd ymm_tmp4, [p_src_row0 + r_tmp0]
4035+ vpbroadcastd ymm_tmp5, [p_src_row1 + r_tmp0]
4036+ lea r_tmp0, [i_xpos + 1 * i_scalex]
4037+ shr r_tmp0, 16
4038+ vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0]
4039+ vpunpcklwd ymm_tmp4, ymm_tmp4, ymm_tmp0
4040+ vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0]
4041+ vpunpcklwd ymm_tmp5, ymm_tmp5, ymm_tmp0
4042+ lea r_tmp0, [i_xpos + 2 * i_scalex]
4043+ lea i_xpos, [i_xpos + 4 * i_scalex]
4044+ shr r_tmp0, 16
4045+ vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0]
4046+ vpblendd ymm_tmp4, ymm_tmp4, ymm_tmp0, 00100010b
4047+ vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0]
4048+ vpblendd ymm_tmp5, ymm_tmp5, ymm_tmp0, 00100010b
4049+ mov r_tmp0, i_xpos
4050+ sub r_tmp0, i_scalex
4051+ shr r_tmp0, 16
4052+ vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0 - 2]
4053+ vpblendw ymm_tmp4, ymm_tmp4, ymm_tmp0, 1000b
4054+ vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0 - 2]
4055+ vpblendw ymm_tmp5, ymm_tmp5, ymm_tmp0, 1000b
4056+ mov r_tmp0, i_xpos
4057+ shr r_tmp0, 16
4058+ vpbroadcastd ymm_tmp2, [p_src_row0 + r_tmp0]
4059+ vpbroadcastd ymm_tmp3, [p_src_row1 + r_tmp0]
4060+ lea r_tmp0, [i_xpos + 1 * i_scalex]
4061+ shr r_tmp0, 16
4062+ vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0]
4063+ vpunpcklwd ymm_tmp2, ymm_tmp2, ymm_tmp0
4064+ vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0]
4065+ vpunpcklwd ymm_tmp3, ymm_tmp3, ymm_tmp0
4066+ lea r_tmp0, [i_xpos + 2 * i_scalex]
4067+ lea i_xpos, [i_xpos + 4 * i_scalex]
4068+ shr r_tmp0, 16
4069+ vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0]
4070+ vpblendd ymm_tmp2, ymm_tmp2, ymm_tmp0, 00100010b
4071+ vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0]
4072+ vpblendd ymm_tmp3, ymm_tmp3, ymm_tmp0, 00100010b
4073+ mov r_tmp0, i_xpos
4074+ sub r_tmp0, i_scalex
4075+ shr r_tmp0, 16
4076+ vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0 - 2]
4077+ vpblendw ymm_tmp2, ymm_tmp2, ymm_tmp0, 1000b
4078+ vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0 - 2]
4079+ vpblendw ymm_tmp3, ymm_tmp3, ymm_tmp0, 1000b
4080+ mov r_tmp0, i_xpos
4081+ shr r_tmp0, 16
4082+ vmovd xmm_tmp0, [p_src_row0 + r_tmp0]
4083+ vmovd xmm_tmp1, [p_src_row1 + r_tmp0]
4084+ lea r_tmp0, [i_xpos + i_scalex]
4085+ shr r_tmp0, 16
4086+ vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 1
4087+ vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 1
4088+ lea r_tmp0, [i_xpos + 2 * i_scalex]
4089+ lea i_xpos, [i_xpos + 4 * i_scalex]
4090+ shr r_tmp0, 16
4091+ vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 2
4092+ vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 2
4093+ mov r_tmp0, i_xpos
4094+ sub r_tmp0, i_scalex
4095+ shr r_tmp0, 16
4096+ vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 3
4097+ vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 3
4098+ vpblendd ymm_tmp4, ymm_tmp4, ymm_tmp0, 00001111b
4099+ vpblendd ymm_tmp5, ymm_tmp5, ymm_tmp1, 00001111b
4100+ mov r_tmp0, i_xpos
4101+ shr r_tmp0, 16
4102+ vmovd xmm_tmp0, [p_src_row0 + r_tmp0]
4103+ vmovd xmm_tmp1, [p_src_row1 + r_tmp0]
4104+ lea r_tmp0, [i_xpos + i_scalex]
4105+ shr r_tmp0, 16
4106+ vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 1
4107+ vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 1
4108+ lea r_tmp0, [i_xpos + 2 * i_scalex]
4109+ lea i_xpos, [i_xpos + 4 * i_scalex]
4110+ shr r_tmp0, 16
4111+ vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 2
4112+ vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 2
4113+ mov r_tmp0, i_xpos
4114+ sub r_tmp0, i_scalex
4115+ shr r_tmp0, 16
4116+ vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 3
4117+ vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 3
4118+ vpblendd ymm_tmp2, ymm_tmp2, ymm_tmp0, 00001111b
4119+ vpblendd ymm_tmp3, ymm_tmp3, ymm_tmp1, 00001111b
4120+ vpunpcklbw ymm_tmp4, ymm_tmp4, ymm_0
4121+ vpunpcklbw ymm_tmp5, ymm_tmp5, ymm_0
4122+ vpmaddwd ymm_tmp4, ymm_tmp4, ymm_xfrac0
4123+ vpmaddwd ymm_tmp5, ymm_tmp5, ymm_xfrac0
4124+ AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp0, ymm_tmp4, ymm_tmp5, ymm_yfrac0, ymm_yfrac1, ymm_tmp1
4125+ vpunpcklbw ymm_tmp4, ymm_tmp2, ymm_0
4126+ vpunpcklbw ymm_tmp5, ymm_tmp3, ymm_0
4127+ vpmaddwd ymm_tmp4, ymm_tmp4, ymm_xfrac1
4128+ vpmaddwd ymm_tmp5, ymm_tmp5, ymm_xfrac1
4129+ AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp1, ymm_tmp4, ymm_tmp5, ymm_yfrac0, ymm_yfrac1, ymm_tmp2
4130+ vpackssdw ymm_tmp0, ymm_tmp0, ymm_tmp1
4131+ vpavgw ymm_tmp0, ymm_tmp0, ymm_0
4132+ vpackuswb ymm_tmp0, ymm_tmp0, ymm_tmp0
4133+ vextracti128 [p_dst], ymm_tmp0, 1
4134+ vmovlps [p_dst + 8], xmm_tmp0
4135+ add p_dst, 16
4136+ vpaddw ymm_xfrac0, ymm_xfrac0, ymm_xfrac_inc
4137+ vpaddw ymm_xfrac1, ymm_xfrac1, ymm_xfrac_inc
4138+ vpand ymm_xfrac0, ymm_xfrac0, ymm_7fff
4139+ vpand ymm_xfrac1, ymm_xfrac1, ymm_7fff
4140+%endmacro
4141+
38574142 ; downsample_16px_macro=%1 b_fast=%2
38584143 %macro AVX2_GeneralBilinearDownsampler_loop 2
38594144 %%height:
@@ -4254,3 +4539,326 @@ WELS_EXTERN GeneralBilinearFastDownsampler_avx2 · 394 unmodified lines
42544539 %undef ymm_xfrac0_begin
42554540 %undef ymm_xfrac1_begin
42564541 %undef ymm_xfrac_inc
4542+
4543+;**************************************************************************************************************
4544+;void GeneralBilinearAccurateDownsampler_avx2 (uint8_t* pDst, int32_t iDstStride, int32_t iDstWidth,
4545+; int32_t iDstHeight, uint8_t* pSrc, int32_t iSrcStride, uint32_t uiScaleX,
4546+; uint32_t uiScaleY);
4547+;
4548+;**************************************************************************************************************
4549+
4550+WELS_EXTERN GeneralBilinearAccurateDownsampler_avx2
4551+ %assign push_num 0
4552+%ifndef X86_32
4553+ push r12
4554+ push r13
4555+ push rbx
4556+ push rbp
4557+ %assign push_num 4
4558+%ifdef WIN64
4559+ push rdi
4560+ push rsi
4561+ %assign push_num push_num + 2
4562+%endif
4563+%endif
4564+ LOAD_7_PARA
4565+ PUSH_XMM 16
4566+ SIGN_EXTENSION r1, r1d
4567+ SIGN_EXTENSION r2, r2d
4568+ SIGN_EXTENSION r3, r3d
4569+ SIGN_EXTENSION r5, r5d
4570+ ZERO_EXTENSION r6d
4571+ sub r1, r2 ; dst_stride - dst_width
4572+ add r6, r6 ; 2 * scalex
4573+%ifdef X86_32
4574+ vmovd xmm0, arg8
4575+ vmovd xmm1, esp
4576+ and esp, -32
4577+ sub esp, 8 * 4 + 8 * 32
4578+ vmovd [esp], xmm1
4579+ %define p_dst r0
4580+ %define i_dst_stride_less_width [esp + 1 * 4]
4581+ %define i_dst_width [esp + 2 * 4]
4582+ %define i_dst_height dword [esp + 3 * 4]
4583+ %define p_src [esp + 4 * 4]
4584+ %define i_src_stride [esp + 5 * 4]
4585+ %define i_scalex r6
4586+ %define i_scalexd r6d
4587+ %define i_scaleyd [esp + 6 * 4]
4588+ %define i_xpos r2
4589+ %define i_ypos [esp + 7 * 4]
4590+ %define i_yposd dword [esp + 7 * 4]
4591+ %define p_src_row0 r3
4592+ %define p_src_row1 r4
4593+ %define i_width_cnt r5
4594+ %define r_tmp0 r1
4595+ %define r_tmp0b r1b
4596+ %define ymm_xpos_frac ymm1
4597+ %define ymm_xpos_frac_inc [esp + 8 * 4]
4598+ %define ymm_xpos_int ymm3
4599+ %define ymm_xpos_int_inc [esp + 8 * 4 + 1 * 32]
4600+ %define ymm_yfrac0 [esp + 8 * 4 + 2 * 32]
4601+ %define ymm_yfrac1 [esp + 8 * 4 + 3 * 32]
4602+ %define xmm_tmp0 xmm7
4603+ %define ymm_tmp0 ymm7
4604+ %define xmm_tmp1 xmm0
4605+ %define ymm_tmp1 ymm0
4606+ %define xmm_tmp2 xmm2
4607+ %define ymm_tmp2 ymm2
4608+ %define xmm_tmp3 xmm4
4609+ %define ymm_tmp3 ymm4
4610+ %define xmm_tmp4 xmm5
4611+ %define ymm_tmp4 ymm5
4612+ %define xmm_tmp5 xmm6
4613+ %define ymm_tmp5 ymm6
4614+ %define ymm_0 [esp + 8 * 4 + 4 * 32]
4615+ %define ymm_7fff [esp + 8 * 4 + 5 * 32]
4616+ %define ymm_xpos_int_begin [esp + 8 * 4 + 6 * 32]
4617+ %define ymm_xpos_frac_begin [esp + 8 * 4 + 7 * 32]
4618+ mov i_dst_stride_less_width, r1
4619+ mov i_dst_width, r2
4620+ mov i_dst_height, r3
4621+ mov p_src, r4
4622+ mov i_src_stride, r5
4623+ vmovd i_scaleyd, xmm0
4624+ vpxor xmm0, xmm0, xmm0
4625+ vmovdqa ymm_0, ymm0
4626+ vpcmpeqw ymm0, ymm0, ymm0
4627+ vpsrlw ymm0, ymm0, 1
4628+ vmovdqa ymm_7fff, ymm0
4629+%else
4630+ %define p_dst r0
4631+ %define i_dst_stride_less_width r1
4632+ %define i_dst_width r2
4633+ %define i_dst_height r3
4634+ %define p_src r4
4635+ %define i_src_stride r5
4636+ %define i_scalex r6
4637+ %define i_scalexd r6d
4638+ %define i_scaleyd dword arg8d
4639+ %define i_xpos r12
4640+ %define i_ypos r13
4641+ %define i_yposd r13d
4642+ %define p_src_row0 rbp
4643+%ifdef WIN64
4644+ %define p_src_row1 rsi
4645+ %define i_width_cnt rdi
4646+%else
4647+ %define p_src_row1 r11
4648+ %define i_width_cnt rax
4649+%endif
4650+ %define r_tmp0 rbx
4651+ %define r_tmp0b bl
4652+ %define ymm_0 ymm0
4653+ %define ymm_xpos_frac ymm1
4654+ %define ymm_xpos_int ymm3
4655+ %define ymm_xpos_frac_inc ymm2
4656+ %define ymm_xpos_int_inc ymm4
4657+ %define ymm_yfrac0 ymm5
4658+ %define ymm_yfrac1 ymm6
4659+ %define xmm_tmp0 xmm7
4660+ %define ymm_tmp0 ymm7
4661+ %define xmm_tmp1 xmm8
4662+ %define ymm_tmp1 ymm8
4663+ %define xmm_tmp2 xmm9
4664+ %define ymm_tmp2 ymm9
4665+ %define xmm_tmp3 xmm10
4666+ %define ymm_tmp3 ymm10
4667+ %define xmm_tmp4 xmm11
4668+ %define ymm_tmp4 ymm11
4669+ %define xmm_tmp5 xmm12
4670+ %define ymm_tmp5 ymm12
4671+ %define ymm_7fff ymm13
4672+ %define ymm_xpos_int_begin ymm14
4673+ %define ymm_xpos_frac_begin ymm15
4674+ vpxor ymm_0, ymm_0, ymm_0
4675+ vpcmpeqw ymm_7fff, ymm_7fff, ymm_7fff
4676+ vpsrlw ymm_7fff, ymm_7fff, 1
4677+%endif
4678+
4679+ sub i_dst_height, 1
4680+ je .final_row
4681+ jl .done
4682+
4683+ mov i_yposd, 1 << 14
4684+ vmovd xmm_tmp0, i_scalexd
4685+ vpbroadcastd ymm_tmp0, xmm_tmp0
4686+ vpslld ymm_tmp1, ymm_tmp0, 2
4687+ vpslld ymm_tmp2, ymm_tmp0, 3
4688+ vpaddd ymm_tmp3, ymm_tmp1, ymm_tmp2
4689+ vpxor ymm_tmp4, ymm_tmp4, ymm_tmp4
4690+ vpblendd ymm_tmp1, ymm_tmp4, ymm_tmp1, 11110000b
4691+ vpblendd ymm_tmp2, ymm_tmp2, ymm_tmp3, 11110000b
4692+ vpaddd ymm_tmp3, ymm_tmp0, ymm_tmp0
4693+ vpblendd ymm_tmp3, ymm_tmp4, ymm_tmp3, 11001100b
4694+ vpblendd ymm_tmp0, ymm_tmp4, ymm_tmp0, 10101010b
4695+ vpaddd ymm_tmp0, ymm_tmp3, ymm_tmp0
4696+ vpaddd ymm_tmp1, ymm_tmp1, ymm_tmp0
4697+ vpaddd ymm_tmp2, ymm_tmp2, ymm_tmp0
4698+ vpcmpeqw ymm_tmp3, ymm_tmp3, ymm_tmp3
4699+ vpsrld ymm_tmp3, ymm_tmp3, 31
4700+ vpslld ymm_tmp3, ymm_tmp3, 15
4701+ vpaddd ymm_tmp1, ymm_tmp1, ymm_tmp3
4702+ vpaddd ymm_tmp2, ymm_tmp2, ymm_tmp3
4703+ vpsrld ymm_xpos_int, ymm_tmp1, 16
4704+ vpsrld ymm_tmp0, ymm_tmp2, 16
4705+ vpackssdw ymm_xpos_int, ymm_xpos_int, ymm_tmp0
4706+ vpermq ymm_xpos_int, ymm_xpos_int, 11011000b
4707+ vpackuswb ymm_xpos_int, ymm_xpos_int, ymm_xpos_int
4708+ vpcmpeqw ymm_tmp3, ymm_tmp3, ymm_tmp3
4709+ vpsubb ymm_tmp0, ymm_xpos_int, ymm_tmp3
4710+ vpunpcklbw ymm_xpos_int, ymm_xpos_int, ymm_tmp0
4711+ vpslld ymm_tmp1, ymm_tmp1, 16
4712+ vpsrld ymm_tmp1, ymm_tmp1, 16
4713+ vpslld ymm_tmp2, ymm_tmp2, 16
4714+ vpsrld ymm_tmp2, ymm_tmp2, 16
4715+ vpackusdw ymm_xpos_frac, ymm_tmp1, ymm_tmp2
4716+ vpermq ymm_xpos_frac, ymm_xpos_frac, 11011000b
4717+ vpsrlw ymm_xpos_frac, ymm_xpos_frac, 1
4718+ vmovd xmm_tmp0, i_scalexd
4719+ vpslld xmm_tmp0, xmm_tmp0, 4
4720+ vpbroadcastw ymm_tmp1, xmm_tmp0
4721+ vpsrlw ymm_tmp1, ymm_tmp1, 1
4722+ vmovdqa ymm_xpos_frac_inc, ymm_tmp1
4723+ vpsrld xmm_tmp0, xmm_tmp0, 16
4724+ vpsubw ymm_tmp0, ymm_tmp0, ymm_tmp3
4725+ vpbroadcastb ymm_tmp0, xmm_tmp0
4726+ vmovdqa ymm_xpos_int_inc, ymm_tmp0
4727+ vmovdqa ymm_xpos_int_begin, ymm_xpos_int
4728+ vmovdqa ymm_xpos_frac_begin, ymm_xpos_frac
4729+
4730+ cmp i_scalex, 4 << 16
4731+ ja .scalex_above4
4732+ cmp i_scalex, 2 << 16
4733+ ja .scalex_above2_beloweq4
4734+ add i_scalex, i_scalex
4735+%xdefine i_scalex2 i_scalex
4736+%undef i_scalex
4737+ AVX2_GeneralBilinearDownsampler_loop AVX2_BilinearAccurateDownsample2xOrLess_16px, 0
4738+ shr i_scalex2, 1
4739+%xdefine i_scalex i_scalex2
4740+%undef i_scalex2
4741+ jmp .final_row
4742+.scalex_above2_beloweq4:
4743+ add i_scalex, i_scalex
4744+%xdefine i_scalex2 i_scalex
4745+%undef i_scalex
4746+ AVX2_GeneralBilinearDownsampler_loop AVX2_BilinearAccurateDownsample4xOrLess_16px, 0
4747+ shr i_scalex2, 1
4748+%xdefine i_scalex i_scalex2
4749+%undef i_scalex2
4750+ jmp .final_row
4751+.scalex_above4:
4752+ cmp i_scalex, 8 << 16
4753+ ja .scalex_above8
4754+ add i_scalex, i_scalex
4755+%xdefine i_scalex2 i_scalex
4756+%undef i_scalex
4757+ AVX2_GeneralBilinearDownsampler_loop AVX2_BilinearAccurateDownsample8xOrLess_16px, 0
4758+ shr i_scalex2, 1
4759+%xdefine i_scalex i_scalex2
4760+%undef i_scalex2
4761+ jmp .final_row
4762+.scalex_above8:
4763+%xdefine ymm_xfrac0 ymm_xpos_frac
4764+%xdefine ymm_xfrac1 ymm_xpos_int
4765+%xdefine ymm_xfrac0_begin ymm_xpos_int_begin
4766+%xdefine ymm_xfrac1_begin ymm_xpos_frac_begin
4767+%xdefine ymm_xfrac_inc ymm_xpos_frac_inc
4768+%undef ymm_xpos_int
4769+%undef ymm_xpos_frac
4770+%undef ymm_xpos_int_begin
4771+%undef ymm_xpos_frac_begin
4772+%undef ymm_xpos_int_inc
4773+%undef ymm_xpos_frac_inc
4774+ AVX2_UnpckXFrac ymm_tmp0, ymm_xfrac1, ymm_xfrac0, ymm_7fff
4775+ vpermq ymm_xfrac0, ymm_tmp0, 01001110b
4776+ vpermq ymm_xfrac1, ymm_xfrac1, 01001110b
4777+ vmovdqa ymm_xfrac0_begin, ymm_xfrac0
4778+ vmovdqa ymm_xfrac1_begin, ymm_xfrac1
4779+ vpcmpeqw ymm_tmp0, ymm_tmp0, ymm_tmp0
4780+ vpmullw ymm_tmp0, ymm_tmp0, ymm_xfrac_inc
4781+ vpunpcklwd ymm_tmp0, ymm_tmp0, ymm_xfrac_inc
4782+ vmovdqa ymm_xfrac_inc, ymm_tmp0
4783+ AVX2_GeneralBilinearDownsampler_loop AVX2_GeneralBilinearAccurateDownsample_16px, 0
4784+
4785+.final_row:
4786+ mov p_src_row0, i_ypos
4787+ shr p_src_row0, 15
4788+ imul p_src_row0, i_src_stride
4789+ add p_src_row0, p_src
4790+ mov i_xpos, 1 << 15
4791+ mov i_width_cnt, i_dst_width
4792+
4793+.final_row_width:
4794+ mov r_tmp0, i_xpos
4795+ shr r_tmp0, 16
4796+ movzx r_tmp0, byte [p_src_row0 + r_tmp0]
4797+ mov [p_dst], r_tmp0b
4798+ add p_dst, 1
4799+ add i_xpos, i_scalex
4800+ sub i_width_cnt, 1
4801+ jg .final_row_width
4802+
4803+.done:
4804+ vzeroupper
4805+%ifdef X86_32
4806+ mov esp, [esp]
4807+%endif
4808+ POP_XMM
4809+ LOAD_7_PARA_POP
4810+%ifndef X86_32
4811+%ifdef WIN64
4812+ pop rsi
4813+ pop rdi
4814+%endif
4815+ pop rbp
4816+ pop rbx
4817+ pop r13
4818+ pop r12
4819+%endif
4820+ ret
4821+%undef p_dst
4822+%undef i_dst_stride_less_width
4823+%undef i_dst_width
4824+%undef i_dst_height
4825+%undef p_src
4826+%undef i_src_stride
4827+%undef i_scalex
4828+%undef i_scalexd
4829+%undef i_scaleyd
4830+%undef i_xpos
4831+%undef i_ypos
4832+%undef i_yposd
4833+%undef p_src_row0
4834+%undef p_src_row1
4835+%undef i_width_cnt
4836+%undef r_tmp0
4837+%undef r_tmp0b
4838+%undef ymm_xpos_frac
4839+%undef ymm_xpos_frac_inc
4840+%undef ymm_xpos_int
4841+%undef ymm_xpos_int_inc
4842+%undef ymm_yfrac0
4843+%undef ymm_yfrac1
4844+%undef xmm_tmp0
4845+%undef ymm_tmp0
4846+%undef xmm_tmp1
4847+%undef ymm_tmp1
4848+%undef xmm_tmp2
4849+%undef ymm_tmp2
4850+%undef xmm_tmp3
4851+%undef ymm_tmp3
4852+%undef xmm_tmp4
4853+%undef ymm_tmp4
4854+%undef xmm_tmp5
4855+%undef ymm_tmp5
4856+%undef ymm_0
4857+%undef ymm_7fff
4858+%undef ymm_xpos_int_begin
4859+%undef ymm_xpos_frac_begin
4860+%undef ymm_xfrac0
4861+%undef ymm_xfrac1
4862+%undef ymm_xfrac0_begin
4863+%undef ymm_xfrac1_begin
4864+%undef ymm_xfrac_inc

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t