History

[Processing/x86] Add an AVX2 implementation of GeneralBilinearAccurateDownsample

Keep track of relative pixel offsets and utilize pshufb to efficiently extract relevant pixels for horizontal scaling ratios <= 8. Because pshufb does not cross 128-bit lanes, the overhead of address calculations and loads is relatively greater as compared with an SSSE3/SSE4.1 implementation. Fall back to a generic approach for ratios > 8. The implementation assumes that data beyond the end of each line, before the next line begins, can be dirtied; which AFAICT is safe with the current usage of these routines. Speedup is ~8.52x/~6.89x (32-bit/64-bit) for horizontal ratios <= 2, ~7.81x/~6.13x for ratios within (2, 4], ~5.81x/~4.52x for ratios within (4, 8], and ~5.06x/~4.09x for ratios > 8 when not memory-bound on Haswell as compared with the current SSE2 implementation.

Sindre Aamås committed 10 years ago

Commit e4902151 parent b43e58a5 files · +616 −0Browse files

Showing the changes to codec/processing/src (4 of 5 files). Show all

Unified · Split

4 files
  1. codec/processing/src/downsample/downsample.cpp +1 −0
  2. codec/processing/src/downsample/downsample.h +4 −0
  3. codec/processing/src/downsample/downsamplefuncs.cpp +1 −0
  4. codec/processing/src/x86/downsample_bilinear.asm +608 −0
codec/processing/src/downsample/downsample.cpp+1−0
@@ -110,6 +110,7 @@ void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int · 109 unmodified lines
110 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_sse41;110 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_sse41;
111 }111 }
112 if (iCpuFlag & WELS_CPU_AVX2) {112 if (iCpuFlag & WELS_CPU_AVX2) {
113 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_avx2;
113 sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearFastDownsamplerWrap_avx2;114 sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearFastDownsamplerWrap_avx2;
114 }115 }
115#endif//X86_ASM116#endif//X86_ASM
codec/processing/src/downsample/downsample.h+4−0
@@ -104,6 +104,7 @@ GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse2; · 103 unmodified lines
104GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_ssse3;104GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_ssse3;
105GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse41;105GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse41;
106GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_avx2;106GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_avx2;
107GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_avx2;
107 108
108SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_ssse3;109SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_ssse3;
109SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_sse4;110SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_sse4;
@@ -126,6 +127,9 @@ void GeneralBilinearAccurateDownsampler_sse41 (uint8_t* pDst, int32_t iDstStride · 16 unmodified lines
126void GeneralBilinearFastDownsampler_avx2 (uint8_t* pDst, int32_t iDstStride, int32_t iDstWidth,127void GeneralBilinearFastDownsampler_avx2 (uint8_t* pDst, int32_t iDstStride, int32_t iDstWidth,
127 int32_t iDstHeight, uint8_t* pSrc, int32_t iSrcStride, uint32_t uiScaleX,128 int32_t iDstHeight, uint8_t* pSrc, int32_t iSrcStride, uint32_t uiScaleX,
128 uint32_t uiScaleY);129 uint32_t uiScaleY);
130void GeneralBilinearAccurateDownsampler_avx2 (uint8_t* pDst, int32_t iDstStride, int32_t iDstWidth,
131 int32_t iDstHeight, uint8_t* pSrc, int32_t iSrcStride, uint32_t uiScaleX,
132 uint32_t uiScaleY);
129 133
130WELSVP_EXTERN_C_END134WELSVP_EXTERN_C_END
131#endif135#endif
codec/processing/src/downsample/downsamplefuncs.cpp+1−0
@@ -285,6 +285,7 @@ DEFINE_GENERAL_BILINEAR_ACCURATE_DOWNSAMPLER_WRAP (sse2) · 284 unmodified lines
285DEFINE_GENERAL_BILINEAR_FAST_DOWNSAMPLER_WRAP (ssse3)285DEFINE_GENERAL_BILINEAR_FAST_DOWNSAMPLER_WRAP (ssse3)
286DEFINE_GENERAL_BILINEAR_ACCURATE_DOWNSAMPLER_WRAP (sse41)286DEFINE_GENERAL_BILINEAR_ACCURATE_DOWNSAMPLER_WRAP (sse41)
287DEFINE_GENERAL_BILINEAR_FAST_DOWNSAMPLER_WRAP (avx2)287DEFINE_GENERAL_BILINEAR_FAST_DOWNSAMPLER_WRAP (avx2)
288DEFINE_GENERAL_BILINEAR_ACCURATE_DOWNSAMPLER_WRAP (avx2)
288#endif //X86_ASM289#endif //X86_ASM
289 290
290#ifdef HAVE_NEON291#ifdef HAVE_NEON
codec/processing/src/x86/downsample_bilinear.asm+608−0
@@ -3854,6 +3854,291 @@ WELS_EXTERN GeneralBilinearAccurateDownsampler_sse41 · 3853 unmodified lines
3854 vpaddw ymm_xfrac1, ymm_xfrac1, ymm_xfrac_inc3854 vpaddw ymm_xfrac1, ymm_xfrac1, ymm_xfrac_inc
3855%endmacro3855%endmacro
3856 3856
3857; xpos_int=%1 xpos_frac=%2 inc_int=%3 inc_frac=%4 7FFFh=%5 tmp=%6,%7
3858%macro AVX2_BilinearIncXposw 7
3859 vpaddb %1, %1, %3
3860 vpaddw %6, %2, %4
3861 vpcmpgtw %7, %2, %6
3862 vpsubb %1, %1, %7 ; add carry
3863 vpand %2, %6, %5
3864%endmacro
3865
3866; res>>29=%1 data0=%2 data1=%3 frac0=%4 frac1=%5 tmp=%6
3867%macro AVX2_LinearAccurateInterpolateVerticalDwords 6
3868 vpshufd %1, %2, 10110001b
3869 vpshufd %6, %3, 10110001b
3870 vpmuludq %1, %1, %4
3871 vpmuludq %6, %6, %5
3872 vpaddq %1, %1, %6
3873 vpmuludq %2, %2, %4
3874 vpmuludq %3, %3, %5
3875 vpaddq %2, %2, %3
3876 vpsllq %1, %1, 3
3877 vpsrlq %2, %2, 29
3878 vpblendd %1, %1, %2, 01010101b
3879%endmacro
3880
3881%macro AVX2_BilinearAccurateDownsample2xOrLess_16px 0
3882 vpshufb ymm_tmp0, ymm_xpos_int, ymm_0
3883 vpsubb ymm_xpos_int, ymm_xpos_int, ymm_tmp0
3884 AVX2_UnpckXFrac ymm_tmp0, ymm_tmp1, ymm_xpos_frac, ymm_7fff
3885 mov r_tmp0, i_xpos
3886 shr r_tmp0, 16
3887 vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3888 vmovdqu xmm_tmp5, [p_src_row1 + r_tmp0]
3889 lea r_tmp0, [i_xpos + 4 * i_scalex2]
3890 lea i_xpos, [i_xpos + 8 * i_scalex2]
3891 shr r_tmp0, 16
3892 vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
3893 vinserti128 ymm_tmp5, ymm_tmp5, [p_src_row1 + r_tmp0], 1
3894 vpshufb ymm_tmp4, ymm_tmp4, ymm_xpos_int
3895 vpshufb ymm_tmp5, ymm_tmp5, ymm_xpos_int
3896 vpunpcklbw ymm_tmp2, ymm_tmp4, ymm_0
3897 vpunpcklbw ymm_tmp3, ymm_tmp5, ymm_0
3898 vpunpckhbw ymm_tmp4, ymm_tmp4, ymm_0
3899 vpunpckhbw ymm_tmp5, ymm_tmp5, ymm_0
3900 vpmaddwd ymm_tmp2, ymm_tmp2, ymm_tmp0
3901 vpmaddwd ymm_tmp3, ymm_tmp3, ymm_tmp0
3902 vpmaddwd ymm_tmp4, ymm_tmp4, ymm_tmp1
3903 vpmaddwd ymm_tmp5, ymm_tmp5, ymm_tmp1
3904 AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp0, ymm_tmp2, ymm_tmp3, ymm_yfrac0, ymm_yfrac1, ymm_tmp1
3905 AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp1, ymm_tmp4, ymm_tmp5, ymm_yfrac0, ymm_yfrac1, ymm_tmp2
3906 vpackssdw ymm_tmp0, ymm_tmp0, ymm_tmp1
3907 vpavgw ymm_tmp0, ymm_tmp0, ymm_0
3908 vpackuswb ymm_tmp0, ymm_tmp0, ymm_tmp0
3909 vmovlps [p_dst], xmm_tmp0
3910 vextracti128 [p_dst + 8], ymm_tmp0, 1
3911 add p_dst, 16
3912 AVX2_BilinearIncXposw ymm_xpos_int, ymm_xpos_frac, ymm_xpos_int_inc, ymm_xpos_frac_inc, ymm_7fff, ymm_tmp0, ymm_tmp1
3913%endmacro
3914
3915%macro AVX2_BilinearAccurateDownsample4xOrLess_16px 0
3916 vbroadcasti128 ymm_tmp0, [shufb_0000000088888888]
3917 vpshufb ymm_tmp0, ymm_xpos_int, ymm_tmp0
3918 vpsubb ymm_xpos_int, ymm_xpos_int, ymm_tmp0
3919 AVX2_UnpckXFrac ymm_tmp0, ymm_tmp1, ymm_xpos_frac, ymm_7fff
3920 mov r_tmp0, i_xpos
3921 shr r_tmp0, 16
3922 vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3923 vmovdqu xmm_tmp2, [p_src_row1 + r_tmp0]
3924 lea r_tmp0, [i_xpos + 4 * i_scalex2]
3925 shr r_tmp0, 16
3926 vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
3927 vinserti128 ymm_tmp2, ymm_tmp2, [p_src_row1 + r_tmp0], 1
3928 lea r_tmp0, [i_xpos + 2 * i_scalex2]
3929 lea i_xpos, [r_tmp0 + 4 * i_scalex2]
3930 shr r_tmp0, 16
3931 vpunpcklbw ymm_tmp3, ymm_xpos_int, [db80h_256]
3932 vpshufb ymm_tmp4, ymm_tmp4, ymm_tmp3
3933 vpshufb ymm_tmp2, ymm_tmp2, ymm_tmp3
3934 vpmaddwd ymm_tmp4, ymm_tmp4, ymm_tmp0
3935 vpmaddwd ymm_tmp2, ymm_tmp2, ymm_tmp0
3936 AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp0, ymm_tmp4, ymm_tmp2, ymm_yfrac0, ymm_yfrac1, ymm_tmp3
3937 vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3938 vmovdqu xmm_tmp2, [p_src_row1 + r_tmp0]
3939 mov r_tmp0, i_xpos
3940 lea i_xpos, [i_xpos + 2 * i_scalex2]
3941 shr r_tmp0, 16
3942 vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
3943 vinserti128 ymm_tmp2, ymm_tmp2, [p_src_row1 + r_tmp0], 1
3944 vpunpckhbw ymm_tmp3, ymm_xpos_int, [db80h_256]
3945 vpshufb ymm_tmp4, ymm_tmp4, ymm_tmp3
3946 vpshufb ymm_tmp2, ymm_tmp2, ymm_tmp3
3947 vpmaddwd ymm_tmp4, ymm_tmp4, ymm_tmp1
3948 vpmaddwd ymm_tmp2, ymm_tmp2, ymm_tmp1
3949 AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp1, ymm_tmp4, ymm_tmp2, ymm_yfrac0, ymm_yfrac1, ymm_tmp3
3950 vpackssdw ymm_tmp0, ymm_tmp0, ymm_tmp1
3951 vpavgw ymm_tmp0, ymm_tmp0, ymm_0
3952 vpackuswb ymm_tmp0, ymm_tmp0, ymm_tmp0
3953 vmovlps [p_dst], xmm_tmp0
3954 vextracti128 [p_dst + 8], ymm_tmp0, 1
3955 add p_dst, 16
3956 AVX2_BilinearIncXposw ymm_xpos_int, ymm_xpos_frac, ymm_xpos_int_inc, ymm_xpos_frac_inc, ymm_7fff, ymm_tmp0, ymm_tmp1
3957%endmacro
3958
3959%macro AVX2_BilinearAccurateDownsample8xOrLess_16px 0
3960 vbroadcasti128 ymm_tmp0, [shufb_000044448888CCCC]
3961 vpshufb ymm_tmp0, ymm_xpos_int, ymm_tmp0
3962 vpsubb ymm_xpos_int, ymm_xpos_int, ymm_tmp0
3963 mov r_tmp0, i_xpos
3964 shr r_tmp0, 16
3965 vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3966 vmovdqu xmm_tmp5, [p_src_row1 + r_tmp0]
3967 lea r_tmp0, [i_xpos + 4 * i_scalex2]
3968 add i_xpos, i_scalex2
3969 shr r_tmp0, 16
3970 vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
3971 vinserti128 ymm_tmp5, ymm_tmp5, [p_src_row1 + r_tmp0], 1
3972 mov r_tmp0, i_xpos
3973 shr r_tmp0, 16
3974 vmovdqu xmm_tmp0, [p_src_row0 + r_tmp0]
3975 vmovdqu xmm_tmp1, [p_src_row1 + r_tmp0]
3976 lea r_tmp0, [i_xpos + 4 * i_scalex2]
3977 add i_xpos, i_scalex2
3978 shr r_tmp0, 16
3979 vinserti128 ymm_tmp0, ymm_tmp0, [p_src_row0 + r_tmp0], 1
3980 vinserti128 ymm_tmp1, ymm_tmp1, [p_src_row1 + r_tmp0], 1
3981 vpunpcklbw ymm_tmp3, ymm_xpos_int, [db80h_256]
3982 vpshufb ymm_tmp4, ymm_tmp4, ymm_tmp3
3983 vpshufb ymm_tmp5, ymm_tmp5, ymm_tmp3
3984 vpshufb ymm_tmp0, ymm_tmp0, ymm_tmp3
3985 vpshufb ymm_tmp1, ymm_tmp1, ymm_tmp3
3986 vpblendd ymm_tmp4, ymm_tmp4, ymm_tmp0, 11001100b
3987 vpblendd ymm_tmp5, ymm_tmp5, ymm_tmp1, 11001100b
3988 AVX2_UnpckXFrac ymm_tmp0, ymm_tmp1, ymm_xpos_frac, ymm_7fff
3989 vpmaddwd ymm_tmp4, ymm_tmp4, ymm_tmp0
3990 vpmaddwd ymm_tmp5, ymm_tmp5, ymm_tmp0
3991 AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp0, ymm_tmp4, ymm_tmp5, ymm_yfrac0, ymm_yfrac1, ymm_tmp3
3992 mov r_tmp0, i_xpos
3993 shr r_tmp0, 16
3994 vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3995 vmovdqu xmm_tmp5, [p_src_row1 + r_tmp0]
3996 lea r_tmp0, [i_xpos + 4 * i_scalex2]
3997 add i_xpos, i_scalex2
3998 shr r_tmp0, 16
3999 vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
4000 vinserti128 ymm_tmp5, ymm_tmp5, [p_src_row1 + r_tmp0], 1
4001 mov r_tmp0, i_xpos
4002 lea i_xpos, [i_xpos + 4 * i_scalex2]
4003 shr r_tmp0, 16
4004 vmovdqu xmm_tmp2, [p_src_row0 + r_tmp0]
4005 vmovdqu xmm_tmp3, [p_src_row1 + r_tmp0]
4006 mov r_tmp0, i_xpos
4007 add i_xpos, i_scalex2
4008 shr r_tmp0, 16
4009 vinserti128 ymm_tmp2, ymm_tmp2, [p_src_row0 + r_tmp0], 1
4010 vinserti128 ymm_tmp3, ymm_tmp3, [p_src_row1 + r_tmp0], 1
4011 vpshufb ymm_tmp4, ymm_tmp4, ymm_xpos_int
4012 vpshufb ymm_tmp5, ymm_tmp5, ymm_xpos_int
4013 vpshufb ymm_tmp2, ymm_tmp2, ymm_xpos_int
4014 vpshufb ymm_tmp3, ymm_tmp3, ymm_xpos_int
4015 vpblendd ymm_tmp4, ymm_tmp4, ymm_tmp2, 10001000b
4016 vpblendd ymm_tmp5, ymm_tmp5, ymm_tmp3, 10001000b
4017 vpunpckhbw ymm_tmp4, ymm_tmp4, ymm_0
4018 vpunpckhbw ymm_tmp5, ymm_tmp5, ymm_0
4019 vpmaddwd ymm_tmp4, ymm_tmp4, ymm_tmp1
4020 vpmaddwd ymm_tmp5, ymm_tmp5, ymm_tmp1
4021 AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp1, ymm_tmp4, ymm_tmp5, ymm_yfrac0, ymm_yfrac1, ymm_tmp3
4022 vpackssdw ymm_tmp0, ymm_tmp0, ymm_tmp1
4023 vpavgw ymm_tmp0, ymm_tmp0, ymm_0
4024 vpackuswb ymm_tmp0, ymm_tmp0, ymm_tmp0
4025 vmovlps [p_dst], xmm_tmp0
4026 vextracti128 [p_dst + 8], ymm_tmp0, 1
4027 add p_dst, 16
4028 AVX2_BilinearIncXposw ymm_xpos_int, ymm_xpos_frac, ymm_xpos_int_inc, ymm_xpos_frac_inc, ymm_7fff, ymm_tmp0, ymm_tmp1
4029%endmacro
4030
4031%macro AVX2_GeneralBilinearAccurateDownsample_16px 0
4032 mov r_tmp0, i_xpos
4033 shr r_tmp0, 16
4034 vpbroadcastd ymm_tmp4, [p_src_row0 + r_tmp0]
4035 vpbroadcastd ymm_tmp5, [p_src_row1 + r_tmp0]
4036 lea r_tmp0, [i_xpos + 1 * i_scalex]
4037 shr r_tmp0, 16
4038 vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0]
4039 vpunpcklwd ymm_tmp4, ymm_tmp4, ymm_tmp0
4040 vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0]
4041 vpunpcklwd ymm_tmp5, ymm_tmp5, ymm_tmp0
4042 lea r_tmp0, [i_xpos + 2 * i_scalex]
4043 lea i_xpos, [i_xpos + 4 * i_scalex]
4044 shr r_tmp0, 16
4045 vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0]
4046 vpblendd ymm_tmp4, ymm_tmp4, ymm_tmp0, 00100010b
4047 vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0]
4048 vpblendd ymm_tmp5, ymm_tmp5, ymm_tmp0, 00100010b
4049 mov r_tmp0, i_xpos
4050 sub r_tmp0, i_scalex
4051 shr r_tmp0, 16
4052 vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0 - 2]
4053 vpblendw ymm_tmp4, ymm_tmp4, ymm_tmp0, 1000b
4054 vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0 - 2]
4055 vpblendw ymm_tmp5, ymm_tmp5, ymm_tmp0, 1000b
4056 mov r_tmp0, i_xpos
4057 shr r_tmp0, 16
4058 vpbroadcastd ymm_tmp2, [p_src_row0 + r_tmp0]
4059 vpbroadcastd ymm_tmp3, [p_src_row1 + r_tmp0]
4060 lea r_tmp0, [i_xpos + 1 * i_scalex]
4061 shr r_tmp0, 16
4062 vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0]
4063 vpunpcklwd ymm_tmp2, ymm_tmp2, ymm_tmp0
4064 vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0]
4065 vpunpcklwd ymm_tmp3, ymm_tmp3, ymm_tmp0
4066 lea r_tmp0, [i_xpos + 2 * i_scalex]
4067 lea i_xpos, [i_xpos + 4 * i_scalex]
4068 shr r_tmp0, 16
4069 vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0]
4070 vpblendd ymm_tmp2, ymm_tmp2, ymm_tmp0, 00100010b
4071 vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0]
4072 vpblendd ymm_tmp3, ymm_tmp3, ymm_tmp0, 00100010b
4073 mov r_tmp0, i_xpos
4074 sub r_tmp0, i_scalex
4075 shr r_tmp0, 16
4076 vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0 - 2]
4077 vpblendw ymm_tmp2, ymm_tmp2, ymm_tmp0, 1000b
4078 vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0 - 2]
4079 vpblendw ymm_tmp3, ymm_tmp3, ymm_tmp0, 1000b
4080 mov r_tmp0, i_xpos
4081 shr r_tmp0, 16
4082 vmovd xmm_tmp0, [p_src_row0 + r_tmp0]
4083 vmovd xmm_tmp1, [p_src_row1 + r_tmp0]
4084 lea r_tmp0, [i_xpos + i_scalex]
4085 shr r_tmp0, 16
4086 vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 1
4087 vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 1
4088 lea r_tmp0, [i_xpos + 2 * i_scalex]
4089 lea i_xpos, [i_xpos + 4 * i_scalex]
4090 shr r_tmp0, 16
4091 vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 2
4092 vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 2
4093 mov r_tmp0, i_xpos
4094 sub r_tmp0, i_scalex
4095 shr r_tmp0, 16
4096 vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 3
4097 vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 3
4098 vpblendd ymm_tmp4, ymm_tmp4, ymm_tmp0, 00001111b
4099 vpblendd ymm_tmp5, ymm_tmp5, ymm_tmp1, 00001111b
4100 mov r_tmp0, i_xpos
4101 shr r_tmp0, 16
4102 vmovd xmm_tmp0, [p_src_row0 + r_tmp0]
4103 vmovd xmm_tmp1, [p_src_row1 + r_tmp0]
4104 lea r_tmp0, [i_xpos + i_scalex]
4105 shr r_tmp0, 16
4106 vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 1
4107 vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 1
4108 lea r_tmp0, [i_xpos + 2 * i_scalex]
4109 lea i_xpos, [i_xpos + 4 * i_scalex]
4110 shr r_tmp0, 16
4111 vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 2
4112 vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 2
4113 mov r_tmp0, i_xpos
4114 sub r_tmp0, i_scalex
4115 shr r_tmp0, 16
4116 vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 3
4117 vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 3
4118 vpblendd ymm_tmp2, ymm_tmp2, ymm_tmp0, 00001111b
4119 vpblendd ymm_tmp3, ymm_tmp3, ymm_tmp1, 00001111b
4120 vpunpcklbw ymm_tmp4, ymm_tmp4, ymm_0
4121 vpunpcklbw ymm_tmp5, ymm_tmp5, ymm_0
4122 vpmaddwd ymm_tmp4, ymm_tmp4, ymm_xfrac0
4123 vpmaddwd ymm_tmp5, ymm_tmp5, ymm_xfrac0
4124 AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp0, ymm_tmp4, ymm_tmp5, ymm_yfrac0, ymm_yfrac1, ymm_tmp1
4125 vpunpcklbw ymm_tmp4, ymm_tmp2, ymm_0
4126 vpunpcklbw ymm_tmp5, ymm_tmp3, ymm_0
4127 vpmaddwd ymm_tmp4, ymm_tmp4, ymm_xfrac1
4128 vpmaddwd ymm_tmp5, ymm_tmp5, ymm_xfrac1
4129 AVX2_LinearAccurateInterpolateVerticalDwords ymm_tmp1, ymm_tmp4, ymm_tmp5, ymm_yfrac0, ymm_yfrac1, ymm_tmp2
4130 vpackssdw ymm_tmp0, ymm_tmp0, ymm_tmp1
4131 vpavgw ymm_tmp0, ymm_tmp0, ymm_0
4132 vpackuswb ymm_tmp0, ymm_tmp0, ymm_tmp0
4133 vextracti128 [p_dst], ymm_tmp0, 1
4134 vmovlps [p_dst + 8], xmm_tmp0
4135 add p_dst, 16
4136 vpaddw ymm_xfrac0, ymm_xfrac0, ymm_xfrac_inc
4137 vpaddw ymm_xfrac1, ymm_xfrac1, ymm_xfrac_inc
4138 vpand ymm_xfrac0, ymm_xfrac0, ymm_7fff
4139 vpand ymm_xfrac1, ymm_xfrac1, ymm_7fff
4140%endmacro
4141
3857; downsample_16px_macro=%1 b_fast=%24142; downsample_16px_macro=%1 b_fast=%2
3858%macro AVX2_GeneralBilinearDownsampler_loop 24143%macro AVX2_GeneralBilinearDownsampler_loop 2
3859%%height:4144%%height:
@@ -4254,3 +4539,326 @@ WELS_EXTERN GeneralBilinearFastDownsampler_avx2 · 394 unmodified lines
4254%undef ymm_xfrac0_begin4539%undef ymm_xfrac0_begin
4255%undef ymm_xfrac1_begin4540%undef ymm_xfrac1_begin
4256%undef ymm_xfrac_inc4541%undef ymm_xfrac_inc
4542
4543;**************************************************************************************************************
4544;void GeneralBilinearAccurateDownsampler_avx2 (uint8_t* pDst, int32_t iDstStride, int32_t iDstWidth,
4545; int32_t iDstHeight, uint8_t* pSrc, int32_t iSrcStride, uint32_t uiScaleX,
4546; uint32_t uiScaleY);
4547;
4548;**************************************************************************************************************
4549
4550WELS_EXTERN GeneralBilinearAccurateDownsampler_avx2
4551 %assign push_num 0
4552%ifndef X86_32
4553 push r12
4554 push r13
4555 push rbx
4556 push rbp
4557 %assign push_num 4
4558%ifdef WIN64
4559 push rdi
4560 push rsi
4561 %assign push_num push_num + 2
4562%endif
4563%endif
4564 LOAD_7_PARA
4565 PUSH_XMM 16
4566 SIGN_EXTENSION r1, r1d
4567 SIGN_EXTENSION r2, r2d
4568 SIGN_EXTENSION r3, r3d
4569 SIGN_EXTENSION r5, r5d
4570 ZERO_EXTENSION r6d
4571 sub r1, r2 ; dst_stride - dst_width
4572 add r6, r6 ; 2 * scalex
4573%ifdef X86_32
4574 vmovd xmm0, arg8
4575 vmovd xmm1, esp
4576 and esp, -32
4577 sub esp, 8 * 4 + 8 * 32
4578 vmovd [esp], xmm1
4579 %define p_dst r0
4580 %define i_dst_stride_less_width [esp + 1 * 4]
4581 %define i_dst_width [esp + 2 * 4]
4582 %define i_dst_height dword [esp + 3 * 4]
4583 %define p_src [esp + 4 * 4]
4584 %define i_src_stride [esp + 5 * 4]
4585 %define i_scalex r6
4586 %define i_scalexd r6d
4587 %define i_scaleyd [esp + 6 * 4]
4588 %define i_xpos r2
4589 %define i_ypos [esp + 7 * 4]
4590 %define i_yposd dword [esp + 7 * 4]
4591 %define p_src_row0 r3
4592 %define p_src_row1 r4
4593 %define i_width_cnt r5
4594 %define r_tmp0 r1
4595 %define r_tmp0b r1b
4596 %define ymm_xpos_frac ymm1
4597 %define ymm_xpos_frac_inc [esp + 8 * 4]
4598 %define ymm_xpos_int ymm3
4599 %define ymm_xpos_int_inc [esp + 8 * 4 + 1 * 32]
4600 %define ymm_yfrac0 [esp + 8 * 4 + 2 * 32]
4601 %define ymm_yfrac1 [esp + 8 * 4 + 3 * 32]
4602 %define xmm_tmp0 xmm7
4603 %define ymm_tmp0 ymm7
4604 %define xmm_tmp1 xmm0
4605 %define ymm_tmp1 ymm0
4606 %define xmm_tmp2 xmm2
4607 %define ymm_tmp2 ymm2
4608 %define xmm_tmp3 xmm4
4609 %define ymm_tmp3 ymm4
4610 %define xmm_tmp4 xmm5
4611 %define ymm_tmp4 ymm5
4612 %define xmm_tmp5 xmm6
4613 %define ymm_tmp5 ymm6
4614 %define ymm_0 [esp + 8 * 4 + 4 * 32]
4615 %define ymm_7fff [esp + 8 * 4 + 5 * 32]
4616 %define ymm_xpos_int_begin [esp + 8 * 4 + 6 * 32]
4617 %define ymm_xpos_frac_begin [esp + 8 * 4 + 7 * 32]
4618 mov i_dst_stride_less_width, r1
4619 mov i_dst_width, r2
4620 mov i_dst_height, r3
4621 mov p_src, r4
4622 mov i_src_stride, r5
4623 vmovd i_scaleyd, xmm0
4624 vpxor xmm0, xmm0, xmm0
4625 vmovdqa ymm_0, ymm0
4626 vpcmpeqw ymm0, ymm0, ymm0
4627 vpsrlw ymm0, ymm0, 1
4628 vmovdqa ymm_7fff, ymm0
4629%else
4630 %define p_dst r0
4631 %define i_dst_stride_less_width r1
4632 %define i_dst_width r2
4633 %define i_dst_height r3
4634 %define p_src r4
4635 %define i_src_stride r5
4636 %define i_scalex r6
4637 %define i_scalexd r6d
4638 %define i_scaleyd dword arg8d
4639 %define i_xpos r12
4640 %define i_ypos r13
4641 %define i_yposd r13d
4642 %define p_src_row0 rbp
4643%ifdef WIN64
4644 %define p_src_row1 rsi
4645 %define i_width_cnt rdi
4646%else
4647 %define p_src_row1 r11
4648 %define i_width_cnt rax
4649%endif
4650 %define r_tmp0 rbx
4651 %define r_tmp0b bl
4652 %define ymm_0 ymm0
4653 %define ymm_xpos_frac ymm1
4654 %define ymm_xpos_int ymm3
4655 %define ymm_xpos_frac_inc ymm2
4656 %define ymm_xpos_int_inc ymm4
4657 %define ymm_yfrac0 ymm5
4658 %define ymm_yfrac1 ymm6
4659 %define xmm_tmp0 xmm7
4660 %define ymm_tmp0 ymm7
4661 %define xmm_tmp1 xmm8
4662 %define ymm_tmp1 ymm8
4663 %define xmm_tmp2 xmm9
4664 %define ymm_tmp2 ymm9
4665 %define xmm_tmp3 xmm10
4666 %define ymm_tmp3 ymm10
4667 %define xmm_tmp4 xmm11
4668 %define ymm_tmp4 ymm11
4669 %define xmm_tmp5 xmm12
4670 %define ymm_tmp5 ymm12
4671 %define ymm_7fff ymm13
4672 %define ymm_xpos_int_begin ymm14
4673 %define ymm_xpos_frac_begin ymm15
4674 vpxor ymm_0, ymm_0, ymm_0
4675 vpcmpeqw ymm_7fff, ymm_7fff, ymm_7fff
4676 vpsrlw ymm_7fff, ymm_7fff, 1
4677%endif
4678
4679 sub i_dst_height, 1
4680 je .final_row
4681 jl .done
4682
4683 mov i_yposd, 1 << 14
4684 vmovd xmm_tmp0, i_scalexd
4685 vpbroadcastd ymm_tmp0, xmm_tmp0
4686 vpslld ymm_tmp1, ymm_tmp0, 2
4687 vpslld ymm_tmp2, ymm_tmp0, 3
4688 vpaddd ymm_tmp3, ymm_tmp1, ymm_tmp2
4689 vpxor ymm_tmp4, ymm_tmp4, ymm_tmp4
4690 vpblendd ymm_tmp1, ymm_tmp4, ymm_tmp1, 11110000b
4691 vpblendd ymm_tmp2, ymm_tmp2, ymm_tmp3, 11110000b
4692 vpaddd ymm_tmp3, ymm_tmp0, ymm_tmp0
4693 vpblendd ymm_tmp3, ymm_tmp4, ymm_tmp3, 11001100b
4694 vpblendd ymm_tmp0, ymm_tmp4, ymm_tmp0, 10101010b
4695 vpaddd ymm_tmp0, ymm_tmp3, ymm_tmp0
4696 vpaddd ymm_tmp1, ymm_tmp1, ymm_tmp0
4697 vpaddd ymm_tmp2, ymm_tmp2, ymm_tmp0
4698 vpcmpeqw ymm_tmp3, ymm_tmp3, ymm_tmp3
4699 vpsrld ymm_tmp3, ymm_tmp3, 31
4700 vpslld ymm_tmp3, ymm_tmp3, 15
4701 vpaddd ymm_tmp1, ymm_tmp1, ymm_tmp3
4702 vpaddd ymm_tmp2, ymm_tmp2, ymm_tmp3
4703 vpsrld ymm_xpos_int, ymm_tmp1, 16
4704 vpsrld ymm_tmp0, ymm_tmp2, 16
4705 vpackssdw ymm_xpos_int, ymm_xpos_int, ymm_tmp0
4706 vpermq ymm_xpos_int, ymm_xpos_int, 11011000b
4707 vpackuswb ymm_xpos_int, ymm_xpos_int, ymm_xpos_int
4708 vpcmpeqw ymm_tmp3, ymm_tmp3, ymm_tmp3
4709 vpsubb ymm_tmp0, ymm_xpos_int, ymm_tmp3
4710 vpunpcklbw ymm_xpos_int, ymm_xpos_int, ymm_tmp0
4711 vpslld ymm_tmp1, ymm_tmp1, 16
4712 vpsrld ymm_tmp1, ymm_tmp1, 16
4713 vpslld ymm_tmp2, ymm_tmp2, 16
4714 vpsrld ymm_tmp2, ymm_tmp2, 16
4715 vpackusdw ymm_xpos_frac, ymm_tmp1, ymm_tmp2
4716 vpermq ymm_xpos_frac, ymm_xpos_frac, 11011000b
4717 vpsrlw ymm_xpos_frac, ymm_xpos_frac, 1
4718 vmovd xmm_tmp0, i_scalexd
4719 vpslld xmm_tmp0, xmm_tmp0, 4
4720 vpbroadcastw ymm_tmp1, xmm_tmp0
4721 vpsrlw ymm_tmp1, ymm_tmp1, 1
4722 vmovdqa ymm_xpos_frac_inc, ymm_tmp1
4723 vpsrld xmm_tmp0, xmm_tmp0, 16
4724 vpsubw ymm_tmp0, ymm_tmp0, ymm_tmp3
4725 vpbroadcastb ymm_tmp0, xmm_tmp0
4726 vmovdqa ymm_xpos_int_inc, ymm_tmp0
4727 vmovdqa ymm_xpos_int_begin, ymm_xpos_int
4728 vmovdqa ymm_xpos_frac_begin, ymm_xpos_frac
4729
4730 cmp i_scalex, 4 << 16
4731 ja .scalex_above4
4732 cmp i_scalex, 2 << 16
4733 ja .scalex_above2_beloweq4
4734 add i_scalex, i_scalex
4735%xdefine i_scalex2 i_scalex
4736%undef i_scalex
4737 AVX2_GeneralBilinearDownsampler_loop AVX2_BilinearAccurateDownsample2xOrLess_16px, 0
4738 shr i_scalex2, 1
4739%xdefine i_scalex i_scalex2
4740%undef i_scalex2
4741 jmp .final_row
4742.scalex_above2_beloweq4:
4743 add i_scalex, i_scalex
4744%xdefine i_scalex2 i_scalex
4745%undef i_scalex
4746 AVX2_GeneralBilinearDownsampler_loop AVX2_BilinearAccurateDownsample4xOrLess_16px, 0
4747 shr i_scalex2, 1
4748%xdefine i_scalex i_scalex2
4749%undef i_scalex2
4750 jmp .final_row
4751.scalex_above4:
4752 cmp i_scalex, 8 << 16
4753 ja .scalex_above8
4754 add i_scalex, i_scalex
4755%xdefine i_scalex2 i_scalex
4756%undef i_scalex
4757 AVX2_GeneralBilinearDownsampler_loop AVX2_BilinearAccurateDownsample8xOrLess_16px, 0
4758 shr i_scalex2, 1
4759%xdefine i_scalex i_scalex2
4760%undef i_scalex2
4761 jmp .final_row
4762.scalex_above8:
4763%xdefine ymm_xfrac0 ymm_xpos_frac
4764%xdefine ymm_xfrac1 ymm_xpos_int
4765%xdefine ymm_xfrac0_begin ymm_xpos_int_begin
4766%xdefine ymm_xfrac1_begin ymm_xpos_frac_begin
4767%xdefine ymm_xfrac_inc ymm_xpos_frac_inc
4768%undef ymm_xpos_int
4769%undef ymm_xpos_frac
4770%undef ymm_xpos_int_begin
4771%undef ymm_xpos_frac_begin
4772%undef ymm_xpos_int_inc
4773%undef ymm_xpos_frac_inc
4774 AVX2_UnpckXFrac ymm_tmp0, ymm_xfrac1, ymm_xfrac0, ymm_7fff
4775 vpermq ymm_xfrac0, ymm_tmp0, 01001110b
4776 vpermq ymm_xfrac1, ymm_xfrac1, 01001110b
4777 vmovdqa ymm_xfrac0_begin, ymm_xfrac0
4778 vmovdqa ymm_xfrac1_begin, ymm_xfrac1
4779 vpcmpeqw ymm_tmp0, ymm_tmp0, ymm_tmp0
4780 vpmullw ymm_tmp0, ymm_tmp0, ymm_xfrac_inc
4781 vpunpcklwd ymm_tmp0, ymm_tmp0, ymm_xfrac_inc
4782 vmovdqa ymm_xfrac_inc, ymm_tmp0
4783 AVX2_GeneralBilinearDownsampler_loop AVX2_GeneralBilinearAccurateDownsample_16px, 0
4784
4785.final_row:
4786 mov p_src_row0, i_ypos
4787 shr p_src_row0, 15
4788 imul p_src_row0, i_src_stride
4789 add p_src_row0, p_src
4790 mov i_xpos, 1 << 15
4791 mov i_width_cnt, i_dst_width
4792
4793.final_row_width:
4794 mov r_tmp0, i_xpos
4795 shr r_tmp0, 16
4796 movzx r_tmp0, byte [p_src_row0 + r_tmp0]
4797 mov [p_dst], r_tmp0b
4798 add p_dst, 1
4799 add i_xpos, i_scalex
4800 sub i_width_cnt, 1
4801 jg .final_row_width
4802
4803.done:
4804 vzeroupper
4805%ifdef X86_32
4806 mov esp, [esp]
4807%endif
4808 POP_XMM
4809 LOAD_7_PARA_POP
4810%ifndef X86_32
4811%ifdef WIN64
4812 pop rsi
4813 pop rdi
4814%endif
4815 pop rbp
4816 pop rbx
4817 pop r13
4818 pop r12
4819%endif
4820 ret
4821%undef p_dst
4822%undef i_dst_stride_less_width
4823%undef i_dst_width
4824%undef i_dst_height
4825%undef p_src
4826%undef i_src_stride
4827%undef i_scalex
4828%undef i_scalexd
4829%undef i_scaleyd
4830%undef i_xpos
4831%undef i_ypos
4832%undef i_yposd
4833%undef p_src_row0
4834%undef p_src_row1
4835%undef i_width_cnt
4836%undef r_tmp0
4837%undef r_tmp0b
4838%undef ymm_xpos_frac
4839%undef ymm_xpos_frac_inc
4840%undef ymm_xpos_int
4841%undef ymm_xpos_int_inc
4842%undef ymm_yfrac0
4843%undef ymm_yfrac1
4844%undef xmm_tmp0
4845%undef ymm_tmp0
4846%undef xmm_tmp1
4847%undef ymm_tmp1
4848%undef xmm_tmp2
4849%undef ymm_tmp2
4850%undef xmm_tmp3
4851%undef ymm_tmp3
4852%undef xmm_tmp4
4853%undef ymm_tmp4
4854%undef xmm_tmp5
4855%undef ymm_tmp5
4856%undef ymm_0
4857%undef ymm_7fff
4858%undef ymm_xpos_int_begin
4859%undef ymm_xpos_frac_begin
4860%undef ymm_xfrac0
4861%undef ymm_xfrac1
4862%undef ymm_xfrac0_begin
4863%undef ymm_xfrac1_begin
4864%undef ymm_xfrac_inc

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t