History

[Processing/x86] Add an AVX2 implementation of GeneralBilinearFastDownsample

Keep track of relative pixel offsets and utilize pshufb to efficiently extract relevant pixels for horizontal scaling ratios <= 8. Because pshufb does not cross 128-bit lanes, the overhead of address calculations and loads is relatively greater as compared with an SSSE3 implementation. Fall back to a generic approach for ratios > 8. The implementation assumes that data beyond the end of each line, before the next line begins, can be dirtied; which AFAICT is safe with the current usage of these routines. Speedup is ~10.42x/~5.23x (32-bit/64-bit) for horizontal ratios <= 2, ~9.49x/~4.64x for ratios within (2, 4], ~6.43x/~3.18x for ratios within (4, 8], and ~5.42x/~2.50x for ratios > 8 when not memory-bound on Haswell as compared with the current SSE2 implementation.

Sindre Aamås committed 10 years ago

Commit b43e58a1 parent b1013095 files · +709 −8Browse files

Showing the changes to codec/processing/src (4 of 5 files). Show all

Unified · Split

4 files
  1. codec/processing/src/downsample/downsample.cpp +3 −0
  2. codec/processing/src/downsample/downsample.h +4 −0
  3. codec/processing/src/downsample/downsamplefuncs.cpp +1 −0
  4. codec/processing/src/x86/downsample_bilinear.asm +699 −8
codec/processing/src/downsample/downsample.cpp+3−0
@@ -109,6 +109,9 @@ void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int · 108 unmodified lines
109 sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_sse4;109 sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_sse4;
110 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_sse41;110 sDownsampleFunc.pfGeneralRatioChroma = GeneralBilinearAccurateDownsamplerWrap_sse41;
111 }111 }
112 if (iCpuFlag & WELS_CPU_AVX2) {
113 sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearFastDownsamplerWrap_avx2;
114 }
112#endif//X86_ASM115#endif//X86_ASM
113 116
114#if defined(HAVE_NEON)117#if defined(HAVE_NEON)
codec/processing/src/downsample/downsample.h+4−0
@@ -103,6 +103,7 @@ GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_sse2; · 102 unmodified lines
103GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse2;103GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse2;
104GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_ssse3;104GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_ssse3;
105GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse41;105GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse41;
106GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_avx2;
106 107
107SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_ssse3;108SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_ssse3;
108SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_sse4;109SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_sse4;
@@ -122,6 +123,9 @@ void GeneralBilinearFastDownsampler_ssse3 (uint8_t* pDst, int32_t iDstStride, in · 13 unmodified lines
122void GeneralBilinearAccurateDownsampler_sse41 (uint8_t* pDst, int32_t iDstStride, int32_t iDstWidth,123void GeneralBilinearAccurateDownsampler_sse41 (uint8_t* pDst, int32_t iDstStride, int32_t iDstWidth,
123 int32_t iDstHeight, uint8_t* pSrc, int32_t iSrcStride, uint32_t uiScaleX,124 int32_t iDstHeight, uint8_t* pSrc, int32_t iSrcStride, uint32_t uiScaleX,
124 uint32_t uiScaleY);125 uint32_t uiScaleY);
126void GeneralBilinearFastDownsampler_avx2 (uint8_t* pDst, int32_t iDstStride, int32_t iDstWidth,
127 int32_t iDstHeight, uint8_t* pSrc, int32_t iSrcStride, uint32_t uiScaleX,
128 uint32_t uiScaleY);
125 129
126WELSVP_EXTERN_C_END130WELSVP_EXTERN_C_END
127#endif131#endif
codec/processing/src/downsample/downsamplefuncs.cpp+1−0
@@ -284,6 +284,7 @@ DEFINE_GENERAL_BILINEAR_FAST_DOWNSAMPLER_WRAP (sse2) · 283 unmodified lines
284DEFINE_GENERAL_BILINEAR_ACCURATE_DOWNSAMPLER_WRAP (sse2)284DEFINE_GENERAL_BILINEAR_ACCURATE_DOWNSAMPLER_WRAP (sse2)
285DEFINE_GENERAL_BILINEAR_FAST_DOWNSAMPLER_WRAP (ssse3)285DEFINE_GENERAL_BILINEAR_FAST_DOWNSAMPLER_WRAP (ssse3)
286DEFINE_GENERAL_BILINEAR_ACCURATE_DOWNSAMPLER_WRAP (sse41)286DEFINE_GENERAL_BILINEAR_ACCURATE_DOWNSAMPLER_WRAP (sse41)
287DEFINE_GENERAL_BILINEAR_FAST_DOWNSAMPLER_WRAP (avx2)
287#endif //X86_ASM288#endif //X86_ASM
288 289
289#ifdef HAVE_NEON290#ifdef HAVE_NEON
codec/processing/src/x86/downsample_bilinear.asm+699−8
@@ -53,15 +53,15 @@ · 52 unmodified lines
53; Local Data (Read Only)53; Local Data (Read Only)
54;***********************************************************************54;***********************************************************************
55 55
56SECTION .rodata align=1656SECTION .rodata align=32
57 57
58;***********************************************************************58;***********************************************************************
59; Various memory constants (trigonometric values or rounding values)59; Various memory constants (trigonometric values or rounding values)
60;***********************************************************************60;***********************************************************************
61 61
62ALIGN 1662ALIGN 32
63db80h_128:63db80h_256:
64 times 16 db 80h64 times 32 db 80h
65shufb_0000000088888888:65shufb_0000000088888888:
66 times 8 db 066 times 8 db 0
67 times 8 db 867 times 8 db 8
@@ -2682,7 +2682,7 @@ WELS_EXTERN DyadicBilinearQuarterDownsampler_sse4 · 2614 unmodified lines
2682 movdqu xmm_tmp3, [p_src_row0 + r_tmp0]2682 movdqu xmm_tmp3, [p_src_row0 + r_tmp0]
2683 movdqu xmm_tmp4, [p_src_row1 + r_tmp0]2683 movdqu xmm_tmp4, [p_src_row1 + r_tmp0]
2684 movdqa xmm_tmp2, xmm_xpos_int2684 movdqa xmm_tmp2, xmm_xpos_int
2685 punpcklbw xmm_tmp2, [db80h_128]2685 punpcklbw xmm_tmp2, [db80h_256]
2686 pshufb xmm_tmp3, xmm_tmp22686 pshufb xmm_tmp3, xmm_tmp2
2687 pshufb xmm_tmp4, xmm_tmp22687 pshufb xmm_tmp4, xmm_tmp2
2688 SSE2_BilinearFastCalcXYFrac xmm_tmp0, xmm_tmp2, xmm_yfrac0, xmm_yfrac12688 SSE2_BilinearFastCalcXYFrac xmm_tmp0, xmm_tmp2, xmm_yfrac0, xmm_yfrac1
@@ -2695,7 +2695,7 @@ WELS_EXTERN DyadicBilinearQuarterDownsampler_sse4 · 6 unmodified lines
2695 movdqu xmm_tmp3, [p_src_row0 + r_tmp0]2695 movdqu xmm_tmp3, [p_src_row0 + r_tmp0]
2696 movdqu xmm_tmp4, [p_src_row1 + r_tmp0]2696 movdqu xmm_tmp4, [p_src_row1 + r_tmp0]
2697 movdqa xmm_tmp2, xmm_xpos_int2697 movdqa xmm_tmp2, xmm_xpos_int
2698 punpckhbw xmm_tmp2, [db80h_128]2698 punpckhbw xmm_tmp2, [db80h_256]
2699 pshufb xmm_tmp3, xmm_tmp22699 pshufb xmm_tmp3, xmm_tmp2
2700 pshufb xmm_tmp4, xmm_tmp22700 pshufb xmm_tmp4, xmm_tmp2
2701 SSE2_BilinearFastCalcXYFrac xmm_tmp1, xmm_tmp2, xmm_yfrac0, xmm_yfrac12701 SSE2_BilinearFastCalcXYFrac xmm_tmp1, xmm_tmp2, xmm_yfrac0, xmm_yfrac1
@@ -2840,7 +2840,7 @@ WELS_EXTERN DyadicBilinearQuarterDownsampler_sse4 · 138 unmodified lines
2840 mov r_tmp0, i_xpos2840 mov r_tmp0, i_xpos
2841 shr r_tmp0, 162841 shr r_tmp0, 16
2842 movdqa xmm_tmp3, xmm_xpos_int2842 movdqa xmm_tmp3, xmm_xpos_int
2843 punpcklbw xmm_tmp3, [db80h_128]2843 punpcklbw xmm_tmp3, [db80h_256]
2844 movdqu xmm_tmp4, [p_src_row0 + r_tmp0]2844 movdqu xmm_tmp4, [p_src_row0 + r_tmp0]
2845 movdqu xmm_tmp2, [p_src_row1 + r_tmp0]2845 movdqu xmm_tmp2, [p_src_row1 + r_tmp0]
2846 lea r_tmp0, [i_xpos + 4 * i_scalex]2846 lea r_tmp0, [i_xpos + 4 * i_scalex]
@@ -2852,7 +2852,7 @@ WELS_EXTERN DyadicBilinearQuarterDownsampler_sse4 · 5 unmodified lines
2852 pmaddwd xmm_tmp2, xmm_tmp02852 pmaddwd xmm_tmp2, xmm_tmp0
2853 SSE41_LinearAccurateInterpolateVerticalDwords xmm_tmp0, xmm_tmp4, xmm_tmp2, xmm_yfrac0, xmm_yfrac1, xmm_tmp32853 SSE41_LinearAccurateInterpolateVerticalDwords xmm_tmp0, xmm_tmp4, xmm_tmp2, xmm_yfrac0, xmm_yfrac1, xmm_tmp3
2854 movdqa xmm_tmp2, xmm_xpos_int2854 movdqa xmm_tmp2, xmm_xpos_int
2855 punpckhbw xmm_tmp2, [db80h_128]2855 punpckhbw xmm_tmp2, [db80h_256]
2856 movdqu xmm_tmp4, [p_src_row0 + r_tmp0]2856 movdqu xmm_tmp4, [p_src_row0 + r_tmp0]
2857 movdqu xmm_tmp3, [p_src_row1 + r_tmp0]2857 movdqu xmm_tmp3, [p_src_row1 + r_tmp0]
2858 pshufb xmm_tmp4, xmm_tmp22858 pshufb xmm_tmp4, xmm_tmp2
@@ -3563,3 +3563,694 @@ WELS_EXTERN GeneralBilinearAccurateDownsampler_sse41 · 704 unmodified lines
3563%undef xmm_xfrac0_begin3563%undef xmm_xfrac0_begin
3564%undef xmm_xfrac1_begin3564%undef xmm_xfrac1_begin
3565%undef xmm_xfrac_inc3565%undef xmm_xfrac_inc
3566
3567; xpos_int=%1 xpos_frac=%2 inc_int+1=%3 inc_frac=%4 tmp=%5
3568%macro AVX2_BilinearIncXposuw 5
3569 vpaddusw %5, %2, %4
3570 vpaddw %2, %2, %4
3571 vpcmpeqw %5, %5, %2
3572 vpaddb %1, %1, %3
3573 vpaddb %1, %1, %5 ; subtract 1 if no carry
3574%endmacro
3575
3576; outl=%1 outh=%2 in=%3 FFFFh/7FFFh=%4
3577%macro AVX2_UnpckXFrac 4
3578 vpxor %1, %3, %4
3579 vpunpckhwd %2, %1, %3
3580 vpunpcklwd %1, %1, %3
3581%endmacro
3582
3583; out0=%1 out1=%2 xfrac=%3 yfrac0=%4 yfrac1=%5
3584%macro AVX2_BilinearFastCalcXYFrac 5
3585 vpmulhuw %2, %3, %5
3586 vpmulhuw %1, %3, %4
3587%endmacro
3588
3589; [in:dwordsl out:bytes] dwordsh=%2 zero=%3
3590%macro AVX2_BilinearFastPackDwordsToBytes 3
3591 vpsrld %1, %1, 14
3592 vpsrld %2, %2, 14
3593 vpackssdw %1, %1, %2
3594 vpavgw %1, %1, %3
3595 vpackuswb %1, %1, %1
3596%endmacro
3597
3598%macro AVX2_BilinearFastDownsample2xOrLess_16px 0
3599 vpshufb ymm_tmp0, ymm_xpos_int, ymm_0
3600 vpsubb ymm_xpos_int, ymm_xpos_int, ymm_tmp0
3601 AVX2_UnpckXFrac ymm_tmp0, ymm_tmp1, ymm_xpos_frac, ymm_ffff
3602 mov r_tmp0, i_xpos
3603 shr r_tmp0, 16
3604 vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3605 vmovdqu xmm_tmp5, [p_src_row1 + r_tmp0]
3606 lea r_tmp0, [i_xpos + 4 * i_scalex2]
3607 lea i_xpos, [i_xpos + 8 * i_scalex2]
3608 shr r_tmp0, 16
3609 vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
3610 vinserti128 ymm_tmp5, ymm_tmp5, [p_src_row1 + r_tmp0], 1
3611 vpshufb ymm_tmp4, ymm_tmp4, ymm_xpos_int
3612 vpshufb ymm_tmp5, ymm_tmp5, ymm_xpos_int
3613 AVX2_BilinearFastCalcXYFrac ymm_tmp0, ymm_tmp2, ymm_tmp0, ymm_yfrac0, ymm_yfrac1
3614 vpunpcklbw ymm_tmp3, ymm_tmp4, ymm_0
3615 vpmaddwd ymm_tmp0, ymm_tmp0, ymm_tmp3
3616 vpunpcklbw ymm_tmp3, ymm_tmp5, ymm_0
3617 vpmaddwd ymm_tmp2, ymm_tmp2, ymm_tmp3
3618 vpaddd ymm_tmp0, ymm_tmp0, ymm_tmp2
3619 AVX2_BilinearFastCalcXYFrac ymm_tmp1, ymm_tmp3, ymm_tmp1, ymm_yfrac0, ymm_yfrac1
3620 vpunpckhbw ymm_tmp2, ymm_tmp4, ymm_0
3621 vpmaddwd ymm_tmp1, ymm_tmp1, ymm_tmp2
3622 vpunpckhbw ymm_tmp2, ymm_tmp5, ymm_0
3623 vpmaddwd ymm_tmp3, ymm_tmp3, ymm_tmp2
3624 vpaddd ymm_tmp1, ymm_tmp1, ymm_tmp3
3625 AVX2_BilinearFastPackDwordsToBytes ymm_tmp0, ymm_tmp1, ymm_0
3626 vmovlps [p_dst], xmm_tmp0
3627 vextracti128 [p_dst + 8], ymm_tmp0, 1
3628 add p_dst, 16
3629 AVX2_BilinearIncXposuw ymm_xpos_int, ymm_xpos_frac, ymm_xpos_int_inc, ymm_xpos_frac_inc, ymm_tmp0
3630%endmacro
3631
3632%macro AVX2_BilinearFastDownsample4xOrLess_16px 0
3633 vbroadcasti128 ymm_tmp0, [shufb_0000000088888888]
3634 vpshufb ymm_tmp0, ymm_xpos_int, ymm_tmp0
3635 vpsubb ymm_xpos_int, ymm_xpos_int, ymm_tmp0
3636 AVX2_UnpckXFrac ymm_tmp0, ymm_tmp1, ymm_xpos_frac, ymm_ffff
3637 mov r_tmp0, i_xpos
3638 shr r_tmp0, 16
3639 vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3640 vmovdqu xmm_tmp3, [p_src_row1 + r_tmp0]
3641 lea r_tmp0, [i_xpos + 4 * i_scalex2]
3642 shr r_tmp0, 16
3643 vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
3644 vinserti128 ymm_tmp3, ymm_tmp3, [p_src_row1 + r_tmp0], 1
3645 lea r_tmp0, [i_xpos + 2 * i_scalex2]
3646 lea i_xpos, [r_tmp0 + 4 * i_scalex2]
3647 shr r_tmp0, 16
3648 vpunpcklbw ymm_tmp2, ymm_xpos_int, ymm_ffff
3649 vpshufb ymm_tmp4, ymm_tmp4, ymm_tmp2
3650 vpshufb ymm_tmp3, ymm_tmp3, ymm_tmp2
3651 AVX2_BilinearFastCalcXYFrac ymm_tmp0, ymm_tmp2, ymm_tmp0, ymm_yfrac0, ymm_yfrac1
3652 vpmaddwd ymm_tmp0, ymm_tmp0, ymm_tmp4
3653 vpmaddwd ymm_tmp2, ymm_tmp2, ymm_tmp3
3654 vpaddd ymm_tmp0, ymm_tmp0, ymm_tmp2
3655 vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3656 vmovdqu xmm_tmp3, [p_src_row1 + r_tmp0]
3657 mov r_tmp0, i_xpos
3658 lea i_xpos, [i_xpos + 2 * i_scalex2]
3659 shr r_tmp0, 16
3660 vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
3661 vinserti128 ymm_tmp3, ymm_tmp3, [p_src_row1 + r_tmp0], 1
3662 vpunpckhbw ymm_tmp2, ymm_xpos_int, ymm_ffff
3663 vpshufb ymm_tmp4, ymm_tmp4, ymm_tmp2
3664 vpshufb ymm_tmp3, ymm_tmp3, ymm_tmp2
3665 AVX2_BilinearFastCalcXYFrac ymm_tmp1, ymm_tmp2, ymm_tmp1, ymm_yfrac0, ymm_yfrac1
3666 vpmaddwd ymm_tmp1, ymm_tmp1, ymm_tmp4
3667 vpmaddwd ymm_tmp2, ymm_tmp2, ymm_tmp3
3668 vpaddd ymm_tmp1, ymm_tmp1, ymm_tmp2
3669 AVX2_BilinearFastPackDwordsToBytes ymm_tmp0, ymm_tmp1, ymm_0
3670 vmovlps [p_dst], xmm_tmp0
3671 vextracti128 [p_dst + 8], ymm_tmp0, 1
3672 add p_dst, 16
3673 AVX2_BilinearIncXposuw ymm_xpos_int, ymm_xpos_frac, ymm_xpos_int_inc, ymm_xpos_frac_inc, ymm_tmp0
3674%endmacro
3675
3676%macro AVX2_BilinearFastDownsample8xOrLess_16px 0
3677 vbroadcasti128 ymm_tmp0, [shufb_000044448888CCCC]
3678 vpshufb ymm_tmp0, ymm_xpos_int, ymm_tmp0
3679 vpsubb ymm_xpos_int, ymm_xpos_int, ymm_tmp0
3680 mov r_tmp0, i_xpos
3681 shr r_tmp0, 16
3682 vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3683 vmovdqu xmm_tmp5, [p_src_row1 + r_tmp0]
3684 lea r_tmp0, [i_xpos + 4 * i_scalex2]
3685 add i_xpos, i_scalex2
3686 shr r_tmp0, 16
3687 vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
3688 vinserti128 ymm_tmp5, ymm_tmp5, [p_src_row1 + r_tmp0], 1
3689 mov r_tmp0, i_xpos
3690 shr r_tmp0, 16
3691 vmovdqu xmm_tmp0, [p_src_row0 + r_tmp0]
3692 vmovdqu xmm_tmp1, [p_src_row1 + r_tmp0]
3693 lea r_tmp0, [i_xpos + 4 * i_scalex2]
3694 add i_xpos, i_scalex2
3695 shr r_tmp0, 16
3696 vinserti128 ymm_tmp0, ymm_tmp0, [p_src_row0 + r_tmp0], 1
3697 vinserti128 ymm_tmp1, ymm_tmp1, [p_src_row1 + r_tmp0], 1
3698 vpunpcklbw ymm_tmp3, ymm_xpos_int, ymm_ffff
3699 vpshufb ymm_tmp4, ymm_tmp4, ymm_tmp3
3700 vpshufb ymm_tmp5, ymm_tmp5, ymm_tmp3
3701 vpshufb ymm_tmp0, ymm_tmp0, ymm_tmp3
3702 vpshufb ymm_tmp1, ymm_tmp1, ymm_tmp3
3703 vpblendd ymm_tmp4, ymm_tmp4, ymm_tmp0, 11001100b
3704 vpblendd ymm_tmp5, ymm_tmp5, ymm_tmp1, 11001100b
3705 AVX2_UnpckXFrac ymm_tmp0, ymm_tmp1, ymm_xpos_frac, ymm_ffff
3706 AVX2_BilinearFastCalcXYFrac ymm_tmp0, ymm_tmp2, ymm_tmp0, ymm_yfrac0, ymm_yfrac1
3707 vpmaddwd ymm_tmp0, ymm_tmp0, ymm_tmp4
3708 vpmaddwd ymm_tmp2, ymm_tmp2, ymm_tmp5
3709 vpaddd ymm_tmp0, ymm_tmp0, ymm_tmp2
3710 mov r_tmp0, i_xpos
3711 shr r_tmp0, 16
3712 vmovdqu xmm_tmp4, [p_src_row0 + r_tmp0]
3713 vmovdqu xmm_tmp5, [p_src_row1 + r_tmp0]
3714 lea r_tmp0, [i_xpos + 4 * i_scalex2]
3715 add i_xpos, i_scalex2
3716 shr r_tmp0, 16
3717 vinserti128 ymm_tmp4, ymm_tmp4, [p_src_row0 + r_tmp0], 1
3718 vinserti128 ymm_tmp5, ymm_tmp5, [p_src_row1 + r_tmp0], 1
3719 mov r_tmp0, i_xpos
3720 lea i_xpos, [i_xpos + 4 * i_scalex2]
3721 shr r_tmp0, 16
3722 vmovdqu xmm_tmp2, [p_src_row0 + r_tmp0]
3723 vmovdqu xmm_tmp3, [p_src_row1 + r_tmp0]
3724 mov r_tmp0, i_xpos
3725 add i_xpos, i_scalex2
3726 shr r_tmp0, 16
3727 vinserti128 ymm_tmp2, ymm_tmp2, [p_src_row0 + r_tmp0], 1
3728 vinserti128 ymm_tmp3, ymm_tmp3, [p_src_row1 + r_tmp0], 1
3729 vpshufb ymm_tmp4, ymm_tmp4, ymm_xpos_int
3730 vpshufb ymm_tmp5, ymm_tmp5, ymm_xpos_int
3731 vpshufb ymm_tmp2, ymm_tmp2, ymm_xpos_int
3732 vpshufb ymm_tmp3, ymm_tmp3, ymm_xpos_int
3733 vpblendd ymm_tmp4, ymm_tmp4, ymm_tmp2, 10001000b
3734 vpblendd ymm_tmp5, ymm_tmp5, ymm_tmp3, 10001000b
3735 vpunpckhbw ymm_tmp4, ymm_tmp4, ymm_0
3736 vpunpckhbw ymm_tmp5, ymm_tmp5, ymm_0
3737 AVX2_BilinearFastCalcXYFrac ymm_tmp1, ymm_tmp3, ymm_tmp1, ymm_yfrac0, ymm_yfrac1
3738 vpmaddwd ymm_tmp1, ymm_tmp1, ymm_tmp4
3739 vpmaddwd ymm_tmp3, ymm_tmp3, ymm_tmp5
3740 vpaddd ymm_tmp1, ymm_tmp1, ymm_tmp3
3741 AVX2_BilinearFastPackDwordsToBytes ymm_tmp0, ymm_tmp1, ymm_0
3742 vmovlps [p_dst], xmm_tmp0
3743 vextracti128 [p_dst + 8], ymm_tmp0, 1
3744 add p_dst, 16
3745 AVX2_BilinearIncXposuw ymm_xpos_int, ymm_xpos_frac, ymm_xpos_int_inc, ymm_xpos_frac_inc, ymm_tmp0
3746%endmacro
3747
3748%macro AVX2_GeneralBilinearFastDownsample_16px 0
3749 mov r_tmp0, i_xpos
3750 shr r_tmp0, 16
3751 vpbroadcastd ymm_tmp4, [p_src_row0 + r_tmp0]
3752 vpbroadcastd ymm_tmp5, [p_src_row1 + r_tmp0]
3753 lea r_tmp0, [i_xpos + 1 * i_scalex]
3754 shr r_tmp0, 16
3755 vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0]
3756 vpunpcklwd ymm_tmp4, ymm_tmp4, ymm_tmp0
3757 vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0]
3758 vpunpcklwd ymm_tmp5, ymm_tmp5, ymm_tmp0
3759 lea r_tmp0, [i_xpos + 2 * i_scalex]
3760 lea i_xpos, [i_xpos + 4 * i_scalex]
3761 shr r_tmp0, 16
3762 vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0]
3763 vpblendd ymm_tmp4, ymm_tmp4, ymm_tmp0, 00100010b
3764 vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0]
3765 vpblendd ymm_tmp5, ymm_tmp5, ymm_tmp0, 00100010b
3766 mov r_tmp0, i_xpos
3767 sub r_tmp0, i_scalex
3768 shr r_tmp0, 16
3769 vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0 - 2]
3770 vpblendw ymm_tmp4, ymm_tmp4, ymm_tmp0, 1000b
3771 vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0 - 2]
3772 vpblendw ymm_tmp5, ymm_tmp5, ymm_tmp0, 1000b
3773 mov r_tmp0, i_xpos
3774 shr r_tmp0, 16
3775 vpbroadcastd ymm_tmp2, [p_src_row0 + r_tmp0]
3776 vpbroadcastd ymm_tmp3, [p_src_row1 + r_tmp0]
3777 lea r_tmp0, [i_xpos + 1 * i_scalex]
3778 shr r_tmp0, 16
3779 vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0]
3780 vpunpcklwd ymm_tmp2, ymm_tmp2, ymm_tmp0
3781 vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0]
3782 vpunpcklwd ymm_tmp3, ymm_tmp3, ymm_tmp0
3783 lea r_tmp0, [i_xpos + 2 * i_scalex]
3784 lea i_xpos, [i_xpos + 4 * i_scalex]
3785 shr r_tmp0, 16
3786 vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0]
3787 vpblendd ymm_tmp2, ymm_tmp2, ymm_tmp0, 00100010b
3788 vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0]
3789 vpblendd ymm_tmp3, ymm_tmp3, ymm_tmp0, 00100010b
3790 mov r_tmp0, i_xpos
3791 sub r_tmp0, i_scalex
3792 shr r_tmp0, 16
3793 vpbroadcastd ymm_tmp0, [p_src_row0 + r_tmp0 - 2]
3794 vpblendw ymm_tmp2, ymm_tmp2, ymm_tmp0, 1000b
3795 vpbroadcastd ymm_tmp0, [p_src_row1 + r_tmp0 - 2]
3796 vpblendw ymm_tmp3, ymm_tmp3, ymm_tmp0, 1000b
3797 mov r_tmp0, i_xpos
3798 shr r_tmp0, 16
3799 vmovd xmm_tmp0, [p_src_row0 + r_tmp0]
3800 vmovd xmm_tmp1, [p_src_row1 + r_tmp0]
3801 lea r_tmp0, [i_xpos + i_scalex]
3802 shr r_tmp0, 16
3803 vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 1
3804 vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 1
3805 lea r_tmp0, [i_xpos + 2 * i_scalex]
3806 lea i_xpos, [i_xpos + 4 * i_scalex]
3807 shr r_tmp0, 16
3808 vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 2
3809 vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 2
3810 mov r_tmp0, i_xpos
3811 sub r_tmp0, i_scalex
3812 shr r_tmp0, 16
3813 vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 3
3814 vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 3
3815 vpblendd ymm_tmp4, ymm_tmp4, ymm_tmp0, 00001111b
3816 vpblendd ymm_tmp5, ymm_tmp5, ymm_tmp1, 00001111b
3817 mov r_tmp0, i_xpos
3818 shr r_tmp0, 16
3819 vmovd xmm_tmp0, [p_src_row0 + r_tmp0]
3820 vmovd xmm_tmp1, [p_src_row1 + r_tmp0]
3821 lea r_tmp0, [i_xpos + i_scalex]
3822 shr r_tmp0, 16
3823 vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 1
3824 vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 1
3825 lea r_tmp0, [i_xpos + 2 * i_scalex]
3826 lea i_xpos, [i_xpos + 4 * i_scalex]
3827 shr r_tmp0, 16
3828 vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 2
3829 vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 2
3830 mov r_tmp0, i_xpos
3831 sub r_tmp0, i_scalex
3832 shr r_tmp0, 16
3833 vpinsrw xmm_tmp0, [p_src_row0 + r_tmp0], 3
3834 vpinsrw xmm_tmp1, [p_src_row1 + r_tmp0], 3
3835 vpblendd ymm_tmp2, ymm_tmp2, ymm_tmp0, 00001111b
3836 vpblendd ymm_tmp3, ymm_tmp3, ymm_tmp1, 00001111b
3837 vpunpcklbw ymm_tmp4, ymm_tmp4, ymm_0
3838 vpunpcklbw ymm_tmp5, ymm_tmp5, ymm_0
3839 AVX2_BilinearFastCalcXYFrac ymm_tmp0, ymm_tmp1, ymm_xfrac0, ymm_yfrac0, ymm_yfrac1
3840 vpmaddwd ymm_tmp0, ymm_tmp0, ymm_tmp4
3841 vpmaddwd ymm_tmp1, ymm_tmp1, ymm_tmp5
3842 vpaddd ymm_tmp0, ymm_tmp0, ymm_tmp1
3843 vpunpcklbw ymm_tmp4, ymm_tmp2, ymm_0
3844 vpunpcklbw ymm_tmp5, ymm_tmp3, ymm_0
3845 AVX2_BilinearFastCalcXYFrac ymm_tmp1, ymm_tmp2, ymm_xfrac1, ymm_yfrac0, ymm_yfrac1
3846 vpmaddwd ymm_tmp1, ymm_tmp1, ymm_tmp4
3847 vpmaddwd ymm_tmp2, ymm_tmp2, ymm_tmp5
3848 vpaddd ymm_tmp1, ymm_tmp1, ymm_tmp2
3849 AVX2_BilinearFastPackDwordsToBytes ymm_tmp0, ymm_tmp1, ymm_0
3850 vpermq ymm_tmp0, ymm_tmp0, 0010b
3851 vmovdqu [p_dst], xmm_tmp0
3852 add p_dst, 16
3853 vpaddw ymm_xfrac0, ymm_xfrac0, ymm_xfrac_inc
3854 vpaddw ymm_xfrac1, ymm_xfrac1, ymm_xfrac_inc
3855%endmacro
3856
3857; downsample_16px_macro=%1 b_fast=%2
3858%macro AVX2_GeneralBilinearDownsampler_loop 2
3859%%height:
3860 mov p_src_row0, i_ypos
3861 shr p_src_row0, 15
3862 imul p_src_row0, i_src_stride
3863 add p_src_row0, p_src
3864 mov p_src_row1, p_src_row0
3865 add p_src_row1, i_src_stride
3866%ifdef X86_32
3867%if %2
3868 vpbroadcastw ymm_tmp1, i_ypos
3869 vpsllw ymm_tmp1, ymm_tmp1, 1
3870 vpsrlw ymm_tmp1, ymm_tmp1, 1
3871 vpcmpeqw ymm_tmp0, ymm_tmp0, ymm_tmp0
3872 vpsrlw ymm_tmp0, ymm_tmp0, 1
3873%else
3874 vpbroadcastd ymm_tmp1, i_ypos
3875 vpslld ymm_tmp1, ymm_tmp1, 17
3876 vpsrld ymm_tmp1, ymm_tmp1, 17
3877 vpcmpeqw ymm_tmp0, ymm_tmp0, ymm_tmp0
3878 vpsrld ymm_tmp0, ymm_tmp0, 17
3879%endif
3880 vpxor ymm_tmp0, ymm_tmp0, ymm_tmp1
3881 vmovdqa ymm_yfrac0, ymm_tmp0
3882 vmovdqa ymm_yfrac1, ymm_tmp1
3883%else
3884 vmovd xmm_tmp0, i_yposd
3885 vpbroadcastw ymm_yfrac1, xmm_tmp0
3886%if %2
3887 vpsllw ymm_yfrac1, ymm_yfrac1, 1
3888 vpsrlw ymm_yfrac1, ymm_yfrac1, 1
3889 vpcmpeqw ymm_yfrac0, ymm_yfrac0, ymm_yfrac0
3890 vpsrlw ymm_yfrac0, ymm_yfrac0, 1
3891%else
3892 vpslld ymm_yfrac1, ymm_yfrac1, 17
3893 vpsrld ymm_yfrac1, ymm_yfrac1, 17
3894 vpcmpeqw ymm_yfrac0, ymm_yfrac0, ymm_yfrac0
3895 vpsrld ymm_yfrac0, ymm_yfrac0, 17
3896%endif
3897 vpxor ymm_yfrac0, ymm_yfrac0, ymm_yfrac1
3898%endif
3899
3900 mov i_xpos, 1 << 15
3901 mov i_width_cnt, i_dst_width
3902 sub i_width_cnt, 1
3903
3904%ifdef ymm_xpos_int
3905 vmovdqa ymm_xpos_int, ymm_xpos_int_begin
3906 vmovdqa ymm_xpos_frac, ymm_xpos_frac_begin
3907%else
3908 vmovdqa ymm_xfrac0, ymm_xfrac0_begin
3909 vmovdqa ymm_xfrac1, ymm_xfrac1_begin
3910%endif
3911
3912%%width:
3913 %1
3914 sub i_width_cnt, 16
3915 jg %%width
3916
3917 lea p_dst, [p_dst + i_width_cnt + 1]
3918%ifdef i_scalex2
3919 mov r_tmp0, i_scalex2
3920 shr r_tmp0, 1
3921 imul i_width_cnt, r_tmp0
3922%else
3923 imul i_width_cnt, i_scalex
3924%endif
3925 add i_xpos, i_width_cnt
3926 shr i_xpos, 16
3927 movzx r_tmp0, byte [p_src_row0 + i_xpos]
3928 mov [p_dst - 1], r_tmp0b
3929%ifdef X86_32
3930 mov r_tmp0, i_scaleyd
3931 add i_yposd, r_tmp0
3932%else
3933 add i_yposd, i_scaleyd
3934%endif
3935 add p_dst, i_dst_stride_less_width
3936 sub i_dst_height, 1
3937 jg %%height
3938%endmacro
3939
3940;**************************************************************************************************************
3941;void GeneralBilinearFastDownsampler_avx2 (uint8_t* pDst, int32_t iDstStride, int32_t iDstWidth,
3942; int32_t iDstHeight, uint8_t* pSrc, int32_t iSrcStride, uint32_t uiScaleX,
3943; uint32_t uiScaleY);
3944;
3945;**************************************************************************************************************
3946
3947WELS_EXTERN GeneralBilinearFastDownsampler_avx2
3948 %assign push_num 0
3949%ifndef X86_32
3950 push r12
3951 push r13
3952 push rbx
3953 push rbp
3954 %assign push_num 4
3955%ifdef WIN64
3956 push rdi
3957 push rsi
3958 %assign push_num push_num + 2
3959%endif
3960%endif
3961 LOAD_7_PARA
3962 PUSH_XMM 16
3963 SIGN_EXTENSION r1, r1d
3964 SIGN_EXTENSION r2, r2d
3965 SIGN_EXTENSION r3, r3d
3966 SIGN_EXTENSION r5, r5d
3967 ZERO_EXTENSION r6d
3968 sub r1, r2 ; dst_stride - dst_width
3969%ifdef X86_32
3970 vmovd xmm0, arg8
3971 vmovd xmm1, esp
3972 and esp, -32
3973 sub esp, 8 * 4 + 8 * 32
3974 vmovd [esp], xmm1
3975 %define p_dst r0
3976 %define i_dst_stride_less_width [esp + 1 * 4]
3977 %define i_dst_width [esp + 2 * 4]
3978 %define i_dst_height dword [esp + 3 * 4]
3979 %define p_src [esp + 4 * 4]
3980 %define i_src_stride [esp + 5 * 4]
3981 %define i_scalex r6
3982 %define i_scalexd r6d
3983 %define i_scaleyd [esp + 6 * 4]
3984 %define i_xpos r2
3985 %define i_ypos [esp + 7 * 4]
3986 %define i_yposd dword [esp + 7 * 4]
3987 %define p_src_row0 r3
3988 %define p_src_row1 r4
3989 %define i_width_cnt r5
3990 %define r_tmp0 r1
3991 %define r_tmp0b r1b
3992 %define ymm_xpos_frac ymm1
3993 %define ymm_xpos_frac_inc [esp + 8 * 4]
3994 %define ymm_xpos_int ymm3
3995 %define ymm_xpos_int_inc [esp + 8 * 4 + 1 * 32]
3996 %define ymm_yfrac0 [esp + 8 * 4 + 2 * 32]
3997 %define ymm_yfrac1 [esp + 8 * 4 + 3 * 32]
3998 %define xmm_tmp0 xmm7
3999 %define ymm_tmp0 ymm7
4000 %define xmm_tmp1 xmm0
4001 %define ymm_tmp1 ymm0
4002 %define xmm_tmp2 xmm2
4003 %define ymm_tmp2 ymm2
4004 %define xmm_tmp3 xmm4
4005 %define ymm_tmp3 ymm4
4006 %define xmm_tmp4 xmm5
4007 %define ymm_tmp4 ymm5
4008 %define xmm_tmp5 xmm6
4009 %define ymm_tmp5 ymm6
4010 %define ymm_0 [esp + 8 * 4 + 4 * 32]
4011 %define ymm_ffff [esp + 8 * 4 + 5 * 32]
4012 %define ymm_xpos_int_begin [esp + 8 * 4 + 6 * 32]
4013 %define ymm_xpos_frac_begin [esp + 8 * 4 + 7 * 32]
4014 mov i_dst_stride_less_width, r1
4015 mov i_dst_width, r2
4016 mov i_dst_height, r3
4017 mov p_src, r4
4018 mov i_src_stride, r5
4019 vmovd i_scaleyd, xmm0
4020 vpxor xmm0, xmm0, xmm0
4021 vmovdqa ymm_0, ymm0
4022 vpcmpeqw ymm_tmp0, ymm_tmp0, ymm_tmp0
4023 vmovdqa ymm_ffff, ymm_tmp0
4024%else
4025 %define p_dst r0
4026 %define i_dst_stride_less_width r1
4027 %define i_dst_width r2
4028 %define i_dst_height r3
4029 %define p_src r4
4030 %define i_src_stride r5
4031 %define i_scalex r6
4032 %define i_scalexd r6d
4033 %define i_scaleyd dword arg8d
4034 %define i_xpos r12
4035 %define i_ypos r13
4036 %define i_yposd r13d
4037 %define p_src_row0 rbp
4038%ifdef WIN64
4039 %define p_src_row1 rsi
4040 %define i_width_cnt rdi
4041%else
4042 %define p_src_row1 r11
4043 %define i_width_cnt rax
4044%endif
4045 %define r_tmp0 rbx
4046 %define r_tmp0b bl
4047 %define ymm_0 ymm0
4048 %define ymm_xpos_frac ymm1
4049 %define ymm_xpos_frac_inc ymm2
4050 %define ymm_xpos_int ymm3
4051 %define ymm_xpos_int_inc ymm4
4052 %define ymm_yfrac0 ymm5
4053 %define ymm_yfrac1 ymm6
4054 %define xmm_tmp0 xmm7
4055 %define ymm_tmp0 ymm7
4056 %define xmm_tmp1 xmm8
4057 %define ymm_tmp1 ymm8
4058 %define xmm_tmp2 xmm9
4059 %define ymm_tmp2 ymm9
4060 %define xmm_tmp3 xmm10
4061 %define ymm_tmp3 ymm10
4062 %define xmm_tmp4 xmm11
4063 %define ymm_tmp4 ymm11
4064 %define xmm_tmp5 xmm12
4065 %define ymm_tmp5 ymm12
4066 %define ymm_ffff ymm13
4067 %define ymm_xpos_int_begin ymm14
4068 %define ymm_xpos_frac_begin ymm15
4069 vpxor ymm_0, ymm_0, ymm_0
4070 vpcmpeqw ymm_ffff, ymm_ffff, ymm_ffff
4071%endif
4072
4073 sub i_dst_height, 1
4074 je .final_row
4075 jl .done
4076
4077 mov i_yposd, 1 << 14
4078 vmovd xmm_tmp0, i_scalexd
4079 vpbroadcastd ymm_tmp0, xmm_tmp0
4080 vpslld ymm_tmp1, ymm_tmp0, 2
4081 vpslld ymm_tmp2, ymm_tmp0, 3
4082 vpaddd ymm_tmp3, ymm_tmp1, ymm_tmp2
4083 vpxor ymm_tmp4, ymm_tmp4, ymm_tmp4
4084 vpblendd ymm_tmp1, ymm_tmp4, ymm_tmp1, 11110000b
4085 vpblendd ymm_tmp2, ymm_tmp2, ymm_tmp3, 11110000b
4086 vpaddd ymm_tmp3, ymm_tmp0, ymm_tmp0
4087 vpblendd ymm_tmp3, ymm_tmp4, ymm_tmp3, 11001100b
4088 vpblendd ymm_tmp0, ymm_tmp4, ymm_tmp0, 10101010b
4089 vpaddd ymm_tmp0, ymm_tmp3, ymm_tmp0
4090 vpaddd ymm_tmp1, ymm_tmp1, ymm_tmp0
4091 vpaddd ymm_tmp2, ymm_tmp2, ymm_tmp0
4092 vpcmpeqw ymm_tmp3, ymm_tmp3, ymm_tmp3
4093 vpsrld ymm_tmp3, ymm_tmp3, 31
4094 vpslld ymm_tmp3, ymm_tmp3, 15
4095 vpaddd ymm_tmp1, ymm_tmp1, ymm_tmp3
4096 vpaddd ymm_tmp2, ymm_tmp2, ymm_tmp3
4097 vpsrld ymm_xpos_int, ymm_tmp1, 16
4098 vpsrld ymm_tmp0, ymm_tmp2, 16
4099 vpackssdw ymm_xpos_int, ymm_xpos_int, ymm_tmp0
4100 vpermq ymm_xpos_int, ymm_xpos_int, 11011000b
4101 vpackuswb ymm_xpos_int, ymm_xpos_int, ymm_xpos_int
4102 vpcmpeqw ymm_tmp3, ymm_tmp3, ymm_tmp3
4103 vpsubb ymm_tmp0, ymm_xpos_int, ymm_tmp3
4104 vpunpcklbw ymm_xpos_int, ymm_xpos_int, ymm_tmp0
4105 vpslld ymm_tmp1, ymm_tmp1, 16
4106 vpsrld ymm_tmp1, ymm_tmp1, 16
4107 vpslld ymm_tmp2, ymm_tmp2, 16
4108 vpsrld ymm_tmp2, ymm_tmp2, 16
4109 vpackusdw ymm_xpos_frac, ymm_tmp1, ymm_tmp2
4110 vpermq ymm_xpos_frac, ymm_xpos_frac, 11011000b
4111 vmovd xmm_tmp0, i_scalexd
4112 vpslld xmm_tmp0, xmm_tmp0, 4
4113 vpbroadcastw ymm_tmp1, xmm_tmp0
4114 vmovdqa ymm_xpos_frac_inc, ymm_tmp1
4115 vpsrld xmm_tmp0, xmm_tmp0, 16
4116 vpsubw ymm_tmp0, ymm_tmp0, ymm_tmp3
4117 vpbroadcastb ymm_tmp0, xmm_tmp0
4118 vmovdqa ymm_xpos_int_inc, ymm_tmp0
4119 vmovdqa ymm_xpos_int_begin, ymm_xpos_int
4120 vmovdqa ymm_xpos_frac_begin, ymm_xpos_frac
4121
4122 cmp i_scalex, 4 << 16
4123 ja .scalex_above4
4124 cmp i_scalex, 2 << 16
4125 ja .scalex_above2_beloweq4
4126 add i_scalex, i_scalex
4127%xdefine i_scalex2 i_scalex
4128%undef i_scalex
4129 AVX2_GeneralBilinearDownsampler_loop AVX2_BilinearFastDownsample2xOrLess_16px, 1
4130 shr i_scalex2, 1
4131%xdefine i_scalex i_scalex2
4132%undef i_scalex2
4133 jmp .final_row
4134.scalex_above2_beloweq4:
4135 add i_scalex, i_scalex
4136%xdefine i_scalex2 i_scalex
4137%undef i_scalex
4138 AVX2_GeneralBilinearDownsampler_loop AVX2_BilinearFastDownsample4xOrLess_16px, 1
4139 shr i_scalex2, 1
4140%xdefine i_scalex i_scalex2
4141%undef i_scalex2
4142 jmp .final_row
4143.scalex_above4:
4144 cmp i_scalex, 8 << 16
4145 ja .scalex_above8
4146 add i_scalex, i_scalex
4147%xdefine i_scalex2 i_scalex
4148%undef i_scalex
4149 AVX2_GeneralBilinearDownsampler_loop AVX2_BilinearFastDownsample8xOrLess_16px, 1
4150 shr i_scalex2, 1
4151%xdefine i_scalex i_scalex2
4152%undef i_scalex2
4153 jmp .final_row
4154.scalex_above8:
4155%xdefine ymm_xfrac0 ymm_xpos_frac
4156%xdefine ymm_xfrac1 ymm_xpos_int
4157%xdefine ymm_xfrac0_begin ymm_xpos_int_begin
4158%xdefine ymm_xfrac1_begin ymm_xpos_frac_begin
4159%xdefine ymm_xfrac_inc ymm_xpos_frac_inc
4160%undef ymm_xpos_int
4161%undef ymm_xpos_frac
4162%undef ymm_xpos_int_begin
4163%undef ymm_xpos_frac_begin
4164%undef ymm_xpos_int_inc
4165%undef ymm_xpos_frac_inc
4166 AVX2_UnpckXFrac ymm_tmp0, ymm_xfrac1, ymm_xfrac0, ymm_ffff
4167 vpermq ymm_xfrac0, ymm_tmp0, 01001110b
4168 vpermq ymm_xfrac1, ymm_xfrac1, 01001110b
4169 vmovdqa ymm_xfrac0_begin, ymm_xfrac0
4170 vmovdqa ymm_xfrac1_begin, ymm_xfrac1
4171 vpcmpeqw ymm_tmp0, ymm_tmp0, ymm_tmp0
4172 vpmullw ymm_tmp0, ymm_tmp0, ymm_xfrac_inc
4173 vpunpcklwd ymm_tmp0, ymm_tmp0, ymm_xfrac_inc
4174 vmovdqa ymm_xfrac_inc, ymm_tmp0
4175 AVX2_GeneralBilinearDownsampler_loop AVX2_GeneralBilinearFastDownsample_16px, 1
4176
4177.final_row:
4178 mov p_src_row0, i_ypos
4179 shr p_src_row0, 15
4180 imul p_src_row0, i_src_stride
4181 add p_src_row0, p_src
4182 mov i_xpos, 1 << 15
4183 mov i_width_cnt, i_dst_width
4184
4185.final_row_width:
4186 mov r_tmp0, i_xpos
4187 shr r_tmp0, 16
4188 movzx r_tmp0, byte [p_src_row0 + r_tmp0]
4189 mov [p_dst], r_tmp0b
4190 add p_dst, 1
4191 add i_xpos, i_scalex
4192 sub i_width_cnt, 1
4193 jg .final_row_width
4194
4195.done:
4196 vzeroupper
4197%ifdef X86_32
4198 mov esp, [esp]
4199%endif
4200 POP_XMM
4201 LOAD_7_PARA_POP
4202%ifndef X86_32
4203%ifdef WIN64
4204 pop rsi
4205 pop rdi
4206%endif
4207 pop rbp
4208 pop rbx
4209 pop r13
4210 pop r12
4211%endif
4212 ret
4213%undef p_dst
4214%undef i_dst_stride_less_width
4215%undef i_dst_width
4216%undef i_dst_height
4217%undef p_src
4218%undef i_src_stride
4219%undef i_scalex
4220%undef i_scalexd
4221%undef i_scaleyd
4222%undef i_xpos
4223%undef i_ypos
4224%undef i_yposd
4225%undef p_src_row0
4226%undef p_src_row1
4227%undef i_width_cnt
4228%undef r_tmp0
4229%undef r_tmp0b
4230%undef ymm_xpos_frac
4231%undef ymm_xpos_frac_inc
4232%undef ymm_xpos_int
4233%undef ymm_xpos_int_inc
4234%undef ymm_yfrac0
4235%undef ymm_yfrac1
4236%undef xmm_tmp0
4237%undef ymm_tmp0
4238%undef xmm_tmp1
4239%undef ymm_tmp1
4240%undef xmm_tmp2
4241%undef ymm_tmp2
4242%undef xmm_tmp3
4243%undef ymm_tmp3
4244%undef xmm_tmp4
4245%undef ymm_tmp4
4246%undef xmm_tmp5
4247%undef ymm_tmp5
4248%undef ymm_ffff
4249%undef ymm_0
4250%undef ymm_xpos_int_begin
4251%undef ymm_xpos_frac_begin
4252%undef ymm_xfrac0
4253%undef ymm_xfrac1
4254%undef ymm_xfrac0_begin
4255%undef ymm_xfrac1_begin
4256%undef ymm_xfrac_inc

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t