History

[Processing/x86] Add an SSSE3 implementation of GeneralBilinearFastDownsample

Keep track of relative pixel offsets and utilize pshufb to efficiently extract relevant pixels for horizontal scaling ratios <= 4. Fall back to a generic approach for ratios > 4. Note that the generic approach can be backported to SSE2. The implementation assumes that data beyond the end of each line, before the next line begins, can be dirtied; which AFAICT is safe with the current usage of these routines. Speedup is ~6.67x/~3.26x (32-bit/64-bit) for horizontal ratios <= 2, ~6.24x/~3.00x for ratios within (2, 4], and ~4.89x/~2.17x for ratios > 4 when not memory-bound on Haswell as compared with the current SSE2 implementation.

Sindre Aamås committed 10 years ago

Commit 1995e031 parent cbaf0876 files · +535 −0Browse files

Showing the changes to codec/processing/src (4 of 6 files). Show all

Unified · Split

4 files
  1. codec/processing/src/downsample/downsample.cpp +1 −0
  2. codec/processing/src/downsample/downsample.h +4 −0
  3. codec/processing/src/downsample/downsamplefuncs.cpp +1 −0
  4. codec/processing/src/x86/downsample_bilinear.asm +521 −0
codec/processing/src/downsample/downsample.cpp+1−0
@@ -100,6 +100,7 @@ void CDownsampling::InitDownsampleFuncs (SDownsampleFuncs& sDownsampleFunc, int · 99 unmodified lines
100 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsamplerWidthx16_ssse3;100 sDownsampleFunc.pfHalfAverage[1] = DyadicBilinearDownsamplerWidthx16_ssse3;
101 sDownsampleFunc.pfOneThirdDownsampler = DyadicBilinearOneThirdDownsampler_ssse3;101 sDownsampleFunc.pfOneThirdDownsampler = DyadicBilinearOneThirdDownsampler_ssse3;
102 sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_ssse3;102 sDownsampleFunc.pfQuarterDownsampler = DyadicBilinearQuarterDownsampler_ssse3;
103 sDownsampleFunc.pfGeneralRatioLuma = GeneralBilinearFastDownsamplerWrap_ssse3;
103 }104 }
104 if (iCpuFlag & WELS_CPU_SSE41) {105 if (iCpuFlag & WELS_CPU_SSE41) {
105 sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsamplerWidthx32_sse4;106 sDownsampleFunc.pfHalfAverage[0] = DyadicBilinearDownsamplerWidthx32_sse4;
codec/processing/src/downsample/downsample.h+4−0
@@ -101,6 +101,7 @@ HalveDownsampleFunc DyadicBilinearDownsamplerWidthx32_sse4; · 100 unmodified lines
101 101
102GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_sse2;102GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_sse2;
103GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse2;103GeneralDownsampleFunc GeneralBilinearAccurateDownsamplerWrap_sse2;
104GeneralDownsampleFunc GeneralBilinearFastDownsamplerWrap_ssse3;
104 105
105SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_ssse3;106SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_ssse3;
106SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_sse4;107SpecificDownsampleFunc DyadicBilinearOneThirdDownsampler_sse4;
@@ -114,6 +115,9 @@ void GeneralBilinearFastDownsampler_sse2 (uint8_t* pDst, const int32_t kiDstStri · 7 unmodified lines
114void GeneralBilinearAccurateDownsampler_sse2 (uint8_t* pDst, const int32_t kiDstStride, const int32_t kiDstWidth,115void GeneralBilinearAccurateDownsampler_sse2 (uint8_t* pDst, const int32_t kiDstStride, const int32_t kiDstWidth,
115 const int32_t kiDstHeight, uint8_t* pSrc, const int32_t kiSrcStride, const uint32_t kuiScaleX,116 const int32_t kiDstHeight, uint8_t* pSrc, const int32_t kiSrcStride, const uint32_t kuiScaleX,
116 const uint32_t kuiScaleY);117 const uint32_t kuiScaleY);
118void GeneralBilinearFastDownsampler_ssse3 (uint8_t* pDst, int32_t iDstStride, int32_t iDstWidth,
119 int32_t iDstHeight, uint8_t* pSrc, int32_t iSrcStride, uint32_t uiScaleX,
120 uint32_t uiScaleY);
117 121
118WELSVP_EXTERN_C_END122WELSVP_EXTERN_C_END
119#endif123#endif
codec/processing/src/downsample/downsamplefuncs.cpp+1−0
@@ -282,6 +282,7 @@ static void GeneralBilinearDownsamplerWrap (uint8_t* pDst, const int32_t kiDstSt · 281 unmodified lines
282#ifdef X86_ASM282#ifdef X86_ASM
283DEFINE_GENERAL_BILINEAR_FAST_DOWNSAMPLER_WRAP (sse2)283DEFINE_GENERAL_BILINEAR_FAST_DOWNSAMPLER_WRAP (sse2)
284DEFINE_GENERAL_BILINEAR_ACCURATE_DOWNSAMPLER_WRAP (sse2)284DEFINE_GENERAL_BILINEAR_ACCURATE_DOWNSAMPLER_WRAP (sse2)
285DEFINE_GENERAL_BILINEAR_FAST_DOWNSAMPLER_WRAP (ssse3)
285#endif //X86_ASM286#endif //X86_ASM
286 287
287#ifdef HAVE_NEON288#ifdef HAVE_NEON
codec/processing/src/x86/downsample_bilinear.asm+521−0
@@ -60,6 +60,16 @@ SECTION .rodata align=16 · 59 unmodified lines
60;***********************************************************************60;***********************************************************************
61 61
62ALIGN 1662ALIGN 16
63db80h_128:
64 times 16 db 80h
65shufb_0000000088888888:
66 times 8 db 0
67 times 8 db 8
68shufb_000044448888CCCC:
69 times 4 db 0
70 times 4 db 4
71 times 4 db 8
72 times 4 db 12
63shufb_mask_low:73shufb_mask_low:
64 db 00h, 80h, 02h, 80h, 04h, 80h, 06h, 80h, 08h, 80h, 0ah, 80h, 0ch, 80h, 0eh, 80h74 db 00h, 80h, 02h, 80h, 04h, 80h, 06h, 80h, 08h, 80h, 0ah, 80h, 0ch, 80h, 0eh, 80h
65shufb_mask_high:75shufb_mask_high:
@@ -2595,3 +2605,514 @@ WELS_EXTERN DyadicBilinearQuarterDownsampler_sse4 · 2529 unmodified lines
2595%endif2605%endif
2596 ret2606 ret
2597 2607
2608; xpos_int=%1 xpos_frac=%2 inc_int+1=%3 inc_frac=%4 tmp=%5
2609%macro SSE2_BilinearIncXposuw 5
2610 movdqa %5, %2
2611 paddw %2, %4
2612 paddusw %5, %4
2613 pcmpeqw %5, %2
2614 paddb %1, %3
2615 paddb %1, %5 ; subtract 1 if no carry
2616%endmacro
2617
2618; outl=%1 outh=%2 in=%3
2619%macro SSE2_UnpckXFracuw 3
2620 pcmpeqw %1, %1
2621 pxor %1, %3
2622 movdqa %2, %1
2623 punpcklwd %1, %3
2624 punpckhwd %2, %3
2625%endmacro
2626
2627; [in:xfrac out:xyfrac0]=%1 [out:xyfrac1]=%2 yfrac0=%3 yfrac1=%4
2628%macro SSE2_BilinearFastCalcXYFrac 4
2629 movdqa %2, %1
2630 pmulhuw %1, %3
2631 pmulhuw %2, %4
2632%endmacro
2633
2634; [in:dwordsl out:bytes] dwordsh=%2 zero=%3
2635%macro SSE2_BilinearFastPackDwordsToBytes 3
2636 psrld %1, 14
2637 psrld %2, 14
2638 packssdw %1, %2
2639 pavgw %1, %3
2640 packuswb %1, %1
2641%endmacro
2642
2643%macro SSSE3_BilinearFastDownsample2xOrLess_8px 0
2644 movdqa xmm_tmp0, xmm_xpos_int
2645 pshufb xmm_tmp0, xmm_0
2646 psubb xmm_xpos_int, xmm_tmp0
2647 SSE2_UnpckXFracuw xmm_tmp0, xmm_tmp1, xmm_xpos_frac
2648 mov r_tmp0, i_xpos
2649 lea i_xpos, [i_xpos + 8 * i_scalex]
2650 shr r_tmp0, 16
2651 movdqu xmm_tmp4, [p_src_row0 + r_tmp0]
2652 pshufb xmm_tmp4, xmm_xpos_int
2653 movdqa xmm_tmp5, xmm_tmp4
2654 punpcklbw xmm_tmp4, xmm_0
2655 punpckhbw xmm_tmp5, xmm_0
2656 SSE2_BilinearFastCalcXYFrac xmm_tmp0, xmm_tmp2, xmm_yfrac0, xmm_yfrac1
2657 SSE2_BilinearFastCalcXYFrac xmm_tmp1, xmm_tmp3, xmm_yfrac0, xmm_yfrac1
2658 pmaddwd xmm_tmp0, xmm_tmp4
2659 pmaddwd xmm_tmp1, xmm_tmp5
2660 movdqu xmm_tmp4, [p_src_row1 + r_tmp0]
2661 pshufb xmm_tmp4, xmm_xpos_int
2662 movdqa xmm_tmp5, xmm_tmp4
2663 punpcklbw xmm_tmp4, xmm_0
2664 punpckhbw xmm_tmp5, xmm_0
2665 pmaddwd xmm_tmp2, xmm_tmp4
2666 pmaddwd xmm_tmp3, xmm_tmp5
2667 paddd xmm_tmp0, xmm_tmp2
2668 paddd xmm_tmp1, xmm_tmp3
2669 SSE2_BilinearFastPackDwordsToBytes xmm_tmp0, xmm_tmp1, xmm_0
2670 movlps [p_dst], xmm_tmp0
2671 add p_dst, 8
2672 SSE2_BilinearIncXposuw xmm_xpos_int, xmm_xpos_frac, xmm_xpos_int_inc, xmm_xpos_frac_inc, xmm_tmp0
2673%endmacro
2674
2675%macro SSSE3_BilinearFastDownsample4xOrLess_8px 0
2676 movdqa xmm_tmp0, xmm_xpos_int
2677 pshufb xmm_tmp0, [shufb_0000000088888888]
2678 psubb xmm_xpos_int, xmm_tmp0
2679 SSE2_UnpckXFracuw xmm_tmp0, xmm_tmp1, xmm_xpos_frac
2680 mov r_tmp0, i_xpos
2681 shr r_tmp0, 16
2682 movdqu xmm_tmp3, [p_src_row0 + r_tmp0]
2683 movdqu xmm_tmp4, [p_src_row1 + r_tmp0]
2684 movdqa xmm_tmp2, xmm_xpos_int
2685 punpcklbw xmm_tmp2, [db80h_128]
2686 pshufb xmm_tmp3, xmm_tmp2
2687 pshufb xmm_tmp4, xmm_tmp2
2688 SSE2_BilinearFastCalcXYFrac xmm_tmp0, xmm_tmp2, xmm_yfrac0, xmm_yfrac1
2689 pmaddwd xmm_tmp0, xmm_tmp3
2690 pmaddwd xmm_tmp2, xmm_tmp4
2691 paddd xmm_tmp0, xmm_tmp2
2692 lea r_tmp0, [i_xpos + 4 * i_scalex]
2693 lea i_xpos, [i_xpos + 8 * i_scalex]
2694 shr r_tmp0, 16
2695 movdqu xmm_tmp3, [p_src_row0 + r_tmp0]
2696 movdqu xmm_tmp4, [p_src_row1 + r_tmp0]
2697 movdqa xmm_tmp2, xmm_xpos_int
2698 punpckhbw xmm_tmp2, [db80h_128]
2699 pshufb xmm_tmp3, xmm_tmp2
2700 pshufb xmm_tmp4, xmm_tmp2
2701 SSE2_BilinearFastCalcXYFrac xmm_tmp1, xmm_tmp2, xmm_yfrac0, xmm_yfrac1
2702 pmaddwd xmm_tmp1, xmm_tmp3
2703 pmaddwd xmm_tmp2, xmm_tmp4
2704 paddd xmm_tmp1, xmm_tmp2
2705 SSE2_BilinearFastPackDwordsToBytes xmm_tmp0, xmm_tmp1, xmm_0
2706 movlps [p_dst], xmm_tmp0
2707 add p_dst, 8
2708 SSE2_BilinearIncXposuw xmm_xpos_int, xmm_xpos_frac, xmm_xpos_int_inc, xmm_xpos_frac_inc, xmm_tmp0
2709%endmacro
2710
2711%macro SSE2_GeneralBilinearFastDownsample_8px 0
2712 mov r_tmp0, i_xpos
2713 shr r_tmp0, 16
2714 movd xmm_tmp3, [p_src_row0 + r_tmp0]
2715 movd xmm_tmp4, [p_src_row1 + r_tmp0]
2716 lea r_tmp0, [i_xpos + i_scalex]
2717 shr r_tmp0, 16
2718 pinsrw xmm_tmp3, [p_src_row0 + r_tmp0], 1
2719 pinsrw xmm_tmp4, [p_src_row1 + r_tmp0], 1
2720 lea r_tmp0, [i_xpos + 2 * i_scalex]
2721 lea i_xpos, [i_xpos + 4 * i_scalex]
2722 shr r_tmp0, 16
2723 pinsrw xmm_tmp3, [p_src_row0 + r_tmp0], 2
2724 pinsrw xmm_tmp4, [p_src_row1 + r_tmp0], 2
2725 mov r_tmp0, i_xpos
2726 sub r_tmp0, i_scalex
2727 shr r_tmp0, 16
2728 pinsrw xmm_tmp3, [p_src_row0 + r_tmp0], 3
2729 pinsrw xmm_tmp4, [p_src_row1 + r_tmp0], 3
2730 punpcklbw xmm_tmp3, xmm_0
2731 punpcklbw xmm_tmp4, xmm_0
2732 movdqa xmm_tmp0, xmm_xfrac0
2733 SSE2_BilinearFastCalcXYFrac xmm_tmp0, xmm_tmp2, xmm_yfrac0, xmm_yfrac1
2734 pmaddwd xmm_tmp0, xmm_tmp3
2735 pmaddwd xmm_tmp2, xmm_tmp4
2736 paddd xmm_tmp0, xmm_tmp2
2737 mov r_tmp0, i_xpos
2738 shr r_tmp0, 16
2739 movd xmm_tmp3, [p_src_row0 + r_tmp0]
2740 movd xmm_tmp4, [p_src_row1 + r_tmp0]
2741 lea r_tmp0, [i_xpos + i_scalex]
2742 shr r_tmp0, 16
2743 pinsrw xmm_tmp3, [p_src_row0 + r_tmp0], 1
2744 pinsrw xmm_tmp4, [p_src_row1 + r_tmp0], 1
2745 lea r_tmp0, [i_xpos + 2 * i_scalex]
2746 lea i_xpos, [i_xpos + 4 * i_scalex]
2747 shr r_tmp0, 16
2748 pinsrw xmm_tmp3, [p_src_row0 + r_tmp0], 2
2749 pinsrw xmm_tmp4, [p_src_row1 + r_tmp0], 2
2750 mov r_tmp0, i_xpos
2751 sub r_tmp0, i_scalex
2752 shr r_tmp0, 16
2753 pinsrw xmm_tmp3, [p_src_row0 + r_tmp0], 3
2754 pinsrw xmm_tmp4, [p_src_row1 + r_tmp0], 3
2755 punpcklbw xmm_tmp3, xmm_0
2756 punpcklbw xmm_tmp4, xmm_0
2757 movdqa xmm_tmp1, xmm_xfrac1
2758 SSE2_BilinearFastCalcXYFrac xmm_tmp1, xmm_tmp2, xmm_yfrac0, xmm_yfrac1
2759 pmaddwd xmm_tmp1, xmm_tmp3
2760 pmaddwd xmm_tmp2, xmm_tmp4
2761 paddd xmm_tmp1, xmm_tmp2
2762 SSE2_BilinearFastPackDwordsToBytes xmm_tmp0, xmm_tmp1, xmm_0
2763 movlps [p_dst], xmm_tmp0
2764 add p_dst, 8
2765 paddw xmm_xfrac0, xmm_xfrac_inc
2766 paddw xmm_xfrac1, xmm_xfrac_inc
2767%endmacro
2768
2769; downsample_8px_macro=%1 b_fast=%2
2770%macro SSE2_GeneralBilinearDownsampler_loop 2
2771%%height:
2772 mov p_src_row0, i_ypos
2773 shr p_src_row0, 15
2774 imul p_src_row0, i_src_stride
2775 add p_src_row0, p_src
2776 mov p_src_row1, p_src_row0
2777 add p_src_row1, i_src_stride
2778 movd xmm_tmp1, i_yposd
2779%if %2
2780 pshuflw xmm_tmp1, xmm_tmp1, 0
2781 psllw xmm_tmp1, 1
2782 psrlw xmm_tmp1, 1
2783%else
2784 pslld xmm_tmp1, 17
2785 psrld xmm_tmp1, 17
2786%endif
2787%ifdef X86_32
2788 pshufd xmm_tmp1, xmm_tmp1, 0
2789 pcmpeqw xmm_tmp0, xmm_tmp0
2790%if %2
2791 psrlw xmm_tmp0, 1
2792%else
2793 psrld xmm_tmp0, 17
2794%endif
2795 pxor xmm_tmp0, xmm_tmp1
2796 movdqa xmm_yfrac0, xmm_tmp0
2797 movdqa xmm_yfrac1, xmm_tmp1
2798%else
2799 pshufd xmm_yfrac1, xmm_tmp1, 0
2800 pcmpeqw xmm_yfrac0, xmm_yfrac0
2801%if %2
2802 psrlw xmm_yfrac0, 1
2803%else
2804 psrld xmm_yfrac0, 17
2805%endif
2806 pxor xmm_yfrac0, xmm_yfrac1
2807%endif
2808
2809 mov i_xpos, 1 << 15
2810 mov i_width_cnt, i_dst_width
2811 sub i_width_cnt, 1
2812
2813%ifdef xmm_xpos_int
2814 movdqa xmm_xpos_int, xmm_xpos_int_begin
2815 movdqa xmm_xpos_frac, xmm_xpos_frac_begin
2816%else
2817 movdqa xmm_xfrac0, xmm_xfrac0_begin
2818 movdqa xmm_xfrac1, xmm_xfrac1_begin
2819%endif
2820
2821%%width:
2822 %1
2823 sub i_width_cnt, 8
2824 jg %%width
2825
2826 lea p_dst, [p_dst + i_width_cnt + 1]
2827 imul i_width_cnt, i_scalex
2828 add i_xpos, i_width_cnt
2829 shr i_xpos, 16
2830 movzx r_tmp0, byte [p_src_row0 + i_xpos]
2831 mov [p_dst - 1], r_tmp0b
2832%ifdef X86_32
2833 mov r_tmp0, i_scaleyd
2834 add i_yposd, r_tmp0
2835%else
2836 add i_yposd, i_scaleyd
2837%endif
2838 add p_dst, i_dst_stride_less_width
2839 sub i_dst_height, 1
2840 jg %%height
2841%endmacro
2842
2843;**************************************************************************************************************
2844;void GeneralBilinearFastDownsampler_ssse3 (uint8_t* pDst, int32_t iDstStride, int32_t iDstWidth,
2845; int32_t iDstHeight, uint8_t* pSrc, int32_t iSrcStride, uint32_t uiScaleX,
2846; uint32_t uiScaleY);
2847;
2848;**************************************************************************************************************
2849
2850WELS_EXTERN GeneralBilinearFastDownsampler_ssse3
2851 %assign push_num 0
2852%ifndef X86_32
2853 push r12
2854 push r13
2855 push rbx
2856 push rbp
2857 %assign push_num 4
2858%ifdef WIN64
2859 push rdi
2860 push rsi
2861 %assign push_num push_num + 2
2862%endif
2863%endif
2864 LOAD_7_PARA
2865 PUSH_XMM 16
2866 SIGN_EXTENSION r1, r1d
2867 SIGN_EXTENSION r2, r2d
2868 SIGN_EXTENSION r3, r3d
2869 SIGN_EXTENSION r5, r5d
2870 ZERO_EXTENSION r6d
2871 sub r1, r2 ; dst_stride - dst_width
2872%ifdef X86_32
2873 movd xmm0, arg8
2874 movd xmm1, esp
2875 and esp, -16
2876 sub esp, 8 * 4 + 7 * 16
2877 movd [esp], xmm1
2878 %define p_dst r0
2879 %define i_dst_stride_less_width [esp + 1 * 4]
2880 %define i_dst_width [esp + 2 * 4]
2881 %define i_dst_height dword [esp + 3 * 4]
2882 %define p_src [esp + 4 * 4]
2883 %define i_src_stride [esp + 5 * 4]
2884 %define i_scalex r6
2885 %define i_scalexd r6d
2886 %define i_scaleyd [esp + 6 * 4]
2887 %define i_xpos r2
2888 %define i_ypos dword [esp + 7 * 4]
2889 %define i_yposd dword [esp + 7 * 4]
2890 %define p_src_row0 r3
2891 %define p_src_row1 r4
2892 %define i_width_cnt r5
2893 %define r_tmp0 r1
2894 %define r_tmp0b r1b
2895 %define xmm_xpos_frac xmm1
2896 %define xmm_xpos_frac_inc [esp + 8 * 4]
2897 %define xmm_xpos_int xmm3
2898 %define xmm_xpos_int_inc [esp + 8 * 4 + 1 * 16]
2899 %define xmm_yfrac0 [esp + 8 * 4 + 2 * 16]
2900 %define xmm_yfrac1 [esp + 8 * 4 + 3 * 16]
2901 %define xmm_tmp0 xmm7
2902 %define xmm_tmp1 xmm0
2903 %define xmm_tmp2 xmm2
2904 %define xmm_tmp3 xmm4
2905 %define xmm_tmp4 xmm5
2906 %define xmm_tmp5 xmm6
2907 %define xmm_0 [esp + 8 * 4 + 4 * 16]
2908 %define xmm_xpos_int_begin [esp + 8 * 4 + 5 * 16]
2909 %define xmm_xpos_frac_begin [esp + 8 * 4 + 6 * 16]
2910 mov i_dst_stride_less_width, r1
2911 mov i_dst_width, r2
2912 mov i_dst_height, r3
2913 mov p_src, r4
2914 mov i_src_stride, r5
2915 movd i_scaleyd, xmm0
2916 pxor xmm_tmp0, xmm_tmp0
2917 movdqa xmm_0, xmm_tmp0
2918%else
2919 %define p_dst r0
2920 %define i_dst_stride_less_width r1
2921 %define i_dst_width r2
2922 %define i_dst_height r3
2923 %define p_src r4
2924 %define i_src_stride r5
2925 %define i_scalex r6
2926 %define i_scalexd r6d
2927 %define i_scaleyd dword arg8d
2928 %define i_xpos r12
2929 %define i_ypos r13
2930 %define i_yposd r13d
2931 %define p_src_row0 rbp
2932%ifdef WIN64
2933 %define p_src_row1 rsi
2934 %define i_width_cnt rdi
2935%else
2936 %define p_src_row1 r11
2937 %define i_width_cnt rax
2938%endif
2939 %define r_tmp0 rbx
2940 %define r_tmp0b bl
2941 %define xmm_0 xmm0
2942 %define xmm_xpos_frac xmm1
2943 %define xmm_xpos_frac_inc xmm8
2944 %define xmm_xpos_int xmm3
2945 %define xmm_xpos_int_inc xmm10
2946 %define xmm_yfrac0 xmm11
2947 %define xmm_yfrac1 xmm12
2948 %define xmm_tmp0 xmm7
2949 %define xmm_tmp1 xmm2
2950 %define xmm_tmp2 xmm9
2951 %define xmm_tmp3 xmm4
2952 %define xmm_tmp4 xmm5
2953 %define xmm_tmp5 xmm6
2954 %define xmm_xpos_int_begin xmm14
2955 %define xmm_xpos_frac_begin xmm15
2956 pxor xmm_0, xmm_0
2957%endif
2958
2959 sub i_dst_height, 1
2960 je .final_row
2961 jl .done
2962
2963 mov i_ypos, 1 << 14
2964 movd xmm_xpos_frac, i_scalexd
2965 pshufd xmm_xpos_frac, xmm_xpos_frac, 0
2966 movdqa xmm_tmp0, xmm_xpos_frac
2967 pslld xmm_tmp0, 2
2968 pslldq xmm_xpos_frac, 4
2969 paddd xmm_tmp0, xmm_xpos_frac
2970 movdqa xmm_tmp1, xmm_xpos_frac
2971 pslldq xmm_tmp1, 4
2972 paddd xmm_xpos_frac, xmm_tmp1
2973 paddd xmm_tmp0, xmm_tmp1
2974 pslldq xmm_tmp1, 4
2975 paddd xmm_xpos_frac, xmm_tmp1
2976 paddd xmm_tmp0, xmm_tmp1
2977 pcmpeqw xmm_tmp1, xmm_tmp1
2978 psrld xmm_tmp1, 31
2979 pslld xmm_tmp1, 15
2980 paddd xmm_xpos_frac, xmm_tmp1
2981 paddd xmm_tmp0, xmm_tmp1
2982 movdqa xmm_xpos_int, xmm_xpos_frac
2983 movdqa xmm_tmp1, xmm_tmp0
2984 psrld xmm_xpos_int, 16
2985 psrld xmm_tmp1, 16
2986 packssdw xmm_xpos_int, xmm_tmp1
2987 packuswb xmm_xpos_int, xmm_xpos_int
2988 movdqa xmm_tmp1, xmm_xpos_int
2989 pcmpeqw xmm_tmp2, xmm_tmp2
2990 psubb xmm_tmp1, xmm_tmp2
2991 punpcklbw xmm_xpos_int, xmm_tmp1
2992 pslld xmm_xpos_frac, 16
2993 pslld xmm_tmp0, 16
2994 psrad xmm_xpos_frac, 16
2995 psrad xmm_tmp0, 16
2996 packssdw xmm_xpos_frac, xmm_tmp0
2997 movd xmm_tmp0, i_scalexd
2998 pslld xmm_tmp0, 3
2999 movdqa xmm_tmp1, xmm_tmp0
3000 punpcklwd xmm_tmp0, xmm_tmp0
3001 pshufd xmm_tmp0, xmm_tmp0, 0
3002 movdqa xmm_xpos_frac_inc, xmm_tmp0
3003 psrld xmm_tmp1, 16
3004 psubw xmm_tmp1, xmm_tmp2
3005 pxor xmm_tmp2, xmm_tmp2
3006 pshufb xmm_tmp1, xmm_tmp2
3007 movdqa xmm_xpos_int_inc, xmm_tmp1
3008 movdqa xmm_xpos_int_begin, xmm_xpos_int
3009 movdqa xmm_xpos_frac_begin, xmm_xpos_frac
3010
3011 cmp i_scalex, 4 << 16
3012 ja .scalex_above4
3013 cmp i_scalex, 2 << 16
3014 ja .scalex_above2_beloweq4
3015 SSE2_GeneralBilinearDownsampler_loop SSSE3_BilinearFastDownsample2xOrLess_8px, 1
3016 jmp .final_row
3017%ifdef X86_32
3018 %undef xmm_yfrac0
3019 %xdefine xmm_yfrac0 xmm_tmp5
3020 %undef xmm_tmp5
3021%endif
3022.scalex_above2_beloweq4:
3023 SSE2_GeneralBilinearDownsampler_loop SSSE3_BilinearFastDownsample4xOrLess_8px, 1
3024 jmp .final_row
3025.scalex_above4:
3026%xdefine xmm_xfrac0 xmm_xpos_frac
3027%xdefine xmm_xfrac1 xmm_xpos_int
3028%xdefine xmm_xfrac0_begin xmm_xpos_int_begin
3029%xdefine xmm_xfrac1_begin xmm_xpos_frac_begin
3030%xdefine xmm_xfrac_inc xmm_xpos_frac_inc
3031%undef xmm_xpos_int
3032%undef xmm_xpos_frac
3033%undef xmm_xpos_int_begin
3034%undef xmm_xpos_frac_begin
3035%undef xmm_xpos_int_inc
3036%undef xmm_xpos_frac_inc
3037 SSE2_UnpckXFracuw xmm_tmp0, xmm_xfrac1, xmm_xfrac0
3038 movdqa xmm_xfrac0, xmm_tmp0
3039 movdqa xmm_xfrac0_begin, xmm_xfrac0
3040 movdqa xmm_xfrac1_begin, xmm_xfrac1
3041 pcmpeqw xmm_tmp0, xmm_tmp0
3042 pmullw xmm_tmp0, xmm_xfrac_inc
3043 punpcklwd xmm_tmp0, xmm_xfrac_inc
3044 movdqa xmm_xfrac_inc, xmm_tmp0
3045 SSE2_GeneralBilinearDownsampler_loop SSE2_GeneralBilinearFastDownsample_8px, 1
3046
3047.final_row:
3048 mov p_src_row0, i_ypos
3049 shr p_src_row0, 15
3050 imul p_src_row0, i_src_stride
3051 add p_src_row0, p_src
3052 mov i_xpos, 1 << 15
3053 mov i_width_cnt, i_dst_width
3054
3055.final_row_width:
3056 mov r_tmp0, i_xpos
3057 shr r_tmp0, 16
3058 movzx r_tmp0, byte [p_src_row0 + r_tmp0]
3059 mov [p_dst], r_tmp0b
3060 add p_dst, 1
3061 add i_xpos, i_scalex
3062 sub i_width_cnt, 1
3063 jg .final_row_width
3064
3065.done:
3066%ifdef X86_32
3067 mov esp, [esp]
3068%endif
3069 POP_XMM
3070 LOAD_7_PARA_POP
3071%ifndef X86_32
3072%ifdef WIN64
3073 pop rsi
3074 pop rdi
3075%endif
3076 pop rbp
3077 pop rbx
3078 pop r13
3079 pop r12
3080%endif
3081 ret
3082%undef p_dst
3083%undef i_dst_stride_less_width
3084%undef i_dst_width
3085%undef i_dst_height
3086%undef p_src
3087%undef i_src_stride
3088%undef i_scalex
3089%undef i_scalexd
3090%undef i_scaleyd
3091%undef i_xpos
3092%undef i_ypos
3093%undef i_yposd
3094%undef p_src_row0
3095%undef p_src_row1
3096%undef i_width_cnt
3097%undef r_tmp0
3098%undef r_tmp0b
3099%undef xmm_0
3100%undef xmm_xpos_frac
3101%undef xmm_xpos_frac_inc
3102%undef xmm_xpos_int
3103%undef xmm_xpos_int_inc
3104%undef xmm_yfrac0
3105%undef xmm_yfrac1
3106%undef xmm_tmp0
3107%undef xmm_tmp1
3108%undef xmm_tmp2
3109%undef xmm_tmp3
3110%undef xmm_tmp4
3111%undef xmm_tmp5
3112%undef xmm_xpos_int_begin
3113%undef xmm_xpos_frac_begin
3114%undef xmm_xfrac0
3115%undef xmm_xfrac1
3116%undef xmm_xfrac0_begin
3117%undef xmm_xfrac1_begin
3118%undef xmm_xfrac_inc

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t