The resulting per-block speedup is measured to between ~1.67x and ~2.37x for horizontal filtering, between ~1.24x and ~1.53x for vertical filtering, and between ~1.33x and ~2.37x for 2-dimensional filtering on Haswell, depending on block size, as compared with the existing SSE2 routines. Outliers are 4-wide vertical and 2-dimensional filtering with a speedups of ~5.69x and ~6.84x respectively on Haswell (with clang-703.0.31) due to the C routines used previously.
Showing the changes to codec/common/src/mc.cpp (1 of 5 files). Show all
Unified · Split
| @@ -44,6 +44,8 @@ · 43 unmodified lines | ||
| 44 | 44 | #include "ls_defines.h" |
| 45 | 45 | #include "macros.h" |
| 46 | 46 | |
| 47 | +namespace { | |
| 48 | + | |
| 47 | 49 | typedef void (*PMcChromaWidthExtFunc) (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, |
| 48 | 50 | const uint8_t* kpABCD, int32_t iHeight); |
| 49 | 51 | typedef void (*PWelsSampleWidthAveragingFunc) (uint8_t*, int32_t, const uint8_t*, int32_t, const uint8_t*, |
| @@ -51,8 +53,6 @@ typedef void (*PWelsSampleWidthAveragingFunc) (uint8_t*, int32_t, const uint8_t* · 1 unmodified line | ||
| 51 | 53 | typedef void (*PWelsMcWidthHeightFunc) (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, |
| 52 | 54 | int32_t iWidth, int32_t iHeight); |
| 53 | 55 | |
| 54 | −namespace WelsCommon { | |
| 55 | − | |
| 56 | 56 | /*------------------weight for chroma fraction pixel interpolation------------------*/ |
| 57 | 57 | //iA = (8 - dx) * (8 - dy); |
| 58 | 58 | //iB = dx * (8 - dy); |
| @@ -710,6 +710,183 @@ void McChroma_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int3 · 651 unmodified lines | ||
| 710 | 710 | McChromaWithFragMv_c (pSrc, iSrcStride, pDst, iDstStride, iMvX, iMvY, iWidth, iHeight); |
| 711 | 711 | } |
| 712 | 712 | |
| 713 | +//***************************************************************************// | |
| 714 | +// SSSE3 implementation // | |
| 715 | +//***************************************************************************// | |
| 716 | + | |
| 717 | +void PixelAvgWidth4Or8Or16_sse2 (uint8_t* pDst, int32_t iDstStride, const uint8_t* pSrcA, int32_t iSrcAStride, | |
| 718 | + const uint8_t* pSrcB, int32_t iSrcBStride, int32_t iWidth, int32_t iHeight) { | |
| 719 | + if (iWidth < 8) { | |
| 720 | + PixelAvgWidthEq4_mmx (pDst, iDstStride, pSrcA, iSrcAStride, pSrcB, iSrcBStride, iHeight); | |
| 721 | + } else if (iWidth == 8) { | |
| 722 | + PixelAvgWidthEq8_mmx (pDst, iDstStride, pSrcA, iSrcAStride, pSrcB, iSrcBStride, iHeight); | |
| 723 | + } else { | |
| 724 | + PixelAvgWidthEq16_sse2 (pDst, iDstStride, pSrcA, iSrcAStride, pSrcB, iSrcBStride, iHeight); | |
| 725 | + } | |
| 726 | +} | |
| 727 | + | |
| 728 | +void McCopy_sse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 729 | + int32_t iWidth, int32_t iHeight) { | |
| 730 | + switch (iWidth) { | |
| 731 | + case 16: return McCopyWidthEq16_sse3 (pSrc, iSrcStride, pDst, iDstStride, iHeight); | |
| 732 | + case 8: return McCopyWidthEq8_sse2 (pSrc, iSrcStride, pDst, iDstStride, iHeight); | |
| 733 | + case 4: return McCopyWidthEq4_c (pSrc, iSrcStride, pDst, iDstStride, iHeight); | |
| 734 | + } | |
| 735 | + return McCopyWidthEq2_c (pSrc, iSrcStride, pDst, iDstStride, iHeight); | |
| 736 | +} | |
| 737 | + | |
| 738 | +void McHorVer22_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 739 | + int32_t iWidth, int32_t iHeight) { | |
| 740 | + ENFORCE_STACK_ALIGN_2D (int16_t, pTmp, 16 + 5, 8, 16); | |
| 741 | + if (iWidth < 8) { | |
| 742 | + McHorVer20Width4U8ToS16_ssse3 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5); | |
| 743 | + McHorVer02Width4S16ToU8_ssse3 (&pTmp[0][0], pDst, iDstStride, iHeight); | |
| 744 | + } else if (iWidth == 8) { | |
| 745 | + McHorVer20Width8U8ToS16_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iHeight + 5); | |
| 746 | + McHorVer02WidthGe8S16ToU8_ssse3 (&pTmp[0][0], sizeof *pTmp, pDst, iDstStride, iWidth, iHeight); | |
| 747 | + } else { | |
| 748 | + McHorVer20Width8U8ToS16_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iHeight + 5); | |
| 749 | + McHorVer02WidthGe8S16ToU8_ssse3 (&pTmp[0][0], sizeof *pTmp, pDst, iDstStride, 8, iHeight); | |
| 750 | + McHorVer20Width8U8ToS16_ssse3 (pSrc + 8, iSrcStride, &pTmp[0][0], sizeof *pTmp, iHeight + 5); | |
| 751 | + McHorVer02WidthGe8S16ToU8_ssse3 (&pTmp[0][0], sizeof *pTmp, pDst + 8, iDstStride, 8, iHeight); | |
| 752 | + } | |
| 753 | +} | |
| 754 | + | |
| 755 | +void McHorVer01_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 756 | + int32_t iWidth, int32_t iHeight) { | |
| 757 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16); | |
| 758 | + McHorVer02_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight); | |
| 759 | + PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc, iSrcStride, | |
| 760 | + &pTmp[0][0], sizeof *pTmp, iWidth, iHeight); | |
| 761 | +} | |
| 762 | + | |
| 763 | +void McHorVer03_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 764 | + int32_t iWidth, int32_t iHeight) { | |
| 765 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16); | |
| 766 | + McHorVer02_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight); | |
| 767 | + PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc + iSrcStride, iSrcStride, | |
| 768 | + &pTmp[0][0], sizeof *pTmp, iWidth, iHeight); | |
| 769 | +} | |
| 770 | + | |
| 771 | +void McHorVer10_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 772 | + int32_t iWidth, int32_t iHeight) { | |
| 773 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16); | |
| 774 | + McHorVer20_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight); | |
| 775 | + PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc, iSrcStride, | |
| 776 | + &pTmp[0][0], sizeof *pTmp, iWidth, iHeight); | |
| 777 | +} | |
| 778 | + | |
| 779 | +void McHorVer11_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 780 | + int32_t iWidth, int32_t iHeight) { | |
| 781 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16); | |
| 782 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16); | |
| 783 | + McHorVer20_ssse3 (pSrc, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight); | |
| 784 | + McHorVer02_ssse3 (pSrc, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight); | |
| 785 | + PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp, | |
| 786 | + &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight); | |
| 787 | +} | |
| 788 | + | |
| 789 | +void McHorVer12_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 790 | + int32_t iWidth, int32_t iHeight) { | |
| 791 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16); | |
| 792 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16); | |
| 793 | + McHorVer02_ssse3 (pSrc, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight); | |
| 794 | + McHorVer22_ssse3 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight); | |
| 795 | + PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pVerTmp[0][0], sizeof *pVerTmp, | |
| 796 | + &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight); | |
| 797 | +} | |
| 798 | + | |
| 799 | +void McHorVer13_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 800 | + int32_t iWidth, int32_t iHeight) { | |
| 801 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16); | |
| 802 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16); | |
| 803 | + McHorVer20_ssse3 (pSrc + iSrcStride, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight); | |
| 804 | + McHorVer02_ssse3 (pSrc, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight); | |
| 805 | + PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp, | |
| 806 | + &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight); | |
| 807 | +} | |
| 808 | + | |
| 809 | +void McHorVer21_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 810 | + int32_t iWidth, int32_t iHeight) { | |
| 811 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16); | |
| 812 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16); | |
| 813 | + McHorVer20_ssse3 (pSrc, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight); | |
| 814 | + McHorVer22_ssse3 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight); | |
| 815 | + PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp, | |
| 816 | + &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight); | |
| 817 | +} | |
| 818 | + | |
| 819 | +void McHorVer23_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 820 | + int32_t iWidth, int32_t iHeight) { | |
| 821 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16); | |
| 822 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16); | |
| 823 | + McHorVer20_ssse3 (pSrc + iSrcStride, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight); | |
| 824 | + McHorVer22_ssse3 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight); | |
| 825 | + PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp, | |
| 826 | + &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight); | |
| 827 | +} | |
| 828 | + | |
| 829 | +void McHorVer30_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 830 | + int32_t iWidth, int32_t iHeight) { | |
| 831 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16); | |
| 832 | + McHorVer20_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight); | |
| 833 | + PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc + 1, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight); | |
| 834 | +} | |
| 835 | + | |
| 836 | +void McHorVer31_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 837 | + int32_t iWidth, int32_t iHeight) { | |
| 838 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16); | |
| 839 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16); | |
| 840 | + McHorVer20_ssse3 (pSrc, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight); | |
| 841 | + McHorVer02_ssse3 (pSrc + 1, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight); | |
| 842 | + PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp, | |
| 843 | + &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight); | |
| 844 | +} | |
| 845 | + | |
| 846 | +void McHorVer32_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 847 | + int32_t iWidth, int32_t iHeight) { | |
| 848 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16); | |
| 849 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16); | |
| 850 | + McHorVer02_ssse3 (pSrc + 1, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight); | |
| 851 | + McHorVer22_ssse3 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight); | |
| 852 | + PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pVerTmp[0][0], sizeof *pVerTmp, | |
| 853 | + &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight); | |
| 854 | +} | |
| 855 | + | |
| 856 | +void McHorVer33_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 857 | + int32_t iWidth, int32_t iHeight) { | |
| 858 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16); | |
| 859 | + ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16); | |
| 860 | + McHorVer20_ssse3 (pSrc + iSrcStride, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight); | |
| 861 | + McHorVer02_ssse3 (pSrc + 1, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight); | |
| 862 | + PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp, | |
| 863 | + &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight); | |
| 864 | +} | |
| 865 | + | |
| 866 | +void McHorVer22Width5Or9Or17_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 867 | + int32_t iWidth, int32_t iHeight) { | |
| 868 | + ENFORCE_STACK_ALIGN_2D (int16_t, pTmp, 17 + 5, WELS_ALIGN(17, 16 / sizeof (int16_t)), 16) | |
| 869 | + if (iWidth > 5) { | |
| 870 | + McHorVer20Width9Or17U8ToS16_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight + 5); | |
| 871 | + McHorVer02WidthGe8S16ToU8_ssse3 (&pTmp[0][0], sizeof *pTmp, pDst, iDstStride, iWidth, iHeight); | |
| 872 | + } else { | |
| 873 | + McHorVer20Width8U8ToS16_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iHeight + 5); | |
| 874 | + McHorVer02Width5S16ToU8_ssse3 (&pTmp[0][0], sizeof *pTmp, pDst, iDstStride, iHeight); | |
| 875 | + } | |
| 876 | +} | |
| 877 | + | |
| 878 | +void McLuma_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, | |
| 879 | + int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight) { | |
| 880 | + static const PWelsMcWidthHeightFunc pWelsMcFunc[4][4] = { | |
| 881 | + {McCopy_sse3, McHorVer01_ssse3, McHorVer02_ssse3, McHorVer03_ssse3}, | |
| 882 | + {McHorVer10_ssse3, McHorVer11_ssse3, McHorVer12_ssse3, McHorVer13_ssse3}, | |
| 883 | + {McHorVer20_ssse3, McHorVer21_ssse3, McHorVer22_ssse3, McHorVer23_ssse3}, | |
| 884 | + {McHorVer30_ssse3, McHorVer31_ssse3, McHorVer32_ssse3, McHorVer33_ssse3}, | |
| 885 | + }; | |
| 886 | + | |
| 887 | + pWelsMcFunc[iMvX & 0x03][iMvY & 0x03] (pSrc, iSrcStride, pDst, iDstStride, iWidth, iHeight); | |
| 888 | +} | |
| 889 | + | |
| 713 | 890 | void McChroma_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, |
| 714 | 891 | int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight) { |
| 715 | 892 | static const PMcChromaWidthExtFunc kpMcChromaWidthFuncs[2] = { |
| @@ -1319,7 +1496,9 @@ void PixelAvg_AArch64_neon (uint8_t* pDst, int32_t iDstStride, const uint8_t* pS · 603 unmodified lines | ||
| 1319 | 1496 | } |
| 1320 | 1497 | #endif |
| 1321 | 1498 | |
| 1322 | −void InitMcFunc (SMcFunc* pMcFuncs, uint32_t uiCpuFlag) { | |
| 1499 | +} // anon ns. | |
| 1500 | + | |
| 1501 | +void WelsCommon::InitMcFunc (SMcFunc* pMcFuncs, uint32_t uiCpuFlag) { | |
| 1323 | 1502 | pMcFuncs->pfLumaHalfpelHor = McHorVer20_c; |
| 1324 | 1503 | pMcFuncs->pfLumaHalfpelVer = McHorVer02_c; |
| 1325 | 1504 | pMcFuncs->pfLumaHalfpelCen = McHorVer22_c; |
| @@ -1338,7 +1517,11 @@ void InitMcFunc (SMcFunc* pMcFuncs, uint32_t uiCpuFlag) { · 12 unmodified lines | ||
| 1338 | 1517 | } |
| 1339 | 1518 | |
| 1340 | 1519 | if (uiCpuFlag & WELS_CPU_SSSE3) { |
| 1520 | + pMcFuncs->pfLumaHalfpelHor = McHorVer20Width5Or9Or17_ssse3; | |
| 1521 | + pMcFuncs->pfLumaHalfpelVer = McHorVer02_ssse3; | |
| 1522 | + pMcFuncs->pfLumaHalfpelCen = McHorVer22Width5Or9Or17_ssse3; | |
| 1341 | 1523 | pMcFuncs->pMcChromaFunc = McChroma_ssse3; |
| 1524 | + pMcFuncs->pMcLumaFunc = McLuma_ssse3; | |
| 1342 | 1525 | } |
| 1343 | 1526 | #endif //(X86_ASM) |
| 1344 | 1527 | |
| @@ -1363,4 +1546,3 @@ void InitMcFunc (SMcFunc* pMcFuncs, uint32_t uiCpuFlag) { · 18 unmodified lines | ||
| 1363 | 1546 | } |
| 1364 | 1547 | #endif |
| 1365 | 1548 | } |
| 1366 | −} // namespace WelsCommon | |