History

[Common] Add AVX2 motion compensation routines

The resulting per-block speedup is measured to between ~1.96x and ~3.27x for horizontal filtering, between ~1.43x and ~2.09x for vertical filtering, and between ~1.47x and ~3.04x for 2-dimensional filtering on Haswell, depending on block size, as compared with the existing SSE2 routines. Outliers are 4-wide vertical and 2-dimensional filtering with a speedups of ~5.94x and ~8.41x respectively on Haswell (with clang-703.0.31) due to the C routines used previously.

Sindre Aamås committed 10 years ago

Commit b397b391 parent 2ecbc514 files · +1673 −1Browse files

Showing the changes to codec/common/src/mc.cpp (1 of 4 files). Show all

Unified · Split

codec/common/src/mc.cpp+171−0
@@ -905,6 +905,169 @@ void McChroma_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int · 904 unmodified lines
905 McChromaWithFragMv_c (pSrc, iSrcStride, pDst, iDstStride, iMvX, iMvY, iWidth, iHeight);905 McChromaWithFragMv_c (pSrc, iSrcStride, pDst, iDstStride, iMvX, iMvY, iWidth, iHeight);
906}906}
907 907
908//***************************************************************************//
909// AVX2 implementation //
910//***************************************************************************//
911
912#ifdef HAVE_AVX2
913
914void McHorVer22_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
915 int32_t iWidth, int32_t iHeight) {
916 ENFORCE_STACK_ALIGN_2D (int16_t, pTmp, 16 + 5, 16, 32);
917 if (iWidth < 8) {
918 McHorVer20Width4U8ToS16_avx2 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5);
919 McHorVer02Width4S16ToU8_avx2 (&pTmp[0][0], pDst, iDstStride, iHeight);
920 } else if (iWidth == 8) {
921 McHorVer20Width8U8ToS16_avx2 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5);
922 McHorVer02Width8S16ToU8_avx2 (&pTmp[0][0], pDst, iDstStride, iHeight);
923 } else {
924 McHorVer20Width16U8ToS16_avx2 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5);
925 McHorVer02Width16Or17S16ToU8_avx2 (&pTmp[0][0], sizeof *pTmp, pDst, iDstStride, iWidth, iHeight);
926 }
927}
928
929void McHorVer01_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
930 int32_t iWidth, int32_t iHeight) {
931 ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16);
932 McHorVer02_avx2 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
933 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc, iSrcStride,
934 &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
935}
936
937void McHorVer03_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
938 int32_t iWidth, int32_t iHeight) {
939 ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16);
940 McHorVer02_avx2 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
941 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc + iSrcStride, iSrcStride,
942 &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
943}
944
945void McHorVer10_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
946 int32_t iWidth, int32_t iHeight) {
947 ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16);
948 McHorVer20_avx2 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
949 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc, iSrcStride,
950 &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
951}
952
953void McHorVer11_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
954 int32_t iWidth, int32_t iHeight) {
955 ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
956 ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
957 McHorVer20_avx2 (pSrc, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
958 McHorVer02_avx2 (pSrc, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
959 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
960 &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
961}
962
963void McHorVer12_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
964 int32_t iWidth, int32_t iHeight) {
965 ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
966 ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16);
967 McHorVer02_avx2 (pSrc, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
968 McHorVer22_avx2 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
969 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pVerTmp[0][0], sizeof *pVerTmp,
970 &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
971}
972
973void McHorVer13_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
974 int32_t iWidth, int32_t iHeight) {
975 ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
976 ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
977 McHorVer20_avx2 (pSrc + iSrcStride, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
978 McHorVer02_avx2 (pSrc, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
979 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
980 &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
981}
982
983void McHorVer21_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
984 int32_t iWidth, int32_t iHeight) {
985 ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
986 ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16);
987 McHorVer20_avx2 (pSrc, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
988 McHorVer22_avx2 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
989 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
990 &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
991}
992
993void McHorVer23_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
994 int32_t iWidth, int32_t iHeight) {
995 ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
996 ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16);
997 McHorVer20_avx2 (pSrc + iSrcStride, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
998 McHorVer22_avx2 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
999 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
1000 &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
1001}
1002
1003void McHorVer30_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
1004 int32_t iWidth, int32_t iHeight) {
1005 ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16);
1006 McHorVer20_avx2 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
1007 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc + 1, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
1008}
1009
1010void McHorVer31_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
1011 int32_t iWidth, int32_t iHeight) {
1012 ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
1013 ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
1014 McHorVer20_avx2 (pSrc, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
1015 McHorVer02_avx2 (pSrc + 1, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
1016 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
1017 &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
1018}
1019
1020void McHorVer32_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
1021 int32_t iWidth, int32_t iHeight) {
1022 ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
1023 ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16);
1024 McHorVer02_avx2 (pSrc + 1, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
1025 McHorVer22_avx2 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
1026 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pVerTmp[0][0], sizeof *pVerTmp,
1027 &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
1028}
1029
1030void McHorVer33_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
1031 int32_t iWidth, int32_t iHeight) {
1032 ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
1033 ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
1034 McHorVer20_avx2 (pSrc + iSrcStride, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
1035 McHorVer02_avx2 (pSrc + 1, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
1036 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
1037 &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
1038}
1039
1040void McHorVer22Width5Or9Or17_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
1041 int32_t iWidth, int32_t iHeight) {
1042 if (iWidth < 9) {
1043 ENFORCE_STACK_ALIGN_2D (int16_t, pTmp, 9 + 5, WELS_ALIGN(5, 16 / sizeof (int16_t)), 16)
1044 McHorVer20Width8U8ToS16_avx2 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5);
1045 McHorVer02Width5S16ToU8_avx2 (&pTmp[0][0], pDst, iDstStride, iHeight);
1046 } else if (iWidth == 9) {
1047 ENFORCE_STACK_ALIGN_2D (int16_t, pTmp, 17 + 5, 16, 32)
1048 McHorVer20Width16U8ToS16_avx2 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5);
1049 McHorVer02Width9S16ToU8_avx2 (&pTmp[0][0], pDst, iDstStride, iHeight);
1050 } else {
1051 ENFORCE_STACK_ALIGN_2D (int16_t, pTmp, 17 + 5, WELS_ALIGN(17, 32 / sizeof (int16_t)), 32)
1052 McHorVer20Width17U8ToS16_avx2 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5);
1053 McHorVer02Width16Or17S16ToU8_avx2 (&pTmp[0][0], sizeof *pTmp, pDst, iDstStride, iWidth, iHeight);
1054 }
1055}
1056
1057void McLuma_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
1058 int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight) {
1059 static const PWelsMcWidthHeightFunc pWelsMcFunc[4][4] = {
1060 {McCopy_sse3, McHorVer01_avx2, McHorVer02_avx2, McHorVer03_avx2},
1061 {McHorVer10_avx2, McHorVer11_avx2, McHorVer12_avx2, McHorVer13_avx2},
1062 {McHorVer20_avx2, McHorVer21_avx2, McHorVer22_avx2, McHorVer23_avx2},
1063 {McHorVer30_avx2, McHorVer31_avx2, McHorVer32_avx2, McHorVer33_avx2},
1064 };
1065
1066 pWelsMcFunc[iMvX & 0x03][iMvY & 0x03] (pSrc, iSrcStride, pDst, iDstStride, iWidth, iHeight);
1067}
1068
1069#endif //HAVE_AVX2
1070
908void PixelAvg_sse2 (uint8_t* pDst, int32_t iDstStride, const uint8_t* pSrcA, int32_t iSrcAStride,1071void PixelAvg_sse2 (uint8_t* pDst, int32_t iDstStride, const uint8_t* pSrcA, int32_t iSrcAStride,
909 const uint8_t* pSrcB, int32_t iSrcBStride, int32_t iWidth, int32_t iHeight) {1072 const uint8_t* pSrcB, int32_t iSrcBStride, int32_t iWidth, int32_t iHeight) {
910 static const PWelsSampleWidthAveragingFunc kpfFuncs[2] = {1073 static const PWelsSampleWidthAveragingFunc kpfFuncs[2] = {
@@ -1523,6 +1686,14 @@ void WelsCommon::InitMcFunc (SMcFunc* pMcFuncs, uint32_t uiCpuFlag) { · 612 unmodified lines
1523 pMcFuncs->pMcChromaFunc = McChroma_ssse3;1686 pMcFuncs->pMcChromaFunc = McChroma_ssse3;
1524 pMcFuncs->pMcLumaFunc = McLuma_ssse3;1687 pMcFuncs->pMcLumaFunc = McLuma_ssse3;
1525 }1688 }
1689#ifdef HAVE_AVX2
1690 if (uiCpuFlag & WELS_CPU_AVX2) {
1691 pMcFuncs->pfLumaHalfpelHor = McHorVer20Width5Or9Or17_avx2;
1692 pMcFuncs->pfLumaHalfpelVer = McHorVer02_avx2;
1693 pMcFuncs->pfLumaHalfpelCen = McHorVer22Width5Or9Or17_avx2;
1694 pMcFuncs->pMcLumaFunc = McLuma_avx2;
1695 }
1696#endif
1526#endif //(X86_ASM)1697#endif //(X86_ASM)
1527 1698
1528#if defined(HAVE_NEON)1699#if defined(HAVE_NEON)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t