History

[Common] Add AVX2 motion compensation routines

The resulting per-block speedup is measured to between ~1.96x and ~3.27x for horizontal filtering, between ~1.43x and ~2.09x for vertical filtering, and between ~1.47x and ~3.04x for 2-dimensional filtering on Haswell, depending on block size, as compared with the existing SSE2 routines. Outliers are 4-wide vertical and 2-dimensional filtering with a speedups of ~5.94x and ~8.41x respectively on Haswell (with clang-703.0.31) due to the C routines used previously.

Sindre Aamås committed 10 years ago

Commit b397b391 parent 2ecbc514 files · +1673 −1Browse files

Showing the changes to codec/common/src/mc.cpp (1 of 4 files). Show all

Unified · Split

codec/common/src/mc.cpp+171−0
@@ -905,6 +905,169 @@ void McChroma_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int · 904 unmodified lines
905905 McChromaWithFragMv_c (pSrc, iSrcStride, pDst, iDstStride, iMvX, iMvY, iWidth, iHeight);
906906 }
907907
908+//***************************************************************************//
909+// AVX2 implementation //
910+//***************************************************************************//
911+
912+#ifdef HAVE_AVX2
913+
914+void McHorVer22_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
915+ int32_t iWidth, int32_t iHeight) {
916+ ENFORCE_STACK_ALIGN_2D (int16_t, pTmp, 16 + 5, 16, 32);
917+ if (iWidth < 8) {
918+ McHorVer20Width4U8ToS16_avx2 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5);
919+ McHorVer02Width4S16ToU8_avx2 (&pTmp[0][0], pDst, iDstStride, iHeight);
920+ } else if (iWidth == 8) {
921+ McHorVer20Width8U8ToS16_avx2 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5);
922+ McHorVer02Width8S16ToU8_avx2 (&pTmp[0][0], pDst, iDstStride, iHeight);
923+ } else {
924+ McHorVer20Width16U8ToS16_avx2 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5);
925+ McHorVer02Width16Or17S16ToU8_avx2 (&pTmp[0][0], sizeof *pTmp, pDst, iDstStride, iWidth, iHeight);
926+ }
927+}
928+
929+void McHorVer01_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
930+ int32_t iWidth, int32_t iHeight) {
931+ ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16);
932+ McHorVer02_avx2 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
933+ PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc, iSrcStride,
934+ &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
935+}
936+
937+void McHorVer03_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
938+ int32_t iWidth, int32_t iHeight) {
939+ ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16);
940+ McHorVer02_avx2 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
941+ PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc + iSrcStride, iSrcStride,
942+ &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
943+}
944+
945+void McHorVer10_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
946+ int32_t iWidth, int32_t iHeight) {
947+ ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16);
948+ McHorVer20_avx2 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
949+ PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc, iSrcStride,
950+ &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
951+}
952+
953+void McHorVer11_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
954+ int32_t iWidth, int32_t iHeight) {
955+ ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
956+ ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
957+ McHorVer20_avx2 (pSrc, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
958+ McHorVer02_avx2 (pSrc, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
959+ PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
960+ &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
961+}
962+
963+void McHorVer12_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
964+ int32_t iWidth, int32_t iHeight) {
965+ ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
966+ ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16);
967+ McHorVer02_avx2 (pSrc, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
968+ McHorVer22_avx2 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
969+ PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pVerTmp[0][0], sizeof *pVerTmp,
970+ &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
971+}
972+
973+void McHorVer13_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
974+ int32_t iWidth, int32_t iHeight) {
975+ ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
976+ ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
977+ McHorVer20_avx2 (pSrc + iSrcStride, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
978+ McHorVer02_avx2 (pSrc, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
979+ PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
980+ &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
981+}
982+
983+void McHorVer21_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
984+ int32_t iWidth, int32_t iHeight) {
985+ ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
986+ ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16);
987+ McHorVer20_avx2 (pSrc, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
988+ McHorVer22_avx2 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
989+ PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
990+ &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
991+}
992+
993+void McHorVer23_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
994+ int32_t iWidth, int32_t iHeight) {
995+ ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
996+ ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16);
997+ McHorVer20_avx2 (pSrc + iSrcStride, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
998+ McHorVer22_avx2 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
999+ PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
1000+ &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
1001+}
1002+
1003+void McHorVer30_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
1004+ int32_t iWidth, int32_t iHeight) {
1005+ ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16);
1006+ McHorVer20_avx2 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
1007+ PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc + 1, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
1008+}
1009+
1010+void McHorVer31_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
1011+ int32_t iWidth, int32_t iHeight) {
1012+ ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
1013+ ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
1014+ McHorVer20_avx2 (pSrc, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
1015+ McHorVer02_avx2 (pSrc + 1, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
1016+ PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
1017+ &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
1018+}
1019+
1020+void McHorVer32_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
1021+ int32_t iWidth, int32_t iHeight) {
1022+ ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
1023+ ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16);
1024+ McHorVer02_avx2 (pSrc + 1, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
1025+ McHorVer22_avx2 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
1026+ PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pVerTmp[0][0], sizeof *pVerTmp,
1027+ &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
1028+}
1029+
1030+void McHorVer33_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
1031+ int32_t iWidth, int32_t iHeight) {
1032+ ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
1033+ ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
1034+ McHorVer20_avx2 (pSrc + iSrcStride, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
1035+ McHorVer02_avx2 (pSrc + 1, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
1036+ PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
1037+ &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
1038+}
1039+
1040+void McHorVer22Width5Or9Or17_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
1041+ int32_t iWidth, int32_t iHeight) {
1042+ if (iWidth < 9) {
1043+ ENFORCE_STACK_ALIGN_2D (int16_t, pTmp, 9 + 5, WELS_ALIGN(5, 16 / sizeof (int16_t)), 16)
1044+ McHorVer20Width8U8ToS16_avx2 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5);
1045+ McHorVer02Width5S16ToU8_avx2 (&pTmp[0][0], pDst, iDstStride, iHeight);
1046+ } else if (iWidth == 9) {
1047+ ENFORCE_STACK_ALIGN_2D (int16_t, pTmp, 17 + 5, 16, 32)
1048+ McHorVer20Width16U8ToS16_avx2 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5);
1049+ McHorVer02Width9S16ToU8_avx2 (&pTmp[0][0], pDst, iDstStride, iHeight);
1050+ } else {
1051+ ENFORCE_STACK_ALIGN_2D (int16_t, pTmp, 17 + 5, WELS_ALIGN(17, 32 / sizeof (int16_t)), 32)
1052+ McHorVer20Width17U8ToS16_avx2 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5);
1053+ McHorVer02Width16Or17S16ToU8_avx2 (&pTmp[0][0], sizeof *pTmp, pDst, iDstStride, iWidth, iHeight);
1054+ }
1055+}
1056+
1057+void McLuma_avx2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
1058+ int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight) {
1059+ static const PWelsMcWidthHeightFunc pWelsMcFunc[4][4] = {
1060+ {McCopy_sse3, McHorVer01_avx2, McHorVer02_avx2, McHorVer03_avx2},
1061+ {McHorVer10_avx2, McHorVer11_avx2, McHorVer12_avx2, McHorVer13_avx2},
1062+ {McHorVer20_avx2, McHorVer21_avx2, McHorVer22_avx2, McHorVer23_avx2},
1063+ {McHorVer30_avx2, McHorVer31_avx2, McHorVer32_avx2, McHorVer33_avx2},
1064+ };
1065+
1066+ pWelsMcFunc[iMvX & 0x03][iMvY & 0x03] (pSrc, iSrcStride, pDst, iDstStride, iWidth, iHeight);
1067+}
1068+
1069+#endif //HAVE_AVX2
1070+
9081071 void PixelAvg_sse2 (uint8_t* pDst, int32_t iDstStride, const uint8_t* pSrcA, int32_t iSrcAStride,
9091072 const uint8_t* pSrcB, int32_t iSrcBStride, int32_t iWidth, int32_t iHeight) {
9101073 static const PWelsSampleWidthAveragingFunc kpfFuncs[2] = {
@@ -1523,6 +1686,14 @@ void WelsCommon::InitMcFunc (SMcFunc* pMcFuncs, uint32_t uiCpuFlag) { · 612 unmodified lines
15231686 pMcFuncs->pMcChromaFunc = McChroma_ssse3;
15241687 pMcFuncs->pMcLumaFunc = McLuma_ssse3;
15251688 }
1689+#ifdef HAVE_AVX2
1690+ if (uiCpuFlag & WELS_CPU_AVX2) {
1691+ pMcFuncs->pfLumaHalfpelHor = McHorVer20Width5Or9Or17_avx2;
1692+ pMcFuncs->pfLumaHalfpelVer = McHorVer02_avx2;
1693+ pMcFuncs->pfLumaHalfpelCen = McHorVer22Width5Or9Or17_avx2;
1694+ pMcFuncs->pMcLumaFunc = McLuma_avx2;
1695+ }
1696+#endif
15261697 #endif //(X86_ASM)
15271698
15281699 #if defined(HAVE_NEON)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t