History

[Common] Add SSSE3 motion compensation routines

The resulting per-block speedup is measured to between ~1.67x and ~2.37x for horizontal filtering, between ~1.24x and ~1.53x for vertical filtering, and between ~1.33x and ~2.37x for 2-dimensional filtering on Haswell, depending on block size, as compared with the existing SSE2 routines. Outliers are 4-wide vertical and 2-dimensional filtering with a speedups of ~5.69x and ~6.84x respectively on Haswell (with clang-703.0.31) due to the C routines used previously.

Sindre Aamås committed 10 years ago

Commit 2ecbc511 parent 80721235 files · +1465 −109Browse files

Showing the changes to codec/common/src/mc.cpp (1 of 5 files). Show all

Unified · Split

codec/common/src/mc.cpp+186−4
@@ -44,6 +44,8 @@ · 43 unmodified lines
44#include "ls_defines.h"44#include "ls_defines.h"
45#include "macros.h"45#include "macros.h"
46 46
47namespace {
48
47typedef void (*PMcChromaWidthExtFunc) (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,49typedef void (*PMcChromaWidthExtFunc) (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
48 const uint8_t* kpABCD, int32_t iHeight);50 const uint8_t* kpABCD, int32_t iHeight);
49typedef void (*PWelsSampleWidthAveragingFunc) (uint8_t*, int32_t, const uint8_t*, int32_t, const uint8_t*,51typedef void (*PWelsSampleWidthAveragingFunc) (uint8_t*, int32_t, const uint8_t*, int32_t, const uint8_t*,
@@ -51,8 +53,6 @@ typedef void (*PWelsSampleWidthAveragingFunc) (uint8_t*, int32_t, const uint8_t* · 1 unmodified line
51typedef void (*PWelsMcWidthHeightFunc) (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,53typedef void (*PWelsMcWidthHeightFunc) (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
52 int32_t iWidth, int32_t iHeight);54 int32_t iWidth, int32_t iHeight);
53 55
54namespace WelsCommon {
55
56/*------------------weight for chroma fraction pixel interpolation------------------*/56/*------------------weight for chroma fraction pixel interpolation------------------*/
57//iA = (8 - dx) * (8 - dy);57//iA = (8 - dx) * (8 - dy);
58//iB = dx * (8 - dy);58//iB = dx * (8 - dy);
@@ -710,6 +710,183 @@ void McChroma_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int3 · 651 unmodified lines
710 McChromaWithFragMv_c (pSrc, iSrcStride, pDst, iDstStride, iMvX, iMvY, iWidth, iHeight);710 McChromaWithFragMv_c (pSrc, iSrcStride, pDst, iDstStride, iMvX, iMvY, iWidth, iHeight);
711}711}
712 712
713//***************************************************************************//
714// SSSE3 implementation //
715//***************************************************************************//
716
717void PixelAvgWidth4Or8Or16_sse2 (uint8_t* pDst, int32_t iDstStride, const uint8_t* pSrcA, int32_t iSrcAStride,
718 const uint8_t* pSrcB, int32_t iSrcBStride, int32_t iWidth, int32_t iHeight) {
719 if (iWidth < 8) {
720 PixelAvgWidthEq4_mmx (pDst, iDstStride, pSrcA, iSrcAStride, pSrcB, iSrcBStride, iHeight);
721 } else if (iWidth == 8) {
722 PixelAvgWidthEq8_mmx (pDst, iDstStride, pSrcA, iSrcAStride, pSrcB, iSrcBStride, iHeight);
723 } else {
724 PixelAvgWidthEq16_sse2 (pDst, iDstStride, pSrcA, iSrcAStride, pSrcB, iSrcBStride, iHeight);
725 }
726}
727
728void McCopy_sse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
729 int32_t iWidth, int32_t iHeight) {
730 switch (iWidth) {
731 case 16: return McCopyWidthEq16_sse3 (pSrc, iSrcStride, pDst, iDstStride, iHeight);
732 case 8: return McCopyWidthEq8_sse2 (pSrc, iSrcStride, pDst, iDstStride, iHeight);
733 case 4: return McCopyWidthEq4_c (pSrc, iSrcStride, pDst, iDstStride, iHeight);
734 }
735 return McCopyWidthEq2_c (pSrc, iSrcStride, pDst, iDstStride, iHeight);
736}
737
738void McHorVer22_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
739 int32_t iWidth, int32_t iHeight) {
740 ENFORCE_STACK_ALIGN_2D (int16_t, pTmp, 16 + 5, 8, 16);
741 if (iWidth < 8) {
742 McHorVer20Width4U8ToS16_ssse3 (pSrc, iSrcStride, &pTmp[0][0], iHeight + 5);
743 McHorVer02Width4S16ToU8_ssse3 (&pTmp[0][0], pDst, iDstStride, iHeight);
744 } else if (iWidth == 8) {
745 McHorVer20Width8U8ToS16_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iHeight + 5);
746 McHorVer02WidthGe8S16ToU8_ssse3 (&pTmp[0][0], sizeof *pTmp, pDst, iDstStride, iWidth, iHeight);
747 } else {
748 McHorVer20Width8U8ToS16_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iHeight + 5);
749 McHorVer02WidthGe8S16ToU8_ssse3 (&pTmp[0][0], sizeof *pTmp, pDst, iDstStride, 8, iHeight);
750 McHorVer20Width8U8ToS16_ssse3 (pSrc + 8, iSrcStride, &pTmp[0][0], sizeof *pTmp, iHeight + 5);
751 McHorVer02WidthGe8S16ToU8_ssse3 (&pTmp[0][0], sizeof *pTmp, pDst + 8, iDstStride, 8, iHeight);
752 }
753}
754
755void McHorVer01_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
756 int32_t iWidth, int32_t iHeight) {
757 ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16);
758 McHorVer02_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
759 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc, iSrcStride,
760 &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
761}
762
763void McHorVer03_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
764 int32_t iWidth, int32_t iHeight) {
765 ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16);
766 McHorVer02_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
767 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc + iSrcStride, iSrcStride,
768 &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
769}
770
771void McHorVer10_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
772 int32_t iWidth, int32_t iHeight) {
773 ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16);
774 McHorVer20_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
775 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc, iSrcStride,
776 &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
777}
778
779void McHorVer11_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
780 int32_t iWidth, int32_t iHeight) {
781 ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
782 ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
783 McHorVer20_ssse3 (pSrc, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
784 McHorVer02_ssse3 (pSrc, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
785 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
786 &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
787}
788
789void McHorVer12_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
790 int32_t iWidth, int32_t iHeight) {
791 ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
792 ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16);
793 McHorVer02_ssse3 (pSrc, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
794 McHorVer22_ssse3 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
795 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pVerTmp[0][0], sizeof *pVerTmp,
796 &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
797}
798
799void McHorVer13_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
800 int32_t iWidth, int32_t iHeight) {
801 ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
802 ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
803 McHorVer20_ssse3 (pSrc + iSrcStride, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
804 McHorVer02_ssse3 (pSrc, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
805 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
806 &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
807}
808
809void McHorVer21_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
810 int32_t iWidth, int32_t iHeight) {
811 ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
812 ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16);
813 McHorVer20_ssse3 (pSrc, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
814 McHorVer22_ssse3 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
815 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
816 &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
817}
818
819void McHorVer23_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
820 int32_t iWidth, int32_t iHeight) {
821 ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
822 ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16);
823 McHorVer20_ssse3 (pSrc + iSrcStride, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
824 McHorVer22_ssse3 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
825 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
826 &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
827}
828
829void McHorVer30_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
830 int32_t iWidth, int32_t iHeight) {
831 ENFORCE_STACK_ALIGN_2D (uint8_t, pTmp, 16, 16, 16);
832 McHorVer20_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
833 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, pSrc + 1, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight);
834}
835
836void McHorVer31_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
837 int32_t iWidth, int32_t iHeight) {
838 ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
839 ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
840 McHorVer20_ssse3 (pSrc, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
841 McHorVer02_ssse3 (pSrc + 1, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
842 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
843 &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
844}
845
846void McHorVer32_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
847 int32_t iWidth, int32_t iHeight) {
848 ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
849 ENFORCE_STACK_ALIGN_2D (uint8_t, pCtrTmp, 16, 16, 16);
850 McHorVer02_ssse3 (pSrc + 1, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
851 McHorVer22_ssse3 (pSrc, iSrcStride, &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
852 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pVerTmp[0][0], sizeof *pVerTmp,
853 &pCtrTmp[0][0], sizeof *pCtrTmp, iWidth, iHeight);
854}
855
856void McHorVer33_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
857 int32_t iWidth, int32_t iHeight) {
858 ENFORCE_STACK_ALIGN_2D (uint8_t, pHorTmp, 16, 16, 16);
859 ENFORCE_STACK_ALIGN_2D (uint8_t, pVerTmp, 16, 16, 16);
860 McHorVer20_ssse3 (pSrc + iSrcStride, iSrcStride, &pHorTmp[0][0], sizeof *pHorTmp, iWidth, iHeight);
861 McHorVer02_ssse3 (pSrc + 1, iSrcStride, &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
862 PixelAvgWidth4Or8Or16_sse2 (pDst, iDstStride, &pHorTmp[0][0], sizeof *pHorTmp,
863 &pVerTmp[0][0], sizeof *pVerTmp, iWidth, iHeight);
864}
865
866void McHorVer22Width5Or9Or17_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
867 int32_t iWidth, int32_t iHeight) {
868 ENFORCE_STACK_ALIGN_2D (int16_t, pTmp, 17 + 5, WELS_ALIGN(17, 16 / sizeof (int16_t)), 16)
869 if (iWidth > 5) {
870 McHorVer20Width9Or17U8ToS16_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iWidth, iHeight + 5);
871 McHorVer02WidthGe8S16ToU8_ssse3 (&pTmp[0][0], sizeof *pTmp, pDst, iDstStride, iWidth, iHeight);
872 } else {
873 McHorVer20Width8U8ToS16_ssse3 (pSrc, iSrcStride, &pTmp[0][0], sizeof *pTmp, iHeight + 5);
874 McHorVer02Width5S16ToU8_ssse3 (&pTmp[0][0], sizeof *pTmp, pDst, iDstStride, iHeight);
875 }
876}
877
878void McLuma_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
879 int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight) {
880 static const PWelsMcWidthHeightFunc pWelsMcFunc[4][4] = {
881 {McCopy_sse3, McHorVer01_ssse3, McHorVer02_ssse3, McHorVer03_ssse3},
882 {McHorVer10_ssse3, McHorVer11_ssse3, McHorVer12_ssse3, McHorVer13_ssse3},
883 {McHorVer20_ssse3, McHorVer21_ssse3, McHorVer22_ssse3, McHorVer23_ssse3},
884 {McHorVer30_ssse3, McHorVer31_ssse3, McHorVer32_ssse3, McHorVer33_ssse3},
885 };
886
887 pWelsMcFunc[iMvX & 0x03][iMvY & 0x03] (pSrc, iSrcStride, pDst, iDstStride, iWidth, iHeight);
888}
889
713void McChroma_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,890void McChroma_ssse3 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
714 int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight) {891 int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight) {
715 static const PMcChromaWidthExtFunc kpMcChromaWidthFuncs[2] = {892 static const PMcChromaWidthExtFunc kpMcChromaWidthFuncs[2] = {
@@ -1319,7 +1496,9 @@ void PixelAvg_AArch64_neon (uint8_t* pDst, int32_t iDstStride, const uint8_t* pS · 603 unmodified lines
1319}1496}
1320#endif1497#endif
1321 1498
1322void InitMcFunc (SMcFunc* pMcFuncs, uint32_t uiCpuFlag) {1499} // anon ns.
1500
1501void WelsCommon::InitMcFunc (SMcFunc* pMcFuncs, uint32_t uiCpuFlag) {
1323 pMcFuncs->pfLumaHalfpelHor = McHorVer20_c;1502 pMcFuncs->pfLumaHalfpelHor = McHorVer20_c;
1324 pMcFuncs->pfLumaHalfpelVer = McHorVer02_c;1503 pMcFuncs->pfLumaHalfpelVer = McHorVer02_c;
1325 pMcFuncs->pfLumaHalfpelCen = McHorVer22_c;1504 pMcFuncs->pfLumaHalfpelCen = McHorVer22_c;
@@ -1338,7 +1517,11 @@ void InitMcFunc (SMcFunc* pMcFuncs, uint32_t uiCpuFlag) { · 12 unmodified lines
1338 }1517 }
1339 1518
1340 if (uiCpuFlag & WELS_CPU_SSSE3) {1519 if (uiCpuFlag & WELS_CPU_SSSE3) {
1520 pMcFuncs->pfLumaHalfpelHor = McHorVer20Width5Or9Or17_ssse3;
1521 pMcFuncs->pfLumaHalfpelVer = McHorVer02_ssse3;
1522 pMcFuncs->pfLumaHalfpelCen = McHorVer22Width5Or9Or17_ssse3;
1341 pMcFuncs->pMcChromaFunc = McChroma_ssse3;1523 pMcFuncs->pMcChromaFunc = McChroma_ssse3;
1524 pMcFuncs->pMcLumaFunc = McLuma_ssse3;
1342 }1525 }
1343#endif //(X86_ASM)1526#endif //(X86_ASM)
1344 1527
@@ -1363,4 +1546,3 @@ void InitMcFunc (SMcFunc* pMcFuncs, uint32_t uiCpuFlag) { · 18 unmodified lines
1363 }1546 }
1364#endif1547#endif
1365}1548}
1366} // namespace WelsCommon

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t