Add Motion Compehension ARM64 Neon Code

dongzhang committed 12 years ago

Commit ad9e2da1 parent b47606a8 files · +5475 −480Browse files

Showing the changes to codec/decoder/core/src/mc.cpp (1 of 8 files). Show all

Unified · Split

@@ -85,13 +85,13 @@ static const uint8_t g_kuiABCD[8][8][4] = { //g_kA[dy][dx], g_kB[dy][dx], g_kC[d · 84 unmodified lines
85	};	85	};
86		86
87	typedef void (PWelsMcWidthHeightFunc) (const uint8_t pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	87	typedef void (PWelsMcWidthHeightFunc) (const uint8_t pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
88	int32_t iWidth, int32_t iHeight);	88	int32_t iWidth, int32_t iHeight);
89		89
90	//***************************************************************************//	90	//***************************************************************************//
91	// C code implementation //	91	// C code implementation //
92	//***************************************************************************//	92	//***************************************************************************//
93	static inline void McCopyWidthEq2_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	93	static inline void McCopyWidthEq2_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
94	int32_t iHeight) {	94	int32_t iHeight) {
95	int32_t i;	95	int32_t i;
96	for (i = 0; i < iHeight; i++) { // iWidth == 2 only for chroma	96	for (i = 0; i < iHeight; i++) { // iWidth == 2 only for chroma
97	ST16A2 (pDst, LD16 (pSrc));	97	ST16A2 (pDst, LD16 (pSrc));
@@ -101,7 +101,7 @@ static inline void McCopyWidthEq2_c (const uint8_t* pSrc, int32_t iSrcStride, ui · 3 unmodified lines
101	}	101	}
102		102
103	static inline void McCopyWidthEq4_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	103	static inline void McCopyWidthEq4_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
104	int32_t iHeight) {	104	int32_t iHeight) {
105	int32_t i;	105	int32_t i;
106	for (i = 0; i < iHeight; i++) {	106	for (i = 0; i < iHeight; i++) {
107	ST32A4 (pDst, LD32 (pSrc));	107	ST32A4 (pDst, LD32 (pSrc));
@@ -111,7 +111,7 @@ static inline void McCopyWidthEq4_c (const uint8_t* pSrc, int32_t iSrcStride, ui · 3 unmodified lines
111	}	111	}
112		112
113	static inline void McCopyWidthEq8_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	113	static inline void McCopyWidthEq8_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
114	int32_t iHeight) {	114	int32_t iHeight) {
115	int32_t i;	115	int32_t i;
116	for (i = 0; i < iHeight; i++) {	116	for (i = 0; i < iHeight; i++) {
117	ST64A8 (pDst, LD64 (pSrc));	117	ST64A8 (pDst, LD64 (pSrc));
@@ -121,7 +121,7 @@ static inline void McCopyWidthEq8_c (const uint8_t* pSrc, int32_t iSrcStride, ui · 3 unmodified lines
121	}	121	}
122		122
123	static inline void McCopyWidthEq16_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	123	static inline void McCopyWidthEq16_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
124	int32_t iHeight) {	124	int32_t iHeight) {
125	int32_t i;	125	int32_t i;
126	for (i = 0; i < iHeight; i++) {	126	for (i = 0; i < iHeight; i++) {
127	ST64A8 (pDst , LD64 (pSrc));	127	ST64A8 (pDst , LD64 (pSrc));
@@ -138,7 +138,7 @@ static inline int32_t HorFilterInput16bit_c (int16_t* pSrc) { · 10 unmodified lines
138	int32_t iPix14 = pSrc[-1] + pSrc[2];	138	int32_t iPix14 = pSrc[-1] + pSrc[2];
139	int32_t iPix23 = pSrc[ 0] + pSrc[1];	139	int32_t iPix23 = pSrc[ 0] + pSrc[1];
140		140
141	return (iPix05 - (iPix14 * 5)+ (iPix23 * 20));	141	return (iPix05 - (iPix14 * 5) + (iPix23 * 20));
142	}	142	}
143	// h: iOffset=1 / v: iOffset=iSrcStride	143	// h: iOffset=1 / v: iOffset=iSrcStride
144	static inline int32_t FilterInput8bitWithStride_c (const uint8_t* pSrc, const int32_t kiOffset) {	144	static inline int32_t FilterInput8bitWithStride_c (const uint8_t* pSrc, const int32_t kiOffset) {
@@ -153,7 +153,7 @@ static inline int32_t FilterInput8bitWithStride_c (const uint8_t* pSrc, const in · 8 unmodified lines
153	}	153	}
154		154
155	static inline void PixelAvg_c (uint8_t* pDst, int32_t iDstStride, const uint8_t* pSrcA, int32_t iSrcAStride,	155	static inline void PixelAvg_c (uint8_t* pDst, int32_t iDstStride, const uint8_t* pSrcA, int32_t iSrcAStride,
156	const uint8_t* pSrcB, int32_t iSrcBStride, int32_t iWidth, int32_t iHeight) {	156	const uint8_t* pSrcB, int32_t iSrcBStride, int32_t iWidth, int32_t iHeight) {
157	int32_t i, j;	157	int32_t i, j;
158	for (i = 0; i < iHeight; i++) {	158	for (i = 0; i < iHeight; i++) {
159	for (j = 0; j < iWidth; j++) {	159	for (j = 0; j < iWidth; j++) {
@@ -165,7 +165,7 @@ static inline void PixelAvg_c (uint8_t* pDst, int32_t iDstStride, const uint8_t* · 5 unmodified lines
165	}	165	}
166	}	166	}
167	static inline void McCopy_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	167	static inline void McCopy_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,
168	int32_t iHeight) {	168	int32_t iHeight) {
169	if (iWidth == 16)	169	if (iWidth == 16)
170	McCopyWidthEq16_c (pSrc, iSrcStride, pDst, iDstStride, iHeight);	170	McCopyWidthEq16_c (pSrc, iSrcStride, pDst, iDstStride, iHeight);
171	else if (iWidth == 8)	171	else if (iWidth == 8)
@@ -176,8 +176,9 @@ static inline void McCopy_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* p · 4 unmodified lines
176	McCopyWidthEq2_c (pSrc, iSrcStride, pDst, iDstStride, iHeight);	176	McCopyWidthEq2_c (pSrc, iSrcStride, pDst, iDstStride, iHeight);
177	}	177	}
178		178
179	static inline void McHorVer20_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	179	static inline void McHorVer20_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
180	int32_t iHeight) {	180	int32_t iWidth,
		181	int32_t iHeight) {
181	int32_t i, j;	182	int32_t i, j;
182	for (i = 0; i < iHeight; i++) {	183	for (i = 0; i < iHeight; i++) {
183	for (j = 0; j < iWidth; j++) {	184	for (j = 0; j < iWidth; j++) {
@@ -188,8 +189,9 @@ static inline void McHorVer20_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_ · 4 unmodified lines
188	}	189	}
189	}	190	}
190		191
191	static inline void McHorVer02_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	192	static inline void McHorVer02_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
192	int32_t iHeight) {	193	int32_t iWidth,
		194	int32_t iHeight) {
193	int32_t i, j;	195	int32_t i, j;
194	for (i = 0; i < iHeight; i++) {	196	for (i = 0; i < iHeight; i++) {
195	for (j = 0; j < iWidth; j++) {	197	for (j = 0; j < iWidth; j++) {
@@ -200,8 +202,9 @@ static inline void McHorVer02_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_ · 4 unmodified lines
200	}	202	}
201	}	203	}
202		204
203	static inline void McHorVer22_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	205	static inline void McHorVer22_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
204	int32_t iHeight) {	206	int32_t iWidth,
		207	int32_t iHeight) {
205	int16_t iTmp[16 + 5]; //16	208	int16_t iTmp[16 + 5]; //16
206	int32_t i, j, k;	209	int32_t i, j, k;
207		210
@@ -218,88 +221,100 @@ static inline void McHorVer22_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_ · 10 unmodified lines
218	}	221	}
219		222
220	/////////////////////luma MC//////////////////////////	223	/////////////////////luma MC//////////////////////////
221	static inline void McHorVer01_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	224	static inline void McHorVer01_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
222	int32_t iHeight) {	225	int32_t iWidth,
		226	int32_t iHeight) {
223	uint8_t uiTmp[256];	227	uint8_t uiTmp[256];
224	McHorVer02_c (pSrc, iSrcStride, uiTmp, 16, iWidth, iHeight);	228	McHorVer02_c (pSrc, iSrcStride, uiTmp, 16, iWidth, iHeight);
225	PixelAvg_c (pDst, iDstStride, pSrc, iSrcStride, uiTmp, 16, iWidth, iHeight);	229	PixelAvg_c (pDst, iDstStride, pSrc, iSrcStride, uiTmp, 16, iWidth, iHeight);
226	}	230	}
227	static inline void McHorVer03_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	231	static inline void McHorVer03_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
228	int32_t iHeight) {	232	int32_t iWidth,
		233	int32_t iHeight) {
229	uint8_t uiTmp[256];	234	uint8_t uiTmp[256];
230	McHorVer02_c (pSrc, iSrcStride, uiTmp, 16, iWidth, iHeight);	235	McHorVer02_c (pSrc, iSrcStride, uiTmp, 16, iWidth, iHeight);
231	PixelAvg_c (pDst, iDstStride, pSrc + iSrcStride, iSrcStride, uiTmp, 16, iWidth, iHeight);	236	PixelAvg_c (pDst, iDstStride, pSrc + iSrcStride, iSrcStride, uiTmp, 16, iWidth, iHeight);
232	}	237	}
233	static inline void McHorVer10_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	238	static inline void McHorVer10_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
234	int32_t iHeight) {	239	int32_t iWidth,
		240	int32_t iHeight) {
235	uint8_t uiTmp[256];	241	uint8_t uiTmp[256];
236	McHorVer20_c (pSrc, iSrcStride, uiTmp, 16, iWidth, iHeight);	242	McHorVer20_c (pSrc, iSrcStride, uiTmp, 16, iWidth, iHeight);
237	PixelAvg_c (pDst, iDstStride, pSrc, iSrcStride, uiTmp, 16, iWidth, iHeight);	243	PixelAvg_c (pDst, iDstStride, pSrc, iSrcStride, uiTmp, 16, iWidth, iHeight);
238	}	244	}
239	static inline void McHorVer11_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	245	static inline void McHorVer11_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
240	int32_t iHeight) {	246	int32_t iWidth,
		247	int32_t iHeight) {
241	uint8_t uiHorTmp[256];	248	uint8_t uiHorTmp[256];
242	uint8_t uiVerTmp[256];	249	uint8_t uiVerTmp[256];
243	McHorVer20_c (pSrc, iSrcStride, uiHorTmp, 16, iWidth, iHeight);	250	McHorVer20_c (pSrc, iSrcStride, uiHorTmp, 16, iWidth, iHeight);
244	McHorVer02_c (pSrc, iSrcStride, uiVerTmp, 16, iWidth, iHeight);	251	McHorVer02_c (pSrc, iSrcStride, uiVerTmp, 16, iWidth, iHeight);
245	PixelAvg_c (pDst, iDstStride, uiHorTmp, 16, uiVerTmp, 16, iWidth, iHeight);	252	PixelAvg_c (pDst, iDstStride, uiHorTmp, 16, uiVerTmp, 16, iWidth, iHeight);
246	}	253	}
247	static inline void McHorVer12_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	254	static inline void McHorVer12_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
248	int32_t iHeight) {	255	int32_t iWidth,
		256	int32_t iHeight) {
249	uint8_t uiVerTmp[256];	257	uint8_t uiVerTmp[256];
250	uint8_t uiCtrTmp[256];	258	uint8_t uiCtrTmp[256];
251	McHorVer02_c (pSrc, iSrcStride, uiVerTmp, 16, iWidth, iHeight);	259	McHorVer02_c (pSrc, iSrcStride, uiVerTmp, 16, iWidth, iHeight);
252	McHorVer22_c (pSrc, iSrcStride, uiCtrTmp, 16, iWidth, iHeight);	260	McHorVer22_c (pSrc, iSrcStride, uiCtrTmp, 16, iWidth, iHeight);
253	PixelAvg_c (pDst, iDstStride, uiVerTmp, 16, uiCtrTmp, 16, iWidth, iHeight);	261	PixelAvg_c (pDst, iDstStride, uiVerTmp, 16, uiCtrTmp, 16, iWidth, iHeight);
254	}	262	}
255	static inline void McHorVer13_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	263	static inline void McHorVer13_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
256	int32_t iHeight) {	264	int32_t iWidth,
		265	int32_t iHeight) {
257	uint8_t uiHorTmp[256];	266	uint8_t uiHorTmp[256];
258	uint8_t uiVerTmp[256];	267	uint8_t uiVerTmp[256];
259	McHorVer20_c (pSrc + iSrcStride, iSrcStride, uiHorTmp, 16, iWidth, iHeight);	268	McHorVer20_c (pSrc + iSrcStride, iSrcStride, uiHorTmp, 16, iWidth, iHeight);
260	McHorVer02_c (pSrc, iSrcStride, uiVerTmp, 16, iWidth, iHeight);	269	McHorVer02_c (pSrc, iSrcStride, uiVerTmp, 16, iWidth, iHeight);
261	PixelAvg_c (pDst, iDstStride, uiHorTmp, 16, uiVerTmp, 16, iWidth, iHeight);	270	PixelAvg_c (pDst, iDstStride, uiHorTmp, 16, uiVerTmp, 16, iWidth, iHeight);
262	}	271	}
263	static inline void McHorVer21_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	272	static inline void McHorVer21_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
264	int32_t iHeight) {	273	int32_t iWidth,
		274	int32_t iHeight) {
265	uint8_t uiHorTmp[256];	275	uint8_t uiHorTmp[256];
266	uint8_t uiCtrTmp[256];	276	uint8_t uiCtrTmp[256];
267	McHorVer20_c (pSrc, iSrcStride, uiHorTmp, 16, iWidth, iHeight);	277	McHorVer20_c (pSrc, iSrcStride, uiHorTmp, 16, iWidth, iHeight);
268	McHorVer22_c (pSrc, iSrcStride, uiCtrTmp, 16, iWidth, iHeight);	278	McHorVer22_c (pSrc, iSrcStride, uiCtrTmp, 16, iWidth, iHeight);
269	PixelAvg_c (pDst, iDstStride, uiHorTmp, 16, uiCtrTmp, 16, iWidth, iHeight);	279	PixelAvg_c (pDst, iDstStride, uiHorTmp, 16, uiCtrTmp, 16, iWidth, iHeight);
270	}	280	}
271	static inline void McHorVer23_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	281	static inline void McHorVer23_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
272	int32_t iHeight) {	282	int32_t iWidth,
		283	int32_t iHeight) {
273	uint8_t uiHorTmp[256];	284	uint8_t uiHorTmp[256];
274	uint8_t uiCtrTmp[256];	285	uint8_t uiCtrTmp[256];
275	McHorVer20_c (pSrc + iSrcStride, iSrcStride, uiHorTmp, 16, iWidth, iHeight);	286	McHorVer20_c (pSrc + iSrcStride, iSrcStride, uiHorTmp, 16, iWidth, iHeight);
276	McHorVer22_c (pSrc, iSrcStride, uiCtrTmp, 16, iWidth, iHeight);	287	McHorVer22_c (pSrc, iSrcStride, uiCtrTmp, 16, iWidth, iHeight);
277	PixelAvg_c (pDst, iDstStride, uiHorTmp, 16, uiCtrTmp, 16, iWidth, iHeight);	288	PixelAvg_c (pDst, iDstStride, uiHorTmp, 16, uiCtrTmp, 16, iWidth, iHeight);
278	}	289	}
279	static inline void McHorVer30_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	290	static inline void McHorVer30_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
280	int32_t iHeight) {	291	int32_t iWidth,
		292	int32_t iHeight) {
281	uint8_t uiHorTmp[256];	293	uint8_t uiHorTmp[256];
282	McHorVer20_c (pSrc, iSrcStride, uiHorTmp, 16, iWidth, iHeight);	294	McHorVer20_c (pSrc, iSrcStride, uiHorTmp, 16, iWidth, iHeight);
283	PixelAvg_c (pDst, iDstStride, pSrc + 1, iSrcStride, uiHorTmp, 16, iWidth, iHeight);	295	PixelAvg_c (pDst, iDstStride, pSrc + 1, iSrcStride, uiHorTmp, 16, iWidth, iHeight);
284	}	296	}
285	static inline void McHorVer31_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	297	static inline void McHorVer31_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
286	int32_t iHeight) {	298	int32_t iWidth,
		299	int32_t iHeight) {
287	uint8_t uiHorTmp[256];	300	uint8_t uiHorTmp[256];
288	uint8_t uiVerTmp[256];	301	uint8_t uiVerTmp[256];
289	McHorVer20_c (pSrc, iSrcStride, uiHorTmp, 16, iWidth, iHeight);	302	McHorVer20_c (pSrc, iSrcStride, uiHorTmp, 16, iWidth, iHeight);
290	McHorVer02_c (pSrc + 1, iSrcStride, uiVerTmp, 16, iWidth, iHeight);	303	McHorVer02_c (pSrc + 1, iSrcStride, uiVerTmp, 16, iWidth, iHeight);
291	PixelAvg_c (pDst, iDstStride, uiHorTmp, 16, uiVerTmp, 16, iWidth, iHeight);	304	PixelAvg_c (pDst, iDstStride, uiHorTmp, 16, uiVerTmp, 16, iWidth, iHeight);
292	}	305	}
293	static inline void McHorVer32_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	306	static inline void McHorVer32_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
294	int32_t iHeight) {	307	int32_t iWidth,
		308	int32_t iHeight) {
295	uint8_t uiVerTmp[256];	309	uint8_t uiVerTmp[256];
296	uint8_t uiCtrTmp[256];	310	uint8_t uiCtrTmp[256];
297	McHorVer02_c (pSrc + 1, iSrcStride, uiVerTmp, 16, iWidth, iHeight);	311	McHorVer02_c (pSrc + 1, iSrcStride, uiVerTmp, 16, iWidth, iHeight);
298	McHorVer22_c (pSrc, iSrcStride, uiCtrTmp, 16, iWidth, iHeight);	312	McHorVer22_c (pSrc, iSrcStride, uiCtrTmp, 16, iWidth, iHeight);
299	PixelAvg_c (pDst, iDstStride, uiVerTmp, 16, uiCtrTmp, 16, iWidth, iHeight);	313	PixelAvg_c (pDst, iDstStride, uiVerTmp, 16, uiCtrTmp, 16, iWidth, iHeight);
300	}	314	}
301	static inline void McHorVer33_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	315	static inline void McHorVer33_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
302	int32_t iHeight) {	316	int32_t iWidth,
		317	int32_t iHeight) {
303	uint8_t uiHorTmp[256];	318	uint8_t uiHorTmp[256];
304	uint8_t uiVerTmp[256];	319	uint8_t uiVerTmp[256];
305	McHorVer20_c (pSrc + iSrcStride, iSrcStride, uiHorTmp, 16, iWidth, iHeight);	320	McHorVer20_c (pSrc + iSrcStride, iSrcStride, uiHorTmp, 16, iWidth, iHeight);
@@ -308,7 +323,7 @@ static inline void McHorVer33_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_ · 2 unmodified lines
308	}	323	}
309		324
310	void McLuma_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	325	void McLuma_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
311	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight)	326	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight)
312	//pSrc has been added the offset of mv	327	//pSrc has been added the offset of mv
313	{	328	{
314	static const PWelsMcWidthHeightFunc pWelsMcFunc[4][4] = { //[x][y]	329	static const PWelsMcWidthHeightFunc pWelsMcFunc[4][4] = { //[x][y]
@@ -326,7 +341,7 @@ static inline void McChromaWithFragMv_c (const uint8_t* pSrc, int32_t iSrcStride · 11 unmodified lines
326	int32_t i, j;	341	int32_t i, j;
327	int32_t iA, iB, iC, iD;	342	int32_t iA, iB, iC, iD;
328	const uint8_t* pSrcNext = pSrc + iSrcStride;	343	const uint8_t* pSrcNext = pSrc + iSrcStride;
329	const uint8_t *pABCD = g_kuiABCD[iMvY & 0x07][iMvX & 0x07];	344	const uint8_t* pABCD = g_kuiABCD[iMvY & 0x07][iMvX & 0x07];
330	iA = pABCD[0];	345	iA = pABCD[0];
331	iB = pABCD[1];	346	iB = pABCD[1];
332	iC = pABCD[2];	347	iC = pABCD[2];
@@ -342,7 +357,7 @@ static inline void McChromaWithFragMv_c (const uint8_t* pSrc, int32_t iSrcStride · 9 unmodified lines
342	}	357	}
343		358
344	void McChroma_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	359	void McChroma_c (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
345	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight)	360	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight)
346	//pSrc has been added the offset of mv	361	//pSrc has been added the offset of mv
347	{	362	{
348	const int32_t kiD8x = iMvX & 0x07;	363	const int32_t kiD8x = iMvX & 0x07;
@@ -376,8 +391,9 @@ static inline void McHorVer22WidthEq16_sse2 (const uint8_t* pSrc, int32_t iSrcSt · 27 unmodified lines
376	McHorVer22WidthEq8_sse2 (&pSrc[8], iSrcStride, &pDst[8], iDstStride, iHeight);	391	McHorVer22WidthEq8_sse2 (&pSrc[8], iSrcStride, &pDst[8], iDstStride, iHeight);
377	}	392	}
378		393
379	static inline void McCopy_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride, int32_t iWidth,	394	static inline void McCopy_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
380	int32_t iHeight) {	395	int32_t iWidth,
		396	int32_t iHeight) {
381	if (iWidth == 16)	397	if (iWidth == 16)
382	McCopyWidthEq16_sse2 (pSrc, iSrcStride, pDst, iDstStride, iHeight);	398	McCopyWidthEq16_sse2 (pSrc, iSrcStride, pDst, iDstStride, iHeight);
383	else if (iWidth == 8)	399	else if (iWidth == 8)
@@ -389,7 +405,7 @@ static inline void McCopy_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t · 5 unmodified lines
389	}	405	}
390		406
391	static inline void McHorVer20_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	407	static inline void McHorVer20_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
392	int32_t iWidth, int32_t iHeight) {	408	int32_t iWidth, int32_t iHeight) {
393	if (iWidth == 16)	409	if (iWidth == 16)
394	McHorVer20WidthEq16_sse2 (pSrc, iSrcStride, pDst, iDstStride, iHeight);	410	McHorVer20WidthEq16_sse2 (pSrc, iSrcStride, pDst, iDstStride, iHeight);
395	else if (iWidth == 8)	411	else if (iWidth == 8)
@@ -399,7 +415,7 @@ static inline void McHorVer20_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 3 unmodified lines
399	}	415	}
400		416
401	static inline void McHorVer02_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	417	static inline void McHorVer02_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
402	int32_t iWidth, int32_t iHeight) {	418	int32_t iWidth, int32_t iHeight) {
403	if (iWidth == 16)	419	if (iWidth == 16)
404	McHorVer02WidthEq16_sse2 (pSrc, iSrcStride, pDst, iDstStride, iHeight);	420	McHorVer02WidthEq16_sse2 (pSrc, iSrcStride, pDst, iDstStride, iHeight);
405	else if (iWidth == 8)	421	else if (iWidth == 8)
@@ -409,7 +425,7 @@ static inline void McHorVer02_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 3 unmodified lines
409	}	425	}
410		426
411	static inline void McHorVer22_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	427	static inline void McHorVer22_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
412	int32_t iWidth, int32_t iHeight) {	428	int32_t iWidth, int32_t iHeight) {
413	if (iWidth == 16)	429	if (iWidth == 16)
414	McHorVer22WidthEq16_sse2 (pSrc, iSrcStride, pDst, iDstStride, iHeight);	430	McHorVer22WidthEq16_sse2 (pSrc, iSrcStride, pDst, iDstStride, iHeight);
415	else if (iWidth == 8)	431	else if (iWidth == 8)
@@ -419,7 +435,7 @@ static inline void McHorVer22_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 3 unmodified lines
419	}	435	}
420		436
421	static inline void McHorVer01_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	437	static inline void McHorVer01_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
422	int32_t iWidth, int32_t iHeight) {	438	int32_t iWidth, int32_t iHeight) {
423	ENFORCE_STACK_ALIGN_1D (uint8_t, pTmp, 256, 16);	439	ENFORCE_STACK_ALIGN_1D (uint8_t, pTmp, 256, 16);
424	if (iWidth == 16) {	440	if (iWidth == 16) {
425	McHorVer02WidthEq16_sse2 (pSrc, iSrcStride, pTmp, 16, iHeight);	441	McHorVer02WidthEq16_sse2 (pSrc, iSrcStride, pTmp, 16, iHeight);
@@ -433,7 +449,7 @@ static inline void McHorVer01_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 7 unmodified lines
433	}	449	}
434	}	450	}
435	static inline void McHorVer03_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	451	static inline void McHorVer03_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
436	int32_t iWidth, int32_t iHeight) {	452	int32_t iWidth, int32_t iHeight) {
437	ENFORCE_STACK_ALIGN_1D (uint8_t, pTmp, 256, 16);	453	ENFORCE_STACK_ALIGN_1D (uint8_t, pTmp, 256, 16);
438	if (iWidth == 16) {	454	if (iWidth == 16) {
439	McHorVer02WidthEq16_sse2 (pSrc, iSrcStride, pTmp, 16, iHeight);	455	McHorVer02WidthEq16_sse2 (pSrc, iSrcStride, pTmp, 16, iHeight);
@@ -447,7 +463,7 @@ static inline void McHorVer03_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 7 unmodified lines
447	}	463	}
448	}	464	}
449	static inline void McHorVer10_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	465	static inline void McHorVer10_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
450	int32_t iWidth, int32_t iHeight) {	466	int32_t iWidth, int32_t iHeight) {
451	ENFORCE_STACK_ALIGN_1D (uint8_t, pTmp, 256, 16);	467	ENFORCE_STACK_ALIGN_1D (uint8_t, pTmp, 256, 16);
452	if (iWidth == 16) {	468	if (iWidth == 16) {
453	McHorVer20WidthEq16_sse2 (pSrc, iSrcStride, pTmp, 16, iHeight);	469	McHorVer20WidthEq16_sse2 (pSrc, iSrcStride, pTmp, 16, iHeight);
@@ -461,7 +477,7 @@ static inline void McHorVer10_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 7 unmodified lines
461	}	477	}
462	}	478	}
463	static inline void McHorVer11_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	479	static inline void McHorVer11_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
464	int32_t iWidth, int32_t iHeight) {	480	int32_t iWidth, int32_t iHeight) {
465	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);	481	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
466	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);	482	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
467	if (iWidth == 16) {	483	if (iWidth == 16) {
@@ -479,7 +495,7 @@ static inline void McHorVer11_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 11 unmodified lines
479	}	495	}
480	}	496	}
481	static inline void McHorVer12_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	497	static inline void McHorVer12_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
482	int32_t iWidth, int32_t iHeight) {	498	int32_t iWidth, int32_t iHeight) {
483	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);	499	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
484	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);	500	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);
485	if (iWidth == 16) {	501	if (iWidth == 16) {
@@ -497,7 +513,7 @@ static inline void McHorVer12_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 11 unmodified lines
497	}	513	}
498	}	514	}
499	static inline void McHorVer13_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	515	static inline void McHorVer13_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
500	int32_t iWidth, int32_t iHeight) {	516	int32_t iWidth, int32_t iHeight) {
501	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);	517	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
502	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);	518	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
503	if (iWidth == 16) {	519	if (iWidth == 16) {
@@ -515,7 +531,7 @@ static inline void McHorVer13_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 11 unmodified lines
515	}	531	}
516	}	532	}
517	static inline void McHorVer21_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	533	static inline void McHorVer21_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
518	int32_t iWidth, int32_t iHeight) {	534	int32_t iWidth, int32_t iHeight) {
519	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);	535	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
520	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);	536	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);
521	if (iWidth == 16) {	537	if (iWidth == 16) {
@@ -533,7 +549,7 @@ static inline void McHorVer21_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 11 unmodified lines
533	}	549	}
534	}	550	}
535	static inline void McHorVer23_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	551	static inline void McHorVer23_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
536	int32_t iWidth, int32_t iHeight) {	552	int32_t iWidth, int32_t iHeight) {
537	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);	553	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
538	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);	554	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);
539	if (iWidth == 16) {	555	if (iWidth == 16) {
@@ -551,7 +567,7 @@ static inline void McHorVer23_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 11 unmodified lines
551	}	567	}
552	}	568	}
553	static inline void McHorVer30_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	569	static inline void McHorVer30_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
554	int32_t iWidth, int32_t iHeight) {	570	int32_t iWidth, int32_t iHeight) {
555	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);	571	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
556	if (iWidth == 16) {	572	if (iWidth == 16) {
557	McHorVer20WidthEq16_sse2 (pSrc, iSrcStride, pHorTmp, 16, iHeight);	573	McHorVer20WidthEq16_sse2 (pSrc, iSrcStride, pHorTmp, 16, iHeight);
@@ -565,7 +581,7 @@ static inline void McHorVer30_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 7 unmodified lines
565	}	581	}
566	}	582	}
567	static inline void McHorVer31_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	583	static inline void McHorVer31_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
568	int32_t iWidth, int32_t iHeight) {	584	int32_t iWidth, int32_t iHeight) {
569	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);	585	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
570	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);	586	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
571	if (iWidth == 16) {	587	if (iWidth == 16) {
@@ -583,7 +599,7 @@ static inline void McHorVer31_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 11 unmodified lines
583	}	599	}
584	}	600	}
585	static inline void McHorVer32_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	601	static inline void McHorVer32_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
586	int32_t iWidth, int32_t iHeight) {	602	int32_t iWidth, int32_t iHeight) {
587	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);	603	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
588	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);	604	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);
589	if (iWidth == 16) {	605	if (iWidth == 16) {
@@ -601,7 +617,7 @@ static inline void McHorVer32_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 11 unmodified lines
601	}	617	}
602	}	618	}
603	static inline void McHorVer33_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	619	static inline void McHorVer33_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
604	int32_t iWidth, int32_t iHeight) {	620	int32_t iWidth, int32_t iHeight) {
605	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);	621	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
606	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);	622	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
607	if (iWidth == 16) {	623	if (iWidth == 16) {
@@ -620,7 +636,7 @@ static inline void McHorVer33_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uin · 12 unmodified lines
620	}	636	}
621		637
622	void McLuma_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	638	void McLuma_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
623	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight)	639	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight)
624	//pSrc has been added the offset of mv	640	//pSrc has been added the offset of mv
625	{	641	{
626	static const PWelsMcWidthHeightFunc pWelsMcFunc[4][4] = { //[x][y]	642	static const PWelsMcWidthHeightFunc pWelsMcFunc[4][4] = { //[x][y]
@@ -634,9 +650,9 @@ void McLuma_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_ · 7 unmodified lines
634	}	650	}
635		651
636	void McChroma_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	652	void McChroma_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
637	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight) {	653	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight) {
638	static const PMcChromaWidthExtFunc kpMcChromaWidthFuncs[2] = {	654	static const PMcChromaWidthExtFunc kpMcChromaWidthFuncs[2] = {
639	McChromaWidthEq4_mmx,	655	McChromaWidthEq4_mmx,
640	McChromaWidthEq8_sse2	656	McChromaWidthEq8_sse2
641	};	657	};
642	const int32_t kiD8x = iMvX & 0x07;	658	const int32_t kiD8x = iMvX & 0x07;
@@ -656,331 +672,528 @@ void McChroma_sse2 (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int3 · 13 unmodified lines
656	// NEON implementation //	672	// NEON implementation //
657	//***************************************************************************//	673	//***************************************************************************//
658	#if defined(HAVE_NEON)	674	#if defined(HAVE_NEON)
659	void McCopy_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	675	void McCopy_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
660	int32_t iWidth, int32_t iHeight)	676	int32_t iWidth, int32_t iHeight) {
661	{
662	if (16 == iWidth)	677	if (16 == iWidth)
663	McCopyWidthEq16_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	678	McCopyWidthEq16_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
664	else if(8 == iWidth)	679	else if (8 == iWidth)
665	McCopyWidthEq8_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	680	McCopyWidthEq8_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
666	else	681	else
667	McCopyWidthEq4_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	682	McCopyWidthEq4_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
668	}	683	}
669	void McHorVer20_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	684	void McHorVer20_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
670	int32_t iWidth, int32_t iHeight)	685	int32_t iWidth, int32_t iHeight) {
671	{
672	if (iWidth == 16)	686	if (iWidth == 16)
673	McHorVer20WidthEq16_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	687	McHorVer20WidthEq16_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
674	else if (iWidth == 8)	688	else if (iWidth == 8)
675	McHorVer20WidthEq8_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	689	McHorVer20WidthEq8_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
676	else if (iWidth == 4)	690	else if (iWidth == 4)
677	McHorVer20WidthEq4_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	691	McHorVer20WidthEq4_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
678	}	692	}
679	void McHorVer02_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	693	void McHorVer02_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
680	int32_t iWidth, int32_t iHeight)	694	int32_t iWidth, int32_t iHeight) {
681	{	695	if (iWidth == 16)
682	if (iWidth == 16)	696	McHorVer02WidthEq16_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
683	McHorVer02WidthEq16_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	697	else if (iWidth == 8)
684	else if (iWidth == 8)	698	McHorVer02WidthEq8_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
685	McHorVer02WidthEq8_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	699	else if (iWidth == 4)
686	else if (iWidth == 4)	700	McHorVer02WidthEq4_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
687	McHorVer02WidthEq4_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	701	}
688	}	702	void McHorVer22_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
689	void McHorVer22_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	703	int32_t iWidth, int32_t iHeight) {
690	int32_t iWidth, int32_t iHeight)	704	if (iWidth == 16)
691	{	705	McHorVer22WidthEq16_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
692	if (iWidth == 16)	706	else if (iWidth == 8)
693	McHorVer22WidthEq16_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	707	McHorVer22WidthEq8_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
694	else if (iWidth == 8)	708	else if (iWidth == 4)
695	McHorVer22WidthEq8_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	709	McHorVer22WidthEq4_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
696	else if (iWidth == 4)
697	McHorVer22WidthEq4_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);
698	}	710	}
699		711
700	void McHorVer01_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	712	void McHorVer01_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
701	int32_t iWidth, int32_t iHeight)	713	int32_t iWidth, int32_t iHeight) {
702	{
703	if (iWidth == 16)	714	if (iWidth == 16)
704	McHorVer01WidthEq16_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	715	McHorVer01WidthEq16_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
705	else if (iWidth == 8)	716	else if (iWidth == 8)
706	McHorVer01WidthEq8_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	717	McHorVer01WidthEq8_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
707	else if (iWidth == 4)	718	else if (iWidth == 4)
708	McHorVer01WidthEq4_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	719	McHorVer01WidthEq4_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
709	}	720	}
710	void McHorVer03_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	721	void McHorVer03_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
711	int32_t iWidth, int32_t iHeight)	722	int32_t iWidth, int32_t iHeight) {
712	{
713	if (iWidth == 16)	723	if (iWidth == 16)
714	McHorVer03WidthEq16_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	724	McHorVer03WidthEq16_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
715	else if (iWidth == 8)	725	else if (iWidth == 8)
716	McHorVer03WidthEq8_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	726	McHorVer03WidthEq8_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
717	else if (iWidth == 4)	727	else if (iWidth == 4)
718	McHorVer03WidthEq4_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	728	McHorVer03WidthEq4_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
719	}	729	}
720	void McHorVer10_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	730	void McHorVer10_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
721	int32_t iWidth, int32_t iHeight)	731	int32_t iWidth, int32_t iHeight) {
722	{
723	if (iWidth == 16)	732	if (iWidth == 16)
724	McHorVer10WidthEq16_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	733	McHorVer10WidthEq16_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
725	else if (iWidth == 8)	734	else if (iWidth == 8)
726	McHorVer10WidthEq8_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	735	McHorVer10WidthEq8_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
727	else if (iWidth == 4)	736	else if (iWidth == 4)
728	McHorVer10WidthEq4_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	737	McHorVer10WidthEq4_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
729	}	738	}
730	void McHorVer11_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	739	void McHorVer11_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
731	int32_t iWidth, int32_t iHeight)	740	int32_t iWidth, int32_t iHeight) {
732	{	741	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
733	ENFORCE_STACK_ALIGN_1D( uint8_t, pHorTmp, 256, 16 );	742	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
734	ENFORCE_STACK_ALIGN_1D( uint8_t, pVerTmp, 256, 16 );	743	if (iWidth == 16) {
		744	McHorVer20WidthEq16_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
		745	McHorVer02WidthEq16_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		746	PixelAvgWidthEq16_neon (pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
		747	} else if (iWidth == 8) {
		748	McHorVer20WidthEq8_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
		749	McHorVer02WidthEq8_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		750	PixelAvgWidthEq8_neon (pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
		751	} else if (iWidth == 4) {
		752	McHorVer20WidthEq4_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
		753	McHorVer02WidthEq4_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		754	PixelAvgWidthEq4_neon (pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
		755	}
		756	}
		757	void McHorVer12_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		758	int32_t iWidth, int32_t iHeight) {
		759	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
		760	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);
		761	if (iWidth == 16) {
		762	McHorVer02WidthEq16_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		763	McHorVer22WidthEq16_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		764	PixelAvgWidthEq16_neon (pDst, iDstStride, pVerTmp, pCtrTmp, iHeight);
		765	} else if (iWidth == 8) {
		766	McHorVer02WidthEq8_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		767	McHorVer22WidthEq8_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		768	PixelAvgWidthEq8_neon (pDst, iDstStride, pVerTmp, pCtrTmp, iHeight);
		769	} else if (iWidth == 4) {
		770	McHorVer02WidthEq4_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		771	McHorVer22WidthEq4_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		772	PixelAvgWidthEq4_neon (pDst, iDstStride, pVerTmp, pCtrTmp, iHeight);
		773	}
		774	}
		775	void McHorVer13_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		776	int32_t iWidth, int32_t iHeight) {
		777	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
		778	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
		779	if (iWidth == 16) {
		780	McHorVer20WidthEq16_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		781	McHorVer02WidthEq16_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		782	PixelAvgWidthEq16_neon (pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
		783	} else if (iWidth == 8) {
		784	McHorVer20WidthEq8_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		785	McHorVer02WidthEq8_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		786	PixelAvgWidthEq8_neon (pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
		787	} else if (iWidth == 4) {
		788	McHorVer20WidthEq4_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		789	McHorVer02WidthEq4_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		790	PixelAvgWidthEq4_neon (pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
		791	}
		792	}
		793	void McHorVer21_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		794	int32_t iWidth, int32_t iHeight) {
		795	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
		796	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);
		797	if (iWidth == 16) {
		798	McHorVer20WidthEq16_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
		799	McHorVer22WidthEq16_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		800	PixelAvgWidthEq16_neon (pDst, iDstStride, pHorTmp, pCtrTmp, iHeight);
		801	} else if (iWidth == 8) {
		802	McHorVer20WidthEq8_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
		803	McHorVer22WidthEq8_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		804	PixelAvgWidthEq8_neon (pDst, iDstStride, pHorTmp, pCtrTmp, iHeight);
		805	} else if (iWidth == 4) {
		806	McHorVer20WidthEq4_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
		807	McHorVer22WidthEq4_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		808	PixelAvgWidthEq4_neon (pDst, iDstStride, pHorTmp, pCtrTmp, iHeight);
		809	}
		810	}
		811	void McHorVer23_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		812	int32_t iWidth, int32_t iHeight) {
		813	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
		814	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);
		815	if (iWidth == 16) {
		816	McHorVer20WidthEq16_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		817	McHorVer22WidthEq16_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		818	PixelAvgWidthEq16_neon (pDst, iDstStride, pHorTmp, pCtrTmp, iHeight);
		819	} else if (iWidth == 8) {
		820	McHorVer20WidthEq8_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		821	McHorVer22WidthEq8_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		822	PixelAvgWidthEq8_neon (pDst, iDstStride, pHorTmp, pCtrTmp, iHeight);
		823	} else if (iWidth == 4) {
		824	McHorVer20WidthEq4_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		825	McHorVer22WidthEq4_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		826	PixelAvgWidthEq4_neon (pDst, iDstStride, pHorTmp, pCtrTmp, iHeight);
		827	}
		828	}
		829	void McHorVer30_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		830	int32_t iWidth, int32_t iHeight) {
735	if (iWidth == 16)	831	if (iWidth == 16)
736	{	832	McHorVer30WidthEq16_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
737	McHorVer20WidthEq16_neon(pSrc, iSrcStride, pHorTmp, 16, iHeight);	833	else if (iWidth == 8)
738	McHorVer02WidthEq16_neon(pSrc, iSrcStride, pVerTmp, 16, iHeight);	834	McHorVer30WidthEq8_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
739	PixelAvgWidthEq16_neon(pDst, iDstStride, pHorTmp, pVerTmp, iHeight);	835	else if (iWidth == 4)
740	}	836	McHorVer30WidthEq4_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
741	else if (iWidth == 8)	837	}
742	{	838	void McHorVer31_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
743	McHorVer20WidthEq8_neon(pSrc, iSrcStride, pHorTmp, 16, iHeight);	839	int32_t iWidth, int32_t iHeight) {
744	McHorVer02WidthEq8_neon(pSrc, iSrcStride, pVerTmp, 16, iHeight);	840	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
745	PixelAvgWidthEq8_neon(pDst, iDstStride, pHorTmp, pVerTmp, iHeight);	841	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
746	}	842	if (iWidth == 16) {
747	else if (iWidth == 4)	843	McHorVer20WidthEq16_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
748	{	844	McHorVer02WidthEq16_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
749	McHorVer20WidthEq4_neon(pSrc, iSrcStride, pHorTmp, 16, iHeight);	845	PixelAvgWidthEq16_neon (pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
750	McHorVer02WidthEq4_neon(pSrc, iSrcStride, pVerTmp, 16, iHeight);	846	} else if (iWidth == 8) {
751	PixelAvgWidthEq4_neon(pDst, iDstStride, pHorTmp, pVerTmp, iHeight);	847	McHorVer20WidthEq8_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
752	}	848	McHorVer02WidthEq8_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
753	}	849	PixelAvgWidthEq8_neon (pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
754	void McHorVer12_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	850	} else if (iWidth == 4) {
755	int32_t iWidth, int32_t iHeight)	851	McHorVer20WidthEq4_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
756	{	852	McHorVer02WidthEq4_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
757	ENFORCE_STACK_ALIGN_1D( uint8_t, pVerTmp, 256, 16 );	853	PixelAvgWidthEq4_neon (pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
758	ENFORCE_STACK_ALIGN_1D( uint8_t, pCtrTmp, 256, 16 );	854	}
		855	}
		856	void McHorVer32_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		857	int32_t iWidth, int32_t iHeight) {
		858	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
		859	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);
		860	if (iWidth == 16) {
		861	McHorVer02WidthEq16_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
		862	McHorVer22WidthEq16_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		863	PixelAvgWidthEq16_neon (pDst, iDstStride, pVerTmp, pCtrTmp, iHeight);
		864	} else if (iWidth == 8) {
		865	McHorVer02WidthEq8_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
		866	McHorVer22WidthEq8_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		867	PixelAvgWidthEq8_neon (pDst, iDstStride, pVerTmp, pCtrTmp, iHeight);
		868	} else if (iWidth == 4) {
		869	McHorVer02WidthEq4_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
		870	McHorVer22WidthEq4_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		871	PixelAvgWidthEq4_neon (pDst, iDstStride, pVerTmp, pCtrTmp, iHeight);
		872	}
		873	}
		874	void McHorVer33_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		875	int32_t iWidth, int32_t iHeight) {
		876	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
		877	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
		878	if (iWidth == 16) {
		879	McHorVer20WidthEq16_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		880	McHorVer02WidthEq16_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
		881	PixelAvgWidthEq16_neon (pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
		882	} else if (iWidth == 8) {
		883	McHorVer20WidthEq8_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		884	McHorVer02WidthEq8_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
		885	PixelAvgWidthEq8_neon (pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
		886	} else if (iWidth == 4) {
		887	McHorVer20WidthEq4_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		888	McHorVer02WidthEq4_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
		889	PixelAvgWidthEq4_neon (pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
		890	}
		891	}
		892
		893	void McLuma_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		894	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight) {
		895	static PWelsMcWidthHeightFunc pWelsMcFunc[4][4] = { //[x][y]
		896	{McCopy_neon, McHorVer01_neon, McHorVer02_neon, McHorVer03_neon},
		897	{McHorVer10_neon, McHorVer11_neon, McHorVer12_neon, McHorVer13_neon},
		898	{McHorVer20_neon, McHorVer21_neon, McHorVer22_neon, McHorVer23_neon},
		899	{McHorVer30_neon, McHorVer31_neon, McHorVer32_neon, McHorVer33_neon},
		900	};
		901	// pSrc += (iMvY >> 2) * iSrcStride + (iMvX >> 2);
		902	pWelsMcFunc[iMvX & 0x03][iMvY & 0x03] (pSrc, iSrcStride, pDst, iDstStride, iWidth, iHeight);
		903	}
		904	void McChroma_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		905	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight) {
		906	if (0 == iMvX && 0 == iMvY) {
		907	if (8 == iWidth)
		908	McCopyWidthEq8_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
		909	else if (iWidth == 4)
		910	McCopyWidthEq4_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
		911	else //here iWidth == 2
		912	McCopyWidthEq2_c (pSrc, iSrcStride, pDst, iDstStride, iHeight);
		913	} else {
		914	const int32_t kiD8x = iMvX & 0x07;
		915	const int32_t kiD8y = iMvY & 0x07;
		916	if (8 == iWidth)
		917	McChromaWidthEq8_neon (pSrc, iSrcStride, pDst, iDstStride, (int32_t*) (g_kuiABCD[kiD8y][kiD8x]), iHeight);
		918	else if (4 == iWidth)
		919	McChromaWidthEq4_neon (pSrc, iSrcStride, pDst, iDstStride, (int32_t*) (g_kuiABCD[kiD8y][kiD8x]), iHeight);
		920	else //here iWidth == 2
		921	McChromaWithFragMv_c (pSrc, iSrcStride, pDst, iDstStride, iMvX, iMvY, iWidth, iHeight);
		922	}
		923	}
		924	#endif
		925	#if defined(HAVE_NEON_AARCH64)
		926	void McCopy_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		927	int32_t iWidth, int32_t iHeight) {
		928	if (16 == iWidth)
		929	McCopyWidthEq16_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
		930	else if (8 == iWidth)
		931	McCopyWidthEq8_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
		932	else
		933	McCopyWidthEq4_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
		934	}
		935	void McHorVer20_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		936	int32_t iWidth, int32_t iHeight) {
759	if (iWidth == 16)	937	if (iWidth == 16)
760	{	938	McHorVer20WidthEq16_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
761	McHorVer02WidthEq16_neon(pSrc, iSrcStride, pVerTmp, 16, iHeight);	939	else if (iWidth == 8)
762	McHorVer22WidthEq16_neon(pSrc, iSrcStride, pCtrTmp, 16, iHeight);	940	McHorVer20WidthEq8_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
763	PixelAvgWidthEq16_neon(pDst, iDstStride, pVerTmp, pCtrTmp, iHeight);	941	else if (iWidth == 4)
764	}	942	McHorVer20WidthEq4_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
765	else if (iWidth == 8)	943	}
766	{	944	void McHorVer02_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
767	McHorVer02WidthEq8_neon(pSrc, iSrcStride, pVerTmp, 16, iHeight);	945	int32_t iWidth, int32_t iHeight) {
768	McHorVer22WidthEq8_neon(pSrc, iSrcStride, pCtrTmp, 16, iHeight);
769	PixelAvgWidthEq8_neon(pDst, iDstStride, pVerTmp, pCtrTmp, iHeight);
770	}
771	else if (iWidth == 4)
772	{
773	McHorVer02WidthEq4_neon(pSrc, iSrcStride, pVerTmp, 16, iHeight);
774	McHorVer22WidthEq4_neon(pSrc, iSrcStride, pCtrTmp, 16, iHeight);
775	PixelAvgWidthEq4_neon(pDst, iDstStride, pVerTmp, pCtrTmp, iHeight);
776	}
777	}
778	void McHorVer13_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
779	int32_t iWidth, int32_t iHeight)
780	{
781	ENFORCE_STACK_ALIGN_1D( uint8_t, pHorTmp, 256, 16 );
782	ENFORCE_STACK_ALIGN_1D( uint8_t, pVerTmp, 256, 16 );
783	if (iWidth == 16)	946	if (iWidth == 16)
784	{	947	McHorVer02WidthEq16_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
785	McHorVer20WidthEq16_neon(pSrc+iSrcStride, iSrcStride, pHorTmp, 16, iHeight);	948	else if (iWidth == 8)
786	McHorVer02WidthEq16_neon(pSrc, iSrcStride, pVerTmp, 16, iHeight);	949	McHorVer02WidthEq8_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
787	PixelAvgWidthEq16_neon(pDst, iDstStride, pHorTmp, pVerTmp, iHeight);	950	else if (iWidth == 4)
788	}	951	McHorVer02WidthEq4_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
789	else if (iWidth == 8)	952	}
790	{	953	void McHorVer22_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
791	McHorVer20WidthEq8_neon(pSrc+iSrcStride, iSrcStride, pHorTmp, 16, iHeight);	954	int32_t iWidth, int32_t iHeight) {
792	McHorVer02WidthEq8_neon(pSrc, iSrcStride, pVerTmp, 16, iHeight);
793	PixelAvgWidthEq8_neon(pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
794	}
795	else if (iWidth == 4)
796	{
797	McHorVer20WidthEq4_neon(pSrc+iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
798	McHorVer02WidthEq4_neon(pSrc, iSrcStride, pVerTmp, 16, iHeight);
799	PixelAvgWidthEq4_neon(pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
800	}
801	}
802	void McHorVer21_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
803	int32_t iWidth, int32_t iHeight)
804	{
805	ENFORCE_STACK_ALIGN_1D( uint8_t, pHorTmp, 256, 16 );
806	ENFORCE_STACK_ALIGN_1D( uint8_t, pCtrTmp, 256, 16 );
807	if (iWidth == 16)	955	if (iWidth == 16)
808	{	956	McHorVer22WidthEq16_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
809	McHorVer20WidthEq16_neon(pSrc, iSrcStride, pHorTmp, 16, iHeight);	957	else if (iWidth == 8)
810	McHorVer22WidthEq16_neon(pSrc, iSrcStride, pCtrTmp, 16, iHeight);	958	McHorVer22WidthEq8_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
811	PixelAvgWidthEq16_neon(pDst, iDstStride, pHorTmp, pCtrTmp, iHeight);	959	else if (iWidth == 4)
812	}	960	McHorVer22WidthEq4_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
813	else if (iWidth == 8)	961	}
814	{	962
815	McHorVer20WidthEq8_neon(pSrc, iSrcStride, pHorTmp, 16, iHeight);	963	void McHorVer01_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
816	McHorVer22WidthEq8_neon(pSrc, iSrcStride, pCtrTmp, 16, iHeight);	964	int32_t iWidth, int32_t iHeight) {
817	PixelAvgWidthEq8_neon(pDst, iDstStride, pHorTmp, pCtrTmp, iHeight);
818	}
819	else if (iWidth == 4)
820	{
821	McHorVer20WidthEq4_neon(pSrc, iSrcStride, pHorTmp, 16, iHeight);
822	McHorVer22WidthEq4_neon(pSrc, iSrcStride, pCtrTmp, 16, iHeight);
823	PixelAvgWidthEq4_neon(pDst, iDstStride, pHorTmp, pCtrTmp, iHeight);
824	}
825	}
826	void McHorVer23_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
827	int32_t iWidth, int32_t iHeight)
828	{
829	ENFORCE_STACK_ALIGN_1D( uint8_t, pHorTmp, 256, 16 );
830	ENFORCE_STACK_ALIGN_1D( uint8_t, pCtrTmp, 256, 16 );
831	if (iWidth == 16)	965	if (iWidth == 16)
832	{	966	McHorVer01WidthEq16_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
833	McHorVer20WidthEq16_neon(pSrc+iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
834	McHorVer22WidthEq16_neon(pSrc, iSrcStride, pCtrTmp, 16, iHeight);
835	PixelAvgWidthEq16_neon(pDst, iDstStride, pHorTmp, pCtrTmp, iHeight);
836	}
837	else if (iWidth == 8)	967	else if (iWidth == 8)
838	{	968	McHorVer01WidthEq8_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
839	McHorVer20WidthEq8_neon(pSrc+iSrcStride, iSrcStride, pHorTmp, 16, iHeight);	969	else if (iWidth == 4)
840	McHorVer22WidthEq8_neon(pSrc, iSrcStride, pCtrTmp, 16, iHeight);	970	McHorVer01WidthEq4_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
841	PixelAvgWidthEq8_neon(pDst, iDstStride, pHorTmp, pCtrTmp, iHeight);	971	}
842	}	972	void McHorVer03_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
843	else if (iWidth == 4)	973	int32_t iWidth, int32_t iHeight) {
844	{
845	McHorVer20WidthEq4_neon(pSrc+iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
846	McHorVer22WidthEq4_neon(pSrc, iSrcStride, pCtrTmp, 16, iHeight);
847	PixelAvgWidthEq4_neon(pDst, iDstStride, pHorTmp, pCtrTmp, iHeight);
848	}
849	}
850	void McHorVer30_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
851	int32_t iWidth, int32_t iHeight)
852	{
853	if (iWidth == 16)	974	if (iWidth == 16)
854	McHorVer30WidthEq16_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	975	McHorVer03WidthEq16_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
855	else if (iWidth == 8)	976	else if (iWidth == 8)
856	McHorVer30WidthEq8_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	977	McHorVer03WidthEq8_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
857	else if (iWidth == 4)	978	else if (iWidth == 4)
858	McHorVer30WidthEq4_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	979	McHorVer03WidthEq4_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
859	}	980	}
860	void McHorVer31_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	981	void McHorVer10_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
861	int32_t iWidth, int32_t iHeight)	982	int32_t iWidth, int32_t iHeight) {
862	{
863	ENFORCE_STACK_ALIGN_1D( uint8_t, pHorTmp, 256, 16 );
864	ENFORCE_STACK_ALIGN_1D( uint8_t, pVerTmp, 256, 16 );
865	if (iWidth == 16) {
866	McHorVer20WidthEq16_neon(pSrc, iSrcStride, pHorTmp, 16, iHeight);
867	McHorVer02WidthEq16_neon(pSrc+1, iSrcStride, pVerTmp, 16, iHeight);
868	PixelAvgWidthEq16_neon(pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
869	}
870	else if (iWidth == 8){
871	McHorVer20WidthEq8_neon(pSrc, iSrcStride, pHorTmp, 16, iHeight);
872	McHorVer02WidthEq8_neon(pSrc+1, iSrcStride, pVerTmp, 16, iHeight);
873	PixelAvgWidthEq8_neon(pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
874	}
875	else if (iWidth == 4)
876	{
877	McHorVer20WidthEq4_neon(pSrc, iSrcStride, pHorTmp, 16, iHeight);
878	McHorVer02WidthEq4_neon(pSrc+1, iSrcStride, pVerTmp, 16, iHeight);
879	PixelAvgWidthEq4_neon(pDst, iDstStride, pHorTmp, pVerTmp, iHeight);
880	}
881	}
882	void McHorVer32_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
883	int32_t iWidth, int32_t iHeight)
884	{
885	ENFORCE_STACK_ALIGN_1D( uint8_t, pVerTmp, 256, 16 );
886	ENFORCE_STACK_ALIGN_1D( uint8_t, pCtrTmp, 256, 16 );
887	if (iWidth == 16)	983	if (iWidth == 16)
888	{	984	McHorVer10WidthEq16_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
889	McHorVer02WidthEq16_neon(pSrc+1, iSrcStride, pVerTmp, 16, iHeight);	985	else if (iWidth == 8)
890	McHorVer22WidthEq16_neon(pSrc, iSrcStride, pCtrTmp, 16, iHeight);	986	McHorVer10WidthEq8_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
891	PixelAvgWidthEq16_neon(pDst, iDstStride, pVerTmp, pCtrTmp, iHeight);	987	else if (iWidth == 4)
892	}	988	McHorVer10WidthEq4_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
893	else if (iWidth == 8)	989	}
894	{	990	void McHorVer11_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
895	McHorVer02WidthEq8_neon(pSrc+1, iSrcStride, pVerTmp, 16, iHeight);	991	int32_t iWidth, int32_t iHeight) {
896	McHorVer22WidthEq8_neon(pSrc, iSrcStride, pCtrTmp, 16, iHeight);	992	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
897	PixelAvgWidthEq8_neon(pDst, iDstStride, pVerTmp, pCtrTmp, iHeight);	993	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
898	}	994	if (iWidth == 16) {
899	else if (iWidth == 4)	995	McHorVer20WidthEq16_AArch64_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
900	{	996	McHorVer02WidthEq16_AArch64_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
901	McHorVer02WidthEq4_neon(pSrc+1, iSrcStride, pVerTmp, 16, iHeight);	997	PixelAvgWidthEq16_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pVerTmp, 16, iHeight);
902	McHorVer22WidthEq4_neon(pSrc, iSrcStride, pCtrTmp, 16, iHeight);	998	} else if (iWidth == 8) {
903	PixelAvgWidthEq4_neon(pDst, iDstStride, pVerTmp, pCtrTmp, iHeight);	999	McHorVer20WidthEq8_AArch64_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
904	}	1000	McHorVer02WidthEq8_AArch64_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
905	}	1001	PixelAvgWidthEq8_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pVerTmp, 16, iHeight);
906	void McHorVer33_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	1002	} else if (iWidth == 4) {
907	int32_t iWidth, int32_t iHeight)	1003	McHorVer20WidthEq4_AArch64_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
908	{	1004	McHorVer02WidthEq4_AArch64_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
909	ENFORCE_STACK_ALIGN_1D( uint8_t, pHorTmp, 256, 16 );	1005	PixelAvgWidthEq4_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pVerTmp, 16, iHeight);
910	ENFORCE_STACK_ALIGN_1D( uint8_t, pVerTmp, 256, 16 );	1006	}
		1007	}
		1008	void McHorVer12_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		1009	int32_t iWidth, int32_t iHeight) {
		1010	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
		1011	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);
		1012	if (iWidth == 16) {
		1013	McHorVer02WidthEq16_AArch64_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		1014	McHorVer22WidthEq16_AArch64_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		1015	PixelAvgWidthEq16_AArch64_neon (pDst, iDstStride, pVerTmp, 16, pCtrTmp, 16, iHeight);
		1016	} else if (iWidth == 8) {
		1017	McHorVer02WidthEq8_AArch64_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		1018	McHorVer22WidthEq8_AArch64_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		1019	PixelAvgWidthEq8_AArch64_neon (pDst, iDstStride, pVerTmp, 16, pCtrTmp, 16, iHeight);
		1020	} else if (iWidth == 4) {
		1021	McHorVer02WidthEq4_AArch64_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		1022	McHorVer22WidthEq4_AArch64_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		1023	PixelAvgWidthEq4_AArch64_neon (pDst, iDstStride, pVerTmp, 16, pCtrTmp, 16, iHeight);
		1024	}
		1025	}
		1026	void McHorVer13_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		1027	int32_t iWidth, int32_t iHeight) {
		1028	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
		1029	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
		1030	if (iWidth == 16) {
		1031	McHorVer20WidthEq16_AArch64_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		1032	McHorVer02WidthEq16_AArch64_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		1033	PixelAvgWidthEq16_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pVerTmp, 16, iHeight);
		1034	} else if (iWidth == 8) {
		1035	McHorVer20WidthEq8_AArch64_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		1036	McHorVer02WidthEq8_AArch64_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		1037	PixelAvgWidthEq8_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pVerTmp, 16, iHeight);
		1038	} else if (iWidth == 4) {
		1039	McHorVer20WidthEq4_AArch64_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		1040	McHorVer02WidthEq4_AArch64_neon (pSrc, iSrcStride, pVerTmp, 16, iHeight);
		1041	PixelAvgWidthEq4_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pVerTmp, 16, iHeight);
		1042	}
		1043	}
		1044	void McHorVer21_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		1045	int32_t iWidth, int32_t iHeight) {
		1046	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
		1047	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);
		1048	if (iWidth == 16) {
		1049	McHorVer20WidthEq16_AArch64_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
		1050	McHorVer22WidthEq16_AArch64_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		1051	PixelAvgWidthEq16_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pCtrTmp, 16, iHeight);
		1052	} else if (iWidth == 8) {
		1053	McHorVer20WidthEq8_AArch64_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
		1054	McHorVer22WidthEq8_AArch64_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		1055	PixelAvgWidthEq8_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pCtrTmp, 16, iHeight);
		1056	} else if (iWidth == 4) {
		1057	McHorVer20WidthEq4_AArch64_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
		1058	McHorVer22WidthEq4_AArch64_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		1059	PixelAvgWidthEq4_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pCtrTmp, 16, iHeight);
		1060	}
		1061	}
		1062	void McHorVer23_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		1063	int32_t iWidth, int32_t iHeight) {
		1064	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
		1065	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);
		1066	if (iWidth == 16) {
		1067	McHorVer20WidthEq16_AArch64_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		1068	McHorVer22WidthEq16_AArch64_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		1069	PixelAvgWidthEq16_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pCtrTmp, 16, iHeight);
		1070	} else if (iWidth == 8) {
		1071	McHorVer20WidthEq8_AArch64_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		1072	McHorVer22WidthEq8_AArch64_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		1073	PixelAvgWidthEq8_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pCtrTmp, 16, iHeight);
		1074	} else if (iWidth == 4) {
		1075	McHorVer20WidthEq4_AArch64_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		1076	McHorVer22WidthEq4_AArch64_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		1077	PixelAvgWidthEq4_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pCtrTmp, 16, iHeight);
		1078	}
		1079	}
		1080	void McHorVer30_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		1081	int32_t iWidth, int32_t iHeight) {
911	if (iWidth == 16)	1082	if (iWidth == 16)
912	{	1083	McHorVer30WidthEq16_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
913	McHorVer20WidthEq16_neon(pSrc+iSrcStride, iSrcStride, pHorTmp, 16, iHeight);	1084	else if (iWidth == 8)
914	McHorVer02WidthEq16_neon(pSrc+1, iSrcStride, pVerTmp, 16, iHeight);	1085	McHorVer30WidthEq8_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
915	PixelAvgWidthEq16_neon(pDst, iDstStride, pHorTmp, pVerTmp, iHeight);	1086	else if (iWidth == 4)
916	}	1087	McHorVer30WidthEq4_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
917	else if (iWidth == 8)	1088	}
918	{	1089	void McHorVer31_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
919	McHorVer20WidthEq8_neon(pSrc+iSrcStride, iSrcStride, pHorTmp, 16, iHeight);	1090	int32_t iWidth, int32_t iHeight) {
920	McHorVer02WidthEq8_neon(pSrc+1, iSrcStride, pVerTmp, 16, iHeight);	1091	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
921	PixelAvgWidthEq8_neon(pDst, iDstStride, pHorTmp, pVerTmp, iHeight);	1092	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
922	}	1093	if (iWidth == 16) {
923	else if (iWidth == 4)	1094	McHorVer20WidthEq16_AArch64_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
924	{	1095	McHorVer02WidthEq16_AArch64_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
925	McHorVer20WidthEq4_neon(pSrc+iSrcStride, iSrcStride, pHorTmp, 16, iHeight);	1096	PixelAvgWidthEq16_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pVerTmp, 16, iHeight);
926	McHorVer02WidthEq4_neon(pSrc+1, iSrcStride, pVerTmp, 16, iHeight);	1097	} else if (iWidth == 8) {
927	PixelAvgWidthEq4_neon(pDst, iDstStride, pHorTmp, pVerTmp, iHeight);	1098	McHorVer20WidthEq8_AArch64_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
928	}	1099	McHorVer02WidthEq8_AArch64_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
929	}	1100	PixelAvgWidthEq8_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pVerTmp, 16, iHeight);
930		1101	} else if (iWidth == 4) {
931	void McLuma_neon(const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,	1102	McHorVer20WidthEq4_AArch64_neon (pSrc, iSrcStride, pHorTmp, 16, iHeight);
932	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight)	1103	McHorVer02WidthEq4_AArch64_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
933	{	1104	PixelAvgWidthEq4_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pVerTmp, 16, iHeight);
934	static PWelsMcWidthHeightFunc pWelsMcFunc[4][4] = //[x][y]	1105	}
935	{	1106	}
936	{McCopy_neon, McHorVer01_neon, McHorVer02_neon, McHorVer03_neon},	1107	void McHorVer32_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
937	{McHorVer10_neon, McHorVer11_neon, McHorVer12_neon, McHorVer13_neon},	1108	int32_t iWidth, int32_t iHeight) {
938	{McHorVer20_neon, McHorVer21_neon, McHorVer22_neon, McHorVer23_neon},	1109	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
939	{McHorVer30_neon, McHorVer31_neon, McHorVer32_neon, McHorVer33_neon},	1110	ENFORCE_STACK_ALIGN_1D (uint8_t, pCtrTmp, 256, 16);
940	};	1111	if (iWidth == 16) {
		1112	McHorVer02WidthEq16_AArch64_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
		1113	McHorVer22WidthEq16_AArch64_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		1114	PixelAvgWidthEq16_AArch64_neon (pDst, iDstStride, pVerTmp, 16, pCtrTmp, 16, iHeight);
		1115	} else if (iWidth == 8) {
		1116	McHorVer02WidthEq8_AArch64_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
		1117	McHorVer22WidthEq8_AArch64_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		1118	PixelAvgWidthEq8_AArch64_neon (pDst, iDstStride, pVerTmp, 16, pCtrTmp, 16, iHeight);
		1119	} else if (iWidth == 4) {
		1120	McHorVer02WidthEq4_AArch64_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
		1121	McHorVer22WidthEq4_AArch64_neon (pSrc, iSrcStride, pCtrTmp, 16, iHeight);
		1122	PixelAvgWidthEq4_AArch64_neon (pDst, iDstStride, pVerTmp, 16, pCtrTmp, 16, iHeight);
		1123	}
		1124	}
		1125	void McHorVer33_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		1126	int32_t iWidth, int32_t iHeight) {
		1127	ENFORCE_STACK_ALIGN_1D (uint8_t, pHorTmp, 256, 16);
		1128	ENFORCE_STACK_ALIGN_1D (uint8_t, pVerTmp, 256, 16);
		1129	if (iWidth == 16) {
		1130	McHorVer20WidthEq16_AArch64_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		1131	McHorVer02WidthEq16_AArch64_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
		1132	PixelAvgWidthEq16_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pVerTmp, 16, iHeight);
		1133	} else if (iWidth == 8) {
		1134	McHorVer20WidthEq8_AArch64_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		1135	McHorVer02WidthEq8_AArch64_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
		1136	PixelAvgWidthEq8_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pVerTmp, 16, iHeight);
		1137	} else if (iWidth == 4) {
		1138	McHorVer20WidthEq4_AArch64_neon (pSrc + iSrcStride, iSrcStride, pHorTmp, 16, iHeight);
		1139	McHorVer02WidthEq4_AArch64_neon (pSrc + 1, iSrcStride, pVerTmp, 16, iHeight);
		1140	PixelAvgWidthEq4_AArch64_neon (pDst, iDstStride, pHorTmp, 16, pVerTmp, 16, iHeight);
		1141	}
		1142	}
		1143
		1144	void McLuma_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
		1145	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight) {
		1146	static PWelsMcWidthHeightFunc pWelsMcFunc[4][4] = { //[x][y]
		1147	{McCopy_AArch64_neon, McHorVer01_AArch64_neon, McHorVer02_AArch64_neon, McHorVer03_AArch64_neon},
		1148	{McHorVer10_AArch64_neon, McHorVer11_AArch64_neon, McHorVer12_AArch64_neon, McHorVer13_AArch64_neon},
		1149	{McHorVer20_AArch64_neon, McHorVer21_AArch64_neon, McHorVer22_AArch64_neon, McHorVer23_AArch64_neon},
		1150	{McHorVer30_AArch64_neon, McHorVer31_AArch64_neon, McHorVer32_AArch64_neon, McHorVer33_AArch64_neon},
		1151	};
941	// pSrc += (iMvY >> 2) * iSrcStride + (iMvX >> 2);	1152	// pSrc += (iMvY >> 2) * iSrcStride + (iMvX >> 2);
942	pWelsMcFunc[iMvX&0x03][iMvY&0x03](pSrc, iSrcStride, pDst, iDstStride, iWidth, iHeight);	1153	pWelsMcFunc[iMvX & 0x03][iMvY & 0x03] (pSrc, iSrcStride, pDst, iDstStride, iWidth, iHeight);
943	}	1154	}
944	void McChroma_neon(const uint8_t pSrc, int32_t iSrcStride, uint8_t pDst, int32_t iDstStride,	1155	void McChroma_AArch64_neon (const uint8_t* pSrc, int32_t iSrcStride, uint8_t* pDst, int32_t iDstStride,
945	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight)	1156	int16_t iMvX, int16_t iMvY, int32_t iWidth, int32_t iHeight) {
946	{	1157	if (0 == iMvX && 0 == iMvY) {
947	if (0 == iMvX && 0 == iMvY)	1158	if (8 == iWidth)
948	{	1159	McCopyWidthEq8_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
949	if(8 == iWidth)	1160	else if (iWidth == 4)
950	McCopyWidthEq8_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	1161	McCopyWidthEq4_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, iHeight);
951	else if(iWidth == 4)	1162	else //here iWidth == 2
952	McCopyWidthEq4_neon(pSrc, iSrcStride, pDst, iDstStride, iHeight);	1163	McCopyWidthEq2_c (pSrc, iSrcStride, pDst, iDstStride, iHeight);
953	else //here iWidth == 2	1164	} else {
954	McCopyWidthEq2_c(pSrc,iSrcStride,pDst,iDstStride,iHeight);	1165	const int32_t kiD8x = iMvX & 0x07;
955	}	1166	const int32_t kiD8y = iMvY & 0x07;
956	else	1167	if (8 == iWidth)
957	{	1168	McChromaWidthEq8_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, (int32_t*) (g_kuiABCD[kiD8y][kiD8x]), iHeight);
958	const int32_t kiD8x = iMvX & 0x07;	1169	else if (4 == iWidth)
959	const int32_t kiD8y = iMvY & 0x07;	1170	McChromaWidthEq4_AArch64_neon (pSrc, iSrcStride, pDst, iDstStride, (int32_t*) (g_kuiABCD[kiD8y][kiD8x]), iHeight);
960	if(8 == iWidth)	1171	else //here iWidth == 2
961	McChromaWidthEq8_neon(pSrc, iSrcStride, pDst, iDstStride, (int32_t*)(g_kuiABCD[kiD8y][kiD8x]), iHeight);	1172	McChromaWithFragMv_c (pSrc, iSrcStride, pDst, iDstStride, iMvX, iMvY, iWidth, iHeight);
962	else if(4 == iWidth)	1173	}
963	McChromaWidthEq4_neon(pSrc, iSrcStride, pDst, iDstStride, (int32_t*)(g_kuiABCD[kiD8y][kiD8x]), iHeight);
964	else //here iWidth == 2
965	McChromaWithFragMv_c(pSrc, iSrcStride, pDst, iDstStride, iMvX, iMvY, iWidth, iHeight);
966	}
967	}	1174	}
968	#endif	1175	#endif
		1176
969	void InitMcFunc (SMcFunc* pMcFunc, int32_t iCpu) {	1177	void InitMcFunc (SMcFunc* pMcFunc, int32_t iCpu) {
970	pMcFunc->pMcLumaFunc = McLuma_c;	1178	pMcFunc->pMcLumaFunc = McLuma_c;
971	pMcFunc->pMcChromaFunc = McChroma_c;	1179	pMcFunc->pMcChromaFunc = McChroma_c;
972		1180
973	#ifdef HAVE_NEON	1181	#ifdef HAVE_NEON
974	if ( iCpu & WELS_CPU_NEON ) {	1182	if (iCpu & WELS_CPU_NEON) {
975	pMcFunc->pMcLumaFunc = McLuma_neon;	1183	pMcFunc->pMcLumaFunc = McLuma_neon;
976	pMcFunc->pMcChromaFunc = McChroma_neon;	1184	pMcFunc->pMcChromaFunc = McChroma_neon;
977	}	1185	}
		1186	#endif
		1187	#ifdef HAVE_NEON_AARCH64
		1188	if (iCpu & WELS_CPU_NEON) {
		1189	pMcFunc->pMcLumaFunc = McLuma_AArch64_neon;
		1190	pMcFunc->pMcChromaFunc = McChroma_AArch64_neon;
		1191	}
978	#endif	1192	#endif
979
980	#if defined (X86_ASM)	1193	#if defined (X86_ASM)
981	if (iCpu & WELS_CPU_SSE2) {	1194	if (iCpu & WELS_CPU_SSE2) {
982	pMcFunc->pMcLumaFunc = McLuma_sse2;	1195	pMcFunc->pMcLumaFunc = McLuma_sse2;
983	pMcFunc->pMcChromaFunc = McChroma_sse2;	1196	pMcFunc->pMcChromaFunc = McChroma_sse2;
984	}	1197	}
985	#endif //(X86_ASM)	1198	#endif //(X86_ASM)
986	}	1199	}

Add Motion Compehension ARM64 Neon Code

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code