Convert all tabs to spaces in assembly sources, unify indentation

Previously the assembly sources had mixed indentation consisting of both spaces and tabs, making it quite hard to read unless the right tab size was used in the editor. Tabs have been interpreted as 4 spaces in most cases, matching the surrounding code.

Martin Storsjö committed 12 years ago

Commit 57f6bcc1 parent faaf62a38 files · +21316 −21316Browse files

Showing the changes to codec/processing/src (7 of 38 files). Show all

Unified · Split

7 files

codec/processing/src/arm/adaptive_quantization.S +46 −46
codec/processing/src/arm/down_sample_neon.S +218 −218
codec/processing/src/arm/pixel_sad_neon.S +17 −17
codec/processing/src/arm/vaa_calc_neon.S +584 −584
codec/processing/src/x86/denoisefilter.asm +170 −170
codec/processing/src/x86/downsample_bilinear.asm +978 −978
codec/processing/src/x86/vaa.asm +1518 −1518

codec/processing/src/arm/adaptive_quantization.S+46−46

@@ -36,17 +36,17 @@ · 35 unmodified lines
36		36
37	#ifdef __APPLE__	37	#ifdef __APPLE__
38	.macro SQR_ADD_16BYTES	38	.macro SQR_ADD_16BYTES
39	vmull.u8 q3, $0, $0	39	vmull.u8 q3, $0, $0
40	vmull.u8 q8, $1, $1	40	vmull.u8 q8, $1, $1
41	vpadal.u16 $2, q3	41	vpadal.u16 $2, q3
42	vpadal.u16 $2, q8	42	vpadal.u16 $2, q8
43	.endm	43	.endm
44	#else	44	#else
45	.macro SQR_ADD_16BYTES arg0, arg1, arg2	45	.macro SQR_ADD_16BYTES arg0, arg1, arg2
46	vmull.u8 q3, \arg0, \arg0	46	vmull.u8 q3, \arg0, \arg0
47	vmull.u8 q8, \arg1, \arg1	47	vmull.u8 q8, \arg1, \arg1
48	vpadal.u16 \arg2, q3	48	vpadal.u16 \arg2, q3
49	vpadal.u16 \arg2, q8	49	vpadal.u16 \arg2, q8
50	.endm	50	.endm
51	#endif	51	#endif
52		52
@@ -54,66 +54,66 @@ · 1 unmodified line
54	WELS_ASM_FUNC_BEGIN SampleVariance16x16_neon	54	WELS_ASM_FUNC_BEGIN SampleVariance16x16_neon
55	stmdb sp!, {r4}	55	stmdb sp!, {r4}
56		56
57	vld1.8 {q15}, [r0], r1 //save the ref data (16bytes)	57	vld1.8 {q15}, [r0], r1 //save the ref data (16bytes)
58	vld1.8 {q14}, [r2], r3 //save the src data (16bytes)	58	vld1.8 {q14}, [r2], r3 //save the src data (16bytes)
59		59
60		60
61	vabd.u8 q13, q14, q15	61	vabd.u8 q13, q14, q15
62	vmull.u8 q12, d27, d27	62	vmull.u8 q12, d27, d27
63	vmull.u8 q11, d26, d26	63	vmull.u8 q11, d26, d26
64	vaddl.u16 q12, d24, d25	64	vaddl.u16 q12, d24, d25
65	vpadal.u16 q12, q11 //sqr	65	vpadal.u16 q12, q11 //sqr
66		66
67	vaddl.u8 q13, d26, d27 //sum	67	vaddl.u8 q13, d26, d27 //sum
68		68
69	vaddl.u8 q10, d28, d29 //sum_cur	69	vaddl.u8 q10, d28, d29 //sum_cur
70		70
71	vmull.u8 q9, d29, d29	71	vmull.u8 q9, d29, d29
72	vmull.u8 q8, d28, d28	72	vmull.u8 q8, d28, d28
73	vaddl.u16 q9, d18, d19 //sqr_cur	73	vaddl.u16 q9, d18, d19 //sqr_cur
74	vpadal.u16 q9, q8	74	vpadal.u16 q9, q8
75		75
76	mov r4, #15	76	mov r4, #15
77	pixel_var_16x16_loop0:	77	pixel_var_16x16_loop0:
78		78
79	vld1.8 {q0}, [r0], r1 //save the ref data (16bytes)	79	vld1.8 {q0}, [r0], r1 //save the ref data (16bytes)
80	vld1.8 {q1}, [r2], r3 //save the src data (16bytes)	80	vld1.8 {q1}, [r2], r3 //save the src data (16bytes)
81		81
82	vabd.u8 q2, q0, q1	82	vabd.u8 q2, q0, q1
83		83
84	//q10 save sum_cur	84	//q10 save sum_cur
85	vpadal.u8 q10, q1	85	vpadal.u8 q10, q1
86		86
87	//q12 save sqr	87	//q12 save sqr
88	SQR_ADD_16BYTES d4, d5, q12	88	SQR_ADD_16BYTES d4, d5, q12
89		89
90	//q13 save sum	90	//q13 save sum
91	vpadal.u8 q13, q2	91	vpadal.u8 q13, q2
92		92
93	subs r4, #1	93	subs r4, #1
94		94
95	//q9 save sqr_cur	95	//q9 save sqr_cur
96	SQR_ADD_16BYTES d2, d3, q9	96	SQR_ADD_16BYTES d2, d3, q9
97		97
98	bne pixel_var_16x16_loop0	98	bne pixel_var_16x16_loop0
99		99
100	vadd.u16 d0, d26, d27 //sum	100	vadd.u16 d0, d26, d27 //sum
101	vadd.u16 d1, d20, d21 //sum_cur	101	vadd.u16 d1, d20, d21 //sum_cur
102	vpaddl.u16 q0, q0	102	vpaddl.u16 q0, q0
103	vadd.u32 d2, d24, d25 //sqr	103	vadd.u32 d2, d24, d25 //sqr
104	vadd.u32 d3, d18, d19 //sqr_cur	104	vadd.u32 d3, d18, d19 //sqr_cur
105	vpadd.u32 d0, d0, d1	105	vpadd.u32 d0, d0, d1
106	vpadd.u32 d1, d2, d3	106	vpadd.u32 d1, d2, d3
107		107
108	ldr r4, [sp, #4]	108	ldr r4, [sp, #4]
109		109
110	vshr.u32 q0, q0, #8	110	vshr.u32 q0, q0, #8
111	vmul.u32 d0, d0	111	vmul.u32 d0, d0
112	vsub.u32 d0, d1, d0	112	vsub.u32 d0, d1, d0
113	vmovl.u32 q0, d0	113	vmovl.u32 q0, d0
114	vst2.16 {d0[0], d1[0]}, [r4]	114	vst2.16 {d0[0], d1[0]}, [r4]
115		115
116	ldmia sp!, {r4}	116	ldmia sp!, {r4}
117		117
118	WELS_ASM_FUNC_END	118	WELS_ASM_FUNC_END
119		119

codec/processing/src/arm/down_sample_neon.S+218−218

@@ -30,313 +30,313 @@ · 29 unmodified lines
30	*	30	*
31	*/	31	*/
32		32
33	#ifdef HAVE_NEON	33	#ifdef HAVE_NEON
34	.text	34	.text
35	#include "arm_arch_common_macro.S"	35	#include "arm_arch_common_macro.S"
36		36
37		37
38	WELS_ASM_FUNC_BEGIN DyadicBilinearDownsampler_neon	38	WELS_ASM_FUNC_BEGIN DyadicBilinearDownsampler_neon
39	stmdb sp!, {r4-r8, lr}	39	stmdb sp!, {r4-r8, lr}
40		40
41	//Get the width and height	41	//Get the width and height
42	ldr r4, [sp, #24] //src_width	42	ldr r4, [sp, #24] //src_width
43	ldr r5, [sp, #28] //src_height	43	ldr r5, [sp, #28] //src_height
44		44
45	//Initialize the register	45	//Initialize the register
46	mov r6, r2	46	mov r6, r2
47	mov r8, r0	47	mov r8, r0
48	mov lr, #0	48	mov lr, #0
49	lsr r5, #1	49	lsr r5, #1
50		50
51	//Save the tailer for the unasigned size	51	//Save the tailer for the unasigned size
52	mla r7, r1, r5, r0	52	mla r7, r1, r5, r0
53	vld1.32 {q15}, [r7]	53	vld1.32 {q15}, [r7]
54		54
55	add r7, r2, r3	55	add r7, r2, r3
56	//processing a colume data	56	//processing a colume data
57	comp_ds_bilinear_loop0:	57	comp_ds_bilinear_loop0:
58		58
59	vld1.8 {q0,q1}, [r2]!	59	vld1.8 {q0,q1}, [r2]!
60	vld1.8 {q2,q3}, [r7]!	60	vld1.8 {q2,q3}, [r7]!
61	vpaddl.u8 q0, q0	61	vpaddl.u8 q0, q0
62	vpaddl.u8 q1, q1	62	vpaddl.u8 q1, q1
63	vpaddl.u8 q2, q2	63	vpaddl.u8 q2, q2
64	vpaddl.u8 q3, q3	64	vpaddl.u8 q3, q3
65	vrshr.u16 q0, #1	65	vrshr.u16 q0, #1
66	vrshr.u16 q1, #1	66	vrshr.u16 q1, #1
67	vrshr.u16 q2, #1	67	vrshr.u16 q2, #1
68	vrshr.u16 q3, #1	68	vrshr.u16 q3, #1
69	vrhadd.u16 q0, q2	69	vrhadd.u16 q0, q2
70	vrhadd.u16 q1, q3	70	vrhadd.u16 q1, q3
71	vmovn.u16 d0, q0	71	vmovn.u16 d0, q0
72	vmovn.u16 d1, q1	72	vmovn.u16 d1, q1
73	vst1.32 {q0}, [r0]!	73	vst1.32 {q0}, [r0]!
74	add lr, #32	74	add lr, #32
75		75
76	cmp lr, r4	76	cmp lr, r4
77	movcs lr, #0	77	movcs lr, #0
78	addcs r6, r6, r3, lsl #1	78	addcs r6, r6, r3, lsl #1
79	movcs r2, r6	79	movcs r2, r6
80	addcs r7, r2, r3	80	addcs r7, r2, r3
81	addcs r8, r1	81	addcs r8, r1
82	movcs r0, r8	82	movcs r0, r8
83	subscs r5, #1	83	subscs r5, #1
84	bne comp_ds_bilinear_loop0	84	bne comp_ds_bilinear_loop0
85		85
86	//restore the tailer for the unasigned size	86	//restore the tailer for the unasigned size
87	vst1.32 {q15}, [r0]	87	vst1.32 {q15}, [r0]
88		88
89	ldmia sp!, {r4-r8,lr}	89	ldmia sp!, {r4-r8,lr}
90	WELS_ASM_FUNC_END	90	WELS_ASM_FUNC_END
91		91
92		92
93	WELS_ASM_FUNC_BEGIN comp_ds_bilinear_w_x8_neon	93	WELS_ASM_FUNC_BEGIN comp_ds_bilinear_w_x8_neon
94	stmdb sp!, {r4-r7, lr}	94	stmdb sp!, {r4-r7, lr}
95		95
96	//Get the width and height	96	//Get the width and height
97	ldr r4, [sp, #20] //src_width	97	ldr r4, [sp, #20] //src_width
98	ldr r5, [sp, #24] //src_height	98	ldr r5, [sp, #24] //src_height
99		99
100	//Get the difference	100	//Get the difference
101	sub lr, r3, r4	101	sub lr, r3, r4
102	sub r1, r1, r4, lsr #1	102	sub r1, r1, r4, lsr #1
103		103
104	lsr r5, #1	104	lsr r5, #1
105		105
106	//processing a colume data	106	//processing a colume data
107	comp_ds_bilinear_w_x8_loop0:	107	comp_ds_bilinear_w_x8_loop0:
108		108
109	lsr r6, r4, #3	109	lsr r6, r4, #3
110	add r7, r2, r3	110	add r7, r2, r3
111	//processing a line data	111	//processing a line data
112	comp_ds_bilinear_w_x8_loop1:	112	comp_ds_bilinear_w_x8_loop1:
113		113
114	vld1.8 {d0}, [r2]!	114	vld1.8 {d0}, [r2]!
115	vld1.8 {d1}, [r7]!	115	vld1.8 {d1}, [r7]!
116	vpaddl.u8 q0, q0	116	vpaddl.u8 q0, q0
117	vrshr.u16 q0, #1	117	vrshr.u16 q0, #1
118	vrhadd.u16 d0, d1	118	vrhadd.u16 d0, d1
119		119
120	vmovn.u16 d0, q0	120	vmovn.u16 d0, q0
121	vst1.32 {d0[0]}, [r0]!	121	vst1.32 {d0[0]}, [r0]!
122	subs r6, #1	122	subs r6, #1
123	bne comp_ds_bilinear_w_x8_loop1	123	bne comp_ds_bilinear_w_x8_loop1
124		124
125	add r2, r7, lr	125	add r2, r7, lr
126	add r0, r1	126	add r0, r1
127	subs r5, #1	127	subs r5, #1
128	bne comp_ds_bilinear_w_x8_loop0	128	bne comp_ds_bilinear_w_x8_loop0
129		129
130	ldmia sp!, {r4-r7,lr}	130	ldmia sp!, {r4-r7,lr}
131	WELS_ASM_FUNC_END	131	WELS_ASM_FUNC_END
132		132
133		133
134	WELS_ASM_FUNC_BEGIN comp_ds_bilinear_w_x16_neon	134	WELS_ASM_FUNC_BEGIN comp_ds_bilinear_w_x16_neon
135	stmdb sp!, {r4-r7, lr}	135	stmdb sp!, {r4-r7, lr}
136		136
137	//Get the width and height	137	//Get the width and height
138	ldr r4, [sp, #20] //src_width	138	ldr r4, [sp, #20] //src_width
139	ldr r5, [sp, #24] //src_height	139	ldr r5, [sp, #24] //src_height
140		140
141	//Get the difference	141	//Get the difference
142	sub lr, r3, r4	142	sub lr, r3, r4
143	sub r1, r1, r4, lsr #1	143	sub r1, r1, r4, lsr #1
144		144
145	lsr r5, #1	145	lsr r5, #1
146		146
147	//processing a colume data	147	//processing a colume data
148	comp_ds_bilinear_w_x16_loop0:	148	comp_ds_bilinear_w_x16_loop0:
149		149
150	lsr r6, r4, #4	150	lsr r6, r4, #4
151	add r7, r2, r3	151	add r7, r2, r3
152	//processing a line data	152	//processing a line data
153	comp_ds_bilinear_w_x16_loop1:	153	comp_ds_bilinear_w_x16_loop1:
154		154
155	vld1.8 {q0}, [r2]!	155	vld1.8 {q0}, [r2]!
156	vld1.8 {q1}, [r7]!	156	vld1.8 {q1}, [r7]!
157	vpaddl.u8 q0, q0	157	vpaddl.u8 q0, q0
158	vpaddl.u8 q1, q1	158	vpaddl.u8 q1, q1
159	vrshr.u16 q0, #1	159	vrshr.u16 q0, #1
160	vrshr.u16 q1, #1	160	vrshr.u16 q1, #1
161	vrhadd.u16 q0, q1	161	vrhadd.u16 q0, q1
162		162
163	vmovn.u16 d0, q0	163	vmovn.u16 d0, q0
164	vst1.32 {d0}, [r0]!	164	vst1.32 {d0}, [r0]!
165	subs r6, #1	165	subs r6, #1
166	bne comp_ds_bilinear_w_x16_loop1	166	bne comp_ds_bilinear_w_x16_loop1
167		167
168	add r2, r7, lr	168	add r2, r7, lr
169	add r0, r1	169	add r0, r1
170	subs r5, #1	170	subs r5, #1
171	bne comp_ds_bilinear_w_x16_loop0	171	bne comp_ds_bilinear_w_x16_loop0
172		172
173	ldmia sp!, {r4-r7,lr}	173	ldmia sp!, {r4-r7,lr}
174	WELS_ASM_FUNC_END	174	WELS_ASM_FUNC_END
175		175
176		176
177	WELS_ASM_FUNC_BEGIN DyadicBilinearDownsamplerWidthx32_neon	177	WELS_ASM_FUNC_BEGIN DyadicBilinearDownsamplerWidthx32_neon
178	stmdb sp!, {r4-r7, lr}	178	stmdb sp!, {r4-r7, lr}
179		179
180	//Get the width and height	180	//Get the width and height
181	ldr r4, [sp, #20] //src_width	181	ldr r4, [sp, #20] //src_width
182	ldr r5, [sp, #24] //src_height	182	ldr r5, [sp, #24] //src_height
183		183
184	//Get the difference	184	//Get the difference
185	sub lr, r3, r4	185	sub lr, r3, r4
186	sub r1, r1, r4, lsr #1	186	sub r1, r1, r4, lsr #1
187		187
188	lsr r5, #1	188	lsr r5, #1
189		189
190	//processing a colume data	190	//processing a colume data
191	comp_ds_bilinear_w_x32_loop0:	191	comp_ds_bilinear_w_x32_loop0:
192		192
193	lsr r6, r4, #5	193	lsr r6, r4, #5
194	add r7, r2, r3	194	add r7, r2, r3
195	//processing a line data	195	//processing a line data
196	comp_ds_bilinear_w_x32_loop1:	196	comp_ds_bilinear_w_x32_loop1:
197		197
198	vld1.8 {q0,q1}, [r2]!	198	vld1.8 {q0,q1}, [r2]!
199	vld1.8 {q2,q3}, [r7]!	199	vld1.8 {q2,q3}, [r7]!
200	vpaddl.u8 q0, q0	200	vpaddl.u8 q0, q0
201	vpaddl.u8 q1, q1	201	vpaddl.u8 q1, q1
202	vpaddl.u8 q2, q2	202	vpaddl.u8 q2, q2
203	vpaddl.u8 q3, q3	203	vpaddl.u8 q3, q3
204	vrshr.u16 q0, #1	204	vrshr.u16 q0, #1
205	vrshr.u16 q1, #1	205	vrshr.u16 q1, #1
206	vrshr.u16 q2, #1	206	vrshr.u16 q2, #1
207	vrshr.u16 q3, #1	207	vrshr.u16 q3, #1
208	vrhadd.u16 q0, q2	208	vrhadd.u16 q0, q2
209	vrhadd.u16 q1, q3	209	vrhadd.u16 q1, q3
210		210
211	vmovn.u16 d0, q0	211	vmovn.u16 d0, q0
212	vmovn.u16 d1, q1	212	vmovn.u16 d1, q1
213	vst1.32 {q0}, [r0]!	213	vst1.32 {q0}, [r0]!
214	subs r6, #1	214	subs r6, #1
215	bne comp_ds_bilinear_w_x32_loop1	215	bne comp_ds_bilinear_w_x32_loop1
216		216
217	add r2, r7, lr	217	add r2, r7, lr
218	add r0, r1	218	add r0, r1
219	subs r5, #1	219	subs r5, #1
220	bne comp_ds_bilinear_w_x32_loop0	220	bne comp_ds_bilinear_w_x32_loop0
221		221
222	ldmia sp!, {r4-r7,lr}	222	ldmia sp!, {r4-r7,lr}
223	WELS_ASM_FUNC_END	223	WELS_ASM_FUNC_END
224		224
225		225
226	WELS_ASM_FUNC_BEGIN GeneralBilinearAccurateDownsampler_neon	226	WELS_ASM_FUNC_BEGIN GeneralBilinearAccurateDownsampler_neon
227	stmdb sp!, {r4-r12, lr}	227	stmdb sp!, {r4-r12, lr}
228		228
229	//Get the data from stack	229	//Get the data from stack
230	ldr r4, [sp, #40] //the addr of src	230	ldr r4, [sp, #40] //the addr of src
231	ldr r5, [sp, #44] //the value of src_stride	231	ldr r5, [sp, #44] //the value of src_stride
232	ldr r6, [sp, #48] //the value of scaleX	232	ldr r6, [sp, #48] //the value of scaleX
233	ldr r7, [sp, #52] //the value of scaleY	233	ldr r7, [sp, #52] //the value of scaleY
234		234
235	mov r10, #32768	235	mov r10, #32768
236	sub r10, #1	236	sub r10, #1
237	and r8, r6, r10 // r8 uinc(scaleX mod 32767)	237	and r8, r6, r10 // r8 uinc(scaleX mod 32767)
238	mov r11, #-1	238	mov r11, #-1
239	mul r11, r8 // r11 -uinc	239	mul r11, r8 // r11 -uinc
240		240
241	vdup.s16 d2, r8	241	vdup.s16 d2, r8
242	vdup.s16 d0, r11	242	vdup.s16 d0, r11
243	vzip.s16 d0, d2 // uinc -uinc uinc -uinc	243	vzip.s16 d0, d2 // uinc -uinc uinc -uinc
244		244
245	and r9, r7, r10 // r9 vinc(scaleY mod 32767)	245	and r9, r7, r10 // r9 vinc(scaleY mod 32767)
246	mov r11, #-1	246	mov r11, #-1
247	mul r11, r9 // r11 -vinc	247	mul r11, r9 // r11 -vinc
248		248
249	vdup.s16 d2, r9	249	vdup.s16 d2, r9
250	vdup.s16 d3, r11	250	vdup.s16 d3, r11
251	vext.8 d5, d3, d2, #4 // vinc vinc -vinc -vinc	251	vext.8 d5, d3, d2, #4 // vinc vinc -vinc -vinc
252		252
253	mov r11, #0x40000000	253	mov r11, #0x40000000
254	mov r12, #0x4000	254	mov r12, #0x4000
255	sub r12, #1	255	sub r12, #1
256	add r11, r12	256	add r11, r12
257	vdup.s32 d1, r11; //init u 16384 16383 16384 16383	257	vdup.s32 d1, r11; //init u 16384 16383 16384 16383
258		258
259	mov r11, #16384	259	mov r11, #16384
260	vdup.s16 d16, r11	260	vdup.s16 d16, r11
261	sub r11, #1	261	sub r11, #1
262	vdup.s16 d17, r11	262	vdup.s16 d17, r11
263	vext.8 d7, d17, d16, #4 //init v 16384 16384 16383 16383	263	vext.8 d7, d17, d16, #4 //init v 16384 16384 16383 16383
264		264
265	veor q14, q14	265	veor q14, q14
266	sub r1, r2 // stride - width	266	sub r1, r2 // stride - width
267	mov r8, #16384 // yInverse	267	mov r8, #16384 // yInverse
268	sub r3, #1	268	sub r3, #1
269		269
270	_HEIGHT:	270	_HEIGHT:
271	ldr r4, [sp, #40] //the addr of src	271	ldr r4, [sp, #40] //the addr of src
272	mov r11, r8	272	mov r11, r8
273	lsr r11, #15	273	lsr r11, #15
274	mul r11, r5	274	mul r11, r5
275	add r11, r4 // get current row address	275	add r11, r4 // get current row address
276	mov r12, r11	276	mov r12, r11
277	add r12, r5	277	add r12, r5
278		278
279	mov r9, #16384 // xInverse	279	mov r9, #16384 // xInverse
280	sub r10, r2, #1	280	sub r10, r2, #1
281	vmov.s16 d6, d1	281	vmov.s16 d6, d1
282		282
283	_WIDTH:	283	_WIDTH:
284	mov lr, r9	284	mov lr, r9
285	lsr lr, #15	285	lsr lr, #15
286	add r4, r11,lr	286	add r4, r11,lr
287	vld2.8 {d28[0],d29[0]}, [r4] //q14: 0000000b0000000a;	287	vld2.8 {d28[0],d29[0]}, [r4] //q14: 0000000b0000000a;
288	add r4, r12,lr	288	add r4, r12,lr
289	vld2.8 {d28[4],d29[4]}, [r4] //q14: 000d000b000c000a;	289	vld2.8 {d28[4],d29[4]}, [r4] //q14: 000d000b000c000a;
290	vzip.32 d28, d29 //q14: 000d000c000b000a;	290	vzip.32 d28, d29 //q14: 000d000c000b000a;
291		291
292	vmull.u16 q13, d6, d7 //q13: init u * init v	292	vmull.u16 q13, d6, d7 //q13: init u * init v
293	vmull.u32 q12, d26,d28	293	vmull.u32 q12, d26,d28
294	vmlal.u32 q12, d27,d29	294	vmlal.u32 q12, d27,d29
295	vqadd.u64 d24, d24,d25	295	vqadd.u64 d24, d24,d25
296	vrshr.u64 d24, #30	296	vrshr.u64 d24, #30
297		297
298	vst1.8 {d24[0]}, [r0]!	298	vst1.8 {d24[0]}, [r0]!
299	add r9, r6	299	add r9, r6
300	vadd.u16 d6, d0 // inc u	300	vadd.u16 d6, d0 // inc u
301	vshl.u16 d6, #1	301	vshl.u16 d6, #1
302	vshr.u16 d6, #1	302	vshr.u16 d6, #1
303	subs r10, #1	303	subs r10, #1
304	bne _WIDTH	304	bne _WIDTH
305		305
306	WIDTH_END:	306	WIDTH_END:
307	lsr r9, #15	307	lsr r9, #15
308	add r4,r11,r9	308	add r4,r11,r9
309	vld1.8 {d24[0]}, [r4]	309	vld1.8 {d24[0]}, [r4]
310	vst1.8 {d24[0]}, [r0]	310	vst1.8 {d24[0]}, [r0]
311	add r0, #1	311	add r0, #1
312	add r8, r7	312	add r8, r7
313	add r0, r1	313	add r0, r1
314	vadd.s16 d7, d5 // inc v	314	vadd.s16 d7, d5 // inc v
315	vshl.u16 d7, #1	315	vshl.u16 d7, #1
316	vshr.u16 d7, #1	316	vshr.u16 d7, #1
317	subs r3, #1	317	subs r3, #1
318	bne _HEIGHT	318	bne _HEIGHT
319		319
320	LAST_ROW:	320	LAST_ROW:
321	ldr r4, [sp, #40] //the addr of src	321	ldr r4, [sp, #40] //the addr of src
322	lsr r8, #15	322	lsr r8, #15
323	mul r8, r5	323	mul r8, r5
324	add r4, r8 // get current row address	324	add r4, r8 // get current row address
325	mov r9, #16384	325	mov r9, #16384
326		326
327	_LAST_ROW_WIDTH:	327	_LAST_ROW_WIDTH:
328	mov r11, r9	328	mov r11, r9
329	lsr r11, #15	329	lsr r11, #15
330		330
331	add r3, r4,r11	331	add r3, r4,r11
332	vld1.8 {d0[0]}, [r3]	332	vld1.8 {d0[0]}, [r3]
333	vst1.8 {d0[0]}, [r0]	333	vst1.8 {d0[0]}, [r0]
334	add r0, #1	334	add r0, #1
335	add r9, r6	335	add r9, r6
336	subs r2, #1	336	subs r2, #1
337	bne _LAST_ROW_WIDTH	337	bne _LAST_ROW_WIDTH
338		338
339	ldmia sp!, {r4-r12, lr}	339	ldmia sp!, {r4-r12, lr}
340	WELS_ASM_FUNC_END	340	WELS_ASM_FUNC_END
341		341
342	#endif	342	#endif

codec/processing/src/arm/pixel_sad_neon.S+17−17

@@ -37,32 +37,32 @@ · 36 unmodified lines
37		37
38	WELS_ASM_FUNC_BEGIN WelsProcessingSampleSad8x8_neon	38	WELS_ASM_FUNC_BEGIN WelsProcessingSampleSad8x8_neon
39	stmdb sp!, {lr}	39	stmdb sp!, {lr}
40	//Loading a horizontal line data (8 bytes)	40	//Loading a horizontal line data (8 bytes)
41	vld1.8 {d0}, [r0], r1	41	vld1.8 {d0}, [r0], r1
42	vld1.8 {d1}, [r2], r3	42	vld1.8 {d1}, [r2], r3
43		43
44	//Do the SAD for 8 bytes	44	//Do the SAD for 8 bytes
45	vabdl.u8 q1, d0, d1	45	vabdl.u8 q1, d0, d1
46		46
47	mov lr, #7	47	mov lr, #7
48	pixel_sad_8x8_loop0:	48	pixel_sad_8x8_loop0:
49		49
50	//Loading a horizontal line data (8 bytes)	50	//Loading a horizontal line data (8 bytes)
51	vld1.8 {d0}, [r0], r1	51	vld1.8 {d0}, [r0], r1
52	vld1.8 {d1}, [r2], r3	52	vld1.8 {d1}, [r2], r3
53		53
54	subs lr, #1	54	subs lr, #1
55		55
56	//Do the SAD for 8 bytes	56	//Do the SAD for 8 bytes
57	vabal.u8 q1, d0, d1	57	vabal.u8 q1, d0, d1
58	bne pixel_sad_8x8_loop0	58	bne pixel_sad_8x8_loop0
59		59
60	vadd.u16 d2, d3	60	vadd.u16 d2, d3
61	vpaddl.u16 d2, d2	61	vpaddl.u16 d2, d2
62	vpaddl.u32 d2, d2	62	vpaddl.u32 d2, d2
63	vmov.u32 r0, d2[0]//TBO...	63	vmov.u32 r0, d2[0]//TBO...
64		64
65	ldmia sp!, {lr}	65	ldmia sp!, {lr}
66	WELS_ASM_FUNC_END	66	WELS_ASM_FUNC_END
67		67
68	#endif	68	#endif

codec/processing/src/arm/vaa_calc_neon.S+584−584

@@ -37,61 +37,61 @@ · 36 unmodified lines
37	#ifdef __APPLE__	37	#ifdef __APPLE__
38		38
39	.macro ABS_SUB_SUM_16BYTES	39	.macro ABS_SUB_SUM_16BYTES
40	vld1.32 {q15}, [$0], $2	40	vld1.32 {q15}, [$0], $2
41	vld1.32 {q14}, [$1], $2	41	vld1.32 {q14}, [$1], $2
42	vabal.u8 $3, d30, d28	42	vabal.u8 $3, d30, d28
43	vabal.u8 $4, d31, d29	43	vabal.u8 $4, d31, d29
44	.endm	44	.endm
45		45
46	.macro ABS_SUB_SUM_8x16BYTES	46	.macro ABS_SUB_SUM_8x16BYTES
47	vld1.32 {q15}, [$0], $2	47	vld1.32 {q15}, [$0], $2
48	vld1.32 {q14}, [$1], $2	48	vld1.32 {q14}, [$1], $2
49	vabdl.u8 $3, d30, d28	49	vabdl.u8 $3, d30, d28
50	vabdl.u8 $4, d31, d29	50	vabdl.u8 $4, d31, d29
51		51
52	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4	52	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4
53	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4	53	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4
54	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4	54	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4
55	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4	55	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4
56	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4	56	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4
57	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4	57	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4
58	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4	58	ABS_SUB_SUM_16BYTES $0, $1, $2, $3, $4
59	.endm	59	.endm
60		60
61	.macro SAD_8X16BITS	61	.macro SAD_8X16BITS
62	vadd.u16 d31, $0, $1	62	vadd.u16 d31, $0, $1
63	vpaddl.u16 d31, d31	63	vpaddl.u16 d31, d31
64	vpaddl.u32 $2, d31	64	vpaddl.u32 $2, d31
65	.endm	65	.endm
66		66
67	#else	67	#else
68		68
69	.macro ABS_SUB_SUM_16BYTES arg0, arg1, arg2, arg3, arg4	69	.macro ABS_SUB_SUM_16BYTES arg0, arg1, arg2, arg3, arg4
70	vld1.32 {q15}, [\arg0], \arg2	70	vld1.32 {q15}, [\arg0], \arg2
71	vld1.32 {q14}, [\arg1], \arg2	71	vld1.32 {q14}, [\arg1], \arg2
72	vabal.u8 \arg3, d30, d28	72	vabal.u8 \arg3, d30, d28
73	vabal.u8 \arg4, d31, d29	73	vabal.u8 \arg4, d31, d29
74	.endm	74	.endm
75		75
76	.macro ABS_SUB_SUM_8x16BYTES arg0, arg1, arg2, arg3, arg4	76	.macro ABS_SUB_SUM_8x16BYTES arg0, arg1, arg2, arg3, arg4
77	vld1.32 {q15}, [\arg0], \arg2	77	vld1.32 {q15}, [\arg0], \arg2
78	vld1.32 {q14}, [\arg1], \arg2	78	vld1.32 {q14}, [\arg1], \arg2
79	vabdl.u8 \arg3, d30, d28	79	vabdl.u8 \arg3, d30, d28
80	vabdl.u8 \arg4, d31, d29	80	vabdl.u8 \arg4, d31, d29
81		81
82	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4	82	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4
83	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4	83	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4
84	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4	84	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4
85	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4	85	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4
86	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4	86	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4
87	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4	87	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4
88	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4	88	ABS_SUB_SUM_16BYTES \arg0, \arg1, \arg2, \arg3, \arg4
89	.endm	89	.endm
90		90
91	.macro SAD_8X16BITS arg0, arg1, arg2	91	.macro SAD_8X16BITS arg0, arg1, arg2
92	vadd.u16 d31, \arg0, \arg1	92	vadd.u16 d31, \arg0, \arg1
93	vpaddl.u16 d31, d31	93	vpaddl.u16 d31, d31
94	vpaddl.u32 \arg2, d31	94	vpaddl.u32 \arg2, d31
95	.endm	95	.endm
96	#endif	96	#endif
97		97
@@ -100,16 +100,16 @@ WELS_ASM_FUNC_BEGIN VAACalcSad_neon · 2 unmodified lines
100		100
101	stmdb sp!, {r4-r8}	101	stmdb sp!, {r4-r8}
102		102
103	ldr r4, [sp, #20] //load pic_stride	103	ldr r4, [sp, #20] //load pic_stride
104	ldr r5, [sp, #28] //load psad8x8	104	ldr r5, [sp, #28] //load psad8x8
105		105
106	//Initial the Q8 register for save the "psadframe"	106	//Initial the Q8 register for save the "psadframe"
107	vmov.s64 q8, #0	107	vmov.s64 q8, #0
108		108
109	//Get the jump distance to use on loop codes	109	//Get the jump distance to use on loop codes
110	lsl r8, r4, #4	110	lsl r8, r4, #4
111	sub r7, r8, #16 //R7 keep the 16*pic_stride-16	111	sub r7, r8, #16 //R7 keep the 16*pic_stride-16
112	sub r8, r2 //R8 keep the 16*pic_stride-pic_width	112	sub r8, r2 //R8 keep the 16*pic_stride-pic_width
113		113
114	vaa_calc_sad_loop0:	114	vaa_calc_sad_loop0:
115		115
@@ -118,70 +118,70 @@ vaa_calc_sad_loop0: · 2 unmodified lines
118		118
119	vaa_calc_sad_loop1:	119	vaa_calc_sad_loop1:
120		120
121	//Process the 16x16 bytes	121	//Process the 16x16 bytes
122	ABS_SUB_SUM_8x16BYTES r0, r1, r4, q0, q1	122	ABS_SUB_SUM_8x16BYTES r0, r1, r4, q0, q1
123	ABS_SUB_SUM_8x16BYTES r0, r1, r4, q2, q3	123	ABS_SUB_SUM_8x16BYTES r0, r1, r4, q2, q3
124		124
125	//Do the SAD	125	//Do the SAD
126	SAD_8X16BITS d0, d1, d0	126	SAD_8X16BITS d0, d1, d0
127	SAD_8X16BITS d2, d3, d1	127	SAD_8X16BITS d2, d3, d1
128	SAD_8X16BITS d4, d5, d2	128	SAD_8X16BITS d4, d5, d2
129	SAD_8X16BITS d6, d7, d3	129	SAD_8X16BITS d6, d7, d3
130		130
131	//Write to "psad8x8" buffer	131	//Write to "psad8x8" buffer
132	vst4.32 {d0[0],d1[0],d2[0],d3[0]}, [r5]!	132	vst4.32 {d0[0],d1[0],d2[0],d3[0]}, [r5]!
133		133
134		134
135	//Adjust the input address	135	//Adjust the input address
136	sub r0, r7	136	sub r0, r7
137	sub r1, r7	137	sub r1, r7
138		138
139	subs r6, #16	139	subs r6, #16
140		140
141	//Save to calculate "psadframe"	141	//Save to calculate "psadframe"
142	vadd.u32 q0, q1	142	vadd.u32 q0, q1
143	vadd.u32 q8, q0	143	vadd.u32 q8, q0
144		144
145	bne vaa_calc_sad_loop1	145	bne vaa_calc_sad_loop1
146		146
147	//Adjust the input address	147	//Adjust the input address
148	add r0, r8	148	add r0, r8
149	add r1, r8	149	add r1, r8
150		150
151	subs r3, #16	151	subs r3, #16
152	bne vaa_calc_sad_loop0	152	bne vaa_calc_sad_loop0
153		153
154	ldr r6, [sp, #24] //load psadframe	154	ldr r6, [sp, #24] //load psadframe
155	vadd.u32 d16, d17	155	vadd.u32 d16, d17
156	vst1.32 {d16[0]}, [r6]	156	vst1.32 {d16[0]}, [r6]
157		157
158	ldmia sp!, {r4-r8}	158	ldmia sp!, {r4-r8}
159		159
160	WELS_ASM_FUNC_END	160	WELS_ASM_FUNC_END
161		161
162		162
163	#ifdef __APPLE__	163	#ifdef __APPLE__
164	.macro SAD_SD_MAD_16BYTES	164	.macro SAD_SD_MAD_16BYTES
165	vld1.32 {q0}, [$0], $2	165	vld1.32 {q0}, [$0], $2
166	vld1.32 {q1}, [$1], $2	166	vld1.32 {q1}, [$1], $2
167		167
168	vpadal.u8 $3, q0	168	vpadal.u8 $3, q0
169	vpadal.u8 $4, q1	169	vpadal.u8 $4, q1
170		170
171	vabd.u8 q0, q0, q1	171	vabd.u8 q0, q0, q1
172	vmax.u8 $5, q0	172	vmax.u8 $5, q0
173	vpadal.u8 $6, q0	173	vpadal.u8 $6, q0
174	.endm	174	.endm
175		175
176	.macro SAD_SD_MAD_8x16BYTES	176	.macro SAD_SD_MAD_8x16BYTES
177	vld1.32 {q0}, [$0], $2	177	vld1.32 {q0}, [$0], $2
178	vld1.32 {q1}, [$1], $2	178	vld1.32 {q1}, [$1], $2
179		179
180	vpaddl.u8 q2, q0	180	vpaddl.u8 q2, q0
181	vpaddl.u8 q3, q1	181	vpaddl.u8 q3, q1
182		182
183	vabd.u8 $3, q0, q1	183	vabd.u8 $3, q0, q1
184	vpaddl.u8 $4, $3 //abs_diff	184	vpaddl.u8 $4, $3 //abs_diff
185		185
186		186
187	SAD_SD_MAD_16BYTES $0,$1,$2,q2,q3,$3,$4	187	SAD_SD_MAD_16BYTES $0,$1,$2,q2,q3,$3,$4
@@ -192,41 +192,41 @@ WELS_ASM_FUNC_END · 4 unmodified lines
192	SAD_SD_MAD_16BYTES $0,$1,$2,q2,q3,$3,$4	192	SAD_SD_MAD_16BYTES $0,$1,$2,q2,q3,$3,$4
193	SAD_SD_MAD_16BYTES $0,$1,$2,q2,q3,$3,$4	193	SAD_SD_MAD_16BYTES $0,$1,$2,q2,q3,$3,$4
194		194
195	vsub.u16 $5, q2, q3	195	vsub.u16 $5, q2, q3
196	.endm	196	.endm
197		197
198	.macro SAD_SD_MAD_CALC	198	.macro SAD_SD_MAD_CALC
199	vpmax.u8 d0, $0, $1 //8bytes	199	vpmax.u8 d0, $0, $1 //8bytes
200	vpmax.u8 d0, d0, d0 //4bytes	200	vpmax.u8 d0, d0, d0 //4bytes
201	vpmax.u8 $2, d0, d0 //2bytes	201	vpmax.u8 $2, d0, d0 //2bytes
202		202
203	vpaddl.u16 $3, $3	203	vpaddl.u16 $3, $3
204	vpaddl.u32 $3, $3	204	vpaddl.u32 $3, $3
205	vpaddl.s16 $4, $4	205	vpaddl.s16 $4, $4
206	vpaddl.s32 $4, $4	206	vpaddl.s32 $4, $4
207	.endm	207	.endm
208	#else	208	#else
209	.macro SAD_SD_MAD_16BYTES arg0, arg1, arg2, arg3, arg4, arg5, arg6	209	.macro SAD_SD_MAD_16BYTES arg0, arg1, arg2, arg3, arg4, arg5, arg6
210	vld1.32 {q0}, [\arg0], \arg2	210	vld1.32 {q0}, [\arg0], \arg2
211	vld1.32 {q1}, [\arg1], \arg2	211	vld1.32 {q1}, [\arg1], \arg2
212		212
213	vpadal.u8 \arg3, q0	213	vpadal.u8 \arg3, q0
214	vpadal.u8 \arg4, q1	214	vpadal.u8 \arg4, q1
215		215
216	vabd.u8 q0, q0, q1	216	vabd.u8 q0, q0, q1
217	vmax.u8 \arg5, q0	217	vmax.u8 \arg5, q0
218	vpadal.u8 \arg6, q0	218	vpadal.u8 \arg6, q0
219	.endm	219	.endm
220		220
221	.macro SAD_SD_MAD_8x16BYTES arg0, arg1, arg2, arg3, arg4, arg5	221	.macro SAD_SD_MAD_8x16BYTES arg0, arg1, arg2, arg3, arg4, arg5
222	vld1.32 {q0}, [\arg0], \arg2	222	vld1.32 {q0}, [\arg0], \arg2
223	vld1.32 {q1}, [\arg1], \arg2	223	vld1.32 {q1}, [\arg1], \arg2
224		224
225	vpaddl.u8 q2, q0	225	vpaddl.u8 q2, q0
226	vpaddl.u8 q3, q1	226	vpaddl.u8 q3, q1
227		227
228	vabd.u8 \arg3, q0, q1	228	vabd.u8 \arg3, q0, q1
229	vpaddl.u8 \arg4, \arg3 //abs_diff	229	vpaddl.u8 \arg4, \arg3 //abs_diff
230		230
231		231
232	SAD_SD_MAD_16BYTES \arg0,\arg1,\arg2,q2,q3,\arg3,\arg4	232	SAD_SD_MAD_16BYTES \arg0,\arg1,\arg2,q2,q3,\arg3,\arg4
@@ -237,18 +237,18 @@ WELS_ASM_FUNC_END · 4 unmodified lines
237	SAD_SD_MAD_16BYTES \arg0,\arg1,\arg2,q2,q3,\arg3,\arg4	237	SAD_SD_MAD_16BYTES \arg0,\arg1,\arg2,q2,q3,\arg3,\arg4
238	SAD_SD_MAD_16BYTES \arg0,\arg1,\arg2,q2,q3,\arg3,\arg4	238	SAD_SD_MAD_16BYTES \arg0,\arg1,\arg2,q2,q3,\arg3,\arg4
239		239
240	vsub.u16 \arg5, q2, q3	240	vsub.u16 \arg5, q2, q3
241	.endm	241	.endm
242		242
243	.macro SAD_SD_MAD_CALC arg0, arg1, arg2, arg3, arg4	243	.macro SAD_SD_MAD_CALC arg0, arg1, arg2, arg3, arg4
244	vpmax.u8 d0, \arg0, \arg1 //8bytes	244	vpmax.u8 d0, \arg0, \arg1 //8bytes
245	vpmax.u8 d0, d0, d0 //4bytes	245	vpmax.u8 d0, d0, d0 //4bytes
246	vpmax.u8 \arg2, d0, d0 //2bytes	246	vpmax.u8 \arg2, d0, d0 //2bytes
247		247
248	vpaddl.u16 \arg3, \arg3	248	vpaddl.u16 \arg3, \arg3
249	vpaddl.u32 \arg3, \arg3	249	vpaddl.u32 \arg3, \arg3
250	vpaddl.s16 \arg4, \arg4	250	vpaddl.s16 \arg4, \arg4
251	vpaddl.s32 \arg4, \arg4	251	vpaddl.s32 \arg4, \arg4
252	.endm	252	.endm
253	#endif	253	#endif
254		254
@@ -256,18 +256,18 @@ WELS_ASM_FUNC_BEGIN VAACalcSadBgd_neon · 1 unmodified line
256		256
257	stmdb sp!, {r4-r10}	257	stmdb sp!, {r4-r10}
258		258
259	ldr r4, [sp, #28] //load pic_stride	259	ldr r4, [sp, #28] //load pic_stride
260	ldr r5, [sp, #36] //load psad8x8	260	ldr r5, [sp, #36] //load psad8x8
261	ldr r6, [sp, #40] //load psd8x8	261	ldr r6, [sp, #40] //load psd8x8
262	ldr r7, [sp, #44] //load pmad8x8	262	ldr r7, [sp, #44] //load pmad8x8
263		263
264	//Initial the Q4 register for save the "psadframe"	264	//Initial the Q4 register for save the "psadframe"
265	vmov.s64 q15, #0	265	vmov.s64 q15, #0
266		266
267	//Get the jump distance to use on loop codes	267	//Get the jump distance to use on loop codes
268	lsl r10, r4, #4	268	lsl r10, r4, #4
269	sub r9, r10, #16 //R9 keep the 16*pic_stride-16	269	sub r9, r10, #16 //R9 keep the 16*pic_stride-16
270	sub r10, r2 //R10 keep the 16*pic_stride-pic_width	270	sub r10, r2 //R10 keep the 16*pic_stride-pic_width
271		271
272	vaa_calc_sad_bgd_loop0:	272	vaa_calc_sad_bgd_loop0:
273		273
@@ -276,391 +276,391 @@ vaa_calc_sad_bgd_loop0: · 2 unmodified lines
276		276
277	vaa_calc_sad_bgd_loop1:	277	vaa_calc_sad_bgd_loop1:
278		278
279	//Process the 16x16 bytes pmad psad psd	279	//Process the 16x16 bytes pmad psad psd
280	SAD_SD_MAD_8x16BYTES r0, r1, r4, q13, q11, q9	280	SAD_SD_MAD_8x16BYTES r0, r1, r4, q13, q11, q9
281	SAD_SD_MAD_8x16BYTES r0, r1, r4, q14, q12, q10	281	SAD_SD_MAD_8x16BYTES r0, r1, r4, q14, q12, q10
282		282
283	SAD_SD_MAD_CALC d26, d27, d16, q11, q9	283	SAD_SD_MAD_CALC d26, d27, d16, q11, q9
284	SAD_SD_MAD_CALC d28, d29, d17, q12, q10	284	SAD_SD_MAD_CALC d28, d29, d17, q12, q10
285		285
286	//Write to "psad8x8" buffer	286	//Write to "psad8x8" buffer
287	vst4.32 {d22[0],d23[0],d24[0],d25[0]}, [r5]!	287	vst4.32 {d22[0],d23[0],d24[0],d25[0]}, [r5]!
288	//Adjust the input address	288	//Adjust the input address
289	sub r0, r9	289	sub r0, r9
290	sub r1, r9	290	sub r1, r9
291	//Write to "psd8x8" buffer	291	//Write to "psd8x8" buffer
292	vst4.32 {d18[0],d19[0],d20[0],d21[0]}, [r6]!	292	vst4.32 {d18[0],d19[0],d20[0],d21[0]}, [r6]!
293	subs r8, #16	293	subs r8, #16
294	//Write to "pmad8x8" buffer	294	//Write to "pmad8x8" buffer
295	vst2.16 {d16[0],d17[0]}, [r7]!	295	vst2.16 {d16[0],d17[0]}, [r7]!
296	//Save to calculate "psadframe"	296	//Save to calculate "psadframe"
297	vadd.u32 q11, q12	297	vadd.u32 q11, q12
298	vadd.u32 q15, q11	298	vadd.u32 q15, q11
299		299
300	bne vaa_calc_sad_bgd_loop1	300	bne vaa_calc_sad_bgd_loop1
301		301
302	//Adjust the input address	302	//Adjust the input address
303	add r0, r10	303	add r0, r10
304	add r1, r10	304	add r1, r10
305		305
306	subs r3, #16	306	subs r3, #16
307	bne vaa_calc_sad_bgd_loop0	307	bne vaa_calc_sad_bgd_loop0
308		308
309	ldr r8, [sp, #32] //load psadframe	309	ldr r8, [sp, #32] //load psadframe
310	vadd.u32 d30, d31	310	vadd.u32 d30, d31
311	vst1.32 {d30[0]}, [r8]	311	vst1.32 {d30[0]}, [r8]
312	ldmia sp!, {r4-r10}	312	ldmia sp!, {r4-r10}
313		313
314	WELS_ASM_FUNC_END	314	WELS_ASM_FUNC_END
315		315
316		316
317	#ifdef __APPLE__	317	#ifdef __APPLE__
318	.macro SSD_MUL_SUM_16BYTES_RESET	318	.macro SSD_MUL_SUM_16BYTES_RESET
319	vmull.u8 $3, $0, $0	319	vmull.u8 $3, $0, $0
320	vpaddl.u16 $2, $3	320	vpaddl.u16 $2, $3
321		321
322	vmull.u8 $3, $1, $1	322	vmull.u8 $3, $1, $1
323	vpadal.u16 $2, $3	323	vpadal.u16 $2, $3
324	.endm	324	.endm
325		325
326	.macro SSD_MUL_SUM_16BYTES	326	.macro SSD_MUL_SUM_16BYTES
327	vmull.u8 $3, $0, $0	327	vmull.u8 $3, $0, $0
328	vpadal.u16 $2, $3	328	vpadal.u16 $2, $3
329		329
330	vmull.u8 $3, $1, $1	330	vmull.u8 $3, $1, $1
331	vpadal.u16 $2, $3	331	vpadal.u16 $2, $3
332	.endm	332	.endm
333		333
334	.macro SAD_SSD_BGD_16	334	.macro SAD_SSD_BGD_16
335	vld1.8 {q0}, [$0], $2 //load cur_row	335	vld1.8 {q0}, [$0], $2 //load cur_row
336		336
337	vpadal.u8 q3, q0 //add cur_row together	337	vpadal.u8 q3, q0 //add cur_row together
338	vpadal.u8 q4, q1 //add ref_row together	338	vpadal.u8 q4, q1 //add ref_row together
339		339
340	vabd.u8 q2, q0, q1 //abs_diff	340	vabd.u8 q2, q0, q1 //abs_diff
341		341
342	vmax.u8 q5, q2 //l_mad for 16 bytes reset for every 8x16	342	vmax.u8 q5, q2 //l_mad for 16 bytes reset for every 8x16
343		343
344	vpadal.u8 $3, q2 //l_sad for 16 bytes reset for every 8x16	344	vpadal.u8 $3, q2 //l_sad for 16 bytes reset for every 8x16
345		345
346	SSD_MUL_SUM_16BYTES d4,d5, q8, q11 //q8 for l_sqiff reset for every 16x16	346	SSD_MUL_SUM_16BYTES d4,d5, q8, q11 //q8 for l_sqiff reset for every 16x16
347		347
348	vld1.8 {q1}, [$1], $2 //load ref_row	348	vld1.8 {q1}, [$1], $2 //load ref_row
349	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16	349	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16
350		350
351	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16	351	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16
352	.endm	352	.endm
353		353
354	//the last row of a 16x16 block	354	//the last row of a 16x16 block
355	.macro SAD_SSD_BGD_16_end	355	.macro SAD_SSD_BGD_16_end
356	vld1.8 {q0}, [$0], $1 //load cur_row	356	vld1.8 {q0}, [$0], $1 //load cur_row
357		357
358	vpadal.u8 q3, q0 //add cur_row together	358	vpadal.u8 q3, q0 //add cur_row together
359	vpadal.u8 q4, q1 //add ref_row together	359	vpadal.u8 q4, q1 //add ref_row together
360		360
361	vabd.u8 q2, q0, q1 //abs_diff	361	vabd.u8 q2, q0, q1 //abs_diff
362		362
363	vmax.u8 q5, q2 //l_mad for 16 bytes reset for every 8x16	363	vmax.u8 q5, q2 //l_mad for 16 bytes reset for every 8x16
364		364
365	vpadal.u8 $2, q2 //l_sad for 16 bytes reset for every 8x16	365	vpadal.u8 $2, q2 //l_sad for 16 bytes reset for every 8x16
366		366
367	SSD_MUL_SUM_16BYTES d4,d5, q8, q11 //q8 for l_sqiff reset for every 16x16	367	SSD_MUL_SUM_16BYTES d4,d5, q8, q11 //q8 for l_sqiff reset for every 16x16
368		368
369	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16	369	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16
370		370
371	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16	371	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16
372	.endm	372	.endm
373		373
374	//for the begin of a 8x16 block, use some instructions to reset the register	374	//for the begin of a 8x16 block, use some instructions to reset the register
375	.macro SAD_SSD_BGD_16_RESET_8x8	375	.macro SAD_SSD_BGD_16_RESET_8x8
376	vld1.8 {q0}, [$0], $2 //load cur_row	376	vld1.8 {q0}, [$0], $2 //load cur_row
377		377
378	vpaddl.u8 q3, q0 //add cur_row together	378	vpaddl.u8 q3, q0 //add cur_row together
379	vpaddl.u8 q4, q1 //add ref_row together	379	vpaddl.u8 q4, q1 //add ref_row together
380		380
381	vabd.u8 q2, q0, q1 //abs_diff	381	vabd.u8 q2, q0, q1 //abs_diff
382		382
383	vmov q5,q2 //calculate max and avoid reset to zero, l_mad for 16 bytes reset for every 8x16	383	vmov q5,q2 //calculate max and avoid reset to zero, l_mad for 16 bytes reset for every 8x16
384		384
385	vpaddl.u8 $3, q2 //l_sad for 16 bytes reset for every 8x16	385	vpaddl.u8 $3, q2 //l_sad for 16 bytes reset for every 8x16
386		386
387		387
388	SSD_MUL_SUM_16BYTES d4,d5, q8, q11 //q8 for l_sqiff reset for every 16x16	388	SSD_MUL_SUM_16BYTES d4,d5, q8, q11 //q8 for l_sqiff reset for every 16x16
389		389
390	vld1.8 {q1}, [$1], $2 //load ref_row	390	vld1.8 {q1}, [$1], $2 //load ref_row
391		391
392	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16	392	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16
393		393
394	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16	394	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16
395	.endm	395	.endm
396		396
397	//for the begin of a 16x16 block, use some instructions to reset the register	397	//for the begin of a 16x16 block, use some instructions to reset the register
398	.macro SAD_SSD_BGD_16_RESET_16x16	398	.macro SAD_SSD_BGD_16_RESET_16x16
399	vld1.8 {q0}, [$0], $2 //load cur_row	399	vld1.8 {q0}, [$0], $2 //load cur_row
400	vld1.8 {q1}, [$1], $2 //load ref_row	400	vld1.8 {q1}, [$1], $2 //load ref_row
401		401
402	vpaddl.u8 q3, q0 //add cur_row together	402	vpaddl.u8 q3, q0 //add cur_row together
403	vpaddl.u8 q4, q1 //add ref_row together	403	vpaddl.u8 q4, q1 //add ref_row together
404		404
405	vabd.u8 q2, q0, q1 //abs_diff	405	vabd.u8 q2, q0, q1 //abs_diff
406		406
407	vmov q5,q2 //calculate max and avoid reset to zero, l_mad for 16 bytes reset for every 8x16	407	vmov q5,q2 //calculate max and avoid reset to zero, l_mad for 16 bytes reset for every 8x16
408		408
409	vpaddl.u8 $3, q2 //l_sad for 16 bytes reset for every 8x16	409	vpaddl.u8 $3, q2 //l_sad for 16 bytes reset for every 8x16
410		410
411	SSD_MUL_SUM_16BYTES_RESET d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16	411	SSD_MUL_SUM_16BYTES_RESET d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16
412		412
413	vld1.8 {q1}, [$1], $2 //load ref_row	413	vld1.8 {q1}, [$1], $2 //load ref_row
414		414
415	vpaddl.u8 q9, q0 //q9 for l_sum reset for every 16x16	415	vpaddl.u8 q9, q0 //q9 for l_sum reset for every 16x16
416		416
417	SSD_MUL_SUM_16BYTES_RESET d0,d1,q10,q11 //q10 for lsqsum reset for every 16x16	417	SSD_MUL_SUM_16BYTES_RESET d0,d1,q10,q11 //q10 for lsqsum reset for every 16x16
418	.endm	418	.endm
419		419
420	//for each 8x16 block	420	//for each 8x16 block
421	.macro SAD_SSD_BGD_CALC_8x16	421	.macro SAD_SSD_BGD_CALC_8x16
422		422
423	vpmax.u8 d10, d10, d11 //4 numbers	423	vpmax.u8 d10, d10, d11 //4 numbers
424	vpmax.u8 d10, d10, d10 //2 numbers	424	vpmax.u8 d10, d10, d10 //2 numbers
425	vpmax.u8 d10, d10, d10 //1 number1	425	vpmax.u8 d10, d10, d10 //1 number1
426		426
427	vmov $0, d10 //d26 d27 keeps the l_mad	427	vmov $0, d10 //d26 d27 keeps the l_mad
428		428
429	//p_sd8x8 fix me	429	//p_sd8x8 fix me
430	vpaddl.u16 q3, q3	430	vpaddl.u16 q3, q3
431	vpaddl.u16 q4, q4	431	vpaddl.u16 q4, q4
432		432
433	vsub.i32 $1, q3, q4	433	vsub.i32 $1, q3, q4
434	vpaddl.u32 $1, $1	434	vpaddl.u32 $1, $1
435		435
436	//psad8x8	436	//psad8x8
437	vpaddl.u16 $2, $2	437	vpaddl.u16 $2, $2
438	vpaddl.u32 $2, $2	438	vpaddl.u32 $2, $2
439		439
440	//psadframe	440	//psadframe
441	vadd.i32 q12, $2	441	vadd.i32 q12, $2
442	.endm	442	.endm
443		443
444	.macro SAD_SSD_BGD_16x16	444	.macro SAD_SSD_BGD_16x16
445	//for one 8x16	445	//for one 8x16
446	SAD_SSD_BGD_16_RESET_16x16 $0, $1, $2, q6	446	SAD_SSD_BGD_16_RESET_16x16 $0, $1, $2, q6
447	SAD_SSD_BGD_16 $0, $1, $2, q6	447	SAD_SSD_BGD_16 $0, $1, $2, q6
448	SAD_SSD_BGD_16 $0, $1, $2, q6	448	SAD_SSD_BGD_16 $0, $1, $2, q6
449	SAD_SSD_BGD_16 $0, $1, $2, q6	449	SAD_SSD_BGD_16 $0, $1, $2, q6
450	SAD_SSD_BGD_16 $0, $1, $2, q6	450	SAD_SSD_BGD_16 $0, $1, $2, q6
451	SAD_SSD_BGD_16 $0, $1, $2, q6	451	SAD_SSD_BGD_16 $0, $1, $2, q6
452	SAD_SSD_BGD_16 $0, $1, $2, q6	452	SAD_SSD_BGD_16 $0, $1, $2, q6
453	SAD_SSD_BGD_16 $0, $1, $2, q6	453	SAD_SSD_BGD_16 $0, $1, $2, q6
454		454
455	SAD_SSD_BGD_CALC_8x16 d26, q14, q6	455	SAD_SSD_BGD_CALC_8x16 d26, q14, q6
456		456
457	//for another 8x16	457	//for another 8x16
458	SAD_SSD_BGD_16_RESET_8x8 $0, $1, $2, q7	458	SAD_SSD_BGD_16_RESET_8x8 $0, $1, $2, q7
459	SAD_SSD_BGD_16 $0, $1, $2, q7	459	SAD_SSD_BGD_16 $0, $1, $2, q7
460	SAD_SSD_BGD_16 $0, $1, $2, q7	460	SAD_SSD_BGD_16 $0, $1, $2, q7
461	SAD_SSD_BGD_16 $0, $1, $2, q7	461	SAD_SSD_BGD_16 $0, $1, $2, q7
462	SAD_SSD_BGD_16 $0, $1, $2, q7	462	SAD_SSD_BGD_16 $0, $1, $2, q7
463	SAD_SSD_BGD_16 $0, $1, $2, q7	463	SAD_SSD_BGD_16 $0, $1, $2, q7
464	SAD_SSD_BGD_16 $0, $1, $2, q7	464	SAD_SSD_BGD_16 $0, $1, $2, q7
465	SAD_SSD_BGD_16_end $0, $2, q7	465	SAD_SSD_BGD_16_end $0, $2, q7
466		466
467	SAD_SSD_BGD_CALC_8x16 d27, q15, q7	467	SAD_SSD_BGD_CALC_8x16 d27, q15, q7
468	.endm	468	.endm
469		469
470	.macro SSD_SAD_SD_MAD_PADDL	470	.macro SSD_SAD_SD_MAD_PADDL
471	vpaddl.s16 $0, $0	471	vpaddl.s16 $0, $0
472	vpaddl.s32 $0, $0	472	vpaddl.s32 $0, $0
473	vadd.i32 $1, $1, $2	473	vadd.i32 $1, $1, $2
474	.endm	474	.endm
475	#else	475	#else
476	.macro SSD_MUL_SUM_16BYTES_RESET arg0, arg1, arg2, arg3	476	.macro SSD_MUL_SUM_16BYTES_RESET arg0, arg1, arg2, arg3
477	vmull.u8 \arg3, \arg0, \arg0	477	vmull.u8 \arg3, \arg0, \arg0
478	vpaddl.u16 \arg2, \arg3	478	vpaddl.u16 \arg2, \arg3
479		479
480	vmull.u8 \arg3, \arg1, \arg1	480	vmull.u8 \arg3, \arg1, \arg1
481	vpadal.u16 \arg2, \arg3	481	vpadal.u16 \arg2, \arg3
482	.endm	482	.endm
483		483
484	.macro SSD_MUL_SUM_16BYTES arg0, arg1, arg2, arg3	484	.macro SSD_MUL_SUM_16BYTES arg0, arg1, arg2, arg3
485	vmull.u8 \arg3, \arg0, \arg0	485	vmull.u8 \arg3, \arg0, \arg0
486	vpadal.u16 \arg2, \arg3	486	vpadal.u16 \arg2, \arg3
487		487
488	vmull.u8 \arg3, \arg1, \arg1	488	vmull.u8 \arg3, \arg1, \arg1
489	vpadal.u16 \arg2, \arg3	489	vpadal.u16 \arg2, \arg3
490	.endm	490	.endm
491		491
492	.macro SAD_SSD_BGD_16 arg0, arg1, arg2, arg3	492	.macro SAD_SSD_BGD_16 arg0, arg1, arg2, arg3
493	vld1.8 {q0}, [\arg0], \arg2 //load cur_row	493	vld1.8 {q0}, [\arg0], \arg2 //load cur_row
494		494
495	vpadal.u8 q3, q0 //add cur_row together	495	vpadal.u8 q3, q0 //add cur_row together
496	vpadal.u8 q4, q1 //add ref_row together	496	vpadal.u8 q4, q1 //add ref_row together
497		497
498	vabd.u8 q2, q0, q1 //abs_diff	498	vabd.u8 q2, q0, q1 //abs_diff
499		499
500	vmax.u8 q5, q2 //l_mad for 16 bytes reset for every 8x16	500	vmax.u8 q5, q2 //l_mad for 16 bytes reset for every 8x16
501		501
502	vpadal.u8 \arg3, q2 //l_sad for 16 bytes reset for every 8x16	502	vpadal.u8 \arg3, q2 //l_sad for 16 bytes reset for every 8x16
503		503
504	SSD_MUL_SUM_16BYTES d4,d5, q8, q11 //q8 for l_sqiff reset for every 16x16	504	SSD_MUL_SUM_16BYTES d4,d5, q8, q11 //q8 for l_sqiff reset for every 16x16
505		505
506	vld1.8 {q1}, [\arg1], \arg2 //load ref_row	506	vld1.8 {q1}, [\arg1], \arg2 //load ref_row
507	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16	507	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16
508		508
509	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16	509	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16
510	.endm	510	.endm
511		511
512	//the last row of a 16x16 block	512	//the last row of a 16x16 block
513	.macro SAD_SSD_BGD_16_end arg0, arg1, arg2	513	.macro SAD_SSD_BGD_16_end arg0, arg1, arg2
514	vld1.8 {q0}, [\arg0], \arg1 //load cur_row	514	vld1.8 {q0}, [\arg0], \arg1 //load cur_row
515		515
516	vpadal.u8 q3, q0 //add cur_row together	516	vpadal.u8 q3, q0 //add cur_row together
517	vpadal.u8 q4, q1 //add ref_row together	517	vpadal.u8 q4, q1 //add ref_row together
518		518
519	vabd.u8 q2, q0, q1 //abs_diff	519	vabd.u8 q2, q0, q1 //abs_diff
520		520
521	vmax.u8 q5, q2 //l_mad for 16 bytes reset for every 8x16	521	vmax.u8 q5, q2 //l_mad for 16 bytes reset for every 8x16
522		522
523	vpadal.u8 \arg2, q2 //l_sad for 16 bytes reset for every 8x16	523	vpadal.u8 \arg2, q2 //l_sad for 16 bytes reset for every 8x16
524		524
525	SSD_MUL_SUM_16BYTES d4,d5, q8, q11 //q8 for l_sqiff reset for every 16x16	525	SSD_MUL_SUM_16BYTES d4,d5, q8, q11 //q8 for l_sqiff reset for every 16x16
526		526
527	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16	527	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16
528		528
529	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16	529	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16
530	.endm	530	.endm
531		531
532	//for the begin of a 8x16 block, use some instructions to reset the register	532	//for the begin of a 8x16 block, use some instructions to reset the register
533	.macro SAD_SSD_BGD_16_RESET_8x8 arg0, arg1, arg2, arg3	533	.macro SAD_SSD_BGD_16_RESET_8x8 arg0, arg1, arg2, arg3
534	vld1.8 {q0}, [\arg0], \arg2 //load cur_row	534	vld1.8 {q0}, [\arg0], \arg2 //load cur_row
535		535
536	vpaddl.u8 q3, q0 //add cur_row together	536	vpaddl.u8 q3, q0 //add cur_row together
537	vpaddl.u8 q4, q1 //add ref_row together	537	vpaddl.u8 q4, q1 //add ref_row together
538		538
539	vabd.u8 q2, q0, q1 //abs_diff	539	vabd.u8 q2, q0, q1 //abs_diff
540		540
541	vmov q5,q2 //calculate max and avoid reset to zero, l_mad for 16 bytes reset for every 8x16	541	vmov q5,q2 //calculate max and avoid reset to zero, l_mad for 16 bytes reset for every 8x16
542		542
543	vpaddl.u8 \arg3, q2 //l_sad for 16 bytes reset for every 8x16	543	vpaddl.u8 \arg3, q2 //l_sad for 16 bytes reset for every 8x16
544		544
545		545
546	SSD_MUL_SUM_16BYTES d4,d5, q8, q11 //q8 for l_sqiff reset for every 16x16	546	SSD_MUL_SUM_16BYTES d4,d5, q8, q11 //q8 for l_sqiff reset for every 16x16
547		547
548	vld1.8 {q1}, [\arg1], \arg2 //load ref_row	548	vld1.8 {q1}, [\arg1], \arg2 //load ref_row
549		549
550	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16	550	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16
551		551
552	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16	552	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16
553	.endm	553	.endm
554		554
555	//for the begin of a 16x16 block, use some instructions to reset the register	555	//for the begin of a 16x16 block, use some instructions to reset the register
556	.macro SAD_SSD_BGD_16_RESET_16x16 arg0, arg1, arg2, arg3	556	.macro SAD_SSD_BGD_16_RESET_16x16 arg0, arg1, arg2, arg3
557	vld1.8 {q0}, [\arg0], \arg2 //load cur_row	557	vld1.8 {q0}, [\arg0], \arg2 //load cur_row
558	vld1.8 {q1}, [\arg1], \arg2 //load ref_row	558	vld1.8 {q1}, [\arg1], \arg2 //load ref_row
559		559
560	vpaddl.u8 q3, q0 //add cur_row together	560	vpaddl.u8 q3, q0 //add cur_row together
561	vpaddl.u8 q4, q1 //add ref_row together	561	vpaddl.u8 q4, q1 //add ref_row together
562		562
563	vabd.u8 q2, q0, q1 //abs_diff	563	vabd.u8 q2, q0, q1 //abs_diff
564		564
565	vmov q5,q2 //calculate max and avoid reset to zero, l_mad for 16 bytes reset for every 8x16	565	vmov q5,q2 //calculate max and avoid reset to zero, l_mad for 16 bytes reset for every 8x16
566		566
567	vpaddl.u8 \arg3, q2 //l_sad for 16 bytes reset for every 8x16	567	vpaddl.u8 \arg3, q2 //l_sad for 16 bytes reset for every 8x16
568		568
569	SSD_MUL_SUM_16BYTES_RESET d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16	569	SSD_MUL_SUM_16BYTES_RESET d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16
570		570
571	vld1.8 {q1}, [\arg1], \arg2 //load ref_row	571	vld1.8 {q1}, [\arg1], \arg2 //load ref_row
572		572
573	vpaddl.u8 q9, q0 //q9 for l_sum reset for every 16x16	573	vpaddl.u8 q9, q0 //q9 for l_sum reset for every 16x16
574		574
575	SSD_MUL_SUM_16BYTES_RESET d0,d1,q10,q11 //q10 for lsqsum reset for every 16x16	575	SSD_MUL_SUM_16BYTES_RESET d0,d1,q10,q11 //q10 for lsqsum reset for every 16x16
576	.endm	576	.endm
577		577
578	//for each 8x16 block	578	//for each 8x16 block
579	.macro SAD_SSD_BGD_CALC_8x16 arg0, arg1, arg2	579	.macro SAD_SSD_BGD_CALC_8x16 arg0, arg1, arg2
580		580
581	vpmax.u8 d10, d10, d11 //4 numbers	581	vpmax.u8 d10, d10, d11 //4 numbers
582	vpmax.u8 d10, d10, d10 //2 numbers	582	vpmax.u8 d10, d10, d10 //2 numbers
583	vpmax.u8 d10, d10, d10 //1 number1	583	vpmax.u8 d10, d10, d10 //1 number1
584		584
585	vmov \arg0, d10 //d26 d27 keeps the l_mad	585	vmov \arg0, d10 //d26 d27 keeps the l_mad
586		586
587	//p_sd8x8	587	//p_sd8x8
588	vpaddl.u16 q3, q3	588	vpaddl.u16 q3, q3
589	vpaddl.u16 q4, q4	589	vpaddl.u16 q4, q4
590		590
591	vsub.i32 \arg1, q3, q4	591	vsub.i32 \arg1, q3, q4
592	vpaddl.u32 \arg1, \arg1	592	vpaddl.u32 \arg1, \arg1
593		593
594	//psad8x8	594	//psad8x8
595	vpaddl.u16 \arg2, \arg2	595	vpaddl.u16 \arg2, \arg2
596	vpaddl.u32 \arg2, \arg2	596	vpaddl.u32 \arg2, \arg2
597		597
598	//psadframe	598	//psadframe
599	vadd.i32 q12, \arg2	599	vadd.i32 q12, \arg2
600	.endm	600	.endm
601		601
602	.macro SAD_SSD_BGD_16x16 arg0, arg1, arg2	602	.macro SAD_SSD_BGD_16x16 arg0, arg1, arg2
603	//for one 8x16	603	//for one 8x16
604	SAD_SSD_BGD_16_RESET_16x16 \arg0, \arg1, \arg2, q6	604	SAD_SSD_BGD_16_RESET_16x16 \arg0, \arg1, \arg2, q6
605	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6	605	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6
606	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6	606	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6
607	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6	607	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6
608	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6	608	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6
609	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6	609	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6
610	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6	610	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6
611	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6	611	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q6
612		612
613	SAD_SSD_BGD_CALC_8x16 d26, q14, q6	613	SAD_SSD_BGD_CALC_8x16 d26, q14, q6
614		614
615	//for another 8x16	615	//for another 8x16
616	SAD_SSD_BGD_16_RESET_8x8 \arg0, \arg1, \arg2, q7	616	SAD_SSD_BGD_16_RESET_8x8 \arg0, \arg1, \arg2, q7
617	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q7	617	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q7
618	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q7	618	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q7
619	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q7	619	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q7
620	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q7	620	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q7
621	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q7	621	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q7
622	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q7	622	SAD_SSD_BGD_16 \arg0, \arg1, \arg2, q7
623	SAD_SSD_BGD_16_end \arg0, \arg2, q7	623	SAD_SSD_BGD_16_end \arg0, \arg2, q7
624		624
625	SAD_SSD_BGD_CALC_8x16 d27, q15, q7	625	SAD_SSD_BGD_CALC_8x16 d27, q15, q7
626	.endm	626	.endm
627		627
628	.macro SSD_SAD_SD_MAD_PADDL arg0, arg1, arg2	628	.macro SSD_SAD_SD_MAD_PADDL arg0, arg1, arg2
629	vpaddl.s16 \arg0, \arg0	629	vpaddl.s16 \arg0, \arg0
630	vpaddl.s32 \arg0, \arg0	630	vpaddl.s32 \arg0, \arg0
631	vadd.i32 \arg1, \arg1, \arg2	631	vadd.i32 \arg1, \arg1, \arg2
632	.endm	632	.endm
633	#endif	633	#endif
634		634
635		635
636	WELS_ASM_FUNC_BEGIN VAACalcSadSsdBgd_neon	636	WELS_ASM_FUNC_BEGIN VAACalcSadSsdBgd_neon
637	stmdb sp!, {r0-r12, r14}	637	stmdb sp!, {r0-r12, r14}
638	vpush {q4-q7}	638	vpush {q4-q7}
639		639
640	ldr r4, [sp, #120] //r4 keeps the pic_stride	640	ldr r4, [sp, #120] //r4 keeps the pic_stride
641		641
642	sub r5, r4, #1	642	sub r5, r4, #1
643	lsl r5, r5, #4 //r5 keeps the little step	643	lsl r5, r5, #4 //r5 keeps the little step
644		644
645	lsl r6, r4, #4	645	lsl r6, r4, #4
646	sub r6, r2, r6 //r6 keeps the big step	646	sub r6, r2, r6 //r6 keeps the big step
647		647
648		648
649	ldr r8, [sp, #128]//psad8x8	649	ldr r8, [sp, #128]//psad8x8
650	ldr r9, [sp, #132]//psum16x16	650	ldr r9, [sp, #132]//psum16x16
651	ldr r10, [sp, #136]//psqsum16x16	651	ldr r10, [sp, #136]//psqsum16x16
652	ldr r11, [sp, #140]//psqdiff16x16	652	ldr r11, [sp, #140]//psqdiff16x16
653	ldr r12, [sp, #144]//p_sd8x8	653	ldr r12, [sp, #144]//p_sd8x8
654	ldr r14, [sp, #148]//p_mad8x8	654	ldr r14, [sp, #148]//p_mad8x8
655		655
656	vmov.i8 q12, #0	656	vmov.i8 q12, #0
657		657
658	vaa_calc_sad_ssd_bgd_height_loop:	658	vaa_calc_sad_ssd_bgd_height_loop:
659		659
660	mov r7, r2	660	mov r7, r2
661	vaa_calc_sad_ssd_bgd_width_loop:	661	vaa_calc_sad_ssd_bgd_width_loop:
662		662
663	//l_sd q14&q15, l_mad q13, l_sad q6 & q7, l_sqdiff q8, l_sum q9, l_sqsum q10	663	//l_sd q14&q15, l_mad q13, l_sad q6 & q7, l_sqdiff q8, l_sum q9, l_sqsum q10
664	SAD_SSD_BGD_16x16 r0,r1,r4	664	SAD_SSD_BGD_16x16 r0,r1,r4
665		665
666	//psad8x8	666	//psad8x8
@@ -694,243 +694,243 @@ vaa_calc_sad_ssd_bgd_width_loop: · 27 unmodified lines
694		694
695	bne vaa_calc_sad_ssd_bgd_width_loop	695	bne vaa_calc_sad_ssd_bgd_width_loop
696		696
697	sub r0, r0, r6 //jump to next 16 x width	697	sub r0, r0, r6 //jump to next 16 x width
698	sub r1, r1, r6 //jump to next 16 x width	698	sub r1, r1, r6 //jump to next 16 x width
699		699
700	subs r3, #16	700	subs r3, #16
701	bne vaa_calc_sad_ssd_bgd_height_loop	701	bne vaa_calc_sad_ssd_bgd_height_loop
702		702
703	//psadframe	703	//psadframe
704	ldr r7, [sp, #124]//psadframe	704	ldr r7, [sp, #124]//psadframe
705		705
706	vadd.i32 d24, d24, d25	706	vadd.i32 d24, d24, d25
707	vst1.32 {d24[0]}, [r7]	707	vst1.32 {d24[0]}, [r7]
708		708
709	vpop {q4-q7}	709	vpop {q4-q7}
710	ldmia sp!, {r0-r12, r14}	710	ldmia sp!, {r0-r12, r14}
711		711
712	WELS_ASM_FUNC_END	712	WELS_ASM_FUNC_END
713		713
714		714
715	#ifdef __APPLE__	715	#ifdef __APPLE__
716	.macro SAD_VAR_16	716	.macro SAD_VAR_16
717	vld1.8 {q0}, [$0], $2 //load cur_row	717	vld1.8 {q0}, [$0], $2 //load cur_row
718		718
719	vpadal.u8 q3, q0 //add cur_row together	719	vpadal.u8 q3, q0 //add cur_row together
720	vpadal.u8 q4, q1 //add ref_row together	720	vpadal.u8 q4, q1 //add ref_row together
721		721
722	vabd.u8 q2, q0, q1 //abs_diff	722	vabd.u8 q2, q0, q1 //abs_diff
723		723
724	vpadal.u8 $3, q2 //l_sad for 16 bytes reset for every 8x16	724	vpadal.u8 $3, q2 //l_sad for 16 bytes reset for every 8x16
725		725
726	vld1.8 {q1}, [$1], $2	726	vld1.8 {q1}, [$1], $2
727		727
728	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16	728	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16
729		729
730	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16	730	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16
731	.endm	731	.endm
732		732
733	.macro SAD_VAR_16_END	733	.macro SAD_VAR_16_END
734	vld1.8 {q0}, [$0], $1 //load cur_row	734	vld1.8 {q0}, [$0], $1 //load cur_row
735		735
736	vpadal.u8 q3, q0 //add cur_row together	736	vpadal.u8 q3, q0 //add cur_row together
737	vpadal.u8 q4, q1 //add ref_row together	737	vpadal.u8 q4, q1 //add ref_row together
738		738
739	vabd.u8 q2, q0, q1 //abs_diff	739	vabd.u8 q2, q0, q1 //abs_diff
740		740
741	vpadal.u8 $2, q2 //l_sad for 16 bytes reset for every 8x16	741	vpadal.u8 $2, q2 //l_sad for 16 bytes reset for every 8x16
742		742
743	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16	743	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16
744		744
745	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16	745	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16
746	.endm	746	.endm
747		747
748	.macro SAD_VAR_16_RESET_16x16	748	.macro SAD_VAR_16_RESET_16x16
749	vld1.8 {q0}, [$0], $2 //load cur_row	749	vld1.8 {q0}, [$0], $2 //load cur_row
750	vld1.8 {q1}, [$1], $2	750	vld1.8 {q1}, [$1], $2
751		751
752	vpaddl.u8 q3, q0 //add cur_row together	752	vpaddl.u8 q3, q0 //add cur_row together
753	vpaddl.u8 q4, q1 //add ref_row together	753	vpaddl.u8 q4, q1 //add ref_row together
754		754
755	vabd.u8 q2, q0, q1 //abs_diff	755	vabd.u8 q2, q0, q1 //abs_diff
756		756
757	vpaddl.u8 $3, q2 //l_sad for 16 bytes reset for every 8x16	757	vpaddl.u8 $3, q2 //l_sad for 16 bytes reset for every 8x16
758		758
759	vld1.8 {q1}, [$1], $2	759	vld1.8 {q1}, [$1], $2
760		760
761	vpaddl.u8 q9, q0 //q9 for l_sum reset for every 16x16	761	vpaddl.u8 q9, q0 //q9 for l_sum reset for every 16x16
762		762
763	SSD_MUL_SUM_16BYTES_RESET d0,d1, q10, q11	763	SSD_MUL_SUM_16BYTES_RESET d0,d1, q10, q11
764	.endm	764	.endm
765		765
766	.macro SAD_VAR_16_RESET_8x8	766	.macro SAD_VAR_16_RESET_8x8
767	vld1.8 {q0}, [$0], $2 //load cur_row	767	vld1.8 {q0}, [$0], $2 //load cur_row
768		768
769	vpaddl.u8 q3, q0 //add cur_row together	769	vpaddl.u8 q3, q0 //add cur_row together
770	vpaddl.u8 q4, q1 //add ref_row together	770	vpaddl.u8 q4, q1 //add ref_row together
771		771
772	vabd.u8 q2, q0, q1 //abs_diff	772	vabd.u8 q2, q0, q1 //abs_diff
773		773
774	vpaddl.u8 $3, q2 //l_sad for 16 bytes reset for every 8x16	774	vpaddl.u8 $3, q2 //l_sad for 16 bytes reset for every 8x16
775		775
776	vld1.8 {q1}, [$1], $2	776	vld1.8 {q1}, [$1], $2
777		777
778	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16	778	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16
779		779
780	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16	780	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16
781	.endm	781	.endm
782		782
783	.macro SAD_VAR_16x16	783	.macro SAD_VAR_16x16
784	//for one 8x16	784	//for one 8x16
785	SAD_VAR_16_RESET_16x16 $0, $1, $2, q6	785	SAD_VAR_16_RESET_16x16 $0, $1, $2, q6
786	SAD_VAR_16 $0, $1, $2, q6	786	SAD_VAR_16 $0, $1, $2, q6
787	SAD_VAR_16 $0, $1, $2, q6	787	SAD_VAR_16 $0, $1, $2, q6
788	SAD_VAR_16 $0, $1, $2, q6	788	SAD_VAR_16 $0, $1, $2, q6
789	SAD_VAR_16 $0, $1, $2, q6	789	SAD_VAR_16 $0, $1, $2, q6
790	SAD_VAR_16 $0, $1, $2, q6	790	SAD_VAR_16 $0, $1, $2, q6
791	SAD_VAR_16 $0, $1, $2, q6	791	SAD_VAR_16 $0, $1, $2, q6
792	SAD_VAR_16 $0, $1, $2, q6	792	SAD_VAR_16 $0, $1, $2, q6
793		793
794	vpaddl.u16 q6, q6	794	vpaddl.u16 q6, q6
795	vpaddl.u32 q6, q6	795	vpaddl.u32 q6, q6
796	vadd.i32 q12, q6	796	vadd.i32 q12, q6
797		797
798	//for another 8x16	798	//for another 8x16
799	SAD_VAR_16_RESET_8x8 $0, $1, $2, q7	799	SAD_VAR_16_RESET_8x8 $0, $1, $2, q7
800	SAD_VAR_16 $0, $1, $2, q7	800	SAD_VAR_16 $0, $1, $2, q7
801	SAD_VAR_16 $0, $1, $2, q7	801	SAD_VAR_16 $0, $1, $2, q7
802	SAD_VAR_16 $0, $1, $2, q7	802	SAD_VAR_16 $0, $1, $2, q7
803	SAD_VAR_16 $0, $1, $2, q7	803	SAD_VAR_16 $0, $1, $2, q7
804	SAD_VAR_16 $0, $1, $2, q7	804	SAD_VAR_16 $0, $1, $2, q7
805	SAD_VAR_16 $0, $1, $2, q7	805	SAD_VAR_16 $0, $1, $2, q7
806	SAD_VAR_16_END $0, $2, q7	806	SAD_VAR_16_END $0, $2, q7
807		807
808	vpaddl.u16 q7, q7	808	vpaddl.u16 q7, q7
809	vpaddl.u32 q7, q7	809	vpaddl.u32 q7, q7
810		810
811	vadd.i32 q12, q7	811	vadd.i32 q12, q7
812	.endm	812	.endm
813	#else	813	#else
814	.macro SAD_VAR_16 arg0, arg1, arg2, arg3	814	.macro SAD_VAR_16 arg0, arg1, arg2, arg3
815	vld1.8 {q0}, [\arg0], \arg2 //load cur_row	815	vld1.8 {q0}, [\arg0], \arg2 //load cur_row
816		816
817	vpadal.u8 q3, q0 //add cur_row together	817	vpadal.u8 q3, q0 //add cur_row together
818	vpadal.u8 q4, q1 //add ref_row together	818	vpadal.u8 q4, q1 //add ref_row together
819		819
820	vabd.u8 q2, q0, q1 //abs_diff	820	vabd.u8 q2, q0, q1 //abs_diff
821		821
822	vpadal.u8 \arg3, q2 //l_sad for 16 bytes reset for every 8x16	822	vpadal.u8 \arg3, q2 //l_sad for 16 bytes reset for every 8x16
823		823
824	vld1.8 {q1}, [\arg1], \arg2	824	vld1.8 {q1}, [\arg1], \arg2
825		825
826	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16	826	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16
827		827
828	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16	828	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16
829	.endm	829	.endm
830		830
831	.macro SAD_VAR_16_END arg0, arg1, arg2	831	.macro SAD_VAR_16_END arg0, arg1, arg2
832	vld1.8 {q0}, [\arg0], \arg1 //load cur_row	832	vld1.8 {q0}, [\arg0], \arg1 //load cur_row
833		833
834	vpadal.u8 q3, q0 //add cur_row together	834	vpadal.u8 q3, q0 //add cur_row together
835	vpadal.u8 q4, q1 //add ref_row together	835	vpadal.u8 q4, q1 //add ref_row together
836		836
837	vabd.u8 q2, q0, q1 //abs_diff	837	vabd.u8 q2, q0, q1 //abs_diff
838		838
839	vpadal.u8 \arg2, q2 //l_sad for 16 bytes reset for every 8x16	839	vpadal.u8 \arg2, q2 //l_sad for 16 bytes reset for every 8x16
840		840
841	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16	841	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16
842		842
843	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16	843	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16
844	.endm	844	.endm
845		845
846		846
847	.macro SAD_VAR_16_RESET_16x16 arg0, arg1, arg2, arg3	847	.macro SAD_VAR_16_RESET_16x16 arg0, arg1, arg2, arg3
848	vld1.8 {q0}, [\arg0], \arg2 //load cur_row	848	vld1.8 {q0}, [\arg0], \arg2 //load cur_row
849	vld1.8 {q1}, [\arg1], \arg2	849	vld1.8 {q1}, [\arg1], \arg2
850		850
851	vpaddl.u8 q3, q0 //add cur_row together	851	vpaddl.u8 q3, q0 //add cur_row together
852	vpaddl.u8 q4, q1 //add ref_row together	852	vpaddl.u8 q4, q1 //add ref_row together
853		853
854	vabd.u8 q2, q0, q1 //abs_diff	854	vabd.u8 q2, q0, q1 //abs_diff
855		855
856	vpaddl.u8 \arg3, q2 //l_sad for 16 bytes reset for every 8x16	856	vpaddl.u8 \arg3, q2 //l_sad for 16 bytes reset for every 8x16
857		857
858	vld1.8 {q1}, [\arg1], \arg2	858	vld1.8 {q1}, [\arg1], \arg2
859		859
860	vpaddl.u8 q9, q0 //q9 for l_sum reset for every 16x16	860	vpaddl.u8 q9, q0 //q9 for l_sum reset for every 16x16
861		861
862	SSD_MUL_SUM_16BYTES_RESET d0,d1, q10, q11	862	SSD_MUL_SUM_16BYTES_RESET d0,d1, q10, q11
863	.endm	863	.endm
864		864
865	.macro SAD_VAR_16_RESET_8x8 arg0, arg1, arg2, arg3	865	.macro SAD_VAR_16_RESET_8x8 arg0, arg1, arg2, arg3
866	vld1.8 {q0}, [\arg0], \arg2 //load cur_row	866	vld1.8 {q0}, [\arg0], \arg2 //load cur_row
867		867
868	vpaddl.u8 q3, q0 //add cur_row together	868	vpaddl.u8 q3, q0 //add cur_row together
869	vpaddl.u8 q4, q1 //add ref_row together	869	vpaddl.u8 q4, q1 //add ref_row together
870		870
871	vabd.u8 q2, q0, q1 //abs_diff	871	vabd.u8 q2, q0, q1 //abs_diff
872		872
873	vpaddl.u8 \arg3, q2 //l_sad for 16 bytes reset for every 8x16	873	vpaddl.u8 \arg3, q2 //l_sad for 16 bytes reset for every 8x16
874		874
875	vld1.8 {q1}, [\arg1], \arg2	875	vld1.8 {q1}, [\arg1], \arg2
876		876
877	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16	877	vpadal.u8 q9, q0 //q9 for l_sum reset for every 16x16
878		878
879	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16	879	SSD_MUL_SUM_16BYTES d0,d1, q10, q11 //q10 for lsqsum reset for every 16x16
880	.endm	880	.endm
881		881
882	.macro SAD_VAR_16x16 arg0, arg1, arg2	882	.macro SAD_VAR_16x16 arg0, arg1, arg2
883	//for one 8x16	883	//for one 8x16
884	SAD_VAR_16_RESET_16x16 \arg0, \arg1, \arg2, q6	884	SAD_VAR_16_RESET_16x16 \arg0, \arg1, \arg2, q6
885	SAD_VAR_16 \arg0, \arg1, \arg2, q6	885	SAD_VAR_16 \arg0, \arg1, \arg2, q6
886	SAD_VAR_16 \arg0, \arg1, \arg2, q6	886	SAD_VAR_16 \arg0, \arg1, \arg2, q6
887	SAD_VAR_16 \arg0, \arg1, \arg2, q6	887	SAD_VAR_16 \arg0, \arg1, \arg2, q6
888	SAD_VAR_16 \arg0, \arg1, \arg2, q6	888	SAD_VAR_16 \arg0, \arg1, \arg2, q6
889	SAD_VAR_16 \arg0, \arg1, \arg2, q6	889	SAD_VAR_16 \arg0, \arg1, \arg2, q6
890	SAD_VAR_16 \arg0, \arg1, \arg2, q6	890	SAD_VAR_16 \arg0, \arg1, \arg2, q6
891	SAD_VAR_16 \arg0, \arg1, \arg2, q6	891	SAD_VAR_16 \arg0, \arg1, \arg2, q6
892		892
893	vpaddl.u16 q6, q6	893	vpaddl.u16 q6, q6
894	vpaddl.u32 q6, q6	894	vpaddl.u32 q6, q6
895	vadd.i32 q12, q6	895	vadd.i32 q12, q6
896		896
897	//for another 8x16	897	//for another 8x16
898	SAD_VAR_16_RESET_8x8 \arg0, \arg1, \arg2, q7	898	SAD_VAR_16_RESET_8x8 \arg0, \arg1, \arg2, q7
899	SAD_VAR_16 \arg0, \arg1, \arg2, q7	899	SAD_VAR_16 \arg0, \arg1, \arg2, q7
900	SAD_VAR_16 \arg0, \arg1, \arg2, q7	900	SAD_VAR_16 \arg0, \arg1, \arg2, q7
901	SAD_VAR_16 \arg0, \arg1, \arg2, q7	901	SAD_VAR_16 \arg0, \arg1, \arg2, q7
902	SAD_VAR_16 \arg0, \arg1, \arg2, q7	902	SAD_VAR_16 \arg0, \arg1, \arg2, q7
903	SAD_VAR_16 \arg0, \arg1, \arg2, q7	903	SAD_VAR_16 \arg0, \arg1, \arg2, q7
904	SAD_VAR_16 \arg0, \arg1, \arg2, q7	904	SAD_VAR_16 \arg0, \arg1, \arg2, q7
905	SAD_VAR_16_END \arg0, \arg2, q7	905	SAD_VAR_16_END \arg0, \arg2, q7
906		906
907	vpaddl.u16 q7, q7	907	vpaddl.u16 q7, q7
908	vpaddl.u32 q7, q7	908	vpaddl.u32 q7, q7
909		909
910	vadd.i32 q12, q7	910	vadd.i32 q12, q7
911	.endm	911	.endm
912	#endif	912	#endif
913		913
914		914
915	WELS_ASM_FUNC_BEGIN VAACalcSadVar_neon	915	WELS_ASM_FUNC_BEGIN VAACalcSadVar_neon
916	stmdb sp!, {r4-r11}	916	stmdb sp!, {r4-r11}
917	vpush {q4}	917	vpush {q4}
918	vpush {q6-q7}	918	vpush {q6-q7}
919		919
920	ldr r4, [sp, #80] //r4 keeps the pic_stride	920	ldr r4, [sp, #80] //r4 keeps the pic_stride
921		921
922	sub r5, r4, #1	922	sub r5, r4, #1
923	lsl r5, r5, #4 //r5 keeps the little step	923	lsl r5, r5, #4 //r5 keeps the little step
924		924
925	lsl r6, r4, #4	925	lsl r6, r4, #4
926	sub r6, r2, r6 //r6 keeps the big step	926	sub r6, r2, r6 //r6 keeps the big step
927		927
928	ldr r7, [sp, #84] //psadframe	928	ldr r7, [sp, #84] //psadframe
929	ldr r8, [sp, #88] //psad8x8	929	ldr r8, [sp, #88] //psad8x8
930	ldr r9, [sp, #92] //psum16x16	930	ldr r9, [sp, #92] //psum16x16
931	ldr r10, [sp, #96] //psqsum16x16	931	ldr r10, [sp, #96] //psqsum16x16
932		932
933	vmov.i8 q12, #0	933	vmov.i8 q12, #0
934	vaa_calc_sad_var_height_loop:	934	vaa_calc_sad_var_height_loop:
935		935
936	mov r11, r2	936	mov r11, r2
@@ -956,154 +956,154 @@ vaa_calc_sad_var_width_loop: · 19 unmodified lines
956		956
957	bne vaa_calc_sad_var_width_loop	957	bne vaa_calc_sad_var_width_loop
958		958
959	sub r0, r0, r6 //jump to next 16 x width	959	sub r0, r0, r6 //jump to next 16 x width
960	sub r1, r1, r6 //jump to next 16 x width	960	sub r1, r1, r6 //jump to next 16 x width
961		961
962	subs r3, #16	962	subs r3, #16
963	bne vaa_calc_sad_var_height_loop	963	bne vaa_calc_sad_var_height_loop
964		964
965	vadd.i32 d24, d24, d25	965	vadd.i32 d24, d24, d25
966	vst1.32 {d24[0]}, [r7]	966	vst1.32 {d24[0]}, [r7]
967		967
968	vpop {q6-q7}	968	vpop {q6-q7}
969	vpop {q4}	969	vpop {q4}
970	ldmia sp!, {r4-r11}	970	ldmia sp!, {r4-r11}
971	WELS_ASM_FUNC_END	971	WELS_ASM_FUNC_END
972		972
973		973
974	#ifdef __APPLE__	974	#ifdef __APPLE__
975	.macro SAD_SSD_16	975	.macro SAD_SSD_16
976	SAD_VAR_16 $0, $1, $2, $3	976	SAD_VAR_16 $0, $1, $2, $3
977		977
978	SSD_MUL_SUM_16BYTES d4,d5,q8, q11	978	SSD_MUL_SUM_16BYTES d4,d5,q8, q11
979	.endm	979	.endm
980		980
981	.macro SAD_SSD_16_END	981	.macro SAD_SSD_16_END
982	SAD_VAR_16_END $0, $1, $2	982	SAD_VAR_16_END $0, $1, $2
983		983
984	SSD_MUL_SUM_16BYTES d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16	984	SSD_MUL_SUM_16BYTES d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16
985	.endm	985	.endm
986		986
987	.macro SAD_SSD_16_RESET_16x16	987	.macro SAD_SSD_16_RESET_16x16
988	SAD_VAR_16_RESET_16x16 $0, $1, $2, $3	988	SAD_VAR_16_RESET_16x16 $0, $1, $2, $3
989		989
990	SSD_MUL_SUM_16BYTES_RESET d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16	990	SSD_MUL_SUM_16BYTES_RESET d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16
991	.endm	991	.endm
992		992
993	.macro SAD_SSD_16_RESET_8x8	993	.macro SAD_SSD_16_RESET_8x8
994	SAD_VAR_16_RESET_8x8 $0, $1, $2, $3	994	SAD_VAR_16_RESET_8x8 $0, $1, $2, $3
995		995
996	SSD_MUL_SUM_16BYTES d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16	996	SSD_MUL_SUM_16BYTES d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16
997	.endm	997	.endm
998		998
999	.macro SAD_SSD_16x16	999	.macro SAD_SSD_16x16
1000	//for one 8x16	1000	//for one 8x16
1001	SAD_SSD_16_RESET_16x16 $0, $1, $2, q6	1001	SAD_SSD_16_RESET_16x16 $0, $1, $2, q6
1002	SAD_SSD_16 $0, $1, $2, q6	1002	SAD_SSD_16 $0, $1, $2, q6
1003	SAD_SSD_16 $0, $1, $2, q6	1003	SAD_SSD_16 $0, $1, $2, q6
1004	SAD_SSD_16 $0, $1, $2, q6	1004	SAD_SSD_16 $0, $1, $2, q6
1005	SAD_SSD_16 $0, $1, $2, q6	1005	SAD_SSD_16 $0, $1, $2, q6
1006	SAD_SSD_16 $0, $1, $2, q6	1006	SAD_SSD_16 $0, $1, $2, q6
1007	SAD_SSD_16 $0, $1, $2, q6	1007	SAD_SSD_16 $0, $1, $2, q6
1008	SAD_SSD_16 $0, $1, $2, q6	1008	SAD_SSD_16 $0, $1, $2, q6
1009		1009
1010	vpaddl.u16 q6, q6	1010	vpaddl.u16 q6, q6
1011	vpaddl.u32 q6, q6	1011	vpaddl.u32 q6, q6
1012	vadd.i32 q12, q6	1012	vadd.i32 q12, q6
1013		1013
1014	//for another 8x16	1014	//for another 8x16
1015	SAD_SSD_16_RESET_8x8 $0, $1, $2, q7	1015	SAD_SSD_16_RESET_8x8 $0, $1, $2, q7
1016	SAD_SSD_16 $0, $1, $2, q7	1016	SAD_SSD_16 $0, $1, $2, q7
1017	SAD_SSD_16 $0, $1, $2, q7	1017	SAD_SSD_16 $0, $1, $2, q7
1018	SAD_SSD_16 $0, $1, $2, q7	1018	SAD_SSD_16 $0, $1, $2, q7
1019	SAD_SSD_16 $0, $1, $2, q7	1019	SAD_SSD_16 $0, $1, $2, q7
1020	SAD_SSD_16 $0, $1, $2, q7	1020	SAD_SSD_16 $0, $1, $2, q7
1021	SAD_SSD_16 $0, $1, $2, q7	1021	SAD_SSD_16 $0, $1, $2, q7
1022	SAD_SSD_16_END $0, $2, q7	1022	SAD_SSD_16_END $0, $2, q7
1023		1023
1024	vpaddl.u16 q7, q7	1024	vpaddl.u16 q7, q7
1025	vpaddl.u32 q7, q7	1025	vpaddl.u32 q7, q7
1026		1026
1027	vadd.i32 q12, q7	1027	vadd.i32 q12, q7
1028	.endm	1028	.endm
1029	#else	1029	#else
1030	.macro SAD_SSD_16 arg0, arg1, arg2, arg3	1030	.macro SAD_SSD_16 arg0, arg1, arg2, arg3
1031	SAD_VAR_16 \arg0, \arg1, \arg2, \arg3	1031	SAD_VAR_16 \arg0, \arg1, \arg2, \arg3
1032		1032
1033	SSD_MUL_SUM_16BYTES d4,d5,q8, q11	1033	SSD_MUL_SUM_16BYTES d4,d5,q8, q11
1034	.endm	1034	.endm
1035		1035
1036	.macro SAD_SSD_16_END arg0, arg1, arg2	1036	.macro SAD_SSD_16_END arg0, arg1, arg2
1037	SAD_VAR_16_END \arg0, \arg1, \arg2	1037	SAD_VAR_16_END \arg0, \arg1, \arg2
1038		1038
1039	SSD_MUL_SUM_16BYTES d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16	1039	SSD_MUL_SUM_16BYTES d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16
1040	.endm	1040	.endm
1041		1041
1042	.macro SAD_SSD_16_RESET_16x16 arg0, arg1, arg2, arg3	1042	.macro SAD_SSD_16_RESET_16x16 arg0, arg1, arg2, arg3
1043	SAD_VAR_16_RESET_16x16 \arg0, \arg1, \arg2, \arg3	1043	SAD_VAR_16_RESET_16x16 \arg0, \arg1, \arg2, \arg3
1044		1044
1045	SSD_MUL_SUM_16BYTES_RESET d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16	1045	SSD_MUL_SUM_16BYTES_RESET d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16
1046	.endm	1046	.endm
1047		1047
1048	.macro SAD_SSD_16_RESET_8x8 arg0, arg1, arg2, arg3	1048	.macro SAD_SSD_16_RESET_8x8 arg0, arg1, arg2, arg3
1049	SAD_VAR_16_RESET_8x8 \arg0, \arg1, \arg2, \arg3	1049	SAD_VAR_16_RESET_8x8 \arg0, \arg1, \arg2, \arg3
1050		1050
1051	SSD_MUL_SUM_16BYTES d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16	1051	SSD_MUL_SUM_16BYTES d4,d5,q8, q11 //q8 for l_sqiff reset for every 16x16
1052	.endm	1052	.endm
1053		1053
1054	.macro SAD_SSD_16x16 arg0, arg1, arg2	1054	.macro SAD_SSD_16x16 arg0, arg1, arg2
1055	//for one 8x16	1055	//for one 8x16
1056	SAD_SSD_16_RESET_16x16 \arg0, \arg1, \arg2, q6	1056	SAD_SSD_16_RESET_16x16 \arg0, \arg1, \arg2, q6
1057	SAD_SSD_16 \arg0, \arg1, \arg2, q6	1057	SAD_SSD_16 \arg0, \arg1, \arg2, q6
1058	SAD_SSD_16 \arg0, \arg1, \arg2, q6	1058	SAD_SSD_16 \arg0, \arg1, \arg2, q6
1059	SAD_SSD_16 \arg0, \arg1, \arg2, q6	1059	SAD_SSD_16 \arg0, \arg1, \arg2, q6
1060	SAD_SSD_16 \arg0, \arg1, \arg2, q6	1060	SAD_SSD_16 \arg0, \arg1, \arg2, q6
1061	SAD_SSD_16 \arg0, \arg1, \arg2, q6	1061	SAD_SSD_16 \arg0, \arg1, \arg2, q6
1062	SAD_SSD_16 \arg0, \arg1, \arg2, q6	1062	SAD_SSD_16 \arg0, \arg1, \arg2, q6
1063	SAD_SSD_16 \arg0, \arg1, \arg2, q6	1063	SAD_SSD_16 \arg0, \arg1, \arg2, q6
1064		1064
1065	vpaddl.u16 q6, q6	1065	vpaddl.u16 q6, q6
1066	vpaddl.u32 q6, q6	1066	vpaddl.u32 q6, q6
1067	vadd.i32 q12, q6	1067	vadd.i32 q12, q6
1068		1068
1069	//for another 8x16	1069	//for another 8x16
1070	SAD_SSD_16_RESET_8x8 \arg0, \arg1, \arg2, q7	1070	SAD_SSD_16_RESET_8x8 \arg0, \arg1, \arg2, q7
1071	SAD_SSD_16 \arg0, \arg1, \arg2, q7	1071	SAD_SSD_16 \arg0, \arg1, \arg2, q7
1072	SAD_SSD_16 \arg0, \arg1, \arg2, q7	1072	SAD_SSD_16 \arg0, \arg1, \arg2, q7
1073	SAD_SSD_16 \arg0, \arg1, \arg2, q7	1073	SAD_SSD_16 \arg0, \arg1, \arg2, q7
1074	SAD_SSD_16 \arg0, \arg1, \arg2, q7	1074	SAD_SSD_16 \arg0, \arg1, \arg2, q7
1075	SAD_SSD_16 \arg0, \arg1, \arg2, q7	1075	SAD_SSD_16 \arg0, \arg1, \arg2, q7
1076	SAD_SSD_16 \arg0, \arg1, \arg2, q7	1076	SAD_SSD_16 \arg0, \arg1, \arg2, q7
1077	SAD_SSD_16_END \arg0, \arg2, q7	1077	SAD_SSD_16_END \arg0, \arg2, q7
1078		1078
1079	vpaddl.u16 q7, q7	1079	vpaddl.u16 q7, q7
1080	vpaddl.u32 q7, q7	1080	vpaddl.u32 q7, q7
1081		1081
1082	vadd.i32 q12, q7	1082	vadd.i32 q12, q7
1083	.endm	1083	.endm
1084	#endif	1084	#endif
1085		1085
1086		1086
1087	WELS_ASM_FUNC_BEGIN VAACalcSadSsd_neon	1087	WELS_ASM_FUNC_BEGIN VAACalcSadSsd_neon
1088	stmdb sp!, {r4-r12}	1088	stmdb sp!, {r4-r12}
1089	vpush {q4}	1089	vpush {q4}
1090	vpush {q6-q7}	1090	vpush {q6-q7}
1091		1091
1092	ldr r4, [sp, #84] //r4 keeps the pic_stride	1092	ldr r4, [sp, #84] //r4 keeps the pic_stride
1093		1093
1094	sub r5, r4, #1	1094	sub r5, r4, #1
1095	lsl r5, r5, #4 //r5 keeps the little step	1095	lsl r5, r5, #4 //r5 keeps the little step
1096		1096
1097	lsl r6, r4, #4	1097	lsl r6, r4, #4
1098	sub r6, r2, r6 //r6 keeps the big step	1098	sub r6, r2, r6 //r6 keeps the big step
1099		1099
1100	ldr r7, [sp, #88] //psadframe	1100	ldr r7, [sp, #88] //psadframe
1101	ldr r8, [sp, #92] //psad8x8	1101	ldr r8, [sp, #92] //psad8x8
1102	ldr r9, [sp, #96] //psum16x16	1102	ldr r9, [sp, #96] //psum16x16
1103	ldr r10, [sp, #100] //psqsum16x16	1103	ldr r10, [sp, #100] //psqsum16x16
1104	ldr r11, [sp, #104] //psqdiff16x16	1104	ldr r11, [sp, #104] //psqdiff16x16
1105		1105
1106	vmov.i8 q12, #0	1106	vmov.i8 q12, #0
1107	vaa_calc_sad_ssd_height_loop:	1107	vaa_calc_sad_ssd_height_loop:
1108		1108
1109	mov r12, r2	1109	mov r12, r2
@@ -1136,18 +1136,18 @@ vaa_calc_sad_ssd_width_loop: · 26 unmodified lines
1136		1136
1137	bne vaa_calc_sad_ssd_width_loop	1137	bne vaa_calc_sad_ssd_width_loop
1138		1138
1139	sub r0, r0, r6 //jump to next 16 x width	1139	sub r0, r0, r6 //jump to next 16 x width
1140	sub r1, r1, r6 //jump to next 16 x width	1140	sub r1, r1, r6 //jump to next 16 x width
1141		1141
1142	subs r3, #16	1142	subs r3, #16
1143	bne vaa_calc_sad_ssd_height_loop	1143	bne vaa_calc_sad_ssd_height_loop
1144		1144
1145	vadd.i32 d24, d24, d25	1145	vadd.i32 d24, d24, d25
1146	vst1.32 {d24[0]}, [r7]	1146	vst1.32 {d24[0]}, [r7]
1147		1147
1148	vpop {q6-q7}	1148	vpop {q6-q7}
1149	vpop {q4}	1149	vpop {q4}
1150	ldmia sp!, {r4-r12}	1150	ldmia sp!, {r4-r12}
1151	WELS_ASM_FUNC_END	1151	WELS_ASM_FUNC_END
1152		1152
1153	#endif	1153	#endif

codec/processing/src/x86/denoisefilter.asm+170−170

@@ -56,217 +56,217 @@ sse2_20 times 8 dw 20 · 55 unmodified lines
56	;***********************************************************************	56	;***********************************************************************
57	SECTION .text	57	SECTION .text
58		58
59	%macro WEIGHT_LINE 9	59	%macro WEIGHT_LINE 9
60	movq %2, %9	60	movq %2, %9
61	punpcklbw %2, %7	61	punpcklbw %2, %7
62	movdqa %8, %2	62	movdqa %8, %2
63		63
64	movdqa %1, %6	64	movdqa %1, %6
65	psubusb %1, %8	65	psubusb %1, %8
66	psubusb %8, %6	66	psubusb %8, %6
67	por %8, %1 ; ABS(curPixel - centerPixel);	67	por %8, %1 ; ABS(curPixel - centerPixel);
68		68
69	movdqa %1, %3	69	movdqa %1, %3
70	psubusb %1, %8	70	psubusb %1, %8
71		71
72	pmullw %1, %1	72	pmullw %1, %1
73	psrlw %1, 5	73	psrlw %1, 5
74	pmullw %2, %1	74	pmullw %2, %1
75	paddusw %4, %1	75	paddusw %4, %1
76	paddusw %5, %2	76	paddusw %5, %2
77	%endmacro	77	%endmacro
78		78
79	%macro WEIGHT_LINE1_UV 4	79	%macro WEIGHT_LINE1_UV 4
80	movdqa %2, %1	80	movdqa %2, %1
81	punpcklbw %2, %4	81	punpcklbw %2, %4
82	paddw %3, %2	82	paddw %3, %2
83		83
84	movdqa %2, %1	84	movdqa %2, %1
85	psrldq %2, 1	85	psrldq %2, 1
86	punpcklbw %2, %4	86	punpcklbw %2, %4
87	paddw %3, %2	87	paddw %3, %2
88		88
89	movdqa %2, %1	89	movdqa %2, %1
90	psrldq %2, 2	90	psrldq %2, 2
91	punpcklbw %2, %4	91	punpcklbw %2, %4
92	psllw %2, 1	92	psllw %2, 1
93	paddw %3, %2	93	paddw %3, %2
94		94
95	movdqa %2, %1	95	movdqa %2, %1
96	psrldq %2, 3	96	psrldq %2, 3
97	punpcklbw %2, %4	97	punpcklbw %2, %4
98	paddw %3, %2	98	paddw %3, %2
99		99
100	movdqa %2, %1	100	movdqa %2, %1
101	psrldq %2, 4	101	psrldq %2, 4
102	punpcklbw %2, %4	102	punpcklbw %2, %4
103	paddw %3, %2	103	paddw %3, %2
104	%endmacro	104	%endmacro
105		105
106	%macro WEIGHT_LINE2_UV 4	106	%macro WEIGHT_LINE2_UV 4
107	movdqa %2, %1	107	movdqa %2, %1
108	punpcklbw %2, %4	108	punpcklbw %2, %4
109	paddw %3, %2	109	paddw %3, %2
110		110
111	movdqa %2, %1	111	movdqa %2, %1
112	psrldq %2, 1	112	psrldq %2, 1
113	punpcklbw %2, %4	113	punpcklbw %2, %4
114	psllw %2, 1	114	psllw %2, 1
115	paddw %3, %2	115	paddw %3, %2
116		116
117	movdqa %2, %1	117	movdqa %2, %1
118	psrldq %2, 2	118	psrldq %2, 2
119	punpcklbw %2, %4	119	punpcklbw %2, %4
120	psllw %2, 2	120	psllw %2, 2
121	paddw %3, %2	121	paddw %3, %2
122		122
123	movdqa %2, %1	123	movdqa %2, %1
124	psrldq %2, 3	124	psrldq %2, 3
125	punpcklbw %2, %4	125	punpcklbw %2, %4
126	psllw %2, 1	126	psllw %2, 1
127	paddw %3, %2	127	paddw %3, %2
128		128
129	movdqa %2, %1	129	movdqa %2, %1
130	psrldq %2, 4	130	psrldq %2, 4
131	punpcklbw %2, %4	131	punpcklbw %2, %4
132	paddw %3, %2	132	paddw %3, %2
133	%endmacro	133	%endmacro
134		134
135	%macro WEIGHT_LINE3_UV 4	135	%macro WEIGHT_LINE3_UV 4
136	movdqa %2, %1	136	movdqa %2, %1
137	punpcklbw %2, %4	137	punpcklbw %2, %4
138	psllw %2, 1	138	psllw %2, 1
139	paddw %3, %2	139	paddw %3, %2
140		140
141	movdqa %2, %1	141	movdqa %2, %1
142	psrldq %2, 1	142	psrldq %2, 1
143	punpcklbw %2, %4	143	punpcklbw %2, %4
144	psllw %2, 2	144	psllw %2, 2
145	paddw %3, %2	145	paddw %3, %2
146		146
147	movdqa %2, %1	147	movdqa %2, %1
148	psrldq %2, 2	148	psrldq %2, 2
149	punpcklbw %2, %4	149	punpcklbw %2, %4
150	pmullw %2, [sse2_20]	150	pmullw %2, [sse2_20]
151	paddw %3, %2	151	paddw %3, %2
152		152
153	movdqa %2, %1	153	movdqa %2, %1
154	psrldq %2, 3	154	psrldq %2, 3
155	punpcklbw %2, %4	155	punpcklbw %2, %4
156	psllw %2, 2	156	psllw %2, 2
157	paddw %3, %2	157	paddw %3, %2
158		158
159	movdqa %2, %1	159	movdqa %2, %1
160	psrldq %2, 4	160	psrldq %2, 4
161	punpcklbw %2, %4	161	punpcklbw %2, %4
162	psllw %2, 1	162	psllw %2, 1
163	paddw %3, %2	163	paddw %3, %2
164	%endmacro	164	%endmacro
165		165
166	;***********************************************************************	166	;***********************************************************************
167	; BilateralLumaFilter8_sse2(uint8_t *pixels, int stride);	167	; BilateralLumaFilter8_sse2(uint8_t *pixels, int stride);
168	;***********************************************************************	168	;***********************************************************************
169	; 1 2 3	169	; 1 2 3
170	; 4 0 5	170	; 4 0 5
171	; 6 7 8	171	; 6 7 8
172	; 0: the center point	172	; 0: the center point
173		173
174	WELS_EXTERN BilateralLumaFilter8_sse2	174	WELS_EXTERN BilateralLumaFilter8_sse2
175		175
176	push r3	176	push r3
177	%assign push_num 1	177	%assign push_num 1
178	LOAD_2_PARA	178	LOAD_2_PARA
179	PUSH_XMM 8	179	PUSH_XMM 8
180		180
181	pxor xmm7, xmm7	181	pxor xmm7, xmm7
182		182
183	mov r3, r0	183	mov r3, r0
184		184
185	movq xmm6, [r0]	185	movq xmm6, [r0]
186	punpcklbw xmm6, xmm7	186	punpcklbw xmm6, xmm7
187	movdqa xmm3, [sse2_32]	187	movdqa xmm3, [sse2_32]
188	pxor xmm4, xmm4 ; nTotWeight	188	pxor xmm4, xmm4 ; nTotWeight
189	pxor xmm5, xmm5 ; nSum	189	pxor xmm5, xmm5 ; nSum
190		190
191	dec r0	191	dec r0
192	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0] ; pixel 4	192	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0] ; pixel 4
193	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0 + 2] ; pixel 5	193	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0 + 2] ; pixel 5
194		194
195	sub r0, r1	195	sub r0, r1
196	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0] ; pixel 1	196	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0] ; pixel 1
197	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0 + 1] ; pixel 2	197	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0 + 1] ; pixel 2
198	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0 + 2] ; pixel 3	198	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0 + 2] ; pixel 3
199		199
200	lea r0, [r0 + r1 * 2]	200	lea r0, [r0 + r1 * 2]
201	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0] ; pixel 6	201	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0] ; pixel 6
202	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0 + 1] ; pixel 7	202	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0 + 1] ; pixel 7
203	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0 + 2] ; pixel 8	203	WEIGHT_LINE xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm0, [r0 + 2] ; pixel 8
204		204
205	pcmpeqw xmm0, xmm0	205	pcmpeqw xmm0, xmm0
206	psrlw xmm0, 15	206	psrlw xmm0, 15
207	psllw xmm0, 8	207	psllw xmm0, 8
208	psubusw xmm0, xmm4	208	psubusw xmm0, xmm4
209	pmullw xmm0, xmm6	209	pmullw xmm0, xmm6
210	paddusw xmm5, xmm0	210	paddusw xmm5, xmm0
211	psrlw xmm5, 8	211	psrlw xmm5, 8
212	packuswb xmm5, xmm5	212	packuswb xmm5, xmm5
213	movq [r3], xmm5	213	movq [r3], xmm5
214		214
215		215
216	POP_XMM	216	POP_XMM
217	pop r3	217	pop r3
218	%assign push_num 0	218	%assign push_num 0
219		219
220	ret	220	ret
221		221
222	;***********************************************************************	222	;***********************************************************************
223	; void WaverageChromaFilter8_sse2(uint8_t *pixels, int stride);	223	; void WaverageChromaFilter8_sse2(uint8_t *pixels, int stride);
224	;***********************************************************************	224	;***********************************************************************
225	;5x5 filter:	225	;5x5 filter:
226	;1 1 2 1 1	226	;1 1 2 1 1
227	;1 2 4 2 1	227	;1 2 4 2 1
228	;2 4 20 4 2	228	;2 4 20 4 2
229	;1 2 4 2 1	229	;1 2 4 2 1
230	;1 1 2 1 1	230	;1 1 2 1 1
231		231
232	WELS_EXTERN WaverageChromaFilter8_sse2	232	WELS_EXTERN WaverageChromaFilter8_sse2
233		233
234	push r3	234	push r3
235		235
236	%assign push_num 1	236	%assign push_num 1
237		237
238	LOAD_2_PARA	238	LOAD_2_PARA
239		239
240	mov r3, r1	240	mov r3, r1
241	add r3, r3	241	add r3, r3
242	sub r0, r3 ; pixels - 2 * stride	242	sub r0, r3 ; pixels - 2 * stride
243	sub r0, 2	243	sub r0, 2
244		244
245	pxor xmm0, xmm0	245	pxor xmm0, xmm0
246	pxor xmm3, xmm3	246	pxor xmm3, xmm3
247		247
248	movdqu xmm1, [r0]	248	movdqu xmm1, [r0]
249	WEIGHT_LINE1_UV xmm1, xmm2, xmm3, xmm0	249	WEIGHT_LINE1_UV xmm1, xmm2, xmm3, xmm0
250		250
251	movdqu xmm1, [r0 + r1]	251	movdqu xmm1, [r0 + r1]
252	WEIGHT_LINE2_UV xmm1, xmm2, xmm3, xmm0	252	WEIGHT_LINE2_UV xmm1, xmm2, xmm3, xmm0
253		253
254	add r0, r3	254	add r0, r3
255	movdqu xmm1, [r0]	255	movdqu xmm1, [r0]
256	WEIGHT_LINE3_UV xmm1, xmm2, xmm3, xmm0	256	WEIGHT_LINE3_UV xmm1, xmm2, xmm3, xmm0
257		257
258	movdqu xmm1, [r0 + r1]	258	movdqu xmm1, [r0 + r1]
259	WEIGHT_LINE2_UV xmm1, xmm2, xmm3, xmm0	259	WEIGHT_LINE2_UV xmm1, xmm2, xmm3, xmm0
260		260
261	movdqu xmm1, [r0 + r1 * 2]	261	movdqu xmm1, [r0 + r1 * 2]
262	WEIGHT_LINE1_UV xmm1, xmm2, xmm3, xmm0	262	WEIGHT_LINE1_UV xmm1, xmm2, xmm3, xmm0
263		263
264	psrlw xmm3, 6	264	psrlw xmm3, 6
265	packuswb xmm3, xmm3	265	packuswb xmm3, xmm3
266	movq [r0 + 2], xmm3	266	movq [r0 + 2], xmm3
267		267
268		268
269	pop r3	269	pop r3
270		270
271	%assign push_num 0	271	%assign push_num 0
272	ret	272	ret

codec/processing/src/x86/downsample_bilinear.asm+978−978

@@ -29,13 +29,13 @@ · 28 unmodified lines
29	;* POSSIBILITY OF SUCH DAMAGE.	29	;* POSSIBILITY OF SUCH DAMAGE.
30	;*	30	;*
31	;*	31	;*
32	;* upsampling.asm	32	;* upsampling.asm
33	;*	33	;*
34	;* Abstract	34	;* Abstract
35	;* SIMD for pixel domain down sampling	35	;* SIMD for pixel domain down sampling
36	;*	36	;*
37	;* History	37	;* History
38	;* 10/22/2009 Created	38	;* 10/22/2009 Created
39	;*	39	;*
40	;*************************************************************************/	40	;*************************************************************************/
41	%include "asm_inc.asm"	41	%include "asm_inc.asm"
@@ -61,9 +61,9 @@ SECTION .rodata align=16 · 19 unmodified lines
61		61
62	ALIGN 16	62	ALIGN 16
63	shufb_mask_low:	63	shufb_mask_low:
64	db 00h, 80h, 02h, 80h, 04h, 80h, 06h, 80h, 08h, 80h, 0ah, 80h, 0ch, 80h, 0eh, 80h	64	db 00h, 80h, 02h, 80h, 04h, 80h, 06h, 80h, 08h, 80h, 0ah, 80h, 0ch, 80h, 0eh, 80h
65	shufb_mask_high:	65	shufb_mask_high:
66	db 01h, 80h, 03h, 80h, 05h, 80h, 07h, 80h, 09h, 80h, 0bh, 80h, 0dh, 80h, 0fh, 80h	66	db 01h, 80h, 03h, 80h, 05h, 80h, 07h, 80h, 09h, 80h, 0bh, 80h, 0dh, 80h, 0fh, 80h
67		67
68		68
69	;***********************************************************************	69	;***********************************************************************
@@ -73,737 +73,737 @@ shufb_mask_high: · 3 unmodified lines
73	SECTION .text	73	SECTION .text
74		74
75	;***********************************************************************	75	;***********************************************************************
76	; void DyadicBilinearDownsamplerWidthx32_sse( unsigned char* pDst, const int iDstStride,	76	; void DyadicBilinearDownsamplerWidthx32_sse( unsigned char* pDst, const int iDstStride,
77	; unsigned char* pSrc, const int iSrcStride,	77	; unsigned char* pSrc, const int iSrcStride,
78	; const int iSrcWidth, const int iSrcHeight );	78	; const int iSrcWidth, const int iSrcHeight );
79	;***********************************************************************	79	;***********************************************************************
80	WELS_EXTERN DyadicBilinearDownsamplerWidthx32_sse	80	WELS_EXTERN DyadicBilinearDownsamplerWidthx32_sse
81	push ebx	81	push ebx
82	push edx	82	push edx
83	push esi	83	push esi
84	push edi	84	push edi
85	push ebp	85	push ebp
86		86
87	mov edi, [esp+24] ; pDst	87	mov edi, [esp+24] ; pDst
88	mov edx, [esp+28] ; iDstStride	88	mov edx, [esp+28] ; iDstStride
89	mov esi, [esp+32] ; pSrc	89	mov esi, [esp+32] ; pSrc
90	mov ecx, [esp+36] ; iSrcStride	90	mov ecx, [esp+36] ; iSrcStride
91	mov ebp, [esp+44] ; iSrcHeight	91	mov ebp, [esp+44] ; iSrcHeight
92		92
93	sar ebp, $01 ; iSrcHeight >> 1	93	sar ebp, $01 ; iSrcHeight >> 1
94		94
95	.yloops:	95	.yloops:
96	mov eax, [esp+40] ; iSrcWidth	96	mov eax, [esp+40] ; iSrcWidth
97	sar eax, $01 ; iSrcWidth >> 1	97	sar eax, $01 ; iSrcWidth >> 1
98	mov ebx, eax ; iDstWidth restored at ebx	98	mov ebx, eax ; iDstWidth restored at ebx
99	sar eax, $04 ; (iSrcWidth >> 1) / 16 ; loop count = num_of_mb	99	sar eax, $04 ; (iSrcWidth >> 1) / 16 ; loop count = num_of_mb
100	neg ebx ; - (iSrcWidth >> 1)	100	neg ebx ; - (iSrcWidth >> 1)
101	; each loop = source bandwidth: 32 bytes	101	; each loop = source bandwidth: 32 bytes
102	.xloops:	102	.xloops:
103	; 1st part horizonal loop: x16 bytes	103	; 1st part horizonal loop: x16 bytes
104	; mem hi<- ->lo	104	; mem hi<- ->lo
105	;1st Line Src: mm0: d D c C b B a A mm1: h H g G f F e E	105	;1st Line Src: mm0: d D c C b B a A mm1: h H g G f F e E
106	;2nd Line Src: mm2: l L k K j J i I mm3: p P o O n N m M	106	;2nd Line Src: mm2: l L k K j J i I mm3: p P o O n N m M
107	;=> target:	107	;=> target:
108	;: H G F E D C B A, P O N M L K J I	108	;: H G F E D C B A, P O N M L K J I
109	;: h g f e d c b a, p o n m l k j i	109	;: h g f e d c b a, p o n m l k j i
110	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;	110	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
111	movq mm0, [esi] ; 1st pSrc line	111	movq mm0, [esi] ; 1st pSrc line
112	movq mm1, [esi+8] ; 1st pSrc line + 8	112	movq mm1, [esi+8] ; 1st pSrc line + 8
113	movq mm2, [esi+ecx] ; 2nd pSrc line	113	movq mm2, [esi+ecx] ; 2nd pSrc line
114	movq mm3, [esi+ecx+8] ; 2nd pSrc line + 8	114	movq mm3, [esi+ecx+8] ; 2nd pSrc line + 8
115		115
116	; to handle mm0, mm1, mm2, mm3	116	; to handle mm0, mm1, mm2, mm3
117	pshufw mm4, mm0, 0d8h ; d D b B c C a A ; 11011000 B	117	pshufw mm4, mm0, 0d8h ; d D b B c C a A ; 11011000 B
118	pshufw mm5, mm4, 04eh ; c C a A d D b B ; 01001110 B	118	pshufw mm5, mm4, 04eh ; c C a A d D b B ; 01001110 B
119	punpcklbw mm4, mm5 ; d c D C b a B A	119	punpcklbw mm4, mm5 ; d c D C b a B A
120	pshufw mm4, mm4, 0d8h ; d c b a D C B A ; 11011000 B: mm4	120	pshufw mm4, mm4, 0d8h ; d c b a D C B A ; 11011000 B: mm4
121		121
122	pshufw mm5, mm1, 0d8h ; h H f F g G e E ; 11011000 B	122	pshufw mm5, mm1, 0d8h ; h H f F g G e E ; 11011000 B
123	pshufw mm6, mm5, 04eh ; g G e E h H f F ; 01001110 B	123	pshufw mm6, mm5, 04eh ; g G e E h H f F ; 01001110 B
124	punpcklbw mm5, mm6 ; h g H G f e F E	124	punpcklbw mm5, mm6 ; h g H G f e F E
125	pshufw mm5, mm5, 0d8h ; h g f e H G F E ; 11011000 B: mm5	125	pshufw mm5, mm5, 0d8h ; h g f e H G F E ; 11011000 B: mm5
126		126
127	pshufw mm6, mm2, 0d8h ; l L j J k K i I ; 11011000 B	127	pshufw mm6, mm2, 0d8h ; l L j J k K i I ; 11011000 B
128	pshufw mm7, mm6, 04eh ; k K i I l L j J ; 01001110 B	128	pshufw mm7, mm6, 04eh ; k K i I l L j J ; 01001110 B
129	punpcklbw mm6, mm7 ; l k L K j i J I	129	punpcklbw mm6, mm7 ; l k L K j i J I
130	pshufw mm6, mm6, 0d8h ; l k j i L K J I ; 11011000 B: mm6	130	pshufw mm6, mm6, 0d8h ; l k j i L K J I ; 11011000 B: mm6
131		131
132	pshufw mm7, mm3, 0d8h ; p P n N o O m M ; 11011000 B	132	pshufw mm7, mm3, 0d8h ; p P n N o O m M ; 11011000 B
133	pshufw mm0, mm7, 04eh ; o O m M p P n N ; 01001110 B	133	pshufw mm0, mm7, 04eh ; o O m M p P n N ; 01001110 B
134	punpcklbw mm7, mm0 ; p o P O n m N M	134	punpcklbw mm7, mm0 ; p o P O n m N M
135	pshufw mm7, mm7, 0d8h ; p o n m P O N M ; 11011000 B: mm7	135	pshufw mm7, mm7, 0d8h ; p o n m P O N M ; 11011000 B: mm7
136		136
137	; to handle mm4, mm5, mm6, mm7	137	; to handle mm4, mm5, mm6, mm7
138	movq mm0, mm4 ;	138	movq mm0, mm4 ;
139	punpckldq mm0, mm5 ; H G F E D C B A	139	punpckldq mm0, mm5 ; H G F E D C B A
140	punpckhdq mm4, mm5 ; h g f e d c b a	140	punpckhdq mm4, mm5 ; h g f e d c b a
141		141
142	movq mm1, mm6	142	movq mm1, mm6
143	punpckldq mm1, mm7 ; P O N M L K J I	143	punpckldq mm1, mm7 ; P O N M L K J I
144	punpckhdq mm6, mm7 ; p o n m l k j i	144	punpckhdq mm6, mm7 ; p o n m l k j i
145		145
146	; avg within MB horizon width (16 x 2 lines)	146	; avg within MB horizon width (16 x 2 lines)
147	pavgb mm0, mm4 ; (A+a+1)>>1, .., (H+h+1)>>1, temp_row1	147	pavgb mm0, mm4 ; (A+a+1)>>1, .., (H+h+1)>>1, temp_row1
148	pavgb mm1, mm6 ; (I+i+1)>>1, .., (P+p+1)>>1, temp_row2	148	pavgb mm1, mm6 ; (I+i+1)>>1, .., (P+p+1)>>1, temp_row2
149	pavgb mm0, mm1 ; (temp_row1+temp_row2+1)>>1, pending here and wait another horizonal part done then write memory once	149	pavgb mm0, mm1 ; (temp_row1+temp_row2+1)>>1, pending here and wait another horizonal part done then write memory once
150		150
151	; 2nd part horizonal loop: x16 bytes	151	; 2nd part horizonal loop: x16 bytes
152	; mem hi<- ->lo	152	; mem hi<- ->lo
153	;1st Line Src: mm0: d D c C b B a A mm1: h H g G f F e E	153	;1st Line Src: mm0: d D c C b B a A mm1: h H g G f F e E
154	;2nd Line Src: mm2: l L k K j J i I mm3: p P o O n N m M	154	;2nd Line Src: mm2: l L k K j J i I mm3: p P o O n N m M
155	;=> target:	155	;=> target:
156	;: H G F E D C B A, P O N M L K J I	156	;: H G F E D C B A, P O N M L K J I
157	;: h g f e d c b a, p o n m l k j i	157	;: h g f e d c b a, p o n m l k j i
158	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;	158	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
159	movq mm1, [esi+16] ; 1st pSrc line + 16	159	movq mm1, [esi+16] ; 1st pSrc line + 16
160	movq mm2, [esi+24] ; 1st pSrc line + 24	160	movq mm2, [esi+24] ; 1st pSrc line + 24
161	movq mm3, [esi+ecx+16] ; 2nd pSrc line + 16	161	movq mm3, [esi+ecx+16] ; 2nd pSrc line + 16
162	movq mm4, [esi+ecx+24] ; 2nd pSrc line + 24	162	movq mm4, [esi+ecx+24] ; 2nd pSrc line + 24
163		163
164	; to handle mm1, mm2, mm3, mm4	164	; to handle mm1, mm2, mm3, mm4
165	pshufw mm5, mm1, 0d8h ; d D b B c C a A ; 11011000 B	165	pshufw mm5, mm1, 0d8h ; d D b B c C a A ; 11011000 B
166	pshufw mm6, mm5, 04eh ; c C a A d D b B ; 01001110 B	166	pshufw mm6, mm5, 04eh ; c C a A d D b B ; 01001110 B
167	punpcklbw mm5, mm6 ; d c D C b a B A	167	punpcklbw mm5, mm6 ; d c D C b a B A
168	pshufw mm5, mm5, 0d8h ; d c b a D C B A ; 11011000 B: mm5	168	pshufw mm5, mm5, 0d8h ; d c b a D C B A ; 11011000 B: mm5
169		169
170	pshufw mm6, mm2, 0d8h ; h H f F g G e E ; 11011000 B	170	pshufw mm6, mm2, 0d8h ; h H f F g G e E ; 11011000 B
171	pshufw mm7, mm6, 04eh ; g G e E h H f F ; 01001110 B	171	pshufw mm7, mm6, 04eh ; g G e E h H f F ; 01001110 B
172	punpcklbw mm6, mm7 ; h g H G f e F E	172	punpcklbw mm6, mm7 ; h g H G f e F E
173	pshufw mm6, mm6, 0d8h ; h g f e H G F E ; 11011000 B: mm6	173	pshufw mm6, mm6, 0d8h ; h g f e H G F E ; 11011000 B: mm6
174		174
175	pshufw mm7, mm3, 0d8h ; l L j J k K i I ; 11011000 B	175	pshufw mm7, mm3, 0d8h ; l L j J k K i I ; 11011000 B
176	pshufw mm1, mm7, 04eh ; k K i I l L j J ; 01001110 B	176	pshufw mm1, mm7, 04eh ; k K i I l L j J ; 01001110 B
177	punpcklbw mm7, mm1 ; l k L K j i J I	177	punpcklbw mm7, mm1 ; l k L K j i J I
178	pshufw mm7, mm7, 0d8h ; l k j i L K J I ; 11011000 B: mm7	178	pshufw mm7, mm7, 0d8h ; l k j i L K J I ; 11011000 B: mm7
179		179
180	pshufw mm1, mm4, 0d8h ; p P n N o O m M ; 11011000 B	180	pshufw mm1, mm4, 0d8h ; p P n N o O m M ; 11011000 B
181	pshufw mm2, mm1, 04eh ; o O m M p P n N ; 01001110 B	181	pshufw mm2, mm1, 04eh ; o O m M p P n N ; 01001110 B
182	punpcklbw mm1, mm2 ; p o P O n m N M	182	punpcklbw mm1, mm2 ; p o P O n m N M
183	pshufw mm1, mm1, 0d8h ; p o n m P O N M ; 11011000 B: mm1	183	pshufw mm1, mm1, 0d8h ; p o n m P O N M ; 11011000 B: mm1
184		184
185	; to handle mm5, mm6, mm7, mm1	185	; to handle mm5, mm6, mm7, mm1
186	movq mm2, mm5	186	movq mm2, mm5
187	punpckldq mm2, mm6 ; H G F E D C B A	187	punpckldq mm2, mm6 ; H G F E D C B A
188	punpckhdq mm5, mm6 ; h g f e d c b a	188	punpckhdq mm5, mm6 ; h g f e d c b a
189		189
190	movq mm3, mm7	190	movq mm3, mm7
191	punpckldq mm3, mm1 ; P O N M L K J I	191	punpckldq mm3, mm1 ; P O N M L K J I
192	punpckhdq mm7, mm1 ; p o n m l k j i	192	punpckhdq mm7, mm1 ; p o n m l k j i
193		193
194	; avg within MB horizon width (16 x 2 lines)	194	; avg within MB horizon width (16 x 2 lines)
195	pavgb mm2, mm5 ; (A+a+1)>>1, .., (H+h+1)>>1, temp_row1	195	pavgb mm2, mm5 ; (A+a+1)>>1, .., (H+h+1)>>1, temp_row1
196	pavgb mm3, mm7 ; (I+i+1)>>1, .., (P+p+1)>>1, temp_row2	196	pavgb mm3, mm7 ; (I+i+1)>>1, .., (P+p+1)>>1, temp_row2
197	pavgb mm2, mm3 ; (temp_row1+temp_row2+1)>>1, done in another 2nd horizonal part	197	pavgb mm2, mm3 ; (temp_row1+temp_row2+1)>>1, done in another 2nd horizonal part
198		198
199	movq [edi ], mm0	199	movq [edi ], mm0
200	movq [edi+8], mm2	200	movq [edi+8], mm2
201		201
202	; next SMB	202	; next SMB
203	lea esi, [esi+32]	203	lea esi, [esi+32]
204	lea edi, [edi+16]	204	lea edi, [edi+16]
205		205
206	dec eax	206	dec eax
207	jg near .xloops	207	jg near .xloops
208		208
209	; next line	209	; next line
210	lea esi, [esi+2*ecx] ; next end of lines	210	lea esi, [esi+2*ecx] ; next end of lines
211	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]	211	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]
212	lea edi, [edi+edx]	212	lea edi, [edi+edx]
213	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]	213	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]
214		214
215	dec ebp	215	dec ebp
216	jg near .yloops	216	jg near .yloops
217		217
218	WELSEMMS	218	WELSEMMS
219	pop ebp	219	pop ebp
220	pop edi	220	pop edi
221	pop esi	221	pop esi
222	pop edx	222	pop edx
223	pop ebx	223	pop ebx
224	ret	224	ret
225		225
226	;***********************************************************************	226	;***********************************************************************
227	; void DyadicBilinearDownsamplerWidthx16_sse( unsigned char* pDst, const int iDstStride,	227	; void DyadicBilinearDownsamplerWidthx16_sse( unsigned char* pDst, const int iDstStride,
228	; unsigned char* pSrc, const int iSrcStride,	228	; unsigned char* pSrc, const int iSrcStride,
229	; const int iSrcWidth, const int iSrcHeight );	229	; const int iSrcWidth, const int iSrcHeight );
230	;***********************************************************************	230	;***********************************************************************
231	WELS_EXTERN DyadicBilinearDownsamplerWidthx16_sse	231	WELS_EXTERN DyadicBilinearDownsamplerWidthx16_sse
232	push ebx	232	push ebx
233	push edx	233	push edx
234	push esi	234	push esi
235	push edi	235	push edi
236	push ebp	236	push ebp
237		237
238	mov edi, [esp+24] ; pDst	238	mov edi, [esp+24] ; pDst
239	mov edx, [esp+28] ; iDstStride	239	mov edx, [esp+28] ; iDstStride
240	mov esi, [esp+32] ; pSrc	240	mov esi, [esp+32] ; pSrc
241	mov ecx, [esp+36] ; iSrcStride	241	mov ecx, [esp+36] ; iSrcStride
242	mov ebp, [esp+44] ; iSrcHeight	242	mov ebp, [esp+44] ; iSrcHeight
243		243
244	sar ebp, $01 ; iSrcHeight >> 1	244	sar ebp, $01 ; iSrcHeight >> 1
245		245
246	.yloops:	246	.yloops:
247	mov eax, [esp+40] ; iSrcWidth	247	mov eax, [esp+40] ; iSrcWidth
248	sar eax, $01 ; iSrcWidth >> 1	248	sar eax, $01 ; iSrcWidth >> 1
249	mov ebx, eax ; iDstWidth restored at ebx	249	mov ebx, eax ; iDstWidth restored at ebx
250	sar eax, $03 ; (iSrcWidth >> 1) / 8 ; loop count = num_of_mb	250	sar eax, $03 ; (iSrcWidth >> 1) / 8 ; loop count = num_of_mb
251	neg ebx ; - (iSrcWidth >> 1)	251	neg ebx ; - (iSrcWidth >> 1)
252	; each loop = source bandwidth: 16 bytes	252	; each loop = source bandwidth: 16 bytes
253	.xloops:	253	.xloops:
254	; 1st part horizonal loop: x16 bytes	254	; 1st part horizonal loop: x16 bytes
255	; mem hi<- ->lo	255	; mem hi<- ->lo
256	;1st Line Src: mm0: d D c C b B a A mm1: h H g G f F e E	256	;1st Line Src: mm0: d D c C b B a A mm1: h H g G f F e E
257	;2nd Line Src: mm2: l L k K j J i I mm3: p P o O n N m M	257	;2nd Line Src: mm2: l L k K j J i I mm3: p P o O n N m M
258	;=> target:	258	;=> target:
259	;: H G F E D C B A, P O N M L K J I	259	;: H G F E D C B A, P O N M L K J I
260	;: h g f e d c b a, p o n m l k j i	260	;: h g f e d c b a, p o n m l k j i
261	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;	261	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
262	movq mm0, [esi] ; 1st pSrc line	262	movq mm0, [esi] ; 1st pSrc line
263	movq mm1, [esi+8] ; 1st pSrc line + 8	263	movq mm1, [esi+8] ; 1st pSrc line + 8
264	movq mm2, [esi+ecx] ; 2nd pSrc line	264	movq mm2, [esi+ecx] ; 2nd pSrc line
265	movq mm3, [esi+ecx+8] ; 2nd pSrc line + 8	265	movq mm3, [esi+ecx+8] ; 2nd pSrc line + 8
266		266
267	; to handle mm0, mm1, mm2, mm3	267	; to handle mm0, mm1, mm2, mm3
268	pshufw mm4, mm0, 0d8h ; d D b B c C a A ; 11011000 B	268	pshufw mm4, mm0, 0d8h ; d D b B c C a A ; 11011000 B
269	pshufw mm5, mm4, 04eh ; c C a A d D b B ; 01001110 B	269	pshufw mm5, mm4, 04eh ; c C a A d D b B ; 01001110 B
270	punpcklbw mm4, mm5 ; d c D C b a B A	270	punpcklbw mm4, mm5 ; d c D C b a B A
271	pshufw mm4, mm4, 0d8h ; d c b a D C B A ; 11011000 B: mm4	271	pshufw mm4, mm4, 0d8h ; d c b a D C B A ; 11011000 B: mm4
272		272
273	pshufw mm5, mm1, 0d8h ; h H f F g G e E ; 11011000 B	273	pshufw mm5, mm1, 0d8h ; h H f F g G e E ; 11011000 B
274	pshufw mm6, mm5, 04eh ; g G e E h H f F ; 01001110 B	274	pshufw mm6, mm5, 04eh ; g G e E h H f F ; 01001110 B
275	punpcklbw mm5, mm6 ; h g H G f e F E	275	punpcklbw mm5, mm6 ; h g H G f e F E
276	pshufw mm5, mm5, 0d8h ; h g f e H G F E ; 11011000 B: mm5	276	pshufw mm5, mm5, 0d8h ; h g f e H G F E ; 11011000 B: mm5
277		277
278	pshufw mm6, mm2, 0d8h ; l L j J k K i I ; 11011000 B	278	pshufw mm6, mm2, 0d8h ; l L j J k K i I ; 11011000 B
279	pshufw mm7, mm6, 04eh ; k K i I l L j J ; 01001110 B	279	pshufw mm7, mm6, 04eh ; k K i I l L j J ; 01001110 B
280	punpcklbw mm6, mm7 ; l k L K j i J I	280	punpcklbw mm6, mm7 ; l k L K j i J I
281	pshufw mm6, mm6, 0d8h ; l k j i L K J I ; 11011000 B: mm6	281	pshufw mm6, mm6, 0d8h ; l k j i L K J I ; 11011000 B: mm6
282		282
283	pshufw mm7, mm3, 0d8h ; p P n N o O m M ; 11011000 B	283	pshufw mm7, mm3, 0d8h ; p P n N o O m M ; 11011000 B
284	pshufw mm0, mm7, 04eh ; o O m M p P n N ; 01001110 B	284	pshufw mm0, mm7, 04eh ; o O m M p P n N ; 01001110 B
285	punpcklbw mm7, mm0 ; p o P O n m N M	285	punpcklbw mm7, mm0 ; p o P O n m N M
286	pshufw mm7, mm7, 0d8h ; p o n m P O N M ; 11011000 B: mm7	286	pshufw mm7, mm7, 0d8h ; p o n m P O N M ; 11011000 B: mm7
287		287
288	; to handle mm4, mm5, mm6, mm7	288	; to handle mm4, mm5, mm6, mm7
289	movq mm0, mm4 ;	289	movq mm0, mm4 ;
290	punpckldq mm0, mm5 ; H G F E D C B A	290	punpckldq mm0, mm5 ; H G F E D C B A
291	punpckhdq mm4, mm5 ; h g f e d c b a	291	punpckhdq mm4, mm5 ; h g f e d c b a
292		292
293	movq mm1, mm6	293	movq mm1, mm6
294	punpckldq mm1, mm7 ; P O N M L K J I	294	punpckldq mm1, mm7 ; P O N M L K J I
295	punpckhdq mm6, mm7 ; p o n m l k j i	295	punpckhdq mm6, mm7 ; p o n m l k j i
296		296
297	; avg within MB horizon width (16 x 2 lines)	297	; avg within MB horizon width (16 x 2 lines)
298	pavgb mm0, mm4 ; (A+a+1)>>1, .., (H+h+1)>>1, temp_row1	298	pavgb mm0, mm4 ; (A+a+1)>>1, .., (H+h+1)>>1, temp_row1
299	pavgb mm1, mm6 ; (I+i+1)>>1, .., (P+p+1)>>1, temp_row2	299	pavgb mm1, mm6 ; (I+i+1)>>1, .., (P+p+1)>>1, temp_row2
300	pavgb mm0, mm1 ; (temp_row1+temp_row2+1)>>1, pending here and wait another horizonal part done then write memory once	300	pavgb mm0, mm1 ; (temp_row1+temp_row2+1)>>1, pending here and wait another horizonal part done then write memory once
301		301
302	movq [edi ], mm0	302	movq [edi ], mm0
303		303
304	; next SMB	304	; next SMB
305	lea esi, [esi+16]	305	lea esi, [esi+16]
306	lea edi, [edi+8]	306	lea edi, [edi+8]
307		307
308	dec eax	308	dec eax
309	jg near .xloops	309	jg near .xloops
310		310
311	; next line	311	; next line
312	lea esi, [esi+2*ecx] ; next end of lines	312	lea esi, [esi+2*ecx] ; next end of lines
313	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]	313	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]
314	lea edi, [edi+edx]	314	lea edi, [edi+edx]
315	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]	315	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]
316		316
317	dec ebp	317	dec ebp
318	jg near .yloops	318	jg near .yloops
319		319
320	WELSEMMS	320	WELSEMMS
321	pop ebp	321	pop ebp
322	pop edi	322	pop edi
323	pop esi	323	pop esi
324	pop edx	324	pop edx
325	pop ebx	325	pop ebx
326	ret	326	ret
327		327
328	;***********************************************************************	328	;***********************************************************************
329	; void DyadicBilinearDownsamplerWidthx8_sse( unsigned char* pDst, const int iDstStride,	329	; void DyadicBilinearDownsamplerWidthx8_sse( unsigned char* pDst, const int iDstStride,
330	; unsigned char* pSrc, const int iSrcStride,	330	; unsigned char* pSrc, const int iSrcStride,
331	; const int iSrcWidth, const int iSrcHeight );	331	; const int iSrcWidth, const int iSrcHeight );
332	;***********************************************************************	332	;***********************************************************************
333	WELS_EXTERN DyadicBilinearDownsamplerWidthx8_sse	333	WELS_EXTERN DyadicBilinearDownsamplerWidthx8_sse
334	push ebx	334	push ebx
335	push edx	335	push edx
336	push esi	336	push esi
337	push edi	337	push edi
338	push ebp	338	push ebp
339		339
340	mov edi, [esp+24] ; pDst	340	mov edi, [esp+24] ; pDst
341	mov edx, [esp+28] ; iDstStride	341	mov edx, [esp+28] ; iDstStride
342	mov esi, [esp+32] ; pSrc	342	mov esi, [esp+32] ; pSrc
343	mov ecx, [esp+36] ; iSrcStride	343	mov ecx, [esp+36] ; iSrcStride
344	mov ebp, [esp+44] ; iSrcHeight	344	mov ebp, [esp+44] ; iSrcHeight
345		345
346	sar ebp, $01 ; iSrcHeight >> 1	346	sar ebp, $01 ; iSrcHeight >> 1
347		347
348	.yloops:	348	.yloops:
349	mov eax, [esp+40] ; iSrcWidth	349	mov eax, [esp+40] ; iSrcWidth
350	sar eax, $01 ; iSrcWidth >> 1	350	sar eax, $01 ; iSrcWidth >> 1
351	mov ebx, eax ; iDstWidth restored at ebx	351	mov ebx, eax ; iDstWidth restored at ebx
352	sar eax, $02 ; (iSrcWidth >> 1) / 4 ; loop count = num_of_mb	352	sar eax, $02 ; (iSrcWidth >> 1) / 4 ; loop count = num_of_mb
353	neg ebx ; - (iSrcWidth >> 1)	353	neg ebx ; - (iSrcWidth >> 1)
354	; each loop = source bandwidth: 8 bytes	354	; each loop = source bandwidth: 8 bytes
355	.xloops:	355	.xloops:
356	; 1st part horizonal loop: x8 bytes	356	; 1st part horizonal loop: x8 bytes
357	; mem hi<- ->lo	357	; mem hi<- ->lo
358	;1st Line Src: mm0: d D c C b B a A	358	;1st Line Src: mm0: d D c C b B a A
359	;2nd Line Src: mm1: h H g G f F e E	359	;2nd Line Src: mm1: h H g G f F e E
360	;=> target:	360	;=> target:
361	;: H G F E D C B A	361	;: H G F E D C B A
362	;: h g f e d c b a	362	;: h g f e d c b a
363	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;	363	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
364	movq mm0, [esi] ; 1st pSrc line	364	movq mm0, [esi] ; 1st pSrc line
365	movq mm1, [esi+ecx] ; 2nd pSrc line	365	movq mm1, [esi+ecx] ; 2nd pSrc line
366		366
367	; to handle mm0, mm1, mm2, mm3	367	; to handle mm0, mm1, mm2, mm3
368	pshufw mm2, mm0, 0d8h ; d D b B c C a A ; 11011000 B	368	pshufw mm2, mm0, 0d8h ; d D b B c C a A ; 11011000 B
369	pshufw mm3, mm2, 04eh ; c C a A d D b B ; 01001110 B	369	pshufw mm3, mm2, 04eh ; c C a A d D b B ; 01001110 B
370	punpcklbw mm2, mm3 ; d c D C b a B A	370	punpcklbw mm2, mm3 ; d c D C b a B A
371	pshufw mm2, mm2, 0d8h ; d c b a D C B A ; 11011000 B: mm4	371	pshufw mm2, mm2, 0d8h ; d c b a D C B A ; 11011000 B: mm4
372		372
373	pshufw mm4, mm1, 0d8h ; h H f F g G e E ; 11011000 B	373	pshufw mm4, mm1, 0d8h ; h H f F g G e E ; 11011000 B
374	pshufw mm5, mm4, 04eh ; g G e E h H f F ; 01001110 B	374	pshufw mm5, mm4, 04eh ; g G e E h H f F ; 01001110 B
375	punpcklbw mm4, mm5 ; h g H G f e F E	375	punpcklbw mm4, mm5 ; h g H G f e F E
376	pshufw mm4, mm4, 0d8h ; h g f e H G F E ; 11011000 B: mm5	376	pshufw mm4, mm4, 0d8h ; h g f e H G F E ; 11011000 B: mm5
377		377
378	; to handle mm2, mm4	378	; to handle mm2, mm4
379	movq mm0, mm2 ;	379	movq mm0, mm2 ;
380	punpckldq mm0, mm4 ; H G F E D C B A	380	punpckldq mm0, mm4 ; H G F E D C B A
381	punpckhdq mm2, mm4 ; h g f e d c b a	381	punpckhdq mm2, mm4 ; h g f e d c b a
382		382
383	; avg within MB horizon width (16 x 2 lines)	383	; avg within MB horizon width (16 x 2 lines)
384	pavgb mm0, mm2 ; (H+h+1)>>1, .., (A+a+1)>>1, temp_row1, 2	384	pavgb mm0, mm2 ; (H+h+1)>>1, .., (A+a+1)>>1, temp_row1, 2
385	pshufw mm1, mm0, 04eh ; 01001110 B	385	pshufw mm1, mm0, 04eh ; 01001110 B
386	pavgb mm0, mm1 ; (temp_row1+temp_row2+1)>>1, pending here and wait another horizonal part done then write memory once	386	pavgb mm0, mm1 ; (temp_row1+temp_row2+1)>>1, pending here and wait another horizonal part done then write memory once
387		387
388	movd [edi], mm0	388	movd [edi], mm0
389		389
390	; next unit	390	; next unit
391	lea esi, [esi+8]	391	lea esi, [esi+8]
392	lea edi, [edi+4]	392	lea edi, [edi+4]
393		393
394	dec eax	394	dec eax
395	jg near .xloops	395	jg near .xloops
396		396
397	; next line	397	; next line
398	lea esi, [esi+2*ecx] ; next end of lines	398	lea esi, [esi+2*ecx] ; next end of lines
399	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]	399	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]
400	lea edi, [edi+edx]	400	lea edi, [edi+edx]
401	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]	401	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]
402		402
403	dec ebp	403	dec ebp
404	jg near .yloops	404	jg near .yloops
405		405
406	WELSEMMS	406	WELSEMMS
407	pop ebp	407	pop ebp
408	pop edi	408	pop edi
409	pop esi	409	pop esi
410	pop edx	410	pop edx
411	pop ebx	411	pop ebx
412	ret	412	ret
413		413
414		414
415		415
416	; got about 50% improvement over DyadicBilinearDownsamplerWidthx32_sse	416	; got about 50% improvement over DyadicBilinearDownsamplerWidthx32_sse
417	;***********************************************************************	417	;***********************************************************************
418	; void DyadicBilinearDownsamplerWidthx32_ssse3( unsigned char* pDst, const int iDstStride,	418	; void DyadicBilinearDownsamplerWidthx32_ssse3( unsigned char* pDst, const int iDstStride,
419	; unsigned char* pSrc, const int iSrcStride,	419	; unsigned char* pSrc, const int iSrcStride,
420	; const int iSrcWidth, const int iSrcHeight );	420	; const int iSrcWidth, const int iSrcHeight );
421	;***********************************************************************	421	;***********************************************************************
422	WELS_EXTERN DyadicBilinearDownsamplerWidthx32_ssse3	422	WELS_EXTERN DyadicBilinearDownsamplerWidthx32_ssse3
423	push ebx	423	push ebx
424	push edx	424	push edx
425	push esi	425	push esi
426	push edi	426	push edi
427	push ebp	427	push ebp
428		428
429	mov edi, [esp+24] ; pDst	429	mov edi, [esp+24] ; pDst
430	mov edx, [esp+28] ; iDstStride	430	mov edx, [esp+28] ; iDstStride
431	mov esi, [esp+32] ; pSrc	431	mov esi, [esp+32] ; pSrc
432	mov ecx, [esp+36] ; iSrcStride	432	mov ecx, [esp+36] ; iSrcStride
433	mov ebp, [esp+44] ; iSrcHeight	433	mov ebp, [esp+44] ; iSrcHeight
434		434
435	sar ebp, $01 ; iSrcHeight >> 1	435	sar ebp, $01 ; iSrcHeight >> 1
436		436
437	movdqa xmm7, [shufb_mask_low] ; mask low	437	movdqa xmm7, [shufb_mask_low] ; mask low
438	movdqa xmm6, [shufb_mask_high] ; mask high	438	movdqa xmm6, [shufb_mask_high] ; mask high
439		439
440	.yloops:	440	.yloops:
441	mov eax, [esp+40] ; iSrcWidth	441	mov eax, [esp+40] ; iSrcWidth
442	sar eax, $01 ; iSrcWidth >> 1	442	sar eax, $01 ; iSrcWidth >> 1
443	mov ebx, eax ; iDstWidth restored at ebx	443	mov ebx, eax ; iDstWidth restored at ebx
444	sar eax, $04 ; (iSrcWidth >> 1) / 16 ; loop count = num_of_mb	444	sar eax, $04 ; (iSrcWidth >> 1) / 16 ; loop count = num_of_mb
445	neg ebx ; - (iSrcWidth >> 1)	445	neg ebx ; - (iSrcWidth >> 1)
446	; each loop = source bandwidth: 32 bytes	446	; each loop = source bandwidth: 32 bytes
447	.xloops:	447	.xloops:
448	; 1st part horizonal loop: x16 bytes	448	; 1st part horizonal loop: x16 bytes
449	; mem hi<- ->lo	449	; mem hi<- ->lo
450	;1st Line Src: xmm0: h H g G f F e E d D c C b B a A	450	;1st Line Src: xmm0: h H g G f F e E d D c C b B a A
451	; xmm1: p P o O n N m M l L k K j J i I	451	; xmm1: p P o O n N m M l L k K j J i I
452	;2nd Line Src: xmm2: h H g G f F e E d D c C b B a A	452	;2nd Line Src: xmm2: h H g G f F e E d D c C b B a A
453	; xmm3: p P o O n N m M l L k K j J i I	453	; xmm3: p P o O n N m M l L k K j J i I
454	;=> target:	454	;=> target:
455	;: P O N M L K J I H G F E D C B A	455	;: P O N M L K J I H G F E D C B A
456	;: p o n m l k j i h g f e d c b a	456	;: p o n m l k j i h g f e d c b a
457	;: P .. A	457	;: P .. A
458	;: p .. a	458	;: p .. a
459		459
460	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;	460	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
461	movdqa xmm0, [esi] ; 1st_src_line	461	movdqa xmm0, [esi] ; 1st_src_line
462	movdqa xmm1, [esi+16] ; 1st_src_line + 16	462	movdqa xmm1, [esi+16] ; 1st_src_line + 16
463	movdqa xmm2, [esi+ecx] ; 2nd_src_line	463	movdqa xmm2, [esi+ecx] ; 2nd_src_line
464	movdqa xmm3, [esi+ecx+16] ; 2nd_src_line + 16	464	movdqa xmm3, [esi+ecx+16] ; 2nd_src_line + 16
465		465
466	; packing & avg	466	; packing & avg
467	movdqa xmm4, xmm0 ; h H g G f F e E d D c C b B a A	467	movdqa xmm4, xmm0 ; h H g G f F e E d D c C b B a A
468	pshufb xmm0, xmm7 ; 0 H 0 G 0 F 0 E 0 D 0 C 0 B 0 A	468	pshufb xmm0, xmm7 ; 0 H 0 G 0 F 0 E 0 D 0 C 0 B 0 A
469	pshufb xmm4, xmm6 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a	469	pshufb xmm4, xmm6 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
470	; another implementation for xmm4 high bits	470	; another implementation for xmm4 high bits
471	; psubb xmm4, xmm0 ; h 0 g 0 f 0 e 0 d 0 c 0 b 0 a 0	471	; psubb xmm4, xmm0 ; h 0 g 0 f 0 e 0 d 0 c 0 b 0 a 0
472	; psrlw xmm4, 8 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a	472	; psrlw xmm4, 8 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
473	pavgb xmm0, xmm4	473	pavgb xmm0, xmm4
474		474
475	movdqa xmm5, xmm1	475	movdqa xmm5, xmm1
476	pshufb xmm1, xmm7	476	pshufb xmm1, xmm7
477	pshufb xmm5, xmm6	477	pshufb xmm5, xmm6
478	; psubb xmm5, xmm1	478	; psubb xmm5, xmm1
479	; psrlw xmm5, 8	479	; psrlw xmm5, 8
480	pavgb xmm1, xmm5	480	pavgb xmm1, xmm5
481		481
482	movdqa xmm4, xmm2	482	movdqa xmm4, xmm2
483	pshufb xmm2, xmm7	483	pshufb xmm2, xmm7
484	pshufb xmm4, xmm6	484	pshufb xmm4, xmm6
485	; psubb xmm4, xmm2	485	; psubb xmm4, xmm2
486	; psrlw xmm4, 8	486	; psrlw xmm4, 8
487	pavgb xmm2, xmm4	487	pavgb xmm2, xmm4
488		488
489	movdqa xmm5, xmm3	489	movdqa xmm5, xmm3
490	pshufb xmm3, xmm7	490	pshufb xmm3, xmm7
491	pshufb xmm5, xmm6	491	pshufb xmm5, xmm6
492	; psubb xmm5, xmm3	492	; psubb xmm5, xmm3
493	; psrlw xmm5, 8	493	; psrlw xmm5, 8
494	pavgb xmm3, xmm5	494	pavgb xmm3, xmm5
495		495
496	packuswb xmm0, xmm1	496	packuswb xmm0, xmm1
497	packuswb xmm2, xmm3	497	packuswb xmm2, xmm3
498	pavgb xmm0, xmm2	498	pavgb xmm0, xmm2
499		499
500	; write pDst	500	; write pDst
501	movdqa [edi], xmm0	501	movdqa [edi], xmm0
502		502
503	; next SMB	503	; next SMB
504	lea esi, [esi+32]	504	lea esi, [esi+32]
505	lea edi, [edi+16]	505	lea edi, [edi+16]
506		506
507	dec eax	507	dec eax
508	jg near .xloops	508	jg near .xloops
509		509
510	; next line	510	; next line
511	lea esi, [esi+2*ecx] ; next end of lines	511	lea esi, [esi+2*ecx] ; next end of lines
512	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]	512	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]
513	lea edi, [edi+edx]	513	lea edi, [edi+edx]
514	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]	514	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]
515		515
516	dec ebp	516	dec ebp
517	jg near .yloops	517	jg near .yloops
518		518
519	pop ebp	519	pop ebp
520	pop edi	520	pop edi
521	pop esi	521	pop esi
522	pop edx	522	pop edx
523	pop ebx	523	pop ebx
524	ret	524	ret
525		525
526	;***********************************************************************	526	;***********************************************************************
527	; void DyadicBilinearDownsamplerWidthx16_ssse3( unsigned char* pDst, const int iDstStride,	527	; void DyadicBilinearDownsamplerWidthx16_ssse3( unsigned char* pDst, const int iDstStride,
528	; unsigned char* pSrc, const int iSrcStride,	528	; unsigned char* pSrc, const int iSrcStride,
529	; const int iSrcWidth, const int iSrcHeight );	529	; const int iSrcWidth, const int iSrcHeight );
530	;***********************************************************************	530	;***********************************************************************
531	WELS_EXTERN DyadicBilinearDownsamplerWidthx16_ssse3	531	WELS_EXTERN DyadicBilinearDownsamplerWidthx16_ssse3
532	push ebx	532	push ebx
533	push edx	533	push edx
534	push esi	534	push esi
535	push edi	535	push edi
536	push ebp	536	push ebp
537		537
538	mov edi, [esp+24] ; pDst	538	mov edi, [esp+24] ; pDst
539	mov edx, [esp+28] ; iDstStride	539	mov edx, [esp+28] ; iDstStride
540	mov esi, [esp+32] ; pSrc	540	mov esi, [esp+32] ; pSrc
541	mov ecx, [esp+36] ; iSrcStride	541	mov ecx, [esp+36] ; iSrcStride
542	mov ebp, [esp+44] ; iSrcHeight	542	mov ebp, [esp+44] ; iSrcHeight
543		543
544	sar ebp, $01 ; iSrcHeight >> 1	544	sar ebp, $01 ; iSrcHeight >> 1
545	movdqa xmm7, [shufb_mask_low] ; mask low	545	movdqa xmm7, [shufb_mask_low] ; mask low
546	movdqa xmm6, [shufb_mask_high] ; mask high	546	movdqa xmm6, [shufb_mask_high] ; mask high
547		547
548	.yloops:	548	.yloops:
549	mov eax, [esp+40] ; iSrcWidth	549	mov eax, [esp+40] ; iSrcWidth
550	sar eax, $01 ; iSrcWidth >> 1	550	sar eax, $01 ; iSrcWidth >> 1
551	mov ebx, eax ; iDstWidth restored at ebx	551	mov ebx, eax ; iDstWidth restored at ebx
552	sar eax, $03 ; (iSrcWidth >> 1) / 8 ; loop count = num_of_mb	552	sar eax, $03 ; (iSrcWidth >> 1) / 8 ; loop count = num_of_mb
553	neg ebx ; - (iSrcWidth >> 1)	553	neg ebx ; - (iSrcWidth >> 1)
554	; each loop = source bandwidth: 16 bytes	554	; each loop = source bandwidth: 16 bytes
555	.xloops:	555	.xloops:
556	; horizonal loop: x16 bytes by source	556	; horizonal loop: x16 bytes by source
557	; mem hi<- ->lo	557	; mem hi<- ->lo
558	;1st line pSrc: xmm0: h H g G f F e E d D c C b B a A	558	;1st line pSrc: xmm0: h H g G f F e E d D c C b B a A
559	;2nd line pSrc: xmm1: p P o O n N m M l L k K j J i I	559	;2nd line pSrc: xmm1: p P o O n N m M l L k K j J i I
560	;=> target:	560	;=> target:
561	;: H G F E D C B A, P O N M L K J I	561	;: H G F E D C B A, P O N M L K J I
562	;: h g f e d c b a, p o n m l k j i	562	;: h g f e d c b a, p o n m l k j i
563		563
564	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;	564	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
565	movdqa xmm0, [esi] ; 1st_src_line	565	movdqa xmm0, [esi] ; 1st_src_line
566	movdqa xmm1, [esi+ecx] ; 2nd_src_line	566	movdqa xmm1, [esi+ecx] ; 2nd_src_line
567		567
568	; packing & avg	568	; packing & avg
569	movdqa xmm2, xmm0 ; h H g G f F e E d D c C b B a A	569	movdqa xmm2, xmm0 ; h H g G f F e E d D c C b B a A
570	pshufb xmm0, xmm7 ; 0 H 0 G 0 F 0 E 0 D 0 C 0 B 0 A	570	pshufb xmm0, xmm7 ; 0 H 0 G 0 F 0 E 0 D 0 C 0 B 0 A
571	pshufb xmm2, xmm6 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a	571	pshufb xmm2, xmm6 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
572	; another implementation for xmm2 high bits	572	; another implementation for xmm2 high bits
573	; psubb xmm2, xmm0 ; h 0 g 0 f 0 e 0 d 0 c 0 b 0 a 0	573	; psubb xmm2, xmm0 ; h 0 g 0 f 0 e 0 d 0 c 0 b 0 a 0
574	; psrlw xmm2, 8 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a	574	; psrlw xmm2, 8 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
575	pavgb xmm0, xmm2	575	pavgb xmm0, xmm2
576		576
577	movdqa xmm3, xmm1	577	movdqa xmm3, xmm1
578	pshufb xmm1, xmm7	578	pshufb xmm1, xmm7
579	pshufb xmm3, xmm6	579	pshufb xmm3, xmm6
580	; psubb xmm3, xmm1	580	; psubb xmm3, xmm1
581	; psrlw xmm3, 8	581	; psrlw xmm3, 8
582	pavgb xmm1, xmm3	582	pavgb xmm1, xmm3
583		583
584	pavgb xmm0, xmm1	584	pavgb xmm0, xmm1
585	packuswb xmm0, xmm1	585	packuswb xmm0, xmm1
586		586
587	; write pDst	587	; write pDst
588	movq [edi], xmm0	588	movq [edi], xmm0
589		589
590	; next SMB	590	; next SMB
591	lea esi, [esi+16]	591	lea esi, [esi+16]
592	lea edi, [edi+8]	592	lea edi, [edi+8]
593		593
594	dec eax	594	dec eax
595	jg near .xloops	595	jg near .xloops
596		596
597	; next line	597	; next line
598	lea esi, [esi+2*ecx] ; next end of lines	598	lea esi, [esi+2*ecx] ; next end of lines
599	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]	599	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]
600	lea edi, [edi+edx]	600	lea edi, [edi+edx]
601	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]	601	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]
602		602
603	dec ebp	603	dec ebp
604	jg near .yloops	604	jg near .yloops
605		605
606	pop ebp	606	pop ebp
607	pop edi	607	pop edi
608	pop esi	608	pop esi
609	pop edx	609	pop edx
610	pop ebx	610	pop ebx
611	ret	611	ret
612		612
613	; got about 65% improvement over DyadicBilinearDownsamplerWidthx32_sse	613	; got about 65% improvement over DyadicBilinearDownsamplerWidthx32_sse
614	;***********************************************************************	614	;***********************************************************************
615	; void DyadicBilinearDownsamplerWidthx32_sse4( unsigned char* pDst, const int iDstStride,	615	; void DyadicBilinearDownsamplerWidthx32_sse4( unsigned char* pDst, const int iDstStride,
616	; unsigned char* pSrc, const int iSrcStride,	616	; unsigned char* pSrc, const int iSrcStride,
617	; const int iSrcWidth, const int iSrcHeight );	617	; const int iSrcWidth, const int iSrcHeight );
618	;***********************************************************************	618	;***********************************************************************
619	WELS_EXTERN DyadicBilinearDownsamplerWidthx32_sse4	619	WELS_EXTERN DyadicBilinearDownsamplerWidthx32_sse4
620	push ebx	620	push ebx
621	push edx	621	push edx
622	push esi	622	push esi
623	push edi	623	push edi
624	push ebp	624	push ebp
625		625
626	mov edi, [esp+24] ; pDst	626	mov edi, [esp+24] ; pDst
627	mov edx, [esp+28] ; iDstStride	627	mov edx, [esp+28] ; iDstStride
628	mov esi, [esp+32] ; pSrc	628	mov esi, [esp+32] ; pSrc
629	mov ecx, [esp+36] ; iSrcStride	629	mov ecx, [esp+36] ; iSrcStride
630	mov ebp, [esp+44] ; iSrcHeight	630	mov ebp, [esp+44] ; iSrcHeight
631		631
632	sar ebp, $01 ; iSrcHeight >> 1	632	sar ebp, $01 ; iSrcHeight >> 1
633		633
634	movdqa xmm7, [shufb_mask_low] ; mask low	634	movdqa xmm7, [shufb_mask_low] ; mask low
635	movdqa xmm6, [shufb_mask_high] ; mask high	635	movdqa xmm6, [shufb_mask_high] ; mask high
636		636
637	.yloops:	637	.yloops:
638	mov eax, [esp+40] ; iSrcWidth	638	mov eax, [esp+40] ; iSrcWidth
639	sar eax, $01 ; iSrcWidth >> 1	639	sar eax, $01 ; iSrcWidth >> 1
640	mov ebx, eax ; iDstWidth restored at ebx	640	mov ebx, eax ; iDstWidth restored at ebx
641	sar eax, $04 ; (iSrcWidth >> 1) / 16 ; loop count = num_of_mb	641	sar eax, $04 ; (iSrcWidth >> 1) / 16 ; loop count = num_of_mb
642	neg ebx ; - (iSrcWidth >> 1)	642	neg ebx ; - (iSrcWidth >> 1)
643	; each loop = source bandwidth: 32 bytes	643	; each loop = source bandwidth: 32 bytes
644	.xloops:	644	.xloops:
645	; 1st part horizonal loop: x16 bytes	645	; 1st part horizonal loop: x16 bytes
646	; mem hi<- ->lo	646	; mem hi<- ->lo
647	;1st Line Src: xmm0: h H g G f F e E d D c C b B a A	647	;1st Line Src: xmm0: h H g G f F e E d D c C b B a A
648	; xmm1: p P o O n N m M l L k K j J i I	648	; xmm1: p P o O n N m M l L k K j J i I
649	;2nd Line Src: xmm2: h H g G f F e E d D c C b B a A	649	;2nd Line Src: xmm2: h H g G f F e E d D c C b B a A
650	; xmm3: p P o O n N m M l L k K j J i I	650	; xmm3: p P o O n N m M l L k K j J i I
651	;=> target:	651	;=> target:
652	;: P O N M L K J I H G F E D C B A	652	;: P O N M L K J I H G F E D C B A
653	;: p o n m l k j i h g f e d c b a	653	;: p o n m l k j i h g f e d c b a
654	;: P .. A	654	;: P .. A
655	;: p .. a	655	;: p .. a
656		656
657	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;	657	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
658	movntdqa xmm0, [esi] ; 1st_src_line	658	movntdqa xmm0, [esi] ; 1st_src_line
659	movntdqa xmm1, [esi+16] ; 1st_src_line + 16	659	movntdqa xmm1, [esi+16] ; 1st_src_line + 16
660	movntdqa xmm2, [esi+ecx] ; 2nd_src_line	660	movntdqa xmm2, [esi+ecx] ; 2nd_src_line
661	movntdqa xmm3, [esi+ecx+16] ; 2nd_src_line + 16	661	movntdqa xmm3, [esi+ecx+16] ; 2nd_src_line + 16
662		662
663	; packing & avg	663	; packing & avg
664	movdqa xmm4, xmm0 ; h H g G f F e E d D c C b B a A	664	movdqa xmm4, xmm0 ; h H g G f F e E d D c C b B a A
665	pshufb xmm0, xmm7 ; 0 H 0 G 0 F 0 E 0 D 0 C 0 B 0 A	665	pshufb xmm0, xmm7 ; 0 H 0 G 0 F 0 E 0 D 0 C 0 B 0 A
666	pshufb xmm4, xmm6 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a	666	pshufb xmm4, xmm6 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
667	; psubb xmm4, xmm0 ; h 0 g 0 f 0 e 0 d 0 c 0 b 0 a 0	667	; psubb xmm4, xmm0 ; h 0 g 0 f 0 e 0 d 0 c 0 b 0 a 0
668	; psrlw xmm4, 8 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a	668	; psrlw xmm4, 8 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
669	pavgb xmm0, xmm4	669	pavgb xmm0, xmm4
670		670
671	movdqa xmm5, xmm1	671	movdqa xmm5, xmm1
672	pshufb xmm1, xmm7	672	pshufb xmm1, xmm7
673	pshufb xmm5, xmm6	673	pshufb xmm5, xmm6
674	; psubb xmm5, xmm1	674	; psubb xmm5, xmm1
675	; psrlw xmm5, 8	675	; psrlw xmm5, 8
676	pavgb xmm1, xmm5	676	pavgb xmm1, xmm5
677		677
678	movdqa xmm4, xmm2	678	movdqa xmm4, xmm2
679	pshufb xmm2, xmm7	679	pshufb xmm2, xmm7
680	pshufb xmm4, xmm6	680	pshufb xmm4, xmm6
681	; psubb xmm4, xmm2	681	; psubb xmm4, xmm2
682	; psrlw xmm4, 8	682	; psrlw xmm4, 8
683	pavgb xmm2, xmm4	683	pavgb xmm2, xmm4
684		684
685	movdqa xmm5, xmm3	685	movdqa xmm5, xmm3
686	pshufb xmm3, xmm7	686	pshufb xmm3, xmm7
687	pshufb xmm5, xmm6	687	pshufb xmm5, xmm6
688	; psubb xmm5, xmm3	688	; psubb xmm5, xmm3
689	; psrlw xmm5, 8	689	; psrlw xmm5, 8
690	pavgb xmm3, xmm5	690	pavgb xmm3, xmm5
691		691
692	packuswb xmm0, xmm1	692	packuswb xmm0, xmm1
693	packuswb xmm2, xmm3	693	packuswb xmm2, xmm3
694	pavgb xmm0, xmm2	694	pavgb xmm0, xmm2
695		695
696	; write pDst	696	; write pDst
697	movdqa [edi], xmm0	697	movdqa [edi], xmm0
698		698
699	; next SMB	699	; next SMB
700	lea esi, [esi+32]	700	lea esi, [esi+32]
701	lea edi, [edi+16]	701	lea edi, [edi+16]
702		702
703	dec eax	703	dec eax
704	jg near .xloops	704	jg near .xloops
705		705
706	; next line	706	; next line
707	lea esi, [esi+2*ecx] ; next end of lines	707	lea esi, [esi+2*ecx] ; next end of lines
708	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]	708	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]
709	lea edi, [edi+edx]	709	lea edi, [edi+edx]
710	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]	710	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]
711		711
712	dec ebp	712	dec ebp
713	jg near .yloops	713	jg near .yloops
714		714
715	pop ebp	715	pop ebp
716	pop edi	716	pop edi
717	pop esi	717	pop esi
718	pop edx	718	pop edx
719	pop ebx	719	pop ebx
720	ret	720	ret
721		721
722	;***********************************************************************	722	;***********************************************************************
723	; void DyadicBilinearDownsamplerWidthx16_sse4( unsigned char* pDst, const int iDstStride,	723	; void DyadicBilinearDownsamplerWidthx16_sse4( unsigned char* pDst, const int iDstStride,
724	; unsigned char* pSrc, const int iSrcStride,	724	; unsigned char* pSrc, const int iSrcStride,
725	; const int iSrcWidth, const int iSrcHeight );	725	; const int iSrcWidth, const int iSrcHeight );
726	;***********************************************************************	726	;***********************************************************************
727	WELS_EXTERN DyadicBilinearDownsamplerWidthx16_sse4	727	WELS_EXTERN DyadicBilinearDownsamplerWidthx16_sse4
728	push ebx	728	push ebx
729	push edx	729	push edx
730	push esi	730	push esi
731	push edi	731	push edi
732	push ebp	732	push ebp
733		733
734	mov edi, [esp+24] ; pDst	734	mov edi, [esp+24] ; pDst
735	mov edx, [esp+28] ; iDstStride	735	mov edx, [esp+28] ; iDstStride
736	mov esi, [esp+32] ; pSrc	736	mov esi, [esp+32] ; pSrc
737	mov ecx, [esp+36] ; iSrcStride	737	mov ecx, [esp+36] ; iSrcStride
738	mov ebp, [esp+44] ; iSrcHeight	738	mov ebp, [esp+44] ; iSrcHeight
739		739
740	sar ebp, $01 ; iSrcHeight >> 1	740	sar ebp, $01 ; iSrcHeight >> 1
741	movdqa xmm7, [shufb_mask_low] ; mask low	741	movdqa xmm7, [shufb_mask_low] ; mask low
742	movdqa xmm6, [shufb_mask_high] ; mask high	742	movdqa xmm6, [shufb_mask_high] ; mask high
743		743
744	.yloops:	744	.yloops:
745	mov eax, [esp+40] ; iSrcWidth	745	mov eax, [esp+40] ; iSrcWidth
746	sar eax, $01 ; iSrcWidth >> 1	746	sar eax, $01 ; iSrcWidth >> 1
747	mov ebx, eax ; iDstWidth restored at ebx	747	mov ebx, eax ; iDstWidth restored at ebx
748	sar eax, $03 ; (iSrcWidth >> 1) / 8 ; loop count = num_of_mb	748	sar eax, $03 ; (iSrcWidth >> 1) / 8 ; loop count = num_of_mb
749	neg ebx ; - (iSrcWidth >> 1)	749	neg ebx ; - (iSrcWidth >> 1)
750	; each loop = source bandwidth: 16 bytes	750	; each loop = source bandwidth: 16 bytes
751	.xloops:	751	.xloops:
752	; horizonal loop: x16 bytes by source	752	; horizonal loop: x16 bytes by source
753	; mem hi<- ->lo	753	; mem hi<- ->lo
754	;1st line pSrc: xmm0: h H g G f F e E d D c C b B a A	754	;1st line pSrc: xmm0: h H g G f F e E d D c C b B a A
755	;2nd line pSrc: xmm1: p P o O n N m M l L k K j J i I	755	;2nd line pSrc: xmm1: p P o O n N m M l L k K j J i I
756	;=> target:	756	;=> target:
757	;: H G F E D C B A, P O N M L K J I	757	;: H G F E D C B A, P O N M L K J I
758	;: h g f e d c b a, p o n m l k j i	758	;: h g f e d c b a, p o n m l k j i
759		759
760	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;	760	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
761	movntdqa xmm0, [esi] ; 1st_src_line	761	movntdqa xmm0, [esi] ; 1st_src_line
762	movntdqa xmm1, [esi+ecx] ; 2nd_src_line	762	movntdqa xmm1, [esi+ecx] ; 2nd_src_line
763		763
764	; packing & avg	764	; packing & avg
765	movdqa xmm2, xmm0 ; h H g G f F e E d D c C b B a A	765	movdqa xmm2, xmm0 ; h H g G f F e E d D c C b B a A
766	pshufb xmm0, xmm7 ; 0 H 0 G 0 F 0 E 0 D 0 C 0 B 0 A	766	pshufb xmm0, xmm7 ; 0 H 0 G 0 F 0 E 0 D 0 C 0 B 0 A
767	pshufb xmm2, xmm6 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a	767	pshufb xmm2, xmm6 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
768	; psubb xmm2, xmm0 ; h 0 g 0 f 0 e 0 d 0 c 0 b 0 a 0	768	; psubb xmm2, xmm0 ; h 0 g 0 f 0 e 0 d 0 c 0 b 0 a 0
769	; psrlw xmm2, 8 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a	769	; psrlw xmm2, 8 ; 0 h 0 g 0 f 0 e 0 d 0 c 0 b 0 a
770	pavgb xmm0, xmm2	770	pavgb xmm0, xmm2
771		771
772	movdqa xmm3, xmm1	772	movdqa xmm3, xmm1
773	pshufb xmm1, xmm7	773	pshufb xmm1, xmm7
774	pshufb xmm3, xmm6	774	pshufb xmm3, xmm6
775	; psubb xmm3, xmm1	775	; psubb xmm3, xmm1
776	; psrlw xmm3, 8	776	; psrlw xmm3, 8
777	pavgb xmm1, xmm3	777	pavgb xmm1, xmm3
778		778
779	pavgb xmm0, xmm1	779	pavgb xmm0, xmm1
780	packuswb xmm0, xmm1	780	packuswb xmm0, xmm1
781		781
782	; write pDst	782	; write pDst
783	movq [edi], xmm0	783	movq [edi], xmm0
784		784
785	; next SMB	785	; next SMB
786	lea esi, [esi+16]	786	lea esi, [esi+16]
787	lea edi, [edi+8]	787	lea edi, [edi+8]
788		788
789	dec eax	789	dec eax
790	jg near .xloops	790	jg near .xloops
791		791
792	; next line	792	; next line
793	lea esi, [esi+2*ecx] ; next end of lines	793	lea esi, [esi+2*ecx] ; next end of lines
794	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]	794	lea esi, [esi+2ebx] ; reset to base 0 [- 2 iDstWidth]
795	lea edi, [edi+edx]	795	lea edi, [edi+edx]
796	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]	796	lea edi, [edi+ebx] ; reset to base 0 [- iDstWidth]
797		797
798	dec ebp	798	dec ebp
799	jg near .yloops	799	jg near .yloops
800		800
801	pop ebp	801	pop ebp
802	pop edi	802	pop edi
803	pop esi	803	pop esi
804	pop edx	804	pop edx
805	pop ebx	805	pop ebx
806	ret	806	ret
807		807
808		808
809		809
@@ -811,395 +811,395 @@ WELS_EXTERN DyadicBilinearDownsamplerWidthx16_sse4 · 1 unmodified line
811		811
812	;**************************************************************************************************************	812	;**************************************************************************************************************
813	;int GeneralBilinearAccurateDownsampler_sse2( unsigned char* pDst, const int iDstStride, const int iDstWidth, const int iDstHeight,	813	;int GeneralBilinearAccurateDownsampler_sse2( unsigned char* pDst, const int iDstStride, const int iDstWidth, const int iDstHeight,
814	; unsigned char* pSrc, const int iSrcStride, const int iSrcWidth, const int iSrcHeight,	814	; unsigned char* pSrc, const int iSrcStride, const int iSrcWidth, const int iSrcHeight,
815	; unsigned int uiScaleX, unsigned int uiScaleY );	815	; unsigned int uiScaleX, unsigned int uiScaleY );
816	;{	816	;{
817	;**************************************************************************************************************	817	;**************************************************************************************************************
818		818
819	WELS_EXTERN GeneralBilinearAccurateDownsampler_sse2	819	WELS_EXTERN GeneralBilinearAccurateDownsampler_sse2
820	push ebp	820	push ebp
821	push esi	821	push esi
822	push edi	822	push edi
823	push ebx	823	push ebx
824	%define pushsize 16	824	%define pushsize 16
825	%define localsize 28	825	%define localsize 28
826	%define pDstData esp + pushsize + localsize + 4	826	%define pDstData esp + pushsize + localsize + 4
827	%define dwDstStride esp + pushsize + localsize + 8	827	%define dwDstStride esp + pushsize + localsize + 8
828	%define dwDstWidth esp + pushsize + localsize + 12	828	%define dwDstWidth esp + pushsize + localsize + 12
829	%define dwDstHeight esp + pushsize + localsize + 16	829	%define dwDstHeight esp + pushsize + localsize + 16
830	%define pSrcData esp + pushsize + localsize + 20	830	%define pSrcData esp + pushsize + localsize + 20
831	%define dwSrcStride esp + pushsize + localsize + 24	831	%define dwSrcStride esp + pushsize + localsize + 24
832	%define dwSrcWidth esp + pushsize + localsize + 28	832	%define dwSrcWidth esp + pushsize + localsize + 28
833	%define dwSrcHeight esp + pushsize + localsize + 32	833	%define dwSrcHeight esp + pushsize + localsize + 32
834	%define scale esp + 0	834	%define scale esp + 0
835	%define uiScaleX esp + pushsize + localsize + 36	835	%define uiScaleX esp + pushsize + localsize + 36
836	%define uiScaleY esp + pushsize + localsize + 40	836	%define uiScaleY esp + pushsize + localsize + 40
837	%define tmpHeight esp + 12	837	%define tmpHeight esp + 12
838	%define yInverse esp + 16	838	%define yInverse esp + 16
839	%define xInverse esp + 20	839	%define xInverse esp + 20
840	%define dstStep esp + 24	840	%define dstStep esp + 24
841	sub esp, localsize	841	sub esp, localsize
842		842
843	pxor xmm0, xmm0	843	pxor xmm0, xmm0
844	mov edx, 32767	844	mov edx, 32767
845	mov eax, [uiScaleX]	845	mov eax, [uiScaleX]
846	and eax, 32767	846	and eax, 32767
847	mov ebx, eax	847	mov ebx, eax
848	neg ebx	848	neg ebx
849	and ebx, 32767	849	and ebx, 32767
850	movd xmm1, eax ; uinc(uiScaleX mod 32767)	850	movd xmm1, eax ; uinc(uiScaleX mod 32767)
851	movd xmm2, ebx ; -uinc	851	movd xmm2, ebx ; -uinc
852	psllq xmm1, 32	852	psllq xmm1, 32
853	por xmm1, xmm2 ; 0 0 uinc -uinc (dword)	853	por xmm1, xmm2 ; 0 0 uinc -uinc (dword)
854	pshufd xmm7, xmm1, 01000100b ; xmm7: uinc -uinc uinc -uinc	854	pshufd xmm7, xmm1, 01000100b ; xmm7: uinc -uinc uinc -uinc
855		855
856	mov eax, [uiScaleY]	856	mov eax, [uiScaleY]
857	and eax, 32767	857	and eax, 32767
858	mov ebx, eax	858	mov ebx, eax
859	neg ebx	859	neg ebx
860	and ebx, 32767	860	and ebx, 32767
861	movd xmm6, eax ; vinc(uiScaleY mod 32767)	861	movd xmm6, eax ; vinc(uiScaleY mod 32767)
862	movd xmm2, ebx ; -vinc	862	movd xmm2, ebx ; -vinc
863	psllq xmm6, 32	863	psllq xmm6, 32
864	por xmm6, xmm2 ; 0 0 vinc -vinc (dword)	864	por xmm6, xmm2 ; 0 0 vinc -vinc (dword)
865	pshufd xmm6, xmm6, 01010000b ; xmm6: vinc vinc -vinc -vinc	865	pshufd xmm6, xmm6, 01010000b ; xmm6: vinc vinc -vinc -vinc
866		866
867	mov edx, 40003fffh	867	mov edx, 40003fffh
868	movd xmm5, edx	868	movd xmm5, edx
869	punpcklwd xmm5, xmm0 ; 16384 16383	869	punpcklwd xmm5, xmm0 ; 16384 16383
870	pshufd xmm5, xmm5, 01000100b ; xmm5: 16384 16383 16384 16383	870	pshufd xmm5, xmm5, 01000100b ; xmm5: 16384 16383 16384 16383
871		871
872		872
873	DOWNSAMPLE:	873	DOWNSAMPLE:
874		874
875	mov eax, [dwDstHeight]	875	mov eax, [dwDstHeight]
876	mov edi, [pDstData]	876	mov edi, [pDstData]
877	mov edx, [dwDstStride]	877	mov edx, [dwDstStride]
878	mov ecx, [dwDstWidth]	878	mov ecx, [dwDstWidth]
879	sub edx, ecx	879	sub edx, ecx
880	mov [dstStep], edx ; stride - width	880	mov [dstStep], edx ; stride - width
881	dec eax	881	dec eax
882	mov [tmpHeight], eax	882	mov [tmpHeight], eax
883	mov eax, 16384	883	mov eax, 16384
884	mov [yInverse], eax	884	mov [yInverse], eax
885		885
886	pshufd xmm4, xmm5, 01010000b ; initial v to 16384 16384 16383 16383	886	pshufd xmm4, xmm5, 01010000b ; initial v to 16384 16384 16383 16383
887		887
888	HEIGHT:	888	HEIGHT:
889	mov eax, [yInverse]	889	mov eax, [yInverse]
890	mov esi, [pSrcData]	890	mov esi, [pSrcData]
891	shr eax, 15	891	shr eax, 15
892	mul dword [dwSrcStride]	892	mul dword [dwSrcStride]
893	add esi, eax ; get current row address	893	add esi, eax ; get current row address
894	mov ebp, esi	894	mov ebp, esi
895	add ebp, [dwSrcStride]	895	add ebp, [dwSrcStride]
896		896
897	mov eax, 16384	897	mov eax, 16384
898	mov [xInverse], eax	898	mov [xInverse], eax
899	mov ecx, [dwDstWidth]	899	mov ecx, [dwDstWidth]
900	dec ecx	900	dec ecx
901		901
902	movdqa xmm3, xmm5 ; initial u to 16384 16383 16384 16383	902	movdqa xmm3, xmm5 ; initial u to 16384 16383 16384 16383
903		903
904	WIDTH:	904	WIDTH:
905	mov eax, [xInverse]	905	mov eax, [xInverse]
906	shr eax, 15	906	shr eax, 15
907		907
908	movd xmm1, [esi+eax] ; xxxxxxba	908	movd xmm1, [esi+eax] ; xxxxxxba
909	movd xmm2, [ebp+eax] ; xxxxxxdc	909	movd xmm2, [ebp+eax] ; xxxxxxdc
910	pxor xmm0, xmm0	910	pxor xmm0, xmm0
911	punpcklwd xmm1, xmm2 ; xxxxdcba	911	punpcklwd xmm1, xmm2 ; xxxxdcba
912	punpcklbw xmm1, xmm0 ; 0d0c0b0a	912	punpcklbw xmm1, xmm0 ; 0d0c0b0a
913	punpcklwd xmm1, xmm0 ; 000d000c000b000a	913	punpcklwd xmm1, xmm0 ; 000d000c000b000a
914		914
915	movdqa xmm2, xmm4 ; xmm2: vv(1-v)(1-v) tmpv	915	movdqa xmm2, xmm4 ; xmm2: vv(1-v)(1-v) tmpv
916	pmaddwd xmm2, xmm3 ; mul u(1-u)u(1-u) on xmm2	916	pmaddwd xmm2, xmm3 ; mul u(1-u)u(1-u) on xmm2
917	movdqa xmm0, xmm2	917	movdqa xmm0, xmm2
918	pmuludq xmm2, xmm1	918	pmuludq xmm2, xmm1
919	psrlq xmm0, 32	919	psrlq xmm0, 32
920	psrlq xmm1, 32	920	psrlq xmm1, 32
921	pmuludq xmm0, xmm1	921	pmuludq xmm0, xmm1
922	paddq xmm2, xmm0	922	paddq xmm2, xmm0
923	pshufd xmm1, xmm2, 00001110b	923	pshufd xmm1, xmm2, 00001110b
924	paddq xmm2, xmm1	924	paddq xmm2, xmm1
925	psrlq xmm2, 29	925	psrlq xmm2, 29
926		926
927	movd eax, xmm2	927	movd eax, xmm2
928	inc eax	928	inc eax
929	shr eax, 1	929	shr eax, 1
930	mov [edi], al	930	mov [edi], al
931	inc edi	931	inc edi
932		932
933	mov eax, [uiScaleX]	933	mov eax, [uiScaleX]
934	add [xInverse], eax	934	add [xInverse], eax
935		935
936	paddw xmm3, xmm7 ; inc u	936	paddw xmm3, xmm7 ; inc u
937	psllw xmm3, 1	937	psllw xmm3, 1
938	psrlw xmm3, 1	938	psrlw xmm3, 1
939		939
940	loop WIDTH	940	loop WIDTH
941		941
942	WIDTH_END:	942	WIDTH_END:
943	mov eax, [xInverse]	943	mov eax, [xInverse]
944	shr eax, 15	944	shr eax, 15
945	mov cl, [esi+eax]	945	mov cl, [esi+eax]
946	mov [edi], cl	946	mov [edi], cl
947	inc edi	947	inc edi
948		948
949	mov eax, [uiScaleY]	949	mov eax, [uiScaleY]
950	add [yInverse], eax	950	add [yInverse], eax
951	add edi, [dstStep]	951	add edi, [dstStep]
952		952
953	paddw xmm4, xmm6 ; inc v	953	paddw xmm4, xmm6 ; inc v
954	psllw xmm4, 1	954	psllw xmm4, 1
955	psrlw xmm4, 1	955	psrlw xmm4, 1
956		956
957	dec dword [tmpHeight]	957	dec dword [tmpHeight]
958	jg HEIGHT	958	jg HEIGHT
959		959
960		960
961	LAST_ROW:	961	LAST_ROW:
962	mov eax, [yInverse]	962	mov eax, [yInverse]
963	mov esi, [pSrcData]	963	mov esi, [pSrcData]
964	shr eax, 15	964	shr eax, 15
965	mul dword [dwSrcStride]	965	mul dword [dwSrcStride]
966	add esi, eax ; get current row address	966	add esi, eax ; get current row address
967		967
968	mov eax, 16384	968	mov eax, 16384
969	mov [xInverse], eax	969	mov [xInverse], eax
970	mov ecx, [dwDstWidth]	970	mov ecx, [dwDstWidth]
971		971
972	LAST_ROW_WIDTH:	972	LAST_ROW_WIDTH:
973	mov eax, [xInverse]	973	mov eax, [xInverse]
974	shr eax, 15	974	shr eax, 15
975		975
976	mov al, [esi+eax]	976	mov al, [esi+eax]
977	mov [edi], al	977	mov [edi], al
978	inc edi	978	inc edi
979		979
980	mov eax, [uiScaleX]	980	mov eax, [uiScaleX]
981	add [xInverse], eax	981	add [xInverse], eax
982		982
983	loop LAST_ROW_WIDTH	983	loop LAST_ROW_WIDTH
984		984
985	LAST_ROW_END:	985	LAST_ROW_END:
986		986
987	add esp, localsize	987	add esp, localsize
988	pop ebx	988	pop ebx
989	pop edi	989	pop edi
990	pop esi	990	pop esi
991	pop ebp	991	pop ebp
992	%undef pushsize	992	%undef pushsize
993	%undef localsize	993	%undef localsize
994	%undef pSrcData	994	%undef pSrcData
995	%undef dwSrcWidth	995	%undef dwSrcWidth
996	%undef dwSrcHeight	996	%undef dwSrcHeight
997	%undef dwSrcStride	997	%undef dwSrcStride
998	%undef pDstData	998	%undef pDstData
999	%undef dwDstWidth	999	%undef dwDstWidth
1000	%undef dwDstHeight	1000	%undef dwDstHeight
1001	%undef dwDstStride	1001	%undef dwDstStride
1002	%undef scale	1002	%undef scale
1003	%undef uiScaleX	1003	%undef uiScaleX
1004	%undef uiScaleY	1004	%undef uiScaleY
1005	%undef tmpHeight	1005	%undef tmpHeight
1006	%undef yInverse	1006	%undef yInverse
1007	%undef xInverse	1007	%undef xInverse
1008	%undef dstStep	1008	%undef dstStep
1009	ret	1009	ret
1010		1010
1011		1011
1012		1012
1013		1013
1014	;**************************************************************************************************************	1014	;**************************************************************************************************************
1015	;int GeneralBilinearFastDownsampler_sse2( unsigned char* pDst, const int iDstStride, const int iDstWidth, const int iDstHeight,	1015	;int GeneralBilinearFastDownsampler_sse2( unsigned char* pDst, const int iDstStride, const int iDstWidth, const int iDstHeight,
1016	; unsigned char* pSrc, const int iSrcStride, const int iSrcWidth, const int iSrcHeight,	1016	; unsigned char* pSrc, const int iSrcStride, const int iSrcWidth, const int iSrcHeight,
1017	; unsigned int uiScaleX, unsigned int uiScaleY );	1017	; unsigned int uiScaleX, unsigned int uiScaleY );
1018	;{	1018	;{
1019	;**************************************************************************************************************	1019	;**************************************************************************************************************
1020		1020
1021	WELS_EXTERN GeneralBilinearFastDownsampler_sse2	1021	WELS_EXTERN GeneralBilinearFastDownsampler_sse2
1022	push ebp	1022	push ebp
1023	push esi	1023	push esi
1024	push edi	1024	push edi
1025	push ebx	1025	push ebx
1026	%define pushsize 16	1026	%define pushsize 16
1027	%define localsize 28	1027	%define localsize 28
1028	%define pDstData esp + pushsize + localsize + 4	1028	%define pDstData esp + pushsize + localsize + 4
1029	%define dwDstStride esp + pushsize + localsize + 8	1029	%define dwDstStride esp + pushsize + localsize + 8
1030	%define dwDstWidth esp + pushsize + localsize + 12	1030	%define dwDstWidth esp + pushsize + localsize + 12
1031	%define dwDstHeight esp + pushsize + localsize + 16	1031	%define dwDstHeight esp + pushsize + localsize + 16
1032	%define pSrcData esp + pushsize + localsize + 20	1032	%define pSrcData esp + pushsize + localsize + 20
1033	%define dwSrcStride esp + pushsize + localsize + 24	1033	%define dwSrcStride esp + pushsize + localsize + 24
1034	%define dwSrcWidth esp + pushsize + localsize + 28	1034	%define dwSrcWidth esp + pushsize + localsize + 28
1035	%define dwSrcHeight esp + pushsize + localsize + 32	1035	%define dwSrcHeight esp + pushsize + localsize + 32
1036	%define scale esp + 0	1036	%define scale esp + 0
1037	%define uiScaleX esp + pushsize + localsize + 36	1037	%define uiScaleX esp + pushsize + localsize + 36
1038	%define uiScaleY esp + pushsize + localsize + 40	1038	%define uiScaleY esp + pushsize + localsize + 40
1039	%define tmpHeight esp + 12	1039	%define tmpHeight esp + 12
1040	%define yInverse esp + 16	1040	%define yInverse esp + 16
1041	%define xInverse esp + 20	1041	%define xInverse esp + 20
1042	%define dstStep esp + 24	1042	%define dstStep esp + 24
1043	sub esp, localsize	1043	sub esp, localsize
1044		1044
1045	pxor xmm0, xmm0	1045	pxor xmm0, xmm0
1046	mov edx, 65535	1046	mov edx, 65535
1047	mov eax, [uiScaleX]	1047	mov eax, [uiScaleX]
1048	and eax, edx	1048	and eax, edx
1049	mov ebx, eax	1049	mov ebx, eax
1050	neg ebx	1050	neg ebx
1051	and ebx, 65535	1051	and ebx, 65535
1052	movd xmm1, eax ; uinc(uiScaleX mod 65536)	1052	movd xmm1, eax ; uinc(uiScaleX mod 65536)
1053	movd xmm2, ebx ; -uinc	1053	movd xmm2, ebx ; -uinc
1054	psllq xmm1, 32	1054	psllq xmm1, 32
1055	por xmm1, xmm2 ; 0 uinc 0 -uinc	1055	por xmm1, xmm2 ; 0 uinc 0 -uinc
1056	pshuflw xmm7, xmm1, 10001000b ; xmm7: uinc -uinc uinc -uinc	1056	pshuflw xmm7, xmm1, 10001000b ; xmm7: uinc -uinc uinc -uinc
1057		1057
1058	mov eax, [uiScaleY]	1058	mov eax, [uiScaleY]
1059	and eax, 32767	1059	and eax, 32767
1060	mov ebx, eax	1060	mov ebx, eax
1061	neg ebx	1061	neg ebx
1062	and ebx, 32767	1062	and ebx, 32767
1063	movd xmm6, eax ; vinc(uiScaleY mod 32767)	1063	movd xmm6, eax ; vinc(uiScaleY mod 32767)
1064	movd xmm2, ebx ; -vinc	1064	movd xmm2, ebx ; -vinc
1065	psllq xmm6, 32	1065	psllq xmm6, 32
1066	por xmm6, xmm2 ; 0 vinc 0 -vinc	1066	por xmm6, xmm2 ; 0 vinc 0 -vinc
1067	pshuflw xmm6, xmm6, 10100000b ; xmm6: vinc vinc -vinc -vinc	1067	pshuflw xmm6, xmm6, 10100000b ; xmm6: vinc vinc -vinc -vinc
1068		1068
1069	mov edx, 80007fffh ; 32768 32767	1069	mov edx, 80007fffh ; 32768 32767
1070	movd xmm5, edx	1070	movd xmm5, edx
1071	pshuflw xmm5, xmm5, 01000100b ; 32768 32767 32768 32767	1071	pshuflw xmm5, xmm5, 01000100b ; 32768 32767 32768 32767
1072	mov ebx, 16384	1072	mov ebx, 16384
1073		1073
1074		1074
1075	FAST_DOWNSAMPLE:	1075	FAST_DOWNSAMPLE:
1076		1076
1077	mov eax, [dwDstHeight]	1077	mov eax, [dwDstHeight]
1078	mov edi, [pDstData]	1078	mov edi, [pDstData]
1079	mov edx, [dwDstStride]	1079	mov edx, [dwDstStride]
1080	mov ecx, [dwDstWidth]	1080	mov ecx, [dwDstWidth]
1081	sub edx, ecx	1081	sub edx, ecx
1082	mov [dstStep], edx ; stride - width	1082	mov [dstStep], edx ; stride - width
1083	dec eax	1083	dec eax
1084	mov [tmpHeight], eax	1084	mov [tmpHeight], eax
1085	mov eax, 16384	1085	mov eax, 16384
1086	mov [yInverse], eax	1086	mov [yInverse], eax
1087		1087
1088	pshuflw xmm4, xmm5, 01010000b	1088	pshuflw xmm4, xmm5, 01010000b
1089	psrlw xmm4, 1 ; initial v to 16384 16384 16383 16383	1089	psrlw xmm4, 1 ; initial v to 16384 16384 16383 16383
1090		1090
1091	FAST_HEIGHT:	1091	FAST_HEIGHT:
1092	mov eax, [yInverse]	1092	mov eax, [yInverse]
1093	mov esi, [pSrcData]	1093	mov esi, [pSrcData]
1094	shr eax, 15	1094	shr eax, 15
1095	mul dword [dwSrcStride]	1095	mul dword [dwSrcStride]
1096	add esi, eax ; get current row address	1096	add esi, eax ; get current row address
1097	mov ebp, esi	1097	mov ebp, esi
1098	add ebp, [dwSrcStride]	1098	add ebp, [dwSrcStride]
1099		1099
1100	mov eax, 32768	1100	mov eax, 32768
1101	mov [xInverse], eax	1101	mov [xInverse], eax
1102	mov ecx, [dwDstWidth]	1102	mov ecx, [dwDstWidth]
1103	dec ecx	1103	dec ecx
1104		1104
1105	movdqa xmm3, xmm5 ; initial u to 32768 32767 32768 32767	1105	movdqa xmm3, xmm5 ; initial u to 32768 32767 32768 32767
1106		1106
1107	FAST_WIDTH:	1107	FAST_WIDTH:
1108	mov eax, [xInverse]	1108	mov eax, [xInverse]
1109	shr eax, 16	1109	shr eax, 16
1110		1110
1111	movd xmm1, [esi+eax] ; xxxxxxba	1111	movd xmm1, [esi+eax] ; xxxxxxba
1112	movd xmm2, [ebp+eax] ; xxxxxxdc	1112	movd xmm2, [ebp+eax] ; xxxxxxdc
1113	punpcklwd xmm1, xmm2 ; xxxxdcba	1113	punpcklwd xmm1, xmm2 ; xxxxdcba
1114	punpcklbw xmm1, xmm0 ; 0d0c0b0a	1114	punpcklbw xmm1, xmm0 ; 0d0c0b0a
1115		1115
1116	movdqa xmm2, xmm4 ; xmm2: vv(1-v)(1-v) tmpv	1116	movdqa xmm2, xmm4 ; xmm2: vv(1-v)(1-v) tmpv
1117	pmulhuw xmm2, xmm3 ; mul u(1-u)u(1-u) on xmm2	1117	pmulhuw xmm2, xmm3 ; mul u(1-u)u(1-u) on xmm2
1118	pmaddwd xmm2, xmm1	1118	pmaddwd xmm2, xmm1
1119	pshufd xmm1, xmm2, 00000001b	1119	pshufd xmm1, xmm2, 00000001b
1120	paddd xmm2, xmm1	1120	paddd xmm2, xmm1
1121	movd xmm1, ebx	1121	movd xmm1, ebx
1122	paddd xmm2, xmm1	1122	paddd xmm2, xmm1
1123	psrld xmm2, 15	1123	psrld xmm2, 15
1124		1124
1125	packuswb xmm2, xmm0	1125	packuswb xmm2, xmm0
1126	movd eax, xmm2	1126	movd eax, xmm2
1127	mov [edi], al	1127	mov [edi], al
1128	inc edi	1128	inc edi
1129		1129
1130	mov eax, [uiScaleX]	1130	mov eax, [uiScaleX]
1131	add [xInverse], eax	1131	add [xInverse], eax
1132		1132
1133	paddw xmm3, xmm7 ; inc u	1133	paddw xmm3, xmm7 ; inc u
1134		1134
1135	loop FAST_WIDTH	1135	loop FAST_WIDTH
1136		1136
1137	FAST_WIDTH_END:	1137	FAST_WIDTH_END:
1138	mov eax, [xInverse]	1138	mov eax, [xInverse]
1139	shr eax, 16	1139	shr eax, 16
1140	mov cl, [esi+eax]	1140	mov cl, [esi+eax]
1141	mov [edi], cl	1141	mov [edi], cl
1142	inc edi	1142	inc edi
1143		1143
1144	mov eax, [uiScaleY]	1144	mov eax, [uiScaleY]
1145	add [yInverse], eax	1145	add [yInverse], eax
1146	add edi, [dstStep]	1146	add edi, [dstStep]
1147		1147
1148	paddw xmm4, xmm6 ; inc v	1148	paddw xmm4, xmm6 ; inc v
1149	psllw xmm4, 1	1149	psllw xmm4, 1
1150	psrlw xmm4, 1	1150	psrlw xmm4, 1
1151		1151
1152	dec dword [tmpHeight]	1152	dec dword [tmpHeight]
1153	jg FAST_HEIGHT	1153	jg FAST_HEIGHT
1154		1154
1155		1155
1156	FAST_LAST_ROW:	1156	FAST_LAST_ROW:
1157	mov eax, [yInverse]	1157	mov eax, [yInverse]
1158	mov esi, [pSrcData]	1158	mov esi, [pSrcData]
1159	shr eax, 15	1159	shr eax, 15
1160	mul dword [dwSrcStride]	1160	mul dword [dwSrcStride]
1161	add esi, eax ; get current row address	1161	add esi, eax ; get current row address
1162		1162
1163	mov eax, 32768	1163	mov eax, 32768
1164	mov [xInverse], eax	1164	mov [xInverse], eax
1165	mov ecx, [dwDstWidth]	1165	mov ecx, [dwDstWidth]
1166		1166
1167	FAST_LAST_ROW_WIDTH:	1167	FAST_LAST_ROW_WIDTH:
1168	mov eax, [xInverse]	1168	mov eax, [xInverse]
1169	shr eax, 16	1169	shr eax, 16
1170		1170
1171	mov al, [esi+eax]	1171	mov al, [esi+eax]
1172	mov [edi], al	1172	mov [edi], al
1173	inc edi	1173	inc edi
1174		1174
1175	mov eax, [uiScaleX]	1175	mov eax, [uiScaleX]
1176	add [xInverse], eax	1176	add [xInverse], eax
1177		1177
1178	loop FAST_LAST_ROW_WIDTH	1178	loop FAST_LAST_ROW_WIDTH
1179		1179
1180	FAST_LAST_ROW_END:	1180	FAST_LAST_ROW_END:
1181		1181
1182	add esp, localsize	1182	add esp, localsize
1183	pop ebx	1183	pop ebx
1184	pop edi	1184	pop edi
1185	pop esi	1185	pop esi
1186	pop ebp	1186	pop ebp
1187	%undef pushsize	1187	%undef pushsize
1188	%undef localsize	1188	%undef localsize
1189	%undef pSrcData	1189	%undef pSrcData
1190	%undef dwSrcWidth	1190	%undef dwSrcWidth
1191	%undef dwSrcHeight	1191	%undef dwSrcHeight
1192	%undef dwSrcStride	1192	%undef dwSrcStride
1193	%undef pDstData	1193	%undef pDstData
1194	%undef dwDstWidth	1194	%undef dwDstWidth
1195	%undef dwDstHeight	1195	%undef dwDstHeight
1196	%undef dwDstStride	1196	%undef dwDstStride
1197	%undef scale	1197	%undef scale
1198	%undef uiScaleX	1198	%undef uiScaleX
1199	%undef uiScaleY	1199	%undef uiScaleY
1200	%undef tmpHeight	1200	%undef tmpHeight
1201	%undef yInverse	1201	%undef yInverse
1202	%undef xInverse	1202	%undef xInverse
1203	%undef dstStep	1203	%undef dstStep
1204	ret	1204	ret
1205	%endif	1205	%endif

codec/processing/src/x86/vaa.asm+1518−1518

@@ -48,100 +48,100 @@ · 47 unmodified lines
48	; Macros and other preprocessor constants	48	; Macros and other preprocessor constants
49	;***********************************************************************	49	;***********************************************************************
50	%macro SUM_SQR_SSE2 3 ; dst, pSrc, zero	50	%macro SUM_SQR_SSE2 3 ; dst, pSrc, zero
51	movdqa %1, %2	51	movdqa %1, %2
52	punpcklbw %1, %3	52	punpcklbw %1, %3
53	punpckhbw %2, %3	53	punpckhbw %2, %3
54	pmaddwd %1, %1	54	pmaddwd %1, %1
55	pmaddwd %2, %2	55	pmaddwd %2, %2
56	paddd %1, %2	56	paddd %1, %2
57	pshufd %2, %1, 04Eh ; 01001110 B	57	pshufd %2, %1, 04Eh ; 01001110 B
58	paddd %1, %2	58	paddd %1, %2
59	pshufd %2, %1, 0B1h ; 10110001 B	59	pshufd %2, %1, 0B1h ; 10110001 B
60	paddd %1, %2	60	paddd %1, %2
61	%endmacro ; END OF SUM_SQR_SSE2	61	%endmacro ; END OF SUM_SQR_SSE2
62		62
63	%macro WELS_SAD_16x2_SSE2 3 ;esi :%1 edi:%2 ebx:%3	63	%macro WELS_SAD_16x2_SSE2 3 ;esi :%1 edi:%2 ebx:%3
64	movdqa xmm1, [%1]	64	movdqa xmm1, [%1]
65	movdqa xmm2, [%2]	65	movdqa xmm2, [%2]
66	movdqa xmm3, [%1+%3]	66	movdqa xmm3, [%1+%3]
67	movdqa xmm4, [%2+%3]	67	movdqa xmm4, [%2+%3]
68	psadbw xmm1, xmm2	68	psadbw xmm1, xmm2
69	psadbw xmm3, xmm4	69	psadbw xmm3, xmm4
70	paddd xmm6, xmm1	70	paddd xmm6, xmm1
71	paddd xmm6, xmm3	71	paddd xmm6, xmm3
72	lea %1, [%1+%3*2]	72	lea %1, [%1+%3*2]
73	lea %2, [%2+%3*2]	73	lea %2, [%2+%3*2]
74	%endmacro	74	%endmacro
75		75
76	; by comparing it outperforms than phaddw(SSSE3) sets	76	; by comparing it outperforms than phaddw(SSSE3) sets
77	%macro SUM_WORD_8x2_SSE2 2 ; dst(pSrc), tmp	77	%macro SUM_WORD_8x2_SSE2 2 ; dst(pSrc), tmp
78	; @sum_8x2 begin	78	; @sum_8x2 begin
79	pshufd %2, %1, 04Eh ; 01001110 B	79	pshufd %2, %1, 04Eh ; 01001110 B
80	paddw %1, %2	80	paddw %1, %2
81	pshuflw %2, %1, 04Eh ; 01001110 B	81	pshuflw %2, %1, 04Eh ; 01001110 B
82	paddw %1, %2	82	paddw %1, %2
83	pshuflw %2, %1, 0B1h ; 10110001 B	83	pshuflw %2, %1, 0B1h ; 10110001 B
84	paddw %1, %2	84	paddw %1, %2
85	; end of @sum_8x2	85	; end of @sum_8x2
86	%endmacro ; END of SUM_WORD_8x2_SSE2	86	%endmacro ; END of SUM_WORD_8x2_SSE2
87		87
88	%macro WELS_SAD_SUM_SQSUM_16x1_SSE2 3 ;esi:%1,edi:%2,ebx:%3	88	%macro WELS_SAD_SUM_SQSUM_16x1_SSE2 3 ;esi:%1,edi:%2,ebx:%3
89	movdqa xmm1, [%1]	89	movdqa xmm1, [%1]
90	movdqa xmm2, [%2]	90	movdqa xmm2, [%2]
91	movdqa xmm3, xmm1	91	movdqa xmm3, xmm1
92	psadbw xmm3, xmm2	92	psadbw xmm3, xmm2
93	paddd xmm6, xmm3	93	paddd xmm6, xmm3
94		94
95	movdqa xmm3, xmm1	95	movdqa xmm3, xmm1
96	psadbw xmm3, xmm0	96	psadbw xmm3, xmm0
97	paddd xmm5, xmm3	97	paddd xmm5, xmm3
98		98
99	movdqa xmm2, xmm1	99	movdqa xmm2, xmm1
100	punpcklbw xmm1, xmm0	100	punpcklbw xmm1, xmm0
101	punpckhbw xmm2, xmm0	101	punpckhbw xmm2, xmm0
102	pmaddwd xmm1, xmm1	102	pmaddwd xmm1, xmm1
103	pmaddwd xmm2, xmm2	103	pmaddwd xmm2, xmm2
104	paddd xmm4, xmm1	104	paddd xmm4, xmm1
105	paddd xmm4, xmm2	105	paddd xmm4, xmm2
106		106
107	add %1, %3	107	add %1, %3
108	add %2, %3	108	add %2, %3
109	%endmacro	109	%endmacro
110		110
111	%macro WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 3 ;esi:%1 edi:%2 ebx:%3	111	%macro WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 3 ;esi:%1 edi:%2 ebx:%3
112	movdqa xmm1, [%1]	112	movdqa xmm1, [%1]
113	movdqa xmm2, [%2]	113	movdqa xmm2, [%2]
114	movdqa xmm3, xmm1	114	movdqa xmm3, xmm1
115	psadbw xmm3, xmm2	115	psadbw xmm3, xmm2
116	paddd xmm7, xmm3 ; sad	116	paddd xmm7, xmm3 ; sad
117		117
118	movdqa xmm3, xmm1	118	movdqa xmm3, xmm1
119	pmaxub xmm3, xmm2	119	pmaxub xmm3, xmm2
120	pminub xmm2, xmm1	120	pminub xmm2, xmm1
121	psubb xmm3, xmm2 ; diff	121	psubb xmm3, xmm2 ; diff
122		122
123	movdqa xmm2, xmm1	123	movdqa xmm2, xmm1
124	psadbw xmm2, xmm0	124	psadbw xmm2, xmm0
125	paddd xmm6, xmm2 ; sum	125	paddd xmm6, xmm2 ; sum
126		126
127	movdqa xmm2, xmm1	127	movdqa xmm2, xmm1
128	punpcklbw xmm1, xmm0	128	punpcklbw xmm1, xmm0
129	punpckhbw xmm2, xmm0	129	punpckhbw xmm2, xmm0
130	pmaddwd xmm1, xmm1	130	pmaddwd xmm1, xmm1
131	pmaddwd xmm2, xmm2	131	pmaddwd xmm2, xmm2
132	paddd xmm5, xmm1	132	paddd xmm5, xmm1
133	paddd xmm5, xmm2 ; sqsum	133	paddd xmm5, xmm2 ; sqsum
134		134
135	movdqa xmm1, xmm3	135	movdqa xmm1, xmm3
136	punpcklbw xmm1, xmm0	136	punpcklbw xmm1, xmm0
137	punpckhbw xmm3, xmm0	137	punpckhbw xmm3, xmm0
138	pmaddwd xmm1, xmm1	138	pmaddwd xmm1, xmm1
139	pmaddwd xmm3, xmm3	139	pmaddwd xmm3, xmm3
140	paddd xmm4, xmm1	140	paddd xmm4, xmm1
141	paddd xmm4, xmm3 ; sqdiff	141	paddd xmm4, xmm3 ; sqdiff
142		142
143	add %1, %3	143	add %1, %3
144	add %2, %3	144	add %2, %3
145	%endmacro	145	%endmacro
146		146
147	%macro WELS_SAD_SD_MAD_16x1_SSE2 7 ;esi:%5 edi:%6 ebx:%7	147	%macro WELS_SAD_SD_MAD_16x1_SSE2 7 ;esi:%5 edi:%6 ebx:%7
@@ -149,40 +149,40 @@ · 1 unmodified line
149	%define sum_cur_reg %2	149	%define sum_cur_reg %2
150	%define sum_ref_reg %3	150	%define sum_ref_reg %3
151	%define mad_reg %4	151	%define mad_reg %4
152	movdqa xmm1, [%5]	152	movdqa xmm1, [%5]
153	movdqa xmm2, [%6]	153	movdqa xmm2, [%6]
154	movdqa xmm3, xmm1	154	movdqa xmm3, xmm1
155	psadbw xmm3, xmm0	155	psadbw xmm3, xmm0
156	paddd sum_cur_reg, xmm3 ; sum_cur	156	paddd sum_cur_reg, xmm3 ; sum_cur
157	movdqa xmm3, xmm2	157	movdqa xmm3, xmm2
158	psadbw xmm3, xmm0	158	psadbw xmm3, xmm0
159	paddd sum_ref_reg, xmm3 ; sum_ref	159	paddd sum_ref_reg, xmm3 ; sum_ref
160		160
161	movdqa xmm3, xmm1	161	movdqa xmm3, xmm1
162	pmaxub xmm3, xmm2	162	pmaxub xmm3, xmm2
163	pminub xmm2, xmm1	163	pminub xmm2, xmm1
164	psubb xmm3, xmm2 ; abs diff	164	psubb xmm3, xmm2 ; abs diff
165	pmaxub mad_reg, xmm3 ; max abs diff	165	pmaxub mad_reg, xmm3 ; max abs diff
166		166
167	psadbw xmm3, xmm0	167	psadbw xmm3, xmm0
168	paddd sad_reg, xmm3 ; sad	168	paddd sad_reg, xmm3 ; sad
169		169
170	add %5, %7	170	add %5, %7
171	add %6, %7	171	add %6, %7
172	%endmacro	172	%endmacro
173		173
174		174
175	%macro WELS_MAX_REG_SSE2 1 ; xmm1, xmm2, xmm3 can be used	175	%macro WELS_MAX_REG_SSE2 1 ; xmm1, xmm2, xmm3 can be used
176	%define max_reg %1	176	%define max_reg %1
177	movdqa xmm1, max_reg	177	movdqa xmm1, max_reg
178	psrldq xmm1, 4	178	psrldq xmm1, 4
179	pmaxub max_reg, xmm1	179	pmaxub max_reg, xmm1
180	movdqa xmm1, max_reg	180	movdqa xmm1, max_reg
181	psrldq xmm1, 2	181	psrldq xmm1, 2
182	pmaxub max_reg, xmm1	182	pmaxub max_reg, xmm1
183	movdqa xmm1, max_reg	183	movdqa xmm1, max_reg
184	psrldq xmm1, 1	184	psrldq xmm1, 1
185	pmaxub max_reg, xmm1	185	pmaxub max_reg, xmm1
186	%endmacro	186	%endmacro
187		187
188	%macro WELS_SAD_BGD_SQDIFF_16x1_SSE2 7 ;esi:%5 edi:%6 ebx:%7	188	%macro WELS_SAD_BGD_SQDIFF_16x1_SSE2 7 ;esi:%5 edi:%6 ebx:%7
@@ -190,50 +190,50 @@ · 1 unmodified line
190	%define sum_reg %2	190	%define sum_reg %2
191	%define mad_reg %3	191	%define mad_reg %3
192	%define sqdiff_reg %4	192	%define sqdiff_reg %4
193	movdqa xmm1, [%5]	193	movdqa xmm1, [%5]
194	movdqa xmm2, xmm1	194	movdqa xmm2, xmm1
195	movdqa xmm3, xmm1	195	movdqa xmm3, xmm1
196	punpcklbw xmm2, xmm0	196	punpcklbw xmm2, xmm0
197	punpckhbw xmm3, xmm0	197	punpckhbw xmm3, xmm0
198	pmaddwd xmm2, xmm2	198	pmaddwd xmm2, xmm2
199	pmaddwd xmm3, xmm3	199	pmaddwd xmm3, xmm3
200	paddd xmm2, xmm3	200	paddd xmm2, xmm3
201	movdqa xmm3, xmm2	201	movdqa xmm3, xmm2
202	psllq xmm2, 32	202	psllq xmm2, 32
203	psrlq xmm3, 32	203	psrlq xmm3, 32
204	psllq xmm3, 32	204	psllq xmm3, 32
205	paddd xmm2, xmm3	205	paddd xmm2, xmm3
206	paddd sad_reg, xmm2 ; sqsum	206	paddd sad_reg, xmm2 ; sqsum
207		207
208	movdqa xmm2, [%6]	208	movdqa xmm2, [%6]
209	movdqa xmm3, xmm1	209	movdqa xmm3, xmm1
210	psadbw xmm3, xmm0	210	psadbw xmm3, xmm0
211	paddd sum_reg, xmm3 ; sum_cur	211	paddd sum_reg, xmm3 ; sum_cur
212	movdqa xmm3, xmm2	212	movdqa xmm3, xmm2
213	psadbw xmm3, xmm0	213	psadbw xmm3, xmm0
214	pslldq xmm3, 4	214	pslldq xmm3, 4
215	paddd sum_reg, xmm3 ; sum_ref	215	paddd sum_reg, xmm3 ; sum_ref
216		216
217	movdqa xmm3, xmm1	217	movdqa xmm3, xmm1
218	pmaxub xmm3, xmm2	218	pmaxub xmm3, xmm2
219	pminub xmm2, xmm1	219	pminub xmm2, xmm1
220	psubb xmm3, xmm2 ; abs diff	220	psubb xmm3, xmm2 ; abs diff
221	pmaxub mad_reg, xmm3 ; max abs diff	221	pmaxub mad_reg, xmm3 ; max abs diff
222		222
223	movdqa xmm1, xmm3	223	movdqa xmm1, xmm3
224	psadbw xmm3, xmm0	224	psadbw xmm3, xmm0
225	paddd sad_reg, xmm3 ; sad	225	paddd sad_reg, xmm3 ; sad
226		226
227	movdqa xmm3, xmm1	227	movdqa xmm3, xmm1
228	punpcklbw xmm1, xmm0	228	punpcklbw xmm1, xmm0
229	punpckhbw xmm3, xmm0	229	punpckhbw xmm3, xmm0
230	pmaddwd xmm1, xmm1	230	pmaddwd xmm1, xmm1
231	pmaddwd xmm3, xmm3	231	pmaddwd xmm3, xmm3
232	paddd sqdiff_reg, xmm1	232	paddd sqdiff_reg, xmm1
233	paddd sqdiff_reg, xmm3 ; sqdiff	233	paddd sqdiff_reg, xmm3 ; sqdiff
234		234
235	add %5, %7	235	add %5, %7
236	add %6, %7	236	add %6, %7
237	%endmacro	237	%endmacro
238		238
239		239
@@ -249,99 +249,99 @@ SECTION .text · 9 unmodified lines
249	; void SampleVariance16x16_sse2( uint8_t * y_ref, int32_t y_ref_stride, uint8_t * y_src, int32_t y_src_stride,SMotionTextureUnit* pMotionTexture );	249	; void SampleVariance16x16_sse2( uint8_t * y_ref, int32_t y_ref_stride, uint8_t * y_src, int32_t y_src_stride,SMotionTextureUnit* pMotionTexture );
250	;***********************************************************************	250	;***********************************************************************
251	WELS_EXTERN SampleVariance16x16_sse2	251	WELS_EXTERN SampleVariance16x16_sse2
252	push esi	252	push esi
253	push edi	253	push edi
254	push ebx	254	push ebx
255		255
256	sub esp, 16	256	sub esp, 16
257	%define SUM [esp]	257	%define SUM [esp]
258	%define SUM_CUR [esp+4]	258	%define SUM_CUR [esp+4]
259	%define SQR [esp+8]	259	%define SQR [esp+8]
260	%define SQR_CUR [esp+12]	260	%define SQR_CUR [esp+12]
261	%define PUSH_SIZE 28 ; 12 + 16	261	%define PUSH_SIZE 28 ; 12 + 16
262		262
263	mov edi, [esp+PUSH_SIZE+4] ; y_ref	263	mov edi, [esp+PUSH_SIZE+4] ; y_ref
264	mov edx, [esp+PUSH_SIZE+8] ; y_ref_stride	264	mov edx, [esp+PUSH_SIZE+8] ; y_ref_stride
265	mov esi, [esp+PUSH_SIZE+12] ; y_src	265	mov esi, [esp+PUSH_SIZE+12] ; y_src
266	mov eax, [esp+PUSH_SIZE+16] ; y_src_stride	266	mov eax, [esp+PUSH_SIZE+16] ; y_src_stride
267	mov ecx, 010h ; height = 16	267	mov ecx, 010h ; height = 16
268		268
269	pxor xmm7, xmm7	269	pxor xmm7, xmm7
270	movdqu SUM, xmm7	270	movdqu SUM, xmm7
271		271
272	.hloops:	272	.hloops:
273	movdqa xmm0, [edi] ; y_ref	273	movdqa xmm0, [edi] ; y_ref
274	movdqa xmm1, [esi] ; y_src	274	movdqa xmm1, [esi] ; y_src
275	movdqa xmm2, xmm0 ; store first for future process	275	movdqa xmm2, xmm0 ; store first for future process
276	movdqa xmm3, xmm1	276	movdqa xmm3, xmm1
277	; sum += diff;	277	; sum += diff;
278	movdqa xmm4, xmm0	278	movdqa xmm4, xmm0
279	psadbw xmm4, xmm1 ; 2 parts, [0,..,15], [64,..,79]	279	psadbw xmm4, xmm1 ; 2 parts, [0,..,15], [64,..,79]
280	; to be continued for sum	280	; to be continued for sum
281	pshufd xmm5, xmm4, 0C6h ; 11000110 B	281	pshufd xmm5, xmm4, 0C6h ; 11000110 B
282	paddw xmm4, xmm5	282	paddw xmm4, xmm5
283	movd ebx, xmm4	283	movd ebx, xmm4
284	add SUM, ebx	284	add SUM, ebx
285		285
286	; sqr += diff * diff;	286	; sqr += diff * diff;
287	pmaxub xmm0, xmm1	287	pmaxub xmm0, xmm1
288	pminub xmm1, xmm2	288	pminub xmm1, xmm2
289	psubb xmm0, xmm1 ; diff	289	psubb xmm0, xmm1 ; diff
290	SUM_SQR_SSE2 xmm1, xmm0, xmm7 ; dst, pSrc, zero	290	SUM_SQR_SSE2 xmm1, xmm0, xmm7 ; dst, pSrc, zero
291	movd ebx, xmm1	291	movd ebx, xmm1
292	add SQR, ebx	292	add SQR, ebx
293		293
294	; sum_cur += y_src[x];	294	; sum_cur += y_src[x];
295	movdqa xmm0, xmm3 ; cur_orig	295	movdqa xmm0, xmm3 ; cur_orig
296	movdqa xmm1, xmm0	296	movdqa xmm1, xmm0
297	punpcklbw xmm0, xmm7	297	punpcklbw xmm0, xmm7
298	punpckhbw xmm1, xmm7	298	punpckhbw xmm1, xmm7
299	paddw xmm0, xmm1 ; 8x2	299	paddw xmm0, xmm1 ; 8x2
300	SUM_WORD_8x2_SSE2 xmm0, xmm1	300	SUM_WORD_8x2_SSE2 xmm0, xmm1
301	movd ebx, xmm0	301	movd ebx, xmm0
302	and ebx, 0ffffh	302	and ebx, 0ffffh
303	add SUM_CUR, ebx	303	add SUM_CUR, ebx
304		304
305	; sqr_cur += y_src[x] * y_src[x];	305	; sqr_cur += y_src[x] * y_src[x];
306	SUM_SQR_SSE2 xmm0, xmm3, xmm7 ; dst, pSrc, zero	306	SUM_SQR_SSE2 xmm0, xmm3, xmm7 ; dst, pSrc, zero
307	movd ebx, xmm0	307	movd ebx, xmm0
308	add SQR_CUR, ebx	308	add SQR_CUR, ebx
309		309
310	lea edi, [edi+edx]	310	lea edi, [edi+edx]
311	lea esi, [esi+eax]	311	lea esi, [esi+eax]
312	dec ecx	312	dec ecx
313	jnz near .hloops	313	jnz near .hloops
314		314
315	mov ebx, 0	315	mov ebx, 0
316	mov bx, word SUM	316	mov bx, word SUM
317	sar ebx, 8	317	sar ebx, 8
318	imul ebx, ebx	318	imul ebx, ebx
319	mov ecx, SQR	319	mov ecx, SQR
320	sar ecx, 8	320	sar ecx, 8
321	sub ecx, ebx	321	sub ecx, ebx
322	mov edi, [esp+PUSH_SIZE+20] ; pMotionTexture	322	mov edi, [esp+PUSH_SIZE+20] ; pMotionTexture
323	mov [edi], cx ; to store uiMotionIndex	323	mov [edi], cx ; to store uiMotionIndex
324	mov ebx, 0	324	mov ebx, 0
325	mov bx, word SUM_CUR	325	mov bx, word SUM_CUR
326	sar ebx, 8	326	sar ebx, 8
327	imul ebx, ebx	327	imul ebx, ebx
328	mov ecx, SQR_CUR	328	mov ecx, SQR_CUR
329	sar ecx, 8	329	sar ecx, 8
330	sub ecx, ebx	330	sub ecx, ebx
331	mov [edi+2], cx ; to store uiTextureIndex	331	mov [edi+2], cx ; to store uiTextureIndex
332		332
333	%undef SUM	333	%undef SUM
334	%undef SUM_CUR	334	%undef SUM_CUR
335	%undef SQR	335	%undef SQR
336	%undef SQR_CUR	336	%undef SQR_CUR
337	%undef PUSH_SIZE	337	%undef PUSH_SIZE
338		338
339	add esp, 16	339	add esp, 16
340	pop ebx	340	pop ebx
341	pop edi	341	pop edi
342	pop esi	342	pop esi
343		343
344	ret	344	ret
345		345
346		346
347		347
@@ -360,67 +360,67 @@ WELS_EXTERN VAACalcSad_sse2 · 12 unmodified lines
360	%define psadframe esp + pushsize + 24	360	%define psadframe esp + pushsize + 24
361	%define psad8x8 esp + pushsize + 28	361	%define psad8x8 esp + pushsize + 28
362	%define pushsize 12	362	%define pushsize 12
363	push esi	363	push esi
364	push edi	364	push edi
365	push ebx	365	push ebx
366	mov esi, [cur_data]	366	mov esi, [cur_data]
367	mov edi, [ref_data]	367	mov edi, [ref_data]
368	mov ebx, [iPicStride]	368	mov ebx, [iPicStride]
369	mov edx, [psad8x8]	369	mov edx, [psad8x8]
370	mov eax, ebx	370	mov eax, ebx
371		371
372	shr dword [iPicWidth], 4 ; iPicWidth/16	372	shr dword [iPicWidth], 4 ; iPicWidth/16
373	shr dword [iPicHeight], 4 ; iPicHeight/16	373	shr dword [iPicHeight], 4 ; iPicHeight/16
374	shl eax, 4 ; iPicStride*16	374	shl eax, 4 ; iPicStride*16
375	pxor xmm0, xmm0	375	pxor xmm0, xmm0
376	pxor xmm7, xmm7 ; iFrameSad	376	pxor xmm7, xmm7 ; iFrameSad
377	height_loop:	377	height_loop:
378	mov ecx, dword [iPicWidth]	378	mov ecx, dword [iPicWidth]
379	push esi	379	push esi
380	push edi	380	push edi
381	width_loop:	381	width_loop:
382	pxor xmm6, xmm6 ;	382	pxor xmm6, xmm6 ;
383	WELS_SAD_16x2_SSE2 esi,edi,ebx	383	WELS_SAD_16x2_SSE2 esi,edi,ebx
384	WELS_SAD_16x2_SSE2 esi,edi,ebx	384	WELS_SAD_16x2_SSE2 esi,edi,ebx
385	WELS_SAD_16x2_SSE2 esi,edi,ebx	385	WELS_SAD_16x2_SSE2 esi,edi,ebx
386	WELS_SAD_16x2_SSE2 esi,edi,ebx	386	WELS_SAD_16x2_SSE2 esi,edi,ebx
387	paddd xmm7, xmm6	387	paddd xmm7, xmm6
388	movd [edx], xmm6	388	movd [edx], xmm6
389	psrldq xmm6, 8	389	psrldq xmm6, 8
390	movd [edx+4], xmm6	390	movd [edx+4], xmm6
391		391
392	pxor xmm6, xmm6	392	pxor xmm6, xmm6
393	WELS_SAD_16x2_SSE2 esi,edi,ebx	393	WELS_SAD_16x2_SSE2 esi,edi,ebx
394	WELS_SAD_16x2_SSE2 esi,edi,ebx	394	WELS_SAD_16x2_SSE2 esi,edi,ebx
395	WELS_SAD_16x2_SSE2 esi,edi,ebx	395	WELS_SAD_16x2_SSE2 esi,edi,ebx
396	WELS_SAD_16x2_SSE2 esi,edi,ebx	396	WELS_SAD_16x2_SSE2 esi,edi,ebx
397	paddd xmm7, xmm6	397	paddd xmm7, xmm6
398	movd [edx+8], xmm6	398	movd [edx+8], xmm6
399	psrldq xmm6, 8	399	psrldq xmm6, 8
400	movd [edx+12], xmm6	400	movd [edx+12], xmm6
401		401
402	add edx, 16	402	add edx, 16
403	sub esi, eax	403	sub esi, eax
404	sub edi, eax	404	sub edi, eax
405	add esi, 16	405	add esi, 16
406	add edi, 16	406	add edi, 16
407		407
408	dec ecx	408	dec ecx
409	jnz width_loop	409	jnz width_loop
410		410
411	pop edi	411	pop edi
412	pop esi	412	pop esi
413	add esi, eax	413	add esi, eax
414	add edi, eax	414	add edi, eax
415		415
416	dec dword [iPicHeight]	416	dec dword [iPicHeight]
417	jnz height_loop	417	jnz height_loop
418		418
419	mov edx, [psadframe]	419	mov edx, [psadframe]
420	movdqa xmm5, xmm7	420	movdqa xmm5, xmm7
421	psrldq xmm7, 8	421	psrldq xmm7, 8
422	paddd xmm7, xmm5	422	paddd xmm7, xmm5
423	movd [edx], xmm7	423	movd [edx], xmm7
424		424
425	%undef cur_data	425	%undef cur_data
426	%undef ref_data	426	%undef ref_data
@@ -430,10 +430,10 @@ width_loop: · 3 unmodified lines
430	%undef psadframe	430	%undef psadframe
431	%undef psad8x8	431	%undef psad8x8
432	%undef pushsize	432	%undef pushsize
433	pop ebx	433	pop ebx
434	pop edi	434	pop edi
435	pop esi	435	pop esi
436	ret	436	ret
437		437
438	%else ;64-bit	438	%else ;64-bit
439		439
@@ -441,98 +441,98 @@ width_loop: · 1 unmodified line
441	; void SampleVariance16x16_sse2( uint8_t * y_ref, int32_t y_ref_stride, uint8_t * y_src, int32_t y_src_stride,SMotionTextureUnit* pMotionTexture );	441	; void SampleVariance16x16_sse2( uint8_t * y_ref, int32_t y_ref_stride, uint8_t * y_src, int32_t y_src_stride,SMotionTextureUnit* pMotionTexture );
442	;***********************************************************************	442	;***********************************************************************
443	WELS_EXTERN SampleVariance16x16_sse2	443	WELS_EXTERN SampleVariance16x16_sse2
444	%define SUM r10;[esp]	444	%define SUM r10;[esp]
445	%define SUM_CUR r11;[esp+4]	445	%define SUM_CUR r11;[esp+4]
446	%define SQR r13;[esp+8]	446	%define SQR r13;[esp+8]
447	%define SQR_CUR r15;[esp+12]	447	%define SQR_CUR r15;[esp+12]
448		448
449	push r12	449	push r12
450	push r13	450	push r13
451	push r14	451	push r14
452	push r15	452	push r15
453	%assign push_num 4	453	%assign push_num 4
454	LOAD_5_PARA	454	LOAD_5_PARA
455	PUSH_XMM 8	455	PUSH_XMM 8
456	SIGN_EXTENSION r1,r1d	456	SIGN_EXTENSION r1,r1d
457	SIGN_EXTENSION r3,r3d	457	SIGN_EXTENSION r3,r3d
458		458
459	mov r12,010h	459	mov r12,010h
460	pxor xmm7, xmm7	460	pxor xmm7, xmm7
461	movq SUM, xmm7	461	movq SUM, xmm7
462	movq SUM_CUR,xmm7	462	movq SUM_CUR,xmm7
463	movq SQR,xmm7	463	movq SQR,xmm7
464	movq SQR_CUR,xmm7	464	movq SQR_CUR,xmm7
465		465
466	.hloops:	466	.hloops:
467	mov r14,0	467	mov r14,0
468	movdqa xmm0, [r0] ; y_ref	468	movdqa xmm0, [r0] ; y_ref
469	movdqa xmm1, [r2] ; y_src	469	movdqa xmm1, [r2] ; y_src
470	movdqa xmm2, xmm0 ; store first for future process	470	movdqa xmm2, xmm0 ; store first for future process
471	movdqa xmm3, xmm1	471	movdqa xmm3, xmm1
472	; sum += diff;	472	; sum += diff;
473	movdqa xmm4, xmm0	473	movdqa xmm4, xmm0
474	psadbw xmm4, xmm1 ; 2 parts, [0,..,15], [64,..,79]	474	psadbw xmm4, xmm1 ; 2 parts, [0,..,15], [64,..,79]
475	; to be continued for sum	475	; to be continued for sum
476	pshufd xmm5, xmm4, 0C6h ; 11000110 B	476	pshufd xmm5, xmm4, 0C6h ; 11000110 B
477	paddw xmm4, xmm5	477	paddw xmm4, xmm5
478	movd r14d, xmm4	478	movd r14d, xmm4
479	add SUM, r14	479	add SUM, r14
480		480
481	; sqr += diff * diff;	481	; sqr += diff * diff;
482	pmaxub xmm0, xmm1	482	pmaxub xmm0, xmm1
483	pminub xmm1, xmm2	483	pminub xmm1, xmm2
484	psubb xmm0, xmm1 ; diff	484	psubb xmm0, xmm1 ; diff
485	SUM_SQR_SSE2 xmm1, xmm0, xmm7 ; dst, pSrc, zero	485	SUM_SQR_SSE2 xmm1, xmm0, xmm7 ; dst, pSrc, zero
486	movd r14d, xmm1	486	movd r14d, xmm1
487	add SQR, r14	487	add SQR, r14
488		488
489	; sum_cur += y_src[x];	489	; sum_cur += y_src[x];
490	movdqa xmm0, xmm3 ; cur_orig	490	movdqa xmm0, xmm3 ; cur_orig
491	movdqa xmm1, xmm0	491	movdqa xmm1, xmm0
492	punpcklbw xmm0, xmm7	492	punpcklbw xmm0, xmm7
493	punpckhbw xmm1, xmm7	493	punpckhbw xmm1, xmm7
494	paddw xmm0, xmm1 ; 8x2	494	paddw xmm0, xmm1 ; 8x2
495	SUM_WORD_8x2_SSE2 xmm0, xmm1	495	SUM_WORD_8x2_SSE2 xmm0, xmm1
496	movd r14d, xmm0	496	movd r14d, xmm0
497	and r14, 0ffffh	497	and r14, 0ffffh
498	add SUM_CUR, r14	498	add SUM_CUR, r14
499		499
500	; sqr_cur += y_src[x] * y_src[x];	500	; sqr_cur += y_src[x] * y_src[x];
501	SUM_SQR_SSE2 xmm0, xmm3, xmm7 ; dst, pSrc, zero	501	SUM_SQR_SSE2 xmm0, xmm3, xmm7 ; dst, pSrc, zero
502	movd r14d, xmm0	502	movd r14d, xmm0
503	add SQR_CUR, r14	503	add SQR_CUR, r14
504		504
505	lea r0, [r0+r1]	505	lea r0, [r0+r1]
506	lea r2, [r2+r3]	506	lea r2, [r2+r3]
507	dec r12	507	dec r12
508	jnz near .hloops	508	jnz near .hloops
509		509
510	mov r0, SUM	510	mov r0, SUM
511	sar r0, 8	511	sar r0, 8
512	imul r0, r0	512	imul r0, r0
513	mov r1, SQR	513	mov r1, SQR
514	sar r1, 8	514	sar r1, 8
515	sub r1, r0	515	sub r1, r0
516	mov [r4], r1w ; to store uiMotionIndex	516	mov [r4], r1w ; to store uiMotionIndex
517	mov r0, SUM_CUR	517	mov r0, SUM_CUR
518	sar r0, 8	518	sar r0, 8
519	imul r0, r0	519	imul r0, r0
520	mov r1, SQR_CUR	520	mov r1, SQR_CUR
521	sar r1, 8	521	sar r1, 8
522	sub r1, r0	522	sub r1, r0
523	mov [r4+2], r1w ; to store uiTextureIndex	523	mov [r4+2], r1w ; to store uiTextureIndex
524		524
525	POP_XMM	525	POP_XMM
526	LOAD_5_PARA_POP	526	LOAD_5_PARA_POP
527	pop r15	527	pop r15
528	pop r14	528	pop r14
529	pop r13	529	pop r13
530	pop r12	530	pop r12
531		531
532		532
533	%assign push_num 0	533	%assign push_num 0
534		534
535	ret	535	ret
536		536
537		537
538	;*************************************************************************************************************	538	;*************************************************************************************************************
@@ -550,69 +550,69 @@ WELS_EXTERN VAACalcSad_sse2 · 11 unmodified lines
550	%define psadframe r5	550	%define psadframe r5
551	%define psad8x8 r6	551	%define psad8x8 r6
552		552
553	push r12	553	push r12
554	push r13	554	push r13
555	%assign push_num 2	555	%assign push_num 2
556	LOAD_7_PARA	556	LOAD_7_PARA
557	PUSH_XMM 8	557	PUSH_XMM 8
558	SIGN_EXTENSION r2,r2d	558	SIGN_EXTENSION r2,r2d
559	SIGN_EXTENSION r3,r3d	559	SIGN_EXTENSION r3,r3d
560	SIGN_EXTENSION r4,r4d	560	SIGN_EXTENSION r4,r4d
561		561
562	mov r12,r4	562	mov r12,r4
563	shr r2, 4 ; iPicWidth/16	563	shr r2, 4 ; iPicWidth/16
564	shr r3, 4 ; iPicHeight/16	564	shr r3, 4 ; iPicHeight/16
565		565
566	shl r12, 4 ; iPicStride*16	566	shl r12, 4 ; iPicStride*16
567	pxor xmm0, xmm0	567	pxor xmm0, xmm0
568	pxor xmm7, xmm7 ; iFrameSad	568	pxor xmm7, xmm7 ; iFrameSad
569	height_loop:	569	height_loop:
570	mov r13, r2	570	mov r13, r2
571	push r0	571	push r0
572	push r1	572	push r1
573	width_loop:	573	width_loop:
574	pxor xmm6, xmm6	574	pxor xmm6, xmm6
575	WELS_SAD_16x2_SSE2 r0,r1,r4	575	WELS_SAD_16x2_SSE2 r0,r1,r4
576	WELS_SAD_16x2_SSE2 r0,r1,r4	576	WELS_SAD_16x2_SSE2 r0,r1,r4
577	WELS_SAD_16x2_SSE2 r0,r1,r4	577	WELS_SAD_16x2_SSE2 r0,r1,r4
578	WELS_SAD_16x2_SSE2 r0,r1,r4	578	WELS_SAD_16x2_SSE2 r0,r1,r4
579	paddd xmm7, xmm6	579	paddd xmm7, xmm6
580	movd [r6], xmm6	580	movd [r6], xmm6
581	psrldq xmm6, 8	581	psrldq xmm6, 8
582	movd [r6+4], xmm6	582	movd [r6+4], xmm6
583		583
584	pxor xmm6, xmm6	584	pxor xmm6, xmm6
585	WELS_SAD_16x2_SSE2 r0,r1,r4	585	WELS_SAD_16x2_SSE2 r0,r1,r4
586	WELS_SAD_16x2_SSE2 r0,r1,r4	586	WELS_SAD_16x2_SSE2 r0,r1,r4
587	WELS_SAD_16x2_SSE2 r0,r1,r4	587	WELS_SAD_16x2_SSE2 r0,r1,r4
588	WELS_SAD_16x2_SSE2 r0,r1,r4	588	WELS_SAD_16x2_SSE2 r0,r1,r4
589	paddd xmm7, xmm6	589	paddd xmm7, xmm6
590	movd [r6+8], xmm6	590	movd [r6+8], xmm6
591	psrldq xmm6, 8	591	psrldq xmm6, 8
592	movd [r6+12], xmm6	592	movd [r6+12], xmm6
593		593
594	add r6, 16	594	add r6, 16
595	sub r0, r12	595	sub r0, r12
596	sub r1, r12	596	sub r1, r12
597	add r0, 16	597	add r0, 16
598	add r1, 16	598	add r1, 16
599		599
600	dec r13	600	dec r13
601	jnz width_loop	601	jnz width_loop
602		602
603	pop r1	603	pop r1
604	pop r0	604	pop r0
605	add r0, r12	605	add r0, r12
606	add r1, r12	606	add r1, r12
607		607
608	dec r3	608	dec r3
609	jnz height_loop	609	jnz height_loop
610		610
611	;mov r13, [psadframe]	611	;mov r13, [psadframe]
612	movdqa xmm5, xmm7	612	movdqa xmm5, xmm7
613	psrldq xmm7, 8	613	psrldq xmm7, 8
614	paddd xmm7, xmm5	614	paddd xmm7, xmm5
615	movd [psadframe], xmm7	615	movd [psadframe], xmm7
616		616
617	%undef cur_data	617	%undef cur_data
618	%undef ref_data	618	%undef ref_data
@@ -622,12 +622,12 @@ width_loop: · 3 unmodified lines
622	%undef psadframe	622	%undef psadframe
623	%undef psad8x8	623	%undef psad8x8
624	%undef pushsize	624	%undef pushsize
625	POP_XMM	625	POP_XMM
626	LOAD_7_PARA_POP	626	LOAD_7_PARA_POP
627	pop r13	627	pop r13
628	pop r12	628	pop r12
629	%assign push_num 0	629	%assign push_num 0
630	ret	630	ret
631		631
632	%endif	632	%endif
633		633
@@ -653,103 +653,103 @@ WELS_EXTERN VAACalcSadVar_sse2 · 19 unmodified lines
653	%define tmp_esi esp + 0	653	%define tmp_esi esp + 0
654	%define tmp_edi esp + 4	654	%define tmp_edi esp + 4
655	%define pushsize 16	655	%define pushsize 16
656	push ebp	656	push ebp
657	push esi	657	push esi
658	push edi	658	push edi
659	push ebx	659	push ebx
660	sub esp, localsize	660	sub esp, localsize
661	mov esi, [cur_data]	661	mov esi, [cur_data]
662	mov edi, [ref_data]	662	mov edi, [ref_data]
663	mov ebx, [iPicStride]	663	mov ebx, [iPicStride]
664	mov edx, [psad8x8]	664	mov edx, [psad8x8]
665	mov eax, ebx	665	mov eax, ebx
666		666
667	shr dword [iPicWidth], 4 ; iPicWidth/16	667	shr dword [iPicWidth], 4 ; iPicWidth/16
668	shr dword [iPicHeight], 4 ; iPicHeight/16	668	shr dword [iPicHeight], 4 ; iPicHeight/16
669	shl eax, 4 ; iPicStride*16	669	shl eax, 4 ; iPicStride*16
670	pxor xmm0, xmm0	670	pxor xmm0, xmm0
671	pxor xmm7, xmm7 ; iFrameSad	671	pxor xmm7, xmm7 ; iFrameSad
672	var_height_loop:	672	var_height_loop:
673	mov ecx, dword [iPicWidth]	673	mov ecx, dword [iPicWidth]
674	mov [tmp_esi], esi	674	mov [tmp_esi], esi
675	mov [tmp_edi], edi	675	mov [tmp_edi], edi
676	var_width_loop:	676	var_width_loop:
677	pxor xmm6, xmm6 ; hiQuad_loQuad pSad8x8	677	pxor xmm6, xmm6 ; hiQuad_loQuad pSad8x8
678	pxor xmm5, xmm5 ; pSum16x16	678	pxor xmm5, xmm5 ; pSum16x16
679	pxor xmm4, xmm4 ; sqsum_16x16	679	pxor xmm4, xmm4 ; sqsum_16x16
680	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	680	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
681	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	681	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
682	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	682	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
683	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	683	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
684	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	684	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
685	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	685	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
686	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	686	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
687	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	687	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
688	paddd xmm7, xmm6	688	paddd xmm7, xmm6
689	movd [edx], xmm6	689	movd [edx], xmm6
690	psrldq xmm6, 8	690	psrldq xmm6, 8
691	movd [edx+4], xmm6	691	movd [edx+4], xmm6
692		692
693	pxor xmm6, xmm6	693	pxor xmm6, xmm6
694	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	694	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
695	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	695	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
696	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	696	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
697	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	697	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
698	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	698	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
699	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	699	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
700	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	700	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
701	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx	701	WELS_SAD_SUM_SQSUM_16x1_SSE2 esi,edi,ebx
702	paddd xmm7, xmm6	702	paddd xmm7, xmm6
703	movd [edx+8], xmm6	703	movd [edx+8], xmm6
704	psrldq xmm6, 8	704	psrldq xmm6, 8
705	movd [edx+12], xmm6	705	movd [edx+12], xmm6
706		706
707	mov ebp, [psum16x16]	707	mov ebp, [psum16x16]
708	movdqa xmm1, xmm5	708	movdqa xmm1, xmm5
709	psrldq xmm1, 8	709	psrldq xmm1, 8
710	paddd xmm5, xmm1	710	paddd xmm5, xmm1
711	movd [ebp], xmm5	711	movd [ebp], xmm5
712	add dword [psum16x16], 4	712	add dword [psum16x16], 4
713		713
714	movdqa xmm5, xmm4	714	movdqa xmm5, xmm4
715	psrldq xmm5, 8	715	psrldq xmm5, 8
716	paddd xmm4, xmm5	716	paddd xmm4, xmm5
717	movdqa xmm3, xmm4	717	movdqa xmm3, xmm4
718	psrldq xmm3, 4	718	psrldq xmm3, 4
719	paddd xmm4, xmm3	719	paddd xmm4, xmm3
720		720
721	mov ebp, [psqsum16x16]	721	mov ebp, [psqsum16x16]
722	movd [ebp], xmm4	722	movd [ebp], xmm4
723	add dword [psqsum16x16], 4	723	add dword [psqsum16x16], 4
724		724
725	add edx, 16	725	add edx, 16
726	sub esi, eax	726	sub esi, eax
727	sub edi, eax	727	sub edi, eax
728	add esi, 16	728	add esi, 16
729	add edi, 16	729	add edi, 16
730		730
731	dec ecx	731	dec ecx
732	jnz var_width_loop	732	jnz var_width_loop
733		733
734	mov esi, [tmp_esi]	734	mov esi, [tmp_esi]
735	mov edi, [tmp_edi]	735	mov edi, [tmp_edi]
736	add esi, eax	736	add esi, eax
737	add edi, eax	737	add edi, eax
738		738
739	dec dword [iPicHeight]	739	dec dword [iPicHeight]
740	jnz var_height_loop	740	jnz var_height_loop
741		741
742	mov edx, [psadframe]	742	mov edx, [psadframe]
743	movdqa xmm5, xmm7	743	movdqa xmm5, xmm7
744	psrldq xmm7, 8	744	psrldq xmm7, 8
745	paddd xmm7, xmm5	745	paddd xmm7, xmm5
746	movd [edx], xmm7	746	movd [edx], xmm7
747		747
748	add esp, localsize	748	add esp, localsize
749	pop ebx	749	pop ebx
750	pop edi	750	pop edi
751	pop esi	751	pop esi
752	pop ebp	752	pop ebp
753	%undef cur_data	753	%undef cur_data
754	%undef ref_data	754	%undef ref_data
755	%undef iPicWidth	755	%undef iPicWidth
@@ -763,7 +763,7 @@ var_width_loop: · 7 unmodified lines
763	%undef tmp_edi	763	%undef tmp_edi
764	%undef pushsize	764	%undef pushsize
765	%undef localsize	765	%undef localsize
766	ret	766	ret
767		767
768	%else ;64-bit	768	%else ;64-bit
769		769
@@ -784,112 +784,112 @@ WELS_EXTERN VAACalcSadVar_sse2 · 14 unmodified lines
784	%define psum16x16 arg8	784	%define psum16x16 arg8
785	%define psqsum16x16 arg9	785	%define psqsum16x16 arg9
786		786
787	push r12	787	push r12
788	push r13	788	push r13
789	push r14	789	push r14
790	push r15	790	push r15
791	%assign push_num 4	791	%assign push_num 4
792	PUSH_XMM 8	792	PUSH_XMM 8
793		793
794	%ifdef WIN64	794	%ifdef WIN64
795	mov r4, arg5 ;iPicStride	795	mov r4, arg5 ;iPicStride
796	mov r5, arg6 ;psad8x8	796	mov r5, arg6 ;psad8x8
797	%endif	797	%endif
798	mov r14,arg7	798	mov r14,arg7
799	SIGN_EXTENSION r2,r2d	799	SIGN_EXTENSION r2,r2d
800	SIGN_EXTENSION r3,r3d	800	SIGN_EXTENSION r3,r3d
801	SIGN_EXTENSION r4,r4d	801	SIGN_EXTENSION r4,r4d
802		802
803	mov r13,r4	803	mov r13,r4
804	shr r2,4	804	shr r2,4
805	shr r3,4	805	shr r3,4
806		806
807	shl r13,4 ; iPicStride*16	807	shl r13,4 ; iPicStride*16
808	pxor xmm0, xmm0	808	pxor xmm0, xmm0
809	pxor xmm7, xmm7 ; iFrameSad	809	pxor xmm7, xmm7 ; iFrameSad
810	var_height_loop:	810	var_height_loop:
811	push r2	811	push r2
812	%assign push_num push_num+1	812	%assign push_num push_num+1
813	mov r11, r0	813	mov r11, r0
814	mov r12, r1	814	mov r12, r1
815	var_width_loop:	815	var_width_loop:
816	pxor xmm6, xmm6 ; hiQuad_loQuad pSad8x8	816	pxor xmm6, xmm6 ; hiQuad_loQuad pSad8x8
817	pxor xmm5, xmm5 ; pSum16x16	817	pxor xmm5, xmm5 ; pSum16x16
818	pxor xmm4, xmm4 ; sqsum_16x16	818	pxor xmm4, xmm4 ; sqsum_16x16
819	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	819	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
820	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	820	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
821	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	821	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
822	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	822	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
823	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	823	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
824	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	824	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
825	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	825	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
826	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	826	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
827	paddd xmm7, xmm6	827	paddd xmm7, xmm6
828	movd [r14], xmm6	828	movd [r14], xmm6
829	psrldq xmm6, 8	829	psrldq xmm6, 8
830	movd [r14+4], xmm6	830	movd [r14+4], xmm6
831		831
832	pxor xmm6, xmm6	832	pxor xmm6, xmm6
833	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	833	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
834	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	834	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
835	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	835	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
836	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	836	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
837	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	837	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
838	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	838	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
839	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	839	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
840	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4	840	WELS_SAD_SUM_SQSUM_16x1_SSE2 r0,r1,r4
841	paddd xmm7, xmm6	841	paddd xmm7, xmm6
842	movd [r14+8], xmm6	842	movd [r14+8], xmm6
843	psrldq xmm6, 8	843	psrldq xmm6, 8
844	movd [r14+12], xmm6	844	movd [r14+12], xmm6
845		845
846	mov r15, psum16x16	846	mov r15, psum16x16
847	movdqa xmm1, xmm5	847	movdqa xmm1, xmm5
848	psrldq xmm1, 8	848	psrldq xmm1, 8
849	paddd xmm5, xmm1	849	paddd xmm5, xmm1
850	movd [r15], xmm5	850	movd [r15], xmm5
851	add dword psum16x16, 4	851	add dword psum16x16, 4
852		852
853	movdqa xmm5, xmm4	853	movdqa xmm5, xmm4
854	psrldq xmm5, 8	854	psrldq xmm5, 8
855	paddd xmm4, xmm5	855	paddd xmm4, xmm5
856	movdqa xmm3, xmm4	856	movdqa xmm3, xmm4
857	psrldq xmm3, 4	857	psrldq xmm3, 4
858	paddd xmm4, xmm3	858	paddd xmm4, xmm3
859		859
860	mov r15, psqsum16x16	860	mov r15, psqsum16x16
861	movd [r15], xmm4	861	movd [r15], xmm4
862	add dword psqsum16x16, 4	862	add dword psqsum16x16, 4
863		863
864	add r14,16	864	add r14,16
865	sub r0, r13	865	sub r0, r13
866	sub r1, r13	866	sub r1, r13
867	add r0, 16	867	add r0, 16
868	add r1, 16	868	add r1, 16
869		869
870	dec r2	870	dec r2
871	jnz var_width_loop	871	jnz var_width_loop
872		872
873	pop r2	873	pop r2
874	%assign push_num push_num-1	874	%assign push_num push_num-1
875	mov r0, r11	875	mov r0, r11
876	mov r1, r12	876	mov r1, r12
877	add r0, r13	877	add r0, r13
878	add r1, r13	878	add r1, r13
879	dec r3	879	dec r3
880	jnz var_height_loop	880	jnz var_height_loop
881		881
882	mov r15, psadframe	882	mov r15, psadframe
883	movdqa xmm5, xmm7	883	movdqa xmm5, xmm7
884	psrldq xmm7, 8	884	psrldq xmm7, 8
885	paddd xmm7, xmm5	885	paddd xmm7, xmm5
886	movd [r15], xmm7	886	movd [r15], xmm7
887		887
888	POP_XMM	888	POP_XMM
889	pop r15	889	pop r15
890	pop r14	890	pop r14
891	pop r13	891	pop r13
892	pop r12	892	pop r12
893	%assign push_num 0	893	%assign push_num 0
894	%undef cur_data	894	%undef cur_data
895	%undef ref_data	895	%undef ref_data
@@ -904,7 +904,7 @@ var_width_loop: · 8 unmodified lines
904	%undef tmp_edi	904	%undef tmp_edi
905	%undef pushsize	905	%undef pushsize
906	%undef localsize	906	%undef localsize
907	ret	907	ret
908		908
909	%endif	909	%endif
910		910
@@ -932,118 +932,118 @@ WELS_EXTERN VAACalcSadSsd_sse2 · 21 unmodified lines
932	%define tmp_edi esp + 4	932	%define tmp_edi esp + 4
933	%define tmp_sadframe esp + 8	933	%define tmp_sadframe esp + 8
934	%define pushsize 16	934	%define pushsize 16
935	push ebp	935	push ebp
936	push esi	936	push esi
937	push edi	937	push edi
938	push ebx	938	push ebx
939	sub esp, localsize	939	sub esp, localsize
940		940
941	mov ecx, [iPicWidth]	941	mov ecx, [iPicWidth]
942	mov ecx, [iPicHeight]	942	mov ecx, [iPicHeight]
943	mov esi, [cur_data]	943	mov esi, [cur_data]
944	mov edi, [ref_data]	944	mov edi, [ref_data]
945	mov ebx, [iPicStride]	945	mov ebx, [iPicStride]
946	mov edx, [psad8x8]	946	mov edx, [psad8x8]
947	mov eax, ebx	947	mov eax, ebx
948		948
949	shr dword [iPicWidth], 4 ; iPicWidth/16	949	shr dword [iPicWidth], 4 ; iPicWidth/16
950	shr dword [iPicHeight], 4 ; iPicHeight/16	950	shr dword [iPicHeight], 4 ; iPicHeight/16
951	shl eax, 4 ; iPicStride*16	951	shl eax, 4 ; iPicStride*16
952	mov ecx, [iPicWidth]	952	mov ecx, [iPicWidth]
953	mov ecx, [iPicHeight]	953	mov ecx, [iPicHeight]
954	pxor xmm0, xmm0	954	pxor xmm0, xmm0
955	movd [tmp_sadframe], xmm0	955	movd [tmp_sadframe], xmm0
956	sqdiff_height_loop:	956	sqdiff_height_loop:
957	mov ecx, dword [iPicWidth]	957	mov ecx, dword [iPicWidth]
958	mov [tmp_esi], esi	958	mov [tmp_esi], esi
959	mov [tmp_edi], edi	959	mov [tmp_edi], edi
960	sqdiff_width_loop:	960	sqdiff_width_loop:
961	pxor xmm7, xmm7 ; hiQuad_loQuad pSad8x8	961	pxor xmm7, xmm7 ; hiQuad_loQuad pSad8x8
962	pxor xmm6, xmm6 ; pSum16x16	962	pxor xmm6, xmm6 ; pSum16x16
963	pxor xmm5, xmm5 ; sqsum_16x16 four dword	963	pxor xmm5, xmm5 ; sqsum_16x16 four dword
964	pxor xmm4, xmm4 ; sqdiff_16x16 four Dword	964	pxor xmm4, xmm4 ; sqdiff_16x16 four Dword
965	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	965	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
966	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	966	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
967	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	967	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
968	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	968	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
969	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	969	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
970	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	970	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
971	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	971	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
972	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	972	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
973	movdqa xmm1, xmm7	973	movdqa xmm1, xmm7
974	movd [edx], xmm7	974	movd [edx], xmm7
975	psrldq xmm7, 8	975	psrldq xmm7, 8
976	paddd xmm1, xmm7	976	paddd xmm1, xmm7
977	movd [edx+4], xmm7	977	movd [edx+4], xmm7
978	movd ebp, xmm1	978	movd ebp, xmm1
979	add [tmp_sadframe], ebp	979	add [tmp_sadframe], ebp
980		980
981	pxor xmm7, xmm7	981	pxor xmm7, xmm7
982	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	982	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
983	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	983	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
984	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	984	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
985	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	985	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
986	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	986	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
987	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	987	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
988	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	988	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
989	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx	989	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 esi,edi,ebx
990	movdqa xmm1, xmm7	990	movdqa xmm1, xmm7
991	movd [edx+8], xmm7	991	movd [edx+8], xmm7
992	psrldq xmm7, 8	992	psrldq xmm7, 8
993	paddd xmm1, xmm7	993	paddd xmm1, xmm7
994	movd [edx+12], xmm7	994	movd [edx+12], xmm7
995	movd ebp, xmm1	995	movd ebp, xmm1
996	add [tmp_sadframe], ebp	996	add [tmp_sadframe], ebp
997		997
998	mov ebp, [psum16x16]	998	mov ebp, [psum16x16]
999	movdqa xmm1, xmm6	999	movdqa xmm1, xmm6
1000	psrldq xmm1, 8	1000	psrldq xmm1, 8
1001	paddd xmm6, xmm1	1001	paddd xmm6, xmm1
1002	movd [ebp], xmm6	1002	movd [ebp], xmm6
1003	add dword [psum16x16], 4	1003	add dword [psum16x16], 4
1004		1004
1005	mov ebp, [psqsum16x16]	1005	mov ebp, [psqsum16x16]
1006	pshufd xmm6, xmm5, 14 ;00001110	1006	pshufd xmm6, xmm5, 14 ;00001110
1007	paddd xmm6, xmm5	1007	paddd xmm6, xmm5
1008	pshufd xmm5, xmm6, 1 ;00000001	1008	pshufd xmm5, xmm6, 1 ;00000001
1009	paddd xmm5, xmm6	1009	paddd xmm5, xmm6
1010	movd [ebp], xmm5	1010	movd [ebp], xmm5
1011	add dword [psqsum16x16], 4	1011	add dword [psqsum16x16], 4
1012		1012
1013	mov ebp, [psqdiff16x16]	1013	mov ebp, [psqdiff16x16]
1014	pshufd xmm5, xmm4, 14 ; 00001110	1014	pshufd xmm5, xmm4, 14 ; 00001110
1015	paddd xmm5, xmm4	1015	paddd xmm5, xmm4
1016	pshufd xmm4, xmm5, 1 ; 00000001	1016	pshufd xmm4, xmm5, 1 ; 00000001
1017	paddd xmm4, xmm5	1017	paddd xmm4, xmm5
1018	movd [ebp], xmm4	1018	movd [ebp], xmm4
1019	add dword [psqdiff16x16], 4	1019	add dword [psqdiff16x16], 4
1020		1020
1021	add edx, 16	1021	add edx, 16
1022	sub esi, eax	1022	sub esi, eax
1023	sub edi, eax	1023	sub edi, eax
1024	add esi, 16	1024	add esi, 16
1025	add edi, 16	1025	add edi, 16
1026		1026
1027	dec ecx	1027	dec ecx
1028	jnz sqdiff_width_loop	1028	jnz sqdiff_width_loop
1029		1029
1030	mov esi, [tmp_esi]	1030	mov esi, [tmp_esi]
1031	mov edi, [tmp_edi]	1031	mov edi, [tmp_edi]
1032	add esi, eax	1032	add esi, eax
1033	add edi, eax	1033	add edi, eax
1034		1034
1035	dec dword [iPicHeight]	1035	dec dword [iPicHeight]
1036	jnz sqdiff_height_loop	1036	jnz sqdiff_height_loop
1037		1037
1038	mov ebx, [tmp_sadframe]	1038	mov ebx, [tmp_sadframe]
1039	mov eax, [psadframe]	1039	mov eax, [psadframe]
1040	mov [eax], ebx	1040	mov [eax], ebx
1041		1041
1042	add esp, localsize	1042	add esp, localsize
1043	pop ebx	1043	pop ebx
1044	pop edi	1044	pop edi
1045	pop esi	1045	pop esi
1046	pop ebp	1046	pop ebp
1047	%undef cur_data	1047	%undef cur_data
1048	%undef ref_data	1048	%undef ref_data
1049	%undef iPicWidth	1049	%undef iPicWidth
@@ -1059,7 +1059,7 @@ sqdiff_width_loop: · 9 unmodified lines
1059	%undef tmp_sadframe	1059	%undef tmp_sadframe
1060	%undef pushsize	1060	%undef pushsize
1061	%undef localsize	1061	%undef localsize
1062	ret	1062	ret
1063		1063
1064	%else	1064	%else
1065		1065
@@ -1083,128 +1083,128 @@ WELS_EXTERN VAACalcSadSsd_sse2 · 17 unmodified lines
1083	%define psqsum16x16 arg9;	1083	%define psqsum16x16 arg9;
1084	%define psqdiff16x16 arg10	1084	%define psqdiff16x16 arg10
1085		1085
1086	push r12	1086	push r12
1087	push r13	1087	push r13
1088	push r14	1088	push r14
1089	push r15	1089	push r15
1090	%assign push_num 4	1090	%assign push_num 4
1091	PUSH_XMM 10	1091	PUSH_XMM 10
1092		1092
1093	%ifdef WIN64	1093	%ifdef WIN64
1094	mov r4,arg5	1094	mov r4,arg5
1095	%endif	1095	%endif
1096	mov r14,arg7	1096	mov r14,arg7
1097	SIGN_EXTENSION r2,r2d	1097	SIGN_EXTENSION r2,r2d
1098	SIGN_EXTENSION r3,r3d	1098	SIGN_EXTENSION r3,r3d
1099	SIGN_EXTENSION r4,r4d	1099	SIGN_EXTENSION r4,r4d
1100		1100
1101	mov r13,r4	1101	mov r13,r4
1102	shr r2,4 ; iPicWidth/16	1102	shr r2,4 ; iPicWidth/16
1103	shr r3,4 ; iPicHeight/16	1103	shr r3,4 ; iPicHeight/16
1104	shl r13,4 ; iPicStride*16	1104	shl r13,4 ; iPicStride*16
1105	pxor xmm0, xmm0	1105	pxor xmm0, xmm0
1106	pxor xmm8, xmm8 ;framesad	1106	pxor xmm8, xmm8 ;framesad
1107	pxor xmm9, xmm9	1107	pxor xmm9, xmm9
1108	sqdiff_height_loop:	1108	sqdiff_height_loop:
1109	;mov ecx, dword [iPicWidth]	1109	;mov ecx, dword [iPicWidth]
1110	;mov r14,r2	1110	;mov r14,r2
1111	push r2	1111	push r2
1112	%assign push_num push_num +1	1112	%assign push_num push_num +1
1113	mov r10, r0	1113	mov r10, r0
1114	mov r11, r1	1114	mov r11, r1
1115	sqdiff_width_loop:	1115	sqdiff_width_loop:
1116	pxor xmm7, xmm7 ; hiQuad_loQuad pSad8x8	1116	pxor xmm7, xmm7 ; hiQuad_loQuad pSad8x8
1117	pxor xmm6, xmm6 ; pSum16x16	1117	pxor xmm6, xmm6 ; pSum16x16
1118	pxor xmm5, xmm5 ; sqsum_16x16 four dword	1118	pxor xmm5, xmm5 ; sqsum_16x16 four dword
1119	pxor xmm4, xmm4 ; sqdiff_16x16 four Dword	1119	pxor xmm4, xmm4 ; sqdiff_16x16 four Dword
1120	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1120	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1121	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1121	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1122	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1122	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1123	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1123	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1124	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1124	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1125	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1125	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1126	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1126	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1127	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1127	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1128	movdqa xmm1, xmm7	1128	movdqa xmm1, xmm7
1129	movd [r14], xmm7	1129	movd [r14], xmm7
1130	psrldq xmm7, 8	1130	psrldq xmm7, 8
1131	paddd xmm1, xmm7	1131	paddd xmm1, xmm7
1132	movd [r14+4], xmm7	1132	movd [r14+4], xmm7
1133	movd r15d, xmm1	1133	movd r15d, xmm1
1134	movd xmm9, r15d	1134	movd xmm9, r15d
1135	paddd xmm8,xmm9	1135	paddd xmm8,xmm9
1136		1136
1137		1137
1138	pxor xmm7, xmm7	1138	pxor xmm7, xmm7
1139	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1139	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1140	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1140	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1141	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1141	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1142	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1142	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1143	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1143	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1144	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1144	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1145	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1145	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1146	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4	1146	WELS_SAD_SUM_SQSUM_SQDIFF_16x1_SSE2 r0,r1,r4
1147	movdqa xmm1, xmm7	1147	movdqa xmm1, xmm7
1148	movd [r14+8], xmm7	1148	movd [r14+8], xmm7
1149	psrldq xmm7, 8	1149	psrldq xmm7, 8
1150	paddd xmm1, xmm7	1150	paddd xmm1, xmm7
1151	movd [r14+12], xmm7	1151	movd [r14+12], xmm7
1152	movd r15d, xmm1	1152	movd r15d, xmm1
1153	movd xmm9, r15d	1153	movd xmm9, r15d
1154	paddd xmm8,xmm9	1154	paddd xmm8,xmm9
1155		1155
1156	mov r15, psum16x16	1156	mov r15, psum16x16
1157	movdqa xmm1, xmm6	1157	movdqa xmm1, xmm6
1158	psrldq xmm1, 8	1158	psrldq xmm1, 8
1159	paddd xmm6, xmm1	1159	paddd xmm6, xmm1
1160	movd [r15], xmm6	1160	movd [r15], xmm6
1161	add dword psum16x16, 4	1161	add dword psum16x16, 4
1162		1162
1163	mov r15, psqsum16x16	1163	mov r15, psqsum16x16
1164	pshufd xmm6, xmm5, 14 ;00001110	1164	pshufd xmm6, xmm5, 14 ;00001110
1165	paddd xmm6, xmm5	1165	paddd xmm6, xmm5
1166	pshufd xmm5, xmm6, 1 ;00000001	1166	pshufd xmm5, xmm6, 1 ;00000001
1167	paddd xmm5, xmm6	1167	paddd xmm5, xmm6
1168	movd [r15], xmm5	1168	movd [r15], xmm5
1169	add dword psqsum16x16, 4	1169	add dword psqsum16x16, 4
1170		1170
1171	mov r15, psqdiff16x16	1171	mov r15, psqdiff16x16
1172	pshufd xmm5, xmm4, 14 ; 00001110	1172	pshufd xmm5, xmm4, 14 ; 00001110
1173	paddd xmm5, xmm4	1173	paddd xmm5, xmm4
1174	pshufd xmm4, xmm5, 1 ; 00000001	1174	pshufd xmm4, xmm5, 1 ; 00000001
1175	paddd xmm4, xmm5	1175	paddd xmm4, xmm5
1176	movd [r15], xmm4	1176	movd [r15], xmm4
1177	add dword psqdiff16x16, 4	1177	add dword psqdiff16x16, 4
1178		1178
1179	add r14,16	1179	add r14,16
1180	sub r0, r13	1180	sub r0, r13
1181	sub r1, r13	1181	sub r1, r13
1182	add r0, 16	1182	add r0, 16
1183	add r1, 16	1183	add r1, 16
1184		1184
1185	dec r2	1185	dec r2
1186	jnz sqdiff_width_loop	1186	jnz sqdiff_width_loop
1187		1187
1188	pop r2	1188	pop r2
1189	%assign push_num push_num -1	1189	%assign push_num push_num -1
1190		1190
1191	mov r0, r10	1191	mov r0, r10
1192	mov r1, r11	1192	mov r1, r11
1193	add r0, r13	1193	add r0, r13
1194	add r1, r13	1194	add r1, r13
1195		1195
1196	dec r3	1196	dec r3
1197	jnz sqdiff_height_loop	1197	jnz sqdiff_height_loop
1198		1198
1199	mov r13, psadframe	1199	mov r13, psadframe
1200	movd [r13], xmm8	1200	movd [r13], xmm8
1201		1201
1202	POP_XMM	1202	POP_XMM
1203	pop r15	1203	pop r15
1204	pop r14	1204	pop r14
1205	pop r13	1205	pop r13
1206	pop r12	1206	pop r12
1207	%assign push_num 0	1207	%assign push_num 0
1208		1208
1209	%undef cur_data	1209	%undef cur_data
1210	%undef ref_data	1210	%undef ref_data
@@ -1221,7 +1221,7 @@ sqdiff_width_loop: · 10 unmodified lines
1221	%undef tmp_sadframe	1221	%undef tmp_sadframe
1222	%undef pushsize	1222	%undef pushsize
1223	%undef localsize	1223	%undef localsize
1224	ret	1224	ret
1225		1225
1226		1226
1227		1227
@@ -1249,145 +1249,145 @@ WELS_EXTERN VAACalcSadBgd_sse2 · 21 unmodified lines
1249	%define tmp_edi esp + 4	1249	%define tmp_edi esp + 4
1250	%define tmp_ecx esp + 8	1250	%define tmp_ecx esp + 8
1251	%define pushsize 16	1251	%define pushsize 16
1252	push ebp	1252	push ebp
1253	push esi	1253	push esi
1254	push edi	1254	push edi
1255	push ebx	1255	push ebx
1256	sub esp, localsize	1256	sub esp, localsize
1257	mov esi, [cur_data]	1257	mov esi, [cur_data]
1258	mov edi, [ref_data]	1258	mov edi, [ref_data]
1259	mov ebx, [iPicStride]	1259	mov ebx, [iPicStride]
1260	mov eax, ebx	1260	mov eax, ebx
1261		1261
1262	shr dword [iPicWidth], 4 ; iPicWidth/16	1262	shr dword [iPicWidth], 4 ; iPicWidth/16
1263	shr dword [iPicHeight], 4 ; iPicHeight/16	1263	shr dword [iPicHeight], 4 ; iPicHeight/16
1264	shl eax, 4 ; iPicStride*16	1264	shl eax, 4 ; iPicStride*16
1265	xor ebp, ebp	1265	xor ebp, ebp
1266	pxor xmm0, xmm0	1266	pxor xmm0, xmm0
1267	bgd_height_loop:	1267	bgd_height_loop:
1268	mov ecx, dword [iPicWidth]	1268	mov ecx, dword [iPicWidth]
1269	mov [tmp_esi], esi	1269	mov [tmp_esi], esi
1270	mov [tmp_edi], edi	1270	mov [tmp_edi], edi
1271	bgd_width_loop:	1271	bgd_width_loop:
1272	pxor xmm7, xmm7 ; pSad8x8	1272	pxor xmm7, xmm7 ; pSad8x8
1273	pxor xmm6, xmm6 ; sum_cur_8x8	1273	pxor xmm6, xmm6 ; sum_cur_8x8
1274	pxor xmm5, xmm5 ; sum_ref_8x8	1274	pxor xmm5, xmm5 ; sum_ref_8x8
1275	pxor xmm4, xmm4 ; pMad8x8	1275	pxor xmm4, xmm4 ; pMad8x8
1276	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1276	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1277	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1277	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1278	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1278	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1279	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1279	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1280	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1280	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1281	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1281	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1282	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1282	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1283	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1283	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1284		1284
1285		1285
1286	mov edx, [p_mad8x8]	1286	mov edx, [p_mad8x8]
1287	WELS_MAX_REG_SSE2 xmm4	1287	WELS_MAX_REG_SSE2 xmm4
1288		1288
1289	;movdqa xmm1, xmm4	1289	;movdqa xmm1, xmm4
1290	;punpcklbw xmm1, xmm0	1290	;punpcklbw xmm1, xmm0
1291	;punpcklwd xmm1, xmm0	1291	;punpcklwd xmm1, xmm0
1292	;movd [edx], xmm1	1292	;movd [edx], xmm1
1293	;punpckhbw xmm4, xmm0	1293	;punpckhbw xmm4, xmm0
1294	;punpcklwd xmm4, xmm0	1294	;punpcklwd xmm4, xmm0
1295	;movd [edx+4], xmm4	1295	;movd [edx+4], xmm4
1296	;add edx, 8	1296	;add edx, 8
1297	;mov [p_mad8x8], edx	1297	;mov [p_mad8x8], edx
1298	mov [tmp_ecx], ecx	1298	mov [tmp_ecx], ecx
1299	movhlps xmm1, xmm4	1299	movhlps xmm1, xmm4
1300	movd ecx, xmm4	1300	movd ecx, xmm4
1301	mov [edx], cl	1301	mov [edx], cl
1302	movd ecx, xmm1	1302	movd ecx, xmm1
1303	mov [edx+1],cl	1303	mov [edx+1],cl
1304	add edx, 2	1304	add edx, 2
1305	mov [p_mad8x8], edx	1305	mov [p_mad8x8], edx
1306		1306
1307		1307
1308	pslldq xmm7, 4	1308	pslldq xmm7, 4
1309	pslldq xmm6, 4	1309	pslldq xmm6, 4
1310	pslldq xmm5, 4	1310	pslldq xmm5, 4
1311		1311
1312		1312
1313	pxor xmm4, xmm4 ; pMad8x8	1313	pxor xmm4, xmm4 ; pMad8x8
1314	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1314	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1315	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1315	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1316	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1316	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1317	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1317	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1318	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1318	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1319	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1319	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1320	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1320	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1321	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx	1321	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,esi ,edi, ebx
1322		1322
1323	mov edx, [p_mad8x8]	1323	mov edx, [p_mad8x8]
1324	WELS_MAX_REG_SSE2 xmm4	1324	WELS_MAX_REG_SSE2 xmm4
1325		1325
1326	;movdqa xmm1, xmm4	1326	;movdqa xmm1, xmm4
1327	;punpcklbw xmm1, xmm0	1327	;punpcklbw xmm1, xmm0
1328	;punpcklwd xmm1, xmm0	1328	;punpcklwd xmm1, xmm0
1329	;movd [edx], xmm1	1329	;movd [edx], xmm1
1330	;punpckhbw xmm4, xmm0	1330	;punpckhbw xmm4, xmm0
1331	;punpcklwd xmm4, xmm0	1331	;punpcklwd xmm4, xmm0
1332	;movd [edx+4], xmm4	1332	;movd [edx+4], xmm4
1333	;add edx, 8	1333	;add edx, 8
1334	;mov [p_mad8x8], edx	1334	;mov [p_mad8x8], edx
1335	movhlps xmm1, xmm4	1335	movhlps xmm1, xmm4
1336	movd ecx, xmm4	1336	movd ecx, xmm4
1337	mov [edx], cl	1337	mov [edx], cl
1338	movd ecx, xmm1	1338	movd ecx, xmm1
1339	mov [edx+1],cl	1339	mov [edx+1],cl
1340	add edx, 2	1340	add edx, 2
1341	mov [p_mad8x8], edx	1341	mov [p_mad8x8], edx
1342		1342
1343	; data in xmm7, xmm6, xmm5: D1 D3 D0 D2	1343	; data in xmm7, xmm6, xmm5: D1 D3 D0 D2
1344		1344
1345	mov edx, [psad8x8]	1345	mov edx, [psad8x8]
1346	pshufd xmm1, xmm7, 10001101b ; D3 D2 D1 D0	1346	pshufd xmm1, xmm7, 10001101b ; D3 D2 D1 D0
1347	movdqa [edx], xmm1	1347	movdqa [edx], xmm1
1348	add edx, 16	1348	add edx, 16
1349	mov [psad8x8], edx ; sad8x8	1349	mov [psad8x8], edx ; sad8x8
1350		1350
1351	paddd xmm1, xmm7 ; D1+3 D3+2 D0+1 D2+0	1351	paddd xmm1, xmm7 ; D1+3 D3+2 D0+1 D2+0
1352	pshufd xmm2, xmm1, 00000011b	1352	pshufd xmm2, xmm1, 00000011b
1353	paddd xmm1, xmm2	1353	paddd xmm1, xmm2
1354	movd edx, xmm1	1354	movd edx, xmm1
1355	add ebp, edx ; sad frame	1355	add ebp, edx ; sad frame
1356		1356
1357	mov edx, [p_sd8x8]	1357	mov edx, [p_sd8x8]
1358	psubd xmm6, xmm5	1358	psubd xmm6, xmm5
1359	pshufd xmm1, xmm6, 10001101b	1359	pshufd xmm1, xmm6, 10001101b
1360	movdqa [edx], xmm1	1360	movdqa [edx], xmm1
1361	add edx, 16	1361	add edx, 16
1362	mov [p_sd8x8], edx	1362	mov [p_sd8x8], edx
1363		1363
1364		1364
1365	add edx, 16	1365	add edx, 16
1366	sub esi, eax	1366	sub esi, eax
1367	sub edi, eax	1367	sub edi, eax
1368	add esi, 16	1368	add esi, 16
1369	add edi, 16	1369	add edi, 16
1370		1370
1371	mov ecx, [tmp_ecx]	1371	mov ecx, [tmp_ecx]
1372	dec ecx	1372	dec ecx
1373	jnz bgd_width_loop	1373	jnz bgd_width_loop
1374		1374
1375	mov esi, [tmp_esi]	1375	mov esi, [tmp_esi]
1376	mov edi, [tmp_edi]	1376	mov edi, [tmp_edi]
1377	add esi, eax	1377	add esi, eax
1378	add edi, eax	1378	add edi, eax
1379		1379
1380	dec dword [iPicHeight]	1380	dec dword [iPicHeight]
1381	jnz bgd_height_loop	1381	jnz bgd_height_loop
1382		1382
1383	mov edx, [psadframe]	1383	mov edx, [psadframe]
1384	mov [edx], ebp	1384	mov [edx], ebp
1385		1385
1386	add esp, localsize	1386	add esp, localsize
1387	pop ebx	1387	pop ebx
1388	pop edi	1388	pop edi
1389	pop esi	1389	pop esi
1390	pop ebp	1390	pop ebp
1391	%undef cur_data	1391	%undef cur_data
1392	%undef ref_data	1392	%undef ref_data
1393	%undef iPicWidth	1393	%undef iPicWidth
@@ -1401,7 +1401,7 @@ bgd_width_loop: · 7 unmodified lines
1401	%undef tmp_edi	1401	%undef tmp_edi
1402	%undef pushsize	1402	%undef pushsize
1403	%undef localsize	1403	%undef localsize
1404	ret	1404	ret
1405		1405
1406		1406
1407		1407
@@ -1431,190 +1431,190 @@ WELS_EXTERN VAACalcSadSsdBgd_sse2 · 23 unmodified lines
1431	%define tmp_sadframe esp + 8	1431	%define tmp_sadframe esp + 8
1432	%define tmp_ecx esp + 12	1432	%define tmp_ecx esp + 12
1433	%define pushsize 16	1433	%define pushsize 16
1434	push ebp	1434	push ebp
1435	push esi	1435	push esi
1436	push edi	1436	push edi
1437	push ebx	1437	push ebx
1438	sub esp, localsize	1438	sub esp, localsize
1439	mov esi, [cur_data]	1439	mov esi, [cur_data]
1440	mov edi, [ref_data]	1440	mov edi, [ref_data]
1441	mov ebx, [iPicStride]	1441	mov ebx, [iPicStride]
1442	mov eax, ebx	1442	mov eax, ebx
1443		1443
1444	shr dword [iPicWidth], 4 ; iPicWidth/16	1444	shr dword [iPicWidth], 4 ; iPicWidth/16
1445	shr dword [iPicHeight], 4 ; iPicHeight/16	1445	shr dword [iPicHeight], 4 ; iPicHeight/16
1446	shl eax, 4 ; iPicStride*16	1446	shl eax, 4 ; iPicStride*16
1447	pxor xmm0, xmm0	1447	pxor xmm0, xmm0
1448	movd [tmp_sadframe], xmm0	1448	movd [tmp_sadframe], xmm0
1449	sqdiff_bgd_height_loop:	1449	sqdiff_bgd_height_loop:
1450	mov ecx, dword [iPicWidth]	1450	mov ecx, dword [iPicWidth]
1451	mov [tmp_esi], esi	1451	mov [tmp_esi], esi
1452	mov [tmp_edi], edi	1452	mov [tmp_edi], edi
1453	sqdiff_bgd_width_loop:	1453	sqdiff_bgd_width_loop:
1454	pxor xmm7, xmm7 ; pSad8x8 interleaves sqsum16x16: sqsum1 sad1 sqsum0 sad0	1454	pxor xmm7, xmm7 ; pSad8x8 interleaves sqsum16x16: sqsum1 sad1 sqsum0 sad0
1455	pxor xmm6, xmm6 ; sum_8x8 interleaves cur and pRef in Dword, Sref1 Scur1 Sref0 Scur0	1455	pxor xmm6, xmm6 ; sum_8x8 interleaves cur and pRef in Dword, Sref1 Scur1 Sref0 Scur0
1456	pxor xmm5, xmm5 ; pMad8x8	1456	pxor xmm5, xmm5 ; pMad8x8
1457	pxor xmm4, xmm4 ; sqdiff_16x16 four Dword	1457	pxor xmm4, xmm4 ; sqdiff_16x16 four Dword
1458	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1458	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1459	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1459	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1460	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1460	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1461	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1461	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1462	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1462	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1463	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1463	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1464	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1464	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1465	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1465	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1466		1466
1467	mov edx, [psad8x8]	1467	mov edx, [psad8x8]
1468	movdqa xmm2, xmm7	1468	movdqa xmm2, xmm7
1469	pshufd xmm1, xmm2, 00001110b	1469	pshufd xmm1, xmm2, 00001110b
1470	movd [edx], xmm2	1470	movd [edx], xmm2
1471	movd [edx+4], xmm1	1471	movd [edx+4], xmm1
1472	add edx, 8	1472	add edx, 8
1473	mov [psad8x8], edx ; sad8x8	1473	mov [psad8x8], edx ; sad8x8
1474		1474
1475	paddd xmm1, xmm2	1475	paddd xmm1, xmm2
1476	movd edx, xmm1	1476	movd edx, xmm1
1477	add [tmp_sadframe], edx ; iFrameSad	1477	add [tmp_sadframe], edx ; iFrameSad
1478		1478
1479	mov edx, [psum16x16]	1479	mov edx, [psum16x16]
1480	movdqa xmm1, xmm6	1480	movdqa xmm1, xmm6
1481	pshufd xmm2, xmm1, 00001110b	1481	pshufd xmm2, xmm1, 00001110b
1482	paddd xmm1, xmm2	1482	paddd xmm1, xmm2
1483	movd [edx], xmm1 ; sum	1483	movd [edx], xmm1 ; sum
1484		1484
1485	mov edx, [p_sd8x8]	1485	mov edx, [p_sd8x8]
1486	pshufd xmm1, xmm6, 11110101b ; Sref1 Sref1 Sref0 Sref0	1486	pshufd xmm1, xmm6, 11110101b ; Sref1 Sref1 Sref0 Sref0
1487	psubd xmm6, xmm1 ; 00 diff1 00 diff0	1487	psubd xmm6, xmm1 ; 00 diff1 00 diff0
1488	pshufd xmm1, xmm6, 00001000b ; xx xx diff1 diff0	1488	pshufd xmm1, xmm6, 00001000b ; xx xx diff1 diff0
1489	movq [edx], xmm1	1489	movq [edx], xmm1
1490	add edx, 8	1490	add edx, 8
1491	mov [p_sd8x8], edx	1491	mov [p_sd8x8], edx
1492		1492
1493	mov edx, [p_mad8x8]	1493	mov edx, [p_mad8x8]
1494	WELS_MAX_REG_SSE2 xmm5	1494	WELS_MAX_REG_SSE2 xmm5
1495	;movdqa xmm1, xmm5	1495	;movdqa xmm1, xmm5
1496	;punpcklbw xmm1, xmm0	1496	;punpcklbw xmm1, xmm0
1497	;punpcklwd xmm1, xmm0	1497	;punpcklwd xmm1, xmm0
1498	;movd [edx], xmm1	1498	;movd [edx], xmm1
1499	;punpckhbw xmm5, xmm0	1499	;punpckhbw xmm5, xmm0
1500	;punpcklwd xmm5, xmm0	1500	;punpcklwd xmm5, xmm0
1501	;movd [edx+4], xmm5	1501	;movd [edx+4], xmm5
1502	;add edx, 8	1502	;add edx, 8
1503	;mov [p_mad8x8], edx	1503	;mov [p_mad8x8], edx
1504	mov [tmp_ecx], ecx	1504	mov [tmp_ecx], ecx
1505	movhlps xmm1, xmm5	1505	movhlps xmm1, xmm5
1506	movd ecx, xmm5	1506	movd ecx, xmm5
1507	mov [edx], cl	1507	mov [edx], cl
1508	movd ecx, xmm1	1508	movd ecx, xmm1
1509	mov [edx+1],cl	1509	mov [edx+1],cl
1510	add edx, 2	1510	add edx, 2
1511	mov [p_mad8x8], edx	1511	mov [p_mad8x8], edx
1512		1512
1513	psrlq xmm7, 32	1513	psrlq xmm7, 32
1514	psllq xmm7, 32 ; clear sad	1514	psllq xmm7, 32 ; clear sad
1515	pxor xmm6, xmm6 ; sum_8x8 interleaves cur and pRef in Dword, Sref1 Scur1 Sref0 Scur0	1515	pxor xmm6, xmm6 ; sum_8x8 interleaves cur and pRef in Dword, Sref1 Scur1 Sref0 Scur0
1516	pxor xmm5, xmm5 ; pMad8x8	1516	pxor xmm5, xmm5 ; pMad8x8
1517	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1517	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1518	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1518	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1519	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1519	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1520	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1520	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1521	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1521	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1522	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1522	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1523	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1523	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1524	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx	1524	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, esi , edi , ebx
1525		1525
1526	mov edx, [psad8x8]	1526	mov edx, [psad8x8]
1527	movdqa xmm2, xmm7	1527	movdqa xmm2, xmm7
1528	pshufd xmm1, xmm2, 00001110b	1528	pshufd xmm1, xmm2, 00001110b
1529	movd [edx], xmm2	1529	movd [edx], xmm2
1530	movd [edx+4], xmm1	1530	movd [edx+4], xmm1
1531	add edx, 8	1531	add edx, 8
1532	mov [psad8x8], edx ; sad8x8	1532	mov [psad8x8], edx ; sad8x8
1533		1533
1534	paddd xmm1, xmm2	1534	paddd xmm1, xmm2
1535	movd edx, xmm1	1535	movd edx, xmm1
1536	add [tmp_sadframe], edx ; iFrameSad	1536	add [tmp_sadframe], edx ; iFrameSad
1537		1537
1538	mov edx, [psum16x16]	1538	mov edx, [psum16x16]
1539	movdqa xmm1, xmm6	1539	movdqa xmm1, xmm6
1540	pshufd xmm2, xmm1, 00001110b	1540	pshufd xmm2, xmm1, 00001110b
1541	paddd xmm1, xmm2	1541	paddd xmm1, xmm2
1542	movd ebp, xmm1 ; sum	1542	movd ebp, xmm1 ; sum
1543	add [edx], ebp	1543	add [edx], ebp
1544	add edx, 4	1544	add edx, 4
1545	mov [psum16x16], edx	1545	mov [psum16x16], edx
1546		1546
1547	mov edx, [psqsum16x16]	1547	mov edx, [psqsum16x16]
1548	psrlq xmm7, 32	1548	psrlq xmm7, 32
1549	pshufd xmm2, xmm7, 00001110b	1549	pshufd xmm2, xmm7, 00001110b
1550	paddd xmm2, xmm7	1550	paddd xmm2, xmm7
1551	movd [edx], xmm2 ; sqsum	1551	movd [edx], xmm2 ; sqsum
1552	add edx, 4	1552	add edx, 4
1553	mov [psqsum16x16], edx	1553	mov [psqsum16x16], edx
1554		1554
1555	mov edx, [p_sd8x8]	1555	mov edx, [p_sd8x8]
1556	pshufd xmm1, xmm6, 11110101b ; Sref1 Sref1 Sref0 Sref0	1556	pshufd xmm1, xmm6, 11110101b ; Sref1 Sref1 Sref0 Sref0
1557	psubd xmm6, xmm1 ; 00 diff1 00 diff0	1557	psubd xmm6, xmm1 ; 00 diff1 00 diff0
1558	pshufd xmm1, xmm6, 00001000b ; xx xx diff1 diff0	1558	pshufd xmm1, xmm6, 00001000b ; xx xx diff1 diff0
1559	movq [edx], xmm1	1559	movq [edx], xmm1
1560	add edx, 8	1560	add edx, 8
1561	mov [p_sd8x8], edx	1561	mov [p_sd8x8], edx
1562		1562
1563	mov edx, [p_mad8x8]	1563	mov edx, [p_mad8x8]
1564	WELS_MAX_REG_SSE2 xmm5	1564	WELS_MAX_REG_SSE2 xmm5
1565	;movdqa xmm1, xmm5	1565	;movdqa xmm1, xmm5
1566	;punpcklbw xmm1, xmm0	1566	;punpcklbw xmm1, xmm0
1567	;punpcklwd xmm1, xmm0	1567	;punpcklwd xmm1, xmm0
1568	;movd [edx], xmm1	1568	;movd [edx], xmm1
1569	;punpckhbw xmm5, xmm0	1569	;punpckhbw xmm5, xmm0
1570	;punpcklwd xmm5, xmm0	1570	;punpcklwd xmm5, xmm0
1571	;movd [edx+4], xmm5	1571	;movd [edx+4], xmm5
1572	;add edx, 8	1572	;add edx, 8
1573	;mov [p_mad8x8], edx	1573	;mov [p_mad8x8], edx
1574	movhlps xmm1, xmm5	1574	movhlps xmm1, xmm5
1575	movd ecx, xmm5	1575	movd ecx, xmm5
1576	mov [edx], cl	1576	mov [edx], cl
1577	movd ecx, xmm1	1577	movd ecx, xmm1
1578	mov [edx+1],cl	1578	mov [edx+1],cl
1579	add edx, 2	1579	add edx, 2
1580	mov [p_mad8x8], edx	1580	mov [p_mad8x8], edx
1581		1581
1582	mov edx, [psqdiff16x16]	1582	mov edx, [psqdiff16x16]
1583	pshufd xmm1, xmm4, 00001110b	1583	pshufd xmm1, xmm4, 00001110b
1584	paddd xmm4, xmm1	1584	paddd xmm4, xmm1
1585	pshufd xmm1, xmm4, 00000001b	1585	pshufd xmm1, xmm4, 00000001b
1586	paddd xmm4, xmm1	1586	paddd xmm4, xmm1
1587	movd [edx], xmm4	1587	movd [edx], xmm4
1588	add edx, 4	1588	add edx, 4
1589	mov [psqdiff16x16], edx	1589	mov [psqdiff16x16], edx
1590		1590
1591	add edx, 16	1591	add edx, 16
1592	sub esi, eax	1592	sub esi, eax
1593	sub edi, eax	1593	sub edi, eax
1594	add esi, 16	1594	add esi, 16
1595	add edi, 16	1595	add edi, 16
1596		1596
1597	mov ecx, [tmp_ecx]	1597	mov ecx, [tmp_ecx]
1598	dec ecx	1598	dec ecx
1599	jnz sqdiff_bgd_width_loop	1599	jnz sqdiff_bgd_width_loop
1600		1600
1601	mov esi, [tmp_esi]	1601	mov esi, [tmp_esi]
1602	mov edi, [tmp_edi]	1602	mov edi, [tmp_edi]
1603	add esi, eax	1603	add esi, eax
1604	add edi, eax	1604	add edi, eax
1605		1605
1606	dec dword [iPicHeight]	1606	dec dword [iPicHeight]
1607	jnz sqdiff_bgd_height_loop	1607	jnz sqdiff_bgd_height_loop
1608		1608
1609	mov edx, [psadframe]	1609	mov edx, [psadframe]
1610	mov ebp, [tmp_sadframe]	1610	mov ebp, [tmp_sadframe]
1611	mov [edx], ebp	1611	mov [edx], ebp
1612		1612
1613	add esp, localsize	1613	add esp, localsize
1614	pop ebx	1614	pop ebx
1615	pop edi	1615	pop edi
1616	pop esi	1616	pop esi
1617	pop ebp	1617	pop ebp
1618	%undef cur_data	1618	%undef cur_data
1619	%undef ref_data	1619	%undef ref_data
1620	%undef iPicWidth	1620	%undef iPicWidth
@@ -1631,7 +1631,7 @@ sqdiff_bgd_width_loop: · 10 unmodified lines
1631	%undef tmp_edi	1631	%undef tmp_edi
1632	%undef pushsize	1632	%undef pushsize
1633	%undef localsize	1633	%undef localsize
1634	ret	1634	ret
1635	%else	1635	%else
1636		1636
1637	;*************************************************************************************************************	1637	;*************************************************************************************************************
@@ -1651,142 +1651,142 @@ WELS_EXTERN VAACalcSadBgd_sse2 · 13 unmodified lines
1651	%define p_sd8x8 arg8;	1651	%define p_sd8x8 arg8;
1652	%define p_mad8x8 arg9;	1652	%define p_mad8x8 arg9;
1653		1653
1654	push r12	1654	push r12
1655	push r13	1655	push r13
1656	push r14	1656	push r14
1657	push r15	1657	push r15
1658	%assign push_num 4	1658	%assign push_num 4
1659	PUSH_XMM 10	1659	PUSH_XMM 10
1660	%ifdef WIN64	1660	%ifdef WIN64
1661	mov r4,arg5	1661	mov r4,arg5
1662	; mov r5,arg6	1662	; mov r5,arg6
1663	%endif	1663	%endif
1664	mov r14,arg7	1664	mov r14,arg7
1665	SIGN_EXTENSION r2,r2d	1665	SIGN_EXTENSION r2,r2d
1666	SIGN_EXTENSION r3,r3d	1666	SIGN_EXTENSION r3,r3d
1667	SIGN_EXTENSION r4,r4d	1667	SIGN_EXTENSION r4,r4d
1668		1668
1669		1669
1670	mov r13,r4	1670	mov r13,r4
1671	mov r15,r0	1671	mov r15,r0
1672	shr r2,4	1672	shr r2,4
1673	shr r3,4	1673	shr r3,4
1674	shl r13,4	1674	shl r13,4
1675	pxor xmm0, xmm0	1675	pxor xmm0, xmm0
1676	pxor xmm8, xmm8	1676	pxor xmm8, xmm8
1677	pxor xmm9, xmm9	1677	pxor xmm9, xmm9
1678	bgd_height_loop:	1678	bgd_height_loop:
1679	;mov ecx, dword [iPicWidth]	1679	;mov ecx, dword [iPicWidth]
1680	push r2	1680	push r2
1681	%assign push_num push_num+1	1681	%assign push_num push_num+1
1682	mov r10, r15	1682	mov r10, r15
1683	mov r11, r1	1683	mov r11, r1
1684	bgd_width_loop:	1684	bgd_width_loop:
1685	pxor xmm7, xmm7 ; pSad8x8	1685	pxor xmm7, xmm7 ; pSad8x8
1686	pxor xmm6, xmm6 ; sum_cur_8x8	1686	pxor xmm6, xmm6 ; sum_cur_8x8
1687	pxor xmm5, xmm5 ; sum_ref_8x8	1687	pxor xmm5, xmm5 ; sum_ref_8x8
1688	pxor xmm4, xmm4 ; pMad8x8	1688	pxor xmm4, xmm4 ; pMad8x8
1689	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1689	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1690	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1690	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1691	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1691	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1692	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1692	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1693	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1693	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1694	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1694	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1695	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1695	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1696	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1696	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1697		1697
1698		1698
1699	mov r14, p_mad8x8	1699	mov r14, p_mad8x8
1700	WELS_MAX_REG_SSE2 xmm4	1700	WELS_MAX_REG_SSE2 xmm4
1701		1701
1702	;mov [tmp_ecx], ecx	1702	;mov [tmp_ecx], ecx
1703	movhlps xmm1, xmm4	1703	movhlps xmm1, xmm4
1704	movd r0d, xmm4	1704	movd r0d, xmm4
1705		1705
1706		1706
1707	mov [r14], r0b	1707	mov [r14], r0b
1708	movd r0d, xmm1	1708	movd r0d, xmm1
1709	mov [r14+1],r0b	1709	mov [r14+1],r0b
1710	add r14, 2	1710	add r14, 2
1711	;mov p_mad8x8, r14	1711	;mov p_mad8x8, r14
1712		1712
1713		1713
1714	pslldq xmm7, 4	1714	pslldq xmm7, 4
1715	pslldq xmm6, 4	1715	pslldq xmm6, 4
1716	pslldq xmm5, 4	1716	pslldq xmm5, 4
1717		1717
1718		1718
1719	pxor xmm4, xmm4 ; pMad8x8	1719	pxor xmm4, xmm4 ; pMad8x8
1720	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1720	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1721	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1721	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1722	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1722	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1723	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1723	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1724	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1724	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1725	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1725	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1726	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1726	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1727	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4	1727	WELS_SAD_SD_MAD_16x1_SSE2 xmm7, xmm6, xmm5, xmm4 ,r15 ,r1, r4
1728		1728
1729	;mov r14, [p_mad8x8]	1729	;mov r14, [p_mad8x8]
1730	WELS_MAX_REG_SSE2 xmm4	1730	WELS_MAX_REG_SSE2 xmm4
1731		1731
1732	movhlps xmm1, xmm4	1732	movhlps xmm1, xmm4
1733	movd r0d, xmm4	1733	movd r0d, xmm4
1734	mov [r14], r0b	1734	mov [r14], r0b
1735	movd r0d, xmm1	1735	movd r0d, xmm1
1736	mov [r14+1],r0b	1736	mov [r14+1],r0b
1737	add r14, 2	1737	add r14, 2
1738	mov p_mad8x8, r14	1738	mov p_mad8x8, r14
1739		1739
1740	; data in xmm7, xmm6, xmm5: D1 D3 D0 D2	1740	; data in xmm7, xmm6, xmm5: D1 D3 D0 D2
1741		1741
1742	mov r14, psad8x8	1742	mov r14, psad8x8
1743	pshufd xmm1, xmm7, 10001101b ; D3 D2 D1 D0	1743	pshufd xmm1, xmm7, 10001101b ; D3 D2 D1 D0
1744	movdqa [r14], xmm1	1744	movdqa [r14], xmm1
1745	add r14, 16	1745	add r14, 16
1746	mov psad8x8, r14 ; sad8x8	1746	mov psad8x8, r14 ; sad8x8
1747		1747
1748	paddd xmm1, xmm7 ; D1+3 D3+2 D0+1 D2+0	1748	paddd xmm1, xmm7 ; D1+3 D3+2 D0+1 D2+0
1749	pshufd xmm2, xmm1, 00000011b	1749	pshufd xmm2, xmm1, 00000011b
1750	paddd xmm1, xmm2	1750	paddd xmm1, xmm2
1751	movd r14d, xmm1	1751	movd r14d, xmm1
1752	movd xmm9, r14d	1752	movd xmm9, r14d
1753	paddd xmm8, xmm9 ; sad frame	1753	paddd xmm8, xmm9 ; sad frame
1754		1754
1755	mov r14, p_sd8x8	1755	mov r14, p_sd8x8
1756	psubd xmm6, xmm5	1756	psubd xmm6, xmm5
1757	pshufd xmm1, xmm6, 10001101b	1757	pshufd xmm1, xmm6, 10001101b
1758	movdqa [r14], xmm1	1758	movdqa [r14], xmm1
1759	add r14, 16	1759	add r14, 16
1760	mov p_sd8x8, r14	1760	mov p_sd8x8, r14
1761		1761
1762		1762
1763	;add edx, 16	1763	;add edx, 16
1764	sub r15, r13	1764	sub r15, r13
1765	sub r1, r13	1765	sub r1, r13
1766	add r15, 16	1766	add r15, 16
1767	add r1, 16	1767	add r1, 16
1768		1768
1769		1769
1770	dec r2	1770	dec r2
1771	jnz bgd_width_loop	1771	jnz bgd_width_loop
1772	pop r2	1772	pop r2
1773	%assign push_num push_num-1	1773	%assign push_num push_num-1
1774	mov r15, r10	1774	mov r15, r10
1775	mov r1, r11	1775	mov r1, r11
1776	add r15, r13	1776	add r15, r13
1777	add r1, r13	1777	add r1, r13
1778		1778
1779	dec r3	1779	dec r3
1780	jnz bgd_height_loop	1780	jnz bgd_height_loop
1781		1781
1782	mov r13, psadframe	1782	mov r13, psadframe
1783	movd [r13], xmm8	1783	movd [r13], xmm8
1784		1784
1785	POP_XMM	1785	POP_XMM
1786	pop r15	1786	pop r15
1787	pop r14	1787	pop r14
1788	pop r13	1788	pop r13
1789	pop r12	1789	pop r12
1790	%assign push_num 0	1790	%assign push_num 0
1791	%undef cur_data	1791	%undef cur_data
1792	%undef ref_data	1792	%undef ref_data
@@ -1801,7 +1801,7 @@ bgd_width_loop: · 8 unmodified lines
1801	%undef tmp_edi	1801	%undef tmp_edi
1802	%undef pushsize	1802	%undef pushsize
1803	%undef localsize	1803	%undef localsize
1804	ret	1804	ret
1805		1805
1806		1806
1807		1807
@@ -1826,189 +1826,189 @@ WELS_EXTERN VAACalcSadSsdBgd_sse2 · 18 unmodified lines
1826	%define p_sd8x8 arg11	1826	%define p_sd8x8 arg11
1827	%define p_mad8x8 arg12	1827	%define p_mad8x8 arg12
1828		1828
1829	push r12	1829	push r12
1830	push r13	1830	push r13
1831	push r14	1831	push r14
1832	push r15	1832	push r15
1833	%assign push_num 4	1833	%assign push_num 4
1834	PUSH_XMM 10	1834	PUSH_XMM 10
1835	%ifdef WIN64	1835	%ifdef WIN64
1836	mov r4,arg5	1836	mov r4,arg5
1837	;mov r5,arg6	1837	;mov r5,arg6
1838	%endif	1838	%endif
1839	SIGN_EXTENSION r2,r2d	1839	SIGN_EXTENSION r2,r2d
1840	SIGN_EXTENSION r3,r3d	1840	SIGN_EXTENSION r3,r3d
1841	SIGN_EXTENSION r4,r4d	1841	SIGN_EXTENSION r4,r4d
1842		1842
1843	mov r13,r4	1843	mov r13,r4
1844	shr r2, 4 ; iPicWidth/16	1844	shr r2, 4 ; iPicWidth/16
1845	shr r3, 4 ; iPicHeight/16	1845	shr r3, 4 ; iPicHeight/16
1846	shl r13, 4 ; iPicStride*16	1846	shl r13, 4 ; iPicStride*16
1847	pxor xmm0, xmm0	1847	pxor xmm0, xmm0
1848	pxor xmm8, xmm8	1848	pxor xmm8, xmm8
1849	pxor xmm9, xmm9	1849	pxor xmm9, xmm9
1850		1850
1851		1851
1852	sqdiff_bgd_height_loop:	1852	sqdiff_bgd_height_loop:
1853	mov r10, r0	1853	mov r10, r0
1854	mov r11, r1	1854	mov r11, r1
1855	push r2	1855	push r2
1856	%assign push_num push_num+1	1856	%assign push_num push_num+1
1857	sqdiff_bgd_width_loop:	1857	sqdiff_bgd_width_loop:
1858		1858
1859	pxor xmm7, xmm7 ; pSad8x8 interleaves sqsum16x16: sqsum1 sad1 sqsum0 sad0	1859	pxor xmm7, xmm7 ; pSad8x8 interleaves sqsum16x16: sqsum1 sad1 sqsum0 sad0
1860	pxor xmm6, xmm6 ; sum_8x8 interleaves cur and pRef in Dword, Sref1 Scur1 Sref0 Scur0	1860	pxor xmm6, xmm6 ; sum_8x8 interleaves cur and pRef in Dword, Sref1 Scur1 Sref0 Scur0
1861	pxor xmm5, xmm5 ; pMad8x8	1861	pxor xmm5, xmm5 ; pMad8x8
1862	pxor xmm4, xmm4 ; sqdiff_16x16 four Dword	1862	pxor xmm4, xmm4 ; sqdiff_16x16 four Dword
1863	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1863	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1864	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1864	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1865	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1865	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1866	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1866	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1867	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1867	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1868	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1868	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1869	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1869	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1870	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1870	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1871		1871
1872	mov r14, psad8x8	1872	mov r14, psad8x8
1873	movdqa xmm2, xmm7	1873	movdqa xmm2, xmm7
1874	pshufd xmm1, xmm2, 00001110b	1874	pshufd xmm1, xmm2, 00001110b
1875	movd [r14], xmm2	1875	movd [r14], xmm2
1876	movd [r14+4], xmm1	1876	movd [r14+4], xmm1
1877	add r14, 8	1877	add r14, 8
1878	mov psad8x8, r14 ; sad8x8	1878	mov psad8x8, r14 ; sad8x8
1879		1879
1880	paddd xmm1, xmm2	1880	paddd xmm1, xmm2
1881	movd r14d, xmm1	1881	movd r14d, xmm1
1882	movd xmm9,r14d	1882	movd xmm9,r14d
1883	paddd xmm8, xmm9 ; iFrameSad	1883	paddd xmm8, xmm9 ; iFrameSad
1884		1884
1885	mov r14, psum16x16	1885	mov r14, psum16x16
1886	movdqa xmm1, xmm6	1886	movdqa xmm1, xmm6
1887	pshufd xmm2, xmm1, 00001110b	1887	pshufd xmm2, xmm1, 00001110b
1888	paddd xmm1, xmm2	1888	paddd xmm1, xmm2
1889	movd [r14], xmm1 ; sum	1889	movd [r14], xmm1 ; sum
1890		1890
1891	mov r14, p_sd8x8	1891	mov r14, p_sd8x8
1892	pshufd xmm1, xmm6, 11110101b ; Sref1 Sref1 Sref0 Sref0	1892	pshufd xmm1, xmm6, 11110101b ; Sref1 Sref1 Sref0 Sref0
1893	psubd xmm6, xmm1 ; 00 diff1 00 diff0	1893	psubd xmm6, xmm1 ; 00 diff1 00 diff0
1894	pshufd xmm1, xmm6, 00001000b ; xx xx diff1 diff0	1894	pshufd xmm1, xmm6, 00001000b ; xx xx diff1 diff0
1895	movq [r14], xmm1	1895	movq [r14], xmm1
1896	add r14, 8	1896	add r14, 8
1897	mov p_sd8x8, r14	1897	mov p_sd8x8, r14
1898		1898
1899	mov r14, p_mad8x8	1899	mov r14, p_mad8x8
1900	WELS_MAX_REG_SSE2 xmm5	1900	WELS_MAX_REG_SSE2 xmm5
1901		1901
1902	movhlps xmm1, xmm5	1902	movhlps xmm1, xmm5
1903	push r0	1903	push r0
1904	movd r0d, xmm5	1904	movd r0d, xmm5
1905	mov [r14], r0b	1905	mov [r14], r0b
1906	movd r0d, xmm1	1906	movd r0d, xmm1
1907	mov [r14+1],r0b	1907	mov [r14+1],r0b
1908	pop r0	1908	pop r0
1909	add r14, 2	1909	add r14, 2
1910	mov p_mad8x8, r14	1910	mov p_mad8x8, r14
1911		1911
1912	psrlq xmm7, 32	1912	psrlq xmm7, 32
1913	psllq xmm7, 32 ; clear sad	1913	psllq xmm7, 32 ; clear sad
1914	pxor xmm6, xmm6 ; sum_8x8 interleaves cur and pRef in Dword, Sref1 Scur1 Sref0 Scur0	1914	pxor xmm6, xmm6 ; sum_8x8 interleaves cur and pRef in Dword, Sref1 Scur1 Sref0 Scur0
1915	pxor xmm5, xmm5 ; pMad8x8	1915	pxor xmm5, xmm5 ; pMad8x8
1916	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1916	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1917	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1917	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1918	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1918	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1919	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1919	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1920	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1920	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1921	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1921	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1922	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1922	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1923	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4	1923	WELS_SAD_BGD_SQDIFF_16x1_SSE2 xmm7, xmm6, xmm5, xmm4, r0 , r1 , r4
1924		1924
1925	mov r14, psad8x8	1925	mov r14, psad8x8
1926	movdqa xmm2, xmm7	1926	movdqa xmm2, xmm7
1927	pshufd xmm1, xmm2, 00001110b	1927	pshufd xmm1, xmm2, 00001110b
1928	movd [r14], xmm2	1928	movd [r14], xmm2
1929	movd [r14+4], xmm1	1929	movd [r14+4], xmm1
1930	add r14, 8	1930	add r14, 8
1931	mov psad8x8, r14 ; sad8x8	1931	mov psad8x8, r14 ; sad8x8
1932		1932
1933	paddd xmm1, xmm2	1933	paddd xmm1, xmm2
1934	movd r14d, xmm1	1934	movd r14d, xmm1
1935	movd xmm9, r14d	1935	movd xmm9, r14d
1936	paddd xmm8, xmm9 ; iFrameSad	1936	paddd xmm8, xmm9 ; iFrameSad
1937		1937
1938	mov r14, psum16x16	1938	mov r14, psum16x16
1939	movdqa xmm1, xmm6	1939	movdqa xmm1, xmm6
1940	pshufd xmm2, xmm1, 00001110b	1940	pshufd xmm2, xmm1, 00001110b
1941	paddd xmm1, xmm2	1941	paddd xmm1, xmm2
1942	movd r15d, xmm1 ; sum	1942	movd r15d, xmm1 ; sum
1943	add [r14], r15d	1943	add [r14], r15d
1944	add r14, 4	1944	add r14, 4
1945	mov psum16x16, r14	1945	mov psum16x16, r14
1946		1946
1947	mov r14, psqsum16x16	1947	mov r14, psqsum16x16
1948	psrlq xmm7, 32	1948	psrlq xmm7, 32
1949	pshufd xmm2, xmm7, 00001110b	1949	pshufd xmm2, xmm7, 00001110b
1950	paddd xmm2, xmm7	1950	paddd xmm2, xmm7
1951	movd [r14], xmm2 ; sqsum	1951	movd [r14], xmm2 ; sqsum
1952	add r14, 4	1952	add r14, 4
1953	mov psqsum16x16, r14	1953	mov psqsum16x16, r14
1954		1954
1955	mov r14, p_sd8x8	1955	mov r14, p_sd8x8
1956	pshufd xmm1, xmm6, 11110101b ; Sref1 Sref1 Sref0 Sref0	1956	pshufd xmm1, xmm6, 11110101b ; Sref1 Sref1 Sref0 Sref0
1957	psubd xmm6, xmm1 ; 00 diff1 00 diff0	1957	psubd xmm6, xmm1 ; 00 diff1 00 diff0
1958	pshufd xmm1, xmm6, 00001000b ; xx xx diff1 diff0	1958	pshufd xmm1, xmm6, 00001000b ; xx xx diff1 diff0
1959	movq [r14], xmm1	1959	movq [r14], xmm1
1960	add r14, 8	1960	add r14, 8
1961	mov p_sd8x8, r14	1961	mov p_sd8x8, r14
1962		1962
1963	mov r14, p_mad8x8	1963	mov r14, p_mad8x8
1964	WELS_MAX_REG_SSE2 xmm5	1964	WELS_MAX_REG_SSE2 xmm5
1965		1965
1966		1966
1967	movhlps xmm1, xmm5	1967	movhlps xmm1, xmm5
1968	push r0	1968	push r0
1969	movd r0d, xmm5	1969	movd r0d, xmm5
1970	mov [r14], r0b	1970	mov [r14], r0b
1971	movd r0d, xmm1	1971	movd r0d, xmm1
1972	mov [r14+1],r0b	1972	mov [r14+1],r0b
1973	pop r0	1973	pop r0
1974	add r14, 2	1974	add r14, 2
1975	mov p_mad8x8, r14	1975	mov p_mad8x8, r14
1976		1976
1977	mov r14, psqdiff16x16	1977	mov r14, psqdiff16x16
1978	pshufd xmm1, xmm4, 00001110b	1978	pshufd xmm1, xmm4, 00001110b
1979	paddd xmm4, xmm1	1979	paddd xmm4, xmm1
1980	pshufd xmm1, xmm4, 00000001b	1980	pshufd xmm1, xmm4, 00000001b
1981	paddd xmm4, xmm1	1981	paddd xmm4, xmm1
1982	movd [r14], xmm4	1982	movd [r14], xmm4
1983	add r14, 4	1983	add r14, 4
1984	mov psqdiff16x16, r14	1984	mov psqdiff16x16, r14
1985		1985
1986	add r14, 16	1986	add r14, 16
1987	sub r0, r13	1987	sub r0, r13
1988	sub r1, r13	1988	sub r1, r13
1989	add r0, 16	1989	add r0, 16
1990	add r1, 16	1990	add r1, 16
1991		1991
1992	dec r2	1992	dec r2
1993	jnz sqdiff_bgd_width_loop	1993	jnz sqdiff_bgd_width_loop
1994	pop r2	1994	pop r2
1995	%assign push_num push_num-1	1995	%assign push_num push_num-1
1996	mov r0, r10	1996	mov r0, r10
1997	mov r1, r11	1997	mov r1, r11
1998	add r0, r13	1998	add r0, r13
1999	add r1, r13	1999	add r1, r13
2000		2000
2001	dec r3	2001	dec r3
2002	jnz sqdiff_bgd_height_loop	2002	jnz sqdiff_bgd_height_loop
2003		2003
2004	mov r14, psadframe	2004	mov r14, psadframe
2005	movd [r14], xmm8	2005	movd [r14], xmm8
2006		2006
2007	POP_XMM	2007	POP_XMM
2008	pop r15	2008	pop r15
2009	pop r14	2009	pop r14
2010	pop r13	2010	pop r13
2011	pop r12	2011	pop r12
2012	%assign push_num 0	2012	%assign push_num 0
2013	%undef cur_data	2013	%undef cur_data
2014	%undef ref_data	2014	%undef ref_data
@@ -2026,5 +2026,5 @@ sqdiff_bgd_width_loop: · 11 unmodified lines
2026	%undef tmp_edi	2026	%undef tmp_edi
2027	%undef pushsize	2027	%undef pushsize
2028	%undef localsize	2028	%undef localsize
2029	ret	2029	ret
2030	%endif	2030	%endif

Convert all tabs to spaces in assembly sources, unify indentation

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code