History

aarch64: Reindent all assembly to 8/24 column indentation

libavcodec/aarch64/vc1dsp_neon.S is skipped here, as it intentionally uses a layered indentation style to visually show how different unrolled/interleaved phases fit together.

Martin Storsjö committed 3 years ago

Commit a76b4091 parent cada4596 files · +1600 −1600Browse files

Unified · Split

6 files
  1. libavcodec/aarch64/aacpsdsp_neon.S +109 −109
  2. libavcodec/aarch64/opusdsp_neon.S +51 −51
  3. libswresample/aarch64/resample.S +40 −40
  4. libswscale/aarch64/hscale.S +1125 −1125
  5. libswscale/aarch64/output.S +165 −165
  6. libswscale/aarch64/yuv2rgb_neon.S +110 −110
libavcodec/aarch64/aacpsdsp_neon.S+109−109
@@ -19,130 +19,130 @@ · 18 unmodified lines
1919 #include "libavutil/aarch64/asm.S"
2020
2121 function ff_ps_add_squares_neon, export=1
22−1: ld1 {v0.4s,v1.4s}, [x1], #32
23− fmul v0.4s, v0.4s, v0.4s
24− fmul v1.4s, v1.4s, v1.4s
25− faddp v2.4s, v0.4s, v1.4s
26− ld1 {v3.4s}, [x0]
27− fadd v3.4s, v3.4s, v2.4s
28− st1 {v3.4s}, [x0], #16
29− subs w2, w2, #4
30− b.gt 1b
22+1: ld1 {v0.4s,v1.4s}, [x1], #32
23+ fmul v0.4s, v0.4s, v0.4s
24+ fmul v1.4s, v1.4s, v1.4s
25+ faddp v2.4s, v0.4s, v1.4s
26+ ld1 {v3.4s}, [x0]
27+ fadd v3.4s, v3.4s, v2.4s
28+ st1 {v3.4s}, [x0], #16
29+ subs w2, w2, #4
30+ b.gt 1b
3131 ret
3232 endfunc
3333
3434 function ff_ps_mul_pair_single_neon, export=1
35−1: ld1 {v0.4s,v1.4s}, [x1], #32
36− ld1 {v2.4s}, [x2], #16
37− zip1 v3.4s, v2.4s, v2.4s
38− zip2 v4.4s, v2.4s, v2.4s
39− fmul v0.4s, v0.4s, v3.4s
40− fmul v1.4s, v1.4s, v4.4s
41− st1 {v0.4s,v1.4s}, [x0], #32
42− subs w3, w3, #4
43− b.gt 1b
35+1: ld1 {v0.4s,v1.4s}, [x1], #32
36+ ld1 {v2.4s}, [x2], #16
37+ zip1 v3.4s, v2.4s, v2.4s
38+ zip2 v4.4s, v2.4s, v2.4s
39+ fmul v0.4s, v0.4s, v3.4s
40+ fmul v1.4s, v1.4s, v4.4s
41+ st1 {v0.4s,v1.4s}, [x0], #32
42+ subs w3, w3, #4
43+ b.gt 1b
4444 ret
4545 endfunc
4646
4747 function ff_ps_stereo_interpolate_neon, export=1
48− ld1 {v0.4s}, [x2]
49− ld1 {v1.4s}, [x3]
50− zip1 v4.4s, v0.4s, v0.4s
51− zip2 v5.4s, v0.4s, v0.4s
52− zip1 v6.4s, v1.4s, v1.4s
53− zip2 v7.4s, v1.4s, v1.4s
54−1: ld1 {v2.2s}, [x0]
55− ld1 {v3.2s}, [x1]
56− fadd v4.4s, v4.4s, v6.4s
57− fadd v5.4s, v5.4s, v7.4s
58− mov v2.d[1], v2.d[0]
59− mov v3.d[1], v3.d[0]
60− fmul v2.4s, v2.4s, v4.4s
61− fmla v2.4s, v3.4s, v5.4s
62− st1 {v2.d}[0], [x0], #8
63− st1 {v2.d}[1], [x1], #8
64− subs w4, w4, #1
65− b.gt 1b
48+ ld1 {v0.4s}, [x2]
49+ ld1 {v1.4s}, [x3]
50+ zip1 v4.4s, v0.4s, v0.4s
51+ zip2 v5.4s, v0.4s, v0.4s
52+ zip1 v6.4s, v1.4s, v1.4s
53+ zip2 v7.4s, v1.4s, v1.4s
54+1: ld1 {v2.2s}, [x0]
55+ ld1 {v3.2s}, [x1]
56+ fadd v4.4s, v4.4s, v6.4s
57+ fadd v5.4s, v5.4s, v7.4s
58+ mov v2.d[1], v2.d[0]
59+ mov v3.d[1], v3.d[0]
60+ fmul v2.4s, v2.4s, v4.4s
61+ fmla v2.4s, v3.4s, v5.4s
62+ st1 {v2.d}[0], [x0], #8
63+ st1 {v2.d}[1], [x1], #8
64+ subs w4, w4, #1
65+ b.gt 1b
6666 ret
6767 endfunc
6868
6969 function ff_ps_stereo_interpolate_ipdopd_neon, export=1
70− ld1 {v0.4s,v1.4s}, [x2]
71− ld1 {v6.4s,v7.4s}, [x3]
72− fneg v2.4s, v1.4s
73− fneg v3.4s, v7.4s
74− zip1 v16.4s, v0.4s, v0.4s
75− zip2 v17.4s, v0.4s, v0.4s
76− zip1 v18.4s, v2.4s, v1.4s
77− zip2 v19.4s, v2.4s, v1.4s
78− zip1 v20.4s, v6.4s, v6.4s
79− zip2 v21.4s, v6.4s, v6.4s
80− zip1 v22.4s, v3.4s, v7.4s
81− zip2 v23.4s, v3.4s, v7.4s
82−1: ld1 {v2.2s}, [x0]
83− ld1 {v3.2s}, [x1]
84− fadd v16.4s, v16.4s, v20.4s
85− fadd v17.4s, v17.4s, v21.4s
86− mov v2.d[1], v2.d[0]
87− mov v3.d[1], v3.d[0]
88− fmul v4.4s, v2.4s, v16.4s
89− fmla v4.4s, v3.4s, v17.4s
90− fadd v18.4s, v18.4s, v22.4s
91− fadd v19.4s, v19.4s, v23.4s
92− ext v2.16b, v2.16b, v2.16b, #4
93− ext v3.16b, v3.16b, v3.16b, #4
94− fmla v4.4s, v2.4s, v18.4s
95− fmla v4.4s, v3.4s, v19.4s
96− st1 {v4.d}[0], [x0], #8
97− st1 {v4.d}[1], [x1], #8
98− subs w4, w4, #1
99− b.gt 1b
70+ ld1 {v0.4s,v1.4s}, [x2]
71+ ld1 {v6.4s,v7.4s}, [x3]
72+ fneg v2.4s, v1.4s
73+ fneg v3.4s, v7.4s
74+ zip1 v16.4s, v0.4s, v0.4s
75+ zip2 v17.4s, v0.4s, v0.4s
76+ zip1 v18.4s, v2.4s, v1.4s
77+ zip2 v19.4s, v2.4s, v1.4s
78+ zip1 v20.4s, v6.4s, v6.4s
79+ zip2 v21.4s, v6.4s, v6.4s
80+ zip1 v22.4s, v3.4s, v7.4s
81+ zip2 v23.4s, v3.4s, v7.4s
82+1: ld1 {v2.2s}, [x0]
83+ ld1 {v3.2s}, [x1]
84+ fadd v16.4s, v16.4s, v20.4s
85+ fadd v17.4s, v17.4s, v21.4s
86+ mov v2.d[1], v2.d[0]
87+ mov v3.d[1], v3.d[0]
88+ fmul v4.4s, v2.4s, v16.4s
89+ fmla v4.4s, v3.4s, v17.4s
90+ fadd v18.4s, v18.4s, v22.4s
91+ fadd v19.4s, v19.4s, v23.4s
92+ ext v2.16b, v2.16b, v2.16b, #4
93+ ext v3.16b, v3.16b, v3.16b, #4
94+ fmla v4.4s, v2.4s, v18.4s
95+ fmla v4.4s, v3.4s, v19.4s
96+ st1 {v4.d}[0], [x0], #8
97+ st1 {v4.d}[1], [x1], #8
98+ subs w4, w4, #1
99+ b.gt 1b
100100 ret
101101 endfunc
102102
103103 function ff_ps_hybrid_analysis_neon, export=1
104− lsl x3, x3, #3
105− ld2 {v0.4s,v1.4s}, [x1], #32
106− ld2 {v2.2s,v3.2s}, [x1], #16
107− ld1 {v24.2s}, [x1], #8
108− ld2 {v4.2s,v5.2s}, [x1], #16
109− ld2 {v6.4s,v7.4s}, [x1]
110− rev64 v6.4s, v6.4s
111− rev64 v7.4s, v7.4s
112− ext v6.16b, v6.16b, v6.16b, #8
113− ext v7.16b, v7.16b, v7.16b, #8
114− rev64 v4.2s, v4.2s
115− rev64 v5.2s, v5.2s
116− mov v2.d[1], v3.d[0]
117− mov v4.d[1], v5.d[0]
118− mov v5.d[1], v2.d[0]
119− mov v3.d[1], v4.d[0]
120− fadd v16.4s, v0.4s, v6.4s
121− fadd v17.4s, v1.4s, v7.4s
122− fsub v18.4s, v1.4s, v7.4s
123− fsub v19.4s, v0.4s, v6.4s
124− fadd v22.4s, v2.4s, v4.4s
125− fsub v23.4s, v5.4s, v3.4s
126− trn1 v20.2d, v22.2d, v23.2d // {re4+re8, re5+re7, im8-im4, im7-im5}
127− trn2 v21.2d, v22.2d, v23.2d // {im4+im8, im5+im7, re4-re8, re5-re7}
128−1: ld2 {v2.4s,v3.4s}, [x2], #32
129− ld2 {v4.2s,v5.2s}, [x2], #16
130− ld1 {v6.2s}, [x2], #8
131− add x2, x2, #8
132− mov v4.d[1], v5.d[0]
133− mov v6.s[1], v6.s[0]
134− fmul v6.2s, v6.2s, v24.2s
135− fmul v0.4s, v2.4s, v16.4s
136− fmul v1.4s, v2.4s, v17.4s
137− fmls v0.4s, v3.4s, v18.4s
138− fmla v1.4s, v3.4s, v19.4s
139− fmla v0.4s, v4.4s, v20.4s
140− fmla v1.4s, v4.4s, v21.4s
141− faddp v0.4s, v0.4s, v1.4s
142− faddp v0.4s, v0.4s, v0.4s
143− fadd v0.2s, v0.2s, v6.2s
144− st1 {v0.2s}, [x0], x3
145− subs w4, w4, #1
146− b.gt 1b
104+ lsl x3, x3, #3
105+ ld2 {v0.4s,v1.4s}, [x1], #32
106+ ld2 {v2.2s,v3.2s}, [x1], #16
107+ ld1 {v24.2s}, [x1], #8
108+ ld2 {v4.2s,v5.2s}, [x1], #16
109+ ld2 {v6.4s,v7.4s}, [x1]
110+ rev64 v6.4s, v6.4s
111+ rev64 v7.4s, v7.4s
112+ ext v6.16b, v6.16b, v6.16b, #8
113+ ext v7.16b, v7.16b, v7.16b, #8
114+ rev64 v4.2s, v4.2s
115+ rev64 v5.2s, v5.2s
116+ mov v2.d[1], v3.d[0]
117+ mov v4.d[1], v5.d[0]
118+ mov v5.d[1], v2.d[0]
119+ mov v3.d[1], v4.d[0]
120+ fadd v16.4s, v0.4s, v6.4s
121+ fadd v17.4s, v1.4s, v7.4s
122+ fsub v18.4s, v1.4s, v7.4s
123+ fsub v19.4s, v0.4s, v6.4s
124+ fadd v22.4s, v2.4s, v4.4s
125+ fsub v23.4s, v5.4s, v3.4s
126+ trn1 v20.2d, v22.2d, v23.2d // {re4+re8, re5+re7, im8-im4, im7-im5}
127+ trn2 v21.2d, v22.2d, v23.2d // {im4+im8, im5+im7, re4-re8, re5-re7}
128+1: ld2 {v2.4s,v3.4s}, [x2], #32
129+ ld2 {v4.2s,v5.2s}, [x2], #16
130+ ld1 {v6.2s}, [x2], #8
131+ add x2, x2, #8
132+ mov v4.d[1], v5.d[0]
133+ mov v6.s[1], v6.s[0]
134+ fmul v6.2s, v6.2s, v24.2s
135+ fmul v0.4s, v2.4s, v16.4s
136+ fmul v1.4s, v2.4s, v17.4s
137+ fmls v0.4s, v3.4s, v18.4s
138+ fmla v1.4s, v3.4s, v19.4s
139+ fmla v0.4s, v4.4s, v20.4s
140+ fmla v1.4s, v4.4s, v21.4s
141+ faddp v0.4s, v0.4s, v1.4s
142+ faddp v0.4s, v0.4s, v0.4s
143+ fadd v0.2s, v0.2s, v6.2s
144+ st1 {v0.2s}, [x0], x3
145+ subs w4, w4, #1
146+ b.gt 1b
147147 ret
148148 endfunc
libavcodec/aarch64/opusdsp_neon.S+51−51
@@ -33,81 +33,81 @@ const tab_x2, align=4 · 32 unmodified lines
3333 endconst
3434
3535 function ff_opus_deemphasis_neon, export=1
36− movrel x4, tab_st
37− ld1 {v4.4s}, [x4]
38− movrel x4, tab_x0
39− ld1 {v5.4s}, [x4]
40− movrel x4, tab_x1
41− ld1 {v6.4s}, [x4]
42− movrel x4, tab_x2
43− ld1 {v7.4s}, [x4]
36+ movrel x4, tab_st
37+ ld1 {v4.4s}, [x4]
38+ movrel x4, tab_x0
39+ ld1 {v5.4s}, [x4]
40+ movrel x4, tab_x1
41+ ld1 {v6.4s}, [x4]
42+ movrel x4, tab_x2
43+ ld1 {v7.4s}, [x4]
4444
45− fmul v0.4s, v4.4s, v0.s[0]
45+ fmul v0.4s, v4.4s, v0.s[0]
4646
47−1: ld1 {v1.4s, v2.4s}, [x1], #32
47+1: ld1 {v1.4s, v2.4s}, [x1], #32
4848
49− fmla v0.4s, v5.4s, v1.s[0]
50− fmul v3.4s, v7.4s, v2.s[2]
49+ fmla v0.4s, v5.4s, v1.s[0]
50+ fmul v3.4s, v7.4s, v2.s[2]
5151
52− fmla v0.4s, v6.4s, v1.s[1]
53− fmla v3.4s, v6.4s, v2.s[1]
52+ fmla v0.4s, v6.4s, v1.s[1]
53+ fmla v3.4s, v6.4s, v2.s[1]
5454
55− fmla v0.4s, v7.4s, v1.s[2]
56− fmla v3.4s, v5.4s, v2.s[0]
55+ fmla v0.4s, v7.4s, v1.s[2]
56+ fmla v3.4s, v5.4s, v2.s[0]
5757
58− fadd v1.4s, v1.4s, v0.4s
59− fadd v2.4s, v2.4s, v3.4s
58+ fadd v1.4s, v1.4s, v0.4s
59+ fadd v2.4s, v2.4s, v3.4s
6060
61− fmla v2.4s, v4.4s, v1.s[3]
61+ fmla v2.4s, v4.4s, v1.s[3]
6262
63− st1 {v1.4s, v2.4s}, [x0], #32
64− fmul v0.4s, v4.4s, v2.s[3]
63+ st1 {v1.4s, v2.4s}, [x0], #32
64+ fmul v0.4s, v4.4s, v2.s[3]
6565
66− subs w2, w2, #8
67− b.gt 1b
66+ subs w2, w2, #8
67+ b.gt 1b
6868
69− mov s0, v2.s[3]
69+ mov s0, v2.s[3]
7070
7171 ret
7272 endfunc
7373
7474 function ff_opus_postfilter_neon, export=1
75− ld1 {v0.4s}, [x2]
76− dup v1.4s, v0.s[1]
77− dup v2.4s, v0.s[2]
78− dup v0.4s, v0.s[0]
75+ ld1 {v0.4s}, [x2]
76+ dup v1.4s, v0.s[1]
77+ dup v2.4s, v0.s[2]
78+ dup v0.4s, v0.s[0]
7979
80− add w1, w1, #2
81− sub x1, x0, x1, lsl #2
80+ add w1, w1, #2
81+ sub x1, x0, x1, lsl #2
8282
83− ld1 {v3.4s}, [x1]
84− fmul v3.4s, v3.4s, v2.4s
83+ ld1 {v3.4s}, [x1]
84+ fmul v3.4s, v3.4s, v2.4s
8585
86−1: add x1, x1, #4
87− ld1 {v4.4s}, [x1]
88− add x1, x1, #4
89− ld1 {v5.4s}, [x1]
90− add x1, x1, #4
91− ld1 {v6.4s}, [x1]
92− add x1, x1, #4
93− ld1 {v7.4s}, [x1]
86+1: add x1, x1, #4
87+ ld1 {v4.4s}, [x1]
88+ add x1, x1, #4
89+ ld1 {v5.4s}, [x1]
90+ add x1, x1, #4
91+ ld1 {v6.4s}, [x1]
92+ add x1, x1, #4
93+ ld1 {v7.4s}, [x1]
9494
95− fmla v3.4s, v7.4s, v2.4s
96− fadd v6.4s, v6.4s, v4.4s
95+ fmla v3.4s, v7.4s, v2.4s
96+ fadd v6.4s, v6.4s, v4.4s
9797
98− ld1 {v4.4s}, [x0]
99− fmla v4.4s, v5.4s, v0.4s
98+ ld1 {v4.4s}, [x0]
99+ fmla v4.4s, v5.4s, v0.4s
100100
101− fmul v6.4s, v6.4s, v1.4s
102− fadd v6.4s, v6.4s, v3.4s
101+ fmul v6.4s, v6.4s, v1.4s
102+ fadd v6.4s, v6.4s, v3.4s
103103
104− fadd v4.4s, v4.4s, v6.4s
105− fmul v3.4s, v7.4s, v2.4s
104+ fadd v4.4s, v4.4s, v6.4s
105+ fmul v3.4s, v7.4s, v2.4s
106106
107− st1 {v4.4s}, [x0], #16
107+ st1 {v4.4s}, [x0], #16
108108
109− subs w3, w3, #4
110− b.gt 1b
109+ subs w3, w3, #4
110+ b.gt 1b
111111
112112 ret
113113 endfunc
libswresample/aarch64/resample.S+40−40
@@ -21,57 +21,57 @@ · 20 unmodified lines
2121 #include "libavutil/aarch64/asm.S"
2222
2323 function ff_resample_common_apply_filter_x4_float_neon, export=1
24− movi v0.4s, #0 // accumulator
25−1: ld1 {v1.4s}, [x1], #16 // src[0..3]
26− ld1 {v2.4s}, [x2], #16 // filter[0..3]
27− fmla v0.4s, v1.4s, v2.4s // accumulator += src[0..3] * filter[0..3]
28− subs w3, w3, #4 // filter_length -= 4
29− b.gt 1b // loop until filter_length
30− faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
31− faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
32− st1 {v0.s}[0], [x0], #4 // write accumulator
24+ movi v0.4s, #0 // accumulator
25+1: ld1 {v1.4s}, [x1], #16 // src[0..3]
26+ ld1 {v2.4s}, [x2], #16 // filter[0..3]
27+ fmla v0.4s, v1.4s, v2.4s // accumulator += src[0..3] * filter[0..3]
28+ subs w3, w3, #4 // filter_length -= 4
29+ b.gt 1b // loop until filter_length
30+ faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
31+ faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
32+ st1 {v0.s}[0], [x0], #4 // write accumulator
3333 ret
3434 endfunc
3535
3636 function ff_resample_common_apply_filter_x8_float_neon, export=1
37− movi v0.4s, #0 // accumulator
38−1: ld1 {v1.4s}, [x1], #16 // src[0..3]
39− ld1 {v2.4s}, [x2], #16 // filter[0..3]
40− ld1 {v3.4s}, [x1], #16 // src[4..7]
41− ld1 {v4.4s}, [x2], #16 // filter[4..7]
42− fmla v0.4s, v1.4s, v2.4s // accumulator += src[0..3] * filter[0..3]
43− fmla v0.4s, v3.4s, v4.4s // accumulator += src[4..7] * filter[4..7]
44− subs w3, w3, #8 // filter_length -= 8
45− b.gt 1b // loop until filter_length
46− faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
47− faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
48− st1 {v0.s}[0], [x0], #4 // write accumulator
37+ movi v0.4s, #0 // accumulator
38+1: ld1 {v1.4s}, [x1], #16 // src[0..3]
39+ ld1 {v2.4s}, [x2], #16 // filter[0..3]
40+ ld1 {v3.4s}, [x1], #16 // src[4..7]
41+ ld1 {v4.4s}, [x2], #16 // filter[4..7]
42+ fmla v0.4s, v1.4s, v2.4s // accumulator += src[0..3] * filter[0..3]
43+ fmla v0.4s, v3.4s, v4.4s // accumulator += src[4..7] * filter[4..7]
44+ subs w3, w3, #8 // filter_length -= 8
45+ b.gt 1b // loop until filter_length
46+ faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
47+ faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
48+ st1 {v0.s}[0], [x0], #4 // write accumulator
4949 ret
5050 endfunc
5151
5252 function ff_resample_common_apply_filter_x4_s16_neon, export=1
53− movi v0.4s, #0 // accumulator
54−1: ld1 {v1.4h}, [x1], #8 // src[0..3]
55− ld1 {v2.4h}, [x2], #8 // filter[0..3]
56− smlal v0.4s, v1.4h, v2.4h // accumulator += src[0..3] * filter[0..3]
57− subs w3, w3, #4 // filter_length -= 4
58− b.gt 1b // loop until filter_length
59− addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
60− addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
61− st1 {v0.s}[0], [x0], #4 // write accumulator
53+ movi v0.4s, #0 // accumulator
54+1: ld1 {v1.4h}, [x1], #8 // src[0..3]
55+ ld1 {v2.4h}, [x2], #8 // filter[0..3]
56+ smlal v0.4s, v1.4h, v2.4h // accumulator += src[0..3] * filter[0..3]
57+ subs w3, w3, #4 // filter_length -= 4
58+ b.gt 1b // loop until filter_length
59+ addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
60+ addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
61+ st1 {v0.s}[0], [x0], #4 // write accumulator
6262 ret
6363 endfunc
6464
6565 function ff_resample_common_apply_filter_x8_s16_neon, export=1
66− movi v0.4s, #0 // accumulator
67−1: ld1 {v1.8h}, [x1], #16 // src[0..7]
68− ld1 {v2.8h}, [x2], #16 // filter[0..7]
69− smlal v0.4s, v1.4h, v2.4h // accumulator += src[0..3] * filter[0..3]
70− smlal2 v0.4s, v1.8h, v2.8h // accumulator += src[4..7] * filter[4..7]
71− subs w3, w3, #8 // filter_length -= 8
72− b.gt 1b // loop until filter_length
73− addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
74− addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
75− st1 {v0.s}[0], [x0], #4 // write accumulator
66+ movi v0.4s, #0 // accumulator
67+1: ld1 {v1.8h}, [x1], #16 // src[0..7]
68+ ld1 {v2.8h}, [x2], #16 // filter[0..7]
69+ smlal v0.4s, v1.4h, v2.4h // accumulator += src[0..3] * filter[0..3]
70+ smlal2 v0.4s, v1.8h, v2.8h // accumulator += src[4..7] * filter[4..7]
71+ subs w3, w3, #8 // filter_length -= 8
72+ b.gt 1b // loop until filter_length
73+ addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
74+ addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
75+ st1 {v0.s}[0], [x0], #4 // write accumulator
7676 ret
7777 endfunc
libswscale/aarch64/hscale.S+1125−1125
@@ -41,53 +41,53 @@ · 40 unmodified lines
4141 ;----------------------------------------------------------------------------- */
4242
4343 function ff_hscale8to15_X8_neon, export=1
44− sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)
45−1: ldr w8, [x5], #4 // filterPos[idx]
46− ldr w0, [x5], #4 // filterPos[idx + 1]
47− ldr w11, [x5], #4 // filterPos[idx + 2]
48− ldr w9, [x5], #4 // filterPos[idx + 3]
49− mov x16, x4 // filter0 = filter
50− add x12, x16, x7 // filter1 = filter0 + filterSize*2
51− add x13, x12, x7 // filter2 = filter1 + filterSize*2
52− add x4, x13, x7 // filter3 = filter2 + filterSize*2
53− movi v0.2d, #0 // val sum part 1 (for dst[0])
54− movi v1.2d, #0 // val sum part 2 (for dst[1])
55− movi v2.2d, #0 // val sum part 3 (for dst[2])
56− movi v3.2d, #0 // val sum part 4 (for dst[3])
57− add x17, x3, w8, uxtw // srcp + filterPos[0]
58− add x8, x3, w0, uxtw // srcp + filterPos[1]
59− add x0, x3, w11, uxtw // srcp + filterPos[2]
60− add x11, x3, w9, uxtw // srcp + filterPos[3]
61− mov w15, w6 // filterSize counter
62−2: ld1 {v4.8b}, [x17], #8 // srcp[filterPos[0] + {0..7}]
63− ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1
64− ld1 {v6.8b}, [x8], #8 // srcp[filterPos[1] + {0..7}]
65− ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize
66− uxtl v4.8h, v4.8b // unpack part 1 to 16-bit
67− smlal v0.4s, v4.4h, v5.4h // v0 accumulates srcp[filterPos[0] + {0..3}] * filter[{0..3}]
68− smlal2 v0.4s, v4.8h, v5.8h // v0 accumulates srcp[filterPos[0] + {4..7}] * filter[{4..7}]
69− ld1 {v16.8b}, [x0], #8 // srcp[filterPos[2] + {0..7}]
70− ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize
71− uxtl v6.8h, v6.8b // unpack part 2 to 16-bit
72− smlal v1.4s, v6.4h, v7.4h // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]
73− uxtl v16.8h, v16.8b // unpack part 3 to 16-bit
74− smlal v2.4s, v16.4h, v17.4h // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]
75− smlal2 v2.4s, v16.8h, v17.8h // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]
76− ld1 {v18.8b}, [x11], #8 // srcp[filterPos[3] + {0..7}]
77− smlal2 v1.4s, v6.8h, v7.8h // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]
78− ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize
79− subs w15, w15, #8 // j -= 8: processed 8/filterSize
80− uxtl v18.8h, v18.8b // unpack part 4 to 16-bit
81− smlal v3.4s, v18.4h, v19.4h // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]
82− smlal2 v3.4s, v18.8h, v19.8h // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]
83− b.gt 2b // inner loop if filterSize not consumed completely
84− addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding
85− addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding
86− addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding
87− subs w2, w2, #4 // dstW -= 4
88− sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values
89− st1 {v0.4h}, [x1], #8 // write to destination part0123
90− b.gt 1b // loop until end of line
44+ sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)
45+1: ldr w8, [x5], #4 // filterPos[idx]
46+ ldr w0, [x5], #4 // filterPos[idx + 1]
47+ ldr w11, [x5], #4 // filterPos[idx + 2]
48+ ldr w9, [x5], #4 // filterPos[idx + 3]
49+ mov x16, x4 // filter0 = filter
50+ add x12, x16, x7 // filter1 = filter0 + filterSize*2
51+ add x13, x12, x7 // filter2 = filter1 + filterSize*2
52+ add x4, x13, x7 // filter3 = filter2 + filterSize*2
53+ movi v0.2d, #0 // val sum part 1 (for dst[0])
54+ movi v1.2d, #0 // val sum part 2 (for dst[1])
55+ movi v2.2d, #0 // val sum part 3 (for dst[2])
56+ movi v3.2d, #0 // val sum part 4 (for dst[3])
57+ add x17, x3, w8, uxtw // srcp + filterPos[0]
58+ add x8, x3, w0, uxtw // srcp + filterPos[1]
59+ add x0, x3, w11, uxtw // srcp + filterPos[2]
60+ add x11, x3, w9, uxtw // srcp + filterPos[3]
61+ mov w15, w6 // filterSize counter
62+2: ld1 {v4.8b}, [x17], #8 // srcp[filterPos[0] + {0..7}]
63+ ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1
64+ ld1 {v6.8b}, [x8], #8 // srcp[filterPos[1] + {0..7}]
65+ ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize
66+ uxtl v4.8h, v4.8b // unpack part 1 to 16-bit
67+ smlal v0.4s, v4.4h, v5.4h // v0 accumulates srcp[filterPos[0] + {0..3}] * filter[{0..3}]
68+ smlal2 v0.4s, v4.8h, v5.8h // v0 accumulates srcp[filterPos[0] + {4..7}] * filter[{4..7}]
69+ ld1 {v16.8b}, [x0], #8 // srcp[filterPos[2] + {0..7}]
70+ ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize
71+ uxtl v6.8h, v6.8b // unpack part 2 to 16-bit
72+ smlal v1.4s, v6.4h, v7.4h // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]
73+ uxtl v16.8h, v16.8b // unpack part 3 to 16-bit
74+ smlal v2.4s, v16.4h, v17.4h // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]
75+ smlal2 v2.4s, v16.8h, v17.8h // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]
76+ ld1 {v18.8b}, [x11], #8 // srcp[filterPos[3] + {0..7}]
77+ smlal2 v1.4s, v6.8h, v7.8h // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]
78+ ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize
79+ subs w15, w15, #8 // j -= 8: processed 8/filterSize
80+ uxtl v18.8h, v18.8b // unpack part 4 to 16-bit
81+ smlal v3.4s, v18.4h, v19.4h // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]
82+ smlal2 v3.4s, v18.8h, v19.8h // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]
83+ b.gt 2b // inner loop if filterSize not consumed completely
84+ addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding
85+ addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding
86+ addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding
87+ subs w2, w2, #4 // dstW -= 4
88+ sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values
89+ st1 {v0.4h}, [x1], #8 // write to destination part0123
90+ b.gt 1b // loop until end of line
9191 ret
9292 endfunc
9393
@@ -103,98 +103,98 @@ function ff_hscale8to15_X4_neon, export=1 · 9 unmodified lines
103103 // This function for filter sizes that are 4 mod 8. In other words, anything that's 0 mod 4 but not
104104 // 0 mod 8. It also assumes that dstW is 0 mod 4.
105105
106− lsl w7, w6, #1 // w7 = filterSize * 2
106+ lsl w7, w6, #1 // w7 = filterSize * 2
107107 1:
108− ldp w8, w9, [x5] // filterPos[idx + 0], [idx + 1]
109− ldp w10, w11, [x5, #8] // filterPos[idx + 2], [idx + 3]
108+ ldp w8, w9, [x5] // filterPos[idx + 0], [idx + 1]
109+ ldp w10, w11, [x5, #8] // filterPos[idx + 2], [idx + 3]
110110
111− movi v16.2d, #0 // initialize accumulator for idx + 0
112− movi v17.2d, #0 // initialize accumulator for idx + 1
113− movi v18.2d, #0 // initialize accumulator for idx + 2
114− movi v19.2d, #0 // initialize accumulator for idx + 3
111+ movi v16.2d, #0 // initialize accumulator for idx + 0
112+ movi v17.2d, #0 // initialize accumulator for idx + 1
113+ movi v18.2d, #0 // initialize accumulator for idx + 2
114+ movi v19.2d, #0 // initialize accumulator for idx + 3
115115
116− mov x12, x4 // filter pointer for idx + 0
117− add x13, x4, x7 // filter pointer for idx + 1
118− add x8, x3, w8, uxtw // srcp + filterPos[idx + 0]
119− add x9, x3, w9, uxtw // srcp + filterPos[idx + 1]
116+ mov x12, x4 // filter pointer for idx + 0
117+ add x13, x4, x7 // filter pointer for idx + 1
118+ add x8, x3, w8, uxtw // srcp + filterPos[idx + 0]
119+ add x9, x3, w9, uxtw // srcp + filterPos[idx + 1]
120120
121− add x14, x13, x7 // filter pointer for idx + 2
122− add x10, x3, w10, uxtw // srcp + filterPos[idx + 2]
123− add x11, x3, w11, uxtw // srcp + filterPos[idx + 3]
121+ add x14, x13, x7 // filter pointer for idx + 2
122+ add x10, x3, w10, uxtw // srcp + filterPos[idx + 2]
123+ add x11, x3, w11, uxtw // srcp + filterPos[idx + 3]
124124
125− mov w0, w6 // copy filterSize to a temp register, w0
126− add x5, x5, #16 // advance the filterPos pointer
127− add x15, x14, x7 // filter pointer for idx + 3
128− mov x16, xzr // temp register for offsetting filter pointers
125+ mov w0, w6 // copy filterSize to a temp register, w0
126+ add x5, x5, #16 // advance the filterPos pointer
127+ add x15, x14, x7 // filter pointer for idx + 3
128+ mov x16, xzr // temp register for offsetting filter pointers
129129
130130 2:
131131 // This section loops over 8-wide chunks of filter size
132− ldr d4, [x8], #8 // load 8 bytes from srcp for idx + 0
133− ldr q0, [x12, x16] // load 8 values, 16 bytes from filter for idx + 0
132+ ldr d4, [x8], #8 // load 8 bytes from srcp for idx + 0
133+ ldr q0, [x12, x16] // load 8 values, 16 bytes from filter for idx + 0
134134
135− ldr d5, [x9], #8 // load 8 bytes from srcp for idx + 1
136− ldr q1, [x13, x16] // load 8 values, 16 bytes from filter for idx + 1
135+ ldr d5, [x9], #8 // load 8 bytes from srcp for idx + 1
136+ ldr q1, [x13, x16] // load 8 values, 16 bytes from filter for idx + 1
137137
138− uxtl v4.8h, v4.8b // unsigned extend long for idx + 0
139− uxtl v5.8h, v5.8b // unsigned extend long for idx + 1
138+ uxtl v4.8h, v4.8b // unsigned extend long for idx + 0
139+ uxtl v5.8h, v5.8b // unsigned extend long for idx + 1
140140
141− ldr d6, [x10], #8 // load 8 bytes from srcp for idx + 2
142− ldr q2, [x14, x16] // load 8 values, 16 bytes from filter for idx + 2
141+ ldr d6, [x10], #8 // load 8 bytes from srcp for idx + 2
142+ ldr q2, [x14, x16] // load 8 values, 16 bytes from filter for idx + 2
143143
144− smlal v16.4s, v0.4h, v4.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 0
145− smlal v17.4s, v1.4h, v5.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 1
144+ smlal v16.4s, v0.4h, v4.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 0
145+ smlal v17.4s, v1.4h, v5.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 1
146146
147− ldr d7, [x11], #8 // load 8 bytes from srcp for idx + 3
148− ldr q3, [x15, x16] // load 8 values, 16 bytes from filter for idx + 3
147+ ldr d7, [x11], #8 // load 8 bytes from srcp for idx + 3
148+ ldr q3, [x15, x16] // load 8 values, 16 bytes from filter for idx + 3
149149
150− sub w0, w0, #8 // decrement the remaining filterSize counter
151− smlal2 v16.4s, v0.8h, v4.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 0
152− smlal2 v17.4s, v1.8h, v5.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 1
153− uxtl v6.8h, v6.8b // unsigned extend long for idx + 2
154− uxtl v7.8h, v7.8b // unsigned extend long for idx + 3
155− smlal v18.4s, v2.4h, v6.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 2
156− smlal v19.4s, v3.4h, v7.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 3
150+ sub w0, w0, #8 // decrement the remaining filterSize counter
151+ smlal2 v16.4s, v0.8h, v4.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 0
152+ smlal2 v17.4s, v1.8h, v5.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 1
153+ uxtl v6.8h, v6.8b // unsigned extend long for idx + 2
154+ uxtl v7.8h, v7.8b // unsigned extend long for idx + 3
155+ smlal v18.4s, v2.4h, v6.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 2
156+ smlal v19.4s, v3.4h, v7.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 3
157157
158− cmp w0, #8 // are there at least 8 more elements in filter to consume?
159− add x16, x16, #16 // advance the offsetting register for filter values
158+ cmp w0, #8 // are there at least 8 more elements in filter to consume?
159+ add x16, x16, #16 // advance the offsetting register for filter values
160160
161− smlal2 v18.4s, v2.8h, v6.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 2
162− smlal2 v19.4s, v3.8h, v7.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 3
161+ smlal2 v18.4s, v2.8h, v6.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 2
162+ smlal2 v19.4s, v3.8h, v7.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 3
163163
164− b.ge 2b // branch back to inner loop
164+ b.ge 2b // branch back to inner loop
165165
166166 // complete the remaining 4 filter elements
167− sub x17, x7, #8 // calculate the offset of the filter pointer for the remaining 4 elements
168−
169− ldr s4, [x8] // load 4 bytes from srcp for idx + 0
170− ldr d0, [x12, x17] // load 4 values, 8 bytes from filter for idx + 0
171− ldr s5, [x9] // load 4 bytes from srcp for idx + 1
172− ldr d1, [x13, x17] // load 4 values, 8 bytes from filter for idx + 1
173−
174− uxtl v4.8h, v4.8b // unsigned extend long for idx + 0
175− uxtl v5.8h, v5.8b // unsigned extend long for idx + 1
176−
177− ldr s6, [x10] // load 4 bytes from srcp for idx + 2
178− ldr d2, [x14, x17] // load 4 values, 8 bytes from filter for idx + 2
179− smlal v16.4s, v0.4h, v4.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 0
180− smlal v17.4s, v1.4h, v5.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 1
181− ldr s7, [x11] // load 4 bytes from srcp for idx + 3
182− ldr d3, [x15, x17] // load 4 values, 8 bytes from filter for idx + 3
183−
184− uxtl v6.8h, v6.8b // unsigned extend long for idx + 2
185− uxtl v7.8h, v7.8b // unsigned extend long for idx + 3
186− addp v16.4s, v16.4s, v17.4s // horizontal pair adding for idx 0,1
187− smlal v18.4s, v2.4h, v6.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 2
188− smlal v19.4s, v3.4h, v7.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 3
189−
190− addp v18.4s, v18.4s, v19.4s // horizontal pair adding for idx 2,3
191− addp v16.4s, v16.4s, v18.4s // final horizontal pair adding producing one vector with results for idx = 0..3
192−
193− subs w2, w2, #4 // dstW -= 4
194− sqshrn v0.4h, v16.4s, #7 // shift and clip the 2x16-bit final values
195− st1 {v0.4h}, [x1], #8 // write to destination idx 0..3
196− add x4, x4, x7, lsl #2 // filter += (filterSize*2) * 4
197− b.gt 1b // loop until end of line
167+ sub x17, x7, #8 // calculate the offset of the filter pointer for the remaining 4 elements
168+
169+ ldr s4, [x8] // load 4 bytes from srcp for idx + 0
170+ ldr d0, [x12, x17] // load 4 values, 8 bytes from filter for idx + 0
171+ ldr s5, [x9] // load 4 bytes from srcp for idx + 1
172+ ldr d1, [x13, x17] // load 4 values, 8 bytes from filter for idx + 1
173+
174+ uxtl v4.8h, v4.8b // unsigned extend long for idx + 0
175+ uxtl v5.8h, v5.8b // unsigned extend long for idx + 1
176+
177+ ldr s6, [x10] // load 4 bytes from srcp for idx + 2
178+ ldr d2, [x14, x17] // load 4 values, 8 bytes from filter for idx + 2
179+ smlal v16.4s, v0.4h, v4.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 0
180+ smlal v17.4s, v1.4h, v5.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 1
181+ ldr s7, [x11] // load 4 bytes from srcp for idx + 3
182+ ldr d3, [x15, x17] // load 4 values, 8 bytes from filter for idx + 3
183+
184+ uxtl v6.8h, v6.8b // unsigned extend long for idx + 2
185+ uxtl v7.8h, v7.8b // unsigned extend long for idx + 3
186+ addp v16.4s, v16.4s, v17.4s // horizontal pair adding for idx 0,1
187+ smlal v18.4s, v2.4h, v6.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 2
188+ smlal v19.4s, v3.4h, v7.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 3
189+
190+ addp v18.4s, v18.4s, v19.4s // horizontal pair adding for idx 2,3
191+ addp v16.4s, v16.4s, v18.4s // final horizontal pair adding producing one vector with results for idx = 0..3
192+
193+ subs w2, w2, #4 // dstW -= 4
194+ sqshrn v0.4h, v16.4s, #7 // shift and clip the 2x16-bit final values
195+ st1 {v0.4h}, [x1], #8 // write to destination idx 0..3
196+ add x4, x4, x7, lsl #2 // filter += (filterSize*2) * 4
197+ b.gt 1b // loop until end of line
198198 ret
199199 endfunc
200200
@@ -219,132 +219,132 @@ function ff_hscale8to15_4_neon, export=1 · 18 unmodified lines
219219 // 3. Complete madd
220220 // 4. Complete remaining iterations when dstW % 8 != 0
221221
222− sub sp, sp, #32 // allocate 32 bytes on the stack
223− cmp w2, #16 // if dstW <16, skip to the last block used for wrapping up
224− b.lt 2f
222+ sub sp, sp, #32 // allocate 32 bytes on the stack
223+ cmp w2, #16 // if dstW <16, skip to the last block used for wrapping up
224+ b.lt 2f
225225
226226 // load 8 values from filterPos to be used as offsets into src
227− ldp w8, w9, [x5] // filterPos[idx + 0], [idx + 1]
228− ldp w10, w11, [x5, #8] // filterPos[idx + 2], [idx + 3]
229− ldp w12, w13, [x5, #16] // filterPos[idx + 4], [idx + 5]
230− ldp w14, w15, [x5, #24] // filterPos[idx + 6], [idx + 7]
231− add x5, x5, #32 // advance filterPos
227+ ldp w8, w9, [x5] // filterPos[idx + 0], [idx + 1]
228+ ldp w10, w11, [x5, #8] // filterPos[idx + 2], [idx + 3]
229+ ldp w12, w13, [x5, #16] // filterPos[idx + 4], [idx + 5]
230+ ldp w14, w15, [x5, #24] // filterPos[idx + 6], [idx + 7]
231+ add x5, x5, #32 // advance filterPos
232232
233233 // gather random access data from src into contiguous memory
234− ldr w8, [x3, w8, uxtw] // src[filterPos[idx + 0]][0..3]
235− ldr w9, [x3, w9, uxtw] // src[filterPos[idx + 1]][0..3]
236− ldr w10, [x3, w10, uxtw] // src[filterPos[idx + 2]][0..3]
237− ldr w11, [x3, w11, uxtw] // src[filterPos[idx + 3]][0..3]
238− ldr w12, [x3, w12, uxtw] // src[filterPos[idx + 4]][0..3]
239− ldr w13, [x3, w13, uxtw] // src[filterPos[idx + 5]][0..3]
240− ldr w14, [x3, w14, uxtw] // src[filterPos[idx + 6]][0..3]
241− ldr w15, [x3, w15, uxtw] // src[filterPos[idx + 7]][0..3]
242− stp w8, w9, [sp] // *scratch_mem = { src[filterPos[idx + 0]][0..3], src[filterPos[idx + 1]][0..3] }
243− stp w10, w11, [sp, #8] // *scratch_mem = { src[filterPos[idx + 2]][0..3], src[filterPos[idx + 3]][0..3] }
244− stp w12, w13, [sp, #16] // *scratch_mem = { src[filterPos[idx + 4]][0..3], src[filterPos[idx + 5]][0..3] }
245− stp w14, w15, [sp, #24] // *scratch_mem = { src[filterPos[idx + 6]][0..3], src[filterPos[idx + 7]][0..3] }
234+ ldr w8, [x3, w8, uxtw] // src[filterPos[idx + 0]][0..3]
235+ ldr w9, [x3, w9, uxtw] // src[filterPos[idx + 1]][0..3]
236+ ldr w10, [x3, w10, uxtw] // src[filterPos[idx + 2]][0..3]
237+ ldr w11, [x3, w11, uxtw] // src[filterPos[idx + 3]][0..3]
238+ ldr w12, [x3, w12, uxtw] // src[filterPos[idx + 4]][0..3]
239+ ldr w13, [x3, w13, uxtw] // src[filterPos[idx + 5]][0..3]
240+ ldr w14, [x3, w14, uxtw] // src[filterPos[idx + 6]][0..3]
241+ ldr w15, [x3, w15, uxtw] // src[filterPos[idx + 7]][0..3]
242+ stp w8, w9, [sp] // *scratch_mem = { src[filterPos[idx + 0]][0..3], src[filterPos[idx + 1]][0..3] }
243+ stp w10, w11, [sp, #8] // *scratch_mem = { src[filterPos[idx + 2]][0..3], src[filterPos[idx + 3]][0..3] }
244+ stp w12, w13, [sp, #16] // *scratch_mem = { src[filterPos[idx + 4]][0..3], src[filterPos[idx + 5]][0..3] }
245+ stp w14, w15, [sp, #24] // *scratch_mem = { src[filterPos[idx + 6]][0..3], src[filterPos[idx + 7]][0..3] }
246246
247247 1:
248− ld4 {v16.8b, v17.8b, v18.8b, v19.8b}, [sp] // transpose 8 bytes each from src into 4 registers
248+ ld4 {v16.8b, v17.8b, v18.8b, v19.8b}, [sp] // transpose 8 bytes each from src into 4 registers
249249
250250 // load 8 values from filterPos to be used as offsets into src
251− ldp w8, w9, [x5] // filterPos[idx + 0][0..3], [idx + 1][0..3], next iteration
252− ldp w10, w11, [x5, #8] // filterPos[idx + 2][0..3], [idx + 3][0..3], next iteration
253− ldp w12, w13, [x5, #16] // filterPos[idx + 4][0..3], [idx + 5][0..3], next iteration
254− ldp w14, w15, [x5, #24] // filterPos[idx + 6][0..3], [idx + 7][0..3], next iteration
251+ ldp w8, w9, [x5] // filterPos[idx + 0][0..3], [idx + 1][0..3], next iteration
252+ ldp w10, w11, [x5, #8] // filterPos[idx + 2][0..3], [idx + 3][0..3], next iteration
253+ ldp w12, w13, [x5, #16] // filterPos[idx + 4][0..3], [idx + 5][0..3], next iteration
254+ ldp w14, w15, [x5, #24] // filterPos[idx + 6][0..3], [idx + 7][0..3], next iteration
255255
256− movi v0.2d, #0 // Clear madd accumulator for idx 0..3
257− movi v5.2d, #0 // Clear madd accumulator for idx 4..7
256+ movi v0.2d, #0 // Clear madd accumulator for idx 0..3
257+ movi v5.2d, #0 // Clear madd accumulator for idx 4..7
258258
259− ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x4], #64 // load filter idx + 0..7
259+ ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x4], #64 // load filter idx + 0..7
260260
261− add x5, x5, #32 // advance filterPos
261+ add x5, x5, #32 // advance filterPos
262262
263263 // interleaved SIMD and prefetching intended to keep ld/st and vector pipelines busy
264− uxtl v16.8h, v16.8b // unsigned extend long, covert src data to 16-bit
265− uxtl v17.8h, v17.8b // unsigned extend long, covert src data to 16-bit
266− ldr w8, [x3, w8, uxtw] // src[filterPos[idx + 0]], next iteration
267− ldr w9, [x3, w9, uxtw] // src[filterPos[idx + 1]], next iteration
268− uxtl v18.8h, v18.8b // unsigned extend long, covert src data to 16-bit
269− uxtl v19.8h, v19.8b // unsigned extend long, covert src data to 16-bit
270− ldr w10, [x3, w10, uxtw] // src[filterPos[idx + 2]], next iteration
271− ldr w11, [x3, w11, uxtw] // src[filterPos[idx + 3]], next iteration
272−
273− smlal v0.4s, v1.4h, v16.4h // multiply accumulate inner loop j = 0, idx = 0..3
274− smlal v0.4s, v2.4h, v17.4h // multiply accumulate inner loop j = 1, idx = 0..3
275− ldr w12, [x3, w12, uxtw] // src[filterPos[idx + 4]], next iteration
276− ldr w13, [x3, w13, uxtw] // src[filterPos[idx + 5]], next iteration
277− smlal v0.4s, v3.4h, v18.4h // multiply accumulate inner loop j = 2, idx = 0..3
278− smlal v0.4s, v4.4h, v19.4h // multiply accumulate inner loop j = 3, idx = 0..3
279− ldr w14, [x3, w14, uxtw] // src[filterPos[idx + 6]], next iteration
280− ldr w15, [x3, w15, uxtw] // src[filterPos[idx + 7]], next iteration
281−
282− smlal2 v5.4s, v1.8h, v16.8h // multiply accumulate inner loop j = 0, idx = 4..7
283− smlal2 v5.4s, v2.8h, v17.8h // multiply accumulate inner loop j = 1, idx = 4..7
284− stp w8, w9, [sp] // *scratch_mem = { src[filterPos[idx + 0]][0..3], src[filterPos[idx + 1]][0..3] }
285− stp w10, w11, [sp, #8] // *scratch_mem = { src[filterPos[idx + 2]][0..3], src[filterPos[idx + 3]][0..3] }
286− smlal2 v5.4s, v3.8h, v18.8h // multiply accumulate inner loop j = 2, idx = 4..7
287− smlal2 v5.4s, v4.8h, v19.8h // multiply accumulate inner loop j = 3, idx = 4..7
288− stp w12, w13, [sp, #16] // *scratch_mem = { src[filterPos[idx + 4]][0..3], src[filterPos[idx + 5]][0..3] }
289− stp w14, w15, [sp, #24] // *scratch_mem = { src[filterPos[idx + 6]][0..3], src[filterPos[idx + 7]][0..3] }
290−
291− sub w2, w2, #8 // dstW -= 8
292− sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values
293− sqshrn v1.4h, v5.4s, #7 // shift and clip the 2x16-bit final values
294− st1 {v0.4h, v1.4h}, [x1], #16 // write to dst[idx + 0..7]
295− cmp w2, #16 // continue on main loop if there are at least 16 iterations left
296− b.ge 1b
264+ uxtl v16.8h, v16.8b // unsigned extend long, covert src data to 16-bit
265+ uxtl v17.8h, v17.8b // unsigned extend long, covert src data to 16-bit
266+ ldr w8, [x3, w8, uxtw] // src[filterPos[idx + 0]], next iteration
267+ ldr w9, [x3, w9, uxtw] // src[filterPos[idx + 1]], next iteration
268+ uxtl v18.8h, v18.8b // unsigned extend long, covert src data to 16-bit
269+ uxtl v19.8h, v19.8b // unsigned extend long, covert src data to 16-bit
270+ ldr w10, [x3, w10, uxtw] // src[filterPos[idx + 2]], next iteration
271+ ldr w11, [x3, w11, uxtw] // src[filterPos[idx + 3]], next iteration
272+
273+ smlal v0.4s, v1.4h, v16.4h // multiply accumulate inner loop j = 0, idx = 0..3
274+ smlal v0.4s, v2.4h, v17.4h // multiply accumulate inner loop j = 1, idx = 0..3
275+ ldr w12, [x3, w12, uxtw] // src[filterPos[idx + 4]], next iteration
276+ ldr w13, [x3, w13, uxtw] // src[filterPos[idx + 5]], next iteration
277+ smlal v0.4s, v3.4h, v18.4h // multiply accumulate inner loop j = 2, idx = 0..3
278+ smlal v0.4s, v4.4h, v19.4h // multiply accumulate inner loop j = 3, idx = 0..3
279+ ldr w14, [x3, w14, uxtw] // src[filterPos[idx + 6]], next iteration
280+ ldr w15, [x3, w15, uxtw] // src[filterPos[idx + 7]], next iteration
281+
282+ smlal2 v5.4s, v1.8h, v16.8h // multiply accumulate inner loop j = 0, idx = 4..7
283+ smlal2 v5.4s, v2.8h, v17.8h // multiply accumulate inner loop j = 1, idx = 4..7
284+ stp w8, w9, [sp] // *scratch_mem = { src[filterPos[idx + 0]][0..3], src[filterPos[idx + 1]][0..3] }
285+ stp w10, w11, [sp, #8] // *scratch_mem = { src[filterPos[idx + 2]][0..3], src[filterPos[idx + 3]][0..3] }
286+ smlal2 v5.4s, v3.8h, v18.8h // multiply accumulate inner loop j = 2, idx = 4..7
287+ smlal2 v5.4s, v4.8h, v19.8h // multiply accumulate inner loop j = 3, idx = 4..7
288+ stp w12, w13, [sp, #16] // *scratch_mem = { src[filterPos[idx + 4]][0..3], src[filterPos[idx + 5]][0..3] }
289+ stp w14, w15, [sp, #24] // *scratch_mem = { src[filterPos[idx + 6]][0..3], src[filterPos[idx + 7]][0..3] }
290+
291+ sub w2, w2, #8 // dstW -= 8
292+ sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values
293+ sqshrn v1.4h, v5.4s, #7 // shift and clip the 2x16-bit final values
294+ st1 {v0.4h, v1.4h}, [x1], #16 // write to dst[idx + 0..7]
295+ cmp w2, #16 // continue on main loop if there are at least 16 iterations left
296+ b.ge 1b
297297
298298 // last full iteration
299− ld4 {v16.8b, v17.8b, v18.8b, v19.8b}, [sp]
300− ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x4], #64 // load filter idx + 0..7
299+ ld4 {v16.8b, v17.8b, v18.8b, v19.8b}, [sp]
300+ ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x4], #64 // load filter idx + 0..7
301301
302− movi v0.2d, #0 // Clear madd accumulator for idx 0..3
303− movi v5.2d, #0 // Clear madd accumulator for idx 4..7
302+ movi v0.2d, #0 // Clear madd accumulator for idx 0..3
303+ movi v5.2d, #0 // Clear madd accumulator for idx 4..7
304304
305− uxtl v16.8h, v16.8b // unsigned extend long, covert src data to 16-bit
306− uxtl v17.8h, v17.8b // unsigned extend long, covert src data to 16-bit
307− uxtl v18.8h, v18.8b // unsigned extend long, covert src data to 16-bit
308− uxtl v19.8h, v19.8b // unsigned extend long, covert src data to 16-bit
305+ uxtl v16.8h, v16.8b // unsigned extend long, covert src data to 16-bit
306+ uxtl v17.8h, v17.8b // unsigned extend long, covert src data to 16-bit
307+ uxtl v18.8h, v18.8b // unsigned extend long, covert src data to 16-bit
308+ uxtl v19.8h, v19.8b // unsigned extend long, covert src data to 16-bit
309309
310− smlal v0.4s, v1.4h, v16.4h // multiply accumulate inner loop j = 0, idx = 0..3
311− smlal v0.4s, v2.4h, v17.4h // multiply accumulate inner loop j = 1, idx = 0..3
312− smlal v0.4s, v3.4h, v18.4h // multiply accumulate inner loop j = 2, idx = 0..3
313− smlal v0.4s, v4.4h, v19.4h // multiply accumulate inner loop j = 3, idx = 0..3
310+ smlal v0.4s, v1.4h, v16.4h // multiply accumulate inner loop j = 0, idx = 0..3
311+ smlal v0.4s, v2.4h, v17.4h // multiply accumulate inner loop j = 1, idx = 0..3
312+ smlal v0.4s, v3.4h, v18.4h // multiply accumulate inner loop j = 2, idx = 0..3
313+ smlal v0.4s, v4.4h, v19.4h // multiply accumulate inner loop j = 3, idx = 0..3
314314
315− smlal2 v5.4s, v1.8h, v16.8h // multiply accumulate inner loop j = 0, idx = 4..7
316− smlal2 v5.4s, v2.8h, v17.8h // multiply accumulate inner loop j = 1, idx = 4..7
317− smlal2 v5.4s, v3.8h, v18.8h // multiply accumulate inner loop j = 2, idx = 4..7
318− smlal2 v5.4s, v4.8h, v19.8h // multiply accumulate inner loop j = 3, idx = 4..7
315+ smlal2 v5.4s, v1.8h, v16.8h // multiply accumulate inner loop j = 0, idx = 4..7
316+ smlal2 v5.4s, v2.8h, v17.8h // multiply accumulate inner loop j = 1, idx = 4..7
317+ smlal2 v5.4s, v3.8h, v18.8h // multiply accumulate inner loop j = 2, idx = 4..7
318+ smlal2 v5.4s, v4.8h, v19.8h // multiply accumulate inner loop j = 3, idx = 4..7
319319
320− subs w2, w2, #8 // dstW -= 8
321− sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values
322− sqshrn v1.4h, v5.4s, #7 // shift and clip the 2x16-bit final values
323− st1 {v0.4h, v1.4h}, [x1], #16 // write to dst[idx + 0..7]
320+ subs w2, w2, #8 // dstW -= 8
321+ sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values
322+ sqshrn v1.4h, v5.4s, #7 // shift and clip the 2x16-bit final values
323+ st1 {v0.4h, v1.4h}, [x1], #16 // write to dst[idx + 0..7]
324324
325− cbnz w2, 2f // if >0 iterations remain, jump to the wrap up section
325+ cbnz w2, 2f // if >0 iterations remain, jump to the wrap up section
326326
327− add sp, sp, #32 // clean up stack
327+ add sp, sp, #32 // clean up stack
328328 ret
329329
330330 // finish up when dstW % 8 != 0 or dstW < 16
331331 2:
332332 // load src
333− ldr w8, [x5], #4 // filterPos[i]
334− add x9, x3, w8, uxtw // calculate the address for src load
335− ld1 {v5.s}[0], [x9] // src[filterPos[i] + 0..3]
333+ ldr w8, [x5], #4 // filterPos[i]
334+ add x9, x3, w8, uxtw // calculate the address for src load
335+ ld1 {v5.s}[0], [x9] // src[filterPos[i] + 0..3]
336336 // load filter
337− ld1 {v6.4h}, [x4], #8 // filter[filterSize * i + 0..3]
337+ ld1 {v6.4h}, [x4], #8 // filter[filterSize * i + 0..3]
338338
339− uxtl v5.8h, v5.8b // unsigned exten long, convert src data to 16-bit
340− smull v0.4s, v5.4h, v6.4h // 4 iterations of src[...] * filter[...]
341− addv s0, v0.4s // add up products of src and filter values
342− sqshrn h0, s0, #7 // shift and clip the 2x16-bit final value
343− st1 {v0.h}[0], [x1], #2 // dst[i] = ...
344− sub w2, w2, #1 // dstW--
345− cbnz w2, 2b
339+ uxtl v5.8h, v5.8b // unsigned exten long, convert src data to 16-bit
340+ smull v0.4s, v5.4h, v6.4h // 4 iterations of src[...] * filter[...]
341+ addv s0, v0.4s // add up products of src and filter values
342+ sqshrn h0, s0, #7 // shift and clip the 2x16-bit final value
343+ st1 {v0.h}[0], [x1], #2 // dst[i] = ...
344+ sub w2, w2, #1 // dstW--
345+ cbnz w2, 2b
346346
347− add sp, sp, #32 // clean up stack
347+ add sp, sp, #32 // clean up stack
348348 ret
349349 endfunc
350350
@@ -357,187 +357,187 @@ function ff_hscale8to19_4_neon, export=1 · 6 unmodified lines
357357 // x5 const int32_t *filterPos
358358 // w6 int filterSize
359359
360− movi v18.4s, #1
361− movi v17.4s, #1
362− shl v18.4s, v18.4s, #19
363− sub v18.4s, v18.4s, v17.4s // max allowed value
360+ movi v18.4s, #1
361+ movi v17.4s, #1
362+ shl v18.4s, v18.4s, #19
363+ sub v18.4s, v18.4s, v17.4s // max allowed value
364364
365− cmp w2, #16
366− b.lt 2f // move to last block
365+ cmp w2, #16
366+ b.lt 2f // move to last block
367367
368− ldp w8, w9, [x5] // filterPos[0], filterPos[1]
369− ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
370− ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
371− ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
372− add x5, x5, #32
368+ ldp w8, w9, [x5] // filterPos[0], filterPos[1]
369+ ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
370+ ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
371+ ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
372+ add x5, x5, #32
373373
374374 // load data from
375− ldr w8, [x3, w8, uxtw]
376− ldr w9, [x3, w9, uxtw]
377− ldr w10, [x3, w10, uxtw]
378− ldr w11, [x3, w11, uxtw]
379− ldr w12, [x3, w12, uxtw]
380− ldr w13, [x3, w13, uxtw]
381− ldr w14, [x3, w14, uxtw]
382− ldr w15, [x3, w15, uxtw]
383−
384− sub sp, sp, #32
385−
386− stp w8, w9, [sp]
387− stp w10, w11, [sp, #8]
388− stp w12, w13, [sp, #16]
389− stp w14, w15, [sp, #24]
375+ ldr w8, [x3, w8, uxtw]
376+ ldr w9, [x3, w9, uxtw]
377+ ldr w10, [x3, w10, uxtw]
378+ ldr w11, [x3, w11, uxtw]
379+ ldr w12, [x3, w12, uxtw]
380+ ldr w13, [x3, w13, uxtw]
381+ ldr w14, [x3, w14, uxtw]
382+ ldr w15, [x3, w15, uxtw]
383+
384+ sub sp, sp, #32
385+
386+ stp w8, w9, [sp]
387+ stp w10, w11, [sp, #8]
388+ stp w12, w13, [sp, #16]
389+ stp w14, w15, [sp, #24]
390390
391391 1:
392− ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [sp]
393− ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]
392+ ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [sp]
393+ ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]
394394 // load filterPositions into registers for next iteration
395395
396− ldp w8, w9, [x5] // filterPos[0], filterPos[1]
397− ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
398− ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
399− ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
400− add x5, x5, #32
401− uxtl v0.8h, v0.8b
402− ldr w8, [x3, w8, uxtw]
403− smull v5.4s, v0.4h, v28.4h // multiply first column of src
404− ldr w9, [x3, w9, uxtw]
405− smull2 v6.4s, v0.8h, v28.8h
406− stp w8, w9, [sp]
407−
408− uxtl v1.8h, v1.8b
409− ldr w10, [x3, w10, uxtw]
410− smlal v5.4s, v1.4h, v29.4h // multiply second column of src
411− ldr w11, [x3, w11, uxtw]
412− smlal2 v6.4s, v1.8h, v29.8h
413− stp w10, w11, [sp, #8]
414−
415− uxtl v2.8h, v2.8b
416− ldr w12, [x3, w12, uxtw]
417− smlal v5.4s, v2.4h, v30.4h // multiply third column of src
418− ldr w13, [x3, w13, uxtw]
419− smlal2 v6.4s, v2.8h, v30.8h
420− stp w12, w13, [sp, #16]
421−
422− uxtl v3.8h, v3.8b
423− ldr w14, [x3, w14, uxtw]
424− smlal v5.4s, v3.4h, v31.4h // multiply fourth column of src
425− ldr w15, [x3, w15, uxtw]
426− smlal2 v6.4s, v3.8h, v31.8h
427− stp w14, w15, [sp, #24]
428−
429− sub w2, w2, #8
430− sshr v5.4s, v5.4s, #3
431− sshr v6.4s, v6.4s, #3
432− smin v5.4s, v5.4s, v18.4s
433− smin v6.4s, v6.4s, v18.4s
434−
435− st1 {v5.4s, v6.4s}, [x1], #32
436− cmp w2, #16
437− b.ge 1b
396+ ldp w8, w9, [x5] // filterPos[0], filterPos[1]
397+ ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
398+ ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
399+ ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
400+ add x5, x5, #32
401+ uxtl v0.8h, v0.8b
402+ ldr w8, [x3, w8, uxtw]
403+ smull v5.4s, v0.4h, v28.4h // multiply first column of src
404+ ldr w9, [x3, w9, uxtw]
405+ smull2 v6.4s, v0.8h, v28.8h
406+ stp w8, w9, [sp]
407+
408+ uxtl v1.8h, v1.8b
409+ ldr w10, [x3, w10, uxtw]
410+ smlal v5.4s, v1.4h, v29.4h // multiply second column of src
411+ ldr w11, [x3, w11, uxtw]
412+ smlal2 v6.4s, v1.8h, v29.8h
413+ stp w10, w11, [sp, #8]
414+
415+ uxtl v2.8h, v2.8b
416+ ldr w12, [x3, w12, uxtw]
417+ smlal v5.4s, v2.4h, v30.4h // multiply third column of src
418+ ldr w13, [x3, w13, uxtw]
419+ smlal2 v6.4s, v2.8h, v30.8h
420+ stp w12, w13, [sp, #16]
421+
422+ uxtl v3.8h, v3.8b
423+ ldr w14, [x3, w14, uxtw]
424+ smlal v5.4s, v3.4h, v31.4h // multiply fourth column of src
425+ ldr w15, [x3, w15, uxtw]
426+ smlal2 v6.4s, v3.8h, v31.8h
427+ stp w14, w15, [sp, #24]
428+
429+ sub w2, w2, #8
430+ sshr v5.4s, v5.4s, #3
431+ sshr v6.4s, v6.4s, #3
432+ smin v5.4s, v5.4s, v18.4s
433+ smin v6.4s, v6.4s, v18.4s
434+
435+ st1 {v5.4s, v6.4s}, [x1], #32
436+ cmp w2, #16
437+ b.ge 1b
438438
439439 // here we make last iteration, without updating the registers
440− ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [sp]
441− ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]
442−
443− uxtl v0.8h, v0.8b
444− uxtl v1.8h, v1.8b
445− smull v5.4s, v0.4h, v28.4h
446− smull2 v6.4s, v0.8h, v28.8h
447− uxtl v2.8h, v2.8b
448− smlal v5.4s, v1.4h, v29.4h
449− smlal2 v6.4s, v1.8h, v29.8h
450− uxtl v3.8h, v3.8b
451− smlal v5.4s, v2.4h, v30.4h
452− smlal2 v6.4s, v2.8h, v30.8h
453− smlal v5.4s, v3.4h, v31.4h
454− smlal2 v6.4s, v3.8h, v31.8h
455−
456− sshr v5.4s, v5.4s, #3
457− sshr v6.4s, v6.4s, #3
458−
459− smin v5.4s, v5.4s, v18.4s
460− smin v6.4s, v6.4s, v18.4s
461−
462− sub w2, w2, #8
463− st1 {v5.4s, v6.4s}, [x1], #32
464− add sp, sp, #32 // restore stack
465− cbnz w2, 2f
440+ ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [sp]
441+ ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]
442+
443+ uxtl v0.8h, v0.8b
444+ uxtl v1.8h, v1.8b
445+ smull v5.4s, v0.4h, v28.4h
446+ smull2 v6.4s, v0.8h, v28.8h
447+ uxtl v2.8h, v2.8b
448+ smlal v5.4s, v1.4h, v29.4h
449+ smlal2 v6.4s, v1.8h, v29.8h
450+ uxtl v3.8h, v3.8b
451+ smlal v5.4s, v2.4h, v30.4h
452+ smlal2 v6.4s, v2.8h, v30.8h
453+ smlal v5.4s, v3.4h, v31.4h
454+ smlal2 v6.4s, v3.8h, v31.8h
455+
456+ sshr v5.4s, v5.4s, #3
457+ sshr v6.4s, v6.4s, #3
458+
459+ smin v5.4s, v5.4s, v18.4s
460+ smin v6.4s, v6.4s, v18.4s
461+
462+ sub w2, w2, #8
463+ st1 {v5.4s, v6.4s}, [x1], #32
464+ add sp, sp, #32 // restore stack
465+ cbnz w2, 2f
466466
467467 ret
468468
469469 2:
470− ldr w8, [x5], #4 // load filterPos
471− add x9, x3, w8, uxtw // src + filterPos
472− ld1 {v0.s}[0], [x9] // load 4 * uint8_t* into one single
473− ld1 {v31.4h}, [x4], #8
474− uxtl v0.8h, v0.8b
475− smull v5.4s, v0.4h, v31.4h
476− saddlv d0, v5.4s
477− sqshrn s0, d0, #3
478− smin v0.4s, v0.4s, v18.4s
479− st1 {v0.s}[0], [x1], #4
480− sub w2, w2, #1
481− cbnz w2, 2b // if iterations remain jump to beginning
470+ ldr w8, [x5], #4 // load filterPos
471+ add x9, x3, w8, uxtw // src + filterPos
472+ ld1 {v0.s}[0], [x9] // load 4 * uint8_t* into one single
473+ ld1 {v31.4h}, [x4], #8
474+ uxtl v0.8h, v0.8b
475+ smull v5.4s, v0.4h, v31.4h
476+ saddlv d0, v5.4s
477+ sqshrn s0, d0, #3
478+ smin v0.4s, v0.4s, v18.4s
479+ st1 {v0.s}[0], [x1], #4
480+ sub w2, w2, #1
481+ cbnz w2, 2b // if iterations remain jump to beginning
482482
483483 ret
484484 endfunc
485485
486486 function ff_hscale8to19_X8_neon, export=1
487− movi v20.4s, #1
488− movi v17.4s, #1
489− shl v20.4s, v20.4s, #19
490− sub v20.4s, v20.4s, v17.4s
487+ movi v20.4s, #1
488+ movi v17.4s, #1
489+ shl v20.4s, v20.4s, #19
490+ sub v20.4s, v20.4s, v17.4s
491491
492− sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)
492+ sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)
493493 1:
494− mov x16, x4 // filter0 = filter
495− ldr w8, [x5], #4 // filterPos[idx]
496− add x12, x16, x7 // filter1 = filter0 + filterSize*2
497− ldr w0, [x5], #4 // filterPos[idx + 1]
498− add x13, x12, x7 // filter2 = filter1 + filterSize*2
499− ldr w11, [x5], #4 // filterPos[idx + 2]
500− add x4, x13, x7 // filter3 = filter2 + filterSize*2
501− ldr w9, [x5], #4 // filterPos[idx + 3]
502− movi v0.2d, #0 // val sum part 1 (for dst[0])
503− movi v1.2d, #0 // val sum part 2 (for dst[1])
504− movi v2.2d, #0 // val sum part 3 (for dst[2])
505− movi v3.2d, #0 // val sum part 4 (for dst[3])
506− add x17, x3, w8, uxtw // srcp + filterPos[0]
507− add x8, x3, w0, uxtw // srcp + filterPos[1]
508− add x0, x3, w11, uxtw // srcp + filterPos[2]
509− add x11, x3, w9, uxtw // srcp + filterPos[3]
510− mov w15, w6 // filterSize counter
511−2: ld1 {v4.8b}, [x17], #8 // srcp[filterPos[0] + {0..7}]
512− ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1
513− uxtl v4.8h, v4.8b // unpack part 1 to 16-bit
514− smlal v0.4s, v4.4h, v5.4h // v0 accumulates srcp[filterPos[0] + {0..3}] * filter[{0..3}]
515− ld1 {v6.8b}, [x8], #8 // srcp[filterPos[1] + {0..7}]
516− smlal2 v0.4s, v4.8h, v5.8h // v0 accumulates srcp[filterPos[0] + {4..7}] * filter[{4..7}]
517− ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize
518− ld1 {v16.8b}, [x0], #8 // srcp[filterPos[2] + {0..7}]
519− uxtl v6.8h, v6.8b // unpack part 2 to 16-bit
520− ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize
521− uxtl v16.8h, v16.8b // unpack part 3 to 16-bit
522− smlal v1.4s, v6.4h, v7.4h // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]
523− ld1 {v18.8b}, [x11], #8 // srcp[filterPos[3] + {0..7}]
524− smlal v2.4s, v16.4h, v17.4h // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]
525− ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize
526− smlal2 v2.4s, v16.8h, v17.8h // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]
527− uxtl v18.8h, v18.8b // unpack part 4 to 16-bit
528− smlal2 v1.4s, v6.8h, v7.8h // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]
529− smlal v3.4s, v18.4h, v19.4h // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]
530− subs w15, w15, #8 // j -= 8: processed 8/filterSize
531− smlal2 v3.4s, v18.8h, v19.8h // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]
532− b.gt 2b // inner loop if filterSize not consumed completely
533− addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding
534− addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding
535− addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding
536− subs w2, w2, #4 // dstW -= 4
537− sshr v0.4s, v0.4s, #3 // shift and clip the 2x16-bit final values
538− smin v0.4s, v0.4s, v20.4s
539− st1 {v0.4s}, [x1], #16 // write to destination part0123
540− b.gt 1b // loop until end of line
494+ mov x16, x4 // filter0 = filter
495+ ldr w8, [x5], #4 // filterPos[idx]
496+ add x12, x16, x7 // filter1 = filter0 + filterSize*2
497+ ldr w0, [x5], #4 // filterPos[idx + 1]
498+ add x13, x12, x7 // filter2 = filter1 + filterSize*2
499+ ldr w11, [x5], #4 // filterPos[idx + 2]
500+ add x4, x13, x7 // filter3 = filter2 + filterSize*2
501+ ldr w9, [x5], #4 // filterPos[idx + 3]
502+ movi v0.2d, #0 // val sum part 1 (for dst[0])
503+ movi v1.2d, #0 // val sum part 2 (for dst[1])
504+ movi v2.2d, #0 // val sum part 3 (for dst[2])
505+ movi v3.2d, #0 // val sum part 4 (for dst[3])
506+ add x17, x3, w8, uxtw // srcp + filterPos[0]
507+ add x8, x3, w0, uxtw // srcp + filterPos[1]
508+ add x0, x3, w11, uxtw // srcp + filterPos[2]
509+ add x11, x3, w9, uxtw // srcp + filterPos[3]
510+ mov w15, w6 // filterSize counter
511+2: ld1 {v4.8b}, [x17], #8 // srcp[filterPos[0] + {0..7}]
512+ ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1
513+ uxtl v4.8h, v4.8b // unpack part 1 to 16-bit
514+ smlal v0.4s, v4.4h, v5.4h // v0 accumulates srcp[filterPos[0] + {0..3}] * filter[{0..3}]
515+ ld1 {v6.8b}, [x8], #8 // srcp[filterPos[1] + {0..7}]
516+ smlal2 v0.4s, v4.8h, v5.8h // v0 accumulates srcp[filterPos[0] + {4..7}] * filter[{4..7}]
517+ ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize
518+ ld1 {v16.8b}, [x0], #8 // srcp[filterPos[2] + {0..7}]
519+ uxtl v6.8h, v6.8b // unpack part 2 to 16-bit
520+ ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize
521+ uxtl v16.8h, v16.8b // unpack part 3 to 16-bit
522+ smlal v1.4s, v6.4h, v7.4h // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]
523+ ld1 {v18.8b}, [x11], #8 // srcp[filterPos[3] + {0..7}]
524+ smlal v2.4s, v16.4h, v17.4h // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]
525+ ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize
526+ smlal2 v2.4s, v16.8h, v17.8h // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]
527+ uxtl v18.8h, v18.8b // unpack part 4 to 16-bit
528+ smlal2 v1.4s, v6.8h, v7.8h // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]
529+ smlal v3.4s, v18.4h, v19.4h // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]
530+ subs w15, w15, #8 // j -= 8: processed 8/filterSize
531+ smlal2 v3.4s, v18.8h, v19.8h // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]
532+ b.gt 2b // inner loop if filterSize not consumed completely
533+ addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding
534+ addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding
535+ addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding
536+ subs w2, w2, #4 // dstW -= 4
537+ sshr v0.4s, v0.4s, #3 // shift and clip the 2x16-bit final values
538+ smin v0.4s, v0.4s, v20.4s
539+ st1 {v0.4s}, [x1], #16 // write to destination part0123
540+ b.gt 1b // loop until end of line
541541 ret
542542 endfunc
543543
@@ -550,91 +550,91 @@ function ff_hscale8to19_X4_neon, export=1 · 6 unmodified lines
550550 // x5 const int32_t *filterPos
551551 // w6 int filterSize
552552
553− movi v20.4s, #1
554− movi v17.4s, #1
555− shl v20.4s, v20.4s, #19
556− sub v20.4s, v20.4s, v17.4s
553+ movi v20.4s, #1
554+ movi v17.4s, #1
555+ shl v20.4s, v20.4s, #19
556+ sub v20.4s, v20.4s, v17.4s
557557
558− lsl w7, w6, #1
558+ lsl w7, w6, #1
559559 1:
560− ldp w8, w9, [x5]
561− ldp w10, w11, [x5, #8]
562−
563− movi v16.2d, #0 // initialize accumulator for idx + 0
564− movi v17.2d, #0 // initialize accumulator for idx + 1
565− movi v18.2d, #0 // initialize accumulator for idx + 2
566− movi v19.2d, #0 // initialize accumulator for idx + 3
567−
568− mov x12, x4 // filter + 0
569− add x13, x4, x7 // filter + 1
570− add x8, x3, w8, uxtw // srcp + filterPos 0
571− add x14, x13, x7 // filter + 2
572− add x9, x3, w9, uxtw // srcp + filterPos 1
573− add x15, x14, x7 // filter + 3
574− add x10, x3, w10, uxtw // srcp + filterPos 2
575− mov w0, w6 // save the filterSize to temporary variable
576− add x11, x3, w11, uxtw // srcp + filterPos 3
577− add x5, x5, #16 // advance filter position
578− mov x16, xzr // clear the register x16 used for offsetting the filter values
560+ ldp w8, w9, [x5]
561+ ldp w10, w11, [x5, #8]
562+
563+ movi v16.2d, #0 // initialize accumulator for idx + 0
564+ movi v17.2d, #0 // initialize accumulator for idx + 1
565+ movi v18.2d, #0 // initialize accumulator for idx + 2
566+ movi v19.2d, #0 // initialize accumulator for idx + 3
567+
568+ mov x12, x4 // filter + 0
569+ add x13, x4, x7 // filter + 1
570+ add x8, x3, w8, uxtw // srcp + filterPos 0
571+ add x14, x13, x7 // filter + 2
572+ add x9, x3, w9, uxtw // srcp + filterPos 1
573+ add x15, x14, x7 // filter + 3
574+ add x10, x3, w10, uxtw // srcp + filterPos 2
575+ mov w0, w6 // save the filterSize to temporary variable
576+ add x11, x3, w11, uxtw // srcp + filterPos 3
577+ add x5, x5, #16 // advance filter position
578+ mov x16, xzr // clear the register x16 used for offsetting the filter values
579579
580580 2:
581− ldr d4, [x8], #8 // load src values for idx 0
582− ldr q31, [x12, x16] // load filter values for idx 0
583− uxtl v4.8h, v4.8b // extend type to match the filter' size
584− ldr d5, [x9], #8 // load src values for idx 1
585− smlal v16.4s, v4.4h, v31.4h // multiplication of lower half for idx 0
586− uxtl v5.8h, v5.8b // extend type to match the filter' size
587− ldr q30, [x13, x16] // load filter values for idx 1
588− smlal2 v16.4s, v4.8h, v31.8h // multiplication of upper half for idx 0
589− ldr d6, [x10], #8 // load src values for idx 2
590− ldr q29, [x14, x16] // load filter values for idx 2
591− smlal v17.4s, v5.4h, v30.4h // multiplication of lower half for idx 1
592− ldr d7, [x11], #8 // load src values for idx 3
593− smlal2 v17.4s, v5.8h, v30.8h // multiplication of upper half for idx 1
594− uxtl v6.8h, v6.8b // extend tpye to matchi the filter's size
595− ldr q28, [x15, x16] // load filter values for idx 3
596− smlal v18.4s, v6.4h, v29.4h // multiplication of lower half for idx 2
597− uxtl v7.8h, v7.8b
598− smlal2 v18.4s, v6.8h, v29.8h // multiplication of upper half for idx 2
599− sub w0, w0, #8
600− smlal v19.4s, v7.4h, v28.4h // multiplication of lower half for idx 3
601− cmp w0, #8
602− smlal2 v19.4s, v7.8h, v28.8h // multiplication of upper half for idx 3
603− add x16, x16, #16 // advance filter values indexing
604−
605− b.ge 2b
581+ ldr d4, [x8], #8 // load src values for idx 0
582+ ldr q31, [x12, x16] // load filter values for idx 0
583+ uxtl v4.8h, v4.8b // extend type to match the filter' size
584+ ldr d5, [x9], #8 // load src values for idx 1
585+ smlal v16.4s, v4.4h, v31.4h // multiplication of lower half for idx 0
586+ uxtl v5.8h, v5.8b // extend type to match the filter' size
587+ ldr q30, [x13, x16] // load filter values for idx 1
588+ smlal2 v16.4s, v4.8h, v31.8h // multiplication of upper half for idx 0
589+ ldr d6, [x10], #8 // load src values for idx 2
590+ ldr q29, [x14, x16] // load filter values for idx 2
591+ smlal v17.4s, v5.4h, v30.4h // multiplication of lower half for idx 1
592+ ldr d7, [x11], #8 // load src values for idx 3
593+ smlal2 v17.4s, v5.8h, v30.8h // multiplication of upper half for idx 1
594+ uxtl v6.8h, v6.8b // extend tpye to matchi the filter's size
595+ ldr q28, [x15, x16] // load filter values for idx 3
596+ smlal v18.4s, v6.4h, v29.4h // multiplication of lower half for idx 2
597+ uxtl v7.8h, v7.8b
598+ smlal2 v18.4s, v6.8h, v29.8h // multiplication of upper half for idx 2
599+ sub w0, w0, #8
600+ smlal v19.4s, v7.4h, v28.4h // multiplication of lower half for idx 3
601+ cmp w0, #8
602+ smlal2 v19.4s, v7.8h, v28.8h // multiplication of upper half for idx 3
603+ add x16, x16, #16 // advance filter values indexing
604+
605+ b.ge 2b
606606
607607
608608 // 4 iterations left
609609
610− sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements
611−
612− ldr s4, [x8] // load src values for idx 0
613− ldr d31, [x12, x17] // load filter values for idx 0
614− uxtl v4.8h, v4.8b // extend type to match the filter' size
615− ldr s5, [x9] // load src values for idx 1
616− smlal v16.4s, v4.4h, v31.4h
617− ldr d30, [x13, x17] // load filter values for idx 1
618− uxtl v5.8h, v5.8b // extend type to match the filter' size
619− ldr s6, [x10] // load src values for idx 2
620− smlal v17.4s, v5.4h, v30.4h
621− uxtl v6.8h, v6.8b // extend type to match the filter's size
622− ldr d29, [x14, x17] // load filter values for idx 2
623− ldr s7, [x11] // load src values for idx 3
624− addp v16.4s, v16.4s, v17.4s
625− uxtl v7.8h, v7.8b
626− ldr d28, [x15, x17] // load filter values for idx 3
627− smlal v18.4s, v6.4h, v29.4h
628− smlal v19.4s, v7.4h, v28.4h
629− subs w2, w2, #4
630− addp v18.4s, v18.4s, v19.4s
631− addp v16.4s, v16.4s, v18.4s
632− sshr v16.4s, v16.4s, #3
633− smin v16.4s, v16.4s, v20.4s
634−
635− st1 {v16.4s}, [x1], #16
636− add x4, x4, x7, lsl #2
637− b.gt 1b
610+ sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements
611+
612+ ldr s4, [x8] // load src values for idx 0
613+ ldr d31, [x12, x17] // load filter values for idx 0
614+ uxtl v4.8h, v4.8b // extend type to match the filter' size
615+ ldr s5, [x9] // load src values for idx 1
616+ smlal v16.4s, v4.4h, v31.4h
617+ ldr d30, [x13, x17] // load filter values for idx 1
618+ uxtl v5.8h, v5.8b // extend type to match the filter' size
619+ ldr s6, [x10] // load src values for idx 2
620+ smlal v17.4s, v5.4h, v30.4h
621+ uxtl v6.8h, v6.8b // extend type to match the filter's size
622+ ldr d29, [x14, x17] // load filter values for idx 2
623+ ldr s7, [x11] // load src values for idx 3
624+ addp v16.4s, v16.4s, v17.4s
625+ uxtl v7.8h, v7.8b
626+ ldr d28, [x15, x17] // load filter values for idx 3
627+ smlal v18.4s, v6.4h, v29.4h
628+ smlal v19.4s, v7.4h, v28.4h
629+ subs w2, w2, #4
630+ addp v18.4s, v18.4s, v19.4s
631+ addp v16.4s, v16.4s, v18.4s
632+ sshr v16.4s, v16.4s, #3
633+ smin v16.4s, v16.4s, v20.4s
634+
635+ st1 {v16.4s}, [x1], #16
636+ add x4, x4, x7, lsl #2
637+ b.gt 1b
638638 ret
639639 endfunc
640640
@@ -647,191 +647,191 @@ function ff_hscale16to15_4_neon_asm, export=1 · 6 unmodified lines
647647 // x5 const int32_t *filterPos
648648 // w6 int filterSize
649649
650− movi v18.4s, #1
651− movi v17.4s, #1
652− shl v18.4s, v18.4s, #15
653− sub v18.4s, v18.4s, v17.4s // max allowed value
654− dup v17.4s, w0 // read shift
655− neg v17.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)
650+ movi v18.4s, #1
651+ movi v17.4s, #1
652+ shl v18.4s, v18.4s, #15
653+ sub v18.4s, v18.4s, v17.4s // max allowed value
654+ dup v17.4s, w0 // read shift
655+ neg v17.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)
656656
657− cmp w2, #16
658− b.lt 2f // move to last block
657+ cmp w2, #16
658+ b.lt 2f // move to last block
659659
660− ldp w8, w9, [x5] // filterPos[0], filterPos[1]
661− ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
662− ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
663− ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
664− add x5, x5, #32
660+ ldp w8, w9, [x5] // filterPos[0], filterPos[1]
661+ ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
662+ ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
663+ ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
664+ add x5, x5, #32
665665
666666 // shift all filterPos left by one, as uint16_t will be read
667− lsl x8, x8, #1
668− lsl x9, x9, #1
669− lsl x10, x10, #1
670− lsl x11, x11, #1
671− lsl x12, x12, #1
672− lsl x13, x13, #1
673− lsl x14, x14, #1
674− lsl x15, x15, #1
667+ lsl x8, x8, #1
668+ lsl x9, x9, #1
669+ lsl x10, x10, #1
670+ lsl x11, x11, #1
671+ lsl x12, x12, #1
672+ lsl x13, x13, #1
673+ lsl x14, x14, #1
674+ lsl x15, x15, #1
675675
676676 // load src with given offset
677− ldr x8, [x3, w8, uxtw]
678− ldr x9, [x3, w9, uxtw]
679− ldr x10, [x3, w10, uxtw]
680− ldr x11, [x3, w11, uxtw]
681− ldr x12, [x3, w12, uxtw]
682− ldr x13, [x3, w13, uxtw]
683− ldr x14, [x3, w14, uxtw]
684− ldr x15, [x3, w15, uxtw]
685−
686− sub sp, sp, #64
677+ ldr x8, [x3, w8, uxtw]
678+ ldr x9, [x3, w9, uxtw]
679+ ldr x10, [x3, w10, uxtw]
680+ ldr x11, [x3, w11, uxtw]
681+ ldr x12, [x3, w12, uxtw]
682+ ldr x13, [x3, w13, uxtw]
683+ ldr x14, [x3, w14, uxtw]
684+ ldr x15, [x3, w15, uxtw]
685+
686+ sub sp, sp, #64
687687 // push src on stack so it can be loaded into vectors later
688− stp x8, x9, [sp]
689− stp x10, x11, [sp, #16]
690− stp x12, x13, [sp, #32]
691− stp x14, x15, [sp, #48]
688+ stp x8, x9, [sp]
689+ stp x10, x11, [sp, #16]
690+ stp x12, x13, [sp, #32]
691+ stp x14, x15, [sp, #48]
692692
693693 1:
694− ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]
695− ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]
694+ ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]
695+ ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]
696696
697697 // Each of blocks does the following:
698698 // Extend src and filter to 32 bits with uxtl and sxtl
699699 // multiply or multiply and accumulate results
700700 // Extending to 32 bits is necessary, as unit16_t values can't
701701 // be represented as int16_t without type promotion.
702− uxtl v26.4s, v0.4h
703− sxtl v27.4s, v28.4h
704− uxtl2 v0.4s, v0.8h
705− mul v5.4s, v26.4s, v27.4s
706− sxtl2 v28.4s, v28.8h
707− uxtl v26.4s, v1.4h
708− mul v6.4s, v0.4s, v28.4s
709−
710− sxtl v27.4s, v29.4h
711− uxtl2 v0.4s, v1.8h
712− mla v5.4s, v27.4s, v26.4s
713− sxtl2 v28.4s, v29.8h
714− uxtl v26.4s, v2.4h
715− mla v6.4s, v28.4s, v0.4s
716−
717− sxtl v27.4s, v30.4h
718− uxtl2 v0.4s, v2.8h
719− mla v5.4s, v27.4s, v26.4s
720− sxtl2 v28.4s, v30.8h
721− uxtl v26.4s, v3.4h
722− mla v6.4s, v28.4s, v0.4s
723−
724− sxtl v27.4s, v31.4h
725− uxtl2 v0.4s, v3.8h
726− mla v5.4s, v27.4s, v26.4s
727− sxtl2 v28.4s, v31.8h
728− sub w2, w2, #8
729− mla v6.4s, v28.4s, v0.4s
730−
731− sshl v5.4s, v5.4s, v17.4s
732− sshl v6.4s, v6.4s, v17.4s
733− smin v5.4s, v5.4s, v18.4s
734− smin v6.4s, v6.4s, v18.4s
735− xtn v5.4h, v5.4s
736− xtn2 v5.8h, v6.4s
737−
738− st1 {v5.8h}, [x1], #16
739− cmp w2, #16
702+ uxtl v26.4s, v0.4h
703+ sxtl v27.4s, v28.4h
704+ uxtl2 v0.4s, v0.8h
705+ mul v5.4s, v26.4s, v27.4s
706+ sxtl2 v28.4s, v28.8h
707+ uxtl v26.4s, v1.4h
708+ mul v6.4s, v0.4s, v28.4s
709+
710+ sxtl v27.4s, v29.4h
711+ uxtl2 v0.4s, v1.8h
712+ mla v5.4s, v27.4s, v26.4s
713+ sxtl2 v28.4s, v29.8h
714+ uxtl v26.4s, v2.4h
715+ mla v6.4s, v28.4s, v0.4s
716+
717+ sxtl v27.4s, v30.4h
718+ uxtl2 v0.4s, v2.8h
719+ mla v5.4s, v27.4s, v26.4s
720+ sxtl2 v28.4s, v30.8h
721+ uxtl v26.4s, v3.4h
722+ mla v6.4s, v28.4s, v0.4s
723+
724+ sxtl v27.4s, v31.4h
725+ uxtl2 v0.4s, v3.8h
726+ mla v5.4s, v27.4s, v26.4s
727+ sxtl2 v28.4s, v31.8h
728+ sub w2, w2, #8
729+ mla v6.4s, v28.4s, v0.4s
730+
731+ sshl v5.4s, v5.4s, v17.4s
732+ sshl v6.4s, v6.4s, v17.4s
733+ smin v5.4s, v5.4s, v18.4s
734+ smin v6.4s, v6.4s, v18.4s
735+ xtn v5.4h, v5.4s
736+ xtn2 v5.8h, v6.4s
737+
738+ st1 {v5.8h}, [x1], #16
739+ cmp w2, #16
740740
741741 // load filterPositions into registers for next iteration
742− ldp w8, w9, [x5] // filterPos[0], filterPos[1]
743− ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
744− ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
745− ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
746− add x5, x5, #32
747−
748− lsl x8, x8, #1
749− lsl x9, x9, #1
750− lsl x10, x10, #1
751− lsl x11, x11, #1
752− lsl x12, x12, #1
753− lsl x13, x13, #1
754− lsl x14, x14, #1
755− lsl x15, x15, #1
756−
757− ldr x8, [x3, w8, uxtw]
758− ldr x9, [x3, w9, uxtw]
759− ldr x10, [x3, w10, uxtw]
760− ldr x11, [x3, w11, uxtw]
761− ldr x12, [x3, w12, uxtw]
762− ldr x13, [x3, w13, uxtw]
763− ldr x14, [x3, w14, uxtw]
764− ldr x15, [x3, w15, uxtw]
765−
766− stp x8, x9, [sp]
767− stp x10, x11, [sp, #16]
768− stp x12, x13, [sp, #32]
769− stp x14, x15, [sp, #48]
770−
771− b.ge 1b
742+ ldp w8, w9, [x5] // filterPos[0], filterPos[1]
743+ ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
744+ ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
745+ ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
746+ add x5, x5, #32
747+
748+ lsl x8, x8, #1
749+ lsl x9, x9, #1
750+ lsl x10, x10, #1
751+ lsl x11, x11, #1
752+ lsl x12, x12, #1
753+ lsl x13, x13, #1
754+ lsl x14, x14, #1
755+ lsl x15, x15, #1
756+
757+ ldr x8, [x3, w8, uxtw]
758+ ldr x9, [x3, w9, uxtw]
759+ ldr x10, [x3, w10, uxtw]
760+ ldr x11, [x3, w11, uxtw]
761+ ldr x12, [x3, w12, uxtw]
762+ ldr x13, [x3, w13, uxtw]
763+ ldr x14, [x3, w14, uxtw]
764+ ldr x15, [x3, w15, uxtw]
765+
766+ stp x8, x9, [sp]
767+ stp x10, x11, [sp, #16]
768+ stp x12, x13, [sp, #32]
769+ stp x14, x15, [sp, #48]
770+
771+ b.ge 1b
772772
773773 // here we make last iteration, without updating the registers
774− ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]
775− ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64
776−
777− uxtl v26.4s, v0.4h
778− sxtl v27.4s, v28.4h
779− uxtl2 v0.4s, v0.8h
780− mul v5.4s, v26.4s, v27.4s
781− sxtl2 v28.4s, v28.8h
782− uxtl v26.4s, v1.4h
783− mul v6.4s, v0.4s, v28.4s
784−
785− sxtl v27.4s, v29.4h
786− uxtl2 v0.4s, v1.8h
787− mla v5.4s, v26.4s, v27.4s
788− sxtl2 v28.4s, v29.8h
789− uxtl v26.4s, v2.4h
790− mla v6.4s, v0.4s, v28.4s
791−
792− sxtl v27.4s, v30.4h
793− uxtl2 v0.4s, v2.8h
794− mla v5.4s, v26.4s, v27.4s
795− sxtl2 v28.4s, v30.8h
796− uxtl v26.4s, v3.4h
797− mla v6.4s, v0.4s, v28.4s
798−
799− sxtl v27.4s, v31.4h
800− uxtl2 v0.4s, v3.8h
801− mla v5.4s, v26.4s, v27.4s
802− sxtl2 v28.4s, v31.8h
803− subs w2, w2, #8
804− mla v6.4s, v0.4s, v28.4s
805−
806− sshl v5.4s, v5.4s, v17.4s
807− sshl v6.4s, v6.4s, v17.4s
808− smin v5.4s, v5.4s, v18.4s
809− smin v6.4s, v6.4s, v18.4s
810− xtn v5.4h, v5.4s
811− xtn2 v5.8h, v6.4s
812−
813− st1 {v5.8h}, [x1], #16
814− add sp, sp, #64 // restore stack
815− cbnz w2, 2f
774+ ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]
775+ ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64
776+
777+ uxtl v26.4s, v0.4h
778+ sxtl v27.4s, v28.4h
779+ uxtl2 v0.4s, v0.8h
780+ mul v5.4s, v26.4s, v27.4s
781+ sxtl2 v28.4s, v28.8h
782+ uxtl v26.4s, v1.4h
783+ mul v6.4s, v0.4s, v28.4s
784+
785+ sxtl v27.4s, v29.4h
786+ uxtl2 v0.4s, v1.8h
787+ mla v5.4s, v26.4s, v27.4s
788+ sxtl2 v28.4s, v29.8h
789+ uxtl v26.4s, v2.4h
790+ mla v6.4s, v0.4s, v28.4s
791+
792+ sxtl v27.4s, v30.4h
793+ uxtl2 v0.4s, v2.8h
794+ mla v5.4s, v26.4s, v27.4s
795+ sxtl2 v28.4s, v30.8h
796+ uxtl v26.4s, v3.4h
797+ mla v6.4s, v0.4s, v28.4s
798+
799+ sxtl v27.4s, v31.4h
800+ uxtl2 v0.4s, v3.8h
801+ mla v5.4s, v26.4s, v27.4s
802+ sxtl2 v28.4s, v31.8h
803+ subs w2, w2, #8
804+ mla v6.4s, v0.4s, v28.4s
805+
806+ sshl v5.4s, v5.4s, v17.4s
807+ sshl v6.4s, v6.4s, v17.4s
808+ smin v5.4s, v5.4s, v18.4s
809+ smin v6.4s, v6.4s, v18.4s
810+ xtn v5.4h, v5.4s
811+ xtn2 v5.8h, v6.4s
812+
813+ st1 {v5.8h}, [x1], #16
814+ add sp, sp, #64 // restore stack
815+ cbnz w2, 2f
816816
817817 ret
818818
819819 2:
820− ldr w8, [x5], #4 // load filterPos
821− lsl w8, w8, #1
822− add x9, x3, w8, uxtw // src + filterPos
823− ld1 {v0.4h}, [x9] // load 4 * uint16_t
824− ld1 {v31.4h}, [x4], #8
825−
826− uxtl v0.4s, v0.4h
827− sxtl v31.4s, v31.4h
828− mul v5.4s, v0.4s, v31.4s
829− addv s0, v5.4s
830− sshl v0.4s, v0.4s, v17.4s
831− smin v0.4s, v0.4s, v18.4s
832− st1 {v0.h}[0], [x1], #2
833− sub w2, w2, #1
834− cbnz w2, 2b // if iterations remain jump to beginning
820+ ldr w8, [x5], #4 // load filterPos
821+ lsl w8, w8, #1
822+ add x9, x3, w8, uxtw // src + filterPos
823+ ld1 {v0.4h}, [x9] // load 4 * uint16_t
824+ ld1 {v31.4h}, [x4], #8
825+
826+ uxtl v0.4s, v0.4h
827+ sxtl v31.4s, v31.4h
828+ mul v5.4s, v0.4s, v31.4s
829+ addv s0, v5.4s
830+ sshl v0.4s, v0.4s, v17.4s
831+ smin v0.4s, v0.4s, v18.4s
832+ st1 {v0.h}[0], [x1], #2
833+ sub w2, w2, #1
834+ cbnz w2, 2b // if iterations remain jump to beginning
835835
836836 ret
837837 endfunc
@@ -845,79 +845,79 @@ function ff_hscale16to15_X8_neon_asm, export=1 · 7 unmodified lines
845845 // x5 const int32_t *filterPos
846846 // w6 int filterSize
847847
848− movi v20.4s, #1
849− movi v21.4s, #1
850− shl v20.4s, v20.4s, #15
851− sub v20.4s, v20.4s, v21.4s
852− dup v21.4s, w0
853− neg v21.4s, v21.4s
854−
855− sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)
856−1: ldr w8, [x5], #4 // filterPos[idx]
857− lsl w8, w8, #1
858− ldr w10, [x5], #4 // filterPos[idx + 1]
859− lsl w10, w10, #1
860− ldr w11, [x5], #4 // filterPos[idx + 2]
861− lsl w11, w11, #1
862− ldr w9, [x5], #4 // filterPos[idx + 3]
863− lsl w9, w9, #1
864− mov x16, x4 // filter0 = filter
865− add x12, x16, x7 // filter1 = filter0 + filterSize*2
866− add x13, x12, x7 // filter2 = filter1 + filterSize*2
867− add x4, x13, x7 // filter3 = filter2 + filterSize*2
868− movi v0.2d, #0 // val sum part 1 (for dst[0])
869− movi v1.2d, #0 // val sum part 2 (for dst[1])
870− movi v2.2d, #0 // val sum part 3 (for dst[2])
871− movi v3.2d, #0 // val sum part 4 (for dst[3])
872− add x17, x3, w8, uxtw // srcp + filterPos[0]
873− add x8, x3, w10, uxtw // srcp + filterPos[1]
874− add x10, x3, w11, uxtw // srcp + filterPos[2]
875− add x11, x3, w9, uxtw // srcp + filterPos[3]
876− mov w15, w6 // filterSize counter
877−2: ld1 {v4.8h}, [x17], #16 // srcp[filterPos[0] + {0..7}]
878− ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1
879− ld1 {v6.8h}, [x8], #16 // srcp[filterPos[1] + {0..7}]
880− ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize
881− uxtl v24.4s, v4.4h // extend srcp lower half to 32 bits to preserve sign
882− sxtl v25.4s, v5.4h // extend filter lower half to 32 bits to match srcp size
883− uxtl2 v4.4s, v4.8h // extend srcp upper half to 32 bits
884− mla v0.4s, v24.4s, v25.4s // multiply accumulate lower half of v4 * v5
885− sxtl2 v5.4s, v5.8h // extend filter upper half to 32 bits
886− uxtl v26.4s, v6.4h // extend srcp lower half to 32 bits
887− mla v0.4s, v4.4s, v5.4s // multiply accumulate upper half of v4 * v5
888− sxtl v27.4s, v7.4h // exted filter lower half
889− uxtl2 v6.4s, v6.8h // extend srcp upper half
890− sxtl2 v7.4s, v7.8h // extend filter upper half
891− ld1 {v16.8h}, [x10], #16 // srcp[filterPos[2] + {0..7}]
892− mla v1.4s, v26.4s, v27.4s // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]
893− ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize
894− uxtl v22.4s, v16.4h // extend srcp lower half
895− sxtl v23.4s, v17.4h // extend filter lower half
896− uxtl2 v16.4s, v16.8h // extend srcp upper half
897− sxtl2 v17.4s, v17.8h // extend filter upper half
898− mla v2.4s, v22.4s, v23.4s // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]
899− mla v2.4s, v16.4s, v17.4s // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]
900− ld1 {v18.8h}, [x11], #16 // srcp[filterPos[3] + {0..7}]
901− mla v1.4s, v6.4s, v7.4s // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]
902− ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize
903− subs w15, w15, #8 // j -= 8: processed 8/filterSize
904− uxtl v28.4s, v18.4h // extend srcp lower half
905− sxtl v29.4s, v19.4h // extend filter lower half
906− uxtl2 v18.4s, v18.8h // extend srcp upper half
907− sxtl2 v19.4s, v19.8h // extend filter upper half
908− mla v3.4s, v28.4s, v29.4s // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]
909− mla v3.4s, v18.4s, v19.4s // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]
910− b.gt 2b // inner loop if filterSize not consumed completely
911− addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding
912− addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding
913− addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding
914− subs w2, w2, #4 // dstW -= 4
915− sshl v0.4s, v0.4s, v21.4s // shift right (effectively rigth, as shift is negative); overflow expected
916− smin v0.4s, v0.4s, v20.4s // apply min (do not use sqshl)
917− xtn v0.4h, v0.4s // narrow down to 16 bits
918−
919− st1 {v0.4h}, [x1], #8 // write to destination part0123
920− b.gt 1b // loop until end of line
848+ movi v20.4s, #1
849+ movi v21.4s, #1
850+ shl v20.4s, v20.4s, #15
851+ sub v20.4s, v20.4s, v21.4s
852+ dup v21.4s, w0
853+ neg v21.4s, v21.4s
854+
855+ sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)
856+1: ldr w8, [x5], #4 // filterPos[idx]
857+ lsl w8, w8, #1
858+ ldr w10, [x5], #4 // filterPos[idx + 1]
859+ lsl w10, w10, #1
860+ ldr w11, [x5], #4 // filterPos[idx + 2]
861+ lsl w11, w11, #1
862+ ldr w9, [x5], #4 // filterPos[idx + 3]
863+ lsl w9, w9, #1
864+ mov x16, x4 // filter0 = filter
865+ add x12, x16, x7 // filter1 = filter0 + filterSize*2
866+ add x13, x12, x7 // filter2 = filter1 + filterSize*2
867+ add x4, x13, x7 // filter3 = filter2 + filterSize*2
868+ movi v0.2d, #0 // val sum part 1 (for dst[0])
869+ movi v1.2d, #0 // val sum part 2 (for dst[1])
870+ movi v2.2d, #0 // val sum part 3 (for dst[2])
871+ movi v3.2d, #0 // val sum part 4 (for dst[3])
872+ add x17, x3, w8, uxtw // srcp + filterPos[0]
873+ add x8, x3, w10, uxtw // srcp + filterPos[1]
874+ add x10, x3, w11, uxtw // srcp + filterPos[2]
875+ add x11, x3, w9, uxtw // srcp + filterPos[3]
876+ mov w15, w6 // filterSize counter
877+2: ld1 {v4.8h}, [x17], #16 // srcp[filterPos[0] + {0..7}]
878+ ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1
879+ ld1 {v6.8h}, [x8], #16 // srcp[filterPos[1] + {0..7}]
880+ ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize
881+ uxtl v24.4s, v4.4h // extend srcp lower half to 32 bits to preserve sign
882+ sxtl v25.4s, v5.4h // extend filter lower half to 32 bits to match srcp size
883+ uxtl2 v4.4s, v4.8h // extend srcp upper half to 32 bits
884+ mla v0.4s, v24.4s, v25.4s // multiply accumulate lower half of v4 * v5
885+ sxtl2 v5.4s, v5.8h // extend filter upper half to 32 bits
886+ uxtl v26.4s, v6.4h // extend srcp lower half to 32 bits
887+ mla v0.4s, v4.4s, v5.4s // multiply accumulate upper half of v4 * v5
888+ sxtl v27.4s, v7.4h // exted filter lower half
889+ uxtl2 v6.4s, v6.8h // extend srcp upper half
890+ sxtl2 v7.4s, v7.8h // extend filter upper half
891+ ld1 {v16.8h}, [x10], #16 // srcp[filterPos[2] + {0..7}]
892+ mla v1.4s, v26.4s, v27.4s // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]
893+ ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize
894+ uxtl v22.4s, v16.4h // extend srcp lower half
895+ sxtl v23.4s, v17.4h // extend filter lower half
896+ uxtl2 v16.4s, v16.8h // extend srcp upper half
897+ sxtl2 v17.4s, v17.8h // extend filter upper half
898+ mla v2.4s, v22.4s, v23.4s // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]
899+ mla v2.4s, v16.4s, v17.4s // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]
900+ ld1 {v18.8h}, [x11], #16 // srcp[filterPos[3] + {0..7}]
901+ mla v1.4s, v6.4s, v7.4s // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]
902+ ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize
903+ subs w15, w15, #8 // j -= 8: processed 8/filterSize
904+ uxtl v28.4s, v18.4h // extend srcp lower half
905+ sxtl v29.4s, v19.4h // extend filter lower half
906+ uxtl2 v18.4s, v18.8h // extend srcp upper half
907+ sxtl2 v19.4s, v19.8h // extend filter upper half
908+ mla v3.4s, v28.4s, v29.4s // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]
909+ mla v3.4s, v18.4s, v19.4s // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]
910+ b.gt 2b // inner loop if filterSize not consumed completely
911+ addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding
912+ addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding
913+ addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding
914+ subs w2, w2, #4 // dstW -= 4
915+ sshl v0.4s, v0.4s, v21.4s // shift right (effectively rigth, as shift is negative); overflow expected
916+ smin v0.4s, v0.4s, v20.4s // apply min (do not use sqshl)
917+ xtn v0.4h, v0.4s // narrow down to 16 bits
918+
919+ st1 {v0.4h}, [x1], #8 // write to destination part0123
920+ b.gt 1b // loop until end of line
921921 ret
922922 endfunc
923923
@@ -930,118 +930,118 @@ function ff_hscale16to15_X4_neon_asm, export=1 · 6 unmodified lines
930930 // x5 const int32_t *filterPos
931931 // w6 int filterSize
932932
933− stp d8, d9, [sp, #-0x20]!
934− stp d10, d11, [sp, #0x10]
933+ stp d8, d9, [sp, #-0x20]!
934+ stp d10, d11, [sp, #0x10]
935935
936− movi v18.4s, #1
937− movi v17.4s, #1
938− shl v18.4s, v18.4s, #15
939− sub v21.4s, v18.4s, v17.4s // max allowed value
940− dup v17.4s, w0 // read shift
941− neg v20.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)
936+ movi v18.4s, #1
937+ movi v17.4s, #1
938+ shl v18.4s, v18.4s, #15
939+ sub v21.4s, v18.4s, v17.4s // max allowed value
940+ dup v17.4s, w0 // read shift
941+ neg v20.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)
942942
943− lsl w7, w6, #1
943+ lsl w7, w6, #1
944944 1:
945− ldp w8, w9, [x5]
946− ldp w10, w11, [x5, #8]
947−
948− movi v16.2d, #0 // initialize accumulator for idx + 0
949− movi v17.2d, #0 // initialize accumulator for idx + 1
950− movi v18.2d, #0 // initialize accumulator for idx + 2
951− movi v19.2d, #0 // initialize accumulator for idx + 3
952−
953− mov x12, x4 // filter + 0
954− add x13, x4, x7 // filter + 1
955− add x8, x3, x8, lsl #1 // srcp + filterPos 0
956− add x14, x13, x7 // filter + 2
957− add x9, x3, x9, lsl #1 // srcp + filterPos 1
958− add x15, x14, x7 // filter + 3
959− add x10, x3, x10, lsl #1 // srcp + filterPos 2
960− mov w0, w6 // save the filterSize to temporary variable
961− add x11, x3, x11, lsl #1 // srcp + filterPos 3
962− add x5, x5, #16 // advance filter position
963− mov x16, xzr // clear the register x16 used for offsetting the filter values
945+ ldp w8, w9, [x5]
946+ ldp w10, w11, [x5, #8]
947+
948+ movi v16.2d, #0 // initialize accumulator for idx + 0
949+ movi v17.2d, #0 // initialize accumulator for idx + 1
950+ movi v18.2d, #0 // initialize accumulator for idx + 2
951+ movi v19.2d, #0 // initialize accumulator for idx + 3
952+
953+ mov x12, x4 // filter + 0
954+ add x13, x4, x7 // filter + 1
955+ add x8, x3, x8, lsl #1 // srcp + filterPos 0
956+ add x14, x13, x7 // filter + 2
957+ add x9, x3, x9, lsl #1 // srcp + filterPos 1
958+ add x15, x14, x7 // filter + 3
959+ add x10, x3, x10, lsl #1 // srcp + filterPos 2
960+ mov w0, w6 // save the filterSize to temporary variable
961+ add x11, x3, x11, lsl #1 // srcp + filterPos 3
962+ add x5, x5, #16 // advance filter position
963+ mov x16, xzr // clear the register x16 used for offsetting the filter values
964964
965965 2:
966− ldr q4, [x8], #16 // load src values for idx 0
967− ldr q5, [x9], #16 // load src values for idx 1
968− uxtl v26.4s, v4.4h
969− uxtl2 v4.4s, v4.8h
970− ldr q31, [x12, x16] // load filter values for idx 0
971− ldr q6, [x10], #16 // load src values for idx 2
972− sxtl v22.4s, v31.4h
973− sxtl2 v31.4s, v31.8h
974− mla v16.4s, v26.4s, v22.4s // multiplication of lower half for idx 0
975− uxtl v25.4s, v5.4h
976− uxtl2 v5.4s, v5.8h
977− ldr q30, [x13, x16] // load filter values for idx 1
978− ldr q7, [x11], #16 // load src values for idx 3
979− mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0
980− uxtl v24.4s, v6.4h
981− sxtl v8.4s, v30.4h
982− sxtl2 v30.4s, v30.8h
983− mla v17.4s, v25.4s, v8.4s // multiplication of lower half for idx 1
984− ldr q29, [x14, x16] // load filter values for idx 2
985− uxtl2 v6.4s, v6.8h
986− sxtl v9.4s, v29.4h
987− sxtl2 v29.4s, v29.8h
988− mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1
989− mla v18.4s, v24.4s, v9.4s // multiplication of lower half for idx 2
990− ldr q28, [x15, x16] // load filter values for idx 3
991− uxtl v23.4s, v7.4h
992− sxtl v10.4s, v28.4h
993− mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2
994− uxtl2 v7.4s, v7.8h
995− sxtl2 v28.4s, v28.8h
996− mla v19.4s, v23.4s, v10.4s // multiplication of lower half for idx 3
997− sub w0, w0, #8
998− cmp w0, #8
999− mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3
1000−
1001− add x16, x16, #16 // advance filter values indexing
1002−
1003− b.ge 2b
966+ ldr q4, [x8], #16 // load src values for idx 0
967+ ldr q5, [x9], #16 // load src values for idx 1
968+ uxtl v26.4s, v4.4h
969+ uxtl2 v4.4s, v4.8h
970+ ldr q31, [x12, x16] // load filter values for idx 0
971+ ldr q6, [x10], #16 // load src values for idx 2
972+ sxtl v22.4s, v31.4h
973+ sxtl2 v31.4s, v31.8h
974+ mla v16.4s, v26.4s, v22.4s // multiplication of lower half for idx 0
975+ uxtl v25.4s, v5.4h
976+ uxtl2 v5.4s, v5.8h
977+ ldr q30, [x13, x16] // load filter values for idx 1
978+ ldr q7, [x11], #16 // load src values for idx 3
979+ mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0
980+ uxtl v24.4s, v6.4h
981+ sxtl v8.4s, v30.4h
982+ sxtl2 v30.4s, v30.8h
983+ mla v17.4s, v25.4s, v8.4s // multiplication of lower half for idx 1
984+ ldr q29, [x14, x16] // load filter values for idx 2
985+ uxtl2 v6.4s, v6.8h
986+ sxtl v9.4s, v29.4h
987+ sxtl2 v29.4s, v29.8h
988+ mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1
989+ mla v18.4s, v24.4s, v9.4s // multiplication of lower half for idx 2
990+ ldr q28, [x15, x16] // load filter values for idx 3
991+ uxtl v23.4s, v7.4h
992+ sxtl v10.4s, v28.4h
993+ mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2
994+ uxtl2 v7.4s, v7.8h
995+ sxtl2 v28.4s, v28.8h
996+ mla v19.4s, v23.4s, v10.4s // multiplication of lower half for idx 3
997+ sub w0, w0, #8
998+ cmp w0, #8
999+ mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3
1000+
1001+ add x16, x16, #16 // advance filter values indexing
1002+
1003+ b.ge 2b
10041004
10051005 // 4 iterations left
10061006
1007− sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements
1008−
1009− ldr d4, [x8] // load src values for idx 0
1010− ldr d31, [x12, x17] // load filter values for idx 0
1011− uxtl v4.4s, v4.4h
1012− sxtl v31.4s, v31.4h
1013− ldr d5, [x9] // load src values for idx 1
1014− mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0
1015− ldr d30, [x13, x17] // load filter values for idx 1
1016− uxtl v5.4s, v5.4h
1017− sxtl v30.4s, v30.4h
1018− ldr d6, [x10] // load src values for idx 2
1019− mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1
1020− ldr d29, [x14, x17] // load filter values for idx 2
1021− uxtl v6.4s, v6.4h
1022− sxtl v29.4s, v29.4h
1023− ldr d7, [x11] // load src values for idx 3
1024− ldr d28, [x15, x17] // load filter values for idx 3
1025− mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2
1026− uxtl v7.4s, v7.4h
1027− sxtl v28.4s, v28.4h
1028− addp v16.4s, v16.4s, v17.4s
1029− mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3
1030− subs w2, w2, #4
1031− addp v18.4s, v18.4s, v19.4s
1032− addp v16.4s, v16.4s, v18.4s
1033− sshl v16.4s, v16.4s, v20.4s
1034− smin v16.4s, v16.4s, v21.4s
1035− xtn v16.4h, v16.4s
1036−
1037− st1 {v16.4h}, [x1], #8
1038− add x4, x4, x7, lsl #2
1039− b.gt 1b
1040−
1041− ldp d8, d9, [sp]
1042− ldp d10, d11, [sp, #0x10]
1043−
1044− add sp, sp, #0x20
1007+ sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements
1008+
1009+ ldr d4, [x8] // load src values for idx 0
1010+ ldr d31, [x12, x17] // load filter values for idx 0
1011+ uxtl v4.4s, v4.4h
1012+ sxtl v31.4s, v31.4h
1013+ ldr d5, [x9] // load src values for idx 1
1014+ mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0
1015+ ldr d30, [x13, x17] // load filter values for idx 1
1016+ uxtl v5.4s, v5.4h
1017+ sxtl v30.4s, v30.4h
1018+ ldr d6, [x10] // load src values for idx 2
1019+ mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1
1020+ ldr d29, [x14, x17] // load filter values for idx 2
1021+ uxtl v6.4s, v6.4h
1022+ sxtl v29.4s, v29.4h
1023+ ldr d7, [x11] // load src values for idx 3
1024+ ldr d28, [x15, x17] // load filter values for idx 3
1025+ mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2
1026+ uxtl v7.4s, v7.4h
1027+ sxtl v28.4s, v28.4h
1028+ addp v16.4s, v16.4s, v17.4s
1029+ mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3
1030+ subs w2, w2, #4
1031+ addp v18.4s, v18.4s, v19.4s
1032+ addp v16.4s, v16.4s, v18.4s
1033+ sshl v16.4s, v16.4s, v20.4s
1034+ smin v16.4s, v16.4s, v21.4s
1035+ xtn v16.4h, v16.4s
1036+
1037+ st1 {v16.4h}, [x1], #8
1038+ add x4, x4, x7, lsl #2
1039+ b.gt 1b
1040+
1041+ ldp d8, d9, [sp]
1042+ ldp d10, d11, [sp, #0x10]
1043+
1044+ add sp, sp, #0x20
10451045
10461046 ret
10471047 endfunc
@@ -1055,188 +1055,188 @@ function ff_hscale16to19_4_neon_asm, export=1 · 7 unmodified lines
10551055 // x5 const int32_t *filterPos
10561056 // w6 int filterSize
10571057
1058− movi v18.4s, #1
1059− movi v17.4s, #1
1060− shl v18.4s, v18.4s, #19
1061− sub v18.4s, v18.4s, v17.4s // max allowed value
1062− dup v17.4s, w0 // read shift
1063− neg v17.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)
1058+ movi v18.4s, #1
1059+ movi v17.4s, #1
1060+ shl v18.4s, v18.4s, #19
1061+ sub v18.4s, v18.4s, v17.4s // max allowed value
1062+ dup v17.4s, w0 // read shift
1063+ neg v17.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)
10641064
1065− cmp w2, #16
1066− b.lt 2f // move to last block
1065+ cmp w2, #16
1066+ b.lt 2f // move to last block
10671067
1068− ldp w8, w9, [x5] // filterPos[0], filterPos[1]
1069− ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
1070− ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
1071− ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
1072− add x5, x5, #32
1068+ ldp w8, w9, [x5] // filterPos[0], filterPos[1]
1069+ ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
1070+ ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
1071+ ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
1072+ add x5, x5, #32
10731073
10741074 // shift all filterPos left by one, as uint16_t will be read
1075− lsl x8, x8, #1
1076− lsl x9, x9, #1
1077− lsl x10, x10, #1
1078− lsl x11, x11, #1
1079− lsl x12, x12, #1
1080− lsl x13, x13, #1
1081− lsl x14, x14, #1
1082− lsl x15, x15, #1
1075+ lsl x8, x8, #1
1076+ lsl x9, x9, #1
1077+ lsl x10, x10, #1
1078+ lsl x11, x11, #1
1079+ lsl x12, x12, #1
1080+ lsl x13, x13, #1
1081+ lsl x14, x14, #1
1082+ lsl x15, x15, #1
10831083
10841084 // load src with given offset
1085− ldr x8, [x3, w8, uxtw]
1086− ldr x9, [x3, w9, uxtw]
1087− ldr x10, [x3, w10, uxtw]
1088− ldr x11, [x3, w11, uxtw]
1089− ldr x12, [x3, w12, uxtw]
1090− ldr x13, [x3, w13, uxtw]
1091− ldr x14, [x3, w14, uxtw]
1092− ldr x15, [x3, w15, uxtw]
1093−
1094− sub sp, sp, #64
1085+ ldr x8, [x3, w8, uxtw]
1086+ ldr x9, [x3, w9, uxtw]
1087+ ldr x10, [x3, w10, uxtw]
1088+ ldr x11, [x3, w11, uxtw]
1089+ ldr x12, [x3, w12, uxtw]
1090+ ldr x13, [x3, w13, uxtw]
1091+ ldr x14, [x3, w14, uxtw]
1092+ ldr x15, [x3, w15, uxtw]
1093+
1094+ sub sp, sp, #64
10951095 // push src on stack so it can be loaded into vectors later
1096− stp x8, x9, [sp]
1097− stp x10, x11, [sp, #16]
1098− stp x12, x13, [sp, #32]
1099− stp x14, x15, [sp, #48]
1096+ stp x8, x9, [sp]
1097+ stp x10, x11, [sp, #16]
1098+ stp x12, x13, [sp, #32]
1099+ stp x14, x15, [sp, #48]
11001100
11011101 1:
1102− ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]
1103− ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]
1102+ ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]
1103+ ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]
11041104
11051105 // Each of blocks does the following:
11061106 // Extend src and filter to 32 bits with uxtl and sxtl
11071107 // multiply or multiply and accumulate results
11081108 // Extending to 32 bits is necessary, as unit16_t values can't
11091109 // be represented as int16_t without type promotion.
1110− uxtl v26.4s, v0.4h
1111− sxtl v27.4s, v28.4h
1112− uxtl2 v0.4s, v0.8h
1113− mul v5.4s, v26.4s, v27.4s
1114− sxtl2 v28.4s, v28.8h
1115− uxtl v26.4s, v1.4h
1116− mul v6.4s, v0.4s, v28.4s
1117−
1118− sxtl v27.4s, v29.4h
1119− uxtl2 v0.4s, v1.8h
1120− mla v5.4s, v27.4s, v26.4s
1121− sxtl2 v28.4s, v29.8h
1122− uxtl v26.4s, v2.4h
1123− mla v6.4s, v28.4s, v0.4s
1124−
1125− sxtl v27.4s, v30.4h
1126− uxtl2 v0.4s, v2.8h
1127− mla v5.4s, v27.4s, v26.4s
1128− sxtl2 v28.4s, v30.8h
1129− uxtl v26.4s, v3.4h
1130− mla v6.4s, v28.4s, v0.4s
1131−
1132− sxtl v27.4s, v31.4h
1133− uxtl2 v0.4s, v3.8h
1134− mla v5.4s, v27.4s, v26.4s
1135− sxtl2 v28.4s, v31.8h
1136− sub w2, w2, #8
1137− mla v6.4s, v28.4s, v0.4s
1138−
1139− sshl v5.4s, v5.4s, v17.4s
1140− sshl v6.4s, v6.4s, v17.4s
1141− smin v5.4s, v5.4s, v18.4s
1142− smin v6.4s, v6.4s, v18.4s
1143−
1144− st1 {v5.4s, v6.4s}, [x1], #32
1145− cmp w2, #16
1110+ uxtl v26.4s, v0.4h
1111+ sxtl v27.4s, v28.4h
1112+ uxtl2 v0.4s, v0.8h
1113+ mul v5.4s, v26.4s, v27.4s
1114+ sxtl2 v28.4s, v28.8h
1115+ uxtl v26.4s, v1.4h
1116+ mul v6.4s, v0.4s, v28.4s
1117+
1118+ sxtl v27.4s, v29.4h
1119+ uxtl2 v0.4s, v1.8h
1120+ mla v5.4s, v27.4s, v26.4s
1121+ sxtl2 v28.4s, v29.8h
1122+ uxtl v26.4s, v2.4h
1123+ mla v6.4s, v28.4s, v0.4s
1124+
1125+ sxtl v27.4s, v30.4h
1126+ uxtl2 v0.4s, v2.8h
1127+ mla v5.4s, v27.4s, v26.4s
1128+ sxtl2 v28.4s, v30.8h
1129+ uxtl v26.4s, v3.4h
1130+ mla v6.4s, v28.4s, v0.4s
1131+
1132+ sxtl v27.4s, v31.4h
1133+ uxtl2 v0.4s, v3.8h
1134+ mla v5.4s, v27.4s, v26.4s
1135+ sxtl2 v28.4s, v31.8h
1136+ sub w2, w2, #8
1137+ mla v6.4s, v28.4s, v0.4s
1138+
1139+ sshl v5.4s, v5.4s, v17.4s
1140+ sshl v6.4s, v6.4s, v17.4s
1141+ smin v5.4s, v5.4s, v18.4s
1142+ smin v6.4s, v6.4s, v18.4s
1143+
1144+ st1 {v5.4s, v6.4s}, [x1], #32
1145+ cmp w2, #16
11461146
11471147 // load filterPositions into registers for next iteration
1148− ldp w8, w9, [x5] // filterPos[0], filterPos[1]
1149− ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
1150− ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
1151− ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
1152− add x5, x5, #32
1153−
1154− lsl x8, x8, #1
1155− lsl x9, x9, #1
1156− lsl x10, x10, #1
1157− lsl x11, x11, #1
1158− lsl x12, x12, #1
1159− lsl x13, x13, #1
1160− lsl x14, x14, #1
1161− lsl x15, x15, #1
1162−
1163− ldr x8, [x3, w8, uxtw]
1164− ldr x9, [x3, w9, uxtw]
1165− ldr x10, [x3, w10, uxtw]
1166− ldr x11, [x3, w11, uxtw]
1167− ldr x12, [x3, w12, uxtw]
1168− ldr x13, [x3, w13, uxtw]
1169− ldr x14, [x3, w14, uxtw]
1170− ldr x15, [x3, w15, uxtw]
1171−
1172− stp x8, x9, [sp]
1173− stp x10, x11, [sp, #16]
1174− stp x12, x13, [sp, #32]
1175− stp x14, x15, [sp, #48]
1176−
1177− b.ge 1b
1148+ ldp w8, w9, [x5] // filterPos[0], filterPos[1]
1149+ ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
1150+ ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
1151+ ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
1152+ add x5, x5, #32
1153+
1154+ lsl x8, x8, #1
1155+ lsl x9, x9, #1
1156+ lsl x10, x10, #1
1157+ lsl x11, x11, #1
1158+ lsl x12, x12, #1
1159+ lsl x13, x13, #1
1160+ lsl x14, x14, #1
1161+ lsl x15, x15, #1
1162+
1163+ ldr x8, [x3, w8, uxtw]
1164+ ldr x9, [x3, w9, uxtw]
1165+ ldr x10, [x3, w10, uxtw]
1166+ ldr x11, [x3, w11, uxtw]
1167+ ldr x12, [x3, w12, uxtw]
1168+ ldr x13, [x3, w13, uxtw]
1169+ ldr x14, [x3, w14, uxtw]
1170+ ldr x15, [x3, w15, uxtw]
1171+
1172+ stp x8, x9, [sp]
1173+ stp x10, x11, [sp, #16]
1174+ stp x12, x13, [sp, #32]
1175+ stp x14, x15, [sp, #48]
1176+
1177+ b.ge 1b
11781178
11791179 // here we make last iteration, without updating the registers
1180− ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]
1181− ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64
1182−
1183− uxtl v26.4s, v0.4h
1184− sxtl v27.4s, v28.4h
1185− uxtl2 v0.4s, v0.8h
1186− mul v5.4s, v26.4s, v27.4s
1187− sxtl2 v28.4s, v28.8h
1188− uxtl v26.4s, v1.4h
1189− mul v6.4s, v0.4s, v28.4s
1190−
1191− sxtl v27.4s, v29.4h
1192− uxtl2 v0.4s, v1.8h
1193− mla v5.4s, v26.4s, v27.4s
1194− sxtl2 v28.4s, v29.8h
1195− uxtl v26.4s, v2.4h
1196− mla v6.4s, v0.4s, v28.4s
1197−
1198− sxtl v27.4s, v30.4h
1199− uxtl2 v0.4s, v2.8h
1200− mla v5.4s, v26.4s, v27.4s
1201− sxtl2 v28.4s, v30.8h
1202− uxtl v26.4s, v3.4h
1203− mla v6.4s, v0.4s, v28.4s
1204−
1205− sxtl v27.4s, v31.4h
1206− uxtl2 v0.4s, v3.8h
1207− mla v5.4s, v26.4s, v27.4s
1208− sxtl2 v28.4s, v31.8h
1209− subs w2, w2, #8
1210− mla v6.4s, v0.4s, v28.4s
1211−
1212− sshl v5.4s, v5.4s, v17.4s
1213− sshl v6.4s, v6.4s, v17.4s
1214−
1215− smin v5.4s, v5.4s, v18.4s
1216− smin v6.4s, v6.4s, v18.4s
1217−
1218− st1 {v5.4s, v6.4s}, [x1], #32
1219− add sp, sp, #64 // restore stack
1220− cbnz w2, 2f
1180+ ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]
1181+ ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64
1182+
1183+ uxtl v26.4s, v0.4h
1184+ sxtl v27.4s, v28.4h
1185+ uxtl2 v0.4s, v0.8h
1186+ mul v5.4s, v26.4s, v27.4s
1187+ sxtl2 v28.4s, v28.8h
1188+ uxtl v26.4s, v1.4h
1189+ mul v6.4s, v0.4s, v28.4s
1190+
1191+ sxtl v27.4s, v29.4h
1192+ uxtl2 v0.4s, v1.8h
1193+ mla v5.4s, v26.4s, v27.4s
1194+ sxtl2 v28.4s, v29.8h
1195+ uxtl v26.4s, v2.4h
1196+ mla v6.4s, v0.4s, v28.4s
1197+
1198+ sxtl v27.4s, v30.4h
1199+ uxtl2 v0.4s, v2.8h
1200+ mla v5.4s, v26.4s, v27.4s
1201+ sxtl2 v28.4s, v30.8h
1202+ uxtl v26.4s, v3.4h
1203+ mla v6.4s, v0.4s, v28.4s
1204+
1205+ sxtl v27.4s, v31.4h
1206+ uxtl2 v0.4s, v3.8h
1207+ mla v5.4s, v26.4s, v27.4s
1208+ sxtl2 v28.4s, v31.8h
1209+ subs w2, w2, #8
1210+ mla v6.4s, v0.4s, v28.4s
1211+
1212+ sshl v5.4s, v5.4s, v17.4s
1213+ sshl v6.4s, v6.4s, v17.4s
1214+
1215+ smin v5.4s, v5.4s, v18.4s
1216+ smin v6.4s, v6.4s, v18.4s
1217+
1218+ st1 {v5.4s, v6.4s}, [x1], #32
1219+ add sp, sp, #64 // restore stack
1220+ cbnz w2, 2f
12211221
12221222 ret
12231223
12241224 2:
1225− ldr w8, [x5], #4 // load filterPos
1226− lsl w8, w8, #1
1227− add x9, x3, w8, uxtw // src + filterPos
1228− ld1 {v0.4h}, [x9] // load 4 * uint16_t
1229− ld1 {v31.4h}, [x4], #8
1230−
1231− uxtl v0.4s, v0.4h
1232− sxtl v31.4s, v31.4h
1233− subs w2, w2, #1
1234− mul v5.4s, v0.4s, v31.4s
1235− addv s0, v5.4s
1236− sshl v0.4s, v0.4s, v17.4s
1237− smin v0.4s, v0.4s, v18.4s
1238− st1 {v0.s}[0], [x1], #4
1239− cbnz w2, 2b // if iterations remain jump to beginning
1225+ ldr w8, [x5], #4 // load filterPos
1226+ lsl w8, w8, #1
1227+ add x9, x3, w8, uxtw // src + filterPos
1228+ ld1 {v0.4h}, [x9] // load 4 * uint16_t
1229+ ld1 {v31.4h}, [x4], #8
1230+
1231+ uxtl v0.4s, v0.4h
1232+ sxtl v31.4s, v31.4h
1233+ subs w2, w2, #1
1234+ mul v5.4s, v0.4s, v31.4s
1235+ addv s0, v5.4s
1236+ sshl v0.4s, v0.4s, v17.4s
1237+ smin v0.4s, v0.4s, v18.4s
1238+ st1 {v0.s}[0], [x1], #4
1239+ cbnz w2, 2b // if iterations remain jump to beginning
12401240
12411241 ret
12421242 endfunc
@@ -1250,77 +1250,77 @@ function ff_hscale16to19_X8_neon_asm, export=1 · 7 unmodified lines
12501250 // x5 const int32_t *filterPos
12511251 // w6 int filterSize
12521252
1253− movi v20.4s, #1
1254− movi v21.4s, #1
1255− shl v20.4s, v20.4s, #19
1256− sub v20.4s, v20.4s, v21.4s
1257− dup v21.4s, w0
1258− neg v21.4s, v21.4s
1259−
1260− sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)
1261−1: ldr w8, [x5], #4 // filterPos[idx]
1262− ldr w10, [x5], #4 // filterPos[idx + 1]
1263− lsl w8, w8, #1
1264− ldr w11, [x5], #4 // filterPos[idx + 2]
1265− ldr w9, [x5], #4 // filterPos[idx + 3]
1266− mov x16, x4 // filter0 = filter
1267− lsl w11, w11, #1
1268− add x12, x16, x7 // filter1 = filter0 + filterSize*2
1269− lsl w9, w9, #1
1270− add x13, x12, x7 // filter2 = filter1 + filterSize*2
1271− lsl w10, w10, #1
1272− add x4, x13, x7 // filter3 = filter2 + filterSize*2
1273− movi v0.2d, #0 // val sum part 1 (for dst[0])
1274− movi v1.2d, #0 // val sum part 2 (for dst[1])
1275− movi v2.2d, #0 // val sum part 3 (for dst[2])
1276− movi v3.2d, #0 // val sum part 4 (for dst[3])
1277− add x17, x3, w8, uxtw // srcp + filterPos[0]
1278− add x8, x3, w10, uxtw // srcp + filterPos[1]
1279− add x10, x3, w11, uxtw // srcp + filterPos[2]
1280− add x11, x3, w9, uxtw // srcp + filterPos[3]
1281− mov w15, w6 // filterSize counter
1282−2: ld1 {v4.8h}, [x17], #16 // srcp[filterPos[0] + {0..7}]
1283− ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1
1284− ld1 {v6.8h}, [x8], #16 // srcp[filterPos[1] + {0..7}]
1285− ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize
1286− uxtl v24.4s, v4.4h // extend srcp lower half to 32 bits to preserve sign
1287− sxtl v25.4s, v5.4h // extend filter lower half to 32 bits to match srcp size
1288− uxtl2 v4.4s, v4.8h // extend srcp upper half to 32 bits
1289− mla v0.4s, v24.4s, v25.4s // multiply accumulate lower half of v4 * v5
1290− sxtl2 v5.4s, v5.8h // extend filter upper half to 32 bits
1291− uxtl v26.4s, v6.4h // extend srcp lower half to 32 bits
1292− mla v0.4s, v4.4s, v5.4s // multiply accumulate upper half of v4 * v5
1293− sxtl v27.4s, v7.4h // exted filter lower half
1294− uxtl2 v6.4s, v6.8h // extend srcp upper half
1295− sxtl2 v7.4s, v7.8h // extend filter upper half
1296− ld1 {v16.8h}, [x10], #16 // srcp[filterPos[2] + {0..7}]
1297− mla v1.4s, v26.4s, v27.4s // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]
1298− ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize
1299− uxtl v22.4s, v16.4h // extend srcp lower half
1300− sxtl v23.4s, v17.4h // extend filter lower half
1301− uxtl2 v16.4s, v16.8h // extend srcp upper half
1302− sxtl2 v17.4s, v17.8h // extend filter upper half
1303− mla v2.4s, v22.4s, v23.4s // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]
1304− mla v2.4s, v16.4s, v17.4s // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]
1305− ld1 {v18.8h}, [x11], #16 // srcp[filterPos[3] + {0..7}]
1306− mla v1.4s, v6.4s, v7.4s // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]
1307− ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize
1308− subs w15, w15, #8 // j -= 8: processed 8/filterSize
1309− uxtl v28.4s, v18.4h // extend srcp lower half
1310− sxtl v29.4s, v19.4h // extend filter lower half
1311− uxtl2 v18.4s, v18.8h // extend srcp upper half
1312− sxtl2 v19.4s, v19.8h // extend filter upper half
1313− mla v3.4s, v28.4s, v29.4s // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]
1314− mla v3.4s, v18.4s, v19.4s // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]
1315− b.gt 2b // inner loop if filterSize not consumed completely
1316− addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding
1317− addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding
1318− addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding
1319− subs w2, w2, #4 // dstW -= 4
1320− sshl v0.4s, v0.4s, v21.4s // shift right (effectively rigth, as shift is negative); overflow expected
1321− smin v0.4s, v0.4s, v20.4s // apply min (do not use sqshl)
1322− st1 {v0.4s}, [x1], #16 // write to destination part0123
1323− b.gt 1b // loop until end of line
1253+ movi v20.4s, #1
1254+ movi v21.4s, #1
1255+ shl v20.4s, v20.4s, #19
1256+ sub v20.4s, v20.4s, v21.4s
1257+ dup v21.4s, w0
1258+ neg v21.4s, v21.4s
1259+
1260+ sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)
1261+1: ldr w8, [x5], #4 // filterPos[idx]
1262+ ldr w10, [x5], #4 // filterPos[idx + 1]
1263+ lsl w8, w8, #1
1264+ ldr w11, [x5], #4 // filterPos[idx + 2]
1265+ ldr w9, [x5], #4 // filterPos[idx + 3]
1266+ mov x16, x4 // filter0 = filter
1267+ lsl w11, w11, #1
1268+ add x12, x16, x7 // filter1 = filter0 + filterSize*2
1269+ lsl w9, w9, #1
1270+ add x13, x12, x7 // filter2 = filter1 + filterSize*2
1271+ lsl w10, w10, #1
1272+ add x4, x13, x7 // filter3 = filter2 + filterSize*2
1273+ movi v0.2d, #0 // val sum part 1 (for dst[0])
1274+ movi v1.2d, #0 // val sum part 2 (for dst[1])
1275+ movi v2.2d, #0 // val sum part 3 (for dst[2])
1276+ movi v3.2d, #0 // val sum part 4 (for dst[3])
1277+ add x17, x3, w8, uxtw // srcp + filterPos[0]
1278+ add x8, x3, w10, uxtw // srcp + filterPos[1]
1279+ add x10, x3, w11, uxtw // srcp + filterPos[2]
1280+ add x11, x3, w9, uxtw // srcp + filterPos[3]
1281+ mov w15, w6 // filterSize counter
1282+2: ld1 {v4.8h}, [x17], #16 // srcp[filterPos[0] + {0..7}]
1283+ ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1
1284+ ld1 {v6.8h}, [x8], #16 // srcp[filterPos[1] + {0..7}]
1285+ ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize
1286+ uxtl v24.4s, v4.4h // extend srcp lower half to 32 bits to preserve sign
1287+ sxtl v25.4s, v5.4h // extend filter lower half to 32 bits to match srcp size
1288+ uxtl2 v4.4s, v4.8h // extend srcp upper half to 32 bits
1289+ mla v0.4s, v24.4s, v25.4s // multiply accumulate lower half of v4 * v5
1290+ sxtl2 v5.4s, v5.8h // extend filter upper half to 32 bits
1291+ uxtl v26.4s, v6.4h // extend srcp lower half to 32 bits
1292+ mla v0.4s, v4.4s, v5.4s // multiply accumulate upper half of v4 * v5
1293+ sxtl v27.4s, v7.4h // exted filter lower half
1294+ uxtl2 v6.4s, v6.8h // extend srcp upper half
1295+ sxtl2 v7.4s, v7.8h // extend filter upper half
1296+ ld1 {v16.8h}, [x10], #16 // srcp[filterPos[2] + {0..7}]
1297+ mla v1.4s, v26.4s, v27.4s // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]
1298+ ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize
1299+ uxtl v22.4s, v16.4h // extend srcp lower half
1300+ sxtl v23.4s, v17.4h // extend filter lower half
1301+ uxtl2 v16.4s, v16.8h // extend srcp upper half
1302+ sxtl2 v17.4s, v17.8h // extend filter upper half
1303+ mla v2.4s, v22.4s, v23.4s // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]
1304+ mla v2.4s, v16.4s, v17.4s // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]
1305+ ld1 {v18.8h}, [x11], #16 // srcp[filterPos[3] + {0..7}]
1306+ mla v1.4s, v6.4s, v7.4s // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]
1307+ ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize
1308+ subs w15, w15, #8 // j -= 8: processed 8/filterSize
1309+ uxtl v28.4s, v18.4h // extend srcp lower half
1310+ sxtl v29.4s, v19.4h // extend filter lower half
1311+ uxtl2 v18.4s, v18.8h // extend srcp upper half
1312+ sxtl2 v19.4s, v19.8h // extend filter upper half
1313+ mla v3.4s, v28.4s, v29.4s // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]
1314+ mla v3.4s, v18.4s, v19.4s // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]
1315+ b.gt 2b // inner loop if filterSize not consumed completely
1316+ addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding
1317+ addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding
1318+ addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding
1319+ subs w2, w2, #4 // dstW -= 4
1320+ sshl v0.4s, v0.4s, v21.4s // shift right (effectively rigth, as shift is negative); overflow expected
1321+ smin v0.4s, v0.4s, v20.4s // apply min (do not use sqshl)
1322+ st1 {v0.4s}, [x1], #16 // write to destination part0123
1323+ b.gt 1b // loop until end of line
13241324 ret
13251325 endfunc
13261326
@@ -1333,117 +1333,117 @@ function ff_hscale16to19_X4_neon_asm, export=1 · 6 unmodified lines
13331333 // x5 const int32_t *filterPos
13341334 // w6 int filterSize
13351335
1336− stp d8, d9, [sp, #-0x20]!
1337− stp d10, d11, [sp, #0x10]
1336+ stp d8, d9, [sp, #-0x20]!
1337+ stp d10, d11, [sp, #0x10]
13381338
1339− movi v18.4s, #1
1340− movi v17.4s, #1
1341− shl v18.4s, v18.4s, #19
1342− sub v21.4s, v18.4s, v17.4s // max allowed value
1343− dup v17.4s, w0 // read shift
1344− neg v20.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)
1339+ movi v18.4s, #1
1340+ movi v17.4s, #1
1341+ shl v18.4s, v18.4s, #19
1342+ sub v21.4s, v18.4s, v17.4s // max allowed value
1343+ dup v17.4s, w0 // read shift
1344+ neg v20.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)
13451345
1346− lsl w7, w6, #1
1346+ lsl w7, w6, #1
13471347 1:
1348− ldp w8, w9, [x5]
1349− ldp w10, w11, [x5, #8]
1350−
1351− movi v16.2d, #0 // initialize accumulator for idx + 0
1352− movi v17.2d, #0 // initialize accumulator for idx + 1
1353− movi v18.2d, #0 // initialize accumulator for idx + 2
1354− movi v19.2d, #0 // initialize accumulator for idx + 3
1355−
1356− mov x12, x4 // filter + 0
1357− add x13, x4, x7 // filter + 1
1358− add x8, x3, x8, lsl #1 // srcp + filterPos 0
1359− add x14, x13, x7 // filter + 2
1360− add x9, x3, x9, lsl #1 // srcp + filterPos 1
1361− add x15, x14, x7 // filter + 3
1362− add x10, x3, x10, lsl #1 // srcp + filterPos 2
1363− mov w0, w6 // save the filterSize to temporary variable
1364− add x11, x3, x11, lsl #1 // srcp + filterPos 3
1365− add x5, x5, #16 // advance filter position
1366− mov x16, xzr // clear the register x16 used for offsetting the filter values
1348+ ldp w8, w9, [x5]
1349+ ldp w10, w11, [x5, #8]
1350+
1351+ movi v16.2d, #0 // initialize accumulator for idx + 0
1352+ movi v17.2d, #0 // initialize accumulator for idx + 1
1353+ movi v18.2d, #0 // initialize accumulator for idx + 2
1354+ movi v19.2d, #0 // initialize accumulator for idx + 3
1355+
1356+ mov x12, x4 // filter + 0
1357+ add x13, x4, x7 // filter + 1
1358+ add x8, x3, x8, lsl #1 // srcp + filterPos 0
1359+ add x14, x13, x7 // filter + 2
1360+ add x9, x3, x9, lsl #1 // srcp + filterPos 1
1361+ add x15, x14, x7 // filter + 3
1362+ add x10, x3, x10, lsl #1 // srcp + filterPos 2
1363+ mov w0, w6 // save the filterSize to temporary variable
1364+ add x11, x3, x11, lsl #1 // srcp + filterPos 3
1365+ add x5, x5, #16 // advance filter position
1366+ mov x16, xzr // clear the register x16 used for offsetting the filter values
13671367
13681368 2:
1369− ldr q4, [x8], #16 // load src values for idx 0
1370− ldr q5, [x9], #16 // load src values for idx 1
1371− uxtl v26.4s, v4.4h
1372− uxtl2 v4.4s, v4.8h
1373− ldr q31, [x12, x16] // load filter values for idx 0
1374− ldr q6, [x10], #16 // load src values for idx 2
1375− sxtl v22.4s, v31.4h
1376− sxtl2 v31.4s, v31.8h
1377− mla v16.4s, v26.4s, v22.4s // multiplication of lower half for idx 0
1378− uxtl v25.4s, v5.4h
1379− uxtl2 v5.4s, v5.8h
1380− ldr q30, [x13, x16] // load filter values for idx 1
1381− ldr q7, [x11], #16 // load src values for idx 3
1382− mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0
1383− uxtl v24.4s, v6.4h
1384− sxtl v8.4s, v30.4h
1385− sxtl2 v30.4s, v30.8h
1386− mla v17.4s, v25.4s, v8.4s // multiplication of lower half for idx 1
1387− ldr q29, [x14, x16] // load filter values for idx 2
1388− uxtl2 v6.4s, v6.8h
1389− sxtl v9.4s, v29.4h
1390− sxtl2 v29.4s, v29.8h
1391− mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1
1392− ldr q28, [x15, x16] // load filter values for idx 3
1393− mla v18.4s, v24.4s, v9.4s // multiplication of lower half for idx 2
1394− uxtl v23.4s, v7.4h
1395− sxtl v10.4s, v28.4h
1396− mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2
1397− uxtl2 v7.4s, v7.8h
1398− sxtl2 v28.4s, v28.8h
1399− mla v19.4s, v23.4s, v10.4s // multiplication of lower half for idx 3
1400− sub w0, w0, #8
1401− cmp w0, #8
1402− mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3
1403−
1404− add x16, x16, #16 // advance filter values indexing
1405−
1406− b.ge 2b
1369+ ldr q4, [x8], #16 // load src values for idx 0
1370+ ldr q5, [x9], #16 // load src values for idx 1
1371+ uxtl v26.4s, v4.4h
1372+ uxtl2 v4.4s, v4.8h
1373+ ldr q31, [x12, x16] // load filter values for idx 0
1374+ ldr q6, [x10], #16 // load src values for idx 2
1375+ sxtl v22.4s, v31.4h
1376+ sxtl2 v31.4s, v31.8h
1377+ mla v16.4s, v26.4s, v22.4s // multiplication of lower half for idx 0
1378+ uxtl v25.4s, v5.4h
1379+ uxtl2 v5.4s, v5.8h
1380+ ldr q30, [x13, x16] // load filter values for idx 1
1381+ ldr q7, [x11], #16 // load src values for idx 3
1382+ mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0
1383+ uxtl v24.4s, v6.4h
1384+ sxtl v8.4s, v30.4h
1385+ sxtl2 v30.4s, v30.8h
1386+ mla v17.4s, v25.4s, v8.4s // multiplication of lower half for idx 1
1387+ ldr q29, [x14, x16] // load filter values for idx 2
1388+ uxtl2 v6.4s, v6.8h
1389+ sxtl v9.4s, v29.4h
1390+ sxtl2 v29.4s, v29.8h
1391+ mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1
1392+ ldr q28, [x15, x16] // load filter values for idx 3
1393+ mla v18.4s, v24.4s, v9.4s // multiplication of lower half for idx 2
1394+ uxtl v23.4s, v7.4h
1395+ sxtl v10.4s, v28.4h
1396+ mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2
1397+ uxtl2 v7.4s, v7.8h
1398+ sxtl2 v28.4s, v28.8h
1399+ mla v19.4s, v23.4s, v10.4s // multiplication of lower half for idx 3
1400+ sub w0, w0, #8
1401+ cmp w0, #8
1402+ mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3
1403+
1404+ add x16, x16, #16 // advance filter values indexing
1405+
1406+ b.ge 2b
14071407
14081408 // 4 iterations left
14091409
1410− sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements
1411−
1412− ldr d4, [x8] // load src values for idx 0
1413− ldr d31, [x12, x17] // load filter values for idx 0
1414− uxtl v4.4s, v4.4h
1415− sxtl v31.4s, v31.4h
1416− ldr d5, [x9] // load src values for idx 1
1417− mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0
1418− ldr d30, [x13, x17] // load filter values for idx 1
1419− uxtl v5.4s, v5.4h
1420− sxtl v30.4s, v30.4h
1421− ldr d6, [x10] // load src values for idx 2
1422− mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1
1423− ldr d29, [x14, x17] // load filter values for idx 2
1424− uxtl v6.4s, v6.4h
1425− sxtl v29.4s, v29.4h
1426− ldr d7, [x11] // load src values for idx 3
1427− ldr d28, [x15, x17] // load filter values for idx 3
1428− mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2
1429− uxtl v7.4s, v7.4h
1430− sxtl v28.4s, v28.4h
1431− addp v16.4s, v16.4s, v17.4s
1432− mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3
1433− subs w2, w2, #4
1434− addp v18.4s, v18.4s, v19.4s
1435− addp v16.4s, v16.4s, v18.4s
1436− sshl v16.4s, v16.4s, v20.4s
1437− smin v16.4s, v16.4s, v21.4s
1438−
1439− st1 {v16.4s}, [x1], #16
1440− add x4, x4, x7, lsl #2
1441− b.gt 1b
1442−
1443− ldp d8, d9, [sp]
1444− ldp d10, d11, [sp, #0x10]
1445−
1446− add sp, sp, #0x20
1410+ sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements
1411+
1412+ ldr d4, [x8] // load src values for idx 0
1413+ ldr d31, [x12, x17] // load filter values for idx 0
1414+ uxtl v4.4s, v4.4h
1415+ sxtl v31.4s, v31.4h
1416+ ldr d5, [x9] // load src values for idx 1
1417+ mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0
1418+ ldr d30, [x13, x17] // load filter values for idx 1
1419+ uxtl v5.4s, v5.4h
1420+ sxtl v30.4s, v30.4h
1421+ ldr d6, [x10] // load src values for idx 2
1422+ mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1
1423+ ldr d29, [x14, x17] // load filter values for idx 2
1424+ uxtl v6.4s, v6.4h
1425+ sxtl v29.4s, v29.4h
1426+ ldr d7, [x11] // load src values for idx 3
1427+ ldr d28, [x15, x17] // load filter values for idx 3
1428+ mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2
1429+ uxtl v7.4s, v7.4h
1430+ sxtl v28.4s, v28.4h
1431+ addp v16.4s, v16.4s, v17.4s
1432+ mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3
1433+ subs w2, w2, #4
1434+ addp v18.4s, v18.4s, v19.4s
1435+ addp v16.4s, v16.4s, v18.4s
1436+ sshl v16.4s, v16.4s, v20.4s
1437+ smin v16.4s, v16.4s, v21.4s
1438+
1439+ st1 {v16.4s}, [x1], #16
1440+ add x4, x4, x7, lsl #2
1441+ b.gt 1b
1442+
1443+ ldp d8, d9, [sp]
1444+ ldp d10, d11, [sp, #0x10]
1445+
1446+ add sp, sp, #0x20
14471447
14481448 ret
14491449 endfunc
libswscale/aarch64/output.S+165−165
@@ -29,178 +29,178 @@ function ff_yuv2planeX_8_neon, export=1 · 28 unmodified lines
2929 // x5 - const uint8_t *dither,
3030 // w6 - int offset
3131
32− ld1 {v0.8b}, [x5] // load 8x8-bit dither
33− and w6, w6, #7
34− cbz w6, 1f // check if offsetting present
35− ext v0.8b, v0.8b, v0.8b, #3 // honor offsetting which can be 0 or 3 only
36−1: uxtl v0.8h, v0.8b // extend dither to 16-bit
37− ushll v1.4s, v0.4h, #12 // extend dither to 32-bit with left shift by 12 (part 1)
38− ushll2 v2.4s, v0.8h, #12 // extend dither to 32-bit with left shift by 12 (part 2)
39− cmp w1, #8 // if filterSize == 8, branch to specialized version
40− b.eq 6f
41− cmp w1, #4 // if filterSize == 4, branch to specialized version
42− b.eq 8f
43− cmp w1, #2 // if filterSize == 2, branch to specialized version
44− b.eq 10f
32+ ld1 {v0.8b}, [x5] // load 8x8-bit dither
33+ and w6, w6, #7
34+ cbz w6, 1f // check if offsetting present
35+ ext v0.8b, v0.8b, v0.8b, #3 // honor offsetting which can be 0 or 3 only
36+1: uxtl v0.8h, v0.8b // extend dither to 16-bit
37+ ushll v1.4s, v0.4h, #12 // extend dither to 32-bit with left shift by 12 (part 1)
38+ ushll2 v2.4s, v0.8h, #12 // extend dither to 32-bit with left shift by 12 (part 2)
39+ cmp w1, #8 // if filterSize == 8, branch to specialized version
40+ b.eq 6f
41+ cmp w1, #4 // if filterSize == 4, branch to specialized version
42+ b.eq 8f
43+ cmp w1, #2 // if filterSize == 2, branch to specialized version
44+ b.eq 10f
4545
4646 // The filter size does not match of the of specialized implementations. It is either even or odd. If it is even
4747 // then use the first section below.
48− mov x7, #0 // i = 0
49− tbnz w1, #0, 4f // if filterSize % 2 != 0 branch to specialized version
48+ mov x7, #0 // i = 0
49+ tbnz w1, #0, 4f // if filterSize % 2 != 0 branch to specialized version
5050 // fs % 2 == 0
51−2: mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
52− mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
53− mov w8, w1 // tmpfilterSize = filterSize
54− mov x9, x2 // srcp = src
55− mov x10, x0 // filterp = filter
56−3: ldp x11, x12, [x9], #16 // get 2 pointers: src[j] and src[j+1]
57− ldr s7, [x10], #4 // read 2x16-bit coeff X and Y at filter[j] and filter[j+1]
58− add x11, x11, x7, lsl #1 // &src[j ][i]
59− add x12, x12, x7, lsl #1 // &src[j+1][i]
60− ld1 {v5.8h}, [x11] // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H
61− ld1 {v6.8h}, [x12] // read 8x16-bit @ src[j+1][i + {0..7}]: I,J,K,L,M,N,O,P
62− smlal v3.4s, v5.4h, v7.h[0] // val0 += {A,B,C,D} * X
63− smlal2 v4.4s, v5.8h, v7.h[0] // val1 += {E,F,G,H} * X
64− smlal v3.4s, v6.4h, v7.h[1] // val0 += {I,J,K,L} * Y
65− smlal2 v4.4s, v6.8h, v7.h[1] // val1 += {M,N,O,P} * Y
66− subs w8, w8, #2 // tmpfilterSize -= 2
67− b.gt 3b // loop until filterSize consumed
68−
69− sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
70− sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
71− uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
72− st1 {v3.8b}, [x3], #8 // write to destination
73− subs w4, w4, #8 // dstW -= 8
74− add x7, x7, #8 // i += 8
75− b.gt 2b // loop until width consumed
51+2: mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
52+ mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
53+ mov w8, w1 // tmpfilterSize = filterSize
54+ mov x9, x2 // srcp = src
55+ mov x10, x0 // filterp = filter
56+3: ldp x11, x12, [x9], #16 // get 2 pointers: src[j] and src[j+1]
57+ ldr s7, [x10], #4 // read 2x16-bit coeff X and Y at filter[j] and filter[j+1]
58+ add x11, x11, x7, lsl #1 // &src[j ][i]
59+ add x12, x12, x7, lsl #1 // &src[j+1][i]
60+ ld1 {v5.8h}, [x11] // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H
61+ ld1 {v6.8h}, [x12] // read 8x16-bit @ src[j+1][i + {0..7}]: I,J,K,L,M,N,O,P
62+ smlal v3.4s, v5.4h, v7.h[0] // val0 += {A,B,C,D} * X
63+ smlal2 v4.4s, v5.8h, v7.h[0] // val1 += {E,F,G,H} * X
64+ smlal v3.4s, v6.4h, v7.h[1] // val0 += {I,J,K,L} * Y
65+ smlal2 v4.4s, v6.8h, v7.h[1] // val1 += {M,N,O,P} * Y
66+ subs w8, w8, #2 // tmpfilterSize -= 2
67+ b.gt 3b // loop until filterSize consumed
68+
69+ sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
70+ sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
71+ uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
72+ st1 {v3.8b}, [x3], #8 // write to destination
73+ subs w4, w4, #8 // dstW -= 8
74+ add x7, x7, #8 // i += 8
75+ b.gt 2b // loop until width consumed
7676 ret
7777
7878 // If filter size is odd (most likely == 1), then use this section.
7979 // fs % 2 != 0
80−4: mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
81− mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
82− mov w8, w1 // tmpfilterSize = filterSize
83− mov x9, x2 // srcp = src
84− mov x10, x0 // filterp = filter
85−5: ldr x11, [x9], #8 // get 1 pointer: src[j]
86− ldr h6, [x10], #2 // read 1 16 bit coeff X at filter[j]
87− add x11, x11, x7, lsl #1 // &src[j ][i]
88− ld1 {v5.8h}, [x11] // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H
89− smlal v3.4s, v5.4h, v6.h[0] // val0 += {A,B,C,D} * X
90− smlal2 v4.4s, v5.8h, v6.h[0] // val1 += {E,F,G,H} * X
91− subs w8, w8, #1 // tmpfilterSize -= 2
92− b.gt 5b // loop until filterSize consumed
93−
94− sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
95− sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
96− uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
97− st1 {v3.8b}, [x3], #8 // write to destination
98− subs w4, w4, #8 // dstW -= 8
99− add x7, x7, #8 // i += 8
100− b.gt 4b // loop until width consumed
80+4: mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
81+ mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
82+ mov w8, w1 // tmpfilterSize = filterSize
83+ mov x9, x2 // srcp = src
84+ mov x10, x0 // filterp = filter
85+5: ldr x11, [x9], #8 // get 1 pointer: src[j]
86+ ldr h6, [x10], #2 // read 1 16 bit coeff X at filter[j]
87+ add x11, x11, x7, lsl #1 // &src[j ][i]
88+ ld1 {v5.8h}, [x11] // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H
89+ smlal v3.4s, v5.4h, v6.h[0] // val0 += {A,B,C,D} * X
90+ smlal2 v4.4s, v5.8h, v6.h[0] // val1 += {E,F,G,H} * X
91+ subs w8, w8, #1 // tmpfilterSize -= 2
92+ b.gt 5b // loop until filterSize consumed
93+
94+ sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
95+ sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
96+ uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
97+ st1 {v3.8b}, [x3], #8 // write to destination
98+ subs w4, w4, #8 // dstW -= 8
99+ add x7, x7, #8 // i += 8
100+ b.gt 4b // loop until width consumed
101101 ret
102102
103103 6: // fs=8
104− ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]
105− ldp x7, x9, [x2, #16] // load 2 pointers: src[j+2] and src[j+3]
106− ldp x10, x11, [x2, #32] // load 2 pointers: src[j+4] and src[j+5]
107− ldp x12, x13, [x2, #48] // load 2 pointers: src[j+6] and src[j+7]
104+ ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]
105+ ldp x7, x9, [x2, #16] // load 2 pointers: src[j+2] and src[j+3]
106+ ldp x10, x11, [x2, #32] // load 2 pointers: src[j+4] and src[j+5]
107+ ldp x12, x13, [x2, #48] // load 2 pointers: src[j+6] and src[j+7]
108108
109109 // load 8x16-bit values for filter[j], where j=0..7
110− ld1 {v6.8h}, [x0]
110+ ld1 {v6.8h}, [x0]
111111 7:
112− mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
113− mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
114−
115− ld1 {v24.8h}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]
116− ld1 {v25.8h}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]
117− ld1 {v26.8h}, [x7], #16 // load 8x16-bit values for src[j + 2][i + {0..7}]
118− ld1 {v27.8h}, [x9], #16 // load 8x16-bit values for src[j + 3][i + {0..7}]
119− ld1 {v28.8h}, [x10], #16 // load 8x16-bit values for src[j + 4][i + {0..7}]
120− ld1 {v29.8h}, [x11], #16 // load 8x16-bit values for src[j + 5][i + {0..7}]
121− ld1 {v30.8h}, [x12], #16 // load 8x16-bit values for src[j + 6][i + {0..7}]
122− ld1 {v31.8h}, [x13], #16 // load 8x16-bit values for src[j + 7][i + {0..7}]
123−
124− smlal v3.4s, v24.4h, v6.h[0] // val0 += src[0][i + {0..3}] * filter[0]
125− smlal2 v4.4s, v24.8h, v6.h[0] // val1 += src[0][i + {4..7}] * filter[0]
126− smlal v3.4s, v25.4h, v6.h[1] // val0 += src[1][i + {0..3}] * filter[1]
127− smlal2 v4.4s, v25.8h, v6.h[1] // val1 += src[1][i + {4..7}] * filter[1]
128− smlal v3.4s, v26.4h, v6.h[2] // val0 += src[2][i + {0..3}] * filter[2]
129− smlal2 v4.4s, v26.8h, v6.h[2] // val1 += src[2][i + {4..7}] * filter[2]
130− smlal v3.4s, v27.4h, v6.h[3] // val0 += src[3][i + {0..3}] * filter[3]
131− smlal2 v4.4s, v27.8h, v6.h[3] // val1 += src[3][i + {4..7}] * filter[3]
132− smlal v3.4s, v28.4h, v6.h[4] // val0 += src[4][i + {0..3}] * filter[4]
133− smlal2 v4.4s, v28.8h, v6.h[4] // val1 += src[4][i + {4..7}] * filter[4]
134− smlal v3.4s, v29.4h, v6.h[5] // val0 += src[5][i + {0..3}] * filter[5]
135− smlal2 v4.4s, v29.8h, v6.h[5] // val1 += src[5][i + {4..7}] * filter[5]
136− smlal v3.4s, v30.4h, v6.h[6] // val0 += src[6][i + {0..3}] * filter[6]
137− smlal2 v4.4s, v30.8h, v6.h[6] // val1 += src[6][i + {4..7}] * filter[6]
138− smlal v3.4s, v31.4h, v6.h[7] // val0 += src[7][i + {0..3}] * filter[7]
139− smlal2 v4.4s, v31.8h, v6.h[7] // val1 += src[7][i + {4..7}] * filter[7]
140−
141− sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
142− sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
143− uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
144− subs w4, w4, #8 // dstW -= 8
145− st1 {v3.8b}, [x3], #8 // write to destination
146− b.gt 7b // loop until width consumed
112+ mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
113+ mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
114+
115+ ld1 {v24.8h}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]
116+ ld1 {v25.8h}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]
117+ ld1 {v26.8h}, [x7], #16 // load 8x16-bit values for src[j + 2][i + {0..7}]
118+ ld1 {v27.8h}, [x9], #16 // load 8x16-bit values for src[j + 3][i + {0..7}]
119+ ld1 {v28.8h}, [x10], #16 // load 8x16-bit values for src[j + 4][i + {0..7}]
120+ ld1 {v29.8h}, [x11], #16 // load 8x16-bit values for src[j + 5][i + {0..7}]
121+ ld1 {v30.8h}, [x12], #16 // load 8x16-bit values for src[j + 6][i + {0..7}]
122+ ld1 {v31.8h}, [x13], #16 // load 8x16-bit values for src[j + 7][i + {0..7}]
123+
124+ smlal v3.4s, v24.4h, v6.h[0] // val0 += src[0][i + {0..3}] * filter[0]
125+ smlal2 v4.4s, v24.8h, v6.h[0] // val1 += src[0][i + {4..7}] * filter[0]
126+ smlal v3.4s, v25.4h, v6.h[1] // val0 += src[1][i + {0..3}] * filter[1]
127+ smlal2 v4.4s, v25.8h, v6.h[1] // val1 += src[1][i + {4..7}] * filter[1]
128+ smlal v3.4s, v26.4h, v6.h[2] // val0 += src[2][i + {0..3}] * filter[2]
129+ smlal2 v4.4s, v26.8h, v6.h[2] // val1 += src[2][i + {4..7}] * filter[2]
130+ smlal v3.4s, v27.4h, v6.h[3] // val0 += src[3][i + {0..3}] * filter[3]
131+ smlal2 v4.4s, v27.8h, v6.h[3] // val1 += src[3][i + {4..7}] * filter[3]
132+ smlal v3.4s, v28.4h, v6.h[4] // val0 += src[4][i + {0..3}] * filter[4]
133+ smlal2 v4.4s, v28.8h, v6.h[4] // val1 += src[4][i + {4..7}] * filter[4]
134+ smlal v3.4s, v29.4h, v6.h[5] // val0 += src[5][i + {0..3}] * filter[5]
135+ smlal2 v4.4s, v29.8h, v6.h[5] // val1 += src[5][i + {4..7}] * filter[5]
136+ smlal v3.4s, v30.4h, v6.h[6] // val0 += src[6][i + {0..3}] * filter[6]
137+ smlal2 v4.4s, v30.8h, v6.h[6] // val1 += src[6][i + {4..7}] * filter[6]
138+ smlal v3.4s, v31.4h, v6.h[7] // val0 += src[7][i + {0..3}] * filter[7]
139+ smlal2 v4.4s, v31.8h, v6.h[7] // val1 += src[7][i + {4..7}] * filter[7]
140+
141+ sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
142+ sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
143+ uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
144+ subs w4, w4, #8 // dstW -= 8
145+ st1 {v3.8b}, [x3], #8 // write to destination
146+ b.gt 7b // loop until width consumed
147147 ret
148148
149149 8: // fs=4
150− ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]
151− ldp x7, x9, [x2, #16] // load 2 pointers: src[j+2] and src[j+3]
150+ ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]
151+ ldp x7, x9, [x2, #16] // load 2 pointers: src[j+2] and src[j+3]
152152
153153 // load 4x16-bit values for filter[j], where j=0..3 and replicated across lanes
154− ld1 {v6.4h}, [x0]
154+ ld1 {v6.4h}, [x0]
155155 9:
156− mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
157− mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
158−
159− ld1 {v24.8h}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]
160− ld1 {v25.8h}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]
161− ld1 {v26.8h}, [x7], #16 // load 8x16-bit values for src[j + 2][i + {0..7}]
162− ld1 {v27.8h}, [x9], #16 // load 8x16-bit values for src[j + 3][i + {0..7}]
163−
164− smlal v3.4s, v24.4h, v6.h[0] // val0 += src[0][i + {0..3}] * filter[0]
165− smlal2 v4.4s, v24.8h, v6.h[0] // val1 += src[0][i + {4..7}] * filter[0]
166− smlal v3.4s, v25.4h, v6.h[1] // val0 += src[1][i + {0..3}] * filter[1]
167− smlal2 v4.4s, v25.8h, v6.h[1] // val1 += src[1][i + {4..7}] * filter[1]
168− smlal v3.4s, v26.4h, v6.h[2] // val0 += src[2][i + {0..3}] * filter[2]
169− smlal2 v4.4s, v26.8h, v6.h[2] // val1 += src[2][i + {4..7}] * filter[2]
170− smlal v3.4s, v27.4h, v6.h[3] // val0 += src[3][i + {0..3}] * filter[3]
171− smlal2 v4.4s, v27.8h, v6.h[3] // val1 += src[3][i + {4..7}] * filter[3]
172−
173− sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
174− sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
175− uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
176− st1 {v3.8b}, [x3], #8 // write to destination
177− subs w4, w4, #8 // dstW -= 8
178− b.gt 9b // loop until width consumed
156+ mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
157+ mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
158+
159+ ld1 {v24.8h}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]
160+ ld1 {v25.8h}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]
161+ ld1 {v26.8h}, [x7], #16 // load 8x16-bit values for src[j + 2][i + {0..7}]
162+ ld1 {v27.8h}, [x9], #16 // load 8x16-bit values for src[j + 3][i + {0..7}]
163+
164+ smlal v3.4s, v24.4h, v6.h[0] // val0 += src[0][i + {0..3}] * filter[0]
165+ smlal2 v4.4s, v24.8h, v6.h[0] // val1 += src[0][i + {4..7}] * filter[0]
166+ smlal v3.4s, v25.4h, v6.h[1] // val0 += src[1][i + {0..3}] * filter[1]
167+ smlal2 v4.4s, v25.8h, v6.h[1] // val1 += src[1][i + {4..7}] * filter[1]
168+ smlal v3.4s, v26.4h, v6.h[2] // val0 += src[2][i + {0..3}] * filter[2]
169+ smlal2 v4.4s, v26.8h, v6.h[2] // val1 += src[2][i + {4..7}] * filter[2]
170+ smlal v3.4s, v27.4h, v6.h[3] // val0 += src[3][i + {0..3}] * filter[3]
171+ smlal2 v4.4s, v27.8h, v6.h[3] // val1 += src[3][i + {4..7}] * filter[3]
172+
173+ sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
174+ sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
175+ uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
176+ st1 {v3.8b}, [x3], #8 // write to destination
177+ subs w4, w4, #8 // dstW -= 8
178+ b.gt 9b // loop until width consumed
179179 ret
180180
181181 10: // fs=2
182− ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]
182+ ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]
183183
184184 // load 2x16-bit values for filter[j], where j=0..1 and replicated across lanes
185− ldr s6, [x0]
185+ ldr s6, [x0]
186186 11:
187− mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
188− mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
189−
190− ld1 {v24.8h}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]
191− ld1 {v25.8h}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]
192−
193− smlal v3.4s, v24.4h, v6.h[0] // val0 += src[0][i + {0..3}] * filter[0]
194− smlal2 v4.4s, v24.8h, v6.h[0] // val1 += src[0][i + {4..7}] * filter[0]
195− smlal v3.4s, v25.4h, v6.h[1] // val0 += src[1][i + {0..3}] * filter[1]
196− smlal2 v4.4s, v25.8h, v6.h[1] // val1 += src[1][i + {4..7}] * filter[1]
197−
198− sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
199− sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
200− uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
201− st1 {v3.8b}, [x3], #8 // write to destination
202− subs w4, w4, #8 // dstW -= 8
203− b.gt 11b // loop until width consumed
187+ mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
188+ mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
189+
190+ ld1 {v24.8h}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]
191+ ld1 {v25.8h}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]
192+
193+ smlal v3.4s, v24.4h, v6.h[0] // val0 += src[0][i + {0..3}] * filter[0]
194+ smlal2 v4.4s, v24.8h, v6.h[0] // val1 += src[0][i + {4..7}] * filter[0]
195+ smlal v3.4s, v25.4h, v6.h[1] // val0 += src[1][i + {0..3}] * filter[1]
196+ smlal2 v4.4s, v25.8h, v6.h[1] // val1 += src[1][i + {4..7}] * filter[1]
197+
198+ sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
199+ sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
200+ uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
201+ st1 {v3.8b}, [x3], #8 // write to destination
202+ subs w4, w4, #8 // dstW -= 8
203+ b.gt 11b // loop until width consumed
204204 ret
205205 endfunc
206206
@@ -210,25 +210,25 @@ function ff_yuv2plane1_8_neon, export=1 · 3 unmodified lines
210210 // w2 - int dstW,
211211 // x3 - const uint8_t *dither,
212212 // w4 - int offset
213− ld1 {v0.8b}, [x3] // load 8x8-bit dither
214− and w4, w4, #7
215− cbz w4, 1f // check if offsetting present
216− ext v0.8b, v0.8b, v0.8b, #3 // honor offsetting which can be 0 or 3 only
217−1: uxtl v0.8h, v0.8b // extend dither to 32-bit
218− uxtl v1.4s, v0.4h
219− uxtl2 v2.4s, v0.8h
213+ ld1 {v0.8b}, [x3] // load 8x8-bit dither
214+ and w4, w4, #7
215+ cbz w4, 1f // check if offsetting present
216+ ext v0.8b, v0.8b, v0.8b, #3 // honor offsetting which can be 0 or 3 only
217+1: uxtl v0.8h, v0.8b // extend dither to 32-bit
218+ uxtl v1.4s, v0.4h
219+ uxtl2 v2.4s, v0.8h
220220 2:
221− ld1 {v3.8h}, [x0], #16 // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H
222− sxtl v4.4s, v3.4h
223− sxtl2 v5.4s, v3.8h
224− add v4.4s, v4.4s, v1.4s
225− add v5.4s, v5.4s, v2.4s
226− sqshrun v4.4h, v4.4s, #6
227− sqshrun2 v4.8h, v5.4s, #6
228−
229− uqshrn v3.8b, v4.8h, #1 // clip8(val>>7)
230− subs w2, w2, #8 // dstW -= 8
231− st1 {v3.8b}, [x1], #8 // write to destination
232− b.gt 2b // loop until width consumed
221+ ld1 {v3.8h}, [x0], #16 // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H
222+ sxtl v4.4s, v3.4h
223+ sxtl2 v5.4s, v3.8h
224+ add v4.4s, v4.4s, v1.4s
225+ add v5.4s, v5.4s, v2.4s
226+ sqshrun v4.4h, v4.4s, #6
227+ sqshrun2 v4.8h, v5.4s, #6
228+
229+ uqshrn v3.8b, v4.8h, #1 // clip8(val>>7)
230+ subs w2, w2, #8 // dstW -= 8
231+ st1 {v3.8b}, [x1], #8 // write to destination
232+ b.gt 2b // loop until width consumed
233233 ret
234234 endfunc
libswscale/aarch64/yuv2rgb_neon.S+110−110
@@ -23,23 +23,23 @@ · 22 unmodified lines
2323
2424 .macro load_yoff_ycoeff yoff ycoeff
2525 #if defined(__APPLE__)
26− ldp w9, w10, [sp, #\yoff]
26+ ldp w9, w10, [sp, #\yoff]
2727 #else
28− ldr w9, [sp, #\yoff]
29− ldr w10, [sp, #\ycoeff]
28+ ldr w9, [sp, #\yoff]
29+ ldr w10, [sp, #\ycoeff]
3030 #endif
3131 .endm
3232
3333 .macro load_args_nv12
34− ldr x8, [sp] // table
35− load_yoff_ycoeff 8, 16 // y_offset, y_coeff
36− ld1 {v1.1d}, [x8]
37− dup v0.8h, w10
38− dup v3.8h, w9
39− sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding)
40− sub w5, w5, w0 // w5 = linesizeY - width (paddingY)
41− sub w7, w7, w0 // w7 = linesizeC - width (paddingC)
42− neg w11, w0
34+ ldr x8, [sp] // table
35+ load_yoff_ycoeff 8, 16 // y_offset, y_coeff
36+ ld1 {v1.1d}, [x8]
37+ dup v0.8h, w10
38+ dup v3.8h, w9
39+ sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding)
40+ sub w5, w5, w0 // w5 = linesizeY - width (paddingY)
41+ sub w7, w7, w0 // w7 = linesizeC - width (paddingC)
42+ neg w11, w0
4343 .endm
4444
4545 .macro load_args_nv21
@@ -47,52 +47,52 @@ · 1 unmodified line
4747 .endm
4848
4949 .macro load_args_yuv420p
50− ldr x13, [sp] // srcV
51− ldr w14, [sp, #8] // linesizeV
52− ldr x8, [sp, #16] // table
53− load_yoff_ycoeff 24, 32 // y_offset, y_coeff
54− ld1 {v1.1d}, [x8]
55− dup v0.8h, w10
56− dup v3.8h, w9
57− sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding)
58− sub w5, w5, w0 // w5 = linesizeY - width (paddingY)
59− sub w7, w7, w0, lsr #1 // w7 = linesizeU - width / 2 (paddingU)
60− sub w14, w14, w0, lsr #1 // w14 = linesizeV - width / 2 (paddingV)
61− lsr w11, w0, #1
62− neg w11, w11
50+ ldr x13, [sp] // srcV
51+ ldr w14, [sp, #8] // linesizeV
52+ ldr x8, [sp, #16] // table
53+ load_yoff_ycoeff 24, 32 // y_offset, y_coeff
54+ ld1 {v1.1d}, [x8]
55+ dup v0.8h, w10
56+ dup v3.8h, w9
57+ sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding)
58+ sub w5, w5, w0 // w5 = linesizeY - width (paddingY)
59+ sub w7, w7, w0, lsr #1 // w7 = linesizeU - width / 2 (paddingU)
60+ sub w14, w14, w0, lsr #1 // w14 = linesizeV - width / 2 (paddingV)
61+ lsr w11, w0, #1
62+ neg w11, w11
6363 .endm
6464
6565 .macro load_args_yuv422p
66− ldr x13, [sp] // srcV
67− ldr w14, [sp, #8] // linesizeV
68− ldr x8, [sp, #16] // table
69− load_yoff_ycoeff 24, 32 // y_offset, y_coeff
70− ld1 {v1.1d}, [x8]
71− dup v0.8h, w10
72− dup v3.8h, w9
73− sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding)
74− sub w5, w5, w0 // w5 = linesizeY - width (paddingY)
75− sub w7, w7, w0, lsr #1 // w7 = linesizeU - width / 2 (paddingU)
76− sub w14, w14, w0, lsr #1 // w14 = linesizeV - width / 2 (paddingV)
66+ ldr x13, [sp] // srcV
67+ ldr w14, [sp, #8] // linesizeV
68+ ldr x8, [sp, #16] // table
69+ load_yoff_ycoeff 24, 32 // y_offset, y_coeff
70+ ld1 {v1.1d}, [x8]
71+ dup v0.8h, w10
72+ dup v3.8h, w9
73+ sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding)
74+ sub w5, w5, w0 // w5 = linesizeY - width (paddingY)
75+ sub w7, w7, w0, lsr #1 // w7 = linesizeU - width / 2 (paddingU)
76+ sub w14, w14, w0, lsr #1 // w14 = linesizeV - width / 2 (paddingV)
7777 .endm
7878
7979 .macro load_chroma_nv12
80− ld2 {v16.8b, v17.8b}, [x6], #16
81− ushll v18.8h, v16.8b, #3
82− ushll v19.8h, v17.8b, #3
80+ ld2 {v16.8b, v17.8b}, [x6], #16
81+ ushll v18.8h, v16.8b, #3
82+ ushll v19.8h, v17.8b, #3
8383 .endm
8484
8585 .macro load_chroma_nv21
86− ld2 {v16.8b, v17.8b}, [x6], #16
87− ushll v19.8h, v16.8b, #3
88− ushll v18.8h, v17.8b, #3
86+ ld2 {v16.8b, v17.8b}, [x6], #16
87+ ushll v19.8h, v16.8b, #3
88+ ushll v18.8h, v17.8b, #3
8989 .endm
9090
9191 .macro load_chroma_yuv420p
92− ld1 {v16.8b}, [ x6], #8
93− ld1 {v17.8b}, [x13], #8
94− ushll v18.8h, v16.8b, #3
95− ushll v19.8h, v17.8b, #3
92+ ld1 {v16.8b}, [ x6], #8
93+ ld1 {v17.8b}, [x13], #8
94+ ushll v18.8h, v16.8b, #3
95+ ushll v19.8h, v17.8b, #3
9696 .endm
9797
9898 .macro load_chroma_yuv422p
@@ -100,9 +100,9 @@ · 1 unmodified line
100100 .endm
101101
102102 .macro increment_nv12
103− ands w15, w1, #1
104− csel w16, w7, w11, ne // incC = (h & 1) ? paddincC : -width
105− add x6, x6, w16, sxtw // srcC += incC
103+ ands w15, w1, #1
104+ csel w16, w7, w11, ne // incC = (h & 1) ? paddincC : -width
105+ add x6, x6, w16, sxtw // srcC += incC
106106 .endm
107107
108108 .macro increment_nv21
@@ -110,100 +110,100 @@ · 1 unmodified line
110110 .endm
111111
112112 .macro increment_yuv420p
113− ands w15, w1, #1
114− csel w16, w7, w11, ne // incU = (h & 1) ? paddincU : -width/2
115− csel w17, w14, w11, ne // incV = (h & 1) ? paddincV : -width/2
116− add x6, x6, w16, sxtw // srcU += incU
117− add x13, x13, w17, sxtw // srcV += incV
113+ ands w15, w1, #1
114+ csel w16, w7, w11, ne // incU = (h & 1) ? paddincU : -width/2
115+ csel w17, w14, w11, ne // incV = (h & 1) ? paddincV : -width/2
116+ add x6, x6, w16, sxtw // srcU += incU
117+ add x13, x13, w17, sxtw // srcV += incV
118118 .endm
119119
120120 .macro increment_yuv422p
121− add x6, x6, w7, sxtw // srcU += incU
122− add x13, x13, w14, sxtw // srcV += incV
121+ add x6, x6, w7, sxtw // srcU += incU
122+ add x13, x13, w14, sxtw // srcV += incV
123123 .endm
124124
125125 .macro compute_rgba r1 g1 b1 a1 r2 g2 b2 a2
126− add v20.8h, v26.8h, v20.8h // Y1 + R1
127− add v21.8h, v27.8h, v21.8h // Y2 + R2
128− add v22.8h, v26.8h, v22.8h // Y1 + G1
129− add v23.8h, v27.8h, v23.8h // Y2 + G2
130− add v24.8h, v26.8h, v24.8h // Y1 + B1
131− add v25.8h, v27.8h, v25.8h // Y2 + B2
132− sqrshrun \r1, v20.8h, #1 // clip_u8((Y1 + R1) >> 1)
133− sqrshrun \r2, v21.8h, #1 // clip_u8((Y2 + R1) >> 1)
134− sqrshrun \g1, v22.8h, #1 // clip_u8((Y1 + G1) >> 1)
135− sqrshrun \g2, v23.8h, #1 // clip_u8((Y2 + G1) >> 1)
136− sqrshrun \b1, v24.8h, #1 // clip_u8((Y1 + B1) >> 1)
137− sqrshrun \b2, v25.8h, #1 // clip_u8((Y2 + B1) >> 1)
138− movi \a1, #255
139− movi \a2, #255
126+ add v20.8h, v26.8h, v20.8h // Y1 + R1
127+ add v21.8h, v27.8h, v21.8h // Y2 + R2
128+ add v22.8h, v26.8h, v22.8h // Y1 + G1
129+ add v23.8h, v27.8h, v23.8h // Y2 + G2
130+ add v24.8h, v26.8h, v24.8h // Y1 + B1
131+ add v25.8h, v27.8h, v25.8h // Y2 + B2
132+ sqrshrun \r1, v20.8h, #1 // clip_u8((Y1 + R1) >> 1)
133+ sqrshrun \r2, v21.8h, #1 // clip_u8((Y2 + R1) >> 1)
134+ sqrshrun \g1, v22.8h, #1 // clip_u8((Y1 + G1) >> 1)
135+ sqrshrun \g2, v23.8h, #1 // clip_u8((Y2 + G1) >> 1)
136+ sqrshrun \b1, v24.8h, #1 // clip_u8((Y1 + B1) >> 1)
137+ sqrshrun \b2, v25.8h, #1 // clip_u8((Y2 + B1) >> 1)
138+ movi \a1, #255
139+ movi \a2, #255
140140 .endm
141141
142142 .macro declare_func ifmt ofmt
143143 function ff_\ifmt\()_to_\ofmt\()_neon, export=1
144144 load_args_\ifmt
145− mov w9, w1
145+ mov w9, w1
146146 1:
147− mov w8, w0 // w8 = width
147+ mov w8, w0 // w8 = width
148148 2:
149− movi v5.8h, #4, lsl #8 // 128 * (1<<3)
149+ movi v5.8h, #4, lsl #8 // 128 * (1<<3)
150150 load_chroma_\ifmt
151− sub v18.8h, v18.8h, v5.8h // U*(1<<3) - 128*(1<<3)
152− sub v19.8h, v19.8h, v5.8h // V*(1<<3) - 128*(1<<3)
153− sqdmulh v20.8h, v19.8h, v1.h[0] // V * v2r (R)
154− sqdmulh v22.8h, v18.8h, v1.h[1] // U * u2g
155− sqdmulh v19.8h, v19.8h, v1.h[2] // V * v2g
156− add v22.8h, v22.8h, v19.8h // U * u2g + V * v2g (G)
157− sqdmulh v24.8h, v18.8h, v1.h[3] // U * u2b (B)
158− zip2 v21.8h, v20.8h, v20.8h // R2
159− zip1 v20.8h, v20.8h, v20.8h // R1
160− zip2 v23.8h, v22.8h, v22.8h // G2
161− zip1 v22.8h, v22.8h, v22.8h // G1
162− zip2 v25.8h, v24.8h, v24.8h // B2
163− zip1 v24.8h, v24.8h, v24.8h // B1
164− ld1 {v2.16b}, [x4], #16 // load luma
165− ushll v26.8h, v2.8b, #3 // Y1*(1<<3)
166− ushll2 v27.8h, v2.16b, #3 // Y2*(1<<3)
167− sub v26.8h, v26.8h, v3.8h // Y1*(1<<3) - y_offset
168− sub v27.8h, v27.8h, v3.8h // Y2*(1<<3) - y_offset
169− sqdmulh v26.8h, v26.8h, v0.8h // ((Y1*(1<<3) - y_offset) * y_coeff) >> 15
170− sqdmulh v27.8h, v27.8h, v0.8h // ((Y2*(1<<3) - y_offset) * y_coeff) >> 15
151+ sub v18.8h, v18.8h, v5.8h // U*(1<<3) - 128*(1<<3)
152+ sub v19.8h, v19.8h, v5.8h // V*(1<<3) - 128*(1<<3)
153+ sqdmulh v20.8h, v19.8h, v1.h[0] // V * v2r (R)
154+ sqdmulh v22.8h, v18.8h, v1.h[1] // U * u2g
155+ sqdmulh v19.8h, v19.8h, v1.h[2] // V * v2g
156+ add v22.8h, v22.8h, v19.8h // U * u2g + V * v2g (G)
157+ sqdmulh v24.8h, v18.8h, v1.h[3] // U * u2b (B)
158+ zip2 v21.8h, v20.8h, v20.8h // R2
159+ zip1 v20.8h, v20.8h, v20.8h // R1
160+ zip2 v23.8h, v22.8h, v22.8h // G2
161+ zip1 v22.8h, v22.8h, v22.8h // G1
162+ zip2 v25.8h, v24.8h, v24.8h // B2
163+ zip1 v24.8h, v24.8h, v24.8h // B1
164+ ld1 {v2.16b}, [x4], #16 // load luma
165+ ushll v26.8h, v2.8b, #3 // Y1*(1<<3)
166+ ushll2 v27.8h, v2.16b, #3 // Y2*(1<<3)
167+ sub v26.8h, v26.8h, v3.8h // Y1*(1<<3) - y_offset
168+ sub v27.8h, v27.8h, v3.8h // Y2*(1<<3) - y_offset
169+ sqdmulh v26.8h, v26.8h, v0.8h // ((Y1*(1<<3) - y_offset) * y_coeff) >> 15
170+ sqdmulh v27.8h, v27.8h, v0.8h // ((Y2*(1<<3) - y_offset) * y_coeff) >> 15
171171
172172 .ifc \ofmt,argb // 1 2 3 0
173− compute_rgba v5.8b,v6.8b,v7.8b,v4.8b, v17.8b,v18.8b,v19.8b,v16.8b
173+ compute_rgba v5.8b,v6.8b,v7.8b,v4.8b, v17.8b,v18.8b,v19.8b,v16.8b
174174 .endif
175175
176176 .ifc \ofmt,rgba // 0 1 2 3
177− compute_rgba v4.8b,v5.8b,v6.8b,v7.8b, v16.8b,v17.8b,v18.8b,v19.8b
177+ compute_rgba v4.8b,v5.8b,v6.8b,v7.8b, v16.8b,v17.8b,v18.8b,v19.8b
178178 .endif
179179
180180 .ifc \ofmt,abgr // 3 2 1 0
181− compute_rgba v7.8b,v6.8b,v5.8b,v4.8b, v19.8b,v18.8b,v17.8b,v16.8b
181+ compute_rgba v7.8b,v6.8b,v5.8b,v4.8b, v19.8b,v18.8b,v17.8b,v16.8b
182182 .endif
183183
184184 .ifc \ofmt,bgra // 2 1 0 3
185− compute_rgba v6.8b,v5.8b,v4.8b,v7.8b, v18.8b,v17.8b,v16.8b,v19.8b
185+ compute_rgba v6.8b,v5.8b,v4.8b,v7.8b, v18.8b,v17.8b,v16.8b,v19.8b
186186 .endif
187187
188− st4 { v4.8b, v5.8b, v6.8b, v7.8b}, [x2], #32
189− st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [x2], #32
190− subs w8, w8, #16 // width -= 16
191− b.gt 2b
192− add x2, x2, w3, sxtw // dst += padding
193− add x4, x4, w5, sxtw // srcY += paddingY
188+ st4 { v4.8b, v5.8b, v6.8b, v7.8b}, [x2], #32
189+ st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [x2], #32
190+ subs w8, w8, #16 // width -= 16
191+ b.gt 2b
192+ add x2, x2, w3, sxtw // dst += padding
193+ add x4, x4, w5, sxtw // srcY += paddingY
194194 increment_\ifmt
195− subs w1, w1, #1 // height -= 1
196− b.gt 1b
197− mov w0, w9
195+ subs w1, w1, #1 // height -= 1
196+ b.gt 1b
197+ mov w0, w9
198198 ret
199199 endfunc
200200 .endm
201201
202202 .macro declare_rgb_funcs ifmt
203− declare_func \ifmt, argb
204− declare_func \ifmt, rgba
205− declare_func \ifmt, abgr
206− declare_func \ifmt, bgra
203+ declare_func \ifmt, argb
204+ declare_func \ifmt, rgba
205+ declare_func \ifmt, abgr
206+ declare_func \ifmt, bgra
207207 .endm
208208
209209 declare_rgb_funcs nv12

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t