History

aarch64: Reindent all assembly to 8/24 column indentation

libavcodec/aarch64/vc1dsp_neon.S is skipped here, as it intentionally uses a layered indentation style to visually show how different unrolled/interleaved phases fit together.

Martin Storsjö committed 3 years ago

Commit a76b4091 parent cada4596 files · +1600 −1600Browse files

Unified · Split

6 files
  1. libavcodec/aarch64/aacpsdsp_neon.S +109 −109
  2. libavcodec/aarch64/opusdsp_neon.S +51 −51
  3. libswresample/aarch64/resample.S +40 −40
  4. libswscale/aarch64/hscale.S +1125 −1125
  5. libswscale/aarch64/output.S +165 −165
  6. libswscale/aarch64/yuv2rgb_neon.S +110 −110
libavcodec/aarch64/aacpsdsp_neon.S+109−109
@@ -19,130 +19,130 @@ · 18 unmodified lines
19#include "libavutil/aarch64/asm.S"19#include "libavutil/aarch64/asm.S"
20 20
21function ff_ps_add_squares_neon, export=121function ff_ps_add_squares_neon, export=1
221: ld1 {v0.4s,v1.4s}, [x1], #32221: ld1 {v0.4s,v1.4s}, [x1], #32
23 fmul v0.4s, v0.4s, v0.4s23 fmul v0.4s, v0.4s, v0.4s
24 fmul v1.4s, v1.4s, v1.4s24 fmul v1.4s, v1.4s, v1.4s
25 faddp v2.4s, v0.4s, v1.4s25 faddp v2.4s, v0.4s, v1.4s
26 ld1 {v3.4s}, [x0]26 ld1 {v3.4s}, [x0]
27 fadd v3.4s, v3.4s, v2.4s27 fadd v3.4s, v3.4s, v2.4s
28 st1 {v3.4s}, [x0], #1628 st1 {v3.4s}, [x0], #16
29 subs w2, w2, #429 subs w2, w2, #4
30 b.gt 1b30 b.gt 1b
31 ret31 ret
32endfunc32endfunc
33 33
34function ff_ps_mul_pair_single_neon, export=134function ff_ps_mul_pair_single_neon, export=1
351: ld1 {v0.4s,v1.4s}, [x1], #32351: ld1 {v0.4s,v1.4s}, [x1], #32
36 ld1 {v2.4s}, [x2], #1636 ld1 {v2.4s}, [x2], #16
37 zip1 v3.4s, v2.4s, v2.4s37 zip1 v3.4s, v2.4s, v2.4s
38 zip2 v4.4s, v2.4s, v2.4s38 zip2 v4.4s, v2.4s, v2.4s
39 fmul v0.4s, v0.4s, v3.4s39 fmul v0.4s, v0.4s, v3.4s
40 fmul v1.4s, v1.4s, v4.4s40 fmul v1.4s, v1.4s, v4.4s
41 st1 {v0.4s,v1.4s}, [x0], #3241 st1 {v0.4s,v1.4s}, [x0], #32
42 subs w3, w3, #442 subs w3, w3, #4
43 b.gt 1b43 b.gt 1b
44 ret44 ret
45endfunc45endfunc
46 46
47function ff_ps_stereo_interpolate_neon, export=147function ff_ps_stereo_interpolate_neon, export=1
48 ld1 {v0.4s}, [x2]48 ld1 {v0.4s}, [x2]
49 ld1 {v1.4s}, [x3]49 ld1 {v1.4s}, [x3]
50 zip1 v4.4s, v0.4s, v0.4s50 zip1 v4.4s, v0.4s, v0.4s
51 zip2 v5.4s, v0.4s, v0.4s51 zip2 v5.4s, v0.4s, v0.4s
52 zip1 v6.4s, v1.4s, v1.4s52 zip1 v6.4s, v1.4s, v1.4s
53 zip2 v7.4s, v1.4s, v1.4s53 zip2 v7.4s, v1.4s, v1.4s
541: ld1 {v2.2s}, [x0]541: ld1 {v2.2s}, [x0]
55 ld1 {v3.2s}, [x1]55 ld1 {v3.2s}, [x1]
56 fadd v4.4s, v4.4s, v6.4s56 fadd v4.4s, v4.4s, v6.4s
57 fadd v5.4s, v5.4s, v7.4s57 fadd v5.4s, v5.4s, v7.4s
58 mov v2.d[1], v2.d[0]58 mov v2.d[1], v2.d[0]
59 mov v3.d[1], v3.d[0]59 mov v3.d[1], v3.d[0]
60 fmul v2.4s, v2.4s, v4.4s60 fmul v2.4s, v2.4s, v4.4s
61 fmla v2.4s, v3.4s, v5.4s61 fmla v2.4s, v3.4s, v5.4s
62 st1 {v2.d}[0], [x0], #862 st1 {v2.d}[0], [x0], #8
63 st1 {v2.d}[1], [x1], #863 st1 {v2.d}[1], [x1], #8
64 subs w4, w4, #164 subs w4, w4, #1
65 b.gt 1b65 b.gt 1b
66 ret66 ret
67endfunc67endfunc
68 68
69function ff_ps_stereo_interpolate_ipdopd_neon, export=169function ff_ps_stereo_interpolate_ipdopd_neon, export=1
70 ld1 {v0.4s,v1.4s}, [x2]70 ld1 {v0.4s,v1.4s}, [x2]
71 ld1 {v6.4s,v7.4s}, [x3]71 ld1 {v6.4s,v7.4s}, [x3]
72 fneg v2.4s, v1.4s72 fneg v2.4s, v1.4s
73 fneg v3.4s, v7.4s73 fneg v3.4s, v7.4s
74 zip1 v16.4s, v0.4s, v0.4s74 zip1 v16.4s, v0.4s, v0.4s
75 zip2 v17.4s, v0.4s, v0.4s75 zip2 v17.4s, v0.4s, v0.4s
76 zip1 v18.4s, v2.4s, v1.4s76 zip1 v18.4s, v2.4s, v1.4s
77 zip2 v19.4s, v2.4s, v1.4s77 zip2 v19.4s, v2.4s, v1.4s
78 zip1 v20.4s, v6.4s, v6.4s78 zip1 v20.4s, v6.4s, v6.4s
79 zip2 v21.4s, v6.4s, v6.4s79 zip2 v21.4s, v6.4s, v6.4s
80 zip1 v22.4s, v3.4s, v7.4s80 zip1 v22.4s, v3.4s, v7.4s
81 zip2 v23.4s, v3.4s, v7.4s81 zip2 v23.4s, v3.4s, v7.4s
821: ld1 {v2.2s}, [x0]821: ld1 {v2.2s}, [x0]
83 ld1 {v3.2s}, [x1]83 ld1 {v3.2s}, [x1]
84 fadd v16.4s, v16.4s, v20.4s84 fadd v16.4s, v16.4s, v20.4s
85 fadd v17.4s, v17.4s, v21.4s85 fadd v17.4s, v17.4s, v21.4s
86 mov v2.d[1], v2.d[0]86 mov v2.d[1], v2.d[0]
87 mov v3.d[1], v3.d[0]87 mov v3.d[1], v3.d[0]
88 fmul v4.4s, v2.4s, v16.4s88 fmul v4.4s, v2.4s, v16.4s
89 fmla v4.4s, v3.4s, v17.4s89 fmla v4.4s, v3.4s, v17.4s
90 fadd v18.4s, v18.4s, v22.4s90 fadd v18.4s, v18.4s, v22.4s
91 fadd v19.4s, v19.4s, v23.4s91 fadd v19.4s, v19.4s, v23.4s
92 ext v2.16b, v2.16b, v2.16b, #492 ext v2.16b, v2.16b, v2.16b, #4
93 ext v3.16b, v3.16b, v3.16b, #493 ext v3.16b, v3.16b, v3.16b, #4
94 fmla v4.4s, v2.4s, v18.4s94 fmla v4.4s, v2.4s, v18.4s
95 fmla v4.4s, v3.4s, v19.4s95 fmla v4.4s, v3.4s, v19.4s
96 st1 {v4.d}[0], [x0], #896 st1 {v4.d}[0], [x0], #8
97 st1 {v4.d}[1], [x1], #897 st1 {v4.d}[1], [x1], #8
98 subs w4, w4, #198 subs w4, w4, #1
99 b.gt 1b99 b.gt 1b
100 ret100 ret
101endfunc101endfunc
102 102
103function ff_ps_hybrid_analysis_neon, export=1103function ff_ps_hybrid_analysis_neon, export=1
104 lsl x3, x3, #3104 lsl x3, x3, #3
105 ld2 {v0.4s,v1.4s}, [x1], #32105 ld2 {v0.4s,v1.4s}, [x1], #32
106 ld2 {v2.2s,v3.2s}, [x1], #16106 ld2 {v2.2s,v3.2s}, [x1], #16
107 ld1 {v24.2s}, [x1], #8107 ld1 {v24.2s}, [x1], #8
108 ld2 {v4.2s,v5.2s}, [x1], #16108 ld2 {v4.2s,v5.2s}, [x1], #16
109 ld2 {v6.4s,v7.4s}, [x1]109 ld2 {v6.4s,v7.4s}, [x1]
110 rev64 v6.4s, v6.4s110 rev64 v6.4s, v6.4s
111 rev64 v7.4s, v7.4s111 rev64 v7.4s, v7.4s
112 ext v6.16b, v6.16b, v6.16b, #8112 ext v6.16b, v6.16b, v6.16b, #8
113 ext v7.16b, v7.16b, v7.16b, #8113 ext v7.16b, v7.16b, v7.16b, #8
114 rev64 v4.2s, v4.2s114 rev64 v4.2s, v4.2s
115 rev64 v5.2s, v5.2s115 rev64 v5.2s, v5.2s
116 mov v2.d[1], v3.d[0]116 mov v2.d[1], v3.d[0]
117 mov v4.d[1], v5.d[0]117 mov v4.d[1], v5.d[0]
118 mov v5.d[1], v2.d[0]118 mov v5.d[1], v2.d[0]
119 mov v3.d[1], v4.d[0]119 mov v3.d[1], v4.d[0]
120 fadd v16.4s, v0.4s, v6.4s120 fadd v16.4s, v0.4s, v6.4s
121 fadd v17.4s, v1.4s, v7.4s121 fadd v17.4s, v1.4s, v7.4s
122 fsub v18.4s, v1.4s, v7.4s122 fsub v18.4s, v1.4s, v7.4s
123 fsub v19.4s, v0.4s, v6.4s123 fsub v19.4s, v0.4s, v6.4s
124 fadd v22.4s, v2.4s, v4.4s124 fadd v22.4s, v2.4s, v4.4s
125 fsub v23.4s, v5.4s, v3.4s125 fsub v23.4s, v5.4s, v3.4s
126 trn1 v20.2d, v22.2d, v23.2d // {re4+re8, re5+re7, im8-im4, im7-im5}126 trn1 v20.2d, v22.2d, v23.2d // {re4+re8, re5+re7, im8-im4, im7-im5}
127 trn2 v21.2d, v22.2d, v23.2d // {im4+im8, im5+im7, re4-re8, re5-re7}127 trn2 v21.2d, v22.2d, v23.2d // {im4+im8, im5+im7, re4-re8, re5-re7}
1281: ld2 {v2.4s,v3.4s}, [x2], #321281: ld2 {v2.4s,v3.4s}, [x2], #32
129 ld2 {v4.2s,v5.2s}, [x2], #16129 ld2 {v4.2s,v5.2s}, [x2], #16
130 ld1 {v6.2s}, [x2], #8130 ld1 {v6.2s}, [x2], #8
131 add x2, x2, #8131 add x2, x2, #8
132 mov v4.d[1], v5.d[0]132 mov v4.d[1], v5.d[0]
133 mov v6.s[1], v6.s[0]133 mov v6.s[1], v6.s[0]
134 fmul v6.2s, v6.2s, v24.2s134 fmul v6.2s, v6.2s, v24.2s
135 fmul v0.4s, v2.4s, v16.4s135 fmul v0.4s, v2.4s, v16.4s
136 fmul v1.4s, v2.4s, v17.4s136 fmul v1.4s, v2.4s, v17.4s
137 fmls v0.4s, v3.4s, v18.4s137 fmls v0.4s, v3.4s, v18.4s
138 fmla v1.4s, v3.4s, v19.4s138 fmla v1.4s, v3.4s, v19.4s
139 fmla v0.4s, v4.4s, v20.4s139 fmla v0.4s, v4.4s, v20.4s
140 fmla v1.4s, v4.4s, v21.4s140 fmla v1.4s, v4.4s, v21.4s
141 faddp v0.4s, v0.4s, v1.4s141 faddp v0.4s, v0.4s, v1.4s
142 faddp v0.4s, v0.4s, v0.4s142 faddp v0.4s, v0.4s, v0.4s
143 fadd v0.2s, v0.2s, v6.2s143 fadd v0.2s, v0.2s, v6.2s
144 st1 {v0.2s}, [x0], x3144 st1 {v0.2s}, [x0], x3
145 subs w4, w4, #1145 subs w4, w4, #1
146 b.gt 1b146 b.gt 1b
147 ret147 ret
148endfunc148endfunc
libavcodec/aarch64/opusdsp_neon.S+51−51
@@ -33,81 +33,81 @@ const tab_x2, align=4 · 32 unmodified lines
33endconst33endconst
34 34
35function ff_opus_deemphasis_neon, export=135function ff_opus_deemphasis_neon, export=1
36 movrel x4, tab_st36 movrel x4, tab_st
37 ld1 {v4.4s}, [x4]37 ld1 {v4.4s}, [x4]
38 movrel x4, tab_x038 movrel x4, tab_x0
39 ld1 {v5.4s}, [x4]39 ld1 {v5.4s}, [x4]
40 movrel x4, tab_x140 movrel x4, tab_x1
41 ld1 {v6.4s}, [x4]41 ld1 {v6.4s}, [x4]
42 movrel x4, tab_x242 movrel x4, tab_x2
43 ld1 {v7.4s}, [x4]43 ld1 {v7.4s}, [x4]
44 44
45 fmul v0.4s, v4.4s, v0.s[0]45 fmul v0.4s, v4.4s, v0.s[0]
46 46
471: ld1 {v1.4s, v2.4s}, [x1], #32471: ld1 {v1.4s, v2.4s}, [x1], #32
48 48
49 fmla v0.4s, v5.4s, v1.s[0]49 fmla v0.4s, v5.4s, v1.s[0]
50 fmul v3.4s, v7.4s, v2.s[2]50 fmul v3.4s, v7.4s, v2.s[2]
51 51
52 fmla v0.4s, v6.4s, v1.s[1]52 fmla v0.4s, v6.4s, v1.s[1]
53 fmla v3.4s, v6.4s, v2.s[1]53 fmla v3.4s, v6.4s, v2.s[1]
54 54
55 fmla v0.4s, v7.4s, v1.s[2]55 fmla v0.4s, v7.4s, v1.s[2]
56 fmla v3.4s, v5.4s, v2.s[0]56 fmla v3.4s, v5.4s, v2.s[0]
57 57
58 fadd v1.4s, v1.4s, v0.4s58 fadd v1.4s, v1.4s, v0.4s
59 fadd v2.4s, v2.4s, v3.4s59 fadd v2.4s, v2.4s, v3.4s
60 60
61 fmla v2.4s, v4.4s, v1.s[3]61 fmla v2.4s, v4.4s, v1.s[3]
62 62
63 st1 {v1.4s, v2.4s}, [x0], #3263 st1 {v1.4s, v2.4s}, [x0], #32
64 fmul v0.4s, v4.4s, v2.s[3]64 fmul v0.4s, v4.4s, v2.s[3]
65 65
66 subs w2, w2, #866 subs w2, w2, #8
67 b.gt 1b67 b.gt 1b
68 68
69 mov s0, v2.s[3]69 mov s0, v2.s[3]
70 70
71 ret71 ret
72endfunc72endfunc
73 73
74function ff_opus_postfilter_neon, export=174function ff_opus_postfilter_neon, export=1
75 ld1 {v0.4s}, [x2]75 ld1 {v0.4s}, [x2]
76 dup v1.4s, v0.s[1]76 dup v1.4s, v0.s[1]
77 dup v2.4s, v0.s[2]77 dup v2.4s, v0.s[2]
78 dup v0.4s, v0.s[0]78 dup v0.4s, v0.s[0]
79 79
80 add w1, w1, #280 add w1, w1, #2
81 sub x1, x0, x1, lsl #281 sub x1, x0, x1, lsl #2
82 82
83 ld1 {v3.4s}, [x1]83 ld1 {v3.4s}, [x1]
84 fmul v3.4s, v3.4s, v2.4s84 fmul v3.4s, v3.4s, v2.4s
85 85
861: add x1, x1, #4861: add x1, x1, #4
87 ld1 {v4.4s}, [x1]87 ld1 {v4.4s}, [x1]
88 add x1, x1, #488 add x1, x1, #4
89 ld1 {v5.4s}, [x1]89 ld1 {v5.4s}, [x1]
90 add x1, x1, #490 add x1, x1, #4
91 ld1 {v6.4s}, [x1]91 ld1 {v6.4s}, [x1]
92 add x1, x1, #492 add x1, x1, #4
93 ld1 {v7.4s}, [x1]93 ld1 {v7.4s}, [x1]
94 94
95 fmla v3.4s, v7.4s, v2.4s95 fmla v3.4s, v7.4s, v2.4s
96 fadd v6.4s, v6.4s, v4.4s96 fadd v6.4s, v6.4s, v4.4s
97 97
98 ld1 {v4.4s}, [x0]98 ld1 {v4.4s}, [x0]
99 fmla v4.4s, v5.4s, v0.4s99 fmla v4.4s, v5.4s, v0.4s
100 100
101 fmul v6.4s, v6.4s, v1.4s101 fmul v6.4s, v6.4s, v1.4s
102 fadd v6.4s, v6.4s, v3.4s102 fadd v6.4s, v6.4s, v3.4s
103 103
104 fadd v4.4s, v4.4s, v6.4s104 fadd v4.4s, v4.4s, v6.4s
105 fmul v3.4s, v7.4s, v2.4s105 fmul v3.4s, v7.4s, v2.4s
106 106
107 st1 {v4.4s}, [x0], #16107 st1 {v4.4s}, [x0], #16
108 108
109 subs w3, w3, #4109 subs w3, w3, #4
110 b.gt 1b110 b.gt 1b
111 111
112 ret112 ret
113endfunc113endfunc
libswresample/aarch64/resample.S+40−40
@@ -21,57 +21,57 @@ · 20 unmodified lines
21#include "libavutil/aarch64/asm.S"21#include "libavutil/aarch64/asm.S"
22 22
23function ff_resample_common_apply_filter_x4_float_neon, export=123function ff_resample_common_apply_filter_x4_float_neon, export=1
24 movi v0.4s, #0 // accumulator24 movi v0.4s, #0 // accumulator
251: ld1 {v1.4s}, [x1], #16 // src[0..3]251: ld1 {v1.4s}, [x1], #16 // src[0..3]
26 ld1 {v2.4s}, [x2], #16 // filter[0..3]26 ld1 {v2.4s}, [x2], #16 // filter[0..3]
27 fmla v0.4s, v1.4s, v2.4s // accumulator += src[0..3] * filter[0..3]27 fmla v0.4s, v1.4s, v2.4s // accumulator += src[0..3] * filter[0..3]
28 subs w3, w3, #4 // filter_length -= 428 subs w3, w3, #4 // filter_length -= 4
29 b.gt 1b // loop until filter_length29 b.gt 1b // loop until filter_length
30 faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values30 faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
31 faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values31 faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
32 st1 {v0.s}[0], [x0], #4 // write accumulator32 st1 {v0.s}[0], [x0], #4 // write accumulator
33 ret33 ret
34endfunc34endfunc
35 35
36function ff_resample_common_apply_filter_x8_float_neon, export=136function ff_resample_common_apply_filter_x8_float_neon, export=1
37 movi v0.4s, #0 // accumulator37 movi v0.4s, #0 // accumulator
381: ld1 {v1.4s}, [x1], #16 // src[0..3]381: ld1 {v1.4s}, [x1], #16 // src[0..3]
39 ld1 {v2.4s}, [x2], #16 // filter[0..3]39 ld1 {v2.4s}, [x2], #16 // filter[0..3]
40 ld1 {v3.4s}, [x1], #16 // src[4..7]40 ld1 {v3.4s}, [x1], #16 // src[4..7]
41 ld1 {v4.4s}, [x2], #16 // filter[4..7]41 ld1 {v4.4s}, [x2], #16 // filter[4..7]
42 fmla v0.4s, v1.4s, v2.4s // accumulator += src[0..3] * filter[0..3]42 fmla v0.4s, v1.4s, v2.4s // accumulator += src[0..3] * filter[0..3]
43 fmla v0.4s, v3.4s, v4.4s // accumulator += src[4..7] * filter[4..7]43 fmla v0.4s, v3.4s, v4.4s // accumulator += src[4..7] * filter[4..7]
44 subs w3, w3, #8 // filter_length -= 844 subs w3, w3, #8 // filter_length -= 8
45 b.gt 1b // loop until filter_length45 b.gt 1b // loop until filter_length
46 faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values46 faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
47 faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values47 faddp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
48 st1 {v0.s}[0], [x0], #4 // write accumulator48 st1 {v0.s}[0], [x0], #4 // write accumulator
49 ret49 ret
50endfunc50endfunc
51 51
52function ff_resample_common_apply_filter_x4_s16_neon, export=152function ff_resample_common_apply_filter_x4_s16_neon, export=1
53 movi v0.4s, #0 // accumulator53 movi v0.4s, #0 // accumulator
541: ld1 {v1.4h}, [x1], #8 // src[0..3]541: ld1 {v1.4h}, [x1], #8 // src[0..3]
55 ld1 {v2.4h}, [x2], #8 // filter[0..3]55 ld1 {v2.4h}, [x2], #8 // filter[0..3]
56 smlal v0.4s, v1.4h, v2.4h // accumulator += src[0..3] * filter[0..3]56 smlal v0.4s, v1.4h, v2.4h // accumulator += src[0..3] * filter[0..3]
57 subs w3, w3, #4 // filter_length -= 457 subs w3, w3, #4 // filter_length -= 4
58 b.gt 1b // loop until filter_length58 b.gt 1b // loop until filter_length
59 addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values59 addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
60 addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values60 addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
61 st1 {v0.s}[0], [x0], #4 // write accumulator61 st1 {v0.s}[0], [x0], #4 // write accumulator
62 ret62 ret
63endfunc63endfunc
64 64
65function ff_resample_common_apply_filter_x8_s16_neon, export=165function ff_resample_common_apply_filter_x8_s16_neon, export=1
66 movi v0.4s, #0 // accumulator66 movi v0.4s, #0 // accumulator
671: ld1 {v1.8h}, [x1], #16 // src[0..7]671: ld1 {v1.8h}, [x1], #16 // src[0..7]
68 ld1 {v2.8h}, [x2], #16 // filter[0..7]68 ld1 {v2.8h}, [x2], #16 // filter[0..7]
69 smlal v0.4s, v1.4h, v2.4h // accumulator += src[0..3] * filter[0..3]69 smlal v0.4s, v1.4h, v2.4h // accumulator += src[0..3] * filter[0..3]
70 smlal2 v0.4s, v1.8h, v2.8h // accumulator += src[4..7] * filter[4..7]70 smlal2 v0.4s, v1.8h, v2.8h // accumulator += src[4..7] * filter[4..7]
71 subs w3, w3, #8 // filter_length -= 871 subs w3, w3, #8 // filter_length -= 8
72 b.gt 1b // loop until filter_length72 b.gt 1b // loop until filter_length
73 addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values73 addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
74 addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values74 addp v0.4s, v0.4s, v0.4s // pair adding of the 4x32-bit accumulated values
75 st1 {v0.s}[0], [x0], #4 // write accumulator75 st1 {v0.s}[0], [x0], #4 // write accumulator
76 ret76 ret
77endfunc77endfunc
libswscale/aarch64/hscale.S+1125−1125
@@ -41,53 +41,53 @@ · 40 unmodified lines
41;----------------------------------------------------------------------------- */41;----------------------------------------------------------------------------- */
42 42
43function ff_hscale8to15_X8_neon, export=143function ff_hscale8to15_X8_neon, export=1
44 sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)44 sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)
451: ldr w8, [x5], #4 // filterPos[idx]451: ldr w8, [x5], #4 // filterPos[idx]
46 ldr w0, [x5], #4 // filterPos[idx + 1]46 ldr w0, [x5], #4 // filterPos[idx + 1]
47 ldr w11, [x5], #4 // filterPos[idx + 2]47 ldr w11, [x5], #4 // filterPos[idx + 2]
48 ldr w9, [x5], #4 // filterPos[idx + 3]48 ldr w9, [x5], #4 // filterPos[idx + 3]
49 mov x16, x4 // filter0 = filter49 mov x16, x4 // filter0 = filter
50 add x12, x16, x7 // filter1 = filter0 + filterSize*250 add x12, x16, x7 // filter1 = filter0 + filterSize*2
51 add x13, x12, x7 // filter2 = filter1 + filterSize*251 add x13, x12, x7 // filter2 = filter1 + filterSize*2
52 add x4, x13, x7 // filter3 = filter2 + filterSize*252 add x4, x13, x7 // filter3 = filter2 + filterSize*2
53 movi v0.2d, #0 // val sum part 1 (for dst[0])53 movi v0.2d, #0 // val sum part 1 (for dst[0])
54 movi v1.2d, #0 // val sum part 2 (for dst[1])54 movi v1.2d, #0 // val sum part 2 (for dst[1])
55 movi v2.2d, #0 // val sum part 3 (for dst[2])55 movi v2.2d, #0 // val sum part 3 (for dst[2])
56 movi v3.2d, #0 // val sum part 4 (for dst[3])56 movi v3.2d, #0 // val sum part 4 (for dst[3])
57 add x17, x3, w8, uxtw // srcp + filterPos[0]57 add x17, x3, w8, uxtw // srcp + filterPos[0]
58 add x8, x3, w0, uxtw // srcp + filterPos[1]58 add x8, x3, w0, uxtw // srcp + filterPos[1]
59 add x0, x3, w11, uxtw // srcp + filterPos[2]59 add x0, x3, w11, uxtw // srcp + filterPos[2]
60 add x11, x3, w9, uxtw // srcp + filterPos[3]60 add x11, x3, w9, uxtw // srcp + filterPos[3]
61 mov w15, w6 // filterSize counter61 mov w15, w6 // filterSize counter
622: ld1 {v4.8b}, [x17], #8 // srcp[filterPos[0] + {0..7}]622: ld1 {v4.8b}, [x17], #8 // srcp[filterPos[0] + {0..7}]
63 ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 163 ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1
64 ld1 {v6.8b}, [x8], #8 // srcp[filterPos[1] + {0..7}]64 ld1 {v6.8b}, [x8], #8 // srcp[filterPos[1] + {0..7}]
65 ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize65 ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize
66 uxtl v4.8h, v4.8b // unpack part 1 to 16-bit66 uxtl v4.8h, v4.8b // unpack part 1 to 16-bit
67 smlal v0.4s, v4.4h, v5.4h // v0 accumulates srcp[filterPos[0] + {0..3}] * filter[{0..3}]67 smlal v0.4s, v4.4h, v5.4h // v0 accumulates srcp[filterPos[0] + {0..3}] * filter[{0..3}]
68 smlal2 v0.4s, v4.8h, v5.8h // v0 accumulates srcp[filterPos[0] + {4..7}] * filter[{4..7}]68 smlal2 v0.4s, v4.8h, v5.8h // v0 accumulates srcp[filterPos[0] + {4..7}] * filter[{4..7}]
69 ld1 {v16.8b}, [x0], #8 // srcp[filterPos[2] + {0..7}]69 ld1 {v16.8b}, [x0], #8 // srcp[filterPos[2] + {0..7}]
70 ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize70 ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize
71 uxtl v6.8h, v6.8b // unpack part 2 to 16-bit71 uxtl v6.8h, v6.8b // unpack part 2 to 16-bit
72 smlal v1.4s, v6.4h, v7.4h // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]72 smlal v1.4s, v6.4h, v7.4h // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]
73 uxtl v16.8h, v16.8b // unpack part 3 to 16-bit73 uxtl v16.8h, v16.8b // unpack part 3 to 16-bit
74 smlal v2.4s, v16.4h, v17.4h // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]74 smlal v2.4s, v16.4h, v17.4h // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]
75 smlal2 v2.4s, v16.8h, v17.8h // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]75 smlal2 v2.4s, v16.8h, v17.8h // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]
76 ld1 {v18.8b}, [x11], #8 // srcp[filterPos[3] + {0..7}]76 ld1 {v18.8b}, [x11], #8 // srcp[filterPos[3] + {0..7}]
77 smlal2 v1.4s, v6.8h, v7.8h // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]77 smlal2 v1.4s, v6.8h, v7.8h // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]
78 ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize78 ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize
79 subs w15, w15, #8 // j -= 8: processed 8/filterSize79 subs w15, w15, #8 // j -= 8: processed 8/filterSize
80 uxtl v18.8h, v18.8b // unpack part 4 to 16-bit80 uxtl v18.8h, v18.8b // unpack part 4 to 16-bit
81 smlal v3.4s, v18.4h, v19.4h // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]81 smlal v3.4s, v18.4h, v19.4h // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]
82 smlal2 v3.4s, v18.8h, v19.8h // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]82 smlal2 v3.4s, v18.8h, v19.8h // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]
83 b.gt 2b // inner loop if filterSize not consumed completely83 b.gt 2b // inner loop if filterSize not consumed completely
84 addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding84 addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding
85 addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding85 addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding
86 addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding86 addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding
87 subs w2, w2, #4 // dstW -= 487 subs w2, w2, #4 // dstW -= 4
88 sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values88 sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values
89 st1 {v0.4h}, [x1], #8 // write to destination part012389 st1 {v0.4h}, [x1], #8 // write to destination part0123
90 b.gt 1b // loop until end of line90 b.gt 1b // loop until end of line
91 ret91 ret
92endfunc92endfunc
93 93
@@ -103,98 +103,98 @@ function ff_hscale8to15_X4_neon, export=1 · 9 unmodified lines
103// This function for filter sizes that are 4 mod 8. In other words, anything that's 0 mod 4 but not103// This function for filter sizes that are 4 mod 8. In other words, anything that's 0 mod 4 but not
104// 0 mod 8. It also assumes that dstW is 0 mod 4.104// 0 mod 8. It also assumes that dstW is 0 mod 4.
105 105
106 lsl w7, w6, #1 // w7 = filterSize * 2106 lsl w7, w6, #1 // w7 = filterSize * 2
1071:1071:
108 ldp w8, w9, [x5] // filterPos[idx + 0], [idx + 1]108 ldp w8, w9, [x5] // filterPos[idx + 0], [idx + 1]
109 ldp w10, w11, [x5, #8] // filterPos[idx + 2], [idx + 3]109 ldp w10, w11, [x5, #8] // filterPos[idx + 2], [idx + 3]
110 110
111 movi v16.2d, #0 // initialize accumulator for idx + 0111 movi v16.2d, #0 // initialize accumulator for idx + 0
112 movi v17.2d, #0 // initialize accumulator for idx + 1112 movi v17.2d, #0 // initialize accumulator for idx + 1
113 movi v18.2d, #0 // initialize accumulator for idx + 2113 movi v18.2d, #0 // initialize accumulator for idx + 2
114 movi v19.2d, #0 // initialize accumulator for idx + 3114 movi v19.2d, #0 // initialize accumulator for idx + 3
115 115
116 mov x12, x4 // filter pointer for idx + 0116 mov x12, x4 // filter pointer for idx + 0
117 add x13, x4, x7 // filter pointer for idx + 1117 add x13, x4, x7 // filter pointer for idx + 1
118 add x8, x3, w8, uxtw // srcp + filterPos[idx + 0]118 add x8, x3, w8, uxtw // srcp + filterPos[idx + 0]
119 add x9, x3, w9, uxtw // srcp + filterPos[idx + 1]119 add x9, x3, w9, uxtw // srcp + filterPos[idx + 1]
120 120
121 add x14, x13, x7 // filter pointer for idx + 2121 add x14, x13, x7 // filter pointer for idx + 2
122 add x10, x3, w10, uxtw // srcp + filterPos[idx + 2]122 add x10, x3, w10, uxtw // srcp + filterPos[idx + 2]
123 add x11, x3, w11, uxtw // srcp + filterPos[idx + 3]123 add x11, x3, w11, uxtw // srcp + filterPos[idx + 3]
124 124
125 mov w0, w6 // copy filterSize to a temp register, w0125 mov w0, w6 // copy filterSize to a temp register, w0
126 add x5, x5, #16 // advance the filterPos pointer126 add x5, x5, #16 // advance the filterPos pointer
127 add x15, x14, x7 // filter pointer for idx + 3127 add x15, x14, x7 // filter pointer for idx + 3
128 mov x16, xzr // temp register for offsetting filter pointers128 mov x16, xzr // temp register for offsetting filter pointers
129 129
1302:1302:
131 // This section loops over 8-wide chunks of filter size131 // This section loops over 8-wide chunks of filter size
132 ldr d4, [x8], #8 // load 8 bytes from srcp for idx + 0132 ldr d4, [x8], #8 // load 8 bytes from srcp for idx + 0
133 ldr q0, [x12, x16] // load 8 values, 16 bytes from filter for idx + 0133 ldr q0, [x12, x16] // load 8 values, 16 bytes from filter for idx + 0
134 134
135 ldr d5, [x9], #8 // load 8 bytes from srcp for idx + 1135 ldr d5, [x9], #8 // load 8 bytes from srcp for idx + 1
136 ldr q1, [x13, x16] // load 8 values, 16 bytes from filter for idx + 1136 ldr q1, [x13, x16] // load 8 values, 16 bytes from filter for idx + 1
137 137
138 uxtl v4.8h, v4.8b // unsigned extend long for idx + 0138 uxtl v4.8h, v4.8b // unsigned extend long for idx + 0
139 uxtl v5.8h, v5.8b // unsigned extend long for idx + 1139 uxtl v5.8h, v5.8b // unsigned extend long for idx + 1
140 140
141 ldr d6, [x10], #8 // load 8 bytes from srcp for idx + 2141 ldr d6, [x10], #8 // load 8 bytes from srcp for idx + 2
142 ldr q2, [x14, x16] // load 8 values, 16 bytes from filter for idx + 2142 ldr q2, [x14, x16] // load 8 values, 16 bytes from filter for idx + 2
143 143
144 smlal v16.4s, v0.4h, v4.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 0144 smlal v16.4s, v0.4h, v4.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 0
145 smlal v17.4s, v1.4h, v5.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 1145 smlal v17.4s, v1.4h, v5.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 1
146 146
147 ldr d7, [x11], #8 // load 8 bytes from srcp for idx + 3147 ldr d7, [x11], #8 // load 8 bytes from srcp for idx + 3
148 ldr q3, [x15, x16] // load 8 values, 16 bytes from filter for idx + 3148 ldr q3, [x15, x16] // load 8 values, 16 bytes from filter for idx + 3
149 149
150 sub w0, w0, #8 // decrement the remaining filterSize counter150 sub w0, w0, #8 // decrement the remaining filterSize counter
151 smlal2 v16.4s, v0.8h, v4.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 0151 smlal2 v16.4s, v0.8h, v4.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 0
152 smlal2 v17.4s, v1.8h, v5.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 1152 smlal2 v17.4s, v1.8h, v5.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 1
153 uxtl v6.8h, v6.8b // unsigned extend long for idx + 2153 uxtl v6.8h, v6.8b // unsigned extend long for idx + 2
154 uxtl v7.8h, v7.8b // unsigned extend long for idx + 3154 uxtl v7.8h, v7.8b // unsigned extend long for idx + 3
155 smlal v18.4s, v2.4h, v6.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 2155 smlal v18.4s, v2.4h, v6.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 2
156 smlal v19.4s, v3.4h, v7.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 3156 smlal v19.4s, v3.4h, v7.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 3
157 157
158 cmp w0, #8 // are there at least 8 more elements in filter to consume?158 cmp w0, #8 // are there at least 8 more elements in filter to consume?
159 add x16, x16, #16 // advance the offsetting register for filter values159 add x16, x16, #16 // advance the offsetting register for filter values
160 160
161 smlal2 v18.4s, v2.8h, v6.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 2161 smlal2 v18.4s, v2.8h, v6.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 2
162 smlal2 v19.4s, v3.8h, v7.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 3162 smlal2 v19.4s, v3.8h, v7.8h // val += src[srcPos + j + 4..7] * filter[fs * i + j + 4..7], idx + 3
163 163
164 b.ge 2b // branch back to inner loop164 b.ge 2b // branch back to inner loop
165 165
166 // complete the remaining 4 filter elements166 // complete the remaining 4 filter elements
167 sub x17, x7, #8 // calculate the offset of the filter pointer for the remaining 4 elements167 sub x17, x7, #8 // calculate the offset of the filter pointer for the remaining 4 elements
168 168
169 ldr s4, [x8] // load 4 bytes from srcp for idx + 0169 ldr s4, [x8] // load 4 bytes from srcp for idx + 0
170 ldr d0, [x12, x17] // load 4 values, 8 bytes from filter for idx + 0170 ldr d0, [x12, x17] // load 4 values, 8 bytes from filter for idx + 0
171 ldr s5, [x9] // load 4 bytes from srcp for idx + 1171 ldr s5, [x9] // load 4 bytes from srcp for idx + 1
172 ldr d1, [x13, x17] // load 4 values, 8 bytes from filter for idx + 1172 ldr d1, [x13, x17] // load 4 values, 8 bytes from filter for idx + 1
173 173
174 uxtl v4.8h, v4.8b // unsigned extend long for idx + 0174 uxtl v4.8h, v4.8b // unsigned extend long for idx + 0
175 uxtl v5.8h, v5.8b // unsigned extend long for idx + 1175 uxtl v5.8h, v5.8b // unsigned extend long for idx + 1
176 176
177 ldr s6, [x10] // load 4 bytes from srcp for idx + 2177 ldr s6, [x10] // load 4 bytes from srcp for idx + 2
178 ldr d2, [x14, x17] // load 4 values, 8 bytes from filter for idx + 2178 ldr d2, [x14, x17] // load 4 values, 8 bytes from filter for idx + 2
179 smlal v16.4s, v0.4h, v4.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 0179 smlal v16.4s, v0.4h, v4.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 0
180 smlal v17.4s, v1.4h, v5.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 1180 smlal v17.4s, v1.4h, v5.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 1
181 ldr s7, [x11] // load 4 bytes from srcp for idx + 3181 ldr s7, [x11] // load 4 bytes from srcp for idx + 3
182 ldr d3, [x15, x17] // load 4 values, 8 bytes from filter for idx + 3182 ldr d3, [x15, x17] // load 4 values, 8 bytes from filter for idx + 3
183 183
184 uxtl v6.8h, v6.8b // unsigned extend long for idx + 2184 uxtl v6.8h, v6.8b // unsigned extend long for idx + 2
185 uxtl v7.8h, v7.8b // unsigned extend long for idx + 3185 uxtl v7.8h, v7.8b // unsigned extend long for idx + 3
186 addp v16.4s, v16.4s, v17.4s // horizontal pair adding for idx 0,1186 addp v16.4s, v16.4s, v17.4s // horizontal pair adding for idx 0,1
187 smlal v18.4s, v2.4h, v6.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 2187 smlal v18.4s, v2.4h, v6.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 2
188 smlal v19.4s, v3.4h, v7.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 3188 smlal v19.4s, v3.4h, v7.4h // val += src[srcPos + j + 0..3] * filter[fs * i + j + 0..3], idx + 3
189 189
190 addp v18.4s, v18.4s, v19.4s // horizontal pair adding for idx 2,3190 addp v18.4s, v18.4s, v19.4s // horizontal pair adding for idx 2,3
191 addp v16.4s, v16.4s, v18.4s // final horizontal pair adding producing one vector with results for idx = 0..3191 addp v16.4s, v16.4s, v18.4s // final horizontal pair adding producing one vector with results for idx = 0..3
192 192
193 subs w2, w2, #4 // dstW -= 4193 subs w2, w2, #4 // dstW -= 4
194 sqshrn v0.4h, v16.4s, #7 // shift and clip the 2x16-bit final values194 sqshrn v0.4h, v16.4s, #7 // shift and clip the 2x16-bit final values
195 st1 {v0.4h}, [x1], #8 // write to destination idx 0..3195 st1 {v0.4h}, [x1], #8 // write to destination idx 0..3
196 add x4, x4, x7, lsl #2 // filter += (filterSize*2) * 4196 add x4, x4, x7, lsl #2 // filter += (filterSize*2) * 4
197 b.gt 1b // loop until end of line197 b.gt 1b // loop until end of line
198 ret198 ret
199endfunc199endfunc
200 200
@@ -219,132 +219,132 @@ function ff_hscale8to15_4_neon, export=1 · 18 unmodified lines
219// 3. Complete madd219// 3. Complete madd
220// 4. Complete remaining iterations when dstW % 8 != 0220// 4. Complete remaining iterations when dstW % 8 != 0
221 221
222 sub sp, sp, #32 // allocate 32 bytes on the stack222 sub sp, sp, #32 // allocate 32 bytes on the stack
223 cmp w2, #16 // if dstW <16, skip to the last block used for wrapping up223 cmp w2, #16 // if dstW <16, skip to the last block used for wrapping up
224 b.lt 2f224 b.lt 2f
225 225
226 // load 8 values from filterPos to be used as offsets into src226 // load 8 values from filterPos to be used as offsets into src
227 ldp w8, w9, [x5] // filterPos[idx + 0], [idx + 1]227 ldp w8, w9, [x5] // filterPos[idx + 0], [idx + 1]
228 ldp w10, w11, [x5, #8] // filterPos[idx + 2], [idx + 3]228 ldp w10, w11, [x5, #8] // filterPos[idx + 2], [idx + 3]
229 ldp w12, w13, [x5, #16] // filterPos[idx + 4], [idx + 5]229 ldp w12, w13, [x5, #16] // filterPos[idx + 4], [idx + 5]
230 ldp w14, w15, [x5, #24] // filterPos[idx + 6], [idx + 7]230 ldp w14, w15, [x5, #24] // filterPos[idx + 6], [idx + 7]
231 add x5, x5, #32 // advance filterPos231 add x5, x5, #32 // advance filterPos
232 232
233 // gather random access data from src into contiguous memory233 // gather random access data from src into contiguous memory
234 ldr w8, [x3, w8, uxtw] // src[filterPos[idx + 0]][0..3]234 ldr w8, [x3, w8, uxtw] // src[filterPos[idx + 0]][0..3]
235 ldr w9, [x3, w9, uxtw] // src[filterPos[idx + 1]][0..3]235 ldr w9, [x3, w9, uxtw] // src[filterPos[idx + 1]][0..3]
236 ldr w10, [x3, w10, uxtw] // src[filterPos[idx + 2]][0..3]236 ldr w10, [x3, w10, uxtw] // src[filterPos[idx + 2]][0..3]
237 ldr w11, [x3, w11, uxtw] // src[filterPos[idx + 3]][0..3]237 ldr w11, [x3, w11, uxtw] // src[filterPos[idx + 3]][0..3]
238 ldr w12, [x3, w12, uxtw] // src[filterPos[idx + 4]][0..3]238 ldr w12, [x3, w12, uxtw] // src[filterPos[idx + 4]][0..3]
239 ldr w13, [x3, w13, uxtw] // src[filterPos[idx + 5]][0..3]239 ldr w13, [x3, w13, uxtw] // src[filterPos[idx + 5]][0..3]
240 ldr w14, [x3, w14, uxtw] // src[filterPos[idx + 6]][0..3]240 ldr w14, [x3, w14, uxtw] // src[filterPos[idx + 6]][0..3]
241 ldr w15, [x3, w15, uxtw] // src[filterPos[idx + 7]][0..3]241 ldr w15, [x3, w15, uxtw] // src[filterPos[idx + 7]][0..3]
242 stp w8, w9, [sp] // *scratch_mem = { src[filterPos[idx + 0]][0..3], src[filterPos[idx + 1]][0..3] }242 stp w8, w9, [sp] // *scratch_mem = { src[filterPos[idx + 0]][0..3], src[filterPos[idx + 1]][0..3] }
243 stp w10, w11, [sp, #8] // *scratch_mem = { src[filterPos[idx + 2]][0..3], src[filterPos[idx + 3]][0..3] }243 stp w10, w11, [sp, #8] // *scratch_mem = { src[filterPos[idx + 2]][0..3], src[filterPos[idx + 3]][0..3] }
244 stp w12, w13, [sp, #16] // *scratch_mem = { src[filterPos[idx + 4]][0..3], src[filterPos[idx + 5]][0..3] }244 stp w12, w13, [sp, #16] // *scratch_mem = { src[filterPos[idx + 4]][0..3], src[filterPos[idx + 5]][0..3] }
245 stp w14, w15, [sp, #24] // *scratch_mem = { src[filterPos[idx + 6]][0..3], src[filterPos[idx + 7]][0..3] }245 stp w14, w15, [sp, #24] // *scratch_mem = { src[filterPos[idx + 6]][0..3], src[filterPos[idx + 7]][0..3] }
246 246
2471:2471:
248 ld4 {v16.8b, v17.8b, v18.8b, v19.8b}, [sp] // transpose 8 bytes each from src into 4 registers248 ld4 {v16.8b, v17.8b, v18.8b, v19.8b}, [sp] // transpose 8 bytes each from src into 4 registers
249 249
250 // load 8 values from filterPos to be used as offsets into src250 // load 8 values from filterPos to be used as offsets into src
251 ldp w8, w9, [x5] // filterPos[idx + 0][0..3], [idx + 1][0..3], next iteration251 ldp w8, w9, [x5] // filterPos[idx + 0][0..3], [idx + 1][0..3], next iteration
252 ldp w10, w11, [x5, #8] // filterPos[idx + 2][0..3], [idx + 3][0..3], next iteration252 ldp w10, w11, [x5, #8] // filterPos[idx + 2][0..3], [idx + 3][0..3], next iteration
253 ldp w12, w13, [x5, #16] // filterPos[idx + 4][0..3], [idx + 5][0..3], next iteration253 ldp w12, w13, [x5, #16] // filterPos[idx + 4][0..3], [idx + 5][0..3], next iteration
254 ldp w14, w15, [x5, #24] // filterPos[idx + 6][0..3], [idx + 7][0..3], next iteration254 ldp w14, w15, [x5, #24] // filterPos[idx + 6][0..3], [idx + 7][0..3], next iteration
255 255
256 movi v0.2d, #0 // Clear madd accumulator for idx 0..3256 movi v0.2d, #0 // Clear madd accumulator for idx 0..3
257 movi v5.2d, #0 // Clear madd accumulator for idx 4..7257 movi v5.2d, #0 // Clear madd accumulator for idx 4..7
258 258
259 ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x4], #64 // load filter idx + 0..7259 ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x4], #64 // load filter idx + 0..7
260 260
261 add x5, x5, #32 // advance filterPos261 add x5, x5, #32 // advance filterPos
262 262
263 // interleaved SIMD and prefetching intended to keep ld/st and vector pipelines busy263 // interleaved SIMD and prefetching intended to keep ld/st and vector pipelines busy
264 uxtl v16.8h, v16.8b // unsigned extend long, covert src data to 16-bit264 uxtl v16.8h, v16.8b // unsigned extend long, covert src data to 16-bit
265 uxtl v17.8h, v17.8b // unsigned extend long, covert src data to 16-bit265 uxtl v17.8h, v17.8b // unsigned extend long, covert src data to 16-bit
266 ldr w8, [x3, w8, uxtw] // src[filterPos[idx + 0]], next iteration266 ldr w8, [x3, w8, uxtw] // src[filterPos[idx + 0]], next iteration
267 ldr w9, [x3, w9, uxtw] // src[filterPos[idx + 1]], next iteration267 ldr w9, [x3, w9, uxtw] // src[filterPos[idx + 1]], next iteration
268 uxtl v18.8h, v18.8b // unsigned extend long, covert src data to 16-bit268 uxtl v18.8h, v18.8b // unsigned extend long, covert src data to 16-bit
269 uxtl v19.8h, v19.8b // unsigned extend long, covert src data to 16-bit269 uxtl v19.8h, v19.8b // unsigned extend long, covert src data to 16-bit
270 ldr w10, [x3, w10, uxtw] // src[filterPos[idx + 2]], next iteration270 ldr w10, [x3, w10, uxtw] // src[filterPos[idx + 2]], next iteration
271 ldr w11, [x3, w11, uxtw] // src[filterPos[idx + 3]], next iteration271 ldr w11, [x3, w11, uxtw] // src[filterPos[idx + 3]], next iteration
272 272
273 smlal v0.4s, v1.4h, v16.4h // multiply accumulate inner loop j = 0, idx = 0..3273 smlal v0.4s, v1.4h, v16.4h // multiply accumulate inner loop j = 0, idx = 0..3
274 smlal v0.4s, v2.4h, v17.4h // multiply accumulate inner loop j = 1, idx = 0..3274 smlal v0.4s, v2.4h, v17.4h // multiply accumulate inner loop j = 1, idx = 0..3
275 ldr w12, [x3, w12, uxtw] // src[filterPos[idx + 4]], next iteration275 ldr w12, [x3, w12, uxtw] // src[filterPos[idx + 4]], next iteration
276 ldr w13, [x3, w13, uxtw] // src[filterPos[idx + 5]], next iteration276 ldr w13, [x3, w13, uxtw] // src[filterPos[idx + 5]], next iteration
277 smlal v0.4s, v3.4h, v18.4h // multiply accumulate inner loop j = 2, idx = 0..3277 smlal v0.4s, v3.4h, v18.4h // multiply accumulate inner loop j = 2, idx = 0..3
278 smlal v0.4s, v4.4h, v19.4h // multiply accumulate inner loop j = 3, idx = 0..3278 smlal v0.4s, v4.4h, v19.4h // multiply accumulate inner loop j = 3, idx = 0..3
279 ldr w14, [x3, w14, uxtw] // src[filterPos[idx + 6]], next iteration279 ldr w14, [x3, w14, uxtw] // src[filterPos[idx + 6]], next iteration
280 ldr w15, [x3, w15, uxtw] // src[filterPos[idx + 7]], next iteration280 ldr w15, [x3, w15, uxtw] // src[filterPos[idx + 7]], next iteration
281 281
282 smlal2 v5.4s, v1.8h, v16.8h // multiply accumulate inner loop j = 0, idx = 4..7282 smlal2 v5.4s, v1.8h, v16.8h // multiply accumulate inner loop j = 0, idx = 4..7
283 smlal2 v5.4s, v2.8h, v17.8h // multiply accumulate inner loop j = 1, idx = 4..7283 smlal2 v5.4s, v2.8h, v17.8h // multiply accumulate inner loop j = 1, idx = 4..7
284 stp w8, w9, [sp] // *scratch_mem = { src[filterPos[idx + 0]][0..3], src[filterPos[idx + 1]][0..3] }284 stp w8, w9, [sp] // *scratch_mem = { src[filterPos[idx + 0]][0..3], src[filterPos[idx + 1]][0..3] }
285 stp w10, w11, [sp, #8] // *scratch_mem = { src[filterPos[idx + 2]][0..3], src[filterPos[idx + 3]][0..3] }285 stp w10, w11, [sp, #8] // *scratch_mem = { src[filterPos[idx + 2]][0..3], src[filterPos[idx + 3]][0..3] }
286 smlal2 v5.4s, v3.8h, v18.8h // multiply accumulate inner loop j = 2, idx = 4..7286 smlal2 v5.4s, v3.8h, v18.8h // multiply accumulate inner loop j = 2, idx = 4..7
287 smlal2 v5.4s, v4.8h, v19.8h // multiply accumulate inner loop j = 3, idx = 4..7287 smlal2 v5.4s, v4.8h, v19.8h // multiply accumulate inner loop j = 3, idx = 4..7
288 stp w12, w13, [sp, #16] // *scratch_mem = { src[filterPos[idx + 4]][0..3], src[filterPos[idx + 5]][0..3] }288 stp w12, w13, [sp, #16] // *scratch_mem = { src[filterPos[idx + 4]][0..3], src[filterPos[idx + 5]][0..3] }
289 stp w14, w15, [sp, #24] // *scratch_mem = { src[filterPos[idx + 6]][0..3], src[filterPos[idx + 7]][0..3] }289 stp w14, w15, [sp, #24] // *scratch_mem = { src[filterPos[idx + 6]][0..3], src[filterPos[idx + 7]][0..3] }
290 290
291 sub w2, w2, #8 // dstW -= 8291 sub w2, w2, #8 // dstW -= 8
292 sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values292 sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values
293 sqshrn v1.4h, v5.4s, #7 // shift and clip the 2x16-bit final values293 sqshrn v1.4h, v5.4s, #7 // shift and clip the 2x16-bit final values
294 st1 {v0.4h, v1.4h}, [x1], #16 // write to dst[idx + 0..7]294 st1 {v0.4h, v1.4h}, [x1], #16 // write to dst[idx + 0..7]
295 cmp w2, #16 // continue on main loop if there are at least 16 iterations left295 cmp w2, #16 // continue on main loop if there are at least 16 iterations left
296 b.ge 1b296 b.ge 1b
297 297
298 // last full iteration298 // last full iteration
299 ld4 {v16.8b, v17.8b, v18.8b, v19.8b}, [sp]299 ld4 {v16.8b, v17.8b, v18.8b, v19.8b}, [sp]
300 ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x4], #64 // load filter idx + 0..7300 ld4 {v1.8h, v2.8h, v3.8h, v4.8h}, [x4], #64 // load filter idx + 0..7
301 301
302 movi v0.2d, #0 // Clear madd accumulator for idx 0..3302 movi v0.2d, #0 // Clear madd accumulator for idx 0..3
303 movi v5.2d, #0 // Clear madd accumulator for idx 4..7303 movi v5.2d, #0 // Clear madd accumulator for idx 4..7
304 304
305 uxtl v16.8h, v16.8b // unsigned extend long, covert src data to 16-bit305 uxtl v16.8h, v16.8b // unsigned extend long, covert src data to 16-bit
306 uxtl v17.8h, v17.8b // unsigned extend long, covert src data to 16-bit306 uxtl v17.8h, v17.8b // unsigned extend long, covert src data to 16-bit
307 uxtl v18.8h, v18.8b // unsigned extend long, covert src data to 16-bit307 uxtl v18.8h, v18.8b // unsigned extend long, covert src data to 16-bit
308 uxtl v19.8h, v19.8b // unsigned extend long, covert src data to 16-bit308 uxtl v19.8h, v19.8b // unsigned extend long, covert src data to 16-bit
309 309
310 smlal v0.4s, v1.4h, v16.4h // multiply accumulate inner loop j = 0, idx = 0..3310 smlal v0.4s, v1.4h, v16.4h // multiply accumulate inner loop j = 0, idx = 0..3
311 smlal v0.4s, v2.4h, v17.4h // multiply accumulate inner loop j = 1, idx = 0..3311 smlal v0.4s, v2.4h, v17.4h // multiply accumulate inner loop j = 1, idx = 0..3
312 smlal v0.4s, v3.4h, v18.4h // multiply accumulate inner loop j = 2, idx = 0..3312 smlal v0.4s, v3.4h, v18.4h // multiply accumulate inner loop j = 2, idx = 0..3
313 smlal v0.4s, v4.4h, v19.4h // multiply accumulate inner loop j = 3, idx = 0..3313 smlal v0.4s, v4.4h, v19.4h // multiply accumulate inner loop j = 3, idx = 0..3
314 314
315 smlal2 v5.4s, v1.8h, v16.8h // multiply accumulate inner loop j = 0, idx = 4..7315 smlal2 v5.4s, v1.8h, v16.8h // multiply accumulate inner loop j = 0, idx = 4..7
316 smlal2 v5.4s, v2.8h, v17.8h // multiply accumulate inner loop j = 1, idx = 4..7316 smlal2 v5.4s, v2.8h, v17.8h // multiply accumulate inner loop j = 1, idx = 4..7
317 smlal2 v5.4s, v3.8h, v18.8h // multiply accumulate inner loop j = 2, idx = 4..7317 smlal2 v5.4s, v3.8h, v18.8h // multiply accumulate inner loop j = 2, idx = 4..7
318 smlal2 v5.4s, v4.8h, v19.8h // multiply accumulate inner loop j = 3, idx = 4..7318 smlal2 v5.4s, v4.8h, v19.8h // multiply accumulate inner loop j = 3, idx = 4..7
319 319
320 subs w2, w2, #8 // dstW -= 8320 subs w2, w2, #8 // dstW -= 8
321 sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values321 sqshrn v0.4h, v0.4s, #7 // shift and clip the 2x16-bit final values
322 sqshrn v1.4h, v5.4s, #7 // shift and clip the 2x16-bit final values322 sqshrn v1.4h, v5.4s, #7 // shift and clip the 2x16-bit final values
323 st1 {v0.4h, v1.4h}, [x1], #16 // write to dst[idx + 0..7]323 st1 {v0.4h, v1.4h}, [x1], #16 // write to dst[idx + 0..7]
324 324
325 cbnz w2, 2f // if >0 iterations remain, jump to the wrap up section325 cbnz w2, 2f // if >0 iterations remain, jump to the wrap up section
326 326
327 add sp, sp, #32 // clean up stack327 add sp, sp, #32 // clean up stack
328 ret328 ret
329 329
330 // finish up when dstW % 8 != 0 or dstW < 16330 // finish up when dstW % 8 != 0 or dstW < 16
3312:3312:
332 // load src332 // load src
333 ldr w8, [x5], #4 // filterPos[i]333 ldr w8, [x5], #4 // filterPos[i]
334 add x9, x3, w8, uxtw // calculate the address for src load334 add x9, x3, w8, uxtw // calculate the address for src load
335 ld1 {v5.s}[0], [x9] // src[filterPos[i] + 0..3]335 ld1 {v5.s}[0], [x9] // src[filterPos[i] + 0..3]
336 // load filter336 // load filter
337 ld1 {v6.4h}, [x4], #8 // filter[filterSize * i + 0..3]337 ld1 {v6.4h}, [x4], #8 // filter[filterSize * i + 0..3]
338 338
339 uxtl v5.8h, v5.8b // unsigned exten long, convert src data to 16-bit339 uxtl v5.8h, v5.8b // unsigned exten long, convert src data to 16-bit
340 smull v0.4s, v5.4h, v6.4h // 4 iterations of src[...] * filter[...]340 smull v0.4s, v5.4h, v6.4h // 4 iterations of src[...] * filter[...]
341 addv s0, v0.4s // add up products of src and filter values341 addv s0, v0.4s // add up products of src and filter values
342 sqshrn h0, s0, #7 // shift and clip the 2x16-bit final value342 sqshrn h0, s0, #7 // shift and clip the 2x16-bit final value
343 st1 {v0.h}[0], [x1], #2 // dst[i] = ...343 st1 {v0.h}[0], [x1], #2 // dst[i] = ...
344 sub w2, w2, #1 // dstW--344 sub w2, w2, #1 // dstW--
345 cbnz w2, 2b345 cbnz w2, 2b
346 346
347 add sp, sp, #32 // clean up stack347 add sp, sp, #32 // clean up stack
348 ret348 ret
349endfunc349endfunc
350 350
@@ -357,187 +357,187 @@ function ff_hscale8to19_4_neon, export=1 · 6 unmodified lines
357 // x5 const int32_t *filterPos357 // x5 const int32_t *filterPos
358 // w6 int filterSize358 // w6 int filterSize
359 359
360 movi v18.4s, #1360 movi v18.4s, #1
361 movi v17.4s, #1361 movi v17.4s, #1
362 shl v18.4s, v18.4s, #19362 shl v18.4s, v18.4s, #19
363 sub v18.4s, v18.4s, v17.4s // max allowed value363 sub v18.4s, v18.4s, v17.4s // max allowed value
364 364
365 cmp w2, #16365 cmp w2, #16
366 b.lt 2f // move to last block366 b.lt 2f // move to last block
367 367
368 ldp w8, w9, [x5] // filterPos[0], filterPos[1]368 ldp w8, w9, [x5] // filterPos[0], filterPos[1]
369 ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]369 ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
370 ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]370 ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
371 ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]371 ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
372 add x5, x5, #32372 add x5, x5, #32
373 373
374 // load data from374 // load data from
375 ldr w8, [x3, w8, uxtw]375 ldr w8, [x3, w8, uxtw]
376 ldr w9, [x3, w9, uxtw]376 ldr w9, [x3, w9, uxtw]
377 ldr w10, [x3, w10, uxtw]377 ldr w10, [x3, w10, uxtw]
378 ldr w11, [x3, w11, uxtw]378 ldr w11, [x3, w11, uxtw]
379 ldr w12, [x3, w12, uxtw]379 ldr w12, [x3, w12, uxtw]
380 ldr w13, [x3, w13, uxtw]380 ldr w13, [x3, w13, uxtw]
381 ldr w14, [x3, w14, uxtw]381 ldr w14, [x3, w14, uxtw]
382 ldr w15, [x3, w15, uxtw]382 ldr w15, [x3, w15, uxtw]
383 383
384 sub sp, sp, #32384 sub sp, sp, #32
385 385
386 stp w8, w9, [sp]386 stp w8, w9, [sp]
387 stp w10, w11, [sp, #8]387 stp w10, w11, [sp, #8]
388 stp w12, w13, [sp, #16]388 stp w12, w13, [sp, #16]
389 stp w14, w15, [sp, #24]389 stp w14, w15, [sp, #24]
390 390
3911:3911:
392 ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [sp]392 ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [sp]
393 ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]393 ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]
394 // load filterPositions into registers for next iteration394 // load filterPositions into registers for next iteration
395 395
396 ldp w8, w9, [x5] // filterPos[0], filterPos[1]396 ldp w8, w9, [x5] // filterPos[0], filterPos[1]
397 ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]397 ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
398 ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]398 ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
399 ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]399 ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
400 add x5, x5, #32400 add x5, x5, #32
401 uxtl v0.8h, v0.8b401 uxtl v0.8h, v0.8b
402 ldr w8, [x3, w8, uxtw]402 ldr w8, [x3, w8, uxtw]
403 smull v5.4s, v0.4h, v28.4h // multiply first column of src403 smull v5.4s, v0.4h, v28.4h // multiply first column of src
404 ldr w9, [x3, w9, uxtw]404 ldr w9, [x3, w9, uxtw]
405 smull2 v6.4s, v0.8h, v28.8h405 smull2 v6.4s, v0.8h, v28.8h
406 stp w8, w9, [sp]406 stp w8, w9, [sp]
407 407
408 uxtl v1.8h, v1.8b408 uxtl v1.8h, v1.8b
409 ldr w10, [x3, w10, uxtw]409 ldr w10, [x3, w10, uxtw]
410 smlal v5.4s, v1.4h, v29.4h // multiply second column of src410 smlal v5.4s, v1.4h, v29.4h // multiply second column of src
411 ldr w11, [x3, w11, uxtw]411 ldr w11, [x3, w11, uxtw]
412 smlal2 v6.4s, v1.8h, v29.8h412 smlal2 v6.4s, v1.8h, v29.8h
413 stp w10, w11, [sp, #8]413 stp w10, w11, [sp, #8]
414 414
415 uxtl v2.8h, v2.8b415 uxtl v2.8h, v2.8b
416 ldr w12, [x3, w12, uxtw]416 ldr w12, [x3, w12, uxtw]
417 smlal v5.4s, v2.4h, v30.4h // multiply third column of src417 smlal v5.4s, v2.4h, v30.4h // multiply third column of src
418 ldr w13, [x3, w13, uxtw]418 ldr w13, [x3, w13, uxtw]
419 smlal2 v6.4s, v2.8h, v30.8h419 smlal2 v6.4s, v2.8h, v30.8h
420 stp w12, w13, [sp, #16]420 stp w12, w13, [sp, #16]
421 421
422 uxtl v3.8h, v3.8b422 uxtl v3.8h, v3.8b
423 ldr w14, [x3, w14, uxtw]423 ldr w14, [x3, w14, uxtw]
424 smlal v5.4s, v3.4h, v31.4h // multiply fourth column of src424 smlal v5.4s, v3.4h, v31.4h // multiply fourth column of src
425 ldr w15, [x3, w15, uxtw]425 ldr w15, [x3, w15, uxtw]
426 smlal2 v6.4s, v3.8h, v31.8h426 smlal2 v6.4s, v3.8h, v31.8h
427 stp w14, w15, [sp, #24]427 stp w14, w15, [sp, #24]
428 428
429 sub w2, w2, #8429 sub w2, w2, #8
430 sshr v5.4s, v5.4s, #3430 sshr v5.4s, v5.4s, #3
431 sshr v6.4s, v6.4s, #3431 sshr v6.4s, v6.4s, #3
432 smin v5.4s, v5.4s, v18.4s432 smin v5.4s, v5.4s, v18.4s
433 smin v6.4s, v6.4s, v18.4s433 smin v6.4s, v6.4s, v18.4s
434 434
435 st1 {v5.4s, v6.4s}, [x1], #32435 st1 {v5.4s, v6.4s}, [x1], #32
436 cmp w2, #16436 cmp w2, #16
437 b.ge 1b437 b.ge 1b
438 438
439 // here we make last iteration, without updating the registers439 // here we make last iteration, without updating the registers
440 ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [sp]440 ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [sp]
441 ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]441 ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]
442 442
443 uxtl v0.8h, v0.8b443 uxtl v0.8h, v0.8b
444 uxtl v1.8h, v1.8b444 uxtl v1.8h, v1.8b
445 smull v5.4s, v0.4h, v28.4h445 smull v5.4s, v0.4h, v28.4h
446 smull2 v6.4s, v0.8h, v28.8h446 smull2 v6.4s, v0.8h, v28.8h
447 uxtl v2.8h, v2.8b447 uxtl v2.8h, v2.8b
448 smlal v5.4s, v1.4h, v29.4h448 smlal v5.4s, v1.4h, v29.4h
449 smlal2 v6.4s, v1.8h, v29.8h449 smlal2 v6.4s, v1.8h, v29.8h
450 uxtl v3.8h, v3.8b450 uxtl v3.8h, v3.8b
451 smlal v5.4s, v2.4h, v30.4h451 smlal v5.4s, v2.4h, v30.4h
452 smlal2 v6.4s, v2.8h, v30.8h452 smlal2 v6.4s, v2.8h, v30.8h
453 smlal v5.4s, v3.4h, v31.4h453 smlal v5.4s, v3.4h, v31.4h
454 smlal2 v6.4s, v3.8h, v31.8h454 smlal2 v6.4s, v3.8h, v31.8h
455 455
456 sshr v5.4s, v5.4s, #3456 sshr v5.4s, v5.4s, #3
457 sshr v6.4s, v6.4s, #3457 sshr v6.4s, v6.4s, #3
458 458
459 smin v5.4s, v5.4s, v18.4s459 smin v5.4s, v5.4s, v18.4s
460 smin v6.4s, v6.4s, v18.4s460 smin v6.4s, v6.4s, v18.4s
461 461
462 sub w2, w2, #8462 sub w2, w2, #8
463 st1 {v5.4s, v6.4s}, [x1], #32463 st1 {v5.4s, v6.4s}, [x1], #32
464 add sp, sp, #32 // restore stack464 add sp, sp, #32 // restore stack
465 cbnz w2, 2f465 cbnz w2, 2f
466 466
467 ret467 ret
468 468
4692:4692:
470 ldr w8, [x5], #4 // load filterPos470 ldr w8, [x5], #4 // load filterPos
471 add x9, x3, w8, uxtw // src + filterPos471 add x9, x3, w8, uxtw // src + filterPos
472 ld1 {v0.s}[0], [x9] // load 4 * uint8_t* into one single472 ld1 {v0.s}[0], [x9] // load 4 * uint8_t* into one single
473 ld1 {v31.4h}, [x4], #8473 ld1 {v31.4h}, [x4], #8
474 uxtl v0.8h, v0.8b474 uxtl v0.8h, v0.8b
475 smull v5.4s, v0.4h, v31.4h475 smull v5.4s, v0.4h, v31.4h
476 saddlv d0, v5.4s476 saddlv d0, v5.4s
477 sqshrn s0, d0, #3477 sqshrn s0, d0, #3
478 smin v0.4s, v0.4s, v18.4s478 smin v0.4s, v0.4s, v18.4s
479 st1 {v0.s}[0], [x1], #4479 st1 {v0.s}[0], [x1], #4
480 sub w2, w2, #1480 sub w2, w2, #1
481 cbnz w2, 2b // if iterations remain jump to beginning481 cbnz w2, 2b // if iterations remain jump to beginning
482 482
483 ret483 ret
484endfunc484endfunc
485 485
486function ff_hscale8to19_X8_neon, export=1486function ff_hscale8to19_X8_neon, export=1
487 movi v20.4s, #1487 movi v20.4s, #1
488 movi v17.4s, #1488 movi v17.4s, #1
489 shl v20.4s, v20.4s, #19489 shl v20.4s, v20.4s, #19
490 sub v20.4s, v20.4s, v17.4s490 sub v20.4s, v20.4s, v17.4s
491 491
492 sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)492 sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)
4931:4931:
494 mov x16, x4 // filter0 = filter494 mov x16, x4 // filter0 = filter
495 ldr w8, [x5], #4 // filterPos[idx]495 ldr w8, [x5], #4 // filterPos[idx]
496 add x12, x16, x7 // filter1 = filter0 + filterSize*2496 add x12, x16, x7 // filter1 = filter0 + filterSize*2
497 ldr w0, [x5], #4 // filterPos[idx + 1]497 ldr w0, [x5], #4 // filterPos[idx + 1]
498 add x13, x12, x7 // filter2 = filter1 + filterSize*2498 add x13, x12, x7 // filter2 = filter1 + filterSize*2
499 ldr w11, [x5], #4 // filterPos[idx + 2]499 ldr w11, [x5], #4 // filterPos[idx + 2]
500 add x4, x13, x7 // filter3 = filter2 + filterSize*2500 add x4, x13, x7 // filter3 = filter2 + filterSize*2
501 ldr w9, [x5], #4 // filterPos[idx + 3]501 ldr w9, [x5], #4 // filterPos[idx + 3]
502 movi v0.2d, #0 // val sum part 1 (for dst[0])502 movi v0.2d, #0 // val sum part 1 (for dst[0])
503 movi v1.2d, #0 // val sum part 2 (for dst[1])503 movi v1.2d, #0 // val sum part 2 (for dst[1])
504 movi v2.2d, #0 // val sum part 3 (for dst[2])504 movi v2.2d, #0 // val sum part 3 (for dst[2])
505 movi v3.2d, #0 // val sum part 4 (for dst[3])505 movi v3.2d, #0 // val sum part 4 (for dst[3])
506 add x17, x3, w8, uxtw // srcp + filterPos[0]506 add x17, x3, w8, uxtw // srcp + filterPos[0]
507 add x8, x3, w0, uxtw // srcp + filterPos[1]507 add x8, x3, w0, uxtw // srcp + filterPos[1]
508 add x0, x3, w11, uxtw // srcp + filterPos[2]508 add x0, x3, w11, uxtw // srcp + filterPos[2]
509 add x11, x3, w9, uxtw // srcp + filterPos[3]509 add x11, x3, w9, uxtw // srcp + filterPos[3]
510 mov w15, w6 // filterSize counter510 mov w15, w6 // filterSize counter
5112: ld1 {v4.8b}, [x17], #8 // srcp[filterPos[0] + {0..7}]5112: ld1 {v4.8b}, [x17], #8 // srcp[filterPos[0] + {0..7}]
512 ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1512 ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1
513 uxtl v4.8h, v4.8b // unpack part 1 to 16-bit513 uxtl v4.8h, v4.8b // unpack part 1 to 16-bit
514 smlal v0.4s, v4.4h, v5.4h // v0 accumulates srcp[filterPos[0] + {0..3}] * filter[{0..3}]514 smlal v0.4s, v4.4h, v5.4h // v0 accumulates srcp[filterPos[0] + {0..3}] * filter[{0..3}]
515 ld1 {v6.8b}, [x8], #8 // srcp[filterPos[1] + {0..7}]515 ld1 {v6.8b}, [x8], #8 // srcp[filterPos[1] + {0..7}]
516 smlal2 v0.4s, v4.8h, v5.8h // v0 accumulates srcp[filterPos[0] + {4..7}] * filter[{4..7}]516 smlal2 v0.4s, v4.8h, v5.8h // v0 accumulates srcp[filterPos[0] + {4..7}] * filter[{4..7}]
517 ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize517 ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize
518 ld1 {v16.8b}, [x0], #8 // srcp[filterPos[2] + {0..7}]518 ld1 {v16.8b}, [x0], #8 // srcp[filterPos[2] + {0..7}]
519 uxtl v6.8h, v6.8b // unpack part 2 to 16-bit519 uxtl v6.8h, v6.8b // unpack part 2 to 16-bit
520 ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize520 ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize
521 uxtl v16.8h, v16.8b // unpack part 3 to 16-bit521 uxtl v16.8h, v16.8b // unpack part 3 to 16-bit
522 smlal v1.4s, v6.4h, v7.4h // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]522 smlal v1.4s, v6.4h, v7.4h // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]
523 ld1 {v18.8b}, [x11], #8 // srcp[filterPos[3] + {0..7}]523 ld1 {v18.8b}, [x11], #8 // srcp[filterPos[3] + {0..7}]
524 smlal v2.4s, v16.4h, v17.4h // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]524 smlal v2.4s, v16.4h, v17.4h // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]
525 ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize525 ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize
526 smlal2 v2.4s, v16.8h, v17.8h // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]526 smlal2 v2.4s, v16.8h, v17.8h // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]
527 uxtl v18.8h, v18.8b // unpack part 4 to 16-bit527 uxtl v18.8h, v18.8b // unpack part 4 to 16-bit
528 smlal2 v1.4s, v6.8h, v7.8h // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]528 smlal2 v1.4s, v6.8h, v7.8h // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]
529 smlal v3.4s, v18.4h, v19.4h // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]529 smlal v3.4s, v18.4h, v19.4h // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]
530 subs w15, w15, #8 // j -= 8: processed 8/filterSize530 subs w15, w15, #8 // j -= 8: processed 8/filterSize
531 smlal2 v3.4s, v18.8h, v19.8h // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]531 smlal2 v3.4s, v18.8h, v19.8h // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]
532 b.gt 2b // inner loop if filterSize not consumed completely532 b.gt 2b // inner loop if filterSize not consumed completely
533 addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding533 addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding
534 addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding534 addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding
535 addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding535 addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding
536 subs w2, w2, #4 // dstW -= 4536 subs w2, w2, #4 // dstW -= 4
537 sshr v0.4s, v0.4s, #3 // shift and clip the 2x16-bit final values537 sshr v0.4s, v0.4s, #3 // shift and clip the 2x16-bit final values
538 smin v0.4s, v0.4s, v20.4s538 smin v0.4s, v0.4s, v20.4s
539 st1 {v0.4s}, [x1], #16 // write to destination part0123539 st1 {v0.4s}, [x1], #16 // write to destination part0123
540 b.gt 1b // loop until end of line540 b.gt 1b // loop until end of line
541 ret541 ret
542endfunc542endfunc
543 543
@@ -550,91 +550,91 @@ function ff_hscale8to19_X4_neon, export=1 · 6 unmodified lines
550 // x5 const int32_t *filterPos550 // x5 const int32_t *filterPos
551 // w6 int filterSize551 // w6 int filterSize
552 552
553 movi v20.4s, #1553 movi v20.4s, #1
554 movi v17.4s, #1554 movi v17.4s, #1
555 shl v20.4s, v20.4s, #19555 shl v20.4s, v20.4s, #19
556 sub v20.4s, v20.4s, v17.4s556 sub v20.4s, v20.4s, v17.4s
557 557
558 lsl w7, w6, #1558 lsl w7, w6, #1
5591:5591:
560 ldp w8, w9, [x5]560 ldp w8, w9, [x5]
561 ldp w10, w11, [x5, #8]561 ldp w10, w11, [x5, #8]
562 562
563 movi v16.2d, #0 // initialize accumulator for idx + 0563 movi v16.2d, #0 // initialize accumulator for idx + 0
564 movi v17.2d, #0 // initialize accumulator for idx + 1564 movi v17.2d, #0 // initialize accumulator for idx + 1
565 movi v18.2d, #0 // initialize accumulator for idx + 2565 movi v18.2d, #0 // initialize accumulator for idx + 2
566 movi v19.2d, #0 // initialize accumulator for idx + 3566 movi v19.2d, #0 // initialize accumulator for idx + 3
567 567
568 mov x12, x4 // filter + 0568 mov x12, x4 // filter + 0
569 add x13, x4, x7 // filter + 1569 add x13, x4, x7 // filter + 1
570 add x8, x3, w8, uxtw // srcp + filterPos 0570 add x8, x3, w8, uxtw // srcp + filterPos 0
571 add x14, x13, x7 // filter + 2571 add x14, x13, x7 // filter + 2
572 add x9, x3, w9, uxtw // srcp + filterPos 1572 add x9, x3, w9, uxtw // srcp + filterPos 1
573 add x15, x14, x7 // filter + 3573 add x15, x14, x7 // filter + 3
574 add x10, x3, w10, uxtw // srcp + filterPos 2574 add x10, x3, w10, uxtw // srcp + filterPos 2
575 mov w0, w6 // save the filterSize to temporary variable575 mov w0, w6 // save the filterSize to temporary variable
576 add x11, x3, w11, uxtw // srcp + filterPos 3576 add x11, x3, w11, uxtw // srcp + filterPos 3
577 add x5, x5, #16 // advance filter position577 add x5, x5, #16 // advance filter position
578 mov x16, xzr // clear the register x16 used for offsetting the filter values578 mov x16, xzr // clear the register x16 used for offsetting the filter values
579 579
5802:5802:
581 ldr d4, [x8], #8 // load src values for idx 0581 ldr d4, [x8], #8 // load src values for idx 0
582 ldr q31, [x12, x16] // load filter values for idx 0582 ldr q31, [x12, x16] // load filter values for idx 0
583 uxtl v4.8h, v4.8b // extend type to match the filter' size583 uxtl v4.8h, v4.8b // extend type to match the filter' size
584 ldr d5, [x9], #8 // load src values for idx 1584 ldr d5, [x9], #8 // load src values for idx 1
585 smlal v16.4s, v4.4h, v31.4h // multiplication of lower half for idx 0585 smlal v16.4s, v4.4h, v31.4h // multiplication of lower half for idx 0
586 uxtl v5.8h, v5.8b // extend type to match the filter' size586 uxtl v5.8h, v5.8b // extend type to match the filter' size
587 ldr q30, [x13, x16] // load filter values for idx 1587 ldr q30, [x13, x16] // load filter values for idx 1
588 smlal2 v16.4s, v4.8h, v31.8h // multiplication of upper half for idx 0588 smlal2 v16.4s, v4.8h, v31.8h // multiplication of upper half for idx 0
589 ldr d6, [x10], #8 // load src values for idx 2589 ldr d6, [x10], #8 // load src values for idx 2
590 ldr q29, [x14, x16] // load filter values for idx 2590 ldr q29, [x14, x16] // load filter values for idx 2
591 smlal v17.4s, v5.4h, v30.4h // multiplication of lower half for idx 1591 smlal v17.4s, v5.4h, v30.4h // multiplication of lower half for idx 1
592 ldr d7, [x11], #8 // load src values for idx 3592 ldr d7, [x11], #8 // load src values for idx 3
593 smlal2 v17.4s, v5.8h, v30.8h // multiplication of upper half for idx 1593 smlal2 v17.4s, v5.8h, v30.8h // multiplication of upper half for idx 1
594 uxtl v6.8h, v6.8b // extend tpye to matchi the filter's size594 uxtl v6.8h, v6.8b // extend tpye to matchi the filter's size
595 ldr q28, [x15, x16] // load filter values for idx 3595 ldr q28, [x15, x16] // load filter values for idx 3
596 smlal v18.4s, v6.4h, v29.4h // multiplication of lower half for idx 2596 smlal v18.4s, v6.4h, v29.4h // multiplication of lower half for idx 2
597 uxtl v7.8h, v7.8b597 uxtl v7.8h, v7.8b
598 smlal2 v18.4s, v6.8h, v29.8h // multiplication of upper half for idx 2598 smlal2 v18.4s, v6.8h, v29.8h // multiplication of upper half for idx 2
599 sub w0, w0, #8599 sub w0, w0, #8
600 smlal v19.4s, v7.4h, v28.4h // multiplication of lower half for idx 3600 smlal v19.4s, v7.4h, v28.4h // multiplication of lower half for idx 3
601 cmp w0, #8601 cmp w0, #8
602 smlal2 v19.4s, v7.8h, v28.8h // multiplication of upper half for idx 3602 smlal2 v19.4s, v7.8h, v28.8h // multiplication of upper half for idx 3
603 add x16, x16, #16 // advance filter values indexing603 add x16, x16, #16 // advance filter values indexing
604 604
605 b.ge 2b605 b.ge 2b
606 606
607 607
608 // 4 iterations left608 // 4 iterations left
609 609
610 sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements610 sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements
611 611
612 ldr s4, [x8] // load src values for idx 0612 ldr s4, [x8] // load src values for idx 0
613 ldr d31, [x12, x17] // load filter values for idx 0613 ldr d31, [x12, x17] // load filter values for idx 0
614 uxtl v4.8h, v4.8b // extend type to match the filter' size614 uxtl v4.8h, v4.8b // extend type to match the filter' size
615 ldr s5, [x9] // load src values for idx 1615 ldr s5, [x9] // load src values for idx 1
616 smlal v16.4s, v4.4h, v31.4h616 smlal v16.4s, v4.4h, v31.4h
617 ldr d30, [x13, x17] // load filter values for idx 1617 ldr d30, [x13, x17] // load filter values for idx 1
618 uxtl v5.8h, v5.8b // extend type to match the filter' size618 uxtl v5.8h, v5.8b // extend type to match the filter' size
619 ldr s6, [x10] // load src values for idx 2619 ldr s6, [x10] // load src values for idx 2
620 smlal v17.4s, v5.4h, v30.4h620 smlal v17.4s, v5.4h, v30.4h
621 uxtl v6.8h, v6.8b // extend type to match the filter's size621 uxtl v6.8h, v6.8b // extend type to match the filter's size
622 ldr d29, [x14, x17] // load filter values for idx 2622 ldr d29, [x14, x17] // load filter values for idx 2
623 ldr s7, [x11] // load src values for idx 3623 ldr s7, [x11] // load src values for idx 3
624 addp v16.4s, v16.4s, v17.4s624 addp v16.4s, v16.4s, v17.4s
625 uxtl v7.8h, v7.8b625 uxtl v7.8h, v7.8b
626 ldr d28, [x15, x17] // load filter values for idx 3626 ldr d28, [x15, x17] // load filter values for idx 3
627 smlal v18.4s, v6.4h, v29.4h627 smlal v18.4s, v6.4h, v29.4h
628 smlal v19.4s, v7.4h, v28.4h628 smlal v19.4s, v7.4h, v28.4h
629 subs w2, w2, #4629 subs w2, w2, #4
630 addp v18.4s, v18.4s, v19.4s630 addp v18.4s, v18.4s, v19.4s
631 addp v16.4s, v16.4s, v18.4s631 addp v16.4s, v16.4s, v18.4s
632 sshr v16.4s, v16.4s, #3632 sshr v16.4s, v16.4s, #3
633 smin v16.4s, v16.4s, v20.4s633 smin v16.4s, v16.4s, v20.4s
634 634
635 st1 {v16.4s}, [x1], #16635 st1 {v16.4s}, [x1], #16
636 add x4, x4, x7, lsl #2636 add x4, x4, x7, lsl #2
637 b.gt 1b637 b.gt 1b
638 ret638 ret
639endfunc639endfunc
640 640
@@ -647,191 +647,191 @@ function ff_hscale16to15_4_neon_asm, export=1 · 6 unmodified lines
647 // x5 const int32_t *filterPos647 // x5 const int32_t *filterPos
648 // w6 int filterSize648 // w6 int filterSize
649 649
650 movi v18.4s, #1650 movi v18.4s, #1
651 movi v17.4s, #1651 movi v17.4s, #1
652 shl v18.4s, v18.4s, #15652 shl v18.4s, v18.4s, #15
653 sub v18.4s, v18.4s, v17.4s // max allowed value653 sub v18.4s, v18.4s, v17.4s // max allowed value
654 dup v17.4s, w0 // read shift654 dup v17.4s, w0 // read shift
655 neg v17.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)655 neg v17.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)
656 656
657 cmp w2, #16657 cmp w2, #16
658 b.lt 2f // move to last block658 b.lt 2f // move to last block
659 659
660 ldp w8, w9, [x5] // filterPos[0], filterPos[1]660 ldp w8, w9, [x5] // filterPos[0], filterPos[1]
661 ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]661 ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
662 ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]662 ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
663 ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]663 ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
664 add x5, x5, #32664 add x5, x5, #32
665 665
666 // shift all filterPos left by one, as uint16_t will be read666 // shift all filterPos left by one, as uint16_t will be read
667 lsl x8, x8, #1667 lsl x8, x8, #1
668 lsl x9, x9, #1668 lsl x9, x9, #1
669 lsl x10, x10, #1669 lsl x10, x10, #1
670 lsl x11, x11, #1670 lsl x11, x11, #1
671 lsl x12, x12, #1671 lsl x12, x12, #1
672 lsl x13, x13, #1672 lsl x13, x13, #1
673 lsl x14, x14, #1673 lsl x14, x14, #1
674 lsl x15, x15, #1674 lsl x15, x15, #1
675 675
676 // load src with given offset676 // load src with given offset
677 ldr x8, [x3, w8, uxtw]677 ldr x8, [x3, w8, uxtw]
678 ldr x9, [x3, w9, uxtw]678 ldr x9, [x3, w9, uxtw]
679 ldr x10, [x3, w10, uxtw]679 ldr x10, [x3, w10, uxtw]
680 ldr x11, [x3, w11, uxtw]680 ldr x11, [x3, w11, uxtw]
681 ldr x12, [x3, w12, uxtw]681 ldr x12, [x3, w12, uxtw]
682 ldr x13, [x3, w13, uxtw]682 ldr x13, [x3, w13, uxtw]
683 ldr x14, [x3, w14, uxtw]683 ldr x14, [x3, w14, uxtw]
684 ldr x15, [x3, w15, uxtw]684 ldr x15, [x3, w15, uxtw]
685 685
686 sub sp, sp, #64686 sub sp, sp, #64
687 // push src on stack so it can be loaded into vectors later687 // push src on stack so it can be loaded into vectors later
688 stp x8, x9, [sp]688 stp x8, x9, [sp]
689 stp x10, x11, [sp, #16]689 stp x10, x11, [sp, #16]
690 stp x12, x13, [sp, #32]690 stp x12, x13, [sp, #32]
691 stp x14, x15, [sp, #48]691 stp x14, x15, [sp, #48]
692 692
6931:6931:
694 ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]694 ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]
695 ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]695 ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]
696 696
697 // Each of blocks does the following:697 // Each of blocks does the following:
698 // Extend src and filter to 32 bits with uxtl and sxtl698 // Extend src and filter to 32 bits with uxtl and sxtl
699 // multiply or multiply and accumulate results699 // multiply or multiply and accumulate results
700 // Extending to 32 bits is necessary, as unit16_t values can't700 // Extending to 32 bits is necessary, as unit16_t values can't
701 // be represented as int16_t without type promotion.701 // be represented as int16_t without type promotion.
702 uxtl v26.4s, v0.4h702 uxtl v26.4s, v0.4h
703 sxtl v27.4s, v28.4h703 sxtl v27.4s, v28.4h
704 uxtl2 v0.4s, v0.8h704 uxtl2 v0.4s, v0.8h
705 mul v5.4s, v26.4s, v27.4s705 mul v5.4s, v26.4s, v27.4s
706 sxtl2 v28.4s, v28.8h706 sxtl2 v28.4s, v28.8h
707 uxtl v26.4s, v1.4h707 uxtl v26.4s, v1.4h
708 mul v6.4s, v0.4s, v28.4s708 mul v6.4s, v0.4s, v28.4s
709 709
710 sxtl v27.4s, v29.4h710 sxtl v27.4s, v29.4h
711 uxtl2 v0.4s, v1.8h711 uxtl2 v0.4s, v1.8h
712 mla v5.4s, v27.4s, v26.4s712 mla v5.4s, v27.4s, v26.4s
713 sxtl2 v28.4s, v29.8h713 sxtl2 v28.4s, v29.8h
714 uxtl v26.4s, v2.4h714 uxtl v26.4s, v2.4h
715 mla v6.4s, v28.4s, v0.4s715 mla v6.4s, v28.4s, v0.4s
716 716
717 sxtl v27.4s, v30.4h717 sxtl v27.4s, v30.4h
718 uxtl2 v0.4s, v2.8h718 uxtl2 v0.4s, v2.8h
719 mla v5.4s, v27.4s, v26.4s719 mla v5.4s, v27.4s, v26.4s
720 sxtl2 v28.4s, v30.8h720 sxtl2 v28.4s, v30.8h
721 uxtl v26.4s, v3.4h721 uxtl v26.4s, v3.4h
722 mla v6.4s, v28.4s, v0.4s722 mla v6.4s, v28.4s, v0.4s
723 723
724 sxtl v27.4s, v31.4h724 sxtl v27.4s, v31.4h
725 uxtl2 v0.4s, v3.8h725 uxtl2 v0.4s, v3.8h
726 mla v5.4s, v27.4s, v26.4s726 mla v5.4s, v27.4s, v26.4s
727 sxtl2 v28.4s, v31.8h727 sxtl2 v28.4s, v31.8h
728 sub w2, w2, #8728 sub w2, w2, #8
729 mla v6.4s, v28.4s, v0.4s729 mla v6.4s, v28.4s, v0.4s
730 730
731 sshl v5.4s, v5.4s, v17.4s731 sshl v5.4s, v5.4s, v17.4s
732 sshl v6.4s, v6.4s, v17.4s732 sshl v6.4s, v6.4s, v17.4s
733 smin v5.4s, v5.4s, v18.4s733 smin v5.4s, v5.4s, v18.4s
734 smin v6.4s, v6.4s, v18.4s734 smin v6.4s, v6.4s, v18.4s
735 xtn v5.4h, v5.4s735 xtn v5.4h, v5.4s
736 xtn2 v5.8h, v6.4s736 xtn2 v5.8h, v6.4s
737 737
738 st1 {v5.8h}, [x1], #16738 st1 {v5.8h}, [x1], #16
739 cmp w2, #16739 cmp w2, #16
740 740
741 // load filterPositions into registers for next iteration741 // load filterPositions into registers for next iteration
742 ldp w8, w9, [x5] // filterPos[0], filterPos[1]742 ldp w8, w9, [x5] // filterPos[0], filterPos[1]
743 ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]743 ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
744 ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]744 ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
745 ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]745 ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
746 add x5, x5, #32746 add x5, x5, #32
747 747
748 lsl x8, x8, #1748 lsl x8, x8, #1
749 lsl x9, x9, #1749 lsl x9, x9, #1
750 lsl x10, x10, #1750 lsl x10, x10, #1
751 lsl x11, x11, #1751 lsl x11, x11, #1
752 lsl x12, x12, #1752 lsl x12, x12, #1
753 lsl x13, x13, #1753 lsl x13, x13, #1
754 lsl x14, x14, #1754 lsl x14, x14, #1
755 lsl x15, x15, #1755 lsl x15, x15, #1
756 756
757 ldr x8, [x3, w8, uxtw]757 ldr x8, [x3, w8, uxtw]
758 ldr x9, [x3, w9, uxtw]758 ldr x9, [x3, w9, uxtw]
759 ldr x10, [x3, w10, uxtw]759 ldr x10, [x3, w10, uxtw]
760 ldr x11, [x3, w11, uxtw]760 ldr x11, [x3, w11, uxtw]
761 ldr x12, [x3, w12, uxtw]761 ldr x12, [x3, w12, uxtw]
762 ldr x13, [x3, w13, uxtw]762 ldr x13, [x3, w13, uxtw]
763 ldr x14, [x3, w14, uxtw]763 ldr x14, [x3, w14, uxtw]
764 ldr x15, [x3, w15, uxtw]764 ldr x15, [x3, w15, uxtw]
765 765
766 stp x8, x9, [sp]766 stp x8, x9, [sp]
767 stp x10, x11, [sp, #16]767 stp x10, x11, [sp, #16]
768 stp x12, x13, [sp, #32]768 stp x12, x13, [sp, #32]
769 stp x14, x15, [sp, #48]769 stp x14, x15, [sp, #48]
770 770
771 b.ge 1b771 b.ge 1b
772 772
773 // here we make last iteration, without updating the registers773 // here we make last iteration, without updating the registers
774 ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]774 ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]
775 ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64775 ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64
776 776
777 uxtl v26.4s, v0.4h777 uxtl v26.4s, v0.4h
778 sxtl v27.4s, v28.4h778 sxtl v27.4s, v28.4h
779 uxtl2 v0.4s, v0.8h779 uxtl2 v0.4s, v0.8h
780 mul v5.4s, v26.4s, v27.4s780 mul v5.4s, v26.4s, v27.4s
781 sxtl2 v28.4s, v28.8h781 sxtl2 v28.4s, v28.8h
782 uxtl v26.4s, v1.4h782 uxtl v26.4s, v1.4h
783 mul v6.4s, v0.4s, v28.4s783 mul v6.4s, v0.4s, v28.4s
784 784
785 sxtl v27.4s, v29.4h785 sxtl v27.4s, v29.4h
786 uxtl2 v0.4s, v1.8h786 uxtl2 v0.4s, v1.8h
787 mla v5.4s, v26.4s, v27.4s787 mla v5.4s, v26.4s, v27.4s
788 sxtl2 v28.4s, v29.8h788 sxtl2 v28.4s, v29.8h
789 uxtl v26.4s, v2.4h789 uxtl v26.4s, v2.4h
790 mla v6.4s, v0.4s, v28.4s790 mla v6.4s, v0.4s, v28.4s
791 791
792 sxtl v27.4s, v30.4h792 sxtl v27.4s, v30.4h
793 uxtl2 v0.4s, v2.8h793 uxtl2 v0.4s, v2.8h
794 mla v5.4s, v26.4s, v27.4s794 mla v5.4s, v26.4s, v27.4s
795 sxtl2 v28.4s, v30.8h795 sxtl2 v28.4s, v30.8h
796 uxtl v26.4s, v3.4h796 uxtl v26.4s, v3.4h
797 mla v6.4s, v0.4s, v28.4s797 mla v6.4s, v0.4s, v28.4s
798 798
799 sxtl v27.4s, v31.4h799 sxtl v27.4s, v31.4h
800 uxtl2 v0.4s, v3.8h800 uxtl2 v0.4s, v3.8h
801 mla v5.4s, v26.4s, v27.4s801 mla v5.4s, v26.4s, v27.4s
802 sxtl2 v28.4s, v31.8h802 sxtl2 v28.4s, v31.8h
803 subs w2, w2, #8803 subs w2, w2, #8
804 mla v6.4s, v0.4s, v28.4s804 mla v6.4s, v0.4s, v28.4s
805 805
806 sshl v5.4s, v5.4s, v17.4s806 sshl v5.4s, v5.4s, v17.4s
807 sshl v6.4s, v6.4s, v17.4s807 sshl v6.4s, v6.4s, v17.4s
808 smin v5.4s, v5.4s, v18.4s808 smin v5.4s, v5.4s, v18.4s
809 smin v6.4s, v6.4s, v18.4s809 smin v6.4s, v6.4s, v18.4s
810 xtn v5.4h, v5.4s810 xtn v5.4h, v5.4s
811 xtn2 v5.8h, v6.4s811 xtn2 v5.8h, v6.4s
812 812
813 st1 {v5.8h}, [x1], #16813 st1 {v5.8h}, [x1], #16
814 add sp, sp, #64 // restore stack814 add sp, sp, #64 // restore stack
815 cbnz w2, 2f815 cbnz w2, 2f
816 816
817 ret817 ret
818 818
8192:8192:
820 ldr w8, [x5], #4 // load filterPos820 ldr w8, [x5], #4 // load filterPos
821 lsl w8, w8, #1821 lsl w8, w8, #1
822 add x9, x3, w8, uxtw // src + filterPos822 add x9, x3, w8, uxtw // src + filterPos
823 ld1 {v0.4h}, [x9] // load 4 * uint16_t823 ld1 {v0.4h}, [x9] // load 4 * uint16_t
824 ld1 {v31.4h}, [x4], #8824 ld1 {v31.4h}, [x4], #8
825 825
826 uxtl v0.4s, v0.4h826 uxtl v0.4s, v0.4h
827 sxtl v31.4s, v31.4h827 sxtl v31.4s, v31.4h
828 mul v5.4s, v0.4s, v31.4s828 mul v5.4s, v0.4s, v31.4s
829 addv s0, v5.4s829 addv s0, v5.4s
830 sshl v0.4s, v0.4s, v17.4s830 sshl v0.4s, v0.4s, v17.4s
831 smin v0.4s, v0.4s, v18.4s831 smin v0.4s, v0.4s, v18.4s
832 st1 {v0.h}[0], [x1], #2832 st1 {v0.h}[0], [x1], #2
833 sub w2, w2, #1833 sub w2, w2, #1
834 cbnz w2, 2b // if iterations remain jump to beginning834 cbnz w2, 2b // if iterations remain jump to beginning
835 835
836 ret836 ret
837endfunc837endfunc
@@ -845,79 +845,79 @@ function ff_hscale16to15_X8_neon_asm, export=1 · 7 unmodified lines
845 // x5 const int32_t *filterPos845 // x5 const int32_t *filterPos
846 // w6 int filterSize846 // w6 int filterSize
847 847
848 movi v20.4s, #1848 movi v20.4s, #1
849 movi v21.4s, #1849 movi v21.4s, #1
850 shl v20.4s, v20.4s, #15850 shl v20.4s, v20.4s, #15
851 sub v20.4s, v20.4s, v21.4s851 sub v20.4s, v20.4s, v21.4s
852 dup v21.4s, w0852 dup v21.4s, w0
853 neg v21.4s, v21.4s853 neg v21.4s, v21.4s
854 854
855 sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)855 sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)
8561: ldr w8, [x5], #4 // filterPos[idx]8561: ldr w8, [x5], #4 // filterPos[idx]
857 lsl w8, w8, #1857 lsl w8, w8, #1
858 ldr w10, [x5], #4 // filterPos[idx + 1]858 ldr w10, [x5], #4 // filterPos[idx + 1]
859 lsl w10, w10, #1859 lsl w10, w10, #1
860 ldr w11, [x5], #4 // filterPos[idx + 2]860 ldr w11, [x5], #4 // filterPos[idx + 2]
861 lsl w11, w11, #1861 lsl w11, w11, #1
862 ldr w9, [x5], #4 // filterPos[idx + 3]862 ldr w9, [x5], #4 // filterPos[idx + 3]
863 lsl w9, w9, #1863 lsl w9, w9, #1
864 mov x16, x4 // filter0 = filter864 mov x16, x4 // filter0 = filter
865 add x12, x16, x7 // filter1 = filter0 + filterSize*2865 add x12, x16, x7 // filter1 = filter0 + filterSize*2
866 add x13, x12, x7 // filter2 = filter1 + filterSize*2866 add x13, x12, x7 // filter2 = filter1 + filterSize*2
867 add x4, x13, x7 // filter3 = filter2 + filterSize*2867 add x4, x13, x7 // filter3 = filter2 + filterSize*2
868 movi v0.2d, #0 // val sum part 1 (for dst[0])868 movi v0.2d, #0 // val sum part 1 (for dst[0])
869 movi v1.2d, #0 // val sum part 2 (for dst[1])869 movi v1.2d, #0 // val sum part 2 (for dst[1])
870 movi v2.2d, #0 // val sum part 3 (for dst[2])870 movi v2.2d, #0 // val sum part 3 (for dst[2])
871 movi v3.2d, #0 // val sum part 4 (for dst[3])871 movi v3.2d, #0 // val sum part 4 (for dst[3])
872 add x17, x3, w8, uxtw // srcp + filterPos[0]872 add x17, x3, w8, uxtw // srcp + filterPos[0]
873 add x8, x3, w10, uxtw // srcp + filterPos[1]873 add x8, x3, w10, uxtw // srcp + filterPos[1]
874 add x10, x3, w11, uxtw // srcp + filterPos[2]874 add x10, x3, w11, uxtw // srcp + filterPos[2]
875 add x11, x3, w9, uxtw // srcp + filterPos[3]875 add x11, x3, w9, uxtw // srcp + filterPos[3]
876 mov w15, w6 // filterSize counter876 mov w15, w6 // filterSize counter
8772: ld1 {v4.8h}, [x17], #16 // srcp[filterPos[0] + {0..7}]8772: ld1 {v4.8h}, [x17], #16 // srcp[filterPos[0] + {0..7}]
878 ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1878 ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1
879 ld1 {v6.8h}, [x8], #16 // srcp[filterPos[1] + {0..7}]879 ld1 {v6.8h}, [x8], #16 // srcp[filterPos[1] + {0..7}]
880 ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize880 ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize
881 uxtl v24.4s, v4.4h // extend srcp lower half to 32 bits to preserve sign881 uxtl v24.4s, v4.4h // extend srcp lower half to 32 bits to preserve sign
882 sxtl v25.4s, v5.4h // extend filter lower half to 32 bits to match srcp size882 sxtl v25.4s, v5.4h // extend filter lower half to 32 bits to match srcp size
883 uxtl2 v4.4s, v4.8h // extend srcp upper half to 32 bits883 uxtl2 v4.4s, v4.8h // extend srcp upper half to 32 bits
884 mla v0.4s, v24.4s, v25.4s // multiply accumulate lower half of v4 * v5884 mla v0.4s, v24.4s, v25.4s // multiply accumulate lower half of v4 * v5
885 sxtl2 v5.4s, v5.8h // extend filter upper half to 32 bits885 sxtl2 v5.4s, v5.8h // extend filter upper half to 32 bits
886 uxtl v26.4s, v6.4h // extend srcp lower half to 32 bits886 uxtl v26.4s, v6.4h // extend srcp lower half to 32 bits
887 mla v0.4s, v4.4s, v5.4s // multiply accumulate upper half of v4 * v5887 mla v0.4s, v4.4s, v5.4s // multiply accumulate upper half of v4 * v5
888 sxtl v27.4s, v7.4h // exted filter lower half888 sxtl v27.4s, v7.4h // exted filter lower half
889 uxtl2 v6.4s, v6.8h // extend srcp upper half889 uxtl2 v6.4s, v6.8h // extend srcp upper half
890 sxtl2 v7.4s, v7.8h // extend filter upper half890 sxtl2 v7.4s, v7.8h // extend filter upper half
891 ld1 {v16.8h}, [x10], #16 // srcp[filterPos[2] + {0..7}]891 ld1 {v16.8h}, [x10], #16 // srcp[filterPos[2] + {0..7}]
892 mla v1.4s, v26.4s, v27.4s // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]892 mla v1.4s, v26.4s, v27.4s // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]
893 ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize893 ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize
894 uxtl v22.4s, v16.4h // extend srcp lower half894 uxtl v22.4s, v16.4h // extend srcp lower half
895 sxtl v23.4s, v17.4h // extend filter lower half895 sxtl v23.4s, v17.4h // extend filter lower half
896 uxtl2 v16.4s, v16.8h // extend srcp upper half896 uxtl2 v16.4s, v16.8h // extend srcp upper half
897 sxtl2 v17.4s, v17.8h // extend filter upper half897 sxtl2 v17.4s, v17.8h // extend filter upper half
898 mla v2.4s, v22.4s, v23.4s // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]898 mla v2.4s, v22.4s, v23.4s // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]
899 mla v2.4s, v16.4s, v17.4s // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]899 mla v2.4s, v16.4s, v17.4s // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]
900 ld1 {v18.8h}, [x11], #16 // srcp[filterPos[3] + {0..7}]900 ld1 {v18.8h}, [x11], #16 // srcp[filterPos[3] + {0..7}]
901 mla v1.4s, v6.4s, v7.4s // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]901 mla v1.4s, v6.4s, v7.4s // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]
902 ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize902 ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize
903 subs w15, w15, #8 // j -= 8: processed 8/filterSize903 subs w15, w15, #8 // j -= 8: processed 8/filterSize
904 uxtl v28.4s, v18.4h // extend srcp lower half904 uxtl v28.4s, v18.4h // extend srcp lower half
905 sxtl v29.4s, v19.4h // extend filter lower half905 sxtl v29.4s, v19.4h // extend filter lower half
906 uxtl2 v18.4s, v18.8h // extend srcp upper half906 uxtl2 v18.4s, v18.8h // extend srcp upper half
907 sxtl2 v19.4s, v19.8h // extend filter upper half907 sxtl2 v19.4s, v19.8h // extend filter upper half
908 mla v3.4s, v28.4s, v29.4s // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]908 mla v3.4s, v28.4s, v29.4s // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]
909 mla v3.4s, v18.4s, v19.4s // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]909 mla v3.4s, v18.4s, v19.4s // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]
910 b.gt 2b // inner loop if filterSize not consumed completely910 b.gt 2b // inner loop if filterSize not consumed completely
911 addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding911 addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding
912 addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding912 addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding
913 addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding913 addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding
914 subs w2, w2, #4 // dstW -= 4914 subs w2, w2, #4 // dstW -= 4
915 sshl v0.4s, v0.4s, v21.4s // shift right (effectively rigth, as shift is negative); overflow expected915 sshl v0.4s, v0.4s, v21.4s // shift right (effectively rigth, as shift is negative); overflow expected
916 smin v0.4s, v0.4s, v20.4s // apply min (do not use sqshl)916 smin v0.4s, v0.4s, v20.4s // apply min (do not use sqshl)
917 xtn v0.4h, v0.4s // narrow down to 16 bits917 xtn v0.4h, v0.4s // narrow down to 16 bits
918 918
919 st1 {v0.4h}, [x1], #8 // write to destination part0123919 st1 {v0.4h}, [x1], #8 // write to destination part0123
920 b.gt 1b // loop until end of line920 b.gt 1b // loop until end of line
921 ret921 ret
922endfunc922endfunc
923 923
@@ -930,118 +930,118 @@ function ff_hscale16to15_X4_neon_asm, export=1 · 6 unmodified lines
930 // x5 const int32_t *filterPos930 // x5 const int32_t *filterPos
931 // w6 int filterSize931 // w6 int filterSize
932 932
933 stp d8, d9, [sp, #-0x20]!933 stp d8, d9, [sp, #-0x20]!
934 stp d10, d11, [sp, #0x10]934 stp d10, d11, [sp, #0x10]
935 935
936 movi v18.4s, #1936 movi v18.4s, #1
937 movi v17.4s, #1937 movi v17.4s, #1
938 shl v18.4s, v18.4s, #15938 shl v18.4s, v18.4s, #15
939 sub v21.4s, v18.4s, v17.4s // max allowed value939 sub v21.4s, v18.4s, v17.4s // max allowed value
940 dup v17.4s, w0 // read shift940 dup v17.4s, w0 // read shift
941 neg v20.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)941 neg v20.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)
942 942
943 lsl w7, w6, #1943 lsl w7, w6, #1
9441:9441:
945 ldp w8, w9, [x5]945 ldp w8, w9, [x5]
946 ldp w10, w11, [x5, #8]946 ldp w10, w11, [x5, #8]
947 947
948 movi v16.2d, #0 // initialize accumulator for idx + 0948 movi v16.2d, #0 // initialize accumulator for idx + 0
949 movi v17.2d, #0 // initialize accumulator for idx + 1949 movi v17.2d, #0 // initialize accumulator for idx + 1
950 movi v18.2d, #0 // initialize accumulator for idx + 2950 movi v18.2d, #0 // initialize accumulator for idx + 2
951 movi v19.2d, #0 // initialize accumulator for idx + 3951 movi v19.2d, #0 // initialize accumulator for idx + 3
952 952
953 mov x12, x4 // filter + 0953 mov x12, x4 // filter + 0
954 add x13, x4, x7 // filter + 1954 add x13, x4, x7 // filter + 1
955 add x8, x3, x8, lsl #1 // srcp + filterPos 0955 add x8, x3, x8, lsl #1 // srcp + filterPos 0
956 add x14, x13, x7 // filter + 2956 add x14, x13, x7 // filter + 2
957 add x9, x3, x9, lsl #1 // srcp + filterPos 1957 add x9, x3, x9, lsl #1 // srcp + filterPos 1
958 add x15, x14, x7 // filter + 3958 add x15, x14, x7 // filter + 3
959 add x10, x3, x10, lsl #1 // srcp + filterPos 2959 add x10, x3, x10, lsl #1 // srcp + filterPos 2
960 mov w0, w6 // save the filterSize to temporary variable960 mov w0, w6 // save the filterSize to temporary variable
961 add x11, x3, x11, lsl #1 // srcp + filterPos 3961 add x11, x3, x11, lsl #1 // srcp + filterPos 3
962 add x5, x5, #16 // advance filter position962 add x5, x5, #16 // advance filter position
963 mov x16, xzr // clear the register x16 used for offsetting the filter values963 mov x16, xzr // clear the register x16 used for offsetting the filter values
964 964
9652:9652:
966 ldr q4, [x8], #16 // load src values for idx 0966 ldr q4, [x8], #16 // load src values for idx 0
967 ldr q5, [x9], #16 // load src values for idx 1967 ldr q5, [x9], #16 // load src values for idx 1
968 uxtl v26.4s, v4.4h968 uxtl v26.4s, v4.4h
969 uxtl2 v4.4s, v4.8h969 uxtl2 v4.4s, v4.8h
970 ldr q31, [x12, x16] // load filter values for idx 0970 ldr q31, [x12, x16] // load filter values for idx 0
971 ldr q6, [x10], #16 // load src values for idx 2971 ldr q6, [x10], #16 // load src values for idx 2
972 sxtl v22.4s, v31.4h972 sxtl v22.4s, v31.4h
973 sxtl2 v31.4s, v31.8h973 sxtl2 v31.4s, v31.8h
974 mla v16.4s, v26.4s, v22.4s // multiplication of lower half for idx 0974 mla v16.4s, v26.4s, v22.4s // multiplication of lower half for idx 0
975 uxtl v25.4s, v5.4h975 uxtl v25.4s, v5.4h
976 uxtl2 v5.4s, v5.8h976 uxtl2 v5.4s, v5.8h
977 ldr q30, [x13, x16] // load filter values for idx 1977 ldr q30, [x13, x16] // load filter values for idx 1
978 ldr q7, [x11], #16 // load src values for idx 3978 ldr q7, [x11], #16 // load src values for idx 3
979 mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0979 mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0
980 uxtl v24.4s, v6.4h980 uxtl v24.4s, v6.4h
981 sxtl v8.4s, v30.4h981 sxtl v8.4s, v30.4h
982 sxtl2 v30.4s, v30.8h982 sxtl2 v30.4s, v30.8h
983 mla v17.4s, v25.4s, v8.4s // multiplication of lower half for idx 1983 mla v17.4s, v25.4s, v8.4s // multiplication of lower half for idx 1
984 ldr q29, [x14, x16] // load filter values for idx 2984 ldr q29, [x14, x16] // load filter values for idx 2
985 uxtl2 v6.4s, v6.8h985 uxtl2 v6.4s, v6.8h
986 sxtl v9.4s, v29.4h986 sxtl v9.4s, v29.4h
987 sxtl2 v29.4s, v29.8h987 sxtl2 v29.4s, v29.8h
988 mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1988 mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1
989 mla v18.4s, v24.4s, v9.4s // multiplication of lower half for idx 2989 mla v18.4s, v24.4s, v9.4s // multiplication of lower half for idx 2
990 ldr q28, [x15, x16] // load filter values for idx 3990 ldr q28, [x15, x16] // load filter values for idx 3
991 uxtl v23.4s, v7.4h991 uxtl v23.4s, v7.4h
992 sxtl v10.4s, v28.4h992 sxtl v10.4s, v28.4h
993 mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2993 mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2
994 uxtl2 v7.4s, v7.8h994 uxtl2 v7.4s, v7.8h
995 sxtl2 v28.4s, v28.8h995 sxtl2 v28.4s, v28.8h
996 mla v19.4s, v23.4s, v10.4s // multiplication of lower half for idx 3996 mla v19.4s, v23.4s, v10.4s // multiplication of lower half for idx 3
997 sub w0, w0, #8997 sub w0, w0, #8
998 cmp w0, #8998 cmp w0, #8
999 mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3999 mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3
1000 1000
1001 add x16, x16, #16 // advance filter values indexing1001 add x16, x16, #16 // advance filter values indexing
1002 1002
1003 b.ge 2b1003 b.ge 2b
1004 1004
1005 // 4 iterations left1005 // 4 iterations left
1006 1006
1007 sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements1007 sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements
1008 1008
1009 ldr d4, [x8] // load src values for idx 01009 ldr d4, [x8] // load src values for idx 0
1010 ldr d31, [x12, x17] // load filter values for idx 01010 ldr d31, [x12, x17] // load filter values for idx 0
1011 uxtl v4.4s, v4.4h1011 uxtl v4.4s, v4.4h
1012 sxtl v31.4s, v31.4h1012 sxtl v31.4s, v31.4h
1013 ldr d5, [x9] // load src values for idx 11013 ldr d5, [x9] // load src values for idx 1
1014 mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 01014 mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0
1015 ldr d30, [x13, x17] // load filter values for idx 11015 ldr d30, [x13, x17] // load filter values for idx 1
1016 uxtl v5.4s, v5.4h1016 uxtl v5.4s, v5.4h
1017 sxtl v30.4s, v30.4h1017 sxtl v30.4s, v30.4h
1018 ldr d6, [x10] // load src values for idx 21018 ldr d6, [x10] // load src values for idx 2
1019 mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 11019 mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1
1020 ldr d29, [x14, x17] // load filter values for idx 21020 ldr d29, [x14, x17] // load filter values for idx 2
1021 uxtl v6.4s, v6.4h1021 uxtl v6.4s, v6.4h
1022 sxtl v29.4s, v29.4h1022 sxtl v29.4s, v29.4h
1023 ldr d7, [x11] // load src values for idx 31023 ldr d7, [x11] // load src values for idx 3
1024 ldr d28, [x15, x17] // load filter values for idx 31024 ldr d28, [x15, x17] // load filter values for idx 3
1025 mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 21025 mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2
1026 uxtl v7.4s, v7.4h1026 uxtl v7.4s, v7.4h
1027 sxtl v28.4s, v28.4h1027 sxtl v28.4s, v28.4h
1028 addp v16.4s, v16.4s, v17.4s1028 addp v16.4s, v16.4s, v17.4s
1029 mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 31029 mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3
1030 subs w2, w2, #41030 subs w2, w2, #4
1031 addp v18.4s, v18.4s, v19.4s1031 addp v18.4s, v18.4s, v19.4s
1032 addp v16.4s, v16.4s, v18.4s1032 addp v16.4s, v16.4s, v18.4s
1033 sshl v16.4s, v16.4s, v20.4s1033 sshl v16.4s, v16.4s, v20.4s
1034 smin v16.4s, v16.4s, v21.4s1034 smin v16.4s, v16.4s, v21.4s
1035 xtn v16.4h, v16.4s1035 xtn v16.4h, v16.4s
1036 1036
1037 st1 {v16.4h}, [x1], #81037 st1 {v16.4h}, [x1], #8
1038 add x4, x4, x7, lsl #21038 add x4, x4, x7, lsl #2
1039 b.gt 1b1039 b.gt 1b
1040 1040
1041 ldp d8, d9, [sp]1041 ldp d8, d9, [sp]
1042 ldp d10, d11, [sp, #0x10]1042 ldp d10, d11, [sp, #0x10]
1043 1043
1044 add sp, sp, #0x201044 add sp, sp, #0x20
1045 1045
1046 ret1046 ret
1047endfunc1047endfunc
@@ -1055,188 +1055,188 @@ function ff_hscale16to19_4_neon_asm, export=1 · 7 unmodified lines
1055 // x5 const int32_t *filterPos1055 // x5 const int32_t *filterPos
1056 // w6 int filterSize1056 // w6 int filterSize
1057 1057
1058 movi v18.4s, #11058 movi v18.4s, #1
1059 movi v17.4s, #11059 movi v17.4s, #1
1060 shl v18.4s, v18.4s, #191060 shl v18.4s, v18.4s, #19
1061 sub v18.4s, v18.4s, v17.4s // max allowed value1061 sub v18.4s, v18.4s, v17.4s // max allowed value
1062 dup v17.4s, w0 // read shift1062 dup v17.4s, w0 // read shift
1063 neg v17.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)1063 neg v17.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)
1064 1064
1065 cmp w2, #161065 cmp w2, #16
1066 b.lt 2f // move to last block1066 b.lt 2f // move to last block
1067 1067
1068 ldp w8, w9, [x5] // filterPos[0], filterPos[1]1068 ldp w8, w9, [x5] // filterPos[0], filterPos[1]
1069 ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]1069 ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
1070 ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]1070 ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
1071 ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]1071 ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
1072 add x5, x5, #321072 add x5, x5, #32
1073 1073
1074 // shift all filterPos left by one, as uint16_t will be read1074 // shift all filterPos left by one, as uint16_t will be read
1075 lsl x8, x8, #11075 lsl x8, x8, #1
1076 lsl x9, x9, #11076 lsl x9, x9, #1
1077 lsl x10, x10, #11077 lsl x10, x10, #1
1078 lsl x11, x11, #11078 lsl x11, x11, #1
1079 lsl x12, x12, #11079 lsl x12, x12, #1
1080 lsl x13, x13, #11080 lsl x13, x13, #1
1081 lsl x14, x14, #11081 lsl x14, x14, #1
1082 lsl x15, x15, #11082 lsl x15, x15, #1
1083 1083
1084 // load src with given offset1084 // load src with given offset
1085 ldr x8, [x3, w8, uxtw]1085 ldr x8, [x3, w8, uxtw]
1086 ldr x9, [x3, w9, uxtw]1086 ldr x9, [x3, w9, uxtw]
1087 ldr x10, [x3, w10, uxtw]1087 ldr x10, [x3, w10, uxtw]
1088 ldr x11, [x3, w11, uxtw]1088 ldr x11, [x3, w11, uxtw]
1089 ldr x12, [x3, w12, uxtw]1089 ldr x12, [x3, w12, uxtw]
1090 ldr x13, [x3, w13, uxtw]1090 ldr x13, [x3, w13, uxtw]
1091 ldr x14, [x3, w14, uxtw]1091 ldr x14, [x3, w14, uxtw]
1092 ldr x15, [x3, w15, uxtw]1092 ldr x15, [x3, w15, uxtw]
1093 1093
1094 sub sp, sp, #641094 sub sp, sp, #64
1095 // push src on stack so it can be loaded into vectors later1095 // push src on stack so it can be loaded into vectors later
1096 stp x8, x9, [sp]1096 stp x8, x9, [sp]
1097 stp x10, x11, [sp, #16]1097 stp x10, x11, [sp, #16]
1098 stp x12, x13, [sp, #32]1098 stp x12, x13, [sp, #32]
1099 stp x14, x15, [sp, #48]1099 stp x14, x15, [sp, #48]
1100 1100
11011:11011:
1102 ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]1102 ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]
1103 ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]1103 ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64 // filter[0..7]
1104 1104
1105 // Each of blocks does the following:1105 // Each of blocks does the following:
1106 // Extend src and filter to 32 bits with uxtl and sxtl1106 // Extend src and filter to 32 bits with uxtl and sxtl
1107 // multiply or multiply and accumulate results1107 // multiply or multiply and accumulate results
1108 // Extending to 32 bits is necessary, as unit16_t values can't1108 // Extending to 32 bits is necessary, as unit16_t values can't
1109 // be represented as int16_t without type promotion.1109 // be represented as int16_t without type promotion.
1110 uxtl v26.4s, v0.4h1110 uxtl v26.4s, v0.4h
1111 sxtl v27.4s, v28.4h1111 sxtl v27.4s, v28.4h
1112 uxtl2 v0.4s, v0.8h1112 uxtl2 v0.4s, v0.8h
1113 mul v5.4s, v26.4s, v27.4s1113 mul v5.4s, v26.4s, v27.4s
1114 sxtl2 v28.4s, v28.8h1114 sxtl2 v28.4s, v28.8h
1115 uxtl v26.4s, v1.4h1115 uxtl v26.4s, v1.4h
1116 mul v6.4s, v0.4s, v28.4s1116 mul v6.4s, v0.4s, v28.4s
1117 1117
1118 sxtl v27.4s, v29.4h1118 sxtl v27.4s, v29.4h
1119 uxtl2 v0.4s, v1.8h1119 uxtl2 v0.4s, v1.8h
1120 mla v5.4s, v27.4s, v26.4s1120 mla v5.4s, v27.4s, v26.4s
1121 sxtl2 v28.4s, v29.8h1121 sxtl2 v28.4s, v29.8h
1122 uxtl v26.4s, v2.4h1122 uxtl v26.4s, v2.4h
1123 mla v6.4s, v28.4s, v0.4s1123 mla v6.4s, v28.4s, v0.4s
1124 1124
1125 sxtl v27.4s, v30.4h1125 sxtl v27.4s, v30.4h
1126 uxtl2 v0.4s, v2.8h1126 uxtl2 v0.4s, v2.8h
1127 mla v5.4s, v27.4s, v26.4s1127 mla v5.4s, v27.4s, v26.4s
1128 sxtl2 v28.4s, v30.8h1128 sxtl2 v28.4s, v30.8h
1129 uxtl v26.4s, v3.4h1129 uxtl v26.4s, v3.4h
1130 mla v6.4s, v28.4s, v0.4s1130 mla v6.4s, v28.4s, v0.4s
1131 1131
1132 sxtl v27.4s, v31.4h1132 sxtl v27.4s, v31.4h
1133 uxtl2 v0.4s, v3.8h1133 uxtl2 v0.4s, v3.8h
1134 mla v5.4s, v27.4s, v26.4s1134 mla v5.4s, v27.4s, v26.4s
1135 sxtl2 v28.4s, v31.8h1135 sxtl2 v28.4s, v31.8h
1136 sub w2, w2, #81136 sub w2, w2, #8
1137 mla v6.4s, v28.4s, v0.4s1137 mla v6.4s, v28.4s, v0.4s
1138 1138
1139 sshl v5.4s, v5.4s, v17.4s1139 sshl v5.4s, v5.4s, v17.4s
1140 sshl v6.4s, v6.4s, v17.4s1140 sshl v6.4s, v6.4s, v17.4s
1141 smin v5.4s, v5.4s, v18.4s1141 smin v5.4s, v5.4s, v18.4s
1142 smin v6.4s, v6.4s, v18.4s1142 smin v6.4s, v6.4s, v18.4s
1143 1143
1144 st1 {v5.4s, v6.4s}, [x1], #321144 st1 {v5.4s, v6.4s}, [x1], #32
1145 cmp w2, #161145 cmp w2, #16
1146 1146
1147 // load filterPositions into registers for next iteration1147 // load filterPositions into registers for next iteration
1148 ldp w8, w9, [x5] // filterPos[0], filterPos[1]1148 ldp w8, w9, [x5] // filterPos[0], filterPos[1]
1149 ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]1149 ldp w10, w11, [x5, #8] // filterPos[2], filterPos[3]
1150 ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]1150 ldp w12, w13, [x5, #16] // filterPos[4], filterPos[5]
1151 ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]1151 ldp w14, w15, [x5, #24] // filterPos[6], filterPos[7]
1152 add x5, x5, #321152 add x5, x5, #32
1153 1153
1154 lsl x8, x8, #11154 lsl x8, x8, #1
1155 lsl x9, x9, #11155 lsl x9, x9, #1
1156 lsl x10, x10, #11156 lsl x10, x10, #1
1157 lsl x11, x11, #11157 lsl x11, x11, #1
1158 lsl x12, x12, #11158 lsl x12, x12, #1
1159 lsl x13, x13, #11159 lsl x13, x13, #1
1160 lsl x14, x14, #11160 lsl x14, x14, #1
1161 lsl x15, x15, #11161 lsl x15, x15, #1
1162 1162
1163 ldr x8, [x3, w8, uxtw]1163 ldr x8, [x3, w8, uxtw]
1164 ldr x9, [x3, w9, uxtw]1164 ldr x9, [x3, w9, uxtw]
1165 ldr x10, [x3, w10, uxtw]1165 ldr x10, [x3, w10, uxtw]
1166 ldr x11, [x3, w11, uxtw]1166 ldr x11, [x3, w11, uxtw]
1167 ldr x12, [x3, w12, uxtw]1167 ldr x12, [x3, w12, uxtw]
1168 ldr x13, [x3, w13, uxtw]1168 ldr x13, [x3, w13, uxtw]
1169 ldr x14, [x3, w14, uxtw]1169 ldr x14, [x3, w14, uxtw]
1170 ldr x15, [x3, w15, uxtw]1170 ldr x15, [x3, w15, uxtw]
1171 1171
1172 stp x8, x9, [sp]1172 stp x8, x9, [sp]
1173 stp x10, x11, [sp, #16]1173 stp x10, x11, [sp, #16]
1174 stp x12, x13, [sp, #32]1174 stp x12, x13, [sp, #32]
1175 stp x14, x15, [sp, #48]1175 stp x14, x15, [sp, #48]
1176 1176
1177 b.ge 1b1177 b.ge 1b
1178 1178
1179 // here we make last iteration, without updating the registers1179 // here we make last iteration, without updating the registers
1180 ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]1180 ld4 {v0.8h, v1.8h, v2.8h, v3.8h}, [sp]
1181 ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #641181 ld4 {v28.8h, v29.8h, v30.8h, v31.8h}, [x4], #64
1182 1182
1183 uxtl v26.4s, v0.4h1183 uxtl v26.4s, v0.4h
1184 sxtl v27.4s, v28.4h1184 sxtl v27.4s, v28.4h
1185 uxtl2 v0.4s, v0.8h1185 uxtl2 v0.4s, v0.8h
1186 mul v5.4s, v26.4s, v27.4s1186 mul v5.4s, v26.4s, v27.4s
1187 sxtl2 v28.4s, v28.8h1187 sxtl2 v28.4s, v28.8h
1188 uxtl v26.4s, v1.4h1188 uxtl v26.4s, v1.4h
1189 mul v6.4s, v0.4s, v28.4s1189 mul v6.4s, v0.4s, v28.4s
1190 1190
1191 sxtl v27.4s, v29.4h1191 sxtl v27.4s, v29.4h
1192 uxtl2 v0.4s, v1.8h1192 uxtl2 v0.4s, v1.8h
1193 mla v5.4s, v26.4s, v27.4s1193 mla v5.4s, v26.4s, v27.4s
1194 sxtl2 v28.4s, v29.8h1194 sxtl2 v28.4s, v29.8h
1195 uxtl v26.4s, v2.4h1195 uxtl v26.4s, v2.4h
1196 mla v6.4s, v0.4s, v28.4s1196 mla v6.4s, v0.4s, v28.4s
1197 1197
1198 sxtl v27.4s, v30.4h1198 sxtl v27.4s, v30.4h
1199 uxtl2 v0.4s, v2.8h1199 uxtl2 v0.4s, v2.8h
1200 mla v5.4s, v26.4s, v27.4s1200 mla v5.4s, v26.4s, v27.4s
1201 sxtl2 v28.4s, v30.8h1201 sxtl2 v28.4s, v30.8h
1202 uxtl v26.4s, v3.4h1202 uxtl v26.4s, v3.4h
1203 mla v6.4s, v0.4s, v28.4s1203 mla v6.4s, v0.4s, v28.4s
1204 1204
1205 sxtl v27.4s, v31.4h1205 sxtl v27.4s, v31.4h
1206 uxtl2 v0.4s, v3.8h1206 uxtl2 v0.4s, v3.8h
1207 mla v5.4s, v26.4s, v27.4s1207 mla v5.4s, v26.4s, v27.4s
1208 sxtl2 v28.4s, v31.8h1208 sxtl2 v28.4s, v31.8h
1209 subs w2, w2, #81209 subs w2, w2, #8
1210 mla v6.4s, v0.4s, v28.4s1210 mla v6.4s, v0.4s, v28.4s
1211 1211
1212 sshl v5.4s, v5.4s, v17.4s1212 sshl v5.4s, v5.4s, v17.4s
1213 sshl v6.4s, v6.4s, v17.4s1213 sshl v6.4s, v6.4s, v17.4s
1214 1214
1215 smin v5.4s, v5.4s, v18.4s1215 smin v5.4s, v5.4s, v18.4s
1216 smin v6.4s, v6.4s, v18.4s1216 smin v6.4s, v6.4s, v18.4s
1217 1217
1218 st1 {v5.4s, v6.4s}, [x1], #321218 st1 {v5.4s, v6.4s}, [x1], #32
1219 add sp, sp, #64 // restore stack1219 add sp, sp, #64 // restore stack
1220 cbnz w2, 2f1220 cbnz w2, 2f
1221 1221
1222 ret1222 ret
1223 1223
12242:12242:
1225 ldr w8, [x5], #4 // load filterPos1225 ldr w8, [x5], #4 // load filterPos
1226 lsl w8, w8, #11226 lsl w8, w8, #1
1227 add x9, x3, w8, uxtw // src + filterPos1227 add x9, x3, w8, uxtw // src + filterPos
1228 ld1 {v0.4h}, [x9] // load 4 * uint16_t1228 ld1 {v0.4h}, [x9] // load 4 * uint16_t
1229 ld1 {v31.4h}, [x4], #81229 ld1 {v31.4h}, [x4], #8
1230 1230
1231 uxtl v0.4s, v0.4h1231 uxtl v0.4s, v0.4h
1232 sxtl v31.4s, v31.4h1232 sxtl v31.4s, v31.4h
1233 subs w2, w2, #11233 subs w2, w2, #1
1234 mul v5.4s, v0.4s, v31.4s1234 mul v5.4s, v0.4s, v31.4s
1235 addv s0, v5.4s1235 addv s0, v5.4s
1236 sshl v0.4s, v0.4s, v17.4s1236 sshl v0.4s, v0.4s, v17.4s
1237 smin v0.4s, v0.4s, v18.4s1237 smin v0.4s, v0.4s, v18.4s
1238 st1 {v0.s}[0], [x1], #41238 st1 {v0.s}[0], [x1], #4
1239 cbnz w2, 2b // if iterations remain jump to beginning1239 cbnz w2, 2b // if iterations remain jump to beginning
1240 1240
1241 ret1241 ret
1242endfunc1242endfunc
@@ -1250,77 +1250,77 @@ function ff_hscale16to19_X8_neon_asm, export=1 · 7 unmodified lines
1250 // x5 const int32_t *filterPos1250 // x5 const int32_t *filterPos
1251 // w6 int filterSize1251 // w6 int filterSize
1252 1252
1253 movi v20.4s, #11253 movi v20.4s, #1
1254 movi v21.4s, #11254 movi v21.4s, #1
1255 shl v20.4s, v20.4s, #191255 shl v20.4s, v20.4s, #19
1256 sub v20.4s, v20.4s, v21.4s1256 sub v20.4s, v20.4s, v21.4s
1257 dup v21.4s, w01257 dup v21.4s, w0
1258 neg v21.4s, v21.4s1258 neg v21.4s, v21.4s
1259 1259
1260 sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)1260 sbfiz x7, x6, #1, #32 // filterSize*2 (*2 because int16)
12611: ldr w8, [x5], #4 // filterPos[idx]12611: ldr w8, [x5], #4 // filterPos[idx]
1262 ldr w10, [x5], #4 // filterPos[idx + 1]1262 ldr w10, [x5], #4 // filterPos[idx + 1]
1263 lsl w8, w8, #11263 lsl w8, w8, #1
1264 ldr w11, [x5], #4 // filterPos[idx + 2]1264 ldr w11, [x5], #4 // filterPos[idx + 2]
1265 ldr w9, [x5], #4 // filterPos[idx + 3]1265 ldr w9, [x5], #4 // filterPos[idx + 3]
1266 mov x16, x4 // filter0 = filter1266 mov x16, x4 // filter0 = filter
1267 lsl w11, w11, #11267 lsl w11, w11, #1
1268 add x12, x16, x7 // filter1 = filter0 + filterSize*21268 add x12, x16, x7 // filter1 = filter0 + filterSize*2
1269 lsl w9, w9, #11269 lsl w9, w9, #1
1270 add x13, x12, x7 // filter2 = filter1 + filterSize*21270 add x13, x12, x7 // filter2 = filter1 + filterSize*2
1271 lsl w10, w10, #11271 lsl w10, w10, #1
1272 add x4, x13, x7 // filter3 = filter2 + filterSize*21272 add x4, x13, x7 // filter3 = filter2 + filterSize*2
1273 movi v0.2d, #0 // val sum part 1 (for dst[0])1273 movi v0.2d, #0 // val sum part 1 (for dst[0])
1274 movi v1.2d, #0 // val sum part 2 (for dst[1])1274 movi v1.2d, #0 // val sum part 2 (for dst[1])
1275 movi v2.2d, #0 // val sum part 3 (for dst[2])1275 movi v2.2d, #0 // val sum part 3 (for dst[2])
1276 movi v3.2d, #0 // val sum part 4 (for dst[3])1276 movi v3.2d, #0 // val sum part 4 (for dst[3])
1277 add x17, x3, w8, uxtw // srcp + filterPos[0]1277 add x17, x3, w8, uxtw // srcp + filterPos[0]
1278 add x8, x3, w10, uxtw // srcp + filterPos[1]1278 add x8, x3, w10, uxtw // srcp + filterPos[1]
1279 add x10, x3, w11, uxtw // srcp + filterPos[2]1279 add x10, x3, w11, uxtw // srcp + filterPos[2]
1280 add x11, x3, w9, uxtw // srcp + filterPos[3]1280 add x11, x3, w9, uxtw // srcp + filterPos[3]
1281 mov w15, w6 // filterSize counter1281 mov w15, w6 // filterSize counter
12822: ld1 {v4.8h}, [x17], #16 // srcp[filterPos[0] + {0..7}]12822: ld1 {v4.8h}, [x17], #16 // srcp[filterPos[0] + {0..7}]
1283 ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 11283 ld1 {v5.8h}, [x16], #16 // load 8x16-bit filter values, part 1
1284 ld1 {v6.8h}, [x8], #16 // srcp[filterPos[1] + {0..7}]1284 ld1 {v6.8h}, [x8], #16 // srcp[filterPos[1] + {0..7}]
1285 ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize1285 ld1 {v7.8h}, [x12], #16 // load 8x16-bit at filter+filterSize
1286 uxtl v24.4s, v4.4h // extend srcp lower half to 32 bits to preserve sign1286 uxtl v24.4s, v4.4h // extend srcp lower half to 32 bits to preserve sign
1287 sxtl v25.4s, v5.4h // extend filter lower half to 32 bits to match srcp size1287 sxtl v25.4s, v5.4h // extend filter lower half to 32 bits to match srcp size
1288 uxtl2 v4.4s, v4.8h // extend srcp upper half to 32 bits1288 uxtl2 v4.4s, v4.8h // extend srcp upper half to 32 bits
1289 mla v0.4s, v24.4s, v25.4s // multiply accumulate lower half of v4 * v51289 mla v0.4s, v24.4s, v25.4s // multiply accumulate lower half of v4 * v5
1290 sxtl2 v5.4s, v5.8h // extend filter upper half to 32 bits1290 sxtl2 v5.4s, v5.8h // extend filter upper half to 32 bits
1291 uxtl v26.4s, v6.4h // extend srcp lower half to 32 bits1291 uxtl v26.4s, v6.4h // extend srcp lower half to 32 bits
1292 mla v0.4s, v4.4s, v5.4s // multiply accumulate upper half of v4 * v51292 mla v0.4s, v4.4s, v5.4s // multiply accumulate upper half of v4 * v5
1293 sxtl v27.4s, v7.4h // exted filter lower half1293 sxtl v27.4s, v7.4h // exted filter lower half
1294 uxtl2 v6.4s, v6.8h // extend srcp upper half1294 uxtl2 v6.4s, v6.8h // extend srcp upper half
1295 sxtl2 v7.4s, v7.8h // extend filter upper half1295 sxtl2 v7.4s, v7.8h // extend filter upper half
1296 ld1 {v16.8h}, [x10], #16 // srcp[filterPos[2] + {0..7}]1296 ld1 {v16.8h}, [x10], #16 // srcp[filterPos[2] + {0..7}]
1297 mla v1.4s, v26.4s, v27.4s // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]1297 mla v1.4s, v26.4s, v27.4s // v1 accumulates srcp[filterPos[1] + {0..3}] * filter[{0..3}]
1298 ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize1298 ld1 {v17.8h}, [x13], #16 // load 8x16-bit at filter+2*filterSize
1299 uxtl v22.4s, v16.4h // extend srcp lower half1299 uxtl v22.4s, v16.4h // extend srcp lower half
1300 sxtl v23.4s, v17.4h // extend filter lower half1300 sxtl v23.4s, v17.4h // extend filter lower half
1301 uxtl2 v16.4s, v16.8h // extend srcp upper half1301 uxtl2 v16.4s, v16.8h // extend srcp upper half
1302 sxtl2 v17.4s, v17.8h // extend filter upper half1302 sxtl2 v17.4s, v17.8h // extend filter upper half
1303 mla v2.4s, v22.4s, v23.4s // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]1303 mla v2.4s, v22.4s, v23.4s // v2 accumulates srcp[filterPos[2] + {0..3}] * filter[{0..3}]
1304 mla v2.4s, v16.4s, v17.4s // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]1304 mla v2.4s, v16.4s, v17.4s // v2 accumulates srcp[filterPos[2] + {4..7}] * filter[{4..7}]
1305 ld1 {v18.8h}, [x11], #16 // srcp[filterPos[3] + {0..7}]1305 ld1 {v18.8h}, [x11], #16 // srcp[filterPos[3] + {0..7}]
1306 mla v1.4s, v6.4s, v7.4s // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]1306 mla v1.4s, v6.4s, v7.4s // v1 accumulates srcp[filterPos[1] + {4..7}] * filter[{4..7}]
1307 ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize1307 ld1 {v19.8h}, [x4], #16 // load 8x16-bit at filter+3*filterSize
1308 subs w15, w15, #8 // j -= 8: processed 8/filterSize1308 subs w15, w15, #8 // j -= 8: processed 8/filterSize
1309 uxtl v28.4s, v18.4h // extend srcp lower half1309 uxtl v28.4s, v18.4h // extend srcp lower half
1310 sxtl v29.4s, v19.4h // extend filter lower half1310 sxtl v29.4s, v19.4h // extend filter lower half
1311 uxtl2 v18.4s, v18.8h // extend srcp upper half1311 uxtl2 v18.4s, v18.8h // extend srcp upper half
1312 sxtl2 v19.4s, v19.8h // extend filter upper half1312 sxtl2 v19.4s, v19.8h // extend filter upper half
1313 mla v3.4s, v28.4s, v29.4s // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]1313 mla v3.4s, v28.4s, v29.4s // v3 accumulates srcp[filterPos[3] + {0..3}] * filter[{0..3}]
1314 mla v3.4s, v18.4s, v19.4s // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]1314 mla v3.4s, v18.4s, v19.4s // v3 accumulates srcp[filterPos[3] + {4..7}] * filter[{4..7}]
1315 b.gt 2b // inner loop if filterSize not consumed completely1315 b.gt 2b // inner loop if filterSize not consumed completely
1316 addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding1316 addp v0.4s, v0.4s, v1.4s // part01 horizontal pair adding
1317 addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding1317 addp v2.4s, v2.4s, v3.4s // part23 horizontal pair adding
1318 addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding1318 addp v0.4s, v0.4s, v2.4s // part0123 horizontal pair adding
1319 subs w2, w2, #4 // dstW -= 41319 subs w2, w2, #4 // dstW -= 4
1320 sshl v0.4s, v0.4s, v21.4s // shift right (effectively rigth, as shift is negative); overflow expected1320 sshl v0.4s, v0.4s, v21.4s // shift right (effectively rigth, as shift is negative); overflow expected
1321 smin v0.4s, v0.4s, v20.4s // apply min (do not use sqshl)1321 smin v0.4s, v0.4s, v20.4s // apply min (do not use sqshl)
1322 st1 {v0.4s}, [x1], #16 // write to destination part01231322 st1 {v0.4s}, [x1], #16 // write to destination part0123
1323 b.gt 1b // loop until end of line1323 b.gt 1b // loop until end of line
1324 ret1324 ret
1325endfunc1325endfunc
1326 1326
@@ -1333,117 +1333,117 @@ function ff_hscale16to19_X4_neon_asm, export=1 · 6 unmodified lines
1333 // x5 const int32_t *filterPos1333 // x5 const int32_t *filterPos
1334 // w6 int filterSize1334 // w6 int filterSize
1335 1335
1336 stp d8, d9, [sp, #-0x20]!1336 stp d8, d9, [sp, #-0x20]!
1337 stp d10, d11, [sp, #0x10]1337 stp d10, d11, [sp, #0x10]
1338 1338
1339 movi v18.4s, #11339 movi v18.4s, #1
1340 movi v17.4s, #11340 movi v17.4s, #1
1341 shl v18.4s, v18.4s, #191341 shl v18.4s, v18.4s, #19
1342 sub v21.4s, v18.4s, v17.4s // max allowed value1342 sub v21.4s, v18.4s, v17.4s // max allowed value
1343 dup v17.4s, w0 // read shift1343 dup v17.4s, w0 // read shift
1344 neg v20.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)1344 neg v20.4s, v17.4s // negate it, so it can be used in sshl (effectively shift right)
1345 1345
1346 lsl w7, w6, #11346 lsl w7, w6, #1
13471:13471:
1348 ldp w8, w9, [x5]1348 ldp w8, w9, [x5]
1349 ldp w10, w11, [x5, #8]1349 ldp w10, w11, [x5, #8]
1350 1350
1351 movi v16.2d, #0 // initialize accumulator for idx + 01351 movi v16.2d, #0 // initialize accumulator for idx + 0
1352 movi v17.2d, #0 // initialize accumulator for idx + 11352 movi v17.2d, #0 // initialize accumulator for idx + 1
1353 movi v18.2d, #0 // initialize accumulator for idx + 21353 movi v18.2d, #0 // initialize accumulator for idx + 2
1354 movi v19.2d, #0 // initialize accumulator for idx + 31354 movi v19.2d, #0 // initialize accumulator for idx + 3
1355 1355
1356 mov x12, x4 // filter + 01356 mov x12, x4 // filter + 0
1357 add x13, x4, x7 // filter + 11357 add x13, x4, x7 // filter + 1
1358 add x8, x3, x8, lsl #1 // srcp + filterPos 01358 add x8, x3, x8, lsl #1 // srcp + filterPos 0
1359 add x14, x13, x7 // filter + 21359 add x14, x13, x7 // filter + 2
1360 add x9, x3, x9, lsl #1 // srcp + filterPos 11360 add x9, x3, x9, lsl #1 // srcp + filterPos 1
1361 add x15, x14, x7 // filter + 31361 add x15, x14, x7 // filter + 3
1362 add x10, x3, x10, lsl #1 // srcp + filterPos 21362 add x10, x3, x10, lsl #1 // srcp + filterPos 2
1363 mov w0, w6 // save the filterSize to temporary variable1363 mov w0, w6 // save the filterSize to temporary variable
1364 add x11, x3, x11, lsl #1 // srcp + filterPos 31364 add x11, x3, x11, lsl #1 // srcp + filterPos 3
1365 add x5, x5, #16 // advance filter position1365 add x5, x5, #16 // advance filter position
1366 mov x16, xzr // clear the register x16 used for offsetting the filter values1366 mov x16, xzr // clear the register x16 used for offsetting the filter values
1367 1367
13682:13682:
1369 ldr q4, [x8], #16 // load src values for idx 01369 ldr q4, [x8], #16 // load src values for idx 0
1370 ldr q5, [x9], #16 // load src values for idx 11370 ldr q5, [x9], #16 // load src values for idx 1
1371 uxtl v26.4s, v4.4h1371 uxtl v26.4s, v4.4h
1372 uxtl2 v4.4s, v4.8h1372 uxtl2 v4.4s, v4.8h
1373 ldr q31, [x12, x16] // load filter values for idx 01373 ldr q31, [x12, x16] // load filter values for idx 0
1374 ldr q6, [x10], #16 // load src values for idx 21374 ldr q6, [x10], #16 // load src values for idx 2
1375 sxtl v22.4s, v31.4h1375 sxtl v22.4s, v31.4h
1376 sxtl2 v31.4s, v31.8h1376 sxtl2 v31.4s, v31.8h
1377 mla v16.4s, v26.4s, v22.4s // multiplication of lower half for idx 01377 mla v16.4s, v26.4s, v22.4s // multiplication of lower half for idx 0
1378 uxtl v25.4s, v5.4h1378 uxtl v25.4s, v5.4h
1379 uxtl2 v5.4s, v5.8h1379 uxtl2 v5.4s, v5.8h
1380 ldr q30, [x13, x16] // load filter values for idx 11380 ldr q30, [x13, x16] // load filter values for idx 1
1381 ldr q7, [x11], #16 // load src values for idx 31381 ldr q7, [x11], #16 // load src values for idx 3
1382 mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 01382 mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0
1383 uxtl v24.4s, v6.4h1383 uxtl v24.4s, v6.4h
1384 sxtl v8.4s, v30.4h1384 sxtl v8.4s, v30.4h
1385 sxtl2 v30.4s, v30.8h1385 sxtl2 v30.4s, v30.8h
1386 mla v17.4s, v25.4s, v8.4s // multiplication of lower half for idx 11386 mla v17.4s, v25.4s, v8.4s // multiplication of lower half for idx 1
1387 ldr q29, [x14, x16] // load filter values for idx 21387 ldr q29, [x14, x16] // load filter values for idx 2
1388 uxtl2 v6.4s, v6.8h1388 uxtl2 v6.4s, v6.8h
1389 sxtl v9.4s, v29.4h1389 sxtl v9.4s, v29.4h
1390 sxtl2 v29.4s, v29.8h1390 sxtl2 v29.4s, v29.8h
1391 mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 11391 mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1
1392 ldr q28, [x15, x16] // load filter values for idx 31392 ldr q28, [x15, x16] // load filter values for idx 3
1393 mla v18.4s, v24.4s, v9.4s // multiplication of lower half for idx 21393 mla v18.4s, v24.4s, v9.4s // multiplication of lower half for idx 2
1394 uxtl v23.4s, v7.4h1394 uxtl v23.4s, v7.4h
1395 sxtl v10.4s, v28.4h1395 sxtl v10.4s, v28.4h
1396 mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 21396 mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2
1397 uxtl2 v7.4s, v7.8h1397 uxtl2 v7.4s, v7.8h
1398 sxtl2 v28.4s, v28.8h1398 sxtl2 v28.4s, v28.8h
1399 mla v19.4s, v23.4s, v10.4s // multiplication of lower half for idx 31399 mla v19.4s, v23.4s, v10.4s // multiplication of lower half for idx 3
1400 sub w0, w0, #81400 sub w0, w0, #8
1401 cmp w0, #81401 cmp w0, #8
1402 mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 31402 mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3
1403 1403
1404 add x16, x16, #16 // advance filter values indexing1404 add x16, x16, #16 // advance filter values indexing
1405 1405
1406 b.ge 2b1406 b.ge 2b
1407 1407
1408 // 4 iterations left1408 // 4 iterations left
1409 1409
1410 sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements1410 sub x17, x7, #8 // step back to wrap up the filter pos for last 4 elements
1411 1411
1412 ldr d4, [x8] // load src values for idx 01412 ldr d4, [x8] // load src values for idx 0
1413 ldr d31, [x12, x17] // load filter values for idx 01413 ldr d31, [x12, x17] // load filter values for idx 0
1414 uxtl v4.4s, v4.4h1414 uxtl v4.4s, v4.4h
1415 sxtl v31.4s, v31.4h1415 sxtl v31.4s, v31.4h
1416 ldr d5, [x9] // load src values for idx 11416 ldr d5, [x9] // load src values for idx 1
1417 mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 01417 mla v16.4s, v4.4s, v31.4s // multiplication of upper half for idx 0
1418 ldr d30, [x13, x17] // load filter values for idx 11418 ldr d30, [x13, x17] // load filter values for idx 1
1419 uxtl v5.4s, v5.4h1419 uxtl v5.4s, v5.4h
1420 sxtl v30.4s, v30.4h1420 sxtl v30.4s, v30.4h
1421 ldr d6, [x10] // load src values for idx 21421 ldr d6, [x10] // load src values for idx 2
1422 mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 11422 mla v17.4s, v5.4s, v30.4s // multiplication of upper half for idx 1
1423 ldr d29, [x14, x17] // load filter values for idx 21423 ldr d29, [x14, x17] // load filter values for idx 2
1424 uxtl v6.4s, v6.4h1424 uxtl v6.4s, v6.4h
1425 sxtl v29.4s, v29.4h1425 sxtl v29.4s, v29.4h
1426 ldr d7, [x11] // load src values for idx 31426 ldr d7, [x11] // load src values for idx 3
1427 ldr d28, [x15, x17] // load filter values for idx 31427 ldr d28, [x15, x17] // load filter values for idx 3
1428 mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 21428 mla v18.4s, v6.4s, v29.4s // multiplication of upper half for idx 2
1429 uxtl v7.4s, v7.4h1429 uxtl v7.4s, v7.4h
1430 sxtl v28.4s, v28.4h1430 sxtl v28.4s, v28.4h
1431 addp v16.4s, v16.4s, v17.4s1431 addp v16.4s, v16.4s, v17.4s
1432 mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 31432 mla v19.4s, v7.4s, v28.4s // multiplication of upper half for idx 3
1433 subs w2, w2, #41433 subs w2, w2, #4
1434 addp v18.4s, v18.4s, v19.4s1434 addp v18.4s, v18.4s, v19.4s
1435 addp v16.4s, v16.4s, v18.4s1435 addp v16.4s, v16.4s, v18.4s
1436 sshl v16.4s, v16.4s, v20.4s1436 sshl v16.4s, v16.4s, v20.4s
1437 smin v16.4s, v16.4s, v21.4s1437 smin v16.4s, v16.4s, v21.4s
1438 1438
1439 st1 {v16.4s}, [x1], #161439 st1 {v16.4s}, [x1], #16
1440 add x4, x4, x7, lsl #21440 add x4, x4, x7, lsl #2
1441 b.gt 1b1441 b.gt 1b
1442 1442
1443 ldp d8, d9, [sp]1443 ldp d8, d9, [sp]
1444 ldp d10, d11, [sp, #0x10]1444 ldp d10, d11, [sp, #0x10]
1445 1445
1446 add sp, sp, #0x201446 add sp, sp, #0x20
1447 1447
1448 ret1448 ret
1449endfunc1449endfunc
libswscale/aarch64/output.S+165−165
@@ -29,178 +29,178 @@ function ff_yuv2planeX_8_neon, export=1 · 28 unmodified lines
29// x5 - const uint8_t *dither,29// x5 - const uint8_t *dither,
30// w6 - int offset30// w6 - int offset
31 31
32 ld1 {v0.8b}, [x5] // load 8x8-bit dither32 ld1 {v0.8b}, [x5] // load 8x8-bit dither
33 and w6, w6, #733 and w6, w6, #7
34 cbz w6, 1f // check if offsetting present34 cbz w6, 1f // check if offsetting present
35 ext v0.8b, v0.8b, v0.8b, #3 // honor offsetting which can be 0 or 3 only35 ext v0.8b, v0.8b, v0.8b, #3 // honor offsetting which can be 0 or 3 only
361: uxtl v0.8h, v0.8b // extend dither to 16-bit361: uxtl v0.8h, v0.8b // extend dither to 16-bit
37 ushll v1.4s, v0.4h, #12 // extend dither to 32-bit with left shift by 12 (part 1)37 ushll v1.4s, v0.4h, #12 // extend dither to 32-bit with left shift by 12 (part 1)
38 ushll2 v2.4s, v0.8h, #12 // extend dither to 32-bit with left shift by 12 (part 2)38 ushll2 v2.4s, v0.8h, #12 // extend dither to 32-bit with left shift by 12 (part 2)
39 cmp w1, #8 // if filterSize == 8, branch to specialized version39 cmp w1, #8 // if filterSize == 8, branch to specialized version
40 b.eq 6f40 b.eq 6f
41 cmp w1, #4 // if filterSize == 4, branch to specialized version41 cmp w1, #4 // if filterSize == 4, branch to specialized version
42 b.eq 8f42 b.eq 8f
43 cmp w1, #2 // if filterSize == 2, branch to specialized version43 cmp w1, #2 // if filterSize == 2, branch to specialized version
44 b.eq 10f44 b.eq 10f
45 45
46// The filter size does not match of the of specialized implementations. It is either even or odd. If it is even46// The filter size does not match of the of specialized implementations. It is either even or odd. If it is even
47// then use the first section below.47// then use the first section below.
48 mov x7, #0 // i = 048 mov x7, #0 // i = 0
49 tbnz w1, #0, 4f // if filterSize % 2 != 0 branch to specialized version49 tbnz w1, #0, 4f // if filterSize % 2 != 0 branch to specialized version
50// fs % 2 == 050// fs % 2 == 0
512: mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value512: mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
52 mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value52 mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
53 mov w8, w1 // tmpfilterSize = filterSize53 mov w8, w1 // tmpfilterSize = filterSize
54 mov x9, x2 // srcp = src54 mov x9, x2 // srcp = src
55 mov x10, x0 // filterp = filter55 mov x10, x0 // filterp = filter
563: ldp x11, x12, [x9], #16 // get 2 pointers: src[j] and src[j+1]563: ldp x11, x12, [x9], #16 // get 2 pointers: src[j] and src[j+1]
57 ldr s7, [x10], #4 // read 2x16-bit coeff X and Y at filter[j] and filter[j+1]57 ldr s7, [x10], #4 // read 2x16-bit coeff X and Y at filter[j] and filter[j+1]
58 add x11, x11, x7, lsl #1 // &src[j ][i]58 add x11, x11, x7, lsl #1 // &src[j ][i]
59 add x12, x12, x7, lsl #1 // &src[j+1][i]59 add x12, x12, x7, lsl #1 // &src[j+1][i]
60 ld1 {v5.8h}, [x11] // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H60 ld1 {v5.8h}, [x11] // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H
61 ld1 {v6.8h}, [x12] // read 8x16-bit @ src[j+1][i + {0..7}]: I,J,K,L,M,N,O,P61 ld1 {v6.8h}, [x12] // read 8x16-bit @ src[j+1][i + {0..7}]: I,J,K,L,M,N,O,P
62 smlal v3.4s, v5.4h, v7.h[0] // val0 += {A,B,C,D} * X62 smlal v3.4s, v5.4h, v7.h[0] // val0 += {A,B,C,D} * X
63 smlal2 v4.4s, v5.8h, v7.h[0] // val1 += {E,F,G,H} * X63 smlal2 v4.4s, v5.8h, v7.h[0] // val1 += {E,F,G,H} * X
64 smlal v3.4s, v6.4h, v7.h[1] // val0 += {I,J,K,L} * Y64 smlal v3.4s, v6.4h, v7.h[1] // val0 += {I,J,K,L} * Y
65 smlal2 v4.4s, v6.8h, v7.h[1] // val1 += {M,N,O,P} * Y65 smlal2 v4.4s, v6.8h, v7.h[1] // val1 += {M,N,O,P} * Y
66 subs w8, w8, #2 // tmpfilterSize -= 266 subs w8, w8, #2 // tmpfilterSize -= 2
67 b.gt 3b // loop until filterSize consumed67 b.gt 3b // loop until filterSize consumed
68 68
69 sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)69 sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
70 sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)70 sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
71 uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)71 uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
72 st1 {v3.8b}, [x3], #8 // write to destination72 st1 {v3.8b}, [x3], #8 // write to destination
73 subs w4, w4, #8 // dstW -= 873 subs w4, w4, #8 // dstW -= 8
74 add x7, x7, #8 // i += 874 add x7, x7, #8 // i += 8
75 b.gt 2b // loop until width consumed75 b.gt 2b // loop until width consumed
76 ret76 ret
77 77
78// If filter size is odd (most likely == 1), then use this section.78// If filter size is odd (most likely == 1), then use this section.
79// fs % 2 != 079// fs % 2 != 0
804: mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value804: mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
81 mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value81 mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
82 mov w8, w1 // tmpfilterSize = filterSize82 mov w8, w1 // tmpfilterSize = filterSize
83 mov x9, x2 // srcp = src83 mov x9, x2 // srcp = src
84 mov x10, x0 // filterp = filter84 mov x10, x0 // filterp = filter
855: ldr x11, [x9], #8 // get 1 pointer: src[j]855: ldr x11, [x9], #8 // get 1 pointer: src[j]
86 ldr h6, [x10], #2 // read 1 16 bit coeff X at filter[j]86 ldr h6, [x10], #2 // read 1 16 bit coeff X at filter[j]
87 add x11, x11, x7, lsl #1 // &src[j ][i]87 add x11, x11, x7, lsl #1 // &src[j ][i]
88 ld1 {v5.8h}, [x11] // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H88 ld1 {v5.8h}, [x11] // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H
89 smlal v3.4s, v5.4h, v6.h[0] // val0 += {A,B,C,D} * X89 smlal v3.4s, v5.4h, v6.h[0] // val0 += {A,B,C,D} * X
90 smlal2 v4.4s, v5.8h, v6.h[0] // val1 += {E,F,G,H} * X90 smlal2 v4.4s, v5.8h, v6.h[0] // val1 += {E,F,G,H} * X
91 subs w8, w8, #1 // tmpfilterSize -= 291 subs w8, w8, #1 // tmpfilterSize -= 2
92 b.gt 5b // loop until filterSize consumed92 b.gt 5b // loop until filterSize consumed
93 93
94 sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)94 sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
95 sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)95 sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
96 uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)96 uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
97 st1 {v3.8b}, [x3], #8 // write to destination97 st1 {v3.8b}, [x3], #8 // write to destination
98 subs w4, w4, #8 // dstW -= 898 subs w4, w4, #8 // dstW -= 8
99 add x7, x7, #8 // i += 899 add x7, x7, #8 // i += 8
100 b.gt 4b // loop until width consumed100 b.gt 4b // loop until width consumed
101 ret101 ret
102 102
1036: // fs=81036: // fs=8
104 ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]104 ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]
105 ldp x7, x9, [x2, #16] // load 2 pointers: src[j+2] and src[j+3]105 ldp x7, x9, [x2, #16] // load 2 pointers: src[j+2] and src[j+3]
106 ldp x10, x11, [x2, #32] // load 2 pointers: src[j+4] and src[j+5]106 ldp x10, x11, [x2, #32] // load 2 pointers: src[j+4] and src[j+5]
107 ldp x12, x13, [x2, #48] // load 2 pointers: src[j+6] and src[j+7]107 ldp x12, x13, [x2, #48] // load 2 pointers: src[j+6] and src[j+7]
108 108
109 // load 8x16-bit values for filter[j], where j=0..7109 // load 8x16-bit values for filter[j], where j=0..7
110 ld1 {v6.8h}, [x0]110 ld1 {v6.8h}, [x0]
1117:1117:
112 mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value112 mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
113 mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value113 mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
114 114
115 ld1 {v24.8h}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]115 ld1 {v24.8h}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]
116 ld1 {v25.8h}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]116 ld1 {v25.8h}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]
117 ld1 {v26.8h}, [x7], #16 // load 8x16-bit values for src[j + 2][i + {0..7}]117 ld1 {v26.8h}, [x7], #16 // load 8x16-bit values for src[j + 2][i + {0..7}]
118 ld1 {v27.8h}, [x9], #16 // load 8x16-bit values for src[j + 3][i + {0..7}]118 ld1 {v27.8h}, [x9], #16 // load 8x16-bit values for src[j + 3][i + {0..7}]
119 ld1 {v28.8h}, [x10], #16 // load 8x16-bit values for src[j + 4][i + {0..7}]119 ld1 {v28.8h}, [x10], #16 // load 8x16-bit values for src[j + 4][i + {0..7}]
120 ld1 {v29.8h}, [x11], #16 // load 8x16-bit values for src[j + 5][i + {0..7}]120 ld1 {v29.8h}, [x11], #16 // load 8x16-bit values for src[j + 5][i + {0..7}]
121 ld1 {v30.8h}, [x12], #16 // load 8x16-bit values for src[j + 6][i + {0..7}]121 ld1 {v30.8h}, [x12], #16 // load 8x16-bit values for src[j + 6][i + {0..7}]
122 ld1 {v31.8h}, [x13], #16 // load 8x16-bit values for src[j + 7][i + {0..7}]122 ld1 {v31.8h}, [x13], #16 // load 8x16-bit values for src[j + 7][i + {0..7}]
123 123
124 smlal v3.4s, v24.4h, v6.h[0] // val0 += src[0][i + {0..3}] * filter[0]124 smlal v3.4s, v24.4h, v6.h[0] // val0 += src[0][i + {0..3}] * filter[0]
125 smlal2 v4.4s, v24.8h, v6.h[0] // val1 += src[0][i + {4..7}] * filter[0]125 smlal2 v4.4s, v24.8h, v6.h[0] // val1 += src[0][i + {4..7}] * filter[0]
126 smlal v3.4s, v25.4h, v6.h[1] // val0 += src[1][i + {0..3}] * filter[1]126 smlal v3.4s, v25.4h, v6.h[1] // val0 += src[1][i + {0..3}] * filter[1]
127 smlal2 v4.4s, v25.8h, v6.h[1] // val1 += src[1][i + {4..7}] * filter[1]127 smlal2 v4.4s, v25.8h, v6.h[1] // val1 += src[1][i + {4..7}] * filter[1]
128 smlal v3.4s, v26.4h, v6.h[2] // val0 += src[2][i + {0..3}] * filter[2]128 smlal v3.4s, v26.4h, v6.h[2] // val0 += src[2][i + {0..3}] * filter[2]
129 smlal2 v4.4s, v26.8h, v6.h[2] // val1 += src[2][i + {4..7}] * filter[2]129 smlal2 v4.4s, v26.8h, v6.h[2] // val1 += src[2][i + {4..7}] * filter[2]
130 smlal v3.4s, v27.4h, v6.h[3] // val0 += src[3][i + {0..3}] * filter[3]130 smlal v3.4s, v27.4h, v6.h[3] // val0 += src[3][i + {0..3}] * filter[3]
131 smlal2 v4.4s, v27.8h, v6.h[3] // val1 += src[3][i + {4..7}] * filter[3]131 smlal2 v4.4s, v27.8h, v6.h[3] // val1 += src[3][i + {4..7}] * filter[3]
132 smlal v3.4s, v28.4h, v6.h[4] // val0 += src[4][i + {0..3}] * filter[4]132 smlal v3.4s, v28.4h, v6.h[4] // val0 += src[4][i + {0..3}] * filter[4]
133 smlal2 v4.4s, v28.8h, v6.h[4] // val1 += src[4][i + {4..7}] * filter[4]133 smlal2 v4.4s, v28.8h, v6.h[4] // val1 += src[4][i + {4..7}] * filter[4]
134 smlal v3.4s, v29.4h, v6.h[5] // val0 += src[5][i + {0..3}] * filter[5]134 smlal v3.4s, v29.4h, v6.h[5] // val0 += src[5][i + {0..3}] * filter[5]
135 smlal2 v4.4s, v29.8h, v6.h[5] // val1 += src[5][i + {4..7}] * filter[5]135 smlal2 v4.4s, v29.8h, v6.h[5] // val1 += src[5][i + {4..7}] * filter[5]
136 smlal v3.4s, v30.4h, v6.h[6] // val0 += src[6][i + {0..3}] * filter[6]136 smlal v3.4s, v30.4h, v6.h[6] // val0 += src[6][i + {0..3}] * filter[6]
137 smlal2 v4.4s, v30.8h, v6.h[6] // val1 += src[6][i + {4..7}] * filter[6]137 smlal2 v4.4s, v30.8h, v6.h[6] // val1 += src[6][i + {4..7}] * filter[6]
138 smlal v3.4s, v31.4h, v6.h[7] // val0 += src[7][i + {0..3}] * filter[7]138 smlal v3.4s, v31.4h, v6.h[7] // val0 += src[7][i + {0..3}] * filter[7]
139 smlal2 v4.4s, v31.8h, v6.h[7] // val1 += src[7][i + {4..7}] * filter[7]139 smlal2 v4.4s, v31.8h, v6.h[7] // val1 += src[7][i + {4..7}] * filter[7]
140 140
141 sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)141 sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
142 sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)142 sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
143 uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)143 uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
144 subs w4, w4, #8 // dstW -= 8144 subs w4, w4, #8 // dstW -= 8
145 st1 {v3.8b}, [x3], #8 // write to destination145 st1 {v3.8b}, [x3], #8 // write to destination
146 b.gt 7b // loop until width consumed146 b.gt 7b // loop until width consumed
147 ret147 ret
148 148
1498: // fs=41498: // fs=4
150 ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]150 ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]
151 ldp x7, x9, [x2, #16] // load 2 pointers: src[j+2] and src[j+3]151 ldp x7, x9, [x2, #16] // load 2 pointers: src[j+2] and src[j+3]
152 152
153 // load 4x16-bit values for filter[j], where j=0..3 and replicated across lanes153 // load 4x16-bit values for filter[j], where j=0..3 and replicated across lanes
154 ld1 {v6.4h}, [x0]154 ld1 {v6.4h}, [x0]
1559:1559:
156 mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value156 mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
157 mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value157 mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
158 158
159 ld1 {v24.8h}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]159 ld1 {v24.8h}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]
160 ld1 {v25.8h}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]160 ld1 {v25.8h}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]
161 ld1 {v26.8h}, [x7], #16 // load 8x16-bit values for src[j + 2][i + {0..7}]161 ld1 {v26.8h}, [x7], #16 // load 8x16-bit values for src[j + 2][i + {0..7}]
162 ld1 {v27.8h}, [x9], #16 // load 8x16-bit values for src[j + 3][i + {0..7}]162 ld1 {v27.8h}, [x9], #16 // load 8x16-bit values for src[j + 3][i + {0..7}]
163 163
164 smlal v3.4s, v24.4h, v6.h[0] // val0 += src[0][i + {0..3}] * filter[0]164 smlal v3.4s, v24.4h, v6.h[0] // val0 += src[0][i + {0..3}] * filter[0]
165 smlal2 v4.4s, v24.8h, v6.h[0] // val1 += src[0][i + {4..7}] * filter[0]165 smlal2 v4.4s, v24.8h, v6.h[0] // val1 += src[0][i + {4..7}] * filter[0]
166 smlal v3.4s, v25.4h, v6.h[1] // val0 += src[1][i + {0..3}] * filter[1]166 smlal v3.4s, v25.4h, v6.h[1] // val0 += src[1][i + {0..3}] * filter[1]
167 smlal2 v4.4s, v25.8h, v6.h[1] // val1 += src[1][i + {4..7}] * filter[1]167 smlal2 v4.4s, v25.8h, v6.h[1] // val1 += src[1][i + {4..7}] * filter[1]
168 smlal v3.4s, v26.4h, v6.h[2] // val0 += src[2][i + {0..3}] * filter[2]168 smlal v3.4s, v26.4h, v6.h[2] // val0 += src[2][i + {0..3}] * filter[2]
169 smlal2 v4.4s, v26.8h, v6.h[2] // val1 += src[2][i + {4..7}] * filter[2]169 smlal2 v4.4s, v26.8h, v6.h[2] // val1 += src[2][i + {4..7}] * filter[2]
170 smlal v3.4s, v27.4h, v6.h[3] // val0 += src[3][i + {0..3}] * filter[3]170 smlal v3.4s, v27.4h, v6.h[3] // val0 += src[3][i + {0..3}] * filter[3]
171 smlal2 v4.4s, v27.8h, v6.h[3] // val1 += src[3][i + {4..7}] * filter[3]171 smlal2 v4.4s, v27.8h, v6.h[3] // val1 += src[3][i + {4..7}] * filter[3]
172 172
173 sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)173 sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
174 sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)174 sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
175 uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)175 uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
176 st1 {v3.8b}, [x3], #8 // write to destination176 st1 {v3.8b}, [x3], #8 // write to destination
177 subs w4, w4, #8 // dstW -= 8177 subs w4, w4, #8 // dstW -= 8
178 b.gt 9b // loop until width consumed178 b.gt 9b // loop until width consumed
179 ret179 ret
180 180
18110: // fs=218110: // fs=2
182 ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]182 ldp x5, x6, [x2] // load 2 pointers: src[j ] and src[j+1]
183 183
184 // load 2x16-bit values for filter[j], where j=0..1 and replicated across lanes184 // load 2x16-bit values for filter[j], where j=0..1 and replicated across lanes
185 ldr s6, [x0]185 ldr s6, [x0]
18611:18611:
187 mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value187 mov v3.16b, v1.16b // initialize accumulator part 1 with dithering value
188 mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value188 mov v4.16b, v2.16b // initialize accumulator part 2 with dithering value
189 189
190 ld1 {v24.8h}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]190 ld1 {v24.8h}, [x5], #16 // load 8x16-bit values for src[j + 0][i + {0..7}]
191 ld1 {v25.8h}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]191 ld1 {v25.8h}, [x6], #16 // load 8x16-bit values for src[j + 1][i + {0..7}]
192 192
193 smlal v3.4s, v24.4h, v6.h[0] // val0 += src[0][i + {0..3}] * filter[0]193 smlal v3.4s, v24.4h, v6.h[0] // val0 += src[0][i + {0..3}] * filter[0]
194 smlal2 v4.4s, v24.8h, v6.h[0] // val1 += src[0][i + {4..7}] * filter[0]194 smlal2 v4.4s, v24.8h, v6.h[0] // val1 += src[0][i + {4..7}] * filter[0]
195 smlal v3.4s, v25.4h, v6.h[1] // val0 += src[1][i + {0..3}] * filter[1]195 smlal v3.4s, v25.4h, v6.h[1] // val0 += src[1][i + {0..3}] * filter[1]
196 smlal2 v4.4s, v25.8h, v6.h[1] // val1 += src[1][i + {4..7}] * filter[1]196 smlal2 v4.4s, v25.8h, v6.h[1] // val1 += src[1][i + {4..7}] * filter[1]
197 197
198 sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)198 sqshrun v3.4h, v3.4s, #16 // clip16(val0>>16)
199 sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)199 sqshrun2 v3.8h, v4.4s, #16 // clip16(val1>>16)
200 uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)200 uqshrn v3.8b, v3.8h, #3 // clip8(val>>19)
201 st1 {v3.8b}, [x3], #8 // write to destination201 st1 {v3.8b}, [x3], #8 // write to destination
202 subs w4, w4, #8 // dstW -= 8202 subs w4, w4, #8 // dstW -= 8
203 b.gt 11b // loop until width consumed203 b.gt 11b // loop until width consumed
204 ret204 ret
205endfunc205endfunc
206 206
@@ -210,25 +210,25 @@ function ff_yuv2plane1_8_neon, export=1 · 3 unmodified lines
210// w2 - int dstW,210// w2 - int dstW,
211// x3 - const uint8_t *dither,211// x3 - const uint8_t *dither,
212// w4 - int offset212// w4 - int offset
213 ld1 {v0.8b}, [x3] // load 8x8-bit dither213 ld1 {v0.8b}, [x3] // load 8x8-bit dither
214 and w4, w4, #7214 and w4, w4, #7
215 cbz w4, 1f // check if offsetting present215 cbz w4, 1f // check if offsetting present
216 ext v0.8b, v0.8b, v0.8b, #3 // honor offsetting which can be 0 or 3 only216 ext v0.8b, v0.8b, v0.8b, #3 // honor offsetting which can be 0 or 3 only
2171: uxtl v0.8h, v0.8b // extend dither to 32-bit2171: uxtl v0.8h, v0.8b // extend dither to 32-bit
218 uxtl v1.4s, v0.4h218 uxtl v1.4s, v0.4h
219 uxtl2 v2.4s, v0.8h219 uxtl2 v2.4s, v0.8h
2202:2202:
221 ld1 {v3.8h}, [x0], #16 // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H221 ld1 {v3.8h}, [x0], #16 // read 8x16-bit @ src[j ][i + {0..7}]: A,B,C,D,E,F,G,H
222 sxtl v4.4s, v3.4h222 sxtl v4.4s, v3.4h
223 sxtl2 v5.4s, v3.8h223 sxtl2 v5.4s, v3.8h
224 add v4.4s, v4.4s, v1.4s224 add v4.4s, v4.4s, v1.4s
225 add v5.4s, v5.4s, v2.4s225 add v5.4s, v5.4s, v2.4s
226 sqshrun v4.4h, v4.4s, #6226 sqshrun v4.4h, v4.4s, #6
227 sqshrun2 v4.8h, v5.4s, #6227 sqshrun2 v4.8h, v5.4s, #6
228 228
229 uqshrn v3.8b, v4.8h, #1 // clip8(val>>7)229 uqshrn v3.8b, v4.8h, #1 // clip8(val>>7)
230 subs w2, w2, #8 // dstW -= 8230 subs w2, w2, #8 // dstW -= 8
231 st1 {v3.8b}, [x1], #8 // write to destination231 st1 {v3.8b}, [x1], #8 // write to destination
232 b.gt 2b // loop until width consumed232 b.gt 2b // loop until width consumed
233 ret233 ret
234endfunc234endfunc
libswscale/aarch64/yuv2rgb_neon.S+110−110
@@ -23,23 +23,23 @@ · 22 unmodified lines
23 23
24.macro load_yoff_ycoeff yoff ycoeff24.macro load_yoff_ycoeff yoff ycoeff
25#if defined(__APPLE__)25#if defined(__APPLE__)
26 ldp w9, w10, [sp, #\yoff]26 ldp w9, w10, [sp, #\yoff]
27#else27#else
28 ldr w9, [sp, #\yoff]28 ldr w9, [sp, #\yoff]
29 ldr w10, [sp, #\ycoeff]29 ldr w10, [sp, #\ycoeff]
30#endif30#endif
31.endm31.endm
32 32
33.macro load_args_nv1233.macro load_args_nv12
34 ldr x8, [sp] // table34 ldr x8, [sp] // table
35 load_yoff_ycoeff 8, 16 // y_offset, y_coeff35 load_yoff_ycoeff 8, 16 // y_offset, y_coeff
36 ld1 {v1.1d}, [x8]36 ld1 {v1.1d}, [x8]
37 dup v0.8h, w1037 dup v0.8h, w10
38 dup v3.8h, w938 dup v3.8h, w9
39 sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding)39 sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding)
40 sub w5, w5, w0 // w5 = linesizeY - width (paddingY)40 sub w5, w5, w0 // w5 = linesizeY - width (paddingY)
41 sub w7, w7, w0 // w7 = linesizeC - width (paddingC)41 sub w7, w7, w0 // w7 = linesizeC - width (paddingC)
42 neg w11, w042 neg w11, w0
43.endm43.endm
44 44
45.macro load_args_nv2145.macro load_args_nv21
@@ -47,52 +47,52 @@ · 1 unmodified line
47.endm47.endm
48 48
49.macro load_args_yuv420p49.macro load_args_yuv420p
50 ldr x13, [sp] // srcV50 ldr x13, [sp] // srcV
51 ldr w14, [sp, #8] // linesizeV51 ldr w14, [sp, #8] // linesizeV
52 ldr x8, [sp, #16] // table52 ldr x8, [sp, #16] // table
53 load_yoff_ycoeff 24, 32 // y_offset, y_coeff53 load_yoff_ycoeff 24, 32 // y_offset, y_coeff
54 ld1 {v1.1d}, [x8]54 ld1 {v1.1d}, [x8]
55 dup v0.8h, w1055 dup v0.8h, w10
56 dup v3.8h, w956 dup v3.8h, w9
57 sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding)57 sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding)
58 sub w5, w5, w0 // w5 = linesizeY - width (paddingY)58 sub w5, w5, w0 // w5 = linesizeY - width (paddingY)
59 sub w7, w7, w0, lsr #1 // w7 = linesizeU - width / 2 (paddingU)59 sub w7, w7, w0, lsr #1 // w7 = linesizeU - width / 2 (paddingU)
60 sub w14, w14, w0, lsr #1 // w14 = linesizeV - width / 2 (paddingV)60 sub w14, w14, w0, lsr #1 // w14 = linesizeV - width / 2 (paddingV)
61 lsr w11, w0, #161 lsr w11, w0, #1
62 neg w11, w1162 neg w11, w11
63.endm63.endm
64 64
65.macro load_args_yuv422p65.macro load_args_yuv422p
66 ldr x13, [sp] // srcV66 ldr x13, [sp] // srcV
67 ldr w14, [sp, #8] // linesizeV67 ldr w14, [sp, #8] // linesizeV
68 ldr x8, [sp, #16] // table68 ldr x8, [sp, #16] // table
69 load_yoff_ycoeff 24, 32 // y_offset, y_coeff69 load_yoff_ycoeff 24, 32 // y_offset, y_coeff
70 ld1 {v1.1d}, [x8]70 ld1 {v1.1d}, [x8]
71 dup v0.8h, w1071 dup v0.8h, w10
72 dup v3.8h, w972 dup v3.8h, w9
73 sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding)73 sub w3, w3, w0, lsl #2 // w3 = linesize - width * 4 (padding)
74 sub w5, w5, w0 // w5 = linesizeY - width (paddingY)74 sub w5, w5, w0 // w5 = linesizeY - width (paddingY)
75 sub w7, w7, w0, lsr #1 // w7 = linesizeU - width / 2 (paddingU)75 sub w7, w7, w0, lsr #1 // w7 = linesizeU - width / 2 (paddingU)
76 sub w14, w14, w0, lsr #1 // w14 = linesizeV - width / 2 (paddingV)76 sub w14, w14, w0, lsr #1 // w14 = linesizeV - width / 2 (paddingV)
77.endm77.endm
78 78
79.macro load_chroma_nv1279.macro load_chroma_nv12
80 ld2 {v16.8b, v17.8b}, [x6], #1680 ld2 {v16.8b, v17.8b}, [x6], #16
81 ushll v18.8h, v16.8b, #381 ushll v18.8h, v16.8b, #3
82 ushll v19.8h, v17.8b, #382 ushll v19.8h, v17.8b, #3
83.endm83.endm
84 84
85.macro load_chroma_nv2185.macro load_chroma_nv21
86 ld2 {v16.8b, v17.8b}, [x6], #1686 ld2 {v16.8b, v17.8b}, [x6], #16
87 ushll v19.8h, v16.8b, #387 ushll v19.8h, v16.8b, #3
88 ushll v18.8h, v17.8b, #388 ushll v18.8h, v17.8b, #3
89.endm89.endm
90 90
91.macro load_chroma_yuv420p91.macro load_chroma_yuv420p
92 ld1 {v16.8b}, [ x6], #892 ld1 {v16.8b}, [ x6], #8
93 ld1 {v17.8b}, [x13], #893 ld1 {v17.8b}, [x13], #8
94 ushll v18.8h, v16.8b, #394 ushll v18.8h, v16.8b, #3
95 ushll v19.8h, v17.8b, #395 ushll v19.8h, v17.8b, #3
96.endm96.endm
97 97
98.macro load_chroma_yuv422p98.macro load_chroma_yuv422p
@@ -100,9 +100,9 @@ · 1 unmodified line
100.endm100.endm
101 101
102.macro increment_nv12102.macro increment_nv12
103 ands w15, w1, #1103 ands w15, w1, #1
104 csel w16, w7, w11, ne // incC = (h & 1) ? paddincC : -width104 csel w16, w7, w11, ne // incC = (h & 1) ? paddincC : -width
105 add x6, x6, w16, sxtw // srcC += incC105 add x6, x6, w16, sxtw // srcC += incC
106.endm106.endm
107 107
108.macro increment_nv21108.macro increment_nv21
@@ -110,100 +110,100 @@ · 1 unmodified line
110.endm110.endm
111 111
112.macro increment_yuv420p112.macro increment_yuv420p
113 ands w15, w1, #1113 ands w15, w1, #1
114 csel w16, w7, w11, ne // incU = (h & 1) ? paddincU : -width/2114 csel w16, w7, w11, ne // incU = (h & 1) ? paddincU : -width/2
115 csel w17, w14, w11, ne // incV = (h & 1) ? paddincV : -width/2115 csel w17, w14, w11, ne // incV = (h & 1) ? paddincV : -width/2
116 add x6, x6, w16, sxtw // srcU += incU116 add x6, x6, w16, sxtw // srcU += incU
117 add x13, x13, w17, sxtw // srcV += incV117 add x13, x13, w17, sxtw // srcV += incV
118.endm118.endm
119 119
120.macro increment_yuv422p120.macro increment_yuv422p
121 add x6, x6, w7, sxtw // srcU += incU121 add x6, x6, w7, sxtw // srcU += incU
122 add x13, x13, w14, sxtw // srcV += incV122 add x13, x13, w14, sxtw // srcV += incV
123.endm123.endm
124 124
125.macro compute_rgba r1 g1 b1 a1 r2 g2 b2 a2125.macro compute_rgba r1 g1 b1 a1 r2 g2 b2 a2
126 add v20.8h, v26.8h, v20.8h // Y1 + R1126 add v20.8h, v26.8h, v20.8h // Y1 + R1
127 add v21.8h, v27.8h, v21.8h // Y2 + R2127 add v21.8h, v27.8h, v21.8h // Y2 + R2
128 add v22.8h, v26.8h, v22.8h // Y1 + G1128 add v22.8h, v26.8h, v22.8h // Y1 + G1
129 add v23.8h, v27.8h, v23.8h // Y2 + G2129 add v23.8h, v27.8h, v23.8h // Y2 + G2
130 add v24.8h, v26.8h, v24.8h // Y1 + B1130 add v24.8h, v26.8h, v24.8h // Y1 + B1
131 add v25.8h, v27.8h, v25.8h // Y2 + B2131 add v25.8h, v27.8h, v25.8h // Y2 + B2
132 sqrshrun \r1, v20.8h, #1 // clip_u8((Y1 + R1) >> 1)132 sqrshrun \r1, v20.8h, #1 // clip_u8((Y1 + R1) >> 1)
133 sqrshrun \r2, v21.8h, #1 // clip_u8((Y2 + R1) >> 1)133 sqrshrun \r2, v21.8h, #1 // clip_u8((Y2 + R1) >> 1)
134 sqrshrun \g1, v22.8h, #1 // clip_u8((Y1 + G1) >> 1)134 sqrshrun \g1, v22.8h, #1 // clip_u8((Y1 + G1) >> 1)
135 sqrshrun \g2, v23.8h, #1 // clip_u8((Y2 + G1) >> 1)135 sqrshrun \g2, v23.8h, #1 // clip_u8((Y2 + G1) >> 1)
136 sqrshrun \b1, v24.8h, #1 // clip_u8((Y1 + B1) >> 1)136 sqrshrun \b1, v24.8h, #1 // clip_u8((Y1 + B1) >> 1)
137 sqrshrun \b2, v25.8h, #1 // clip_u8((Y2 + B1) >> 1)137 sqrshrun \b2, v25.8h, #1 // clip_u8((Y2 + B1) >> 1)
138 movi \a1, #255138 movi \a1, #255
139 movi \a2, #255139 movi \a2, #255
140.endm140.endm
141 141
142.macro declare_func ifmt ofmt142.macro declare_func ifmt ofmt
143function ff_\ifmt\()_to_\ofmt\()_neon, export=1143function ff_\ifmt\()_to_\ofmt\()_neon, export=1
144 load_args_\ifmt144 load_args_\ifmt
145 mov w9, w1145 mov w9, w1
1461:1461:
147 mov w8, w0 // w8 = width147 mov w8, w0 // w8 = width
1482:1482:
149 movi v5.8h, #4, lsl #8 // 128 * (1<<3)149 movi v5.8h, #4, lsl #8 // 128 * (1<<3)
150 load_chroma_\ifmt150 load_chroma_\ifmt
151 sub v18.8h, v18.8h, v5.8h // U*(1<<3) - 128*(1<<3)151 sub v18.8h, v18.8h, v5.8h // U*(1<<3) - 128*(1<<3)
152 sub v19.8h, v19.8h, v5.8h // V*(1<<3) - 128*(1<<3)152 sub v19.8h, v19.8h, v5.8h // V*(1<<3) - 128*(1<<3)
153 sqdmulh v20.8h, v19.8h, v1.h[0] // V * v2r (R)153 sqdmulh v20.8h, v19.8h, v1.h[0] // V * v2r (R)
154 sqdmulh v22.8h, v18.8h, v1.h[1] // U * u2g154 sqdmulh v22.8h, v18.8h, v1.h[1] // U * u2g
155 sqdmulh v19.8h, v19.8h, v1.h[2] // V * v2g155 sqdmulh v19.8h, v19.8h, v1.h[2] // V * v2g
156 add v22.8h, v22.8h, v19.8h // U * u2g + V * v2g (G)156 add v22.8h, v22.8h, v19.8h // U * u2g + V * v2g (G)
157 sqdmulh v24.8h, v18.8h, v1.h[3] // U * u2b (B)157 sqdmulh v24.8h, v18.8h, v1.h[3] // U * u2b (B)
158 zip2 v21.8h, v20.8h, v20.8h // R2158 zip2 v21.8h, v20.8h, v20.8h // R2
159 zip1 v20.8h, v20.8h, v20.8h // R1159 zip1 v20.8h, v20.8h, v20.8h // R1
160 zip2 v23.8h, v22.8h, v22.8h // G2160 zip2 v23.8h, v22.8h, v22.8h // G2
161 zip1 v22.8h, v22.8h, v22.8h // G1161 zip1 v22.8h, v22.8h, v22.8h // G1
162 zip2 v25.8h, v24.8h, v24.8h // B2162 zip2 v25.8h, v24.8h, v24.8h // B2
163 zip1 v24.8h, v24.8h, v24.8h // B1163 zip1 v24.8h, v24.8h, v24.8h // B1
164 ld1 {v2.16b}, [x4], #16 // load luma164 ld1 {v2.16b}, [x4], #16 // load luma
165 ushll v26.8h, v2.8b, #3 // Y1*(1<<3)165 ushll v26.8h, v2.8b, #3 // Y1*(1<<3)
166 ushll2 v27.8h, v2.16b, #3 // Y2*(1<<3)166 ushll2 v27.8h, v2.16b, #3 // Y2*(1<<3)
167 sub v26.8h, v26.8h, v3.8h // Y1*(1<<3) - y_offset167 sub v26.8h, v26.8h, v3.8h // Y1*(1<<3) - y_offset
168 sub v27.8h, v27.8h, v3.8h // Y2*(1<<3) - y_offset168 sub v27.8h, v27.8h, v3.8h // Y2*(1<<3) - y_offset
169 sqdmulh v26.8h, v26.8h, v0.8h // ((Y1*(1<<3) - y_offset) * y_coeff) >> 15169 sqdmulh v26.8h, v26.8h, v0.8h // ((Y1*(1<<3) - y_offset) * y_coeff) >> 15
170 sqdmulh v27.8h, v27.8h, v0.8h // ((Y2*(1<<3) - y_offset) * y_coeff) >> 15170 sqdmulh v27.8h, v27.8h, v0.8h // ((Y2*(1<<3) - y_offset) * y_coeff) >> 15
171 171
172.ifc \ofmt,argb // 1 2 3 0172.ifc \ofmt,argb // 1 2 3 0
173 compute_rgba v5.8b,v6.8b,v7.8b,v4.8b, v17.8b,v18.8b,v19.8b,v16.8b173 compute_rgba v5.8b,v6.8b,v7.8b,v4.8b, v17.8b,v18.8b,v19.8b,v16.8b
174.endif174.endif
175 175
176.ifc \ofmt,rgba // 0 1 2 3176.ifc \ofmt,rgba // 0 1 2 3
177 compute_rgba v4.8b,v5.8b,v6.8b,v7.8b, v16.8b,v17.8b,v18.8b,v19.8b177 compute_rgba v4.8b,v5.8b,v6.8b,v7.8b, v16.8b,v17.8b,v18.8b,v19.8b
178.endif178.endif
179 179
180.ifc \ofmt,abgr // 3 2 1 0180.ifc \ofmt,abgr // 3 2 1 0
181 compute_rgba v7.8b,v6.8b,v5.8b,v4.8b, v19.8b,v18.8b,v17.8b,v16.8b181 compute_rgba v7.8b,v6.8b,v5.8b,v4.8b, v19.8b,v18.8b,v17.8b,v16.8b
182.endif182.endif
183 183
184.ifc \ofmt,bgra // 2 1 0 3184.ifc \ofmt,bgra // 2 1 0 3
185 compute_rgba v6.8b,v5.8b,v4.8b,v7.8b, v18.8b,v17.8b,v16.8b,v19.8b185 compute_rgba v6.8b,v5.8b,v4.8b,v7.8b, v18.8b,v17.8b,v16.8b,v19.8b
186.endif186.endif
187 187
188 st4 { v4.8b, v5.8b, v6.8b, v7.8b}, [x2], #32188 st4 { v4.8b, v5.8b, v6.8b, v7.8b}, [x2], #32
189 st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [x2], #32189 st4 {v16.8b,v17.8b,v18.8b,v19.8b}, [x2], #32
190 subs w8, w8, #16 // width -= 16190 subs w8, w8, #16 // width -= 16
191 b.gt 2b191 b.gt 2b
192 add x2, x2, w3, sxtw // dst += padding192 add x2, x2, w3, sxtw // dst += padding
193 add x4, x4, w5, sxtw // srcY += paddingY193 add x4, x4, w5, sxtw // srcY += paddingY
194 increment_\ifmt194 increment_\ifmt
195 subs w1, w1, #1 // height -= 1195 subs w1, w1, #1 // height -= 1
196 b.gt 1b196 b.gt 1b
197 mov w0, w9197 mov w0, w9
198 ret198 ret
199endfunc199endfunc
200.endm200.endm
201 201
202.macro declare_rgb_funcs ifmt202.macro declare_rgb_funcs ifmt
203 declare_func \ifmt, argb203 declare_func \ifmt, argb
204 declare_func \ifmt, rgba204 declare_func \ifmt, rgba
205 declare_func \ifmt, abgr205 declare_func \ifmt, abgr
206 declare_func \ifmt, bgra206 declare_func \ifmt, bgra
207.endm207.endm
208 208
209declare_rgb_funcs nv12209declare_rgb_funcs nv12

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t