History

avcodec/hevcdsp: ARM NEON optimized qpel functions

uses comma as macro parameter separator

Seppo Tomperi committed 11 years ago

Commit 63ca0fe1 parent d1e2d593 files · +1160 −1Browse files

Unified · Split

3 files
  1. libavcodec/arm/Makefile +2 −1
  2. libavcodec/arm/hevcdsp_init_neon.c +159 −0
  3. libavcodec/arm/hevcdsp_qpel_neon.S +999 −0
libavcodec/arm/Makefile+2−1
@@ -138,7 +138,8 @@ NEON-OBJS-$(CONFIG_DCA_DECODER) += arm/dcadsp_neon.o \ · 137 unmodified lines
138138 arm/synth_filter_neon.o
139139 NEON-OBJS-$(CONFIG_HEVC_DECODER) += arm/hevcdsp_init_neon.o \
140140 arm/hevcdsp_deblock_neon.o \
141− arm/hevcdsp_idct_neon.o
141+ arm/hevcdsp_idct_neon.o \
142+ arm/hevcdsp_qpel_neon.o
142143 NEON-OBJS-$(CONFIG_RV30_DECODER) += arm/rv34dsp_neon.o
143144 NEON-OBJS-$(CONFIG_RV40_DECODER) += arm/rv34dsp_neon.o \
144145 arm/rv40dsp_neon.o
libavcodec/arm/hevcdsp_init_neon.c+159−0
@@ -42,10 +42,110 @@ void ff_hevc_transform_add_16x16_neon_8(uint8_t *_dst, int16_t *coeffs, · 41 unmodified lines
4242 void ff_hevc_transform_add_32x32_neon_8(uint8_t *_dst, int16_t *coeffs,
4343 ptrdiff_t stride);
4444
45+#define PUT_PIXELS(name) \
46+ void name(int16_t *dst, uint8_t *src, \
47+ ptrdiff_t srcstride, int height, \
48+ intptr_t mx, intptr_t my, int width)
49+PUT_PIXELS(ff_hevc_put_pixels_w2_neon_8);
50+PUT_PIXELS(ff_hevc_put_pixels_w4_neon_8);
51+PUT_PIXELS(ff_hevc_put_pixels_w6_neon_8);
52+PUT_PIXELS(ff_hevc_put_pixels_w8_neon_8);
53+PUT_PIXELS(ff_hevc_put_pixels_w12_neon_8);
54+PUT_PIXELS(ff_hevc_put_pixels_w16_neon_8);
55+PUT_PIXELS(ff_hevc_put_pixels_w24_neon_8);
56+PUT_PIXELS(ff_hevc_put_pixels_w32_neon_8);
57+PUT_PIXELS(ff_hevc_put_pixels_w48_neon_8);
58+PUT_PIXELS(ff_hevc_put_pixels_w64_neon_8);
59+#undef PUT_PIXELS
60+
61+static void (*put_hevc_qpel_neon[4][4])(int16_t *dst, ptrdiff_t dststride, uint8_t *src, ptrdiff_t srcstride,
62+ int height, int width);
63+static void (*put_hevc_qpel_uw_neon[4][4])(uint8_t *dst, ptrdiff_t dststride, uint8_t *_src, ptrdiff_t _srcstride,
64+ int width, int height, int16_t* src2, ptrdiff_t src2stride);
65+void ff_hevc_put_qpel_neon_wrapper(int16_t *dst, uint8_t *src, ptrdiff_t srcstride,
66+ int height, intptr_t mx, intptr_t my, int width);
67+void ff_hevc_put_qpel_uni_neon_wrapper(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, ptrdiff_t srcstride,
68+ int height, intptr_t mx, intptr_t my, int width);
69+void ff_hevc_put_qpel_bi_neon_wrapper(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, ptrdiff_t srcstride,
70+ int16_t *src2,
71+ int height, intptr_t mx, intptr_t my, int width);
72+#define QPEL_FUNC(name) \
73+ void name(int16_t *dst, ptrdiff_t dststride, uint8_t *src, ptrdiff_t srcstride, \
74+ int height, int width)
75+
76+QPEL_FUNC(ff_hevc_put_qpel_v1_neon_8);
77+QPEL_FUNC(ff_hevc_put_qpel_v2_neon_8);
78+QPEL_FUNC(ff_hevc_put_qpel_v3_neon_8);
79+QPEL_FUNC(ff_hevc_put_qpel_h1_neon_8);
80+QPEL_FUNC(ff_hevc_put_qpel_h2_neon_8);
81+QPEL_FUNC(ff_hevc_put_qpel_h3_neon_8);
82+QPEL_FUNC(ff_hevc_put_qpel_h1v1_neon_8);
83+QPEL_FUNC(ff_hevc_put_qpel_h1v2_neon_8);
84+QPEL_FUNC(ff_hevc_put_qpel_h1v3_neon_8);
85+QPEL_FUNC(ff_hevc_put_qpel_h2v1_neon_8);
86+QPEL_FUNC(ff_hevc_put_qpel_h2v2_neon_8);
87+QPEL_FUNC(ff_hevc_put_qpel_h2v3_neon_8);
88+QPEL_FUNC(ff_hevc_put_qpel_h3v1_neon_8);
89+QPEL_FUNC(ff_hevc_put_qpel_h3v2_neon_8);
90+QPEL_FUNC(ff_hevc_put_qpel_h3v3_neon_8);
91+#undef QPEL_FUNC
92+
93+#define QPEL_FUNC_UW_PIX(name) \
94+ void name(uint8_t *dst, ptrdiff_t dststride, uint8_t *_src, ptrdiff_t _srcstride, \
95+ int height, intptr_t mx, intptr_t my, int width);
96+QPEL_FUNC_UW_PIX(ff_hevc_put_qpel_uw_pixels_w4_neon_8);
97+QPEL_FUNC_UW_PIX(ff_hevc_put_qpel_uw_pixels_w8_neon_8);
98+QPEL_FUNC_UW_PIX(ff_hevc_put_qpel_uw_pixels_w16_neon_8);
99+QPEL_FUNC_UW_PIX(ff_hevc_put_qpel_uw_pixels_w24_neon_8);
100+QPEL_FUNC_UW_PIX(ff_hevc_put_qpel_uw_pixels_w32_neon_8);
101+QPEL_FUNC_UW_PIX(ff_hevc_put_qpel_uw_pixels_w48_neon_8);
102+QPEL_FUNC_UW_PIX(ff_hevc_put_qpel_uw_pixels_w64_neon_8);
103+#undef QPEL_FUNC_UW_PIX
104+
105+#define QPEL_FUNC_UW(name) \
106+ void name(uint8_t *dst, ptrdiff_t dststride, uint8_t *_src, ptrdiff_t _srcstride, \
107+ int width, int height, int16_t* src2, ptrdiff_t src2stride);
108+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_pixels_neon_8);
109+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_v1_neon_8);
110+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_v2_neon_8);
111+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_v3_neon_8);
112+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_h1_neon_8);
113+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_h2_neon_8);
114+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_h3_neon_8);
115+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_h1v1_neon_8);
116+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_h1v2_neon_8);
117+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_h1v3_neon_8);
118+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_h2v1_neon_8);
119+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_h2v2_neon_8);
120+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_h2v3_neon_8);
121+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_h3v1_neon_8);
122+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_h3v2_neon_8);
123+QPEL_FUNC_UW(ff_hevc_put_qpel_uw_h3v3_neon_8);
124+#undef QPEL_FUNC_UW
125+
126+void ff_hevc_put_qpel_neon_wrapper(int16_t *dst, uint8_t *src, ptrdiff_t srcstride,
127+ int height, intptr_t mx, intptr_t my, int width) {
128+
129+ put_hevc_qpel_neon[my][mx](dst, MAX_PB_SIZE, src, srcstride, height, width);
130+}
131+
132+void ff_hevc_put_qpel_uni_neon_wrapper(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, ptrdiff_t srcstride,
133+ int height, intptr_t mx, intptr_t my, int width) {
134+
135+ put_hevc_qpel_uw_neon[my][mx](dst, dststride, src, srcstride, width, height, NULL, 0);
136+}
137+
138+void ff_hevc_put_qpel_bi_neon_wrapper(uint8_t *dst, ptrdiff_t dststride, uint8_t *src, ptrdiff_t srcstride,
139+ int16_t *src2,
140+ int height, intptr_t mx, intptr_t my, int width) {
141+ put_hevc_qpel_uw_neon[my][mx](dst, dststride, src, srcstride, width, height, src2, MAX_PB_SIZE);
142+}
143+
45144 static av_cold void hevcdsp_init_neon(HEVCDSPContext *c, const int bit_depth)
46145 {
47146 #if HAVE_NEON
48147 if (bit_depth == 8) {
148+ int x;
49149 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_neon;
50150 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_neon;
51151 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_neon;
@@ -61,6 +161,65 @@ static av_cold void hevcdsp_init_neon(HEVCDSPContext *c, const int bit_depth) · 9 unmodified lines
61161 c->transform_add[2] = ff_hevc_transform_add_16x16_neon_8;
62162 c->transform_add[3] = ff_hevc_transform_add_32x32_neon_8;
63163 c->idct_4x4_luma = ff_hevc_transform_luma_4x4_neon_8;
164+ put_hevc_qpel_neon[1][0] = ff_hevc_put_qpel_v1_neon_8;
165+ put_hevc_qpel_neon[2][0] = ff_hevc_put_qpel_v2_neon_8;
166+ put_hevc_qpel_neon[3][0] = ff_hevc_put_qpel_v3_neon_8;
167+ put_hevc_qpel_neon[0][1] = ff_hevc_put_qpel_h1_neon_8;
168+ put_hevc_qpel_neon[0][2] = ff_hevc_put_qpel_h2_neon_8;
169+ put_hevc_qpel_neon[0][3] = ff_hevc_put_qpel_h3_neon_8;
170+ put_hevc_qpel_neon[1][1] = ff_hevc_put_qpel_h1v1_neon_8;
171+ put_hevc_qpel_neon[1][2] = ff_hevc_put_qpel_h2v1_neon_8;
172+ put_hevc_qpel_neon[1][3] = ff_hevc_put_qpel_h3v1_neon_8;
173+ put_hevc_qpel_neon[2][1] = ff_hevc_put_qpel_h1v2_neon_8;
174+ put_hevc_qpel_neon[2][2] = ff_hevc_put_qpel_h2v2_neon_8;
175+ put_hevc_qpel_neon[2][3] = ff_hevc_put_qpel_h3v2_neon_8;
176+ put_hevc_qpel_neon[3][1] = ff_hevc_put_qpel_h1v3_neon_8;
177+ put_hevc_qpel_neon[3][2] = ff_hevc_put_qpel_h2v3_neon_8;
178+ put_hevc_qpel_neon[3][3] = ff_hevc_put_qpel_h3v3_neon_8;
179+ put_hevc_qpel_uw_neon[1][0] = ff_hevc_put_qpel_uw_v1_neon_8;
180+ put_hevc_qpel_uw_neon[2][0] = ff_hevc_put_qpel_uw_v2_neon_8;
181+ put_hevc_qpel_uw_neon[3][0] = ff_hevc_put_qpel_uw_v3_neon_8;
182+ put_hevc_qpel_uw_neon[0][1] = ff_hevc_put_qpel_uw_h1_neon_8;
183+ put_hevc_qpel_uw_neon[0][2] = ff_hevc_put_qpel_uw_h2_neon_8;
184+ put_hevc_qpel_uw_neon[0][3] = ff_hevc_put_qpel_uw_h3_neon_8;
185+ put_hevc_qpel_uw_neon[1][1] = ff_hevc_put_qpel_uw_h1v1_neon_8;
186+ put_hevc_qpel_uw_neon[1][2] = ff_hevc_put_qpel_uw_h2v1_neon_8;
187+ put_hevc_qpel_uw_neon[1][3] = ff_hevc_put_qpel_uw_h3v1_neon_8;
188+ put_hevc_qpel_uw_neon[2][1] = ff_hevc_put_qpel_uw_h1v2_neon_8;
189+ put_hevc_qpel_uw_neon[2][2] = ff_hevc_put_qpel_uw_h2v2_neon_8;
190+ put_hevc_qpel_uw_neon[2][3] = ff_hevc_put_qpel_uw_h3v2_neon_8;
191+ put_hevc_qpel_uw_neon[3][1] = ff_hevc_put_qpel_uw_h1v3_neon_8;
192+ put_hevc_qpel_uw_neon[3][2] = ff_hevc_put_qpel_uw_h2v3_neon_8;
193+ put_hevc_qpel_uw_neon[3][3] = ff_hevc_put_qpel_uw_h3v3_neon_8;
194+ for (x = 0; x < 10; x++) {
195+ c->put_hevc_qpel[x][1][0] = ff_hevc_put_qpel_neon_wrapper;
196+ c->put_hevc_qpel[x][0][1] = ff_hevc_put_qpel_neon_wrapper;
197+ c->put_hevc_qpel[x][1][1] = ff_hevc_put_qpel_neon_wrapper;
198+ c->put_hevc_qpel_uni[x][1][0] = ff_hevc_put_qpel_uni_neon_wrapper;
199+ c->put_hevc_qpel_uni[x][0][1] = ff_hevc_put_qpel_uni_neon_wrapper;
200+ c->put_hevc_qpel_uni[x][1][1] = ff_hevc_put_qpel_uni_neon_wrapper;
201+ c->put_hevc_qpel_bi[x][1][0] = ff_hevc_put_qpel_bi_neon_wrapper;
202+ c->put_hevc_qpel_bi[x][0][1] = ff_hevc_put_qpel_bi_neon_wrapper;
203+ c->put_hevc_qpel_bi[x][1][1] = ff_hevc_put_qpel_bi_neon_wrapper;
204+ }
205+ c->put_hevc_qpel[0][0][0] = ff_hevc_put_pixels_w2_neon_8;
206+ c->put_hevc_qpel[1][0][0] = ff_hevc_put_pixels_w4_neon_8;
207+ c->put_hevc_qpel[2][0][0] = ff_hevc_put_pixels_w6_neon_8;
208+ c->put_hevc_qpel[3][0][0] = ff_hevc_put_pixels_w8_neon_8;
209+ c->put_hevc_qpel[4][0][0] = ff_hevc_put_pixels_w12_neon_8;
210+ c->put_hevc_qpel[5][0][0] = ff_hevc_put_pixels_w16_neon_8;
211+ c->put_hevc_qpel[6][0][0] = ff_hevc_put_pixels_w24_neon_8;
212+ c->put_hevc_qpel[7][0][0] = ff_hevc_put_pixels_w32_neon_8;
213+ c->put_hevc_qpel[8][0][0] = ff_hevc_put_pixels_w48_neon_8;
214+ c->put_hevc_qpel[9][0][0] = ff_hevc_put_pixels_w64_neon_8;
215+
216+ c->put_hevc_qpel_uni[1][0][0] = ff_hevc_put_qpel_uw_pixels_w4_neon_8;
217+ c->put_hevc_qpel_uni[3][0][0] = ff_hevc_put_qpel_uw_pixels_w8_neon_8;
218+ c->put_hevc_qpel_uni[5][0][0] = ff_hevc_put_qpel_uw_pixels_w16_neon_8;
219+ c->put_hevc_qpel_uni[6][0][0] = ff_hevc_put_qpel_uw_pixels_w24_neon_8;
220+ c->put_hevc_qpel_uni[7][0][0] = ff_hevc_put_qpel_uw_pixels_w32_neon_8;
221+ c->put_hevc_qpel_uni[8][0][0] = ff_hevc_put_qpel_uw_pixels_w48_neon_8;
222+ c->put_hevc_qpel_uni[9][0][0] = ff_hevc_put_qpel_uw_pixels_w64_neon_8;
64223 }
65224 #endif // HAVE_NEON
66225 }
libavcodec/arm/hevcdsp_qpel_neon.S+999−0
@@ -0,0 +1,999 @@
1+/*
2+ * Copyright (c) 2014 - 2015 Seppo Tomperi <seppo.tomperi@vtt.fi>
3+ *
4+ * This file is part of FFmpeg.
5+ *
6+ * FFmpeg is free software; you can redistribute it and/or
7+ * modify it under the terms of the GNU Lesser General Public
8+ * License as published by the Free Software Foundation; either
9+ * version 2.1 of the License, or (at your option) any later version.
10+ *
11+ * FFmpeg is distributed in the hope that it will be useful,
12+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
13+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+ * Lesser General Public License for more details.
15+ *
16+ * You should have received a copy of the GNU Lesser General Public
17+ * License along with FFmpeg; if not, write to the Free Software
18+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19+ */
20+
21+#include "libavutil/arm/asm.S"
22+#include "neon.S"
23+
24+#define MAX_PB_SIZE #64
25+
26+.macro regshuffle_d8
27+ vmov d16, d17
28+ vmov d17, d18
29+ vmov d18, d19
30+ vmov d19, d20
31+ vmov d20, d21
32+ vmov d21, d22
33+ vmov d22, d23
34+.endm
35+
36+.macro regshuffle_q8
37+ vmov q0, q1
38+ vmov q1, q2
39+ vmov q2, q3
40+ vmov q3, q4
41+ vmov q4, q5
42+ vmov q5, q6
43+ vmov q6, q7
44+.endm
45+
46+.macro vextin8
47+ pld [r2]
48+ vld1.8 {q11}, [r2], r3
49+ vext.8 d16, d22, d23, #1
50+ vext.8 d17, d22, d23, #2
51+ vext.8 d18, d22, d23, #3
52+ vext.8 d19, d22, d23, #4
53+ vext.8 d20, d22, d23, #5
54+ vext.8 d21, d22, d23, #6
55+ vext.8 d22, d22, d23, #7
56+.endm
57+
58+.macro loadin8
59+ pld [r2]
60+ vld1.8 {d16}, [r2], r3
61+ pld [r2]
62+ vld1.8 {d17}, [r2], r3
63+ pld [r2]
64+ vld1.8 {d18}, [r2], r3
65+ pld [r2]
66+ vld1.8 {d19}, [r2], r3
67+ pld [r2]
68+ vld1.8 {d20}, [r2], r3
69+ pld [r2]
70+ vld1.8 {d21}, [r2], r3
71+ pld [r2]
72+ vld1.8 {d22}, [r2], r3
73+ pld [r2]
74+ vld1.8 {d23}, [r2], r3
75+.endm
76+
77+.macro qpel_filter_1_32b
78+ vmov.i16 d16, #58
79+ vmov.i16 d17, #10
80+ vmull.s16 q9, d6, d16 // 58 * d0
81+ vmull.s16 q10, d7, d16 // 58 * d1
82+ vmov.i16 d16, #17
83+ vmull.s16 q11, d4, d17 // 10 * c0
84+ vmull.s16 q12, d5, d17 // 10 * c1
85+ vmov.i16 d17, #5
86+ vmull.s16 q13, d8, d16 // 17 * e0
87+ vmull.s16 q14, d9, d16 // 17 * e1
88+ vmull.s16 q15, d10, d17 // 5 * f0
89+ vmull.s16 q8, d11, d17 // 5 * f1
90+ vsub.s32 q9, q11 // 58 * d0 - 10 * c0
91+ vsub.s32 q10, q12 // 58 * d1 - 10 * c1
92+ vshll.s16 q11, d2, #2 // 4 * b0
93+ vshll.s16 q12, d3, #2 // 4 * b1
94+ vadd.s32 q9, q13 // 58 * d0 - 10 * c0 + 17 * e0
95+ vadd.s32 q10, q14 // 58 * d1 - 10 * c1 + 17 * e1
96+ vsubl.s16 q13, d12, d0 // g0 - a0
97+ vsubl.s16 q14, d13, d1 // g1 - a1
98+ vadd.s32 q9, q11 // 58 * d0 - 10 * c0 + 17 * e0 + 4 * b0
99+ vadd.s32 q10, q12 // 58 * d1 - 10 * c1 + 17 * e1 + 4 * b1
100+ vsub.s32 q13, q15 // g0 - a0 - 5 * f0
101+ vsub.s32 q14, q8 // g1 - a1 - 5 * f1
102+ vadd.s32 q9, q13 // 58 * d0 - 10 * c0 + 17 * e0 + 4 * b0 + g0 - a0 - 5 * f0
103+ vadd.s32 q10, q14 // 58 * d1 - 10 * c1 + 17 * e1 + 4 * b1 + g1 - a1 - 5 * f1
104+ vqshrn.s32 d16, q9, #6
105+ vqshrn.s32 d17, q10, #6
106+.endm
107+
108+// input q0 - q7
109+// output q8
110+.macro qpel_filter_2_32b
111+ vmov.i32 q8, #11
112+ vaddl.s16 q9, d6, d8 // d0 + e0
113+ vaddl.s16 q10, d7, d9 // d1 + e1
114+ vaddl.s16 q11, d4, d10 // c0 + f0
115+ vaddl.s16 q12, d5, d11 // c1 + f1
116+ vmul.s32 q11, q8 // 11 * (c0 + f0)
117+ vmul.s32 q12, q8 // 11 * (c1 + f1)
118+ vmov.i32 q8, #40
119+ vaddl.s16 q15, d2, d12 // b0 + g0
120+ vmul.s32 q9, q8 // 40 * (d0 + e0)
121+ vmul.s32 q10, q8 // 40 * (d1 + e1)
122+ vaddl.s16 q8, d3, d13 // b1 + g1
123+ vaddl.s16 q13, d0, d14 // a0 + h0
124+ vaddl.s16 q14, d1, d15 // a1 + h1
125+ vshl.s32 q15, #2 // 4*(b0+g0)
126+ vshl.s32 q8, #2 // 4*(b1+g1)
127+ vadd.s32 q11, q13 // 11 * (c0 + f0) + a0 + h0
128+ vadd.s32 q12, q14 // 11 * (c1 + f1) + a1 + h1
129+ vadd.s32 q9, q15 // 40 * (d0 + e0) + 4*(b0+g0)
130+ vadd.s32 q10, q8 // 40 * (d1 + e1) + 4*(b1+g1)
131+ vsub.s32 q9, q11 // 40 * (d0 + e0) + 4*(b0+g0) - (11 * (c0 + f0) + a0 + h0)
132+ vsub.s32 q10, q12 // 40 * (d1 + e1) + 4*(b1+g1) - (11 * (c1 + f1) + a1 + h1)
133+ vqshrn.s32 d16, q9, #6
134+ vqshrn.s32 d17, q10, #6
135+.endm
136+
137+.macro qpel_filter_3_32b
138+ vmov.i16 d16, #58
139+ vmov.i16 d17, #10
140+ vmull.s16 q9, d8, d16 // 58 * d0
141+ vmull.s16 q10, d9, d16 // 58 * d1
142+ vmov.i16 d16, #17
143+ vmull.s16 q11, d10, d17 // 10 * c0
144+ vmull.s16 q12, d11, d17 // 10 * c1
145+ vmov.i16 d17, #5
146+ vmull.s16 q13, d6, d16 // 17 * e0
147+ vmull.s16 q14, d7, d16 // 17 * e1
148+ vmull.s16 q15, d4, d17 // 5 * f0
149+ vmull.s16 q8, d5, d17 // 5 * f1
150+ vsub.s32 q9, q11 // 58 * d0 - 10 * c0
151+ vsub.s32 q10, q12 // 58 * d1 - 10 * c1
152+ vshll.s16 q11, d12, #2 // 4 * b0
153+ vshll.s16 q12, d13, #2 // 4 * b1
154+ vadd.s32 q9, q13 // 58 * d0 - 10 * c0 + 17 * e0
155+ vadd.s32 q10, q14 // 58 * d1 - 10 * c1 + 17 * e1
156+ vsubl.s16 q13, d2, d14 // g0 - a0
157+ vsubl.s16 q14, d3, d15 // g1 - a1
158+ vadd.s32 q9, q11 // 58 * d0 - 10 * c0 + 17 * e0 + 4 * b0
159+ vadd.s32 q10, q12 // 58 * d1 - 10 * c1 + 17 * e1 + 4 * b1
160+ vsub.s32 q13, q15 // g0 - a0 - 5 * f0
161+ vsub.s32 q14, q8 // g1 - a1 - 5 * f1
162+ vadd.s32 q9, q13 // 58 * d0 - 10 * c0 + 17 * e0 + 4 * b0 + g0 - a0 - 5 * f0
163+ vadd.s32 q10, q14 // 58 * d1 - 10 * c1 + 17 * e1 + 4 * b1 + g1 - a1 - 5 * f1
164+ vqshrn.s32 d16, q9, #6
165+ vqshrn.s32 d17, q10, #6
166+.endm
167+
168+.macro qpel_filter_1 out=q7
169+ vmov.u8 d24, #58
170+ vmov.u8 d25, #10
171+ vshll.u8 q13, d20, #4 // 16*e
172+ vshll.u8 q14, d21, #2 // 4*f
173+ vmull.u8 \out, d19, d24 // 58*d
174+ vaddw.u8 q13, q13, d20 // 17*e
175+ vmull.u8 q15, d18, d25 // 10*c
176+ vaddw.u8 q14, q14, d21 // 5*f
177+ vsubl.u8 q12, d22, d16 // g - a
178+ vadd.u16 \out, q13 // 58d + 17e
179+ vshll.u8 q13, d17, #2 // 4*b
180+ vadd.u16 q15, q14 // 10*c + 5*f
181+ vadd.s16 q13, q12 // - a + 4*b + g
182+ vsub.s16 \out, q15 // -10*c + 58*d + 17*e -5*f
183+ vadd.s16 \out, q13 // -a + 4*b -10*c + 58*d + 17*e -5*f
184+.endm
185+
186+.macro qpel_filter_2 out=q7
187+ vmov.i16 q12, #10
188+ vmov.i16 q14, #11
189+ vaddl.u8 q13, d19, d20 // d + e
190+ vaddl.u8 q15, d18, d21 // c + f
191+ vmul.u16 q13, q12 // 10 * (d+e)
192+ vmul.u16 q15, q14 // 11 * ( c + f)
193+ vaddl.u8 \out, d17, d22 // b + g
194+ vaddl.u8 q12, d16, d23 // a + h
195+ vadd.u16 \out, q13 // b + 10 * (d + e) + g
196+ vadd.s16 q12, q15
197+ vshl.u16 \out, #2 // 4 * (b + 10 * (d + e) + g)
198+ vsub.s16 \out, q12
199+.endm
200+
201+.macro qpel_filter_3 out=q7
202+ vmov.u8 d24, #58
203+ vmov.u8 d25, #10
204+ vshll.u8 q13, d19, #4 // 16*e
205+ vshll.u8 q14, d18, #2 // 4*f
206+ vmull.u8 \out, d20, d24 // 58*d
207+ vaddw.u8 q13, q13, d19 // 17*e
208+ vmull.u8 q15, d21, d25 // 10*c
209+ vaddw.u8 q14, q14, d18 // 5*f
210+ vsubl.u8 q12, d17, d23 // g - a
211+ vadd.u16 \out, q13 // 58d + 17e
212+ vshll.u8 q13, d22, #2 // 4*b
213+ vadd.u16 q15, q14 // 10*c + 5*f
214+ vadd.s16 q13, q12 // - a + 4*b + g
215+ vsub.s16 \out, q15 // -10*c + 58*d + 17*e -5*f
216+ vadd.s16 \out, q13 // -a + 4*b -10*c + 58*d + 17*e -5*f
217+.endm
218+
219+.macro hevc_put_qpel_vX_neon_8 filter
220+ push {r4, r5, r6, r7}
221+ ldr r4, [sp, #16] // height
222+ ldr r5, [sp, #20] // width
223+ vpush {d8-d15}
224+ sub r2, r2, r3, lsl #1
225+ sub r2, r3
226+ mov r12, r4
227+ mov r6, r0
228+ mov r7, r2
229+ lsl r1, #1
230+0: loadin8
231+ cmp r5, #4
232+ beq 4f
233+8: subs r4, #1
234+ \filter
235+ vst1.16 {q7}, [r0], r1
236+ regshuffle_d8
237+ vld1.8 {d23}, [r2], r3
238+ bne 8b
239+ subs r5, #8
240+ beq 99f
241+ mov r4, r12
242+ add r6, #16
243+ mov r0, r6
244+ add r7, #8
245+ mov r2, r7
246+ b 0b
247+4: subs r4, #1
248+ \filter
249+ vst1.16 d14, [r0], r1
250+ regshuffle_d8
251+ vld1.32 {d23[0]}, [r2], r3
252+ bne 4b
253+99: vpop {d8-d15}
254+ pop {r4, r5, r6, r7}
255+ bx lr
256+.endm
257+
258+.macro hevc_put_qpel_uw_vX_neon_8 filter
259+ push {r4-r10}
260+ ldr r5, [sp, #28] // width
261+ ldr r4, [sp, #32] // height
262+ ldr r8, [sp, #36] // src2
263+ ldr r9, [sp, #40] // src2stride
264+ vpush {d8-d15}
265+ sub r2, r2, r3, lsl #1
266+ sub r2, r3
267+ mov r12, r4
268+ mov r6, r0
269+ mov r7, r2
270+ cmp r8, #0
271+ bne .Lbi\@
272+0: loadin8
273+ cmp r5, #4
274+ beq 4f
275+8: subs r4, #1
276+ \filter
277+ vqrshrun.s16 d0, q7, #6
278+ vst1.8 d0, [r0], r1
279+ regshuffle_d8
280+ vld1.8 {d23}, [r2], r3
281+ bne 8b
282+ subs r5, #8
283+ beq 99f
284+ mov r4, r12
285+ add r6, #8
286+ mov r0, r6
287+ add r7, #8
288+ mov r2, r7
289+ b 0b
290+4: subs r4, #1
291+ \filter
292+ vqrshrun.s16 d0, q7, #6
293+ vst1.32 d0[0], [r0], r1
294+ regshuffle_d8
295+ vld1.32 {d23[0]}, [r2], r3
296+ bne 4b
297+ b 99f
298+.Lbi\@: lsl r9, #1
299+ mov r10, r8
300+0: loadin8
301+ cmp r5, #4
302+ beq 4f
303+8: subs r4, #1
304+ \filter
305+ vld1.16 {q0}, [r8], r9
306+ vqadd.s16 q0, q7
307+ vqrshrun.s16 d0, q0, #7
308+ vst1.8 d0, [r0], r1
309+ regshuffle_d8
310+ vld1.8 {d23}, [r2], r3
311+ bne 8b
312+ subs r5, #8
313+ beq 99f
314+ mov r4, r12
315+ add r6, #8
316+ mov r0, r6
317+ add r10, #16
318+ mov r8, r10
319+ add r7, #8
320+ mov r2, r7
321+ b 0b
322+4: subs r4, #1
323+ \filter
324+ vld1.16 d0, [r8], r9
325+ vqadd.s16 d0, d14
326+ vqrshrun.s16 d0, q0, #7
327+ vst1.32 d0[0], [r0], r1
328+ regshuffle_d8
329+ vld1.32 {d23[0]}, [r2], r3
330+ bne 4b
331+99: vpop {d8-d15}
332+ pop {r4-r10}
333+ bx lr
334+.endm
335+
336+function ff_hevc_put_qpel_v1_neon_8, export=1
337+ hevc_put_qpel_vX_neon_8 qpel_filter_1
338+endfunc
339+
340+function ff_hevc_put_qpel_v2_neon_8, export=1
341+ hevc_put_qpel_vX_neon_8 qpel_filter_2
342+endfunc
343+
344+function ff_hevc_put_qpel_v3_neon_8, export=1
345+ hevc_put_qpel_vX_neon_8 qpel_filter_3
346+endfunc
347+
348+
349+function ff_hevc_put_qpel_uw_v1_neon_8, export=1
350+ hevc_put_qpel_uw_vX_neon_8 qpel_filter_1
351+endfunc
352+
353+function ff_hevc_put_qpel_uw_v2_neon_8, export=1
354+ hevc_put_qpel_uw_vX_neon_8 qpel_filter_2
355+endfunc
356+
357+function ff_hevc_put_qpel_uw_v3_neon_8, export=1
358+ hevc_put_qpel_uw_vX_neon_8 qpel_filter_3
359+endfunc
360+
361+.macro hevc_put_qpel_hX_neon_8 filter
362+ push {r4, r5, r6, r7}
363+ ldr r4, [sp, #16] // height
364+ ldr r5, [sp, #20] // width
365+
366+ vpush {d8-d15}
367+ sub r2, #4
368+ lsl r1, #1
369+ mov r12, r4
370+ mov r6, r0
371+ mov r7, r2
372+ cmp r5, #4
373+ beq 4f
374+8: subs r4, #1
375+ vextin8
376+ \filter
377+ vst1.16 {q7}, [r0], r1
378+ bne 8b
379+ subs r5, #8
380+ beq 99f
381+ mov r4, r12
382+ add r6, #16
383+ mov r0, r6
384+ add r7, #8
385+ mov r2, r7
386+ cmp r5, #4
387+ bne 8b
388+4: subs r4, #1
389+ vextin8
390+ \filter
391+ vst1.16 d14, [r0], r1
392+ bne 4b
393+99: vpop {d8-d15}
394+ pop {r4, r5, r6, r7}
395+ bx lr
396+.endm
397+
398+.macro hevc_put_qpel_uw_hX_neon_8 filter
399+ push {r4-r10}
400+ ldr r5, [sp, #28] // width
401+ ldr r4, [sp, #32] // height
402+ ldr r8, [sp, #36] // src2
403+ ldr r9, [sp, #40] // src2stride
404+ vpush {d8-d15}
405+ sub r2, #4
406+ mov r12, r4
407+ mov r6, r0
408+ mov r7, r2
409+ cmp r8, #0
410+ bne .Lbi\@
411+ cmp r5, #4
412+ beq 4f
413+8: subs r4, #1
414+ vextin8
415+ \filter
416+ vqrshrun.s16 d0, q7, #6
417+ vst1.8 d0, [r0], r1
418+ bne 8b
419+ subs r5, #8
420+ beq 99f
421+ mov r4, r12
422+ add r6, #8
423+ mov r0, r6
424+ add r7, #8
425+ mov r2, r7
426+ cmp r5, #4
427+ bne 8b
428+4: subs r4, #1
429+ vextin8
430+ \filter
431+ vqrshrun.s16 d0, q7, #6
432+ vst1.32 d0[0], [r0], r1
433+ bne 4b
434+ b 99f
435+.Lbi\@:
436+ lsl r9, #1
437+ cmp r5, #4
438+ beq 4f
439+ mov r10, r8
440+8: subs r4, #1
441+ vextin8
442+ \filter
443+ vld1.16 {q0}, [r8], r9
444+ vqadd.s16 q0, q7
445+ vqrshrun.s16 d0, q0, #7
446+ vst1.8 d0, [r0], r1
447+ bne 8b
448+ subs r5, #8
449+ beq 99f
450+ mov r4, r12
451+ add r6, #8
452+ add r10, #16
453+ mov r8, r10
454+ mov r0, r6
455+ add r7, #8
456+ mov r2, r7
457+ cmp r5, #4
458+ bne 8b
459+4: subs r4, #1
460+ vextin8
461+ \filter
462+ vld1.16 d0, [r8], r9
463+ vqadd.s16 d0, d14
464+ vqrshrun.s16 d0, q0, #7
465+ vst1.32 d0[0], [r0], r1
466+ bne 4b
467+99: vpop {d8-d15}
468+ pop {r4-r10}
469+ bx lr
470+.endm
471+
472+function ff_hevc_put_qpel_h1_neon_8, export=1
473+ hevc_put_qpel_hX_neon_8 qpel_filter_1
474+endfunc
475+
476+function ff_hevc_put_qpel_h2_neon_8, export=1
477+ hevc_put_qpel_hX_neon_8 qpel_filter_2
478+endfunc
479+
480+function ff_hevc_put_qpel_h3_neon_8, export=1
481+ hevc_put_qpel_hX_neon_8 qpel_filter_3
482+endfunc
483+
484+
485+function ff_hevc_put_qpel_uw_h1_neon_8, export=1
486+ hevc_put_qpel_uw_hX_neon_8 qpel_filter_1
487+endfunc
488+
489+function ff_hevc_put_qpel_uw_h2_neon_8, export=1
490+ hevc_put_qpel_uw_hX_neon_8 qpel_filter_2
491+endfunc
492+
493+function ff_hevc_put_qpel_uw_h3_neon_8, export=1
494+ hevc_put_qpel_uw_hX_neon_8 qpel_filter_3
495+endfunc
496+
497+.macro hevc_put_qpel_hXvY_neon_8 filterh filterv
498+ push {r4, r5, r6, r7}
499+ ldr r4, [sp, #16] // height
500+ ldr r5, [sp, #20] // width
501+
502+ vpush {d8-d15}
503+ sub r2, #4
504+ sub r2, r2, r3, lsl #1
505+ sub r2, r3 // extra_before 3
506+ lsl r1, #1
507+ mov r12, r4
508+ mov r6, r0
509+ mov r7, r2
510+0: vextin8
511+ \filterh q0
512+ vextin8
513+ \filterh q1
514+ vextin8
515+ \filterh q2
516+ vextin8
517+ \filterh q3
518+ vextin8
519+ \filterh q4
520+ vextin8
521+ \filterh q5
522+ vextin8
523+ \filterh q6
524+ vextin8
525+ \filterh q7
526+ cmp r5, #4
527+ beq 4f
528+8: subs r4, #1
529+ \filterv
530+ vst1.16 {q8}, [r0], r1
531+ regshuffle_q8
532+ vextin8
533+ \filterh q7
534+ bne 8b
535+ subs r5, #8
536+ beq 99f
537+ mov r4, r12
538+ add r6, #16
539+ mov r0, r6
540+ add r7, #8
541+ mov r2, r7
542+ b 0b
543+4: subs r4, #1
544+ \filterv
545+ vst1.16 d16, [r0], r1
546+ regshuffle_q8
547+ vextin8
548+ \filterh q7
549+ bne 4b
550+99: vpop {d8-d15}
551+ pop {r4, r5, r6, r7}
552+ bx lr
553+.endm
554+
555+.macro hevc_put_qpel_uw_hXvY_neon_8 filterh filterv
556+ push {r4-r10}
557+ ldr r5, [sp, #28] // width
558+ ldr r4, [sp, #32] // height
559+ ldr r8, [sp, #36] // src2
560+ ldr r9, [sp, #40] // src2stride
561+ vpush {d8-d15}
562+ sub r2, #4
563+ sub r2, r2, r3, lsl #1
564+ sub r2, r3 // extra_before 3
565+ mov r12, r4
566+ mov r6, r0
567+ mov r7, r2
568+ cmp r8, #0
569+ bne .Lbi\@
570+0: vextin8
571+ \filterh q0
572+ vextin8
573+ \filterh q1
574+ vextin8
575+ \filterh q2
576+ vextin8
577+ \filterh q3
578+ vextin8
579+ \filterh q4
580+ vextin8
581+ \filterh q5
582+ vextin8
583+ \filterh q6
584+ vextin8
585+ \filterh q7
586+ cmp r5, #4
587+ beq 4f
588+8: subs r4, #1
589+ \filterv
590+ vqrshrun.s16 d0, q8, #6
591+ vst1.8 d0, [r0], r1
592+ regshuffle_q8
593+ vextin8
594+ \filterh q7
595+ bne 8b
596+ subs r5, #8
597+ beq 99f
598+ mov r4, r12
599+ add r6, #8
600+ mov r0, r6
601+ add r7, #8
602+ mov r2, r7
603+ b 0b
604+4: subs r4, #1
605+ \filterv
606+ vqrshrun.s16 d0, q8, #6
607+ vst1.32 d0[0], [r0], r1
608+ regshuffle_q8
609+ vextin8
610+ \filterh q7
611+ bne 4b
612+ b 99f
613+.Lbi\@: lsl r9, #1
614+ mov r10, r8
615+0: vextin8
616+ \filterh q0
617+ vextin8
618+ \filterh q1
619+ vextin8
620+ \filterh q2
621+ vextin8
622+ \filterh q3
623+ vextin8
624+ \filterh q4
625+ vextin8
626+ \filterh q5
627+ vextin8
628+ \filterh q6
629+ vextin8
630+ \filterh q7
631+ cmp r5, #4
632+ beq 4f
633+8: subs r4, #1
634+ \filterv
635+ vld1.16 {q0}, [r8], r9
636+ vqadd.s16 q0, q8
637+ vqrshrun.s16 d0, q0, #7
638+ vst1.8 d0, [r0], r1
639+ regshuffle_q8
640+ vextin8
641+ \filterh q7
642+ bne 8b
643+ subs r5, #8
644+ beq 99f
645+ mov r4, r12
646+ add r6, #8
647+ mov r0, r6
648+ add r10, #16
649+ mov r8, r10
650+ add r7, #8
651+ mov r2, r7
652+ b 0b
653+4: subs r4, #1
654+ \filterv
655+ vld1.16 d0, [r8], r9
656+ vqadd.s16 d0, d16
657+ vqrshrun.s16 d0, q0, #7
658+ vst1.32 d0[0], [r0], r1
659+ regshuffle_q8
660+ vextin8
661+ \filterh q7
662+ bne 4b
663+99: vpop {d8-d15}
664+ pop {r4-r10}
665+ bx lr
666+.endm
667+
668+
669+function ff_hevc_put_qpel_h1v1_neon_8, export=1
670+ hevc_put_qpel_hXvY_neon_8 qpel_filter_1 qpel_filter_1_32b
671+endfunc
672+
673+function ff_hevc_put_qpel_h2v1_neon_8, export=1
674+ hevc_put_qpel_hXvY_neon_8 qpel_filter_2 qpel_filter_1_32b
675+endfunc
676+
677+function ff_hevc_put_qpel_h3v1_neon_8, export=1
678+ hevc_put_qpel_hXvY_neon_8 qpel_filter_3 qpel_filter_1_32b
679+endfunc
680+
681+function ff_hevc_put_qpel_h1v2_neon_8, export=1
682+ hevc_put_qpel_hXvY_neon_8 qpel_filter_1 qpel_filter_2_32b
683+endfunc
684+
685+function ff_hevc_put_qpel_h2v2_neon_8, export=1
686+ hevc_put_qpel_hXvY_neon_8 qpel_filter_2 qpel_filter_2_32b
687+endfunc
688+
689+function ff_hevc_put_qpel_h3v2_neon_8, export=1
690+ hevc_put_qpel_hXvY_neon_8 qpel_filter_3 qpel_filter_2_32b
691+endfunc
692+
693+function ff_hevc_put_qpel_h1v3_neon_8, export=1
694+ hevc_put_qpel_hXvY_neon_8 qpel_filter_1 qpel_filter_3_32b
695+endfunc
696+
697+function ff_hevc_put_qpel_h2v3_neon_8, export=1
698+ hevc_put_qpel_hXvY_neon_8 qpel_filter_2 qpel_filter_3_32b
699+endfunc
700+
701+function ff_hevc_put_qpel_h3v3_neon_8, export=1
702+ hevc_put_qpel_hXvY_neon_8 qpel_filter_3 qpel_filter_3_32b
703+endfunc
704+
705+
706+function ff_hevc_put_qpel_uw_h1v1_neon_8, export=1
707+ hevc_put_qpel_uw_hXvY_neon_8 qpel_filter_1 qpel_filter_1_32b
708+endfunc
709+
710+function ff_hevc_put_qpel_uw_h2v1_neon_8, export=1
711+ hevc_put_qpel_uw_hXvY_neon_8 qpel_filter_2 qpel_filter_1_32b
712+endfunc
713+
714+function ff_hevc_put_qpel_uw_h3v1_neon_8, export=1
715+ hevc_put_qpel_uw_hXvY_neon_8 qpel_filter_3 qpel_filter_1_32b
716+endfunc
717+
718+function ff_hevc_put_qpel_uw_h1v2_neon_8, export=1
719+ hevc_put_qpel_uw_hXvY_neon_8 qpel_filter_1 qpel_filter_2_32b
720+endfunc
721+
722+function ff_hevc_put_qpel_uw_h2v2_neon_8, export=1
723+ hevc_put_qpel_uw_hXvY_neon_8 qpel_filter_2 qpel_filter_2_32b
724+endfunc
725+
726+function ff_hevc_put_qpel_uw_h3v2_neon_8, export=1
727+ hevc_put_qpel_uw_hXvY_neon_8 qpel_filter_3 qpel_filter_2_32b
728+endfunc
729+
730+function ff_hevc_put_qpel_uw_h1v3_neon_8, export=1
731+ hevc_put_qpel_uw_hXvY_neon_8 qpel_filter_1 qpel_filter_3_32b
732+endfunc
733+
734+function ff_hevc_put_qpel_uw_h2v3_neon_8, export=1
735+ hevc_put_qpel_uw_hXvY_neon_8 qpel_filter_2 qpel_filter_3_32b
736+endfunc
737+
738+function ff_hevc_put_qpel_uw_h3v3_neon_8, export=1
739+ hevc_put_qpel_uw_hXvY_neon_8 qpel_filter_3 qpel_filter_3_32b
740+endfunc
741+
742+.macro init_put_pixels
743+ pld [r1]
744+ pld [r1, r2]
745+ mov r12, MAX_PB_SIZE
746+ lsl r12, #1
747+.endm
748+
749+function ff_hevc_put_pixels_w2_neon_8, export=1
750+ init_put_pixels
751+ vmov.u8 d5, #255
752+ vshr.u64 d5, #32
753+0: subs r3, #1
754+ vld1.32 {d0[0]}, [r1], r2
755+ pld [r1]
756+ vld1.32 d6, [r0]
757+ vshll.u8 q0, d0, #6
758+ vbit d6, d0, d5
759+ vst1.32 d6, [r0], r12
760+ bne 0b
761+ bx lr
762+endfunc
763+
764+function ff_hevc_put_pixels_w4_neon_8, export=1
765+ init_put_pixels
766+0: subs r3, #2
767+ vld1.32 {d0[0]}, [r1], r2
768+ vld1.32 {d0[1]}, [r1], r2
769+ pld [r1]
770+ pld [r1, r2]
771+ vshll.u8 q0, d0, #6
772+ vst1.64 {d0}, [r0], r12
773+ vst1.64 {d1}, [r0], r12
774+ bne 0b
775+ bx lr
776+endfunc
777+
778+function ff_hevc_put_pixels_w6_neon_8, export=1
779+ init_put_pixels
780+ vmov.u8 q10, #255
781+ vshr.u64 d21, #32
782+0: subs r3, #1
783+ vld1.16 {d0}, [r1], r2
784+ pld [r1]
785+ vshll.u8 q0, d0, #6
786+ vld1.8 {q12}, [r0]
787+ vbit q12, q0, q10
788+ vst1.8 {q12}, [r0], r12
789+ bne 0b
790+ bx lr
791+endfunc
792+
793+function ff_hevc_put_pixels_w8_neon_8, export=1
794+ init_put_pixels
795+0: subs r3, #2
796+ vld1.8 {d0}, [r1], r2
797+ vld1.8 {d2}, [r1], r2
798+ pld [r1]
799+ pld [r1, r2]
800+ vshll.u8 q0, d0, #6
801+ vshll.u8 q1, d2, #6
802+ vst1.16 {q0}, [r0], r12
803+ vst1.16 {q1}, [r0], r12
804+ bne 0b
805+ bx lr
806+endfunc
807+
808+function ff_hevc_put_pixels_w12_neon_8, export=1
809+ init_put_pixels
810+0: subs r3, #2
811+ vld1.64 {d0}, [r1]
812+ add r1, #8
813+ vld1.32 {d1[0]}, [r1], r2
814+ sub r1, #8
815+ vld1.64 {d2}, [r1]
816+ add r1, #8
817+ vld1.32 {d1[1]}, [r1], r2
818+ sub r1, #8
819+ pld [r1]
820+ pld [r1, r2]
821+ vshll.u8 q8, d0, #6
822+ vshll.u8 q9, d1, #6
823+ vshll.u8 q10, d2, #6
824+ vmov d22, d19
825+ vst1.64 {d16, d17, d18}, [r0], r12
826+ vst1.64 {d20, d21, d22}, [r0], r12
827+ bne 0b
828+ bx lr
829+endfunc
830+
831+function ff_hevc_put_pixels_w16_neon_8, export=1
832+ init_put_pixels
833+0: subs r3, #2
834+ vld1.8 {q0}, [r1], r2
835+ vld1.8 {q1}, [r1], r2
836+ pld [r1]
837+ pld [r1, r2]
838+ vshll.u8 q8, d0, #6
839+ vshll.u8 q9, d1, #6
840+ vshll.u8 q10, d2, #6
841+ vshll.u8 q11, d3, #6
842+ vst1.8 {q8, q9}, [r0], r12
843+ vst1.8 {q10, q11}, [r0], r12
844+ bne 0b
845+ bx lr
846+endfunc
847+
848+function ff_hevc_put_pixels_w24_neon_8, export=1
849+ init_put_pixels
850+0: subs r3, #1
851+ vld1.8 {d0, d1, d2}, [r1], r2
852+ pld [r1]
853+ vshll.u8 q10, d0, #6
854+ vshll.u8 q11, d1, #6
855+ vshll.u8 q12, d2, #6
856+ vstm r0, {q10, q11, q12}
857+ add r0, r12
858+ bne 0b
859+ bx lr
860+endfunc
861+
862+function ff_hevc_put_pixels_w32_neon_8, export=1
863+ init_put_pixels
864+0: subs r3, #1
865+ vld1.8 {q0, q1}, [r1], r2
866+ pld [r1]
867+ vshll.u8 q8, d0, #6
868+ vshll.u8 q9, d1, #6
869+ vshll.u8 q10, d2, #6
870+ vshll.u8 q11, d3, #6
871+ vstm r0, {q8, q9, q10, q11}
872+ add r0, r12
873+ bne 0b
874+ bx lr
875+endfunc
876+
877+function ff_hevc_put_pixels_w48_neon_8, export=1
878+ init_put_pixels
879+0: subs r3, #1
880+ vld1.8 {q0, q1}, [r1]
881+ add r1, #32
882+ vld1.8 {q2}, [r1], r2
883+ sub r1, #32
884+ pld [r1]
885+ vshll.u8 q8, d0, #6
886+ vshll.u8 q9, d1, #6
887+ vshll.u8 q10, d2, #6
888+ vshll.u8 q11, d3, #6
889+ vshll.u8 q12, d4, #6
890+ vshll.u8 q13, d5, #6
891+ vstm r0, {q8, q9, q10, q11, q12, q13}
892+ add r0, r12
893+ bne 0b
894+ bx lr
895+endfunc
896+
897+function ff_hevc_put_pixels_w64_neon_8, export=1
898+ init_put_pixels
899+0: subs r3, #1
900+ vld1.8 {q0, q1}, [r1]
901+ add r1, #32
902+ vld1.8 {q2, q3}, [r1], r2
903+ sub r1, #32
904+ pld [r1]
905+ vshll.u8 q8, d0, #6
906+ vshll.u8 q9, d1, #6
907+ vshll.u8 q10, d2, #6
908+ vshll.u8 q11, d3, #6
909+ vshll.u8 q12, d4, #6
910+ vshll.u8 q13, d5, #6
911+ vshll.u8 q14, d6, #6
912+ vshll.u8 q15, d7, #6
913+ vstm r0, {q8, q9, q10, q11, q12, q13, q14, q15}
914+ add r0, r12
915+ bne 0b
916+ bx lr
917+endfunc
918+
919+function ff_hevc_put_qpel_uw_pixels_neon_8, export=1
920+ push {r4-r9}
921+ ldr r5, [sp, #24] // width
922+ ldr r4, [sp, #28] // height
923+ ldr r8, [sp, #32] // src2
924+ ldr r9, [sp, #36] // src2stride
925+ vpush {d8-d15}
926+ cmp r8, #0
927+ bne 2f
928+1: subs r4, #1
929+ vld1.8 {d0}, [r2], r3
930+ vst1.8 d0, [r0], r1
931+ bne 1b
932+ vpop {d8-d15}
933+ pop {r4-r9}
934+ bx lr
935+2: subs r4, #1
936+ vld1.8 {d0}, [r2], r3
937+ vld1.16 {q1}, [r8], r9
938+ vshll.u8 q0, d0, #6
939+ vqadd.s16 q0, q1
940+ vqrshrun.s16 d0, q0, #7
941+ vst1.8 d0, [r0], r1
942+ bne 2b
943+ vpop {d8-d15}
944+ pop {r4-r9}
945+ bx lr
946+endfunc
947+
948+.macro put_qpel_uw_pixels width, regs, regs2, regs3, regs4
949+function ff_hevc_put_qpel_uw_pixels_w\width\()_neon_8, export=1
950+ ldr r12, [sp] // height
951+1: subs r12, #4
952+ vld1.32 {\regs} , [r2], r3
953+ vld1.32 {\regs2} , [r2], r3
954+ vld1.32 {\regs3} , [r2], r3
955+ vld1.32 {\regs4} , [r2], r3
956+ vst1.32 {\regs} , [r0], r1
957+ vst1.32 {\regs2} , [r0], r1
958+ vst1.32 {\regs3} , [r0], r1
959+ vst1.32 {\regs4} , [r0], r1
960+ bne 1b
961+ bx lr
962+endfunc
963+.endm
964+
965+.macro put_qpel_uw_pixels_m width, regs, regs2, regs3, regs4
966+function ff_hevc_put_qpel_uw_pixels_w\width\()_neon_8, export=1
967+ push {r4-r5}
968+ ldr r12, [sp, #8] // height
969+1: subs r12, #2
970+ mov r4, r2
971+ vld1.32 {\regs} , [r2]!
972+ vld1.32 {\regs2} , [r2]
973+ add r2, r4, r3
974+ mov r4, r2
975+ vld1.32 {\regs3} , [r2]!
976+ vld1.32 {\regs4} , [r2]
977+ add r2, r4, r3
978+ mov r5, r0
979+ vst1.32 {\regs} , [r0]!
980+ vst1.32 {\regs2} , [r0]
981+ add r0, r5, r1
982+ mov r5, r0
983+ vst1.32 {\regs3} , [r0]!
984+ vst1.32 {\regs4} , [r0]
985+ add r0, r5, r1
986+ bne 1b
987+ pop {r4-r5}
988+ bx lr
989+endfunc
990+.endm
991+
992+put_qpel_uw_pixels 4, d0[0], d0[1], d1[0], d1[1]
993+put_qpel_uw_pixels 8, d0, d1, d2, d3
994+put_qpel_uw_pixels_m 12, d0, d1[0], d2, d3[0]
995+put_qpel_uw_pixels 16, q0, q1, q2, q3
996+put_qpel_uw_pixels 24, d0-d2, d3-d5, d16-d18, d19-d21
997+put_qpel_uw_pixels 32, q0-q1, q2-q3, q8-q9, q10-q11
998+put_qpel_uw_pixels_m 48, q0-q1, q2, q8-q9, q10
999+put_qpel_uw_pixels_m 64, q0-q1, q2-q3, q8-q9, q10-q11

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t