| @@ -0,0 +1,368 @@ |
| | 1 | /* |
| | 2 | * Armv8 Neon optimizations for libjpeg-turbo |
| | 3 | * |
| | 4 | * Copyright (C) 2009-2011, Nokia Corporation and/or its subsidiary(-ies). |
| | 5 | * All Rights Reserved. |
| | 6 | * Author: Siarhei Siamashka <siarhei.siamashka@nokia.com> |
| | 7 | * Copyright (C) 2013-2014, Linaro Limited. All Rights Reserved. |
| | 8 | * Author: Ragesh Radhakrishnan <ragesh.r@linaro.org> |
| | 9 | * Copyright (C) 2014-2016, 2020, D. R. Commander. All Rights Reserved. |
| | 10 | * Copyright (C) 2015-2016, 2018, Matthieu Darbois. All Rights Reserved. |
| | 11 | * Copyright (C) 2016, Siarhei Siamashka. All Rights Reserved. |
| | 12 | * |
| | 13 | * This software is provided 'as-is', without any express or implied |
| | 14 | * warranty. In no event will the authors be held liable for any damages |
| | 15 | * arising from the use of this software. |
| | 16 | * |
| | 17 | * Permission is granted to anyone to use this software for any purpose, |
| | 18 | * including commercial applications, and to alter it and redistribute it |
| | 19 | * freely, subject to the following restrictions: |
| | 20 | * |
| | 21 | * 1. The origin of this software must not be misrepresented; you must not |
| | 22 | * claim that you wrote the original software. If you use this software |
| | 23 | * in a product, an acknowledgment in the product documentation would be |
| | 24 | * appreciated but is not required. |
| | 25 | * 2. Altered source versions must be plainly marked as such, and must not be |
| | 26 | * misrepresented as being the original software. |
| | 27 | * 3. This notice may not be removed or altered from any source distribution. |
| | 28 | */ |
| | 29 | |
| | 30 | #include "libavutil/aarch64/asm.S" |
| | 31 | #include "neon.S" |
| | 32 | |
| | 33 | // #define EIGHT_BIT_SAMPLES |
| | 34 | |
| | 35 | /* Constants for jsimd_fdct_islow_neon() */ |
| | 36 | |
| | 37 | #define F_0_298 2446 /* FIX(0.298631336) */ |
| | 38 | #define F_0_390 3196 /* FIX(0.390180644) */ |
| | 39 | #define F_0_541 4433 /* FIX(0.541196100) */ |
| | 40 | #define F_0_765 6270 /* FIX(0.765366865) */ |
| | 41 | #define F_0_899 7373 /* FIX(0.899976223) */ |
| | 42 | #define F_1_175 9633 /* FIX(1.175875602) */ |
| | 43 | #define F_1_501 12299 /* FIX(1.501321110) */ |
| | 44 | #define F_1_847 15137 /* FIX(1.847759065) */ |
| | 45 | #define F_1_961 16069 /* FIX(1.961570560) */ |
| | 46 | #define F_2_053 16819 /* FIX(2.053119869) */ |
| | 47 | #define F_2_562 20995 /* FIX(2.562915447) */ |
| | 48 | #define F_3_072 25172 /* FIX(3.072711026) */ |
| | 49 | |
| | 50 | const jsimd_fdct_islow_neon_consts, align=4 |
| | 51 | .short F_0_298 |
| | 52 | .short -F_0_390 |
| | 53 | .short F_0_541 |
| | 54 | .short F_0_765 |
| | 55 | .short - F_0_899 |
| | 56 | .short F_1_175 |
| | 57 | .short F_1_501 |
| | 58 | .short - F_1_847 |
| | 59 | .short - F_1_961 |
| | 60 | .short F_2_053 |
| | 61 | .short - F_2_562 |
| | 62 | .short F_3_072 |
| | 63 | .short 0 /* padding */ |
| | 64 | .short 0 |
| | 65 | .short 0 |
| | 66 | .short 0 |
| | 67 | endconst |
| | 68 | |
| | 69 | #undef F_0_298 |
| | 70 | #undef F_0_390 |
| | 71 | #undef F_0_541 |
| | 72 | #undef F_0_765 |
| | 73 | #undef F_0_899 |
| | 74 | #undef F_1_175 |
| | 75 | #undef F_1_501 |
| | 76 | #undef F_1_847 |
| | 77 | #undef F_1_961 |
| | 78 | #undef F_2_053 |
| | 79 | #undef F_2_562 |
| | 80 | #undef F_3_072 |
| | 81 | |
| | 82 | /*****************************************************************************/ |
| | 83 | |
| | 84 | /* |
| | 85 | * jsimd_fdct_islow_neon |
| | 86 | * |
| | 87 | * This file contains a slower but more accurate integer implementation of the |
| | 88 | * forward DCT (Discrete Cosine Transform). The following code is based |
| | 89 | * directly on the IJG''s original jfdctint.c; see the jfdctint.c for |
| | 90 | * more details. |
| | 91 | */ |
| | 92 | |
| | 93 | #define CONST_BITS 13 |
| | 94 | #ifdef EIGHT_BIT_SAMPLES |
| | 95 | #define PASS1_BITS 2 |
| | 96 | #else |
| | 97 | #define PASS1_BITS 1 /* lose a little precision to avoid overflow */ |
| | 98 | #endif |
| | 99 | |
| | 100 | #define DESCALE_P1 (CONST_BITS - PASS1_BITS) |
| | 101 | #define DESCALE_P2 (CONST_BITS + PASS1_BITS) |
| | 102 | |
| | 103 | #define XFIX_P_0_298 v0.h[0] |
| | 104 | #define XFIX_N_0_390 v0.h[1] |
| | 105 | #define XFIX_P_0_541 v0.h[2] |
| | 106 | #define XFIX_P_0_765 v0.h[3] |
| | 107 | #define XFIX_N_0_899 v0.h[4] |
| | 108 | #define XFIX_P_1_175 v0.h[5] |
| | 109 | #define XFIX_P_1_501 v0.h[6] |
| | 110 | #define XFIX_N_1_847 v0.h[7] |
| | 111 | #define XFIX_N_1_961 v1.h[0] |
| | 112 | #define XFIX_P_2_053 v1.h[1] |
| | 113 | #define XFIX_N_2_562 v1.h[2] |
| | 114 | #define XFIX_P_3_072 v1.h[3] |
| | 115 | |
| | 116 | function ff_fdct_neon, export=1 |
| | 117 | |
| | 118 | DATA .req x0 |
| | 119 | TMP .req x9 |
| | 120 | |
| | 121 | /* Load constants */ |
| | 122 | movrel TMP, jsimd_fdct_islow_neon_consts |
| | 123 | ld1 {v0.8h, v1.8h}, [TMP] |
| | 124 | |
| | 125 | /* Load all DATA into Neon registers with the following allocation: |
| | 126 | * 0 1 2 3 | 4 5 6 7 |
| | 127 | * ---------+-------- |
| | 128 | * 0 | d16 | d17 | v16.8h |
| | 129 | * 1 | d18 | d19 | v17.8h |
| | 130 | * 2 | d20 | d21 | v18.8h |
| | 131 | * 3 | d22 | d23 | v19.8h |
| | 132 | * 4 | d24 | d25 | v20.8h |
| | 133 | * 5 | d26 | d27 | v21.8h |
| | 134 | * 6 | d28 | d29 | v22.8h |
| | 135 | * 7 | d30 | d31 | v23.8h |
| | 136 | */ |
| | 137 | |
| | 138 | ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [DATA], 64 |
| | 139 | ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [DATA] |
| | 140 | sub DATA, DATA, #64 |
| | 141 | |
| | 142 | /* Transpose */ |
| | 143 | transpose_8x8H v16, v17, v18, v19, v20, v21, v22, v23, v31, v2 |
| | 144 | |
| | 145 | /* 1-D FDCT */ |
| | 146 | add v24.8h, v16.8h, v23.8h /* tmp0 = dataptr[0] + dataptr[7]; */ |
| | 147 | sub v31.8h, v16.8h, v23.8h /* tmp7 = dataptr[0] - dataptr[7]; */ |
| | 148 | add v25.8h, v17.8h, v22.8h /* tmp1 = dataptr[1] + dataptr[6]; */ |
| | 149 | sub v30.8h, v17.8h, v22.8h /* tmp6 = dataptr[1] - dataptr[6]; */ |
| | 150 | add v26.8h, v18.8h, v21.8h /* tmp2 = dataptr[2] + dataptr[5]; */ |
| | 151 | sub v29.8h, v18.8h, v21.8h /* tmp5 = dataptr[2] - dataptr[5]; */ |
| | 152 | add v27.8h, v19.8h, v20.8h /* tmp3 = dataptr[3] + dataptr[4]; */ |
| | 153 | sub v28.8h, v19.8h, v20.8h /* tmp4 = dataptr[3] - dataptr[4]; */ |
| | 154 | |
| | 155 | /* Even part */ |
| | 156 | add v4.8h, v24.8h, v27.8h /* tmp10 = tmp0 + tmp3; */ |
| | 157 | sub v5.8h, v24.8h, v27.8h /* tmp13 = tmp0 - tmp3; */ |
| | 158 | add v6.8h, v25.8h, v26.8h /* tmp11 = tmp1 + tmp2; */ |
| | 159 | sub v7.8h, v25.8h, v26.8h /* tmp12 = tmp1 - tmp2; */ |
| | 160 | |
| | 161 | add v16.8h, v4.8h, v6.8h /* tmp10 + tmp11 */ |
| | 162 | sub v20.8h, v4.8h, v6.8h /* tmp10 - tmp11 */ |
| | 163 | |
| | 164 | add v18.8h, v7.8h, v5.8h /* tmp12 + tmp13 */ |
| | 165 | |
| | 166 | shl v16.8h, v16.8h, #PASS1_BITS /* dataptr[0] = (DCTELEM)LEFT_SHIFT(tmp10 + tmp11, PASS1_BITS); */ |
| | 167 | shl v20.8h, v20.8h, #PASS1_BITS /* dataptr[4] = (DCTELEM)LEFT_SHIFT(tmp10 - tmp11, PASS1_BITS); */ |
| | 168 | |
| | 169 | smull2 v24.4s, v18.8h, XFIX_P_0_541 /* z1 hi = MULTIPLY(tmp12 + tmp13, XFIX_P_0_541); */ |
| | 170 | smull v18.4s, v18.4h, XFIX_P_0_541 /* z1 lo = MULTIPLY(tmp12 + tmp13, XFIX_P_0_541); */ |
| | 171 | mov v22.16b, v18.16b |
| | 172 | mov v25.16b, v24.16b |
| | 173 | |
| | 174 | smlal v18.4s, v5.4h, XFIX_P_0_765 /* lo z1 + MULTIPLY(tmp13, XFIX_P_0_765) */ |
| | 175 | smlal2 v24.4s, v5.8h, XFIX_P_0_765 /* hi z1 + MULTIPLY(tmp13, XFIX_P_0_765) */ |
| | 176 | smlal v22.4s, v7.4h, XFIX_N_1_847 /* lo z1 + MULTIPLY(tmp12, XFIX_N_1_847) */ |
| | 177 | smlal2 v25.4s, v7.8h, XFIX_N_1_847 /* hi z1 + MULTIPLY(tmp12, XFIX_N_1_847) */ |
| | 178 | |
| | 179 | rshrn v18.4h, v18.4s, #DESCALE_P1 |
| | 180 | rshrn v22.4h, v22.4s, #DESCALE_P1 |
| | 181 | rshrn2 v18.8h, v24.4s, #DESCALE_P1 /* dataptr[2] = (DCTELEM)DESCALE(z1 + MULTIPLY(tmp13, XFIX_P_0_765), CONST_BITS-PASS1_BITS); */ |
| | 182 | rshrn2 v22.8h, v25.4s, #DESCALE_P1 /* dataptr[6] = (DCTELEM)DESCALE(z1 + MULTIPLY(tmp12, XFIX_N_1_847), CONST_BITS-PASS1_BITS); */ |
| | 183 | |
| | 184 | /* Odd part */ |
| | 185 | add v2.8h, v28.8h, v31.8h /* z1 = tmp4 + tmp7; */ |
| | 186 | add v3.8h, v29.8h, v30.8h /* z2 = tmp5 + tmp6; */ |
| | 187 | add v6.8h, v28.8h, v30.8h /* z3 = tmp4 + tmp6; */ |
| | 188 | add v7.8h, v29.8h, v31.8h /* z4 = tmp5 + tmp7; */ |
| | 189 | smull v4.4s, v6.4h, XFIX_P_1_175 /* z5 lo = z3 lo * XFIX_P_1_175 */ |
| | 190 | smull2 v5.4s, v6.8h, XFIX_P_1_175 |
| | 191 | smlal v4.4s, v7.4h, XFIX_P_1_175 /* z5 = MULTIPLY(z3 + z4, FIX_1_175875602); */ |
| | 192 | smlal2 v5.4s, v7.8h, XFIX_P_1_175 |
| | 193 | |
| | 194 | smull2 v24.4s, v28.8h, XFIX_P_0_298 |
| | 195 | smull2 v25.4s, v29.8h, XFIX_P_2_053 |
| | 196 | smull2 v26.4s, v30.8h, XFIX_P_3_072 |
| | 197 | smull2 v27.4s, v31.8h, XFIX_P_1_501 |
| | 198 | smull v23.4s, v28.4h, XFIX_P_0_298 /* tmp4 = MULTIPLY(tmp4, FIX_0_298631336); */ |
| | 199 | smull v21.4s, v29.4h, XFIX_P_2_053 /* tmp5 = MULTIPLY(tmp5, FIX_2_053119869); */ |
| | 200 | smull v19.4s, v30.4h, XFIX_P_3_072 /* tmp6 = MULTIPLY(tmp6, FIX_3_072711026); */ |
| | 201 | smull v17.4s, v31.4h, XFIX_P_1_501 /* tmp7 = MULTIPLY(tmp7, FIX_1_501321110); */ |
| | 202 | |
| | 203 | smull2 v28.4s, v2.8h, XFIX_N_0_899 |
| | 204 | smull2 v29.4s, v3.8h, XFIX_N_2_562 |
| | 205 | smull2 v30.4s, v6.8h, XFIX_N_1_961 |
| | 206 | smull2 v31.4s, v7.8h, XFIX_N_0_390 |
| | 207 | smull v2.4s, v2.4h, XFIX_N_0_899 /* z1 = MULTIPLY(z1, -FIX_0_899976223); */ |
| | 208 | smull v3.4s, v3.4h, XFIX_N_2_562 /* z2 = MULTIPLY(z2, -FIX_2_562915447); */ |
| | 209 | smull v6.4s, v6.4h, XFIX_N_1_961 /* z3 = MULTIPLY(z3, -FIX_1_961570560); */ |
| | 210 | smull v7.4s, v7.4h, XFIX_N_0_390 /* z4 = MULTIPLY(z4, -FIX_0_390180644); */ |
| | 211 | |
| | 212 | add v6.4s, v6.4s, v4.4s /* z3 += z5 */ |
| | 213 | add v30.4s, v30.4s, v5.4s |
| | 214 | add v7.4s, v7.4s, v4.4s /* z4 += z5 */ |
| | 215 | add v31.4s, v31.4s, v5.4s |
| | 216 | |
| | 217 | add v23.4s, v23.4s, v2.4s /* tmp4 += z1 */ |
| | 218 | add v24.4s, v24.4s, v28.4s |
| | 219 | add v21.4s, v21.4s, v3.4s /* tmp5 += z2 */ |
| | 220 | add v25.4s, v25.4s, v29.4s |
| | 221 | add v19.4s, v19.4s, v6.4s /* tmp6 += z3 */ |
| | 222 | add v26.4s, v26.4s, v30.4s |
| | 223 | add v17.4s, v17.4s, v7.4s /* tmp7 += z4 */ |
| | 224 | add v27.4s, v27.4s, v31.4s |
| | 225 | |
| | 226 | add v23.4s, v23.4s, v6.4s /* tmp4 += z3 */ |
| | 227 | add v24.4s, v24.4s, v30.4s |
| | 228 | add v21.4s, v21.4s, v7.4s /* tmp5 += z4 */ |
| | 229 | add v25.4s, v25.4s, v31.4s |
| | 230 | add v19.4s, v19.4s, v3.4s /* tmp6 += z2 */ |
| | 231 | add v26.4s, v26.4s, v29.4s |
| | 232 | add v17.4s, v17.4s, v2.4s /* tmp7 += z1 */ |
| | 233 | add v27.4s, v27.4s, v28.4s |
| | 234 | |
| | 235 | rshrn v23.4h, v23.4s, #DESCALE_P1 |
| | 236 | rshrn v21.4h, v21.4s, #DESCALE_P1 |
| | 237 | rshrn v19.4h, v19.4s, #DESCALE_P1 |
| | 238 | rshrn v17.4h, v17.4s, #DESCALE_P1 |
| | 239 | rshrn2 v23.8h, v24.4s, #DESCALE_P1 /* dataptr[7] = (DCTELEM)DESCALE(tmp4 + z1 + z3, CONST_BITS-PASS1_BITS); */ |
| | 240 | rshrn2 v21.8h, v25.4s, #DESCALE_P1 /* dataptr[5] = (DCTELEM)DESCALE(tmp5 + z2 + z4, CONST_BITS-PASS1_BITS); */ |
| | 241 | rshrn2 v19.8h, v26.4s, #DESCALE_P1 /* dataptr[3] = (DCTELEM)DESCALE(tmp6 + z2 + z3, CONST_BITS-PASS1_BITS); */ |
| | 242 | rshrn2 v17.8h, v27.4s, #DESCALE_P1 /* dataptr[1] = (DCTELEM)DESCALE(tmp7 + z1 + z4, CONST_BITS-PASS1_BITS); */ |
| | 243 | |
| | 244 | /* Transpose */ |
| | 245 | transpose_8x8H v16, v17, v18, v19, v20, v21, v22, v23, v31, v2 |
| | 246 | |
| | 247 | /* 1-D FDCT */ |
| | 248 | add v24.8h, v16.8h, v23.8h /* tmp0 = dataptr[0] + dataptr[7]; */ |
| | 249 | sub v31.8h, v16.8h, v23.8h /* tmp7 = dataptr[0] - dataptr[7]; */ |
| | 250 | add v25.8h, v17.8h, v22.8h /* tmp1 = dataptr[1] + dataptr[6]; */ |
| | 251 | sub v30.8h, v17.8h, v22.8h /* tmp6 = dataptr[1] - dataptr[6]; */ |
| | 252 | add v26.8h, v18.8h, v21.8h /* tmp2 = dataptr[2] + dataptr[5]; */ |
| | 253 | sub v29.8h, v18.8h, v21.8h /* tmp5 = dataptr[2] - dataptr[5]; */ |
| | 254 | add v27.8h, v19.8h, v20.8h /* tmp3 = dataptr[3] + dataptr[4]; */ |
| | 255 | sub v28.8h, v19.8h, v20.8h /* tmp4 = dataptr[3] - dataptr[4]; */ |
| | 256 | |
| | 257 | /* Even part */ |
| | 258 | add v4.8h, v24.8h, v27.8h /* tmp10 = tmp0 + tmp3; */ |
| | 259 | sub v5.8h, v24.8h, v27.8h /* tmp13 = tmp0 - tmp3; */ |
| | 260 | add v6.8h, v25.8h, v26.8h /* tmp11 = tmp1 + tmp2; */ |
| | 261 | sub v7.8h, v25.8h, v26.8h /* tmp12 = tmp1 - tmp2; */ |
| | 262 | |
| | 263 | add v16.8h, v4.8h, v6.8h /* tmp10 + tmp11 */ |
| | 264 | sub v20.8h, v4.8h, v6.8h /* tmp10 - tmp11 */ |
| | 265 | |
| | 266 | add v18.8h, v7.8h, v5.8h /* tmp12 + tmp13 */ |
| | 267 | |
| | 268 | srshr v16.8h, v16.8h, #PASS1_BITS /* dataptr[0] = (DCTELEM)DESCALE(tmp10 + tmp11, PASS1_BITS); */ |
| | 269 | srshr v20.8h, v20.8h, #PASS1_BITS /* dataptr[4] = (DCTELEM)DESCALE(tmp10 - tmp11, PASS1_BITS); */ |
| | 270 | |
| | 271 | smull2 v24.4s, v18.8h, XFIX_P_0_541 /* z1 hi = MULTIPLY(tmp12 + tmp13, XFIX_P_0_541); */ |
| | 272 | smull v18.4s, v18.4h, XFIX_P_0_541 /* z1 lo = MULTIPLY(tmp12 + tmp13, XFIX_P_0_541); */ |
| | 273 | mov v22.16b, v18.16b |
| | 274 | mov v25.16b, v24.16b |
| | 275 | |
| | 276 | smlal v18.4s, v5.4h, XFIX_P_0_765 /* lo z1 + MULTIPLY(tmp13, XFIX_P_0_765) */ |
| | 277 | smlal2 v24.4s, v5.8h, XFIX_P_0_765 /* hi z1 + MULTIPLY(tmp13, XFIX_P_0_765) */ |
| | 278 | smlal v22.4s, v7.4h, XFIX_N_1_847 /* lo z1 + MULTIPLY(tmp12, XFIX_N_1_847) */ |
| | 279 | smlal2 v25.4s, v7.8h, XFIX_N_1_847 /* hi z1 + MULTIPLY(tmp12, XFIX_N_1_847) */ |
| | 280 | |
| | 281 | rshrn v18.4h, v18.4s, #DESCALE_P2 |
| | 282 | rshrn v22.4h, v22.4s, #DESCALE_P2 |
| | 283 | rshrn2 v18.8h, v24.4s, #DESCALE_P2 /* dataptr[2] = (DCTELEM)DESCALE(z1 + MULTIPLY(tmp13, XFIX_P_0_765), CONST_BITS+PASS1_BITS); */ |
| | 284 | rshrn2 v22.8h, v25.4s, #DESCALE_P2 /* dataptr[6] = (DCTELEM)DESCALE(z1 + MULTIPLY(tmp12, XFIX_N_1_847), CONST_BITS+PASS1_BITS); */ |
| | 285 | |
| | 286 | /* Odd part */ |
| | 287 | add v2.8h, v28.8h, v31.8h /* z1 = tmp4 + tmp7; */ |
| | 288 | add v3.8h, v29.8h, v30.8h /* z2 = tmp5 + tmp6; */ |
| | 289 | add v6.8h, v28.8h, v30.8h /* z3 = tmp4 + tmp6; */ |
| | 290 | add v7.8h, v29.8h, v31.8h /* z4 = tmp5 + tmp7; */ |
| | 291 | |
| | 292 | smull v4.4s, v6.4h, XFIX_P_1_175 /* z5 lo = z3 lo * XFIX_P_1_175 */ |
| | 293 | smull2 v5.4s, v6.8h, XFIX_P_1_175 |
| | 294 | smlal v4.4s, v7.4h, XFIX_P_1_175 /* z5 = MULTIPLY(z3 + z4, FIX_1_175875602); */ |
| | 295 | smlal2 v5.4s, v7.8h, XFIX_P_1_175 |
| | 296 | |
| | 297 | smull2 v24.4s, v28.8h, XFIX_P_0_298 |
| | 298 | smull2 v25.4s, v29.8h, XFIX_P_2_053 |
| | 299 | smull2 v26.4s, v30.8h, XFIX_P_3_072 |
| | 300 | smull2 v27.4s, v31.8h, XFIX_P_1_501 |
| | 301 | smull v23.4s, v28.4h, XFIX_P_0_298 /* tmp4 = MULTIPLY(tmp4, FIX_0_298631336); */ |
| | 302 | smull v21.4s, v29.4h, XFIX_P_2_053 /* tmp5 = MULTIPLY(tmp5, FIX_2_053119869); */ |
| | 303 | smull v19.4s, v30.4h, XFIX_P_3_072 /* tmp6 = MULTIPLY(tmp6, FIX_3_072711026); */ |
| | 304 | smull v17.4s, v31.4h, XFIX_P_1_501 /* tmp7 = MULTIPLY(tmp7, FIX_1_501321110); */ |
| | 305 | |
| | 306 | smull2 v28.4s, v2.8h, XFIX_N_0_899 |
| | 307 | smull2 v29.4s, v3.8h, XFIX_N_2_562 |
| | 308 | smull2 v30.4s, v6.8h, XFIX_N_1_961 |
| | 309 | smull2 v31.4s, v7.8h, XFIX_N_0_390 |
| | 310 | smull v2.4s, v2.4h, XFIX_N_0_899 /* z1 = MULTIPLY(z1, -FIX_0_899976223); */ |
| | 311 | smull v3.4s, v3.4h, XFIX_N_2_562 /* z2 = MULTIPLY(z2, -FIX_2_562915447); */ |
| | 312 | smull v6.4s, v6.4h, XFIX_N_1_961 /* z3 = MULTIPLY(z3, -FIX_1_961570560); */ |
| | 313 | smull v7.4s, v7.4h, XFIX_N_0_390 /* z4 = MULTIPLY(z4, -FIX_0_390180644); */ |
| | 314 | |
| | 315 | add v6.4s, v6.4s, v4.4s /* z3 += z5 */ |
| | 316 | add v30.4s, v30.4s, v5.4s |
| | 317 | add v7.4s, v7.4s, v4.4s /* z4 += z5 */ |
| | 318 | add v31.4s, v31.4s, v5.4s |
| | 319 | |
| | 320 | add v23.4s, v23.4s, v2.4s /* tmp4 += z1 */ |
| | 321 | add v24.4s, v24.4s, v28.4s |
| | 322 | add v21.4s, v21.4s, v3.4s /* tmp5 += z2 */ |
| | 323 | add v25.4s, v25.4s, v29.4s |
| | 324 | add v19.4s, v19.4s, v6.4s /* tmp6 += z3 */ |
| | 325 | add v26.4s, v26.4s, v30.4s |
| | 326 | add v17.4s, v17.4s, v7.4s /* tmp7 += z4 */ |
| | 327 | add v27.4s, v27.4s, v31.4s |
| | 328 | |
| | 329 | add v23.4s, v23.4s, v6.4s /* tmp4 += z3 */ |
| | 330 | add v24.4s, v24.4s, v30.4s |
| | 331 | add v21.4s, v21.4s, v7.4s /* tmp5 += z4 */ |
| | 332 | add v25.4s, v25.4s, v31.4s |
| | 333 | add v19.4s, v19.4s, v3.4s /* tmp6 += z2 */ |
| | 334 | add v26.4s, v26.4s, v29.4s |
| | 335 | add v17.4s, v17.4s, v2.4s /* tmp7 += z1 */ |
| | 336 | add v27.4s, v27.4s, v28.4s |
| | 337 | |
| | 338 | rshrn v23.4h, v23.4s, #DESCALE_P2 |
| | 339 | rshrn v21.4h, v21.4s, #DESCALE_P2 |
| | 340 | rshrn v19.4h, v19.4s, #DESCALE_P2 |
| | 341 | rshrn v17.4h, v17.4s, #DESCALE_P2 |
| | 342 | rshrn2 v23.8h, v24.4s, #DESCALE_P2 /* dataptr[7] = (DCTELEM)DESCALE(tmp4 + z1 + z3, CONST_BITS+PASS1_BITS); */ |
| | 343 | rshrn2 v21.8h, v25.4s, #DESCALE_P2 /* dataptr[5] = (DCTELEM)DESCALE(tmp5 + z2 + z4, CONST_BITS+PASS1_BITS); */ |
| | 344 | rshrn2 v19.8h, v26.4s, #DESCALE_P2 /* dataptr[3] = (DCTELEM)DESCALE(tmp6 + z2 + z3, CONST_BITS+PASS1_BITS); */ |
| | 345 | rshrn2 v17.8h, v27.4s, #DESCALE_P2 /* dataptr[1] = (DCTELEM)DESCALE(tmp7 + z1 + z4, CONST_BITS+PASS1_BITS); */ |
| | 346 | |
| | 347 | /* Store results */ |
| | 348 | st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [DATA], 64 |
| | 349 | st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [DATA] |
| | 350 | |
| | 351 | ret |
| | 352 | |
| | 353 | .unreq DATA |
| | 354 | .unreq TMP |
| | 355 | endfunc |
| | 356 | |
| | 357 | #undef XFIX_P_0_298 |
| | 358 | #undef XFIX_N_0_390 |
| | 359 | #undef XFIX_P_0_541 |
| | 360 | #undef XFIX_P_0_765 |
| | 361 | #undef XFIX_N_0_899 |
| | 362 | #undef XFIX_P_1_175 |
| | 363 | #undef XFIX_P_1_501 |
| | 364 | #undef XFIX_N_1_847 |
| | 365 | #undef XFIX_N_1_961 |
| | 366 | #undef XFIX_P_2_053 |
| | 367 | #undef XFIX_N_2_562 |
| | 368 | #undef XFIX_P_3_072 |