aarch64 A55: chrRangeFromJpeg16_1920_c: 32684.2 chrRangeFromJpeg16_1920_neon: 8431.2 (3.88x) chrRangeToJpeg16_1920_c: 24996.8 chrRangeToJpeg16_1920_neon: 9395.0 (2.66x) lumRangeFromJpeg16_1920_c: 17305.2 lumRangeFromJpeg16_1920_neon: 4586.5 (3.77x) lumRangeToJpeg16_1920_c: 21144.8 lumRangeToJpeg16_1920_neon: 5069.8 (4.17x) aarch64 A76: chrRangeFromJpeg16_1920_c: 11523.8 chrRangeFromJpeg16_1920_neon: 3367.5 (3.42x) chrRangeToJpeg16_1920_c: 11655.2 chrRangeToJpeg16_1920_neon: 4087.2 (2.85x) lumRangeFromJpeg16_1920_c: 5762.0 lumRangeFromJpeg16_1920_neon: 1815.8 (3.17x) lumRangeToJpeg16_1920_c: 5946.2 lumRangeToJpeg16_1920_neon: 2148.2 (2.77x)
Unified · Split
| @@ -20,12 +20,42 @@ · 19 unmodified lines | |||
| 20 | 20 | ||
| 21 | #include "libavutil/aarch64/asm.S" | 21 | #include "libavutil/aarch64/asm.S" |
| 22 | 22 | ||
| 23 | .macro lumConvertRange fromto | 23 | .macro lumConvertRange fromto, bit_depth |
| 24 | function ff_lumRange\fromto\()Jpeg_neon, export=1 | 24 | function ff_lumRange\fromto\()Jpeg\bit_depth\()_neon, export=1 |
| 25 | // x0 int16_t *dst | 25 | // x0 int16_t *dst |
| 26 | // w1 int width | 26 | // w1 int width |
| 27 | // w2 uint32_t coeff | 27 | // w2 uint32_t coeff |
| 28 | // x3 int64_t offset | 28 | // x3 int64_t offset |
| 29 | .if \bit_depth == 16 | ||
| 30 | .ifc \fromto, To | ||
| 31 | movi v25.4s, #1 | ||
| 32 | movi v24.4s, #1<<3, lsl #16 | ||
| 33 | sub v24.4s, v24.4s, v25.4s | ||
| 34 | .endif | ||
| 35 | dup v25.4s, w2 | ||
| 36 | dup v26.2d, x3 | ||
| 37 | 1: | ||
| 38 | ld1 {v0.4s, v1.4s}, [x0] | ||
| 39 | mov v16.16b, v26.16b | ||
| 40 | mov v17.16b, v26.16b | ||
| 41 | mov v18.16b, v26.16b | ||
| 42 | mov v19.16b, v26.16b | ||
| 43 | smlal v16.2d, v0.2s, v25.2s | ||
| 44 | smlal2 v17.2d, v0.4s, v25.4s | ||
| 45 | smlal v18.2d, v1.2s, v25.2s | ||
| 46 | smlal2 v19.2d, v1.4s, v25.4s | ||
| 47 | shrn v0.2s, v16.2d, 18 | ||
| 48 | shrn2 v0.4s, v17.2d, 18 | ||
| 49 | shrn v1.2s, v18.2d, 18 | ||
| 50 | shrn2 v1.4s, v19.2d, 18 | ||
| 51 | subs w1, w1, #8 | ||
| 52 | .ifc \fromto, To | ||
| 53 | smin v0.4s, v0.4s, v24.4s | ||
| 54 | smin v1.4s, v1.4s, v24.4s | ||
| 55 | .endif | ||
| 56 | st1 {v0.4s, v1.4s}, [x0], #32 | ||
| 57 | b.gt 1b | ||
| 58 | .else | ||
| 29 | dup v25.4s, w2 | 59 | dup v25.4s, w2 |
| 30 | dup v26.4s, w3 | 60 | dup v26.4s, w3 |
| 31 | 1: | 61 | 1: |
| @@ -46,17 +76,64 @@ function ff_lumRange\fromto\()Jpeg_neon, export=1 · 14 unmodified lines | |||
| 46 | subs w1, w1, #8 | 76 | subs w1, w1, #8 |
| 47 | st1 {v0.8h}, [x0], #16 | 77 | st1 {v0.8h}, [x0], #16 |
| 48 | b.gt 1b | 78 | b.gt 1b |
| 79 | .endif | ||
| 49 | ret | 80 | ret |
| 50 | endfunc | 81 | endfunc |
| 51 | .endm | 82 | .endm |
| 52 | 83 | ||
| 53 | .macro chrConvertRange fromto | 84 | .macro chrConvertRange fromto, bit_depth |
| 54 | function ff_chrRange\fromto\()Jpeg_neon, export=1 | 85 | function ff_chrRange\fromto\()Jpeg\bit_depth\()_neon, export=1 |
| 55 | // x0 int16_t *dstU | 86 | // x0 int16_t *dstU |
| 56 | // x1 int16_t *dstV | 87 | // x1 int16_t *dstV |
| 57 | // w2 int width | 88 | // w2 int width |
| 58 | // w3 uint32_t coeff | 89 | // w3 uint32_t coeff |
| 59 | // x4 int64_t offset | 90 | // x4 int64_t offset |
| 91 | .if \bit_depth == 16 | ||
| 92 | .ifc \fromto, To | ||
| 93 | movi v25.4s, #1 | ||
| 94 | movi v24.4s, #1<<3, lsl #16 | ||
| 95 | sub v24.4s, v24.4s, v25.4s | ||
| 96 | .endif | ||
| 97 | dup v25.4s, w3 | ||
| 98 | dup v26.2d, x4 | ||
| 99 | 1: | ||
| 100 | ld1 {v0.4s, v1.4s}, [x0] | ||
| 101 | ld1 {v2.4s, v3.4s}, [x1] | ||
| 102 | mov v16.16b, v26.16b | ||
| 103 | mov v17.16b, v26.16b | ||
| 104 | mov v18.16b, v26.16b | ||
| 105 | mov v19.16b, v26.16b | ||
| 106 | mov v20.16b, v26.16b | ||
| 107 | mov v21.16b, v26.16b | ||
| 108 | mov v22.16b, v26.16b | ||
| 109 | mov v23.16b, v26.16b | ||
| 110 | smlal v16.2d, v0.2s, v25.2s | ||
| 111 | smlal2 v17.2d, v0.4s, v25.4s | ||
| 112 | smlal v18.2d, v1.2s, v25.2s | ||
| 113 | smlal2 v19.2d, v1.4s, v25.4s | ||
| 114 | smlal v20.2d, v2.2s, v25.2s | ||
| 115 | smlal2 v21.2d, v2.4s, v25.4s | ||
| 116 | smlal v22.2d, v3.2s, v25.2s | ||
| 117 | smlal2 v23.2d, v3.4s, v25.4s | ||
| 118 | shrn v0.2s, v16.2d, 18 | ||
| 119 | shrn2 v0.4s, v17.2d, 18 | ||
| 120 | shrn v1.2s, v18.2d, 18 | ||
| 121 | shrn2 v1.4s, v19.2d, 18 | ||
| 122 | shrn v2.2s, v20.2d, 18 | ||
| 123 | shrn2 v2.4s, v21.2d, 18 | ||
| 124 | shrn v3.2s, v22.2d, 18 | ||
| 125 | shrn2 v3.4s, v23.2d, 18 | ||
| 126 | subs w2, w2, #8 | ||
| 127 | .ifc \fromto, To | ||
| 128 | smin v0.4s, v0.4s, v24.4s | ||
| 129 | smin v1.4s, v1.4s, v24.4s | ||
| 130 | smin v2.4s, v2.4s, v24.4s | ||
| 131 | smin v3.4s, v3.4s, v24.4s | ||
| 132 | .endif | ||
| 133 | st1 {v0.4s, v1.4s}, [x0], #32 | ||
| 134 | st1 {v2.4s, v3.4s}, [x1], #32 | ||
| 135 | b.gt 1b | ||
| 136 | .else | ||
| 60 | dup v25.4s, w3 | 137 | dup v25.4s, w3 |
| 61 | dup v26.4s, w4 | 138 | dup v26.4s, w4 |
| 62 | 1: | 139 | 1: |
| @@ -89,11 +166,16 @@ function ff_chrRange\fromto\()Jpeg_neon, export=1 · 26 unmodified lines | |||
| 89 | st1 {v0.8h}, [x0], #16 | 166 | st1 {v0.8h}, [x0], #16 |
| 90 | st1 {v1.8h}, [x1], #16 | 167 | st1 {v1.8h}, [x1], #16 |
| 91 | b.gt 1b | 168 | b.gt 1b |
| 169 | .endif | ||
| 92 | ret | 170 | ret |
| 93 | endfunc | 171 | endfunc |
| 94 | .endm | 172 | .endm |
| 95 | 173 | ||
| 96 | lumConvertRange To | 174 | lumConvertRange To, 8 |
| 97 | chrConvertRange To | 175 | lumConvertRange To, 16 |
| 98 | lumConvertRange From | 176 | chrConvertRange To, 8 |
| 99 | chrConvertRange From | 177 | chrConvertRange To, 16 |
| 178 | lumConvertRange From, 8 | ||
| 179 | lumConvertRange From, 16 | ||
| 180 | chrConvertRange From, 8 | ||
| 181 | chrConvertRange From, 16 | ||
| @@ -218,14 +218,22 @@ NEON_INPUT(bgra32); · 217 unmodified lines | |||
| 218 | NEON_INPUT(rgb24); | 218 | NEON_INPUT(rgb24); |
| 219 | NEON_INPUT(rgba32); | 219 | NEON_INPUT(rgba32); |
| 220 | 220 | ||
| 221 | void ff_lumRangeFromJpeg_neon(int16_t *dst, int width, | 221 | void ff_lumRangeFromJpeg8_neon(int16_t *dst, int width, |
| 222 | uint32_t coeff, int64_t offset); | ||
| 223 | void ff_chrRangeFromJpeg8_neon(int16_t *dstU, int16_t *dstV, int width, | ||
| 224 | uint32_t coeff, int64_t offset); | ||
| 225 | void ff_lumRangeToJpeg8_neon(int16_t *dst, int width, | ||
| 226 | uint32_t coeff, int64_t offset); | ||
| 227 | void ff_chrRangeToJpeg8_neon(int16_t *dstU, int16_t *dstV, int width, | ||
| 228 | uint32_t coeff, int64_t offset); | ||
| 229 | void ff_lumRangeFromJpeg16_neon(int16_t *dst, int width, | ||
| 230 | uint32_t coeff, int64_t offset); | ||
| 231 | void ff_chrRangeFromJpeg16_neon(int16_t *dstU, int16_t *dstV, int width, | ||
| 232 | uint32_t coeff, int64_t offset); | ||
| 233 | void ff_lumRangeToJpeg16_neon(int16_t *dst, int width, | ||
| 222 | uint32_t coeff, int64_t offset); | 234 | uint32_t coeff, int64_t offset); |
| 223 | void ff_chrRangeFromJpeg_neon(int16_t *dstU, int16_t *dstV, int width, | 235 | void ff_chrRangeToJpeg16_neon(int16_t *dstU, int16_t *dstV, int width, |
| 224 | uint32_t coeff, int64_t offset); | 236 | uint32_t coeff, int64_t offset); |
| 225 | void ff_lumRangeToJpeg_neon(int16_t *dst, int width, | ||
| 226 | uint32_t coeff, int64_t offset); | ||
| 227 | void ff_chrRangeToJpeg_neon(int16_t *dstU, int16_t *dstV, int width, | ||
| 228 | uint32_t coeff, int64_t offset); | ||
| 229 | 237 | ||
| 230 | av_cold void ff_sws_init_range_convert_aarch64(SwsInternal *c) | 238 | av_cold void ff_sws_init_range_convert_aarch64(SwsInternal *c) |
| 231 | { | 239 | { |
| @@ -234,11 +242,19 @@ av_cold void ff_sws_init_range_convert_aarch64(SwsInternal *c) · 2 unmodified lines | |||
| 234 | if (have_neon(cpu_flags)) { | 242 | if (have_neon(cpu_flags)) { |
| 235 | if (c->dstBpc <= 14) { | 243 | if (c->dstBpc <= 14) { |
| 236 | if (c->opts.src_range) { | 244 | if (c->opts.src_range) { |
| 237 | c->lumConvertRange = ff_lumRangeFromJpeg_neon; | 245 | c->lumConvertRange = ff_lumRangeFromJpeg8_neon; |
| 238 | c->chrConvertRange = ff_chrRangeFromJpeg_neon; | 246 | c->chrConvertRange = ff_chrRangeFromJpeg8_neon; |
| 239 | } else { | 247 | } else { |
| 240 | c->lumConvertRange = ff_lumRangeToJpeg_neon; | 248 | c->lumConvertRange = ff_lumRangeToJpeg8_neon; |
| 241 | c->chrConvertRange = ff_chrRangeToJpeg_neon; | 249 | c->chrConvertRange = ff_chrRangeToJpeg8_neon; |
| 250 | } | ||
| 251 | } else { | ||
| 252 | if (c->opts.src_range) { | ||
| 253 | c->lumConvertRange = ff_lumRangeFromJpeg16_neon; | ||
| 254 | c->chrConvertRange = ff_chrRangeFromJpeg16_neon; | ||
| 255 | } else { | ||
| 256 | c->lumConvertRange = ff_lumRangeToJpeg16_neon; | ||
| 257 | c->chrConvertRange = ff_chrRangeToJpeg16_neon; | ||
| 242 | } | 258 | } |
| 243 | } | 259 | } |
| 244 | } | 260 | } |