The max height is currently documented as 16; the max difference per pixel is 255, and a .8h element can easily contain 16*255, thus keep accumulating in two .8h vectors, and just do the final accumulationat the end. This should work for heights up to 256. This requires a minor register renumbering in ff_pix_abs16_xy2_neon. Before: Cortex A53 A72 A73 Graviton 3 pix_abs_0_0_neon: 97.7 47.0 37.5 22.7 pix_abs_0_1_neon: 154.0 59.0 52.0 25.0 pix_abs_0_3_neon: 179.7 96.7 87.5 41.2 After: pix_abs_0_0_neon: 96.0 39.2 31.2 22.0 pix_abs_0_1_neon: 150.7 59.7 46.2 23.7 pix_abs_0_3_neon: 175.7 83.7 81.7 38.2
Unified · Split
| @@ -27,15 +27,16 @@ function ff_pix_abs16_neon, export=1 · 26 unmodified lines | |||
| 27 | // x3 ptrdiff_t stride | 27 | // x3 ptrdiff_t stride |
| 28 | // w4 int h | 28 | // w4 int h |
| 29 | cmp w4, #4 // if h < 4, jump to completion section | 29 | cmp w4, #4 // if h < 4, jump to completion section |
| 30 | movi v18.4S, #0 // clear result accumulator | 30 | movi v16.8h, #0 // clear result accumulator |
| 31 | movi v17.8h, #0 // clear result accumulator | ||
| 31 | b.lt 2f | 32 | b.lt 2f |
| 32 | 1: | 33 | 1: |
| 33 | ld1 {v0.16b}, [x1], x3 // load pix1 | 34 | ld1 {v0.16b}, [x1], x3 // load pix1 |
| 34 | ld1 {v4.16b}, [x2], x3 // load pix2 | 35 | ld1 {v4.16b}, [x2], x3 // load pix2 |
| 35 | ld1 {v1.16b}, [x1], x3 // load pix1 | 36 | ld1 {v1.16b}, [x1], x3 // load pix1 |
| 36 | ld1 {v5.16b}, [x2], x3 // load pix2 | 37 | ld1 {v5.16b}, [x2], x3 // load pix2 |
| 37 | uabdl v16.8h, v0.8b, v4.8b // absolute difference accumulate | 38 | uabal v16.8h, v0.8b, v4.8b // absolute difference accumulate |
| 38 | uabdl2 v17.8h, v0.16b, v4.16b | 39 | uabal2 v17.8h, v0.16b, v4.16b |
| 39 | ld1 {v2.16b}, [x1], x3 // load pix1 | 40 | ld1 {v2.16b}, [x1], x3 // load pix1 |
| 40 | ld1 {v6.16b}, [x2], x3 // load pix2 | 41 | ld1 {v6.16b}, [x2], x3 // load pix2 |
| 41 | uabal v16.8h, v1.8b, v5.8b // absolute difference accumulate | 42 | uabal v16.8h, v1.8b, v5.8b // absolute difference accumulate |
| @@ -48,27 +49,26 @@ function ff_pix_abs16_neon, export=1 · 6 unmodified lines | |||
| 48 | uabal v16.8h, v3.8b, v7.8b | 49 | uabal v16.8h, v3.8b, v7.8b |
| 49 | uabal2 v17.8h, v3.16b, v7.16b | 50 | uabal2 v17.8h, v3.16b, v7.16b |
| 50 | cmp w4, #4 // if h >= 4, loop | 51 | cmp w4, #4 // if h >= 4, loop |
| 51 | add v16.8h, v16.8h, v17.8h | ||
| 52 | uaddlv s16, v16.8h // add up everything in v16 accumulator | ||
| 53 | add d18, d16, d18 // add to the end result register | ||
| 54 | 52 | ||
| 55 | b.ge 1b | 53 | b.ge 1b |
| 56 | cbnz w4, 2f // if iterations remain, jump to completion section | 54 | cbnz w4, 2f // if iterations remain, jump to completion section |
| 57 | 55 | ||
| 58 | fmov w0, s18 // copy result to general purpose register | 56 | add v16.8h, v16.8h, v17.8h |
| 57 | uaddlv s16, v16.8h // add up everything in v16 accumulator | ||
| 58 | fmov w0, s16 // copy result to general purpose register | ||
| 59 | ret | 59 | ret |
| 60 | 60 | ||
| 61 | 2: | 61 | 2: |
| 62 | ld1 {v0.16b}, [x1], x3 // load pix1 | 62 | ld1 {v0.16b}, [x1], x3 // load pix1 |
| 63 | ld1 {v4.16b}, [x2], x3 // load pix2 | 63 | ld1 {v4.16b}, [x2], x3 // load pix2 |
| 64 | uabdl v16.8h, v0.8b, v4.8b // absolute difference accumulate | ||
| 65 | uabal2 v16.8h, v0.16b, v4.16b | ||
| 66 | subs w4, w4, #1 // h -= 1 | 64 | subs w4, w4, #1 // h -= 1 |
| 67 | addv h16, v16.8h // add up v16 | 65 | uabal v16.8h, v0.8b, v4.8b // absolute difference accumulate |
| 68 | add d18, d16, d18 // add to result | 66 | uabal2 v17.8h, v0.16b, v4.16b |
| 69 | b.ne 2b | 67 | b.ne 2b |
| 70 | 68 | ||
| 71 | fmov w0, s18 // copy result to general purpose register | 69 | add v16.8h, v16.8h, v17.8h |
| 70 | uaddlv s16, v16.8h // add up everything in v16 accumulator | ||
| 71 | fmov w0, s16 // copy result to general purpose register | ||
| 72 | ret | 72 | ret |
| 73 | endfunc | 73 | endfunc |
| 74 | 74 | ||
| @@ -80,7 +80,8 @@ function ff_pix_abs16_xy2_neon, export=1 · 5 unmodified lines | |||
| 80 | // w4 int h | 80 | // w4 int h |
| 81 | 81 | ||
| 82 | add x5, x2, x3 // use x5 to hold uint8_t *pix3 | 82 | add x5, x2, x3 // use x5 to hold uint8_t *pix3 |
| 83 | movi v0.2d, #0 // initialize the result register | 83 | movi v21.8h, #0 // initialize the result register |
| 84 | movi v22.8h, #0 // initialize the result register | ||
| 84 | 85 | ||
| 85 | // Load initial pix2 values for either the unrolled version or completion version. | 86 | // Load initial pix2 values for either the unrolled version or completion version. |
| 86 | ldur q4, [x2, #1] // load pix2+1 | 87 | ldur q4, [x2, #1] // load pix2+1 |
| @@ -119,15 +120,15 @@ function ff_pix_abs16_xy2_neon, export=1 · 32 unmodified lines | |||
| 119 | uaddl v2.8h, v6.8b, v7.8b // pix3 + pix3+1 0..7 | 120 | uaddl v2.8h, v6.8b, v7.8b // pix3 + pix3+1 0..7 |
| 120 | uaddl2 v3.8h, v6.16b, v7.16b // pix3 + pix3+1 8..15 | 121 | uaddl2 v3.8h, v6.16b, v7.16b // pix3 + pix3+1 8..15 |
| 121 | 122 | ||
| 122 | ldur q22, [x5, #1] // load pix3+1 | 123 | ldur q7, [x5, #1] // load pix3+1 |
| 123 | 124 | ||
| 124 | add v26.8h, v30.8h, v2.8h // add up 0..7, using pix2 + pix2+1 values from pix3 above | 125 | add v26.8h, v30.8h, v2.8h // add up 0..7, using pix2 + pix2+1 values from pix3 above |
| 125 | add v27.8h, v31.8h, v3.8h // add up 8..15, using pix2 + pix2+1 values from pix3 above | 126 | add v27.8h, v31.8h, v3.8h // add up 8..15, using pix2 + pix2+1 values from pix3 above |
| 126 | 127 | ||
| 127 | uabdl v24.8h, v1.8b, v23.8b // absolute difference 0..7, i=0 | 128 | uabal v21.8h, v1.8b, v23.8b // absolute difference 0..7, i=0 |
| 128 | uabdl2 v23.8h, v1.16b, v23.16b // absolute difference 8..15, i=0 | 129 | uabal2 v22.8h, v1.16b, v23.16b // absolute difference 8..15, i=0 |
| 129 | 130 | ||
| 130 | ld1 {v21.16b}, [x5], x3 // load pix3 | 131 | ld1 {v6.16b}, [x5], x3 // load pix3 |
| 131 | ld1 {v20.16b}, [x1], x3 // load pix1 | 132 | ld1 {v20.16b}, [x1], x3 // load pix1 |
| 132 | 133 | ||
| 133 | rshrn v26.8b, v26.8h, #2 // shift right 2 0..7 (rounding shift right) | 134 | rshrn v26.8b, v26.8h, #2 // shift right 2 0..7 (rounding shift right) |
| @@ -140,33 +141,33 @@ function ff_pix_abs16_xy2_neon, export=1 · 6 unmodified lines | |||
| 140 | rshrn v28.8b, v28.8h, #2 // shift right 2 0..7 (rounding shift right) | 141 | rshrn v28.8b, v28.8h, #2 // shift right 2 0..7 (rounding shift right) |
| 141 | rshrn2 v28.16b, v29.8h, #2 // shift right 2 8..15 | 142 | rshrn2 v28.16b, v29.8h, #2 // shift right 2 8..15 |
| 142 | 143 | ||
| 143 | uabal v24.8h, v16.8b, v26.8b // absolute difference 0..7, i=1 | 144 | uabal v21.8h, v16.8b, v26.8b // absolute difference 0..7, i=1 |
| 144 | uabal2 v23.8h, v16.16b, v26.16b // absolute difference 8..15, i=1 | 145 | uabal2 v22.8h, v16.16b, v26.16b // absolute difference 8..15, i=1 |
| 145 | 146 | ||
| 146 | uaddl v2.8h, v21.8b, v22.8b // pix3 + pix3+1 0..7 | 147 | uaddl v2.8h, v6.8b, v7.8b // pix3 + pix3+1 0..7 |
| 147 | uaddl2 v3.8h, v21.16b, v22.16b // pix3 + pix3+1 8..15 | 148 | uaddl2 v3.8h, v6.16b, v7.16b // pix3 + pix3+1 8..15 |
| 148 | add v30.8h, v4.8h, v2.8h // add up 0..7, using pix2 + pix2+1 values from pix3 above | 149 | add v30.8h, v4.8h, v2.8h // add up 0..7, using pix2 + pix2+1 values from pix3 above |
| 149 | add v31.8h, v5.8h, v3.8h // add up 8..15, using pix2 + pix2+1 values from pix3 above | 150 | add v31.8h, v5.8h, v3.8h // add up 8..15, using pix2 + pix2+1 values from pix3 above |
| 150 | rshrn v30.8b, v30.8h, #2 // shift right 2 0..7 (rounding shift right) | 151 | rshrn v30.8b, v30.8h, #2 // shift right 2 0..7 (rounding shift right) |
| 151 | rshrn2 v30.16b, v31.8h, #2 // shift right 2 8..15 | 152 | rshrn2 v30.16b, v31.8h, #2 // shift right 2 8..15 |
| 152 | 153 | ||
| 153 | uabal v24.8h, v17.8b, v28.8b // absolute difference 0..7, i=2 | ||
| 154 | uabal2 v23.8h, v17.16b, v28.16b // absolute difference 8..15, i=2 | ||
| 155 | |||
| 156 | sub w4, w4, #4 // h -= 4 | 154 | sub w4, w4, #4 // h -= 4 |
| 157 | 155 | ||
| 158 | uabal v24.8h, v20.8b, v30.8b // absolute difference 0..7, i=3 | 156 | uabal v21.8h, v17.8b, v28.8b // absolute difference 0..7, i=2 |
| 159 | uabal2 v23.8h, v20.16b, v30.16b // absolute difference 8..15, i=3 | 157 | uabal2 v22.8h, v17.16b, v28.16b // absolute difference 8..15, i=2 |
| 160 | 158 | ||
| 161 | cmp w4, #4 // loop if h >= 4 | 159 | cmp w4, #4 // loop if h >= 4 |
| 162 | 160 | ||
| 163 | add v4.8h, v23.8h, v24.8h | 161 | |
| 164 | uaddlv s4, v4.8h // finish adding up accumulated values | 162 | uabal v21.8h, v20.8b, v30.8b // absolute difference 0..7, i=3 |
| 165 | add d0, d0, d4 // add the value to the top level accumulator | 163 | uabal2 v22.8h, v20.16b, v30.16b // absolute difference 8..15, i=3 |
| 166 | 164 | ||
| 167 | b.ge 1b | 165 | b.ge 1b |
| 168 | cbnz w4, 2f // if iterations remain jump to completion section | 166 | cbnz w4, 2f // if iterations remain jump to completion section |
| 169 | 167 | ||
| 168 | add v4.8h, v21.8h, v22.8h | ||
| 169 | uaddlv s0, v4.8h // finish adding up accumulated values | ||
| 170 | |||
| 170 | fmov w0, s0 // copy result to general purpose register | 171 | fmov w0, s0 // copy result to general purpose register |
| 171 | ret | 172 | ret |
| 172 | 2: | 173 | 2: |
| @@ -182,20 +183,18 @@ function ff_pix_abs16_xy2_neon, export=1 · 9 unmodified lines | |||
| 182 | add v16.8h, v2.8h, v18.8h // add up 0..7, using pix2 + pix2+1 values from previous iteration | 183 | add v16.8h, v2.8h, v18.8h // add up 0..7, using pix2 + pix2+1 values from previous iteration |
| 183 | add v17.8h, v3.8h, v19.8h // add up 8..15, using pix2 + pix2+1 values from previous iteration | 184 | add v17.8h, v3.8h, v19.8h // add up 8..15, using pix2 + pix2+1 values from previous iteration |
| 184 | // divide by 4 to compute the average of values summed above | 185 | // divide by 4 to compute the average of values summed above |
| 185 | urshr v16.8h, v16.8h, #2 // shift right by 2 0..7 (rounding shift right) | 186 | rshrn v16.8b, v16.8h, #2 // shift right by 2 0..7 (rounding shift right) |
| 186 | urshr v17.8h, v17.8h, #2 // shift right by 2 8..15 | 187 | rshrn2 v16.16b, v17.8h, #2 // shift right by 2 8..15 |
| 187 | |||
| 188 | uxtl2 v7.8h, v1.16b // 8->16 bits pix1 8..15 | ||
| 189 | uxtl v1.8h, v1.8b // 8->16 bits pix1 0..7 | ||
| 190 | 188 | ||
| 191 | uabd v6.8h, v1.8h, v16.8h // absolute difference 0..7 | 189 | uabal v21.8h, v1.8b, v16.8b // absolute difference 0..7 |
| 192 | uaba v6.8h, v7.8h, v17.8h // absolute difference accumulate 8..15 | 190 | uabal2 v22.8h, v1.16b, v16.16b // absolute difference accumulate 8..15 |
| 193 | mov v2.16b, v18.16b // pix3 -> pix2 | 191 | mov v2.16b, v18.16b // pix3 -> pix2 |
| 194 | mov v3.16b, v19.16b // pix3+1 -> pix2+1 | 192 | mov v3.16b, v19.16b // pix3+1 -> pix2+1 |
| 195 | uaddlv s6, v6.8h // add up accumulator in v6 | ||
| 196 | add d0, d0, d6 // add to the final result | ||
| 197 | 193 | ||
| 198 | b.ne 2b // loop if h > 0 | 194 | b.ne 2b // loop if h > 0 |
| 195 | |||
| 196 | add v4.8h, v21.8h, v22.8h | ||
| 197 | uaddlv s0, v4.8h // finish adding up accumulated values | ||
| 199 | fmov w0, s0 // copy result to general purpose register | 198 | fmov w0, s0 // copy result to general purpose register |
| 200 | ret | 199 | ret |
| 201 | endfunc | 200 | endfunc |
| @@ -209,7 +208,8 @@ function ff_pix_abs16_x2_neon, export=1 · 7 unmodified lines | |||
| 209 | 208 | ||
| 210 | cmp w4, #4 | 209 | cmp w4, #4 |
| 211 | // initialize buffers | 210 | // initialize buffers |
| 212 | movi d20, #0 | 211 | movi v16.8h, #0 |
| 212 | movi v17.8h, #0 | ||
| 213 | add x5, x2, #1 // pix2 + 1 | 213 | add x5, x2, #1 // pix2 + 1 |
| 214 | b.lt 2f | 214 | b.lt 2f |
| 215 | 215 | ||
| @@ -224,9 +224,9 @@ function ff_pix_abs16_x2_neon, export=1 · 8 unmodified lines | |||
| 224 | ld1 {v2.16b}, [x5], x3 | 224 | ld1 {v2.16b}, [x5], x3 |
| 225 | urhadd v30.16b, v1.16b, v2.16b | 225 | urhadd v30.16b, v1.16b, v2.16b |
| 226 | ld1 {v0.16b}, [x1], x3 | 226 | ld1 {v0.16b}, [x1], x3 |
| 227 | uabdl v16.8h, v0.8b, v30.8b | 227 | uabal v16.8h, v0.8b, v30.8b |
| 228 | ld1 {v4.16b}, [x2], x3 | 228 | ld1 {v4.16b}, [x2], x3 |
| 229 | uabdl2 v17.8h, v0.16b, v30.16b | 229 | uabal2 v17.8h, v0.16b, v30.16b |
| 230 | ld1 {v5.16b}, [x5], x3 | 230 | ld1 {v5.16b}, [x5], x3 |
| 231 | urhadd v29.16b, v4.16b, v5.16b | 231 | urhadd v29.16b, v4.16b, v5.16b |
| 232 | ld1 {v3.16b}, [x1], x3 | 232 | ld1 {v3.16b}, [x1], x3 |
| @@ -238,20 +238,15 @@ function ff_pix_abs16_x2_neon, export=1 · 5 unmodified lines | |||
| 238 | ld1 {v6.16b}, [x1], x3 | 238 | ld1 {v6.16b}, [x1], x3 |
| 239 | uabal v16.8h, v6.8b, v28.8b | 239 | uabal v16.8h, v6.8b, v28.8b |
| 240 | ld1 {v24.16b}, [x2], x3 | 240 | ld1 {v24.16b}, [x2], x3 |
| 241 | sub w4, w4, #4 | ||
| 241 | uabal2 v17.8h, v6.16b, v28.16b | 242 | uabal2 v17.8h, v6.16b, v28.16b |
| 242 | ld1 {v25.16b}, [x5], x3 | 243 | ld1 {v25.16b}, [x5], x3 |
| 243 | urhadd v27.16b, v24.16b, v25.16b | 244 | urhadd v27.16b, v24.16b, v25.16b |
| 244 | ld1 {v23.16b}, [x1], x3 | 245 | ld1 {v23.16b}, [x1], x3 |
| 246 | cmp w4, #4 | ||
| 245 | uabal v16.8h, v23.8b, v27.8b | 247 | uabal v16.8h, v23.8b, v27.8b |
| 246 | uabal2 v17.8h, v23.16b, v27.16b | 248 | uabal2 v17.8h, v23.16b, v27.16b |
| 247 | 249 | ||
| 248 | sub w4, w4, #4 | ||
| 249 | |||
| 250 | add v16.8h, v16.8h, v17.8h | ||
| 251 | uaddlv s16, v16.8h | ||
| 252 | cmp w4, #4 | ||
| 253 | add d20, d20, d16 | ||
| 254 | |||
| 255 | b.ge 1b | 250 | b.ge 1b |
| 256 | cbz w4, 3f | 251 | cbz w4, 3f |
| 257 | 252 | ||
| @@ -259,18 +254,19 @@ function ff_pix_abs16_x2_neon, export=1 · 1 unmodified line | |||
| 259 | 2: | 254 | 2: |
| 260 | ld1 {v1.16b}, [x2], x3 | 255 | ld1 {v1.16b}, [x2], x3 |
| 261 | ld1 {v2.16b}, [x5], x3 | 256 | ld1 {v2.16b}, [x5], x3 |
| 257 | subs w4, w4, #1 | ||
| 262 | urhadd v29.16b, v1.16b, v2.16b | 258 | urhadd v29.16b, v1.16b, v2.16b |
| 263 | ld1 {v0.16b}, [x1], x3 | 259 | ld1 {v0.16b}, [x1], x3 |
| 264 | uabd v28.16b, v0.16b, v29.16b | 260 | uabal v16.8h, v0.8b, v29.8b |
| 261 | uabal2 v17.8h, v0.16b, v29.16b | ||
| 265 | 262 | ||
| 266 | uaddlv h28, v28.16b | ||
| 267 | subs w4, w4, #1 | ||
| 268 | 263 | ||
| 269 | add d20, d20, d28 | ||
| 270 | b.ne 2b | 264 | b.ne 2b |
| 271 | 265 | ||
| 272 | 3: | 266 | 3: |
| 273 | fmov w0, s20 | 267 | add v16.8h, v16.8h, v17.8h |
| 268 | uaddlv s16, v16.8h | ||
| 269 | fmov w0, s16 | ||
| 274 | 270 | ||
| 275 | ret | 271 | ret |
| 276 | endfunc | 272 | endfunc |