int8_t[] is enough for offset_table of 8 bit streams. On rpi5: Before After hevc_sao_band_8_8_c: 252.3 ( 1.00x) 252.3 ( 1.00x) hevc_sao_band_8_8_neon: 95.8 ( 2.63x) 61.0 ( 4.57x) hevc_sao_band_16_8_c: 875.2 ( 1.00x) 864.9 ( 1.00x) hevc_sao_band_16_8_neon: 317.5 ( 2.76x) 150.0 ( 6.26x) hevc_sao_band_32_8_c: 3853.5 ( 1.00x) 3871.6 ( 1.00x) hevc_sao_band_32_8_neon: 1222.3 ( 3.15x) 550.6 ( 7.39) hevc_sao_band_48_8_c: 8203.6 ( 1.00x) 8182.6 ( 1.00x) hevc_sao_band_48_8_neon: 2685.7 ( 3.05x) 1185.8 ( 7.36x) hevc_sao_band_64_8_c: 14023.0 ( 1.00x) 14038.9 ( 1.00x) hevc_sao_band_64_8_neon: 4783.2 ( 2.93x) 2078.4 ( 7.15x)
Unified · Split
| @@ -28,6 +28,10 @@ void ff_h26x_sao_band_filter_8x8_8_neon(uint8_t *_dst, const uint8_t *_src, · 27 unmodified lines | ||
| 28 | 28 | ptrdiff_t stride_dst, ptrdiff_t stride_src, |
| 29 | 29 | const int16_t *sao_offset_val, int sao_left_class, |
| 30 | 30 | int width, int height); |
| 31 | +void ff_h26x_sao_band_filter_16x16_8_neon(uint8_t *_dst, const uint8_t *_src, | |
| 32 | + ptrdiff_t stride_dst, ptrdiff_t stride_src, | |
| 33 | + const int16_t *sao_offset_val, int sao_left_class, | |
| 34 | + int width, int height); | |
| 31 | 35 | void ff_hevc_sao_edge_filter_16x16_8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst, |
| 32 | 36 | const int16_t *sao_offset_val, int eo, int width, int height); |
| 33 | 37 | void ff_hevc_sao_edge_filter_8x8_8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst, |
| @@ -35,48 +35,67 @@ · 34 unmodified lines | ||
| 35 | 35 | // int16_t *sao_offset_val, int sao_left_class, |
| 36 | 36 | // int width, int height) |
| 37 | 37 | function ff_h26x_sao_band_filter_8x8_8_neon, export=1 |
| 38 | − stp xzr, xzr, [sp, #-64]! | |
| 38 | + stp xzr, xzr, [sp, #-32]! | |
| 39 | 39 | stp xzr, xzr, [sp, #16] |
| 40 | − stp xzr, xzr, [sp, #32] | |
| 41 | − stp xzr, xzr, [sp, #48] | |
| 42 | 40 | mov w8, #4 |
| 43 | −0: ldrsh x9, [x4, x8, lsl #1] // sao_offset_val[k+1] | |
| 44 | − subs w8, w8, #1 | |
| 45 | − add w10, w8, w5 // k + sao_left_class | |
| 41 | +0: | |
| 42 | + ldrsh x9, [x4, x8, lsl #1] // sao_offset_val[k+1] | |
| 43 | + subs w8, w8, #1 | |
| 44 | + add w10, w8, w5 // k + sao_left_class | |
| 46 | 45 | and w10, w10, #0x1F |
| 47 | − strh w9, [sp, x10, lsl #1] | |
| 46 | + strb w9, [sp, x10] | |
| 48 | 47 | bne 0b |
| 49 | − add w6, w6, #7 | |
| 50 | − bic w6, w6, #7 | |
| 51 | − ld1 {v16.16b-v19.16b}, [sp], #64 | |
| 52 | − sub x2, x2, x6 | |
| 53 | − sub x3, x3, x6 | |
| 54 | − movi v20.8h, #1 | |
| 55 | −1: mov w8, w6 // beginning of line | |
| 56 | −2: // Simple layout for accessing 16bit values | |
| 57 | − // with 8bit LUT. | |
| 58 | − // | |
| 59 | − // 00 01 02 03 04 05 06 07 | |
| 60 | − // +-----------------------------------> | |
| 61 | − // |xDE#xAD|xCA#xFE|xBE#xEF|xFE#xED|.... | |
| 62 | − // +-----------------------------------> | |
| 63 | − // i-0 i-1 i-2 i-3 | |
| 64 | − ld1 {v2.8b}, [x1], #8 // dst[x] = av_clip_pixel(src[x] + offset_table[src[x] >> shift]); | |
| 65 | − subs w8, w8, #8 | |
| 66 | − uxtl v0.8h, v2.8b // load src[x] | |
| 67 | − ushr v2.8h, v0.8h, #3 // >> BIT_DEPTH - 3 | |
| 68 | − shl v1.8h, v2.8h, #1 // low (x2, accessing short) | |
| 69 | − add v3.8h, v1.8h, v20.8h // +1 access upper short | |
| 70 | − sli v1.8h, v3.8h, #8 // shift insert index to upper byte | |
| 71 | − tbx v2.16b, {v16.16b-v19.16b}, v1.16b // table | |
| 72 | − add v1.8h, v0.8h, v2.8h // src[x] + table | |
| 73 | − sqxtun v4.8b, v1.8h // clip + narrow | |
| 74 | − st1 {v4.8b}, [x0], #8 // store | |
| 75 | − // done 8 pixels | |
| 48 | + ldp q16, q17, [sp], #32 | |
| 49 | +1: | |
| 50 | + ld1 {v2.8b}, [x1], x3 | |
| 51 | + subs w7, w7, #1 | |
| 52 | + uxtl v0.8h, v2.8b | |
| 53 | + ushr v3.8b, v2.8b, #3 // >> BIT_DEPTH - 3 | |
| 54 | + tbl v3.8b, {v16.16b-v17.16b}, v3.8b | |
| 55 | + sxtl v2.8h, v3.8b | |
| 56 | + add v0.8h, v0.8h, v2.8h // src[x] + table | |
| 57 | + sqxtun v0.8b, v0.8h // clip + narrow | |
| 58 | + st1 {v0.8b}, [x0], x2 | |
| 59 | + bne 1b | |
| 60 | + ret | |
| 61 | +endfunc | |
| 62 | + | |
| 63 | +function ff_h26x_sao_band_filter_16x16_8_neon, export=1 | |
| 64 | + stp xzr, xzr, [sp, #-32]! | |
| 65 | + stp xzr, xzr, [sp, #16] | |
| 66 | + mov w8, #4 | |
| 67 | +0: | |
| 68 | + ldrsh x9, [x4, x8, lsl #1] // sao_offset_val[k+1] | |
| 69 | + subs w8, w8, #1 | |
| 70 | + add w10, w8, w5 // k + sao_left_class | |
| 71 | + and w10, w10, #0x1F | |
| 72 | + strb w9, [sp, x10] | |
| 73 | + bne 0b | |
| 74 | + add w6, w6, #15 | |
| 75 | + bic w6, w6, #15 | |
| 76 | + ldp q16, q17, [sp], #32 | |
| 77 | + sub x2, x2, x6 | |
| 78 | + sub x3, x3, x6 | |
| 79 | +1: | |
| 80 | + mov w8, w6 // beginning of line | |
| 81 | +2: | |
| 82 | + ldr q2, [x1], #16 | |
| 83 | + subs w8, w8, #16 | |
| 84 | + uxtl v0.8h, v2.8b | |
| 85 | + uxtl2 v1.8h, v2.16b | |
| 86 | + ushr v3.16b, v2.16b, #3 // >> BIT_DEPTH - 3 | |
| 87 | + tbl v3.16b, {v16.16b-v17.16b}, v3.16b | |
| 88 | + sxtl v2.8h, v3.8b | |
| 89 | + sxtl2 v3.8h, v3.16b | |
| 90 | + add v0.8h, v0.8h, v2.8h // src[x] + table | |
| 91 | + add v1.8h, v1.8h, v3.8h | |
| 92 | + sqxtun v0.8b, v0.8h // clip + narrow | |
| 93 | + sqxtun2 v0.16b, v1.8h | |
| 94 | + str q0, [x0], #16 | |
| 76 | 95 | bne 2b |
| 77 | − subs w7, w7, #1 // finished line, prep. new | |
| 78 | − add x0, x0, x2 // dst += stride_dst | |
| 79 | − add x1, x1, x3 // src += stride_src | |
| 96 | + subs w7, w7, #1 | |
| 97 | + add x0, x0, x2 // dst += stride_dst | |
| 98 | + add x1, x1, x3 // src += stride_src | |
| 80 | 99 | bne 1b |
| 81 | 100 | ret |
| 82 | 101 | endfunc |
| @@ -157,11 +157,11 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) · 156 unmodified lines | ||
| 157 | 157 | c->idct_dc[2] = ff_hevc_idct_16x16_dc_8_neon; |
| 158 | 158 | c->idct_dc[3] = ff_hevc_idct_32x32_dc_8_neon; |
| 159 | 159 | c->transform_4x4_luma = ff_hevc_transform_luma_4x4_neon_8; |
| 160 | − c->sao_band_filter[0] = | |
| 160 | + c->sao_band_filter[0] = ff_h26x_sao_band_filter_8x8_8_neon; | |
| 161 | 161 | c->sao_band_filter[1] = |
| 162 | 162 | c->sao_band_filter[2] = |
| 163 | 163 | c->sao_band_filter[3] = |
| 164 | − c->sao_band_filter[4] = ff_h26x_sao_band_filter_8x8_8_neon; | |
| 164 | + c->sao_band_filter[4] = ff_h26x_sao_band_filter_16x16_8_neon; | |
| 165 | 165 | c->sao_edge_filter[0] = ff_hevc_sao_edge_filter_8x8_8_neon; |
| 166 | 166 | c->sao_edge_filter[1] = |
| 167 | 167 | c->sao_edge_filter[2] = |
| @@ -197,8 +197,9 @@ void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd) · 196 unmodified lines | ||
| 197 | 197 | c->inter.dmvr[1][1] = ff_vvc_dmvr_hv_8_neon; |
| 198 | 198 | c->inter.apply_bdof = apply_bdof_8; |
| 199 | 199 | |
| 200 | − for (int i = 0; i < FF_ARRAY_ELEMS(c->sao.band_filter); i++) | |
| 201 | − c->sao.band_filter[i] = ff_h26x_sao_band_filter_8x8_8_neon; | |
| 200 | + c->sao.band_filter[0] = ff_h26x_sao_band_filter_8x8_8_neon; | |
| 201 | + for (int i = 1; i < FF_ARRAY_ELEMS(c->sao.band_filter); i++) | |
| 202 | + c->sao.band_filter[i] = ff_h26x_sao_band_filter_16x16_8_neon; | |
| 202 | 203 | c->sao.edge_filter[0] = ff_vvc_sao_edge_filter_8x8_8_neon; |
| 203 | 204 | for (int i = 1; i < FF_ARRAY_ELEMS(c->sao.edge_filter); i++) |
| 204 | 205 | c->sao.edge_filter[i] = ff_vvc_sao_edge_filter_16x16_8_neon; |