int8_t[] is enough for offset_table of 8 bit streams. On rpi5: Before After hevc_sao_band_8_8_c: 252.3 ( 1.00x) 252.3 ( 1.00x) hevc_sao_band_8_8_neon: 95.8 ( 2.63x) 61.0 ( 4.57x) hevc_sao_band_16_8_c: 875.2 ( 1.00x) 864.9 ( 1.00x) hevc_sao_band_16_8_neon: 317.5 ( 2.76x) 150.0 ( 6.26x) hevc_sao_band_32_8_c: 3853.5 ( 1.00x) 3871.6 ( 1.00x) hevc_sao_band_32_8_neon: 1222.3 ( 3.15x) 550.6 ( 7.39) hevc_sao_band_48_8_c: 8203.6 ( 1.00x) 8182.6 ( 1.00x) hevc_sao_band_48_8_neon: 2685.7 ( 3.05x) 1185.8 ( 7.36x) hevc_sao_band_64_8_c: 14023.0 ( 1.00x) 14038.9 ( 1.00x) hevc_sao_band_64_8_neon: 4783.2 ( 2.93x) 2078.4 ( 7.15x)
Unified · Split
| @@ -28,6 +28,10 @@ void ff_h26x_sao_band_filter_8x8_8_neon(uint8_t *_dst, const uint8_t *_src, · 27 unmodified lines | |||
| 28 | ptrdiff_t stride_dst, ptrdiff_t stride_src, | 28 | ptrdiff_t stride_dst, ptrdiff_t stride_src, |
| 29 | const int16_t *sao_offset_val, int sao_left_class, | 29 | const int16_t *sao_offset_val, int sao_left_class, |
| 30 | int width, int height); | 30 | int width, int height); |
| 31 | void ff_h26x_sao_band_filter_16x16_8_neon(uint8_t *_dst, const uint8_t *_src, | ||
| 32 | ptrdiff_t stride_dst, ptrdiff_t stride_src, | ||
| 33 | const int16_t *sao_offset_val, int sao_left_class, | ||
| 34 | int width, int height); | ||
| 31 | void ff_hevc_sao_edge_filter_16x16_8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst, | 35 | void ff_hevc_sao_edge_filter_16x16_8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst, |
| 32 | const int16_t *sao_offset_val, int eo, int width, int height); | 36 | const int16_t *sao_offset_val, int eo, int width, int height); |
| 33 | void ff_hevc_sao_edge_filter_8x8_8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst, | 37 | void ff_hevc_sao_edge_filter_8x8_8_neon(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst, |
| @@ -35,48 +35,67 @@ · 34 unmodified lines | |||
| 35 | // int16_t *sao_offset_val, int sao_left_class, | 35 | // int16_t *sao_offset_val, int sao_left_class, |
| 36 | // int width, int height) | 36 | // int width, int height) |
| 37 | function ff_h26x_sao_band_filter_8x8_8_neon, export=1 | 37 | function ff_h26x_sao_band_filter_8x8_8_neon, export=1 |
| 38 | stp xzr, xzr, [sp, #-64]! | 38 | stp xzr, xzr, [sp, #-32]! |
| 39 | stp xzr, xzr, [sp, #16] | 39 | stp xzr, xzr, [sp, #16] |
| 40 | stp xzr, xzr, [sp, #32] | ||
| 41 | stp xzr, xzr, [sp, #48] | ||
| 42 | mov w8, #4 | 40 | mov w8, #4 |
| 43 | 0: ldrsh x9, [x4, x8, lsl #1] // sao_offset_val[k+1] | 41 | 0: |
| 44 | subs w8, w8, #1 | 42 | ldrsh x9, [x4, x8, lsl #1] // sao_offset_val[k+1] |
| 45 | add w10, w8, w5 // k + sao_left_class | 43 | subs w8, w8, #1 |
| 44 | add w10, w8, w5 // k + sao_left_class | ||
| 46 | and w10, w10, #0x1F | 45 | and w10, w10, #0x1F |
| 47 | strh w9, [sp, x10, lsl #1] | 46 | strb w9, [sp, x10] |
| 48 | bne 0b | 47 | bne 0b |
| 49 | add w6, w6, #7 | 48 | ldp q16, q17, [sp], #32 |
| 50 | bic w6, w6, #7 | 49 | 1: |
| 51 | ld1 {v16.16b-v19.16b}, [sp], #64 | 50 | ld1 {v2.8b}, [x1], x3 |
| 52 | sub x2, x2, x6 | 51 | subs w7, w7, #1 |
| 53 | sub x3, x3, x6 | 52 | uxtl v0.8h, v2.8b |
| 54 | movi v20.8h, #1 | 53 | ushr v3.8b, v2.8b, #3 // >> BIT_DEPTH - 3 |
| 55 | 1: mov w8, w6 // beginning of line | 54 | tbl v3.8b, {v16.16b-v17.16b}, v3.8b |
| 56 | 2: // Simple layout for accessing 16bit values | 55 | sxtl v2.8h, v3.8b |
| 57 | // with 8bit LUT. | 56 | add v0.8h, v0.8h, v2.8h // src[x] + table |
| 58 | // | 57 | sqxtun v0.8b, v0.8h // clip + narrow |
| 59 | // 00 01 02 03 04 05 06 07 | 58 | st1 {v0.8b}, [x0], x2 |
| 60 | // +-----------------------------------> | 59 | bne 1b |
| 61 | // |xDE#xAD|xCA#xFE|xBE#xEF|xFE#xED|.... | 60 | ret |
| 62 | // +-----------------------------------> | 61 | endfunc |
| 63 | // i-0 i-1 i-2 i-3 | 62 | |
| 64 | ld1 {v2.8b}, [x1], #8 // dst[x] = av_clip_pixel(src[x] + offset_table[src[x] >> shift]); | 63 | function ff_h26x_sao_band_filter_16x16_8_neon, export=1 |
| 65 | subs w8, w8, #8 | 64 | stp xzr, xzr, [sp, #-32]! |
| 66 | uxtl v0.8h, v2.8b // load src[x] | 65 | stp xzr, xzr, [sp, #16] |
| 67 | ushr v2.8h, v0.8h, #3 // >> BIT_DEPTH - 3 | 66 | mov w8, #4 |
| 68 | shl v1.8h, v2.8h, #1 // low (x2, accessing short) | 67 | 0: |
| 69 | add v3.8h, v1.8h, v20.8h // +1 access upper short | 68 | ldrsh x9, [x4, x8, lsl #1] // sao_offset_val[k+1] |
| 70 | sli v1.8h, v3.8h, #8 // shift insert index to upper byte | 69 | subs w8, w8, #1 |
| 71 | tbx v2.16b, {v16.16b-v19.16b}, v1.16b // table | 70 | add w10, w8, w5 // k + sao_left_class |
| 72 | add v1.8h, v0.8h, v2.8h // src[x] + table | 71 | and w10, w10, #0x1F |
| 73 | sqxtun v4.8b, v1.8h // clip + narrow | 72 | strb w9, [sp, x10] |
| 74 | st1 {v4.8b}, [x0], #8 // store | 73 | bne 0b |
| 75 | // done 8 pixels | 74 | add w6, w6, #15 |
| 75 | bic w6, w6, #15 | ||
| 76 | ldp q16, q17, [sp], #32 | ||
| 77 | sub x2, x2, x6 | ||
| 78 | sub x3, x3, x6 | ||
| 79 | 1: | ||
| 80 | mov w8, w6 // beginning of line | ||
| 81 | 2: | ||
| 82 | ldr q2, [x1], #16 | ||
| 83 | subs w8, w8, #16 | ||
| 84 | uxtl v0.8h, v2.8b | ||
| 85 | uxtl2 v1.8h, v2.16b | ||
| 86 | ushr v3.16b, v2.16b, #3 // >> BIT_DEPTH - 3 | ||
| 87 | tbl v3.16b, {v16.16b-v17.16b}, v3.16b | ||
| 88 | sxtl v2.8h, v3.8b | ||
| 89 | sxtl2 v3.8h, v3.16b | ||
| 90 | add v0.8h, v0.8h, v2.8h // src[x] + table | ||
| 91 | add v1.8h, v1.8h, v3.8h | ||
| 92 | sqxtun v0.8b, v0.8h // clip + narrow | ||
| 93 | sqxtun2 v0.16b, v1.8h | ||
| 94 | str q0, [x0], #16 | ||
| 76 | bne 2b | 95 | bne 2b |
| 77 | subs w7, w7, #1 // finished line, prep. new | 96 | subs w7, w7, #1 |
| 78 | add x0, x0, x2 // dst += stride_dst | 97 | add x0, x0, x2 // dst += stride_dst |
| 79 | add x1, x1, x3 // src += stride_src | 98 | add x1, x1, x3 // src += stride_src |
| 80 | bne 1b | 99 | bne 1b |
| 81 | ret | 100 | ret |
| 82 | endfunc | 101 | endfunc |
| @@ -157,11 +157,11 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) · 156 unmodified lines | |||
| 157 | c->idct_dc[2] = ff_hevc_idct_16x16_dc_8_neon; | 157 | c->idct_dc[2] = ff_hevc_idct_16x16_dc_8_neon; |
| 158 | c->idct_dc[3] = ff_hevc_idct_32x32_dc_8_neon; | 158 | c->idct_dc[3] = ff_hevc_idct_32x32_dc_8_neon; |
| 159 | c->transform_4x4_luma = ff_hevc_transform_luma_4x4_neon_8; | 159 | c->transform_4x4_luma = ff_hevc_transform_luma_4x4_neon_8; |
| 160 | c->sao_band_filter[0] = | 160 | c->sao_band_filter[0] = ff_h26x_sao_band_filter_8x8_8_neon; |
| 161 | c->sao_band_filter[1] = | 161 | c->sao_band_filter[1] = |
| 162 | c->sao_band_filter[2] = | 162 | c->sao_band_filter[2] = |
| 163 | c->sao_band_filter[3] = | 163 | c->sao_band_filter[3] = |
| 164 | c->sao_band_filter[4] = ff_h26x_sao_band_filter_8x8_8_neon; | 164 | c->sao_band_filter[4] = ff_h26x_sao_band_filter_16x16_8_neon; |
| 165 | c->sao_edge_filter[0] = ff_hevc_sao_edge_filter_8x8_8_neon; | 165 | c->sao_edge_filter[0] = ff_hevc_sao_edge_filter_8x8_8_neon; |
| 166 | c->sao_edge_filter[1] = | 166 | c->sao_edge_filter[1] = |
| 167 | c->sao_edge_filter[2] = | 167 | c->sao_edge_filter[2] = |
| @@ -197,8 +197,9 @@ void ff_vvc_dsp_init_aarch64(VVCDSPContext *const c, const int bd) · 196 unmodified lines | |||
| 197 | c->inter.dmvr[1][1] = ff_vvc_dmvr_hv_8_neon; | 197 | c->inter.dmvr[1][1] = ff_vvc_dmvr_hv_8_neon; |
| 198 | c->inter.apply_bdof = apply_bdof_8; | 198 | c->inter.apply_bdof = apply_bdof_8; |
| 199 | 199 | ||
| 200 | for (int i = 0; i < FF_ARRAY_ELEMS(c->sao.band_filter); i++) | 200 | c->sao.band_filter[0] = ff_h26x_sao_band_filter_8x8_8_neon; |
| 201 | c->sao.band_filter[i] = ff_h26x_sao_band_filter_8x8_8_neon; | 201 | for (int i = 1; i < FF_ARRAY_ELEMS(c->sao.band_filter); i++) |
| 202 | c->sao.band_filter[i] = ff_h26x_sao_band_filter_16x16_8_neon; | ||
| 202 | c->sao.edge_filter[0] = ff_vvc_sao_edge_filter_8x8_8_neon; | 203 | c->sao.edge_filter[0] = ff_vvc_sao_edge_filter_8x8_8_neon; |
| 203 | for (int i = 1; i < FF_ARRAY_ELEMS(c->sao.edge_filter); i++) | 204 | for (int i = 1; i < FF_ARRAY_ELEMS(c->sao.edge_filter); i++) |
| 204 | c->sao.edge_filter[i] = ff_vvc_sao_edge_filter_16x16_8_neon; | 205 | c->sao.edge_filter[i] = ff_vvc_sao_edge_filter_16x16_8_neon; |