Unified · Split
| @@ -19,6 +19,24 @@ · 18 unmodified lines | ||
| 19 | 19 | #include "vpx/vpx_integer.h" |
| 20 | 20 | #include "vpx_dsp/vpx_dsp_common.h" |
| 21 | 21 | |
| 22 | +// Support for these xN intrinsics is lacking in older versions of GCC. | |
| 23 | +#if defined(__GNUC__) && !defined(__clang__) | |
| 24 | +#if __GNUC__ < 8 || defined(__arm__) | |
| 25 | +static INLINE uint8x16x2_t vld1q_u8_x2(uint8_t const *ptr) { | |
| 26 | + uint8x16x2_t res = { { vld1q_u8(ptr + 0 * 16), vld1q_u8(ptr + 1 * 16) } }; | |
| 27 | + return res; | |
| 28 | +} | |
| 29 | +#endif | |
| 30 | + | |
| 31 | +#if __GNUC__ < 9 || defined(__arm__) | |
| 32 | +static INLINE uint8x16x3_t vld1q_u8_x3(uint8_t const *ptr) { | |
| 33 | + uint8x16x3_t res = { { vld1q_u8(ptr + 0 * 16), vld1q_u8(ptr + 1 * 16), | |
| 34 | + vld1q_u8(ptr + 2 * 16) } }; | |
| 35 | + return res; | |
| 36 | +} | |
| 37 | +#endif | |
| 38 | +#endif | |
| 39 | + | |
| 22 | 40 | static INLINE int16x4_t create_s16x4_neon(const int16_t c0, const int16_t c1, |
| 23 | 41 | const int16_t c2, const int16_t c3) { |
| 24 | 42 | return vcreate_s16((uint16_t)c0 | ((uint32_t)c1 << 16) | |
| @@ -14,6 +14,7 @@ · 13 unmodified lines | ||
| 14 | 14 | #include "./vpx_config.h" |
| 15 | 15 | #include "./vpx_dsp_rtcd.h" |
| 16 | 16 | #include "vpx/vpx_integer.h" |
| 17 | +#include "vpx_dsp/arm/mem_neon.h" | |
| 17 | 18 | #include "vpx_dsp/arm/transpose_neon.h" |
| 18 | 19 | #include "vpx_dsp/arm/vpx_convolve8_neon.h" |
| 19 | 20 | #include "vpx_ports/mem.h" |
| @@ -52,6 +53,112 @@ static INLINE void store_u8_8x8(uint8_t *s, const ptrdiff_t p, · 32 unmodified lines | ||
| 52 | 53 | vst1_u8(s, s7); |
| 53 | 54 | } |
| 54 | 55 | |
| 56 | +#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ | |
| 57 | + (__ARM_FEATURE_DOTPROD == 1) | |
| 58 | + | |
| 59 | +DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = { | |
| 60 | + 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, | |
| 61 | + 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, | |
| 62 | + 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 | |
| 63 | +}; | |
| 64 | + | |
| 65 | +void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride, | |
| 66 | + uint8_t *dst, ptrdiff_t dst_stride, | |
| 67 | + const InterpKernel *filter, int x0_q4, | |
| 68 | + int x_step_q4, int y0_q4, int y_step_q4, int w, | |
| 69 | + int h) { | |
| 70 | + const int8x8_t filters = vmovn_s16(vld1q_s16(filter[x0_q4])); | |
| 71 | + const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[x0_q4]), 128); | |
| 72 | + const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp)); | |
| 73 | + const uint8x16_t range_limit = vdupq_n_u8(128); | |
| 74 | + uint8x16_t s0, s1, s2, s3; | |
| 75 | + | |
| 76 | + assert(!((intptr_t)dst & 3)); | |
| 77 | + assert(!(dst_stride & 3)); | |
| 78 | + assert(x_step_q4 == 16); | |
| 79 | + | |
| 80 | + (void)x_step_q4; | |
| 81 | + (void)y0_q4; | |
| 82 | + (void)y_step_q4; | |
| 83 | + | |
| 84 | + src -= 3; | |
| 85 | + | |
| 86 | + if (w == 4) { | |
| 87 | + const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl); | |
| 88 | + do { | |
| 89 | + int32x4_t t0, t1, t2, t3; | |
| 90 | + int16x8_t t01, t23; | |
| 91 | + uint8x8_t d01, d23; | |
| 92 | + | |
| 93 | + s0 = vld1q_u8(src); | |
| 94 | + src += src_stride; | |
| 95 | + s1 = vld1q_u8(src); | |
| 96 | + src += src_stride; | |
| 97 | + s2 = vld1q_u8(src); | |
| 98 | + src += src_stride; | |
| 99 | + s3 = vld1q_u8(src); | |
| 100 | + src += src_stride; | |
| 101 | + | |
| 102 | + t0 = convolve8_4_dot(s0, filters, correction, range_limit, permute_tbl); | |
| 103 | + t1 = convolve8_4_dot(s1, filters, correction, range_limit, permute_tbl); | |
| 104 | + t2 = convolve8_4_dot(s2, filters, correction, range_limit, permute_tbl); | |
| 105 | + t3 = convolve8_4_dot(s3, filters, correction, range_limit, permute_tbl); | |
| 106 | + | |
| 107 | + t01 = vcombine_s16(vqmovn_s32(t0), vqmovn_s32(t1)); | |
| 108 | + t23 = vcombine_s16(vqmovn_s32(t2), vqmovn_s32(t3)); | |
| 109 | + d01 = vqrshrun_n_s16(t01, 7); | |
| 110 | + d23 = vqrshrun_n_s16(t23, 7); | |
| 111 | + | |
| 112 | + vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0); | |
| 113 | + dst += dst_stride; | |
| 114 | + vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1); | |
| 115 | + dst += dst_stride; | |
| 116 | + vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0); | |
| 117 | + dst += dst_stride; | |
| 118 | + vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1); | |
| 119 | + dst += dst_stride; | |
| 120 | + h -= 4; | |
| 121 | + } while (h > 0); | |
| 122 | + } else { | |
| 123 | + const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl); | |
| 124 | + const uint8_t *s; | |
| 125 | + uint8_t *d; | |
| 126 | + int width; | |
| 127 | + uint8x8_t d0, d1, d2, d3; | |
| 128 | + | |
| 129 | + do { | |
| 130 | + width = w; | |
| 131 | + s = src; | |
| 132 | + d = dst; | |
| 133 | + do { | |
| 134 | + s0 = vld1q_u8(s + 0 * src_stride); | |
| 135 | + s1 = vld1q_u8(s + 1 * src_stride); | |
| 136 | + s2 = vld1q_u8(s + 2 * src_stride); | |
| 137 | + s3 = vld1q_u8(s + 3 * src_stride); | |
| 138 | + | |
| 139 | + d0 = convolve8_8_dot(s0, filters, correction, range_limit, permute_tbl); | |
| 140 | + d1 = convolve8_8_dot(s1, filters, correction, range_limit, permute_tbl); | |
| 141 | + d2 = convolve8_8_dot(s2, filters, correction, range_limit, permute_tbl); | |
| 142 | + d3 = convolve8_8_dot(s3, filters, correction, range_limit, permute_tbl); | |
| 143 | + | |
| 144 | + vst1_u8(d + 0 * dst_stride, d0); | |
| 145 | + vst1_u8(d + 1 * dst_stride, d1); | |
| 146 | + vst1_u8(d + 2 * dst_stride, d2); | |
| 147 | + vst1_u8(d + 3 * dst_stride, d3); | |
| 148 | + | |
| 149 | + s += 8; | |
| 150 | + d += 8; | |
| 151 | + width -= 8; | |
| 152 | + } while (width > 0); | |
| 153 | + src += 4 * src_stride; | |
| 154 | + dst += 4 * dst_stride; | |
| 155 | + h -= 4; | |
| 156 | + } while (h > 0); | |
| 157 | + } | |
| 158 | +} | |
| 159 | + | |
| 160 | +#else | |
| 161 | + | |
| 55 | 162 | void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride, |
| 56 | 163 | uint8_t *dst, ptrdiff_t dst_stride, |
| 57 | 164 | const InterpKernel *filter, int x0_q4, |
| @@ -305,6 +412,8 @@ void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride, · 247 unmodified lines | ||
| 305 | 412 | } |
| 306 | 413 | } |
| 307 | 414 | |
| 415 | +#endif | |
| 416 | + | |
| 308 | 417 | void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride, |
| 309 | 418 | uint8_t *dst, ptrdiff_t dst_stride, |
| 310 | 419 | const InterpKernel *filter, int x0_q4, |
| @@ -72,6 +72,69 @@ static INLINE void load_u8_16x8(const uint8_t *s, const ptrdiff_t p, · 71 unmodified lines | ||
| 72 | 72 | *s7 = vld1q_u8(s); |
| 73 | 73 | } |
| 74 | 74 | |
| 75 | +#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ | |
| 76 | + (__ARM_FEATURE_DOTPROD == 1) | |
| 77 | + | |
| 78 | +static INLINE int32x4_t convolve8_4_dot(uint8x16_t samples, | |
| 79 | + const int8x8_t filters, | |
| 80 | + const int32x4_t correction, | |
| 81 | + const uint8x16_t range_limit, | |
| 82 | + const uint8x16x2_t permute_tbl) { | |
| 83 | + int8x16_t clamped_samples, permuted_samples[2]; | |
| 84 | + int32x4_t sum; | |
| 85 | + | |
| 86 | + /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */ | |
| 87 | + clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit)); | |
| 88 | + | |
| 89 | + /* Permute samples ready for dot product. */ | |
| 90 | + /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */ | |
| 91 | + permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]); | |
| 92 | + /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */ | |
| 93 | + permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]); | |
| 94 | + | |
| 95 | + /* Accumulate dot product into 'correction' to account for range clamp. */ | |
| 96 | + sum = vdotq_lane_s32(correction, permuted_samples[0], filters, 0); | |
| 97 | + sum = vdotq_lane_s32(sum, permuted_samples[1], filters, 1); | |
| 98 | + | |
| 99 | + /* Narrowing and packing is performed by the caller. */ | |
| 100 | + return sum; | |
| 101 | +} | |
| 102 | + | |
| 103 | +static INLINE uint8x8_t convolve8_8_dot(uint8x16_t samples, | |
| 104 | + const int8x8_t filters, | |
| 105 | + const int32x4_t correction, | |
| 106 | + const uint8x16_t range_limit, | |
| 107 | + const uint8x16x3_t permute_tbl) { | |
| 108 | + int8x16_t clamped_samples, permuted_samples[3]; | |
| 109 | + int32x4_t sum0, sum1; | |
| 110 | + int16x8_t sum; | |
| 111 | + | |
| 112 | + /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */ | |
| 113 | + clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit)); | |
| 114 | + | |
| 115 | + /* Permute samples ready for dot product. */ | |
| 116 | + /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */ | |
| 117 | + permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]); | |
| 118 | + /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */ | |
| 119 | + permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]); | |
| 120 | + /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */ | |
| 121 | + permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]); | |
| 122 | + | |
| 123 | + /* Accumulate dot product into 'correction' to account for range clamp. */ | |
| 124 | + /* First 4 output values. */ | |
| 125 | + sum0 = vdotq_lane_s32(correction, permuted_samples[0], filters, 0); | |
| 126 | + sum0 = vdotq_lane_s32(sum0, permuted_samples[1], filters, 1); | |
| 127 | + /* Second 4 output values. */ | |
| 128 | + sum1 = vdotq_lane_s32(correction, permuted_samples[1], filters, 0); | |
| 129 | + sum1 = vdotq_lane_s32(sum1, permuted_samples[2], filters, 1); | |
| 130 | + | |
| 131 | + /* Narrow and re-pack. */ | |
| 132 | + sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1)); | |
| 133 | + return vqrshrun_n_s16(sum, 7); | |
| 134 | +} | |
| 135 | + | |
| 136 | +#endif | |
| 137 | + | |
| 75 | 138 | static INLINE int16x4_t convolve8_4(const int16x4_t s0, const int16x4_t s1, |
| 76 | 139 | const int16x4_t s2, const int16x4_t s3, |
| 77 | 140 | const int16x4_t s4, const int16x4_t s5, |