History

Merge "Implement horizontal convolution using Neon SDOT instruction"

James Zern committed 5 years ago

Commit 7ef83142 parents 12a1491 c1f77a33 files · +190 −0Browse files

Unified · Split

3 files
  1. vpx_dsp/arm/mem_neon.h +18 −0
  2. vpx_dsp/arm/vpx_convolve8_neon.c +109 −0
  3. vpx_dsp/arm/vpx_convolve8_neon.h +63 −0
vpx_dsp/arm/mem_neon.h+18−0
@@ -19,6 +19,24 @@ · 18 unmodified lines
19#include "vpx/vpx_integer.h"19#include "vpx/vpx_integer.h"
20#include "vpx_dsp/vpx_dsp_common.h"20#include "vpx_dsp/vpx_dsp_common.h"
21 21
22// Support for these xN intrinsics is lacking in older versions of GCC.
23#if defined(__GNUC__) && !defined(__clang__)
24#if __GNUC__ < 8 || defined(__arm__)
25static INLINE uint8x16x2_t vld1q_u8_x2(uint8_t const *ptr) {
26 uint8x16x2_t res = { { vld1q_u8(ptr + 0 * 16), vld1q_u8(ptr + 1 * 16) } };
27 return res;
28}
29#endif
30
31#if __GNUC__ < 9 || defined(__arm__)
32static INLINE uint8x16x3_t vld1q_u8_x3(uint8_t const *ptr) {
33 uint8x16x3_t res = { { vld1q_u8(ptr + 0 * 16), vld1q_u8(ptr + 1 * 16),
34 vld1q_u8(ptr + 2 * 16) } };
35 return res;
36}
37#endif
38#endif
39
22static INLINE int16x4_t create_s16x4_neon(const int16_t c0, const int16_t c1,40static INLINE int16x4_t create_s16x4_neon(const int16_t c0, const int16_t c1,
23 const int16_t c2, const int16_t c3) {41 const int16_t c2, const int16_t c3) {
24 return vcreate_s16((uint16_t)c0 | ((uint32_t)c1 << 16) |42 return vcreate_s16((uint16_t)c0 | ((uint32_t)c1 << 16) |
vpx_dsp/arm/vpx_convolve8_neon.c+109−0
@@ -14,6 +14,7 @@ · 13 unmodified lines
14#include "./vpx_config.h"14#include "./vpx_config.h"
15#include "./vpx_dsp_rtcd.h"15#include "./vpx_dsp_rtcd.h"
16#include "vpx/vpx_integer.h"16#include "vpx/vpx_integer.h"
17#include "vpx_dsp/arm/mem_neon.h"
17#include "vpx_dsp/arm/transpose_neon.h"18#include "vpx_dsp/arm/transpose_neon.h"
18#include "vpx_dsp/arm/vpx_convolve8_neon.h"19#include "vpx_dsp/arm/vpx_convolve8_neon.h"
19#include "vpx_ports/mem.h"20#include "vpx_ports/mem.h"
@@ -52,6 +53,112 @@ static INLINE void store_u8_8x8(uint8_t *s, const ptrdiff_t p, · 32 unmodified lines
52 vst1_u8(s, s7);53 vst1_u8(s, s7);
53}54}
54 55
56#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
57 (__ARM_FEATURE_DOTPROD == 1)
58
59DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {
60 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
61 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
62 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
63};
64
65void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
66 uint8_t *dst, ptrdiff_t dst_stride,
67 const InterpKernel *filter, int x0_q4,
68 int x_step_q4, int y0_q4, int y_step_q4, int w,
69 int h) {
70 const int8x8_t filters = vmovn_s16(vld1q_s16(filter[x0_q4]));
71 const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[x0_q4]), 128);
72 const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
73 const uint8x16_t range_limit = vdupq_n_u8(128);
74 uint8x16_t s0, s1, s2, s3;
75
76 assert(!((intptr_t)dst & 3));
77 assert(!(dst_stride & 3));
78 assert(x_step_q4 == 16);
79
80 (void)x_step_q4;
81 (void)y0_q4;
82 (void)y_step_q4;
83
84 src -= 3;
85
86 if (w == 4) {
87 const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
88 do {
89 int32x4_t t0, t1, t2, t3;
90 int16x8_t t01, t23;
91 uint8x8_t d01, d23;
92
93 s0 = vld1q_u8(src);
94 src += src_stride;
95 s1 = vld1q_u8(src);
96 src += src_stride;
97 s2 = vld1q_u8(src);
98 src += src_stride;
99 s3 = vld1q_u8(src);
100 src += src_stride;
101
102 t0 = convolve8_4_dot(s0, filters, correction, range_limit, permute_tbl);
103 t1 = convolve8_4_dot(s1, filters, correction, range_limit, permute_tbl);
104 t2 = convolve8_4_dot(s2, filters, correction, range_limit, permute_tbl);
105 t3 = convolve8_4_dot(s3, filters, correction, range_limit, permute_tbl);
106
107 t01 = vcombine_s16(vqmovn_s32(t0), vqmovn_s32(t1));
108 t23 = vcombine_s16(vqmovn_s32(t2), vqmovn_s32(t3));
109 d01 = vqrshrun_n_s16(t01, 7);
110 d23 = vqrshrun_n_s16(t23, 7);
111
112 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
113 dst += dst_stride;
114 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
115 dst += dst_stride;
116 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
117 dst += dst_stride;
118 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
119 dst += dst_stride;
120 h -= 4;
121 } while (h > 0);
122 } else {
123 const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
124 const uint8_t *s;
125 uint8_t *d;
126 int width;
127 uint8x8_t d0, d1, d2, d3;
128
129 do {
130 width = w;
131 s = src;
132 d = dst;
133 do {
134 s0 = vld1q_u8(s + 0 * src_stride);
135 s1 = vld1q_u8(s + 1 * src_stride);
136 s2 = vld1q_u8(s + 2 * src_stride);
137 s3 = vld1q_u8(s + 3 * src_stride);
138
139 d0 = convolve8_8_dot(s0, filters, correction, range_limit, permute_tbl);
140 d1 = convolve8_8_dot(s1, filters, correction, range_limit, permute_tbl);
141 d2 = convolve8_8_dot(s2, filters, correction, range_limit, permute_tbl);
142 d3 = convolve8_8_dot(s3, filters, correction, range_limit, permute_tbl);
143
144 vst1_u8(d + 0 * dst_stride, d0);
145 vst1_u8(d + 1 * dst_stride, d1);
146 vst1_u8(d + 2 * dst_stride, d2);
147 vst1_u8(d + 3 * dst_stride, d3);
148
149 s += 8;
150 d += 8;
151 width -= 8;
152 } while (width > 0);
153 src += 4 * src_stride;
154 dst += 4 * dst_stride;
155 h -= 4;
156 } while (h > 0);
157 }
158}
159
160#else
161
55void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,162void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
56 uint8_t *dst, ptrdiff_t dst_stride,163 uint8_t *dst, ptrdiff_t dst_stride,
57 const InterpKernel *filter, int x0_q4,164 const InterpKernel *filter, int x0_q4,
@@ -305,6 +412,8 @@ void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride, · 247 unmodified lines
305 }412 }
306}413}
307 414
415#endif
416
308void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,417void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
309 uint8_t *dst, ptrdiff_t dst_stride,418 uint8_t *dst, ptrdiff_t dst_stride,
310 const InterpKernel *filter, int x0_q4,419 const InterpKernel *filter, int x0_q4,
vpx_dsp/arm/vpx_convolve8_neon.h+63−0
@@ -72,6 +72,69 @@ static INLINE void load_u8_16x8(const uint8_t *s, const ptrdiff_t p, · 71 unmodified lines
72 *s7 = vld1q_u8(s);72 *s7 = vld1q_u8(s);
73}73}
74 74
75#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
76 (__ARM_FEATURE_DOTPROD == 1)
77
78static INLINE int32x4_t convolve8_4_dot(uint8x16_t samples,
79 const int8x8_t filters,
80 const int32x4_t correction,
81 const uint8x16_t range_limit,
82 const uint8x16x2_t permute_tbl) {
83 int8x16_t clamped_samples, permuted_samples[2];
84 int32x4_t sum;
85
86 /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
87 clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
88
89 /* Permute samples ready for dot product. */
90 /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
91 permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
92 /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
93 permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
94
95 /* Accumulate dot product into 'correction' to account for range clamp. */
96 sum = vdotq_lane_s32(correction, permuted_samples[0], filters, 0);
97 sum = vdotq_lane_s32(sum, permuted_samples[1], filters, 1);
98
99 /* Narrowing and packing is performed by the caller. */
100 return sum;
101}
102
103static INLINE uint8x8_t convolve8_8_dot(uint8x16_t samples,
104 const int8x8_t filters,
105 const int32x4_t correction,
106 const uint8x16_t range_limit,
107 const uint8x16x3_t permute_tbl) {
108 int8x16_t clamped_samples, permuted_samples[3];
109 int32x4_t sum0, sum1;
110 int16x8_t sum;
111
112 /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
113 clamped_samples = vreinterpretq_s8_u8(vsubq_u8(samples, range_limit));
114
115 /* Permute samples ready for dot product. */
116 /* { 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 } */
117 permuted_samples[0] = vqtbl1q_s8(clamped_samples, permute_tbl.val[0]);
118 /* { 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10 } */
119 permuted_samples[1] = vqtbl1q_s8(clamped_samples, permute_tbl.val[1]);
120 /* { 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14 } */
121 permuted_samples[2] = vqtbl1q_s8(clamped_samples, permute_tbl.val[2]);
122
123 /* Accumulate dot product into 'correction' to account for range clamp. */
124 /* First 4 output values. */
125 sum0 = vdotq_lane_s32(correction, permuted_samples[0], filters, 0);
126 sum0 = vdotq_lane_s32(sum0, permuted_samples[1], filters, 1);
127 /* Second 4 output values. */
128 sum1 = vdotq_lane_s32(correction, permuted_samples[1], filters, 0);
129 sum1 = vdotq_lane_s32(sum1, permuted_samples[2], filters, 1);
130
131 /* Narrow and re-pack. */
132 sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
133 return vqrshrun_n_s16(sum, 7);
134}
135
136#endif
137
75static INLINE int16x4_t convolve8_4(const int16x4_t s0, const int16x4_t s1,138static INLINE int16x4_t convolve8_4(const int16x4_t s0, const int16x4_t s1,
76 const int16x4_t s2, const int16x4_t s3,139 const int16x4_t s2, const int16x4_t s3,
77 const int16x4_t s4, const int16x4_t s5,140 const int16x4_t s4, const int16x4_t s5,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t