Merge changes I2e86b005,I971c6261,I87fe4dad

* changes: Use 'ptrdiff_t' instead of 'int' for pointer offset parameters Implement vpx_convolve8_avg_vert_neon using SDOT instruction Merge transpose and permute in Neon SDOT vertical convolution

James Zern committed 5 years ago

Commit 76a4e442 parents 2c6a617 dbda0323 files · +589 −215Browse files

Unified · Split

3 files

vpx_dsp/arm/mem_neon.h +8 −6
vpx_dsp/arm/vpx_convolve8_neon.c +543 −209
vpx_dsp/arm/vpx_convolve8_neon.h +38 −0

vpx_dsp/arm/mem_neon.h+8−6

@@ -113,7 +113,8 @@ static INLINE void uint32_to_mem(uint8_t *buf, uint32_t a) { · 112 unmodified lines
113	113	}
114	114
115	115	// Load 2 sets of 4 bytes when alignment is not guaranteed.
116		−static INLINE uint8x8_t load_unaligned_u8(const uint8_t *buf, int stride) {
	116	+static INLINE uint8x8_t load_unaligned_u8(const uint8_t *buf,
	117	+ ptrdiff_t stride) {
117	118	uint32_t a;
118	119	uint32x2_t a_u32 = vdup_n_u32(0);
119	120	if (stride == 4) return vld1_u8(buf);
@@ -126,7 +127,7 @@ static INLINE uint8x8_t load_unaligned_u8(const uint8_t *buf, int stride) { · 6 unmodified lines
126	127	}
127	128
128	129	// Store 2 sets of 4 bytes when alignment is not guaranteed.
129		−static INLINE void store_unaligned_u8(uint8_t *buf, int stride,
	130	+static INLINE void store_unaligned_u8(uint8_t *buf, ptrdiff_t stride,
130	131	const uint8x8_t a) {
131	132	const uint32x2_t a_u32 = vreinterpret_u32_u8(a);
132	133	if (stride == 4) {
@@ -139,7 +140,8 @@ static INLINE void store_unaligned_u8(uint8_t *buf, int stride, · 6 unmodified lines
139	140	}
140	141
141	142	// Load 4 sets of 4 bytes when alignment is not guaranteed.
142		−static INLINE uint8x16_t load_unaligned_u8q(const uint8_t *buf, int stride) {
	143	+static INLINE uint8x16_t load_unaligned_u8q(const uint8_t *buf,
	144	+ ptrdiff_t stride) {
143	145	uint32_t a;
144	146	uint32x4_t a_u32 = vdupq_n_u32(0);
145	147	if (stride == 4) return vld1q_u8(buf);
@@ -159,7 +161,7 @@ static INLINE uint8x16_t load_unaligned_u8q(const uint8_t *buf, int stride) { · 13 unmodified lines
159	161	}
160	162
161	163	// Store 4 sets of 4 bytes when alignment is not guaranteed.
162		−static INLINE void store_unaligned_u8q(uint8_t *buf, int stride,
	164	+static INLINE void store_unaligned_u8q(uint8_t *buf, ptrdiff_t stride,
163	165	const uint8x16_t a) {
164	166	const uint32x4_t a_u32 = vreinterpretq_u32_u8(a);
165	167	if (stride == 4) {
@@ -176,7 +178,7 @@ static INLINE void store_unaligned_u8q(uint8_t *buf, int stride, · 10 unmodified lines
176	178	}
177	179
178	180	// Load 2 sets of 4 bytes when alignment is guaranteed.
179		−static INLINE uint8x8_t load_u8(const uint8_t *buf, int stride) {
	181	+static INLINE uint8x8_t load_u8(const uint8_t *buf, ptrdiff_t stride) {
180	182	uint32x2_t a = vdup_n_u32(0);
181	183
182	184	assert(!((intptr_t)buf % sizeof(uint32_t)));
@@ -189,7 +191,7 @@ static INLINE uint8x8_t load_u8(const uint8_t *buf, int stride) { · 6 unmodified lines
189	191	}
190	192
191	193	// Store 2 sets of 4 bytes when alignment is guaranteed.
192		−static INLINE void store_u8(uint8_t *buf, int stride, const uint8x8_t a) {
	194	+static INLINE void store_u8(uint8_t *buf, ptrdiff_t stride, const uint8x8_t a) {
193	195	uint32x2_t a_u32 = vreinterpret_u32_u8(a);
194	196
195	197	assert(!((intptr_t)buf % sizeof(uint32_t)));

vpx_dsp/arm/vpx_convolve8_neon.c+543−209

@@ -31,37 +31,72 @@ · 30 unmodified lines
31	31	// instructions. This optimization is much faster in speed unit test, but slowed
32	32	// down the whole decoder by 5%.
33	33
34		−static INLINE void store_u8_8x8(uint8_t *s, const ptrdiff_t p,
35		− const uint8x8_t s0, const uint8x8_t s1,
36		− const uint8x8_t s2, const uint8x8_t s3,
37		− const uint8x8_t s4, const uint8x8_t s5,
38		− const uint8x8_t s6, const uint8x8_t s7) {
39		− vst1_u8(s, s0);
40		− s += p;
41		− vst1_u8(s, s1);
42		− s += p;
43		− vst1_u8(s, s2);
44		− s += p;
45		− vst1_u8(s, s3);
46		− s += p;
47		− vst1_u8(s, s4);
48		− s += p;
49		− vst1_u8(s, s5);
50		− s += p;
51		− vst1_u8(s, s6);
52		− s += p;
53		− vst1_u8(s, s7);
54		−}
55		−
56	34	#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
57	35	(__ARM_FEATURE_DOTPROD == 1)
58		−
59	36	DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {
60	37	0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
61	38	4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
62	39	8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
63	40	};
64	41
	42	+DECLARE_ALIGNED(16, static const uint8_t, dot_prod_tran_concat_tbl[32]) = {
	43	+ 0, 8, 16, 24, 1, 9, 17, 25, 2, 10, 18, 26, 3, 11, 19, 27,
	44	+ 4, 12, 20, 28, 5, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31
	45	+};
	46	+
	47	+DECLARE_ALIGNED(16, static const uint8_t, dot_prod_merge_block_tbl[48]) = {
	48	+ /* Shift left and insert new last column in transposed 4x4 block. */
	49	+ 1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28,
	50	+ /* Shift left and insert two new columns in transposed 4x4 block. */
	51	+ 2, 3, 16, 17, 6, 7, 20, 21, 10, 11, 24, 25, 14, 15, 28, 29,
	52	+ /* Shift left and insert three new columns in transposed 4x4 block. */
	53	+ 3, 16, 17, 18, 7, 20, 21, 22, 11, 24, 25, 26, 15, 28, 29, 30
	54	+};
	55	+
	56	+static INLINE void transpose_concat_4x4(int8x8_t a0, int8x8_t a1,
	57	+ int8x8_t a2, int8x8_t a3,
	58	+ int8x16_t *b,
	59	+ const uint8x16_t permute_tbl) {
	60	+ /* Transpose 8-bit elements and concatenate result rows as follows:
	61	+ * a0: 00, 01, 02, 03, XX, XX, XX, XX
	62	+ * a1: 10, 11, 12, 13, XX, XX, XX, XX
	63	+ * a2: 20, 21, 22, 23, XX, XX, XX, XX
	64	+ * a3: 30, 31, 32, 33, XX, XX, XX, XX
	65	+ *
	66	+ * b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
	67	+ *
	68	+ * The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
	69	+ * as an argument is preferable to loading it directly from memory as this
	70	+ * inline helper is called many times from the same parent function.
	71	+ */
	72	+
	73	+ int8x16x2_t samples = { { vcombine_s8(a0, a1), vcombine_s8(a2, a3) } };
	74	+ *b = vqtbl2q_s8(samples, permute_tbl);
	75	+}
	76	+
	77	+static INLINE void transpose_concat_8x4(int8x8_t a0, int8x8_t a1,
	78	+ int8x8_t a2, int8x8_t a3,
	79	+ int8x16_t b0, int8x16_t b1,
	80	+ const uint8x16x2_t permute_tbl) {
	81	+ /* Transpose 8-bit elements and concatenate result rows as follows:
	82	+ * a0: 00, 01, 02, 03, 04, 05, 06, 07
	83	+ * a1: 10, 11, 12, 13, 14, 15, 16, 17
	84	+ * a2: 20, 21, 22, 23, 24, 25, 26, 27
	85	+ * a3: 30, 31, 32, 33, 34, 35, 36, 37
	86	+ *
	87	+ * b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
	88	+ * b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37
	89	+ *
	90	+ * The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
	91	+ * as an argument is preferable to loading it directly from memory as this
	92	+ * inline helper is called many times from the same parent function.
	93	+ */
	94	+
	95	+ int8x16x2_t samples = { { vcombine_s8(a0, a1), vcombine_s8(a2, a3) } };
	96	+ *b0 = vqtbl2q_s8(samples, permute_tbl.val[0]);
	97	+ *b1 = vqtbl2q_s8(samples, permute_tbl.val[1]);
	98	+}
	99	+
65	100	void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
66	101	uint8_t *dst, ptrdiff_t dst_stride,
67	102	const InterpKernel *filter, int x0_q4,
@@ -268,8 +303,491 @@ void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride, · 200 unmodified lines
268	303	}
269	304	}
270	305
	306	+void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
	307	+ uint8_t *dst, ptrdiff_t dst_stride,
	308	+ const InterpKernel *filter, int x0_q4,
	309	+ int x_step_q4, int y0_q4, int y_step_q4, int w,
	310	+ int h) {
	311	+ const int8x8_t filters = vmovn_s16(vld1q_s16(filter[y0_q4]));
	312	+ const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[y0_q4]), 128);
	313	+ const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
	314	+ const uint8x8_t range_limit = vdup_n_u8(128);
	315	+ const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(dot_prod_merge_block_tbl);
	316	+ uint8x8_t t0, t1, t2, t3, t4, t5, t6;
	317	+ int8x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
	318	+ int8x16x2_t samples_LUT;
	319	+
	320	+ assert(!((intptr_t)dst & 3));
	321	+ assert(!(dst_stride & 3));
	322	+ assert(y_step_q4 == 16);
	323	+
	324	+ (void)x0_q4;
	325	+ (void)x_step_q4;
	326	+ (void)y_step_q4;
	327	+
	328	+ src -= 3 * src_stride;
	329	+
	330	+ if (w == 4) {
	331	+ const uint8x16_t tran_concat_tbl = vld1q_u8(dot_prod_tran_concat_tbl);
	332	+ int8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s78910;
	333	+ int32x4_t d0, d1, d2, d3;
	334	+ uint8x8_t d01, d23;
	335	+
	336	+ load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
	337	+ src += 4 * src_stride;
	338	+ t4 = vld1_u8(src);
	339	+ src += src_stride;
	340	+ t5 = vld1_u8(src);
	341	+ src += src_stride;
	342	+ t6 = vld1_u8(src);
	343	+ src += src_stride;
	344	+
	345	+ /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
	346	+ s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
	347	+ s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
	348	+ s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
	349	+ s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
	350	+ s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
	351	+ s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
	352	+ s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
	353	+ s7 = vdup_n_s8(0);
	354	+ s8 = vdup_n_s8(0);
	355	+ s9 = vdup_n_s8(0);
	356	+
	357	+ /* This operation combines a conventional transpose and the sample permute
	358	+ * (see horizontal case) required before computing the dot product.
	359	+ */
	360	+ transpose_concat_4x4(&s0, &s1, &s2, &s3, &s0123, tran_concat_tbl);
	361	+ transpose_concat_4x4(&s1, &s2, &s3, &s4, &s1234, tran_concat_tbl);
	362	+ transpose_concat_4x4(&s2, &s3, &s4, &s5, &s2345, tran_concat_tbl);
	363	+ transpose_concat_4x4(&s3, &s4, &s5, &s6, &s3456, tran_concat_tbl);
	364	+ transpose_concat_4x4(&s4, &s5, &s6, &s7, &s4567, tran_concat_tbl);
	365	+ transpose_concat_4x4(&s5, &s6, &s7, &s8, &s5678, tran_concat_tbl);
	366	+ transpose_concat_4x4(&s6, &s7, &s8, &s9, &s6789, tran_concat_tbl);
	367	+
	368	+ do {
	369	+ uint8x8_t t7, t8, t9, t10;
	370	+
	371	+ load_u8_8x4(src, src_stride, &t7, &t8, &t9, &t10);
	372	+
	373	+ s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
	374	+ s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
	375	+ s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
	376	+ s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
	377	+
	378	+ transpose_concat_4x4(&s7, &s8, &s9, &s10, &s78910, tran_concat_tbl);
	379	+
	380	+ /* Merge new data into block from previous iteration. */
	381	+ samples_LUT.val[0] = s3456;
	382	+ samples_LUT.val[1] = s78910;
	383	+ s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
	384	+ s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
	385	+ s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
	386	+
	387	+ d0 = convolve8_4_dot_partial(s0123, s4567, correction, filters);
	388	+ d1 = convolve8_4_dot_partial(s1234, s5678, correction, filters);
	389	+ d2 = convolve8_4_dot_partial(s2345, s6789, correction, filters);
	390	+ d3 = convolve8_4_dot_partial(s3456, s78910, correction, filters);
	391	+
	392	+ d01 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d1)), 7);
	393	+ d23 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d3)), 7);
	394	+
	395	+ vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
	396	+ dst += dst_stride;
	397	+ vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
	398	+ dst += dst_stride;
	399	+ vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
	400	+ dst += dst_stride;
	401	+ vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
	402	+ dst += dst_stride;
	403	+
	404	+ /* Prepare block for next iteration - re-using as much as possible. */
	405	+ /* Shuffle everything up four rows. */
	406	+ s0123 = s4567;
	407	+ s1234 = s5678;
	408	+ s2345 = s6789;
	409	+ s3456 = s78910;
	410	+
	411	+ src += 4 * src_stride;
	412	+ h -= 4;
	413	+ } while (h > 0);
	414	+ } else {
	415	+ const uint8x16x2_t tran_concat_tbl = vld1q_u8_x2(dot_prod_tran_concat_tbl);
	416	+ int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi,
	417	+ s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo,
	418	+ s6789_hi, s78910_lo, s78910_hi;
	419	+ uint8x8_t d0, d1, d2, d3;
	420	+ const uint8_t *s;
	421	+ uint8_t *d;
	422	+ int height;
	423	+
	424	+ do {
	425	+ height = h;
	426	+ s = src;
	427	+ d = dst;
	428	+
	429	+ load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
	430	+ s += 4 * src_stride;
	431	+ t4 = vld1_u8(s);
	432	+ s += src_stride;
	433	+ t5 = vld1_u8(s);
	434	+ s += src_stride;
	435	+ t6 = vld1_u8(s);
	436	+ s += src_stride;
	437	+
	438	+ /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
	439	+ s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
	440	+ s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
	441	+ s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
	442	+ s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
	443	+ s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
	444	+ s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
	445	+ s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
	446	+ s7 = vdup_n_s8(0);
	447	+ s8 = vdup_n_s8(0);
	448	+ s9 = vdup_n_s8(0);
	449	+
	450	+ /* This operation combines a conventional transpose and the sample permute
	451	+ * (see horizontal case) required before computing the dot product.
	452	+ */
	453	+ transpose_concat_8x4(&s0, &s1, &s2, &s3, &s0123_lo, &s0123_hi,
	454	+ tran_concat_tbl);
	455	+ transpose_concat_8x4(&s1, &s2, &s3, &s4, &s1234_lo, &s1234_hi,
	456	+ tran_concat_tbl);
	457	+ transpose_concat_8x4(&s2, &s3, &s4, &s5, &s2345_lo, &s2345_hi,
	458	+ tran_concat_tbl);
	459	+ transpose_concat_8x4(&s3, &s4, &s5, &s6, &s3456_lo, &s3456_hi,
	460	+ tran_concat_tbl);
	461	+ transpose_concat_8x4(&s4, &s5, &s6, &s7, &s4567_lo, &s4567_hi,
	462	+ tran_concat_tbl);
	463	+ transpose_concat_8x4(&s5, &s6, &s7, &s8, &s5678_lo, &s5678_hi,
	464	+ tran_concat_tbl);
	465	+ transpose_concat_8x4(&s6, &s7, &s8, &s9, &s6789_lo, &s6789_hi,
	466	+ tran_concat_tbl);
	467	+
	468	+ do {
	469	+ uint8x8_t t7, t8, t9, t10;
	470	+
	471	+ load_u8_8x4(s, src_stride, &t7, &t8, &t9, &t10);
	472	+
	473	+ s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
	474	+ s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
	475	+ s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
	476	+ s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
	477	+
	478	+ transpose_concat_8x4(&s7, &s8, &s9, &s10, &s78910_lo, &s78910_hi,
	479	+ tran_concat_tbl);
	480	+
	481	+ /* Merge new data into block from previous iteration. */
	482	+ samples_LUT.val[0] = s3456_lo;
	483	+ samples_LUT.val[1] = s78910_lo;
	484	+ s4567_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
	485	+ s5678_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
	486	+ s6789_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
	487	+
	488	+ samples_LUT.val[0] = s3456_hi;
	489	+ samples_LUT.val[1] = s78910_hi;
	490	+ s4567_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
	491	+ s5678_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
	492	+ s6789_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
	493	+
	494	+ d0 = convolve8_8_dot_partial(s0123_lo, s4567_lo, s0123_hi, s4567_hi,
	495	+ correction, filters);
	496	+ d1 = convolve8_8_dot_partial(s1234_lo, s5678_lo, s1234_hi, s5678_hi,
	497	+ correction, filters);
	498	+ d2 = convolve8_8_dot_partial(s2345_lo, s6789_lo, s2345_hi, s6789_hi,
	499	+ correction, filters);
	500	+ d3 = convolve8_8_dot_partial(s3456_lo, s78910_lo, s3456_hi, s78910_hi,
	501	+ correction, filters);
	502	+ vst1_u8(d + 0 * dst_stride, d0);
	503	+ vst1_u8(d + 1 * dst_stride, d1);
	504	+ vst1_u8(d + 2 * dst_stride, d2);
	505	+ vst1_u8(d + 3 * dst_stride, d3);
	506	+
	507	+ /* Prepare block for next iteration - re-using as much as possible. */
	508	+ /* Shuffle everything up four rows. */
	509	+ s0123_lo = s4567_lo;
	510	+ s0123_hi = s4567_hi;
	511	+ s1234_lo = s5678_lo;
	512	+ s1234_hi = s5678_hi;
	513	+ s2345_lo = s6789_lo;
	514	+ s2345_hi = s6789_hi;
	515	+ s3456_lo = s78910_lo;
	516	+ s3456_hi = s78910_hi;
	517	+
	518	+ s += 4 * src_stride;
	519	+ d += 4 * dst_stride;
	520	+ height -= 4;
	521	+ } while (height > 0);
	522	+ src += 8;
	523	+ dst += 8;
	524	+ w -= 8;
	525	+ } while (w > 0);
	526	+ }
	527	+}
	528	+
	529	+void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
	530	+ uint8_t *dst, ptrdiff_t dst_stride,
	531	+ const InterpKernel *filter, int x0_q4,
	532	+ int x_step_q4, int y0_q4, int y_step_q4, int w,
	533	+ int h) {
	534	+ const int8x8_t filters = vmovn_s16(vld1q_s16(filter[y0_q4]));
	535	+ const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[y0_q4]), 128);
	536	+ const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
	537	+ const uint8x8_t range_limit = vdup_n_u8(128);
	538	+ const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(dot_prod_merge_block_tbl);
	539	+ uint8x8_t t0, t1, t2, t3, t4, t5, t6;
	540	+ int8x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
	541	+ int8x16x2_t samples_LUT;
	542	+
	543	+ assert(!((intptr_t)dst & 3));
	544	+ assert(!(dst_stride & 3));
	545	+ assert(y_step_q4 == 16);
	546	+
	547	+ (void)x0_q4;
	548	+ (void)x_step_q4;
	549	+ (void)y_step_q4;
	550	+
	551	+ src -= 3 * src_stride;
	552	+
	553	+ if (w == 4) {
	554	+ const uint8x16_t tran_concat_tbl = vld1q_u8(dot_prod_tran_concat_tbl);
	555	+ int8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s78910;
	556	+ int32x4_t d0, d1, d2, d3;
	557	+ uint8x8_t d01, d23, dd01, dd23;
	558	+
	559	+ load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
	560	+ src += 4 * src_stride;
	561	+ t4 = vld1_u8(src);
	562	+ src += src_stride;
	563	+ t5 = vld1_u8(src);
	564	+ src += src_stride;
	565	+ t6 = vld1_u8(src);
	566	+ src += src_stride;
	567	+
	568	+ /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
	569	+ s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
	570	+ s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
	571	+ s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
	572	+ s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
	573	+ s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
	574	+ s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
	575	+ s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
	576	+ s7 = vdup_n_s8(0);
	577	+ s8 = vdup_n_s8(0);
	578	+ s9 = vdup_n_s8(0);
	579	+
	580	+ /* This operation combines a conventional transpose and the sample permute
	581	+ * (see horizontal case) required before computing the dot product.
	582	+ */
	583	+ transpose_concat_4x4(&s0, &s1, &s2, &s3, &s0123, tran_concat_tbl);
	584	+ transpose_concat_4x4(&s1, &s2, &s3, &s4, &s1234, tran_concat_tbl);
	585	+ transpose_concat_4x4(&s2, &s3, &s4, &s5, &s2345, tran_concat_tbl);
	586	+ transpose_concat_4x4(&s3, &s4, &s5, &s6, &s3456, tran_concat_tbl);
	587	+ transpose_concat_4x4(&s4, &s5, &s6, &s7, &s4567, tran_concat_tbl);
	588	+ transpose_concat_4x4(&s5, &s6, &s7, &s8, &s5678, tran_concat_tbl);
	589	+ transpose_concat_4x4(&s6, &s7, &s8, &s9, &s6789, tran_concat_tbl);
	590	+
	591	+ do {
	592	+ uint8x8_t t7, t8, t9, t10;
	593	+
	594	+ load_u8_8x4(src, src_stride, &t7, &t8, &t9, &t10);
	595	+
	596	+ s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
	597	+ s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
	598	+ s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
	599	+ s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
	600	+
	601	+ transpose_concat_4x4(&s7, &s8, &s9, &s10, &s78910, tran_concat_tbl);
	602	+
	603	+ /* Merge new data into block from previous iteration. */
	604	+ samples_LUT.val[0] = s3456;
	605	+ samples_LUT.val[1] = s78910;
	606	+ s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
	607	+ s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
	608	+ s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
	609	+
	610	+ d0 = convolve8_4_dot_partial(s0123, s4567, correction, filters);
	611	+ d1 = convolve8_4_dot_partial(s1234, s5678, correction, filters);
	612	+ d2 = convolve8_4_dot_partial(s2345, s6789, correction, filters);
	613	+ d3 = convolve8_4_dot_partial(s3456, s78910, correction, filters);
	614	+
	615	+ d01 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d1)), 7);
	616	+ d23 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d3)), 7);
	617	+
	618	+ dd01 = load_u8(dst + 0 * dst_stride, dst_stride);
	619	+ dd23 = load_u8(dst + 2 * dst_stride, dst_stride);
	620	+ d01 = vrhadd_u8(d01, dd01);
	621	+ d23 = vrhadd_u8(d23, dd23);
	622	+
	623	+ vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
	624	+ dst += dst_stride;
	625	+ vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
	626	+ dst += dst_stride;
	627	+ vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
	628	+ dst += dst_stride;
	629	+ vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
	630	+ dst += dst_stride;
	631	+
	632	+ /* Prepare block for next iteration - re-using as much as possible. */
	633	+ /* Shuffle everything up four rows. */
	634	+ s0123 = s4567;
	635	+ s1234 = s5678;
	636	+ s2345 = s6789;
	637	+ s3456 = s78910;
	638	+
	639	+ src += 4 * src_stride;
	640	+ h -= 4;
	641	+ } while (h > 0);
	642	+ } else {
	643	+ const uint8x16x2_t tran_concat_tbl = vld1q_u8_x2(dot_prod_tran_concat_tbl);
	644	+ int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi,
	645	+ s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo,
	646	+ s6789_hi, s78910_lo, s78910_hi;
	647	+ uint8x8_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
	648	+ const uint8_t *s;
	649	+ uint8_t *d;
	650	+ int height;
	651	+
	652	+ do {
	653	+ height = h;
	654	+ s = src;
	655	+ d = dst;
	656	+
	657	+ load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
	658	+ s += 4 * src_stride;
	659	+ t4 = vld1_u8(s);
	660	+ s += src_stride;
	661	+ t5 = vld1_u8(s);
	662	+ s += src_stride;
	663	+ t6 = vld1_u8(s);
	664	+ s += src_stride;
	665	+
	666	+ /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
	667	+ s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
	668	+ s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
	669	+ s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
	670	+ s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
	671	+ s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
	672	+ s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
	673	+ s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
	674	+ s7 = vdup_n_s8(0);
	675	+ s8 = vdup_n_s8(0);
	676	+ s9 = vdup_n_s8(0);
	677	+
	678	+ /* This operation combines a conventional transpose and the sample permute
	679	+ * (see horizontal case) required before computing the dot product.
	680	+ */
	681	+ transpose_concat_8x4(&s0, &s1, &s2, &s3, &s0123_lo, &s0123_hi,
	682	+ tran_concat_tbl);
	683	+ transpose_concat_8x4(&s1, &s2, &s3, &s4, &s1234_lo, &s1234_hi,
	684	+ tran_concat_tbl);
	685	+ transpose_concat_8x4(&s2, &s3, &s4, &s5, &s2345_lo, &s2345_hi,
	686	+ tran_concat_tbl);
	687	+ transpose_concat_8x4(&s3, &s4, &s5, &s6, &s3456_lo, &s3456_hi,
	688	+ tran_concat_tbl);
	689	+ transpose_concat_8x4(&s4, &s5, &s6, &s7, &s4567_lo, &s4567_hi,
	690	+ tran_concat_tbl);
	691	+ transpose_concat_8x4(&s5, &s6, &s7, &s8, &s5678_lo, &s5678_hi,
	692	+ tran_concat_tbl);
	693	+ transpose_concat_8x4(&s6, &s7, &s8, &s9, &s6789_lo, &s6789_hi,
	694	+ tran_concat_tbl);
	695	+
	696	+ do {
	697	+ uint8x8_t t7, t8, t9, t10;
	698	+
	699	+ load_u8_8x4(s, src_stride, &t7, &t8, &t9, &t10);
	700	+
	701	+ s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
	702	+ s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
	703	+ s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
	704	+ s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
	705	+
	706	+ transpose_concat_8x4(&s7, &s8, &s9, &s10, &s78910_lo, &s78910_hi,
	707	+ tran_concat_tbl);
	708	+
	709	+ /* Merge new data into block from previous iteration. */
	710	+ samples_LUT.val[0] = s3456_lo;
	711	+ samples_LUT.val[1] = s78910_lo;
	712	+ s4567_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
	713	+ s5678_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
	714	+ s6789_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
	715	+
	716	+ samples_LUT.val[0] = s3456_hi;
	717	+ samples_LUT.val[1] = s78910_hi;
	718	+ s4567_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
	719	+ s5678_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
	720	+ s6789_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
	721	+
	722	+ d0 = convolve8_8_dot_partial(s0123_lo, s4567_lo, s0123_hi, s4567_hi,
	723	+ correction, filters);
	724	+ d1 = convolve8_8_dot_partial(s1234_lo, s5678_lo, s1234_hi, s5678_hi,
	725	+ correction, filters);
	726	+ d2 = convolve8_8_dot_partial(s2345_lo, s6789_lo, s2345_hi, s6789_hi,
	727	+ correction, filters);
	728	+ d3 = convolve8_8_dot_partial(s3456_lo, s78910_lo, s3456_hi, s78910_hi,
	729	+ correction, filters);
	730	+
	731	+ dd0 = vld1_u8(d + 0 * dst_stride);
	732	+ dd1 = vld1_u8(d + 1 * dst_stride);
	733	+ dd2 = vld1_u8(d + 2 * dst_stride);
	734	+ dd3 = vld1_u8(d + 3 * dst_stride);
	735	+ d0 = vrhadd_u8(d0, dd0);
	736	+ d1 = vrhadd_u8(d1, dd1);
	737	+ d2 = vrhadd_u8(d2, dd2);
	738	+ d3 = vrhadd_u8(d3, dd3);
	739	+
	740	+ vst1_u8(d + 0 * dst_stride, d0);
	741	+ vst1_u8(d + 1 * dst_stride, d1);
	742	+ vst1_u8(d + 2 * dst_stride, d2);
	743	+ vst1_u8(d + 3 * dst_stride, d3);
	744	+
	745	+ /* Prepare block for next iteration - re-using as much as possible. */
	746	+ /* Shuffle everything up four rows. */
	747	+ s0123_lo = s4567_lo;
	748	+ s0123_hi = s4567_hi;
	749	+ s1234_lo = s5678_lo;
	750	+ s1234_hi = s5678_hi;
	751	+ s2345_lo = s6789_lo;
	752	+ s2345_hi = s6789_hi;
	753	+ s3456_lo = s78910_lo;
	754	+ s3456_hi = s78910_hi;
	755	+
	756	+ s += 4 * src_stride;
	757	+ d += 4 * dst_stride;
	758	+ height -= 4;
	759	+ } while (height > 0);
	760	+ src += 8;
	761	+ dst += 8;
	762	+ w -= 8;
	763	+ } while (w > 0);
	764	+ }
	765	+}
	766	+
271	767	#else
272	768
	769	+static INLINE void store_u8_8x8(uint8_t *s, const ptrdiff_t p,
	770	+ const uint8x8_t s0, const uint8x8_t s1,
	771	+ const uint8x8_t s2, const uint8x8_t s3,
	772	+ const uint8x8_t s4, const uint8x8_t s5,
	773	+ const uint8x8_t s6, const uint8x8_t s7) {
	774	+ vst1_u8(s, s0);
	775	+ s += p;
	776	+ vst1_u8(s, s1);
	777	+ s += p;
	778	+ vst1_u8(s, s2);
	779	+ s += p;
	780	+ vst1_u8(s, s3);
	781	+ s += p;
	782	+ vst1_u8(s, s4);
	783	+ s += p;
	784	+ vst1_u8(s, s5);
	785	+ s += p;
	786	+ vst1_u8(s, s6);
	787	+ s += p;
	788	+ vst1_u8(s, s7);
	789	+}
	790	+
273	791	void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
274	792	uint8_t *dst, ptrdiff_t dst_stride,
275	793	const InterpKernel *filter, int x0_q4,
@@ -813,190 +1331,6 @@ void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride, · 537 unmodified lines
813	1331	}
814	1332	}
815	1333
816		−#endif
817		−
818		−#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
819		− (__ARM_FEATURE_DOTPROD == 1)
820		−
821		−void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
822		− uint8_t *dst, ptrdiff_t dst_stride,
823		− const InterpKernel *filter, int x0_q4,
824		− int x_step_q4, int y0_q4, int y_step_q4, int w,
825		− int h) {
826		− const int8x8_t filters = vmovn_s16(vld1q_s16(filter[y0_q4]));
827		− const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[y0_q4]), 128);
828		− const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
829		− const uint8x16_t range_limit = vdupq_n_u8(128);
830		−
831		− assert(!((intptr_t)dst & 3));
832		− assert(!(dst_stride & 3));
833		− assert(y_step_q4 == 16);
834		−
835		− (void)x0_q4;
836		− (void)x_step_q4;
837		− (void)y_step_q4;
838		−
839		− src -= 3 * src_stride;
840		−
841		− if (w == 4) {
842		− const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
843		− uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, d01, d23;
844		− uint8x16_t s0, s1, s2, s3;
845		− int32x4_t d0, d1, d2, d3;
846		−
847		− load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
848		− transpose_u8_4x8(&t0, &t1, &t2, &t3, t4, t5, t6, t7);
849		− src += 8 * src_stride;
850		−
851		− do {
852		− load_u8_8x4(src, src_stride, &t8, &t9, &t10, &t11);
853		− transpose_u8_8x4(&t8, &t9, &t10, &t11);
854		− s0 = vcombine_u8(t0, t8);
855		− s1 = vcombine_u8(t1, t9);
856		− s2 = vcombine_u8(t2, t10);
857		− s3 = vcombine_u8(t3, t11);
858		−
859		− d0 = convolve8_4_dot(s0, filters, correction, range_limit, permute_tbl);
860		− d1 = convolve8_4_dot(s1, filters, correction, range_limit, permute_tbl);
861		− d2 = convolve8_4_dot(s2, filters, correction, range_limit, permute_tbl);
862		− d3 = convolve8_4_dot(s3, filters, correction, range_limit, permute_tbl);
863		−
864		− d01 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d1)), 7);
865		− d23 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d3)), 7);
866		− transpose_u8_4x4(&d01, &d23);
867		−
868		− vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
869		− dst += dst_stride;
870		− vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
871		− dst += dst_stride;
872		− vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
873		− dst += dst_stride;
874		− vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
875		− dst += dst_stride;
876		−
877		− t0 = vext_u8(t0, t8, 4);
878		− t1 = vext_u8(t1, t9, 4);
879		− t2 = vext_u8(t2, t10, 4);
880		− t3 = vext_u8(t3, t11, 4);
881		− src += 4 * src_stride;
882		− h -= 4;
883		− } while (h > 0);
884		− } else if (h == 4) {
885		− const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
886		− uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, d04, d15, d26, d37;
887		− uint8x16_t s0, s1, s2, s3, s4, s5, s6, s7;
888		− int32x4_t d0, d1, d2, d3, d4, d5, d6, d7;
889		− const uint8_t *s;
890		− uint8_t *d;
891		−
892		− do {
893		− s = src;
894		− d = dst;
895		− load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
896		− s += 8 * src_stride;
897		− t8 = vld1_u8(s);
898		− s += src_stride;
899		− t9 = vld1_u8(s);
900		− s += src_stride;
901		− t10 = vld1_u8(s);
902		− s += src_stride;
903		−
904		− transpose_u8_8x16(t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10,
905		− vdup_n_u8(0), vdup_n_u8(0), vdup_n_u8(0), vdup_n_u8(0),
906		− vdup_n_u8(0), &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
907		−
908		− d0 = convolve8_4_dot(s0, filters, correction, range_limit, permute_tbl);
909		− d1 = convolve8_4_dot(s1, filters, correction, range_limit, permute_tbl);
910		− d2 = convolve8_4_dot(s2, filters, correction, range_limit, permute_tbl);
911		− d3 = convolve8_4_dot(s3, filters, correction, range_limit, permute_tbl);
912		− d4 = convolve8_4_dot(s4, filters, correction, range_limit, permute_tbl);
913		− d5 = convolve8_4_dot(s5, filters, correction, range_limit, permute_tbl);
914		− d6 = convolve8_4_dot(s6, filters, correction, range_limit, permute_tbl);
915		− d7 = convolve8_4_dot(s7, filters, correction, range_limit, permute_tbl);
916		−
917		− d04 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d4)), 7);
918		− d15 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d1), vqmovn_s32(d5)), 7);
919		− d26 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d6)), 7);
920		− d37 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d3), vqmovn_s32(d7)), 7);
921		−
922		− transpose_u8_8x4(&d04, &d15, &d26, &d37);
923		−
924		− vst1_u8(d, d04);
925		− d += dst_stride;
926		− vst1_u8(d, d15);
927		− d += dst_stride;
928		− vst1_u8(d, d26);
929		− d += dst_stride;
930		− vst1_u8(d, d37);
931		− d += dst_stride;
932		−
933		− src += 8;
934		− dst += 8;
935		− w -= 8;
936		− } while (w > 0);
937		− } else {
938		− const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
939		− uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t14,
940		− t15, d0, d1, d2, d3, d4, d5, d6, d7;
941		− uint8x16_t s0, s1, s2, s3, s4, s5, s6, s7;
942		− const uint8_t *s;
943		− uint8_t *d;
944		− int height;
945		−
946		− do {
947		− height = h;
948		− s = src;
949		− d = dst;
950		− load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
951		− transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
952		− s += 8 * src_stride;
953		−
954		− do {
955		− load_u8_8x8(s, src_stride, &t8, &t9, &t10, &t11, &t12, &t13, &t14,
956		− &t15);
957		− transpose_u8_8x8(&t8, &t9, &t10, &t11, &t12, &t13, &t14, &t15);
958		− s0 = vcombine_u8(t0, t8);
959		− s1 = vcombine_u8(t1, t9);
960		− s2 = vcombine_u8(t2, t10);
961		− s3 = vcombine_u8(t3, t11);
962		− s4 = vcombine_u8(t4, t12);
963		− s5 = vcombine_u8(t5, t13);
964		− s6 = vcombine_u8(t6, t14);
965		− s7 = vcombine_u8(t7, t15);
966		−
967		− d0 = convolve8_8_dot(s0, filters, correction, range_limit, permute_tbl);
968		− d1 = convolve8_8_dot(s1, filters, correction, range_limit, permute_tbl);
969		− d2 = convolve8_8_dot(s2, filters, correction, range_limit, permute_tbl);
970		− d3 = convolve8_8_dot(s3, filters, correction, range_limit, permute_tbl);
971		− d4 = convolve8_8_dot(s4, filters, correction, range_limit, permute_tbl);
972		− d5 = convolve8_8_dot(s5, filters, correction, range_limit, permute_tbl);
973		− d6 = convolve8_8_dot(s6, filters, correction, range_limit, permute_tbl);
974		− d7 = convolve8_8_dot(s7, filters, correction, range_limit, permute_tbl);
975		−
976		− transpose_u8_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
977		− store_u8_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
978		−
979		− t0 = t8;
980		− t1 = t9;
981		− t2 = t10;
982		− t3 = t11;
983		− t4 = t12;
984		− t5 = t13;
985		− t6 = t14;
986		− t7 = t15;
987		− s += 8 * src_stride;
988		− d += 8 * dst_stride;
989		− height -= 8;
990		− } while (height > 0);
991		− src += 8;
992		− dst += 8;
993		− w -= 8;
994		− } while (w > 0);
995		− }
996		−}
997		−
998		−#else
999		−
1000	1334	void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
1001	1335	uint8_t *dst, ptrdiff_t dst_stride,
1002	1336	const InterpKernel *filter, int x0_q4,
@@ -1169,8 +1503,6 @@ void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride, · 166 unmodified lines
1169	1503	}
1170	1504	}
1171	1505
1172		−#endif
1173		−
1174	1506	void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
1175	1507	uint8_t *dst, ptrdiff_t dst_stride,
1176	1508	const InterpKernel *filter, int x0_q4,
@@ -1361,3 +1693,5 @@ void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride, · 184 unmodified lines
1361	1693	} while (w != 0);
1362	1694	}
1363	1695	}
	1696	+
	1697	+#endif

vpx_dsp/arm/vpx_convolve8_neon.h+38−0

@@ -75,6 +75,21 @@ static INLINE void load_u8_16x8(const uint8_t *s, const ptrdiff_t p, · 74 unmodified lines
75	75	#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
76	76	(__ARM_FEATURE_DOTPROD == 1)
77	77
	78	+static INLINE int32x4_t convolve8_4_dot_partial(const int8x16_t samples_lo,
	79	+ const int8x16_t samples_hi,
	80	+ const int32x4_t correction,
	81	+ const int8x8_t filters) {
	82	+ /* Sample range-clamping and permutation are performed by the caller. */
	83	+ int32x4_t sum;
	84	+
	85	+ /* Accumulate dot product into 'correction' to account for range clamp. */
	86	+ sum = vdotq_lane_s32(correction, samples_lo, filters, 0);
	87	+ sum = vdotq_lane_s32(sum, samples_hi, filters, 1);
	88	+
	89	+ /* Narrowing and packing is performed by the caller. */
	90	+ return sum;
	91	+}
	92	+
78	93	static INLINE int32x4_t convolve8_4_dot(uint8x16_t samples,
79	94	const int8x8_t filters,
80	95	const int32x4_t correction,
@@ -100,6 +115,29 @@ static INLINE int32x4_t convolve8_4_dot(uint8x16_t samples, · 19 unmodified lines
100	115	return sum;
101	116	}
102	117
	118	+static INLINE uint8x8_t convolve8_8_dot_partial(const int8x16_t samples0_lo,
	119	+ const int8x16_t samples0_hi,
	120	+ const int8x16_t samples1_lo,
	121	+ const int8x16_t samples1_hi,
	122	+ const int32x4_t correction,
	123	+ const int8x8_t filters) {
	124	+ /* Sample range-clamping and permutation are performed by the caller. */
	125	+ int32x4_t sum0, sum1;
	126	+ int16x8_t sum;
	127	+
	128	+ /* Accumulate dot product into 'correction' to account for range clamp. */
	129	+ /* First 4 output values. */
	130	+ sum0 = vdotq_lane_s32(correction, samples0_lo, filters, 0);
	131	+ sum0 = vdotq_lane_s32(sum0, samples0_hi, filters, 1);
	132	+ /* Second 4 output values. */
	133	+ sum1 = vdotq_lane_s32(correction, samples1_lo, filters, 0);
	134	+ sum1 = vdotq_lane_s32(sum1, samples1_hi, filters, 1);
	135	+
	136	+ /* Narrow and re-pack. */
	137	+ sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
	138	+ return vqrshrun_n_s16(sum, 7);
	139	+}
	140	+
103	141	static INLINE uint8x8_t convolve8_8_dot(uint8x16_t samples,
104	142	const int8x8_t filters,
105	143	const int32x4_t correction,

Merge changes I2e86b005,I971c6261,I87fe4dad

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code