History

Merge changes I2e86b005,I971c6261,I87fe4dad

* changes: Use 'ptrdiff_t' instead of 'int' for pointer offset parameters Implement vpx_convolve8_avg_vert_neon using SDOT instruction Merge transpose and permute in Neon SDOT vertical convolution

James Zern committed 5 years ago

Commit 76a4e442 parents 2c6a617 dbda0323 files · +589 −215Browse files

Unified · Split

3 files
  1. vpx_dsp/arm/mem_neon.h +8 −6
  2. vpx_dsp/arm/vpx_convolve8_neon.c +543 −209
  3. vpx_dsp/arm/vpx_convolve8_neon.h +38 −0
vpx_dsp/arm/mem_neon.h+8−6
@@ -113,7 +113,8 @@ static INLINE void uint32_to_mem(uint8_t *buf, uint32_t a) { · 112 unmodified lines
113}113}
114 114
115// Load 2 sets of 4 bytes when alignment is not guaranteed.115// Load 2 sets of 4 bytes when alignment is not guaranteed.
116static INLINE uint8x8_t load_unaligned_u8(const uint8_t *buf, int stride) {116static INLINE uint8x8_t load_unaligned_u8(const uint8_t *buf,
117 ptrdiff_t stride) {
117 uint32_t a;118 uint32_t a;
118 uint32x2_t a_u32 = vdup_n_u32(0);119 uint32x2_t a_u32 = vdup_n_u32(0);
119 if (stride == 4) return vld1_u8(buf);120 if (stride == 4) return vld1_u8(buf);
@@ -126,7 +127,7 @@ static INLINE uint8x8_t load_unaligned_u8(const uint8_t *buf, int stride) { · 6 unmodified lines
126}127}
127 128
128// Store 2 sets of 4 bytes when alignment is not guaranteed.129// Store 2 sets of 4 bytes when alignment is not guaranteed.
129static INLINE void store_unaligned_u8(uint8_t *buf, int stride,130static INLINE void store_unaligned_u8(uint8_t *buf, ptrdiff_t stride,
130 const uint8x8_t a) {131 const uint8x8_t a) {
131 const uint32x2_t a_u32 = vreinterpret_u32_u8(a);132 const uint32x2_t a_u32 = vreinterpret_u32_u8(a);
132 if (stride == 4) {133 if (stride == 4) {
@@ -139,7 +140,8 @@ static INLINE void store_unaligned_u8(uint8_t *buf, int stride, · 6 unmodified lines
139}140}
140 141
141// Load 4 sets of 4 bytes when alignment is not guaranteed.142// Load 4 sets of 4 bytes when alignment is not guaranteed.
142static INLINE uint8x16_t load_unaligned_u8q(const uint8_t *buf, int stride) {143static INLINE uint8x16_t load_unaligned_u8q(const uint8_t *buf,
144 ptrdiff_t stride) {
143 uint32_t a;145 uint32_t a;
144 uint32x4_t a_u32 = vdupq_n_u32(0);146 uint32x4_t a_u32 = vdupq_n_u32(0);
145 if (stride == 4) return vld1q_u8(buf);147 if (stride == 4) return vld1q_u8(buf);
@@ -159,7 +161,7 @@ static INLINE uint8x16_t load_unaligned_u8q(const uint8_t *buf, int stride) { · 13 unmodified lines
159}161}
160 162
161// Store 4 sets of 4 bytes when alignment is not guaranteed.163// Store 4 sets of 4 bytes when alignment is not guaranteed.
162static INLINE void store_unaligned_u8q(uint8_t *buf, int stride,164static INLINE void store_unaligned_u8q(uint8_t *buf, ptrdiff_t stride,
163 const uint8x16_t a) {165 const uint8x16_t a) {
164 const uint32x4_t a_u32 = vreinterpretq_u32_u8(a);166 const uint32x4_t a_u32 = vreinterpretq_u32_u8(a);
165 if (stride == 4) {167 if (stride == 4) {
@@ -176,7 +178,7 @@ static INLINE void store_unaligned_u8q(uint8_t *buf, int stride, · 10 unmodified lines
176}178}
177 179
178// Load 2 sets of 4 bytes when alignment is guaranteed.180// Load 2 sets of 4 bytes when alignment is guaranteed.
179static INLINE uint8x8_t load_u8(const uint8_t *buf, int stride) {181static INLINE uint8x8_t load_u8(const uint8_t *buf, ptrdiff_t stride) {
180 uint32x2_t a = vdup_n_u32(0);182 uint32x2_t a = vdup_n_u32(0);
181 183
182 assert(!((intptr_t)buf % sizeof(uint32_t)));184 assert(!((intptr_t)buf % sizeof(uint32_t)));
@@ -189,7 +191,7 @@ static INLINE uint8x8_t load_u8(const uint8_t *buf, int stride) { · 6 unmodified lines
189}191}
190 192
191// Store 2 sets of 4 bytes when alignment is guaranteed.193// Store 2 sets of 4 bytes when alignment is guaranteed.
192static INLINE void store_u8(uint8_t *buf, int stride, const uint8x8_t a) {194static INLINE void store_u8(uint8_t *buf, ptrdiff_t stride, const uint8x8_t a) {
193 uint32x2_t a_u32 = vreinterpret_u32_u8(a);195 uint32x2_t a_u32 = vreinterpret_u32_u8(a);
194 196
195 assert(!((intptr_t)buf % sizeof(uint32_t)));197 assert(!((intptr_t)buf % sizeof(uint32_t)));
vpx_dsp/arm/vpx_convolve8_neon.c+543−209
@@ -31,37 +31,72 @@ · 30 unmodified lines
31// instructions. This optimization is much faster in speed unit test, but slowed31// instructions. This optimization is much faster in speed unit test, but slowed
32// down the whole decoder by 5%.32// down the whole decoder by 5%.
33 33
34static INLINE void store_u8_8x8(uint8_t *s, const ptrdiff_t p,
35 const uint8x8_t s0, const uint8x8_t s1,
36 const uint8x8_t s2, const uint8x8_t s3,
37 const uint8x8_t s4, const uint8x8_t s5,
38 const uint8x8_t s6, const uint8x8_t s7) {
39 vst1_u8(s, s0);
40 s += p;
41 vst1_u8(s, s1);
42 s += p;
43 vst1_u8(s, s2);
44 s += p;
45 vst1_u8(s, s3);
46 s += p;
47 vst1_u8(s, s4);
48 s += p;
49 vst1_u8(s, s5);
50 s += p;
51 vst1_u8(s, s6);
52 s += p;
53 vst1_u8(s, s7);
54}
55
56#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \34#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
57 (__ARM_FEATURE_DOTPROD == 1)35 (__ARM_FEATURE_DOTPROD == 1)
58
59DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {36DECLARE_ALIGNED(16, static const uint8_t, dot_prod_permute_tbl[48]) = {
60 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,37 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6,
61 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,38 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10,
62 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 1439 8, 9, 10, 11, 9, 10, 11, 12, 10, 11, 12, 13, 11, 12, 13, 14
63};40};
64 41
42DECLARE_ALIGNED(16, static const uint8_t, dot_prod_tran_concat_tbl[32]) = {
43 0, 8, 16, 24, 1, 9, 17, 25, 2, 10, 18, 26, 3, 11, 19, 27,
44 4, 12, 20, 28, 5, 13, 21, 29, 6, 14, 22, 30, 7, 15, 23, 31
45};
46
47DECLARE_ALIGNED(16, static const uint8_t, dot_prod_merge_block_tbl[48]) = {
48 /* Shift left and insert new last column in transposed 4x4 block. */
49 1, 2, 3, 16, 5, 6, 7, 20, 9, 10, 11, 24, 13, 14, 15, 28,
50 /* Shift left and insert two new columns in transposed 4x4 block. */
51 2, 3, 16, 17, 6, 7, 20, 21, 10, 11, 24, 25, 14, 15, 28, 29,
52 /* Shift left and insert three new columns in transposed 4x4 block. */
53 3, 16, 17, 18, 7, 20, 21, 22, 11, 24, 25, 26, 15, 28, 29, 30
54};
55
56static INLINE void transpose_concat_4x4(int8x8_t *a0, int8x8_t *a1,
57 int8x8_t *a2, int8x8_t *a3,
58 int8x16_t *b,
59 const uint8x16_t permute_tbl) {
60 /* Transpose 8-bit elements and concatenate result rows as follows:
61 * a0: 00, 01, 02, 03, XX, XX, XX, XX
62 * a1: 10, 11, 12, 13, XX, XX, XX, XX
63 * a2: 20, 21, 22, 23, XX, XX, XX, XX
64 * a3: 30, 31, 32, 33, XX, XX, XX, XX
65 *
66 * b: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
67 *
68 * The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
69 * as an argument is preferable to loading it directly from memory as this
70 * inline helper is called many times from the same parent function.
71 */
72
73 int8x16x2_t samples = { { vcombine_s8(*a0, *a1), vcombine_s8(*a2, *a3) } };
74 *b = vqtbl2q_s8(samples, permute_tbl);
75}
76
77static INLINE void transpose_concat_8x4(int8x8_t *a0, int8x8_t *a1,
78 int8x8_t *a2, int8x8_t *a3,
79 int8x16_t *b0, int8x16_t *b1,
80 const uint8x16x2_t permute_tbl) {
81 /* Transpose 8-bit elements and concatenate result rows as follows:
82 * a0: 00, 01, 02, 03, 04, 05, 06, 07
83 * a1: 10, 11, 12, 13, 14, 15, 16, 17
84 * a2: 20, 21, 22, 23, 24, 25, 26, 27
85 * a3: 30, 31, 32, 33, 34, 35, 36, 37
86 *
87 * b0: 00, 10, 20, 30, 01, 11, 21, 31, 02, 12, 22, 32, 03, 13, 23, 33
88 * b1: 04, 14, 24, 34, 05, 15, 25, 35, 06, 16, 26, 36, 07, 17, 27, 37
89 *
90 * The 'permute_tbl' is always 'dot_prod_tran_concat_tbl' above. Passing it
91 * as an argument is preferable to loading it directly from memory as this
92 * inline helper is called many times from the same parent function.
93 */
94
95 int8x16x2_t samples = { { vcombine_s8(*a0, *a1), vcombine_s8(*a2, *a3) } };
96 *b0 = vqtbl2q_s8(samples, permute_tbl.val[0]);
97 *b1 = vqtbl2q_s8(samples, permute_tbl.val[1]);
98}
99
65void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,100void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
66 uint8_t *dst, ptrdiff_t dst_stride,101 uint8_t *dst, ptrdiff_t dst_stride,
67 const InterpKernel *filter, int x0_q4,102 const InterpKernel *filter, int x0_q4,
@@ -268,8 +303,491 @@ void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride, · 200 unmodified lines
268 }303 }
269}304}
270 305
306void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
307 uint8_t *dst, ptrdiff_t dst_stride,
308 const InterpKernel *filter, int x0_q4,
309 int x_step_q4, int y0_q4, int y_step_q4, int w,
310 int h) {
311 const int8x8_t filters = vmovn_s16(vld1q_s16(filter[y0_q4]));
312 const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[y0_q4]), 128);
313 const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
314 const uint8x8_t range_limit = vdup_n_u8(128);
315 const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(dot_prod_merge_block_tbl);
316 uint8x8_t t0, t1, t2, t3, t4, t5, t6;
317 int8x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
318 int8x16x2_t samples_LUT;
319
320 assert(!((intptr_t)dst & 3));
321 assert(!(dst_stride & 3));
322 assert(y_step_q4 == 16);
323
324 (void)x0_q4;
325 (void)x_step_q4;
326 (void)y_step_q4;
327
328 src -= 3 * src_stride;
329
330 if (w == 4) {
331 const uint8x16_t tran_concat_tbl = vld1q_u8(dot_prod_tran_concat_tbl);
332 int8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s78910;
333 int32x4_t d0, d1, d2, d3;
334 uint8x8_t d01, d23;
335
336 load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
337 src += 4 * src_stride;
338 t4 = vld1_u8(src);
339 src += src_stride;
340 t5 = vld1_u8(src);
341 src += src_stride;
342 t6 = vld1_u8(src);
343 src += src_stride;
344
345 /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
346 s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
347 s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
348 s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
349 s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
350 s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
351 s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
352 s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
353 s7 = vdup_n_s8(0);
354 s8 = vdup_n_s8(0);
355 s9 = vdup_n_s8(0);
356
357 /* This operation combines a conventional transpose and the sample permute
358 * (see horizontal case) required before computing the dot product.
359 */
360 transpose_concat_4x4(&s0, &s1, &s2, &s3, &s0123, tran_concat_tbl);
361 transpose_concat_4x4(&s1, &s2, &s3, &s4, &s1234, tran_concat_tbl);
362 transpose_concat_4x4(&s2, &s3, &s4, &s5, &s2345, tran_concat_tbl);
363 transpose_concat_4x4(&s3, &s4, &s5, &s6, &s3456, tran_concat_tbl);
364 transpose_concat_4x4(&s4, &s5, &s6, &s7, &s4567, tran_concat_tbl);
365 transpose_concat_4x4(&s5, &s6, &s7, &s8, &s5678, tran_concat_tbl);
366 transpose_concat_4x4(&s6, &s7, &s8, &s9, &s6789, tran_concat_tbl);
367
368 do {
369 uint8x8_t t7, t8, t9, t10;
370
371 load_u8_8x4(src, src_stride, &t7, &t8, &t9, &t10);
372
373 s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
374 s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
375 s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
376 s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
377
378 transpose_concat_4x4(&s7, &s8, &s9, &s10, &s78910, tran_concat_tbl);
379
380 /* Merge new data into block from previous iteration. */
381 samples_LUT.val[0] = s3456;
382 samples_LUT.val[1] = s78910;
383 s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
384 s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
385 s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
386
387 d0 = convolve8_4_dot_partial(s0123, s4567, correction, filters);
388 d1 = convolve8_4_dot_partial(s1234, s5678, correction, filters);
389 d2 = convolve8_4_dot_partial(s2345, s6789, correction, filters);
390 d3 = convolve8_4_dot_partial(s3456, s78910, correction, filters);
391
392 d01 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d1)), 7);
393 d23 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d3)), 7);
394
395 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
396 dst += dst_stride;
397 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
398 dst += dst_stride;
399 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
400 dst += dst_stride;
401 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
402 dst += dst_stride;
403
404 /* Prepare block for next iteration - re-using as much as possible. */
405 /* Shuffle everything up four rows. */
406 s0123 = s4567;
407 s1234 = s5678;
408 s2345 = s6789;
409 s3456 = s78910;
410
411 src += 4 * src_stride;
412 h -= 4;
413 } while (h > 0);
414 } else {
415 const uint8x16x2_t tran_concat_tbl = vld1q_u8_x2(dot_prod_tran_concat_tbl);
416 int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi,
417 s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo,
418 s6789_hi, s78910_lo, s78910_hi;
419 uint8x8_t d0, d1, d2, d3;
420 const uint8_t *s;
421 uint8_t *d;
422 int height;
423
424 do {
425 height = h;
426 s = src;
427 d = dst;
428
429 load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
430 s += 4 * src_stride;
431 t4 = vld1_u8(s);
432 s += src_stride;
433 t5 = vld1_u8(s);
434 s += src_stride;
435 t6 = vld1_u8(s);
436 s += src_stride;
437
438 /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
439 s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
440 s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
441 s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
442 s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
443 s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
444 s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
445 s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
446 s7 = vdup_n_s8(0);
447 s8 = vdup_n_s8(0);
448 s9 = vdup_n_s8(0);
449
450 /* This operation combines a conventional transpose and the sample permute
451 * (see horizontal case) required before computing the dot product.
452 */
453 transpose_concat_8x4(&s0, &s1, &s2, &s3, &s0123_lo, &s0123_hi,
454 tran_concat_tbl);
455 transpose_concat_8x4(&s1, &s2, &s3, &s4, &s1234_lo, &s1234_hi,
456 tran_concat_tbl);
457 transpose_concat_8x4(&s2, &s3, &s4, &s5, &s2345_lo, &s2345_hi,
458 tran_concat_tbl);
459 transpose_concat_8x4(&s3, &s4, &s5, &s6, &s3456_lo, &s3456_hi,
460 tran_concat_tbl);
461 transpose_concat_8x4(&s4, &s5, &s6, &s7, &s4567_lo, &s4567_hi,
462 tran_concat_tbl);
463 transpose_concat_8x4(&s5, &s6, &s7, &s8, &s5678_lo, &s5678_hi,
464 tran_concat_tbl);
465 transpose_concat_8x4(&s6, &s7, &s8, &s9, &s6789_lo, &s6789_hi,
466 tran_concat_tbl);
467
468 do {
469 uint8x8_t t7, t8, t9, t10;
470
471 load_u8_8x4(s, src_stride, &t7, &t8, &t9, &t10);
472
473 s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
474 s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
475 s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
476 s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
477
478 transpose_concat_8x4(&s7, &s8, &s9, &s10, &s78910_lo, &s78910_hi,
479 tran_concat_tbl);
480
481 /* Merge new data into block from previous iteration. */
482 samples_LUT.val[0] = s3456_lo;
483 samples_LUT.val[1] = s78910_lo;
484 s4567_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
485 s5678_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
486 s6789_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
487
488 samples_LUT.val[0] = s3456_hi;
489 samples_LUT.val[1] = s78910_hi;
490 s4567_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
491 s5678_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
492 s6789_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
493
494 d0 = convolve8_8_dot_partial(s0123_lo, s4567_lo, s0123_hi, s4567_hi,
495 correction, filters);
496 d1 = convolve8_8_dot_partial(s1234_lo, s5678_lo, s1234_hi, s5678_hi,
497 correction, filters);
498 d2 = convolve8_8_dot_partial(s2345_lo, s6789_lo, s2345_hi, s6789_hi,
499 correction, filters);
500 d3 = convolve8_8_dot_partial(s3456_lo, s78910_lo, s3456_hi, s78910_hi,
501 correction, filters);
502 vst1_u8(d + 0 * dst_stride, d0);
503 vst1_u8(d + 1 * dst_stride, d1);
504 vst1_u8(d + 2 * dst_stride, d2);
505 vst1_u8(d + 3 * dst_stride, d3);
506
507 /* Prepare block for next iteration - re-using as much as possible. */
508 /* Shuffle everything up four rows. */
509 s0123_lo = s4567_lo;
510 s0123_hi = s4567_hi;
511 s1234_lo = s5678_lo;
512 s1234_hi = s5678_hi;
513 s2345_lo = s6789_lo;
514 s2345_hi = s6789_hi;
515 s3456_lo = s78910_lo;
516 s3456_hi = s78910_hi;
517
518 s += 4 * src_stride;
519 d += 4 * dst_stride;
520 height -= 4;
521 } while (height > 0);
522 src += 8;
523 dst += 8;
524 w -= 8;
525 } while (w > 0);
526 }
527}
528
529void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
530 uint8_t *dst, ptrdiff_t dst_stride,
531 const InterpKernel *filter, int x0_q4,
532 int x_step_q4, int y0_q4, int y_step_q4, int w,
533 int h) {
534 const int8x8_t filters = vmovn_s16(vld1q_s16(filter[y0_q4]));
535 const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[y0_q4]), 128);
536 const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
537 const uint8x8_t range_limit = vdup_n_u8(128);
538 const uint8x16x3_t merge_block_tbl = vld1q_u8_x3(dot_prod_merge_block_tbl);
539 uint8x8_t t0, t1, t2, t3, t4, t5, t6;
540 int8x8_t s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10;
541 int8x16x2_t samples_LUT;
542
543 assert(!((intptr_t)dst & 3));
544 assert(!(dst_stride & 3));
545 assert(y_step_q4 == 16);
546
547 (void)x0_q4;
548 (void)x_step_q4;
549 (void)y_step_q4;
550
551 src -= 3 * src_stride;
552
553 if (w == 4) {
554 const uint8x16_t tran_concat_tbl = vld1q_u8(dot_prod_tran_concat_tbl);
555 int8x16_t s0123, s1234, s2345, s3456, s4567, s5678, s6789, s78910;
556 int32x4_t d0, d1, d2, d3;
557 uint8x8_t d01, d23, dd01, dd23;
558
559 load_u8_8x4(src, src_stride, &t0, &t1, &t2, &t3);
560 src += 4 * src_stride;
561 t4 = vld1_u8(src);
562 src += src_stride;
563 t5 = vld1_u8(src);
564 src += src_stride;
565 t6 = vld1_u8(src);
566 src += src_stride;
567
568 /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
569 s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
570 s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
571 s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
572 s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
573 s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
574 s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
575 s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
576 s7 = vdup_n_s8(0);
577 s8 = vdup_n_s8(0);
578 s9 = vdup_n_s8(0);
579
580 /* This operation combines a conventional transpose and the sample permute
581 * (see horizontal case) required before computing the dot product.
582 */
583 transpose_concat_4x4(&s0, &s1, &s2, &s3, &s0123, tran_concat_tbl);
584 transpose_concat_4x4(&s1, &s2, &s3, &s4, &s1234, tran_concat_tbl);
585 transpose_concat_4x4(&s2, &s3, &s4, &s5, &s2345, tran_concat_tbl);
586 transpose_concat_4x4(&s3, &s4, &s5, &s6, &s3456, tran_concat_tbl);
587 transpose_concat_4x4(&s4, &s5, &s6, &s7, &s4567, tran_concat_tbl);
588 transpose_concat_4x4(&s5, &s6, &s7, &s8, &s5678, tran_concat_tbl);
589 transpose_concat_4x4(&s6, &s7, &s8, &s9, &s6789, tran_concat_tbl);
590
591 do {
592 uint8x8_t t7, t8, t9, t10;
593
594 load_u8_8x4(src, src_stride, &t7, &t8, &t9, &t10);
595
596 s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
597 s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
598 s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
599 s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
600
601 transpose_concat_4x4(&s7, &s8, &s9, &s10, &s78910, tran_concat_tbl);
602
603 /* Merge new data into block from previous iteration. */
604 samples_LUT.val[0] = s3456;
605 samples_LUT.val[1] = s78910;
606 s4567 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
607 s5678 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
608 s6789 = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
609
610 d0 = convolve8_4_dot_partial(s0123, s4567, correction, filters);
611 d1 = convolve8_4_dot_partial(s1234, s5678, correction, filters);
612 d2 = convolve8_4_dot_partial(s2345, s6789, correction, filters);
613 d3 = convolve8_4_dot_partial(s3456, s78910, correction, filters);
614
615 d01 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d1)), 7);
616 d23 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d3)), 7);
617
618 dd01 = load_u8(dst + 0 * dst_stride, dst_stride);
619 dd23 = load_u8(dst + 2 * dst_stride, dst_stride);
620 d01 = vrhadd_u8(d01, dd01);
621 d23 = vrhadd_u8(d23, dd23);
622
623 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
624 dst += dst_stride;
625 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
626 dst += dst_stride;
627 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
628 dst += dst_stride;
629 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
630 dst += dst_stride;
631
632 /* Prepare block for next iteration - re-using as much as possible. */
633 /* Shuffle everything up four rows. */
634 s0123 = s4567;
635 s1234 = s5678;
636 s2345 = s6789;
637 s3456 = s78910;
638
639 src += 4 * src_stride;
640 h -= 4;
641 } while (h > 0);
642 } else {
643 const uint8x16x2_t tran_concat_tbl = vld1q_u8_x2(dot_prod_tran_concat_tbl);
644 int8x16_t s0123_lo, s0123_hi, s1234_lo, s1234_hi, s2345_lo, s2345_hi,
645 s3456_lo, s3456_hi, s4567_lo, s4567_hi, s5678_lo, s5678_hi, s6789_lo,
646 s6789_hi, s78910_lo, s78910_hi;
647 uint8x8_t d0, d1, d2, d3, dd0, dd1, dd2, dd3;
648 const uint8_t *s;
649 uint8_t *d;
650 int height;
651
652 do {
653 height = h;
654 s = src;
655 d = dst;
656
657 load_u8_8x4(s, src_stride, &t0, &t1, &t2, &t3);
658 s += 4 * src_stride;
659 t4 = vld1_u8(s);
660 s += src_stride;
661 t5 = vld1_u8(s);
662 s += src_stride;
663 t6 = vld1_u8(s);
664 s += src_stride;
665
666 /* Clamp sample range to [-128, 127] for 8-bit signed dot product. */
667 s0 = vreinterpret_s8_u8(vsub_u8(t0, range_limit));
668 s1 = vreinterpret_s8_u8(vsub_u8(t1, range_limit));
669 s2 = vreinterpret_s8_u8(vsub_u8(t2, range_limit));
670 s3 = vreinterpret_s8_u8(vsub_u8(t3, range_limit));
671 s4 = vreinterpret_s8_u8(vsub_u8(t4, range_limit));
672 s5 = vreinterpret_s8_u8(vsub_u8(t5, range_limit));
673 s6 = vreinterpret_s8_u8(vsub_u8(t6, range_limit));
674 s7 = vdup_n_s8(0);
675 s8 = vdup_n_s8(0);
676 s9 = vdup_n_s8(0);
677
678 /* This operation combines a conventional transpose and the sample permute
679 * (see horizontal case) required before computing the dot product.
680 */
681 transpose_concat_8x4(&s0, &s1, &s2, &s3, &s0123_lo, &s0123_hi,
682 tran_concat_tbl);
683 transpose_concat_8x4(&s1, &s2, &s3, &s4, &s1234_lo, &s1234_hi,
684 tran_concat_tbl);
685 transpose_concat_8x4(&s2, &s3, &s4, &s5, &s2345_lo, &s2345_hi,
686 tran_concat_tbl);
687 transpose_concat_8x4(&s3, &s4, &s5, &s6, &s3456_lo, &s3456_hi,
688 tran_concat_tbl);
689 transpose_concat_8x4(&s4, &s5, &s6, &s7, &s4567_lo, &s4567_hi,
690 tran_concat_tbl);
691 transpose_concat_8x4(&s5, &s6, &s7, &s8, &s5678_lo, &s5678_hi,
692 tran_concat_tbl);
693 transpose_concat_8x4(&s6, &s7, &s8, &s9, &s6789_lo, &s6789_hi,
694 tran_concat_tbl);
695
696 do {
697 uint8x8_t t7, t8, t9, t10;
698
699 load_u8_8x4(s, src_stride, &t7, &t8, &t9, &t10);
700
701 s7 = vreinterpret_s8_u8(vsub_u8(t7, range_limit));
702 s8 = vreinterpret_s8_u8(vsub_u8(t8, range_limit));
703 s9 = vreinterpret_s8_u8(vsub_u8(t9, range_limit));
704 s10 = vreinterpret_s8_u8(vsub_u8(t10, range_limit));
705
706 transpose_concat_8x4(&s7, &s8, &s9, &s10, &s78910_lo, &s78910_hi,
707 tran_concat_tbl);
708
709 /* Merge new data into block from previous iteration. */
710 samples_LUT.val[0] = s3456_lo;
711 samples_LUT.val[1] = s78910_lo;
712 s4567_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
713 s5678_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
714 s6789_lo = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
715
716 samples_LUT.val[0] = s3456_hi;
717 samples_LUT.val[1] = s78910_hi;
718 s4567_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[0]);
719 s5678_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[1]);
720 s6789_hi = vqtbl2q_s8(samples_LUT, merge_block_tbl.val[2]);
721
722 d0 = convolve8_8_dot_partial(s0123_lo, s4567_lo, s0123_hi, s4567_hi,
723 correction, filters);
724 d1 = convolve8_8_dot_partial(s1234_lo, s5678_lo, s1234_hi, s5678_hi,
725 correction, filters);
726 d2 = convolve8_8_dot_partial(s2345_lo, s6789_lo, s2345_hi, s6789_hi,
727 correction, filters);
728 d3 = convolve8_8_dot_partial(s3456_lo, s78910_lo, s3456_hi, s78910_hi,
729 correction, filters);
730
731 dd0 = vld1_u8(d + 0 * dst_stride);
732 dd1 = vld1_u8(d + 1 * dst_stride);
733 dd2 = vld1_u8(d + 2 * dst_stride);
734 dd3 = vld1_u8(d + 3 * dst_stride);
735 d0 = vrhadd_u8(d0, dd0);
736 d1 = vrhadd_u8(d1, dd1);
737 d2 = vrhadd_u8(d2, dd2);
738 d3 = vrhadd_u8(d3, dd3);
739
740 vst1_u8(d + 0 * dst_stride, d0);
741 vst1_u8(d + 1 * dst_stride, d1);
742 vst1_u8(d + 2 * dst_stride, d2);
743 vst1_u8(d + 3 * dst_stride, d3);
744
745 /* Prepare block for next iteration - re-using as much as possible. */
746 /* Shuffle everything up four rows. */
747 s0123_lo = s4567_lo;
748 s0123_hi = s4567_hi;
749 s1234_lo = s5678_lo;
750 s1234_hi = s5678_hi;
751 s2345_lo = s6789_lo;
752 s2345_hi = s6789_hi;
753 s3456_lo = s78910_lo;
754 s3456_hi = s78910_hi;
755
756 s += 4 * src_stride;
757 d += 4 * dst_stride;
758 height -= 4;
759 } while (height > 0);
760 src += 8;
761 dst += 8;
762 w -= 8;
763 } while (w > 0);
764 }
765}
766
271#else767#else
272 768
769static INLINE void store_u8_8x8(uint8_t *s, const ptrdiff_t p,
770 const uint8x8_t s0, const uint8x8_t s1,
771 const uint8x8_t s2, const uint8x8_t s3,
772 const uint8x8_t s4, const uint8x8_t s5,
773 const uint8x8_t s6, const uint8x8_t s7) {
774 vst1_u8(s, s0);
775 s += p;
776 vst1_u8(s, s1);
777 s += p;
778 vst1_u8(s, s2);
779 s += p;
780 vst1_u8(s, s3);
781 s += p;
782 vst1_u8(s, s4);
783 s += p;
784 vst1_u8(s, s5);
785 s += p;
786 vst1_u8(s, s6);
787 s += p;
788 vst1_u8(s, s7);
789}
790
273void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,791void vpx_convolve8_horiz_neon(const uint8_t *src, ptrdiff_t src_stride,
274 uint8_t *dst, ptrdiff_t dst_stride,792 uint8_t *dst, ptrdiff_t dst_stride,
275 const InterpKernel *filter, int x0_q4,793 const InterpKernel *filter, int x0_q4,
@@ -813,190 +1331,6 @@ void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride, · 537 unmodified lines
813 }1331 }
814}1332}
815 1333
816#endif
817
818#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
819 (__ARM_FEATURE_DOTPROD == 1)
820
821void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
822 uint8_t *dst, ptrdiff_t dst_stride,
823 const InterpKernel *filter, int x0_q4,
824 int x_step_q4, int y0_q4, int y_step_q4, int w,
825 int h) {
826 const int8x8_t filters = vmovn_s16(vld1q_s16(filter[y0_q4]));
827 const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[y0_q4]), 128);
828 const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp));
829 const uint8x16_t range_limit = vdupq_n_u8(128);
830
831 assert(!((intptr_t)dst & 3));
832 assert(!(dst_stride & 3));
833 assert(y_step_q4 == 16);
834
835 (void)x0_q4;
836 (void)x_step_q4;
837 (void)y_step_q4;
838
839 src -= 3 * src_stride;
840
841 if (w == 4) {
842 const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
843 uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, d01, d23;
844 uint8x16_t s0, s1, s2, s3;
845 int32x4_t d0, d1, d2, d3;
846
847 load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
848 transpose_u8_4x8(&t0, &t1, &t2, &t3, t4, t5, t6, t7);
849 src += 8 * src_stride;
850
851 do {
852 load_u8_8x4(src, src_stride, &t8, &t9, &t10, &t11);
853 transpose_u8_8x4(&t8, &t9, &t10, &t11);
854 s0 = vcombine_u8(t0, t8);
855 s1 = vcombine_u8(t1, t9);
856 s2 = vcombine_u8(t2, t10);
857 s3 = vcombine_u8(t3, t11);
858
859 d0 = convolve8_4_dot(s0, filters, correction, range_limit, permute_tbl);
860 d1 = convolve8_4_dot(s1, filters, correction, range_limit, permute_tbl);
861 d2 = convolve8_4_dot(s2, filters, correction, range_limit, permute_tbl);
862 d3 = convolve8_4_dot(s3, filters, correction, range_limit, permute_tbl);
863
864 d01 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d1)), 7);
865 d23 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d3)), 7);
866 transpose_u8_4x4(&d01, &d23);
867
868 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0);
869 dst += dst_stride;
870 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0);
871 dst += dst_stride;
872 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1);
873 dst += dst_stride;
874 vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1);
875 dst += dst_stride;
876
877 t0 = vext_u8(t0, t8, 4);
878 t1 = vext_u8(t1, t9, 4);
879 t2 = vext_u8(t2, t10, 4);
880 t3 = vext_u8(t3, t11, 4);
881 src += 4 * src_stride;
882 h -= 4;
883 } while (h > 0);
884 } else if (h == 4) {
885 const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl);
886 uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, d04, d15, d26, d37;
887 uint8x16_t s0, s1, s2, s3, s4, s5, s6, s7;
888 int32x4_t d0, d1, d2, d3, d4, d5, d6, d7;
889 const uint8_t *s;
890 uint8_t *d;
891
892 do {
893 s = src;
894 d = dst;
895 load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
896 s += 8 * src_stride;
897 t8 = vld1_u8(s);
898 s += src_stride;
899 t9 = vld1_u8(s);
900 s += src_stride;
901 t10 = vld1_u8(s);
902 s += src_stride;
903
904 transpose_u8_8x16(t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10,
905 vdup_n_u8(0), vdup_n_u8(0), vdup_n_u8(0), vdup_n_u8(0),
906 vdup_n_u8(0), &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7);
907
908 d0 = convolve8_4_dot(s0, filters, correction, range_limit, permute_tbl);
909 d1 = convolve8_4_dot(s1, filters, correction, range_limit, permute_tbl);
910 d2 = convolve8_4_dot(s2, filters, correction, range_limit, permute_tbl);
911 d3 = convolve8_4_dot(s3, filters, correction, range_limit, permute_tbl);
912 d4 = convolve8_4_dot(s4, filters, correction, range_limit, permute_tbl);
913 d5 = convolve8_4_dot(s5, filters, correction, range_limit, permute_tbl);
914 d6 = convolve8_4_dot(s6, filters, correction, range_limit, permute_tbl);
915 d7 = convolve8_4_dot(s7, filters, correction, range_limit, permute_tbl);
916
917 d04 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d4)), 7);
918 d15 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d1), vqmovn_s32(d5)), 7);
919 d26 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d6)), 7);
920 d37 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d3), vqmovn_s32(d7)), 7);
921
922 transpose_u8_8x4(&d04, &d15, &d26, &d37);
923
924 vst1_u8(d, d04);
925 d += dst_stride;
926 vst1_u8(d, d15);
927 d += dst_stride;
928 vst1_u8(d, d26);
929 d += dst_stride;
930 vst1_u8(d, d37);
931 d += dst_stride;
932
933 src += 8;
934 dst += 8;
935 w -= 8;
936 } while (w > 0);
937 } else {
938 const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl);
939 uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t14,
940 t15, d0, d1, d2, d3, d4, d5, d6, d7;
941 uint8x16_t s0, s1, s2, s3, s4, s5, s6, s7;
942 const uint8_t *s;
943 uint8_t *d;
944 int height;
945
946 do {
947 height = h;
948 s = src;
949 d = dst;
950 load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
951 transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7);
952 s += 8 * src_stride;
953
954 do {
955 load_u8_8x8(s, src_stride, &t8, &t9, &t10, &t11, &t12, &t13, &t14,
956 &t15);
957 transpose_u8_8x8(&t8, &t9, &t10, &t11, &t12, &t13, &t14, &t15);
958 s0 = vcombine_u8(t0, t8);
959 s1 = vcombine_u8(t1, t9);
960 s2 = vcombine_u8(t2, t10);
961 s3 = vcombine_u8(t3, t11);
962 s4 = vcombine_u8(t4, t12);
963 s5 = vcombine_u8(t5, t13);
964 s6 = vcombine_u8(t6, t14);
965 s7 = vcombine_u8(t7, t15);
966
967 d0 = convolve8_8_dot(s0, filters, correction, range_limit, permute_tbl);
968 d1 = convolve8_8_dot(s1, filters, correction, range_limit, permute_tbl);
969 d2 = convolve8_8_dot(s2, filters, correction, range_limit, permute_tbl);
970 d3 = convolve8_8_dot(s3, filters, correction, range_limit, permute_tbl);
971 d4 = convolve8_8_dot(s4, filters, correction, range_limit, permute_tbl);
972 d5 = convolve8_8_dot(s5, filters, correction, range_limit, permute_tbl);
973 d6 = convolve8_8_dot(s6, filters, correction, range_limit, permute_tbl);
974 d7 = convolve8_8_dot(s7, filters, correction, range_limit, permute_tbl);
975
976 transpose_u8_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7);
977 store_u8_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7);
978
979 t0 = t8;
980 t1 = t9;
981 t2 = t10;
982 t3 = t11;
983 t4 = t12;
984 t5 = t13;
985 t6 = t14;
986 t7 = t15;
987 s += 8 * src_stride;
988 d += 8 * dst_stride;
989 height -= 8;
990 } while (height > 0);
991 src += 8;
992 dst += 8;
993 w -= 8;
994 } while (w > 0);
995 }
996}
997
998#else
999
1000void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,1334void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
1001 uint8_t *dst, ptrdiff_t dst_stride,1335 uint8_t *dst, ptrdiff_t dst_stride,
1002 const InterpKernel *filter, int x0_q4,1336 const InterpKernel *filter, int x0_q4,
@@ -1169,8 +1503,6 @@ void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride, · 166 unmodified lines
1169 }1503 }
1170}1504}
1171 1505
1172#endif
1173
1174void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride,1506void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride,
1175 uint8_t *dst, ptrdiff_t dst_stride,1507 uint8_t *dst, ptrdiff_t dst_stride,
1176 const InterpKernel *filter, int x0_q4,1508 const InterpKernel *filter, int x0_q4,
@@ -1361,3 +1693,5 @@ void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride, · 184 unmodified lines
1361 } while (w != 0);1693 } while (w != 0);
1362 }1694 }
1363}1695}
1696
1697#endif
vpx_dsp/arm/vpx_convolve8_neon.h+38−0
@@ -75,6 +75,21 @@ static INLINE void load_u8_16x8(const uint8_t *s, const ptrdiff_t p, · 74 unmodified lines
75#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \75#if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \
76 (__ARM_FEATURE_DOTPROD == 1)76 (__ARM_FEATURE_DOTPROD == 1)
77 77
78static INLINE int32x4_t convolve8_4_dot_partial(const int8x16_t samples_lo,
79 const int8x16_t samples_hi,
80 const int32x4_t correction,
81 const int8x8_t filters) {
82 /* Sample range-clamping and permutation are performed by the caller. */
83 int32x4_t sum;
84
85 /* Accumulate dot product into 'correction' to account for range clamp. */
86 sum = vdotq_lane_s32(correction, samples_lo, filters, 0);
87 sum = vdotq_lane_s32(sum, samples_hi, filters, 1);
88
89 /* Narrowing and packing is performed by the caller. */
90 return sum;
91}
92
78static INLINE int32x4_t convolve8_4_dot(uint8x16_t samples,93static INLINE int32x4_t convolve8_4_dot(uint8x16_t samples,
79 const int8x8_t filters,94 const int8x8_t filters,
80 const int32x4_t correction,95 const int32x4_t correction,
@@ -100,6 +115,29 @@ static INLINE int32x4_t convolve8_4_dot(uint8x16_t samples, · 19 unmodified lines
100 return sum;115 return sum;
101}116}
102 117
118static INLINE uint8x8_t convolve8_8_dot_partial(const int8x16_t samples0_lo,
119 const int8x16_t samples0_hi,
120 const int8x16_t samples1_lo,
121 const int8x16_t samples1_hi,
122 const int32x4_t correction,
123 const int8x8_t filters) {
124 /* Sample range-clamping and permutation are performed by the caller. */
125 int32x4_t sum0, sum1;
126 int16x8_t sum;
127
128 /* Accumulate dot product into 'correction' to account for range clamp. */
129 /* First 4 output values. */
130 sum0 = vdotq_lane_s32(correction, samples0_lo, filters, 0);
131 sum0 = vdotq_lane_s32(sum0, samples0_hi, filters, 1);
132 /* Second 4 output values. */
133 sum1 = vdotq_lane_s32(correction, samples1_lo, filters, 0);
134 sum1 = vdotq_lane_s32(sum1, samples1_hi, filters, 1);
135
136 /* Narrow and re-pack. */
137 sum = vcombine_s16(vqmovn_s32(sum0), vqmovn_s32(sum1));
138 return vqrshrun_n_s16(sum, 7);
139}
140
103static INLINE uint8x8_t convolve8_8_dot(uint8x16_t samples,141static INLINE uint8x8_t convolve8_8_dot(uint8x16_t samples,
104 const int8x8_t filters,142 const int8x8_t filters,
105 const int32x4_t correction,143 const int32x4_t correction,

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t