Add an alternative AArch64 implementation of vpx_convolve8_vert_neon for targets that implement the Armv8.4-A SDOT (signed dot product) instruction. The existing MLA-based implementation of vpx_convolve8_vert_neon is retained and used on target CPUs that do not implement the SDOT instruction (or CPUs executing in AArch32 mode). The availability of the SDOT instruction is indicated by the feature macro __ARM_FEATURE_DOTPROD.
Unified · Split
| @@ -704,6 +704,188 @@ void vpx_convolve8_avg_horiz_neon(const uint8_t *src, ptrdiff_t src_stride, · 703 unmodified lines | |||
| 704 | } | 704 | } |
| 705 | } | 705 | } |
| 706 | 706 | ||
| 707 | #if defined(__aarch64__) && defined(__ARM_FEATURE_DOTPROD) && \ | ||
| 708 | (__ARM_FEATURE_DOTPROD == 1) | ||
| 709 | |||
| 710 | void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride, | ||
| 711 | uint8_t *dst, ptrdiff_t dst_stride, | ||
| 712 | const InterpKernel *filter, int x0_q4, | ||
| 713 | int x_step_q4, int y0_q4, int y_step_q4, int w, | ||
| 714 | int h) { | ||
| 715 | const int8x8_t filters = vmovn_s16(vld1q_s16(filter[y0_q4])); | ||
| 716 | const int16x8_t correct_tmp = vmulq_n_s16(vld1q_s16(filter[y0_q4]), 128); | ||
| 717 | const int32x4_t correction = vdupq_n_s32((int32_t)vaddvq_s16(correct_tmp)); | ||
| 718 | const uint8x16_t range_limit = vdupq_n_u8(128); | ||
| 719 | |||
| 720 | assert(!((intptr_t)dst & 3)); | ||
| 721 | assert(!(dst_stride & 3)); | ||
| 722 | assert(y_step_q4 == 16); | ||
| 723 | |||
| 724 | (void)x0_q4; | ||
| 725 | (void)x_step_q4; | ||
| 726 | (void)y_step_q4; | ||
| 727 | |||
| 728 | src -= 3 * src_stride; | ||
| 729 | |||
| 730 | if (w == 4) { | ||
| 731 | const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl); | ||
| 732 | uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, d01, d23; | ||
| 733 | uint8x16_t s0, s1, s2, s3; | ||
| 734 | int32x4_t d0, d1, d2, d3; | ||
| 735 | |||
| 736 | load_u8_8x8(src, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); | ||
| 737 | transpose_u8_4x8(&t0, &t1, &t2, &t3, t4, t5, t6, t7); | ||
| 738 | src += 8 * src_stride; | ||
| 739 | |||
| 740 | do { | ||
| 741 | load_u8_8x4(src, src_stride, &t8, &t9, &t10, &t11); | ||
| 742 | transpose_u8_8x4(&t8, &t9, &t10, &t11); | ||
| 743 | s0 = vcombine_u8(t0, t8); | ||
| 744 | s1 = vcombine_u8(t1, t9); | ||
| 745 | s2 = vcombine_u8(t2, t10); | ||
| 746 | s3 = vcombine_u8(t3, t11); | ||
| 747 | |||
| 748 | d0 = convolve8_4_dot(s0, filters, correction, range_limit, permute_tbl); | ||
| 749 | d1 = convolve8_4_dot(s1, filters, correction, range_limit, permute_tbl); | ||
| 750 | d2 = convolve8_4_dot(s2, filters, correction, range_limit, permute_tbl); | ||
| 751 | d3 = convolve8_4_dot(s3, filters, correction, range_limit, permute_tbl); | ||
| 752 | |||
| 753 | d01 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d1)), 7); | ||
| 754 | d23 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d3)), 7); | ||
| 755 | transpose_u8_4x4(&d01, &d23); | ||
| 756 | |||
| 757 | vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 0); | ||
| 758 | dst += dst_stride; | ||
| 759 | vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 0); | ||
| 760 | dst += dst_stride; | ||
| 761 | vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d01), 1); | ||
| 762 | dst += dst_stride; | ||
| 763 | vst1_lane_u32((uint32_t *)dst, vreinterpret_u32_u8(d23), 1); | ||
| 764 | dst += dst_stride; | ||
| 765 | |||
| 766 | t0 = vext_u8(t0, t8, 4); | ||
| 767 | t1 = vext_u8(t1, t9, 4); | ||
| 768 | t2 = vext_u8(t2, t10, 4); | ||
| 769 | t3 = vext_u8(t3, t11, 4); | ||
| 770 | src += 4 * src_stride; | ||
| 771 | h -= 4; | ||
| 772 | } while (h > 0); | ||
| 773 | } else if (h == 4) { | ||
| 774 | const uint8x16x2_t permute_tbl = vld1q_u8_x2(dot_prod_permute_tbl); | ||
| 775 | uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, d04, d15, d26, d37; | ||
| 776 | uint8x16_t s0, s1, s2, s3, s4, s5, s6, s7; | ||
| 777 | int32x4_t d0, d1, d2, d3, d4, d5, d6, d7; | ||
| 778 | const uint8_t *s; | ||
| 779 | uint8_t *d; | ||
| 780 | |||
| 781 | do { | ||
| 782 | s = src; | ||
| 783 | d = dst; | ||
| 784 | load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); | ||
| 785 | s += 8 * src_stride; | ||
| 786 | t8 = vld1_u8(s); | ||
| 787 | s += src_stride; | ||
| 788 | t9 = vld1_u8(s); | ||
| 789 | s += src_stride; | ||
| 790 | t10 = vld1_u8(s); | ||
| 791 | s += src_stride; | ||
| 792 | |||
| 793 | transpose_u8_8x16(t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, | ||
| 794 | vdup_n_u8(0), vdup_n_u8(0), vdup_n_u8(0), vdup_n_u8(0), | ||
| 795 | vdup_n_u8(0), &s0, &s1, &s2, &s3, &s4, &s5, &s6, &s7); | ||
| 796 | |||
| 797 | d0 = convolve8_4_dot(s0, filters, correction, range_limit, permute_tbl); | ||
| 798 | d1 = convolve8_4_dot(s1, filters, correction, range_limit, permute_tbl); | ||
| 799 | d2 = convolve8_4_dot(s2, filters, correction, range_limit, permute_tbl); | ||
| 800 | d3 = convolve8_4_dot(s3, filters, correction, range_limit, permute_tbl); | ||
| 801 | d4 = convolve8_4_dot(s4, filters, correction, range_limit, permute_tbl); | ||
| 802 | d5 = convolve8_4_dot(s5, filters, correction, range_limit, permute_tbl); | ||
| 803 | d6 = convolve8_4_dot(s6, filters, correction, range_limit, permute_tbl); | ||
| 804 | d7 = convolve8_4_dot(s7, filters, correction, range_limit, permute_tbl); | ||
| 805 | |||
| 806 | d04 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d0), vqmovn_s32(d4)), 7); | ||
| 807 | d15 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d1), vqmovn_s32(d5)), 7); | ||
| 808 | d26 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d2), vqmovn_s32(d6)), 7); | ||
| 809 | d37 = vqrshrun_n_s16(vcombine_s16(vqmovn_s32(d3), vqmovn_s32(d7)), 7); | ||
| 810 | |||
| 811 | transpose_u8_8x4(&d04, &d15, &d26, &d37); | ||
| 812 | |||
| 813 | vst1_u8(d, d04); | ||
| 814 | d += dst_stride; | ||
| 815 | vst1_u8(d, d15); | ||
| 816 | d += dst_stride; | ||
| 817 | vst1_u8(d, d26); | ||
| 818 | d += dst_stride; | ||
| 819 | vst1_u8(d, d37); | ||
| 820 | d += dst_stride; | ||
| 821 | |||
| 822 | src += 8; | ||
| 823 | dst += 8; | ||
| 824 | w -= 8; | ||
| 825 | } while (w > 0); | ||
| 826 | } else { | ||
| 827 | const uint8x16x3_t permute_tbl = vld1q_u8_x3(dot_prod_permute_tbl); | ||
| 828 | uint8x8_t t0, t1, t2, t3, t4, t5, t6, t7, t8, t9, t10, t11, t12, t13, t14, | ||
| 829 | t15, d0, d1, d2, d3, d4, d5, d6, d7; | ||
| 830 | uint8x16_t s0, s1, s2, s3, s4, s5, s6, s7; | ||
| 831 | const uint8_t *s; | ||
| 832 | uint8_t *d; | ||
| 833 | int height; | ||
| 834 | |||
| 835 | do { | ||
| 836 | height = h; | ||
| 837 | s = src; | ||
| 838 | d = dst; | ||
| 839 | load_u8_8x8(s, src_stride, &t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); | ||
| 840 | transpose_u8_8x8(&t0, &t1, &t2, &t3, &t4, &t5, &t6, &t7); | ||
| 841 | s += 8 * src_stride; | ||
| 842 | |||
| 843 | do { | ||
| 844 | load_u8_8x8(s, src_stride, &t8, &t9, &t10, &t11, &t12, &t13, &t14, | ||
| 845 | &t15); | ||
| 846 | transpose_u8_8x8(&t8, &t9, &t10, &t11, &t12, &t13, &t14, &t15); | ||
| 847 | s0 = vcombine_u8(t0, t8); | ||
| 848 | s1 = vcombine_u8(t1, t9); | ||
| 849 | s2 = vcombine_u8(t2, t10); | ||
| 850 | s3 = vcombine_u8(t3, t11); | ||
| 851 | s4 = vcombine_u8(t4, t12); | ||
| 852 | s5 = vcombine_u8(t5, t13); | ||
| 853 | s6 = vcombine_u8(t6, t14); | ||
| 854 | s7 = vcombine_u8(t7, t15); | ||
| 855 | |||
| 856 | d0 = convolve8_8_dot(s0, filters, correction, range_limit, permute_tbl); | ||
| 857 | d1 = convolve8_8_dot(s1, filters, correction, range_limit, permute_tbl); | ||
| 858 | d2 = convolve8_8_dot(s2, filters, correction, range_limit, permute_tbl); | ||
| 859 | d3 = convolve8_8_dot(s3, filters, correction, range_limit, permute_tbl); | ||
| 860 | d4 = convolve8_8_dot(s4, filters, correction, range_limit, permute_tbl); | ||
| 861 | d5 = convolve8_8_dot(s5, filters, correction, range_limit, permute_tbl); | ||
| 862 | d6 = convolve8_8_dot(s6, filters, correction, range_limit, permute_tbl); | ||
| 863 | d7 = convolve8_8_dot(s7, filters, correction, range_limit, permute_tbl); | ||
| 864 | |||
| 865 | transpose_u8_8x8(&d0, &d1, &d2, &d3, &d4, &d5, &d6, &d7); | ||
| 866 | store_u8_8x8(d, dst_stride, d0, d1, d2, d3, d4, d5, d6, d7); | ||
| 867 | |||
| 868 | t0 = t8; | ||
| 869 | t1 = t9; | ||
| 870 | t2 = t10; | ||
| 871 | t3 = t11; | ||
| 872 | t4 = t12; | ||
| 873 | t5 = t13; | ||
| 874 | t6 = t14; | ||
| 875 | t7 = t15; | ||
| 876 | s += 8 * src_stride; | ||
| 877 | d += 8 * dst_stride; | ||
| 878 | height -= 8; | ||
| 879 | } while (height > 0); | ||
| 880 | src += 8; | ||
| 881 | dst += 8; | ||
| 882 | w -= 8; | ||
| 883 | } while (w > 0); | ||
| 884 | } | ||
| 885 | } | ||
| 886 | |||
| 887 | #else | ||
| 888 | |||
| 707 | void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride, | 889 | void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride, |
| 708 | uint8_t *dst, ptrdiff_t dst_stride, | 890 | uint8_t *dst, ptrdiff_t dst_stride, |
| 709 | const InterpKernel *filter, int x0_q4, | 891 | const InterpKernel *filter, int x0_q4, |
| @@ -876,6 +1058,8 @@ void vpx_convolve8_vert_neon(const uint8_t *src, ptrdiff_t src_stride, · 166 unmodified lines | |||
| 876 | } | 1058 | } |
| 877 | } | 1059 | } |
| 878 | 1060 | ||
| 1061 | #endif | ||
| 1062 | |||
| 879 | void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride, | 1063 | void vpx_convolve8_avg_vert_neon(const uint8_t *src, ptrdiff_t src_stride, |
| 880 | uint8_t *dst, ptrdiff_t dst_stride, | 1064 | uint8_t *dst, ptrdiff_t dst_stride, |
| 881 | const InterpKernel *filter, int x0_q4, | 1065 | const InterpKernel *filter, int x0_q4, |