checkasm bench: put_hevc_epel_uni_hv4_8_c: 204.7 put_hevc_epel_uni_hv4_8_i8mm: 70.2 put_hevc_epel_uni_hv6_8_c: 378.2 put_hevc_epel_uni_hv6_8_i8mm: 131.9 put_hevc_epel_uni_hv8_8_c: 637.7 put_hevc_epel_uni_hv8_8_i8mm: 137.9 put_hevc_epel_uni_hv12_8_c: 1301.9 put_hevc_epel_uni_hv12_8_i8mm: 314.2 put_hevc_epel_uni_hv16_8_c: 2203.4 put_hevc_epel_uni_hv16_8_i8mm: 454.7 put_hevc_epel_uni_hv24_8_c: 4848.2 put_hevc_epel_uni_hv24_8_i8mm: 1065.2 put_hevc_epel_uni_hv32_8_c: 8517.4 put_hevc_epel_uni_hv32_8_i8mm: 1898.4 put_hevc_epel_uni_hv48_8_c: 19591.7 put_hevc_epel_uni_hv48_8_i8mm: 4107.2 put_hevc_epel_uni_hv64_8_c: 33880.2 put_hevc_epel_uni_hv64_8_i8mm: 6568.7
Unified · Split
| @@ -682,6 +682,300 @@ function ff_hevc_put_hevc_epel_h64_8_neon_i8mm, export=1 · 681 unmodified lines | |||
| 682 | ret | 682 | ret |
| 683 | endfunc | 683 | endfunc |
| 684 | 684 | ||
| 685 | function ff_hevc_put_hevc_epel_uni_hv4_8_neon_i8mm, export=1 | ||
| 686 | add w10, w4, #3 | ||
| 687 | lsl x10, x10, #7 | ||
| 688 | sub sp, sp, x10 // tmp_array | ||
| 689 | str x30, [sp, #-48]! | ||
| 690 | stp x4, x6, [sp, #16] | ||
| 691 | stp x0, x1, [sp, #32] | ||
| 692 | add x0, sp, #48 | ||
| 693 | sub x1, x2, x3 | ||
| 694 | mov x2, x3 | ||
| 695 | add w3, w4, #3 | ||
| 696 | mov x4, x5 | ||
| 697 | bl X(ff_hevc_put_hevc_epel_h4_8_neon_i8mm) | ||
| 698 | ldp x4, x6, [sp, #16] | ||
| 699 | ldp x0, x1, [sp, #32] | ||
| 700 | ldr x30, [sp], #48 | ||
| 701 | load_epel_filterh x6, x5 | ||
| 702 | mov x10, #(MAX_PB_SIZE * 2) | ||
| 703 | ld1 {v16.4h}, [sp], x10 | ||
| 704 | ld1 {v17.4h}, [sp], x10 | ||
| 705 | ld1 {v18.4h}, [sp], x10 | ||
| 706 | .macro calc src0, src1, src2, src3 | ||
| 707 | ld1 {\src3\().4h}, [sp], x10 | ||
| 708 | calc_epelh v4, \src0, \src1, \src2, \src3 | ||
| 709 | sqrshrun v4.8b, v4.8h, #6 | ||
| 710 | subs w4, w4, #1 | ||
| 711 | st1 {v4.s}[0], [x0], x1 | ||
| 712 | .endm | ||
| 713 | 1: calc_all4 | ||
| 714 | .purgem calc | ||
| 715 | 2: ret | ||
| 716 | endfunc | ||
| 717 | |||
| 718 | function ff_hevc_put_hevc_epel_uni_hv6_8_neon_i8mm, export=1 | ||
| 719 | add w10, w4, #3 | ||
| 720 | lsl x10, x10, #7 | ||
| 721 | sub sp, sp, x10 // tmp_array | ||
| 722 | str x30, [sp, #-48]! | ||
| 723 | stp x4, x6, [sp, #16] | ||
| 724 | stp x0, x1, [sp, #32] | ||
| 725 | add x0, sp, #48 | ||
| 726 | sub x1, x2, x3 | ||
| 727 | mov x2, x3 | ||
| 728 | add w3, w4, #3 | ||
| 729 | mov x4, x5 | ||
| 730 | bl X(ff_hevc_put_hevc_epel_h6_8_neon_i8mm) | ||
| 731 | ldp x4, x6, [sp, #16] | ||
| 732 | ldp x0, x1, [sp, #32] | ||
| 733 | ldr x30, [sp], #48 | ||
| 734 | load_epel_filterh x6, x5 | ||
| 735 | sub x1, x1, #4 | ||
| 736 | mov x10, #(MAX_PB_SIZE * 2) | ||
| 737 | ld1 {v16.8h}, [sp], x10 | ||
| 738 | ld1 {v17.8h}, [sp], x10 | ||
| 739 | ld1 {v18.8h}, [sp], x10 | ||
| 740 | .macro calc src0, src1, src2, src3 | ||
| 741 | ld1 {\src3\().8h}, [sp], x10 | ||
| 742 | calc_epelh v4, \src0, \src1, \src2, \src3 | ||
| 743 | calc_epelh2 v4, v5, \src0, \src1, \src2, \src3 | ||
| 744 | sqrshrun v4.8b, v4.8h, #6 | ||
| 745 | st1 {v4.s}[0], [x0], #4 | ||
| 746 | subs w4, w4, #1 | ||
| 747 | st1 {v4.h}[2], [x0], x1 | ||
| 748 | .endm | ||
| 749 | 1: calc_all4 | ||
| 750 | .purgem calc | ||
| 751 | 2: ret | ||
| 752 | endfunc | ||
| 753 | |||
| 754 | function ff_hevc_put_hevc_epel_uni_hv8_8_neon_i8mm, export=1 | ||
| 755 | add w10, w4, #3 | ||
| 756 | lsl x10, x10, #7 | ||
| 757 | sub sp, sp, x10 // tmp_array | ||
| 758 | str x30, [sp, #-48]! | ||
| 759 | stp x4, x6, [sp, #16] | ||
| 760 | stp x0, x1, [sp, #32] | ||
| 761 | add x0, sp, #48 | ||
| 762 | sub x1, x2, x3 | ||
| 763 | mov x2, x3 | ||
| 764 | add w3, w4, #3 | ||
| 765 | mov x4, x5 | ||
| 766 | bl X(ff_hevc_put_hevc_epel_h8_8_neon_i8mm) | ||
| 767 | ldp x4, x6, [sp, #16] | ||
| 768 | ldp x0, x1, [sp, #32] | ||
| 769 | ldr x30, [sp], #48 | ||
| 770 | load_epel_filterh x6, x5 | ||
| 771 | mov x10, #(MAX_PB_SIZE * 2) | ||
| 772 | ld1 {v16.8h}, [sp], x10 | ||
| 773 | ld1 {v17.8h}, [sp], x10 | ||
| 774 | ld1 {v18.8h}, [sp], x10 | ||
| 775 | .macro calc src0, src1, src2, src3 | ||
| 776 | ld1 {\src3\().8h}, [sp], x10 | ||
| 777 | calc_epelh v4, \src0, \src1, \src2, \src3 | ||
| 778 | calc_epelh2 v4, v5, \src0, \src1, \src2, \src3 | ||
| 779 | sqrshrun v4.8b, v4.8h, #6 | ||
| 780 | subs w4, w4, #1 | ||
| 781 | st1 {v4.8b}, [x0], x1 | ||
| 782 | .endm | ||
| 783 | 1: calc_all4 | ||
| 784 | .purgem calc | ||
| 785 | 2: ret | ||
| 786 | endfunc | ||
| 787 | |||
| 788 | function ff_hevc_put_hevc_epel_uni_hv12_8_neon_i8mm, export=1 | ||
| 789 | add w10, w4, #3 | ||
| 790 | lsl x10, x10, #7 | ||
| 791 | sub sp, sp, x10 // tmp_array | ||
| 792 | str x30, [sp, #-48]! | ||
| 793 | stp x4, x6, [sp, #16] | ||
| 794 | stp x0, x1, [sp, #32] | ||
| 795 | add x0, sp, #48 | ||
| 796 | sub x1, x2, x3 | ||
| 797 | mov x2, x3 | ||
| 798 | add w3, w4, #3 | ||
| 799 | mov x4, x5 | ||
| 800 | bl X(ff_hevc_put_hevc_epel_h12_8_neon_i8mm) | ||
| 801 | ldp x4, x6, [sp, #16] | ||
| 802 | ldp x0, x1, [sp, #32] | ||
| 803 | ldr x30, [sp], #48 | ||
| 804 | load_epel_filterh x6, x5 | ||
| 805 | sub x1, x1, #8 | ||
| 806 | mov x10, #(MAX_PB_SIZE * 2) | ||
| 807 | ld1 {v16.8h, v17.8h}, [sp], x10 | ||
| 808 | ld1 {v18.8h, v19.8h}, [sp], x10 | ||
| 809 | ld1 {v20.8h, v21.8h}, [sp], x10 | ||
| 810 | .macro calc src0, src1, src2, src3, src4, src5, src6, src7 | ||
| 811 | ld1 {\src6\().8h, \src7\().8h}, [sp], x10 | ||
| 812 | calc_epelh v4, \src0, \src2, \src4, \src6 | ||
| 813 | calc_epelh2 v4, v5, \src0, \src2, \src4, \src6 | ||
| 814 | calc_epelh v5, \src1, \src3, \src5, \src7 | ||
| 815 | sqrshrun v4.8b, v4.8h, #6 | ||
| 816 | sqrshrun2 v4.16b, v5.8h, #6 | ||
| 817 | st1 {v4.8b}, [x0], #8 | ||
| 818 | st1 {v4.s}[2], [x0], x1 | ||
| 819 | subs w4, w4, #1 | ||
| 820 | .endm | ||
| 821 | 1: calc_all8 | ||
| 822 | .purgem calc | ||
| 823 | 2: ret | ||
| 824 | endfunc | ||
| 825 | |||
| 826 | function ff_hevc_put_hevc_epel_uni_hv16_8_neon_i8mm, export=1 | ||
| 827 | add w10, w4, #3 | ||
| 828 | lsl x10, x10, #7 | ||
| 829 | sub sp, sp, x10 // tmp_array | ||
| 830 | str x30, [sp, #-48]! | ||
| 831 | stp x4, x6, [sp, #16] | ||
| 832 | stp x0, x1, [sp, #32] | ||
| 833 | add x0, sp, #48 | ||
| 834 | sub x1, x2, x3 | ||
| 835 | mov x2, x3 | ||
| 836 | add w3, w4, #3 | ||
| 837 | mov x4, x5 | ||
| 838 | bl X(ff_hevc_put_hevc_epel_h16_8_neon_i8mm) | ||
| 839 | ldp x4, x6, [sp, #16] | ||
| 840 | ldp x0, x1, [sp, #32] | ||
| 841 | ldr x30, [sp], #48 | ||
| 842 | load_epel_filterh x6, x5 | ||
| 843 | mov x10, #(MAX_PB_SIZE * 2) | ||
| 844 | ld1 {v16.8h, v17.8h}, [sp], x10 | ||
| 845 | ld1 {v18.8h, v19.8h}, [sp], x10 | ||
| 846 | ld1 {v20.8h, v21.8h}, [sp], x10 | ||
| 847 | .macro calc src0, src1, src2, src3, src4, src5, src6, src7 | ||
| 848 | ld1 {\src6\().8h, \src7\().8h}, [sp], x10 | ||
| 849 | calc_epelh v4, \src0, \src2, \src4, \src6 | ||
| 850 | calc_epelh2 v4, v5, \src0, \src2, \src4, \src6 | ||
| 851 | calc_epelh v5, \src1, \src3, \src5, \src7 | ||
| 852 | calc_epelh2 v5, v6, \src1, \src3, \src5, \src7 | ||
| 853 | sqrshrun v4.8b, v4.8h, #6 | ||
| 854 | sqrshrun2 v4.16b, v5.8h, #6 | ||
| 855 | subs w4, w4, #1 | ||
| 856 | st1 {v4.16b}, [x0], x1 | ||
| 857 | .endm | ||
| 858 | 1: calc_all8 | ||
| 859 | .purgem calc | ||
| 860 | 2: ret | ||
| 861 | endfunc | ||
| 862 | |||
| 863 | function ff_hevc_put_hevc_epel_uni_hv24_8_neon_i8mm, export=1 | ||
| 864 | add w10, w4, #3 | ||
| 865 | lsl x10, x10, #7 | ||
| 866 | sub sp, sp, x10 // tmp_array | ||
| 867 | str x30, [sp, #-48]! | ||
| 868 | stp x4, x6, [sp, #16] | ||
| 869 | stp x0, x1, [sp, #32] | ||
| 870 | add x0, sp, #48 | ||
| 871 | sub x1, x2, x3 | ||
| 872 | mov x2, x3 | ||
| 873 | add w3, w4, #3 | ||
| 874 | mov x4, x5 | ||
| 875 | bl X(ff_hevc_put_hevc_epel_h24_8_neon_i8mm) | ||
| 876 | ldp x4, x6, [sp, #16] | ||
| 877 | ldp x0, x1, [sp, #32] | ||
| 878 | ldr x30, [sp], #48 | ||
| 879 | load_epel_filterh x6, x5 | ||
| 880 | mov x10, #(MAX_PB_SIZE * 2) | ||
| 881 | ld1 {v16.8h, v17.8h, v18.8h}, [sp], x10 | ||
| 882 | ld1 {v19.8h, v20.8h, v21.8h}, [sp], x10 | ||
| 883 | ld1 {v22.8h, v23.8h, v24.8h}, [sp], x10 | ||
| 884 | .macro calc src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11 | ||
| 885 | ld1 {\src9\().8h, \src10\().8h, \src11\().8h}, [sp], x10 | ||
| 886 | calc_epelh v4, \src0, \src3, \src6, \src9 | ||
| 887 | calc_epelh2 v4, v5, \src0, \src3, \src6, \src9 | ||
| 888 | calc_epelh v5, \src1, \src4, \src7, \src10 | ||
| 889 | calc_epelh2 v5, v6, \src1, \src4, \src7, \src10 | ||
| 890 | calc_epelh v6, \src2, \src5, \src8, \src11 | ||
| 891 | calc_epelh2 v6, v7, \src2, \src5, \src8, \src11 | ||
| 892 | sqrshrun v4.8b, v4.8h, #6 | ||
| 893 | sqrshrun v5.8b, v5.8h, #6 | ||
| 894 | sqrshrun v6.8b, v6.8h, #6 | ||
| 895 | subs w4, w4, #1 | ||
| 896 | st1 {v4.8b, v5.8b, v6.8b}, [x0], x1 | ||
| 897 | .endm | ||
| 898 | 1: calc_all12 | ||
| 899 | .purgem calc | ||
| 900 | 2: ret | ||
| 901 | endfunc | ||
| 902 | |||
| 903 | function ff_hevc_put_hevc_epel_uni_hv32_8_neon_i8mm, export=1 | ||
| 904 | stp x5, x6, [sp, #-64]! | ||
| 905 | stp x3, x4, [sp, #16] | ||
| 906 | stp x1, x2, [sp, #32] | ||
| 907 | stp x0, x30, [sp, #48] | ||
| 908 | mov x7, #16 | ||
| 909 | bl X(ff_hevc_put_hevc_epel_uni_hv16_8_neon_i8mm) | ||
| 910 | ldp x5, x6, [sp] | ||
| 911 | ldp x3, x4, [sp, #16] | ||
| 912 | ldp x1, x2, [sp, #32] | ||
| 913 | ldr x0, [sp, #48] | ||
| 914 | add x0, x0, #16 | ||
| 915 | add x2, x2, #16 | ||
| 916 | mov x7, #16 | ||
| 917 | bl X(ff_hevc_put_hevc_epel_uni_hv16_8_neon_i8mm) | ||
| 918 | ldr x30, [sp, #56] | ||
| 919 | add sp, sp, #64 | ||
| 920 | ret | ||
| 921 | endfunc | ||
| 922 | |||
| 923 | function ff_hevc_put_hevc_epel_uni_hv48_8_neon_i8mm, export=1 | ||
| 924 | stp x5, x6, [sp, #-64]! | ||
| 925 | stp x3, x4, [sp, #16] | ||
| 926 | stp x1, x2, [sp, #32] | ||
| 927 | stp x0, x30, [sp, #48] | ||
| 928 | mov x7, #24 | ||
| 929 | bl X(ff_hevc_put_hevc_epel_uni_hv24_8_neon_i8mm) | ||
| 930 | ldp x5, x6, [sp] | ||
| 931 | ldp x3, x4, [sp, #16] | ||
| 932 | ldp x1, x2, [sp, #32] | ||
| 933 | ldr x0, [sp, #48] | ||
| 934 | add x0, x0, #24 | ||
| 935 | add x2, x2, #24 | ||
| 936 | mov x7, #24 | ||
| 937 | bl X(ff_hevc_put_hevc_epel_uni_hv24_8_neon_i8mm) | ||
| 938 | ldr x30, [sp, #56] | ||
| 939 | add sp, sp, #64 | ||
| 940 | ret | ||
| 941 | endfunc | ||
| 942 | |||
| 943 | function ff_hevc_put_hevc_epel_uni_hv64_8_neon_i8mm, export=1 | ||
| 944 | stp x5, x6, [sp, #-64]! | ||
| 945 | stp x3, x4, [sp, #16] | ||
| 946 | stp x1, x2, [sp, #32] | ||
| 947 | stp x0, x30, [sp, #48] | ||
| 948 | mov x7, #16 | ||
| 949 | bl X(ff_hevc_put_hevc_epel_uni_hv16_8_neon_i8mm) | ||
| 950 | ldp x5, x6, [sp] | ||
| 951 | ldp x3, x4, [sp, #16] | ||
| 952 | ldp x1, x2, [sp, #32] | ||
| 953 | ldr x0, [sp, #48] | ||
| 954 | add x0, x0, #16 | ||
| 955 | add x2, x2, #16 | ||
| 956 | mov x7, #16 | ||
| 957 | bl X(ff_hevc_put_hevc_epel_uni_hv16_8_neon_i8mm) | ||
| 958 | ldp x5, x6, [sp] | ||
| 959 | ldp x3, x4, [sp, #16] | ||
| 960 | ldp x1, x2, [sp, #32] | ||
| 961 | ldr x0, [sp, #48] | ||
| 962 | add x0, x0, #32 | ||
| 963 | add x2, x2, #32 | ||
| 964 | mov x7, #16 | ||
| 965 | bl X(ff_hevc_put_hevc_epel_uni_hv16_8_neon_i8mm) | ||
| 966 | ldp x5, x6, [sp] | ||
| 967 | ldp x3, x4, [sp, #16] | ||
| 968 | ldp x1, x2, [sp, #32] | ||
| 969 | ldr x0, [sp, #48] | ||
| 970 | add x0, x0, #48 | ||
| 971 | add x2, x2, #48 | ||
| 972 | mov x7, #16 | ||
| 973 | bl X(ff_hevc_put_hevc_epel_uni_hv16_8_neon_i8mm) | ||
| 974 | ldr x30, [sp, #56] | ||
| 975 | add sp, sp, #64 | ||
| 976 | ret | ||
| 977 | endfunc | ||
| 978 | |||
| 685 | .macro EPEL_UNI_W_H_HEADER | 979 | .macro EPEL_UNI_W_H_HEADER |
| 686 | ldr x12, [sp] | 980 | ldr x12, [sp] |
| 687 | sub x2, x2, #1 | 981 | sub x2, x2, #1 |
| @@ -165,6 +165,10 @@ NEON8_FNPROTO(epel_uni_v, (uint8_t *dst, ptrdiff_t dststride, · 164 unmodified lines | |||
| 165 | const uint8_t *src, ptrdiff_t srcstride, | 165 | const uint8_t *src, ptrdiff_t srcstride, |
| 166 | int height, intptr_t mx, intptr_t my, int width),); | 166 | int height, intptr_t mx, intptr_t my, int width),); |
| 167 | 167 | ||
| 168 | NEON8_FNPROTO(epel_uni_hv, (uint8_t *dst, ptrdiff_t _dststride, | ||
| 169 | const uint8_t *src, ptrdiff_t srcstride, | ||
| 170 | int height, intptr_t mx, intptr_t my, int width), _i8mm); | ||
| 171 | |||
| 168 | NEON8_FNPROTO(epel_uni_w_v, (uint8_t *_dst, ptrdiff_t _dststride, | 172 | NEON8_FNPROTO(epel_uni_w_v, (uint8_t *_dst, ptrdiff_t _dststride, |
| 169 | const uint8_t *_src, ptrdiff_t _srcstride, | 173 | const uint8_t *_src, ptrdiff_t _srcstride, |
| 170 | int height, int denom, int wx, int ox, | 174 | int height, int denom, int wx, int ox, |
| @@ -298,6 +302,7 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) · 127 unmodified lines | |||
| 298 | 302 | ||
| 299 | if (have_i8mm(cpu_flags)) { | 303 | if (have_i8mm(cpu_flags)) { |
| 300 | NEON8_FNASSIGN(c->put_hevc_epel, 0, 1, epel_h, _i8mm); | 304 | NEON8_FNASSIGN(c->put_hevc_epel, 0, 1, epel_h, _i8mm); |
| 305 | NEON8_FNASSIGN(c->put_hevc_epel_uni, 1, 1, epel_uni_hv, _i8mm); | ||
| 301 | NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 0, 1, epel_uni_w_h ,_i8mm); | 306 | NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 0, 1, epel_uni_w_h ,_i8mm); |
| 302 | NEON8_FNASSIGN(c->put_hevc_qpel, 0, 1, qpel_h, _i8mm); | 307 | NEON8_FNASSIGN(c->put_hevc_qpel, 0, 1, qpel_h, _i8mm); |
| 303 | NEON8_FNASSIGN(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h, _i8mm); | 308 | NEON8_FNASSIGN(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h, _i8mm); |