put_hevc_epel_uni_w_hv4_8_c: 254.6 put_hevc_epel_uni_w_hv4_8_i8mm: 102.9 put_hevc_epel_uni_w_hv6_8_c: 411.6 put_hevc_epel_uni_w_hv6_8_i8mm: 221.6 put_hevc_epel_uni_w_hv8_8_c: 669.4 put_hevc_epel_uni_w_hv8_8_i8mm: 214.9 put_hevc_epel_uni_w_hv12_8_c: 1412.6 put_hevc_epel_uni_w_hv12_8_i8mm: 481.4 put_hevc_epel_uni_w_hv16_8_c: 2425.4 put_hevc_epel_uni_w_hv16_8_i8mm: 647.4 put_hevc_epel_uni_w_hv24_8_c: 5384.1 put_hevc_epel_uni_w_hv24_8_i8mm: 1450.6 put_hevc_epel_uni_w_hv32_8_c: 9470.9 put_hevc_epel_uni_w_hv32_8_i8mm: 2497.1 put_hevc_epel_uni_w_hv48_8_c: 20930.1 put_hevc_epel_uni_w_hv48_8_i8mm: 5635.9 put_hevc_epel_uni_w_hv64_8_c: 36682.9 put_hevc_epel_uni_w_hv64_8_i8mm: 9712.6
Unified · Split
| @@ -717,6 +717,674 @@ function ff_hevc_put_hevc_epel_uni_w_h64_8_neon_i8mm, export=1 · 716 unmodified lines | |||
| 717 | ret | 717 | ret |
| 718 | endfunc | 718 | endfunc |
| 719 | 719 | ||
| 720 | .macro epel_uni_w_hv_start | ||
| 721 | mov x15, x5 //denom | ||
| 722 | mov x16, x6 //wx | ||
| 723 | mov x17, x7 //ox | ||
| 724 | add w15, w15, #6 //shift = denom+6 | ||
| 725 | |||
| 726 | |||
| 727 | ldp x5, x6, [sp] | ||
| 728 | ldr x7, [sp, #16] | ||
| 729 | |||
| 730 | stp d14, d15, [sp, #-64]! | ||
| 731 | stp d8, d9, [sp, #16] | ||
| 732 | stp d10, d11, [sp, #32] | ||
| 733 | stp d12, d13, [sp, #48] | ||
| 734 | |||
| 735 | dup v13.8h, w16 //wx | ||
| 736 | dup v14.4s, w17 //ox | ||
| 737 | |||
| 738 | mov w17, #1 | ||
| 739 | lsl w17, w17, w15 | ||
| 740 | lsr w17, w17, #1 | ||
| 741 | dup v15.4s, w17 | ||
| 742 | |||
| 743 | neg w15, w15 // -shift | ||
| 744 | dup v12.4s, w15 //shift | ||
| 745 | .endm | ||
| 746 | |||
| 747 | .macro epel_uni_w_hv_end | ||
| 748 | smull v28.4s, v4.4h, v13.4h | ||
| 749 | smull2 v29.4s, v4.8h, v13.8h | ||
| 750 | add v28.4s, v28.4s, v15.4s | ||
| 751 | add v29.4s, v29.4s, v15.4s | ||
| 752 | sshl v28.4s, v28.4s, v12.4s | ||
| 753 | sshl v29.4s, v29.4s, v12.4s | ||
| 754 | add v28.4s, v28.4s, v14.4s | ||
| 755 | add v29.4s, v29.4s, v14.4s | ||
| 756 | sqxtn v4.4h, v28.4s | ||
| 757 | sqxtn2 v4.8h, v29.4s | ||
| 758 | .endm | ||
| 759 | |||
| 760 | .macro epel_uni_w_hv_end2 | ||
| 761 | smull v28.4s, v4.4h, v13.4h | ||
| 762 | smull2 v29.4s, v4.8h, v13.8h | ||
| 763 | smull v30.4s, v5.4h, v13.4h | ||
| 764 | smull2 v31.4s, v5.8h, v13.8h | ||
| 765 | add v28.4s, v28.4s, v15.4s | ||
| 766 | add v29.4s, v29.4s, v15.4s | ||
| 767 | add v30.4s, v30.4s, v15.4s | ||
| 768 | add v31.4s, v31.4s, v15.4s | ||
| 769 | |||
| 770 | sshl v28.4s, v28.4s, v12.4s | ||
| 771 | sshl v29.4s, v29.4s, v12.4s | ||
| 772 | sshl v30.4s, v30.4s, v12.4s | ||
| 773 | sshl v31.4s, v31.4s, v12.4s | ||
| 774 | |||
| 775 | add v28.4s, v28.4s, v14.4s | ||
| 776 | add v29.4s, v29.4s, v14.4s | ||
| 777 | add v30.4s, v30.4s, v14.4s | ||
| 778 | add v31.4s, v31.4s, v14.4s | ||
| 779 | |||
| 780 | sqxtn v4.4h, v28.4s | ||
| 781 | sqxtn2 v4.8h, v29.4s | ||
| 782 | sqxtn v5.4h, v30.4s | ||
| 783 | sqxtn2 v5.8h, v31.4s | ||
| 784 | .endm | ||
| 785 | |||
| 786 | .macro epel_uni_w_hv_end3 | ||
| 787 | smull v1.4s, v4.4h, v13.4h | ||
| 788 | smull2 v2.4s, v4.8h, v13.8h | ||
| 789 | smull v28.4s, v5.4h, v13.4h | ||
| 790 | smull2 v29.4s, v5.8h, v13.8h | ||
| 791 | smull v30.4s, v6.4h, v13.4h | ||
| 792 | smull2 v31.4s, v6.8h, v13.8h | ||
| 793 | add v1.4s, v1.4s, v15.4s | ||
| 794 | add v2.4s, v2.4s, v15.4s | ||
| 795 | add v28.4s, v28.4s, v15.4s | ||
| 796 | add v29.4s, v29.4s, v15.4s | ||
| 797 | add v30.4s, v30.4s, v15.4s | ||
| 798 | add v31.4s, v31.4s, v15.4s | ||
| 799 | |||
| 800 | sshl v1.4s, v1.4s, v12.4s | ||
| 801 | sshl v2.4s, v2.4s, v12.4s | ||
| 802 | sshl v28.4s, v28.4s, v12.4s | ||
| 803 | sshl v29.4s, v29.4s, v12.4s | ||
| 804 | sshl v30.4s, v30.4s, v12.4s | ||
| 805 | sshl v31.4s, v31.4s, v12.4s | ||
| 806 | add v1.4s, v1.4s, v14.4s | ||
| 807 | add v2.4s, v2.4s, v14.4s | ||
| 808 | add v28.4s, v28.4s, v14.4s | ||
| 809 | add v29.4s, v29.4s, v14.4s | ||
| 810 | add v30.4s, v30.4s, v14.4s | ||
| 811 | add v31.4s, v31.4s, v14.4s | ||
| 812 | |||
| 813 | sqxtn v4.4h, v1.4s | ||
| 814 | sqxtn2 v4.8h, v2.4s | ||
| 815 | sqxtn v5.4h, v28.4s | ||
| 816 | sqxtn2 v5.8h, v29.4s | ||
| 817 | sqxtn v6.4h, v30.4s | ||
| 818 | sqxtn2 v6.8h, v31.4s | ||
| 819 | .endm | ||
| 820 | |||
| 821 | .macro calc_epelh dst, src0, src1, src2, src3 | ||
| 822 | smull \dst\().4s, \src0\().4h, v0.h[0] | ||
| 823 | smlal \dst\().4s, \src1\().4h, v0.h[1] | ||
| 824 | smlal \dst\().4s, \src2\().4h, v0.h[2] | ||
| 825 | smlal \dst\().4s, \src3\().4h, v0.h[3] | ||
| 826 | sqshrn \dst\().4h, \dst\().4s, #6 | ||
| 827 | .endm | ||
| 828 | |||
| 829 | .macro calc_epelh2 dst, tmp, src0, src1, src2, src3 | ||
| 830 | smull2 \tmp\().4s, \src0\().8h, v0.h[0] | ||
| 831 | smlal2 \tmp\().4s, \src1\().8h, v0.h[1] | ||
| 832 | smlal2 \tmp\().4s, \src2\().8h, v0.h[2] | ||
| 833 | smlal2 \tmp\().4s, \src3\().8h, v0.h[3] | ||
| 834 | sqshrn2 \dst\().8h, \tmp\().4s, #6 | ||
| 835 | .endm | ||
| 836 | |||
| 837 | .macro load_epel_filterh freg, xreg | ||
| 838 | movrel \xreg, epel_filters | ||
| 839 | add \xreg, \xreg, \freg, lsl #2 | ||
| 840 | ld1 {v0.8b}, [\xreg] | ||
| 841 | sxtl v0.8h, v0.8b | ||
| 842 | .endm | ||
| 843 | |||
| 844 | function ff_hevc_put_hevc_epel_uni_w_hv4_8_neon_i8mm, export=1 | ||
| 845 | epel_uni_w_hv_start | ||
| 846 | sxtw x4, w4 | ||
| 847 | |||
| 848 | add x10, x4, #3 | ||
| 849 | lsl x10, x10, #7 | ||
| 850 | sub sp, sp, x10 // tmp_array | ||
| 851 | str x30, [sp, #-48]! | ||
| 852 | stp x4, x6, [sp, #16] | ||
| 853 | stp x0, x1, [sp, #32] | ||
| 854 | add x0, sp, #48 | ||
| 855 | sub x1, x2, x3 | ||
| 856 | mov x2, x3 | ||
| 857 | add x3, x4, #3 | ||
| 858 | mov x4, x5 | ||
| 859 | bl X(ff_hevc_put_hevc_epel_h4_8_neon_i8mm) | ||
| 860 | ldp x4, x6, [sp, #16] | ||
| 861 | ldp x0, x1, [sp, #32] | ||
| 862 | ldr x30, [sp], #48 | ||
| 863 | load_epel_filterh x6, x5 | ||
| 864 | mov x10, #(MAX_PB_SIZE * 2) | ||
| 865 | ld1 {v16.4h}, [sp], x10 | ||
| 866 | ld1 {v17.4h}, [sp], x10 | ||
| 867 | ld1 {v18.4h}, [sp], x10 | ||
| 868 | 1: ld1 {v19.4h}, [sp], x10 | ||
| 869 | subs x4, x4, #1 | ||
| 870 | calc_epelh v4, v16, v17, v18, v19 | ||
| 871 | epel_uni_w_hv_end | ||
| 872 | sqxtun v4.8b, v4.8h | ||
| 873 | str s4, [x0] | ||
| 874 | add x0, x0, x1 | ||
| 875 | b.eq 2f | ||
| 876 | |||
| 877 | ld1 {v16.4h}, [sp], x10 | ||
| 878 | subs x4, x4, #1 | ||
| 879 | calc_epelh v4, v17, v18, v19, v16 | ||
| 880 | epel_uni_w_hv_end | ||
| 881 | sqxtun v4.8b, v4.8h | ||
| 882 | str s4, [x0] | ||
| 883 | add x0, x0, x1 | ||
| 884 | b.eq 2f | ||
| 885 | |||
| 886 | ld1 {v17.4h}, [sp], x10 | ||
| 887 | subs x4, x4, #1 | ||
| 888 | calc_epelh v4, v18, v19, v16, v17 | ||
| 889 | epel_uni_w_hv_end | ||
| 890 | sqxtun v4.8b, v4.8h | ||
| 891 | str s4, [x0] | ||
| 892 | add x0, x0, x1 | ||
| 893 | b.eq 2f | ||
| 894 | |||
| 895 | ld1 {v18.4h}, [sp], x10 | ||
| 896 | subs x4, x4, #1 | ||
| 897 | calc_epelh v4, v19, v16, v17, v18 | ||
| 898 | epel_uni_w_hv_end | ||
| 899 | sqxtun v4.8b, v4.8h | ||
| 900 | str s4, [x0] | ||
| 901 | add x0, x0, x1 | ||
| 902 | b.ne 1b | ||
| 903 | 2: | ||
| 904 | ldp d8, d9, [sp, #16] | ||
| 905 | ldp d10, d11, [sp, #32] | ||
| 906 | ldp d12, d13, [sp, #48] | ||
| 907 | ldp d14, d15, [sp], #64 | ||
| 908 | ret | ||
| 909 | endfunc | ||
| 910 | |||
| 911 | function ff_hevc_put_hevc_epel_uni_w_hv6_8_neon_i8mm, export=1 | ||
| 912 | epel_uni_w_hv_start | ||
| 913 | sxtw x4, w4 | ||
| 914 | |||
| 915 | add x10, x4, #3 | ||
| 916 | lsl x10, x10, #7 | ||
| 917 | sub sp, sp, x10 // tmp_array | ||
| 918 | str x30, [sp, #-48]! | ||
| 919 | stp x4, x6, [sp, #16] | ||
| 920 | stp x0, x1, [sp, #32] | ||
| 921 | add x0, sp, #48 | ||
| 922 | sub x1, x2, x3 | ||
| 923 | mov x2, x3 | ||
| 924 | add x3, x4, #3 | ||
| 925 | mov x4, x5 | ||
| 926 | bl X(ff_hevc_put_hevc_epel_h6_8_neon_i8mm) | ||
| 927 | ldp x4, x6, [sp, #16] | ||
| 928 | ldp x0, x1, [sp, #32] | ||
| 929 | ldr x30, [sp], #48 | ||
| 930 | load_epel_filterh x6, x5 | ||
| 931 | sub x1, x1, #4 | ||
| 932 | mov x10, #(MAX_PB_SIZE * 2) | ||
| 933 | ld1 {v16.8h}, [sp], x10 | ||
| 934 | ld1 {v17.8h}, [sp], x10 | ||
| 935 | ld1 {v18.8h}, [sp], x10 | ||
| 936 | 1: ld1 {v19.8h}, [sp], x10 | ||
| 937 | subs x4, x4, #1 | ||
| 938 | calc_epelh v4, v16, v17, v18, v19 | ||
| 939 | calc_epelh2 v4, v5, v16, v17, v18, v19 | ||
| 940 | epel_uni_w_hv_end | ||
| 941 | sqxtun v4.8b, v4.8h | ||
| 942 | st1 {v4.s}[0], [x0], #4 | ||
| 943 | st1 {v4.h}[2], [x0], x1 | ||
| 944 | b.eq 2f | ||
| 945 | |||
| 946 | ld1 {v16.8h}, [sp], x10 | ||
| 947 | subs x4, x4, #1 | ||
| 948 | calc_epelh v4, v17, v18, v19, v16 | ||
| 949 | calc_epelh2 v4, v5, v17, v18, v19, v16 | ||
| 950 | epel_uni_w_hv_end | ||
| 951 | sqxtun v4.8b, v4.8h | ||
| 952 | st1 {v4.s}[0], [x0], #4 | ||
| 953 | st1 {v4.h}[2], [x0], x1 | ||
| 954 | b.eq 2f | ||
| 955 | |||
| 956 | ld1 {v17.8h}, [sp], x10 | ||
| 957 | subs x4, x4, #1 | ||
| 958 | calc_epelh v4, v18, v19, v16, v17 | ||
| 959 | calc_epelh2 v4, v5, v18, v19, v16, v17 | ||
| 960 | epel_uni_w_hv_end | ||
| 961 | sqxtun v4.8b, v4.8h | ||
| 962 | st1 {v4.s}[0], [x0], #4 | ||
| 963 | st1 {v4.h}[2], [x0], x1 | ||
| 964 | b.eq 2f | ||
| 965 | |||
| 966 | ld1 {v18.8h}, [sp], x10 | ||
| 967 | subs x4, x4, #1 | ||
| 968 | calc_epelh v4, v19, v16, v17, v18 | ||
| 969 | calc_epelh2 v4, v5, v19, v16, v17, v18 | ||
| 970 | epel_uni_w_hv_end | ||
| 971 | sqxtun v4.8b, v4.8h | ||
| 972 | st1 {v4.s}[0], [x0], #4 | ||
| 973 | st1 {v4.h}[2], [x0], x1 | ||
| 974 | b.ne 1b | ||
| 975 | 2: | ||
| 976 | ldp d8, d9, [sp, #16] | ||
| 977 | ldp d10, d11, [sp, #32] | ||
| 978 | ldp d12, d13, [sp, #48] | ||
| 979 | ldp d14, d15, [sp], #64 | ||
| 980 | ret | ||
| 981 | endfunc | ||
| 982 | |||
| 983 | function ff_hevc_put_hevc_epel_uni_w_hv8_8_neon_i8mm, export=1 | ||
| 984 | epel_uni_w_hv_start | ||
| 985 | sxtw x4, w4 | ||
| 986 | |||
| 987 | add x10, x4, #3 | ||
| 988 | lsl x10, x10, #7 | ||
| 989 | sub sp, sp, x10 // tmp_array | ||
| 990 | str x30, [sp, #-48]! | ||
| 991 | stp x4, x6, [sp, #16] | ||
| 992 | stp x0, x1, [sp, #32] | ||
| 993 | add x0, sp, #48 | ||
| 994 | sub x1, x2, x3 | ||
| 995 | mov x2, x3 | ||
| 996 | add x3, x4, #3 | ||
| 997 | mov x4, x5 | ||
| 998 | bl X(ff_hevc_put_hevc_epel_h8_8_neon_i8mm) | ||
| 999 | ldp x4, x6, [sp, #16] | ||
| 1000 | ldp x0, x1, [sp, #32] | ||
| 1001 | ldr x30, [sp], #48 | ||
| 1002 | load_epel_filterh x6, x5 | ||
| 1003 | mov x10, #(MAX_PB_SIZE * 2) | ||
| 1004 | ld1 {v16.8h}, [sp], x10 | ||
| 1005 | ld1 {v17.8h}, [sp], x10 | ||
| 1006 | ld1 {v18.8h}, [sp], x10 | ||
| 1007 | 1: ld1 {v19.8h}, [sp], x10 | ||
| 1008 | subs x4, x4, #1 | ||
| 1009 | calc_epelh v4, v16, v17, v18, v19 | ||
| 1010 | calc_epelh2 v4, v5, v16, v17, v18, v19 | ||
| 1011 | epel_uni_w_hv_end | ||
| 1012 | sqxtun v4.8b, v4.8h | ||
| 1013 | st1 {v4.8b}, [x0], x1 | ||
| 1014 | b.eq 2f | ||
| 1015 | |||
| 1016 | ld1 {v16.8h}, [sp], x10 | ||
| 1017 | subs x4, x4, #1 | ||
| 1018 | calc_epelh v4, v17, v18, v19, v16 | ||
| 1019 | calc_epelh2 v4, v5, v17, v18, v19, v16 | ||
| 1020 | epel_uni_w_hv_end | ||
| 1021 | sqxtun v4.8b, v4.8h | ||
| 1022 | st1 {v4.8b}, [x0], x1 | ||
| 1023 | b.eq 2f | ||
| 1024 | |||
| 1025 | ld1 {v17.8h}, [sp], x10 | ||
| 1026 | subs x4, x4, #1 | ||
| 1027 | calc_epelh v4, v18, v19, v16, v17 | ||
| 1028 | calc_epelh2 v4, v5, v18, v19, v16, v17 | ||
| 1029 | epel_uni_w_hv_end | ||
| 1030 | sqxtun v4.8b, v4.8h | ||
| 1031 | st1 {v4.8b}, [x0], x1 | ||
| 1032 | b.eq 2f | ||
| 1033 | |||
| 1034 | ld1 {v18.8h}, [sp], x10 | ||
| 1035 | subs x4, x4, #1 | ||
| 1036 | calc_epelh v4, v19, v16, v17, v18 | ||
| 1037 | calc_epelh2 v4, v5, v19, v16, v17, v18 | ||
| 1038 | epel_uni_w_hv_end | ||
| 1039 | sqxtun v4.8b, v4.8h | ||
| 1040 | st1 {v4.8b}, [x0], x1 | ||
| 1041 | b.ne 1b | ||
| 1042 | 2: | ||
| 1043 | ldp d8, d9, [sp, #16] | ||
| 1044 | ldp d10, d11, [sp, #32] | ||
| 1045 | ldp d12, d13, [sp, #48] | ||
| 1046 | ldp d14, d15, [sp], #64 | ||
| 1047 | ret | ||
| 1048 | endfunc | ||
| 1049 | |||
| 1050 | function ff_hevc_put_hevc_epel_uni_w_hv12_8_neon_i8mm, export=1 | ||
| 1051 | epel_uni_w_hv_start | ||
| 1052 | sxtw x4, w4 | ||
| 1053 | |||
| 1054 | add x10, x4, #3 | ||
| 1055 | lsl x10, x10, #7 | ||
| 1056 | sub sp, sp, x10 // tmp_array | ||
| 1057 | str x30, [sp, #-48]! | ||
| 1058 | stp x4, x6, [sp, #16] | ||
| 1059 | stp x0, x1, [sp, #32] | ||
| 1060 | add x0, sp, #48 | ||
| 1061 | sub x1, x2, x3 | ||
| 1062 | mov x2, x3 | ||
| 1063 | add x3, x4, #3 | ||
| 1064 | mov x4, x5 | ||
| 1065 | bl X(ff_hevc_put_hevc_epel_h12_8_neon_i8mm) | ||
| 1066 | ldp x4, x6, [sp, #16] | ||
| 1067 | ldp x0, x1, [sp, #32] | ||
| 1068 | ldr x30, [sp], #48 | ||
| 1069 | load_epel_filterh x6, x5 | ||
| 1070 | sub x1, x1, #8 | ||
| 1071 | mov x10, #(MAX_PB_SIZE * 2) | ||
| 1072 | ld1 {v16.8h, v17.8h}, [sp], x10 | ||
| 1073 | ld1 {v18.8h, v19.8h}, [sp], x10 | ||
| 1074 | ld1 {v20.8h, v21.8h}, [sp], x10 | ||
| 1075 | 1: ld1 {v22.8h, v23.8h}, [sp], x10 | ||
| 1076 | subs x4, x4, #1 | ||
| 1077 | calc_epelh v4, v16, v18, v20, v22 | ||
| 1078 | calc_epelh2 v4, v5, v16, v18, v20, v22 | ||
| 1079 | calc_epelh v5, v17, v19, v21, v23 | ||
| 1080 | epel_uni_w_hv_end2 | ||
| 1081 | sqxtun v4.8b, v4.8h | ||
| 1082 | sqxtun2 v4.16b, v5.8h | ||
| 1083 | st1 {v4.8b}, [x0], #8 | ||
| 1084 | st1 {v4.s}[2], [x0], x1 | ||
| 1085 | b.eq 2f | ||
| 1086 | |||
| 1087 | ld1 {v16.8h, v17.8h}, [sp], x10 | ||
| 1088 | subs x4, x4, #1 | ||
| 1089 | calc_epelh v4, v18, v20, v22, v16 | ||
| 1090 | calc_epelh2 v4, v5, v18, v20, v22, v16 | ||
| 1091 | calc_epelh v5, v19, v21, v23, v17 | ||
| 1092 | epel_uni_w_hv_end2 | ||
| 1093 | sqxtun v4.8b, v4.8h | ||
| 1094 | sqxtun2 v4.16b, v5.8h | ||
| 1095 | st1 {v4.8b}, [x0], #8 | ||
| 1096 | st1 {v4.s}[2], [x0], x1 | ||
| 1097 | b.eq 2f | ||
| 1098 | |||
| 1099 | ld1 {v18.8h, v19.8h}, [sp], x10 | ||
| 1100 | subs x4, x4, #1 | ||
| 1101 | calc_epelh v4, v20, v22, v16, v18 | ||
| 1102 | calc_epelh2 v4, v5, v20, v22, v16, v18 | ||
| 1103 | calc_epelh v5, v21, v23, v17, v19 | ||
| 1104 | epel_uni_w_hv_end2 | ||
| 1105 | sqxtun v4.8b, v4.8h | ||
| 1106 | sqxtun2 v4.16b, v5.8h | ||
| 1107 | st1 {v4.8b}, [x0], #8 | ||
| 1108 | st1 {v4.s}[2], [x0], x1 | ||
| 1109 | b.eq 2f | ||
| 1110 | |||
| 1111 | ld1 {v20.8h, v21.8h}, [sp], x10 | ||
| 1112 | subs x4, x4, #1 | ||
| 1113 | calc_epelh v4, v22, v16, v18, v20 | ||
| 1114 | calc_epelh2 v4, v5, v22, v16, v18, v20 | ||
| 1115 | calc_epelh v5, v23, v17, v19, v21 | ||
| 1116 | epel_uni_w_hv_end2 | ||
| 1117 | sqxtun v4.8b, v4.8h | ||
| 1118 | sqxtun2 v4.16b, v5.8h | ||
| 1119 | st1 {v4.8b}, [x0], #8 | ||
| 1120 | st1 {v4.s}[2], [x0], x1 | ||
| 1121 | b.ne 1b | ||
| 1122 | 2: | ||
| 1123 | ldp d8, d9, [sp, #16] | ||
| 1124 | ldp d10, d11, [sp, #32] | ||
| 1125 | ldp d12, d13, [sp, #48] | ||
| 1126 | ldp d14, d15, [sp], #64 | ||
| 1127 | ret | ||
| 1128 | endfunc | ||
| 1129 | |||
| 1130 | function ff_hevc_put_hevc_epel_uni_w_hv16_8_neon_i8mm, export=1 | ||
| 1131 | epel_uni_w_hv_start | ||
| 1132 | sxtw x4, w4 | ||
| 1133 | |||
| 1134 | add x10, x4, #3 | ||
| 1135 | lsl x10, x10, #7 | ||
| 1136 | sub sp, sp, x10 // tmp_array | ||
| 1137 | str x30, [sp, #-48]! | ||
| 1138 | stp x4, x6, [sp, #16] | ||
| 1139 | stp x0, x1, [sp, #32] | ||
| 1140 | add x0, sp, #48 | ||
| 1141 | sub x1, x2, x3 | ||
| 1142 | mov x2, x3 | ||
| 1143 | add x3, x4, #3 | ||
| 1144 | mov x4, x5 | ||
| 1145 | bl X(ff_hevc_put_hevc_epel_h16_8_neon_i8mm) | ||
| 1146 | ldp x4, x6, [sp, #16] | ||
| 1147 | ldp x0, x1, [sp, #32] | ||
| 1148 | ldr x30, [sp], #48 | ||
| 1149 | load_epel_filterh x6, x5 | ||
| 1150 | mov x10, #(MAX_PB_SIZE * 2) | ||
| 1151 | ld1 {v16.8h, v17.8h}, [sp], x10 | ||
| 1152 | ld1 {v18.8h, v19.8h}, [sp], x10 | ||
| 1153 | ld1 {v20.8h, v21.8h}, [sp], x10 | ||
| 1154 | 1: ld1 {v22.8h, v23.8h}, [sp], x10 | ||
| 1155 | subs x4, x4, #1 | ||
| 1156 | calc_epelh v4, v16, v18, v20, v22 | ||
| 1157 | calc_epelh2 v4, v5, v16, v18, v20, v22 | ||
| 1158 | calc_epelh v5, v17, v19, v21, v23 | ||
| 1159 | calc_epelh2 v5, v6, v17, v19, v21, v23 | ||
| 1160 | epel_uni_w_hv_end2 | ||
| 1161 | sqxtun v4.8b, v4.8h | ||
| 1162 | sqxtun2 v4.16b, v5.8h | ||
| 1163 | st1 {v4.16b}, [x0], x1 | ||
| 1164 | b.eq 2f | ||
| 1165 | |||
| 1166 | ld1 {v16.8h, v17.8h}, [sp], x10 | ||
| 1167 | subs x4, x4, #1 | ||
| 1168 | calc_epelh v4, v18, v20, v22, v16 | ||
| 1169 | calc_epelh2 v4, v5, v18, v20, v22, v16 | ||
| 1170 | calc_epelh v5, v19, v21, v23, v17 | ||
| 1171 | calc_epelh2 v5, v6, v19, v21, v23, v17 | ||
| 1172 | epel_uni_w_hv_end2 | ||
| 1173 | sqxtun v4.8b, v4.8h | ||
| 1174 | sqxtun2 v4.16b, v5.8h | ||
| 1175 | st1 {v4.16b}, [x0], x1 | ||
| 1176 | b.eq 2f | ||
| 1177 | |||
| 1178 | ld1 {v18.8h, v19.8h}, [sp], x10 | ||
| 1179 | subs x4, x4, #1 | ||
| 1180 | calc_epelh v4, v20, v22, v16, v18 | ||
| 1181 | calc_epelh2 v4, v5, v20, v22, v16, v18 | ||
| 1182 | calc_epelh v5, v21, v23, v17, v19 | ||
| 1183 | calc_epelh2 v5, v6, v21, v23, v17, v19 | ||
| 1184 | epel_uni_w_hv_end2 | ||
| 1185 | sqxtun v4.8b, v4.8h | ||
| 1186 | sqxtun2 v4.16b, v5.8h | ||
| 1187 | st1 {v4.16b}, [x0], x1 | ||
| 1188 | b.eq 2f | ||
| 1189 | |||
| 1190 | ld1 {v20.8h, v21.8h}, [sp], x10 | ||
| 1191 | subs x4, x4, #1 | ||
| 1192 | calc_epelh v4, v22, v16, v18, v20 | ||
| 1193 | calc_epelh2 v4, v5, v22, v16, v18, v20 | ||
| 1194 | calc_epelh v5, v23, v17, v19, v21 | ||
| 1195 | calc_epelh2 v5, v6, v23, v17, v19, v21 | ||
| 1196 | epel_uni_w_hv_end2 | ||
| 1197 | sqxtun v4.8b, v4.8h | ||
| 1198 | sqxtun2 v4.16b, v5.8h | ||
| 1199 | st1 {v4.16b}, [x0], x1 | ||
| 1200 | b.ne 1b | ||
| 1201 | 2: | ||
| 1202 | ldp d8, d9, [sp, #16] | ||
| 1203 | ldp d10, d11, [sp, #32] | ||
| 1204 | ldp d12, d13, [sp, #48] | ||
| 1205 | ldp d14, d15, [sp], #64 | ||
| 1206 | ret | ||
| 1207 | endfunc | ||
| 1208 | |||
| 1209 | function ff_hevc_put_hevc_epel_uni_w_hv24_8_neon_i8mm, export=1 | ||
| 1210 | epel_uni_w_hv_start | ||
| 1211 | sxtw x4, w4 | ||
| 1212 | |||
| 1213 | add x10, x4, #3 | ||
| 1214 | lsl x10, x10, #7 | ||
| 1215 | sub sp, sp, x10 // tmp_array | ||
| 1216 | str x30, [sp, #-48]! | ||
| 1217 | stp x4, x6, [sp, #16] | ||
| 1218 | stp x0, x1, [sp, #32] | ||
| 1219 | add x0, sp, #48 | ||
| 1220 | sub x1, x2, x3 | ||
| 1221 | mov x2, x3 | ||
| 1222 | add x3, x4, #3 | ||
| 1223 | mov x4, x5 | ||
| 1224 | bl X(ff_hevc_put_hevc_epel_h24_8_neon_i8mm) | ||
| 1225 | ldp x4, x6, [sp, #16] | ||
| 1226 | ldp x0, x1, [sp, #32] | ||
| 1227 | ldr x30, [sp], #48 | ||
| 1228 | load_epel_filterh x6, x5 | ||
| 1229 | mov x10, #(MAX_PB_SIZE * 2) | ||
| 1230 | ld1 {v16.8h, v17.8h, v18.8h}, [sp], x10 | ||
| 1231 | ld1 {v19.8h, v20.8h, v21.8h}, [sp], x10 | ||
| 1232 | ld1 {v22.8h, v23.8h, v24.8h}, [sp], x10 | ||
| 1233 | 1: ld1 {v25.8h, v26.8h, v27.8h}, [sp], x10 | ||
| 1234 | subs x4, x4, #1 | ||
| 1235 | calc_epelh v4, v16, v19, v22, v25 | ||
| 1236 | calc_epelh2 v4, v5, v16, v19, v22, v25 | ||
| 1237 | calc_epelh v5, v17, v20, v23, v26 | ||
| 1238 | calc_epelh2 v5, v6, v17, v20, v23, v26 | ||
| 1239 | calc_epelh v6, v18, v21, v24, v27 | ||
| 1240 | calc_epelh2 v6, v7, v18, v21, v24, v27 | ||
| 1241 | |||
| 1242 | epel_uni_w_hv_end3 | ||
| 1243 | sqxtun v4.8b, v4.8h | ||
| 1244 | sqxtun v5.8b, v5.8h | ||
| 1245 | sqxtun v6.8b, v6.8h | ||
| 1246 | st1 {v4.8b, v5.8b, v6.8b}, [x0], x1 | ||
| 1247 | b.eq 2f | ||
| 1248 | |||
| 1249 | ld1 {v16.8h, v17.8h, v18.8h}, [sp], x10 | ||
| 1250 | subs x4, x4, #1 | ||
| 1251 | calc_epelh v4, v19, v22, v25, v16 | ||
| 1252 | calc_epelh2 v4, v5, v19, v22, v25, v16 | ||
| 1253 | calc_epelh v5, v20, v23, v26, v17 | ||
| 1254 | calc_epelh2 v5, v6, v20, v23, v26, v17 | ||
| 1255 | calc_epelh v6, v21, v24, v27, v18 | ||
| 1256 | calc_epelh2 v6, v7, v21, v24, v27, v18 | ||
| 1257 | epel_uni_w_hv_end3 | ||
| 1258 | |||
| 1259 | sqxtun v4.8b, v4.8h | ||
| 1260 | sqxtun v5.8b, v5.8h | ||
| 1261 | sqxtun v6.8b, v6.8h | ||
| 1262 | st1 {v4.8b, v5.8b, v6.8b}, [x0], x1 | ||
| 1263 | b.eq 2f | ||
| 1264 | |||
| 1265 | ld1 {v19.8h, v20.8h, v21.8h}, [sp], x10 | ||
| 1266 | subs x4, x4, #1 | ||
| 1267 | calc_epelh v4, v22, v25, v16, v19 | ||
| 1268 | calc_epelh2 v4, v5, v22, v25, v16, v19 | ||
| 1269 | calc_epelh v5, v23, v26, v17, v20 | ||
| 1270 | calc_epelh2 v5, v6, v23, v26, v17, v20 | ||
| 1271 | calc_epelh v6, v24, v27, v18, v21 | ||
| 1272 | calc_epelh2 v6, v7, v24, v27, v18, v21 | ||
| 1273 | epel_uni_w_hv_end3 | ||
| 1274 | |||
| 1275 | sqxtun v4.8b, v4.8h | ||
| 1276 | sqxtun v5.8b, v5.8h | ||
| 1277 | sqxtun v6.8b, v6.8h | ||
| 1278 | st1 {v4.8b, v5.8b, v6.8b}, [x0], x1 | ||
| 1279 | b.eq 2f | ||
| 1280 | |||
| 1281 | ld1 {v22.8h, v23.8h, v24.8h}, [sp], x10 | ||
| 1282 | subs x4, x4, #1 | ||
| 1283 | calc_epelh v4, v25, v16, v19, v22 | ||
| 1284 | calc_epelh2 v4, v5, v25, v16, v19, v22 | ||
| 1285 | calc_epelh v5, v26, v17, v20, v23 | ||
| 1286 | calc_epelh2 v5, v6, v26, v17, v20, v23 | ||
| 1287 | calc_epelh v6, v27, v18, v21, v24 | ||
| 1288 | calc_epelh2 v6, v7, v27, v18, v21, v24 | ||
| 1289 | epel_uni_w_hv_end3 | ||
| 1290 | |||
| 1291 | sqxtun v4.8b, v4.8h | ||
| 1292 | sqxtun v5.8b, v5.8h | ||
| 1293 | sqxtun v6.8b, v6.8h | ||
| 1294 | st1 {v4.8b, v5.8b, v6.8b}, [x0], x1 | ||
| 1295 | b.ne 1b | ||
| 1296 | 2: | ||
| 1297 | ldp d8, d9, [sp, #16] | ||
| 1298 | ldp d10, d11, [sp, #32] | ||
| 1299 | ldp d12, d13, [sp, #48] | ||
| 1300 | ldp d14, d15, [sp], #64 | ||
| 1301 | ret | ||
| 1302 | endfunc | ||
| 1303 | |||
| 1304 | function ff_hevc_put_hevc_epel_uni_w_hv32_8_neon_i8mm, export=1 | ||
| 1305 | ldp x15, x16, [sp] | ||
| 1306 | mov x17, #16 | ||
| 1307 | stp x15, x16, [sp, #-96]! | ||
| 1308 | stp x0, x30, [sp, #16] | ||
| 1309 | stp x1, x2, [sp, #32] | ||
| 1310 | stp x3, x4, [sp, #48] | ||
| 1311 | stp x5, x6, [sp, #64] | ||
| 1312 | stp x17, x7, [sp, #80] | ||
| 1313 | |||
| 1314 | bl X(ff_hevc_put_hevc_epel_uni_w_hv16_8_neon_i8mm) | ||
| 1315 | ldp x0, x30, [sp, #16] | ||
| 1316 | ldp x1, x2, [sp, #32] | ||
| 1317 | ldp x3, x4, [sp, #48] | ||
| 1318 | ldp x5, x6, [sp, #64] | ||
| 1319 | ldp x17, x7, [sp, #80] | ||
| 1320 | ldp x15, x16, [sp], #96 | ||
| 1321 | add x0, x0, #16 | ||
| 1322 | add x2, x2, #16 | ||
| 1323 | mov x17, #16 | ||
| 1324 | stp x15, x16, [sp, #-32]! | ||
| 1325 | stp x17, x30, [sp, #16] | ||
| 1326 | bl X(ff_hevc_put_hevc_epel_uni_w_hv16_8_neon_i8mm) | ||
| 1327 | ldp x17, x30, [sp, #16] | ||
| 1328 | ldp x15, x16, [sp], #32 | ||
| 1329 | ret | ||
| 1330 | endfunc | ||
| 1331 | |||
| 1332 | function ff_hevc_put_hevc_epel_uni_w_hv48_8_neon_i8mm, export=1 | ||
| 1333 | ldp x15, x16, [sp] | ||
| 1334 | mov x17, #24 | ||
| 1335 | stp x15, x16, [sp, #-96]! | ||
| 1336 | stp x0, x30, [sp, #16] | ||
| 1337 | stp x1, x2, [sp, #32] | ||
| 1338 | stp x3, x4, [sp, #48] | ||
| 1339 | stp x5, x6, [sp, #64] | ||
| 1340 | stp x17, x7, [sp, #80] | ||
| 1341 | bl X(ff_hevc_put_hevc_epel_uni_w_hv24_8_neon_i8mm) | ||
| 1342 | ldp x0, x30, [sp, #16] | ||
| 1343 | ldp x1, x2, [sp, #32] | ||
| 1344 | ldp x3, x4, [sp, #48] | ||
| 1345 | ldp x5, x6, [sp, #64] | ||
| 1346 | ldp x17, x7, [sp, #80] | ||
| 1347 | ldp x15, x16, [sp], #96 | ||
| 1348 | add x0, x0, #24 | ||
| 1349 | add x2, x2, #24 | ||
| 1350 | mov x17, #24 | ||
| 1351 | stp x15, x16, [sp, #-32]! | ||
| 1352 | stp x17, x30, [sp, #16] | ||
| 1353 | bl X(ff_hevc_put_hevc_epel_uni_w_hv24_8_neon_i8mm) | ||
| 1354 | ldp x17, x30, [sp, #16] | ||
| 1355 | ldp x15, x16, [sp], #32 | ||
| 1356 | ret | ||
| 1357 | endfunc | ||
| 1358 | |||
| 1359 | function ff_hevc_put_hevc_epel_uni_w_hv64_8_neon_i8mm, export=1 | ||
| 1360 | ldp x15, x16, [sp] | ||
| 1361 | mov x17, #32 | ||
| 1362 | stp x15, x16, [sp, #-96]! | ||
| 1363 | stp x0, x30, [sp, #16] | ||
| 1364 | stp x1, x2, [sp, #32] | ||
| 1365 | stp x3, x4, [sp, #48] | ||
| 1366 | stp x5, x6, [sp, #64] | ||
| 1367 | stp x17, x7, [sp, #80] | ||
| 1368 | |||
| 1369 | bl X(ff_hevc_put_hevc_epel_uni_w_hv32_8_neon_i8mm) | ||
| 1370 | ldp x0, x30, [sp, #16] | ||
| 1371 | ldp x1, x2, [sp, #32] | ||
| 1372 | ldp x3, x4, [sp, #48] | ||
| 1373 | ldp x5, x6, [sp, #64] | ||
| 1374 | ldp x17, x7, [sp, #80] | ||
| 1375 | ldp x15, x16, [sp], #96 | ||
| 1376 | add x0, x0, #32 | ||
| 1377 | add x2, x2, #32 | ||
| 1378 | mov x17, #32 | ||
| 1379 | stp x15, x16, [sp, #-32]! | ||
| 1380 | stp x17, x30, [sp, #16] | ||
| 1381 | bl X(ff_hevc_put_hevc_epel_uni_w_hv32_8_neon_i8mm) | ||
| 1382 | ldp x17, x30, [sp, #16] | ||
| 1383 | ldp x15, x16, [sp], #32 | ||
| 1384 | ret | ||
| 1385 | endfunc | ||
| 1386 | |||
| 1387 | |||
| 720 | #endif | 1388 | #endif |
| 721 | 1389 | ||
| 722 | 1390 | ||
| @@ -189,6 +189,11 @@ NEON8_FNPROTO(qpel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride, · 188 unmodified lines | |||
| 189 | int height, int denom, int wx, int ox, | 189 | int height, int denom, int wx, int ox, |
| 190 | intptr_t mx, intptr_t my, int width), _i8mm); | 190 | intptr_t mx, intptr_t my, int width), _i8mm); |
| 191 | 191 | ||
| 192 | NEON8_FNPROTO(epel_uni_w_hv, (uint8_t *_dst, ptrdiff_t _dststride, | ||
| 193 | const uint8_t *_src, ptrdiff_t _srcstride, | ||
| 194 | int height, int denom, int wx, int ox, | ||
| 195 | intptr_t mx, intptr_t my, int width), _i8mm); | ||
| 196 | |||
| 192 | NEON8_FNPROTO_PARTIAL_5(qpel_uni_w_hv, (uint8_t *_dst, ptrdiff_t _dststride, | 197 | NEON8_FNPROTO_PARTIAL_5(qpel_uni_w_hv, (uint8_t *_dst, ptrdiff_t _dststride, |
| 193 | const uint8_t *_src, ptrdiff_t _srcstride, | 198 | const uint8_t *_src, ptrdiff_t _srcstride, |
| 194 | int height, int denom, int wx, int ox, | 199 | int height, int denom, int wx, int ox, |
| @@ -291,6 +296,7 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) · 96 unmodified lines | |||
| 291 | NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 0, 1, epel_uni_w_h ,_i8mm); | 296 | NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 0, 1, epel_uni_w_h ,_i8mm); |
| 292 | NEON8_FNASSIGN(c->put_hevc_qpel, 0, 1, qpel_h, _i8mm); | 297 | NEON8_FNASSIGN(c->put_hevc_qpel, 0, 1, qpel_h, _i8mm); |
| 293 | NEON8_FNASSIGN(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h, _i8mm); | 298 | NEON8_FNASSIGN(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h, _i8mm); |
| 299 | NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 1, 1, epel_uni_w_hv, _i8mm); | ||
| 294 | NEON8_FNASSIGN_PARTIAL_5(c->put_hevc_qpel_uni_w, 1, 1, qpel_uni_w_hv, _i8mm); | 300 | NEON8_FNASSIGN_PARTIAL_5(c->put_hevc_qpel_uni_w, 1, 1, qpel_uni_w_hv, _i8mm); |
| 295 | } | 301 | } |
| 296 | 302 | ||