put_hevc_epel_uni_w_hv4_8_c: 254.6 put_hevc_epel_uni_w_hv4_8_i8mm: 102.9 put_hevc_epel_uni_w_hv6_8_c: 411.6 put_hevc_epel_uni_w_hv6_8_i8mm: 221.6 put_hevc_epel_uni_w_hv8_8_c: 669.4 put_hevc_epel_uni_w_hv8_8_i8mm: 214.9 put_hevc_epel_uni_w_hv12_8_c: 1412.6 put_hevc_epel_uni_w_hv12_8_i8mm: 481.4 put_hevc_epel_uni_w_hv16_8_c: 2425.4 put_hevc_epel_uni_w_hv16_8_i8mm: 647.4 put_hevc_epel_uni_w_hv24_8_c: 5384.1 put_hevc_epel_uni_w_hv24_8_i8mm: 1450.6 put_hevc_epel_uni_w_hv32_8_c: 9470.9 put_hevc_epel_uni_w_hv32_8_i8mm: 2497.1 put_hevc_epel_uni_w_hv48_8_c: 20930.1 put_hevc_epel_uni_w_hv48_8_i8mm: 5635.9 put_hevc_epel_uni_w_hv64_8_c: 36682.9 put_hevc_epel_uni_w_hv64_8_i8mm: 9712.6
Unified · Split
| @@ -717,6 +717,674 @@ function ff_hevc_put_hevc_epel_uni_w_h64_8_neon_i8mm, export=1 · 716 unmodified lines | ||
| 717 | 717 | ret |
| 718 | 718 | endfunc |
| 719 | 719 | |
| 720 | +.macro epel_uni_w_hv_start | |
| 721 | + mov x15, x5 //denom | |
| 722 | + mov x16, x6 //wx | |
| 723 | + mov x17, x7 //ox | |
| 724 | + add w15, w15, #6 //shift = denom+6 | |
| 725 | + | |
| 726 | + | |
| 727 | + ldp x5, x6, [sp] | |
| 728 | + ldr x7, [sp, #16] | |
| 729 | + | |
| 730 | + stp d14, d15, [sp, #-64]! | |
| 731 | + stp d8, d9, [sp, #16] | |
| 732 | + stp d10, d11, [sp, #32] | |
| 733 | + stp d12, d13, [sp, #48] | |
| 734 | + | |
| 735 | + dup v13.8h, w16 //wx | |
| 736 | + dup v14.4s, w17 //ox | |
| 737 | + | |
| 738 | + mov w17, #1 | |
| 739 | + lsl w17, w17, w15 | |
| 740 | + lsr w17, w17, #1 | |
| 741 | + dup v15.4s, w17 | |
| 742 | + | |
| 743 | + neg w15, w15 // -shift | |
| 744 | + dup v12.4s, w15 //shift | |
| 745 | +.endm | |
| 746 | + | |
| 747 | +.macro epel_uni_w_hv_end | |
| 748 | + smull v28.4s, v4.4h, v13.4h | |
| 749 | + smull2 v29.4s, v4.8h, v13.8h | |
| 750 | + add v28.4s, v28.4s, v15.4s | |
| 751 | + add v29.4s, v29.4s, v15.4s | |
| 752 | + sshl v28.4s, v28.4s, v12.4s | |
| 753 | + sshl v29.4s, v29.4s, v12.4s | |
| 754 | + add v28.4s, v28.4s, v14.4s | |
| 755 | + add v29.4s, v29.4s, v14.4s | |
| 756 | + sqxtn v4.4h, v28.4s | |
| 757 | + sqxtn2 v4.8h, v29.4s | |
| 758 | +.endm | |
| 759 | + | |
| 760 | +.macro epel_uni_w_hv_end2 | |
| 761 | + smull v28.4s, v4.4h, v13.4h | |
| 762 | + smull2 v29.4s, v4.8h, v13.8h | |
| 763 | + smull v30.4s, v5.4h, v13.4h | |
| 764 | + smull2 v31.4s, v5.8h, v13.8h | |
| 765 | + add v28.4s, v28.4s, v15.4s | |
| 766 | + add v29.4s, v29.4s, v15.4s | |
| 767 | + add v30.4s, v30.4s, v15.4s | |
| 768 | + add v31.4s, v31.4s, v15.4s | |
| 769 | + | |
| 770 | + sshl v28.4s, v28.4s, v12.4s | |
| 771 | + sshl v29.4s, v29.4s, v12.4s | |
| 772 | + sshl v30.4s, v30.4s, v12.4s | |
| 773 | + sshl v31.4s, v31.4s, v12.4s | |
| 774 | + | |
| 775 | + add v28.4s, v28.4s, v14.4s | |
| 776 | + add v29.4s, v29.4s, v14.4s | |
| 777 | + add v30.4s, v30.4s, v14.4s | |
| 778 | + add v31.4s, v31.4s, v14.4s | |
| 779 | + | |
| 780 | + sqxtn v4.4h, v28.4s | |
| 781 | + sqxtn2 v4.8h, v29.4s | |
| 782 | + sqxtn v5.4h, v30.4s | |
| 783 | + sqxtn2 v5.8h, v31.4s | |
| 784 | +.endm | |
| 785 | + | |
| 786 | +.macro epel_uni_w_hv_end3 | |
| 787 | + smull v1.4s, v4.4h, v13.4h | |
| 788 | + smull2 v2.4s, v4.8h, v13.8h | |
| 789 | + smull v28.4s, v5.4h, v13.4h | |
| 790 | + smull2 v29.4s, v5.8h, v13.8h | |
| 791 | + smull v30.4s, v6.4h, v13.4h | |
| 792 | + smull2 v31.4s, v6.8h, v13.8h | |
| 793 | + add v1.4s, v1.4s, v15.4s | |
| 794 | + add v2.4s, v2.4s, v15.4s | |
| 795 | + add v28.4s, v28.4s, v15.4s | |
| 796 | + add v29.4s, v29.4s, v15.4s | |
| 797 | + add v30.4s, v30.4s, v15.4s | |
| 798 | + add v31.4s, v31.4s, v15.4s | |
| 799 | + | |
| 800 | + sshl v1.4s, v1.4s, v12.4s | |
| 801 | + sshl v2.4s, v2.4s, v12.4s | |
| 802 | + sshl v28.4s, v28.4s, v12.4s | |
| 803 | + sshl v29.4s, v29.4s, v12.4s | |
| 804 | + sshl v30.4s, v30.4s, v12.4s | |
| 805 | + sshl v31.4s, v31.4s, v12.4s | |
| 806 | + add v1.4s, v1.4s, v14.4s | |
| 807 | + add v2.4s, v2.4s, v14.4s | |
| 808 | + add v28.4s, v28.4s, v14.4s | |
| 809 | + add v29.4s, v29.4s, v14.4s | |
| 810 | + add v30.4s, v30.4s, v14.4s | |
| 811 | + add v31.4s, v31.4s, v14.4s | |
| 812 | + | |
| 813 | + sqxtn v4.4h, v1.4s | |
| 814 | + sqxtn2 v4.8h, v2.4s | |
| 815 | + sqxtn v5.4h, v28.4s | |
| 816 | + sqxtn2 v5.8h, v29.4s | |
| 817 | + sqxtn v6.4h, v30.4s | |
| 818 | + sqxtn2 v6.8h, v31.4s | |
| 819 | +.endm | |
| 820 | + | |
| 821 | +.macro calc_epelh dst, src0, src1, src2, src3 | |
| 822 | + smull \dst\().4s, \src0\().4h, v0.h[0] | |
| 823 | + smlal \dst\().4s, \src1\().4h, v0.h[1] | |
| 824 | + smlal \dst\().4s, \src2\().4h, v0.h[2] | |
| 825 | + smlal \dst\().4s, \src3\().4h, v0.h[3] | |
| 826 | + sqshrn \dst\().4h, \dst\().4s, #6 | |
| 827 | +.endm | |
| 828 | + | |
| 829 | +.macro calc_epelh2 dst, tmp, src0, src1, src2, src3 | |
| 830 | + smull2 \tmp\().4s, \src0\().8h, v0.h[0] | |
| 831 | + smlal2 \tmp\().4s, \src1\().8h, v0.h[1] | |
| 832 | + smlal2 \tmp\().4s, \src2\().8h, v0.h[2] | |
| 833 | + smlal2 \tmp\().4s, \src3\().8h, v0.h[3] | |
| 834 | + sqshrn2 \dst\().8h, \tmp\().4s, #6 | |
| 835 | +.endm | |
| 836 | + | |
| 837 | +.macro load_epel_filterh freg, xreg | |
| 838 | + movrel \xreg, epel_filters | |
| 839 | + add \xreg, \xreg, \freg, lsl #2 | |
| 840 | + ld1 {v0.8b}, [\xreg] | |
| 841 | + sxtl v0.8h, v0.8b | |
| 842 | +.endm | |
| 843 | + | |
| 844 | +function ff_hevc_put_hevc_epel_uni_w_hv4_8_neon_i8mm, export=1 | |
| 845 | + epel_uni_w_hv_start | |
| 846 | + sxtw x4, w4 | |
| 847 | + | |
| 848 | + add x10, x4, #3 | |
| 849 | + lsl x10, x10, #7 | |
| 850 | + sub sp, sp, x10 // tmp_array | |
| 851 | + str x30, [sp, #-48]! | |
| 852 | + stp x4, x6, [sp, #16] | |
| 853 | + stp x0, x1, [sp, #32] | |
| 854 | + add x0, sp, #48 | |
| 855 | + sub x1, x2, x3 | |
| 856 | + mov x2, x3 | |
| 857 | + add x3, x4, #3 | |
| 858 | + mov x4, x5 | |
| 859 | + bl X(ff_hevc_put_hevc_epel_h4_8_neon_i8mm) | |
| 860 | + ldp x4, x6, [sp, #16] | |
| 861 | + ldp x0, x1, [sp, #32] | |
| 862 | + ldr x30, [sp], #48 | |
| 863 | + load_epel_filterh x6, x5 | |
| 864 | + mov x10, #(MAX_PB_SIZE * 2) | |
| 865 | + ld1 {v16.4h}, [sp], x10 | |
| 866 | + ld1 {v17.4h}, [sp], x10 | |
| 867 | + ld1 {v18.4h}, [sp], x10 | |
| 868 | +1: ld1 {v19.4h}, [sp], x10 | |
| 869 | + subs x4, x4, #1 | |
| 870 | + calc_epelh v4, v16, v17, v18, v19 | |
| 871 | + epel_uni_w_hv_end | |
| 872 | + sqxtun v4.8b, v4.8h | |
| 873 | + str s4, [x0] | |
| 874 | + add x0, x0, x1 | |
| 875 | + b.eq 2f | |
| 876 | + | |
| 877 | + ld1 {v16.4h}, [sp], x10 | |
| 878 | + subs x4, x4, #1 | |
| 879 | + calc_epelh v4, v17, v18, v19, v16 | |
| 880 | + epel_uni_w_hv_end | |
| 881 | + sqxtun v4.8b, v4.8h | |
| 882 | + str s4, [x0] | |
| 883 | + add x0, x0, x1 | |
| 884 | + b.eq 2f | |
| 885 | + | |
| 886 | + ld1 {v17.4h}, [sp], x10 | |
| 887 | + subs x4, x4, #1 | |
| 888 | + calc_epelh v4, v18, v19, v16, v17 | |
| 889 | + epel_uni_w_hv_end | |
| 890 | + sqxtun v4.8b, v4.8h | |
| 891 | + str s4, [x0] | |
| 892 | + add x0, x0, x1 | |
| 893 | + b.eq 2f | |
| 894 | + | |
| 895 | + ld1 {v18.4h}, [sp], x10 | |
| 896 | + subs x4, x4, #1 | |
| 897 | + calc_epelh v4, v19, v16, v17, v18 | |
| 898 | + epel_uni_w_hv_end | |
| 899 | + sqxtun v4.8b, v4.8h | |
| 900 | + str s4, [x0] | |
| 901 | + add x0, x0, x1 | |
| 902 | + b.ne 1b | |
| 903 | +2: | |
| 904 | + ldp d8, d9, [sp, #16] | |
| 905 | + ldp d10, d11, [sp, #32] | |
| 906 | + ldp d12, d13, [sp, #48] | |
| 907 | + ldp d14, d15, [sp], #64 | |
| 908 | + ret | |
| 909 | +endfunc | |
| 910 | + | |
| 911 | +function ff_hevc_put_hevc_epel_uni_w_hv6_8_neon_i8mm, export=1 | |
| 912 | + epel_uni_w_hv_start | |
| 913 | + sxtw x4, w4 | |
| 914 | + | |
| 915 | + add x10, x4, #3 | |
| 916 | + lsl x10, x10, #7 | |
| 917 | + sub sp, sp, x10 // tmp_array | |
| 918 | + str x30, [sp, #-48]! | |
| 919 | + stp x4, x6, [sp, #16] | |
| 920 | + stp x0, x1, [sp, #32] | |
| 921 | + add x0, sp, #48 | |
| 922 | + sub x1, x2, x3 | |
| 923 | + mov x2, x3 | |
| 924 | + add x3, x4, #3 | |
| 925 | + mov x4, x5 | |
| 926 | + bl X(ff_hevc_put_hevc_epel_h6_8_neon_i8mm) | |
| 927 | + ldp x4, x6, [sp, #16] | |
| 928 | + ldp x0, x1, [sp, #32] | |
| 929 | + ldr x30, [sp], #48 | |
| 930 | + load_epel_filterh x6, x5 | |
| 931 | + sub x1, x1, #4 | |
| 932 | + mov x10, #(MAX_PB_SIZE * 2) | |
| 933 | + ld1 {v16.8h}, [sp], x10 | |
| 934 | + ld1 {v17.8h}, [sp], x10 | |
| 935 | + ld1 {v18.8h}, [sp], x10 | |
| 936 | +1: ld1 {v19.8h}, [sp], x10 | |
| 937 | + subs x4, x4, #1 | |
| 938 | + calc_epelh v4, v16, v17, v18, v19 | |
| 939 | + calc_epelh2 v4, v5, v16, v17, v18, v19 | |
| 940 | + epel_uni_w_hv_end | |
| 941 | + sqxtun v4.8b, v4.8h | |
| 942 | + st1 {v4.s}[0], [x0], #4 | |
| 943 | + st1 {v4.h}[2], [x0], x1 | |
| 944 | + b.eq 2f | |
| 945 | + | |
| 946 | + ld1 {v16.8h}, [sp], x10 | |
| 947 | + subs x4, x4, #1 | |
| 948 | + calc_epelh v4, v17, v18, v19, v16 | |
| 949 | + calc_epelh2 v4, v5, v17, v18, v19, v16 | |
| 950 | + epel_uni_w_hv_end | |
| 951 | + sqxtun v4.8b, v4.8h | |
| 952 | + st1 {v4.s}[0], [x0], #4 | |
| 953 | + st1 {v4.h}[2], [x0], x1 | |
| 954 | + b.eq 2f | |
| 955 | + | |
| 956 | + ld1 {v17.8h}, [sp], x10 | |
| 957 | + subs x4, x4, #1 | |
| 958 | + calc_epelh v4, v18, v19, v16, v17 | |
| 959 | + calc_epelh2 v4, v5, v18, v19, v16, v17 | |
| 960 | + epel_uni_w_hv_end | |
| 961 | + sqxtun v4.8b, v4.8h | |
| 962 | + st1 {v4.s}[0], [x0], #4 | |
| 963 | + st1 {v4.h}[2], [x0], x1 | |
| 964 | + b.eq 2f | |
| 965 | + | |
| 966 | + ld1 {v18.8h}, [sp], x10 | |
| 967 | + subs x4, x4, #1 | |
| 968 | + calc_epelh v4, v19, v16, v17, v18 | |
| 969 | + calc_epelh2 v4, v5, v19, v16, v17, v18 | |
| 970 | + epel_uni_w_hv_end | |
| 971 | + sqxtun v4.8b, v4.8h | |
| 972 | + st1 {v4.s}[0], [x0], #4 | |
| 973 | + st1 {v4.h}[2], [x0], x1 | |
| 974 | + b.ne 1b | |
| 975 | +2: | |
| 976 | + ldp d8, d9, [sp, #16] | |
| 977 | + ldp d10, d11, [sp, #32] | |
| 978 | + ldp d12, d13, [sp, #48] | |
| 979 | + ldp d14, d15, [sp], #64 | |
| 980 | + ret | |
| 981 | +endfunc | |
| 982 | + | |
| 983 | +function ff_hevc_put_hevc_epel_uni_w_hv8_8_neon_i8mm, export=1 | |
| 984 | + epel_uni_w_hv_start | |
| 985 | + sxtw x4, w4 | |
| 986 | + | |
| 987 | + add x10, x4, #3 | |
| 988 | + lsl x10, x10, #7 | |
| 989 | + sub sp, sp, x10 // tmp_array | |
| 990 | + str x30, [sp, #-48]! | |
| 991 | + stp x4, x6, [sp, #16] | |
| 992 | + stp x0, x1, [sp, #32] | |
| 993 | + add x0, sp, #48 | |
| 994 | + sub x1, x2, x3 | |
| 995 | + mov x2, x3 | |
| 996 | + add x3, x4, #3 | |
| 997 | + mov x4, x5 | |
| 998 | + bl X(ff_hevc_put_hevc_epel_h8_8_neon_i8mm) | |
| 999 | + ldp x4, x6, [sp, #16] | |
| 1000 | + ldp x0, x1, [sp, #32] | |
| 1001 | + ldr x30, [sp], #48 | |
| 1002 | + load_epel_filterh x6, x5 | |
| 1003 | + mov x10, #(MAX_PB_SIZE * 2) | |
| 1004 | + ld1 {v16.8h}, [sp], x10 | |
| 1005 | + ld1 {v17.8h}, [sp], x10 | |
| 1006 | + ld1 {v18.8h}, [sp], x10 | |
| 1007 | +1: ld1 {v19.8h}, [sp], x10 | |
| 1008 | + subs x4, x4, #1 | |
| 1009 | + calc_epelh v4, v16, v17, v18, v19 | |
| 1010 | + calc_epelh2 v4, v5, v16, v17, v18, v19 | |
| 1011 | + epel_uni_w_hv_end | |
| 1012 | + sqxtun v4.8b, v4.8h | |
| 1013 | + st1 {v4.8b}, [x0], x1 | |
| 1014 | + b.eq 2f | |
| 1015 | + | |
| 1016 | + ld1 {v16.8h}, [sp], x10 | |
| 1017 | + subs x4, x4, #1 | |
| 1018 | + calc_epelh v4, v17, v18, v19, v16 | |
| 1019 | + calc_epelh2 v4, v5, v17, v18, v19, v16 | |
| 1020 | + epel_uni_w_hv_end | |
| 1021 | + sqxtun v4.8b, v4.8h | |
| 1022 | + st1 {v4.8b}, [x0], x1 | |
| 1023 | + b.eq 2f | |
| 1024 | + | |
| 1025 | + ld1 {v17.8h}, [sp], x10 | |
| 1026 | + subs x4, x4, #1 | |
| 1027 | + calc_epelh v4, v18, v19, v16, v17 | |
| 1028 | + calc_epelh2 v4, v5, v18, v19, v16, v17 | |
| 1029 | + epel_uni_w_hv_end | |
| 1030 | + sqxtun v4.8b, v4.8h | |
| 1031 | + st1 {v4.8b}, [x0], x1 | |
| 1032 | + b.eq 2f | |
| 1033 | + | |
| 1034 | + ld1 {v18.8h}, [sp], x10 | |
| 1035 | + subs x4, x4, #1 | |
| 1036 | + calc_epelh v4, v19, v16, v17, v18 | |
| 1037 | + calc_epelh2 v4, v5, v19, v16, v17, v18 | |
| 1038 | + epel_uni_w_hv_end | |
| 1039 | + sqxtun v4.8b, v4.8h | |
| 1040 | + st1 {v4.8b}, [x0], x1 | |
| 1041 | + b.ne 1b | |
| 1042 | +2: | |
| 1043 | + ldp d8, d9, [sp, #16] | |
| 1044 | + ldp d10, d11, [sp, #32] | |
| 1045 | + ldp d12, d13, [sp, #48] | |
| 1046 | + ldp d14, d15, [sp], #64 | |
| 1047 | + ret | |
| 1048 | +endfunc | |
| 1049 | + | |
| 1050 | +function ff_hevc_put_hevc_epel_uni_w_hv12_8_neon_i8mm, export=1 | |
| 1051 | + epel_uni_w_hv_start | |
| 1052 | + sxtw x4, w4 | |
| 1053 | + | |
| 1054 | + add x10, x4, #3 | |
| 1055 | + lsl x10, x10, #7 | |
| 1056 | + sub sp, sp, x10 // tmp_array | |
| 1057 | + str x30, [sp, #-48]! | |
| 1058 | + stp x4, x6, [sp, #16] | |
| 1059 | + stp x0, x1, [sp, #32] | |
| 1060 | + add x0, sp, #48 | |
| 1061 | + sub x1, x2, x3 | |
| 1062 | + mov x2, x3 | |
| 1063 | + add x3, x4, #3 | |
| 1064 | + mov x4, x5 | |
| 1065 | + bl X(ff_hevc_put_hevc_epel_h12_8_neon_i8mm) | |
| 1066 | + ldp x4, x6, [sp, #16] | |
| 1067 | + ldp x0, x1, [sp, #32] | |
| 1068 | + ldr x30, [sp], #48 | |
| 1069 | + load_epel_filterh x6, x5 | |
| 1070 | + sub x1, x1, #8 | |
| 1071 | + mov x10, #(MAX_PB_SIZE * 2) | |
| 1072 | + ld1 {v16.8h, v17.8h}, [sp], x10 | |
| 1073 | + ld1 {v18.8h, v19.8h}, [sp], x10 | |
| 1074 | + ld1 {v20.8h, v21.8h}, [sp], x10 | |
| 1075 | +1: ld1 {v22.8h, v23.8h}, [sp], x10 | |
| 1076 | + subs x4, x4, #1 | |
| 1077 | + calc_epelh v4, v16, v18, v20, v22 | |
| 1078 | + calc_epelh2 v4, v5, v16, v18, v20, v22 | |
| 1079 | + calc_epelh v5, v17, v19, v21, v23 | |
| 1080 | + epel_uni_w_hv_end2 | |
| 1081 | + sqxtun v4.8b, v4.8h | |
| 1082 | + sqxtun2 v4.16b, v5.8h | |
| 1083 | + st1 {v4.8b}, [x0], #8 | |
| 1084 | + st1 {v4.s}[2], [x0], x1 | |
| 1085 | + b.eq 2f | |
| 1086 | + | |
| 1087 | + ld1 {v16.8h, v17.8h}, [sp], x10 | |
| 1088 | + subs x4, x4, #1 | |
| 1089 | + calc_epelh v4, v18, v20, v22, v16 | |
| 1090 | + calc_epelh2 v4, v5, v18, v20, v22, v16 | |
| 1091 | + calc_epelh v5, v19, v21, v23, v17 | |
| 1092 | + epel_uni_w_hv_end2 | |
| 1093 | + sqxtun v4.8b, v4.8h | |
| 1094 | + sqxtun2 v4.16b, v5.8h | |
| 1095 | + st1 {v4.8b}, [x0], #8 | |
| 1096 | + st1 {v4.s}[2], [x0], x1 | |
| 1097 | + b.eq 2f | |
| 1098 | + | |
| 1099 | + ld1 {v18.8h, v19.8h}, [sp], x10 | |
| 1100 | + subs x4, x4, #1 | |
| 1101 | + calc_epelh v4, v20, v22, v16, v18 | |
| 1102 | + calc_epelh2 v4, v5, v20, v22, v16, v18 | |
| 1103 | + calc_epelh v5, v21, v23, v17, v19 | |
| 1104 | + epel_uni_w_hv_end2 | |
| 1105 | + sqxtun v4.8b, v4.8h | |
| 1106 | + sqxtun2 v4.16b, v5.8h | |
| 1107 | + st1 {v4.8b}, [x0], #8 | |
| 1108 | + st1 {v4.s}[2], [x0], x1 | |
| 1109 | + b.eq 2f | |
| 1110 | + | |
| 1111 | + ld1 {v20.8h, v21.8h}, [sp], x10 | |
| 1112 | + subs x4, x4, #1 | |
| 1113 | + calc_epelh v4, v22, v16, v18, v20 | |
| 1114 | + calc_epelh2 v4, v5, v22, v16, v18, v20 | |
| 1115 | + calc_epelh v5, v23, v17, v19, v21 | |
| 1116 | + epel_uni_w_hv_end2 | |
| 1117 | + sqxtun v4.8b, v4.8h | |
| 1118 | + sqxtun2 v4.16b, v5.8h | |
| 1119 | + st1 {v4.8b}, [x0], #8 | |
| 1120 | + st1 {v4.s}[2], [x0], x1 | |
| 1121 | + b.ne 1b | |
| 1122 | +2: | |
| 1123 | + ldp d8, d9, [sp, #16] | |
| 1124 | + ldp d10, d11, [sp, #32] | |
| 1125 | + ldp d12, d13, [sp, #48] | |
| 1126 | + ldp d14, d15, [sp], #64 | |
| 1127 | + ret | |
| 1128 | +endfunc | |
| 1129 | + | |
| 1130 | +function ff_hevc_put_hevc_epel_uni_w_hv16_8_neon_i8mm, export=1 | |
| 1131 | + epel_uni_w_hv_start | |
| 1132 | + sxtw x4, w4 | |
| 1133 | + | |
| 1134 | + add x10, x4, #3 | |
| 1135 | + lsl x10, x10, #7 | |
| 1136 | + sub sp, sp, x10 // tmp_array | |
| 1137 | + str x30, [sp, #-48]! | |
| 1138 | + stp x4, x6, [sp, #16] | |
| 1139 | + stp x0, x1, [sp, #32] | |
| 1140 | + add x0, sp, #48 | |
| 1141 | + sub x1, x2, x3 | |
| 1142 | + mov x2, x3 | |
| 1143 | + add x3, x4, #3 | |
| 1144 | + mov x4, x5 | |
| 1145 | + bl X(ff_hevc_put_hevc_epel_h16_8_neon_i8mm) | |
| 1146 | + ldp x4, x6, [sp, #16] | |
| 1147 | + ldp x0, x1, [sp, #32] | |
| 1148 | + ldr x30, [sp], #48 | |
| 1149 | + load_epel_filterh x6, x5 | |
| 1150 | + mov x10, #(MAX_PB_SIZE * 2) | |
| 1151 | + ld1 {v16.8h, v17.8h}, [sp], x10 | |
| 1152 | + ld1 {v18.8h, v19.8h}, [sp], x10 | |
| 1153 | + ld1 {v20.8h, v21.8h}, [sp], x10 | |
| 1154 | +1: ld1 {v22.8h, v23.8h}, [sp], x10 | |
| 1155 | + subs x4, x4, #1 | |
| 1156 | + calc_epelh v4, v16, v18, v20, v22 | |
| 1157 | + calc_epelh2 v4, v5, v16, v18, v20, v22 | |
| 1158 | + calc_epelh v5, v17, v19, v21, v23 | |
| 1159 | + calc_epelh2 v5, v6, v17, v19, v21, v23 | |
| 1160 | + epel_uni_w_hv_end2 | |
| 1161 | + sqxtun v4.8b, v4.8h | |
| 1162 | + sqxtun2 v4.16b, v5.8h | |
| 1163 | + st1 {v4.16b}, [x0], x1 | |
| 1164 | + b.eq 2f | |
| 1165 | + | |
| 1166 | + ld1 {v16.8h, v17.8h}, [sp], x10 | |
| 1167 | + subs x4, x4, #1 | |
| 1168 | + calc_epelh v4, v18, v20, v22, v16 | |
| 1169 | + calc_epelh2 v4, v5, v18, v20, v22, v16 | |
| 1170 | + calc_epelh v5, v19, v21, v23, v17 | |
| 1171 | + calc_epelh2 v5, v6, v19, v21, v23, v17 | |
| 1172 | + epel_uni_w_hv_end2 | |
| 1173 | + sqxtun v4.8b, v4.8h | |
| 1174 | + sqxtun2 v4.16b, v5.8h | |
| 1175 | + st1 {v4.16b}, [x0], x1 | |
| 1176 | + b.eq 2f | |
| 1177 | + | |
| 1178 | + ld1 {v18.8h, v19.8h}, [sp], x10 | |
| 1179 | + subs x4, x4, #1 | |
| 1180 | + calc_epelh v4, v20, v22, v16, v18 | |
| 1181 | + calc_epelh2 v4, v5, v20, v22, v16, v18 | |
| 1182 | + calc_epelh v5, v21, v23, v17, v19 | |
| 1183 | + calc_epelh2 v5, v6, v21, v23, v17, v19 | |
| 1184 | + epel_uni_w_hv_end2 | |
| 1185 | + sqxtun v4.8b, v4.8h | |
| 1186 | + sqxtun2 v4.16b, v5.8h | |
| 1187 | + st1 {v4.16b}, [x0], x1 | |
| 1188 | + b.eq 2f | |
| 1189 | + | |
| 1190 | + ld1 {v20.8h, v21.8h}, [sp], x10 | |
| 1191 | + subs x4, x4, #1 | |
| 1192 | + calc_epelh v4, v22, v16, v18, v20 | |
| 1193 | + calc_epelh2 v4, v5, v22, v16, v18, v20 | |
| 1194 | + calc_epelh v5, v23, v17, v19, v21 | |
| 1195 | + calc_epelh2 v5, v6, v23, v17, v19, v21 | |
| 1196 | + epel_uni_w_hv_end2 | |
| 1197 | + sqxtun v4.8b, v4.8h | |
| 1198 | + sqxtun2 v4.16b, v5.8h | |
| 1199 | + st1 {v4.16b}, [x0], x1 | |
| 1200 | + b.ne 1b | |
| 1201 | +2: | |
| 1202 | + ldp d8, d9, [sp, #16] | |
| 1203 | + ldp d10, d11, [sp, #32] | |
| 1204 | + ldp d12, d13, [sp, #48] | |
| 1205 | + ldp d14, d15, [sp], #64 | |
| 1206 | + ret | |
| 1207 | +endfunc | |
| 1208 | + | |
| 1209 | +function ff_hevc_put_hevc_epel_uni_w_hv24_8_neon_i8mm, export=1 | |
| 1210 | + epel_uni_w_hv_start | |
| 1211 | + sxtw x4, w4 | |
| 1212 | + | |
| 1213 | + add x10, x4, #3 | |
| 1214 | + lsl x10, x10, #7 | |
| 1215 | + sub sp, sp, x10 // tmp_array | |
| 1216 | + str x30, [sp, #-48]! | |
| 1217 | + stp x4, x6, [sp, #16] | |
| 1218 | + stp x0, x1, [sp, #32] | |
| 1219 | + add x0, sp, #48 | |
| 1220 | + sub x1, x2, x3 | |
| 1221 | + mov x2, x3 | |
| 1222 | + add x3, x4, #3 | |
| 1223 | + mov x4, x5 | |
| 1224 | + bl X(ff_hevc_put_hevc_epel_h24_8_neon_i8mm) | |
| 1225 | + ldp x4, x6, [sp, #16] | |
| 1226 | + ldp x0, x1, [sp, #32] | |
| 1227 | + ldr x30, [sp], #48 | |
| 1228 | + load_epel_filterh x6, x5 | |
| 1229 | + mov x10, #(MAX_PB_SIZE * 2) | |
| 1230 | + ld1 {v16.8h, v17.8h, v18.8h}, [sp], x10 | |
| 1231 | + ld1 {v19.8h, v20.8h, v21.8h}, [sp], x10 | |
| 1232 | + ld1 {v22.8h, v23.8h, v24.8h}, [sp], x10 | |
| 1233 | +1: ld1 {v25.8h, v26.8h, v27.8h}, [sp], x10 | |
| 1234 | + subs x4, x4, #1 | |
| 1235 | + calc_epelh v4, v16, v19, v22, v25 | |
| 1236 | + calc_epelh2 v4, v5, v16, v19, v22, v25 | |
| 1237 | + calc_epelh v5, v17, v20, v23, v26 | |
| 1238 | + calc_epelh2 v5, v6, v17, v20, v23, v26 | |
| 1239 | + calc_epelh v6, v18, v21, v24, v27 | |
| 1240 | + calc_epelh2 v6, v7, v18, v21, v24, v27 | |
| 1241 | + | |
| 1242 | + epel_uni_w_hv_end3 | |
| 1243 | + sqxtun v4.8b, v4.8h | |
| 1244 | + sqxtun v5.8b, v5.8h | |
| 1245 | + sqxtun v6.8b, v6.8h | |
| 1246 | + st1 {v4.8b, v5.8b, v6.8b}, [x0], x1 | |
| 1247 | + b.eq 2f | |
| 1248 | + | |
| 1249 | + ld1 {v16.8h, v17.8h, v18.8h}, [sp], x10 | |
| 1250 | + subs x4, x4, #1 | |
| 1251 | + calc_epelh v4, v19, v22, v25, v16 | |
| 1252 | + calc_epelh2 v4, v5, v19, v22, v25, v16 | |
| 1253 | + calc_epelh v5, v20, v23, v26, v17 | |
| 1254 | + calc_epelh2 v5, v6, v20, v23, v26, v17 | |
| 1255 | + calc_epelh v6, v21, v24, v27, v18 | |
| 1256 | + calc_epelh2 v6, v7, v21, v24, v27, v18 | |
| 1257 | + epel_uni_w_hv_end3 | |
| 1258 | + | |
| 1259 | + sqxtun v4.8b, v4.8h | |
| 1260 | + sqxtun v5.8b, v5.8h | |
| 1261 | + sqxtun v6.8b, v6.8h | |
| 1262 | + st1 {v4.8b, v5.8b, v6.8b}, [x0], x1 | |
| 1263 | + b.eq 2f | |
| 1264 | + | |
| 1265 | + ld1 {v19.8h, v20.8h, v21.8h}, [sp], x10 | |
| 1266 | + subs x4, x4, #1 | |
| 1267 | + calc_epelh v4, v22, v25, v16, v19 | |
| 1268 | + calc_epelh2 v4, v5, v22, v25, v16, v19 | |
| 1269 | + calc_epelh v5, v23, v26, v17, v20 | |
| 1270 | + calc_epelh2 v5, v6, v23, v26, v17, v20 | |
| 1271 | + calc_epelh v6, v24, v27, v18, v21 | |
| 1272 | + calc_epelh2 v6, v7, v24, v27, v18, v21 | |
| 1273 | + epel_uni_w_hv_end3 | |
| 1274 | + | |
| 1275 | + sqxtun v4.8b, v4.8h | |
| 1276 | + sqxtun v5.8b, v5.8h | |
| 1277 | + sqxtun v6.8b, v6.8h | |
| 1278 | + st1 {v4.8b, v5.8b, v6.8b}, [x0], x1 | |
| 1279 | + b.eq 2f | |
| 1280 | + | |
| 1281 | + ld1 {v22.8h, v23.8h, v24.8h}, [sp], x10 | |
| 1282 | + subs x4, x4, #1 | |
| 1283 | + calc_epelh v4, v25, v16, v19, v22 | |
| 1284 | + calc_epelh2 v4, v5, v25, v16, v19, v22 | |
| 1285 | + calc_epelh v5, v26, v17, v20, v23 | |
| 1286 | + calc_epelh2 v5, v6, v26, v17, v20, v23 | |
| 1287 | + calc_epelh v6, v27, v18, v21, v24 | |
| 1288 | + calc_epelh2 v6, v7, v27, v18, v21, v24 | |
| 1289 | + epel_uni_w_hv_end3 | |
| 1290 | + | |
| 1291 | + sqxtun v4.8b, v4.8h | |
| 1292 | + sqxtun v5.8b, v5.8h | |
| 1293 | + sqxtun v6.8b, v6.8h | |
| 1294 | + st1 {v4.8b, v5.8b, v6.8b}, [x0], x1 | |
| 1295 | + b.ne 1b | |
| 1296 | +2: | |
| 1297 | + ldp d8, d9, [sp, #16] | |
| 1298 | + ldp d10, d11, [sp, #32] | |
| 1299 | + ldp d12, d13, [sp, #48] | |
| 1300 | + ldp d14, d15, [sp], #64 | |
| 1301 | + ret | |
| 1302 | +endfunc | |
| 1303 | + | |
| 1304 | +function ff_hevc_put_hevc_epel_uni_w_hv32_8_neon_i8mm, export=1 | |
| 1305 | + ldp x15, x16, [sp] | |
| 1306 | + mov x17, #16 | |
| 1307 | + stp x15, x16, [sp, #-96]! | |
| 1308 | + stp x0, x30, [sp, #16] | |
| 1309 | + stp x1, x2, [sp, #32] | |
| 1310 | + stp x3, x4, [sp, #48] | |
| 1311 | + stp x5, x6, [sp, #64] | |
| 1312 | + stp x17, x7, [sp, #80] | |
| 1313 | + | |
| 1314 | + bl X(ff_hevc_put_hevc_epel_uni_w_hv16_8_neon_i8mm) | |
| 1315 | + ldp x0, x30, [sp, #16] | |
| 1316 | + ldp x1, x2, [sp, #32] | |
| 1317 | + ldp x3, x4, [sp, #48] | |
| 1318 | + ldp x5, x6, [sp, #64] | |
| 1319 | + ldp x17, x7, [sp, #80] | |
| 1320 | + ldp x15, x16, [sp], #96 | |
| 1321 | + add x0, x0, #16 | |
| 1322 | + add x2, x2, #16 | |
| 1323 | + mov x17, #16 | |
| 1324 | + stp x15, x16, [sp, #-32]! | |
| 1325 | + stp x17, x30, [sp, #16] | |
| 1326 | + bl X(ff_hevc_put_hevc_epel_uni_w_hv16_8_neon_i8mm) | |
| 1327 | + ldp x17, x30, [sp, #16] | |
| 1328 | + ldp x15, x16, [sp], #32 | |
| 1329 | + ret | |
| 1330 | +endfunc | |
| 1331 | + | |
| 1332 | +function ff_hevc_put_hevc_epel_uni_w_hv48_8_neon_i8mm, export=1 | |
| 1333 | + ldp x15, x16, [sp] | |
| 1334 | + mov x17, #24 | |
| 1335 | + stp x15, x16, [sp, #-96]! | |
| 1336 | + stp x0, x30, [sp, #16] | |
| 1337 | + stp x1, x2, [sp, #32] | |
| 1338 | + stp x3, x4, [sp, #48] | |
| 1339 | + stp x5, x6, [sp, #64] | |
| 1340 | + stp x17, x7, [sp, #80] | |
| 1341 | + bl X(ff_hevc_put_hevc_epel_uni_w_hv24_8_neon_i8mm) | |
| 1342 | + ldp x0, x30, [sp, #16] | |
| 1343 | + ldp x1, x2, [sp, #32] | |
| 1344 | + ldp x3, x4, [sp, #48] | |
| 1345 | + ldp x5, x6, [sp, #64] | |
| 1346 | + ldp x17, x7, [sp, #80] | |
| 1347 | + ldp x15, x16, [sp], #96 | |
| 1348 | + add x0, x0, #24 | |
| 1349 | + add x2, x2, #24 | |
| 1350 | + mov x17, #24 | |
| 1351 | + stp x15, x16, [sp, #-32]! | |
| 1352 | + stp x17, x30, [sp, #16] | |
| 1353 | + bl X(ff_hevc_put_hevc_epel_uni_w_hv24_8_neon_i8mm) | |
| 1354 | + ldp x17, x30, [sp, #16] | |
| 1355 | + ldp x15, x16, [sp], #32 | |
| 1356 | + ret | |
| 1357 | +endfunc | |
| 1358 | + | |
| 1359 | +function ff_hevc_put_hevc_epel_uni_w_hv64_8_neon_i8mm, export=1 | |
| 1360 | + ldp x15, x16, [sp] | |
| 1361 | + mov x17, #32 | |
| 1362 | + stp x15, x16, [sp, #-96]! | |
| 1363 | + stp x0, x30, [sp, #16] | |
| 1364 | + stp x1, x2, [sp, #32] | |
| 1365 | + stp x3, x4, [sp, #48] | |
| 1366 | + stp x5, x6, [sp, #64] | |
| 1367 | + stp x17, x7, [sp, #80] | |
| 1368 | + | |
| 1369 | + bl X(ff_hevc_put_hevc_epel_uni_w_hv32_8_neon_i8mm) | |
| 1370 | + ldp x0, x30, [sp, #16] | |
| 1371 | + ldp x1, x2, [sp, #32] | |
| 1372 | + ldp x3, x4, [sp, #48] | |
| 1373 | + ldp x5, x6, [sp, #64] | |
| 1374 | + ldp x17, x7, [sp, #80] | |
| 1375 | + ldp x15, x16, [sp], #96 | |
| 1376 | + add x0, x0, #32 | |
| 1377 | + add x2, x2, #32 | |
| 1378 | + mov x17, #32 | |
| 1379 | + stp x15, x16, [sp, #-32]! | |
| 1380 | + stp x17, x30, [sp, #16] | |
| 1381 | + bl X(ff_hevc_put_hevc_epel_uni_w_hv32_8_neon_i8mm) | |
| 1382 | + ldp x17, x30, [sp, #16] | |
| 1383 | + ldp x15, x16, [sp], #32 | |
| 1384 | + ret | |
| 1385 | +endfunc | |
| 1386 | + | |
| 1387 | + | |
| 720 | 1388 | #endif |
| 721 | 1389 | |
| 722 | 1390 | |
| @@ -189,6 +189,11 @@ NEON8_FNPROTO(qpel_uni_w_h, (uint8_t *_dst, ptrdiff_t _dststride, · 188 unmodified lines | ||
| 189 | 189 | int height, int denom, int wx, int ox, |
| 190 | 190 | intptr_t mx, intptr_t my, int width), _i8mm); |
| 191 | 191 | |
| 192 | +NEON8_FNPROTO(epel_uni_w_hv, (uint8_t *_dst, ptrdiff_t _dststride, | |
| 193 | + const uint8_t *_src, ptrdiff_t _srcstride, | |
| 194 | + int height, int denom, int wx, int ox, | |
| 195 | + intptr_t mx, intptr_t my, int width), _i8mm); | |
| 196 | + | |
| 192 | 197 | NEON8_FNPROTO_PARTIAL_5(qpel_uni_w_hv, (uint8_t *_dst, ptrdiff_t _dststride, |
| 193 | 198 | const uint8_t *_src, ptrdiff_t _srcstride, |
| 194 | 199 | int height, int denom, int wx, int ox, |
| @@ -291,6 +296,7 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) · 96 unmodified lines | ||
| 291 | 296 | NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 0, 1, epel_uni_w_h ,_i8mm); |
| 292 | 297 | NEON8_FNASSIGN(c->put_hevc_qpel, 0, 1, qpel_h, _i8mm); |
| 293 | 298 | NEON8_FNASSIGN(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h, _i8mm); |
| 299 | + NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 1, 1, epel_uni_w_hv, _i8mm); | |
| 294 | 300 | NEON8_FNASSIGN_PARTIAL_5(c->put_hevc_qpel_uni_w, 1, 1, qpel_uni_w_hv, _i8mm); |
| 295 | 301 | } |
| 296 | 302 | |