As the plain neon qpel_h functions process two rows at a time, we need to allocate storage for h+8 rows instead of h+7. By allocating storage for h+8 rows, incrementing the stack pointer won't end up at the right spot in the end. Store the intended final stack pointer value in a register x14 which we store on the stack. AWS Graviton 3: put_hevc_qpel_hv4_8_c: 386.0 put_hevc_qpel_hv4_8_neon: 125.7 put_hevc_qpel_hv4_8_i8mm: 83.2 put_hevc_qpel_hv6_8_c: 749.0 put_hevc_qpel_hv6_8_neon: 207.0 put_hevc_qpel_hv6_8_i8mm: 166.0 put_hevc_qpel_hv8_8_c: 1305.2 put_hevc_qpel_hv8_8_neon: 216.5 put_hevc_qpel_hv8_8_i8mm: 213.0 put_hevc_qpel_hv12_8_c: 2570.5 put_hevc_qpel_hv12_8_neon: 480.0 put_hevc_qpel_hv12_8_i8mm: 398.2 put_hevc_qpel_hv16_8_c: 4158.7 put_hevc_qpel_hv16_8_neon: 659.7 put_hevc_qpel_hv16_8_i8mm: 593.5 put_hevc_qpel_hv24_8_c: 8626.7 put_hevc_qpel_hv24_8_neon: 1653.5 put_hevc_qpel_hv24_8_i8mm: 1398.7 put_hevc_qpel_hv32_8_c: 14646.0 put_hevc_qpel_hv32_8_neon: 2566.2 put_hevc_qpel_hv32_8_i8mm: 2287.5 put_hevc_qpel_hv48_8_c: 31072.5 put_hevc_qpel_hv48_8_neon: 6228.5 put_hevc_qpel_hv48_8_i8mm: 5291.0 put_hevc_qpel_hv64_8_c: 53847.2 put_hevc_qpel_hv64_8_neon: 9856.7 put_hevc_qpel_hv64_8_i8mm: 8831.0
Showing the changes to libavcodec (2 of 2 files). Show all
Unified · Split
| @@ -265,6 +265,10 @@ NEON8_FNPROTO(qpel_v, (int16_t *dst, · 264 unmodified lines | ||
| 265 | 265 | const uint8_t *src, ptrdiff_t srcstride, |
| 266 | 266 | int height, intptr_t mx, intptr_t my, int width),); |
| 267 | 267 | |
| 268 | +NEON8_FNPROTO(qpel_hv, (int16_t *dst, | |
| 269 | + const uint8_t *src, ptrdiff_t srcstride, | |
| 270 | + int height, intptr_t mx, intptr_t my, int width),); | |
| 271 | + | |
| 268 | 272 | NEON8_FNPROTO(qpel_hv, (int16_t *dst, |
| 269 | 273 | const uint8_t *src, ptrdiff_t srcstride, |
| 270 | 274 | int height, intptr_t mx, intptr_t my, int width), _i8mm); |
| @@ -436,6 +440,8 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) · 165 unmodified lines | ||
| 436 | 440 | |
| 437 | 441 | NEON8_FNASSIGN_SHARED_32(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h,); |
| 438 | 442 | |
| 443 | + NEON8_FNASSIGN(c->put_hevc_qpel, 1, 1, qpel_hv,); | |
| 444 | + | |
| 439 | 445 | if (have_i8mm(cpu_flags)) { |
| 440 | 446 | NEON8_FNASSIGN(c->put_hevc_epel, 0, 1, epel_h, _i8mm); |
| 441 | 447 | NEON8_FNASSIGN(c->put_hevc_epel, 1, 1, epel_hv, _i8mm); |
| @@ -3804,7 +3804,8 @@ function hevc_put_hevc_qpel_hv4_8_end_neon · 3803 unmodified lines | ||
| 3804 | 3804 | .endm |
| 3805 | 3805 | 1: calc_all |
| 3806 | 3806 | .purgem calc |
| 3807 | −2: ret | |
| 3807 | +2: mov sp, x14 | |
| 3808 | + ret | |
| 3808 | 3809 | endfunc |
| 3809 | 3810 | |
| 3810 | 3811 | function hevc_put_hevc_qpel_hv6_8_end_neon |
| @@ -3831,7 +3832,8 @@ function hevc_put_hevc_qpel_hv6_8_end_neon · 20 unmodified lines | ||
| 3831 | 3832 | .endm |
| 3832 | 3833 | 1: calc_all |
| 3833 | 3834 | .purgem calc |
| 3834 | −2: ret | |
| 3835 | +2: mov sp, x14 | |
| 3836 | + ret | |
| 3835 | 3837 | endfunc |
| 3836 | 3838 | |
| 3837 | 3839 | function hevc_put_hevc_qpel_hv8_8_end_neon |
| @@ -3857,7 +3859,8 @@ function hevc_put_hevc_qpel_hv8_8_end_neon · 19 unmodified lines | ||
| 3857 | 3859 | .endm |
| 3858 | 3860 | 1: calc_all |
| 3859 | 3861 | .purgem calc |
| 3860 | −2: ret | |
| 3862 | +2: mov sp, x14 | |
| 3863 | + ret | |
| 3861 | 3864 | endfunc |
| 3862 | 3865 | |
| 3863 | 3866 | function hevc_put_hevc_qpel_hv12_8_end_neon |
| @@ -3882,7 +3885,8 @@ function hevc_put_hevc_qpel_hv12_8_end_neon · 18 unmodified lines | ||
| 3882 | 3885 | .endm |
| 3883 | 3886 | 1: calc_all2 |
| 3884 | 3887 | .purgem calc |
| 3885 | −2: ret | |
| 3888 | +2: mov sp, x14 | |
| 3889 | + ret | |
| 3886 | 3890 | endfunc |
| 3887 | 3891 | |
| 3888 | 3892 | function hevc_put_hevc_qpel_hv16_8_end_neon |
| @@ -3906,7 +3910,8 @@ function hevc_put_hevc_qpel_hv16_8_end_neon · 17 unmodified lines | ||
| 3906 | 3910 | .endm |
| 3907 | 3911 | 1: calc_all2 |
| 3908 | 3912 | .purgem calc |
| 3909 | −2: ret | |
| 3913 | +2: mov sp, x14 | |
| 3914 | + ret | |
| 3910 | 3915 | endfunc |
| 3911 | 3916 | |
| 3912 | 3917 | function hevc_put_hevc_qpel_hv32_8_end_neon |
| @@ -3937,162 +3942,187 @@ function hevc_put_hevc_qpel_hv32_8_end_neon · 24 unmodified lines | ||
| 3937 | 3942 | add sp, sp, #32 |
| 3938 | 3943 | subs w6, w6, #16 |
| 3939 | 3944 | b.hi 0b |
| 3940 | − add w10, w3, #6 | |
| 3941 | − add sp, sp, #64 // discard rest of first line | |
| 3942 | − lsl x10, x10, #7 | |
| 3943 | − add sp, sp, x10 // tmp_array without first line | |
| 3945 | + mov sp, x14 | |
| 3944 | 3946 | ret |
| 3945 | 3947 | endfunc |
| 3946 | 3948 | |
| 3947 | −#if HAVE_I8MM | |
| 3948 | −ENABLE_I8MM | |
| 3949 | −function ff_hevc_put_hevc_qpel_hv4_8_neon_i8mm, export=1 | |
| 3950 | − add w10, w3, #7 | |
| 3949 | +.macro qpel_hv suffix | |
| 3950 | +function ff_hevc_put_hevc_qpel_hv4_8_\suffix, export=1 | |
| 3951 | + add w10, w3, #8 | |
| 3951 | 3952 | mov x7, #128 |
| 3952 | 3953 | lsl x10, x10, #7 |
| 3954 | + mov x14, sp | |
| 3953 | 3955 | sub sp, sp, x10 // tmp_array |
| 3954 | − stp x5, x30, [sp, #-32]! | |
| 3955 | − stp x0, x3, [sp, #16] | |
| 3956 | − add x0, sp, #32 | |
| 3956 | + stp x5, x30, [sp, #-48]! | |
| 3957 | + stp x0, x3, [sp, #16] | |
| 3958 | + str x14, [sp, #32] | |
| 3959 | + add x0, sp, #48 | |
| 3957 | 3960 | sub x1, x1, x2, lsl #1 |
| 3958 | 3961 | add x3, x3, #7 |
| 3959 | 3962 | sub x1, x1, x2 |
| 3960 | − bl X(ff_hevc_put_hevc_qpel_h4_8_neon_i8mm) | |
| 3961 | − ldp x0, x3, [sp, #16] | |
| 3962 | − ldp x5, x30, [sp], #32 | |
| 3963 | + bl X(ff_hevc_put_hevc_qpel_h4_8_\suffix) | |
| 3964 | + ldr x14, [sp, #32] | |
| 3965 | + ldp x0, x3, [sp, #16] | |
| 3966 | + ldp x5, x30, [sp], #48 | |
| 3963 | 3967 | b hevc_put_hevc_qpel_hv4_8_end_neon |
| 3964 | 3968 | endfunc |
| 3965 | 3969 | |
| 3966 | −function ff_hevc_put_hevc_qpel_hv6_8_neon_i8mm, export=1 | |
| 3967 | − add w10, w3, #7 | |
| 3970 | +function ff_hevc_put_hevc_qpel_hv6_8_\suffix, export=1 | |
| 3971 | + add w10, w3, #8 | |
| 3968 | 3972 | mov x7, #128 |
| 3969 | 3973 | lsl x10, x10, #7 |
| 3974 | + mov x14, sp | |
| 3970 | 3975 | sub sp, sp, x10 // tmp_array |
| 3971 | − stp x5, x30, [sp, #-32]! | |
| 3972 | − stp x0, x3, [sp, #16] | |
| 3973 | − add x0, sp, #32 | |
| 3976 | + stp x5, x30, [sp, #-48]! | |
| 3977 | + stp x0, x3, [sp, #16] | |
| 3978 | + str x14, [sp, #32] | |
| 3979 | + add x0, sp, #48 | |
| 3974 | 3980 | sub x1, x1, x2, lsl #1 |
| 3975 | 3981 | add x3, x3, #7 |
| 3976 | 3982 | sub x1, x1, x2 |
| 3977 | − bl X(ff_hevc_put_hevc_qpel_h6_8_neon_i8mm) | |
| 3978 | − ldp x0, x3, [sp, #16] | |
| 3979 | − ldp x5, x30, [sp], #32 | |
| 3983 | + bl X(ff_hevc_put_hevc_qpel_h6_8_\suffix) | |
| 3984 | + ldr x14, [sp, #32] | |
| 3985 | + ldp x0, x3, [sp, #16] | |
| 3986 | + ldp x5, x30, [sp], #48 | |
| 3980 | 3987 | b hevc_put_hevc_qpel_hv6_8_end_neon |
| 3981 | 3988 | endfunc |
| 3982 | 3989 | |
| 3983 | −function ff_hevc_put_hevc_qpel_hv8_8_neon_i8mm, export=1 | |
| 3984 | − add w10, w3, #7 | |
| 3990 | +function ff_hevc_put_hevc_qpel_hv8_8_\suffix, export=1 | |
| 3991 | + add w10, w3, #8 | |
| 3985 | 3992 | lsl x10, x10, #7 |
| 3986 | 3993 | sub x1, x1, x2, lsl #1 |
| 3994 | + mov x14, sp | |
| 3987 | 3995 | sub sp, sp, x10 // tmp_array |
| 3988 | − stp x5, x30, [sp, #-32]! | |
| 3989 | − stp x0, x3, [sp, #16] | |
| 3990 | − add x0, sp, #32 | |
| 3996 | + stp x5, x30, [sp, #-48]! | |
| 3997 | + stp x0, x3, [sp, #16] | |
| 3998 | + str x14, [sp, #32] | |
| 3999 | + add x0, sp, #48 | |
| 3991 | 4000 | add x3, x3, #7 |
| 3992 | 4001 | sub x1, x1, x2 |
| 3993 | − bl X(ff_hevc_put_hevc_qpel_h8_8_neon_i8mm) | |
| 3994 | − ldp x0, x3, [sp, #16] | |
| 3995 | − ldp x5, x30, [sp], #32 | |
| 4002 | + bl X(ff_hevc_put_hevc_qpel_h8_8_\suffix) | |
| 4003 | + ldr x14, [sp, #32] | |
| 4004 | + ldp x0, x3, [sp, #16] | |
| 4005 | + ldp x5, x30, [sp], #48 | |
| 3996 | 4006 | b hevc_put_hevc_qpel_hv8_8_end_neon |
| 3997 | 4007 | endfunc |
| 3998 | 4008 | |
| 3999 | −function ff_hevc_put_hevc_qpel_hv12_8_neon_i8mm, export=1 | |
| 4000 | − add w10, w3, #7 | |
| 4009 | +function ff_hevc_put_hevc_qpel_hv12_8_\suffix, export=1 | |
| 4010 | + add w10, w3, #8 | |
| 4001 | 4011 | lsl x10, x10, #7 |
| 4002 | 4012 | sub x1, x1, x2, lsl #1 |
| 4013 | + mov x14, sp | |
| 4003 | 4014 | sub sp, sp, x10 // tmp_array |
| 4004 | − stp x5, x30, [sp, #-32]! | |
| 4005 | − stp x0, x3, [sp, #16] | |
| 4006 | − add x0, sp, #32 | |
| 4015 | + stp x5, x30, [sp, #-48]! | |
| 4016 | + stp x0, x3, [sp, #16] | |
| 4017 | + str x14, [sp, #32] | |
| 4018 | + add x0, sp, #48 | |
| 4007 | 4019 | add x3, x3, #7 |
| 4008 | 4020 | sub x1, x1, x2 |
| 4009 | − bl X(ff_hevc_put_hevc_qpel_h12_8_neon_i8mm) | |
| 4010 | − ldp x0, x3, [sp, #16] | |
| 4011 | − ldp x5, x30, [sp], #32 | |
| 4021 | + mov w6, #12 | |
| 4022 | + bl X(ff_hevc_put_hevc_qpel_h12_8_\suffix) | |
| 4023 | + ldr x14, [sp, #32] | |
| 4024 | + ldp x0, x3, [sp, #16] | |
| 4025 | + ldp x5, x30, [sp], #48 | |
| 4012 | 4026 | b hevc_put_hevc_qpel_hv12_8_end_neon |
| 4013 | 4027 | endfunc |
| 4014 | 4028 | |
| 4015 | −function ff_hevc_put_hevc_qpel_hv16_8_neon_i8mm, export=1 | |
| 4016 | − add w10, w3, #7 | |
| 4029 | +function ff_hevc_put_hevc_qpel_hv16_8_\suffix, export=1 | |
| 4030 | + add w10, w3, #8 | |
| 4017 | 4031 | lsl x10, x10, #7 |
| 4018 | 4032 | sub x1, x1, x2, lsl #1 |
| 4033 | + mov x14, sp | |
| 4019 | 4034 | sub sp, sp, x10 // tmp_array |
| 4020 | − stp x5, x30, [sp, #-32]! | |
| 4021 | − stp x0, x3, [sp, #16] | |
| 4035 | + stp x5, x30, [sp, #-48]! | |
| 4036 | + stp x0, x3, [sp, #16] | |
| 4037 | + str x14, [sp, #32] | |
| 4022 | 4038 | add x3, x3, #7 |
| 4023 | − add x0, sp, #32 | |
| 4039 | + add x0, sp, #48 | |
| 4024 | 4040 | sub x1, x1, x2 |
| 4025 | − bl X(ff_hevc_put_hevc_qpel_h16_8_neon_i8mm) | |
| 4026 | − ldp x0, x3, [sp, #16] | |
| 4027 | − ldp x5, x30, [sp], #32 | |
| 4041 | + bl X(ff_hevc_put_hevc_qpel_h16_8_\suffix) | |
| 4042 | + ldr x14, [sp, #32] | |
| 4043 | + ldp x0, x3, [sp, #16] | |
| 4044 | + ldp x5, x30, [sp], #48 | |
| 4028 | 4045 | b hevc_put_hevc_qpel_hv16_8_end_neon |
| 4029 | 4046 | endfunc |
| 4030 | 4047 | |
| 4031 | −function ff_hevc_put_hevc_qpel_hv24_8_neon_i8mm, export=1 | |
| 4048 | +function ff_hevc_put_hevc_qpel_hv24_8_\suffix, export=1 | |
| 4032 | 4049 | stp x4, x5, [sp, #-64]! |
| 4033 | 4050 | stp x2, x3, [sp, #16] |
| 4034 | 4051 | stp x0, x1, [sp, #32] |
| 4035 | 4052 | str x30, [sp, #48] |
| 4036 | − bl X(ff_hevc_put_hevc_qpel_hv12_8_neon_i8mm) | |
| 4053 | + bl X(ff_hevc_put_hevc_qpel_hv12_8_\suffix) | |
| 4037 | 4054 | ldp x0, x1, [sp, #32] |
| 4038 | 4055 | ldp x2, x3, [sp, #16] |
| 4039 | 4056 | ldp x4, x5, [sp], #48 |
| 4040 | 4057 | add x1, x1, #12 |
| 4041 | 4058 | add x0, x0, #24 |
| 4042 | − bl X(ff_hevc_put_hevc_qpel_hv12_8_neon_i8mm) | |
| 4059 | + bl X(ff_hevc_put_hevc_qpel_hv12_8_\suffix) | |
| 4043 | 4060 | ldr x30, [sp], #16 |
| 4044 | 4061 | ret |
| 4045 | 4062 | endfunc |
| 4046 | 4063 | |
| 4047 | −function ff_hevc_put_hevc_qpel_hv32_8_neon_i8mm, export=1 | |
| 4048 | − add w10, w3, #7 | |
| 4064 | +function ff_hevc_put_hevc_qpel_hv32_8_\suffix, export=1 | |
| 4065 | + add w10, w3, #8 | |
| 4049 | 4066 | sub x1, x1, x2, lsl #1 |
| 4050 | 4067 | lsl x10, x10, #7 |
| 4051 | 4068 | sub x1, x1, x2 |
| 4069 | + mov x14, sp | |
| 4052 | 4070 | sub sp, sp, x10 // tmp_array |
| 4053 | − stp x5, x30, [sp, #-32]! | |
| 4054 | − stp x0, x3, [sp, #16] | |
| 4071 | + stp x5, x30, [sp, #-48]! | |
| 4072 | + stp x0, x3, [sp, #16] | |
| 4073 | + str x14, [sp, #32] | |
| 4055 | 4074 | add x3, x3, #7 |
| 4056 | − add x0, sp, #32 | |
| 4057 | − bl X(ff_hevc_put_hevc_qpel_h32_8_neon_i8mm) | |
| 4058 | − ldp x0, x3, [sp, #16] | |
| 4059 | − ldp x5, x30, [sp], #32 | |
| 4075 | + add x0, sp, #48 | |
| 4076 | + mov w6, #32 | |
| 4077 | + bl X(ff_hevc_put_hevc_qpel_h32_8_\suffix) | |
| 4078 | + ldr x14, [sp, #32] | |
| 4079 | + ldp x0, x3, [sp, #16] | |
| 4080 | + ldp x5, x30, [sp], #48 | |
| 4060 | 4081 | b hevc_put_hevc_qpel_hv32_8_end_neon |
| 4061 | 4082 | endfunc |
| 4062 | 4083 | |
| 4063 | −function ff_hevc_put_hevc_qpel_hv48_8_neon_i8mm, export=1 | |
| 4084 | +function ff_hevc_put_hevc_qpel_hv48_8_\suffix, export=1 | |
| 4064 | 4085 | stp x4, x5, [sp, #-64]! |
| 4065 | 4086 | stp x2, x3, [sp, #16] |
| 4066 | 4087 | stp x0, x1, [sp, #32] |
| 4067 | 4088 | str x30, [sp, #48] |
| 4068 | − bl X(ff_hevc_put_hevc_qpel_hv24_8_neon_i8mm) | |
| 4089 | + bl X(ff_hevc_put_hevc_qpel_hv24_8_\suffix) | |
| 4069 | 4090 | ldp x0, x1, [sp, #32] |
| 4070 | 4091 | ldp x2, x3, [sp, #16] |
| 4071 | 4092 | ldp x4, x5, [sp], #48 |
| 4072 | 4093 | add x1, x1, #24 |
| 4073 | 4094 | add x0, x0, #48 |
| 4074 | − bl X(ff_hevc_put_hevc_qpel_hv24_8_neon_i8mm) | |
| 4095 | + bl X(ff_hevc_put_hevc_qpel_hv24_8_\suffix) | |
| 4075 | 4096 | ldr x30, [sp], #16 |
| 4076 | 4097 | ret |
| 4077 | 4098 | endfunc |
| 4078 | 4099 | |
| 4079 | −function ff_hevc_put_hevc_qpel_hv64_8_neon_i8mm, export=1 | |
| 4100 | +function ff_hevc_put_hevc_qpel_hv64_8_\suffix, export=1 | |
| 4080 | 4101 | stp x4, x5, [sp, #-64]! |
| 4081 | 4102 | stp x2, x3, [sp, #16] |
| 4082 | 4103 | stp x0, x1, [sp, #32] |
| 4083 | 4104 | str x30, [sp, #48] |
| 4084 | 4105 | mov x6, #32 |
| 4085 | − bl X(ff_hevc_put_hevc_qpel_hv32_8_neon_i8mm) | |
| 4106 | + bl X(ff_hevc_put_hevc_qpel_hv32_8_\suffix) | |
| 4086 | 4107 | ldp x0, x1, [sp, #32] |
| 4087 | 4108 | ldp x2, x3, [sp, #16] |
| 4088 | 4109 | ldp x4, x5, [sp], #48 |
| 4089 | 4110 | add x1, x1, #32 |
| 4090 | 4111 | add x0, x0, #64 |
| 4091 | 4112 | mov x6, #32 |
| 4092 | − bl X(ff_hevc_put_hevc_qpel_hv32_8_neon_i8mm) | |
| 4113 | + bl X(ff_hevc_put_hevc_qpel_hv32_8_\suffix) | |
| 4093 | 4114 | ldr x30, [sp], #16 |
| 4094 | 4115 | ret |
| 4095 | 4116 | endfunc |
| 4117 | +.endm | |
| 4118 | + | |
| 4119 | +qpel_hv neon | |
| 4120 | + | |
| 4121 | +#if HAVE_I8MM | |
| 4122 | +ENABLE_I8MM | |
| 4123 | + | |
| 4124 | +qpel_hv neon_i8mm | |
| 4125 | + | |
| 4096 | 4126 | DISABLE_I8MM |
| 4097 | 4127 | #endif |
| 4098 | 4128 | |