History

aarch64: hevc: Produce plain neon versions of qpel_hv

As the plain neon qpel_h functions process two rows at a time, we need to allocate storage for h+8 rows instead of h+7. By allocating storage for h+8 rows, incrementing the stack pointer won't end up at the right spot in the end. Store the intended final stack pointer value in a register x14 which we store on the stack. AWS Graviton 3: put_hevc_qpel_hv4_8_c: 386.0 put_hevc_qpel_hv4_8_neon: 125.7 put_hevc_qpel_hv4_8_i8mm: 83.2 put_hevc_qpel_hv6_8_c: 749.0 put_hevc_qpel_hv6_8_neon: 207.0 put_hevc_qpel_hv6_8_i8mm: 166.0 put_hevc_qpel_hv8_8_c: 1305.2 put_hevc_qpel_hv8_8_neon: 216.5 put_hevc_qpel_hv8_8_i8mm: 213.0 put_hevc_qpel_hv12_8_c: 2570.5 put_hevc_qpel_hv12_8_neon: 480.0 put_hevc_qpel_hv12_8_i8mm: 398.2 put_hevc_qpel_hv16_8_c: 4158.7 put_hevc_qpel_hv16_8_neon: 659.7 put_hevc_qpel_hv16_8_i8mm: 593.5 put_hevc_qpel_hv24_8_c: 8626.7 put_hevc_qpel_hv24_8_neon: 1653.5 put_hevc_qpel_hv24_8_i8mm: 1398.7 put_hevc_qpel_hv32_8_c: 14646.0 put_hevc_qpel_hv32_8_neon: 2566.2 put_hevc_qpel_hv32_8_i8mm: 2287.5 put_hevc_qpel_hv48_8_c: 31072.5 put_hevc_qpel_hv48_8_neon: 6228.5 put_hevc_qpel_hv48_8_i8mm: 5291.0 put_hevc_qpel_hv64_8_c: 53847.2 put_hevc_qpel_hv64_8_neon: 9856.7 put_hevc_qpel_hv64_8_i8mm: 8831.0

Martin Storsjö committed 2 years ago

Commit 5cbeefc1 parent 20c38f42 files · +104 −68Browse files

Unified · Split

2 files
  1. libavcodec/aarch64/hevcdsp_init_aarch64.c +6 −0
  2. libavcodec/aarch64/hevcdsp_qpel_neon.S +98 −68
libavcodec/aarch64/hevcdsp_init_aarch64.c+6−0
@@ -265,6 +265,10 @@ NEON8_FNPROTO(qpel_v, (int16_t *dst, · 264 unmodified lines
265 const uint8_t *src, ptrdiff_t srcstride,265 const uint8_t *src, ptrdiff_t srcstride,
266 int height, intptr_t mx, intptr_t my, int width),);266 int height, intptr_t mx, intptr_t my, int width),);
267 267
268NEON8_FNPROTO(qpel_hv, (int16_t *dst,
269 const uint8_t *src, ptrdiff_t srcstride,
270 int height, intptr_t mx, intptr_t my, int width),);
271
268NEON8_FNPROTO(qpel_hv, (int16_t *dst,272NEON8_FNPROTO(qpel_hv, (int16_t *dst,
269 const uint8_t *src, ptrdiff_t srcstride,273 const uint8_t *src, ptrdiff_t srcstride,
270 int height, intptr_t mx, intptr_t my, int width), _i8mm);274 int height, intptr_t mx, intptr_t my, int width), _i8mm);
@@ -436,6 +440,8 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) · 165 unmodified lines
436 440
437 NEON8_FNASSIGN_SHARED_32(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h,);441 NEON8_FNASSIGN_SHARED_32(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h,);
438 442
443 NEON8_FNASSIGN(c->put_hevc_qpel, 1, 1, qpel_hv,);
444
439 if (have_i8mm(cpu_flags)) {445 if (have_i8mm(cpu_flags)) {
440 NEON8_FNASSIGN(c->put_hevc_epel, 0, 1, epel_h, _i8mm);446 NEON8_FNASSIGN(c->put_hevc_epel, 0, 1, epel_h, _i8mm);
441 NEON8_FNASSIGN(c->put_hevc_epel, 1, 1, epel_hv, _i8mm);447 NEON8_FNASSIGN(c->put_hevc_epel, 1, 1, epel_hv, _i8mm);
libavcodec/aarch64/hevcdsp_qpel_neon.S+98−68
@@ -3804,7 +3804,8 @@ function hevc_put_hevc_qpel_hv4_8_end_neon · 3803 unmodified lines
3804.endm3804.endm
38051: calc_all38051: calc_all
3806.purgem calc3806.purgem calc
38072: ret38072: mov sp, x14
3808 ret
3808endfunc3809endfunc
3809 3810
3810function hevc_put_hevc_qpel_hv6_8_end_neon3811function hevc_put_hevc_qpel_hv6_8_end_neon
@@ -3831,7 +3832,8 @@ function hevc_put_hevc_qpel_hv6_8_end_neon · 20 unmodified lines
3831.endm3832.endm
38321: calc_all38331: calc_all
3833.purgem calc3834.purgem calc
38342: ret38352: mov sp, x14
3836 ret
3835endfunc3837endfunc
3836 3838
3837function hevc_put_hevc_qpel_hv8_8_end_neon3839function hevc_put_hevc_qpel_hv8_8_end_neon
@@ -3857,7 +3859,8 @@ function hevc_put_hevc_qpel_hv8_8_end_neon · 19 unmodified lines
3857.endm3859.endm
38581: calc_all38601: calc_all
3859.purgem calc3861.purgem calc
38602: ret38622: mov sp, x14
3863 ret
3861endfunc3864endfunc
3862 3865
3863function hevc_put_hevc_qpel_hv12_8_end_neon3866function hevc_put_hevc_qpel_hv12_8_end_neon
@@ -3882,7 +3885,8 @@ function hevc_put_hevc_qpel_hv12_8_end_neon · 18 unmodified lines
3882.endm3885.endm
38831: calc_all238861: calc_all2
3884.purgem calc3887.purgem calc
38852: ret38882: mov sp, x14
3889 ret
3886endfunc3890endfunc
3887 3891
3888function hevc_put_hevc_qpel_hv16_8_end_neon3892function hevc_put_hevc_qpel_hv16_8_end_neon
@@ -3906,7 +3910,8 @@ function hevc_put_hevc_qpel_hv16_8_end_neon · 17 unmodified lines
3906.endm3910.endm
39071: calc_all239111: calc_all2
3908.purgem calc3912.purgem calc
39092: ret39132: mov sp, x14
3914 ret
3910endfunc3915endfunc
3911 3916
3912function hevc_put_hevc_qpel_hv32_8_end_neon3917function hevc_put_hevc_qpel_hv32_8_end_neon
@@ -3937,162 +3942,187 @@ function hevc_put_hevc_qpel_hv32_8_end_neon · 24 unmodified lines
3937 add sp, sp, #323942 add sp, sp, #32
3938 subs w6, w6, #163943 subs w6, w6, #16
3939 b.hi 0b3944 b.hi 0b
3940 add w10, w3, #63945 mov sp, x14
3941 add sp, sp, #64 // discard rest of first line
3942 lsl x10, x10, #7
3943 add sp, sp, x10 // tmp_array without first line
3944 ret3946 ret
3945endfunc3947endfunc
3946 3948
3947#if HAVE_I8MM3949.macro qpel_hv suffix
3948ENABLE_I8MM3950function ff_hevc_put_hevc_qpel_hv4_8_\suffix, export=1
3949function ff_hevc_put_hevc_qpel_hv4_8_neon_i8mm, export=13951 add w10, w3, #8
3950 add w10, w3, #7
3951 mov x7, #1283952 mov x7, #128
3952 lsl x10, x10, #73953 lsl x10, x10, #7
3954 mov x14, sp
3953 sub sp, sp, x10 // tmp_array3955 sub sp, sp, x10 // tmp_array
3954 stp x5, x30, [sp, #-32]!3956 stp x5, x30, [sp, #-48]!
3955 stp x0, x3, [sp, #16]3957 stp x0, x3, [sp, #16]
3956 add x0, sp, #323958 str x14, [sp, #32]
3959 add x0, sp, #48
3957 sub x1, x1, x2, lsl #13960 sub x1, x1, x2, lsl #1
3958 add x3, x3, #73961 add x3, x3, #7
3959 sub x1, x1, x23962 sub x1, x1, x2
3960 bl X(ff_hevc_put_hevc_qpel_h4_8_neon_i8mm)3963 bl X(ff_hevc_put_hevc_qpel_h4_8_\suffix)
3961 ldp x0, x3, [sp, #16]3964 ldr x14, [sp, #32]
3962 ldp x5, x30, [sp], #323965 ldp x0, x3, [sp, #16]
3966 ldp x5, x30, [sp], #48
3963 b hevc_put_hevc_qpel_hv4_8_end_neon3967 b hevc_put_hevc_qpel_hv4_8_end_neon
3964endfunc3968endfunc
3965 3969
3966function ff_hevc_put_hevc_qpel_hv6_8_neon_i8mm, export=13970function ff_hevc_put_hevc_qpel_hv6_8_\suffix, export=1
3967 add w10, w3, #73971 add w10, w3, #8
3968 mov x7, #1283972 mov x7, #128
3969 lsl x10, x10, #73973 lsl x10, x10, #7
3974 mov x14, sp
3970 sub sp, sp, x10 // tmp_array3975 sub sp, sp, x10 // tmp_array
3971 stp x5, x30, [sp, #-32]!3976 stp x5, x30, [sp, #-48]!
3972 stp x0, x3, [sp, #16]3977 stp x0, x3, [sp, #16]
3973 add x0, sp, #323978 str x14, [sp, #32]
3979 add x0, sp, #48
3974 sub x1, x1, x2, lsl #13980 sub x1, x1, x2, lsl #1
3975 add x3, x3, #73981 add x3, x3, #7
3976 sub x1, x1, x23982 sub x1, x1, x2
3977 bl X(ff_hevc_put_hevc_qpel_h6_8_neon_i8mm)3983 bl X(ff_hevc_put_hevc_qpel_h6_8_\suffix)
3978 ldp x0, x3, [sp, #16]3984 ldr x14, [sp, #32]
3979 ldp x5, x30, [sp], #323985 ldp x0, x3, [sp, #16]
3986 ldp x5, x30, [sp], #48
3980 b hevc_put_hevc_qpel_hv6_8_end_neon3987 b hevc_put_hevc_qpel_hv6_8_end_neon
3981endfunc3988endfunc
3982 3989
3983function ff_hevc_put_hevc_qpel_hv8_8_neon_i8mm, export=13990function ff_hevc_put_hevc_qpel_hv8_8_\suffix, export=1
3984 add w10, w3, #73991 add w10, w3, #8
3985 lsl x10, x10, #73992 lsl x10, x10, #7
3986 sub x1, x1, x2, lsl #13993 sub x1, x1, x2, lsl #1
3994 mov x14, sp
3987 sub sp, sp, x10 // tmp_array3995 sub sp, sp, x10 // tmp_array
3988 stp x5, x30, [sp, #-32]!3996 stp x5, x30, [sp, #-48]!
3989 stp x0, x3, [sp, #16]3997 stp x0, x3, [sp, #16]
3990 add x0, sp, #323998 str x14, [sp, #32]
3999 add x0, sp, #48
3991 add x3, x3, #74000 add x3, x3, #7
3992 sub x1, x1, x24001 sub x1, x1, x2
3993 bl X(ff_hevc_put_hevc_qpel_h8_8_neon_i8mm)4002 bl X(ff_hevc_put_hevc_qpel_h8_8_\suffix)
3994 ldp x0, x3, [sp, #16]4003 ldr x14, [sp, #32]
3995 ldp x5, x30, [sp], #324004 ldp x0, x3, [sp, #16]
4005 ldp x5, x30, [sp], #48
3996 b hevc_put_hevc_qpel_hv8_8_end_neon4006 b hevc_put_hevc_qpel_hv8_8_end_neon
3997endfunc4007endfunc
3998 4008
3999function ff_hevc_put_hevc_qpel_hv12_8_neon_i8mm, export=14009function ff_hevc_put_hevc_qpel_hv12_8_\suffix, export=1
4000 add w10, w3, #74010 add w10, w3, #8
4001 lsl x10, x10, #74011 lsl x10, x10, #7
4002 sub x1, x1, x2, lsl #14012 sub x1, x1, x2, lsl #1
4013 mov x14, sp
4003 sub sp, sp, x10 // tmp_array4014 sub sp, sp, x10 // tmp_array
4004 stp x5, x30, [sp, #-32]!4015 stp x5, x30, [sp, #-48]!
4005 stp x0, x3, [sp, #16]4016 stp x0, x3, [sp, #16]
4006 add x0, sp, #324017 str x14, [sp, #32]
4018 add x0, sp, #48
4007 add x3, x3, #74019 add x3, x3, #7
4008 sub x1, x1, x24020 sub x1, x1, x2
4009 bl X(ff_hevc_put_hevc_qpel_h12_8_neon_i8mm)4021 mov w6, #12
4010 ldp x0, x3, [sp, #16]4022 bl X(ff_hevc_put_hevc_qpel_h12_8_\suffix)
4011 ldp x5, x30, [sp], #324023 ldr x14, [sp, #32]
4024 ldp x0, x3, [sp, #16]
4025 ldp x5, x30, [sp], #48
4012 b hevc_put_hevc_qpel_hv12_8_end_neon4026 b hevc_put_hevc_qpel_hv12_8_end_neon
4013endfunc4027endfunc
4014 4028
4015function ff_hevc_put_hevc_qpel_hv16_8_neon_i8mm, export=14029function ff_hevc_put_hevc_qpel_hv16_8_\suffix, export=1
4016 add w10, w3, #74030 add w10, w3, #8
4017 lsl x10, x10, #74031 lsl x10, x10, #7
4018 sub x1, x1, x2, lsl #14032 sub x1, x1, x2, lsl #1
4033 mov x14, sp
4019 sub sp, sp, x10 // tmp_array4034 sub sp, sp, x10 // tmp_array
4020 stp x5, x30, [sp, #-32]!4035 stp x5, x30, [sp, #-48]!
4021 stp x0, x3, [sp, #16]4036 stp x0, x3, [sp, #16]
4037 str x14, [sp, #32]
4022 add x3, x3, #74038 add x3, x3, #7
4023 add x0, sp, #324039 add x0, sp, #48
4024 sub x1, x1, x24040 sub x1, x1, x2
4025 bl X(ff_hevc_put_hevc_qpel_h16_8_neon_i8mm)4041 bl X(ff_hevc_put_hevc_qpel_h16_8_\suffix)
4026 ldp x0, x3, [sp, #16]4042 ldr x14, [sp, #32]
4027 ldp x5, x30, [sp], #324043 ldp x0, x3, [sp, #16]
4044 ldp x5, x30, [sp], #48
4028 b hevc_put_hevc_qpel_hv16_8_end_neon4045 b hevc_put_hevc_qpel_hv16_8_end_neon
4029endfunc4046endfunc
4030 4047
4031function ff_hevc_put_hevc_qpel_hv24_8_neon_i8mm, export=14048function ff_hevc_put_hevc_qpel_hv24_8_\suffix, export=1
4032 stp x4, x5, [sp, #-64]!4049 stp x4, x5, [sp, #-64]!
4033 stp x2, x3, [sp, #16]4050 stp x2, x3, [sp, #16]
4034 stp x0, x1, [sp, #32]4051 stp x0, x1, [sp, #32]
4035 str x30, [sp, #48]4052 str x30, [sp, #48]
4036 bl X(ff_hevc_put_hevc_qpel_hv12_8_neon_i8mm)4053 bl X(ff_hevc_put_hevc_qpel_hv12_8_\suffix)
4037 ldp x0, x1, [sp, #32]4054 ldp x0, x1, [sp, #32]
4038 ldp x2, x3, [sp, #16]4055 ldp x2, x3, [sp, #16]
4039 ldp x4, x5, [sp], #484056 ldp x4, x5, [sp], #48
4040 add x1, x1, #124057 add x1, x1, #12
4041 add x0, x0, #244058 add x0, x0, #24
4042 bl X(ff_hevc_put_hevc_qpel_hv12_8_neon_i8mm)4059 bl X(ff_hevc_put_hevc_qpel_hv12_8_\suffix)
4043 ldr x30, [sp], #164060 ldr x30, [sp], #16
4044 ret4061 ret
4045endfunc4062endfunc
4046 4063
4047function ff_hevc_put_hevc_qpel_hv32_8_neon_i8mm, export=14064function ff_hevc_put_hevc_qpel_hv32_8_\suffix, export=1
4048 add w10, w3, #74065 add w10, w3, #8
4049 sub x1, x1, x2, lsl #14066 sub x1, x1, x2, lsl #1
4050 lsl x10, x10, #74067 lsl x10, x10, #7
4051 sub x1, x1, x24068 sub x1, x1, x2
4069 mov x14, sp
4052 sub sp, sp, x10 // tmp_array4070 sub sp, sp, x10 // tmp_array
4053 stp x5, x30, [sp, #-32]!4071 stp x5, x30, [sp, #-48]!
4054 stp x0, x3, [sp, #16]4072 stp x0, x3, [sp, #16]
4073 str x14, [sp, #32]
4055 add x3, x3, #74074 add x3, x3, #7
4056 add x0, sp, #324075 add x0, sp, #48
4057 bl X(ff_hevc_put_hevc_qpel_h32_8_neon_i8mm)4076 mov w6, #32
4058 ldp x0, x3, [sp, #16]4077 bl X(ff_hevc_put_hevc_qpel_h32_8_\suffix)
4059 ldp x5, x30, [sp], #324078 ldr x14, [sp, #32]
4079 ldp x0, x3, [sp, #16]
4080 ldp x5, x30, [sp], #48
4060 b hevc_put_hevc_qpel_hv32_8_end_neon4081 b hevc_put_hevc_qpel_hv32_8_end_neon
4061endfunc4082endfunc
4062 4083
4063function ff_hevc_put_hevc_qpel_hv48_8_neon_i8mm, export=14084function ff_hevc_put_hevc_qpel_hv48_8_\suffix, export=1
4064 stp x4, x5, [sp, #-64]!4085 stp x4, x5, [sp, #-64]!
4065 stp x2, x3, [sp, #16]4086 stp x2, x3, [sp, #16]
4066 stp x0, x1, [sp, #32]4087 stp x0, x1, [sp, #32]
4067 str x30, [sp, #48]4088 str x30, [sp, #48]
4068 bl X(ff_hevc_put_hevc_qpel_hv24_8_neon_i8mm)4089 bl X(ff_hevc_put_hevc_qpel_hv24_8_\suffix)
4069 ldp x0, x1, [sp, #32]4090 ldp x0, x1, [sp, #32]
4070 ldp x2, x3, [sp, #16]4091 ldp x2, x3, [sp, #16]
4071 ldp x4, x5, [sp], #484092 ldp x4, x5, [sp], #48
4072 add x1, x1, #244093 add x1, x1, #24
4073 add x0, x0, #484094 add x0, x0, #48
4074 bl X(ff_hevc_put_hevc_qpel_hv24_8_neon_i8mm)4095 bl X(ff_hevc_put_hevc_qpel_hv24_8_\suffix)
4075 ldr x30, [sp], #164096 ldr x30, [sp], #16
4076 ret4097 ret
4077endfunc4098endfunc
4078 4099
4079function ff_hevc_put_hevc_qpel_hv64_8_neon_i8mm, export=14100function ff_hevc_put_hevc_qpel_hv64_8_\suffix, export=1
4080 stp x4, x5, [sp, #-64]!4101 stp x4, x5, [sp, #-64]!
4081 stp x2, x3, [sp, #16]4102 stp x2, x3, [sp, #16]
4082 stp x0, x1, [sp, #32]4103 stp x0, x1, [sp, #32]
4083 str x30, [sp, #48]4104 str x30, [sp, #48]
4084 mov x6, #324105 mov x6, #32
4085 bl X(ff_hevc_put_hevc_qpel_hv32_8_neon_i8mm)4106 bl X(ff_hevc_put_hevc_qpel_hv32_8_\suffix)
4086 ldp x0, x1, [sp, #32]4107 ldp x0, x1, [sp, #32]
4087 ldp x2, x3, [sp, #16]4108 ldp x2, x3, [sp, #16]
4088 ldp x4, x5, [sp], #484109 ldp x4, x5, [sp], #48
4089 add x1, x1, #324110 add x1, x1, #32
4090 add x0, x0, #644111 add x0, x0, #64
4091 mov x6, #324112 mov x6, #32
4092 bl X(ff_hevc_put_hevc_qpel_hv32_8_neon_i8mm)4113 bl X(ff_hevc_put_hevc_qpel_hv32_8_\suffix)
4093 ldr x30, [sp], #164114 ldr x30, [sp], #16
4094 ret4115 ret
4095endfunc4116endfunc
4117.endm
4118
4119qpel_hv neon
4120
4121#if HAVE_I8MM
4122ENABLE_I8MM
4123
4124qpel_hv neon_i8mm
4125
4096DISABLE_I8MM4126DISABLE_I8MM
4097#endif4127#endif
4098 4128

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t