History

aarch64: hevc: Produce plain neon versions of qpel_hv

As the plain neon qpel_h functions process two rows at a time, we need to allocate storage for h+8 rows instead of h+7. By allocating storage for h+8 rows, incrementing the stack pointer won't end up at the right spot in the end. Store the intended final stack pointer value in a register x14 which we store on the stack. AWS Graviton 3: put_hevc_qpel_hv4_8_c: 386.0 put_hevc_qpel_hv4_8_neon: 125.7 put_hevc_qpel_hv4_8_i8mm: 83.2 put_hevc_qpel_hv6_8_c: 749.0 put_hevc_qpel_hv6_8_neon: 207.0 put_hevc_qpel_hv6_8_i8mm: 166.0 put_hevc_qpel_hv8_8_c: 1305.2 put_hevc_qpel_hv8_8_neon: 216.5 put_hevc_qpel_hv8_8_i8mm: 213.0 put_hevc_qpel_hv12_8_c: 2570.5 put_hevc_qpel_hv12_8_neon: 480.0 put_hevc_qpel_hv12_8_i8mm: 398.2 put_hevc_qpel_hv16_8_c: 4158.7 put_hevc_qpel_hv16_8_neon: 659.7 put_hevc_qpel_hv16_8_i8mm: 593.5 put_hevc_qpel_hv24_8_c: 8626.7 put_hevc_qpel_hv24_8_neon: 1653.5 put_hevc_qpel_hv24_8_i8mm: 1398.7 put_hevc_qpel_hv32_8_c: 14646.0 put_hevc_qpel_hv32_8_neon: 2566.2 put_hevc_qpel_hv32_8_i8mm: 2287.5 put_hevc_qpel_hv48_8_c: 31072.5 put_hevc_qpel_hv48_8_neon: 6228.5 put_hevc_qpel_hv48_8_i8mm: 5291.0 put_hevc_qpel_hv64_8_c: 53847.2 put_hevc_qpel_hv64_8_neon: 9856.7 put_hevc_qpel_hv64_8_i8mm: 8831.0

Martin Storsjö committed 2 years ago

Commit 5cbeefc1 parent 20c38f42 files · +104 −68Browse files

Showing the changes to libavcodec (2 of 2 files). Show all

Unified · Split

2 files
  1. libavcodec/aarch64/hevcdsp_init_aarch64.c +6 −0
  2. libavcodec/aarch64/hevcdsp_qpel_neon.S +98 −68
libavcodec/aarch64/hevcdsp_init_aarch64.c+6−0
@@ -265,6 +265,10 @@ NEON8_FNPROTO(qpel_v, (int16_t *dst, · 264 unmodified lines
265265 const uint8_t *src, ptrdiff_t srcstride,
266266 int height, intptr_t mx, intptr_t my, int width),);
267267
268+NEON8_FNPROTO(qpel_hv, (int16_t *dst,
269+ const uint8_t *src, ptrdiff_t srcstride,
270+ int height, intptr_t mx, intptr_t my, int width),);
271+
268272 NEON8_FNPROTO(qpel_hv, (int16_t *dst,
269273 const uint8_t *src, ptrdiff_t srcstride,
270274 int height, intptr_t mx, intptr_t my, int width), _i8mm);
@@ -436,6 +440,8 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) · 165 unmodified lines
436440
437441 NEON8_FNASSIGN_SHARED_32(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h,);
438442
443+ NEON8_FNASSIGN(c->put_hevc_qpel, 1, 1, qpel_hv,);
444+
439445 if (have_i8mm(cpu_flags)) {
440446 NEON8_FNASSIGN(c->put_hevc_epel, 0, 1, epel_h, _i8mm);
441447 NEON8_FNASSIGN(c->put_hevc_epel, 1, 1, epel_hv, _i8mm);
libavcodec/aarch64/hevcdsp_qpel_neon.S+98−68
@@ -3804,7 +3804,8 @@ function hevc_put_hevc_qpel_hv4_8_end_neon · 3803 unmodified lines
38043804 .endm
38053805 1: calc_all
38063806 .purgem calc
3807−2: ret
3807+2: mov sp, x14
3808+ ret
38083809 endfunc
38093810
38103811 function hevc_put_hevc_qpel_hv6_8_end_neon
@@ -3831,7 +3832,8 @@ function hevc_put_hevc_qpel_hv6_8_end_neon · 20 unmodified lines
38313832 .endm
38323833 1: calc_all
38333834 .purgem calc
3834−2: ret
3835+2: mov sp, x14
3836+ ret
38353837 endfunc
38363838
38373839 function hevc_put_hevc_qpel_hv8_8_end_neon
@@ -3857,7 +3859,8 @@ function hevc_put_hevc_qpel_hv8_8_end_neon · 19 unmodified lines
38573859 .endm
38583860 1: calc_all
38593861 .purgem calc
3860−2: ret
3862+2: mov sp, x14
3863+ ret
38613864 endfunc
38623865
38633866 function hevc_put_hevc_qpel_hv12_8_end_neon
@@ -3882,7 +3885,8 @@ function hevc_put_hevc_qpel_hv12_8_end_neon · 18 unmodified lines
38823885 .endm
38833886 1: calc_all2
38843887 .purgem calc
3885−2: ret
3888+2: mov sp, x14
3889+ ret
38863890 endfunc
38873891
38883892 function hevc_put_hevc_qpel_hv16_8_end_neon
@@ -3906,7 +3910,8 @@ function hevc_put_hevc_qpel_hv16_8_end_neon · 17 unmodified lines
39063910 .endm
39073911 1: calc_all2
39083912 .purgem calc
3909−2: ret
3913+2: mov sp, x14
3914+ ret
39103915 endfunc
39113916
39123917 function hevc_put_hevc_qpel_hv32_8_end_neon
@@ -3937,162 +3942,187 @@ function hevc_put_hevc_qpel_hv32_8_end_neon · 24 unmodified lines
39373942 add sp, sp, #32
39383943 subs w6, w6, #16
39393944 b.hi 0b
3940− add w10, w3, #6
3941− add sp, sp, #64 // discard rest of first line
3942− lsl x10, x10, #7
3943− add sp, sp, x10 // tmp_array without first line
3945+ mov sp, x14
39443946 ret
39453947 endfunc
39463948
3947−#if HAVE_I8MM
3948−ENABLE_I8MM
3949−function ff_hevc_put_hevc_qpel_hv4_8_neon_i8mm, export=1
3950− add w10, w3, #7
3949+.macro qpel_hv suffix
3950+function ff_hevc_put_hevc_qpel_hv4_8_\suffix, export=1
3951+ add w10, w3, #8
39513952 mov x7, #128
39523953 lsl x10, x10, #7
3954+ mov x14, sp
39533955 sub sp, sp, x10 // tmp_array
3954− stp x5, x30, [sp, #-32]!
3955− stp x0, x3, [sp, #16]
3956− add x0, sp, #32
3956+ stp x5, x30, [sp, #-48]!
3957+ stp x0, x3, [sp, #16]
3958+ str x14, [sp, #32]
3959+ add x0, sp, #48
39573960 sub x1, x1, x2, lsl #1
39583961 add x3, x3, #7
39593962 sub x1, x1, x2
3960− bl X(ff_hevc_put_hevc_qpel_h4_8_neon_i8mm)
3961− ldp x0, x3, [sp, #16]
3962− ldp x5, x30, [sp], #32
3963+ bl X(ff_hevc_put_hevc_qpel_h4_8_\suffix)
3964+ ldr x14, [sp, #32]
3965+ ldp x0, x3, [sp, #16]
3966+ ldp x5, x30, [sp], #48
39633967 b hevc_put_hevc_qpel_hv4_8_end_neon
39643968 endfunc
39653969
3966−function ff_hevc_put_hevc_qpel_hv6_8_neon_i8mm, export=1
3967− add w10, w3, #7
3970+function ff_hevc_put_hevc_qpel_hv6_8_\suffix, export=1
3971+ add w10, w3, #8
39683972 mov x7, #128
39693973 lsl x10, x10, #7
3974+ mov x14, sp
39703975 sub sp, sp, x10 // tmp_array
3971− stp x5, x30, [sp, #-32]!
3972− stp x0, x3, [sp, #16]
3973− add x0, sp, #32
3976+ stp x5, x30, [sp, #-48]!
3977+ stp x0, x3, [sp, #16]
3978+ str x14, [sp, #32]
3979+ add x0, sp, #48
39743980 sub x1, x1, x2, lsl #1
39753981 add x3, x3, #7
39763982 sub x1, x1, x2
3977− bl X(ff_hevc_put_hevc_qpel_h6_8_neon_i8mm)
3978− ldp x0, x3, [sp, #16]
3979− ldp x5, x30, [sp], #32
3983+ bl X(ff_hevc_put_hevc_qpel_h6_8_\suffix)
3984+ ldr x14, [sp, #32]
3985+ ldp x0, x3, [sp, #16]
3986+ ldp x5, x30, [sp], #48
39803987 b hevc_put_hevc_qpel_hv6_8_end_neon
39813988 endfunc
39823989
3983−function ff_hevc_put_hevc_qpel_hv8_8_neon_i8mm, export=1
3984− add w10, w3, #7
3990+function ff_hevc_put_hevc_qpel_hv8_8_\suffix, export=1
3991+ add w10, w3, #8
39853992 lsl x10, x10, #7
39863993 sub x1, x1, x2, lsl #1
3994+ mov x14, sp
39873995 sub sp, sp, x10 // tmp_array
3988− stp x5, x30, [sp, #-32]!
3989− stp x0, x3, [sp, #16]
3990− add x0, sp, #32
3996+ stp x5, x30, [sp, #-48]!
3997+ stp x0, x3, [sp, #16]
3998+ str x14, [sp, #32]
3999+ add x0, sp, #48
39914000 add x3, x3, #7
39924001 sub x1, x1, x2
3993− bl X(ff_hevc_put_hevc_qpel_h8_8_neon_i8mm)
3994− ldp x0, x3, [sp, #16]
3995− ldp x5, x30, [sp], #32
4002+ bl X(ff_hevc_put_hevc_qpel_h8_8_\suffix)
4003+ ldr x14, [sp, #32]
4004+ ldp x0, x3, [sp, #16]
4005+ ldp x5, x30, [sp], #48
39964006 b hevc_put_hevc_qpel_hv8_8_end_neon
39974007 endfunc
39984008
3999−function ff_hevc_put_hevc_qpel_hv12_8_neon_i8mm, export=1
4000− add w10, w3, #7
4009+function ff_hevc_put_hevc_qpel_hv12_8_\suffix, export=1
4010+ add w10, w3, #8
40014011 lsl x10, x10, #7
40024012 sub x1, x1, x2, lsl #1
4013+ mov x14, sp
40034014 sub sp, sp, x10 // tmp_array
4004− stp x5, x30, [sp, #-32]!
4005− stp x0, x3, [sp, #16]
4006− add x0, sp, #32
4015+ stp x5, x30, [sp, #-48]!
4016+ stp x0, x3, [sp, #16]
4017+ str x14, [sp, #32]
4018+ add x0, sp, #48
40074019 add x3, x3, #7
40084020 sub x1, x1, x2
4009− bl X(ff_hevc_put_hevc_qpel_h12_8_neon_i8mm)
4010− ldp x0, x3, [sp, #16]
4011− ldp x5, x30, [sp], #32
4021+ mov w6, #12
4022+ bl X(ff_hevc_put_hevc_qpel_h12_8_\suffix)
4023+ ldr x14, [sp, #32]
4024+ ldp x0, x3, [sp, #16]
4025+ ldp x5, x30, [sp], #48
40124026 b hevc_put_hevc_qpel_hv12_8_end_neon
40134027 endfunc
40144028
4015−function ff_hevc_put_hevc_qpel_hv16_8_neon_i8mm, export=1
4016− add w10, w3, #7
4029+function ff_hevc_put_hevc_qpel_hv16_8_\suffix, export=1
4030+ add w10, w3, #8
40174031 lsl x10, x10, #7
40184032 sub x1, x1, x2, lsl #1
4033+ mov x14, sp
40194034 sub sp, sp, x10 // tmp_array
4020− stp x5, x30, [sp, #-32]!
4021− stp x0, x3, [sp, #16]
4035+ stp x5, x30, [sp, #-48]!
4036+ stp x0, x3, [sp, #16]
4037+ str x14, [sp, #32]
40224038 add x3, x3, #7
4023− add x0, sp, #32
4039+ add x0, sp, #48
40244040 sub x1, x1, x2
4025− bl X(ff_hevc_put_hevc_qpel_h16_8_neon_i8mm)
4026− ldp x0, x3, [sp, #16]
4027− ldp x5, x30, [sp], #32
4041+ bl X(ff_hevc_put_hevc_qpel_h16_8_\suffix)
4042+ ldr x14, [sp, #32]
4043+ ldp x0, x3, [sp, #16]
4044+ ldp x5, x30, [sp], #48
40284045 b hevc_put_hevc_qpel_hv16_8_end_neon
40294046 endfunc
40304047
4031−function ff_hevc_put_hevc_qpel_hv24_8_neon_i8mm, export=1
4048+function ff_hevc_put_hevc_qpel_hv24_8_\suffix, export=1
40324049 stp x4, x5, [sp, #-64]!
40334050 stp x2, x3, [sp, #16]
40344051 stp x0, x1, [sp, #32]
40354052 str x30, [sp, #48]
4036− bl X(ff_hevc_put_hevc_qpel_hv12_8_neon_i8mm)
4053+ bl X(ff_hevc_put_hevc_qpel_hv12_8_\suffix)
40374054 ldp x0, x1, [sp, #32]
40384055 ldp x2, x3, [sp, #16]
40394056 ldp x4, x5, [sp], #48
40404057 add x1, x1, #12
40414058 add x0, x0, #24
4042− bl X(ff_hevc_put_hevc_qpel_hv12_8_neon_i8mm)
4059+ bl X(ff_hevc_put_hevc_qpel_hv12_8_\suffix)
40434060 ldr x30, [sp], #16
40444061 ret
40454062 endfunc
40464063
4047−function ff_hevc_put_hevc_qpel_hv32_8_neon_i8mm, export=1
4048− add w10, w3, #7
4064+function ff_hevc_put_hevc_qpel_hv32_8_\suffix, export=1
4065+ add w10, w3, #8
40494066 sub x1, x1, x2, lsl #1
40504067 lsl x10, x10, #7
40514068 sub x1, x1, x2
4069+ mov x14, sp
40524070 sub sp, sp, x10 // tmp_array
4053− stp x5, x30, [sp, #-32]!
4054− stp x0, x3, [sp, #16]
4071+ stp x5, x30, [sp, #-48]!
4072+ stp x0, x3, [sp, #16]
4073+ str x14, [sp, #32]
40554074 add x3, x3, #7
4056− add x0, sp, #32
4057− bl X(ff_hevc_put_hevc_qpel_h32_8_neon_i8mm)
4058− ldp x0, x3, [sp, #16]
4059− ldp x5, x30, [sp], #32
4075+ add x0, sp, #48
4076+ mov w6, #32
4077+ bl X(ff_hevc_put_hevc_qpel_h32_8_\suffix)
4078+ ldr x14, [sp, #32]
4079+ ldp x0, x3, [sp, #16]
4080+ ldp x5, x30, [sp], #48
40604081 b hevc_put_hevc_qpel_hv32_8_end_neon
40614082 endfunc
40624083
4063−function ff_hevc_put_hevc_qpel_hv48_8_neon_i8mm, export=1
4084+function ff_hevc_put_hevc_qpel_hv48_8_\suffix, export=1
40644085 stp x4, x5, [sp, #-64]!
40654086 stp x2, x3, [sp, #16]
40664087 stp x0, x1, [sp, #32]
40674088 str x30, [sp, #48]
4068− bl X(ff_hevc_put_hevc_qpel_hv24_8_neon_i8mm)
4089+ bl X(ff_hevc_put_hevc_qpel_hv24_8_\suffix)
40694090 ldp x0, x1, [sp, #32]
40704091 ldp x2, x3, [sp, #16]
40714092 ldp x4, x5, [sp], #48
40724093 add x1, x1, #24
40734094 add x0, x0, #48
4074− bl X(ff_hevc_put_hevc_qpel_hv24_8_neon_i8mm)
4095+ bl X(ff_hevc_put_hevc_qpel_hv24_8_\suffix)
40754096 ldr x30, [sp], #16
40764097 ret
40774098 endfunc
40784099
4079−function ff_hevc_put_hevc_qpel_hv64_8_neon_i8mm, export=1
4100+function ff_hevc_put_hevc_qpel_hv64_8_\suffix, export=1
40804101 stp x4, x5, [sp, #-64]!
40814102 stp x2, x3, [sp, #16]
40824103 stp x0, x1, [sp, #32]
40834104 str x30, [sp, #48]
40844105 mov x6, #32
4085− bl X(ff_hevc_put_hevc_qpel_hv32_8_neon_i8mm)
4106+ bl X(ff_hevc_put_hevc_qpel_hv32_8_\suffix)
40864107 ldp x0, x1, [sp, #32]
40874108 ldp x2, x3, [sp, #16]
40884109 ldp x4, x5, [sp], #48
40894110 add x1, x1, #32
40904111 add x0, x0, #64
40914112 mov x6, #32
4092− bl X(ff_hevc_put_hevc_qpel_hv32_8_neon_i8mm)
4113+ bl X(ff_hevc_put_hevc_qpel_hv32_8_\suffix)
40934114 ldr x30, [sp], #16
40944115 ret
40954116 endfunc
4117+.endm
4118+
4119+qpel_hv neon
4120+
4121+#if HAVE_I8MM
4122+ENABLE_I8MM
4123+
4124+qpel_hv neon_i8mm
4125+
40964126 DISABLE_I8MM
40974127 #endif
40984128

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t