History

codec/aarch64/hevc:add idct_32x32_neon

got 73% speed up (run_count=1000, CPU=Cortex A53) idct_32x32_neon: 4826 idct_32x32_c: 18236 idct_32x32_neon: 4824 idct_32x32_c: 18149 idct_32x32_neon: 4937 idct_32x32_c: 18333

xufuji456 committed 3 years ago

Commit 00a062b1 parent 4dffa562 files · +266 −28Browse files

Unified · Split

2 files
  1. libavcodec/aarch64/hevcdsp_idct_neon.S +261 −28
  2. libavcodec/aarch64/hevcdsp_init_aarch64.c +5 −0
libavcodec/aarch64/hevcdsp_idct_neon.S+261−28
@@ -6,6 +6,7 @@ · 5 unmodified lines
66 * Ported from arm/hevcdsp_idct_neon.S by
77 * Copyright (c) 2020 Reimar Döffinger
88 * Copyright (c) 2023 J. Dekker <jdek@itanimul.li>
9+ * Copyright (c) 2023 xu fulong <839789740@qq.com>
910 *
1011 * This file is part of FFmpeg.
1112 *
@@ -477,34 +478,52 @@ endfunc · 465 unmodified lines
477478 sqrshrn2 \out3\().8h, \in7, \shift
478479 .endm
479480
480−.macro transpose16_4x4_2 r0, r1, r2, r3
481+// use temp register to transpose, then we can reuse it
482+.macro transpose16_4x4_2 r0, r1, r2, r3, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5
481483 // lower halves
482− trn1 v2.4h, \r0\().4h, \r1\().4h
483− trn2 v3.4h, \r0\().4h, \r1\().4h
484− trn1 v4.4h, \r2\().4h, \r3\().4h
485− trn2 v5.4h, \r2\().4h, \r3\().4h
486− trn1 v6.2s, v2.2s, v4.2s
487− trn2 v7.2s, v2.2s, v4.2s
488− trn1 v2.2s, v3.2s, v5.2s
489− trn2 v4.2s, v3.2s, v5.2s
490− mov \r0\().d[0], v6.d[0]
491− mov \r2\().d[0], v7.d[0]
492− mov \r1\().d[0], v2.d[0]
493− mov \r3\().d[0], v4.d[0]
484+ trn1 \tmp0\().4h, \r0\().4h, \r1\().4h
485+ trn2 \tmp1\().4h, \r0\().4h, \r1\().4h
486+ trn1 \tmp2\().4h, \r2\().4h, \r3\().4h
487+ trn2 \tmp3\().4h, \r2\().4h, \r3\().4h
488+ trn1 \tmp4\().2s, \tmp0\().2s, \tmp2\().2s
489+ trn2 \tmp5\().2s, \tmp0\().2s, \tmp2\().2s
490+ trn1 \tmp0\().2s, \tmp1\().2s, \tmp3\().2s
491+ trn2 \tmp2\().2s, \tmp1\().2s, \tmp3\().2s
492+ mov \r0\().d[0], \tmp4\().d[0]
493+ mov \r2\().d[0], \tmp5\().d[0]
494+ mov \r1\().d[0], \tmp0\().d[0]
495+ mov \r3\().d[0], \tmp2\().d[0]
494496
495497 // upper halves in reverse order
496− trn1 v2.8h, \r3\().8h, \r2\().8h
497− trn2 v3.8h, \r3\().8h, \r2\().8h
498− trn1 v4.8h, \r1\().8h, \r0\().8h
499− trn2 v5.8h, \r1\().8h, \r0\().8h
500− trn1 v6.4s, v2.4s, v4.4s
501− trn2 v7.4s, v2.4s, v4.4s
502− trn1 v2.4s, v3.4s, v5.4s
503− trn2 v4.4s, v3.4s, v5.4s
504− mov \r3\().d[1], v6.d[1]
505− mov \r1\().d[1], v7.d[1]
506− mov \r2\().d[1], v2.d[1]
507− mov \r0\().d[1], v4.d[1]
498+ trn1 \tmp0\().8h, \r3\().8h, \r2\().8h
499+ trn2 \tmp1\().8h, \r3\().8h, \r2\().8h
500+ trn1 \tmp2\().8h, \r1\().8h, \r0\().8h
501+ trn2 \tmp3\().8h, \r1\().8h, \r0\().8h
502+ trn1 \tmp4\().4s, \tmp0\().4s, \tmp2\().4s
503+ trn2 \tmp5\().4s, \tmp0\().4s, \tmp2\().4s
504+ trn1 \tmp0\().4s, \tmp1\().4s, \tmp3\().4s
505+ trn2 \tmp2\().4s, \tmp1\().4s, \tmp3\().4s
506+ mov \r3\().d[1], \tmp4\().d[1]
507+ mov \r1\().d[1], \tmp5\().d[1]
508+ mov \r2\().d[1], \tmp0\().d[1]
509+ mov \r0\().d[1], \tmp2\().d[1]
510+.endm
511+
512+// stores in0, in2, in4, in6 ascending from off1 and
513+// stores in1, in3, in5, in7 descending from off2
514+.macro store_to_stack off1, off2, in0, in2, in4, in6, in7, in5, in3, in1
515+ add x1, sp, #\off1
516+ add x3, sp, #\off2
517+ mov x2, #-16
518+ mov x4, #16
519+ st1 {\in0}, [x1], x4
520+ st1 {\in1}, [x3], x2
521+ st1 {\in2}, [x1], x4
522+ st1 {\in3}, [x3], x2
523+ st1 {\in4}, [x1], x4
524+ st1 {\in5}, [x3], x2
525+ st1 {\in6}, [x1]
526+ st1 {\in7}, [x3]
508527 .endm
509528
510529 .macro tr_16x4 name, shift, offset, step
@@ -543,27 +562,34 @@ function func_tr_16x4_\name · 32 unmodified lines
543562
544563 add x4, sp, #\offset
545564 ld1 {v16.4s-v19.4s}, [x4], #64
546−
547565 butterfly16 v16.4s, v21.4s, v17.4s, v22.4s, v18.4s, v23.4s, v19.4s, v24.4s
566+ .if \shift > 0
548567 scale v29, v30, v31, v24, v20.4s, v16.4s, v21.4s, v17.4s, v22.4s, v18.4s, v23.4s, v19.4s, \shift
549− transpose16_4x4_2 v29, v30, v31, v24
568+ transpose16_4x4_2 v29, v30, v31, v24, v2, v3, v4, v5, v6, v7
550569 mov x1, x6
551570 add x3, x6, #(24 +3*32)
552571 mov x2, #32
553572 mov x4, #-32
554573 store16 v29.d, v30.d, v31.d, v24.d, x4
574+ .else
575+ store_to_stack \offset, (\offset + 240), v20.4s, v21.4s, v22.4s, v23.4s, v19.4s, v18.4s, v17.4s, v16.4s
576+ .endif
555577
556578 add x4, sp, #(\offset + 64)
557579 ld1 {v16.4s-v19.4s}, [x4]
558580 butterfly16 v16.4s, v25.4s, v17.4s, v26.4s, v18.4s, v27.4s, v19.4s, v28.4s
581+ .if \shift > 0
559582 scale v29, v30, v31, v20, v20.4s, v16.4s, v25.4s, v17.4s, v26.4s, v18.4s, v27.4s, v19.4s, \shift
560− transpose16_4x4_2 v29, v30, v31, v20
583+ transpose16_4x4_2 v29, v30, v31, v20, v2, v3, v4, v5, v6, v7
561584
562585 add x1, x6, #8
563586 add x3, x6, #(16 + 3 * 32)
564587 mov x2, #32
565588 mov x4, #-32
566589 store16 v29.d, v30.d, v31.d, v20.d, x4
590+ .else
591+ store_to_stack (\offset + 64), (\offset + 176), v20.4s, v25.4s, v26.4s, v27.4s, v19.4s, v18.4s, v17.4s, v16.4s
592+ .endif
567593
568594 ret
569595 endfunc
@@ -596,6 +622,203 @@ function ff_hevc_idct_16x16_\bitdepth\()_neon, export=1 · 26 unmodified lines
596622 endfunc
597623 .endm
598624
625+.macro load32
626+ add x1, x5, #64
627+ add x3, x1, #128
628+ mov x2, #256
629+ ld1 {v4.d}[0], [x1], x2
630+ ld1 {v4.d}[1], [x3], x2
631+ ld1 {v5.d}[0], [x1], x2
632+ ld1 {v5.d}[1], [x3], x2
633+ ld1 {v6.d}[0], [x1], x2
634+ ld1 {v6.d}[1], [x3], x2
635+ ld1 {v7.d}[0], [x1], x2
636+ ld1 {v7.d}[1], [x3], x2
637+ ld1 {v16.d}[0], [x1], x2
638+ ld1 {v16.d}[1], [x3], x2
639+ ld1 {v17.d}[0], [x1], x2
640+ ld1 {v17.d}[1], [x3], x2
641+ ld1 {v18.d}[0], [x1], x2
642+ ld1 {v18.d}[1], [x3], x2
643+ ld1 {v19.d}[0], [x1], x2
644+ ld1 {v19.d}[1], [x3], x2
645+.endm
646+
647+.macro add_member32 in, t0, t1, t2, t3, op0, op1, op2, op3, p
648+ sum_sub v24.4s, \in, \t0, \op0, \p
649+ sum_sub v25.4s, \in, \t1, \op1, \p
650+ sum_sub v26.4s, \in, \t2, \op2, \p
651+ sum_sub v27.4s, \in, \t3, \op3, \p
652+.endm
653+
654+.macro butterfly32 in0, in1, in2, in3, out
655+ add \out, \in0, \in1
656+ sub \in0, \in0, \in1
657+ add \in1, \in2, \in3
658+ sub \in2, \in2, \in3
659+.endm
660+
661+.macro multiply in
662+ smull v24.4s, v4.4h, \in\().h[0]
663+ smull v25.4s, v4.4h, \in\().h[1]
664+ smull v26.4s, v4.4h, \in\().h[2]
665+ smull v27.4s, v4.4h, \in\().h[3]
666+.endm
667+
668+.macro scale_store shift
669+ ld1 {v28.8h-v31.8h}, [x4], #64
670+ butterfly32 v28.4s, v24.4s, v29.4s, v25.4s, v2.4s
671+ butterfly32 v30.4s, v26.4s, v31.4s, v27.4s, v3.4s
672+ scale v20, v21, v22, v23, v2.4s, v28.4s, v24.4s, v29.4s, v3.4s, v30.4s, v26.4s, v31.4s, \shift
673+
674+ transpose16_4x4_2 v20, v21, v22, v23, v24, v25, v26, v27, v28, v29
675+ store16 v20.d, v21.d, v22.d, v23.d, x8
676+
677+ // reload coefficients
678+ ld1 {v2.4h-v3.4h}, [x9]
679+.endm
680+
681+function tr_block1
682+ multiply v0
683+ add_member32 v4.8h, v0.h[1], v1.h[0], v1.h[3], v2.h[2], +, +, +, +, 2
684+ add_member32 v5.4h, v0.h[2], v1.h[3], v3.h[0], v3.h[2], +, +, +, -
685+ add_member32 v5.8h, v0.h[3], v2.h[2], v3.h[2], v1.h[3], +, +, -, -, 2
686+ add_member32 v6.4h, v1.h[0], v3.h[1], v2.h[1], v0.h[0], +, +, -, -
687+ add_member32 v6.8h, v1.h[1], v3.h[3], v1.h[0], v1.h[2], +, -, -, -, 2
688+ add_member32 v7.4h, v1.h[2], v3.h[0], v0.h[0], v3.h[1], +, -, -, -
689+ add_member32 v7.8h, v1.h[3], v2.h[1], v1.h[1], v2.h[3], +, -, -, +, 2
690+ add_member32 v16.4h, v2.h[0], v1.h[2], v2.h[2], v1.h[0], +, -, -, +
691+ add_member32 v16.8h, v2.h[1], v0.h[3], v3.h[3], v0.h[2], +, -, -, +, 2
692+ add_member32 v17.4h, v2.h[2], v0.h[1], v2.h[3], v2.h[1], +, -, +, +
693+ add_member32 v17.8h, v2.h[3], v0.h[2], v1.h[2], v3.h[3], +, -, +, -, 2
694+ add_member32 v18.4h, v3.h[0], v1.h[1], v0.h[1], v2.h[0], +, -, +, -
695+ add_member32 v18.8h, v3.h[1], v2.h[0], v0.h[3], v0.h[1], +, -, +, -, 2
696+ add_member32 v19.4h, v3.h[2], v2.h[3], v2.h[0], v1.h[1], +, -, +, -
697+ add_member32 v19.8h, v3.h[3], v3.h[2], v3.h[1], v3.h[0], +, -, +, -, 2
698+ ret
699+endfunc
700+
701+function tr_block2
702+ multiply v1
703+ add_member32 v4.8h, v3.h[1], v3.h[3], v3.h[0], v2.h[1], +, -, -, -, 2
704+ add_member32 v5.4h, v2.h[1], v1.h[0], v0.h[0], v1.h[1], -, -, -, -
705+ add_member32 v5.8h, v0.h[0], v1.h[2], v3.h[1], v2.h[3], -, -, -, +, 2
706+ add_member32 v6.4h, v2.h[0], v3.h[2], v1.h[1], v0.h[3], -, +, +, +
707+ add_member32 v6.8h, v3.h[2], v0.h[3], v1.h[3], v3.h[1], +, +, +, -, 2
708+ add_member32 v7.4h, v1.h[1], v1.h[3], v2.h[3], v0.h[0], +, +, -, -
709+ add_member32 v7.8h, v0.h[3], v3.h[1], v0.h[1], v3.h[3], +, -, -, +, 2
710+ add_member32 v16.4h, v3.h[0], v0.h[2], v3.h[2], v0.h[1], +, -, -, +
711+ add_member32 v16.8h, v2.h[2], v2.h[0], v1.h[0], v3.h[2], -, -, +, +, 2
712+ add_member32 v17.4h, v0.h[1], v3.h[0], v2.h[0], v0.h[2], -, +, +, -
713+ add_member32 v17.8h, v1.h[3], v0.h[1], v2.h[2], v3.h[0], -, +, -, -, 2
714+ add_member32 v18.4h, v3.h[3], v2.h[1], v0.h[2], v1.h[0], +, +, -, +
715+ add_member32 v18.8h, v1.h[2], v2.h[3], v3.h[3], v2.h[2], +, -, -, +, 2
716+ add_member32 v19.4h, v0.h[2], v0.h[1], v0.h[3], v1.h[2], +, -, +, -
717+ add_member32 v19.8h, v2.h[3], v2.h[2], v2.h[1], v2.h[0], +, -, +, -, 2
718+ ret
719+endfunc
720+
721+function tr_block3
722+ multiply v2
723+ add_member32 v4.8h, v1.h[2], v0.h[3], v0.h[0], v0.h[2], -, -, -, -, 2
724+ add_member32 v5.4h, v2.h[2], v3.h[3], v2.h[3], v1.h[2], -, -, +, +
725+ add_member32 v5.8h, v1.h[0], v0.h[2], v2.h[1], v3.h[3], +, +, +, -, 2
726+ add_member32 v6.4h, v3.h[0], v2.h[2], v0.h[1], v1.h[3], +, -, -, -
727+ add_member32 v6.8h, v0.h[2], v2.h[0], v3.h[0], v0.h[0], -, -, +, +, 2
728+ add_member32 v7.4h, v3.h[2], v1.h[0], v2.h[0], v2.h[2], -, +, +, -
729+ add_member32 v7.8h, v0.h[0], v3.h[2], v0.h[2], v3.h[0], +, +, -, -, 2
730+ add_member32 v16.4h, v3.h[3], v0.h[1], v3.h[1], v0.h[3], -, -, +, +
731+ add_member32 v16.8h, v0.h[1], v2.h[3], v1.h[3], v1.h[1], -, +, +, -, 2
732+ add_member32 v17.4h, v3.h[1], v1.h[3], v0.h[3], v3.h[2], +, +, -, +
733+ add_member32 v17.8h, v0.h[3], v1.h[1], v3.h[2], v2.h[0], +, -, +, +, 2
734+ add_member32 v18.4h, v2.h[3], v3.h[1], v1.h[2], v0.h[1], -, -, +, -
735+ add_member32 v18.8h, v1.h[1], v0.h[0], v1.h[0], v2.h[1], -, +, -, +, 2
736+ add_member32 v19.4h, v2.h[1], v3.h[0], v3.h[3], v3.h[1], +, -, +, +
737+ add_member32 v19.8h, v1.h[3], v1.h[2], v1.h[1], v1.h[0], +, -, +, -, 2
738+ ret
739+endfunc
740+
741+function tr_block4
742+ multiply v3
743+ add_member32 v4.8h, v1.h[1], v2.h[0], v2.h[3], v3.h[2], -, -, -, -, 2
744+ add_member32 v5.4h, v0.h[0], v0.h[3], v2.h[0], v3.h[1], +, +, +, +
745+ add_member32 v5.8h, v2.h[0], v0.h[0], v1.h[1], v3.h[0], -, -, -, -, 2
746+ add_member32 v6.4h, v3.h[3], v1.h[2], v0.h[2], v2.h[3], +, +, +, +
747+ add_member32 v6.8h, v2.h[1], v2.h[3], v0.h[0], v2.h[2], +, -, -, -, 2
748+ add_member32 v7.4h, v0.h[2], v3.h[3], v0.h[3], v2.h[1], -, -, +, +
749+ add_member32 v7.8h, v1.h[0], v2.h[2], v1.h[2], v2.h[0], +, +, -, -, 2
750+ add_member32 v16.4h, v2.h[3], v1.h[1], v2.h[1], v1.h[3], -, -, +, +
751+ add_member32 v16.8h, v3.h[1], v0.h[1], v3.h[0], v1.h[2], -, +, -, -, 2
752+ add_member32 v17.4h, v1.h[2], v1.h[0], v3.h[3], v1.h[1], +, -, +, +
753+ add_member32 v17.8h, v0.h[1], v2.h[1], v3.h[1], v1.h[0], -, +, +, -, 2
754+ add_member32 v18.4h, v1.h[3], v3.h[2], v2.h[2], v0.h[3], +, -, -, +
755+ add_member32 v18.8h, v3.h[2], v3.h[0], v1.h[3], v0.h[2], -, -, +, -, 2
756+ add_member32 v19.4h, v2.h[2], v1.h[3], v1.h[0], v0.h[1], -, +, -, +
757+ add_member32 v19.8h, v0.h[3], v0.h[2], v0.h[1], v0.h[0], +, -, +, -, 2
758+ ret
759+endfunc
760+
761+.macro tr_32x4 name, shift
762+function func_tr_32x4_\name
763+ mov x10, x30
764+ bl func_tr_16x4_noscale
765+
766+ load32
767+ movrel x9, trans, 32
768+ ld1 {v0.4h-v1.4h}, [x9], #16
769+ ld1 {v2.4h-v3.4h}, [x9]
770+ add x4, sp, #2048
771+ mov x2, #64
772+ mov x8, #-64
773+
774+ bl tr_block1
775+ mov x1, x11
776+ add x3, x11, #(56 + 3 * 64)
777+ scale_store \shift
778+
779+ bl tr_block2
780+ add x1, x11, #8
781+ add x3, x11, #(48 + 3 * 64)
782+ scale_store \shift
783+
784+ bl tr_block3
785+ add x1, x11, #16
786+ add x3, x11, #(40 + 3 * 64)
787+ scale_store \shift
788+
789+ bl tr_block4
790+ add x1, x11, #24
791+ add x3, x11, #(32 + 3 * 64)
792+ scale_store \shift
793+
794+ br x10
795+endfunc
796+.endm
797+
798+.macro idct_32x32 bitdepth
799+function ff_hevc_idct_32x32_\bitdepth\()_neon, export=1
800+ mov x15, x30
801+ // allocate a temp buffer
802+ sub sp, sp, #2432
803+
804+.irp i, 0, 1, 2, 3, 4, 5, 6, 7
805+ add x5, x0, #(8 * \i)
806+ add x11, sp, #(8 * \i * 32)
807+ bl func_tr_32x4_firstpass
808+.endr
809+
810+.irp i, 0, 1, 2, 3, 4, 5, 6, 7
811+ add x5, sp, #(8 * \i)
812+ add x11, x0, #(8 * \i * 32)
813+ bl func_tr_32x4_secondpass_\bitdepth
814+.endr
815+
816+ add sp, sp, #2432
817+ mov x30, x15
818+ ret
819+endfunc
820+.endm
821+
599822 idct_4x4 8
600823 idct_4x4 10
601824
@@ -605,10 +828,20 @@ idct_8x8 10 · 3 unmodified lines
605828 tr_16x4 firstpass, 7, 512, 1
606829 tr_16x4 secondpass_8, 20 - 8, 512, 1
607830 tr_16x4 secondpass_10, 20 - 10, 512, 1
831+tr_16x4 noscale, 0, 2048, 4
608832
609833 idct_16x16 8
610834 idct_16x16 10
611835
836+.ltorg
837+tr_32x4 firstpass, 7
838+tr_32x4 secondpass_8, 20 - 8
839+tr_32x4 secondpass_10, 20 - 10
840+.ltorg
841+
842+idct_32x32 8
843+idct_32x32 10
844+
612845 // void ff_hevc_idct_NxN_dc_DEPTH_neon(int16_t *coeffs)
613846 .macro idct_dc size, bitdepth
614847 function ff_hevc_idct_\size\()x\size\()_dc_\bitdepth\()_neon, export=1
libavcodec/aarch64/hevcdsp_init_aarch64.c+5−0
@@ -1,5 +1,6 @@
11 /*
22 * Copyright (c) 2020 Reimar Döffinger
3+ * Copyright (c) 2023 xu fulong <839789740@qq.com>
34 *
45 * This file is part of FFmpeg.
56 *
@@ -67,6 +68,8 @@ void ff_hevc_idct_8x8_8_neon(int16_t *coeffs, int col_limit); · 61 unmodified lines
6768 void ff_hevc_idct_8x8_10_neon(int16_t *coeffs, int col_limit);
6869 void ff_hevc_idct_16x16_8_neon(int16_t *coeffs, int col_limit);
6970 void ff_hevc_idct_16x16_10_neon(int16_t *coeffs, int col_limit);
71+void ff_hevc_idct_32x32_8_neon(int16_t *coeffs, int col_limit);
72+void ff_hevc_idct_32x32_10_neon(int16_t *coeffs, int col_limit);
7073 void ff_hevc_idct_4x4_dc_8_neon(int16_t *coeffs);
7174 void ff_hevc_idct_8x8_dc_8_neon(int16_t *coeffs);
7275 void ff_hevc_idct_16x16_dc_8_neon(int16_t *coeffs);
@@ -138,6 +141,7 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) · 65 unmodified lines
138141 c->idct[0] = ff_hevc_idct_4x4_8_neon;
139142 c->idct[1] = ff_hevc_idct_8x8_8_neon;
140143 c->idct[2] = ff_hevc_idct_16x16_8_neon;
144+ c->idct[3] = ff_hevc_idct_32x32_8_neon;
141145 c->idct_dc[0] = ff_hevc_idct_4x4_dc_8_neon;
142146 c->idct_dc[1] = ff_hevc_idct_8x8_dc_8_neon;
143147 c->idct_dc[2] = ff_hevc_idct_16x16_dc_8_neon;
@@ -190,6 +194,7 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) · 46 unmodified lines
190194 c->idct[0] = ff_hevc_idct_4x4_10_neon;
191195 c->idct[1] = ff_hevc_idct_8x8_10_neon;
192196 c->idct[2] = ff_hevc_idct_16x16_10_neon;
197+ c->idct[3] = ff_hevc_idct_32x32_10_neon;
193198 c->idct_dc[0] = ff_hevc_idct_4x4_dc_10_neon;
194199 c->idct_dc[1] = ff_hevc_idct_8x8_dc_10_neon;
195200 c->idct_dc[2] = ff_hevc_idct_16x16_dc_10_neon;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t