History

Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Will Deacon: "There's a reasonable amount here and the juicy details are all below. It's worth noting that the MTE/KASAN changes strayed outside of our usual directories due to core mm changes and some associated changes to some other architectures; Andrew asked for us to carry these [1] rather that take them via the -mm tree. Summary: - Optimise SVE switching for CPUs with 128-bit implementations. - Fix output format from SVE selftest. - Add support for versions v1.2 and 1.3 of the SMC calling convention. - Allow Pointer Authentication to be configured independently for kernel and userspace. - PMU driver cleanups for managing IRQ affinity and exposing event attributes via sysfs. - KASAN optimisations for both hardware tagging (MTE) and out-of-line software tagging implementations. - Relax frame record alignment requirements to facilitate 8-byte alignment with KASAN and Clang. - Cleanup of page-table definitions and removal of unused memory types. - Reduction of ARCH_DMA_MINALIGN back to 64 bytes. - Refactoring of our instruction decoding routines and addition of some missing encodings. - Move entry code moved into C and hardened against harmful compiler instrumentation. - Update booting requirements for the FEAT_HCX feature, added to v8.7 of the architecture. - Fix resume from idle when pNMI is being used. - Additional CPU sanity checks for MTE and preparatory changes for systems where not all of the CPUs support 32-bit EL0. - Update our kernel string routines to the latest Cortex Strings implementation. - Big cleanup of our cache maintenance routines, which were confusingly named and inconsistent in their implementations. - Tweak linker flags so that GDB can understand vmlinux when using RELR relocations. - Boot path cleanups to enable early initialisation of per-cpu operations needed by KCSAN. - Non-critical fixes and miscellaneous cleanup" * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (150 commits) arm64: tlb: fix the TTL value of tlb_get_level arm64: Restrict undef hook for cpufeature registers arm64/mm: Rename ARM64_SWAPPER_USES_SECTION_MAPS arm64: insn: avoid circular include dependency arm64: smp: Bump debugging information print down to KERN_DEBUG drivers/perf: fix the missed ida_simple_remove() in ddr_perf_probe() perf/arm-cmn: Fix invalid pointer when access dtc object sharing the same IRQ number arm64: suspend: Use cpuidle context helpers in cpu_suspend() PSCI: Use cpuidle context helpers in psci_cpu_suspend_enter() arm64: Convert cpu_do_idle() to using cpuidle context helpers arm64: Add cpuidle context save/restore helpers arm64: head: fix code comments in set_cpu_boot_mode_flag arm64: mm: drop unused __pa(__idmap_text_start) arm64: mm: fix the count comments in compute_indices arm64/mm: Fix ttbr0 values stored in struct thread_info for software-pan arm64: mm: Pass original fault address to handle_mm_fault() arm64/mm: Drop SECTION_[SHIFT|SIZE|MASK] arm64/mm: Use CONT_PMD_SHIFT for ARM64_MEMSTART_SHIFT arm64/mm: Drop SWAPPER_INIT_MAP_SIZE arm64: Conditionally configure PTR_AUTH key of the kernel. ...

Linus Torvalds committed 5 years ago

Commit 9840cfc2 parents 17cfb9d 3d1bf78158 files · +3370 −2563Browse files

Unified · Split

158 files
  1. Documentation/arm64/booting.rst +6 −0
  2. Makefile +1 −1
  3. arch/alpha/include/asm/page.h +3 −3
  4. arch/arm/include/asm/cpuidle.h +5 −0
  5. arch/arm/kernel/perf_event_v7.c +2 −2
  6. arch/arm64/Kconfig +19 −14
  7. arch/arm64/Makefile +1 −1
  8. arch/arm64/include/asm/alternative-macros.h +1 −8
  9. arch/arm64/include/asm/arch_gicv3.h +2 −1
  10. arch/arm64/include/asm/asm-prototypes.h +6 −0
  11. arch/arm64/include/asm/asm_pointer_auth.h +29 −20
  12. arch/arm64/include/asm/assembler.h +61 −37
  13. arch/arm64/include/asm/cache.h +1 −1
  14. arch/arm64/include/asm/cacheflush.h +42 −29
  15. arch/arm64/include/asm/cpu.h +25 −20
  16. arch/arm64/include/asm/cpufeature.h +14 −1
  17. arch/arm64/include/asm/cpuidle.h +35 −0
  18. arch/arm64/include/asm/efi.h +1 −1
  19. arch/arm64/include/asm/exception.h +26 −8
  20. arch/arm64/include/asm/fpsimd.h +1 −1
  21. arch/arm64/include/asm/fpsimdmacros.h +3 −1
  22. arch/arm64/include/asm/insn-def.h +9 −0
  23. arch/arm64/include/asm/insn.h +61 −6
  24. arch/arm64/include/asm/kernel-pgtable.h +8 −11
  25. arch/arm64/include/asm/kvm_asm.h +1 −0
  26. arch/arm64/include/asm/kvm_mmu.h +4 −3
  27. arch/arm64/include/asm/linkage.h +8 −0
  28. arch/arm64/include/asm/memory.h +2 −4
  29. arch/arm64/include/asm/mmu_context.h +2 −2
  30. arch/arm64/include/asm/module.lds.h +15 −2
  31. arch/arm64/include/asm/mte-kasan.h +67 −26
  32. arch/arm64/include/asm/mte.h +4 −0
  33. arch/arm64/include/asm/page.h +7 −3
  34. arch/arm64/include/asm/patching.h +13 −0
  35. arch/arm64/include/asm/perf_event.h +5 −0
  36. arch/arm64/include/asm/pgtable-hwdef.h +0 −7
  37. arch/arm64/include/asm/pgtable-prot.h +0 −1
  38. arch/arm64/include/asm/pgtable.h +2 −3
  39. arch/arm64/include/asm/pointer_auth.h +33 −26
  40. arch/arm64/include/asm/processor.h +8 −8
  41. arch/arm64/include/asm/scs.h +4 −4
  42. arch/arm64/include/asm/sdei.h +7 −3
  43. arch/arm64/include/asm/smp.h +0 −2
  44. arch/arm64/include/asm/stacktrace.h +16 −16
  45. arch/arm64/include/asm/sysreg.h +0 −2
  46. arch/arm64/include/asm/tlb.h +4 −0
  47. arch/arm64/kernel/Makefile +9 −2
  48. arch/arm64/kernel/acpi.c +17 −5
  49. arch/arm64/kernel/alternative.c +1 −1
  50. arch/arm64/kernel/asm-offsets.c +14 −1
  51. arch/arm64/kernel/cpufeature.c +162 −48
  52. arch/arm64/kernel/cpuinfo.c +32 −26
  53. arch/arm64/kernel/efi-entry.S +5 −4
  54. arch/arm64/kernel/entry-common.c +238 −18
  55. arch/arm64/kernel/entry-fpsimd.S +15 −7
  56. arch/arm64/kernel/entry.S +96 −273
  57. arch/arm64/kernel/fpsimd.c +4 −2
  58. arch/arm64/kernel/ftrace.c +1 −0
  59. arch/arm64/kernel/head.S +44 −32
  60. arch/arm64/kernel/hibernate-asm.S +4 −3
  61. arch/arm64/kernel/hibernate.c +12 −8
  62. arch/arm64/kernel/idle.c +46 −0
  63. arch/arm64/kernel/idreg-override.c +2 −1
  64. arch/arm64/kernel/image-vars.h +1 −1
  65. arch/arm64/kernel/jump_label.c +1 −0
  66. arch/arm64/kernel/kaslr.c +9 −3
  67. arch/arm64/kernel/kgdb.c +1 −0
  68. arch/arm64/kernel/machine_kexec.c +20 −10
  69. arch/arm64/kernel/patching.c +150 −0
  70. arch/arm64/kernel/perf_callchain.c +1 −1
  71. arch/arm64/kernel/perf_event.c +35 −5
  72. arch/arm64/kernel/probes/kprobes.c +10 −8
  73. arch/arm64/kernel/probes/simulate-insn.c +1 −0
  74. arch/arm64/kernel/probes/uprobes.c +1 −1
  75. arch/arm64/kernel/process.c +23 −76
  76. arch/arm64/kernel/ptrace.c +1 −1
  77. arch/arm64/kernel/sdei.c +12 −52
  78. arch/arm64/kernel/setup.c +1 −7
  79. arch/arm64/kernel/signal.c +26 −0
  80. arch/arm64/kernel/smccc-call.S +83 −0
  81. arch/arm64/kernel/smp.c +7 −9
  82. arch/arm64/kernel/smp_spin_table.c +4 −3
  83. arch/arm64/kernel/stacktrace.c +8 −10
  84. arch/arm64/kernel/suspend.c +11 −1
  85. arch/arm64/kernel/sys_compat.c +1 −1
  86. arch/arm64/kernel/traps.c +100 −35
  87. arch/arm64/kvm/arm.c +11 −2
  88. arch/arm64/kvm/hyp/nvhe/cache.S +2 −2
  89. arch/arm64/kvm/hyp/nvhe/setup.c +2 −1
  90. arch/arm64/kvm/hyp/nvhe/tlb.c +1 −1
  91. arch/arm64/kvm/hyp/pgtable.c +10 −3
  92. arch/arm64/lib/Makefile +3 −1
  93. arch/arm64/lib/clear_user.S +27 −20
  94. arch/arm64/lib/insn.c +4 −245
  95. arch/arm64/lib/kasan_sw_tags.S +76 −0
  96. arch/arm64/lib/memchr.S +53 −12
  97. arch/arm64/lib/memcmp.S +119 −227
  98. arch/arm64/lib/memcpy.S +229 −43
  99. arch/arm64/lib/memmove.S +0 −189
  100. arch/arm64/lib/mte.S +20 −0
  101. arch/arm64/lib/strcmp.S +121 −168
  102. arch/arm64/lib/strlen.S +173 −85
  103. arch/arm64/lib/strncmp.S +184 −222
  104. arch/arm64/lib/uaccess_flushcache.c +2 −2
  105. arch/arm64/mm/cache.S +81 −77
  106. arch/arm64/mm/context.c +2 −4
  107. arch/arm64/mm/fault.c +37 −13
  108. arch/arm64/mm/flush.c +14 −15
  109. arch/arm64/mm/init.c +7 −0
  110. arch/arm64/mm/mmu.c +7 −6
  111. arch/arm64/mm/proc.S +9 −15
  112. arch/arm64/mm/ptdump.c +0 −4
  113. arch/arm64/net/bpf_jit_comp.c +1 −0
  114. arch/arm64/tools/cpucaps +2 −1
  115. arch/ia64/include/asm/page.h +3 −3
  116. arch/m68k/include/asm/page_no.h +3 −3
  117. arch/s390/include/asm/page.h +3 −3
  118. arch/x86/include/asm/page.h +3 −3
  119. drivers/firmware/psci/psci.c +7 −2
  120. drivers/firmware/smccc/smccc.c +4 −0
  121. drivers/misc/lkdtm/bugs.c +3 −3
  122. drivers/perf/arm-cci.c +2 −2
  123. drivers/perf/arm-ccn.c +2 −4
  124. drivers/perf/arm-cmn.c +4 −9
  125. drivers/perf/arm_dmc620_pmu.c +2 −3
  126. drivers/perf/arm_dsu_pmu.c +2 −6
  127. drivers/perf/arm_pmu.c +6 −10
  128. drivers/perf/arm_smmuv3_pmu.c +15 −20
  129. drivers/perf/arm_spe_pmu.c +4 −8
  130. drivers/perf/fsl_imx8_ddr_perf.c +8 −10
  131. drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c +1 −4
  132. drivers/perf/hisilicon/hisi_uncore_hha_pmu.c +3 −6
  133. drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c +1 −4
  134. drivers/perf/hisilicon/hisi_uncore_pa_pmu.c +1 −4
  135. drivers/perf/hisilicon/hisi_uncore_pmu.c +3 −3
  136. drivers/perf/hisilicon/hisi_uncore_pmu.h +1 −1
  137. drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c +2 −5
  138. drivers/perf/qcom_l2_pmu.c +4 −7
  139. drivers/perf/qcom_l3_pmu.c +5 −10
  140. drivers/perf/thunderx2_pmu.c +1 −3
  141. drivers/perf/xgene_pmu.c +7 −10
  142. include/linux/arm-smccc.h +86 −2
  143. include/linux/gfp.h +15 −3
  144. include/linux/highmem.h +16 −27
  145. include/linux/interrupt.h +2 −33
  146. include/linux/kasan.h +38 −26
  147. include/linux/page-flags.h +9 −0
  148. include/linux/perf_event.h +6 −0
  149. include/trace/events/mmflags.h +8 −1
  150. kernel/irq/manage.c +32 −1
  151. mm/kasan/common.c +2 −2
  152. mm/kasan/hw_tags.c +32 −0
  153. mm/kasan/sw_tags.c +7 −0
  154. mm/mempool.c +4 −2
  155. mm/page_alloc.c +39 −27
  156. scripts/Makefile.kasan +1 −0
  157. scripts/tools-support-relr.sh +2 −1
  158. tools/testing/selftests/arm64/fp/sve-probe-vls.c +1 −1
Documentation/arm64/booting.rst+6−0
@@ -277,6 +277,12 @@ Before jumping into the kernel, the following conditions must be met: · 276 unmodified lines
277277
278278 - SCR_EL3.FGTEn (bit 27) must be initialised to 0b1.
279279
280+ For CPUs with support for HCRX_EL2 (FEAT_HCX) present:
281+
282+ - If EL3 is present and the kernel is entered at EL2:
283+
284+ - SCR_EL3.HXEn (bit 38) must be initialised to 0b1.
285+
280286 For CPUs with Advanced SIMD and floating point support:
281287
282288 - If EL3 is present:
Makefile+1−1
@@ -1039,7 +1039,7 @@ LDFLAGS_vmlinux += $(call ld-option, -X,) · 1038 unmodified lines
10391039 endif
10401040
10411041 ifeq ($(CONFIG_RELR),y)
1042−LDFLAGS_vmlinux += --pack-dyn-relocs=relr
1042+LDFLAGS_vmlinux += --pack-dyn-relocs=relr --use-android-relr-tags
10431043 endif
10441044
10451045 # We never want expected sections to be placed heuristically by the
arch/alpha/include/asm/page.h+3−3
@@ -17,9 +17,9 @@ · 16 unmodified lines
1717 extern void clear_page(void *page);
1818 #define clear_user_page(page, vaddr, pg) clear_page(page)
1919
20−#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \
21− alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vmaddr)
22−#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
20+#define alloc_zeroed_user_highpage_movable(vma, vaddr) \
21+ alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vmaddr)
22+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE
2323
2424 extern void copy_page(void * _to, void * _from);
2525 #define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
arch/arm/include/asm/cpuidle.h+5−0
@@ -50,4 +50,9 @@ extern int arm_cpuidle_suspend(int index); · 49 unmodified lines
5050
5151 extern int arm_cpuidle_init(int cpu);
5252
53+struct arm_cpuidle_irq_context { };
54+
55+#define arm_cpuidle_save_irq_context(c) (void)c
56+#define arm_cpuidle_restore_irq_context(c) (void)c
57+
5358 #endif
arch/arm/kernel/perf_event_v7.c+2−2
@@ -773,10 +773,10 @@ static inline void armv7pmu_write_counter(struct perf_event *event, u64 value) · 772 unmodified lines
773773 pr_err("CPU%u writing wrong counter %d\n",
774774 smp_processor_id(), idx);
775775 } else if (idx == ARMV7_IDX_CYCLE_COUNTER) {
776− asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value));
776+ asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" ((u32)value));
777777 } else {
778778 armv7_pmnc_select_counter(idx);
779− asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value));
779+ asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" ((u32)value));
780780 }
781781 }
782782
arch/arm64/Kconfig+19−14
@@ -1481,12 +1481,6 @@ menu "ARMv8.3 architectural features" · 1480 unmodified lines
14811481 config ARM64_PTR_AUTH
14821482 bool "Enable support for pointer authentication"
14831483 default y
1484− depends on (CC_HAS_SIGN_RETURN_ADDRESS || CC_HAS_BRANCH_PROT_PAC_RET) && AS_HAS_PAC
1485− # Modern compilers insert a .note.gnu.property section note for PAC
1486− # which is only understood by binutils starting with version 2.33.1.
1487− depends on LD_IS_LLD || LD_VERSION >= 23301 || (CC_IS_GCC && GCC_VERSION < 90100)
1488− depends on !CC_IS_CLANG || AS_HAS_CFI_NEGATE_RA_STATE
1489− depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS)
14901484 help
14911485 Pointer authentication (part of the ARMv8.3 Extensions) provides
14921486 instructions for signing and authenticating pointers against secret
@@ -1498,13 +1492,6 @@ config ARM64_PTR_AUTH · 5 unmodified lines
14981492 for each process at exec() time, with these keys being
14991493 context-switched along with the process.
15001494
1501− If the compiler supports the -mbranch-protection or
1502− -msign-return-address flag (e.g. GCC 7 or later), then this option
1503− will also cause the kernel itself to be compiled with return address
1504− protection. In this case, and if the target hardware is known to
1505− support pointer authentication, then CONFIG_STACKPROTECTOR can be
1506− disabled with minimal loss of protection.
1507−
15081495 The feature is detected at runtime. If the feature is not present in
15091496 hardware it will not be advertised to userspace/KVM guest nor will it
15101497 be enabled.
@@ -1515,6 +1502,24 @@ config ARM64_PTR_AUTH · 4 unmodified lines
15151502 but with the feature disabled. On such a system, this option should
15161503 not be selected.
15171504
1505+config ARM64_PTR_AUTH_KERNEL
1506+ bool "Use pointer authentication for kernel"
1507+ default y
1508+ depends on ARM64_PTR_AUTH
1509+ depends on (CC_HAS_SIGN_RETURN_ADDRESS || CC_HAS_BRANCH_PROT_PAC_RET) && AS_HAS_PAC
1510+ # Modern compilers insert a .note.gnu.property section note for PAC
1511+ # which is only understood by binutils starting with version 2.33.1.
1512+ depends on LD_IS_LLD || LD_VERSION >= 23301 || (CC_IS_GCC && GCC_VERSION < 90100)
1513+ depends on !CC_IS_CLANG || AS_HAS_CFI_NEGATE_RA_STATE
1514+ depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS)
1515+ help
1516+ If the compiler supports the -mbranch-protection or
1517+ -msign-return-address flag (e.g. GCC 7 or later), then this option
1518+ will cause the kernel itself to be compiled with return address
1519+ protection. In this case, and if the target hardware is known to
1520+ support pointer authentication, then CONFIG_STACKPROTECTOR can be
1521+ disabled with minimal loss of protection.
1522+
15181523 This feature works with FUNCTION_GRAPH_TRACER option only if
15191524 DYNAMIC_FTRACE_WITH_REGS is enabled.
15201525
@@ -1606,7 +1611,7 @@ config ARM64_BTI_KERNEL · 85 unmodified lines
16061611 bool "Use Branch Target Identification for kernel"
16071612 default y
16081613 depends on ARM64_BTI
1609− depends on ARM64_PTR_AUTH
1614+ depends on ARM64_PTR_AUTH_KERNEL
16101615 depends on CC_HAS_BRANCH_PROT_PAC_RET_BTI
16111616 # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94697
16121617 depends on !CC_IS_GCC || GCC_VERSION >= 100100
arch/arm64/Makefile+1−1
@@ -70,7 +70,7 @@ endif · 69 unmodified lines
7070 # off, this will be overridden if we are using branch protection.
7171 branch-prot-flags-y += $(call cc-option,-mbranch-protection=none)
7272
73−ifeq ($(CONFIG_ARM64_PTR_AUTH),y)
73+ifeq ($(CONFIG_ARM64_PTR_AUTH_KERNEL),y)
7474 branch-prot-flags-$(CONFIG_CC_HAS_SIGN_RETURN_ADDRESS) := -msign-return-address=all
7575 # We enable additional protection for leaf functions as there is some
7676 # narrow potential for ROP protection benefits and no substantial
arch/arm64/include/asm/alternative-macros.h+1−8
@@ -3,12 +3,10 @@ · 2 unmodified lines
33 #define __ASM_ALTERNATIVE_MACROS_H
44
55 #include <asm/cpucaps.h>
6+#include <asm/insn-def.h>
67
78 #define ARM64_CB_PATCH ARM64_NCAPS
89
9−/* A64 instructions are always 32 bits. */
10−#define AARCH64_INSN_SIZE 4
11−
1210 #ifndef __ASSEMBLY__
1311
1412 #include <linux/stringify.h>
@@ -197,11 +195,6 @@ alternative_endif · 182 unmodified lines
197195 #define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \
198196 alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
199197
200−.macro user_alt, label, oldinstr, newinstr, cond
201−9999: alternative_insn "\oldinstr", "\newinstr", \cond
202− _asm_extable 9999b, \label
203−.endm
204−
205198 #endif /* __ASSEMBLY__ */
206199
207200 /*
arch/arm64/include/asm/arch_gicv3.h+2−1
@@ -124,7 +124,8 @@ static inline u32 gic_read_rpr(void) · 123 unmodified lines
124124 #define gic_read_lpir(c) readq_relaxed(c)
125125 #define gic_write_lpir(v, c) writeq_relaxed(v, c)
126126
127−#define gic_flush_dcache_to_poc(a,l) __flush_dcache_area((a), (l))
127+#define gic_flush_dcache_to_poc(a,l) \
128+ dcache_clean_inval_poc((unsigned long)(a), (unsigned long)(a)+(l))
128129
129130 #define gits_read_baser(c) readq_relaxed(c)
130131 #define gits_write_baser(v, c) writeq_relaxed(v, c)
arch/arm64/include/asm/asm-prototypes.h+6−0
@@ -23,4 +23,10 @@ long long __ashlti3(long long a, int b); · 22 unmodified lines
2323 long long __ashrti3(long long a, int b);
2424 long long __lshrti3(long long a, int b);
2525
26+/*
27+ * This function uses a custom calling convention and cannot be called from C so
28+ * this prototype is not entirely accurate.
29+ */
30+void __hwasan_tag_mismatch(unsigned long addr, unsigned long access_info);
31+
2632 #endif /* __ASM_PROTOTYPES_H */
arch/arm64/include/asm/asm_pointer_auth.h+29−20
@@ -7,19 +7,7 @@ · 6 unmodified lines
77 #include <asm/cpufeature.h>
88 #include <asm/sysreg.h>
99
10−#ifdef CONFIG_ARM64_PTR_AUTH
11−/*
12− * thread.keys_user.ap* as offset exceeds the #imm offset range
13− * so use the base value of ldp as thread.keys_user and offset as
14− * thread.keys_user.ap*.
15− */
16− .macro __ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3
17− mov \tmp1, #THREAD_KEYS_USER
18− add \tmp1, \tsk, \tmp1
19− ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIA]
20− msr_s SYS_APIAKEYLO_EL1, \tmp2
21− msr_s SYS_APIAKEYHI_EL1, \tmp3
22− .endm
10+#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
2311
2412 .macro __ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3
2513 mov \tmp1, #THREAD_KEYS_KERNEL
@@ -42,6 +30,33 @@ alternative_if ARM64_HAS_ADDRESS_AUTH · 16 unmodified lines
4230 alternative_else_nop_endif
4331 .endm
4432
33+#else /* CONFIG_ARM64_PTR_AUTH_KERNEL */
34+
35+ .macro __ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3
36+ .endm
37+
38+ .macro ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3
39+ .endm
40+
41+ .macro ptrauth_keys_install_kernel tsk, tmp1, tmp2, tmp3
42+ .endm
43+
44+#endif /* CONFIG_ARM64_PTR_AUTH_KERNEL */
45+
46+#ifdef CONFIG_ARM64_PTR_AUTH
47+/*
48+ * thread.keys_user.ap* as offset exceeds the #imm offset range
49+ * so use the base value of ldp as thread.keys_user and offset as
50+ * thread.keys_user.ap*.
51+ */
52+ .macro __ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3
53+ mov \tmp1, #THREAD_KEYS_USER
54+ add \tmp1, \tsk, \tmp1
55+ ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIA]
56+ msr_s SYS_APIAKEYLO_EL1, \tmp2
57+ msr_s SYS_APIAKEYHI_EL1, \tmp3
58+ .endm
59+
4560 .macro __ptrauth_keys_init_cpu tsk, tmp1, tmp2, tmp3
4661 mrs \tmp1, id_aa64isar1_el1
4762 ubfx \tmp1, \tmp1, #ID_AA64ISAR1_APA_SHIFT, #8
@@ -64,17 +79,11 @@ alternative_else_nop_endif · 16 unmodified lines
6479 .Lno_addr_auth\@:
6580 .endm
6681
67−#else /* CONFIG_ARM64_PTR_AUTH */
82+#else /* !CONFIG_ARM64_PTR_AUTH */
6883
6984 .macro ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3
7085 .endm
7186
72− .macro ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3
73− .endm
74−
75− .macro ptrauth_keys_install_kernel tsk, tmp1, tmp2, tmp3
76− .endm
77−
7887 #endif /* CONFIG_ARM64_PTR_AUTH */
7988
8089 #endif /* __ASM_ASM_POINTER_AUTH_H */
arch/arm64/include/asm/assembler.h+61−37
@@ -130,15 +130,27 @@ alternative_endif · 129 unmodified lines
130130 .endm
131131
132132 /*
133− * Emit an entry into the exception table
133+ * Create an exception table entry for `insn`, which will branch to `fixup`
134+ * when an unhandled fault is taken.
134135 */
135− .macro _asm_extable, from, to
136+ .macro _asm_extable, insn, fixup
136137 .pushsection __ex_table, "a"
137138 .align 3
138− .long (\from - .), (\to - .)
139+ .long (\insn - .), (\fixup - .)
139140 .popsection
140141 .endm
141142
143+/*
144+ * Create an exception table entry for `insn` if `fixup` is provided. Otherwise
145+ * do nothing.
146+ */
147+ .macro _cond_extable, insn, fixup
148+ .ifnc \fixup,
149+ _asm_extable \insn, \fixup
150+ .endif
151+ .endm
152+
153+
142154 #define USER(l, x...) \
143155 9999: x; \
144156 _asm_extable 9999b, l
@@ -232,15 +244,23 @@ lr .req x30 // link register · 87 unmodified lines
232244 * @dst: destination register
233245 */
234246 #if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__)
235− .macro this_cpu_offset, dst
247+ .macro get_this_cpu_offset, dst
236248 mrs \dst, tpidr_el2
237249 .endm
238250 #else
239− .macro this_cpu_offset, dst
251+ .macro get_this_cpu_offset, dst
240252 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
241253 mrs \dst, tpidr_el1
242254 alternative_else
243255 mrs \dst, tpidr_el2
256+alternative_endif
257+ .endm
258+
259+ .macro set_this_cpu_offset, src
260+alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
261+ msr tpidr_el1, \src
262+alternative_else
263+ msr tpidr_el2, \src
244264 alternative_endif
245265 .endm
246266 #endif
@@ -253,7 +273,7 @@ alternative_endif · 6 unmodified lines
253273 .macro adr_this_cpu, dst, sym, tmp
254274 adrp \tmp, \sym
255275 add \dst, \tmp, #:lo12:\sym
256− this_cpu_offset \tmp
276+ get_this_cpu_offset \tmp
257277 add \dst, \dst, \tmp
258278 .endm
259279
@@ -264,7 +284,7 @@ alternative_endif · 4 unmodified lines
264284 */
265285 .macro ldr_this_cpu dst, sym, tmp
266286 adr_l \dst, \sym
267− this_cpu_offset \tmp
287+ get_this_cpu_offset \tmp
268288 ldr \dst, [\dst, \tmp]
269289 .endm
270290
@@ -375,51 +395,53 @@ alternative_cb_end · 104 unmodified lines
375395 bfi \tcr, \tmp0, \pos, #3
376396 .endm
377397
398+ .macro __dcache_op_workaround_clean_cache, op, addr
399+alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
400+ dc \op, \addr
401+alternative_else
402+ dc civac, \addr
403+alternative_endif
404+ .endm
405+
378406 /*
379407 * Macro to perform a data cache maintenance for the interval
380− * [kaddr, kaddr + size)
408+ * [start, end)
381409 *
382410 * op: operation passed to dc instruction
383411 * domain: domain used in dsb instruciton
384− * kaddr: starting virtual address of the region
385− * size: size of the region
386− * Corrupts: kaddr, size, tmp1, tmp2
412+ * start: starting virtual address of the region
413+ * end: end virtual address of the region
414+ * fixup: optional label to branch to on user fault
415+ * Corrupts: start, end, tmp1, tmp2
387416 */
388− .macro __dcache_op_workaround_clean_cache, op, kaddr
389−alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
390− dc \op, \kaddr
391−alternative_else
392− dc civac, \kaddr
393−alternative_endif
394− .endm
395−
396− .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
417+ .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup
397418 dcache_line_size \tmp1, \tmp2
398− add \size, \kaddr, \size
399419 sub \tmp2, \tmp1, #1
400− bic \kaddr, \kaddr, \tmp2
401−9998:
420+ bic \start, \start, \tmp2
421+.Ldcache_op\@:
402422 .ifc \op, cvau
403− __dcache_op_workaround_clean_cache \op, \kaddr
423+ __dcache_op_workaround_clean_cache \op, \start
404424 .else
405425 .ifc \op, cvac
406− __dcache_op_workaround_clean_cache \op, \kaddr
426+ __dcache_op_workaround_clean_cache \op, \start
407427 .else
408428 .ifc \op, cvap
409− sys 3, c7, c12, 1, \kaddr // dc cvap
429+ sys 3, c7, c12, 1, \start // dc cvap
410430 .else
411431 .ifc \op, cvadp
412− sys 3, c7, c13, 1, \kaddr // dc cvadp
432+ sys 3, c7, c13, 1, \start // dc cvadp
413433 .else
414− dc \op, \kaddr
434+ dc \op, \start
415435 .endif
416436 .endif
417437 .endif
418438 .endif
419− add \kaddr, \kaddr, \tmp1
420− cmp \kaddr, \size
421− b.lo 9998b
439+ add \start, \start, \tmp1
440+ cmp \start, \end
441+ b.lo .Ldcache_op\@
422442 dsb \domain
443+
444+ _cond_extable .Ldcache_op\@, \fixup
423445 .endm
424446
425447 /*
@@ -427,20 +449,22 @@ alternative_endif · 1 unmodified line
427449 * [start, end)
428450 *
429451 * start, end: virtual addresses describing the region
430− * label: A label to branch to on user fault.
452+ * fixup: optional label to branch to on user fault
431453 * Corrupts: tmp1, tmp2
432454 */
433− .macro invalidate_icache_by_line start, end, tmp1, tmp2, label
455+ .macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup
434456 icache_line_size \tmp1, \tmp2
435457 sub \tmp2, \tmp1, #1
436458 bic \tmp2, \start, \tmp2
437−9997:
438−USER(\label, ic ivau, \tmp2) // invalidate I line PoU
459+.Licache_op\@:
460+ ic ivau, \tmp2 // invalidate I line PoU
439461 add \tmp2, \tmp2, \tmp1
440462 cmp \tmp2, \end
441− b.lo 9997b
463+ b.lo .Licache_op\@
442464 dsb ish
443465 isb
466+
467+ _cond_extable .Licache_op\@, \fixup
444468 .endm
445469
446470 /*
@@ -745,7 +769,7 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU · 298 unmodified lines
745769 cbz \tmp, \lbl
746770 #endif
747771 adr_l \tmp, irq_stat + IRQ_CPUSTAT_SOFTIRQ_PENDING
748− this_cpu_offset \tmp2
772+ get_this_cpu_offset \tmp2
749773 ldr w\tmp, [\tmp, \tmp2]
750774 cbnz w\tmp, \lbl // yield on pending softirq in task context
751775 .Lnoyield_\@:
arch/arm64/include/asm/cache.h+1−1
@@ -47,7 +47,7 @@ · 46 unmodified lines
4747 * cache before the transfer is done, causing old data to be seen by
4848 * the CPU.
4949 */
50−#define ARCH_DMA_MINALIGN (128)
50+#define ARCH_DMA_MINALIGN L1_CACHE_BYTES
5151
5252 #ifdef CONFIG_KASAN_SW_TAGS
5353 #define ARCH_SLAB_MINALIGN (1ULL << KASAN_SHADOW_SCALE_SHIFT)
arch/arm64/include/asm/cacheflush.h+42−29
@@ -30,45 +30,58 @@ · 29 unmodified lines
3030 * the implementation assumes non-aliasing VIPT D-cache and (aliasing)
3131 * VIPT I-cache.
3232 *
33− * flush_icache_range(start, end)
33+ * All functions below apply to the interval [start, end)
34+ * - start - virtual start address (inclusive)
35+ * - end - virtual end address (exclusive)
3436 *
35− * Ensure coherency between the I-cache and the D-cache in the
36− * region described by start, end.
37− * - start - virtual start address
38− * - end - virtual end address
37+ * caches_clean_inval_pou(start, end)
3938 *
40− * invalidate_icache_range(start, end)
39+ * Ensure coherency between the I-cache and the D-cache region to
40+ * the Point of Unification.
4141 *
42− * Invalidate the I-cache in the region described by start, end.
43− * - start - virtual start address
44− * - end - virtual end address
42+ * caches_clean_inval_user_pou(start, end)
4543 *
46− * __flush_cache_user_range(start, end)
44+ * Ensure coherency between the I-cache and the D-cache region to
45+ * the Point of Unification.
46+ * Use only if the region might access user memory.
4747 *
48− * Ensure coherency between the I-cache and the D-cache in the
49− * region described by start, end.
50− * - start - virtual start address
51− * - end - virtual end address
48+ * icache_inval_pou(start, end)
5249 *
53− * __flush_dcache_area(kaddr, size)
50+ * Invalidate I-cache region to the Point of Unification.
5451 *
55− * Ensure that the data held in page is written back.
56− * - kaddr - page address
57− * - size - region size
52+ * dcache_clean_inval_poc(start, end)
53+ *
54+ * Clean and invalidate D-cache region to the Point of Coherency.
55+ *
56+ * dcache_inval_poc(start, end)
57+ *
58+ * Invalidate D-cache region to the Point of Coherency.
59+ *
60+ * dcache_clean_poc(start, end)
61+ *
62+ * Clean D-cache region to the Point of Coherency.
63+ *
64+ * dcache_clean_pop(start, end)
65+ *
66+ * Clean D-cache region to the Point of Persistence.
67+ *
68+ * dcache_clean_pou(start, end)
69+ *
70+ * Clean D-cache region to the Point of Unification.
5871 */
59−extern void __flush_icache_range(unsigned long start, unsigned long end);
60−extern int invalidate_icache_range(unsigned long start, unsigned long end);
61−extern void __flush_dcache_area(void *addr, size_t len);
62−extern void __inval_dcache_area(void *addr, size_t len);
63−extern void __clean_dcache_area_poc(void *addr, size_t len);
64−extern void __clean_dcache_area_pop(void *addr, size_t len);
65−extern void __clean_dcache_area_pou(void *addr, size_t len);
66−extern long __flush_cache_user_range(unsigned long start, unsigned long end);
67−extern void sync_icache_aliases(void *kaddr, unsigned long len);
72+extern void caches_clean_inval_pou(unsigned long start, unsigned long end);
73+extern void icache_inval_pou(unsigned long start, unsigned long end);
74+extern void dcache_clean_inval_poc(unsigned long start, unsigned long end);
75+extern void dcache_inval_poc(unsigned long start, unsigned long end);
76+extern void dcache_clean_poc(unsigned long start, unsigned long end);
77+extern void dcache_clean_pop(unsigned long start, unsigned long end);
78+extern void dcache_clean_pou(unsigned long start, unsigned long end);
79+extern long caches_clean_inval_user_pou(unsigned long start, unsigned long end);
80+extern void sync_icache_aliases(unsigned long start, unsigned long end);
6881
6982 static inline void flush_icache_range(unsigned long start, unsigned long end)
7083 {
71− __flush_icache_range(start, end);
84+ caches_clean_inval_pou(start, end);
7285
7386 /*
7487 * IPI all online CPUs so that they undergo a context synchronization
@@ -122,7 +135,7 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *, · 47 unmodified lines
122135 #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1
123136 extern void flush_dcache_page(struct page *);
124137
125−static __always_inline void __flush_icache_all(void)
138+static __always_inline void icache_inval_all_pou(void)
126139 {
127140 if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC))
128141 return;
arch/arm64/include/asm/cpu.h+25−20
@@ -12,26 +12,7 @@ · 11 unmodified lines
1212 /*
1313 * Records attributes of an individual CPU.
1414 */
15−struct cpuinfo_arm64 {
16− struct cpu cpu;
17− struct kobject kobj;
18− u32 reg_ctr;
19− u32 reg_cntfrq;
20− u32 reg_dczid;
21− u32 reg_midr;
22− u32 reg_revidr;
23−
24− u64 reg_id_aa64dfr0;
25− u64 reg_id_aa64dfr1;
26− u64 reg_id_aa64isar0;
27− u64 reg_id_aa64isar1;
28− u64 reg_id_aa64mmfr0;
29− u64 reg_id_aa64mmfr1;
30− u64 reg_id_aa64mmfr2;
31− u64 reg_id_aa64pfr0;
32− u64 reg_id_aa64pfr1;
33− u64 reg_id_aa64zfr0;
34−
15+struct cpuinfo_32bit {
3516 u32 reg_id_dfr0;
3617 u32 reg_id_dfr1;
3718 u32 reg_id_isar0;
@@ -54,6 +35,30 @@ struct cpuinfo_arm64 { · 16 unmodified lines
5435 u32 reg_mvfr0;
5536 u32 reg_mvfr1;
5637 u32 reg_mvfr2;
38+};
39+
40+struct cpuinfo_arm64 {
41+ struct cpu cpu;
42+ struct kobject kobj;
43+ u64 reg_ctr;
44+ u64 reg_cntfrq;
45+ u64 reg_dczid;
46+ u64 reg_midr;
47+ u64 reg_revidr;
48+ u64 reg_gmid;
49+
50+ u64 reg_id_aa64dfr0;
51+ u64 reg_id_aa64dfr1;
52+ u64 reg_id_aa64isar0;
53+ u64 reg_id_aa64isar1;
54+ u64 reg_id_aa64mmfr0;
55+ u64 reg_id_aa64mmfr1;
56+ u64 reg_id_aa64mmfr2;
57+ u64 reg_id_aa64pfr0;
58+ u64 reg_id_aa64pfr1;
59+ u64 reg_id_aa64zfr0;
60+
61+ struct cpuinfo_32bit aarch32;
5762
5863 /* pseudo-ZCR for recording maximum ZCR_EL1 LEN value: */
5964 u64 reg_zcr;
arch/arm64/include/asm/cpufeature.h+14−1
@@ -619,6 +619,13 @@ static inline bool id_aa64pfr0_sve(u64 pfr0) · 618 unmodified lines
619619 return val > 0;
620620 }
621621
622+static inline bool id_aa64pfr1_mte(u64 pfr1)
623+{
624+ u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_MTE_SHIFT);
625+
626+ return val >= ID_AA64PFR1_MTE;
627+}
628+
622629 void __init setup_cpu_features(void);
623630 void check_local_cpu_capabilities(void);
624631
@@ -630,9 +637,15 @@ static inline bool cpu_supports_mixed_endian_el0(void) · 5 unmodified lines
630637 return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));
631638 }
632639
640+const struct cpumask *system_32bit_el0_cpumask(void);
641+DECLARE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0);
642+
633643 static inline bool system_supports_32bit_el0(void)
634644 {
635− return cpus_have_const_cap(ARM64_HAS_32BIT_EL0);
645+ u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
646+
647+ return static_branch_unlikely(&arm64_mismatched_32bit_el0) ||
648+ id_aa64pfr0_32bit_el0(pfr0);
636649 }
637650
638651 static inline bool system_supports_4kb_granule(void)
arch/arm64/include/asm/cpuidle.h+35−0
@@ -18,4 +18,39 @@ static inline int arm_cpuidle_suspend(int index) · 17 unmodified lines
1818 return -EOPNOTSUPP;
1919 }
2020 #endif
21+
22+#ifdef CONFIG_ARM64_PSEUDO_NMI
23+#include <asm/arch_gicv3.h>
24+
25+struct arm_cpuidle_irq_context {
26+ unsigned long pmr;
27+ unsigned long daif_bits;
28+};
29+
30+#define arm_cpuidle_save_irq_context(__c) \
31+ do { \
32+ struct arm_cpuidle_irq_context *c = __c; \
33+ if (system_uses_irq_prio_masking()) { \
34+ c->daif_bits = read_sysreg(daif); \
35+ write_sysreg(c->daif_bits | PSR_I_BIT | PSR_F_BIT, \
36+ daif); \
37+ c->pmr = gic_read_pmr(); \
38+ gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); \
39+ } \
40+ } while (0)
41+
42+#define arm_cpuidle_restore_irq_context(__c) \
43+ do { \
44+ struct arm_cpuidle_irq_context *c = __c; \
45+ if (system_uses_irq_prio_masking()) { \
46+ gic_write_pmr(c->pmr); \
47+ write_sysreg(c->daif_bits, daif); \
48+ } \
49+ } while (0)
50+#else
51+struct arm_cpuidle_irq_context { };
52+
53+#define arm_cpuidle_save_irq_context(c) (void)c
54+#define arm_cpuidle_restore_irq_context(c) (void)c
55+#endif
2156 #endif
arch/arm64/include/asm/efi.h+1−1
@@ -137,7 +137,7 @@ void efi_virtmap_unload(void); · 136 unmodified lines
137137
138138 static inline void efi_capsule_flush_cache_range(void *addr, int size)
139139 {
140− __flush_dcache_area(addr, size);
140+ dcache_clean_inval_poc((unsigned long)addr, (unsigned long)addr + size);
141141 }
142142
143143 #endif /* _ASM_EFI_H */
arch/arm64/include/asm/exception.h+26−8
@@ -31,20 +31,35 @@ static inline u32 disr_to_esr(u64 disr) · 30 unmodified lines
3131 return esr;
3232 }
3333
34−asmlinkage void el1_sync_handler(struct pt_regs *regs);
35−asmlinkage void el0_sync_handler(struct pt_regs *regs);
36−asmlinkage void el0_sync_compat_handler(struct pt_regs *regs);
34+asmlinkage void handle_bad_stack(struct pt_regs *regs);
3735
38−asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs);
39−asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs);
36+asmlinkage void el1t_64_sync_handler(struct pt_regs *regs);
37+asmlinkage void el1t_64_irq_handler(struct pt_regs *regs);
38+asmlinkage void el1t_64_fiq_handler(struct pt_regs *regs);
39+asmlinkage void el1t_64_error_handler(struct pt_regs *regs);
40+
41+asmlinkage void el1h_64_sync_handler(struct pt_regs *regs);
42+asmlinkage void el1h_64_irq_handler(struct pt_regs *regs);
43+asmlinkage void el1h_64_fiq_handler(struct pt_regs *regs);
44+asmlinkage void el1h_64_error_handler(struct pt_regs *regs);
45+
46+asmlinkage void el0t_64_sync_handler(struct pt_regs *regs);
47+asmlinkage void el0t_64_irq_handler(struct pt_regs *regs);
48+asmlinkage void el0t_64_fiq_handler(struct pt_regs *regs);
49+asmlinkage void el0t_64_error_handler(struct pt_regs *regs);
50+
51+asmlinkage void el0t_32_sync_handler(struct pt_regs *regs);
52+asmlinkage void el0t_32_irq_handler(struct pt_regs *regs);
53+asmlinkage void el0t_32_fiq_handler(struct pt_regs *regs);
54+asmlinkage void el0t_32_error_handler(struct pt_regs *regs);
55+
56+asmlinkage void call_on_irq_stack(struct pt_regs *regs,
57+ void (*func)(struct pt_regs *));
4058 asmlinkage void enter_from_user_mode(void);
4159 asmlinkage void exit_to_user_mode(void);
42−void arm64_enter_nmi(struct pt_regs *regs);
43−void arm64_exit_nmi(struct pt_regs *regs);
4460 void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs);
4561 void do_undefinstr(struct pt_regs *regs);
4662 void do_bti(struct pt_regs *regs);
47−asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr);
4863 void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr,
4964 struct pt_regs *regs);
5065 void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs);
@@ -57,4 +72,7 @@ void do_cp15instr(unsigned int esr, struct pt_regs *regs); · 6 unmodified lines
5772 void do_el0_svc(struct pt_regs *regs);
5873 void do_el0_svc_compat(struct pt_regs *regs);
5974 void do_ptrauth_fault(struct pt_regs *regs, unsigned int esr);
75+void do_serror(struct pt_regs *regs, unsigned int esr);
76+
77+void panic_bad_stack(struct pt_regs *regs, unsigned int esr, unsigned long far);
6078 #endif /* __ASM_EXCEPTION_H */
arch/arm64/include/asm/fpsimd.h+1−1
@@ -69,7 +69,7 @@ static inline void *sve_pffr(struct thread_struct *thread) · 68 unmodified lines
6969 extern void sve_save_state(void *state, u32 *pfpsr);
7070 extern void sve_load_state(void const *state, u32 const *pfpsr,
7171 unsigned long vq_minus_1);
72−extern void sve_flush_live(void);
72+extern void sve_flush_live(unsigned long vq_minus_1);
7373 extern void sve_load_from_fpsimd_state(struct user_fpsimd_state const *state,
7474 unsigned long vq_minus_1);
7575 extern unsigned int sve_get_vl(void);
arch/arm64/include/asm/fpsimdmacros.h+3−1
@@ -213,8 +213,10 @@ · 212 unmodified lines
213213 mov v\nz\().16b, v\nz\().16b
214214 .endm
215215
216−.macro sve_flush
216+.macro sve_flush_z
217217 _for n, 0, 31, _sve_flush_z \n
218+.endm
219+.macro sve_flush_p_ffr
218220 _for n, 0, 15, _sve_pfalse \n
219221 _sve_wrffr 0
220222 .endm
arch/arm64/include/asm/insn-def.h+9−0
@@ -0,0 +1,9 @@
1+/* SPDX-License-Identifier: GPL-2.0-only */
2+
3+#ifndef __ASM_INSN_DEF_H
4+#define __ASM_INSN_DEF_H
5+
6+/* A64 instructions are always 32 bits. */
7+#define AARCH64_INSN_SIZE 4
8+
9+#endif /* __ASM_INSN_DEF_H */
arch/arm64/include/asm/insn.h+61−6
@@ -10,7 +10,7 @@ · 9 unmodified lines
1010 #include <linux/build_bug.h>
1111 #include <linux/types.h>
1212
13−#include <asm/alternative.h>
13+#include <asm/insn-def.h>
1414
1515 #ifndef __ASSEMBLY__
1616 /*
@@ -30,6 +30,7 @@ · 13 unmodified lines
3030 */
3131 enum aarch64_insn_encoding_class {
3232 AARCH64_INSN_CLS_UNKNOWN, /* UNALLOCATED */
33+ AARCH64_INSN_CLS_SVE, /* SVE instructions */
3334 AARCH64_INSN_CLS_DP_IMM, /* Data processing - immediate */
3435 AARCH64_INSN_CLS_DP_REG, /* Data processing - register */
3536 AARCH64_INSN_CLS_DP_FPSIMD, /* Data processing - SIMD and FP */
@@ -294,6 +295,12 @@ __AARCH64_INSN_FUNCS(adr, 0x9F000000, 0x10000000) · 258 unmodified lines
294295 __AARCH64_INSN_FUNCS(adrp, 0x9F000000, 0x90000000)
295296 __AARCH64_INSN_FUNCS(prfm, 0x3FC00000, 0x39800000)
296297 __AARCH64_INSN_FUNCS(prfm_lit, 0xFF000000, 0xD8000000)
298+__AARCH64_INSN_FUNCS(store_imm, 0x3FC00000, 0x39000000)
299+__AARCH64_INSN_FUNCS(load_imm, 0x3FC00000, 0x39400000)
300+__AARCH64_INSN_FUNCS(store_pre, 0x3FE00C00, 0x38000C00)
301+__AARCH64_INSN_FUNCS(load_pre, 0x3FE00C00, 0x38400C00)
302+__AARCH64_INSN_FUNCS(store_post, 0x3FE00C00, 0x38000400)
303+__AARCH64_INSN_FUNCS(load_post, 0x3FE00C00, 0x38400400)
297304 __AARCH64_INSN_FUNCS(str_reg, 0x3FE0EC00, 0x38206800)
298305 __AARCH64_INSN_FUNCS(ldadd, 0x3F20FC00, 0x38200000)
299306 __AARCH64_INSN_FUNCS(ldr_reg, 0x3FE0EC00, 0x38606800)
@@ -302,6 +309,8 @@ __AARCH64_INSN_FUNCS(ldrsw_lit, 0xFF000000, 0x98000000) · 2 unmodified lines
302309 __AARCH64_INSN_FUNCS(exclusive, 0x3F800000, 0x08000000)
303310 __AARCH64_INSN_FUNCS(load_ex, 0x3F400000, 0x08400000)
304311 __AARCH64_INSN_FUNCS(store_ex, 0x3F400000, 0x08000000)
312+__AARCH64_INSN_FUNCS(stp, 0x7FC00000, 0x29000000)
313+__AARCH64_INSN_FUNCS(ldp, 0x7FC00000, 0x29400000)
305314 __AARCH64_INSN_FUNCS(stp_post, 0x7FC00000, 0x28800000)
306315 __AARCH64_INSN_FUNCS(ldp_post, 0x7FC00000, 0x28C00000)
307316 __AARCH64_INSN_FUNCS(stp_pre, 0x7FC00000, 0x29800000)
@@ -334,6 +343,7 @@ __AARCH64_INSN_FUNCS(rev64, 0x7FFFFC00, 0x5AC00C00) · 26 unmodified lines
334343 __AARCH64_INSN_FUNCS(and, 0x7F200000, 0x0A000000)
335344 __AARCH64_INSN_FUNCS(bic, 0x7F200000, 0x0A200000)
336345 __AARCH64_INSN_FUNCS(orr, 0x7F200000, 0x2A000000)
346+__AARCH64_INSN_FUNCS(mov_reg, 0x7FE0FFE0, 0x2A0003E0)
337347 __AARCH64_INSN_FUNCS(orn, 0x7F200000, 0x2A200000)
338348 __AARCH64_INSN_FUNCS(eor, 0x7F200000, 0x4A000000)
339349 __AARCH64_INSN_FUNCS(eon, 0x7F200000, 0x4A200000)
@@ -368,6 +378,14 @@ __AARCH64_INSN_FUNCS(eret_auth, 0xFFFFFBFF, 0xD69F0BFF) · 28 unmodified lines
368378 __AARCH64_INSN_FUNCS(mrs, 0xFFF00000, 0xD5300000)
369379 __AARCH64_INSN_FUNCS(msr_imm, 0xFFF8F01F, 0xD500401F)
370380 __AARCH64_INSN_FUNCS(msr_reg, 0xFFF00000, 0xD5100000)
381+__AARCH64_INSN_FUNCS(dmb, 0xFFFFF0FF, 0xD50330BF)
382+__AARCH64_INSN_FUNCS(dsb_base, 0xFFFFF0FF, 0xD503309F)
383+__AARCH64_INSN_FUNCS(dsb_nxs, 0xFFFFF3FF, 0xD503323F)
384+__AARCH64_INSN_FUNCS(isb, 0xFFFFF0FF, 0xD50330DF)
385+__AARCH64_INSN_FUNCS(sb, 0xFFFFFFFF, 0xD50330FF)
386+__AARCH64_INSN_FUNCS(clrex, 0xFFFFF0FF, 0xD503305F)
387+__AARCH64_INSN_FUNCS(ssbb, 0xFFFFFFFF, 0xD503309F)
388+__AARCH64_INSN_FUNCS(pssbb, 0xFFFFFFFF, 0xD503349F)
371389
372390 #undef __AARCH64_INSN_FUNCS
373391
@@ -379,8 +397,47 @@ static inline bool aarch64_insn_is_adr_adrp(u32 insn) · 5 unmodified lines
379397 return aarch64_insn_is_adr(insn) || aarch64_insn_is_adrp(insn);
380398 }
381399
382−int aarch64_insn_read(void *addr, u32 *insnp);
383−int aarch64_insn_write(void *addr, u32 insn);
400+static inline bool aarch64_insn_is_dsb(u32 insn)
401+{
402+ return aarch64_insn_is_dsb_base(insn) || aarch64_insn_is_dsb_nxs(insn);
403+}
404+
405+static inline bool aarch64_insn_is_barrier(u32 insn)
406+{
407+ return aarch64_insn_is_dmb(insn) || aarch64_insn_is_dsb(insn) ||
408+ aarch64_insn_is_isb(insn) || aarch64_insn_is_sb(insn) ||
409+ aarch64_insn_is_clrex(insn) || aarch64_insn_is_ssbb(insn) ||
410+ aarch64_insn_is_pssbb(insn);
411+}
412+
413+static inline bool aarch64_insn_is_store_single(u32 insn)
414+{
415+ return aarch64_insn_is_store_imm(insn) ||
416+ aarch64_insn_is_store_pre(insn) ||
417+ aarch64_insn_is_store_post(insn);
418+}
419+
420+static inline bool aarch64_insn_is_store_pair(u32 insn)
421+{
422+ return aarch64_insn_is_stp(insn) ||
423+ aarch64_insn_is_stp_pre(insn) ||
424+ aarch64_insn_is_stp_post(insn);
425+}
426+
427+static inline bool aarch64_insn_is_load_single(u32 insn)
428+{
429+ return aarch64_insn_is_load_imm(insn) ||
430+ aarch64_insn_is_load_pre(insn) ||
431+ aarch64_insn_is_load_post(insn);
432+}
433+
434+static inline bool aarch64_insn_is_load_pair(u32 insn)
435+{
436+ return aarch64_insn_is_ldp(insn) ||
437+ aarch64_insn_is_ldp_pre(insn) ||
438+ aarch64_insn_is_ldp_post(insn);
439+}
440+
384441 enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn);
385442 bool aarch64_insn_uses_literal(u32 insn);
386443 bool aarch64_insn_is_branch(u32 insn);
@@ -487,9 +544,6 @@ u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base, · 100 unmodified lines
487544 s32 aarch64_get_branch_offset(u32 insn);
488545 u32 aarch64_set_branch_offset(u32 insn, s32 offset);
489546
490−int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
491−int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt);
492−
493547 s32 aarch64_insn_adrp_get_offset(u32 insn);
494548 u32 aarch64_insn_adrp_set_offset(u32 insn, s32 offset);
495549
@@ -506,6 +560,7 @@ u32 aarch32_insn_mcr_extract_crm(u32 insn); · 10 unmodified lines
506560
507561 typedef bool (pstate_check_t)(unsigned long);
508562 extern pstate_check_t * const aarch32_opcode_cond_checks[16];
563+
509564 #endif /* __ASSEMBLY__ */
510565
511566 #endif /* __ASM_INSN_H */
arch/arm64/include/asm/kernel-pgtable.h+8−11
@@ -18,9 +18,9 @@ · 17 unmodified lines
1818 * 64K (section size = 512M).
1919 */
2020 #ifdef CONFIG_ARM64_4K_PAGES
21−#define ARM64_SWAPPER_USES_SECTION_MAPS 1
21+#define ARM64_KERNEL_USES_PMD_MAPS 1
2222 #else
23−#define ARM64_SWAPPER_USES_SECTION_MAPS 0
23+#define ARM64_KERNEL_USES_PMD_MAPS 0
2424 #endif
2525
2626 /*
@@ -33,7 +33,7 @@ · 6 unmodified lines
3333 * VA range, so pages required to map highest possible PA are reserved in all
3434 * cases.
3535 */
36−#if ARM64_SWAPPER_USES_SECTION_MAPS
36+#if ARM64_KERNEL_USES_PMD_MAPS
3737 #define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - 1)
3838 #define IDMAP_PGTABLE_LEVELS (ARM64_HW_PGTABLE_LEVELS(PHYS_MASK_SHIFT) - 1)
3939 #else
@@ -90,9 +90,9 @@ · 50 unmodified lines
9090 #define IDMAP_DIR_SIZE (IDMAP_PGTABLE_LEVELS * PAGE_SIZE)
9191
9292 /* Initial memory map size */
93−#if ARM64_SWAPPER_USES_SECTION_MAPS
94−#define SWAPPER_BLOCK_SHIFT SECTION_SHIFT
95−#define SWAPPER_BLOCK_SIZE SECTION_SIZE
93+#if ARM64_KERNEL_USES_PMD_MAPS
94+#define SWAPPER_BLOCK_SHIFT PMD_SHIFT
95+#define SWAPPER_BLOCK_SIZE PMD_SIZE
9696 #define SWAPPER_TABLE_SHIFT PUD_SHIFT
9797 #else
9898 #define SWAPPER_BLOCK_SHIFT PAGE_SHIFT
@@ -100,16 +100,13 @@ · 1 unmodified line
100100 #define SWAPPER_TABLE_SHIFT PMD_SHIFT
101101 #endif
102102
103−/* The size of the initial kernel direct mapping */
104−#define SWAPPER_INIT_MAP_SIZE (_AC(1, UL) << SWAPPER_TABLE_SHIFT)
105−
106103 /*
107104 * Initial memory map attributes.
108105 */
109106 #define SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED)
110107 #define SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S)
111108
112−#if ARM64_SWAPPER_USES_SECTION_MAPS
109+#if ARM64_KERNEL_USES_PMD_MAPS
113110 #define SWAPPER_MM_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS)
114111 #else
115112 #define SWAPPER_MM_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS)
@@ -125,7 +122,7 @@ · 9 unmodified lines
125122 #if defined(CONFIG_ARM64_4K_PAGES)
126123 #define ARM64_MEMSTART_SHIFT PUD_SHIFT
127124 #elif defined(CONFIG_ARM64_16K_PAGES)
128−#define ARM64_MEMSTART_SHIFT (PMD_SHIFT + 5)
125+#define ARM64_MEMSTART_SHIFT CONT_PMD_SHIFT
129126 #else
130127 #define ARM64_MEMSTART_SHIFT PMD_SHIFT
131128 #endif
arch/arm64/include/asm/kvm_asm.h+1−0
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 #define __ARM_KVM_ASM_H__
99
1010 #include <asm/hyp_image.h>
11+#include <asm/insn.h>
1112 #include <asm/virt.h>
1213
1314 #define ARM_EXIT_WITH_SERROR_BIT 31
arch/arm64/include/asm/kvm_mmu.h+4−3
@@ -180,7 +180,8 @@ static inline void *__kvm_vector_slot2addr(void *base, · 179 unmodified lines
180180
181181 struct kvm;
182182
183−#define kvm_flush_dcache_to_poc(a,l) __flush_dcache_area((a), (l))
183+#define kvm_flush_dcache_to_poc(a,l) \
184+ dcache_clean_inval_poc((unsigned long)(a), (unsigned long)(a)+(l))
184185
185186 static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu)
186187 {
@@ -208,12 +209,12 @@ static inline void __invalidate_icache_guest_page(kvm_pfn_t pfn, · 21 unmodified lines
208209 {
209210 if (icache_is_aliasing()) {
210211 /* any kind of VIPT cache */
211− __flush_icache_all();
212+ icache_inval_all_pou();
212213 } else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) {
213214 /* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */
214215 void *va = page_address(pfn_to_page(pfn));
215216
216− invalidate_icache_range((unsigned long)va,
217+ icache_inval_pou((unsigned long)va,
217218 (unsigned long)va + size);
218219 }
219220 }
arch/arm64/include/asm/linkage.h+8−0
@@ -56,8 +56,16 @@ · 55 unmodified lines
5656 SYM_FUNC_START_ALIAS(__pi_##x); \
5757 SYM_FUNC_START_WEAK(x)
5858
59+#define SYM_FUNC_START_WEAK_ALIAS_PI(x) \
60+ SYM_FUNC_START_ALIAS(__pi_##x); \
61+ SYM_START(x, SYM_L_WEAK, SYM_A_ALIGN)
62+
5963 #define SYM_FUNC_END_PI(x) \
6064 SYM_FUNC_END(x); \
6165 SYM_FUNC_END_ALIAS(__pi_##x)
6266
67+#define SYM_FUNC_END_ALIAS_PI(x) \
68+ SYM_FUNC_END_ALIAS(x); \
69+ SYM_FUNC_END_ALIAS(__pi_##x)
70+
6371 #endif
arch/arm64/include/asm/memory.h+2−4
@@ -135,10 +135,8 @@ · 134 unmodified lines
135135 #define MT_NORMAL 0
136136 #define MT_NORMAL_TAGGED 1
137137 #define MT_NORMAL_NC 2
138−#define MT_NORMAL_WT 3
139−#define MT_DEVICE_nGnRnE 4
140−#define MT_DEVICE_nGnRE 5
141−#define MT_DEVICE_GRE 6
138+#define MT_DEVICE_nGnRnE 3
139+#define MT_DEVICE_nGnRE 4
142140
143141 /*
144142 * Memory types for Stage-2 translation
arch/arm64/include/asm/mmu_context.h+2−2
@@ -177,9 +177,9 @@ static inline void update_saved_ttbr0(struct task_struct *tsk, · 176 unmodified lines
177177 return;
178178
179179 if (mm == &init_mm)
180− ttbr = __pa_symbol(reserved_pg_dir);
180+ ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir));
181181 else
182− ttbr = virt_to_phys(mm->pgd) | ASID(mm) << 48;
182+ ttbr = phys_to_ttbr(virt_to_phys(mm->pgd)) | ASID(mm) << 48;
183183
184184 WRITE_ONCE(task_thread_info(tsk)->ttbr0, ttbr);
185185 }
arch/arm64/include/asm/module.lds.h+15−2
@@ -1,7 +1,20 @@
1−#ifdef CONFIG_ARM64_MODULE_PLTS
21 SECTIONS {
2+#ifdef CONFIG_ARM64_MODULE_PLTS
33 .plt 0 (NOLOAD) : { BYTE(0) }
44 .init.plt 0 (NOLOAD) : { BYTE(0) }
55 .text.ftrace_trampoline 0 (NOLOAD) : { BYTE(0) }
6−}
76 #endif
7+
8+#ifdef CONFIG_KASAN_SW_TAGS
9+ /*
10+ * Outlined checks go into comdat-deduplicated sections named .text.hot.
11+ * Because they are in comdats they are not combined by the linker and
12+ * we otherwise end up with multiple sections with the same .text.hot
13+ * name in the .ko file. The kernel module loader warns if it sees
14+ * multiple sections with the same name so we use this sections
15+ * directive to force them into a single section and silence the
16+ * warning.
17+ */
18+ .text.hot : { *(.text.hot) }
19+#endif
20+}
arch/arm64/include/asm/mte-kasan.h+67−26
@@ -48,43 +48,84 @@ static inline u8 mte_get_random_tag(void) · 47 unmodified lines
4848 return mte_get_ptr_tag(addr);
4949 }
5050
51+static inline u64 __stg_post(u64 p)
52+{
53+ asm volatile(__MTE_PREAMBLE "stg %0, [%0], #16"
54+ : "+r"(p)
55+ :
56+ : "memory");
57+ return p;
58+}
59+
60+static inline u64 __stzg_post(u64 p)
61+{
62+ asm volatile(__MTE_PREAMBLE "stzg %0, [%0], #16"
63+ : "+r"(p)
64+ :
65+ : "memory");
66+ return p;
67+}
68+
69+static inline void __dc_gva(u64 p)
70+{
71+ asm volatile(__MTE_PREAMBLE "dc gva, %0" : : "r"(p) : "memory");
72+}
73+
74+static inline void __dc_gzva(u64 p)
75+{
76+ asm volatile(__MTE_PREAMBLE "dc gzva, %0" : : "r"(p) : "memory");
77+}
78+
5179 /*
5280 * Assign allocation tags for a region of memory based on the pointer tag.
5381 * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and
54− * size must be non-zero and MTE_GRANULE_SIZE aligned.
82+ * size must be MTE_GRANULE_SIZE aligned.
5583 */
56−static inline void mte_set_mem_tag_range(void *addr, size_t size,
57− u8 tag, bool init)
84+static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag,
85+ bool init)
5886 {
59− u64 curr, end;
87+ u64 curr, mask, dczid_bs, end1, end2, end3;
6088
61− if (!size)
62− return;
89+ /* Read DC G(Z)VA block size from the system register. */
90+ dczid_bs = 4ul << (read_cpuid(DCZID_EL0) & 0xf);
6391
6492 curr = (u64)__tag_set(addr, tag);
65− end = curr + size;
93+ mask = dczid_bs - 1;
94+ /* STG/STZG up to the end of the first block. */
95+ end1 = curr | mask;
96+ end3 = curr + size;
97+ /* DC GVA / GZVA in [end1, end2) */
98+ end2 = end3 & ~mask;
6699
67100 /*
68− * 'asm volatile' is required to prevent the compiler to move
69− * the statement outside of the loop.
101+ * The following code uses STG on the first DC GVA block even if the
102+ * start address is aligned - it appears to be faster than an alignment
103+ * check + conditional branch. Also, if the range size is at least 2 DC
104+ * GVA blocks, the first two loops can use post-condition to save one
105+ * branch each.
70106 */
71− if (init) {
72− do {
73− asm volatile(__MTE_PREAMBLE "stzg %0, [%0]"
74− :
75− : "r" (curr)
76− : "memory");
77− curr += MTE_GRANULE_SIZE;
78− } while (curr != end);
79− } else {
80− do {
81− asm volatile(__MTE_PREAMBLE "stg %0, [%0]"
82− :
83− : "r" (curr)
84− : "memory");
85− curr += MTE_GRANULE_SIZE;
86− } while (curr != end);
87− }
107+#define SET_MEMTAG_RANGE(stg_post, dc_gva) \
108+ do { \
109+ if (size >= 2 * dczid_bs) { \
110+ do { \
111+ curr = stg_post(curr); \
112+ } while (curr < end1); \
113+ \
114+ do { \
115+ dc_gva(curr); \
116+ curr += dczid_bs; \
117+ } while (curr < end2); \
118+ } \
119+ \
120+ while (curr < end3) \
121+ curr = stg_post(curr); \
122+ } while (0)
123+
124+ if (init)
125+ SET_MEMTAG_RANGE(__stzg_post, __dc_gzva);
126+ else
127+ SET_MEMTAG_RANGE(__stg_post, __dc_gva);
128+#undef SET_MEMTAG_RANGE
88129 }
89130
90131 void mte_enable_kernel_sync(void);
arch/arm64/include/asm/mte.h+4−0
@@ -37,6 +37,7 @@ void mte_free_tag_storage(char *storage); · 36 unmodified lines
3737 /* track which pages have valid allocation tags */
3838 #define PG_mte_tagged PG_arch_2
3939
40+void mte_zero_clear_page_tags(void *addr);
4041 void mte_sync_tags(pte_t *ptep, pte_t pte);
4142 void mte_copy_page_tags(void *kto, const void *kfrom);
4243 void mte_thread_init_user(void);
@@ -53,6 +54,9 @@ int mte_ptrace_copy_tags(struct task_struct *child, long request, · 10 unmodified lines
5354 /* unused if !CONFIG_ARM64_MTE, silence the compiler */
5455 #define PG_mte_tagged 0
5556
57+static inline void mte_zero_clear_page_tags(void *addr)
58+{
59+}
5660 static inline void mte_sync_tags(pte_t *ptep, pte_t pte)
5761 {
5862 }
arch/arm64/include/asm/page.h+7−3
@@ -13,6 +13,7 @@ · 12 unmodified lines
1313 #ifndef __ASSEMBLY__
1414
1515 #include <linux/personality.h> /* for READ_IMPLIES_EXEC */
16+#include <linux/types.h> /* for gfp_t */
1617 #include <asm/pgtable-types.h>
1718
1819 struct page;
@@ -28,9 +29,12 @@ void copy_user_highpage(struct page *to, struct page *from, · 9 unmodified lines
2829 void copy_highpage(struct page *to, struct page *from);
2930 #define __HAVE_ARCH_COPY_HIGHPAGE
3031
31−#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \
32− alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr)
33−#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
32+struct page *alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma,
33+ unsigned long vaddr);
34+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE
35+
36+void tag_clear_highpage(struct page *to);
37+#define __HAVE_ARCH_TAG_CLEAR_HIGHPAGE
3438
3539 #define clear_user_page(page, vaddr, pg) clear_page(page)
3640 #define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
arch/arm64/include/asm/patching.h+13−0
@@ -0,0 +1,13 @@
1+/* SPDX-License-Identifier: GPL-2.0-only */
2+#ifndef __ASM_PATCHING_H
3+#define __ASM_PATCHING_H
4+
5+#include <linux/types.h>
6+
7+int aarch64_insn_read(void *addr, u32 *insnp);
8+int aarch64_insn_write(void *addr, u32 insn);
9+
10+int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
11+int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt);
12+
13+#endif /* __ASM_PATCHING_H */
arch/arm64/include/asm/perf_event.h+5−0
@@ -239,6 +239,11 @@ · 238 unmodified lines
239239 /* PMMIR_EL1.SLOTS mask */
240240 #define ARMV8_PMU_SLOTS_MASK 0xff
241241
242+#define ARMV8_PMU_BUS_SLOTS_SHIFT 8
243+#define ARMV8_PMU_BUS_SLOTS_MASK 0xff
244+#define ARMV8_PMU_BUS_WIDTH_SHIFT 16
245+#define ARMV8_PMU_BUS_WIDTH_MASK 0xf
246+
242247 #ifdef CONFIG_PERF_EVENTS
243248 struct pt_regs;
244249 extern unsigned long perf_instruction_pointer(struct pt_regs *regs);
arch/arm64/include/asm/pgtable-hwdef.h+0−7
@@ -71,13 +71,6 @@ · 70 unmodified lines
7171 #define PGDIR_MASK (~(PGDIR_SIZE-1))
7272 #define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT))
7373
74−/*
75− * Section address mask and size definitions.
76− */
77−#define SECTION_SHIFT PMD_SHIFT
78−#define SECTION_SIZE (_AC(1, UL) << SECTION_SHIFT)
79−#define SECTION_MASK (~(SECTION_SIZE-1))
80−
8174 /*
8275 * Contiguous page definitions.
8376 */
arch/arm64/include/asm/pgtable-prot.h+0−1
@@ -55,7 +55,6 @@ extern bool arm64_use_ng_mappings; · 54 unmodified lines
5555 #define PROT_DEVICE_nGnRnE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRnE))
5656 #define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRE))
5757 #define PROT_NORMAL_NC (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_NC))
58−#define PROT_NORMAL_WT (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_WT))
5958 #define PROT_NORMAL (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL))
6059 #define PROT_NORMAL_TAGGED (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_TAGGED))
6160
arch/arm64/include/asm/pgtable.h+2−3
@@ -511,13 +511,12 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, · 510 unmodified lines
511511
512512 #define pmd_none(pmd) (!pmd_val(pmd))
513513
514−#define pmd_bad(pmd) (!(pmd_val(pmd) & PMD_TABLE_BIT))
515−
516514 #define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
517515 PMD_TYPE_TABLE)
518516 #define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
519517 PMD_TYPE_SECT)
520518 #define pmd_leaf(pmd) pmd_sect(pmd)
519+#define pmd_bad(pmd) (!pmd_table(pmd))
521520
522521 #define pmd_leaf_size(pmd) (pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE)
523522 #define pte_leaf_size(pte) (pte_cont(pte) ? CONT_PTE_SIZE : PAGE_SIZE)
@@ -604,7 +603,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) · 80 unmodified lines
604603 pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e))
605604
606605 #define pud_none(pud) (!pud_val(pud))
607−#define pud_bad(pud) (!(pud_val(pud) & PUD_TABLE_BIT))
606+#define pud_bad(pud) (!pud_table(pud))
608607 #define pud_present(pud) pte_present(pud_pte(pud))
609608 #define pud_leaf(pud) pud_sect(pud)
610609 #define pud_valid(pud) pte_valid(pud_pte(pud))
arch/arm64/include/asm/pointer_auth.h+33−26
@@ -31,10 +31,6 @@ struct ptrauth_keys_user { · 30 unmodified lines
3131 struct ptrauth_key apga;
3232 };
3333
34−struct ptrauth_keys_kernel {
35− struct ptrauth_key apia;
36−};
37−
3834 #define __ptrauth_key_install_nosync(k, v) \
3935 do { \
4036 struct ptrauth_key __pki_v = (v); \
@@ -42,6 +38,29 @@ do { \ · 1 unmodified line
4238 write_sysreg_s(__pki_v.hi, SYS_ ## k ## KEYHI_EL1); \
4339 } while (0)
4440
41+#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
42+
43+struct ptrauth_keys_kernel {
44+ struct ptrauth_key apia;
45+};
46+
47+static __always_inline void ptrauth_keys_init_kernel(struct ptrauth_keys_kernel *keys)
48+{
49+ if (system_supports_address_auth())
50+ get_random_bytes(&keys->apia, sizeof(keys->apia));
51+}
52+
53+static __always_inline void ptrauth_keys_switch_kernel(struct ptrauth_keys_kernel *keys)
54+{
55+ if (!system_supports_address_auth())
56+ return;
57+
58+ __ptrauth_key_install_nosync(APIA, keys->apia);
59+ isb();
60+}
61+
62+#endif /* CONFIG_ARM64_PTR_AUTH_KERNEL */
63+
4564 static inline void ptrauth_keys_install_user(struct ptrauth_keys_user *keys)
4665 {
4766 if (system_supports_address_auth()) {
@@ -69,21 +88,6 @@ static inline void ptrauth_keys_init_user(struct ptrauth_keys_user *keys) · 21 unmodified lines
6988 ptrauth_keys_install_user(keys);
7089 }
7190
72−static __always_inline void ptrauth_keys_init_kernel(struct ptrauth_keys_kernel *keys)
73−{
74− if (system_supports_address_auth())
75− get_random_bytes(&keys->apia, sizeof(keys->apia));
76−}
77−
78−static __always_inline void ptrauth_keys_switch_kernel(struct ptrauth_keys_kernel *keys)
79−{
80− if (!system_supports_address_auth())
81− return;
82−
83− __ptrauth_key_install_nosync(APIA, keys->apia);
84− isb();
85−}
86−
8791 extern int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg);
8892
8993 extern int ptrauth_set_enabled_keys(struct task_struct *tsk, unsigned long keys,
@@ -121,11 +125,6 @@ static __always_inline void ptrauth_enable(void) · 31 unmodified lines
121125 #define ptrauth_thread_switch_user(tsk) \
122126 ptrauth_keys_install_user(&(tsk)->thread.keys_user)
123127
124−#define ptrauth_thread_init_kernel(tsk) \
125− ptrauth_keys_init_kernel(&(tsk)->thread.keys_kernel)
126−#define ptrauth_thread_switch_kernel(tsk) \
127− ptrauth_keys_switch_kernel(&(tsk)->thread.keys_kernel)
128−
129128 #else /* CONFIG_ARM64_PTR_AUTH */
130129 #define ptrauth_enable()
131130 #define ptrauth_prctl_reset_keys(tsk, arg) (-EINVAL)
@@ -134,11 +133,19 @@ static __always_inline void ptrauth_enable(void) · 2 unmodified lines
134133 #define ptrauth_strip_insn_pac(lr) (lr)
135134 #define ptrauth_suspend_exit()
136135 #define ptrauth_thread_init_user()
137−#define ptrauth_thread_init_kernel(tsk)
138136 #define ptrauth_thread_switch_user(tsk)
139−#define ptrauth_thread_switch_kernel(tsk)
140137 #endif /* CONFIG_ARM64_PTR_AUTH */
141138
139+#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
140+#define ptrauth_thread_init_kernel(tsk) \
141+ ptrauth_keys_init_kernel(&(tsk)->thread.keys_kernel)
142+#define ptrauth_thread_switch_kernel(tsk) \
143+ ptrauth_keys_switch_kernel(&(tsk)->thread.keys_kernel)
144+#else
145+#define ptrauth_thread_init_kernel(tsk)
146+#define ptrauth_thread_switch_kernel(tsk)
147+#endif /* CONFIG_ARM64_PTR_AUTH_KERNEL */
148+
142149 #define PR_PAC_ENABLED_KEYS_MASK \
143150 (PR_PAC_APIAKEY | PR_PAC_APIBKEY | PR_PAC_APDAKEY | PR_PAC_APDBKEY)
144151
arch/arm64/include/asm/processor.h+8−8
@@ -148,8 +148,10 @@ struct thread_struct { · 147 unmodified lines
148148 struct debug_info debug; /* debugging */
149149 #ifdef CONFIG_ARM64_PTR_AUTH
150150 struct ptrauth_keys_user keys_user;
151+#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
151152 struct ptrauth_keys_kernel keys_kernel;
152153 #endif
154+#endif
153155 #ifdef CONFIG_ARM64_MTE
154156 u64 gcr_user_excl;
155157 #endif
@@ -257,8 +259,6 @@ void set_task_sctlr_el1(u64 sctlr); · 101 unmodified lines
257259 extern struct task_struct *cpu_switch_to(struct task_struct *prev,
258260 struct task_struct *next);
259261
260−asmlinkage void arm64_preempt_schedule_irq(void);
261−
262262 #define task_pt_regs(p) \
263263 ((struct pt_regs *)(THREAD_SIZE + task_stack_page(p)) - 1)
264264
@@ -329,13 +329,13 @@ long get_tagged_addr_ctrl(struct task_struct *task); · 64 unmodified lines
329329 * of header definitions for the use of task_stack_page.
330330 */
331331
332−#define current_top_of_stack() \
333−({ \
334− struct stack_info _info; \
335− BUG_ON(!on_accessible_stack(current, current_stack_pointer, &_info)); \
336− _info.high; \
332+#define current_top_of_stack() \
333+({ \
334+ struct stack_info _info; \
335+ BUG_ON(!on_accessible_stack(current, current_stack_pointer, 1, &_info)); \
336+ _info.high; \
337337 })
338−#define on_thread_stack() (on_task_stack(current, current_stack_pointer, NULL))
338+#define on_thread_stack() (on_task_stack(current, current_stack_pointer, 1, NULL))
339339
340340 #endif /* __ASSEMBLY__ */
341341 #endif /* __ASM_PROCESSOR_H */
arch/arm64/include/asm/scs.h+4−4
@@ -9,18 +9,18 @@ · 8 unmodified lines
99 #ifdef CONFIG_SHADOW_CALL_STACK
1010 scs_sp .req x18
1111
12− .macro scs_load tsk, tmp
12+ .macro scs_load tsk
1313 ldr scs_sp, [\tsk, #TSK_TI_SCS_SP]
1414 .endm
1515
16− .macro scs_save tsk, tmp
16+ .macro scs_save tsk
1717 str scs_sp, [\tsk, #TSK_TI_SCS_SP]
1818 .endm
1919 #else
20− .macro scs_load tsk, tmp
20+ .macro scs_load tsk
2121 .endm
2222
23− .macro scs_save tsk, tmp
23+ .macro scs_save tsk
2424 .endm
2525 #endif /* CONFIG_SHADOW_CALL_STACK */
2626
arch/arm64/include/asm/sdei.h+7−3
@@ -37,13 +37,17 @@ struct sdei_registered_event; · 36 unmodified lines
3737 asmlinkage unsigned long __sdei_handler(struct pt_regs *regs,
3838 struct sdei_registered_event *arg);
3939
40+unsigned long do_sdei_event(struct pt_regs *regs,
41+ struct sdei_registered_event *arg);
42+
4043 unsigned long sdei_arch_get_entry_point(int conduit);
4144 #define sdei_arch_get_entry_point(x) sdei_arch_get_entry_point(x)
4245
4346 struct stack_info;
4447
45−bool _on_sdei_stack(unsigned long sp, struct stack_info *info);
46−static inline bool on_sdei_stack(unsigned long sp,
48+bool _on_sdei_stack(unsigned long sp, unsigned long size,
49+ struct stack_info *info);
50+static inline bool on_sdei_stack(unsigned long sp, unsigned long size,
4751 struct stack_info *info)
4852 {
4953 if (!IS_ENABLED(CONFIG_VMAP_STACK))
@@ -51,7 +55,7 @@ static inline bool on_sdei_stack(unsigned long sp, · 1 unmodified line
5155 if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE))
5256 return false;
5357 if (in_nmi())
54− return _on_sdei_stack(sp, info);
58+ return _on_sdei_stack(sp, size, info);
5559
5660 return false;
5761 }
arch/arm64/include/asm/smp.h+0−2
@@ -73,12 +73,10 @@ asmlinkage void secondary_start_kernel(void); · 72 unmodified lines
7373
7474 /*
7575 * Initial data for bringing up a secondary CPU.
76− * @stack - sp for the secondary CPU
7776 * @status - Result passed back from the secondary CPU to
7877 * indicate failure.
7978 */
8079 struct secondary_data {
81− void *stack;
8280 struct task_struct *task;
8381 long status;
8482 };
arch/arm64/include/asm/stacktrace.h+16−16
@@ -69,14 +69,14 @@ extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, · 68 unmodified lines
6969
7070 DECLARE_PER_CPU(unsigned long *, irq_stack_ptr);
7171
72−static inline bool on_stack(unsigned long sp, unsigned long low,
73− unsigned long high, enum stack_type type,
74− struct stack_info *info)
72+static inline bool on_stack(unsigned long sp, unsigned long size,
73+ unsigned long low, unsigned long high,
74+ enum stack_type type, struct stack_info *info)
7575 {
7676 if (!low)
7777 return false;
7878
79− if (sp < low || sp >= high)
79+ if (sp < low || sp + size < sp || sp + size > high)
8080 return false;
8181
8282 if (info) {
@@ -87,38 +87,38 @@ static inline bool on_stack(unsigned long sp, unsigned long low, · 4 unmodified lines
8787 return true;
8888 }
8989
90−static inline bool on_irq_stack(unsigned long sp,
90+static inline bool on_irq_stack(unsigned long sp, unsigned long size,
9191 struct stack_info *info)
9292 {
9393 unsigned long low = (unsigned long)raw_cpu_read(irq_stack_ptr);
9494 unsigned long high = low + IRQ_STACK_SIZE;
9595
96− return on_stack(sp, low, high, STACK_TYPE_IRQ, info);
96+ return on_stack(sp, size, low, high, STACK_TYPE_IRQ, info);
9797 }
9898
9999 static inline bool on_task_stack(const struct task_struct *tsk,
100− unsigned long sp,
100+ unsigned long sp, unsigned long size,
101101 struct stack_info *info)
102102 {
103103 unsigned long low = (unsigned long)task_stack_page(tsk);
104104 unsigned long high = low + THREAD_SIZE;
105105
106− return on_stack(sp, low, high, STACK_TYPE_TASK, info);
106+ return on_stack(sp, size, low, high, STACK_TYPE_TASK, info);
107107 }
108108
109109 #ifdef CONFIG_VMAP_STACK
110110 DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
111111
112−static inline bool on_overflow_stack(unsigned long sp,
112+static inline bool on_overflow_stack(unsigned long sp, unsigned long size,
113113 struct stack_info *info)
114114 {
115115 unsigned long low = (unsigned long)raw_cpu_ptr(overflow_stack);
116116 unsigned long high = low + OVERFLOW_STACK_SIZE;
117117
118− return on_stack(sp, low, high, STACK_TYPE_OVERFLOW, info);
118+ return on_stack(sp, size, low, high, STACK_TYPE_OVERFLOW, info);
119119 }
120120 #else
121−static inline bool on_overflow_stack(unsigned long sp,
121+static inline bool on_overflow_stack(unsigned long sp, unsigned long size,
122122 struct stack_info *info) { return false; }
123123 #endif
124124
@@ -128,21 +128,21 @@ static inline bool on_overflow_stack(unsigned long sp, · 3 unmodified lines
128128 * context.
129129 */
130130 static inline bool on_accessible_stack(const struct task_struct *tsk,
131− unsigned long sp,
131+ unsigned long sp, unsigned long size,
132132 struct stack_info *info)
133133 {
134134 if (info)
135135 info->type = STACK_TYPE_UNKNOWN;
136136
137− if (on_task_stack(tsk, sp, info))
137+ if (on_task_stack(tsk, sp, size, info))
138138 return true;
139139 if (tsk != current || preemptible())
140140 return false;
141− if (on_irq_stack(sp, info))
141+ if (on_irq_stack(sp, size, info))
142142 return true;
143− if (on_overflow_stack(sp, info))
143+ if (on_overflow_stack(sp, size, info))
144144 return true;
145− if (on_sdei_stack(sp, info))
145+ if (on_sdei_stack(sp, size, info))
146146 return true;
147147
148148 return false;
arch/arm64/include/asm/sysreg.h+0−2
@@ -703,9 +703,7 @@ · 702 unmodified lines
703703 /* MAIR_ELx memory attributes (used by Linux) */
704704 #define MAIR_ATTR_DEVICE_nGnRnE UL(0x00)
705705 #define MAIR_ATTR_DEVICE_nGnRE UL(0x04)
706−#define MAIR_ATTR_DEVICE_GRE UL(0x0c)
707706 #define MAIR_ATTR_NORMAL_NC UL(0x44)
708−#define MAIR_ATTR_NORMAL_WT UL(0xbb)
709707 #define MAIR_ATTR_NORMAL_TAGGED UL(0xf0)
710708 #define MAIR_ATTR_NORMAL UL(0xff)
711709 #define MAIR_ATTR_MASK UL(0xff)
arch/arm64/include/asm/tlb.h+4−0
@@ -28,6 +28,10 @@ static void tlb_flush(struct mmu_gather *tlb); · 27 unmodified lines
2828 */
2929 static inline int tlb_get_level(struct mmu_gather *tlb)
3030 {
31+ /* The TTL field is only valid for the leaf entry. */
32+ if (tlb->freed_tables)
33+ return 0;
34+
3135 if (tlb->cleared_ptes && !(tlb->cleared_pmds ||
3236 tlb->cleared_puds ||
3337 tlb->cleared_p4ds))
arch/arm64/kernel/Makefile+9−2
@@ -14,15 +14,22 @@ CFLAGS_REMOVE_return_address.o = $(CC_FLAGS_FTRACE) · 13 unmodified lines
1414 CFLAGS_REMOVE_syscall.o = -fstack-protector -fstack-protector-strong
1515 CFLAGS_syscall.o += -fno-stack-protector
1616
17+# It's not safe to invoke KCOV when portions of the kernel environment aren't
18+# available or are out-of-sync with HW state. Since `noinstr` doesn't always
19+# inhibit KCOV instrumentation, disable it for the entire compilation unit.
20+KCOV_INSTRUMENT_entry.o := n
21+KCOV_INSTRUMENT_idle.o := n
22+
1723 # Object file lists.
1824 obj-y := debug-monitors.o entry.o irq.o fpsimd.o \
1925 entry-common.o entry-fpsimd.o process.o ptrace.o \
2026 setup.o signal.o sys.o stacktrace.o time.o traps.o \
21− io.o vdso.o hyp-stub.o psci.o cpu_ops.o insn.o \
27+ io.o vdso.o hyp-stub.o psci.o cpu_ops.o \
2228 return_address.o cpuinfo.o cpu_errata.o \
2329 cpufeature.o alternative.o cacheinfo.o \
2430 smp.o smp_spin_table.o topology.o smccc-call.o \
25− syscall.o proton-pack.o idreg-override.o
31+ syscall.o proton-pack.o idreg-override.o idle.o \
32+ patching.o
2633
2734 targets += efi-entry.o
2835
arch/arm64/kernel/acpi.c+17−5
@@ -239,6 +239,18 @@ done: · 238 unmodified lines
239239 }
240240 }
241241
242+static pgprot_t __acpi_get_writethrough_mem_attribute(void)
243+{
244+ /*
245+ * Although UEFI specifies the use of Normal Write-through for
246+ * EFI_MEMORY_WT, it is seldom used in practice and not implemented
247+ * by most (all?) CPUs. Rather than allocate a MAIR just for this
248+ * purpose, emit a warning and use Normal Non-cacheable instead.
249+ */
250+ pr_warn_once("No MAIR allocation for EFI_MEMORY_WT; treating as Normal Non-cacheable\n");
251+ return __pgprot(PROT_NORMAL_NC);
252+}
253+
242254 pgprot_t __acpi_get_mem_attribute(phys_addr_t addr)
243255 {
244256 /*
@@ -246,7 +258,7 @@ pgprot_t __acpi_get_mem_attribute(phys_addr_t addr) · 1 unmodified line
246258 * types" of UEFI 2.5 section 2.3.6.1, each EFI memory type is
247259 * mapped to a corresponding MAIR attribute encoding.
248260 * The EFI memory attribute advises all possible capabilities
249− * of a memory region. We use the most efficient capability.
261+ * of a memory region.
250262 */
251263
252264 u64 attr;
@@ -254,10 +266,10 @@ pgprot_t __acpi_get_mem_attribute(phys_addr_t addr) · 1 unmodified line
254266 attr = efi_mem_attributes(addr);
255267 if (attr & EFI_MEMORY_WB)
256268 return PAGE_KERNEL;
257− if (attr & EFI_MEMORY_WT)
258− return __pgprot(PROT_NORMAL_WT);
259269 if (attr & EFI_MEMORY_WC)
260270 return __pgprot(PROT_NORMAL_NC);
271+ if (attr & EFI_MEMORY_WT)
272+ return __acpi_get_writethrough_mem_attribute();
261273 return __pgprot(PROT_DEVICE_nGnRnE);
262274 }
263275
@@ -340,10 +352,10 @@ void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) · 76 unmodified lines
340352 default:
341353 if (region->attribute & EFI_MEMORY_WB)
342354 prot = PAGE_KERNEL;
343− else if (region->attribute & EFI_MEMORY_WT)
344− prot = __pgprot(PROT_NORMAL_WT);
345355 else if (region->attribute & EFI_MEMORY_WC)
346356 prot = __pgprot(PROT_NORMAL_NC);
357+ else if (region->attribute & EFI_MEMORY_WT)
358+ prot = __acpi_get_writethrough_mem_attribute();
347359 }
348360 }
349361 return __ioremap(phys, size, prot);
arch/arm64/kernel/alternative.c+1−1
@@ -181,7 +181,7 @@ static void __nocfi __apply_alternatives(struct alt_region *region, bool is_modu · 180 unmodified lines
181181 */
182182 if (!is_module) {
183183 dsb(ish);
184− __flush_icache_all();
184+ icache_inval_all_pou();
185185 isb();
186186
187187 /* Ignore ARM64_CB bit from feature mask */
arch/arm64/kernel/asm-offsets.c+14−1
@@ -27,6 +27,7 @@ · 26 unmodified lines
2727 int main(void)
2828 {
2929 DEFINE(TSK_ACTIVE_MM, offsetof(struct task_struct, active_mm));
30+ DEFINE(TSK_CPU, offsetof(struct task_struct, cpu));
3031 BLANK();
3132 DEFINE(TSK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags));
3233 DEFINE(TSK_TI_PREEMPT, offsetof(struct task_struct, thread_info.preempt_count));
@@ -46,6 +47,8 @@ int main(void) · 13 unmodified lines
4647 DEFINE(THREAD_SCTLR_USER, offsetof(struct task_struct, thread.sctlr_user));
4748 #ifdef CONFIG_ARM64_PTR_AUTH
4849 DEFINE(THREAD_KEYS_USER, offsetof(struct task_struct, thread.keys_user));
50+#endif
51+#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
4952 DEFINE(THREAD_KEYS_KERNEL, offsetof(struct task_struct, thread.keys_kernel));
5053 #endif
5154 #ifdef CONFIG_ARM64_MTE
@@ -99,7 +102,6 @@ int main(void) · 47 unmodified lines
99102 DEFINE(SOFTIRQ_SHIFT, SOFTIRQ_SHIFT);
100103 DEFINE(IRQ_CPUSTAT_SOFTIRQ_PENDING, offsetof(irq_cpustat_t, __softirq_pending));
101104 BLANK();
102− DEFINE(CPU_BOOT_STACK, offsetof(struct secondary_data, stack));
103105 DEFINE(CPU_BOOT_TASK, offsetof(struct secondary_data, task));
104106 BLANK();
105107 DEFINE(FTR_OVR_VAL_OFFSET, offsetof(struct arm64_ftr_override, val));
@@ -138,6 +140,15 @@ int main(void) · 32 unmodified lines
138140 DEFINE(ARM_SMCCC_RES_X2_OFFS, offsetof(struct arm_smccc_res, a2));
139141 DEFINE(ARM_SMCCC_QUIRK_ID_OFFS, offsetof(struct arm_smccc_quirk, id));
140142 DEFINE(ARM_SMCCC_QUIRK_STATE_OFFS, offsetof(struct arm_smccc_quirk, state));
143+ DEFINE(ARM_SMCCC_1_2_REGS_X0_OFFS, offsetof(struct arm_smccc_1_2_regs, a0));
144+ DEFINE(ARM_SMCCC_1_2_REGS_X2_OFFS, offsetof(struct arm_smccc_1_2_regs, a2));
145+ DEFINE(ARM_SMCCC_1_2_REGS_X4_OFFS, offsetof(struct arm_smccc_1_2_regs, a4));
146+ DEFINE(ARM_SMCCC_1_2_REGS_X6_OFFS, offsetof(struct arm_smccc_1_2_regs, a6));
147+ DEFINE(ARM_SMCCC_1_2_REGS_X8_OFFS, offsetof(struct arm_smccc_1_2_regs, a8));
148+ DEFINE(ARM_SMCCC_1_2_REGS_X10_OFFS, offsetof(struct arm_smccc_1_2_regs, a10));
149+ DEFINE(ARM_SMCCC_1_2_REGS_X12_OFFS, offsetof(struct arm_smccc_1_2_regs, a12));
150+ DEFINE(ARM_SMCCC_1_2_REGS_X14_OFFS, offsetof(struct arm_smccc_1_2_regs, a14));
151+ DEFINE(ARM_SMCCC_1_2_REGS_X16_OFFS, offsetof(struct arm_smccc_1_2_regs, a16));
141152 BLANK();
142153 DEFINE(HIBERN_PBE_ORIG, offsetof(struct pbe, orig_address));
143154 DEFINE(HIBERN_PBE_ADDR, offsetof(struct pbe, address));
@@ -153,7 +164,9 @@ int main(void) · 9 unmodified lines
153164 #endif
154165 #ifdef CONFIG_ARM64_PTR_AUTH
155166 DEFINE(PTRAUTH_USER_KEY_APIA, offsetof(struct ptrauth_keys_user, apia));
167+#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL
156168 DEFINE(PTRAUTH_KERNEL_KEY_APIA, offsetof(struct ptrauth_keys_kernel, apia));
169+#endif
157170 BLANK();
158171 #endif
159172 return 0;
arch/arm64/kernel/cpufeature.c+162−48
@@ -76,6 +76,7 @@ · 75 unmodified lines
7676 #include <asm/cpufeature.h>
7777 #include <asm/cpu_ops.h>
7878 #include <asm/fpsimd.h>
79+#include <asm/insn.h>
7980 #include <asm/kvm_host.h>
8081 #include <asm/mmu_context.h>
8182 #include <asm/mte.h>
@@ -107,6 +108,24 @@ DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE); · 25 unmodified lines
107108 bool arm64_use_ng_mappings = false;
108109 EXPORT_SYMBOL(arm64_use_ng_mappings);
109110
111+/*
112+ * Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs
113+ * support it?
114+ */
115+static bool __read_mostly allow_mismatched_32bit_el0;
116+
117+/*
118+ * Static branch enabled only if allow_mismatched_32bit_el0 is set and we have
119+ * seen at least one CPU capable of 32-bit EL0.
120+ */
121+DEFINE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0);
122+
123+/*
124+ * Mask of CPUs supporting 32-bit EL0.
125+ * Only valid if arm64_mismatched_32bit_el0 is enabled.
126+ */
127+static cpumask_var_t cpu_32bit_el0_mask __cpumask_var_read_mostly;
128+
110129 /*
111130 * Flag to indicate if we have computed the system wide
112131 * capabilities based on the boot time active CPUs. This
@@ -400,6 +419,11 @@ static const struct arm64_ftr_bits ftr_dczid[] = { · 287 unmodified lines
400419 ARM64_FTR_END,
401420 };
402421
422+static const struct arm64_ftr_bits ftr_gmid[] = {
423+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, SYS_GMID_EL1_BS_SHIFT, 4, 0),
424+ ARM64_FTR_END,
425+};
426+
403427 static const struct arm64_ftr_bits ftr_id_isar0[] = {
404428 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DIVIDE_SHIFT, 4, 0),
405429 ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DEBUG_SHIFT, 4, 0),
@@ -617,6 +641,9 @@ static const struct __ftr_reg_entry { · 211 unmodified lines
617641 /* Op1 = 0, CRn = 1, CRm = 2 */
618642 ARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr),
619643
644+ /* Op1 = 1, CRn = 0, CRm = 0 */
645+ ARM64_FTR_REG(SYS_GMID_EL1, ftr_gmid),
646+
620647 /* Op1 = 3, CRn = 0, CRm = 0 */
621648 { SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 },
622649 ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),
@@ -767,7 +794,7 @@ static void __init sort_ftr_regs(void) · 144 unmodified lines
767794 * Any bits that are not covered by an arm64_ftr_bits entry are considered
768795 * RES0 for the system-wide value, and must strictly match.
769796 */
770−static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new)
797+static void init_cpu_ftr_reg(u32 sys_reg, u64 new)
771798 {
772799 u64 val = 0;
773800 u64 strict_mask = ~0x0ULL;
@@ -863,6 +890,31 @@ static void __init init_cpu_hwcaps_indirect_list(void) · 89 unmodified lines
863890
864891 static void __init setup_boot_cpu_capabilities(void);
865892
893+static void init_32bit_cpu_features(struct cpuinfo_32bit *info)
894+{
895+ init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
896+ init_cpu_ftr_reg(SYS_ID_DFR1_EL1, info->reg_id_dfr1);
897+ init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0);
898+ init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1);
899+ init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2);
900+ init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3);
901+ init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4);
902+ init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5);
903+ init_cpu_ftr_reg(SYS_ID_ISAR6_EL1, info->reg_id_isar6);
904+ init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0);
905+ init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1);
906+ init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2);
907+ init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3);
908+ init_cpu_ftr_reg(SYS_ID_MMFR4_EL1, info->reg_id_mmfr4);
909+ init_cpu_ftr_reg(SYS_ID_MMFR5_EL1, info->reg_id_mmfr5);
910+ init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0);
911+ init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1);
912+ init_cpu_ftr_reg(SYS_ID_PFR2_EL1, info->reg_id_pfr2);
913+ init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0);
914+ init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1);
915+ init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
916+}
917+
866918 void __init init_cpu_features(struct cpuinfo_arm64 *info)
867919 {
868920 /* Before we start using the tables, make sure it is sorted */
@@ -882,35 +934,17 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) · 13 unmodified lines
882934 init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);
883935 init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0);
884936
885− if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
886− init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
887− init_cpu_ftr_reg(SYS_ID_DFR1_EL1, info->reg_id_dfr1);
888− init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0);
889− init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1);
890− init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2);
891− init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3);
892− init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4);
893− init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5);
894− init_cpu_ftr_reg(SYS_ID_ISAR6_EL1, info->reg_id_isar6);
895− init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0);
896− init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1);
897− init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2);
898− init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3);
899− init_cpu_ftr_reg(SYS_ID_MMFR4_EL1, info->reg_id_mmfr4);
900− init_cpu_ftr_reg(SYS_ID_MMFR5_EL1, info->reg_id_mmfr5);
901− init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0);
902− init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1);
903− init_cpu_ftr_reg(SYS_ID_PFR2_EL1, info->reg_id_pfr2);
904− init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0);
905− init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1);
906− init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
907− }
937+ if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0))
938+ init_32bit_cpu_features(&info->aarch32);
908939
909940 if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) {
910941 init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr);
911942 sve_init_vq_map();
912943 }
913944
945+ if (id_aa64pfr1_mte(info->reg_id_aa64pfr1))
946+ init_cpu_ftr_reg(SYS_GMID_EL1, info->reg_gmid);
947+
914948 /*
915949 * Initialize the indirect array of CPU hwcaps capabilities pointers
916950 * before we handle the boot CPU below.
@@ -975,20 +1009,28 @@ static void relax_cpu_ftr_reg(u32 sys_id, int field) · 58 unmodified lines
9751009 WARN_ON(!ftrp->width);
9761010 }
9771011
978−static int update_32bit_cpu_features(int cpu, struct cpuinfo_arm64 *info,
979− struct cpuinfo_arm64 *boot)
1012+static void lazy_init_32bit_cpu_features(struct cpuinfo_arm64 *info,
1013+ struct cpuinfo_arm64 *boot)
1014+{
1015+ static bool boot_cpu_32bit_regs_overridden = false;
1016+
1017+ if (!allow_mismatched_32bit_el0 || boot_cpu_32bit_regs_overridden)
1018+ return;
1019+
1020+ if (id_aa64pfr0_32bit_el0(boot->reg_id_aa64pfr0))
1021+ return;
1022+
1023+ boot->aarch32 = info->aarch32;
1024+ init_32bit_cpu_features(&boot->aarch32);
1025+ boot_cpu_32bit_regs_overridden = true;
1026+}
1027+
1028+static int update_32bit_cpu_features(int cpu, struct cpuinfo_32bit *info,
1029+ struct cpuinfo_32bit *boot)
9801030 {
9811031 int taint = 0;
9821032 u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
9831033
984− /*
985− * If we don't have AArch32 at all then skip the checks entirely
986− * as the register values may be UNKNOWN and we're not going to be
987− * using them for anything.
988− */
989− if (!id_aa64pfr0_32bit_el0(pfr0))
990− return taint;
991−
9921034 /*
9931035 * If we don't have AArch32 at EL1, then relax the strictness of
9941036 * EL1-dependent register fields to avoid spurious sanity check fails.
@@ -1135,10 +1177,29 @@ void update_cpu_features(int cpu, · 140 unmodified lines
11351177 }
11361178
11371179 /*
1180+ * The kernel uses the LDGM/STGM instructions and the number of tags
1181+ * they read/write depends on the GMID_EL1.BS field. Check that the
1182+ * value is the same on all CPUs.
1183+ */
1184+ if (IS_ENABLED(CONFIG_ARM64_MTE) &&
1185+ id_aa64pfr1_mte(info->reg_id_aa64pfr1)) {
1186+ taint |= check_update_ftr_reg(SYS_GMID_EL1, cpu,
1187+ info->reg_gmid, boot->reg_gmid);
1188+ }
1189+
1190+ /*
1191+ * If we don't have AArch32 at all then skip the checks entirely
1192+ * as the register values may be UNKNOWN and we're not going to be
1193+ * using them for anything.
1194+ *
11381195 * This relies on a sanitised view of the AArch64 ID registers
11391196 * (e.g. SYS_ID_AA64PFR0_EL1), so we call it last.
11401197 */
1141− taint |= update_32bit_cpu_features(cpu, info, boot);
1198+ if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
1199+ lazy_init_32bit_cpu_features(info, boot);
1200+ taint |= update_32bit_cpu_features(cpu, &info->aarch32,
1201+ &boot->aarch32);
1202+ }
11421203
11431204 /*
11441205 * Mismatched CPU features are a recipe for disaster. Don't even
@@ -1248,6 +1309,28 @@ has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) · 103 unmodified lines
12481309 return feature_matches(val, entry);
12491310 }
12501311
1312+const struct cpumask *system_32bit_el0_cpumask(void)
1313+{
1314+ if (!system_supports_32bit_el0())
1315+ return cpu_none_mask;
1316+
1317+ if (static_branch_unlikely(&arm64_mismatched_32bit_el0))
1318+ return cpu_32bit_el0_mask;
1319+
1320+ return cpu_possible_mask;
1321+}
1322+
1323+static bool has_32bit_el0(const struct arm64_cpu_capabilities *entry, int scope)
1324+{
1325+ if (!has_cpuid_feature(entry, scope))
1326+ return allow_mismatched_32bit_el0;
1327+
1328+ if (scope == SCOPE_SYSTEM)
1329+ pr_info("detected: 32-bit EL0 Support\n");
1330+
1331+ return true;
1332+}
1333+
12511334 static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, int scope)
12521335 {
12531336 bool has_sre;
@@ -1866,10 +1949,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 612 unmodified lines
18661949 .cpu_enable = cpu_copy_el2regs,
18671950 },
18681951 {
1869− .desc = "32-bit EL0 Support",
1870− .capability = ARM64_HAS_32BIT_EL0,
1952+ .capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE,
18711953 .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1872− .matches = has_cpuid_feature,
1954+ .matches = has_32bit_el0,
18731955 .sys_reg = SYS_ID_AA64PFR0_EL1,
18741956 .sign = FTR_UNSIGNED,
18751957 .field_pos = ID_AA64PFR0_EL0_SHIFT,
@@ -2378,7 +2460,7 @@ static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = { · 502 unmodified lines
23782460 {},
23792461 };
23802462
2381−static void __init cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap)
2463+static void cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap)
23822464 {
23832465 switch (cap->hwcap_type) {
23842466 case CAP_HWCAP:
@@ -2423,7 +2505,7 @@ static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap) · 38 unmodified lines
24232505 return rc;
24242506 }
24252507
2426−static void __init setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps)
2508+static void setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps)
24272509 {
24282510 /* We support emulation of accesses to CPU ID feature registers */
24292511 cpu_set_named_feature(CPUID);
@@ -2598,7 +2680,7 @@ static void check_early_cpu_features(void) · 168 unmodified lines
25982680 }
25992681
26002682 static void
2601−verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps)
2683+__verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps)
26022684 {
26032685
26042686 for (; caps->matches; caps++)
@@ -2609,6 +2691,14 @@ verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps) · 4 unmodified lines
26092691 }
26102692 }
26112693
2694+static void verify_local_elf_hwcaps(void)
2695+{
2696+ __verify_local_elf_hwcaps(arm64_elf_hwcaps);
2697+
2698+ if (id_aa64pfr0_32bit_el0(read_cpuid(ID_AA64PFR0_EL1)))
2699+ __verify_local_elf_hwcaps(compat_elf_hwcaps);
2700+}
2701+
26122702 static void verify_sve_features(void)
26132703 {
26142704 u64 safe_zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
@@ -2673,11 +2763,7 @@ static void verify_local_cpu_capabilities(void) · 58 unmodified lines
26732763 * on all secondary CPUs.
26742764 */
26752765 verify_local_cpu_caps(SCOPE_ALL & ~SCOPE_BOOT_CPU);
2676−
2677− verify_local_elf_hwcaps(arm64_elf_hwcaps);
2678−
2679− if (system_supports_32bit_el0())
2680− verify_local_elf_hwcaps(compat_elf_hwcaps);
2766+ verify_local_elf_hwcaps();
26812767
26822768 if (system_supports_sve())
26832769 verify_sve_features();
@@ -2812,6 +2898,34 @@ void __init setup_cpu_features(void) · 128 unmodified lines
28122898 ARCH_DMA_MINALIGN);
28132899 }
28142900
2901+static int enable_mismatched_32bit_el0(unsigned int cpu)
2902+{
2903+ struct cpuinfo_arm64 *info = &per_cpu(cpu_data, cpu);
2904+ bool cpu_32bit = id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0);
2905+
2906+ if (cpu_32bit) {
2907+ cpumask_set_cpu(cpu, cpu_32bit_el0_mask);
2908+ static_branch_enable_cpuslocked(&arm64_mismatched_32bit_el0);
2909+ setup_elf_hwcaps(compat_elf_hwcaps);
2910+ }
2911+
2912+ return 0;
2913+}
2914+
2915+static int __init init_32bit_el0_mask(void)
2916+{
2917+ if (!allow_mismatched_32bit_el0)
2918+ return 0;
2919+
2920+ if (!zalloc_cpumask_var(&cpu_32bit_el0_mask, GFP_KERNEL))
2921+ return -ENOMEM;
2922+
2923+ return cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
2924+ "arm64/mismatched_32bit_el0:online",
2925+ enable_mismatched_32bit_el0, NULL);
2926+}
2927+subsys_initcall_sync(init_32bit_el0_mask);
2928+
28152929 static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap)
28162930 {
28172931 cpu_replace_ttbr1(lm_alias(swapper_pg_dir));
@@ -2905,8 +3019,8 @@ static int emulate_mrs(struct pt_regs *regs, u32 insn) · 87 unmodified lines
29053019 }
29063020
29073021 static struct undef_hook mrs_hook = {
2908− .instr_mask = 0xfff00000,
2909− .instr_val = 0xd5300000,
3022+ .instr_mask = 0xffff0000,
3023+ .instr_val = 0xd5380000,
29103024 .pstate_mask = PSR_AA32_MODE_MASK,
29113025 .pstate_val = PSR_MODE_EL0t,
29123026 .fn = emulate_mrs,
arch/arm64/kernel/cpuinfo.c+32−26
@@ -246,7 +246,7 @@ static struct kobj_type cpuregs_kobj_type = { · 245 unmodified lines
246246 struct cpuinfo_arm64 *info = kobj_to_cpuinfo(kobj); \
247247 \
248248 if (info->reg_midr) \
249− return sprintf(buf, "0x%016x\n", info->reg_##_field); \
249+ return sprintf(buf, "0x%016llx\n", info->reg_##_field); \
250250 else \
251251 return 0; \
252252 } \
@@ -344,6 +344,32 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info) · 91 unmodified lines
344344 pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu);
345345 }
346346
347+static void __cpuinfo_store_cpu_32bit(struct cpuinfo_32bit *info)
348+{
349+ info->reg_id_dfr0 = read_cpuid(ID_DFR0_EL1);
350+ info->reg_id_dfr1 = read_cpuid(ID_DFR1_EL1);
351+ info->reg_id_isar0 = read_cpuid(ID_ISAR0_EL1);
352+ info->reg_id_isar1 = read_cpuid(ID_ISAR1_EL1);
353+ info->reg_id_isar2 = read_cpuid(ID_ISAR2_EL1);
354+ info->reg_id_isar3 = read_cpuid(ID_ISAR3_EL1);
355+ info->reg_id_isar4 = read_cpuid(ID_ISAR4_EL1);
356+ info->reg_id_isar5 = read_cpuid(ID_ISAR5_EL1);
357+ info->reg_id_isar6 = read_cpuid(ID_ISAR6_EL1);
358+ info->reg_id_mmfr0 = read_cpuid(ID_MMFR0_EL1);
359+ info->reg_id_mmfr1 = read_cpuid(ID_MMFR1_EL1);
360+ info->reg_id_mmfr2 = read_cpuid(ID_MMFR2_EL1);
361+ info->reg_id_mmfr3 = read_cpuid(ID_MMFR3_EL1);
362+ info->reg_id_mmfr4 = read_cpuid(ID_MMFR4_EL1);
363+ info->reg_id_mmfr5 = read_cpuid(ID_MMFR5_EL1);
364+ info->reg_id_pfr0 = read_cpuid(ID_PFR0_EL1);
365+ info->reg_id_pfr1 = read_cpuid(ID_PFR1_EL1);
366+ info->reg_id_pfr2 = read_cpuid(ID_PFR2_EL1);
367+
368+ info->reg_mvfr0 = read_cpuid(MVFR0_EL1);
369+ info->reg_mvfr1 = read_cpuid(MVFR1_EL1);
370+ info->reg_mvfr2 = read_cpuid(MVFR2_EL1);
371+}
372+
347373 static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
348374 {
349375 info->reg_cntfrq = arch_timer_get_cntfrq();
@@ -371,31 +397,11 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) · 21 unmodified lines
371397 info->reg_id_aa64pfr1 = read_cpuid(ID_AA64PFR1_EL1);
372398 info->reg_id_aa64zfr0 = read_cpuid(ID_AA64ZFR0_EL1);
373399
374− /* Update the 32bit ID registers only if AArch32 is implemented */
375− if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
376− info->reg_id_dfr0 = read_cpuid(ID_DFR0_EL1);
377− info->reg_id_dfr1 = read_cpuid(ID_DFR1_EL1);
378− info->reg_id_isar0 = read_cpuid(ID_ISAR0_EL1);
379− info->reg_id_isar1 = read_cpuid(ID_ISAR1_EL1);
380− info->reg_id_isar2 = read_cpuid(ID_ISAR2_EL1);
381− info->reg_id_isar3 = read_cpuid(ID_ISAR3_EL1);
382− info->reg_id_isar4 = read_cpuid(ID_ISAR4_EL1);
383− info->reg_id_isar5 = read_cpuid(ID_ISAR5_EL1);
384− info->reg_id_isar6 = read_cpuid(ID_ISAR6_EL1);
385− info->reg_id_mmfr0 = read_cpuid(ID_MMFR0_EL1);
386− info->reg_id_mmfr1 = read_cpuid(ID_MMFR1_EL1);
387− info->reg_id_mmfr2 = read_cpuid(ID_MMFR2_EL1);
388− info->reg_id_mmfr3 = read_cpuid(ID_MMFR3_EL1);
389− info->reg_id_mmfr4 = read_cpuid(ID_MMFR4_EL1);
390− info->reg_id_mmfr5 = read_cpuid(ID_MMFR5_EL1);
391− info->reg_id_pfr0 = read_cpuid(ID_PFR0_EL1);
392− info->reg_id_pfr1 = read_cpuid(ID_PFR1_EL1);
393− info->reg_id_pfr2 = read_cpuid(ID_PFR2_EL1);
394−
395− info->reg_mvfr0 = read_cpuid(MVFR0_EL1);
396− info->reg_mvfr1 = read_cpuid(MVFR1_EL1);
397− info->reg_mvfr2 = read_cpuid(MVFR2_EL1);
398− }
400+ if (id_aa64pfr1_mte(info->reg_id_aa64pfr1))
401+ info->reg_gmid = read_cpuid(GMID_EL1);
402+
403+ if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0))
404+ __cpuinfo_store_cpu_32bit(&info->aarch32);
399405
400406 if (IS_ENABLED(CONFIG_ARM64_SVE) &&
401407 id_aa64pfr0_sve(info->reg_id_aa64pfr0))
arch/arm64/kernel/efi-entry.S+5−4
@@ -28,7 +28,8 @@ SYM_CODE_START(efi_enter_kernel) · 27 unmodified lines
2828 * stale icache entries from before relocation.
2929 */
3030 ldr w1, =kernel_size
31− bl __clean_dcache_area_poc
31+ add x1, x0, x1
32+ bl dcache_clean_poc
3233 ic ialluis
3334
3435 /*
@@ -36,8 +37,8 @@ SYM_CODE_START(efi_enter_kernel) · 1 unmodified line
3637 * so that we can safely disable the MMU and caches.
3738 */
3839 adr x0, 0f
39− ldr w1, 3f
40− bl __clean_dcache_area_poc
40+ adr x1, 3f
41+ bl dcache_clean_poc
4142 0:
4243 /* Turn off Dcache and MMU */
4344 mrs x0, CurrentEL
@@ -64,5 +65,5 @@ SYM_CODE_START(efi_enter_kernel) · 20 unmodified lines
6465 mov x2, xzr
6566 mov x3, xzr
6667 br x19
68+3:
6769 SYM_CODE_END(efi_enter_kernel)
68−3: .long . - 0b
arch/arm64/kernel/entry-common.c+238−18
@@ -6,7 +6,11 @@ · 5 unmodified lines
66 */
77
88 #include <linux/context_tracking.h>
9+#include <linux/linkage.h>
10+#include <linux/lockdep.h>
911 #include <linux/ptrace.h>
12+#include <linux/sched.h>
13+#include <linux/sched/debug.h>
1014 #include <linux/thread_info.h>
1115
1216 #include <asm/cpufeature.h>
@@ -15,7 +19,11 @@ · 2 unmodified lines
1519 #include <asm/exception.h>
1620 #include <asm/kprobes.h>
1721 #include <asm/mmu.h>
22+#include <asm/processor.h>
23+#include <asm/sdei.h>
24+#include <asm/stacktrace.h>
1825 #include <asm/sysreg.h>
26+#include <asm/system_misc.h>
1927
2028 /*
2129 * This is intended to match the logic in irqentry_enter(), handling the kernel
@@ -67,7 +75,7 @@ static void noinstr exit_to_kernel_mode(struct pt_regs *regs) · 45 unmodified lines
6775 }
6876 }
6977
70−void noinstr arm64_enter_nmi(struct pt_regs *regs)
78+static void noinstr arm64_enter_nmi(struct pt_regs *regs)
7179 {
7280 regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
7381
@@ -80,7 +88,7 @@ void noinstr arm64_enter_nmi(struct pt_regs *regs) · 6 unmodified lines
8088 ftrace_nmi_enter();
8189 }
8290
83−void noinstr arm64_exit_nmi(struct pt_regs *regs)
91+static void noinstr arm64_exit_nmi(struct pt_regs *regs)
8492 {
8593 bool restore = regs->lockdep_hardirqs;
8694
@@ -97,7 +105,7 @@ void noinstr arm64_exit_nmi(struct pt_regs *regs) · 10 unmodified lines
97105 __nmi_exit();
98106 }
99107
100−asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)
108+static void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs)
101109 {
102110 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
103111 arm64_enter_nmi(regs);
@@ -105,7 +113,7 @@ asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs) · 1 unmodified line
105113 enter_from_kernel_mode(regs);
106114 }
107115
108−asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)
116+static void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs)
109117 {
110118 if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs))
111119 arm64_exit_nmi(regs);
@@ -113,6 +121,65 @@ asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs) · 1 unmodified line
113121 exit_to_kernel_mode(regs);
114122 }
115123
124+static void __sched arm64_preempt_schedule_irq(void)
125+{
126+ lockdep_assert_irqs_disabled();
127+
128+ /*
129+ * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC
130+ * priority masking is used the GIC irqchip driver will clear DAIF.IF
131+ * using gic_arch_enable_irqs() for normal IRQs. If anything is set in
132+ * DAIF we must have handled an NMI, so skip preemption.
133+ */
134+ if (system_uses_irq_prio_masking() && read_sysreg(daif))
135+ return;
136+
137+ /*
138+ * Preempting a task from an IRQ means we leave copies of PSTATE
139+ * on the stack. cpufeature's enable calls may modify PSTATE, but
140+ * resuming one of these preempted tasks would undo those changes.
141+ *
142+ * Only allow a task to be preempted once cpufeatures have been
143+ * enabled.
144+ */
145+ if (system_capabilities_finalized())
146+ preempt_schedule_irq();
147+}
148+
149+static void do_interrupt_handler(struct pt_regs *regs,
150+ void (*handler)(struct pt_regs *))
151+{
152+ if (on_thread_stack())
153+ call_on_irq_stack(regs, handler);
154+ else
155+ handler(regs);
156+}
157+
158+extern void (*handle_arch_irq)(struct pt_regs *);
159+extern void (*handle_arch_fiq)(struct pt_regs *);
160+
161+static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector,
162+ unsigned int esr)
163+{
164+ arm64_enter_nmi(regs);
165+
166+ console_verbose();
167+
168+ pr_crit("Unhandled %s exception on CPU%d, ESR 0x%08x -- %s\n",
169+ vector, smp_processor_id(), esr,
170+ esr_get_class_string(esr));
171+
172+ __show_regs(regs);
173+ panic("Unhandled exception");
174+}
175+
176+#define UNHANDLED(el, regsize, vector) \
177+asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \
178+{ \
179+ const char *desc = #regsize "-bit " #el " " #vector; \
180+ __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \
181+}
182+
116183 #ifdef CONFIG_ARM64_ERRATUM_1463225
117184 static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);
118185
@@ -162,6 +229,11 @@ static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) · 43 unmodified lines
162229 }
163230 #endif /* CONFIG_ARM64_ERRATUM_1463225 */
164231
232+UNHANDLED(el1t, 64, sync)
233+UNHANDLED(el1t, 64, irq)
234+UNHANDLED(el1t, 64, fiq)
235+UNHANDLED(el1t, 64, error)
236+
165237 static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr)
166238 {
167239 unsigned long far = read_sysreg(far_el1);
@@ -193,15 +265,6 @@ static void noinstr el1_undef(struct pt_regs *regs) · 25 unmodified lines
193265 exit_to_kernel_mode(regs);
194266 }
195267
196−static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr)
197−{
198− enter_from_kernel_mode(regs);
199− local_daif_inherit(regs);
200− bad_mode(regs, 0, esr);
201− local_daif_mask();
202− exit_to_kernel_mode(regs);
203−}
204−
205268 static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs)
206269 {
207270 regs->lockdep_hardirqs = lockdep_hardirqs_enabled();
@@ -245,7 +308,7 @@ static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) · 37 unmodified lines
245308 exit_to_kernel_mode(regs);
246309 }
247310
248−asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs)
311+asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs)
249312 {
250313 unsigned long esr = read_sysreg(esr_el1);
251314
@@ -275,10 +338,50 @@ asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs) · 23 unmodified lines
275338 el1_fpac(regs, esr);
276339 break;
277340 default:
278− el1_inv(regs, esr);
341+ __panic_unhandled(regs, "64-bit el1h sync", esr);
279342 }
280343 }
281344
345+static void noinstr el1_interrupt(struct pt_regs *regs,
346+ void (*handler)(struct pt_regs *))
347+{
348+ write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
349+
350+ enter_el1_irq_or_nmi(regs);
351+ do_interrupt_handler(regs, handler);
352+
353+ /*
354+ * Note: thread_info::preempt_count includes both thread_info::count
355+ * and thread_info::need_resched, and is not equivalent to
356+ * preempt_count().
357+ */
358+ if (IS_ENABLED(CONFIG_PREEMPTION) &&
359+ READ_ONCE(current_thread_info()->preempt_count) == 0)
360+ arm64_preempt_schedule_irq();
361+
362+ exit_el1_irq_or_nmi(regs);
363+}
364+
365+asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs)
366+{
367+ el1_interrupt(regs, handle_arch_irq);
368+}
369+
370+asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs)
371+{
372+ el1_interrupt(regs, handle_arch_fiq);
373+}
374+
375+asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs)
376+{
377+ unsigned long esr = read_sysreg(esr_el1);
378+
379+ local_daif_restore(DAIF_ERRCTX);
380+ arm64_enter_nmi(regs);
381+ do_serror(regs, esr);
382+ arm64_exit_nmi(regs);
383+}
384+
282385 asmlinkage void noinstr enter_from_user_mode(void)
283386 {
284387 lockdep_hardirqs_off(CALLER_ADDR0);
@@ -398,7 +501,7 @@ static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr) · 113 unmodified lines
398501
399502 enter_from_user_mode();
400503 do_debug_exception(far, esr, regs);
401− local_daif_restore(DAIF_PROCCTX_NOIRQ);
504+ local_daif_restore(DAIF_PROCCTX);
402505 }
403506
404507 static void noinstr el0_svc(struct pt_regs *regs)
@@ -415,7 +518,7 @@ static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) · 10 unmodified lines
415518 do_ptrauth_fault(regs, esr);
416519 }
417520
418−asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs)
521+asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs)
419522 {
420523 unsigned long esr = read_sysreg(esr_el1);
421524
@@ -468,6 +571,56 @@ asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs) · 46 unmodified lines
468571 }
469572 }
470573
574+static void noinstr el0_interrupt(struct pt_regs *regs,
575+ void (*handler)(struct pt_regs *))
576+{
577+ enter_from_user_mode();
578+
579+ write_sysreg(DAIF_PROCCTX_NOIRQ, daif);
580+
581+ if (regs->pc & BIT(55))
582+ arm64_apply_bp_hardening();
583+
584+ do_interrupt_handler(regs, handler);
585+}
586+
587+static void noinstr __el0_irq_handler_common(struct pt_regs *regs)
588+{
589+ el0_interrupt(regs, handle_arch_irq);
590+}
591+
592+asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs)
593+{
594+ __el0_irq_handler_common(regs);
595+}
596+
597+static void noinstr __el0_fiq_handler_common(struct pt_regs *regs)
598+{
599+ el0_interrupt(regs, handle_arch_fiq);
600+}
601+
602+asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs)
603+{
604+ __el0_fiq_handler_common(regs);
605+}
606+
607+static void __el0_error_handler_common(struct pt_regs *regs)
608+{
609+ unsigned long esr = read_sysreg(esr_el1);
610+
611+ enter_from_user_mode();
612+ local_daif_restore(DAIF_ERRCTX);
613+ arm64_enter_nmi(regs);
614+ do_serror(regs, esr);
615+ arm64_exit_nmi(regs);
616+ local_daif_restore(DAIF_PROCCTX);
617+}
618+
619+asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs)
620+{
621+ __el0_error_handler_common(regs);
622+}
623+
471624 #ifdef CONFIG_COMPAT
472625 static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr)
473626 {
@@ -483,7 +636,7 @@ static void noinstr el0_svc_compat(struct pt_regs *regs) · 9 unmodified lines
483636 do_el0_svc_compat(regs);
484637 }
485638
486−asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs)
639+asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs)
487640 {
488641 unsigned long esr = read_sysreg(esr_el1);
489642
@@ -526,4 +679,71 @@ asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs) · 36 unmodified lines
526679 el0_inv(regs, esr);
527680 }
528681 }
682+
683+asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs)
684+{
685+ __el0_irq_handler_common(regs);
686+}
687+
688+asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs)
689+{
690+ __el0_fiq_handler_common(regs);
691+}
692+
693+asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs)
694+{
695+ __el0_error_handler_common(regs);
696+}
697+#else /* CONFIG_COMPAT */
698+UNHANDLED(el0t, 32, sync)
699+UNHANDLED(el0t, 32, irq)
700+UNHANDLED(el0t, 32, fiq)
701+UNHANDLED(el0t, 32, error)
529702 #endif /* CONFIG_COMPAT */
703+
704+#ifdef CONFIG_VMAP_STACK
705+asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs)
706+{
707+ unsigned int esr = read_sysreg(esr_el1);
708+ unsigned long far = read_sysreg(far_el1);
709+
710+ arm64_enter_nmi(regs);
711+ panic_bad_stack(regs, esr, far);
712+}
713+#endif /* CONFIG_VMAP_STACK */
714+
715+#ifdef CONFIG_ARM_SDE_INTERFACE
716+asmlinkage noinstr unsigned long
717+__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
718+{
719+ unsigned long ret;
720+
721+ /*
722+ * We didn't take an exception to get here, so the HW hasn't
723+ * set/cleared bits in PSTATE that we may rely on.
724+ *
725+ * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to
726+ * whether PSTATE bits are inherited unchanged or generated from
727+ * scratch, and the TF-A implementation always clears PAN and always
728+ * clears UAO. There are no other known implementations.
729+ *
730+ * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how
731+ * PSTATE is modified upon architectural exceptions, and so PAN is
732+ * either inherited or set per SCTLR_ELx.SPAN, and UAO is always
733+ * cleared.
734+ *
735+ * We must explicitly reset PAN to the expected state, including
736+ * clearing it when the host isn't using it, in case a VM had it set.
737+ */
738+ if (system_uses_hw_pan())
739+ set_pstate_pan(1);
740+ else if (cpu_has_pan())
741+ set_pstate_pan(0);
742+
743+ arm64_enter_nmi(regs);
744+ ret = do_sdei_event(regs, arg);
745+ arm64_exit_nmi(regs);
746+
747+ return ret;
748+}
749+#endif /* CONFIG_ARM_SDE_INTERFACE */
arch/arm64/kernel/entry-fpsimd.S+15−7
@@ -63,16 +63,24 @@ SYM_FUNC_END(sve_set_vq) · 62 unmodified lines
6363 * and the rest zeroed. All the other SVE registers will be zeroed.
6464 */
6565 SYM_FUNC_START(sve_load_from_fpsimd_state)
66− sve_load_vq x1, x2, x3
67− fpsimd_restore x0, 8
68− _for n, 0, 15, _sve_pfalse \n
69− _sve_wrffr 0
70− ret
66+ sve_load_vq x1, x2, x3
67+ fpsimd_restore x0, 8
68+ sve_flush_p_ffr
69+ ret
7170 SYM_FUNC_END(sve_load_from_fpsimd_state)
7271
73−/* Zero all SVE registers but the first 128-bits of each vector */
72+/*
73+ * Zero all SVE registers but the first 128-bits of each vector
74+ *
75+ * VQ must already be configured by caller, any further updates of VQ
76+ * will need to ensure that the register state remains valid.
77+ *
78+ * x0 = VQ - 1
79+ */
7480 SYM_FUNC_START(sve_flush_live)
75− sve_flush
81+ cbz x0, 1f // A VQ-1 of 0 is 128 bits so no extra Z state
82+ sve_flush_z
83+1: sve_flush_p_ffr
7684 ret
7785 SYM_FUNC_END(sve_flush_live)
7886
arch/arm64/kernel/entry.S+96−273
@@ -33,12 +33,6 @@ · 32 unmodified lines
3333 * Context tracking and irqflag tracing need to instrument transitions between
3434 * user and kernel mode.
3535 */
36− .macro user_exit_irqoff
37−#if defined(CONFIG_CONTEXT_TRACKING) || defined(CONFIG_TRACE_IRQFLAGS)
38− bl enter_from_user_mode
39−#endif
40− .endm
41−
4236 .macro user_enter_irqoff
4337 #if defined(CONFIG_CONTEXT_TRACKING) || defined(CONFIG_TRACE_IRQFLAGS)
4438 bl exit_to_user_mode
@@ -51,16 +45,7 @@ · 6 unmodified lines
5145 .endr
5246 .endm
5347
54−/*
55− * Bad Abort numbers
56− *-----------------
57− */
58−#define BAD_SYNC 0
59−#define BAD_IRQ 1
60−#define BAD_FIQ 2
61−#define BAD_ERROR 3
62−
63− .macro kernel_ventry, el, label, regsize = 64
48+ .macro kernel_ventry, el:req, ht:req, regsize:req, label:req
6449 .align 7
6550 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
6651 .if \el == 0
@@ -87,7 +72,7 @@ alternative_else_nop_endif · 20 unmodified lines
8772 tbnz x0, #THREAD_SHIFT, 0f
8873 sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0
8974 sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp
90− b el\()\el\()_\label
75+ b el\el\ht\()_\regsize\()_\label
9176
9277 0:
9378 /*
@@ -119,7 +104,7 @@ alternative_else_nop_endif · 25 unmodified lines
119104 sub sp, sp, x0
120105 mrs x0, tpidrro_el0
121106 #endif
122− b el\()\el\()_\label
107+ b el\el\ht\()_\regsize\()_\label
123108 .endm
124109
125110 .macro tramp_alias, dst, sym
@@ -275,7 +260,7 @@ alternative_else_nop_endif · 149 unmodified lines
275260
276261 mte_set_kernel_gcr x22, x23
277262
278− scs_load tsk, x20
263+ scs_load tsk
279264 .else
280265 add x21, sp, #PT_REGS_SIZE
281266 get_current_task tsk
@@ -285,7 +270,7 @@ alternative_else_nop_endif · 3 unmodified lines
285270 stp lr, x21, [sp, #S_LR]
286271
287272 /*
288− * For exceptions from EL0, create a terminal frame record.
273+ * For exceptions from EL0, create a final frame record.
289274 * For exceptions from EL1, create a synthetic frame record so the
290275 * interrupted code shows up in the backtrace.
291276 */
@@ -375,7 +360,7 @@ alternative_if ARM64_WORKAROUND_845719 · 83 unmodified lines
375360 alternative_else_nop_endif
376361 #endif
377362 3:
378− scs_save tsk, x0
363+ scs_save tsk
379364
380365 #ifdef CONFIG_ARM64_PTR_AUTH
381366 alternative_if ARM64_HAS_ADDRESS_AUTH
@@ -486,63 +471,12 @@ SYM_CODE_START_LOCAL(__swpan_exit_el0) · 104 unmodified lines
486471 SYM_CODE_END(__swpan_exit_el0)
487472 #endif
488473
489− .macro irq_stack_entry
490− mov x19, sp // preserve the original sp
491−#ifdef CONFIG_SHADOW_CALL_STACK
492− mov x24, scs_sp // preserve the original shadow stack
493−#endif
494−
495− /*
496− * Compare sp with the base of the task stack.
497− * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack,
498− * and should switch to the irq stack.
499− */
500− ldr x25, [tsk, TSK_STACK]
501− eor x25, x25, x19
502− and x25, x25, #~(THREAD_SIZE - 1)
503− cbnz x25, 9998f
504−
505− ldr_this_cpu x25, irq_stack_ptr, x26
506− mov x26, #IRQ_STACK_SIZE
507− add x26, x25, x26
508−
509− /* switch to the irq stack */
510− mov sp, x26
511−
512−#ifdef CONFIG_SHADOW_CALL_STACK
513− /* also switch to the irq shadow stack */
514− ldr_this_cpu scs_sp, irq_shadow_call_stack_ptr, x26
515−#endif
516−
517−9998:
518− .endm
519−
520− /*
521− * The callee-saved regs (x19-x29) should be preserved between
522− * irq_stack_entry and irq_stack_exit, but note that kernel_entry
523− * uses x20-x23 to store data for later use.
524− */
525− .macro irq_stack_exit
526− mov sp, x19
527−#ifdef CONFIG_SHADOW_CALL_STACK
528− mov scs_sp, x24
529−#endif
530− .endm
531−
532474 /* GPRs used by entry code */
533475 tsk .req x28 // current thread_info
534476
535477 /*
536478 * Interrupt handling.
537479 */
538− .macro irq_handler, handler:req
539− ldr_l x1, \handler
540− mov x0, sp
541− irq_stack_entry
542− blr x1
543− irq_stack_exit
544− .endm
545−
546480 .macro gic_prio_kentry_setup, tmp:req
547481 #ifdef CONFIG_ARM64_PSEUDO_NMI
548482 alternative_if ARM64_HAS_IRQ_PRIO_MASKING
@@ -552,45 +486,6 @@ tsk .req x28 // current thread_info · 3 unmodified lines
552486 #endif
553487 .endm
554488
555− .macro el1_interrupt_handler, handler:req
556− enable_da
557−
558− mov x0, sp
559− bl enter_el1_irq_or_nmi
560−
561− irq_handler \handler
562−
563−#ifdef CONFIG_PREEMPTION
564− ldr x24, [tsk, #TSK_TI_PREEMPT] // get preempt count
565−alternative_if ARM64_HAS_IRQ_PRIO_MASKING
566− /*
567− * DA were cleared at start of handling, and IF are cleared by
568− * the GIC irqchip driver using gic_arch_enable_irqs() for
569− * normal IRQs. If anything is set, it means we come back from
570− * an NMI instead of a normal IRQ, so skip preemption
571− */
572− mrs x0, daif
573− orr x24, x24, x0
574−alternative_else_nop_endif
575− cbnz x24, 1f // preempt count != 0 || NMI return path
576− bl arm64_preempt_schedule_irq // irq en/disable is done inside
577−1:
578−#endif
579−
580− mov x0, sp
581− bl exit_el1_irq_or_nmi
582− .endm
583−
584− .macro el0_interrupt_handler, handler:req
585− user_exit_irqoff
586− enable_da
587−
588− tbz x22, #55, 1f
589− bl do_el0_irq_bp_hardening
590−1:
591− irq_handler \handler
592− .endm
593−
594489 .text
595490
596491 /*
@@ -600,32 +495,25 @@ alternative_else_nop_endif · 3 unmodified lines
600495
601496 .align 11
602497 SYM_CODE_START(vectors)
603− kernel_ventry 1, sync_invalid // Synchronous EL1t
604− kernel_ventry 1, irq_invalid // IRQ EL1t
605− kernel_ventry 1, fiq_invalid // FIQ EL1t
606− kernel_ventry 1, error_invalid // Error EL1t
607−
608− kernel_ventry 1, sync // Synchronous EL1h
609− kernel_ventry 1, irq // IRQ EL1h
610− kernel_ventry 1, fiq // FIQ EL1h
611− kernel_ventry 1, error // Error EL1h
612−
613− kernel_ventry 0, sync // Synchronous 64-bit EL0
614− kernel_ventry 0, irq // IRQ 64-bit EL0
615− kernel_ventry 0, fiq // FIQ 64-bit EL0
616− kernel_ventry 0, error // Error 64-bit EL0
617−
618−#ifdef CONFIG_COMPAT
619− kernel_ventry 0, sync_compat, 32 // Synchronous 32-bit EL0
620− kernel_ventry 0, irq_compat, 32 // IRQ 32-bit EL0
621− kernel_ventry 0, fiq_compat, 32 // FIQ 32-bit EL0
622− kernel_ventry 0, error_compat, 32 // Error 32-bit EL0
623−#else
624− kernel_ventry 0, sync_invalid, 32 // Synchronous 32-bit EL0
625− kernel_ventry 0, irq_invalid, 32 // IRQ 32-bit EL0
626− kernel_ventry 0, fiq_invalid, 32 // FIQ 32-bit EL0
627− kernel_ventry 0, error_invalid, 32 // Error 32-bit EL0
628−#endif
498+ kernel_ventry 1, t, 64, sync // Synchronous EL1t
499+ kernel_ventry 1, t, 64, irq // IRQ EL1t
500+ kernel_ventry 1, t, 64, fiq // FIQ EL1h
501+ kernel_ventry 1, t, 64, error // Error EL1t
502+
503+ kernel_ventry 1, h, 64, sync // Synchronous EL1h
504+ kernel_ventry 1, h, 64, irq // IRQ EL1h
505+ kernel_ventry 1, h, 64, fiq // FIQ EL1h
506+ kernel_ventry 1, h, 64, error // Error EL1h
507+
508+ kernel_ventry 0, t, 64, sync // Synchronous 64-bit EL0
509+ kernel_ventry 0, t, 64, irq // IRQ 64-bit EL0
510+ kernel_ventry 0, t, 64, fiq // FIQ 64-bit EL0
511+ kernel_ventry 0, t, 64, error // Error 64-bit EL0
512+
513+ kernel_ventry 0, t, 32, sync // Synchronous 32-bit EL0
514+ kernel_ventry 0, t, 32, irq // IRQ 32-bit EL0
515+ kernel_ventry 0, t, 32, fiq // FIQ 32-bit EL0
516+ kernel_ventry 0, t, 32, error // Error 32-bit EL0
629517 SYM_CODE_END(vectors)
630518
631519 #ifdef CONFIG_VMAP_STACK
@@ -656,147 +544,46 @@ __bad_stack: · 24 unmodified lines
656544 ASM_BUG()
657545 #endif /* CONFIG_VMAP_STACK */
658546
659−/*
660− * Invalid mode handlers
661− */
662− .macro inv_entry, el, reason, regsize = 64
547+
548+ .macro entry_handler el:req, ht:req, regsize:req, label:req
549+SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label)
663550 kernel_entry \el, \regsize
664551 mov x0, sp
665− mov x1, #\reason
666− mrs x2, esr_el1
667− bl bad_mode
668− ASM_BUG()
552+ bl el\el\ht\()_\regsize\()_\label\()_handler
553+ .if \el == 0
554+ b ret_to_user
555+ .else
556+ b ret_to_kernel
557+ .endif
558+SYM_CODE_END(el\el\ht\()_\regsize\()_\label)
669559 .endm
670560
671−SYM_CODE_START_LOCAL(el0_sync_invalid)
672− inv_entry 0, BAD_SYNC
673−SYM_CODE_END(el0_sync_invalid)
674−
675−SYM_CODE_START_LOCAL(el0_irq_invalid)
676− inv_entry 0, BAD_IRQ
677−SYM_CODE_END(el0_irq_invalid)
678−
679−SYM_CODE_START_LOCAL(el0_fiq_invalid)
680− inv_entry 0, BAD_FIQ
681−SYM_CODE_END(el0_fiq_invalid)
682−
683−SYM_CODE_START_LOCAL(el0_error_invalid)
684− inv_entry 0, BAD_ERROR
685−SYM_CODE_END(el0_error_invalid)
686−
687−SYM_CODE_START_LOCAL(el1_sync_invalid)
688− inv_entry 1, BAD_SYNC
689−SYM_CODE_END(el1_sync_invalid)
690−
691−SYM_CODE_START_LOCAL(el1_irq_invalid)
692− inv_entry 1, BAD_IRQ
693−SYM_CODE_END(el1_irq_invalid)
694−
695−SYM_CODE_START_LOCAL(el1_fiq_invalid)
696− inv_entry 1, BAD_FIQ
697−SYM_CODE_END(el1_fiq_invalid)
698−
699−SYM_CODE_START_LOCAL(el1_error_invalid)
700− inv_entry 1, BAD_ERROR
701−SYM_CODE_END(el1_error_invalid)
702−
703561 /*
704− * EL1 mode handlers.
562+ * Early exception handlers
705563 */
706− .align 6
707−SYM_CODE_START_LOCAL_NOALIGN(el1_sync)
708− kernel_entry 1
709− mov x0, sp
710− bl el1_sync_handler
711− kernel_exit 1
712−SYM_CODE_END(el1_sync)
713−
714− .align 6
715−SYM_CODE_START_LOCAL_NOALIGN(el1_irq)
716− kernel_entry 1
717− el1_interrupt_handler handle_arch_irq
718− kernel_exit 1
719−SYM_CODE_END(el1_irq)
720−
721−SYM_CODE_START_LOCAL_NOALIGN(el1_fiq)
722− kernel_entry 1
723− el1_interrupt_handler handle_arch_fiq
724− kernel_exit 1
725−SYM_CODE_END(el1_fiq)
726−
727−/*
728− * EL0 mode handlers.
729− */
730− .align 6
731−SYM_CODE_START_LOCAL_NOALIGN(el0_sync)
732− kernel_entry 0
733− mov x0, sp
734− bl el0_sync_handler
735− b ret_to_user
736−SYM_CODE_END(el0_sync)
737−
738−#ifdef CONFIG_COMPAT
739− .align 6
740−SYM_CODE_START_LOCAL_NOALIGN(el0_sync_compat)
741− kernel_entry 0, 32
742− mov x0, sp
743− bl el0_sync_compat_handler
744− b ret_to_user
745−SYM_CODE_END(el0_sync_compat)
746−
747− .align 6
748−SYM_CODE_START_LOCAL_NOALIGN(el0_irq_compat)
749− kernel_entry 0, 32
750− b el0_irq_naked
751−SYM_CODE_END(el0_irq_compat)
752−
753−SYM_CODE_START_LOCAL_NOALIGN(el0_fiq_compat)
754− kernel_entry 0, 32
755− b el0_fiq_naked
756−SYM_CODE_END(el0_fiq_compat)
757−
758−SYM_CODE_START_LOCAL_NOALIGN(el0_error_compat)
759− kernel_entry 0, 32
760− b el0_error_naked
761−SYM_CODE_END(el0_error_compat)
762−#endif
763−
764− .align 6
765−SYM_CODE_START_LOCAL_NOALIGN(el0_irq)
766− kernel_entry 0
767−el0_irq_naked:
768− el0_interrupt_handler handle_arch_irq
769− b ret_to_user
770−SYM_CODE_END(el0_irq)
771−
772−SYM_CODE_START_LOCAL_NOALIGN(el0_fiq)
773− kernel_entry 0
774−el0_fiq_naked:
775− el0_interrupt_handler handle_arch_fiq
776− b ret_to_user
777−SYM_CODE_END(el0_fiq)
778−
779−SYM_CODE_START_LOCAL(el1_error)
780− kernel_entry 1
781− mrs x1, esr_el1
782− enable_dbg
783− mov x0, sp
784− bl do_serror
564+ entry_handler 1, t, 64, sync
565+ entry_handler 1, t, 64, irq
566+ entry_handler 1, t, 64, fiq
567+ entry_handler 1, t, 64, error
568+
569+ entry_handler 1, h, 64, sync
570+ entry_handler 1, h, 64, irq
571+ entry_handler 1, h, 64, fiq
572+ entry_handler 1, h, 64, error
573+
574+ entry_handler 0, t, 64, sync
575+ entry_handler 0, t, 64, irq
576+ entry_handler 0, t, 64, fiq
577+ entry_handler 0, t, 64, error
578+
579+ entry_handler 0, t, 32, sync
580+ entry_handler 0, t, 32, irq
581+ entry_handler 0, t, 32, fiq
582+ entry_handler 0, t, 32, error
583+
584+SYM_CODE_START_LOCAL(ret_to_kernel)
785585 kernel_exit 1
786−SYM_CODE_END(el1_error)
787−
788−SYM_CODE_START_LOCAL(el0_error)
789− kernel_entry 0
790−el0_error_naked:
791− mrs x25, esr_el1
792− user_exit_irqoff
793− enable_dbg
794− mov x0, sp
795− mov x1, x25
796− bl do_serror
797− enable_da
798− b ret_to_user
799−SYM_CODE_END(el0_error)
586+SYM_CODE_END(ret_to_kernel)
800587
801588 /*
802589 * "slow" syscall return path.
@@ -979,8 +766,8 @@ SYM_FUNC_START(cpu_switch_to) · 176 unmodified lines
979766 mov sp, x9
980767 msr sp_el0, x1
981768 ptrauth_keys_install_kernel x1, x8, x9, x10
982− scs_save x0, x8
983− scs_load x1, x8
769+ scs_save x0
770+ scs_load x1
984771 ret
985772 SYM_FUNC_END(cpu_switch_to)
986773 NOKPROBE(cpu_switch_to)
@@ -998,6 +785,42 @@ SYM_CODE_START(ret_from_fork) · 11 unmodified lines
998785 SYM_CODE_END(ret_from_fork)
999786 NOKPROBE(ret_from_fork)
1000787
788+/*
789+ * void call_on_irq_stack(struct pt_regs *regs,
790+ * void (*func)(struct pt_regs *));
791+ *
792+ * Calls func(regs) using this CPU's irq stack and shadow irq stack.
793+ */
794+SYM_FUNC_START(call_on_irq_stack)
795+#ifdef CONFIG_SHADOW_CALL_STACK
796+ stp scs_sp, xzr, [sp, #-16]!
797+ ldr_this_cpu scs_sp, irq_shadow_call_stack_ptr, x17
798+#endif
799+ /* Create a frame record to save our LR and SP (implicit in FP) */
800+ stp x29, x30, [sp, #-16]!
801+ mov x29, sp
802+
803+ ldr_this_cpu x16, irq_stack_ptr, x17
804+ mov x15, #IRQ_STACK_SIZE
805+ add x16, x16, x15
806+
807+ /* Move to the new stack and call the function there */
808+ mov sp, x16
809+ blr x1
810+
811+ /*
812+ * Restore the SP from the FP, and restore the FP and LR from the frame
813+ * record.
814+ */
815+ mov sp, x29
816+ ldp x29, x30, [sp], #16
817+#ifdef CONFIG_SHADOW_CALL_STACK
818+ ldp scs_sp, xzr, [sp], #16
819+#endif
820+ ret
821+SYM_FUNC_END(call_on_irq_stack)
822+NOKPROBE(call_on_irq_stack)
823+
1001824 #ifdef CONFIG_ARM_SDE_INTERFACE
1002825
1003826 #include <asm/sdei.h>
arch/arm64/kernel/fpsimd.c+4−2
@@ -957,8 +957,10 @@ void do_sve_acc(unsigned int esr, struct pt_regs *regs) · 956 unmodified lines
957957 * disabling the trap, otherwise update our in-memory copy.
958958 */
959959 if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
960− sve_set_vq(sve_vq_from_vl(current->thread.sve_vl) - 1);
961− sve_flush_live();
960+ unsigned long vq_minus_one =
961+ sve_vq_from_vl(current->thread.sve_vl) - 1;
962+ sve_set_vq(vq_minus_one);
963+ sve_flush_live(vq_minus_one);
962964 fpsimd_bind_task_to_cpu();
963965 } else {
964966 fpsimd_to_sve(current);
arch/arm64/kernel/ftrace.c+1−0
@@ -15,6 +15,7 @@ · 14 unmodified lines
1515 #include <asm/debug-monitors.h>
1616 #include <asm/ftrace.h>
1717 #include <asm/insn.h>
18+#include <asm/patching.h>
1819
1920 #ifdef CONFIG_DYNAMIC_FTRACE
2021 /*
arch/arm64/kernel/head.S+44−32
@@ -16,6 +16,7 @@ · 15 unmodified lines
1616 #include <asm/asm_pointer_auth.h>
1717 #include <asm/assembler.h>
1818 #include <asm/boot.h>
19+#include <asm/bug.h>
1920 #include <asm/ptrace.h>
2021 #include <asm/asm-offsets.h>
2122 #include <asm/cache.h>
@@ -117,8 +118,8 @@ SYM_CODE_START_LOCAL(preserve_boot_args) · 95 unmodified lines
117118 dmb sy // needed before dc ivac with
118119 // MMU off
119120
120− mov x1, #0x20 // 4 x 8 bytes
121− b __inval_dcache_area // tail call
121+ add x1, x0, #0x20 // 4 x 8 bytes
122+ b dcache_inval_poc // tail call
122123 SYM_CODE_END(preserve_boot_args)
123124
124125 /*
@@ -195,7 +196,7 @@ SYM_CODE_END(preserve_boot_args) · 70 unmodified lines
195196 and \iend, \iend, \istart // iend = (vend >> shift) & (ptrs - 1)
196197 mov \istart, \ptrs
197198 mul \istart, \istart, \count
198− add \iend, \iend, \istart // iend += (count - 1) * ptrs
199+ add \iend, \iend, \istart // iend += count * ptrs
199200 // our entries span multiple tables
200201
201202 lsr \istart, \vstart, \shift
@@ -268,8 +269,7 @@ SYM_FUNC_START_LOCAL(__create_page_tables) · 66 unmodified lines
268269 */
269270 adrp x0, init_pg_dir
270271 adrp x1, init_pg_end
271− sub x1, x1, x0
272− bl __inval_dcache_area
272+ bl dcache_inval_poc
273273
274274 /*
275275 * Clear the init page tables.
@@ -354,7 +354,6 @@ SYM_FUNC_START_LOCAL(__create_page_tables) · 78 unmodified lines
354354 #endif
355355 1:
356356 ldr_l x4, idmap_ptrs_per_pgd
357− mov x5, x3 // __pa(__idmap_text_start)
358357 adr_l x6, __idmap_text_end // __pa(__idmap_text_end)
359358
360359 map_memory x0, x1, x3, x6, x7, x3, x4, x10, x11, x12, x13, x14
@@ -382,39 +381,57 @@ SYM_FUNC_START_LOCAL(__create_page_tables) · 21 unmodified lines
382381
383382 adrp x0, idmap_pg_dir
384383 adrp x1, idmap_pg_end
385− sub x1, x1, x0
386− bl __inval_dcache_area
384+ bl dcache_inval_poc
387385
388386 adrp x0, init_pg_dir
389387 adrp x1, init_pg_end
390− sub x1, x1, x0
391− bl __inval_dcache_area
388+ bl dcache_inval_poc
392389
393390 ret x28
394391 SYM_FUNC_END(__create_page_tables)
395392
393+ /*
394+ * Initialize CPU registers with task-specific and cpu-specific context.
395+ *
396+ * Create a final frame record at task_pt_regs(current)->stackframe, so
397+ * that the unwinder can identify the final frame record of any task by
398+ * its location in the task stack. We reserve the entire pt_regs space
399+ * for consistency with user tasks and kthreads.
400+ */
401+ .macro init_cpu_task tsk, tmp1, tmp2
402+ msr sp_el0, \tsk
403+
404+ ldr \tmp1, [\tsk, #TSK_STACK]
405+ add sp, \tmp1, #THREAD_SIZE
406+ sub sp, sp, #PT_REGS_SIZE
407+
408+ stp xzr, xzr, [sp, #S_STACKFRAME]
409+ add x29, sp, #S_STACKFRAME
410+
411+ scs_load \tsk
412+
413+ adr_l \tmp1, __per_cpu_offset
414+ ldr w\tmp2, [\tsk, #TSK_CPU]
415+ ldr \tmp1, [\tmp1, \tmp2, lsl #3]
416+ set_this_cpu_offset \tmp1
417+ .endm
418+
396419 /*
397420 * The following fragment of code is executed with the MMU enabled.
398421 *
399422 * x0 = __PHYS_OFFSET
400423 */
401424 SYM_FUNC_START_LOCAL(__primary_switched)
402− adrp x4, init_thread_union
403− add sp, x4, #THREAD_SIZE
404− adr_l x5, init_task
405− msr sp_el0, x5 // Save thread_info
425+ adr_l x4, init_task
426+ init_cpu_task x4, x5, x6
406427
407428 adr_l x8, vectors // load VBAR_EL1 with virtual
408429 msr vbar_el1, x8 // vector table address
409430 isb
410431
411− stp xzr, x30, [sp, #-16]!
432+ stp x29, x30, [sp, #-16]!
412433 mov x29, sp
413434
414−#ifdef CONFIG_SHADOW_CALL_STACK
415− adr_l scs_sp, init_shadow_call_stack // Set shadow call stack
416−#endif
417−
418435 str_l x21, __fdt_pointer, x5 // Save FDT pointer
419436
420437 ldr_l x4, kimage_vaddr // Save the offset between
@@ -446,10 +463,9 @@ SYM_FUNC_START_LOCAL(__primary_switched) · 25 unmodified lines
446463 0:
447464 #endif
448465 bl switch_to_vhe // Prefer VHE if possible
449− add sp, sp, #16
450− mov x29, #0
451− mov x30, #0
452− b start_kernel
466+ ldp x29, x30, [sp], #16
467+ bl start_kernel
468+ ASM_BUG()
453469 SYM_FUNC_END(__primary_switched)
454470
455471 .pushsection ".rodata", "a"
@@ -551,7 +567,7 @@ SYM_FUNC_START_LOCAL(set_cpu_boot_mode_flag) · 95 unmodified lines
551567 cmp w0, #BOOT_CPU_MODE_EL2
552568 b.ne 1f
553569 add x1, x1, #4
554−1: str w0, [x1] // This CPU has booted in EL1
570+1: str w0, [x1] // Save CPU boot mode
555571 dmb sy
556572 dc ivac, x1 // Invalidate potentially stale cache line
557573 ret
@@ -632,21 +648,17 @@ SYM_FUNC_START_LOCAL(__secondary_switched) · 74 unmodified lines
632648 isb
633649
634650 adr_l x0, secondary_data
635− ldr x1, [x0, #CPU_BOOT_STACK] // get secondary_data.stack
636− cbz x1, __secondary_too_slow
637− mov sp, x1
638651 ldr x2, [x0, #CPU_BOOT_TASK]
639652 cbz x2, __secondary_too_slow
640− msr sp_el0, x2
641− scs_load x2, x3
642− mov x29, #0
643− mov x30, #0
653+
654+ init_cpu_task x2, x1, x3
644655
645656 #ifdef CONFIG_ARM64_PTR_AUTH
646657 ptrauth_keys_init_cpu x2, x3, x4, x5
647658 #endif
648659
649− b secondary_start_kernel
660+ bl secondary_start_kernel
661+ ASM_BUG()
650662 SYM_FUNC_END(__secondary_switched)
651663
652664 SYM_FUNC_START_LOCAL(__secondary_too_slow)
arch/arm64/kernel/hibernate-asm.S+4−3
@@ -45,7 +45,7 @@ · 44 unmodified lines
4545 * Because this code has to be copied to a 'safe' page, it can't call out to
4646 * other functions by PC-relative address. Also remember that it may be
4747 * mid-way through over-writing other functions. For this reason it contains
48− * code from flush_icache_range() and uses the copy_page() macro.
48+ * code from caches_clean_inval_pou() and uses the copy_page() macro.
4949 *
5050 * This 'safe' page is mapped via ttbr0, and executed from there. This function
5151 * switches to a copy of the linear map in ttbr1, performs the restore, then
@@ -87,11 +87,12 @@ SYM_CODE_START(swsusp_arch_suspend_exit) · 35 unmodified lines
8787 copy_page x0, x1, x2, x3, x4, x5, x6, x7, x8, x9
8888
8989 add x1, x10, #PAGE_SIZE
90− /* Clean the copied page to PoU - based on flush_icache_range() */
90+ /* Clean the copied page to PoU - based on caches_clean_inval_pou() */
9191 raw_dcache_line_size x2, x3
9292 sub x3, x2, #1
9393 bic x4, x10, x3
94−2: dc cvau, x4 /* clean D line / unified line */
94+2: /* clean D line / unified line */
95+alternative_insn "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE
9596 add x4, x4, x2
9697 cmp x4, x1
9798 b.lo 2b
arch/arm64/kernel/hibernate.c+12−8
@@ -210,7 +210,7 @@ static int create_safe_exec_page(void *src_start, size_t length, · 209 unmodified lines
210210 return -ENOMEM;
211211
212212 memcpy(page, src_start, length);
213− __flush_icache_range((unsigned long)page, (unsigned long)page + length);
213+ caches_clean_inval_pou((unsigned long)page, (unsigned long)page + length);
214214 rc = trans_pgd_idmap_page(&trans_info, &trans_ttbr0, &t0sz, page);
215215 if (rc)
216216 return rc;
@@ -240,8 +240,6 @@ static int create_safe_exec_page(void *src_start, size_t length, · 23 unmodified lines
240240 return 0;
241241 }
242242
243−#define dcache_clean_range(start, end) __flush_dcache_area(start, (end - start))
244−
245243 #ifdef CONFIG_ARM64_MTE
246244
247245 static DEFINE_XARRAY(mte_pages);
@@ -383,13 +381,18 @@ int swsusp_arch_suspend(void) · 135 unmodified lines
383381 ret = swsusp_save();
384382 } else {
385383 /* Clean kernel core startup/idle code to PoC*/
386− dcache_clean_range(__mmuoff_data_start, __mmuoff_data_end);
387− dcache_clean_range(__idmap_text_start, __idmap_text_end);
384+ dcache_clean_inval_poc((unsigned long)__mmuoff_data_start,
385+ (unsigned long)__mmuoff_data_end);
386+ dcache_clean_inval_poc((unsigned long)__idmap_text_start,
387+ (unsigned long)__idmap_text_end);
388388
389389 /* Clean kvm setup code to PoC? */
390390 if (el2_reset_needed()) {
391− dcache_clean_range(__hyp_idmap_text_start, __hyp_idmap_text_end);
392− dcache_clean_range(__hyp_text_start, __hyp_text_end);
391+ dcache_clean_inval_poc(
392+ (unsigned long)__hyp_idmap_text_start,
393+ (unsigned long)__hyp_idmap_text_end);
394+ dcache_clean_inval_poc((unsigned long)__hyp_text_start,
395+ (unsigned long)__hyp_text_end);
393396 }
394397
395398 swsusp_mte_restore_tags();
@@ -474,7 +477,8 @@ int swsusp_arch_resume(void) · 78 unmodified lines
474477 * The hibernate exit text contains a set of el2 vectors, that will
475478 * be executed at el2 with the mmu off in order to reload hyp-stub.
476479 */
477− __flush_dcache_area(hibernate_exit, exit_size);
480+ dcache_clean_inval_poc((unsigned long)hibernate_exit,
481+ (unsigned long)hibernate_exit + exit_size);
478482
479483 /*
480484 * KASLR will cause the el2 vectors to be in a different location in
arch/arm64/kernel/idle.c+46−0
@@ -0,0 +1,46 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+/*
3+ * Low-level idle sequences
4+ */
5+
6+#include <linux/cpu.h>
7+#include <linux/irqflags.h>
8+
9+#include <asm/barrier.h>
10+#include <asm/cpuidle.h>
11+#include <asm/cpufeature.h>
12+#include <asm/sysreg.h>
13+
14+/*
15+ * cpu_do_idle()
16+ *
17+ * Idle the processor (wait for interrupt).
18+ *
19+ * If the CPU supports priority masking we must do additional work to
20+ * ensure that interrupts are not masked at the PMR (because the core will
21+ * not wake up if we block the wake up signal in the interrupt controller).
22+ */
23+void noinstr cpu_do_idle(void)
24+{
25+ struct arm_cpuidle_irq_context context;
26+
27+ arm_cpuidle_save_irq_context(&context);
28+
29+ dsb(sy);
30+ wfi();
31+
32+ arm_cpuidle_restore_irq_context(&context);
33+}
34+
35+/*
36+ * This is our default idle handler.
37+ */
38+void noinstr arch_cpu_idle(void)
39+{
40+ /*
41+ * This should do all the clock switching and wait for interrupt
42+ * tricks
43+ */
44+ cpu_do_idle();
45+ raw_local_irq_enable();
46+}
arch/arm64/kernel/idreg-override.c+2−1
@@ -237,7 +237,8 @@ asmlinkage void __init init_feature_override(void) · 236 unmodified lines
237237
238238 for (i = 0; i < ARRAY_SIZE(regs); i++) {
239239 if (regs[i]->override)
240− __flush_dcache_area(regs[i]->override,
240+ dcache_clean_inval_poc((unsigned long)regs[i]->override,
241+ (unsigned long)regs[i]->override +
241242 sizeof(*regs[i]->override));
242243 }
243244 }
arch/arm64/kernel/image-vars.h+1−1
@@ -35,7 +35,7 @@ __efistub_strnlen = __pi_strnlen; · 34 unmodified lines
3535 __efistub_strcmp = __pi_strcmp;
3636 __efistub_strncmp = __pi_strncmp;
3737 __efistub_strrchr = __pi_strrchr;
38−__efistub___clean_dcache_area_poc = __pi___clean_dcache_area_poc;
38+__efistub_dcache_clean_poc = __pi_dcache_clean_poc;
3939
4040 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
4141 __efistub___memcpy = __pi_memcpy;
arch/arm64/kernel/jump_label.c+1−0
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 #include <linux/kernel.h>
99 #include <linux/jump_label.h>
1010 #include <asm/insn.h>
11+#include <asm/patching.h>
1112
1213 void arch_jump_label_transform(struct jump_entry *entry,
1314 enum jump_label_type type)
arch/arm64/kernel/kaslr.c+9−3
@@ -72,7 +72,9 @@ u64 __init kaslr_early_init(void) · 71 unmodified lines
7272 * we end up running with module randomization disabled.
7373 */
7474 module_alloc_base = (u64)_etext - MODULES_VSIZE;
75− __flush_dcache_area(&module_alloc_base, sizeof(module_alloc_base));
75+ dcache_clean_inval_poc((unsigned long)&module_alloc_base,
76+ (unsigned long)&module_alloc_base +
77+ sizeof(module_alloc_base));
7678
7779 /*
7880 * Try to map the FDT early. If this fails, we simply bail,
@@ -170,8 +172,12 @@ u64 __init kaslr_early_init(void) · 91 unmodified lines
170172 module_alloc_base += (module_range * (seed & ((1 << 21) - 1))) >> 21;
171173 module_alloc_base &= PAGE_MASK;
172174
173− __flush_dcache_area(&module_alloc_base, sizeof(module_alloc_base));
174− __flush_dcache_area(&memstart_offset_seed, sizeof(memstart_offset_seed));
175+ dcache_clean_inval_poc((unsigned long)&module_alloc_base,
176+ (unsigned long)&module_alloc_base +
177+ sizeof(module_alloc_base));
178+ dcache_clean_inval_poc((unsigned long)&memstart_offset_seed,
179+ (unsigned long)&memstart_offset_seed +
180+ sizeof(memstart_offset_seed));
175181
176182 return offset;
177183 }
arch/arm64/kernel/kgdb.c+1−0
@@ -17,6 +17,7 @@ · 16 unmodified lines
1717
1818 #include <asm/debug-monitors.h>
1919 #include <asm/insn.h>
20+#include <asm/patching.h>
2021 #include <asm/traps.h>
2122
2223 struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = {
arch/arm64/kernel/machine_kexec.c+20−10
@@ -68,10 +68,16 @@ int machine_kexec_post_load(struct kimage *kimage) · 67 unmodified lines
6868 kimage->arch.kern_reloc = __pa(reloc_code);
6969 kexec_image_info(kimage);
7070
71− /* Flush the reloc_code in preparation for its execution. */
72− __flush_dcache_area(reloc_code, arm64_relocate_new_kernel_size);
73− flush_icache_range((uintptr_t)reloc_code, (uintptr_t)reloc_code +
74− arm64_relocate_new_kernel_size);
71+ /*
72+ * For execution with the MMU off, reloc_code needs to be cleaned to the
73+ * PoC and invalidated from the I-cache.
74+ */
75+ dcache_clean_inval_poc((unsigned long)reloc_code,
76+ (unsigned long)reloc_code +
77+ arm64_relocate_new_kernel_size);
78+ icache_inval_pou((uintptr_t)reloc_code,
79+ (uintptr_t)reloc_code +
80+ arm64_relocate_new_kernel_size);
7581
7682 return 0;
7783 }
@@ -102,16 +108,18 @@ static void kexec_list_flush(struct kimage *kimage) · 24 unmodified lines
102108
103109 for (entry = &kimage->head; ; entry++) {
104110 unsigned int flag;
105− void *addr;
111+ unsigned long addr;
106112
107113 /* flush the list entries. */
108− __flush_dcache_area(entry, sizeof(kimage_entry_t));
114+ dcache_clean_inval_poc((unsigned long)entry,
115+ (unsigned long)entry +
116+ sizeof(kimage_entry_t));
109117
110118 flag = *entry & IND_FLAGS;
111119 if (flag == IND_DONE)
112120 break;
113121
114− addr = phys_to_virt(*entry & PAGE_MASK);
122+ addr = (unsigned long)phys_to_virt(*entry & PAGE_MASK);
115123
116124 switch (flag) {
117125 case IND_INDIRECTION:
@@ -120,7 +128,7 @@ static void kexec_list_flush(struct kimage *kimage) · 2 unmodified lines
120128 break;
121129 case IND_SOURCE:
122130 /* flush the source pages. */
123− __flush_dcache_area(addr, PAGE_SIZE);
131+ dcache_clean_inval_poc(addr, addr + PAGE_SIZE);
124132 break;
125133 case IND_DESTINATION:
126134 break;
@@ -147,8 +155,10 @@ static void kexec_segment_flush(const struct kimage *kimage) · 20 unmodified lines
147155 kimage->segment[i].memsz,
148156 kimage->segment[i].memsz / PAGE_SIZE);
149157
150− __flush_dcache_area(phys_to_virt(kimage->segment[i].mem),
151− kimage->segment[i].memsz);
158+ dcache_clean_inval_poc(
159+ (unsigned long)phys_to_virt(kimage->segment[i].mem),
160+ (unsigned long)phys_to_virt(kimage->segment[i].mem) +
161+ kimage->segment[i].memsz);
152162 }
153163 }
154164
arch/arm64/kernel/patching.c+150−0
@@ -0,0 +1,150 @@
1+// SPDX-License-Identifier: GPL-2.0-only
2+#include <linux/kernel.h>
3+#include <linux/mm.h>
4+#include <linux/smp.h>
5+#include <linux/spinlock.h>
6+#include <linux/stop_machine.h>
7+#include <linux/uaccess.h>
8+
9+#include <asm/cacheflush.h>
10+#include <asm/fixmap.h>
11+#include <asm/insn.h>
12+#include <asm/kprobes.h>
13+#include <asm/patching.h>
14+#include <asm/sections.h>
15+
16+static DEFINE_RAW_SPINLOCK(patch_lock);
17+
18+static bool is_exit_text(unsigned long addr)
19+{
20+ /* discarded with init text/data */
21+ return system_state < SYSTEM_RUNNING &&
22+ addr >= (unsigned long)__exittext_begin &&
23+ addr < (unsigned long)__exittext_end;
24+}
25+
26+static bool is_image_text(unsigned long addr)
27+{
28+ return core_kernel_text(addr) || is_exit_text(addr);
29+}
30+
31+static void __kprobes *patch_map(void *addr, int fixmap)
32+{
33+ unsigned long uintaddr = (uintptr_t) addr;
34+ bool image = is_image_text(uintaddr);
35+ struct page *page;
36+
37+ if (image)
38+ page = phys_to_page(__pa_symbol(addr));
39+ else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
40+ page = vmalloc_to_page(addr);
41+ else
42+ return addr;
43+
44+ BUG_ON(!page);
45+ return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
46+ (uintaddr & ~PAGE_MASK));
47+}
48+
49+static void __kprobes patch_unmap(int fixmap)
50+{
51+ clear_fixmap(fixmap);
52+}
53+/*
54+ * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
55+ * little-endian.
56+ */
57+int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
58+{
59+ int ret;
60+ __le32 val;
61+
62+ ret = copy_from_kernel_nofault(&val, addr, AARCH64_INSN_SIZE);
63+ if (!ret)
64+ *insnp = le32_to_cpu(val);
65+
66+ return ret;
67+}
68+
69+static int __kprobes __aarch64_insn_write(void *addr, __le32 insn)
70+{
71+ void *waddr = addr;
72+ unsigned long flags = 0;
73+ int ret;
74+
75+ raw_spin_lock_irqsave(&patch_lock, flags);
76+ waddr = patch_map(addr, FIX_TEXT_POKE0);
77+
78+ ret = copy_to_kernel_nofault(waddr, &insn, AARCH64_INSN_SIZE);
79+
80+ patch_unmap(FIX_TEXT_POKE0);
81+ raw_spin_unlock_irqrestore(&patch_lock, flags);
82+
83+ return ret;
84+}
85+
86+int __kprobes aarch64_insn_write(void *addr, u32 insn)
87+{
88+ return __aarch64_insn_write(addr, cpu_to_le32(insn));
89+}
90+
91+int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
92+{
93+ u32 *tp = addr;
94+ int ret;
95+
96+ /* A64 instructions must be word aligned */
97+ if ((uintptr_t)tp & 0x3)
98+ return -EINVAL;
99+
100+ ret = aarch64_insn_write(tp, insn);
101+ if (ret == 0)
102+ caches_clean_inval_pou((uintptr_t)tp,
103+ (uintptr_t)tp + AARCH64_INSN_SIZE);
104+
105+ return ret;
106+}
107+
108+struct aarch64_insn_patch {
109+ void **text_addrs;
110+ u32 *new_insns;
111+ int insn_cnt;
112+ atomic_t cpu_count;
113+};
114+
115+static int __kprobes aarch64_insn_patch_text_cb(void *arg)
116+{
117+ int i, ret = 0;
118+ struct aarch64_insn_patch *pp = arg;
119+
120+ /* The first CPU becomes master */
121+ if (atomic_inc_return(&pp->cpu_count) == 1) {
122+ for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
123+ ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
124+ pp->new_insns[i]);
125+ /* Notify other processors with an additional increment. */
126+ atomic_inc(&pp->cpu_count);
127+ } else {
128+ while (atomic_read(&pp->cpu_count) <= num_online_cpus())
129+ cpu_relax();
130+ isb();
131+ }
132+
133+ return ret;
134+}
135+
136+int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
137+{
138+ struct aarch64_insn_patch patch = {
139+ .text_addrs = addrs,
140+ .new_insns = insns,
141+ .insn_cnt = cnt,
142+ .cpu_count = ATOMIC_INIT(0),
143+ };
144+
145+ if (cnt <= 0)
146+ return -EINVAL;
147+
148+ return stop_machine_cpuslocked(aarch64_insn_patch_text_cb, &patch,
149+ cpu_online_mask);
150+}
arch/arm64/kernel/perf_callchain.c+1−1
@@ -116,7 +116,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, · 115 unmodified lines
116116 tail = (struct frame_tail __user *)regs->regs[29];
117117
118118 while (entry->nr < entry->max_stack &&
119− tail && !((unsigned long)tail & 0xf))
119+ tail && !((unsigned long)tail & 0x7))
120120 tail = user_backtrace(tail, entry);
121121 } else {
122122 #ifdef CONFIG_COMPAT
arch/arm64/kernel/perf_event.c+35−5
@@ -165,10 +165,7 @@ armv8pmu_events_sysfs_show(struct device *dev, · 164 unmodified lines
165165 }
166166
167167 #define ARMV8_EVENT_ATTR(name, config) \
168− (&((struct perf_pmu_events_attr) { \
169− .attr = __ATTR(name, 0444, armv8pmu_events_sysfs_show, NULL), \
170− .id = config, \
171− }).attr.attr)
168+ PMU_EVENT_ATTR_ID(name, armv8pmu_events_sysfs_show, config)
172169
173170 static struct attribute *armv8_pmuv3_event_attrs[] = {
174171 ARMV8_EVENT_ATTR(sw_incr, ARMV8_PMUV3_PERFCTR_SW_INCR),
@@ -312,13 +309,46 @@ static ssize_t slots_show(struct device *dev, struct device_attribute *attr, · 137 unmodified lines
312309 struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
313310 u32 slots = cpu_pmu->reg_pmmir & ARMV8_PMU_SLOTS_MASK;
314311
315− return snprintf(page, PAGE_SIZE, "0x%08x\n", slots);
312+ return sysfs_emit(page, "0x%08x\n", slots);
316313 }
317314
318315 static DEVICE_ATTR_RO(slots);
319316
317+static ssize_t bus_slots_show(struct device *dev, struct device_attribute *attr,
318+ char *page)
319+{
320+ struct pmu *pmu = dev_get_drvdata(dev);
321+ struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
322+ u32 bus_slots = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_SLOTS_SHIFT)
323+ & ARMV8_PMU_BUS_SLOTS_MASK;
324+
325+ return sysfs_emit(page, "0x%08x\n", bus_slots);
326+}
327+
328+static DEVICE_ATTR_RO(bus_slots);
329+
330+static ssize_t bus_width_show(struct device *dev, struct device_attribute *attr,
331+ char *page)
332+{
333+ struct pmu *pmu = dev_get_drvdata(dev);
334+ struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu);
335+ u32 bus_width = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_WIDTH_SHIFT)
336+ & ARMV8_PMU_BUS_WIDTH_MASK;
337+ u32 val = 0;
338+
339+ /* Encoded as Log2(number of bytes), plus one */
340+ if (bus_width > 2 && bus_width < 13)
341+ val = 1 << (bus_width - 1);
342+
343+ return sysfs_emit(page, "0x%08x\n", val);
344+}
345+
346+static DEVICE_ATTR_RO(bus_width);
347+
320348 static struct attribute *armv8_pmuv3_caps_attrs[] = {
321349 &dev_attr_slots.attr,
350+ &dev_attr_bus_slots.attr,
351+ &dev_attr_bus_width.attr,
322352 NULL,
323353 };
324354
arch/arm64/kernel/probes/kprobes.c+10−8
@@ -7,26 +7,28 @@ · 6 unmodified lines
77 * Copyright (C) 2013 Linaro Limited.
88 * Author: Sandeepa Prabhu <sandeepa.prabhu@linaro.org>
99 */
10+#include <linux/extable.h>
1011 #include <linux/kasan.h>
1112 #include <linux/kernel.h>
1213 #include <linux/kprobes.h>
13−#include <linux/extable.h>
14−#include <linux/slab.h>
15−#include <linux/stop_machine.h>
1614 #include <linux/sched/debug.h>
1715 #include <linux/set_memory.h>
16+#include <linux/slab.h>
17+#include <linux/stop_machine.h>
1818 #include <linux/stringify.h>
19+#include <linux/uaccess.h>
1920 #include <linux/vmalloc.h>
20−#include <asm/traps.h>
21−#include <asm/ptrace.h>
21+
2222 #include <asm/cacheflush.h>
23−#include <asm/debug-monitors.h>
2423 #include <asm/daifflags.h>
25−#include <asm/system_misc.h>
24+#include <asm/debug-monitors.h>
2625 #include <asm/insn.h>
27−#include <linux/uaccess.h>
2826 #include <asm/irq.h>
27+#include <asm/patching.h>
28+#include <asm/ptrace.h>
2929 #include <asm/sections.h>
30+#include <asm/system_misc.h>
31+#include <asm/traps.h>
3032
3133 #include "decode-insn.h"
3234
arch/arm64/kernel/probes/simulate-insn.c+1−0
@@ -10,6 +10,7 @@ · 9 unmodified lines
1010 #include <linux/kprobes.h>
1111
1212 #include <asm/ptrace.h>
13+#include <asm/traps.h>
1314
1415 #include "simulate-insn.h"
1516
arch/arm64/kernel/probes/uprobes.c+1−1
@@ -21,7 +21,7 @@ void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, · 20 unmodified lines
2121 memcpy(dst, src, len);
2222
2323 /* flush caches (dcache/icache) */
24− sync_icache_aliases(dst, len);
24+ sync_icache_aliases((unsigned long)dst, (unsigned long)dst + len);
2525
2626 kunmap_atomic(xol_page_kaddr);
2727 }
arch/arm64/kernel/process.c+23−76
@@ -18,7 +18,6 @@ · 17 unmodified lines
1818 #include <linux/sched/task.h>
1919 #include <linux/sched/task_stack.h>
2020 #include <linux/kernel.h>
21−#include <linux/lockdep.h>
2221 #include <linux/mman.h>
2322 #include <linux/mm.h>
2423 #include <linux/nospec.h>
@@ -46,7 +45,6 @@ · 21 unmodified lines
4645 #include <linux/prctl.h>
4746
4847 #include <asm/alternative.h>
49−#include <asm/arch_gicv3.h>
5048 #include <asm/compat.h>
5149 #include <asm/cpufeature.h>
5250 #include <asm/cacheflush.h>
@@ -74,63 +72,6 @@ EXPORT_SYMBOL_GPL(pm_power_off); · 21 unmodified lines
7472
7573 void (*arm_pm_restart)(enum reboot_mode reboot_mode, const char *cmd);
7674
77−static void noinstr __cpu_do_idle(void)
78−{
79− dsb(sy);
80− wfi();
81−}
82−
83−static void noinstr __cpu_do_idle_irqprio(void)
84−{
85− unsigned long pmr;
86− unsigned long daif_bits;
87−
88− daif_bits = read_sysreg(daif);
89− write_sysreg(daif_bits | PSR_I_BIT | PSR_F_BIT, daif);
90−
91− /*
92− * Unmask PMR before going idle to make sure interrupts can
93− * be raised.
94− */
95− pmr = gic_read_pmr();
96− gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET);
97−
98− __cpu_do_idle();
99−
100− gic_write_pmr(pmr);
101− write_sysreg(daif_bits, daif);
102−}
103−
104−/*
105− * cpu_do_idle()
106− *
107− * Idle the processor (wait for interrupt).
108− *
109− * If the CPU supports priority masking we must do additional work to
110− * ensure that interrupts are not masked at the PMR (because the core will
111− * not wake up if we block the wake up signal in the interrupt controller).
112− */
113−void noinstr cpu_do_idle(void)
114−{
115− if (system_uses_irq_prio_masking())
116− __cpu_do_idle_irqprio();
117− else
118− __cpu_do_idle();
119−}
120−
121−/*
122− * This is our default idle handler.
123− */
124−void noinstr arch_cpu_idle(void)
125−{
126− /*
127− * This should do all the clock switching and wait for interrupt
128− * tricks
129− */
130− cpu_do_idle();
131− raw_local_irq_enable();
132−}
133−
13475 #ifdef CONFIG_HOTPLUG_CPU
13576 void arch_cpu_idle_dead(void)
13677 {
@@ -435,6 +376,11 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, · 298 unmodified lines
435376 }
436377 p->thread.cpu_context.pc = (unsigned long)ret_from_fork;
437378 p->thread.cpu_context.sp = (unsigned long)childregs;
379+ /*
380+ * For the benefit of the unwinder, set up childregs->stackframe
381+ * as the final frame for the new task.
382+ */
383+ p->thread.cpu_context.fp = (unsigned long)childregs->stackframe;
438384
439385 ptrace_hw_copy_thread(p);
440386
@@ -527,6 +473,15 @@ static void erratum_1418040_thread_switch(struct task_struct *prev, · 86 unmodified lines
527473 write_sysreg(val, cntkctl_el1);
528474 }
529475
476+static void compat_thread_switch(struct task_struct *next)
477+{
478+ if (!is_compat_thread(task_thread_info(next)))
479+ return;
480+
481+ if (static_branch_unlikely(&arm64_mismatched_32bit_el0))
482+ set_tsk_thread_flag(next, TIF_NOTIFY_RESUME);
483+}
484+
530485 static void update_sctlr_el1(u64 sctlr)
531486 {
532487 /*
@@ -568,6 +523,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev, · 35 unmodified lines
568523 ssbs_thread_switch(next);
569524 erratum_1418040_thread_switch(prev, next);
570525 ptrauth_thread_switch_user(next);
526+ compat_thread_switch(next);
571527
572528 /*
573529 * Complete any pending TLB or cache maintenance on this CPU in case
@@ -633,8 +589,15 @@ unsigned long arch_align_stack(unsigned long sp) · 59 unmodified lines
633589 */
634590 void arch_setup_new_exec(void)
635591 {
636− current->mm->context.flags = is_compat_task() ? MMCF_AARCH32 : 0;
592+ unsigned long mmflags = 0;
593+
594+ if (is_compat_task()) {
595+ mmflags = MMCF_AARCH32;
596+ if (static_branch_unlikely(&arm64_mismatched_32bit_el0))
597+ set_tsk_thread_flag(current, TIF_NOTIFY_RESUME);
598+ }
637599
600+ current->mm->context.flags = mmflags;
638601 ptrauth_thread_init_user();
639602 mte_thread_init_user();
640603
@@ -724,22 +687,6 @@ static int __init tagged_addr_init(void) · 83 unmodified lines
724687 core_initcall(tagged_addr_init);
725688 #endif /* CONFIG_ARM64_TAGGED_ADDR_ABI */
726689
727−asmlinkage void __sched arm64_preempt_schedule_irq(void)
728−{
729− lockdep_assert_irqs_disabled();
730−
731− /*
732− * Preempting a task from an IRQ means we leave copies of PSTATE
733− * on the stack. cpufeature's enable calls may modify PSTATE, but
734− * resuming one of these preempted tasks would undo those changes.
735− *
736− * Only allow a task to be preempted once cpufeatures have been
737− * enabled.
738− */
739− if (system_capabilities_finalized())
740− preempt_schedule_irq();
741−}
742−
743690 #ifdef CONFIG_BINFMT_ELF
744691 int arch_elf_adjust_prot(int prot, const struct arch_elf_state *state,
745692 bool has_interp, bool is_interp)
arch/arm64/kernel/ptrace.c+1−1
@@ -122,7 +122,7 @@ static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) · 121 unmodified lines
122122 {
123123 return ((addr & ~(THREAD_SIZE - 1)) ==
124124 (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) ||
125− on_irq_stack(addr, NULL);
125+ on_irq_stack(addr, sizeof(unsigned long), NULL);
126126 }
127127
128128 /**
arch/arm64/kernel/sdei.c+12−52
@@ -162,31 +162,33 @@ static int init_sdei_scs(void) · 161 unmodified lines
162162 return err;
163163 }
164164
165−static bool on_sdei_normal_stack(unsigned long sp, struct stack_info *info)
165+static bool on_sdei_normal_stack(unsigned long sp, unsigned long size,
166+ struct stack_info *info)
166167 {
167168 unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
168169 unsigned long high = low + SDEI_STACK_SIZE;
169170
170− return on_stack(sp, low, high, STACK_TYPE_SDEI_NORMAL, info);
171+ return on_stack(sp, size, low, high, STACK_TYPE_SDEI_NORMAL, info);
171172 }
172173
173−static bool on_sdei_critical_stack(unsigned long sp, struct stack_info *info)
174+static bool on_sdei_critical_stack(unsigned long sp, unsigned long size,
175+ struct stack_info *info)
174176 {
175177 unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
176178 unsigned long high = low + SDEI_STACK_SIZE;
177179
178− return on_stack(sp, low, high, STACK_TYPE_SDEI_CRITICAL, info);
180+ return on_stack(sp, size, low, high, STACK_TYPE_SDEI_CRITICAL, info);
179181 }
180182
181−bool _on_sdei_stack(unsigned long sp, struct stack_info *info)
183+bool _on_sdei_stack(unsigned long sp, unsigned long size, struct stack_info *info)
182184 {
183185 if (!IS_ENABLED(CONFIG_VMAP_STACK))
184186 return false;
185187
186− if (on_sdei_critical_stack(sp, info))
188+ if (on_sdei_critical_stack(sp, size, info))
187189 return true;
188190
189− if (on_sdei_normal_stack(sp, info))
191+ if (on_sdei_normal_stack(sp, size, info))
190192 return true;
191193
192194 return false;
@@ -231,13 +233,13 @@ out_err: · 38 unmodified lines
231233 }
232234
233235 /*
234− * __sdei_handler() returns one of:
236+ * do_sdei_event() returns one of:
235237 * SDEI_EV_HANDLED - success, return to the interrupted context.
236238 * SDEI_EV_FAILED - failure, return this error code to firmare.
237239 * virtual-address - success, return to this address.
238240 */
239−static __kprobes unsigned long _sdei_handler(struct pt_regs *regs,
240− struct sdei_registered_event *arg)
241+unsigned long __kprobes do_sdei_event(struct pt_regs *regs,
242+ struct sdei_registered_event *arg)
241243 {
242244 u32 mode;
243245 int i, err = 0;
@@ -292,45 +294,3 @@ static __kprobes unsigned long _sdei_handler(struct pt_regs *regs, · 48 unmodified lines
292294
293295 return vbar + 0x480;
294296 }
295−
296−static void __kprobes notrace __sdei_pstate_entry(void)
297−{
298− /*
299− * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to
300− * whether PSTATE bits are inherited unchanged or generated from
301− * scratch, and the TF-A implementation always clears PAN and always
302− * clears UAO. There are no other known implementations.
303− *
304− * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how
305− * PSTATE is modified upon architectural exceptions, and so PAN is
306− * either inherited or set per SCTLR_ELx.SPAN, and UAO is always
307− * cleared.
308− *
309− * We must explicitly reset PAN to the expected state, including
310− * clearing it when the host isn't using it, in case a VM had it set.
311− */
312− if (system_uses_hw_pan())
313− set_pstate_pan(1);
314− else if (cpu_has_pan())
315− set_pstate_pan(0);
316−}
317−
318−asmlinkage noinstr unsigned long
319−__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg)
320−{
321− unsigned long ret;
322−
323− /*
324− * We didn't take an exception to get here, so the HW hasn't
325− * set/cleared bits in PSTATE that we may rely on. Initialize PAN.
326− */
327− __sdei_pstate_entry();
328−
329− arm64_enter_nmi(regs);
330−
331− ret = _sdei_handler(regs, arg);
332−
333− arm64_exit_nmi(regs);
334−
335− return ret;
336−}
arch/arm64/kernel/setup.c+1−7
@@ -87,12 +87,6 @@ void __init smp_setup_processor_id(void) · 86 unmodified lines
8787 u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
8888 set_cpu_logical_map(0, mpidr);
8989
90− /*
91− * clear __my_cpu_offset on boot CPU to avoid hang caused by
92− * using percpu variable early, for example, lockdep will
93− * access percpu variable inside lock_release
94− */
95− set_my_cpu_offset(0);
9690 pr_info("Booting Linux on physical CPU 0x%010lx [0x%08x]\n",
9791 (unsigned long)mpidr, read_cpuid_id());
9892 }
@@ -381,7 +375,7 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) · 282 unmodified lines
381375 * faults in case uaccess_enable() is inadvertently called by the init
382376 * thread.
383377 */
384− init_task.thread_info.ttbr0 = __pa_symbol(reserved_pg_dir);
378+ init_task.thread_info.ttbr0 = phys_to_ttbr(__pa_symbol(reserved_pg_dir));
385379 #endif
386380
387381 if (boot_args[1] || boot_args[2] || boot_args[3]) {
arch/arm64/kernel/signal.c+26−0
@@ -911,6 +911,19 @@ static void do_signal(struct pt_regs *regs) · 910 unmodified lines
911911 restore_saved_sigmask();
912912 }
913913
914+static bool cpu_affinity_invalid(struct pt_regs *regs)
915+{
916+ if (!compat_user_mode(regs))
917+ return false;
918+
919+ /*
920+ * We're preemptible, but a reschedule will cause us to check the
921+ * affinity again.
922+ */
923+ return !cpumask_test_cpu(raw_smp_processor_id(),
924+ system_32bit_el0_cpumask());
925+}
926+
914927 asmlinkage void do_notify_resume(struct pt_regs *regs,
915928 unsigned long thread_flags)
916929 {
@@ -938,6 +951,19 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, · 21 unmodified lines
938951 if (thread_flags & _TIF_NOTIFY_RESUME) {
939952 tracehook_notify_resume(regs);
940953 rseq_handle_notify_resume(NULL, regs);
954+
955+ /*
956+ * If we reschedule after checking the affinity
957+ * then we must ensure that TIF_NOTIFY_RESUME
958+ * is set so that we check the affinity again.
959+ * Since tracehook_notify_resume() clears the
960+ * flag, ensure that the compiler doesn't move
961+ * it after the affinity check.
962+ */
963+ barrier();
964+
965+ if (cpu_affinity_invalid(regs))
966+ force_sig(SIGKILL);
941967 }
942968
943969 if (thread_flags & _TIF_FOREIGN_FPSTATE)
arch/arm64/kernel/smccc-call.S+83−0
@@ -7,8 +7,34 @@ · 6 unmodified lines
77
88 #include <asm/asm-offsets.h>
99 #include <asm/assembler.h>
10+#include <asm/thread_info.h>
11+
12+/*
13+ * If we have SMCCC v1.3 and (as is likely) no SVE state in
14+ * the registers then set the SMCCC hint bit to say there's no
15+ * need to preserve it. Do this by directly adjusting the SMCCC
16+ * function value which is already stored in x0 ready to be called.
17+ */
18+SYM_FUNC_START(__arm_smccc_sve_check)
19+
20+ ldr_l x16, smccc_has_sve_hint
21+ cbz x16, 2f
22+
23+ get_current_task x16
24+ ldr x16, [x16, #TSK_TI_FLAGS]
25+ tbnz x16, #TIF_FOREIGN_FPSTATE, 1f // Any live FP state?
26+ tbnz x16, #TIF_SVE, 2f // Does that state include SVE?
27+
28+1: orr x0, x0, ARM_SMCCC_1_3_SVE_HINT
29+
30+2: ret
31+SYM_FUNC_END(__arm_smccc_sve_check)
32+EXPORT_SYMBOL(__arm_smccc_sve_check)
1033
1134 .macro SMCCC instr
35+alternative_if ARM64_SVE
36+ bl __arm_smccc_sve_check
37+alternative_else_nop_endif
1238 \instr #0
1339 ldr x4, [sp]
1440 stp x0, x1, [x4, #ARM_SMCCC_RES_X0_OFFS]
@@ -43,3 +69,60 @@ SYM_FUNC_START(__arm_smccc_hvc) · 28 unmodified lines
4369 SMCCC hvc
4470 SYM_FUNC_END(__arm_smccc_hvc)
4571 EXPORT_SYMBOL(__arm_smccc_hvc)
72+
73+ .macro SMCCC_1_2 instr
74+ /* Save `res` and free a GPR that won't be clobbered */
75+ stp x1, x19, [sp, #-16]!
76+
77+ /* Ensure `args` won't be clobbered while loading regs in next step */
78+ mov x19, x0
79+
80+ /* Load the registers x0 - x17 from the struct arm_smccc_1_2_regs */
81+ ldp x0, x1, [x19, #ARM_SMCCC_1_2_REGS_X0_OFFS]
82+ ldp x2, x3, [x19, #ARM_SMCCC_1_2_REGS_X2_OFFS]
83+ ldp x4, x5, [x19, #ARM_SMCCC_1_2_REGS_X4_OFFS]
84+ ldp x6, x7, [x19, #ARM_SMCCC_1_2_REGS_X6_OFFS]
85+ ldp x8, x9, [x19, #ARM_SMCCC_1_2_REGS_X8_OFFS]
86+ ldp x10, x11, [x19, #ARM_SMCCC_1_2_REGS_X10_OFFS]
87+ ldp x12, x13, [x19, #ARM_SMCCC_1_2_REGS_X12_OFFS]
88+ ldp x14, x15, [x19, #ARM_SMCCC_1_2_REGS_X14_OFFS]
89+ ldp x16, x17, [x19, #ARM_SMCCC_1_2_REGS_X16_OFFS]
90+
91+ \instr #0
92+
93+ /* Load the `res` from the stack */
94+ ldr x19, [sp]
95+
96+ /* Store the registers x0 - x17 into the result structure */
97+ stp x0, x1, [x19, #ARM_SMCCC_1_2_REGS_X0_OFFS]
98+ stp x2, x3, [x19, #ARM_SMCCC_1_2_REGS_X2_OFFS]
99+ stp x4, x5, [x19, #ARM_SMCCC_1_2_REGS_X4_OFFS]
100+ stp x6, x7, [x19, #ARM_SMCCC_1_2_REGS_X6_OFFS]
101+ stp x8, x9, [x19, #ARM_SMCCC_1_2_REGS_X8_OFFS]
102+ stp x10, x11, [x19, #ARM_SMCCC_1_2_REGS_X10_OFFS]
103+ stp x12, x13, [x19, #ARM_SMCCC_1_2_REGS_X12_OFFS]
104+ stp x14, x15, [x19, #ARM_SMCCC_1_2_REGS_X14_OFFS]
105+ stp x16, x17, [x19, #ARM_SMCCC_1_2_REGS_X16_OFFS]
106+
107+ /* Restore original x19 */
108+ ldp xzr, x19, [sp], #16
109+ ret
110+.endm
111+
112+/*
113+ * void arm_smccc_1_2_hvc(const struct arm_smccc_1_2_regs *args,
114+ * struct arm_smccc_1_2_regs *res);
115+ */
116+SYM_FUNC_START(arm_smccc_1_2_hvc)
117+ SMCCC_1_2 hvc
118+SYM_FUNC_END(arm_smccc_1_2_hvc)
119+EXPORT_SYMBOL(arm_smccc_1_2_hvc)
120+
121+/*
122+ * void arm_smccc_1_2_smc(const struct arm_smccc_1_2_regs *args,
123+ * struct arm_smccc_1_2_regs *res);
124+ */
125+SYM_FUNC_START(arm_smccc_1_2_smc)
126+ SMCCC_1_2 smc
127+SYM_FUNC_END(arm_smccc_1_2_smc)
128+EXPORT_SYMBOL(arm_smccc_1_2_smc)
arch/arm64/kernel/smp.c+7−9
@@ -120,9 +120,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) · 119 unmodified lines
120120 * page tables.
121121 */
122122 secondary_data.task = idle;
123− secondary_data.stack = task_stack_page(idle) + THREAD_SIZE;
124123 update_cpu_boot_status(CPU_MMU_OFF);
125− __flush_dcache_area(&secondary_data, sizeof(secondary_data));
126124
127125 /* Now bring the CPU into our world */
128126 ret = boot_secondary(cpu, idle);
@@ -142,8 +140,6 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) · 13 unmodified lines
142140
143141 pr_crit("CPU%u: failed to come online\n", cpu);
144142 secondary_data.task = NULL;
145− secondary_data.stack = NULL;
146− __flush_dcache_area(&secondary_data, sizeof(secondary_data));
147143 status = READ_ONCE(secondary_data.status);
148144 if (status == CPU_MMU_OFF)
149145 status = READ_ONCE(__early_cpu_boot_status);
@@ -202,10 +198,7 @@ asmlinkage notrace void secondary_start_kernel(void) · 52 unmodified lines
202198 u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
203199 struct mm_struct *mm = &init_mm;
204200 const struct cpu_operations *ops;
205− unsigned int cpu;
206−
207− cpu = task_cpu(current);
208− set_my_cpu_offset(per_cpu_offset(cpu));
201+ unsigned int cpu = smp_processor_id();
209202
210203 /*
211204 * All kernel threads share the same mm context; grab a
@@ -351,7 +344,7 @@ void __cpu_die(unsigned int cpu) · 139 unmodified lines
351344 pr_crit("CPU%u: cpu didn't die\n", cpu);
352345 return;
353346 }
354− pr_notice("CPU%u: shutdown\n", cpu);
347+ pr_debug("CPU%u: shutdown\n", cpu);
355348
356349 /*
357350 * Now that the dying CPU is beyond the point of no return w.r.t.
@@ -451,6 +444,11 @@ void __init smp_cpus_done(unsigned int max_cpus) · 93 unmodified lines
451444
452445 void __init smp_prepare_boot_cpu(void)
453446 {
447+ /*
448+ * The runtime per-cpu areas have been allocated by
449+ * setup_per_cpu_areas(), and CPU0's boot time per-cpu area will be
450+ * freed shortly, so we must move over to the runtime per-cpu area.
451+ */
454452 set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
455453 cpuinfo_store_boot_cpu();
456454
arch/arm64/kernel/smp_spin_table.c+4−3
@@ -36,7 +36,7 @@ static void write_pen_release(u64 val) · 35 unmodified lines
3636 unsigned long size = sizeof(secondary_holding_pen_release);
3737
3838 secondary_holding_pen_release = val;
39− __flush_dcache_area(start, size);
39+ dcache_clean_inval_poc((unsigned long)start, (unsigned long)start + size);
4040 }
4141
4242
@@ -90,8 +90,9 @@ static int smp_spin_table_cpu_prepare(unsigned int cpu) · 47 unmodified lines
9090 * the boot protocol.
9191 */
9292 writeq_relaxed(pa_holding_pen, release_addr);
93− __flush_dcache_area((__force void *)release_addr,
94− sizeof(*release_addr));
93+ dcache_clean_inval_poc((__force unsigned long)release_addr,
94+ (__force unsigned long)release_addr +
95+ sizeof(*release_addr));
9596
9697 /*
9798 * Send an event to wake up the secondary CPU.
arch/arm64/kernel/stacktrace.c+8−10
@@ -68,13 +68,17 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) · 67 unmodified lines
6868 unsigned long fp = frame->fp;
6969 struct stack_info info;
7070
71− if (fp & 0xf)
72− return -EINVAL;
73−
7471 if (!tsk)
7572 tsk = current;
7673
77− if (!on_accessible_stack(tsk, fp, &info))
74+ /* Final frame; nothing to unwind */
75+ if (fp == (unsigned long)task_pt_regs(tsk)->stackframe)
76+ return -ENOENT;
77+
78+ if (fp & 0x7)
79+ return -EINVAL;
80+
81+ if (!on_accessible_stack(tsk, fp, 16, &info))
7882 return -EINVAL;
7983
8084 if (test_bit(info.type, frame->stacks_done))
@@ -128,12 +132,6 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) · 47 unmodified lines
128132
129133 frame->pc = ptrauth_strip_insn_pac(frame->pc);
130134
131− /*
132− * This is a terminal record, so we have finished unwinding.
133− */
134− if (!frame->fp && !frame->pc)
135− return -ENOENT;
136−
137135 return 0;
138136 }
139137 NOKPROBE_SYMBOL(unwind_frame);
arch/arm64/kernel/suspend.c+11−1
@@ -7,6 +7,7 @@ · 6 unmodified lines
77 #include <asm/alternative.h>
88 #include <asm/cacheflush.h>
99 #include <asm/cpufeature.h>
10+#include <asm/cpuidle.h>
1011 #include <asm/daifflags.h>
1112 #include <asm/debug-monitors.h>
1213 #include <asm/exec.h>
@@ -91,6 +92,7 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) · 78 unmodified lines
9192 int ret = 0;
9293 unsigned long flags;
9394 struct sleep_stack_data state;
95+ struct arm_cpuidle_irq_context context;
9496
9597 /* Report any MTE async fault before going to suspend */
9698 mte_suspend_enter();
@@ -103,12 +105,18 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) · 6 unmodified lines
103105 flags = local_daif_save();
104106
105107 /*
106− * Function graph tracer state gets incosistent when the kernel
108+ * Function graph tracer state gets inconsistent when the kernel
107109 * calls functions that never return (aka suspend finishers) hence
108110 * disable graph tracing during their execution.
109111 */
110112 pause_graph_tracing();
111113
114+ /*
115+ * Switch to using DAIF.IF instead of PMR in order to reliably
116+ * resume if we're using pseudo-NMIs.
117+ */
118+ arm_cpuidle_save_irq_context(&context);
119+
112120 if (__cpu_suspend_enter(&state)) {
113121 /* Call the suspend finisher */
114122 ret = fn(arg);
@@ -126,6 +134,8 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) · 11 unmodified lines
126134 RCU_NONIDLE(__cpu_suspend_exit());
127135 }
128136
137+ arm_cpuidle_restore_irq_context(&context);
138+
129139 unpause_graph_tracing();
130140
131141 /*
arch/arm64/kernel/sys_compat.c+1−1
@@ -41,7 +41,7 @@ __do_compat_cache_op(unsigned long start, unsigned long end) · 40 unmodified lines
4141 dsb(ish);
4242 }
4343
44− ret = __flush_cache_user_range(start, start + chunk);
44+ ret = caches_clean_inval_user_pou(start, start + chunk);
4545 if (ret)
4646 return ret;
4747
arch/arm64/kernel/traps.c+100−35
@@ -38,6 +38,7 @@ · 37 unmodified lines
3838 #include <asm/extable.h>
3939 #include <asm/insn.h>
4040 #include <asm/kprobes.h>
41+#include <asm/patching.h>
4142 #include <asm/traps.h>
4243 #include <asm/smp.h>
4344 #include <asm/stack_pointer.h>
@@ -45,11 +46,102 @@ · 1 unmodified line
4546 #include <asm/system_misc.h>
4647 #include <asm/sysreg.h>
4748
48−static const char *handler[] = {
49− "Synchronous Abort",
50− "IRQ",
51− "FIQ",
52− "Error"
49+static bool __kprobes __check_eq(unsigned long pstate)
50+{
51+ return (pstate & PSR_Z_BIT) != 0;
52+}
53+
54+static bool __kprobes __check_ne(unsigned long pstate)
55+{
56+ return (pstate & PSR_Z_BIT) == 0;
57+}
58+
59+static bool __kprobes __check_cs(unsigned long pstate)
60+{
61+ return (pstate & PSR_C_BIT) != 0;
62+}
63+
64+static bool __kprobes __check_cc(unsigned long pstate)
65+{
66+ return (pstate & PSR_C_BIT) == 0;
67+}
68+
69+static bool __kprobes __check_mi(unsigned long pstate)
70+{
71+ return (pstate & PSR_N_BIT) != 0;
72+}
73+
74+static bool __kprobes __check_pl(unsigned long pstate)
75+{
76+ return (pstate & PSR_N_BIT) == 0;
77+}
78+
79+static bool __kprobes __check_vs(unsigned long pstate)
80+{
81+ return (pstate & PSR_V_BIT) != 0;
82+}
83+
84+static bool __kprobes __check_vc(unsigned long pstate)
85+{
86+ return (pstate & PSR_V_BIT) == 0;
87+}
88+
89+static bool __kprobes __check_hi(unsigned long pstate)
90+{
91+ pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */
92+ return (pstate & PSR_C_BIT) != 0;
93+}
94+
95+static bool __kprobes __check_ls(unsigned long pstate)
96+{
97+ pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */
98+ return (pstate & PSR_C_BIT) == 0;
99+}
100+
101+static bool __kprobes __check_ge(unsigned long pstate)
102+{
103+ pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */
104+ return (pstate & PSR_N_BIT) == 0;
105+}
106+
107+static bool __kprobes __check_lt(unsigned long pstate)
108+{
109+ pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */
110+ return (pstate & PSR_N_BIT) != 0;
111+}
112+
113+static bool __kprobes __check_gt(unsigned long pstate)
114+{
115+ /*PSR_N_BIT ^= PSR_V_BIT */
116+ unsigned long temp = pstate ^ (pstate << 3);
117+
118+ temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */
119+ return (temp & PSR_N_BIT) == 0;
120+}
121+
122+static bool __kprobes __check_le(unsigned long pstate)
123+{
124+ /*PSR_N_BIT ^= PSR_V_BIT */
125+ unsigned long temp = pstate ^ (pstate << 3);
126+
127+ temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */
128+ return (temp & PSR_N_BIT) != 0;
129+}
130+
131+static bool __kprobes __check_al(unsigned long pstate)
132+{
133+ return true;
134+}
135+
136+/*
137+ * Note that the ARMv8 ARM calls condition code 0b1111 "nv", but states that
138+ * it behaves identically to 0b1110 ("al").
139+ */
140+pstate_check_t * const aarch32_opcode_cond_checks[16] = {
141+ __check_eq, __check_ne, __check_cs, __check_cc,
142+ __check_mi, __check_pl, __check_vs, __check_vc,
143+ __check_hi, __check_ls, __check_ge, __check_lt,
144+ __check_gt, __check_le, __check_al, __check_al
53145 };
54146
55147 int show_unhandled_signals = 0;
@@ -750,28 +842,9 @@ const char *esr_get_class_string(u32 esr) · 694 unmodified lines
750842 return esr_class_str[ESR_ELx_EC(esr)];
751843 }
752844
753−/*
754− * bad_mode handles the impossible case in the exception vector. This is always
755− * fatal.
756− */
757−asmlinkage void notrace bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
758−{
759− arm64_enter_nmi(regs);
760−
761− console_verbose();
762−
763− pr_crit("Bad mode in %s handler detected on CPU%d, code 0x%08x -- %s\n",
764− handler[reason], smp_processor_id(), esr,
765− esr_get_class_string(esr));
766−
767− __show_regs(regs);
768− local_daif_mask();
769− panic("bad mode");
770−}
771−
772845 /*
773846 * bad_el0_sync handles unexpected, but potentially recoverable synchronous
774− * exceptions taken from EL0. Unlike bad_mode, this returns.
847+ * exceptions taken from EL0.
775848 */
776849 void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
777850 {
@@ -789,15 +862,11 @@ void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) · 11 unmodified lines
789862 DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
790863 __aligned(16);
791864
792−asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs)
865+void panic_bad_stack(struct pt_regs *regs, unsigned int esr, unsigned long far)
793866 {
794867 unsigned long tsk_stk = (unsigned long)current->stack;
795868 unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
796869 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
797− unsigned int esr = read_sysreg(esr_el1);
798− unsigned long far = read_sysreg(far_el1);
799−
800− arm64_enter_nmi(regs);
801870
802871 console_verbose();
803872 pr_emerg("Insufficient stack space to handle exception!");
@@ -870,15 +939,11 @@ bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr) · 66 unmodified lines
870939 }
871940 }
872941
873−asmlinkage void noinstr do_serror(struct pt_regs *regs, unsigned int esr)
942+void do_serror(struct pt_regs *regs, unsigned int esr)
874943 {
875− arm64_enter_nmi(regs);
876−
877944 /* non-RAS errors are not containable */
878945 if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr))
879946 arm64_serror_panic(regs, esr);
880−
881− arm64_exit_nmi(regs);
882947 }
883948
884949 /* GENERIC_BUG traps */
arch/arm64/kvm/arm.c+11−2
@@ -692,6 +692,15 @@ static void check_vcpu_requests(struct kvm_vcpu *vcpu) · 691 unmodified lines
692692 }
693693 }
694694
695+static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu)
696+{
697+ if (likely(!vcpu_mode_is_32bit(vcpu)))
698+ return false;
699+
700+ return !system_supports_32bit_el0() ||
701+ static_branch_unlikely(&arm64_mismatched_32bit_el0);
702+}
703+
695704 /**
696705 * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code
697706 * @vcpu: The VCPU pointer
@@ -877,7 +886,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) · 179 unmodified lines
877886 * with the asymmetric AArch32 case), return to userspace with
878887 * a fatal error.
879888 */
880− if (!system_supports_32bit_el0() && vcpu_mode_is_32bit(vcpu)) {
889+ if (vcpu_mode_is_bad_32bit(vcpu)) {
881890 /*
882891 * As we have caught the guest red-handed, decide that
883892 * it isn't fit for purpose anymore by making the vcpu
@@ -1078,7 +1087,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, · 194 unmodified lines
10781087 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
10791088 stage2_unmap_vm(vcpu->kvm);
10801089 else
1081− __flush_icache_all();
1090+ icache_inval_all_pou();
10821091 }
10831092
10841093 vcpu_reset_hcr(vcpu);
arch/arm64/kvm/hyp/nvhe/cache.S+2−2
@@ -7,7 +7,7 @@ · 6 unmodified lines
77 #include <asm/assembler.h>
88 #include <asm/alternative.h>
99
10−SYM_FUNC_START_PI(__flush_dcache_area)
10+SYM_FUNC_START_PI(dcache_clean_inval_poc)
1111 dcache_by_line_op civac, sy, x0, x1, x2, x3
1212 ret
13−SYM_FUNC_END_PI(__flush_dcache_area)
13+SYM_FUNC_END_PI(dcache_clean_inval_poc)
arch/arm64/kvm/hyp/nvhe/setup.c+2−1
@@ -134,7 +134,8 @@ static void update_nvhe_init_params(void) · 133 unmodified lines
134134 for (i = 0; i < hyp_nr_cpus; i++) {
135135 params = per_cpu_ptr(&kvm_init_params, i);
136136 params->pgd_pa = __hyp_pa(pkvm_pgtable.pgd);
137− __flush_dcache_area(params, sizeof(*params));
137+ dcache_clean_inval_poc((unsigned long)params,
138+ (unsigned long)params + sizeof(*params));
138139 }
139140 }
140141
arch/arm64/kvm/hyp/nvhe/tlb.c+1−1
@@ -104,7 +104,7 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, · 103 unmodified lines
104104 * you should be running with VHE enabled.
105105 */
106106 if (icache_is_vpipt())
107− __flush_icache_all();
107+ icache_inval_all_pou();
108108
109109 __tlb_switch_to_host(&cxt);
110110 }
arch/arm64/kvm/hyp/pgtable.c+10−3
@@ -839,8 +839,11 @@ static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 838 unmodified lines
839839 stage2_put_pte(ptep, mmu, addr, level, mm_ops);
840840
841841 if (need_flush) {
842− __flush_dcache_area(kvm_pte_follow(pte, mm_ops),
843− kvm_granule_size(level));
842+ kvm_pte_t *pte_follow = kvm_pte_follow(pte, mm_ops);
843+
844+ dcache_clean_inval_poc((unsigned long)pte_follow,
845+ (unsigned long)pte_follow +
846+ kvm_granule_size(level));
844847 }
845848
846849 if (childp)
@@ -988,11 +991,15 @@ static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, · 141 unmodified lines
988991 struct kvm_pgtable *pgt = arg;
989992 struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
990993 kvm_pte_t pte = *ptep;
994+ kvm_pte_t *pte_follow;
991995
992996 if (!kvm_pte_valid(pte) || !stage2_pte_cacheable(pgt, pte))
993997 return 0;
994998
995− __flush_dcache_area(kvm_pte_follow(pte, mm_ops), kvm_granule_size(level));
999+ pte_follow = kvm_pte_follow(pte, mm_ops);
1000+ dcache_clean_inval_poc((unsigned long)pte_follow,
1001+ (unsigned long)pte_follow +
1002+ kvm_granule_size(level));
9961003 return 0;
9971004 }
9981005
arch/arm64/lib/Makefile+3−1
@@ -1,7 +1,7 @@
11 # SPDX-License-Identifier: GPL-2.0
22 lib-y := clear_user.o delay.o copy_from_user.o \
33 copy_to_user.o copy_in_user.o copy_page.o \
4− clear_page.o csum.o memchr.o memcpy.o memmove.o \
4+ clear_page.o csum.o insn.o memchr.o memcpy.o \
55 memset.o memcmp.o strcmp.o strncmp.o strlen.o \
66 strnlen.o strchr.o strrchr.o tishift.o
77
@@ -18,3 +18,5 @@ obj-$(CONFIG_CRC32) += crc32.o · 10 unmodified lines
1818 obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
1919
2020 obj-$(CONFIG_ARM64_MTE) += mte.o
21+
22+obj-$(CONFIG_KASAN_SW_TAGS) += kasan_sw_tags.o
arch/arm64/lib/clear_user.S+27−20
@@ -1,12 +1,9 @@
11 /* SPDX-License-Identifier: GPL-2.0-only */
22 /*
3− * Based on arch/arm/lib/clear_user.S
4− *
5− * Copyright (C) 2012 ARM Ltd.
3+ * Copyright (C) 2021 Arm Ltd.
64 */
7−#include <linux/linkage.h>
85
9−#include <asm/asm-uaccess.h>
6+#include <linux/linkage.h>
107 #include <asm/assembler.h>
118
129 .text
@@ -19,25 +16,33 @@ · 6 unmodified lines
1916 *
2017 * Alignment fixed up by hardware.
2118 */
19+
20+ .p2align 4
21+ // Alignment is for the loop, but since the prologue (including BTI)
22+ // is also 16 bytes we can keep any padding outside the function
2223 SYM_FUNC_START(__arch_clear_user)
23− mov x2, x1 // save the size for fixup return
24+ add x2, x0, x1
2425 subs x1, x1, #8
2526 b.mi 2f
2627 1:
27−user_ldst 9f, sttr, xzr, x0, 8
28+USER(9f, sttr xzr, [x0])
29+ add x0, x0, #8
2830 subs x1, x1, #8
29− b.pl 1b
30−2: adds x1, x1, #4
31− b.mi 3f
32−user_ldst 9f, sttr, wzr, x0, 4
33− sub x1, x1, #4
34−3: adds x1, x1, #2
35− b.mi 4f
36−user_ldst 9f, sttrh, wzr, x0, 2
37− sub x1, x1, #2
38−4: adds x1, x1, #1
39− b.mi 5f
40−user_ldst 9f, sttrb, wzr, x0, 0
31+ b.hi 1b
32+USER(9f, sttr xzr, [x2, #-8])
33+ mov x0, #0
34+ ret
35+
36+2: tbz x1, #2, 3f
37+USER(9f, sttr wzr, [x0])
38+USER(8f, sttr wzr, [x2, #-4])
39+ mov x0, #0
40+ ret
41+
42+3: tbz x1, #1, 4f
43+USER(9f, sttrh wzr, [x0])
44+4: tbz x1, #0, 5f
45+USER(7f, sttrb wzr, [x2, #-1])
4146 5: mov x0, #0
4247 ret
4348 SYM_FUNC_END(__arch_clear_user)
@@ -45,6 +50,8 @@ EXPORT_SYMBOL(__arch_clear_user) · 1 unmodified line
4550
4651 .section .fixup,"ax"
4752 .align 2
48−9: mov x0, x2 // return the original size
53+7: sub x0, x2, #5 // Adjust for faulting on the final byte...
54+8: add x0, x0, #4 // ...or the second word of the 4-7 byte case
55+9: sub x0, x2, x0
4956 ret
5057 .previous
arch/arm64/lib/insn.crenamed from arch/arm64/kernel/insn.c+4−245
@@ -7,21 +7,14 @@ · 6 unmodified lines
77 */
88 #include <linux/bitops.h>
99 #include <linux/bug.h>
10−#include <linux/compiler.h>
11−#include <linux/kernel.h>
12−#include <linux/mm.h>
13−#include <linux/smp.h>
14−#include <linux/spinlock.h>
15−#include <linux/stop_machine.h>
10+#include <linux/printk.h>
11+#include <linux/sizes.h>
1612 #include <linux/types.h>
17−#include <linux/uaccess.h>
1813
19−#include <asm/cacheflush.h>
2014 #include <asm/debug-monitors.h>
21−#include <asm/fixmap.h>
15+#include <asm/errno.h>
2216 #include <asm/insn.h>
2317 #include <asm/kprobes.h>
24−#include <asm/sections.h>
2518
2619 #define AARCH64_INSN_SF_BIT BIT(31)
2720 #define AARCH64_INSN_N_BIT BIT(22)
@@ -30,7 +23,7 @@ · 2 unmodified lines
3023 static const int aarch64_insn_encoding_class[] = {
3124 AARCH64_INSN_CLS_UNKNOWN,
3225 AARCH64_INSN_CLS_UNKNOWN,
33− AARCH64_INSN_CLS_UNKNOWN,
26+ AARCH64_INSN_CLS_SVE,
3427 AARCH64_INSN_CLS_UNKNOWN,
3528 AARCH64_INSN_CLS_LDST,
3629 AARCH64_INSN_CLS_DP_REG,
@@ -83,81 +76,6 @@ bool aarch64_insn_is_branch_imm(u32 insn) · 46 unmodified lines
8376 aarch64_insn_is_bcond(insn));
8477 }
8578
86−static DEFINE_RAW_SPINLOCK(patch_lock);
87−
88−static bool is_exit_text(unsigned long addr)
89−{
90− /* discarded with init text/data */
91− return system_state < SYSTEM_RUNNING &&
92− addr >= (unsigned long)__exittext_begin &&
93− addr < (unsigned long)__exittext_end;
94−}
95−
96−static bool is_image_text(unsigned long addr)
97−{
98− return core_kernel_text(addr) || is_exit_text(addr);
99−}
100−
101−static void __kprobes *patch_map(void *addr, int fixmap)
102−{
103− unsigned long uintaddr = (uintptr_t) addr;
104− bool image = is_image_text(uintaddr);
105− struct page *page;
106−
107− if (image)
108− page = phys_to_page(__pa_symbol(addr));
109− else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX))
110− page = vmalloc_to_page(addr);
111− else
112− return addr;
113−
114− BUG_ON(!page);
115− return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
116− (uintaddr & ~PAGE_MASK));
117−}
118−
119−static void __kprobes patch_unmap(int fixmap)
120−{
121− clear_fixmap(fixmap);
122−}
123−/*
124− * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
125− * little-endian.
126− */
127−int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
128−{
129− int ret;
130− __le32 val;
131−
132− ret = copy_from_kernel_nofault(&val, addr, AARCH64_INSN_SIZE);
133− if (!ret)
134− *insnp = le32_to_cpu(val);
135−
136− return ret;
137−}
138−
139−static int __kprobes __aarch64_insn_write(void *addr, __le32 insn)
140−{
141− void *waddr = addr;
142− unsigned long flags = 0;
143− int ret;
144−
145− raw_spin_lock_irqsave(&patch_lock, flags);
146− waddr = patch_map(addr, FIX_TEXT_POKE0);
147−
148− ret = copy_to_kernel_nofault(waddr, &insn, AARCH64_INSN_SIZE);
149−
150− patch_unmap(FIX_TEXT_POKE0);
151− raw_spin_unlock_irqrestore(&patch_lock, flags);
152−
153− return ret;
154−}
155−
156−int __kprobes aarch64_insn_write(void *addr, u32 insn)
157−{
158− return __aarch64_insn_write(addr, cpu_to_le32(insn));
159−}
160−
16179 bool __kprobes aarch64_insn_uses_literal(u32 insn)
16280 {
16381 /* ldr/ldrsw (literal), prfm */
@@ -187,67 +105,6 @@ bool __kprobes aarch64_insn_is_branch(u32 insn) · 23 unmodified lines
187105 aarch64_insn_is_bcond(insn);
188106 }
189107
190−int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
191−{
192− u32 *tp = addr;
193− int ret;
194−
195− /* A64 instructions must be word aligned */
196− if ((uintptr_t)tp & 0x3)
197− return -EINVAL;
198−
199− ret = aarch64_insn_write(tp, insn);
200− if (ret == 0)
201− __flush_icache_range((uintptr_t)tp,
202− (uintptr_t)tp + AARCH64_INSN_SIZE);
203−
204− return ret;
205−}
206−
207−struct aarch64_insn_patch {
208− void **text_addrs;
209− u32 *new_insns;
210− int insn_cnt;
211− atomic_t cpu_count;
212−};
213−
214−static int __kprobes aarch64_insn_patch_text_cb(void *arg)
215−{
216− int i, ret = 0;
217− struct aarch64_insn_patch *pp = arg;
218−
219− /* The first CPU becomes master */
220− if (atomic_inc_return(&pp->cpu_count) == 1) {
221− for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
222− ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
223− pp->new_insns[i]);
224− /* Notify other processors with an additional increment. */
225− atomic_inc(&pp->cpu_count);
226− } else {
227− while (atomic_read(&pp->cpu_count) <= num_online_cpus())
228− cpu_relax();
229− isb();
230− }
231−
232− return ret;
233−}
234−
235−int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
236−{
237− struct aarch64_insn_patch patch = {
238− .text_addrs = addrs,
239− .new_insns = insns,
240− .insn_cnt = cnt,
241− .cpu_count = ATOMIC_INIT(0),
242− };
243−
244− if (cnt <= 0)
245− return -EINVAL;
246−
247− return stop_machine_cpuslocked(aarch64_insn_patch_text_cb, &patch,
248− cpu_online_mask);
249−}
250−
251108 static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,
252109 u32 *maskp, int *shiftp)
253110 {
@@ -1432,104 +1289,6 @@ u32 aarch32_insn_mcr_extract_crm(u32 insn) · 1178 unmodified lines
14321289 return insn & CRM_MASK;
14331290 }
14341291
1435−static bool __kprobes __check_eq(unsigned long pstate)
1436−{
1437− return (pstate & PSR_Z_BIT) != 0;
1438−}
1439−
1440−static bool __kprobes __check_ne(unsigned long pstate)
1441−{
1442− return (pstate & PSR_Z_BIT) == 0;
1443−}
1444−
1445−static bool __kprobes __check_cs(unsigned long pstate)
1446−{
1447− return (pstate & PSR_C_BIT) != 0;
1448−}
1449−
1450−static bool __kprobes __check_cc(unsigned long pstate)
1451−{
1452− return (pstate & PSR_C_BIT) == 0;
1453−}
1454−
1455−static bool __kprobes __check_mi(unsigned long pstate)
1456−{
1457− return (pstate & PSR_N_BIT) != 0;
1458−}
1459−
1460−static bool __kprobes __check_pl(unsigned long pstate)
1461−{
1462− return (pstate & PSR_N_BIT) == 0;
1463−}
1464−
1465−static bool __kprobes __check_vs(unsigned long pstate)
1466−{
1467− return (pstate & PSR_V_BIT) != 0;
1468−}
1469−
1470−static bool __kprobes __check_vc(unsigned long pstate)
1471−{
1472− return (pstate & PSR_V_BIT) == 0;
1473−}
1474−
1475−static bool __kprobes __check_hi(unsigned long pstate)
1476−{
1477− pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */
1478− return (pstate & PSR_C_BIT) != 0;
1479−}
1480−
1481−static bool __kprobes __check_ls(unsigned long pstate)
1482−{
1483− pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */
1484− return (pstate & PSR_C_BIT) == 0;
1485−}
1486−
1487−static bool __kprobes __check_ge(unsigned long pstate)
1488−{
1489− pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */
1490− return (pstate & PSR_N_BIT) == 0;
1491−}
1492−
1493−static bool __kprobes __check_lt(unsigned long pstate)
1494−{
1495− pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */
1496− return (pstate & PSR_N_BIT) != 0;
1497−}
1498−
1499−static bool __kprobes __check_gt(unsigned long pstate)
1500−{
1501− /*PSR_N_BIT ^= PSR_V_BIT */
1502− unsigned long temp = pstate ^ (pstate << 3);
1503−
1504− temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */
1505− return (temp & PSR_N_BIT) == 0;
1506−}
1507−
1508−static bool __kprobes __check_le(unsigned long pstate)
1509−{
1510− /*PSR_N_BIT ^= PSR_V_BIT */
1511− unsigned long temp = pstate ^ (pstate << 3);
1512−
1513− temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */
1514− return (temp & PSR_N_BIT) != 0;
1515−}
1516−
1517−static bool __kprobes __check_al(unsigned long pstate)
1518−{
1519− return true;
1520−}
1521−
1522−/*
1523− * Note that the ARMv8 ARM calls condition code 0b1111 "nv", but states that
1524− * it behaves identically to 0b1110 ("al").
1525− */
1526−pstate_check_t * const aarch32_opcode_cond_checks[16] = {
1527− __check_eq, __check_ne, __check_cs, __check_cc,
1528− __check_mi, __check_pl, __check_vs, __check_vc,
1529− __check_hi, __check_ls, __check_ge, __check_lt,
1530− __check_gt, __check_le, __check_al, __check_al
1531−};
1532−
15331292 static bool range_of_ones(u64 val)
15341293 {
15351294 /* Doesn't handle full ones or full zeroes */
arch/arm64/lib/kasan_sw_tags.S+76−0
@@ -0,0 +1,76 @@
1+/* SPDX-License-Identifier: GPL-2.0-only */
2+/*
3+ * Copyright (C) 2020 Google LLC
4+ */
5+
6+#include <linux/linkage.h>
7+#include <asm/assembler.h>
8+
9+/*
10+ * Report a tag mismatch detected by tag-based KASAN.
11+ *
12+ * A compiler-generated thunk calls this with a non-AAPCS calling
13+ * convention. Upon entry to this function, registers are as follows:
14+ *
15+ * x0: fault address (see below for restore)
16+ * x1: fault description (see below for restore)
17+ * x2 to x15: callee-saved
18+ * x16 to x17: safe to clobber
19+ * x18 to x30: callee-saved
20+ * sp: pre-decremented by 256 bytes (see below for restore)
21+ *
22+ * The caller has decremented the SP by 256 bytes, and created a
23+ * structure on the stack as follows:
24+ *
25+ * sp + 0..15: x0 and x1 to be restored
26+ * sp + 16..231: free for use
27+ * sp + 232..247: x29 and x30 (same as in GPRs)
28+ * sp + 248..255: free for use
29+ *
30+ * Note that this is not a struct pt_regs.
31+ *
32+ * To call a regular AAPCS function we must save x2 to x15 (which we can
33+ * store in the gaps), and create a frame record (for which we can use
34+ * x29 and x30 spilled by the caller as those match the GPRs).
35+ *
36+ * The caller expects x0 and x1 to be restored from the structure, and
37+ * for the structure to be removed from the stack (i.e. the SP must be
38+ * incremented by 256 prior to return).
39+ */
40+SYM_CODE_START(__hwasan_tag_mismatch)
41+#ifdef BTI_C
42+ BTI_C
43+#endif
44+ add x29, sp, #232
45+ stp x2, x3, [sp, #8 * 2]
46+ stp x4, x5, [sp, #8 * 4]
47+ stp x6, x7, [sp, #8 * 6]
48+ stp x8, x9, [sp, #8 * 8]
49+ stp x10, x11, [sp, #8 * 10]
50+ stp x12, x13, [sp, #8 * 12]
51+ stp x14, x15, [sp, #8 * 14]
52+#ifndef CONFIG_SHADOW_CALL_STACK
53+ str x18, [sp, #8 * 18]
54+#endif
55+
56+ mov x2, x30
57+ bl kasan_tag_mismatch
58+
59+ ldp x0, x1, [sp]
60+ ldp x2, x3, [sp, #8 * 2]
61+ ldp x4, x5, [sp, #8 * 4]
62+ ldp x6, x7, [sp, #8 * 6]
63+ ldp x8, x9, [sp, #8 * 8]
64+ ldp x10, x11, [sp, #8 * 10]
65+ ldp x12, x13, [sp, #8 * 12]
66+ ldp x14, x15, [sp, #8 * 14]
67+#ifndef CONFIG_SHADOW_CALL_STACK
68+ ldr x18, [sp, #8 * 18]
69+#endif
70+ ldp x29, x30, [sp, #8 * 29]
71+
72+ /* remove the structure from the stack */
73+ add sp, sp, #256
74+ ret
75+SYM_CODE_END(__hwasan_tag_mismatch)
76+EXPORT_SYMBOL(__hwasan_tag_mismatch)
arch/arm64/lib/memchr.S+53−12
@@ -1,9 +1,6 @@
11 /* SPDX-License-Identifier: GPL-2.0-only */
22 /*
3− * Based on arch/arm/lib/memchr.S
4− *
5− * Copyright (C) 1995-2000 Russell King
6− * Copyright (C) 2013 ARM Ltd.
3+ * Copyright (C) 2021 Arm Ltd.
74 */
85
96 #include <linux/linkage.h>
@@ -19,16 +16,60 @@ · 9 unmodified lines
1916 * Returns:
2017 * x0 - address of first occurrence of 'c' or 0
2118 */
19+
20+#define L(label) .L ## label
21+
22+#define REP8_01 0x0101010101010101
23+#define REP8_7f 0x7f7f7f7f7f7f7f7f
24+
25+#define srcin x0
26+#define chrin w1
27+#define cntin x2
28+
29+#define result x0
30+
31+#define wordcnt x3
32+#define rep01 x4
33+#define repchr x5
34+#define cur_word x6
35+#define cur_byte w6
36+#define tmp x7
37+#define tmp2 x8
38+
39+ .p2align 4
40+ nop
2241 SYM_FUNC_START_WEAK_PI(memchr)
23− and w1, w1, #0xff
24−1: subs x2, x2, #1
25− b.mi 2f
26− ldrb w3, [x0], #1
27− cmp w3, w1
28− b.ne 1b
29− sub x0, x0, #1
42+ and chrin, chrin, #0xff
43+ lsr wordcnt, cntin, #3
44+ cbz wordcnt, L(byte_loop)
45+ mov rep01, #REP8_01
46+ mul repchr, x1, rep01
47+ and cntin, cntin, #7
48+L(word_loop):
49+ ldr cur_word, [srcin], #8
50+ sub wordcnt, wordcnt, #1
51+ eor cur_word, cur_word, repchr
52+ sub tmp, cur_word, rep01
53+ orr tmp2, cur_word, #REP8_7f
54+ bics tmp, tmp, tmp2
55+ b.ne L(found_word)
56+ cbnz wordcnt, L(word_loop)
57+L(byte_loop):
58+ cbz cntin, L(not_found)
59+ ldrb cur_byte, [srcin], #1
60+ sub cntin, cntin, #1
61+ cmp cur_byte, chrin
62+ b.ne L(byte_loop)
63+ sub srcin, srcin, #1
64+ ret
65+L(found_word):
66+CPU_LE( rev tmp, tmp)
67+ clz tmp, tmp
68+ sub tmp, tmp, #64
69+ add result, srcin, tmp, asr #3
3070 ret
31−2: mov x0, #0
71+L(not_found):
72+ mov result, #0
3273 ret
3374 SYM_FUNC_END_PI(memchr)
3475 EXPORT_SYMBOL_NOKASAN(memchr)
arch/arm64/lib/memcmp.S+119−227
@@ -1,247 +1,139 @@
11 /* SPDX-License-Identifier: GPL-2.0-only */
22 /*
3− * Copyright (C) 2013 ARM Ltd.
4− * Copyright (C) 2013 Linaro.
3+ * Copyright (c) 2013-2021, Arm Limited.
54 *
6− * This code is based on glibc cortex strings work originally authored by Linaro
7− * be found @
8− *
9− * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
10− * files/head:/src/aarch64/
5+ * Adapted from the original at:
6+ * https://github.com/ARM-software/optimized-routines/blob/e823e3abf5f89ecb/string/aarch64/memcmp.S
117 */
128
139 #include <linux/linkage.h>
1410 #include <asm/assembler.h>
1511
16−/*
17−* compare memory areas(when two memory areas' offset are different,
18−* alignment handled by the hardware)
19−*
20−* Parameters:
21−* x0 - const memory area 1 pointer
22−* x1 - const memory area 2 pointer
23−* x2 - the maximal compare byte length
24−* Returns:
25−* x0 - a compare result, maybe less than, equal to, or greater than ZERO
26−*/
12+/* Assumptions:
13+ *
14+ * ARMv8-a, AArch64, unaligned accesses.
15+ */
16+
17+#define L(label) .L ## label
2718
2819 /* Parameters and result. */
29−src1 .req x0
30−src2 .req x1
31−limit .req x2
32−result .req x0
20+#define src1 x0
21+#define src2 x1
22+#define limit x2
23+#define result w0
3324
3425 /* Internal variables. */
35−data1 .req x3
36−data1w .req w3
37−data2 .req x4
38−data2w .req w4
39−has_nul .req x5
40−diff .req x6
41−endloop .req x7
42−tmp1 .req x8
43−tmp2 .req x9
44−tmp3 .req x10
45−pos .req x11
46−limit_wd .req x12
47−mask .req x13
26+#define data1 x3
27+#define data1w w3
28+#define data1h x4
29+#define data2 x5
30+#define data2w w5
31+#define data2h x6
32+#define tmp1 x7
33+#define tmp2 x8
4834
4935 SYM_FUNC_START_WEAK_PI(memcmp)
50− cbz limit, .Lret0
51− eor tmp1, src1, src2
52− tst tmp1, #7
53− b.ne .Lmisaligned8
54− ands tmp1, src1, #7
55− b.ne .Lmutual_align
56− sub limit_wd, limit, #1 /* limit != 0, so no underflow. */
57− lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */
58− /*
59− * The input source addresses are at alignment boundary.
60− * Directly compare eight bytes each time.
61− */
62−.Lloop_aligned:
63− ldr data1, [src1], #8
64− ldr data2, [src2], #8
65−.Lstart_realigned:
66− subs limit_wd, limit_wd, #1
67− eor diff, data1, data2 /* Non-zero if differences found. */
68− csinv endloop, diff, xzr, cs /* Last Dword or differences. */
69− cbz endloop, .Lloop_aligned
70−
71− /* Not reached the limit, must have found a diff. */
72− tbz limit_wd, #63, .Lnot_limit
73−
74− /* Limit % 8 == 0 => the diff is in the last 8 bytes. */
75− ands limit, limit, #7
76− b.eq .Lnot_limit
77− /*
78− * The remained bytes less than 8. It is needed to extract valid data
79− * from last eight bytes of the intended memory range.
80− */
81− lsl limit, limit, #3 /* bytes-> bits. */
82− mov mask, #~0
83−CPU_BE( lsr mask, mask, limit )
84−CPU_LE( lsl mask, mask, limit )
85− bic data1, data1, mask
86− bic data2, data2, mask
87−
88− orr diff, diff, mask
89− b .Lnot_limit
90−
91−.Lmutual_align:
92− /*
93− * Sources are mutually aligned, but are not currently at an
94− * alignment boundary. Round down the addresses and then mask off
95− * the bytes that precede the start point.
96− */
97− bic src1, src1, #7
98− bic src2, src2, #7
99− ldr data1, [src1], #8
100− ldr data2, [src2], #8
101− /*
102− * We can not add limit with alignment offset(tmp1) here. Since the
103− * addition probably make the limit overflown.
104− */
105− sub limit_wd, limit, #1/*limit != 0, so no underflow.*/
106− and tmp3, limit_wd, #7
107− lsr limit_wd, limit_wd, #3
108− add tmp3, tmp3, tmp1
109− add limit_wd, limit_wd, tmp3, lsr #3
110− add limit, limit, tmp1/* Adjust the limit for the extra. */
111−
112− lsl tmp1, tmp1, #3/* Bytes beyond alignment -> bits.*/
113− neg tmp1, tmp1/* Bits to alignment -64. */
114− mov tmp2, #~0
115− /*mask off the non-intended bytes before the start address.*/
116−CPU_BE( lsl tmp2, tmp2, tmp1 )/*Big-endian.Early bytes are at MSB*/
117− /* Little-endian. Early bytes are at LSB. */
118−CPU_LE( lsr tmp2, tmp2, tmp1 )
119−
120− orr data1, data1, tmp2
121− orr data2, data2, tmp2
122− b .Lstart_realigned
123−
124− /*src1 and src2 have different alignment offset.*/
125−.Lmisaligned8:
126− cmp limit, #8
127− b.lo .Ltiny8proc /*limit < 8: compare byte by byte*/
128−
129− and tmp1, src1, #7
130− neg tmp1, tmp1
131− add tmp1, tmp1, #8/*valid length in the first 8 bytes of src1*/
132− and tmp2, src2, #7
133− neg tmp2, tmp2
134− add tmp2, tmp2, #8/*valid length in the first 8 bytes of src2*/
135− subs tmp3, tmp1, tmp2
136− csel pos, tmp1, tmp2, hi /*Choose the maximum.*/
137−
138− sub limit, limit, pos
139− /*compare the proceeding bytes in the first 8 byte segment.*/
140−.Ltinycmp:
141− ldrb data1w, [src1], #1
142− ldrb data2w, [src2], #1
143− subs pos, pos, #1
144− ccmp data1w, data2w, #0, ne /* NZCV = 0b0000. */
145− b.eq .Ltinycmp
146− cbnz pos, 1f /*diff occurred before the last byte.*/
147− cmp data1w, data2w
148− b.eq .Lstart_align
149−1:
150− sub result, data1, data2
36+ subs limit, limit, 8
37+ b.lo L(less8)
38+
39+ ldr data1, [src1], 8
40+ ldr data2, [src2], 8
41+ cmp data1, data2
42+ b.ne L(return)
43+
44+ subs limit, limit, 8
45+ b.gt L(more16)
46+
47+ ldr data1, [src1, limit]
48+ ldr data2, [src2, limit]
49+ b L(return)
50+
51+L(more16):
52+ ldr data1, [src1], 8
53+ ldr data2, [src2], 8
54+ cmp data1, data2
55+ bne L(return)
56+
57+ /* Jump directly to comparing the last 16 bytes for 32 byte (or less)
58+ strings. */
59+ subs limit, limit, 16
60+ b.ls L(last_bytes)
61+
62+ /* We overlap loads between 0-32 bytes at either side of SRC1 when we
63+ try to align, so limit it only to strings larger than 128 bytes. */
64+ cmp limit, 96
65+ b.ls L(loop16)
66+
67+ /* Align src1 and adjust src2 with bytes not yet done. */
68+ and tmp1, src1, 15
69+ add limit, limit, tmp1
70+ sub src1, src1, tmp1
71+ sub src2, src2, tmp1
72+
73+ /* Loop performing 16 bytes per iteration using aligned src1.
74+ Limit is pre-decremented by 16 and must be larger than zero.
75+ Exit if <= 16 bytes left to do or if the data is not equal. */
76+ .p2align 4
77+L(loop16):
78+ ldp data1, data1h, [src1], 16
79+ ldp data2, data2h, [src2], 16
80+ subs limit, limit, 16
81+ ccmp data1, data2, 0, hi
82+ ccmp data1h, data2h, 0, eq
83+ b.eq L(loop16)
84+
85+ cmp data1, data2
86+ bne L(return)
87+ mov data1, data1h
88+ mov data2, data2h
89+ cmp data1, data2
90+ bne L(return)
91+
92+ /* Compare last 1-16 bytes using unaligned access. */
93+L(last_bytes):
94+ add src1, src1, limit
95+ add src2, src2, limit
96+ ldp data1, data1h, [src1]
97+ ldp data2, data2h, [src2]
98+ cmp data1, data2
99+ bne L(return)
100+ mov data1, data1h
101+ mov data2, data2h
102+ cmp data1, data2
103+
104+ /* Compare data bytes and set return value to 0, -1 or 1. */
105+L(return):
106+#ifndef __AARCH64EB__
107+ rev data1, data1
108+ rev data2, data2
109+#endif
110+ cmp data1, data2
111+L(ret_eq):
112+ cset result, ne
113+ cneg result, result, lo
151114 ret
152115
153−.Lstart_align:
154− lsr limit_wd, limit, #3
155− cbz limit_wd, .Lremain8
156−
157− ands xzr, src1, #7
158− b.eq .Lrecal_offset
159− /*process more leading bytes to make src1 aligned...*/
160− add src1, src1, tmp3 /*backwards src1 to alignment boundary*/
161− add src2, src2, tmp3
162− sub limit, limit, tmp3
163− lsr limit_wd, limit, #3
164− cbz limit_wd, .Lremain8
165− /*load 8 bytes from aligned SRC1..*/
166− ldr data1, [src1], #8
167− ldr data2, [src2], #8
168−
169− subs limit_wd, limit_wd, #1
170− eor diff, data1, data2 /*Non-zero if differences found.*/
171− csinv endloop, diff, xzr, ne
172− cbnz endloop, .Lunequal_proc
173− /*How far is the current SRC2 from the alignment boundary...*/
174− and tmp3, tmp3, #7
175−
176−.Lrecal_offset:/*src1 is aligned now..*/
177− neg pos, tmp3
178−.Lloopcmp_proc:
179− /*
180− * Divide the eight bytes into two parts. First,backwards the src2
181− * to an alignment boundary,load eight bytes and compare from
182− * the SRC2 alignment boundary. If all 8 bytes are equal,then start
183− * the second part's comparison. Otherwise finish the comparison.
184− * This special handle can garantee all the accesses are in the
185− * thread/task space in avoid to overrange access.
186− */
187− ldr data1, [src1,pos]
188− ldr data2, [src2,pos]
189− eor diff, data1, data2 /* Non-zero if differences found. */
190− cbnz diff, .Lnot_limit
191−
192− /*The second part process*/
193− ldr data1, [src1], #8
194− ldr data2, [src2], #8
195− eor diff, data1, data2 /* Non-zero if differences found. */
196− subs limit_wd, limit_wd, #1
197− csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/
198− cbz endloop, .Lloopcmp_proc
199−.Lunequal_proc:
200− cbz diff, .Lremain8
201−
202−/* There is difference occurred in the latest comparison. */
203−.Lnot_limit:
204−/*
205−* For little endian,reverse the low significant equal bits into MSB,then
206−* following CLZ can find how many equal bits exist.
207−*/
208−CPU_LE( rev diff, diff )
209−CPU_LE( rev data1, data1 )
210−CPU_LE( rev data2, data2 )
211−
212− /*
213− * The MS-non-zero bit of DIFF marks either the first bit
214− * that is different, or the end of the significant data.
215− * Shifting left now will bring the critical information into the
216− * top bits.
217− */
218− clz pos, diff
219− lsl data1, data1, pos
220− lsl data2, data2, pos
221− /*
222− * We need to zero-extend (char is unsigned) the value and then
223− * perform a signed subtraction.
224− */
225− lsr data1, data1, #56
226− sub result, data1, data2, lsr #56
116+ .p2align 4
117+ /* Compare up to 8 bytes. Limit is [-8..-1]. */
118+L(less8):
119+ adds limit, limit, 4
120+ b.lo L(less4)
121+ ldr data1w, [src1], 4
122+ ldr data2w, [src2], 4
123+ cmp data1w, data2w
124+ b.ne L(return)
125+ sub limit, limit, 4
126+L(less4):
127+ adds limit, limit, 4
128+ beq L(ret_eq)
129+L(byte_loop):
130+ ldrb data1w, [src1], 1
131+ ldrb data2w, [src2], 1
132+ subs limit, limit, 1
133+ ccmp data1w, data2w, 0, ne /* NZCV = 0b0000. */
134+ b.eq L(byte_loop)
135+ sub result, data1w, data2w
227136 ret
228137
229−.Lremain8:
230− /* Limit % 8 == 0 =>. all data are equal.*/
231− ands limit, limit, #7
232− b.eq .Lret0
233−
234−.Ltiny8proc:
235− ldrb data1w, [src1], #1
236− ldrb data2w, [src2], #1
237− subs limit, limit, #1
238−
239− ccmp data1w, data2w, #0, ne /* NZCV = 0b0000. */
240− b.eq .Ltiny8proc
241− sub result, data1, data2
242− ret
243−.Lret0:
244− mov result, #0
245− ret
246138 SYM_FUNC_END_PI(memcmp)
247139 EXPORT_SYMBOL_NOKASAN(memcmp)
arch/arm64/lib/memcpy.S+229−43
@@ -1,66 +1,252 @@
11 /* SPDX-License-Identifier: GPL-2.0-only */
22 /*
3− * Copyright (C) 2013 ARM Ltd.
4− * Copyright (C) 2013 Linaro.
3+ * Copyright (c) 2012-2021, Arm Limited.
54 *
6− * This code is based on glibc cortex strings work originally authored by Linaro
7− * be found @
8− *
9− * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
10− * files/head:/src/aarch64/
5+ * Adapted from the original at:
6+ * https://github.com/ARM-software/optimized-routines/blob/afd6244a1f8d9229/string/aarch64/memcpy.S
117 */
128
139 #include <linux/linkage.h>
1410 #include <asm/assembler.h>
15−#include <asm/cache.h>
1611
17−/*
18− * Copy a buffer from src to dest (alignment handled by the hardware)
12+/* Assumptions:
13+ *
14+ * ARMv8-a, AArch64, unaligned accesses.
1915 *
20− * Parameters:
21− * x0 - dest
22− * x1 - src
23− * x2 - n
24− * Returns:
25− * x0 - dest
2616 */
27− .macro ldrb1 reg, ptr, val
28− ldrb \reg, [\ptr], \val
29− .endm
30−
31− .macro strb1 reg, ptr, val
32− strb \reg, [\ptr], \val
33− .endm
3417
35− .macro ldrh1 reg, ptr, val
36− ldrh \reg, [\ptr], \val
37− .endm
18+#define L(label) .L ## label
3819
39− .macro strh1 reg, ptr, val
40− strh \reg, [\ptr], \val
41− .endm
20+#define dstin x0
21+#define src x1
22+#define count x2
23+#define dst x3
24+#define srcend x4
25+#define dstend x5
26+#define A_l x6
27+#define A_lw w6
28+#define A_h x7
29+#define B_l x8
30+#define B_lw w8
31+#define B_h x9
32+#define C_l x10
33+#define C_lw w10
34+#define C_h x11
35+#define D_l x12
36+#define D_h x13
37+#define E_l x14
38+#define E_h x15
39+#define F_l x16
40+#define F_h x17
41+#define G_l count
42+#define G_h dst
43+#define H_l src
44+#define H_h srcend
45+#define tmp1 x14
4246
43− .macro ldr1 reg, ptr, val
44− ldr \reg, [\ptr], \val
45− .endm
47+/* This implementation handles overlaps and supports both memcpy and memmove
48+ from a single entry point. It uses unaligned accesses and branchless
49+ sequences to keep the code small, simple and improve performance.
4650
47− .macro str1 reg, ptr, val
48− str \reg, [\ptr], \val
49− .endm
51+ Copies are split into 3 main cases: small copies of up to 32 bytes, medium
52+ copies of up to 128 bytes, and large copies. The overhead of the overlap
53+ check is negligible since it is only required for large copies.
5054
51− .macro ldp1 reg1, reg2, ptr, val
52− ldp \reg1, \reg2, [\ptr], \val
53− .endm
54−
55− .macro stp1 reg1, reg2, ptr, val
56− stp \reg1, \reg2, [\ptr], \val
57− .endm
55+ Large copies use a software pipelined loop processing 64 bytes per iteration.
56+ The destination pointer is 16-byte aligned to minimize unaligned accesses.
57+ The loop tail is handled by always copying 64 bytes from the end.
58+*/
5859
60+SYM_FUNC_START_ALIAS(__memmove)
61+SYM_FUNC_START_WEAK_ALIAS_PI(memmove)
5962 SYM_FUNC_START_ALIAS(__memcpy)
6063 SYM_FUNC_START_WEAK_PI(memcpy)
61−#include "copy_template.S"
64+ add srcend, src, count
65+ add dstend, dstin, count
66+ cmp count, 128
67+ b.hi L(copy_long)
68+ cmp count, 32
69+ b.hi L(copy32_128)
70+
71+ /* Small copies: 0..32 bytes. */
72+ cmp count, 16
73+ b.lo L(copy16)
74+ ldp A_l, A_h, [src]
75+ ldp D_l, D_h, [srcend, -16]
76+ stp A_l, A_h, [dstin]
77+ stp D_l, D_h, [dstend, -16]
78+ ret
79+
80+ /* Copy 8-15 bytes. */
81+L(copy16):
82+ tbz count, 3, L(copy8)
83+ ldr A_l, [src]
84+ ldr A_h, [srcend, -8]
85+ str A_l, [dstin]
86+ str A_h, [dstend, -8]
87+ ret
88+
89+ .p2align 3
90+ /* Copy 4-7 bytes. */
91+L(copy8):
92+ tbz count, 2, L(copy4)
93+ ldr A_lw, [src]
94+ ldr B_lw, [srcend, -4]
95+ str A_lw, [dstin]
96+ str B_lw, [dstend, -4]
97+ ret
98+
99+ /* Copy 0..3 bytes using a branchless sequence. */
100+L(copy4):
101+ cbz count, L(copy0)
102+ lsr tmp1, count, 1
103+ ldrb A_lw, [src]
104+ ldrb C_lw, [srcend, -1]
105+ ldrb B_lw, [src, tmp1]
106+ strb A_lw, [dstin]
107+ strb B_lw, [dstin, tmp1]
108+ strb C_lw, [dstend, -1]
109+L(copy0):
110+ ret
111+
112+ .p2align 4
113+ /* Medium copies: 33..128 bytes. */
114+L(copy32_128):
115+ ldp A_l, A_h, [src]
116+ ldp B_l, B_h, [src, 16]
117+ ldp C_l, C_h, [srcend, -32]
118+ ldp D_l, D_h, [srcend, -16]
119+ cmp count, 64
120+ b.hi L(copy128)
121+ stp A_l, A_h, [dstin]
122+ stp B_l, B_h, [dstin, 16]
123+ stp C_l, C_h, [dstend, -32]
124+ stp D_l, D_h, [dstend, -16]
62125 ret
126+
127+ .p2align 4
128+ /* Copy 65..128 bytes. */
129+L(copy128):
130+ ldp E_l, E_h, [src, 32]
131+ ldp F_l, F_h, [src, 48]
132+ cmp count, 96
133+ b.ls L(copy96)
134+ ldp G_l, G_h, [srcend, -64]
135+ ldp H_l, H_h, [srcend, -48]
136+ stp G_l, G_h, [dstend, -64]
137+ stp H_l, H_h, [dstend, -48]
138+L(copy96):
139+ stp A_l, A_h, [dstin]
140+ stp B_l, B_h, [dstin, 16]
141+ stp E_l, E_h, [dstin, 32]
142+ stp F_l, F_h, [dstin, 48]
143+ stp C_l, C_h, [dstend, -32]
144+ stp D_l, D_h, [dstend, -16]
145+ ret
146+
147+ .p2align 4
148+ /* Copy more than 128 bytes. */
149+L(copy_long):
150+ /* Use backwards copy if there is an overlap. */
151+ sub tmp1, dstin, src
152+ cbz tmp1, L(copy0)
153+ cmp tmp1, count
154+ b.lo L(copy_long_backwards)
155+
156+ /* Copy 16 bytes and then align dst to 16-byte alignment. */
157+
158+ ldp D_l, D_h, [src]
159+ and tmp1, dstin, 15
160+ bic dst, dstin, 15
161+ sub src, src, tmp1
162+ add count, count, tmp1 /* Count is now 16 too large. */
163+ ldp A_l, A_h, [src, 16]
164+ stp D_l, D_h, [dstin]
165+ ldp B_l, B_h, [src, 32]
166+ ldp C_l, C_h, [src, 48]
167+ ldp D_l, D_h, [src, 64]!
168+ subs count, count, 128 + 16 /* Test and readjust count. */
169+ b.ls L(copy64_from_end)
170+
171+L(loop64):
172+ stp A_l, A_h, [dst, 16]
173+ ldp A_l, A_h, [src, 16]
174+ stp B_l, B_h, [dst, 32]
175+ ldp B_l, B_h, [src, 32]
176+ stp C_l, C_h, [dst, 48]
177+ ldp C_l, C_h, [src, 48]
178+ stp D_l, D_h, [dst, 64]!
179+ ldp D_l, D_h, [src, 64]!
180+ subs count, count, 64
181+ b.hi L(loop64)
182+
183+ /* Write the last iteration and copy 64 bytes from the end. */
184+L(copy64_from_end):
185+ ldp E_l, E_h, [srcend, -64]
186+ stp A_l, A_h, [dst, 16]
187+ ldp A_l, A_h, [srcend, -48]
188+ stp B_l, B_h, [dst, 32]
189+ ldp B_l, B_h, [srcend, -32]
190+ stp C_l, C_h, [dst, 48]
191+ ldp C_l, C_h, [srcend, -16]
192+ stp D_l, D_h, [dst, 64]
193+ stp E_l, E_h, [dstend, -64]
194+ stp A_l, A_h, [dstend, -48]
195+ stp B_l, B_h, [dstend, -32]
196+ stp C_l, C_h, [dstend, -16]
197+ ret
198+
199+ .p2align 4
200+
201+ /* Large backwards copy for overlapping copies.
202+ Copy 16 bytes and then align dst to 16-byte alignment. */
203+L(copy_long_backwards):
204+ ldp D_l, D_h, [srcend, -16]
205+ and tmp1, dstend, 15
206+ sub srcend, srcend, tmp1
207+ sub count, count, tmp1
208+ ldp A_l, A_h, [srcend, -16]
209+ stp D_l, D_h, [dstend, -16]
210+ ldp B_l, B_h, [srcend, -32]
211+ ldp C_l, C_h, [srcend, -48]
212+ ldp D_l, D_h, [srcend, -64]!
213+ sub dstend, dstend, tmp1
214+ subs count, count, 128
215+ b.ls L(copy64_from_start)
216+
217+L(loop64_backwards):
218+ stp A_l, A_h, [dstend, -16]
219+ ldp A_l, A_h, [srcend, -16]
220+ stp B_l, B_h, [dstend, -32]
221+ ldp B_l, B_h, [srcend, -32]
222+ stp C_l, C_h, [dstend, -48]
223+ ldp C_l, C_h, [srcend, -48]
224+ stp D_l, D_h, [dstend, -64]!
225+ ldp D_l, D_h, [srcend, -64]!
226+ subs count, count, 64
227+ b.hi L(loop64_backwards)
228+
229+ /* Write the last iteration and copy 64 bytes from the start. */
230+L(copy64_from_start):
231+ ldp G_l, G_h, [src, 48]
232+ stp A_l, A_h, [dstend, -16]
233+ ldp A_l, A_h, [src, 32]
234+ stp B_l, B_h, [dstend, -32]
235+ ldp B_l, B_h, [src, 16]
236+ stp C_l, C_h, [dstend, -48]
237+ ldp C_l, C_h, [src]
238+ stp D_l, D_h, [dstend, -64]
239+ stp G_l, G_h, [dstin, 48]
240+ stp A_l, A_h, [dstin, 32]
241+ stp B_l, B_h, [dstin, 16]
242+ stp C_l, C_h, [dstin]
243+ ret
244+
63245 SYM_FUNC_END_PI(memcpy)
64246 EXPORT_SYMBOL(memcpy)
65247 SYM_FUNC_END_ALIAS(__memcpy)
66248 EXPORT_SYMBOL(__memcpy)
249+SYM_FUNC_END_ALIAS_PI(memmove)
250+EXPORT_SYMBOL(memmove)
251+SYM_FUNC_END_ALIAS(__memmove)
252+EXPORT_SYMBOL(__memmove)
arch/arm64/lib/memmove.S+0−189
@@ -1,189 +0,0 @@
1−/* SPDX-License-Identifier: GPL-2.0-only */
2−/*
3− * Copyright (C) 2013 ARM Ltd.
4− * Copyright (C) 2013 Linaro.
5− *
6− * This code is based on glibc cortex strings work originally authored by Linaro
7− * be found @
8− *
9− * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
10− * files/head:/src/aarch64/
11− */
12−
13−#include <linux/linkage.h>
14−#include <asm/assembler.h>
15−#include <asm/cache.h>
16−
17−/*
18− * Move a buffer from src to test (alignment handled by the hardware).
19− * If dest <= src, call memcpy, otherwise copy in reverse order.
20− *
21− * Parameters:
22− * x0 - dest
23− * x1 - src
24− * x2 - n
25− * Returns:
26− * x0 - dest
27− */
28−dstin .req x0
29−src .req x1
30−count .req x2
31−tmp1 .req x3
32−tmp1w .req w3
33−tmp2 .req x4
34−tmp2w .req w4
35−tmp3 .req x5
36−tmp3w .req w5
37−dst .req x6
38−
39−A_l .req x7
40−A_h .req x8
41−B_l .req x9
42−B_h .req x10
43−C_l .req x11
44−C_h .req x12
45−D_l .req x13
46−D_h .req x14
47−
48−SYM_FUNC_START_ALIAS(__memmove)
49−SYM_FUNC_START_WEAK_PI(memmove)
50− cmp dstin, src
51− b.lo __memcpy
52− add tmp1, src, count
53− cmp dstin, tmp1
54− b.hs __memcpy /* No overlap. */
55−
56− add dst, dstin, count
57− add src, src, count
58− cmp count, #16
59− b.lo .Ltail15 /*probably non-alignment accesses.*/
60−
61− ands tmp2, src, #15 /* Bytes to reach alignment. */
62− b.eq .LSrcAligned
63− sub count, count, tmp2
64− /*
65− * process the aligned offset length to make the src aligned firstly.
66− * those extra instructions' cost is acceptable. It also make the
67− * coming accesses are based on aligned address.
68− */
69− tbz tmp2, #0, 1f
70− ldrb tmp1w, [src, #-1]!
71− strb tmp1w, [dst, #-1]!
72−1:
73− tbz tmp2, #1, 2f
74− ldrh tmp1w, [src, #-2]!
75− strh tmp1w, [dst, #-2]!
76−2:
77− tbz tmp2, #2, 3f
78− ldr tmp1w, [src, #-4]!
79− str tmp1w, [dst, #-4]!
80−3:
81− tbz tmp2, #3, .LSrcAligned
82− ldr tmp1, [src, #-8]!
83− str tmp1, [dst, #-8]!
84−
85−.LSrcAligned:
86− cmp count, #64
87− b.ge .Lcpy_over64
88−
89− /*
90− * Deal with small copies quickly by dropping straight into the
91− * exit block.
92− */
93−.Ltail63:
94− /*
95− * Copy up to 48 bytes of data. At this point we only need the
96− * bottom 6 bits of count to be accurate.
97− */
98− ands tmp1, count, #0x30
99− b.eq .Ltail15
100− cmp tmp1w, #0x20
101− b.eq 1f
102− b.lt 2f
103− ldp A_l, A_h, [src, #-16]!
104− stp A_l, A_h, [dst, #-16]!
105−1:
106− ldp A_l, A_h, [src, #-16]!
107− stp A_l, A_h, [dst, #-16]!
108−2:
109− ldp A_l, A_h, [src, #-16]!
110− stp A_l, A_h, [dst, #-16]!
111−
112−.Ltail15:
113− tbz count, #3, 1f
114− ldr tmp1, [src, #-8]!
115− str tmp1, [dst, #-8]!
116−1:
117− tbz count, #2, 2f
118− ldr tmp1w, [src, #-4]!
119− str tmp1w, [dst, #-4]!
120−2:
121− tbz count, #1, 3f
122− ldrh tmp1w, [src, #-2]!
123− strh tmp1w, [dst, #-2]!
124−3:
125− tbz count, #0, .Lexitfunc
126− ldrb tmp1w, [src, #-1]
127− strb tmp1w, [dst, #-1]
128−
129−.Lexitfunc:
130− ret
131−
132−.Lcpy_over64:
133− subs count, count, #128
134− b.ge .Lcpy_body_large
135− /*
136− * Less than 128 bytes to copy, so handle 64 bytes here and then jump
137− * to the tail.
138− */
139− ldp A_l, A_h, [src, #-16]
140− stp A_l, A_h, [dst, #-16]
141− ldp B_l, B_h, [src, #-32]
142− ldp C_l, C_h, [src, #-48]
143− stp B_l, B_h, [dst, #-32]
144− stp C_l, C_h, [dst, #-48]
145− ldp D_l, D_h, [src, #-64]!
146− stp D_l, D_h, [dst, #-64]!
147−
148− tst count, #0x3f
149− b.ne .Ltail63
150− ret
151−
152− /*
153− * Critical loop. Start at a new cache line boundary. Assuming
154− * 64 bytes per line this ensures the entire loop is in one line.
155− */
156− .p2align L1_CACHE_SHIFT
157−.Lcpy_body_large:
158− /* pre-load 64 bytes data. */
159− ldp A_l, A_h, [src, #-16]
160− ldp B_l, B_h, [src, #-32]
161− ldp C_l, C_h, [src, #-48]
162− ldp D_l, D_h, [src, #-64]!
163−1:
164− /*
165− * interlace the load of next 64 bytes data block with store of the last
166− * loaded 64 bytes data.
167− */
168− stp A_l, A_h, [dst, #-16]
169− ldp A_l, A_h, [src, #-16]
170− stp B_l, B_h, [dst, #-32]
171− ldp B_l, B_h, [src, #-32]
172− stp C_l, C_h, [dst, #-48]
173− ldp C_l, C_h, [src, #-48]
174− stp D_l, D_h, [dst, #-64]!
175− ldp D_l, D_h, [src, #-64]!
176− subs count, count, #64
177− b.ge 1b
178− stp A_l, A_h, [dst, #-16]
179− stp B_l, B_h, [dst, #-32]
180− stp C_l, C_h, [dst, #-48]
181− stp D_l, D_h, [dst, #-64]!
182−
183− tst count, #0x3f
184− b.ne .Ltail63
185− ret
186−SYM_FUNC_END_PI(memmove)
187−EXPORT_SYMBOL(memmove)
188−SYM_FUNC_END_ALIAS(__memmove)
189−EXPORT_SYMBOL(__memmove)
arch/arm64/lib/mte.S+20−0
@@ -36,6 +36,26 @@ SYM_FUNC_START(mte_clear_page_tags) · 35 unmodified lines
3636 ret
3737 SYM_FUNC_END(mte_clear_page_tags)
3838
39+/*
40+ * Zero the page and tags at the same time
41+ *
42+ * Parameters:
43+ * x0 - address to the beginning of the page
44+ */
45+SYM_FUNC_START(mte_zero_clear_page_tags)
46+ mrs x1, dczid_el0
47+ and w1, w1, #0xf
48+ mov x2, #4
49+ lsl x1, x2, x1
50+ and x0, x0, #(1 << MTE_TAG_SHIFT) - 1 // clear the tag
51+
52+1: dc gzva, x0
53+ add x0, x0, x1
54+ tst x0, #(PAGE_SIZE - 1)
55+ b.ne 1b
56+ ret
57+SYM_FUNC_END(mte_zero_clear_page_tags)
58+
3959 /*
4060 * Copy the tags from the source page to the destination one
4161 * x0 - address of the destination page
arch/arm64/lib/strcmp.S+121−168
@@ -1,84 +1,123 @@
11 /* SPDX-License-Identifier: GPL-2.0-only */
22 /*
3− * Copyright (C) 2013 ARM Ltd.
4− * Copyright (C) 2013 Linaro.
3+ * Copyright (c) 2012-2021, Arm Limited.
54 *
6− * This code is based on glibc cortex strings work originally authored by Linaro
7− * be found @
8− *
9− * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
10− * files/head:/src/aarch64/
5+ * Adapted from the original at:
6+ * https://github.com/ARM-software/optimized-routines/blob/afd6244a1f8d9229/string/aarch64/strcmp.S
117 */
128
139 #include <linux/linkage.h>
1410 #include <asm/assembler.h>
1511
16−/*
17− * compare two strings
12+/* Assumptions:
1813 *
19− * Parameters:
20− * x0 - const string 1 pointer
21− * x1 - const string 2 pointer
22− * Returns:
23− * x0 - an integer less than, equal to, or greater than zero
24− * if s1 is found, respectively, to be less than, to match,
25− * or be greater than s2.
14+ * ARMv8-a, AArch64
2615 */
2716
17+#define L(label) .L ## label
18+
2819 #define REP8_01 0x0101010101010101
2920 #define REP8_7f 0x7f7f7f7f7f7f7f7f
3021 #define REP8_80 0x8080808080808080
3122
3223 /* Parameters and result. */
33−src1 .req x0
34−src2 .req x1
35−result .req x0
24+#define src1 x0
25+#define src2 x1
26+#define result x0
3627
3728 /* Internal variables. */
38−data1 .req x2
39−data1w .req w2
40−data2 .req x3
41−data2w .req w3
42−has_nul .req x4
43−diff .req x5
44−syndrome .req x6
45−tmp1 .req x7
46−tmp2 .req x8
47−tmp3 .req x9
48−zeroones .req x10
49−pos .req x11
50−
29+#define data1 x2
30+#define data1w w2
31+#define data2 x3
32+#define data2w w3
33+#define has_nul x4
34+#define diff x5
35+#define syndrome x6
36+#define tmp1 x7
37+#define tmp2 x8
38+#define tmp3 x9
39+#define zeroones x10
40+#define pos x11
41+
42+ /* Start of performance-critical section -- one 64B cache line. */
43+ .align 6
5144 SYM_FUNC_START_WEAK_PI(strcmp)
5245 eor tmp1, src1, src2
5346 mov zeroones, #REP8_01
5447 tst tmp1, #7
55− b.ne .Lmisaligned8
48+ b.ne L(misaligned8)
5649 ands tmp1, src1, #7
57− b.ne .Lmutual_align
58−
59− /*
60− * NUL detection works on the principle that (X - 1) & (~X) & 0x80
61− * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
62− * can be done in parallel across the entire word.
63− */
64−.Lloop_aligned:
50+ b.ne L(mutual_align)
51+ /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
52+ (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
53+ can be done in parallel across the entire word. */
54+L(loop_aligned):
6555 ldr data1, [src1], #8
6656 ldr data2, [src2], #8
67−.Lstart_realigned:
57+L(start_realigned):
6858 sub tmp1, data1, zeroones
6959 orr tmp2, data1, #REP8_7f
7060 eor diff, data1, data2 /* Non-zero if differences found. */
7161 bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
7262 orr syndrome, diff, has_nul
73− cbz syndrome, .Lloop_aligned
74− b .Lcal_cmpresult
63+ cbz syndrome, L(loop_aligned)
64+ /* End of performance-critical section -- one 64B cache line. */
65+
66+L(end):
67+#ifndef __AARCH64EB__
68+ rev syndrome, syndrome
69+ rev data1, data1
70+ /* The MS-non-zero bit of the syndrome marks either the first bit
71+ that is different, or the top bit of the first zero byte.
72+ Shifting left now will bring the critical information into the
73+ top bits. */
74+ clz pos, syndrome
75+ rev data2, data2
76+ lsl data1, data1, pos
77+ lsl data2, data2, pos
78+ /* But we need to zero-extend (char is unsigned) the value and then
79+ perform a signed 32-bit subtraction. */
80+ lsr data1, data1, #56
81+ sub result, data1, data2, lsr #56
82+ ret
83+#else
84+ /* For big-endian we cannot use the trick with the syndrome value
85+ as carry-propagation can corrupt the upper bits if the trailing
86+ bytes in the string contain 0x01. */
87+ /* However, if there is no NUL byte in the dword, we can generate
88+ the result directly. We can't just subtract the bytes as the
89+ MSB might be significant. */
90+ cbnz has_nul, 1f
91+ cmp data1, data2
92+ cset result, ne
93+ cneg result, result, lo
94+ ret
95+1:
96+ /* Re-compute the NUL-byte detection, using a byte-reversed value. */
97+ rev tmp3, data1
98+ sub tmp1, tmp3, zeroones
99+ orr tmp2, tmp3, #REP8_7f
100+ bic has_nul, tmp1, tmp2
101+ rev has_nul, has_nul
102+ orr syndrome, diff, has_nul
103+ clz pos, syndrome
104+ /* The MS-non-zero bit of the syndrome marks either the first bit
105+ that is different, or the top bit of the first zero byte.
106+ Shifting left now will bring the critical information into the
107+ top bits. */
108+ lsl data1, data1, pos
109+ lsl data2, data2, pos
110+ /* But we need to zero-extend (char is unsigned) the value and then
111+ perform a signed 32-bit subtraction. */
112+ lsr data1, data1, #56
113+ sub result, data1, data2, lsr #56
114+ ret
115+#endif
75116
76−.Lmutual_align:
77− /*
78− * Sources are mutually aligned, but are not currently at an
79− * alignment boundary. Round down the addresses and then mask off
80− * the bytes that preceed the start point.
81− */
117+L(mutual_align):
118+ /* Sources are mutually aligned, but are not currently at an
119+ alignment boundary. Round down the addresses and then mask off
120+ the bytes that preceed the start point. */
82121 bic src1, src1, #7
83122 bic src2, src2, #7
84123 lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */
@@ -86,138 +125,52 @@ SYM_FUNC_START_WEAK_PI(strcmp) · 1 unmodified line
86125 neg tmp1, tmp1 /* Bits to alignment -64. */
87126 ldr data2, [src2], #8
88127 mov tmp2, #~0
128+#ifdef __AARCH64EB__
89129 /* Big-endian. Early bytes are at MSB. */
90−CPU_BE( lsl tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */
130+ lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
131+#else
91132 /* Little-endian. Early bytes are at LSB. */
92−CPU_LE( lsr tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */
93−
133+ lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */
134+#endif
94135 orr data1, data1, tmp2
95136 orr data2, data2, tmp2
96− b .Lstart_realigned
97−
98−.Lmisaligned8:
99− /*
100− * Get the align offset length to compare per byte first.
101− * After this process, one string's address will be aligned.
102− */
103− and tmp1, src1, #7
104− neg tmp1, tmp1
105− add tmp1, tmp1, #8
106− and tmp2, src2, #7
107− neg tmp2, tmp2
108− add tmp2, tmp2, #8
109− subs tmp3, tmp1, tmp2
110− csel pos, tmp1, tmp2, hi /*Choose the maximum. */
111−.Ltinycmp:
137+ b L(start_realigned)
138+
139+L(misaligned8):
140+ /* Align SRC1 to 8 bytes and then compare 8 bytes at a time, always
141+ checking to make sure that we don't access beyond page boundary in
142+ SRC2. */
143+ tst src1, #7
144+ b.eq L(loop_misaligned)
145+L(do_misaligned):
112146 ldrb data1w, [src1], #1
113147 ldrb data2w, [src2], #1
114− subs pos, pos, #1
115− ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */
116− ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */
117− b.eq .Ltinycmp
118− cbnz pos, 1f /*find the null or unequal...*/
119148 cmp data1w, #1
120− ccmp data1w, data2w, #0, cs
121− b.eq .Lstart_align /*the last bytes are equal....*/
122−1:
123− sub result, data1, data2
124− ret
125−
126−.Lstart_align:
127− ands xzr, src1, #7
128− b.eq .Lrecal_offset
129− /*process more leading bytes to make str1 aligned...*/
130− add src1, src1, tmp3
131− add src2, src2, tmp3
132− /*load 8 bytes from aligned str1 and non-aligned str2..*/
149+ ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */
150+ b.ne L(done)
151+ tst src1, #7
152+ b.ne L(do_misaligned)
153+
154+L(loop_misaligned):
155+ /* Test if we are within the last dword of the end of a 4K page. If
156+ yes then jump back to the misaligned loop to copy a byte at a time. */
157+ and tmp1, src2, #0xff8
158+ eor tmp1, tmp1, #0xff8
159+ cbz tmp1, L(do_misaligned)
133160 ldr data1, [src1], #8
134161 ldr data2, [src2], #8
135162
136163 sub tmp1, data1, zeroones
137164 orr tmp2, data1, #REP8_7f
138− bic has_nul, tmp1, tmp2
139− eor diff, data1, data2 /* Non-zero if differences found. */
140− orr syndrome, diff, has_nul
141− cbnz syndrome, .Lcal_cmpresult
142− /*How far is the current str2 from the alignment boundary...*/
143− and tmp3, tmp3, #7
144−.Lrecal_offset:
145− neg pos, tmp3
146−.Lloopcmp_proc:
147− /*
148− * Divide the eight bytes into two parts. First,backwards the src2
149− * to an alignment boundary,load eight bytes from the SRC2 alignment
150− * boundary,then compare with the relative bytes from SRC1.
151− * If all 8 bytes are equal,then start the second part's comparison.
152− * Otherwise finish the comparison.
153− * This special handle can garantee all the accesses are in the
154− * thread/task space in avoid to overrange access.
155− */
156− ldr data1, [src1,pos]
157− ldr data2, [src2,pos]
158− sub tmp1, data1, zeroones
159− orr tmp2, data1, #REP8_7f
160− bic has_nul, tmp1, tmp2
161− eor diff, data1, data2 /* Non-zero if differences found. */
162− orr syndrome, diff, has_nul
163− cbnz syndrome, .Lcal_cmpresult
164−
165− /*The second part process*/
166− ldr data1, [src1], #8
167− ldr data2, [src2], #8
168− sub tmp1, data1, zeroones
169− orr tmp2, data1, #REP8_7f
170− bic has_nul, tmp1, tmp2
171− eor diff, data1, data2 /* Non-zero if differences found. */
165+ eor diff, data1, data2 /* Non-zero if differences found. */
166+ bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
172167 orr syndrome, diff, has_nul
173− cbz syndrome, .Lloopcmp_proc
168+ cbz syndrome, L(loop_misaligned)
169+ b L(end)
174170
175−.Lcal_cmpresult:
176− /*
177− * reversed the byte-order as big-endian,then CLZ can find the most
178− * significant zero bits.
179− */
180−CPU_LE( rev syndrome, syndrome )
181−CPU_LE( rev data1, data1 )
182−CPU_LE( rev data2, data2 )
183−
184− /*
185− * For big-endian we cannot use the trick with the syndrome value
186− * as carry-propagation can corrupt the upper bits if the trailing
187− * bytes in the string contain 0x01.
188− * However, if there is no NUL byte in the dword, we can generate
189− * the result directly. We cannot just subtract the bytes as the
190− * MSB might be significant.
191− */
192−CPU_BE( cbnz has_nul, 1f )
193−CPU_BE( cmp data1, data2 )
194−CPU_BE( cset result, ne )
195−CPU_BE( cneg result, result, lo )
196−CPU_BE( ret )
197−CPU_BE( 1: )
198− /*Re-compute the NUL-byte detection, using a byte-reversed value. */
199−CPU_BE( rev tmp3, data1 )
200−CPU_BE( sub tmp1, tmp3, zeroones )
201−CPU_BE( orr tmp2, tmp3, #REP8_7f )
202−CPU_BE( bic has_nul, tmp1, tmp2 )
203−CPU_BE( rev has_nul, has_nul )
204−CPU_BE( orr syndrome, diff, has_nul )
205−
206− clz pos, syndrome
207− /*
208− * The MS-non-zero bit of the syndrome marks either the first bit
209− * that is different, or the top bit of the first zero byte.
210− * Shifting left now will bring the critical information into the
211− * top bits.
212− */
213− lsl data1, data1, pos
214− lsl data2, data2, pos
215− /*
216− * But we need to zero-extend (char is unsigned) the value and then
217− * perform a signed 32-bit subtraction.
218− */
219− lsr data1, data1, #56
220− sub result, data1, data2, lsr #56
171+L(done):
172+ sub result, data1, data2
221173 ret
174+
222175 SYM_FUNC_END_PI(strcmp)
223176 EXPORT_SYMBOL_NOKASAN(strcmp)
arch/arm64/lib/strlen.S+173−85
@@ -1,115 +1,203 @@
11 /* SPDX-License-Identifier: GPL-2.0-only */
22 /*
3− * Copyright (C) 2013 ARM Ltd.
4− * Copyright (C) 2013 Linaro.
3+ * Copyright (c) 2013-2021, Arm Limited.
54 *
6− * This code is based on glibc cortex strings work originally authored by Linaro
7− * be found @
8− *
9− * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
10− * files/head:/src/aarch64/
5+ * Adapted from the original at:
6+ * https://github.com/ARM-software/optimized-routines/blob/98e4d6a5c13c8e54/string/aarch64/strlen.S
117 */
128
139 #include <linux/linkage.h>
1410 #include <asm/assembler.h>
1511
16−/*
17− * calculate the length of a string
12+/* Assumptions:
1813 *
19− * Parameters:
20− * x0 - const string pointer
21− * Returns:
22− * x0 - the return length of specific string
14+ * ARMv8-a, AArch64, unaligned accesses, min page size 4k.
2315 */
2416
17+#define L(label) .L ## label
18+
2519 /* Arguments and results. */
26−srcin .req x0
27−len .req x0
20+#define srcin x0
21+#define len x0
2822
2923 /* Locals and temporaries. */
30−src .req x1
31−data1 .req x2
32−data2 .req x3
33−data2a .req x4
34−has_nul1 .req x5
35−has_nul2 .req x6
36−tmp1 .req x7
37−tmp2 .req x8
38−tmp3 .req x9
39−tmp4 .req x10
40−zeroones .req x11
41−pos .req x12
24+#define src x1
25+#define data1 x2
26+#define data2 x3
27+#define has_nul1 x4
28+#define has_nul2 x5
29+#define tmp1 x4
30+#define tmp2 x5
31+#define tmp3 x6
32+#define tmp4 x7
33+#define zeroones x8
34+
35+ /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
36+ (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
37+ can be done in parallel across the entire word. A faster check
38+ (X - 1) & 0x80 is zero for non-NUL ASCII characters, but gives
39+ false hits for characters 129..255. */
4240
4341 #define REP8_01 0x0101010101010101
4442 #define REP8_7f 0x7f7f7f7f7f7f7f7f
4543 #define REP8_80 0x8080808080808080
4644
45+#define MIN_PAGE_SIZE 4096
46+
47+ /* Since strings are short on average, we check the first 16 bytes
48+ of the string for a NUL character. In order to do an unaligned ldp
49+ safely we have to do a page cross check first. If there is a NUL
50+ byte we calculate the length from the 2 8-byte words using
51+ conditional select to reduce branch mispredictions (it is unlikely
52+ strlen will be repeatedly called on strings with the same length).
53+
54+ If the string is longer than 16 bytes, we align src so don't need
55+ further page cross checks, and process 32 bytes per iteration
56+ using the fast NUL check. If we encounter non-ASCII characters,
57+ fallback to a second loop using the full NUL check.
58+
59+ If the page cross check fails, we read 16 bytes from an aligned
60+ address, remove any characters before the string, and continue
61+ in the main loop using aligned loads. Since strings crossing a
62+ page in the first 16 bytes are rare (probability of
63+ 16/MIN_PAGE_SIZE ~= 0.4%), this case does not need to be optimized.
64+
65+ AArch64 systems have a minimum page size of 4k. We don't bother
66+ checking for larger page sizes - the cost of setting up the correct
67+ page size is just not worth the extra gain from a small reduction in
68+ the cases taking the slow path. Note that we only care about
69+ whether the first fetch, which may be misaligned, crosses a page
70+ boundary. */
71+
4772 SYM_FUNC_START_WEAK_PI(strlen)
48− mov zeroones, #REP8_01
49− bic src, srcin, #15
50− ands tmp1, srcin, #15
51− b.ne .Lmisaligned
52− /*
53− * NUL detection works on the principle that (X - 1) & (~X) & 0x80
54− * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
55− * can be done in parallel across the entire word.
56− */
57− /*
58− * The inner loop deals with two Dwords at a time. This has a
59− * slightly higher start-up cost, but we should win quite quickly,
60− * especially on cores with a high number of issue slots per
61− * cycle, as we get much better parallelism out of the operations.
62− */
63−.Lloop:
64− ldp data1, data2, [src], #16
65−.Lrealigned:
73+ and tmp1, srcin, MIN_PAGE_SIZE - 1
74+ mov zeroones, REP8_01
75+ cmp tmp1, MIN_PAGE_SIZE - 16
76+ b.gt L(page_cross)
77+ ldp data1, data2, [srcin]
78+#ifdef __AARCH64EB__
79+ /* For big-endian, carry propagation (if the final byte in the
80+ string is 0x01) means we cannot use has_nul1/2 directly.
81+ Since we expect strings to be small and early-exit,
82+ byte-swap the data now so has_null1/2 will be correct. */
83+ rev data1, data1
84+ rev data2, data2
85+#endif
6686 sub tmp1, data1, zeroones
67− orr tmp2, data1, #REP8_7f
87+ orr tmp2, data1, REP8_7f
6888 sub tmp3, data2, zeroones
69− orr tmp4, data2, #REP8_7f
70− bic has_nul1, tmp1, tmp2
71− bics has_nul2, tmp3, tmp4
72− ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */
73− b.eq .Lloop
89+ orr tmp4, data2, REP8_7f
90+ bics has_nul1, tmp1, tmp2
91+ bic has_nul2, tmp3, tmp4
92+ ccmp has_nul2, 0, 0, eq
93+ beq L(main_loop_entry)
94+
95+ /* Enter with C = has_nul1 == 0. */
96+ csel has_nul1, has_nul1, has_nul2, cc
97+ mov len, 8
98+ rev has_nul1, has_nul1
99+ clz tmp1, has_nul1
100+ csel len, xzr, len, cc
101+ add len, len, tmp1, lsr 3
102+ ret
74103
104+ /* The inner loop processes 32 bytes per iteration and uses the fast
105+ NUL check. If we encounter non-ASCII characters, use a second
106+ loop with the accurate NUL check. */
107+ .p2align 4
108+L(main_loop_entry):
109+ bic src, srcin, 15
110+ sub src, src, 16
111+L(main_loop):
112+ ldp data1, data2, [src, 32]!
113+L(page_cross_entry):
114+ sub tmp1, data1, zeroones
115+ sub tmp3, data2, zeroones
116+ orr tmp2, tmp1, tmp3
117+ tst tmp2, zeroones, lsl 7
118+ bne 1f
119+ ldp data1, data2, [src, 16]
120+ sub tmp1, data1, zeroones
121+ sub tmp3, data2, zeroones
122+ orr tmp2, tmp1, tmp3
123+ tst tmp2, zeroones, lsl 7
124+ beq L(main_loop)
125+ add src, src, 16
126+1:
127+ /* The fast check failed, so do the slower, accurate NUL check. */
128+ orr tmp2, data1, REP8_7f
129+ orr tmp4, data2, REP8_7f
130+ bics has_nul1, tmp1, tmp2
131+ bic has_nul2, tmp3, tmp4
132+ ccmp has_nul2, 0, 0, eq
133+ beq L(nonascii_loop)
134+
135+ /* Enter with C = has_nul1 == 0. */
136+L(tail):
137+#ifdef __AARCH64EB__
138+ /* For big-endian, carry propagation (if the final byte in the
139+ string is 0x01) means we cannot use has_nul1/2 directly. The
140+ easiest way to get the correct byte is to byte-swap the data
141+ and calculate the syndrome a second time. */
142+ csel data1, data1, data2, cc
143+ rev data1, data1
144+ sub tmp1, data1, zeroones
145+ orr tmp2, data1, REP8_7f
146+ bic has_nul1, tmp1, tmp2
147+#else
148+ csel has_nul1, has_nul1, has_nul2, cc
149+#endif
75150 sub len, src, srcin
76− cbz has_nul1, .Lnul_in_data2
77−CPU_BE( mov data2, data1 ) /*prepare data to re-calculate the syndrome*/
78− sub len, len, #8
79− mov has_nul2, has_nul1
80−.Lnul_in_data2:
81− /*
82− * For big-endian, carry propagation (if the final byte in the
83− * string is 0x01) means we cannot use has_nul directly. The
84− * easiest way to get the correct byte is to byte-swap the data
85− * and calculate the syndrome a second time.
86− */
87−CPU_BE( rev data2, data2 )
88−CPU_BE( sub tmp1, data2, zeroones )
89−CPU_BE( orr tmp2, data2, #REP8_7f )
90−CPU_BE( bic has_nul2, tmp1, tmp2 )
91−
92− sub len, len, #8
93− rev has_nul2, has_nul2
94− clz pos, has_nul2
95− add len, len, pos, lsr #3 /* Bits to bytes. */
151+ rev has_nul1, has_nul1
152+ add tmp2, len, 8
153+ clz tmp1, has_nul1
154+ csel len, len, tmp2, cc
155+ add len, len, tmp1, lsr 3
96156 ret
97157
98−.Lmisaligned:
99− cmp tmp1, #8
100− neg tmp1, tmp1
101− ldp data1, data2, [src], #16
102− lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */
103− mov tmp2, #~0
104− /* Big-endian. Early bytes are at MSB. */
105−CPU_BE( lsl tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */
158+L(nonascii_loop):
159+ ldp data1, data2, [src, 16]!
160+ sub tmp1, data1, zeroones
161+ orr tmp2, data1, REP8_7f
162+ sub tmp3, data2, zeroones
163+ orr tmp4, data2, REP8_7f
164+ bics has_nul1, tmp1, tmp2
165+ bic has_nul2, tmp3, tmp4
166+ ccmp has_nul2, 0, 0, eq
167+ bne L(tail)
168+ ldp data1, data2, [src, 16]!
169+ sub tmp1, data1, zeroones
170+ orr tmp2, data1, REP8_7f
171+ sub tmp3, data2, zeroones
172+ orr tmp4, data2, REP8_7f
173+ bics has_nul1, tmp1, tmp2
174+ bic has_nul2, tmp3, tmp4
175+ ccmp has_nul2, 0, 0, eq
176+ beq L(nonascii_loop)
177+ b L(tail)
178+
179+ /* Load 16 bytes from [srcin & ~15] and force the bytes that precede
180+ srcin to 0x7f, so we ignore any NUL bytes before the string.
181+ Then continue in the aligned loop. */
182+L(page_cross):
183+ bic src, srcin, 15
184+ ldp data1, data2, [src]
185+ lsl tmp1, srcin, 3
186+ mov tmp4, -1
187+#ifdef __AARCH64EB__
188+ /* Big-endian. Early bytes are at MSB. */
189+ lsr tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */
190+#else
106191 /* Little-endian. Early bytes are at LSB. */
107−CPU_LE( lsr tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */
192+ lsl tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */
193+#endif
194+ orr tmp1, tmp1, REP8_80
195+ orn data1, data1, tmp1
196+ orn tmp2, data2, tmp1
197+ tst srcin, 8
198+ csel data1, data1, tmp4, eq
199+ csel data2, data2, tmp2, eq
200+ b L(page_cross_entry)
108201
109− orr data1, data1, tmp2
110− orr data2a, data2, tmp2
111− csinv data1, data1, xzr, le
112− csel data2, data2, data2a, le
113− b .Lrealigned
114202 SYM_FUNC_END_PI(strlen)
115203 EXPORT_SYMBOL_NOKASAN(strlen)
arch/arm64/lib/strncmp.S+184−222
@@ -1,299 +1,261 @@
11 /* SPDX-License-Identifier: GPL-2.0-only */
22 /*
3− * Copyright (C) 2013 ARM Ltd.
4− * Copyright (C) 2013 Linaro.
3+ * Copyright (c) 2013-2021, Arm Limited.
54 *
6− * This code is based on glibc cortex strings work originally authored by Linaro
7− * be found @
8− *
9− * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/
10− * files/head:/src/aarch64/
5+ * Adapted from the original at:
6+ * https://github.com/ARM-software/optimized-routines/blob/e823e3abf5f89ecb/string/aarch64/strncmp.S
117 */
128
139 #include <linux/linkage.h>
1410 #include <asm/assembler.h>
1511
16−/*
17− * compare two strings
12+/* Assumptions:
1813 *
19− * Parameters:
20− * x0 - const string 1 pointer
21− * x1 - const string 2 pointer
22− * x2 - the maximal length to be compared
23− * Returns:
24− * x0 - an integer less than, equal to, or greater than zero if s1 is found,
25− * respectively, to be less than, to match, or be greater than s2.
14+ * ARMv8-a, AArch64
2615 */
2716
17+#define L(label) .L ## label
18+
2819 #define REP8_01 0x0101010101010101
2920 #define REP8_7f 0x7f7f7f7f7f7f7f7f
3021 #define REP8_80 0x8080808080808080
3122
3223 /* Parameters and result. */
33−src1 .req x0
34−src2 .req x1
35−limit .req x2
36−result .req x0
24+#define src1 x0
25+#define src2 x1
26+#define limit x2
27+#define result x0
3728
3829 /* Internal variables. */
39−data1 .req x3
40−data1w .req w3
41−data2 .req x4
42−data2w .req w4
43−has_nul .req x5
44−diff .req x6
45−syndrome .req x7
46−tmp1 .req x8
47−tmp2 .req x9
48−tmp3 .req x10
49−zeroones .req x11
50−pos .req x12
51−limit_wd .req x13
52−mask .req x14
53−endloop .req x15
30+#define data1 x3
31+#define data1w w3
32+#define data2 x4
33+#define data2w w4
34+#define has_nul x5
35+#define diff x6
36+#define syndrome x7
37+#define tmp1 x8
38+#define tmp2 x9
39+#define tmp3 x10
40+#define zeroones x11
41+#define pos x12
42+#define limit_wd x13
43+#define mask x14
44+#define endloop x15
45+#define count mask
5446
5547 SYM_FUNC_START_WEAK_PI(strncmp)
56− cbz limit, .Lret0
48+ cbz limit, L(ret0)
5749 eor tmp1, src1, src2
5850 mov zeroones, #REP8_01
5951 tst tmp1, #7
60− b.ne .Lmisaligned8
61− ands tmp1, src1, #7
62− b.ne .Lmutual_align
52+ and count, src1, #7
53+ b.ne L(misaligned8)
54+ cbnz count, L(mutual_align)
6355 /* Calculate the number of full and partial words -1. */
64− /*
65− * when limit is mulitply of 8, if not sub 1,
66− * the judgement of last dword will wrong.
67− */
68− sub limit_wd, limit, #1 /* limit != 0, so no underflow. */
69− lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */
56+ sub limit_wd, limit, #1 /* limit != 0, so no underflow. */
57+ lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */
7058
71− /*
72− * NUL detection works on the principle that (X - 1) & (~X) & 0x80
73− * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
74− * can be done in parallel across the entire word.
75− */
76−.Lloop_aligned:
59+ /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
60+ (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
61+ can be done in parallel across the entire word. */
62+ .p2align 4
63+L(loop_aligned):
7764 ldr data1, [src1], #8
7865 ldr data2, [src2], #8
79−.Lstart_realigned:
66+L(start_realigned):
8067 subs limit_wd, limit_wd, #1
8168 sub tmp1, data1, zeroones
8269 orr tmp2, data1, #REP8_7f
83− eor diff, data1, data2 /* Non-zero if differences found. */
84− csinv endloop, diff, xzr, pl /* Last Dword or differences.*/
85− bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
70+ eor diff, data1, data2 /* Non-zero if differences found. */
71+ csinv endloop, diff, xzr, pl /* Last Dword or differences. */
72+ bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
8673 ccmp endloop, #0, #0, eq
87− b.eq .Lloop_aligned
74+ b.eq L(loop_aligned)
75+ /* End of main loop */
8876
89− /*Not reached the limit, must have found the end or a diff. */
90− tbz limit_wd, #63, .Lnot_limit
77+ /* Not reached the limit, must have found the end or a diff. */
78+ tbz limit_wd, #63, L(not_limit)
9179
9280 /* Limit % 8 == 0 => all bytes significant. */
9381 ands limit, limit, #7
94− b.eq .Lnot_limit
82+ b.eq L(not_limit)
9583
96− lsl limit, limit, #3 /* Bits -> bytes. */
84+ lsl limit, limit, #3 /* Bits -> bytes. */
9785 mov mask, #~0
98−CPU_BE( lsr mask, mask, limit )
99−CPU_LE( lsl mask, mask, limit )
86+#ifdef __AARCH64EB__
87+ lsr mask, mask, limit
88+#else
89+ lsl mask, mask, limit
90+#endif
10091 bic data1, data1, mask
10192 bic data2, data2, mask
10293
10394 /* Make sure that the NUL byte is marked in the syndrome. */
10495 orr has_nul, has_nul, mask
10596
106−.Lnot_limit:
97+L(not_limit):
10798 orr syndrome, diff, has_nul
108− b .Lcal_cmpresult
10999
110−.Lmutual_align:
111− /*
112− * Sources are mutually aligned, but are not currently at an
113− * alignment boundary. Round down the addresses and then mask off
114− * the bytes that precede the start point.
115− * We also need to adjust the limit calculations, but without
116− * overflowing if the limit is near ULONG_MAX.
117− */
100+#ifndef __AARCH64EB__
101+ rev syndrome, syndrome
102+ rev data1, data1
103+ /* The MS-non-zero bit of the syndrome marks either the first bit
104+ that is different, or the top bit of the first zero byte.
105+ Shifting left now will bring the critical information into the
106+ top bits. */
107+ clz pos, syndrome
108+ rev data2, data2
109+ lsl data1, data1, pos
110+ lsl data2, data2, pos
111+ /* But we need to zero-extend (char is unsigned) the value and then
112+ perform a signed 32-bit subtraction. */
113+ lsr data1, data1, #56
114+ sub result, data1, data2, lsr #56
115+ ret
116+#else
117+ /* For big-endian we cannot use the trick with the syndrome value
118+ as carry-propagation can corrupt the upper bits if the trailing
119+ bytes in the string contain 0x01. */
120+ /* However, if there is no NUL byte in the dword, we can generate
121+ the result directly. We can't just subtract the bytes as the
122+ MSB might be significant. */
123+ cbnz has_nul, 1f
124+ cmp data1, data2
125+ cset result, ne
126+ cneg result, result, lo
127+ ret
128+1:
129+ /* Re-compute the NUL-byte detection, using a byte-reversed value. */
130+ rev tmp3, data1
131+ sub tmp1, tmp3, zeroones
132+ orr tmp2, tmp3, #REP8_7f
133+ bic has_nul, tmp1, tmp2
134+ rev has_nul, has_nul
135+ orr syndrome, diff, has_nul
136+ clz pos, syndrome
137+ /* The MS-non-zero bit of the syndrome marks either the first bit
138+ that is different, or the top bit of the first zero byte.
139+ Shifting left now will bring the critical information into the
140+ top bits. */
141+ lsl data1, data1, pos
142+ lsl data2, data2, pos
143+ /* But we need to zero-extend (char is unsigned) the value and then
144+ perform a signed 32-bit subtraction. */
145+ lsr data1, data1, #56
146+ sub result, data1, data2, lsr #56
147+ ret
148+#endif
149+
150+L(mutual_align):
151+ /* Sources are mutually aligned, but are not currently at an
152+ alignment boundary. Round down the addresses and then mask off
153+ the bytes that precede the start point.
154+ We also need to adjust the limit calculations, but without
155+ overflowing if the limit is near ULONG_MAX. */
118156 bic src1, src1, #7
119157 bic src2, src2, #7
120158 ldr data1, [src1], #8
121− neg tmp3, tmp1, lsl #3 /* 64 - bits(bytes beyond align). */
159+ neg tmp3, count, lsl #3 /* 64 - bits(bytes beyond align). */
122160 ldr data2, [src2], #8
123161 mov tmp2, #~0
124− sub limit_wd, limit, #1 /* limit != 0, so no underflow. */
162+ sub limit_wd, limit, #1 /* limit != 0, so no underflow. */
163+#ifdef __AARCH64EB__
125164 /* Big-endian. Early bytes are at MSB. */
126−CPU_BE( lsl tmp2, tmp2, tmp3 ) /* Shift (tmp1 & 63). */
165+ lsl tmp2, tmp2, tmp3 /* Shift (count & 63). */
166+#else
127167 /* Little-endian. Early bytes are at LSB. */
128−CPU_LE( lsr tmp2, tmp2, tmp3 ) /* Shift (tmp1 & 63). */
129−
168+ lsr tmp2, tmp2, tmp3 /* Shift (count & 63). */
169+#endif
130170 and tmp3, limit_wd, #7
131171 lsr limit_wd, limit_wd, #3
132− /* Adjust the limit. Only low 3 bits used, so overflow irrelevant.*/
133− add limit, limit, tmp1
134− add tmp3, tmp3, tmp1
172+ /* Adjust the limit. Only low 3 bits used, so overflow irrelevant. */
173+ add limit, limit, count
174+ add tmp3, tmp3, count
135175 orr data1, data1, tmp2
136176 orr data2, data2, tmp2
137177 add limit_wd, limit_wd, tmp3, lsr #3
138− b .Lstart_realigned
178+ b L(start_realigned)
179+
180+ .p2align 4
181+ /* Don't bother with dwords for up to 16 bytes. */
182+L(misaligned8):
183+ cmp limit, #16
184+ b.hs L(try_misaligned_words)
139185
140−/*when src1 offset is not equal to src2 offset...*/
141−.Lmisaligned8:
142− cmp limit, #8
143− b.lo .Ltiny8proc /*limit < 8... */
144− /*
145− * Get the align offset length to compare per byte first.
146− * After this process, one string's address will be aligned.*/
147− and tmp1, src1, #7
148− neg tmp1, tmp1
149− add tmp1, tmp1, #8
150− and tmp2, src2, #7
151− neg tmp2, tmp2
152− add tmp2, tmp2, #8
153− subs tmp3, tmp1, tmp2
154− csel pos, tmp1, tmp2, hi /*Choose the maximum. */
155− /*
156− * Here, limit is not less than 8, so directly run .Ltinycmp
157− * without checking the limit.*/
158− sub limit, limit, pos
159−.Ltinycmp:
186+L(byte_loop):
187+ /* Perhaps we can do better than this. */
160188 ldrb data1w, [src1], #1
161189 ldrb data2w, [src2], #1
162− subs pos, pos, #1
163− ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */
164− ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */
165− b.eq .Ltinycmp
166− cbnz pos, 1f /*find the null or unequal...*/
167− cmp data1w, #1
168− ccmp data1w, data2w, #0, cs
169− b.eq .Lstart_align /*the last bytes are equal....*/
170−1:
190+ subs limit, limit, #1
191+ ccmp data1w, #1, #0, hi /* NZCV = 0b0000. */
192+ ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */
193+ b.eq L(byte_loop)
194+L(done):
171195 sub result, data1, data2
172196 ret
173−
174−.Lstart_align:
197+ /* Align the SRC1 to a dword by doing a bytewise compare and then do
198+ the dword loop. */
199+L(try_misaligned_words):
175200 lsr limit_wd, limit, #3
176− cbz limit_wd, .Lremain8
177− /*process more leading bytes to make str1 aligned...*/
178− ands xzr, src1, #7
179− b.eq .Lrecal_offset
180− add src1, src1, tmp3 /*tmp3 is positive in this branch.*/
181− add src2, src2, tmp3
182− ldr data1, [src1], #8
183− ldr data2, [src2], #8
201+ cbz count, L(do_misaligned)
184202
185− sub limit, limit, tmp3
203+ neg count, count
204+ and count, count, #7
205+ sub limit, limit, count
186206 lsr limit_wd, limit, #3
187− subs limit_wd, limit_wd, #1
188207
189− sub tmp1, data1, zeroones
190− orr tmp2, data1, #REP8_7f
191− eor diff, data1, data2 /* Non-zero if differences found. */
192− csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/
193− bics has_nul, tmp1, tmp2
194− ccmp endloop, #0, #0, eq /*has_null is ZERO: no null byte*/
195− b.ne .Lunequal_proc
196− /*How far is the current str2 from the alignment boundary...*/
197− and tmp3, tmp3, #7
198−.Lrecal_offset:
199− neg pos, tmp3
200−.Lloopcmp_proc:
201− /*
202− * Divide the eight bytes into two parts. First,backwards the src2
203− * to an alignment boundary,load eight bytes from the SRC2 alignment
204− * boundary,then compare with the relative bytes from SRC1.
205− * If all 8 bytes are equal,then start the second part's comparison.
206− * Otherwise finish the comparison.
207− * This special handle can garantee all the accesses are in the
208− * thread/task space in avoid to overrange access.
209− */
210− ldr data1, [src1,pos]
211− ldr data2, [src2,pos]
212− sub tmp1, data1, zeroones
213− orr tmp2, data1, #REP8_7f
214− bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
215− eor diff, data1, data2 /* Non-zero if differences found. */
216− csinv endloop, diff, xzr, eq
217− cbnz endloop, .Lunequal_proc
208+L(page_end_loop):
209+ ldrb data1w, [src1], #1
210+ ldrb data2w, [src2], #1
211+ cmp data1w, #1
212+ ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */
213+ b.ne L(done)
214+ subs count, count, #1
215+ b.hi L(page_end_loop)
216+
217+L(do_misaligned):
218+ /* Prepare ourselves for the next page crossing. Unlike the aligned
219+ loop, we fetch 1 less dword because we risk crossing bounds on
220+ SRC2. */
221+ mov count, #8
222+ subs limit_wd, limit_wd, #1
223+ b.lo L(done_loop)
224+L(loop_misaligned):
225+ and tmp2, src2, #0xff8
226+ eor tmp2, tmp2, #0xff8
227+ cbz tmp2, L(page_end_loop)
218228
219− /*The second part process*/
220229 ldr data1, [src1], #8
221230 ldr data2, [src2], #8
222− subs limit_wd, limit_wd, #1
223231 sub tmp1, data1, zeroones
224232 orr tmp2, data1, #REP8_7f
225− eor diff, data1, data2 /* Non-zero if differences found. */
226− csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/
227− bics has_nul, tmp1, tmp2
228− ccmp endloop, #0, #0, eq /*has_null is ZERO: no null byte*/
229− b.eq .Lloopcmp_proc
230−
231−.Lunequal_proc:
232− orr syndrome, diff, has_nul
233− cbz syndrome, .Lremain8
234−.Lcal_cmpresult:
235− /*
236− * reversed the byte-order as big-endian,then CLZ can find the most
237− * significant zero bits.
238− */
239−CPU_LE( rev syndrome, syndrome )
240−CPU_LE( rev data1, data1 )
241−CPU_LE( rev data2, data2 )
242− /*
243− * For big-endian we cannot use the trick with the syndrome value
244− * as carry-propagation can corrupt the upper bits if the trailing
245− * bytes in the string contain 0x01.
246− * However, if there is no NUL byte in the dword, we can generate
247− * the result directly. We can't just subtract the bytes as the
248− * MSB might be significant.
249− */
250−CPU_BE( cbnz has_nul, 1f )
251−CPU_BE( cmp data1, data2 )
252−CPU_BE( cset result, ne )
253−CPU_BE( cneg result, result, lo )
254−CPU_BE( ret )
255−CPU_BE( 1: )
256− /* Re-compute the NUL-byte detection, using a byte-reversed value.*/
257−CPU_BE( rev tmp3, data1 )
258−CPU_BE( sub tmp1, tmp3, zeroones )
259−CPU_BE( orr tmp2, tmp3, #REP8_7f )
260−CPU_BE( bic has_nul, tmp1, tmp2 )
261−CPU_BE( rev has_nul, has_nul )
262−CPU_BE( orr syndrome, diff, has_nul )
263− /*
264− * The MS-non-zero bit of the syndrome marks either the first bit
265− * that is different, or the top bit of the first zero byte.
266− * Shifting left now will bring the critical information into the
267− * top bits.
268− */
269− clz pos, syndrome
270− lsl data1, data1, pos
271− lsl data2, data2, pos
272− /*
273− * But we need to zero-extend (char is unsigned) the value and then
274− * perform a signed 32-bit subtraction.
275− */
276− lsr data1, data1, #56
277− sub result, data1, data2, lsr #56
278− ret
279−
280−.Lremain8:
281− /* Limit % 8 == 0 => all bytes significant. */
282− ands limit, limit, #7
283− b.eq .Lret0
284−.Ltiny8proc:
285− ldrb data1w, [src1], #1
286− ldrb data2w, [src2], #1
287− subs limit, limit, #1
233+ eor diff, data1, data2 /* Non-zero if differences found. */
234+ bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
235+ ccmp diff, #0, #0, eq
236+ b.ne L(not_limit)
237+ subs limit_wd, limit_wd, #1
238+ b.pl L(loop_misaligned)
288239
289− ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */
290− ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */
291− b.eq .Ltiny8proc
292− sub result, data1, data2
293− ret
240+L(done_loop):
241+ /* We found a difference or a NULL before the limit was reached. */
242+ and limit, limit, #7
243+ cbz limit, L(not_limit)
244+ /* Read the last word. */
245+ sub src1, src1, 8
246+ sub src2, src2, 8
247+ ldr data1, [src1, limit]
248+ ldr data2, [src2, limit]
249+ sub tmp1, data1, zeroones
250+ orr tmp2, data1, #REP8_7f
251+ eor diff, data1, data2 /* Non-zero if differences found. */
252+ bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */
253+ ccmp diff, #0, #0, eq
254+ b.ne L(not_limit)
294255
295−.Lret0:
256+L(ret0):
296257 mov result, #0
297258 ret
259+
298260 SYM_FUNC_END_PI(strncmp)
299261 EXPORT_SYMBOL_NOKASAN(strncmp)
arch/arm64/lib/uaccess_flushcache.c+2−2
@@ -15,7 +15,7 @@ void memcpy_flushcache(void *dst, const void *src, size_t cnt) · 14 unmodified lines
1515 * barrier to order the cache maintenance against the memcpy.
1616 */
1717 memcpy(dst, src, cnt);
18− __clean_dcache_area_pop(dst, cnt);
18+ dcache_clean_pop((unsigned long)dst, (unsigned long)dst + cnt);
1919 }
2020 EXPORT_SYMBOL_GPL(memcpy_flushcache);
2121
@@ -33,6 +33,6 @@ unsigned long __copy_user_flushcache(void *to, const void __user *from, · 11 unmodified lines
3333 rc = raw_copy_from_user(to, from, n);
3434
3535 /* See above */
36− __clean_dcache_area_pop(to, n - rc);
36+ dcache_clean_pop((unsigned long)to, (unsigned long)to + n - rc);
3737 return rc;
3838 }
arch/arm64/mm/cache.S+81−77
@@ -15,7 +15,7 @@ · 14 unmodified lines
1515 #include <asm/asm-uaccess.h>
1616
1717 /*
18− * flush_icache_range(start,end)
18+ * caches_clean_inval_pou_macro(start,end) [fixup]
1919 *
2020 * Ensure that the I and D caches are coherent within specified region.
2121 * This is typically used when code has been written to a memory region,
@@ -23,12 +23,27 @@ · 1 unmodified line
2323 *
2424 * - start - virtual start address of region
2525 * - end - virtual end address of region
26+ * - fixup - optional label to branch to on user fault
2627 */
27−SYM_FUNC_START(__flush_icache_range)
28− /* FALLTHROUGH */
28+.macro caches_clean_inval_pou_macro, fixup
29+alternative_if ARM64_HAS_CACHE_IDC
30+ dsb ishst
31+ b .Ldc_skip_\@
32+alternative_else_nop_endif
33+ mov x2, x0
34+ mov x3, x1
35+ dcache_by_line_op cvau, ish, x2, x3, x4, x5, \fixup
36+.Ldc_skip_\@:
37+alternative_if ARM64_HAS_CACHE_DIC
38+ isb
39+ b .Lic_skip_\@
40+alternative_else_nop_endif
41+ invalidate_icache_by_line x0, x1, x2, x3, \fixup
42+.Lic_skip_\@:
43+.endm
2944
3045 /*
31− * __flush_cache_user_range(start,end)
46+ * caches_clean_inval_pou(start,end)
3247 *
3348 * Ensure that the I and D caches are coherent within specified region.
3449 * This is typically used when code has been written to a memory region,
@@ -37,117 +52,103 @@ SYM_FUNC_START(__flush_icache_range) · 2 unmodified lines
3752 * - start - virtual start address of region
3853 * - end - virtual end address of region
3954 */
40−SYM_FUNC_START(__flush_cache_user_range)
55+SYM_FUNC_START(caches_clean_inval_pou)
56+ caches_clean_inval_pou_macro
57+ ret
58+SYM_FUNC_END(caches_clean_inval_pou)
59+
60+/*
61+ * caches_clean_inval_user_pou(start,end)
62+ *
63+ * Ensure that the I and D caches are coherent within specified region.
64+ * This is typically used when code has been written to a memory region,
65+ * and will be executed.
66+ *
67+ * - start - virtual start address of region
68+ * - end - virtual end address of region
69+ */
70+SYM_FUNC_START(caches_clean_inval_user_pou)
4171 uaccess_ttbr0_enable x2, x3, x4
42−alternative_if ARM64_HAS_CACHE_IDC
43− dsb ishst
44− b 7f
45−alternative_else_nop_endif
46− dcache_line_size x2, x3
47− sub x3, x2, #1
48− bic x4, x0, x3
49−1:
50−user_alt 9f, "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE
51− add x4, x4, x2
52− cmp x4, x1
53− b.lo 1b
54− dsb ish
5572
56−7:
57−alternative_if ARM64_HAS_CACHE_DIC
58− isb
59− b 8f
60−alternative_else_nop_endif
61− invalidate_icache_by_line x0, x1, x2, x3, 9f
62−8: mov x0, #0
73+ caches_clean_inval_pou_macro 2f
74+ mov x0, xzr
6375 1:
6476 uaccess_ttbr0_disable x1, x2
6577 ret
66−9:
78+2:
6779 mov x0, #-EFAULT
6880 b 1b
69−SYM_FUNC_END(__flush_icache_range)
70−SYM_FUNC_END(__flush_cache_user_range)
81+SYM_FUNC_END(caches_clean_inval_user_pou)
7182
7283 /*
73− * invalidate_icache_range(start,end)
84+ * icache_inval_pou(start,end)
7485 *
7586 * Ensure that the I cache is invalid within specified region.
7687 *
7788 * - start - virtual start address of region
7889 * - end - virtual end address of region
7990 */
80−SYM_FUNC_START(invalidate_icache_range)
91+SYM_FUNC_START(icache_inval_pou)
8192 alternative_if ARM64_HAS_CACHE_DIC
82− mov x0, xzr
8393 isb
8494 ret
8595 alternative_else_nop_endif
8696
87− uaccess_ttbr0_enable x2, x3, x4
88−
89− invalidate_icache_by_line x0, x1, x2, x3, 2f
90− mov x0, xzr
91−1:
92− uaccess_ttbr0_disable x1, x2
97+ invalidate_icache_by_line x0, x1, x2, x3
9398 ret
94−2:
95− mov x0, #-EFAULT
96− b 1b
97−SYM_FUNC_END(invalidate_icache_range)
99+SYM_FUNC_END(icache_inval_pou)
98100
99101 /*
100− * __flush_dcache_area(kaddr, size)
102+ * dcache_clean_inval_poc(start, end)
101103 *
102− * Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
104+ * Ensure that any D-cache lines for the interval [start, end)
103105 * are cleaned and invalidated to the PoC.
104106 *
105− * - kaddr - kernel address
106− * - size - size in question
107+ * - start - virtual start address of region
108+ * - end - virtual end address of region
107109 */
108−SYM_FUNC_START_PI(__flush_dcache_area)
110+SYM_FUNC_START_PI(dcache_clean_inval_poc)
109111 dcache_by_line_op civac, sy, x0, x1, x2, x3
110112 ret
111−SYM_FUNC_END_PI(__flush_dcache_area)
113+SYM_FUNC_END_PI(dcache_clean_inval_poc)
112114
113115 /*
114− * __clean_dcache_area_pou(kaddr, size)
116+ * dcache_clean_pou(start, end)
115117 *
116− * Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
118+ * Ensure that any D-cache lines for the interval [start, end)
117119 * are cleaned to the PoU.
118120 *
119− * - kaddr - kernel address
120− * - size - size in question
121+ * - start - virtual start address of region
122+ * - end - virtual end address of region
121123 */
122−SYM_FUNC_START(__clean_dcache_area_pou)
124+SYM_FUNC_START(dcache_clean_pou)
123125 alternative_if ARM64_HAS_CACHE_IDC
124126 dsb ishst
125127 ret
126128 alternative_else_nop_endif
127129 dcache_by_line_op cvau, ish, x0, x1, x2, x3
128130 ret
129−SYM_FUNC_END(__clean_dcache_area_pou)
131+SYM_FUNC_END(dcache_clean_pou)
130132
131133 /*
132− * __inval_dcache_area(kaddr, size)
134+ * dcache_inval_poc(start, end)
133135 *
134− * Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
136+ * Ensure that any D-cache lines for the interval [start, end)
135137 * are invalidated. Any partial lines at the ends of the interval are
136138 * also cleaned to PoC to prevent data loss.
137139 *
138− * - kaddr - kernel address
139− * - size - size in question
140+ * - start - kernel start address of region
141+ * - end - kernel end address of region
140142 */
141143 SYM_FUNC_START_LOCAL(__dma_inv_area)
142−SYM_FUNC_START_PI(__inval_dcache_area)
144+SYM_FUNC_START_PI(dcache_inval_poc)
143145 /* FALLTHROUGH */
144146
145147 /*
146− * __dma_inv_area(start, size)
148+ * __dma_inv_area(start, end)
147149 * - start - virtual start address of region
148− * - size - size in question
150+ * - end - virtual end address of region
149151 */
150− add x1, x1, x0
151152 dcache_line_size x2, x3
152153 sub x3, x2, #1
153154 tst x1, x3 // end cache line aligned?
@@ -165,48 +166,48 @@ SYM_FUNC_START_PI(__inval_dcache_area) · 11 unmodified lines
165166 b.lo 2b
166167 dsb sy
167168 ret
168−SYM_FUNC_END_PI(__inval_dcache_area)
169+SYM_FUNC_END_PI(dcache_inval_poc)
169170 SYM_FUNC_END(__dma_inv_area)
170171
171172 /*
172− * __clean_dcache_area_poc(kaddr, size)
173+ * dcache_clean_poc(start, end)
173174 *
174− * Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
175+ * Ensure that any D-cache lines for the interval [start, end)
175176 * are cleaned to the PoC.
176177 *
177− * - kaddr - kernel address
178− * - size - size in question
178+ * - start - virtual start address of region
179+ * - end - virtual end address of region
179180 */
180181 SYM_FUNC_START_LOCAL(__dma_clean_area)
181−SYM_FUNC_START_PI(__clean_dcache_area_poc)
182+SYM_FUNC_START_PI(dcache_clean_poc)
182183 /* FALLTHROUGH */
183184
184185 /*
185− * __dma_clean_area(start, size)
186+ * __dma_clean_area(start, end)
186187 * - start - virtual start address of region
187− * - size - size in question
188+ * - end - virtual end address of region
188189 */
189190 dcache_by_line_op cvac, sy, x0, x1, x2, x3
190191 ret
191−SYM_FUNC_END_PI(__clean_dcache_area_poc)
192+SYM_FUNC_END_PI(dcache_clean_poc)
192193 SYM_FUNC_END(__dma_clean_area)
193194
194195 /*
195− * __clean_dcache_area_pop(kaddr, size)
196+ * dcache_clean_pop(start, end)
196197 *
197− * Ensure that any D-cache lines for the interval [kaddr, kaddr+size)
198+ * Ensure that any D-cache lines for the interval [start, end)
198199 * are cleaned to the PoP.
199200 *
200− * - kaddr - kernel address
201− * - size - size in question
201+ * - start - virtual start address of region
202+ * - end - virtual end address of region
202203 */
203−SYM_FUNC_START_PI(__clean_dcache_area_pop)
204+SYM_FUNC_START_PI(dcache_clean_pop)
204205 alternative_if_not ARM64_HAS_DCPOP
205− b __clean_dcache_area_poc
206+ b dcache_clean_poc
206207 alternative_else_nop_endif
207208 dcache_by_line_op cvap, sy, x0, x1, x2, x3
208209 ret
209−SYM_FUNC_END_PI(__clean_dcache_area_pop)
210+SYM_FUNC_END_PI(dcache_clean_pop)
210211
211212 /*
212213 * __dma_flush_area(start, size)
@@ -217,6 +218,7 @@ SYM_FUNC_END_PI(__clean_dcache_area_pop) · 4 unmodified lines
217218 * - size - size in question
218219 */
219220 SYM_FUNC_START_PI(__dma_flush_area)
221+ add x1, x0, x1
220222 dcache_by_line_op civac, sy, x0, x1, x2, x3
221223 ret
222224 SYM_FUNC_END_PI(__dma_flush_area)
@@ -228,6 +230,7 @@ SYM_FUNC_END_PI(__dma_flush_area) · 5 unmodified lines
228230 * - dir - DMA direction
229231 */
230232 SYM_FUNC_START_PI(__dma_map_area)
233+ add x1, x0, x1
231234 cmp w2, #DMA_FROM_DEVICE
232235 b.eq __dma_inv_area
233236 b __dma_clean_area
@@ -240,6 +243,7 @@ SYM_FUNC_END_PI(__dma_map_area) · 6 unmodified lines
240243 * - dir - DMA direction
241244 */
242245 SYM_FUNC_START_PI(__dma_unmap_area)
246+ add x1, x0, x1
243247 cmp w2, #DMA_TO_DEVICE
244248 b.ne __dma_inv_area
245249 ret
arch/arm64/mm/context.c+2−4
@@ -402,14 +402,12 @@ static int asids_init(void) · 401 unmodified lines
402402 {
403403 asid_bits = get_cpu_asid_bits();
404404 atomic64_set(&asid_generation, ASID_FIRST_VERSION);
405− asid_map = kcalloc(BITS_TO_LONGS(NUM_USER_ASIDS), sizeof(*asid_map),
406− GFP_KERNEL);
405+ asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL);
407406 if (!asid_map)
408407 panic("Failed to allocate bitmap for %lu ASIDs\n",
409408 NUM_USER_ASIDS);
410409
411− pinned_asid_map = kcalloc(BITS_TO_LONGS(NUM_USER_ASIDS),
412− sizeof(*pinned_asid_map), GFP_KERNEL);
410+ pinned_asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL);
413411 nr_pinned_asids = 0;
414412
415413 /*
arch/arm64/mm/fault.c+37−13
@@ -99,6 +99,8 @@ static void mem_abort_decode(unsigned int esr) · 98 unmodified lines
9999 pr_alert(" EA = %lu, S1PTW = %lu\n",
100100 (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT,
101101 (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT);
102+ pr_alert(" FSC = 0x%02x: %s\n", (esr & ESR_ELx_FSC),
103+ esr_to_fault_info(esr)->name);
102104
103105 if (esr_is_data_abort(esr))
104106 data_abort_decode(esr);
@@ -232,13 +234,17 @@ static bool is_el1_instruction_abort(unsigned int esr) · 127 unmodified lines
232234 return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR;
233235 }
234236
237+static bool is_el1_data_abort(unsigned int esr)
238+{
239+ return ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_CUR;
240+}
241+
235242 static inline bool is_el1_permission_fault(unsigned long addr, unsigned int esr,
236243 struct pt_regs *regs)
237244 {
238− unsigned int ec = ESR_ELx_EC(esr);
239245 unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE;
240246
241− if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR)
247+ if (!is_el1_data_abort(esr) && !is_el1_instruction_abort(esr))
242248 return false;
243249
244250 if (fsc_type == ESR_ELx_FSC_PERM)
@@ -258,7 +264,7 @@ static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr, · 13 unmodified lines
258264 unsigned long flags;
259265 u64 par, dfsc;
260266
261− if (ESR_ELx_EC(esr) != ESR_ELx_EC_DABT_CUR ||
267+ if (!is_el1_data_abort(esr) ||
262268 (esr & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT)
263269 return false;
264270
@@ -346,10 +352,9 @@ static void do_tag_recovery(unsigned long addr, unsigned int esr, · 81 unmodified lines
346352
347353 static bool is_el1_mte_sync_tag_check_fault(unsigned int esr)
348354 {
349− unsigned int ec = ESR_ELx_EC(esr);
350355 unsigned int fsc = esr & ESR_ELx_FSC;
351356
352− if (ec != ESR_ELx_EC_DABT_CUR)
357+ if (!is_el1_data_abort(esr))
353358 return false;
354359
355360 if (fsc == ESR_ELx_FSC_MTE)
@@ -504,7 +509,7 @@ static vm_fault_t __do_page_fault(struct mm_struct *mm, unsigned long addr, · 148 unmodified lines
504509 */
505510 if (!(vma->vm_flags & vm_flags))
506511 return VM_FAULT_BADACCESS;
507− return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags, regs);
512+ return handle_mm_fault(vma, addr, mm_flags, regs);
508513 }
509514
510515 static bool is_el0_instruction_abort(unsigned int esr)
@@ -836,13 +841,6 @@ void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs) · 325 unmodified lines
836841 }
837842 NOKPROBE_SYMBOL(do_mem_abort);
838843
839−void do_el0_irq_bp_hardening(void)
840−{
841− /* PC has already been checked in entry.S */
842− arm64_apply_bp_hardening();
843−}
844−NOKPROBE_SYMBOL(do_el0_irq_bp_hardening);
845−
846844 void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
847845 {
848846 arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN,
@@ -921,3 +919,29 @@ void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr, · 72 unmodified lines
921919 debug_exception_exit(regs);
922920 }
923921 NOKPROBE_SYMBOL(do_debug_exception);
922+
923+/*
924+ * Used during anonymous page fault handling.
925+ */
926+struct page *alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma,
927+ unsigned long vaddr)
928+{
929+ gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO;
930+
931+ /*
932+ * If the page is mapped with PROT_MTE, initialise the tags at the
933+ * point of allocation and page zeroing as this is usually faster than
934+ * separate DC ZVA and STGM.
935+ */
936+ if (vma->vm_flags & VM_MTE)
937+ flags |= __GFP_ZEROTAGS;
938+
939+ return alloc_page_vma(flags, vma, vaddr);
940+}
941+
942+void tag_clear_highpage(struct page *page)
943+{
944+ mte_zero_clear_page_tags(page_address(page));
945+ page_kasan_tag_reset(page);
946+ set_bit(PG_mte_tagged, &page->flags);
947+}
arch/arm64/mm/flush.c+14−15
@@ -14,28 +14,25 @@ · 13 unmodified lines
1414 #include <asm/cache.h>
1515 #include <asm/tlbflush.h>
1616
17−void sync_icache_aliases(void *kaddr, unsigned long len)
17+void sync_icache_aliases(unsigned long start, unsigned long end)
1818 {
19− unsigned long addr = (unsigned long)kaddr;
20−
2119 if (icache_is_aliasing()) {
22− __clean_dcache_area_pou(kaddr, len);
23− __flush_icache_all();
20+ dcache_clean_pou(start, end);
21+ icache_inval_all_pou();
2422 } else {
2523 /*
2624 * Don't issue kick_all_cpus_sync() after I-cache invalidation
2725 * for user mappings.
2826 */
29− __flush_icache_range(addr, addr + len);
27+ caches_clean_inval_pou(start, end);
3028 }
3129 }
3230
33−static void flush_ptrace_access(struct vm_area_struct *vma, struct page *page,
34− unsigned long uaddr, void *kaddr,
35− unsigned long len)
31+static void flush_ptrace_access(struct vm_area_struct *vma, unsigned long start,
32+ unsigned long end)
3633 {
3734 if (vma->vm_flags & VM_EXEC)
38− sync_icache_aliases(kaddr, len);
35+ sync_icache_aliases(start, end);
3936 }
4037
4138 /*
@@ -48,7 +45,7 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page, · 6 unmodified lines
4845 unsigned long len)
4946 {
5047 memcpy(dst, src, len);
51− flush_ptrace_access(vma, page, uaddr, dst, len);
48+ flush_ptrace_access(vma, (unsigned long)dst, (unsigned long)dst + len);
5249 }
5350
5451 void __sync_icache_dcache(pte_t pte)
@@ -56,7 +53,9 @@ void __sync_icache_dcache(pte_t pte) · 1 unmodified line
5653 struct page *page = pte_page(pte);
5754
5855 if (!test_bit(PG_dcache_clean, &page->flags)) {
59− sync_icache_aliases(page_address(page), page_size(page));
56+ sync_icache_aliases((unsigned long)page_address(page),
57+ (unsigned long)page_address(page) +
58+ page_size(page));
6059 set_bit(PG_dcache_clean, &page->flags);
6160 }
6261 }
@@ -77,20 +76,20 @@ EXPORT_SYMBOL(flush_dcache_page); · 14 unmodified lines
7776 /*
7877 * Additional functions defined in assembly.
7978 */
80−EXPORT_SYMBOL(__flush_icache_range);
79+EXPORT_SYMBOL(caches_clean_inval_pou);
8180
8281 #ifdef CONFIG_ARCH_HAS_PMEM_API
8382 void arch_wb_cache_pmem(void *addr, size_t size)
8483 {
8584 /* Ensure order against any prior non-cacheable writes */
8685 dmb(osh);
87− __clean_dcache_area_pop(addr, size);
86+ dcache_clean_pop((unsigned long)addr, (unsigned long)addr + size);
8887 }
8988 EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
9089
9190 void arch_invalidate_pmem(void *addr, size_t size)
9291 {
93− __inval_dcache_area(addr, size);
92+ dcache_inval_poc((unsigned long)addr, (unsigned long)addr + size);
9493 }
9594 EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
9695 #endif
arch/arm64/mm/init.c+7−0
@@ -499,6 +499,13 @@ void __init mem_init(void) · 498 unmodified lines
499499 BUILD_BUG_ON(TASK_SIZE_32 > DEFAULT_MAP_WINDOW_64);
500500 #endif
501501
502+ /*
503+ * Selected page table levels should match when derived from
504+ * scratch using the virtual address range and page size.
505+ */
506+ BUILD_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) !=
507+ CONFIG_PGTABLE_LEVELS);
508+
502509 if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) {
503510 extern int sysctl_overcommit_memory;
504511 /*
arch/arm64/mm/mmu.c+7−6
@@ -228,7 +228,7 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, · 227 unmodified lines
228228 next = pmd_addr_end(addr, end);
229229
230230 /* try section mapping first */
231− if (((addr | next | phys) & ~SECTION_MASK) == 0 &&
231+ if (((addr | next | phys) & ~PMD_MASK) == 0 &&
232232 (flags & NO_BLOCK_MAPPINGS) == 0) {
233233 pmd_set_huge(pmdp, phys, prot);
234234
@@ -1114,14 +1114,14 @@ static void free_empty_tables(unsigned long addr, unsigned long end, · 879 unmodified lines
11141114 }
11151115 #endif
11161116
1117−#if !ARM64_SWAPPER_USES_SECTION_MAPS
1117+#if !ARM64_KERNEL_USES_PMD_MAPS
11181118 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
11191119 struct vmem_altmap *altmap)
11201120 {
11211121 WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));
11221122 return vmemmap_populate_basepages(start, end, node, altmap);
11231123 }
1124−#else /* !ARM64_SWAPPER_USES_SECTION_MAPS */
1124+#else /* !ARM64_KERNEL_USES_PMD_MAPS */
11251125 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
11261126 struct vmem_altmap *altmap)
11271127 {
@@ -1166,17 +1166,18 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, · 38 unmodified lines
11661166
11671167 return 0;
11681168 }
1169−#endif /* !ARM64_SWAPPER_USES_SECTION_MAPS */
1169+#endif /* !ARM64_KERNEL_USES_PMD_MAPS */
1170+
1171+#ifdef CONFIG_MEMORY_HOTPLUG
11701172 void vmemmap_free(unsigned long start, unsigned long end,
11711173 struct vmem_altmap *altmap)
11721174 {
1173−#ifdef CONFIG_MEMORY_HOTPLUG
11741175 WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END));
11751176
11761177 unmap_hotplug_range(start, end, true, altmap);
11771178 free_empty_tables(start, end, VMEMMAP_START, VMEMMAP_END);
1178−#endif
11791179 }
1180+#endif /* CONFIG_MEMORY_HOTPLUG */
11801181
11811182 static inline pud_t *fixmap_pud(unsigned long addr)
11821183 {
arch/arm64/mm/proc.S+9−15
@@ -46,9 +46,13 @@ · 45 unmodified lines
4646 #endif
4747
4848 #ifdef CONFIG_KASAN_HW_TAGS
49−#define TCR_KASAN_HW_FLAGS SYS_TCR_EL1_TCMA1 | TCR_TBI1 | TCR_TBID1
49+#define TCR_MTE_FLAGS SYS_TCR_EL1_TCMA1 | TCR_TBI1 | TCR_TBID1
5050 #else
51−#define TCR_KASAN_HW_FLAGS 0
51+/*
52+ * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on
53+ * TBI being enabled at EL1.
54+ */
55+#define TCR_MTE_FLAGS TCR_TBI1 | TCR_TBID1
5256 #endif
5357
5458 /*
@@ -58,10 +62,8 @@ · 3 unmodified lines
5862 #define MAIR_EL1_SET \
5963 (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \
6064 MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \
61− MAIR_ATTRIDX(MAIR_ATTR_DEVICE_GRE, MT_DEVICE_GRE) | \
6265 MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \
6366 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \
64− MAIR_ATTRIDX(MAIR_ATTR_NORMAL_WT, MT_NORMAL_WT) | \
6567 MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED))
6668
6769 #ifdef CONFIG_CPU_PM
@@ -83,11 +85,7 @@ SYM_FUNC_START(cpu_do_suspend) · 15 unmodified lines
8385 mrs x9, mdscr_el1
8486 mrs x10, oslsr_el1
8587 mrs x11, sctlr_el1
86−alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
87− mrs x12, tpidr_el1
88−alternative_else
89− mrs x12, tpidr_el2
90−alternative_endif
88+ get_this_cpu_offset x12
9189 mrs x13, sp_el0
9290 stp x2, x3, [x0]
9391 stp x4, x5, [x0, #16]
@@ -145,11 +143,7 @@ SYM_FUNC_START(cpu_do_resume) · 51 unmodified lines
145143 msr mdscr_el1, x10
146144
147145 msr sctlr_el1, x12
148−alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
149− msr tpidr_el1, x13
150−alternative_else
151− msr tpidr_el2, x13
152−alternative_endif
146+ set_this_cpu_offset x13
153147 msr sp_el0, x14
154148 /*
155149 * Restore oslsr_el1 by writing oslar_el1
@@ -464,7 +458,7 @@ SYM_FUNC_START(__cpu_setup) · 308 unmodified lines
464458 msr_s SYS_TFSRE0_EL1, xzr
465459
466460 /* set the TCR_EL1 bits */
467− mov_q x10, TCR_KASAN_HW_FLAGS
461+ mov_q x10, TCR_MTE_FLAGS
468462 orr tcr, tcr, x10
469463 1:
470464 #endif
arch/arm64/mm/ptdump.c+0−4
@@ -157,10 +157,6 @@ static const struct prot_bits pte_bits[] = { · 156 unmodified lines
157157 .mask = PTE_ATTRINDX_MASK,
158158 .val = PTE_ATTRINDX(MT_DEVICE_nGnRE),
159159 .set = "DEVICE/nGnRE",
160− }, {
161− .mask = PTE_ATTRINDX_MASK,
162− .val = PTE_ATTRINDX(MT_DEVICE_GRE),
163− .set = "DEVICE/GRE",
164160 }, {
165161 .mask = PTE_ATTRINDX_MASK,
166162 .val = PTE_ATTRINDX(MT_NORMAL_NC),
arch/arm64/net/bpf_jit_comp.c+1−0
@@ -16,6 +16,7 @@ · 15 unmodified lines
1616 #include <asm/byteorder.h>
1717 #include <asm/cacheflush.h>
1818 #include <asm/debug-monitors.h>
19+#include <asm/insn.h>
1920 #include <asm/set_memory.h>
2021
2122 #include "bpf_jit.h"
arch/arm64/tools/cpucaps+2−1
@@ -3,7 +3,8 @@ · 2 unmodified lines
33 # Internal CPU capabilities constants, keep this list sorted
44
55 BTI
6−HAS_32BIT_EL0
6+# Unreliable: use system_supports_32bit_el0() instead.
7+HAS_32BIT_EL0_DO_NOT_USE
78 HAS_32BIT_EL1
89 HAS_ADDRESS_AUTH
910 HAS_ADDRESS_AUTH_ARCH
arch/ia64/include/asm/page.h+3−3
@@ -82,16 +82,16 @@ do { \ · 81 unmodified lines
8282 } while (0)
8383
8484
85−#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \
85+#define alloc_zeroed_user_highpage_movable(vma, vaddr) \
8686 ({ \
8787 struct page *page = alloc_page_vma( \
88− GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr); \
88+ GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr); \
8989 if (page) \
9090 flush_dcache_page(page); \
9191 page; \
9292 })
9393
94−#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
94+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE
9595
9696 #define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT)
9797
arch/m68k/include/asm/page_no.h+3−3
@@ -13,9 +13,9 @@ extern unsigned long memory_end; · 12 unmodified lines
1313 #define clear_user_page(page, vaddr, pg) clear_page(page)
1414 #define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
1515
16−#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \
17− alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr)
18−#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
16+#define alloc_zeroed_user_highpage_movable(vma, vaddr) \
17+ alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr)
18+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE
1919
2020 #define __pa(vaddr) ((unsigned long)(vaddr))
2121 #define __va(paddr) ((void *)((unsigned long)(paddr)))
arch/s390/include/asm/page.h+3−3
@@ -68,9 +68,9 @@ static inline void copy_page(void *to, void *from) · 67 unmodified lines
6868 #define clear_user_page(page, vaddr, pg) clear_page(page)
6969 #define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
7070
71−#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \
72− alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr)
73−#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
71+#define alloc_zeroed_user_highpage_movable(vma, vaddr) \
72+ alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr)
73+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE
7474
7575 /*
7676 * These are used to make use of C type-checking..
arch/x86/include/asm/page.h+3−3
@@ -34,9 +34,9 @@ static inline void copy_user_page(void *to, void *from, unsigned long vaddr, · 33 unmodified lines
3434 copy_page(to, from);
3535 }
3636
37−#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \
38− alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr)
39−#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
37+#define alloc_zeroed_user_highpage_movable(vma, vaddr) \
38+ alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr)
39+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE
4040
4141 #ifndef __pa
4242 #define __pa(x) __phys_addr((unsigned long)(x))
drivers/firmware/psci/psci.c+7−2
@@ -335,10 +335,15 @@ int psci_cpu_suspend_enter(u32 state) · 334 unmodified lines
335335 {
336336 int ret;
337337
338− if (!psci_power_state_loses_context(state))
338+ if (!psci_power_state_loses_context(state)) {
339+ struct arm_cpuidle_irq_context context;
340+
341+ arm_cpuidle_save_irq_context(&context);
339342 ret = psci_ops.cpu_suspend(state, 0);
340− else
343+ arm_cpuidle_restore_irq_context(&context);
344+ } else {
341345 ret = cpu_suspend(state, psci_suspend_finisher);
346+ }
342347
343348 return ret;
344349 }
drivers/firmware/smccc/smccc.c+4−0
@@ -15,6 +15,7 @@ static u32 smccc_version = ARM_SMCCC_VERSION_1_0; · 14 unmodified lines
1515 static enum arm_smccc_conduit smccc_conduit = SMCCC_CONDUIT_NONE;
1616
1717 bool __ro_after_init smccc_trng_available = false;
18+u64 __ro_after_init smccc_has_sve_hint = false;
1819
1920 void __init arm_smccc_version_init(u32 version, enum arm_smccc_conduit conduit)
2021 {
@@ -22,6 +23,9 @@ void __init arm_smccc_version_init(u32 version, enum arm_smccc_conduit conduit) · 1 unmodified line
2223 smccc_conduit = conduit;
2324
2425 smccc_trng_available = smccc_probe_trng();
26+ if (IS_ENABLED(CONFIG_ARM64_SVE) &&
27+ smccc_version >= ARM_SMCCC_VERSION_1_3)
28+ smccc_has_sve_hint = true;
2529 }
2630
2731 enum arm_smccc_conduit arm_smccc_1_1_get_conduit(void)
drivers/misc/lkdtm/bugs.c+3−3
@@ -463,7 +463,7 @@ void lkdtm_DOUBLE_FAULT(void) · 462 unmodified lines
463463 #ifdef CONFIG_ARM64
464464 static noinline void change_pac_parameters(void)
465465 {
466− if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH)) {
466+ if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL)) {
467467 /* Reset the keys of current task */
468468 ptrauth_thread_init_kernel(current);
469469 ptrauth_thread_switch_kernel(current);
@@ -477,8 +477,8 @@ noinline void lkdtm_CORRUPT_PAC(void) · 7 unmodified lines
477477 #define CORRUPT_PAC_ITERATE 10
478478 int i;
479479
480− if (!IS_ENABLED(CONFIG_ARM64_PTR_AUTH))
481− pr_err("FAIL: kernel not built with CONFIG_ARM64_PTR_AUTH\n");
480+ if (!IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL))
481+ pr_err("FAIL: kernel not built with CONFIG_ARM64_PTR_AUTH_KERNEL\n");
482482
483483 if (!system_supports_address_auth()) {
484484 pr_err("FAIL: CPU lacks pointer authentication feature\n");
drivers/perf/arm-cci.c+2−2
@@ -37,7 +37,7 @@ · 36 unmodified lines
3737
3838 #define CCI_PMU_CNTR_SIZE(model) ((model)->cntr_size)
3939 #define CCI_PMU_CNTR_BASE(model, idx) ((idx) * CCI_PMU_CNTR_SIZE(model))
40−#define CCI_PMU_CNTR_MASK ((1ULL << 32) -1)
40+#define CCI_PMU_CNTR_MASK ((1ULL << 32) - 1)
4141 #define CCI_PMU_CNTR_LAST(cci_pmu) (cci_pmu->num_cntrs - 1)
4242
4343 #define CCI_PMU_MAX_HW_CNTRS(model) \
@@ -806,7 +806,7 @@ static int pmu_get_event_idx(struct cci_pmu_hw_events *hw, struct perf_event *ev · 762 unmodified lines
806806 return cci_pmu->model->get_event_idx(cci_pmu, hw, cci_event);
807807
808808 /* Generic code to find an unused idx from the mask */
809− for(idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++)
809+ for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++)
810810 if (!test_and_set_bit(idx, hw->used_mask))
811811 return idx;
812812
drivers/perf/arm-ccn.c+2−4
@@ -1211,7 +1211,7 @@ static int arm_ccn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) · 1210 unmodified lines
12111211 perf_pmu_migrate_context(&dt->pmu, cpu, target);
12121212 dt->cpu = target;
12131213 if (ccn->irq)
1214− WARN_ON(irq_set_affinity_hint(ccn->irq, cpumask_of(dt->cpu)));
1214+ WARN_ON(irq_set_affinity(ccn->irq, cpumask_of(dt->cpu)));
12151215 return 0;
12161216 }
12171217
@@ -1291,7 +1291,7 @@ static int arm_ccn_pmu_init(struct arm_ccn *ccn) · 73 unmodified lines
12911291
12921292 /* Also make sure that the overflow interrupt is handled by this CPU */
12931293 if (ccn->irq) {
1294− err = irq_set_affinity_hint(ccn->irq, cpumask_of(ccn->dt.cpu));
1294+ err = irq_set_affinity(ccn->irq, cpumask_of(ccn->dt.cpu));
12951295 if (err) {
12961296 dev_err(ccn->dev, "Failed to set interrupt affinity!\n");
12971297 goto error_set_affinity;
@@ -1325,8 +1325,6 @@ static void arm_ccn_pmu_cleanup(struct arm_ccn *ccn) · 27 unmodified lines
13251325
13261326 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE,
13271327 &ccn->dt.node);
1328− if (ccn->irq)
1329− irq_set_affinity_hint(ccn->irq, NULL);
13301328 for (i = 0; i < ccn->num_xps; i++)
13311329 writel(0, ccn->xp[i].base + CCN_XP_DT_CONTROL);
13321330 writel(0, ccn->dt.base + CCN_DT_PMCR);
drivers/perf/arm-cmn.c+4−9
@@ -31,7 +31,7 @@ · 30 unmodified lines
3131 #define CMN_CI_CHILD_COUNT GENMASK_ULL(15, 0)
3232 #define CMN_CI_CHILD_PTR_OFFSET GENMASK_ULL(31, 16)
3333
34−#define CMN_CHILD_NODE_ADDR GENMASK(27,0)
34+#define CMN_CHILD_NODE_ADDR GENMASK(27, 0)
3535 #define CMN_CHILD_NODE_EXTERNAL BIT(31)
3636
3737 #define CMN_ADDR_NODE_PTR GENMASK(27, 14)
@@ -1162,7 +1162,7 @@ static int arm_cmn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) · 1124 unmodified lines
11621162
11631163 perf_pmu_migrate_context(&cmn->pmu, cpu, target);
11641164 for (i = 0; i < cmn->num_dtcs; i++)
1165− irq_set_affinity_hint(cmn->dtc[i].irq, cpumask_of(target));
1165+ irq_set_affinity(cmn->dtc[i].irq, cpumask_of(target));
11661166 cmn->cpu = target;
11671167 return 0;
11681168 }
@@ -1212,7 +1212,7 @@ static int arm_cmn_init_irqs(struct arm_cmn *cmn) · 43 unmodified lines
12121212 irq = cmn->dtc[i].irq;
12131213 for (j = i; j--; ) {
12141214 if (cmn->dtc[j].irq == irq) {
1215− cmn->dtc[j].irq_friend = j - i;
1215+ cmn->dtc[j].irq_friend = i - j;
12161216 goto next;
12171217 }
12181218 }
@@ -1222,7 +1222,7 @@ static int arm_cmn_init_irqs(struct arm_cmn *cmn) · 3 unmodified lines
12221222 if (err)
12231223 return err;
12241224
1225− err = irq_set_affinity_hint(irq, cpumask_of(cmn->cpu));
1225+ err = irq_set_affinity(irq, cpumask_of(cmn->cpu));
12261226 if (err)
12271227 return err;
12281228 next:
@@ -1568,16 +1568,11 @@ static int arm_cmn_probe(struct platform_device *pdev) · 339 unmodified lines
15681568 static int arm_cmn_remove(struct platform_device *pdev)
15691569 {
15701570 struct arm_cmn *cmn = platform_get_drvdata(pdev);
1571− int i;
15721571
15731572 writel_relaxed(0, cmn->dtc[0].base + CMN_DT_DTC_CTL);
15741573
15751574 perf_pmu_unregister(&cmn->pmu);
15761575 cpuhp_state_remove_instance(arm_cmn_hp_state, &cmn->cpuhp_node);
1577−
1578− for (i = 0; i < cmn->num_dtcs; i++)
1579− irq_set_affinity_hint(cmn->dtc[i].irq, NULL);
1580−
15811576 return 0;
15821577 }
15831578
drivers/perf/arm_dmc620_pmu.c+2−3
@@ -421,7 +421,7 @@ static struct dmc620_pmu_irq *__dmc620_pmu_get_irq(int irq_num) · 420 unmodified lines
421421 if (ret)
422422 goto out_free_aff;
423423
424− ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu));
424+ ret = irq_set_affinity(irq_num, cpumask_of(irq->cpu));
425425 if (ret)
426426 goto out_free_irq;
427427
@@ -475,7 +475,6 @@ static void dmc620_pmu_put_irq(struct dmc620_pmu *dmc620_pmu) · 47 unmodified lines
475475 list_del(&irq->irqs_node);
476476 mutex_unlock(&dmc620_pmu_irqs_lock);
477477
478− WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL));
479478 free_irq(irq->irq_num, irq);
480479 cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &irq->node);
481480 kfree(irq);
@@ -622,7 +621,7 @@ static int dmc620_pmu_cpu_teardown(unsigned int cpu, · 140 unmodified lines
622621 perf_pmu_migrate_context(&dmc620_pmu->pmu, irq->cpu, target);
623622 mutex_unlock(&dmc620_pmu_irqs_lock);
624623
625− WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target)));
624+ WARN_ON(irq_set_affinity(irq->irq_num, cpumask_of(target)));
626625 irq->cpu = target;
627626
628627 return 0;
drivers/perf/arm_dsu_pmu.c+2−6
@@ -687,7 +687,7 @@ static void dsu_pmu_probe_pmu(struct dsu_pmu *dsu_pmu) · 686 unmodified lines
687687 static void dsu_pmu_set_active_cpu(int cpu, struct dsu_pmu *dsu_pmu)
688688 {
689689 cpumask_set_cpu(cpu, &dsu_pmu->active_cpu);
690− if (irq_set_affinity_hint(dsu_pmu->irq, &dsu_pmu->active_cpu))
690+ if (irq_set_affinity(dsu_pmu->irq, &dsu_pmu->active_cpu))
691691 pr_warn("Failed to set irq affinity to %d\n", cpu);
692692 }
693693
@@ -769,7 +769,6 @@ static int dsu_pmu_device_probe(struct platform_device *pdev) · 75 unmodified lines
769769 if (rc) {
770770 cpuhp_state_remove_instance(dsu_pmu_cpuhp_state,
771771 &dsu_pmu->cpuhp_node);
772− irq_set_affinity_hint(dsu_pmu->irq, NULL);
773772 }
774773
775774 return rc;
@@ -781,7 +780,6 @@ static int dsu_pmu_device_remove(struct platform_device *pdev) · 5 unmodified lines
781780
782781 perf_pmu_unregister(&dsu_pmu->pmu);
783782 cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node);
784− irq_set_affinity_hint(dsu_pmu->irq, NULL);
785783
786784 return 0;
787785 }
@@ -840,10 +838,8 @@ static int dsu_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) · 52 unmodified lines
840838
841839 dst = dsu_pmu_get_online_cpu_any_but(dsu_pmu, cpu);
842840 /* If there are no active CPUs in the DSU, leave IRQ disabled */
843− if (dst >= nr_cpu_ids) {
844− irq_set_affinity_hint(dsu_pmu->irq, NULL);
841+ if (dst >= nr_cpu_ids)
845842 return 0;
846− }
847843
848844 perf_pmu_migrate_context(&dsu_pmu->pmu, cpu, dst);
849845 dsu_pmu_set_active_cpu(dst, dsu_pmu);
drivers/perf/arm_pmu.c+6−10
@@ -563,14 +563,14 @@ static int armpmu_filter_match(struct perf_event *event) · 562 unmodified lines
563563 return ret;
564564 }
565565
566−static ssize_t armpmu_cpumask_show(struct device *dev,
567− struct device_attribute *attr, char *buf)
566+static ssize_t cpus_show(struct device *dev,
567+ struct device_attribute *attr, char *buf)
568568 {
569569 struct arm_pmu *armpmu = to_arm_pmu(dev_get_drvdata(dev));
570570 return cpumap_print_to_pagebuf(true, buf, &armpmu->supported_cpus);
571571 }
572572
573−static DEVICE_ATTR(cpus, S_IRUGO, armpmu_cpumask_show, NULL);
573+static DEVICE_ATTR_RO(cpus);
574574
575575 static struct attribute *armpmu_common_attrs[] = {
576576 &dev_attr_cpus.attr,
@@ -644,11 +644,9 @@ int armpmu_request_irq(int irq, int cpu) · 67 unmodified lines
644644 }
645645
646646 irq_flags = IRQF_PERCPU |
647− IRQF_NOBALANCING |
647+ IRQF_NOBALANCING | IRQF_NO_AUTOEN |
648648 IRQF_NO_THREAD;
649649
650− irq_set_status_flags(irq, IRQ_NOAUTOEN);
651−
652650 err = request_nmi(irq, handler, irq_flags, "arm-pmu",
653651 per_cpu_ptr(&cpu_armpmu, cpu));
654652
@@ -670,7 +668,7 @@ int armpmu_request_irq(int irq, int cpu) · 15 unmodified lines
670668 &cpu_armpmu);
671669 irq_ops = &percpu_pmuirq_ops;
672670 } else {
673− has_nmi= true;
671+ has_nmi = true;
674672 irq_ops = &percpu_pmunmi_ops;
675673 }
676674 } else {
@@ -869,10 +867,8 @@ static struct arm_pmu *__armpmu_alloc(gfp_t flags) · 192 unmodified lines
869867 int cpu;
870868
871869 pmu = kzalloc(sizeof(*pmu), flags);
872− if (!pmu) {
873− pr_info("failed to allocate PMU device!\n");
870+ if (!pmu)
874871 goto out;
875− }
876872
877873 pmu->hw_events = alloc_percpu_gfp(struct pmu_hw_events, flags);
878874 if (!pmu->hw_events) {
drivers/perf/arm_smmuv3_pmu.c+15−20
@@ -277,7 +277,7 @@ static int smmu_pmu_apply_event_filter(struct smmu_pmu *smmu_pmu, · 276 unmodified lines
277277 struct perf_event *event, int idx)
278278 {
279279 u32 span, sid;
280− unsigned int num_ctrs = smmu_pmu->num_counters;
280+ unsigned int cur_idx, num_ctrs = smmu_pmu->num_counters;
281281 bool filter_en = !!get_filter_enable(event);
282282
283283 span = filter_en ? get_filter_span(event) :
@@ -285,17 +285,19 @@ static int smmu_pmu_apply_event_filter(struct smmu_pmu *smmu_pmu, · 1 unmodified line
285285 sid = filter_en ? get_filter_stream_id(event) :
286286 SMMU_PMCG_DEFAULT_FILTER_SID;
287287
288− /* Support individual filter settings */
289− if (!smmu_pmu->global_filter) {
288+ cur_idx = find_first_bit(smmu_pmu->used_counters, num_ctrs);
289+ /*
290+ * Per-counter filtering, or scheduling the first globally-filtered
291+ * event into an empty PMU so idx == 0 and it works out equivalent.
292+ */
293+ if (!smmu_pmu->global_filter || cur_idx == num_ctrs) {
290294 smmu_pmu_set_event_filter(event, idx, span, sid);
291295 return 0;
292296 }
293297
294− /* Requested settings same as current global settings*/
295− idx = find_first_bit(smmu_pmu->used_counters, num_ctrs);
296− if (idx == num_ctrs ||
297− smmu_pmu_check_global_filter(smmu_pmu->events[idx], event)) {
298− smmu_pmu_set_event_filter(event, 0, span, sid);
298+ /* Otherwise, must match whatever's currently scheduled */
299+ if (smmu_pmu_check_global_filter(smmu_pmu->events[cur_idx], event)) {
300+ smmu_pmu_set_evtyper(smmu_pmu, idx, get_event(event));
299301 return 0;
300302 }
301303
@@ -509,11 +511,8 @@ static ssize_t smmu_pmu_event_show(struct device *dev, · 207 unmodified lines
509511 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
510512 }
511513
512−#define SMMU_EVENT_ATTR(name, config) \
513− (&((struct perf_pmu_events_attr) { \
514− .attr = __ATTR(name, 0444, smmu_pmu_event_show, NULL), \
515− .id = config, \
516− }).attr.attr)
514+#define SMMU_EVENT_ATTR(name, config) \
515+ PMU_EVENT_ATTR_ID(name, smmu_pmu_event_show, config)
517516
518517 static struct attribute *smmu_pmu_events[] = {
519518 SMMU_EVENT_ATTR(cycles, 0),
@@ -628,7 +627,7 @@ static int smmu_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) · 108 unmodified lines
628627
629628 perf_pmu_migrate_context(&smmu_pmu->pmu, cpu, target);
630629 smmu_pmu->on_cpu = target;
631− WARN_ON(irq_set_affinity_hint(smmu_pmu->irq, cpumask_of(target)));
630+ WARN_ON(irq_set_affinity(smmu_pmu->irq, cpumask_of(target)));
632631
633632 return 0;
634633 }
@@ -839,15 +838,14 @@ static int smmu_pmu_probe(struct platform_device *pdev) · 204 unmodified lines
839838
840839 /* Pick one CPU to be the preferred one to use */
841840 smmu_pmu->on_cpu = raw_smp_processor_id();
842− WARN_ON(irq_set_affinity_hint(smmu_pmu->irq,
843− cpumask_of(smmu_pmu->on_cpu)));
841+ WARN_ON(irq_set_affinity(smmu_pmu->irq, cpumask_of(smmu_pmu->on_cpu)));
844842
845843 err = cpuhp_state_add_instance_nocalls(cpuhp_state_num,
846844 &smmu_pmu->node);
847845 if (err) {
848846 dev_err(dev, "Error %d registering hotplug, PMU @%pa\n",
849847 err, &res_0->start);
850− goto out_clear_affinity;
848+ return err;
851849 }
852850
853851 err = perf_pmu_register(&smmu_pmu->pmu, name, -1);
@@ -866,8 +864,6 @@ static int smmu_pmu_probe(struct platform_device *pdev) · 12 unmodified lines
866864
867865 out_unregister:
868866 cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node);
869−out_clear_affinity:
870− irq_set_affinity_hint(smmu_pmu->irq, NULL);
871867 return err;
872868 }
873869
@@ -877,7 +873,6 @@ static int smmu_pmu_remove(struct platform_device *pdev) · 3 unmodified lines
877873
878874 perf_pmu_unregister(&smmu_pmu->pmu);
879875 cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node);
880− irq_set_affinity_hint(smmu_pmu->irq, NULL);
881876
882877 return 0;
883878 }
drivers/perf/arm_spe_pmu.c+4−8
@@ -231,15 +231,14 @@ static const struct attribute_group arm_spe_pmu_format_group = { · 230 unmodified lines
231231 .attrs = arm_spe_pmu_formats_attr,
232232 };
233233
234−static ssize_t arm_spe_pmu_get_attr_cpumask(struct device *dev,
235− struct device_attribute *attr,
236− char *buf)
234+static ssize_t cpumask_show(struct device *dev,
235+ struct device_attribute *attr, char *buf)
237236 {
238237 struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev);
239238
240239 return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus);
241240 }
242−static DEVICE_ATTR(cpumask, S_IRUGO, arm_spe_pmu_get_attr_cpumask, NULL);
241+static DEVICE_ATTR_RO(cpumask);
243242
244243 static struct attribute *arm_spe_pmu_attrs[] = {
245244 &dev_attr_cpumask.attr,
@@ -1044,7 +1043,6 @@ static void __arm_spe_pmu_dev_probe(void *info) · 798 unmodified lines
10441043 spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features);
10451044
10461045 spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED;
1047− return;
10481046 }
10491047
10501048 static void __arm_spe_pmu_reset_local(void)
@@ -1190,10 +1188,8 @@ static int arm_spe_pmu_device_probe(struct platform_device *pdev) · 139 unmodified lines
11901188 }
11911189
11921190 spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL);
1193− if (!spe_pmu) {
1194− dev_err(dev, "failed to allocate spe_pmu\n");
1191+ if (!spe_pmu)
11951192 return -ENOMEM;
1196− }
11971193
11981194 spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle));
11991195 if (!spe_pmu->handle)
drivers/perf/fsl_imx8_ddr_perf.c+8−10
@@ -222,11 +222,8 @@ ddr_pmu_event_show(struct device *dev, struct device_attribute *attr, · 221 unmodified lines
222222 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
223223 }
224224
225−#define IMX8_DDR_PMU_EVENT_ATTR(_name, _id) \
226− (&((struct perf_pmu_events_attr[]) { \
227− { .attr = __ATTR(_name, 0444, ddr_pmu_event_show, NULL),\
228− .id = _id, } \
229− })[0].attr.attr)
225+#define IMX8_DDR_PMU_EVENT_ATTR(_name, _id) \
226+ PMU_EVENT_ATTR_ID(_name, ddr_pmu_event_show, _id)
230227
231228 static struct attribute *ddr_perf_events_attrs[] = {
232229 IMX8_DDR_PMU_EVENT_ATTR(cycles, EVENT_CYCLES_ID),
@@ -674,7 +671,7 @@ static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node) · 441 unmodified lines
674671 perf_pmu_migrate_context(&pmu->pmu, cpu, target);
675672 pmu->cpu = target;
676673
677− WARN_ON(irq_set_affinity_hint(pmu->irq, cpumask_of(pmu->cpu)));
674+ WARN_ON(irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)));
678675
679676 return 0;
680677 }
@@ -705,8 +702,10 @@ static int ddr_perf_probe(struct platform_device *pdev) · 24 unmodified lines
705702
706703 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d",
707704 num);
708− if (!name)
709− return -ENOMEM;
705+ if (!name) {
706+ ret = -ENOMEM;
707+ goto cpuhp_state_err;
708+ }
710709
711710 pmu->devtype_data = of_device_get_match_data(&pdev->dev);
712711
@@ -749,7 +748,7 @@ static int ddr_perf_probe(struct platform_device *pdev) · 36 unmodified lines
749748 }
750749
751750 pmu->irq = irq;
752− ret = irq_set_affinity_hint(pmu->irq, cpumask_of(pmu->cpu));
751+ ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu));
753752 if (ret) {
754753 dev_err(pmu->dev, "Failed to set interrupt affinity!\n");
755754 goto ddr_perf_err;
@@ -777,7 +776,6 @@ static int ddr_perf_remove(struct platform_device *pdev) · 21 unmodified lines
777776
778777 cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
779778 cpuhp_remove_multi_state(pmu->cpuhp_state);
780− irq_set_affinity_hint(pmu->irq, NULL);
781779
782780 perf_pmu_unregister(&pmu->pmu);
783781
drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c+1−4
@@ -2,7 +2,7 @@ · 1 unmodified line
22 /*
33 * HiSilicon SoC DDRC uncore Hardware event counters support
44 *
5− * Copyright (C) 2017 Hisilicon Limited
5+ * Copyright (C) 2017 HiSilicon Limited
66 * Author: Shaokun Zhang <zhangshaokun@hisilicon.com>
77 * Anurup M <anurup.m@huawei.com>
88 *
@@ -537,7 +537,6 @@ static int hisi_ddrc_pmu_probe(struct platform_device *pdev) · 528 unmodified lines
537537 dev_err(ddrc_pmu->dev, "DDRC PMU register failed!\n");
538538 cpuhp_state_remove_instance_nocalls(
539539 CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE, &ddrc_pmu->node);
540− irq_set_affinity_hint(ddrc_pmu->irq, NULL);
541540 }
542541
543542 return ret;
@@ -550,8 +549,6 @@ static int hisi_ddrc_pmu_remove(struct platform_device *pdev) · 6 unmodified lines
550549 perf_pmu_unregister(&ddrc_pmu->pmu);
551550 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE,
552551 &ddrc_pmu->node);
553− irq_set_affinity_hint(ddrc_pmu->irq, NULL);
554−
555552 return 0;
556553 }
557554
drivers/perf/hisilicon/hisi_uncore_hha_pmu.c+3−6
@@ -2,7 +2,7 @@ · 1 unmodified line
22 /*
33 * HiSilicon SoC HHA uncore Hardware event counters support
44 *
5− * Copyright (C) 2017 Hisilicon Limited
5+ * Copyright (C) 2017 HiSilicon Limited
66 * Author: Shaokun Zhang <zhangshaokun@hisilicon.com>
77 * Anurup M <anurup.m@huawei.com>
88 *
@@ -90,7 +90,7 @@ static void hisi_hha_pmu_config_ds(struct perf_event *event) · 81 unmodified lines
9090
9191 val = readl(hha_pmu->base + HHA_DATSRC_CTRL);
9292 val |= HHA_DATSRC_SKT_EN;
93− writel(ds_skt, hha_pmu->base + HHA_DATSRC_CTRL);
93+ writel(val, hha_pmu->base + HHA_DATSRC_CTRL);
9494 }
9595 }
9696
@@ -104,7 +104,7 @@ static void hisi_hha_pmu_clear_ds(struct perf_event *event) · 7 unmodified lines
104104
105105 val = readl(hha_pmu->base + HHA_DATSRC_CTRL);
106106 val &= ~HHA_DATSRC_SKT_EN;
107− writel(ds_skt, hha_pmu->base + HHA_DATSRC_CTRL);
107+ writel(val, hha_pmu->base + HHA_DATSRC_CTRL);
108108 }
109109 }
110110
@@ -540,7 +540,6 @@ static int hisi_hha_pmu_probe(struct platform_device *pdev) · 429 unmodified lines
540540 dev_err(hha_pmu->dev, "HHA PMU register failed!\n");
541541 cpuhp_state_remove_instance_nocalls(
542542 CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE, &hha_pmu->node);
543− irq_set_affinity_hint(hha_pmu->irq, NULL);
544543 }
545544
546545 return ret;
@@ -553,8 +552,6 @@ static int hisi_hha_pmu_remove(struct platform_device *pdev) · 6 unmodified lines
553552 perf_pmu_unregister(&hha_pmu->pmu);
554553 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE,
555554 &hha_pmu->node);
556− irq_set_affinity_hint(hha_pmu->irq, NULL);
557−
558555 return 0;
559556 }
560557
drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c+1−4
@@ -2,7 +2,7 @@ · 1 unmodified line
22 /*
33 * HiSilicon SoC L3C uncore Hardware event counters support
44 *
5− * Copyright (C) 2017 Hisilicon Limited
5+ * Copyright (C) 2017 HiSilicon Limited
66 * Author: Anurup M <anurup.m@huawei.com>
77 * Shaokun Zhang <zhangshaokun@hisilicon.com>
88 *
@@ -578,7 +578,6 @@ static int hisi_l3c_pmu_probe(struct platform_device *pdev) · 569 unmodified lines
578578 dev_err(l3c_pmu->dev, "L3C PMU register failed!\n");
579579 cpuhp_state_remove_instance_nocalls(
580580 CPUHP_AP_PERF_ARM_HISI_L3_ONLINE, &l3c_pmu->node);
581− irq_set_affinity_hint(l3c_pmu->irq, NULL);
582581 }
583582
584583 return ret;
@@ -591,8 +590,6 @@ static int hisi_l3c_pmu_remove(struct platform_device *pdev) · 6 unmodified lines
591590 perf_pmu_unregister(&l3c_pmu->pmu);
592591 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE,
593592 &l3c_pmu->node);
594− irq_set_affinity_hint(l3c_pmu->irq, NULL);
595−
596593 return 0;
597594 }
598595
drivers/perf/hisilicon/hisi_uncore_pa_pmu.c+1−4
@@ -333,7 +333,7 @@ static struct attribute *hisi_pa_pmu_identifier_attrs[] = { · 332 unmodified lines
333333 NULL
334334 };
335335
336−static struct attribute_group hisi_pa_pmu_identifier_group = {
336+static const struct attribute_group hisi_pa_pmu_identifier_group = {
337337 .attrs = hisi_pa_pmu_identifier_attrs,
338338 };
339339
@@ -436,7 +436,6 @@ static int hisi_pa_pmu_probe(struct platform_device *pdev) · 96 unmodified lines
436436 dev_err(pa_pmu->dev, "PMU register failed, ret = %d\n", ret);
437437 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_PA_ONLINE,
438438 &pa_pmu->node);
439− irq_set_affinity_hint(pa_pmu->irq, NULL);
440439 return ret;
441440 }
442441
@@ -451,8 +450,6 @@ static int hisi_pa_pmu_remove(struct platform_device *pdev) · 8 unmodified lines
451450 perf_pmu_unregister(&pa_pmu->pmu);
452451 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_PA_ONLINE,
453452 &pa_pmu->node);
454− irq_set_affinity_hint(pa_pmu->irq, NULL);
455−
456453 return 0;
457454 }
458455
drivers/perf/hisilicon/hisi_uncore_pmu.c+3−3
@@ -2,7 +2,7 @@ · 1 unmodified line
22 /*
33 * HiSilicon SoC Hardware event counters support
44 *
5− * Copyright (C) 2017 Hisilicon Limited
5+ * Copyright (C) 2017 HiSilicon Limited
66 * Author: Anurup M <anurup.m@huawei.com>
77 * Shaokun Zhang <zhangshaokun@hisilicon.com>
88 *
@@ -488,7 +488,7 @@ int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) · 479 unmodified lines
488488 hisi_pmu->on_cpu = cpu;
489489
490490 /* Overflow interrupt also should use the same CPU */
491− WARN_ON(irq_set_affinity_hint(hisi_pmu->irq, cpumask_of(cpu)));
491+ WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu)));
492492
493493 return 0;
494494 }
@@ -521,7 +521,7 @@ int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) · 26 unmodified lines
521521 perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target);
522522 /* Use this CPU for event counting */
523523 hisi_pmu->on_cpu = target;
524− WARN_ON(irq_set_affinity_hint(hisi_pmu->irq, cpumask_of(target)));
524+ WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target)));
525525
526526 return 0;
527527 }
drivers/perf/hisilicon/hisi_uncore_pmu.h+1−1
@@ -2,7 +2,7 @@ · 1 unmodified line
22 /*
33 * HiSilicon SoC Hardware event counters support
44 *
5− * Copyright (C) 2017 Hisilicon Limited
5+ * Copyright (C) 2017 HiSilicon Limited
66 * Author: Anurup M <anurup.m@huawei.com>
77 * Shaokun Zhang <zhangshaokun@hisilicon.com>
88 *
drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c+2−5
@@ -2,7 +2,7 @@ · 1 unmodified line
22 /*
33 * HiSilicon SLLC uncore Hardware event counters support
44 *
5− * Copyright (C) 2020 Hisilicon Limited
5+ * Copyright (C) 2020 HiSilicon Limited
66 * Author: Shaokun Zhang <zhangshaokun@hisilicon.com>
77 *
88 * This code is based on the uncore PMUs like arm-cci and arm-ccn.
@@ -366,7 +366,7 @@ static struct attribute *hisi_sllc_pmu_identifier_attrs[] = { · 357 unmodified lines
366366 NULL
367367 };
368368
369−static struct attribute_group hisi_sllc_pmu_identifier_group = {
369+static const struct attribute_group hisi_sllc_pmu_identifier_group = {
370370 .attrs = hisi_sllc_pmu_identifier_attrs,
371371 };
372372
@@ -465,7 +465,6 @@ static int hisi_sllc_pmu_probe(struct platform_device *pdev) · 92 unmodified lines
465465 dev_err(sllc_pmu->dev, "PMU register failed, ret = %d\n", ret);
466466 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE,
467467 &sllc_pmu->node);
468− irq_set_affinity_hint(sllc_pmu->irq, NULL);
469468 return ret;
470469 }
471470
@@ -481,8 +480,6 @@ static int hisi_sllc_pmu_remove(struct platform_device *pdev) · 9 unmodified lines
481480 perf_pmu_unregister(&sllc_pmu->pmu);
482481 cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE,
483482 &sllc_pmu->node);
484− irq_set_affinity_hint(sllc_pmu->irq, NULL);
485−
486483 return 0;
487484 }
488485
drivers/perf/qcom_l2_pmu.c+4−7
@@ -679,11 +679,8 @@ static ssize_t l2cache_pmu_event_show(struct device *dev, · 678 unmodified lines
679679 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
680680 }
681681
682−#define L2CACHE_EVENT_ATTR(_name, _id) \
683− (&((struct perf_pmu_events_attr[]) { \
684− { .attr = __ATTR(_name, 0444, l2cache_pmu_event_show, NULL), \
685− .id = _id, } \
686− })[0].attr.attr)
682+#define L2CACHE_EVENT_ATTR(_name, _id) \
683+ PMU_EVENT_ATTR_ID(_name, l2cache_pmu_event_show, _id)
687684
688685 static struct attribute *l2_cache_pmu_events[] = {
689686 L2CACHE_EVENT_ATTR(cycles, L2_EVENT_CYCLES),
@@ -869,14 +866,14 @@ static int l2_cache_pmu_probe_cluster(struct device *dev, void *data) · 179 unmodified lines
869866 irq = platform_get_irq(sdev, 0);
870867 if (irq < 0)
871868 return irq;
872− irq_set_status_flags(irq, IRQ_NOAUTOEN);
873869 cluster->irq = irq;
874870
875871 cluster->l2cache_pmu = l2cache_pmu;
876872 cluster->on_cpu = -1;
877873
878874 err = devm_request_irq(&pdev->dev, irq, l2_cache_handle_irq,
879− IRQF_NOBALANCING | IRQF_NO_THREAD,
875+ IRQF_NOBALANCING | IRQF_NO_THREAD |
876+ IRQF_NO_AUTOEN,
880877 "l2-cache-pmu", cluster);
881878 if (err) {
882879 dev_err(&pdev->dev,
drivers/perf/qcom_l3_pmu.c+5−10
@@ -647,10 +647,7 @@ static ssize_t l3cache_pmu_event_show(struct device *dev, · 646 unmodified lines
647647 }
648648
649649 #define L3CACHE_EVENT_ATTR(_name, _id) \
650− (&((struct perf_pmu_events_attr[]) { \
651− { .attr = __ATTR(_name, 0444, l3cache_pmu_event_show, NULL), \
652− .id = _id, } \
653− })[0].attr.attr)
650+ PMU_EVENT_ATTR_ID(_name, l3cache_pmu_event_show, _id)
654651
655652 static struct attribute *qcom_l3_cache_pmu_events[] = {
656653 L3CACHE_EVENT_ATTR(cycles, L3_EVENT_CYCLES),
@@ -670,15 +667,15 @@ static const struct attribute_group qcom_l3_cache_pmu_events_group = { · 13 unmodified lines
670667
671668 /* cpumask */
672669
673−static ssize_t qcom_l3_cache_pmu_cpumask_show(struct device *dev,
674− struct device_attribute *attr, char *buf)
670+static ssize_t cpumask_show(struct device *dev,
671+ struct device_attribute *attr, char *buf)
675672 {
676673 struct l3cache_pmu *l3pmu = to_l3cache_pmu(dev_get_drvdata(dev));
677674
678675 return cpumap_print_to_pagebuf(true, buf, &l3pmu->cpumask);
679676 }
680677
681−static DEVICE_ATTR(cpumask, 0444, qcom_l3_cache_pmu_cpumask_show, NULL);
678+static DEVICE_ATTR_RO(cpumask);
682679
683680 static struct attribute *qcom_l3_cache_pmu_cpumask_attrs[] = {
684681 &dev_attr_cpumask.attr,
@@ -767,10 +764,8 @@ static int qcom_l3_cache_pmu_probe(struct platform_device *pdev) · 82 unmodified lines
767764
768765 memrc = platform_get_resource(pdev, IORESOURCE_MEM, 0);
769766 l3pmu->regs = devm_ioremap_resource(&pdev->dev, memrc);
770− if (IS_ERR(l3pmu->regs)) {
771− dev_err(&pdev->dev, "Can't map PMU @%pa\n", &memrc->start);
767+ if (IS_ERR(l3pmu->regs))
772768 return PTR_ERR(l3pmu->regs);
773− }
774769
775770 qcom_l3_cache__init(l3pmu);
776771
drivers/perf/thunderx2_pmu.c+1−3
@@ -817,10 +817,8 @@ static struct tx2_uncore_pmu *tx2_uncore_pmu_init_dev(struct device *dev, · 816 unmodified lines
817817 }
818818
819819 base = devm_ioremap_resource(dev, &res);
820− if (IS_ERR(base)) {
821− dev_err(dev, "PMU type %d: Fail to map resource\n", type);
820+ if (IS_ERR(base))
822821 return NULL;
823− }
824822
825823 tx2_pmu = devm_kzalloc(dev, sizeof(*tx2_pmu), GFP_KERNEL);
826824 if (!tx2_pmu)
drivers/perf/xgene_pmu.c+7−10
@@ -278,17 +278,14 @@ static const struct attribute_group mc_pmu_v3_format_attr_group = { · 277 unmodified lines
278278 static ssize_t xgene_pmu_event_show(struct device *dev,
279279 struct device_attribute *attr, char *buf)
280280 {
281− struct dev_ext_attribute *eattr;
281+ struct perf_pmu_events_attr *pmu_attr =
282+ container_of(attr, struct perf_pmu_events_attr, attr);
282283
283− eattr = container_of(attr, struct dev_ext_attribute, attr);
284− return sysfs_emit(buf, "config=0x%lx\n", (unsigned long) eattr->var);
284+ return sysfs_emit(buf, "config=0x%llx\n", pmu_attr->id);
285285 }
286286
287287 #define XGENE_PMU_EVENT_ATTR(_name, _config) \
288− (&((struct dev_ext_attribute[]) { \
289− { .attr = __ATTR(_name, S_IRUGO, xgene_pmu_event_show, NULL), \
290− .var = (void *) _config, } \
291− })[0].attr.attr)
288+ PMU_EVENT_ATTR_ID(_name, xgene_pmu_event_show, _config)
292289
293290 static struct attribute *l3c_pmu_events_attrs[] = {
294291 XGENE_PMU_EVENT_ATTR(cycle-count, 0x00),
@@ -604,15 +601,15 @@ static const struct attribute_group mc_pmu_v3_events_attr_group = { · 309 unmodified lines
604601 /*
605602 * sysfs cpumask attributes
606603 */
607−static ssize_t xgene_pmu_cpumask_show(struct device *dev,
608− struct device_attribute *attr, char *buf)
604+static ssize_t cpumask_show(struct device *dev,
605+ struct device_attribute *attr, char *buf)
609606 {
610607 struct xgene_pmu_dev *pmu_dev = to_pmu_dev(dev_get_drvdata(dev));
611608
612609 return cpumap_print_to_pagebuf(true, buf, &pmu_dev->parent->cpu);
613610 }
614611
615−static DEVICE_ATTR(cpumask, S_IRUGO, xgene_pmu_cpumask_show, NULL);
612+static DEVICE_ATTR_RO(cpumask);
616613
617614 static struct attribute *xgene_pmu_cpumask_attrs[] = {
618615 &dev_attr_cpumask.attr,
include/linux/arm-smccc.h+86−2
@@ -63,6 +63,9 @@ · 62 unmodified lines
6363 #define ARM_SMCCC_VERSION_1_0 0x10000
6464 #define ARM_SMCCC_VERSION_1_1 0x10001
6565 #define ARM_SMCCC_VERSION_1_2 0x10002
66+#define ARM_SMCCC_VERSION_1_3 0x10003
67+
68+#define ARM_SMCCC_1_3_SVE_HINT 0x10000
6669
6770 #define ARM_SMCCC_VERSION_FUNC_ID \
6871 ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \
@@ -216,6 +219,8 @@ u32 arm_smccc_get_version(void); · 147 unmodified lines
216219
217220 void __init arm_smccc_version_init(u32 version, enum arm_smccc_conduit conduit);
218221
222+extern u64 smccc_has_sve_hint;
223+
219224 /**
220225 * struct arm_smccc_res - Result from SMC/HVC call
221226 * @a0-a3 result values from registers 0 to 3
@@ -227,6 +232,61 @@ struct arm_smccc_res { · 5 unmodified lines
227232 unsigned long a3;
228233 };
229234
235+#ifdef CONFIG_ARM64
236+/**
237+ * struct arm_smccc_1_2_regs - Arguments for or Results from SMC/HVC call
238+ * @a0-a17 argument values from registers 0 to 17
239+ */
240+struct arm_smccc_1_2_regs {
241+ unsigned long a0;
242+ unsigned long a1;
243+ unsigned long a2;
244+ unsigned long a3;
245+ unsigned long a4;
246+ unsigned long a5;
247+ unsigned long a6;
248+ unsigned long a7;
249+ unsigned long a8;
250+ unsigned long a9;
251+ unsigned long a10;
252+ unsigned long a11;
253+ unsigned long a12;
254+ unsigned long a13;
255+ unsigned long a14;
256+ unsigned long a15;
257+ unsigned long a16;
258+ unsigned long a17;
259+};
260+
261+/**
262+ * arm_smccc_1_2_hvc() - make HVC calls
263+ * @args: arguments passed via struct arm_smccc_1_2_regs
264+ * @res: result values via struct arm_smccc_1_2_regs
265+ *
266+ * This function is used to make HVC calls following SMC Calling Convention
267+ * v1.2 or above. The content of the supplied param are copied from the
268+ * structure to registers prior to the HVC instruction. The return values
269+ * are updated with the content from registers on return from the HVC
270+ * instruction.
271+ */
272+asmlinkage void arm_smccc_1_2_hvc(const struct arm_smccc_1_2_regs *args,
273+ struct arm_smccc_1_2_regs *res);
274+
275+/**
276+ * arm_smccc_1_2_smc() - make SMC calls
277+ * @args: arguments passed via struct arm_smccc_1_2_regs
278+ * @res: result values via struct arm_smccc_1_2_regs
279+ *
280+ * This function is used to make SMC calls following SMC Calling Convention
281+ * v1.2 or above. The content of the supplied param are copied from the
282+ * structure to registers prior to the SMC instruction. The return values
283+ * are updated with the content from registers on return from the SMC
284+ * instruction.
285+ */
286+asmlinkage void arm_smccc_1_2_smc(const struct arm_smccc_1_2_regs *args,
287+ struct arm_smccc_1_2_regs *res);
288+#endif
289+
230290 /**
231291 * struct arm_smccc_quirk - Contains quirk information
232292 * @id: quirk identification
@@ -240,6 +300,15 @@ struct arm_smccc_quirk { · 7 unmodified lines
240300 } state;
241301 };
242302
303+/**
304+ * __arm_smccc_sve_check() - Set the SVE hint bit when doing SMC calls
305+ *
306+ * Sets the SMCCC hint bit to indicate if there is live state in the SVE
307+ * registers, this modifies x0 in place and should never be called from C
308+ * code.
309+ */
310+asmlinkage unsigned long __arm_smccc_sve_check(unsigned long x0);
311+
243312 /**
244313 * __arm_smccc_smc() - make SMC calls
245314 * @a0-a7: arguments passed in registers 0 to 7
@@ -297,6 +366,20 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, · 51 unmodified lines
297366
298367 #endif
299368
369+/* nVHE hypervisor doesn't have a current thread so needs separate checks */
370+#if defined(CONFIG_ARM64_SVE) && !defined(__KVM_NVHE_HYPERVISOR__)
371+
372+#define SMCCC_SVE_CHECK ALTERNATIVE("nop \n", "bl __arm_smccc_sve_check \n", \
373+ ARM64_SVE)
374+#define smccc_sve_clobbers "x16", "x30", "cc",
375+
376+#else
377+
378+#define SMCCC_SVE_CHECK
379+#define smccc_sve_clobbers
380+
381+#endif
382+
300383 #define ___count_args(_0, _1, _2, _3, _4, _5, _6, _7, _8, x, ...) x
301384
302385 #define __count_args(...) \
@@ -364,7 +447,7 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, · 61 unmodified lines
364447
365448 #define ___constraints(count) \
366449 : __constraint_read_ ## count \
367− : "memory"
450+ : smccc_sve_clobbers "memory"
368451 #define __constraints(count) ___constraints(count)
369452
370453 /*
@@ -379,7 +462,8 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, · 8 unmodified lines
379462 register unsigned long r2 asm("r2"); \
380463 register unsigned long r3 asm("r3"); \
381464 __declare_args(__count_args(__VA_ARGS__), __VA_ARGS__); \
382− asm volatile(inst "\n" : \
465+ asm volatile(SMCCC_SVE_CHECK \
466+ inst "\n" : \
383467 "=r" (r0), "=r" (r1), "=r" (r2), "=r" (r3) \
384468 __constraints(__count_args(__VA_ARGS__))); \
385469 if (___res) \
include/linux/gfp.h+15−3
@@ -53,8 +53,10 @@ struct vm_area_struct; · 52 unmodified lines
5353 #define ___GFP_HARDWALL 0x100000u
5454 #define ___GFP_THISNODE 0x200000u
5555 #define ___GFP_ACCOUNT 0x400000u
56+#define ___GFP_ZEROTAGS 0x800000u
57+#define ___GFP_SKIP_KASAN_POISON 0x1000000u
5658 #ifdef CONFIG_LOCKDEP
57−#define ___GFP_NOLOCKDEP 0x800000u
59+#define ___GFP_NOLOCKDEP 0x2000000u
5860 #else
5961 #define ___GFP_NOLOCKDEP 0
6062 #endif
@@ -229,16 +231,25 @@ struct vm_area_struct; · 168 unmodified lines
229231 * %__GFP_COMP address compound page metadata.
230232 *
231233 * %__GFP_ZERO returns a zeroed page on success.
234+ *
235+ * %__GFP_ZEROTAGS returns a page with zeroed memory tags on success, if
236+ * __GFP_ZERO is set.
237+ *
238+ * %__GFP_SKIP_KASAN_POISON returns a page which does not need to be poisoned
239+ * on deallocation. Typically used for userspace pages. Currently only has an
240+ * effect in HW tags mode.
232241 */
233242 #define __GFP_NOWARN ((__force gfp_t)___GFP_NOWARN)
234243 #define __GFP_COMP ((__force gfp_t)___GFP_COMP)
235244 #define __GFP_ZERO ((__force gfp_t)___GFP_ZERO)
245+#define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS)
246+#define __GFP_SKIP_KASAN_POISON ((__force gfp_t)___GFP_SKIP_KASAN_POISON)
236247
237248 /* Disable lockdep for GFP context tracking */
238249 #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP)
239250
240251 /* Room for N __GFP_FOO bits */
241−#define __GFP_BITS_SHIFT (23 + IS_ENABLED(CONFIG_LOCKDEP))
252+#define __GFP_BITS_SHIFT (25 + IS_ENABLED(CONFIG_LOCKDEP))
242253 #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
243254
244255 /**
@@ -319,7 +330,8 @@ struct vm_area_struct; · 74 unmodified lines
319330 #define GFP_DMA __GFP_DMA
320331 #define GFP_DMA32 __GFP_DMA32
321332 #define GFP_HIGHUSER (GFP_USER | __GFP_HIGHMEM)
322−#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE)
333+#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE | \
334+ __GFP_SKIP_KASAN_POISON)
323335 #define GFP_TRANSHUGE_LIGHT ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \
324336 __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM)
325337 #define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM)
include/linux/highmem.h+16−27
@@ -152,28 +152,24 @@ static inline void clear_user_highpage(struct page *page, unsigned long vaddr) · 151 unmodified lines
152152 }
153153 #endif
154154
155−#ifndef __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
155+#ifndef __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE
156156 /**
157− * __alloc_zeroed_user_highpage - Allocate a zeroed HIGHMEM page for a VMA with caller-specified movable GFP flags
158− * @movableflags: The GFP flags related to the pages future ability to move like __GFP_MOVABLE
157+ * alloc_zeroed_user_highpage_movable - Allocate a zeroed HIGHMEM page for a VMA that the caller knows can move
159158 * @vma: The VMA the page is to be allocated for
160159 * @vaddr: The virtual address the page will be inserted into
161160 *
162− * This function will allocate a page for a VMA but the caller is expected
163− * to specify via movableflags whether the page will be movable in the
164− * future or not
161+ * This function will allocate a page for a VMA that the caller knows will
162+ * be able to migrate in the future using move_pages() or reclaimed
165163 *
166164 * An architecture may override this function by defining
167− * __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE and providing their own
165+ * __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE and providing their own
168166 * implementation.
169167 */
170168 static inline struct page *
171−__alloc_zeroed_user_highpage(gfp_t movableflags,
172− struct vm_area_struct *vma,
173− unsigned long vaddr)
169+alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma,
170+ unsigned long vaddr)
174171 {
175− struct page *page = alloc_page_vma(GFP_HIGHUSER | movableflags,
176− vma, vaddr);
172+ struct page *page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr);
177173
178174 if (page)
179175 clear_user_highpage(page, vaddr);
@@ -182,21 +178,6 @@ __alloc_zeroed_user_highpage(gfp_t movableflags, · 2 unmodified lines
182178 }
183179 #endif
184180
185−/**
186− * alloc_zeroed_user_highpage_movable - Allocate a zeroed HIGHMEM page for a VMA that the caller knows can move
187− * @vma: The VMA the page is to be allocated for
188− * @vaddr: The virtual address the page will be inserted into
189− *
190− * This function will allocate a page for a VMA that the caller knows will
191− * be able to migrate in the future using move_pages() or reclaimed
192− */
193−static inline struct page *
194−alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma,
195− unsigned long vaddr)
196−{
197− return __alloc_zeroed_user_highpage(__GFP_MOVABLE, vma, vaddr);
198−}
199−
200181 static inline void clear_highpage(struct page *page)
201182 {
202183 void *kaddr = kmap_atomic(page);
@@ -204,6 +185,14 @@ static inline void clear_highpage(struct page *page) · 1 unmodified line
204185 kunmap_atomic(kaddr);
205186 }
206187
188+#ifndef __HAVE_ARCH_TAG_CLEAR_HIGHPAGE
189+
190+static inline void tag_clear_highpage(struct page *page)
191+{
192+}
193+
194+#endif
195+
207196 /*
208197 * If we pass in a base or tail page, we can zero up to PAGE_SIZE.
209198 * If we pass in a head page, we can zero up to the size of the compound page.
include/linux/interrupt.h+2−33
@@ -319,39 +319,8 @@ struct irq_affinity_desc { · 318 unmodified lines
319319
320320 extern cpumask_var_t irq_default_affinity;
321321
322−/* Internal implementation. Use the helpers below */
323−extern int __irq_set_affinity(unsigned int irq, const struct cpumask *cpumask,
324− bool force);
325−
326−/**
327− * irq_set_affinity - Set the irq affinity of a given irq
328− * @irq: Interrupt to set affinity
329− * @cpumask: cpumask
330− *
331− * Fails if cpumask does not contain an online CPU
332− */
333−static inline int
334−irq_set_affinity(unsigned int irq, const struct cpumask *cpumask)
335−{
336− return __irq_set_affinity(irq, cpumask, false);
337−}
338−
339−/**
340− * irq_force_affinity - Force the irq affinity of a given irq
341− * @irq: Interrupt to set affinity
342− * @cpumask: cpumask
343− *
344− * Same as irq_set_affinity, but without checking the mask against
345− * online cpus.
346− *
347− * Solely for low level cpu hotplug code, where we need to make per
348− * cpu interrupts affine before the cpu becomes online.
349− */
350−static inline int
351−irq_force_affinity(unsigned int irq, const struct cpumask *cpumask)
352−{
353− return __irq_set_affinity(irq, cpumask, true);
354−}
322+extern int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask);
323+extern int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask);
355324
356325 extern int irq_can_set_affinity(unsigned int irq);
357326 extern int irq_select_affinity(unsigned int irq);
include/linux/kasan.h+38−26
@@ -2,6 +2,7 @@ · 1 unmodified line
22 #ifndef _LINUX_KASAN_H
33 #define _LINUX_KASAN_H
44
5+#include <linux/bug.h>
56 #include <linux/static_key.h>
67 #include <linux/types.h>
78
@@ -79,14 +80,6 @@ static inline void kasan_disable_current(void) {} · 71 unmodified lines
7980
8081 #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
8182
82−#ifdef CONFIG_KASAN
83−
84−struct kasan_cache {
85− int alloc_meta_offset;
86− int free_meta_offset;
87− bool is_kmalloc;
88−};
89−
9083 #ifdef CONFIG_KASAN_HW_TAGS
9184
9285 DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled);
@@ -101,11 +94,14 @@ static inline bool kasan_has_integrated_init(void) · 8 unmodified lines
10194 return kasan_enabled();
10295 }
10396
97+void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags);
98+void kasan_free_pages(struct page *page, unsigned int order);
99+
104100 #else /* CONFIG_KASAN_HW_TAGS */
105101
106102 static inline bool kasan_enabled(void)
107103 {
108− return true;
104+ return IS_ENABLED(CONFIG_KASAN);
109105 }
110106
111107 static inline bool kasan_has_integrated_init(void)
@@ -113,8 +109,30 @@ static inline bool kasan_has_integrated_init(void) · 1 unmodified line
113109 return false;
114110 }
115111
112+static __always_inline void kasan_alloc_pages(struct page *page,
113+ unsigned int order, gfp_t flags)
114+{
115+ /* Only available for integrated init. */
116+ BUILD_BUG();
117+}
118+
119+static __always_inline void kasan_free_pages(struct page *page,
120+ unsigned int order)
121+{
122+ /* Only available for integrated init. */
123+ BUILD_BUG();
124+}
125+
116126 #endif /* CONFIG_KASAN_HW_TAGS */
117127
128+#ifdef CONFIG_KASAN
129+
130+struct kasan_cache {
131+ int alloc_meta_offset;
132+ int free_meta_offset;
133+ bool is_kmalloc;
134+};
135+
118136 slab_flags_t __kasan_never_merge(void);
119137 static __always_inline slab_flags_t kasan_never_merge(void)
120138 {
@@ -130,20 +148,20 @@ static __always_inline void kasan_unpoison_range(const void *addr, size_t size) · 9 unmodified lines
130148 __kasan_unpoison_range(addr, size);
131149 }
132150
133−void __kasan_alloc_pages(struct page *page, unsigned int order, bool init);
134−static __always_inline void kasan_alloc_pages(struct page *page,
151+void __kasan_poison_pages(struct page *page, unsigned int order, bool init);
152+static __always_inline void kasan_poison_pages(struct page *page,
135153 unsigned int order, bool init)
136154 {
137155 if (kasan_enabled())
138− __kasan_alloc_pages(page, order, init);
156+ __kasan_poison_pages(page, order, init);
139157 }
140158
141−void __kasan_free_pages(struct page *page, unsigned int order, bool init);
142−static __always_inline void kasan_free_pages(struct page *page,
143− unsigned int order, bool init)
159+void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init);
160+static __always_inline void kasan_unpoison_pages(struct page *page,
161+ unsigned int order, bool init)
144162 {
145163 if (kasan_enabled())
146− __kasan_free_pages(page, order, init);
164+ __kasan_unpoison_pages(page, order, init);
147165 }
148166
149167 void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
@@ -285,21 +303,15 @@ void kasan_restore_multi_shot(bool enabled); · 135 unmodified lines
285303
286304 #else /* CONFIG_KASAN */
287305
288−static inline bool kasan_enabled(void)
289−{
290− return false;
291−}
292−static inline bool kasan_has_integrated_init(void)
293−{
294− return false;
295−}
296306 static inline slab_flags_t kasan_never_merge(void)
297307 {
298308 return 0;
299309 }
300310 static inline void kasan_unpoison_range(const void *address, size_t size) {}
301−static inline void kasan_alloc_pages(struct page *page, unsigned int order, bool init) {}
302−static inline void kasan_free_pages(struct page *page, unsigned int order, bool init) {}
311+static inline void kasan_poison_pages(struct page *page, unsigned int order,
312+ bool init) {}
313+static inline void kasan_unpoison_pages(struct page *page, unsigned int order,
314+ bool init) {}
303315 static inline void kasan_cache_create(struct kmem_cache *cache,
304316 unsigned int *size,
305317 slab_flags_t *flags) {}
include/linux/page-flags.h+9−0
@@ -137,6 +137,9 @@ enum pageflags { · 136 unmodified lines
137137 #endif
138138 #ifdef CONFIG_64BIT
139139 PG_arch_2,
140+#endif
141+#ifdef CONFIG_KASAN_HW_TAGS
142+ PG_skip_kasan_poison,
140143 #endif
141144 __NR_PAGEFLAGS,
142145
@@ -443,6 +446,12 @@ TESTCLEARFLAG(Young, young, PF_ANY) · 300 unmodified lines
443446 PAGEFLAG(Idle, idle, PF_ANY)
444447 #endif
445448
449+#ifdef CONFIG_KASAN_HW_TAGS
450+PAGEFLAG(SkipKASanPoison, skip_kasan_poison, PF_HEAD)
451+#else
452+PAGEFLAG_FALSE(SkipKASanPoison)
453+#endif
454+
446455 /*
447456 * PageReported() is used to track reported free pages within the Buddy
448457 * allocator. We can use the non-atomic version of the test and set
include/linux/perf_event.h+6−0
@@ -1576,6 +1576,12 @@ static struct perf_pmu_events_attr _var = { \ · 1575 unmodified lines
15761576 .event_str = _str, \
15771577 };
15781578
1579+#define PMU_EVENT_ATTR_ID(_name, _show, _id) \
1580+ (&((struct perf_pmu_events_attr[]) { \
1581+ { .attr = __ATTR(_name, 0444, _show, NULL), \
1582+ .id = _id, } \
1583+ })[0].attr.attr)
1584+
15791585 #define PMU_FORMAT_ATTR(_name, _format) \
15801586 static ssize_t \
15811587 _name##_show(struct device *dev, \
include/trace/events/mmflags.h+8−1
@@ -85,6 +85,12 @@ · 84 unmodified lines
8585 #define IF_HAVE_PG_ARCH_2(flag,string)
8686 #endif
8787
88+#ifdef CONFIG_KASAN_HW_TAGS
89+#define IF_HAVE_PG_SKIP_KASAN_POISON(flag,string) ,{1UL << flag, string}
90+#else
91+#define IF_HAVE_PG_SKIP_KASAN_POISON(flag,string)
92+#endif
93+
8894 #define __def_pageflag_names \
8995 {1UL << PG_locked, "locked" }, \
9096 {1UL << PG_waiters, "waiters" }, \
@@ -112,7 +118,8 @@ IF_HAVE_PG_UNCACHED(PG_uncached, "uncached" ) \ · 21 unmodified lines
112118 IF_HAVE_PG_HWPOISON(PG_hwpoison, "hwpoison" ) \
113119 IF_HAVE_PG_IDLE(PG_young, "young" ) \
114120 IF_HAVE_PG_IDLE(PG_idle, "idle" ) \
115−IF_HAVE_PG_ARCH_2(PG_arch_2, "arch_2" )
121+IF_HAVE_PG_ARCH_2(PG_arch_2, "arch_2" ) \
122+IF_HAVE_PG_SKIP_KASAN_POISON(PG_skip_kasan_poison, "skip_kasan_poison")
116123
117124 #define show_page_flags(flags) \
118125 (flags) ? __print_flags(flags, "|", \
kernel/irq/manage.c+32−1
@@ -441,7 +441,8 @@ out_unlock: · 440 unmodified lines
441441 return ret;
442442 }
443443
444−int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force)
444+static int __irq_set_affinity(unsigned int irq, const struct cpumask *mask,
445+ bool force)
445446 {
446447 struct irq_desc *desc = irq_to_desc(irq);
447448 unsigned long flags;
@@ -456,6 +457,36 @@ int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force) · 8 unmodified lines
456457 return ret;
457458 }
458459
460+/**
461+ * irq_set_affinity - Set the irq affinity of a given irq
462+ * @irq: Interrupt to set affinity
463+ * @cpumask: cpumask
464+ *
465+ * Fails if cpumask does not contain an online CPU
466+ */
467+int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask)
468+{
469+ return __irq_set_affinity(irq, cpumask, false);
470+}
471+EXPORT_SYMBOL_GPL(irq_set_affinity);
472+
473+/**
474+ * irq_force_affinity - Force the irq affinity of a given irq
475+ * @irq: Interrupt to set affinity
476+ * @cpumask: cpumask
477+ *
478+ * Same as irq_set_affinity, but without checking the mask against
479+ * online cpus.
480+ *
481+ * Solely for low level cpu hotplug code, where we need to make per
482+ * cpu interrupts affine before the cpu becomes online.
483+ */
484+int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask)
485+{
486+ return __irq_set_affinity(irq, cpumask, true);
487+}
488+EXPORT_SYMBOL_GPL(irq_force_affinity);
489+
459490 int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m)
460491 {
461492 unsigned long flags;
mm/kasan/common.c+2−2
@@ -97,7 +97,7 @@ slab_flags_t __kasan_never_merge(void) · 96 unmodified lines
9797 return 0;
9898 }
9999
100−void __kasan_alloc_pages(struct page *page, unsigned int order, bool init)
100+void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init)
101101 {
102102 u8 tag;
103103 unsigned long i;
@@ -111,7 +111,7 @@ void __kasan_alloc_pages(struct page *page, unsigned int order, bool init) · 7 unmodified lines
111111 kasan_unpoison(page_address(page), PAGE_SIZE << order, init);
112112 }
113113
114−void __kasan_free_pages(struct page *page, unsigned int order, bool init)
114+void __kasan_poison_pages(struct page *page, unsigned int order, bool init)
115115 {
116116 if (likely(!PageHighMem(page)))
117117 kasan_poison(page_address(page), PAGE_SIZE << order,
mm/kasan/hw_tags.c+32−0
@@ -238,6 +238,38 @@ struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, · 237 unmodified lines
238238 return &alloc_meta->free_track[0];
239239 }
240240
241+void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags)
242+{
243+ /*
244+ * This condition should match the one in post_alloc_hook() in
245+ * page_alloc.c.
246+ */
247+ bool init = !want_init_on_free() && want_init_on_alloc(flags);
248+
249+ if (flags & __GFP_SKIP_KASAN_POISON)
250+ SetPageSkipKASanPoison(page);
251+
252+ if (flags & __GFP_ZEROTAGS) {
253+ int i;
254+
255+ for (i = 0; i != 1 << order; ++i)
256+ tag_clear_highpage(page + i);
257+ } else {
258+ kasan_unpoison_pages(page, order, init);
259+ }
260+}
261+
262+void kasan_free_pages(struct page *page, unsigned int order)
263+{
264+ /*
265+ * This condition should match the one in free_pages_prepare() in
266+ * page_alloc.c.
267+ */
268+ bool init = want_init_on_free();
269+
270+ kasan_poison_pages(page, order, init);
271+}
272+
241273 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
242274
243275 void kasan_set_tagging_report_once(bool state)
mm/kasan/sw_tags.c+7−0
@@ -207,3 +207,10 @@ struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, · 206 unmodified lines
207207
208208 return &alloc_meta->free_track[i];
209209 }
210+
211+void kasan_tag_mismatch(unsigned long addr, unsigned long access_info,
212+ unsigned long ret_ip)
213+{
214+ kasan_report(addr, 1 << (access_info & 0xf), access_info & 0x10,
215+ ret_ip);
216+}
mm/mempool.c+4−2
@@ -106,7 +106,8 @@ static __always_inline void kasan_poison_element(mempool_t *pool, void *element) · 105 unmodified lines
106106 if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
107107 kasan_slab_free_mempool(element);
108108 else if (pool->alloc == mempool_alloc_pages)
109− kasan_free_pages(element, (unsigned long)pool->pool_data, false);
109+ kasan_poison_pages(element, (unsigned long)pool->pool_data,
110+ false);
110111 }
111112
112113 static void kasan_unpoison_element(mempool_t *pool, void *element)
@@ -114,7 +115,8 @@ static void kasan_unpoison_element(mempool_t *pool, void *element) · 1 unmodified line
114115 if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
115116 kasan_unpoison_range(element, __ksize(element));
116117 else if (pool->alloc == mempool_alloc_pages)
117− kasan_alloc_pages(element, (unsigned long)pool->pool_data, false);
118+ kasan_unpoison_pages(element, (unsigned long)pool->pool_data,
119+ false);
118120 }
119121
120122 static __always_inline void add_element(mempool_t *pool, void *element)
mm/page_alloc.c+39−27
@@ -382,7 +382,7 @@ int page_group_by_mobility_disabled __read_mostly; · 381 unmodified lines
382382 static DEFINE_STATIC_KEY_TRUE(deferred_pages);
383383
384384 /*
385− * Calling kasan_free_pages() only after deferred memory initialization
385+ * Calling kasan_poison_pages() only after deferred memory initialization
386386 * has completed. Poisoning pages during deferred memory init will greatly
387387 * lengthen the process and cause problem in large memory systems as the
388388 * deferred pages initialization is done with interrupt disabled.
@@ -394,15 +394,12 @@ static DEFINE_STATIC_KEY_TRUE(deferred_pages); · 5 unmodified lines
394394 * on-demand allocation and then freed again before the deferred pages
395395 * initialization is done, but this is not likely to happen.
396396 */
397−static inline void kasan_free_nondeferred_pages(struct page *page, int order,
398− bool init, fpi_t fpi_flags)
397+static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags)
399398 {
400− if (static_branch_unlikely(&deferred_pages))
401− return;
402− if (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
403− (fpi_flags & FPI_SKIP_KASAN_POISON))
404− return;
405− kasan_free_pages(page, order, init);
399+ return static_branch_unlikely(&deferred_pages) ||
400+ (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
401+ (fpi_flags & FPI_SKIP_KASAN_POISON)) ||
402+ PageSkipKASanPoison(page);
406403 }
407404
408405 /* Returns true if the struct page for the pfn is uninitialised */
@@ -453,13 +450,11 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn) · 44 unmodified lines
453450 return false;
454451 }
455452 #else
456−static inline void kasan_free_nondeferred_pages(struct page *page, int order,
457− bool init, fpi_t fpi_flags)
453+static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags)
458454 {
459− if (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
460− (fpi_flags & FPI_SKIP_KASAN_POISON))
461− return;
462− kasan_free_pages(page, order, init);
455+ return (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
456+ (fpi_flags & FPI_SKIP_KASAN_POISON)) ||
457+ PageSkipKASanPoison(page);
463458 }
464459
465460 static inline bool early_page_uninitialised(unsigned long pfn)
@@ -1226,10 +1221,16 @@ out: · 760 unmodified lines
12261221 return ret;
12271222 }
12281223
1229−static void kernel_init_free_pages(struct page *page, int numpages)
1224+static void kernel_init_free_pages(struct page *page, int numpages, bool zero_tags)
12301225 {
12311226 int i;
12321227
1228+ if (zero_tags) {
1229+ for (i = 0; i < numpages; i++)
1230+ tag_clear_highpage(page + i);
1231+ return;
1232+ }
1233+
12331234 /* s390's use of memset() could override KASAN redzones. */
12341235 kasan_disable_current();
12351236 for (i = 0; i < numpages; i++) {
@@ -1245,7 +1246,7 @@ static __always_inline bool free_pages_prepare(struct page *page, · 9 unmodified lines
12451246 unsigned int order, bool check_free, fpi_t fpi_flags)
12461247 {
12471248 int bad = 0;
1248− bool init;
1249+ bool skip_kasan_poison = should_skip_kasan_poison(page, fpi_flags);
12491250
12501251 VM_BUG_ON_PAGE(PageTail(page), page);
12511252
@@ -1314,10 +1315,17 @@ static __always_inline bool free_pages_prepare(struct page *page, · 62 unmodified lines
13141315 * With hardware tag-based KASAN, memory tags must be set before the
13151316 * page becomes unavailable via debug_pagealloc or arch_free_page.
13161317 */
1317− init = want_init_on_free();
1318− if (init && !kasan_has_integrated_init())
1319− kernel_init_free_pages(page, 1 << order);
1320− kasan_free_nondeferred_pages(page, order, init, fpi_flags);
1318+ if (kasan_has_integrated_init()) {
1319+ if (!skip_kasan_poison)
1320+ kasan_free_pages(page, order);
1321+ } else {
1322+ bool init = want_init_on_free();
1323+
1324+ if (init)
1325+ kernel_init_free_pages(page, 1 << order, false);
1326+ if (!skip_kasan_poison)
1327+ kasan_poison_pages(page, order, init);
1328+ }
13211329
13221330 /*
13231331 * arch_free_page() can make the page's contents inaccessible. s390
@@ -2324,8 +2332,6 @@ static bool check_new_pages(struct page *page, unsigned int order) · 1000 unmodified lines
23242332 inline void post_alloc_hook(struct page *page, unsigned int order,
23252333 gfp_t gfp_flags)
23262334 {
2327− bool init;
2328−
23292335 set_page_private(page, 0);
23302336 set_page_refcounted(page);
23312337
@@ -2344,10 +2350,16 @@ inline void post_alloc_hook(struct page *page, unsigned int order, · 12 unmodified lines
23442350 * kasan_alloc_pages and kernel_init_free_pages must be
23452351 * kept together to avoid discrepancies in behavior.
23462352 */
2347− init = !want_init_on_free() && want_init_on_alloc(gfp_flags);
2348− kasan_alloc_pages(page, order, init);
2349− if (init && !kasan_has_integrated_init())
2350− kernel_init_free_pages(page, 1 << order);
2353+ if (kasan_has_integrated_init()) {
2354+ kasan_alloc_pages(page, order, gfp_flags);
2355+ } else {
2356+ bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags);
2357+
2358+ kasan_unpoison_pages(page, order, init);
2359+ if (init)
2360+ kernel_init_free_pages(page, 1 << order,
2361+ gfp_flags & __GFP_ZEROTAGS);
2362+ }
23512363
23522364 set_page_owner(page, order, gfp_flags);
23532365 }
scripts/Makefile.kasan+1−0
@@ -50,6 +50,7 @@ endif · 49 unmodified lines
5050 CFLAGS_KASAN := -fsanitize=kernel-hwaddress \
5151 $(call cc-param,hwasan-instrument-stack=$(stack_enable)) \
5252 $(call cc-param,hwasan-use-short-granules=0) \
53+ $(call cc-param,hwasan-inline-all-checks=0) \
5354 $(instrumentation_flags)
5455
5556 endif # CONFIG_KASAN_SW_TAGS
scripts/tools-support-relr.sh+2−1
@@ -7,7 +7,8 @@ trap "rm -f $tmp_file.o $tmp_file $tmp_file.bin" EXIT · 6 unmodified lines
77 cat << "END" | $CC -c -x c - -o $tmp_file.o >/dev/null 2>&1
88 void *p = &p;
99 END
10−$LD $tmp_file.o -shared -Bsymbolic --pack-dyn-relocs=relr -o $tmp_file
10+$LD $tmp_file.o -shared -Bsymbolic --pack-dyn-relocs=relr \
11+ --use-android-relr-tags -o $tmp_file
1112
1213 # Despite printing an error message, GNU nm still exits with exit code 0 if it
1314 # sees a relr section. So we need to check that nothing is printed to stderr.
tools/testing/selftests/arm64/fp/sve-probe-vls.c+1−1
@@ -25,7 +25,7 @@ int main(int argc, char **argv) · 24 unmodified lines
2525 ksft_set_plan(2);
2626
2727 if (!(getauxval(AT_HWCAP) & HWCAP_SVE))
28− ksft_exit_skip("SVE not available");
28+ ksft_exit_skip("SVE not available\n");
2929
3030 /*
3131 * Enumerate up to SVE_VQ_MAX vector lengths

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t