Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Catalin Marinas: "The bulk is in-kernel pointer authentication, activity monitors and lots of asm symbol annotations. I also queued the sys_mremap() patch commenting the asymmetry in the address untagging. Summary: - In-kernel Pointer Authentication support (previously only offered to user space). - ARM Activity Monitors (AMU) extension support allowing better CPU utilisation numbers for the scheduler (frequency invariance). - Memory hot-remove support for arm64. - Lots of asm annotations (SYM_*) in preparation for the in-kernel Branch Target Identification (BTI) support. - arm64 perf updates: ARMv8.5-PMU 64-bit counters, refactoring the PMU init callbacks, support for new DT compatibles. - IPv6 header checksum optimisation. - Fixes: SDEI (software delegated exception interface) double-lock on hibernate with shared events. - Minor clean-ups and refactoring: cpu_ops accessor, cpu_do_switch_mm() converted to C, cpufeature finalisation helper. - sys_mremap() comment explaining the asymmetric address untagging behaviour" * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (81 commits) mm/mremap: Add comment explaining the untagging behaviour of mremap() arm64: head: Convert install_el2_stub to SYM_INNER_LABEL arm64: Introduce get_cpu_ops() helper function arm64: Rename cpu_read_ops() to init_cpu_ops() arm64: Declare ACPI parking protocol CPU operation if needed arm64: move kimage_vaddr to .rodata arm64: use mov_q instead of literal ldr arm64: Kconfig: verify binutils support for ARM64_PTR_AUTH lkdtm: arm64: test kernel pointer authentication arm64: compile the kernel with ptrauth return address signing kconfig: Add support for 'as-option' arm64: suspend: restore the kernel ptrauth keys arm64: __show_regs: strip PAC from lr in printk arm64: unwind: strip PAC from kernel addresses arm64: mask PAC bits of __builtin_return_address arm64: initialize ptrauth keys for kernel booting task arm64: initialize and switch ptrauth kernel keys arm64: enable ptrauth earlier arm64: cpufeature: handle conflicts based on capability arm64: cpufeature: Move cpu capability helpers inside C file ...

Linus Torvalds committed 6 years ago

Commit 3cd86a52 parents a8222fd b2a84de88 files · +2192 −700Browse files

Unified · Split

88 files

Documentation/arm64/amu.rst +112 −0
Documentation/arm64/booting.rst +14 −0
Documentation/arm64/index.rst +1 −0
arch/arm64/Kconfig +66 −3
arch/arm64/Makefile +11 −0
arch/arm64/crypto/aes-ce.S +2 −2
arch/arm64/crypto/aes-modes.S +24 −24
arch/arm64/crypto/aes-neon.S +2 −2
arch/arm64/crypto/ghash-ce-core.S +8 −8
arch/arm64/include/asm/asm_pointer_auth.h +65 −0
arch/arm64/include/asm/assembler.h +10 −6
arch/arm64/include/asm/checksum.h +6 −1
arch/arm64/include/asm/compiler.h +24 −0
arch/arm64/include/asm/cpu_ops.h +4 −4
arch/arm64/include/asm/cpucaps.h +4 −1
arch/arm64/include/asm/cpufeature.h +96 −29
arch/arm64/include/asm/esr.h +1 −1
arch/arm64/include/asm/kvm_arm.h +1 −0
arch/arm64/include/asm/kvm_asm.h +4 −0
arch/arm64/include/asm/kvm_mmu.h +4 −5
arch/arm64/include/asm/memory.h +1 −0
arch/arm64/include/asm/mmu.h +6 −4
arch/arm64/include/asm/mmu_context.h +2 −0
arch/arm64/include/asm/page.h +4 −0
arch/arm64/include/asm/perf_event.h +2 −1
arch/arm64/include/asm/pointer_auth.h +22 −28
arch/arm64/include/asm/proc-fns.h +0 −2
arch/arm64/include/asm/processor.h +2 −1
arch/arm64/include/asm/smp.h +12 −0
arch/arm64/include/asm/stackprotector.h +5 −0
arch/arm64/include/asm/sysreg.h +48 −0
arch/arm64/include/asm/topology.h +9 −0
arch/arm64/kernel/Makefile +1 −1
arch/arm64/kernel/armv8_deprecated.c +1 −1
arch/arm64/kernel/asm-offsets.c +16 −0
arch/arm64/kernel/cpu-reset.S +1 −1
arch/arm64/kernel/cpu_errata.c +8 −10
arch/arm64/kernel/cpu_ops.c +9 −2
arch/arm64/kernel/cpufeature.c +142 −23
arch/arm64/kernel/cpuidle.c +5 −4
arch/arm64/kernel/entry-common.c +1 −1
arch/arm64/kernel/entry-ftrace.S +24 −24
arch/arm64/kernel/entry.S +64 −57
arch/arm64/kernel/head.S +46 −40
arch/arm64/kernel/hibernate-asm.S +0 −2
arch/arm64/kernel/hyp-stub.S +1 −1
arch/arm64/kernel/machine_kexec_file.c +1 −1
arch/arm64/kernel/perf_event.c +181 −157
arch/arm64/kernel/pointer_auth.c +2 −5
arch/arm64/kernel/process.c +3 −2
arch/arm64/kernel/ptrace.c +8 −8
arch/arm64/kernel/relocate_kernel.S +1 −3
arch/arm64/kernel/setup.c +5 −3
arch/arm64/kernel/sleep.S +2 −0
arch/arm64/kernel/smp.c +92 −67
arch/arm64/kernel/stacktrace.c +4 −1
arch/arm64/kernel/topology.c +180 −0
arch/arm64/kernel/vdso/sigreturn.S +2 −2
arch/arm64/kernel/vdso32/sigreturn.S +8 −15
arch/arm64/kvm/hyp-init.S +8 −10
arch/arm64/kvm/hyp.S +2 −2
arch/arm64/kvm/hyp/fpsimd.S +4 −4
arch/arm64/kvm/hyp/hyp-entry.S +16 −11
arch/arm64/kvm/hyp/switch.c +20 −8
arch/arm64/kvm/hyp/sysreg-sr.c +4 −4
arch/arm64/kvm/hyp/tlb.c +4 −4
arch/arm64/kvm/sys_regs.c +101 −2
arch/arm64/lib/csum.c +27 −0
arch/arm64/lib/strcmp.S +1 −1
arch/arm64/mm/context.c +30 −2
arch/arm64/mm/mmu.c +370 −9
arch/arm64/mm/proc.S +64 −40
arch/arm64/mm/ptdump_debugfs.c +4 −0
drivers/base/arch_topology.c +12 −0
drivers/clocksource/arm_arch_timer.c +15 −3
drivers/cpufreq/cpufreq.c +20 −0
drivers/firmware/arm_sdei.c +36 −35
drivers/misc/lkdtm/bugs.c +36 −0
drivers/misc/lkdtm/core.c +1 −0
drivers/misc/lkdtm/lkdtm.h +1 −0
drivers/perf/arm-ccn.c +10 −10
drivers/perf/arm_spe_pmu.c +1 −1
include/linux/arch_topology.h +2 −0
include/linux/cpufreq.h +5 −0
include/linux/perf/arm_pmu.h +1 −0
include/linux/stackprotector.h +1 −1
mm/mremap.c +10 −0
scripts/Kconfig.include +6 −0

Documentation/arm64/amu.rst+112−0

@@ -0,0 +1,112 @@
	1	+=======================================================
	2	+Activity Monitors Unit (AMU) extension in AArch64 Linux
	3	+=======================================================
	4	+
	5	+Author: Ionela Voinescu <ionela.voinescu@arm.com>
	6	+
	7	+Date: 2019-09-10
	8	+
	9	+This document briefly describes the provision of Activity Monitors Unit
	10	+support in AArch64 Linux.
	11	+
	12	+
	13	+Architecture overview
	14	+---------------------
	15	+
	16	+The activity monitors extension is an optional extension introduced by the
	17	+ARMv8.4 CPU architecture.
	18	+
	19	+The activity monitors unit, implemented in each CPU, provides performance
	20	+counters intended for system management use. The AMU extension provides a
	21	+system register interface to the counter registers and also supports an
	22	+optional external memory-mapped interface.
	23	+
	24	+Version 1 of the Activity Monitors architecture implements a counter group
	25	+of four fixed and architecturally defined 64-bit event counters.
	26	+ - CPU cycle counter: increments at the frequency of the CPU.
	27	+ - Constant counter: increments at the fixed frequency of the system
	28	+ clock.
	29	+ - Instructions retired: increments with every architecturally executed
	30	+ instruction.
	31	+ - Memory stall cycles: counts instruction dispatch stall cycles caused by
	32	+ misses in the last level cache within the clock domain.
	33	+
	34	+When in WFI or WFE these counters do not increment.
	35	+
	36	+The Activity Monitors architecture provides space for up to 16 architected
	37	+event counters. Future versions of the architecture may use this space to
	38	+implement additional architected event counters.
	39	+
	40	+Additionally, version 1 implements a counter group of up to 16 auxiliary
	41	+64-bit event counters.
	42	+
	43	+On cold reset all counters reset to 0.
	44	+
	45	+
	46	+Basic support
	47	+-------------
	48	+
	49	+The kernel can safely run a mix of CPUs with and without support for the
	50	+activity monitors extension. Therefore, when CONFIG_ARM64_AMU_EXTN is
	51	+selected we unconditionally enable the capability to allow any late CPU
	52	+(secondary or hotplugged) to detect and use the feature.
	53	+
	54	+When the feature is detected on a CPU, we flag the availability of the
	55	+feature but this does not guarantee the correct functionality of the
	56	+counters, only the presence of the extension.
	57	+
	58	+Firmware (code running at higher exception levels, e.g. arm-tf) support is
	59	+needed to:
	60	+ - Enable access for lower exception levels (EL2 and EL1) to the AMU
	61	+ registers.
	62	+ - Enable the counters. If not enabled these will read as 0.
	63	+ - Save/restore the counters before/after the CPU is being put/brought up
	64	+ from the 'off' power state.
	65	+
	66	+When using kernels that have this feature enabled but boot with broken
	67	+firmware the user may experience panics or lockups when accessing the
	68	+counter registers. Even if these symptoms are not observed, the values
	69	+returned by the register reads might not correctly reflect reality. Most
	70	+commonly, the counters will read as 0, indicating that they are not
	71	+enabled.
	72	+
	73	+If proper support is not provided in firmware it's best to disable
	74	+CONFIG_ARM64_AMU_EXTN. To be noted that for security reasons, this does not
	75	+bypass the setting of AMUSERENR_EL0 to trap accesses from EL0 (userspace) to
	76	+EL1 (kernel). Therefore, firmware should still ensure accesses to AMU registers
	77	+are not trapped in EL2/EL3.
	78	+
	79	+The fixed counters of AMUv1 are accessible though the following system
	80	+register definitions:
	81	+ - SYS_AMEVCNTR0_CORE_EL0
	82	+ - SYS_AMEVCNTR0_CONST_EL0
	83	+ - SYS_AMEVCNTR0_INST_RET_EL0
	84	+ - SYS_AMEVCNTR0_MEM_STALL_EL0
	85	+
	86	+Auxiliary platform specific counters can be accessed using
	87	+SYS_AMEVCNTR1_EL0(n), where n is a value between 0 and 15.
	88	+
	89	+Details can be found in: arch/arm64/include/asm/sysreg.h.
	90	+
	91	+
	92	+Userspace access
	93	+----------------
	94	+
	95	+Currently, access from userspace to the AMU registers is disabled due to:
	96	+ - Security reasons: they might expose information about code executed in
	97	+ secure mode.
	98	+ - Purpose: AMU counters are intended for system management use.
	99	+
	100	+Also, the presence of the feature is not visible to userspace.
	101	+
	102	+
	103	+Virtualization
	104	+--------------
	105	+
	106	+Currently, access from userspace (EL0) and kernelspace (EL1) on the KVM
	107	+guest side is disabled due to:
	108	+ - Security reasons: they might expose information about code executed
	109	+ by other guests or the host.
	110	+
	111	+Any attempt to access the AMU registers will result in an UNDEFINED
	112	+exception being injected into the guest.

Documentation/arm64/booting.rst+14−0

@@ -248,6 +248,20 @@ Before jumping into the kernel, the following conditions must be met: · 247 unmodified lines
248	248	- HCR_EL2.APK (bit 40) must be initialised to 0b1
249	249	- HCR_EL2.API (bit 41) must be initialised to 0b1
250	250
	251	+ For CPUs with Activity Monitors Unit v1 (AMUv1) extension present:
	252	+ - If EL3 is present:
	253	+ CPTR_EL3.TAM (bit 30) must be initialised to 0b0
	254	+ CPTR_EL2.TAM (bit 30) must be initialised to 0b0
	255	+ AMCNTENSET0_EL0 must be initialised to 0b1111
	256	+ AMCNTENSET1_EL0 must be initialised to a platform specific value
	257	+ having 0b1 set for the corresponding bit for each of the auxiliary
	258	+ counters present.
	259	+ - If the kernel is entered at EL1:
	260	+ AMCNTENSET0_EL0 must be initialised to 0b1111
	261	+ AMCNTENSET1_EL0 must be initialised to a platform specific value
	262	+ having 0b1 set for the corresponding bit for each of the auxiliary
	263	+ counters present.
	264	+
251	265	The requirements described above for CPU mode, caches, MMUs, architected
252	266	timers, coherency and system registers apply to all CPUs. All CPUs must
253	267	enter the kernel in the same exception level.

Documentation/arm64/index.rst+1−0

@@ -6,6 +6,7 @@ ARM64 Architecture · 5 unmodified lines
6	6	:maxdepth: 1
7	7
8	8	acpi_object_usage
	9	+ amu
9	10	arm-acpi
10	11	booting
11	12	cpu-feature-registers

arch/arm64/Kconfig+66−3

@@ -117,6 +117,7 @@ config ARM64 · 116 unmodified lines
117	117	select HAVE_ALIGNED_STRUCT_PAGE if SLUB
118	118	select HAVE_ARCH_AUDITSYSCALL
119	119	select HAVE_ARCH_BITREVERSE
	120	+ select HAVE_ARCH_COMPILER_H
120	121	select HAVE_ARCH_HUGE_VMAP
121	122	select HAVE_ARCH_JUMP_LABEL
122	123	select HAVE_ARCH_JUMP_LABEL_RELATIVE
@@ -280,6 +281,9 @@ config ZONE_DMA32 · 157 unmodified lines
280	281	config ARCH_ENABLE_MEMORY_HOTPLUG
281	282	def_bool y
282	283
	284	+config ARCH_ENABLE_MEMORY_HOTREMOVE
	285	+ def_bool y
	286	+
283	287	config SMP
284	288	def_bool y
285	289
@@ -951,11 +955,11 @@ config HOTPLUG_CPU · 665 unmodified lines
951	955
952	956	# Common NUMA Features
953	957	config NUMA
954		− bool "Numa Memory Allocation and Scheduler Support"
	958	+ bool "NUMA Memory Allocation and Scheduler Support"
955	959	select ACPI_NUMA if ACPI
956	960	select OF_NUMA
957	961	help
958		− Enable NUMA (Non Uniform Memory Access) support.
	962	+ Enable NUMA (Non-Uniform Memory Access) support.
959	963
960	964	The kernel will try to allocate memory used by a CPU on the
961	965	local memory of the CPU and add some more
@@ -1497,6 +1501,9 @@ config ARM64_PTR_AUTH · 535 unmodified lines
1497	1501	bool "Enable support for pointer authentication"
1498	1502	default y
1499	1503	depends on !KVM \|\| ARM64_VHE
	1504	+ depends on (CC_HAS_SIGN_RETURN_ADDRESS \|\| CC_HAS_BRANCH_PROT_PAC_RET) && AS_HAS_PAC
	1505	+ depends on CC_IS_GCC \|\| (CC_IS_CLANG && AS_HAS_CFI_NEGATE_RA_STATE)
	1506	+ depends on (!FUNCTION_GRAPH_TRACER \|\| DYNAMIC_FTRACE_WITH_REGS)
1500	1507	help
1501	1508	Pointer authentication (part of the ARMv8.3 Extensions) provides
1502	1509	instructions for signing and authenticating pointers against secret
@@ -1504,16 +1511,72 @@ config ARM64_PTR_AUTH · 1 unmodified line
1504	1511	and other attacks.
1505	1512
1506	1513	This option enables these instructions at EL0 (i.e. for userspace).
1507		−
1508	1514	Choosing this option will cause the kernel to initialise secret keys
1509	1515	for each process at exec() time, with these keys being
1510	1516	context-switched along with the process.
1511	1517
	1518	+ If the compiler supports the -mbranch-protection or
	1519	+ -msign-return-address flag (e.g. GCC 7 or later), then this option
	1520	+ will also cause the kernel itself to be compiled with return address
	1521	+ protection. In this case, and if the target hardware is known to
	1522	+ support pointer authentication, then CONFIG_STACKPROTECTOR can be
	1523	+ disabled with minimal loss of protection.
	1524	+
1512	1525	The feature is detected at runtime. If the feature is not present in
1513	1526	hardware it will not be advertised to userspace/KVM guest nor will it
1514	1527	be enabled. However, KVM guest also require VHE mode and hence
1515	1528	CONFIG_ARM64_VHE=y option to use this feature.
1516	1529
	1530	+ If the feature is present on the boot CPU but not on a late CPU, then
	1531	+ the late CPU will be parked. Also, if the boot CPU does not have
	1532	+ address auth and the late CPU has then the late CPU will still boot
	1533	+ but with the feature disabled. On such a system, this option should
	1534	+ not be selected.
	1535	+
	1536	+ This feature works with FUNCTION_GRAPH_TRACER option only if
	1537	+ DYNAMIC_FTRACE_WITH_REGS is enabled.
	1538	+
	1539	+config CC_HAS_BRANCH_PROT_PAC_RET
	1540	+ # GCC 9 or later, clang 8 or later
	1541	+ def_bool $(cc-option,-mbranch-protection=pac-ret+leaf)
	1542	+
	1543	+config CC_HAS_SIGN_RETURN_ADDRESS
	1544	+ # GCC 7, 8
	1545	+ def_bool $(cc-option,-msign-return-address=all)
	1546	+
	1547	+config AS_HAS_PAC
	1548	+ def_bool $(as-option,-Wa$(comma)-march=armv8.3-a)
	1549	+
	1550	+config AS_HAS_CFI_NEGATE_RA_STATE
	1551	+ def_bool $(as-instr,.cfi_startproc\n.cfi_negate_ra_state\n.cfi_endproc\n)
	1552	+
	1553	+endmenu
	1554	+
	1555	+menu "ARMv8.4 architectural features"
	1556	+
	1557	+config ARM64_AMU_EXTN
	1558	+ bool "Enable support for the Activity Monitors Unit CPU extension"
	1559	+ default y
	1560	+ help
	1561	+ The activity monitors extension is an optional extension introduced
	1562	+ by the ARMv8.4 CPU architecture. This enables support for version 1
	1563	+ of the activity monitors architecture, AMUv1.
	1564	+
	1565	+ To enable the use of this extension on CPUs that implement it, say Y.
	1566	+
	1567	+ Note that for architectural reasons, firmware _must_ implement AMU
	1568	+ support when running on CPUs that present the activity monitors
	1569	+ extension. The required support is present in:
	1570	+ * Version 1.5 and later of the ARM Trusted Firmware
	1571	+
	1572	+ For kernels that have this configuration enabled but boot with broken
	1573	+ firmware, you may need to say N here until the firmware is fixed.
	1574	+ Otherwise you may experience firmware panics or lockups when
	1575	+ accessing the counter registers. Even if you are not observing these
	1576	+ symptoms, the values returned by the register reads might not
	1577	+ correctly reflect reality. Most commonly, the value read will be 0,
	1578	+ indicating that the counter is not enabled.
	1579	+
1517	1580	endmenu
1518	1581
1519	1582	menu "ARMv8.5 architectural features"

arch/arm64/Makefile+11−0

@@ -65,6 +65,17 @@ stack_protector_prepare: prepare0 · 64 unmodified lines
65	65	include/generated/asm-offsets.h))
66	66	endif
67	67
	68	+ifeq ($(CONFIG_ARM64_PTR_AUTH),y)
	69	+branch-prot-flags-$(CONFIG_CC_HAS_SIGN_RETURN_ADDRESS) := -msign-return-address=all
	70	+branch-prot-flags-$(CONFIG_CC_HAS_BRANCH_PROT_PAC_RET) := -mbranch-protection=pac-ret+leaf
	71	+# -march=armv8.3-a enables the non-nops instructions for PAC, to avoid the
	72	+# compiler to generate them and consequently to break the single image contract
	73	+# we pass it only to the assembler. This option is utilized only in case of non
	74	+# integrated assemblers.
	75	+branch-prot-flags-$(CONFIG_AS_HAS_PAC) += -Wa,-march=armv8.3-a
	76	+KBUILD_CFLAGS += $(branch-prot-flags-y)
	77	+endif
	78	+
68	79	ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
69	80	KBUILD_CPPFLAGS += -mbig-endian
70	81	CHECKFLAGS += -D__AARCH64EB__

arch/arm64/crypto/aes-ce.S+2−2

@@ -9,8 +9,8 @@ · 8 unmodified lines
9	9	#include <linux/linkage.h>
10	10	#include <asm/assembler.h>
11	11
12		−#define AES_ENTRY(func) SYM_FUNC_START(ce_ ## func)
13		−#define AES_ENDPROC(func) SYM_FUNC_END(ce_ ## func)
	12	+#define AES_FUNC_START(func) SYM_FUNC_START(ce_ ## func)
	13	+#define AES_FUNC_END(func) SYM_FUNC_END(ce_ ## func)
14	14
15	15	.arch armv8-a+crypto
16	16

arch/arm64/crypto/aes-modes.S+24−24

@@ -51,7 +51,7 @@ SYM_FUNC_END(aes_decrypt_block5x) · 50 unmodified lines
51	51	* int blocks)
52	52	*/
53	53
54		−AES_ENTRY(aes_ecb_encrypt)
	54	+AES_FUNC_START(aes_ecb_encrypt)
55	55	stp x29, x30, [sp, #-16]!
56	56	mov x29, sp
57	57
@@ -79,10 +79,10 @@ ST5( st1 {v4.16b}, [x0], #16 ) · 21 unmodified lines
79	79	.Lecbencout:
80	80	ldp x29, x30, [sp], #16
81	81	ret
82		−AES_ENDPROC(aes_ecb_encrypt)
	82	+AES_FUNC_END(aes_ecb_encrypt)
83	83
84	84
85		−AES_ENTRY(aes_ecb_decrypt)
	85	+AES_FUNC_START(aes_ecb_decrypt)
86	86	stp x29, x30, [sp, #-16]!
87	87	mov x29, sp
88	88
@@ -110,7 +110,7 @@ ST5( st1 {v4.16b}, [x0], #16 ) · 21 unmodified lines
110	110	.Lecbdecout:
111	111	ldp x29, x30, [sp], #16
112	112	ret
113		−AES_ENDPROC(aes_ecb_decrypt)
	113	+AES_FUNC_END(aes_ecb_decrypt)
114	114
115	115
116	116	/*
@@ -126,7 +126,7 @@ AES_ENDPROC(aes_ecb_decrypt) · 9 unmodified lines
126	126	* u32 const rk2[]);
127	127	*/
128	128
129		−AES_ENTRY(aes_essiv_cbc_encrypt)
	129	+AES_FUNC_START(aes_essiv_cbc_encrypt)
130	130	ld1 {v4.16b}, [x5] /* get iv */
131	131
132	132	mov w8, #14 /* AES-256: 14 rounds */
@@ -135,7 +135,7 @@ AES_ENTRY(aes_essiv_cbc_encrypt) · 2 unmodified lines
135	135	enc_switch_key w3, x2, x6
136	136	b .Lcbcencloop4x
137	137
138		−AES_ENTRY(aes_cbc_encrypt)
	138	+AES_FUNC_START(aes_cbc_encrypt)
139	139	ld1 {v4.16b}, [x5] /* get iv */
140	140	enc_prepare w3, x2, x6
141	141
@@ -167,10 +167,10 @@ AES_ENTRY(aes_cbc_encrypt) · 25 unmodified lines
167	167	.Lcbcencout:
168	168	st1 {v4.16b}, [x5] /* return iv */
169	169	ret
170		−AES_ENDPROC(aes_cbc_encrypt)
171		−AES_ENDPROC(aes_essiv_cbc_encrypt)
	170	+AES_FUNC_END(aes_cbc_encrypt)
	171	+AES_FUNC_END(aes_essiv_cbc_encrypt)
172	172
173		−AES_ENTRY(aes_essiv_cbc_decrypt)
	173	+AES_FUNC_START(aes_essiv_cbc_decrypt)
174	174	stp x29, x30, [sp, #-16]!
175	175	mov x29, sp
176	176
@@ -181,7 +181,7 @@ AES_ENTRY(aes_essiv_cbc_decrypt) · 4 unmodified lines
181	181	encrypt_block cbciv, w8, x6, x7, w9
182	182	b .Lessivcbcdecstart
183	183
184		−AES_ENTRY(aes_cbc_decrypt)
	184	+AES_FUNC_START(aes_cbc_decrypt)
185	185	stp x29, x30, [sp, #-16]!
186	186	mov x29, sp
187	187
@@ -238,8 +238,8 @@ ST5( st1 {v4.16b}, [x0], #16 ) · 50 unmodified lines
238	238	st1 {cbciv.16b}, [x5] /* return iv */
239	239	ldp x29, x30, [sp], #16
240	240	ret
241		−AES_ENDPROC(aes_cbc_decrypt)
242		−AES_ENDPROC(aes_essiv_cbc_decrypt)
	241	+AES_FUNC_END(aes_cbc_decrypt)
	242	+AES_FUNC_END(aes_essiv_cbc_decrypt)
243	243
244	244
245	245	/*
@@ -249,7 +249,7 @@ AES_ENDPROC(aes_essiv_cbc_decrypt) · 3 unmodified lines
249	249	* int rounds, int bytes, u8 const iv[])
250	250	*/
251	251
252		−AES_ENTRY(aes_cbc_cts_encrypt)
	252	+AES_FUNC_START(aes_cbc_cts_encrypt)
253	253	adr_l x8, .Lcts_permute_table
254	254	sub x4, x4, #16
255	255	add x9, x8, #32
@@ -276,9 +276,9 @@ AES_ENTRY(aes_cbc_cts_encrypt) · 20 unmodified lines
276	276	st1 {v0.16b}, [x4] /* overlapping stores */
277	277	st1 {v1.16b}, [x0]
278	278	ret
279		−AES_ENDPROC(aes_cbc_cts_encrypt)
	279	+AES_FUNC_END(aes_cbc_cts_encrypt)
280	280
281		−AES_ENTRY(aes_cbc_cts_decrypt)
	281	+AES_FUNC_START(aes_cbc_cts_decrypt)
282	282	adr_l x8, .Lcts_permute_table
283	283	sub x4, x4, #16
284	284	add x9, x8, #32
@@ -305,7 +305,7 @@ AES_ENTRY(aes_cbc_cts_decrypt) · 20 unmodified lines
305	305	st1 {v2.16b}, [x4] /* overlapping stores */
306	306	st1 {v0.16b}, [x0]
307	307	ret
308		−AES_ENDPROC(aes_cbc_cts_decrypt)
	308	+AES_FUNC_END(aes_cbc_cts_decrypt)
309	309
310	310	.section ".rodata", "a"
311	311	.align 6
@@ -324,7 +324,7 @@ AES_ENDPROC(aes_cbc_cts_decrypt) · 12 unmodified lines
324	324	* int blocks, u8 ctr[])
325	325	*/
326	326
327		−AES_ENTRY(aes_ctr_encrypt)
	327	+AES_FUNC_START(aes_ctr_encrypt)
328	328	stp x29, x30, [sp, #-16]!
329	329	mov x29, sp
330	330
@@ -409,7 +409,7 @@ ST5( st1 {v4.16b}, [x0], #16 ) · 78 unmodified lines
409	409	rev x7, x7
410	410	ins vctr.d[0], x7
411	411	b .Lctrcarrydone
412		−AES_ENDPROC(aes_ctr_encrypt)
	412	+AES_FUNC_END(aes_ctr_encrypt)
413	413
414	414
415	415	/*
@@ -433,7 +433,7 @@ AES_ENDPROC(aes_ctr_encrypt) · 17 unmodified lines
433	433	uzp1 xtsmask.4s, xtsmask.4s, \tmp\().4s
434	434	.endm
435	435
436		−AES_ENTRY(aes_xts_encrypt)
	436	+AES_FUNC_START(aes_xts_encrypt)
437	437	stp x29, x30, [sp, #-16]!
438	438	mov x29, sp
439	439
@@ -518,9 +518,9 @@ AES_ENTRY(aes_xts_encrypt) · 78 unmodified lines
518	518	st1 {v2.16b}, [x4] /* overlapping stores */
519	519	mov w4, wzr
520	520	b .Lxtsencctsout
521		−AES_ENDPROC(aes_xts_encrypt)
	521	+AES_FUNC_END(aes_xts_encrypt)
522	522
523		−AES_ENTRY(aes_xts_decrypt)
	523	+AES_FUNC_START(aes_xts_decrypt)
524	524	stp x29, x30, [sp, #-16]!
525	525	mov x29, sp
526	526
@@ -612,13 +612,13 @@ AES_ENTRY(aes_xts_decrypt) · 85 unmodified lines
612	612	st1 {v2.16b}, [x4] /* overlapping stores */
613	613	mov w4, wzr
614	614	b .Lxtsdecctsout
615		−AES_ENDPROC(aes_xts_decrypt)
	615	+AES_FUNC_END(aes_xts_decrypt)
616	616
617	617	/*
618	618	* aes_mac_update(u8 const in[], u32 const rk[], int rounds,
619	619	* int blocks, u8 dg[], int enc_before, int enc_after)
620	620	*/
621		−AES_ENTRY(aes_mac_update)
	621	+AES_FUNC_START(aes_mac_update)
622	622	frame_push 6
623	623
624	624	mov x19, x0
@@ -676,4 +676,4 @@ AES_ENTRY(aes_mac_update) · 51 unmodified lines
676	676	ld1 {v0.16b}, [x23] /* get dg */
677	677	enc_prepare w21, x20, x0
678	678	b .Lmacloop4x
679		−AES_ENDPROC(aes_mac_update)
	679	+AES_FUNC_END(aes_mac_update)

arch/arm64/crypto/aes-neon.S+2−2

@@ -8,8 +8,8 @@ · 7 unmodified lines
8	8	#include <linux/linkage.h>
9	9	#include <asm/assembler.h>
10	10
11		−#define AES_ENTRY(func) SYM_FUNC_START(neon_ ## func)
12		−#define AES_ENDPROC(func) SYM_FUNC_END(neon_ ## func)
	11	+#define AES_FUNC_START(func) SYM_FUNC_START(neon_ ## func)
	12	+#define AES_FUNC_END(func) SYM_FUNC_END(neon_ ## func)
13	13
14	14	xtsmask .req v7
15	15	cbciv .req v7

arch/arm64/crypto/ghash-ce-core.S+8−8

@@ -587,20 +587,20 @@ CPU_LE( rev w8, w8 ) · 586 unmodified lines
587	587	* struct ghash_key const *k, u64 dg[], u8 ctr[],
588	588	* int rounds, u8 tag)
589	589	*/
590		−ENTRY(pmull_gcm_encrypt)
	590	+SYM_FUNC_START(pmull_gcm_encrypt)
591	591	pmull_gcm_do_crypt 1
592		−ENDPROC(pmull_gcm_encrypt)
	592	+SYM_FUNC_END(pmull_gcm_encrypt)
593	593
594	594	/*
595	595	* void pmull_gcm_decrypt(int blocks, u8 dst[], const u8 src[],
596	596	* struct ghash_key const *k, u64 dg[], u8 ctr[],
597	597	* int rounds, u8 tag)
598	598	*/
599		−ENTRY(pmull_gcm_decrypt)
	599	+SYM_FUNC_START(pmull_gcm_decrypt)
600	600	pmull_gcm_do_crypt 0
601		−ENDPROC(pmull_gcm_decrypt)
	601	+SYM_FUNC_END(pmull_gcm_decrypt)
602	602
603		−pmull_gcm_ghash_4x:
	603	+SYM_FUNC_START_LOCAL(pmull_gcm_ghash_4x)
604	604	movi MASK.16b, #0xe1
605	605	shl MASK.2d, MASK.2d, #57
606	606
@@ -681,9 +681,9 @@ pmull_gcm_ghash_4x: · 74 unmodified lines
681	681	eor XL.16b, XL.16b, T2.16b
682	682
683	683	ret
684		−ENDPROC(pmull_gcm_ghash_4x)
	684	+SYM_FUNC_END(pmull_gcm_ghash_4x)
685	685
686		−pmull_gcm_enc_4x:
	686	+SYM_FUNC_START_LOCAL(pmull_gcm_enc_4x)
687	687	ld1 {KS0.16b}, [x5] // load upper counter
688	688	sub w10, w8, #4
689	689	sub w11, w8, #3
@@ -746,7 +746,7 @@ pmull_gcm_enc_4x: · 56 unmodified lines
746	746	eor INP3.16b, INP3.16b, KS3.16b
747	747
748	748	ret
749		−ENDPROC(pmull_gcm_enc_4x)
	749	+SYM_FUNC_END(pmull_gcm_enc_4x)
750	750
751	751	.section ".rodata", "a"
752	752	.align 6

arch/arm64/include/asm/asm_pointer_auth.h+65−0

@@ -0,0 +1,65 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef __ASM_ASM_POINTER_AUTH_H
	3	+#define __ASM_ASM_POINTER_AUTH_H
	4	+
	5	+#include <asm/alternative.h>
	6	+#include <asm/asm-offsets.h>
	7	+#include <asm/cpufeature.h>
	8	+#include <asm/sysreg.h>
	9	+
	10	+#ifdef CONFIG_ARM64_PTR_AUTH
	11	+/*
	12	+ * thread.keys_user.ap* as offset exceeds the #imm offset range
	13	+ * so use the base value of ldp as thread.keys_user and offset as
	14	+ * thread.keys_user.ap*.
	15	+ */
	16	+ .macro ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3
	17	+ mov \tmp1, #THREAD_KEYS_USER
	18	+ add \tmp1, \tsk, \tmp1
	19	+alternative_if_not ARM64_HAS_ADDRESS_AUTH
	20	+ b .Laddr_auth_skip_\@
	21	+alternative_else_nop_endif
	22	+ ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIA]
	23	+ msr_s SYS_APIAKEYLO_EL1, \tmp2
	24	+ msr_s SYS_APIAKEYHI_EL1, \tmp3
	25	+ ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIB]
	26	+ msr_s SYS_APIBKEYLO_EL1, \tmp2
	27	+ msr_s SYS_APIBKEYHI_EL1, \tmp3
	28	+ ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APDA]
	29	+ msr_s SYS_APDAKEYLO_EL1, \tmp2
	30	+ msr_s SYS_APDAKEYHI_EL1, \tmp3
	31	+ ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APDB]
	32	+ msr_s SYS_APDBKEYLO_EL1, \tmp2
	33	+ msr_s SYS_APDBKEYHI_EL1, \tmp3
	34	+.Laddr_auth_skip_\@:
	35	+alternative_if ARM64_HAS_GENERIC_AUTH
	36	+ ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APGA]
	37	+ msr_s SYS_APGAKEYLO_EL1, \tmp2
	38	+ msr_s SYS_APGAKEYHI_EL1, \tmp3
	39	+alternative_else_nop_endif
	40	+ .endm
	41	+
	42	+ .macro ptrauth_keys_install_kernel tsk, sync, tmp1, tmp2, tmp3
	43	+alternative_if ARM64_HAS_ADDRESS_AUTH
	44	+ mov \tmp1, #THREAD_KEYS_KERNEL
	45	+ add \tmp1, \tsk, \tmp1
	46	+ ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_KERNEL_KEY_APIA]
	47	+ msr_s SYS_APIAKEYLO_EL1, \tmp2
	48	+ msr_s SYS_APIAKEYHI_EL1, \tmp3
	49	+ .if \sync == 1
	50	+ isb
	51	+ .endif
	52	+alternative_else_nop_endif
	53	+ .endm
	54	+
	55	+#else /* CONFIG_ARM64_PTR_AUTH */
	56	+
	57	+ .macro ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3
	58	+ .endm
	59	+
	60	+ .macro ptrauth_keys_install_kernel tsk, sync, tmp1, tmp2, tmp3
	61	+ .endm
	62	+
	63	+#endif /* CONFIG_ARM64_PTR_AUTH */
	64	+
	65	+#endif /* __ASM_ASM_POINTER_AUTH_H */

arch/arm64/include/asm/assembler.h+10−6

@@ -256,12 +256,6 @@ alternative_endif · 255 unmodified lines
256	256	ldr \rd, [\rn, #VMA_VM_MM]
257	257	.endm
258	258
259		−/*
260		− * mmid - get context id from mm pointer (mm->context.id)
261		− */
262		− .macro mmid, rd, rn
263		− ldr \rd, [\rn, #MM_CONTEXT_ID]
264		− .endm
265	259	/*
266	260	* read_ctr - read CTR_EL0. If the system has mismatched register fields,
267	261	* provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val
@@ -430,6 +424,16 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU · 162 unmodified lines
430	424	9000:
431	425	.endm
432	426
	427	+/*
	428	+ * reset_amuserenr_el0 - reset AMUSERENR_EL0 if AMUv1 present
	429	+ */
	430	+ .macro reset_amuserenr_el0, tmpreg
	431	+ mrs \tmpreg, id_aa64pfr0_el1 // Check ID_AA64PFR0_EL1
	432	+ ubfx \tmpreg, \tmpreg, #ID_AA64PFR0_AMU_SHIFT, #4
	433	+ cbz \tmpreg, .Lskip_\@ // Skip if no AMU present
	434	+ msr_s SYS_AMUSERENR_EL0, xzr // Disable AMU access from EL0
	435	+.Lskip_\@:
	436	+ .endm
433	437	/*
434	438	* copy_page - copy src to dest using temp registers t1-t8
435	439	*/

arch/arm64/include/asm/checksum.h+6−1

@@ -5,7 +5,12 @@ · 4 unmodified lines
5	5	#ifndef __ASM_CHECKSUM_H
6	6	#define __ASM_CHECKSUM_H
7	7
8		−#include <linux/types.h>
	8	+#include <linux/in6.h>
	9	+
	10	+#define _HAVE_ARCH_IPV6_CSUM
	11	+__sum16 csum_ipv6_magic(const struct in6_addr *saddr,
	12	+ const struct in6_addr *daddr,
	13	+ __u32 len, __u8 proto, __wsum sum);
9	14
10	15	static inline __sum16 csum_fold(__wsum csum)
11	16	{

arch/arm64/include/asm/compiler.h+24−0

@@ -0,0 +1,24 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef __ASM_COMPILER_H
	3	+#define __ASM_COMPILER_H
	4	+
	5	+#if defined(CONFIG_ARM64_PTR_AUTH)
	6	+
	7	+/*
	8	+ * The EL0/EL1 pointer bits used by a pointer authentication code.
	9	+ * This is dependent on TBI0/TBI1 being enabled, or bits 63:56 would also apply.
	10	+ */
	11	+#define ptrauth_user_pac_mask() GENMASK_ULL(54, vabits_actual)
	12	+#define ptrauth_kernel_pac_mask() GENMASK_ULL(63, vabits_actual)
	13	+
	14	+/* Valid for EL0 TTBR0 and EL1 TTBR1 instruction pointers */
	15	+#define ptrauth_clear_pac(ptr) \
	16	+ ((ptr & BIT_ULL(55)) ? (ptr \| ptrauth_kernel_pac_mask()) : \
	17	+ (ptr & ~ptrauth_user_pac_mask()))
	18	+
	19	+#define __builtin_return_address(val) \
	20	+ (void *)(ptrauth_clear_pac((unsigned long)__builtin_return_address(val)))
	21	+
	22	+#endif /* CONFIG_ARM64_PTR_AUTH */
	23	+
	24	+#endif /* __ASM_COMPILER_H */

arch/arm64/include/asm/cpu_ops.h+4−4

@@ -55,12 +55,12 @@ struct cpu_operations { · 54 unmodified lines
55	55	#endif
56	56	};
57	57
58		−extern const struct cpu_operations *cpu_ops[NR_CPUS];
59		−int __init cpu_read_ops(int cpu);
	58	+int __init init_cpu_ops(int cpu);
	59	+extern const struct cpu_operations *get_cpu_ops(int cpu);
60	60
61		−static inline void __init cpu_read_bootcpu_ops(void)
	61	+static inline void __init init_bootcpu_ops(void)
62	62	{
63		− cpu_read_ops(0);
	63	+ init_cpu_ops(0);
64	64	}
65	65
66	66	#endif /* ifndef __ASM_CPU_OPS_H */

arch/arm64/include/asm/cpucaps.h+4−1

@@ -58,7 +58,10 @@ · 57 unmodified lines
58	58	#define ARM64_WORKAROUND_SPECULATIVE_AT_NVHE 48
59	59	#define ARM64_HAS_E0PD 49
60	60	#define ARM64_HAS_RNG 50
	61	+#define ARM64_HAS_AMU_EXTN 51
	62	+#define ARM64_HAS_ADDRESS_AUTH 52
	63	+#define ARM64_HAS_GENERIC_AUTH 53
61	64
62		−#define ARM64_NCAPS 51
	65	+#define ARM64_NCAPS 54
63	66
64	67	#endif /* __ASM_CPUCAPS_H */

arch/arm64/include/asm/cpufeature.h+96−29

@@ -208,6 +208,10 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; · 207 unmodified lines
208	208	* In some non-typical cases either both (a) and (b), or neither,
209	209	* should be permitted. This can be described by including neither
210	210	* or both flags in the capability's type field.
	211	+ *
	212	+ * In case of a conflict, the CPU is prevented from booting. If the
	213	+ * ARM64_CPUCAP_PANIC_ON_CONFLICT flag is specified for the capability,
	214	+ * then a kernel panic is triggered.
211	215	*/
212	216
213	217
@@ -240,6 +244,8 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; · 26 unmodified lines
240	244	#define ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU ((u16)BIT(4))
241	245	/* Is it safe for a late CPU to miss this capability when system has it */
242	246	#define ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU ((u16)BIT(5))
	247	+/* Panic when a conflict is detected */
	248	+#define ARM64_CPUCAP_PANIC_ON_CONFLICT ((u16)BIT(6))
243	249
244	250	/*
245	251	* CPU errata workarounds that need to be enabled at boot time if one or
@@ -279,9 +285,20 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0; · 33 unmodified lines
279	285
280	286	/*
281	287	* CPU feature used early in the boot based on the boot CPU. All secondary
282		− * CPUs must match the state of the capability as detected by the boot CPU.
	288	+ * CPUs must match the state of the capability as detected by the boot CPU. In
	289	+ * case of a conflict, a kernel panic is triggered.
	290	+ */
	291	+#define ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE \
	292	+ (ARM64_CPUCAP_SCOPE_BOOT_CPU \| ARM64_CPUCAP_PANIC_ON_CONFLICT)
	293	+
	294	+/*
	295	+ * CPU feature used early in the boot based on the boot CPU. It is safe for a
	296	+ * late CPU to have this feature even though the boot CPU hasn't enabled it,
	297	+ * although the feature will not be used by Linux in this case. If the boot CPU
	298	+ * has enabled this feature already, then every late CPU must have it.
283	299	*/
284		−#define ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE ARM64_CPUCAP_SCOPE_BOOT_CPU
	300	+#define ARM64_CPUCAP_BOOT_CPU_FEATURE \
	301	+ (ARM64_CPUCAP_SCOPE_BOOT_CPU \| ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU)
285	302
286	303	struct arm64_cpu_capabilities {
287	304	const char *desc;
@@ -340,18 +357,6 @@ static inline int cpucap_default_scope(const struct arm64_cpu_capabilities *cap) · 52 unmodified lines
340	357	return cap->type & ARM64_CPUCAP_SCOPE_MASK;
341	358	}
342	359
343		−static inline bool
344		−cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
345		−{
346		− return !!(cap->type & ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU);
347		−}
348		−
349		−static inline bool
350		−cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities *cap)
351		−{
352		− return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU);
353		−}
354		−
355	360	/*
356	361	* Generic helper for handling capabilties with multiple (match,enable) pairs
357	362	* of call backs, sharing the same capability bit.
@@ -390,14 +395,16 @@ unsigned long cpu_get_elf_hwcap2(void); · 32 unmodified lines
390	395	#define cpu_set_named_feature(name) cpu_set_feature(cpu_feature(name))
391	396	#define cpu_have_named_feature(name) cpu_have_feature(cpu_feature(name))
392	397
393		−/* System capability check for constant caps */
394		−static __always_inline bool __cpus_have_const_cap(int num)
	398	+static __always_inline bool system_capabilities_finalized(void)
395	399	{
396		− if (num >= ARM64_NCAPS)
397		− return false;
398		− return static_branch_unlikely(&cpu_hwcap_keys[num]);
	400	+ return static_branch_likely(&arm64_const_caps_ready);
399	401	}
400	402
	403	+/*
	404	+ * Test for a capability with a runtime check.
	405	+ *
	406	+ * Before the capability is detected, this returns false.
	407	+ */
401	408	static inline bool cpus_have_cap(unsigned int num)
402	409	{
403	410	if (num >= ARM64_NCAPS)
@@ -405,14 +412,53 @@ static inline bool cpus_have_cap(unsigned int num) · 1 unmodified line
405	412	return test_bit(num, cpu_hwcaps);
406	413	}
407	414
	415	+/*
	416	+ * Test for a capability without a runtime check.
	417	+ *
	418	+ * Before capabilities are finalized, this returns false.
	419	+ * After capabilities are finalized, this is patched to avoid a runtime check.
	420	+ *
	421	+ * @num must be a compile-time constant.
	422	+ */
	423	+static __always_inline bool __cpus_have_const_cap(int num)
	424	+{
	425	+ if (num >= ARM64_NCAPS)
	426	+ return false;
	427	+ return static_branch_unlikely(&cpu_hwcap_keys[num]);
	428	+}
	429	+
	430	+/*
	431	+ * Test for a capability, possibly with a runtime check.
	432	+ *
	433	+ * Before capabilities are finalized, this behaves as cpus_have_cap().
	434	+ * After capabilities are finalized, this is patched to avoid a runtime check.
	435	+ *
	436	+ * @num must be a compile-time constant.
	437	+ */
408	438	static __always_inline bool cpus_have_const_cap(int num)
409	439	{
410		− if (static_branch_likely(&arm64_const_caps_ready))
	440	+ if (system_capabilities_finalized())
411	441	return __cpus_have_const_cap(num);
412	442	else
413	443	return cpus_have_cap(num);
414	444	}
415	445
	446	+/*
	447	+ * Test for a capability without a runtime check.
	448	+ *
	449	+ * Before capabilities are finalized, this will BUG().
	450	+ * After capabilities are finalized, this is patched to avoid a runtime check.
	451	+ *
	452	+ * @num must be a compile-time constant.
	453	+ */
	454	+static __always_inline bool cpus_have_final_cap(int num)
	455	+{
	456	+ if (system_capabilities_finalized())
	457	+ return __cpus_have_const_cap(num);
	458	+ else
	459	+ BUG();
	460	+}
	461	+
416	462	static inline void cpus_set_cap(unsigned int num)
417	463	{
418	464	if (num >= ARM64_NCAPS) {
@@ -447,6 +493,29 @@ cpuid_feature_extract_unsigned_field(u64 features, int field) · 28 unmodified lines
447	493	return cpuid_feature_extract_unsigned_field_width(features, field, 4);
448	494	}
449	495
	496	+/*
	497	+ * Fields that identify the version of the Performance Monitors Extension do
	498	+ * not follow the standard ID scheme. See ARM DDI 0487E.a page D13-2825,
	499	+ * "Alternative ID scheme used for the Performance Monitors Extension version".
	500	+ */
	501	+static inline u64 __attribute_const__
	502	+cpuid_feature_cap_perfmon_field(u64 features, int field, u64 cap)
	503	+{
	504	+ u64 val = cpuid_feature_extract_unsigned_field(features, field);
	505	+ u64 mask = GENMASK_ULL(field + 3, field);
	506	+
	507	+ /* Treat IMPLEMENTATION DEFINED functionality as unimplemented */
	508	+ if (val == 0xf)
	509	+ val = 0;
	510	+
	511	+ if (val > cap) {
	512	+ features &= ~mask;
	513	+ features \|= (cap << field) & mask;
	514	+ }
	515	+
	516	+ return features;
	517	+}
	518	+
450	519	static inline u64 arm64_ftr_mask(const struct arm64_ftr_bits *ftrp)
451	520	{
452	521	return (u64)GENMASK(ftrp->shift + ftrp->width - 1, ftrp->shift);
@@ -590,15 +659,13 @@ static __always_inline bool system_supports_cnp(void) · 137 unmodified lines
590	659	static inline bool system_supports_address_auth(void)
591	660	{
592	661	return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) &&
593		− (cpus_have_const_cap(ARM64_HAS_ADDRESS_AUTH_ARCH) \|\|
594		− cpus_have_const_cap(ARM64_HAS_ADDRESS_AUTH_IMP_DEF));
	662	+ cpus_have_const_cap(ARM64_HAS_ADDRESS_AUTH);
595	663	}
596	664
597	665	static inline bool system_supports_generic_auth(void)
598	666	{
599	667	return IS_ENABLED(CONFIG_ARM64_PTR_AUTH) &&
600		− (cpus_have_const_cap(ARM64_HAS_GENERIC_AUTH_ARCH) \|\|
601		− cpus_have_const_cap(ARM64_HAS_GENERIC_AUTH_IMP_DEF));
	668	+ cpus_have_const_cap(ARM64_HAS_GENERIC_AUTH);
602	669	}
603	670
604	671	static inline bool system_uses_irq_prio_masking(void)
@@ -613,11 +680,6 @@ static inline bool system_has_prio_mask_debugging(void) · 8 unmodified lines
613	680	system_uses_irq_prio_masking();
614	681	}
615	682
616		−static inline bool system_capabilities_finalized(void)
617		−{
618		− return static_branch_likely(&arm64_const_caps_ready);
619		−}
620		−
621	683	#define ARM64_BP_HARDEN_UNKNOWN -1
622	684	#define ARM64_BP_HARDEN_WA_NEEDED 0
623	685	#define ARM64_BP_HARDEN_NOT_REQUIRED 1
@@ -678,6 +740,11 @@ static inline bool cpu_has_hw_af(void) · 54 unmodified lines
678	740	ID_AA64MMFR1_HADBS_SHIFT);
679	741	}
680	742
	743	+#ifdef CONFIG_ARM64_AMU_EXTN
	744	+/* Check whether the cpu supports the Activity Monitors Unit (AMU) */
	745	+extern bool cpu_has_amu_feat(int cpu);
	746	+#endif
	747	+
681	748	#endif /* __ASSEMBLY__ */
682	749
683	750	#endif

arch/arm64/include/asm/esr.h+1−1

@@ -60,7 +60,7 @@ · 59 unmodified lines
60	60	#define ESR_ELx_EC_BKPT32 (0x38)
61	61	/* Unallocated EC: 0x39 */
62	62	#define ESR_ELx_EC_VECTOR32 (0x3A) /* EL2 only */
63		−/* Unallocted EC: 0x3B */
	63	+/* Unallocated EC: 0x3B */
64	64	#define ESR_ELx_EC_BRK64 (0x3C)
65	65	/* Unallocated EC: 0x3D - 0x3F */
66	66	#define ESR_ELx_EC_MAX (0x3F)

arch/arm64/include/asm/kvm_arm.h+1−0

@@ -267,6 +267,7 @@ · 266 unmodified lines
267	267
268	268	/* Hyp Coprocessor Trap Register */
269	269	#define CPTR_EL2_TCPAC (1 << 31)
	270	+#define CPTR_EL2_TAM (1 << 30)
270	271	#define CPTR_EL2_TTA (1 << 20)
271	272	#define CPTR_EL2_TFP (1 << CPTR_EL2_TFP_SHIFT)
272	273	#define CPTR_EL2_TZ (1 << 8)

arch/arm64/include/asm/kvm_asm.h+4−0

@@ -36,6 +36,8 @@ · 35 unmodified lines
36	36	*/
37	37	#define KVM_VECTOR_PREAMBLE (2 * AARCH64_INSN_SIZE)
38	38
	39	+#define __SMCCC_WORKAROUND_1_SMC_SZ 36
	40	+
39	41	#ifndef __ASSEMBLY__
40	42
41	43	#include <linux/mm.h>
@@ -75,6 +77,8 @@ extern void __vgic_v3_init_lrs(void); · 33 unmodified lines
75	77
76	78	extern u32 __kvm_get_mdcr_el2(void);
77	79
	80	+extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ];
	81	+
78	82	/* Home-grown __this_cpu_{ptr,read} variants that always work at HYP */
79	83	#define __hyp_this_cpu_ptr(sym) \
80	84	({ \

arch/arm64/include/asm/kvm_mmu.h+4−5

@@ -481,7 +481,7 @@ static inline void *kvm_get_hyp_vector(void) · 480 unmodified lines
481	481	int slot = -1;
482	482
483	483	if (cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR) && data->fn) {
484		− vect = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs_start));
	484	+ vect = kern_hyp_va(kvm_ksym_ref(__bp_harden_hyp_vecs));
485	485	slot = data->hyp_vectors_slot;
486	486	}
487	487
@@ -510,14 +510,13 @@ static inline int kvm_map_vectors(void) · 22 unmodified lines
510	510	* HBP + HEL2 -> use hardened vertors and use exec mapping
511	511	*/
512	512	if (cpus_have_const_cap(ARM64_HARDEN_BRANCH_PREDICTOR)) {
513		− __kvm_bp_vect_base = kvm_ksym_ref(__bp_harden_hyp_vecs_start);
	513	+ __kvm_bp_vect_base = kvm_ksym_ref(__bp_harden_hyp_vecs);
514	514	__kvm_bp_vect_base = kern_hyp_va(__kvm_bp_vect_base);
515	515	}
516	516
517	517	if (cpus_have_const_cap(ARM64_HARDEN_EL2_VECTORS)) {
518		− phys_addr_t vect_pa = __pa_symbol(__bp_harden_hyp_vecs_start);
519		− unsigned long size = (__bp_harden_hyp_vecs_end -
520		− __bp_harden_hyp_vecs_start);
	518	+ phys_addr_t vect_pa = __pa_symbol(__bp_harden_hyp_vecs);
	519	+ unsigned long size = __BP_HARDEN_HYP_VECS_SZ;
521	520
522	521	/*
523	522	* Always allocate a spare vector slot, as we don't

arch/arm64/include/asm/memory.h+1−0

@@ -54,6 +54,7 @@ · 53 unmodified lines
54	54	#define MODULES_VADDR (BPF_JIT_REGION_END)
55	55	#define MODULES_VSIZE (SZ_128M)
56	56	#define VMEMMAP_START (-VMEMMAP_SIZE - SZ_2M)
	57	+#define VMEMMAP_END (VMEMMAP_START + VMEMMAP_SIZE)
57	58	#define PCI_IO_END (VMEMMAP_START - SZ_2M)
58	59	#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
59	60	#define FIXADDR_TOP (PCI_IO_START - SZ_2M)

arch/arm64/include/asm/mmu.h+6−4

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	13	#define TTBR_ASID_MASK (UL(0xffff) << 48)
14	14
15	15	#define BP_HARDEN_EL2_SLOTS 4
	16	+#define __BP_HARDEN_HYP_VECS_SZ (BP_HARDEN_EL2_SLOTS * SZ_2K)
16	17
17	18	#ifndef __ASSEMBLY__
18	19
@@ -23,9 +24,9 @@ typedef struct { · 4 unmodified lines
23	24	} mm_context_t;
24	25
25	26	/*
26		− * This macro is only used by the TLBI code, which cannot race with an
27		− * ASID change and therefore doesn't need to reload the counter using
28		− * atomic64_read.
	27	+ * This macro is only used by the TLBI and low-level switch_mm() code,
	28	+ * neither of which can race with an ASID change. We therefore don't
	29	+ * need to reload the counter using atomic64_read().
29	30	*/
30	31	#define ASID(mm) ((mm)->context.id.counter & 0xffff)
31	32
@@ -43,7 +44,8 @@ struct bp_hardening_data { · 11 unmodified lines
43	44
44	45	#if (defined(CONFIG_HARDEN_BRANCH_PREDICTOR) \|\| \
45	46	defined(CONFIG_HARDEN_EL2_VECTORS))
46		−extern char __bp_harden_hyp_vecs_start[], __bp_harden_hyp_vecs_end[];
	47	+
	48	+extern char __bp_harden_hyp_vecs[];
47	49	extern atomic_t arm64_el2_vector_last_slot;
48	50	#endif /* CONFIG_HARDEN_BRANCH_PREDICTOR \|\| CONFIG_HARDEN_EL2_VECTORS */
49	51

arch/arm64/include/asm/mmu_context.h+2−0

@@ -46,6 +46,8 @@ static inline void cpu_set_reserved_ttbr0(void) · 45 unmodified lines
46	46	isb();
47	47	}
48	48
	49	+void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm);
	50	+
49	51	static inline void cpu_switch_mm(pgd_t pgd, struct mm_struct mm)
50	52	{
51	53	BUG_ON(pgd == swapper_pg_dir);

arch/arm64/include/asm/page.h+4−0

@@ -21,6 +21,10 @@ extern void __cpu_copy_user_page(void to, const void from, · 20 unmodified lines
21	21	extern void copy_page(void to, const void from);
22	22	extern void clear_page(void *to);
23	23
	24	+#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \
	25	+ alloc_page_vma(GFP_HIGHUSER \| __GFP_ZERO \| movableflags, vma, vaddr)
	26	+#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE
	27	+
24	28	#define clear_user_page(addr,vaddr,pg) __cpu_clear_user_page(addr, vaddr)
25	29	#define copy_user_page(to,from,vaddr,pg) __cpu_copy_user_page(to, from, vaddr)
26	30

arch/arm64/include/asm/perf_event.h+2−1

@@ -176,9 +176,10 @@ · 175 unmodified lines
176	176	#define ARMV8_PMU_PMCR_X (1 << 4) /* Export to ETM */
177	177	#define ARMV8_PMU_PMCR_DP (1 << 5) /* Disable CCNT if non-invasive debug*/
178	178	#define ARMV8_PMU_PMCR_LC (1 << 6) /* Overflow on 64 bit cycle counter */
	179	+#define ARMV8_PMU_PMCR_LP (1 << 7) /* Long event counter enable */
179	180	#define ARMV8_PMU_PMCR_N_SHIFT 11 /* Number of counters supported */
180	181	#define ARMV8_PMU_PMCR_N_MASK 0x1f
181		−#define ARMV8_PMU_PMCR_MASK 0x7f /* Mask for writable bits */
	182	+#define ARMV8_PMU_PMCR_MASK 0xff /* Mask for writable bits */
182	183
183	184	/*
184	185	* PMOVSR: counters overflow flag status reg

arch/arm64/include/asm/pointer_auth.h+22−28

@@ -22,7 +22,7 @@ struct ptrauth_key { · 21 unmodified lines
22	22	* We give each process its own keys, which are shared by all threads. The keys
23	23	* are inherited upon fork(), and reinitialised upon exec*().
24	24	*/
25		−struct ptrauth_keys {
	25	+struct ptrauth_keys_user {
26	26	struct ptrauth_key apia;
27	27	struct ptrauth_key apib;
28	28	struct ptrauth_key apda;
@@ -30,7 +30,11 @@ struct ptrauth_keys { · 1 unmodified line
30	30	struct ptrauth_key apga;
31	31	};
32	32
33		−static inline void ptrauth_keys_init(struct ptrauth_keys *keys)
	33	+struct ptrauth_keys_kernel {
	34	+ struct ptrauth_key apia;
	35	+};
	36	+
	37	+static inline void ptrauth_keys_init_user(struct ptrauth_keys_user *keys)
34	38	{
35	39	if (system_supports_address_auth()) {
36	40	get_random_bytes(&keys->apia, sizeof(keys->apia));
@@ -50,48 +54,38 @@ do { \ · 13 unmodified lines
50	54	write_sysreg_s(__pki_v.hi, SYS_ ## k ## KEYHI_EL1); \
51	55	} while (0)
52	56
53		−static inline void ptrauth_keys_switch(struct ptrauth_keys *keys)
	57	+static __always_inline void ptrauth_keys_init_kernel(struct ptrauth_keys_kernel *keys)
54	58	{
55		− if (system_supports_address_auth()) {
56		− __ptrauth_key_install(APIA, keys->apia);
57		− __ptrauth_key_install(APIB, keys->apib);
58		− __ptrauth_key_install(APDA, keys->apda);
59		− __ptrauth_key_install(APDB, keys->apdb);
60		− }
	59	+ if (system_supports_address_auth())
	60	+ get_random_bytes(&keys->apia, sizeof(keys->apia));
	61	+}
61	62
62		− if (system_supports_generic_auth())
63		− __ptrauth_key_install(APGA, keys->apga);
	63	+static __always_inline void ptrauth_keys_switch_kernel(struct ptrauth_keys_kernel *keys)
	64	+{
	65	+ if (system_supports_address_auth())
	66	+ __ptrauth_key_install(APIA, keys->apia);
64	67	}
65	68
66	69	extern int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg);
67	70
68		−/*
69		− * The EL0 pointer bits used by a pointer authentication code.
70		− * This is dependent on TBI0 being enabled, or bits 63:56 would also apply.
71		− */
72		−#define ptrauth_user_pac_mask() GENMASK(54, vabits_actual)
73		−
74		−/* Only valid for EL0 TTBR0 instruction pointers */
75	71	static inline unsigned long ptrauth_strip_insn_pac(unsigned long ptr)
76	72	{
77		− return ptr & ~ptrauth_user_pac_mask();
	73	+ return ptrauth_clear_pac(ptr);
78	74	}
79	75
80	76	#define ptrauth_thread_init_user(tsk) \
81		−do { \
82		− struct task_struct *__ptiu_tsk = (tsk); \
83		− ptrauth_keys_init(&__ptiu_tsk->thread.keys_user); \
84		− ptrauth_keys_switch(&__ptiu_tsk->thread.keys_user); \
85		−} while (0)
86		−
87		−#define ptrauth_thread_switch(tsk) \
88		− ptrauth_keys_switch(&(tsk)->thread.keys_user)
	77	+ ptrauth_keys_init_user(&(tsk)->thread.keys_user)
	78	+#define ptrauth_thread_init_kernel(tsk) \
	79	+ ptrauth_keys_init_kernel(&(tsk)->thread.keys_kernel)
	80	+#define ptrauth_thread_switch_kernel(tsk) \
	81	+ ptrauth_keys_switch_kernel(&(tsk)->thread.keys_kernel)
89	82
90	83	#else /* CONFIG_ARM64_PTR_AUTH */
91	84	#define ptrauth_prctl_reset_keys(tsk, arg) (-EINVAL)
92	85	#define ptrauth_strip_insn_pac(lr) (lr)
93	86	#define ptrauth_thread_init_user(tsk)
94		−#define ptrauth_thread_switch(tsk)
	87	+#define ptrauth_thread_init_kernel(tsk)
	88	+#define ptrauth_thread_switch_kernel(tsk)
95	89	#endif /* CONFIG_ARM64_PTR_AUTH */
96	90
97	91	#endif /* __ASM_POINTER_AUTH_H */

arch/arm64/include/asm/proc-fns.h+0−2

@@ -13,11 +13,9 @@ · 12 unmodified lines
13	13
14	14	#include <asm/page.h>
15	15
16		−struct mm_struct;
17	16	struct cpu_suspend_ctx;
18	17
19	18	extern void cpu_do_idle(void);
20		−extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm);
21	19	extern void cpu_do_suspend(struct cpu_suspend_ctx *ptr);
22	20	extern u64 cpu_do_resume(phys_addr_t ptr, u64 idmap_ttbr);
23	21

arch/arm64/include/asm/processor.h+2−1

@@ -148,7 +148,8 @@ struct thread_struct { · 147 unmodified lines
148	148	unsigned long fault_code; /* ESR_EL1 value */
149	149	struct debug_info debug; /* debugging */
150	150	#ifdef CONFIG_ARM64_PTR_AUTH
151		− struct ptrauth_keys keys_user;
	151	+ struct ptrauth_keys_user keys_user;
	152	+ struct ptrauth_keys_kernel keys_kernel;
152	153	#endif
153	154	};
154	155

arch/arm64/include/asm/smp.h+12−0

@@ -23,6 +23,14 @@ · 22 unmodified lines
23	23	#define CPU_STUCK_REASON_52_BIT_VA (UL(1) << CPU_STUCK_REASON_SHIFT)
24	24	#define CPU_STUCK_REASON_NO_GRAN (UL(2) << CPU_STUCK_REASON_SHIFT)
25	25
	26	+/* Possible options for __cpu_setup */
	27	+/* Option to setup primary cpu */
	28	+#define ARM64_CPU_BOOT_PRIMARY (1)
	29	+/* Option to setup secondary cpus */
	30	+#define ARM64_CPU_BOOT_SECONDARY (2)
	31	+/* Option to setup cpus for different cpu run time services */
	32	+#define ARM64_CPU_RUNTIME (3)
	33	+
26	34	#ifndef __ASSEMBLY__
27	35
28	36	#include <asm/percpu.h>
@@ -30,6 +38,7 @@ · 1 unmodified line
30	38	#include <linux/threads.h>
31	39	#include <linux/cpumask.h>
32	40	#include <linux/thread_info.h>
	41	+#include <asm/pointer_auth.h>
33	42
34	43	DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);
35	44
@@ -87,6 +96,9 @@ asmlinkage void secondary_start_kernel(void); · 51 unmodified lines
87	96	struct secondary_data {
88	97	void *stack;
89	98	struct task_struct *task;
	99	+#ifdef CONFIG_ARM64_PTR_AUTH
	100	+ struct ptrauth_keys_kernel ptrauth_key;
	101	+#endif
90	102	long status;
91	103	};
92	104

arch/arm64/include/asm/stackprotector.h+5−0

@@ -15,6 +15,7 @@ · 14 unmodified lines
15	15
16	16	#include <linux/random.h>
17	17	#include <linux/version.h>
	18	+#include <asm/pointer_auth.h>
18	19
19	20	extern unsigned long __stack_chk_guard;
20	21
@@ -26,6 +27,7 @@ extern unsigned long __stack_chk_guard; · 5 unmodified lines
26	27	*/
27	28	static __always_inline void boot_init_stack_canary(void)
28	29	{
	30	+#if defined(CONFIG_STACKPROTECTOR)
29	31	unsigned long canary;
30	32
31	33	/* Try to get a semi random initial value. */
@@ -36,6 +38,9 @@ static __always_inline void boot_init_stack_canary(void) · 4 unmodified lines
36	38	current->stack_canary = canary;
37	39	if (!IS_ENABLED(CONFIG_STACKPROTECTOR_PER_TASK))
38	40	__stack_chk_guard = current->stack_canary;
	41	+#endif
	42	+ ptrauth_thread_init_kernel(current);
	43	+ ptrauth_thread_switch_kernel(current);
39	44	}
40	45
41	46	#endif /* _ASM_STACKPROTECTOR_H */

arch/arm64/include/asm/sysreg.h+48−0

@@ -386,6 +386,42 @@ · 385 unmodified lines
386	386	#define SYS_TPIDR_EL0 sys_reg(3, 3, 13, 0, 2)
387	387	#define SYS_TPIDRRO_EL0 sys_reg(3, 3, 13, 0, 3)
388	388
	389	+/* Definitions for system register interface to AMU for ARMv8.4 onwards */
	390	+#define SYS_AM_EL0(crm, op2) sys_reg(3, 3, 13, (crm), (op2))
	391	+#define SYS_AMCR_EL0 SYS_AM_EL0(2, 0)
	392	+#define SYS_AMCFGR_EL0 SYS_AM_EL0(2, 1)
	393	+#define SYS_AMCGCR_EL0 SYS_AM_EL0(2, 2)
	394	+#define SYS_AMUSERENR_EL0 SYS_AM_EL0(2, 3)
	395	+#define SYS_AMCNTENCLR0_EL0 SYS_AM_EL0(2, 4)
	396	+#define SYS_AMCNTENSET0_EL0 SYS_AM_EL0(2, 5)
	397	+#define SYS_AMCNTENCLR1_EL0 SYS_AM_EL0(3, 0)
	398	+#define SYS_AMCNTENSET1_EL0 SYS_AM_EL0(3, 1)
	399	+
	400	+/*
	401	+ * Group 0 of activity monitors (architected):
	402	+ * op0 op1 CRn CRm op2
	403	+ * Counter: 11 011 1101 010:n<3> n<2:0>
	404	+ * Type: 11 011 1101 011:n<3> n<2:0>
	405	+ * n: 0-15
	406	+ *
	407	+ * Group 1 of activity monitors (auxiliary):
	408	+ * op0 op1 CRn CRm op2
	409	+ * Counter: 11 011 1101 110:n<3> n<2:0>
	410	+ * Type: 11 011 1101 111:n<3> n<2:0>
	411	+ * n: 0-15
	412	+ */
	413	+
	414	+#define SYS_AMEVCNTR0_EL0(n) SYS_AM_EL0(4 + ((n) >> 3), (n) & 7)
	415	+#define SYS_AMEVTYPE0_EL0(n) SYS_AM_EL0(6 + ((n) >> 3), (n) & 7)
	416	+#define SYS_AMEVCNTR1_EL0(n) SYS_AM_EL0(12 + ((n) >> 3), (n) & 7)
	417	+#define SYS_AMEVTYPE1_EL0(n) SYS_AM_EL0(14 + ((n) >> 3), (n) & 7)
	418	+
	419	+/* AMU v1: Fixed (architecturally defined) activity monitors */
	420	+#define SYS_AMEVCNTR0_CORE_EL0 SYS_AMEVCNTR0_EL0(0)
	421	+#define SYS_AMEVCNTR0_CONST_EL0 SYS_AMEVCNTR0_EL0(1)
	422	+#define SYS_AMEVCNTR0_INST_RET_EL0 SYS_AMEVCNTR0_EL0(2)
	423	+#define SYS_AMEVCNTR0_MEM_STALL SYS_AMEVCNTR0_EL0(3)
	424	+
389	425	#define SYS_CNTFRQ_EL0 sys_reg(3, 3, 14, 0, 0)
390	426
391	427	#define SYS_CNTP_TVAL_EL0 sys_reg(3, 3, 14, 2, 0)
@@ -598,6 +634,7 @@ · 206 unmodified lines
598	634	#define ID_AA64PFR0_CSV3_SHIFT 60
599	635	#define ID_AA64PFR0_CSV2_SHIFT 56
600	636	#define ID_AA64PFR0_DIT_SHIFT 48
	637	+#define ID_AA64PFR0_AMU_SHIFT 44
601	638	#define ID_AA64PFR0_SVE_SHIFT 32
602	639	#define ID_AA64PFR0_RAS_SHIFT 28
603	640	#define ID_AA64PFR0_GIC_SHIFT 24
@@ -608,6 +645,7 @@ · 4 unmodified lines
608	645	#define ID_AA64PFR0_EL1_SHIFT 4
609	646	#define ID_AA64PFR0_EL0_SHIFT 0
610	647
	648	+#define ID_AA64PFR0_AMU 0x1
611	649	#define ID_AA64PFR0_SVE 0x1
612	650	#define ID_AA64PFR0_RAS_V1 0x1
613	651	#define ID_AA64PFR0_FP_NI 0xf
@@ -702,6 +740,16 @@ · 88 unmodified lines
702	740	#define ID_AA64DFR0_TRACEVER_SHIFT 4
703	741	#define ID_AA64DFR0_DEBUGVER_SHIFT 0
704	742
	743	+#define ID_AA64DFR0_PMUVER_8_0 0x1
	744	+#define ID_AA64DFR0_PMUVER_8_1 0x4
	745	+#define ID_AA64DFR0_PMUVER_8_4 0x5
	746	+#define ID_AA64DFR0_PMUVER_8_5 0x6
	747	+#define ID_AA64DFR0_PMUVER_IMP_DEF 0xf
	748	+
	749	+#define ID_DFR0_PERFMON_SHIFT 24
	750	+
	751	+#define ID_DFR0_PERFMON_8_1 0x4
	752	+
705	753	#define ID_ISAR5_RDM_SHIFT 24
706	754	#define ID_ISAR5_CRC32_SHIFT 16
707	755	#define ID_ISAR5_SHA2_SHIFT 12

arch/arm64/include/asm/topology.h+9−0

@@ -16,6 +16,15 @@ int pcibus_to_node(struct pci_bus *bus); · 15 unmodified lines
16	16
17	17	#include <linux/arch_topology.h>
18	18
	19	+#ifdef CONFIG_ARM64_AMU_EXTN
	20	+/*
	21	+ * Replace task scheduler's default counter-based
	22	+ * frequency-invariance scale factor setting.
	23	+ */
	24	+void topology_scale_freq_tick(void);
	25	+#define arch_scale_freq_tick topology_scale_freq_tick
	26	+#endif /* CONFIG_ARM64_AMU_EXTN */
	27	+
19	28	/* Replace task scheduler's default frequency-invariant accounting */
20	29	#define arch_scale_freq_capacity topology_get_freq_scale
21	30

arch/arm64/kernel/Makefile+1−1

@@ -21,7 +21,7 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ · 20 unmodified lines
21	21	smp.o smp_spin_table.o topology.o smccc-call.o \
22	22	syscall.o
23	23
24		−extra-$(CONFIG_EFI) := efi-entry.o
	24	+targets += efi-entry.o
25	25
26	26	OBJCOPYFLAGS := --prefix-symbols=__efistub_
27	27	$(obj)/%.stub.o: $(obj)/%.o FORCE

arch/arm64/kernel/armv8_deprecated.c+1−1

@@ -630,7 +630,7 @@ static int __init armv8_deprecated_init(void) · 629 unmodified lines
630	630	register_insn_emulation(&cp15_barrier_ops);
631	631
632	632	if (IS_ENABLED(CONFIG_SETEND_EMULATION)) {
633		− if(system_supports_mixed_endian_el0())
	633	+ if (system_supports_mixed_endian_el0())
634	634	register_insn_emulation(&setend_ops);
635	635	else
636	636	pr_info("setend instruction emulation is not supported on this system\n");

arch/arm64/kernel/asm-offsets.c+16−0

@@ -40,6 +40,10 @@ int main(void) · 39 unmodified lines
40	40	#endif
41	41	BLANK();
42	42	DEFINE(THREAD_CPU_CONTEXT, offsetof(struct task_struct, thread.cpu_context));
	43	+#ifdef CONFIG_ARM64_PTR_AUTH
	44	+ DEFINE(THREAD_KEYS_USER, offsetof(struct task_struct, thread.keys_user));
	45	+ DEFINE(THREAD_KEYS_KERNEL, offsetof(struct task_struct, thread.keys_kernel));
	46	+#endif
43	47	BLANK();
44	48	DEFINE(S_X0, offsetof(struct pt_regs, regs[0]));
45	49	DEFINE(S_X2, offsetof(struct pt_regs, regs[2]));
@@ -88,6 +92,9 @@ int main(void) · 42 unmodified lines
88	92	BLANK();
89	93	DEFINE(CPU_BOOT_STACK, offsetof(struct secondary_data, stack));
90	94	DEFINE(CPU_BOOT_TASK, offsetof(struct secondary_data, task));
	95	+#ifdef CONFIG_ARM64_PTR_AUTH
	96	+ DEFINE(CPU_BOOT_PTRAUTH_KEY, offsetof(struct secondary_data, ptrauth_key));
	97	+#endif
91	98	BLANK();
92	99	#ifdef CONFIG_KVM_ARM_HOST
93	100	DEFINE(VCPU_CONTEXT, offsetof(struct kvm_vcpu, arch.ctxt));
@@ -127,6 +134,15 @@ int main(void) · 33 unmodified lines
127	134	#ifdef CONFIG_ARM_SDE_INTERFACE
128	135	DEFINE(SDEI_EVENT_INTREGS, offsetof(struct sdei_registered_event, interrupted_regs));
129	136	DEFINE(SDEI_EVENT_PRIORITY, offsetof(struct sdei_registered_event, priority));
	137	+#endif
	138	+#ifdef CONFIG_ARM64_PTR_AUTH
	139	+ DEFINE(PTRAUTH_USER_KEY_APIA, offsetof(struct ptrauth_keys_user, apia));
	140	+ DEFINE(PTRAUTH_USER_KEY_APIB, offsetof(struct ptrauth_keys_user, apib));
	141	+ DEFINE(PTRAUTH_USER_KEY_APDA, offsetof(struct ptrauth_keys_user, apda));
	142	+ DEFINE(PTRAUTH_USER_KEY_APDB, offsetof(struct ptrauth_keys_user, apdb));
	143	+ DEFINE(PTRAUTH_USER_KEY_APGA, offsetof(struct ptrauth_keys_user, apga));
	144	+ DEFINE(PTRAUTH_KERNEL_KEY_APIA, offsetof(struct ptrauth_keys_kernel, apia));
	145	+ BLANK();
130	146	#endif
131	147	return 0;
132	148	}

arch/arm64/kernel/cpu-reset.S+1−1

@@ -32,7 +32,7 @@ · 31 unmodified lines
32	32	ENTRY(__cpu_soft_restart)
33	33	/* Clear sctlr_el1 flags. */
34	34	mrs x12, sctlr_el1
35		− ldr x13, =SCTLR_ELx_FLAGS
	35	+ mov_q x13, SCTLR_ELx_FLAGS
36	36	bic x12, x12, x13
37	37	pre_disable_mmu_workaround
38	38	msr sctlr_el1, x12

arch/arm64/kernel/cpu_errata.c+8−10

@@ -11,6 +11,7 @@ · 10 unmodified lines
11	11	#include <asm/cpu.h>
12	12	#include <asm/cputype.h>
13	13	#include <asm/cpufeature.h>
	14	+#include <asm/kvm_asm.h>
14	15	#include <asm/smp_plat.h>
15	16
16	17	static bool __maybe_unused
@@ -113,13 +114,10 @@ atomic_t arm64_el2_vector_last_slot = ATOMIC_INIT(-1); · 96 unmodified lines
113	114	DEFINE_PER_CPU_READ_MOSTLY(struct bp_hardening_data, bp_hardening_data);
114	115
115	116	#ifdef CONFIG_KVM_INDIRECT_VECTORS
116		−extern char __smccc_workaround_1_smc_start[];
117		−extern char __smccc_workaround_1_smc_end[];
118		−
119	117	static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start,
120	118	const char *hyp_vecs_end)
121	119	{
122		− void dst = lm_alias(__bp_harden_hyp_vecs_start + slot SZ_2K);
	120	+ void dst = lm_alias(__bp_harden_hyp_vecs + slot SZ_2K);
123	121	int i;
124	122
125	123	for (i = 0; i < SZ_2K; i += 0x80)
@@ -163,9 +161,6 @@ static void install_bp_hardening_cb(bp_hardening_cb_t fn, · 37 unmodified lines
163	161	raw_spin_unlock(&bp_lock);
164	162	}
165	163	#else
166		−#define __smccc_workaround_1_smc_start NULL
167		−#define __smccc_workaround_1_smc_end NULL
168		−
169	164	static void install_bp_hardening_cb(bp_hardening_cb_t fn,
170	165	const char *hyp_vecs_start,
171	166	const char *hyp_vecs_end)
@@ -176,7 +171,7 @@ static void install_bp_hardening_cb(bp_hardening_cb_t fn, · 4 unmodified lines
176	171
177	172	#include <linux/arm-smccc.h>
178	173
179		−static void call_smc_arch_workaround_1(void)
	174	+static void __maybe_unused call_smc_arch_workaround_1(void)
180	175	{
181	176	arm_smccc_1_1_smc(ARM_SMCCC_ARCH_WORKAROUND_1, NULL);
182	177	}
@@ -239,11 +234,14 @@ static int detect_harden_bp_fw(void) · 56 unmodified lines
239	234	smccc_end = NULL;
240	235	break;
241	236
	237	+#if IS_ENABLED(CONFIG_KVM_ARM_HOST)
242	238	case SMCCC_CONDUIT_SMC:
243	239	cb = call_smc_arch_workaround_1;
244		− smccc_start = __smccc_workaround_1_smc_start;
245		− smccc_end = __smccc_workaround_1_smc_end;
	240	+ smccc_start = __smccc_workaround_1_smc;
	241	+ smccc_end = __smccc_workaround_1_smc +
	242	+ __SMCCC_WORKAROUND_1_SMC_SZ;
246	243	break;
	244	+#endif
247	245
248	246	default:
249	247	return -1;

arch/arm64/kernel/cpu_ops.c+9−2

@@ -15,10 +15,12 @@ · 14 unmodified lines
15	15	#include <asm/smp_plat.h>
16	16
17	17	extern const struct cpu_operations smp_spin_table_ops;
	18	+#ifdef CONFIG_ARM64_ACPI_PARKING_PROTOCOL
18	19	extern const struct cpu_operations acpi_parking_protocol_ops;
	20	+#endif
19	21	extern const struct cpu_operations cpu_psci_ops;
20	22
21		−const struct cpu_operations *cpu_ops[NR_CPUS] __ro_after_init;
	23	+static const struct cpu_operations *cpu_ops[NR_CPUS] __ro_after_init;
22	24
23	25	static const struct cpu_operations *const dt_supported_cpu_ops[] __initconst = {
24	26	&smp_spin_table_ops,
@@ -94,7 +96,7 @@ static const char *__init cpu_read_enable_method(int cpu) · 69 unmodified lines
94	96	/*
95	97	* Read a cpu's enable method and record it in cpu_ops.
96	98	*/
97		−int __init cpu_read_ops(int cpu)
	99	+int __init init_cpu_ops(int cpu)
98	100	{
99	101	const char *enable_method = cpu_read_enable_method(cpu);
100	102
@@ -109,3 +111,8 @@ int __init cpu_read_ops(int cpu) · 8 unmodified lines
109	111
110	112	return 0;
111	113	}
	114	+
	115	+const struct cpu_operations *get_cpu_ops(int cpu)
	116	+{
	117	+ return cpu_ops[cpu];
	118	+}

arch/arm64/kernel/cpufeature.c+142−23

@@ -116,6 +116,8 @@ cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused) · 115 unmodified lines
116	116
117	117	static void cpu_enable_cnp(struct arm64_cpu_capabilities const *cap);
118	118
	119	+static bool __system_matches_cap(unsigned int n);
	120	+
119	121	/*
120	122	* NOTE: Any changes to the visibility of features should be kept in
121	123	* sync with the documentation of the CPU feature register ABI.
@@ -163,6 +165,7 @@ static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = { · 41 unmodified lines
163	165	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV3_SHIFT, 4, 0),
164	166	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_CSV2_SHIFT, 4, 0),
165	167	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_DIT_SHIFT, 4, 0),
	168	+ ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_AMU_SHIFT, 4, 0),
166	169	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
167	170	FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_SVE_SHIFT, 4, 0),
168	171	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_RAS_SHIFT, 4, 0),
@@ -551,7 +554,7 @@ static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new) · 382 unmodified lines
551	554
552	555	BUG_ON(!reg);
553	556
554		− for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
	557	+ for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
555	558	u64 ftr_mask = arm64_ftr_mask(ftrp);
556	559	s64 ftr_new = arm64_ftr_value(ftrp, new);
557	560
@@ -1222,6 +1225,57 @@ static bool has_hw_dbm(const struct arm64_cpu_capabilities *cap, · 664 unmodified lines
1222	1225
1223	1226	#endif
1224	1227
	1228	+#ifdef CONFIG_ARM64_AMU_EXTN
	1229	+
	1230	+/*
	1231	+ * The "amu_cpus" cpumask only signals that the CPU implementation for the
	1232	+ * flagged CPUs supports the Activity Monitors Unit (AMU) but does not provide
	1233	+ * information regarding all the events that it supports. When a CPU bit is
	1234	+ * set in the cpumask, the user of this feature can only rely on the presence
	1235	+ * of the 4 fixed counters for that CPU. But this does not guarantee that the
	1236	+ * counters are enabled or access to these counters is enabled by code
	1237	+ * executed at higher exception levels (firmware).
	1238	+ */
	1239	+static struct cpumask amu_cpus __read_mostly;
	1240	+
	1241	+bool cpu_has_amu_feat(int cpu)
	1242	+{
	1243	+ return cpumask_test_cpu(cpu, &amu_cpus);
	1244	+}
	1245	+
	1246	+/* Initialize the use of AMU counters for frequency invariance */
	1247	+extern void init_cpu_freq_invariance_counters(void);
	1248	+
	1249	+static void cpu_amu_enable(struct arm64_cpu_capabilities const *cap)
	1250	+{
	1251	+ if (has_cpuid_feature(cap, SCOPE_LOCAL_CPU)) {
	1252	+ pr_info("detected CPU%d: Activity Monitors Unit (AMU)\n",
	1253	+ smp_processor_id());
	1254	+ cpumask_set_cpu(smp_processor_id(), &amu_cpus);
	1255	+ init_cpu_freq_invariance_counters();
	1256	+ }
	1257	+}
	1258	+
	1259	+static bool has_amu(const struct arm64_cpu_capabilities *cap,
	1260	+ int __unused)
	1261	+{
	1262	+ /*
	1263	+ * The AMU extension is a non-conflicting feature: the kernel can
	1264	+ * safely run a mix of CPUs with and without support for the
	1265	+ * activity monitors extension. Therefore, unconditionally enable
	1266	+ * the capability to allow any late CPU to use the feature.
	1267	+ *
	1268	+ * With this feature unconditionally enabled, the cpu_enable
	1269	+ * function will be called for all CPUs that match the criteria,
	1270	+ * including secondary and hotplugged, marking this feature as
	1271	+ * present on that respective CPU. The enable function will also
	1272	+ * print a detection message.
	1273	+ */
	1274	+
	1275	+ return true;
	1276	+}
	1277	+#endif
	1278	+
1225	1279	#ifdef CONFIG_ARM64_VHE
1226	1280	static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused)
1227	1281	{
@@ -1316,10 +1370,18 @@ static void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused) · 88 unmodified lines
1316	1370	#endif /* CONFIG_ARM64_RAS_EXTN */
1317	1371
1318	1372	#ifdef CONFIG_ARM64_PTR_AUTH
1319		−static void cpu_enable_address_auth(struct arm64_cpu_capabilities const *cap)
	1373	+static bool has_address_auth(const struct arm64_cpu_capabilities *entry,
	1374	+ int __unused)
1320	1375	{
1321		− sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ENIA \| SCTLR_ELx_ENIB \|
1322		− SCTLR_ELx_ENDA \| SCTLR_ELx_ENDB);
	1376	+ return __system_matches_cap(ARM64_HAS_ADDRESS_AUTH_ARCH) \|\|
	1377	+ __system_matches_cap(ARM64_HAS_ADDRESS_AUTH_IMP_DEF);
	1378	+}
	1379	+
	1380	+static bool has_generic_auth(const struct arm64_cpu_capabilities *entry,
	1381	+ int __unused)
	1382	+{
	1383	+ return __system_matches_cap(ARM64_HAS_GENERIC_AUTH_ARCH) \|\|
	1384	+ __system_matches_cap(ARM64_HAS_GENERIC_AUTH_IMP_DEF);
1323	1385	}
1324	1386	#endif /* CONFIG_ARM64_PTR_AUTH */
1325	1387
@@ -1347,6 +1409,25 @@ static bool can_use_gic_priorities(const struct arm64_cpu_capabilities *entry, · 21 unmodified lines
1347	1409	}
1348	1410	#endif
1349	1411
	1412	+/* Internal helper functions to match cpu capability type */
	1413	+static bool
	1414	+cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
	1415	+{
	1416	+ return !!(cap->type & ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU);
	1417	+}
	1418	+
	1419	+static bool
	1420	+cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities *cap)
	1421	+{
	1422	+ return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU);
	1423	+}
	1424	+
	1425	+static bool
	1426	+cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap)
	1427	+{
	1428	+ return !!(cap->type & ARM64_CPUCAP_PANIC_ON_CONFLICT);
	1429	+}
	1430	+
1350	1431	static const struct arm64_cpu_capabilities arm64_features[] = {
1351	1432	{
1352	1433	.desc = "GIC system register CPU interface",
@@ -1499,6 +1580,24 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 146 unmodified lines
1499	1580	.cpu_enable = cpu_clear_disr,
1500	1581	},
1501	1582	#endif /* CONFIG_ARM64_RAS_EXTN */
	1583	+#ifdef CONFIG_ARM64_AMU_EXTN
	1584	+ {
	1585	+ /*
	1586	+ * The feature is enabled by default if CONFIG_ARM64_AMU_EXTN=y.
	1587	+ * Therefore, don't provide .desc as we don't want the detection
	1588	+ * message to be shown until at least one CPU is detected to
	1589	+ * support the feature.
	1590	+ */
	1591	+ .capability = ARM64_HAS_AMU_EXTN,
	1592	+ .type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
	1593	+ .matches = has_amu,
	1594	+ .sys_reg = SYS_ID_AA64PFR0_EL1,
	1595	+ .sign = FTR_UNSIGNED,
	1596	+ .field_pos = ID_AA64PFR0_AMU_SHIFT,
	1597	+ .min_field_value = ID_AA64PFR0_AMU,
	1598	+ .cpu_enable = cpu_amu_enable,
	1599	+ },
	1600	+#endif /* CONFIG_ARM64_AMU_EXTN */
1502	1601	{
1503	1602	.desc = "Data cache clean to the PoU not required for I/D coherence",
1504	1603	.capability = ARM64_HAS_CACHE_IDC,
@@ -1592,24 +1691,27 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 87 unmodified lines
1592	1691	{
1593	1692	.desc = "Address authentication (architected algorithm)",
1594	1693	.capability = ARM64_HAS_ADDRESS_AUTH_ARCH,
1595		− .type = ARM64_CPUCAP_SYSTEM_FEATURE,
	1694	+ .type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
1596	1695	.sys_reg = SYS_ID_AA64ISAR1_EL1,
1597	1696	.sign = FTR_UNSIGNED,
1598	1697	.field_pos = ID_AA64ISAR1_APA_SHIFT,
1599	1698	.min_field_value = ID_AA64ISAR1_APA_ARCHITECTED,
1600	1699	.matches = has_cpuid_feature,
1601		− .cpu_enable = cpu_enable_address_auth,
1602	1700	},
1603	1701	{
1604	1702	.desc = "Address authentication (IMP DEF algorithm)",
1605	1703	.capability = ARM64_HAS_ADDRESS_AUTH_IMP_DEF,
1606		− .type = ARM64_CPUCAP_SYSTEM_FEATURE,
	1704	+ .type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
1607	1705	.sys_reg = SYS_ID_AA64ISAR1_EL1,
1608	1706	.sign = FTR_UNSIGNED,
1609	1707	.field_pos = ID_AA64ISAR1_API_SHIFT,
1610	1708	.min_field_value = ID_AA64ISAR1_API_IMP_DEF,
1611	1709	.matches = has_cpuid_feature,
1612		− .cpu_enable = cpu_enable_address_auth,
	1710	+ },
	1711	+ {
	1712	+ .capability = ARM64_HAS_ADDRESS_AUTH,
	1713	+ .type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
	1714	+ .matches = has_address_auth,
1613	1715	},
1614	1716	{
1615	1717	.desc = "Generic authentication (architected algorithm)",
@@ -1631,6 +1733,11 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 15 unmodified lines
1631	1733	.min_field_value = ID_AA64ISAR1_GPI_IMP_DEF,
1632	1734	.matches = has_cpuid_feature,
1633	1735	},
	1736	+ {
	1737	+ .capability = ARM64_HAS_GENERIC_AUTH,
	1738	+ .type = ARM64_CPUCAP_SYSTEM_FEATURE,
	1739	+ .matches = has_generic_auth,
	1740	+ },
1634	1741	#endif /* CONFIG_ARM64_PTR_AUTH */
1635	1742	#ifdef CONFIG_ARM64_PSEUDO_NMI
1636	1743	{
@@ -1980,10 +2087,8 @@ static void __init enable_cpu_capabilities(u16 scope_mask) · 343 unmodified lines
1980	2087	* Run through the list of capabilities to check for conflicts.
1981	2088	* If the system has already detected a capability, take necessary
1982	2089	* action on this CPU.
1983		− *
1984		− * Returns "false" on conflicts.
1985	2090	*/
1986		−static bool verify_local_cpu_caps(u16 scope_mask)
	2091	+static void verify_local_cpu_caps(u16 scope_mask)
1987	2092	{
1988	2093	int i;
1989	2094	bool cpu_has_cap, system_has_cap;
@@ -2028,10 +2133,12 @@ static bool verify_local_cpu_caps(u16 scope_mask) · 38 unmodified lines
2028	2133	pr_crit("CPU%d: Detected conflict for capability %d (%s), System: %d, CPU: %d\n",
2029	2134	smp_processor_id(), caps->capability,
2030	2135	caps->desc, system_has_cap, cpu_has_cap);
2031		− return false;
2032		− }
2033	2136
2034		− return true;
	2137	+ if (cpucap_panic_on_conflict(caps))
	2138	+ cpu_panic_kernel();
	2139	+ else
	2140	+ cpu_die_early();
	2141	+ }
2035	2142	}
2036	2143
2037	2144	/*
@@ -2041,12 +2148,8 @@ static bool verify_local_cpu_caps(u16 scope_mask) · 3 unmodified lines
2041	2148	static void check_early_cpu_features(void)
2042	2149	{
2043	2150	verify_cpu_asid_bits();
2044		− /*
2045		− * Early features are used by the kernel already. If there
2046		− * is a conflict, we cannot proceed further.
2047		− */
2048		− if (!verify_local_cpu_caps(SCOPE_BOOT_CPU))
2049		− cpu_panic_kernel();
	2151	+
	2152	+ verify_local_cpu_caps(SCOPE_BOOT_CPU);
2050	2153	}
2051	2154
2052	2155	static void
@@ -2094,8 +2197,7 @@ static void verify_local_cpu_capabilities(void) · 41 unmodified lines
2094	2197	* check_early_cpu_features(), as they need to be verified
2095	2198	* on all secondary CPUs.
2096	2199	*/
2097		− if (!verify_local_cpu_caps(SCOPE_ALL & ~SCOPE_BOOT_CPU))
2098		− cpu_die_early();
	2200	+ verify_local_cpu_caps(SCOPE_ALL & ~SCOPE_BOOT_CPU);
2099	2201
2100	2202	verify_local_elf_hwcaps(arm64_elf_hwcaps);
2101	2203
@@ -2146,6 +2248,23 @@ bool this_cpu_has_cap(unsigned int n) · 44 unmodified lines
2146	2248	return false;
2147	2249	}
2148	2250
	2251	+/*
	2252	+ * This helper function is used in a narrow window when,
	2253	+ * - The system wide safe registers are set with all the SMP CPUs and,
	2254	+ * - The SYSTEM_FEATURE cpu_hwcaps may not have been set.
	2255	+ * In all other cases cpus_have_{const_}cap() should be used.
	2256	+ */
	2257	+static bool __system_matches_cap(unsigned int n)
	2258	+{
	2259	+ if (n < ARM64_NCAPS) {
	2260	+ const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[n];
	2261	+
	2262	+ if (cap)
	2263	+ return cap->matches(cap, SCOPE_SYSTEM);
	2264	+ }
	2265	+ return false;
	2266	+}
	2267	+
2149	2268	void cpu_set_feature(unsigned int num)
2150	2269	{
2151	2270	WARN_ON(num >= MAX_CPU_FEATURES);
@@ -2218,7 +2337,7 @@ void __init setup_cpu_features(void) · 66 unmodified lines
2218	2337	static bool __maybe_unused
2219	2338	cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused)
2220	2339	{
2221		− return (cpus_have_const_cap(ARM64_HAS_PAN) && !cpus_have_const_cap(ARM64_HAS_UAO));
	2340	+ return (__system_matches_cap(ARM64_HAS_PAN) && !__system_matches_cap(ARM64_HAS_UAO));
2222	2341	}
2223	2342
2224	2343	static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap)

arch/arm64/kernel/cpuidle.c+5−4

@@ -18,11 +18,11 @@ · 17 unmodified lines
18	18
19	19	int arm_cpuidle_init(unsigned int cpu)
20	20	{
	21	+ const struct cpu_operations *ops = get_cpu_ops(cpu);
21	22	int ret = -EOPNOTSUPP;
22	23
23		− if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_suspend &&
24		− cpu_ops[cpu]->cpu_init_idle)
25		− ret = cpu_ops[cpu]->cpu_init_idle(cpu);
	24	+ if (ops && ops->cpu_suspend && ops->cpu_init_idle)
	25	+ ret = ops->cpu_init_idle(cpu);
26	26
27	27	return ret;
28	28	}
@@ -37,8 +37,9 @@ int arm_cpuidle_init(unsigned int cpu) · 8 unmodified lines
37	37	int arm_cpuidle_suspend(int index)
38	38	{
39	39	int cpu = smp_processor_id();
	40	+ const struct cpu_operations *ops = get_cpu_ops(cpu);
40	41
41		− return cpu_ops[cpu]->cpu_suspend(index);
	42	+ return ops->cpu_suspend(index);
42	43	}
43	44
44	45	#ifdef CONFIG_ACPI

arch/arm64/kernel/entry-common.c+1−1

@@ -175,7 +175,7 @@ NOKPROBE_SYMBOL(el0_pc); · 174 unmodified lines
175	175	static void notrace el0_sp(struct pt_regs *regs, unsigned long esr)
176	176	{
177	177	user_exit_irqoff();
178		− local_daif_restore(DAIF_PROCCTX_NOIRQ);
	178	+ local_daif_restore(DAIF_PROCCTX);
179	179	do_sp_pc_abort(regs->sp, esr, regs);
180	180	}
181	181	NOKPROBE_SYMBOL(el0_sp);

arch/arm64/kernel/entry-ftrace.S+24−24

@@ -75,27 +75,27 @@ · 74 unmodified lines
75	75	add x29, sp, #S_STACKFRAME
76	76	.endm
77	77
78		−ENTRY(ftrace_regs_caller)
	78	+SYM_CODE_START(ftrace_regs_caller)
79	79	ftrace_regs_entry 1
80	80	b ftrace_common
81		−ENDPROC(ftrace_regs_caller)
	81	+SYM_CODE_END(ftrace_regs_caller)
82	82
83		−ENTRY(ftrace_caller)
	83	+SYM_CODE_START(ftrace_caller)
84	84	ftrace_regs_entry 0
85	85	b ftrace_common
86		−ENDPROC(ftrace_caller)
	86	+SYM_CODE_END(ftrace_caller)
87	87
88		−ENTRY(ftrace_common)
	88	+SYM_CODE_START(ftrace_common)
89	89	sub x0, x30, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
90	90	mov x1, x9 // parent_ip (callsite's LR)
91	91	ldr_l x2, function_trace_op // op
92	92	mov x3, sp // regs
93	93
94		−GLOBAL(ftrace_call)
	94	+SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
95	95	bl ftrace_stub
96	96
97	97	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
98		−GLOBAL(ftrace_graph_call) // ftrace_graph_caller();
	98	+SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller();
99	99	nop // If enabled, this will be replaced
100	100	// "b ftrace_graph_caller"
101	101	#endif
@@ -122,17 +122,17 @@ ftrace_common_return: · 20 unmodified lines
122	122	add sp, sp, #S_FRAME_SIZE + 16
123	123
124	124	ret x9
125		−ENDPROC(ftrace_common)
	125	+SYM_CODE_END(ftrace_common)
126	126
127	127	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
128		−ENTRY(ftrace_graph_caller)
	128	+SYM_CODE_START(ftrace_graph_caller)
129	129	ldr x0, [sp, #S_PC]
130	130	sub x0, x0, #AARCH64_INSN_SIZE // ip (callsite's BL insn)
131	131	add x1, sp, #S_LR // parent_ip (callsite's LR)
132	132	ldr x2, [sp, #S_FRAME_SIZE] // parent fp (callsite's FP)
133	133	bl prepare_ftrace_return
134	134	b ftrace_common_return
135		−ENDPROC(ftrace_graph_caller)
	135	+SYM_CODE_END(ftrace_graph_caller)
136	136	#endif
137	137
138	138	#else /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
@@ -218,7 +218,7 @@ ENDPROC(ftrace_graph_caller) · 79 unmodified lines
218	218	* - tracer function to probe instrumented function's entry,
219	219	* - ftrace_graph_caller to set up an exit hook
220	220	*/
221		−ENTRY(_mcount)
	221	+SYM_FUNC_START(_mcount)
222	222	mcount_enter
223	223
224	224	ldr_l x2, ftrace_trace_function
@@ -242,7 +242,7 @@ skip_ftrace_call: // } · 17 unmodified lines
242	242	b.ne ftrace_graph_caller // ftrace_graph_caller();
243	243	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
244	244	mcount_exit
245		−ENDPROC(_mcount)
	245	+SYM_FUNC_END(_mcount)
246	246	EXPORT_SYMBOL(_mcount)
247	247	NOKPROBE(_mcount)
248	248
@@ -253,9 +253,9 @@ NOKPROBE(_mcount) · 4 unmodified lines
253	253	* and later on, NOP to branch to ftrace_caller() when enabled or branch to
254	254	* NOP when disabled per-function base.
255	255	*/
256		−ENTRY(_mcount)
	256	+SYM_FUNC_START(_mcount)
257	257	ret
258		−ENDPROC(_mcount)
	258	+SYM_FUNC_END(_mcount)
259	259	EXPORT_SYMBOL(_mcount)
260	260	NOKPROBE(_mcount)
261	261
@@ -268,24 +268,24 @@ NOKPROBE(_mcount) · 6 unmodified lines
268	268	* - tracer function to probe instrumented function's entry,
269	269	* - ftrace_graph_caller to set up an exit hook
270	270	*/
271		−ENTRY(ftrace_caller)
	271	+SYM_FUNC_START(ftrace_caller)
272	272	mcount_enter
273	273
274	274	mcount_get_pc0 x0 // function's pc
275	275	mcount_get_lr x1 // function's lr
276	276
277		−GLOBAL(ftrace_call) // tracer(pc, lr);
	277	+SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL) // tracer(pc, lr);
278	278	nop // This will be replaced with "bl xxx"
279	279	// where xxx can be any kind of tracer.
280	280
281	281	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
282		−GLOBAL(ftrace_graph_call) // ftrace_graph_caller();
	282	+SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller();
283	283	nop // If enabled, this will be replaced
284	284	// "b ftrace_graph_caller"
285	285	#endif
286	286
287	287	mcount_exit
288		−ENDPROC(ftrace_caller)
	288	+SYM_FUNC_END(ftrace_caller)
289	289	#endif /* CONFIG_DYNAMIC_FTRACE */
290	290
291	291	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
@@ -298,20 +298,20 @@ ENDPROC(ftrace_caller) · 6 unmodified lines
298	298	* the call stack in order to intercept instrumented function's return path
299	299	* and run return_to_handler() later on its exit.
300	300	*/
301		−ENTRY(ftrace_graph_caller)
	301	+SYM_FUNC_START(ftrace_graph_caller)
302	302	mcount_get_pc x0 // function's pc
303	303	mcount_get_lr_addr x1 // pointer to function's saved lr
304	304	mcount_get_parent_fp x2 // parent's fp
305	305	bl prepare_ftrace_return // prepare_ftrace_return(pc, &lr, fp)
306	306
307	307	mcount_exit
308		−ENDPROC(ftrace_graph_caller)
	308	+SYM_FUNC_END(ftrace_graph_caller)
309	309	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
310	310	#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
311	311
312		−ENTRY(ftrace_stub)
	312	+SYM_FUNC_START(ftrace_stub)
313	313	ret
314		−ENDPROC(ftrace_stub)
	314	+SYM_FUNC_END(ftrace_stub)
315	315
316	316	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
317	317	/*
@@ -320,7 +320,7 @@ ENDPROC(ftrace_stub) · 2 unmodified lines
320	320	* Run ftrace_return_to_handler() before going back to parent.
321	321	* @fp is checked against the value passed by ftrace_graph_caller().
322	322	*/
323		−ENTRY(return_to_handler)
	323	+SYM_CODE_START(return_to_handler)
324	324	/* save return value regs */
325	325	sub sp, sp, #64
326	326	stp x0, x1, [sp]
@@ -340,5 +340,5 @@ ENTRY(return_to_handler) · 13 unmodified lines
340	340	add sp, sp, #64
341	341
342	342	ret
343		−END(return_to_handler)
	343	+SYM_CODE_END(return_to_handler)
344	344	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */

arch/arm64/kernel/entry.S+64−57

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <asm/alternative.h>
15	15	#include <asm/assembler.h>
16	16	#include <asm/asm-offsets.h>
	17	+#include <asm/asm_pointer_auth.h>
17	18	#include <asm/cpufeature.h>
18	19	#include <asm/errno.h>
19	20	#include <asm/esr.h>
@@ -177,6 +178,7 @@ alternative_cb_end · 157 unmodified lines
177	178
178	179	apply_ssbd 1, x22, x23
179	180
	181	+ ptrauth_keys_install_kernel tsk, 1, x20, x22, x23
180	182	.else
181	183	add x21, sp, #S_FRAME_SIZE
182	184	get_current_task tsk
@@ -341,6 +343,9 @@ alternative_else_nop_endif · 158 unmodified lines
341	343	msr cntkctl_el1, x1
342	344	4:
343	345	#endif
	346	+ /* No kernel C function calls after this as user keys are set. */
	347	+ ptrauth_keys_install_user tsk, x0, x1, x2
	348	+
344	349	apply_ssbd 0, x0, x1
345	350	.endif
346	351
@@ -465,7 +470,7 @@ alternative_endif · 118 unmodified lines
465	470	.pushsection ".entry.text", "ax"
466	471
467	472	.align 11
468		−ENTRY(vectors)
	473	+SYM_CODE_START(vectors)
469	474	kernel_ventry 1, sync_invalid // Synchronous EL1t
470	475	kernel_ventry 1, irq_invalid // IRQ EL1t
471	476	kernel_ventry 1, fiq_invalid // FIQ EL1t
@@ -492,7 +497,7 @@ ENTRY(vectors) · 20 unmodified lines
492	497	kernel_ventry 0, fiq_invalid, 32 // FIQ 32-bit EL0
493	498	kernel_ventry 0, error_invalid, 32 // Error 32-bit EL0
494	499	#endif
495		−END(vectors)
	500	+SYM_CODE_END(vectors)
496	501
497	502	#ifdef CONFIG_VMAP_STACK
498	503	/*
@@ -534,57 +539,57 @@ __bad_stack: · 35 unmodified lines
534	539	ASM_BUG()
535	540	.endm
536	541
537		−el0_sync_invalid:
	542	+SYM_CODE_START_LOCAL(el0_sync_invalid)
538	543	inv_entry 0, BAD_SYNC
539		−ENDPROC(el0_sync_invalid)
	544	+SYM_CODE_END(el0_sync_invalid)
540	545
541		−el0_irq_invalid:
	546	+SYM_CODE_START_LOCAL(el0_irq_invalid)
542	547	inv_entry 0, BAD_IRQ
543		−ENDPROC(el0_irq_invalid)
	548	+SYM_CODE_END(el0_irq_invalid)
544	549
545		−el0_fiq_invalid:
	550	+SYM_CODE_START_LOCAL(el0_fiq_invalid)
546	551	inv_entry 0, BAD_FIQ
547		−ENDPROC(el0_fiq_invalid)
	552	+SYM_CODE_END(el0_fiq_invalid)
548	553
549		−el0_error_invalid:
	554	+SYM_CODE_START_LOCAL(el0_error_invalid)
550	555	inv_entry 0, BAD_ERROR
551		−ENDPROC(el0_error_invalid)
	556	+SYM_CODE_END(el0_error_invalid)
552	557
553	558	#ifdef CONFIG_COMPAT
554		−el0_fiq_invalid_compat:
	559	+SYM_CODE_START_LOCAL(el0_fiq_invalid_compat)
555	560	inv_entry 0, BAD_FIQ, 32
556		−ENDPROC(el0_fiq_invalid_compat)
	561	+SYM_CODE_END(el0_fiq_invalid_compat)
557	562	#endif
558	563
559		−el1_sync_invalid:
	564	+SYM_CODE_START_LOCAL(el1_sync_invalid)
560	565	inv_entry 1, BAD_SYNC
561		−ENDPROC(el1_sync_invalid)
	566	+SYM_CODE_END(el1_sync_invalid)
562	567
563		−el1_irq_invalid:
	568	+SYM_CODE_START_LOCAL(el1_irq_invalid)
564	569	inv_entry 1, BAD_IRQ
565		−ENDPROC(el1_irq_invalid)
	570	+SYM_CODE_END(el1_irq_invalid)
566	571
567		−el1_fiq_invalid:
	572	+SYM_CODE_START_LOCAL(el1_fiq_invalid)
568	573	inv_entry 1, BAD_FIQ
569		−ENDPROC(el1_fiq_invalid)
	574	+SYM_CODE_END(el1_fiq_invalid)
570	575
571		−el1_error_invalid:
	576	+SYM_CODE_START_LOCAL(el1_error_invalid)
572	577	inv_entry 1, BAD_ERROR
573		−ENDPROC(el1_error_invalid)
	578	+SYM_CODE_END(el1_error_invalid)
574	579
575	580	/*
576	581	* EL1 mode handlers.
577	582	*/
578	583	.align 6
579		−el1_sync:
	584	+SYM_CODE_START_LOCAL_NOALIGN(el1_sync)
580	585	kernel_entry 1
581	586	mov x0, sp
582	587	bl el1_sync_handler
583	588	kernel_exit 1
584		−ENDPROC(el1_sync)
	589	+SYM_CODE_END(el1_sync)
585	590
586	591	.align 6
587		−el1_irq:
	592	+SYM_CODE_START_LOCAL_NOALIGN(el1_irq)
588	593	kernel_entry 1
589	594	gic_prio_irq_setup pmr=x20, tmp=x1
590	595	enable_da_f
@@ -639,42 +644,42 @@ alternative_else_nop_endif · 48 unmodified lines
639	644	#endif
640	645
641	646	kernel_exit 1
642		−ENDPROC(el1_irq)
	647	+SYM_CODE_END(el1_irq)
643	648
644	649	/*
645	650	* EL0 mode handlers.
646	651	*/
647	652	.align 6
648		−el0_sync:
	653	+SYM_CODE_START_LOCAL_NOALIGN(el0_sync)
649	654	kernel_entry 0
650	655	mov x0, sp
651	656	bl el0_sync_handler
652	657	b ret_to_user
653		−ENDPROC(el0_sync)
	658	+SYM_CODE_END(el0_sync)
654	659
655	660	#ifdef CONFIG_COMPAT
656	661	.align 6
657		−el0_sync_compat:
	662	+SYM_CODE_START_LOCAL_NOALIGN(el0_sync_compat)
658	663	kernel_entry 0, 32
659	664	mov x0, sp
660	665	bl el0_sync_compat_handler
661	666	b ret_to_user
662		−ENDPROC(el0_sync_compat)
	667	+SYM_CODE_END(el0_sync_compat)
663	668
664	669	.align 6
665		−el0_irq_compat:
	670	+SYM_CODE_START_LOCAL_NOALIGN(el0_irq_compat)
666	671	kernel_entry 0, 32
667	672	b el0_irq_naked
668		−ENDPROC(el0_irq_compat)
	673	+SYM_CODE_END(el0_irq_compat)
669	674
670		−el0_error_compat:
	675	+SYM_CODE_START_LOCAL_NOALIGN(el0_error_compat)
671	676	kernel_entry 0, 32
672	677	b el0_error_naked
673		−ENDPROC(el0_error_compat)
	678	+SYM_CODE_END(el0_error_compat)
674	679	#endif
675	680
676	681	.align 6
677		−el0_irq:
	682	+SYM_CODE_START_LOCAL_NOALIGN(el0_irq)
678	683	kernel_entry 0
679	684	el0_irq_naked:
680	685	gic_prio_irq_setup pmr=x20, tmp=x0
@@ -696,9 +701,9 @@ el0_irq_naked: · 15 unmodified lines
696	701	bl trace_hardirqs_on
697	702	#endif
698	703	b ret_to_user
699		−ENDPROC(el0_irq)
	704	+SYM_CODE_END(el0_irq)
700	705
701		−el1_error:
	706	+SYM_CODE_START_LOCAL(el1_error)
702	707	kernel_entry 1
703	708	mrs x1, esr_el1
704	709	gic_prio_kentry_setup tmp=x2
@@ -706,9 +711,9 @@ el1_error: · 1 unmodified line
706	711	mov x0, sp
707	712	bl do_serror
708	713	kernel_exit 1
709		−ENDPROC(el1_error)
	714	+SYM_CODE_END(el1_error)
710	715
711		−el0_error:
	716	+SYM_CODE_START_LOCAL(el0_error)
712	717	kernel_entry 0
713	718	el0_error_naked:
714	719	mrs x25, esr_el1
@@ -720,7 +725,7 @@ el0_error_naked: · 5 unmodified lines
720	725	bl do_serror
721	726	enable_da_f
722	727	b ret_to_user
723		−ENDPROC(el0_error)
	728	+SYM_CODE_END(el0_error)
724	729
725	730	/*
726	731	* Ok, we need to do extra processing, enter the slow path.
@@ -832,7 +837,7 @@ alternative_else_nop_endif · 105 unmodified lines
832	837	.endm
833	838
834	839	.align 11
835		−ENTRY(tramp_vectors)
	840	+SYM_CODE_START_NOALIGN(tramp_vectors)
836	841	.space 0x400
837	842
838	843	tramp_ventry
@@ -844,24 +849,24 @@ ENTRY(tramp_vectors) · 5 unmodified lines
844	849	tramp_ventry 32
845	850	tramp_ventry 32
846	851	tramp_ventry 32
847		−END(tramp_vectors)
	852	+SYM_CODE_END(tramp_vectors)
848	853
849		−ENTRY(tramp_exit_native)
	854	+SYM_CODE_START(tramp_exit_native)
850	855	tramp_exit
851		−END(tramp_exit_native)
	856	+SYM_CODE_END(tramp_exit_native)
852	857
853		−ENTRY(tramp_exit_compat)
	858	+SYM_CODE_START(tramp_exit_compat)
854	859	tramp_exit 32
855		−END(tramp_exit_compat)
	860	+SYM_CODE_END(tramp_exit_compat)
856	861
857	862	.ltorg
858	863	.popsection // .entry.tramp.text
859	864	#ifdef CONFIG_RANDOMIZE_BASE
860	865	.pushsection ".rodata", "a"
861	866	.align PAGE_SHIFT
862		− .globl __entry_tramp_data_start
863		−__entry_tramp_data_start:
	867	+SYM_DATA_START(__entry_tramp_data_start)
864	868	.quad vectors
	869	+SYM_DATA_END(__entry_tramp_data_start)
865	870	.popsection // .rodata
866	871	#endif /* CONFIG_RANDOMIZE_BASE */
867	872	#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
@@ -874,7 +879,7 @@ __entry_tramp_data_start: · 6 unmodified lines
874	879	* Previous and next are guaranteed not to be the same.
875	880	*
876	881	*/
877		−ENTRY(cpu_switch_to)
	882	+SYM_FUNC_START(cpu_switch_to)
878	883	mov x10, #THREAD_CPU_CONTEXT
879	884	add x8, x0, x10
880	885	mov x9, sp
@@ -895,21 +900,22 @@ ENTRY(cpu_switch_to) · 14 unmodified lines
895	900	ldr lr, [x8]
896	901	mov sp, x9
897	902	msr sp_el0, x1
	903	+ ptrauth_keys_install_kernel x1, 1, x8, x9, x10
898	904	ret
899		−ENDPROC(cpu_switch_to)
	905	+SYM_FUNC_END(cpu_switch_to)
900	906	NOKPROBE(cpu_switch_to)
901	907
902	908	/*
903	909	* This is how we return from a fork.
904	910	*/
905		−ENTRY(ret_from_fork)
	911	+SYM_CODE_START(ret_from_fork)
906	912	bl schedule_tail
907	913	cbz x19, 1f // not a kernel thread
908	914	mov x0, x20
909	915	blr x19
910	916	1: get_current_task tsk
911	917	b ret_to_user
912		−ENDPROC(ret_from_fork)
	918	+SYM_CODE_END(ret_from_fork)
913	919	NOKPROBE(ret_from_fork)
914	920
915	921	#ifdef CONFIG_ARM_SDE_INTERFACE
@@ -938,7 +944,7 @@ NOKPROBE(ret_from_fork) · 22 unmodified lines
938	944	*/
939	945	.ltorg
940	946	.pushsection ".entry.tramp.text", "ax"
941		−ENTRY(__sdei_asm_entry_trampoline)
	947	+SYM_CODE_START(__sdei_asm_entry_trampoline)
942	948	mrs x4, ttbr1_el1
943	949	tbz x4, #USER_ASID_BIT, 1f
944	950
@@ -960,7 +966,7 @@ ENTRY(__sdei_asm_entry_trampoline) · 15 unmodified lines
960	966	ldr x4, =__sdei_asm_handler
961	967	#endif
962	968	br x4
963		−ENDPROC(__sdei_asm_entry_trampoline)
	969	+SYM_CODE_END(__sdei_asm_entry_trampoline)
964	970	NOKPROBE(__sdei_asm_entry_trampoline)
965	971
966	972	/*
@@ -970,21 +976,22 @@ NOKPROBE(__sdei_asm_entry_trampoline) · 3 unmodified lines
970	976	* x2: exit_mode
971	977	* x4: struct sdei_registered_event argument from registration time.
972	978	*/
973		−ENTRY(__sdei_asm_exit_trampoline)
	979	+SYM_CODE_START(__sdei_asm_exit_trampoline)
974	980	ldr x4, [x4, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)]
975	981	cbnz x4, 1f
976	982
977	983	tramp_unmap_kernel tmp=x4
978	984
979	985	1: sdei_handler_exit exit_mode=x2
980		−ENDPROC(__sdei_asm_exit_trampoline)
	986	+SYM_CODE_END(__sdei_asm_exit_trampoline)
981	987	NOKPROBE(__sdei_asm_exit_trampoline)
982	988	.ltorg
983	989	.popsection // .entry.tramp.text
984	990	#ifdef CONFIG_RANDOMIZE_BASE
985	991	.pushsection ".rodata", "a"
986		−__sdei_asm_trampoline_next_handler:
	992	+SYM_DATA_START(__sdei_asm_trampoline_next_handler)
987	993	.quad __sdei_asm_handler
	994	+SYM_DATA_END(__sdei_asm_trampoline_next_handler)
988	995	.popsection // .rodata
989	996	#endif /* CONFIG_RANDOMIZE_BASE */
990	997	#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
@@ -1002,7 +1009,7 @@ __sdei_asm_trampoline_next_handler: · 11 unmodified lines
1002	1009	* follow SMC-CC. We save (or retrieve) all the registers as the handler may
1003	1010	* want them.
1004	1011	*/
1005		−ENTRY(__sdei_asm_handler)
	1012	+SYM_CODE_START(__sdei_asm_handler)
1006	1013	stp x2, x3, [x1, #SDEI_EVENT_INTREGS + S_PC]
1007	1014	stp x4, x5, [x1, #SDEI_EVENT_INTREGS + 16 * 2]
1008	1015	stp x6, x7, [x1, #SDEI_EVENT_INTREGS + 16 * 3]
@@ -1085,6 +1092,6 @@ alternative_else_nop_endif · 76 unmodified lines
1085	1092	tramp_alias dst=x5, sym=__sdei_asm_exit_trampoline
1086	1093	br x5
1087	1094	#endif
1088		−ENDPROC(__sdei_asm_handler)
	1095	+SYM_CODE_END(__sdei_asm_handler)
1089	1096	NOKPROBE(__sdei_asm_handler)
1090	1097	#endif /* CONFIG_ARM_SDE_INTERFACE */

arch/arm64/kernel/head.S+46−40

@@ -105,7 +105,7 @@ pe_header: · 104 unmodified lines
105	105	* x24 __primary_switch() .. relocate_kernel()
106	106	* current RELR displacement
107	107	*/
108		−ENTRY(stext)
	108	+SYM_CODE_START(stext)
109	109	bl preserve_boot_args
110	110	bl el2_setup // Drop to EL1, w0=cpu_boot_mode
111	111	adrp x23, __PHYS_OFFSET
@@ -118,14 +118,15 @@ ENTRY(stext) · 6 unmodified lines
118	118	* On return, the CPU will be ready for the MMU to be turned on and
119	119	* the TCR will have been set.
120	120	*/
	121	+ mov x0, #ARM64_CPU_BOOT_PRIMARY
121	122	bl __cpu_setup // initialise processor
122	123	b __primary_switch
123		−ENDPROC(stext)
	124	+SYM_CODE_END(stext)
124	125
125	126	/*
126	127	* Preserve the arguments passed by the bootloader in x0 .. x3
127	128	*/
128		−preserve_boot_args:
	129	+SYM_CODE_START_LOCAL(preserve_boot_args)
129	130	mov x21, x0 // x21=FDT
130	131
131	132	adr_l x0, boot_args // record the contents of
@@ -137,7 +138,7 @@ preserve_boot_args: · 5 unmodified lines
137	138
138	139	mov x1, #0x20 // 4 x 8 bytes
139	140	b __inval_dcache_area // tail call
140		−ENDPROC(preserve_boot_args)
	141	+SYM_CODE_END(preserve_boot_args)
141	142
142	143	/*
143	144	* Macro to create a table entry to the next page.
@@ -275,7 +276,7 @@ ENDPROC(preserve_boot_args) · 131 unmodified lines
275	276	* - first few MB of the kernel linear mapping to jump to once the MMU has
276	277	* been enabled
277	278	*/
278		−__create_page_tables:
	279	+SYM_FUNC_START_LOCAL(__create_page_tables)
279	280	mov x28, lr
280	281
281	282	/*
@@ -403,15 +404,14 @@ __create_page_tables: · 121 unmodified lines
403	404	bl __inval_dcache_area
404	405
405	406	ret x28
406		−ENDPROC(__create_page_tables)
407		− .ltorg
	407	+SYM_FUNC_END(__create_page_tables)
408	408
409	409	/*
410	410	* The following fragment of code is executed with the MMU enabled.
411	411	*
412	412	* x0 = __PHYS_OFFSET
413	413	*/
414		−__primary_switched:
	414	+SYM_FUNC_START_LOCAL(__primary_switched)
415	415	adrp x4, init_thread_union
416	416	add sp, x4, #THREAD_SIZE
417	417	adr_l x5, init_task
@@ -456,7 +456,14 @@ __primary_switched: · 38 unmodified lines
456	456	mov x29, #0
457	457	mov x30, #0
458	458	b start_kernel
459		−ENDPROC(__primary_switched)
	459	+SYM_FUNC_END(__primary_switched)
	460	+
	461	+ .pushsection ".rodata", "a"
	462	+SYM_DATA_START(kimage_vaddr)
	463	+ .quad _text - TEXT_OFFSET
	464	+SYM_DATA_END(kimage_vaddr)
	465	+EXPORT_SYMBOL(kimage_vaddr)
	466	+ .popsection
460	467
461	468	/*
462	469	* end early head section, begin head code that is also used for
@@ -464,10 +471,6 @@ ENDPROC(__primary_switched) · 1 unmodified line
464	471	*/
465	472	.section ".idmap.text","awx"
466	473
467		−ENTRY(kimage_vaddr)
468		− .quad _text - TEXT_OFFSET
469		−EXPORT_SYMBOL(kimage_vaddr)
470		−
471	474	/*
472	475	* If we're fortunate enough to boot at EL2, ensure that the world is
473	476	* sane before dropping to EL1.
@@ -475,7 +478,7 @@ EXPORT_SYMBOL(kimage_vaddr) · 1 unmodified line
475	478	* Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in w0 if
476	479	* booted in EL1 or EL2 respectively.
477	480	*/
478		−ENTRY(el2_setup)
	481	+SYM_FUNC_START(el2_setup)
479	482	msr SPsel, #1 // We want to use SP_EL{1,2}
480	483	mrs x0, CurrentEL
481	484	cmp x0, #CurrentEL_EL2
@@ -599,7 +602,7 @@ set_hcr: · 117 unmodified lines
599	602	isb
600	603	ret
601	604
602		−install_el2_stub:
	605	+SYM_INNER_LABEL(install_el2_stub, SYM_L_LOCAL)
603	606	/*
604	607	* When VHE is not in use, early init of EL2 and EL1 needs to be
605	608	* done here.
@@ -636,13 +639,13 @@ install_el2_stub: · 30 unmodified lines
636	639	msr elr_el2, lr
637	640	mov w0, #BOOT_CPU_MODE_EL2 // This CPU booted in EL2
638	641	eret
639		−ENDPROC(el2_setup)
	642	+SYM_FUNC_END(el2_setup)
640	643
641	644	/*
642	645	* Sets the __boot_cpu_mode flag depending on the CPU boot mode passed
643	646	* in w0. See arch/arm64/include/asm/virt.h for more info.
644	647	*/
645		−set_cpu_boot_mode_flag:
	648	+SYM_FUNC_START_LOCAL(set_cpu_boot_mode_flag)
646	649	adr_l x1, __boot_cpu_mode
647	650	cmp w0, #BOOT_CPU_MODE_EL2
648	651	b.ne 1f
@@ -651,7 +654,7 @@ set_cpu_boot_mode_flag: · 2 unmodified lines
651	654	dmb sy
652	655	dc ivac, x1 // Invalidate potentially stale cache line
653	656	ret
654		−ENDPROC(set_cpu_boot_mode_flag)
	657	+SYM_FUNC_END(set_cpu_boot_mode_flag)
655	658
656	659	/*
657	660	* These values are written with the MMU off, but read with the MMU on.
@@ -667,15 +670,17 @@ ENDPROC(set_cpu_boot_mode_flag) · 9 unmodified lines
667	670	* This is not in .bss, because we set it sufficiently early that the boot-time
668	671	* zeroing of .bss would clobber it.
669	672	*/
670		−ENTRY(__boot_cpu_mode)
	673	+SYM_DATA_START(__boot_cpu_mode)
671	674	.long BOOT_CPU_MODE_EL2
672	675	.long BOOT_CPU_MODE_EL1
	676	+SYM_DATA_END(__boot_cpu_mode)
673	677	/*
674	678	* The booting CPU updates the failed status @__early_cpu_boot_status,
675	679	* with MMU turned off.
676	680	*/
677		−ENTRY(__early_cpu_boot_status)
	681	+SYM_DATA_START(__early_cpu_boot_status)
678	682	.quad 0
	683	+SYM_DATA_END(__early_cpu_boot_status)
679	684
680	685	.popsection
681	686
@@ -683,7 +688,7 @@ ENTRY(__early_cpu_boot_status) · 1 unmodified line
683	688	* This provides a "holding pen" for platforms to hold all secondary
684	689	* cores are held until we're ready for them to initialise.
685	690	*/
686		−ENTRY(secondary_holding_pen)
	691	+SYM_FUNC_START(secondary_holding_pen)
687	692	bl el2_setup // Drop to EL1, w0=cpu_boot_mode
688	693	bl set_cpu_boot_mode_flag
689	694	mrs x0, mpidr_el1
@@ -695,31 +700,32 @@ pen: ldr x4, [x3] · 5 unmodified lines
695	700	b.eq secondary_startup
696	701	wfe
697	702	b pen
698		−ENDPROC(secondary_holding_pen)
	703	+SYM_FUNC_END(secondary_holding_pen)
699	704
700	705	/*
701	706	* Secondary entry point that jumps straight into the kernel. Only to
702	707	* be used where CPUs are brought online dynamically by the kernel.
703	708	*/
704		−ENTRY(secondary_entry)
	709	+SYM_FUNC_START(secondary_entry)
705	710	bl el2_setup // Drop to EL1
706	711	bl set_cpu_boot_mode_flag
707	712	b secondary_startup
708		−ENDPROC(secondary_entry)
	713	+SYM_FUNC_END(secondary_entry)
709	714
710		−secondary_startup:
	715	+SYM_FUNC_START_LOCAL(secondary_startup)
711	716	/*
712	717	* Common entry point for secondary CPUs.
713	718	*/
714	719	bl __cpu_secondary_check52bitva
	720	+ mov x0, #ARM64_CPU_BOOT_SECONDARY
715	721	bl __cpu_setup // initialise processor
716	722	adrp x1, swapper_pg_dir
717	723	bl __enable_mmu
718	724	ldr x8, =__secondary_switched
719	725	br x8
720		−ENDPROC(secondary_startup)
	726	+SYM_FUNC_END(secondary_startup)
721	727
722		−__secondary_switched:
	728	+SYM_FUNC_START_LOCAL(__secondary_switched)
723	729	adr_l x5, vectors
724	730	msr vbar_el1, x5
725	731	isb
@@ -734,13 +740,13 @@ __secondary_switched: · 8 unmodified lines
734	740	mov x29, #0
735	741	mov x30, #0
736	742	b secondary_start_kernel
737		−ENDPROC(__secondary_switched)
	743	+SYM_FUNC_END(__secondary_switched)
738	744
739		−__secondary_too_slow:
	745	+SYM_FUNC_START_LOCAL(__secondary_too_slow)
740	746	wfe
741	747	wfi
742	748	b __secondary_too_slow
743		−ENDPROC(__secondary_too_slow)
	749	+SYM_FUNC_END(__secondary_too_slow)
744	750
745	751	/*
746	752	* The booting CPU updates the failed status @__early_cpu_boot_status,
@@ -772,7 +778,7 @@ ENDPROC(__secondary_too_slow) · 25 unmodified lines
772	778	* Checks if the selected granule size is supported by the CPU.
773	779	* If it isn't, park the CPU
774	780	*/
775		−ENTRY(__enable_mmu)
	781	+SYM_FUNC_START(__enable_mmu)
776	782	mrs x2, ID_AA64MMFR0_EL1
777	783	ubfx x2, x2, #ID_AA64MMFR0_TGRAN_SHIFT, 4
778	784	cmp x2, #ID_AA64MMFR0_TGRAN_SUPPORTED
@@ -796,9 +802,9 @@ ENTRY(__enable_mmu) · 17 unmodified lines
796	802	dsb nsh
797	803	isb
798	804	ret
799		−ENDPROC(__enable_mmu)
	805	+SYM_FUNC_END(__enable_mmu)
800	806
801		−ENTRY(__cpu_secondary_check52bitva)
	807	+SYM_FUNC_START(__cpu_secondary_check52bitva)
802	808	#ifdef CONFIG_ARM64_VA_BITS_52
803	809	ldr_l x0, vabits_actual
804	810	cmp x0, #52
@@ -816,9 +822,9 @@ ENTRY(__cpu_secondary_check52bitva) · 11 unmodified lines
816	822
817	823	#endif
818	824	2: ret
819		−ENDPROC(__cpu_secondary_check52bitva)
	825	+SYM_FUNC_END(__cpu_secondary_check52bitva)
820	826
821		−__no_granule_support:
	827	+SYM_FUNC_START_LOCAL(__no_granule_support)
822	828	/* Indicate that this CPU can't boot and is stuck in the kernel */
823	829	update_early_cpu_boot_status \
824	830	CPU_STUCK_IN_KERNEL \| CPU_STUCK_REASON_NO_GRAN, x1, x2
@@ -826,10 +832,10 @@ __no_granule_support: · 1 unmodified line
826	832	wfe
827	833	wfi
828	834	b 1b
829		−ENDPROC(__no_granule_support)
	835	+SYM_FUNC_END(__no_granule_support)
830	836
831	837	#ifdef CONFIG_RELOCATABLE
832		−__relocate_kernel:
	838	+SYM_FUNC_START_LOCAL(__relocate_kernel)
833	839	/*
834	840	* Iterate over each entry in the relocation table, and apply the
835	841	* relocations in place.
@@ -931,10 +937,10 @@ __relocate_kernel: · 95 unmodified lines
931	937	#endif
932	938	ret
933	939
934		−ENDPROC(__relocate_kernel)
	940	+SYM_FUNC_END(__relocate_kernel)
935	941	#endif
936	942
937		−__primary_switch:
	943	+SYM_FUNC_START_LOCAL(__primary_switch)
938	944	#ifdef CONFIG_RANDOMIZE_BASE
939	945	mov x19, x0 // preserve new SCTLR_EL1 value
940	946	mrs x20, sctlr_el1 // preserve old SCTLR_EL1 value
@@ -977,4 +983,4 @@ __primary_switch: · 36 unmodified lines
977	983	ldr x8, =__primary_switched
978	984	adrp x0, __PHYS_OFFSET
979	985	br x8
980		−ENDPROC(__primary_switch)
	986	+SYM_FUNC_END(__primary_switch)

arch/arm64/kernel/hibernate-asm.S+0−2

@@ -110,8 +110,6 @@ ENTRY(swsusp_arch_suspend_exit) · 109 unmodified lines
110	110	cbz x24, 3f /* Do we need to re-initialise EL2? */
111	111	hvc #0
112	112	3: ret
113		−
114		− .ltorg
115	113	ENDPROC(swsusp_arch_suspend_exit)
116	114
117	115	/*

arch/arm64/kernel/hyp-stub.S+1−1

@@ -63,7 +63,7 @@ el1_sync: · 62 unmodified lines
63	63	beq 9f // Nothing to reset!
64	64
65	65	/* Someone called kvm_call_hyp() against the hyp-stub... */
66		− ldr x0, =HVC_STUB_ERR
	66	+ mov_q x0, HVC_STUB_ERR
67	67	eret
68	68
69	69	9: mov x0, xzr

arch/arm64/kernel/machine_kexec_file.c+1−1

@@ -121,7 +121,7 @@ static int setup_dtb(struct kimage *image, · 120 unmodified lines
121	121
122	122	/* add kaslr-seed */
123	123	ret = fdt_delprop(dtb, off, FDT_PROP_KASLR_SEED);
124		− if (ret == -FDT_ERR_NOTFOUND)
	124	+ if (ret == -FDT_ERR_NOTFOUND)
125	125	ret = 0;
126	126	else if (ret)
127	127	goto out;

arch/arm64/kernel/perf_event.c+181−157

@@ -285,6 +285,17 @@ static struct attribute_group armv8_pmuv3_format_attr_group = { · 284 unmodified lines
285	285	#define ARMV8_IDX_COUNTER_LAST(cpu_pmu) \
286	286	(ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
287	287
	288	+
	289	+/*
	290	+ * We unconditionally enable ARMv8.5-PMU long event counter support
	291	+ * (64-bit events) where supported. Indicate if this arm_pmu has long
	292	+ * event counter support.
	293	+ */
	294	+static bool armv8pmu_has_long_event(struct arm_pmu *cpu_pmu)
	295	+{
	296	+ return (cpu_pmu->pmuver >= ID_AA64DFR0_PMUVER_8_5);
	297	+}
	298	+
288	299	/*
289	300	* We must chain two programmable counters for 64 bit events,
290	301	* except when we have allocated the 64bit cycle counter (for CPU
@@ -294,9 +305,11 @@ static struct attribute_group armv8_pmuv3_format_attr_group = { · 3 unmodified lines
294	305	static inline bool armv8pmu_event_is_chained(struct perf_event *event)
295	306	{
296	307	int idx = event->hw.idx;
	308	+ struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
297	309
298	310	return !WARN_ON(idx < 0) &&
299	311	armv8pmu_event_is_64bit(event) &&
	312	+ !armv8pmu_has_long_event(cpu_pmu) &&
300	313	(idx != ARMV8_IDX_CYCLE_COUNTER);
301	314	}
302	315
@@ -345,7 +358,7 @@ static inline void armv8pmu_select_counter(int idx) · 42 unmodified lines
345	358	isb();
346	359	}
347	360
348		−static inline u32 armv8pmu_read_evcntr(int idx)
	361	+static inline u64 armv8pmu_read_evcntr(int idx)
349	362	{
350	363	armv8pmu_select_counter(idx);
351	364	return read_sysreg(pmxevcntr_el0);
@@ -362,6 +375,44 @@ static inline u64 armv8pmu_read_hw_counter(struct perf_event *event) · 10 unmodified lines
362	375	return val;
363	376	}
364	377
	378	+/*
	379	+ * The cycle counter is always a 64-bit counter. When ARMV8_PMU_PMCR_LP
	380	+ * is set the event counters also become 64-bit counters. Unless the
	381	+ * user has requested a long counter (attr.config1) then we want to
	382	+ * interrupt upon 32-bit overflow - we achieve this by applying a bias.
	383	+ */
	384	+static bool armv8pmu_event_needs_bias(struct perf_event *event)
	385	+{
	386	+ struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
	387	+ struct hw_perf_event *hwc = &event->hw;
	388	+ int idx = hwc->idx;
	389	+
	390	+ if (armv8pmu_event_is_64bit(event))
	391	+ return false;
	392	+
	393	+ if (armv8pmu_has_long_event(cpu_pmu) \|\|
	394	+ idx == ARMV8_IDX_CYCLE_COUNTER)
	395	+ return true;
	396	+
	397	+ return false;
	398	+}
	399	+
	400	+static u64 armv8pmu_bias_long_counter(struct perf_event *event, u64 value)
	401	+{
	402	+ if (armv8pmu_event_needs_bias(event))
	403	+ value \|= GENMASK(63, 32);
	404	+
	405	+ return value;
	406	+}
	407	+
	408	+static u64 armv8pmu_unbias_long_counter(struct perf_event *event, u64 value)
	409	+{
	410	+ if (armv8pmu_event_needs_bias(event))
	411	+ value &= ~GENMASK(63, 32);
	412	+
	413	+ return value;
	414	+}
	415	+
365	416	static u64 armv8pmu_read_counter(struct perf_event *event)
366	417	{
367	418	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
@@ -377,10 +428,10 @@ static u64 armv8pmu_read_counter(struct perf_event *event) · 9 unmodified lines
377	428	else
378	429	value = armv8pmu_read_hw_counter(event);
379	430
380		− return value;
	431	+ return armv8pmu_unbias_long_counter(event, value);
381	432	}
382	433
383		−static inline void armv8pmu_write_evcntr(int idx, u32 value)
	434	+static inline void armv8pmu_write_evcntr(int idx, u64 value)
384	435	{
385	436	armv8pmu_select_counter(idx);
386	437	write_sysreg(value, pmxevcntr_el0);
@@ -405,20 +456,14 @@ static void armv8pmu_write_counter(struct perf_event *event, u64 value) · 18 unmodified lines
405	456	struct hw_perf_event *hwc = &event->hw;
406	457	int idx = hwc->idx;
407	458
	459	+ value = armv8pmu_bias_long_counter(event, value);
	460	+
408	461	if (!armv8pmu_counter_valid(cpu_pmu, idx))
409	462	pr_err("CPU%u writing wrong counter %d\n",
410	463	smp_processor_id(), idx);
411		− else if (idx == ARMV8_IDX_CYCLE_COUNTER) {
412		− /*
413		− * The cycles counter is really a 64-bit counter.
414		− * When treating it as a 32-bit counter, we only count
415		− * the lower 32 bits, and set the upper 32-bits so that
416		− * we get an interrupt upon 32-bit overflow.
417		− */
418		− if (!armv8pmu_event_is_64bit(event))
419		− value \|= 0xffffffff00000000ULL;
	464	+ else if (idx == ARMV8_IDX_CYCLE_COUNTER)
420	465	write_sysreg(value, pmccntr_el0);
421		− } else
	466	+ else
422	467	armv8pmu_write_hw_counter(event, value);
423	468	}
424	469
@@ -450,86 +495,74 @@ static inline void armv8pmu_write_event_type(struct perf_event *event) · 25 unmodified lines
450	495	}
451	496	}
452	497
453		−static inline int armv8pmu_enable_counter(int idx)
	498	+static u32 armv8pmu_event_cnten_mask(struct perf_event *event)
454	499	{
455		− u32 counter = ARMV8_IDX_TO_COUNTER(idx);
456		− write_sysreg(BIT(counter), pmcntenset_el0);
457		− return idx;
	500	+ int counter = ARMV8_IDX_TO_COUNTER(event->hw.idx);
	501	+ u32 mask = BIT(counter);
	502	+
	503	+ if (armv8pmu_event_is_chained(event))
	504	+ mask \|= BIT(counter - 1);
	505	+ return mask;
	506	+}
	507	+
	508	+static inline void armv8pmu_enable_counter(u32 mask)
	509	+{
	510	+ write_sysreg(mask, pmcntenset_el0);
458	511	}
459	512
460	513	static inline void armv8pmu_enable_event_counter(struct perf_event *event)
461	514	{
462	515	struct perf_event_attr *attr = &event->attr;
463		− int idx = event->hw.idx;
464		− u32 counter_bits = BIT(ARMV8_IDX_TO_COUNTER(idx));
	516	+ u32 mask = armv8pmu_event_cnten_mask(event);
465	517
466		− if (armv8pmu_event_is_chained(event))
467		− counter_bits \|= BIT(ARMV8_IDX_TO_COUNTER(idx - 1));
468		−
469		− kvm_set_pmu_events(counter_bits, attr);
	518	+ kvm_set_pmu_events(mask, attr);
470	519
471	520	/* We rely on the hypervisor switch code to enable guest counters */
472		− if (!kvm_pmu_counter_deferred(attr)) {
473		− armv8pmu_enable_counter(idx);
474		− if (armv8pmu_event_is_chained(event))
475		− armv8pmu_enable_counter(idx - 1);
476		− }
	521	+ if (!kvm_pmu_counter_deferred(attr))
	522	+ armv8pmu_enable_counter(mask);
477	523	}
478	524
479		−static inline int armv8pmu_disable_counter(int idx)
	525	+static inline void armv8pmu_disable_counter(u32 mask)
480	526	{
481		− u32 counter = ARMV8_IDX_TO_COUNTER(idx);
482		− write_sysreg(BIT(counter), pmcntenclr_el0);
483		− return idx;
	527	+ write_sysreg(mask, pmcntenclr_el0);
484	528	}
485	529
486	530	static inline void armv8pmu_disable_event_counter(struct perf_event *event)
487	531	{
488		− struct hw_perf_event *hwc = &event->hw;
489	532	struct perf_event_attr *attr = &event->attr;
490		− int idx = hwc->idx;
491		− u32 counter_bits = BIT(ARMV8_IDX_TO_COUNTER(idx));
492		−
493		− if (armv8pmu_event_is_chained(event))
494		− counter_bits \|= BIT(ARMV8_IDX_TO_COUNTER(idx - 1));
	533	+ u32 mask = armv8pmu_event_cnten_mask(event);
495	534
496		− kvm_clr_pmu_events(counter_bits);
	535	+ kvm_clr_pmu_events(mask);
497	536
498	537	/* We rely on the hypervisor switch code to disable guest counters */
499		− if (!kvm_pmu_counter_deferred(attr)) {
500		− if (armv8pmu_event_is_chained(event))
501		− armv8pmu_disable_counter(idx - 1);
502		− armv8pmu_disable_counter(idx);
503		− }
	538	+ if (!kvm_pmu_counter_deferred(attr))
	539	+ armv8pmu_disable_counter(mask);
504	540	}
505	541
506		−static inline int armv8pmu_enable_intens(int idx)
	542	+static inline void armv8pmu_enable_intens(u32 mask)
507	543	{
508		− u32 counter = ARMV8_IDX_TO_COUNTER(idx);
509		− write_sysreg(BIT(counter), pmintenset_el1);
510		− return idx;
	544	+ write_sysreg(mask, pmintenset_el1);
511	545	}
512	546
513		−static inline int armv8pmu_enable_event_irq(struct perf_event *event)
	547	+static inline void armv8pmu_enable_event_irq(struct perf_event *event)
514	548	{
515		− return armv8pmu_enable_intens(event->hw.idx);
	549	+ u32 counter = ARMV8_IDX_TO_COUNTER(event->hw.idx);
	550	+ armv8pmu_enable_intens(BIT(counter));
516	551	}
517	552
518		−static inline int armv8pmu_disable_intens(int idx)
	553	+static inline void armv8pmu_disable_intens(u32 mask)
519	554	{
520		− u32 counter = ARMV8_IDX_TO_COUNTER(idx);
521		− write_sysreg(BIT(counter), pmintenclr_el1);
	555	+ write_sysreg(mask, pmintenclr_el1);
522	556	isb();
523	557	/* Clear the overflow flag in case an interrupt is pending. */
524		− write_sysreg(BIT(counter), pmovsclr_el0);
	558	+ write_sysreg(mask, pmovsclr_el0);
525	559	isb();
526		−
527		− return idx;
528	560	}
529	561
530		−static inline int armv8pmu_disable_event_irq(struct perf_event *event)
	562	+static inline void armv8pmu_disable_event_irq(struct perf_event *event)
531	563	{
532		− return armv8pmu_disable_intens(event->hw.idx);
	564	+ u32 counter = ARMV8_IDX_TO_COUNTER(event->hw.idx);
	565	+ armv8pmu_disable_intens(BIT(counter));
533	566	}
534	567
535	568	static inline u32 armv8pmu_getreset_flags(void)
@@ -743,7 +776,8 @@ static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc, · 207 unmodified lines
743	776	/*
744	777	* Otherwise use events counters
745	778	*/
746		− if (armv8pmu_event_is_64bit(event))
	779	+ if (armv8pmu_event_is_64bit(event) &&
	780	+ !armv8pmu_has_long_event(cpu_pmu))
747	781	return armv8pmu_get_chain_idx(cpuc, cpu_pmu);
748	782	else
749	783	return armv8pmu_get_single_idx(cpuc, cpu_pmu);
@@ -815,13 +849,11 @@ static int armv8pmu_filter_match(struct perf_event *event) · 65 unmodified lines
815	849	static void armv8pmu_reset(void *info)
816	850	{
817	851	struct arm_pmu cpu_pmu = (struct arm_pmu )info;
818		− u32 idx, nb_cnt = cpu_pmu->num_events;
	852	+ u32 pmcr;
819	853
820	854	/* The counter and interrupt enable registers are unknown at reset. */
821		− for (idx = ARMV8_IDX_CYCLE_COUNTER; idx < nb_cnt; ++idx) {
822		− armv8pmu_disable_counter(idx);
823		− armv8pmu_disable_intens(idx);
824		− }
	855	+ armv8pmu_disable_counter(U32_MAX);
	856	+ armv8pmu_disable_intens(U32_MAX);
825	857
826	858	/* Clear the counters we flip at guest entry/exit */
827	859	kvm_clr_pmu_events(U32_MAX);
@@ -830,8 +862,13 @@ static void armv8pmu_reset(void *info) · 2 unmodified lines
830	862	* Initialize & Reset PMNC. Request overflow interrupt for
831	863	* 64 bit cycle counter but cheat in armv8pmu_write_counter().
832	864	*/
833		− armv8pmu_pmcr_write(ARMV8_PMU_PMCR_P \| ARMV8_PMU_PMCR_C \|
834		− ARMV8_PMU_PMCR_LC);
	865	+ pmcr = ARMV8_PMU_PMCR_P \| ARMV8_PMU_PMCR_C \| ARMV8_PMU_PMCR_LC;
	866	+
	867	+ /* Enable long event counter support where available */
	868	+ if (armv8pmu_has_long_event(cpu_pmu))
	869	+ pmcr \|= ARMV8_PMU_PMCR_LP;
	870	+
	871	+ armv8pmu_pmcr_write(pmcr);
835	872	}
836	873
837	874	static int __armv8_pmuv3_map_event(struct perf_event *event,
@@ -914,6 +951,7 @@ static void __armv8pmu_probe_pmu(void *info) · 76 unmodified lines
914	951	if (pmuver == 0xf \|\| pmuver == 0)
915	952	return;
916	953
	954	+ cpu_pmu->pmuver = pmuver;
917	955	probe->present = true;
918	956
919	957	/* Read the nb of CNTx counters supported from PMNC */
@@ -953,7 +991,10 @@ static int armv8pmu_probe_pmu(struct arm_pmu *cpu_pmu) · 33 unmodified lines
953	991	return probe.present ? 0 : -ENODEV;
954	992	}
955	993
956		−static int armv8_pmu_init(struct arm_pmu *cpu_pmu)
	994	+static int armv8_pmu_init(struct arm_pmu cpu_pmu, char name,
	995	+ int (map_event)(struct perf_event event),
	996	+ const struct attribute_group *events,
	997	+ const struct attribute_group *format)
957	998	{
958	999	int ret = armv8pmu_probe_pmu(cpu_pmu);
959	1000	if (ret)
@@ -972,144 +1013,127 @@ static int armv8_pmu_init(struct arm_pmu *cpu_pmu) · 12 unmodified lines
972	1013	cpu_pmu->set_event_filter = armv8pmu_set_event_filter;
973	1014	cpu_pmu->filter_match = armv8pmu_filter_match;
974	1015
	1016	+ cpu_pmu->name = name;
	1017	+ cpu_pmu->map_event = map_event;
	1018	+ cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] = events ?
	1019	+ events : &armv8_pmuv3_events_attr_group;
	1020	+ cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] = format ?
	1021	+ format : &armv8_pmuv3_format_attr_group;
	1022	+
975	1023	return 0;
976	1024	}
977	1025
978	1026	static int armv8_pmuv3_init(struct arm_pmu *cpu_pmu)
979	1027	{
980		− int ret = armv8_pmu_init(cpu_pmu);
981		− if (ret)
982		− return ret;
983		−
984		− cpu_pmu->name = "armv8_pmuv3";
985		− cpu_pmu->map_event = armv8_pmuv3_map_event;
986		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
987		− &armv8_pmuv3_events_attr_group;
988		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
989		− &armv8_pmuv3_format_attr_group;
	1028	+ return armv8_pmu_init(cpu_pmu, "armv8_pmuv3",
	1029	+ armv8_pmuv3_map_event, NULL, NULL);
	1030	+}
990	1031
991		− return 0;
	1032	+static int armv8_a34_pmu_init(struct arm_pmu *cpu_pmu)
	1033	+{
	1034	+ return armv8_pmu_init(cpu_pmu, "armv8_cortex_a34",
	1035	+ armv8_pmuv3_map_event, NULL, NULL);
992	1036	}
993	1037
994	1038	static int armv8_a35_pmu_init(struct arm_pmu *cpu_pmu)
995	1039	{
996		− int ret = armv8_pmu_init(cpu_pmu);
997		− if (ret)
998		− return ret;
999		−
1000		− cpu_pmu->name = "armv8_cortex_a35";
1001		− cpu_pmu->map_event = armv8_a53_map_event;
1002		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1003		− &armv8_pmuv3_events_attr_group;
1004		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1005		− &armv8_pmuv3_format_attr_group;
1006		−
1007		− return 0;
	1040	+ return armv8_pmu_init(cpu_pmu, "armv8_cortex_a35",
	1041	+ armv8_a53_map_event, NULL, NULL);
1008	1042	}
1009	1043
1010	1044	static int armv8_a53_pmu_init(struct arm_pmu *cpu_pmu)
1011	1045	{
1012		− int ret = armv8_pmu_init(cpu_pmu);
1013		− if (ret)
1014		− return ret;
1015		−
1016		− cpu_pmu->name = "armv8_cortex_a53";
1017		− cpu_pmu->map_event = armv8_a53_map_event;
1018		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1019		− &armv8_pmuv3_events_attr_group;
1020		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1021		− &armv8_pmuv3_format_attr_group;
	1046	+ return armv8_pmu_init(cpu_pmu, "armv8_cortex_a53",
	1047	+ armv8_a53_map_event, NULL, NULL);
	1048	+}
1022	1049
1023		− return 0;
	1050	+static int armv8_a55_pmu_init(struct arm_pmu *cpu_pmu)
	1051	+{
	1052	+ return armv8_pmu_init(cpu_pmu, "armv8_cortex_a55",
	1053	+ armv8_pmuv3_map_event, NULL, NULL);
1024	1054	}
1025	1055
1026	1056	static int armv8_a57_pmu_init(struct arm_pmu *cpu_pmu)
1027	1057	{
1028		− int ret = armv8_pmu_init(cpu_pmu);
1029		− if (ret)
1030		− return ret;
1031		−
1032		− cpu_pmu->name = "armv8_cortex_a57";
1033		− cpu_pmu->map_event = armv8_a57_map_event;
1034		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1035		− &armv8_pmuv3_events_attr_group;
1036		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1037		− &armv8_pmuv3_format_attr_group;
	1058	+ return armv8_pmu_init(cpu_pmu, "armv8_cortex_a57",
	1059	+ armv8_a57_map_event, NULL, NULL);
	1060	+}
1038	1061
1039		− return 0;
	1062	+static int armv8_a65_pmu_init(struct arm_pmu *cpu_pmu)
	1063	+{
	1064	+ return armv8_pmu_init(cpu_pmu, "armv8_cortex_a65",
	1065	+ armv8_pmuv3_map_event, NULL, NULL);
1040	1066	}
1041	1067
1042	1068	static int armv8_a72_pmu_init(struct arm_pmu *cpu_pmu)
1043	1069	{
1044		− int ret = armv8_pmu_init(cpu_pmu);
1045		− if (ret)
1046		− return ret;
1047		−
1048		− cpu_pmu->name = "armv8_cortex_a72";
1049		− cpu_pmu->map_event = armv8_a57_map_event;
1050		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1051		− &armv8_pmuv3_events_attr_group;
1052		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1053		− &armv8_pmuv3_format_attr_group;
1054		−
1055		− return 0;
	1070	+ return armv8_pmu_init(cpu_pmu, "armv8_cortex_a72",
	1071	+ armv8_a57_map_event, NULL, NULL);
1056	1072	}
1057	1073
1058	1074	static int armv8_a73_pmu_init(struct arm_pmu *cpu_pmu)
1059	1075	{
1060		− int ret = armv8_pmu_init(cpu_pmu);
1061		− if (ret)
1062		− return ret;
1063		−
1064		− cpu_pmu->name = "armv8_cortex_a73";
1065		− cpu_pmu->map_event = armv8_a73_map_event;
1066		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1067		− &armv8_pmuv3_events_attr_group;
1068		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1069		− &armv8_pmuv3_format_attr_group;
	1076	+ return armv8_pmu_init(cpu_pmu, "armv8_cortex_a73",
	1077	+ armv8_a73_map_event, NULL, NULL);
	1078	+}
1070	1079
1071		− return 0;
	1080	+static int armv8_a75_pmu_init(struct arm_pmu *cpu_pmu)
	1081	+{
	1082	+ return armv8_pmu_init(cpu_pmu, "armv8_cortex_a75",
	1083	+ armv8_pmuv3_map_event, NULL, NULL);
1072	1084	}
1073	1085
1074		−static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)
	1086	+static int armv8_a76_pmu_init(struct arm_pmu *cpu_pmu)
1075	1087	{
1076		− int ret = armv8_pmu_init(cpu_pmu);
1077		− if (ret)
1078		− return ret;
	1088	+ return armv8_pmu_init(cpu_pmu, "armv8_cortex_a76",
	1089	+ armv8_pmuv3_map_event, NULL, NULL);
	1090	+}
1079	1091
1080		− cpu_pmu->name = "armv8_cavium_thunder";
1081		− cpu_pmu->map_event = armv8_thunder_map_event;
1082		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1083		− &armv8_pmuv3_events_attr_group;
1084		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1085		− &armv8_pmuv3_format_attr_group;
	1092	+static int armv8_a77_pmu_init(struct arm_pmu *cpu_pmu)
	1093	+{
	1094	+ return armv8_pmu_init(cpu_pmu, "armv8_cortex_a77",
	1095	+ armv8_pmuv3_map_event, NULL, NULL);
	1096	+}
1086	1097
1087		− return 0;
	1098	+static int armv8_e1_pmu_init(struct arm_pmu *cpu_pmu)
	1099	+{
	1100	+ return armv8_pmu_init(cpu_pmu, "armv8_neoverse_e1",
	1101	+ armv8_pmuv3_map_event, NULL, NULL);
1088	1102	}
1089	1103
1090		−static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu)
	1104	+static int armv8_n1_pmu_init(struct arm_pmu *cpu_pmu)
1091	1105	{
1092		− int ret = armv8_pmu_init(cpu_pmu);
1093		− if (ret)
1094		− return ret;
	1106	+ return armv8_pmu_init(cpu_pmu, "armv8_neoverse_n1",
	1107	+ armv8_pmuv3_map_event, NULL, NULL);
	1108	+}
1095	1109
1096		− cpu_pmu->name = "armv8_brcm_vulcan";
1097		− cpu_pmu->map_event = armv8_vulcan_map_event;
1098		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_EVENTS] =
1099		− &armv8_pmuv3_events_attr_group;
1100		− cpu_pmu->attr_groups[ARMPMU_ATTR_GROUP_FORMATS] =
1101		− &armv8_pmuv3_format_attr_group;
	1110	+static int armv8_thunder_pmu_init(struct arm_pmu *cpu_pmu)
	1111	+{
	1112	+ return armv8_pmu_init(cpu_pmu, "armv8_cavium_thunder",
	1113	+ armv8_thunder_map_event, NULL, NULL);
	1114	+}
1102	1115
1103		− return 0;
	1116	+static int armv8_vulcan_pmu_init(struct arm_pmu *cpu_pmu)
	1117	+{
	1118	+ return armv8_pmu_init(cpu_pmu, "armv8_brcm_vulcan",
	1119	+ armv8_vulcan_map_event, NULL, NULL);
1104	1120	}
1105	1121
1106	1122	static const struct of_device_id armv8_pmu_of_device_ids[] = {
1107	1123	{.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_init},
	1124	+ {.compatible = "arm,cortex-a34-pmu", .data = armv8_a34_pmu_init},
1108	1125	{.compatible = "arm,cortex-a35-pmu", .data = armv8_a35_pmu_init},
1109	1126	{.compatible = "arm,cortex-a53-pmu", .data = armv8_a53_pmu_init},
	1127	+ {.compatible = "arm,cortex-a55-pmu", .data = armv8_a55_pmu_init},
1110	1128	{.compatible = "arm,cortex-a57-pmu", .data = armv8_a57_pmu_init},
	1129	+ {.compatible = "arm,cortex-a65-pmu", .data = armv8_a65_pmu_init},
1111	1130	{.compatible = "arm,cortex-a72-pmu", .data = armv8_a72_pmu_init},
1112	1131	{.compatible = "arm,cortex-a73-pmu", .data = armv8_a73_pmu_init},
	1132	+ {.compatible = "arm,cortex-a75-pmu", .data = armv8_a75_pmu_init},
	1133	+ {.compatible = "arm,cortex-a76-pmu", .data = armv8_a76_pmu_init},
	1134	+ {.compatible = "arm,cortex-a77-pmu", .data = armv8_a77_pmu_init},
	1135	+ {.compatible = "arm,neoverse-e1-pmu", .data = armv8_e1_pmu_init},
	1136	+ {.compatible = "arm,neoverse-n1-pmu", .data = armv8_n1_pmu_init},
1113	1137	{.compatible = "cavium,thunder-pmu", .data = armv8_thunder_pmu_init},
1114	1138	{.compatible = "brcm,vulcan-pmu", .data = armv8_vulcan_pmu_init},
1115	1139	{},

arch/arm64/kernel/pointer_auth.c+2−5

@@ -9,7 +9,7 @@ · 8 unmodified lines
9	9
10	10	int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg)
11	11	{
12		− struct ptrauth_keys *keys = &tsk->thread.keys_user;
	12	+ struct ptrauth_keys_user *keys = &tsk->thread.keys_user;
13	13	unsigned long addr_key_mask = PR_PAC_APIAKEY \| PR_PAC_APIBKEY \|
14	14	PR_PAC_APDAKEY \| PR_PAC_APDBKEY;
15	15	unsigned long key_mask = addr_key_mask \| PR_PAC_APGAKEY;
@@ -18,8 +18,7 @@ int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg) · 2 unmodified lines
18	18	return -EINVAL;
19	19
20	20	if (!arg) {
21		− ptrauth_keys_init(keys);
22		− ptrauth_keys_switch(keys);
	21	+ ptrauth_keys_init_user(keys);
23	22	return 0;
24	23	}
25	24
@@ -41,7 +40,5 @@ int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg) · 15 unmodified lines
41	40	if (arg & PR_PAC_APGAKEY)
42	41	get_random_bytes(&keys->apga, sizeof(keys->apga));
43	42
44		− ptrauth_keys_switch(keys);
45		−
46	43	return 0;
47	44	}

arch/arm64/kernel/process.c+3−2

@@ -262,7 +262,7 @@ void __show_regs(struct pt_regs *regs) · 261 unmodified lines
262	262
263	263	if (!user_mode(regs)) {
264	264	printk("pc : %pS\n", (void *)regs->pc);
265		− printk("lr : %pS\n", (void *)lr);
	265	+ printk("lr : %pS\n", (void *)ptrauth_strip_insn_pac(lr));
266	266	} else {
267	267	printk("pc : %016llx\n", regs->pc);
268	268	printk("lr : %016llx\n", lr);
@@ -376,6 +376,8 @@ int copy_thread_tls(unsigned long clone_flags, unsigned long stack_start, · 107 unmodified lines
376	376	*/
377	377	fpsimd_flush_task_state(p);
378	378
	379	+ ptrauth_thread_init_kernel(p);
	380	+
379	381	if (likely(!(p->flags & PF_KTHREAD))) {
380	382	childregs = current_pt_regs();
381	383	childregs->regs[0] = 0;
@@ -512,7 +514,6 @@ __notrace_funcgraph struct task_struct __switch_to(struct task_struct prev, · 130 unmodified lines
512	514	contextidr_thread_switch(next);
513	515	entry_task_switch(next);
514	516	uao_thread_switch(next);
515		− ptrauth_thread_switch(next);
516	517	ssbs_thread_switch(next);
517	518
518	519	/*

arch/arm64/kernel/ptrace.c+8−8

@@ -999,7 +999,7 @@ static struct ptrauth_key pac_key_from_user(__uint128_t ukey) · 998 unmodified lines
999	999	}
1000	1000
1001	1001	static void pac_address_keys_to_user(struct user_pac_address_keys *ukeys,
1002		− const struct ptrauth_keys *keys)
	1002	+ const struct ptrauth_keys_user *keys)
1003	1003	{
1004	1004	ukeys->apiakey = pac_key_to_user(&keys->apia);
1005	1005	ukeys->apibkey = pac_key_to_user(&keys->apib);
@@ -1007,7 +1007,7 @@ static void pac_address_keys_to_user(struct user_pac_address_keys *ukeys, · 1 unmodified line
1007	1007	ukeys->apdbkey = pac_key_to_user(&keys->apdb);
1008	1008	}
1009	1009
1010		−static void pac_address_keys_from_user(struct ptrauth_keys *keys,
	1010	+static void pac_address_keys_from_user(struct ptrauth_keys_user *keys,
1011	1011	const struct user_pac_address_keys *ukeys)
1012	1012	{
1013	1013	keys->apia = pac_key_from_user(ukeys->apiakey);
@@ -1021,7 +1021,7 @@ static int pac_address_keys_get(struct task_struct *target, · 7 unmodified lines
1021	1021	unsigned int pos, unsigned int count,
1022	1022	void kbuf, void __user ubuf)
1023	1023	{
1024		− struct ptrauth_keys *keys = &target->thread.keys_user;
	1024	+ struct ptrauth_keys_user *keys = &target->thread.keys_user;
1025	1025	struct user_pac_address_keys user_keys;
1026	1026
1027	1027	if (!system_supports_address_auth())
@@ -1038,7 +1038,7 @@ static int pac_address_keys_set(struct task_struct *target, · 10 unmodified lines
1038	1038	unsigned int pos, unsigned int count,
1039	1039	const void kbuf, const void __user ubuf)
1040	1040	{
1041		− struct ptrauth_keys *keys = &target->thread.keys_user;
	1041	+ struct ptrauth_keys_user *keys = &target->thread.keys_user;
1042	1042	struct user_pac_address_keys user_keys;
1043	1043	int ret;
1044	1044
@@ -1056,12 +1056,12 @@ static int pac_address_keys_set(struct task_struct *target, · 11 unmodified lines
1056	1056	}
1057	1057
1058	1058	static void pac_generic_keys_to_user(struct user_pac_generic_keys *ukeys,
1059		− const struct ptrauth_keys *keys)
	1059	+ const struct ptrauth_keys_user *keys)
1060	1060	{
1061	1061	ukeys->apgakey = pac_key_to_user(&keys->apga);
1062	1062	}
1063	1063
1064		−static void pac_generic_keys_from_user(struct ptrauth_keys *keys,
	1064	+static void pac_generic_keys_from_user(struct ptrauth_keys_user *keys,
1065	1065	const struct user_pac_generic_keys *ukeys)
1066	1066	{
1067	1067	keys->apga = pac_key_from_user(ukeys->apgakey);
@@ -1072,7 +1072,7 @@ static int pac_generic_keys_get(struct task_struct *target, · 4 unmodified lines
1072	1072	unsigned int pos, unsigned int count,
1073	1073	void kbuf, void __user ubuf)
1074	1074	{
1075		− struct ptrauth_keys *keys = &target->thread.keys_user;
	1075	+ struct ptrauth_keys_user *keys = &target->thread.keys_user;
1076	1076	struct user_pac_generic_keys user_keys;
1077	1077
1078	1078	if (!system_supports_generic_auth())
@@ -1089,7 +1089,7 @@ static int pac_generic_keys_set(struct task_struct *target, · 10 unmodified lines
1089	1089	unsigned int pos, unsigned int count,
1090	1090	const void kbuf, const void __user ubuf)
1091	1091	{
1092		− struct ptrauth_keys *keys = &target->thread.keys_user;
	1092	+ struct ptrauth_keys_user *keys = &target->thread.keys_user;
1093	1093	struct user_pac_generic_keys user_keys;
1094	1094	int ret;
1095	1095

arch/arm64/kernel/relocate_kernel.S+1−3

@@ -41,7 +41,7 @@ ENTRY(arm64_relocate_new_kernel) · 40 unmodified lines
41	41	cmp x0, #CurrentEL_EL2
42	42	b.ne 1f
43	43	mrs x0, sctlr_el2
44		− ldr x1, =SCTLR_ELx_FLAGS
	44	+ mov_q x1, SCTLR_ELx_FLAGS
45	45	bic x0, x0, x1
46	46	pre_disable_mmu_workaround
47	47	msr sctlr_el2, x0
@@ -113,8 +113,6 @@ ENTRY(arm64_relocate_new_kernel) · 65 unmodified lines
113	113
114	114	ENDPROC(arm64_relocate_new_kernel)
115	115
116		−.ltorg
117		−
118	116	.align 3 /* To keep the 64-bit values below naturally aligned. */
119	117
120	118	.Lcopy_end:

arch/arm64/kernel/setup.c+5−3

@@ -344,7 +344,7 @@ void __init setup_arch(char **cmdline_p) · 343 unmodified lines
344	344	else
345	345	psci_acpi_init();
346	346
347		− cpu_read_bootcpu_ops();
	347	+ init_bootcpu_ops();
348	348	smp_init_cpus();
349	349	smp_build_mpidr_hash();
350	350
@@ -371,8 +371,10 @@ void __init setup_arch(char **cmdline_p) · 20 unmodified lines
371	371	static inline bool cpu_can_disable(unsigned int cpu)
372	372	{
373	373	#ifdef CONFIG_HOTPLUG_CPU
374		− if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_can_disable)
375		− return cpu_ops[cpu]->cpu_can_disable(cpu);
	374	+ const struct cpu_operations *ops = get_cpu_ops(cpu);
	375	+
	376	+ if (ops && ops->cpu_can_disable)
	377	+ return ops->cpu_can_disable(cpu);
376	378	#endif
377	379	return false;
378	380	}

arch/arm64/kernel/sleep.S+2−0

@@ -3,6 +3,7 @@ · 2 unmodified lines
3	3	#include <linux/linkage.h>
4	4	#include <asm/asm-offsets.h>
5	5	#include <asm/assembler.h>
	6	+#include <asm/smp.h>
6	7
7	8	.text
8	9	/*
@@ -99,6 +100,7 @@ ENDPROC(__cpu_suspend_enter) · 90 unmodified lines
99	100	.pushsection ".idmap.text", "awx"
100	101	ENTRY(cpu_resume)
101	102	bl el2_setup // if in EL2 drop to EL1 cleanly
	103	+ mov x0, #ARM64_CPU_RUNTIME
102	104	bl __cpu_setup
103	105	/* enable the MMU early - so we can access sleep_save_stash by va */
104	106	adrp x1, swapper_pg_dir

arch/arm64/kernel/smp.c+92−67

@@ -93,8 +93,10 @@ static inline int op_cpu_kill(unsigned int cpu) · 92 unmodified lines
93	93	*/
94	94	static int boot_secondary(unsigned int cpu, struct task_struct *idle)
95	95	{
96		− if (cpu_ops[cpu]->cpu_boot)
97		− return cpu_ops[cpu]->cpu_boot(cpu);
	96	+ const struct cpu_operations *ops = get_cpu_ops(cpu);
	97	+
	98	+ if (ops->cpu_boot)
	99	+ return ops->cpu_boot(cpu);
98	100
99	101	return -EOPNOTSUPP;
100	102	}
@@ -112,63 +114,66 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) · 11 unmodified lines
112	114	*/
113	115	secondary_data.task = idle;
114	116	secondary_data.stack = task_stack_page(idle) + THREAD_SIZE;
	117	+#if defined(CONFIG_ARM64_PTR_AUTH)
	118	+ secondary_data.ptrauth_key.apia.lo = idle->thread.keys_kernel.apia.lo;
	119	+ secondary_data.ptrauth_key.apia.hi = idle->thread.keys_kernel.apia.hi;
	120	+#endif
115	121	update_cpu_boot_status(CPU_MMU_OFF);
116	122	__flush_dcache_area(&secondary_data, sizeof(secondary_data));
117	123
118		− /*
119		− * Now bring the CPU into our world.
120		− */
	124	+ /* Now bring the CPU into our world */
121	125	ret = boot_secondary(cpu, idle);
122		− if (ret == 0) {
123		− /*
124		− * CPU was successfully started, wait for it to come online or
125		− * time out.
126		− */
127		− wait_for_completion_timeout(&cpu_running,
128		− msecs_to_jiffies(5000));
129		−
130		− if (!cpu_online(cpu)) {
131		− pr_crit("CPU%u: failed to come online\n", cpu);
132		− ret = -EIO;
133		− }
134		− } else {
	126	+ if (ret) {
135	127	pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
136	128	return ret;
137	129	}
138	130
	131	+ /*
	132	+ * CPU was successfully started, wait for it to come online or
	133	+ * time out.
	134	+ */
	135	+ wait_for_completion_timeout(&cpu_running,
	136	+ msecs_to_jiffies(5000));
	137	+ if (cpu_online(cpu))
	138	+ return 0;
	139	+
	140	+ pr_crit("CPU%u: failed to come online\n", cpu);
139	141	secondary_data.task = NULL;
140	142	secondary_data.stack = NULL;
	143	+#if defined(CONFIG_ARM64_PTR_AUTH)
	144	+ secondary_data.ptrauth_key.apia.lo = 0;
	145	+ secondary_data.ptrauth_key.apia.hi = 0;
	146	+#endif
141	147	__flush_dcache_area(&secondary_data, sizeof(secondary_data));
142	148	status = READ_ONCE(secondary_data.status);
143		− if (ret && status) {
144		−
145		− if (status == CPU_MMU_OFF)
146		− status = READ_ONCE(__early_cpu_boot_status);
	149	+ if (status == CPU_MMU_OFF)
	150	+ status = READ_ONCE(__early_cpu_boot_status);
147	151
148		− switch (status & CPU_BOOT_STATUS_MASK) {
149		− default:
150		− pr_err("CPU%u: failed in unknown state : 0x%lx\n",
151		− cpu, status);
152		− cpus_stuck_in_kernel++;
153		− break;
154		− case CPU_KILL_ME:
155		− if (!op_cpu_kill(cpu)) {
156		− pr_crit("CPU%u: died during early boot\n", cpu);
157		− break;
158		− }
159		− pr_crit("CPU%u: may not have shut down cleanly\n", cpu);
160		− /* Fall through */
161		− case CPU_STUCK_IN_KERNEL:
162		− pr_crit("CPU%u: is stuck in kernel\n", cpu);
163		− if (status & CPU_STUCK_REASON_52_BIT_VA)
164		− pr_crit("CPU%u: does not support 52-bit VAs\n", cpu);
165		− if (status & CPU_STUCK_REASON_NO_GRAN)
166		− pr_crit("CPU%u: does not support %luK granule \n", cpu, PAGE_SIZE / SZ_1K);
167		− cpus_stuck_in_kernel++;
	152	+ switch (status & CPU_BOOT_STATUS_MASK) {
	153	+ default:
	154	+ pr_err("CPU%u: failed in unknown state : 0x%lx\n",
	155	+ cpu, status);
	156	+ cpus_stuck_in_kernel++;
	157	+ break;
	158	+ case CPU_KILL_ME:
	159	+ if (!op_cpu_kill(cpu)) {
	160	+ pr_crit("CPU%u: died during early boot\n", cpu);
168	161	break;
169		− case CPU_PANIC_KERNEL:
170		− panic("CPU%u detected unsupported configuration\n", cpu);
171	162	}
	163	+ pr_crit("CPU%u: may not have shut down cleanly\n", cpu);
	164	+ /* Fall through */
	165	+ case CPU_STUCK_IN_KERNEL:
	166	+ pr_crit("CPU%u: is stuck in kernel\n", cpu);
	167	+ if (status & CPU_STUCK_REASON_52_BIT_VA)
	168	+ pr_crit("CPU%u: does not support 52-bit VAs\n", cpu);
	169	+ if (status & CPU_STUCK_REASON_NO_GRAN) {
	170	+ pr_crit("CPU%u: does not support %luK granule\n",
	171	+ cpu, PAGE_SIZE / SZ_1K);
	172	+ }
	173	+ cpus_stuck_in_kernel++;
	174	+ break;
	175	+ case CPU_PANIC_KERNEL:
	176	+ panic("CPU%u detected unsupported configuration\n", cpu);
172	177	}
173	178
174	179	return ret;
@@ -196,6 +201,7 @@ asmlinkage notrace void secondary_start_kernel(void) · 21 unmodified lines
196	201	{
197	202	u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
198	203	struct mm_struct *mm = &init_mm;
	204	+ const struct cpu_operations *ops;
199	205	unsigned int cpu;
200	206
201	207	cpu = task_cpu(current);
@@ -227,8 +233,9 @@ asmlinkage notrace void secondary_start_kernel(void) · 25 unmodified lines
227	233	*/
228	234	check_local_cpu_capabilities();
229	235
230		− if (cpu_ops[cpu]->cpu_postboot)
231		− cpu_ops[cpu]->cpu_postboot();
	236	+ ops = get_cpu_ops(cpu);
	237	+ if (ops->cpu_postboot)
	238	+ ops->cpu_postboot();
232	239
233	240	/*
234	241	* Log the CPU info before it is marked online and might get read.
@@ -266,19 +273,21 @@ asmlinkage notrace void secondary_start_kernel(void) · 31 unmodified lines
266	273	#ifdef CONFIG_HOTPLUG_CPU
267	274	static int op_cpu_disable(unsigned int cpu)
268	275	{
	276	+ const struct cpu_operations *ops = get_cpu_ops(cpu);
	277	+
269	278	/*
270	279	* If we don't have a cpu_die method, abort before we reach the point
271	280	* of no return. CPU0 may not have an cpu_ops, so test for it.
272	281	*/
273		− if (!cpu_ops[cpu] \|\| !cpu_ops[cpu]->cpu_die)
	282	+ if (!ops \|\| !ops->cpu_die)
274	283	return -EOPNOTSUPP;
275	284
276	285	/*
277	286	* We may need to abort a hot unplug for some other mechanism-specific
278	287	* reason.
279	288	*/
280		− if (cpu_ops[cpu]->cpu_disable)
281		− return cpu_ops[cpu]->cpu_disable(cpu);
	289	+ if (ops->cpu_disable)
	290	+ return ops->cpu_disable(cpu);
282	291
283	292	return 0;
284	293	}
@@ -314,15 +323,17 @@ int __cpu_disable(void) · 29 unmodified lines
314	323
315	324	static int op_cpu_kill(unsigned int cpu)
316	325	{
	326	+ const struct cpu_operations *ops = get_cpu_ops(cpu);
	327	+
317	328	/*
318	329	* If we have no means of synchronising with the dying CPU, then assume
319	330	* that it is really dead. We can only wait for an arbitrary length of
320	331	* time and hope that it's dead, so let's skip the wait and just hope.
321	332	*/
322		− if (!cpu_ops[cpu]->cpu_kill)
	333	+ if (!ops->cpu_kill)
323	334	return 0;
324	335
325		− return cpu_ops[cpu]->cpu_kill(cpu);
	336	+ return ops->cpu_kill(cpu);
326	337	}
327	338
328	339	/*
@@ -357,6 +368,7 @@ void __cpu_die(unsigned int cpu) · 28 unmodified lines
357	368	void cpu_die(void)
358	369	{
359	370	unsigned int cpu = smp_processor_id();
	371	+ const struct cpu_operations *ops = get_cpu_ops(cpu);
360	372
361	373	idle_task_exit();
362	374
@@ -370,12 +382,22 @@ void cpu_die(void) · 7 unmodified lines
370	382	* mechanism must perform all required cache maintenance to ensure that
371	383	* no dirty lines are lost in the process of shutting down the CPU.
372	384	*/
373		− cpu_ops[cpu]->cpu_die(cpu);
	385	+ ops->cpu_die(cpu);
374	386
375	387	BUG();
376	388	}
377	389	#endif
378	390
	391	+static void __cpu_try_die(int cpu)
	392	+{
	393	+#ifdef CONFIG_HOTPLUG_CPU
	394	+ const struct cpu_operations *ops = get_cpu_ops(cpu);
	395	+
	396	+ if (ops && ops->cpu_die)
	397	+ ops->cpu_die(cpu);
	398	+#endif
	399	+}
	400	+
379	401	/*
380	402	* Kill the calling secondary CPU, early in bringup before it is turned
381	403	* online.
@@ -389,12 +411,11 @@ void cpu_die_early(void) · 7 unmodified lines
389	411	/* Mark this CPU absent */
390	412	set_cpu_present(cpu, 0);
391	413
392		−#ifdef CONFIG_HOTPLUG_CPU
393		− update_cpu_boot_status(CPU_KILL_ME);
394		− /* Check if we can park ourselves */
395		− if (cpu_ops[cpu] && cpu_ops[cpu]->cpu_die)
396		− cpu_ops[cpu]->cpu_die(cpu);
397		−#endif
	414	+ if (IS_ENABLED(CONFIG_HOTPLUG_CPU)) {
	415	+ update_cpu_boot_status(CPU_KILL_ME);
	416	+ __cpu_try_die(cpu);
	417	+ }
	418	+
398	419	update_cpu_boot_status(CPU_STUCK_IN_KERNEL);
399	420
400	421	cpu_park_loop();
@@ -488,10 +509,13 @@ static bool __init is_mpidr_duplicate(unsigned int cpu, u64 hwid) · 87 unmodified lines
488	509	*/
489	510	static int __init smp_cpu_setup(int cpu)
490	511	{
491		− if (cpu_read_ops(cpu))
	512	+ const struct cpu_operations *ops;
	513	+
	514	+ if (init_cpu_ops(cpu))
492	515	return -ENODEV;
493	516
494		− if (cpu_ops[cpu]->cpu_init(cpu))
	517	+ ops = get_cpu_ops(cpu);
	518	+ if (ops->cpu_init(cpu))
495	519	return -ENODEV;
496	520
497	521	set_cpu_possible(cpu, true);
@@ -714,6 +738,7 @@ void __init smp_init_cpus(void) · 216 unmodified lines
714	738
715	739	void __init smp_prepare_cpus(unsigned int max_cpus)
716	740	{
	741	+ const struct cpu_operations *ops;
717	742	int err;
718	743	unsigned int cpu;
719	744	unsigned int this_cpu;
@@ -744,10 +769,11 @@ void __init smp_prepare_cpus(unsigned int max_cpus) · 24 unmodified lines
744	769	if (cpu == smp_processor_id())
745	770	continue;
746	771
747		− if (!cpu_ops[cpu])
	772	+ ops = get_cpu_ops(cpu);
	773	+ if (!ops)
748	774	continue;
749	775
750		− err = cpu_ops[cpu]->cpu_prepare(cpu);
	776	+ err = ops->cpu_prepare(cpu);
751	777	if (err)
752	778	continue;
753	779
@@ -863,10 +889,8 @@ static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs) · 109 unmodified lines
863	889	local_irq_disable();
864	890	sdei_mask_local_cpu();
865	891
866		−#ifdef CONFIG_HOTPLUG_CPU
867		− if (cpu_ops[cpu]->cpu_die)
868		− cpu_ops[cpu]->cpu_die(cpu);
869		−#endif
	892	+ if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
	893	+ __cpu_try_die(cpu);
870	894
871	895	/* just in case */
872	896	cpu_park_loop();
@@ -1059,8 +1083,9 @@ static bool have_cpu_die(void) · 186 unmodified lines
1059	1083	{
1060	1084	#ifdef CONFIG_HOTPLUG_CPU
1061	1085	int any_cpu = raw_smp_processor_id();
	1086	+ const struct cpu_operations *ops = get_cpu_ops(any_cpu);
1062	1087
1063		− if (cpu_ops[any_cpu] && cpu_ops[any_cpu]->cpu_die)
	1088	+ if (ops && ops->cpu_die)
1064	1089	return true;
1065	1090	#endif
1066	1091	return false;

arch/arm64/kernel/stacktrace.c+4−1

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <linux/stacktrace.h>
15	15
16	16	#include <asm/irq.h>
	17	+#include <asm/pointer_auth.h>
17	18	#include <asm/stack_pointer.h>
18	19	#include <asm/stacktrace.h>
19	20
@@ -86,7 +87,7 @@ int notrace unwind_frame(struct task_struct tsk, struct stackframe frame) · 66 unmodified lines
86	87
87	88	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
88	89	if (tsk->ret_stack &&
89		− (frame->pc == (unsigned long)return_to_handler)) {
	90	+ (ptrauth_strip_insn_pac(frame->pc) == (unsigned long)return_to_handler)) {
90	91	struct ftrace_ret_stack *ret_stack;
91	92	/*
92	93	* This is a case where function graph tracer has
@@ -101,6 +102,8 @@ int notrace unwind_frame(struct task_struct tsk, struct stackframe frame) · 8 unmodified lines
101	102	}
102	103	#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
103	104
	105	+ frame->pc = ptrauth_strip_insn_pac(frame->pc);
	106	+
104	107	/*
105	108	* Frames created upon entry from EL0 have NULL FP and PC values, so
106	109	* don't bother reporting these. Frames created by __noreturn functions

arch/arm64/kernel/topology.c+180−0

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include <linux/acpi.h>
15	15	#include <linux/arch_topology.h>
16	16	#include <linux/cacheinfo.h>
	17	+#include <linux/cpufreq.h>
17	18	#include <linux/init.h>
18	19	#include <linux/percpu.h>
19	20
@@ -120,4 +121,183 @@ int __init parse_acpi_topology(void) · 100 unmodified lines
120	121	}
121	122	#endif
122	123
	124	+#ifdef CONFIG_ARM64_AMU_EXTN
123	125
	126	+#undef pr_fmt
	127	+#define pr_fmt(fmt) "AMU: " fmt
	128	+
	129	+static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, arch_max_freq_scale);
	130	+static DEFINE_PER_CPU(u64, arch_const_cycles_prev);
	131	+static DEFINE_PER_CPU(u64, arch_core_cycles_prev);
	132	+static cpumask_var_t amu_fie_cpus;
	133	+
	134	+/* Initialize counter reference per-cpu variables for the current CPU */
	135	+void init_cpu_freq_invariance_counters(void)
	136	+{
	137	+ this_cpu_write(arch_core_cycles_prev,
	138	+ read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0));
	139	+ this_cpu_write(arch_const_cycles_prev,
	140	+ read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0));
	141	+}
	142	+
	143	+static int validate_cpu_freq_invariance_counters(int cpu)
	144	+{
	145	+ u64 max_freq_hz, ratio;
	146	+
	147	+ if (!cpu_has_amu_feat(cpu)) {
	148	+ pr_debug("CPU%d: counters are not supported.\n", cpu);
	149	+ return -EINVAL;
	150	+ }
	151	+
	152	+ if (unlikely(!per_cpu(arch_const_cycles_prev, cpu) \|\|
	153	+ !per_cpu(arch_core_cycles_prev, cpu))) {
	154	+ pr_debug("CPU%d: cycle counters are not enabled.\n", cpu);
	155	+ return -EINVAL;
	156	+ }
	157	+
	158	+ /* Convert maximum frequency from KHz to Hz and validate */
	159	+ max_freq_hz = cpufreq_get_hw_max_freq(cpu) * 1000;
	160	+ if (unlikely(!max_freq_hz)) {
	161	+ pr_debug("CPU%d: invalid maximum frequency.\n", cpu);
	162	+ return -EINVAL;
	163	+ }
	164	+
	165	+ /*
	166	+ * Pre-compute the fixed ratio between the frequency of the constant
	167	+ * counter and the maximum frequency of the CPU.
	168	+ *
	169	+ * const_freq
	170	+ * arch_max_freq_scale = ---------------- * SCHED_CAPACITY_SCALE²
	171	+ * cpuinfo_max_freq
	172	+ *
	173	+ * We use a factor of 2 * SCHED_CAPACITY_SHIFT -> SCHED_CAPACITY_SCALE²
	174	+ * in order to ensure a good resolution for arch_max_freq_scale for
	175	+ * very low arch timer frequencies (down to the KHz range which should
	176	+ * be unlikely).
	177	+ */
	178	+ ratio = (u64)arch_timer_get_rate() << (2 * SCHED_CAPACITY_SHIFT);
	179	+ ratio = div64_u64(ratio, max_freq_hz);
	180	+ if (!ratio) {
	181	+ WARN_ONCE(1, "System timer frequency too low.\n");
	182	+ return -EINVAL;
	183	+ }
	184	+
	185	+ per_cpu(arch_max_freq_scale, cpu) = (unsigned long)ratio;
	186	+
	187	+ return 0;
	188	+}
	189	+
	190	+static inline bool
	191	+enable_policy_freq_counters(int cpu, cpumask_var_t valid_cpus)
	192	+{
	193	+ struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
	194	+
	195	+ if (!policy) {
	196	+ pr_debug("CPU%d: No cpufreq policy found.\n", cpu);
	197	+ return false;
	198	+ }
	199	+
	200	+ if (cpumask_subset(policy->related_cpus, valid_cpus))
	201	+ cpumask_or(amu_fie_cpus, policy->related_cpus,
	202	+ amu_fie_cpus);
	203	+
	204	+ cpufreq_cpu_put(policy);
	205	+
	206	+ return true;
	207	+}
	208	+
	209	+static DEFINE_STATIC_KEY_FALSE(amu_fie_key);
	210	+#define amu_freq_invariant() static_branch_unlikely(&amu_fie_key)
	211	+
	212	+static int __init init_amu_fie(void)
	213	+{
	214	+ cpumask_var_t valid_cpus;
	215	+ bool have_policy = false;
	216	+ int ret = 0;
	217	+ int cpu;
	218	+
	219	+ if (!zalloc_cpumask_var(&valid_cpus, GFP_KERNEL))
	220	+ return -ENOMEM;
	221	+
	222	+ if (!zalloc_cpumask_var(&amu_fie_cpus, GFP_KERNEL)) {
	223	+ ret = -ENOMEM;
	224	+ goto free_valid_mask;
	225	+ }
	226	+
	227	+ for_each_present_cpu(cpu) {
	228	+ if (validate_cpu_freq_invariance_counters(cpu))
	229	+ continue;
	230	+ cpumask_set_cpu(cpu, valid_cpus);
	231	+ have_policy \|= enable_policy_freq_counters(cpu, valid_cpus);
	232	+ }
	233	+
	234	+ /*
	235	+ * If we are not restricted by cpufreq policies, we only enable
	236	+ * the use of the AMU feature for FIE if all CPUs support AMU.
	237	+ * Otherwise, enable_policy_freq_counters has already enabled
	238	+ * policy cpus.
	239	+ */
	240	+ if (!have_policy && cpumask_equal(valid_cpus, cpu_present_mask))
	241	+ cpumask_or(amu_fie_cpus, amu_fie_cpus, valid_cpus);
	242	+
	243	+ if (!cpumask_empty(amu_fie_cpus)) {
	244	+ pr_info("CPUs[%*pbl]: counters will be used for FIE.",
	245	+ cpumask_pr_args(amu_fie_cpus));
	246	+ static_branch_enable(&amu_fie_key);
	247	+ }
	248	+
	249	+free_valid_mask:
	250	+ free_cpumask_var(valid_cpus);
	251	+
	252	+ return ret;
	253	+}
	254	+late_initcall_sync(init_amu_fie);
	255	+
	256	+bool arch_freq_counters_available(struct cpumask *cpus)
	257	+{
	258	+ return amu_freq_invariant() &&
	259	+ cpumask_subset(cpus, amu_fie_cpus);
	260	+}
	261	+
	262	+void topology_scale_freq_tick(void)
	263	+{
	264	+ u64 prev_core_cnt, prev_const_cnt;
	265	+ u64 core_cnt, const_cnt, scale;
	266	+ int cpu = smp_processor_id();
	267	+
	268	+ if (!amu_freq_invariant())
	269	+ return;
	270	+
	271	+ if (!cpumask_test_cpu(cpu, amu_fie_cpus))
	272	+ return;
	273	+
	274	+ const_cnt = read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0);
	275	+ core_cnt = read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0);
	276	+ prev_const_cnt = this_cpu_read(arch_const_cycles_prev);
	277	+ prev_core_cnt = this_cpu_read(arch_core_cycles_prev);
	278	+
	279	+ if (unlikely(core_cnt <= prev_core_cnt \|\|
	280	+ const_cnt <= prev_const_cnt))
	281	+ goto store_and_exit;
	282	+
	283	+ /*
	284	+ * /\core arch_max_freq_scale
	285	+ * scale = ------- * --------------------
	286	+ * /\const SCHED_CAPACITY_SCALE
	287	+ *
	288	+ * See validate_cpu_freq_invariance_counters() for details on
	289	+ * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT.
	290	+ */
	291	+ scale = core_cnt - prev_core_cnt;
	292	+ scale *= this_cpu_read(arch_max_freq_scale);
	293	+ scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT,
	294	+ const_cnt - prev_const_cnt);
	295	+
	296	+ scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE);
	297	+ this_cpu_write(freq_scale, (unsigned long)scale);
	298	+
	299	+store_and_exit:
	300	+ this_cpu_write(arch_core_cycles_prev, core_cnt);
	301	+ this_cpu_write(arch_const_cycles_prev, const_cnt);
	302	+}
	303	+#endif /* CONFIG_ARM64_AMU_EXTN */

arch/arm64/kernel/vdso/sigreturn.S+2−2

@@ -14,7 +14,7 @@ · 13 unmodified lines
14	14	.text
15	15
16	16	nop
17		−ENTRY(__kernel_rt_sigreturn)
	17	+SYM_FUNC_START(__kernel_rt_sigreturn)
18	18	.cfi_startproc
19	19	.cfi_signal_frame
20	20	.cfi_def_cfa x29, 0
@@ -23,4 +23,4 @@ ENTRY(__kernel_rt_sigreturn) · 2 unmodified lines
23	23	mov x8, #__NR_rt_sigreturn
24	24	svc #0
25	25	.cfi_endproc
26		−ENDPROC(__kernel_rt_sigreturn)
	26	+SYM_FUNC_END(__kernel_rt_sigreturn)

arch/arm64/kernel/vdso32/sigreturn.S+8−15

@@ -10,13 +10,6 @@ · 9 unmodified lines
10	10	#include <asm/asm-offsets.h>
11	11	#include <asm/unistd.h>
12	12
13		−#define ARM_ENTRY(name) \
14		− ENTRY(name)
15		−
16		−#define ARM_ENDPROC(name) \
17		− .type name, %function; \
18		− END(name)
19		−
20	13	.text
21	14
22	15	.arm
@@ -24,39 +17,39 @@ · 1 unmodified line
24	17	.save {r0-r15}
25	18	.pad #COMPAT_SIGFRAME_REGS_OFFSET
26	19	nop
27		−ARM_ENTRY(__kernel_sigreturn_arm)
	20	+SYM_FUNC_START(__kernel_sigreturn_arm)
28	21	mov r7, #__NR_compat_sigreturn
29	22	svc #0
30	23	.fnend
31		−ARM_ENDPROC(__kernel_sigreturn_arm)
	24	+SYM_FUNC_END(__kernel_sigreturn_arm)
32	25
33	26	.fnstart
34	27	.save {r0-r15}
35	28	.pad #COMPAT_RT_SIGFRAME_REGS_OFFSET
36	29	nop
37		−ARM_ENTRY(__kernel_rt_sigreturn_arm)
	30	+SYM_FUNC_START(__kernel_rt_sigreturn_arm)
38	31	mov r7, #__NR_compat_rt_sigreturn
39	32	svc #0
40	33	.fnend
41		−ARM_ENDPROC(__kernel_rt_sigreturn_arm)
	34	+SYM_FUNC_END(__kernel_rt_sigreturn_arm)
42	35
43	36	.thumb
44	37	.fnstart
45	38	.save {r0-r15}
46	39	.pad #COMPAT_SIGFRAME_REGS_OFFSET
47	40	nop
48		−ARM_ENTRY(__kernel_sigreturn_thumb)
	41	+SYM_FUNC_START(__kernel_sigreturn_thumb)
49	42	mov r7, #__NR_compat_sigreturn
50	43	svc #0
51	44	.fnend
52		−ARM_ENDPROC(__kernel_sigreturn_thumb)
	45	+SYM_FUNC_END(__kernel_sigreturn_thumb)
53	46
54	47	.fnstart
55	48	.save {r0-r15}
56	49	.pad #COMPAT_RT_SIGFRAME_REGS_OFFSET
57	50	nop
58		−ARM_ENTRY(__kernel_rt_sigreturn_thumb)
	51	+SYM_FUNC_START(__kernel_rt_sigreturn_thumb)
59	52	mov r7, #__NR_compat_rt_sigreturn
60	53	svc #0
61	54	.fnend
62		−ARM_ENDPROC(__kernel_rt_sigreturn_thumb)
	55	+SYM_FUNC_END(__kernel_rt_sigreturn_thumb)

arch/arm64/kvm/hyp-init.S+8−10

@@ -18,7 +18,7 @@ · 17 unmodified lines
18	18
19	19	.align 11
20	20
21		−ENTRY(__kvm_hyp_init)
	21	+SYM_CODE_START(__kvm_hyp_init)
22	22	ventry __invalid // Synchronous EL2t
23	23	ventry __invalid // IRQ EL2t
24	24	ventry __invalid // FIQ EL2t
@@ -60,7 +60,7 @@ alternative_else_nop_endif · 35 unmodified lines
60	60	msr ttbr0_el2, x4
61	61
62	62	mrs x4, tcr_el1
63		− ldr x5, =TCR_EL2_MASK
	63	+ mov_q x5, TCR_EL2_MASK
64	64	and x4, x4, x5
65	65	mov x5, #TCR_EL2_RES1
66	66	orr x4, x4, x5
@@ -102,7 +102,7 @@ alternative_else_nop_endif · 35 unmodified lines
102	102	* as well as the EE bit on BE. Drop the A flag since the compiler
103	103	* is allowed to generate unaligned accesses.
104	104	*/
105		− ldr x4, =(SCTLR_EL2_RES1 \| (SCTLR_ELx_FLAGS & ~SCTLR_ELx_A))
	105	+ mov_q x4, (SCTLR_EL2_RES1 \| (SCTLR_ELx_FLAGS & ~SCTLR_ELx_A))
106	106	CPU_BE( orr x4, x4, #SCTLR_ELx_EE)
107	107	msr sctlr_el2, x4
108	108	isb
@@ -117,9 +117,9 @@ CPU_BE( orr x4, x4, #SCTLR_ELx_EE) · 8 unmodified lines
117	117
118	118	/* Hello, World! */
119	119	eret
120		−ENDPROC(__kvm_hyp_init)
	120	+SYM_CODE_END(__kvm_hyp_init)
121	121
122		−ENTRY(__kvm_handle_stub_hvc)
	122	+SYM_CODE_START(__kvm_handle_stub_hvc)
123	123	cmp x0, #HVC_SOFT_RESTART
124	124	b.ne 1f
125	125
@@ -142,7 +142,7 @@ reset: · 16 unmodified lines
142	142	* case we coming via HVC_SOFT_RESTART.
143	143	*/
144	144	mrs x5, sctlr_el2
145		− ldr x6, =SCTLR_ELx_FLAGS
	145	+ mov_q x6, SCTLR_ELx_FLAGS
146	146	bic x5, x5, x6 // Clear SCTL_M and etc
147	147	pre_disable_mmu_workaround
148	148	msr sctlr_el2, x5
@@ -155,11 +155,9 @@ reset: · 6 unmodified lines
155	155	eret
156	156
157	157	1: /* Bad stub call */
158		− ldr x0, =HVC_STUB_ERR
	158	+ mov_q x0, HVC_STUB_ERR
159	159	eret
160	160
161		−ENDPROC(__kvm_handle_stub_hvc)
162		−
163		− .ltorg
	161	+SYM_CODE_END(__kvm_handle_stub_hvc)
164	162
165	163	.popsection

arch/arm64/kvm/hyp.S+2−2

@@ -28,7 +28,7 @@ · 27 unmodified lines
28	28	* and is used to implement hyp stubs in the same way as in
29	29	* arch/arm64/kernel/hyp_stub.S.
30	30	*/
31		−ENTRY(__kvm_call_hyp)
	31	+SYM_FUNC_START(__kvm_call_hyp)
32	32	hvc #0
33	33	ret
34		−ENDPROC(__kvm_call_hyp)
	34	+SYM_FUNC_END(__kvm_call_hyp)

arch/arm64/kvm/hyp/fpsimd.S+4−4

@@ -11,12 +11,12 @@ · 10 unmodified lines
11	11	.text
12	12	.pushsection .hyp.text, "ax"
13	13
14		−ENTRY(__fpsimd_save_state)
	14	+SYM_FUNC_START(__fpsimd_save_state)
15	15	fpsimd_save x0, 1
16	16	ret
17		−ENDPROC(__fpsimd_save_state)
	17	+SYM_FUNC_END(__fpsimd_save_state)
18	18
19		−ENTRY(__fpsimd_restore_state)
	19	+SYM_FUNC_START(__fpsimd_restore_state)
20	20	fpsimd_restore x0, 1
21	21	ret
22		−ENDPROC(__fpsimd_restore_state)
	22	+SYM_FUNC_END(__fpsimd_restore_state)

arch/arm64/kvm/hyp/hyp-entry.S+16−11

@@ -180,7 +180,7 @@ el2_error: · 179 unmodified lines
180	180	eret
181	181	sb
182	182
183		−ENTRY(__hyp_do_panic)
	183	+SYM_FUNC_START(__hyp_do_panic)
184	184	mov lr, #(PSR_F_BIT \| PSR_I_BIT \| PSR_A_BIT \| PSR_D_BIT \|\
185	185	PSR_MODE_EL1h)
186	186	msr spsr_el2, lr
@@ -188,18 +188,19 @@ ENTRY(__hyp_do_panic) · 1 unmodified line
188	188	msr elr_el2, lr
189	189	eret
190	190	sb
191		−ENDPROC(__hyp_do_panic)
	191	+SYM_FUNC_END(__hyp_do_panic)
192	192
193		−ENTRY(__hyp_panic)
	193	+SYM_CODE_START(__hyp_panic)
194	194	get_host_ctxt x0, x1
195	195	b hyp_panic
196		−ENDPROC(__hyp_panic)
	196	+SYM_CODE_END(__hyp_panic)
197	197
198	198	.macro invalid_vector label, target = __hyp_panic
199	199	.align 2
	200	+SYM_CODE_START(\label)
200	201	\label:
201	202	b \target
202		−ENDPROC(\label)
	203	+SYM_CODE_END(\label)
203	204	.endm
204	205
205	206	/* None of these should ever happen */
@@ -246,7 +247,7 @@ check_preamble_length 661b, 662b · 40 unmodified lines
246	247	check_preamble_length 661b, 662b
247	248	.endm
248	249
249		−ENTRY(__kvm_hyp_vector)
	250	+SYM_CODE_START(__kvm_hyp_vector)
250	251	invalid_vect el2t_sync_invalid // Synchronous EL2t
251	252	invalid_vect el2t_irq_invalid // IRQ EL2t
252	253	invalid_vect el2t_fiq_invalid // FIQ EL2t
@@ -266,7 +267,7 @@ ENTRY(__kvm_hyp_vector) · 13 unmodified lines
266	267	valid_vect el1_irq // IRQ 32-bit EL1
267	268	invalid_vect el1_fiq_invalid // FIQ 32-bit EL1
268	269	valid_vect el1_error // Error 32-bit EL1
269		−ENDPROC(__kvm_hyp_vector)
	270	+SYM_CODE_END(__kvm_hyp_vector)
270	271
271	272	#ifdef CONFIG_KVM_INDIRECT_VECTORS
272	273	.macro hyp_ventry
@@ -311,15 +312,17 @@ alternative_cb_end · 38 unmodified lines
311	312	.endm
312	313
313	314	.align 11
314		−ENTRY(__bp_harden_hyp_vecs_start)
	315	+SYM_CODE_START(__bp_harden_hyp_vecs)
315	316	.rept BP_HARDEN_EL2_SLOTS
316	317	generate_vectors
317	318	.endr
318		−ENTRY(__bp_harden_hyp_vecs_end)
	319	+1: .org __bp_harden_hyp_vecs + __BP_HARDEN_HYP_VECS_SZ
	320	+ .org 1b
	321	+SYM_CODE_END(__bp_harden_hyp_vecs)
319	322
320	323	.popsection
321	324
322		−ENTRY(__smccc_workaround_1_smc_start)
	325	+SYM_CODE_START(__smccc_workaround_1_smc)
323	326	esb
324	327	sub sp, sp, #(8 * 4)
325	328	stp x2, x3, [sp, #(8 * 0)]
@@ -329,5 +332,7 @@ ENTRY(__smccc_workaround_1_smc_start) · 3 unmodified lines
329	332	ldp x2, x3, [sp, #(8 * 0)]
330	333	ldp x0, x1, [sp, #(8 * 2)]
331	334	add sp, sp, #(8 * 4)
332		−ENTRY(__smccc_workaround_1_smc_end)
	335	+1: .org __smccc_workaround_1_smc + __SMCCC_WORKAROUND_1_SMC_SZ
	336	+ .org 1b
	337	+SYM_CODE_END(__smccc_workaround_1_smc)
333	338	#endif

arch/arm64/kvm/hyp/switch.c+20−8

@@ -98,6 +98,18 @@ static void activate_traps_vhe(struct kvm_vcpu *vcpu) · 97 unmodified lines
98	98	val = read_sysreg(cpacr_el1);
99	99	val \|= CPACR_EL1_TTA;
100	100	val &= ~CPACR_EL1_ZEN;
	101	+
	102	+ /*
	103	+ * With VHE (HCR.E2H == 1), accesses to CPACR_EL1 are routed to
	104	+ * CPTR_EL2. In general, CPACR_EL1 has the same layout as CPTR_EL2,
	105	+ * except for some missing controls, such as TAM.
	106	+ * In this case, CPTR_EL2.TAM has the same position with or without
	107	+ * VHE (HCR.E2H == 1) which allows us to use here the CPTR_EL2.TAM
	108	+ * shift value for trapping the AMU accesses.
	109	+ */
	110	+
	111	+ val \|= CPTR_EL2_TAM;
	112	+
101	113	if (update_fp_enabled(vcpu)) {
102	114	if (vcpu_has_sve(vcpu))
103	115	val \|= CPACR_EL1_ZEN;
@@ -119,7 +131,7 @@ static void __hyp_text __activate_traps_nvhe(struct kvm_vcpu *vcpu) · 15 unmodified lines
119	131	__activate_traps_common(vcpu);
120	132
121	133	val = CPTR_EL2_DEFAULT;
122		− val \|= CPTR_EL2_TTA \| CPTR_EL2_TZ;
	134	+ val \|= CPTR_EL2_TTA \| CPTR_EL2_TZ \| CPTR_EL2_TAM;
123	135	if (!update_fp_enabled(vcpu)) {
124	136	val \|= CPTR_EL2_TFP;
125	137	__activate_traps_fpsimd32(vcpu);
@@ -127,7 +139,7 @@ static void __hyp_text __activate_traps_nvhe(struct kvm_vcpu *vcpu) · 1 unmodified line
127	139
128	140	write_sysreg(val, cptr_el2);
129	141
130		− if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) {
	142	+ if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) {
131	143	struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt;
132	144
133	145	isb();
@@ -146,12 +158,12 @@ static void __hyp_text __activate_traps(struct kvm_vcpu *vcpu) · 12 unmodified lines
146	158	{
147	159	u64 hcr = vcpu->arch.hcr_el2;
148	160
149		− if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM))
	161	+ if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM))
150	162	hcr \|= HCR_TVM;
151	163
152	164	write_sysreg(hcr, hcr_el2);
153	165
154		− if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE))
	166	+ if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE))
155	167	write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2);
156	168
157	169	if (has_vhe())
@@ -181,7 +193,7 @@ static void __hyp_text __deactivate_traps_nvhe(void) · 23 unmodified lines
181	193	{
182	194	u64 mdcr_el2 = read_sysreg(mdcr_el2);
183	195
184		− if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) {
	196	+ if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) {
185	197	u64 val;
186	198
187	199	/*
@@ -328,7 +340,7 @@ static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu) · 140 unmodified lines
328	340	* resolve the IPA using the AT instruction.
329	341	*/
330	342	if (!(esr & ESR_ELx_S1PTW) &&
331		− (cpus_have_const_cap(ARM64_WORKAROUND_834220) \|\|
	343	+ (cpus_have_final_cap(ARM64_WORKAROUND_834220) \|\|
332	344	(esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) {
333	345	if (!__translate_far_to_hpfar(far, &hpfar))
334	346	return false;
@@ -498,7 +510,7 @@ static bool __hyp_text fixup_guest_exit(struct kvm_vcpu vcpu, u64 exit_code) · 163 unmodified lines
498	510	if (*exit_code != ARM_EXCEPTION_TRAP)
499	511	goto exit;
500	512
501		− if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) &&
	513	+ if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM) &&
502	514	kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_SYS64 &&
503	515	handle_tx2_tvm(vcpu))
504	516	return true;
@@ -555,7 +567,7 @@ exit: · 50 unmodified lines
555	567
556	568	static inline bool __hyp_text __needs_ssbd_off(struct kvm_vcpu *vcpu)
557	569	{
558		− if (!cpus_have_const_cap(ARM64_SSBD))
	570	+ if (!cpus_have_final_cap(ARM64_SSBD))
559	571	return false;
560	572
561	573	return !(vcpu->arch.workaround_flags & VCPU_WORKAROUND_2_FLAG);

arch/arm64/kvm/hyp/sysreg-sr.c+4−4

@@ -71,7 +71,7 @@ static void __hyp_text __sysreg_save_el2_return_state(struct kvm_cpu_context *ct · 70 unmodified lines
71	71	ctxt->gp_regs.regs.pc = read_sysreg_el2(SYS_ELR);
72	72	ctxt->gp_regs.regs.pstate = read_sysreg_el2(SYS_SPSR);
73	73
74		− if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN))
	74	+ if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
75	75	ctxt->sys_regs[DISR_EL1] = read_sysreg_s(SYS_VDISR_EL2);
76	76	}
77	77
@@ -118,7 +118,7 @@ static void __hyp_text __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt) · 40 unmodified lines
118	118	write_sysreg(ctxt->sys_regs[MPIDR_EL1], vmpidr_el2);
119	119	write_sysreg(ctxt->sys_regs[CSSELR_EL1], csselr_el1);
120	120
121		− if (!cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) {
	121	+ if (!cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) {
122	122	write_sysreg_el1(ctxt->sys_regs[SCTLR_EL1], SYS_SCTLR);
123	123	write_sysreg_el1(ctxt->sys_regs[TCR_EL1], SYS_TCR);
124	124	} else if (!ctxt->__hyp_running_vcpu) {
@@ -149,7 +149,7 @@ static void __hyp_text __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt) · 24 unmodified lines
149	149	write_sysreg(ctxt->sys_regs[PAR_EL1], par_el1);
150	150	write_sysreg(ctxt->sys_regs[TPIDR_EL1], tpidr_el1);
151	151
152		− if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE) &&
	152	+ if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE) &&
153	153	ctxt->__hyp_running_vcpu) {
154	154	/*
155	155	* Must only be done for host registers, hence the context
@@ -194,7 +194,7 @@ __sysreg_restore_el2_return_state(struct kvm_cpu_context *ctxt) · 38 unmodified lines
194	194	write_sysreg_el2(ctxt->gp_regs.regs.pc, SYS_ELR);
195	195	write_sysreg_el2(pstate, SYS_SPSR);
196	196
197		− if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN))
	197	+ if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
198	198	write_sysreg_s(ctxt->sys_regs[DISR_EL1], SYS_VDISR_EL2);
199	199	}
200	200

arch/arm64/kvm/hyp/tlb.c+4−4

@@ -23,7 +23,7 @@ static void __hyp_text __tlb_switch_to_guest_vhe(struct kvm *kvm, · 22 unmodified lines
23	23
24	24	local_irq_save(cxt->flags);
25	25
26		− if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_VHE)) {
	26	+ if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_VHE)) {
27	27	/*
28	28	* For CPUs that are affected by ARM errata 1165522 or 1530923,
29	29	* we cannot trust stage-1 to be in a correct state at that
@@ -63,7 +63,7 @@ static void __hyp_text __tlb_switch_to_guest_vhe(struct kvm *kvm, · 33 unmodified lines
63	63	static void __hyp_text __tlb_switch_to_guest_nvhe(struct kvm *kvm,
64	64	struct tlb_inv_context *cxt)
65	65	{
66		− if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) {
	66	+ if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) {
67	67	u64 val;
68	68
69	69	/*
@@ -103,7 +103,7 @@ static void __hyp_text __tlb_switch_to_host_vhe(struct kvm *kvm, · 33 unmodified lines
103	103	write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
104	104	isb();
105	105
106		− if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_VHE)) {
	106	+ if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_VHE)) {
107	107	/* Restore the registers to what they were */
108	108	write_sysreg_el1(cxt->tcr, SYS_TCR);
109	109	write_sysreg_el1(cxt->sctlr, SYS_SCTLR);
@@ -117,7 +117,7 @@ static void __hyp_text __tlb_switch_to_host_nvhe(struct kvm *kvm, · 7 unmodified lines
117	117	{
118	118	write_sysreg(0, vttbr_el2);
119	119
120		− if (cpus_have_const_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) {
	120	+ if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT_NVHE)) {
121	121	/* Ensure write of the host VMID */
122	122	isb();
123	123	/* Restore the host's TCR_EL1 */

arch/arm64/kvm/sys_regs.c+101−2

@@ -1003,6 +1003,20 @@ static bool access_pmuserenr(struct kvm_vcpu vcpu, struct sys_reg_params p, · 1002 unmodified lines
1003	1003	{ SYS_DESC(SYS_PMEVTYPERn_EL0(n)), \
1004	1004	access_pmu_evtyper, reset_unknown, (PMEVTYPER0_EL0 + n), }
1005	1005
	1006	+static bool access_amu(struct kvm_vcpu vcpu, struct sys_reg_params p,
	1007	+ const struct sys_reg_desc *r)
	1008	+{
	1009	+ kvm_inject_undefined(vcpu);
	1010	+
	1011	+ return false;
	1012	+}
	1013	+
	1014	+/* Macro to expand the AMU counter and type registers*/
	1015	+#define AMU_AMEVCNTR0_EL0(n) { SYS_DESC(SYS_AMEVCNTR0_EL0(n)), access_amu }
	1016	+#define AMU_AMEVTYPE0_EL0(n) { SYS_DESC(SYS_AMEVTYPE0_EL0(n)), access_amu }
	1017	+#define AMU_AMEVCNTR1_EL0(n) { SYS_DESC(SYS_AMEVCNTR1_EL0(n)), access_amu }
	1018	+#define AMU_AMEVTYPE1_EL0(n) { SYS_DESC(SYS_AMEVTYPE1_EL0(n)), access_amu }
	1019	+
1006	1020	static bool trap_ptrauth(struct kvm_vcpu *vcpu,
1007	1021	struct sys_reg_params *p,
1008	1022	const struct sys_reg_desc *rd)
@@ -1078,13 +1092,25 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, · 69 unmodified lines
1078	1092	(u32)r->CRn, (u32)r->CRm, (u32)r->Op2);
1079	1093	u64 val = raz ? 0 : read_sanitised_ftr_reg(id);
1080	1094
1081		− if (id == SYS_ID_AA64PFR0_EL1 && !vcpu_has_sve(vcpu)) {
1082		− val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT);
	1095	+ if (id == SYS_ID_AA64PFR0_EL1) {
	1096	+ if (!vcpu_has_sve(vcpu))
	1097	+ val &= ~(0xfUL << ID_AA64PFR0_SVE_SHIFT);
	1098	+ val &= ~(0xfUL << ID_AA64PFR0_AMU_SHIFT);
1083	1099	} else if (id == SYS_ID_AA64ISAR1_EL1 && !vcpu_has_ptrauth(vcpu)) {
1084	1100	val &= ~((0xfUL << ID_AA64ISAR1_APA_SHIFT) \|
1085	1101	(0xfUL << ID_AA64ISAR1_API_SHIFT) \|
1086	1102	(0xfUL << ID_AA64ISAR1_GPA_SHIFT) \|
1087	1103	(0xfUL << ID_AA64ISAR1_GPI_SHIFT));
	1104	+ } else if (id == SYS_ID_AA64DFR0_EL1) {
	1105	+ /* Limit guests to PMUv3 for ARMv8.1 */
	1106	+ val = cpuid_feature_cap_perfmon_field(val,
	1107	+ ID_AA64DFR0_PMUVER_SHIFT,
	1108	+ ID_AA64DFR0_PMUVER_8_1);
	1109	+ } else if (id == SYS_ID_DFR0_EL1) {
	1110	+ /* Limit guests to PMUv3 for ARMv8.1 */
	1111	+ val = cpuid_feature_cap_perfmon_field(val,
	1112	+ ID_DFR0_PERFMON_SHIFT,
	1113	+ ID_DFR0_PERFMON_8_1);
1088	1114	}
1089	1115
1090	1116	return val;
@@ -1565,6 +1591,79 @@ static const struct sys_reg_desc sys_reg_descs[] = { · 474 unmodified lines
1565	1591	{ SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
1566	1592	{ SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
1567	1593
	1594	+ { SYS_DESC(SYS_AMCR_EL0), access_amu },
	1595	+ { SYS_DESC(SYS_AMCFGR_EL0), access_amu },
	1596	+ { SYS_DESC(SYS_AMCGCR_EL0), access_amu },
	1597	+ { SYS_DESC(SYS_AMUSERENR_EL0), access_amu },
	1598	+ { SYS_DESC(SYS_AMCNTENCLR0_EL0), access_amu },
	1599	+ { SYS_DESC(SYS_AMCNTENSET0_EL0), access_amu },
	1600	+ { SYS_DESC(SYS_AMCNTENCLR1_EL0), access_amu },
	1601	+ { SYS_DESC(SYS_AMCNTENSET1_EL0), access_amu },
	1602	+ AMU_AMEVCNTR0_EL0(0),
	1603	+ AMU_AMEVCNTR0_EL0(1),
	1604	+ AMU_AMEVCNTR0_EL0(2),
	1605	+ AMU_AMEVCNTR0_EL0(3),
	1606	+ AMU_AMEVCNTR0_EL0(4),
	1607	+ AMU_AMEVCNTR0_EL0(5),
	1608	+ AMU_AMEVCNTR0_EL0(6),
	1609	+ AMU_AMEVCNTR0_EL0(7),
	1610	+ AMU_AMEVCNTR0_EL0(8),
	1611	+ AMU_AMEVCNTR0_EL0(9),
	1612	+ AMU_AMEVCNTR0_EL0(10),
	1613	+ AMU_AMEVCNTR0_EL0(11),
	1614	+ AMU_AMEVCNTR0_EL0(12),
	1615	+ AMU_AMEVCNTR0_EL0(13),
	1616	+ AMU_AMEVCNTR0_EL0(14),
	1617	+ AMU_AMEVCNTR0_EL0(15),
	1618	+ AMU_AMEVTYPE0_EL0(0),
	1619	+ AMU_AMEVTYPE0_EL0(1),
	1620	+ AMU_AMEVTYPE0_EL0(2),
	1621	+ AMU_AMEVTYPE0_EL0(3),
	1622	+ AMU_AMEVTYPE0_EL0(4),
	1623	+ AMU_AMEVTYPE0_EL0(5),
	1624	+ AMU_AMEVTYPE0_EL0(6),
	1625	+ AMU_AMEVTYPE0_EL0(7),
	1626	+ AMU_AMEVTYPE0_EL0(8),
	1627	+ AMU_AMEVTYPE0_EL0(9),
	1628	+ AMU_AMEVTYPE0_EL0(10),
	1629	+ AMU_AMEVTYPE0_EL0(11),
	1630	+ AMU_AMEVTYPE0_EL0(12),
	1631	+ AMU_AMEVTYPE0_EL0(13),
	1632	+ AMU_AMEVTYPE0_EL0(14),
	1633	+ AMU_AMEVTYPE0_EL0(15),
	1634	+ AMU_AMEVCNTR1_EL0(0),
	1635	+ AMU_AMEVCNTR1_EL0(1),
	1636	+ AMU_AMEVCNTR1_EL0(2),
	1637	+ AMU_AMEVCNTR1_EL0(3),
	1638	+ AMU_AMEVCNTR1_EL0(4),
	1639	+ AMU_AMEVCNTR1_EL0(5),
	1640	+ AMU_AMEVCNTR1_EL0(6),
	1641	+ AMU_AMEVCNTR1_EL0(7),
	1642	+ AMU_AMEVCNTR1_EL0(8),
	1643	+ AMU_AMEVCNTR1_EL0(9),
	1644	+ AMU_AMEVCNTR1_EL0(10),
	1645	+ AMU_AMEVCNTR1_EL0(11),
	1646	+ AMU_AMEVCNTR1_EL0(12),
	1647	+ AMU_AMEVCNTR1_EL0(13),
	1648	+ AMU_AMEVCNTR1_EL0(14),
	1649	+ AMU_AMEVCNTR1_EL0(15),
	1650	+ AMU_AMEVTYPE1_EL0(0),
	1651	+ AMU_AMEVTYPE1_EL0(1),
	1652	+ AMU_AMEVTYPE1_EL0(2),
	1653	+ AMU_AMEVTYPE1_EL0(3),
	1654	+ AMU_AMEVTYPE1_EL0(4),
	1655	+ AMU_AMEVTYPE1_EL0(5),
	1656	+ AMU_AMEVTYPE1_EL0(6),
	1657	+ AMU_AMEVTYPE1_EL0(7),
	1658	+ AMU_AMEVTYPE1_EL0(8),
	1659	+ AMU_AMEVTYPE1_EL0(9),
	1660	+ AMU_AMEVTYPE1_EL0(10),
	1661	+ AMU_AMEVTYPE1_EL0(11),
	1662	+ AMU_AMEVTYPE1_EL0(12),
	1663	+ AMU_AMEVTYPE1_EL0(13),
	1664	+ AMU_AMEVTYPE1_EL0(14),
	1665	+ AMU_AMEVTYPE1_EL0(15),
	1666	+
1568	1667	{ SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },
1569	1668	{ SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer },
1570	1669	{ SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer },

arch/arm64/lib/csum.c+27−0

@@ -124,3 +124,30 @@ unsigned int do_csum(const unsigned char *buff, int len) · 123 unmodified lines
124	124
125	125	return sum >> 16;
126	126	}
	127	+
	128	+__sum16 csum_ipv6_magic(const struct in6_addr *saddr,
	129	+ const struct in6_addr *daddr,
	130	+ __u32 len, __u8 proto, __wsum csum)
	131	+{
	132	+ __uint128_t src, dst;
	133	+ u64 sum = (__force u64)csum;
	134	+
	135	+ src = (const __uint128_t )saddr->s6_addr;
	136	+ dst = (const __uint128_t )daddr->s6_addr;
	137	+
	138	+ sum += (__force u32)htonl(len);
	139	+#ifdef __LITTLE_ENDIAN
	140	+ sum += (u32)proto << 24;
	141	+#else
	142	+ sum += proto;
	143	+#endif
	144	+ src += (src >> 64) \| (src << 64);
	145	+ dst += (dst >> 64) \| (dst << 64);
	146	+
	147	+ sum = accumulate(sum, src >> 64);
	148	+ sum = accumulate(sum, dst >> 64);
	149	+
	150	+ sum += ((sum >> 32) \| (sum << 32));
	151	+ return csum_fold((__force __wsum)(sum >> 32));
	152	+}
	153	+EXPORT_SYMBOL(csum_ipv6_magic);

arch/arm64/lib/strcmp.S+1−1

@@ -186,7 +186,7 @@ CPU_LE( rev data2, data2 ) · 185 unmodified lines
186	186	* as carry-propagation can corrupt the upper bits if the trailing
187	187	* bytes in the string contain 0x01.
188	188	* However, if there is no NUL byte in the dword, we can generate
189		− * the result directly. We ca not just subtract the bytes as the
	189	+ * the result directly. We cannot just subtract the bytes as the
190	190	* MSB might be significant.
191	191	*/
192	192	CPU_BE( cbnz has_nul, 1f )

arch/arm64/mm/context.c+30−2

@@ -6,6 +6,7 @@ · 5 unmodified lines
6	6	* Copyright (C) 2012 ARM Ltd.
7	7	*/
8	8
	9	+#include <linux/bitfield.h>
9	10	#include <linux/bitops.h>
10	11	#include <linux/sched.h>
11	12	#include <linux/slab.h>
@@ -254,10 +255,37 @@ switch_mm_fastpath: · 242 unmodified lines
254	255	/* Errata workaround post TTBRx_EL1 update. */
255	256	asmlinkage void post_ttbr_update_workaround(void)
256	257	{
	258	+ if (!IS_ENABLED(CONFIG_CAVIUM_ERRATUM_27456))
	259	+ return;
	260	+
257	261	asm(ALTERNATIVE("nop; nop; nop",
258	262	"ic iallu; dsb nsh; isb",
259		− ARM64_WORKAROUND_CAVIUM_27456,
260		− CONFIG_CAVIUM_ERRATUM_27456));
	263	+ ARM64_WORKAROUND_CAVIUM_27456));
	264	+}
	265	+
	266	+void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm)
	267	+{
	268	+ unsigned long ttbr1 = read_sysreg(ttbr1_el1);
	269	+ unsigned long asid = ASID(mm);
	270	+ unsigned long ttbr0 = phys_to_ttbr(pgd_phys);
	271	+
	272	+ /* Skip CNP for the reserved ASID */
	273	+ if (system_supports_cnp() && asid)
	274	+ ttbr0 \|= TTBR_CNP_BIT;
	275	+
	276	+ /* SW PAN needs a copy of the ASID in TTBR0 for entry */
	277	+ if (IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN))
	278	+ ttbr0 \|= FIELD_PREP(TTBR_ASID_MASK, asid);
	279	+
	280	+ /* Set ASID in TTBR1 since TCR.A1 is set */
	281	+ ttbr1 &= ~TTBR_ASID_MASK;
	282	+ ttbr1 \|= FIELD_PREP(TTBR_ASID_MASK, asid);
	283	+
	284	+ write_sysreg(ttbr1, ttbr1_el1);
	285	+ isb();
	286	+ write_sysreg(ttbr0, ttbr0_el1);
	287	+ isb();
	288	+ post_ttbr_update_workaround();
261	289	}
262	290
263	291	static int asids_update_limit(void)

arch/arm64/mm/mmu.c+370−9

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	17	#include <linux/mman.h>
18	18	#include <linux/nodemask.h>
19	19	#include <linux/memblock.h>
	20	+#include <linux/memory.h>
20	21	#include <linux/fs.h>
21	22	#include <linux/io.h>
22	23	#include <linux/mm.h>
@@ -724,6 +725,312 @@ int kern_addr_valid(unsigned long addr) · 701 unmodified lines
724	725
725	726	return pfn_valid(pte_pfn(pte));
726	727	}
	728	+
	729	+#ifdef CONFIG_MEMORY_HOTPLUG
	730	+static void free_hotplug_page_range(struct page *page, size_t size)
	731	+{
	732	+ WARN_ON(PageReserved(page));
	733	+ free_pages((unsigned long)page_address(page), get_order(size));
	734	+}
	735	+
	736	+static void free_hotplug_pgtable_page(struct page *page)
	737	+{
	738	+ free_hotplug_page_range(page, PAGE_SIZE);
	739	+}
	740	+
	741	+static bool pgtable_range_aligned(unsigned long start, unsigned long end,
	742	+ unsigned long floor, unsigned long ceiling,
	743	+ unsigned long mask)
	744	+{
	745	+ start &= mask;
	746	+ if (start < floor)
	747	+ return false;
	748	+
	749	+ if (ceiling) {
	750	+ ceiling &= mask;
	751	+ if (!ceiling)
	752	+ return false;
	753	+ }
	754	+
	755	+ if (end - 1 > ceiling - 1)
	756	+ return false;
	757	+ return true;
	758	+}
	759	+
	760	+static void unmap_hotplug_pte_range(pmd_t *pmdp, unsigned long addr,
	761	+ unsigned long end, bool free_mapped)
	762	+{
	763	+ pte_t *ptep, pte;
	764	+
	765	+ do {
	766	+ ptep = pte_offset_kernel(pmdp, addr);
	767	+ pte = READ_ONCE(*ptep);
	768	+ if (pte_none(pte))
	769	+ continue;
	770	+
	771	+ WARN_ON(!pte_present(pte));
	772	+ pte_clear(&init_mm, addr, ptep);
	773	+ flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
	774	+ if (free_mapped)
	775	+ free_hotplug_page_range(pte_page(pte), PAGE_SIZE);
	776	+ } while (addr += PAGE_SIZE, addr < end);
	777	+}
	778	+
	779	+static void unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr,
	780	+ unsigned long end, bool free_mapped)
	781	+{
	782	+ unsigned long next;
	783	+ pmd_t *pmdp, pmd;
	784	+
	785	+ do {
	786	+ next = pmd_addr_end(addr, end);
	787	+ pmdp = pmd_offset(pudp, addr);
	788	+ pmd = READ_ONCE(*pmdp);
	789	+ if (pmd_none(pmd))
	790	+ continue;
	791	+
	792	+ WARN_ON(!pmd_present(pmd));
	793	+ if (pmd_sect(pmd)) {
	794	+ pmd_clear(pmdp);
	795	+
	796	+ /*
	797	+ * One TLBI should be sufficient here as the PMD_SIZE
	798	+ * range is mapped with a single block entry.
	799	+ */
	800	+ flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
	801	+ if (free_mapped)
	802	+ free_hotplug_page_range(pmd_page(pmd),
	803	+ PMD_SIZE);
	804	+ continue;
	805	+ }
	806	+ WARN_ON(!pmd_table(pmd));
	807	+ unmap_hotplug_pte_range(pmdp, addr, next, free_mapped);
	808	+ } while (addr = next, addr < end);
	809	+}
	810	+
	811	+static void unmap_hotplug_pud_range(p4d_t *p4dp, unsigned long addr,
	812	+ unsigned long end, bool free_mapped)
	813	+{
	814	+ unsigned long next;
	815	+ pud_t *pudp, pud;
	816	+
	817	+ do {
	818	+ next = pud_addr_end(addr, end);
	819	+ pudp = pud_offset(p4dp, addr);
	820	+ pud = READ_ONCE(*pudp);
	821	+ if (pud_none(pud))
	822	+ continue;
	823	+
	824	+ WARN_ON(!pud_present(pud));
	825	+ if (pud_sect(pud)) {
	826	+ pud_clear(pudp);
	827	+
	828	+ /*
	829	+ * One TLBI should be sufficient here as the PUD_SIZE
	830	+ * range is mapped with a single block entry.
	831	+ */
	832	+ flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
	833	+ if (free_mapped)
	834	+ free_hotplug_page_range(pud_page(pud),
	835	+ PUD_SIZE);
	836	+ continue;
	837	+ }
	838	+ WARN_ON(!pud_table(pud));
	839	+ unmap_hotplug_pmd_range(pudp, addr, next, free_mapped);
	840	+ } while (addr = next, addr < end);
	841	+}
	842	+
	843	+static void unmap_hotplug_p4d_range(pgd_t *pgdp, unsigned long addr,
	844	+ unsigned long end, bool free_mapped)
	845	+{
	846	+ unsigned long next;
	847	+ p4d_t *p4dp, p4d;
	848	+
	849	+ do {
	850	+ next = p4d_addr_end(addr, end);
	851	+ p4dp = p4d_offset(pgdp, addr);
	852	+ p4d = READ_ONCE(*p4dp);
	853	+ if (p4d_none(p4d))
	854	+ continue;
	855	+
	856	+ WARN_ON(!p4d_present(p4d));
	857	+ unmap_hotplug_pud_range(p4dp, addr, next, free_mapped);
	858	+ } while (addr = next, addr < end);
	859	+}
	860	+
	861	+static void unmap_hotplug_range(unsigned long addr, unsigned long end,
	862	+ bool free_mapped)
	863	+{
	864	+ unsigned long next;
	865	+ pgd_t *pgdp, pgd;
	866	+
	867	+ do {
	868	+ next = pgd_addr_end(addr, end);
	869	+ pgdp = pgd_offset_k(addr);
	870	+ pgd = READ_ONCE(*pgdp);
	871	+ if (pgd_none(pgd))
	872	+ continue;
	873	+
	874	+ WARN_ON(!pgd_present(pgd));
	875	+ unmap_hotplug_p4d_range(pgdp, addr, next, free_mapped);
	876	+ } while (addr = next, addr < end);
	877	+}
	878	+
	879	+static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,
	880	+ unsigned long end, unsigned long floor,
	881	+ unsigned long ceiling)
	882	+{
	883	+ pte_t *ptep, pte;
	884	+ unsigned long i, start = addr;
	885	+
	886	+ do {
	887	+ ptep = pte_offset_kernel(pmdp, addr);
	888	+ pte = READ_ONCE(*ptep);
	889	+
	890	+ /*
	891	+ * This is just a sanity check here which verifies that
	892	+ * pte clearing has been done by earlier unmap loops.
	893	+ */
	894	+ WARN_ON(!pte_none(pte));
	895	+ } while (addr += PAGE_SIZE, addr < end);
	896	+
	897	+ if (!pgtable_range_aligned(start, end, floor, ceiling, PMD_MASK))
	898	+ return;
	899	+
	900	+ /*
	901	+ * Check whether we can free the pte page if the rest of the
	902	+ * entries are empty. Overlap with other regions have been
	903	+ * handled by the floor/ceiling check.
	904	+ */
	905	+ ptep = pte_offset_kernel(pmdp, 0UL);
	906	+ for (i = 0; i < PTRS_PER_PTE; i++) {
	907	+ if (!pte_none(READ_ONCE(ptep[i])))
	908	+ return;
	909	+ }
	910	+
	911	+ pmd_clear(pmdp);
	912	+ __flush_tlb_kernel_pgtable(start);
	913	+ free_hotplug_pgtable_page(virt_to_page(ptep));
	914	+}
	915	+
	916	+static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,
	917	+ unsigned long end, unsigned long floor,
	918	+ unsigned long ceiling)
	919	+{
	920	+ pmd_t *pmdp, pmd;
	921	+ unsigned long i, next, start = addr;
	922	+
	923	+ do {
	924	+ next = pmd_addr_end(addr, end);
	925	+ pmdp = pmd_offset(pudp, addr);
	926	+ pmd = READ_ONCE(*pmdp);
	927	+ if (pmd_none(pmd))
	928	+ continue;
	929	+
	930	+ WARN_ON(!pmd_present(pmd) \|\| !pmd_table(pmd) \|\| pmd_sect(pmd));
	931	+ free_empty_pte_table(pmdp, addr, next, floor, ceiling);
	932	+ } while (addr = next, addr < end);
	933	+
	934	+ if (CONFIG_PGTABLE_LEVELS <= 2)
	935	+ return;
	936	+
	937	+ if (!pgtable_range_aligned(start, end, floor, ceiling, PUD_MASK))
	938	+ return;
	939	+
	940	+ /*
	941	+ * Check whether we can free the pmd page if the rest of the
	942	+ * entries are empty. Overlap with other regions have been
	943	+ * handled by the floor/ceiling check.
	944	+ */
	945	+ pmdp = pmd_offset(pudp, 0UL);
	946	+ for (i = 0; i < PTRS_PER_PMD; i++) {
	947	+ if (!pmd_none(READ_ONCE(pmdp[i])))
	948	+ return;
	949	+ }
	950	+
	951	+ pud_clear(pudp);
	952	+ __flush_tlb_kernel_pgtable(start);
	953	+ free_hotplug_pgtable_page(virt_to_page(pmdp));
	954	+}
	955	+
	956	+static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,
	957	+ unsigned long end, unsigned long floor,
	958	+ unsigned long ceiling)
	959	+{
	960	+ pud_t *pudp, pud;
	961	+ unsigned long i, next, start = addr;
	962	+
	963	+ do {
	964	+ next = pud_addr_end(addr, end);
	965	+ pudp = pud_offset(p4dp, addr);
	966	+ pud = READ_ONCE(*pudp);
	967	+ if (pud_none(pud))
	968	+ continue;
	969	+
	970	+ WARN_ON(!pud_present(pud) \|\| !pud_table(pud) \|\| pud_sect(pud));
	971	+ free_empty_pmd_table(pudp, addr, next, floor, ceiling);
	972	+ } while (addr = next, addr < end);
	973	+
	974	+ if (CONFIG_PGTABLE_LEVELS <= 3)
	975	+ return;
	976	+
	977	+ if (!pgtable_range_aligned(start, end, floor, ceiling, PGDIR_MASK))
	978	+ return;
	979	+
	980	+ /*
	981	+ * Check whether we can free the pud page if the rest of the
	982	+ * entries are empty. Overlap with other regions have been
	983	+ * handled by the floor/ceiling check.
	984	+ */
	985	+ pudp = pud_offset(p4dp, 0UL);
	986	+ for (i = 0; i < PTRS_PER_PUD; i++) {
	987	+ if (!pud_none(READ_ONCE(pudp[i])))
	988	+ return;
	989	+ }
	990	+
	991	+ p4d_clear(p4dp);
	992	+ __flush_tlb_kernel_pgtable(start);
	993	+ free_hotplug_pgtable_page(virt_to_page(pudp));
	994	+}
	995	+
	996	+static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr,
	997	+ unsigned long end, unsigned long floor,
	998	+ unsigned long ceiling)
	999	+{
	1000	+ unsigned long next;
	1001	+ p4d_t *p4dp, p4d;
	1002	+
	1003	+ do {
	1004	+ next = p4d_addr_end(addr, end);
	1005	+ p4dp = p4d_offset(pgdp, addr);
	1006	+ p4d = READ_ONCE(*p4dp);
	1007	+ if (p4d_none(p4d))
	1008	+ continue;
	1009	+
	1010	+ WARN_ON(!p4d_present(p4d));
	1011	+ free_empty_pud_table(p4dp, addr, next, floor, ceiling);
	1012	+ } while (addr = next, addr < end);
	1013	+}
	1014	+
	1015	+static void free_empty_tables(unsigned long addr, unsigned long end,
	1016	+ unsigned long floor, unsigned long ceiling)
	1017	+{
	1018	+ unsigned long next;
	1019	+ pgd_t *pgdp, pgd;
	1020	+
	1021	+ do {
	1022	+ next = pgd_addr_end(addr, end);
	1023	+ pgdp = pgd_offset_k(addr);
	1024	+ pgd = READ_ONCE(*pgdp);
	1025	+ if (pgd_none(pgd))
	1026	+ continue;
	1027	+
	1028	+ WARN_ON(!pgd_present(pgd));
	1029	+ free_empty_p4d_table(pgdp, addr, next, floor, ceiling);
	1030	+ } while (addr = next, addr < end);
	1031	+}
	1032	+#endif
	1033	+
727	1034	#ifdef CONFIG_SPARSEMEM_VMEMMAP
728	1035	#if !ARM64_SWAPPER_USES_SECTION_MAPS
729	1036	int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node,
@@ -771,6 +1078,12 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, · 41 unmodified lines
771	1078	void vmemmap_free(unsigned long start, unsigned long end,
772	1079	struct vmem_altmap *altmap)
773	1080	{
	1081	+#ifdef CONFIG_MEMORY_HOTPLUG
	1082	+ WARN_ON((start < VMEMMAP_START) \|\| (end > VMEMMAP_END));
	1083	+
	1084	+ unmap_hotplug_range(start, end, true);
	1085	+ free_empty_tables(start, end, VMEMMAP_START, VMEMMAP_END);
	1086	+#endif
774	1087	}
775	1088	#endif /* CONFIG_SPARSEMEM_VMEMMAP */
776	1089
@@ -1049,10 +1362,21 @@ int p4d_free_pud_page(p4d_t *p4d, unsigned long addr) · 272 unmodified lines
1049	1362	}
1050	1363
1051	1364	#ifdef CONFIG_MEMORY_HOTPLUG
	1365	+static void __remove_pgd_mapping(pgd_t *pgdir, unsigned long start, u64 size)
	1366	+{
	1367	+ unsigned long end = start + size;
	1368	+
	1369	+ WARN_ON(pgdir != init_mm.pgd);
	1370	+ WARN_ON((start < PAGE_OFFSET) \|\| (end > PAGE_END));
	1371	+
	1372	+ unmap_hotplug_range(start, end, false);
	1373	+ free_empty_tables(start, end, PAGE_OFFSET, PAGE_END);
	1374	+}
	1375	+
1052	1376	int arch_add_memory(int nid, u64 start, u64 size,
1053	1377	struct mhp_restrictions *restrictions)
1054	1378	{
1055		− int flags = 0;
	1379	+ int ret, flags = 0;
1056	1380
1057	1381	if (rodata_full \|\| debug_pagealloc_enabled())
1058	1382	flags = NO_BLOCK_MAPPINGS \| NO_CONT_MAPPINGS;
@@ -1062,22 +1386,59 @@ int arch_add_memory(int nid, u64 start, u64 size, · 3 unmodified lines
1062	1386
1063	1387	memblock_clear_nomap(start, size);
1064	1388
1065		− return __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT,
	1389	+ ret = __add_pages(nid, start >> PAGE_SHIFT, size >> PAGE_SHIFT,
1066	1390	restrictions);
	1391	+ if (ret)
	1392	+ __remove_pgd_mapping(swapper_pg_dir,
	1393	+ __phys_to_virt(start), size);
	1394	+ return ret;
1067	1395	}
	1396	+
1068	1397	void arch_remove_memory(int nid, u64 start, u64 size,
1069	1398	struct vmem_altmap *altmap)
1070	1399	{
1071	1400	unsigned long start_pfn = start >> PAGE_SHIFT;
1072	1401	unsigned long nr_pages = size >> PAGE_SHIFT;
1073	1402
1074		− /*
1075		− * FIXME: Cleanup page tables (also in arch_add_memory() in case
1076		− * adding fails). Until then, this function should only be used
1077		− * during memory hotplug (adding memory), not for memory
1078		− * unplug. ARCH_ENABLE_MEMORY_HOTREMOVE must not be
1079		− * unlocked yet.
1080		− */
1081	1403	__remove_pages(start_pfn, nr_pages, altmap);
	1404	+ __remove_pgd_mapping(swapper_pg_dir, __phys_to_virt(start), size);
	1405	+}
	1406	+
	1407	+/*
	1408	+ * This memory hotplug notifier helps prevent boot memory from being
	1409	+ * inadvertently removed as it blocks pfn range offlining process in
	1410	+ * __offline_pages(). Hence this prevents both offlining as well as
	1411	+ * removal process for boot memory which is initially always online.
	1412	+ * In future if and when boot memory could be removed, this notifier
	1413	+ * should be dropped and free_hotplug_page_range() should handle any
	1414	+ * reserved pages allocated during boot.
	1415	+ */
	1416	+static int prevent_bootmem_remove_notifier(struct notifier_block *nb,
	1417	+ unsigned long action, void *data)
	1418	+{
	1419	+ struct mem_section *ms;
	1420	+ struct memory_notify *arg = data;
	1421	+ unsigned long end_pfn = arg->start_pfn + arg->nr_pages;
	1422	+ unsigned long pfn = arg->start_pfn;
	1423	+
	1424	+ if (action != MEM_GOING_OFFLINE)
	1425	+ return NOTIFY_OK;
	1426	+
	1427	+ for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
	1428	+ ms = __pfn_to_section(pfn);
	1429	+ if (early_section(ms))
	1430	+ return NOTIFY_BAD;
	1431	+ }
	1432	+ return NOTIFY_OK;
	1433	+}
	1434	+
	1435	+static struct notifier_block prevent_bootmem_remove_nb = {
	1436	+ .notifier_call = prevent_bootmem_remove_notifier,
	1437	+};
	1438	+
	1439	+static int __init prevent_bootmem_remove_init(void)
	1440	+{
	1441	+ return register_memory_notifier(&prevent_bootmem_remove_nb);
1082	1442	}
	1443	+device_initcall(prevent_bootmem_remove_init);
1083	1444	#endif

arch/arm64/mm/proc.S+64−40

@@ -11,11 +11,13 @@ · 10 unmodified lines
11	11	#include <linux/linkage.h>
12	12	#include <asm/assembler.h>
13	13	#include <asm/asm-offsets.h>
	14	+#include <asm/asm_pointer_auth.h>
14	15	#include <asm/hwcap.h>
15	16	#include <asm/pgtable.h>
16	17	#include <asm/pgtable-hwdef.h>
17	18	#include <asm/cpufeature.h>
18	19	#include <asm/alternative.h>
	20	+#include <asm/smp.h>
19	21
20	22	#ifdef CONFIG_ARM64_64K_PAGES
21	23	#define TCR_TG_FLAGS TCR_TG0_64K \| TCR_TG1_64K
@@ -131,45 +133,19 @@ alternative_endif · 109 unmodified lines
131	133	ubfx x11, x11, #1, #1
132	134	msr oslar_el1, x11
133	135	reset_pmuserenr_el0 x0 // Disable PMU access from EL0
	136	+ reset_amuserenr_el0 x0 // Disable AMU access from EL0
134	137
135	138	alternative_if ARM64_HAS_RAS_EXTN
136	139	msr_s SYS_DISR_EL1, xzr
137	140	alternative_else_nop_endif
138	141
	142	+ ptrauth_keys_install_kernel x14, 0, x1, x2, x3
139	143	isb
140	144	ret
141	145	SYM_FUNC_END(cpu_do_resume)
142	146	.popsection
143	147	#endif
144	148
145		−/*
146		− * cpu_do_switch_mm(pgd_phys, tsk)
147		− *
148		− * Set the translation table base pointer to be pgd_phys.
149		− *
150		− * - pgd_phys - physical address of new TTB
151		− */
152		−SYM_FUNC_START(cpu_do_switch_mm)
153		− mrs x2, ttbr1_el1
154		− mmid x1, x1 // get mm->context.id
155		− phys_to_ttbr x3, x0
156		−
157		−alternative_if ARM64_HAS_CNP
158		− cbz x1, 1f // skip CNP for reserved ASID
159		− orr x3, x3, #TTBR_CNP_BIT
160		−1:
161		−alternative_else_nop_endif
162		−#ifdef CONFIG_ARM64_SW_TTBR0_PAN
163		− bfi x3, x1, #48, #16 // set the ASID field in TTBR0
164		−#endif
165		− bfi x2, x1, #48, #16 // set the ASID
166		− msr ttbr1_el1, x2 // in TTBR1 (since TCR.A1 is set)
167		− isb
168		− msr ttbr0_el1, x3 // now update TTBR0
169		− isb
170		− b post_ttbr_update_workaround // Back to C code...
171		−SYM_FUNC_END(cpu_do_switch_mm)
172		−
173	149	.pushsection ".idmap.text", "awx"
174	150
175	151	.macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2
@@ -408,35 +384,37 @@ SYM_FUNC_END(idmap_kpti_install_ng_mappings) · 232 unmodified lines
408	384	/*
409	385	* __cpu_setup
410	386	*
411		− * Initialise the processor for turning the MMU on. Return in x0 the
412		− * value of the SCTLR_EL1 register.
	387	+ * Initialise the processor for turning the MMU on.
	388	+ *
	389	+ * Input:
	390	+ * x0 with a flag ARM64_CPU_BOOT_PRIMARY/ARM64_CPU_BOOT_SECONDARY/ARM64_CPU_RUNTIME.
	391	+ * Output:
	392	+ * Return in x0 the value of the SCTLR_EL1 register.
413	393	*/
414	394	.pushsection ".idmap.text", "awx"
415	395	SYM_FUNC_START(__cpu_setup)
416	396	tlbi vmalle1 // Invalidate local TLB
417	397	dsb nsh
418	398
419		− mov x0, #3 << 20
420		− msr cpacr_el1, x0 // Enable FP/ASIMD
421		− mov x0, #1 << 12 // Reset mdscr_el1 and disable
422		− msr mdscr_el1, x0 // access to the DCC from EL0
	399	+ mov x1, #3 << 20
	400	+ msr cpacr_el1, x1 // Enable FP/ASIMD
	401	+ mov x1, #1 << 12 // Reset mdscr_el1 and disable
	402	+ msr mdscr_el1, x1 // access to the DCC from EL0
423	403	isb // Unmask debug exceptions now,
424	404	enable_dbg // since this is per-cpu
425		− reset_pmuserenr_el0 x0 // Disable PMU access from EL0
	405	+ reset_pmuserenr_el0 x1 // Disable PMU access from EL0
	406	+ reset_amuserenr_el0 x1 // Disable AMU access from EL0
	407	+
426	408	/*
427	409	* Memory region attributes
428	410	*/
429	411	mov_q x5, MAIR_EL1_SET
430	412	msr mair_el1, x5
431		− /*
432		− * Prepare SCTLR
433		− */
434		− mov_q x0, SCTLR_EL1_SET
435	413	/*
436	414	* Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for
437	415	* both user and kernel.
438	416	*/
439		− ldr x10, =TCR_TxSZ(VA_BITS) \| TCR_CACHE_FLAGS \| TCR_SMP_FLAGS \| \
	417	+ mov_q x10, TCR_TxSZ(VA_BITS) \| TCR_CACHE_FLAGS \| TCR_SMP_FLAGS \| \
440	418	TCR_TG_FLAGS \| TCR_KASLR_FLAGS \| TCR_ASID16 \| \
441	419	TCR_TBI0 \| TCR_A1 \| TCR_KASAN_FLAGS
442	420	tcr_clear_errata_bits x10, x9, x5
@@ -468,5 +446,51 @@ SYM_FUNC_START(__cpu_setup) · 25 unmodified lines
468	446	1:
469	447	#endif /* CONFIG_ARM64_HW_AFDBM */
470	448	msr tcr_el1, x10
	449	+ mov x1, x0
	450	+ /*
	451	+ * Prepare SCTLR
	452	+ */
	453	+ mov_q x0, SCTLR_EL1_SET
	454	+
	455	+#ifdef CONFIG_ARM64_PTR_AUTH
	456	+ /* No ptrauth setup for run time cpus */
	457	+ cmp x1, #ARM64_CPU_RUNTIME
	458	+ b.eq 3f
	459	+
	460	+ /* Check if the CPU supports ptrauth */
	461	+ mrs x2, id_aa64isar1_el1
	462	+ ubfx x2, x2, #ID_AA64ISAR1_APA_SHIFT, #8
	463	+ cbz x2, 3f
	464	+
	465	+ /*
	466	+ * The primary cpu keys are reset here and can be
	467	+ * re-initialised with some proper values later.
	468	+ */
	469	+ msr_s SYS_APIAKEYLO_EL1, xzr
	470	+ msr_s SYS_APIAKEYHI_EL1, xzr
	471	+
	472	+ /* Just enable ptrauth for primary cpu */
	473	+ cmp x1, #ARM64_CPU_BOOT_PRIMARY
	474	+ b.eq 2f
	475	+
	476	+ /* if !system_supports_address_auth() then skip enable */
	477	+alternative_if_not ARM64_HAS_ADDRESS_AUTH
	478	+ b 3f
	479	+alternative_else_nop_endif
	480	+
	481	+ /* Install ptrauth key for secondary cpus */
	482	+ adr_l x2, secondary_data
	483	+ ldr x3, [x2, #CPU_BOOT_TASK] // get secondary_data.task
	484	+ cbz x3, 2f // check for slow booting cpus
	485	+ ldp x3, x4, [x2, #CPU_BOOT_PTRAUTH_KEY]
	486	+ msr_s SYS_APIAKEYLO_EL1, x3
	487	+ msr_s SYS_APIAKEYHI_EL1, x4
	488	+
	489	+2: /* Enable ptrauth instructions */
	490	+ ldr x2, =SCTLR_ELx_ENIA \| SCTLR_ELx_ENIB \| \
	491	+ SCTLR_ELx_ENDA \| SCTLR_ELx_ENDB
	492	+ orr x0, x0, x2
	493	+3:
	494	+#endif
471	495	ret // return to head.S
472	496	SYM_FUNC_END(__cpu_setup)

arch/arm64/mm/ptdump_debugfs.c+4−0

@@ -1,5 +1,6 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2	2	#include <linux/debugfs.h>
	3	+#include <linux/memory_hotplug.h>
3	4	#include <linux/seq_file.h>
4	5
5	6	#include <asm/ptdump.h>
@@ -7,7 +8,10 @@ · 1 unmodified line
7	8	static int ptdump_show(struct seq_file m, void v)
8	9	{
9	10	struct ptdump_info *info = m->private;
	11	+
	12	+ get_online_mems();
10	13	ptdump_walk(m, info);
	14	+ put_online_mems();
11	15	return 0;
12	16	}
13	17	DEFINE_SHOW_ATTRIBUTE(ptdump);

drivers/base/arch_topology.c+12−0

@@ -21,6 +21,10 @@ · 20 unmodified lines
21	21	#include <linux/sched.h>
22	22	#include <linux/smp.h>
23	23
	24	+__weak bool arch_freq_counters_available(struct cpumask *cpus)
	25	+{
	26	+ return false;
	27	+}
24	28	DEFINE_PER_CPU(unsigned long, freq_scale) = SCHED_CAPACITY_SCALE;
25	29
26	30	void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq,
@@ -29,6 +33,14 @@ void arch_set_freq_scale(struct cpumask *cpus, unsigned long cur_freq, · 2 unmodified lines
29	33	unsigned long scale;
30	34	int i;
31	35
	36	+ /*
	37	+ * If the use of counters for FIE is enabled, just return as we don't
	38	+ * want to update the scale factor with information from CPUFREQ.
	39	+ * Instead the scale factor will be updated from arch_scale_freq_tick.
	40	+ */
	41	+ if (arch_freq_counters_available(cpus))
	42	+ return;
	43	+
32	44	scale = (cur_freq << SCHED_CAPACITY_SHIFT) / max_freq;
33	45
34	46	for_each_cpu(i, cpus)

drivers/clocksource/arm_arch_timer.c+15−3

@@ -889,6 +889,17 @@ static int arch_timer_starting_cpu(unsigned int cpu) · 888 unmodified lines
889	889	return 0;
890	890	}
891	891
	892	+static int validate_timer_rate(void)
	893	+{
	894	+ if (!arch_timer_rate)
	895	+ return -EINVAL;
	896	+
	897	+ /* Arch timer frequency < 1MHz can cause trouble */
	898	+ WARN_ON(arch_timer_rate < 1000000);
	899	+
	900	+ return 0;
	901	+}
	902	+
892	903	/*
893	904	* For historical reasons, when probing with DT we use whichever (non-zero)
894	905	* rate was probed first, and don't verify that others match. If the first node
@@ -904,7 +915,7 @@ static void arch_timer_of_configure_rate(u32 rate, struct device_node *np) · 9 unmodified lines
904	915	arch_timer_rate = rate;
905	916
906	917	/* Check the timer frequency. */
907		− if (arch_timer_rate == 0)
	918	+ if (validate_timer_rate())
908	919	pr_warn("frequency not available\n");
909	920	}
910	921
@@ -1598,9 +1609,10 @@ static int __init arch_timer_acpi_init(struct acpi_table_header *table) · 687 unmodified lines
1598	1609	* CNTFRQ value. This must be correct.
1599	1610	*/
1600	1611	arch_timer_rate = arch_timer_get_cntfrq();
1601		− if (!arch_timer_rate) {
	1612	+ ret = validate_timer_rate();
	1613	+ if (ret) {
1602	1614	pr_err(FW_BUG "frequency not available.\n");
1603		− return -EINVAL;
	1615	+ return ret;
1604	1616	}
1605	1617
1606	1618	arch_timer_uses_ppi = arch_timer_select_ppi();

drivers/cpufreq/cpufreq.c+20−0

@@ -1733,6 +1733,26 @@ unsigned int cpufreq_quick_get_max(unsigned int cpu) · 1732 unmodified lines
1733	1733	}
1734	1734	EXPORT_SYMBOL(cpufreq_quick_get_max);
1735	1735
	1736	+/**
	1737	+ * cpufreq_get_hw_max_freq - get the max hardware frequency of the CPU
	1738	+ * @cpu: CPU number
	1739	+ *
	1740	+ * The default return value is the max_freq field of cpuinfo.
	1741	+ */
	1742	+__weak unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
	1743	+{
	1744	+ struct cpufreq_policy *policy = cpufreq_cpu_get(cpu);
	1745	+ unsigned int ret_freq = 0;
	1746	+
	1747	+ if (policy) {
	1748	+ ret_freq = policy->cpuinfo.max_freq;
	1749	+ cpufreq_cpu_put(policy);
	1750	+ }
	1751	+
	1752	+ return ret_freq;
	1753	+}
	1754	+EXPORT_SYMBOL(cpufreq_get_hw_max_freq);
	1755	+
1736	1756	static unsigned int __cpufreq_get(struct cpufreq_policy *policy)
1737	1757	{
1738	1758	if (unlikely(policy_is_inactive(policy)))

drivers/firmware/arm_sdei.c+36−35

@@ -267,26 +267,19 @@ static struct sdei_event *sdei_event_create(u32 event_num, · 266 unmodified lines
267	267	event->private_registered = regs;
268	268	}
269	269
270		− if (sdei_event_find(event_num)) {
271		− kfree(event->registered);
272		− kfree(event);
273		− event = ERR_PTR(-EBUSY);
274		− } else {
275		− spin_lock(&sdei_list_lock);
276		− list_add(&event->list, &sdei_list);
277		− spin_unlock(&sdei_list_lock);
278		− }
	270	+ spin_lock(&sdei_list_lock);
	271	+ list_add(&event->list, &sdei_list);
	272	+ spin_unlock(&sdei_list_lock);
279	273
280	274	return event;
281	275	}
282	276
283		−static void sdei_event_destroy(struct sdei_event *event)
	277	+static void sdei_event_destroy_llocked(struct sdei_event *event)
284	278	{
285	279	lockdep_assert_held(&sdei_events_lock);
	280	+ lockdep_assert_held(&sdei_list_lock);
286	281
287		− spin_lock(&sdei_list_lock);
288	282	list_del(&event->list);
289		− spin_unlock(&sdei_list_lock);
290	283
291	284	if (event->type == SDEI_EVENT_TYPE_SHARED)
292	285	kfree(event->registered);
@@ -296,6 +289,13 @@ static void sdei_event_destroy(struct sdei_event *event) · 3 unmodified lines
296	289	kfree(event);
297	290	}
298	291
	292	+static void sdei_event_destroy(struct sdei_event *event)
	293	+{
	294	+ spin_lock(&sdei_list_lock);
	295	+ sdei_event_destroy_llocked(event);
	296	+ spin_unlock(&sdei_list_lock);
	297	+}
	298	+
299	299	static int sdei_api_get_version(u64 *version)
300	300	{
301	301	return invoke_sdei_fn(SDEI_1_0_FN_SDEI_VERSION, 0, 0, 0, 0, 0, version);
@@ -412,14 +412,19 @@ int sdei_event_enable(u32 event_num) · 110 unmodified lines
412	412	return -ENOENT;
413	413	}
414	414
415		− spin_lock(&sdei_list_lock);
416		− event->reenable = true;
417		− spin_unlock(&sdei_list_lock);
418	415
	416	+ cpus_read_lock();
419	417	if (event->type == SDEI_EVENT_TYPE_SHARED)
420	418	err = sdei_api_event_enable(event->event_num);
421	419	else
422	420	err = sdei_do_cross_call(_local_event_enable, event);
	421	+
	422	+ if (!err) {
	423	+ spin_lock(&sdei_list_lock);
	424	+ event->reenable = true;
	425	+ spin_unlock(&sdei_list_lock);
	426	+ }
	427	+ cpus_read_unlock();
423	428	mutex_unlock(&sdei_events_lock);
424	429
425	430	return err;
@@ -491,11 +496,6 @@ static int _sdei_event_unregister(struct sdei_event *event) · 65 unmodified lines
491	496	{
492	497	lockdep_assert_held(&sdei_events_lock);
493	498
494		− spin_lock(&sdei_list_lock);
495		− event->reregister = false;
496		− event->reenable = false;
497		− spin_unlock(&sdei_list_lock);
498		−
499	499	if (event->type == SDEI_EVENT_TYPE_SHARED)
500	500	return sdei_api_event_unregister(event->event_num);
501	501
@@ -518,6 +518,11 @@ int sdei_event_unregister(u32 event_num) · 16 unmodified lines
518	518	break;
519	519	}
520	520
	521	+ spin_lock(&sdei_list_lock);
	522	+ event->reregister = false;
	523	+ event->reenable = false;
	524	+ spin_unlock(&sdei_list_lock);
	525	+
521	526	err = _sdei_event_unregister(event);
522	527	if (err)
523	528	break;
@@ -585,26 +590,15 @@ static int _sdei_event_register(struct sdei_event *event) · 61 unmodified lines
585	590
586	591	lockdep_assert_held(&sdei_events_lock);
587	592
588		− spin_lock(&sdei_list_lock);
589		− event->reregister = true;
590		− spin_unlock(&sdei_list_lock);
591		−
592	593	if (event->type == SDEI_EVENT_TYPE_SHARED)
593	594	return sdei_api_event_register(event->event_num,
594	595	sdei_entry_point,
595	596	event->registered,
596	597	SDEI_EVENT_REGISTER_RM_ANY, 0);
597	598
598		−
599	599	err = sdei_do_cross_call(_local_event_register, event);
600		− if (err) {
601		− spin_lock(&sdei_list_lock);
602		− event->reregister = false;
603		− event->reenable = false;
604		− spin_unlock(&sdei_list_lock);
605		−
	600	+ if (err)
606	601	sdei_do_cross_call(_local_event_unregister, event);
607		− }
608	602
609	603	return err;
610	604	}
@@ -632,12 +626,18 @@ int sdei_event_register(u32 event_num, sdei_event_callback cb, void arg) · 21 unmodified lines
632	626	break;
633	627	}
634	628
	629	+ cpus_read_lock();
635	630	err = _sdei_event_register(event);
636	631	if (err) {
637	632	sdei_event_destroy(event);
638	633	pr_warn("Failed to register event %u: %d\n", event_num,
639	634	err);
	635	+ } else {
	636	+ spin_lock(&sdei_list_lock);
	637	+ event->reregister = true;
	638	+ spin_unlock(&sdei_list_lock);
640	639	}
	640	+ cpus_read_unlock();
641	641	} while (0);
642	642	mutex_unlock(&sdei_events_lock);
643	643
@@ -645,16 +645,17 @@ int sdei_event_register(u32 event_num, sdei_event_callback cb, void arg) · 1 unmodified line
645	645	}
646	646	EXPORT_SYMBOL(sdei_event_register);
647	647
648		−static int sdei_reregister_event(struct sdei_event *event)
	648	+static int sdei_reregister_event_llocked(struct sdei_event *event)
649	649	{
650	650	int err;
651	651
652	652	lockdep_assert_held(&sdei_events_lock);
	653	+ lockdep_assert_held(&sdei_list_lock);
653	654
654	655	err = _sdei_event_register(event);
655	656	if (err) {
656	657	pr_err("Failed to re-register event %u\n", event->event_num);
657		− sdei_event_destroy(event);
	658	+ sdei_event_destroy_llocked(event);
658	659	return err;
659	660	}
660	661
@@ -683,7 +684,7 @@ static int sdei_reregister_shared(void) · 22 unmodified lines
683	684	continue;
684	685
685	686	if (event->reregister) {
686		− err = sdei_reregister_event(event);
	687	+ err = sdei_reregister_event_llocked(event);
687	688	if (err)
688	689	break;
689	690	}

drivers/misc/lkdtm/bugs.c+36−0

@@ -378,3 +378,39 @@ void lkdtm_DOUBLE_FAULT(void) · 377 unmodified lines
378	378	pr_err("XFAIL: this test is ia32-only\n");
379	379	#endif
380	380	}
	381	+
	382	+#ifdef CONFIG_ARM64_PTR_AUTH
	383	+static noinline void change_pac_parameters(void)
	384	+{
	385	+ /* Reset the keys of current task */
	386	+ ptrauth_thread_init_kernel(current);
	387	+ ptrauth_thread_switch_kernel(current);
	388	+}
	389	+
	390	+#define CORRUPT_PAC_ITERATE 10
	391	+noinline void lkdtm_CORRUPT_PAC(void)
	392	+{
	393	+ int i;
	394	+
	395	+ if (!system_supports_address_auth()) {
	396	+ pr_err("FAIL: arm64 pointer authentication feature not present\n");
	397	+ return;
	398	+ }
	399	+
	400	+ pr_info("Change the PAC parameters to force function return failure\n");
	401	+ /*
	402	+ * Pac is a hash value computed from input keys, return address and
	403	+ * stack pointer. As pac has fewer bits so there is a chance of
	404	+ * collision, so iterate few times to reduce the collision probability.
	405	+ */
	406	+ for (i = 0; i < CORRUPT_PAC_ITERATE; i++)
	407	+ change_pac_parameters();
	408	+
	409	+ pr_err("FAIL: %s test failed. Kernel may be unstable from here\n", __func__);
	410	+}
	411	+#else /* !CONFIG_ARM64_PTR_AUTH */
	412	+noinline void lkdtm_CORRUPT_PAC(void)
	413	+{
	414	+ pr_err("FAIL: arm64 pointer authentication config disabled\n");
	415	+}
	416	+#endif

drivers/misc/lkdtm/core.c+1−0

@@ -116,6 +116,7 @@ static const struct crashtype crashtypes[] = { · 115 unmodified lines
116	116	CRASHTYPE(STACK_GUARD_PAGE_LEADING),
117	117	CRASHTYPE(STACK_GUARD_PAGE_TRAILING),
118	118	CRASHTYPE(UNSET_SMEP),
	119	+ CRASHTYPE(CORRUPT_PAC),
119	120	CRASHTYPE(UNALIGNED_LOAD_STORE_WRITE),
120	121	CRASHTYPE(OVERWRITE_ALLOCATION),
121	122	CRASHTYPE(WRITE_AFTER_FREE),

drivers/misc/lkdtm/lkdtm.h+1−0

@@ -31,6 +31,7 @@ void lkdtm_UNSET_SMEP(void); · 30 unmodified lines
31	31	#ifdef CONFIG_X86_32
32	32	void lkdtm_DOUBLE_FAULT(void);
33	33	#endif
	34	+void lkdtm_CORRUPT_PAC(void);
34	35
35	36	/* lkdtm_heap.c */
36	37	void __init lkdtm_heap_init(void);

drivers/perf/arm-ccn.c+10−10

@@ -328,15 +328,15 @@ static ssize_t arm_ccn_pmu_event_show(struct device *dev, · 327 unmodified lines
328	328	struct arm_ccn_pmu_event, attr);
329	329	ssize_t res;
330	330
331		− res = snprintf(buf, PAGE_SIZE, "type=0x%x", event->type);
	331	+ res = scnprintf(buf, PAGE_SIZE, "type=0x%x", event->type);
332	332	if (event->event)
333		− res += snprintf(buf + res, PAGE_SIZE - res, ",event=0x%x",
	333	+ res += scnprintf(buf + res, PAGE_SIZE - res, ",event=0x%x",
334	334	event->event);
335	335	if (event->def)
336		− res += snprintf(buf + res, PAGE_SIZE - res, ",%s",
	336	+ res += scnprintf(buf + res, PAGE_SIZE - res, ",%s",
337	337	event->def);
338	338	if (event->mask)
339		− res += snprintf(buf + res, PAGE_SIZE - res, ",mask=0x%x",
	339	+ res += scnprintf(buf + res, PAGE_SIZE - res, ",mask=0x%x",
340	340	event->mask);
341	341
342	342	/* Arguments required by an event */
@@ -344,25 +344,25 @@ static ssize_t arm_ccn_pmu_event_show(struct device *dev, · 1 unmodified line
344	344	case CCN_TYPE_CYCLES:
345	345	break;
346	346	case CCN_TYPE_XP:
347		− res += snprintf(buf + res, PAGE_SIZE - res,
	347	+ res += scnprintf(buf + res, PAGE_SIZE - res,
348	348	",xp=?,vc=?");
349	349	if (event->event == CCN_EVENT_WATCHPOINT)
350		− res += snprintf(buf + res, PAGE_SIZE - res,
	350	+ res += scnprintf(buf + res, PAGE_SIZE - res,
351	351	",port=?,dir=?,cmp_l=?,cmp_h=?,mask=?");
352	352	else
353		− res += snprintf(buf + res, PAGE_SIZE - res,
	353	+ res += scnprintf(buf + res, PAGE_SIZE - res,
354	354	",bus=?");
355	355
356	356	break;
357	357	case CCN_TYPE_MN:
358		− res += snprintf(buf + res, PAGE_SIZE - res, ",node=%d", ccn->mn_id);
	358	+ res += scnprintf(buf + res, PAGE_SIZE - res, ",node=%d", ccn->mn_id);
359	359	break;
360	360	default:
361		− res += snprintf(buf + res, PAGE_SIZE - res, ",node=?");
	361	+ res += scnprintf(buf + res, PAGE_SIZE - res, ",node=?");
362	362	break;
363	363	}
364	364
365		− res += snprintf(buf + res, PAGE_SIZE - res, "\n");
	365	+ res += scnprintf(buf + res, PAGE_SIZE - res, "\n");
366	366
367	367	return res;
368	368	}

drivers/perf/arm_spe_pmu.c+1−1

@@ -831,7 +831,7 @@ static void arm_spe_pmu_setup_aux(struct perf_event event, void **pages, · 830 unmodified lines
831	831	* parts and give userspace a fighting chance of getting some
832	832	* useful data out of it.
833	833	*/
834		− if (!nr_pages \|\| (snapshot && (nr_pages & 1)))
	834	+ if (snapshot && (nr_pages & 1))
835	835	return NULL;
836	836
837	837	if (cpu == -1)

include/linux/arch_topology.h+2−0

@@ -30,6 +30,8 @@ static inline unsigned long topology_get_freq_scale(int cpu) · 29 unmodified lines
30	30	return per_cpu(freq_scale, cpu);
31	31	}
32	32
	33	+bool arch_freq_counters_available(struct cpumask *cpus);
	34	+
33	35	DECLARE_PER_CPU(unsigned long, thermal_pressure);
34	36
35	37	static inline unsigned long topology_get_thermal_pressure(int cpu)

include/linux/cpufreq.h+5−0

@@ -205,6 +205,7 @@ static inline bool policy_is_shared(struct cpufreq_policy *policy) · 204 unmodified lines
205	205	unsigned int cpufreq_get(unsigned int cpu);
206	206	unsigned int cpufreq_quick_get(unsigned int cpu);
207	207	unsigned int cpufreq_quick_get_max(unsigned int cpu);
	208	+unsigned int cpufreq_get_hw_max_freq(unsigned int cpu);
208	209	void disable_cpufreq(void);
209	210
210	211	u64 get_cpu_idle_time(unsigned int cpu, u64 *wall, int io_busy);
@@ -232,6 +233,10 @@ static inline unsigned int cpufreq_quick_get_max(unsigned int cpu) · 21 unmodified lines
232	233	{
233	234	return 0;
234	235	}
	236	+static inline unsigned int cpufreq_get_hw_max_freq(unsigned int cpu)
	237	+{
	238	+ return 0;
	239	+}
235	240	static inline void disable_cpufreq(void) { }
236	241	#endif
237	242

include/linux/perf/arm_pmu.h+1−0

@@ -80,6 +80,7 @@ struct arm_pmu { · 79 unmodified lines
80	80	struct pmu pmu;
81	81	cpumask_t supported_cpus;
82	82	char *name;
	83	+ int pmuver;
83	84	irqreturn_t (handle_irq)(struct arm_pmu pmu);
84	85	void (enable)(struct perf_event event);
85	86	void (disable)(struct perf_event event);

include/linux/stackprotector.h+1−1

@@ -6,7 +6,7 @@ · 5 unmodified lines
6	6	#include <linux/sched.h>
7	7	#include <linux/random.h>
8	8
9		−#ifdef CONFIG_STACKPROTECTOR
	9	+#if defined(CONFIG_STACKPROTECTOR) \|\| defined(CONFIG_ARM64_PTR_AUTH)
10	10	# include <asm/stackprotector.h>
11	11	#else
12	12	static inline void boot_init_stack_canary(void)

mm/mremap.c+10−0

@@ -606,6 +606,16 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, · 605 unmodified lines
606	606	LIST_HEAD(uf_unmap_early);
607	607	LIST_HEAD(uf_unmap);
608	608
	609	+ /*
	610	+ * There is a deliberate asymmetry here: we strip the pointer tag
	611	+ * from the old address but leave the new address alone. This is
	612	+ * for consistency with mmap(), where we prevent the creation of
	613	+ * aliasing mappings in userspace by leaving the tag bits of the
	614	+ * mapping address intact. A non-zero tag will cause the subsequent
	615	+ * range checks to reject the address as invalid.
	616	+ *
	617	+ * See Documentation/arm64/tagged-address-abi.rst for more information.
	618	+ */
609	619	addr = untagged_addr(addr);
610	620
611	621	if (flags & ~(MREMAP_FIXED \| MREMAP_MAYMOVE))

scripts/Kconfig.include+6−0

@@ -31,6 +31,12 @@ cc-option = $(success,$(CC) -Werror $(CLANG_FLAGS) $(1) -S -x c /dev/null -o /de · 30 unmodified lines
31	31	# Return y if the linker supports <flag>, n otherwise
32	32	ld-option = $(success,$(LD) -v $(1))
33	33
	34	+# $(as-option,<flag>)
	35	+# /dev/zero is used as output instead of /dev/null as some assembler cribs when
	36	+# both input and output are same. Also both of them have same write behaviour so
	37	+# can be easily substituted.
	38	+as-option = $(success, $(CC) $(CLANG_FLAGS) $(1) -c -x assembler /dev/null -o /dev/zero)
	39	+
34	40	# $(as-instr,<instr>)
35	41	# Return y if the assembler supports <instr>, n otherwise
36	42	as-instr = $(success,printf "%b\n" "$(1)" \| $(CC) $(CLANG_FLAGS) -c -x assembler -o /dev/null -)