Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Will Deacon: "A bunch of good stuff in here. Worth noting is that we've pulled in the x86/mm branch from -tip so that we can make use of the core ioremap changes which allow us to put down huge mappings in the vmalloc area without screwing up the TLB. Much of the positive diffstat is because of the rseq selftest for arm64. Summary: - Wire up support for qspinlock, replacing our trusty ticket lock code - Add an IPI to flush_icache_range() to ensure that stale instructions fetched into the pipeline are discarded along with the I-cache lines - Support for the GCC "stackleak" plugin - Support for restartable sequences, plus an arm64 port for the selftest - Kexec/kdump support on systems booting with ACPI - Rewrite of our syscall entry code in C, which allows us to zero the GPRs on entry from userspace - Support for chained PMU counters, allowing 64-bit event counters to be constructed on current CPUs - Ensure scheduler topology information is kept up-to-date with CPU hotplug events - Re-enable support for huge vmalloc/IO mappings now that the core code has the correct hooks to use break-before-make sequences - Miscellaneous, non-critical fixes and cleanups" * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (90 commits) arm64: alternative: Use true and false for boolean values arm64: kexec: Add comment to explain use of __flush_icache_range() arm64: sdei: Mark sdei stack helper functions as static arm64, kaslr: export offset in VMCOREINFO ELF notes arm64: perf: Add cap_user_time aarch64 efi/libstub: Only disable stackleak plugin for arm64 arm64: drop unused kernel_neon_begin_partial() macro arm64: kexec: machine_kexec should call __flush_icache_range arm64: svc: Ensure hardirq tracing is updated before return arm64: mm: Export __sync_icache_dcache() for xen-privcmd drivers/perf: arm-ccn: Use devm_ioremap_resource() to map memory arm64: Add support for STACKLEAK gcc plugin arm64: Add stack information to on_accessible_stack drivers/perf: hisi: update the sccl_id/ccl_id when MT is supported arm64: fix ACPI dependencies rseq/selftests: Add support for arm64 arm64: acpi: fix alignment fault in accessing ACPI efi/arm: map UEFI memory map even w/o runtime services enabled efi/arm: preserve early mapping of UEFI memory map longer for BGRT drivers: acpi: add dependency of EFI for arm64 ...

Linus Torvalds committed 8 years ago

Commit 1202f4f2 parents d0055f3 3c4d91391 files · +2154 −1001Browse files

Unified · Split

91 files

MAINTAINERS +1 −0
arch/arm/include/asm/kvm_emulate.h +7 −7
arch/arm/kernel/perf_event_v6.c +10 −4
arch/arm/kernel/perf_event_v7.c +12 −3
arch/arm/kernel/perf_event_xscale.c +12 −6
arch/arm64/Kconfig +18 −0
arch/arm64/Makefile +5 −4
arch/arm64/include/asm/Kbuild +1 −0
arch/arm64/include/asm/acpi.h +20 −9
arch/arm64/include/asm/barrier.h +13 −0
arch/arm64/include/asm/cache.h +4 −0
arch/arm64/include/asm/cacheflush.h +26 −1
arch/arm64/include/asm/cpucaps.h +2 −1
arch/arm64/include/asm/fpsimd.h +16 −1
arch/arm64/include/asm/insn.h +0 −2
arch/arm64/include/asm/kvm_emulate.h +5 −5
arch/arm64/include/asm/neon.h +0 −7
arch/arm64/include/asm/numa.h +4 −0
arch/arm64/include/asm/processor.h +18 −3
arch/arm64/include/asm/ptrace.h +53 −26
arch/arm64/include/asm/sdei.h +6 −3
arch/arm64/include/asm/spinlock.h +1 −116
arch/arm64/include/asm/spinlock_types.h +1 −16
arch/arm64/include/asm/stacktrace.h +61 −12
arch/arm64/include/asm/syscall.h +7 −1
arch/arm64/include/asm/syscall_wrapper.h +80 −0
arch/arm64/include/asm/sysreg.h +10 −20
arch/arm64/include/asm/tlbflush.h +7 −0
arch/arm64/include/asm/topology.h +3 −1
arch/arm64/include/asm/unistd.h +1 −1
arch/arm64/include/asm/unistd32.h +15 −13
arch/arm64/kernel/Makefile +3 −2
arch/arm64/kernel/acpi.c +3 −8
arch/arm64/kernel/acpi_numa.c +53 −35
arch/arm64/kernel/alternative.c +2 −2
arch/arm64/kernel/armv8_deprecated.c +16 −16
arch/arm64/kernel/cpu-reset.h +5 −4
arch/arm64/kernel/cpu_errata.c +20 −10
arch/arm64/kernel/cpufeature.c +2 −2
arch/arm64/kernel/entry.S +23 −137
arch/arm64/kernel/entry32.S +0 −121
arch/arm64/kernel/fpsimd.c +0 −19
arch/arm64/kernel/insn.c +3 −67
arch/arm64/kernel/machine_kexec.c +11 −4
arch/arm64/kernel/perf_event.c +231 −50
arch/arm64/kernel/process.c +32 −10
arch/arm64/kernel/ptrace.c +36 −25
arch/arm64/kernel/sdei.c +39 −9
arch/arm64/kernel/setup.c +38 −0
arch/arm64/kernel/signal.c +6 −2
arch/arm64/kernel/signal32.c +15 −9
arch/arm64/kernel/smp.c +33 −11
arch/arm64/kernel/stacktrace.c +1 −1
arch/arm64/kernel/sys.c +16 −13
arch/arm64/kernel/sys32.c +116 −19
arch/arm64/kernel/syscall.c +139 −0
arch/arm64/kernel/topology.c +41 −17
arch/arm64/kernel/traps.c +1 −17
arch/arm64/kvm/guest.c +7 −7
arch/arm64/kvm/hyp/Makefile +2 −1
arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c +1 −1
arch/arm64/kvm/regmap.c +11 −11
arch/arm64/kvm/reset.c +2 −2
arch/arm64/mm/cache.S +2 −2
arch/arm64/mm/fault.c +1 −1
arch/arm64/mm/flush.c +2 −1
arch/arm64/mm/mmu.c +44 −4
arch/arm64/mm/numa.c +21 −8
arch/arm64/mm/ptdump_debugfs.c +1 −12
arch/ia64/Kconfig +1 −0
arch/x86/Kconfig +1 −0
drivers/acpi/Kconfig +5 −3
drivers/firmware/efi/arm-init.c +0 −1
drivers/firmware/efi/arm-runtime.c +10 −8
drivers/firmware/efi/libstub/Makefile +5 −2
drivers/perf/arm-cci.c +26 −12
drivers/perf/arm-ccn.c +3 −11
drivers/perf/arm_pmu.c +22 −16
drivers/perf/arm_pmu_platform.c +1 −1
drivers/perf/hisilicon/hisi_uncore_pmu.c +7 −5
fs/statfs.c +12 −2
include/linux/compat.h +11 −0
include/linux/perf/arm_pmu.h +8 −3
include/linux/syscalls.h +11 −0
include/uapi/asm-generic/unistd.h +3 −1
kernel/events/core.c +2 −2
scripts/Kbuild.include +2 −2
tools/testing/selftests/rseq/param_test.c +20 −0
tools/testing/selftests/rseq/rseq-arm64.h +594 −0
tools/testing/selftests/rseq/rseq.h +2 −0
virt/kvm/arm/aarch32.c +10 −10

MAINTAINERS+1−0

@@ -2270,6 +2270,7 @@ L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers) · 2269 unmodified lines
2270	2270	T: git git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux.git
2271	2271	S: Maintained
2272	2272	F: arch/arm64/
	2273	+X: arch/arm64/boot/dts/
2273	2274	F: Documentation/arm64/
2274	2275
2275	2276	AS3645A LED FLASH CONTROLLER DRIVER

arch/arm/include/asm/kvm_emulate.h+7−7

@@ -26,13 +26,13 @@ · 25 unmodified lines
26	26	#include <asm/cputype.h>
27	27
28	28	/* arm64 compatibility macros */
29		−#define COMPAT_PSR_MODE_ABT ABT_MODE
30		−#define COMPAT_PSR_MODE_UND UND_MODE
31		−#define COMPAT_PSR_T_BIT PSR_T_BIT
32		−#define COMPAT_PSR_I_BIT PSR_I_BIT
33		−#define COMPAT_PSR_A_BIT PSR_A_BIT
34		−#define COMPAT_PSR_E_BIT PSR_E_BIT
35		−#define COMPAT_PSR_IT_MASK PSR_IT_MASK
	29	+#define PSR_AA32_MODE_ABT ABT_MODE
	30	+#define PSR_AA32_MODE_UND UND_MODE
	31	+#define PSR_AA32_T_BIT PSR_T_BIT
	32	+#define PSR_AA32_I_BIT PSR_I_BIT
	33	+#define PSR_AA32_A_BIT PSR_A_BIT
	34	+#define PSR_AA32_E_BIT PSR_E_BIT
	35	+#define PSR_AA32_IT_MASK PSR_IT_MASK
36	36
37	37	unsigned long vcpu_reg(struct kvm_vcpu vcpu, u8 reg_num);
38	38

arch/arm/kernel/perf_event_v6.c+10−4

@@ -233,7 +233,7 @@ armv6_pmcr_counter_has_overflowed(unsigned long pmcr, · 232 unmodified lines
233	233	return ret;
234	234	}
235	235
236		−static inline u32 armv6pmu_read_counter(struct perf_event *event)
	236	+static inline u64 armv6pmu_read_counter(struct perf_event *event)
237	237	{
238	238	struct hw_perf_event *hwc = &event->hw;
239	239	int counter = hwc->idx;
@@ -251,7 +251,7 @@ static inline u32 armv6pmu_read_counter(struct perf_event *event) · 11 unmodified lines
251	251	return value;
252	252	}
253	253
254		−static inline void armv6pmu_write_counter(struct perf_event *event, u32 value)
	254	+static inline void armv6pmu_write_counter(struct perf_event *event, u64 value)
255	255	{
256	256	struct hw_perf_event *hwc = &event->hw;
257	257	int counter = hwc->idx;
@@ -411,6 +411,12 @@ armv6pmu_get_event_idx(struct pmu_hw_events *cpuc, · 153 unmodified lines
411	411	}
412	412	}
413	413
	414	+static void armv6pmu_clear_event_idx(struct pmu_hw_events *cpuc,
	415	+ struct perf_event *event)
	416	+{
	417	+ clear_bit(event->hw.idx, cpuc->used_mask);
	418	+}
	419	+
414	420	static void armv6pmu_disable_event(struct perf_event *event)
415	421	{
416	422	unsigned long val, mask, evt, flags;
@@ -491,11 +497,11 @@ static void armv6pmu_init(struct arm_pmu *cpu_pmu) · 74 unmodified lines
491	497	cpu_pmu->read_counter = armv6pmu_read_counter;
492	498	cpu_pmu->write_counter = armv6pmu_write_counter;
493	499	cpu_pmu->get_event_idx = armv6pmu_get_event_idx;
	500	+ cpu_pmu->clear_event_idx = armv6pmu_clear_event_idx;
494	501	cpu_pmu->start = armv6pmu_start;
495	502	cpu_pmu->stop = armv6pmu_stop;
496	503	cpu_pmu->map_event = armv6_map_event;
497	504	cpu_pmu->num_events = 3;
498		− cpu_pmu->max_period = (1LLU << 32) - 1;
499	505	}
500	506
501	507	static int armv6_1136_pmu_init(struct arm_pmu *cpu_pmu)
@@ -542,11 +548,11 @@ static int armv6mpcore_pmu_init(struct arm_pmu *cpu_pmu) · 40 unmodified lines
542	548	cpu_pmu->read_counter = armv6pmu_read_counter;
543	549	cpu_pmu->write_counter = armv6pmu_write_counter;
544	550	cpu_pmu->get_event_idx = armv6pmu_get_event_idx;
	551	+ cpu_pmu->clear_event_idx = armv6pmu_clear_event_idx;
545	552	cpu_pmu->start = armv6pmu_start;
546	553	cpu_pmu->stop = armv6pmu_stop;
547	554	cpu_pmu->map_event = armv6mpcore_map_event;
548	555	cpu_pmu->num_events = 3;
549		− cpu_pmu->max_period = (1LLU << 32) - 1;
550	556
551	557	return 0;
552	558	}

arch/arm/kernel/perf_event_v7.c+12−3

@@ -743,7 +743,7 @@ static inline void armv7_pmnc_select_counter(int idx) · 742 unmodified lines
743	743	isb();
744	744	}
745	745
746		−static inline u32 armv7pmu_read_counter(struct perf_event *event)
	746	+static inline u64 armv7pmu_read_counter(struct perf_event *event)
747	747	{
748	748	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
749	749	struct hw_perf_event *hwc = &event->hw;
@@ -763,7 +763,7 @@ static inline u32 armv7pmu_read_counter(struct perf_event *event) · 13 unmodified lines
763	763	return value;
764	764	}
765	765
766		−static inline void armv7pmu_write_counter(struct perf_event *event, u32 value)
	766	+static inline void armv7pmu_write_counter(struct perf_event *event, u64 value)
767	767	{
768	768	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
769	769	struct hw_perf_event *hwc = &event->hw;
@@ -1058,6 +1058,12 @@ static int armv7pmu_get_event_idx(struct pmu_hw_events *cpuc, · 288 unmodified lines
1058	1058	return -EAGAIN;
1059	1059	}
1060	1060
	1061	+static void armv7pmu_clear_event_idx(struct pmu_hw_events *cpuc,
	1062	+ struct perf_event *event)
	1063	+{
	1064	+ clear_bit(event->hw.idx, cpuc->used_mask);
	1065	+}
	1066	+
1061	1067	/*
1062	1068	* Add an event filter to a given event. This will only work for PMUv2 PMUs.
1063	1069	*/
@@ -1167,10 +1173,10 @@ static void armv7pmu_init(struct arm_pmu *cpu_pmu) · 103 unmodified lines
1167	1173	cpu_pmu->read_counter = armv7pmu_read_counter;
1168	1174	cpu_pmu->write_counter = armv7pmu_write_counter;
1169	1175	cpu_pmu->get_event_idx = armv7pmu_get_event_idx;
	1176	+ cpu_pmu->clear_event_idx = armv7pmu_clear_event_idx;
1170	1177	cpu_pmu->start = armv7pmu_start;
1171	1178	cpu_pmu->stop = armv7pmu_stop;
1172	1179	cpu_pmu->reset = armv7pmu_reset;
1173		− cpu_pmu->max_period = (1LLU << 32) - 1;
1174	1180	};
1175	1181
1176	1182	static void armv7_read_num_pmnc_events(void *info)
@@ -1638,6 +1644,7 @@ static void krait_pmu_clear_event_idx(struct pmu_hw_events *cpuc, · 461 unmodified lines
1638	1644	bool venum_event = EVENT_VENUM(hwc->config_base);
1639	1645	bool krait_event = EVENT_CPU(hwc->config_base);
1640	1646
	1647	+ armv7pmu_clear_event_idx(cpuc, event);
1641	1648	if (venum_event \|\| krait_event) {
1642	1649	bit = krait_event_to_bit(event, region, group);
1643	1650	clear_bit(bit, cpuc->used_mask);
@@ -1967,6 +1974,7 @@ static void scorpion_pmu_clear_event_idx(struct pmu_hw_events *cpuc, · 323 unmodified lines
1967	1974	bool venum_event = EVENT_VENUM(hwc->config_base);
1968	1975	bool scorpion_event = EVENT_CPU(hwc->config_base);
1969	1976
	1977	+ armv7pmu_clear_event_idx(cpuc, event);
1970	1978	if (venum_event \|\| scorpion_event) {
1971	1979	bit = scorpion_event_to_bit(event, region, group);
1972	1980	clear_bit(bit, cpuc->used_mask);
@@ -2030,6 +2038,7 @@ static struct platform_driver armv7_pmu_driver = { · 57 unmodified lines
2030	2038	.driver = {
2031	2039	.name = "armv7-pmu",
2032	2040	.of_match_table = armv7_pmu_of_device_ids,
	2041	+ .suppress_bind_attrs = true,
2033	2042	},
2034	2043	.probe = armv7_pmu_device_probe,
2035	2044	};

arch/arm/kernel/perf_event_xscale.c+12−6

@@ -292,6 +292,12 @@ xscale1pmu_get_event_idx(struct pmu_hw_events *cpuc, · 291 unmodified lines
292	292	}
293	293	}
294	294
	295	+static void xscalepmu_clear_event_idx(struct pmu_hw_events *cpuc,
	296	+ struct perf_event *event)
	297	+{
	298	+ clear_bit(event->hw.idx, cpuc->used_mask);
	299	+}
	300	+
295	301	static void xscale1pmu_start(struct arm_pmu *cpu_pmu)
296	302	{
297	303	unsigned long flags, val;
@@ -316,7 +322,7 @@ static void xscale1pmu_stop(struct arm_pmu *cpu_pmu) · 18 unmodified lines
316	322	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
317	323	}
318	324
319		−static inline u32 xscale1pmu_read_counter(struct perf_event *event)
	325	+static inline u64 xscale1pmu_read_counter(struct perf_event *event)
320	326	{
321	327	struct hw_perf_event *hwc = &event->hw;
322	328	int counter = hwc->idx;
@@ -337,7 +343,7 @@ static inline u32 xscale1pmu_read_counter(struct perf_event *event) · 14 unmodified lines
337	343	return val;
338	344	}
339	345
340		−static inline void xscale1pmu_write_counter(struct perf_event *event, u32 val)
	346	+static inline void xscale1pmu_write_counter(struct perf_event *event, u64 val)
341	347	{
342	348	struct hw_perf_event *hwc = &event->hw;
343	349	int counter = hwc->idx;
@@ -370,11 +376,11 @@ static int xscale1pmu_init(struct arm_pmu *cpu_pmu) · 26 unmodified lines
370	376	cpu_pmu->read_counter = xscale1pmu_read_counter;
371	377	cpu_pmu->write_counter = xscale1pmu_write_counter;
372	378	cpu_pmu->get_event_idx = xscale1pmu_get_event_idx;
	379	+ cpu_pmu->clear_event_idx = xscalepmu_clear_event_idx;
373	380	cpu_pmu->start = xscale1pmu_start;
374	381	cpu_pmu->stop = xscale1pmu_stop;
375	382	cpu_pmu->map_event = xscale_map_event;
376	383	cpu_pmu->num_events = 3;
377		− cpu_pmu->max_period = (1LLU << 32) - 1;
378	384
379	385	return 0;
380	386	}
@@ -679,7 +685,7 @@ static void xscale2pmu_stop(struct arm_pmu *cpu_pmu) · 298 unmodified lines
679	685	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
680	686	}
681	687
682		−static inline u32 xscale2pmu_read_counter(struct perf_event *event)
	688	+static inline u64 xscale2pmu_read_counter(struct perf_event *event)
683	689	{
684	690	struct hw_perf_event *hwc = &event->hw;
685	691	int counter = hwc->idx;
@@ -706,7 +712,7 @@ static inline u32 xscale2pmu_read_counter(struct perf_event *event) · 20 unmodified lines
706	712	return val;
707	713	}
708	714
709		−static inline void xscale2pmu_write_counter(struct perf_event *event, u32 val)
	715	+static inline void xscale2pmu_write_counter(struct perf_event *event, u64 val)
710	716	{
711	717	struct hw_perf_event *hwc = &event->hw;
712	718	int counter = hwc->idx;
@@ -739,11 +745,11 @@ static int xscale2pmu_init(struct arm_pmu *cpu_pmu) · 26 unmodified lines
739	745	cpu_pmu->read_counter = xscale2pmu_read_counter;
740	746	cpu_pmu->write_counter = xscale2pmu_write_counter;
741	747	cpu_pmu->get_event_idx = xscale2pmu_get_event_idx;
	748	+ cpu_pmu->clear_event_idx = xscalepmu_clear_event_idx;
742	749	cpu_pmu->start = xscale2pmu_start;
743	750	cpu_pmu->stop = xscale2pmu_stop;
744	751	cpu_pmu->map_event = xscale_map_event;
745	752	cpu_pmu->num_events = 5;
746		− cpu_pmu->max_period = (1LLU << 32) - 1;
747	753
748	754	return 0;
749	755	}

arch/arm64/Kconfig+18−0

@@ -24,6 +24,7 @@ config ARM64 · 23 unmodified lines
24	24	select ARCH_HAS_SG_CHAIN
25	25	select ARCH_HAS_STRICT_KERNEL_RWX
26	26	select ARCH_HAS_STRICT_MODULE_RWX
	27	+ select ARCH_HAS_SYSCALL_WRAPPER
27	28	select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
28	29	select ARCH_HAVE_NMI_SAFE_CMPXCHG
29	30	select ARCH_INLINE_READ_LOCK if !PREEMPT
@@ -42,8 +43,19 @@ config ARM64 · 12 unmodified lines
42	43	select ARCH_INLINE_WRITE_UNLOCK_BH if !PREEMPT
43	44	select ARCH_INLINE_WRITE_UNLOCK_IRQ if !PREEMPT
44	45	select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE if !PREEMPT
	46	+ select ARCH_INLINE_SPIN_TRYLOCK if !PREEMPT
	47	+ select ARCH_INLINE_SPIN_TRYLOCK_BH if !PREEMPT
	48	+ select ARCH_INLINE_SPIN_LOCK if !PREEMPT
	49	+ select ARCH_INLINE_SPIN_LOCK_BH if !PREEMPT
	50	+ select ARCH_INLINE_SPIN_LOCK_IRQ if !PREEMPT
	51	+ select ARCH_INLINE_SPIN_LOCK_IRQSAVE if !PREEMPT
	52	+ select ARCH_INLINE_SPIN_UNLOCK if !PREEMPT
	53	+ select ARCH_INLINE_SPIN_UNLOCK_BH if !PREEMPT
	54	+ select ARCH_INLINE_SPIN_UNLOCK_IRQ if !PREEMPT
	55	+ select ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE if !PREEMPT
45	56	select ARCH_USE_CMPXCHG_LOCKREF
46	57	select ARCH_USE_QUEUED_RWLOCKS
	58	+ select ARCH_USE_QUEUED_SPINLOCKS
47	59	select ARCH_SUPPORTS_MEMORY_FAILURE
48	60	select ARCH_SUPPORTS_ATOMIC_RMW
49	61	select ARCH_SUPPORTS_INT128 if GCC_VERSION >= 50000 \|\| CC_IS_CLANG
@@ -97,6 +109,7 @@ config ARM64 · 47 unmodified lines
97	109	select HAVE_ARCH_MMAP_RND_BITS
98	110	select HAVE_ARCH_MMAP_RND_COMPAT_BITS if COMPAT
99	111	select HAVE_ARCH_SECCOMP_FILTER
	112	+ select HAVE_ARCH_STACKLEAK
100	113	select HAVE_ARCH_THREAD_STRUCT_WHITELIST
101	114	select HAVE_ARCH_TRACEHOOK
102	115	select HAVE_ARCH_TRANSPARENT_HUGEPAGE
@@ -128,6 +141,7 @@ config ARM64 · 25 unmodified lines
128	141	select HAVE_PERF_USER_STACK_DUMP
129	142	select HAVE_REGS_AND_STACK_ACCESS_API
130	143	select HAVE_RCU_TABLE_FREE
	144	+ select HAVE_RSEQ
131	145	select HAVE_STACKPROTECTOR
132	146	select HAVE_SYSCALL_TRACEPOINTS
133	147	select HAVE_KPROBES
@@ -773,6 +787,9 @@ config ARCH_SPARSEMEM_DEFAULT · 639 unmodified lines
773	787	config ARCH_SELECT_MEMORY_MODEL
774	788	def_bool ARCH_SPARSEMEM_ENABLE
775	789
	790	+config ARCH_FLATMEM_ENABLE
	791	+ def_bool !NUMA
	792	+
776	793	config HAVE_ARCH_PFN_VALID
777	794	def_bool ARCH_HAS_HOLES_MEMORYMODEL \|\| !SPARSEMEM
778	795
@@ -1244,6 +1261,7 @@ config EFI · 465 unmodified lines
1244	1261	bool "UEFI runtime support"
1245	1262	depends on OF && !CPU_BIG_ENDIAN
1246	1263	depends on KERNEL_MODE_NEON
	1264	+ select ARCH_SUPPORTS_ACPI
1247	1265	select LIBFDT
1248	1266	select UCS2_STRING
1249	1267	select EFI_PARAMS_FROM_FDT

arch/arm64/Makefile+5−4

@@ -60,15 +60,16 @@ ifeq ($(CONFIG_CPU_BIG_ENDIAN), y) · 59 unmodified lines
60	60	KBUILD_CPPFLAGS += -mbig-endian
61	61	CHECKFLAGS += -D__AARCH64EB__
62	62	AS += -EB
63		−# We must use the linux target here, since distributions don't tend to package
64		−# the ELF linker scripts with binutils, and this results in a build failure.
65		−LDFLAGS += -EB -maarch64linuxb
	63	+# Prefer the baremetal ELF build target, but not all toolchains include
	64	+# it so fall back to the standard linux version if needed.
	65	+LDFLAGS += -EB $(call ld-option, -maarch64elfb, -maarch64linuxb)
66	66	UTS_MACHINE := aarch64_be
67	67	else
68	68	KBUILD_CPPFLAGS += -mlittle-endian
69	69	CHECKFLAGS += -D__AARCH64EL__
70	70	AS += -EL
71		−LDFLAGS += -EL -maarch64linux # See comment above
	71	+# Same as above, prefer ELF but fall back to linux target if needed.
	72	+LDFLAGS += -EL $(call ld-option, -maarch64elf, -maarch64linux)
72	73	UTS_MACHINE := aarch64
73	74	endif
74	75

arch/arm64/include/asm/Kbuild+1−0

@@ -16,6 +16,7 @@ generic-y += mm-arch-hooks.h · 15 unmodified lines
16	16	generic-y += msi.h
17	17	generic-y += preempt.h
18	18	generic-y += qrwlock.h
	19	+generic-y += qspinlock.h
19	20	generic-y += rwsem.h
20	21	generic-y += segment.h
21	22	generic-y += serial.h

arch/arm64/include/asm/acpi.h+20−9

@@ -12,10 +12,12 @@ · 11 unmodified lines
12	12	#ifndef _ASM_ACPI_H
13	13	#define _ASM_ACPI_H
14	14
	15	+#include <linux/efi.h>
15	16	#include <linux/memblock.h>
16	17	#include <linux/psci.h>
17	18
18	19	#include <asm/cputype.h>
	20	+#include <asm/io.h>
19	21	#include <asm/smp_plat.h>
20	22	#include <asm/tlbflush.h>
21	23
@@ -29,18 +31,22 @@ · 7 unmodified lines
29	31
30	32	/* Basic configuration for ACPI */
31	33	#ifdef CONFIG_ACPI
	34	+pgprot_t __acpi_get_mem_attribute(phys_addr_t addr);
	35	+
32	36	/* ACPI table mapping after acpi_permanent_mmap is set */
33	37	static inline void __iomem *acpi_os_ioremap(acpi_physical_address phys,
34	38	acpi_size size)
35	39	{
	40	+ /* For normal memory we already have a cacheable mapping. */
	41	+ if (memblock_is_map_memory(phys))
	42	+ return (void __iomem *)__phys_to_virt(phys);
	43	+
36	44	/*
37		− * EFI's reserve_regions() call adds memory with the WB attribute
38		− * to memblock via early_init_dt_add_memory_arch().
	45	+ * We should still honor the memory's attribute here because
	46	+ * crash dump kernel possibly excludes some ACPI (reclaim)
	47	+ * regions from memblock list.
39	48	*/
40		− if (!memblock_is_memory(phys))
41		− return ioremap(phys, size);
42		−
43		− return ioremap_cache(phys, size);
	49	+ return __ioremap(phys, size, __acpi_get_mem_attribute(phys));
44	50	}
45	51	#define acpi_os_ioremap acpi_os_ioremap
46	52
@@ -129,15 +135,20 @@ static inline const char *acpi_get_enable_method(int cpu) · 82 unmodified lines
129	135	* for compatibility.
130	136	*/
131	137	#define acpi_disable_cmcff 1
132		−pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr);
	138	+static inline pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr)
	139	+{
	140	+ return __acpi_get_mem_attribute(addr);
	141	+}
133	142	#endif /* CONFIG_ACPI_APEI */
134	143
135	144	#ifdef CONFIG_ACPI_NUMA
136	145	int arm64_acpi_numa_init(void);
137		−int acpi_numa_get_nid(unsigned int cpu, u64 hwid);
	146	+int acpi_numa_get_nid(unsigned int cpu);
	147	+void acpi_map_cpus_to_nodes(void);
138	148	#else
139	149	static inline int arm64_acpi_numa_init(void) { return -ENOSYS; }
140		−static inline int acpi_numa_get_nid(unsigned int cpu, u64 hwid) { return NUMA_NO_NODE; }
	150	+static inline int acpi_numa_get_nid(unsigned int cpu) { return NUMA_NO_NODE; }
	151	+static inline void acpi_map_cpus_to_nodes(void) { }
141	152	#endif /* CONFIG_ACPI_NUMA */
142	153
143	154	#define ACPI_TABLE_UPGRADE_MAX_PHYS MEMBLOCK_ALLOC_ACCESSIBLE

arch/arm64/include/asm/barrier.h+13−0

@@ -128,6 +128,19 @@ do { \ · 127 unmodified lines
128	128	__u.__val; \
129	129	})
130	130
	131	+#define smp_cond_load_relaxed(ptr, cond_expr) \
	132	+({ \
	133	+ typeof(ptr) __PTR = (ptr); \
	134	+ typeof(*ptr) VAL; \
	135	+ for (;;) { \
	136	+ VAL = READ_ONCE(*__PTR); \
	137	+ if (cond_expr) \
	138	+ break; \
	139	+ __cmpwait_relaxed(__PTR, VAL); \
	140	+ } \
	141	+ VAL; \
	142	+})
	143	+
131	144	#define smp_cond_load_acquire(ptr, cond_expr) \
132	145	({ \
133	146	typeof(ptr) __PTR = (ptr); \

arch/arm64/include/asm/cache.h+4−0

@@ -21,12 +21,16 @@ · 20 unmodified lines
21	21	#define CTR_L1IP_SHIFT 14
22	22	#define CTR_L1IP_MASK 3
23	23	#define CTR_DMINLINE_SHIFT 16
	24	+#define CTR_IMINLINE_SHIFT 0
24	25	#define CTR_ERG_SHIFT 20
25	26	#define CTR_CWG_SHIFT 24
26	27	#define CTR_CWG_MASK 15
27	28	#define CTR_IDC_SHIFT 28
28	29	#define CTR_DIC_SHIFT 29
29	30
	31	+#define CTR_CACHE_MINLINE_MASK \
	32	+ (0xf << CTR_DMINLINE_SHIFT \| 0xf << CTR_IMINLINE_SHIFT)
	33	+
30	34	#define CTR_L1IP(ctr) (((ctr) >> CTR_L1IP_SHIFT) & CTR_L1IP_MASK)
31	35
32	36	#define ICACHE_POLICY_VPIPT 0

arch/arm64/include/asm/cacheflush.h+26−1

@@ -19,6 +19,7 @@ · 18 unmodified lines
19	19	#ifndef __ASM_CACHEFLUSH_H
20	20	#define __ASM_CACHEFLUSH_H
21	21
	22	+#include <linux/kgdb.h>
22	23	#include <linux/mm.h>
23	24
24	25	/*
@@ -71,7 +72,7 @@ · 46 unmodified lines
71	72	* - kaddr - page address
72	73	* - size - region size
73	74	*/
74		−extern void flush_icache_range(unsigned long start, unsigned long end);
	75	+extern void __flush_icache_range(unsigned long start, unsigned long end);
75	76	extern int invalidate_icache_range(unsigned long start, unsigned long end);
76	77	extern void __flush_dcache_area(void *addr, size_t len);
77	78	extern void __inval_dcache_area(void *addr, size_t len);
@@ -81,6 +82,30 @@ extern void __clean_dcache_area_pou(void *addr, size_t len); · 3 unmodified lines
81	82	extern long __flush_cache_user_range(unsigned long start, unsigned long end);
82	83	extern void sync_icache_aliases(void *kaddr, unsigned long len);
83	84
	85	+static inline void flush_icache_range(unsigned long start, unsigned long end)
	86	+{
	87	+ __flush_icache_range(start, end);
	88	+
	89	+ /*
	90	+ * IPI all online CPUs so that they undergo a context synchronization
	91	+ * event and are forced to refetch the new instructions.
	92	+ */
	93	+#ifdef CONFIG_KGDB
	94	+ /*
	95	+ * KGDB performs cache maintenance with interrupts disabled, so we
	96	+ * will deadlock trying to IPI the secondary CPUs. In theory, we can
	97	+ * set CACHE_FLUSH_IS_SAFE to 0 to avoid this known issue, but that
	98	+ * just means that KGDB will elide the maintenance altogether! As it
	99	+ * turns out, KGDB uses IPIs to round-up the secondary CPUs during
	100	+ * the patching operation, so we don't need extra IPIs here anyway.
	101	+ * In which case, add a KGDB-specific bodge and return early.
	102	+ */
	103	+ if (kgdb_connected && irqs_disabled())
	104	+ return;
	105	+#endif
	106	+ kick_all_cpus_sync();
	107	+}
	108	+
84	109	static inline void flush_cache_mm(struct mm_struct *mm)
85	110	{
86	111	}

arch/arm64/include/asm/cpucaps.h+2−1

@@ -49,7 +49,8 @@ · 48 unmodified lines
49	49	#define ARM64_HAS_CACHE_DIC 28
50	50	#define ARM64_HW_DBM 29
51	51	#define ARM64_SSBD 30
	52	+#define ARM64_MISMATCHED_CACHE_TYPE 31
52	53
53		−#define ARM64_NCAPS 31
	54	+#define ARM64_NCAPS 32
54	55
55	56	#endif /* __ASM_CPUCAPS_H */

arch/arm64/include/asm/fpsimd.h+16−1

@@ -16,13 +16,15 @@ · 15 unmodified lines
16	16	#ifndef __ASM_FP_H
17	17	#define __ASM_FP_H
18	18
19		−#include <asm/ptrace.h>
20	19	#include <asm/errno.h>
	20	+#include <asm/ptrace.h>
21	21	#include <asm/processor.h>
22	22	#include <asm/sigcontext.h>
	23	+#include <asm/sysreg.h>
23	24
24	25	#ifndef __ASSEMBLY__
25	26
	27	+#include <linux/build_bug.h>
26	28	#include <linux/cache.h>
27	29	#include <linux/init.h>
28	30	#include <linux/stddef.h>
@@ -102,6 +104,16 @@ extern int sve_set_vector_length(struct task_struct *task, · 73 unmodified lines
102	104	extern int sve_set_current_vl(unsigned long arg);
103	105	extern int sve_get_current_vl(void);
104	106
	107	+static inline void sve_user_disable(void)
	108	+{
	109	+ sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0);
	110	+}
	111	+
	112	+static inline void sve_user_enable(void)
	113	+{
	114	+ sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
	115	+}
	116	+
105	117	/*
106	118	* Probing and setup functions.
107	119	* Calls to these functions must be serialised with one another.
@@ -128,6 +140,9 @@ static inline int sve_get_current_vl(void) · 20 unmodified lines
128	140	return -EINVAL;
129	141	}
130	142
	143	+static inline void sve_user_disable(void) { BUILD_BUG(); }
	144	+static inline void sve_user_enable(void) { BUILD_BUG(); }
	145	+
131	146	static inline void sve_init_vq_map(void) { }
132	147	static inline void sve_update_vq_map(void) { }
133	148	static inline int sve_verify_vq_map(void) { return 0; }

arch/arm64/include/asm/insn.h+0−2

@@ -446,8 +446,6 @@ u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base, · 445 unmodified lines
446	446	s32 aarch64_get_branch_offset(u32 insn);
447	447	u32 aarch64_set_branch_offset(u32 insn, s32 offset);
448	448
449		−bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn);
450		−
451	449	int aarch64_insn_patch_text_nosync(void *addr, u32 insn);
452	450	int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt);
453	451

arch/arm64/include/asm/kvm_emulate.h+5−5

@@ -140,7 +140,7 @@ static inline void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr) · 139 unmodified lines
140	140
141	141	static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu)
142	142	{
143		− *vcpu_cpsr(vcpu) \|= COMPAT_PSR_T_BIT;
	143	+ *vcpu_cpsr(vcpu) \|= PSR_AA32_T_BIT;
144	144	}
145	145
146	146	/*
@@ -190,8 +190,8 @@ static inline bool vcpu_mode_priv(const struct kvm_vcpu *vcpu) · 43 unmodified lines
190	190	u32 mode;
191	191
192	192	if (vcpu_mode_is_32bit(vcpu)) {
193		− mode = *vcpu_cpsr(vcpu) & COMPAT_PSR_MODE_MASK;
194		− return mode > COMPAT_PSR_MODE_USR;
	193	+ mode = *vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK;
	194	+ return mode > PSR_AA32_MODE_USR;
195	195	}
196	196
197	197	mode = *vcpu_cpsr(vcpu) & PSR_MODE_MASK;
@@ -329,7 +329,7 @@ static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) · 131 unmodified lines
329	329	static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)
330	330	{
331	331	if (vcpu_mode_is_32bit(vcpu)) {
332		− *vcpu_cpsr(vcpu) \|= COMPAT_PSR_E_BIT;
	332	+ *vcpu_cpsr(vcpu) \|= PSR_AA32_E_BIT;
333	333	} else {
334	334	u64 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1);
335	335	sctlr \|= (1 << 25);
@@ -340,7 +340,7 @@ static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu) · 4 unmodified lines
340	340	static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu)
341	341	{
342	342	if (vcpu_mode_is_32bit(vcpu))
343		− return !!(*vcpu_cpsr(vcpu) & COMPAT_PSR_E_BIT);
	343	+ return !!(*vcpu_cpsr(vcpu) & PSR_AA32_E_BIT);
344	344
345	345	return !!(vcpu_read_sys_reg(vcpu, SCTLR_EL1) & (1 << 25));
346	346	}

arch/arm64/include/asm/neon.h+0−7

@@ -19,11 +19,4 @@ · 18 unmodified lines
19	19	void kernel_neon_begin(void);
20	20	void kernel_neon_end(void);
21	21
22		−/*
23		− * Temporary macro to allow the crypto code to compile. Note that the
24		− * semantics of kernel_neon_begin_partial() are now different from the
25		− * original as it does not allow being called in an interrupt context.
26		− */
27		−#define kernel_neon_begin_partial(num_regs) kernel_neon_begin()
28		−
29	22	#endif /* ! __ASM_NEON_H */

arch/arm64/include/asm/numa.h+4−0

@@ -35,10 +35,14 @@ void __init numa_set_distance(int from, int to, int distance); · 34 unmodified lines
35	35	void __init numa_free_distance(void);
36	36	void __init early_map_cpu_to_node(unsigned int cpu, int nid);
37	37	void numa_store_cpu_info(unsigned int cpu);
	38	+void numa_add_cpu(unsigned int cpu);
	39	+void numa_remove_cpu(unsigned int cpu);
38	40
39	41	#else /* CONFIG_NUMA */
40	42
41	43	static inline void numa_store_cpu_info(unsigned int cpu) { }
	44	+static inline void numa_add_cpu(unsigned int cpu) { }
	45	+static inline void numa_remove_cpu(unsigned int cpu) { }
42	46	static inline void arm64_numa_init(void) { }
43	47	static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
44	48

arch/arm64/include/asm/processor.h+18−3

@@ -182,12 +182,12 @@ static inline void compat_start_thread(struct pt_regs *regs, unsigned long pc, · 181 unmodified lines
182	182	unsigned long sp)
183	183	{
184	184	start_thread_common(regs, pc);
185		− regs->pstate = COMPAT_PSR_MODE_USR;
	185	+ regs->pstate = PSR_AA32_MODE_USR;
186	186	if (pc & 1)
187		− regs->pstate \|= COMPAT_PSR_T_BIT;
	187	+ regs->pstate \|= PSR_AA32_T_BIT;
188	188
189	189	#ifdef __AARCH64EB__
190		− regs->pstate \|= COMPAT_PSR_E_BIT;
	190	+ regs->pstate \|= PSR_AA32_E_BIT;
191	191	#endif
192	192
193	193	regs->compat_sp = sp;
@@ -266,5 +266,20 @@ extern void __init minsigstksz_setup(void); · 72 unmodified lines
266	266	#define SVE_SET_VL(arg) sve_set_current_vl(arg)
267	267	#define SVE_GET_VL() sve_get_current_vl()
268	268
	269	+/*
	270	+ * For CONFIG_GCC_PLUGIN_STACKLEAK
	271	+ *
	272	+ * These need to be macros because otherwise we get stuck in a nightmare
	273	+ * of header definitions for the use of task_stack_page.
	274	+ */
	275	+
	276	+#define current_top_of_stack() \
	277	+({ \
	278	+ struct stack_info _info; \
	279	+ BUG_ON(!on_accessible_stack(current, current_stack_pointer, &_info)); \
	280	+ _info.high; \
	281	+})
	282	+#define on_thread_stack() (on_task_stack(current, current_stack_pointer, NULL))
	283	+
269	284	#endif /* __ASSEMBLY__ */
270	285	#endif /* __ASM_PROCESSOR_H */

arch/arm64/include/asm/ptrace.h+53−26

@@ -35,36 +35,39 @@ · 34 unmodified lines
35	35	#define COMPAT_PTRACE_GETHBPREGS 29
36	36	#define COMPAT_PTRACE_SETHBPREGS 30
37	37
38		−/* AArch32 CPSR bits */
39		−#define COMPAT_PSR_MODE_MASK 0x0000001f
40		−#define COMPAT_PSR_MODE_USR 0x00000010
41		−#define COMPAT_PSR_MODE_FIQ 0x00000011
42		−#define COMPAT_PSR_MODE_IRQ 0x00000012
43		−#define COMPAT_PSR_MODE_SVC 0x00000013
44		−#define COMPAT_PSR_MODE_ABT 0x00000017
45		−#define COMPAT_PSR_MODE_HYP 0x0000001a
46		−#define COMPAT_PSR_MODE_UND 0x0000001b
47		−#define COMPAT_PSR_MODE_SYS 0x0000001f
48		−#define COMPAT_PSR_T_BIT 0x00000020
49		−#define COMPAT_PSR_F_BIT 0x00000040
50		−#define COMPAT_PSR_I_BIT 0x00000080
51		−#define COMPAT_PSR_A_BIT 0x00000100
52		−#define COMPAT_PSR_E_BIT 0x00000200
53		−#define COMPAT_PSR_J_BIT 0x01000000
54		−#define COMPAT_PSR_Q_BIT 0x08000000
55		−#define COMPAT_PSR_V_BIT 0x10000000
56		−#define COMPAT_PSR_C_BIT 0x20000000
57		−#define COMPAT_PSR_Z_BIT 0x40000000
58		−#define COMPAT_PSR_N_BIT 0x80000000
59		−#define COMPAT_PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */
60		−#define COMPAT_PSR_GE_MASK 0x000f0000
	38	+/* SPSR_ELx bits for exceptions taken from AArch32 */
	39	+#define PSR_AA32_MODE_MASK 0x0000001f
	40	+#define PSR_AA32_MODE_USR 0x00000010
	41	+#define PSR_AA32_MODE_FIQ 0x00000011
	42	+#define PSR_AA32_MODE_IRQ 0x00000012
	43	+#define PSR_AA32_MODE_SVC 0x00000013
	44	+#define PSR_AA32_MODE_ABT 0x00000017
	45	+#define PSR_AA32_MODE_HYP 0x0000001a
	46	+#define PSR_AA32_MODE_UND 0x0000001b
	47	+#define PSR_AA32_MODE_SYS 0x0000001f
	48	+#define PSR_AA32_T_BIT 0x00000020
	49	+#define PSR_AA32_F_BIT 0x00000040
	50	+#define PSR_AA32_I_BIT 0x00000080
	51	+#define PSR_AA32_A_BIT 0x00000100
	52	+#define PSR_AA32_E_BIT 0x00000200
	53	+#define PSR_AA32_DIT_BIT 0x01000000
	54	+#define PSR_AA32_Q_BIT 0x08000000
	55	+#define PSR_AA32_V_BIT 0x10000000
	56	+#define PSR_AA32_C_BIT 0x20000000
	57	+#define PSR_AA32_Z_BIT 0x40000000
	58	+#define PSR_AA32_N_BIT 0x80000000
	59	+#define PSR_AA32_IT_MASK 0x0600fc00 /* If-Then execution state mask */
	60	+#define PSR_AA32_GE_MASK 0x000f0000
61	61
62	62	#ifdef CONFIG_CPU_BIG_ENDIAN
63		−#define COMPAT_PSR_ENDSTATE COMPAT_PSR_E_BIT
	63	+#define PSR_AA32_ENDSTATE PSR_AA32_E_BIT
64	64	#else
65		−#define COMPAT_PSR_ENDSTATE 0
	65	+#define PSR_AA32_ENDSTATE 0
66	66	#endif
67	67
	68	+/* AArch32 CPSR bits, as seen in AArch32 */
	69	+#define COMPAT_PSR_DIT_BIT 0x00200000
	70	+
68	71	/*
69	72	* These are 'magic' values for PTRACE_PEEKUSR that return info about where a
70	73	* process is located in memory.
@@ -111,6 +114,30 @@ · 40 unmodified lines
111	114	#define compat_sp_fiq regs[29]
112	115	#define compat_lr_fiq regs[30]
113	116
	117	+static inline unsigned long compat_psr_to_pstate(const unsigned long psr)
	118	+{
	119	+ unsigned long pstate;
	120	+
	121	+ pstate = psr & ~COMPAT_PSR_DIT_BIT;
	122	+
	123	+ if (psr & COMPAT_PSR_DIT_BIT)
	124	+ pstate \|= PSR_AA32_DIT_BIT;
	125	+
	126	+ return pstate;
	127	+}
	128	+
	129	+static inline unsigned long pstate_to_compat_psr(const unsigned long pstate)
	130	+{
	131	+ unsigned long psr;
	132	+
	133	+ psr = pstate & ~PSR_AA32_DIT_BIT;
	134	+
	135	+ if (pstate & PSR_AA32_DIT_BIT)
	136	+ psr \|= COMPAT_PSR_DIT_BIT;
	137	+
	138	+ return psr;
	139	+}
	140	+
114	141	/*
115	142	* This struct defines the way the registers are stored on the stack during an
116	143	* exception. Note that sizeof(struct pt_regs) has to be a multiple of 16 (for
@@ -156,7 +183,7 @@ static inline void forget_syscall(struct pt_regs *regs) · 39 unmodified lines
156	183
157	184	#ifdef CONFIG_COMPAT
158	185	#define compat_thumb_mode(regs) \
159		− (((regs)->pstate & COMPAT_PSR_T_BIT))
	186	+ (((regs)->pstate & PSR_AA32_T_BIT))
160	187	#else
161	188	#define compat_thumb_mode(regs) (0)
162	189	#endif

arch/arm64/include/asm/sdei.h+6−3

@@ -40,15 +40,18 @@ asmlinkage unsigned long __sdei_handler(struct pt_regs *regs, · 39 unmodified lines
40	40	unsigned long sdei_arch_get_entry_point(int conduit);
41	41	#define sdei_arch_get_entry_point(x) sdei_arch_get_entry_point(x)
42	42
43		−bool _on_sdei_stack(unsigned long sp);
44		−static inline bool on_sdei_stack(unsigned long sp)
	43	+struct stack_info;
	44	+
	45	+bool _on_sdei_stack(unsigned long sp, struct stack_info *info);
	46	+static inline bool on_sdei_stack(unsigned long sp,
	47	+ struct stack_info *info)
45	48	{
46	49	if (!IS_ENABLED(CONFIG_VMAP_STACK))
47	50	return false;
48	51	if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE))
49	52	return false;
50	53	if (in_nmi())
51		− return _on_sdei_stack(sp);
	54	+ return _on_sdei_stack(sp, info);
52	55
53	56	return false;
54	57	}

arch/arm64/include/asm/spinlock.h+1−116

@@ -16,123 +16,8 @@ · 15 unmodified lines
16	16	#ifndef __ASM_SPINLOCK_H
17	17	#define __ASM_SPINLOCK_H
18	18
19		−#include <asm/lse.h>
20		−#include <asm/spinlock_types.h>
21		−#include <asm/processor.h>
22		−
23		−/*
24		− * Spinlock implementation.
25		− *
26		− * The memory barriers are implicit with the load-acquire and store-release
27		− * instructions.
28		− */
29		−
30		−static inline void arch_spin_lock(arch_spinlock_t *lock)
31		−{
32		− unsigned int tmp;
33		− arch_spinlock_t lockval, newval;
34		−
35		− asm volatile(
36		− /* Atomically increment the next ticket. */
37		− ARM64_LSE_ATOMIC_INSN(
38		− /* LL/SC */
39		−" prfm pstl1strm, %3\n"
40		−"1: ldaxr %w0, %3\n"
41		−" add %w1, %w0, %w5\n"
42		−" stxr %w2, %w1, %3\n"
43		−" cbnz %w2, 1b\n",
44		− /* LSE atomics */
45		−" mov %w2, %w5\n"
46		−" ldadda %w2, %w0, %3\n"
47		− __nops(3)
48		− )
49		−
50		− /* Did we get the lock? */
51		−" eor %w1, %w0, %w0, ror #16\n"
52		−" cbz %w1, 3f\n"
53		− /*
54		− * No: spin on the owner. Send a local event to avoid missing an
55		− * unlock before the exclusive load.
56		− */
57		−" sevl\n"
58		−"2: wfe\n"
59		−" ldaxrh %w2, %4\n"
60		−" eor %w1, %w2, %w0, lsr #16\n"
61		−" cbnz %w1, 2b\n"
62		− /* We got the lock. Critical section starts here. */
63		−"3:"
64		− : "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock)
65		− : "Q" (lock->owner), "I" (1 << TICKET_SHIFT)
66		− : "memory");
67		−}
68		−
69		−static inline int arch_spin_trylock(arch_spinlock_t *lock)
70		−{
71		− unsigned int tmp;
72		− arch_spinlock_t lockval;
73		−
74		− asm volatile(ARM64_LSE_ATOMIC_INSN(
75		− /* LL/SC */
76		− " prfm pstl1strm, %2\n"
77		− "1: ldaxr %w0, %2\n"
78		− " eor %w1, %w0, %w0, ror #16\n"
79		− " cbnz %w1, 2f\n"
80		− " add %w0, %w0, %3\n"
81		− " stxr %w1, %w0, %2\n"
82		− " cbnz %w1, 1b\n"
83		− "2:",
84		− /* LSE atomics */
85		− " ldr %w0, %2\n"
86		− " eor %w1, %w0, %w0, ror #16\n"
87		− " cbnz %w1, 1f\n"
88		− " add %w1, %w0, %3\n"
89		− " casa %w0, %w1, %2\n"
90		− " sub %w1, %w1, %3\n"
91		− " eor %w1, %w1, %w0\n"
92		− "1:")
93		− : "=&r" (lockval), "=&r" (tmp), "+Q" (*lock)
94		− : "I" (1 << TICKET_SHIFT)
95		− : "memory");
96		−
97		− return !tmp;
98		−}
99		−
100		−static inline void arch_spin_unlock(arch_spinlock_t *lock)
101		−{
102		− unsigned long tmp;
103		−
104		− asm volatile(ARM64_LSE_ATOMIC_INSN(
105		− /* LL/SC */
106		− " ldrh %w1, %0\n"
107		− " add %w1, %w1, #1\n"
108		− " stlrh %w1, %0",
109		− /* LSE atomics */
110		− " mov %w1, #1\n"
111		− " staddlh %w1, %0\n"
112		− __nops(1))
113		− : "=Q" (lock->owner), "=&r" (tmp)
114		− :
115		− : "memory");
116		−}
117		−
118		−static inline int arch_spin_value_unlocked(arch_spinlock_t lock)
119		−{
120		− return lock.owner == lock.next;
121		−}
122		−
123		−static inline int arch_spin_is_locked(arch_spinlock_t *lock)
124		−{
125		− return !arch_spin_value_unlocked(READ_ONCE(*lock));
126		−}
127		−
128		−static inline int arch_spin_is_contended(arch_spinlock_t *lock)
129		−{
130		− arch_spinlock_t lockval = READ_ONCE(*lock);
131		− return (lockval.next - lockval.owner) > 1;
132		−}
133		−#define arch_spin_is_contended arch_spin_is_contended
134		−
135	19	#include <asm/qrwlock.h>
	20	+#include <asm/qspinlock.h>
136	21
137	22	/* See include/linux/spinlock.h */
138	23	#define smp_mb__after_spinlock() smp_mb()

arch/arm64/include/asm/spinlock_types.h+1−16

@@ -20,22 +20,7 @@ · 19 unmodified lines
20	20	# error "please don't include this file directly"
21	21	#endif
22	22
23		−#include <linux/types.h>
24		−
25		−#define TICKET_SHIFT 16
26		−
27		−typedef struct {
28		−#ifdef __AARCH64EB__
29		− u16 next;
30		− u16 owner;
31		−#else
32		− u16 owner;
33		− u16 next;
34		−#endif
35		−} __aligned(4) arch_spinlock_t;
36		−
37		−#define __ARCH_SPIN_LOCK_UNLOCKED { 0 , 0 }
38		−
	23	+#include <asm-generic/qspinlock_types.h>
39	24	#include <asm-generic/qrwlock_types.h>
40	25
41	26	#endif

arch/arm64/include/asm/stacktrace.h+61−12

@@ -32,6 +32,21 @@ struct stackframe { · 31 unmodified lines
32	32	#endif
33	33	};
34	34
	35	+enum stack_type {
	36	+ STACK_TYPE_UNKNOWN,
	37	+ STACK_TYPE_TASK,
	38	+ STACK_TYPE_IRQ,
	39	+ STACK_TYPE_OVERFLOW,
	40	+ STACK_TYPE_SDEI_NORMAL,
	41	+ STACK_TYPE_SDEI_CRITICAL,
	42	+};
	43	+
	44	+struct stack_info {
	45	+ unsigned long low;
	46	+ unsigned long high;
	47	+ enum stack_type type;
	48	+};
	49	+
35	50	extern int unwind_frame(struct task_struct tsk, struct stackframe frame);
36	51	extern void walk_stackframe(struct task_struct tsk, struct stackframe frame,
37	52	int (fn)(struct stackframe , void ), void data);
@@ -39,7 +54,8 @@ extern void dump_backtrace(struct pt_regs regs, struct task_struct tsk); · 1 unmodified line
39	54
40	55	DECLARE_PER_CPU(unsigned long *, irq_stack_ptr);
41	56
42		−static inline bool on_irq_stack(unsigned long sp)
	57	+static inline bool on_irq_stack(unsigned long sp,
	58	+ struct stack_info *info)
43	59	{
44	60	unsigned long low = (unsigned long)raw_cpu_read(irq_stack_ptr);
45	61	unsigned long high = low + IRQ_STACK_SIZE;
@@ -47,46 +63,79 @@ static inline bool on_irq_stack(unsigned long sp) · 1 unmodified line
47	63	if (!low)
48	64	return false;
49	65
50		− return (low <= sp && sp < high);
	66	+ if (sp < low \|\| sp >= high)
	67	+ return false;
	68	+
	69	+ if (info) {
	70	+ info->low = low;
	71	+ info->high = high;
	72	+ info->type = STACK_TYPE_IRQ;
	73	+ }
	74	+
	75	+ return true;
51	76	}
52	77
53		−static inline bool on_task_stack(struct task_struct *tsk, unsigned long sp)
	78	+static inline bool on_task_stack(struct task_struct *tsk, unsigned long sp,
	79	+ struct stack_info *info)
54	80	{
55	81	unsigned long low = (unsigned long)task_stack_page(tsk);
56	82	unsigned long high = low + THREAD_SIZE;
57	83
58		− return (low <= sp && sp < high);
	84	+ if (sp < low \|\| sp >= high)
	85	+ return false;
	86	+
	87	+ if (info) {
	88	+ info->low = low;
	89	+ info->high = high;
	90	+ info->type = STACK_TYPE_TASK;
	91	+ }
	92	+
	93	+ return true;
59	94	}
60	95
61	96	#ifdef CONFIG_VMAP_STACK
62	97	DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack);
63	98
64		−static inline bool on_overflow_stack(unsigned long sp)
	99	+static inline bool on_overflow_stack(unsigned long sp,
	100	+ struct stack_info *info)
65	101	{
66	102	unsigned long low = (unsigned long)raw_cpu_ptr(overflow_stack);
67	103	unsigned long high = low + OVERFLOW_STACK_SIZE;
68	104
69		− return (low <= sp && sp < high);
	105	+ if (sp < low \|\| sp >= high)
	106	+ return false;
	107	+
	108	+ if (info) {
	109	+ info->low = low;
	110	+ info->high = high;
	111	+ info->type = STACK_TYPE_OVERFLOW;
	112	+ }
	113	+
	114	+ return true;
70	115	}
71	116	#else
72		−static inline bool on_overflow_stack(unsigned long sp) { return false; }
	117	+static inline bool on_overflow_stack(unsigned long sp,
	118	+ struct stack_info *info) { return false; }
73	119	#endif
74	120
	121	+
75	122	/*
76	123	* We can only safely access per-cpu stacks from current in a non-preemptible
77	124	* context.
78	125	*/
79		−static inline bool on_accessible_stack(struct task_struct *tsk, unsigned long sp)
	126	+static inline bool on_accessible_stack(struct task_struct *tsk,
	127	+ unsigned long sp,
	128	+ struct stack_info *info)
80	129	{
81		− if (on_task_stack(tsk, sp))
	130	+ if (on_task_stack(tsk, sp, info))
82	131	return true;
83	132	if (tsk != current \|\| preemptible())
84	133	return false;
85		− if (on_irq_stack(sp))
	134	+ if (on_irq_stack(sp, info))
86	135	return true;
87		− if (on_overflow_stack(sp))
	136	+ if (on_overflow_stack(sp, info))
88	137	return true;
89		− if (on_sdei_stack(sp))
	138	+ if (on_sdei_stack(sp, info))
90	139	return true;
91	140
92	141	return false;

arch/arm64/include/asm/syscall.h+7−1

@@ -20,7 +20,13 @@ · 19 unmodified lines
20	20	#include <linux/compat.h>
21	21	#include <linux/err.h>
22	22
23		−extern const void *sys_call_table[];
	23	+typedef long (syscall_fn_t)(struct pt_regs regs);
	24	+
	25	+extern const syscall_fn_t sys_call_table[];
	26	+
	27	+#ifdef CONFIG_COMPAT
	28	+extern const syscall_fn_t compat_sys_call_table[];
	29	+#endif
24	30
25	31	static inline int syscall_get_nr(struct task_struct *task,
26	32	struct pt_regs *regs)

arch/arm64/include/asm/syscall_wrapper.h+80−0

@@ -0,0 +1,80 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/*
	3	+ * syscall_wrapper.h - arm64 specific wrappers to syscall definitions
	4	+ *
	5	+ * Based on arch/x86/include/asm_syscall_wrapper.h
	6	+ */
	7	+
	8	+#ifndef __ASM_SYSCALL_WRAPPER_H
	9	+#define __ASM_SYSCALL_WRAPPER_H
	10	+
	11	+#define SC_ARM64_REGS_TO_ARGS(x, ...) \
	12	+ __MAP(x,__SC_ARGS \
	13	+ ,,regs->regs[0],,regs->regs[1],,regs->regs[2] \
	14	+ ,,regs->regs[3],,regs->regs[4],,regs->regs[5])
	15	+
	16	+#ifdef CONFIG_COMPAT
	17	+
	18	+#define COMPAT_SYSCALL_DEFINEx(x, name, ...) \
	19	+ asmlinkage long __arm64_compat_sys##name(const struct pt_regs *regs); \
	20	+ ALLOW_ERROR_INJECTION(__arm64_compat_sys##name, ERRNO); \
	21	+ static long __se_compat_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)); \
	22	+ static inline long __do_compat_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)); \
	23	+ asmlinkage long __arm64_compat_sys##name(const struct pt_regs *regs) \
	24	+ { \
	25	+ return __se_compat_sys##name(SC_ARM64_REGS_TO_ARGS(x,__VA_ARGS__)); \
	26	+ } \
	27	+ static long __se_compat_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)) \
	28	+ { \
	29	+ return __do_compat_sys##name(__MAP(x,__SC_DELOUSE,__VA_ARGS__)); \
	30	+ } \
	31	+ static inline long __do_compat_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__))
	32	+
	33	+#define COMPAT_SYSCALL_DEFINE0(sname) \
	34	+ asmlinkage long __arm64_compat_sys_##sname(void); \
	35	+ ALLOW_ERROR_INJECTION(__arm64_compat_sys_##sname, ERRNO); \
	36	+ asmlinkage long __arm64_compat_sys_##sname(void)
	37	+
	38	+#define COND_SYSCALL_COMPAT(name) \
	39	+ cond_syscall(__arm64_compat_sys_##name);
	40	+
	41	+#define COMPAT_SYS_NI(name) \
	42	+ SYSCALL_ALIAS(__arm64_compat_sys_##name, sys_ni_posix_timers);
	43	+
	44	+#endif /* CONFIG_COMPAT */
	45	+
	46	+#define __SYSCALL_DEFINEx(x, name, ...) \
	47	+ asmlinkage long __arm64_sys##name(const struct pt_regs *regs); \
	48	+ ALLOW_ERROR_INJECTION(__arm64_sys##name, ERRNO); \
	49	+ static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)); \
	50	+ static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)); \
	51	+ asmlinkage long __arm64_sys##name(const struct pt_regs *regs) \
	52	+ { \
	53	+ return __se_sys##name(SC_ARM64_REGS_TO_ARGS(x,__VA_ARGS__)); \
	54	+ } \
	55	+ static long __se_sys##name(__MAP(x,__SC_LONG,__VA_ARGS__)) \
	56	+ { \
	57	+ long ret = __do_sys##name(__MAP(x,__SC_CAST,__VA_ARGS__)); \
	58	+ __MAP(x,__SC_TEST,__VA_ARGS__); \
	59	+ __PROTECT(x, ret,__MAP(x,__SC_ARGS,__VA_ARGS__)); \
	60	+ return ret; \
	61	+ } \
	62	+ static inline long __do_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__))
	63	+
	64	+#ifndef SYSCALL_DEFINE0
	65	+#define SYSCALL_DEFINE0(sname) \
	66	+ SYSCALL_METADATA(_##sname, 0); \
	67	+ asmlinkage long __arm64_sys_##sname(void); \
	68	+ ALLOW_ERROR_INJECTION(__arm64_sys_##sname, ERRNO); \
	69	+ asmlinkage long __arm64_sys_##sname(void)
	70	+#endif
	71	+
	72	+#ifndef COND_SYSCALL
	73	+#define COND_SYSCALL(name) cond_syscall(__arm64_sys_##name)
	74	+#endif
	75	+
	76	+#ifndef SYS_NI
	77	+#define SYS_NI(name) SYSCALL_ALIAS(__arm64_sys_##name, sys_ni_posix_timers);
	78	+#endif
	79	+
	80	+#endif /* __ASM_SYSCALL_WRAPPER_H */

arch/arm64/include/asm/sysreg.h+10−20

@@ -436,7 +436,8 @@ · 435 unmodified lines
436	436	#define SCTLR_EL2_RES0 ((1 << 6) \| (1 << 7) \| (1 << 8) \| (1 << 9) \| \
437	437	(1 << 10) \| (1 << 13) \| (1 << 14) \| (1 << 15) \| \
438	438	(1 << 17) \| (1 << 20) \| (1 << 24) \| (1 << 26) \| \
439		− (1 << 27) \| (1 << 30) \| (1 << 31))
	439	+ (1 << 27) \| (1 << 30) \| (1 << 31) \| \
	440	+ (0xffffffffUL << 32))
440	441
441	442	#ifdef CONFIG_CPU_BIG_ENDIAN
442	443	#define ENDIAN_SET_EL2 SCTLR_ELx_EE
@@ -452,9 +453,9 @@ · 9 unmodified lines
452	453	SCTLR_ELx_SA \| SCTLR_ELx_I \| SCTLR_ELx_WXN \| \
453	454	ENDIAN_CLEAR_EL2 \| SCTLR_EL2_RES0)
454	455
455		−/* Check all the bits are accounted for */
456		−#define SCTLR_EL2_BUILD_BUG_ON_MISSING_BITS BUILD_BUG_ON((SCTLR_EL2_SET ^ SCTLR_EL2_CLEAR) != ~0)
457		−
	456	+#if (SCTLR_EL2_SET ^ SCTLR_EL2_CLEAR) != 0xffffffffffffffff
	457	+#error "Inconsistent SCTLR_EL2 set/clear bits"
	458	+#endif
458	459
459	460	/* SCTLR_EL1 specific flags. */
460	461	#define SCTLR_EL1_UCI (1 << 26)
@@ -473,7 +474,8 @@ · 12 unmodified lines
473	474	#define SCTLR_EL1_RES1 ((1 << 11) \| (1 << 20) \| (1 << 22) \| (1 << 28) \| \
474	475	(1 << 29))
475	476	#define SCTLR_EL1_RES0 ((1 << 6) \| (1 << 10) \| (1 << 13) \| (1 << 17) \| \
476		− (1 << 27) \| (1 << 30) \| (1 << 31))
	477	+ (1 << 27) \| (1 << 30) \| (1 << 31) \| \
	478	+ (0xffffffffUL << 32))
477	479
478	480	#ifdef CONFIG_CPU_BIG_ENDIAN
479	481	#define ENDIAN_SET_EL1 (SCTLR_EL1_E0E \| SCTLR_ELx_EE)
@@ -492,8 +494,9 @@ · 12 unmodified lines
492	494	SCTLR_EL1_UMA \| SCTLR_ELx_WXN \| ENDIAN_CLEAR_EL1 \|\
493	495	SCTLR_EL1_RES0)
494	496
495		−/* Check all the bits are accounted for */
496		−#define SCTLR_EL1_BUILD_BUG_ON_MISSING_BITS BUILD_BUG_ON((SCTLR_EL1_SET ^ SCTLR_EL1_CLEAR) != ~0)
	497	+#if (SCTLR_EL1_SET ^ SCTLR_EL1_CLEAR) != 0xffffffffffffffff
	498	+#error "Inconsistent SCTLR_EL1 set/clear bits"
	499	+#endif
497	500
498	501	/* id_aa64isar0 */
499	502	#define ID_AA64ISAR0_TS_SHIFT 52
@@ -739,19 +742,6 @@ asm( · 239 unmodified lines
739	742	write_sysreg(__scs_new, sysreg); \
740	743	} while (0)
741	744
742		−static inline void config_sctlr_el1(u32 clear, u32 set)
743		−{
744		− u32 val;
745		−
746		− SCTLR_EL2_BUILD_BUG_ON_MISSING_BITS;
747		− SCTLR_EL1_BUILD_BUG_ON_MISSING_BITS;
748		−
749		− val = read_sysreg(sctlr_el1);
750		− val &= ~clear;
751		− val \|= set;
752		− write_sysreg(val, sctlr_el1);
753		−}
754		−
755	745	#endif
756	746
757	747	#endif /* __ASM_SYSREG_H */

arch/arm64/include/asm/tlbflush.h+7−0

@@ -218,6 +218,13 @@ static inline void __flush_tlb_pgtable(struct mm_struct *mm, · 217 unmodified lines
218	218	dsb(ish);
219	219	}
220	220
	221	+static inline void __flush_tlb_kernel_pgtable(unsigned long kaddr)
	222	+{
	223	+ unsigned long addr = __TLBI_VADDR(kaddr, 0);
	224	+
	225	+ __tlbi(vaae1is, addr);
	226	+ dsb(ish);
	227	+}
221	228	#endif
222	229
223	230	#endif

arch/arm64/include/asm/topology.h+3−1

@@ -11,7 +11,7 @@ struct cpu_topology { · 10 unmodified lines
11	11	int llc_id;
12	12	cpumask_t thread_sibling;
13	13	cpumask_t core_sibling;
14		− cpumask_t llc_siblings;
	14	+ cpumask_t llc_sibling;
15	15	};
16	16
17	17	extern struct cpu_topology cpu_topology[NR_CPUS];
@@ -20,9 +20,11 @@ extern struct cpu_topology cpu_topology[NR_CPUS]; · 2 unmodified lines
20	20	#define topology_core_id(cpu) (cpu_topology[cpu].core_id)
21	21	#define topology_core_cpumask(cpu) (&cpu_topology[cpu].core_sibling)
22	22	#define topology_sibling_cpumask(cpu) (&cpu_topology[cpu].thread_sibling)
	23	+#define topology_llc_cpumask(cpu) (&cpu_topology[cpu].llc_sibling)
23	24
24	25	void init_cpu_topology(void);
25	26	void store_cpu_topology(unsigned int cpuid);
	27	+void remove_cpu_topology(unsigned int cpuid);
26	28	const struct cpumask *cpu_coregroup_mask(int cpu);
27	29
28	30	#ifdef CONFIG_NUMA

arch/arm64/include/asm/unistd.h+1−1

@@ -43,7 +43,7 @@ · 42 unmodified lines
43	43	#define __ARM_NR_compat_cacheflush (__ARM_NR_COMPAT_BASE+2)
44	44	#define __ARM_NR_compat_set_tls (__ARM_NR_COMPAT_BASE+5)
45	45
46		−#define __NR_compat_syscalls 398
	46	+#define __NR_compat_syscalls 399
47	47	#endif
48	48
49	49	#define __ARCH_WANT_SYS_CLONE

arch/arm64/include/asm/unistd32.h+15−13

@@ -260,7 +260,7 @@ __SYSCALL(117, sys_ni_syscall) · 259 unmodified lines
260	260	#define __NR_fsync 118
261	261	__SYSCALL(__NR_fsync, sys_fsync)
262	262	#define __NR_sigreturn 119
263		−__SYSCALL(__NR_sigreturn, compat_sys_sigreturn_wrapper)
	263	+__SYSCALL(__NR_sigreturn, compat_sys_sigreturn)
264	264	#define __NR_clone 120
265	265	__SYSCALL(__NR_clone, sys_clone)
266	266	#define __NR_setdomainname 121
@@ -368,7 +368,7 @@ __SYSCALL(__NR_getresgid, sys_getresgid16) · 101 unmodified lines
368	368	#define __NR_prctl 172
369	369	__SYSCALL(__NR_prctl, sys_prctl)
370	370	#define __NR_rt_sigreturn 173
371		−__SYSCALL(__NR_rt_sigreturn, compat_sys_rt_sigreturn_wrapper)
	371	+__SYSCALL(__NR_rt_sigreturn, compat_sys_rt_sigreturn)
372	372	#define __NR_rt_sigaction 174
373	373	__SYSCALL(__NR_rt_sigaction, compat_sys_rt_sigaction)
374	374	#define __NR_rt_sigprocmask 175
@@ -382,9 +382,9 @@ __SYSCALL(__NR_rt_sigqueueinfo, compat_sys_rt_sigqueueinfo) · 7 unmodified lines
382	382	#define __NR_rt_sigsuspend 179
383	383	__SYSCALL(__NR_rt_sigsuspend, compat_sys_rt_sigsuspend)
384	384	#define __NR_pread64 180
385		−__SYSCALL(__NR_pread64, compat_sys_pread64_wrapper)
	385	+__SYSCALL(__NR_pread64, compat_sys_aarch32_pread64)
386	386	#define __NR_pwrite64 181
387		−__SYSCALL(__NR_pwrite64, compat_sys_pwrite64_wrapper)
	387	+__SYSCALL(__NR_pwrite64, compat_sys_aarch32_pwrite64)
388	388	#define __NR_chown 182
389	389	__SYSCALL(__NR_chown, sys_chown16)
390	390	#define __NR_getcwd 183
@@ -406,11 +406,11 @@ __SYSCALL(__NR_vfork, sys_vfork) · 15 unmodified lines
406	406	#define __NR_ugetrlimit 191 /* SuS compliant getrlimit */
407	407	__SYSCALL(__NR_ugetrlimit, compat_sys_getrlimit) /* SuS compliant getrlimit */
408	408	#define __NR_mmap2 192
409		−__SYSCALL(__NR_mmap2, compat_sys_mmap2_wrapper)
	409	+__SYSCALL(__NR_mmap2, compat_sys_aarch32_mmap2)
410	410	#define __NR_truncate64 193
411		−__SYSCALL(__NR_truncate64, compat_sys_truncate64_wrapper)
	411	+__SYSCALL(__NR_truncate64, compat_sys_aarch32_truncate64)
412	412	#define __NR_ftruncate64 194
413		−__SYSCALL(__NR_ftruncate64, compat_sys_ftruncate64_wrapper)
	413	+__SYSCALL(__NR_ftruncate64, compat_sys_aarch32_ftruncate64)
414	414	#define __NR_stat64 195
415	415	__SYSCALL(__NR_stat64, sys_stat64)
416	416	#define __NR_lstat64 196
@@ -472,7 +472,7 @@ __SYSCALL(223, sys_ni_syscall) · 55 unmodified lines
472	472	#define __NR_gettid 224
473	473	__SYSCALL(__NR_gettid, sys_gettid)
474	474	#define __NR_readahead 225
475		−__SYSCALL(__NR_readahead, compat_sys_readahead_wrapper)
	475	+__SYSCALL(__NR_readahead, compat_sys_aarch32_readahead)
476	476	#define __NR_setxattr 226
477	477	__SYSCALL(__NR_setxattr, sys_setxattr)
478	478	#define __NR_lsetxattr 227
@@ -554,15 +554,15 @@ __SYSCALL(__NR_clock_getres, compat_sys_clock_getres) · 75 unmodified lines
554	554	#define __NR_clock_nanosleep 265
555	555	__SYSCALL(__NR_clock_nanosleep, compat_sys_clock_nanosleep)
556	556	#define __NR_statfs64 266
557		−__SYSCALL(__NR_statfs64, compat_sys_statfs64_wrapper)
	557	+__SYSCALL(__NR_statfs64, compat_sys_aarch32_statfs64)
558	558	#define __NR_fstatfs64 267
559		−__SYSCALL(__NR_fstatfs64, compat_sys_fstatfs64_wrapper)
	559	+__SYSCALL(__NR_fstatfs64, compat_sys_aarch32_fstatfs64)
560	560	#define __NR_tgkill 268
561	561	__SYSCALL(__NR_tgkill, sys_tgkill)
562	562	#define __NR_utimes 269
563	563	__SYSCALL(__NR_utimes, compat_sys_utimes)
564	564	#define __NR_arm_fadvise64_64 270
565		−__SYSCALL(__NR_arm_fadvise64_64, compat_sys_fadvise64_64_wrapper)
	565	+__SYSCALL(__NR_arm_fadvise64_64, compat_sys_aarch32_fadvise64_64)
566	566	#define __NR_pciconfig_iobase 271
567	567	__SYSCALL(__NR_pciconfig_iobase, sys_pciconfig_iobase)
568	568	#define __NR_pciconfig_read 272
@@ -704,7 +704,7 @@ __SYSCALL(__NR_get_robust_list, compat_sys_get_robust_list) · 135 unmodified lines
704	704	#define __NR_splice 340
705	705	__SYSCALL(__NR_splice, sys_splice)
706	706	#define __NR_sync_file_range2 341
707		−__SYSCALL(__NR_sync_file_range2, compat_sys_sync_file_range2_wrapper)
	707	+__SYSCALL(__NR_sync_file_range2, compat_sys_aarch32_sync_file_range2)
708	708	#define __NR_tee 342
709	709	__SYSCALL(__NR_tee, sys_tee)
710	710	#define __NR_vmsplice 343
@@ -726,7 +726,7 @@ __SYSCALL(__NR_timerfd_create, sys_timerfd_create) · 15 unmodified lines
726	726	#define __NR_eventfd 351
727	727	__SYSCALL(__NR_eventfd, sys_eventfd)
728	728	#define __NR_fallocate 352
729		−__SYSCALL(__NR_fallocate, compat_sys_fallocate_wrapper)
	729	+__SYSCALL(__NR_fallocate, compat_sys_aarch32_fallocate)
730	730	#define __NR_timerfd_settime 353
731	731	__SYSCALL(__NR_timerfd_settime, compat_sys_timerfd_settime)
732	732	#define __NR_timerfd_gettime 354
@@ -817,6 +817,8 @@ __SYSCALL(__NR_pkey_alloc, sys_pkey_alloc) · 84 unmodified lines
817	817	__SYSCALL(__NR_pkey_free, sys_pkey_free)
818	818	#define __NR_statx 397
819	819	__SYSCALL(__NR_statx, sys_statx)
	820	+#define __NR_rseq 398
	821	+__SYSCALL(__NR_rseq, sys_rseq)
820	822
821	823	/*
822	824	* Please add new compat syscalls above this comment and update

arch/arm64/kernel/Makefile+3−2

@@ -18,7 +18,8 @@ arm64-obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ · 17 unmodified lines
18	18	hyp-stub.o psci.o cpu_ops.o insn.o \
19	19	return_address.o cpuinfo.o cpu_errata.o \
20	20	cpufeature.o alternative.o cacheinfo.o \
21		− smp.o smp_spin_table.o topology.o smccc-call.o
	21	+ smp.o smp_spin_table.o topology.o smccc-call.o \
	22	+ syscall.o
22	23
23	24	extra-$(CONFIG_EFI) := efi-entry.o
24	25
@@ -27,7 +28,7 @@ $(obj)/%.stub.o: $(obj)/%.o FORCE · 2 unmodified lines
27	28	$(call if_changed,objcopy)
28	29
29	30	arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
30		− sys_compat.o entry32.o
	31	+ sys_compat.o
31	32	arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o
32	33	arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o
33	34	arm64-obj-$(CONFIG_ARM64_MODULE_PLTS) += module-plts.o

arch/arm64/kernel/acpi.c+3−8

@@ -18,6 +18,7 @@ · 17 unmodified lines
18	18	#include <linux/acpi.h>
19	19	#include <linux/bootmem.h>
20	20	#include <linux/cpumask.h>
	21	+#include <linux/efi.h>
21	22	#include <linux/efi-bgrt.h>
22	23	#include <linux/init.h>
23	24	#include <linux/irq.h>
@@ -29,13 +30,9 @@ · 5 unmodified lines
29	30
30	31	#include <asm/cputype.h>
31	32	#include <asm/cpu_ops.h>
	33	+#include <asm/pgtable.h>
32	34	#include <asm/smp_plat.h>
33	35
34		−#ifdef CONFIG_ACPI_APEI
35		−# include <linux/efi.h>
36		−# include <asm/pgtable.h>
37		−#endif
38		−
39	36	int acpi_noirq = 1; /* skip ACPI IRQ initialization */
40	37	int acpi_disabled = 1;
41	38	EXPORT_SYMBOL(acpi_disabled);
@@ -239,8 +236,7 @@ done: · 197 unmodified lines
239	236	}
240	237	}
241	238
242		−#ifdef CONFIG_ACPI_APEI
243		−pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr)
	239	+pgprot_t __acpi_get_mem_attribute(phys_addr_t addr)
244	240	{
245	241	/*
246	242	* According to "Table 8 Map: EFI memory types to AArch64 memory
@@ -261,4 +257,3 @@ pgprot_t arch_apei_get_mem_attribute(phys_addr_t addr) · 14 unmodified lines
261	257	return __pgprot(PROT_NORMAL_NC);
262	258	return __pgprot(PROT_DEVICE_nGnRnE);
263	259	}
264		−#endif

arch/arm64/kernel/acpi_numa.c+53−35

@@ -26,36 +26,73 @@ · 25 unmodified lines
26	26	#include <linux/module.h>
27	27	#include <linux/topology.h>
28	28
29		−#include <acpi/processor.h>
30	29	#include <asm/numa.h>
31	30
32		−static int cpus_in_srat;
	31	+static int acpi_early_node_map[NR_CPUS] __initdata = { NUMA_NO_NODE };
33	32
34		−struct __node_cpu_hwid {
35		− u32 node_id; /* logical node containing this CPU */
36		− u64 cpu_hwid; /* MPIDR for this CPU */
37		−};
	33	+int __init acpi_numa_get_nid(unsigned int cpu)
	34	+{
	35	+ return acpi_early_node_map[cpu];
	36	+}
	37	+
	38	+static inline int get_cpu_for_acpi_id(u32 uid)
	39	+{
	40	+ int cpu;
	41	+
	42	+ for (cpu = 0; cpu < nr_cpu_ids; cpu++)
	43	+ if (uid == get_acpi_id_for_cpu(cpu))
	44	+ return cpu;
38	45
39		−static struct __node_cpu_hwid early_node_cpu_hwid[NR_CPUS] = {
40		−[0 ... NR_CPUS - 1] = {NUMA_NO_NODE, PHYS_CPUID_INVALID} };
	46	+ return -EINVAL;
	47	+}
41	48
42		−int acpi_numa_get_nid(unsigned int cpu, u64 hwid)
	49	+static int __init acpi_parse_gicc_pxm(struct acpi_subtable_header *header,
	50	+ const unsigned long end)
43	51	{
44		− int i;
	52	+ struct acpi_srat_gicc_affinity *pa;
	53	+ int cpu, pxm, node;
45	54
46		− for (i = 0; i < cpus_in_srat; i++) {
47		− if (hwid == early_node_cpu_hwid[i].cpu_hwid)
48		− return early_node_cpu_hwid[i].node_id;
49		− }
	55	+ if (srat_disabled())
	56	+ return -EINVAL;
	57	+
	58	+ pa = (struct acpi_srat_gicc_affinity *)header;
	59	+ if (!pa)
	60	+ return -EINVAL;
	61	+
	62	+ if (!(pa->flags & ACPI_SRAT_GICC_ENABLED))
	63	+ return 0;
50	64
51		− return NUMA_NO_NODE;
	65	+ pxm = pa->proximity_domain;
	66	+ node = pxm_to_node(pxm);
	67	+
	68	+ /*
	69	+ * If we can't map the UID to a logical cpu this
	70	+ * means that the UID is not part of possible cpus
	71	+ * so we do not need a NUMA mapping for it, skip
	72	+ * the SRAT entry and keep parsing.
	73	+ */
	74	+ cpu = get_cpu_for_acpi_id(pa->acpi_processor_uid);
	75	+ if (cpu < 0)
	76	+ return 0;
	77	+
	78	+ acpi_early_node_map[cpu] = node;
	79	+ pr_info("SRAT: PXM %d -> MPIDR 0x%llx -> Node %d\n", pxm,
	80	+ cpu_logical_map(cpu), node);
	81	+
	82	+ return 0;
	83	+}
	84	+
	85	+void __init acpi_map_cpus_to_nodes(void)
	86	+{
	87	+ acpi_table_parse_entries(ACPI_SIG_SRAT, sizeof(struct acpi_table_srat),
	88	+ ACPI_SRAT_TYPE_GICC_AFFINITY,
	89	+ acpi_parse_gicc_pxm, 0);
52	90	}
53	91
54	92	/* Callback for Proximity Domain -> ACPI processor UID mapping */
55	93	void __init acpi_numa_gicc_affinity_init(struct acpi_srat_gicc_affinity *pa)
56	94	{
57	95	int pxm, node;
58		− phys_cpuid_t mpidr;
59	96
60	97	if (srat_disabled())
61	98	return;
@@ -70,12 +107,6 @@ void __init acpi_numa_gicc_affinity_init(struct acpi_srat_gicc_affinity *pa) · 8 unmodified lines
70	107	if (!(pa->flags & ACPI_SRAT_GICC_ENABLED))
71	108	return;
72	109
73		− if (cpus_in_srat >= NR_CPUS) {
74		− pr_warn_once("SRAT: cpu_to_node_map[%d] is too small, may not be able to use all cpus\n",
75		− NR_CPUS);
76		− return;
77		− }
78		−
79	110	pxm = pa->proximity_domain;
80	111	node = acpi_map_pxm_to_node(pxm);
81	112
@@ -85,20 +116,7 @@ void __init acpi_numa_gicc_affinity_init(struct acpi_srat_gicc_affinity *pa) · 3 unmodified lines
85	116	return;
86	117	}
87	118
88		− mpidr = acpi_map_madt_entry(pa->acpi_processor_uid);
89		− if (mpidr == PHYS_CPUID_INVALID) {
90		− pr_err("SRAT: PXM %d with ACPI ID %d has no valid MPIDR in MADT\n",
91		− pxm, pa->acpi_processor_uid);
92		− bad_srat();
93		− return;
94		− }
95		−
96		− early_node_cpu_hwid[cpus_in_srat].node_id = node;
97		− early_node_cpu_hwid[cpus_in_srat].cpu_hwid = mpidr;
98	119	node_set(node, numa_nodes_parsed);
99		− cpus_in_srat++;
100		− pr_info("SRAT: PXM %d -> MPIDR 0x%Lx -> Node %d\n",
101		− pxm, mpidr, node);
102	120	}
103	121
104	122	int __init arm64_acpi_numa_init(void)

arch/arm64/kernel/alternative.c+2−2

@@ -47,11 +47,11 @@ static bool branch_insn_requires_update(struct alt_instr *alt, unsigned long pc) · 46 unmodified lines
47	47	unsigned long replptr;
48	48
49	49	if (kernel_text_address(pc))
50		− return 1;
	50	+ return true;
51	51
52	52	replptr = (unsigned long)ALT_REPL_PTR(alt);
53	53	if (pc >= replptr && pc <= (replptr + alt->alt_len))
54		− return 0;
	54	+ return false;
55	55
56	56	/*
57	57	* Branching into another alternate sequence is doomed, and

arch/arm64/kernel/armv8_deprecated.c+16−16

@@ -441,8 +441,8 @@ static struct undef_hook swp_hooks[] = { · 440 unmodified lines
441	441	{
442	442	.instr_mask = 0x0fb00ff0,
443	443	.instr_val = 0x01000090,
444		− .pstate_mask = COMPAT_PSR_MODE_MASK,
445		− .pstate_val = COMPAT_PSR_MODE_USR,
	444	+ .pstate_mask = PSR_AA32_MODE_MASK,
	445	+ .pstate_val = PSR_AA32_MODE_USR,
446	446	.fn = swp_handler
447	447	},
448	448	{ }
@@ -511,9 +511,9 @@ ret: · 62 unmodified lines
511	511	static int cp15_barrier_set_hw_mode(bool enable)
512	512	{
513	513	if (enable)
514		− config_sctlr_el1(0, SCTLR_EL1_CP15BEN);
	514	+ sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_CP15BEN);
515	515	else
516		− config_sctlr_el1(SCTLR_EL1_CP15BEN, 0);
	516	+ sysreg_clear_set(sctlr_el1, SCTLR_EL1_CP15BEN, 0);
517	517	return 0;
518	518	}
519	519
@@ -521,15 +521,15 @@ static struct undef_hook cp15_barrier_hooks[] = { · 1 unmodified line
521	521	{
522	522	.instr_mask = 0x0fff0fdf,
523	523	.instr_val = 0x0e070f9a,
524		− .pstate_mask = COMPAT_PSR_MODE_MASK,
525		− .pstate_val = COMPAT_PSR_MODE_USR,
	524	+ .pstate_mask = PSR_AA32_MODE_MASK,
	525	+ .pstate_val = PSR_AA32_MODE_USR,
526	526	.fn = cp15barrier_handler,
527	527	},
528	528	{
529	529	.instr_mask = 0x0fff0fff,
530	530	.instr_val = 0x0e070f95,
531		− .pstate_mask = COMPAT_PSR_MODE_MASK,
532		− .pstate_val = COMPAT_PSR_MODE_USR,
	531	+ .pstate_mask = PSR_AA32_MODE_MASK,
	532	+ .pstate_val = PSR_AA32_MODE_USR,
533	533	.fn = cp15barrier_handler,
534	534	},
535	535	{ }
@@ -548,9 +548,9 @@ static int setend_set_hw_mode(bool enable) · 12 unmodified lines
548	548	return -EINVAL;
549	549
550	550	if (enable)
551		− config_sctlr_el1(SCTLR_EL1_SED, 0);
	551	+ sysreg_clear_set(sctlr_el1, SCTLR_EL1_SED, 0);
552	552	else
553		− config_sctlr_el1(0, SCTLR_EL1_SED);
	553	+ sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_SED);
554	554	return 0;
555	555	}
556	556
@@ -562,10 +562,10 @@ static int compat_setend_handler(struct pt_regs *regs, u32 big_endian) · 5 unmodified lines
562	562
563	563	if (big_endian) {
564	564	insn = "setend be";
565		− regs->pstate \|= COMPAT_PSR_E_BIT;
	565	+ regs->pstate \|= PSR_AA32_E_BIT;
566	566	} else {
567	567	insn = "setend le";
568		− regs->pstate &= ~COMPAT_PSR_E_BIT;
	568	+ regs->pstate &= ~PSR_AA32_E_BIT;
569	569	}
570	570
571	571	trace_instruction_emulation(insn, regs->pc);
@@ -593,16 +593,16 @@ static struct undef_hook setend_hooks[] = { · 21 unmodified lines
593	593	{
594	594	.instr_mask = 0xfffffdff,
595	595	.instr_val = 0xf1010000,
596		− .pstate_mask = COMPAT_PSR_MODE_MASK,
597		− .pstate_val = COMPAT_PSR_MODE_USR,
	596	+ .pstate_mask = PSR_AA32_MODE_MASK,
	597	+ .pstate_val = PSR_AA32_MODE_USR,
598	598	.fn = a32_setend_handler,
599	599	},
600	600	{
601	601	/* Thumb mode */
602	602	.instr_mask = 0x0000fff7,
603	603	.instr_val = 0x0000b650,
604		− .pstate_mask = (COMPAT_PSR_T_BIT \| COMPAT_PSR_MODE_MASK),
605		− .pstate_val = (COMPAT_PSR_T_BIT \| COMPAT_PSR_MODE_USR),
	604	+ .pstate_mask = (PSR_AA32_T_BIT \| PSR_AA32_MODE_MASK),
	605	+ .pstate_val = (PSR_AA32_T_BIT \| PSR_AA32_MODE_USR),
606	606	.fn = t16_setend_handler,
607	607	},
608	608	{}

arch/arm64/kernel/cpu-reset.h+5−4

@@ -16,13 +16,14 @@ · 15 unmodified lines
16	16	void __cpu_soft_restart(unsigned long el2_switch, unsigned long entry,
17	17	unsigned long arg0, unsigned long arg1, unsigned long arg2);
18	18
19		−static inline void __noreturn cpu_soft_restart(unsigned long el2_switch,
20		− unsigned long entry, unsigned long arg0, unsigned long arg1,
21		− unsigned long arg2)
	19	+static inline void __noreturn cpu_soft_restart(unsigned long entry,
	20	+ unsigned long arg0,
	21	+ unsigned long arg1,
	22	+ unsigned long arg2)
22	23	{
23	24	typeof(__cpu_soft_restart) *restart;
24	25
25		− el2_switch = el2_switch && !is_kernel_in_hyp_mode() &&
	26	+ unsigned long el2_switch = !is_kernel_in_hyp_mode() &&
26	27	is_hyp_mode_available();
27	28	restart = (void *)__pa_symbol(__cpu_soft_restart);
28	29

arch/arm64/kernel/cpu_errata.c+20−10

@@ -65,19 +65,24 @@ is_kryo_midr(const struct arm64_cpu_capabilities *entry, int scope) · 64 unmodified lines
65	65	}
66	66
67	67	static bool
68		−has_mismatched_cache_line_size(const struct arm64_cpu_capabilities *entry,
69		− int scope)
	68	+has_mismatched_cache_type(const struct arm64_cpu_capabilities *entry,
	69	+ int scope)
70	70	{
	71	+ u64 mask = CTR_CACHE_MINLINE_MASK;
	72	+
	73	+ /* Skip matching the min line sizes for cache type check */
	74	+ if (entry->capability == ARM64_MISMATCHED_CACHE_TYPE)
	75	+ mask ^= arm64_ftr_reg_ctrel0.strict_mask;
	76	+
71	77	WARN_ON(scope != SCOPE_LOCAL_CPU \|\| preemptible());
72		− return (read_cpuid_cachetype() & arm64_ftr_reg_ctrel0.strict_mask) !=
73		− (arm64_ftr_reg_ctrel0.sys_val & arm64_ftr_reg_ctrel0.strict_mask);
	78	+ return (read_cpuid_cachetype() & mask) !=
	79	+ (arm64_ftr_reg_ctrel0.sys_val & mask);
74	80	}
75	81
76	82	static void
77	83	cpu_enable_trap_ctr_access(const struct arm64_cpu_capabilities *__unused)
78	84	{
79		− /* Clear SCTLR_EL1.UCT */
80		− config_sctlr_el1(SCTLR_EL1_UCT, 0);
	85	+ sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
81	86	}
82	87
83	88	atomic_t arm64_el2_vector_last_slot = ATOMIC_INIT(-1);
@@ -101,7 +106,7 @@ static void __copy_hyp_vect_bpi(int slot, const char *hyp_vecs_start, · 17 unmodified lines
101	106	for (i = 0; i < SZ_2K; i += 0x80)
102	107	memcpy(dst + i, hyp_vecs_start, hyp_vecs_end - hyp_vecs_start);
103	108
104		− flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
	109	+ __flush_icache_range((uintptr_t)dst, (uintptr_t)dst + SZ_2K);
105	110	}
106	111
107	112	static void __install_bp_hardening_cb(bp_hardening_cb_t fn,
@@ -613,7 +618,14 @@ const struct arm64_cpu_capabilities arm64_errata[] = { · 505 unmodified lines
613	618	{
614	619	.desc = "Mismatched cache line size",
615	620	.capability = ARM64_MISMATCHED_CACHE_LINE_SIZE,
616		− .matches = has_mismatched_cache_line_size,
	621	+ .matches = has_mismatched_cache_type,
	622	+ .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
	623	+ .cpu_enable = cpu_enable_trap_ctr_access,
	624	+ },
	625	+ {
	626	+ .desc = "Mismatched cache type",
	627	+ .capability = ARM64_MISMATCHED_CACHE_TYPE,
	628	+ .matches = has_mismatched_cache_type,
617	629	.type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
618	630	.cpu_enable = cpu_enable_trap_ctr_access,
619	631	},
@@ -649,7 +661,6 @@ const struct arm64_cpu_capabilities arm64_errata[] = { · 29 unmodified lines
649	661	#ifdef CONFIG_HARDEN_BRANCH_PREDICTOR
650	662	{
651	663	.capability = ARM64_HARDEN_BRANCH_PREDICTOR,
652		− .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
653	664	.cpu_enable = enable_smccc_arch_workaround_1,
654	665	ERRATA_MIDR_RANGE_LIST(arm64_bp_harden_smccc_cpus),
655	666	},
@@ -658,7 +669,6 @@ const struct arm64_cpu_capabilities arm64_errata[] = { · 2 unmodified lines
658	669	{
659	670	.desc = "EL2 vector hardening",
660	671	.capability = ARM64_HARDEN_EL2_VECTORS,
661		− .type = ARM64_CPUCAP_LOCAL_CPU_ERRATUM,
662	672	ERRATA_MIDR_RANGE_LIST(arm64_harden_el2_vectors),
663	673	},
664	674	#endif

arch/arm64/kernel/cpufeature.c+2−2

@@ -214,7 +214,7 @@ static const struct arm64_ftr_bits ftr_ctr[] = { · 213 unmodified lines
214	214	* If we have differing I-cache policies, report it as the weakest - VIPT.
215	215	*/
216	216	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_EXACT, 14, 2, ICACHE_POLICY_VIPT), /* L1Ip */
217		− ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0), /* IminLine */
	217	+ ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_IMINLINE_SHIFT, 4, 0),
218	218	ARM64_FTR_END,
219	219	};
220	220
@@ -1723,7 +1723,7 @@ static int emulate_mrs(struct pt_regs *regs, u32 insn) · 1502 unmodified lines
1723	1723	static struct undef_hook mrs_hook = {
1724	1724	.instr_mask = 0xfff00000,
1725	1725	.instr_val = 0xd5300000,
1726		− .pstate_mask = COMPAT_PSR_MODE_MASK,
	1726	+ .pstate_mask = PSR_AA32_MODE_MASK,
1727	1727	.pstate_val = PSR_MODE_EL0t,
1728	1728	.fn = emulate_mrs,
1729	1729	};

arch/arm64/kernel/entry.S+23−137

@@ -41,19 +41,9 @@ · 40 unmodified lines
41	41	* Context tracking subsystem. Used to instrument transitions
42	42	* between user and kernel mode.
43	43	*/
44		− .macro ct_user_exit, syscall = 0
	44	+ .macro ct_user_exit
45	45	#ifdef CONFIG_CONTEXT_TRACKING
46	46	bl context_tracking_user_exit
47		− .if \syscall == 1
48		− /*
49		− * Save/restore needed during syscalls. Restore syscall arguments from
50		− * the values already saved on stack during kernel_entry.
51		− */
52		− ldp x0, x1, [sp]
53		− ldp x2, x3, [sp, #S_X2]
54		− ldp x4, x5, [sp, #S_X4]
55		− ldp x6, x7, [sp, #S_X6]
56		− .endif
57	47	#endif
58	48	.endm
59	49
@@ -63,6 +53,12 @@ · 3 unmodified lines
63	53	#endif
64	54	.endm
65	55
	56	+ .macro clear_gp_regs
	57	+ .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29
	58	+ mov x\n, xzr
	59	+ .endr
	60	+ .endm
	61	+
66	62	/*
67	63	* Bad Abort numbers
68	64	*-----------------
@@ -140,20 +136,21 @@ alternative_else_nop_endif · 71 unmodified lines
140	136
141	137	// This macro corrupts x0-x3. It is the caller's duty
142	138	// to save/restore them if required.
143		− .macro apply_ssbd, state, targ, tmp1, tmp2
	139	+ .macro apply_ssbd, state, tmp1, tmp2
144	140	#ifdef CONFIG_ARM64_SSBD
145	141	alternative_cb arm64_enable_wa2_handling
146		− b \targ
	142	+ b .L__asm_ssbd_skip\@
147	143	alternative_cb_end
148	144	ldr_this_cpu \tmp2, arm64_ssbd_callback_required, \tmp1
149		− cbz \tmp2, \targ
	145	+ cbz \tmp2, .L__asm_ssbd_skip\@
150	146	ldr \tmp2, [tsk, #TSK_TI_FLAGS]
151		− tbnz \tmp2, #TIF_SSBD, \targ
	147	+ tbnz \tmp2, #TIF_SSBD, .L__asm_ssbd_skip\@
152	148	mov w0, #ARM_SMCCC_ARCH_WORKAROUND_2
153	149	mov w1, #\state
154	150	alternative_cb arm64_update_smccc_conduit
155	151	nop // Patched to SMC/HVC #0
156	152	alternative_cb_end
	153	+.L__asm_ssbd_skip\@:
157	154	#endif
158	155	.endm
159	156
@@ -178,20 +175,14 @@ alternative_cb_end · 18 unmodified lines
178	175	stp x28, x29, [sp, #16 * 14]
179	176
180	177	.if \el == 0
	178	+ clear_gp_regs
181	179	mrs x21, sp_el0
182	180	ldr_this_cpu tsk, __entry_task, x20 // Ensure MDSCR_EL1.SS is clear,
183	181	ldr x19, [tsk, #TSK_TI_FLAGS] // since we can unmask debug
184	182	disable_step_tsk x19, x20 // exceptions when scheduling.
185	183
186		− apply_ssbd 1, 1f, x22, x23
187		−
188		−#ifdef CONFIG_ARM64_SSBD
189		− ldp x0, x1, [sp, #16 * 0]
190		− ldp x2, x3, [sp, #16 * 1]
191		−#endif
192		−1:
	184	+ apply_ssbd 1, x22, x23
193	185
194		− mov x29, xzr // fp pointed to user-space
195	186	.else
196	187	add x21, sp, #S_FRAME_SIZE
197	188	get_thread_info tsk
@@ -331,8 +322,7 @@ alternative_if ARM64_WORKAROUND_845719 · 133 unmodified lines
331	322	alternative_else_nop_endif
332	323	#endif
333	324	3:
334		− apply_ssbd 0, 5f, x0, x1
335		−5:
	325	+ apply_ssbd 0, x0, x1
336	326	.endif
337	327
338	328	msr elr_el1, x21 // set up the return data
@@ -720,14 +710,9 @@ el0_sync_compat: · 381 unmodified lines
720	710	b.ge el0_dbg
721	711	b el0_inv
722	712	el0_svc_compat:
723		− /*
724		− * AArch32 syscall handling
725		− */
726		− ldr x16, [tsk, #TSK_TI_FLAGS] // load thread flags
727		− adrp stbl, compat_sys_call_table // load compat syscall table pointer
728		− mov wscno, w7 // syscall number in w7 (r7)
729		− mov wsc_nr, #__NR_compat_syscalls
730		− b el0_svc_naked
	713	+ mov x0, sp
	714	+ bl el0_svc_compat_handler
	715	+ b ret_to_user
731	716
732	717	.align 6
733	718	el0_irq_compat:
@@ -896,25 +881,6 @@ el0_error_naked: · 162 unmodified lines
896	881	b ret_to_user
897	882	ENDPROC(el0_error)
898	883
899		−
900		−/*
901		− * This is the fast syscall return path. We do as little as possible here,
902		− * and this includes saving x0 back into the kernel stack.
903		− */
904		−ret_fast_syscall:
905		− disable_daif
906		− str x0, [sp, #S_X0] // returned x0
907		− ldr x1, [tsk, #TSK_TI_FLAGS] // re-check for syscall tracing
908		− and x2, x1, #_TIF_SYSCALL_WORK
909		− cbnz x2, ret_fast_syscall_trace
910		− and x2, x1, #_TIF_WORK_MASK
911		− cbnz x2, work_pending
912		− enable_step_tsk x1, x2
913		− kernel_exit 0
914		−ret_fast_syscall_trace:
915		− enable_daif
916		− b __sys_trace_return_skipped // we already saved x0
917		−
918	884	/*
919	885	* Ok, we need to do extra processing, enter the slow path.
920	886	*/
@@ -936,6 +902,9 @@ ret_to_user: · 15 unmodified lines
936	902	cbnz x2, work_pending
937	903	finish_ret_to_user:
938	904	enable_step_tsk x1, x2
	905	+#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
	906	+ bl stackleak_erase
	907	+#endif
939	908	kernel_exit 0
940	909	ENDPROC(ret_to_user)
941	910
@@ -944,85 +913,10 @@ ENDPROC(ret_to_user) · 2 unmodified lines
944	913	*/
945	914	.align 6
946	915	el0_svc:
947		− ldr x16, [tsk, #TSK_TI_FLAGS] // load thread flags
948		− adrp stbl, sys_call_table // load syscall table pointer
949		− mov wscno, w8 // syscall number in w8
950		− mov wsc_nr, #__NR_syscalls
951		−
952		−#ifdef CONFIG_ARM64_SVE
953		−alternative_if_not ARM64_SVE
954		− b el0_svc_naked
955		−alternative_else_nop_endif
956		− tbz x16, #TIF_SVE, el0_svc_naked // Skip unless TIF_SVE set:
957		− bic x16, x16, #_TIF_SVE // discard SVE state
958		− str x16, [tsk, #TSK_TI_FLAGS]
959		−
960		− /*
961		− * task_fpsimd_load() won't be called to update CPACR_EL1 in
962		− * ret_to_user unless TIF_FOREIGN_FPSTATE is still set, which only
963		− * happens if a context switch or kernel_neon_begin() or context
964		− * modification (sigreturn, ptrace) intervenes.
965		− * So, ensure that CPACR_EL1 is already correct for the fast-path case:
966		− */
967		− mrs x9, cpacr_el1
968		− bic x9, x9, #CPACR_EL1_ZEN_EL0EN // disable SVE for el0
969		− msr cpacr_el1, x9 // synchronised by eret to el0
970		−#endif
971		−
972		−el0_svc_naked: // compat entry point
973		− stp x0, xscno, [sp, #S_ORIG_X0] // save the original x0 and syscall number
974		− enable_daif
975		− ct_user_exit 1
976		−
977		− tst x16, #_TIF_SYSCALL_WORK // check for syscall hooks
978		− b.ne __sys_trace
979		− cmp wscno, wsc_nr // check upper syscall limit
980		− b.hs ni_sys
981		− mask_nospec64 xscno, xsc_nr, x19 // enforce bounds for syscall number
982		− ldr x16, [stbl, xscno, lsl #3] // address in the syscall table
983		− blr x16 // call sys_* routine
984		− b ret_fast_syscall
985		−ni_sys:
986		− mov x0, sp
987		− bl do_ni_syscall
988		− b ret_fast_syscall
989		−ENDPROC(el0_svc)
990		−
991		− /*
992		− * This is the really slow path. We're going to be doing context
993		− * switches, and waiting for our parent to respond.
994		− */
995		−__sys_trace:
996		− cmp wscno, #NO_SYSCALL // user-issued syscall(-1)?
997		− b.ne 1f
998		− mov x0, #-ENOSYS // set default errno if so
999		− str x0, [sp, #S_X0]
1000		−1: mov x0, sp
1001		− bl syscall_trace_enter
1002		− cmp w0, #NO_SYSCALL // skip the syscall?
1003		− b.eq __sys_trace_return_skipped
1004		− mov wscno, w0 // syscall number (possibly new)
1005		− mov x1, sp // pointer to regs
1006		− cmp wscno, wsc_nr // check upper syscall limit
1007		− b.hs __ni_sys_trace
1008		− ldp x0, x1, [sp] // restore the syscall args
1009		− ldp x2, x3, [sp, #S_X2]
1010		− ldp x4, x5, [sp, #S_X4]
1011		− ldp x6, x7, [sp, #S_X6]
1012		− ldr x16, [stbl, xscno, lsl #3] // address in the syscall table
1013		− blr x16 // call sys_* routine
1014		−
1015		−__sys_trace_return:
1016		− str x0, [sp, #S_X0] // save returned x0
1017		−__sys_trace_return_skipped:
1018	916	mov x0, sp
1019		− bl syscall_trace_exit
	917	+ bl el0_svc_handler
1020	918	b ret_to_user
1021		−
1022		−__ni_sys_trace:
1023		− mov x0, sp
1024		− bl do_ni_syscall
1025		− b __sys_trace_return
	919	+ENDPROC(el0_svc)
1026	920
1027	921	.popsection // .entry.text
1028	922
@@ -1137,14 +1031,6 @@ __entry_tramp_data_start: · 108 unmodified lines
1137	1031	#endif /* CONFIG_RANDOMIZE_BASE */
1138	1032	#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
1139	1033
1140		−/*
1141		− * Special system call wrappers.
1142		− */
1143		−ENTRY(sys_rt_sigreturn_wrapper)
1144		− mov x0, sp
1145		− b sys_rt_sigreturn
1146		−ENDPROC(sys_rt_sigreturn_wrapper)
1147		−
1148	1034	/*
1149	1035	* Register switch for AArch64. The callee-saved registers need to be saved
1150	1036	* and restored. On entry:

arch/arm64/kernel/entry32.S+0−121

@@ -1,121 +0,0 @@
1		−/*
2		− * Compat system call wrappers
3		− *
4		− * Copyright (C) 2012 ARM Ltd.
5		− * Authors: Will Deacon <will.deacon@arm.com>
6		− * Catalin Marinas <catalin.marinas@arm.com>
7		− *
8		− * This program is free software; you can redistribute it and/or modify
9		− * it under the terms of the GNU General Public License version 2 as
10		− * published by the Free Software Foundation.
11		− *
12		− * This program is distributed in the hope that it will be useful,
13		− * but WITHOUT ANY WARRANTY; without even the implied warranty of
14		− * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15		− * GNU General Public License for more details.
16		− *
17		− * You should have received a copy of the GNU General Public License
18		− * along with this program. If not, see <http://www.gnu.org/licenses/>.
19		− */
20		−
21		−#include <linux/linkage.h>
22		−#include <linux/const.h>
23		−
24		−#include <asm/assembler.h>
25		−#include <asm/asm-offsets.h>
26		−#include <asm/errno.h>
27		−#include <asm/page.h>
28		−
29		−/*
30		− * System call wrappers for the AArch32 compatibility layer.
31		− */
32		−
33		−ENTRY(compat_sys_sigreturn_wrapper)
34		− mov x0, sp
35		− b compat_sys_sigreturn
36		−ENDPROC(compat_sys_sigreturn_wrapper)
37		−
38		−ENTRY(compat_sys_rt_sigreturn_wrapper)
39		− mov x0, sp
40		− b compat_sys_rt_sigreturn
41		−ENDPROC(compat_sys_rt_sigreturn_wrapper)
42		−
43		−ENTRY(compat_sys_statfs64_wrapper)
44		− mov w3, #84
45		− cmp w1, #88
46		− csel w1, w3, w1, eq
47		− b compat_sys_statfs64
48		−ENDPROC(compat_sys_statfs64_wrapper)
49		−
50		−ENTRY(compat_sys_fstatfs64_wrapper)
51		− mov w3, #84
52		− cmp w1, #88
53		− csel w1, w3, w1, eq
54		− b compat_sys_fstatfs64
55		−ENDPROC(compat_sys_fstatfs64_wrapper)
56		−
57		−/*
58		− * Note: off_4k (w5) is always in units of 4K. If we can't do the
59		− * requested offset because it is not page-aligned, we return -EINVAL.
60		− */
61		−ENTRY(compat_sys_mmap2_wrapper)
62		−#if PAGE_SHIFT > 12
63		− tst w5, #~PAGE_MASK >> 12
64		− b.ne 1f
65		− lsr w5, w5, #PAGE_SHIFT - 12
66		−#endif
67		− b sys_mmap_pgoff
68		−1: mov x0, #-EINVAL
69		− ret
70		−ENDPROC(compat_sys_mmap2_wrapper)
71		−
72		−/*
73		− * Wrappers for AArch32 syscalls that either take 64-bit parameters
74		− * in registers or that take 32-bit parameters which require sign
75		− * extension.
76		− */
77		−ENTRY(compat_sys_pread64_wrapper)
78		− regs_to_64 x3, x4, x5
79		− b sys_pread64
80		−ENDPROC(compat_sys_pread64_wrapper)
81		−
82		−ENTRY(compat_sys_pwrite64_wrapper)
83		− regs_to_64 x3, x4, x5
84		− b sys_pwrite64
85		−ENDPROC(compat_sys_pwrite64_wrapper)
86		−
87		−ENTRY(compat_sys_truncate64_wrapper)
88		− regs_to_64 x1, x2, x3
89		− b sys_truncate
90		−ENDPROC(compat_sys_truncate64_wrapper)
91		−
92		−ENTRY(compat_sys_ftruncate64_wrapper)
93		− regs_to_64 x1, x2, x3
94		− b sys_ftruncate
95		−ENDPROC(compat_sys_ftruncate64_wrapper)
96		−
97		−ENTRY(compat_sys_readahead_wrapper)
98		− regs_to_64 x1, x2, x3
99		− mov w2, w4
100		− b sys_readahead
101		−ENDPROC(compat_sys_readahead_wrapper)
102		−
103		−ENTRY(compat_sys_fadvise64_64_wrapper)
104		− mov w6, w1
105		− regs_to_64 x1, x2, x3
106		− regs_to_64 x2, x4, x5
107		− mov w3, w6
108		− b sys_fadvise64_64
109		−ENDPROC(compat_sys_fadvise64_64_wrapper)
110		−
111		−ENTRY(compat_sys_sync_file_range2_wrapper)
112		− regs_to_64 x2, x2, x3
113		− regs_to_64 x3, x4, x5
114		− b sys_sync_file_range2
115		−ENDPROC(compat_sys_sync_file_range2_wrapper)
116		−
117		−ENTRY(compat_sys_fallocate_wrapper)
118		− regs_to_64 x2, x2, x3
119		− regs_to_64 x3, x4, x5
120		− b sys_fallocate
121		−ENDPROC(compat_sys_fallocate_wrapper)

arch/arm64/kernel/fpsimd.c+0−19

@@ -159,25 +159,6 @@ static void sve_free(struct task_struct *task) · 158 unmodified lines
159	159	__sve_free(task);
160	160	}
161	161
162		−static void change_cpacr(u64 val, u64 mask)
163		−{
164		− u64 cpacr = read_sysreg(CPACR_EL1);
165		− u64 new = (cpacr & ~mask) \| val;
166		−
167		− if (new != cpacr)
168		− write_sysreg(new, CPACR_EL1);
169		−}
170		−
171		−static void sve_user_disable(void)
172		−{
173		− change_cpacr(0, CPACR_EL1_ZEN_EL0EN);
174		−}
175		−
176		−static void sve_user_enable(void)
177		−{
178		− change_cpacr(CPACR_EL1_ZEN_EL0EN, CPACR_EL1_ZEN_EL0EN);
179		−}
180		−
181	162	/*
182	163	* TIF_SVE controls whether a task can use SVE without trapping while
183	164	* in userspace, and also the way a task's FPSIMD/SVE state is stored

arch/arm64/kernel/insn.c+3−67

@@ -149,20 +149,6 @@ int __kprobes aarch64_insn_write(void *addr, u32 insn) · 148 unmodified lines
149	149	return __aarch64_insn_write(addr, cpu_to_le32(insn));
150	150	}
151	151
152		−static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn)
153		−{
154		− if (aarch64_get_insn_class(insn) != AARCH64_INSN_CLS_BR_SYS)
155		− return false;
156		−
157		− return aarch64_insn_is_b(insn) \|\|
158		− aarch64_insn_is_bl(insn) \|\|
159		− aarch64_insn_is_svc(insn) \|\|
160		− aarch64_insn_is_hvc(insn) \|\|
161		− aarch64_insn_is_smc(insn) \|\|
162		− aarch64_insn_is_brk(insn) \|\|
163		− aarch64_insn_is_nop(insn);
164		−}
165		−
166	152	bool __kprobes aarch64_insn_uses_literal(u32 insn)
167	153	{
168	154	/* ldr/ldrsw (literal), prfm */
@@ -189,22 +175,6 @@ bool __kprobes aarch64_insn_is_branch(u32 insn) · 20 unmodified lines
189	175	aarch64_insn_is_bcond(insn);
190	176	}
191	177
192		−/*
193		− * ARM Architecture Reference Manual for ARMv8 Profile-A, Issue A.a
194		− * Section B2.6.5 "Concurrent modification and execution of instructions":
195		− * Concurrent modification and execution of instructions can lead to the
196		− * resulting instruction performing any behavior that can be achieved by
197		− * executing any sequence of instructions that can be executed from the
198		− * same Exception level, except where the instruction before modification
199		− * and the instruction after modification is a B, BL, NOP, BKPT, SVC, HVC,
200		− * or SMC instruction.
201		− */
202		−bool __kprobes aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn)
203		−{
204		− return __aarch64_insn_hotpatch_safe(old_insn) &&
205		− __aarch64_insn_hotpatch_safe(new_insn);
206		−}
207		−
208	178	int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn)
209	179	{
210	180	u32 *tp = addr;
@@ -216,8 +186,8 @@ int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn) · 5 unmodified lines
216	186
217	187	ret = aarch64_insn_write(tp, insn);
218	188	if (ret == 0)
219		− flush_icache_range((uintptr_t)tp,
220		− (uintptr_t)tp + AARCH64_INSN_SIZE);
	189	+ __flush_icache_range((uintptr_t)tp,
	190	+ (uintptr_t)tp + AARCH64_INSN_SIZE);
221	191
222	192	return ret;
223	193	}
@@ -239,11 +209,6 @@ static int __kprobes aarch64_insn_patch_text_cb(void *arg) · 15 unmodified lines
239	209	for (i = 0; ret == 0 && i < pp->insn_cnt; i++)
240	210	ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i],
241	211	pp->new_insns[i]);
242		− /*
243		− * aarch64_insn_patch_text_nosync() calls flush_icache_range(),
244		− * which ends with "dsb; isb" pair guaranteeing global
245		− * visibility.
246		− */
247	212	/* Notify other processors with an additional increment. */
248	213	atomic_inc(&pp->cpu_count);
249	214	} else {
@@ -255,8 +220,7 @@ static int __kprobes aarch64_insn_patch_text_cb(void *arg) · 5 unmodified lines
255	220	return ret;
256	221	}
257	222
258		−static
259		−int __kprobes aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt)
	223	+int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
260	224	{
261	225	struct aarch64_insn_patch patch = {
262	226	.text_addrs = addrs,
@@ -272,34 +236,6 @@ int __kprobes aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt) · 9 unmodified lines
272	236	cpu_online_mask);
273	237	}
274	238
275		−int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt)
276		−{
277		− int ret;
278		− u32 insn;
279		−
280		− /* Unsafe to patch multiple instructions without synchronizaiton */
281		− if (cnt == 1) {
282		− ret = aarch64_insn_read(addrs[0], &insn);
283		− if (ret)
284		− return ret;
285		−
286		− if (aarch64_insn_hotpatch_safe(insn, insns[0])) {
287		− /*
288		− * ARMv8 architecture doesn't guarantee all CPUs see
289		− * the new instruction after returning from function
290		− * aarch64_insn_patch_text_nosync(). So send IPIs to
291		− * all other CPUs to achieve instruction
292		− * synchronization.
293		− */
294		− ret = aarch64_insn_patch_text_nosync(addrs[0], insns[0]);
295		− kick_all_cpus_sync();
296		− return ret;
297		− }
298		− }
299		−
300		− return aarch64_insn_patch_text_sync(addrs, insns, cnt);
301		−}
302		−
303	239	static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,
304	240	u32 maskp, int shiftp)
305	241	{

arch/arm64/kernel/machine_kexec.c+11−4

@@ -184,8 +184,15 @@ void machine_kexec(struct kimage *kimage) · 183 unmodified lines
184	184
185	185	/* Flush the reboot_code_buffer in preparation for its execution. */
186	186	__flush_dcache_area(reboot_code_buffer, arm64_relocate_new_kernel_size);
187		− flush_icache_range((uintptr_t)reboot_code_buffer,
188		− arm64_relocate_new_kernel_size);
	187	+
	188	+ /*
	189	+ * Although we've killed off the secondary CPUs, we don't update
	190	+ * the online mask if we're handling a crash kernel and consequently
	191	+ * need to avoid flush_icache_range(), which will attempt to IPI
	192	+ * the offline CPUs. Therefore, we must use the __* variant here.
	193	+ */
	194	+ __flush_icache_range((uintptr_t)reboot_code_buffer,
	195	+ arm64_relocate_new_kernel_size);
189	196
190	197	/* Flush the kimage list and its buffers. */
191	198	kexec_list_flush(kimage);
@@ -207,8 +214,7 @@ void machine_kexec(struct kimage *kimage) · 15 unmodified lines
207	214	* relocation is complete.
208	215	*/
209	216
210		− cpu_soft_restart(kimage != kexec_crash_image,
211		− reboot_code_buffer_phys, kimage->head, kimage->start, 0);
	217	+ cpu_soft_restart(reboot_code_buffer_phys, kimage->head, kimage->start, 0);
212	218
213	219	BUG(); /* Should never get here. */
214	220	}
@@ -361,4 +367,5 @@ void arch_crash_save_vmcoreinfo(void) · 146 unmodified lines
361	367	kimage_voffset);
362	368	vmcoreinfo_append_str("NUMBER(PHYS_OFFSET)=0x%llx\n",
363	369	PHYS_OFFSET);
	370	+ vmcoreinfo_append_str("KERNELOFFSET=%lx\n", kaslr_offset());
364	371	}

arch/arm64/kernel/perf_event.c+231−50

@@ -25,6 +25,7 @@ · 24 unmodified lines
25	25	#include <asm/virt.h>
26	26
27	27	#include <linux/acpi.h>
	28	+#include <linux/clocksource.h>
28	29	#include <linux/of.h>
29	30	#include <linux/perf/arm_pmu.h>
30	31	#include <linux/platform_device.h>
@@ -446,9 +447,16 @@ static struct attribute_group armv8_pmuv3_events_attr_group = { · 415 unmodified lines
446	447	};
447	448
448	449	PMU_FORMAT_ATTR(event, "config:0-15");
	450	+PMU_FORMAT_ATTR(long, "config1:0");
	451	+
	452	+static inline bool armv8pmu_event_is_64bit(struct perf_event *event)
	453	+{
	454	+ return event->attr.config1 & 0x1;
	455	+}
449	456
450	457	static struct attribute *armv8_pmuv3_format_attrs[] = {
451	458	&format_attr_event.attr,
	459	+ &format_attr_long.attr,
452	460	NULL,
453	461	};
454	462
@@ -465,6 +473,21 @@ static struct attribute_group armv8_pmuv3_format_attr_group = { · 10 unmodified lines
465	473	#define ARMV8_IDX_COUNTER_LAST(cpu_pmu) \
466	474	(ARMV8_IDX_CYCLE_COUNTER + cpu_pmu->num_events - 1)
467	475
	476	+/*
	477	+ * We must chain two programmable counters for 64 bit events,
	478	+ * except when we have allocated the 64bit cycle counter (for CPU
	479	+ * cycles event). This must be called only when the event has
	480	+ * a counter allocated.
	481	+ */
	482	+static inline bool armv8pmu_event_is_chained(struct perf_event *event)
	483	+{
	484	+ int idx = event->hw.idx;
	485	+
	486	+ return !WARN_ON(idx < 0) &&
	487	+ armv8pmu_event_is_64bit(event) &&
	488	+ (idx != ARMV8_IDX_CYCLE_COUNTER);
	489	+}
	490	+
468	491	/*
469	492	* ARMv8 low level PMU access
470	493	*/
@@ -503,34 +526,68 @@ static inline int armv8pmu_counter_has_overflowed(u32 pmnc, int idx) · 32 unmodified lines
503	526	return pmnc & BIT(ARMV8_IDX_TO_COUNTER(idx));
504	527	}
505	528
506		−static inline int armv8pmu_select_counter(int idx)
	529	+static inline void armv8pmu_select_counter(int idx)
507	530	{
508	531	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
509	532	write_sysreg(counter, pmselr_el0);
510	533	isb();
	534	+}
511	535
512		− return idx;
	536	+static inline u32 armv8pmu_read_evcntr(int idx)
	537	+{
	538	+ armv8pmu_select_counter(idx);
	539	+ return read_sysreg(pmxevcntr_el0);
	540	+}
	541	+
	542	+static inline u64 armv8pmu_read_hw_counter(struct perf_event *event)
	543	+{
	544	+ int idx = event->hw.idx;
	545	+ u64 val = 0;
	546	+
	547	+ val = armv8pmu_read_evcntr(idx);
	548	+ if (armv8pmu_event_is_chained(event))
	549	+ val = (val << 32) \| armv8pmu_read_evcntr(idx - 1);
	550	+ return val;
513	551	}
514	552
515		−static inline u32 armv8pmu_read_counter(struct perf_event *event)
	553	+static inline u64 armv8pmu_read_counter(struct perf_event *event)
516	554	{
517	555	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
518	556	struct hw_perf_event *hwc = &event->hw;
519	557	int idx = hwc->idx;
520		− u32 value = 0;
	558	+ u64 value = 0;
521	559
522	560	if (!armv8pmu_counter_valid(cpu_pmu, idx))
523	561	pr_err("CPU%u reading wrong counter %d\n",
524	562	smp_processor_id(), idx);
525	563	else if (idx == ARMV8_IDX_CYCLE_COUNTER)
526	564	value = read_sysreg(pmccntr_el0);
527		− else if (armv8pmu_select_counter(idx) == idx)
528		− value = read_sysreg(pmxevcntr_el0);
	565	+ else
	566	+ value = armv8pmu_read_hw_counter(event);
529	567
530	568	return value;
531	569	}
532	570
533		−static inline void armv8pmu_write_counter(struct perf_event *event, u32 value)
	571	+static inline void armv8pmu_write_evcntr(int idx, u32 value)
	572	+{
	573	+ armv8pmu_select_counter(idx);
	574	+ write_sysreg(value, pmxevcntr_el0);
	575	+}
	576	+
	577	+static inline void armv8pmu_write_hw_counter(struct perf_event *event,
	578	+ u64 value)
	579	+{
	580	+ int idx = event->hw.idx;
	581	+
	582	+ if (armv8pmu_event_is_chained(event)) {
	583	+ armv8pmu_write_evcntr(idx, upper_32_bits(value));
	584	+ armv8pmu_write_evcntr(idx - 1, lower_32_bits(value));
	585	+ } else {
	586	+ armv8pmu_write_evcntr(idx, value);
	587	+ }
	588	+}
	589	+
	590	+static inline void armv8pmu_write_counter(struct perf_event *event, u64 value)
534	591	{
535	592	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
536	593	struct hw_perf_event *hwc = &event->hw;
@@ -541,22 +598,43 @@ static inline void armv8pmu_write_counter(struct perf_event *event, u32 value) · 4 unmodified lines
541	598	smp_processor_id(), idx);
542	599	else if (idx == ARMV8_IDX_CYCLE_COUNTER) {
543	600	/*
544		− * Set the upper 32bits as this is a 64bit counter but we only
545		− * count using the lower 32bits and we want an interrupt when
546		− * it overflows.
	601	+ * The cycles counter is really a 64-bit counter.
	602	+ * When treating it as a 32-bit counter, we only count
	603	+ * the lower 32 bits, and set the upper 32-bits so that
	604	+ * we get an interrupt upon 32-bit overflow.
547	605	*/
548		− u64 value64 = 0xffffffff00000000ULL \| value;
549		−
550		− write_sysreg(value64, pmccntr_el0);
551		− } else if (armv8pmu_select_counter(idx) == idx)
552		− write_sysreg(value, pmxevcntr_el0);
	606	+ if (!armv8pmu_event_is_64bit(event))
	607	+ value \|= 0xffffffff00000000ULL;
	608	+ write_sysreg(value, pmccntr_el0);
	609	+ } else
	610	+ armv8pmu_write_hw_counter(event, value);
553	611	}
554	612
555	613	static inline void armv8pmu_write_evtype(int idx, u32 val)
556	614	{
557		− if (armv8pmu_select_counter(idx) == idx) {
558		− val &= ARMV8_PMU_EVTYPE_MASK;
559		− write_sysreg(val, pmxevtyper_el0);
	615	+ armv8pmu_select_counter(idx);
	616	+ val &= ARMV8_PMU_EVTYPE_MASK;
	617	+ write_sysreg(val, pmxevtyper_el0);
	618	+}
	619	+
	620	+static inline void armv8pmu_write_event_type(struct perf_event *event)
	621	+{
	622	+ struct hw_perf_event *hwc = &event->hw;
	623	+ int idx = hwc->idx;
	624	+
	625	+ /*
	626	+ * For chained events, the low counter is programmed to count
	627	+ * the event of interest and the high counter is programmed
	628	+ * with CHAIN event code with filters set to count at all ELs.
	629	+ */
	630	+ if (armv8pmu_event_is_chained(event)) {
	631	+ u32 chain_evt = ARMV8_PMUV3_PERFCTR_CHAIN \|
	632	+ ARMV8_PMU_INCLUDE_EL2;
	633	+
	634	+ armv8pmu_write_evtype(idx - 1, hwc->config_base);
	635	+ armv8pmu_write_evtype(idx, chain_evt);
	636	+ } else {
	637	+ armv8pmu_write_evtype(idx, hwc->config_base);
560	638	}
561	639	}
562	640
@@ -567,6 +645,16 @@ static inline int armv8pmu_enable_counter(int idx) · 4 unmodified lines
567	645	return idx;
568	646	}
569	647
	648	+static inline void armv8pmu_enable_event_counter(struct perf_event *event)
	649	+{
	650	+ int idx = event->hw.idx;
	651	+
	652	+ armv8pmu_enable_counter(idx);
	653	+ if (armv8pmu_event_is_chained(event))
	654	+ armv8pmu_enable_counter(idx - 1);
	655	+ isb();
	656	+}
	657	+
570	658	static inline int armv8pmu_disable_counter(int idx)
571	659	{
572	660	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
@@ -574,6 +662,16 @@ static inline int armv8pmu_disable_counter(int idx) · 1 unmodified line
574	662	return idx;
575	663	}
576	664
	665	+static inline void armv8pmu_disable_event_counter(struct perf_event *event)
	666	+{
	667	+ struct hw_perf_event *hwc = &event->hw;
	668	+ int idx = hwc->idx;
	669	+
	670	+ if (armv8pmu_event_is_chained(event))
	671	+ armv8pmu_disable_counter(idx - 1);
	672	+ armv8pmu_disable_counter(idx);
	673	+}
	674	+
577	675	static inline int armv8pmu_enable_intens(int idx)
578	676	{
579	677	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
@@ -581,6 +679,11 @@ static inline int armv8pmu_enable_intens(int idx) · 1 unmodified line
581	679	return idx;
582	680	}
583	681
	682	+static inline int armv8pmu_enable_event_irq(struct perf_event *event)
	683	+{
	684	+ return armv8pmu_enable_intens(event->hw.idx);
	685	+}
	686	+
584	687	static inline int armv8pmu_disable_intens(int idx)
585	688	{
586	689	u32 counter = ARMV8_IDX_TO_COUNTER(idx);
@@ -593,6 +696,11 @@ static inline int armv8pmu_disable_intens(int idx) · 6 unmodified lines
593	696	return idx;
594	697	}
595	698
	699	+static inline int armv8pmu_disable_event_irq(struct perf_event *event)
	700	+{
	701	+ return armv8pmu_disable_intens(event->hw.idx);
	702	+}
	703	+
596	704	static inline u32 armv8pmu_getreset_flags(void)
597	705	{
598	706	u32 value;
@@ -610,10 +718,8 @@ static inline u32 armv8pmu_getreset_flags(void) · 11 unmodified lines
610	718	static void armv8pmu_enable_event(struct perf_event *event)
611	719	{
612	720	unsigned long flags;
613		− struct hw_perf_event *hwc = &event->hw;
614	721	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
615	722	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
616		− int idx = hwc->idx;
617	723
618	724	/*
619	725	* Enable counter and interrupt, and set the counter to count
@@ -624,22 +730,22 @@ static void armv8pmu_enable_event(struct perf_event *event) · 4 unmodified lines
624	730	/*
625	731	* Disable counter
626	732	*/
627		− armv8pmu_disable_counter(idx);
	733	+ armv8pmu_disable_event_counter(event);
628	734
629	735	/*
630	736	* Set event (if destined for PMNx counters).
631	737	*/
632		− armv8pmu_write_evtype(idx, hwc->config_base);
	738	+ armv8pmu_write_event_type(event);
633	739
634	740	/*
635	741	* Enable interrupt for this counter
636	742	*/
637		− armv8pmu_enable_intens(idx);
	743	+ armv8pmu_enable_event_irq(event);
638	744
639	745	/*
640	746	* Enable counter
641	747	*/
642		− armv8pmu_enable_counter(idx);
	748	+ armv8pmu_enable_event_counter(event);
643	749
644	750	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
645	751	}
@@ -647,10 +753,8 @@ static void armv8pmu_enable_event(struct perf_event *event) · 1 unmodified line
647	753	static void armv8pmu_disable_event(struct perf_event *event)
648	754	{
649	755	unsigned long flags;
650		− struct hw_perf_event *hwc = &event->hw;
651	756	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
652	757	struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
653		− int idx = hwc->idx;
654	758
655	759	/*
656	760	* Disable counter and interrupt
@@ -660,16 +764,38 @@ static void armv8pmu_disable_event(struct perf_event *event) · 3 unmodified lines
660	764	/*
661	765	* Disable counter
662	766	*/
663		− armv8pmu_disable_counter(idx);
	767	+ armv8pmu_disable_event_counter(event);
664	768
665	769	/*
666	770	* Disable interrupt for this counter
667	771	*/
668		− armv8pmu_disable_intens(idx);
	772	+ armv8pmu_disable_event_irq(event);
669	773
670	774	raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
671	775	}
672	776
	777	+static void armv8pmu_start(struct arm_pmu *cpu_pmu)
	778	+{
	779	+ unsigned long flags;
	780	+ struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
	781	+
	782	+ raw_spin_lock_irqsave(&events->pmu_lock, flags);
	783	+ /* Enable all counters */
	784	+ armv8pmu_pmcr_write(armv8pmu_pmcr_read() \| ARMV8_PMU_PMCR_E);
	785	+ raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
	786	+}
	787	+
	788	+static void armv8pmu_stop(struct arm_pmu *cpu_pmu)
	789	+{
	790	+ unsigned long flags;
	791	+ struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
	792	+
	793	+ raw_spin_lock_irqsave(&events->pmu_lock, flags);
	794	+ /* Disable all counters */
	795	+ armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E);
	796	+ raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
	797	+}
	798	+
673	799	static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu)
674	800	{
675	801	u32 pmovsr;
@@ -694,6 +820,11 @@ static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu) · 18 unmodified lines
694	820	*/
695	821	regs = get_irq_regs();
696	822
	823	+ /*
	824	+ * Stop the PMU while processing the counter overflows
	825	+ * to prevent skews in group events.
	826	+ */
	827	+ armv8pmu_stop(cpu_pmu);
697	828	for (idx = 0; idx < cpu_pmu->num_events; ++idx) {
698	829	struct perf_event *event = cpuc->events[idx];
699	830	struct hw_perf_event *hwc;
@@ -718,6 +849,7 @@ static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu) · 18 unmodified lines
718	849	if (perf_event_overflow(event, &data, regs))
719	850	cpu_pmu->disable(event);
720	851	}
	852	+ armv8pmu_start(cpu_pmu);
721	853
722	854	/*
723	855	* Handle the pending perf events.
@@ -731,32 +863,42 @@ static irqreturn_t armv8pmu_handle_irq(struct arm_pmu *cpu_pmu) · 7 unmodified lines
731	863	return IRQ_HANDLED;
732	864	}
733	865
734		−static void armv8pmu_start(struct arm_pmu *cpu_pmu)
	866	+static int armv8pmu_get_single_idx(struct pmu_hw_events *cpuc,
	867	+ struct arm_pmu *cpu_pmu)
735	868	{
736		− unsigned long flags;
737		− struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
	869	+ int idx;
738	870
739		− raw_spin_lock_irqsave(&events->pmu_lock, flags);
740		− /* Enable all counters */
741		− armv8pmu_pmcr_write(armv8pmu_pmcr_read() \| ARMV8_PMU_PMCR_E);
742		− raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
	871	+ for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; idx ++) {
	872	+ if (!test_and_set_bit(idx, cpuc->used_mask))
	873	+ return idx;
	874	+ }
	875	+ return -EAGAIN;
743	876	}
744	877
745		−static void armv8pmu_stop(struct arm_pmu *cpu_pmu)
	878	+static int armv8pmu_get_chain_idx(struct pmu_hw_events *cpuc,
	879	+ struct arm_pmu *cpu_pmu)
746	880	{
747		− unsigned long flags;
748		− struct pmu_hw_events *events = this_cpu_ptr(cpu_pmu->hw_events);
	881	+ int idx;
749	882
750		− raw_spin_lock_irqsave(&events->pmu_lock, flags);
751		− /* Disable all counters */
752		− armv8pmu_pmcr_write(armv8pmu_pmcr_read() & ~ARMV8_PMU_PMCR_E);
753		− raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
	883	+ /*
	884	+ * Chaining requires two consecutive event counters, where
	885	+ * the lower idx must be even.
	886	+ */
	887	+ for (idx = ARMV8_IDX_COUNTER0 + 1; idx < cpu_pmu->num_events; idx += 2) {
	888	+ if (!test_and_set_bit(idx, cpuc->used_mask)) {
	889	+ /* Check if the preceding even counter is available */
	890	+ if (!test_and_set_bit(idx - 1, cpuc->used_mask))
	891	+ return idx;
	892	+ /* Release the Odd counter */
	893	+ clear_bit(idx, cpuc->used_mask);
	894	+ }
	895	+ }
	896	+ return -EAGAIN;
754	897	}
755	898
756	899	static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc,
757	900	struct perf_event *event)
758	901	{
759		− int idx;
760	902	struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu);
761	903	struct hw_perf_event *hwc = &event->hw;
762	904	unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT;
@@ -770,13 +912,20 @@ static int armv8pmu_get_event_idx(struct pmu_hw_events *cpuc, · 7 unmodified lines
770	912	/*
771	913	* Otherwise use events counters
772	914	*/
773		− for (idx = ARMV8_IDX_COUNTER0; idx < cpu_pmu->num_events; ++idx) {
774		− if (!test_and_set_bit(idx, cpuc->used_mask))
775		− return idx;
776		− }
	915	+ if (armv8pmu_event_is_64bit(event))
	916	+ return armv8pmu_get_chain_idx(cpuc, cpu_pmu);
	917	+ else
	918	+ return armv8pmu_get_single_idx(cpuc, cpu_pmu);
	919	+}
777	920
778		− /* The counters are all in use. */
779		− return -EAGAIN;
	921	+static void armv8pmu_clear_event_idx(struct pmu_hw_events *cpuc,
	922	+ struct perf_event *event)
	923	+{
	924	+ int idx = event->hw.idx;
	925	+
	926	+ clear_bit(idx, cpuc->used_mask);
	927	+ if (armv8pmu_event_is_chained(event))
	928	+ clear_bit(idx - 1, cpuc->used_mask);
780	929	}
781	930
782	931	/*
@@ -851,6 +1000,9 @@ static int __armv8_pmuv3_map_event(struct perf_event *event, · 68 unmodified lines
851	1000	&armv8_pmuv3_perf_cache_map,
852	1001	ARMV8_PMU_EVTYPE_EVENT);
853	1002
	1003	+ if (armv8pmu_event_is_64bit(event))
	1004	+ event->hw.flags \|= ARMPMU_EVT_64BIT;
	1005	+
854	1006	/* Onl expose micro/arch events supported by this PMU */
855	1007	if ((hw_event_id > 0) && (hw_event_id < ARMV8_PMUV3_MAX_COMMON_EVENTS)
856	1008	&& test_bit(hw_event_id, armpmu->pmceid_bitmap)) {
@@ -957,10 +1109,10 @@ static int armv8_pmu_init(struct arm_pmu *cpu_pmu) · 100 unmodified lines
957	1109	cpu_pmu->read_counter = armv8pmu_read_counter,
958	1110	cpu_pmu->write_counter = armv8pmu_write_counter,
959	1111	cpu_pmu->get_event_idx = armv8pmu_get_event_idx,
	1112	+ cpu_pmu->clear_event_idx = armv8pmu_clear_event_idx,
960	1113	cpu_pmu->start = armv8pmu_start,
961	1114	cpu_pmu->stop = armv8pmu_stop,
962	1115	cpu_pmu->reset = armv8pmu_reset,
963		− cpu_pmu->max_period = (1LLU << 32) - 1,
964	1116	cpu_pmu->set_event_filter = armv8pmu_set_event_filter;
965	1117
966	1118	return 0;
@@ -1127,3 +1279,32 @@ static int __init armv8_pmu_driver_init(void) · 160 unmodified lines
1127	1279	return arm_pmu_acpi_probe(armv8_pmuv3_init);
1128	1280	}
1129	1281	device_initcall(armv8_pmu_driver_init)
	1282	+
	1283	+void arch_perf_update_userpage(struct perf_event *event,
	1284	+ struct perf_event_mmap_page *userpg, u64 now)
	1285	+{
	1286	+ u32 freq;
	1287	+ u32 shift;
	1288	+
	1289	+ /*
	1290	+ * Internal timekeeping for enabled/running/stopped times
	1291	+ * is always computed with the sched_clock.
	1292	+ */
	1293	+ freq = arch_timer_get_rate();
	1294	+ userpg->cap_user_time = 1;
	1295	+
	1296	+ clocks_calc_mult_shift(&userpg->time_mult, &shift, freq,
	1297	+ NSEC_PER_SEC, 0);
	1298	+ /*
	1299	+ * time_shift is not expected to be greater than 31 due to
	1300	+ * the original published conversion algorithm shifting a
	1301	+ * 32-bit value (now specifies a 64-bit value) - refer
	1302	+ * perf_event_mmap_page documentation in perf_event.h.
	1303	+ */
	1304	+ if (shift == 32) {
	1305	+ shift = 31;
	1306	+ userpg->time_mult >>= 1;
	1307	+ }
	1308	+ userpg->time_shift = (u16)shift;
	1309	+ userpg->time_offset = -now;
	1310	+}

arch/arm64/kernel/process.c+32−10

@@ -177,16 +177,16 @@ static void print_pstate(struct pt_regs *regs) · 176 unmodified lines
177	177	if (compat_user_mode(regs)) {
178	178	printk("pstate: %08llx (%c%c%c%c %c %s %s %c%c%c)\n",
179	179	pstate,
180		− pstate & COMPAT_PSR_N_BIT ? 'N' : 'n',
181		− pstate & COMPAT_PSR_Z_BIT ? 'Z' : 'z',
182		− pstate & COMPAT_PSR_C_BIT ? 'C' : 'c',
183		− pstate & COMPAT_PSR_V_BIT ? 'V' : 'v',
184		− pstate & COMPAT_PSR_Q_BIT ? 'Q' : 'q',
185		− pstate & COMPAT_PSR_T_BIT ? "T32" : "A32",
186		− pstate & COMPAT_PSR_E_BIT ? "BE" : "LE",
187		− pstate & COMPAT_PSR_A_BIT ? 'A' : 'a',
188		− pstate & COMPAT_PSR_I_BIT ? 'I' : 'i',
189		− pstate & COMPAT_PSR_F_BIT ? 'F' : 'f');
	180	+ pstate & PSR_AA32_N_BIT ? 'N' : 'n',
	181	+ pstate & PSR_AA32_Z_BIT ? 'Z' : 'z',
	182	+ pstate & PSR_AA32_C_BIT ? 'C' : 'c',
	183	+ pstate & PSR_AA32_V_BIT ? 'V' : 'v',
	184	+ pstate & PSR_AA32_Q_BIT ? 'Q' : 'q',
	185	+ pstate & PSR_AA32_T_BIT ? "T32" : "A32",
	186	+ pstate & PSR_AA32_E_BIT ? "BE" : "LE",
	187	+ pstate & PSR_AA32_A_BIT ? 'A' : 'a',
	188	+ pstate & PSR_AA32_I_BIT ? 'I' : 'i',
	189	+ pstate & PSR_AA32_F_BIT ? 'F' : 'f');
190	190	} else {
191	191	printk("pstate: %08llx (%c%c%c%c %c%c%c%c %cPAN %cUAO)\n",
192	192	pstate,
@@ -493,3 +493,25 @@ void arch_setup_new_exec(void) · 300 unmodified lines
493	493	{
494	494	current->mm->context.flags = is_compat_task() ? MMCF_AARCH32 : 0;
495	495	}
	496	+
	497	+#ifdef CONFIG_GCC_PLUGIN_STACKLEAK
	498	+void __used stackleak_check_alloca(unsigned long size)
	499	+{
	500	+ unsigned long stack_left;
	501	+ unsigned long current_sp = current_stack_pointer;
	502	+ struct stack_info info;
	503	+
	504	+ BUG_ON(!on_accessible_stack(current, current_sp, &info));
	505	+
	506	+ stack_left = current_sp - info.low;
	507	+
	508	+ /*
	509	+ * There's a good chance we're almost out of stack space if this
	510	+ * is true. Using panic() over BUG() is more likely to give
	511	+ * reliable debugging output.
	512	+ */
	513	+ if (size >= stack_left)
	514	+ panic("alloca() over the kernel stack boundary\n");
	515	+}
	516	+EXPORT_SYMBOL(stackleak_check_alloca);
	517	+#endif

arch/arm64/kernel/ptrace.c+36−25

@@ -132,7 +132,7 @@ static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) · 131 unmodified lines
132	132	{
133	133	return ((addr & ~(THREAD_SIZE - 1)) ==
134	134	(kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) \|\|
135		− on_irq_stack(addr);
	135	+ on_irq_stack(addr, NULL);
136	136	}
137	137
138	138	/**
@@ -277,19 +277,22 @@ static int ptrace_hbp_set_event(unsigned int note_type, · 138 unmodified lines
277	277
278	278	switch (note_type) {
279	279	case NT_ARM_HW_BREAK:
280		− if (idx < ARM_MAX_BRP) {
281		− tsk->thread.debug.hbp_break[idx] = bp;
282		− err = 0;
283		− }
	280	+ if (idx >= ARM_MAX_BRP)
	281	+ goto out;
	282	+ idx = array_index_nospec(idx, ARM_MAX_BRP);
	283	+ tsk->thread.debug.hbp_break[idx] = bp;
	284	+ err = 0;
284	285	break;
285	286	case NT_ARM_HW_WATCH:
286		− if (idx < ARM_MAX_WRP) {
287		− tsk->thread.debug.hbp_watch[idx] = bp;
288		− err = 0;
289		− }
	287	+ if (idx >= ARM_MAX_WRP)
	288	+ goto out;
	289	+ idx = array_index_nospec(idx, ARM_MAX_WRP);
	290	+ tsk->thread.debug.hbp_watch[idx] = bp;
	291	+ err = 0;
290	292	break;
291	293	}
292	294
	295	+out:
293	296	return err;
294	297	}
295	298
@@ -1076,6 +1079,7 @@ static int compat_gpr_get(struct task_struct *target, · 780 unmodified lines
1076	1079	break;
1077	1080	case 16:
1078	1081	reg = task_pt_regs(target)->pstate;
	1082	+ reg = pstate_to_compat_psr(reg);
1079	1083	break;
1080	1084	case 17:
1081	1085	reg = task_pt_regs(target)->orig_x0;
@@ -1143,6 +1147,7 @@ static int compat_gpr_set(struct task_struct *target, · 61 unmodified lines
1143	1147	newregs.pc = reg;
1144	1148	break;
1145	1149	case 16:
	1150	+ reg = compat_psr_to_pstate(reg);
1146	1151	newregs.pstate = reg;
1147	1152	break;
1148	1153	case 17:
@@ -1629,7 +1634,7 @@ static void tracehook_report_syscall(struct pt_regs *regs, · 480 unmodified lines
1629	1634	regs->regs[regno] = saved_reg;
1630	1635	}
1631	1636
1632		−asmlinkage int syscall_trace_enter(struct pt_regs *regs)
	1637	+int syscall_trace_enter(struct pt_regs *regs)
1633	1638	{
1634	1639	if (test_thread_flag(TIF_SYSCALL_TRACE))
1635	1640	tracehook_report_syscall(regs, PTRACE_SYSCALL_ENTER);
@@ -1647,7 +1652,7 @@ asmlinkage int syscall_trace_enter(struct pt_regs *regs) · 11 unmodified lines
1647	1652	return regs->syscallno;
1648	1653	}
1649	1654
1650		−asmlinkage void syscall_trace_exit(struct pt_regs *regs)
	1655	+void syscall_trace_exit(struct pt_regs *regs)
1651	1656	{
1652	1657	audit_syscall_exit(regs);
1653	1658
@@ -1656,18 +1661,24 @@ asmlinkage void syscall_trace_exit(struct pt_regs *regs) · 2 unmodified lines
1656	1661
1657	1662	if (test_thread_flag(TIF_SYSCALL_TRACE))
1658	1663	tracehook_report_syscall(regs, PTRACE_SYSCALL_EXIT);
	1664	+
	1665	+ rseq_syscall(regs);
1659	1666	}
1660	1667
1661	1668	/*
1662		− * Bits which are always architecturally RES0 per ARM DDI 0487A.h
	1669	+ * SPSR_ELx bits which are always architecturally RES0 per ARM DDI 0487C.a
	1670	+ * We also take into account DIT (bit 24), which is not yet documented, and
	1671	+ * treat PAN and UAO as RES0 bits, as they are meaningless at EL0, and may be
	1672	+ * allocated an EL0 meaning in future.
1663	1673	* Userspace cannot use these until they have an architectural meaning.
	1674	+ * Note that this follows the SPSR_ELx format, not the AArch32 PSR format.
1664	1675	* We also reserve IL for the kernel; SS is handled dynamically.
1665	1676	*/
1666	1677	#define SPSR_EL1_AARCH64_RES0_BITS \
1667		− (GENMASK_ULL(63,32) \| GENMASK_ULL(27, 22) \| GENMASK_ULL(20, 10) \| \
1668		− GENMASK_ULL(5, 5))
	1678	+ (GENMASK_ULL(63,32) \| GENMASK_ULL(27, 25) \| GENMASK_ULL(23, 22) \| \
	1679	+ GENMASK_ULL(20, 10) \| GENMASK_ULL(5, 5))
1669	1680	#define SPSR_EL1_AARCH32_RES0_BITS \
1670		− (GENMASK_ULL(63,32) \| GENMASK_ULL(24, 22) \| GENMASK_ULL(20,20))
	1681	+ (GENMASK_ULL(63,32) \| GENMASK_ULL(23, 22) \| GENMASK_ULL(20,20))
1671	1682
1672	1683	static int valid_compat_regs(struct user_pt_regs *regs)
1673	1684	{
@@ -1675,15 +1686,15 @@ static int valid_compat_regs(struct user_pt_regs *regs) · 1 unmodified line
1675	1686
1676	1687	if (!system_supports_mixed_endian_el0()) {
1677	1688	if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
1678		− regs->pstate \|= COMPAT_PSR_E_BIT;
	1689	+ regs->pstate \|= PSR_AA32_E_BIT;
1679	1690	else
1680		− regs->pstate &= ~COMPAT_PSR_E_BIT;
	1691	+ regs->pstate &= ~PSR_AA32_E_BIT;
1681	1692	}
1682	1693
1683	1694	if (user_mode(regs) && (regs->pstate & PSR_MODE32_BIT) &&
1684		− (regs->pstate & COMPAT_PSR_A_BIT) == 0 &&
1685		− (regs->pstate & COMPAT_PSR_I_BIT) == 0 &&
1686		− (regs->pstate & COMPAT_PSR_F_BIT) == 0) {
	1695	+ (regs->pstate & PSR_AA32_A_BIT) == 0 &&
	1696	+ (regs->pstate & PSR_AA32_I_BIT) == 0 &&
	1697	+ (regs->pstate & PSR_AA32_F_BIT) == 0) {
1687	1698	return 1;
1688	1699	}
1689	1700
@@ -1691,11 +1702,11 @@ static int valid_compat_regs(struct user_pt_regs *regs) · 1 unmodified line
1691	1702	* Force PSR to a valid 32-bit EL0t, preserving the same bits as
1692	1703	* arch/arm.
1693	1704	*/
1694		− regs->pstate &= COMPAT_PSR_N_BIT \| COMPAT_PSR_Z_BIT \|
1695		− COMPAT_PSR_C_BIT \| COMPAT_PSR_V_BIT \|
1696		− COMPAT_PSR_Q_BIT \| COMPAT_PSR_IT_MASK \|
1697		− COMPAT_PSR_GE_MASK \| COMPAT_PSR_E_BIT \|
1698		− COMPAT_PSR_T_BIT;
	1705	+ regs->pstate &= PSR_AA32_N_BIT \| PSR_AA32_Z_BIT \|
	1706	+ PSR_AA32_C_BIT \| PSR_AA32_V_BIT \|
	1707	+ PSR_AA32_Q_BIT \| PSR_AA32_IT_MASK \|
	1708	+ PSR_AA32_GE_MASK \| PSR_AA32_E_BIT \|
	1709	+ PSR_AA32_T_BIT;
1699	1710	regs->pstate \|= PSR_MODE32_BIT;
1700	1711
1701	1712	return 0;

arch/arm64/kernel/sdei.c+39−9

@@ -13,6 +13,7 @@ · 12 unmodified lines
13	13	#include <asm/mmu.h>
14	14	#include <asm/ptrace.h>
15	15	#include <asm/sections.h>
	16	+#include <asm/stacktrace.h>
16	17	#include <asm/sysreg.h>
17	18	#include <asm/vmap_stack.h>
18	19
@@ -88,23 +89,52 @@ static int init_sdei_stacks(void) · 69 unmodified lines
88	89	return err;
89	90	}
90	91
91		−bool _on_sdei_stack(unsigned long sp)
	92	+static bool on_sdei_normal_stack(unsigned long sp, struct stack_info *info)
92	93	{
93		− unsigned long low, high;
	94	+ unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
	95	+ unsigned long high = low + SDEI_STACK_SIZE;
94	96
95		− if (!IS_ENABLED(CONFIG_VMAP_STACK))
	97	+ if (sp < low \|\| sp >= high)
96	98	return false;
97	99
98		− low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
99		− high = low + SDEI_STACK_SIZE;
	100	+ if (info) {
	101	+ info->low = low;
	102	+ info->high = high;
	103	+ info->type = STACK_TYPE_SDEI_NORMAL;
	104	+ }
100	105
101		− if (low <= sp && sp < high)
	106	+ return true;
	107	+}
	108	+
	109	+static bool on_sdei_critical_stack(unsigned long sp, struct stack_info *info)
	110	+{
	111	+ unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr);
	112	+ unsigned long high = low + SDEI_STACK_SIZE;
	113	+
	114	+ if (sp < low \|\| sp >= high)
	115	+ return false;
	116	+
	117	+ if (info) {
	118	+ info->low = low;
	119	+ info->high = high;
	120	+ info->type = STACK_TYPE_SDEI_CRITICAL;
	121	+ }
	122	+
	123	+ return true;
	124	+}
	125	+
	126	+bool _on_sdei_stack(unsigned long sp, struct stack_info *info)
	127	+{
	128	+ if (!IS_ENABLED(CONFIG_VMAP_STACK))
	129	+ return false;
	130	+
	131	+ if (on_sdei_critical_stack(sp, info))
102	132	return true;
103	133
104		− low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
105		− high = low + SDEI_STACK_SIZE;
	134	+ if (on_sdei_normal_stack(sp, info))
	135	+ return true;
106	136
107		− return (low <= sp && sp < high);
	137	+ return false;
108	138	}
109	139
110	140	unsigned long sdei_arch_get_entry_point(int conduit)

arch/arm64/kernel/setup.c+38−0

@@ -241,6 +241,44 @@ static void __init request_standard_resources(void) · 240 unmodified lines
241	241	}
242	242	}
243	243
	244	+static int __init reserve_memblock_reserved_regions(void)
	245	+{
	246	+ phys_addr_t start, end, roundup_end = 0;
	247	+ struct resource mem, res;
	248	+ u64 i;
	249	+
	250	+ for_each_reserved_mem_region(i, &start, &end) {
	251	+ if (end <= roundup_end)
	252	+ continue; /* done already */
	253	+
	254	+ start = __pfn_to_phys(PFN_DOWN(start));
	255	+ end = __pfn_to_phys(PFN_UP(end)) - 1;
	256	+ roundup_end = end;
	257	+
	258	+ res = kzalloc(sizeof(*res), GFP_ATOMIC);
	259	+ if (WARN_ON(!res))
	260	+ return -ENOMEM;
	261	+ res->start = start;
	262	+ res->end = end;
	263	+ res->name = "reserved";
	264	+ res->flags = IORESOURCE_MEM;
	265	+
	266	+ mem = request_resource_conflict(&iomem_resource, res);
	267	+ /*
	268	+ * We expected memblock_reserve() regions to conflict with
	269	+ * memory created by request_standard_resources().
	270	+ */
	271	+ if (WARN_ON_ONCE(!mem))
	272	+ continue;
	273	+ kfree(res);
	274	+
	275	+ reserve_region_with_split(mem, start, end, "reserved");
	276	+ }
	277	+
	278	+ return 0;
	279	+}
	280	+arch_initcall(reserve_memblock_reserved_regions);
	281	+
244	282	u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID };
245	283
246	284	void __init setup_arch(char **cmdline_p)

arch/arm64/kernel/signal.c+6−2

@@ -539,8 +539,9 @@ static int restore_sigframe(struct pt_regs *regs, · 538 unmodified lines
539	539	return err;
540	540	}
541	541
542		−asmlinkage long sys_rt_sigreturn(struct pt_regs *regs)
	542	+SYSCALL_DEFINE0(rt_sigreturn)
543	543	{
	544	+ struct pt_regs *regs = current_pt_regs();
544	545	struct rt_sigframe __user *frame;
545	546
546	547	/* Always make any pending restarted system calls return -EINTR */
@@ -802,6 +803,8 @@ static void handle_signal(struct ksignal ksig, struct pt_regs regs) · 255 unmodified lines
802	803	int usig = ksig->sig;
803	804	int ret;
804	805
	806	+ rseq_signal_deliver(ksig, regs);
	807	+
805	808	/*
806	809	* Set up the stack frame
807	810	*/
@@ -910,7 +913,7 @@ static void do_signal(struct pt_regs *regs) · 102 unmodified lines
910	913	}
911	914
912	915	asmlinkage void do_notify_resume(struct pt_regs *regs,
913		− unsigned int thread_flags)
	916	+ unsigned long thread_flags)
914	917	{
915	918	/*
916	919	* The assembly code enters us with IRQs off, but it hasn't
@@ -940,6 +943,7 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, · 23 unmodified lines
940	943	if (thread_flags & _TIF_NOTIFY_RESUME) {
941	944	clear_thread_flag(TIF_NOTIFY_RESUME);
942	945	tracehook_notify_resume(regs);
	946	+ rseq_handle_notify_resume(NULL, regs);
943	947	}
944	948
945	949	if (thread_flags & _TIF_FOREIGN_FPSTATE)

arch/arm64/kernel/signal32.c+15−9

@@ -243,6 +243,7 @@ static int compat_restore_sigframe(struct pt_regs *regs, · 242 unmodified lines
243	243	int err;
244	244	sigset_t set;
245	245	struct compat_aux_sigframe __user *aux;
	246	+ unsigned long psr;
246	247
247	248	err = get_sigset_t(&set, &sf->uc.uc_sigmask);
248	249	if (err == 0) {
@@ -266,7 +267,9 @@ static int compat_restore_sigframe(struct pt_regs *regs, · 17 unmodified lines
266	267	__get_user_error(regs->compat_sp, &sf->uc.uc_mcontext.arm_sp, err);
267	268	__get_user_error(regs->compat_lr, &sf->uc.uc_mcontext.arm_lr, err);
268	269	__get_user_error(regs->pc, &sf->uc.uc_mcontext.arm_pc, err);
269		− __get_user_error(regs->pstate, &sf->uc.uc_mcontext.arm_cpsr, err);
	270	+ __get_user_error(psr, &sf->uc.uc_mcontext.arm_cpsr, err);
	271	+
	272	+ regs->pstate = compat_psr_to_pstate(psr);
270	273
271	274	/*
272	275	* Avoid compat_sys_sigreturn() restarting.
@@ -282,8 +285,9 @@ static int compat_restore_sigframe(struct pt_regs *regs, · 9 unmodified lines
282	285	return err;
283	286	}
284	287
285		−asmlinkage int compat_sys_sigreturn(struct pt_regs *regs)
	288	+COMPAT_SYSCALL_DEFINE0(sigreturn)
286	289	{
	290	+ struct pt_regs *regs = current_pt_regs();
287	291	struct compat_sigframe __user *frame;
288	292
289	293	/* Always make any pending restarted system calls return -EINTR */
@@ -312,8 +316,9 @@ badframe: · 22 unmodified lines
312	316	return 0;
313	317	}
314	318
315		−asmlinkage int compat_sys_rt_sigreturn(struct pt_regs *regs)
	319	+COMPAT_SYSCALL_DEFINE0(rt_sigreturn)
316	320	{
	321	+ struct pt_regs *regs = current_pt_regs();
317	322	struct compat_rt_sigframe __user *frame;
318	323
319	324	/* Always make any pending restarted system calls return -EINTR */
@@ -372,22 +377,22 @@ static void compat_setup_return(struct pt_regs regs, struct k_sigaction ka, · 52 unmodified lines
372	377	{
373	378	compat_ulong_t handler = ptr_to_compat(ka->sa.sa_handler);
374	379	compat_ulong_t retcode;
375		− compat_ulong_t spsr = regs->pstate & ~(PSR_f \| COMPAT_PSR_E_BIT);
	380	+ compat_ulong_t spsr = regs->pstate & ~(PSR_f \| PSR_AA32_E_BIT);
376	381	int thumb;
377	382
378	383	/* Check if the handler is written for ARM or Thumb */
379	384	thumb = handler & 1;
380	385
381	386	if (thumb)
382		− spsr \|= COMPAT_PSR_T_BIT;
	387	+ spsr \|= PSR_AA32_T_BIT;
383	388	else
384		− spsr &= ~COMPAT_PSR_T_BIT;
	389	+ spsr &= ~PSR_AA32_T_BIT;
385	390
386	391	/* The IT state must be cleared for both ARM and Thumb-2 */
387		− spsr &= ~COMPAT_PSR_IT_MASK;
	392	+ spsr &= ~PSR_AA32_IT_MASK;
388	393
389	394	/* Restore the original endianness */
390		− spsr \|= COMPAT_PSR_ENDSTATE;
	395	+ spsr \|= PSR_AA32_ENDSTATE;
391	396
392	397	if (ka->sa.sa_flags & SA_RESTORER) {
393	398	retcode = ptr_to_compat(ka->sa.sa_restorer);
@@ -414,6 +419,7 @@ static int compat_setup_sigframe(struct compat_sigframe __user *sf, · 20 unmodified lines
414	419	struct pt_regs regs, sigset_t set)
415	420	{
416	421	struct compat_aux_sigframe __user *aux;
	422	+ unsigned long psr = pstate_to_compat_psr(regs->pstate);
417	423	int err = 0;
418	424
419	425	__put_user_error(regs->regs[0], &sf->uc.uc_mcontext.arm_r0, err);
@@ -432,7 +438,7 @@ static int compat_setup_sigframe(struct compat_sigframe __user *sf, · 12 unmodified lines
432	438	__put_user_error(regs->compat_sp, &sf->uc.uc_mcontext.arm_sp, err);
433	439	__put_user_error(regs->compat_lr, &sf->uc.uc_mcontext.arm_lr, err);
434	440	__put_user_error(regs->pc, &sf->uc.uc_mcontext.arm_pc, err);
435		− __put_user_error(regs->pstate, &sf->uc.uc_mcontext.arm_cpsr, err);
	441	+ __put_user_error(psr, &sf->uc.uc_mcontext.arm_cpsr, err);
436	442
437	443	__put_user_error((compat_ulong_t)0, &sf->uc.uc_mcontext.trap_no, err);
438	444	/* set the compat FSR WnR */

arch/arm64/kernel/smp.c+33−11

@@ -225,6 +225,7 @@ asmlinkage notrace void secondary_start_kernel(void) · 224 unmodified lines
225	225	notify_cpu_starting(cpu);
226	226
227	227	store_cpu_topology(cpu);
	228	+ numa_add_cpu(cpu);
228	229
229	230	/*
230	231	* OK, now it's safe to let the boot CPU continue. Wait for
@@ -278,6 +279,9 @@ int __cpu_disable(void) · 47 unmodified lines
278	279	if (ret)
279	280	return ret;
280	281
	282	+ remove_cpu_topology(cpu);
	283	+ numa_remove_cpu(cpu);
	284	+
281	285	/*
282	286	* Take this CPU offline. Once we clear this, we can't return,
283	287	* and we must not schedule until we're ready to give up the cpu.
@@ -518,7 +522,6 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) · 234 unmodified lines
518	522	}
519	523	bootcpu_valid = true;
520	524	cpu_madt_gicc[0] = *processor;
521		− early_map_cpu_to_node(0, acpi_numa_get_nid(0, hwid));
522	525	return;
523	526	}
524	527
@@ -541,8 +544,6 @@ acpi_map_gic_cpu_interface(struct acpi_madt_generic_interrupt *processor) · 16 unmodified lines
541	544	*/
542	545	acpi_set_mailbox_entry(cpu_count, processor);
543	546
544		− early_map_cpu_to_node(cpu_count, acpi_numa_get_nid(cpu_count, hwid));
545		−
546	547	cpu_count++;
547	548	}
548	549
@@ -562,8 +563,34 @@ acpi_parse_gic_cpu_interface(struct acpi_subtable_header *header, · 13 unmodified lines
562	563
563	564	return 0;
564	565	}
	566	+
	567	+static void __init acpi_parse_and_init_cpus(void)
	568	+{
	569	+ int i;
	570	+
	571	+ /*
	572	+ * do a walk of MADT to determine how many CPUs
	573	+ * we have including disabled CPUs, and get information
	574	+ * we need for SMP init.
	575	+ */
	576	+ acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
	577	+ acpi_parse_gic_cpu_interface, 0);
	578	+
	579	+ /*
	580	+ * In ACPI, SMP and CPU NUMA information is provided in separate
	581	+ * static tables, namely the MADT and the SRAT.
	582	+ *
	583	+ * Thus, it is simpler to first create the cpu logical map through
	584	+ * an MADT walk and then map the logical cpus to their node ids
	585	+ * as separate steps.
	586	+ */
	587	+ acpi_map_cpus_to_nodes();
	588	+
	589	+ for (i = 0; i < nr_cpu_ids; i++)
	590	+ early_map_cpu_to_node(i, acpi_numa_get_nid(i));
	591	+}
565	592	#else
566		−#define acpi_table_parse_madt(...) do { } while (0)
	593	+#define acpi_parse_and_init_cpus(...) do { } while (0)
567	594	#endif
568	595
569	596	/*
@@ -636,13 +663,7 @@ void __init smp_init_cpus(void) · 66 unmodified lines
636	663	if (acpi_disabled)
637	664	of_parse_and_init_cpus();
638	665	else
639		− /*
640		− * do a walk of MADT to determine how many CPUs
641		− * we have including disabled CPUs, and get information
642		− * we need for SMP init
643		− */
644		− acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT,
645		− acpi_parse_gic_cpu_interface, 0);
	666	+ acpi_parse_and_init_cpus();
646	667
647	668	if (cpu_count > nr_cpu_ids)
648	669	pr_warn("Number of cores (%d) exceeds configured maximum of %u - clipping\n",
@@ -679,6 +700,7 @@ void __init smp_prepare_cpus(unsigned int max_cpus) · 30 unmodified lines
679	700	this_cpu = smp_processor_id();
680	701	store_cpu_topology(this_cpu);
681	702	numa_store_cpu_info(this_cpu);
	703	+ numa_add_cpu(this_cpu);
682	704
683	705	/*
684	706	* If UP is mandated by "nosmp" (which implies "maxcpus=0"), don't set

arch/arm64/kernel/stacktrace.c+1−1

@@ -50,7 +50,7 @@ int notrace unwind_frame(struct task_struct tsk, struct stackframe frame) · 49 unmodified lines
50	50	if (!tsk)
51	51	tsk = current;
52	52
53		− if (!on_accessible_stack(tsk, fp))
	53	+ if (!on_accessible_stack(tsk, fp, NULL))
54	54	return -EINVAL;
55	55
56	56	frame->fp = READ_ONCE_NOCHECK((unsigned long )(fp));

arch/arm64/kernel/sys.c+16−13

@@ -25,11 +25,13 @@ · 24 unmodified lines
25	25	#include <linux/sched.h>
26	26	#include <linux/slab.h>
27	27	#include <linux/syscalls.h>
	28	+
28	29	#include <asm/cpufeature.h>
	30	+#include <asm/syscall.h>
29	31
30		−asmlinkage long sys_mmap(unsigned long addr, unsigned long len,
31		− unsigned long prot, unsigned long flags,
32		− unsigned long fd, off_t off)
	32	+SYSCALL_DEFINE6(mmap, unsigned long, addr, unsigned long, len,
	33	+ unsigned long, prot, unsigned long, flags,
	34	+ unsigned long, fd, off_t, off)
33	35	{
34	36	if (offset_in_page(off) != 0)
35	37	return -EINVAL;
@@ -42,24 +44,25 @@ SYSCALL_DEFINE1(arm64_personality, unsigned int, personality) · 6 unmodified lines
42	44	if (personality(personality) == PER_LINUX32 &&
43	45	!system_supports_32bit_el0())
44	46	return -EINVAL;
45		− return sys_personality(personality);
	47	+ return ksys_personality(personality);
46	48	}
47	49
48	50	/*
49	51	* Wrappers to pass the pt_regs argument.
50	52	*/
51		−asmlinkage long sys_rt_sigreturn_wrapper(void);
52		−#define sys_rt_sigreturn sys_rt_sigreturn_wrapper
53	53	#define sys_personality sys_arm64_personality
54	54
	55	+asmlinkage long sys_ni_syscall(const struct pt_regs *);
	56	+#define __arm64_sys_ni_syscall sys_ni_syscall
	57	+
55	58	#undef __SYSCALL
56		−#define __SYSCALL(nr, sym) [nr] = sym,
	59	+#define __SYSCALL(nr, sym) asmlinkage long __arm64_##sym(const struct pt_regs *);
	60	+#include <asm/unistd.h>
57	61
58		−/*
59		− * The sys_call_table array must be 4K aligned to be accessible from
60		− * kernel/entry.S.
61		− */
62		−void * const sys_call_table[__NR_syscalls] __aligned(4096) = {
63		− [0 ... __NR_syscalls - 1] = sys_ni_syscall,
	62	+#undef __SYSCALL
	63	+#define __SYSCALL(nr, sym) [nr] = (syscall_fn_t)__arm64_##sym,
	64	+
	65	+const syscall_fn_t sys_call_table[__NR_syscalls] = {
	66	+ [0 ... __NR_syscalls - 1] = (syscall_fn_t)sys_ni_syscall,
64	67	#include <asm/unistd.h>
65	68	};

arch/arm64/kernel/sys32.c+116−19

@@ -22,31 +22,128 @@ · 21 unmodified lines
22	22	*/
23	23	#define __COMPAT_SYSCALL_NR
24	24
	25	+#include <linux/compat.h>
25	26	#include <linux/compiler.h>
26	27	#include <linux/syscalls.h>
27	28
28		−asmlinkage long compat_sys_sigreturn_wrapper(void);
29		−asmlinkage long compat_sys_rt_sigreturn_wrapper(void);
30		−asmlinkage long compat_sys_statfs64_wrapper(void);
31		−asmlinkage long compat_sys_fstatfs64_wrapper(void);
32		−asmlinkage long compat_sys_pread64_wrapper(void);
33		−asmlinkage long compat_sys_pwrite64_wrapper(void);
34		−asmlinkage long compat_sys_truncate64_wrapper(void);
35		−asmlinkage long compat_sys_ftruncate64_wrapper(void);
36		−asmlinkage long compat_sys_readahead_wrapper(void);
37		−asmlinkage long compat_sys_fadvise64_64_wrapper(void);
38		−asmlinkage long compat_sys_sync_file_range2_wrapper(void);
39		−asmlinkage long compat_sys_fallocate_wrapper(void);
40		−asmlinkage long compat_sys_mmap2_wrapper(void);
	29	+#include <asm/syscall.h>
41	30
42		−#undef __SYSCALL
43		−#define __SYSCALL(nr, sym) [nr] = sym,
	31	+asmlinkage long compat_sys_sigreturn(void);
	32	+asmlinkage long compat_sys_rt_sigreturn(void);
	33	+
	34	+COMPAT_SYSCALL_DEFINE3(aarch32_statfs64, const char __user *, pathname,
	35	+ compat_size_t, sz, struct compat_statfs64 __user *, buf)
	36	+{
	37	+ /*
	38	+ * 32-bit ARM applies an OABI compatibility fixup to statfs64 and
	39	+ * fstatfs64 regardless of whether OABI is in use, and therefore
	40	+ * arbitrary binaries may rely upon it, so we must do the same.
	41	+ * For more details, see commit:
	42	+ *
	43	+ * 713c481519f19df9 ("[ARM] 3108/2: old ABI compat: statfs64 and
	44	+ * fstatfs64")
	45	+ */
	46	+ if (sz == 88)
	47	+ sz = 84;
	48	+
	49	+ return kcompat_sys_statfs64(pathname, sz, buf);
	50	+}
	51	+
	52	+COMPAT_SYSCALL_DEFINE3(aarch32_fstatfs64, unsigned int, fd, compat_size_t, sz,
	53	+ struct compat_statfs64 __user *, buf)
	54	+{
	55	+ /* see aarch32_statfs64 */
	56	+ if (sz == 88)
	57	+ sz = 84;
	58	+
	59	+ return kcompat_sys_fstatfs64(fd, sz, buf);
	60	+}
44	61
45	62	/*
46		− * The sys_call_table array must be 4K aligned to be accessible from
47		− * kernel/entry.S.
	63	+ * Note: off_4k is always in units of 4K. If we can't do the
	64	+ * requested offset because it is not page-aligned, we return -EINVAL.
48	65	*/
49		−void * const compat_sys_call_table[__NR_compat_syscalls] __aligned(4096) = {
50		− [0 ... __NR_compat_syscalls - 1] = sys_ni_syscall,
	66	+COMPAT_SYSCALL_DEFINE6(aarch32_mmap2, unsigned long, addr, unsigned long, len,
	67	+ unsigned long, prot, unsigned long, flags,
	68	+ unsigned long, fd, unsigned long, off_4k)
	69	+{
	70	+ if (off_4k & (~PAGE_MASK >> 12))
	71	+ return -EINVAL;
	72	+
	73	+ off_4k >>= (PAGE_SHIFT - 12);
	74	+
	75	+ return ksys_mmap_pgoff(addr, len, prot, flags, fd, off_4k);
	76	+}
	77	+
	78	+#ifdef CONFIG_CPU_BIG_ENDIAN
	79	+#define arg_u32p(name) u32, name##_hi, u32, name##_lo
	80	+#else
	81	+#define arg_u32p(name) u32, name##_lo, u32, name##_hi
	82	+#endif
	83	+
	84	+#define arg_u64(name) (((u64)name##_hi << 32) \| name##_lo)
	85	+
	86	+COMPAT_SYSCALL_DEFINE6(aarch32_pread64, unsigned int, fd, char __user *, buf,
	87	+ size_t, count, u32, __pad, arg_u32p(pos))
	88	+{
	89	+ return ksys_pread64(fd, buf, count, arg_u64(pos));
	90	+}
	91	+
	92	+COMPAT_SYSCALL_DEFINE6(aarch32_pwrite64, unsigned int, fd,
	93	+ const char __user *, buf, size_t, count, u32, __pad,
	94	+ arg_u32p(pos))
	95	+{
	96	+ return ksys_pwrite64(fd, buf, count, arg_u64(pos));
	97	+}
	98	+
	99	+COMPAT_SYSCALL_DEFINE4(aarch32_truncate64, const char __user *, pathname,
	100	+ u32, __pad, arg_u32p(length))
	101	+{
	102	+ return ksys_truncate(pathname, arg_u64(length));
	103	+}
	104	+
	105	+COMPAT_SYSCALL_DEFINE4(aarch32_ftruncate64, unsigned int, fd, u32, __pad,
	106	+ arg_u32p(length))
	107	+{
	108	+ return ksys_ftruncate(fd, arg_u64(length));
	109	+}
	110	+
	111	+COMPAT_SYSCALL_DEFINE5(aarch32_readahead, int, fd, u32, __pad,
	112	+ arg_u32p(offset), size_t, count)
	113	+{
	114	+ return ksys_readahead(fd, arg_u64(offset), count);
	115	+}
	116	+
	117	+COMPAT_SYSCALL_DEFINE6(aarch32_fadvise64_64, int, fd, int, advice,
	118	+ arg_u32p(offset), arg_u32p(len))
	119	+{
	120	+ return ksys_fadvise64_64(fd, arg_u64(offset), arg_u64(len), advice);
	121	+}
	122	+
	123	+COMPAT_SYSCALL_DEFINE6(aarch32_sync_file_range2, int, fd, unsigned int, flags,
	124	+ arg_u32p(offset), arg_u32p(nbytes))
	125	+{
	126	+ return ksys_sync_file_range(fd, arg_u64(offset), arg_u64(nbytes),
	127	+ flags);
	128	+}
	129	+
	130	+COMPAT_SYSCALL_DEFINE6(aarch32_fallocate, int, fd, int, mode,
	131	+ arg_u32p(offset), arg_u32p(len))
	132	+{
	133	+ return ksys_fallocate(fd, mode, arg_u64(offset), arg_u64(len));
	134	+}
	135	+
	136	+asmlinkage long sys_ni_syscall(const struct pt_regs *);
	137	+#define __arm64_sys_ni_syscall sys_ni_syscall
	138	+
	139	+#undef __SYSCALL
	140	+#define __SYSCALL(nr, sym) asmlinkage long __arm64_##sym(const struct pt_regs *);
	141	+#include <asm/unistd32.h>
	142	+
	143	+#undef __SYSCALL
	144	+#define __SYSCALL(nr, sym) [nr] = (syscall_fn_t)__arm64_##sym,
	145	+
	146	+const syscall_fn_t compat_sys_call_table[__NR_compat_syscalls] = {
	147	+ [0 ... __NR_compat_syscalls - 1] = (syscall_fn_t)sys_ni_syscall,
51	148	#include <asm/unistd32.h>
52	149	};

arch/arm64/kernel/syscall.c+139−0

@@ -0,0 +1,139 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include <linux/compiler.h>
	4	+#include <linux/context_tracking.h>
	5	+#include <linux/errno.h>
	6	+#include <linux/nospec.h>
	7	+#include <linux/ptrace.h>
	8	+#include <linux/syscalls.h>
	9	+
	10	+#include <asm/daifflags.h>
	11	+#include <asm/fpsimd.h>
	12	+#include <asm/syscall.h>
	13	+#include <asm/thread_info.h>
	14	+#include <asm/unistd.h>
	15	+
	16	+long compat_arm_syscall(struct pt_regs *regs);
	17	+
	18	+long sys_ni_syscall(void);
	19	+
	20	+asmlinkage long do_ni_syscall(struct pt_regs *regs)
	21	+{
	22	+#ifdef CONFIG_COMPAT
	23	+ long ret;
	24	+ if (is_compat_task()) {
	25	+ ret = compat_arm_syscall(regs);
	26	+ if (ret != -ENOSYS)
	27	+ return ret;
	28	+ }
	29	+#endif
	30	+
	31	+ return sys_ni_syscall();
	32	+}
	33	+
	34	+static long __invoke_syscall(struct pt_regs *regs, syscall_fn_t syscall_fn)
	35	+{
	36	+ return syscall_fn(regs);
	37	+}
	38	+
	39	+static void invoke_syscall(struct pt_regs *regs, unsigned int scno,
	40	+ unsigned int sc_nr,
	41	+ const syscall_fn_t syscall_table[])
	42	+{
	43	+ long ret;
	44	+
	45	+ if (scno < sc_nr) {
	46	+ syscall_fn_t syscall_fn;
	47	+ syscall_fn = syscall_table[array_index_nospec(scno, sc_nr)];
	48	+ ret = __invoke_syscall(regs, syscall_fn);
	49	+ } else {
	50	+ ret = do_ni_syscall(regs);
	51	+ }
	52	+
	53	+ regs->regs[0] = ret;
	54	+}
	55	+
	56	+static inline bool has_syscall_work(unsigned long flags)
	57	+{
	58	+ return unlikely(flags & _TIF_SYSCALL_WORK);
	59	+}
	60	+
	61	+int syscall_trace_enter(struct pt_regs *regs);
	62	+void syscall_trace_exit(struct pt_regs *regs);
	63	+
	64	+static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
	65	+ const syscall_fn_t syscall_table[])
	66	+{
	67	+ unsigned long flags = current_thread_info()->flags;
	68	+
	69	+ regs->orig_x0 = regs->regs[0];
	70	+ regs->syscallno = scno;
	71	+
	72	+ local_daif_restore(DAIF_PROCCTX);
	73	+ user_exit();
	74	+
	75	+ if (has_syscall_work(flags)) {
	76	+ /* set default errno for user-issued syscall(-1) */
	77	+ if (scno == NO_SYSCALL)
	78	+ regs->regs[0] = -ENOSYS;
	79	+ scno = syscall_trace_enter(regs);
	80	+ if (scno == NO_SYSCALL)
	81	+ goto trace_exit;
	82	+ }
	83	+
	84	+ invoke_syscall(regs, scno, sc_nr, syscall_table);
	85	+
	86	+ /*
	87	+ * The tracing status may have changed under our feet, so we have to
	88	+ * check again. However, if we were tracing entry, then we always trace
	89	+ * exit regardless, as the old entry assembly did.
	90	+ */
	91	+ if (!has_syscall_work(flags) && !IS_ENABLED(CONFIG_DEBUG_RSEQ)) {
	92	+ local_daif_mask();
	93	+ flags = current_thread_info()->flags;
	94	+ if (!has_syscall_work(flags)) {
	95	+ /*
	96	+ * We're off to userspace, where interrupts are
	97	+ * always enabled after we restore the flags from
	98	+ * the SPSR.
	99	+ */
	100	+ trace_hardirqs_on();
	101	+ return;
	102	+ }
	103	+ local_daif_restore(DAIF_PROCCTX);
	104	+ }
	105	+
	106	+trace_exit:
	107	+ syscall_trace_exit(regs);
	108	+}
	109	+
	110	+static inline void sve_user_discard(void)
	111	+{
	112	+ if (!system_supports_sve())
	113	+ return;
	114	+
	115	+ clear_thread_flag(TIF_SVE);
	116	+
	117	+ /*
	118	+ * task_fpsimd_load() won't be called to update CPACR_EL1 in
	119	+ * ret_to_user unless TIF_FOREIGN_FPSTATE is still set, which only
	120	+ * happens if a context switch or kernel_neon_begin() or context
	121	+ * modification (sigreturn, ptrace) intervenes.
	122	+ * So, ensure that CPACR_EL1 is already correct for the fast-path case.
	123	+ */
	124	+ sve_user_disable();
	125	+}
	126	+
	127	+asmlinkage void el0_svc_handler(struct pt_regs *regs)
	128	+{
	129	+ sve_user_discard();
	130	+ el0_svc_common(regs, regs->regs[8], __NR_syscalls, sys_call_table);
	131	+}
	132	+
	133	+#ifdef CONFIG_COMPAT
	134	+asmlinkage void el0_svc_compat_handler(struct pt_regs *regs)
	135	+{
	136	+ el0_svc_common(regs, regs->regs[7], __NR_compat_syscalls,
	137	+ compat_sys_call_table);
	138	+}
	139	+#endif

arch/arm64/kernel/topology.c+41−17

@@ -215,11 +215,16 @@ EXPORT_SYMBOL_GPL(cpu_topology); · 214 unmodified lines
215	215
216	216	const struct cpumask *cpu_coregroup_mask(int cpu)
217	217	{
218		− const cpumask_t *core_mask = &cpu_topology[cpu].core_sibling;
	218	+ const cpumask_t *core_mask = cpumask_of_node(cpu_to_node(cpu));
219	219
	220	+ /* Find the smaller of NUMA, core or LLC siblings */
	221	+ if (cpumask_subset(&cpu_topology[cpu].core_sibling, core_mask)) {
	222	+ /* not numa in package, lets use the package siblings */
	223	+ core_mask = &cpu_topology[cpu].core_sibling;
	224	+ }
220	225	if (cpu_topology[cpu].llc_id != -1) {
221		− if (cpumask_subset(&cpu_topology[cpu].llc_siblings, core_mask))
222		− core_mask = &cpu_topology[cpu].llc_siblings;
	226	+ if (cpumask_subset(&cpu_topology[cpu].llc_sibling, core_mask))
	227	+ core_mask = &cpu_topology[cpu].llc_sibling;
223	228	}
224	229
225	230	return core_mask;
@@ -231,27 +236,25 @@ static void update_siblings_masks(unsigned int cpuid) · 5 unmodified lines
231	236	int cpu;
232	237
233	238	/* update core and thread sibling masks */
234		− for_each_possible_cpu(cpu) {
	239	+ for_each_online_cpu(cpu) {
235	240	cpu_topo = &cpu_topology[cpu];
236	241
237	242	if (cpuid_topo->llc_id == cpu_topo->llc_id) {
238		− cpumask_set_cpu(cpu, &cpuid_topo->llc_siblings);
239		− cpumask_set_cpu(cpuid, &cpu_topo->llc_siblings);
	243	+ cpumask_set_cpu(cpu, &cpuid_topo->llc_sibling);
	244	+ cpumask_set_cpu(cpuid, &cpu_topo->llc_sibling);
240	245	}
241	246
242	247	if (cpuid_topo->package_id != cpu_topo->package_id)
243	248	continue;
244	249
245	250	cpumask_set_cpu(cpuid, &cpu_topo->core_sibling);
246		− if (cpu != cpuid)
247		− cpumask_set_cpu(cpu, &cpuid_topo->core_sibling);
	251	+ cpumask_set_cpu(cpu, &cpuid_topo->core_sibling);
248	252
249	253	if (cpuid_topo->core_id != cpu_topo->core_id)
250	254	continue;
251	255
252	256	cpumask_set_cpu(cpuid, &cpu_topo->thread_sibling);
253		− if (cpu != cpuid)
254		− cpumask_set_cpu(cpu, &cpuid_topo->thread_sibling);
	257	+ cpumask_set_cpu(cpu, &cpuid_topo->thread_sibling);
255	258	}
256	259	}
257	260
@@ -293,6 +296,19 @@ topology_populated: · 35 unmodified lines
293	296	update_siblings_masks(cpuid);
294	297	}
295	298
	299	+static void clear_cpu_topology(int cpu)
	300	+{
	301	+ struct cpu_topology *cpu_topo = &cpu_topology[cpu];
	302	+
	303	+ cpumask_clear(&cpu_topo->llc_sibling);
	304	+ cpumask_set_cpu(cpu, &cpu_topo->llc_sibling);
	305	+
	306	+ cpumask_clear(&cpu_topo->core_sibling);
	307	+ cpumask_set_cpu(cpu, &cpu_topo->core_sibling);
	308	+ cpumask_clear(&cpu_topo->thread_sibling);
	309	+ cpumask_set_cpu(cpu, &cpu_topo->thread_sibling);
	310	+}
	311	+
296	312	static void __init reset_cpu_topology(void)
297	313	{
298	314	unsigned int cpu;
@@ -303,18 +319,26 @@ static void __init reset_cpu_topology(void) · 4 unmodified lines
303	319	cpu_topo->thread_id = -1;
304	320	cpu_topo->core_id = 0;
305	321	cpu_topo->package_id = -1;
306		−
307	322	cpu_topo->llc_id = -1;
308		− cpumask_clear(&cpu_topo->llc_siblings);
309		− cpumask_set_cpu(cpu, &cpu_topo->llc_siblings);
310	323
311		− cpumask_clear(&cpu_topo->core_sibling);
312		− cpumask_set_cpu(cpu, &cpu_topo->core_sibling);
313		− cpumask_clear(&cpu_topo->thread_sibling);
314		− cpumask_set_cpu(cpu, &cpu_topo->thread_sibling);
	324	+ clear_cpu_topology(cpu);
315	325	}
316	326	}
317	327
	328	+void remove_cpu_topology(unsigned int cpu)
	329	+{
	330	+ int sibling;
	331	+
	332	+ for_each_cpu(sibling, topology_core_cpumask(cpu))
	333	+ cpumask_clear_cpu(cpu, topology_core_cpumask(sibling));
	334	+ for_each_cpu(sibling, topology_sibling_cpumask(cpu))
	335	+ cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling));
	336	+ for_each_cpu(sibling, topology_llc_cpumask(cpu))
	337	+ cpumask_clear_cpu(cpu, topology_llc_cpumask(sibling));
	338	+
	339	+ clear_cpu_topology(cpu);
	340	+}
	341	+
318	342	#ifdef CONFIG_ACPI
319	343	/*
320	344	* Propagate the topology information of the processor_topology_node tree to the

arch/arm64/kernel/traps.c+1−17

@@ -411,7 +411,7 @@ asmlinkage void __exception do_undefinstr(struct pt_regs *regs) · 410 unmodified lines
411	411
412	412	void cpu_enable_cache_maint_trap(const struct arm64_cpu_capabilities *__unused)
413	413	{
414		− config_sctlr_el1(SCTLR_EL1_UCI, 0);
	414	+ sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCI, 0);
415	415	}
416	416
417	417	#define __user_cache_maint(insn, address, res) \
@@ -547,22 +547,6 @@ asmlinkage void __exception do_sysinstr(unsigned int esr, struct pt_regs *regs) · 129 unmodified lines
547	547	do_undefinstr(regs);
548	548	}
549	549
550		−long compat_arm_syscall(struct pt_regs *regs);
551		−
552		−asmlinkage long do_ni_syscall(struct pt_regs *regs)
553		−{
554		−#ifdef CONFIG_COMPAT
555		− long ret;
556		− if (is_compat_task()) {
557		− ret = compat_arm_syscall(regs);
558		− if (ret != -ENOSYS)
559		− return ret;
560		− }
561		−#endif
562		−
563		− return sys_ni_syscall();
564		−}
565		−
566	550	static const char *esr_class_str[] = {
567	551	[0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC",
568	552	[ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized",

arch/arm64/kvm/guest.c+7−7

@@ -107,14 +107,14 @@ static int set_core_reg(struct kvm_vcpu vcpu, const struct kvm_one_reg reg) · 106 unmodified lines
107	107	}
108	108
109	109	if (off == KVM_REG_ARM_CORE_REG(regs.pstate)) {
110		− u32 mode = ((u32 )valp) & COMPAT_PSR_MODE_MASK;
	110	+ u32 mode = ((u32 )valp) & PSR_AA32_MODE_MASK;
111	111	switch (mode) {
112		− case COMPAT_PSR_MODE_USR:
113		− case COMPAT_PSR_MODE_FIQ:
114		− case COMPAT_PSR_MODE_IRQ:
115		− case COMPAT_PSR_MODE_SVC:
116		− case COMPAT_PSR_MODE_ABT:
117		− case COMPAT_PSR_MODE_UND:
	112	+ case PSR_AA32_MODE_USR:
	113	+ case PSR_AA32_MODE_FIQ:
	114	+ case PSR_AA32_MODE_IRQ:
	115	+ case PSR_AA32_MODE_SVC:
	116	+ case PSR_AA32_MODE_ABT:
	117	+ case PSR_AA32_MODE_UND:
118	118	case PSR_MODE_EL0t:
119	119	case PSR_MODE_EL1t:
120	120	case PSR_MODE_EL1h:

arch/arm64/kvm/hyp/Makefile+2−1

@@ -3,7 +3,8 @@ · 2 unmodified lines
3	3	# Makefile for Kernel-based Virtual Machine module, HYP part
4	4	#
5	5
6		−ccflags-y += -fno-stack-protector -DDISABLE_BRANCH_PROFILING
	6	+ccflags-y += -fno-stack-protector -DDISABLE_BRANCH_PROFILING \
	7	+ $(DISABLE_STACKLEAK_PLUGIN)
7	8
8	9	KVM=../../../../virt/kvm
9	10

arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c+1−1

@@ -27,7 +27,7 @@ · 26 unmodified lines
27	27	static bool __hyp_text __is_be(struct kvm_vcpu *vcpu)
28	28	{
29	29	if (vcpu_mode_is_32bit(vcpu))
30		− return !!(read_sysreg_el2(spsr) & COMPAT_PSR_E_BIT);
	30	+ return !!(read_sysreg_el2(spsr) & PSR_AA32_E_BIT);
31	31
32	32	return !!(read_sysreg(SCTLR_EL1) & SCTLR_ELx_EE);
33	33	}

arch/arm64/kvm/regmap.c+11−11

@@ -112,22 +112,22 @@ static const unsigned long vcpu_reg_offsets[VCPU_NR_MODES][16] = { · 111 unmodified lines
112	112	unsigned long vcpu_reg32(const struct kvm_vcpu vcpu, u8 reg_num)
113	113	{
114	114	unsigned long reg_array = (unsigned long )&vcpu->arch.ctxt.gp_regs.regs;
115		− unsigned long mode = *vcpu_cpsr(vcpu) & COMPAT_PSR_MODE_MASK;
	115	+ unsigned long mode = *vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK;
116	116
117	117	switch (mode) {
118		− case COMPAT_PSR_MODE_USR ... COMPAT_PSR_MODE_SVC:
	118	+ case PSR_AA32_MODE_USR ... PSR_AA32_MODE_SVC:
119	119	mode &= ~PSR_MODE32_BIT; /* 0 ... 3 */
120	120	break;
121	121
122		− case COMPAT_PSR_MODE_ABT:
	122	+ case PSR_AA32_MODE_ABT:
123	123	mode = 4;
124	124	break;
125	125
126		− case COMPAT_PSR_MODE_UND:
	126	+ case PSR_AA32_MODE_UND:
127	127	mode = 5;
128	128	break;
129	129
130		− case COMPAT_PSR_MODE_SYS:
	130	+ case PSR_AA32_MODE_SYS:
131	131	mode = 0; /* SYS maps to USR */
132	132	break;
133	133
@@ -143,13 +143,13 @@ unsigned long vcpu_reg32(const struct kvm_vcpu vcpu, u8 reg_num) · 9 unmodified lines
143	143	*/
144	144	static int vcpu_spsr32_mode(const struct kvm_vcpu *vcpu)
145	145	{
146		− unsigned long mode = *vcpu_cpsr(vcpu) & COMPAT_PSR_MODE_MASK;
	146	+ unsigned long mode = *vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK;
147	147	switch (mode) {
148		− case COMPAT_PSR_MODE_SVC: return KVM_SPSR_SVC;
149		− case COMPAT_PSR_MODE_ABT: return KVM_SPSR_ABT;
150		− case COMPAT_PSR_MODE_UND: return KVM_SPSR_UND;
151		− case COMPAT_PSR_MODE_IRQ: return KVM_SPSR_IRQ;
152		− case COMPAT_PSR_MODE_FIQ: return KVM_SPSR_FIQ;
	148	+ case PSR_AA32_MODE_SVC: return KVM_SPSR_SVC;
	149	+ case PSR_AA32_MODE_ABT: return KVM_SPSR_ABT;
	150	+ case PSR_AA32_MODE_UND: return KVM_SPSR_UND;
	151	+ case PSR_AA32_MODE_IRQ: return KVM_SPSR_IRQ;
	152	+ case PSR_AA32_MODE_FIQ: return KVM_SPSR_FIQ;
153	153	default: BUG();
154	154	}
155	155	}

arch/arm64/kvm/reset.c+2−2

@@ -42,8 +42,8 @@ static const struct kvm_regs default_regs_reset = { · 41 unmodified lines
42	42	};
43	43
44	44	static const struct kvm_regs default_regs_reset32 = {
45		− .regs.pstate = (COMPAT_PSR_MODE_SVC \| COMPAT_PSR_A_BIT \|
46		− COMPAT_PSR_I_BIT \| COMPAT_PSR_F_BIT),
	45	+ .regs.pstate = (PSR_AA32_MODE_SVC \| PSR_AA32_A_BIT \|
	46	+ PSR_AA32_I_BIT \| PSR_AA32_F_BIT),
47	47	};
48	48
49	49	static bool cpu_has_32bit_el1(void)

arch/arm64/mm/cache.S+2−2

@@ -35,7 +35,7 @@ · 34 unmodified lines
35	35	* - start - virtual start address of region
36	36	* - end - virtual end address of region
37	37	*/
38		−ENTRY(flush_icache_range)
	38	+ENTRY(__flush_icache_range)
39	39	/* FALLTHROUGH */
40	40
41	41	/*
@@ -77,7 +77,7 @@ alternative_else_nop_endif · 35 unmodified lines
77	77	9:
78	78	mov x0, #-EFAULT
79	79	b 1b
80		−ENDPROC(flush_icache_range)
	80	+ENDPROC(__flush_icache_range)
81	81	ENDPROC(__flush_cache_user_range)
82	82
83	83	/*

arch/arm64/mm/fault.c+1−1

@@ -874,7 +874,7 @@ void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) · 873 unmodified lines
874	874	*/
875	875	WARN_ON_ONCE(in_interrupt());
876	876
877		− config_sctlr_el1(SCTLR_EL1_SPAN, 0);
	877	+ sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0);
878	878	asm(SET_PSTATE_PAN(1));
879	879	}
880	880	#endif /* CONFIG_ARM64_PAN */

arch/arm64/mm/flush.c+2−1

@@ -66,6 +66,7 @@ void __sync_icache_dcache(pte_t pte) · 65 unmodified lines
66	66	sync_icache_aliases(page_address(page),
67	67	PAGE_SIZE << compound_order(page));
68	68	}
	69	+EXPORT_SYMBOL_GPL(__sync_icache_dcache);
69	70
70	71	/*
71	72	* This function is called when a page has been modified by the kernel. Mark
@@ -82,7 +83,7 @@ EXPORT_SYMBOL(flush_dcache_page); · 10 unmodified lines
82	83	/*
83	84	* Additional functions defined in assembly.
84	85	*/
85		−EXPORT_SYMBOL(flush_icache_range);
	86	+EXPORT_SYMBOL(__flush_icache_range);
86	87
87	88	#ifdef CONFIG_ARCH_HAS_PMEM_API
88	89	void arch_wb_cache_pmem(void *addr, size_t size)

arch/arm64/mm/mmu.c+44−4

@@ -45,6 +45,7 @@ · 44 unmodified lines
45	45	#include <asm/memblock.h>
46	46	#include <asm/mmu_context.h>
47	47	#include <asm/ptdump.h>
	48	+#include <asm/tlbflush.h>
48	49
49	50	#define NO_BLOCK_MAPPINGS BIT(0)
50	51	#define NO_CONT_MAPPINGS BIT(1)
@@ -977,12 +978,51 @@ int pmd_clear_huge(pmd_t *pmdp) · 926 unmodified lines
977	978	return 1;
978	979	}
979	980
980		−int pud_free_pmd_page(pud_t *pud, unsigned long addr)
	981	+int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr)
981	982	{
982		− return pud_none(*pud);
	983	+ pte_t *table;
	984	+ pmd_t pmd;
	985	+
	986	+ pmd = READ_ONCE(*pmdp);
	987	+
	988	+ /* No-op for empty entry and WARN_ON for valid entry */
	989	+ if (!pmd_present(pmd) \|\| !pmd_table(pmd)) {
	990	+ VM_WARN_ON(!pmd_table(pmd));
	991	+ return 1;
	992	+ }
	993	+
	994	+ table = pte_offset_kernel(pmdp, addr);
	995	+ pmd_clear(pmdp);
	996	+ __flush_tlb_kernel_pgtable(addr);
	997	+ pte_free_kernel(NULL, table);
	998	+ return 1;
983	999	}
984	1000
985		−int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
	1001	+int pud_free_pmd_page(pud_t *pudp, unsigned long addr)
986	1002	{
987		− return pmd_none(*pmd);
	1003	+ pmd_t *table;
	1004	+ pmd_t *pmdp;
	1005	+ pud_t pud;
	1006	+ unsigned long next, end;
	1007	+
	1008	+ pud = READ_ONCE(*pudp);
	1009	+
	1010	+ /* No-op for empty entry and WARN_ON for valid entry */
	1011	+ if (!pud_present(pud) \|\| !pud_table(pud)) {
	1012	+ VM_WARN_ON(!pud_table(pud));
	1013	+ return 1;
	1014	+ }
	1015	+
	1016	+ table = pmd_offset(pudp, addr);
	1017	+ pmdp = table;
	1018	+ next = addr;
	1019	+ end = addr + PUD_SIZE;
	1020	+ do {
	1021	+ pmd_free_pte_page(pmdp, next);
	1022	+ } while (pmdp++, next += PMD_SIZE, next != end);
	1023	+
	1024	+ pud_clear(pudp);
	1025	+ __flush_tlb_kernel_pgtable(addr);
	1026	+ pmd_free(NULL, table);
	1027	+ return 1;
988	1028	}

arch/arm64/mm/numa.c+21−8

@@ -70,19 +70,32 @@ EXPORT_SYMBOL(cpumask_of_node); · 69 unmodified lines
70	70
71	71	#endif
72	72
73		−static void map_cpu_to_node(unsigned int cpu, int nid)
	73	+static void numa_update_cpu(unsigned int cpu, bool remove)
74	74	{
75		− set_cpu_numa_node(cpu, nid);
76		− if (nid >= 0)
	75	+ int nid = cpu_to_node(cpu);
	76	+
	77	+ if (nid == NUMA_NO_NODE)
	78	+ return;
	79	+
	80	+ if (remove)
	81	+ cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]);
	82	+ else
77	83	cpumask_set_cpu(cpu, node_to_cpumask_map[nid]);
78	84	}
79	85
80		−void numa_clear_node(unsigned int cpu)
	86	+void numa_add_cpu(unsigned int cpu)
81	87	{
82		− int nid = cpu_to_node(cpu);
	88	+ numa_update_cpu(cpu, false);
	89	+}
83	90
84		− if (nid >= 0)
85		− cpumask_clear_cpu(cpu, node_to_cpumask_map[nid]);
	91	+void numa_remove_cpu(unsigned int cpu)
	92	+{
	93	+ numa_update_cpu(cpu, true);
	94	+}
	95	+
	96	+void numa_clear_node(unsigned int cpu)
	97	+{
	98	+ numa_remove_cpu(cpu);
86	99	set_cpu_numa_node(cpu, NUMA_NO_NODE);
87	100	}
88	101
@@ -116,7 +129,7 @@ static void __init setup_node_to_cpumask_map(void) · 27 unmodified lines
116	129	*/
117	130	void numa_store_cpu_info(unsigned int cpu)
118	131	{
119		− map_cpu_to_node(cpu, cpu_to_node_map[cpu]);
	132	+ set_cpu_numa_node(cpu, cpu_to_node_map[cpu]);
120	133	}
121	134
122	135	void __init early_map_cpu_to_node(unsigned int cpu, int nid)

arch/arm64/mm/ptdump_debugfs.c+1−12

@@ -10,18 +10,7 @@ static int ptdump_show(struct seq_file m, void v) · 9 unmodified lines
10	10	ptdump_walk_pgd(m, info);
11	11	return 0;
12	12	}
13		−
14		−static int ptdump_open(struct inode inode, struct file file)
15		−{
16		− return single_open(file, ptdump_show, inode->i_private);
17		−}
18		−
19		−static const struct file_operations ptdump_fops = {
20		− .open = ptdump_open,
21		− .read = seq_read,
22		− .llseek = seq_lseek,
23		− .release = single_release,
24		−};
	13	+DEFINE_SHOW_ATTRIBUTE(ptdump);
25	14
26	15	int ptdump_debugfs_register(struct ptdump_info info, const char name)
27	16	{

arch/ia64/Kconfig+1−0

@@ -16,6 +16,7 @@ config IA64 · 15 unmodified lines
16	16	select ARCH_MIGHT_HAVE_PC_SERIO
17	17	select PCI if (!IA64_HP_SIM)
18	18	select ACPI if (!IA64_HP_SIM)
	19	+ select ARCH_SUPPORTS_ACPI if (!IA64_HP_SIM)
19	20	select ACPI_SYSTEM_POWER_STATES_SUPPORT if ACPI
20	21	select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI
21	22	select HAVE_UNSTABLE_SCHED_CLOCK

arch/x86/Kconfig+1−0

@@ -75,6 +75,7 @@ config X86 · 74 unmodified lines
75	75	select ARCH_MIGHT_HAVE_ACPI_PDC if ACPI
76	76	select ARCH_MIGHT_HAVE_PC_PARPORT
77	77	select ARCH_MIGHT_HAVE_PC_SERIO
	78	+ select ARCH_SUPPORTS_ACPI
78	79	select ARCH_SUPPORTS_ATOMIC_RMW
79	80	select ARCH_SUPPORTS_NUMA_BALANCING if X86_64
80	81	select ARCH_USE_BUILTIN_BSWAP

drivers/acpi/Kconfig+5−3

@@ -5,11 +5,10 @@ · 4 unmodified lines
5	5
6	6	menuconfig ACPI
7	7	bool "ACPI (Advanced Configuration and Power Interface) Support"
8		− depends on !IA64_HP_SIM
9		− depends on IA64 \|\| X86 \|\| ARM64
	8	+ depends on ARCH_SUPPORTS_ACPI
10	9	depends on PCI
11	10	select PNP
12		− default y if (IA64 \|\| X86)
	11	+ default y if X86
13	12	help
14	13	Advanced Configuration and Power Interface (ACPI) support for
15	14	Linux requires an ACPI-compliant platform (hardware/firmware),
@@ -41,6 +40,9 @@ menuconfig ACPI · 25 unmodified lines
41	40	<http://www.acpi.info>
42	41	<http://www.uefi.org/acpi/specs>
43	42
	43	+config ARCH_SUPPORTS_ACPI
	44	+ bool
	45	+
44	46	if ACPI
45	47
46	48	config ACPI_LEGACY_TABLES_LOOKUP

drivers/firmware/efi/arm-init.c+0−1

@@ -259,7 +259,6 @@ void __init efi_init(void) · 258 unmodified lines
259	259
260	260	reserve_regions();
261	261	efi_esrt_init();
262		− efi_memmap_unmap();
263	262
264	263	memblock_reserve(params.mmap & PAGE_MASK,
265	264	PAGE_ALIGN(params.mmap_size +

drivers/firmware/efi/arm-runtime.c+10−8

@@ -110,11 +110,20 @@ static int __init arm_enable_runtime_services(void) · 109 unmodified lines
110	110	{
111	111	u64 mapsize;
112	112
113		− if (!efi_enabled(EFI_BOOT)) {
	113	+ if (!efi_enabled(EFI_BOOT) \|\| !efi_enabled(EFI_MEMMAP)) {
114	114	pr_info("EFI services will not be available.\n");
115	115	return 0;
116	116	}
117	117
	118	+ efi_memmap_unmap();
	119	+
	120	+ mapsize = efi.memmap.desc_size * efi.memmap.nr_map;
	121	+
	122	+ if (efi_memmap_init_late(efi.memmap.phys_map, mapsize)) {
	123	+ pr_err("Failed to remap EFI memory map\n");
	124	+ return 0;
	125	+ }
	126	+
118	127	if (efi_runtime_disabled()) {
119	128	pr_info("EFI runtime services will be disabled.\n");
120	129	return 0;
@@ -127,13 +136,6 @@ static int __init arm_enable_runtime_services(void) · 6 unmodified lines
127	136
128	137	pr_info("Remapping and enabling EFI services.\n");
129	138
130		− mapsize = efi.memmap.desc_size * efi.memmap.nr_map;
131		−
132		− if (efi_memmap_init_late(efi.memmap.phys_map, mapsize)) {
133		− pr_err("Failed to remap EFI memory map\n");
134		− return -ENOMEM;
135		− }
136		−
137	139	if (!efi_virtmap_init()) {
138	140	pr_err("UEFI virtual mapping missing or invalid -- runtime services will not be available\n");
139	141	return -ENOMEM;

drivers/firmware/efi/libstub/Makefile+5−2

@@ -11,7 +11,10 @@ cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ -O2 \ · 10 unmodified lines
11	11	-fPIC -fno-strict-aliasing -mno-red-zone \
12	12	-mno-mmx -mno-sse -fshort-wchar
13	13
14		−cflags-$(CONFIG_ARM64) := $(subst -pg,,$(KBUILD_CFLAGS)) -fpie
	14	+# arm64 uses the full KBUILD_CFLAGS so it's necessary to explicitly
	15	+# disable the stackleak plugin
	16	+cflags-$(CONFIG_ARM64) := $(subst -pg,,$(KBUILD_CFLAGS)) -fpie \
	17	+ $(DISABLE_STACKLEAK_PLUGIN)
15	18	cflags-$(CONFIG_ARM) := $(subst -pg,,$(KBUILD_CFLAGS)) \
16	19	-fno-builtin -fpic -mno-single-pic-base
17	20
@@ -20,7 +23,7 @@ cflags-$(CONFIG_EFI_ARMSTUB) += -I$(srctree)/scripts/dtc/libfdt · 2 unmodified lines
20	23	KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \
21	24	-D__NO_FORTIFY \
22	25	$(call cc-option,-ffreestanding) \
23		− $(call cc-option,-fno-stack-protector)
	26	+ $(call cc-option,-fno-stack-protector) \
24	27
25	28	GCOV_PROFILE := n
26	29	KASAN_SANITIZE := n

drivers/perf/arm-cci.c+26−12

@@ -53,6 +53,16 @@ enum { · 52 unmodified lines
53	53	CCI_IF_MAX,
54	54	};
55	55
	56	+#define NUM_HW_CNTRS_CII_4XX 4
	57	+#define NUM_HW_CNTRS_CII_5XX 8
	58	+#define NUM_HW_CNTRS_MAX NUM_HW_CNTRS_CII_5XX
	59	+
	60	+#define FIXED_HW_CNTRS_CII_4XX 1
	61	+#define FIXED_HW_CNTRS_CII_5XX 0
	62	+#define FIXED_HW_CNTRS_MAX FIXED_HW_CNTRS_CII_4XX
	63	+
	64	+#define HW_CNTRS_MAX (NUM_HW_CNTRS_MAX + FIXED_HW_CNTRS_MAX)
	65	+
56	66	struct event_range {
57	67	u32 min;
58	68	u32 max;
@@ -633,8 +643,7 @@ static void cci_pmu_sync_counters(struct cci_pmu *cci_pmu) · 574 unmodified lines
633	643	{
634	644	int i;
635	645	struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events;
636		−
637		− DECLARE_BITMAP(mask, cci_pmu->num_cntrs);
	646	+ DECLARE_BITMAP(mask, HW_CNTRS_MAX);
638	647
639	648	bitmap_zero(mask, cci_pmu->num_cntrs);
640	649	for_each_set_bit(i, cci_pmu->hw_events.used_mask, cci_pmu->num_cntrs) {
@@ -940,7 +949,7 @@ static void pmu_write_counters(struct cci_pmu cci_pmu, unsigned long mask) · 299 unmodified lines
940	949	static void cci5xx_pmu_write_counters(struct cci_pmu cci_pmu, unsigned long mask)
941	950	{
942	951	int i;
943		− DECLARE_BITMAP(saved_mask, cci_pmu->num_cntrs);
	952	+ DECLARE_BITMAP(saved_mask, HW_CNTRS_MAX);
944	953
945	954	bitmap_zero(saved_mask, cci_pmu->num_cntrs);
946	955	pmu_save_counters(cci_pmu, saved_mask);
@@ -1245,7 +1254,7 @@ static int validate_group(struct perf_event *event) · 298 unmodified lines
1245	1254	{
1246	1255	struct perf_event sibling, leader = event->group_leader;
1247	1256	struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1248		− unsigned long mask[BITS_TO_LONGS(cci_pmu->num_cntrs)];
	1257	+ unsigned long mask[BITS_TO_LONGS(HW_CNTRS_MAX)];
1249	1258	struct cci_pmu_hw_events fake_pmu = {
1250	1259	/*
1251	1260	* Initialise the fake PMU. We only need to populate the
@@ -1403,6 +1412,11 @@ static int cci_pmu_init(struct cci_pmu cci_pmu, struct platform_device pdev) · 151 unmodified lines
1403	1412	char *name = model->name;
1404	1413	u32 num_cntrs;
1405	1414
	1415	+ if (WARN_ON(model->num_hw_cntrs > NUM_HW_CNTRS_MAX))
	1416	+ return -EINVAL;
	1417	+ if (WARN_ON(model->fixed_hw_cntrs > FIXED_HW_CNTRS_MAX))
	1418	+ return -EINVAL;
	1419	+
1406	1420	pmu_event_attr_group.attrs = model->event_attrs;
1407	1421	pmu_format_attr_group.attrs = model->format_attrs;
1408	1422
@@ -1455,8 +1469,8 @@ static __maybe_unused struct cci_pmu_model cci_pmu_models[] = { · 46 unmodified lines
1455	1469	#ifdef CONFIG_ARM_CCI400_PMU
1456	1470	[CCI400_R0] = {
1457	1471	.name = "CCI_400",
1458		− .fixed_hw_cntrs = 1, /* Cycle counter */
1459		− .num_hw_cntrs = 4,
	1472	+ .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_4XX, /* Cycle counter */
	1473	+ .num_hw_cntrs = NUM_HW_CNTRS_CII_4XX,
1460	1474	.cntr_size = SZ_4K,
1461	1475	.format_attrs = cci400_pmu_format_attrs,
1462	1476	.event_attrs = cci400_r0_pmu_event_attrs,
@@ -1475,8 +1489,8 @@ static __maybe_unused struct cci_pmu_model cci_pmu_models[] = { · 12 unmodified lines
1475	1489	},
1476	1490	[CCI400_R1] = {
1477	1491	.name = "CCI_400_r1",
1478		− .fixed_hw_cntrs = 1, /* Cycle counter */
1479		− .num_hw_cntrs = 4,
	1492	+ .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_4XX, /* Cycle counter */
	1493	+ .num_hw_cntrs = NUM_HW_CNTRS_CII_4XX,
1480	1494	.cntr_size = SZ_4K,
1481	1495	.format_attrs = cci400_pmu_format_attrs,
1482	1496	.event_attrs = cci400_r1_pmu_event_attrs,
@@ -1497,8 +1511,8 @@ static __maybe_unused struct cci_pmu_model cci_pmu_models[] = { · 14 unmodified lines
1497	1511	#ifdef CONFIG_ARM_CCI5xx_PMU
1498	1512	[CCI500_R0] = {
1499	1513	.name = "CCI_500",
1500		− .fixed_hw_cntrs = 0,
1501		− .num_hw_cntrs = 8,
	1514	+ .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_5XX,
	1515	+ .num_hw_cntrs = NUM_HW_CNTRS_CII_5XX,
1502	1516	.cntr_size = SZ_64K,
1503	1517	.format_attrs = cci5xx_pmu_format_attrs,
1504	1518	.event_attrs = cci5xx_pmu_event_attrs,
@@ -1521,8 +1535,8 @@ static __maybe_unused struct cci_pmu_model cci_pmu_models[] = { · 16 unmodified lines
1521	1535	},
1522	1536	[CCI550_R0] = {
1523	1537	.name = "CCI_550",
1524		− .fixed_hw_cntrs = 0,
1525		− .num_hw_cntrs = 8,
	1538	+ .fixed_hw_cntrs = FIXED_HW_CNTRS_CII_5XX,
	1539	+ .num_hw_cntrs = NUM_HW_CNTRS_CII_5XX,
1526	1540	.cntr_size = SZ_64K,
1527	1541	.format_attrs = cci5xx_pmu_format_attrs,
1528	1542	.event_attrs = cci5xx_pmu_event_attrs,

drivers/perf/arm-ccn.c+3−11

@@ -1485,17 +1485,9 @@ static int arm_ccn_probe(struct platform_device *pdev) · 1484 unmodified lines
1485	1485	platform_set_drvdata(pdev, ccn);
1486	1486
1487	1487	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1488		− if (!res)
1489		− return -EINVAL;
1490		−
1491		− if (!devm_request_mem_region(ccn->dev, res->start,
1492		− resource_size(res), pdev->name))
1493		− return -EBUSY;
1494		−
1495		− ccn->base = devm_ioremap(ccn->dev, res->start,
1496		− resource_size(res));
1497		− if (!ccn->base)
1498		− return -EFAULT;
	1488	+ ccn->base = devm_ioremap_resource(ccn->dev, res);
	1489	+ if (IS_ERR(ccn->base))
	1490	+ return PTR_ERR(ccn->base);
1499	1491
1500	1492	res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
1501	1493	if (!res)

drivers/perf/arm_pmu.c+22−16

@@ -28,6 +28,14 @@ · 27 unmodified lines
28	28	static DEFINE_PER_CPU(struct arm_pmu *, cpu_armpmu);
29	29	static DEFINE_PER_CPU(int, cpu_irq);
30	30
	31	+static inline u64 arm_pmu_event_max_period(struct perf_event *event)
	32	+{
	33	+ if (event->hw.flags & ARMPMU_EVT_64BIT)
	34	+ return GENMASK_ULL(63, 0);
	35	+ else
	36	+ return GENMASK_ULL(31, 0);
	37	+}
	38	+
31	39	static int
32	40	armpmu_map_cache_event(const unsigned (*cache_map)
33	41	[PERF_COUNT_HW_CACHE_MAX]
@@ -114,8 +122,10 @@ int armpmu_event_set_period(struct perf_event *event) · 80 unmodified lines
114	122	struct hw_perf_event *hwc = &event->hw;
115	123	s64 left = local64_read(&hwc->period_left);
116	124	s64 period = hwc->sample_period;
	125	+ u64 max_period;
117	126	int ret = 0;
118	127
	128	+ max_period = arm_pmu_event_max_period(event);
119	129	if (unlikely(left <= -period)) {
120	130	left = period;
121	131	local64_set(&hwc->period_left, left);
@@ -136,12 +146,12 @@ int armpmu_event_set_period(struct perf_event *event) · 14 unmodified lines
136	146	* effect we are reducing max_period to account for
137	147	* interrupt latency (and we are being very conservative).
138	148	*/
139		− if (left > (armpmu->max_period >> 1))
140		− left = armpmu->max_period >> 1;
	149	+ if (left > (max_period >> 1))
	150	+ left = (max_period >> 1);
141	151
142	152	local64_set(&hwc->prev_count, (u64)-left);
143	153
144		− armpmu->write_counter(event, (u64)(-left) & 0xffffffff);
	154	+ armpmu->write_counter(event, (u64)(-left) & max_period);
145	155
146	156	perf_event_update_userpage(event);
147	157
@@ -153,6 +163,7 @@ u64 armpmu_event_update(struct perf_event *event) · 5 unmodified lines
153	163	struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
154	164	struct hw_perf_event *hwc = &event->hw;
155	165	u64 delta, prev_raw_count, new_raw_count;
	166	+ u64 max_period = arm_pmu_event_max_period(event);
156	167
157	168	again:
158	169	prev_raw_count = local64_read(&hwc->prev_count);
@@ -162,7 +173,7 @@ again: · 3 unmodified lines
162	173	new_raw_count) != prev_raw_count)
163	174	goto again;
164	175
165		− delta = (new_raw_count - prev_raw_count) & armpmu->max_period;
	176	+ delta = (new_raw_count - prev_raw_count) & max_period;
166	177
167	178	local64_add(delta, &event->count);
168	179	local64_sub(delta, &hwc->period_left);
@@ -227,11 +238,10 @@ armpmu_del(struct perf_event *event, int flags) · 58 unmodified lines
227	238
228	239	armpmu_stop(event, PERF_EF_UPDATE);
229	240	hw_events->events[idx] = NULL;
230		− clear_bit(idx, hw_events->used_mask);
231		− if (armpmu->clear_event_idx)
232		− armpmu->clear_event_idx(hw_events, event);
233		−
	241	+ armpmu->clear_event_idx(hw_events, event);
234	242	perf_event_update_userpage(event);
	243	+ /* Clear the allocated counter */
	244	+ hwc->idx = -1;
235	245	}
236	246
237	247	static int
@@ -360,6 +370,7 @@ __hw_perf_event_init(struct perf_event *event) · 122 unmodified lines
360	370	struct hw_perf_event *hwc = &event->hw;
361	371	int mapping;
362	372
	373	+ hwc->flags = 0;
363	374	mapping = armpmu->map_event(event);
364	375
365	376	if (mapping < 0) {
@@ -402,7 +413,7 @@ __hw_perf_event_init(struct perf_event *event) · 36 unmodified lines
402	413	* is far less likely to overtake the previous one unless
403	414	* you have some serious IRQ latency issues.
404	415	*/
405		− hwc->sample_period = armpmu->max_period >> 1;
	416	+ hwc->sample_period = arm_pmu_event_max_period(event) >> 1;
406	417	hwc->last_period = hwc->sample_period;
407	418	local64_set(&hwc->period_left, hwc->sample_period);
408	419	}
@@ -654,14 +665,9 @@ static void cpu_pm_pmu_setup(struct arm_pmu *armpmu, unsigned long cmd) · 245 unmodified lines
654	665	int idx;
655	666
656	667	for (idx = 0; idx < armpmu->num_events; idx++) {
657		− /*
658		− * If the counter is not used skip it, there is no
659		− * need of stopping/restarting it.
660		− */
661		− if (!test_bit(idx, hw_events->used_mask))
662		− continue;
663		−
664	668	event = hw_events->events[idx];
	669	+ if (!event)
	670	+ continue;
665	671
666	672	switch (cmd) {
667	673	case CPU_PM_ENTER:

drivers/perf/arm_pmu_platform.c+1−1

@@ -160,7 +160,7 @@ static int pmu_parse_irqs(struct arm_pmu *pmu) · 159 unmodified lines
160	160	static int armpmu_request_irqs(struct arm_pmu *armpmu)
161	161	{
162	162	struct pmu_hw_events __percpu *hw_events = armpmu->hw_events;
163		− int cpu, err;
	163	+ int cpu, err = 0;
164	164
165	165	for_each_cpu(cpu, &armpmu->supported_cpus) {
166	166	int irq = per_cpu(hw_events->irq, cpu);

drivers/perf/hisilicon/hisi_uncore_pmu.c+7−5

@@ -350,19 +350,21 @@ void hisi_uncore_pmu_disable(struct pmu *pmu) · 349 unmodified lines
350	350
351	351	/*
352	352	* Read Super CPU cluster and CPU cluster ID from MPIDR_EL1.
353		− * If multi-threading is supported, SCCL_ID is in MPIDR[aff3] and CCL_ID
354		− * is in MPIDR[aff2]; if not, SCCL_ID is in MPIDR[aff2] and CCL_ID is
355		− * in MPIDR[aff1]. If this changes in future, this shall be updated.
	353	+ * If multi-threading is supported, CCL_ID is the low 3-bits in MPIDR[Aff2]
	354	+ * and SCCL_ID is the upper 5-bits of Aff2 field; if not, SCCL_ID
	355	+ * is in MPIDR[Aff2] and CCL_ID is in MPIDR[Aff1].
356	356	*/
357	357	static void hisi_read_sccl_and_ccl_id(int sccl_id, int ccl_id)
358	358	{
359	359	u64 mpidr = read_cpuid_mpidr();
360	360
361	361	if (mpidr & MPIDR_MT_BITMASK) {
	362	+ int aff2 = MPIDR_AFFINITY_LEVEL(mpidr, 2);
	363	+
362	364	if (sccl_id)
363		− *sccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 3);
	365	+ *sccl_id = aff2 >> 3;
364	366	if (ccl_id)
365		− *ccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
	367	+ *ccl_id = aff2 & 0x7;
366	368	} else {
367	369	if (sccl_id)
368	370	*sccl_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);

fs/statfs.c+12−2

@@ -335,7 +335,7 @@ static int put_compat_statfs64(struct compat_statfs64 __user *ubuf, struct kstat · 334 unmodified lines
335	335	return 0;
336	336	}
337	337
338		−COMPAT_SYSCALL_DEFINE3(statfs64, const char __user , pathname, compat_size_t, sz, struct compat_statfs64 __user , buf)
	338	+int kcompat_sys_statfs64(const char __user * pathname, compat_size_t sz, struct compat_statfs64 __user * buf)
339	339	{
340	340	struct kstatfs tmp;
341	341	int error;
@@ -349,7 +349,12 @@ COMPAT_SYSCALL_DEFINE3(statfs64, const char __user *, pathname, compat_size_t, s · 7 unmodified lines
349	349	return error;
350	350	}
351	351
352		−COMPAT_SYSCALL_DEFINE3(fstatfs64, unsigned int, fd, compat_size_t, sz, struct compat_statfs64 __user *, buf)
	352	+COMPAT_SYSCALL_DEFINE3(statfs64, const char __user , pathname, compat_size_t, sz, struct compat_statfs64 __user , buf)
	353	+{
	354	+ return kcompat_sys_statfs64(pathname, sz, buf);
	355	+}
	356	+
	357	+int kcompat_sys_fstatfs64(unsigned int fd, compat_size_t sz, struct compat_statfs64 __user * buf)
353	358	{
354	359	struct kstatfs tmp;
355	360	int error;
@@ -363,6 +368,11 @@ COMPAT_SYSCALL_DEFINE3(fstatfs64, unsigned int, fd, compat_size_t, sz, struct co · 7 unmodified lines
363	368	return error;
364	369	}
365	370
	371	+COMPAT_SYSCALL_DEFINE3(fstatfs64, unsigned int, fd, compat_size_t, sz, struct compat_statfs64 __user *, buf)
	372	+{
	373	+ return kcompat_sys_fstatfs64(fd, sz, buf);
	374	+}
	375	+
366	376	/*
367	377	* This is a copy of sys_ustat, just dealing with a structure layout.
368	378	* Given how simple this syscall is that apporach is more maintainable

include/linux/compat.h+11−0

@@ -1019,6 +1019,17 @@ static inline struct compat_timeval ns_to_compat_timeval(s64 nsec) · 1018 unmodified lines
1019	1019	return ctv;
1020	1020	}
1021	1021
	1022	+/*
	1023	+ * Kernel code should not call compat syscalls (i.e., compat_sys_xyzyyz())
	1024	+ * directly. Instead, use one of the functions which work equivalently, such
	1025	+ * as the kcompat_sys_xyzyyz() functions prototyped below.
	1026	+ */
	1027	+
	1028	+int kcompat_sys_statfs64(const char __user * pathname, compat_size_t sz,
	1029	+ struct compat_statfs64 __user * buf);
	1030	+int kcompat_sys_fstatfs64(unsigned int fd, compat_size_t sz,
	1031	+ struct compat_statfs64 __user * buf);
	1032	+
1022	1033	#else /* !CONFIG_COMPAT */
1023	1034
1024	1035	#define is_compat_task() (0)

include/linux/perf/arm_pmu.h+8−3

@@ -25,6 +25,12 @@ · 24 unmodified lines
25	25	*/
26	26	#define ARMPMU_MAX_HWEVENTS 32
27	27
	28	+/*
	29	+ * ARM PMU hw_event flags
	30	+ */
	31	+/* Event uses a 64bit counter */
	32	+#define ARMPMU_EVT_64BIT 1
	33	+
28	34	#define HW_OP_UNSUPPORTED 0xFFFF
29	35	#define C(_x) PERF_COUNT_HW_CACHE_##_x
30	36	#define CACHE_OP_UNSUPPORTED 0xFFFF
@@ -87,14 +93,13 @@ struct arm_pmu { · 56 unmodified lines
87	93	struct perf_event *event);
88	94	int (set_event_filter)(struct hw_perf_event evt,
89	95	struct perf_event_attr *attr);
90		− u32 (read_counter)(struct perf_event event);
91		− void (write_counter)(struct perf_event event, u32 val);
	96	+ u64 (read_counter)(struct perf_event event);
	97	+ void (write_counter)(struct perf_event event, u64 val);
92	98	void (start)(struct arm_pmu );
93	99	void (stop)(struct arm_pmu );
94	100	void (reset)(void );
95	101	int (map_event)(struct perf_event event);
96	102	int num_events;
97		− u64 max_period;
98	103	bool secure_access; /* 32-bit ARM only */
99	104	#define ARMV8_PMUV3_MAX_COMMON_EVENTS 0x40
100	105	DECLARE_BITMAP(pmceid_bitmap, ARMV8_PMUV3_MAX_COMMON_EVENTS);

include/linux/syscalls.h+11−0

@@ -81,6 +81,7 @@ union bpf_attr; · 80 unmodified lines
81	81	#include <linux/unistd.h>
82	82	#include <linux/quota.h>
83	83	#include <linux/key.h>
	84	+#include <linux/personality.h>
84	85	#include <trace/syscall.h>
85	86
86	87	#ifdef CONFIG_ARCH_HAS_SYSCALL_WRAPPER
@@ -1282,4 +1283,14 @@ static inline long ksys_truncate(const char __user *pathname, loff_t length) · 1195 unmodified lines
1282	1283	return do_sys_truncate(pathname, length);
1283	1284	}
1284	1285
	1286	+static inline unsigned int ksys_personality(unsigned int personality)
	1287	+{
	1288	+ unsigned int old = current->personality;
	1289	+
	1290	+ if (personality != 0xffffffff)
	1291	+ set_personality(personality);
	1292	+
	1293	+ return old;
	1294	+}
	1295	+
1285	1296	#endif

include/uapi/asm-generic/unistd.h+3−1

@@ -734,9 +734,11 @@ __SYSCALL(__NR_pkey_free, sys_pkey_free) · 733 unmodified lines
734	734	__SYSCALL(__NR_statx, sys_statx)
735	735	#define __NR_io_pgetevents 292
736	736	__SC_COMP(__NR_io_pgetevents, sys_io_pgetevents, compat_sys_io_pgetevents)
	737	+#define __NR_rseq 293
	738	+__SYSCALL(__NR_rseq, sys_rseq)
737	739
738	740	#undef __NR_syscalls
739		−#define __NR_syscalls 293
	741	+#define __NR_syscalls 294
740	742
741	743	/*
742	744	* 32 bit systems traditionally used different

kernel/events/core.c+2−2

@@ -5246,8 +5246,8 @@ void perf_event_update_userpage(struct perf_event *event) · 5245 unmodified lines
5246	5246
5247	5247	userpg = rb->user_page;
5248	5248	/*
5249		− * Disable preemption so as to not let the corresponding user-space
5250		− * spin too long if we get preempted.
	5249	+ * Disable preemption to guarantee consistent time stamps are stored to
	5250	+ * the user page.
5251	5251	*/
5252	5252	preempt_disable();
5253	5253	++userpg->lock;

scripts/Kbuild.include+2−2

@@ -163,8 +163,8 @@ cc-ldoption = $(call try-run,\ · 162 unmodified lines
163	163	$(CC) $(1) $(KBUILD_CPPFLAGS) $(CC_OPTION_CFLAGS) -nostdlib -x c /dev/null -o "$$TMP",$(1),$(2))
164	164
165	165	# ld-option
166		−# Usage: LDFLAGS += $(call ld-option, -X)
167		−ld-option = $(call try-run, $(LD) $(LDFLAGS) $(1) -v,$(1),$(2))
	166	+# Usage: LDFLAGS += $(call ld-option, -X, -Y)
	167	+ld-option = $(call try-run, $(LD) $(LDFLAGS) $(1) -v,$(1),$(2),$(3))
168	168
169	169	# ar-option
170	170	# Usage: KBUILD_ARFLAGS := $(call ar-option,D)

tools/testing/selftests/rseq/param_test.c+20−0

@@ -138,6 +138,26 @@ unsigned int yield_mod_cnt, nr_abort; · 137 unmodified lines
138	138	"bne 222b\n\t" \
139	139	"333:\n\t"
140	140
	141	+#elif defined(__AARCH64EL__)
	142	+
	143	+#define RSEQ_INJECT_INPUT \
	144	+ , [loop_cnt_1] "Qo" (loop_cnt[1]) \
	145	+ , [loop_cnt_2] "Qo" (loop_cnt[2]) \
	146	+ , [loop_cnt_3] "Qo" (loop_cnt[3]) \
	147	+ , [loop_cnt_4] "Qo" (loop_cnt[4]) \
	148	+ , [loop_cnt_5] "Qo" (loop_cnt[5]) \
	149	+ , [loop_cnt_6] "Qo" (loop_cnt[6])
	150	+
	151	+#define INJECT_ASM_REG RSEQ_ASM_TMP_REG32
	152	+
	153	+#define RSEQ_INJECT_ASM(n) \
	154	+ " ldr " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n" \
	155	+ " cbz " INJECT_ASM_REG ", 333f\n" \
	156	+ "222:\n" \
	157	+ " sub " INJECT_ASM_REG ", " INJECT_ASM_REG ", #1\n" \
	158	+ " cbnz " INJECT_ASM_REG ", 222b\n" \
	159	+ "333:\n"
	160	+
141	161	#elif __PPC__
142	162
143	163	#define RSEQ_INJECT_INPUT \

tools/testing/selftests/rseq/rseq-arm64.h+594−0

@@ -0,0 +1,594 @@
	1	+/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
	2	+/*
	3	+ * rseq-arm64.h
	4	+ *
	5	+ * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	6	+ * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
	7	+ */
	8	+
	9	+#define RSEQ_SIG 0xd428bc00 /* BRK #0x45E0 */
	10	+
	11	+#define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
	12	+#define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
	13	+#define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
	14	+
	15	+#define rseq_smp_load_acquire(p) \
	16	+__extension__ ({ \
	17	+ __typeof(*p) ____p1; \
	18	+ switch (sizeof(*p)) { \
	19	+ case 1: \
	20	+ asm volatile ("ldarb %w0, %1" \
	21	+ : "=r" ((__u8 )p) \
	22	+ : "Q" (*p) : "memory"); \
	23	+ break; \
	24	+ case 2: \
	25	+ asm volatile ("ldarh %w0, %1" \
	26	+ : "=r" ((__u16 )p) \
	27	+ : "Q" (*p) : "memory"); \
	28	+ break; \
	29	+ case 4: \
	30	+ asm volatile ("ldar %w0, %1" \
	31	+ : "=r" ((__u32 )p) \
	32	+ : "Q" (*p) : "memory"); \
	33	+ break; \
	34	+ case 8: \
	35	+ asm volatile ("ldar %0, %1" \
	36	+ : "=r" ((__u64 )p) \
	37	+ : "Q" (*p) : "memory"); \
	38	+ break; \
	39	+ } \
	40	+ ____p1; \
	41	+})
	42	+
	43	+#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
	44	+
	45	+#define rseq_smp_store_release(p, v) \
	46	+do { \
	47	+ switch (sizeof(*p)) { \
	48	+ case 1: \
	49	+ asm volatile ("stlrb %w1, %0" \
	50	+ : "=Q" (*p) \
	51	+ : "r" ((__u8)v) \
	52	+ : "memory"); \
	53	+ break; \
	54	+ case 2: \
	55	+ asm volatile ("stlrh %w1, %0" \
	56	+ : "=Q" (*p) \
	57	+ : "r" ((__u16)v) \
	58	+ : "memory"); \
	59	+ break; \
	60	+ case 4: \
	61	+ asm volatile ("stlr %w1, %0" \
	62	+ : "=Q" (*p) \
	63	+ : "r" ((__u32)v) \
	64	+ : "memory"); \
	65	+ break; \
	66	+ case 8: \
	67	+ asm volatile ("stlr %1, %0" \
	68	+ : "=Q" (*p) \
	69	+ : "r" ((__u64)v) \
	70	+ : "memory"); \
	71	+ break; \
	72	+ } \
	73	+} while (0)
	74	+
	75	+#ifdef RSEQ_SKIP_FASTPATH
	76	+#include "rseq-skip.h"
	77	+#else /* !RSEQ_SKIP_FASTPATH */
	78	+
	79	+#define RSEQ_ASM_TMP_REG32 "w15"
	80	+#define RSEQ_ASM_TMP_REG "x15"
	81	+#define RSEQ_ASM_TMP_REG_2 "x14"
	82	+
	83	+#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
	84	+ post_commit_offset, abort_ip) \
	85	+ " .pushsection __rseq_table, \"aw\"\n" \
	86	+ " .balign 32\n" \
	87	+ __rseq_str(label) ":\n" \
	88	+ " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
	89	+ " .quad " __rseq_str(start_ip) ", " \
	90	+ __rseq_str(post_commit_offset) ", " \
	91	+ __rseq_str(abort_ip) "\n" \
	92	+ " .popsection\n"
	93	+
	94	+#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	95	+ __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
	96	+ (post_commit_ip - start_ip), abort_ip)
	97	+
	98	+#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	99	+ RSEQ_INJECT_ASM(1) \
	100	+ " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
	101	+ " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
	102	+ ", :lo12:" __rseq_str(cs_label) "\n" \
	103	+ " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
	104	+ __rseq_str(label) ":\n"
	105	+
	106	+#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
	107	+ " b 222f\n" \
	108	+ " .inst " __rseq_str(RSEQ_SIG) "\n" \
	109	+ __rseq_str(label) ":\n" \
	110	+ " b %l[" __rseq_str(abort_label) "]\n" \
	111	+ "222:\n"
	112	+
	113	+#define RSEQ_ASM_OP_STORE(value, var) \
	114	+ " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
	115	+
	116	+#define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
	117	+ " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
	118	+
	119	+#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
	120	+ RSEQ_ASM_OP_STORE(value, var) \
	121	+ __rseq_str(post_commit_label) ":\n"
	122	+
	123	+#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
	124	+ RSEQ_ASM_OP_STORE_RELEASE(value, var) \
	125	+ __rseq_str(post_commit_label) ":\n"
	126	+
	127	+#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
	128	+ " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
	129	+ " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
	130	+ ", %[" __rseq_str(expect) "]\n" \
	131	+ " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
	132	+
	133	+#define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
	134	+ " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
	135	+ " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
	136	+ ", %w[" __rseq_str(expect) "]\n" \
	137	+ " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
	138	+
	139	+#define RSEQ_ASM_OP_CMPNE(var, expect, label) \
	140	+ " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
	141	+ " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
	142	+ ", %[" __rseq_str(expect) "]\n" \
	143	+ " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
	144	+
	145	+#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
	146	+ RSEQ_INJECT_ASM(2) \
	147	+ RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
	148	+
	149	+#define RSEQ_ASM_OP_R_LOAD(var) \
	150	+ " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
	151	+
	152	+#define RSEQ_ASM_OP_R_STORE(var) \
	153	+ " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
	154	+
	155	+#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
	156	+ " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
	157	+ ", %[" __rseq_str(offset) "]]\n"
	158	+
	159	+#define RSEQ_ASM_OP_R_ADD(count) \
	160	+ " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
	161	+ ", %[" __rseq_str(count) "]\n"
	162	+
	163	+#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
	164	+ " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
	165	+ __rseq_str(post_commit_label) ":\n"
	166	+
	167	+#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
	168	+ " cbz %[" __rseq_str(len) "], 333f\n" \
	169	+ " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
	170	+ "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
	171	+ " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
	172	+ ", " RSEQ_ASM_TMP_REG_2 "]\n" \
	173	+ " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
	174	+ ", " RSEQ_ASM_TMP_REG_2 "]\n" \
	175	+ " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
	176	+ "333:\n"
	177	+
	178	+static inline __attribute__((always_inline))
	179	+int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
	180	+{
	181	+ RSEQ_INJECT_C(9)
	182	+
	183	+ __asm__ __volatile__ goto (
	184	+ RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
	185	+ RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	186	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	187	+ RSEQ_INJECT_ASM(3)
	188	+ RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
	189	+ RSEQ_INJECT_ASM(4)
	190	+#ifdef RSEQ_COMPARE_TWICE
	191	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	192	+ RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
	193	+#endif
	194	+ RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
	195	+ RSEQ_INJECT_ASM(5)
	196	+ RSEQ_ASM_DEFINE_ABORT(4, abort)
	197	+ : /* gcc asm goto does not allow outputs */
	198	+ : [cpu_id] "r" (cpu),
	199	+ [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
	200	+ [rseq_cs] "m" (__rseq_abi.rseq_cs),
	201	+ [v] "Qo" (*v),
	202	+ [expect] "r" (expect),
	203	+ [newv] "r" (newv)
	204	+ RSEQ_INJECT_INPUT
	205	+ : "memory", RSEQ_ASM_TMP_REG
	206	+ : abort, cmpfail
	207	+#ifdef RSEQ_COMPARE_TWICE
	208	+ , error1, error2
	209	+#endif
	210	+ );
	211	+
	212	+ return 0;
	213	+abort:
	214	+ RSEQ_INJECT_FAILED
	215	+ return -1;
	216	+cmpfail:
	217	+ return 1;
	218	+#ifdef RSEQ_COMPARE_TWICE
	219	+error1:
	220	+ rseq_bug("cpu_id comparison failed");
	221	+error2:
	222	+ rseq_bug("expected value comparison failed");
	223	+#endif
	224	+}
	225	+
	226	+static inline __attribute__((always_inline))
	227	+int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
	228	+ off_t voffp, intptr_t *load, int cpu)
	229	+{
	230	+ RSEQ_INJECT_C(9)
	231	+
	232	+ __asm__ __volatile__ goto (
	233	+ RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
	234	+ RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	235	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	236	+ RSEQ_INJECT_ASM(3)
	237	+ RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
	238	+ RSEQ_INJECT_ASM(4)
	239	+#ifdef RSEQ_COMPARE_TWICE
	240	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	241	+ RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
	242	+#endif
	243	+ RSEQ_ASM_OP_R_LOAD(v)
	244	+ RSEQ_ASM_OP_R_STORE(load)
	245	+ RSEQ_ASM_OP_R_LOAD_OFF(voffp)
	246	+ RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
	247	+ RSEQ_INJECT_ASM(5)
	248	+ RSEQ_ASM_DEFINE_ABORT(4, abort)
	249	+ : /* gcc asm goto does not allow outputs */
	250	+ : [cpu_id] "r" (cpu),
	251	+ [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
	252	+ [rseq_cs] "m" (__rseq_abi.rseq_cs),
	253	+ [v] "Qo" (*v),
	254	+ [expectnot] "r" (expectnot),
	255	+ [load] "Qo" (*load),
	256	+ [voffp] "r" (voffp)
	257	+ RSEQ_INJECT_INPUT
	258	+ : "memory", RSEQ_ASM_TMP_REG
	259	+ : abort, cmpfail
	260	+#ifdef RSEQ_COMPARE_TWICE
	261	+ , error1, error2
	262	+#endif
	263	+ );
	264	+ return 0;
	265	+abort:
	266	+ RSEQ_INJECT_FAILED
	267	+ return -1;
	268	+cmpfail:
	269	+ return 1;
	270	+#ifdef RSEQ_COMPARE_TWICE
	271	+error1:
	272	+ rseq_bug("cpu_id comparison failed");
	273	+error2:
	274	+ rseq_bug("expected value comparison failed");
	275	+#endif
	276	+}
	277	+
	278	+static inline __attribute__((always_inline))
	279	+int rseq_addv(intptr_t *v, intptr_t count, int cpu)
	280	+{
	281	+ RSEQ_INJECT_C(9)
	282	+
	283	+ __asm__ __volatile__ goto (
	284	+ RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
	285	+ RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	286	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	287	+ RSEQ_INJECT_ASM(3)
	288	+#ifdef RSEQ_COMPARE_TWICE
	289	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	290	+#endif
	291	+ RSEQ_ASM_OP_R_LOAD(v)
	292	+ RSEQ_ASM_OP_R_ADD(count)
	293	+ RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
	294	+ RSEQ_INJECT_ASM(4)
	295	+ RSEQ_ASM_DEFINE_ABORT(4, abort)
	296	+ : /* gcc asm goto does not allow outputs */
	297	+ : [cpu_id] "r" (cpu),
	298	+ [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
	299	+ [rseq_cs] "m" (__rseq_abi.rseq_cs),
	300	+ [v] "Qo" (*v),
	301	+ [count] "r" (count)
	302	+ RSEQ_INJECT_INPUT
	303	+ : "memory", RSEQ_ASM_TMP_REG
	304	+ : abort
	305	+#ifdef RSEQ_COMPARE_TWICE
	306	+ , error1
	307	+#endif
	308	+ );
	309	+ return 0;
	310	+abort:
	311	+ RSEQ_INJECT_FAILED
	312	+ return -1;
	313	+#ifdef RSEQ_COMPARE_TWICE
	314	+error1:
	315	+ rseq_bug("cpu_id comparison failed");
	316	+#endif
	317	+}
	318	+
	319	+static inline __attribute__((always_inline))
	320	+int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
	321	+ intptr_t *v2, intptr_t newv2,
	322	+ intptr_t newv, int cpu)
	323	+{
	324	+ RSEQ_INJECT_C(9)
	325	+
	326	+ __asm__ __volatile__ goto (
	327	+ RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
	328	+ RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	329	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	330	+ RSEQ_INJECT_ASM(3)
	331	+ RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
	332	+ RSEQ_INJECT_ASM(4)
	333	+#ifdef RSEQ_COMPARE_TWICE
	334	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	335	+ RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
	336	+#endif
	337	+ RSEQ_ASM_OP_STORE(newv2, v2)
	338	+ RSEQ_INJECT_ASM(5)
	339	+ RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
	340	+ RSEQ_INJECT_ASM(6)
	341	+ RSEQ_ASM_DEFINE_ABORT(4, abort)
	342	+ : /* gcc asm goto does not allow outputs */
	343	+ : [cpu_id] "r" (cpu),
	344	+ [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
	345	+ [rseq_cs] "m" (__rseq_abi.rseq_cs),
	346	+ [expect] "r" (expect),
	347	+ [v] "Qo" (*v),
	348	+ [newv] "r" (newv),
	349	+ [v2] "Qo" (*v2),
	350	+ [newv2] "r" (newv2)
	351	+ RSEQ_INJECT_INPUT
	352	+ : "memory", RSEQ_ASM_TMP_REG
	353	+ : abort, cmpfail
	354	+#ifdef RSEQ_COMPARE_TWICE
	355	+ , error1, error2
	356	+#endif
	357	+ );
	358	+
	359	+ return 0;
	360	+abort:
	361	+ RSEQ_INJECT_FAILED
	362	+ return -1;
	363	+cmpfail:
	364	+ return 1;
	365	+#ifdef RSEQ_COMPARE_TWICE
	366	+error1:
	367	+ rseq_bug("cpu_id comparison failed");
	368	+error2:
	369	+ rseq_bug("expected value comparison failed");
	370	+#endif
	371	+}
	372	+
	373	+static inline __attribute__((always_inline))
	374	+int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
	375	+ intptr_t *v2, intptr_t newv2,
	376	+ intptr_t newv, int cpu)
	377	+{
	378	+ RSEQ_INJECT_C(9)
	379	+
	380	+ __asm__ __volatile__ goto (
	381	+ RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
	382	+ RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	383	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	384	+ RSEQ_INJECT_ASM(3)
	385	+ RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
	386	+ RSEQ_INJECT_ASM(4)
	387	+#ifdef RSEQ_COMPARE_TWICE
	388	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	389	+ RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
	390	+#endif
	391	+ RSEQ_ASM_OP_STORE(newv2, v2)
	392	+ RSEQ_INJECT_ASM(5)
	393	+ RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
	394	+ RSEQ_INJECT_ASM(6)
	395	+ RSEQ_ASM_DEFINE_ABORT(4, abort)
	396	+ : /* gcc asm goto does not allow outputs */
	397	+ : [cpu_id] "r" (cpu),
	398	+ [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
	399	+ [rseq_cs] "m" (__rseq_abi.rseq_cs),
	400	+ [expect] "r" (expect),
	401	+ [v] "Qo" (*v),
	402	+ [newv] "r" (newv),
	403	+ [v2] "Qo" (*v2),
	404	+ [newv2] "r" (newv2)
	405	+ RSEQ_INJECT_INPUT
	406	+ : "memory", RSEQ_ASM_TMP_REG
	407	+ : abort, cmpfail
	408	+#ifdef RSEQ_COMPARE_TWICE
	409	+ , error1, error2
	410	+#endif
	411	+ );
	412	+
	413	+ return 0;
	414	+abort:
	415	+ RSEQ_INJECT_FAILED
	416	+ return -1;
	417	+cmpfail:
	418	+ return 1;
	419	+#ifdef RSEQ_COMPARE_TWICE
	420	+error1:
	421	+ rseq_bug("cpu_id comparison failed");
	422	+error2:
	423	+ rseq_bug("expected value comparison failed");
	424	+#endif
	425	+}
	426	+
	427	+static inline __attribute__((always_inline))
	428	+int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
	429	+ intptr_t *v2, intptr_t expect2,
	430	+ intptr_t newv, int cpu)
	431	+{
	432	+ RSEQ_INJECT_C(9)
	433	+
	434	+ __asm__ __volatile__ goto (
	435	+ RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
	436	+ RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	437	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	438	+ RSEQ_INJECT_ASM(3)
	439	+ RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
	440	+ RSEQ_INJECT_ASM(4)
	441	+ RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
	442	+ RSEQ_INJECT_ASM(5)
	443	+#ifdef RSEQ_COMPARE_TWICE
	444	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	445	+ RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
	446	+ RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
	447	+#endif
	448	+ RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
	449	+ RSEQ_INJECT_ASM(6)
	450	+ RSEQ_ASM_DEFINE_ABORT(4, abort)
	451	+ : /* gcc asm goto does not allow outputs */
	452	+ : [cpu_id] "r" (cpu),
	453	+ [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
	454	+ [rseq_cs] "m" (__rseq_abi.rseq_cs),
	455	+ [v] "Qo" (*v),
	456	+ [expect] "r" (expect),
	457	+ [v2] "Qo" (*v2),
	458	+ [expect2] "r" (expect2),
	459	+ [newv] "r" (newv)
	460	+ RSEQ_INJECT_INPUT
	461	+ : "memory", RSEQ_ASM_TMP_REG
	462	+ : abort, cmpfail
	463	+#ifdef RSEQ_COMPARE_TWICE
	464	+ , error1, error2, error3
	465	+#endif
	466	+ );
	467	+
	468	+ return 0;
	469	+abort:
	470	+ RSEQ_INJECT_FAILED
	471	+ return -1;
	472	+cmpfail:
	473	+ return 1;
	474	+#ifdef RSEQ_COMPARE_TWICE
	475	+error1:
	476	+ rseq_bug("cpu_id comparison failed");
	477	+error2:
	478	+ rseq_bug("expected value comparison failed");
	479	+error3:
	480	+ rseq_bug("2nd expected value comparison failed");
	481	+#endif
	482	+}
	483	+
	484	+static inline __attribute__((always_inline))
	485	+int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
	486	+ void dst, void src, size_t len,
	487	+ intptr_t newv, int cpu)
	488	+{
	489	+ RSEQ_INJECT_C(9)
	490	+
	491	+ __asm__ __volatile__ goto (
	492	+ RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
	493	+ RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	494	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	495	+ RSEQ_INJECT_ASM(3)
	496	+ RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
	497	+ RSEQ_INJECT_ASM(4)
	498	+#ifdef RSEQ_COMPARE_TWICE
	499	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	500	+ RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
	501	+#endif
	502	+ RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
	503	+ RSEQ_INJECT_ASM(5)
	504	+ RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
	505	+ RSEQ_INJECT_ASM(6)
	506	+ RSEQ_ASM_DEFINE_ABORT(4, abort)
	507	+ : /* gcc asm goto does not allow outputs */
	508	+ : [cpu_id] "r" (cpu),
	509	+ [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
	510	+ [rseq_cs] "m" (__rseq_abi.rseq_cs),
	511	+ [expect] "r" (expect),
	512	+ [v] "Qo" (*v),
	513	+ [newv] "r" (newv),
	514	+ [dst] "r" (dst),
	515	+ [src] "r" (src),
	516	+ [len] "r" (len)
	517	+ RSEQ_INJECT_INPUT
	518	+ : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
	519	+ : abort, cmpfail
	520	+#ifdef RSEQ_COMPARE_TWICE
	521	+ , error1, error2
	522	+#endif
	523	+ );
	524	+
	525	+ return 0;
	526	+abort:
	527	+ RSEQ_INJECT_FAILED
	528	+ return -1;
	529	+cmpfail:
	530	+ return 1;
	531	+#ifdef RSEQ_COMPARE_TWICE
	532	+error1:
	533	+ rseq_bug("cpu_id comparison failed");
	534	+error2:
	535	+ rseq_bug("expected value comparison failed");
	536	+#endif
	537	+}
	538	+
	539	+static inline __attribute__((always_inline))
	540	+int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
	541	+ void dst, void src, size_t len,
	542	+ intptr_t newv, int cpu)
	543	+{
	544	+ RSEQ_INJECT_C(9)
	545	+
	546	+ __asm__ __volatile__ goto (
	547	+ RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
	548	+ RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	549	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	550	+ RSEQ_INJECT_ASM(3)
	551	+ RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
	552	+ RSEQ_INJECT_ASM(4)
	553	+#ifdef RSEQ_COMPARE_TWICE
	554	+ RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	555	+ RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
	556	+#endif
	557	+ RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
	558	+ RSEQ_INJECT_ASM(5)
	559	+ RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
	560	+ RSEQ_INJECT_ASM(6)
	561	+ RSEQ_ASM_DEFINE_ABORT(4, abort)
	562	+ : /* gcc asm goto does not allow outputs */
	563	+ : [cpu_id] "r" (cpu),
	564	+ [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
	565	+ [rseq_cs] "m" (__rseq_abi.rseq_cs),
	566	+ [expect] "r" (expect),
	567	+ [v] "Qo" (*v),
	568	+ [newv] "r" (newv),
	569	+ [dst] "r" (dst),
	570	+ [src] "r" (src),
	571	+ [len] "r" (len)
	572	+ RSEQ_INJECT_INPUT
	573	+ : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
	574	+ : abort, cmpfail
	575	+#ifdef RSEQ_COMPARE_TWICE
	576	+ , error1, error2
	577	+#endif
	578	+ );
	579	+
	580	+ return 0;
	581	+abort:
	582	+ RSEQ_INJECT_FAILED
	583	+ return -1;
	584	+cmpfail:
	585	+ return 1;
	586	+#ifdef RSEQ_COMPARE_TWICE
	587	+error1:
	588	+ rseq_bug("cpu_id comparison failed");
	589	+error2:
	590	+ rseq_bug("expected value comparison failed");
	591	+#endif
	592	+}
	593	+
	594	+#endif /* !RSEQ_SKIP_FASTPATH */

tools/testing/selftests/rseq/rseq.h+2−0

@@ -71,6 +71,8 @@ extern __thread volatile struct rseq __rseq_abi; · 70 unmodified lines
71	71	#include <rseq-x86.h>
72	72	#elif defined(__ARMEL__)
73	73	#include <rseq-arm.h>
	74	+#elif defined (__AARCH64EL__)
	75	+#include <rseq-arm64.h>
74	76	#elif defined(__PPC__)
75	77	#include <rseq-ppc.h>
76	78	#elif defined(__mips__)

virt/kvm/arm/aarch32.c+10−10

@@ -108,9 +108,9 @@ static void __hyp_text kvm_adjust_itstate(struct kvm_vcpu *vcpu) · 107 unmodified lines
108	108	{
109	109	unsigned long itbits, cond;
110	110	unsigned long cpsr = *vcpu_cpsr(vcpu);
111		− bool is_arm = !(cpsr & COMPAT_PSR_T_BIT);
	111	+ bool is_arm = !(cpsr & PSR_AA32_T_BIT);
112	112
113		− if (is_arm \|\| !(cpsr & COMPAT_PSR_IT_MASK))
	113	+ if (is_arm \|\| !(cpsr & PSR_AA32_IT_MASK))
114	114	return;
115	115
116	116	cond = (cpsr & 0xe000) >> 13;
@@ -123,7 +123,7 @@ static void __hyp_text kvm_adjust_itstate(struct kvm_vcpu *vcpu) · 6 unmodified lines
123	123	else
124	124	itbits = (itbits << 1) & 0x1f;
125	125
126		− cpsr &= ~COMPAT_PSR_IT_MASK;
	126	+ cpsr &= ~PSR_AA32_IT_MASK;
127	127	cpsr \|= cond << 13;
128	128	cpsr \|= (itbits & 0x1c) << (10 - 2);
129	129	cpsr \|= (itbits & 0x3) << 25;
@@ -138,7 +138,7 @@ void __hyp_text kvm_skip_instr32(struct kvm_vcpu *vcpu, bool is_wide_instr) · 8 unmodified lines
138	138	{
139	139	bool is_thumb;
140	140
141		− is_thumb = !!(*vcpu_cpsr(vcpu) & COMPAT_PSR_T_BIT);
	141	+ is_thumb = !!(*vcpu_cpsr(vcpu) & PSR_AA32_T_BIT);
142	142	if (is_thumb && !is_wide_instr)
143	143	*vcpu_pc(vcpu) += 2;
144	144	else
@@ -164,16 +164,16 @@ static void prepare_fault32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) · 19 unmodified lines
164	164	{
165	165	unsigned long cpsr;
166	166	unsigned long new_spsr_value = *vcpu_cpsr(vcpu);
167		− bool is_thumb = (new_spsr_value & COMPAT_PSR_T_BIT);
	167	+ bool is_thumb = (new_spsr_value & PSR_AA32_T_BIT);
168	168	u32 return_offset = return_offsets[vect_offset >> 2][is_thumb];
169	169	u32 sctlr = vcpu_cp15(vcpu, c1_SCTLR);
170	170
171		− cpsr = mode \| COMPAT_PSR_I_BIT;
	171	+ cpsr = mode \| PSR_AA32_I_BIT;
172	172
173	173	if (sctlr & (1 << 30))
174		− cpsr \|= COMPAT_PSR_T_BIT;
	174	+ cpsr \|= PSR_AA32_T_BIT;
175	175	if (sctlr & (1 << 25))
176		− cpsr \|= COMPAT_PSR_E_BIT;
	176	+ cpsr \|= PSR_AA32_E_BIT;
177	177
178	178	*vcpu_cpsr(vcpu) = cpsr;
179	179
@@ -192,7 +192,7 @@ static void prepare_fault32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) · 12 unmodified lines
192	192
193	193	void kvm_inject_undef32(struct kvm_vcpu *vcpu)
194	194	{
195		− prepare_fault32(vcpu, COMPAT_PSR_MODE_UND, 4);
	195	+ prepare_fault32(vcpu, PSR_AA32_MODE_UND, 4);
196	196	}
197	197
198	198	/*
@@ -216,7 +216,7 @@ static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, · 17 unmodified lines
216	216	fsr = &vcpu_cp15(vcpu, c5_DFSR);
217	217	}
218	218
219		− prepare_fault32(vcpu, COMPAT_PSR_MODE_ABT \| COMPAT_PSR_A_BIT, vect_offset);
	219	+ prepare_fault32(vcpu, PSR_AA32_MODE_ABT \| PSR_AA32_A_BIT, vect_offset);
220	220
221	221	*far = addr;
222	222