Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux

Pull arm64 updates from Catalin Marinas: - Expose tag address bits in siginfo. The original arm64 ABI did not expose any of the bits 63:56 of a tagged address in siginfo. In the presence of user ASAN or MTE, this information may be useful. The implementation is generic to other architectures supporting tags (like SPARC ADI, subject to wiring up the arch code). The user will have to opt in via sigaction(SA_EXPOSE_TAGBITS) so that the extra bits, if available, become visible in si_addr. - Default to 32-bit wide ZONE_DMA. Previously, ZONE_DMA was set to the lowest 1GB to cope with the Raspberry Pi 4 limitations, to the detriment of other platforms. With these changes, the kernel scans the Device Tree dma-ranges and the ACPI IORT information before deciding on a smaller ZONE_DMA. - Strengthen READ_ONCE() to acquire when CONFIG_LTO=y. When building with LTO, there is an increased risk of the compiler converting an address dependency headed by a READ_ONCE() invocation into a control dependency and consequently allowing for harmful reordering by the CPU. - Add CPPC FFH support using arm64 AMU counters. - set_fs() removal on arm64. This renders the User Access Override (UAO) ARMv8 feature unnecessary. - Perf updates: PMU driver for the ARM DMC-620 memory controller, sysfs identifier file for SMMUv3, stop event counters support for i.MX8MP, enable the perf events-based hard lockup detector. - Reorganise the kernel VA space slightly so that 52-bit VA configurations can use more virtual address space. - Improve the robustness of the arm64 memory offline event notifier. - Pad the Image header to 64K following the EFI header definition updated recently to increase the section alignment to 64K. - Support CONFIG_CMDLINE_EXTEND on arm64. - Do not use tagged PC in the kernel (TCR_EL1.TBID1==1), freeing up 8 bits for PtrAuth. - Switch to vmapped shadow call stacks. - Miscellaneous clean-ups. * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (78 commits) perf/imx_ddr: Add system PMU identifier for userspace bindings: perf: imx-ddr: add compatible string arm64: Fix build failure when HARDLOCKUP_DETECTOR_PERF is enabled arm64: mte: fix prctl(PR_GET_TAGGED_ADDR_CTRL) if TCF0=NONE arm64: mark __system_matches_cap as __maybe_unused arm64: uaccess: remove vestigal UAO support arm64: uaccess: remove redundant PAN toggling arm64: uaccess: remove addr_limit_user_check() arm64: uaccess: remove set_fs() arm64: uaccess cleanup macro naming arm64: uaccess: split user/kernel routines arm64: uaccess: refactor __{get,put}_user arm64: uaccess: simplify __copy_user_flushcache() arm64: uaccess: rename privileged uaccess routines arm64: sdei: explicitly simulate PAN/UAO entry arm64: sdei: move uaccess logic to arch/arm64/ arm64: head.S: always initialize PSTATE arm64: head.S: cleanup SCTLR_ELx initialization arm64: head.S: rename el2_setup -> init_kernel_el arm64: add C wrappers for SET_PSTATE_*() ...

Linus Torvalds committed 5 years ago

Commit 0ca2ce82 parents 5865924 d889797114 files · +2603 −1314Browse files

Unified · Split

114 files

Documentation/arm64/kasan-offsets.sh +1 −2
Documentation/arm64/memory.rst +21 −24
Documentation/arm64/tagged-pointers.rst +19 −6
Documentation/devicetree/bindings/perf/fsl-imx-ddr.yaml +3 −0
arch/alpha/include/uapi/asm/signal.h +0 −14
arch/arm/include/asm/signal.h +2 −0
arch/arm/include/uapi/asm/signal.h +3 −24
arch/arm64/Kconfig +37 −33
arch/arm64/include/asm/alternative-macros.h +217 −0
arch/arm64/include/asm/alternative.h +2 −265
arch/arm64/include/asm/asm-uaccess.h +32 −3
arch/arm64/include/asm/cpucaps.h +2 −3
arch/arm64/include/asm/cpufeature.h +22 −1
arch/arm64/include/asm/exception.h +1 −1
arch/arm64/include/asm/exec.h +0 −1
arch/arm64/include/asm/futex.h +4 −4
arch/arm64/include/asm/insn.h +1 −2
arch/arm64/include/asm/kernel-pgtable.h +0 −6
arch/arm64/include/asm/kprobes.h +0 −7
arch/arm64/include/asm/memory.h +10 −12
arch/arm64/include/asm/mmu_context.h +3 −3
arch/arm64/include/asm/pgtable-hwdef.h +1 −0
arch/arm64/include/asm/pgtable.h +2 −1
arch/arm64/include/asm/processor.h +1 −3
arch/arm64/include/asm/ptrace.h +6 −2
arch/arm64/include/asm/rwonce.h +73 −0
arch/arm64/include/asm/signal.h +25 −0
arch/arm64/include/asm/sysreg.h +16 −6
arch/arm64/include/asm/system_misc.h +1 −1
arch/arm64/include/asm/thread_info.h +2 −8
arch/arm64/include/asm/topology.h +3 −1
arch/arm64/include/asm/traps.h +3 −3
arch/arm64/include/asm/uaccess.h +75 −103
arch/arm64/kernel/Makefile +0 −1
arch/arm64/kernel/alternative.c +4 −3
arch/arm64/kernel/armv8_deprecated.c +2 −2
arch/arm64/kernel/asm-offsets.c +1 −2
arch/arm64/kernel/cpufeature.c +22 −37
arch/arm64/kernel/debug-monitors.c +2 −3
arch/arm64/kernel/efi-header.S +49 −35
arch/arm64/kernel/entry-common.c +0 −2
arch/arm64/kernel/entry.S +10 −21
arch/arm64/kernel/head.S +33 −37
arch/arm64/kernel/irq.c +21 −0
arch/arm64/kernel/kaslr.c +18 −8
arch/arm64/kernel/mte.c +2 −1
arch/arm64/kernel/perf_event.c +39 −2
arch/arm64/kernel/probes/kprobes.c +12 −41
arch/arm64/kernel/process.c +8 −21
arch/arm64/kernel/proton-pack.c +3 −2
arch/arm64/kernel/ptrace.c +2 −5
arch/arm64/kernel/scs.c +0 −16
arch/arm64/kernel/sdei.c +108 −11
arch/arm64/kernel/setup.c +3 −3
arch/arm64/kernel/signal.c +0 −3
arch/arm64/kernel/sleep.S +1 −1
arch/arm64/kernel/smp.c +7 −8
arch/arm64/kernel/suspend.c +0 −1
arch/arm64/kernel/sys_compat.c +2 −3
arch/arm64/kernel/syscall.c +1 −1
arch/arm64/kernel/topology.c +110 −26
arch/arm64/kernel/traps.c +15 −14
arch/arm64/kernel/vdso/Makefile +1 −1
arch/arm64/kernel/vdso32/Makefile +1 −1
arch/arm64/kernel/vmlinux.lds.S +5 −7
arch/arm64/lib/clear_user.S +4 −4
arch/arm64/lib/copy_from_user.S +4 −4
arch/arm64/lib/copy_in_user.S +8 −8
arch/arm64/lib/copy_to_user.S +4 −4
arch/arm64/lib/mte.S +3 −3
arch/arm64/lib/uaccess_flushcache.c +1 −3
arch/arm64/mm/fault.c +50 −38
arch/arm64/mm/init.c +35 −21
arch/arm64/mm/mmu.c +117 −32
arch/arm64/mm/proc.S +3 −3
arch/h8300/include/uapi/asm/signal.h +0 −24
arch/ia64/include/uapi/asm/signal.h +0 −24
arch/m68k/include/uapi/asm/signal.h +0 −24
arch/mips/include/uapi/asm/signal.h +0 −12
arch/parisc/include/asm/signal.h +2 −0
arch/parisc/include/uapi/asm/signal.h +1 −33
arch/powerpc/include/uapi/asm/signal.h +0 −24
arch/s390/include/uapi/asm/signal.h +0 −24
arch/sparc/include/uapi/asm/signal.h +1 −3
arch/x86/include/uapi/asm/signal.h +0 −24
arch/x86/kernel/signal_compat.c +0 −7
arch/xtensa/include/uapi/asm/signal.h +0 −24
drivers/acpi/arm64/iort.c +55 −0
drivers/firmware/arm_sdei.c +0 −14
drivers/of/address.c +42 −0
drivers/of/unittest.c +21 −0
drivers/perf/Kconfig +7 −0
drivers/perf/Makefile +1 −0
drivers/perf/arm_dmc620_pmu.c +748 −0
drivers/perf/arm_dsu_pmu.c +0 −3
drivers/perf/arm_pmu.c +5 −0
drivers/perf/arm_smmuv3_pmu.c +39 −0
drivers/perf/fsl_imx8_ddr_perf.c +109 −26
drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c +16 −0
drivers/perf/hisilicon/hisi_uncore_hha_pmu.c +16 −0
drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c +16 −0
drivers/perf/hisilicon/hisi_uncore_pmu.c +10 −0
drivers/perf/hisilicon/hisi_uncore_pmu.h +7 −0
include/linux/acpi_iort.h +4 −0
include/linux/mmzone.h +0 −20
include/linux/of.h +7 −0
include/linux/perf/arm_pmu.h +2 −0
include/linux/scs.h +6 −10
include/linux/signal.h +14 −0
include/linux/signal_types.h +12 −0
include/uapi/asm-generic/signal-defs.h +63 −0
include/uapi/asm-generic/signal.h +0 −29
kernel/scs.c +60 −11
kernel/signal.c +40 −0

Documentation/arm64/kasan-offsets.sh+1−2

@@ -1,12 +1,11 @@
1	1	#!/bin/sh
2	2
3	3	# Print out the KASAN_SHADOW_OFFSETS required to place the KASAN SHADOW
4		−# start address at the mid-point of the kernel VA space
	4	+# start address at the top of the linear region
5	5
6	6	print_kasan_offset () {
7	7	printf "%02d\t" $1
8	8	printf "0x%08x00000000\n" $(( (0xffffffff & (-1 << ($1 - 1 - 32))) \
9		− + (1 << ($1 - 32 - $2)) \
10	9	- (1 << (64 - 32 - $2)) ))
11	10	}
12	11

Documentation/arm64/memory.rst+21−24

@@ -32,17 +32,16 @@ AArch64 Linux memory layout with 4KB pages + 4 levels (48-bit):: · 31 unmodified lines
32	32	-----------------------------------------------------------------------
33	33	0000000000000000 0000ffffffffffff 256TB user
34	34	ffff000000000000 ffff7fffffffffff 128TB kernel logical memory map
35		− ffff800000000000 ffff9fffffffffff 32TB kasan shadow region
36		− ffffa00000000000 ffffa00007ffffff 128MB bpf jit region
37		− ffffa00008000000 ffffa0000fffffff 128MB modules
38		− ffffa00010000000 fffffdffbffeffff ~93TB vmalloc
39		− fffffdffbfff0000 fffffdfffe5f8fff ~998MB [guard region]
40		− fffffdfffe5f9000 fffffdfffe9fffff 4124KB fixed mappings
41		− fffffdfffea00000 fffffdfffebfffff 2MB [guard region]
42		− fffffdfffec00000 fffffdffffbfffff 16MB PCI I/O space
43		− fffffdffffc00000 fffffdffffdfffff 2MB [guard region]
44		− fffffdffffe00000 ffffffffffdfffff 2TB vmemmap
45		− ffffffffffe00000 ffffffffffffffff 2MB [guard region]
	35	+ [ffff600000000000 ffff7fffffffffff] 32TB [kasan shadow region]
	36	+ ffff800000000000 ffff800007ffffff 128MB bpf jit region
	37	+ ffff800008000000 ffff80000fffffff 128MB modules
	38	+ ffff800010000000 fffffbffefffffff 124TB vmalloc
	39	+ fffffbfff0000000 fffffbfffdffffff 224MB fixed mappings (top down)
	40	+ fffffbfffe000000 fffffbfffe7fffff 8MB [guard region]
	41	+ fffffbfffe800000 fffffbffff7fffff 16MB PCI I/O space
	42	+ fffffbffff800000 fffffbffffffffff 8MB [guard region]
	43	+ fffffc0000000000 fffffdffffffffff 2TB vmemmap
	44	+ fffffe0000000000 ffffffffffffffff 2TB [guard region]
46	45
47	46
48	47	AArch64 Linux memory layout with 64KB pages + 3 levels (52-bit with HW support)::
@@ -50,19 +49,17 @@ AArch64 Linux memory layout with 64KB pages + 3 levels (52-bit with HW support): · 1 unmodified line
50	49	Start End Size Use
51	50	-----------------------------------------------------------------------
52	51	0000000000000000 000fffffffffffff 4PB user
53		− fff0000000000000 fff7ffffffffffff 2PB kernel logical memory map
54		− fff8000000000000 fffd9fffffffffff 1440TB [gap]
55		− fffda00000000000 ffff9fffffffffff 512TB kasan shadow region
56		− ffffa00000000000 ffffa00007ffffff 128MB bpf jit region
57		− ffffa00008000000 ffffa0000fffffff 128MB modules
58		− ffffa00010000000 fffff81ffffeffff ~88TB vmalloc
59		− fffff81fffff0000 fffffc1ffe58ffff ~3TB [guard region]
60		− fffffc1ffe590000 fffffc1ffe9fffff 4544KB fixed mappings
61		− fffffc1ffea00000 fffffc1ffebfffff 2MB [guard region]
62		− fffffc1ffec00000 fffffc1fffbfffff 16MB PCI I/O space
63		− fffffc1fffc00000 fffffc1fffdfffff 2MB [guard region]
64		− fffffc1fffe00000 ffffffffffdfffff 3968GB vmemmap
65		− ffffffffffe00000 ffffffffffffffff 2MB [guard region]
	52	+ fff0000000000000 ffff7fffffffffff ~4PB kernel logical memory map
	53	+ [fffd800000000000 ffff7fffffffffff] 512TB [kasan shadow region]
	54	+ ffff800000000000 ffff800007ffffff 128MB bpf jit region
	55	+ ffff800008000000 ffff80000fffffff 128MB modules
	56	+ ffff800010000000 fffffbffefffffff 124TB vmalloc
	57	+ fffffbfff0000000 fffffbfffdffffff 224MB fixed mappings (top down)
	58	+ fffffbfffe000000 fffffbfffe7fffff 8MB [guard region]
	59	+ fffffbfffe800000 fffffbffff7fffff 16MB PCI I/O space
	60	+ fffffbffff800000 fffffbffffffffff 8MB [guard region]
	61	+ fffffc0000000000 ffffffdfffffffff ~4TB vmemmap
	62	+ ffffffe000000000 ffffffffffffffff 128GB [guard region]
66	63
67	64
68	65	Translation table lookup with 4KB pages::

Documentation/arm64/tagged-pointers.rst+19−6

@@ -53,12 +53,25 @@ visibility. · 52 unmodified lines
53	53	Preserving tags
54	54	---------------
55	55
56		−Non-zero tags are not preserved when delivering signals. This means that
57		−signal handlers in applications making use of tags cannot rely on the
58		−tag information for user virtual addresses being maintained for fields
59		−inside siginfo_t. One exception to this rule is for signals raised in
60		−response to watchpoint debug exceptions, where the tag information will
61		−be preserved.
	56	+When delivering signals, non-zero tags are not preserved in
	57	+siginfo.si_addr unless the flag SA_EXPOSE_TAGBITS was set in
	58	+sigaction.sa_flags when the signal handler was installed. This means
	59	+that signal handlers in applications making use of tags cannot rely
	60	+on the tag information for user virtual addresses being maintained
	61	+in these fields unless the flag was set.
	62	+
	63	+Due to architecture limitations, bits 63:60 of the fault address
	64	+are not preserved in response to synchronous tag check faults
	65	+(SEGV_MTESERR) even if SA_EXPOSE_TAGBITS was set. Applications should
	66	+treat the values of these bits as undefined in order to accommodate
	67	+future architecture revisions which may preserve the bits.
	68	+
	69	+For signals raised in response to watchpoint debug exceptions, the
	70	+tag information will be preserved regardless of the SA_EXPOSE_TAGBITS
	71	+flag setting.
	72	+
	73	+Non-zero tags are never preserved in sigcontext.fault_address
	74	+regardless of the SA_EXPOSE_TAGBITS flag setting.
62	75
63	76	The architecture prevents the use of a tagged PC, so the upper byte will
64	77	be set to a sign-extension of bit 55 on exception return.

Documentation/devicetree/bindings/perf/fsl-imx-ddr.yaml+3−0

@@ -15,6 +15,9 @@ properties: · 14 unmodified lines
15	15	- enum:
16	16	- fsl,imx8-ddr-pmu
17	17	- fsl,imx8m-ddr-pmu
	18	+ - fsl,imx8mq-ddr-pmu
	19	+ - fsl,imx8mm-ddr-pmu
	20	+ - fsl,imx8mn-ddr-pmu
18	21	- fsl,imx8mp-ddr-pmu
19	22	- items:
20	23	- enum:

arch/alpha/include/uapi/asm/signal.h+0−14

@@ -60,20 +60,6 @@ typedef unsigned long sigset_t; · 59 unmodified lines
60	60	#define SIGRTMIN 32
61	61	#define SIGRTMAX _NSIG
62	62
63		−/*
64		− * SA_FLAGS values:
65		− *
66		− * SA_ONSTACK indicates that a registered stack_t will be used.
67		− * SA_RESTART flag to get restarting signals (which were the default long ago)
68		− * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop.
69		− * SA_RESETHAND clears the handler when the signal is delivered.
70		− * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies.
71		− * SA_NODEFER prevents the current signal from being masked in the handler.
72		− *
73		− * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single
74		− * Unix names RESETHAND and NODEFER respectively.
75		− */
76		−
77	63	#define SA_ONSTACK 0x00000001
78	64	#define SA_RESTART 0x00000002
79	65	#define SA_NOCLDSTOP 0x00000004

arch/arm/include/asm/signal.h+2−0

@@ -17,6 +17,8 @@ typedef struct { · 16 unmodified lines
17	17	unsigned long sig[_NSIG_WORDS];
18	18	} sigset_t;
19	19
	20	+#define __ARCH_UAPI_SA_FLAGS (SA_THIRTYTWO \| SA_RESTORER)
	21	+
20	22	#define __ARCH_HAS_SA_RESTORER
21	23
22	24	#include <asm/sigcontext.h>

arch/arm/include/uapi/asm/signal.h+3−24

@@ -60,33 +60,12 @@ typedef unsigned long sigset_t; · 59 unmodified lines
60	60	#define SIGSWI 32
61	61
62	62	/*
63		− * SA_FLAGS values:
64		− *
65		− * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop.
66		− * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies.
67		− * SA_SIGINFO deliver the signal with SIGINFO structs
68		− * SA_THIRTYTWO delivers the signal in 32-bit mode, even if the task
69		− * is running in 26-bit.
70		− * SA_ONSTACK allows alternate signal stacks (see sigaltstack(2)).
71		− * SA_RESTART flag to get restarting signals (which were the default long ago)
72		− * SA_NODEFER prevents the current signal from being masked in the handler.
73		− * SA_RESETHAND clears the handler when the signal is delivered.
74		− *
75		− * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single
76		− * Unix names RESETHAND and NODEFER respectively.
	63	+ * SA_THIRTYTWO historically meant deliver the signal in 32-bit mode, even if
	64	+ * the task is running in 26-bit. But since the kernel no longer supports
	65	+ * 26-bit mode, the flag has no effect.
77	66	*/
78		−#define SA_NOCLDSTOP 0x00000001
79		−#define SA_NOCLDWAIT 0x00000002
80		−#define SA_SIGINFO 0x00000004
81	67	#define SA_THIRTYTWO 0x02000000
82	68	#define SA_RESTORER 0x04000000
83		−#define SA_ONSTACK 0x08000000
84		−#define SA_RESTART 0x10000000
85		−#define SA_NODEFER 0x40000000
86		−#define SA_RESETHAND 0x80000000
87		−
88		−#define SA_NOMASK SA_NODEFER
89		−#define SA_ONESHOT SA_RESETHAND
90	69
91	70	#define MINSIGSTKSZ 2048
92	71	#define SIGSTKSZ 8192

arch/arm64/Kconfig+37−33

@@ -171,6 +171,8 @@ config ARM64 · 170 unmodified lines
171	171	select HAVE_NMI
172	172	select HAVE_PATA_PLATFORM
173	173	select HAVE_PERF_EVENTS
	174	+ select HAVE_PERF_EVENTS_NMI if ARM64_PSEUDO_NMI && HW_PERF_EVENTS
	175	+ select HAVE_HARDLOCKUP_DETECTOR_PERF if PERF_EVENTS && HAVE_PERF_EVENTS_NMI
174	176	select HAVE_PERF_REGS
175	177	select HAVE_PERF_USER_STACK_DUMP
176	178	select HAVE_REGS_AND_STACK_ACCESS_API
@@ -196,7 +198,6 @@ config ARM64 · 19 unmodified lines
196	198	select PCI_SYSCALL if PCI
197	199	select POWER_RESET
198	200	select POWER_SUPPLY
199		− select SET_FS
200	201	select SPARSE_IRQ
201	202	select SWIOTLB
202	203	select SYSCTL_EXCEPTION_TRACE
@@ -332,16 +333,16 @@ config BROKEN_GAS_INST · 129 unmodified lines
332	333	config KASAN_SHADOW_OFFSET
333	334	hex
334	335	depends on KASAN
335		− default 0xdfffa00000000000 if (ARM64_VA_BITS_48 \|\| ARM64_VA_BITS_52) && !KASAN_SW_TAGS
336		− default 0xdfffd00000000000 if ARM64_VA_BITS_47 && !KASAN_SW_TAGS
337		− default 0xdffffe8000000000 if ARM64_VA_BITS_42 && !KASAN_SW_TAGS
338		− default 0xdfffffd000000000 if ARM64_VA_BITS_39 && !KASAN_SW_TAGS
339		− default 0xdffffffa00000000 if ARM64_VA_BITS_36 && !KASAN_SW_TAGS
340		− default 0xefff900000000000 if (ARM64_VA_BITS_48 \|\| ARM64_VA_BITS_52) && KASAN_SW_TAGS
341		− default 0xefffc80000000000 if ARM64_VA_BITS_47 && KASAN_SW_TAGS
342		− default 0xeffffe4000000000 if ARM64_VA_BITS_42 && KASAN_SW_TAGS
343		− default 0xefffffc800000000 if ARM64_VA_BITS_39 && KASAN_SW_TAGS
344		− default 0xeffffff900000000 if ARM64_VA_BITS_36 && KASAN_SW_TAGS
	336	+ default 0xdfff800000000000 if (ARM64_VA_BITS_48 \|\| ARM64_VA_BITS_52) && !KASAN_SW_TAGS
	337	+ default 0xdfffc00000000000 if ARM64_VA_BITS_47 && !KASAN_SW_TAGS
	338	+ default 0xdffffe0000000000 if ARM64_VA_BITS_42 && !KASAN_SW_TAGS
	339	+ default 0xdfffffc000000000 if ARM64_VA_BITS_39 && !KASAN_SW_TAGS
	340	+ default 0xdffffff800000000 if ARM64_VA_BITS_36 && !KASAN_SW_TAGS
	341	+ default 0xefff800000000000 if (ARM64_VA_BITS_48 \|\| ARM64_VA_BITS_52) && KASAN_SW_TAGS
	342	+ default 0xefffc00000000000 if ARM64_VA_BITS_47 && KASAN_SW_TAGS
	343	+ default 0xeffffe0000000000 if ARM64_VA_BITS_42 && KASAN_SW_TAGS
	344	+ default 0xefffffc000000000 if ARM64_VA_BITS_39 && KASAN_SW_TAGS
	345	+ default 0xeffffff800000000 if ARM64_VA_BITS_36 && KASAN_SW_TAGS
345	346	default 0xffffffffffffffff
346	347
347	348	source "arch/arm64/Kconfig.platforms"
@@ -1389,6 +1390,9 @@ config ARM64_PAN · 1041 unmodified lines
1389	1390	The feature is detected at runtime, and will remain as a 'nop'
1390	1391	instruction if the cpu does not implement the feature.
1391	1392
	1393	+config AS_HAS_LDAPR
	1394	+ def_bool $(as-instr,.arch_extension rcpc)
	1395	+
1392	1396	config ARM64_LSE_ATOMICS
1393	1397	bool
1394	1398	default ARM64_USE_LSE_ATOMICS
@@ -1426,27 +1430,6 @@ endmenu · 31 unmodified lines
1426	1430
1427	1431	menu "ARMv8.2 architectural features"
1428	1432
1429		−config ARM64_UAO
1430		− bool "Enable support for User Access Override (UAO)"
1431		− default y
1432		− help
1433		− User Access Override (UAO; part of the ARMv8.2 Extensions)
1434		− causes the 'unprivileged' variant of the load/store instructions to
1435		− be overridden to be privileged.
1436		−
1437		− This option changes get_user() and friends to use the 'unprivileged'
1438		− variant of the load/store instructions. This ensures that user-space
1439		− really did have access to the supplied memory. When addr_limit is
1440		− set to kernel memory the UAO bit will be set, allowing privileged
1441		− access to kernel memory.
1442		−
1443		− Choosing this option will cause copy_to_user() et al to use user-space
1444		− memory permissions.
1445		−
1446		− The feature is detected at runtime, the kernel will use the
1447		− regular load/store instructions if the cpu does not implement the
1448		− feature.
1449		−
1450	1433	config ARM64_PMEM
1451	1434	bool "Enable support for persistent memory"
1452	1435	select ARCH_HAS_PMEM_API
@@ -1847,15 +1830,36 @@ config CMDLINE · 394 unmodified lines
1847	1830	entering them here. As a minimum, you should specify the the
1848	1831	root device (e.g. root=/dev/nfs).
1849	1832
	1833	+choice
	1834	+ prompt "Kernel command line type" if CMDLINE != ""
	1835	+ default CMDLINE_FROM_BOOTLOADER
	1836	+ help
	1837	+ Choose how the kernel will handle the provided default kernel
	1838	+ command line string.
	1839	+
	1840	+config CMDLINE_FROM_BOOTLOADER
	1841	+ bool "Use bootloader kernel arguments if available"
	1842	+ help
	1843	+ Uses the command-line options passed by the boot loader. If
	1844	+ the boot loader doesn't provide any, the default kernel command
	1845	+ string provided in CMDLINE will be used.
	1846	+
	1847	+config CMDLINE_EXTEND
	1848	+ bool "Extend bootloader kernel arguments"
	1849	+ help
	1850	+ The command-line arguments provided by the boot loader will be
	1851	+ appended to the default kernel command string.
	1852	+
1850	1853	config CMDLINE_FORCE
1851	1854	bool "Always use the default kernel command string"
1852		− depends on CMDLINE != ""
1853	1855	help
1854	1856	Always use the default kernel command string, even if the boot
1855	1857	loader passes other arguments to the kernel.
1856	1858	This is useful if you cannot or don't want to change the
1857	1859	command-line options your boot loader passes to the kernel.
1858	1860
	1861	+endchoice
	1862	+
1859	1863	config EFI_STUB
1860	1864	bool
1861	1865

arch/arm64/include/asm/alternative-macros.h+217−0

@@ -0,0 +1,217 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef __ASM_ALTERNATIVE_MACROS_H
	3	+#define __ASM_ALTERNATIVE_MACROS_H
	4	+
	5	+#include <asm/cpucaps.h>
	6	+
	7	+#define ARM64_CB_PATCH ARM64_NCAPS
	8	+
	9	+/* A64 instructions are always 32 bits. */
	10	+#define AARCH64_INSN_SIZE 4
	11	+
	12	+#ifndef __ASSEMBLY__
	13	+
	14	+#include <linux/stringify.h>
	15	+
	16	+#define ALTINSTR_ENTRY(feature) \
	17	+ " .word 661b - .\n" /* label */ \
	18	+ " .word 663f - .\n" /* new instruction */ \
	19	+ " .hword " __stringify(feature) "\n" /* feature bit */ \
	20	+ " .byte 662b-661b\n" /* source len */ \
	21	+ " .byte 664f-663f\n" /* replacement len */
	22	+
	23	+#define ALTINSTR_ENTRY_CB(feature, cb) \
	24	+ " .word 661b - .\n" /* label */ \
	25	+ " .word " __stringify(cb) "- .\n" /* callback */ \
	26	+ " .hword " __stringify(feature) "\n" /* feature bit */ \
	27	+ " .byte 662b-661b\n" /* source len */ \
	28	+ " .byte 664f-663f\n" /* replacement len */
	29	+
	30	+/*
	31	+ * alternative assembly primitive:
	32	+ *
	33	+ * If any of these .org directive fail, it means that insn1 and insn2
	34	+ * don't have the same length. This used to be written as
	35	+ *
	36	+ * .if ((664b-663b) != (662b-661b))
	37	+ * .error "Alternatives instruction length mismatch"
	38	+ * .endif
	39	+ *
	40	+ * but most assemblers die if insn1 or insn2 have a .inst. This should
	41	+ * be fixed in a binutils release posterior to 2.25.51.0.2 (anything
	42	+ * containing commit 4e4d08cf7399b606 or c1baaddf8861).
	43	+ *
	44	+ * Alternatives with callbacks do not generate replacement instructions.
	45	+ */
	46	+#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled) \
	47	+ ".if "__stringify(cfg_enabled)" == 1\n" \
	48	+ "661:\n\t" \
	49	+ oldinstr "\n" \
	50	+ "662:\n" \
	51	+ ".pushsection .altinstructions,\"a\"\n" \
	52	+ ALTINSTR_ENTRY(feature) \
	53	+ ".popsection\n" \
	54	+ ".subsection 1\n" \
	55	+ "663:\n\t" \
	56	+ newinstr "\n" \
	57	+ "664:\n\t" \
	58	+ ".org . - (664b-663b) + (662b-661b)\n\t" \
	59	+ ".org . - (662b-661b) + (664b-663b)\n\t" \
	60	+ ".previous\n" \
	61	+ ".endif\n"
	62	+
	63	+#define __ALTERNATIVE_CFG_CB(oldinstr, feature, cfg_enabled, cb) \
	64	+ ".if "__stringify(cfg_enabled)" == 1\n" \
	65	+ "661:\n\t" \
	66	+ oldinstr "\n" \
	67	+ "662:\n" \
	68	+ ".pushsection .altinstructions,\"a\"\n" \
	69	+ ALTINSTR_ENTRY_CB(feature, cb) \
	70	+ ".popsection\n" \
	71	+ "663:\n\t" \
	72	+ "664:\n\t" \
	73	+ ".endif\n"
	74	+
	75	+#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
	76	+ __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
	77	+
	78	+#define ALTERNATIVE_CB(oldinstr, cb) \
	79	+ __ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_PATCH, 1, cb)
	80	+#else
	81	+
	82	+#include <asm/assembler.h>
	83	+
	84	+.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
	85	+ .word \orig_offset - .
	86	+ .word \alt_offset - .
	87	+ .hword \feature
	88	+ .byte \orig_len
	89	+ .byte \alt_len
	90	+.endm
	91	+
	92	+.macro alternative_insn insn1, insn2, cap, enable = 1
	93	+ .if \enable
	94	+661: \insn1
	95	+662: .pushsection .altinstructions, "a"
	96	+ altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
	97	+ .popsection
	98	+ .subsection 1
	99	+663: \insn2
	100	+664: .previous
	101	+ .org . - (664b-663b) + (662b-661b)
	102	+ .org . - (662b-661b) + (664b-663b)
	103	+ .endif
	104	+.endm
	105	+
	106	+/*
	107	+ * Alternative sequences
	108	+ *
	109	+ * The code for the case where the capability is not present will be
	110	+ * assembled and linked as normal. There are no restrictions on this
	111	+ * code.
	112	+ *
	113	+ * The code for the case where the capability is present will be
	114	+ * assembled into a special section to be used for dynamic patching.
	115	+ * Code for that case must:
	116	+ *
	117	+ * 1. Be exactly the same length (in bytes) as the default code
	118	+ * sequence.
	119	+ *
	120	+ * 2. Not contain a branch target that is used outside of the
	121	+ * alternative sequence it is defined in (branches into an
	122	+ * alternative sequence are not fixed up).
	123	+ */
	124	+
	125	+/*
	126	+ * Begin an alternative code sequence.
	127	+ */
	128	+.macro alternative_if_not cap
	129	+ .set .Lasm_alt_mode, 0
	130	+ .pushsection .altinstructions, "a"
	131	+ altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
	132	+ .popsection
	133	+661:
	134	+.endm
	135	+
	136	+.macro alternative_if cap
	137	+ .set .Lasm_alt_mode, 1
	138	+ .pushsection .altinstructions, "a"
	139	+ altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f
	140	+ .popsection
	141	+ .subsection 1
	142	+ .align 2 /* So GAS knows label 661 is suitably aligned */
	143	+661:
	144	+.endm
	145	+
	146	+.macro alternative_cb cb
	147	+ .set .Lasm_alt_mode, 0
	148	+ .pushsection .altinstructions, "a"
	149	+ altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0
	150	+ .popsection
	151	+661:
	152	+.endm
	153	+
	154	+/*
	155	+ * Provide the other half of the alternative code sequence.
	156	+ */
	157	+.macro alternative_else
	158	+662:
	159	+ .if .Lasm_alt_mode==0
	160	+ .subsection 1
	161	+ .else
	162	+ .previous
	163	+ .endif
	164	+663:
	165	+.endm
	166	+
	167	+/*
	168	+ * Complete an alternative code sequence.
	169	+ */
	170	+.macro alternative_endif
	171	+664:
	172	+ .if .Lasm_alt_mode==0
	173	+ .previous
	174	+ .endif
	175	+ .org . - (664b-663b) + (662b-661b)
	176	+ .org . - (662b-661b) + (664b-663b)
	177	+.endm
	178	+
	179	+/*
	180	+ * Callback-based alternative epilogue
	181	+ */
	182	+.macro alternative_cb_end
	183	+662:
	184	+.endm
	185	+
	186	+/*
	187	+ * Provides a trivial alternative or default sequence consisting solely
	188	+ * of NOPs. The number of NOPs is chosen automatically to match the
	189	+ * previous case.
	190	+ */
	191	+.macro alternative_else_nop_endif
	192	+alternative_else
	193	+ nops (662b-661b) / AARCH64_INSN_SIZE
	194	+alternative_endif
	195	+.endm
	196	+
	197	+#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \
	198	+ alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
	199	+
	200	+.macro user_alt, label, oldinstr, newinstr, cond
	201	+9999: alternative_insn "\oldinstr", "\newinstr", \cond
	202	+ _asm_extable 9999b, \label
	203	+.endm
	204	+
	205	+#endif /* __ASSEMBLY__ */
	206	+
	207	+/*
	208	+ * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature));
	209	+ *
	210	+ * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO));
	211	+ * N.B. If CONFIG_FOO is specified, but not selected, the whole block
	212	+ * will be omitted, including oldinstr.
	213	+ */
	214	+#define ALTERNATIVE(oldinstr, newinstr, ...) \
	215	+ _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
	216	+
	217	+#endif /* __ASM_ALTERNATIVE_MACROS_H */

arch/arm64/include/asm/alternative.h+2−265

@@ -2,17 +2,13 @@ · 1 unmodified line
2	2	#ifndef __ASM_ALTERNATIVE_H
3	3	#define __ASM_ALTERNATIVE_H
4	4
5		−#include <asm/cpucaps.h>
6		−#include <asm/insn.h>
7		−
8		−#define ARM64_CB_PATCH ARM64_NCAPS
	5	+#include <asm/alternative-macros.h>
9	6
10	7	#ifndef __ASSEMBLY__
11	8
12	9	#include <linux/init.h>
13	10	#include <linux/types.h>
14	11	#include <linux/stddef.h>
15		−#include <linux/stringify.h>
16	12
17	13	struct alt_instr {
18	14	s32 orig_offset; /* offset to original instruction */
@@ -35,264 +31,5 @@ void apply_alternatives_module(void *start, size_t length); · 16 unmodified lines
35	31	static inline void apply_alternatives_module(void *start, size_t length) { }
36	32	#endif
37	33
38		−#define ALTINSTR_ENTRY(feature) \
39		− " .word 661b - .\n" /* label */ \
40		− " .word 663f - .\n" /* new instruction */ \
41		− " .hword " __stringify(feature) "\n" /* feature bit */ \
42		− " .byte 662b-661b\n" /* source len */ \
43		− " .byte 664f-663f\n" /* replacement len */
44		−
45		−#define ALTINSTR_ENTRY_CB(feature, cb) \
46		− " .word 661b - .\n" /* label */ \
47		− " .word " __stringify(cb) "- .\n" /* callback */ \
48		− " .hword " __stringify(feature) "\n" /* feature bit */ \
49		− " .byte 662b-661b\n" /* source len */ \
50		− " .byte 664f-663f\n" /* replacement len */
51		−
52		−/*
53		− * alternative assembly primitive:
54		− *
55		− * If any of these .org directive fail, it means that insn1 and insn2
56		− * don't have the same length. This used to be written as
57		− *
58		− * .if ((664b-663b) != (662b-661b))
59		− * .error "Alternatives instruction length mismatch"
60		− * .endif
61		− *
62		− * but most assemblers die if insn1 or insn2 have a .inst. This should
63		− * be fixed in a binutils release posterior to 2.25.51.0.2 (anything
64		− * containing commit 4e4d08cf7399b606 or c1baaddf8861).
65		− *
66		− * Alternatives with callbacks do not generate replacement instructions.
67		− */
68		−#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled) \
69		− ".if "__stringify(cfg_enabled)" == 1\n" \
70		− "661:\n\t" \
71		− oldinstr "\n" \
72		− "662:\n" \
73		− ".pushsection .altinstructions,\"a\"\n" \
74		− ALTINSTR_ENTRY(feature) \
75		− ".popsection\n" \
76		− ".subsection 1\n" \
77		− "663:\n\t" \
78		− newinstr "\n" \
79		− "664:\n\t" \
80		− ".org . - (664b-663b) + (662b-661b)\n\t" \
81		− ".org . - (662b-661b) + (664b-663b)\n\t" \
82		− ".previous\n" \
83		− ".endif\n"
84		−
85		−#define __ALTERNATIVE_CFG_CB(oldinstr, feature, cfg_enabled, cb) \
86		− ".if "__stringify(cfg_enabled)" == 1\n" \
87		− "661:\n\t" \
88		− oldinstr "\n" \
89		− "662:\n" \
90		− ".pushsection .altinstructions,\"a\"\n" \
91		− ALTINSTR_ENTRY_CB(feature, cb) \
92		− ".popsection\n" \
93		− "663:\n\t" \
94		− "664:\n\t" \
95		− ".endif\n"
96		−
97		−#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...) \
98		− __ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
99		−
100		−#define ALTERNATIVE_CB(oldinstr, cb) \
101		− __ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_PATCH, 1, cb)
102		−#else
103		−
104		−#include <asm/assembler.h>
105		−
106		−.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
107		− .word \orig_offset - .
108		− .word \alt_offset - .
109		− .hword \feature
110		− .byte \orig_len
111		− .byte \alt_len
112		−.endm
113		−
114		−.macro alternative_insn insn1, insn2, cap, enable = 1
115		− .if \enable
116		−661: \insn1
117		−662: .pushsection .altinstructions, "a"
118		− altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
119		− .popsection
120		− .subsection 1
121		−663: \insn2
122		−664: .previous
123		− .org . - (664b-663b) + (662b-661b)
124		− .org . - (662b-661b) + (664b-663b)
125		− .endif
126		−.endm
127		−
128		−/*
129		− * Alternative sequences
130		− *
131		− * The code for the case where the capability is not present will be
132		− * assembled and linked as normal. There are no restrictions on this
133		− * code.
134		− *
135		− * The code for the case where the capability is present will be
136		− * assembled into a special section to be used for dynamic patching.
137		− * Code for that case must:
138		− *
139		− * 1. Be exactly the same length (in bytes) as the default code
140		− * sequence.
141		− *
142		− * 2. Not contain a branch target that is used outside of the
143		− * alternative sequence it is defined in (branches into an
144		− * alternative sequence are not fixed up).
145		− */
146		−
147		−/*
148		− * Begin an alternative code sequence.
149		− */
150		−.macro alternative_if_not cap
151		− .set .Lasm_alt_mode, 0
152		− .pushsection .altinstructions, "a"
153		− altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
154		− .popsection
155		−661:
156		−.endm
157		−
158		−.macro alternative_if cap
159		− .set .Lasm_alt_mode, 1
160		− .pushsection .altinstructions, "a"
161		− altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f
162		− .popsection
163		− .subsection 1
164		− .align 2 /* So GAS knows label 661 is suitably aligned */
165		−661:
166		−.endm
167		−
168		−.macro alternative_cb cb
169		− .set .Lasm_alt_mode, 0
170		− .pushsection .altinstructions, "a"
171		− altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0
172		− .popsection
173		−661:
174		−.endm
175		−
176		−/*
177		− * Provide the other half of the alternative code sequence.
178		− */
179		−.macro alternative_else
180		−662:
181		− .if .Lasm_alt_mode==0
182		− .subsection 1
183		− .else
184		− .previous
185		− .endif
186		−663:
187		−.endm
188		−
189		−/*
190		− * Complete an alternative code sequence.
191		− */
192		−.macro alternative_endif
193		−664:
194		− .if .Lasm_alt_mode==0
195		− .previous
196		− .endif
197		− .org . - (664b-663b) + (662b-661b)
198		− .org . - (662b-661b) + (664b-663b)
199		−.endm
200		−
201		−/*
202		− * Callback-based alternative epilogue
203		− */
204		−.macro alternative_cb_end
205		−662:
206		−.endm
207		−
208		−/*
209		− * Provides a trivial alternative or default sequence consisting solely
210		− * of NOPs. The number of NOPs is chosen automatically to match the
211		− * previous case.
212		− */
213		−.macro alternative_else_nop_endif
214		−alternative_else
215		− nops (662b-661b) / AARCH64_INSN_SIZE
216		−alternative_endif
217		−.endm
218		−
219		−#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \
220		− alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
221		−
222		−.macro user_alt, label, oldinstr, newinstr, cond
223		−9999: alternative_insn "\oldinstr", "\newinstr", \cond
224		− _asm_extable 9999b, \label
225		−.endm
226		−
227		−/*
228		− * Generate the assembly for UAO alternatives with exception table entries.
229		− * This is complicated as there is no post-increment or pair versions of the
230		− * unprivileged instructions, and USER() only works for single instructions.
231		− */
232		−#ifdef CONFIG_ARM64_UAO
233		− .macro uao_ldp l, reg1, reg2, addr, post_inc
234		− alternative_if_not ARM64_HAS_UAO
235		−8888: ldp \reg1, \reg2, [\addr], \post_inc;
236		−8889: nop;
237		− nop;
238		− alternative_else
239		− ldtr \reg1, [\addr];
240		− ldtr \reg2, [\addr, #8];
241		− add \addr, \addr, \post_inc;
242		− alternative_endif
243		−
244		− _asm_extable 8888b,\l;
245		− _asm_extable 8889b,\l;
246		− .endm
247		−
248		− .macro uao_stp l, reg1, reg2, addr, post_inc
249		− alternative_if_not ARM64_HAS_UAO
250		−8888: stp \reg1, \reg2, [\addr], \post_inc;
251		−8889: nop;
252		− nop;
253		− alternative_else
254		− sttr \reg1, [\addr];
255		− sttr \reg2, [\addr, #8];
256		− add \addr, \addr, \post_inc;
257		− alternative_endif
258		−
259		− _asm_extable 8888b,\l;
260		− _asm_extable 8889b,\l;
261		− .endm
262		−
263		− .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
264		− alternative_if_not ARM64_HAS_UAO
265		−8888: \inst \reg, [\addr], \post_inc;
266		− nop;
267		− alternative_else
268		− \alt_inst \reg, [\addr];
269		− add \addr, \addr, \post_inc;
270		− alternative_endif
271		−
272		− _asm_extable 8888b,\l;
273		− .endm
274		−#else
275		− .macro uao_ldp l, reg1, reg2, addr, post_inc
276		− USER(\l, ldp \reg1, \reg2, [\addr], \post_inc)
277		− .endm
278		− .macro uao_stp l, reg1, reg2, addr, post_inc
279		− USER(\l, stp \reg1, \reg2, [\addr], \post_inc)
280		− .endm
281		− .macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
282		− USER(\l, \inst \reg, [\addr], \post_inc)
283		− .endm
284		−#endif
285		−
286		−#endif /* __ASSEMBLY__ */
287		−
288		−/*
289		− * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature));
290		− *
291		− * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO));
292		− * N.B. If CONFIG_FOO is specified, but not selected, the whole block
293		− * will be omitted, including oldinstr.
294		− */
295		−#define ALTERNATIVE(oldinstr, newinstr, ...) \
296		− _ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
297		−
	34	+#endif /* __ASSEMBLY__ */
298	35	#endif /* __ASM_ALTERNATIVE_H */

arch/arm64/include/asm/asm-uaccess.h+32−3

@@ -2,7 +2,7 @@ · 1 unmodified line
2	2	#ifndef __ASM_ASM_UACCESS_H
3	3	#define __ASM_ASM_UACCESS_H
4	4
5		−#include <asm/alternative.h>
	5	+#include <asm/alternative-macros.h>
6	6	#include <asm/kernel-pgtable.h>
7	7	#include <asm/mmu.h>
8	8	#include <asm/sysreg.h>
@@ -15,10 +15,10 @@ · 6 unmodified lines
15	15	.macro __uaccess_ttbr0_disable, tmp1
16	16	mrs \tmp1, ttbr1_el1 // swapper_pg_dir
17	17	bic \tmp1, \tmp1, #TTBR_ASID_MASK
18		− sub \tmp1, \tmp1, #RESERVED_TTBR0_SIZE // reserved_ttbr0 just before swapper_pg_dir
	18	+ sub \tmp1, \tmp1, #PAGE_SIZE // reserved_pg_dir just before swapper_pg_dir
19	19	msr ttbr0_el1, \tmp1 // set reserved TTBR0_EL1
20	20	isb
21		− add \tmp1, \tmp1, #RESERVED_TTBR0_SIZE
	21	+ add \tmp1, \tmp1, #PAGE_SIZE
22	22	msr ttbr1_el1, \tmp1 // set reserved ASID
23	23	isb
24	24	.endm
@@ -58,4 +58,33 @@ alternative_else_nop_endif · 33 unmodified lines
58	58	.endm
59	59	#endif
60	60
	61	+/*
	62	+ * Generate the assembly for LDTR/STTR with exception table entries.
	63	+ * This is complicated as there is no post-increment or pair versions of the
	64	+ * unprivileged instructions, and USER() only works for single instructions.
	65	+ */
	66	+ .macro user_ldp l, reg1, reg2, addr, post_inc
	67	+8888: ldtr \reg1, [\addr];
	68	+8889: ldtr \reg2, [\addr, #8];
	69	+ add \addr, \addr, \post_inc;
	70	+
	71	+ _asm_extable 8888b,\l;
	72	+ _asm_extable 8889b,\l;
	73	+ .endm
	74	+
	75	+ .macro user_stp l, reg1, reg2, addr, post_inc
	76	+8888: sttr \reg1, [\addr];
	77	+8889: sttr \reg2, [\addr, #8];
	78	+ add \addr, \addr, \post_inc;
	79	+
	80	+ _asm_extable 8888b,\l;
	81	+ _asm_extable 8889b,\l;
	82	+ .endm
	83	+
	84	+ .macro user_ldst l, inst, reg, addr, post_inc
	85	+8888: \inst \reg, [\addr];
	86	+ add \addr, \addr, \post_inc;
	87	+
	88	+ _asm_extable 8888b,\l;
	89	+ .endm
61	90	#endif

arch/arm64/include/asm/cpucaps.h+2−3

@@ -16,8 +16,6 @@ · 15 unmodified lines
16	16	#define ARM64_WORKAROUND_CAVIUM_23154 6
17	17	#define ARM64_WORKAROUND_834220 7
18	18	#define ARM64_HAS_NO_HW_PREFETCH 8
19		−#define ARM64_HAS_UAO 9
20		−#define ARM64_ALT_PAN_NOT_UAO 10
21	19	#define ARM64_HAS_VIRT_HOST_EXTN 11
22	20	#define ARM64_WORKAROUND_CAVIUM_27456 12
23	21	#define ARM64_HAS_32BIT_EL0 13
@@ -66,7 +64,8 @@ · 42 unmodified lines
66	64	#define ARM64_HAS_TLB_RANGE 56
67	65	#define ARM64_MTE 57
68	66	#define ARM64_WORKAROUND_1508412 58
	67	+#define ARM64_HAS_LDAPR 59
69	68
70		−#define ARM64_NCAPS 59
	69	+#define ARM64_NCAPS 60
71	70
72	71	#endif /* __ASM_CPUCAPS_H */

arch/arm64/include/asm/cpufeature.h+22−1

@@ -669,10 +669,16 @@ static __always_inline bool system_supports_fpsimd(void) · 668 unmodified lines
669	669	return !cpus_have_const_cap(ARM64_HAS_NO_FPSIMD);
670	670	}
671	671
	672	+static inline bool system_uses_hw_pan(void)
	673	+{
	674	+ return IS_ENABLED(CONFIG_ARM64_PAN) &&
	675	+ cpus_have_const_cap(ARM64_HAS_PAN);
	676	+}
	677	+
672	678	static inline bool system_uses_ttbr0_pan(void)
673	679	{
674	680	return IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN) &&
675		− !cpus_have_const_cap(ARM64_HAS_PAN);
	681	+ !system_uses_hw_pan();
676	682	}
677	683
678	684	static __always_inline bool system_supports_sve(void)
@@ -764,11 +770,26 @@ static inline bool cpu_has_hw_af(void) · 85 unmodified lines
764	770	ID_AA64MMFR1_HADBS_SHIFT);
765	771	}
766	772
	773	+static inline bool cpu_has_pan(void)
	774	+{
	775	+ u64 mmfr1 = read_cpuid(ID_AA64MMFR1_EL1);
	776	+ return cpuid_feature_extract_unsigned_field(mmfr1,
	777	+ ID_AA64MMFR1_PAN_SHIFT);
	778	+}
	779	+
767	780	#ifdef CONFIG_ARM64_AMU_EXTN
768	781	/* Check whether the cpu supports the Activity Monitors Unit (AMU) */
769	782	extern bool cpu_has_amu_feat(int cpu);
	783	+#else
	784	+static inline bool cpu_has_amu_feat(int cpu)
	785	+{
	786	+ return false;
	787	+}
770	788	#endif
771	789
	790	+/* Get a cpu that supports the Activity Monitors Unit (AMU) */
	791	+extern int get_cpu_with_amu_feat(void);
	792	+
772	793	static inline unsigned int get_vmid_bits(u64 mmfr1)
773	794	{
774	795	int vmid_bits;

arch/arm64/include/asm/exception.h+1−1

@@ -37,7 +37,7 @@ asmlinkage void enter_from_user_mode(void); · 36 unmodified lines
37	37	asmlinkage void exit_to_user_mode(void);
38	38	void arm64_enter_nmi(struct pt_regs *regs);
39	39	void arm64_exit_nmi(struct pt_regs *regs);
40		−void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs);
	40	+void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs);
41	41	void do_undefinstr(struct pt_regs *regs);
42	42	void do_bti(struct pt_regs *regs);
43	43	asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr);

arch/arm64/include/asm/exec.h+0−1

@@ -10,6 +10,5 @@ · 9 unmodified lines
10	10	#include <linux/sched.h>
11	11
12	12	extern unsigned long arch_align_stack(unsigned long sp);
13		−void uao_thread_switch(struct task_struct *next);
14	13
15	14	#endif /* __ASM_EXEC_H */

arch/arm64/include/asm/futex.h+4−4

@@ -16,7 +16,7 @@ · 15 unmodified lines
16	16	do { \
17	17	unsigned int loops = FUTEX_MAX_LOOPS; \
18	18	\
19		− uaccess_enable(); \
	19	+ uaccess_enable_privileged(); \
20	20	asm volatile( \
21	21	" prfm pstl1strm, %2\n" \
22	22	"1: ldxr %w1, %2\n" \
@@ -39,7 +39,7 @@ do { \ · 16 unmodified lines
39	39	"+r" (loops) \
40	40	: "r" (oparg), "Ir" (-EFAULT), "Ir" (-EAGAIN) \
41	41	: "memory"); \
42		− uaccess_disable(); \
	42	+ uaccess_disable_privileged(); \
43	43	} while (0)
44	44
45	45	static inline int
@@ -95,7 +95,7 @@ futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user _uaddr, · 49 unmodified lines
95	95	return -EFAULT;
96	96
97	97	uaddr = __uaccess_mask_ptr(_uaddr);
98		− uaccess_enable();
	98	+ uaccess_enable_privileged();
99	99	asm volatile("// futex_atomic_cmpxchg_inatomic\n"
100	100	" prfm pstl1strm, %2\n"
101	101	"1: ldxr %w1, %2\n"
@@ -118,7 +118,7 @@ futex_atomic_cmpxchg_inatomic(u32 uval, u32 __user _uaddr, · 16 unmodified lines
118	118	: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
119	119	: "r" (oldval), "r" (newval), "Ir" (-EFAULT), "Ir" (-EAGAIN)
120	120	: "memory");
121		− uaccess_disable();
	121	+ uaccess_disable_privileged();
122	122
123	123	if (!ret)
124	124	*uval = val;

arch/arm64/include/asm/insn.h+1−2

@@ -10,8 +10,7 @@ · 9 unmodified lines
10	10	#include <linux/build_bug.h>
11	11	#include <linux/types.h>
12	12
13		−/* A64 instructions are always 32 bits. */
14		−#define AARCH64_INSN_SIZE 4
	13	+#include <asm/alternative.h>
15	14
16	15	#ifndef __ASSEMBLY__
17	16	/*

arch/arm64/include/asm/kernel-pgtable.h+0−6

@@ -89,12 +89,6 @@ · 88 unmodified lines
89	89	#define INIT_DIR_SIZE (PAGE_SIZE * EARLY_PAGES(KIMAGE_VADDR, _end))
90	90	#define IDMAP_DIR_SIZE (IDMAP_PGTABLE_LEVELS * PAGE_SIZE)
91	91
92		−#ifdef CONFIG_ARM64_SW_TTBR0_PAN
93		−#define RESERVED_TTBR0_SIZE (PAGE_SIZE)
94		−#else
95		−#define RESERVED_TTBR0_SIZE (0)
96		−#endif
97		−
98	92	/* Initial memory map size */
99	93	#if ARM64_SWAPPER_USES_SECTION_MAPS
100	94	#define SWAPPER_BLOCK_SHIFT SECTION_SHIFT

arch/arm64/include/asm/kprobes.h+0−7

@@ -28,18 +28,11 @@ struct prev_kprobe { · 27 unmodified lines
28	28	unsigned int status;
29	29	};
30	30
31		−/* Single step context for kprobe */
32		−struct kprobe_step_ctx {
33		− unsigned long ss_pending;
34		− unsigned long match_addr;
35		−};
36		−
37	31	/* per-cpu kprobe control block */
38	32	struct kprobe_ctlblk {
39	33	unsigned int kprobe_status;
40	34	unsigned long saved_irqflag;
41	35	struct prev_kprobe prev_kprobe;
42		− struct kprobe_step_ctx ss_ctx;
43	36	};
44	37
45	38	void arch_remove_kprobe(struct kprobe *);

arch/arm64/include/asm/memory.h+10−12

@@ -30,8 +30,8 @@ · 29 unmodified lines
30	30	* keep a constant PAGE_OFFSET and "fallback" to using the higher end
31	31	* of the VMEMMAP where 52-bit support is not available in hardware.
32	32	*/
33		−#define VMEMMAP_SIZE ((_PAGE_END(VA_BITS_MIN) - PAGE_OFFSET) \
34		− >> (PAGE_SHIFT - STRUCT_PAGE_MAX_SHIFT))
	33	+#define VMEMMAP_SHIFT (PAGE_SHIFT - STRUCT_PAGE_MAX_SHIFT)
	34	+#define VMEMMAP_SIZE ((_PAGE_END(VA_BITS_MIN) - PAGE_OFFSET) >> VMEMMAP_SHIFT)
35	35
36	36	/*
37	37	* PAGE_OFFSET - the virtual address of the start of the linear map, at the
@@ -44,17 +44,17 @@ · 6 unmodified lines
44	44	#define _PAGE_OFFSET(va) (-(UL(1) << (va)))
45	45	#define PAGE_OFFSET (_PAGE_OFFSET(VA_BITS))
46	46	#define KIMAGE_VADDR (MODULES_END)
47		−#define BPF_JIT_REGION_START (KASAN_SHADOW_END)
	47	+#define BPF_JIT_REGION_START (_PAGE_END(VA_BITS_MIN))
48	48	#define BPF_JIT_REGION_SIZE (SZ_128M)
49	49	#define BPF_JIT_REGION_END (BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE)
50	50	#define MODULES_END (MODULES_VADDR + MODULES_VSIZE)
51	51	#define MODULES_VADDR (BPF_JIT_REGION_END)
52	52	#define MODULES_VSIZE (SZ_128M)
53		−#define VMEMMAP_START (-VMEMMAP_SIZE - SZ_2M)
	53	+#define VMEMMAP_START (-(UL(1) << (VA_BITS - VMEMMAP_SHIFT)))
54	54	#define VMEMMAP_END (VMEMMAP_START + VMEMMAP_SIZE)
55		−#define PCI_IO_END (VMEMMAP_START - SZ_2M)
	55	+#define PCI_IO_END (VMEMMAP_START - SZ_8M)
56	56	#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
57		−#define FIXADDR_TOP (PCI_IO_START - SZ_2M)
	57	+#define FIXADDR_TOP (VMEMMAP_START - SZ_32M)
58	58
59	59	#if VA_BITS > 48
60	60	#define VA_BITS_MIN (48)
@@ -76,10 +76,11 @@ · 15 unmodified lines
76	76	#define KASAN_SHADOW_OFFSET _AC(CONFIG_KASAN_SHADOW_OFFSET, UL)
77	77	#define KASAN_SHADOW_END ((UL(1) << (64 - KASAN_SHADOW_SCALE_SHIFT)) \
78	78	+ KASAN_SHADOW_OFFSET)
	79	+#define PAGE_END (KASAN_SHADOW_END - (1UL << (vabits_actual - KASAN_SHADOW_SCALE_SHIFT)))
79	80	#define KASAN_THREAD_SHIFT 1
80	81	#else
81	82	#define KASAN_THREAD_SHIFT 0
82		−#define KASAN_SHADOW_END (_PAGE_END(VA_BITS_MIN))
	83	+#define PAGE_END (_PAGE_END(VA_BITS_MIN))
83	84	#endif /* CONFIG_KASAN */
84	85
85	86	#define MIN_THREAD_SHIFT (14 + KASAN_THREAD_SHIFT)
@@ -167,7 +168,6 @@ · 81 unmodified lines
167	168	#include <asm/bug.h>
168	169
169	170	extern u64 vabits_actual;
170		−#define PAGE_END (_PAGE_END(vabits_actual))
171	171
172	172	extern s64 memstart_addr;
173	173	/* PHYS_OFFSET - the physical address of the start of memory. */
@@ -238,11 +238,9 @@ static inline const void __tag_set(const void addr, u8 tag) · 64 unmodified lines
238	238
239	239
240	240	/*
241		− * The linear kernel range starts at the bottom of the virtual address
242		− * space. Testing the top bit for the start of the region is a
243		− * sufficient check and avoids having to worry about the tag.
	241	+ * The linear kernel range starts at the bottom of the virtual address space.
244	242	*/
245		−#define __is_lm_address(addr) (!(((u64)addr) & BIT(vabits_actual - 1)))
	243	+#define __is_lm_address(addr) (((u64)(addr) & ~PAGE_OFFSET) < (PAGE_END - PAGE_OFFSET))
246	244
247	245	#define __lm_to_phys(addr) (((addr) & ~PAGE_OFFSET) + PHYS_OFFSET)
248	246	#define __kimg_to_phys(addr) ((addr) - kimage_voffset)

arch/arm64/include/asm/mmu_context.h+3−3

@@ -36,11 +36,11 @@ static inline void contextidr_thread_switch(struct task_struct *next) · 35 unmodified lines
36	36	}
37	37
38	38	/*
39		− * Set TTBR0 to empty_zero_page. No translations will be possible via TTBR0.
	39	+ * Set TTBR0 to reserved_pg_dir. No translations will be possible via TTBR0.
40	40	*/
41	41	static inline void cpu_set_reserved_ttbr0(void)
42	42	{
43		− unsigned long ttbr = phys_to_ttbr(__pa_symbol(empty_zero_page));
	43	+ unsigned long ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir));
44	44
45	45	write_sysreg(ttbr, ttbr0_el1);
46	46	isb();
@@ -195,7 +195,7 @@ static inline void update_saved_ttbr0(struct task_struct *tsk, · 148 unmodified lines
195	195	return;
196	196
197	197	if (mm == &init_mm)
198		− ttbr = __pa_symbol(empty_zero_page);
	198	+ ttbr = __pa_symbol(reserved_pg_dir);
199	199	else
200	200	ttbr = virt_to_phys(mm->pgd) \| ASID(mm) << 48;
201	201

arch/arm64/include/asm/pgtable-hwdef.h+1−0

@@ -260,6 +260,7 @@ · 259 unmodified lines
260	260	#define TCR_TBI1 (UL(1) << 38)
261	261	#define TCR_HA (UL(1) << 39)
262	262	#define TCR_HD (UL(1) << 40)
	263	+#define TCR_TBID1 (UL(1) << 52)
263	264	#define TCR_NFD0 (UL(1) << 53)
264	265	#define TCR_NFD1 (UL(1) << 54)
265	266	#define TCR_E0PD0 (UL(1) << 55)

arch/arm64/include/asm/pgtable.h+2−1

@@ -22,7 +22,7 @@ · 21 unmodified lines
22	22	* and fixed mappings
23	23	*/
24	24	#define VMALLOC_START (MODULES_END)
25		−#define VMALLOC_END (- PUD_SIZE - VMEMMAP_SIZE - SZ_64K)
	25	+#define VMALLOC_END (VMEMMAP_START - SZ_256M)
26	26
27	27	#define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
28	28
@@ -527,6 +527,7 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; · 498 unmodified lines
527	527	extern pgd_t idmap_pg_dir[PTRS_PER_PGD];
528	528	extern pgd_t idmap_pg_end[];
529	529	extern pgd_t tramp_pg_dir[PTRS_PER_PGD];
	530	+extern pgd_t reserved_pg_dir[PTRS_PER_PGD];
530	531
531	532	extern void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd);
532	533

arch/arm64/include/asm/processor.h+1−3

@@ -8,9 +8,6 @@ · 7 unmodified lines
8	8	#ifndef __ASM_PROCESSOR_H
9	9	#define __ASM_PROCESSOR_H
10	10
11		−#define KERNEL_DS UL(-1)
12		−#define USER_DS ((UL(1) << VA_BITS) - 1)
13		−
14	11	/*
15	12	* On arm64 systems, unaligned accesses by the CPU are cheap, and so there is
16	13	* no point in shifting all network buffers by 2 bytes just to make some IP
@@ -48,6 +45,7 @@ · 31 unmodified lines
48	45
49	46	#define DEFAULT_MAP_WINDOW_64 (UL(1) << VA_BITS_MIN)
50	47	#define TASK_SIZE_64 (UL(1) << vabits_actual)
	48	+#define TASK_SIZE_MAX (UL(1) << VA_BITS)
51	49
52	50	#ifdef CONFIG_COMPAT
53	51	#if defined(CONFIG_ARM64_64K_PAGES) && defined(CONFIG_KUSER_HELPERS)

arch/arm64/include/asm/ptrace.h+6−2

@@ -16,6 +16,11 @@ · 15 unmodified lines
16	16	#define CurrentEL_EL1 (1 << 2)
17	17	#define CurrentEL_EL2 (2 << 2)
18	18
	19	+#define INIT_PSTATE_EL1 \
	20	+ (PSR_D_BIT \| PSR_A_BIT \| PSR_I_BIT \| PSR_F_BIT \| PSR_MODE_EL1h)
	21	+#define INIT_PSTATE_EL2 \
	22	+ (PSR_D_BIT \| PSR_A_BIT \| PSR_I_BIT \| PSR_F_BIT \| PSR_MODE_EL2h)
	23	+
19	24	/*
20	25	* PMR values used to mask/unmask interrupts.
21	26	*
@@ -188,8 +193,7 @@ struct pt_regs { · 166 unmodified lines
188	193	s32 syscallno;
189	194	u32 unused2;
190	195	#endif
191		−
192		− u64 orig_addr_limit;
	196	+ u64 sdei_ttbr1;
193	197	/* Only valid when ARM64_HAS_IRQ_PRIO_MASKING is enabled. */
194	198	u64 pmr_save;
195	199	u64 stackframe[2];

arch/arm64/include/asm/rwonce.h+73−0

@@ -0,0 +1,73 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/*
	3	+ * Copyright (C) 2020 Google LLC.
	4	+ */
	5	+#ifndef __ASM_RWONCE_H
	6	+#define __ASM_RWONCE_H
	7	+
	8	+#ifdef CONFIG_LTO
	9	+
	10	+#include <linux/compiler_types.h>
	11	+#include <asm/alternative-macros.h>
	12	+
	13	+#ifndef BUILD_VDSO
	14	+
	15	+#ifdef CONFIG_AS_HAS_LDAPR
	16	+#define __LOAD_RCPC(sfx, regs...) \
	17	+ ALTERNATIVE( \
	18	+ "ldar" #sfx "\t" #regs, \
	19	+ ".arch_extension rcpc\n" \
	20	+ "ldapr" #sfx "\t" #regs, \
	21	+ ARM64_HAS_LDAPR)
	22	+#else
	23	+#define __LOAD_RCPC(sfx, regs...) "ldar" #sfx "\t" #regs
	24	+#endif /* CONFIG_AS_HAS_LDAPR */
	25	+
	26	+/*
	27	+ * When building with LTO, there is an increased risk of the compiler
	28	+ * converting an address dependency headed by a READ_ONCE() invocation
	29	+ * into a control dependency and consequently allowing for harmful
	30	+ * reordering by the CPU.
	31	+ *
	32	+ * Ensure that such transformations are harmless by overriding the generic
	33	+ * READ_ONCE() definition with one that provides RCpc acquire semantics
	34	+ * when building with LTO.
	35	+ */
	36	+#define __READ_ONCE(x) \
	37	+({ \
	38	+ typeof(&(x)) __x = &(x); \
	39	+ int atomic = 1; \
	40	+ union { __unqual_scalar_typeof(*__x) __val; char __c[1]; } __u; \
	41	+ switch (sizeof(x)) { \
	42	+ case 1: \
	43	+ asm volatile(__LOAD_RCPC(b, %w0, %1) \
	44	+ : "=r" ((__u8 )__u.__c) \
	45	+ : "Q" (*__x) : "memory"); \
	46	+ break; \
	47	+ case 2: \
	48	+ asm volatile(__LOAD_RCPC(h, %w0, %1) \
	49	+ : "=r" ((__u16 )__u.__c) \
	50	+ : "Q" (*__x) : "memory"); \
	51	+ break; \
	52	+ case 4: \
	53	+ asm volatile(__LOAD_RCPC(, %w0, %1) \
	54	+ : "=r" ((__u32 )__u.__c) \
	55	+ : "Q" (*__x) : "memory"); \
	56	+ break; \
	57	+ case 8: \
	58	+ asm volatile(__LOAD_RCPC(, %0, %1) \
	59	+ : "=r" ((__u64 )__u.__c) \
	60	+ : "Q" (*__x) : "memory"); \
	61	+ break; \
	62	+ default: \
	63	+ atomic = 0; \
	64	+ } \
	65	+ atomic ? (typeof(__x))__u.__val : ((volatile typeof(__x))__x);\
	66	+})
	67	+
	68	+#endif /* !BUILD_VDSO */
	69	+#endif /* CONFIG_LTO */
	70	+
	71	+#include <asm-generic/rwonce.h>
	72	+
	73	+#endif /* __ASM_RWONCE_H */

arch/arm64/include/asm/signal.h+25−0

@@ -0,0 +1,25 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+#ifndef __ARM64_ASM_SIGNAL_H
	3	+#define __ARM64_ASM_SIGNAL_H
	4	+
	5	+#include <asm/memory.h>
	6	+#include <uapi/asm/signal.h>
	7	+#include <uapi/asm/siginfo.h>
	8	+
	9	+static inline void __user arch_untagged_si_addr(void __user addr,
	10	+ unsigned long sig,
	11	+ unsigned long si_code)
	12	+{
	13	+ /*
	14	+ * For historical reasons, all bits of the fault address are exposed as
	15	+ * address bits for watchpoint exceptions. New architectures should
	16	+ * handle the tag bits consistently.
	17	+ */
	18	+ if (sig == SIGTRAP && si_code == TRAP_BRKPT)
	19	+ return addr;
	20	+
	21	+ return untagged_addr(addr);
	22	+}
	23	+#define arch_untagged_si_addr arch_untagged_si_addr
	24	+
	25	+#endif

arch/arm64/include/asm/sysreg.h+16−6

@@ -98,6 +98,10 @@ · 97 unmodified lines
98	98	#define SET_PSTATE_SSBS(x) __emit_inst(0xd500401f \| PSTATE_SSBS \| ((!!x) << PSTATE_Imm_shift))
99	99	#define SET_PSTATE_TCO(x) __emit_inst(0xd500401f \| PSTATE_TCO \| ((!!x) << PSTATE_Imm_shift))
100	100
	101	+#define set_pstate_pan(x) asm volatile(SET_PSTATE_PAN(x))
	102	+#define set_pstate_uao(x) asm volatile(SET_PSTATE_UAO(x))
	103	+#define set_pstate_ssbs(x) asm volatile(SET_PSTATE_SSBS(x))
	104	+
101	105	#define __SYS_BARRIER_INSN(CRm, op2, Rt) \
102	106	__emit_inst(0xd5000000 \| sys_insn(0, 3, 3, (CRm), (op2)) \| ((Rt) & 0x1f))
103	107
@@ -582,6 +586,9 @@ · 478 unmodified lines
582	586	#define ENDIAN_SET_EL2 0
583	587	#endif
584	588
	589	+#define INIT_SCTLR_EL2_MMU_OFF \
	590	+ (SCTLR_EL2_RES1 \| ENDIAN_SET_EL2)
	591	+
585	592	/* SCTLR_EL1 specific flags. */
586	593	#define SCTLR_EL1_ATA0 (BIT(42))
587	594
@@ -615,12 +622,15 @@ · 27 unmodified lines
615	622	#define ENDIAN_SET_EL1 0
616	623	#endif
617	624
618		−#define SCTLR_EL1_SET (SCTLR_ELx_M \| SCTLR_ELx_C \| SCTLR_ELx_SA \|\
619		− SCTLR_EL1_SA0 \| SCTLR_EL1_SED \| SCTLR_ELx_I \|\
620		− SCTLR_EL1_DZE \| SCTLR_EL1_UCT \|\
621		− SCTLR_EL1_NTWE \| SCTLR_ELx_IESB \| SCTLR_EL1_SPAN \|\
622		− SCTLR_ELx_ITFSB\| SCTLR_ELx_ATA \| SCTLR_EL1_ATA0 \|\
623		− ENDIAN_SET_EL1 \| SCTLR_EL1_UCI \| SCTLR_EL1_RES1)
	625	+#define INIT_SCTLR_EL1_MMU_OFF \
	626	+ (ENDIAN_SET_EL1 \| SCTLR_EL1_RES1)
	627	+
	628	+#define INIT_SCTLR_EL1_MMU_ON \
	629	+ (SCTLR_ELx_M \| SCTLR_ELx_C \| SCTLR_ELx_SA \| SCTLR_EL1_SA0 \| \
	630	+ SCTLR_EL1_SED \| SCTLR_ELx_I \| SCTLR_EL1_DZE \| SCTLR_EL1_UCT \| \
	631	+ SCTLR_EL1_NTWE \| SCTLR_ELx_IESB \| SCTLR_EL1_SPAN \| SCTLR_ELx_ITFSB \| \
	632	+ SCTLR_ELx_ATA \| SCTLR_EL1_ATA0 \| ENDIAN_SET_EL1 \| SCTLR_EL1_UCI \| \
	633	+ SCTLR_EL1_RES1)
624	634
625	635	/* MAIR_ELx memory attributes (used by Linux) */
626	636	#define MAIR_ATTR_DEVICE_nGnRnE UL(0x00)

arch/arm64/include/asm/system_misc.h+1−1

@@ -22,7 +22,7 @@ void die(const char msg, struct pt_regs regs, int err); · 21 unmodified lines
22	22
23	23	struct siginfo;
24	24	void arm64_notify_die(const char str, struct pt_regs regs,
25		− int signo, int sicode, void __user *addr,
	25	+ int signo, int sicode, unsigned long far,
26	26	int err);
27	27
28	28	void hook_debug_fault_code(int nr, int (*fn)(unsigned long, unsigned int,

arch/arm64/include/asm/thread_info.h+2−8

@@ -18,14 +18,11 @@ struct task_struct; · 17 unmodified lines
18	18	#include <asm/stack_pointer.h>
19	19	#include <asm/types.h>
20	20
21		−typedef unsigned long mm_segment_t;
22		−
23	21	/*
24	22	* low level task data that entry.S needs immediate access to.
25	23	*/
26	24	struct thread_info {
27	25	unsigned long flags; /* low level flags */
28		− mm_segment_t addr_limit; /* address limit */
29	26	#ifdef CONFIG_ARM64_SW_TTBR0_PAN
30	27	u64 ttbr0; /* saved TTBR0_EL1 */
31	28	#endif
@@ -66,8 +63,7 @@ void arch_release_task_struct(struct task_struct *tsk); · 34 unmodified lines
66	63	#define TIF_NOTIFY_RESUME 2 /* callback before returning to user */
67	64	#define TIF_FOREIGN_FPSTATE 3 /* CPU's FP state is not current's */
68	65	#define TIF_UPROBE 4 /* uprobe breakpoint or singlestep */
69		−#define TIF_FSCHECK 5 /* Check FS is USER_DS on return */
70		−#define TIF_MTE_ASYNC_FAULT 6 /* MTE Asynchronous Tag Check Fault */
	66	+#define TIF_MTE_ASYNC_FAULT 5 /* MTE Asynchronous Tag Check Fault */
71	67	#define TIF_SYSCALL_TRACE 8 /* syscall trace active */
72	68	#define TIF_SYSCALL_AUDIT 9 /* syscall auditing */
73	69	#define TIF_SYSCALL_TRACEPOINT 10 /* syscall tracepoint for ftrace */
@@ -93,7 +89,6 @@ void arch_release_task_struct(struct task_struct *tsk); · 19 unmodified lines
93	89	#define _TIF_SECCOMP (1 << TIF_SECCOMP)
94	90	#define _TIF_SYSCALL_EMU (1 << TIF_SYSCALL_EMU)
95	91	#define _TIF_UPROBE (1 << TIF_UPROBE)
96		−#define _TIF_FSCHECK (1 << TIF_FSCHECK)
97	92	#define _TIF_SINGLESTEP (1 << TIF_SINGLESTEP)
98	93	#define _TIF_32BIT (1 << TIF_32BIT)
99	94	#define _TIF_SVE (1 << TIF_SVE)
@@ -101,7 +96,7 @@ void arch_release_task_struct(struct task_struct *tsk); · 1 unmodified line
101	96
102	97	#define _TIF_WORK_MASK (_TIF_NEED_RESCHED \| _TIF_SIGPENDING \| \
103	98	_TIF_NOTIFY_RESUME \| _TIF_FOREIGN_FPSTATE \| \
104		− _TIF_UPROBE \| _TIF_FSCHECK \| _TIF_MTE_ASYNC_FAULT)
	99	+ _TIF_UPROBE \| _TIF_MTE_ASYNC_FAULT)
105	100
106	101	#define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE \| _TIF_SYSCALL_AUDIT \| \
107	102	_TIF_SYSCALL_TRACEPOINT \| _TIF_SECCOMP \| \
@@ -119,7 +114,6 @@ void arch_release_task_struct(struct task_struct *tsk); · 11 unmodified lines
119	114	{ \
120	115	.flags = _TIF_FOREIGN_FPSTATE, \
121	116	.preempt_count = INIT_PREEMPT_COUNT, \
122		− .addr_limit = KERNEL_DS, \
123	117	INIT_SCS \
124	118	}
125	119

arch/arm64/include/asm/topology.h+3−1

@@ -16,12 +16,14 @@ int pcibus_to_node(struct pci_bus *bus); · 15 unmodified lines
16	16
17	17	#include <linux/arch_topology.h>
18	18
	19	+void update_freq_counters_refs(void);
	20	+void topology_scale_freq_tick(void);
	21	+
19	22	#ifdef CONFIG_ARM64_AMU_EXTN
20	23	/*
21	24	* Replace task scheduler's default counter-based
22	25	* frequency-invariance scale factor setting.
23	26	*/
24		−void topology_scale_freq_tick(void);
25	27	#define arch_scale_freq_tick topology_scale_freq_tick
26	28	#endif /* CONFIG_ARM64_AMU_EXTN */
27	29

arch/arm64/include/asm/traps.h+3−3

@@ -26,9 +26,9 @@ void register_undef_hook(struct undef_hook *hook); · 25 unmodified lines
26	26	void unregister_undef_hook(struct undef_hook *hook);
27	27	void force_signal_inject(int signal, int code, unsigned long address, unsigned int err);
28	28	void arm64_notify_segfault(unsigned long addr);
29		−void arm64_force_sig_fault(int signo, int code, void __user addr, const char str);
30		−void arm64_force_sig_mceerr(int code, void __user addr, short lsb, const char str);
31		−void arm64_force_sig_ptrace_errno_trap(int errno, void __user addr, const char str);
	29	+void arm64_force_sig_fault(int signo, int code, unsigned long far, const char *str);
	30	+void arm64_force_sig_mceerr(int code, unsigned long far, short lsb, const char *str);
	31	+void arm64_force_sig_ptrace_errno_trap(int errno, unsigned long far, const char *str);
32	32
33	33	/*
34	34	* Move regs->pc to next instruction and do necessary setup before it

arch/arm64/include/asm/uaccess.h+75−103

@@ -24,44 +24,18 @@ · 23 unmodified lines
24	24	#include <asm/memory.h>
25	25	#include <asm/extable.h>
26	26
27		−#define get_fs() (current_thread_info()->addr_limit)
28		−
29		−static inline void set_fs(mm_segment_t fs)
30		−{
31		− current_thread_info()->addr_limit = fs;
32		−
33		− /*
34		− * Prevent a mispredicted conditional call to set_fs from forwarding
35		− * the wrong address limit to access_ok under speculation.
36		− */
37		− spec_bar();
38		−
39		− /* On user-mode return, check fs is correct */
40		− set_thread_flag(TIF_FSCHECK);
41		−
42		− /*
43		− * Enable/disable UAO so that copy_to_user() etc can access
44		− * kernel memory with the unprivileged instructions.
45		− */
46		− if (IS_ENABLED(CONFIG_ARM64_UAO) && fs == KERNEL_DS)
47		− asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO));
48		− else
49		− asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO,
50		− CONFIG_ARM64_UAO));
51		−}
52		−
53		−#define uaccess_kernel() (get_fs() == KERNEL_DS)
	27	+#define HAVE_GET_KERNEL_NOFAULT
54	28
55	29	/*
56	30	* Test whether a block of memory is a valid user space address.
57	31	* Returns 1 if the range is valid, 0 otherwise.
58	32	*
59	33	* This is equivalent to the following test:
60		− * (u65)addr + (u65)size <= (u65)current->addr_limit + 1
	34	+ * (u65)addr + (u65)size <= (u65)TASK_SIZE_MAX
61	35	*/
62	36	static inline unsigned long __range_ok(const void __user *addr, unsigned long size)
63	37	{
64		− unsigned long ret, limit = current_thread_info()->addr_limit;
	38	+ unsigned long ret, limit = TASK_SIZE_MAX - 1;
65	39
66	40	/*
67	41	* Asynchronous I/O running in a kernel thread does not have the
@@ -94,7 +68,6 @@ static inline unsigned long __range_ok(const void __user *addr, unsigned long si · 26 unmodified lines
94	68	}
95	69
96	70	#define access_ok(addr, size) __range_ok(addr, size)
97		−#define user_addr_max get_fs
98	71
99	72	#define _ASM_EXTABLE(from, to) \
100	73	" .pushsection __ex_table, \"a\"\n" \
@@ -113,8 +86,8 @@ static inline void __uaccess_ttbr0_disable(void) · 12 unmodified lines
113	86	local_irq_save(flags);
114	87	ttbr = read_sysreg(ttbr1_el1);
115	88	ttbr &= ~TTBR_ASID_MASK;
116		− /* reserved_ttbr0 placed before swapper_pg_dir */
117		− write_sysreg(ttbr - RESERVED_TTBR0_SIZE, ttbr0_el1);
	89	+ /* reserved_pg_dir placed before swapper_pg_dir */
	90	+ write_sysreg(ttbr - PAGE_SIZE, ttbr0_el1);
118	91	isb();
119	92	/* Set reserved ASID */
120	93	write_sysreg(ttbr, ttbr1_el1);
@@ -186,47 +159,26 @@ static inline void __uaccess_enable_hw_pan(void) · 65 unmodified lines
186	159	CONFIG_ARM64_PAN));
187	160	}
188	161
189		−#define __uaccess_disable(alt) \
190		−do { \
191		− if (!uaccess_ttbr0_disable()) \
192		− asm(ALTERNATIVE("nop", SET_PSTATE_PAN(1), alt, \
193		− CONFIG_ARM64_PAN)); \
194		−} while (0)
195		−
196		−#define __uaccess_enable(alt) \
197		−do { \
198		− if (!uaccess_ttbr0_enable()) \
199		− asm(ALTERNATIVE("nop", SET_PSTATE_PAN(0), alt, \
200		− CONFIG_ARM64_PAN)); \
201		−} while (0)
202		−
203		−static inline void uaccess_disable(void)
	162	+static inline void uaccess_disable_privileged(void)
204	163	{
205		− __uaccess_disable(ARM64_HAS_PAN);
206		−}
	164	+ if (uaccess_ttbr0_disable())
	165	+ return;
207	166
208		−static inline void uaccess_enable(void)
209		−{
210		− __uaccess_enable(ARM64_HAS_PAN);
	167	+ __uaccess_enable_hw_pan();
211	168	}
212	169
213		−/*
214		− * These functions are no-ops when UAO is present.
215		− */
216		−static inline void uaccess_disable_not_uao(void)
	170	+static inline void uaccess_enable_privileged(void)
217	171	{
218		− __uaccess_disable(ARM64_ALT_PAN_NOT_UAO);
219		−}
	172	+ if (uaccess_ttbr0_enable())
	173	+ return;
220	174
221		−static inline void uaccess_enable_not_uao(void)
222		−{
223		− __uaccess_enable(ARM64_ALT_PAN_NOT_UAO);
	175	+ __uaccess_disable_hw_pan();
224	176	}
225	177
226	178	/*
227		− * Sanitise a uaccess pointer such that it becomes NULL if above the
228		− * current addr_limit. In case the pointer is tagged (has the top byte set),
229		− * untag the pointer before checking.
	179	+ * Sanitise a uaccess pointer such that it becomes NULL if above the maximum
	180	+ * user address. In case the pointer is tagged (has the top byte set), untag
	181	+ * the pointer before checking.
230	182	*/
231	183	#define uaccess_mask_ptr(ptr) (__typeof__(ptr))__uaccess_mask_ptr(ptr)
232	184	static inline void __user __uaccess_mask_ptr(const void __user ptr)
@@ -237,7 +189,7 @@ static inline void __user __uaccess_mask_ptr(const void __user ptr) · 4 unmodified lines
237	189	" bics xzr, %3, %2\n"
238	190	" csel %0, %1, xzr, eq\n"
239	191	: "=&r" (safe_ptr)
240		− : "r" (ptr), "r" (current_thread_info()->addr_limit),
	192	+ : "r" (ptr), "r" (TASK_SIZE_MAX - 1),
241	193	"r" (untagged_addr(ptr))
242	194	: "cc");
243	195
@@ -253,10 +205,9 @@ static inline void __user __uaccess_mask_ptr(const void __user ptr) · 9 unmodified lines
253	205	* The "__xxx_error" versions set the third argument to -EFAULT if an error
254	206	* occurs, and leave it unchanged on success.
255	207	*/
256		−#define __get_user_asm(instr, alt_instr, reg, x, addr, err, feature) \
	208	+#define __get_mem_asm(load, reg, x, addr, err) \
257	209	asm volatile( \
258		− "1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \
259		− alt_instr " " reg "1, [%2]\n", feature) \
	210	+ "1: " load " " reg "1, [%2]\n" \
260	211	"2:\n" \
261	212	" .section .fixup, \"ax\"\n" \
262	213	" .align 2\n" \
@@ -268,35 +219,36 @@ static inline void __user __uaccess_mask_ptr(const void __user ptr) · 5 unmodified lines
268	219	: "+r" (err), "=&r" (x) \
269	220	: "r" (addr), "i" (-EFAULT))
270	221
271		−#define __raw_get_user(x, ptr, err) \
	222	+#define __raw_get_mem(ldr, x, ptr, err) \
272	223	do { \
273	224	unsigned long __gu_val; \
274		− __chk_user_ptr(ptr); \
275		− uaccess_enable_not_uao(); \
276	225	switch (sizeof(*(ptr))) { \
277	226	case 1: \
278		− __get_user_asm("ldrb", "ldtrb", "%w", __gu_val, (ptr), \
279		− (err), ARM64_HAS_UAO); \
	227	+ __get_mem_asm(ldr "b", "%w", __gu_val, (ptr), (err)); \
280	228	break; \
281	229	case 2: \
282		− __get_user_asm("ldrh", "ldtrh", "%w", __gu_val, (ptr), \
283		− (err), ARM64_HAS_UAO); \
	230	+ __get_mem_asm(ldr "h", "%w", __gu_val, (ptr), (err)); \
284	231	break; \
285	232	case 4: \
286		− __get_user_asm("ldr", "ldtr", "%w", __gu_val, (ptr), \
287		− (err), ARM64_HAS_UAO); \
	233	+ __get_mem_asm(ldr, "%w", __gu_val, (ptr), (err)); \
288	234	break; \
289	235	case 8: \
290		− __get_user_asm("ldr", "ldtr", "%x", __gu_val, (ptr), \
291		− (err), ARM64_HAS_UAO); \
	236	+ __get_mem_asm(ldr, "%x", __gu_val, (ptr), (err)); \
292	237	break; \
293	238	default: \
294	239	BUILD_BUG(); \
295	240	} \
296		− uaccess_disable_not_uao(); \
297	241	(x) = (__force __typeof__(*(ptr)))__gu_val; \
298	242	} while (0)
299	243
	244	+#define __raw_get_user(x, ptr, err) \
	245	+do { \
	246	+ __chk_user_ptr(ptr); \
	247	+ uaccess_ttbr0_enable(); \
	248	+ __raw_get_mem("ldtr", x, ptr, err); \
	249	+ uaccess_ttbr0_disable(); \
	250	+} while (0)
	251	+
300	252	#define __get_user_error(x, ptr, err) \
301	253	do { \
302	254	__typeof__((ptr)) __user __p = (ptr); \
@@ -318,10 +270,19 @@ do { \ · 15 unmodified lines
318	270
319	271	#define get_user __get_user
320	272
321		−#define __put_user_asm(instr, alt_instr, reg, x, addr, err, feature) \
	273	+#define __get_kernel_nofault(dst, src, type, err_label) \
	274	+do { \
	275	+ int __gkn_err = 0; \
	276	+ \
	277	+ __raw_get_mem("ldr", ((type )(dst)), \
	278	+ (__force type *)(src), __gkn_err); \
	279	+ if (unlikely(__gkn_err)) \
	280	+ goto err_label; \
	281	+} while (0)
	282	+
	283	+#define __put_mem_asm(store, reg, x, addr, err) \
322	284	asm volatile( \
323		− "1:"ALTERNATIVE(instr " " reg "1, [%2]\n", \
324		− alt_instr " " reg "1, [%2]\n", feature) \
	285	+ "1: " store " " reg "1, [%2]\n" \
325	286	"2:\n" \
326	287	" .section .fixup,\"ax\"\n" \
327	288	" .align 2\n" \
@@ -332,32 +293,33 @@ do { \ · 4 unmodified lines
332	293	: "+r" (err) \
333	294	: "r" (x), "r" (addr), "i" (-EFAULT))
334	295
335		−#define __raw_put_user(x, ptr, err) \
	296	+#define __raw_put_mem(str, x, ptr, err) \
336	297	do { \
337	298	__typeof__(*(ptr)) __pu_val = (x); \
338		− __chk_user_ptr(ptr); \
339		− uaccess_enable_not_uao(); \
340	299	switch (sizeof(*(ptr))) { \
341	300	case 1: \
342		− __put_user_asm("strb", "sttrb", "%w", __pu_val, (ptr), \
343		− (err), ARM64_HAS_UAO); \
	301	+ __put_mem_asm(str "b", "%w", __pu_val, (ptr), (err)); \
344	302	break; \
345	303	case 2: \
346		− __put_user_asm("strh", "sttrh", "%w", __pu_val, (ptr), \
347		− (err), ARM64_HAS_UAO); \
	304	+ __put_mem_asm(str "h", "%w", __pu_val, (ptr), (err)); \
348	305	break; \
349	306	case 4: \
350		− __put_user_asm("str", "sttr", "%w", __pu_val, (ptr), \
351		− (err), ARM64_HAS_UAO); \
	307	+ __put_mem_asm(str, "%w", __pu_val, (ptr), (err)); \
352	308	break; \
353	309	case 8: \
354		− __put_user_asm("str", "sttr", "%x", __pu_val, (ptr), \
355		− (err), ARM64_HAS_UAO); \
	310	+ __put_mem_asm(str, "%x", __pu_val, (ptr), (err)); \
356	311	break; \
357	312	default: \
358	313	BUILD_BUG(); \
359	314	} \
360		− uaccess_disable_not_uao(); \
	315	+} while (0)
	316	+
	317	+#define __raw_put_user(x, ptr, err) \
	318	+do { \
	319	+ __chk_user_ptr(ptr); \
	320	+ uaccess_ttbr0_enable(); \
	321	+ __raw_put_mem("sttr", x, ptr, err); \
	322	+ uaccess_ttbr0_disable(); \
361	323	} while (0)
362	324
363	325	#define __put_user_error(x, ptr, err) \
@@ -381,14 +343,24 @@ do { \ · 17 unmodified lines
381	343
382	344	#define put_user __put_user
383	345
	346	+#define __put_kernel_nofault(dst, src, type, err_label) \
	347	+do { \
	348	+ int __pkn_err = 0; \
	349	+ \
	350	+ __raw_put_mem("str", ((type )(src)), \
	351	+ (__force type *)(dst), __pkn_err); \
	352	+ if (unlikely(__pkn_err)) \
	353	+ goto err_label; \
	354	+} while(0)
	355	+
384	356	extern unsigned long __must_check __arch_copy_from_user(void to, const void __user from, unsigned long n);
385	357	#define raw_copy_from_user(to, from, n) \
386	358	({ \
387	359	unsigned long __acfu_ret; \
388		− uaccess_enable_not_uao(); \
	360	+ uaccess_ttbr0_enable(); \
389	361	__acfu_ret = __arch_copy_from_user((to), \
390	362	__uaccess_mask_ptr(from), (n)); \
391		− uaccess_disable_not_uao(); \
	363	+ uaccess_ttbr0_disable(); \
392	364	__acfu_ret; \
393	365	})
394	366
@@ -396,10 +368,10 @@ extern unsigned long __must_check __arch_copy_to_user(void __user *to, const voi · 1 unmodified line
396	368	#define raw_copy_to_user(to, from, n) \
397	369	({ \
398	370	unsigned long __actu_ret; \
399		− uaccess_enable_not_uao(); \
	371	+ uaccess_ttbr0_enable(); \
400	372	__actu_ret = __arch_copy_to_user(__uaccess_mask_ptr(to), \
401	373	(from), (n)); \
402		− uaccess_disable_not_uao(); \
	374	+ uaccess_ttbr0_disable(); \
403	375	__actu_ret; \
404	376	})
405	377
@@ -407,10 +379,10 @@ extern unsigned long __must_check __arch_copy_in_user(void __user *to, const voi · 1 unmodified line
407	379	#define raw_copy_in_user(to, from, n) \
408	380	({ \
409	381	unsigned long __aciu_ret; \
410		− uaccess_enable_not_uao(); \
	382	+ uaccess_ttbr0_enable(); \
411	383	__aciu_ret = __arch_copy_in_user(__uaccess_mask_ptr(to), \
412	384	__uaccess_mask_ptr(from), (n)); \
413		− uaccess_disable_not_uao(); \
	385	+ uaccess_ttbr0_disable(); \
414	386	__aciu_ret; \
415	387	})
416	388
@@ -421,9 +393,9 @@ extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned lo · 4 unmodified lines
421	393	static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n)
422	394	{
423	395	if (access_ok(to, n)) {
424		− uaccess_enable_not_uao();
	396	+ uaccess_ttbr0_enable();
425	397	n = __arch_clear_user(__uaccess_mask_ptr(to), n);
426		− uaccess_disable_not_uao();
	398	+ uaccess_ttbr0_disable();
427	399	}
428	400	return n;
429	401	}

arch/arm64/kernel/Makefile+0−1

@@ -58,7 +58,6 @@ obj-$(CONFIG_CRASH_DUMP) += crash_dump.o · 57 unmodified lines
58	58	obj-$(CONFIG_CRASH_CORE) += crash_core.o
59	59	obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o
60	60	obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o
61		−obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
62	61	obj-$(CONFIG_ARM64_MTE) += mte.o
63	62
64	63	obj-y += vdso/ probes/

arch/arm64/kernel/alternative.c+4−3

@@ -21,7 +21,8 @@ · 20 unmodified lines
21	21	#define ALT_ORIG_PTR(a) __ALT_PTR(a, orig_offset)
22	22	#define ALT_REPL_PTR(a) __ALT_PTR(a, alt_offset)
23	23
24		−static int all_alternatives_applied;
	24	+/* Volatile, as we may be patching the guts of READ_ONCE() */
	25	+static volatile int all_alternatives_applied;
25	26
26	27	static DECLARE_BITMAP(applied_alternatives, ARM64_NCAPS);
27	28
@@ -205,7 +206,7 @@ static int __apply_alternatives_multi_stop(void *unused) · 177 unmodified lines
205	206
206	207	/* We always have a CPU 0 at this point (__init) */
207	208	if (smp_processor_id()) {
208		− while (!READ_ONCE(all_alternatives_applied))
	209	+ while (!all_alternatives_applied)
209	210	cpu_relax();
210	211	isb();
211	212	} else {
@@ -217,7 +218,7 @@ static int __apply_alternatives_multi_stop(void *unused) · 5 unmodified lines
217	218	BUG_ON(all_alternatives_applied);
218	219	__apply_alternatives(&region, false, remaining_capabilities);
219	220	/* Barriers provided by the cache flushing */
220		− WRITE_ONCE(all_alternatives_applied, 1);
	221	+ all_alternatives_applied = 1;
221	222	}
222	223
223	224	return 0;

arch/arm64/kernel/armv8_deprecated.c+2−2

@@ -277,7 +277,7 @@ static void __init register_insn_emulation_sysctl(void) · 276 unmodified lines
277	277
278	278	#define __user_swpX_asm(data, addr, res, temp, temp2, B) \
279	279	do { \
280		− uaccess_enable(); \
	280	+ uaccess_enable_privileged(); \
281	281	__asm__ __volatile__( \
282	282	" mov %w3, %w7\n" \
283	283	"0: ldxr"B" %w2, [%4]\n" \
@@ -302,7 +302,7 @@ do { \ · 18 unmodified lines
302	302	"i" (-EFAULT), \
303	303	"i" (__SWP_LL_SC_LOOPS) \
304	304	: "memory"); \
305		− uaccess_disable(); \
	305	+ uaccess_disable_privileged(); \
306	306	} while (0)
307	307
308	308	#define __user_swp_asm(data, addr, res, temp, temp2) \

arch/arm64/kernel/asm-offsets.c+1−2

@@ -30,7 +30,6 @@ int main(void) · 29 unmodified lines
30	30	BLANK();
31	31	DEFINE(TSK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags));
32	32	DEFINE(TSK_TI_PREEMPT, offsetof(struct task_struct, thread_info.preempt_count));
33		− DEFINE(TSK_TI_ADDR_LIMIT, offsetof(struct task_struct, thread_info.addr_limit));
34	33	#ifdef CONFIG_ARM64_SW_TTBR0_PAN
35	34	DEFINE(TSK_TI_TTBR0, offsetof(struct task_struct, thread_info.ttbr0));
36	35	#endif
@@ -70,7 +69,7 @@ int main(void) · 33 unmodified lines
70	69	DEFINE(S_PSTATE, offsetof(struct pt_regs, pstate));
71	70	DEFINE(S_PC, offsetof(struct pt_regs, pc));
72	71	DEFINE(S_SYSCALLNO, offsetof(struct pt_regs, syscallno));
73		− DEFINE(S_ORIG_ADDR_LIMIT, offsetof(struct pt_regs, orig_addr_limit));
	72	+ DEFINE(S_SDEI_TTBR1, offsetof(struct pt_regs, sdei_ttbr1));
74	73	DEFINE(S_PMR_SAVE, offsetof(struct pt_regs, pmr_save));
75	74	DEFINE(S_STACKFRAME, offsetof(struct pt_regs, stackframe));
76	75	DEFINE(S_FRAME_SIZE, sizeof(struct pt_regs));

arch/arm64/kernel/cpufeature.c+22−37

@@ -153,10 +153,6 @@ EXPORT_SYMBOL(cpu_hwcap_keys); · 152 unmodified lines
153	153	.width = 0, \
154	154	}
155	155
156		−/* meta feature for alternatives */
157		−static bool __maybe_unused
158		−cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused);
159		−
160	156	static void cpu_enable_cnp(struct arm64_cpu_capabilities const *cap);
161	157
162	158	static bool __system_matches_cap(unsigned int n);
@@ -1528,8 +1524,10 @@ bool cpu_has_amu_feat(int cpu) · 1365 unmodified lines
1528	1524	return cpumask_test_cpu(cpu, &amu_cpus);
1529	1525	}
1530	1526
1531		−/* Initialize the use of AMU counters for frequency invariance */
1532		−extern void init_cpu_freq_invariance_counters(void);
	1527	+int get_cpu_with_amu_feat(void)
	1528	+{
	1529	+ return cpumask_any(&amu_cpus);
	1530	+}
1533	1531
1534	1532	static void cpu_amu_enable(struct arm64_cpu_capabilities const *cap)
1535	1533	{
@@ -1537,7 +1535,7 @@ static void cpu_amu_enable(struct arm64_cpu_capabilities const *cap) · 1 unmodified line
1537	1535	pr_info("detected CPU%d: Activity Monitors Unit (AMU)\n",
1538	1536	smp_processor_id());
1539	1537	cpumask_set_cpu(smp_processor_id(), &amu_cpus);
1540		− init_cpu_freq_invariance_counters();
	1538	+ update_freq_counters_refs();
1541	1539	}
1542	1540	}
1543	1541
@@ -1559,6 +1557,11 @@ static bool has_amu(const struct arm64_cpu_capabilities *cap, · 15 unmodified lines
1559	1557
1560	1558	return true;
1561	1559	}
	1560	+#else
	1561	+int get_cpu_with_amu_feat(void)
	1562	+{
	1563	+ return nr_cpu_ids;
	1564	+}
1562	1565	#endif
1563	1566
1564	1567	#ifdef CONFIG_ARM64_VHE
@@ -1600,7 +1603,7 @@ static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) · 35 unmodified lines
1600	1603	WARN_ON_ONCE(in_interrupt());
1601	1604
1602	1605	sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0);
1603		− asm(SET_PSTATE_PAN(1));
	1606	+ set_pstate_pan(1);
1604	1607	}
1605	1608	#endif /* CONFIG_ARM64_PAN */
1606	1609
@@ -1770,28 +1773,6 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 163 unmodified lines
1770	1773	.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
1771	1774	.matches = has_no_hw_prefetch,
1772	1775	},
1773		−#ifdef CONFIG_ARM64_UAO
1774		− {
1775		− .desc = "User Access Override",
1776		− .capability = ARM64_HAS_UAO,
1777		− .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1778		− .matches = has_cpuid_feature,
1779		− .sys_reg = SYS_ID_AA64MMFR2_EL1,
1780		− .field_pos = ID_AA64MMFR2_UAO_SHIFT,
1781		− .min_field_value = 1,
1782		− /*
1783		− * We rely on stop_machine() calling uao_thread_switch() to set
1784		− * UAO immediately after patching.
1785		− */
1786		− },
1787		−#endif /* CONFIG_ARM64_UAO */
1788		−#ifdef CONFIG_ARM64_PAN
1789		− {
1790		− .capability = ARM64_ALT_PAN_NOT_UAO,
1791		− .type = ARM64_CPUCAP_SYSTEM_FEATURE,
1792		− .matches = cpufeature_pan_not_uao,
1793		− },
1794		−#endif /* CONFIG_ARM64_PAN */
1795	1776	#ifdef CONFIG_ARM64_VHE
1796	1777	{
1797	1778	.desc = "Virtualization Host Extensions",
@@ -2138,6 +2119,16 @@ static const struct arm64_cpu_capabilities arm64_features[] = { · 340 unmodified lines
2138	2119	.cpu_enable = cpu_enable_mte,
2139	2120	},
2140	2121	#endif /* CONFIG_ARM64_MTE */
	2122	+ {
	2123	+ .desc = "RCpc load-acquire (LDAPR)",
	2124	+ .capability = ARM64_HAS_LDAPR,
	2125	+ .type = ARM64_CPUCAP_SYSTEM_FEATURE,
	2126	+ .sys_reg = SYS_ID_AA64ISAR1_EL1,
	2127	+ .sign = FTR_UNSIGNED,
	2128	+ .field_pos = ID_AA64ISAR1_LRCPC_SHIFT,
	2129	+ .matches = has_cpuid_feature,
	2130	+ .min_field_value = 1,
	2131	+ },
2141	2132	{},
2142	2133	};
2143	2134
@@ -2652,7 +2643,7 @@ bool this_cpu_has_cap(unsigned int n) · 508 unmodified lines
2652	2643	* - The SYSTEM_FEATURE cpu_hwcaps may not have been set.
2653	2644	* In all other cases cpus_have_{const_}cap() should be used.
2654	2645	*/
2655		−static bool __system_matches_cap(unsigned int n)
	2646	+static bool __maybe_unused __system_matches_cap(unsigned int n)
2656	2647	{
2657	2648	if (n < ARM64_NCAPS) {
2658	2649	const struct arm64_cpu_capabilities *cap = cpu_hwcaps_ptrs[n];
@@ -2732,12 +2723,6 @@ void __init setup_cpu_features(void) · 73 unmodified lines
2732	2723	ARCH_DMA_MINALIGN);
2733	2724	}
2734	2725
2735		−static bool __maybe_unused
2736		−cpufeature_pan_not_uao(const struct arm64_cpu_capabilities *entry, int __unused)
2737		−{
2738		− return (__system_matches_cap(ARM64_HAS_PAN) && !__system_matches_cap(ARM64_HAS_UAO));
2739		−}
2740		−
2741	2726	static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap)
2742	2727	{
2743	2728	cpu_replace_ttbr1(lm_alias(swapper_pg_dir));

arch/arm64/kernel/debug-monitors.c+2−3

@@ -234,9 +234,8 @@ static void send_user_sigtrap(int si_code) · 233 unmodified lines
234	234	if (interrupts_enabled(regs))
235	235	local_irq_enable();
236	236
237		− arm64_force_sig_fault(SIGTRAP, si_code,
238		− (void __user *)instruction_pointer(regs),
239		− "User debug trap");
	237	+ arm64_force_sig_fault(SIGTRAP, si_code, instruction_pointer(regs),
	238	+ "User debug trap");
240	239	}
241	240
242	241	static int single_step_handler(unsigned long unused, unsigned int esr,

arch/arm64/kernel/efi-header.S+49−35

@@ -7,30 +7,48 @@ · 6 unmodified lines
7	7	#include <linux/pe.h>
8	8	#include <linux/sizes.h>
9	9
	10	+ .macro efi_signature_nop
	11	+#ifdef CONFIG_EFI
	12	+.L_head:
	13	+ /*
	14	+ * This ccmp instruction has no meaningful effect except that
	15	+ * its opcode forms the magic "MZ" signature required by UEFI.
	16	+ */
	17	+ ccmp x18, #0, #0xd, pl
	18	+#else
	19	+ /*
	20	+ * Bootloaders may inspect the opcode at the start of the kernel
	21	+ * image to decide if the kernel is capable of booting via UEFI.
	22	+ * So put an ordinary NOP here, not the "MZ.." pseudo-nop above.
	23	+ */
	24	+ nop
	25	+#endif
	26	+ .endm
	27	+
10	28	.macro __EFI_PE_HEADER
	29	+#ifdef CONFIG_EFI
	30	+ .set .Lpe_header_offset, . - .L_head
11	31	.long PE_MAGIC
12		−coff_header:
13	32	.short IMAGE_FILE_MACHINE_ARM64 // Machine
14		− .short section_count // NumberOfSections
	33	+ .short .Lsection_count // NumberOfSections
15	34	.long 0 // TimeDateStamp
16	35	.long 0 // PointerToSymbolTable
17	36	.long 0 // NumberOfSymbols
18		− .short section_table - optional_header // SizeOfOptionalHeader
	37	+ .short .Lsection_table - .Loptional_header // SizeOfOptionalHeader
19	38	.short IMAGE_FILE_DEBUG_STRIPPED \| \
20	39	IMAGE_FILE_EXECUTABLE_IMAGE \| \
21	40	IMAGE_FILE_LINE_NUMS_STRIPPED // Characteristics
22	41
23		−optional_header:
	42	+.Loptional_header:
24	43	.short PE_OPT_MAGIC_PE32PLUS // PE32+ format
25	44	.byte 0x02 // MajorLinkerVersion
26	45	.byte 0x14 // MinorLinkerVersion
27		− .long __initdata_begin - efi_header_end // SizeOfCode
	46	+ .long __initdata_begin - .Lefi_header_end // SizeOfCode
28	47	.long __pecoff_data_size // SizeOfInitializedData
29	48	.long 0 // SizeOfUninitializedData
30		− .long __efistub_efi_pe_entry - _head // AddressOfEntryPoint
31		− .long efi_header_end - _head // BaseOfCode
	49	+ .long __efistub_efi_pe_entry - .L_head // AddressOfEntryPoint
	50	+ .long .Lefi_header_end - .L_head // BaseOfCode
32	51
33		−extra_header_fields:
34	52	.quad 0 // ImageBase
35	53	.long SEGMENT_ALIGN // SectionAlignment
36	54	.long PECOFF_FILE_ALIGNMENT // FileAlignment
@@ -42,10 +60,10 @@ extra_header_fields: · 5 unmodified lines
42	60	.short 0 // MinorSubsystemVersion
43	61	.long 0 // Win32VersionValue
44	62
45		− .long _end - _head // SizeOfImage
	63	+ .long _end - .L_head // SizeOfImage
46	64
47	65	// Everything before the kernel image is considered part of the header
48		− .long efi_header_end - _head // SizeOfHeaders
	66	+ .long .Lefi_header_end - .L_head // SizeOfHeaders
49	67	.long 0 // CheckSum
50	68	.short IMAGE_SUBSYSTEM_EFI_APPLICATION // Subsystem
51	69	.short 0 // DllCharacteristics
@@ -54,7 +72,7 @@ extra_header_fields: · 2 unmodified lines
54	72	.quad 0 // SizeOfHeapReserve
55	73	.quad 0 // SizeOfHeapCommit
56	74	.long 0 // LoaderFlags
57		− .long (section_table - .) / 8 // NumberOfRvaAndSizes
	75	+ .long (.Lsection_table - .) / 8 // NumberOfRvaAndSizes
58	76
59	77	.quad 0 // ExportTable
60	78	.quad 0 // ImportTable
@@ -64,17 +82,17 @@ extra_header_fields: · 3 unmodified lines
64	82	.quad 0 // BaseRelocationTable
65	83
66	84	#ifdef CONFIG_DEBUG_EFI
67		− .long efi_debug_table - _head // DebugTable
68		− .long efi_debug_table_size
	85	+ .long .Lefi_debug_table - .L_head // DebugTable
	86	+ .long .Lefi_debug_table_size
69	87	#endif
70	88
71	89	// Section table
72		−section_table:
	90	+.Lsection_table:
73	91	.ascii ".text\0\0\0"
74		− .long __initdata_begin - efi_header_end // VirtualSize
75		− .long efi_header_end - _head // VirtualAddress
76		− .long __initdata_begin - efi_header_end // SizeOfRawData
77		− .long efi_header_end - _head // PointerToRawData
	92	+ .long __initdata_begin - .Lefi_header_end // VirtualSize
	93	+ .long .Lefi_header_end - .L_head // VirtualAddress
	94	+ .long __initdata_begin - .Lefi_header_end // SizeOfRawData
	95	+ .long .Lefi_header_end - .L_head // PointerToRawData
78	96
79	97	.long 0 // PointerToRelocations
80	98	.long 0 // PointerToLineNumbers
@@ -86,9 +104,9 @@ section_table: · 5 unmodified lines
86	104
87	105	.ascii ".data\0\0\0"
88	106	.long __pecoff_data_size // VirtualSize
89		− .long __initdata_begin - _head // VirtualAddress
	107	+ .long __initdata_begin - .L_head // VirtualAddress
90	108	.long __pecoff_data_rawsize // SizeOfRawData
91		− .long __initdata_begin - _head // PointerToRawData
	109	+ .long __initdata_begin - .L_head // PointerToRawData
92	110
93	111	.long 0 // PointerToRelocations
94	112	.long 0 // PointerToLineNumbers
@@ -98,7 +116,7 @@ section_table: · 3 unmodified lines
98	116	IMAGE_SCN_MEM_READ \| \
99	117	IMAGE_SCN_MEM_WRITE // Characteristics
100	118
101		− .set section_count, (. - section_table) / 40
	119	+ .set .Lsection_count, (. - .Lsection_table) / 40
102	120
103	121	#ifdef CONFIG_DEBUG_EFI
104	122	/*
@@ -114,21 +132,21 @@ section_table: · 9 unmodified lines
114	132	__INITRODATA
115	133
116	134	.align 2
117		−efi_debug_table:
	135	+.Lefi_debug_table:
118	136	// EFI_IMAGE_DEBUG_DIRECTORY_ENTRY
119	137	.long 0 // Characteristics
120	138	.long 0 // TimeDateStamp
121	139	.short 0 // MajorVersion
122	140	.short 0 // MinorVersion
123	141	.long IMAGE_DEBUG_TYPE_CODEVIEW // Type
124		− .long efi_debug_entry_size // SizeOfData
	142	+ .long .Lefi_debug_entry_size // SizeOfData
125	143	.long 0 // RVA
126		− .long efi_debug_entry - _head // FileOffset
	144	+ .long .Lefi_debug_entry - .L_head // FileOffset
127	145
128		− .set efi_debug_table_size, . - efi_debug_table
	146	+ .set .Lefi_debug_table_size, . - .Lefi_debug_table
129	147	.previous
130	148
131		−efi_debug_entry:
	149	+.Lefi_debug_entry:
132	150	// EFI_IMAGE_DEBUG_CODEVIEW_NB10_ENTRY
133	151	.ascii "NB10" // Signature
134	152	.long 0 // Unknown
@@ -137,16 +155,12 @@ efi_debug_entry: · 2 unmodified lines
137	155
138	156	.asciz VMLINUX_PATH
139	157
140		− .set efi_debug_entry_size, . - efi_debug_entry
	158	+ .set .Lefi_debug_entry_size, . - .Lefi_debug_entry
141	159	#endif
142	160
143		− /*
144		− * EFI will load .text onwards at the 4k section alignment
145		− * described in the PE/COFF header. To ensure that instruction
146		− * sequences using an adrp and a :lo12: immediate will function
147		− * correctly at this alignment, we must ensure that .text is
148		− * placed at a 4k boundary in the Image to begin with.
149		− */
150	161	.balign SEGMENT_ALIGN
151		−efi_header_end:
	162	+.Lefi_header_end:
	163	+#else
	164	+ .set .Lpe_header_offset, 0x0
	165	+#endif
152	166	.endm

arch/arm64/kernel/entry-common.c+0−2

@@ -115,7 +115,6 @@ static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) · 114 unmodified lines
115	115
116	116	enter_from_kernel_mode(regs);
117	117	local_daif_inherit(regs);
118		− far = untagged_addr(far);
119	118	do_mem_abort(far, esr, regs);
120	119	local_daif_mask();
121	120	exit_to_kernel_mode(regs);
@@ -256,7 +255,6 @@ static void noinstr el0_da(struct pt_regs *regs, unsigned long esr) · 134 unmodified lines
256	255
257	256	enter_from_user_mode();
258	257	local_daif_restore(DAIF_PROCCTX);
259		− far = untagged_addr(far);
260	258	do_mem_abort(far, esr, regs);
261	259	}
262	260

arch/arm64/kernel/entry.S+10−21

@@ -216,12 +216,6 @@ alternative_else_nop_endif · 215 unmodified lines
216	216	.else
217	217	add x21, sp, #S_FRAME_SIZE
218	218	get_current_task tsk
219		− /* Save the task's original addr_limit and set USER_DS */
220		− ldr x20, [tsk, #TSK_TI_ADDR_LIMIT]
221		− str x20, [sp, #S_ORIG_ADDR_LIMIT]
222		− mov x20, #USER_DS
223		− str x20, [tsk, #TSK_TI_ADDR_LIMIT]
224		− /* No need to reset PSTATE.UAO, hardware's already set it to 0 for us */
225	219	.endif /* \el == 0 */
226	220	mrs x22, elr_el1
227	221	mrs x23, spsr_el1
@@ -279,12 +273,6 @@ alternative_else_nop_endif · 51 unmodified lines
279	273	.macro kernel_exit, el
280	274	.if \el != 0
281	275	disable_daif
282		−
283		− /* Restore the task's original addr_limit. */
284		− ldr x20, [sp, #S_ORIG_ADDR_LIMIT]
285		− str x20, [tsk, #TSK_TI_ADDR_LIMIT]
286		−
287		− /* No need to restore UAO, it will be restored from SPSR_EL1 */
288	276	.endif
289	277
290	278	/* Restore pmr */
@@ -438,7 +426,7 @@ SYM_CODE_END(__swpan_exit_el0) · 147 unmodified lines
438	426
439	427	#ifdef CONFIG_SHADOW_CALL_STACK
440	428	/* also switch to the irq shadow stack */
441		− adr_this_cpu scs_sp, irq_shadow_call_stack, x26
	429	+ ldr_this_cpu scs_sp, irq_shadow_call_stack_ptr, x26
442	430	#endif
443	431
444	432	9998:
@@ -773,9 +761,10 @@ SYM_CODE_END(ret_to_user) · 328 unmodified lines
773	761	*/
774	762	.pushsection ".entry.tramp.text", "ax"
775	763
	764	+ // Move from tramp_pg_dir to swapper_pg_dir
776	765	.macro tramp_map_kernel, tmp
777	766	mrs \tmp, ttbr1_el1
778		− add \tmp, \tmp, #(PAGE_SIZE + RESERVED_TTBR0_SIZE)
	767	+ add \tmp, \tmp, #(2 * PAGE_SIZE)
779	768	bic \tmp, \tmp, #USER_ASID_FLAG
780	769	msr ttbr1_el1, \tmp
781	770	#ifdef CONFIG_QCOM_FALKOR_ERRATUM_1003
@@ -792,9 +781,10 @@ alternative_else_nop_endif · 10 unmodified lines
792	781	#endif /* CONFIG_QCOM_FALKOR_ERRATUM_1003 */
793	782	.endm
794	783
	784	+ // Move from swapper_pg_dir to tramp_pg_dir
795	785	.macro tramp_unmap_kernel, tmp
796	786	mrs \tmp, ttbr1_el1
797		− sub \tmp, \tmp, #(PAGE_SIZE + RESERVED_TTBR0_SIZE)
	787	+ sub \tmp, \tmp, #(2 * PAGE_SIZE)
798	788	orr \tmp, \tmp, #USER_ASID_FLAG
799	789	msr ttbr1_el1, \tmp
800	790	/*
@@ -965,10 +955,9 @@ SYM_CODE_START(__sdei_asm_entry_trampoline) · 164 unmodified lines
965	955	mov x4, xzr
966	956
967	957	/*
968		− * Use reg->interrupted_regs.addr_limit to remember whether to unmap
969		− * the kernel on exit.
	958	+ * Remember whether to unmap the kernel on exit.
970	959	*/
971		−1: str x4, [x1, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)]
	960	+1: str x4, [x1, #(SDEI_EVENT_INTREGS + S_SDEI_TTBR1)]
972	961
973	962	#ifdef CONFIG_RANDOMIZE_BASE
974	963	adr x4, tramp_vectors + PAGE_SIZE
@@ -989,7 +978,7 @@ NOKPROBE(__sdei_asm_entry_trampoline) · 14 unmodified lines
989	978	* x4: struct sdei_registered_event argument from registration time.
990	979	*/
991	980	SYM_CODE_START(__sdei_asm_exit_trampoline)
992		− ldr x4, [x4, #(SDEI_EVENT_INTREGS + S_ORIG_ADDR_LIMIT)]
	981	+ ldr x4, [x4, #(SDEI_EVENT_INTREGS + S_SDEI_TTBR1)]
993	982	cbnz x4, 1f
994	983
995	984	tramp_unmap_kernel tmp=x4
@@ -1063,9 +1052,9 @@ SYM_CODE_START(__sdei_asm_handler) · 67 unmodified lines
1063	1052	#ifdef CONFIG_SHADOW_CALL_STACK
1064	1053	/* Use a separate shadow call stack for normal and critical events */
1065	1054	cbnz w4, 3f
1066		− adr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_normal, tmp=x6
	1055	+ ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_normal_ptr, tmp=x6
1067	1056	b 4f
1068		−3: adr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_critical, tmp=x6
	1057	+3: ldr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_critical_ptr, tmp=x6
1069	1058	4:
1070	1059	#endif
1071	1060

arch/arm64/kernel/head.S+33−37

@@ -58,21 +58,11 @@ · 57 unmodified lines
58	58	* in the entry routines.
59	59	*/
60	60	__HEAD
61		−_head:
62	61	/*
63	62	* DO NOT MODIFY. Image header expected by Linux boot-loaders.
64	63	*/
65		−#ifdef CONFIG_EFI
66		− /*
67		− * This add instruction has no meaningful effect except that
68		− * its opcode forms the magic "MZ" signature required by UEFI.
69		− */
70		− add x13, x18, #0x16
71		− b primary_entry
72		−#else
	64	+ efi_signature_nop // special NOP to identity as PE/COFF executable
73	65	b primary_entry // branch to kernel start, magic
74		− .long 0 // reserved
75		−#endif
76	66	.quad 0 // Image load offset from start of RAM, little-endian
77	67	le64sym _kernel_size_le // Effective size of kernel image, little-endian
78	68	le64sym _kernel_flags_le // Informative flags, little-endian
@@ -80,14 +70,9 @@ _head: · 1 unmodified line
80	70	.quad 0 // reserved
81	71	.quad 0 // reserved
82	72	.ascii ARM64_IMAGE_MAGIC // Magic number
83		−#ifdef CONFIG_EFI
84		− .long pe_header - _head // Offset to the PE header.
	73	+ .long .Lpe_header_offset // Offset to the PE header.
85	74
86		−pe_header:
87	75	__EFI_PE_HEADER
88		−#else
89		− .long 0 // reserved
90		−#endif
91	76
92	77	__INIT
93	78
@@ -104,7 +89,7 @@ pe_header: · 10 unmodified lines
104	89	*/
105	90	SYM_CODE_START(primary_entry)
106	91	bl preserve_boot_args
107		− bl el2_setup // Drop to EL1, w0=cpu_boot_mode
	92	+ bl init_kernel_el // w0=cpu_boot_mode
108	93	adrp x23, __PHYS_OFFSET
109	94	and x23, x23, MIN_KIMG_ALIGN - 1 // KASLR offset, defaults to 0
110	95	bl set_cpu_boot_mode_flag
@@ -482,24 +467,33 @@ EXPORT_SYMBOL(kimage_vaddr) · 371 unmodified lines
482	467	.section ".idmap.text","awx"
483	468
484	469	/*
485		− * If we're fortunate enough to boot at EL2, ensure that the world is
486		− * sane before dropping to EL1.
	470	+ * Starting from EL2 or EL1, configure the CPU to execute at the highest
	471	+ * reachable EL supported by the kernel in a chosen default state. If dropping
	472	+ * from EL2 to EL1, configure EL2 before configuring EL1.
	473	+ *
	474	+ * Since we cannot always rely on ERET synchronizing writes to sysregs (e.g. if
	475	+ * SCTLR_ELx.EOS is clear), we place an ISB prior to ERET.
487	476	*
488	477	* Returns either BOOT_CPU_MODE_EL1 or BOOT_CPU_MODE_EL2 in w0 if
489	478	* booted in EL1 or EL2 respectively.
490	479	*/
491		−SYM_FUNC_START(el2_setup)
492		− msr SPsel, #1 // We want to use SP_EL{1,2}
	480	+SYM_FUNC_START(init_kernel_el)
493	481	mrs x0, CurrentEL
494	482	cmp x0, #CurrentEL_EL2
495		− b.eq 1f
496		− mov_q x0, (SCTLR_EL1_RES1 \| ENDIAN_SET_EL1)
	483	+ b.eq init_el2
	484	+
	485	+SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
	486	+ mov_q x0, INIT_SCTLR_EL1_MMU_OFF
497	487	msr sctlr_el1, x0
498		− mov w0, #BOOT_CPU_MODE_EL1 // This cpu booted in EL1
499	488	isb
500		− ret
	489	+ mov_q x0, INIT_PSTATE_EL1
	490	+ msr spsr_el1, x0
	491	+ msr elr_el1, lr
	492	+ mov w0, #BOOT_CPU_MODE_EL1
	493	+ eret
501	494
502		−1: mov_q x0, (SCTLR_EL2_RES1 \| ENDIAN_SET_EL2)
	495	+SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
	496	+ mov_q x0, INIT_SCTLR_EL2_MMU_OFF
503	497	msr sctlr_el2, x0
504	498
505	499	#ifdef CONFIG_ARM64_VHE
@@ -608,9 +602,12 @@ set_hcr: · 102 unmodified lines
608	602
609	603	cbz x2, install_el2_stub
610	604
611		− mov w0, #BOOT_CPU_MODE_EL2 // This CPU booted in EL2
612	605	isb
613		− ret
	606	+ mov_q x0, INIT_PSTATE_EL2
	607	+ msr spsr_el2, x0
	608	+ msr elr_el2, lr
	609	+ mov w0, #BOOT_CPU_MODE_EL2
	610	+ eret
614	611
615	612	SYM_INNER_LABEL(install_el2_stub, SYM_L_LOCAL)
616	613	/*
@@ -620,7 +617,7 @@ SYM_INNER_LABEL(install_el2_stub, SYM_L_LOCAL) · 3 unmodified lines
620	617	* requires no configuration, and all non-hyp-specific EL2 setup
621	618	* will be done via the _EL1 system register aliases in __cpu_setup.
622	619	*/
623		− mov_q x0, (SCTLR_EL1_RES1 \| ENDIAN_SET_EL1)
	620	+ mov_q x0, INIT_SCTLR_EL1_MMU_OFF
624	621	msr sctlr_el1, x0
625	622
626	623	/* Coprocessor traps. */
@@ -642,14 +639,13 @@ SYM_INNER_LABEL(install_el2_stub, SYM_L_LOCAL) · 15 unmodified lines
642	639	7: adr_l x0, __hyp_stub_vectors
643	640	msr vbar_el2, x0
644	641
645		− /* spsr */
646		− mov x0, #(PSR_F_BIT \| PSR_I_BIT \| PSR_A_BIT \| PSR_D_BIT \|\
647		− PSR_MODE_EL1h)
	642	+ isb
	643	+ mov x0, #INIT_PSTATE_EL1
648	644	msr spsr_el2, x0
649	645	msr elr_el2, lr
650		− mov w0, #BOOT_CPU_MODE_EL2 // This CPU booted in EL2
	646	+ mov w0, #BOOT_CPU_MODE_EL2
651	647	eret
652		−SYM_FUNC_END(el2_setup)
	648	+SYM_FUNC_END(init_kernel_el)
653	649
654	650	/*
655	651	* Sets the __boot_cpu_mode flag depending on the CPU boot mode passed
@@ -699,7 +695,7 @@ SYM_DATA_END(__early_cpu_boot_status) · 43 unmodified lines
699	695	* cores are held until we're ready for them to initialise.
700	696	*/
701	697	SYM_FUNC_START(secondary_holding_pen)
702		− bl el2_setup // Drop to EL1, w0=cpu_boot_mode
	698	+ bl init_kernel_el // w0=cpu_boot_mode
703	699	bl set_cpu_boot_mode_flag
704	700	mrs x0, mpidr_el1
705	701	mov_q x1, MPIDR_HWID_BITMASK
@@ -717,7 +713,7 @@ SYM_FUNC_END(secondary_holding_pen) · 11 unmodified lines
717	713	* be used where CPUs are brought online dynamically by the kernel.
718	714	*/
719	715	SYM_FUNC_START(secondary_entry)
720		− bl el2_setup // Drop to EL1
	716	+ bl init_kernel_el // w0=cpu_boot_mode
721	717	bl set_cpu_boot_mode_flag
722	718	b secondary_startup
723	719	SYM_FUNC_END(secondary_entry)

arch/arm64/kernel/irq.c+21−0

@@ -17,6 +17,7 @@ · 16 unmodified lines
17	17	#include <linux/init.h>
18	18	#include <linux/irqchip.h>
19	19	#include <linux/kprobes.h>
	20	+#include <linux/scs.h>
20	21	#include <linux/seq_file.h>
21	22	#include <linux/vmalloc.h>
22	23	#include <asm/daifflags.h>
@@ -27,6 +28,25 @@ DEFINE_PER_CPU(struct nmi_ctx, nmi_contexts); · 4 unmodified lines
27	28
28	29	DEFINE_PER_CPU(unsigned long *, irq_stack_ptr);
29	30
	31	+
	32	+DECLARE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
	33	+
	34	+#ifdef CONFIG_SHADOW_CALL_STACK
	35	+DEFINE_PER_CPU(unsigned long *, irq_shadow_call_stack_ptr);
	36	+#endif
	37	+
	38	+static void init_irq_scs(void)
	39	+{
	40	+ int cpu;
	41	+
	42	+ if (!IS_ENABLED(CONFIG_SHADOW_CALL_STACK))
	43	+ return;
	44	+
	45	+ for_each_possible_cpu(cpu)
	46	+ per_cpu(irq_shadow_call_stack_ptr, cpu) =
	47	+ scs_alloc(cpu_to_node(cpu));
	48	+}
	49	+
30	50	#ifdef CONFIG_VMAP_STACK
31	51	static void init_irq_stacks(void)
32	52	{
@@ -54,6 +74,7 @@ static void init_irq_stacks(void) · 21 unmodified lines
54	74	void __init init_IRQ(void)
55	75	{
56	76	init_irq_stacks();
	77	+ init_irq_scs();
57	78	irqchip_init();
58	79	if (!handle_arch_irq)
59	80	panic("No interrupt controller found.");

arch/arm64/kernel/kaslr.c+18−8

@@ -50,10 +50,16 @@ static __init u64 get_kaslr_seed(void *fdt) · 49 unmodified lines
50	50	return ret;
51	51	}
52	52
53		−static __init const u8 kaslr_get_cmdline(void fdt)
	53	+static __init bool cmdline_contains_nokaslr(const u8 *cmdline)
54	54	{
55		− static __initconst const u8 default_cmdline[] = CONFIG_CMDLINE;
	55	+ const u8 *str;
56	56
	57	+ str = strstr(cmdline, "nokaslr");
	58	+ return str == cmdline \|\| (str > cmdline && *(str - 1) == ' ');
	59	+}
	60	+
	61	+static __init bool is_kaslr_disabled_cmdline(void *fdt)
	62	+{
57	63	if (!IS_ENABLED(CONFIG_CMDLINE_FORCE)) {
58	64	int node;
59	65	const u8 *prop;
@@ -65,10 +71,17 @@ static __init const u8 kaslr_get_cmdline(void fdt) · 5 unmodified lines
65	71	prop = fdt_getprop(fdt, node, "bootargs", NULL);
66	72	if (!prop)
67	73	goto out;
68		− return prop;
	74	+
	75	+ if (cmdline_contains_nokaslr(prop))
	76	+ return true;
	77	+
	78	+ if (IS_ENABLED(CONFIG_CMDLINE_EXTEND))
	79	+ goto out;
	80	+
	81	+ return false;
69	82	}
70	83	out:
71		− return default_cmdline;
	84	+ return cmdline_contains_nokaslr(CONFIG_CMDLINE);
72	85	}
73	86
74	87	/*
@@ -83,7 +96,6 @@ u64 __init kaslr_early_init(u64 dt_phys) · 8 unmodified lines
83	96	{
84	97	void *fdt;
85	98	u64 seed, offset, mask, module_range;
86		− const u8 cmdline, str;
87	99	unsigned long raw;
88	100	int size;
89	101
@@ -115,9 +127,7 @@ u64 __init kaslr_early_init(u64 dt_phys) · 25 unmodified lines
115	127	* Check if 'nokaslr' appears on the command line, and
116	128	* return 0 if that is the case.
117	129	*/
118		− cmdline = kaslr_get_cmdline(fdt);
119		− str = strstr(cmdline, "nokaslr");
120		− if (str == cmdline \|\| (str > cmdline && *(str - 1) == ' ')) {
	130	+ if (is_kaslr_disabled_cmdline(fdt)) {
121	131	kaslr_status = KASLR_DISABLED_CMDLINE;
122	132	return 0;
123	133	}

arch/arm64/kernel/mte.c+2−1

@@ -189,7 +189,8 @@ long get_mte_ctrl(struct task_struct *task) · 188 unmodified lines
189	189
190	190	switch (task->thread.sctlr_tcf0) {
191	191	case SCTLR_EL1_TCF0_NONE:
192		− return PR_MTE_TCF_NONE;
	192	+ ret \|= PR_MTE_TCF_NONE;
	193	+ break;
193	194	case SCTLR_EL1_TCF0_SYNC:
194	195	ret \|= PR_MTE_TCF_SYNC;
195	196	break;

arch/arm64/kernel/perf_event.c+39−2

@@ -23,6 +23,8 @@ · 22 unmodified lines
23	23	#include <linux/platform_device.h>
24	24	#include <linux/sched_clock.h>
25	25	#include <linux/smp.h>
	26	+#include <linux/nmi.h>
	27	+#include <linux/cpufreq.h>
26	28
27	29	/* ARMv8 Cortex-A53 specific event types. */
28	30	#define ARMV8_A53_PERFCTR_PREF_LINEFILL 0xC2
@@ -1248,10 +1250,21 @@ static struct platform_driver armv8_pmu_driver = { · 1219 unmodified lines
1248	1250
1249	1251	static int __init armv8_pmu_driver_init(void)
1250	1252	{
	1253	+ int ret;
	1254	+
1251	1255	if (acpi_disabled)
1252		− return platform_driver_register(&armv8_pmu_driver);
	1256	+ ret = platform_driver_register(&armv8_pmu_driver);
1253	1257	else
1254		− return arm_pmu_acpi_probe(armv8_pmuv3_init);
	1258	+ ret = arm_pmu_acpi_probe(armv8_pmuv3_init);
	1259	+
	1260	+ /*
	1261	+ * Try to re-initialize lockup detector after PMU init in
	1262	+ * case PMU events are triggered via NMIs.
	1263	+ */
	1264	+ if (ret == 0 && arm_pmu_irq_is_nmi())
	1265	+ lockup_detector_init();
	1266	+
	1267	+ return ret;
1255	1268	}
1256	1269	device_initcall(armv8_pmu_driver_init)
1257	1270
@@ -1309,3 +1322,27 @@ void arch_perf_update_userpage(struct perf_event *event, · 51 unmodified lines
1309	1322	userpg->cap_user_time_zero = 1;
1310	1323	userpg->cap_user_time_short = 1;
1311	1324	}
	1325	+
	1326	+#ifdef CONFIG_HARDLOCKUP_DETECTOR_PERF
	1327	+/*
	1328	+ * Safe maximum CPU frequency in case a particular platform doesn't implement
	1329	+ * cpufreq driver. Although, architecture doesn't put any restrictions on
	1330	+ * maximum frequency but 5 GHz seems to be safe maximum given the available
	1331	+ * Arm CPUs in the market which are clocked much less than 5 GHz. On the other
	1332	+ * hand, we can't make it much higher as it would lead to a large hard-lockup
	1333	+ * detection timeout on parts which are running slower (eg. 1GHz on
	1334	+ * Developerbox) and doesn't possess a cpufreq driver.
	1335	+ */
	1336	+#define SAFE_MAX_CPU_FREQ 5000000000UL // 5 GHz
	1337	+u64 hw_nmi_get_sample_period(int watchdog_thresh)
	1338	+{
	1339	+ unsigned int cpu = smp_processor_id();
	1340	+ unsigned long max_cpu_freq;
	1341	+
	1342	+ max_cpu_freq = cpufreq_get_hw_max_freq(cpu) * 1000UL;
	1343	+ if (!max_cpu_freq)
	1344	+ max_cpu_freq = SAFE_MAX_CPU_FREQ;
	1345	+
	1346	+ return (u64)max_cpu_freq * watchdog_thresh;
	1347	+}
	1348	+#endif

arch/arm64/kernel/probes/kprobes.c+12−41

@@ -34,7 +34,7 @@ DEFINE_PER_CPU(struct kprobe *, current_kprobe) = NULL; · 33 unmodified lines
34	34	DEFINE_PER_CPU(struct kprobe_ctlblk, kprobe_ctlblk);
35	35
36	36	static void __kprobes
37		−post_kprobe_handler(struct kprobe_ctlblk , struct pt_regs );
	37	+post_kprobe_handler(struct kprobe , struct kprobe_ctlblk , struct pt_regs *);
38	38
39	39	static void __kprobes arch_prepare_ss_slot(struct kprobe *p)
40	40	{
@@ -68,7 +68,7 @@ static void __kprobes arch_simulate_insn(struct kprobe p, struct pt_regs regs) · 27 unmodified lines
68	68	p->ainsn.api.handler((u32)p->opcode, (long)p->addr, regs);
69	69
70	70	/* single step simulated, now go for post processing */
71		− post_kprobe_handler(kcb, regs);
	71	+ post_kprobe_handler(p, kcb, regs);
72	72	}
73	73
74	74	int __kprobes arch_prepare_kprobe(struct kprobe *p)
@@ -177,19 +177,6 @@ static void __kprobes kprobes_restore_local_irqflag(struct kprobe_ctlblk *kcb, · 102 unmodified lines
177	177	regs->pstate \|= kcb->saved_irqflag;
178	178	}
179	179
180		−static void __kprobes
181		−set_ss_context(struct kprobe_ctlblk *kcb, unsigned long addr)
182		−{
183		− kcb->ss_ctx.ss_pending = true;
184		− kcb->ss_ctx.match_addr = addr + sizeof(kprobe_opcode_t);
185		−}
186		−
187		−static void __kprobes clear_ss_context(struct kprobe_ctlblk *kcb)
188		−{
189		− kcb->ss_ctx.ss_pending = false;
190		− kcb->ss_ctx.match_addr = 0;
191		−}
192		−
193	180	static void __kprobes setup_singlestep(struct kprobe *p,
194	181	struct pt_regs *regs,
195	182	struct kprobe_ctlblk *kcb, int reenter)
@@ -209,7 +196,6 @@ static void __kprobes setup_singlestep(struct kprobe *p, · 13 unmodified lines
209	196	/* prepare for single stepping */
210	197	slot = (unsigned long)p->ainsn.api.insn;
211	198
212		− set_ss_context(kcb, slot); /* mark pending ss */
213	199	kprobes_save_local_irqflag(kcb, regs);
214	200	instruction_pointer_set(regs, slot);
215	201	} else {
@@ -243,13 +229,8 @@ static int __kprobes reenter_kprobe(struct kprobe *p, · 27 unmodified lines
243	229	}
244	230
245	231	static void __kprobes
246		−post_kprobe_handler(struct kprobe_ctlblk kcb, struct pt_regs regs)
	232	+post_kprobe_handler(struct kprobe cur, struct kprobe_ctlblk kcb, struct pt_regs *regs)
247	233	{
248		− struct kprobe *cur = kprobe_running();
249		−
250		− if (!cur)
251		− return;
252		−
253	234	/* return addr restore if non-branching insn */
254	235	if (cur->ainsn.api.restore != 0)
255	236	instruction_pointer_set(regs, cur->ainsn.api.restore);
@@ -364,33 +345,23 @@ static void __kprobes kprobe_handler(struct pt_regs *regs) · 108 unmodified lines
364	345	*/
365	346	}
366	347
367		−static int __kprobes
368		−kprobe_ss_hit(struct kprobe_ctlblk *kcb, unsigned long addr)
369		−{
370		− if ((kcb->ss_ctx.ss_pending)
371		− && (kcb->ss_ctx.match_addr == addr)) {
372		− clear_ss_context(kcb); /* clear pending ss */
373		− return DBG_HOOK_HANDLED;
374		− }
375		− /* not ours, kprobes should ignore it */
376		− return DBG_HOOK_ERROR;
377		−}
378		−
379	348	static int __kprobes
380	349	kprobe_breakpoint_ss_handler(struct pt_regs *regs, unsigned int esr)
381	350	{
382	351	struct kprobe_ctlblk *kcb = get_kprobe_ctlblk();
383		− int retval;
384		−
385		− /* return error if this is not our step */
386		− retval = kprobe_ss_hit(kcb, instruction_pointer(regs));
	352	+ unsigned long addr = instruction_pointer(regs);
	353	+ struct kprobe *cur = kprobe_running();
387	354
388		− if (retval == DBG_HOOK_HANDLED) {
	355	+ if (cur && (kcb->kprobe_status == KPROBE_HIT_SS)
	356	+ && ((unsigned long)&cur->ainsn.api.insn[1] == addr)) {
389	357	kprobes_restore_local_irqflag(kcb, regs);
390		− post_kprobe_handler(kcb, regs);
	358	+ post_kprobe_handler(cur, kcb, regs);
	359	+
	360	+ return DBG_HOOK_HANDLED;
391	361	}
392	362
393		− return retval;
	363	+ /* not ours, kprobes should ignore it */
	364	+ return DBG_HOOK_ERROR;
394	365	}
395	366
396	367	static struct break_hook kprobes_break_ss_hook = {

arch/arm64/kernel/process.c+8−21

@@ -422,16 +422,15 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, · 421 unmodified lines
422	422	if (clone_flags & CLONE_SETTLS)
423	423	p->thread.uw.tp_value = tls;
424	424	} else {
	425	+ /*
	426	+ * A kthread has no context to ERET to, so ensure any buggy
	427	+ * ERET is treated as an illegal exception return.
	428	+ *
	429	+ * When a user task is created from a kthread, childregs will
	430	+ * be initialized by start_thread() or start_compat_thread().
	431	+ */
425	432	memset(childregs, 0, sizeof(struct pt_regs));
426		− childregs->pstate = PSR_MODE_EL1h;
427		− if (IS_ENABLED(CONFIG_ARM64_UAO) &&
428		− cpus_have_const_cap(ARM64_HAS_UAO))
429		− childregs->pstate \|= PSR_UAO_BIT;
430		−
431		− spectre_v4_enable_task_mitigation(p);
432		−
433		− if (system_uses_irq_prio_masking())
434		− childregs->pmr_save = GIC_PRIO_IRQON;
	433	+ childregs->pstate = PSR_MODE_EL1h \| PSR_IL_BIT;
435	434
436	435	p->thread.cpu_context.x19 = stack_start;
437	436	p->thread.cpu_context.x20 = stk_sz;
@@ -461,17 +460,6 @@ static void tls_thread_switch(struct task_struct *next) · 23 unmodified lines
461	460	write_sysreg(*task_user_tls(next), tpidr_el0);
462	461	}
463	462
464		−/* Restore the UAO state depending on next's addr_limit */
465		−void uao_thread_switch(struct task_struct *next)
466		−{
467		− if (IS_ENABLED(CONFIG_ARM64_UAO)) {
468		− if (task_thread_info(next)->addr_limit == KERNEL_DS)
469		− asm(ALTERNATIVE("nop", SET_PSTATE_UAO(1), ARM64_HAS_UAO));
470		− else
471		− asm(ALTERNATIVE("nop", SET_PSTATE_UAO(0), ARM64_HAS_UAO));
472		− }
473		−}
474		−
475	463	/*
476	464	* Force SSBS state on context-switch, since it may be lost after migrating
477	465	* from a CPU which treats the bit as RES0 in a heterogeneous system.
@@ -554,7 +542,6 @@ __notrace_funcgraph struct task_struct __switch_to(struct task_struct prev, · 76 unmodified lines
554	542	hw_breakpoint_thread_switch(next);
555	543	contextidr_thread_switch(next);
556	544	entry_task_switch(next);
557		− uao_thread_switch(next);
558	545	ssbs_thread_switch(next);
559	546	erratum_1418040_thread_switch(prev, next);
560	547

arch/arm64/kernel/proton-pack.c+3−2

@@ -24,6 +24,7 @@ · 23 unmodified lines
24	24	#include <linux/prctl.h>
25	25	#include <linux/sched/task_stack.h>
26	26
	27	+#include <asm/insn.h>
27	28	#include <asm/spectre.h>
28	29	#include <asm/traps.h>
29	30
@@ -538,12 +539,12 @@ static enum mitigation_state spectre_v4_enable_hw_mitigation(void) · 508 unmodified lines
538	539
539	540	if (spectre_v4_mitigations_off()) {
540	541	sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_DSSBS);
541		− asm volatile(SET_PSTATE_SSBS(1));
	542	+ set_pstate_ssbs(1);
542	543	return SPECTRE_VULNERABLE;
543	544	}
544	545
545	546	/* SCTLR_EL1.DSSBS was initialised to 0 during boot */
546		− asm volatile(SET_PSTATE_SSBS(0));
	547	+ set_pstate_ssbs(0);
547	548	return SPECTRE_MITIGATED;
548	549	}
549	550

arch/arm64/kernel/ptrace.c+2−5

@@ -192,14 +192,11 @@ static void ptrace_hbptriggered(struct perf_event *bp, · 191 unmodified lines
192	192	break;
193	193	}
194	194	}
195		− arm64_force_sig_ptrace_errno_trap(si_errno,
196		− (void __user *)bkpt->trigger,
	195	+ arm64_force_sig_ptrace_errno_trap(si_errno, bkpt->trigger,
197	196	desc);
198	197	}
199	198	#endif
200		− arm64_force_sig_fault(SIGTRAP, TRAP_HWBKPT,
201		− (void __user *)(bkpt->trigger),
202		− desc);
	199	+ arm64_force_sig_fault(SIGTRAP, TRAP_HWBKPT, bkpt->trigger, desc);
203	200	}
204	201
205	202	/*

arch/arm64/kernel/scs.c+0−16

@@ -1,16 +0,0 @@
1		−// SPDX-License-Identifier: GPL-2.0
2		−/*
3		− * Shadow Call Stack support.
4		− *
5		− * Copyright (C) 2019 Google LLC
6		− */
7		−
8		−#include <linux/percpu.h>
9		−#include <linux/scs.h>
10		−
11		−DEFINE_SCS(irq_shadow_call_stack);
12		−
13		−#ifdef CONFIG_ARM_SDE_INTERFACE
14		−DEFINE_SCS(sdei_shadow_call_stack_normal);
15		−DEFINE_SCS(sdei_shadow_call_stack_critical);
16		−#endif

arch/arm64/kernel/sdei.c+108−11

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#include <linux/hardirq.h>
8	8	#include <linux/irqflags.h>
9	9	#include <linux/sched/task_stack.h>
	10	+#include <linux/scs.h>
10	11	#include <linux/uaccess.h>
11	12
12	13	#include <asm/alternative.h>
@@ -38,6 +39,14 @@ DEFINE_PER_CPU(unsigned long *, sdei_stack_normal_ptr); · 25 unmodified lines
38	39	DEFINE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
39	40	#endif
40	41
	42	+DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
	43	+DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);
	44	+
	45	+#ifdef CONFIG_SHADOW_CALL_STACK
	46	+DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
	47	+DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);
	48	+#endif
	49	+
41	50	static void _free_sdei_stack(unsigned long * __percpu *ptr, int cpu)
42	51	{
43	52	unsigned long *p;
@@ -53,6 +62,9 @@ static void free_sdei_stacks(void) · 9 unmodified lines
53	62	{
54	63	int cpu;
55	64
	65	+ if (!IS_ENABLED(CONFIG_VMAP_STACK))
	66	+ return;
	67	+
56	68	for_each_possible_cpu(cpu) {
57	69	_free_sdei_stack(&sdei_stack_normal_ptr, cpu);
58	70	_free_sdei_stack(&sdei_stack_critical_ptr, cpu);
@@ -76,6 +88,9 @@ static int init_sdei_stacks(void) · 17 unmodified lines
76	88	int cpu;
77	89	int err = 0;
78	90
	91	+ if (!IS_ENABLED(CONFIG_VMAP_STACK))
	92	+ return 0;
	93	+
79	94	for_each_possible_cpu(cpu) {
80	95	err = _init_sdei_stack(&sdei_stack_normal_ptr, cpu);
81	96	if (err)
@@ -91,6 +106,62 @@ static int init_sdei_stacks(void) · 9 unmodified lines
91	106	return err;
92	107	}
93	108
	109	+static void _free_sdei_scs(unsigned long * __percpu *ptr, int cpu)
	110	+{
	111	+ void *s;
	112	+
	113	+ s = per_cpu(*ptr, cpu);
	114	+ if (s) {
	115	+ per_cpu(*ptr, cpu) = NULL;
	116	+ scs_free(s);
	117	+ }
	118	+}
	119	+
	120	+static void free_sdei_scs(void)
	121	+{
	122	+ int cpu;
	123	+
	124	+ for_each_possible_cpu(cpu) {
	125	+ _free_sdei_scs(&sdei_shadow_call_stack_normal_ptr, cpu);
	126	+ _free_sdei_scs(&sdei_shadow_call_stack_critical_ptr, cpu);
	127	+ }
	128	+}
	129	+
	130	+static int _init_sdei_scs(unsigned long * __percpu *ptr, int cpu)
	131	+{
	132	+ void *s;
	133	+
	134	+ s = scs_alloc(cpu_to_node(cpu));
	135	+ if (!s)
	136	+ return -ENOMEM;
	137	+ per_cpu(*ptr, cpu) = s;
	138	+
	139	+ return 0;
	140	+}
	141	+
	142	+static int init_sdei_scs(void)
	143	+{
	144	+ int cpu;
	145	+ int err = 0;
	146	+
	147	+ if (!IS_ENABLED(CONFIG_SHADOW_CALL_STACK))
	148	+ return 0;
	149	+
	150	+ for_each_possible_cpu(cpu) {
	151	+ err = _init_sdei_scs(&sdei_shadow_call_stack_normal_ptr, cpu);
	152	+ if (err)
	153	+ break;
	154	+ err = _init_sdei_scs(&sdei_shadow_call_stack_critical_ptr, cpu);
	155	+ if (err)
	156	+ break;
	157	+ }
	158	+
	159	+ if (err)
	160	+ free_sdei_scs();
	161	+
	162	+ return err;
	163	+}
	164	+
94	165	static bool on_sdei_normal_stack(unsigned long sp, struct stack_info *info)
95	166	{
96	167	unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr);
@@ -131,13 +202,14 @@ unsigned long sdei_arch_get_entry_point(int conduit) · 34 unmodified lines
131	202	*/
132	203	if (is_hyp_mode_available() && !is_kernel_in_hyp_mode()) {
133	204	pr_err("Not supported on this hardware/boot configuration\n");
134		− return 0;
	205	+ goto out_err;
135	206	}
136	207
137		− if (IS_ENABLED(CONFIG_VMAP_STACK)) {
138		− if (init_sdei_stacks())
139		− return 0;
140		− }
	208	+ if (init_sdei_stacks())
	209	+ goto out_err;
	210	+
	211	+ if (init_sdei_scs())
	212	+ goto out_err_free_stacks;
141	213
142	214	sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
143	215
@@ -152,6 +224,10 @@ unsigned long sdei_arch_get_entry_point(int conduit) · 8 unmodified lines
152	224	#endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
153	225	return (unsigned long)__sdei_asm_handler;
154	226
	227	+out_err_free_stacks:
	228	+ free_sdei_stacks();
	229	+out_err:
	230	+ return 0;
155	231	}
156	232
157	233	/*
@@ -179,12 +255,6 @@ static __kprobes unsigned long _sdei_handler(struct pt_regs *regs, · 21 unmodified lines
179	255	sdei_api_event_context(i, &regs->regs[i]);
180	256	}
181	257
182		− /*
183		− * We didn't take an exception to get here, set PAN. UAO will be cleared
184		− * by sdei_event_handler()s force_uaccess_begin() call.
185		− */
186		− __uaccess_enable_hw_pan();
187		−
188	258	err = sdei_event_handler(regs, arg);
189	259	if (err)
190	260	return SDEI_EV_FAILED;
@@ -223,12 +293,39 @@ static __kprobes unsigned long _sdei_handler(struct pt_regs *regs, · 32 unmodified lines
223	293	return vbar + 0x480;
224	294	}
225	295
	296	+static void __kprobes notrace __sdei_pstate_entry(void)
	297	+{
	298	+ /*
	299	+ * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to
	300	+ * whether PSTATE bits are inherited unchanged or generated from
	301	+ * scratch, and the TF-A implementation always clears PAN and always
	302	+ * clears UAO. There are no other known implementations.
	303	+ *
	304	+ * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how
	305	+ * PSTATE is modified upon architectural exceptions, and so PAN is
	306	+ * either inherited or set per SCTLR_ELx.SPAN, and UAO is always
	307	+ * cleared.
	308	+ *
	309	+ * We must explicitly reset PAN to the expected state, including
	310	+ * clearing it when the host isn't using it, in case a VM had it set.
	311	+ */
	312	+ if (system_uses_hw_pan())
	313	+ set_pstate_pan(1);
	314	+ else if (cpu_has_pan())
	315	+ set_pstate_pan(0);
	316	+}
226	317
227	318	asmlinkage noinstr unsigned long
228	319	__sdei_handler(struct pt_regs regs, struct sdei_registered_event arg)
229	320	{
230	321	unsigned long ret;
231	322
	323	+ /*
	324	+ * We didn't take an exception to get here, so the HW hasn't
	325	+ * set/cleared bits in PSTATE that we may rely on. Initialize PAN.
	326	+ */
	327	+ __sdei_pstate_entry();
	328	+
232	329	arm64_enter_nmi(regs);
233	330
234	331	ret = _sdei_handler(regs, arg);

arch/arm64/kernel/setup.c+3−3

@@ -206,7 +206,7 @@ static void __init request_standard_resources(void) · 205 unmodified lines
206	206	unsigned long i = 0;
207	207	size_t res_size;
208	208
209		− kernel_code.start = __pa_symbol(_text);
	209	+ kernel_code.start = __pa_symbol(_stext);
210	210	kernel_code.end = __pa_symbol(__init_begin - 1);
211	211	kernel_data.start = __pa_symbol(_sdata);
212	212	kernel_data.end = __pa_symbol(_end - 1);
@@ -283,7 +283,7 @@ u64 cpu_logical_map(int cpu) · 70 unmodified lines
283	283
284	284	void __init __no_sanitize_address setup_arch(char **cmdline_p)
285	285	{
286		− init_mm.start_code = (unsigned long) _text;
	286	+ init_mm.start_code = (unsigned long) _stext;
287	287	init_mm.end_code = (unsigned long) _etext;
288	288	init_mm.end_data = (unsigned long) _edata;
289	289	init_mm.brk = (unsigned long) _end;
@@ -366,7 +366,7 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) · 76 unmodified lines
366	366	* faults in case uaccess_enable() is inadvertently called by the init
367	367	* thread.
368	368	*/
369		− init_task.thread_info.ttbr0 = __pa_symbol(empty_zero_page);
	369	+ init_task.thread_info.ttbr0 = __pa_symbol(reserved_pg_dir);
370	370	#endif
371	371
372	372	if (boot_args[1] \|\| boot_args[2] \|\| boot_args[3]) {

arch/arm64/kernel/signal.c+0−3

@@ -922,9 +922,6 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, · 921 unmodified lines
922	922	trace_hardirqs_off();
923	923
924	924	do {
925		− /* Check valid user FS if needed */
926		− addr_limit_user_check();
927		−
928	925	if (thread_flags & _TIF_NEED_RESCHED) {
929	926	/* Unmask Debug and SError for the next task */
930	927	local_daif_restore(DAIF_PROCCTX_NOIRQ);

arch/arm64/kernel/sleep.S+1−1

@@ -99,7 +99,7 @@ SYM_FUNC_END(__cpu_suspend_enter) · 98 unmodified lines
99	99
100	100	.pushsection ".idmap.text", "awx"
101	101	SYM_CODE_START(cpu_resume)
102		− bl el2_setup // if in EL2 drop to EL1 cleanly
	102	+ bl init_kernel_el
103	103	bl __cpu_setup
104	104	/* enable the MMU early - so we can access sleep_save_stash by va */
105	105	adrp x1, swapper_pg_dir

arch/arm64/kernel/smp.c+7−8

@@ -787,14 +787,13 @@ void __init smp_prepare_cpus(unsigned int max_cpus) · 786 unmodified lines
787	787	}
788	788
789	789	static const char *ipi_types[NR_IPI] __tracepoint_string = {
790		−#define S(x,s) [x] = s
791		− S(IPI_RESCHEDULE, "Rescheduling interrupts"),
792		− S(IPI_CALL_FUNC, "Function call interrupts"),
793		− S(IPI_CPU_STOP, "CPU stop interrupts"),
794		− S(IPI_CPU_CRASH_STOP, "CPU stop (for crash dump) interrupts"),
795		− S(IPI_TIMER, "Timer broadcast interrupts"),
796		− S(IPI_IRQ_WORK, "IRQ work interrupts"),
797		− S(IPI_WAKEUP, "CPU wake-up interrupts"),
	790	+ [IPI_RESCHEDULE] = "Rescheduling interrupts",
	791	+ [IPI_CALL_FUNC] = "Function call interrupts",
	792	+ [IPI_CPU_STOP] = "CPU stop interrupts",
	793	+ [IPI_CPU_CRASH_STOP] = "CPU stop (for crash dump) interrupts",
	794	+ [IPI_TIMER] = "Timer broadcast interrupts",
	795	+ [IPI_IRQ_WORK] = "IRQ work interrupts",
	796	+ [IPI_WAKEUP] = "CPU wake-up interrupts",
798	797	};
799	798
800	799	static void smp_cross_call(const struct cpumask *target, unsigned int ipinr);

arch/arm64/kernel/suspend.c+0−1

@@ -58,7 +58,6 @@ void notrace __cpu_suspend_exit(void) · 57 unmodified lines
58	58	* features that might not have been set correctly.
59	59	*/
60	60	__uaccess_enable_hw_pan();
61		− uao_thread_switch(current);
62	61
63	62	/*
64	63	* Restore HW breakpoint registers to sane values

arch/arm64/kernel/sys_compat.c+2−3

@@ -68,7 +68,7 @@ do_compat_cache_op(unsigned long start, unsigned long end, int flags) · 67 unmodified lines
68	68	*/
69	69	long compat_arm_syscall(struct pt_regs *regs, int scno)
70	70	{
71		− void __user *addr;
	71	+ unsigned long addr;
72	72
73	73	switch (scno) {
74	74	/*
@@ -111,8 +111,7 @@ long compat_arm_syscall(struct pt_regs *regs, int scno) · 36 unmodified lines
111	111	break;
112	112	}
113	113
114		− addr = (void __user *)instruction_pointer(regs) -
115		− (compat_thumb_mode(regs) ? 2 : 4);
	114	+ addr = instruction_pointer(regs) - (compat_thumb_mode(regs) ? 2 : 4);
116	115
117	116	arm64_notify_die("Oops - bad compat syscall(2)", regs,
118	117	SIGILL, ILL_ILLTRP, addr, scno);

arch/arm64/kernel/syscall.c+1−1

@@ -122,7 +122,7 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr, · 121 unmodified lines
122	122	cortex_a76_erratum_1463225_svc_handler();
123	123	local_daif_restore(DAIF_PROCCTX);
124	124
125		− if (system_supports_mte() && (flags & _TIF_MTE_ASYNC_FAULT)) {
	125	+ if (flags & _TIF_MTE_ASYNC_FAULT) {
126	126	/*
127	127	* Process the asynchronous tag check fault before the actual
128	128	* syscall. do_notify_resume() will send a signal to userspace

arch/arm64/kernel/topology.c+110−26

@@ -124,6 +124,12 @@ int __init parse_acpi_topology(void) · 123 unmodified lines
124	124	#endif
125	125
126	126	#ifdef CONFIG_ARM64_AMU_EXTN
	127	+#define read_corecnt() read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0)
	128	+#define read_constcnt() read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0)
	129	+#else
	130	+#define read_corecnt() (0UL)
	131	+#define read_constcnt() (0UL)
	132	+#endif
127	133
128	134	#undef pr_fmt
129	135	#define pr_fmt(fmt) "AMU: " fmt
@@ -133,54 +139,58 @@ static DEFINE_PER_CPU(u64, arch_const_cycles_prev); · 3 unmodified lines
133	139	static DEFINE_PER_CPU(u64, arch_core_cycles_prev);
134	140	static cpumask_var_t amu_fie_cpus;
135	141
136		−/* Initialize counter reference per-cpu variables for the current CPU */
137		−void init_cpu_freq_invariance_counters(void)
	142	+void update_freq_counters_refs(void)
138	143	{
139		− this_cpu_write(arch_core_cycles_prev,
140		− read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0));
141		− this_cpu_write(arch_const_cycles_prev,
142		− read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0));
	144	+ this_cpu_write(arch_core_cycles_prev, read_corecnt());
	145	+ this_cpu_write(arch_const_cycles_prev, read_constcnt());
143	146	}
144	147
145		−static int validate_cpu_freq_invariance_counters(int cpu)
	148	+static inline bool freq_counters_valid(int cpu)
146	149	{
147		− u64 max_freq_hz, ratio;
	150	+ if ((cpu >= nr_cpu_ids) \|\| !cpumask_test_cpu(cpu, cpu_present_mask))
	151	+ return false;
148	152
149	153	if (!cpu_has_amu_feat(cpu)) {
150	154	pr_debug("CPU%d: counters are not supported.\n", cpu);
151		− return -EINVAL;
	155	+ return false;
152	156	}
153	157
154	158	if (unlikely(!per_cpu(arch_const_cycles_prev, cpu) \|\|
155	159	!per_cpu(arch_core_cycles_prev, cpu))) {
156	160	pr_debug("CPU%d: cycle counters are not enabled.\n", cpu);
157		− return -EINVAL;
	161	+ return false;
158	162	}
159	163
160		− /* Convert maximum frequency from KHz to Hz and validate */
161		− max_freq_hz = cpufreq_get_hw_max_freq(cpu) * 1000;
162		− if (unlikely(!max_freq_hz)) {
163		− pr_debug("CPU%d: invalid maximum frequency.\n", cpu);
	164	+ return true;
	165	+}
	166	+
	167	+static int freq_inv_set_max_ratio(int cpu, u64 max_rate, u64 ref_rate)
	168	+{
	169	+ u64 ratio;
	170	+
	171	+ if (unlikely(!max_rate \|\| !ref_rate)) {
	172	+ pr_debug("CPU%d: invalid maximum or reference frequency.\n",
	173	+ cpu);
164	174	return -EINVAL;
165	175	}
166	176
167	177	/*
168	178	* Pre-compute the fixed ratio between the frequency of the constant
169		− * counter and the maximum frequency of the CPU.
	179	+ * reference counter and the maximum frequency of the CPU.
170	180	*
171		− * const_freq
172		− * arch_max_freq_scale = ---------------- * SCHED_CAPACITY_SCALE²
173		− * cpuinfo_max_freq
	181	+ * ref_rate
	182	+ * arch_max_freq_scale = ---------- * SCHED_CAPACITY_SCALE²
	183	+ * max_rate
174	184	*
175	185	* We use a factor of 2 * SCHED_CAPACITY_SHIFT -> SCHED_CAPACITY_SCALE²
176	186	* in order to ensure a good resolution for arch_max_freq_scale for
177		− * very low arch timer frequencies (down to the KHz range which should
	187	+ * very low reference frequencies (down to the KHz range which should
178	188	* be unlikely).
179	189	*/
180		− ratio = (u64)arch_timer_get_rate() << (2 * SCHED_CAPACITY_SHIFT);
181		− ratio = div64_u64(ratio, max_freq_hz);
	190	+ ratio = ref_rate << (2 * SCHED_CAPACITY_SHIFT);
	191	+ ratio = div64_u64(ratio, max_rate);
182	192	if (!ratio) {
183		− WARN_ONCE(1, "System timer frequency too low.\n");
	193	+ WARN_ONCE(1, "Reference frequency too low.\n");
184	194	return -EINVAL;
185	195	}
186	196
@@ -227,8 +237,12 @@ static int __init init_amu_fie(void) · 40 unmodified lines
227	237	}
228	238
229	239	for_each_present_cpu(cpu) {
230		− if (validate_cpu_freq_invariance_counters(cpu))
	240	+ if (!freq_counters_valid(cpu) \|\|
	241	+ freq_inv_set_max_ratio(cpu,
	242	+ cpufreq_get_hw_max_freq(cpu) * 1000,
	243	+ arch_timer_get_rate()))
231	244	continue;
	245	+
232	246	cpumask_set_cpu(cpu, valid_cpus);
233	247	have_policy \|= enable_policy_freq_counters(cpu, valid_cpus);
234	248	}
@@ -280,11 +294,14 @@ void topology_scale_freq_tick(void) · 45 unmodified lines
280	294	if (!cpumask_test_cpu(cpu, amu_fie_cpus))
281	295	return;
282	296
283		− const_cnt = read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0);
284		− core_cnt = read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0);
285	297	prev_const_cnt = this_cpu_read(arch_const_cycles_prev);
286	298	prev_core_cnt = this_cpu_read(arch_core_cycles_prev);
287	299
	300	+ update_freq_counters_refs();
	301	+
	302	+ const_cnt = this_cpu_read(arch_const_cycles_prev);
	303	+ core_cnt = this_cpu_read(arch_core_cycles_prev);
	304	+
288	305	if (unlikely(core_cnt <= prev_core_cnt \|\|
289	306	const_cnt <= prev_const_cnt))
290	307	goto store_and_exit;
@@ -309,4 +326,71 @@ store_and_exit: · 18 unmodified lines
309	326	this_cpu_write(arch_core_cycles_prev, core_cnt);
310	327	this_cpu_write(arch_const_cycles_prev, const_cnt);
311	328	}
312		−#endif /* CONFIG_ARM64_AMU_EXTN */
	329	+
	330	+#ifdef CONFIG_ACPI_CPPC_LIB
	331	+#include <acpi/cppc_acpi.h>
	332	+
	333	+static void cpu_read_corecnt(void *val)
	334	+{
	335	+ (u64 )val = read_corecnt();
	336	+}
	337	+
	338	+static void cpu_read_constcnt(void *val)
	339	+{
	340	+ (u64 )val = read_constcnt();
	341	+}
	342	+
	343	+static inline
	344	+int counters_read_on_cpu(int cpu, smp_call_func_t func, u64 *val)
	345	+{
	346	+ /*
	347	+ * Abort call on counterless CPU or when interrupts are
	348	+ * disabled - can lead to deadlock in smp sync call.
	349	+ */
	350	+ if (!cpu_has_amu_feat(cpu))
	351	+ return -EOPNOTSUPP;
	352	+
	353	+ if (WARN_ON_ONCE(irqs_disabled()))
	354	+ return -EPERM;
	355	+
	356	+ smp_call_function_single(cpu, func, val, 1);
	357	+
	358	+ return 0;
	359	+}
	360	+
	361	+/*
	362	+ * Refer to drivers/acpi/cppc_acpi.c for the description of the functions
	363	+ * below.
	364	+ */
	365	+bool cpc_ffh_supported(void)
	366	+{
	367	+ return freq_counters_valid(get_cpu_with_amu_feat());
	368	+}
	369	+
	370	+int cpc_read_ffh(int cpu, struct cpc_reg reg, u64 val)
	371	+{
	372	+ int ret = -EOPNOTSUPP;
	373	+
	374	+ switch ((u64)reg->address) {
	375	+ case 0x0:
	376	+ ret = counters_read_on_cpu(cpu, cpu_read_corecnt, val);
	377	+ break;
	378	+ case 0x1:
	379	+ ret = counters_read_on_cpu(cpu, cpu_read_constcnt, val);
	380	+ break;
	381	+ }
	382	+
	383	+ if (!ret) {
	384	+ *val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1,
	385	+ reg->bit_offset);
	386	+ *val >>= reg->bit_offset;
	387	+ }
	388	+
	389	+ return ret;
	390	+}
	391	+
	392	+int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val)
	393	+{
	394	+ return -EOPNOTSUPP;
	395	+}
	396	+#endif /* CONFIG_ACPI_CPPC_LIB */

arch/arm64/kernel/traps.c+15−14

@@ -171,32 +171,32 @@ static void arm64_show_signal(int signo, const char *str) · 170 unmodified lines
171	171	__show_regs(regs);
172	172	}
173	173
174		−void arm64_force_sig_fault(int signo, int code, void __user *addr,
	174	+void arm64_force_sig_fault(int signo, int code, unsigned long far,
175	175	const char *str)
176	176	{
177	177	arm64_show_signal(signo, str);
178	178	if (signo == SIGKILL)
179	179	force_sig(SIGKILL);
180	180	else
181		− force_sig_fault(signo, code, addr);
	181	+ force_sig_fault(signo, code, (void __user *)far);
182	182	}
183	183
184		−void arm64_force_sig_mceerr(int code, void __user *addr, short lsb,
	184	+void arm64_force_sig_mceerr(int code, unsigned long far, short lsb,
185	185	const char *str)
186	186	{
187	187	arm64_show_signal(SIGBUS, str);
188		− force_sig_mceerr(code, addr, lsb);
	188	+ force_sig_mceerr(code, (void __user *)far, lsb);
189	189	}
190	190
191		−void arm64_force_sig_ptrace_errno_trap(int errno, void __user *addr,
	191	+void arm64_force_sig_ptrace_errno_trap(int errno, unsigned long far,
192	192	const char *str)
193	193	{
194	194	arm64_show_signal(SIGTRAP, str);
195		− force_sig_ptrace_errno_trap(errno, addr);
	195	+ force_sig_ptrace_errno_trap(errno, (void __user *)far);
196	196	}
197	197
198	198	void arm64_notify_die(const char str, struct pt_regs regs,
199		− int signo, int sicode, void __user *addr,
	199	+ int signo, int sicode, unsigned long far,
200	200	int err)
201	201	{
202	202	if (user_mode(regs)) {
@@ -204,7 +204,7 @@ void arm64_notify_die(const char str, struct pt_regs regs, · 1 unmodified line
204	204	current->thread.fault_address = 0;
205	205	current->thread.fault_code = err;
206	206
207		− arm64_force_sig_fault(signo, sicode, addr, str);
	207	+ arm64_force_sig_fault(signo, sicode, far, str);
208	208	} else {
209	209	die(str, regs, err);
210	210	}
@@ -375,7 +375,7 @@ void force_signal_inject(int signal, int code, unsigned long address, unsigned i · 164 unmodified lines
375	375	signal = SIGKILL;
376	376	}
377	377
378		− arm64_notify_die(desc, regs, signal, code, (void __user *)address, err);
	378	+ arm64_notify_die(desc, regs, signal, code, address, err);
379	379	}
380	380
381	381	/*
@@ -386,7 +386,7 @@ void arm64_notify_segfault(unsigned long addr) · 4 unmodified lines
386	386	int code;
387	387
388	388	mmap_read_lock(current->mm);
389		− if (find_vma(current->mm, addr) == NULL)
	389	+ if (find_vma(current->mm, untagged_addr(addr)) == NULL)
390	390	code = SEGV_MAPERR;
391	391	else
392	392	code = SEGV_ACCERR;
@@ -449,12 +449,13 @@ NOKPROBE_SYMBOL(do_ptrauth_fault); · 56 unmodified lines
449	449
450	450	static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs)
451	451	{
452		− unsigned long address;
	452	+ unsigned long tagged_address, address;
453	453	int rt = ESR_ELx_SYS64_ISS_RT(esr);
454	454	int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
455	455	int ret = 0;
456	456
457		− address = untagged_addr(pt_regs_read_reg(regs, rt));
	457	+ tagged_address = pt_regs_read_reg(regs, rt);
	458	+ address = untagged_addr(tagged_address);
458	459
459	460	switch (crm) {
460	461	case ESR_ELx_SYS64_ISS_CRM_DC_CVAU: /* DC CVAU, gets promoted */
@@ -481,7 +482,7 @@ static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs) · 20 unmodified lines
481	482	}
482	483
483	484	if (ret)
484		− arm64_notify_segfault(address);
	485	+ arm64_notify_segfault(tagged_address);
485	486	else
486	487	arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
487	488	}
@@ -775,7 +776,7 @@ asmlinkage void notrace bad_mode(struct pt_regs *regs, int reason, unsigned int · 287 unmodified lines
775	776	*/
776	777	void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
777	778	{
778		− void __user pc = (void __user )instruction_pointer(regs);
	779	+ unsigned long pc = instruction_pointer(regs);
779	780
780	781	current->thread.fault_address = 0;
781	782	current->thread.fault_code = esr;

arch/arm64/kernel/vdso/Makefile+1−1

@@ -28,7 +28,7 @@ ldflags-y := -shared -nostdlib -soname=linux-vdso.so.1 --hash-style=sysv \ · 27 unmodified lines
28	28	$(btildflags-y) -T
29	29
30	30	ccflags-y := -fno-common -fno-builtin -fno-stack-protector -ffixed-x18
31		−ccflags-y += -DDISABLE_BRANCH_PROFILING
	31	+ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO
32	32
33	33	CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) $(GCC_PLUGINS_CFLAGS)
34	34	KASAN_SANITIZE := n

arch/arm64/kernel/vdso32/Makefile+1−1

@@ -48,7 +48,7 @@ cc32-as-instr = $(call try-run,\ · 47 unmodified lines
48	48	# As a result we set our own flags here.
49	49
50	50	# KBUILD_CPPFLAGS and NOSTDINC_FLAGS from top-level Makefile
51		−VDSO_CPPFLAGS := -D__KERNEL__ -nostdinc -isystem $(shell $(CC_COMPAT) -print-file-name=include)
	51	+VDSO_CPPFLAGS := -DBUILD_VDSO -D__KERNEL__ -nostdinc -isystem $(shell $(CC_COMPAT) -print-file-name=include)
52	52	VDSO_CPPFLAGS += $(LINUXINCLUDE)
53	53
54	54	# Common C and assembly flags

arch/arm64/kernel/vmlinux.lds.S+5−7

@@ -121,7 +121,7 @@ SECTIONS · 120 unmodified lines
121	121	_text = .;
122	122	HEAD_TEXT
123	123	}
124		− .text : { /* Real text segment */
	124	+ .text : ALIGN(SEGMENT_ALIGN) { /* Real text segment */
125	125	_stext = .; /* Text and read-only data */
126	126	IRQENTRY_TEXT
127	127	SOFTIRQENTRY_TEXT
@@ -164,13 +164,11 @@ SECTIONS · 36 unmodified lines
164	164	. += PAGE_SIZE;
165	165	#endif
166	166
167		−#ifdef CONFIG_ARM64_SW_TTBR0_PAN
168		− reserved_ttbr0 = .;
169		− . += RESERVED_TTBR0_SIZE;
170		−#endif
	167	+ reserved_pg_dir = .;
	168	+ . += PAGE_SIZE;
	169	+
171	170	swapper_pg_dir = .;
172	171	. += PAGE_SIZE;
173		− swapper_pg_end = .;
174	172
175	173	. = ALIGN(SEGMENT_ALIGN);
176	174	__init_begin = .;
@@ -201,7 +199,7 @@ SECTIONS · 24 unmodified lines
201	199	INIT_CALLS
202	200	CON_INITCALL
203	201	INIT_RAM_FS
204		− (.init.rodata. .init.bss) /* from the EFI stub */
	202	+ (.init.altinstructions .init.bss) / from the EFI stub */
205	203	}
206	204	.exit.data : {
207	205	EXIT_DATA

arch/arm64/lib/clear_user.S+4−4

@@ -24,20 +24,20 @@ SYM_FUNC_START(__arch_clear_user) · 23 unmodified lines
24	24	subs x1, x1, #8
25	25	b.mi 2f
26	26	1:
27		−uao_user_alternative 9f, str, sttr, xzr, x0, 8
	27	+user_ldst 9f, sttr, xzr, x0, 8
28	28	subs x1, x1, #8
29	29	b.pl 1b
30	30	2: adds x1, x1, #4
31	31	b.mi 3f
32		−uao_user_alternative 9f, str, sttr, wzr, x0, 4
	32	+user_ldst 9f, sttr, wzr, x0, 4
33	33	sub x1, x1, #4
34	34	3: adds x1, x1, #2
35	35	b.mi 4f
36		−uao_user_alternative 9f, strh, sttrh, wzr, x0, 2
	36	+user_ldst 9f, sttrh, wzr, x0, 2
37	37	sub x1, x1, #2
38	38	4: adds x1, x1, #1
39	39	b.mi 5f
40		−uao_user_alternative 9f, strb, sttrb, wzr, x0, 0
	40	+user_ldst 9f, sttrb, wzr, x0, 0
41	41	5: mov x0, #0
42	42	ret
43	43	SYM_FUNC_END(__arch_clear_user)

arch/arm64/lib/copy_from_user.S+4−4

@@ -21,7 +21,7 @@ · 20 unmodified lines
21	21	*/
22	22
23	23	.macro ldrb1 reg, ptr, val
24		− uao_user_alternative 9998f, ldrb, ldtrb, \reg, \ptr, \val
	24	+ user_ldst 9998f, ldtrb, \reg, \ptr, \val
25	25	.endm
26	26
27	27	.macro strb1 reg, ptr, val
@@ -29,7 +29,7 @@ · 1 unmodified line
29	29	.endm
30	30
31	31	.macro ldrh1 reg, ptr, val
32		− uao_user_alternative 9998f, ldrh, ldtrh, \reg, \ptr, \val
	32	+ user_ldst 9998f, ldtrh, \reg, \ptr, \val
33	33	.endm
34	34
35	35	.macro strh1 reg, ptr, val
@@ -37,7 +37,7 @@ · 1 unmodified line
37	37	.endm
38	38
39	39	.macro ldr1 reg, ptr, val
40		− uao_user_alternative 9998f, ldr, ldtr, \reg, \ptr, \val
	40	+ user_ldst 9998f, ldtr, \reg, \ptr, \val
41	41	.endm
42	42
43	43	.macro str1 reg, ptr, val
@@ -45,7 +45,7 @@ · 1 unmodified line
45	45	.endm
46	46
47	47	.macro ldp1 reg1, reg2, ptr, val
48		− uao_ldp 9998f, \reg1, \reg2, \ptr, \val
	48	+ user_ldp 9998f, \reg1, \reg2, \ptr, \val
49	49	.endm
50	50
51	51	.macro stp1 reg1, reg2, ptr, val

arch/arm64/lib/copy_in_user.S+8−8

@@ -22,35 +22,35 @@ · 21 unmodified lines
22	22	* x0 - bytes not copied
23	23	*/
24	24	.macro ldrb1 reg, ptr, val
25		− uao_user_alternative 9998f, ldrb, ldtrb, \reg, \ptr, \val
	25	+ user_ldst 9998f, ldtrb, \reg, \ptr, \val
26	26	.endm
27	27
28	28	.macro strb1 reg, ptr, val
29		− uao_user_alternative 9998f, strb, sttrb, \reg, \ptr, \val
	29	+ user_ldst 9998f, sttrb, \reg, \ptr, \val
30	30	.endm
31	31
32	32	.macro ldrh1 reg, ptr, val
33		− uao_user_alternative 9998f, ldrh, ldtrh, \reg, \ptr, \val
	33	+ user_ldst 9998f, ldtrh, \reg, \ptr, \val
34	34	.endm
35	35
36	36	.macro strh1 reg, ptr, val
37		− uao_user_alternative 9998f, strh, sttrh, \reg, \ptr, \val
	37	+ user_ldst 9998f, sttrh, \reg, \ptr, \val
38	38	.endm
39	39
40	40	.macro ldr1 reg, ptr, val
41		− uao_user_alternative 9998f, ldr, ldtr, \reg, \ptr, \val
	41	+ user_ldst 9998f, ldtr, \reg, \ptr, \val
42	42	.endm
43	43
44	44	.macro str1 reg, ptr, val
45		− uao_user_alternative 9998f, str, sttr, \reg, \ptr, \val
	45	+ user_ldst 9998f, sttr, \reg, \ptr, \val
46	46	.endm
47	47
48	48	.macro ldp1 reg1, reg2, ptr, val
49		− uao_ldp 9998f, \reg1, \reg2, \ptr, \val
	49	+ user_ldp 9998f, \reg1, \reg2, \ptr, \val
50	50	.endm
51	51
52	52	.macro stp1 reg1, reg2, ptr, val
53		− uao_stp 9998f, \reg1, \reg2, \ptr, \val
	53	+ user_stp 9998f, \reg1, \reg2, \ptr, \val
54	54	.endm
55	55
56	56	end .req x5

arch/arm64/lib/copy_to_user.S+4−4

@@ -24,7 +24,7 @@ · 23 unmodified lines
24	24	.endm
25	25
26	26	.macro strb1 reg, ptr, val
27		− uao_user_alternative 9998f, strb, sttrb, \reg, \ptr, \val
	27	+ user_ldst 9998f, sttrb, \reg, \ptr, \val
28	28	.endm
29	29
30	30	.macro ldrh1 reg, ptr, val
@@ -32,7 +32,7 @@ · 1 unmodified line
32	32	.endm
33	33
34	34	.macro strh1 reg, ptr, val
35		− uao_user_alternative 9998f, strh, sttrh, \reg, \ptr, \val
	35	+ user_ldst 9998f, sttrh, \reg, \ptr, \val
36	36	.endm
37	37
38	38	.macro ldr1 reg, ptr, val
@@ -40,7 +40,7 @@ · 1 unmodified line
40	40	.endm
41	41
42	42	.macro str1 reg, ptr, val
43		− uao_user_alternative 9998f, str, sttr, \reg, \ptr, \val
	43	+ user_ldst 9998f, sttr, \reg, \ptr, \val
44	44	.endm
45	45
46	46	.macro ldp1 reg1, reg2, ptr, val
@@ -48,7 +48,7 @@ · 1 unmodified line
48	48	.endm
49	49
50	50	.macro stp1 reg1, reg2, ptr, val
51		− uao_stp 9998f, \reg1, \reg2, \ptr, \val
	51	+ user_stp 9998f, \reg1, \reg2, \ptr, \val
52	52	.endm
53	53
54	54	end .req x5

arch/arm64/lib/mte.S+3−3

@@ -4,7 +4,7 @@ · 3 unmodified lines
4	4	*/
5	5	#include <linux/linkage.h>
6	6
7		−#include <asm/alternative.h>
	7	+#include <asm/asm-uaccess.h>
8	8	#include <asm/assembler.h>
9	9	#include <asm/mte.h>
10	10	#include <asm/page.h>
@@ -67,7 +67,7 @@ SYM_FUNC_START(mte_copy_tags_from_user) · 56 unmodified lines
67	67	mov x3, x1
68	68	cbz x2, 2f
69	69	1:
70		− uao_user_alternative 2f, ldrb, ldtrb, w4, x1, 0
	70	+ user_ldst 2f, ldtrb, w4, x1, 0
71	71	lsl x4, x4, #MTE_TAG_SHIFT
72	72	stg x4, [x0], #MTE_GRANULE_SIZE
73	73	add x1, x1, #1
@@ -94,7 +94,7 @@ SYM_FUNC_START(mte_copy_tags_to_user) · 20 unmodified lines
94	94	1:
95	95	ldg x4, [x1]
96	96	ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
97		− uao_user_alternative 2f, strb, sttrb, w4, x0, 0
	97	+ user_ldst 2f, sttrb, w4, x0, 0
98	98	add x0, x0, #1
99	99	add x1, x1, #MTE_GRANULE_SIZE
100	100	subs x2, x2, #1

arch/arm64/lib/uaccess_flushcache.c+1−3

@@ -30,9 +30,7 @@ unsigned long __copy_user_flushcache(void to, const void __user from, · 29 unmodified lines
30	30	{
31	31	unsigned long rc;
32	32
33		− uaccess_enable_not_uao();
34		− rc = __arch_copy_from_user(to, from, n);
35		− uaccess_disable_not_uao();
	33	+ rc = raw_copy_from_user(to, from, n);
36	34
37	35	/* See above */
38	36	__clean_dcache_area_pop(to, n - rc);

arch/arm64/mm/fault.c+50−38

@@ -40,7 +40,7 @@ · 39 unmodified lines
40	40	#include <asm/traps.h>
41	41
42	42	struct fault_info {
43		− int (*fn)(unsigned long addr, unsigned int esr,
	43	+ int (*fn)(unsigned long far, unsigned int esr,
44	44	struct pt_regs *regs);
45	45	int sig;
46	46	int code;
@@ -385,8 +385,11 @@ static void set_thread_esr(unsigned long address, unsigned int esr) · 338 unmodified lines
385	385	current->thread.fault_code = esr;
386	386	}
387	387
388		−static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs)
	388	+static void do_bad_area(unsigned long far, unsigned int esr,
	389	+ struct pt_regs *regs)
389	390	{
	391	+ unsigned long addr = untagged_addr(far);
	392	+
390	393	/*
391	394	* If we are in kernel mode at this point, we have no context to
392	395	* handle this fault with.
@@ -395,8 +398,7 @@ static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *re · 2 unmodified lines
395	398	const struct fault_info *inf = esr_to_fault_info(esr);
396	399
397	400	set_thread_esr(addr, esr);
398		− arm64_force_sig_fault(inf->sig, inf->code, (void __user *)addr,
399		− inf->name);
	401	+ arm64_force_sig_fault(inf->sig, inf->code, far, inf->name);
400	402	} else {
401	403	__do_kernel_fault(addr, esr, regs);
402	404	}
@@ -448,7 +450,7 @@ static bool is_write_abort(unsigned int esr) · 45 unmodified lines
448	450	return (esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM);
449	451	}
450	452
451		−static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
	453	+static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
452	454	struct pt_regs *regs)
453	455	{
454	456	const struct fault_info *inf;
@@ -456,6 +458,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, · 1 unmodified line
456	458	vm_fault_t fault;
457	459	unsigned long vm_flags = VM_ACCESS_FLAGS;
458	460	unsigned int mm_flags = FAULT_FLAG_DEFAULT;
	461	+ unsigned long addr = untagged_addr(far);
459	462
460	463	if (kprobe_page_fault(regs, esr))
461	464	return 0;
@@ -479,11 +482,6 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, · 17 unmodified lines
479	482	}
480	483
481	484	if (is_ttbr0_addr(addr) && is_el1_permission_fault(addr, esr, regs)) {
482		− /* regs->orig_addr_limit may be 0 if we entered from EL0 */
483		− if (regs->orig_addr_limit == KERNEL_DS)
484		− die_kernel_fault("access to user memory with fs=KERNEL_DS",
485		− addr, esr, regs);
486		−
487	485	if (is_el1_instruction_abort(esr))
488	486	die_kernel_fault("execution of user memory",
489	487	addr, esr, regs);
@@ -567,8 +565,7 @@ retry: · 77 unmodified lines
567	565	* We had some memory, but were unable to successfully fix up
568	566	* this page fault.
569	567	*/
570		− arm64_force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)addr,
571		− inf->name);
	568	+ arm64_force_sig_fault(SIGBUS, BUS_ADRERR, far, inf->name);
572	569	} else if (fault & (VM_FAULT_HWPOISON_LARGE \| VM_FAULT_HWPOISON)) {
573	570	unsigned int lsb;
574	571
@@ -576,8 +573,7 @@ retry: · 1 unmodified line
576	573	if (fault & VM_FAULT_HWPOISON_LARGE)
577	574	lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
578	575
579		− arm64_force_sig_mceerr(BUS_MCEERR_AR, (void __user *)addr, lsb,
580		− inf->name);
	576	+ arm64_force_sig_mceerr(BUS_MCEERR_AR, far, lsb, inf->name);
581	577	} else {
582	578	/*
583	579	* Something tried to access memory that isn't in our memory
@@ -585,8 +581,7 @@ retry: · 1 unmodified line
585	581	*/
586	582	arm64_force_sig_fault(SIGSEGV,
587	583	fault == VM_FAULT_BADACCESS ? SEGV_ACCERR : SEGV_MAPERR,
588		− (void __user *)addr,
589		− inf->name);
	584	+ far, inf->name);
590	585	}
591	586
592	587	return 0;
@@ -596,33 +591,35 @@ no_context: · 3 unmodified lines
596	591	return 0;
597	592	}
598	593
599		−static int __kprobes do_translation_fault(unsigned long addr,
	594	+static int __kprobes do_translation_fault(unsigned long far,
600	595	unsigned int esr,
601	596	struct pt_regs *regs)
602	597	{
	598	+ unsigned long addr = untagged_addr(far);
	599	+
603	600	if (is_ttbr0_addr(addr))
604		− return do_page_fault(addr, esr, regs);
	601	+ return do_page_fault(far, esr, regs);
605	602
606		− do_bad_area(addr, esr, regs);
	603	+ do_bad_area(far, esr, regs);
607	604	return 0;
608	605	}
609	606
610		−static int do_alignment_fault(unsigned long addr, unsigned int esr,
	607	+static int do_alignment_fault(unsigned long far, unsigned int esr,
611	608	struct pt_regs *regs)
612	609	{
613		− do_bad_area(addr, esr, regs);
	610	+ do_bad_area(far, esr, regs);
614	611	return 0;
615	612	}
616	613
617		−static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
	614	+static int do_bad(unsigned long far, unsigned int esr, struct pt_regs *regs)
618	615	{
619	616	return 1; /* "fault" */
620	617	}
621	618
622		−static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs)
	619	+static int do_sea(unsigned long far, unsigned int esr, struct pt_regs *regs)
623	620	{
624	621	const struct fault_info *inf;
625		− void __user *siaddr;
	622	+ unsigned long siaddr;
626	623
627	624	inf = esr_to_fault_info(esr);
628	625
@@ -634,19 +631,30 @@ static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs) · 5 unmodified lines
634	631	return 0;
635	632	}
636	633
637		− if (esr & ESR_ELx_FnV)
638		− siaddr = NULL;
639		− else
640		− siaddr = (void __user *)addr;
	634	+ if (esr & ESR_ELx_FnV) {
	635	+ siaddr = 0;
	636	+ } else {
	637	+ /*
	638	+ * The architecture specifies that the tag bits of FAR_EL1 are
	639	+ * UNKNOWN for synchronous external aborts. Mask them out now
	640	+ * so that userspace doesn't see them.
	641	+ */
	642	+ siaddr = untagged_addr(far);
	643	+ }
641	644	arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr);
642	645
643	646	return 0;
644	647	}
645	648
646		−static int do_tag_check_fault(unsigned long addr, unsigned int esr,
	649	+static int do_tag_check_fault(unsigned long far, unsigned int esr,
647	650	struct pt_regs *regs)
648	651	{
649		− do_bad_area(addr, esr, regs);
	652	+ /*
	653	+ * The architecture specifies that bits 63:60 of FAR_EL1 are UNKNOWN for tag
	654	+ * check faults. Mask them out now so that userspace doesn't see them.
	655	+ */
	656	+ far &= (1UL << 60) - 1;
	657	+ do_bad_area(far, esr, regs);
650	658	return 0;
651	659	}
652	660
@@ -717,11 +725,12 @@ static const struct fault_info fault_info[] = { · 64 unmodified lines
717	725	{ do_bad, SIGKILL, SI_KERNEL, "unknown 63" },
718	726	};
719	727
720		−void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
	728	+void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs)
721	729	{
722	730	const struct fault_info *inf = esr_to_fault_info(esr);
	731	+ unsigned long addr = untagged_addr(far);
723	732
724		− if (!inf->fn(addr, esr, regs))
	733	+ if (!inf->fn(far, esr, regs))
725	734	return;
726	735
727	736	if (!user_mode(regs)) {
@@ -730,8 +739,12 @@ void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs) · 2 unmodified lines
730	739	show_pte(addr);
731	740	}
732	741
733		− arm64_notify_die(inf->name, regs,
734		− inf->sig, inf->code, (void __user *)addr, esr);
	742	+ /*
	743	+ * At this point we have an unrecognized fault type whose tag bits may
	744	+ * have been defined as UNKNOWN. Therefore we only expose the untagged
	745	+ * address to the signal handler.
	746	+ */
	747	+ arm64_notify_die(inf->name, regs, inf->sig, inf->code, addr, esr);
735	748	}
736	749	NOKPROBE_SYMBOL(do_mem_abort);
737	750
@@ -744,8 +757,8 @@ NOKPROBE_SYMBOL(do_el0_irq_bp_hardening); · 6 unmodified lines
744	757
745	758	void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
746	759	{
747		− arm64_notify_die("SP/PC alignment exception", regs,
748		− SIGBUS, BUS_ADRALN, (void __user *)addr, esr);
	760	+ arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN,
	761	+ addr, esr);
749	762	}
750	763	NOKPROBE_SYMBOL(do_sp_pc_abort);
751	764
@@ -846,8 +859,7 @@ void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr, · 94 unmodified lines
846	859	arm64_apply_bp_hardening();
847	860
848	861	if (inf->fn(addr_if_watchpoint, esr, regs)) {
849		− arm64_notify_die(inf->name, regs,
850		− inf->sig, inf->code, (void __user *)pc, esr);
	862	+ arm64_notify_die(inf->name, regs, inf->sig, inf->code, pc, esr);
851	863	}
852	864
853	865	debug_exception_exit(regs);

arch/arm64/mm/init.c+35−21

@@ -29,6 +29,7 @@ · 28 unmodified lines
29	29	#include <linux/kexec.h>
30	30	#include <linux/crash_dump.h>
31	31	#include <linux/hugetlb.h>
	32	+#include <linux/acpi_iort.h>
32	33
33	34	#include <asm/boot.h>
34	35	#include <asm/fixmap.h>
@@ -42,8 +43,6 @@ · 7 unmodified lines
42	43	#include <asm/tlb.h>
43	44	#include <asm/alternative.h>
44	45
45		−#define ARM64_ZONE_DMA_BITS 30
46		−
47	46	/*
48	47	* We need to be able to catch inadvertent references to memstart_addr
49	48	* that occur (potentially in generic code) before arm64_memblock_init()
@@ -175,21 +174,34 @@ static void __init reserve_elfcorehdr(void) · 125 unmodified lines
175	174	#endif /* CONFIG_CRASH_DUMP */
176	175
177	176	/*
178		− * Return the maximum physical address for a zone with a given address size
179		− * limit. It currently assumes that for memory starting above 4G, 32-bit
180		− * devices will use a DMA offset.
	177	+ * Return the maximum physical address for a zone accessible by the given bits
	178	+ * limit. If DRAM starts above 32-bit, expand the zone to the maximum
	179	+ * available memory, otherwise cap it at 32-bit.
181	180	*/
182	181	static phys_addr_t __init max_zone_phys(unsigned int zone_bits)
183	182	{
184		− phys_addr_t offset = memblock_start_of_DRAM() & GENMASK_ULL(63, zone_bits);
185		− return min(offset + (1ULL << zone_bits), memblock_end_of_DRAM());
	183	+ phys_addr_t zone_mask = DMA_BIT_MASK(zone_bits);
	184	+ phys_addr_t phys_start = memblock_start_of_DRAM();
	185	+
	186	+ if (phys_start > U32_MAX)
	187	+ zone_mask = PHYS_ADDR_MAX;
	188	+ else if (phys_start > zone_mask)
	189	+ zone_mask = U32_MAX;
	190	+
	191	+ return min(zone_mask, memblock_end_of_DRAM() - 1) + 1;
186	192	}
187	193
188	194	static void __init zone_sizes_init(unsigned long min, unsigned long max)
189	195	{
190	196	unsigned long max_zone_pfns[MAX_NR_ZONES] = {0};
	197	+ unsigned int __maybe_unused acpi_zone_dma_bits;
	198	+ unsigned int __maybe_unused dt_zone_dma_bits;
191	199
192	200	#ifdef CONFIG_ZONE_DMA
	201	+ acpi_zone_dma_bits = fls64(acpi_iort_dma_get_max_cpu_address());
	202	+ dt_zone_dma_bits = fls64(of_dma_get_max_cpu_address(NULL));
	203	+ zone_dma_bits = min3(32U, dt_zone_dma_bits, acpi_zone_dma_bits);
	204	+ arm64_dma_phys_limit = max_zone_phys(zone_dma_bits);
193	205	max_zone_pfns[ZONE_DMA] = PFN_DOWN(arm64_dma_phys_limit);
194	206	#endif
195	207	#ifdef CONFIG_ZONE_DMA32
@@ -269,7 +281,7 @@ static void __init fdt_enforce_memory_region(void) · 73 unmodified lines
269	281
270	282	void __init arm64_memblock_init(void)
271	283	{
272		− const s64 linear_region_size = BIT(vabits_actual - 1);
	284	+ const s64 linear_region_size = PAGE_END - _PAGE_OFFSET(vabits_actual);
273	285
274	286	/* Handle linux,usable-memory-range property */
275	287	fdt_enforce_memory_region();
@@ -348,15 +360,18 @@ void __init arm64_memblock_init(void) · 72 unmodified lines
348	360
349	361	if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
350	362	extern u16 memstart_offset_seed;
351		− u64 range = linear_region_size -
352		− (memblock_end_of_DRAM() - memblock_start_of_DRAM());
	363	+ u64 mmfr0 = read_cpuid(ID_AA64MMFR0_EL1);
	364	+ int parange = cpuid_feature_extract_unsigned_field(
	365	+ mmfr0, ID_AA64MMFR0_PARANGE_SHIFT);
	366	+ s64 range = linear_region_size -
	367	+ BIT(id_aa64mmfr0_parange_to_phys_shift(parange));
353	368
354	369	/*
355	370	* If the size of the linear region exceeds, by a sufficient
356		− * margin, the size of the region that the available physical
357		− * memory spans, randomize the linear region as well.
	371	+ * margin, the size of the region that the physical memory can
	372	+ * span, randomize the linear region as well.
358	373	*/
359		− if (memstart_offset_seed > 0 && range >= ARM64_MEMSTART_ALIGN) {
	374	+ if (memstart_offset_seed > 0 && range >= (s64)ARM64_MEMSTART_ALIGN) {
360	375	range /= ARM64_MEMSTART_ALIGN;
361	376	memstart_addr -= ARM64_MEMSTART_ALIGN *
362	377	((range * memstart_offset_seed) >> 16);
@@ -367,7 +382,7 @@ void __init arm64_memblock_init(void) · 4 unmodified lines
367	382	* Register the kernel text, kernel data, initrd, and initial
368	383	* pagetables with memblock.
369	384	*/
370		− memblock_reserve(__pa_symbol(_text), _end - _text);
	385	+ memblock_reserve(__pa_symbol(_stext), _end - _stext);
371	386	if (IS_ENABLED(CONFIG_BLK_DEV_INITRD) && phys_initrd_size) {
372	387	/* the generic initrd code expects virtual addresses */
373	388	initrd_start = __phys_to_virt(phys_initrd_start);
@@ -376,18 +391,11 @@ void __init arm64_memblock_init(void) · 2 unmodified lines
376	391
377	392	early_init_fdt_scan_reserved_mem();
378	393
379		− if (IS_ENABLED(CONFIG_ZONE_DMA)) {
380		− zone_dma_bits = ARM64_ZONE_DMA_BITS;
381		− arm64_dma_phys_limit = max_zone_phys(ARM64_ZONE_DMA_BITS);
382		− }
383		−
384	394	if (IS_ENABLED(CONFIG_ZONE_DMA32))
385	395	arm64_dma32_phys_limit = max_zone_phys(32);
386	396	else
387	397	arm64_dma32_phys_limit = PHYS_MASK + 1;
388	398
389		− reserve_crashkernel();
390		−
391	399	reserve_elfcorehdr();
392	400
393	401	high_memory = __va(memblock_end_of_DRAM() - 1) + 1;
@@ -427,6 +435,12 @@ void __init bootmem_init(void) · 33 unmodified lines
427	435	sparse_init();
428	436	zone_sizes_init(min, max);
429	437
	438	+ /*
	439	+ * request_standard_resources() depends on crashkernel's memory being
	440	+ * reserved, so do it here.
	441	+ */
	442	+ reserve_crashkernel();
	443	+
430	444	memblock_dump_all();
431	445	}
432	446

arch/arm64/mm/mmu.c+117−32

@@ -464,20 +464,35 @@ void __init mark_linear_text_alias_ro(void) · 463 unmodified lines
464	464	/*
465	465	* Remove the write permissions from the linear alias of .text/.rodata
466	466	*/
467		− update_mapping_prot(__pa_symbol(_text), (unsigned long)lm_alias(_text),
468		− (unsigned long)__init_begin - (unsigned long)_text,
	467	+ update_mapping_prot(__pa_symbol(_stext), (unsigned long)lm_alias(_stext),
	468	+ (unsigned long)__init_begin - (unsigned long)_stext,
469	469	PAGE_KERNEL_RO);
470	470	}
471	471
	472	+static bool crash_mem_map __initdata;
	473	+
	474	+static int __init enable_crash_mem_map(char *arg)
	475	+{
	476	+ /*
	477	+ * Proper parameter parsing is done by reserve_crashkernel(). We only
	478	+ * need to know if the linear map has to avoid block mappings so that
	479	+ * the crashkernel reservations can be unmapped later.
	480	+ */
	481	+ crash_mem_map = true;
	482	+
	483	+ return 0;
	484	+}
	485	+early_param("crashkernel", enable_crash_mem_map);
	486	+
472	487	static void __init map_mem(pgd_t *pgdp)
473	488	{
474		− phys_addr_t kernel_start = __pa_symbol(_text);
	489	+ phys_addr_t kernel_start = __pa_symbol(_stext);
475	490	phys_addr_t kernel_end = __pa_symbol(__init_begin);
476	491	phys_addr_t start, end;
477	492	int flags = 0;
478	493	u64 i;
479	494
480		− if (rodata_full \|\| debug_pagealloc_enabled())
	495	+ if (rodata_full \|\| crash_mem_map \|\| debug_pagealloc_enabled())
481	496	flags = NO_BLOCK_MAPPINGS \| NO_CONT_MAPPINGS;
482	497
483	498	/*
@@ -487,11 +502,6 @@ static void __init map_mem(pgd_t *pgdp) · 3 unmodified lines
487	502	* the following for-loop
488	503	*/
489	504	memblock_mark_nomap(kernel_start, kernel_end - kernel_start);
490		−#ifdef CONFIG_KEXEC_CORE
491		− if (crashk_res.end)
492		− memblock_mark_nomap(crashk_res.start,
493		− resource_size(&crashk_res));
494		−#endif
495	505
496	506	/* map all the memory banks */
497	507	for_each_mem_range(i, &start, &end) {
@@ -506,7 +516,7 @@ static void __init map_mem(pgd_t *pgdp) · 8 unmodified lines
506	516	}
507	517
508	518	/*
509		− * Map the linear alias of the [_text, __init_begin) interval
	519	+ * Map the linear alias of the [_stext, __init_begin) interval
510	520	* as non-executable now, and remove the write permission in
511	521	* mark_linear_text_alias_ro() below (which will be called after
512	522	* alternative patching has completed). This makes the contents
@@ -518,21 +528,6 @@ static void __init map_mem(pgd_t *pgdp) · 5 unmodified lines
518	528	__map_memblock(pgdp, kernel_start, kernel_end,
519	529	PAGE_KERNEL, NO_CONT_MAPPINGS);
520	530	memblock_clear_nomap(kernel_start, kernel_end - kernel_start);
521		−
522		−#ifdef CONFIG_KEXEC_CORE
523		− /*
524		− * Use page-level mappings here so that we can shrink the region
525		− * in page granularity and put back unused memory to buddy system
526		− * through /sys/kernel/kexec_crash_size interface.
527		− */
528		− if (crashk_res.end) {
529		− __map_memblock(pgdp, crashk_res.start, crashk_res.end + 1,
530		− PAGE_KERNEL,
531		− NO_BLOCK_MAPPINGS \| NO_CONT_MAPPINGS);
532		− memblock_clear_nomap(crashk_res.start,
533		− resource_size(&crashk_res));
534		− }
535		−#endif
536	531	}
537	532
538	533	void mark_rodata_ro(void)
@@ -665,7 +660,7 @@ static void __init map_kernel(pgd_t *pgdp) · 126 unmodified lines
665	660	* Only rodata will be remapped with different permissions later on,
666	661	* all other segments are allowed to use contiguous mappings.
667	662	*/
668		− map_kernel_segment(pgdp, _text, _etext, text_prot, &vmlinux_text, 0,
	663	+ map_kernel_segment(pgdp, _stext, _etext, text_prot, &vmlinux_text, 0,
669	664	VM_NO_GUARD);
670	665	map_kernel_segment(pgdp, __start_rodata, __inittext_begin, PAGE_KERNEL,
671	666	&vmlinux_rodata, NO_CONT_MAPPINGS, VM_NO_GUARD);
@@ -1132,8 +1127,11 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, · 460 unmodified lines
1132	1127	void *p = NULL;
1133	1128
1134	1129	p = vmemmap_alloc_block_buf(PMD_SIZE, node, altmap);
1135		− if (!p)
1136		− return -ENOMEM;
	1130	+ if (!p) {
	1131	+ if (vmemmap_populate_basepages(addr, next, node, altmap))
	1132	+ return -ENOMEM;
	1133	+ continue;
	1134	+ }
1137	1135
1138	1136	pmd_set_huge(pmdp, __pa(p), __pgprot(PROT_SECT_NORMAL));
1139	1137	} else
@@ -1510,13 +1508,43 @@ static int prevent_bootmem_remove_notifier(struct notifier_block *nb, · 370 unmodified lines
1510	1508	unsigned long end_pfn = arg->start_pfn + arg->nr_pages;
1511	1509	unsigned long pfn = arg->start_pfn;
1512	1510
1513		− if (action != MEM_GOING_OFFLINE)
	1511	+ if ((action != MEM_GOING_OFFLINE) && (action != MEM_OFFLINE))
1514	1512	return NOTIFY_OK;
1515	1513
1516	1514	for (; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
	1515	+ unsigned long start = PFN_PHYS(pfn);
	1516	+ unsigned long end = start + (1UL << PA_SECTION_SHIFT);
	1517	+
1517	1518	ms = __pfn_to_section(pfn);
1518		− if (early_section(ms))
	1519	+ if (!early_section(ms))
	1520	+ continue;
	1521	+
	1522	+ if (action == MEM_GOING_OFFLINE) {
	1523	+ /*
	1524	+ * Boot memory removal is not supported. Prevent
	1525	+ * it via blocking any attempted offline request
	1526	+ * for the boot memory and just report it.
	1527	+ */
	1528	+ pr_warn("Boot memory [%lx %lx] offlining attempted\n", start, end);
1519	1529	return NOTIFY_BAD;
	1530	+ } else if (action == MEM_OFFLINE) {
	1531	+ /*
	1532	+ * This should have never happened. Boot memory
	1533	+ * offlining should have been prevented by this
	1534	+ * very notifier. Probably some memory removal
	1535	+ * procedure might have changed which would then
	1536	+ * require further debug.
	1537	+ */
	1538	+ pr_err("Boot memory [%lx %lx] offlined\n", start, end);
	1539	+
	1540	+ /*
	1541	+ * Core memory hotplug does not process a return
	1542	+ * code from the notifier for MEM_OFFLINE events.
	1543	+ * The error condition has been reported. Return
	1544	+ * from here as if ignored.
	1545	+ */
	1546	+ return NOTIFY_DONE;
	1547	+ }
1520	1548	}
1521	1549	return NOTIFY_OK;
1522	1550	}
@@ -1525,9 +1553,66 @@ static struct notifier_block prevent_bootmem_remove_nb = { · 2 unmodified lines
1525	1553	.notifier_call = prevent_bootmem_remove_notifier,
1526	1554	};
1527	1555
	1556	+/*
	1557	+ * This ensures that boot memory sections on the platform are online
	1558	+ * from early boot. Memory sections could not be prevented from being
	1559	+ * offlined, unless for some reason they are not online to begin with.
	1560	+ * This helps validate the basic assumption on which the above memory
	1561	+ * event notifier works to prevent boot memory section offlining and
	1562	+ * its possible removal.
	1563	+ */
	1564	+static void validate_bootmem_online(void)
	1565	+{
	1566	+ phys_addr_t start, end, addr;
	1567	+ struct mem_section *ms;
	1568	+ u64 i;
	1569	+
	1570	+ /*
	1571	+ * Scanning across all memblock might be expensive
	1572	+ * on some big memory systems. Hence enable this
	1573	+ * validation only with DEBUG_VM.
	1574	+ */
	1575	+ if (!IS_ENABLED(CONFIG_DEBUG_VM))
	1576	+ return;
	1577	+
	1578	+ for_each_mem_range(i, &start, &end) {
	1579	+ for (addr = start; addr < end; addr += (1UL << PA_SECTION_SHIFT)) {
	1580	+ ms = __pfn_to_section(PHYS_PFN(addr));
	1581	+
	1582	+ /*
	1583	+ * All memory ranges in the system at this point
	1584	+ * should have been marked as early sections.
	1585	+ */
	1586	+ WARN_ON(!early_section(ms));
	1587	+
	1588	+ /*
	1589	+ * Memory notifier mechanism here to prevent boot
	1590	+ * memory offlining depends on the fact that each
	1591	+ * early section memory on the system is initially
	1592	+ * online. Otherwise a given memory section which
	1593	+ * is already offline will be overlooked and can
	1594	+ * be removed completely. Call out such sections.
	1595	+ */
	1596	+ if (!online_section(ms))
	1597	+ pr_err("Boot memory [%llx %llx] is offline, can be removed\n",
	1598	+ addr, addr + (1UL << PA_SECTION_SHIFT));
	1599	+ }
	1600	+ }
	1601	+}
	1602	+
1528	1603	static int __init prevent_bootmem_remove_init(void)
1529	1604	{
1530		− return register_memory_notifier(&prevent_bootmem_remove_nb);
	1605	+ int ret = 0;
	1606	+
	1607	+ if (!IS_ENABLED(CONFIG_MEMORY_HOTREMOVE))
	1608	+ return ret;
	1609	+
	1610	+ validate_bootmem_online();
	1611	+ ret = register_memory_notifier(&prevent_bootmem_remove_nb);
	1612	+ if (ret)
	1613	+ pr_err("%s: Notifier registration failed %d\n", __func__, ret);
	1614	+
	1615	+ return ret;
1531	1616	}
1532		−device_initcall(prevent_bootmem_remove_init);
	1617	+early_initcall(prevent_bootmem_remove_init);
1533	1618	#endif

arch/arm64/mm/proc.S+3−3

@@ -40,7 +40,7 @@ · 39 unmodified lines
40	40	#define TCR_CACHE_FLAGS TCR_IRGN_WBWA \| TCR_ORGN_WBWA
41	41
42	42	#ifdef CONFIG_KASAN_SW_TAGS
43		−#define TCR_KASAN_FLAGS TCR_TBI1
	43	+#define TCR_KASAN_FLAGS TCR_TBI1 \| TCR_TBID1
44	44	#else
45	45	#define TCR_KASAN_FLAGS 0
46	46	#endif
@@ -168,7 +168,7 @@ SYM_FUNC_END(cpu_do_resume) · 121 unmodified lines
168	168	.pushsection ".idmap.text", "awx"
169	169
170	170	.macro __idmap_cpu_set_reserved_ttbr1, tmp1, tmp2
171		− adrp \tmp1, empty_zero_page
	171	+ adrp \tmp1, reserved_pg_dir
172	172	phys_to_ttbr \tmp2, \tmp1
173	173	offset_ttbr1 \tmp2, \tmp1
174	174	msr ttbr1_el1, \tmp2
@@ -489,6 +489,6 @@ SYM_FUNC_START(__cpu_setup) · 314 unmodified lines
489	489	/*
490	490	* Prepare SCTLR
491	491	*/
492		− mov_q x0, SCTLR_EL1_SET
	492	+ mov_q x0, INIT_SCTLR_EL1_MMU_ON
493	493	ret // return to head.S
494	494	SYM_FUNC_END(__cpu_setup)

arch/h8300/include/uapi/asm/signal.h+0−24

@@ -57,30 +57,6 @@ typedef unsigned long sigset_t; · 56 unmodified lines
57	57	#define SIGRTMIN 32
58	58	#define SIGRTMAX _NSIG
59	59
60		−/*
61		− * SA_FLAGS values:
62		− *
63		− * SA_ONSTACK indicates that a registered stack_t will be used.
64		− * SA_RESTART flag to get restarting signals (which were the default long ago)
65		− * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop.
66		− * SA_RESETHAND clears the handler when the signal is delivered.
67		− * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies.
68		− * SA_NODEFER prevents the current signal from being masked in the handler.
69		− *
70		− * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single
71		− * Unix names RESETHAND and NODEFER respectively.
72		− */
73		−#define SA_NOCLDSTOP 0x00000001
74		−#define SA_NOCLDWAIT 0x00000002 /* not supported yet */
75		−#define SA_SIGINFO 0x00000004
76		−#define SA_ONSTACK 0x08000000
77		−#define SA_RESTART 0x10000000
78		−#define SA_NODEFER 0x40000000
79		−#define SA_RESETHAND 0x80000000
80		−
81		−#define SA_NOMASK SA_NODEFER
82		−#define SA_ONESHOT SA_RESETHAND
83		−
84	60	#define SA_RESTORER 0x04000000
85	61
86	62	#define MINSIGSTKSZ 2048

arch/ia64/include/uapi/asm/signal.h+0−24

@@ -53,30 +53,6 @@ · 52 unmodified lines
53	53	#define SIGRTMIN 32
54	54	#define SIGRTMAX _NSIG
55	55
56		−/*
57		− * SA_FLAGS values:
58		− *
59		− * SA_ONSTACK indicates that a registered stack_t will be used.
60		− * SA_RESTART flag to get restarting signals (which were the default long ago)
61		− * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop.
62		− * SA_RESETHAND clears the handler when the signal is delivered.
63		− * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies.
64		− * SA_NODEFER prevents the current signal from being masked in the handler.
65		− *
66		− * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single
67		− * Unix names RESETHAND and NODEFER respectively.
68		− */
69		−#define SA_NOCLDSTOP 0x00000001
70		−#define SA_NOCLDWAIT 0x00000002
71		−#define SA_SIGINFO 0x00000004
72		−#define SA_ONSTACK 0x08000000
73		−#define SA_RESTART 0x10000000
74		−#define SA_NODEFER 0x40000000
75		−#define SA_RESETHAND 0x80000000
76		−
77		−#define SA_NOMASK SA_NODEFER
78		−#define SA_ONESHOT SA_RESETHAND
79		−
80	56	#define SA_RESTORER 0x04000000
81	57
82	58	/*

arch/m68k/include/uapi/asm/signal.h+0−24

@@ -57,30 +57,6 @@ typedef unsigned long sigset_t; · 56 unmodified lines
57	57	#define SIGRTMIN 32
58	58	#define SIGRTMAX _NSIG
59	59
60		−/*
61		− * SA_FLAGS values:
62		− *
63		− * SA_ONSTACK indicates that a registered stack_t will be used.
64		− * SA_RESTART flag to get restarting signals (which were the default long ago)
65		− * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop.
66		− * SA_RESETHAND clears the handler when the signal is delivered.
67		− * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies.
68		− * SA_NODEFER prevents the current signal from being masked in the handler.
69		− *
70		− * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single
71		− * Unix names RESETHAND and NODEFER respectively.
72		− */
73		−#define SA_NOCLDSTOP 0x00000001
74		−#define SA_NOCLDWAIT 0x00000002
75		−#define SA_SIGINFO 0x00000004
76		−#define SA_ONSTACK 0x08000000
77		−#define SA_RESTART 0x10000000
78		−#define SA_NODEFER 0x40000000
79		−#define SA_RESETHAND 0x80000000
80		−
81		−#define SA_NOMASK SA_NODEFER
82		−#define SA_ONESHOT SA_RESETHAND
83		−
84	60	#define MINSIGSTKSZ 2048
85	61	#define SIGSTKSZ 8192
86	62

arch/mips/include/uapi/asm/signal.h+0−12

@@ -62,18 +62,6 @@ typedef unsigned long old_sigset_t; /* at least 32 bits */ · 61 unmodified lines
62	62	#define SIGRTMAX _NSIG
63	63
64	64	/*
65		− * SA_FLAGS values:
66		− *
67		− * SA_ONSTACK indicates that a registered stack_t will be used.
68		− * SA_RESTART flag to get restarting signals (which were the default long ago)
69		− * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop.
70		− * SA_RESETHAND clears the handler when the signal is delivered.
71		− * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies.
72		− * SA_NODEFER prevents the current signal from being masked in the handler.
73		− *
74		− * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single
75		− * Unix names RESETHAND and NODEFER respectively.
76		− *
77	65	* SA_RESTORER used to be defined as 0x04000000 but only the O32 ABI ever
78	66	* supported its use and no libc was using it, so the entire sa-restorer
79	67	* functionality was removed with lmo commit 39bffc12c3580ab for 2.5.48

arch/parisc/include/asm/signal.h+2−0

@@ -21,6 +21,8 @@ typedef struct { · 20 unmodified lines
21	21	unsigned long sig[_NSIG_WORDS];
22	22	} sigset_t;
23	23
	24	+#define __ARCH_UAPI_SA_FLAGS _SA_SIGGFAULT
	25	+
24	26	#include <asm/sigcontext.h>
25	27
26	28	#endif /* !__ASSEMBLY */

arch/parisc/include/uapi/asm/signal.h+1−33

@@ -41,19 +41,6 @@ · 40 unmodified lines
41	41	#define SIGRTMIN 32
42	42	#define SIGRTMAX _NSIG
43	43
44		−/*
45		− * SA_FLAGS values:
46		− *
47		− * SA_ONSTACK indicates that a registered stack_t will be used.
48		− * SA_RESTART flag to get restarting signals (which were the default long ago)
49		− * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop.
50		− * SA_RESETHAND clears the handler when the signal is delivered.
51		− * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies.
52		− * SA_NODEFER prevents the current signal from being masked in the handler.
53		− *
54		− * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single
55		− * Unix names RESETHAND and NODEFER respectively.
56		− */
57	44	#define SA_ONSTACK 0x00000001
58	45	#define SA_RESETHAND 0x00000004
59	46	#define SA_NOCLDSTOP 0x00000008
@@ -68,14 +55,7 @@ · 8 unmodified lines
68	55	#define MINSIGSTKSZ 2048
69	56	#define SIGSTKSZ 8192
70	57
71		−
72		−#define SIG_BLOCK 0 /* for blocking signals */
73		−#define SIG_UNBLOCK 1 /* for unblocking signals */
74		−#define SIG_SETMASK 2 /* for setting the signal mask */
75		−
76		−#define SIG_DFL ((__sighandler_t)0) /* default signal handling */
77		−#define SIG_IGN ((__sighandler_t)1) /* ignore signal */
78		−#define SIG_ERR ((__sighandler_t)-1) /* error return from signal */
	58	+#include <asm-generic/signal-defs.h>
79	59
80	60	# ifndef __ASSEMBLY__
81	61
@@ -84,18 +64,6 @@ · 2 unmodified lines
84	64	/* Avoid too many header ordering problems. */
85	65	struct siginfo;
86	66
87		−/* Type of a signal handler. */
88		−#if defined(__LP64__)
89		−/* function pointers on 64-bit parisc are pointers to little structs and the
90		− * compiler doesn't support code which changes or tests the address of
91		− * the function in the little struct. This is really ugly -PB
92		− */
93		−typedef char __user *__sighandler_t;
94		−#else
95		−typedef void __signalfn_t(int);
96		−typedef __signalfn_t __user *__sighandler_t;
97		−#endif
98		−
99	67	typedef struct sigaltstack {
100	68	void __user *ss_sp;
101	69	int ss_flags;

arch/powerpc/include/uapi/asm/signal.h+0−24

@@ -60,30 +60,6 @@ typedef struct { · 59 unmodified lines
60	60	#define SIGRTMIN 32
61	61	#define SIGRTMAX _NSIG
62	62
63		−/*
64		− * SA_FLAGS values:
65		− *
66		− * SA_ONSTACK is not currently supported, but will allow sigaltstack(2).
67		− * SA_RESTART flag to get restarting signals (which were the default long ago)
68		− * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop.
69		− * SA_RESETHAND clears the handler when the signal is delivered.
70		− * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies.
71		− * SA_NODEFER prevents the current signal from being masked in the handler.
72		− *
73		− * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single
74		− * Unix names RESETHAND and NODEFER respectively.
75		− */
76		−#define SA_NOCLDSTOP 0x00000001U
77		−#define SA_NOCLDWAIT 0x00000002U
78		−#define SA_SIGINFO 0x00000004U
79		−#define SA_ONSTACK 0x08000000U
80		−#define SA_RESTART 0x10000000U
81		−#define SA_NODEFER 0x40000000U
82		−#define SA_RESETHAND 0x80000000U
83		−
84		−#define SA_NOMASK SA_NODEFER
85		−#define SA_ONESHOT SA_RESETHAND
86		−
87	63	#define SA_RESTORER 0x04000000U
88	64
89	65	#define MINSIGSTKSZ 2048

arch/s390/include/uapi/asm/signal.h+0−24

@@ -65,30 +65,6 @@ typedef unsigned long sigset_t; · 64 unmodified lines
65	65	#define SIGRTMIN 32
66	66	#define SIGRTMAX _NSIG
67	67
68		−/*
69		− * SA_FLAGS values:
70		− *
71		− * SA_ONSTACK indicates that a registered stack_t will be used.
72		− * SA_RESTART flag to get restarting signals (which were the default long ago)
73		− * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop.
74		− * SA_RESETHAND clears the handler when the signal is delivered.
75		− * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies.
76		− * SA_NODEFER prevents the current signal from being masked in the handler.
77		− *
78		− * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single
79		− * Unix names RESETHAND and NODEFER respectively.
80		− */
81		−#define SA_NOCLDSTOP 0x00000001
82		−#define SA_NOCLDWAIT 0x00000002
83		−#define SA_SIGINFO 0x00000004
84		−#define SA_ONSTACK 0x08000000
85		−#define SA_RESTART 0x10000000
86		−#define SA_NODEFER 0x40000000
87		−#define SA_RESETHAND 0x80000000
88		−
89		−#define SA_NOMASK SA_NODEFER
90		−#define SA_ONESHOT SA_RESETHAND
91		−
92	68	#define SA_RESTORER 0x04000000
93	69
94	70	#define MINSIGSTKSZ 2048

arch/sparc/include/uapi/asm/signal.h+1−3

@@ -137,13 +137,11 @@ struct sigstack { · 136 unmodified lines
137	137	#define SA_STACK _SV_SSTACK
138	138	#define SA_ONSTACK _SV_SSTACK
139	139	#define SA_RESTART _SV_INTR
140		−#define SA_ONESHOT _SV_RESET
	140	+#define SA_RESETHAND _SV_RESET
141	141	#define SA_NODEFER 0x20u
142	142	#define SA_NOCLDWAIT 0x100u
143	143	#define SA_SIGINFO 0x200u
144	144
145		−#define SA_NOMASK SA_NODEFER
146		−
147	145	#define SIG_BLOCK 0x01 /* for blocking signals */
148	146	#define SIG_UNBLOCK 0x02 /* for unblocking signals */
149	147	#define SIG_SETMASK 0x04 /* for setting the signal mask */

arch/x86/include/uapi/asm/signal.h+0−24

@@ -62,30 +62,6 @@ typedef unsigned long sigset_t; · 61 unmodified lines
62	62	#define SIGRTMIN 32
63	63	#define SIGRTMAX _NSIG
64	64
65		−/*
66		− * SA_FLAGS values:
67		− *
68		− * SA_ONSTACK indicates that a registered stack_t will be used.
69		− * SA_RESTART flag to get restarting signals (which were the default long ago)
70		− * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop.
71		− * SA_RESETHAND clears the handler when the signal is delivered.
72		− * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies.
73		− * SA_NODEFER prevents the current signal from being masked in the handler.
74		− *
75		− * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single
76		− * Unix names RESETHAND and NODEFER respectively.
77		− */
78		−#define SA_NOCLDSTOP 0x00000001u
79		−#define SA_NOCLDWAIT 0x00000002u
80		−#define SA_SIGINFO 0x00000004u
81		−#define SA_ONSTACK 0x08000000u
82		−#define SA_RESTART 0x10000000u
83		−#define SA_NODEFER 0x40000000u
84		−#define SA_RESETHAND 0x80000000u
85		−
86		−#define SA_NOMASK SA_NODEFER
87		−#define SA_ONESHOT SA_RESETHAND
88		−
89	65	#define SA_RESTORER 0x04000000
90	66
91	67	#define MINSIGSTKSZ 2048

arch/x86/kernel/signal_compat.c+0−7

@@ -165,16 +165,9 @@ void sigaction_compat_abi(struct k_sigaction act, struct k_sigaction oact) · 164 unmodified lines
165	165	{
166	166	signal_compat_build_tests();
167	167
168		− /* Don't leak in-kernel non-uapi flags to user-space */
169		− if (oact)
170		− oact->sa.sa_flags &= ~(SA_IA32_ABI \| SA_X32_ABI);
171		−
172	168	if (!act)
173	169	return;
174	170
175		− /* Don't let flags to be set from userspace */
176		− act->sa.sa_flags &= ~(SA_IA32_ABI \| SA_X32_ABI);
177		−
178	171	if (in_ia32_syscall())
179	172	act->sa.sa_flags \|= SA_IA32_ABI;
180	173	if (in_x32_syscall())

arch/xtensa/include/uapi/asm/signal.h+0−24

@@ -72,30 +72,6 @@ typedef struct { · 71 unmodified lines
72	72	#define SIGRTMIN 32
73	73	#define SIGRTMAX (_NSIG-1)
74	74
75		−/*
76		− * SA_FLAGS values:
77		− *
78		− * SA_ONSTACK indicates that a registered stack_t will be used.
79		− * SA_RESTART flag to get restarting signals (which were the default long ago)
80		− * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop.
81		− * SA_RESETHAND clears the handler when the signal is delivered.
82		− * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies.
83		− * SA_NODEFER prevents the current signal from being masked in the handler.
84		− *
85		− * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single
86		− * Unix names RESETHAND and NODEFER respectively.
87		− */
88		−#define SA_NOCLDSTOP 0x00000001
89		−#define SA_NOCLDWAIT 0x00000002 /* not supported yet */
90		−#define SA_SIGINFO 0x00000004
91		−#define SA_ONSTACK 0x08000000
92		−#define SA_RESTART 0x10000000
93		−#define SA_NODEFER 0x40000000
94		−#define SA_RESETHAND 0x80000000
95		−
96		−#define SA_NOMASK SA_NODEFER
97		−#define SA_ONESHOT SA_RESETHAND
98		−
99	75	#define SA_RESTORER 0x04000000
100	76
101	77	#define MINSIGSTKSZ 2048

drivers/acpi/arm64/iort.c+55−0

@@ -1720,3 +1720,58 @@ void __init acpi_iort_init(void) · 1719 unmodified lines
1720	1720
1721	1721	iort_init_platform_devices();
1722	1722	}
	1723	+
	1724	+#ifdef CONFIG_ZONE_DMA
	1725	+/*
	1726	+ * Extract the highest CPU physical address accessible to all DMA masters in
	1727	+ * the system. PHYS_ADDR_MAX is returned when no constrained device is found.
	1728	+ */
	1729	+phys_addr_t __init acpi_iort_dma_get_max_cpu_address(void)
	1730	+{
	1731	+ phys_addr_t limit = PHYS_ADDR_MAX;
	1732	+ struct acpi_iort_node node, end;
	1733	+ struct acpi_table_iort *iort;
	1734	+ acpi_status status;
	1735	+ int i;
	1736	+
	1737	+ if (acpi_disabled)
	1738	+ return limit;
	1739	+
	1740	+ status = acpi_get_table(ACPI_SIG_IORT, 0,
	1741	+ (struct acpi_table_header **)&iort);
	1742	+ if (ACPI_FAILURE(status))
	1743	+ return limit;
	1744	+
	1745	+ node = ACPI_ADD_PTR(struct acpi_iort_node, iort, iort->node_offset);
	1746	+ end = ACPI_ADD_PTR(struct acpi_iort_node, iort, iort->header.length);
	1747	+
	1748	+ for (i = 0; i < iort->node_count; i++) {
	1749	+ if (node >= end)
	1750	+ break;
	1751	+
	1752	+ switch (node->type) {
	1753	+ struct acpi_iort_named_component *ncomp;
	1754	+ struct acpi_iort_root_complex *rc;
	1755	+ phys_addr_t local_limit;
	1756	+
	1757	+ case ACPI_IORT_NODE_NAMED_COMPONENT:
	1758	+ ncomp = (struct acpi_iort_named_component *)node->node_data;
	1759	+ local_limit = DMA_BIT_MASK(ncomp->memory_address_limit);
	1760	+ limit = min_not_zero(limit, local_limit);
	1761	+ break;
	1762	+
	1763	+ case ACPI_IORT_NODE_PCI_ROOT_COMPLEX:
	1764	+ if (node->revision < 1)
	1765	+ break;
	1766	+
	1767	+ rc = (struct acpi_iort_root_complex *)node->node_data;
	1768	+ local_limit = DMA_BIT_MASK(rc->memory_address_limit);
	1769	+ limit = min_not_zero(limit, local_limit);
	1770	+ break;
	1771	+ }
	1772	+ node = ACPI_ADD_PTR(struct acpi_iort_node, node, node->length);
	1773	+ }
	1774	+ acpi_put_table(&iort->header);
	1775	+ return limit;
	1776	+}
	1777	+#endif

drivers/firmware/arm_sdei.c+0−14

@@ -31,7 +31,6 @@ · 30 unmodified lines
31	31	#include <linux/slab.h>
32	32	#include <linux/smp.h>
33	33	#include <linux/spinlock.h>
34		−#include <linux/uaccess.h>
35	34
36	35	/*
37	36	* The call to use to reach the firmware.
@@ -1092,26 +1091,13 @@ int sdei_event_handler(struct pt_regs *regs, · 1054 unmodified lines
1092	1091	struct sdei_registered_event *arg)
1093	1092	{
1094	1093	int err;
1095		− mm_segment_t orig_addr_limit;
1096	1094	u32 event_num = arg->event_num;
1097	1095
1098		− /*
1099		− * Save restore 'fs'.
1100		− * The architecture's entry code save/restores 'fs' when taking an
1101		− * exception from the kernel. This ensures addr_limit isn't inherited
1102		− * if you interrupted something that allowed the uaccess routines to
1103		− * access kernel memory.
1104		− * Do the same here because this doesn't come via the same entry code.
1105		− */
1106		− orig_addr_limit = force_uaccess_begin();
1107		−
1108	1096	err = arg->callback(event_num, regs, arg->callback_arg);
1109	1097	if (err)
1110	1098	pr_err_ratelimited("event %u on CPU %u failed with error: %d\n",
1111	1099	event_num, smp_processor_id(), err);
1112	1100
1113		− force_uaccess_end(orig_addr_limit);
1114		−
1115	1101	return err;
1116	1102	}
1117	1103	NOKPROBE_SYMBOL(sdei_event_handler);

drivers/of/address.c+42−0

@@ -1024,6 +1024,48 @@ out: · 1023 unmodified lines
1024	1024	}
1025	1025	#endif /* CONFIG_HAS_DMA */
1026	1026
	1027	+/**
	1028	+ * of_dma_get_max_cpu_address - Gets highest CPU address suitable for DMA
	1029	+ * @np: The node to start searching from or NULL to start from the root
	1030	+ *
	1031	+ * Gets the highest CPU physical address that is addressable by all DMA masters
	1032	+ * in the sub-tree pointed by np, or the whole tree if NULL is passed. If no
	1033	+ * DMA constrained device is found, it returns PHYS_ADDR_MAX.
	1034	+ */
	1035	+phys_addr_t __init of_dma_get_max_cpu_address(struct device_node *np)
	1036	+{
	1037	+ phys_addr_t max_cpu_addr = PHYS_ADDR_MAX;
	1038	+ struct of_range_parser parser;
	1039	+ phys_addr_t subtree_max_addr;
	1040	+ struct device_node *child;
	1041	+ struct of_range range;
	1042	+ const __be32 *ranges;
	1043	+ u64 cpu_end = 0;
	1044	+ int len;
	1045	+
	1046	+ if (!np)
	1047	+ np = of_root;
	1048	+
	1049	+ ranges = of_get_property(np, "dma-ranges", &len);
	1050	+ if (ranges && len) {
	1051	+ of_dma_range_parser_init(&parser, np);
	1052	+ for_each_of_range(&parser, &range)
	1053	+ if (range.cpu_addr + range.size > cpu_end)
	1054	+ cpu_end = range.cpu_addr + range.size - 1;
	1055	+
	1056	+ if (max_cpu_addr > cpu_end)
	1057	+ max_cpu_addr = cpu_end;
	1058	+ }
	1059	+
	1060	+ for_each_available_child_of_node(np, child) {
	1061	+ subtree_max_addr = of_dma_get_max_cpu_address(child);
	1062	+ if (max_cpu_addr > subtree_max_addr)
	1063	+ max_cpu_addr = subtree_max_addr;
	1064	+ }
	1065	+
	1066	+ return max_cpu_addr;
	1067	+}
	1068	+
1027	1069	/**
1028	1070	* of_dma_is_coherent - Check if device is coherent
1029	1071	* @np: device node

drivers/of/unittest.c+21−0

@@ -869,6 +869,26 @@ static void __init of_unittest_changeset(void) · 868 unmodified lines
869	869	#endif
870	870	}
871	871
	872	+static void __init of_unittest_dma_get_max_cpu_address(void)
	873	+{
	874	+ struct device_node *np;
	875	+ phys_addr_t cpu_addr;
	876	+
	877	+ if (!IS_ENABLED(CONFIG_OF_ADDRESS))
	878	+ return;
	879	+
	880	+ np = of_find_node_by_path("/testcase-data/address-tests");
	881	+ if (!np) {
	882	+ pr_err("missing testcase data\n");
	883	+ return;
	884	+ }
	885	+
	886	+ cpu_addr = of_dma_get_max_cpu_address(np);
	887	+ unittest(cpu_addr == 0x4fffffff,
	888	+ "of_dma_get_max_cpu_address: wrong CPU addr %pad (expecting %x)\n",
	889	+ &cpu_addr, 0x4fffffff);
	890	+}
	891	+
872	892	static void __init of_unittest_dma_ranges_one(const char *path,
873	893	u64 expect_dma_addr, u64 expect_paddr)
874	894	{
@@ -3266,6 +3286,7 @@ static int __init of_unittest(void) · 2391 unmodified lines
3266	3286	of_unittest_changeset();
3267	3287	of_unittest_parse_interrupts();
3268	3288	of_unittest_parse_interrupts_extended();
	3289	+ of_unittest_dma_get_max_cpu_address();
3269	3290	of_unittest_parse_dma_ranges();
3270	3291	of_unittest_pci_dma_ranges();
3271	3292	of_unittest_match_node();

drivers/perf/Kconfig+7−0

@@ -130,6 +130,13 @@ config ARM_SPE_PMU · 129 unmodified lines
130	130	Extension, which provides periodic sampling of operations in
131	131	the CPU pipeline and reports this via the perf AUX interface.
132	132
	133	+config ARM_DMC620_PMU
	134	+ tristate "Enable PMU support for the ARM DMC-620 memory controller"
	135	+ depends on (ARM64 && ACPI) \|\| COMPILE_TEST
	136	+ help
	137	+ Support for PMU events monitoring on the ARM DMC-620 memory
	138	+ controller.
	139	+
133	140	source "drivers/perf/hisilicon/Kconfig"
134	141
135	142	endmenu

drivers/perf/Makefile+1−0

@@ -13,3 +13,4 @@ obj-$(CONFIG_QCOM_L3_PMU) += qcom_l3_pmu.o · 12 unmodified lines
13	13	obj-$(CONFIG_THUNDERX2_PMU) += thunderx2_pmu.o
14	14	obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
15	15	obj-$(CONFIG_ARM_SPE_PMU) += arm_spe_pmu.o
	16	+obj-$(CONFIG_ARM_DMC620_PMU) += arm_dmc620_pmu.o

drivers/perf/arm_dmc620_pmu.c+748−0

@@ -0,0 +1,748 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * ARM DMC-620 memory controller PMU driver
	4	+ *
	5	+ * Copyright (C) 2020 Ampere Computing LLC.
	6	+ */
	7	+
	8	+#define DMC620_PMUNAME "arm_dmc620"
	9	+#define DMC620_DRVNAME DMC620_PMUNAME "_pmu"
	10	+#define pr_fmt(fmt) DMC620_DRVNAME ": " fmt
	11	+
	12	+#include <linux/acpi.h>
	13	+#include <linux/bitfield.h>
	14	+#include <linux/bitops.h>
	15	+#include <linux/cpuhotplug.h>
	16	+#include <linux/cpumask.h>
	17	+#include <linux/device.h>
	18	+#include <linux/errno.h>
	19	+#include <linux/interrupt.h>
	20	+#include <linux/irq.h>
	21	+#include <linux/kernel.h>
	22	+#include <linux/list.h>
	23	+#include <linux/module.h>
	24	+#include <linux/mutex.h>
	25	+#include <linux/perf_event.h>
	26	+#include <linux/platform_device.h>
	27	+#include <linux/printk.h>
	28	+#include <linux/rculist.h>
	29	+#include <linux/refcount.h>
	30	+
	31	+#define DMC620_PA_SHIFT 12
	32	+#define DMC620_CNT_INIT 0x80000000
	33	+#define DMC620_CNT_MAX_PERIOD 0xffffffff
	34	+#define DMC620_PMU_CLKDIV2_MAX_COUNTERS 8
	35	+#define DMC620_PMU_CLK_MAX_COUNTERS 2
	36	+#define DMC620_PMU_MAX_COUNTERS \
	37	+ (DMC620_PMU_CLKDIV2_MAX_COUNTERS + DMC620_PMU_CLK_MAX_COUNTERS)
	38	+
	39	+/*
	40	+ * The PMU registers start at 0xA00 in the DMC-620 memory map, and these
	41	+ * offsets are relative to that base.
	42	+ *
	43	+ * Each counter has a group of control/value registers, and the
	44	+ * DMC620_PMU_COUNTERn offsets are within a counter group.
	45	+ *
	46	+ * The counter registers groups start at 0xA10.
	47	+ */
	48	+#define DMC620_PMU_OVERFLOW_STATUS_CLKDIV2 0x8
	49	+#define DMC620_PMU_OVERFLOW_STATUS_CLKDIV2_MASK \
	50	+ (DMC620_PMU_CLKDIV2_MAX_COUNTERS - 1)
	51	+#define DMC620_PMU_OVERFLOW_STATUS_CLK 0xC
	52	+#define DMC620_PMU_OVERFLOW_STATUS_CLK_MASK \
	53	+ (DMC620_PMU_CLK_MAX_COUNTERS - 1)
	54	+#define DMC620_PMU_COUNTERS_BASE 0x10
	55	+#define DMC620_PMU_COUNTERn_MASK_31_00 0x0
	56	+#define DMC620_PMU_COUNTERn_MASK_63_32 0x4
	57	+#define DMC620_PMU_COUNTERn_MATCH_31_00 0x8
	58	+#define DMC620_PMU_COUNTERn_MATCH_63_32 0xC
	59	+#define DMC620_PMU_COUNTERn_CONTROL 0x10
	60	+#define DMC620_PMU_COUNTERn_CONTROL_ENABLE BIT(0)
	61	+#define DMC620_PMU_COUNTERn_CONTROL_INVERT BIT(1)
	62	+#define DMC620_PMU_COUNTERn_CONTROL_EVENT_MUX GENMASK(6, 2)
	63	+#define DMC620_PMU_COUNTERn_CONTROL_INCR_MUX GENMASK(8, 7)
	64	+#define DMC620_PMU_COUNTERn_VALUE 0x20
	65	+/* Offset of the registers for a given counter, relative to 0xA00 */
	66	+#define DMC620_PMU_COUNTERn_OFFSET(n) \
	67	+ (DMC620_PMU_COUNTERS_BASE + 0x28 * (n))
	68	+
	69	+static LIST_HEAD(dmc620_pmu_irqs);
	70	+static DEFINE_MUTEX(dmc620_pmu_irqs_lock);
	71	+
	72	+struct dmc620_pmu_irq {
	73	+ struct hlist_node node;
	74	+ struct list_head pmus_node;
	75	+ struct list_head irqs_node;
	76	+ refcount_t refcount;
	77	+ unsigned int irq_num;
	78	+ unsigned int cpu;
	79	+};
	80	+
	81	+struct dmc620_pmu {
	82	+ struct pmu pmu;
	83	+
	84	+ void __iomem *base;
	85	+ struct dmc620_pmu_irq *irq;
	86	+ struct list_head pmus_node;
	87	+
	88	+ /*
	89	+ * We put all clkdiv2 and clk counters to a same array.
	90	+ * The first DMC620_PMU_CLKDIV2_MAX_COUNTERS bits belong to
	91	+ * clkdiv2 counters, the last DMC620_PMU_CLK_MAX_COUNTERS
	92	+ * belong to clk counters.
	93	+ */
	94	+ DECLARE_BITMAP(used_mask, DMC620_PMU_MAX_COUNTERS);
	95	+ struct perf_event *events[DMC620_PMU_MAX_COUNTERS];
	96	+};
	97	+
	98	+#define to_dmc620_pmu(p) (container_of(p, struct dmc620_pmu, pmu))
	99	+
	100	+static int cpuhp_state_num;
	101	+
	102	+struct dmc620_pmu_event_attr {
	103	+ struct device_attribute attr;
	104	+ u8 clkdiv2;
	105	+ u8 eventid;
	106	+};
	107	+
	108	+static ssize_t
	109	+dmc620_pmu_event_show(struct device *dev,
	110	+ struct device_attribute attr, char page)
	111	+{
	112	+ struct dmc620_pmu_event_attr *eattr;
	113	+
	114	+ eattr = container_of(attr, typeof(*eattr), attr);
	115	+
	116	+ return sprintf(page, "event=0x%x,clkdiv2=0x%x\n", eattr->eventid, eattr->clkdiv2);
	117	+}
	118	+
	119	+#define DMC620_PMU_EVENT_ATTR(_name, _eventid, _clkdiv2) \
	120	+ (&((struct dmc620_pmu_event_attr[]) {{ \
	121	+ .attr = __ATTR(_name, 0444, dmc620_pmu_event_show, NULL), \
	122	+ .clkdiv2 = _clkdiv2, \
	123	+ .eventid = _eventid, \
	124	+ }})[0].attr.attr)
	125	+
	126	+static struct attribute *dmc620_pmu_events_attrs[] = {
	127	+ /* clkdiv2 events list */
	128	+ DMC620_PMU_EVENT_ATTR(clkdiv2_cycle_count, 0x0, 1),
	129	+ DMC620_PMU_EVENT_ATTR(clkdiv2_allocate, 0x1, 1),
	130	+ DMC620_PMU_EVENT_ATTR(clkdiv2_queue_depth, 0x2, 1),
	131	+ DMC620_PMU_EVENT_ATTR(clkdiv2_waiting_for_wr_data, 0x3, 1),
	132	+ DMC620_PMU_EVENT_ATTR(clkdiv2_read_backlog, 0x4, 1),
	133	+ DMC620_PMU_EVENT_ATTR(clkdiv2_waiting_for_mi, 0x5, 1),
	134	+ DMC620_PMU_EVENT_ATTR(clkdiv2_hazard_resolution, 0x6, 1),
	135	+ DMC620_PMU_EVENT_ATTR(clkdiv2_enqueue, 0x7, 1),
	136	+ DMC620_PMU_EVENT_ATTR(clkdiv2_arbitrate, 0x8, 1),
	137	+ DMC620_PMU_EVENT_ATTR(clkdiv2_lrank_turnaround_activate, 0x9, 1),
	138	+ DMC620_PMU_EVENT_ATTR(clkdiv2_prank_turnaround_activate, 0xa, 1),
	139	+ DMC620_PMU_EVENT_ATTR(clkdiv2_read_depth, 0xb, 1),
	140	+ DMC620_PMU_EVENT_ATTR(clkdiv2_write_depth, 0xc, 1),
	141	+ DMC620_PMU_EVENT_ATTR(clkdiv2_highigh_qos_depth, 0xd, 1),
	142	+ DMC620_PMU_EVENT_ATTR(clkdiv2_high_qos_depth, 0xe, 1),
	143	+ DMC620_PMU_EVENT_ATTR(clkdiv2_medium_qos_depth, 0xf, 1),
	144	+ DMC620_PMU_EVENT_ATTR(clkdiv2_low_qos_depth, 0x10, 1),
	145	+ DMC620_PMU_EVENT_ATTR(clkdiv2_activate, 0x11, 1),
	146	+ DMC620_PMU_EVENT_ATTR(clkdiv2_rdwr, 0x12, 1),
	147	+ DMC620_PMU_EVENT_ATTR(clkdiv2_refresh, 0x13, 1),
	148	+ DMC620_PMU_EVENT_ATTR(clkdiv2_training_request, 0x14, 1),
	149	+ DMC620_PMU_EVENT_ATTR(clkdiv2_t_mac_tracker, 0x15, 1),
	150	+ DMC620_PMU_EVENT_ATTR(clkdiv2_bk_fsm_tracker, 0x16, 1),
	151	+ DMC620_PMU_EVENT_ATTR(clkdiv2_bk_open_tracker, 0x17, 1),
	152	+ DMC620_PMU_EVENT_ATTR(clkdiv2_ranks_in_pwr_down, 0x18, 1),
	153	+ DMC620_PMU_EVENT_ATTR(clkdiv2_ranks_in_sref, 0x19, 1),
	154	+
	155	+ /* clk events list */
	156	+ DMC620_PMU_EVENT_ATTR(clk_cycle_count, 0x0, 0),
	157	+ DMC620_PMU_EVENT_ATTR(clk_request, 0x1, 0),
	158	+ DMC620_PMU_EVENT_ATTR(clk_upload_stall, 0x2, 0),
	159	+ NULL,
	160	+};
	161	+
	162	+static struct attribute_group dmc620_pmu_events_attr_group = {
	163	+ .name = "events",
	164	+ .attrs = dmc620_pmu_events_attrs,
	165	+};
	166	+
	167	+/* User ABI */
	168	+#define ATTR_CFG_FLD_mask_CFG config
	169	+#define ATTR_CFG_FLD_mask_LO 0
	170	+#define ATTR_CFG_FLD_mask_HI 44
	171	+#define ATTR_CFG_FLD_match_CFG config1
	172	+#define ATTR_CFG_FLD_match_LO 0
	173	+#define ATTR_CFG_FLD_match_HI 44
	174	+#define ATTR_CFG_FLD_invert_CFG config2
	175	+#define ATTR_CFG_FLD_invert_LO 0
	176	+#define ATTR_CFG_FLD_invert_HI 0
	177	+#define ATTR_CFG_FLD_incr_CFG config2
	178	+#define ATTR_CFG_FLD_incr_LO 1
	179	+#define ATTR_CFG_FLD_incr_HI 2
	180	+#define ATTR_CFG_FLD_event_CFG config2
	181	+#define ATTR_CFG_FLD_event_LO 3
	182	+#define ATTR_CFG_FLD_event_HI 8
	183	+#define ATTR_CFG_FLD_clkdiv2_CFG config2
	184	+#define ATTR_CFG_FLD_clkdiv2_LO 9
	185	+#define ATTR_CFG_FLD_clkdiv2_HI 9
	186	+
	187	+#define __GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \
	188	+ (lo) == (hi) ? #cfg ":" #lo "\n" : #cfg ":" #lo "-" #hi
	189	+
	190	+#define _GEN_PMU_FORMAT_ATTR(cfg, lo, hi) \
	191	+ __GEN_PMU_FORMAT_ATTR(cfg, lo, hi)
	192	+
	193	+#define GEN_PMU_FORMAT_ATTR(name) \
	194	+ PMU_FORMAT_ATTR(name, \
	195	+ _GEN_PMU_FORMAT_ATTR(ATTR_CFG_FLD_##name##_CFG, \
	196	+ ATTR_CFG_FLD_##name##_LO, \
	197	+ ATTR_CFG_FLD_##name##_HI))
	198	+
	199	+#define _ATTR_CFG_GET_FLD(attr, cfg, lo, hi) \
	200	+ ((((attr)->cfg) >> lo) & GENMASK_ULL(hi - lo, 0))
	201	+
	202	+#define ATTR_CFG_GET_FLD(attr, name) \
	203	+ _ATTR_CFG_GET_FLD(attr, \
	204	+ ATTR_CFG_FLD_##name##_CFG, \
	205	+ ATTR_CFG_FLD_##name##_LO, \
	206	+ ATTR_CFG_FLD_##name##_HI)
	207	+
	208	+GEN_PMU_FORMAT_ATTR(mask);
	209	+GEN_PMU_FORMAT_ATTR(match);
	210	+GEN_PMU_FORMAT_ATTR(invert);
	211	+GEN_PMU_FORMAT_ATTR(incr);
	212	+GEN_PMU_FORMAT_ATTR(event);
	213	+GEN_PMU_FORMAT_ATTR(clkdiv2);
	214	+
	215	+static struct attribute *dmc620_pmu_formats_attrs[] = {
	216	+ &format_attr_mask.attr,
	217	+ &format_attr_match.attr,
	218	+ &format_attr_invert.attr,
	219	+ &format_attr_incr.attr,
	220	+ &format_attr_event.attr,
	221	+ &format_attr_clkdiv2.attr,
	222	+ NULL,
	223	+};
	224	+
	225	+static struct attribute_group dmc620_pmu_format_attr_group = {
	226	+ .name = "format",
	227	+ .attrs = dmc620_pmu_formats_attrs,
	228	+};
	229	+
	230	+static const struct attribute_group *dmc620_pmu_attr_groups[] = {
	231	+ &dmc620_pmu_events_attr_group,
	232	+ &dmc620_pmu_format_attr_group,
	233	+ NULL,
	234	+};
	235	+
	236	+static inline
	237	+u32 dmc620_pmu_creg_read(struct dmc620_pmu *dmc620_pmu,
	238	+ unsigned int idx, unsigned int reg)
	239	+{
	240	+ return readl(dmc620_pmu->base + DMC620_PMU_COUNTERn_OFFSET(idx) + reg);
	241	+}
	242	+
	243	+static inline
	244	+void dmc620_pmu_creg_write(struct dmc620_pmu *dmc620_pmu,
	245	+ unsigned int idx, unsigned int reg, u32 val)
	246	+{
	247	+ writel(val, dmc620_pmu->base + DMC620_PMU_COUNTERn_OFFSET(idx) + reg);
	248	+}
	249	+
	250	+static
	251	+unsigned int dmc620_event_to_counter_control(struct perf_event *event)
	252	+{
	253	+ struct perf_event_attr *attr = &event->attr;
	254	+ unsigned int reg = 0;
	255	+
	256	+ reg \|= FIELD_PREP(DMC620_PMU_COUNTERn_CONTROL_INVERT,
	257	+ ATTR_CFG_GET_FLD(attr, invert));
	258	+ reg \|= FIELD_PREP(DMC620_PMU_COUNTERn_CONTROL_EVENT_MUX,
	259	+ ATTR_CFG_GET_FLD(attr, event));
	260	+ reg \|= FIELD_PREP(DMC620_PMU_COUNTERn_CONTROL_INCR_MUX,
	261	+ ATTR_CFG_GET_FLD(attr, incr));
	262	+
	263	+ return reg;
	264	+}
	265	+
	266	+static int dmc620_get_event_idx(struct perf_event *event)
	267	+{
	268	+ struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
	269	+ int idx, start_idx, end_idx;
	270	+
	271	+ if (ATTR_CFG_GET_FLD(&event->attr, clkdiv2)) {
	272	+ start_idx = 0;
	273	+ end_idx = DMC620_PMU_CLKDIV2_MAX_COUNTERS;
	274	+ } else {
	275	+ start_idx = DMC620_PMU_CLKDIV2_MAX_COUNTERS;
	276	+ end_idx = DMC620_PMU_MAX_COUNTERS;
	277	+ }
	278	+
	279	+ for (idx = start_idx; idx < end_idx; ++idx) {
	280	+ if (!test_and_set_bit(idx, dmc620_pmu->used_mask))
	281	+ return idx;
	282	+ }
	283	+
	284	+ /* The counters are all in use. */
	285	+ return -EAGAIN;
	286	+}
	287	+
	288	+static inline
	289	+u64 dmc620_pmu_read_counter(struct perf_event *event)
	290	+{
	291	+ struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
	292	+
	293	+ return dmc620_pmu_creg_read(dmc620_pmu,
	294	+ event->hw.idx, DMC620_PMU_COUNTERn_VALUE);
	295	+}
	296	+
	297	+static void dmc620_pmu_event_update(struct perf_event *event)
	298	+{
	299	+ struct hw_perf_event *hwc = &event->hw;
	300	+ u64 delta, prev_count, new_count;
	301	+
	302	+ do {
	303	+ /* We may also be called from the irq handler */
	304	+ prev_count = local64_read(&hwc->prev_count);
	305	+ new_count = dmc620_pmu_read_counter(event);
	306	+ } while (local64_cmpxchg(&hwc->prev_count,
	307	+ prev_count, new_count) != prev_count);
	308	+ delta = (new_count - prev_count) & DMC620_CNT_MAX_PERIOD;
	309	+ local64_add(delta, &event->count);
	310	+}
	311	+
	312	+static void dmc620_pmu_event_set_period(struct perf_event *event)
	313	+{
	314	+ struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
	315	+
	316	+ local64_set(&event->hw.prev_count, DMC620_CNT_INIT);
	317	+ dmc620_pmu_creg_write(dmc620_pmu,
	318	+ event->hw.idx, DMC620_PMU_COUNTERn_VALUE, DMC620_CNT_INIT);
	319	+}
	320	+
	321	+static void dmc620_pmu_enable_counter(struct perf_event *event)
	322	+{
	323	+ struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
	324	+ u32 reg;
	325	+
	326	+ reg = dmc620_event_to_counter_control(event) \| DMC620_PMU_COUNTERn_CONTROL_ENABLE;
	327	+ dmc620_pmu_creg_write(dmc620_pmu,
	328	+ event->hw.idx, DMC620_PMU_COUNTERn_CONTROL, reg);
	329	+}
	330	+
	331	+static void dmc620_pmu_disable_counter(struct perf_event *event)
	332	+{
	333	+ struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
	334	+
	335	+ dmc620_pmu_creg_write(dmc620_pmu,
	336	+ event->hw.idx, DMC620_PMU_COUNTERn_CONTROL, 0);
	337	+}
	338	+
	339	+static irqreturn_t dmc620_pmu_handle_irq(int irq_num, void *data)
	340	+{
	341	+ struct dmc620_pmu_irq *irq = data;
	342	+ struct dmc620_pmu *dmc620_pmu;
	343	+ irqreturn_t ret = IRQ_NONE;
	344	+
	345	+ rcu_read_lock();
	346	+ list_for_each_entry_rcu(dmc620_pmu, &irq->pmus_node, pmus_node) {
	347	+ unsigned long status;
	348	+ struct perf_event *event;
	349	+ unsigned int idx;
	350	+
	351	+ /*
	352	+ * HW doesn't provide a control to atomically disable all counters.
	353	+ * To prevent race condition (overflow happens while clearing status register),
	354	+ * disable all events before continuing
	355	+ */
	356	+ for (idx = 0; idx < DMC620_PMU_MAX_COUNTERS; idx++) {
	357	+ event = dmc620_pmu->events[idx];
	358	+ if (!event)
	359	+ continue;
	360	+ dmc620_pmu_disable_counter(event);
	361	+ }
	362	+
	363	+ status = readl(dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2);
	364	+ status \|= (readl(dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK) <<
	365	+ DMC620_PMU_CLKDIV2_MAX_COUNTERS);
	366	+ if (status) {
	367	+ for_each_set_bit(idx, &status,
	368	+ DMC620_PMU_MAX_COUNTERS) {
	369	+ event = dmc620_pmu->events[idx];
	370	+ if (WARN_ON_ONCE(!event))
	371	+ continue;
	372	+ dmc620_pmu_event_update(event);
	373	+ dmc620_pmu_event_set_period(event);
	374	+ }
	375	+
	376	+ if (status & DMC620_PMU_OVERFLOW_STATUS_CLKDIV2_MASK)
	377	+ writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2);
	378	+
	379	+ if ((status >> DMC620_PMU_CLKDIV2_MAX_COUNTERS) &
	380	+ DMC620_PMU_OVERFLOW_STATUS_CLK_MASK)
	381	+ writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK);
	382	+ }
	383	+
	384	+ for (idx = 0; idx < DMC620_PMU_MAX_COUNTERS; idx++) {
	385	+ event = dmc620_pmu->events[idx];
	386	+ if (!event)
	387	+ continue;
	388	+ if (!(event->hw.state & PERF_HES_STOPPED))
	389	+ dmc620_pmu_enable_counter(event);
	390	+ }
	391	+
	392	+ ret = IRQ_HANDLED;
	393	+ }
	394	+ rcu_read_unlock();
	395	+
	396	+ return ret;
	397	+}
	398	+
	399	+static struct dmc620_pmu_irq *__dmc620_pmu_get_irq(int irq_num)
	400	+{
	401	+ struct dmc620_pmu_irq *irq;
	402	+ int ret;
	403	+
	404	+ list_for_each_entry(irq, &dmc620_pmu_irqs, irqs_node)
	405	+ if (irq->irq_num == irq_num && refcount_inc_not_zero(&irq->refcount))
	406	+ return irq;
	407	+
	408	+ irq = kzalloc(sizeof(*irq), GFP_KERNEL);
	409	+ if (!irq)
	410	+ return ERR_PTR(-ENOMEM);
	411	+
	412	+ INIT_LIST_HEAD(&irq->pmus_node);
	413	+
	414	+ /* Pick one CPU to be the preferred one to use */
	415	+ irq->cpu = raw_smp_processor_id();
	416	+ refcount_set(&irq->refcount, 1);
	417	+
	418	+ ret = request_irq(irq_num, dmc620_pmu_handle_irq,
	419	+ IRQF_NOBALANCING \| IRQF_NO_THREAD,
	420	+ "dmc620-pmu", irq);
	421	+ if (ret)
	422	+ goto out_free_aff;
	423	+
	424	+ ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu));
	425	+ if (ret)
	426	+ goto out_free_irq;
	427	+
	428	+ ret = cpuhp_state_add_instance_nocalls(cpuhp_state_num, &irq->node);
	429	+ if (ret)
	430	+ goto out_free_irq;
	431	+
	432	+ irq->irq_num = irq_num;
	433	+ list_add(&irq->irqs_node, &dmc620_pmu_irqs);
	434	+
	435	+ return irq;
	436	+
	437	+out_free_irq:
	438	+ free_irq(irq_num, irq);
	439	+out_free_aff:
	440	+ kfree(irq);
	441	+ return ERR_PTR(ret);
	442	+}
	443	+
	444	+static int dmc620_pmu_get_irq(struct dmc620_pmu *dmc620_pmu, int irq_num)
	445	+{
	446	+ struct dmc620_pmu_irq *irq;
	447	+
	448	+ mutex_lock(&dmc620_pmu_irqs_lock);
	449	+ irq = __dmc620_pmu_get_irq(irq_num);
	450	+ mutex_unlock(&dmc620_pmu_irqs_lock);
	451	+
	452	+ if (IS_ERR(irq))
	453	+ return PTR_ERR(irq);
	454	+
	455	+ dmc620_pmu->irq = irq;
	456	+ mutex_lock(&dmc620_pmu_irqs_lock);
	457	+ list_add_rcu(&dmc620_pmu->pmus_node, &irq->pmus_node);
	458	+ mutex_unlock(&dmc620_pmu_irqs_lock);
	459	+
	460	+ return 0;
	461	+}
	462	+
	463	+static void dmc620_pmu_put_irq(struct dmc620_pmu *dmc620_pmu)
	464	+{
	465	+ struct dmc620_pmu_irq *irq = dmc620_pmu->irq;
	466	+
	467	+ mutex_lock(&dmc620_pmu_irqs_lock);
	468	+ list_del_rcu(&dmc620_pmu->pmus_node);
	469	+
	470	+ if (!refcount_dec_and_test(&irq->refcount)) {
	471	+ mutex_unlock(&dmc620_pmu_irqs_lock);
	472	+ return;
	473	+ }
	474	+
	475	+ list_del(&irq->irqs_node);
	476	+ mutex_unlock(&dmc620_pmu_irqs_lock);
	477	+
	478	+ WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL));
	479	+ free_irq(irq->irq_num, irq);
	480	+ cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &irq->node);
	481	+ kfree(irq);
	482	+}
	483	+
	484	+static int dmc620_pmu_event_init(struct perf_event *event)
	485	+{
	486	+ struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
	487	+ struct hw_perf_event *hwc = &event->hw;
	488	+ struct perf_event *sibling;
	489	+
	490	+ if (event->attr.type != event->pmu->type)
	491	+ return -ENOENT;
	492	+
	493	+ /*
	494	+ * DMC 620 PMUs are shared across all cpus and cannot
	495	+ * support task bound and sampling events.
	496	+ */
	497	+ if (is_sampling_event(event) \|\|
	498	+ event->attach_state & PERF_ATTACH_TASK) {
	499	+ dev_dbg(dmc620_pmu->pmu.dev,
	500	+ "Can't support per-task counters\n");
	501	+ return -EOPNOTSUPP;
	502	+ }
	503	+
	504	+ /*
	505	+ * Many perf core operations (eg. events rotation) operate on a
	506	+ * single CPU context. This is obvious for CPU PMUs, where one
	507	+ * expects the same sets of events being observed on all CPUs,
	508	+ * but can lead to issues for off-core PMUs, where each
	509	+ * event could be theoretically assigned to a different CPU. To
	510	+ * mitigate this, we enforce CPU assignment to one, selected
	511	+ * processor.
	512	+ */
	513	+ event->cpu = dmc620_pmu->irq->cpu;
	514	+ if (event->cpu < 0)
	515	+ return -EINVAL;
	516	+
	517	+ /*
	518	+ * We can't atomically disable all HW counters so only one event allowed,
	519	+ * although software events are acceptable.
	520	+ */
	521	+ if (event->group_leader != event &&
	522	+ !is_software_event(event->group_leader))
	523	+ return -EINVAL;
	524	+
	525	+ for_each_sibling_event(sibling, event->group_leader) {
	526	+ if (sibling != event &&
	527	+ !is_software_event(sibling))
	528	+ return -EINVAL;
	529	+ }
	530	+
	531	+ hwc->idx = -1;
	532	+ return 0;
	533	+}
	534	+
	535	+static void dmc620_pmu_read(struct perf_event *event)
	536	+{
	537	+ dmc620_pmu_event_update(event);
	538	+}
	539	+
	540	+static void dmc620_pmu_start(struct perf_event *event, int flags)
	541	+{
	542	+ event->hw.state = 0;
	543	+ dmc620_pmu_event_set_period(event);
	544	+ dmc620_pmu_enable_counter(event);
	545	+}
	546	+
	547	+static void dmc620_pmu_stop(struct perf_event *event, int flags)
	548	+{
	549	+ if (event->hw.state & PERF_HES_STOPPED)
	550	+ return;
	551	+
	552	+ dmc620_pmu_disable_counter(event);
	553	+ dmc620_pmu_event_update(event);
	554	+ event->hw.state \|= PERF_HES_STOPPED \| PERF_HES_UPTODATE;
	555	+}
	556	+
	557	+static int dmc620_pmu_add(struct perf_event *event, int flags)
	558	+{
	559	+ struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
	560	+ struct perf_event_attr *attr = &event->attr;
	561	+ struct hw_perf_event *hwc = &event->hw;
	562	+ int idx;
	563	+ u64 reg;
	564	+
	565	+ idx = dmc620_get_event_idx(event);
	566	+ if (idx < 0)
	567	+ return idx;
	568	+
	569	+ hwc->idx = idx;
	570	+ dmc620_pmu->events[idx] = event;
	571	+ hwc->state = PERF_HES_STOPPED \| PERF_HES_UPTODATE;
	572	+
	573	+ reg = ATTR_CFG_GET_FLD(attr, mask);
	574	+ dmc620_pmu_creg_write(dmc620_pmu,
	575	+ idx, DMC620_PMU_COUNTERn_MASK_31_00, lower_32_bits(reg));
	576	+ dmc620_pmu_creg_write(dmc620_pmu,
	577	+ idx, DMC620_PMU_COUNTERn_MASK_63_32, upper_32_bits(reg));
	578	+
	579	+ reg = ATTR_CFG_GET_FLD(attr, match);
	580	+ dmc620_pmu_creg_write(dmc620_pmu,
	581	+ idx, DMC620_PMU_COUNTERn_MATCH_31_00, lower_32_bits(reg));
	582	+ dmc620_pmu_creg_write(dmc620_pmu,
	583	+ idx, DMC620_PMU_COUNTERn_MATCH_63_32, upper_32_bits(reg));
	584	+
	585	+ if (flags & PERF_EF_START)
	586	+ dmc620_pmu_start(event, PERF_EF_RELOAD);
	587	+
	588	+ perf_event_update_userpage(event);
	589	+ return 0;
	590	+}
	591	+
	592	+static void dmc620_pmu_del(struct perf_event *event, int flags)
	593	+{
	594	+ struct dmc620_pmu *dmc620_pmu = to_dmc620_pmu(event->pmu);
	595	+ struct hw_perf_event *hwc = &event->hw;
	596	+ int idx = hwc->idx;
	597	+
	598	+ dmc620_pmu_stop(event, PERF_EF_UPDATE);
	599	+ dmc620_pmu->events[idx] = NULL;
	600	+ clear_bit(idx, dmc620_pmu->used_mask);
	601	+ perf_event_update_userpage(event);
	602	+}
	603	+
	604	+static int dmc620_pmu_cpu_teardown(unsigned int cpu,
	605	+ struct hlist_node *node)
	606	+{
	607	+ struct dmc620_pmu_irq *irq;
	608	+ struct dmc620_pmu *dmc620_pmu;
	609	+ unsigned int target;
	610	+
	611	+ irq = hlist_entry_safe(node, struct dmc620_pmu_irq, node);
	612	+ if (cpu != irq->cpu)
	613	+ return 0;
	614	+
	615	+ target = cpumask_any_but(cpu_online_mask, cpu);
	616	+ if (target >= nr_cpu_ids)
	617	+ return 0;
	618	+
	619	+ /* We're only reading, but this isn't the place to be involving RCU */
	620	+ mutex_lock(&dmc620_pmu_irqs_lock);
	621	+ list_for_each_entry(dmc620_pmu, &irq->pmus_node, pmus_node)
	622	+ perf_pmu_migrate_context(&dmc620_pmu->pmu, irq->cpu, target);
	623	+ mutex_unlock(&dmc620_pmu_irqs_lock);
	624	+
	625	+ WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target)));
	626	+ irq->cpu = target;
	627	+
	628	+ return 0;
	629	+}
	630	+
	631	+static int dmc620_pmu_device_probe(struct platform_device *pdev)
	632	+{
	633	+ struct dmc620_pmu *dmc620_pmu;
	634	+ struct resource *res;
	635	+ char *name;
	636	+ int irq_num;
	637	+ int i, ret;
	638	+
	639	+ dmc620_pmu = devm_kzalloc(&pdev->dev,
	640	+ sizeof(struct dmc620_pmu), GFP_KERNEL);
	641	+ if (!dmc620_pmu)
	642	+ return -ENOMEM;
	643	+
	644	+ platform_set_drvdata(pdev, dmc620_pmu);
	645	+
	646	+ dmc620_pmu->pmu = (struct pmu) {
	647	+ .module = THIS_MODULE,
	648	+ .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
	649	+ .task_ctx_nr = perf_invalid_context,
	650	+ .event_init = dmc620_pmu_event_init,
	651	+ .add = dmc620_pmu_add,
	652	+ .del = dmc620_pmu_del,
	653	+ .start = dmc620_pmu_start,
	654	+ .stop = dmc620_pmu_stop,
	655	+ .read = dmc620_pmu_read,
	656	+ .attr_groups = dmc620_pmu_attr_groups,
	657	+ };
	658	+
	659	+ res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	660	+ dmc620_pmu->base = devm_ioremap_resource(&pdev->dev, res);
	661	+ if (IS_ERR(dmc620_pmu->base))
	662	+ return PTR_ERR(dmc620_pmu->base);
	663	+
	664	+ /* Make sure device is reset before enabling interrupt */
	665	+ for (i = 0; i < DMC620_PMU_MAX_COUNTERS; i++)
	666	+ dmc620_pmu_creg_write(dmc620_pmu, i, DMC620_PMU_COUNTERn_CONTROL, 0);
	667	+ writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLKDIV2);
	668	+ writel(0, dmc620_pmu->base + DMC620_PMU_OVERFLOW_STATUS_CLK);
	669	+
	670	+ irq_num = platform_get_irq(pdev, 0);
	671	+ if (irq_num < 0)
	672	+ return irq_num;
	673	+
	674	+ ret = dmc620_pmu_get_irq(dmc620_pmu, irq_num);
	675	+ if (ret)
	676	+ return ret;
	677	+
	678	+ name = devm_kasprintf(&pdev->dev, GFP_KERNEL,
	679	+ "%s_%llx", DMC620_PMUNAME,
	680	+ (u64)(res->start >> DMC620_PA_SHIFT));
	681	+ if (!name) {
	682	+ dev_err(&pdev->dev,
	683	+ "Create name failed, PMU @%pa\n", &res->start);
	684	+ goto out_teardown_dev;
	685	+ }
	686	+
	687	+ ret = perf_pmu_register(&dmc620_pmu->pmu, name, -1);
	688	+ if (ret)
	689	+ goto out_teardown_dev;
	690	+
	691	+ return 0;
	692	+
	693	+out_teardown_dev:
	694	+ dmc620_pmu_put_irq(dmc620_pmu);
	695	+ synchronize_rcu();
	696	+ return ret;
	697	+}
	698	+
	699	+static int dmc620_pmu_device_remove(struct platform_device *pdev)
	700	+{
	701	+ struct dmc620_pmu *dmc620_pmu = platform_get_drvdata(pdev);
	702	+
	703	+ dmc620_pmu_put_irq(dmc620_pmu);
	704	+
	705	+ /* perf will synchronise RCU before devres can free dmc620_pmu */
	706	+ perf_pmu_unregister(&dmc620_pmu->pmu);
	707	+
	708	+ return 0;
	709	+}
	710	+
	711	+static const struct acpi_device_id dmc620_acpi_match[] = {
	712	+ { "ARMHD620", 0},
	713	+ {},
	714	+};
	715	+MODULE_DEVICE_TABLE(acpi, dmc620_acpi_match);
	716	+static struct platform_driver dmc620_pmu_driver = {
	717	+ .driver = {
	718	+ .name = DMC620_DRVNAME,
	719	+ .acpi_match_table = dmc620_acpi_match,
	720	+ },
	721	+ .probe = dmc620_pmu_device_probe,
	722	+ .remove = dmc620_pmu_device_remove,
	723	+};
	724	+
	725	+static int __init dmc620_pmu_init(void)
	726	+{
	727	+ cpuhp_state_num = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
	728	+ DMC620_DRVNAME,
	729	+ NULL,
	730	+ dmc620_pmu_cpu_teardown);
	731	+ if (cpuhp_state_num < 0)
	732	+ return cpuhp_state_num;
	733	+
	734	+ return platform_driver_register(&dmc620_pmu_driver);
	735	+}
	736	+
	737	+static void __exit dmc620_pmu_exit(void)
	738	+{
	739	+ platform_driver_unregister(&dmc620_pmu_driver);
	740	+ cpuhp_remove_multi_state(cpuhp_state_num);
	741	+}
	742	+
	743	+module_init(dmc620_pmu_init);
	744	+module_exit(dmc620_pmu_exit);
	745	+
	746	+MODULE_DESCRIPTION("Perf driver for the ARM DMC-620 memory controller");
	747	+MODULE_AUTHOR("Tuan Phan <tuanphan@os.amperecomputing.com");
	748	+MODULE_LICENSE("GPL v2");

drivers/perf/arm_dsu_pmu.c+0−3

@@ -716,9 +716,6 @@ static int dsu_pmu_device_probe(struct platform_device *pdev) · 715 unmodified lines
716	716	if (IS_ERR(dsu_pmu))
717	717	return PTR_ERR(dsu_pmu);
718	718
719		− if (IS_ERR_OR_NULL(fwnode))
720		− return -ENOENT;
721		−
722	719	if (is_of_node(fwnode))
723	720	rc = dsu_pmu_dt_get_cpus(&pdev->dev, &dsu_pmu->associated_cpus);
724	721	else if (is_acpi_device_node(fwnode))

drivers/perf/arm_pmu.c+5−0

@@ -726,6 +726,11 @@ static int armpmu_get_cpu_irq(struct arm_pmu *pmu, int cpu) · 725 unmodified lines
726	726	return per_cpu(hw_events->irq, cpu);
727	727	}
728	728
	729	+bool arm_pmu_irq_is_nmi(void)
	730	+{
	731	+ return has_nmi;
	732	+}
	733	+
729	734	/*
730	735	* PMU hardware loses all context when a CPU goes offline.
731	736	* When a CPU is hotplugged back in, since some hardware registers are

drivers/perf/arm_smmuv3_pmu.c+39−0

@@ -74,6 +74,7 @@ · 73 unmodified lines
74	74	#define SMMU_PMCG_CFGR_NCTR GENMASK(5, 0)
75	75	#define SMMU_PMCG_CR 0xE04
76	76	#define SMMU_PMCG_CR_ENABLE BIT(0)
	77	+#define SMMU_PMCG_IIDR 0xE08
77	78	#define SMMU_PMCG_CEID0 0xE20
78	79	#define SMMU_PMCG_CEID1 0xE28
79	80	#define SMMU_PMCG_IRQ_CTRL 0xE50
@@ -112,6 +113,7 @@ struct smmu_pmu { · 32 unmodified lines
112	113	void __iomem *reloc_base;
113	114	u64 counter_mask;
114	115	u32 options;
	116	+ u32 iidr;
115	117	bool global_filter;
116	118	};
117	119
@@ -552,6 +554,40 @@ static struct attribute_group smmu_pmu_events_group = { · 434 unmodified lines
552	554	.is_visible = smmu_pmu_event_is_visible,
553	555	};
554	556
	557	+static ssize_t smmu_pmu_identifier_attr_show(struct device *dev,
	558	+ struct device_attribute *attr,
	559	+ char *page)
	560	+{
	561	+ struct smmu_pmu *smmu_pmu = to_smmu_pmu(dev_get_drvdata(dev));
	562	+
	563	+ return snprintf(page, PAGE_SIZE, "0x%08x\n", smmu_pmu->iidr);
	564	+}
	565	+
	566	+static umode_t smmu_pmu_identifier_attr_visible(struct kobject *kobj,
	567	+ struct attribute *attr,
	568	+ int n)
	569	+{
	570	+ struct device *dev = kobj_to_dev(kobj);
	571	+ struct smmu_pmu *smmu_pmu = to_smmu_pmu(dev_get_drvdata(dev));
	572	+
	573	+ if (!smmu_pmu->iidr)
	574	+ return 0;
	575	+ return attr->mode;
	576	+}
	577	+
	578	+static struct device_attribute smmu_pmu_identifier_attr =
	579	+ __ATTR(identifier, 0444, smmu_pmu_identifier_attr_show, NULL);
	580	+
	581	+static struct attribute *smmu_pmu_identifier_attrs[] = {
	582	+ &smmu_pmu_identifier_attr.attr,
	583	+ NULL
	584	+};
	585	+
	586	+static struct attribute_group smmu_pmu_identifier_group = {
	587	+ .attrs = smmu_pmu_identifier_attrs,
	588	+ .is_visible = smmu_pmu_identifier_attr_visible,
	589	+};
	590	+
555	591	/* Formats */
556	592	PMU_FORMAT_ATTR(event, "config:0-15");
557	593	PMU_FORMAT_ATTR(filter_stream_id, "config1:0-31");
@@ -575,6 +611,7 @@ static const struct attribute_group *smmu_pmu_attr_grps[] = { · 17 unmodified lines
575	611	&smmu_pmu_cpumask_group,
576	612	&smmu_pmu_events_group,
577	613	&smmu_pmu_format_group,
	614	+ &smmu_pmu_identifier_group,
578	615	NULL
579	616	};
580	617
@@ -795,6 +832,8 @@ static int smmu_pmu_probe(struct platform_device *pdev) · 214 unmodified lines
795	832	return err;
796	833	}
797	834
	835	+ smmu_pmu->iidr = readl_relaxed(smmu_pmu->reg_base + SMMU_PMCG_IIDR);
	836	+
798	837	name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "smmuv3_pmcg_%llx",
799	838	(res_0->start) >> SMMU_PMCG_PA_SHIFT);
800	839	if (!name) {

drivers/perf/fsl_imx8_ddr_perf.c+109−26

@@ -50,6 +50,7 @@ static DEFINE_IDA(ddr_ida); · 49 unmodified lines
50	50
51	51	struct fsl_ddr_devtype_data {
52	52	unsigned int quirks; /* quirks needed for different DDR Perf core */
	53	+ const char identifier; / system PMU identifier for userspace */
53	54	};
54	55
55	56	static const struct fsl_ddr_devtype_data imx8_devtype_data;
@@ -58,13 +59,32 @@ static const struct fsl_ddr_devtype_data imx8m_devtype_data = { · 2 unmodified lines
58	59	.quirks = DDR_CAP_AXI_ID_FILTER,
59	60	};
60	61
	62	+static const struct fsl_ddr_devtype_data imx8mq_devtype_data = {
	63	+ .quirks = DDR_CAP_AXI_ID_FILTER,
	64	+ .identifier = "i.MX8MQ",
	65	+};
	66	+
	67	+static const struct fsl_ddr_devtype_data imx8mm_devtype_data = {
	68	+ .quirks = DDR_CAP_AXI_ID_FILTER,
	69	+ .identifier = "i.MX8MM",
	70	+};
	71	+
	72	+static const struct fsl_ddr_devtype_data imx8mn_devtype_data = {
	73	+ .quirks = DDR_CAP_AXI_ID_FILTER,
	74	+ .identifier = "i.MX8MN",
	75	+};
	76	+
61	77	static const struct fsl_ddr_devtype_data imx8mp_devtype_data = {
62	78	.quirks = DDR_CAP_AXI_ID_FILTER_ENHANCED,
	79	+ .identifier = "i.MX8MP",
63	80	};
64	81
65	82	static const struct of_device_id imx_ddr_pmu_dt_ids[] = {
66	83	{ .compatible = "fsl,imx8-ddr-pmu", .data = &imx8_devtype_data},
67	84	{ .compatible = "fsl,imx8m-ddr-pmu", .data = &imx8m_devtype_data},
	85	+ { .compatible = "fsl,imx8mq-ddr-pmu", .data = &imx8mq_devtype_data},
	86	+ { .compatible = "fsl,imx8mm-ddr-pmu", .data = &imx8mm_devtype_data},
	87	+ { .compatible = "fsl,imx8mn-ddr-pmu", .data = &imx8mn_devtype_data},
68	88	{ .compatible = "fsl,imx8mp-ddr-pmu", .data = &imx8mp_devtype_data},
69	89	{ /* sentinel */ }
70	90	};
@@ -84,6 +104,40 @@ struct ddr_pmu { · 13 unmodified lines
84	104	int id;
85	105	};
86	106
	107	+static ssize_t ddr_perf_identifier_show(struct device *dev,
	108	+ struct device_attribute *attr,
	109	+ char *page)
	110	+{
	111	+ struct ddr_pmu *pmu = dev_get_drvdata(dev);
	112	+
	113	+ return sprintf(page, "%s\n", pmu->devtype_data->identifier);
	114	+}
	115	+
	116	+static umode_t ddr_perf_identifier_attr_visible(struct kobject *kobj,
	117	+ struct attribute *attr,
	118	+ int n)
	119	+{
	120	+ struct device *dev = kobj_to_dev(kobj);
	121	+ struct ddr_pmu *pmu = dev_get_drvdata(dev);
	122	+
	123	+ if (!pmu->devtype_data->identifier)
	124	+ return 0;
	125	+ return attr->mode;
	126	+};
	127	+
	128	+static struct device_attribute ddr_perf_identifier_attr =
	129	+ __ATTR(identifier, 0444, ddr_perf_identifier_show, NULL);
	130	+
	131	+static struct attribute *ddr_perf_identifier_attrs[] = {
	132	+ &ddr_perf_identifier_attr.attr,
	133	+ NULL,
	134	+};
	135	+
	136	+static struct attribute_group ddr_perf_identifier_attr_group = {
	137	+ .attrs = ddr_perf_identifier_attrs,
	138	+ .is_visible = ddr_perf_identifier_attr_visible,
	139	+};
	140	+
87	141	enum ddr_perf_filter_capabilities {
88	142	PERF_CAP_AXI_ID_FILTER = 0,
89	143	PERF_CAP_AXI_ID_FILTER_ENHANCED,
@@ -237,6 +291,7 @@ static const struct attribute_group *attr_groups[] = { · 147 unmodified lines
237	291	&ddr_perf_format_attr_group,
238	292	&ddr_perf_cpumask_attr_group,
239	293	&ddr_perf_filter_cap_attr_group,
	294	+ &ddr_perf_identifier_attr_group,
240	295	NULL,
241	296	};
242	297
@@ -361,25 +416,6 @@ static int ddr_perf_event_init(struct perf_event *event) · 118 unmodified lines
361	416	return 0;
362	417	}
363	418
364		−
365		−static void ddr_perf_event_update(struct perf_event *event)
366		−{
367		− struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
368		− struct hw_perf_event *hwc = &event->hw;
369		− u64 delta, prev_raw_count, new_raw_count;
370		− int counter = hwc->idx;
371		−
372		− do {
373		− prev_raw_count = local64_read(&hwc->prev_count);
374		− new_raw_count = ddr_perf_read_counter(pmu, counter);
375		− } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
376		− new_raw_count) != prev_raw_count);
377		−
378		− delta = (new_raw_count - prev_raw_count) & 0xFFFFFFFF;
379		−
380		− local64_add(delta, &event->count);
381		−}
382		−
383	419	static void ddr_perf_counter_enable(struct ddr_pmu *pmu, int config,
384	420	int counter, bool enable)
385	421	{
@@ -404,6 +440,56 @@ static void ddr_perf_counter_enable(struct ddr_pmu *pmu, int config, · 18 unmodified lines
404	440	}
405	441	}
406	442
	443	+static bool ddr_perf_counter_overflow(struct ddr_pmu *pmu, int counter)
	444	+{
	445	+ int val;
	446	+
	447	+ val = readl_relaxed(pmu->base + counter * 4 + COUNTER_CNTL);
	448	+
	449	+ return val & CNTL_OVER;
	450	+}
	451	+
	452	+static void ddr_perf_counter_clear(struct ddr_pmu *pmu, int counter)
	453	+{
	454	+ u8 reg = counter * 4 + COUNTER_CNTL;
	455	+ int val;
	456	+
	457	+ val = readl_relaxed(pmu->base + reg);
	458	+ val &= ~CNTL_CLEAR;
	459	+ writel(val, pmu->base + reg);
	460	+
	461	+ val \|= CNTL_CLEAR;
	462	+ writel(val, pmu->base + reg);
	463	+}
	464	+
	465	+static void ddr_perf_event_update(struct perf_event *event)
	466	+{
	467	+ struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
	468	+ struct hw_perf_event *hwc = &event->hw;
	469	+ u64 new_raw_count;
	470	+ int counter = hwc->idx;
	471	+ int ret;
	472	+
	473	+ new_raw_count = ddr_perf_read_counter(pmu, counter);
	474	+ local64_add(new_raw_count, &event->count);
	475	+
	476	+ /*
	477	+ * For legacy SoCs: event counter continue counting when overflow,
	478	+ * no need to clear the counter.
	479	+ * For new SoCs: event counter stop counting when overflow, need
	480	+ * clear counter to let it count again.
	481	+ */
	482	+ if (counter != EVENT_CYCLES_COUNTER) {
	483	+ ret = ddr_perf_counter_overflow(pmu, counter);
	484	+ if (ret)
	485	+ dev_warn_ratelimited(pmu->dev, "events lost due to counter overflow (config 0x%llx)\n",
	486	+ event->attr.config);
	487	+ }
	488	+
	489	+ /* clear counter every time for both cycle counter and event counter */
	490	+ ddr_perf_counter_clear(pmu, counter);
	491	+}
	492	+
407	493	static void ddr_perf_event_start(struct perf_event *event, int flags)
408	494	{
409	495	struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
@@ -537,7 +623,7 @@ static irqreturn_t ddr_perf_irq_handler(int irq, void *p) · 127 unmodified lines
537	623	{
538	624	int i;
539	625	struct ddr_pmu pmu = (struct ddr_pmu ) p;
540		− struct perf_event event, cycle_event = NULL;
	626	+ struct perf_event *event;
541	627
542	628	/* all counter will stop if cycle counter disabled */
543	629	ddr_perf_counter_enable(pmu,
@@ -547,7 +633,9 @@ static irqreturn_t ddr_perf_irq_handler(int irq, void *p) · 3 unmodified lines
547	633	/*
548	634	* When the cycle counter overflows, all counters are stopped,
549	635	* and an IRQ is raised. If any other counter overflows, it
550		− * continues counting, and no IRQ is raised.
	636	+ * continues counting, and no IRQ is raised. But for new SoCs,
	637	+ * such as i.MX8MP, event counter would stop when overflow, so
	638	+ * we need use cycle counter to stop overflow of event counter.
551	639	*
552	640	* Cycles occur at least 4 times as often as other events, so we
553	641	* can update all events on a cycle counter overflow and not
@@ -562,17 +650,12 @@ static irqreturn_t ddr_perf_irq_handler(int irq, void *p) · 8 unmodified lines
562	650	event = pmu->events[i];
563	651
564	652	ddr_perf_event_update(event);
565		−
566		− if (event->hw.idx == EVENT_CYCLES_COUNTER)
567		− cycle_event = event;
568	653	}
569	654
570	655	ddr_perf_counter_enable(pmu,
571	656	EVENT_CYCLES_ID,
572	657	EVENT_CYCLES_COUNTER,
573	658	true);
574		− if (cycle_event)
575		− ddr_perf_event_update(cycle_event);
576	659
577	660	return IRQ_HANDLED;
578	661	}

drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c+16−0

@@ -33,6 +33,7 @@ · 32 unmodified lines
33	33	#define DDRC_INT_MASK 0x6c8
34	34	#define DDRC_INT_STATUS 0x6cc
35	35	#define DDRC_INT_CLEAR 0x6d0
	36	+#define DDRC_VERSION 0x710
36	37
37	38	/* DDRC has 8-counters */
38	39	#define DDRC_NR_COUNTERS 0x8
@@ -267,6 +268,8 @@ static int hisi_ddrc_pmu_init_data(struct platform_device *pdev, · 228 unmodified lines
267	268	return PTR_ERR(ddrc_pmu->base);
268	269	}
269	270
	271	+ ddrc_pmu->identifier = readl(ddrc_pmu->base + DDRC_VERSION);
	272	+
270	273	return 0;
271	274	}
272	275
@@ -308,10 +311,23 @@ static const struct attribute_group hisi_ddrc_pmu_cpumask_attr_group = { · 35 unmodified lines
308	311	.attrs = hisi_ddrc_pmu_cpumask_attrs,
309	312	};
310	313
	314	+static struct device_attribute hisi_ddrc_pmu_identifier_attr =
	315	+ __ATTR(identifier, 0444, hisi_uncore_pmu_identifier_attr_show, NULL);
	316	+
	317	+static struct attribute *hisi_ddrc_pmu_identifier_attrs[] = {
	318	+ &hisi_ddrc_pmu_identifier_attr.attr,
	319	+ NULL
	320	+};
	321	+
	322	+static struct attribute_group hisi_ddrc_pmu_identifier_group = {
	323	+ .attrs = hisi_ddrc_pmu_identifier_attrs,
	324	+};
	325	+
311	326	static const struct attribute_group *hisi_ddrc_pmu_attr_groups[] = {
312	327	&hisi_ddrc_pmu_format_group,
313	328	&hisi_ddrc_pmu_events_group,
314	329	&hisi_ddrc_pmu_cpumask_attr_group,
	330	+ &hisi_ddrc_pmu_identifier_group,
315	331	NULL,
316	332	};
317	333

drivers/perf/hisilicon/hisi_uncore_hha_pmu.c+16−0

@@ -23,6 +23,7 @@ · 22 unmodified lines
23	23	#define HHA_INT_MASK 0x0804
24	24	#define HHA_INT_STATUS 0x0808
25	25	#define HHA_INT_CLEAR 0x080C
	26	+#define HHA_VERSION 0x1cf0
26	27	#define HHA_PERF_CTRL 0x1E00
27	28	#define HHA_EVENT_CTRL 0x1E04
28	29	#define HHA_EVENT_TYPE0 0x1E80
@@ -261,6 +262,8 @@ static int hisi_hha_pmu_init_data(struct platform_device *pdev, · 232 unmodified lines
261	262	return PTR_ERR(hha_pmu->base);
262	263	}
263	264
	265	+ hha_pmu->identifier = readl(hha_pmu->base + HHA_VERSION);
	266	+
264	267	return 0;
265	268	}
266	269
@@ -320,10 +323,23 @@ static const struct attribute_group hisi_hha_pmu_cpumask_attr_group = { · 53 unmodified lines
320	323	.attrs = hisi_hha_pmu_cpumask_attrs,
321	324	};
322	325
	326	+static struct device_attribute hisi_hha_pmu_identifier_attr =
	327	+ __ATTR(identifier, 0444, hisi_uncore_pmu_identifier_attr_show, NULL);
	328	+
	329	+static struct attribute *hisi_hha_pmu_identifier_attrs[] = {
	330	+ &hisi_hha_pmu_identifier_attr.attr,
	331	+ NULL
	332	+};
	333	+
	334	+static struct attribute_group hisi_hha_pmu_identifier_group = {
	335	+ .attrs = hisi_hha_pmu_identifier_attrs,
	336	+};
	337	+
323	338	static const struct attribute_group *hisi_hha_pmu_attr_groups[] = {
324	339	&hisi_hha_pmu_format_group,
325	340	&hisi_hha_pmu_events_group,
326	341	&hisi_hha_pmu_cpumask_attr_group,
	342	+ &hisi_hha_pmu_identifier_group,
327	343	NULL,
328	344	};
329	345

drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c+16−0

@@ -25,6 +25,7 @@ · 24 unmodified lines
25	25	#define L3C_INT_STATUS 0x0808
26	26	#define L3C_INT_CLEAR 0x080c
27	27	#define L3C_EVENT_CTRL 0x1c00
	28	+#define L3C_VERSION 0x1cf0
28	29	#define L3C_EVENT_TYPE0 0x1d00
29	30	/*
30	31	* Each counter is 48-bits and [48:63] are reserved
@@ -264,6 +265,8 @@ static int hisi_l3c_pmu_init_data(struct platform_device *pdev, · 233 unmodified lines
264	265	return PTR_ERR(l3c_pmu->base);
265	266	}
266	267
	268	+ l3c_pmu->identifier = readl(l3c_pmu->base + L3C_VERSION);
	269	+
267	270	return 0;
268	271	}
269	272
@@ -310,10 +313,23 @@ static const struct attribute_group hisi_l3c_pmu_cpumask_attr_group = { · 40 unmodified lines
310	313	.attrs = hisi_l3c_pmu_cpumask_attrs,
311	314	};
312	315
	316	+static struct device_attribute hisi_l3c_pmu_identifier_attr =
	317	+ __ATTR(identifier, 0444, hisi_uncore_pmu_identifier_attr_show, NULL);
	318	+
	319	+static struct attribute *hisi_l3c_pmu_identifier_attrs[] = {
	320	+ &hisi_l3c_pmu_identifier_attr.attr,
	321	+ NULL
	322	+};
	323	+
	324	+static struct attribute_group hisi_l3c_pmu_identifier_group = {
	325	+ .attrs = hisi_l3c_pmu_identifier_attrs,
	326	+};
	327	+
313	328	static const struct attribute_group *hisi_l3c_pmu_attr_groups[] = {
314	329	&hisi_l3c_pmu_format_group,
315	330	&hisi_l3c_pmu_events_group,
316	331	&hisi_l3c_pmu_cpumask_attr_group,
	332	+ &hisi_l3c_pmu_identifier_group,
317	333	NULL,
318	334	};
319	335

drivers/perf/hisilicon/hisi_uncore_pmu.c+10−0

@@ -119,6 +119,16 @@ int hisi_uncore_pmu_get_event_idx(struct perf_event *event) · 118 unmodified lines
119	119	}
120	120	EXPORT_SYMBOL_GPL(hisi_uncore_pmu_get_event_idx);
121	121
	122	+ssize_t hisi_uncore_pmu_identifier_attr_show(struct device *dev,
	123	+ struct device_attribute *attr,
	124	+ char *page)
	125	+{
	126	+ struct hisi_pmu *hisi_pmu = to_hisi_pmu(dev_get_drvdata(dev));
	127	+
	128	+ return snprintf(page, PAGE_SIZE, "0x%08x\n", hisi_pmu->identifier);
	129	+}
	130	+EXPORT_SYMBOL_GPL(hisi_uncore_pmu_identifier_attr_show);
	131	+
122	132	static void hisi_uncore_pmu_clear_event_idx(struct hisi_pmu *hisi_pmu, int idx)
123	133	{
124	134	if (!hisi_uncore_pmu_counter_valid(hisi_pmu, idx)) {

drivers/perf/hisilicon/hisi_uncore_pmu.h+7−0

@@ -75,6 +75,7 @@ struct hisi_pmu { · 74 unmodified lines
75	75	int counter_bits;
76	76	/* check event code range */
77	77	int check_event;
	78	+ u32 identifier;
78	79	};
79	80
80	81	int hisi_uncore_pmu_counter_valid(struct hisi_pmu *hisi_pmu, int idx);
@@ -97,4 +98,10 @@ ssize_t hisi_cpumask_sysfs_show(struct device *dev, · 16 unmodified lines
97	98	struct device_attribute attr, char buf);
98	99	int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node);
99	100	int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node);
	101	+
	102	+ssize_t hisi_uncore_pmu_identifier_attr_show(struct device *dev,
	103	+ struct device_attribute *attr,
	104	+ char *page);
	105	+
	106	+
100	107	#endif /* __HISI_UNCORE_PMU_H__ */

include/linux/acpi_iort.h+4−0

@@ -38,6 +38,7 @@ void iort_dma_setup(struct device dev, u64 dma_addr, u64 *size); · 37 unmodified lines
38	38	const struct iommu_ops iort_iommu_configure_id(struct device dev,
39	39	const u32 *id_in);
40	40	int iort_iommu_msi_get_resv_regions(struct device dev, struct list_head head);
	41	+phys_addr_t acpi_iort_dma_get_max_cpu_address(void);
41	42	#else
42	43	static inline void acpi_iort_init(void) { }
43	44	static inline u32 iort_msi_map_id(struct device *dev, u32 id)
@@ -55,6 +56,9 @@ static inline const struct iommu_ops *iort_iommu_configure_id( · 11 unmodified lines
55	56	static inline
56	57	int iort_iommu_msi_get_resv_regions(struct device dev, struct list_head head)
57	58	{ return 0; }
	59	+
	60	+static inline phys_addr_t acpi_iort_dma_get_max_cpu_address(void)
	61	+{ return PHYS_ADDR_MAX; }
58	62	#endif
59	63
60	64	#endif /* __ACPI_IORT_H__ */

include/linux/mmzone.h+0−20

@@ -354,26 +354,6 @@ enum zone_type { · 353 unmodified lines
354	354	* DMA mask is assumed when ZONE_DMA32 is defined. Some 64-bit
355	355	* platforms may need both zones as they support peripherals with
356	356	* different DMA addressing limitations.
357		− *
358		− * Some examples:
359		− *
360		− * - i386 and x86_64 have a fixed 16M ZONE_DMA and ZONE_DMA32 for the
361		− * rest of the lower 4G.
362		− *
363		− * - arm only uses ZONE_DMA, the size, up to 4G, may vary depending on
364		− * the specific device.
365		− *
366		− * - arm64 has a fixed 1G ZONE_DMA and ZONE_DMA32 for the rest of the
367		− * lower 4G.
368		− *
369		− * - powerpc only uses ZONE_DMA, the size, up to 2G, may vary
370		− * depending on the specific device.
371		− *
372		− * - s390 uses ZONE_DMA fixed to the lower 2G.
373		− *
374		− * - ia64 and riscv only use ZONE_DMA32.
375		− *
376		− * - parisc uses neither.
377	357	*/
378	358	#ifdef CONFIG_ZONE_DMA
379	359	ZONE_DMA,

include/linux/of.h+7−0

@@ -558,6 +558,8 @@ int of_map_id(struct device_node *np, u32 id, · 557 unmodified lines
558	558	const char map_name, const char map_mask_name,
559	559	struct device_node *target, u32 id_out);
560	560
	561	+phys_addr_t of_dma_get_max_cpu_address(struct device_node *np);
	562	+
561	563	#else /* CONFIG_OF */
562	564
563	565	static inline void of_core_init(void)
@@ -995,6 +997,11 @@ static inline int of_map_id(struct device_node *np, u32 id, · 431 unmodified lines
995	997	return -EINVAL;
996	998	}
997	999
	1000	+static inline phys_addr_t of_dma_get_max_cpu_address(struct device_node *np)
	1001	+{
	1002	+ return PHYS_ADDR_MAX;
	1003	+}
	1004	+
998	1005	#define of_match_ptr(_ptr) NULL
999	1006	#define of_match_node(_matches, _node) NULL
1000	1007	#endif /* CONFIG_OF */

include/linux/perf/arm_pmu.h+2−0

@@ -163,6 +163,8 @@ int arm_pmu_acpi_probe(armpmu_init_fn init_fn); · 162 unmodified lines
163	163	static inline int arm_pmu_acpi_probe(armpmu_init_fn init_fn) { return 0; }
164	164	#endif
165	165
	166	+bool arm_pmu_irq_is_nmi(void);
	167	+
166	168	/* Internal functions only for core arm_pmu code */
167	169	struct arm_pmu *armpmu_alloc(void);
168	170	struct arm_pmu *armpmu_alloc_atomic(void);

include/linux/scs.h+6−10

@@ -15,24 +15,18 @@ · 14 unmodified lines
15	15
16	16	#ifdef CONFIG_SHADOW_CALL_STACK
17	17
18		−/*
19		− * In testing, 1 KiB shadow stack size (i.e. 128 stack frames on a 64-bit
20		− * architecture) provided ~40% safety margin on stack usage while keeping
21		− * memory allocation overhead reasonable.
22		− */
23		−#define SCS_SIZE SZ_1K
	18	+#define SCS_ORDER 0
	19	+#define SCS_SIZE (PAGE_SIZE << SCS_ORDER)
24	20	#define GFP_SCS (GFP_KERNEL \| __GFP_ZERO)
25	21
26	22	/* An illegal pointer value to mark the end of the shadow stack. */
27	23	#define SCS_END_MAGIC (0x5f6UL + POISON_POINTER_DELTA)
28	24
29		−/* Allocate a static per-CPU shadow stack */
30		−#define DEFINE_SCS(name) \
31		− DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], name) \
32		−
33	25	#define task_scs(tsk) (task_thread_info(tsk)->scs_base)
34	26	#define task_scs_sp(tsk) (task_thread_info(tsk)->scs_sp)
35	27
	28	+void *scs_alloc(int node);
	29	+void scs_free(void *s);
36	30	void scs_init(void);
37	31	int scs_prepare(struct task_struct *tsk, int node);
38	32	void scs_release(struct task_struct *tsk);
@@ -61,6 +55,8 @@ static inline bool task_scs_end_corrupted(struct task_struct *tsk) · 22 unmodified lines
61	55
62	56	#else /* CONFIG_SHADOW_CALL_STACK */
63	57
	58	+static inline void *scs_alloc(int node) { return NULL; }
	59	+static inline void scs_free(void *s) {}
64	60	static inline void scs_init(void) {}
65	61	static inline void scs_task_reset(struct task_struct *tsk) {}
66	62	static inline int scs_prepare(struct task_struct *tsk, int node) { return 0; }

include/linux/signal.h+14−0

@@ -469,4 +469,18 @@ struct seq_file; · 468 unmodified lines
469	469	extern void render_sigset_t(struct seq_file , const char , sigset_t *);
470	470	#endif
471	471
	472	+#ifndef arch_untagged_si_addr
	473	+/*
	474	+ * Given a fault address and a signal and si_code which correspond to the
	475	+ * _sigfault union member, returns the address that must appear in si_addr if
	476	+ * the signal handler does not have SA_EXPOSE_TAGBITS enabled in sa_flags.
	477	+ */
	478	+static inline void __user arch_untagged_si_addr(void __user addr,
	479	+ unsigned long sig,
	480	+ unsigned long si_code)
	481	+{
	482	+ return addr;
	483	+}
	484	+#endif
	485	+
472	486	#endif /* _LINUX_SIGNAL_H */

include/linux/signal_types.h+12−0

@@ -68,4 +68,16 @@ struct ksignal { · 67 unmodified lines
68	68	int sig;
69	69	};
70	70
	71	+#ifndef __ARCH_UAPI_SA_FLAGS
	72	+#ifdef SA_RESTORER
	73	+#define __ARCH_UAPI_SA_FLAGS SA_RESTORER
	74	+#else
	75	+#define __ARCH_UAPI_SA_FLAGS 0
	76	+#endif
	77	+#endif
	78	+
	79	+#define UAPI_SA_FLAGS \
	80	+ (SA_NOCLDSTOP \| SA_NOCLDWAIT \| SA_SIGINFO \| SA_ONSTACK \| SA_RESTART \| \
	81	+ SA_NODEFER \| SA_RESETHAND \| SA_EXPOSE_TAGBITS \| __ARCH_UAPI_SA_FLAGS)
	82	+
71	83	#endif /* _LINUX_SIGNAL_TYPES_H */

include/uapi/asm-generic/signal-defs.h+63−0

@@ -4,6 +4,69 @@ · 3 unmodified lines
4	4
5	5	#include <linux/compiler.h>
6	6
	7	+/*
	8	+ * SA_FLAGS values:
	9	+ *
	10	+ * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop.
	11	+ * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies.
	12	+ * SA_SIGINFO delivers the signal with SIGINFO structs.
	13	+ * SA_ONSTACK indicates that a registered stack_t will be used.
	14	+ * SA_RESTART flag to get restarting signals (which were the default long ago)
	15	+ * SA_NODEFER prevents the current signal from being masked in the handler.
	16	+ * SA_RESETHAND clears the handler when the signal is delivered.
	17	+ * SA_UNSUPPORTED is a flag bit that will never be supported. Kernels from
	18	+ * before the introduction of SA_UNSUPPORTED did not clear unknown bits from
	19	+ * sa_flags when read using the oldact argument to sigaction and rt_sigaction,
	20	+ * so this bit allows flag bit support to be detected from userspace while
	21	+ * allowing an old kernel to be distinguished from a kernel that supports every
	22	+ * flag bit.
	23	+ * SA_EXPOSE_TAGBITS exposes an architecture-defined set of tag bits in
	24	+ * siginfo.si_addr.
	25	+ *
	26	+ * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single
	27	+ * Unix names RESETHAND and NODEFER respectively.
	28	+ */
	29	+#ifndef SA_NOCLDSTOP
	30	+#define SA_NOCLDSTOP 0x00000001
	31	+#endif
	32	+#ifndef SA_NOCLDWAIT
	33	+#define SA_NOCLDWAIT 0x00000002
	34	+#endif
	35	+#ifndef SA_SIGINFO
	36	+#define SA_SIGINFO 0x00000004
	37	+#endif
	38	+/* 0x00000008 used on alpha, mips, parisc */
	39	+/* 0x00000010 used on alpha, parisc */
	40	+/* 0x00000020 used on alpha, parisc, sparc */
	41	+/* 0x00000040 used on alpha, parisc */
	42	+/* 0x00000080 used on parisc */
	43	+/* 0x00000100 used on sparc */
	44	+/* 0x00000200 used on sparc */
	45	+#define SA_UNSUPPORTED 0x00000400
	46	+#define SA_EXPOSE_TAGBITS 0x00000800
	47	+/* 0x00010000 used on mips */
	48	+/* 0x01000000 used on x86 */
	49	+/* 0x02000000 used on x86 */
	50	+/*
	51	+ * New architectures should not define the obsolete
	52	+ * SA_RESTORER 0x04000000
	53	+ */
	54	+#ifndef SA_ONSTACK
	55	+#define SA_ONSTACK 0x08000000
	56	+#endif
	57	+#ifndef SA_RESTART
	58	+#define SA_RESTART 0x10000000
	59	+#endif
	60	+#ifndef SA_NODEFER
	61	+#define SA_NODEFER 0x40000000
	62	+#endif
	63	+#ifndef SA_RESETHAND
	64	+#define SA_RESETHAND 0x80000000
	65	+#endif
	66	+
	67	+#define SA_NOMASK SA_NODEFER
	68	+#define SA_ONESHOT SA_RESETHAND
	69	+
7	70	#ifndef SIG_BLOCK
8	71	#define SIG_BLOCK 0 /* for blocking signals */
9	72	#endif

include/uapi/asm-generic/signal.h+0−29

@@ -52,35 +52,6 @@ · 51 unmodified lines
52	52	#define SIGRTMAX _NSIG
53	53	#endif
54	54
55		−/*
56		− * SA_FLAGS values:
57		− *
58		− * SA_ONSTACK indicates that a registered stack_t will be used.
59		− * SA_RESTART flag to get restarting signals (which were the default long ago)
60		− * SA_NOCLDSTOP flag to turn off SIGCHLD when children stop.
61		− * SA_RESETHAND clears the handler when the signal is delivered.
62		− * SA_NOCLDWAIT flag on SIGCHLD to inhibit zombies.
63		− * SA_NODEFER prevents the current signal from being masked in the handler.
64		− *
65		− * SA_ONESHOT and SA_NOMASK are the historical Linux names for the Single
66		− * Unix names RESETHAND and NODEFER respectively.
67		− */
68		−#define SA_NOCLDSTOP 0x00000001
69		−#define SA_NOCLDWAIT 0x00000002
70		−#define SA_SIGINFO 0x00000004
71		−#define SA_ONSTACK 0x08000000
72		−#define SA_RESTART 0x10000000
73		−#define SA_NODEFER 0x40000000
74		−#define SA_RESETHAND 0x80000000
75		−
76		−#define SA_NOMASK SA_NODEFER
77		−#define SA_ONESHOT SA_RESETHAND
78		−
79		−/*
80		− * New architectures should not define the obsolete
81		− * SA_RESTORER 0x04000000
82		− */
83		−
84	55	#if !defined MINSIGSTKSZ \|\| !defined SIGSTKSZ
85	56	#define MINSIGSTKSZ 2048
86	57	#define SIGSTKSZ 8192

kernel/scs.c+60−11

@@ -5,26 +5,49 @@ · 4 unmodified lines
5	5	* Copyright (C) 2019 Google LLC
6	6	*/
7	7
	8	+#include <linux/cpuhotplug.h>
8	9	#include <linux/kasan.h>
9	10	#include <linux/mm.h>
10	11	#include <linux/scs.h>
11		−#include <linux/slab.h>
	12	+#include <linux/vmalloc.h>
12	13	#include <linux/vmstat.h>
13	14
14		−static struct kmem_cache *scs_cache;
15		−
16	15	static void __scs_account(void *s, int account)
17	16	{
18		− struct page *scs_page = virt_to_page(s);
	17	+ struct page *scs_page = vmalloc_to_page(s);
19	18
20	19	mod_node_page_state(page_pgdat(scs_page), NR_KERNEL_SCS_KB,
21	20	account * (SCS_SIZE / SZ_1K));
22	21	}
23	22
24		−static void *scs_alloc(int node)
	23	+/* Matches NR_CACHED_STACKS for VMAP_STACK */
	24	+#define NR_CACHED_SCS 2
	25	+static DEFINE_PER_CPU(void *, scs_cache[NR_CACHED_SCS]);
	26	+
	27	+static void *__scs_alloc(int node)
25	28	{
26		− void *s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node);
	29	+ int i;
	30	+ void *s;
	31	+
	32	+ for (i = 0; i < NR_CACHED_SCS; i++) {
	33	+ s = this_cpu_xchg(scs_cache[i], NULL);
	34	+ if (s) {
	35	+ kasan_unpoison_vmalloc(s, SCS_SIZE);
	36	+ memset(s, 0, SCS_SIZE);
	37	+ return s;
	38	+ }
	39	+ }
	40	+
	41	+ return __vmalloc_node_range(SCS_SIZE, 1, VMALLOC_START, VMALLOC_END,
	42	+ GFP_SCS, PAGE_KERNEL, 0, node,
	43	+ __builtin_return_address(0));
	44	+}
27	45
	46	+void *scs_alloc(int node)
	47	+{
	48	+ void *s;
	49	+
	50	+ s = __scs_alloc(node);
28	51	if (!s)
29	52	return NULL;
30	53
@@ -34,21 +57,47 @@ static void *scs_alloc(int node) · 3 unmodified lines
34	57	* Poison the allocation to catch unintentional accesses to
35	58	* the shadow stack when KASAN is enabled.
36	59	*/
37		− kasan_poison_object_data(scs_cache, s);
	60	+ kasan_poison_vmalloc(s, SCS_SIZE);
38	61	__scs_account(s, 1);
39	62	return s;
40	63	}
41	64
42		−static void scs_free(void *s)
	65	+void scs_free(void *s)
43	66	{
	67	+ int i;
	68	+
44	69	__scs_account(s, -1);
45		− kasan_unpoison_object_data(scs_cache, s);
46		− kmem_cache_free(scs_cache, s);
	70	+
	71	+ /*
	72	+ * We cannot sleep as this can be called in interrupt context,
	73	+ * so use this_cpu_cmpxchg to update the cache, and vfree_atomic
	74	+ * to free the stack.
	75	+ */
	76	+
	77	+ for (i = 0; i < NR_CACHED_SCS; i++)
	78	+ if (this_cpu_cmpxchg(scs_cache[i], 0, s) == NULL)
	79	+ return;
	80	+
	81	+ vfree_atomic(s);
	82	+}
	83	+
	84	+static int scs_cleanup(unsigned int cpu)
	85	+{
	86	+ int i;
	87	+ void **cache = per_cpu_ptr(scs_cache, cpu);
	88	+
	89	+ for (i = 0; i < NR_CACHED_SCS; i++) {
	90	+ vfree(cache[i]);
	91	+ cache[i] = NULL;
	92	+ }
	93	+
	94	+ return 0;
47	95	}
48	96
49	97	void __init scs_init(void)
50	98	{
51		− scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL);
	99	+ cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "scs:scs_cache", NULL,
	100	+ scs_cleanup);
52	101	}
53	102
54	103	int scs_prepare(struct task_struct *tsk, int node)

kernel/signal.c+40−0

@@ -2524,6 +2524,26 @@ static int ptrace_signal(int signr, kernel_siginfo_t *info) · 2523 unmodified lines
2524	2524	return signr;
2525	2525	}
2526	2526
	2527	+static void hide_si_addr_tag_bits(struct ksignal *ksig)
	2528	+{
	2529	+ switch (siginfo_layout(ksig->sig, ksig->info.si_code)) {
	2530	+ case SIL_FAULT:
	2531	+ case SIL_FAULT_MCEERR:
	2532	+ case SIL_FAULT_BNDERR:
	2533	+ case SIL_FAULT_PKUERR:
	2534	+ ksig->info.si_addr = arch_untagged_si_addr(
	2535	+ ksig->info.si_addr, ksig->sig, ksig->info.si_code);
	2536	+ break;
	2537	+ case SIL_KILL:
	2538	+ case SIL_TIMER:
	2539	+ case SIL_POLL:
	2540	+ case SIL_CHLD:
	2541	+ case SIL_RT:
	2542	+ case SIL_SYS:
	2543	+ break;
	2544	+ }
	2545	+}
	2546	+
2527	2547	bool get_signal(struct ksignal *ksig)
2528	2548	{
2529	2549	struct sighand_struct *sighand = current->sighand;
@@ -2761,6 +2781,10 @@ relock: · 231 unmodified lines
2761	2781	spin_unlock_irq(&sighand->siglock);
2762	2782
2763	2783	ksig->sig = signr;
	2784	+
	2785	+ if (!(ksig->ka.sa.sa_flags & SA_EXPOSE_TAGBITS))
	2786	+ hide_si_addr_tag_bits(ksig);
	2787	+
2764	2788	return ksig->sig > 0;
2765	2789	}
2766	2790
@@ -3985,6 +4009,22 @@ int do_sigaction(int sig, struct k_sigaction act, struct k_sigaction oact) · 1218 unmodified lines
3985	4009	if (oact)
3986	4010	oact = k;
3987	4011
	4012	+ /*
	4013	+ * Make sure that we never accidentally claim to support SA_UNSUPPORTED,
	4014	+ * e.g. by having an architecture use the bit in their uapi.
	4015	+ */
	4016	+ BUILD_BUG_ON(UAPI_SA_FLAGS & SA_UNSUPPORTED);
	4017	+
	4018	+ /*
	4019	+ * Clear unknown flag bits in order to allow userspace to detect missing
	4020	+ * support for flag bits and to allow the kernel to use non-uapi bits
	4021	+ * internally.
	4022	+ */
	4023	+ if (act)
	4024	+ act->sa.sa_flags &= UAPI_SA_FLAGS;
	4025	+ if (oact)
	4026	+ oact->sa.sa_flags &= UAPI_SA_FLAGS;
	4027	+
3988	4028	sigaction_compat_abi(act, oact);
3989	4029
3990	4030	if (act) {