Merge tag 'riscv-for-linus-6.13-mw1' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux

Pull RISC-v updates from Palmer Dabbelt: - Support for pointer masking in userspace - Support for probing vector misaligned access performance - Support for qspinlock on systems with Zacas and Zabha * tag 'riscv-for-linus-6.13-mw1' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux: (38 commits) RISC-V: Remove unnecessary include from compat.h riscv: Fix default misaligned access trap riscv: Add qspinlock support dt-bindings: riscv: Add Ziccrse ISA extension description riscv: Add ISA extension parsing for Ziccrse asm-generic: ticket-lock: Add separate ticket-lock.h asm-generic: ticket-lock: Reuse arch_spinlock_t of qspinlock riscv: Implement xchg8/16() using Zabha riscv: Implement arch_cmpxchg128() using Zacas riscv: Improve zacas fully-ordered cmpxchg() riscv: Implement cmpxchg8/16() using Zabha dt-bindings: riscv: Add Zabha ISA extension description riscv: Implement cmpxchg32/64() using Zacas riscv: Do not fail to build on byte/halfword operations with Zawrs riscv: Move cpufeature.h macros into their own header KVM: riscv: selftests: Add Smnpm and Ssnpm to get-reg-list test RISC-V: KVM: Allow Smnpm and Ssnpm extensions for guests riscv: hwprobe: Export the Supm ISA extension riscv: selftests: Add a pointer masking test riscv: Allow ptrace control of the tagged address ABI ...

Linus Torvalds committed 1 year ago

Commit 91dbbe62 parents c946969 8d4f1e053 files · +1789 −282Browse files

Unified · Split

53 files

Documentation/arch/riscv/hwprobe.rst +19 −0
Documentation/arch/riscv/uabi.rst +16 −0
Documentation/devicetree/bindings/riscv/extensions.yaml +30 −0
Documentation/features/locking/queued-spinlocks/arch-support.txt +1 −1
arch/riscv/Kconfig +136 −2
arch/riscv/Makefile +6 −0
arch/riscv/configs/defconfig +1 −0
arch/riscv/include/asm/Kbuild +3 −1
arch/riscv/include/asm/cmpxchg.h +208 −78
arch/riscv/include/asm/compat.h +0 −1
arch/riscv/include/asm/cpufeature-macros.h +66 −0
arch/riscv/include/asm/cpufeature.h +14 −59
arch/riscv/include/asm/csr.h +16 −0
arch/riscv/include/asm/entry-common.h +1 −0
arch/riscv/include/asm/hwcap.h +7 −0
arch/riscv/include/asm/hwprobe.h +1 −1
arch/riscv/include/asm/mmu.h +7 −0
arch/riscv/include/asm/mmu_context.h +13 −0
arch/riscv/include/asm/processor.h +9 −0
arch/riscv/include/asm/spinlock.h +47 −0
arch/riscv/include/asm/switch_to.h +19 −0
arch/riscv/include/asm/uaccess.h +38 −5
arch/riscv/include/asm/vector.h +2 −0
arch/riscv/include/uapi/asm/hwprobe.h +6 −0
arch/riscv/include/uapi/asm/kvm.h +2 −0
arch/riscv/kernel/Makefile +2 −1
arch/riscv/kernel/copy-unaligned.h +5 −0
arch/riscv/kernel/cpufeature.c +13 −3
arch/riscv/kernel/fpu.S +2 −2
arch/riscv/kernel/process.c +154 −0
arch/riscv/kernel/ptrace.c +42 −0
arch/riscv/kernel/setup.c +37 −0
arch/riscv/kernel/smpboot.c +0 −2
arch/riscv/kernel/suspend.c +2 −2
arch/riscv/kernel/sys_hwprobe.c +44 −0
arch/riscv/kernel/traps_misaligned.c +128 −11
arch/riscv/kernel/unaligned_access_speed.c +148 −8
arch/riscv/kernel/vdso/Makefile +5 −4
arch/riscv/kernel/vec-copy-unaligned.S +58 −0
arch/riscv/kernel/vector.c +1 −1
arch/riscv/kvm/vcpu_onereg.c +4 −0
drivers/cpuidle/cpuidle-riscv-sbi.c +9 −2
include/asm-generic/qspinlock.h +2 −0
include/asm-generic/spinlock.h +1 −86
include/asm-generic/spinlock_types.h +2 −10
include/asm-generic/ticket_spinlock.h +105 −0
include/uapi/linux/elf.h +1 −0
include/uapi/linux/prctl.h +4 −1
tools/testing/selftests/kvm/riscv/get-reg-list.c +8 −0
tools/testing/selftests/riscv/Makefile +1 −1
tools/testing/selftests/riscv/abi/.gitignore +1 −0
tools/testing/selftests/riscv/abi/Makefile +10 −0
tools/testing/selftests/riscv/abi/pointer_masking.c +332 −0

Documentation/arch/riscv/hwprobe.rst+19−0

@@ -239,6 +239,9 @@ The following keys are defined: · 238 unmodified lines
239	239	ratified in commit 98918c844281 ("Merge pull request #1217 from
240	240	riscv/zawrs") of riscv-isa-manual.
241	241
	242	+ * :c:macro:`RISCV_HWPROBE_EXT_SUPM`: The Supm extension is supported as
	243	+ defined in version 1.0 of the RISC-V Pointer Masking extensions.
	244	+
242	245	* :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: Deprecated. Returns similar values to
243	246	:c:macro:`RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF`, but the key was
244	247	mistakenly classified as a bitmask rather than a value.
@@ -274,3 +277,19 @@ The following keys are defined: · 29 unmodified lines
274	277	represent the highest userspace virtual address usable.
275	278
276	279	* :c:macro:`RISCV_HWPROBE_KEY_TIME_CSR_FREQ`: Frequency (in Hz) of `time CSR`.
	280	+
	281	+* :c:macro:`RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF`: An enum value describing the
	282	+ performance of misaligned vector accesses on the selected set of processors.
	283	+
	284	+ * :c:macro:`RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN`: The performance of misaligned
	285	+ vector accesses is unknown.
	286	+
	287	+ * :c:macro:`RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW`: 32-bit misaligned accesses using vector
	288	+ registers are slower than the equivalent quantity of byte accesses via vector registers.
	289	+ Misaligned accesses may be supported directly in hardware, or trapped and emulated by software.
	290	+
	291	+ * :c:macro:`RISCV_HWPROBE_MISALIGNED_VECTOR_FAST`: 32-bit misaligned accesses using vector
	292	+ registers are faster than the equivalent quantity of byte accesses via vector registers.
	293	+
	294	+ * :c:macro:`RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED`: Misaligned vector accesses are
	295	+ not supported at all and will generate a misaligned address fault.

Documentation/arch/riscv/uabi.rst+16−0

@@ -68,3 +68,19 @@ Misaligned accesses · 67 unmodified lines
68	68	Misaligned scalar accesses are supported in userspace, but they may perform
69	69	poorly. Misaligned vector accesses are only supported if the Zicclsm extension
70	70	is supported.
	71	+
	72	+Pointer masking
	73	+---------------
	74	+
	75	+Support for pointer masking in userspace (the Supm extension) is provided via
	76	+the ``PR_SET_TAGGED_ADDR_CTRL`` and ``PR_GET_TAGGED_ADDR_CTRL`` ``prctl()``
	77	+operations. Pointer masking is disabled by default. To enable it, userspace
	78	+must call ``PR_SET_TAGGED_ADDR_CTRL`` with the ``PR_PMLEN`` field set to the
	79	+number of mask/tag bits needed by the application. ``PR_PMLEN`` is interpreted
	80	+as a lower bound; if the kernel is unable to satisfy the request, the
	81	+``PR_SET_TAGGED_ADDR_CTRL`` operation will fail. The actual number of tag bits
	82	+is returned in ``PR_PMLEN`` by the ``PR_GET_TAGGED_ADDR_CTRL`` operation.
	83	+
	84	+Additionally, when pointer masking is enabled (``PR_PMLEN`` is greater than 0),
	85	+a tagged address ABI is supported, with the same interface and behavior as
	86	+documented for AArch64 (Documentation/arch/arm64/tagged-address-abi.rst).

Documentation/devicetree/bindings/riscv/extensions.yaml+30−0

@@ -128,6 +128,18 @@ properties: · 127 unmodified lines
128	128	changes to interrupts as frozen at commit ccbddab ("Merge pull
129	129	request #42 from riscv/jhauser-2023-RC4") of riscv-aia.
130	130
	131	+ - const: smmpm
	132	+ description: \|
	133	+ The standard Smmpm extension for M-mode pointer masking as
	134	+ ratified at commit d70011dde6c2 ("Update to ratified state")
	135	+ of riscv-j-extension.
	136	+
	137	+ - const: smnpm
	138	+ description: \|
	139	+ The standard Smnpm extension for next-mode pointer masking as
	140	+ ratified at commit d70011dde6c2 ("Update to ratified state")
	141	+ of riscv-j-extension.
	142	+
131	143	- const: smstateen
132	144	description: \|
133	145	The standard Smstateen extension for controlling access to CSRs
@@ -147,6 +159,12 @@ properties: · 13 unmodified lines
147	159	and mode-based filtering as ratified at commit 01d1df0 ("Add ability
148	160	to manually trigger workflow. (#2)") of riscv-count-overflow.
149	161
	162	+ - const: ssnpm
	163	+ description: \|
	164	+ The standard Ssnpm extension for next-mode pointer masking as
	165	+ ratified at commit d70011dde6c2 ("Update to ratified state")
	166	+ of riscv-j-extension.
	167	+
150	168	- const: sstc
151	169	description: \|
152	170	The standard Sstc supervisor-level extension for time compare as
@@ -178,6 +196,12 @@ properties: · 25 unmodified lines
178	196	as ratified at commit 4a69197e5617 ("Update to ratified state") of
179	197	riscv-svvptc.
180	198
	199	+ - const: zabha
	200	+ description: \|
	201	+ The Zabha extension for Byte and Halfword Atomic Memory Operations
	202	+ as ratified at commit 49f49c842ff9 ("Update to Rafified state") of
	203	+ riscv-zabha.
	204	+
181	205	- const: zacas
182	206	description: \|
183	207	The Zacas extension for Atomic Compare-and-Swap (CAS) instructions
@@ -290,6 +314,12 @@ properties: · 106 unmodified lines
290	314	in commit 64074bc ("Update version numbers for Zfh/Zfinx") of
291	315	riscv-isa-manual.
292	316
	317	+ - const: ziccrse
	318	+ description:
	319	+ The standard Ziccrse extension which provides forward progress
	320	+ guarantee on LR/SC sequences, as ratified in commit b1d806605f87
	321	+ ("Updated to ratified state.") of the riscv profiles specification.
	322	+
293	323	- const: zk
294	324	description:
295	325	The standard Zk Standard Scalar cryptography extension as ratified

Documentation/features/locking/queued-spinlocks/arch-support.txt+1−1

@@ -20,7 +20,7 @@ · 19 unmodified lines
20	20	\| openrisc: \| ok \|
21	21	\| parisc: \| TODO \|
22	22	\| powerpc: \| ok \|
23		− \| riscv: \| TODO \|
	23	+ \| riscv: \| ok \|
24	24	\| s390: \| TODO \|
25	25	\| sh: \| TODO \|
26	26	\| sparc: \| ok \|

arch/riscv/Kconfig+136−2

@@ -83,6 +83,7 @@ config RISCV · 82 unmodified lines
83	83	select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
84	84	select ARCH_WANTS_NO_INSTR
85	85	select ARCH_WANTS_THP_SWAP if HAVE_ARCH_TRANSPARENT_HUGEPAGE
	86	+ select ARCH_WEAK_RELEASE_ACQUIRE if ARCH_USE_QUEUED_SPINLOCKS
86	87	select BINFMT_FLAT_NO_DATA_START_OFFSET if !MMU
87	88	select BUILDTIME_TABLE_SORT if MMU
88	89	select CLINT_TIMER if RISCV_M_MODE
@@ -116,6 +117,7 @@ config RISCV · 27 unmodified lines
116	117	select GENERIC_VDSO_TIME_NS if HAVE_GENERIC_VDSO
117	118	select HARDIRQS_SW_RESEND
118	119	select HAS_IOPORT if MMU
	120	+ select HAVE_ALIGNED_STRUCT_PAGE
119	121	select HAVE_ARCH_AUDITSYSCALL
120	122	select HAVE_ARCH_HUGE_VMALLOC if HAVE_ARCH_HUGE_VMAP
121	123	select HAVE_ARCH_HUGE_VMAP if MMU && 64BIT
@@ -507,6 +509,39 @@ config NODES_SHIFT · 385 unmodified lines
507	509	Specify the maximum number of NUMA Nodes available on the target
508	510	system. Increases memory reserved to accommodate various tables.
509	511
	512	+choice
	513	+ prompt "RISC-V spinlock type"
	514	+ default RISCV_COMBO_SPINLOCKS
	515	+
	516	+config RISCV_TICKET_SPINLOCKS
	517	+ bool "Using ticket spinlock"
	518	+
	519	+config RISCV_QUEUED_SPINLOCKS
	520	+ bool "Using queued spinlock"
	521	+ depends on SMP && MMU && NONPORTABLE
	522	+ select ARCH_USE_QUEUED_SPINLOCKS
	523	+ help
	524	+ The queued spinlock implementation requires the forward progress
	525	+ guarantee of cmpxchg()/xchg() atomic operations: CAS with Zabha or
	526	+ LR/SC with Ziccrse provide such guarantee.
	527	+
	528	+ Select this if and only if Zabha or Ziccrse is available on your
	529	+ platform, RISCV_QUEUED_SPINLOCKS must not be selected for platforms
	530	+ without one of those extensions.
	531	+
	532	+ If unsure, select RISCV_COMBO_SPINLOCKS, which will use qspinlocks
	533	+ when supported and otherwise ticket spinlocks.
	534	+
	535	+config RISCV_COMBO_SPINLOCKS
	536	+ bool "Using combo spinlock"
	537	+ depends on SMP && MMU
	538	+ select ARCH_USE_QUEUED_SPINLOCKS
	539	+ help
	540	+ Embed both queued spinlock and ticket lock so that the spinlock
	541	+ implementation can be chosen at runtime.
	542	+
	543	+endchoice
	544	+
510	545	config RISCV_ALTERNATIVE
511	546	bool
512	547	depends on !XIP_KERNEL
@@ -532,6 +567,17 @@ config RISCV_ISA_C · 19 unmodified lines
532	567
533	568	If you don't know what to do here, say Y.
534	569
	570	+config RISCV_ISA_SUPM
	571	+ bool "Supm extension for userspace pointer masking"
	572	+ depends on 64BIT
	573	+ default y
	574	+ help
	575	+ Add support for pointer masking in userspace (Supm) when the
	576	+ underlying hardware extension (Smnpm or Ssnpm) is detected at boot.
	577	+
	578	+ If this option is disabled, userspace will be unable to use
	579	+ the prctl(PR_{SET,GET}_TAGGED_ADDR_CTRL) API.
	580	+
535	581	config RISCV_ISA_SVNAPOT
536	582	bool "Svnapot extension support for supervisor mode NAPOT pages"
537	583	depends on 64BIT && MMU
@@ -633,6 +679,40 @@ config RISCV_ISA_ZAWRS · 95 unmodified lines
633	679	use of these instructions in the kernel when the Zawrs extension is
634	680	detected at boot.
635	681
	682	+config TOOLCHAIN_HAS_ZABHA
	683	+ bool
	684	+ default y
	685	+ depends on !64BIT \|\| $(cc-option,-mabi=lp64 -march=rv64ima_zabha)
	686	+ depends on !32BIT \|\| $(cc-option,-mabi=ilp32 -march=rv32ima_zabha)
	687	+ depends on AS_HAS_OPTION_ARCH
	688	+
	689	+config RISCV_ISA_ZABHA
	690	+ bool "Zabha extension support for atomic byte/halfword operations"
	691	+ depends on TOOLCHAIN_HAS_ZABHA
	692	+ depends on RISCV_ALTERNATIVE
	693	+ default y
	694	+ help
	695	+ Enable the use of the Zabha ISA-extension to implement kernel
	696	+ byte/halfword atomic memory operations when it is detected at boot.
	697	+
	698	+ If you don't know what to do here, say Y.
	699	+
	700	+config TOOLCHAIN_HAS_ZACAS
	701	+ bool
	702	+ default y
	703	+ depends on !64BIT \|\| $(cc-option,-mabi=lp64 -march=rv64ima_zacas)
	704	+ depends on !32BIT \|\| $(cc-option,-mabi=ilp32 -march=rv32ima_zacas)
	705	+ depends on AS_HAS_OPTION_ARCH
	706	+
	707	+config RISCV_ISA_ZACAS
	708	+ bool "Zacas extension support for atomic CAS"
	709	+ depends on TOOLCHAIN_HAS_ZACAS
	710	+ depends on RISCV_ALTERNATIVE
	711	+ default y
	712	+ help
	713	+ Enable the use of the Zacas ISA-extension to implement kernel atomic
	714	+ cmpxchg operations when it is detected at boot.
	715	+
636	716	If you don't know what to do here, say Y.
637	717
638	718	config TOOLCHAIN_HAS_ZBB
@@ -786,10 +866,24 @@ config THREAD_SIZE_ORDER · 147 unmodified lines
786	866
787	867	config RISCV_MISALIGNED
788	868	bool
	869	+ help
	870	+ Embed support for detecting and emulating misaligned
	871	+ scalar or vector loads and stores.
	872	+
	873	+config RISCV_SCALAR_MISALIGNED
	874	+ bool
	875	+ select RISCV_MISALIGNED
789	876	select SYSCTL_ARCH_UNALIGN_ALLOW
790	877	help
791	878	Embed support for emulating misaligned loads and stores.
792	879
	880	+config RISCV_VECTOR_MISALIGNED
	881	+ bool
	882	+ select RISCV_MISALIGNED
	883	+ depends on RISCV_ISA_V
	884	+ help
	885	+ Enable detecting support for vector misaligned loads and stores.
	886	+
793	887	choice
794	888	prompt "Unaligned Accesses Support"
795	889	default RISCV_PROBE_UNALIGNED_ACCESS
@@ -801,7 +895,7 @@ choice · 5 unmodified lines
801	895
802	896	config RISCV_PROBE_UNALIGNED_ACCESS
803	897	bool "Probe for hardware unaligned access support"
804		− select RISCV_MISALIGNED
	898	+ select RISCV_SCALAR_MISALIGNED
805	899	help
806	900	During boot, the kernel will run a series of tests to determine the
807	901	speed of unaligned accesses. This probing will dynamically determine
@@ -812,7 +906,7 @@ config RISCV_PROBE_UNALIGNED_ACCESS · 4 unmodified lines
812	906
813	907	config RISCV_EMULATED_UNALIGNED_ACCESS
814	908	bool "Emulate unaligned access where system support is missing"
815		− select RISCV_MISALIGNED
	909	+ select RISCV_SCALAR_MISALIGNED
816	910	help
817	911	If unaligned memory accesses trap into the kernel as they are not
818	912	supported by the system, the kernel will emulate the unaligned
@@ -841,6 +935,46 @@ config RISCV_EFFICIENT_UNALIGNED_ACCESS · 22 unmodified lines
841	935
842	936	endchoice
843	937
	938	+choice
	939	+ prompt "Vector unaligned Accesses Support"
	940	+ depends on RISCV_ISA_V
	941	+ default RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
	942	+ help
	943	+ This determines the level of support for vector unaligned accesses. This
	944	+ information is used by the kernel to perform optimizations. It is also
	945	+ exposed to user space via the hwprobe syscall. The hardware will be
	946	+ probed at boot by default.
	947	+
	948	+config RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
	949	+ bool "Probe speed of vector unaligned accesses"
	950	+ select RISCV_VECTOR_MISALIGNED
	951	+ depends on RISCV_ISA_V
	952	+ help
	953	+ During boot, the kernel will run a series of tests to determine the
	954	+ speed of vector unaligned accesses if they are supported. This probing
	955	+ will dynamically determine the speed of vector unaligned accesses on
	956	+ the underlying system if they are supported.
	957	+
	958	+config RISCV_SLOW_VECTOR_UNALIGNED_ACCESS
	959	+ bool "Assume the system supports slow vector unaligned memory accesses"
	960	+ depends on NONPORTABLE
	961	+ help
	962	+ Assume that the system supports slow vector unaligned memory accesses. The
	963	+ kernel and userspace programs may not be able to run at all on systems
	964	+ that do not support unaligned memory accesses.
	965	+
	966	+config RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
	967	+ bool "Assume the system supports fast vector unaligned memory accesses"
	968	+ depends on NONPORTABLE
	969	+ help
	970	+ Assume that the system supports fast vector unaligned memory accesses. When
	971	+ enabled, this option improves the performance of the kernel on such
	972	+ systems. However, the kernel and userspace programs will run much more
	973	+ slowly, or will not be able to run at all, on systems that do not
	974	+ support efficient unaligned memory accesses.
	975	+
	976	+endchoice
	977	+
844	978	source "arch/riscv/Kconfig.vendor"
845	979
846	980	endmenu # "Platform type"

arch/riscv/Makefile+6−0

@@ -82,6 +82,12 @@ else · 81 unmodified lines
82	82	riscv-march-$(CONFIG_TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI) := $(riscv-march-y)_zicsr_zifencei
83	83	endif
84	84
	85	+# Check if the toolchain supports Zacas
	86	+riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZACAS) := $(riscv-march-y)_zacas
	87	+
	88	+# Check if the toolchain supports Zabha
	89	+riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZABHA) := $(riscv-march-y)_zabha
	90	+
85	91	# Remove F,D,V from isa string for all. Keep extensions between "fd" and "v" by
86	92	# matching non-v and non-multi-letter extensions out with the filter ([^v_]*)
87	93	KBUILD_CFLAGS += -march=$(shell echo $(riscv-march-y) \| sed -E 's/(rv32ima\|rv64ima)fd([^v_]*)v?/\1\2/')

arch/riscv/configs/defconfig+1−0

@@ -167,6 +167,7 @@ CONFIG_PINCTRL_SOPHGO_CV1800B=y · 166 unmodified lines
167	167	CONFIG_PINCTRL_SOPHGO_CV1812H=y
168	168	CONFIG_PINCTRL_SOPHGO_SG2000=y
169	169	CONFIG_PINCTRL_SOPHGO_SG2002=y
	170	+CONFIG_GPIO_DWAPB=y
170	171	CONFIG_GPIO_SIFIVE=y
171	172	CONFIG_POWER_RESET_GPIO_RESTART=y
172	173	CONFIG_SENSORS_SFCTEMP=m

arch/riscv/include/asm/Kbuild+3−1

@@ -6,10 +6,12 @@ generic-y += early_ioremap.h · 5 unmodified lines
6	6	generic-y += flat.h
7	7	generic-y += kvm_para.h
8	8	generic-y += mmzone.h
	9	+generic-y += mcs_spinlock.h
9	10	generic-y += parport.h
10		−generic-y += spinlock.h
11	11	generic-y += spinlock_types.h
	12	+generic-y += ticket_spinlock.h
12	13	generic-y += qrwlock.h
13	14	generic-y += qrwlock_types.h
	15	+generic-y += qspinlock.h
14	16	generic-y += user.h
15	17	generic-y += vmlinux.lds.h

arch/riscv/include/asm/cmpxchg.h+208−78

@@ -12,30 +12,43 @@ · 11 unmodified lines
12	12	#include <asm/fence.h>
13	13	#include <asm/hwcap.h>
14	14	#include <asm/insn-def.h>
15		−
16		−#define __arch_xchg_masked(sc_sfx, prepend, append, r, p, n) \
17		−({ \
18		− u32 __ptr32b = (u32 )((ulong)(p) & ~0x3); \
19		− ulong __s = ((ulong)(p) & (0x4 - sizeof(p))) BITS_PER_BYTE; \
20		− ulong __mask = GENMASK(((sizeof(p)) BITS_PER_BYTE) - 1, 0) \
21		− << __s; \
22		− ulong __newx = (ulong)(n) << __s; \
23		− ulong __retx; \
24		− ulong __rc; \
25		− \
26		− __asm__ __volatile__ ( \
27		− prepend \
28		− "0: lr.w %0, %2\n" \
29		− " and %1, %0, %z4\n" \
30		− " or %1, %1, %z3\n" \
31		− " sc.w" sc_sfx " %1, %1, %2\n" \
32		− " bnez %1, 0b\n" \
33		− append \
34		− : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
35		− : "rJ" (__newx), "rJ" (~__mask) \
36		− : "memory"); \
37		− \
38		− r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
	15	+#include <asm/cpufeature-macros.h>
	16	+
	17	+#define __arch_xchg_masked(sc_sfx, swap_sfx, prepend, sc_append, \
	18	+ swap_append, r, p, n) \
	19	+({ \
	20	+ if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \
	21	+ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA)) { \
	22	+ __asm__ __volatile__ ( \
	23	+ prepend \
	24	+ " amoswap" swap_sfx " %0, %z2, %1\n" \
	25	+ swap_append \
	26	+ : "=&r" (r), "+A" (*(p)) \
	27	+ : "rJ" (n) \
	28	+ : "memory"); \
	29	+ } else { \
	30	+ u32 __ptr32b = (u32 )((ulong)(p) & ~0x3); \
	31	+ ulong __s = ((ulong)(p) & (0x4 - sizeof(p))) BITS_PER_BYTE; \
	32	+ ulong __mask = GENMASK(((sizeof(p)) BITS_PER_BYTE) - 1, 0) \
	33	+ << __s; \
	34	+ ulong __newx = (ulong)(n) << __s; \
	35	+ ulong __retx; \
	36	+ ulong __rc; \
	37	+ \
	38	+ __asm__ __volatile__ ( \
	39	+ prepend \
	40	+ "0: lr.w %0, %2\n" \
	41	+ " and %1, %0, %z4\n" \
	42	+ " or %1, %1, %z3\n" \
	43	+ " sc.w" sc_sfx " %1, %1, %2\n" \
	44	+ " bnez %1, 0b\n" \
	45	+ sc_append \
	46	+ : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
	47	+ : "rJ" (__newx), "rJ" (~__mask) \
	48	+ : "memory"); \
	49	+ \
	50	+ r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
	51	+ } \
39	52	})
40	53
41	54	#define __arch_xchg(sfx, prepend, append, r, p, n) \
@@ -58,8 +71,13 @@ · 16 unmodified lines
58	71	\
59	72	switch (sizeof(*__ptr)) { \
60	73	case 1: \
	74	+ __arch_xchg_masked(sc_sfx, ".b" swap_sfx, \
	75	+ prepend, sc_append, swap_append, \
	76	+ __ret, __ptr, __new); \
	77	+ break; \
61	78	case 2: \
62		− __arch_xchg_masked(sc_sfx, prepend, sc_append, \
	79	+ __arch_xchg_masked(sc_sfx, ".h" swap_sfx, \
	80	+ prepend, sc_append, swap_append, \
63	81	__ret, __ptr, __new); \
64	82	break; \
65	83	case 4: \
@@ -106,55 +124,90 @@ · 40 unmodified lines
106	124	* store NEW in MEM. Return the initial value in MEM. Success is
107	125	* indicated by comparing RETURN with OLD.
108	126	*/
109		−
110		−#define __arch_cmpxchg_masked(sc_sfx, prepend, append, r, p, o, n) \
111		−({ \
112		− u32 __ptr32b = (u32 )((ulong)(p) & ~0x3); \
113		− ulong __s = ((ulong)(p) & (0x4 - sizeof(p))) BITS_PER_BYTE; \
114		− ulong __mask = GENMASK(((sizeof(p)) BITS_PER_BYTE) - 1, 0) \
115		− << __s; \
116		− ulong __newx = (ulong)(n) << __s; \
117		− ulong __oldx = (ulong)(o) << __s; \
118		− ulong __retx; \
119		− ulong __rc; \
120		− \
121		− __asm__ __volatile__ ( \
122		− prepend \
123		− "0: lr.w %0, %2\n" \
124		− " and %1, %0, %z5\n" \
125		− " bne %1, %z3, 1f\n" \
126		− " and %1, %0, %z6\n" \
127		− " or %1, %1, %z4\n" \
128		− " sc.w" sc_sfx " %1, %1, %2\n" \
129		− " bnez %1, 0b\n" \
130		− append \
131		− "1:\n" \
132		− : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
133		− : "rJ" ((long)__oldx), "rJ" (__newx), \
134		− "rJ" (__mask), "rJ" (~__mask) \
135		− : "memory"); \
136		− \
137		− r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
	127	+#define __arch_cmpxchg_masked(sc_sfx, cas_sfx, \
	128	+ sc_prepend, sc_append, \
	129	+ cas_prepend, cas_append, \
	130	+ r, p, o, n) \
	131	+({ \
	132	+ if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \
	133	+ IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \
	134	+ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA) && \
	135	+ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \
	136	+ r = o; \
	137	+ \
	138	+ __asm__ __volatile__ ( \
	139	+ cas_prepend \
	140	+ " amocas" cas_sfx " %0, %z2, %1\n" \
	141	+ cas_append \
	142	+ : "+&r" (r), "+A" (*(p)) \
	143	+ : "rJ" (n) \
	144	+ : "memory"); \
	145	+ } else { \
	146	+ u32 __ptr32b = (u32 )((ulong)(p) & ~0x3); \
	147	+ ulong __s = ((ulong)(p) & (0x4 - sizeof(p))) BITS_PER_BYTE; \
	148	+ ulong __mask = GENMASK(((sizeof(p)) BITS_PER_BYTE) - 1, 0) \
	149	+ << __s; \
	150	+ ulong __newx = (ulong)(n) << __s; \
	151	+ ulong __oldx = (ulong)(o) << __s; \
	152	+ ulong __retx; \
	153	+ ulong __rc; \
	154	+ \
	155	+ __asm__ __volatile__ ( \
	156	+ sc_prepend \
	157	+ "0: lr.w %0, %2\n" \
	158	+ " and %1, %0, %z5\n" \
	159	+ " bne %1, %z3, 1f\n" \
	160	+ " and %1, %0, %z6\n" \
	161	+ " or %1, %1, %z4\n" \
	162	+ " sc.w" sc_sfx " %1, %1, %2\n" \
	163	+ " bnez %1, 0b\n" \
	164	+ sc_append \
	165	+ "1:\n" \
	166	+ : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
	167	+ : "rJ" ((long)__oldx), "rJ" (__newx), \
	168	+ "rJ" (__mask), "rJ" (~__mask) \
	169	+ : "memory"); \
	170	+ \
	171	+ r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
	172	+ } \
138	173	})
139	174
140		−#define __arch_cmpxchg(lr_sfx, sc_sfx, prepend, append, r, p, co, o, n) \
	175	+#define __arch_cmpxchg(lr_sfx, sc_sfx, cas_sfx, \
	176	+ sc_prepend, sc_append, \
	177	+ cas_prepend, cas_append, \
	178	+ r, p, co, o, n) \
141	179	({ \
142		− register unsigned int __rc; \
	180	+ if (IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \
	181	+ riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \
	182	+ r = o; \
143	183	\
144		− __asm__ __volatile__ ( \
145		− prepend \
146		− "0: lr" lr_sfx " %0, %2\n" \
147		− " bne %0, %z3, 1f\n" \
148		− " sc" sc_sfx " %1, %z4, %2\n" \
149		− " bnez %1, 0b\n" \
150		− append \
151		− "1:\n" \
152		− : "=&r" (r), "=&r" (__rc), "+A" (*(p)) \
153		− : "rJ" (co o), "rJ" (n) \
154		− : "memory"); \
	184	+ __asm__ __volatile__ ( \
	185	+ cas_prepend \
	186	+ " amocas" cas_sfx " %0, %z2, %1\n" \
	187	+ cas_append \
	188	+ : "+&r" (r), "+A" (*(p)) \
	189	+ : "rJ" (n) \
	190	+ : "memory"); \
	191	+ } else { \
	192	+ register unsigned int __rc; \
	193	+ \
	194	+ __asm__ __volatile__ ( \
	195	+ sc_prepend \
	196	+ "0: lr" lr_sfx " %0, %2\n" \
	197	+ " bne %0, %z3, 1f\n" \
	198	+ " sc" sc_sfx " %1, %z4, %2\n" \
	199	+ " bnez %1, 0b\n" \
	200	+ sc_append \
	201	+ "1:\n" \
	202	+ : "=&r" (r), "=&r" (__rc), "+A" (*(p)) \
	203	+ : "rJ" (co o), "rJ" (n) \
	204	+ : "memory"); \
	205	+ } \
155	206	})
156	207
157		−#define _arch_cmpxchg(ptr, old, new, sc_sfx, prepend, append) \
	208	+#define _arch_cmpxchg(ptr, old, new, sc_sfx, cas_sfx, \
	209	+ sc_prepend, sc_append, \
	210	+ cas_prepend, cas_append) \
158	211	({ \
159	212	__typeof__(ptr) __ptr = (ptr); \
160	213	__typeof__(*(__ptr)) __old = (old); \
@@ -163,17 +216,28 @@ · 2 unmodified lines
163	216	\
164	217	switch (sizeof(*__ptr)) { \
165	218	case 1: \
	219	+ __arch_cmpxchg_masked(sc_sfx, ".b" cas_sfx, \
	220	+ sc_prepend, sc_append, \
	221	+ cas_prepend, cas_append, \
	222	+ __ret, __ptr, __old, __new); \
	223	+ break; \
166	224	case 2: \
167		− __arch_cmpxchg_masked(sc_sfx, prepend, append, \
168		− __ret, __ptr, __old, __new); \
	225	+ __arch_cmpxchg_masked(sc_sfx, ".h" cas_sfx, \
	226	+ sc_prepend, sc_append, \
	227	+ cas_prepend, cas_append, \
	228	+ __ret, __ptr, __old, __new); \
169	229	break; \
170	230	case 4: \
171		− __arch_cmpxchg(".w", ".w" sc_sfx, prepend, append, \
172		− __ret, __ptr, (long), __old, __new); \
	231	+ __arch_cmpxchg(".w", ".w" sc_sfx, ".w" cas_sfx, \
	232	+ sc_prepend, sc_append, \
	233	+ cas_prepend, cas_append, \
	234	+ __ret, __ptr, (long), __old, __new); \
173	235	break; \
174	236	case 8: \
175		− __arch_cmpxchg(".d", ".d" sc_sfx, prepend, append, \
176		− __ret, __ptr, /**/, __old, __new); \
	237	+ __arch_cmpxchg(".d", ".d" sc_sfx, ".d" cas_sfx, \
	238	+ sc_prepend, sc_append, \
	239	+ cas_prepend, cas_append, \
	240	+ __ret, __ptr, /**/, __old, __new); \
177	241	break; \
178	242	default: \
179	243	BUILD_BUG(); \
@@ -181,17 +245,40 @@ · 1 unmodified line
181	245	(__typeof__(*(__ptr)))__ret; \
182	246	})
183	247
	248	+/*
	249	+ * These macros are here to improve the readability of the arch_cmpxchg_XXX()
	250	+ * macros.
	251	+ */
	252	+#define SC_SFX(x) x
	253	+#define CAS_SFX(x) x
	254	+#define SC_PREPEND(x) x
	255	+#define SC_APPEND(x) x
	256	+#define CAS_PREPEND(x) x
	257	+#define CAS_APPEND(x) x
	258	+
184	259	#define arch_cmpxchg_relaxed(ptr, o, n) \
185		− _arch_cmpxchg((ptr), (o), (n), "", "", "")
	260	+ _arch_cmpxchg((ptr), (o), (n), \
	261	+ SC_SFX(""), CAS_SFX(""), \
	262	+ SC_PREPEND(""), SC_APPEND(""), \
	263	+ CAS_PREPEND(""), CAS_APPEND(""))
186	264
187	265	#define arch_cmpxchg_acquire(ptr, o, n) \
188		− _arch_cmpxchg((ptr), (o), (n), "", "", RISCV_ACQUIRE_BARRIER)
	266	+ _arch_cmpxchg((ptr), (o), (n), \
	267	+ SC_SFX(""), CAS_SFX(""), \
	268	+ SC_PREPEND(""), SC_APPEND(RISCV_ACQUIRE_BARRIER), \
	269	+ CAS_PREPEND(""), CAS_APPEND(RISCV_ACQUIRE_BARRIER))
189	270
190	271	#define arch_cmpxchg_release(ptr, o, n) \
191		− _arch_cmpxchg((ptr), (o), (n), "", RISCV_RELEASE_BARRIER, "")
	272	+ _arch_cmpxchg((ptr), (o), (n), \
	273	+ SC_SFX(""), CAS_SFX(""), \
	274	+ SC_PREPEND(RISCV_RELEASE_BARRIER), SC_APPEND(""), \
	275	+ CAS_PREPEND(RISCV_RELEASE_BARRIER), CAS_APPEND(""))
192	276
193	277	#define arch_cmpxchg(ptr, o, n) \
194		− _arch_cmpxchg((ptr), (o), (n), ".rl", "", " fence rw, rw\n")
	278	+ _arch_cmpxchg((ptr), (o), (n), \
	279	+ SC_SFX(".rl"), CAS_SFX(".aqrl"), \
	280	+ SC_PREPEND(""), SC_APPEND(RISCV_FULL_BARRIER), \
	281	+ CAS_PREPEND(""), CAS_APPEND(""))
195	282
196	283	#define arch_cmpxchg_local(ptr, o, n) \
197	284	arch_cmpxchg_relaxed((ptr), (o), (n))
@@ -226,6 +313,44 @@ · 28 unmodified lines
226	313	arch_cmpxchg_release((ptr), (o), (n)); \
227	314	})
228	315
	316	+#if defined(CONFIG_64BIT) && defined(CONFIG_RISCV_ISA_ZACAS)
	317	+
	318	+#define system_has_cmpxchg128() riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)
	319	+
	320	+union __u128_halves {
	321	+ u128 full;
	322	+ struct {
	323	+ u64 low, high;
	324	+ };
	325	+};
	326	+
	327	+#define __arch_cmpxchg128(p, o, n, cas_sfx) \
	328	+({ \
	329	+ __typeof__(*(p)) __o = (o); \
	330	+ union __u128_halves __hn = { .full = (n) }; \
	331	+ union __u128_halves __ho = { .full = (__o) }; \
	332	+ register unsigned long t1 asm ("t1") = __hn.low; \
	333	+ register unsigned long t2 asm ("t2") = __hn.high; \
	334	+ register unsigned long t3 asm ("t3") = __ho.low; \
	335	+ register unsigned long t4 asm ("t4") = __ho.high; \
	336	+ \
	337	+ __asm__ __volatile__ ( \
	338	+ " amocas.q" cas_sfx " %0, %z3, %2" \
	339	+ : "+&r" (t3), "+&r" (t4), "+A" (*(p)) \
	340	+ : "rJ" (t1), "rJ" (t2) \
	341	+ : "memory"); \
	342	+ \
	343	+ ((u128)t4 << 64) \| t3; \
	344	+})
	345	+
	346	+#define arch_cmpxchg128(ptr, o, n) \
	347	+ __arch_cmpxchg128((ptr), (o), (n), ".aqrl")
	348	+
	349	+#define arch_cmpxchg128_local(ptr, o, n) \
	350	+ __arch_cmpxchg128((ptr), (o), (n), "")
	351	+
	352	+#endif /* CONFIG_64BIT && CONFIG_RISCV_ISA_ZACAS */
	353	+
229	354	#ifdef CONFIG_RISCV_ISA_ZAWRS
230	355	/*
231	356	* Despite wrs.nto being "WRS-with-no-timeout", in the absence of changes to
@@ -245,6 +370,11 @@ static __always_inline void __cmpwait(volatile void *ptr, · 13 unmodified lines
245	370	: : : : no_zawrs);
246	371
247	372	switch (size) {
	373	+ case 1:
	374	+ fallthrough;
	375	+ case 2:
	376	+ /* RISC-V doesn't have lr instructions on byte and half-word. */
	377	+ goto no_zawrs;
248	378	case 4:
249	379	asm volatile(
250	380	" lr.w %0, %1\n"

arch/riscv/include/asm/compat.h+0−1

@@ -9,7 +9,6 @@ · 8 unmodified lines
9	9	*/
10	10	#include <linux/types.h>
11	11	#include <linux/sched.h>
12		−#include <linux/sched/task_stack.h>
13	12	#include <asm-generic/compat.h>
14	13
15	14	static inline int is_compat_task(void)

arch/riscv/include/asm/cpufeature-macros.h+66−0

@@ -0,0 +1,66 @@
	1	+/* SPDX-License-Identifier: GPL-2.0-only */
	2	+/*
	3	+ * Copyright 2022-2024 Rivos, Inc
	4	+ */
	5	+
	6	+#ifndef _ASM_CPUFEATURE_MACROS_H
	7	+#define _ASM_CPUFEATURE_MACROS_H
	8	+
	9	+#include <asm/hwcap.h>
	10	+#include <asm/alternative-macros.h>
	11	+
	12	+#define STANDARD_EXT 0
	13	+
	14	+bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit);
	15	+#define riscv_isa_extension_available(isa_bitmap, ext) \
	16	+ __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_##ext)
	17	+
	18	+static __always_inline bool __riscv_has_extension_likely(const unsigned long vendor,
	19	+ const unsigned long ext)
	20	+{
	21	+ asm goto(ALTERNATIVE("j %l[l_no]", "nop", %[vendor], %[ext], 1)
	22	+ :
	23	+ : [vendor] "i" (vendor), [ext] "i" (ext)
	24	+ :
	25	+ : l_no);
	26	+
	27	+ return true;
	28	+l_no:
	29	+ return false;
	30	+}
	31	+
	32	+static __always_inline bool __riscv_has_extension_unlikely(const unsigned long vendor,
	33	+ const unsigned long ext)
	34	+{
	35	+ asm goto(ALTERNATIVE("nop", "j %l[l_yes]", %[vendor], %[ext], 1)
	36	+ :
	37	+ : [vendor] "i" (vendor), [ext] "i" (ext)
	38	+ :
	39	+ : l_yes);
	40	+
	41	+ return false;
	42	+l_yes:
	43	+ return true;
	44	+}
	45	+
	46	+static __always_inline bool riscv_has_extension_unlikely(const unsigned long ext)
	47	+{
	48	+ compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
	49	+
	50	+ if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
	51	+ return __riscv_has_extension_unlikely(STANDARD_EXT, ext);
	52	+
	53	+ return __riscv_isa_extension_available(NULL, ext);
	54	+}
	55	+
	56	+static __always_inline bool riscv_has_extension_likely(const unsigned long ext)
	57	+{
	58	+ compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
	59	+
	60	+ if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
	61	+ return __riscv_has_extension_likely(STANDARD_EXT, ext);
	62	+
	63	+ return __riscv_isa_extension_available(NULL, ext);
	64	+}
	65	+
	66	+#endif /* _ASM_CPUFEATURE_MACROS_H */

arch/riscv/include/asm/cpufeature.h+14−59

@@ -8,9 +8,12 @@ · 7 unmodified lines
8	8
9	9	#include <linux/bitmap.h>
10	10	#include <linux/jump_label.h>
	11	+#include <linux/workqueue.h>
	12	+#include <linux/kconfig.h>
	13	+#include <linux/percpu-defs.h>
	14	+#include <linux/threads.h>
11	15	#include <asm/hwcap.h>
12		−#include <asm/alternative-macros.h>
13		−#include <asm/errno.h>
	16	+#include <asm/cpufeature-macros.h>
14	17
15	18	/*
16	19	* These are probed via a device_initcall(), via either the SBI or directly
@@ -31,7 +34,7 @@ DECLARE_PER_CPU(struct riscv_cpuinfo, riscv_cpuinfo); · 14 unmodified lines
31	34	/* Per-cpu ISA extensions. */
32	35	extern struct riscv_isainfo hart_isa[NR_CPUS];
33	36
34		−void riscv_user_isa_enable(void);
	37	+void __init riscv_user_isa_enable(void);
35	38
36	39	#define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size, _validate) { \
37	40	.name = #_name, \
@@ -58,8 +61,9 @@ void riscv_user_isa_enable(void); · 20 unmodified lines
58	61	#define __RISCV_ISA_EXT_SUPERSET_VALIDATE(_name, _id, _sub_exts, _validate) \
59	62	_RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), _validate)
60	63
61		−#if defined(CONFIG_RISCV_MISALIGNED)
62	64	bool check_unaligned_access_emulated_all_cpus(void);
	65	+#if defined(CONFIG_RISCV_SCALAR_MISALIGNED)
	66	+void check_unaligned_access_emulated(struct work_struct *work __always_unused);
63	67	void unaligned_emulation_finish(void);
64	68	bool unaligned_ctl_available(void);
65	69	DECLARE_PER_CPU(long, misaligned_access_speed);
@@ -70,6 +74,12 @@ static inline bool unaligned_ctl_available(void) · 4 unmodified lines
70	74	}
71	75	#endif
72	76
	77	+bool check_vector_unaligned_access_emulated_all_cpus(void);
	78	+#if defined(CONFIG_RISCV_VECTOR_MISALIGNED)
	79	+void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused);
	80	+DECLARE_PER_CPU(long, vector_misaligned_access);
	81	+#endif
	82	+
73	83	#if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)
74	84	DECLARE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key);
75	85
@@ -103,61 +113,6 @@ extern const size_t riscv_isa_ext_count; · 27 unmodified lines
103	113	extern bool riscv_isa_fallback;
104	114
105	115	unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap);
106		−
107		−#define STANDARD_EXT 0
108		−
109		−bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit);
110		−#define riscv_isa_extension_available(isa_bitmap, ext) \
111		− __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_##ext)
112		−
113		−static __always_inline bool __riscv_has_extension_likely(const unsigned long vendor,
114		− const unsigned long ext)
115		−{
116		− asm goto(ALTERNATIVE("j %l[l_no]", "nop", %[vendor], %[ext], 1)
117		− :
118		− : [vendor] "i" (vendor), [ext] "i" (ext)
119		− :
120		− : l_no);
121		−
122		− return true;
123		−l_no:
124		− return false;
125		−}
126		−
127		−static __always_inline bool __riscv_has_extension_unlikely(const unsigned long vendor,
128		− const unsigned long ext)
129		−{
130		− asm goto(ALTERNATIVE("nop", "j %l[l_yes]", %[vendor], %[ext], 1)
131		− :
132		− : [vendor] "i" (vendor), [ext] "i" (ext)
133		− :
134		− : l_yes);
135		−
136		− return false;
137		−l_yes:
138		− return true;
139		−}
140		−
141		−static __always_inline bool riscv_has_extension_unlikely(const unsigned long ext)
142		−{
143		− compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
144		−
145		− if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
146		− return __riscv_has_extension_unlikely(STANDARD_EXT, ext);
147		−
148		− return __riscv_isa_extension_available(NULL, ext);
149		−}
150		−
151		−static __always_inline bool riscv_has_extension_likely(const unsigned long ext)
152		−{
153		− compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
154		−
155		− if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
156		− return __riscv_has_extension_likely(STANDARD_EXT, ext);
157		−
158		− return __riscv_isa_extension_available(NULL, ext);
159		−}
160		−
161	116	static __always_inline bool riscv_cpu_has_extension_likely(int cpu, const unsigned long ext)
162	117	{
163	118	compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");

arch/riscv/include/asm/csr.h+16−0

@@ -119,6 +119,10 @@ · 118 unmodified lines
119	119
120	120	/* HSTATUS flags */
121	121	#ifdef CONFIG_64BIT
	122	+#define HSTATUS_HUPMM _AC(0x3000000000000, UL)
	123	+#define HSTATUS_HUPMM_PMLEN_0 _AC(0x0000000000000, UL)
	124	+#define HSTATUS_HUPMM_PMLEN_7 _AC(0x2000000000000, UL)
	125	+#define HSTATUS_HUPMM_PMLEN_16 _AC(0x3000000000000, UL)
122	126	#define HSTATUS_VSXL _AC(0x300000000, UL)
123	127	#define HSTATUS_VSXL_SHIFT 32
124	128	#endif
@@ -195,6 +199,10 @@ · 70 unmodified lines
195	199	/* xENVCFG flags */
196	200	#define ENVCFG_STCE (_AC(1, ULL) << 63)
197	201	#define ENVCFG_PBMTE (_AC(1, ULL) << 62)
	202	+#define ENVCFG_PMM (_AC(0x3, ULL) << 32)
	203	+#define ENVCFG_PMM_PMLEN_0 (_AC(0x0, ULL) << 32)
	204	+#define ENVCFG_PMM_PMLEN_7 (_AC(0x2, ULL) << 32)
	205	+#define ENVCFG_PMM_PMLEN_16 (_AC(0x3, ULL) << 32)
198	206	#define ENVCFG_CBZE (_AC(1, UL) << 7)
199	207	#define ENVCFG_CBCFE (_AC(1, UL) << 6)
200	208	#define ENVCFG_CBIE_SHIFT 4
@@ -216,6 +224,12 @@ · 15 unmodified lines
216	224	#define SMSTATEEN0_SSTATEEN0_SHIFT 63
217	225	#define SMSTATEEN0_SSTATEEN0 (_ULL(1) << SMSTATEEN0_SSTATEEN0_SHIFT)
218	226
	227	+/* mseccfg bits */
	228	+#define MSECCFG_PMM ENVCFG_PMM
	229	+#define MSECCFG_PMM_PMLEN_0 ENVCFG_PMM_PMLEN_0
	230	+#define MSECCFG_PMM_PMLEN_7 ENVCFG_PMM_PMLEN_7
	231	+#define MSECCFG_PMM_PMLEN_16 ENVCFG_PMM_PMLEN_16
	232	+
219	233	/* symbolic CSR names: */
220	234	#define CSR_CYCLE 0xc00
221	235	#define CSR_TIME 0xc01
@@ -382,6 +396,8 @@ · 160 unmodified lines
382	396	#define CSR_MIP 0x344
383	397	#define CSR_PMPCFG0 0x3a0
384	398	#define CSR_PMPADDR0 0x3b0
	399	+#define CSR_MSECCFG 0x747
	400	+#define CSR_MSECCFGH 0x757
385	401	#define CSR_MVENDORID 0xf11
386	402	#define CSR_MARCHID 0xf12
387	403	#define CSR_MIMPID 0xf13

arch/riscv/include/asm/entry-common.h+1−0

@@ -33,6 +33,7 @@ static inline int handle_misaligned_load(struct pt_regs *regs) · 32 unmodified lines
33	33	{
34	34	return -1;
35	35	}
	36	+
36	37	static inline int handle_misaligned_store(struct pt_regs *regs)
37	38	{
38	39	return -1;

arch/riscv/include/asm/hwcap.h+7−0

@@ -93,6 +93,11 @@ · 92 unmodified lines
93	93	#define RISCV_ISA_EXT_ZCMOP 84
94	94	#define RISCV_ISA_EXT_ZAWRS 85
95	95	#define RISCV_ISA_EXT_SVVPTC 86
	96	+#define RISCV_ISA_EXT_SMMPM 87
	97	+#define RISCV_ISA_EXT_SMNPM 88
	98	+#define RISCV_ISA_EXT_SSNPM 89
	99	+#define RISCV_ISA_EXT_ZABHA 90
	100	+#define RISCV_ISA_EXT_ZICCRSE 91
96	101
97	102	#define RISCV_ISA_EXT_XLINUXENVCFG 127
98	103
@@ -101,8 +106,10 @@ · 2 unmodified lines
101	106
102	107	#ifdef CONFIG_RISCV_M_MODE
103	108	#define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SMAIA
	109	+#define RISCV_ISA_EXT_SUPM RISCV_ISA_EXT_SMNPM
104	110	#else
105	111	#define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SSAIA
	112	+#define RISCV_ISA_EXT_SUPM RISCV_ISA_EXT_SSNPM
106	113	#endif
107	114
108	115	#endif /* _ASM_RISCV_HWCAP_H */

arch/riscv/include/asm/hwprobe.h+1−1

@@ -8,7 +8,7 @@ · 7 unmodified lines
8	8
9	9	#include <uapi/asm/hwprobe.h>
10	10
11		−#define RISCV_HWPROBE_MAX_KEY 9
	11	+#define RISCV_HWPROBE_MAX_KEY 10
12	12
13	13	static inline bool riscv_hwprobe_key_is_valid(__s64 key)
14	14	{

arch/riscv/include/asm/mmu.h+7−0

@@ -25,9 +25,16 @@ typedef struct { · 24 unmodified lines
25	25	#ifdef CONFIG_BINFMT_ELF_FDPIC
26	26	unsigned long exec_fdpic_loadmap;
27	27	unsigned long interp_fdpic_loadmap;
	28	+#endif
	29	+ unsigned long flags;
	30	+#ifdef CONFIG_RISCV_ISA_SUPM
	31	+ u8 pmlen;
28	32	#endif
29	33	} mm_context_t;
30	34
	35	+/* Lock the pointer masking mode because this mm is multithreaded */
	36	+#define MM_CONTEXT_LOCK_PMLEN 0
	37	+
31	38	#define cntx2asid(cntx) ((cntx) & SATP_ASID_MASK)
32	39	#define cntx2version(cntx) ((cntx) & ~SATP_ASID_MASK)
33	40

arch/riscv/include/asm/mmu_context.h+13−0

@@ -20,6 +20,9 @@ void switch_mm(struct mm_struct prev, struct mm_struct next, · 19 unmodified lines
20	20	static inline void activate_mm(struct mm_struct *prev,
21	21	struct mm_struct *next)
22	22	{
	23	+#ifdef CONFIG_RISCV_ISA_SUPM
	24	+ next->context.pmlen = 0;
	25	+#endif
23	26	switch_mm(prev, next, NULL);
24	27	}
25	28
@@ -30,11 +33,21 @@ static inline int init_new_context(struct task_struct *tsk, · 4 unmodified lines
30	33	#ifdef CONFIG_MMU
31	34	atomic_long_set(&mm->context.id, 0);
32	35	#endif
	36	+ if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM))
	37	+ clear_bit(MM_CONTEXT_LOCK_PMLEN, &mm->context.flags);
33	38	return 0;
34	39	}
35	40
36	41	DECLARE_STATIC_KEY_FALSE(use_asid_allocator);
37	42
	43	+#ifdef CONFIG_RISCV_ISA_SUPM
	44	+#define mm_untag_mask mm_untag_mask
	45	+static inline unsigned long mm_untag_mask(struct mm_struct *mm)
	46	+{
	47	+ return -1UL >> mm->context.pmlen;
	48	+}
	49	+#endif
	50	+
38	51	#include <asm-generic/mmu_context.h>
39	52
40	53	#endif /* _ASM_RISCV_MMU_CONTEXT_H */

arch/riscv/include/asm/processor.h+9−0

@@ -102,6 +102,7 @@ struct thread_struct { · 101 unmodified lines
102	102	unsigned long s[12]; /* s[0]: frame pointer */
103	103	struct __riscv_d_ext_state fstate;
104	104	unsigned long bad_cause;
	105	+ unsigned long envcfg;
105	106	u32 riscv_v_flags;
106	107	u32 vstate_ctrl;
107	108	struct __riscv_v_ext_state vstate;
@@ -177,6 +178,14 @@ extern int set_unalign_ctl(struct task_struct *tsk, unsigned int val); · 69 unmodified lines
177	178	#define RISCV_SET_ICACHE_FLUSH_CTX(arg1, arg2) riscv_set_icache_flush_ctx(arg1, arg2)
178	179	extern int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long per_thread);
179	180
	181	+#ifdef CONFIG_RISCV_ISA_SUPM
	182	+/* PR_{SET,GET}_TAGGED_ADDR_CTRL prctl */
	183	+long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg);
	184	+long get_tagged_addr_ctrl(struct task_struct *task);
	185	+#define SET_TAGGED_ADDR_CTRL(arg) set_tagged_addr_ctrl(current, arg)
	186	+#define GET_TAGGED_ADDR_CTRL() get_tagged_addr_ctrl(current)
	187	+#endif
	188	+
180	189	#endif /* __ASSEMBLY__ */
181	190
182	191	#endif /* _ASM_RISCV_PROCESSOR_H */

arch/riscv/include/asm/spinlock.h+47−0

@@ -0,0 +1,47 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+
	3	+#ifndef __ASM_RISCV_SPINLOCK_H
	4	+#define __ASM_RISCV_SPINLOCK_H
	5	+
	6	+#ifdef CONFIG_RISCV_COMBO_SPINLOCKS
	7	+#define _Q_PENDING_LOOPS (1 << 9)
	8	+
	9	+#define __no_arch_spinlock_redefine
	10	+#include <asm/ticket_spinlock.h>
	11	+#include <asm/qspinlock.h>
	12	+#include <asm/jump_label.h>
	13	+
	14	+/*
	15	+ * TODO: Use an alternative instead of a static key when we are able to parse
	16	+ * the extensions string earlier in the boot process.
	17	+ */
	18	+DECLARE_STATIC_KEY_TRUE(qspinlock_key);
	19	+
	20	+#define SPINLOCK_BASE_DECLARE(op, type, type_lock) \
	21	+static __always_inline type arch_spin_##op(type_lock lock) \
	22	+{ \
	23	+ if (static_branch_unlikely(&qspinlock_key)) \
	24	+ return queued_spin_##op(lock); \
	25	+ return ticket_spin_##op(lock); \
	26	+}
	27	+
	28	+SPINLOCK_BASE_DECLARE(lock, void, arch_spinlock_t *)
	29	+SPINLOCK_BASE_DECLARE(unlock, void, arch_spinlock_t *)
	30	+SPINLOCK_BASE_DECLARE(is_locked, int, arch_spinlock_t *)
	31	+SPINLOCK_BASE_DECLARE(is_contended, int, arch_spinlock_t *)
	32	+SPINLOCK_BASE_DECLARE(trylock, bool, arch_spinlock_t *)
	33	+SPINLOCK_BASE_DECLARE(value_unlocked, int, arch_spinlock_t)
	34	+
	35	+#elif defined(CONFIG_RISCV_QUEUED_SPINLOCKS)
	36	+
	37	+#include <asm/qspinlock.h>
	38	+
	39	+#else
	40	+
	41	+#include <asm/ticket_spinlock.h>
	42	+
	43	+#endif
	44	+
	45	+#include <asm/qrwlock.h>
	46	+
	47	+#endif /* __ASM_RISCV_SPINLOCK_H */

arch/riscv/include/asm/switch_to.h+19−0

@@ -70,6 +70,24 @@ static __always_inline bool has_fpu(void) { return false; } · 69 unmodified lines
70	70	#define __switch_to_fpu(__prev, __next) do { } while (0)
71	71	#endif
72	72
	73	+static inline void envcfg_update_bits(struct task_struct *task,
	74	+ unsigned long mask, unsigned long val)
	75	+{
	76	+ unsigned long envcfg;
	77	+
	78	+ envcfg = (task->thread.envcfg & ~mask) \| val;
	79	+ task->thread.envcfg = envcfg;
	80	+ if (task == current)
	81	+ csr_write(CSR_ENVCFG, envcfg);
	82	+}
	83	+
	84	+static inline void __switch_to_envcfg(struct task_struct *next)
	85	+{
	86	+ asm volatile (ALTERNATIVE("nop", "csrw " __stringify(CSR_ENVCFG) ", %0",
	87	+ 0, RISCV_ISA_EXT_XLINUXENVCFG, 1)
	88	+ :: "r" (next->thread.envcfg) : "memory");
	89	+}
	90	+
73	91	extern struct task_struct __switch_to(struct task_struct ,
74	92	struct task_struct *);
75	93
@@ -103,6 +121,7 @@ do { \ · 27 unmodified lines
103	121	__switch_to_vector(__prev, __next); \
104	122	if (switch_to_should_flush_icache(__next)) \
105	123	local_flush_icache_all(); \
	124	+ __switch_to_envcfg(__next); \
106	125	((last) = __switch_to(__prev, __next)); \
107	126	} while (0)
108	127

arch/riscv/include/asm/uaccess.h+38−5

@@ -9,8 +9,41 @@ · 8 unmodified lines
9	9	#define _ASM_RISCV_UACCESS_H
10	10
11	11	#include <asm/asm-extable.h>
	12	+#include <asm/cpufeature.h>
12	13	#include <asm/pgtable.h> /* for TASK_SIZE */
13	14
	15	+#ifdef CONFIG_RISCV_ISA_SUPM
	16	+static inline unsigned long __untagged_addr_remote(struct mm_struct *mm, unsigned long addr)
	17	+{
	18	+ if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM)) {
	19	+ u8 pmlen = mm->context.pmlen;
	20	+
	21	+ /* Virtual addresses are sign-extended; physical addresses are zero-extended. */
	22	+ if (IS_ENABLED(CONFIG_MMU))
	23	+ return (long)(addr << pmlen) >> pmlen;
	24	+ else
	25	+ return (addr << pmlen) >> pmlen;
	26	+ }
	27	+
	28	+ return addr;
	29	+}
	30	+
	31	+#define untagged_addr(addr) ({ \
	32	+ unsigned long __addr = (__force unsigned long)(addr); \
	33	+ (__force __typeof__(addr))__untagged_addr_remote(current->mm, __addr); \
	34	+})
	35	+
	36	+#define untagged_addr_remote(mm, addr) ({ \
	37	+ unsigned long __addr = (__force unsigned long)(addr); \
	38	+ mmap_assert_locked(mm); \
	39	+ (__force __typeof__(addr))__untagged_addr_remote(mm, __addr); \
	40	+})
	41	+
	42	+#define access_ok(addr, size) likely(__access_ok(untagged_addr(addr), size))
	43	+#else
	44	+#define untagged_addr(addr) (addr)
	45	+#endif
	46	+
14	47	/*
15	48	* User space memory access functions
16	49	*/
@@ -130,7 +163,7 @@ do { \ · 113 unmodified lines
130	163	*/
131	164	#define __get_user(x, ptr) \
132	165	({ \
133		− const __typeof__((ptr)) __user __gu_ptr = (ptr); \
	166	+ const __typeof__((ptr)) __user __gu_ptr = untagged_addr(ptr); \
134	167	long __gu_err = 0; \
135	168	\
136	169	__chk_user_ptr(__gu_ptr); \
@@ -246,7 +279,7 @@ do { \ · 109 unmodified lines
246	279	*/
247	280	#define __put_user(x, ptr) \
248	281	({ \
249		− __typeof__((ptr)) __user __gu_ptr = (ptr); \
	282	+ __typeof__((ptr)) __user __gu_ptr = untagged_addr(ptr); \
250	283	__typeof__(*__gu_ptr) __val = (x); \
251	284	long __pu_err = 0; \
252	285	\
@@ -293,13 +326,13 @@ unsigned long __must_check __asm_copy_from_user(void *to, · 40 unmodified lines
293	326	static inline unsigned long
294	327	raw_copy_from_user(void to, const void __user from, unsigned long n)
295	328	{
296		− return __asm_copy_from_user(to, from, n);
	329	+ return __asm_copy_from_user(to, untagged_addr(from), n);
297	330	}
298	331
299	332	static inline unsigned long
300	333	raw_copy_to_user(void __user to, const void from, unsigned long n)
301	334	{
302		− return __asm_copy_to_user(to, from, n);
	335	+ return __asm_copy_to_user(untagged_addr(to), from, n);
303	336	}
304	337
305	338	extern long strncpy_from_user(char dest, const char __user src, long count);
@@ -314,7 +347,7 @@ unsigned long __must_check clear_user(void __user *to, unsigned long n) · 8 unmodified lines
314	347	{
315	348	might_fault();
316	349	return access_ok(to, n) ?
317		− __clear_user(to, n) : n;
	350	+ __clear_user(untagged_addr(to), n) : n;
318	351	}
319	352
320	353	#define __get_kernel_nofault(dst, src, type, err_label) \

arch/riscv/include/asm/vector.h+2−0

@@ -21,6 +21,7 @@ · 20 unmodified lines
21	21
22	22	extern unsigned long riscv_v_vsize;
23	23	int riscv_v_setup_vsize(void);
	24	+bool insn_is_vector(u32 insn_buf);
24	25	bool riscv_v_first_use_handler(struct pt_regs *regs);
25	26	void kernel_vector_begin(void);
26	27	void kernel_vector_end(void);
@@ -268,6 +269,7 @@ struct pt_regs; · 241 unmodified lines
268	269
269	270	static inline int riscv_v_setup_vsize(void) { return -EOPNOTSUPP; }
270	271	static __always_inline bool has_vector(void) { return false; }
	272	+static __always_inline bool insn_is_vector(u32 insn_buf) { return false; }
271	273	static inline bool riscv_v_first_use_handler(struct pt_regs *regs) { return false; }
272	274	static inline bool riscv_v_vstate_query(struct pt_regs *regs) { return false; }
273	275	static inline bool riscv_v_vstate_ctrl_user_allowed(void) { return false; }

arch/riscv/include/uapi/asm/hwprobe.h+6−0

@@ -72,6 +72,7 @@ struct riscv_hwprobe { · 71 unmodified lines
72	72	#define RISCV_HWPROBE_EXT_ZCF (1ULL << 46)
73	73	#define RISCV_HWPROBE_EXT_ZCMOP (1ULL << 47)
74	74	#define RISCV_HWPROBE_EXT_ZAWRS (1ULL << 48)
	75	+#define RISCV_HWPROBE_EXT_SUPM (1ULL << 49)
75	76	#define RISCV_HWPROBE_KEY_CPUPERF_0 5
76	77	#define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0)
77	78	#define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0)
@@ -88,6 +89,11 @@ struct riscv_hwprobe { · 10 unmodified lines
88	89	#define RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW 2
89	90	#define RISCV_HWPROBE_MISALIGNED_SCALAR_FAST 3
90	91	#define RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED 4
	92	+#define RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF 10
	93	+#define RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN 0
	94	+#define RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW 2
	95	+#define RISCV_HWPROBE_MISALIGNED_VECTOR_FAST 3
	96	+#define RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED 4
91	97	/* Increase RISCV_HWPROBE_MAX_KEY when adding items. */
92	98
93	99	/* Flags */

arch/riscv/include/uapi/asm/kvm.h+2−0

@@ -175,6 +175,8 @@ enum KVM_RISCV_ISA_EXT_ID { · 174 unmodified lines
175	175	KVM_RISCV_ISA_EXT_ZCF,
176	176	KVM_RISCV_ISA_EXT_ZCMOP,
177	177	KVM_RISCV_ISA_EXT_ZAWRS,
	178	+ KVM_RISCV_ISA_EXT_SMNPM,
	179	+ KVM_RISCV_ISA_EXT_SSNPM,
178	180	KVM_RISCV_ISA_EXT_MAX,
179	181	};
180	182

arch/riscv/kernel/Makefile+2−1

@@ -75,7 +75,8 @@ obj-$(CONFIG_MMU) += vdso.o vdso/ · 74 unmodified lines
75	75
76	76	obj-$(CONFIG_RISCV_MISALIGNED) += traps_misaligned.o
77	77	obj-$(CONFIG_RISCV_MISALIGNED) += unaligned_access_speed.o
78		−obj-$(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) += copy-unaligned.o
	78	+obj-$(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) += copy-unaligned.o
	79	+obj-$(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS) += vec-copy-unaligned.o
79	80
80	81	obj-$(CONFIG_FPU) += fpu.o
81	82	obj-$(CONFIG_FPU) += kernel_mode_fpu.o

arch/riscv/kernel/copy-unaligned.h+5−0

@@ -10,4 +10,9 @@ · 9 unmodified lines
10	10	void __riscv_copy_words_unaligned(void dst, const void src, size_t size);
11	11	void __riscv_copy_bytes_unaligned(void dst, const void src, size_t size);
12	12
	13	+#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
	14	+void __riscv_copy_vec_words_unaligned(void dst, const void src, size_t size);
	15	+void __riscv_copy_vec_bytes_unaligned(void dst, const void src, size_t size);
	16	+#endif
	17	+
13	18	#endif /* __RISCV_KERNEL_COPY_UNALIGNED_H */

arch/riscv/kernel/cpufeature.c+13−3

@@ -29,6 +29,8 @@ · 28 unmodified lines
29	29
30	30	#define NUM_ALPHA_EXTS ('z' - 'a' + 1)
31	31
	32	+static bool any_cpu_has_zicboz;
	33	+
32	34	unsigned long elf_hwcap __read_mostly;
33	35
34	36	/* Host ISA bitmap */
@@ -99,6 +101,7 @@ static int riscv_ext_zicboz_validate(const struct riscv_isa_ext_data *data, · 64 unmodified lines
99	101	pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n");
100	102	return -EINVAL;
101	103	}
	104	+ any_cpu_has_zicboz = true;
102	105	return 0;
103	106	}
104	107
@@ -315,6 +318,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { · 210 unmodified lines
315	318	riscv_ext_zicbom_validate),
316	319	__RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts,
317	320	riscv_ext_zicboz_validate),
	321	+ __RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE),
318	322	__RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR),
319	323	__RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND),
320	324	__RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR),
@@ -323,6 +327,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { · 2 unmodified lines
323	327	__RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE),
324	328	__RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM),
325	329	__RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP),
	330	+ __RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA),
326	331	__RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS),
327	332	__RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS),
328	333	__RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA),
@@ -375,9 +380,12 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { · 46 unmodified lines
375	380	__RISCV_ISA_EXT_BUNDLE(zvksg, riscv_zvksg_bundled_exts),
376	381	__RISCV_ISA_EXT_DATA(zvkt, RISCV_ISA_EXT_ZVKT),
377	382	__RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA),
	383	+ __RISCV_ISA_EXT_DATA(smmpm, RISCV_ISA_EXT_SMMPM),
	384	+ __RISCV_ISA_EXT_SUPERSET(smnpm, RISCV_ISA_EXT_SMNPM, riscv_xlinuxenvcfg_exts),
378	385	__RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN),
379	386	__RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA),
380	387	__RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF),
	388	+ __RISCV_ISA_EXT_SUPERSET(ssnpm, RISCV_ISA_EXT_SSNPM, riscv_xlinuxenvcfg_exts),
381	389	__RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC),
382	390	__RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),
383	391	__RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT),
@@ -918,10 +926,12 @@ unsigned long riscv_get_elf_hwcap(void) · 534 unmodified lines
918	926	return hwcap;
919	927	}
920	928
921		−void riscv_user_isa_enable(void)
	929	+void __init riscv_user_isa_enable(void)
922	930	{
923		− if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_ZICBOZ))
924		− csr_set(CSR_ENVCFG, ENVCFG_CBZE);
	931	+ if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOZ))
	932	+ current->thread.envcfg \|= ENVCFG_CBZE;
	933	+ else if (any_cpu_has_zicboz)
	934	+ pr_warn("Zicboz disabled as it is unavailable on some harts\n");
925	935	}
926	936
927	937	#ifdef CONFIG_RISCV_ALTERNATIVE

arch/riscv/kernel/fpu.S+2−2

@@ -170,7 +170,7 @@ SYM_FUNC_END(__fstate_restore) · 169 unmodified lines
170	170	__access_func(f31)
171	171
172	172
173		−#ifdef CONFIG_RISCV_MISALIGNED
	173	+#ifdef CONFIG_RISCV_SCALAR_MISALIGNED
174	174
175	175	/*
176	176	* Disable compressed instructions set to keep a constant offset between FP
@@ -224,4 +224,4 @@ SYM_FUNC_START(get_f64_reg) · 47 unmodified lines
224	224	fp_access_epilogue
225	225	SYM_FUNC_END(get_f64_reg)
226	226
227		−#endif /* CONFIG_RISCV_MISALIGNED */
	227	+#endif /* CONFIG_RISCV_SCALAR_MISALIGNED */

arch/riscv/kernel/process.c+154−0

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	* Copyright (C) 2017 SiFive
8	8	*/
9	9
	10	+#include <linux/bitfield.h>
10	11	#include <linux/cpu.h>
11	12	#include <linux/kernel.h>
12	13	#include <linux/sched.h>
@@ -180,6 +181,10 @@ void flush_thread(void) · 167 unmodified lines
180	181	memset(&current->thread.vstate, 0, sizeof(struct __riscv_v_ext_state));
181	182	clear_tsk_thread_flag(current, TIF_RISCV_V_DEFER_RESTORE);
182	183	#endif
	184	+#ifdef CONFIG_RISCV_ISA_SUPM
	185	+ if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM))
	186	+ envcfg_update_bits(current, ENVCFG_PMM, ENVCFG_PMM_PMLEN_0);
	187	+#endif
183	188	}
184	189
185	190	void arch_release_task_struct(struct task_struct *tsk)
@@ -208,6 +213,10 @@ int copy_thread(struct task_struct p, const struct kernel_clone_args args) · 22 unmodified lines
208	213	unsigned long tls = args->tls;
209	214	struct pt_regs *childregs = task_pt_regs(p);
210	215
	216	+ /* Ensure all threads in this mm have the same pointer masking mode. */
	217	+ if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM) && p->mm && (clone_flags & CLONE_VM))
	218	+ set_bit(MM_CONTEXT_LOCK_PMLEN, &p->mm->context.flags);
	219	+
211	220	memset(&p->thread.s, 0, sizeof(p->thread.s));
212	221
213	222	/* p->thread holds context to be restored by __switch_to() */
@@ -242,3 +251,148 @@ void __init arch_task_cache_init(void) · 28 unmodified lines
242	251	{
243	252	riscv_v_setup_ctx_cache();
244	253	}
	254	+
	255	+#ifdef CONFIG_RISCV_ISA_SUPM
	256	+enum {
	257	+ PMLEN_0 = 0,
	258	+ PMLEN_7 = 7,
	259	+ PMLEN_16 = 16,
	260	+};
	261	+
	262	+static bool have_user_pmlen_7;
	263	+static bool have_user_pmlen_16;
	264	+
	265	+/*
	266	+ * Control the relaxed ABI allowing tagged user addresses into the kernel.
	267	+ */
	268	+static unsigned int tagged_addr_disabled;
	269	+
	270	+long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg)
	271	+{
	272	+ unsigned long valid_mask = PR_PMLEN_MASK \| PR_TAGGED_ADDR_ENABLE;
	273	+ struct thread_info *ti = task_thread_info(task);
	274	+ struct mm_struct *mm = task->mm;
	275	+ unsigned long pmm;
	276	+ u8 pmlen;
	277	+
	278	+ if (is_compat_thread(ti))
	279	+ return -EINVAL;
	280	+
	281	+ if (arg & ~valid_mask)
	282	+ return -EINVAL;
	283	+
	284	+ /*
	285	+ * Prefer the smallest PMLEN that satisfies the user's request,
	286	+ * in case choosing a larger PMLEN has a performance impact.
	287	+ */
	288	+ pmlen = FIELD_GET(PR_PMLEN_MASK, arg);
	289	+ if (pmlen == PMLEN_0) {
	290	+ pmm = ENVCFG_PMM_PMLEN_0;
	291	+ } else if (pmlen <= PMLEN_7 && have_user_pmlen_7) {
	292	+ pmlen = PMLEN_7;
	293	+ pmm = ENVCFG_PMM_PMLEN_7;
	294	+ } else if (pmlen <= PMLEN_16 && have_user_pmlen_16) {
	295	+ pmlen = PMLEN_16;
	296	+ pmm = ENVCFG_PMM_PMLEN_16;
	297	+ } else {
	298	+ return -EINVAL;
	299	+ }
	300	+
	301	+ /*
	302	+ * Do not allow the enabling of the tagged address ABI if globally
	303	+ * disabled via sysctl abi.tagged_addr_disabled, if pointer masking
	304	+ * is disabled for userspace.
	305	+ */
	306	+ if (arg & PR_TAGGED_ADDR_ENABLE && (tagged_addr_disabled \|\| !pmlen))
	307	+ return -EINVAL;
	308	+
	309	+ if (!(arg & PR_TAGGED_ADDR_ENABLE))
	310	+ pmlen = PMLEN_0;
	311	+
	312	+ if (mmap_write_lock_killable(mm))
	313	+ return -EINTR;
	314	+
	315	+ if (test_bit(MM_CONTEXT_LOCK_PMLEN, &mm->context.flags) && mm->context.pmlen != pmlen) {
	316	+ mmap_write_unlock(mm);
	317	+ return -EBUSY;
	318	+ }
	319	+
	320	+ envcfg_update_bits(task, ENVCFG_PMM, pmm);
	321	+ mm->context.pmlen = pmlen;
	322	+
	323	+ mmap_write_unlock(mm);
	324	+
	325	+ return 0;
	326	+}
	327	+
	328	+long get_tagged_addr_ctrl(struct task_struct *task)
	329	+{
	330	+ struct thread_info *ti = task_thread_info(task);
	331	+ long ret = 0;
	332	+
	333	+ if (is_compat_thread(ti))
	334	+ return -EINVAL;
	335	+
	336	+ /*
	337	+ * The mm context's pmlen is set only when the tagged address ABI is
	338	+ * enabled, so the effective PMLEN must be extracted from envcfg.PMM.
	339	+ */
	340	+ switch (task->thread.envcfg & ENVCFG_PMM) {
	341	+ case ENVCFG_PMM_PMLEN_7:
	342	+ ret = FIELD_PREP(PR_PMLEN_MASK, PMLEN_7);
	343	+ break;
	344	+ case ENVCFG_PMM_PMLEN_16:
	345	+ ret = FIELD_PREP(PR_PMLEN_MASK, PMLEN_16);
	346	+ break;
	347	+ }
	348	+
	349	+ if (task->mm->context.pmlen)
	350	+ ret \|= PR_TAGGED_ADDR_ENABLE;
	351	+
	352	+ return ret;
	353	+}
	354	+
	355	+static bool try_to_set_pmm(unsigned long value)
	356	+{
	357	+ csr_set(CSR_ENVCFG, value);
	358	+ return (csr_read_clear(CSR_ENVCFG, ENVCFG_PMM) & ENVCFG_PMM) == value;
	359	+}
	360	+
	361	+/*
	362	+ * Global sysctl to disable the tagged user addresses support. This control
	363	+ * only prevents the tagged address ABI enabling via prctl() and does not
	364	+ * disable it for tasks that already opted in to the relaxed ABI.
	365	+ */
	366	+
	367	+static struct ctl_table tagged_addr_sysctl_table[] = {
	368	+ {
	369	+ .procname = "tagged_addr_disabled",
	370	+ .mode = 0644,
	371	+ .data = &tagged_addr_disabled,
	372	+ .maxlen = sizeof(int),
	373	+ .proc_handler = proc_dointvec_minmax,
	374	+ .extra1 = SYSCTL_ZERO,
	375	+ .extra2 = SYSCTL_ONE,
	376	+ },
	377	+};
	378	+
	379	+static int __init tagged_addr_init(void)
	380	+{
	381	+ if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM))
	382	+ return 0;
	383	+
	384	+ /*
	385	+ * envcfg.PMM is a WARL field. Detect which values are supported.
	386	+ * Assume the supported PMLEN values are the same on all harts.
	387	+ */
	388	+ csr_clear(CSR_ENVCFG, ENVCFG_PMM);
	389	+ have_user_pmlen_7 = try_to_set_pmm(ENVCFG_PMM_PMLEN_7);
	390	+ have_user_pmlen_16 = try_to_set_pmm(ENVCFG_PMM_PMLEN_16);
	391	+
	392	+ if (!register_sysctl("abi", tagged_addr_sysctl_table))
	393	+ return -EINVAL;
	394	+
	395	+ return 0;
	396	+}
	397	+core_initcall(tagged_addr_init);
	398	+#endif /* CONFIG_RISCV_ISA_SUPM */

arch/riscv/kernel/ptrace.c+42−0

@@ -28,6 +28,9 @@ enum riscv_regset { · 27 unmodified lines
28	28	#ifdef CONFIG_RISCV_ISA_V
29	29	REGSET_V,
30	30	#endif
	31	+#ifdef CONFIG_RISCV_ISA_SUPM
	32	+ REGSET_TAGGED_ADDR_CTRL,
	33	+#endif
31	34	};
32	35
33	36	static int riscv_gpr_get(struct task_struct *target,
@@ -152,6 +155,35 @@ static int riscv_vr_set(struct task_struct *target, · 118 unmodified lines
152	155	}
153	156	#endif
154	157
	158	+#ifdef CONFIG_RISCV_ISA_SUPM
	159	+static int tagged_addr_ctrl_get(struct task_struct *target,
	160	+ const struct user_regset *regset,
	161	+ struct membuf to)
	162	+{
	163	+ long ctrl = get_tagged_addr_ctrl(target);
	164	+
	165	+ if (IS_ERR_VALUE(ctrl))
	166	+ return ctrl;
	167	+
	168	+ return membuf_write(&to, &ctrl, sizeof(ctrl));
	169	+}
	170	+
	171	+static int tagged_addr_ctrl_set(struct task_struct *target,
	172	+ const struct user_regset *regset,
	173	+ unsigned int pos, unsigned int count,
	174	+ const void kbuf, const void __user ubuf)
	175	+{
	176	+ int ret;
	177	+ long ctrl;
	178	+
	179	+ ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl, 0, -1);
	180	+ if (ret)
	181	+ return ret;
	182	+
	183	+ return set_tagged_addr_ctrl(target, ctrl);
	184	+}
	185	+#endif
	186	+
155	187	static const struct user_regset riscv_user_regset[] = {
156	188	[REGSET_X] = {
157	189	.core_note_type = NT_PRSTATUS,
@@ -182,6 +214,16 @@ static const struct user_regset riscv_user_regset[] = { · 24 unmodified lines
182	214	.set = riscv_vr_set,
183	215	},
184	216	#endif
	217	+#ifdef CONFIG_RISCV_ISA_SUPM
	218	+ [REGSET_TAGGED_ADDR_CTRL] = {
	219	+ .core_note_type = NT_RISCV_TAGGED_ADDR_CTRL,
	220	+ .n = 1,
	221	+ .size = sizeof(long),
	222	+ .align = sizeof(long),
	223	+ .regset_get = tagged_addr_ctrl_get,
	224	+ .set = tagged_addr_ctrl_set,
	225	+ },
	226	+#endif
185	227	};
186	228
187	229	static const struct user_regset_view riscv_user_native_view = {

arch/riscv/kernel/setup.c+37−0

@@ -244,6 +244,42 @@ static void __init parse_dtb(void) · 243 unmodified lines
244	244	#endif
245	245	}
246	246
	247	+#if defined(CONFIG_RISCV_COMBO_SPINLOCKS)
	248	+DEFINE_STATIC_KEY_TRUE(qspinlock_key);
	249	+EXPORT_SYMBOL(qspinlock_key);
	250	+#endif
	251	+
	252	+static void __init riscv_spinlock_init(void)
	253	+{
	254	+ char *using_ext = NULL;
	255	+
	256	+ if (IS_ENABLED(CONFIG_RISCV_TICKET_SPINLOCKS)) {
	257	+ pr_info("Ticket spinlock: enabled\n");
	258	+ return;
	259	+ }
	260	+
	261	+ if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) &&
	262	+ IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) &&
	263	+ riscv_isa_extension_available(NULL, ZABHA) &&
	264	+ riscv_isa_extension_available(NULL, ZACAS)) {
	265	+ using_ext = "using Zabha";
	266	+ } else if (riscv_isa_extension_available(NULL, ZICCRSE)) {
	267	+ using_ext = "using Ziccrse";
	268	+ }
	269	+#if defined(CONFIG_RISCV_COMBO_SPINLOCKS)
	270	+ else {
	271	+ static_branch_disable(&qspinlock_key);
	272	+ pr_info("Ticket spinlock: enabled\n");
	273	+ return;
	274	+ }
	275	+#endif
	276	+
	277	+ if (!using_ext)
	278	+ pr_err("Queued spinlock without Zabha or Ziccrse");
	279	+ else
	280	+ pr_info("Queued spinlock %s: enabled\n", using_ext);
	281	+}
	282	+
247	283	extern void __init init_rt_signal_env(void);
248	284
249	285	void __init setup_arch(char **cmdline_p)
@@ -297,6 +333,7 @@ void __init setup_arch(char **cmdline_p) · 47 unmodified lines
297	333	riscv_set_dma_cache_alignment();
298	334
299	335	riscv_user_isa_enable();
	336	+ riscv_spinlock_init();
300	337	}
301	338
302	339	bool arch_cpu_is_hotpluggable(int cpu)

arch/riscv/kernel/smpboot.c+0−2

@@ -233,8 +233,6 @@ asmlinkage __visible void smp_callin(void) · 232 unmodified lines
233	233	numa_add_cpu(curr_cpuid);
234	234	set_cpu_online(curr_cpuid, true);
235	235
236		− riscv_user_isa_enable();
237		−
238	236	/*
239	237	* Remote cache and TLB flushes are ignored while the CPU is offline,
240	238	* so flush them both right now just in case.

arch/riscv/kernel/suspend.c+2−2

@@ -14,7 +14,7 @@ · 13 unmodified lines
14	14
15	15	void suspend_save_csrs(struct suspend_context *context)
16	16	{
17		− if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_XLINUXENVCFG))
	17	+ if (riscv_has_extension_unlikely(RISCV_ISA_EXT_XLINUXENVCFG))
18	18	context->envcfg = csr_read(CSR_ENVCFG);
19	19	context->tvec = csr_read(CSR_TVEC);
20	20	context->ie = csr_read(CSR_IE);
@@ -37,7 +37,7 @@ void suspend_save_csrs(struct suspend_context *context) · 16 unmodified lines
37	37	void suspend_restore_csrs(struct suspend_context *context)
38	38	{
39	39	csr_write(CSR_SCRATCH, 0);
40		− if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_XLINUXENVCFG))
	40	+ if (riscv_has_extension_unlikely(RISCV_ISA_EXT_XLINUXENVCFG))
41	41	csr_write(CSR_ENVCFG, context->envcfg);
42	42	csr_write(CSR_TVEC, context->tvec);
43	43	csr_write(CSR_IE, context->ie);

arch/riscv/kernel/sys_hwprobe.c+44−0

@@ -150,6 +150,9 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, · 149 unmodified lines
150	150	EXT_KEY(ZFH);
151	151	EXT_KEY(ZFHMIN);
152	152	}
	153	+
	154	+ if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM))
	155	+ EXT_KEY(SUPM);
153	156	#undef EXT_KEY
154	157	}
155	158
@@ -201,6 +204,43 @@ static u64 hwprobe_misaligned(const struct cpumask *cpus) · 45 unmodified lines
201	204	}
202	205	#endif
203	206
	207	+#ifdef CONFIG_RISCV_VECTOR_MISALIGNED
	208	+static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
	209	+{
	210	+ int cpu;
	211	+ u64 perf = -1ULL;
	212	+
	213	+ /* Return if supported or not even if speed wasn't probed */
	214	+ for_each_cpu(cpu, cpus) {
	215	+ int this_perf = per_cpu(vector_misaligned_access, cpu);
	216	+
	217	+ if (perf == -1ULL)
	218	+ perf = this_perf;
	219	+
	220	+ if (perf != this_perf) {
	221	+ perf = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
	222	+ break;
	223	+ }
	224	+ }
	225	+
	226	+ if (perf == -1ULL)
	227	+ return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
	228	+
	229	+ return perf;
	230	+}
	231	+#else
	232	+static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
	233	+{
	234	+ if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS))
	235	+ return RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
	236	+
	237	+ if (IS_ENABLED(CONFIG_RISCV_SLOW_VECTOR_UNALIGNED_ACCESS))
	238	+ return RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
	239	+
	240	+ return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
	241	+}
	242	+#endif
	243	+
204	244	static void hwprobe_one_pair(struct riscv_hwprobe *pair,
205	245	const struct cpumask *cpus)
206	246	{
@@ -229,6 +269,10 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, · 22 unmodified lines
229	269	pair->value = hwprobe_misaligned(cpus);
230	270	break;
231	271
	272	+ case RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF:
	273	+ pair->value = hwprobe_vec_misaligned(cpus);
	274	+ break;
	275	+
232	276	case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE:
233	277	pair->value = 0;
234	278	if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ))

arch/riscv/kernel/traps_misaligned.c+128−11

@@ -16,6 +16,7 @@ · 15 unmodified lines
16	16	#include <asm/entry-common.h>
17	17	#include <asm/hwprobe.h>
18	18	#include <asm/cpufeature.h>
	19	+#include <asm/vector.h>
19	20
20	21	#define INSN_MATCH_LB 0x3
21	22	#define INSN_MASK_LB 0x707f
@@ -320,12 +321,37 @@ union reg_data { · 298 unmodified lines
320	321	u64 data_u64;
321	322	};
322	323
323		−static bool unaligned_ctl __read_mostly;
324		−
325	324	/* sysctl hooks */
326	325	int unaligned_enabled __read_mostly = 1; /* Enabled by default */
327	326
328		−int handle_misaligned_load(struct pt_regs *regs)
	327	+#ifdef CONFIG_RISCV_VECTOR_MISALIGNED
	328	+static int handle_vector_misaligned_load(struct pt_regs *regs)
	329	+{
	330	+ unsigned long epc = regs->epc;
	331	+ unsigned long insn;
	332	+
	333	+ if (get_insn(regs, epc, &insn))
	334	+ return -1;
	335	+
	336	+ /* Only return 0 when in check_vector_unaligned_access_emulated */
	337	+ if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
	338	+ *this_cpu_ptr(&vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
	339	+ regs->epc = epc + INSN_LEN(insn);
	340	+ return 0;
	341	+ }
	342	+
	343	+ /* If vector instruction we don't emulate it yet */
	344	+ regs->epc = epc;
	345	+ return -1;
	346	+}
	347	+#else
	348	+static int handle_vector_misaligned_load(struct pt_regs *regs)
	349	+{
	350	+ return -1;
	351	+}
	352	+#endif
	353	+
	354	+static int handle_scalar_misaligned_load(struct pt_regs *regs)
329	355	{
330	356	union reg_data val;
331	357	unsigned long epc = regs->epc;
@@ -433,7 +459,7 @@ int handle_misaligned_load(struct pt_regs *regs) · 101 unmodified lines
433	459	return 0;
434	460	}
435	461
436		−int handle_misaligned_store(struct pt_regs *regs)
	462	+static int handle_scalar_misaligned_store(struct pt_regs *regs)
437	463	{
438	464	union reg_data val;
439	465	unsigned long epc = regs->epc;
@@ -524,11 +550,96 @@ int handle_misaligned_store(struct pt_regs *regs) · 84 unmodified lines
524	550	return 0;
525	551	}
526	552
527		−static bool check_unaligned_access_emulated(int cpu)
	553	+int handle_misaligned_load(struct pt_regs *regs)
	554	+{
	555	+ unsigned long epc = regs->epc;
	556	+ unsigned long insn;
	557	+
	558	+ if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) {
	559	+ if (get_insn(regs, epc, &insn))
	560	+ return -1;
	561	+
	562	+ if (insn_is_vector(insn))
	563	+ return handle_vector_misaligned_load(regs);
	564	+ }
	565	+
	566	+ if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
	567	+ return handle_scalar_misaligned_load(regs);
	568	+
	569	+ return -1;
	570	+}
	571	+
	572	+int handle_misaligned_store(struct pt_regs *regs)
528	573	{
	574	+ if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
	575	+ return handle_scalar_misaligned_store(regs);
	576	+
	577	+ return -1;
	578	+}
	579	+
	580	+#ifdef CONFIG_RISCV_VECTOR_MISALIGNED
	581	+void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused)
	582	+{
	583	+ long *mas_ptr = this_cpu_ptr(&vector_misaligned_access);
	584	+ unsigned long tmp_var;
	585	+
	586	+ *mas_ptr = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
	587	+
	588	+ kernel_vector_begin();
	589	+ /*
	590	+ * In pre-13.0.0 versions of GCC, vector registers cannot appear in
	591	+ * the clobber list. This inline asm clobbers v0, but since we do not
	592	+ * currently build the kernel with V enabled, the v0 clobber arg is not
	593	+ * needed (as the compiler will not emit vector code itself). If the kernel
	594	+ * is changed to build with V enabled, the clobber arg will need to be
	595	+ * added here.
	596	+ */
	597	+ __asm__ __volatile__ (
	598	+ ".balign 4\n\t"
	599	+ ".option push\n\t"
	600	+ ".option arch, +zve32x\n\t"
	601	+ " vsetivli zero, 1, e16, m1, ta, ma\n\t" // Vectors of 16b
	602	+ " vle16.v v0, (%[ptr])\n\t" // Load bytes
	603	+ ".option pop\n\t"
	604	+ : : [ptr] "r" ((u8 *)&tmp_var + 1));
	605	+ kernel_vector_end();
	606	+}
	607	+
	608	+bool check_vector_unaligned_access_emulated_all_cpus(void)
	609	+{
	610	+ int cpu;
	611	+
	612	+ if (!has_vector()) {
	613	+ for_each_online_cpu(cpu)
	614	+ per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
	615	+ return false;
	616	+ }
	617	+
	618	+ schedule_on_each_cpu(check_vector_unaligned_access_emulated);
	619	+
	620	+ for_each_online_cpu(cpu)
	621	+ if (per_cpu(vector_misaligned_access, cpu)
	622	+ == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
	623	+ return false;
	624	+
	625	+ return true;
	626	+}
	627	+#else
	628	+bool check_vector_unaligned_access_emulated_all_cpus(void)
	629	+{
	630	+ return false;
	631	+}
	632	+#endif
	633	+
	634	+#ifdef CONFIG_RISCV_SCALAR_MISALIGNED
	635	+
	636	+static bool unaligned_ctl __read_mostly;
	637	+
	638	+void check_unaligned_access_emulated(struct work_struct *work __always_unused)
	639	+{
	640	+ int cpu = smp_processor_id();
529	641	long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);
530	642	unsigned long tmp_var, tmp_val;
531		− bool misaligned_emu_detected;
532	643
533	644	*mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
534	645
@@ -536,19 +647,16 @@ static bool check_unaligned_access_emulated(int cpu) · 1 unmodified line
536	647	" "REG_L" %[tmp], 1(%[ptr])\n"
537	648	: [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory");
538	649
539		− misaligned_emu_detected = (*mas_ptr == RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED);
540	650	/*
541	651	* If unaligned_ctl is already set, this means that we detected that all
542	652	* CPUS uses emulated misaligned access at boot time. If that changed
543	653	* when hotplugging the new cpu, this is something we don't handle.
544	654	*/
545		− if (unlikely(unaligned_ctl && !misaligned_emu_detected)) {
	655	+ if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) {
546	656	pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n");
547	657	while (true)
548	658	cpu_relax();
549	659	}
550		−
551		− return misaligned_emu_detected;
552	660	}
553	661
554	662	bool check_unaligned_access_emulated_all_cpus(void)
@@ -560,8 +668,11 @@ bool check_unaligned_access_emulated_all_cpus(void) · 5 unmodified lines
560	668	* accesses emulated since tasks requesting such control can run on any
561	669	* CPU.
562	670	*/
	671	+ schedule_on_each_cpu(check_unaligned_access_emulated);
	672	+
563	673	for_each_online_cpu(cpu)
564		− if (!check_unaligned_access_emulated(cpu))
	674	+ if (per_cpu(misaligned_access_speed, cpu)
	675	+ != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED)
565	676	return false;
566	677
567	678	unaligned_ctl = true;
@@ -572,3 +683,9 @@ bool unaligned_ctl_available(void) · 4 unmodified lines
572	683	{
573	684	return unaligned_ctl;
574	685	}
	686	+#else
	687	+bool check_unaligned_access_emulated_all_cpus(void)
	688	+{
	689	+ return false;
	690	+}
	691	+#endif

arch/riscv/kernel/unaligned_access_speed.c+148−8

@@ -6,11 +6,13 @@ · 5 unmodified lines
6	6	#include <linux/cpu.h>
7	7	#include <linux/cpumask.h>
8	8	#include <linux/jump_label.h>
	9	+#include <linux/kthread.h>
9	10	#include <linux/mm.h>
10	11	#include <linux/smp.h>
11	12	#include <linux/types.h>
12	13	#include <asm/cpufeature.h>
13	14	#include <asm/hwprobe.h>
	15	+#include <asm/vector.h>
14	16
15	17	#include "copy-unaligned.h"
16	18
@@ -19,7 +21,8 @@ · 2 unmodified lines
19	21	#define MISALIGNED_BUFFER_ORDER get_order(MISALIGNED_BUFFER_SIZE)
20	22	#define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80)
21	23
22		−DEFINE_PER_CPU(long, misaligned_access_speed);
	24	+DEFINE_PER_CPU(long, misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
	25	+DEFINE_PER_CPU(long, vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
23	26
24	27	#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
25	28	static cpumask_t fast_misaligned_access;
@@ -191,6 +194,7 @@ static int riscv_online_cpu(unsigned int cpu) · 165 unmodified lines
191	194	if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN)
192	195	goto exit;
193	196
	197	+ check_unaligned_access_emulated(NULL);
194	198	buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
195	199	if (!buf) {
196	200	pr_warn("Allocation failure, not measuring misaligned performance\n");
@@ -259,23 +263,159 @@ out: · 62 unmodified lines
259	263	kfree(bufs);
260	264	return 0;
261	265	}
	266	+#else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */
	267	+static int check_unaligned_access_speed_all_cpus(void)
	268	+{
	269	+ return 0;
	270	+}
	271	+#endif
262	272
263		−static int check_unaligned_access_all_cpus(void)
	273	+#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
	274	+static void check_vector_unaligned_access(struct work_struct *work __always_unused)
264	275	{
265		− bool all_cpus_emulated = check_unaligned_access_emulated_all_cpus();
	276	+ int cpu = smp_processor_id();
	277	+ u64 start_cycles, end_cycles;
	278	+ u64 word_cycles;
	279	+ u64 byte_cycles;
	280	+ int ratio;
	281	+ unsigned long start_jiffies, now;
	282	+ struct page *page;
	283	+ void *dst;
	284	+ void *src;
	285	+ long speed = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
266	286
267		− if (!all_cpus_emulated)
268		− return check_unaligned_access_speed_all_cpus();
	287	+ if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
	288	+ return;
	289	+
	290	+ page = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
	291	+ if (!page) {
	292	+ pr_warn("Allocation failure, not measuring vector misaligned performance\n");
	293	+ return;
	294	+ }
	295	+
	296	+ /* Make an unaligned destination buffer. */
	297	+ dst = (void *)((unsigned long)page_address(page) \| 0x1);
	298	+ /* Unalign src as well, but differently (off by 1 + 2 = 3). */
	299	+ src = dst + (MISALIGNED_BUFFER_SIZE / 2);
	300	+ src += 2;
	301	+ word_cycles = -1ULL;
	302	+
	303	+ /* Do a warmup. */
	304	+ kernel_vector_begin();
	305	+ __riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE);
269	306
	307	+ start_jiffies = jiffies;
	308	+ while ((now = jiffies) == start_jiffies)
	309	+ cpu_relax();
	310	+
	311	+ /*
	312	+ * For a fixed amount of time, repeatedly try the function, and take
	313	+ * the best time in cycles as the measurement.
	314	+ */
	315	+ while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) {
	316	+ start_cycles = get_cycles64();
	317	+ /* Ensure the CSR read can't reorder WRT to the copy. */
	318	+ mb();
	319	+ __riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE);
	320	+ /* Ensure the copy ends before the end time is snapped. */
	321	+ mb();
	322	+ end_cycles = get_cycles64();
	323	+ if ((end_cycles - start_cycles) < word_cycles)
	324	+ word_cycles = end_cycles - start_cycles;
	325	+ }
	326	+
	327	+ byte_cycles = -1ULL;
	328	+ __riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE);
	329	+ start_jiffies = jiffies;
	330	+ while ((now = jiffies) == start_jiffies)
	331	+ cpu_relax();
	332	+
	333	+ while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) {
	334	+ start_cycles = get_cycles64();
	335	+ /* Ensure the CSR read can't reorder WRT to the copy. */
	336	+ mb();
	337	+ __riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE);
	338	+ /* Ensure the copy ends before the end time is snapped. */
	339	+ mb();
	340	+ end_cycles = get_cycles64();
	341	+ if ((end_cycles - start_cycles) < byte_cycles)
	342	+ byte_cycles = end_cycles - start_cycles;
	343	+ }
	344	+
	345	+ kernel_vector_end();
	346	+
	347	+ /* Don't divide by zero. */
	348	+ if (!word_cycles \|\| !byte_cycles) {
	349	+ pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned vector access speed\n",
	350	+ cpu);
	351	+
	352	+ return;
	353	+ }
	354	+
	355	+ if (word_cycles < byte_cycles)
	356	+ speed = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
	357	+
	358	+ ratio = div_u64((byte_cycles * 100), word_cycles);
	359	+ pr_info("cpu%d: Ratio of vector byte access time to vector unaligned word access is %d.%02d, unaligned accesses are %s\n",
	360	+ cpu,
	361	+ ratio / 100,
	362	+ ratio % 100,
	363	+ (speed == RISCV_HWPROBE_MISALIGNED_VECTOR_FAST) ? "fast" : "slow");
	364	+
	365	+ per_cpu(vector_misaligned_access, cpu) = speed;
	366	+}
	367	+
	368	+static int riscv_online_cpu_vec(unsigned int cpu)
	369	+{
	370	+ if (!has_vector())
	371	+ return 0;
	372	+
	373	+ if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED)
	374	+ return 0;
	375	+
	376	+ check_vector_unaligned_access_emulated(NULL);
	377	+ check_vector_unaligned_access(NULL);
270	378	return 0;
271	379	}
272		−#else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */
273		−static int check_unaligned_access_all_cpus(void)
	380	+
	381	+/* Measure unaligned access speed on all CPUs present at boot in parallel. */
	382	+static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused)
274	383	{
275		− check_unaligned_access_emulated_all_cpus();
	384	+ schedule_on_each_cpu(check_vector_unaligned_access);
	385	+
	386	+ /*
	387	+ * Setup hotplug callbacks for any new CPUs that come online or go
	388	+ * offline.
	389	+ */
	390	+ cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online",
	391	+ riscv_online_cpu_vec, NULL);
276	392
277	393	return 0;
278	394	}
	395	+#else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */
	396	+static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused)
	397	+{
	398	+ return 0;
	399	+}
279	400	#endif
280	401
	402	+static int check_unaligned_access_all_cpus(void)
	403	+{
	404	+ bool all_cpus_emulated, all_cpus_vec_unsupported;
	405	+
	406	+ all_cpus_emulated = check_unaligned_access_emulated_all_cpus();
	407	+ all_cpus_vec_unsupported = check_vector_unaligned_access_emulated_all_cpus();
	408	+
	409	+ if (!all_cpus_vec_unsupported &&
	410	+ IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) {
	411	+ kthread_run(vec_check_unaligned_access_speed_all_cpus,
	412	+ NULL, "vec_check_unaligned_access_speed_all_cpus");
	413	+ }
	414	+
	415	+ if (!all_cpus_emulated)
	416	+ return check_unaligned_access_speed_all_cpus();
	417	+
	418	+ return 0;
	419	+}
	420	+
281	421	arch_initcall(check_unaligned_access_all_cpus);

arch/riscv/kernel/vdso/Makefile+5−4

@@ -45,7 +45,7 @@ $(obj)/vdso.o: $(obj)/vdso.so · 44 unmodified lines
45	45
46	46	# link rule for the .so file, .lds has to be first
47	47	$(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE
48		− $(call if_changed,vdsold)
	48	+ $(call if_changed,vdsold_and_check)
49	49	LDFLAGS_vdso.so.dbg = -shared -soname=linux-vdso.so.1 \
50	50	--build-id=sha1 --hash-style=both --eh-frame-hdr
51	51
@@ -65,7 +65,8 @@ include/generated/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE · 13 unmodified lines
65	65	# actual build commands
66	66	# The DSO images are built using a special linker script
67	67	# Make sure only to export the intended __vdso_xxx symbol offsets.
68		−quiet_cmd_vdsold = VDSOLD $@
69		− cmd_vdsold = $(LD) $(ld_flags) -T $(filter-out FORCE,$^) -o $@.tmp && \
	68	+quiet_cmd_vdsold_and_check = VDSOLD $@
	69	+ cmd_vdsold_and_check = $(LD) $(ld_flags) -T $(filter-out FORCE,$^) -o $@.tmp && \
70	70	$(OBJCOPY) $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \
71		− rm $@.tmp
	71	+ rm $@.tmp && \
	72	+ $(cmd_vdso_check)

arch/riscv/kernel/vec-copy-unaligned.S+58−0

@@ -0,0 +1,58 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/* Copyright (C) 2024 Rivos Inc. */
	3	+
	4	+#include <linux/args.h>
	5	+#include <linux/linkage.h>
	6	+#include <asm/asm.h>
	7	+
	8	+ .text
	9	+
	10	+#define WORD_EEW 32
	11	+
	12	+#define WORD_SEW CONCATENATE(e, WORD_EEW)
	13	+#define VEC_L CONCATENATE(vle, WORD_EEW).v
	14	+#define VEC_S CONCATENATE(vle, WORD_EEW).v
	15	+
	16	+/* void __riscv_copy_vec_words_unaligned(void , const void , size_t) */
	17	+/* Performs a memcpy without aligning buffers, using word loads and stores. */
	18	+/* Note: The size is truncated to a multiple of WORD_EEW */
	19	+SYM_FUNC_START(__riscv_copy_vec_words_unaligned)
	20	+ andi a4, a2, ~(WORD_EEW-1)
	21	+ beqz a4, 2f
	22	+ add a3, a1, a4
	23	+ .option push
	24	+ .option arch, +zve32x
	25	+1:
	26	+ vsetivli t0, 8, WORD_SEW, m8, ta, ma
	27	+ VEC_L v0, (a1)
	28	+ VEC_S v0, (a0)
	29	+ addi a0, a0, WORD_EEW
	30	+ addi a1, a1, WORD_EEW
	31	+ bltu a1, a3, 1b
	32	+
	33	+2:
	34	+ .option pop
	35	+ ret
	36	+SYM_FUNC_END(__riscv_copy_vec_words_unaligned)
	37	+
	38	+/* void __riscv_copy_vec_bytes_unaligned(void , const void , size_t) */
	39	+/* Performs a memcpy without aligning buffers, using only byte accesses. */
	40	+/* Note: The size is truncated to a multiple of 8 */
	41	+SYM_FUNC_START(__riscv_copy_vec_bytes_unaligned)
	42	+ andi a4, a2, ~(8-1)
	43	+ beqz a4, 2f
	44	+ add a3, a1, a4
	45	+ .option push
	46	+ .option arch, +zve32x
	47	+1:
	48	+ vsetivli t0, 8, e8, m8, ta, ma
	49	+ vle8.v v0, (a1)
	50	+ vse8.v v0, (a0)
	51	+ addi a0, a0, 8
	52	+ addi a1, a1, 8
	53	+ bltu a1, a3, 1b
	54	+
	55	+2:
	56	+ .option pop
	57	+ ret
	58	+SYM_FUNC_END(__riscv_copy_vec_bytes_unaligned)

arch/riscv/kernel/vector.c+1−1

@@ -66,7 +66,7 @@ void __init riscv_v_setup_ctx_cache(void) · 65 unmodified lines
66	66	#endif
67	67	}
68	68
69		−static bool insn_is_vector(u32 insn_buf)
	69	+bool insn_is_vector(u32 insn_buf)
70	70	{
71	71	u32 opcode = insn_buf & __INSN_OPCODE_MASK;
72	72	u32 width, csr;

arch/riscv/kvm/vcpu_onereg.c+4−0

@@ -34,9 +34,11 @@ static const unsigned long kvm_isa_ext_arr[] = { · 33 unmodified lines
34	34	[KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m,
35	35	[KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v,
36	36	/* Multi letter extensions (alphabetically sorted) */
	37	+ [KVM_RISCV_ISA_EXT_SMNPM] = RISCV_ISA_EXT_SSNPM,
37	38	KVM_ISA_EXT_ARR(SMSTATEEN),
38	39	KVM_ISA_EXT_ARR(SSAIA),
39	40	KVM_ISA_EXT_ARR(SSCOFPMF),
	41	+ KVM_ISA_EXT_ARR(SSNPM),
40	42	KVM_ISA_EXT_ARR(SSTC),
41	43	KVM_ISA_EXT_ARR(SVINVAL),
42	44	KVM_ISA_EXT_ARR(SVNAPOT),
@@ -127,8 +129,10 @@ static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) · 84 unmodified lines
127	129	case KVM_RISCV_ISA_EXT_C:
128	130	case KVM_RISCV_ISA_EXT_I:
129	131	case KVM_RISCV_ISA_EXT_M:
	132	+ case KVM_RISCV_ISA_EXT_SMNPM:
130	133	/* There is not architectural config bit to disable sscofpmf completely */
131	134	case KVM_RISCV_ISA_EXT_SSCOFPMF:
	135	+ case KVM_RISCV_ISA_EXT_SSNPM:
132	136	case KVM_RISCV_ISA_EXT_SSTC:
133	137	case KVM_RISCV_ISA_EXT_SVINVAL:
134	138	case KVM_RISCV_ISA_EXT_SVNAPOT:

drivers/cpuidle/cpuidle-riscv-sbi.c+9−2

@@ -26,6 +26,7 @@ · 25 unmodified lines
26	26	#include <asm/smp.h>
27	27	#include <asm/suspend.h>
28	28
	29	+#include "cpuidle.h"
29	30	#include "dt_idle_states.h"
30	31	#include "dt_idle_genpd.h"
31	32
@@ -329,6 +330,9 @@ static int sbi_cpuidle_init_cpu(struct device *dev, int cpu) · 297 unmodified lines
329	330	return ret;
330	331	}
331	332
	333	+ if (cpuidle_disabled())
	334	+ return 0;
	335	+
332	336	ret = cpuidle_register(drv, NULL);
333	337	if (ret)
334	338	goto deinit;
@@ -538,7 +542,10 @@ static int sbi_cpuidle_probe(struct platform_device *pdev) · 203 unmodified lines
538	542	/* Setup CPU hotplut notifiers */
539	543	sbi_idle_init_cpuhp();
540	544
541		− pr_info("idle driver registered for all CPUs\n");
	545	+ if (cpuidle_disabled())
	546	+ pr_info("cpuidle is disabled\n");
	547	+ else
	548	+ pr_info("idle driver registered for all CPUs\n");
542	549
543	550	return 0;
544	551
@@ -582,4 +589,4 @@ static int __init sbi_cpuidle_init(void) · 37 unmodified lines
582	589
583	590	return 0;
584	591	}
585		−device_initcall(sbi_cpuidle_init);
	592	+arch_initcall(sbi_cpuidle_init);

include/asm-generic/qspinlock.h+2−0

@@ -136,6 +136,7 @@ static __always_inline bool virt_spin_lock(struct qspinlock *lock) · 135 unmodified lines
136	136	}
137	137	#endif
138	138
	139	+#ifndef __no_arch_spinlock_redefine
139	140	/*
140	141	* Remapping spinlock architecture specific functions to the corresponding
141	142	* queued spinlock functions.
@@ -146,5 +147,6 @@ static __always_inline bool virt_spin_lock(struct qspinlock *lock) · 4 unmodified lines
146	147	#define arch_spin_lock(l) queued_spin_lock(l)
147	148	#define arch_spin_trylock(l) queued_spin_trylock(l)
148	149	#define arch_spin_unlock(l) queued_spin_unlock(l)
	150	+#endif
149	151
150	152	#endif /* __ASM_GENERIC_QSPINLOCK_H */

include/asm-generic/spinlock.h+1−86

@@ -1,94 +1,9 @@
1	1	/* SPDX-License-Identifier: GPL-2.0 */
2	2
3		−/*
4		− * 'Generic' ticket-lock implementation.
5		− *
6		− * It relies on atomic_fetch_add() having well defined forward progress
7		− * guarantees under contention. If your architecture cannot provide this, stick
8		− * to a test-and-set lock.
9		− *
10		− * It also relies on atomic_fetch_add() being safe vs smp_store_release() on a
11		− * sub-word of the value. This is generally true for anything LL/SC although
12		− * you'd be hard pressed to find anything useful in architecture specifications
13		− * about this. If your architecture cannot do this you might be better off with
14		− * a test-and-set.
15		− *
16		− * It further assumes atomic__release() + atomic__acquire() is RCpc and hence
17		− * uses atomic_fetch_add() which is RCsc to create an RCsc hot path, along with
18		− * a full fence after the spin to upgrade the otherwise-RCpc
19		− * atomic_cond_read_acquire().
20		− *
21		− * The implementation uses smp_cond_load_acquire() to spin, so if the
22		− * architecture has WFE like instructions to sleep instead of poll for word
23		− * modifications be sure to implement that (see ARM64 for example).
24		− *
25		− */
26		−
27	3	#ifndef __ASM_GENERIC_SPINLOCK_H
28	4	#define __ASM_GENERIC_SPINLOCK_H
29	5
30		−#include <linux/atomic.h>
31		−#include <asm-generic/spinlock_types.h>
32		−
33		−static __always_inline void arch_spin_lock(arch_spinlock_t *lock)
34		−{
35		− u32 val = atomic_fetch_add(1<<16, lock);
36		− u16 ticket = val >> 16;
37		−
38		− if (ticket == (u16)val)
39		− return;
40		−
41		− /*
42		− * atomic_cond_read_acquire() is RCpc, but rather than defining a
43		− * custom cond_read_rcsc() here we just emit a full fence. We only
44		− * need the prior reads before subsequent writes ordering from
45		− * smb_mb(), but as atomic_cond_read_acquire() just emits reads and we
46		− * have no outstanding writes due to the atomic_fetch_add() the extra
47		− * orderings are free.
48		− */
49		− atomic_cond_read_acquire(lock, ticket == (u16)VAL);
50		− smp_mb();
51		−}
52		−
53		−static __always_inline bool arch_spin_trylock(arch_spinlock_t *lock)
54		−{
55		− u32 old = atomic_read(lock);
56		−
57		− if ((old >> 16) != (old & 0xffff))
58		− return false;
59		−
60		− return atomic_try_cmpxchg(lock, &old, old + (1<<16)); /* SC, for RCsc */
61		−}
62		−
63		−static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
64		−{
65		− u16 ptr = (u16 )lock + IS_ENABLED(CONFIG_CPU_BIG_ENDIAN);
66		− u32 val = atomic_read(lock);
67		−
68		− smp_store_release(ptr, (u16)val + 1);
69		−}
70		−
71		−static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
72		−{
73		− u32 val = lock.counter;
74		−
75		− return ((val >> 16) == (val & 0xffff));
76		−}
77		−
78		−static __always_inline int arch_spin_is_locked(arch_spinlock_t *lock)
79		−{
80		− arch_spinlock_t val = READ_ONCE(*lock);
81		−
82		− return !arch_spin_value_unlocked(val);
83		−}
84		−
85		−static __always_inline int arch_spin_is_contended(arch_spinlock_t *lock)
86		−{
87		− u32 val = atomic_read(lock);
88		−
89		− return (s16)((val >> 16) - (val & 0xffff)) > 1;
90		−}
91		−
	6	+#include <asm-generic/ticket_spinlock.h>
92	7	#include <asm/qrwlock.h>
93	8
94	9	#endif /* __ASM_GENERIC_SPINLOCK_H */

include/asm-generic/spinlock_types.h+2−10

@@ -3,15 +3,7 @@ · 2 unmodified lines
3	3	#ifndef __ASM_GENERIC_SPINLOCK_TYPES_H
4	4	#define __ASM_GENERIC_SPINLOCK_TYPES_H
5	5
6		−#include <linux/types.h>
7		−typedef atomic_t arch_spinlock_t;
8		−
9		−/*
10		− * qrwlock_types depends on arch_spinlock_t, so we must typedef that before the
11		− * include.
12		− */
13		−#include <asm/qrwlock_types.h>
14		−
15		−#define __ARCH_SPIN_LOCK_UNLOCKED ATOMIC_INIT(0)
	6	+#include <asm-generic/qspinlock_types.h>
	7	+#include <asm-generic/qrwlock_types.h>
16	8
17	9	#endif /* __ASM_GENERIC_SPINLOCK_TYPES_H */

include/asm-generic/ticket_spinlock.h+105−0

@@ -0,0 +1,105 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+
	3	+/*
	4	+ * 'Generic' ticket-lock implementation.
	5	+ *
	6	+ * It relies on atomic_fetch_add() having well defined forward progress
	7	+ * guarantees under contention. If your architecture cannot provide this, stick
	8	+ * to a test-and-set lock.
	9	+ *
	10	+ * It also relies on atomic_fetch_add() being safe vs smp_store_release() on a
	11	+ * sub-word of the value. This is generally true for anything LL/SC although
	12	+ * you'd be hard pressed to find anything useful in architecture specifications
	13	+ * about this. If your architecture cannot do this you might be better off with
	14	+ * a test-and-set.
	15	+ *
	16	+ * It further assumes atomic__release() + atomic__acquire() is RCpc and hence
	17	+ * uses atomic_fetch_add() which is RCsc to create an RCsc hot path, along with
	18	+ * a full fence after the spin to upgrade the otherwise-RCpc
	19	+ * atomic_cond_read_acquire().
	20	+ *
	21	+ * The implementation uses smp_cond_load_acquire() to spin, so if the
	22	+ * architecture has WFE like instructions to sleep instead of poll for word
	23	+ * modifications be sure to implement that (see ARM64 for example).
	24	+ *
	25	+ */
	26	+
	27	+#ifndef __ASM_GENERIC_TICKET_SPINLOCK_H
	28	+#define __ASM_GENERIC_TICKET_SPINLOCK_H
	29	+
	30	+#include <linux/atomic.h>
	31	+#include <asm-generic/spinlock_types.h>
	32	+
	33	+static __always_inline void ticket_spin_lock(arch_spinlock_t *lock)
	34	+{
	35	+ u32 val = atomic_fetch_add(1<<16, &lock->val);
	36	+ u16 ticket = val >> 16;
	37	+
	38	+ if (ticket == (u16)val)
	39	+ return;
	40	+
	41	+ /*
	42	+ * atomic_cond_read_acquire() is RCpc, but rather than defining a
	43	+ * custom cond_read_rcsc() here we just emit a full fence. We only
	44	+ * need the prior reads before subsequent writes ordering from
	45	+ * smb_mb(), but as atomic_cond_read_acquire() just emits reads and we
	46	+ * have no outstanding writes due to the atomic_fetch_add() the extra
	47	+ * orderings are free.
	48	+ */
	49	+ atomic_cond_read_acquire(&lock->val, ticket == (u16)VAL);
	50	+ smp_mb();
	51	+}
	52	+
	53	+static __always_inline bool ticket_spin_trylock(arch_spinlock_t *lock)
	54	+{
	55	+ u32 old = atomic_read(&lock->val);
	56	+
	57	+ if ((old >> 16) != (old & 0xffff))
	58	+ return false;
	59	+
	60	+ return atomic_try_cmpxchg(&lock->val, &old, old + (1<<16)); /* SC, for RCsc */
	61	+}
	62	+
	63	+static __always_inline void ticket_spin_unlock(arch_spinlock_t *lock)
	64	+{
	65	+ u16 ptr = (u16 )lock + IS_ENABLED(CONFIG_CPU_BIG_ENDIAN);
	66	+ u32 val = atomic_read(&lock->val);
	67	+
	68	+ smp_store_release(ptr, (u16)val + 1);
	69	+}
	70	+
	71	+static __always_inline int ticket_spin_value_unlocked(arch_spinlock_t lock)
	72	+{
	73	+ u32 val = lock.val.counter;
	74	+
	75	+ return ((val >> 16) == (val & 0xffff));
	76	+}
	77	+
	78	+static __always_inline int ticket_spin_is_locked(arch_spinlock_t *lock)
	79	+{
	80	+ arch_spinlock_t val = READ_ONCE(*lock);
	81	+
	82	+ return !ticket_spin_value_unlocked(val);
	83	+}
	84	+
	85	+static __always_inline int ticket_spin_is_contended(arch_spinlock_t *lock)
	86	+{
	87	+ u32 val = atomic_read(&lock->val);
	88	+
	89	+ return (s16)((val >> 16) - (val & 0xffff)) > 1;
	90	+}
	91	+
	92	+#ifndef __no_arch_spinlock_redefine
	93	+/*
	94	+ * Remapping spinlock architecture specific functions to the corresponding
	95	+ * ticket spinlock functions.
	96	+ */
	97	+#define arch_spin_is_locked(l) ticket_spin_is_locked(l)
	98	+#define arch_spin_is_contended(l) ticket_spin_is_contended(l)
	99	+#define arch_spin_value_unlocked(l) ticket_spin_value_unlocked(l)
	100	+#define arch_spin_lock(l) ticket_spin_lock(l)
	101	+#define arch_spin_trylock(l) ticket_spin_trylock(l)
	102	+#define arch_spin_unlock(l) ticket_spin_unlock(l)
	103	+#endif
	104	+
	105	+#endif /* __ASM_GENERIC_TICKET_SPINLOCK_H */

include/uapi/linux/elf.h+1−0

@@ -451,6 +451,7 @@ typedef struct elf64_shdr { · 450 unmodified lines
451	451	#define NT_MIPS_MSA 0x802 /* MIPS SIMD registers */
452	452	#define NT_RISCV_CSR 0x900 /* RISC-V Control and Status Registers */
453	453	#define NT_RISCV_VECTOR 0x901 /* RISC-V vector registers */
	454	+#define NT_RISCV_TAGGED_ADDR_CTRL 0x902 /* RISC-V tagged address control (prctl()) */
454	455	#define NT_LOONGARCH_CPUCFG 0xa00 /* LoongArch CPU config registers */
455	456	#define NT_LOONGARCH_CSR 0xa01 /* LoongArch control and status registers */
456	457	#define NT_LOONGARCH_LSX 0xa02 /* LoongArch Loongson SIMD Extension registers */

include/uapi/linux/prctl.h+4−1

@@ -230,7 +230,7 @@ struct prctl_mm_map { · 229 unmodified lines
230	230	# define PR_PAC_APDBKEY (1UL << 3)
231	231	# define PR_PAC_APGAKEY (1UL << 4)
232	232
233		−/* Tagged user address controls for arm64 */
	233	+/* Tagged user address controls for arm64 and RISC-V */
234	234	#define PR_SET_TAGGED_ADDR_CTRL 55
235	235	#define PR_GET_TAGGED_ADDR_CTRL 56
236	236	# define PR_TAGGED_ADDR_ENABLE (1UL << 0)
@@ -244,6 +244,9 @@ struct prctl_mm_map { · 7 unmodified lines
244	244	# define PR_MTE_TAG_MASK (0xffffUL << PR_MTE_TAG_SHIFT)
245	245	/* Unused; kept only for source compatibility */
246	246	# define PR_MTE_TCF_SHIFT 1
	247	+/* RISC-V pointer masking tag length */
	248	+# define PR_PMLEN_SHIFT 24
	249	+# define PR_PMLEN_MASK (0x7fUL << PR_PMLEN_SHIFT)
247	250
248	251	/* Control reclaim behavior when allocating memory */
249	252	#define PR_SET_IO_FLUSHER 57

tools/testing/selftests/kvm/riscv/get-reg-list.c+8−0

@@ -41,9 +41,11 @@ bool filter_reg(__u64 reg) · 40 unmodified lines
41	41	case KVM_REG_RISCV_ISA_EXT \| KVM_REG_RISCV_ISA_SINGLE \| KVM_RISCV_ISA_EXT_I:
42	42	case KVM_REG_RISCV_ISA_EXT \| KVM_REG_RISCV_ISA_SINGLE \| KVM_RISCV_ISA_EXT_M:
43	43	case KVM_REG_RISCV_ISA_EXT \| KVM_REG_RISCV_ISA_SINGLE \| KVM_RISCV_ISA_EXT_V:
	44	+ case KVM_REG_RISCV_ISA_EXT \| KVM_REG_RISCV_ISA_SINGLE \| KVM_RISCV_ISA_EXT_SMNPM:
44	45	case KVM_REG_RISCV_ISA_EXT \| KVM_REG_RISCV_ISA_SINGLE \| KVM_RISCV_ISA_EXT_SMSTATEEN:
45	46	case KVM_REG_RISCV_ISA_EXT \| KVM_REG_RISCV_ISA_SINGLE \| KVM_RISCV_ISA_EXT_SSAIA:
46	47	case KVM_REG_RISCV_ISA_EXT \| KVM_REG_RISCV_ISA_SINGLE \| KVM_RISCV_ISA_EXT_SSCOFPMF:
	48	+ case KVM_REG_RISCV_ISA_EXT \| KVM_REG_RISCV_ISA_SINGLE \| KVM_RISCV_ISA_EXT_SSNPM:
47	49	case KVM_REG_RISCV_ISA_EXT \| KVM_REG_RISCV_ISA_SINGLE \| KVM_RISCV_ISA_EXT_SSTC:
48	50	case KVM_REG_RISCV_ISA_EXT \| KVM_REG_RISCV_ISA_SINGLE \| KVM_RISCV_ISA_EXT_SVINVAL:
49	51	case KVM_REG_RISCV_ISA_EXT \| KVM_REG_RISCV_ISA_SINGLE \| KVM_RISCV_ISA_EXT_SVNAPOT:
@@ -414,9 +416,11 @@ static const char *isa_ext_single_id_to_str(__u64 reg_off) · 364 unmodified lines
414	416	KVM_ISA_EXT_ARR(I),
415	417	KVM_ISA_EXT_ARR(M),
416	418	KVM_ISA_EXT_ARR(V),
	419	+ KVM_ISA_EXT_ARR(SMNPM),
417	420	KVM_ISA_EXT_ARR(SMSTATEEN),
418	421	KVM_ISA_EXT_ARR(SSAIA),
419	422	KVM_ISA_EXT_ARR(SSCOFPMF),
	423	+ KVM_ISA_EXT_ARR(SSNPM),
420	424	KVM_ISA_EXT_ARR(SSTC),
421	425	KVM_ISA_EXT_ARR(SVINVAL),
422	426	KVM_ISA_EXT_ARR(SVNAPOT),
@@ -946,8 +950,10 @@ KVM_ISA_EXT_SUBLIST_CONFIG(aia, AIA); · 523 unmodified lines
946	950	KVM_ISA_EXT_SUBLIST_CONFIG(fp_f, FP_F);
947	951	KVM_ISA_EXT_SUBLIST_CONFIG(fp_d, FP_D);
948	952	KVM_ISA_EXT_SIMPLE_CONFIG(h, H);
	953	+KVM_ISA_EXT_SIMPLE_CONFIG(smnpm, SMNPM);
949	954	KVM_ISA_EXT_SUBLIST_CONFIG(smstateen, SMSTATEEN);
950	955	KVM_ISA_EXT_SIMPLE_CONFIG(sscofpmf, SSCOFPMF);
	956	+KVM_ISA_EXT_SIMPLE_CONFIG(ssnpm, SSNPM);
951	957	KVM_ISA_EXT_SIMPLE_CONFIG(sstc, SSTC);
952	958	KVM_ISA_EXT_SIMPLE_CONFIG(svinval, SVINVAL);
953	959	KVM_ISA_EXT_SIMPLE_CONFIG(svnapot, SVNAPOT);
@@ -1009,8 +1015,10 @@ struct vcpu_reg_list *vcpu_configs[] = { · 55 unmodified lines
1009	1015	&config_fp_f,
1010	1016	&config_fp_d,
1011	1017	&config_h,
	1018	+ &config_smnpm,
1012	1019	&config_smstateen,
1013	1020	&config_sscofpmf,
	1021	+ &config_ssnpm,
1014	1022	&config_sstc,
1015	1023	&config_svinval,
1016	1024	&config_svnapot,

tools/testing/selftests/riscv/Makefile+1−1

@@ -5,7 +5,7 @@ · 4 unmodified lines
5	5	ARCH ?= $(shell uname -m 2>/dev/null \|\| echo not)
6	6
7	7	ifneq (,$(filter $(ARCH),riscv))
8		−RISCV_SUBTARGETS ?= hwprobe vector mm sigreturn
	8	+RISCV_SUBTARGETS ?= abi hwprobe mm sigreturn vector
9	9	else
10	10	RISCV_SUBTARGETS :=
11	11	endif

tools/testing/selftests/riscv/abi/.gitignore+1−0

@@ -0,0 +1 @@
	1	+pointer_masking

tools/testing/selftests/riscv/abi/Makefile+10−0

@@ -0,0 +1,10 @@
	1	+# SPDX-License-Identifier: GPL-2.0
	2	+
	3	+CFLAGS += -I$(top_srcdir)/tools/include
	4	+
	5	+TEST_GEN_PROGS := pointer_masking
	6	+
	7	+include ../../lib.mk
	8	+
	9	+$(OUTPUT)/pointer_masking: pointer_masking.c
	10	+ $(CC) -static -o$@ $(CFLAGS) $(LDFLAGS) $^

tools/testing/selftests/riscv/abi/pointer_masking.c+332−0

@@ -0,0 +1,332 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+
	3	+#include <errno.h>
	4	+#include <fcntl.h>
	5	+#include <setjmp.h>
	6	+#include <signal.h>
	7	+#include <stdbool.h>
	8	+#include <sys/prctl.h>
	9	+#include <sys/wait.h>
	10	+#include <unistd.h>
	11	+
	12	+#include "../../kselftest.h"
	13	+
	14	+#ifndef PR_PMLEN_SHIFT
	15	+#define PR_PMLEN_SHIFT 24
	16	+#endif
	17	+#ifndef PR_PMLEN_MASK
	18	+#define PR_PMLEN_MASK (0x7fUL << PR_PMLEN_SHIFT)
	19	+#endif
	20	+
	21	+static int dev_zero;
	22	+
	23	+static int pipefd[2];
	24	+
	25	+static sigjmp_buf jmpbuf;
	26	+
	27	+static void sigsegv_handler(int sig)
	28	+{
	29	+ siglongjmp(jmpbuf, 1);
	30	+}
	31	+
	32	+static int min_pmlen;
	33	+static int max_pmlen;
	34	+
	35	+static inline bool valid_pmlen(int pmlen)
	36	+{
	37	+ return pmlen == 0 \|\| pmlen == 7 \|\| pmlen == 16;
	38	+}
	39	+
	40	+static void test_pmlen(void)
	41	+{
	42	+ ksft_print_msg("Testing available PMLEN values\n");
	43	+
	44	+ for (int request = 0; request <= 16; request++) {
	45	+ int pmlen, ret;
	46	+
	47	+ ret = prctl(PR_SET_TAGGED_ADDR_CTRL, request << PR_PMLEN_SHIFT, 0, 0, 0);
	48	+ if (ret)
	49	+ goto pr_set_error;
	50	+
	51	+ ret = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
	52	+ ksft_test_result(ret >= 0, "PMLEN=%d PR_GET_TAGGED_ADDR_CTRL\n", request);
	53	+ if (ret < 0)
	54	+ goto pr_get_error;
	55	+
	56	+ pmlen = (ret & PR_PMLEN_MASK) >> PR_PMLEN_SHIFT;
	57	+ ksft_test_result(pmlen >= request, "PMLEN=%d constraint\n", request);
	58	+ ksft_test_result(valid_pmlen(pmlen), "PMLEN=%d validity\n", request);
	59	+
	60	+ if (min_pmlen == 0)
	61	+ min_pmlen = pmlen;
	62	+ if (max_pmlen < pmlen)
	63	+ max_pmlen = pmlen;
	64	+
	65	+ continue;
	66	+
	67	+pr_set_error:
	68	+ ksft_test_result_skip("PMLEN=%d PR_GET_TAGGED_ADDR_CTRL\n", request);
	69	+pr_get_error:
	70	+ ksft_test_result_skip("PMLEN=%d constraint\n", request);
	71	+ ksft_test_result_skip("PMLEN=%d validity\n", request);
	72	+ }
	73	+
	74	+ if (max_pmlen == 0)
	75	+ ksft_exit_fail_msg("Failed to enable pointer masking\n");
	76	+}
	77	+
	78	+static int set_tagged_addr_ctrl(int pmlen, bool tagged_addr_abi)
	79	+{
	80	+ int arg, ret;
	81	+
	82	+ arg = pmlen << PR_PMLEN_SHIFT \| tagged_addr_abi;
	83	+ ret = prctl(PR_SET_TAGGED_ADDR_CTRL, arg, 0, 0, 0);
	84	+ if (!ret) {
	85	+ ret = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
	86	+ if (ret == arg)
	87	+ return 0;
	88	+ }
	89	+
	90	+ return ret < 0 ? -errno : -ENODATA;
	91	+}
	92	+
	93	+static void test_dereference_pmlen(int pmlen)
	94	+{
	95	+ static volatile int i;
	96	+ volatile int *p;
	97	+ int ret;
	98	+
	99	+ ret = set_tagged_addr_ctrl(pmlen, false);
	100	+ if (ret)
	101	+ return ksft_test_result_error("PMLEN=%d setup (%d)\n", pmlen, ret);
	102	+
	103	+ i = pmlen;
	104	+
	105	+ if (pmlen) {
	106	+ p = (volatile int *)((uintptr_t)&i \| 1UL << (__riscv_xlen - pmlen));
	107	+
	108	+ /* These dereferences should succeed. */
	109	+ if (sigsetjmp(jmpbuf, 1))
	110	+ return ksft_test_result_fail("PMLEN=%d valid tag\n", pmlen);
	111	+ if (*p != pmlen)
	112	+ return ksft_test_result_fail("PMLEN=%d bad value\n", pmlen);
	113	+ ++*p;
	114	+ }
	115	+
	116	+ p = (volatile int *)((uintptr_t)&i \| 1UL << (__riscv_xlen - pmlen - 1));
	117	+
	118	+ /* These dereferences should raise SIGSEGV. */
	119	+ if (sigsetjmp(jmpbuf, 1))
	120	+ return ksft_test_result_pass("PMLEN=%d dereference\n", pmlen);
	121	+ ++*p;
	122	+ ksft_test_result_fail("PMLEN=%d invalid tag\n", pmlen);
	123	+}
	124	+
	125	+static void test_dereference(void)
	126	+{
	127	+ ksft_print_msg("Testing userspace pointer dereference\n");
	128	+
	129	+ signal(SIGSEGV, sigsegv_handler);
	130	+
	131	+ test_dereference_pmlen(0);
	132	+ test_dereference_pmlen(min_pmlen);
	133	+ test_dereference_pmlen(max_pmlen);
	134	+
	135	+ signal(SIGSEGV, SIG_DFL);
	136	+}
	137	+
	138	+static void execve_child_sigsegv_handler(int sig)
	139	+{
	140	+ exit(42);
	141	+}
	142	+
	143	+static int execve_child(void)
	144	+{
	145	+ static volatile int i;
	146	+ volatile int p = (volatile int )((uintptr_t)&i \| 1UL << (__riscv_xlen - 7));
	147	+
	148	+ signal(SIGSEGV, execve_child_sigsegv_handler);
	149	+
	150	+ /* This dereference should raise SIGSEGV. */
	151	+ return *p;
	152	+}
	153	+
	154	+static void test_fork_exec(void)
	155	+{
	156	+ int ret, status;
	157	+
	158	+ ksft_print_msg("Testing fork/exec behavior\n");
	159	+
	160	+ ret = set_tagged_addr_ctrl(min_pmlen, false);
	161	+ if (ret)
	162	+ return ksft_test_result_error("setup (%d)\n", ret);
	163	+
	164	+ if (fork()) {
	165	+ wait(&status);
	166	+ ksft_test_result(WIFEXITED(status) && WEXITSTATUS(status) == 42,
	167	+ "dereference after fork\n");
	168	+ } else {
	169	+ static volatile int i = 42;
	170	+ volatile int *p;
	171	+
	172	+ p = (volatile int *)((uintptr_t)&i \| 1UL << (__riscv_xlen - min_pmlen));
	173	+
	174	+ /* This dereference should succeed. */
	175	+ exit(*p);
	176	+ }
	177	+
	178	+ if (fork()) {
	179	+ wait(&status);
	180	+ ksft_test_result(WIFEXITED(status) && WEXITSTATUS(status) == 42,
	181	+ "dereference after fork+exec\n");
	182	+ } else {
	183	+ /* Will call execve_child(). */
	184	+ execve("/proc/self/exe", (char *const []) { "", NULL }, NULL);
	185	+ }
	186	+}
	187	+
	188	+static void test_tagged_addr_abi_sysctl(void)
	189	+{
	190	+ char value;
	191	+ int fd;
	192	+
	193	+ ksft_print_msg("Testing tagged address ABI sysctl\n");
	194	+
	195	+ fd = open("/proc/sys/abi/tagged_addr_disabled", O_WRONLY);
	196	+ if (fd < 0) {
	197	+ ksft_test_result_skip("failed to open sysctl file\n");
	198	+ ksft_test_result_skip("failed to open sysctl file\n");
	199	+ return;
	200	+ }
	201	+
	202	+ value = '1';
	203	+ pwrite(fd, &value, 1, 0);
	204	+ ksft_test_result(set_tagged_addr_ctrl(min_pmlen, true) == -EINVAL,
	205	+ "sysctl disabled\n");
	206	+
	207	+ value = '0';
	208	+ pwrite(fd, &value, 1, 0);
	209	+ ksft_test_result(set_tagged_addr_ctrl(min_pmlen, true) == 0,
	210	+ "sysctl enabled\n");
	211	+
	212	+ set_tagged_addr_ctrl(0, false);
	213	+
	214	+ close(fd);
	215	+}
	216	+
	217	+static void test_tagged_addr_abi_pmlen(int pmlen)
	218	+{
	219	+ int i, *p, ret;
	220	+
	221	+ i = ~pmlen;
	222	+
	223	+ if (pmlen) {
	224	+ p = (int *)((uintptr_t)&i \| 1UL << (__riscv_xlen - pmlen));
	225	+
	226	+ ret = set_tagged_addr_ctrl(pmlen, false);
	227	+ if (ret)
	228	+ return ksft_test_result_error("PMLEN=%d ABI disabled setup (%d)\n",
	229	+ pmlen, ret);
	230	+
	231	+ ret = write(pipefd[1], p, sizeof(*p));
	232	+ if (ret >= 0 \|\| errno != EFAULT)
	233	+ return ksft_test_result_fail("PMLEN=%d ABI disabled write\n", pmlen);
	234	+
	235	+ ret = read(dev_zero, p, sizeof(*p));
	236	+ if (ret >= 0 \|\| errno != EFAULT)
	237	+ return ksft_test_result_fail("PMLEN=%d ABI disabled read\n", pmlen);
	238	+
	239	+ if (i != ~pmlen)
	240	+ return ksft_test_result_fail("PMLEN=%d ABI disabled value\n", pmlen);
	241	+
	242	+ ret = set_tagged_addr_ctrl(pmlen, true);
	243	+ if (ret)
	244	+ return ksft_test_result_error("PMLEN=%d ABI enabled setup (%d)\n",
	245	+ pmlen, ret);
	246	+
	247	+ ret = write(pipefd[1], p, sizeof(*p));
	248	+ if (ret != sizeof(*p))
	249	+ return ksft_test_result_fail("PMLEN=%d ABI enabled write\n", pmlen);
	250	+
	251	+ ret = read(dev_zero, p, sizeof(*p));
	252	+ if (ret != sizeof(*p))
	253	+ return ksft_test_result_fail("PMLEN=%d ABI enabled read\n", pmlen);
	254	+
	255	+ if (i)
	256	+ return ksft_test_result_fail("PMLEN=%d ABI enabled value\n", pmlen);
	257	+
	258	+ i = ~pmlen;
	259	+ } else {
	260	+ /* The tagged address ABI cannot be enabled when PMLEN == 0. */
	261	+ ret = set_tagged_addr_ctrl(pmlen, true);
	262	+ if (ret != -EINVAL)
	263	+ return ksft_test_result_error("PMLEN=%d ABI setup (%d)\n",
	264	+ pmlen, ret);
	265	+ }
	266	+
	267	+ p = (int *)((uintptr_t)&i \| 1UL << (__riscv_xlen - pmlen - 1));
	268	+
	269	+ ret = write(pipefd[1], p, sizeof(*p));
	270	+ if (ret >= 0 \|\| errno != EFAULT)
	271	+ return ksft_test_result_fail("PMLEN=%d invalid tag write (%d)\n", pmlen, errno);
	272	+
	273	+ ret = read(dev_zero, p, sizeof(*p));
	274	+ if (ret >= 0 \|\| errno != EFAULT)
	275	+ return ksft_test_result_fail("PMLEN=%d invalid tag read\n", pmlen);
	276	+
	277	+ if (i != ~pmlen)
	278	+ return ksft_test_result_fail("PMLEN=%d invalid tag value\n", pmlen);
	279	+
	280	+ ksft_test_result_pass("PMLEN=%d tagged address ABI\n", pmlen);
	281	+}
	282	+
	283	+static void test_tagged_addr_abi(void)
	284	+{
	285	+ ksft_print_msg("Testing tagged address ABI\n");
	286	+
	287	+ test_tagged_addr_abi_pmlen(0);
	288	+ test_tagged_addr_abi_pmlen(min_pmlen);
	289	+ test_tagged_addr_abi_pmlen(max_pmlen);
	290	+}
	291	+
	292	+static struct test_info {
	293	+ unsigned int nr_tests;
	294	+ void (*test_fn)(void);
	295	+} tests[] = {
	296	+ { .nr_tests = 17 * 3, test_pmlen },
	297	+ { .nr_tests = 3, test_dereference },
	298	+ { .nr_tests = 2, test_fork_exec },
	299	+ { .nr_tests = 2, test_tagged_addr_abi_sysctl },
	300	+ { .nr_tests = 3, test_tagged_addr_abi },
	301	+};
	302	+
	303	+int main(int argc, char **argv)
	304	+{
	305	+ unsigned int plan = 0;
	306	+ int ret;
	307	+
	308	+ /* Check if this is the child process after execve(). */
	309	+ if (!argv[0][0])
	310	+ return execve_child();
	311	+
	312	+ dev_zero = open("/dev/zero", O_RDWR);
	313	+ if (dev_zero < 0)
	314	+ return 1;
	315	+
	316	+ /* Write to a pipe so the kernel must dereference the buffer pointer. */
	317	+ ret = pipe(pipefd);
	318	+ if (ret)
	319	+ return 1;
	320	+
	321	+ ksft_print_header();
	322	+
	323	+ for (int i = 0; i < ARRAY_SIZE(tests); i++)
	324	+ plan += tests[i].nr_tests;
	325	+
	326	+ ksft_set_plan(plan);
	327	+
	328	+ for (int i = 0; i < ARRAY_SIZE(tests); i++)
	329	+ tests[i].test_fn();
	330	+
	331	+ ksft_finished();
	332	+}