History

Merge tag 'riscv-for-linus-6.13-mw1' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux

Pull RISC-v updates from Palmer Dabbelt: - Support for pointer masking in userspace - Support for probing vector misaligned access performance - Support for qspinlock on systems with Zacas and Zabha * tag 'riscv-for-linus-6.13-mw1' of git://git.kernel.org/pub/scm/linux/kernel/git/riscv/linux: (38 commits) RISC-V: Remove unnecessary include from compat.h riscv: Fix default misaligned access trap riscv: Add qspinlock support dt-bindings: riscv: Add Ziccrse ISA extension description riscv: Add ISA extension parsing for Ziccrse asm-generic: ticket-lock: Add separate ticket-lock.h asm-generic: ticket-lock: Reuse arch_spinlock_t of qspinlock riscv: Implement xchg8/16() using Zabha riscv: Implement arch_cmpxchg128() using Zacas riscv: Improve zacas fully-ordered cmpxchg() riscv: Implement cmpxchg8/16() using Zabha dt-bindings: riscv: Add Zabha ISA extension description riscv: Implement cmpxchg32/64() using Zacas riscv: Do not fail to build on byte/halfword operations with Zawrs riscv: Move cpufeature.h macros into their own header KVM: riscv: selftests: Add Smnpm and Ssnpm to get-reg-list test RISC-V: KVM: Allow Smnpm and Ssnpm extensions for guests riscv: hwprobe: Export the Supm ISA extension riscv: selftests: Add a pointer masking test riscv: Allow ptrace control of the tagged address ABI ...

Linus Torvalds committed 1 year ago

Commit 91dbbe62 parents c946969 8d4f1e053 files · +1789 −282Browse files

Unified · Split

53 files
  1. Documentation/arch/riscv/hwprobe.rst +19 −0
  2. Documentation/arch/riscv/uabi.rst +16 −0
  3. Documentation/devicetree/bindings/riscv/extensions.yaml +30 −0
  4. Documentation/features/locking/queued-spinlocks/arch-support.txt +1 −1
  5. arch/riscv/Kconfig +136 −2
  6. arch/riscv/Makefile +6 −0
  7. arch/riscv/configs/defconfig +1 −0
  8. arch/riscv/include/asm/Kbuild +3 −1
  9. arch/riscv/include/asm/cmpxchg.h +208 −78
  10. arch/riscv/include/asm/compat.h +0 −1
  11. arch/riscv/include/asm/cpufeature-macros.h +66 −0
  12. arch/riscv/include/asm/cpufeature.h +14 −59
  13. arch/riscv/include/asm/csr.h +16 −0
  14. arch/riscv/include/asm/entry-common.h +1 −0
  15. arch/riscv/include/asm/hwcap.h +7 −0
  16. arch/riscv/include/asm/hwprobe.h +1 −1
  17. arch/riscv/include/asm/mmu.h +7 −0
  18. arch/riscv/include/asm/mmu_context.h +13 −0
  19. arch/riscv/include/asm/processor.h +9 −0
  20. arch/riscv/include/asm/spinlock.h +47 −0
  21. arch/riscv/include/asm/switch_to.h +19 −0
  22. arch/riscv/include/asm/uaccess.h +38 −5
  23. arch/riscv/include/asm/vector.h +2 −0
  24. arch/riscv/include/uapi/asm/hwprobe.h +6 −0
  25. arch/riscv/include/uapi/asm/kvm.h +2 −0
  26. arch/riscv/kernel/Makefile +2 −1
  27. arch/riscv/kernel/copy-unaligned.h +5 −0
  28. arch/riscv/kernel/cpufeature.c +13 −3
  29. arch/riscv/kernel/fpu.S +2 −2
  30. arch/riscv/kernel/process.c +154 −0
  31. arch/riscv/kernel/ptrace.c +42 −0
  32. arch/riscv/kernel/setup.c +37 −0
  33. arch/riscv/kernel/smpboot.c +0 −2
  34. arch/riscv/kernel/suspend.c +2 −2
  35. arch/riscv/kernel/sys_hwprobe.c +44 −0
  36. arch/riscv/kernel/traps_misaligned.c +128 −11
  37. arch/riscv/kernel/unaligned_access_speed.c +148 −8
  38. arch/riscv/kernel/vdso/Makefile +5 −4
  39. arch/riscv/kernel/vec-copy-unaligned.S +58 −0
  40. arch/riscv/kernel/vector.c +1 −1
  41. arch/riscv/kvm/vcpu_onereg.c +4 −0
  42. drivers/cpuidle/cpuidle-riscv-sbi.c +9 −2
  43. include/asm-generic/qspinlock.h +2 −0
  44. include/asm-generic/spinlock.h +1 −86
  45. include/asm-generic/spinlock_types.h +2 −10
  46. include/asm-generic/ticket_spinlock.h +105 −0
  47. include/uapi/linux/elf.h +1 −0
  48. include/uapi/linux/prctl.h +4 −1
  49. tools/testing/selftests/kvm/riscv/get-reg-list.c +8 −0
  50. tools/testing/selftests/riscv/Makefile +1 −1
  51. tools/testing/selftests/riscv/abi/.gitignore +1 −0
  52. tools/testing/selftests/riscv/abi/Makefile +10 −0
  53. tools/testing/selftests/riscv/abi/pointer_masking.c +332 −0
Documentation/arch/riscv/hwprobe.rst+19−0
@@ -239,6 +239,9 @@ The following keys are defined: · 238 unmodified lines
239 ratified in commit 98918c844281 ("Merge pull request #1217 from239 ratified in commit 98918c844281 ("Merge pull request #1217 from
240 riscv/zawrs") of riscv-isa-manual.240 riscv/zawrs") of riscv-isa-manual.
241 241
242 * :c:macro:`RISCV_HWPROBE_EXT_SUPM`: The Supm extension is supported as
243 defined in version 1.0 of the RISC-V Pointer Masking extensions.
244
242* :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: Deprecated. Returns similar values to245* :c:macro:`RISCV_HWPROBE_KEY_CPUPERF_0`: Deprecated. Returns similar values to
243 :c:macro:`RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF`, but the key was246 :c:macro:`RISCV_HWPROBE_KEY_MISALIGNED_SCALAR_PERF`, but the key was
244 mistakenly classified as a bitmask rather than a value.247 mistakenly classified as a bitmask rather than a value.
@@ -274,3 +277,19 @@ The following keys are defined: · 29 unmodified lines
274 represent the highest userspace virtual address usable.277 represent the highest userspace virtual address usable.
275 278
276* :c:macro:`RISCV_HWPROBE_KEY_TIME_CSR_FREQ`: Frequency (in Hz) of `time CSR`.279* :c:macro:`RISCV_HWPROBE_KEY_TIME_CSR_FREQ`: Frequency (in Hz) of `time CSR`.
280
281* :c:macro:`RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF`: An enum value describing the
282 performance of misaligned vector accesses on the selected set of processors.
283
284 * :c:macro:`RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN`: The performance of misaligned
285 vector accesses is unknown.
286
287 * :c:macro:`RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW`: 32-bit misaligned accesses using vector
288 registers are slower than the equivalent quantity of byte accesses via vector registers.
289 Misaligned accesses may be supported directly in hardware, or trapped and emulated by software.
290
291 * :c:macro:`RISCV_HWPROBE_MISALIGNED_VECTOR_FAST`: 32-bit misaligned accesses using vector
292 registers are faster than the equivalent quantity of byte accesses via vector registers.
293
294 * :c:macro:`RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED`: Misaligned vector accesses are
295 not supported at all and will generate a misaligned address fault.
Documentation/arch/riscv/uabi.rst+16−0
@@ -68,3 +68,19 @@ Misaligned accesses · 67 unmodified lines
68Misaligned scalar accesses are supported in userspace, but they may perform68Misaligned scalar accesses are supported in userspace, but they may perform
69poorly. Misaligned vector accesses are only supported if the Zicclsm extension69poorly. Misaligned vector accesses are only supported if the Zicclsm extension
70is supported.70is supported.
71
72Pointer masking
73---------------
74
75Support for pointer masking in userspace (the Supm extension) is provided via
76the ``PR_SET_TAGGED_ADDR_CTRL`` and ``PR_GET_TAGGED_ADDR_CTRL`` ``prctl()``
77operations. Pointer masking is disabled by default. To enable it, userspace
78must call ``PR_SET_TAGGED_ADDR_CTRL`` with the ``PR_PMLEN`` field set to the
79number of mask/tag bits needed by the application. ``PR_PMLEN`` is interpreted
80as a lower bound; if the kernel is unable to satisfy the request, the
81``PR_SET_TAGGED_ADDR_CTRL`` operation will fail. The actual number of tag bits
82is returned in ``PR_PMLEN`` by the ``PR_GET_TAGGED_ADDR_CTRL`` operation.
83
84Additionally, when pointer masking is enabled (``PR_PMLEN`` is greater than 0),
85a tagged address ABI is supported, with the same interface and behavior as
86documented for AArch64 (Documentation/arch/arm64/tagged-address-abi.rst).
Documentation/devicetree/bindings/riscv/extensions.yaml+30−0
@@ -128,6 +128,18 @@ properties: · 127 unmodified lines
128 changes to interrupts as frozen at commit ccbddab ("Merge pull128 changes to interrupts as frozen at commit ccbddab ("Merge pull
129 request #42 from riscv/jhauser-2023-RC4") of riscv-aia.129 request #42 from riscv/jhauser-2023-RC4") of riscv-aia.
130 130
131 - const: smmpm
132 description: |
133 The standard Smmpm extension for M-mode pointer masking as
134 ratified at commit d70011dde6c2 ("Update to ratified state")
135 of riscv-j-extension.
136
137 - const: smnpm
138 description: |
139 The standard Smnpm extension for next-mode pointer masking as
140 ratified at commit d70011dde6c2 ("Update to ratified state")
141 of riscv-j-extension.
142
131 - const: smstateen143 - const: smstateen
132 description: |144 description: |
133 The standard Smstateen extension for controlling access to CSRs145 The standard Smstateen extension for controlling access to CSRs
@@ -147,6 +159,12 @@ properties: · 13 unmodified lines
147 and mode-based filtering as ratified at commit 01d1df0 ("Add ability159 and mode-based filtering as ratified at commit 01d1df0 ("Add ability
148 to manually trigger workflow. (#2)") of riscv-count-overflow.160 to manually trigger workflow. (#2)") of riscv-count-overflow.
149 161
162 - const: ssnpm
163 description: |
164 The standard Ssnpm extension for next-mode pointer masking as
165 ratified at commit d70011dde6c2 ("Update to ratified state")
166 of riscv-j-extension.
167
150 - const: sstc168 - const: sstc
151 description: |169 description: |
152 The standard Sstc supervisor-level extension for time compare as170 The standard Sstc supervisor-level extension for time compare as
@@ -178,6 +196,12 @@ properties: · 25 unmodified lines
178 as ratified at commit 4a69197e5617 ("Update to ratified state") of196 as ratified at commit 4a69197e5617 ("Update to ratified state") of
179 riscv-svvptc.197 riscv-svvptc.
180 198
199 - const: zabha
200 description: |
201 The Zabha extension for Byte and Halfword Atomic Memory Operations
202 as ratified at commit 49f49c842ff9 ("Update to Rafified state") of
203 riscv-zabha.
204
181 - const: zacas205 - const: zacas
182 description: |206 description: |
183 The Zacas extension for Atomic Compare-and-Swap (CAS) instructions207 The Zacas extension for Atomic Compare-and-Swap (CAS) instructions
@@ -290,6 +314,12 @@ properties: · 106 unmodified lines
290 in commit 64074bc ("Update version numbers for Zfh/Zfinx") of314 in commit 64074bc ("Update version numbers for Zfh/Zfinx") of
291 riscv-isa-manual.315 riscv-isa-manual.
292 316
317 - const: ziccrse
318 description:
319 The standard Ziccrse extension which provides forward progress
320 guarantee on LR/SC sequences, as ratified in commit b1d806605f87
321 ("Updated to ratified state.") of the riscv profiles specification.
322
293 - const: zk323 - const: zk
294 description:324 description:
295 The standard Zk Standard Scalar cryptography extension as ratified325 The standard Zk Standard Scalar cryptography extension as ratified
Documentation/features/locking/queued-spinlocks/arch-support.txt+1−1
@@ -20,7 +20,7 @@ · 19 unmodified lines
20 | openrisc: | ok |20 | openrisc: | ok |
21 | parisc: | TODO |21 | parisc: | TODO |
22 | powerpc: | ok |22 | powerpc: | ok |
23 | riscv: | TODO |23 | riscv: | ok |
24 | s390: | TODO |24 | s390: | TODO |
25 | sh: | TODO |25 | sh: | TODO |
26 | sparc: | ok |26 | sparc: | ok |
arch/riscv/Kconfig+136−2
@@ -83,6 +83,7 @@ config RISCV · 82 unmodified lines
83 select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP83 select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
84 select ARCH_WANTS_NO_INSTR84 select ARCH_WANTS_NO_INSTR
85 select ARCH_WANTS_THP_SWAP if HAVE_ARCH_TRANSPARENT_HUGEPAGE85 select ARCH_WANTS_THP_SWAP if HAVE_ARCH_TRANSPARENT_HUGEPAGE
86 select ARCH_WEAK_RELEASE_ACQUIRE if ARCH_USE_QUEUED_SPINLOCKS
86 select BINFMT_FLAT_NO_DATA_START_OFFSET if !MMU87 select BINFMT_FLAT_NO_DATA_START_OFFSET if !MMU
87 select BUILDTIME_TABLE_SORT if MMU88 select BUILDTIME_TABLE_SORT if MMU
88 select CLINT_TIMER if RISCV_M_MODE89 select CLINT_TIMER if RISCV_M_MODE
@@ -116,6 +117,7 @@ config RISCV · 27 unmodified lines
116 select GENERIC_VDSO_TIME_NS if HAVE_GENERIC_VDSO117 select GENERIC_VDSO_TIME_NS if HAVE_GENERIC_VDSO
117 select HARDIRQS_SW_RESEND118 select HARDIRQS_SW_RESEND
118 select HAS_IOPORT if MMU119 select HAS_IOPORT if MMU
120 select HAVE_ALIGNED_STRUCT_PAGE
119 select HAVE_ARCH_AUDITSYSCALL121 select HAVE_ARCH_AUDITSYSCALL
120 select HAVE_ARCH_HUGE_VMALLOC if HAVE_ARCH_HUGE_VMAP122 select HAVE_ARCH_HUGE_VMALLOC if HAVE_ARCH_HUGE_VMAP
121 select HAVE_ARCH_HUGE_VMAP if MMU && 64BIT123 select HAVE_ARCH_HUGE_VMAP if MMU && 64BIT
@@ -507,6 +509,39 @@ config NODES_SHIFT · 385 unmodified lines
507 Specify the maximum number of NUMA Nodes available on the target509 Specify the maximum number of NUMA Nodes available on the target
508 system. Increases memory reserved to accommodate various tables.510 system. Increases memory reserved to accommodate various tables.
509 511
512choice
513 prompt "RISC-V spinlock type"
514 default RISCV_COMBO_SPINLOCKS
515
516config RISCV_TICKET_SPINLOCKS
517 bool "Using ticket spinlock"
518
519config RISCV_QUEUED_SPINLOCKS
520 bool "Using queued spinlock"
521 depends on SMP && MMU && NONPORTABLE
522 select ARCH_USE_QUEUED_SPINLOCKS
523 help
524 The queued spinlock implementation requires the forward progress
525 guarantee of cmpxchg()/xchg() atomic operations: CAS with Zabha or
526 LR/SC with Ziccrse provide such guarantee.
527
528 Select this if and only if Zabha or Ziccrse is available on your
529 platform, RISCV_QUEUED_SPINLOCKS must not be selected for platforms
530 without one of those extensions.
531
532 If unsure, select RISCV_COMBO_SPINLOCKS, which will use qspinlocks
533 when supported and otherwise ticket spinlocks.
534
535config RISCV_COMBO_SPINLOCKS
536 bool "Using combo spinlock"
537 depends on SMP && MMU
538 select ARCH_USE_QUEUED_SPINLOCKS
539 help
540 Embed both queued spinlock and ticket lock so that the spinlock
541 implementation can be chosen at runtime.
542
543endchoice
544
510config RISCV_ALTERNATIVE545config RISCV_ALTERNATIVE
511 bool546 bool
512 depends on !XIP_KERNEL547 depends on !XIP_KERNEL
@@ -532,6 +567,17 @@ config RISCV_ISA_C · 19 unmodified lines
532 567
533 If you don't know what to do here, say Y.568 If you don't know what to do here, say Y.
534 569
570config RISCV_ISA_SUPM
571 bool "Supm extension for userspace pointer masking"
572 depends on 64BIT
573 default y
574 help
575 Add support for pointer masking in userspace (Supm) when the
576 underlying hardware extension (Smnpm or Ssnpm) is detected at boot.
577
578 If this option is disabled, userspace will be unable to use
579 the prctl(PR_{SET,GET}_TAGGED_ADDR_CTRL) API.
580
535config RISCV_ISA_SVNAPOT581config RISCV_ISA_SVNAPOT
536 bool "Svnapot extension support for supervisor mode NAPOT pages"582 bool "Svnapot extension support for supervisor mode NAPOT pages"
537 depends on 64BIT && MMU583 depends on 64BIT && MMU
@@ -633,6 +679,40 @@ config RISCV_ISA_ZAWRS · 95 unmodified lines
633 use of these instructions in the kernel when the Zawrs extension is679 use of these instructions in the kernel when the Zawrs extension is
634 detected at boot.680 detected at boot.
635 681
682config TOOLCHAIN_HAS_ZABHA
683 bool
684 default y
685 depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zabha)
686 depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zabha)
687 depends on AS_HAS_OPTION_ARCH
688
689config RISCV_ISA_ZABHA
690 bool "Zabha extension support for atomic byte/halfword operations"
691 depends on TOOLCHAIN_HAS_ZABHA
692 depends on RISCV_ALTERNATIVE
693 default y
694 help
695 Enable the use of the Zabha ISA-extension to implement kernel
696 byte/halfword atomic memory operations when it is detected at boot.
697
698 If you don't know what to do here, say Y.
699
700config TOOLCHAIN_HAS_ZACAS
701 bool
702 default y
703 depends on !64BIT || $(cc-option,-mabi=lp64 -march=rv64ima_zacas)
704 depends on !32BIT || $(cc-option,-mabi=ilp32 -march=rv32ima_zacas)
705 depends on AS_HAS_OPTION_ARCH
706
707config RISCV_ISA_ZACAS
708 bool "Zacas extension support for atomic CAS"
709 depends on TOOLCHAIN_HAS_ZACAS
710 depends on RISCV_ALTERNATIVE
711 default y
712 help
713 Enable the use of the Zacas ISA-extension to implement kernel atomic
714 cmpxchg operations when it is detected at boot.
715
636 If you don't know what to do here, say Y.716 If you don't know what to do here, say Y.
637 717
638config TOOLCHAIN_HAS_ZBB718config TOOLCHAIN_HAS_ZBB
@@ -786,10 +866,24 @@ config THREAD_SIZE_ORDER · 147 unmodified lines
786 866
787config RISCV_MISALIGNED867config RISCV_MISALIGNED
788 bool868 bool
869 help
870 Embed support for detecting and emulating misaligned
871 scalar or vector loads and stores.
872
873config RISCV_SCALAR_MISALIGNED
874 bool
875 select RISCV_MISALIGNED
789 select SYSCTL_ARCH_UNALIGN_ALLOW876 select SYSCTL_ARCH_UNALIGN_ALLOW
790 help877 help
791 Embed support for emulating misaligned loads and stores.878 Embed support for emulating misaligned loads and stores.
792 879
880config RISCV_VECTOR_MISALIGNED
881 bool
882 select RISCV_MISALIGNED
883 depends on RISCV_ISA_V
884 help
885 Enable detecting support for vector misaligned loads and stores.
886
793choice887choice
794 prompt "Unaligned Accesses Support"888 prompt "Unaligned Accesses Support"
795 default RISCV_PROBE_UNALIGNED_ACCESS889 default RISCV_PROBE_UNALIGNED_ACCESS
@@ -801,7 +895,7 @@ choice · 5 unmodified lines
801 895
802config RISCV_PROBE_UNALIGNED_ACCESS896config RISCV_PROBE_UNALIGNED_ACCESS
803 bool "Probe for hardware unaligned access support"897 bool "Probe for hardware unaligned access support"
804 select RISCV_MISALIGNED898 select RISCV_SCALAR_MISALIGNED
805 help899 help
806 During boot, the kernel will run a series of tests to determine the900 During boot, the kernel will run a series of tests to determine the
807 speed of unaligned accesses. This probing will dynamically determine901 speed of unaligned accesses. This probing will dynamically determine
@@ -812,7 +906,7 @@ config RISCV_PROBE_UNALIGNED_ACCESS · 4 unmodified lines
812 906
813config RISCV_EMULATED_UNALIGNED_ACCESS907config RISCV_EMULATED_UNALIGNED_ACCESS
814 bool "Emulate unaligned access where system support is missing"908 bool "Emulate unaligned access where system support is missing"
815 select RISCV_MISALIGNED909 select RISCV_SCALAR_MISALIGNED
816 help910 help
817 If unaligned memory accesses trap into the kernel as they are not911 If unaligned memory accesses trap into the kernel as they are not
818 supported by the system, the kernel will emulate the unaligned912 supported by the system, the kernel will emulate the unaligned
@@ -841,6 +935,46 @@ config RISCV_EFFICIENT_UNALIGNED_ACCESS · 22 unmodified lines
841 935
842endchoice936endchoice
843 937
938choice
939 prompt "Vector unaligned Accesses Support"
940 depends on RISCV_ISA_V
941 default RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
942 help
943 This determines the level of support for vector unaligned accesses. This
944 information is used by the kernel to perform optimizations. It is also
945 exposed to user space via the hwprobe syscall. The hardware will be
946 probed at boot by default.
947
948config RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
949 bool "Probe speed of vector unaligned accesses"
950 select RISCV_VECTOR_MISALIGNED
951 depends on RISCV_ISA_V
952 help
953 During boot, the kernel will run a series of tests to determine the
954 speed of vector unaligned accesses if they are supported. This probing
955 will dynamically determine the speed of vector unaligned accesses on
956 the underlying system if they are supported.
957
958config RISCV_SLOW_VECTOR_UNALIGNED_ACCESS
959 bool "Assume the system supports slow vector unaligned memory accesses"
960 depends on NONPORTABLE
961 help
962 Assume that the system supports slow vector unaligned memory accesses. The
963 kernel and userspace programs may not be able to run at all on systems
964 that do not support unaligned memory accesses.
965
966config RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
967 bool "Assume the system supports fast vector unaligned memory accesses"
968 depends on NONPORTABLE
969 help
970 Assume that the system supports fast vector unaligned memory accesses. When
971 enabled, this option improves the performance of the kernel on such
972 systems. However, the kernel and userspace programs will run much more
973 slowly, or will not be able to run at all, on systems that do not
974 support efficient unaligned memory accesses.
975
976endchoice
977
844source "arch/riscv/Kconfig.vendor"978source "arch/riscv/Kconfig.vendor"
845 979
846endmenu # "Platform type"980endmenu # "Platform type"
arch/riscv/Makefile+6−0
@@ -82,6 +82,12 @@ else · 81 unmodified lines
82riscv-march-$(CONFIG_TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI) := $(riscv-march-y)_zicsr_zifencei82riscv-march-$(CONFIG_TOOLCHAIN_NEEDS_EXPLICIT_ZICSR_ZIFENCEI) := $(riscv-march-y)_zicsr_zifencei
83endif83endif
84 84
85# Check if the toolchain supports Zacas
86riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZACAS) := $(riscv-march-y)_zacas
87
88# Check if the toolchain supports Zabha
89riscv-march-$(CONFIG_TOOLCHAIN_HAS_ZABHA) := $(riscv-march-y)_zabha
90
85# Remove F,D,V from isa string for all. Keep extensions between "fd" and "v" by91# Remove F,D,V from isa string for all. Keep extensions between "fd" and "v" by
86# matching non-v and non-multi-letter extensions out with the filter ([^v_]*)92# matching non-v and non-multi-letter extensions out with the filter ([^v_]*)
87KBUILD_CFLAGS += -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/')93KBUILD_CFLAGS += -march=$(shell echo $(riscv-march-y) | sed -E 's/(rv32ima|rv64ima)fd([^v_]*)v?/\1\2/')
arch/riscv/configs/defconfig+1−0
@@ -167,6 +167,7 @@ CONFIG_PINCTRL_SOPHGO_CV1800B=y · 166 unmodified lines
167CONFIG_PINCTRL_SOPHGO_CV1812H=y167CONFIG_PINCTRL_SOPHGO_CV1812H=y
168CONFIG_PINCTRL_SOPHGO_SG2000=y168CONFIG_PINCTRL_SOPHGO_SG2000=y
169CONFIG_PINCTRL_SOPHGO_SG2002=y169CONFIG_PINCTRL_SOPHGO_SG2002=y
170CONFIG_GPIO_DWAPB=y
170CONFIG_GPIO_SIFIVE=y171CONFIG_GPIO_SIFIVE=y
171CONFIG_POWER_RESET_GPIO_RESTART=y172CONFIG_POWER_RESET_GPIO_RESTART=y
172CONFIG_SENSORS_SFCTEMP=m173CONFIG_SENSORS_SFCTEMP=m
arch/riscv/include/asm/Kbuild+3−1
@@ -6,10 +6,12 @@ generic-y += early_ioremap.h · 5 unmodified lines
6generic-y += flat.h6generic-y += flat.h
7generic-y += kvm_para.h7generic-y += kvm_para.h
8generic-y += mmzone.h8generic-y += mmzone.h
9generic-y += mcs_spinlock.h
9generic-y += parport.h10generic-y += parport.h
10generic-y += spinlock.h
11generic-y += spinlock_types.h11generic-y += spinlock_types.h
12generic-y += ticket_spinlock.h
12generic-y += qrwlock.h13generic-y += qrwlock.h
13generic-y += qrwlock_types.h14generic-y += qrwlock_types.h
15generic-y += qspinlock.h
14generic-y += user.h16generic-y += user.h
15generic-y += vmlinux.lds.h17generic-y += vmlinux.lds.h
arch/riscv/include/asm/cmpxchg.h+208−78
@@ -12,30 +12,43 @@ · 11 unmodified lines
12#include <asm/fence.h>12#include <asm/fence.h>
13#include <asm/hwcap.h>13#include <asm/hwcap.h>
14#include <asm/insn-def.h>14#include <asm/insn-def.h>
15 15#include <asm/cpufeature-macros.h>
16#define __arch_xchg_masked(sc_sfx, prepend, append, r, p, n) \16
17({ \17#define __arch_xchg_masked(sc_sfx, swap_sfx, prepend, sc_append, \
18 u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \18 swap_append, r, p, n) \
19 ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \19({ \
20 ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \20 if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \
21 << __s; \21 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA)) { \
22 ulong __newx = (ulong)(n) << __s; \22 __asm__ __volatile__ ( \
23 ulong __retx; \23 prepend \
24 ulong __rc; \24 " amoswap" swap_sfx " %0, %z2, %1\n" \
25 \25 swap_append \
26 __asm__ __volatile__ ( \26 : "=&r" (r), "+A" (*(p)) \
27 prepend \27 : "rJ" (n) \
28 "0: lr.w %0, %2\n" \28 : "memory"); \
29 " and %1, %0, %z4\n" \29 } else { \
30 " or %1, %1, %z3\n" \30 u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \
31 " sc.w" sc_sfx " %1, %1, %2\n" \31 ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \
32 " bnez %1, 0b\n" \32 ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \
33 append \33 << __s; \
34 : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \34 ulong __newx = (ulong)(n) << __s; \
35 : "rJ" (__newx), "rJ" (~__mask) \35 ulong __retx; \
36 : "memory"); \36 ulong __rc; \
37 \37 \
38 r = (__typeof__(*(p)))((__retx & __mask) >> __s); \38 __asm__ __volatile__ ( \
39 prepend \
40 "0: lr.w %0, %2\n" \
41 " and %1, %0, %z4\n" \
42 " or %1, %1, %z3\n" \
43 " sc.w" sc_sfx " %1, %1, %2\n" \
44 " bnez %1, 0b\n" \
45 sc_append \
46 : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
47 : "rJ" (__newx), "rJ" (~__mask) \
48 : "memory"); \
49 \
50 r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
51 } \
39})52})
40 53
41#define __arch_xchg(sfx, prepend, append, r, p, n) \54#define __arch_xchg(sfx, prepend, append, r, p, n) \
@@ -58,8 +71,13 @@ · 16 unmodified lines
58 \71 \
59 switch (sizeof(*__ptr)) { \72 switch (sizeof(*__ptr)) { \
60 case 1: \73 case 1: \
74 __arch_xchg_masked(sc_sfx, ".b" swap_sfx, \
75 prepend, sc_append, swap_append, \
76 __ret, __ptr, __new); \
77 break; \
61 case 2: \78 case 2: \
62 __arch_xchg_masked(sc_sfx, prepend, sc_append, \79 __arch_xchg_masked(sc_sfx, ".h" swap_sfx, \
80 prepend, sc_append, swap_append, \
63 __ret, __ptr, __new); \81 __ret, __ptr, __new); \
64 break; \82 break; \
65 case 4: \83 case 4: \
@@ -106,55 +124,90 @@ · 40 unmodified lines
106 * store NEW in MEM. Return the initial value in MEM. Success is124 * store NEW in MEM. Return the initial value in MEM. Success is
107 * indicated by comparing RETURN with OLD.125 * indicated by comparing RETURN with OLD.
108 */126 */
109 127#define __arch_cmpxchg_masked(sc_sfx, cas_sfx, \
110#define __arch_cmpxchg_masked(sc_sfx, prepend, append, r, p, o, n) \128 sc_prepend, sc_append, \
111({ \129 cas_prepend, cas_append, \
112 u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \130 r, p, o, n) \
113 ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \131({ \
114 ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \132 if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) && \
115 << __s; \133 IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \
116 ulong __newx = (ulong)(n) << __s; \134 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZABHA) && \
117 ulong __oldx = (ulong)(o) << __s; \135 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \
118 ulong __retx; \136 r = o; \
119 ulong __rc; \137 \
120 \138 __asm__ __volatile__ ( \
121 __asm__ __volatile__ ( \139 cas_prepend \
122 prepend \140 " amocas" cas_sfx " %0, %z2, %1\n" \
123 "0: lr.w %0, %2\n" \141 cas_append \
124 " and %1, %0, %z5\n" \142 : "+&r" (r), "+A" (*(p)) \
125 " bne %1, %z3, 1f\n" \143 : "rJ" (n) \
126 " and %1, %0, %z6\n" \144 : "memory"); \
127 " or %1, %1, %z4\n" \145 } else { \
128 " sc.w" sc_sfx " %1, %1, %2\n" \146 u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \
129 " bnez %1, 0b\n" \147 ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \
130 append \148 ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \
131 "1:\n" \149 << __s; \
132 : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \150 ulong __newx = (ulong)(n) << __s; \
133 : "rJ" ((long)__oldx), "rJ" (__newx), \151 ulong __oldx = (ulong)(o) << __s; \
134 "rJ" (__mask), "rJ" (~__mask) \152 ulong __retx; \
135 : "memory"); \153 ulong __rc; \
136 \154 \
137 r = (__typeof__(*(p)))((__retx & __mask) >> __s); \155 __asm__ __volatile__ ( \
156 sc_prepend \
157 "0: lr.w %0, %2\n" \
158 " and %1, %0, %z5\n" \
159 " bne %1, %z3, 1f\n" \
160 " and %1, %0, %z6\n" \
161 " or %1, %1, %z4\n" \
162 " sc.w" sc_sfx " %1, %1, %2\n" \
163 " bnez %1, 0b\n" \
164 sc_append \
165 "1:\n" \
166 : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
167 : "rJ" ((long)__oldx), "rJ" (__newx), \
168 "rJ" (__mask), "rJ" (~__mask) \
169 : "memory"); \
170 \
171 r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
172 } \
138})173})
139 174
140#define __arch_cmpxchg(lr_sfx, sc_sfx, prepend, append, r, p, co, o, n) \175#define __arch_cmpxchg(lr_sfx, sc_sfx, cas_sfx, \
176 sc_prepend, sc_append, \
177 cas_prepend, cas_append, \
178 r, p, co, o, n) \
141({ \179({ \
142 register unsigned int __rc; \180 if (IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) && \
181 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)) { \
182 r = o; \
143 \183 \
144 __asm__ __volatile__ ( \184 __asm__ __volatile__ ( \
145 prepend \185 cas_prepend \
146 "0: lr" lr_sfx " %0, %2\n" \186 " amocas" cas_sfx " %0, %z2, %1\n" \
147 " bne %0, %z3, 1f\n" \187 cas_append \
148 " sc" sc_sfx " %1, %z4, %2\n" \188 : "+&r" (r), "+A" (*(p)) \
149 " bnez %1, 0b\n" \189 : "rJ" (n) \
150 append \190 : "memory"); \
151 "1:\n" \191 } else { \
152 : "=&r" (r), "=&r" (__rc), "+A" (*(p)) \192 register unsigned int __rc; \
153 : "rJ" (co o), "rJ" (n) \193 \
154 : "memory"); \194 __asm__ __volatile__ ( \
195 sc_prepend \
196 "0: lr" lr_sfx " %0, %2\n" \
197 " bne %0, %z3, 1f\n" \
198 " sc" sc_sfx " %1, %z4, %2\n" \
199 " bnez %1, 0b\n" \
200 sc_append \
201 "1:\n" \
202 : "=&r" (r), "=&r" (__rc), "+A" (*(p)) \
203 : "rJ" (co o), "rJ" (n) \
204 : "memory"); \
205 } \
155})206})
156 207
157#define _arch_cmpxchg(ptr, old, new, sc_sfx, prepend, append) \208#define _arch_cmpxchg(ptr, old, new, sc_sfx, cas_sfx, \
209 sc_prepend, sc_append, \
210 cas_prepend, cas_append) \
158({ \211({ \
159 __typeof__(ptr) __ptr = (ptr); \212 __typeof__(ptr) __ptr = (ptr); \
160 __typeof__(*(__ptr)) __old = (old); \213 __typeof__(*(__ptr)) __old = (old); \
@@ -163,17 +216,28 @@ · 2 unmodified lines
163 \216 \
164 switch (sizeof(*__ptr)) { \217 switch (sizeof(*__ptr)) { \
165 case 1: \218 case 1: \
219 __arch_cmpxchg_masked(sc_sfx, ".b" cas_sfx, \
220 sc_prepend, sc_append, \
221 cas_prepend, cas_append, \
222 __ret, __ptr, __old, __new); \
223 break; \
166 case 2: \224 case 2: \
167 __arch_cmpxchg_masked(sc_sfx, prepend, append, \225 __arch_cmpxchg_masked(sc_sfx, ".h" cas_sfx, \
168 __ret, __ptr, __old, __new); \226 sc_prepend, sc_append, \
227 cas_prepend, cas_append, \
228 __ret, __ptr, __old, __new); \
169 break; \229 break; \
170 case 4: \230 case 4: \
171 __arch_cmpxchg(".w", ".w" sc_sfx, prepend, append, \231 __arch_cmpxchg(".w", ".w" sc_sfx, ".w" cas_sfx, \
172 __ret, __ptr, (long), __old, __new); \232 sc_prepend, sc_append, \
233 cas_prepend, cas_append, \
234 __ret, __ptr, (long), __old, __new); \
173 break; \235 break; \
174 case 8: \236 case 8: \
175 __arch_cmpxchg(".d", ".d" sc_sfx, prepend, append, \237 __arch_cmpxchg(".d", ".d" sc_sfx, ".d" cas_sfx, \
176 __ret, __ptr, /**/, __old, __new); \238 sc_prepend, sc_append, \
239 cas_prepend, cas_append, \
240 __ret, __ptr, /**/, __old, __new); \
177 break; \241 break; \
178 default: \242 default: \
179 BUILD_BUG(); \243 BUILD_BUG(); \
@@ -181,17 +245,40 @@ · 1 unmodified line
181 (__typeof__(*(__ptr)))__ret; \245 (__typeof__(*(__ptr)))__ret; \
182})246})
183 247
248/*
249 * These macros are here to improve the readability of the arch_cmpxchg_XXX()
250 * macros.
251 */
252#define SC_SFX(x) x
253#define CAS_SFX(x) x
254#define SC_PREPEND(x) x
255#define SC_APPEND(x) x
256#define CAS_PREPEND(x) x
257#define CAS_APPEND(x) x
258
184#define arch_cmpxchg_relaxed(ptr, o, n) \259#define arch_cmpxchg_relaxed(ptr, o, n) \
185 _arch_cmpxchg((ptr), (o), (n), "", "", "")260 _arch_cmpxchg((ptr), (o), (n), \
261 SC_SFX(""), CAS_SFX(""), \
262 SC_PREPEND(""), SC_APPEND(""), \
263 CAS_PREPEND(""), CAS_APPEND(""))
186 264
187#define arch_cmpxchg_acquire(ptr, o, n) \265#define arch_cmpxchg_acquire(ptr, o, n) \
188 _arch_cmpxchg((ptr), (o), (n), "", "", RISCV_ACQUIRE_BARRIER)266 _arch_cmpxchg((ptr), (o), (n), \
267 SC_SFX(""), CAS_SFX(""), \
268 SC_PREPEND(""), SC_APPEND(RISCV_ACQUIRE_BARRIER), \
269 CAS_PREPEND(""), CAS_APPEND(RISCV_ACQUIRE_BARRIER))
189 270
190#define arch_cmpxchg_release(ptr, o, n) \271#define arch_cmpxchg_release(ptr, o, n) \
191 _arch_cmpxchg((ptr), (o), (n), "", RISCV_RELEASE_BARRIER, "")272 _arch_cmpxchg((ptr), (o), (n), \
273 SC_SFX(""), CAS_SFX(""), \
274 SC_PREPEND(RISCV_RELEASE_BARRIER), SC_APPEND(""), \
275 CAS_PREPEND(RISCV_RELEASE_BARRIER), CAS_APPEND(""))
192 276
193#define arch_cmpxchg(ptr, o, n) \277#define arch_cmpxchg(ptr, o, n) \
194 _arch_cmpxchg((ptr), (o), (n), ".rl", "", " fence rw, rw\n")278 _arch_cmpxchg((ptr), (o), (n), \
279 SC_SFX(".rl"), CAS_SFX(".aqrl"), \
280 SC_PREPEND(""), SC_APPEND(RISCV_FULL_BARRIER), \
281 CAS_PREPEND(""), CAS_APPEND(""))
195 282
196#define arch_cmpxchg_local(ptr, o, n) \283#define arch_cmpxchg_local(ptr, o, n) \
197 arch_cmpxchg_relaxed((ptr), (o), (n))284 arch_cmpxchg_relaxed((ptr), (o), (n))
@@ -226,6 +313,44 @@ · 28 unmodified lines
226 arch_cmpxchg_release((ptr), (o), (n)); \313 arch_cmpxchg_release((ptr), (o), (n)); \
227})314})
228 315
316#if defined(CONFIG_64BIT) && defined(CONFIG_RISCV_ISA_ZACAS)
317
318#define system_has_cmpxchg128() riscv_has_extension_unlikely(RISCV_ISA_EXT_ZACAS)
319
320union __u128_halves {
321 u128 full;
322 struct {
323 u64 low, high;
324 };
325};
326
327#define __arch_cmpxchg128(p, o, n, cas_sfx) \
328({ \
329 __typeof__(*(p)) __o = (o); \
330 union __u128_halves __hn = { .full = (n) }; \
331 union __u128_halves __ho = { .full = (__o) }; \
332 register unsigned long t1 asm ("t1") = __hn.low; \
333 register unsigned long t2 asm ("t2") = __hn.high; \
334 register unsigned long t3 asm ("t3") = __ho.low; \
335 register unsigned long t4 asm ("t4") = __ho.high; \
336 \
337 __asm__ __volatile__ ( \
338 " amocas.q" cas_sfx " %0, %z3, %2" \
339 : "+&r" (t3), "+&r" (t4), "+A" (*(p)) \
340 : "rJ" (t1), "rJ" (t2) \
341 : "memory"); \
342 \
343 ((u128)t4 << 64) | t3; \
344})
345
346#define arch_cmpxchg128(ptr, o, n) \
347 __arch_cmpxchg128((ptr), (o), (n), ".aqrl")
348
349#define arch_cmpxchg128_local(ptr, o, n) \
350 __arch_cmpxchg128((ptr), (o), (n), "")
351
352#endif /* CONFIG_64BIT && CONFIG_RISCV_ISA_ZACAS */
353
229#ifdef CONFIG_RISCV_ISA_ZAWRS354#ifdef CONFIG_RISCV_ISA_ZAWRS
230/*355/*
231 * Despite wrs.nto being "WRS-with-no-timeout", in the absence of changes to356 * Despite wrs.nto being "WRS-with-no-timeout", in the absence of changes to
@@ -245,6 +370,11 @@ static __always_inline void __cmpwait(volatile void *ptr, · 13 unmodified lines
245 : : : : no_zawrs);370 : : : : no_zawrs);
246 371
247 switch (size) {372 switch (size) {
373 case 1:
374 fallthrough;
375 case 2:
376 /* RISC-V doesn't have lr instructions on byte and half-word. */
377 goto no_zawrs;
248 case 4:378 case 4:
249 asm volatile(379 asm volatile(
250 " lr.w %0, %1\n"380 " lr.w %0, %1\n"
arch/riscv/include/asm/compat.h+0−1
@@ -9,7 +9,6 @@ · 8 unmodified lines
9 */9 */
10#include <linux/types.h>10#include <linux/types.h>
11#include <linux/sched.h>11#include <linux/sched.h>
12#include <linux/sched/task_stack.h>
13#include <asm-generic/compat.h>12#include <asm-generic/compat.h>
14 13
15static inline int is_compat_task(void)14static inline int is_compat_task(void)
arch/riscv/include/asm/cpufeature-macros.h+66−0
@@ -0,0 +1,66 @@
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright 2022-2024 Rivos, Inc
4 */
5
6#ifndef _ASM_CPUFEATURE_MACROS_H
7#define _ASM_CPUFEATURE_MACROS_H
8
9#include <asm/hwcap.h>
10#include <asm/alternative-macros.h>
11
12#define STANDARD_EXT 0
13
14bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit);
15#define riscv_isa_extension_available(isa_bitmap, ext) \
16 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_##ext)
17
18static __always_inline bool __riscv_has_extension_likely(const unsigned long vendor,
19 const unsigned long ext)
20{
21 asm goto(ALTERNATIVE("j %l[l_no]", "nop", %[vendor], %[ext], 1)
22 :
23 : [vendor] "i" (vendor), [ext] "i" (ext)
24 :
25 : l_no);
26
27 return true;
28l_no:
29 return false;
30}
31
32static __always_inline bool __riscv_has_extension_unlikely(const unsigned long vendor,
33 const unsigned long ext)
34{
35 asm goto(ALTERNATIVE("nop", "j %l[l_yes]", %[vendor], %[ext], 1)
36 :
37 : [vendor] "i" (vendor), [ext] "i" (ext)
38 :
39 : l_yes);
40
41 return false;
42l_yes:
43 return true;
44}
45
46static __always_inline bool riscv_has_extension_unlikely(const unsigned long ext)
47{
48 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
49
50 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
51 return __riscv_has_extension_unlikely(STANDARD_EXT, ext);
52
53 return __riscv_isa_extension_available(NULL, ext);
54}
55
56static __always_inline bool riscv_has_extension_likely(const unsigned long ext)
57{
58 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
59
60 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
61 return __riscv_has_extension_likely(STANDARD_EXT, ext);
62
63 return __riscv_isa_extension_available(NULL, ext);
64}
65
66#endif /* _ASM_CPUFEATURE_MACROS_H */
arch/riscv/include/asm/cpufeature.h+14−59
@@ -8,9 +8,12 @@ · 7 unmodified lines
8 8
9#include <linux/bitmap.h>9#include <linux/bitmap.h>
10#include <linux/jump_label.h>10#include <linux/jump_label.h>
11#include <linux/workqueue.h>
12#include <linux/kconfig.h>
13#include <linux/percpu-defs.h>
14#include <linux/threads.h>
11#include <asm/hwcap.h>15#include <asm/hwcap.h>
12#include <asm/alternative-macros.h>16#include <asm/cpufeature-macros.h>
13#include <asm/errno.h>
14 17
15/*18/*
16 * These are probed via a device_initcall(), via either the SBI or directly19 * These are probed via a device_initcall(), via either the SBI or directly
@@ -31,7 +34,7 @@ DECLARE_PER_CPU(struct riscv_cpuinfo, riscv_cpuinfo); · 14 unmodified lines
31/* Per-cpu ISA extensions. */34/* Per-cpu ISA extensions. */
32extern struct riscv_isainfo hart_isa[NR_CPUS];35extern struct riscv_isainfo hart_isa[NR_CPUS];
33 36
34void riscv_user_isa_enable(void);37void __init riscv_user_isa_enable(void);
35 38
36#define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size, _validate) { \39#define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size, _validate) { \
37 .name = #_name, \40 .name = #_name, \
@@ -58,8 +61,9 @@ void riscv_user_isa_enable(void); · 20 unmodified lines
58#define __RISCV_ISA_EXT_SUPERSET_VALIDATE(_name, _id, _sub_exts, _validate) \61#define __RISCV_ISA_EXT_SUPERSET_VALIDATE(_name, _id, _sub_exts, _validate) \
59 _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), _validate)62 _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), _validate)
60 63
61#if defined(CONFIG_RISCV_MISALIGNED)
62bool check_unaligned_access_emulated_all_cpus(void);64bool check_unaligned_access_emulated_all_cpus(void);
65#if defined(CONFIG_RISCV_SCALAR_MISALIGNED)
66void check_unaligned_access_emulated(struct work_struct *work __always_unused);
63void unaligned_emulation_finish(void);67void unaligned_emulation_finish(void);
64bool unaligned_ctl_available(void);68bool unaligned_ctl_available(void);
65DECLARE_PER_CPU(long, misaligned_access_speed);69DECLARE_PER_CPU(long, misaligned_access_speed);
@@ -70,6 +74,12 @@ static inline bool unaligned_ctl_available(void) · 4 unmodified lines
70}74}
71#endif75#endif
72 76
77bool check_vector_unaligned_access_emulated_all_cpus(void);
78#if defined(CONFIG_RISCV_VECTOR_MISALIGNED)
79void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused);
80DECLARE_PER_CPU(long, vector_misaligned_access);
81#endif
82
73#if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)83#if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)
74DECLARE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key);84DECLARE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key);
75 85
@@ -103,61 +113,6 @@ extern const size_t riscv_isa_ext_count; · 27 unmodified lines
103extern bool riscv_isa_fallback;113extern bool riscv_isa_fallback;
104 114
105unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap);115unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap);
106
107#define STANDARD_EXT 0
108
109bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit);
110#define riscv_isa_extension_available(isa_bitmap, ext) \
111 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_##ext)
112
113static __always_inline bool __riscv_has_extension_likely(const unsigned long vendor,
114 const unsigned long ext)
115{
116 asm goto(ALTERNATIVE("j %l[l_no]", "nop", %[vendor], %[ext], 1)
117 :
118 : [vendor] "i" (vendor), [ext] "i" (ext)
119 :
120 : l_no);
121
122 return true;
123l_no:
124 return false;
125}
126
127static __always_inline bool __riscv_has_extension_unlikely(const unsigned long vendor,
128 const unsigned long ext)
129{
130 asm goto(ALTERNATIVE("nop", "j %l[l_yes]", %[vendor], %[ext], 1)
131 :
132 : [vendor] "i" (vendor), [ext] "i" (ext)
133 :
134 : l_yes);
135
136 return false;
137l_yes:
138 return true;
139}
140
141static __always_inline bool riscv_has_extension_unlikely(const unsigned long ext)
142{
143 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
144
145 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
146 return __riscv_has_extension_unlikely(STANDARD_EXT, ext);
147
148 return __riscv_isa_extension_available(NULL, ext);
149}
150
151static __always_inline bool riscv_has_extension_likely(const unsigned long ext)
152{
153 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
154
155 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE))
156 return __riscv_has_extension_likely(STANDARD_EXT, ext);
157
158 return __riscv_isa_extension_available(NULL, ext);
159}
160
161static __always_inline bool riscv_cpu_has_extension_likely(int cpu, const unsigned long ext)116static __always_inline bool riscv_cpu_has_extension_likely(int cpu, const unsigned long ext)
162{117{
163 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");118 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
arch/riscv/include/asm/csr.h+16−0
@@ -119,6 +119,10 @@ · 118 unmodified lines
119 119
120/* HSTATUS flags */120/* HSTATUS flags */
121#ifdef CONFIG_64BIT121#ifdef CONFIG_64BIT
122#define HSTATUS_HUPMM _AC(0x3000000000000, UL)
123#define HSTATUS_HUPMM_PMLEN_0 _AC(0x0000000000000, UL)
124#define HSTATUS_HUPMM_PMLEN_7 _AC(0x2000000000000, UL)
125#define HSTATUS_HUPMM_PMLEN_16 _AC(0x3000000000000, UL)
122#define HSTATUS_VSXL _AC(0x300000000, UL)126#define HSTATUS_VSXL _AC(0x300000000, UL)
123#define HSTATUS_VSXL_SHIFT 32127#define HSTATUS_VSXL_SHIFT 32
124#endif128#endif
@@ -195,6 +199,10 @@ · 70 unmodified lines
195/* xENVCFG flags */199/* xENVCFG flags */
196#define ENVCFG_STCE (_AC(1, ULL) << 63)200#define ENVCFG_STCE (_AC(1, ULL) << 63)
197#define ENVCFG_PBMTE (_AC(1, ULL) << 62)201#define ENVCFG_PBMTE (_AC(1, ULL) << 62)
202#define ENVCFG_PMM (_AC(0x3, ULL) << 32)
203#define ENVCFG_PMM_PMLEN_0 (_AC(0x0, ULL) << 32)
204#define ENVCFG_PMM_PMLEN_7 (_AC(0x2, ULL) << 32)
205#define ENVCFG_PMM_PMLEN_16 (_AC(0x3, ULL) << 32)
198#define ENVCFG_CBZE (_AC(1, UL) << 7)206#define ENVCFG_CBZE (_AC(1, UL) << 7)
199#define ENVCFG_CBCFE (_AC(1, UL) << 6)207#define ENVCFG_CBCFE (_AC(1, UL) << 6)
200#define ENVCFG_CBIE_SHIFT 4208#define ENVCFG_CBIE_SHIFT 4
@@ -216,6 +224,12 @@ · 15 unmodified lines
216#define SMSTATEEN0_SSTATEEN0_SHIFT 63224#define SMSTATEEN0_SSTATEEN0_SHIFT 63
217#define SMSTATEEN0_SSTATEEN0 (_ULL(1) << SMSTATEEN0_SSTATEEN0_SHIFT)225#define SMSTATEEN0_SSTATEEN0 (_ULL(1) << SMSTATEEN0_SSTATEEN0_SHIFT)
218 226
227/* mseccfg bits */
228#define MSECCFG_PMM ENVCFG_PMM
229#define MSECCFG_PMM_PMLEN_0 ENVCFG_PMM_PMLEN_0
230#define MSECCFG_PMM_PMLEN_7 ENVCFG_PMM_PMLEN_7
231#define MSECCFG_PMM_PMLEN_16 ENVCFG_PMM_PMLEN_16
232
219/* symbolic CSR names: */233/* symbolic CSR names: */
220#define CSR_CYCLE 0xc00234#define CSR_CYCLE 0xc00
221#define CSR_TIME 0xc01235#define CSR_TIME 0xc01
@@ -382,6 +396,8 @@ · 160 unmodified lines
382#define CSR_MIP 0x344396#define CSR_MIP 0x344
383#define CSR_PMPCFG0 0x3a0397#define CSR_PMPCFG0 0x3a0
384#define CSR_PMPADDR0 0x3b0398#define CSR_PMPADDR0 0x3b0
399#define CSR_MSECCFG 0x747
400#define CSR_MSECCFGH 0x757
385#define CSR_MVENDORID 0xf11401#define CSR_MVENDORID 0xf11
386#define CSR_MARCHID 0xf12402#define CSR_MARCHID 0xf12
387#define CSR_MIMPID 0xf13403#define CSR_MIMPID 0xf13
arch/riscv/include/asm/entry-common.h+1−0
@@ -33,6 +33,7 @@ static inline int handle_misaligned_load(struct pt_regs *regs) · 32 unmodified lines
33{33{
34 return -1;34 return -1;
35}35}
36
36static inline int handle_misaligned_store(struct pt_regs *regs)37static inline int handle_misaligned_store(struct pt_regs *regs)
37{38{
38 return -1;39 return -1;
arch/riscv/include/asm/hwcap.h+7−0
@@ -93,6 +93,11 @@ · 92 unmodified lines
93#define RISCV_ISA_EXT_ZCMOP 8493#define RISCV_ISA_EXT_ZCMOP 84
94#define RISCV_ISA_EXT_ZAWRS 8594#define RISCV_ISA_EXT_ZAWRS 85
95#define RISCV_ISA_EXT_SVVPTC 8695#define RISCV_ISA_EXT_SVVPTC 86
96#define RISCV_ISA_EXT_SMMPM 87
97#define RISCV_ISA_EXT_SMNPM 88
98#define RISCV_ISA_EXT_SSNPM 89
99#define RISCV_ISA_EXT_ZABHA 90
100#define RISCV_ISA_EXT_ZICCRSE 91
96 101
97#define RISCV_ISA_EXT_XLINUXENVCFG 127102#define RISCV_ISA_EXT_XLINUXENVCFG 127
98 103
@@ -101,8 +106,10 @@ · 2 unmodified lines
101 106
102#ifdef CONFIG_RISCV_M_MODE107#ifdef CONFIG_RISCV_M_MODE
103#define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SMAIA108#define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SMAIA
109#define RISCV_ISA_EXT_SUPM RISCV_ISA_EXT_SMNPM
104#else110#else
105#define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SSAIA111#define RISCV_ISA_EXT_SxAIA RISCV_ISA_EXT_SSAIA
112#define RISCV_ISA_EXT_SUPM RISCV_ISA_EXT_SSNPM
106#endif113#endif
107 114
108#endif /* _ASM_RISCV_HWCAP_H */115#endif /* _ASM_RISCV_HWCAP_H */
arch/riscv/include/asm/hwprobe.h+1−1
@@ -8,7 +8,7 @@ · 7 unmodified lines
8 8
9#include <uapi/asm/hwprobe.h>9#include <uapi/asm/hwprobe.h>
10 10
11#define RISCV_HWPROBE_MAX_KEY 911#define RISCV_HWPROBE_MAX_KEY 10
12 12
13static inline bool riscv_hwprobe_key_is_valid(__s64 key)13static inline bool riscv_hwprobe_key_is_valid(__s64 key)
14{14{
arch/riscv/include/asm/mmu.h+7−0
@@ -25,9 +25,16 @@ typedef struct { · 24 unmodified lines
25#ifdef CONFIG_BINFMT_ELF_FDPIC25#ifdef CONFIG_BINFMT_ELF_FDPIC
26 unsigned long exec_fdpic_loadmap;26 unsigned long exec_fdpic_loadmap;
27 unsigned long interp_fdpic_loadmap;27 unsigned long interp_fdpic_loadmap;
28#endif
29 unsigned long flags;
30#ifdef CONFIG_RISCV_ISA_SUPM
31 u8 pmlen;
28#endif32#endif
29} mm_context_t;33} mm_context_t;
30 34
35/* Lock the pointer masking mode because this mm is multithreaded */
36#define MM_CONTEXT_LOCK_PMLEN 0
37
31#define cntx2asid(cntx) ((cntx) & SATP_ASID_MASK)38#define cntx2asid(cntx) ((cntx) & SATP_ASID_MASK)
32#define cntx2version(cntx) ((cntx) & ~SATP_ASID_MASK)39#define cntx2version(cntx) ((cntx) & ~SATP_ASID_MASK)
33 40
arch/riscv/include/asm/mmu_context.h+13−0
@@ -20,6 +20,9 @@ void switch_mm(struct mm_struct *prev, struct mm_struct *next, · 19 unmodified lines
20static inline void activate_mm(struct mm_struct *prev,20static inline void activate_mm(struct mm_struct *prev,
21 struct mm_struct *next)21 struct mm_struct *next)
22{22{
23#ifdef CONFIG_RISCV_ISA_SUPM
24 next->context.pmlen = 0;
25#endif
23 switch_mm(prev, next, NULL);26 switch_mm(prev, next, NULL);
24}27}
25 28
@@ -30,11 +33,21 @@ static inline int init_new_context(struct task_struct *tsk, · 4 unmodified lines
30#ifdef CONFIG_MMU33#ifdef CONFIG_MMU
31 atomic_long_set(&mm->context.id, 0);34 atomic_long_set(&mm->context.id, 0);
32#endif35#endif
36 if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM))
37 clear_bit(MM_CONTEXT_LOCK_PMLEN, &mm->context.flags);
33 return 0;38 return 0;
34}39}
35 40
36DECLARE_STATIC_KEY_FALSE(use_asid_allocator);41DECLARE_STATIC_KEY_FALSE(use_asid_allocator);
37 42
43#ifdef CONFIG_RISCV_ISA_SUPM
44#define mm_untag_mask mm_untag_mask
45static inline unsigned long mm_untag_mask(struct mm_struct *mm)
46{
47 return -1UL >> mm->context.pmlen;
48}
49#endif
50
38#include <asm-generic/mmu_context.h>51#include <asm-generic/mmu_context.h>
39 52
40#endif /* _ASM_RISCV_MMU_CONTEXT_H */53#endif /* _ASM_RISCV_MMU_CONTEXT_H */
arch/riscv/include/asm/processor.h+9−0
@@ -102,6 +102,7 @@ struct thread_struct { · 101 unmodified lines
102 unsigned long s[12]; /* s[0]: frame pointer */102 unsigned long s[12]; /* s[0]: frame pointer */
103 struct __riscv_d_ext_state fstate;103 struct __riscv_d_ext_state fstate;
104 unsigned long bad_cause;104 unsigned long bad_cause;
105 unsigned long envcfg;
105 u32 riscv_v_flags;106 u32 riscv_v_flags;
106 u32 vstate_ctrl;107 u32 vstate_ctrl;
107 struct __riscv_v_ext_state vstate;108 struct __riscv_v_ext_state vstate;
@@ -177,6 +178,14 @@ extern int set_unalign_ctl(struct task_struct *tsk, unsigned int val); · 69 unmodified lines
177#define RISCV_SET_ICACHE_FLUSH_CTX(arg1, arg2) riscv_set_icache_flush_ctx(arg1, arg2)178#define RISCV_SET_ICACHE_FLUSH_CTX(arg1, arg2) riscv_set_icache_flush_ctx(arg1, arg2)
178extern int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long per_thread);179extern int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long per_thread);
179 180
181#ifdef CONFIG_RISCV_ISA_SUPM
182/* PR_{SET,GET}_TAGGED_ADDR_CTRL prctl */
183long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg);
184long get_tagged_addr_ctrl(struct task_struct *task);
185#define SET_TAGGED_ADDR_CTRL(arg) set_tagged_addr_ctrl(current, arg)
186#define GET_TAGGED_ADDR_CTRL() get_tagged_addr_ctrl(current)
187#endif
188
180#endif /* __ASSEMBLY__ */189#endif /* __ASSEMBLY__ */
181 190
182#endif /* _ASM_RISCV_PROCESSOR_H */191#endif /* _ASM_RISCV_PROCESSOR_H */
arch/riscv/include/asm/spinlock.h+47−0
@@ -0,0 +1,47 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2
3#ifndef __ASM_RISCV_SPINLOCK_H
4#define __ASM_RISCV_SPINLOCK_H
5
6#ifdef CONFIG_RISCV_COMBO_SPINLOCKS
7#define _Q_PENDING_LOOPS (1 << 9)
8
9#define __no_arch_spinlock_redefine
10#include <asm/ticket_spinlock.h>
11#include <asm/qspinlock.h>
12#include <asm/jump_label.h>
13
14/*
15 * TODO: Use an alternative instead of a static key when we are able to parse
16 * the extensions string earlier in the boot process.
17 */
18DECLARE_STATIC_KEY_TRUE(qspinlock_key);
19
20#define SPINLOCK_BASE_DECLARE(op, type, type_lock) \
21static __always_inline type arch_spin_##op(type_lock lock) \
22{ \
23 if (static_branch_unlikely(&qspinlock_key)) \
24 return queued_spin_##op(lock); \
25 return ticket_spin_##op(lock); \
26}
27
28SPINLOCK_BASE_DECLARE(lock, void, arch_spinlock_t *)
29SPINLOCK_BASE_DECLARE(unlock, void, arch_spinlock_t *)
30SPINLOCK_BASE_DECLARE(is_locked, int, arch_spinlock_t *)
31SPINLOCK_BASE_DECLARE(is_contended, int, arch_spinlock_t *)
32SPINLOCK_BASE_DECLARE(trylock, bool, arch_spinlock_t *)
33SPINLOCK_BASE_DECLARE(value_unlocked, int, arch_spinlock_t)
34
35#elif defined(CONFIG_RISCV_QUEUED_SPINLOCKS)
36
37#include <asm/qspinlock.h>
38
39#else
40
41#include <asm/ticket_spinlock.h>
42
43#endif
44
45#include <asm/qrwlock.h>
46
47#endif /* __ASM_RISCV_SPINLOCK_H */
arch/riscv/include/asm/switch_to.h+19−0
@@ -70,6 +70,24 @@ static __always_inline bool has_fpu(void) { return false; } · 69 unmodified lines
70#define __switch_to_fpu(__prev, __next) do { } while (0)70#define __switch_to_fpu(__prev, __next) do { } while (0)
71#endif71#endif
72 72
73static inline void envcfg_update_bits(struct task_struct *task,
74 unsigned long mask, unsigned long val)
75{
76 unsigned long envcfg;
77
78 envcfg = (task->thread.envcfg & ~mask) | val;
79 task->thread.envcfg = envcfg;
80 if (task == current)
81 csr_write(CSR_ENVCFG, envcfg);
82}
83
84static inline void __switch_to_envcfg(struct task_struct *next)
85{
86 asm volatile (ALTERNATIVE("nop", "csrw " __stringify(CSR_ENVCFG) ", %0",
87 0, RISCV_ISA_EXT_XLINUXENVCFG, 1)
88 :: "r" (next->thread.envcfg) : "memory");
89}
90
73extern struct task_struct *__switch_to(struct task_struct *,91extern struct task_struct *__switch_to(struct task_struct *,
74 struct task_struct *);92 struct task_struct *);
75 93
@@ -103,6 +121,7 @@ do { \ · 27 unmodified lines
103 __switch_to_vector(__prev, __next); \121 __switch_to_vector(__prev, __next); \
104 if (switch_to_should_flush_icache(__next)) \122 if (switch_to_should_flush_icache(__next)) \
105 local_flush_icache_all(); \123 local_flush_icache_all(); \
124 __switch_to_envcfg(__next); \
106 ((last) = __switch_to(__prev, __next)); \125 ((last) = __switch_to(__prev, __next)); \
107} while (0)126} while (0)
108 127
arch/riscv/include/asm/uaccess.h+38−5
@@ -9,8 +9,41 @@ · 8 unmodified lines
9#define _ASM_RISCV_UACCESS_H9#define _ASM_RISCV_UACCESS_H
10 10
11#include <asm/asm-extable.h>11#include <asm/asm-extable.h>
12#include <asm/cpufeature.h>
12#include <asm/pgtable.h> /* for TASK_SIZE */13#include <asm/pgtable.h> /* for TASK_SIZE */
13 14
15#ifdef CONFIG_RISCV_ISA_SUPM
16static inline unsigned long __untagged_addr_remote(struct mm_struct *mm, unsigned long addr)
17{
18 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM)) {
19 u8 pmlen = mm->context.pmlen;
20
21 /* Virtual addresses are sign-extended; physical addresses are zero-extended. */
22 if (IS_ENABLED(CONFIG_MMU))
23 return (long)(addr << pmlen) >> pmlen;
24 else
25 return (addr << pmlen) >> pmlen;
26 }
27
28 return addr;
29}
30
31#define untagged_addr(addr) ({ \
32 unsigned long __addr = (__force unsigned long)(addr); \
33 (__force __typeof__(addr))__untagged_addr_remote(current->mm, __addr); \
34})
35
36#define untagged_addr_remote(mm, addr) ({ \
37 unsigned long __addr = (__force unsigned long)(addr); \
38 mmap_assert_locked(mm); \
39 (__force __typeof__(addr))__untagged_addr_remote(mm, __addr); \
40})
41
42#define access_ok(addr, size) likely(__access_ok(untagged_addr(addr), size))
43#else
44#define untagged_addr(addr) (addr)
45#endif
46
14/*47/*
15 * User space memory access functions48 * User space memory access functions
16 */49 */
@@ -130,7 +163,7 @@ do { \ · 113 unmodified lines
130 */163 */
131#define __get_user(x, ptr) \164#define __get_user(x, ptr) \
132({ \165({ \
133 const __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \166 const __typeof__(*(ptr)) __user *__gu_ptr = untagged_addr(ptr); \
134 long __gu_err = 0; \167 long __gu_err = 0; \
135 \168 \
136 __chk_user_ptr(__gu_ptr); \169 __chk_user_ptr(__gu_ptr); \
@@ -246,7 +279,7 @@ do { \ · 109 unmodified lines
246 */279 */
247#define __put_user(x, ptr) \280#define __put_user(x, ptr) \
248({ \281({ \
249 __typeof__(*(ptr)) __user *__gu_ptr = (ptr); \282 __typeof__(*(ptr)) __user *__gu_ptr = untagged_addr(ptr); \
250 __typeof__(*__gu_ptr) __val = (x); \283 __typeof__(*__gu_ptr) __val = (x); \
251 long __pu_err = 0; \284 long __pu_err = 0; \
252 \285 \
@@ -293,13 +326,13 @@ unsigned long __must_check __asm_copy_from_user(void *to, · 40 unmodified lines
293static inline unsigned long326static inline unsigned long
294raw_copy_from_user(void *to, const void __user *from, unsigned long n)327raw_copy_from_user(void *to, const void __user *from, unsigned long n)
295{328{
296 return __asm_copy_from_user(to, from, n);329 return __asm_copy_from_user(to, untagged_addr(from), n);
297}330}
298 331
299static inline unsigned long332static inline unsigned long
300raw_copy_to_user(void __user *to, const void *from, unsigned long n)333raw_copy_to_user(void __user *to, const void *from, unsigned long n)
301{334{
302 return __asm_copy_to_user(to, from, n);335 return __asm_copy_to_user(untagged_addr(to), from, n);
303}336}
304 337
305extern long strncpy_from_user(char *dest, const char __user *src, long count);338extern long strncpy_from_user(char *dest, const char __user *src, long count);
@@ -314,7 +347,7 @@ unsigned long __must_check clear_user(void __user *to, unsigned long n) · 8 unmodified lines
314{347{
315 might_fault();348 might_fault();
316 return access_ok(to, n) ?349 return access_ok(to, n) ?
317 __clear_user(to, n) : n;350 __clear_user(untagged_addr(to), n) : n;
318}351}
319 352
320#define __get_kernel_nofault(dst, src, type, err_label) \353#define __get_kernel_nofault(dst, src, type, err_label) \
arch/riscv/include/asm/vector.h+2−0
@@ -21,6 +21,7 @@ · 20 unmodified lines
21 21
22extern unsigned long riscv_v_vsize;22extern unsigned long riscv_v_vsize;
23int riscv_v_setup_vsize(void);23int riscv_v_setup_vsize(void);
24bool insn_is_vector(u32 insn_buf);
24bool riscv_v_first_use_handler(struct pt_regs *regs);25bool riscv_v_first_use_handler(struct pt_regs *regs);
25void kernel_vector_begin(void);26void kernel_vector_begin(void);
26void kernel_vector_end(void);27void kernel_vector_end(void);
@@ -268,6 +269,7 @@ struct pt_regs; · 241 unmodified lines
268 269
269static inline int riscv_v_setup_vsize(void) { return -EOPNOTSUPP; }270static inline int riscv_v_setup_vsize(void) { return -EOPNOTSUPP; }
270static __always_inline bool has_vector(void) { return false; }271static __always_inline bool has_vector(void) { return false; }
272static __always_inline bool insn_is_vector(u32 insn_buf) { return false; }
271static inline bool riscv_v_first_use_handler(struct pt_regs *regs) { return false; }273static inline bool riscv_v_first_use_handler(struct pt_regs *regs) { return false; }
272static inline bool riscv_v_vstate_query(struct pt_regs *regs) { return false; }274static inline bool riscv_v_vstate_query(struct pt_regs *regs) { return false; }
273static inline bool riscv_v_vstate_ctrl_user_allowed(void) { return false; }275static inline bool riscv_v_vstate_ctrl_user_allowed(void) { return false; }
arch/riscv/include/uapi/asm/hwprobe.h+6−0
@@ -72,6 +72,7 @@ struct riscv_hwprobe { · 71 unmodified lines
72#define RISCV_HWPROBE_EXT_ZCF (1ULL << 46)72#define RISCV_HWPROBE_EXT_ZCF (1ULL << 46)
73#define RISCV_HWPROBE_EXT_ZCMOP (1ULL << 47)73#define RISCV_HWPROBE_EXT_ZCMOP (1ULL << 47)
74#define RISCV_HWPROBE_EXT_ZAWRS (1ULL << 48)74#define RISCV_HWPROBE_EXT_ZAWRS (1ULL << 48)
75#define RISCV_HWPROBE_EXT_SUPM (1ULL << 49)
75#define RISCV_HWPROBE_KEY_CPUPERF_0 576#define RISCV_HWPROBE_KEY_CPUPERF_0 5
76#define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0)77#define RISCV_HWPROBE_MISALIGNED_UNKNOWN (0 << 0)
77#define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0)78#define RISCV_HWPROBE_MISALIGNED_EMULATED (1 << 0)
@@ -88,6 +89,11 @@ struct riscv_hwprobe { · 10 unmodified lines
88#define RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW 289#define RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW 2
89#define RISCV_HWPROBE_MISALIGNED_SCALAR_FAST 390#define RISCV_HWPROBE_MISALIGNED_SCALAR_FAST 3
90#define RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED 491#define RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED 4
92#define RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF 10
93#define RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN 0
94#define RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW 2
95#define RISCV_HWPROBE_MISALIGNED_VECTOR_FAST 3
96#define RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED 4
91/* Increase RISCV_HWPROBE_MAX_KEY when adding items. */97/* Increase RISCV_HWPROBE_MAX_KEY when adding items. */
92 98
93/* Flags */99/* Flags */
arch/riscv/include/uapi/asm/kvm.h+2−0
@@ -175,6 +175,8 @@ enum KVM_RISCV_ISA_EXT_ID { · 174 unmodified lines
175 KVM_RISCV_ISA_EXT_ZCF,175 KVM_RISCV_ISA_EXT_ZCF,
176 KVM_RISCV_ISA_EXT_ZCMOP,176 KVM_RISCV_ISA_EXT_ZCMOP,
177 KVM_RISCV_ISA_EXT_ZAWRS,177 KVM_RISCV_ISA_EXT_ZAWRS,
178 KVM_RISCV_ISA_EXT_SMNPM,
179 KVM_RISCV_ISA_EXT_SSNPM,
178 KVM_RISCV_ISA_EXT_MAX,180 KVM_RISCV_ISA_EXT_MAX,
179};181};
180 182
arch/riscv/kernel/Makefile+2−1
@@ -75,7 +75,8 @@ obj-$(CONFIG_MMU) += vdso.o vdso/ · 74 unmodified lines
75 75
76obj-$(CONFIG_RISCV_MISALIGNED) += traps_misaligned.o76obj-$(CONFIG_RISCV_MISALIGNED) += traps_misaligned.o
77obj-$(CONFIG_RISCV_MISALIGNED) += unaligned_access_speed.o77obj-$(CONFIG_RISCV_MISALIGNED) += unaligned_access_speed.o
78obj-$(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) += copy-unaligned.o78obj-$(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) += copy-unaligned.o
79obj-$(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS) += vec-copy-unaligned.o
79 80
80obj-$(CONFIG_FPU) += fpu.o81obj-$(CONFIG_FPU) += fpu.o
81obj-$(CONFIG_FPU) += kernel_mode_fpu.o82obj-$(CONFIG_FPU) += kernel_mode_fpu.o
arch/riscv/kernel/copy-unaligned.h+5−0
@@ -10,4 +10,9 @@ · 9 unmodified lines
10void __riscv_copy_words_unaligned(void *dst, const void *src, size_t size);10void __riscv_copy_words_unaligned(void *dst, const void *src, size_t size);
11void __riscv_copy_bytes_unaligned(void *dst, const void *src, size_t size);11void __riscv_copy_bytes_unaligned(void *dst, const void *src, size_t size);
12 12
13#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
14void __riscv_copy_vec_words_unaligned(void *dst, const void *src, size_t size);
15void __riscv_copy_vec_bytes_unaligned(void *dst, const void *src, size_t size);
16#endif
17
13#endif /* __RISCV_KERNEL_COPY_UNALIGNED_H */18#endif /* __RISCV_KERNEL_COPY_UNALIGNED_H */
arch/riscv/kernel/cpufeature.c+13−3
@@ -29,6 +29,8 @@ · 28 unmodified lines
29 29
30#define NUM_ALPHA_EXTS ('z' - 'a' + 1)30#define NUM_ALPHA_EXTS ('z' - 'a' + 1)
31 31
32static bool any_cpu_has_zicboz;
33
32unsigned long elf_hwcap __read_mostly;34unsigned long elf_hwcap __read_mostly;
33 35
34/* Host ISA bitmap */36/* Host ISA bitmap */
@@ -99,6 +101,7 @@ static int riscv_ext_zicboz_validate(const struct riscv_isa_ext_data *data, · 64 unmodified lines
99 pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n");101 pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n");
100 return -EINVAL;102 return -EINVAL;
101 }103 }
104 any_cpu_has_zicboz = true;
102 return 0;105 return 0;
103}106}
104 107
@@ -315,6 +318,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { · 210 unmodified lines
315 riscv_ext_zicbom_validate),318 riscv_ext_zicbom_validate),
316 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts,319 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts,
317 riscv_ext_zicboz_validate),320 riscv_ext_zicboz_validate),
321 __RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE),
318 __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR),322 __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR),
319 __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND),323 __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND),
320 __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR),324 __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR),
@@ -323,6 +327,7 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { · 2 unmodified lines
323 __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE),327 __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE),
324 __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM),328 __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM),
325 __RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP),329 __RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP),
330 __RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA),
326 __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS),331 __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS),
327 __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS),332 __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS),
328 __RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA),333 __RISCV_ISA_EXT_DATA(zfa, RISCV_ISA_EXT_ZFA),
@@ -375,9 +380,12 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { · 46 unmodified lines
375 __RISCV_ISA_EXT_BUNDLE(zvksg, riscv_zvksg_bundled_exts),380 __RISCV_ISA_EXT_BUNDLE(zvksg, riscv_zvksg_bundled_exts),
376 __RISCV_ISA_EXT_DATA(zvkt, RISCV_ISA_EXT_ZVKT),381 __RISCV_ISA_EXT_DATA(zvkt, RISCV_ISA_EXT_ZVKT),
377 __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA),382 __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA),
383 __RISCV_ISA_EXT_DATA(smmpm, RISCV_ISA_EXT_SMMPM),
384 __RISCV_ISA_EXT_SUPERSET(smnpm, RISCV_ISA_EXT_SMNPM, riscv_xlinuxenvcfg_exts),
378 __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN),385 __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN),
379 __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA),386 __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA),
380 __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF),387 __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF),
388 __RISCV_ISA_EXT_SUPERSET(ssnpm, RISCV_ISA_EXT_SSNPM, riscv_xlinuxenvcfg_exts),
381 __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC),389 __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC),
382 __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),390 __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),
383 __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT),391 __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT),
@@ -918,10 +926,12 @@ unsigned long riscv_get_elf_hwcap(void) · 534 unmodified lines
918 return hwcap;926 return hwcap;
919}927}
920 928
921void riscv_user_isa_enable(void)929void __init riscv_user_isa_enable(void)
922{930{
923 if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_ZICBOZ))931 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOZ))
924 csr_set(CSR_ENVCFG, ENVCFG_CBZE);932 current->thread.envcfg |= ENVCFG_CBZE;
933 else if (any_cpu_has_zicboz)
934 pr_warn("Zicboz disabled as it is unavailable on some harts\n");
925}935}
926 936
927#ifdef CONFIG_RISCV_ALTERNATIVE937#ifdef CONFIG_RISCV_ALTERNATIVE
arch/riscv/kernel/fpu.S+2−2
@@ -170,7 +170,7 @@ SYM_FUNC_END(__fstate_restore) · 169 unmodified lines
170 __access_func(f31)170 __access_func(f31)
171 171
172 172
173#ifdef CONFIG_RISCV_MISALIGNED173#ifdef CONFIG_RISCV_SCALAR_MISALIGNED
174 174
175/*175/*
176 * Disable compressed instructions set to keep a constant offset between FP176 * Disable compressed instructions set to keep a constant offset between FP
@@ -224,4 +224,4 @@ SYM_FUNC_START(get_f64_reg) · 47 unmodified lines
224 fp_access_epilogue224 fp_access_epilogue
225SYM_FUNC_END(get_f64_reg)225SYM_FUNC_END(get_f64_reg)
226 226
227#endif /* CONFIG_RISCV_MISALIGNED */227#endif /* CONFIG_RISCV_SCALAR_MISALIGNED */
arch/riscv/kernel/process.c+154−0
@@ -7,6 +7,7 @@ · 6 unmodified lines
7 * Copyright (C) 2017 SiFive7 * Copyright (C) 2017 SiFive
8 */8 */
9 9
10#include <linux/bitfield.h>
10#include <linux/cpu.h>11#include <linux/cpu.h>
11#include <linux/kernel.h>12#include <linux/kernel.h>
12#include <linux/sched.h>13#include <linux/sched.h>
@@ -180,6 +181,10 @@ void flush_thread(void) · 167 unmodified lines
180 memset(&current->thread.vstate, 0, sizeof(struct __riscv_v_ext_state));181 memset(&current->thread.vstate, 0, sizeof(struct __riscv_v_ext_state));
181 clear_tsk_thread_flag(current, TIF_RISCV_V_DEFER_RESTORE);182 clear_tsk_thread_flag(current, TIF_RISCV_V_DEFER_RESTORE);
182#endif183#endif
184#ifdef CONFIG_RISCV_ISA_SUPM
185 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM))
186 envcfg_update_bits(current, ENVCFG_PMM, ENVCFG_PMM_PMLEN_0);
187#endif
183}188}
184 189
185void arch_release_task_struct(struct task_struct *tsk)190void arch_release_task_struct(struct task_struct *tsk)
@@ -208,6 +213,10 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) · 22 unmodified lines
208 unsigned long tls = args->tls;213 unsigned long tls = args->tls;
209 struct pt_regs *childregs = task_pt_regs(p);214 struct pt_regs *childregs = task_pt_regs(p);
210 215
216 /* Ensure all threads in this mm have the same pointer masking mode. */
217 if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM) && p->mm && (clone_flags & CLONE_VM))
218 set_bit(MM_CONTEXT_LOCK_PMLEN, &p->mm->context.flags);
219
211 memset(&p->thread.s, 0, sizeof(p->thread.s));220 memset(&p->thread.s, 0, sizeof(p->thread.s));
212 221
213 /* p->thread holds context to be restored by __switch_to() */222 /* p->thread holds context to be restored by __switch_to() */
@@ -242,3 +251,148 @@ void __init arch_task_cache_init(void) · 28 unmodified lines
242{251{
243 riscv_v_setup_ctx_cache();252 riscv_v_setup_ctx_cache();
244}253}
254
255#ifdef CONFIG_RISCV_ISA_SUPM
256enum {
257 PMLEN_0 = 0,
258 PMLEN_7 = 7,
259 PMLEN_16 = 16,
260};
261
262static bool have_user_pmlen_7;
263static bool have_user_pmlen_16;
264
265/*
266 * Control the relaxed ABI allowing tagged user addresses into the kernel.
267 */
268static unsigned int tagged_addr_disabled;
269
270long set_tagged_addr_ctrl(struct task_struct *task, unsigned long arg)
271{
272 unsigned long valid_mask = PR_PMLEN_MASK | PR_TAGGED_ADDR_ENABLE;
273 struct thread_info *ti = task_thread_info(task);
274 struct mm_struct *mm = task->mm;
275 unsigned long pmm;
276 u8 pmlen;
277
278 if (is_compat_thread(ti))
279 return -EINVAL;
280
281 if (arg & ~valid_mask)
282 return -EINVAL;
283
284 /*
285 * Prefer the smallest PMLEN that satisfies the user's request,
286 * in case choosing a larger PMLEN has a performance impact.
287 */
288 pmlen = FIELD_GET(PR_PMLEN_MASK, arg);
289 if (pmlen == PMLEN_0) {
290 pmm = ENVCFG_PMM_PMLEN_0;
291 } else if (pmlen <= PMLEN_7 && have_user_pmlen_7) {
292 pmlen = PMLEN_7;
293 pmm = ENVCFG_PMM_PMLEN_7;
294 } else if (pmlen <= PMLEN_16 && have_user_pmlen_16) {
295 pmlen = PMLEN_16;
296 pmm = ENVCFG_PMM_PMLEN_16;
297 } else {
298 return -EINVAL;
299 }
300
301 /*
302 * Do not allow the enabling of the tagged address ABI if globally
303 * disabled via sysctl abi.tagged_addr_disabled, if pointer masking
304 * is disabled for userspace.
305 */
306 if (arg & PR_TAGGED_ADDR_ENABLE && (tagged_addr_disabled || !pmlen))
307 return -EINVAL;
308
309 if (!(arg & PR_TAGGED_ADDR_ENABLE))
310 pmlen = PMLEN_0;
311
312 if (mmap_write_lock_killable(mm))
313 return -EINTR;
314
315 if (test_bit(MM_CONTEXT_LOCK_PMLEN, &mm->context.flags) && mm->context.pmlen != pmlen) {
316 mmap_write_unlock(mm);
317 return -EBUSY;
318 }
319
320 envcfg_update_bits(task, ENVCFG_PMM, pmm);
321 mm->context.pmlen = pmlen;
322
323 mmap_write_unlock(mm);
324
325 return 0;
326}
327
328long get_tagged_addr_ctrl(struct task_struct *task)
329{
330 struct thread_info *ti = task_thread_info(task);
331 long ret = 0;
332
333 if (is_compat_thread(ti))
334 return -EINVAL;
335
336 /*
337 * The mm context's pmlen is set only when the tagged address ABI is
338 * enabled, so the effective PMLEN must be extracted from envcfg.PMM.
339 */
340 switch (task->thread.envcfg & ENVCFG_PMM) {
341 case ENVCFG_PMM_PMLEN_7:
342 ret = FIELD_PREP(PR_PMLEN_MASK, PMLEN_7);
343 break;
344 case ENVCFG_PMM_PMLEN_16:
345 ret = FIELD_PREP(PR_PMLEN_MASK, PMLEN_16);
346 break;
347 }
348
349 if (task->mm->context.pmlen)
350 ret |= PR_TAGGED_ADDR_ENABLE;
351
352 return ret;
353}
354
355static bool try_to_set_pmm(unsigned long value)
356{
357 csr_set(CSR_ENVCFG, value);
358 return (csr_read_clear(CSR_ENVCFG, ENVCFG_PMM) & ENVCFG_PMM) == value;
359}
360
361/*
362 * Global sysctl to disable the tagged user addresses support. This control
363 * only prevents the tagged address ABI enabling via prctl() and does not
364 * disable it for tasks that already opted in to the relaxed ABI.
365 */
366
367static struct ctl_table tagged_addr_sysctl_table[] = {
368 {
369 .procname = "tagged_addr_disabled",
370 .mode = 0644,
371 .data = &tagged_addr_disabled,
372 .maxlen = sizeof(int),
373 .proc_handler = proc_dointvec_minmax,
374 .extra1 = SYSCTL_ZERO,
375 .extra2 = SYSCTL_ONE,
376 },
377};
378
379static int __init tagged_addr_init(void)
380{
381 if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM))
382 return 0;
383
384 /*
385 * envcfg.PMM is a WARL field. Detect which values are supported.
386 * Assume the supported PMLEN values are the same on all harts.
387 */
388 csr_clear(CSR_ENVCFG, ENVCFG_PMM);
389 have_user_pmlen_7 = try_to_set_pmm(ENVCFG_PMM_PMLEN_7);
390 have_user_pmlen_16 = try_to_set_pmm(ENVCFG_PMM_PMLEN_16);
391
392 if (!register_sysctl("abi", tagged_addr_sysctl_table))
393 return -EINVAL;
394
395 return 0;
396}
397core_initcall(tagged_addr_init);
398#endif /* CONFIG_RISCV_ISA_SUPM */
arch/riscv/kernel/ptrace.c+42−0
@@ -28,6 +28,9 @@ enum riscv_regset { · 27 unmodified lines
28#ifdef CONFIG_RISCV_ISA_V28#ifdef CONFIG_RISCV_ISA_V
29 REGSET_V,29 REGSET_V,
30#endif30#endif
31#ifdef CONFIG_RISCV_ISA_SUPM
32 REGSET_TAGGED_ADDR_CTRL,
33#endif
31};34};
32 35
33static int riscv_gpr_get(struct task_struct *target,36static int riscv_gpr_get(struct task_struct *target,
@@ -152,6 +155,35 @@ static int riscv_vr_set(struct task_struct *target, · 118 unmodified lines
152}155}
153#endif156#endif
154 157
158#ifdef CONFIG_RISCV_ISA_SUPM
159static int tagged_addr_ctrl_get(struct task_struct *target,
160 const struct user_regset *regset,
161 struct membuf to)
162{
163 long ctrl = get_tagged_addr_ctrl(target);
164
165 if (IS_ERR_VALUE(ctrl))
166 return ctrl;
167
168 return membuf_write(&to, &ctrl, sizeof(ctrl));
169}
170
171static int tagged_addr_ctrl_set(struct task_struct *target,
172 const struct user_regset *regset,
173 unsigned int pos, unsigned int count,
174 const void *kbuf, const void __user *ubuf)
175{
176 int ret;
177 long ctrl;
178
179 ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &ctrl, 0, -1);
180 if (ret)
181 return ret;
182
183 return set_tagged_addr_ctrl(target, ctrl);
184}
185#endif
186
155static const struct user_regset riscv_user_regset[] = {187static const struct user_regset riscv_user_regset[] = {
156 [REGSET_X] = {188 [REGSET_X] = {
157 .core_note_type = NT_PRSTATUS,189 .core_note_type = NT_PRSTATUS,
@@ -182,6 +214,16 @@ static const struct user_regset riscv_user_regset[] = { · 24 unmodified lines
182 .set = riscv_vr_set,214 .set = riscv_vr_set,
183 },215 },
184#endif216#endif
217#ifdef CONFIG_RISCV_ISA_SUPM
218 [REGSET_TAGGED_ADDR_CTRL] = {
219 .core_note_type = NT_RISCV_TAGGED_ADDR_CTRL,
220 .n = 1,
221 .size = sizeof(long),
222 .align = sizeof(long),
223 .regset_get = tagged_addr_ctrl_get,
224 .set = tagged_addr_ctrl_set,
225 },
226#endif
185};227};
186 228
187static const struct user_regset_view riscv_user_native_view = {229static const struct user_regset_view riscv_user_native_view = {
arch/riscv/kernel/setup.c+37−0
@@ -244,6 +244,42 @@ static void __init parse_dtb(void) · 243 unmodified lines
244#endif244#endif
245}245}
246 246
247#if defined(CONFIG_RISCV_COMBO_SPINLOCKS)
248DEFINE_STATIC_KEY_TRUE(qspinlock_key);
249EXPORT_SYMBOL(qspinlock_key);
250#endif
251
252static void __init riscv_spinlock_init(void)
253{
254 char *using_ext = NULL;
255
256 if (IS_ENABLED(CONFIG_RISCV_TICKET_SPINLOCKS)) {
257 pr_info("Ticket spinlock: enabled\n");
258 return;
259 }
260
261 if (IS_ENABLED(CONFIG_RISCV_ISA_ZABHA) &&
262 IS_ENABLED(CONFIG_RISCV_ISA_ZACAS) &&
263 riscv_isa_extension_available(NULL, ZABHA) &&
264 riscv_isa_extension_available(NULL, ZACAS)) {
265 using_ext = "using Zabha";
266 } else if (riscv_isa_extension_available(NULL, ZICCRSE)) {
267 using_ext = "using Ziccrse";
268 }
269#if defined(CONFIG_RISCV_COMBO_SPINLOCKS)
270 else {
271 static_branch_disable(&qspinlock_key);
272 pr_info("Ticket spinlock: enabled\n");
273 return;
274 }
275#endif
276
277 if (!using_ext)
278 pr_err("Queued spinlock without Zabha or Ziccrse");
279 else
280 pr_info("Queued spinlock %s: enabled\n", using_ext);
281}
282
247extern void __init init_rt_signal_env(void);283extern void __init init_rt_signal_env(void);
248 284
249void __init setup_arch(char **cmdline_p)285void __init setup_arch(char **cmdline_p)
@@ -297,6 +333,7 @@ void __init setup_arch(char **cmdline_p) · 47 unmodified lines
297 riscv_set_dma_cache_alignment();333 riscv_set_dma_cache_alignment();
298 334
299 riscv_user_isa_enable();335 riscv_user_isa_enable();
336 riscv_spinlock_init();
300}337}
301 338
302bool arch_cpu_is_hotpluggable(int cpu)339bool arch_cpu_is_hotpluggable(int cpu)
arch/riscv/kernel/smpboot.c+0−2
@@ -233,8 +233,6 @@ asmlinkage __visible void smp_callin(void) · 232 unmodified lines
233 numa_add_cpu(curr_cpuid);233 numa_add_cpu(curr_cpuid);
234 set_cpu_online(curr_cpuid, true);234 set_cpu_online(curr_cpuid, true);
235 235
236 riscv_user_isa_enable();
237
238 /*236 /*
239 * Remote cache and TLB flushes are ignored while the CPU is offline,237 * Remote cache and TLB flushes are ignored while the CPU is offline,
240 * so flush them both right now just in case.238 * so flush them both right now just in case.
arch/riscv/kernel/suspend.c+2−2
@@ -14,7 +14,7 @@ · 13 unmodified lines
14 14
15void suspend_save_csrs(struct suspend_context *context)15void suspend_save_csrs(struct suspend_context *context)
16{16{
17 if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_XLINUXENVCFG))17 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_XLINUXENVCFG))
18 context->envcfg = csr_read(CSR_ENVCFG);18 context->envcfg = csr_read(CSR_ENVCFG);
19 context->tvec = csr_read(CSR_TVEC);19 context->tvec = csr_read(CSR_TVEC);
20 context->ie = csr_read(CSR_IE);20 context->ie = csr_read(CSR_IE);
@@ -37,7 +37,7 @@ void suspend_save_csrs(struct suspend_context *context) · 16 unmodified lines
37void suspend_restore_csrs(struct suspend_context *context)37void suspend_restore_csrs(struct suspend_context *context)
38{38{
39 csr_write(CSR_SCRATCH, 0);39 csr_write(CSR_SCRATCH, 0);
40 if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_XLINUXENVCFG))40 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_XLINUXENVCFG))
41 csr_write(CSR_ENVCFG, context->envcfg);41 csr_write(CSR_ENVCFG, context->envcfg);
42 csr_write(CSR_TVEC, context->tvec);42 csr_write(CSR_TVEC, context->tvec);
43 csr_write(CSR_IE, context->ie);43 csr_write(CSR_IE, context->ie);
arch/riscv/kernel/sys_hwprobe.c+44−0
@@ -150,6 +150,9 @@ static void hwprobe_isa_ext0(struct riscv_hwprobe *pair, · 149 unmodified lines
150 EXT_KEY(ZFH);150 EXT_KEY(ZFH);
151 EXT_KEY(ZFHMIN);151 EXT_KEY(ZFHMIN);
152 }152 }
153
154 if (IS_ENABLED(CONFIG_RISCV_ISA_SUPM))
155 EXT_KEY(SUPM);
153#undef EXT_KEY156#undef EXT_KEY
154 }157 }
155 158
@@ -201,6 +204,43 @@ static u64 hwprobe_misaligned(const struct cpumask *cpus) · 45 unmodified lines
201}204}
202#endif205#endif
203 206
207#ifdef CONFIG_RISCV_VECTOR_MISALIGNED
208static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
209{
210 int cpu;
211 u64 perf = -1ULL;
212
213 /* Return if supported or not even if speed wasn't probed */
214 for_each_cpu(cpu, cpus) {
215 int this_perf = per_cpu(vector_misaligned_access, cpu);
216
217 if (perf == -1ULL)
218 perf = this_perf;
219
220 if (perf != this_perf) {
221 perf = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
222 break;
223 }
224 }
225
226 if (perf == -1ULL)
227 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
228
229 return perf;
230}
231#else
232static u64 hwprobe_vec_misaligned(const struct cpumask *cpus)
233{
234 if (IS_ENABLED(CONFIG_RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS))
235 return RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
236
237 if (IS_ENABLED(CONFIG_RISCV_SLOW_VECTOR_UNALIGNED_ACCESS))
238 return RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
239
240 return RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
241}
242#endif
243
204static void hwprobe_one_pair(struct riscv_hwprobe *pair,244static void hwprobe_one_pair(struct riscv_hwprobe *pair,
205 const struct cpumask *cpus)245 const struct cpumask *cpus)
206{246{
@@ -229,6 +269,10 @@ static void hwprobe_one_pair(struct riscv_hwprobe *pair, · 22 unmodified lines
229 pair->value = hwprobe_misaligned(cpus);269 pair->value = hwprobe_misaligned(cpus);
230 break;270 break;
231 271
272 case RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF:
273 pair->value = hwprobe_vec_misaligned(cpus);
274 break;
275
232 case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE:276 case RISCV_HWPROBE_KEY_ZICBOZ_BLOCK_SIZE:
233 pair->value = 0;277 pair->value = 0;
234 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ))278 if (hwprobe_ext0_has(cpus, RISCV_HWPROBE_EXT_ZICBOZ))
arch/riscv/kernel/traps_misaligned.c+128−11
@@ -16,6 +16,7 @@ · 15 unmodified lines
16#include <asm/entry-common.h>16#include <asm/entry-common.h>
17#include <asm/hwprobe.h>17#include <asm/hwprobe.h>
18#include <asm/cpufeature.h>18#include <asm/cpufeature.h>
19#include <asm/vector.h>
19 20
20#define INSN_MATCH_LB 0x321#define INSN_MATCH_LB 0x3
21#define INSN_MASK_LB 0x707f22#define INSN_MASK_LB 0x707f
@@ -320,12 +321,37 @@ union reg_data { · 298 unmodified lines
320 u64 data_u64;321 u64 data_u64;
321};322};
322 323
323static bool unaligned_ctl __read_mostly;
324
325/* sysctl hooks */324/* sysctl hooks */
326int unaligned_enabled __read_mostly = 1; /* Enabled by default */325int unaligned_enabled __read_mostly = 1; /* Enabled by default */
327 326
328int handle_misaligned_load(struct pt_regs *regs)327#ifdef CONFIG_RISCV_VECTOR_MISALIGNED
328static int handle_vector_misaligned_load(struct pt_regs *regs)
329{
330 unsigned long epc = regs->epc;
331 unsigned long insn;
332
333 if (get_insn(regs, epc, &insn))
334 return -1;
335
336 /* Only return 0 when in check_vector_unaligned_access_emulated */
337 if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
338 *this_cpu_ptr(&vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
339 regs->epc = epc + INSN_LEN(insn);
340 return 0;
341 }
342
343 /* If vector instruction we don't emulate it yet */
344 regs->epc = epc;
345 return -1;
346}
347#else
348static int handle_vector_misaligned_load(struct pt_regs *regs)
349{
350 return -1;
351}
352#endif
353
354static int handle_scalar_misaligned_load(struct pt_regs *regs)
329{355{
330 union reg_data val;356 union reg_data val;
331 unsigned long epc = regs->epc;357 unsigned long epc = regs->epc;
@@ -433,7 +459,7 @@ int handle_misaligned_load(struct pt_regs *regs) · 101 unmodified lines
433 return 0;459 return 0;
434}460}
435 461
436int handle_misaligned_store(struct pt_regs *regs)462static int handle_scalar_misaligned_store(struct pt_regs *regs)
437{463{
438 union reg_data val;464 union reg_data val;
439 unsigned long epc = regs->epc;465 unsigned long epc = regs->epc;
@@ -524,11 +550,96 @@ int handle_misaligned_store(struct pt_regs *regs) · 84 unmodified lines
524 return 0;550 return 0;
525}551}
526 552
527static bool check_unaligned_access_emulated(int cpu)553int handle_misaligned_load(struct pt_regs *regs)
554{
555 unsigned long epc = regs->epc;
556 unsigned long insn;
557
558 if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) {
559 if (get_insn(regs, epc, &insn))
560 return -1;
561
562 if (insn_is_vector(insn))
563 return handle_vector_misaligned_load(regs);
564 }
565
566 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
567 return handle_scalar_misaligned_load(regs);
568
569 return -1;
570}
571
572int handle_misaligned_store(struct pt_regs *regs)
528{573{
574 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
575 return handle_scalar_misaligned_store(regs);
576
577 return -1;
578}
579
580#ifdef CONFIG_RISCV_VECTOR_MISALIGNED
581void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused)
582{
583 long *mas_ptr = this_cpu_ptr(&vector_misaligned_access);
584 unsigned long tmp_var;
585
586 *mas_ptr = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
587
588 kernel_vector_begin();
589 /*
590 * In pre-13.0.0 versions of GCC, vector registers cannot appear in
591 * the clobber list. This inline asm clobbers v0, but since we do not
592 * currently build the kernel with V enabled, the v0 clobber arg is not
593 * needed (as the compiler will not emit vector code itself). If the kernel
594 * is changed to build with V enabled, the clobber arg will need to be
595 * added here.
596 */
597 __asm__ __volatile__ (
598 ".balign 4\n\t"
599 ".option push\n\t"
600 ".option arch, +zve32x\n\t"
601 " vsetivli zero, 1, e16, m1, ta, ma\n\t" // Vectors of 16b
602 " vle16.v v0, (%[ptr])\n\t" // Load bytes
603 ".option pop\n\t"
604 : : [ptr] "r" ((u8 *)&tmp_var + 1));
605 kernel_vector_end();
606}
607
608bool check_vector_unaligned_access_emulated_all_cpus(void)
609{
610 int cpu;
611
612 if (!has_vector()) {
613 for_each_online_cpu(cpu)
614 per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
615 return false;
616 }
617
618 schedule_on_each_cpu(check_vector_unaligned_access_emulated);
619
620 for_each_online_cpu(cpu)
621 if (per_cpu(vector_misaligned_access, cpu)
622 == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
623 return false;
624
625 return true;
626}
627#else
628bool check_vector_unaligned_access_emulated_all_cpus(void)
629{
630 return false;
631}
632#endif
633
634#ifdef CONFIG_RISCV_SCALAR_MISALIGNED
635
636static bool unaligned_ctl __read_mostly;
637
638void check_unaligned_access_emulated(struct work_struct *work __always_unused)
639{
640 int cpu = smp_processor_id();
529 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);641 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);
530 unsigned long tmp_var, tmp_val;642 unsigned long tmp_var, tmp_val;
531 bool misaligned_emu_detected;
532 643
533 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;644 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
534 645
@@ -536,19 +647,16 @@ static bool check_unaligned_access_emulated(int cpu) · 1 unmodified line
536 " "REG_L" %[tmp], 1(%[ptr])\n"647 " "REG_L" %[tmp], 1(%[ptr])\n"
537 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory");648 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory");
538 649
539 misaligned_emu_detected = (*mas_ptr == RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED);
540 /*650 /*
541 * If unaligned_ctl is already set, this means that we detected that all651 * If unaligned_ctl is already set, this means that we detected that all
542 * CPUS uses emulated misaligned access at boot time. If that changed652 * CPUS uses emulated misaligned access at boot time. If that changed
543 * when hotplugging the new cpu, this is something we don't handle.653 * when hotplugging the new cpu, this is something we don't handle.
544 */654 */
545 if (unlikely(unaligned_ctl && !misaligned_emu_detected)) {655 if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) {
546 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n");656 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n");
547 while (true)657 while (true)
548 cpu_relax();658 cpu_relax();
549 }659 }
550
551 return misaligned_emu_detected;
552}660}
553 661
554bool check_unaligned_access_emulated_all_cpus(void)662bool check_unaligned_access_emulated_all_cpus(void)
@@ -560,8 +668,11 @@ bool check_unaligned_access_emulated_all_cpus(void) · 5 unmodified lines
560 * accesses emulated since tasks requesting such control can run on any668 * accesses emulated since tasks requesting such control can run on any
561 * CPU.669 * CPU.
562 */670 */
671 schedule_on_each_cpu(check_unaligned_access_emulated);
672
563 for_each_online_cpu(cpu)673 for_each_online_cpu(cpu)
564 if (!check_unaligned_access_emulated(cpu))674 if (per_cpu(misaligned_access_speed, cpu)
675 != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED)
565 return false;676 return false;
566 677
567 unaligned_ctl = true;678 unaligned_ctl = true;
@@ -572,3 +683,9 @@ bool unaligned_ctl_available(void) · 4 unmodified lines
572{683{
573 return unaligned_ctl;684 return unaligned_ctl;
574}685}
686#else
687bool check_unaligned_access_emulated_all_cpus(void)
688{
689 return false;
690}
691#endif
arch/riscv/kernel/unaligned_access_speed.c+148−8
@@ -6,11 +6,13 @@ · 5 unmodified lines
6#include <linux/cpu.h>6#include <linux/cpu.h>
7#include <linux/cpumask.h>7#include <linux/cpumask.h>
8#include <linux/jump_label.h>8#include <linux/jump_label.h>
9#include <linux/kthread.h>
9#include <linux/mm.h>10#include <linux/mm.h>
10#include <linux/smp.h>11#include <linux/smp.h>
11#include <linux/types.h>12#include <linux/types.h>
12#include <asm/cpufeature.h>13#include <asm/cpufeature.h>
13#include <asm/hwprobe.h>14#include <asm/hwprobe.h>
15#include <asm/vector.h>
14 16
15#include "copy-unaligned.h"17#include "copy-unaligned.h"
16 18
@@ -19,7 +21,8 @@ · 2 unmodified lines
19#define MISALIGNED_BUFFER_ORDER get_order(MISALIGNED_BUFFER_SIZE)21#define MISALIGNED_BUFFER_ORDER get_order(MISALIGNED_BUFFER_SIZE)
20#define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80)22#define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80)
21 23
22DEFINE_PER_CPU(long, misaligned_access_speed);24DEFINE_PER_CPU(long, misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
25DEFINE_PER_CPU(long, vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
23 26
24#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS27#ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
25static cpumask_t fast_misaligned_access;28static cpumask_t fast_misaligned_access;
@@ -191,6 +194,7 @@ static int riscv_online_cpu(unsigned int cpu) · 165 unmodified lines
191 if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN)194 if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN)
192 goto exit;195 goto exit;
193 196
197 check_unaligned_access_emulated(NULL);
194 buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);198 buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
195 if (!buf) {199 if (!buf) {
196 pr_warn("Allocation failure, not measuring misaligned performance\n");200 pr_warn("Allocation failure, not measuring misaligned performance\n");
@@ -259,23 +263,159 @@ out: · 62 unmodified lines
259 kfree(bufs);263 kfree(bufs);
260 return 0;264 return 0;
261}265}
266#else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */
267static int check_unaligned_access_speed_all_cpus(void)
268{
269 return 0;
270}
271#endif
262 272
263static int check_unaligned_access_all_cpus(void)273#ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
274static void check_vector_unaligned_access(struct work_struct *work __always_unused)
264{275{
265 bool all_cpus_emulated = check_unaligned_access_emulated_all_cpus();276 int cpu = smp_processor_id();
277 u64 start_cycles, end_cycles;
278 u64 word_cycles;
279 u64 byte_cycles;
280 int ratio;
281 unsigned long start_jiffies, now;
282 struct page *page;
283 void *dst;
284 void *src;
285 long speed = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
266 286
267 if (!all_cpus_emulated)287 if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
268 return check_unaligned_access_speed_all_cpus();288 return;
289
290 page = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
291 if (!page) {
292 pr_warn("Allocation failure, not measuring vector misaligned performance\n");
293 return;
294 }
295
296 /* Make an unaligned destination buffer. */
297 dst = (void *)((unsigned long)page_address(page) | 0x1);
298 /* Unalign src as well, but differently (off by 1 + 2 = 3). */
299 src = dst + (MISALIGNED_BUFFER_SIZE / 2);
300 src += 2;
301 word_cycles = -1ULL;
302
303 /* Do a warmup. */
304 kernel_vector_begin();
305 __riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE);
269 306
307 start_jiffies = jiffies;
308 while ((now = jiffies) == start_jiffies)
309 cpu_relax();
310
311 /*
312 * For a fixed amount of time, repeatedly try the function, and take
313 * the best time in cycles as the measurement.
314 */
315 while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) {
316 start_cycles = get_cycles64();
317 /* Ensure the CSR read can't reorder WRT to the copy. */
318 mb();
319 __riscv_copy_vec_words_unaligned(dst, src, MISALIGNED_COPY_SIZE);
320 /* Ensure the copy ends before the end time is snapped. */
321 mb();
322 end_cycles = get_cycles64();
323 if ((end_cycles - start_cycles) < word_cycles)
324 word_cycles = end_cycles - start_cycles;
325 }
326
327 byte_cycles = -1ULL;
328 __riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE);
329 start_jiffies = jiffies;
330 while ((now = jiffies) == start_jiffies)
331 cpu_relax();
332
333 while (time_before(jiffies, now + (1 << MISALIGNED_ACCESS_JIFFIES_LG2))) {
334 start_cycles = get_cycles64();
335 /* Ensure the CSR read can't reorder WRT to the copy. */
336 mb();
337 __riscv_copy_vec_bytes_unaligned(dst, src, MISALIGNED_COPY_SIZE);
338 /* Ensure the copy ends before the end time is snapped. */
339 mb();
340 end_cycles = get_cycles64();
341 if ((end_cycles - start_cycles) < byte_cycles)
342 byte_cycles = end_cycles - start_cycles;
343 }
344
345 kernel_vector_end();
346
347 /* Don't divide by zero. */
348 if (!word_cycles || !byte_cycles) {
349 pr_warn("cpu%d: rdtime lacks granularity needed to measure unaligned vector access speed\n",
350 cpu);
351
352 return;
353 }
354
355 if (word_cycles < byte_cycles)
356 speed = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
357
358 ratio = div_u64((byte_cycles * 100), word_cycles);
359 pr_info("cpu%d: Ratio of vector byte access time to vector unaligned word access is %d.%02d, unaligned accesses are %s\n",
360 cpu,
361 ratio / 100,
362 ratio % 100,
363 (speed == RISCV_HWPROBE_MISALIGNED_VECTOR_FAST) ? "fast" : "slow");
364
365 per_cpu(vector_misaligned_access, cpu) = speed;
366}
367
368static int riscv_online_cpu_vec(unsigned int cpu)
369{
370 if (!has_vector())
371 return 0;
372
373 if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED)
374 return 0;
375
376 check_vector_unaligned_access_emulated(NULL);
377 check_vector_unaligned_access(NULL);
270 return 0;378 return 0;
271}379}
272#else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */380
273static int check_unaligned_access_all_cpus(void)381/* Measure unaligned access speed on all CPUs present at boot in parallel. */
382static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused)
274{383{
275 check_unaligned_access_emulated_all_cpus();384 schedule_on_each_cpu(check_vector_unaligned_access);
385
386 /*
387 * Setup hotplug callbacks for any new CPUs that come online or go
388 * offline.
389 */
390 cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online",
391 riscv_online_cpu_vec, NULL);
276 392
277 return 0;393 return 0;
278}394}
395#else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */
396static int vec_check_unaligned_access_speed_all_cpus(void *unused __always_unused)
397{
398 return 0;
399}
279#endif400#endif
280 401
402static int check_unaligned_access_all_cpus(void)
403{
404 bool all_cpus_emulated, all_cpus_vec_unsupported;
405
406 all_cpus_emulated = check_unaligned_access_emulated_all_cpus();
407 all_cpus_vec_unsupported = check_vector_unaligned_access_emulated_all_cpus();
408
409 if (!all_cpus_vec_unsupported &&
410 IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) {
411 kthread_run(vec_check_unaligned_access_speed_all_cpus,
412 NULL, "vec_check_unaligned_access_speed_all_cpus");
413 }
414
415 if (!all_cpus_emulated)
416 return check_unaligned_access_speed_all_cpus();
417
418 return 0;
419}
420
281arch_initcall(check_unaligned_access_all_cpus);421arch_initcall(check_unaligned_access_all_cpus);
arch/riscv/kernel/vdso/Makefile+5−4
@@ -45,7 +45,7 @@ $(obj)/vdso.o: $(obj)/vdso.so · 44 unmodified lines
45 45
46# link rule for the .so file, .lds has to be first46# link rule for the .so file, .lds has to be first
47$(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE47$(obj)/vdso.so.dbg: $(obj)/vdso.lds $(obj-vdso) FORCE
48 $(call if_changed,vdsold)48 $(call if_changed,vdsold_and_check)
49LDFLAGS_vdso.so.dbg = -shared -soname=linux-vdso.so.1 \49LDFLAGS_vdso.so.dbg = -shared -soname=linux-vdso.so.1 \
50 --build-id=sha1 --hash-style=both --eh-frame-hdr50 --build-id=sha1 --hash-style=both --eh-frame-hdr
51 51
@@ -65,7 +65,8 @@ include/generated/vdso-offsets.h: $(obj)/vdso.so.dbg FORCE · 13 unmodified lines
65# actual build commands65# actual build commands
66# The DSO images are built using a special linker script66# The DSO images are built using a special linker script
67# Make sure only to export the intended __vdso_xxx symbol offsets.67# Make sure only to export the intended __vdso_xxx symbol offsets.
68quiet_cmd_vdsold = VDSOLD $@68quiet_cmd_vdsold_and_check = VDSOLD $@
69 cmd_vdsold = $(LD) $(ld_flags) -T $(filter-out FORCE,$^) -o $@.tmp && \69 cmd_vdsold_and_check = $(LD) $(ld_flags) -T $(filter-out FORCE,$^) -o $@.tmp && \
70 $(OBJCOPY) $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \70 $(OBJCOPY) $(patsubst %, -G __vdso_%, $(vdso-syms)) $@.tmp $@ && \
71 rm $@.tmp71 rm $@.tmp && \
72 $(cmd_vdso_check)
arch/riscv/kernel/vec-copy-unaligned.S+58−0
@@ -0,0 +1,58 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Copyright (C) 2024 Rivos Inc. */
3
4#include <linux/args.h>
5#include <linux/linkage.h>
6#include <asm/asm.h>
7
8 .text
9
10#define WORD_EEW 32
11
12#define WORD_SEW CONCATENATE(e, WORD_EEW)
13#define VEC_L CONCATENATE(vle, WORD_EEW).v
14#define VEC_S CONCATENATE(vle, WORD_EEW).v
15
16/* void __riscv_copy_vec_words_unaligned(void *, const void *, size_t) */
17/* Performs a memcpy without aligning buffers, using word loads and stores. */
18/* Note: The size is truncated to a multiple of WORD_EEW */
19SYM_FUNC_START(__riscv_copy_vec_words_unaligned)
20 andi a4, a2, ~(WORD_EEW-1)
21 beqz a4, 2f
22 add a3, a1, a4
23 .option push
24 .option arch, +zve32x
251:
26 vsetivli t0, 8, WORD_SEW, m8, ta, ma
27 VEC_L v0, (a1)
28 VEC_S v0, (a0)
29 addi a0, a0, WORD_EEW
30 addi a1, a1, WORD_EEW
31 bltu a1, a3, 1b
32
332:
34 .option pop
35 ret
36SYM_FUNC_END(__riscv_copy_vec_words_unaligned)
37
38/* void __riscv_copy_vec_bytes_unaligned(void *, const void *, size_t) */
39/* Performs a memcpy without aligning buffers, using only byte accesses. */
40/* Note: The size is truncated to a multiple of 8 */
41SYM_FUNC_START(__riscv_copy_vec_bytes_unaligned)
42 andi a4, a2, ~(8-1)
43 beqz a4, 2f
44 add a3, a1, a4
45 .option push
46 .option arch, +zve32x
471:
48 vsetivli t0, 8, e8, m8, ta, ma
49 vle8.v v0, (a1)
50 vse8.v v0, (a0)
51 addi a0, a0, 8
52 addi a1, a1, 8
53 bltu a1, a3, 1b
54
552:
56 .option pop
57 ret
58SYM_FUNC_END(__riscv_copy_vec_bytes_unaligned)
arch/riscv/kernel/vector.c+1−1
@@ -66,7 +66,7 @@ void __init riscv_v_setup_ctx_cache(void) · 65 unmodified lines
66#endif66#endif
67}67}
68 68
69static bool insn_is_vector(u32 insn_buf)69bool insn_is_vector(u32 insn_buf)
70{70{
71 u32 opcode = insn_buf & __INSN_OPCODE_MASK;71 u32 opcode = insn_buf & __INSN_OPCODE_MASK;
72 u32 width, csr;72 u32 width, csr;
arch/riscv/kvm/vcpu_onereg.c+4−0
@@ -34,9 +34,11 @@ static const unsigned long kvm_isa_ext_arr[] = { · 33 unmodified lines
34 [KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m,34 [KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m,
35 [KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v,35 [KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v,
36 /* Multi letter extensions (alphabetically sorted) */36 /* Multi letter extensions (alphabetically sorted) */
37 [KVM_RISCV_ISA_EXT_SMNPM] = RISCV_ISA_EXT_SSNPM,
37 KVM_ISA_EXT_ARR(SMSTATEEN),38 KVM_ISA_EXT_ARR(SMSTATEEN),
38 KVM_ISA_EXT_ARR(SSAIA),39 KVM_ISA_EXT_ARR(SSAIA),
39 KVM_ISA_EXT_ARR(SSCOFPMF),40 KVM_ISA_EXT_ARR(SSCOFPMF),
41 KVM_ISA_EXT_ARR(SSNPM),
40 KVM_ISA_EXT_ARR(SSTC),42 KVM_ISA_EXT_ARR(SSTC),
41 KVM_ISA_EXT_ARR(SVINVAL),43 KVM_ISA_EXT_ARR(SVINVAL),
42 KVM_ISA_EXT_ARR(SVNAPOT),44 KVM_ISA_EXT_ARR(SVNAPOT),
@@ -127,8 +129,10 @@ static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) · 84 unmodified lines
127 case KVM_RISCV_ISA_EXT_C:129 case KVM_RISCV_ISA_EXT_C:
128 case KVM_RISCV_ISA_EXT_I:130 case KVM_RISCV_ISA_EXT_I:
129 case KVM_RISCV_ISA_EXT_M:131 case KVM_RISCV_ISA_EXT_M:
132 case KVM_RISCV_ISA_EXT_SMNPM:
130 /* There is not architectural config bit to disable sscofpmf completely */133 /* There is not architectural config bit to disable sscofpmf completely */
131 case KVM_RISCV_ISA_EXT_SSCOFPMF:134 case KVM_RISCV_ISA_EXT_SSCOFPMF:
135 case KVM_RISCV_ISA_EXT_SSNPM:
132 case KVM_RISCV_ISA_EXT_SSTC:136 case KVM_RISCV_ISA_EXT_SSTC:
133 case KVM_RISCV_ISA_EXT_SVINVAL:137 case KVM_RISCV_ISA_EXT_SVINVAL:
134 case KVM_RISCV_ISA_EXT_SVNAPOT:138 case KVM_RISCV_ISA_EXT_SVNAPOT:
drivers/cpuidle/cpuidle-riscv-sbi.c+9−2
@@ -26,6 +26,7 @@ · 25 unmodified lines
26#include <asm/smp.h>26#include <asm/smp.h>
27#include <asm/suspend.h>27#include <asm/suspend.h>
28 28
29#include "cpuidle.h"
29#include "dt_idle_states.h"30#include "dt_idle_states.h"
30#include "dt_idle_genpd.h"31#include "dt_idle_genpd.h"
31 32
@@ -329,6 +330,9 @@ static int sbi_cpuidle_init_cpu(struct device *dev, int cpu) · 297 unmodified lines
329 return ret;330 return ret;
330 }331 }
331 332
333 if (cpuidle_disabled())
334 return 0;
335
332 ret = cpuidle_register(drv, NULL);336 ret = cpuidle_register(drv, NULL);
333 if (ret)337 if (ret)
334 goto deinit;338 goto deinit;
@@ -538,7 +542,10 @@ static int sbi_cpuidle_probe(struct platform_device *pdev) · 203 unmodified lines
538 /* Setup CPU hotplut notifiers */542 /* Setup CPU hotplut notifiers */
539 sbi_idle_init_cpuhp();543 sbi_idle_init_cpuhp();
540 544
541 pr_info("idle driver registered for all CPUs\n");545 if (cpuidle_disabled())
546 pr_info("cpuidle is disabled\n");
547 else
548 pr_info("idle driver registered for all CPUs\n");
542 549
543 return 0;550 return 0;
544 551
@@ -582,4 +589,4 @@ static int __init sbi_cpuidle_init(void) · 37 unmodified lines
582 589
583 return 0;590 return 0;
584}591}
585device_initcall(sbi_cpuidle_init);592arch_initcall(sbi_cpuidle_init);
include/asm-generic/qspinlock.h+2−0
@@ -136,6 +136,7 @@ static __always_inline bool virt_spin_lock(struct qspinlock *lock) · 135 unmodified lines
136}136}
137#endif137#endif
138 138
139#ifndef __no_arch_spinlock_redefine
139/*140/*
140 * Remapping spinlock architecture specific functions to the corresponding141 * Remapping spinlock architecture specific functions to the corresponding
141 * queued spinlock functions.142 * queued spinlock functions.
@@ -146,5 +147,6 @@ static __always_inline bool virt_spin_lock(struct qspinlock *lock) · 4 unmodified lines
146#define arch_spin_lock(l) queued_spin_lock(l)147#define arch_spin_lock(l) queued_spin_lock(l)
147#define arch_spin_trylock(l) queued_spin_trylock(l)148#define arch_spin_trylock(l) queued_spin_trylock(l)
148#define arch_spin_unlock(l) queued_spin_unlock(l)149#define arch_spin_unlock(l) queued_spin_unlock(l)
150#endif
149 151
150#endif /* __ASM_GENERIC_QSPINLOCK_H */152#endif /* __ASM_GENERIC_QSPINLOCK_H */
include/asm-generic/spinlock.h+1−86
@@ -1,94 +1,9 @@
1/* SPDX-License-Identifier: GPL-2.0 */1/* SPDX-License-Identifier: GPL-2.0 */
2 2
3/*
4 * 'Generic' ticket-lock implementation.
5 *
6 * It relies on atomic_fetch_add() having well defined forward progress
7 * guarantees under contention. If your architecture cannot provide this, stick
8 * to a test-and-set lock.
9 *
10 * It also relies on atomic_fetch_add() being safe vs smp_store_release() on a
11 * sub-word of the value. This is generally true for anything LL/SC although
12 * you'd be hard pressed to find anything useful in architecture specifications
13 * about this. If your architecture cannot do this you might be better off with
14 * a test-and-set.
15 *
16 * It further assumes atomic_*_release() + atomic_*_acquire() is RCpc and hence
17 * uses atomic_fetch_add() which is RCsc to create an RCsc hot path, along with
18 * a full fence after the spin to upgrade the otherwise-RCpc
19 * atomic_cond_read_acquire().
20 *
21 * The implementation uses smp_cond_load_acquire() to spin, so if the
22 * architecture has WFE like instructions to sleep instead of poll for word
23 * modifications be sure to implement that (see ARM64 for example).
24 *
25 */
26
27#ifndef __ASM_GENERIC_SPINLOCK_H3#ifndef __ASM_GENERIC_SPINLOCK_H
28#define __ASM_GENERIC_SPINLOCK_H4#define __ASM_GENERIC_SPINLOCK_H
29 5
30#include <linux/atomic.h>6#include <asm-generic/ticket_spinlock.h>
31#include <asm-generic/spinlock_types.h>
32
33static __always_inline void arch_spin_lock(arch_spinlock_t *lock)
34{
35 u32 val = atomic_fetch_add(1<<16, lock);
36 u16 ticket = val >> 16;
37
38 if (ticket == (u16)val)
39 return;
40
41 /*
42 * atomic_cond_read_acquire() is RCpc, but rather than defining a
43 * custom cond_read_rcsc() here we just emit a full fence. We only
44 * need the prior reads before subsequent writes ordering from
45 * smb_mb(), but as atomic_cond_read_acquire() just emits reads and we
46 * have no outstanding writes due to the atomic_fetch_add() the extra
47 * orderings are free.
48 */
49 atomic_cond_read_acquire(lock, ticket == (u16)VAL);
50 smp_mb();
51}
52
53static __always_inline bool arch_spin_trylock(arch_spinlock_t *lock)
54{
55 u32 old = atomic_read(lock);
56
57 if ((old >> 16) != (old & 0xffff))
58 return false;
59
60 return atomic_try_cmpxchg(lock, &old, old + (1<<16)); /* SC, for RCsc */
61}
62
63static __always_inline void arch_spin_unlock(arch_spinlock_t *lock)
64{
65 u16 *ptr = (u16 *)lock + IS_ENABLED(CONFIG_CPU_BIG_ENDIAN);
66 u32 val = atomic_read(lock);
67
68 smp_store_release(ptr, (u16)val + 1);
69}
70
71static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
72{
73 u32 val = lock.counter;
74
75 return ((val >> 16) == (val & 0xffff));
76}
77
78static __always_inline int arch_spin_is_locked(arch_spinlock_t *lock)
79{
80 arch_spinlock_t val = READ_ONCE(*lock);
81
82 return !arch_spin_value_unlocked(val);
83}
84
85static __always_inline int arch_spin_is_contended(arch_spinlock_t *lock)
86{
87 u32 val = atomic_read(lock);
88
89 return (s16)((val >> 16) - (val & 0xffff)) > 1;
90}
91
92#include <asm/qrwlock.h>7#include <asm/qrwlock.h>
93 8
94#endif /* __ASM_GENERIC_SPINLOCK_H */9#endif /* __ASM_GENERIC_SPINLOCK_H */
include/asm-generic/spinlock_types.h+2−10
@@ -3,15 +3,7 @@ · 2 unmodified lines
3#ifndef __ASM_GENERIC_SPINLOCK_TYPES_H3#ifndef __ASM_GENERIC_SPINLOCK_TYPES_H
4#define __ASM_GENERIC_SPINLOCK_TYPES_H4#define __ASM_GENERIC_SPINLOCK_TYPES_H
5 5
6#include <linux/types.h>6#include <asm-generic/qspinlock_types.h>
7typedef atomic_t arch_spinlock_t;7#include <asm-generic/qrwlock_types.h>
8
9/*
10 * qrwlock_types depends on arch_spinlock_t, so we must typedef that before the
11 * include.
12 */
13#include <asm/qrwlock_types.h>
14
15#define __ARCH_SPIN_LOCK_UNLOCKED ATOMIC_INIT(0)
16 8
17#endif /* __ASM_GENERIC_SPINLOCK_TYPES_H */9#endif /* __ASM_GENERIC_SPINLOCK_TYPES_H */
include/asm-generic/ticket_spinlock.h+105−0
@@ -0,0 +1,105 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2
3/*
4 * 'Generic' ticket-lock implementation.
5 *
6 * It relies on atomic_fetch_add() having well defined forward progress
7 * guarantees under contention. If your architecture cannot provide this, stick
8 * to a test-and-set lock.
9 *
10 * It also relies on atomic_fetch_add() being safe vs smp_store_release() on a
11 * sub-word of the value. This is generally true for anything LL/SC although
12 * you'd be hard pressed to find anything useful in architecture specifications
13 * about this. If your architecture cannot do this you might be better off with
14 * a test-and-set.
15 *
16 * It further assumes atomic_*_release() + atomic_*_acquire() is RCpc and hence
17 * uses atomic_fetch_add() which is RCsc to create an RCsc hot path, along with
18 * a full fence after the spin to upgrade the otherwise-RCpc
19 * atomic_cond_read_acquire().
20 *
21 * The implementation uses smp_cond_load_acquire() to spin, so if the
22 * architecture has WFE like instructions to sleep instead of poll for word
23 * modifications be sure to implement that (see ARM64 for example).
24 *
25 */
26
27#ifndef __ASM_GENERIC_TICKET_SPINLOCK_H
28#define __ASM_GENERIC_TICKET_SPINLOCK_H
29
30#include <linux/atomic.h>
31#include <asm-generic/spinlock_types.h>
32
33static __always_inline void ticket_spin_lock(arch_spinlock_t *lock)
34{
35 u32 val = atomic_fetch_add(1<<16, &lock->val);
36 u16 ticket = val >> 16;
37
38 if (ticket == (u16)val)
39 return;
40
41 /*
42 * atomic_cond_read_acquire() is RCpc, but rather than defining a
43 * custom cond_read_rcsc() here we just emit a full fence. We only
44 * need the prior reads before subsequent writes ordering from
45 * smb_mb(), but as atomic_cond_read_acquire() just emits reads and we
46 * have no outstanding writes due to the atomic_fetch_add() the extra
47 * orderings are free.
48 */
49 atomic_cond_read_acquire(&lock->val, ticket == (u16)VAL);
50 smp_mb();
51}
52
53static __always_inline bool ticket_spin_trylock(arch_spinlock_t *lock)
54{
55 u32 old = atomic_read(&lock->val);
56
57 if ((old >> 16) != (old & 0xffff))
58 return false;
59
60 return atomic_try_cmpxchg(&lock->val, &old, old + (1<<16)); /* SC, for RCsc */
61}
62
63static __always_inline void ticket_spin_unlock(arch_spinlock_t *lock)
64{
65 u16 *ptr = (u16 *)lock + IS_ENABLED(CONFIG_CPU_BIG_ENDIAN);
66 u32 val = atomic_read(&lock->val);
67
68 smp_store_release(ptr, (u16)val + 1);
69}
70
71static __always_inline int ticket_spin_value_unlocked(arch_spinlock_t lock)
72{
73 u32 val = lock.val.counter;
74
75 return ((val >> 16) == (val & 0xffff));
76}
77
78static __always_inline int ticket_spin_is_locked(arch_spinlock_t *lock)
79{
80 arch_spinlock_t val = READ_ONCE(*lock);
81
82 return !ticket_spin_value_unlocked(val);
83}
84
85static __always_inline int ticket_spin_is_contended(arch_spinlock_t *lock)
86{
87 u32 val = atomic_read(&lock->val);
88
89 return (s16)((val >> 16) - (val & 0xffff)) > 1;
90}
91
92#ifndef __no_arch_spinlock_redefine
93/*
94 * Remapping spinlock architecture specific functions to the corresponding
95 * ticket spinlock functions.
96 */
97#define arch_spin_is_locked(l) ticket_spin_is_locked(l)
98#define arch_spin_is_contended(l) ticket_spin_is_contended(l)
99#define arch_spin_value_unlocked(l) ticket_spin_value_unlocked(l)
100#define arch_spin_lock(l) ticket_spin_lock(l)
101#define arch_spin_trylock(l) ticket_spin_trylock(l)
102#define arch_spin_unlock(l) ticket_spin_unlock(l)
103#endif
104
105#endif /* __ASM_GENERIC_TICKET_SPINLOCK_H */
include/uapi/linux/elf.h+1−0
@@ -451,6 +451,7 @@ typedef struct elf64_shdr { · 450 unmodified lines
451#define NT_MIPS_MSA 0x802 /* MIPS SIMD registers */451#define NT_MIPS_MSA 0x802 /* MIPS SIMD registers */
452#define NT_RISCV_CSR 0x900 /* RISC-V Control and Status Registers */452#define NT_RISCV_CSR 0x900 /* RISC-V Control and Status Registers */
453#define NT_RISCV_VECTOR 0x901 /* RISC-V vector registers */453#define NT_RISCV_VECTOR 0x901 /* RISC-V vector registers */
454#define NT_RISCV_TAGGED_ADDR_CTRL 0x902 /* RISC-V tagged address control (prctl()) */
454#define NT_LOONGARCH_CPUCFG 0xa00 /* LoongArch CPU config registers */455#define NT_LOONGARCH_CPUCFG 0xa00 /* LoongArch CPU config registers */
455#define NT_LOONGARCH_CSR 0xa01 /* LoongArch control and status registers */456#define NT_LOONGARCH_CSR 0xa01 /* LoongArch control and status registers */
456#define NT_LOONGARCH_LSX 0xa02 /* LoongArch Loongson SIMD Extension registers */457#define NT_LOONGARCH_LSX 0xa02 /* LoongArch Loongson SIMD Extension registers */
include/uapi/linux/prctl.h+4−1
@@ -230,7 +230,7 @@ struct prctl_mm_map { · 229 unmodified lines
230# define PR_PAC_APDBKEY (1UL << 3)230# define PR_PAC_APDBKEY (1UL << 3)
231# define PR_PAC_APGAKEY (1UL << 4)231# define PR_PAC_APGAKEY (1UL << 4)
232 232
233/* Tagged user address controls for arm64 */233/* Tagged user address controls for arm64 and RISC-V */
234#define PR_SET_TAGGED_ADDR_CTRL 55234#define PR_SET_TAGGED_ADDR_CTRL 55
235#define PR_GET_TAGGED_ADDR_CTRL 56235#define PR_GET_TAGGED_ADDR_CTRL 56
236# define PR_TAGGED_ADDR_ENABLE (1UL << 0)236# define PR_TAGGED_ADDR_ENABLE (1UL << 0)
@@ -244,6 +244,9 @@ struct prctl_mm_map { · 7 unmodified lines
244# define PR_MTE_TAG_MASK (0xffffUL << PR_MTE_TAG_SHIFT)244# define PR_MTE_TAG_MASK (0xffffUL << PR_MTE_TAG_SHIFT)
245/* Unused; kept only for source compatibility */245/* Unused; kept only for source compatibility */
246# define PR_MTE_TCF_SHIFT 1246# define PR_MTE_TCF_SHIFT 1
247/* RISC-V pointer masking tag length */
248# define PR_PMLEN_SHIFT 24
249# define PR_PMLEN_MASK (0x7fUL << PR_PMLEN_SHIFT)
247 250
248/* Control reclaim behavior when allocating memory */251/* Control reclaim behavior when allocating memory */
249#define PR_SET_IO_FLUSHER 57252#define PR_SET_IO_FLUSHER 57
tools/testing/selftests/kvm/riscv/get-reg-list.c+8−0
@@ -41,9 +41,11 @@ bool filter_reg(__u64 reg) · 40 unmodified lines
41 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_I:41 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_I:
42 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_M:42 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_M:
43 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_V:43 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_V:
44 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SMNPM:
44 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SMSTATEEN:45 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SMSTATEEN:
45 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SSAIA:46 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SSAIA:
46 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SSCOFPMF:47 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SSCOFPMF:
48 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SSNPM:
47 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SSTC:49 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SSTC:
48 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVINVAL:50 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVINVAL:
49 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVNAPOT:51 case KVM_REG_RISCV_ISA_EXT | KVM_REG_RISCV_ISA_SINGLE | KVM_RISCV_ISA_EXT_SVNAPOT:
@@ -414,9 +416,11 @@ static const char *isa_ext_single_id_to_str(__u64 reg_off) · 364 unmodified lines
414 KVM_ISA_EXT_ARR(I),416 KVM_ISA_EXT_ARR(I),
415 KVM_ISA_EXT_ARR(M),417 KVM_ISA_EXT_ARR(M),
416 KVM_ISA_EXT_ARR(V),418 KVM_ISA_EXT_ARR(V),
419 KVM_ISA_EXT_ARR(SMNPM),
417 KVM_ISA_EXT_ARR(SMSTATEEN),420 KVM_ISA_EXT_ARR(SMSTATEEN),
418 KVM_ISA_EXT_ARR(SSAIA),421 KVM_ISA_EXT_ARR(SSAIA),
419 KVM_ISA_EXT_ARR(SSCOFPMF),422 KVM_ISA_EXT_ARR(SSCOFPMF),
423 KVM_ISA_EXT_ARR(SSNPM),
420 KVM_ISA_EXT_ARR(SSTC),424 KVM_ISA_EXT_ARR(SSTC),
421 KVM_ISA_EXT_ARR(SVINVAL),425 KVM_ISA_EXT_ARR(SVINVAL),
422 KVM_ISA_EXT_ARR(SVNAPOT),426 KVM_ISA_EXT_ARR(SVNAPOT),
@@ -946,8 +950,10 @@ KVM_ISA_EXT_SUBLIST_CONFIG(aia, AIA); · 523 unmodified lines
946KVM_ISA_EXT_SUBLIST_CONFIG(fp_f, FP_F);950KVM_ISA_EXT_SUBLIST_CONFIG(fp_f, FP_F);
947KVM_ISA_EXT_SUBLIST_CONFIG(fp_d, FP_D);951KVM_ISA_EXT_SUBLIST_CONFIG(fp_d, FP_D);
948KVM_ISA_EXT_SIMPLE_CONFIG(h, H);952KVM_ISA_EXT_SIMPLE_CONFIG(h, H);
953KVM_ISA_EXT_SIMPLE_CONFIG(smnpm, SMNPM);
949KVM_ISA_EXT_SUBLIST_CONFIG(smstateen, SMSTATEEN);954KVM_ISA_EXT_SUBLIST_CONFIG(smstateen, SMSTATEEN);
950KVM_ISA_EXT_SIMPLE_CONFIG(sscofpmf, SSCOFPMF);955KVM_ISA_EXT_SIMPLE_CONFIG(sscofpmf, SSCOFPMF);
956KVM_ISA_EXT_SIMPLE_CONFIG(ssnpm, SSNPM);
951KVM_ISA_EXT_SIMPLE_CONFIG(sstc, SSTC);957KVM_ISA_EXT_SIMPLE_CONFIG(sstc, SSTC);
952KVM_ISA_EXT_SIMPLE_CONFIG(svinval, SVINVAL);958KVM_ISA_EXT_SIMPLE_CONFIG(svinval, SVINVAL);
953KVM_ISA_EXT_SIMPLE_CONFIG(svnapot, SVNAPOT);959KVM_ISA_EXT_SIMPLE_CONFIG(svnapot, SVNAPOT);
@@ -1009,8 +1015,10 @@ struct vcpu_reg_list *vcpu_configs[] = { · 55 unmodified lines
1009 &config_fp_f,1015 &config_fp_f,
1010 &config_fp_d,1016 &config_fp_d,
1011 &config_h,1017 &config_h,
1018 &config_smnpm,
1012 &config_smstateen,1019 &config_smstateen,
1013 &config_sscofpmf,1020 &config_sscofpmf,
1021 &config_ssnpm,
1014 &config_sstc,1022 &config_sstc,
1015 &config_svinval,1023 &config_svinval,
1016 &config_svnapot,1024 &config_svnapot,
tools/testing/selftests/riscv/Makefile+1−1
@@ -5,7 +5,7 @@ · 4 unmodified lines
5ARCH ?= $(shell uname -m 2>/dev/null || echo not)5ARCH ?= $(shell uname -m 2>/dev/null || echo not)
6 6
7ifneq (,$(filter $(ARCH),riscv))7ifneq (,$(filter $(ARCH),riscv))
8RISCV_SUBTARGETS ?= hwprobe vector mm sigreturn8RISCV_SUBTARGETS ?= abi hwprobe mm sigreturn vector
9else9else
10RISCV_SUBTARGETS :=10RISCV_SUBTARGETS :=
11endif11endif
tools/testing/selftests/riscv/abi/.gitignore+1−0
@@ -0,0 +1 @@
1pointer_masking
tools/testing/selftests/riscv/abi/Makefile+10−0
@@ -0,0 +1,10 @@
1# SPDX-License-Identifier: GPL-2.0
2
3CFLAGS += -I$(top_srcdir)/tools/include
4
5TEST_GEN_PROGS := pointer_masking
6
7include ../../lib.mk
8
9$(OUTPUT)/pointer_masking: pointer_masking.c
10 $(CC) -static -o$@ $(CFLAGS) $(LDFLAGS) $^
tools/testing/selftests/riscv/abi/pointer_masking.c+332−0
@@ -0,0 +1,332 @@
1// SPDX-License-Identifier: GPL-2.0-only
2
3#include <errno.h>
4#include <fcntl.h>
5#include <setjmp.h>
6#include <signal.h>
7#include <stdbool.h>
8#include <sys/prctl.h>
9#include <sys/wait.h>
10#include <unistd.h>
11
12#include "../../kselftest.h"
13
14#ifndef PR_PMLEN_SHIFT
15#define PR_PMLEN_SHIFT 24
16#endif
17#ifndef PR_PMLEN_MASK
18#define PR_PMLEN_MASK (0x7fUL << PR_PMLEN_SHIFT)
19#endif
20
21static int dev_zero;
22
23static int pipefd[2];
24
25static sigjmp_buf jmpbuf;
26
27static void sigsegv_handler(int sig)
28{
29 siglongjmp(jmpbuf, 1);
30}
31
32static int min_pmlen;
33static int max_pmlen;
34
35static inline bool valid_pmlen(int pmlen)
36{
37 return pmlen == 0 || pmlen == 7 || pmlen == 16;
38}
39
40static void test_pmlen(void)
41{
42 ksft_print_msg("Testing available PMLEN values\n");
43
44 for (int request = 0; request <= 16; request++) {
45 int pmlen, ret;
46
47 ret = prctl(PR_SET_TAGGED_ADDR_CTRL, request << PR_PMLEN_SHIFT, 0, 0, 0);
48 if (ret)
49 goto pr_set_error;
50
51 ret = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
52 ksft_test_result(ret >= 0, "PMLEN=%d PR_GET_TAGGED_ADDR_CTRL\n", request);
53 if (ret < 0)
54 goto pr_get_error;
55
56 pmlen = (ret & PR_PMLEN_MASK) >> PR_PMLEN_SHIFT;
57 ksft_test_result(pmlen >= request, "PMLEN=%d constraint\n", request);
58 ksft_test_result(valid_pmlen(pmlen), "PMLEN=%d validity\n", request);
59
60 if (min_pmlen == 0)
61 min_pmlen = pmlen;
62 if (max_pmlen < pmlen)
63 max_pmlen = pmlen;
64
65 continue;
66
67pr_set_error:
68 ksft_test_result_skip("PMLEN=%d PR_GET_TAGGED_ADDR_CTRL\n", request);
69pr_get_error:
70 ksft_test_result_skip("PMLEN=%d constraint\n", request);
71 ksft_test_result_skip("PMLEN=%d validity\n", request);
72 }
73
74 if (max_pmlen == 0)
75 ksft_exit_fail_msg("Failed to enable pointer masking\n");
76}
77
78static int set_tagged_addr_ctrl(int pmlen, bool tagged_addr_abi)
79{
80 int arg, ret;
81
82 arg = pmlen << PR_PMLEN_SHIFT | tagged_addr_abi;
83 ret = prctl(PR_SET_TAGGED_ADDR_CTRL, arg, 0, 0, 0);
84 if (!ret) {
85 ret = prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0);
86 if (ret == arg)
87 return 0;
88 }
89
90 return ret < 0 ? -errno : -ENODATA;
91}
92
93static void test_dereference_pmlen(int pmlen)
94{
95 static volatile int i;
96 volatile int *p;
97 int ret;
98
99 ret = set_tagged_addr_ctrl(pmlen, false);
100 if (ret)
101 return ksft_test_result_error("PMLEN=%d setup (%d)\n", pmlen, ret);
102
103 i = pmlen;
104
105 if (pmlen) {
106 p = (volatile int *)((uintptr_t)&i | 1UL << (__riscv_xlen - pmlen));
107
108 /* These dereferences should succeed. */
109 if (sigsetjmp(jmpbuf, 1))
110 return ksft_test_result_fail("PMLEN=%d valid tag\n", pmlen);
111 if (*p != pmlen)
112 return ksft_test_result_fail("PMLEN=%d bad value\n", pmlen);
113 ++*p;
114 }
115
116 p = (volatile int *)((uintptr_t)&i | 1UL << (__riscv_xlen - pmlen - 1));
117
118 /* These dereferences should raise SIGSEGV. */
119 if (sigsetjmp(jmpbuf, 1))
120 return ksft_test_result_pass("PMLEN=%d dereference\n", pmlen);
121 ++*p;
122 ksft_test_result_fail("PMLEN=%d invalid tag\n", pmlen);
123}
124
125static void test_dereference(void)
126{
127 ksft_print_msg("Testing userspace pointer dereference\n");
128
129 signal(SIGSEGV, sigsegv_handler);
130
131 test_dereference_pmlen(0);
132 test_dereference_pmlen(min_pmlen);
133 test_dereference_pmlen(max_pmlen);
134
135 signal(SIGSEGV, SIG_DFL);
136}
137
138static void execve_child_sigsegv_handler(int sig)
139{
140 exit(42);
141}
142
143static int execve_child(void)
144{
145 static volatile int i;
146 volatile int *p = (volatile int *)((uintptr_t)&i | 1UL << (__riscv_xlen - 7));
147
148 signal(SIGSEGV, execve_child_sigsegv_handler);
149
150 /* This dereference should raise SIGSEGV. */
151 return *p;
152}
153
154static void test_fork_exec(void)
155{
156 int ret, status;
157
158 ksft_print_msg("Testing fork/exec behavior\n");
159
160 ret = set_tagged_addr_ctrl(min_pmlen, false);
161 if (ret)
162 return ksft_test_result_error("setup (%d)\n", ret);
163
164 if (fork()) {
165 wait(&status);
166 ksft_test_result(WIFEXITED(status) && WEXITSTATUS(status) == 42,
167 "dereference after fork\n");
168 } else {
169 static volatile int i = 42;
170 volatile int *p;
171
172 p = (volatile int *)((uintptr_t)&i | 1UL << (__riscv_xlen - min_pmlen));
173
174 /* This dereference should succeed. */
175 exit(*p);
176 }
177
178 if (fork()) {
179 wait(&status);
180 ksft_test_result(WIFEXITED(status) && WEXITSTATUS(status) == 42,
181 "dereference after fork+exec\n");
182 } else {
183 /* Will call execve_child(). */
184 execve("/proc/self/exe", (char *const []) { "", NULL }, NULL);
185 }
186}
187
188static void test_tagged_addr_abi_sysctl(void)
189{
190 char value;
191 int fd;
192
193 ksft_print_msg("Testing tagged address ABI sysctl\n");
194
195 fd = open("/proc/sys/abi/tagged_addr_disabled", O_WRONLY);
196 if (fd < 0) {
197 ksft_test_result_skip("failed to open sysctl file\n");
198 ksft_test_result_skip("failed to open sysctl file\n");
199 return;
200 }
201
202 value = '1';
203 pwrite(fd, &value, 1, 0);
204 ksft_test_result(set_tagged_addr_ctrl(min_pmlen, true) == -EINVAL,
205 "sysctl disabled\n");
206
207 value = '0';
208 pwrite(fd, &value, 1, 0);
209 ksft_test_result(set_tagged_addr_ctrl(min_pmlen, true) == 0,
210 "sysctl enabled\n");
211
212 set_tagged_addr_ctrl(0, false);
213
214 close(fd);
215}
216
217static void test_tagged_addr_abi_pmlen(int pmlen)
218{
219 int i, *p, ret;
220
221 i = ~pmlen;
222
223 if (pmlen) {
224 p = (int *)((uintptr_t)&i | 1UL << (__riscv_xlen - pmlen));
225
226 ret = set_tagged_addr_ctrl(pmlen, false);
227 if (ret)
228 return ksft_test_result_error("PMLEN=%d ABI disabled setup (%d)\n",
229 pmlen, ret);
230
231 ret = write(pipefd[1], p, sizeof(*p));
232 if (ret >= 0 || errno != EFAULT)
233 return ksft_test_result_fail("PMLEN=%d ABI disabled write\n", pmlen);
234
235 ret = read(dev_zero, p, sizeof(*p));
236 if (ret >= 0 || errno != EFAULT)
237 return ksft_test_result_fail("PMLEN=%d ABI disabled read\n", pmlen);
238
239 if (i != ~pmlen)
240 return ksft_test_result_fail("PMLEN=%d ABI disabled value\n", pmlen);
241
242 ret = set_tagged_addr_ctrl(pmlen, true);
243 if (ret)
244 return ksft_test_result_error("PMLEN=%d ABI enabled setup (%d)\n",
245 pmlen, ret);
246
247 ret = write(pipefd[1], p, sizeof(*p));
248 if (ret != sizeof(*p))
249 return ksft_test_result_fail("PMLEN=%d ABI enabled write\n", pmlen);
250
251 ret = read(dev_zero, p, sizeof(*p));
252 if (ret != sizeof(*p))
253 return ksft_test_result_fail("PMLEN=%d ABI enabled read\n", pmlen);
254
255 if (i)
256 return ksft_test_result_fail("PMLEN=%d ABI enabled value\n", pmlen);
257
258 i = ~pmlen;
259 } else {
260 /* The tagged address ABI cannot be enabled when PMLEN == 0. */
261 ret = set_tagged_addr_ctrl(pmlen, true);
262 if (ret != -EINVAL)
263 return ksft_test_result_error("PMLEN=%d ABI setup (%d)\n",
264 pmlen, ret);
265 }
266
267 p = (int *)((uintptr_t)&i | 1UL << (__riscv_xlen - pmlen - 1));
268
269 ret = write(pipefd[1], p, sizeof(*p));
270 if (ret >= 0 || errno != EFAULT)
271 return ksft_test_result_fail("PMLEN=%d invalid tag write (%d)\n", pmlen, errno);
272
273 ret = read(dev_zero, p, sizeof(*p));
274 if (ret >= 0 || errno != EFAULT)
275 return ksft_test_result_fail("PMLEN=%d invalid tag read\n", pmlen);
276
277 if (i != ~pmlen)
278 return ksft_test_result_fail("PMLEN=%d invalid tag value\n", pmlen);
279
280 ksft_test_result_pass("PMLEN=%d tagged address ABI\n", pmlen);
281}
282
283static void test_tagged_addr_abi(void)
284{
285 ksft_print_msg("Testing tagged address ABI\n");
286
287 test_tagged_addr_abi_pmlen(0);
288 test_tagged_addr_abi_pmlen(min_pmlen);
289 test_tagged_addr_abi_pmlen(max_pmlen);
290}
291
292static struct test_info {
293 unsigned int nr_tests;
294 void (*test_fn)(void);
295} tests[] = {
296 { .nr_tests = 17 * 3, test_pmlen },
297 { .nr_tests = 3, test_dereference },
298 { .nr_tests = 2, test_fork_exec },
299 { .nr_tests = 2, test_tagged_addr_abi_sysctl },
300 { .nr_tests = 3, test_tagged_addr_abi },
301};
302
303int main(int argc, char **argv)
304{
305 unsigned int plan = 0;
306 int ret;
307
308 /* Check if this is the child process after execve(). */
309 if (!argv[0][0])
310 return execve_child();
311
312 dev_zero = open("/dev/zero", O_RDWR);
313 if (dev_zero < 0)
314 return 1;
315
316 /* Write to a pipe so the kernel must dereference the buffer pointer. */
317 ret = pipe(pipefd);
318 if (ret)
319 return 1;
320
321 ksft_print_header();
322
323 for (int i = 0; i < ARRAY_SIZE(tests); i++)
324 plan += tests[i].nr_tests;
325
326 ksft_set_plan(plan);
327
328 for (int i = 0; i < ARRAY_SIZE(tests); i++)
329 tests[i].test_fn();
330
331 ksft_finished();
332}