History

Merge tag 'x86_shstk_for_6.6-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 shadow stack support from Dave Hansen: "This is the long awaited x86 shadow stack support, part of Intel's Control-flow Enforcement Technology (CET). CET consists of two related security features: shadow stacks and indirect branch tracking. This series implements just the shadow stack part of this feature, and just for userspace. The main use case for shadow stack is providing protection against return oriented programming attacks. It works by maintaining a secondary (shadow) stack using a special memory type that has protections against modification. When executing a CALL instruction, the processor pushes the return address to both the normal stack and to the special permission shadow stack. Upon RET, the processor pops the shadow stack copy and compares it to the normal stack copy. For more information, refer to the links below for the earlier versions of this patch set" Link: https://lore.kernel.org/lkml/20220130211838.8382-1-rick.p.edgecombe@intel.com/ Link: https://lore.kernel.org/lkml/20230613001108.3040476-1-rick.p.edgecombe@intel.com/ * tag 'x86_shstk_for_6.6-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (47 commits) x86/shstk: Change order of __user in type x86/ibt: Convert IBT selftest to asm x86/shstk: Don't retry vm_munmap() on -EINTR x86/kbuild: Fix Documentation/ reference x86/shstk: Move arch detail comment out of core mm x86/shstk: Add ARCH_SHSTK_STATUS x86/shstk: Add ARCH_SHSTK_UNLOCK x86: Add PTRACE interface for shadow stack selftests/x86: Add shadow stack test x86/cpufeatures: Enable CET CR4 bit for shadow stack x86/shstk: Wire in shadow stack interface x86: Expose thread features in /proc/$PID/status x86/shstk: Support WRSS for userspace x86/shstk: Introduce map_shadow_stack syscall x86/shstk: Check that signal frame is shadow stack mem x86/shstk: Check that SSP is aligned on sigreturn x86/shstk: Handle signals for shadow stack x86/shstk: Introduce routines modifying shstk x86/shstk: Handle thread shadow stack x86/shstk: Add user-mode shadow stack support ...

Linus Torvalds committed 3 years ago

Commit df577212 parents b97d64c 1fe428d118 files · +2790 −308Browse files

Unified · Split

118 files
  1. Documentation/arch/x86/index.rst +1 −0
  2. Documentation/arch/x86/shstk.rst +179 −0
  3. Documentation/filesystems/proc.rst +1 −0
  4. Documentation/mm/arch_pgtable_helpers.rst +10 −2
  5. arch/Kconfig +8 −0
  6. arch/alpha/include/asm/pgtable.h +1 −1
  7. arch/arc/include/asm/hugepage.h +1 −1
  8. arch/arc/include/asm/pgtable-bits-arcv2.h +1 −1
  9. arch/arm/include/asm/pgtable-3level.h +1 −1
  10. arch/arm/include/asm/pgtable.h +1 −1
  11. arch/arm/kernel/signal.c +1 −1
  12. arch/arm64/include/asm/pgtable.h +2 −2
  13. arch/arm64/kernel/signal.c +1 −1
  14. arch/arm64/kernel/signal32.c +1 −1
  15. arch/arm64/mm/trans_pgd.c +2 −2
  16. arch/csky/include/asm/pgtable.h +1 −1
  17. arch/hexagon/include/asm/pgtable.h +1 −1
  18. arch/ia64/include/asm/pgtable.h +1 −1
  19. arch/loongarch/include/asm/pgtable.h +2 −2
  20. arch/m68k/include/asm/mcf_pgtable.h +1 −1
  21. arch/m68k/include/asm/motorola_pgtable.h +1 −1
  22. arch/m68k/include/asm/sun3_pgtable.h +1 −1
  23. arch/microblaze/include/asm/pgtable.h +1 −1
  24. arch/mips/include/asm/pgtable.h +3 −3
  25. arch/nios2/include/asm/pgtable.h +1 −1
  26. arch/openrisc/include/asm/pgtable.h +1 −1
  27. arch/parisc/include/asm/pgtable.h +1 −1
  28. arch/powerpc/include/asm/book3s/32/pgtable.h +1 −1
  29. arch/powerpc/include/asm/book3s/64/pgtable.h +2 −2
  30. arch/powerpc/include/asm/nohash/32/pgtable.h +2 −2
  31. arch/powerpc/include/asm/nohash/32/pte-8xx.h +2 −2
  32. arch/powerpc/include/asm/nohash/64/pgtable.h +1 −1
  33. arch/riscv/include/asm/pgtable.h +3 −3
  34. arch/s390/Kconfig +1 −0
  35. arch/s390/include/asm/hugetlb.h +1 −1
  36. arch/s390/include/asm/pgtable.h +2 −2
  37. arch/s390/mm/pageattr.c +2 −2
  38. arch/sh/include/asm/pgtable_32.h +2 −2
  39. arch/sparc/include/asm/pgtable_32.h +1 −1
  40. arch/sparc/include/asm/pgtable_64.h +3 −3
  41. arch/sparc/kernel/signal32.c +1 −1
  42. arch/sparc/kernel/signal_64.c +1 −1
  43. arch/um/include/asm/pgtable.h +1 −1
  44. arch/x86/Kconfig +24 −0
  45. arch/x86/Kconfig.assembler +5 −0
  46. arch/x86/entry/syscalls/syscall_64.tbl +1 −0
  47. arch/x86/include/asm/cpufeatures.h +2 −0
  48. arch/x86/include/asm/disabled-features.h +14 −2
  49. arch/x86/include/asm/fpu/api.h +9 −0
  50. arch/x86/include/asm/fpu/regset.h +4 −3
  51. arch/x86/include/asm/fpu/sched.h +2 −1
  52. arch/x86/include/asm/fpu/types.h +14 −2
  53. arch/x86/include/asm/fpu/xstate.h +4 −2
  54. arch/x86/include/asm/idtentry.h +1 −1
  55. arch/x86/include/asm/mmu_context.h +2 −0
  56. arch/x86/include/asm/pgtable.h +266 −36
  57. arch/x86/include/asm/pgtable_types.h +36 −8
  58. arch/x86/include/asm/processor.h +8 −0
  59. arch/x86/include/asm/shstk.h +38 −0
  60. arch/x86/include/asm/special_insns.h +13 −0
  61. arch/x86/include/asm/tlbflush.h +2 −1
  62. arch/x86/include/asm/trap_pf.h +2 −0
  63. arch/x86/include/asm/traps.h +14 −1
  64. arch/x86/include/uapi/asm/mman.h +4 −0
  65. arch/x86/include/uapi/asm/prctl.h +12 −0
  66. arch/x86/kernel/Makefile +5 −0
  67. arch/x86/kernel/cet.c +131 −0
  68. arch/x86/kernel/cpu/common.c +27 −8
  69. arch/x86/kernel/cpu/cpuid-deps.c +1 −0
  70. arch/x86/kernel/cpu/proc.c +23 −0
  71. arch/x86/kernel/fpu/core.c +53 −1
  72. arch/x86/kernel/fpu/regset.c +81 −0
  73. arch/x86/kernel/fpu/xstate.c +43 −47
  74. arch/x86/kernel/ibt_selftest.S +17 −0
  75. arch/x86/kernel/idt.c +1 −1
  76. arch/x86/kernel/process.c +20 −1
  77. arch/x86/kernel/process_64.c +8 −0
  78. arch/x86/kernel/ptrace.c +12 −0
  79. arch/x86/kernel/shstk.c +550 −0
  80. arch/x86/kernel/signal.c +1 −0
  81. arch/x86/kernel/signal_32.c +1 −1
  82. arch/x86/kernel/signal_64.c +7 −1
  83. arch/x86/kernel/sys_x86_64.c +5 −1
  84. arch/x86/kernel/traps.c +0 −87
  85. arch/x86/mm/fault.c +22 −0
  86. arch/x86/mm/pat/set_memory.c +2 −2
  87. arch/x86/mm/pgtable.c +40 −0
  88. arch/x86/xen/enlighten_pv.c +1 −1
  89. arch/x86/xen/mmu_pv.c +1 −1
  90. arch/x86/xen/xen-asm.S +1 −1
  91. arch/xtensa/include/asm/pgtable.h +1 −1
  92. fs/aio.c +1 −1
  93. fs/proc/array.c +6 −0
  94. fs/proc/task_mmu.c +3 −0
  95. include/asm-generic/hugetlb.h +1 −1
  96. include/linux/mm.h +39 −8
  97. include/linux/mman.h +4 −0
  98. include/linux/pgtable.h +28 −0
  99. include/linux/proc_fs.h +1 −0
  100. include/linux/syscalls.h +1 −0
  101. include/uapi/asm-generic/siginfo.h +2 −1
  102. include/uapi/linux/elf.h +2 −0
  103. ipc/shm.c +1 −1
  104. kernel/sys_ni.c +1 −0
  105. mm/debug_vm_pgtable.c +6 −6
  106. mm/gup.c +1 −1
  107. mm/huge_memory.c +6 −5
  108. mm/internal.h +2 −2
  109. mm/memory.c +3 −2
  110. mm/migrate.c +1 −1
  111. mm/migrate_device.c +1 −1
  112. mm/mmap.c +7 −7
  113. mm/mprotect.c +1 −1
  114. mm/nommu.c +2 −2
  115. mm/userfaultfd.c +1 −1
  116. mm/util.c +1 −1
  117. tools/testing/selftests/x86/Makefile +1 −1
  118. tools/testing/selftests/x86/test_shadow_stack.c +884 −0
Documentation/arch/x86/index.rst+1−0
@@ -22,6 +22,7 @@ x86-specific Documentation · 21 unmodified lines
22 mtrr22 mtrr
23 pat23 pat
24 intel-hfi24 intel-hfi
25 shstk
25 iommu26 iommu
26 intel_txt27 intel_txt
27 amd-memory-encryption28 amd-memory-encryption
Documentation/arch/x86/shstk.rst+179−0
@@ -0,0 +1,179 @@
1.. SPDX-License-Identifier: GPL-2.0
2
3======================================================
4Control-flow Enforcement Technology (CET) Shadow Stack
5======================================================
6
7CET Background
8==============
9
10Control-flow Enforcement Technology (CET) covers several related x86 processor
11features that provide protection against control flow hijacking attacks. CET
12can protect both applications and the kernel.
13
14CET introduces shadow stack and indirect branch tracking (IBT). A shadow stack
15is a secondary stack allocated from memory which cannot be directly modified by
16applications. When executing a CALL instruction, the processor pushes the
17return address to both the normal stack and the shadow stack. Upon
18function return, the processor pops the shadow stack copy and compares it
19to the normal stack copy. If the two differ, the processor raises a
20control-protection fault. IBT verifies indirect CALL/JMP targets are intended
21as marked by the compiler with 'ENDBR' opcodes. Not all CPU's have both Shadow
22Stack and Indirect Branch Tracking. Today in the 64-bit kernel, only userspace
23shadow stack and kernel IBT are supported.
24
25Requirements to use Shadow Stack
26================================
27
28To use userspace shadow stack you need HW that supports it, a kernel
29configured with it and userspace libraries compiled with it.
30
31The kernel Kconfig option is X86_USER_SHADOW_STACK. When compiled in, shadow
32stacks can be disabled at runtime with the kernel parameter: nousershstk.
33
34To build a user shadow stack enabled kernel, Binutils v2.29 or LLVM v6 or later
35are required.
36
37At run time, /proc/cpuinfo shows CET features if the processor supports
38CET. "user_shstk" means that userspace shadow stack is supported on the current
39kernel and HW.
40
41Application Enabling
42====================
43
44An application's CET capability is marked in its ELF note and can be verified
45from readelf/llvm-readelf output::
46
47 readelf -n <application> | grep -a SHSTK
48 properties: x86 feature: SHSTK
49
50The kernel does not process these applications markers directly. Applications
51or loaders must enable CET features using the interface described in section 4.
52Typically this would be done in dynamic loader or static runtime objects, as is
53the case in GLIBC.
54
55Enabling arch_prctl()'s
56=======================
57
58Elf features should be enabled by the loader using the below arch_prctl's. They
59are only supported in 64 bit user applications. These operate on the features
60on a per-thread basis. The enablement status is inherited on clone, so if the
61feature is enabled on the first thread, it will propagate to all the thread's
62in an app.
63
64arch_prctl(ARCH_SHSTK_ENABLE, unsigned long feature)
65 Enable a single feature specified in 'feature'. Can only operate on
66 one feature at a time.
67
68arch_prctl(ARCH_SHSTK_DISABLE, unsigned long feature)
69 Disable a single feature specified in 'feature'. Can only operate on
70 one feature at a time.
71
72arch_prctl(ARCH_SHSTK_LOCK, unsigned long features)
73 Lock in features at their current enabled or disabled status. 'features'
74 is a mask of all features to lock. All bits set are processed, unset bits
75 are ignored. The mask is ORed with the existing value. So any feature bits
76 set here cannot be enabled or disabled afterwards.
77
78arch_prctl(ARCH_SHSTK_UNLOCK, unsigned long features)
79 Unlock features. 'features' is a mask of all features to unlock. All
80 bits set are processed, unset bits are ignored. Only works via ptrace.
81
82arch_prctl(ARCH_SHSTK_STATUS, unsigned long addr)
83 Copy the currently enabled features to the address passed in addr. The
84 features are described using the bits passed into the others in
85 'features'.
86
87The return values are as follows. On success, return 0. On error, errno can
88be::
89
90 -EPERM if any of the passed feature are locked.
91 -ENOTSUPP if the feature is not supported by the hardware or
92 kernel.
93 -EINVAL arguments (non existing feature, etc)
94 -EFAULT if could not copy information back to userspace
95
96The feature's bits supported are::
97
98 ARCH_SHSTK_SHSTK - Shadow stack
99 ARCH_SHSTK_WRSS - WRSS
100
101Currently shadow stack and WRSS are supported via this interface. WRSS
102can only be enabled with shadow stack, and is automatically disabled
103if shadow stack is disabled.
104
105Proc Status
106===========
107To check if an application is actually running with shadow stack, the
108user can read the /proc/$PID/status. It will report "wrss" or "shstk"
109depending on what is enabled. The lines look like this::
110
111 x86_Thread_features: shstk wrss
112 x86_Thread_features_locked: shstk wrss
113
114Implementation of the Shadow Stack
115==================================
116
117Shadow Stack Size
118-----------------
119
120A task's shadow stack is allocated from memory to a fixed size of
121MIN(RLIMIT_STACK, 4 GB). In other words, the shadow stack is allocated to
122the maximum size of the normal stack, but capped to 4 GB. In the case
123of the clone3 syscall, there is a stack size passed in and shadow stack
124uses this instead of the rlimit.
125
126Signal
127------
128
129The main program and its signal handlers use the same shadow stack. Because
130the shadow stack stores only return addresses, a large shadow stack covers
131the condition that both the program stack and the signal alternate stack run
132out.
133
134When a signal happens, the old pre-signal state is pushed on the stack. When
135shadow stack is enabled, the shadow stack specific state is pushed onto the
136shadow stack. Today this is only the old SSP (shadow stack pointer), pushed
137in a special format with bit 63 set. On sigreturn this old SSP token is
138verified and restored by the kernel. The kernel will also push the normal
139restorer address to the shadow stack to help userspace avoid a shadow stack
140violation on the sigreturn path that goes through the restorer.
141
142So the shadow stack signal frame format is as follows::
143
144 |1...old SSP| - Pointer to old pre-signal ssp in sigframe token format
145 (bit 63 set to 1)
146 | ...| - Other state may be added in the future
147
148
14932 bit ABI signals are not supported in shadow stack processes. Linux prevents
15032 bit execution while shadow stack is enabled by the allocating shadow stacks
151outside of the 32 bit address space. When execution enters 32 bit mode, either
152via far call or returning to userspace, a #GP is generated by the hardware
153which, will be delivered to the process as a segfault. When transitioning to
154userspace the register's state will be as if the userspace ip being returned to
155caused the segfault.
156
157Fork
158----
159
160The shadow stack's vma has VM_SHADOW_STACK flag set; its PTEs are required
161to be read-only and dirty. When a shadow stack PTE is not RO and dirty, a
162shadow access triggers a page fault with the shadow stack access bit set
163in the page fault error code.
164
165When a task forks a child, its shadow stack PTEs are copied and both the
166parent's and the child's shadow stack PTEs are cleared of the dirty bit.
167Upon the next shadow stack access, the resulting shadow stack page fault
168is handled by page copy/re-use.
169
170When a pthread child is created, the kernel allocates a new shadow stack
171for the new thread. New shadow stack creation behaves like mmap() with respect
172to ASLR behavior. Similarly, on thread exit the thread's shadow stack is
173disabled.
174
175Exec
176----
177
178On exec, shadow stack features are disabled by the kernel. At which point,
179userspace can choose to re-enable, or lock them.
Documentation/filesystems/proc.rst+1−0
@@ -566,6 +566,7 @@ encoded manner. The codes are the following: · 565 unmodified lines
566 mt arm64 MTE allocation tags are enabled566 mt arm64 MTE allocation tags are enabled
567 um userfaultfd missing tracking567 um userfaultfd missing tracking
568 uw userfaultfd wr-protect tracking568 uw userfaultfd wr-protect tracking
569 ss shadow stack page
569 == =======================================570 == =======================================
570 571
571Note that there is no guarantee that every flag and associated mnemonic will572Note that there is no guarantee that every flag and associated mnemonic will
Documentation/mm/arch_pgtable_helpers.rst+10−2
@@ -46,7 +46,11 @@ PTE Page Table Helpers · 45 unmodified lines
46+---------------------------+--------------------------------------------------+46+---------------------------+--------------------------------------------------+
47| pte_mkclean | Creates a clean PTE |47| pte_mkclean | Creates a clean PTE |
48+---------------------------+--------------------------------------------------+48+---------------------------+--------------------------------------------------+
49| pte_mkwrite | Creates a writable PTE |49| pte_mkwrite | Creates a writable PTE of the type specified by |
50| | the VMA. |
51+---------------------------+--------------------------------------------------+
52| pte_mkwrite_novma | Creates a writable PTE, of the conventional type |
53| | of writable. |
50+---------------------------+--------------------------------------------------+54+---------------------------+--------------------------------------------------+
51| pte_wrprotect | Creates a write protected PTE |55| pte_wrprotect | Creates a write protected PTE |
52+---------------------------+--------------------------------------------------+56+---------------------------+--------------------------------------------------+
@@ -118,7 +122,11 @@ PMD Page Table Helpers · 65 unmodified lines
118+---------------------------+--------------------------------------------------+122+---------------------------+--------------------------------------------------+
119| pmd_mkclean | Creates a clean PMD |123| pmd_mkclean | Creates a clean PMD |
120+---------------------------+--------------------------------------------------+124+---------------------------+--------------------------------------------------+
121| pmd_mkwrite | Creates a writable PMD |125| pmd_mkwrite | Creates a writable PMD of the type specified by |
126| | the VMA. |
127+---------------------------+--------------------------------------------------+
128| pmd_mkwrite_novma | Creates a writable PMD, of the conventional type |
129| | of writable. |
122+---------------------------+--------------------------------------------------+130+---------------------------+--------------------------------------------------+
123| pmd_wrprotect | Creates a write protected PMD |131| pmd_wrprotect | Creates a write protected PMD |
124+---------------------------+--------------------------------------------------+132+---------------------------+--------------------------------------------------+
arch/Kconfig+8−0
@@ -931,6 +931,14 @@ config HAVE_ARCH_HUGE_VMALLOC · 930 unmodified lines
931config ARCH_WANT_HUGE_PMD_SHARE931config ARCH_WANT_HUGE_PMD_SHARE
932 bool932 bool
933 933
934# Archs that want to use pmd_mkwrite on kernel memory need it defined even
935# if there are no userspace memory management features that use it
936config ARCH_WANT_KERNEL_PMD_MKWRITE
937 bool
938
939config ARCH_WANT_PMD_MKWRITE
940 def_bool TRANSPARENT_HUGEPAGE || ARCH_WANT_KERNEL_PMD_MKWRITE
941
934config HAVE_ARCH_SOFT_DIRTY942config HAVE_ARCH_SOFT_DIRTY
935 bool943 bool
936 944
arch/alpha/include/asm/pgtable.h+1−1
@@ -256,7 +256,7 @@ extern inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; · 255 unmodified lines
256extern inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) |= _PAGE_FOW; return pte; }256extern inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) |= _PAGE_FOW; return pte; }
257extern inline pte_t pte_mkclean(pte_t pte) { pte_val(pte) &= ~(__DIRTY_BITS); return pte; }257extern inline pte_t pte_mkclean(pte_t pte) { pte_val(pte) &= ~(__DIRTY_BITS); return pte; }
258extern inline pte_t pte_mkold(pte_t pte) { pte_val(pte) &= ~(__ACCESS_BITS); return pte; }258extern inline pte_t pte_mkold(pte_t pte) { pte_val(pte) &= ~(__ACCESS_BITS); return pte; }
259extern inline pte_t pte_mkwrite(pte_t pte) { pte_val(pte) &= ~_PAGE_FOW; return pte; }259extern inline pte_t pte_mkwrite_novma(pte_t pte){ pte_val(pte) &= ~_PAGE_FOW; return pte; }
260extern inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= __DIRTY_BITS; return pte; }260extern inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= __DIRTY_BITS; return pte; }
261extern inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= __ACCESS_BITS; return pte; }261extern inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= __ACCESS_BITS; return pte; }
262 262
arch/arc/include/asm/hugepage.h+1−1
@@ -21,7 +21,7 @@ static inline pmd_t pte_pmd(pte_t pte) · 20 unmodified lines
21}21}
22 22
23#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))23#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
24#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))24#define pmd_mkwrite_novma(pmd) pte_pmd(pte_mkwrite_novma(pmd_pte(pmd)))
25#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))25#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
26#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))26#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
27#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))27#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
arch/arc/include/asm/pgtable-bits-arcv2.h+1−1
@@ -87,7 +87,7 @@ · 86 unmodified lines
87 87
88PTE_BIT_FUNC(mknotpresent, &= ~(_PAGE_PRESENT));88PTE_BIT_FUNC(mknotpresent, &= ~(_PAGE_PRESENT));
89PTE_BIT_FUNC(wrprotect, &= ~(_PAGE_WRITE));89PTE_BIT_FUNC(wrprotect, &= ~(_PAGE_WRITE));
90PTE_BIT_FUNC(mkwrite, |= (_PAGE_WRITE));90PTE_BIT_FUNC(mkwrite_novma, |= (_PAGE_WRITE));
91PTE_BIT_FUNC(mkclean, &= ~(_PAGE_DIRTY));91PTE_BIT_FUNC(mkclean, &= ~(_PAGE_DIRTY));
92PTE_BIT_FUNC(mkdirty, |= (_PAGE_DIRTY));92PTE_BIT_FUNC(mkdirty, |= (_PAGE_DIRTY));
93PTE_BIT_FUNC(mkold, &= ~(_PAGE_ACCESSED));93PTE_BIT_FUNC(mkold, &= ~(_PAGE_ACCESSED));
arch/arm/include/asm/pgtable-3level.h+1−1
@@ -202,7 +202,7 @@ static inline pmd_t pmd_##fn(pmd_t pmd) { pmd_val(pmd) op; return pmd; } · 201 unmodified lines
202 202
203PMD_BIT_FUNC(wrprotect, |= L_PMD_SECT_RDONLY);203PMD_BIT_FUNC(wrprotect, |= L_PMD_SECT_RDONLY);
204PMD_BIT_FUNC(mkold, &= ~PMD_SECT_AF);204PMD_BIT_FUNC(mkold, &= ~PMD_SECT_AF);
205PMD_BIT_FUNC(mkwrite, &= ~L_PMD_SECT_RDONLY);205PMD_BIT_FUNC(mkwrite_novma, &= ~L_PMD_SECT_RDONLY);
206PMD_BIT_FUNC(mkdirty, |= L_PMD_SECT_DIRTY);206PMD_BIT_FUNC(mkdirty, |= L_PMD_SECT_DIRTY);
207PMD_BIT_FUNC(mkclean, &= ~L_PMD_SECT_DIRTY);207PMD_BIT_FUNC(mkclean, &= ~L_PMD_SECT_DIRTY);
208PMD_BIT_FUNC(mkyoung, |= PMD_SECT_AF);208PMD_BIT_FUNC(mkyoung, |= PMD_SECT_AF);
arch/arm/include/asm/pgtable.h+1−1
@@ -228,7 +228,7 @@ static inline pte_t pte_wrprotect(pte_t pte) · 227 unmodified lines
228 return set_pte_bit(pte, __pgprot(L_PTE_RDONLY));228 return set_pte_bit(pte, __pgprot(L_PTE_RDONLY));
229}229}
230 230
231static inline pte_t pte_mkwrite(pte_t pte)231static inline pte_t pte_mkwrite_novma(pte_t pte)
232{232{
233 return clear_pte_bit(pte, __pgprot(L_PTE_RDONLY));233 return clear_pte_bit(pte, __pgprot(L_PTE_RDONLY));
234}234}
arch/arm/kernel/signal.c+1−1
@@ -682,7 +682,7 @@ asmlinkage void do_rseq_syscall(struct pt_regs *regs) · 681 unmodified lines
682 */682 */
683static_assert(NSIGILL == 11);683static_assert(NSIGILL == 11);
684static_assert(NSIGFPE == 15);684static_assert(NSIGFPE == 15);
685static_assert(NSIGSEGV == 9);685static_assert(NSIGSEGV == 10);
686static_assert(NSIGBUS == 5);686static_assert(NSIGBUS == 5);
687static_assert(NSIGTRAP == 6);687static_assert(NSIGTRAP == 6);
688static_assert(NSIGCHLD == 6);688static_assert(NSIGCHLD == 6);
arch/arm64/include/asm/pgtable.h+2−2
@@ -181,7 +181,7 @@ static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot) · 180 unmodified lines
181 return pmd;181 return pmd;
182}182}
183 183
184static inline pte_t pte_mkwrite(pte_t pte)184static inline pte_t pte_mkwrite_novma(pte_t pte)
185{185{
186 pte = set_pte_bit(pte, __pgprot(PTE_WRITE));186 pte = set_pte_bit(pte, __pgprot(PTE_WRITE));
187 pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));187 pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
@@ -487,7 +487,7 @@ static inline int pmd_trans_huge(pmd_t pmd) · 299 unmodified lines
487#define pmd_cont(pmd) pte_cont(pmd_pte(pmd))487#define pmd_cont(pmd) pte_cont(pmd_pte(pmd))
488#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))488#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
489#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))489#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
490#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))490#define pmd_mkwrite_novma(pmd) pte_pmd(pte_mkwrite_novma(pmd_pte(pmd)))
491#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))491#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
492#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))492#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
493#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))493#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
arch/arm64/kernel/signal.c+1−1
@@ -1344,7 +1344,7 @@ void __init minsigstksz_setup(void) · 1343 unmodified lines
1344 */1344 */
1345static_assert(NSIGILL == 11);1345static_assert(NSIGILL == 11);
1346static_assert(NSIGFPE == 15);1346static_assert(NSIGFPE == 15);
1347static_assert(NSIGSEGV == 9);1347static_assert(NSIGSEGV == 10);
1348static_assert(NSIGBUS == 5);1348static_assert(NSIGBUS == 5);
1349static_assert(NSIGTRAP == 6);1349static_assert(NSIGTRAP == 6);
1350static_assert(NSIGCHLD == 6);1350static_assert(NSIGCHLD == 6);
arch/arm64/kernel/signal32.c+1−1
@@ -460,7 +460,7 @@ void compat_setup_restart_syscall(struct pt_regs *regs) · 459 unmodified lines
460 */460 */
461static_assert(NSIGILL == 11);461static_assert(NSIGILL == 11);
462static_assert(NSIGFPE == 15);462static_assert(NSIGFPE == 15);
463static_assert(NSIGSEGV == 9);463static_assert(NSIGSEGV == 10);
464static_assert(NSIGBUS == 5);464static_assert(NSIGBUS == 5);
465static_assert(NSIGTRAP == 6);465static_assert(NSIGTRAP == 6);
466static_assert(NSIGCHLD == 6);466static_assert(NSIGCHLD == 6);
arch/arm64/mm/trans_pgd.c+2−2
@@ -41,7 +41,7 @@ static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) · 40 unmodified lines
41 * read only (code, rodata). Clear the RDONLY bit from41 * read only (code, rodata). Clear the RDONLY bit from
42 * the temporary mappings we use during restore.42 * the temporary mappings we use during restore.
43 */43 */
44 set_pte(dst_ptep, pte_mkwrite(pte));44 set_pte(dst_ptep, pte_mkwrite_novma(pte));
45 } else if ((debug_pagealloc_enabled() ||45 } else if ((debug_pagealloc_enabled() ||
46 is_kfence_address((void *)addr)) && !pte_none(pte)) {46 is_kfence_address((void *)addr)) && !pte_none(pte)) {
47 /*47 /*
@@ -55,7 +55,7 @@ static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) · 7 unmodified lines
55 */55 */
56 BUG_ON(!pfn_valid(pte_pfn(pte)));56 BUG_ON(!pfn_valid(pte_pfn(pte)));
57 57
58 set_pte(dst_ptep, pte_mkpresent(pte_mkwrite(pte)));58 set_pte(dst_ptep, pte_mkpresent(pte_mkwrite_novma(pte)));
59 }59 }
60}60}
61 61
arch/csky/include/asm/pgtable.h+1−1
@@ -176,7 +176,7 @@ static inline pte_t pte_mkold(pte_t pte) · 175 unmodified lines
176 return pte;176 return pte;
177}177}
178 178
179static inline pte_t pte_mkwrite(pte_t pte)179static inline pte_t pte_mkwrite_novma(pte_t pte)
180{180{
181 pte_val(pte) |= _PAGE_WRITE;181 pte_val(pte) |= _PAGE_WRITE;
182 if (pte_val(pte) & _PAGE_MODIFIED)182 if (pte_val(pte) & _PAGE_MODIFIED)
arch/hexagon/include/asm/pgtable.h+1−1
@@ -300,7 +300,7 @@ static inline pte_t pte_wrprotect(pte_t pte) · 299 unmodified lines
300}300}
301 301
302/* pte_mkwrite - mark page as writable */302/* pte_mkwrite - mark page as writable */
303static inline pte_t pte_mkwrite(pte_t pte)303static inline pte_t pte_mkwrite_novma(pte_t pte)
304{304{
305 pte_val(pte) |= _PAGE_WRITE;305 pte_val(pte) |= _PAGE_WRITE;
306 return pte;306 return pte;
arch/ia64/include/asm/pgtable.h+1−1
@@ -269,7 +269,7 @@ ia64_phys_addr_valid (unsigned long addr) · 268 unmodified lines
269 * access rights:269 * access rights:
270 */270 */
271#define pte_wrprotect(pte) (__pte(pte_val(pte) & ~_PAGE_AR_RW))271#define pte_wrprotect(pte) (__pte(pte_val(pte) & ~_PAGE_AR_RW))
272#define pte_mkwrite(pte) (__pte(pte_val(pte) | _PAGE_AR_RW))272#define pte_mkwrite_novma(pte) (__pte(pte_val(pte) | _PAGE_AR_RW))
273#define pte_mkold(pte) (__pte(pte_val(pte) & ~_PAGE_A))273#define pte_mkold(pte) (__pte(pte_val(pte) & ~_PAGE_A))
274#define pte_mkyoung(pte) (__pte(pte_val(pte) | _PAGE_A))274#define pte_mkyoung(pte) (__pte(pte_val(pte) | _PAGE_A))
275#define pte_mkclean(pte) (__pte(pte_val(pte) & ~_PAGE_D))275#define pte_mkclean(pte) (__pte(pte_val(pte) & ~_PAGE_D))
arch/loongarch/include/asm/pgtable.h+2−2
@@ -384,7 +384,7 @@ static inline pte_t pte_mkdirty(pte_t pte) · 383 unmodified lines
384 return pte;384 return pte;
385}385}
386 386
387static inline pte_t pte_mkwrite(pte_t pte)387static inline pte_t pte_mkwrite_novma(pte_t pte)
388{388{
389 pte_val(pte) |= _PAGE_WRITE;389 pte_val(pte) |= _PAGE_WRITE;
390 if (pte_val(pte) & _PAGE_MODIFIED)390 if (pte_val(pte) & _PAGE_MODIFIED)
@@ -493,7 +493,7 @@ static inline int pmd_write(pmd_t pmd) · 102 unmodified lines
493 return !!(pmd_val(pmd) & _PAGE_WRITE);493 return !!(pmd_val(pmd) & _PAGE_WRITE);
494}494}
495 495
496static inline pmd_t pmd_mkwrite(pmd_t pmd)496static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
497{497{
498 pmd_val(pmd) |= _PAGE_WRITE;498 pmd_val(pmd) |= _PAGE_WRITE;
499 if (pmd_val(pmd) & _PAGE_MODIFIED)499 if (pmd_val(pmd) & _PAGE_MODIFIED)
arch/m68k/include/asm/mcf_pgtable.h+1−1
@@ -210,7 +210,7 @@ static inline pte_t pte_mkold(pte_t pte) · 209 unmodified lines
210 return pte;210 return pte;
211}211}
212 212
213static inline pte_t pte_mkwrite(pte_t pte)213static inline pte_t pte_mkwrite_novma(pte_t pte)
214{214{
215 pte_val(pte) |= CF_PAGE_WRITABLE;215 pte_val(pte) |= CF_PAGE_WRITABLE;
216 return pte;216 return pte;
arch/m68k/include/asm/motorola_pgtable.h+1−1
@@ -156,7 +156,7 @@ static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; · 155 unmodified lines
156static inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) |= _PAGE_RONLY; return pte; }156static inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) |= _PAGE_RONLY; return pte; }
157static inline pte_t pte_mkclean(pte_t pte) { pte_val(pte) &= ~_PAGE_DIRTY; return pte; }157static inline pte_t pte_mkclean(pte_t pte) { pte_val(pte) &= ~_PAGE_DIRTY; return pte; }
158static inline pte_t pte_mkold(pte_t pte) { pte_val(pte) &= ~_PAGE_ACCESSED; return pte; }158static inline pte_t pte_mkold(pte_t pte) { pte_val(pte) &= ~_PAGE_ACCESSED; return pte; }
159static inline pte_t pte_mkwrite(pte_t pte) { pte_val(pte) &= ~_PAGE_RONLY; return pte; }159static inline pte_t pte_mkwrite_novma(pte_t pte){ pte_val(pte) &= ~_PAGE_RONLY; return pte; }
160static inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= _PAGE_DIRTY; return pte; }160static inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= _PAGE_DIRTY; return pte; }
161static inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= _PAGE_ACCESSED; return pte; }161static inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= _PAGE_ACCESSED; return pte; }
162static inline pte_t pte_mknocache(pte_t pte)162static inline pte_t pte_mknocache(pte_t pte)
arch/m68k/include/asm/sun3_pgtable.h+1−1
@@ -144,7 +144,7 @@ static inline int pte_young(pte_t pte) { return pte_val(pte) & SUN3_PAGE_ACCESS · 143 unmodified lines
144static inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) &= ~SUN3_PAGE_WRITEABLE; return pte; }144static inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) &= ~SUN3_PAGE_WRITEABLE; return pte; }
145static inline pte_t pte_mkclean(pte_t pte) { pte_val(pte) &= ~SUN3_PAGE_MODIFIED; return pte; }145static inline pte_t pte_mkclean(pte_t pte) { pte_val(pte) &= ~SUN3_PAGE_MODIFIED; return pte; }
146static inline pte_t pte_mkold(pte_t pte) { pte_val(pte) &= ~SUN3_PAGE_ACCESSED; return pte; }146static inline pte_t pte_mkold(pte_t pte) { pte_val(pte) &= ~SUN3_PAGE_ACCESSED; return pte; }
147static inline pte_t pte_mkwrite(pte_t pte) { pte_val(pte) |= SUN3_PAGE_WRITEABLE; return pte; }147static inline pte_t pte_mkwrite_novma(pte_t pte){ pte_val(pte) |= SUN3_PAGE_WRITEABLE; return pte; }
148static inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= SUN3_PAGE_MODIFIED; return pte; }148static inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= SUN3_PAGE_MODIFIED; return pte; }
149static inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= SUN3_PAGE_ACCESSED; return pte; }149static inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= SUN3_PAGE_ACCESSED; return pte; }
150static inline pte_t pte_mknocache(pte_t pte) { pte_val(pte) |= SUN3_PAGE_NOCACHE; return pte; }150static inline pte_t pte_mknocache(pte_t pte) { pte_val(pte) |= SUN3_PAGE_NOCACHE; return pte; }
arch/microblaze/include/asm/pgtable.h+1−1
@@ -266,7 +266,7 @@ static inline pte_t pte_mkread(pte_t pte) \ · 265 unmodified lines
266 { pte_val(pte) |= _PAGE_USER; return pte; }266 { pte_val(pte) |= _PAGE_USER; return pte; }
267static inline pte_t pte_mkexec(pte_t pte) \267static inline pte_t pte_mkexec(pte_t pte) \
268 { pte_val(pte) |= _PAGE_USER | _PAGE_EXEC; return pte; }268 { pte_val(pte) |= _PAGE_USER | _PAGE_EXEC; return pte; }
269static inline pte_t pte_mkwrite(pte_t pte) \269static inline pte_t pte_mkwrite_novma(pte_t pte) \
270 { pte_val(pte) |= _PAGE_RW; return pte; }270 { pte_val(pte) |= _PAGE_RW; return pte; }
271static inline pte_t pte_mkdirty(pte_t pte) \271static inline pte_t pte_mkdirty(pte_t pte) \
272 { pte_val(pte) |= _PAGE_DIRTY; return pte; }272 { pte_val(pte) |= _PAGE_DIRTY; return pte; }
arch/mips/include/asm/pgtable.h+3−3
@@ -319,7 +319,7 @@ static inline pte_t pte_mkold(pte_t pte) · 318 unmodified lines
319 return pte;319 return pte;
320}320}
321 321
322static inline pte_t pte_mkwrite(pte_t pte)322static inline pte_t pte_mkwrite_novma(pte_t pte)
323{323{
324 pte.pte_low |= _PAGE_WRITE;324 pte.pte_low |= _PAGE_WRITE;
325 if (pte.pte_low & _PAGE_MODIFIED) {325 if (pte.pte_low & _PAGE_MODIFIED) {
@@ -374,7 +374,7 @@ static inline pte_t pte_mkold(pte_t pte) · 48 unmodified lines
374 return pte;374 return pte;
375}375}
376 376
377static inline pte_t pte_mkwrite(pte_t pte)377static inline pte_t pte_mkwrite_novma(pte_t pte)
378{378{
379 pte_val(pte) |= _PAGE_WRITE;379 pte_val(pte) |= _PAGE_WRITE;
380 if (pte_val(pte) & _PAGE_MODIFIED)380 if (pte_val(pte) & _PAGE_MODIFIED)
@@ -646,7 +646,7 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd) · 265 unmodified lines
646 return pmd;646 return pmd;
647}647}
648 648
649static inline pmd_t pmd_mkwrite(pmd_t pmd)649static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
650{650{
651 pmd_val(pmd) |= _PAGE_WRITE;651 pmd_val(pmd) |= _PAGE_WRITE;
652 if (pmd_val(pmd) & _PAGE_MODIFIED)652 if (pmd_val(pmd) & _PAGE_MODIFIED)
arch/nios2/include/asm/pgtable.h+1−1
@@ -129,7 +129,7 @@ static inline pte_t pte_mkold(pte_t pte) · 128 unmodified lines
129 return pte;129 return pte;
130}130}
131 131
132static inline pte_t pte_mkwrite(pte_t pte)132static inline pte_t pte_mkwrite_novma(pte_t pte)
133{133{
134 pte_val(pte) |= _PAGE_WRITE;134 pte_val(pte) |= _PAGE_WRITE;
135 return pte;135 return pte;
arch/openrisc/include/asm/pgtable.h+1−1
@@ -250,7 +250,7 @@ static inline pte_t pte_mkold(pte_t pte) · 249 unmodified lines
250 return pte;250 return pte;
251}251}
252 252
253static inline pte_t pte_mkwrite(pte_t pte)253static inline pte_t pte_mkwrite_novma(pte_t pte)
254{254{
255 pte_val(pte) |= _PAGE_WRITE;255 pte_val(pte) |= _PAGE_WRITE;
256 return pte;256 return pte;
arch/parisc/include/asm/pgtable.h+1−1
@@ -322,7 +322,7 @@ static inline pte_t pte_mkold(pte_t pte) { pte_val(pte) &= ~_PAGE_ACCESSED; retu · 321 unmodified lines
322static inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) &= ~_PAGE_WRITE; return pte; }322static inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) &= ~_PAGE_WRITE; return pte; }
323static inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= _PAGE_DIRTY; return pte; }323static inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= _PAGE_DIRTY; return pte; }
324static inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= _PAGE_ACCESSED; return pte; }324static inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= _PAGE_ACCESSED; return pte; }
325static inline pte_t pte_mkwrite(pte_t pte) { pte_val(pte) |= _PAGE_WRITE; return pte; }325static inline pte_t pte_mkwrite_novma(pte_t pte) { pte_val(pte) |= _PAGE_WRITE; return pte; }
326static inline pte_t pte_mkspecial(pte_t pte) { pte_val(pte) |= _PAGE_SPECIAL; return pte; }326static inline pte_t pte_mkspecial(pte_t pte) { pte_val(pte) |= _PAGE_SPECIAL; return pte; }
327 327
328/*328/*
arch/powerpc/include/asm/book3s/32/pgtable.h+1−1
@@ -493,7 +493,7 @@ static inline pte_t pte_mkpte(pte_t pte) · 492 unmodified lines
493 return pte;493 return pte;
494}494}
495 495
496static inline pte_t pte_mkwrite(pte_t pte)496static inline pte_t pte_mkwrite_novma(pte_t pte)
497{497{
498 return __pte(pte_val(pte) | _PAGE_RW);498 return __pte(pte_val(pte) | _PAGE_RW);
499}499}
arch/powerpc/include/asm/book3s/64/pgtable.h+2−2
@@ -596,7 +596,7 @@ static inline pte_t pte_mkexec(pte_t pte) · 595 unmodified lines
596 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_EXEC));596 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_EXEC));
597}597}
598 598
599static inline pte_t pte_mkwrite(pte_t pte)599static inline pte_t pte_mkwrite_novma(pte_t pte)
600{600{
601 /*601 /*
602 * write implies read, hence set both602 * write implies read, hence set both
@@ -1088,7 +1088,7 @@ static inline pte_t *pmdp_ptep(pmd_t *pmd) · 485 unmodified lines
1088#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))1088#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
1089#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))1089#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
1090#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))1090#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
1091#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))1091#define pmd_mkwrite_novma(pmd) pte_pmd(pte_mkwrite_novma(pmd_pte(pmd)))
1092 1092
1093#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY1093#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
1094#define pmd_soft_dirty(pmd) pte_soft_dirty(pmd_pte(pmd))1094#define pmd_soft_dirty(pmd) pte_soft_dirty(pmd_pte(pmd))
arch/powerpc/include/asm/nohash/32/pgtable.h+2−2
@@ -170,8 +170,8 @@ void unmap_kernel_page(unsigned long va); · 169 unmodified lines
170#define pte_clear(mm, addr, ptep) \170#define pte_clear(mm, addr, ptep) \
171 do { pte_update(mm, addr, ptep, ~0, 0, 0); } while (0)171 do { pte_update(mm, addr, ptep, ~0, 0, 0); } while (0)
172 172
173#ifndef pte_mkwrite173#ifndef pte_mkwrite_novma
174static inline pte_t pte_mkwrite(pte_t pte)174static inline pte_t pte_mkwrite_novma(pte_t pte)
175{175{
176 return __pte(pte_val(pte) | _PAGE_RW);176 return __pte(pte_val(pte) | _PAGE_RW);
177}177}
arch/powerpc/include/asm/nohash/32/pte-8xx.h+2−2
@@ -101,12 +101,12 @@ static inline int pte_write(pte_t pte) · 100 unmodified lines
101 101
102#define pte_write pte_write102#define pte_write pte_write
103 103
104static inline pte_t pte_mkwrite(pte_t pte)104static inline pte_t pte_mkwrite_novma(pte_t pte)
105{105{
106 return __pte(pte_val(pte) & ~_PAGE_RO);106 return __pte(pte_val(pte) & ~_PAGE_RO);
107}107}
108 108
109#define pte_mkwrite pte_mkwrite109#define pte_mkwrite_novma pte_mkwrite_novma
110 110
111static inline bool pte_user(pte_t pte)111static inline bool pte_user(pte_t pte)
112{112{
arch/powerpc/include/asm/nohash/64/pgtable.h+1−1
@@ -85,7 +85,7 @@ · 84 unmodified lines
85#ifndef __ASSEMBLY__85#ifndef __ASSEMBLY__
86/* pte_clear moved to later in this file */86/* pte_clear moved to later in this file */
87 87
88static inline pte_t pte_mkwrite(pte_t pte)88static inline pte_t pte_mkwrite_novma(pte_t pte)
89{89{
90 return __pte(pte_val(pte) | _PAGE_RW);90 return __pte(pte_val(pte) | _PAGE_RW);
91}91}
arch/riscv/include/asm/pgtable.h+3−3
@@ -380,7 +380,7 @@ static inline pte_t pte_wrprotect(pte_t pte) · 379 unmodified lines
380 380
381/* static inline pte_t pte_mkread(pte_t pte) */381/* static inline pte_t pte_mkread(pte_t pte) */
382 382
383static inline pte_t pte_mkwrite(pte_t pte)383static inline pte_t pte_mkwrite_novma(pte_t pte)
384{384{
385 return __pte(pte_val(pte) | _PAGE_WRITE);385 return __pte(pte_val(pte) | _PAGE_WRITE);
386}386}
@@ -677,9 +677,9 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) · 290 unmodified lines
677 return pte_pmd(pte_mkyoung(pmd_pte(pmd)));677 return pte_pmd(pte_mkyoung(pmd_pte(pmd)));
678}678}
679 679
680static inline pmd_t pmd_mkwrite(pmd_t pmd)680static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
681{681{
682 return pte_pmd(pte_mkwrite(pmd_pte(pmd)));682 return pte_pmd(pte_mkwrite_novma(pmd_pte(pmd)));
683}683}
684 684
685static inline pmd_t pmd_wrprotect(pmd_t pmd)685static inline pmd_t pmd_wrprotect(pmd_t pmd)
arch/s390/Kconfig+1−0
@@ -127,6 +127,7 @@ config S390 · 126 unmodified lines
127 select ARCH_WANTS_NO_INSTR127 select ARCH_WANTS_NO_INSTR
128 select ARCH_WANT_DEFAULT_BPF_JIT128 select ARCH_WANT_DEFAULT_BPF_JIT
129 select ARCH_WANT_IPC_PARSE_VERSION129 select ARCH_WANT_IPC_PARSE_VERSION
130 select ARCH_WANT_KERNEL_PMD_MKWRITE
130 select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP131 select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
131 select BUILDTIME_TABLE_SORT132 select BUILDTIME_TABLE_SORT
132 select CLONE_BACKWARDS2133 select CLONE_BACKWARDS2
arch/s390/include/asm/hugetlb.h+1−1
@@ -104,7 +104,7 @@ static inline int huge_pte_dirty(pte_t pte) · 103 unmodified lines
104 104
105static inline pte_t huge_pte_mkwrite(pte_t pte)105static inline pte_t huge_pte_mkwrite(pte_t pte)
106{106{
107 return pte_mkwrite(pte);107 return pte_mkwrite_novma(pte);
108}108}
109 109
110static inline pte_t huge_pte_mkdirty(pte_t pte)110static inline pte_t huge_pte_mkdirty(pte_t pte)
arch/s390/include/asm/pgtable.h+2−2
@@ -1001,7 +1001,7 @@ static inline pte_t pte_wrprotect(pte_t pte) · 1000 unmodified lines
1001 return set_pte_bit(pte, __pgprot(_PAGE_PROTECT));1001 return set_pte_bit(pte, __pgprot(_PAGE_PROTECT));
1002}1002}
1003 1003
1004static inline pte_t pte_mkwrite(pte_t pte)1004static inline pte_t pte_mkwrite_novma(pte_t pte)
1005{1005{
1006 pte = set_pte_bit(pte, __pgprot(_PAGE_WRITE));1006 pte = set_pte_bit(pte, __pgprot(_PAGE_WRITE));
1007 if (pte_val(pte) & _PAGE_DIRTY)1007 if (pte_val(pte) & _PAGE_DIRTY)
@@ -1498,7 +1498,7 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd) · 490 unmodified lines
1498 return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));1498 return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
1499}1499}
1500 1500
1501static inline pmd_t pmd_mkwrite(pmd_t pmd)1501static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
1502{1502{
1503 pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_WRITE));1503 pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_WRITE));
1504 if (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY)1504 if (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY)
arch/s390/mm/pageattr.c+2−2
@@ -98,7 +98,7 @@ static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, · 97 unmodified lines
98 if (flags & SET_MEMORY_RO)98 if (flags & SET_MEMORY_RO)
99 new = pte_wrprotect(new);99 new = pte_wrprotect(new);
100 else if (flags & SET_MEMORY_RW)100 else if (flags & SET_MEMORY_RW)
101 new = pte_mkwrite(pte_mkdirty(new));101 new = pte_mkwrite_novma(pte_mkdirty(new));
102 if (flags & SET_MEMORY_NX)102 if (flags & SET_MEMORY_NX)
103 new = set_pte_bit(new, __pgprot(_PAGE_NOEXEC));103 new = set_pte_bit(new, __pgprot(_PAGE_NOEXEC));
104 else if (flags & SET_MEMORY_X)104 else if (flags & SET_MEMORY_X)
@@ -156,7 +156,7 @@ static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, · 51 unmodified lines
156 if (flags & SET_MEMORY_RO)156 if (flags & SET_MEMORY_RO)
157 new = pmd_wrprotect(new);157 new = pmd_wrprotect(new);
158 else if (flags & SET_MEMORY_RW)158 else if (flags & SET_MEMORY_RW)
159 new = pmd_mkwrite(pmd_mkdirty(new));159 new = pmd_mkwrite_novma(pmd_mkdirty(new));
160 if (flags & SET_MEMORY_NX)160 if (flags & SET_MEMORY_NX)
161 new = set_pmd_bit(new, __pgprot(_SEGMENT_ENTRY_NOEXEC));161 new = set_pmd_bit(new, __pgprot(_SEGMENT_ENTRY_NOEXEC));
162 else if (flags & SET_MEMORY_X)162 else if (flags & SET_MEMORY_X)
arch/sh/include/asm/pgtable_32.h+2−2
@@ -358,11 +358,11 @@ static inline pte_t pte_##fn(pte_t pte) { pte.pte_##h op; return pte; } · 357 unmodified lines
358 * kernel permissions), we attempt to couple them a bit more sanely here.358 * kernel permissions), we attempt to couple them a bit more sanely here.
359 */359 */
360PTE_BIT_FUNC(high, wrprotect, &= ~(_PAGE_EXT_USER_WRITE | _PAGE_EXT_KERN_WRITE));360PTE_BIT_FUNC(high, wrprotect, &= ~(_PAGE_EXT_USER_WRITE | _PAGE_EXT_KERN_WRITE));
361PTE_BIT_FUNC(high, mkwrite, |= _PAGE_EXT_USER_WRITE | _PAGE_EXT_KERN_WRITE);361PTE_BIT_FUNC(high, mkwrite_novma, |= _PAGE_EXT_USER_WRITE | _PAGE_EXT_KERN_WRITE);
362PTE_BIT_FUNC(high, mkhuge, |= _PAGE_SZHUGE);362PTE_BIT_FUNC(high, mkhuge, |= _PAGE_SZHUGE);
363#else363#else
364PTE_BIT_FUNC(low, wrprotect, &= ~_PAGE_RW);364PTE_BIT_FUNC(low, wrprotect, &= ~_PAGE_RW);
365PTE_BIT_FUNC(low, mkwrite, |= _PAGE_RW);365PTE_BIT_FUNC(low, mkwrite_novma, |= _PAGE_RW);
366PTE_BIT_FUNC(low, mkhuge, |= _PAGE_SZHUGE);366PTE_BIT_FUNC(low, mkhuge, |= _PAGE_SZHUGE);
367#endif367#endif
368 368
arch/sparc/include/asm/pgtable_32.h+1−1
@@ -239,7 +239,7 @@ static inline pte_t pte_mkold(pte_t pte) · 238 unmodified lines
239 return __pte(pte_val(pte) & ~SRMMU_REF);239 return __pte(pte_val(pte) & ~SRMMU_REF);
240}240}
241 241
242static inline pte_t pte_mkwrite(pte_t pte)242static inline pte_t pte_mkwrite_novma(pte_t pte)
243{243{
244 return __pte(pte_val(pte) | SRMMU_WRITE);244 return __pte(pte_val(pte) | SRMMU_WRITE);
245}245}
arch/sparc/include/asm/pgtable_64.h+3−3
@@ -518,7 +518,7 @@ static inline pte_t pte_mkclean(pte_t pte) · 517 unmodified lines
518 return __pte(val);518 return __pte(val);
519}519}
520 520
521static inline pte_t pte_mkwrite(pte_t pte)521static inline pte_t pte_mkwrite_novma(pte_t pte)
522{522{
523 unsigned long val = pte_val(pte), mask;523 unsigned long val = pte_val(pte), mask;
524 524
@@ -773,11 +773,11 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) · 248 unmodified lines
773 return __pmd(pte_val(pte));773 return __pmd(pte_val(pte));
774}774}
775 775
776static inline pmd_t pmd_mkwrite(pmd_t pmd)776static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
777{777{
778 pte_t pte = __pte(pmd_val(pmd));778 pte_t pte = __pte(pmd_val(pmd));
779 779
780 pte = pte_mkwrite(pte);780 pte = pte_mkwrite_novma(pte);
781 781
782 return __pmd(pte_val(pte));782 return __pmd(pte_val(pte));
783}783}
arch/sparc/kernel/signal32.c+1−1
@@ -753,7 +753,7 @@ out: · 752 unmodified lines
753 */753 */
754static_assert(NSIGILL == 11);754static_assert(NSIGILL == 11);
755static_assert(NSIGFPE == 15);755static_assert(NSIGFPE == 15);
756static_assert(NSIGSEGV == 9);756static_assert(NSIGSEGV == 10);
757static_assert(NSIGBUS == 5);757static_assert(NSIGBUS == 5);
758static_assert(NSIGTRAP == 6);758static_assert(NSIGTRAP == 6);
759static_assert(NSIGCHLD == 6);759static_assert(NSIGCHLD == 6);
arch/sparc/kernel/signal_64.c+1−1
@@ -562,7 +562,7 @@ void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long · 561 unmodified lines
562 */562 */
563static_assert(NSIGILL == 11);563static_assert(NSIGILL == 11);
564static_assert(NSIGFPE == 15);564static_assert(NSIGFPE == 15);
565static_assert(NSIGSEGV == 9);565static_assert(NSIGSEGV == 10);
566static_assert(NSIGBUS == 5);566static_assert(NSIGBUS == 5);
567static_assert(NSIGTRAP == 6);567static_assert(NSIGTRAP == 6);
568static_assert(NSIGCHLD == 6);568static_assert(NSIGCHLD == 6);
arch/um/include/asm/pgtable.h+1−1
@@ -207,7 +207,7 @@ static inline pte_t pte_mkyoung(pte_t pte) · 206 unmodified lines
207 return(pte);207 return(pte);
208}208}
209 209
210static inline pte_t pte_mkwrite(pte_t pte)210static inline pte_t pte_mkwrite_novma(pte_t pte)
211{211{
212 if (unlikely(pte_get_bits(pte, _PAGE_RW)))212 if (unlikely(pte_get_bits(pte, _PAGE_RW)))
213 return pte;213 return pte;
arch/x86/Kconfig+24−0
@@ -1815,6 +1815,11 @@ config CC_HAS_IBT · 1814 unmodified lines
1815 (CC_IS_CLANG && CLANG_VERSION >= 140000)) && \1815 (CC_IS_CLANG && CLANG_VERSION >= 140000)) && \
1816 $(as-instr,endbr64)1816 $(as-instr,endbr64)
1817 1817
1818config X86_CET
1819 def_bool n
1820 help
1821 CET features configured (Shadow stack or IBT)
1822
1818config X86_KERNEL_IBT1823config X86_KERNEL_IBT
1819 prompt "Indirect Branch Tracking"1824 prompt "Indirect Branch Tracking"
1820 def_bool y1825 def_bool y
@@ -1822,6 +1827,7 @@ config X86_KERNEL_IBT · 1 unmodified line
1822 # https://github.com/llvm/llvm-project/commit/9d7001eba9c4cb311e03cd8cdc231f9e579f2d0f1827 # https://github.com/llvm/llvm-project/commit/9d7001eba9c4cb311e03cd8cdc231f9e579f2d0f
1823 depends on !LD_IS_LLD || LLD_VERSION >= 1400001828 depends on !LD_IS_LLD || LLD_VERSION >= 140000
1824 select OBJTOOL1829 select OBJTOOL
1830 select X86_CET
1825 help1831 help
1826 Build the kernel with support for Indirect Branch Tracking, a1832 Build the kernel with support for Indirect Branch Tracking, a
1827 hardware support course-grain forward-edge Control Flow Integrity1833 hardware support course-grain forward-edge Control Flow Integrity
@@ -1915,6 +1921,24 @@ config X86_SGX · 87 unmodified lines
1915 1921
1916 If unsure, say N.1922 If unsure, say N.
1917 1923
1924config X86_USER_SHADOW_STACK
1925 bool "X86 userspace shadow stack"
1926 depends on AS_WRUSS
1927 depends on X86_64
1928 select ARCH_USES_HIGH_VMA_FLAGS
1929 select X86_CET
1930 help
1931 Shadow stack protection is a hardware feature that detects function
1932 return address corruption. This helps mitigate ROP attacks.
1933 Applications must be enabled to use it, and old userspace does not
1934 get protection "for free".
1935
1936 CPUs supporting shadow stacks were first released in 2020.
1937
1938 See Documentation/arch/x86/shstk.rst for more information.
1939
1940 If unsure, say N.
1941
1918config EFI1942config EFI
1919 bool "EFI runtime service support"1943 bool "EFI runtime service support"
1920 depends on ACPI1944 depends on ACPI
arch/x86/Kconfig.assembler+5−0
@@ -24,3 +24,8 @@ config AS_GFNI · 23 unmodified lines
24 def_bool $(as-instr,vgf2p8mulb %xmm0$(comma)%xmm1$(comma)%xmm2)24 def_bool $(as-instr,vgf2p8mulb %xmm0$(comma)%xmm1$(comma)%xmm2)
25 help25 help
26 Supported by binutils >= 2.30 and LLVM integrated assembler26 Supported by binutils >= 2.30 and LLVM integrated assembler
27
28config AS_WRUSS
29 def_bool $(as-instr,wrussq %rax$(comma)(%rbx))
30 help
31 Supported by binutils >= 2.31 and LLVM integrated assembler
arch/x86/entry/syscalls/syscall_64.tbl+1−0
@@ -374,6 +374,7 @@ · 373 unmodified lines
374450 common set_mempolicy_home_node sys_set_mempolicy_home_node374450 common set_mempolicy_home_node sys_set_mempolicy_home_node
375451 common cachestat sys_cachestat375451 common cachestat sys_cachestat
376452 common fchmodat2 sys_fchmodat2376452 common fchmodat2 sys_fchmodat2
377453 64 map_shadow_stack sys_map_shadow_stack
377 378
378#379#
379# Due to a historical design error, certain syscalls are numbered differently380# Due to a historical design error, certain syscalls are numbered differently
arch/x86/include/asm/cpufeatures.h+2−0
@@ -307,6 +307,7 @@ · 306 unmodified lines
307#define X86_FEATURE_MSR_TSX_CTRL (11*32+20) /* "" MSR IA32_TSX_CTRL (Intel) implemented */307#define X86_FEATURE_MSR_TSX_CTRL (11*32+20) /* "" MSR IA32_TSX_CTRL (Intel) implemented */
308#define X86_FEATURE_SMBA (11*32+21) /* "" Slow Memory Bandwidth Allocation */308#define X86_FEATURE_SMBA (11*32+21) /* "" Slow Memory Bandwidth Allocation */
309#define X86_FEATURE_BMEC (11*32+22) /* "" Bandwidth Monitoring Event Configuration */309#define X86_FEATURE_BMEC (11*32+22) /* "" Bandwidth Monitoring Event Configuration */
310#define X86_FEATURE_USER_SHSTK (11*32+23) /* Shadow stack support for user mode applications */
310 311
311#define X86_FEATURE_SRSO (11*32+24) /* "" AMD BTB untrain RETs */312#define X86_FEATURE_SRSO (11*32+24) /* "" AMD BTB untrain RETs */
312#define X86_FEATURE_SRSO_ALIAS (11*32+25) /* "" AMD BTB untrain RETs through aliasing */313#define X86_FEATURE_SRSO_ALIAS (11*32+25) /* "" AMD BTB untrain RETs through aliasing */
@@ -383,6 +384,7 @@ · 70 unmodified lines
383#define X86_FEATURE_OSPKE (16*32+ 4) /* OS Protection Keys Enable */384#define X86_FEATURE_OSPKE (16*32+ 4) /* OS Protection Keys Enable */
384#define X86_FEATURE_WAITPKG (16*32+ 5) /* UMONITOR/UMWAIT/TPAUSE Instructions */385#define X86_FEATURE_WAITPKG (16*32+ 5) /* UMONITOR/UMWAIT/TPAUSE Instructions */
385#define X86_FEATURE_AVX512_VBMI2 (16*32+ 6) /* Additional AVX512 Vector Bit Manipulation Instructions */386#define X86_FEATURE_AVX512_VBMI2 (16*32+ 6) /* Additional AVX512 Vector Bit Manipulation Instructions */
387#define X86_FEATURE_SHSTK (16*32+ 7) /* "" Shadow stack */
386#define X86_FEATURE_GFNI (16*32+ 8) /* Galois Field New Instructions */388#define X86_FEATURE_GFNI (16*32+ 8) /* Galois Field New Instructions */
387#define X86_FEATURE_VAES (16*32+ 9) /* Vector AES */389#define X86_FEATURE_VAES (16*32+ 9) /* Vector AES */
388#define X86_FEATURE_VPCLMULQDQ (16*32+10) /* Carry-Less Multiplication Double Quadword */390#define X86_FEATURE_VPCLMULQDQ (16*32+10) /* Carry-Less Multiplication Double Quadword */
arch/x86/include/asm/disabled-features.h+14−2
@@ -105,6 +105,18 @@ · 104 unmodified lines
105# define DISABLE_TDX_GUEST (1 << (X86_FEATURE_TDX_GUEST & 31))105# define DISABLE_TDX_GUEST (1 << (X86_FEATURE_TDX_GUEST & 31))
106#endif106#endif
107 107
108#ifdef CONFIG_X86_USER_SHADOW_STACK
109#define DISABLE_USER_SHSTK 0
110#else
111#define DISABLE_USER_SHSTK (1 << (X86_FEATURE_USER_SHSTK & 31))
112#endif
113
114#ifdef CONFIG_X86_KERNEL_IBT
115#define DISABLE_IBT 0
116#else
117#define DISABLE_IBT (1 << (X86_FEATURE_IBT & 31))
118#endif
119
108/*120/*
109 * Make sure to add features to the correct mask121 * Make sure to add features to the correct mask
110 */122 */
@@ -120,7 +132,7 @@ · 9 unmodified lines
120#define DISABLED_MASK9 (DISABLE_SGX)132#define DISABLED_MASK9 (DISABLE_SGX)
121#define DISABLED_MASK10 0133#define DISABLED_MASK10 0
122#define DISABLED_MASK11 (DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET| \134#define DISABLED_MASK11 (DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET| \
123 DISABLE_CALL_DEPTH_TRACKING)135 DISABLE_CALL_DEPTH_TRACKING|DISABLE_USER_SHSTK)
124#define DISABLED_MASK12 (DISABLE_LAM)136#define DISABLED_MASK12 (DISABLE_LAM)
125#define DISABLED_MASK13 0137#define DISABLED_MASK13 0
126#define DISABLED_MASK14 0138#define DISABLED_MASK14 0
@@ -128,7 +140,7 @@ · 1 unmodified line
128#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP| \140#define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP| \
129 DISABLE_ENQCMD)141 DISABLE_ENQCMD)
130#define DISABLED_MASK17 0142#define DISABLED_MASK17 0
131#define DISABLED_MASK18 0143#define DISABLED_MASK18 (DISABLE_IBT)
132#define DISABLED_MASK19 0144#define DISABLED_MASK19 0
133#define DISABLED_MASK20 0145#define DISABLED_MASK20 0
134#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 21)146#define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 21)
arch/x86/include/asm/fpu/api.h+9−0
@@ -82,6 +82,15 @@ static inline void fpregs_unlock(void) · 81 unmodified lines
82 preempt_enable();82 preempt_enable();
83}83}
84 84
85/*
86 * FPU state gets lazily restored before returning to userspace. So when in the
87 * kernel, the valid FPU state may be kept in the buffer. This function will force
88 * restore all the fpu state to the registers early if needed, and lock them from
89 * being automatically saved/restored. Then FPU state can be modified safely in the
90 * registers, before unlocking with fpregs_unlock().
91 */
92void fpregs_lock_and_load(void);
93
85#ifdef CONFIG_X86_DEBUG_FPU94#ifdef CONFIG_X86_DEBUG_FPU
86extern void fpregs_assert_state_consistent(void);95extern void fpregs_assert_state_consistent(void);
87#else96#else
arch/x86/include/asm/fpu/regset.h+4−3
@@ -7,11 +7,12 @@ · 6 unmodified lines
7 7
8#include <linux/regset.h>8#include <linux/regset.h>
9 9
10extern user_regset_active_fn regset_fpregs_active, regset_xregset_fpregs_active;10extern user_regset_active_fn regset_fpregs_active, regset_xregset_fpregs_active,
11 ssp_active;
11extern user_regset_get2_fn fpregs_get, xfpregs_get, fpregs_soft_get,12extern user_regset_get2_fn fpregs_get, xfpregs_get, fpregs_soft_get,
12 xstateregs_get;13 xstateregs_get, ssp_get;
13extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set,14extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set,
14 xstateregs_set;15 xstateregs_set, ssp_set;
15 16
16/*17/*
17 * xstateregs_active == regset_fpregs_active. Please refer to the comment18 * xstateregs_active == regset_fpregs_active. Please refer to the comment
arch/x86/include/asm/fpu/sched.h+2−1
@@ -11,7 +11,8 @@ · 10 unmodified lines
11 11
12extern void save_fpregs_to_fpstate(struct fpu *fpu);12extern void save_fpregs_to_fpstate(struct fpu *fpu);
13extern void fpu__drop(struct fpu *fpu);13extern void fpu__drop(struct fpu *fpu);
14extern int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal);14extern int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal,
15 unsigned long shstk_addr);
15extern void fpu_flush_thread(void);16extern void fpu_flush_thread(void);
16 17
17/*18/*
arch/x86/include/asm/fpu/types.h+14−2
@@ -115,8 +115,8 @@ enum xfeature { · 114 unmodified lines
115 XFEATURE_PT_UNIMPLEMENTED_SO_FAR,115 XFEATURE_PT_UNIMPLEMENTED_SO_FAR,
116 XFEATURE_PKRU,116 XFEATURE_PKRU,
117 XFEATURE_PASID,117 XFEATURE_PASID,
118 XFEATURE_RSRVD_COMP_11,118 XFEATURE_CET_USER,
119 XFEATURE_RSRVD_COMP_12,119 XFEATURE_CET_KERNEL_UNUSED,
120 XFEATURE_RSRVD_COMP_13,120 XFEATURE_RSRVD_COMP_13,
121 XFEATURE_RSRVD_COMP_14,121 XFEATURE_RSRVD_COMP_14,
122 XFEATURE_LBR,122 XFEATURE_LBR,
@@ -138,6 +138,8 @@ enum xfeature { · 15 unmodified lines
138#define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR)138#define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR)
139#define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU)139#define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU)
140#define XFEATURE_MASK_PASID (1 << XFEATURE_PASID)140#define XFEATURE_MASK_PASID (1 << XFEATURE_PASID)
141#define XFEATURE_MASK_CET_USER (1 << XFEATURE_CET_USER)
142#define XFEATURE_MASK_CET_KERNEL (1 << XFEATURE_CET_KERNEL_UNUSED)
141#define XFEATURE_MASK_LBR (1 << XFEATURE_LBR)143#define XFEATURE_MASK_LBR (1 << XFEATURE_LBR)
142#define XFEATURE_MASK_XTILE_CFG (1 << XFEATURE_XTILE_CFG)144#define XFEATURE_MASK_XTILE_CFG (1 << XFEATURE_XTILE_CFG)
143#define XFEATURE_MASK_XTILE_DATA (1 << XFEATURE_XTILE_DATA)145#define XFEATURE_MASK_XTILE_DATA (1 << XFEATURE_XTILE_DATA)
@@ -252,6 +254,16 @@ struct pkru_state { · 108 unmodified lines
252 u32 pad;254 u32 pad;
253} __packed;255} __packed;
254 256
257/*
258 * State component 11 is Control-flow Enforcement user states
259 */
260struct cet_user_state {
261 /* user control-flow settings */
262 u64 user_cet;
263 /* user shadow stack pointer */
264 u64 user_ssp;
265};
266
255/*267/*
256 * State component 15: Architectural LBR configuration state.268 * State component 15: Architectural LBR configuration state.
257 * The size of Arch LBR state depends on the number of LBRs (lbr_depth).269 * The size of Arch LBR state depends on the number of LBRs (lbr_depth).
arch/x86/include/asm/fpu/xstate.h+4−2
@@ -50,7 +50,8 @@ · 49 unmodified lines
50#define XFEATURE_MASK_USER_DYNAMIC XFEATURE_MASK_XTILE_DATA50#define XFEATURE_MASK_USER_DYNAMIC XFEATURE_MASK_XTILE_DATA
51 51
52/* All currently supported supervisor features */52/* All currently supported supervisor features */
53#define XFEATURE_MASK_SUPERVISOR_SUPPORTED (XFEATURE_MASK_PASID)53#define XFEATURE_MASK_SUPERVISOR_SUPPORTED (XFEATURE_MASK_PASID | \
54 XFEATURE_MASK_CET_USER)
54 55
55/*56/*
56 * A supervisor state component may not always contain valuable information,57 * A supervisor state component may not always contain valuable information,
@@ -77,7 +78,8 @@ · 20 unmodified lines
77 * Unsupported supervisor features. When a supervisor feature in this mask is78 * Unsupported supervisor features. When a supervisor feature in this mask is
78 * supported in the future, move it to the supported supervisor feature mask.79 * supported in the future, move it to the supported supervisor feature mask.
79 */80 */
80#define XFEATURE_MASK_SUPERVISOR_UNSUPPORTED (XFEATURE_MASK_PT)81#define XFEATURE_MASK_SUPERVISOR_UNSUPPORTED (XFEATURE_MASK_PT | \
82 XFEATURE_MASK_CET_KERNEL)
81 83
82/* All supervisor states including supported and unsupported states. */84/* All supervisor states including supported and unsupported states. */
83#define XFEATURE_MASK_SUPERVISOR_ALL (XFEATURE_MASK_SUPERVISOR_SUPPORTED | \85#define XFEATURE_MASK_SUPERVISOR_ALL (XFEATURE_MASK_SUPERVISOR_SUPPORTED | \
arch/x86/include/asm/idtentry.h+1−1
@@ -614,7 +614,7 @@ DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_DF, xenpv_exc_double_fault); · 613 unmodified lines
614#endif614#endif
615 615
616/* #CP */616/* #CP */
617#ifdef CONFIG_X86_KERNEL_IBT617#ifdef CONFIG_X86_CET
618DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_CP, exc_control_protection);618DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_CP, exc_control_protection);
619#endif619#endif
620 620
arch/x86/include/asm/mmu_context.h+2−0
@@ -186,6 +186,8 @@ do { \ · 185 unmodified lines
186#else186#else
187#define deactivate_mm(tsk, mm) \187#define deactivate_mm(tsk, mm) \
188do { \188do { \
189 if (!tsk->vfork_done) \
190 shstk_free(tsk); \
189 load_gs_index(0); \191 load_gs_index(0); \
190 loadsegment(fs, 0); \192 loadsegment(fs, 0); \
191} while (0)193} while (0)
arch/x86/include/asm/pgtable.h+266−36
@@ -125,9 +125,15 @@ extern pmdval_t early_pmd_flags; · 124 unmodified lines
125 * The following only work if pte_present() is true.125 * The following only work if pte_present() is true.
126 * Undefined behaviour if not..126 * Undefined behaviour if not..
127 */127 */
128static inline int pte_dirty(pte_t pte)128static inline bool pte_dirty(pte_t pte)
129{129{
130 return pte_flags(pte) & _PAGE_DIRTY;130 return pte_flags(pte) & _PAGE_DIRTY_BITS;
131}
132
133static inline bool pte_shstk(pte_t pte)
134{
135 return cpu_feature_enabled(X86_FEATURE_SHSTK) &&
136 (pte_flags(pte) & (_PAGE_RW | _PAGE_DIRTY)) == _PAGE_DIRTY;
131}137}
132 138
133static inline int pte_young(pte_t pte)139static inline int pte_young(pte_t pte)
@@ -135,9 +141,16 @@ static inline int pte_young(pte_t pte) · 1 unmodified line
135 return pte_flags(pte) & _PAGE_ACCESSED;141 return pte_flags(pte) & _PAGE_ACCESSED;
136}142}
137 143
138static inline int pmd_dirty(pmd_t pmd)144static inline bool pmd_dirty(pmd_t pmd)
145{
146 return pmd_flags(pmd) & _PAGE_DIRTY_BITS;
147}
148
149static inline bool pmd_shstk(pmd_t pmd)
139{150{
140 return pmd_flags(pmd) & _PAGE_DIRTY;151 return cpu_feature_enabled(X86_FEATURE_SHSTK) &&
152 (pmd_flags(pmd) & (_PAGE_RW | _PAGE_DIRTY | _PAGE_PSE)) ==
153 (_PAGE_DIRTY | _PAGE_PSE);
141}154}
142 155
143#define pmd_young pmd_young156#define pmd_young pmd_young
@@ -146,9 +159,9 @@ static inline int pmd_young(pmd_t pmd) · 2 unmodified lines
146 return pmd_flags(pmd) & _PAGE_ACCESSED;159 return pmd_flags(pmd) & _PAGE_ACCESSED;
147}160}
148 161
149static inline int pud_dirty(pud_t pud)162static inline bool pud_dirty(pud_t pud)
150{163{
151 return pud_flags(pud) & _PAGE_DIRTY;164 return pud_flags(pud) & _PAGE_DIRTY_BITS;
152}165}
153 166
154static inline int pud_young(pud_t pud)167static inline int pud_young(pud_t pud)
@@ -158,7 +171,27 @@ static inline int pud_young(pud_t pud) · 3 unmodified lines
158 171
159static inline int pte_write(pte_t pte)172static inline int pte_write(pte_t pte)
160{173{
161 return pte_flags(pte) & _PAGE_RW;174 /*
175 * Shadow stack pages are logically writable, but do not have
176 * _PAGE_RW. Check for them separately from _PAGE_RW itself.
177 */
178 return (pte_flags(pte) & _PAGE_RW) || pte_shstk(pte);
179}
180
181#define pmd_write pmd_write
182static inline int pmd_write(pmd_t pmd)
183{
184 /*
185 * Shadow stack pages are logically writable, but do not have
186 * _PAGE_RW. Check for them separately from _PAGE_RW itself.
187 */
188 return (pmd_flags(pmd) & _PAGE_RW) || pmd_shstk(pmd);
189}
190
191#define pud_write pud_write
192static inline int pud_write(pud_t pud)
193{
194 return pud_flags(pud) & _PAGE_RW;
162}195}
163 196
164static inline int pte_huge(pte_t pte)197static inline int pte_huge(pte_t pte)
@@ -292,9 +325,63 @@ static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear) · 127 unmodified lines
292 return native_make_pte(v & ~clear);325 return native_make_pte(v & ~clear);
293}326}
294 327
328/*
329 * Write protection operations can result in Dirty=1,Write=0 PTEs. But in the
330 * case of X86_FEATURE_USER_SHSTK, these PTEs denote shadow stack memory. So
331 * when creating dirty, write-protected memory, a software bit is used:
332 * _PAGE_BIT_SAVED_DIRTY. The following functions take a PTE and transition the
333 * Dirty bit to SavedDirty, and vice-vesra.
334 *
335 * This shifting is only done if needed. In the case of shifting
336 * Dirty->SavedDirty, the condition is if the PTE is Write=0. In the case of
337 * shifting SavedDirty->Dirty, the condition is Write=1.
338 */
339static inline pgprotval_t mksaveddirty_shift(pgprotval_t v)
340{
341 pgprotval_t cond = (~v >> _PAGE_BIT_RW) & 1;
342
343 v |= ((v >> _PAGE_BIT_DIRTY) & cond) << _PAGE_BIT_SAVED_DIRTY;
344 v &= ~(cond << _PAGE_BIT_DIRTY);
345
346 return v;
347}
348
349static inline pgprotval_t clear_saveddirty_shift(pgprotval_t v)
350{
351 pgprotval_t cond = (v >> _PAGE_BIT_RW) & 1;
352
353 v |= ((v >> _PAGE_BIT_SAVED_DIRTY) & cond) << _PAGE_BIT_DIRTY;
354 v &= ~(cond << _PAGE_BIT_SAVED_DIRTY);
355
356 return v;
357}
358
359static inline pte_t pte_mksaveddirty(pte_t pte)
360{
361 pteval_t v = native_pte_val(pte);
362
363 v = mksaveddirty_shift(v);
364 return native_make_pte(v);
365}
366
367static inline pte_t pte_clear_saveddirty(pte_t pte)
368{
369 pteval_t v = native_pte_val(pte);
370
371 v = clear_saveddirty_shift(v);
372 return native_make_pte(v);
373}
374
295static inline pte_t pte_wrprotect(pte_t pte)375static inline pte_t pte_wrprotect(pte_t pte)
296{376{
297 return pte_clear_flags(pte, _PAGE_RW);377 pte = pte_clear_flags(pte, _PAGE_RW);
378
379 /*
380 * Blindly clearing _PAGE_RW might accidentally create
381 * a shadow stack PTE (Write=0,Dirty=1). Move the hardware
382 * dirty value to the software bit, if present.
383 */
384 return pte_mksaveddirty(pte);
298}385}
299 386
300#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP387#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
@@ -332,7 +419,7 @@ static inline pte_t pte_clear_uffd_wp(pte_t pte) · 31 unmodified lines
332 419
333static inline pte_t pte_mkclean(pte_t pte)420static inline pte_t pte_mkclean(pte_t pte)
334{421{
335 return pte_clear_flags(pte, _PAGE_DIRTY);422 return pte_clear_flags(pte, _PAGE_DIRTY_BITS);
336}423}
337 424
338static inline pte_t pte_mkold(pte_t pte)425static inline pte_t pte_mkold(pte_t pte)
@@ -347,7 +434,16 @@ static inline pte_t pte_mkexec(pte_t pte) · 8 unmodified lines
347 434
348static inline pte_t pte_mkdirty(pte_t pte)435static inline pte_t pte_mkdirty(pte_t pte)
349{436{
350 return pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);437 pte = pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
438
439 return pte_mksaveddirty(pte);
440}
441
442static inline pte_t pte_mkwrite_shstk(pte_t pte)
443{
444 pte = pte_clear_flags(pte, _PAGE_RW);
445
446 return pte_set_flags(pte, _PAGE_DIRTY);
351}447}
352 448
353static inline pte_t pte_mkyoung(pte_t pte)449static inline pte_t pte_mkyoung(pte_t pte)
@@ -355,11 +451,15 @@ static inline pte_t pte_mkyoung(pte_t pte) · 1 unmodified line
355 return pte_set_flags(pte, _PAGE_ACCESSED);451 return pte_set_flags(pte, _PAGE_ACCESSED);
356}452}
357 453
358static inline pte_t pte_mkwrite(pte_t pte)454static inline pte_t pte_mkwrite_novma(pte_t pte)
359{455{
360 return pte_set_flags(pte, _PAGE_RW);456 return pte_set_flags(pte, _PAGE_RW);
361}457}
362 458
459struct vm_area_struct;
460pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma);
461#define pte_mkwrite pte_mkwrite
462
363static inline pte_t pte_mkhuge(pte_t pte)463static inline pte_t pte_mkhuge(pte_t pte)
364{464{
365 return pte_set_flags(pte, _PAGE_PSE);465 return pte_set_flags(pte, _PAGE_PSE);
@@ -404,9 +504,34 @@ static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear) · 38 unmodified lines
404 return native_make_pmd(v & ~clear);504 return native_make_pmd(v & ~clear);
405}505}
406 506
507/* See comments above mksaveddirty_shift() */
508static inline pmd_t pmd_mksaveddirty(pmd_t pmd)
509{
510 pmdval_t v = native_pmd_val(pmd);
511
512 v = mksaveddirty_shift(v);
513 return native_make_pmd(v);
514}
515
516/* See comments above mksaveddirty_shift() */
517static inline pmd_t pmd_clear_saveddirty(pmd_t pmd)
518{
519 pmdval_t v = native_pmd_val(pmd);
520
521 v = clear_saveddirty_shift(v);
522 return native_make_pmd(v);
523}
524
407static inline pmd_t pmd_wrprotect(pmd_t pmd)525static inline pmd_t pmd_wrprotect(pmd_t pmd)
408{526{
409 return pmd_clear_flags(pmd, _PAGE_RW);527 pmd = pmd_clear_flags(pmd, _PAGE_RW);
528
529 /*
530 * Blindly clearing _PAGE_RW might accidentally create
531 * a shadow stack PMD (RW=0, Dirty=1). Move the hardware
532 * dirty value to the software bit.
533 */
534 return pmd_mksaveddirty(pmd);
410}535}
411 536
412#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP537#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
@@ -433,12 +558,21 @@ static inline pmd_t pmd_mkold(pmd_t pmd) · 20 unmodified lines
433 558
434static inline pmd_t pmd_mkclean(pmd_t pmd)559static inline pmd_t pmd_mkclean(pmd_t pmd)
435{560{
436 return pmd_clear_flags(pmd, _PAGE_DIRTY);561 return pmd_clear_flags(pmd, _PAGE_DIRTY_BITS);
437}562}
438 563
439static inline pmd_t pmd_mkdirty(pmd_t pmd)564static inline pmd_t pmd_mkdirty(pmd_t pmd)
440{565{
441 return pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);566 pmd = pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
567
568 return pmd_mksaveddirty(pmd);
569}
570
571static inline pmd_t pmd_mkwrite_shstk(pmd_t pmd)
572{
573 pmd = pmd_clear_flags(pmd, _PAGE_RW);
574
575 return pmd_set_flags(pmd, _PAGE_DIRTY);
442}576}
443 577
444static inline pmd_t pmd_mkdevmap(pmd_t pmd)578static inline pmd_t pmd_mkdevmap(pmd_t pmd)
@@ -456,11 +590,14 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) · 11 unmodified lines
456 return pmd_set_flags(pmd, _PAGE_ACCESSED);590 return pmd_set_flags(pmd, _PAGE_ACCESSED);
457}591}
458 592
459static inline pmd_t pmd_mkwrite(pmd_t pmd)593static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
460{594{
461 return pmd_set_flags(pmd, _PAGE_RW);595 return pmd_set_flags(pmd, _PAGE_RW);
462}596}
463 597
598pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
599#define pmd_mkwrite pmd_mkwrite
600
464static inline pud_t pud_set_flags(pud_t pud, pudval_t set)601static inline pud_t pud_set_flags(pud_t pud, pudval_t set)
465{602{
466 pudval_t v = native_pud_val(pud);603 pudval_t v = native_pud_val(pud);
@@ -475,6 +612,24 @@ static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear) · 8 unmodified lines
475 return native_make_pud(v & ~clear);612 return native_make_pud(v & ~clear);
476}613}
477 614
615/* See comments above mksaveddirty_shift() */
616static inline pud_t pud_mksaveddirty(pud_t pud)
617{
618 pudval_t v = native_pud_val(pud);
619
620 v = mksaveddirty_shift(v);
621 return native_make_pud(v);
622}
623
624/* See comments above mksaveddirty_shift() */
625static inline pud_t pud_clear_saveddirty(pud_t pud)
626{
627 pudval_t v = native_pud_val(pud);
628
629 v = clear_saveddirty_shift(v);
630 return native_make_pud(v);
631}
632
478static inline pud_t pud_mkold(pud_t pud)633static inline pud_t pud_mkold(pud_t pud)
479{634{
480 return pud_clear_flags(pud, _PAGE_ACCESSED);635 return pud_clear_flags(pud, _PAGE_ACCESSED);
@@ -482,17 +637,26 @@ static inline pud_t pud_mkold(pud_t pud) · 1 unmodified line
482 637
483static inline pud_t pud_mkclean(pud_t pud)638static inline pud_t pud_mkclean(pud_t pud)
484{639{
485 return pud_clear_flags(pud, _PAGE_DIRTY);640 return pud_clear_flags(pud, _PAGE_DIRTY_BITS);
486}641}
487 642
488static inline pud_t pud_wrprotect(pud_t pud)643static inline pud_t pud_wrprotect(pud_t pud)
489{644{
490 return pud_clear_flags(pud, _PAGE_RW);645 pud = pud_clear_flags(pud, _PAGE_RW);
646
647 /*
648 * Blindly clearing _PAGE_RW might accidentally create
649 * a shadow stack PUD (RW=0, Dirty=1). Move the hardware
650 * dirty value to the software bit.
651 */
652 return pud_mksaveddirty(pud);
491}653}
492 654
493static inline pud_t pud_mkdirty(pud_t pud)655static inline pud_t pud_mkdirty(pud_t pud)
494{656{
495 return pud_set_flags(pud, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);657 pud = pud_set_flags(pud, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
658
659 return pud_mksaveddirty(pud);
496}660}
497 661
498static inline pud_t pud_mkdevmap(pud_t pud)662static inline pud_t pud_mkdevmap(pud_t pud)
@@ -512,7 +676,9 @@ static inline pud_t pud_mkyoung(pud_t pud) · 13 unmodified lines
512 676
513static inline pud_t pud_mkwrite(pud_t pud)677static inline pud_t pud_mkwrite(pud_t pud)
514{678{
515 return pud_set_flags(pud, _PAGE_RW);679 pud = pud_set_flags(pud, _PAGE_RW);
680
681 return pud_clear_saveddirty(pud);
516}682}
517 683
518#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY684#ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
@@ -629,6 +795,7 @@ static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask); · 110 unmodified lines
629static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)795static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
630{796{
631 pteval_t val = pte_val(pte), oldval = val;797 pteval_t val = pte_val(pte), oldval = val;
798 pte_t pte_result;
632 799
633 /*800 /*
634 * Chop off the NX bit (if present), and add the NX portion of801 * Chop off the NX bit (if present), and add the NX portion of
@@ -637,17 +804,54 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) · 2 unmodified lines
637 val &= _PAGE_CHG_MASK;804 val &= _PAGE_CHG_MASK;
638 val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK;805 val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK;
639 val = flip_protnone_guard(oldval, val, PTE_PFN_MASK);806 val = flip_protnone_guard(oldval, val, PTE_PFN_MASK);
640 return __pte(val);807
808 pte_result = __pte(val);
809
810 /*
811 * To avoid creating Write=0,Dirty=1 PTEs, pte_modify() needs to avoid:
812 * 1. Marking Write=0 PTEs Dirty=1
813 * 2. Marking Dirty=1 PTEs Write=0
814 *
815 * The first case cannot happen because the _PAGE_CHG_MASK will filter
816 * out any Dirty bit passed in newprot. Handle the second case by
817 * going through the mksaveddirty exercise. Only do this if the old
818 * value was Write=1 to avoid doing this on Shadow Stack PTEs.
819 */
820 if (oldval & _PAGE_RW)
821 pte_result = pte_mksaveddirty(pte_result);
822 else
823 pte_result = pte_clear_saveddirty(pte_result);
824
825 return pte_result;
641}826}
642 827
643static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)828static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
644{829{
645 pmdval_t val = pmd_val(pmd), oldval = val;830 pmdval_t val = pmd_val(pmd), oldval = val;
831 pmd_t pmd_result;
646 832
647 val &= _HPAGE_CHG_MASK;833 val &= (_HPAGE_CHG_MASK & ~_PAGE_DIRTY);
648 val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;834 val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
649 val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);835 val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);
650 return __pmd(val);836
837 pmd_result = __pmd(val);
838
839 /*
840 * To avoid creating Write=0,Dirty=1 PMDs, pte_modify() needs to avoid:
841 * 1. Marking Write=0 PMDs Dirty=1
842 * 2. Marking Dirty=1 PMDs Write=0
843 *
844 * The first case cannot happen because the _PAGE_CHG_MASK will filter
845 * out any Dirty bit passed in newprot. Handle the second case by
846 * going through the mksaveddirty exercise. Only do this if the old
847 * value was Write=1 to avoid doing this on Shadow Stack PTEs.
848 */
849 if (oldval & _PAGE_RW)
850 pmd_result = pmd_mksaveddirty(pmd_result);
851 else
852 pmd_result = pmd_clear_saveddirty(pmd_result);
853
854 return pmd_result;
651}855}
652 856
653/*857/*
@@ -831,7 +1035,14 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) · 177 unmodified lines
831 * (Currently stuck as a macro because of indirect forward reference1035 * (Currently stuck as a macro because of indirect forward reference
832 * to linux/mm.h:page_to_nid())1036 * to linux/mm.h:page_to_nid())
833 */1037 */
834#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))1038#define mk_pte(page, pgprot) \
1039({ \
1040 pgprot_t __pgprot = pgprot; \
1041 \
1042 WARN_ON_ONCE((pgprot_val(__pgprot) & (_PAGE_DIRTY | _PAGE_RW)) == \
1043 _PAGE_DIRTY); \
1044 pfn_pte(page_to_pfn(page), __pgprot); \
1045})
835 1046
836static inline int pmd_bad(pmd_t pmd)1047static inline int pmd_bad(pmd_t pmd)
837{1048{
@@ -1090,7 +1301,17 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, · 252 unmodified lines
1090static inline void ptep_set_wrprotect(struct mm_struct *mm,1301static inline void ptep_set_wrprotect(struct mm_struct *mm,
1091 unsigned long addr, pte_t *ptep)1302 unsigned long addr, pte_t *ptep)
1092{1303{
1093 clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);1304 /*
1305 * Avoid accidentally creating shadow stack PTEs
1306 * (Write=0,Dirty=1). Use cmpxchg() to prevent races with
1307 * the hardware setting Dirty=1.
1308 */
1309 pte_t old_pte, new_pte;
1310
1311 old_pte = READ_ONCE(*ptep);
1312 do {
1313 new_pte = pte_wrprotect(old_pte);
1314 } while (!try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, *(long *)&new_pte));
1094}1315}
1095 1316
1096#define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0)1317#define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0)
@@ -1116,12 +1337,6 @@ extern int pmdp_clear_flush_young(struct vm_area_struct *vma, · 19 unmodified lines
1116 unsigned long address, pmd_t *pmdp);1337 unsigned long address, pmd_t *pmdp);
1117 1338
1118 1339
1119#define pmd_write pmd_write
1120static inline int pmd_write(pmd_t pmd)
1121{
1122 return pmd_flags(pmd) & _PAGE_RW;
1123}
1124
1125#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR1340#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
1126static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr,1341static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr,
1127 pmd_t *pmdp)1342 pmd_t *pmdp)
@@ -1148,13 +1363,17 @@ static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, · 20 unmodified lines
1148static inline void pmdp_set_wrprotect(struct mm_struct *mm,1363static inline void pmdp_set_wrprotect(struct mm_struct *mm,
1149 unsigned long addr, pmd_t *pmdp)1364 unsigned long addr, pmd_t *pmdp)
1150{1365{
1151 clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);1366 /*
1152}1367 * Avoid accidentally creating shadow stack PTEs
1368 * (Write=0,Dirty=1). Use cmpxchg() to prevent races with
1369 * the hardware setting Dirty=1.
1370 */
1371 pmd_t old_pmd, new_pmd;
1153 1372
1154#define pud_write pud_write1373 old_pmd = READ_ONCE(*pmdp);
1155static inline int pud_write(pud_t pud)1374 do {
1156{1375 new_pmd = pmd_wrprotect(old_pmd);
1157 return pud_flags(pud) & _PAGE_RW;1376 } while (!try_cmpxchg((long *)pmdp, (long *)&old_pmd, *(long *)&new_pmd));
1158}1377}
1159 1378
1160#ifndef pmdp_establish1379#ifndef pmdp_establish
@@ -1412,6 +1631,11 @@ static inline bool __pte_access_permitted(unsigned long pteval, bool write) · 251 unmodified lines
1412{1631{
1413 unsigned long need_pte_bits = _PAGE_PRESENT|_PAGE_USER;1632 unsigned long need_pte_bits = _PAGE_PRESENT|_PAGE_USER;
1414 1633
1634 /*
1635 * Write=0,Dirty=1 PTEs are shadow stack, which the kernel
1636 * shouldn't generally allow access to, but since they
1637 * are already Write=0, the below logic covers both cases.
1638 */
1415 if (write)1639 if (write)
1416 need_pte_bits |= _PAGE_RW;1640 need_pte_bits |= _PAGE_RW;
1417 1641
@@ -1453,6 +1677,12 @@ static inline bool arch_has_hw_pte_young(void) · 35 unmodified lines
1453 return true;1677 return true;
1454}1678}
1455 1679
1680#define arch_check_zapped_pte arch_check_zapped_pte
1681void arch_check_zapped_pte(struct vm_area_struct *vma, pte_t pte);
1682
1683#define arch_check_zapped_pmd arch_check_zapped_pmd
1684void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd);
1685
1456#ifdef CONFIG_XEN_PV1686#ifdef CONFIG_XEN_PV
1457#define arch_has_hw_nonleaf_pmd_young arch_has_hw_nonleaf_pmd_young1687#define arch_has_hw_nonleaf_pmd_young arch_has_hw_nonleaf_pmd_young
1458static inline bool arch_has_hw_nonleaf_pmd_young(void)1688static inline bool arch_has_hw_nonleaf_pmd_young(void)
arch/x86/include/asm/pgtable_types.h+36−8
@@ -21,7 +21,8 @@ · 20 unmodified lines
21#define _PAGE_BIT_SOFTW2 10 /* " */21#define _PAGE_BIT_SOFTW2 10 /* " */
22#define _PAGE_BIT_SOFTW3 11 /* " */22#define _PAGE_BIT_SOFTW3 11 /* " */
23#define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */23#define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */
24#define _PAGE_BIT_SOFTW4 58 /* available for programmer */24#define _PAGE_BIT_SOFTW4 57 /* available for programmer */
25#define _PAGE_BIT_SOFTW5 58 /* available for programmer */
25#define _PAGE_BIT_PKEY_BIT0 59 /* Protection Keys, bit 1/4 */26#define _PAGE_BIT_PKEY_BIT0 59 /* Protection Keys, bit 1/4 */
26#define _PAGE_BIT_PKEY_BIT1 60 /* Protection Keys, bit 2/4 */27#define _PAGE_BIT_PKEY_BIT1 60 /* Protection Keys, bit 2/4 */
27#define _PAGE_BIT_PKEY_BIT2 61 /* Protection Keys, bit 3/4 */28#define _PAGE_BIT_PKEY_BIT2 61 /* Protection Keys, bit 3/4 */
@@ -34,6 +35,13 @@ · 6 unmodified lines
34#define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */35#define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */
35#define _PAGE_BIT_DEVMAP _PAGE_BIT_SOFTW436#define _PAGE_BIT_DEVMAP _PAGE_BIT_SOFTW4
36 37
38#ifdef CONFIG_X86_64
39#define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW5 /* Saved Dirty bit */
40#else
41/* Shared with _PAGE_BIT_UFFD_WP which is not supported on 32 bit */
42#define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW2 /* Saved Dirty bit */
43#endif
44
37/* If _PAGE_BIT_PRESENT is clear, we use these: */45/* If _PAGE_BIT_PRESENT is clear, we use these: */
38/* - if the user mapped it with PROT_NONE; pte_present gives true */46/* - if the user mapped it with PROT_NONE; pte_present gives true */
39#define _PAGE_BIT_PROTNONE _PAGE_BIT_GLOBAL47#define _PAGE_BIT_PROTNONE _PAGE_BIT_GLOBAL
@@ -117,6 +125,18 @@ · 77 unmodified lines
117#define _PAGE_SOFTW4 (_AT(pteval_t, 0))125#define _PAGE_SOFTW4 (_AT(pteval_t, 0))
118#endif126#endif
119 127
128/*
129 * The hardware requires shadow stack to be Write=0,Dirty=1. However,
130 * there are valid cases where the kernel might create read-only PTEs that
131 * are dirty (e.g., fork(), mprotect(), uffd-wp(), soft-dirty tracking). In
132 * this case, the _PAGE_SAVED_DIRTY bit is used instead of the HW-dirty bit,
133 * to avoid creating a wrong "shadow stack" PTEs. Such PTEs have
134 * (Write=0,SavedDirty=1,Dirty=0) set.
135 */
136#define _PAGE_SAVED_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_SAVED_DIRTY)
137
138#define _PAGE_DIRTY_BITS (_PAGE_DIRTY | _PAGE_SAVED_DIRTY)
139
120#define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE)140#define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE)
121 141
122/*142/*
@@ -125,10 +145,10 @@ · 2 unmodified lines
125 * instance, and is *not* included in this mask since145 * instance, and is *not* included in this mask since
126 * pte_modify() does modify it.146 * pte_modify() does modify it.
127 */147 */
128#define _COMMON_PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \148#define _COMMON_PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \
129 _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY |\149 _PAGE_SPECIAL | _PAGE_ACCESSED | \
130 _PAGE_SOFT_DIRTY | _PAGE_DEVMAP | _PAGE_ENC | \150 _PAGE_DIRTY_BITS | _PAGE_SOFT_DIRTY | \
131 _PAGE_UFFD_WP)151 _PAGE_DEVMAP | _PAGE_ENC | _PAGE_UFFD_WP)
132#define _PAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PAT)152#define _PAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PAT)
133#define _HPAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_PAT_LARGE)153#define _HPAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_PAT_LARGE)
134 154
@@ -189,14 +209,22 @@ enum page_cache_mode { · 54 unmodified lines
189 209
190#define __PAGE_KERNEL (__PP|__RW| 0|___A|__NX|___D| 0|___G)210#define __PAGE_KERNEL (__PP|__RW| 0|___A|__NX|___D| 0|___G)
191#define __PAGE_KERNEL_EXEC (__PP|__RW| 0|___A| 0|___D| 0|___G)211#define __PAGE_KERNEL_EXEC (__PP|__RW| 0|___A| 0|___D| 0|___G)
212
213/*
214 * Page tables needs to have Write=1 in order for any lower PTEs to be
215 * writable. This includes shadow stack memory (Write=0, Dirty=1)
216 */
192#define _KERNPG_TABLE_NOENC (__PP|__RW| 0|___A| 0|___D| 0| 0)217#define _KERNPG_TABLE_NOENC (__PP|__RW| 0|___A| 0|___D| 0| 0)
193#define _KERNPG_TABLE (__PP|__RW| 0|___A| 0|___D| 0| 0| _ENC)218#define _KERNPG_TABLE (__PP|__RW| 0|___A| 0|___D| 0| 0| _ENC)
194#define _PAGE_TABLE_NOENC (__PP|__RW|_USR|___A| 0|___D| 0| 0)219#define _PAGE_TABLE_NOENC (__PP|__RW|_USR|___A| 0|___D| 0| 0)
195#define _PAGE_TABLE (__PP|__RW|_USR|___A| 0|___D| 0| 0| _ENC)220#define _PAGE_TABLE (__PP|__RW|_USR|___A| 0|___D| 0| 0| _ENC)
196#define __PAGE_KERNEL_RO (__PP| 0| 0|___A|__NX|___D| 0|___G)221
197#define __PAGE_KERNEL_ROX (__PP| 0| 0|___A| 0|___D| 0|___G)222#define __PAGE_KERNEL_RO (__PP| 0| 0|___A|__NX| 0| 0|___G)
223#define __PAGE_KERNEL_ROX (__PP| 0| 0|___A| 0| 0| 0|___G)
224#define __PAGE_KERNEL (__PP|__RW| 0|___A|__NX|___D| 0|___G)
225#define __PAGE_KERNEL_EXEC (__PP|__RW| 0|___A| 0|___D| 0|___G)
198#define __PAGE_KERNEL_NOCACHE (__PP|__RW| 0|___A|__NX|___D| 0|___G| __NC)226#define __PAGE_KERNEL_NOCACHE (__PP|__RW| 0|___A|__NX|___D| 0|___G| __NC)
199#define __PAGE_KERNEL_VVAR (__PP| 0|_USR|___A|__NX|___D| 0|___G)227#define __PAGE_KERNEL_VVAR (__PP| 0|_USR|___A|__NX| 0| 0|___G)
200#define __PAGE_KERNEL_LARGE (__PP|__RW| 0|___A|__NX|___D|_PSE|___G)228#define __PAGE_KERNEL_LARGE (__PP|__RW| 0|___A|__NX|___D|_PSE|___G)
201#define __PAGE_KERNEL_LARGE_EXEC (__PP|__RW| 0|___A| 0|___D|_PSE|___G)229#define __PAGE_KERNEL_LARGE_EXEC (__PP|__RW| 0|___A| 0|___D|_PSE|___G)
202#define __PAGE_KERNEL_WP (__PP|__RW| 0|___A|__NX|___D| 0|___G| __WP)230#define __PAGE_KERNEL_WP (__PP|__RW| 0|___A|__NX|___D| 0|___G| __WP)
arch/x86/include/asm/processor.h+8−0
@@ -28,6 +28,7 @@ struct vm86; · 27 unmodified lines
28#include <asm/unwind_hints.h>28#include <asm/unwind_hints.h>
29#include <asm/vmxfeatures.h>29#include <asm/vmxfeatures.h>
30#include <asm/vdso/processor.h>30#include <asm/vdso/processor.h>
31#include <asm/shstk.h>
31 32
32#include <linux/personality.h>33#include <linux/personality.h>
33#include <linux/cache.h>34#include <linux/cache.h>
@@ -474,6 +475,13 @@ struct thread_struct { · 440 unmodified lines
474 */475 */
475 u32 pkru;476 u32 pkru;
476 477
478#ifdef CONFIG_X86_USER_SHADOW_STACK
479 unsigned long features;
480 unsigned long features_locked;
481
482 struct thread_shstk shstk;
483#endif
484
477 /* Floating point and extended processor state */485 /* Floating point and extended processor state */
478 struct fpu fpu;486 struct fpu fpu;
479 /*487 /*
arch/x86/include/asm/shstk.h+38−0
@@ -0,0 +1,38 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_SHSTK_H
3#define _ASM_X86_SHSTK_H
4
5#ifndef __ASSEMBLY__
6#include <linux/types.h>
7
8struct task_struct;
9struct ksignal;
10
11#ifdef CONFIG_X86_USER_SHADOW_STACK
12struct thread_shstk {
13 u64 base;
14 u64 size;
15};
16
17long shstk_prctl(struct task_struct *task, int option, unsigned long arg2);
18void reset_thread_features(void);
19unsigned long shstk_alloc_thread_stack(struct task_struct *p, unsigned long clone_flags,
20 unsigned long stack_size);
21void shstk_free(struct task_struct *p);
22int setup_signal_shadow_stack(struct ksignal *ksig);
23int restore_signal_shadow_stack(void);
24#else
25static inline long shstk_prctl(struct task_struct *task, int option,
26 unsigned long arg2) { return -EINVAL; }
27static inline void reset_thread_features(void) {}
28static inline unsigned long shstk_alloc_thread_stack(struct task_struct *p,
29 unsigned long clone_flags,
30 unsigned long stack_size) { return 0; }
31static inline void shstk_free(struct task_struct *p) {}
32static inline int setup_signal_shadow_stack(struct ksignal *ksig) { return 0; }
33static inline int restore_signal_shadow_stack(void) { return 0; }
34#endif /* CONFIG_X86_USER_SHADOW_STACK */
35
36#endif /* __ASSEMBLY__ */
37
38#endif /* _ASM_X86_SHSTK_H */
arch/x86/include/asm/special_insns.h+13−0
@@ -202,6 +202,19 @@ static inline void clwb(volatile void *__p) · 201 unmodified lines
202 : [pax] "a" (p));202 : [pax] "a" (p));
203}203}
204 204
205#ifdef CONFIG_X86_USER_SHADOW_STACK
206static inline int write_user_shstk_64(u64 __user *addr, u64 val)
207{
208 asm_volatile_goto("1: wrussq %[val], (%[addr])\n"
209 _ASM_EXTABLE(1b, %l[fail])
210 :: [addr] "r" (addr), [val] "r" (val)
211 :: fail);
212 return 0;
213fail:
214 return -EFAULT;
215}
216#endif /* CONFIG_X86_USER_SHADOW_STACK */
217
205#define nop() asm volatile ("nop")218#define nop() asm volatile ("nop")
206 219
207static inline void serialize(void)220static inline void serialize(void)
arch/x86/include/asm/tlbflush.h+2−1
@@ -306,7 +306,8 @@ static inline bool pte_flags_need_flush(unsigned long oldflags, · 305 unmodified lines
306 const pteval_t flush_on_clear = _PAGE_DIRTY | _PAGE_PRESENT |306 const pteval_t flush_on_clear = _PAGE_DIRTY | _PAGE_PRESENT |
307 _PAGE_ACCESSED;307 _PAGE_ACCESSED;
308 const pteval_t software_flags = _PAGE_SOFTW1 | _PAGE_SOFTW2 |308 const pteval_t software_flags = _PAGE_SOFTW1 | _PAGE_SOFTW2 |
309 _PAGE_SOFTW3 | _PAGE_SOFTW4;309 _PAGE_SOFTW3 | _PAGE_SOFTW4 |
310 _PAGE_SAVED_DIRTY;
310 const pteval_t flush_on_change = _PAGE_RW | _PAGE_USER | _PAGE_PWT |311 const pteval_t flush_on_change = _PAGE_RW | _PAGE_USER | _PAGE_PWT |
311 _PAGE_PCD | _PAGE_PSE | _PAGE_GLOBAL | _PAGE_PAT |312 _PAGE_PCD | _PAGE_PSE | _PAGE_GLOBAL | _PAGE_PAT |
312 _PAGE_PAT_LARGE | _PAGE_PKEY_BIT0 | _PAGE_PKEY_BIT1 |313 _PAGE_PAT_LARGE | _PAGE_PKEY_BIT0 | _PAGE_PKEY_BIT1 |
arch/x86/include/asm/trap_pf.h+2−0
@@ -11,6 +11,7 @@ · 10 unmodified lines
11 * bit 3 == 1: use of reserved bit detected11 * bit 3 == 1: use of reserved bit detected
12 * bit 4 == 1: fault was an instruction fetch12 * bit 4 == 1: fault was an instruction fetch
13 * bit 5 == 1: protection keys block access13 * bit 5 == 1: protection keys block access
14 * bit 6 == 1: shadow stack access fault
14 * bit 15 == 1: SGX MMU page-fault15 * bit 15 == 1: SGX MMU page-fault
15 */16 */
16enum x86_pf_error_code {17enum x86_pf_error_code {
@@ -20,6 +21,7 @@ enum x86_pf_error_code { · 3 unmodified lines
20 X86_PF_RSVD = 1 << 3,21 X86_PF_RSVD = 1 << 3,
21 X86_PF_INSTR = 1 << 4,22 X86_PF_INSTR = 1 << 4,
22 X86_PF_PK = 1 << 5,23 X86_PF_PK = 1 << 5,
24 X86_PF_SHSTK = 1 << 6,
23 X86_PF_SGX = 1 << 15,25 X86_PF_SGX = 1 << 15,
24};26};
25 27
arch/x86/include/asm/traps.h+14−1
@@ -18,7 +18,8 @@ void __init trap_init(void); · 17 unmodified lines
18asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *eregs);18asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *eregs);
19#endif19#endif
20 20
21extern bool ibt_selftest(void);21extern int ibt_selftest(void);
22extern int ibt_selftest_noendbr(void);
22 23
23#ifdef CONFIG_X86_F00F_BUG24#ifdef CONFIG_X86_F00F_BUG
24/* For handling the FOOF bug */25/* For handling the FOOF bug */
@@ -47,4 +48,16 @@ void __noreturn handle_stack_overflow(struct pt_regs *regs, · 22 unmodified lines
47 struct stack_info *info);48 struct stack_info *info);
48#endif49#endif
49 50
51static inline void cond_local_irq_enable(struct pt_regs *regs)
52{
53 if (regs->flags & X86_EFLAGS_IF)
54 local_irq_enable();
55}
56
57static inline void cond_local_irq_disable(struct pt_regs *regs)
58{
59 if (regs->flags & X86_EFLAGS_IF)
60 local_irq_disable();
61}
62
50#endif /* _ASM_X86_TRAPS_H */63#endif /* _ASM_X86_TRAPS_H */
arch/x86/include/uapi/asm/mman.h+4−0
@@ -3,6 +3,7 @@ · 2 unmodified lines
3#define _ASM_X86_MMAN_H3#define _ASM_X86_MMAN_H
4 4
5#define MAP_32BIT 0x40 /* only give out 32bit addresses */5#define MAP_32BIT 0x40 /* only give out 32bit addresses */
6#define MAP_ABOVE4G 0x80 /* only map above 4GB */
6 7
7#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS8#ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
8#define arch_calc_vm_prot_bits(prot, key) ( \9#define arch_calc_vm_prot_bits(prot, key) ( \
@@ -12,6 +13,9 @@ · 3 unmodified lines
12 ((key) & 0x8 ? VM_PKEY_BIT3 : 0))13 ((key) & 0x8 ? VM_PKEY_BIT3 : 0))
13#endif14#endif
14 15
16/* Flags for map_shadow_stack(2) */
17#define SHADOW_STACK_SET_TOKEN (1ULL << 0) /* Set up a restore token in the shadow stack */
18
15#include <asm-generic/mman.h>19#include <asm-generic/mman.h>
16 20
17#endif /* _ASM_X86_MMAN_H */21#endif /* _ASM_X86_MMAN_H */
arch/x86/include/uapi/asm/prctl.h+12−0
@@ -23,9 +23,21 @@ · 22 unmodified lines
23#define ARCH_MAP_VDSO_32 0x200223#define ARCH_MAP_VDSO_32 0x2002
24#define ARCH_MAP_VDSO_64 0x200324#define ARCH_MAP_VDSO_64 0x2003
25 25
26/* Don't use 0x3001-0x3004 because of old glibcs */
27
26#define ARCH_GET_UNTAG_MASK 0x400128#define ARCH_GET_UNTAG_MASK 0x4001
27#define ARCH_ENABLE_TAGGED_ADDR 0x400229#define ARCH_ENABLE_TAGGED_ADDR 0x4002
28#define ARCH_GET_MAX_TAG_BITS 0x400330#define ARCH_GET_MAX_TAG_BITS 0x4003
29#define ARCH_FORCE_TAGGED_SVA 0x400431#define ARCH_FORCE_TAGGED_SVA 0x4004
30 32
33#define ARCH_SHSTK_ENABLE 0x5001
34#define ARCH_SHSTK_DISABLE 0x5002
35#define ARCH_SHSTK_LOCK 0x5003
36#define ARCH_SHSTK_UNLOCK 0x5004
37#define ARCH_SHSTK_STATUS 0x5005
38
39/* ARCH_SHSTK_ features bits */
40#define ARCH_SHSTK_SHSTK (1ULL << 0)
41#define ARCH_SHSTK_WRSS (1ULL << 1)
42
31#endif /* _ASM_X86_PRCTL_H */43#endif /* _ASM_X86_PRCTL_H */
arch/x86/kernel/Makefile+5−0
@@ -48,6 +48,7 @@ obj-y += process_$(BITS).o signal.o signal_$(BITS).o · 47 unmodified lines
48obj-y += traps.o idt.o irq.o irq_$(BITS).o dumpstack_$(BITS).o48obj-y += traps.o idt.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
49obj-y += time.o ioport.o dumpstack.o nmi.o49obj-y += time.o ioport.o dumpstack.o nmi.o
50obj-$(CONFIG_MODIFY_LDT_SYSCALL) += ldt.o50obj-$(CONFIG_MODIFY_LDT_SYSCALL) += ldt.o
51obj-$(CONFIG_X86_KERNEL_IBT) += ibt_selftest.o
51obj-y += setup.o x86_init.o i8259.o irqinit.o52obj-y += setup.o x86_init.o i8259.o irqinit.o
52obj-$(CONFIG_JUMP_LABEL) += jump_label.o53obj-$(CONFIG_JUMP_LABEL) += jump_label.o
53obj-$(CONFIG_IRQ_WORK) += irq_work.o54obj-$(CONFIG_IRQ_WORK) += irq_work.o
@@ -144,6 +145,10 @@ obj-$(CONFIG_CFI_CLANG) += cfi.o · 90 unmodified lines
144 145
145obj-$(CONFIG_CALL_THUNKS) += callthunks.o146obj-$(CONFIG_CALL_THUNKS) += callthunks.o
146 147
148obj-$(CONFIG_X86_CET) += cet.o
149
150obj-$(CONFIG_X86_USER_SHADOW_STACK) += shstk.o
151
147###152###
148# 64 bit specific files153# 64 bit specific files
149ifeq ($(CONFIG_X86_64),y)154ifeq ($(CONFIG_X86_64),y)
arch/x86/kernel/cet.c+131−0
@@ -0,0 +1,131 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/ptrace.h>
4#include <asm/bugs.h>
5#include <asm/traps.h>
6
7enum cp_error_code {
8 CP_EC = (1 << 15) - 1,
9
10 CP_RET = 1,
11 CP_IRET = 2,
12 CP_ENDBR = 3,
13 CP_RSTRORSSP = 4,
14 CP_SETSSBSY = 5,
15
16 CP_ENCL = 1 << 15,
17};
18
19static const char cp_err[][10] = {
20 [0] = "unknown",
21 [1] = "near ret",
22 [2] = "far/iret",
23 [3] = "endbranch",
24 [4] = "rstorssp",
25 [5] = "setssbsy",
26};
27
28static const char *cp_err_string(unsigned long error_code)
29{
30 unsigned int cpec = error_code & CP_EC;
31
32 if (cpec >= ARRAY_SIZE(cp_err))
33 cpec = 0;
34 return cp_err[cpec];
35}
36
37static void do_unexpected_cp(struct pt_regs *regs, unsigned long error_code)
38{
39 WARN_ONCE(1, "Unexpected %s #CP, error_code: %s\n",
40 user_mode(regs) ? "user mode" : "kernel mode",
41 cp_err_string(error_code));
42}
43
44static DEFINE_RATELIMIT_STATE(cpf_rate, DEFAULT_RATELIMIT_INTERVAL,
45 DEFAULT_RATELIMIT_BURST);
46
47static void do_user_cp_fault(struct pt_regs *regs, unsigned long error_code)
48{
49 struct task_struct *tsk;
50 unsigned long ssp;
51
52 /*
53 * An exception was just taken from userspace. Since interrupts are disabled
54 * here, no scheduling should have messed with the registers yet and they
55 * will be whatever is live in userspace. So read the SSP before enabling
56 * interrupts so locking the fpregs to do it later is not required.
57 */
58 rdmsrl(MSR_IA32_PL3_SSP, ssp);
59
60 cond_local_irq_enable(regs);
61
62 tsk = current;
63 tsk->thread.error_code = error_code;
64 tsk->thread.trap_nr = X86_TRAP_CP;
65
66 /* Ratelimit to prevent log spamming. */
67 if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
68 __ratelimit(&cpf_rate)) {
69 pr_emerg("%s[%d] control protection ip:%lx sp:%lx ssp:%lx error:%lx(%s)%s",
70 tsk->comm, task_pid_nr(tsk),
71 regs->ip, regs->sp, ssp, error_code,
72 cp_err_string(error_code),
73 error_code & CP_ENCL ? " in enclave" : "");
74 print_vma_addr(KERN_CONT " in ", regs->ip);
75 pr_cont("\n");
76 }
77
78 force_sig_fault(SIGSEGV, SEGV_CPERR, (void __user *)0);
79 cond_local_irq_disable(regs);
80}
81
82static __ro_after_init bool ibt_fatal = true;
83
84static void do_kernel_cp_fault(struct pt_regs *regs, unsigned long error_code)
85{
86 if ((error_code & CP_EC) != CP_ENDBR) {
87 do_unexpected_cp(regs, error_code);
88 return;
89 }
90
91 if (unlikely(regs->ip == (unsigned long)&ibt_selftest_noendbr)) {
92 regs->ax = 0;
93 return;
94 }
95
96 pr_err("Missing ENDBR: %pS\n", (void *)instruction_pointer(regs));
97 if (!ibt_fatal) {
98 printk(KERN_DEFAULT CUT_HERE);
99 __warn(__FILE__, __LINE__, (void *)regs->ip, TAINT_WARN, regs, NULL);
100 return;
101 }
102 BUG();
103}
104
105static int __init ibt_setup(char *str)
106{
107 if (!strcmp(str, "off"))
108 setup_clear_cpu_cap(X86_FEATURE_IBT);
109
110 if (!strcmp(str, "warn"))
111 ibt_fatal = false;
112
113 return 1;
114}
115
116__setup("ibt=", ibt_setup);
117
118DEFINE_IDTENTRY_ERRORCODE(exc_control_protection)
119{
120 if (user_mode(regs)) {
121 if (cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
122 do_user_cp_fault(regs, error_code);
123 else
124 do_unexpected_cp(regs, error_code);
125 } else {
126 if (cpu_feature_enabled(X86_FEATURE_IBT))
127 do_kernel_cp_fault(regs, error_code);
128 else
129 do_unexpected_cp(regs, error_code);
130 }
131}
arch/x86/kernel/cpu/common.c+27−8
@@ -587,27 +587,43 @@ __noendbr void ibt_restore(u64 save) · 586 unmodified lines
587 587
588static __always_inline void setup_cet(struct cpuinfo_x86 *c)588static __always_inline void setup_cet(struct cpuinfo_x86 *c)
589{589{
590 u64 msr = CET_ENDBR_EN;590 bool user_shstk, kernel_ibt;
591 591
592 if (!HAS_KERNEL_IBT ||592 if (!IS_ENABLED(CONFIG_X86_CET))
593 !cpu_feature_enabled(X86_FEATURE_IBT))
594 return;593 return;
595 594
596 wrmsrl(MSR_IA32_S_CET, msr);595 kernel_ibt = HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT);
596 user_shstk = cpu_feature_enabled(X86_FEATURE_SHSTK) &&
597 IS_ENABLED(CONFIG_X86_USER_SHADOW_STACK);
598
599 if (!kernel_ibt && !user_shstk)
600 return;
601
602 if (user_shstk)
603 set_cpu_cap(c, X86_FEATURE_USER_SHSTK);
604
605 if (kernel_ibt)
606 wrmsrl(MSR_IA32_S_CET, CET_ENDBR_EN);
607 else
608 wrmsrl(MSR_IA32_S_CET, 0);
609
597 cr4_set_bits(X86_CR4_CET);610 cr4_set_bits(X86_CR4_CET);
598 611
599 if (!ibt_selftest()) {612 if (kernel_ibt && ibt_selftest()) {
600 pr_err("IBT selftest: Failed!\n");613 pr_err("IBT selftest: Failed!\n");
601 wrmsrl(MSR_IA32_S_CET, 0);614 wrmsrl(MSR_IA32_S_CET, 0);
602 setup_clear_cpu_cap(X86_FEATURE_IBT);615 setup_clear_cpu_cap(X86_FEATURE_IBT);
603 return;
604 }616 }
605}617}
606 618
607__noendbr void cet_disable(void)619__noendbr void cet_disable(void)
608{620{
609 if (cpu_feature_enabled(X86_FEATURE_IBT))621 if (!(cpu_feature_enabled(X86_FEATURE_IBT) ||
610 wrmsrl(MSR_IA32_S_CET, 0);622 cpu_feature_enabled(X86_FEATURE_SHSTK)))
623 return;
624
625 wrmsrl(MSR_IA32_S_CET, 0);
626 wrmsrl(MSR_IA32_U_CET, 0);
611}627}
612 628
613/*629/*
@@ -1491,6 +1507,9 @@ static void __init cpu_parse_early_param(void) · 877 unmodified lines
1491 if (cmdline_find_option_bool(boot_command_line, "noxsaves"))1507 if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
1492 setup_clear_cpu_cap(X86_FEATURE_XSAVES);1508 setup_clear_cpu_cap(X86_FEATURE_XSAVES);
1493 1509
1510 if (cmdline_find_option_bool(boot_command_line, "nousershstk"))
1511 setup_clear_cpu_cap(X86_FEATURE_USER_SHSTK);
1512
1494 arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));1513 arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));
1495 if (arglen <= 0)1514 if (arglen <= 0)
1496 return;1515 return;
arch/x86/kernel/cpu/cpuid-deps.c+1−0
@@ -81,6 +81,7 @@ static const struct cpuid_dep cpuid_deps[] = { · 80 unmodified lines
81 { X86_FEATURE_XFD, X86_FEATURE_XSAVES },81 { X86_FEATURE_XFD, X86_FEATURE_XSAVES },
82 { X86_FEATURE_XFD, X86_FEATURE_XGETBV1 },82 { X86_FEATURE_XFD, X86_FEATURE_XGETBV1 },
83 { X86_FEATURE_AMX_TILE, X86_FEATURE_XFD },83 { X86_FEATURE_AMX_TILE, X86_FEATURE_XFD },
84 { X86_FEATURE_SHSTK, X86_FEATURE_XSAVES },
84 {}85 {}
85};86};
86 87
arch/x86/kernel/cpu/proc.c+23−0
@@ -4,6 +4,8 @@ · 3 unmodified lines
4#include <linux/string.h>4#include <linux/string.h>
5#include <linux/seq_file.h>5#include <linux/seq_file.h>
6#include <linux/cpufreq.h>6#include <linux/cpufreq.h>
7#include <asm/prctl.h>
8#include <linux/proc_fs.h>
7 9
8#include "cpu.h"10#include "cpu.h"
9 11
@@ -175,3 +177,24 @@ const struct seq_operations cpuinfo_op = { · 165 unmodified lines
175 .stop = c_stop,177 .stop = c_stop,
176 .show = show_cpuinfo,178 .show = show_cpuinfo,
177};179};
180
181#ifdef CONFIG_X86_USER_SHADOW_STACK
182static void dump_x86_features(struct seq_file *m, unsigned long features)
183{
184 if (features & ARCH_SHSTK_SHSTK)
185 seq_puts(m, "shstk ");
186 if (features & ARCH_SHSTK_WRSS)
187 seq_puts(m, "wrss ");
188}
189
190void arch_proc_pid_thread_features(struct seq_file *m, struct task_struct *task)
191{
192 seq_puts(m, "x86_Thread_features:\t");
193 dump_x86_features(m, task->thread.features);
194 seq_putc(m, '\n');
195
196 seq_puts(m, "x86_Thread_features_locked:\t");
197 dump_x86_features(m, task->thread.features_locked);
198 seq_putc(m, '\n');
199}
200#endif /* CONFIG_X86_USER_SHADOW_STACK */
arch/x86/kernel/fpu/core.c+53−1
@@ -552,8 +552,36 @@ static inline void fpu_inherit_perms(struct fpu *dst_fpu) · 551 unmodified lines
552 }552 }
553}553}
554 554
555/* A passed ssp of zero will not cause any update */
556static int update_fpu_shstk(struct task_struct *dst, unsigned long ssp)
557{
558#ifdef CONFIG_X86_USER_SHADOW_STACK
559 struct cet_user_state *xstate;
560
561 /* If ssp update is not needed. */
562 if (!ssp)
563 return 0;
564
565 xstate = get_xsave_addr(&dst->thread.fpu.fpstate->regs.xsave,
566 XFEATURE_CET_USER);
567
568 /*
569 * If there is a non-zero ssp, then 'dst' must be configured with a shadow
570 * stack and the fpu state should be up to date since it was just copied
571 * from the parent in fpu_clone(). So there must be a valid non-init CET
572 * state location in the buffer.
573 */
574 if (WARN_ON_ONCE(!xstate))
575 return 1;
576
577 xstate->user_ssp = (u64)ssp;
578#endif
579 return 0;
580}
581
555/* Clone current's FPU state on fork */582/* Clone current's FPU state on fork */
556int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal)583int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal,
584 unsigned long ssp)
557{585{
558 struct fpu *src_fpu = &current->thread.fpu;586 struct fpu *src_fpu = &current->thread.fpu;
559 struct fpu *dst_fpu = &dst->thread.fpu;587 struct fpu *dst_fpu = &dst->thread.fpu;
@@ -613,6 +641,12 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal) · 53 unmodified lines
613 if (use_xsave())641 if (use_xsave())
614 dst_fpu->fpstate->regs.xsave.header.xfeatures &= ~XFEATURE_MASK_PASID;642 dst_fpu->fpstate->regs.xsave.header.xfeatures &= ~XFEATURE_MASK_PASID;
615 643
644 /*
645 * Update shadow stack pointer, in case it changed during clone.
646 */
647 if (update_fpu_shstk(dst, ssp))
648 return 1;
649
616 trace_x86_fpu_copy_src(src_fpu);650 trace_x86_fpu_copy_src(src_fpu);
617 trace_x86_fpu_copy_dst(dst_fpu);651 trace_x86_fpu_copy_dst(dst_fpu);
618 652
@@ -753,6 +787,24 @@ void switch_fpu_return(void) · 134 unmodified lines
753}787}
754EXPORT_SYMBOL_GPL(switch_fpu_return);788EXPORT_SYMBOL_GPL(switch_fpu_return);
755 789
790void fpregs_lock_and_load(void)
791{
792 /*
793 * fpregs_lock() only disables preemption (mostly). So modifying state
794 * in an interrupt could screw up some in progress fpregs operation.
795 * Warn about it.
796 */
797 WARN_ON_ONCE(!irq_fpu_usable());
798 WARN_ON_ONCE(current->flags & PF_KTHREAD);
799
800 fpregs_lock();
801
802 fpregs_assert_state_consistent();
803
804 if (test_thread_flag(TIF_NEED_FPU_LOAD))
805 fpregs_restore_userregs();
806}
807
756#ifdef CONFIG_X86_DEBUG_FPU808#ifdef CONFIG_X86_DEBUG_FPU
757/*809/*
758 * If current FPU state according to its tracking (loaded FPU context on this810 * If current FPU state according to its tracking (loaded FPU context on this
arch/x86/kernel/fpu/regset.c+81−0
@@ -8,6 +8,7 @@ · 7 unmodified lines
8#include <asm/fpu/api.h>8#include <asm/fpu/api.h>
9#include <asm/fpu/signal.h>9#include <asm/fpu/signal.h>
10#include <asm/fpu/regset.h>10#include <asm/fpu/regset.h>
11#include <asm/prctl.h>
11 12
12#include "context.h"13#include "context.h"
13#include "internal.h"14#include "internal.h"
@@ -174,6 +175,86 @@ out: · 160 unmodified lines
174 return ret;175 return ret;
175}176}
176 177
178#ifdef CONFIG_X86_USER_SHADOW_STACK
179int ssp_active(struct task_struct *target, const struct user_regset *regset)
180{
181 if (target->thread.features & ARCH_SHSTK_SHSTK)
182 return regset->n;
183
184 return 0;
185}
186
187int ssp_get(struct task_struct *target, const struct user_regset *regset,
188 struct membuf to)
189{
190 struct fpu *fpu = &target->thread.fpu;
191 struct cet_user_state *cetregs;
192
193 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
194 return -ENODEV;
195
196 sync_fpstate(fpu);
197 cetregs = get_xsave_addr(&fpu->fpstate->regs.xsave, XFEATURE_CET_USER);
198 if (WARN_ON(!cetregs)) {
199 /*
200 * This shouldn't ever be NULL because shadow stack was
201 * verified to be enabled above. This means
202 * MSR_IA32_U_CET.CET_SHSTK_EN should be 1 and so
203 * XFEATURE_CET_USER should not be in the init state.
204 */
205 return -ENODEV;
206 }
207
208 return membuf_write(&to, (unsigned long *)&cetregs->user_ssp,
209 sizeof(cetregs->user_ssp));
210}
211
212int ssp_set(struct task_struct *target, const struct user_regset *regset,
213 unsigned int pos, unsigned int count,
214 const void *kbuf, const void __user *ubuf)
215{
216 struct fpu *fpu = &target->thread.fpu;
217 struct xregs_state *xsave = &fpu->fpstate->regs.xsave;
218 struct cet_user_state *cetregs;
219 unsigned long user_ssp;
220 int r;
221
222 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) ||
223 !ssp_active(target, regset))
224 return -ENODEV;
225
226 if (pos != 0 || count != sizeof(user_ssp))
227 return -EINVAL;
228
229 r = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &user_ssp, 0, -1);
230 if (r)
231 return r;
232
233 /*
234 * Some kernel instructions (IRET, etc) can cause exceptions in the case
235 * of disallowed CET register values. Just prevent invalid values.
236 */
237 if (user_ssp >= TASK_SIZE_MAX || !IS_ALIGNED(user_ssp, 8))
238 return -EINVAL;
239
240 fpu_force_restore(fpu);
241
242 cetregs = get_xsave_addr(xsave, XFEATURE_CET_USER);
243 if (WARN_ON(!cetregs)) {
244 /*
245 * This shouldn't ever be NULL because shadow stack was
246 * verified to be enabled above. This means
247 * MSR_IA32_U_CET.CET_SHSTK_EN should be 1 and so
248 * XFEATURE_CET_USER should not be in the init state.
249 */
250 return -ENODEV;
251 }
252
253 cetregs->user_ssp = user_ssp;
254 return 0;
255}
256#endif /* CONFIG_X86_USER_SHADOW_STACK */
257
177#if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION258#if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
178 259
179/*260/*
arch/x86/kernel/fpu/xstate.c+43−47
@@ -39,26 +39,26 @@ · 38 unmodified lines
39 */39 */
40static const char *xfeature_names[] =40static const char *xfeature_names[] =
41{41{
42 "x87 floating point registers" ,42 "x87 floating point registers",
43 "SSE registers" ,43 "SSE registers",
44 "AVX registers" ,44 "AVX registers",
45 "MPX bounds registers" ,45 "MPX bounds registers",
46 "MPX CSR" ,46 "MPX CSR",
47 "AVX-512 opmask" ,47 "AVX-512 opmask",
48 "AVX-512 Hi256" ,48 "AVX-512 Hi256",
49 "AVX-512 ZMM_Hi256" ,49 "AVX-512 ZMM_Hi256",
50 "Processor Trace (unused)" ,50 "Processor Trace (unused)",
51 "Protection Keys User registers",51 "Protection Keys User registers",
52 "PASID state",52 "PASID state",
53 "unknown xstate feature" ,53 "Control-flow User registers",
54 "unknown xstate feature" ,54 "Control-flow Kernel registers (unused)",
55 "unknown xstate feature" ,55 "unknown xstate feature",
56 "unknown xstate feature" ,56 "unknown xstate feature",
57 "unknown xstate feature" ,57 "unknown xstate feature",
58 "unknown xstate feature" ,58 "unknown xstate feature",
59 "AMX Tile config" ,59 "AMX Tile config",
60 "AMX Tile data" ,60 "AMX Tile data",
61 "unknown xstate feature" ,61 "unknown xstate feature",
62};62};
63 63
64static unsigned short xsave_cpuid_features[] __initdata = {64static unsigned short xsave_cpuid_features[] __initdata = {
@@ -73,6 +73,7 @@ static unsigned short xsave_cpuid_features[] __initdata = { · 8 unmodified lines
73 [XFEATURE_PT_UNIMPLEMENTED_SO_FAR] = X86_FEATURE_INTEL_PT,73 [XFEATURE_PT_UNIMPLEMENTED_SO_FAR] = X86_FEATURE_INTEL_PT,
74 [XFEATURE_PKRU] = X86_FEATURE_PKU,74 [XFEATURE_PKRU] = X86_FEATURE_PKU,
75 [XFEATURE_PASID] = X86_FEATURE_ENQCMD,75 [XFEATURE_PASID] = X86_FEATURE_ENQCMD,
76 [XFEATURE_CET_USER] = X86_FEATURE_SHSTK,
76 [XFEATURE_XTILE_CFG] = X86_FEATURE_AMX_TILE,77 [XFEATURE_XTILE_CFG] = X86_FEATURE_AMX_TILE,
77 [XFEATURE_XTILE_DATA] = X86_FEATURE_AMX_TILE,78 [XFEATURE_XTILE_DATA] = X86_FEATURE_AMX_TILE,
78};79};
@@ -276,6 +277,7 @@ static void __init print_xstate_features(void) · 197 unmodified lines
276 print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);277 print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
277 print_xstate_feature(XFEATURE_MASK_PKRU);278 print_xstate_feature(XFEATURE_MASK_PKRU);
278 print_xstate_feature(XFEATURE_MASK_PASID);279 print_xstate_feature(XFEATURE_MASK_PASID);
280 print_xstate_feature(XFEATURE_MASK_CET_USER);
279 print_xstate_feature(XFEATURE_MASK_XTILE_CFG);281 print_xstate_feature(XFEATURE_MASK_XTILE_CFG);
280 print_xstate_feature(XFEATURE_MASK_XTILE_DATA);282 print_xstate_feature(XFEATURE_MASK_XTILE_DATA);
281}283}
@@ -344,6 +346,7 @@ static __init void os_xrstor_booting(struct xregs_state *xstate) · 62 unmodified lines
344 XFEATURE_MASK_BNDREGS | \346 XFEATURE_MASK_BNDREGS | \
345 XFEATURE_MASK_BNDCSR | \347 XFEATURE_MASK_BNDCSR | \
346 XFEATURE_MASK_PASID | \348 XFEATURE_MASK_PASID | \
349 XFEATURE_MASK_CET_USER | \
347 XFEATURE_MASK_XTILE)350 XFEATURE_MASK_XTILE)
348 351
349/*352/*
@@ -446,14 +449,15 @@ static void __init __xstate_dump_leaves(void) · 96 unmodified lines
446 } \449 } \
447} while (0)450} while (0)
448 451
449#define XCHECK_SZ(sz, nr, nr_macro, __struct) do { \452#define XCHECK_SZ(sz, nr, __struct) ({ \
450 if ((nr == nr_macro) && \453 if (WARN_ONCE(sz != sizeof(__struct), \
451 WARN_ONCE(sz != sizeof(__struct), \454 "[%s]: struct is %zu bytes, cpu state %d bytes\n", \
452 "%s: struct is %zu bytes, cpu state %d bytes\n", \455 xfeature_names[nr], sizeof(__struct), sz)) { \
453 __stringify(nr_macro), sizeof(__struct), sz)) { \
454 __xstate_dump_leaves(); \456 __xstate_dump_leaves(); \
455 } \457 } \
456} while (0)458 true; \
459})
460
457 461
458/**462/**
459 * check_xtile_data_against_struct - Check tile data state size.463 * check_xtile_data_against_struct - Check tile data state size.
@@ -527,36 +531,28 @@ static bool __init check_xstate_against_struct(int nr) · 67 unmodified lines
527 * Ask the CPU for the size of the state.531 * Ask the CPU for the size of the state.
528 */532 */
529 int sz = xfeature_size(nr);533 int sz = xfeature_size(nr);
534
530 /*535 /*
531 * Match each CPU state with the corresponding software536 * Match each CPU state with the corresponding software
532 * structure.537 * structure.
533 */538 */
534 XCHECK_SZ(sz, nr, XFEATURE_YMM, struct ymmh_struct);539 switch (nr) {
535 XCHECK_SZ(sz, nr, XFEATURE_BNDREGS, struct mpx_bndreg_state);540 case XFEATURE_YMM: return XCHECK_SZ(sz, nr, struct ymmh_struct);
536 XCHECK_SZ(sz, nr, XFEATURE_BNDCSR, struct mpx_bndcsr_state);541 case XFEATURE_BNDREGS: return XCHECK_SZ(sz, nr, struct mpx_bndreg_state);
537 XCHECK_SZ(sz, nr, XFEATURE_OPMASK, struct avx_512_opmask_state);542 case XFEATURE_BNDCSR: return XCHECK_SZ(sz, nr, struct mpx_bndcsr_state);
538 XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state);543 case XFEATURE_OPMASK: return XCHECK_SZ(sz, nr, struct avx_512_opmask_state);
539 XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM, struct avx_512_hi16_state);544 case XFEATURE_ZMM_Hi256: return XCHECK_SZ(sz, nr, struct avx_512_zmm_uppers_state);
540 XCHECK_SZ(sz, nr, XFEATURE_PKRU, struct pkru_state);545 case XFEATURE_Hi16_ZMM: return XCHECK_SZ(sz, nr, struct avx_512_hi16_state);
541 XCHECK_SZ(sz, nr, XFEATURE_PASID, struct ia32_pasid_state);546 case XFEATURE_PKRU: return XCHECK_SZ(sz, nr, struct pkru_state);
542 XCHECK_SZ(sz, nr, XFEATURE_XTILE_CFG, struct xtile_cfg);547 case XFEATURE_PASID: return XCHECK_SZ(sz, nr, struct ia32_pasid_state);
543 548 case XFEATURE_XTILE_CFG: return XCHECK_SZ(sz, nr, struct xtile_cfg);
544 /* The tile data size varies between implementations. */549 case XFEATURE_CET_USER: return XCHECK_SZ(sz, nr, struct cet_user_state);
545 if (nr == XFEATURE_XTILE_DATA)550 case XFEATURE_XTILE_DATA: check_xtile_data_against_struct(sz); return true;
546 check_xtile_data_against_struct(sz);551 default:
547
548 /*
549 * Make *SURE* to add any feature numbers in below if
550 * there are "holes" in the xsave state component
551 * numbers.
552 */
553 if ((nr < XFEATURE_YMM) ||
554 (nr >= XFEATURE_MAX) ||
555 (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) ||
556 ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_RSRVD_COMP_16))) {
557 XSTATE_WARN_ON(1, "No structure for xstate: %d\n", nr);552 XSTATE_WARN_ON(1, "No structure for xstate: %d\n", nr);
558 return false;553 return false;
559 }554 }
555
560 return true;556 return true;
561}557}
562 558
arch/x86/kernel/ibt_selftest.S+17−0
@@ -0,0 +1,17 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#include <linux/linkage.h>
3#include <linux/objtool.h>
4#include <asm/nospec-branch.h>
5
6SYM_CODE_START(ibt_selftest_noendbr)
7 ANNOTATE_NOENDBR
8 UNWIND_HINT_FUNC
9 /* #CP handler sets %ax to 0 */
10 RET
11SYM_CODE_END(ibt_selftest_noendbr)
12
13SYM_FUNC_START(ibt_selftest)
14 lea ibt_selftest_noendbr(%rip), %rax
15 ANNOTATE_RETPOLINE_SAFE
16 jmp *%rax
17SYM_FUNC_END(ibt_selftest)
arch/x86/kernel/idt.c+1−1
@@ -107,7 +107,7 @@ static const __initconst struct idt_data def_idts[] = { · 106 unmodified lines
107 ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE),107 ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE),
108#endif108#endif
109 109
110#ifdef CONFIG_X86_KERNEL_IBT110#ifdef CONFIG_X86_CET
111 INTG(X86_TRAP_CP, asm_exc_control_protection),111 INTG(X86_TRAP_CP, asm_exc_control_protection),
112#endif112#endif
113 113
arch/x86/kernel/process.c+20−1
@@ -51,6 +51,7 @@ · 50 unmodified lines
51#include <asm/unwind.h>51#include <asm/unwind.h>
52#include <asm/tdx.h>52#include <asm/tdx.h>
53#include <asm/mmu_context.h>53#include <asm/mmu_context.h>
54#include <asm/shstk.h>
54 55
55#include "process.h"56#include "process.h"
56 57
@@ -122,6 +123,7 @@ void exit_thread(struct task_struct *tsk) · 65 unmodified lines
122 123
123 free_vm86(t);124 free_vm86(t);
124 125
126 shstk_free(tsk);
125 fpu__drop(fpu);127 fpu__drop(fpu);
126}128}
127 129
@@ -162,6 +164,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) · 34 unmodified lines
162 struct inactive_task_frame *frame;164 struct inactive_task_frame *frame;
163 struct fork_frame *fork_frame;165 struct fork_frame *fork_frame;
164 struct pt_regs *childregs;166 struct pt_regs *childregs;
167 unsigned long new_ssp;
165 int ret = 0;168 int ret = 0;
166 169
167 childregs = task_pt_regs(p);170 childregs = task_pt_regs(p);
@@ -199,7 +202,16 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) · 31 unmodified lines
199 frame->flags = X86_EFLAGS_FIXED;202 frame->flags = X86_EFLAGS_FIXED;
200#endif203#endif
201 204
202 fpu_clone(p, clone_flags, args->fn);205 /*
206 * Allocate a new shadow stack for thread if needed. If shadow stack,
207 * is disabled, new_ssp will remain 0, and fpu_clone() will know not to
208 * update it.
209 */
210 new_ssp = shstk_alloc_thread_stack(p, clone_flags, args->stack_size);
211 if (IS_ERR_VALUE(new_ssp))
212 return PTR_ERR((void *)new_ssp);
213
214 fpu_clone(p, clone_flags, args->fn, new_ssp);
203 215
204 /* Kernel thread ? */216 /* Kernel thread ? */
205 if (unlikely(p->flags & PF_KTHREAD)) {217 if (unlikely(p->flags & PF_KTHREAD)) {
@@ -245,6 +257,13 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) · 39 unmodified lines
245 if (!ret && unlikely(test_tsk_thread_flag(current, TIF_IO_BITMAP)))257 if (!ret && unlikely(test_tsk_thread_flag(current, TIF_IO_BITMAP)))
246 io_bitmap_share(p);258 io_bitmap_share(p);
247 259
260 /*
261 * If copy_thread() if failing, don't leak the shadow stack possibly
262 * allocated in shstk_alloc_thread_stack() above.
263 */
264 if (ret)
265 shstk_free(p);
266
248 return ret;267 return ret;
249}268}
250 269
arch/x86/kernel/process_64.c+8−0
@@ -515,6 +515,8 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip, · 514 unmodified lines
515 load_gs_index(__USER_DS);515 load_gs_index(__USER_DS);
516 }516 }
517 517
518 reset_thread_features();
519
518 loadsegment(fs, 0);520 loadsegment(fs, 0);
519 loadsegment(es, _ds);521 loadsegment(es, _ds);
520 loadsegment(ds, _ds);522 loadsegment(ds, _ds);
@@ -894,6 +896,12 @@ long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2) · 373 unmodified lines
894 else896 else
895 return put_user(LAM_U57_BITS, (unsigned long __user *)arg2);897 return put_user(LAM_U57_BITS, (unsigned long __user *)arg2);
896#endif898#endif
899 case ARCH_SHSTK_ENABLE:
900 case ARCH_SHSTK_DISABLE:
901 case ARCH_SHSTK_LOCK:
902 case ARCH_SHSTK_UNLOCK:
903 case ARCH_SHSTK_STATUS:
904 return shstk_prctl(task, option, arg2);
897 default:905 default:
898 ret = -EINVAL;906 ret = -EINVAL;
899 break;907 break;
arch/x86/kernel/ptrace.c+12−0
@@ -58,6 +58,7 @@ enum x86_regset_64 { · 57 unmodified lines
58 REGSET64_FP,58 REGSET64_FP,
59 REGSET64_IOPERM,59 REGSET64_IOPERM,
60 REGSET64_XSTATE,60 REGSET64_XSTATE,
61 REGSET64_SSP,
61};62};
62 63
63#define REGSET_GENERAL \64#define REGSET_GENERAL \
@@ -1267,6 +1268,17 @@ static struct user_regset x86_64_regsets[] __ro_after_init = { · 1203 unmodified lines
1267 .active = ioperm_active,1268 .active = ioperm_active,
1268 .regset_get = ioperm_get1269 .regset_get = ioperm_get
1269 },1270 },
1271#ifdef CONFIG_X86_USER_SHADOW_STACK
1272 [REGSET64_SSP] = {
1273 .core_note_type = NT_X86_SHSTK,
1274 .n = 1,
1275 .size = sizeof(u64),
1276 .align = sizeof(u64),
1277 .active = ssp_active,
1278 .regset_get = ssp_get,
1279 .set = ssp_set
1280 },
1281#endif
1270};1282};
1271 1283
1272static const struct user_regset_view user_x86_64_view = {1284static const struct user_regset_view user_x86_64_view = {
arch/x86/kernel/shstk.c+550−0
@@ -0,0 +1,550 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * shstk.c - Intel shadow stack support
4 *
5 * Copyright (c) 2021, Intel Corporation.
6 * Yu-cheng Yu <yu-cheng.yu@intel.com>
7 */
8
9#include <linux/sched.h>
10#include <linux/bitops.h>
11#include <linux/types.h>
12#include <linux/mm.h>
13#include <linux/mman.h>
14#include <linux/slab.h>
15#include <linux/uaccess.h>
16#include <linux/sched/signal.h>
17#include <linux/compat.h>
18#include <linux/sizes.h>
19#include <linux/user.h>
20#include <linux/syscalls.h>
21#include <asm/msr.h>
22#include <asm/fpu/xstate.h>
23#include <asm/fpu/types.h>
24#include <asm/shstk.h>
25#include <asm/special_insns.h>
26#include <asm/fpu/api.h>
27#include <asm/prctl.h>
28
29#define SS_FRAME_SIZE 8
30
31static bool features_enabled(unsigned long features)
32{
33 return current->thread.features & features;
34}
35
36static void features_set(unsigned long features)
37{
38 current->thread.features |= features;
39}
40
41static void features_clr(unsigned long features)
42{
43 current->thread.features &= ~features;
44}
45
46/*
47 * Create a restore token on the shadow stack. A token is always 8-byte
48 * and aligned to 8.
49 */
50static int create_rstor_token(unsigned long ssp, unsigned long *token_addr)
51{
52 unsigned long addr;
53
54 /* Token must be aligned */
55 if (!IS_ALIGNED(ssp, 8))
56 return -EINVAL;
57
58 addr = ssp - SS_FRAME_SIZE;
59
60 /*
61 * SSP is aligned, so reserved bits and mode bit are a zero, just mark
62 * the token 64-bit.
63 */
64 ssp |= BIT(0);
65
66 if (write_user_shstk_64((u64 __user *)addr, (u64)ssp))
67 return -EFAULT;
68
69 if (token_addr)
70 *token_addr = addr;
71
72 return 0;
73}
74
75/*
76 * VM_SHADOW_STACK will have a guard page. This helps userspace protect
77 * itself from attacks. The reasoning is as follows:
78 *
79 * The shadow stack pointer(SSP) is moved by CALL, RET, and INCSSPQ. The
80 * INCSSP instruction can increment the shadow stack pointer. It is the
81 * shadow stack analog of an instruction like:
82 *
83 * addq $0x80, %rsp
84 *
85 * However, there is one important difference between an ADD on %rsp
86 * and INCSSP. In addition to modifying SSP, INCSSP also reads from the
87 * memory of the first and last elements that were "popped". It can be
88 * thought of as acting like this:
89 *
90 * READ_ONCE(ssp); // read+discard top element on stack
91 * ssp += nr_to_pop * 8; // move the shadow stack
92 * READ_ONCE(ssp-8); // read+discard last popped stack element
93 *
94 * The maximum distance INCSSP can move the SSP is 2040 bytes, before
95 * it would read the memory. Therefore a single page gap will be enough
96 * to prevent any operation from shifting the SSP to an adjacent stack,
97 * since it would have to land in the gap at least once, causing a
98 * fault.
99 */
100static unsigned long alloc_shstk(unsigned long addr, unsigned long size,
101 unsigned long token_offset, bool set_res_tok)
102{
103 int flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_ABOVE4G;
104 struct mm_struct *mm = current->mm;
105 unsigned long mapped_addr, unused;
106
107 if (addr)
108 flags |= MAP_FIXED_NOREPLACE;
109
110 mmap_write_lock(mm);
111 mapped_addr = do_mmap(NULL, addr, size, PROT_READ, flags,
112 VM_SHADOW_STACK | VM_WRITE, 0, &unused, NULL);
113 mmap_write_unlock(mm);
114
115 if (!set_res_tok || IS_ERR_VALUE(mapped_addr))
116 goto out;
117
118 if (create_rstor_token(mapped_addr + token_offset, NULL)) {
119 vm_munmap(mapped_addr, size);
120 return -EINVAL;
121 }
122
123out:
124 return mapped_addr;
125}
126
127static unsigned long adjust_shstk_size(unsigned long size)
128{
129 if (size)
130 return PAGE_ALIGN(size);
131
132 return PAGE_ALIGN(min_t(unsigned long long, rlimit(RLIMIT_STACK), SZ_4G));
133}
134
135static void unmap_shadow_stack(u64 base, u64 size)
136{
137 int r;
138
139 r = vm_munmap(base, size);
140
141 /*
142 * mmap_write_lock_killable() failed with -EINTR. This means
143 * the process is about to die and have it's MM cleaned up.
144 * This task shouldn't ever make it back to userspace. In this
145 * case it is ok to leak a shadow stack, so just exit out.
146 */
147 if (r == -EINTR)
148 return;
149
150 /*
151 * For all other types of vm_munmap() failure, either the
152 * system is out of memory or there is bug.
153 */
154 WARN_ON_ONCE(r);
155}
156
157static int shstk_setup(void)
158{
159 struct thread_shstk *shstk = &current->thread.shstk;
160 unsigned long addr, size;
161
162 /* Already enabled */
163 if (features_enabled(ARCH_SHSTK_SHSTK))
164 return 0;
165
166 /* Also not supported for 32 bit and x32 */
167 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) || in_32bit_syscall())
168 return -EOPNOTSUPP;
169
170 size = adjust_shstk_size(0);
171 addr = alloc_shstk(0, size, 0, false);
172 if (IS_ERR_VALUE(addr))
173 return PTR_ERR((void *)addr);
174
175 fpregs_lock_and_load();
176 wrmsrl(MSR_IA32_PL3_SSP, addr + size);
177 wrmsrl(MSR_IA32_U_CET, CET_SHSTK_EN);
178 fpregs_unlock();
179
180 shstk->base = addr;
181 shstk->size = size;
182 features_set(ARCH_SHSTK_SHSTK);
183
184 return 0;
185}
186
187void reset_thread_features(void)
188{
189 memset(&current->thread.shstk, 0, sizeof(struct thread_shstk));
190 current->thread.features = 0;
191 current->thread.features_locked = 0;
192}
193
194unsigned long shstk_alloc_thread_stack(struct task_struct *tsk, unsigned long clone_flags,
195 unsigned long stack_size)
196{
197 struct thread_shstk *shstk = &tsk->thread.shstk;
198 unsigned long addr, size;
199
200 /*
201 * If shadow stack is not enabled on the new thread, skip any
202 * switch to a new shadow stack.
203 */
204 if (!features_enabled(ARCH_SHSTK_SHSTK))
205 return 0;
206
207 /*
208 * For CLONE_VM, except vfork, the child needs a separate shadow
209 * stack.
210 */
211 if ((clone_flags & (CLONE_VFORK | CLONE_VM)) != CLONE_VM)
212 return 0;
213
214 size = adjust_shstk_size(stack_size);
215 addr = alloc_shstk(0, size, 0, false);
216 if (IS_ERR_VALUE(addr))
217 return addr;
218
219 shstk->base = addr;
220 shstk->size = size;
221
222 return addr + size;
223}
224
225static unsigned long get_user_shstk_addr(void)
226{
227 unsigned long long ssp;
228
229 fpregs_lock_and_load();
230
231 rdmsrl(MSR_IA32_PL3_SSP, ssp);
232
233 fpregs_unlock();
234
235 return ssp;
236}
237
238#define SHSTK_DATA_BIT BIT(63)
239
240static int put_shstk_data(u64 __user *addr, u64 data)
241{
242 if (WARN_ON_ONCE(data & SHSTK_DATA_BIT))
243 return -EINVAL;
244
245 /*
246 * Mark the high bit so that the sigframe can't be processed as a
247 * return address.
248 */
249 if (write_user_shstk_64(addr, data | SHSTK_DATA_BIT))
250 return -EFAULT;
251 return 0;
252}
253
254static int get_shstk_data(unsigned long *data, unsigned long __user *addr)
255{
256 unsigned long ldata;
257
258 if (unlikely(get_user(ldata, addr)))
259 return -EFAULT;
260
261 if (!(ldata & SHSTK_DATA_BIT))
262 return -EINVAL;
263
264 *data = ldata & ~SHSTK_DATA_BIT;
265
266 return 0;
267}
268
269static int shstk_push_sigframe(unsigned long *ssp)
270{
271 unsigned long target_ssp = *ssp;
272
273 /* Token must be aligned */
274 if (!IS_ALIGNED(target_ssp, 8))
275 return -EINVAL;
276
277 *ssp -= SS_FRAME_SIZE;
278 if (put_shstk_data((void __user *)*ssp, target_ssp))
279 return -EFAULT;
280
281 return 0;
282}
283
284static int shstk_pop_sigframe(unsigned long *ssp)
285{
286 struct vm_area_struct *vma;
287 unsigned long token_addr;
288 bool need_to_check_vma;
289 int err = 1;
290
291 /*
292 * It is possible for the SSP to be off the end of a shadow stack by 4
293 * or 8 bytes. If the shadow stack is at the start of a page or 4 bytes
294 * before it, it might be this case, so check that the address being
295 * read is actually shadow stack.
296 */
297 if (!IS_ALIGNED(*ssp, 8))
298 return -EINVAL;
299
300 need_to_check_vma = PAGE_ALIGN(*ssp) == *ssp;
301
302 if (need_to_check_vma)
303 mmap_read_lock_killable(current->mm);
304
305 err = get_shstk_data(&token_addr, (unsigned long __user *)*ssp);
306 if (unlikely(err))
307 goto out_err;
308
309 if (need_to_check_vma) {
310 vma = find_vma(current->mm, *ssp);
311 if (!vma || !(vma->vm_flags & VM_SHADOW_STACK)) {
312 err = -EFAULT;
313 goto out_err;
314 }
315
316 mmap_read_unlock(current->mm);
317 }
318
319 /* Restore SSP aligned? */
320 if (unlikely(!IS_ALIGNED(token_addr, 8)))
321 return -EINVAL;
322
323 /* SSP in userspace? */
324 if (unlikely(token_addr >= TASK_SIZE_MAX))
325 return -EINVAL;
326
327 *ssp = token_addr;
328
329 return 0;
330out_err:
331 if (need_to_check_vma)
332 mmap_read_unlock(current->mm);
333 return err;
334}
335
336int setup_signal_shadow_stack(struct ksignal *ksig)
337{
338 void __user *restorer = ksig->ka.sa.sa_restorer;
339 unsigned long ssp;
340 int err;
341
342 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) ||
343 !features_enabled(ARCH_SHSTK_SHSTK))
344 return 0;
345
346 if (!restorer)
347 return -EINVAL;
348
349 ssp = get_user_shstk_addr();
350 if (unlikely(!ssp))
351 return -EINVAL;
352
353 err = shstk_push_sigframe(&ssp);
354 if (unlikely(err))
355 return err;
356
357 /* Push restorer address */
358 ssp -= SS_FRAME_SIZE;
359 err = write_user_shstk_64((u64 __user *)ssp, (u64)restorer);
360 if (unlikely(err))
361 return -EFAULT;
362
363 fpregs_lock_and_load();
364 wrmsrl(MSR_IA32_PL3_SSP, ssp);
365 fpregs_unlock();
366
367 return 0;
368}
369
370int restore_signal_shadow_stack(void)
371{
372 unsigned long ssp;
373 int err;
374
375 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) ||
376 !features_enabled(ARCH_SHSTK_SHSTK))
377 return 0;
378
379 ssp = get_user_shstk_addr();
380 if (unlikely(!ssp))
381 return -EINVAL;
382
383 err = shstk_pop_sigframe(&ssp);
384 if (unlikely(err))
385 return err;
386
387 fpregs_lock_and_load();
388 wrmsrl(MSR_IA32_PL3_SSP, ssp);
389 fpregs_unlock();
390
391 return 0;
392}
393
394void shstk_free(struct task_struct *tsk)
395{
396 struct thread_shstk *shstk = &tsk->thread.shstk;
397
398 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) ||
399 !features_enabled(ARCH_SHSTK_SHSTK))
400 return;
401
402 /*
403 * When fork() with CLONE_VM fails, the child (tsk) already has a
404 * shadow stack allocated, and exit_thread() calls this function to
405 * free it. In this case the parent (current) and the child share
406 * the same mm struct.
407 */
408 if (!tsk->mm || tsk->mm != current->mm)
409 return;
410
411 unmap_shadow_stack(shstk->base, shstk->size);
412}
413
414static int wrss_control(bool enable)
415{
416 u64 msrval;
417
418 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
419 return -EOPNOTSUPP;
420
421 /*
422 * Only enable WRSS if shadow stack is enabled. If shadow stack is not
423 * enabled, WRSS will already be disabled, so don't bother clearing it
424 * when disabling.
425 */
426 if (!features_enabled(ARCH_SHSTK_SHSTK))
427 return -EPERM;
428
429 /* Already enabled/disabled? */
430 if (features_enabled(ARCH_SHSTK_WRSS) == enable)
431 return 0;
432
433 fpregs_lock_and_load();
434 rdmsrl(MSR_IA32_U_CET, msrval);
435
436 if (enable) {
437 features_set(ARCH_SHSTK_WRSS);
438 msrval |= CET_WRSS_EN;
439 } else {
440 features_clr(ARCH_SHSTK_WRSS);
441 if (!(msrval & CET_WRSS_EN))
442 goto unlock;
443
444 msrval &= ~CET_WRSS_EN;
445 }
446
447 wrmsrl(MSR_IA32_U_CET, msrval);
448
449unlock:
450 fpregs_unlock();
451
452 return 0;
453}
454
455static int shstk_disable(void)
456{
457 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
458 return -EOPNOTSUPP;
459
460 /* Already disabled? */
461 if (!features_enabled(ARCH_SHSTK_SHSTK))
462 return 0;
463
464 fpregs_lock_and_load();
465 /* Disable WRSS too when disabling shadow stack */
466 wrmsrl(MSR_IA32_U_CET, 0);
467 wrmsrl(MSR_IA32_PL3_SSP, 0);
468 fpregs_unlock();
469
470 shstk_free(current);
471 features_clr(ARCH_SHSTK_SHSTK | ARCH_SHSTK_WRSS);
472
473 return 0;
474}
475
476SYSCALL_DEFINE3(map_shadow_stack, unsigned long, addr, unsigned long, size, unsigned int, flags)
477{
478 bool set_tok = flags & SHADOW_STACK_SET_TOKEN;
479 unsigned long aligned_size;
480
481 if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
482 return -EOPNOTSUPP;
483
484 if (flags & ~SHADOW_STACK_SET_TOKEN)
485 return -EINVAL;
486
487 /* If there isn't space for a token */
488 if (set_tok && size < 8)
489 return -ENOSPC;
490
491 if (addr && addr < SZ_4G)
492 return -ERANGE;
493
494 /*
495 * An overflow would result in attempting to write the restore token
496 * to the wrong location. Not catastrophic, but just return the right
497 * error code and block it.
498 */
499 aligned_size = PAGE_ALIGN(size);
500 if (aligned_size < size)
501 return -EOVERFLOW;
502
503 return alloc_shstk(addr, aligned_size, size, set_tok);
504}
505
506long shstk_prctl(struct task_struct *task, int option, unsigned long arg2)
507{
508 unsigned long features = arg2;
509
510 if (option == ARCH_SHSTK_STATUS) {
511 return put_user(task->thread.features, (unsigned long __user *)arg2);
512 }
513
514 if (option == ARCH_SHSTK_LOCK) {
515 task->thread.features_locked |= features;
516 return 0;
517 }
518
519 /* Only allow via ptrace */
520 if (task != current) {
521 if (option == ARCH_SHSTK_UNLOCK && IS_ENABLED(CONFIG_CHECKPOINT_RESTORE)) {
522 task->thread.features_locked &= ~features;
523 return 0;
524 }
525 return -EINVAL;
526 }
527
528 /* Do not allow to change locked features */
529 if (features & task->thread.features_locked)
530 return -EPERM;
531
532 /* Only support enabling/disabling one feature at a time. */
533 if (hweight_long(features) > 1)
534 return -EINVAL;
535
536 if (option == ARCH_SHSTK_DISABLE) {
537 if (features & ARCH_SHSTK_WRSS)
538 return wrss_control(false);
539 if (features & ARCH_SHSTK_SHSTK)
540 return shstk_disable();
541 return -EINVAL;
542 }
543
544 /* Handle ARCH_SHSTK_ENABLE */
545 if (features & ARCH_SHSTK_SHSTK)
546 return shstk_setup();
547 if (features & ARCH_SHSTK_WRSS)
548 return wrss_control(true);
549 return -EINVAL;
550}
arch/x86/kernel/signal.c+1−0
@@ -40,6 +40,7 @@ · 39 unmodified lines
40#include <asm/syscall.h>40#include <asm/syscall.h>
41#include <asm/sigframe.h>41#include <asm/sigframe.h>
42#include <asm/signal.h>42#include <asm/signal.h>
43#include <asm/shstk.h>
43 44
44static inline int is_ia32_compat_frame(struct ksignal *ksig)45static inline int is_ia32_compat_frame(struct ksignal *ksig)
45{46{
arch/x86/kernel/signal_32.c+1−1
@@ -402,7 +402,7 @@ Efault: · 401 unmodified lines
402*/402*/
403static_assert(NSIGILL == 11);403static_assert(NSIGILL == 11);
404static_assert(NSIGFPE == 15);404static_assert(NSIGFPE == 15);
405static_assert(NSIGSEGV == 9);405static_assert(NSIGSEGV == 10);
406static_assert(NSIGBUS == 5);406static_assert(NSIGBUS == 5);
407static_assert(NSIGTRAP == 6);407static_assert(NSIGTRAP == 6);
408static_assert(NSIGCHLD == 6);408static_assert(NSIGCHLD == 6);
arch/x86/kernel/signal_64.c+7−1
@@ -175,6 +175,9 @@ int x64_setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs) · 174 unmodified lines
175 frame = get_sigframe(ksig, regs, sizeof(struct rt_sigframe), &fp);175 frame = get_sigframe(ksig, regs, sizeof(struct rt_sigframe), &fp);
176 uc_flags = frame_uc_flags(regs);176 uc_flags = frame_uc_flags(regs);
177 177
178 if (setup_signal_shadow_stack(ksig))
179 return -EFAULT;
180
178 if (!user_access_begin(frame, sizeof(*frame)))181 if (!user_access_begin(frame, sizeof(*frame)))
179 return -EFAULT;182 return -EFAULT;
180 183
@@ -260,6 +263,9 @@ SYSCALL_DEFINE0(rt_sigreturn) · 79 unmodified lines
260 if (!restore_sigcontext(regs, &frame->uc.uc_mcontext, uc_flags))263 if (!restore_sigcontext(regs, &frame->uc.uc_mcontext, uc_flags))
261 goto badframe;264 goto badframe;
262 265
266 if (restore_signal_shadow_stack())
267 goto badframe;
268
263 if (restore_altstack(&frame->uc.uc_stack))269 if (restore_altstack(&frame->uc.uc_stack))
264 goto badframe;270 goto badframe;
265 271
@@ -403,7 +409,7 @@ void sigaction_compat_abi(struct k_sigaction *act, struct k_sigaction *oact) · 137 unmodified lines
403*/409*/
404static_assert(NSIGILL == 11);410static_assert(NSIGILL == 11);
405static_assert(NSIGFPE == 15);411static_assert(NSIGFPE == 15);
406static_assert(NSIGSEGV == 9);412static_assert(NSIGSEGV == 10);
407static_assert(NSIGBUS == 5);413static_assert(NSIGBUS == 5);
408static_assert(NSIGTRAP == 6);414static_assert(NSIGTRAP == 6);
409static_assert(NSIGCHLD == 6);415static_assert(NSIGCHLD == 6);
arch/x86/kernel/sys_x86_64.c+5−1
@@ -193,7 +193,11 @@ get_unmapped_area: · 192 unmodified lines
193 193
194 info.flags = VM_UNMAPPED_AREA_TOPDOWN;194 info.flags = VM_UNMAPPED_AREA_TOPDOWN;
195 info.length = len;195 info.length = len;
196 info.low_limit = PAGE_SIZE;196 if (!in_32bit_syscall() && (flags & MAP_ABOVE4G))
197 info.low_limit = SZ_4G;
198 else
199 info.low_limit = PAGE_SIZE;
200
197 info.high_limit = get_mmap_base(0);201 info.high_limit = get_mmap_base(0);
198 202
199 /*203 /*
arch/x86/kernel/traps.c+0−87
@@ -77,18 +77,6 @@ · 76 unmodified lines
77 77
78DECLARE_BITMAP(system_vectors, NR_VECTORS);78DECLARE_BITMAP(system_vectors, NR_VECTORS);
79 79
80static inline void cond_local_irq_enable(struct pt_regs *regs)
81{
82 if (regs->flags & X86_EFLAGS_IF)
83 local_irq_enable();
84}
85
86static inline void cond_local_irq_disable(struct pt_regs *regs)
87{
88 if (regs->flags & X86_EFLAGS_IF)
89 local_irq_disable();
90}
91
92__always_inline int is_valid_bugaddr(unsigned long addr)80__always_inline int is_valid_bugaddr(unsigned long addr)
93{81{
94 if (addr < TASK_SIZE_MAX)82 if (addr < TASK_SIZE_MAX)
@@ -213,81 +201,6 @@ DEFINE_IDTENTRY(exc_overflow) · 118 unmodified lines
213 do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL);201 do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL);
214}202}
215 203
216#ifdef CONFIG_X86_KERNEL_IBT
217
218static __ro_after_init bool ibt_fatal = true;
219
220extern void ibt_selftest_ip(void); /* code label defined in asm below */
221
222enum cp_error_code {
223 CP_EC = (1 << 15) - 1,
224
225 CP_RET = 1,
226 CP_IRET = 2,
227 CP_ENDBR = 3,
228 CP_RSTRORSSP = 4,
229 CP_SETSSBSY = 5,
230
231 CP_ENCL = 1 << 15,
232};
233
234DEFINE_IDTENTRY_ERRORCODE(exc_control_protection)
235{
236 if (!cpu_feature_enabled(X86_FEATURE_IBT)) {
237 pr_err("Unexpected #CP\n");
238 BUG();
239 }
240
241 if (WARN_ON_ONCE(user_mode(regs) || (error_code & CP_EC) != CP_ENDBR))
242 return;
243
244 if (unlikely(regs->ip == (unsigned long)&ibt_selftest_ip)) {
245 regs->ax = 0;
246 return;
247 }
248
249 pr_err("Missing ENDBR: %pS\n", (void *)instruction_pointer(regs));
250 if (!ibt_fatal) {
251 printk(KERN_DEFAULT CUT_HERE);
252 __warn(__FILE__, __LINE__, (void *)regs->ip, TAINT_WARN, regs, NULL);
253 return;
254 }
255 BUG();
256}
257
258/* Must be noinline to ensure uniqueness of ibt_selftest_ip. */
259noinline bool ibt_selftest(void)
260{
261 unsigned long ret;
262
263 asm (" lea ibt_selftest_ip(%%rip), %%rax\n\t"
264 ANNOTATE_RETPOLINE_SAFE
265 " jmp *%%rax\n\t"
266 "ibt_selftest_ip:\n\t"
267 UNWIND_HINT_FUNC
268 ANNOTATE_NOENDBR
269 " nop\n\t"
270
271 : "=a" (ret) : : "memory");
272
273 return !ret;
274}
275
276static int __init ibt_setup(char *str)
277{
278 if (!strcmp(str, "off"))
279 setup_clear_cpu_cap(X86_FEATURE_IBT);
280
281 if (!strcmp(str, "warn"))
282 ibt_fatal = false;
283
284 return 1;
285}
286
287__setup("ibt=", ibt_setup);
288
289#endif /* CONFIG_X86_KERNEL_IBT */
290
291#ifdef CONFIG_X86_F00F_BUG204#ifdef CONFIG_X86_F00F_BUG
292void handle_invalid_op(struct pt_regs *regs)205void handle_invalid_op(struct pt_regs *regs)
293#else206#else
arch/x86/mm/fault.c+22−0
@@ -1112,8 +1112,22 @@ access_error(unsigned long error_code, struct vm_area_struct *vma) · 1111 unmodified lines
1112 (error_code & X86_PF_INSTR), foreign))1112 (error_code & X86_PF_INSTR), foreign))
1113 return 1;1113 return 1;
1114 1114
1115 /*
1116 * Shadow stack accesses (PF_SHSTK=1) are only permitted to
1117 * shadow stack VMAs. All other accesses result in an error.
1118 */
1119 if (error_code & X86_PF_SHSTK) {
1120 if (unlikely(!(vma->vm_flags & VM_SHADOW_STACK)))
1121 return 1;
1122 if (unlikely(!(vma->vm_flags & VM_WRITE)))
1123 return 1;
1124 return 0;
1125 }
1126
1115 if (error_code & X86_PF_WRITE) {1127 if (error_code & X86_PF_WRITE) {
1116 /* write, present and write, not present: */1128 /* write, present and write, not present: */
1129 if (unlikely(vma->vm_flags & VM_SHADOW_STACK))
1130 return 1;
1117 if (unlikely(!(vma->vm_flags & VM_WRITE)))1131 if (unlikely(!(vma->vm_flags & VM_WRITE)))
1118 return 1;1132 return 1;
1119 return 0;1133 return 0;
@@ -1305,6 +1319,14 @@ void do_user_addr_fault(struct pt_regs *regs, · 185 unmodified lines
1305 1319
1306 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);1320 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
1307 1321
1322 /*
1323 * Read-only permissions can not be expressed in shadow stack PTEs.
1324 * Treat all shadow stack accesses as WRITE faults. This ensures
1325 * that the MM will prepare everything (e.g., break COW) such that
1326 * maybe_mkwrite() can create a proper shadow stack PTE.
1327 */
1328 if (error_code & X86_PF_SHSTK)
1329 flags |= FAULT_FLAG_WRITE;
1308 if (error_code & X86_PF_WRITE)1330 if (error_code & X86_PF_WRITE)
1309 flags |= FAULT_FLAG_WRITE;1331 flags |= FAULT_FLAG_WRITE;
1310 if (error_code & X86_PF_INSTR)1332 if (error_code & X86_PF_INSTR)
arch/x86/mm/pat/set_memory.c+2−2
@@ -2074,12 +2074,12 @@ int set_memory_nx(unsigned long addr, int numpages) · 2073 unmodified lines
2074 2074
2075int set_memory_ro(unsigned long addr, int numpages)2075int set_memory_ro(unsigned long addr, int numpages)
2076{2076{
2077 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);2077 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW | _PAGE_DIRTY), 0);
2078}2078}
2079 2079
2080int set_memory_rox(unsigned long addr, int numpages)2080int set_memory_rox(unsigned long addr, int numpages)
2081{2081{
2082 pgprot_t clr = __pgprot(_PAGE_RW);2082 pgprot_t clr = __pgprot(_PAGE_RW | _PAGE_DIRTY);
2083 2083
2084 if (__supported_pte_mask & _PAGE_NX)2084 if (__supported_pte_mask & _PAGE_NX)
2085 clr.pgprot |= _PAGE_NX;2085 clr.pgprot |= _PAGE_NX;
arch/x86/mm/pgtable.c+40−0
@@ -881,3 +881,43 @@ int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) · 880 unmodified lines
881 881
882#endif /* CONFIG_X86_64 */882#endif /* CONFIG_X86_64 */
883#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */883#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
884
885pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma)
886{
887 if (vma->vm_flags & VM_SHADOW_STACK)
888 return pte_mkwrite_shstk(pte);
889
890 pte = pte_mkwrite_novma(pte);
891
892 return pte_clear_saveddirty(pte);
893}
894
895pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
896{
897 if (vma->vm_flags & VM_SHADOW_STACK)
898 return pmd_mkwrite_shstk(pmd);
899
900 pmd = pmd_mkwrite_novma(pmd);
901
902 return pmd_clear_saveddirty(pmd);
903}
904
905void arch_check_zapped_pte(struct vm_area_struct *vma, pte_t pte)
906{
907 /*
908 * Hardware before shadow stack can (rarely) set Dirty=1
909 * on a Write=0 PTE. So the below condition
910 * only indicates a software bug when shadow stack is
911 * supported by the HW. This checking is covered in
912 * pte_shstk().
913 */
914 VM_WARN_ON_ONCE(!(vma->vm_flags & VM_SHADOW_STACK) &&
915 pte_shstk(pte));
916}
917
918void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd)
919{
920 /* See note in arch_check_zapped_pte() */
921 VM_WARN_ON_ONCE(!(vma->vm_flags & VM_SHADOW_STACK) &&
922 pmd_shstk(pmd));
923}
arch/x86/xen/enlighten_pv.c+1−1
@@ -694,7 +694,7 @@ static struct trap_array_entry trap_array[] = { · 693 unmodified lines
694 TRAP_ENTRY(exc_coprocessor_error, false ),694 TRAP_ENTRY(exc_coprocessor_error, false ),
695 TRAP_ENTRY(exc_alignment_check, false ),695 TRAP_ENTRY(exc_alignment_check, false ),
696 TRAP_ENTRY(exc_simd_coprocessor_error, false ),696 TRAP_ENTRY(exc_simd_coprocessor_error, false ),
697#ifdef CONFIG_X86_KERNEL_IBT697#ifdef CONFIG_X86_CET
698 TRAP_ENTRY(exc_control_protection, false ),698 TRAP_ENTRY(exc_control_protection, false ),
699#endif699#endif
700};700};
arch/x86/xen/mmu_pv.c+1−1
@@ -166,7 +166,7 @@ void make_lowmem_page_readwrite(void *vaddr) · 165 unmodified lines
166 if (pte == NULL)166 if (pte == NULL)
167 return; /* vaddr missing */167 return; /* vaddr missing */
168 168
169 ptev = pte_mkwrite(*pte);169 ptev = pte_mkwrite_novma(*pte);
170 170
171 if (HYPERVISOR_update_va_mapping(address, ptev, 0))171 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
172 BUG();172 BUG();
arch/x86/xen/xen-asm.S+1−1
@@ -148,7 +148,7 @@ xen_pv_trap asm_exc_page_fault · 147 unmodified lines
148xen_pv_trap asm_exc_spurious_interrupt_bug148xen_pv_trap asm_exc_spurious_interrupt_bug
149xen_pv_trap asm_exc_coprocessor_error149xen_pv_trap asm_exc_coprocessor_error
150xen_pv_trap asm_exc_alignment_check150xen_pv_trap asm_exc_alignment_check
151#ifdef CONFIG_X86_KERNEL_IBT151#ifdef CONFIG_X86_CET
152xen_pv_trap asm_exc_control_protection152xen_pv_trap asm_exc_control_protection
153#endif153#endif
154#ifdef CONFIG_X86_MCE154#ifdef CONFIG_X86_MCE
arch/xtensa/include/asm/pgtable.h+1−1
@@ -262,7 +262,7 @@ static inline pte_t pte_mkdirty(pte_t pte) · 261 unmodified lines
262 { pte_val(pte) |= _PAGE_DIRTY; return pte; }262 { pte_val(pte) |= _PAGE_DIRTY; return pte; }
263static inline pte_t pte_mkyoung(pte_t pte)263static inline pte_t pte_mkyoung(pte_t pte)
264 { pte_val(pte) |= _PAGE_ACCESSED; return pte; }264 { pte_val(pte) |= _PAGE_ACCESSED; return pte; }
265static inline pte_t pte_mkwrite(pte_t pte)265static inline pte_t pte_mkwrite_novma(pte_t pte)
266 { pte_val(pte) |= _PAGE_WRITABLE; return pte; }266 { pte_val(pte) |= _PAGE_WRITABLE; return pte; }
267 267
268#define pgprot_noncached(prot) \268#define pgprot_noncached(prot) \
fs/aio.c+1−1
@@ -558,7 +558,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) · 557 unmodified lines
558 558
559 ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size,559 ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size,
560 PROT_READ | PROT_WRITE,560 PROT_READ | PROT_WRITE,
561 MAP_SHARED, 0, &unused, NULL);561 MAP_SHARED, 0, 0, &unused, NULL);
562 mmap_write_unlock(mm);562 mmap_write_unlock(mm);
563 if (IS_ERR((void *)ctx->mmap_base)) {563 if (IS_ERR((void *)ctx->mmap_base)) {
564 ctx->mmap_size = 0;564 ctx->mmap_size = 0;
fs/proc/array.c+6−0
@@ -431,6 +431,11 @@ static inline void task_untag_mask(struct seq_file *m, struct mm_struct *mm) · 430 unmodified lines
431 seq_printf(m, "untag_mask:\t%#lx\n", mm_untag_mask(mm));431 seq_printf(m, "untag_mask:\t%#lx\n", mm_untag_mask(mm));
432}432}
433 433
434__weak void arch_proc_pid_thread_features(struct seq_file *m,
435 struct task_struct *task)
436{
437}
438
434int proc_pid_status(struct seq_file *m, struct pid_namespace *ns,439int proc_pid_status(struct seq_file *m, struct pid_namespace *ns,
435 struct pid *pid, struct task_struct *task)440 struct pid *pid, struct task_struct *task)
436{441{
@@ -455,6 +460,7 @@ int proc_pid_status(struct seq_file *m, struct pid_namespace *ns, · 18 unmodified lines
455 task_cpus_allowed(m, task);460 task_cpus_allowed(m, task);
456 cpuset_task_status_allowed(m, task);461 cpuset_task_status_allowed(m, task);
457 task_context_switch_counts(m, task);462 task_context_switch_counts(m, task);
463 arch_proc_pid_thread_features(m, task);
458 return 0;464 return 0;
459}465}
460 466
fs/proc/task_mmu.c+3−0
@@ -692,6 +692,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma) · 691 unmodified lines
692#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR692#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
693 [ilog2(VM_UFFD_MINOR)] = "ui",693 [ilog2(VM_UFFD_MINOR)] = "ui",
694#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */694#endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
695#ifdef CONFIG_X86_USER_SHADOW_STACK
696 [ilog2(VM_SHADOW_STACK)] = "ss",
697#endif
695 };698 };
696 size_t i;699 size_t i;
697 700
include/asm-generic/hugetlb.h+1−1
@@ -22,7 +22,7 @@ static inline unsigned long huge_pte_dirty(pte_t pte) · 21 unmodified lines
22 22
23static inline pte_t huge_pte_mkwrite(pte_t pte)23static inline pte_t huge_pte_mkwrite(pte_t pte)
24{24{
25 return pte_mkwrite(pte);25 return pte_mkwrite_novma(pte);
26}26}
27 27
28#ifndef __HAVE_ARCH_HUGE_PTE_WRPROTECT28#ifndef __HAVE_ARCH_HUGE_PTE_WRPROTECT
include/linux/mm.h+39−8
@@ -319,11 +319,13 @@ extern unsigned int kobjsize(const void *objp); · 318 unmodified lines
319#define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit architectures */319#define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit architectures */
320#define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit architectures */320#define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit architectures */
321#define VM_HIGH_ARCH_BIT_4 36 /* bit only usable on 64-bit architectures */321#define VM_HIGH_ARCH_BIT_4 36 /* bit only usable on 64-bit architectures */
322#define VM_HIGH_ARCH_BIT_5 37 /* bit only usable on 64-bit architectures */
322#define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0)323#define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0)
323#define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1)324#define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1)
324#define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2)325#define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2)
325#define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3)326#define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3)
326#define VM_HIGH_ARCH_4 BIT(VM_HIGH_ARCH_BIT_4)327#define VM_HIGH_ARCH_4 BIT(VM_HIGH_ARCH_BIT_4)
328#define VM_HIGH_ARCH_5 BIT(VM_HIGH_ARCH_BIT_5)
327#endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */329#endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */
328 330
329#ifdef CONFIG_ARCH_HAS_PKEYS331#ifdef CONFIG_ARCH_HAS_PKEYS
@@ -339,6 +341,21 @@ extern unsigned int kobjsize(const void *objp); · 9 unmodified lines
339#endif341#endif
340#endif /* CONFIG_ARCH_HAS_PKEYS */342#endif /* CONFIG_ARCH_HAS_PKEYS */
341 343
344#ifdef CONFIG_X86_USER_SHADOW_STACK
345/*
346 * VM_SHADOW_STACK should not be set with VM_SHARED because of lack of
347 * support core mm.
348 *
349 * These VMAs will get a single end guard page. This helps userspace protect
350 * itself from attacks. A single page is enough for current shadow stack archs
351 * (x86). See the comments near alloc_shstk() in arch/x86/kernel/shstk.c
352 * for more details on the guard size.
353 */
354# define VM_SHADOW_STACK VM_HIGH_ARCH_5
355#else
356# define VM_SHADOW_STACK VM_NONE
357#endif
358
342#if defined(CONFIG_X86)359#if defined(CONFIG_X86)
343# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */360# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
344#elif defined(CONFIG_PPC)361#elif defined(CONFIG_PPC)
@@ -370,7 +387,7 @@ extern unsigned int kobjsize(const void *objp); · 25 unmodified lines
370#endif387#endif
371 388
372#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR389#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
373# define VM_UFFD_MINOR_BIT 37390# define VM_UFFD_MINOR_BIT 38
374# define VM_UFFD_MINOR BIT(VM_UFFD_MINOR_BIT) /* UFFD minor faults */391# define VM_UFFD_MINOR BIT(VM_UFFD_MINOR_BIT) /* UFFD minor faults */
375#else /* !CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */392#else /* !CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
376# define VM_UFFD_MINOR VM_NONE393# define VM_UFFD_MINOR VM_NONE
@@ -397,6 +414,8 @@ extern unsigned int kobjsize(const void *objp); · 20 unmodified lines
397#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS414#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
398#endif415#endif
399 416
417#define VM_STARTGAP_FLAGS (VM_GROWSDOWN | VM_SHADOW_STACK)
418
400#ifdef CONFIG_STACK_GROWSUP419#ifdef CONFIG_STACK_GROWSUP
401#define VM_STACK VM_GROWSUP420#define VM_STACK VM_GROWSUP
402#define VM_STACK_EARLY VM_GROWSDOWN421#define VM_STACK_EARLY VM_GROWSDOWN
@@ -1309,7 +1328,7 @@ static inline unsigned long thp_size(struct page *page) · 906 unmodified lines
1309static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)1328static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
1310{1329{
1311 if (likely(vma->vm_flags & VM_WRITE))1330 if (likely(vma->vm_flags & VM_WRITE))
1312 pte = pte_mkwrite(pte);1331 pte = pte_mkwrite(pte, vma);
1313 return pte;1332 return pte;
1314}1333}
1315 1334
@@ -3265,7 +3284,8 @@ extern unsigned long mmap_region(struct file *file, unsigned long addr, · 1949 unmodified lines
3265 struct list_head *uf);3284 struct list_head *uf);
3266extern unsigned long do_mmap(struct file *file, unsigned long addr,3285extern unsigned long do_mmap(struct file *file, unsigned long addr,
3267 unsigned long len, unsigned long prot, unsigned long flags,3286 unsigned long len, unsigned long prot, unsigned long flags,
3268 unsigned long pgoff, unsigned long *populate, struct list_head *uf);3287 vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate,
3288 struct list_head *uf);
3269extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,3289extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
3270 unsigned long start, size_t len, struct list_head *uf,3290 unsigned long start, size_t len, struct list_head *uf,
3271 bool unlock);3291 bool unlock);
@@ -3353,15 +3373,26 @@ struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr) · 81 unmodified lines
3353 return mtree_load(&mm->mm_mt, addr);3373 return mtree_load(&mm->mm_mt, addr);
3354}3374}
3355 3375
3376static inline unsigned long stack_guard_start_gap(struct vm_area_struct *vma)
3377{
3378 if (vma->vm_flags & VM_GROWSDOWN)
3379 return stack_guard_gap;
3380
3381 /* See reasoning around the VM_SHADOW_STACK definition */
3382 if (vma->vm_flags & VM_SHADOW_STACK)
3383 return PAGE_SIZE;
3384
3385 return 0;
3386}
3387
3356static inline unsigned long vm_start_gap(struct vm_area_struct *vma)3388static inline unsigned long vm_start_gap(struct vm_area_struct *vma)
3357{3389{
3390 unsigned long gap = stack_guard_start_gap(vma);
3358 unsigned long vm_start = vma->vm_start;3391 unsigned long vm_start = vma->vm_start;
3359 3392
3360 if (vma->vm_flags & VM_GROWSDOWN) {3393 vm_start -= gap;
3361 vm_start -= stack_guard_gap;3394 if (vm_start > vma->vm_start)
3362 if (vm_start > vma->vm_start)3395 vm_start = 0;
3363 vm_start = 0;
3364 }
3365 return vm_start;3396 return vm_start;
3366}3397}
3367 3398
include/linux/mman.h+4−0
@@ -15,6 +15,9 @@ · 14 unmodified lines
15#ifndef MAP_32BIT15#ifndef MAP_32BIT
16#define MAP_32BIT 016#define MAP_32BIT 0
17#endif17#endif
18#ifndef MAP_ABOVE4G
19#define MAP_ABOVE4G 0
20#endif
18#ifndef MAP_HUGE_2MB21#ifndef MAP_HUGE_2MB
19#define MAP_HUGE_2MB 022#define MAP_HUGE_2MB 0
20#endif23#endif
@@ -50,6 +53,7 @@ · 29 unmodified lines
50 | MAP_STACK \53 | MAP_STACK \
51 | MAP_HUGETLB \54 | MAP_HUGETLB \
52 | MAP_32BIT \55 | MAP_32BIT \
56 | MAP_ABOVE4G \
53 | MAP_HUGE_2MB \57 | MAP_HUGE_2MB \
54 | MAP_HUGE_1GB)58 | MAP_HUGE_1GB)
55 59
include/linux/pgtable.h+28−0
@@ -371,6 +371,20 @@ static inline bool arch_has_hw_pte_young(void) · 370 unmodified lines
371}371}
372#endif372#endif
373 373
374#ifndef arch_check_zapped_pte
375static inline void arch_check_zapped_pte(struct vm_area_struct *vma,
376 pte_t pte)
377{
378}
379#endif
380
381#ifndef arch_check_zapped_pmd
382static inline void arch_check_zapped_pmd(struct vm_area_struct *vma,
383 pmd_t pmd)
384{
385}
386#endif
387
374#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR388#ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR
375static inline pte_t ptep_get_and_clear(struct mm_struct *mm,389static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
376 unsigned long address,390 unsigned long address,
@@ -577,6 +591,20 @@ extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, · 200 unmodified lines
577 pud_t *pudp);591 pud_t *pudp);
578#endif592#endif
579 593
594#ifndef pte_mkwrite
595static inline pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma)
596{
597 return pte_mkwrite_novma(pte);
598}
599#endif
600
601#if defined(CONFIG_ARCH_WANT_PMD_MKWRITE) && !defined(pmd_mkwrite)
602static inline pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
603{
604 return pmd_mkwrite_novma(pmd);
605}
606#endif
607
580#ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT608#ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT
581struct mm_struct;609struct mm_struct;
582static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)610static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
include/linux/proc_fs.h+1−0
@@ -159,6 +159,7 @@ int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns, · 158 unmodified lines
159#endif /* CONFIG_PROC_PID_ARCH_STATUS */159#endif /* CONFIG_PROC_PID_ARCH_STATUS */
160 160
161void arch_report_meminfo(struct seq_file *m);161void arch_report_meminfo(struct seq_file *m);
162void arch_proc_pid_thread_features(struct seq_file *m, struct task_struct *task);
162 163
163#else /* CONFIG_PROC_FS */164#else /* CONFIG_PROC_FS */
164 165
include/linux/syscalls.h+1−0
@@ -939,6 +939,7 @@ asmlinkage long sys_set_mempolicy_home_node(unsigned long start, unsigned long l · 938 unmodified lines
939asmlinkage long sys_cachestat(unsigned int fd,939asmlinkage long sys_cachestat(unsigned int fd,
940 struct cachestat_range __user *cstat_range,940 struct cachestat_range __user *cstat_range,
941 struct cachestat __user *cstat, unsigned int flags);941 struct cachestat __user *cstat, unsigned int flags);
942asmlinkage long sys_map_shadow_stack(unsigned long addr, unsigned long size, unsigned int flags);
942 943
943/*944/*
944 * Architecture-specific system calls945 * Architecture-specific system calls
include/uapi/asm-generic/siginfo.h+2−1
@@ -242,7 +242,8 @@ typedef struct siginfo { · 241 unmodified lines
242#define SEGV_ADIPERR 7 /* Precise MCD exception */242#define SEGV_ADIPERR 7 /* Precise MCD exception */
243#define SEGV_MTEAERR 8 /* Asynchronous ARM MTE error */243#define SEGV_MTEAERR 8 /* Asynchronous ARM MTE error */
244#define SEGV_MTESERR 9 /* Synchronous ARM MTE exception */244#define SEGV_MTESERR 9 /* Synchronous ARM MTE exception */
245#define NSIGSEGV 9245#define SEGV_CPERR 10 /* Control protection fault */
246#define NSIGSEGV 10
246 247
247/*248/*
248 * SIGBUS si_codes249 * SIGBUS si_codes
include/uapi/linux/elf.h+2−0
@@ -409,6 +409,8 @@ typedef struct elf64_shdr { · 408 unmodified lines
409#define NT_386_TLS 0x200 /* i386 TLS slots (struct user_desc) */409#define NT_386_TLS 0x200 /* i386 TLS slots (struct user_desc) */
410#define NT_386_IOPERM 0x201 /* x86 io permission bitmap (1=deny) */410#define NT_386_IOPERM 0x201 /* x86 io permission bitmap (1=deny) */
411#define NT_X86_XSTATE 0x202 /* x86 extended state using xsave */411#define NT_X86_XSTATE 0x202 /* x86 extended state using xsave */
412/* Old binutils treats 0x203 as a CET state */
413#define NT_X86_SHSTK 0x204 /* x86 SHSTK state */
412#define NT_S390_HIGH_GPRS 0x300 /* s390 upper register halves */414#define NT_S390_HIGH_GPRS 0x300 /* s390 upper register halves */
413#define NT_S390_TIMER 0x301 /* s390 timer register */415#define NT_S390_TIMER 0x301 /* s390 timer register */
414#define NT_S390_TODCMP 0x302 /* s390 TOD clock comparator register */416#define NT_S390_TODCMP 0x302 /* s390 TOD clock comparator register */
ipc/shm.c+1−1
@@ -1662,7 +1662,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, · 1661 unmodified lines
1662 goto invalid;1662 goto invalid;
1663 }1663 }
1664 1664
1665 addr = do_mmap(file, addr, size, prot, flags, 0, &populate, NULL);1665 addr = do_mmap(file, addr, size, prot, flags, 0, 0, &populate, NULL);
1666 *raddr = addr;1666 *raddr = addr;
1667 err = 0;1667 err = 0;
1668 if (IS_ERR_VALUE(addr))1668 if (IS_ERR_VALUE(addr))
kernel/sys_ni.c+1−0
@@ -274,6 +274,7 @@ COND_SYSCALL(vm86old); · 273 unmodified lines
274COND_SYSCALL(modify_ldt);274COND_SYSCALL(modify_ldt);
275COND_SYSCALL(vm86);275COND_SYSCALL(vm86);
276COND_SYSCALL(kexec_file_load);276COND_SYSCALL(kexec_file_load);
277COND_SYSCALL(map_shadow_stack);
277 278
278/* s390 */279/* s390 */
279COND_SYSCALL(s390_pci_mmio_read);280COND_SYSCALL(s390_pci_mmio_read);
mm/debug_vm_pgtable.c+6−6
@@ -109,10 +109,10 @@ static void __init pte_basic_tests(struct pgtable_debug_args *args, int idx) · 108 unmodified lines
109 WARN_ON(!pte_same(pte, pte));109 WARN_ON(!pte_same(pte, pte));
110 WARN_ON(!pte_young(pte_mkyoung(pte_mkold(pte))));110 WARN_ON(!pte_young(pte_mkyoung(pte_mkold(pte))));
111 WARN_ON(!pte_dirty(pte_mkdirty(pte_mkclean(pte))));111 WARN_ON(!pte_dirty(pte_mkdirty(pte_mkclean(pte))));
112 WARN_ON(!pte_write(pte_mkwrite(pte_wrprotect(pte))));112 WARN_ON(!pte_write(pte_mkwrite(pte_wrprotect(pte), args->vma)));
113 WARN_ON(pte_young(pte_mkold(pte_mkyoung(pte))));113 WARN_ON(pte_young(pte_mkold(pte_mkyoung(pte))));
114 WARN_ON(pte_dirty(pte_mkclean(pte_mkdirty(pte))));114 WARN_ON(pte_dirty(pte_mkclean(pte_mkdirty(pte))));
115 WARN_ON(pte_write(pte_wrprotect(pte_mkwrite(pte))));115 WARN_ON(pte_write(pte_wrprotect(pte_mkwrite(pte, args->vma))));
116 WARN_ON(pte_dirty(pte_wrprotect(pte_mkclean(pte))));116 WARN_ON(pte_dirty(pte_wrprotect(pte_mkclean(pte))));
117 WARN_ON(!pte_dirty(pte_wrprotect(pte_mkdirty(pte))));117 WARN_ON(!pte_dirty(pte_wrprotect(pte_mkdirty(pte))));
118}118}
@@ -156,7 +156,7 @@ static void __init pte_advanced_tests(struct pgtable_debug_args *args) · 37 unmodified lines
156 pte = pte_mkclean(pte);156 pte = pte_mkclean(pte);
157 set_pte_at(args->mm, args->vaddr, args->ptep, pte);157 set_pte_at(args->mm, args->vaddr, args->ptep, pte);
158 flush_dcache_page(page);158 flush_dcache_page(page);
159 pte = pte_mkwrite(pte);159 pte = pte_mkwrite(pte, args->vma);
160 pte = pte_mkdirty(pte);160 pte = pte_mkdirty(pte);
161 ptep_set_access_flags(args->vma, args->vaddr, args->ptep, pte, 1);161 ptep_set_access_flags(args->vma, args->vaddr, args->ptep, pte, 1);
162 pte = ptep_get(args->ptep);162 pte = ptep_get(args->ptep);
@@ -202,10 +202,10 @@ static void __init pmd_basic_tests(struct pgtable_debug_args *args, int idx) · 39 unmodified lines
202 WARN_ON(!pmd_same(pmd, pmd));202 WARN_ON(!pmd_same(pmd, pmd));
203 WARN_ON(!pmd_young(pmd_mkyoung(pmd_mkold(pmd))));203 WARN_ON(!pmd_young(pmd_mkyoung(pmd_mkold(pmd))));
204 WARN_ON(!pmd_dirty(pmd_mkdirty(pmd_mkclean(pmd))));204 WARN_ON(!pmd_dirty(pmd_mkdirty(pmd_mkclean(pmd))));
205 WARN_ON(!pmd_write(pmd_mkwrite(pmd_wrprotect(pmd))));205 WARN_ON(!pmd_write(pmd_mkwrite(pmd_wrprotect(pmd), args->vma)));
206 WARN_ON(pmd_young(pmd_mkold(pmd_mkyoung(pmd))));206 WARN_ON(pmd_young(pmd_mkold(pmd_mkyoung(pmd))));
207 WARN_ON(pmd_dirty(pmd_mkclean(pmd_mkdirty(pmd))));207 WARN_ON(pmd_dirty(pmd_mkclean(pmd_mkdirty(pmd))));
208 WARN_ON(pmd_write(pmd_wrprotect(pmd_mkwrite(pmd))));208 WARN_ON(pmd_write(pmd_wrprotect(pmd_mkwrite(pmd, args->vma))));
209 WARN_ON(pmd_dirty(pmd_wrprotect(pmd_mkclean(pmd))));209 WARN_ON(pmd_dirty(pmd_wrprotect(pmd_mkclean(pmd))));
210 WARN_ON(!pmd_dirty(pmd_wrprotect(pmd_mkdirty(pmd))));210 WARN_ON(!pmd_dirty(pmd_wrprotect(pmd_mkdirty(pmd))));
211 /*211 /*
@@ -256,7 +256,7 @@ static void __init pmd_advanced_tests(struct pgtable_debug_args *args) · 44 unmodified lines
256 pmd = pmd_mkclean(pmd);256 pmd = pmd_mkclean(pmd);
257 set_pmd_at(args->mm, vaddr, args->pmdp, pmd);257 set_pmd_at(args->mm, vaddr, args->pmdp, pmd);
258 flush_dcache_page(page);258 flush_dcache_page(page);
259 pmd = pmd_mkwrite(pmd);259 pmd = pmd_mkwrite(pmd, args->vma);
260 pmd = pmd_mkdirty(pmd);260 pmd = pmd_mkdirty(pmd);
261 pmdp_set_access_flags(args->vma, vaddr, args->pmdp, pmd, 1);261 pmdp_set_access_flags(args->vma, vaddr, args->pmdp, pmd, 1);
262 pmd = READ_ONCE(*args->pmdp);262 pmd = READ_ONCE(*args->pmdp);
mm/gup.c+1−1
@@ -1051,7 +1051,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) · 1050 unmodified lines
1051 !writable_file_mapping_allowed(vma, gup_flags))1051 !writable_file_mapping_allowed(vma, gup_flags))
1052 return -EFAULT;1052 return -EFAULT;
1053 1053
1054 if (!(vm_flags & VM_WRITE)) {1054 if (!(vm_flags & VM_WRITE) || (vm_flags & VM_SHADOW_STACK)) {
1055 if (!(gup_flags & FOLL_FORCE))1055 if (!(gup_flags & FOLL_FORCE))
1056 return -EFAULT;1056 return -EFAULT;
1057 /* hugetlb does not support FOLL_FORCE|FOLL_WRITE. */1057 /* hugetlb does not support FOLL_FORCE|FOLL_WRITE. */
mm/huge_memory.c+6−5
@@ -551,7 +551,7 @@ __setup("transparent_hugepage=", setup_transparent_hugepage); · 550 unmodified lines
551pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)551pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
552{552{
553 if (likely(vma->vm_flags & VM_WRITE))553 if (likely(vma->vm_flags & VM_WRITE))
554 pmd = pmd_mkwrite(pmd);554 pmd = pmd_mkwrite(pmd, vma);
555 return pmd;555 return pmd;
556}556}
557 557
@@ -1566,7 +1566,7 @@ out_map: · 1008 unmodified lines
1566 pmd = pmd_modify(oldpmd, vma->vm_page_prot);1566 pmd = pmd_modify(oldpmd, vma->vm_page_prot);
1567 pmd = pmd_mkyoung(pmd);1567 pmd = pmd_mkyoung(pmd);
1568 if (writable)1568 if (writable)
1569 pmd = pmd_mkwrite(pmd);1569 pmd = pmd_mkwrite(pmd, vma);
1570 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);1570 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);
1571 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);1571 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
1572 spin_unlock(vmf->ptl);1572 spin_unlock(vmf->ptl);
@@ -1675,6 +1675,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · 102 unmodified lines
1675 */1675 */
1676 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd,1676 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd,
1677 tlb->fullmm);1677 tlb->fullmm);
1678 arch_check_zapped_pmd(vma, orig_pmd);
1678 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);1679 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1679 if (vma_is_special_huge(vma)) {1680 if (vma_is_special_huge(vma)) {
1680 if (arch_needs_pgtable_deposit())1681 if (arch_needs_pgtable_deposit())
@@ -1919,7 +1920,7 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · 238 unmodified lines
1919 /* See change_pte_range(). */1920 /* See change_pte_range(). */
1920 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) &&1921 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) &&
1921 can_change_pmd_writable(vma, addr, entry))1922 can_change_pmd_writable(vma, addr, entry))
1922 entry = pmd_mkwrite(entry);1923 entry = pmd_mkwrite(entry, vma);
1923 1924
1924 ret = HPAGE_PMD_NR;1925 ret = HPAGE_PMD_NR;
1925 set_pmd_at(mm, addr, pmd, entry);1926 set_pmd_at(mm, addr, pmd, entry);
@@ -2233,7 +2234,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, · 307 unmodified lines
2233 } else {2234 } else {
2234 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot));2235 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot));
2235 if (write)2236 if (write)
2236 entry = pte_mkwrite(entry);2237 entry = pte_mkwrite(entry, vma);
2237 if (anon_exclusive)2238 if (anon_exclusive)
2238 SetPageAnonExclusive(page + i);2239 SetPageAnonExclusive(page + i);
2239 if (!young)2240 if (!young)
@@ -3265,7 +3266,7 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) · 1025 unmodified lines
3265 if (pmd_swp_soft_dirty(*pvmw->pmd))3266 if (pmd_swp_soft_dirty(*pvmw->pmd))
3266 pmde = pmd_mksoft_dirty(pmde);3267 pmde = pmd_mksoft_dirty(pmde);
3267 if (is_writable_migration_entry(entry))3268 if (is_writable_migration_entry(entry))
3268 pmde = pmd_mkwrite(pmde);3269 pmde = pmd_mkwrite(pmde, vma);
3269 if (pmd_swp_uffd_wp(*pvmw->pmd))3270 if (pmd_swp_uffd_wp(*pvmw->pmd))
3270 pmde = pmd_mkuffd_wp(pmde);3271 pmde = pmd_mkuffd_wp(pmde);
3271 if (!is_migration_entry_young(entry))3272 if (!is_migration_entry_young(entry))
mm/internal.h+2−2
@@ -556,14 +556,14 @@ static inline bool is_exec_mapping(vm_flags_t flags) · 555 unmodified lines
556}556}
557 557
558/*558/*
559 * Stack area - automatically grows in one direction559 * Stack area (including shadow stacks)
560 *560 *
561 * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:561 * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
562 * do_mmap() forbids all other combinations.562 * do_mmap() forbids all other combinations.
563 */563 */
564static inline bool is_stack_mapping(vm_flags_t flags)564static inline bool is_stack_mapping(vm_flags_t flags)
565{565{
566 return (flags & VM_STACK) == VM_STACK;566 return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK);
567}567}
568 568
569/*569/*
mm/memory.c+3−2
@@ -1430,6 +1430,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, · 1429 unmodified lines
1430 continue;1430 continue;
1431 ptent = ptep_get_and_clear_full(mm, addr, pte,1431 ptent = ptep_get_and_clear_full(mm, addr, pte,
1432 tlb->fullmm);1432 tlb->fullmm);
1433 arch_check_zapped_pte(vma, ptent);
1433 tlb_remove_tlb_entry(tlb, pte, addr);1434 tlb_remove_tlb_entry(tlb, pte, addr);
1434 zap_install_uffd_wp_if_needed(vma, addr, pte, details,1435 zap_install_uffd_wp_if_needed(vma, addr, pte, details,
1435 ptent);1436 ptent);
@@ -4124,7 +4125,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) · 2688 unmodified lines
4124 entry = mk_pte(&folio->page, vma->vm_page_prot);4125 entry = mk_pte(&folio->page, vma->vm_page_prot);
4125 entry = pte_sw_mkyoung(entry);4126 entry = pte_sw_mkyoung(entry);
4126 if (vma->vm_flags & VM_WRITE)4127 if (vma->vm_flags & VM_WRITE)
4127 entry = pte_mkwrite(pte_mkdirty(entry));4128 entry = pte_mkwrite(pte_mkdirty(entry), vma);
4128 4129
4129 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,4130 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
4130 &vmf->ptl);4131 &vmf->ptl);
@@ -4842,7 +4843,7 @@ out_map: · 711 unmodified lines
4842 pte = pte_modify(old_pte, vma->vm_page_prot);4843 pte = pte_modify(old_pte, vma->vm_page_prot);
4843 pte = pte_mkyoung(pte);4844 pte = pte_mkyoung(pte);
4844 if (writable)4845 if (writable)
4845 pte = pte_mkwrite(pte);4846 pte = pte_mkwrite(pte, vma);
4846 ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte);4847 ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte);
4847 update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);4848 update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
4848 pte_unmap_unlock(vmf->pte, vmf->ptl);4849 pte_unmap_unlock(vmf->pte, vmf->ptl);
mm/migrate.c+1−1
@@ -220,7 +220,7 @@ static bool remove_migration_pte(struct folio *folio, · 219 unmodified lines
220 if (folio_test_dirty(folio) && is_migration_entry_dirty(entry))220 if (folio_test_dirty(folio) && is_migration_entry_dirty(entry))
221 pte = pte_mkdirty(pte);221 pte = pte_mkdirty(pte);
222 if (is_writable_migration_entry(entry))222 if (is_writable_migration_entry(entry))
223 pte = pte_mkwrite(pte);223 pte = pte_mkwrite(pte, vma);
224 else if (pte_swp_uffd_wp(old_pte))224 else if (pte_swp_uffd_wp(old_pte))
225 pte = pte_mkuffd_wp(pte);225 pte = pte_mkuffd_wp(pte);
226 226
mm/migrate_device.c+1−1
@@ -624,7 +624,7 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate, · 623 unmodified lines
624 }624 }
625 entry = mk_pte(page, vma->vm_page_prot);625 entry = mk_pte(page, vma->vm_page_prot);
626 if (vma->vm_flags & VM_WRITE)626 if (vma->vm_flags & VM_WRITE)
627 entry = pte_mkwrite(pte_mkdirty(entry));627 entry = pte_mkwrite(pte_mkdirty(entry), vma);
628 }628 }
629 629
630 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);630 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
mm/mmap.c+7−7
@@ -1182,11 +1182,11 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, · 1181 unmodified lines
1182 */1182 */
1183unsigned long do_mmap(struct file *file, unsigned long addr,1183unsigned long do_mmap(struct file *file, unsigned long addr,
1184 unsigned long len, unsigned long prot,1184 unsigned long len, unsigned long prot,
1185 unsigned long flags, unsigned long pgoff,1185 unsigned long flags, vm_flags_t vm_flags,
1186 unsigned long *populate, struct list_head *uf)1186 unsigned long pgoff, unsigned long *populate,
1187 struct list_head *uf)
1187{1188{
1188 struct mm_struct *mm = current->mm;1189 struct mm_struct *mm = current->mm;
1189 vm_flags_t vm_flags;
1190 int pkey = 0;1190 int pkey = 0;
1191 1191
1192 *populate = 0;1192 *populate = 0;
@@ -1246,7 +1246,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, · 53 unmodified lines
1246 * to. we assume access permissions have been handled by the open1246 * to. we assume access permissions have been handled by the open
1247 * of the memory object, so we don't do any here.1247 * of the memory object, so we don't do any here.
1248 */1248 */
1249 vm_flags = calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |1249 vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
1250 mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;1250 mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
1251 1251
1252 if (flags & MAP_LOCKED)1252 if (flags & MAP_LOCKED)
@@ -1564,7 +1564,7 @@ retry: · 311 unmodified lines
1564 gap = mas.index;1564 gap = mas.index;
1565 gap += (info->align_offset - gap) & info->align_mask;1565 gap += (info->align_offset - gap) & info->align_mask;
1566 tmp = mas_next(&mas, ULONG_MAX);1566 tmp = mas_next(&mas, ULONG_MAX);
1567 if (tmp && (tmp->vm_flags & VM_GROWSDOWN)) { /* Avoid prev check if possible */1567 if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */
1568 if (vm_start_gap(tmp) < gap + length - 1) {1568 if (vm_start_gap(tmp) < gap + length - 1) {
1569 low_limit = tmp->vm_end;1569 low_limit = tmp->vm_end;
1570 mas_reset(&mas);1570 mas_reset(&mas);
@@ -1616,7 +1616,7 @@ retry: · 45 unmodified lines
1616 gap -= (gap - info->align_offset) & info->align_mask;1616 gap -= (gap - info->align_offset) & info->align_mask;
1617 gap_end = mas.last;1617 gap_end = mas.last;
1618 tmp = mas_next(&mas, ULONG_MAX);1618 tmp = mas_next(&mas, ULONG_MAX);
1619 if (tmp && (tmp->vm_flags & VM_GROWSDOWN)) { /* Avoid prev check if possible */1619 if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */
1620 if (vm_start_gap(tmp) <= gap_end) {1620 if (vm_start_gap(tmp) <= gap_end) {
1621 high_limit = vm_start_gap(tmp);1621 high_limit = vm_start_gap(tmp);
1622 mas_reset(&mas);1622 mas_reset(&mas);
@@ -2998,7 +2998,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, · 1375 unmodified lines
2998 2998
2999 file = get_file(vma->vm_file);2999 file = get_file(vma->vm_file);
3000 ret = do_mmap(vma->vm_file, start, size,3000 ret = do_mmap(vma->vm_file, start, size,
3001 prot, flags, pgoff, &populate, NULL);3001 prot, flags, 0, pgoff, &populate, NULL);
3002 fput(file);3002 fput(file);
3003out:3003out:
3004 mmap_write_unlock(mm);3004 mmap_write_unlock(mm);
mm/mprotect.c+1−1
@@ -185,7 +185,7 @@ static long change_pte_range(struct mmu_gather *tlb, · 184 unmodified lines
185 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&185 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
186 !pte_write(ptent) &&186 !pte_write(ptent) &&
187 can_change_pte_writable(vma, addr, ptent))187 can_change_pte_writable(vma, addr, ptent))
188 ptent = pte_mkwrite(ptent);188 ptent = pte_mkwrite(ptent, vma);
189 189
190 ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);190 ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);
191 if (pte_needs_flush(oldpte, ptent))191 if (pte_needs_flush(oldpte, ptent))
mm/nommu.c+2−2
@@ -1016,6 +1016,7 @@ unsigned long do_mmap(struct file *file, · 1015 unmodified lines
1016 unsigned long len,1016 unsigned long len,
1017 unsigned long prot,1017 unsigned long prot,
1018 unsigned long flags,1018 unsigned long flags,
1019 vm_flags_t vm_flags,
1019 unsigned long pgoff,1020 unsigned long pgoff,
1020 unsigned long *populate,1021 unsigned long *populate,
1021 struct list_head *uf)1022 struct list_head *uf)
@@ -1023,7 +1024,6 @@ unsigned long do_mmap(struct file *file, · 1 unmodified line
1023 struct vm_area_struct *vma;1024 struct vm_area_struct *vma;
1024 struct vm_region *region;1025 struct vm_region *region;
1025 struct rb_node *rb;1026 struct rb_node *rb;
1026 vm_flags_t vm_flags;
1027 unsigned long capabilities, result;1027 unsigned long capabilities, result;
1028 int ret;1028 int ret;
1029 VMA_ITERATOR(vmi, current->mm, 0);1029 VMA_ITERATOR(vmi, current->mm, 0);
@@ -1043,7 +1043,7 @@ unsigned long do_mmap(struct file *file, · 13 unmodified lines
1043 1043
1044 /* we've determined that we can make the mapping, now translate what we1044 /* we've determined that we can make the mapping, now translate what we
1045 * now know into VMA flags */1045 * now know into VMA flags */
1046 vm_flags = determine_vm_flags(file, prot, flags, capabilities);1046 vm_flags |= determine_vm_flags(file, prot, flags, capabilities);
1047 1047
1048 1048
1049 /* we're going to need to record the mapping */1049 /* we're going to need to record the mapping */
mm/userfaultfd.c+1−1
@@ -86,7 +86,7 @@ int mfill_atomic_install_pte(pmd_t *dst_pmd, · 85 unmodified lines
86 if (page_in_cache && !vm_shared)86 if (page_in_cache && !vm_shared)
87 writable = false;87 writable = false;
88 if (writable)88 if (writable)
89 _dst_pte = pte_mkwrite(_dst_pte);89 _dst_pte = pte_mkwrite(_dst_pte, dst_vma);
90 if (flags & MFILL_ATOMIC_WP)90 if (flags & MFILL_ATOMIC_WP)
91 _dst_pte = pte_mkuffd_wp(_dst_pte);91 _dst_pte = pte_mkuffd_wp(_dst_pte);
92 92
mm/util.c+1−1
@@ -543,7 +543,7 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, · 542 unmodified lines
543 if (!ret) {543 if (!ret) {
544 if (mmap_write_lock_killable(mm))544 if (mmap_write_lock_killable(mm))
545 return -EINTR;545 return -EINTR;
546 ret = do_mmap(file, addr, len, prot, flag, pgoff, &populate,546 ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate,
547 &uf);547 &uf);
548 mmap_write_unlock(mm);548 mmap_write_unlock(mm);
549 userfaultfd_unmap_complete(mm, &uf);549 userfaultfd_unmap_complete(mm, &uf);
tools/testing/selftests/x86/Makefile+1−1
@@ -18,7 +18,7 @@ TARGETS_C_32BIT_ONLY := entry_from_vm86 test_syscall_vdso unwind_vdso \ · 17 unmodified lines
18 test_FCMOV test_FCOMI test_FISTTP \18 test_FCMOV test_FCOMI test_FISTTP \
19 vdso_restorer19 vdso_restorer
20TARGETS_C_64BIT_ONLY := fsgsbase sysret_rip syscall_numbering \20TARGETS_C_64BIT_ONLY := fsgsbase sysret_rip syscall_numbering \
21 corrupt_xstate_header amx lam21 corrupt_xstate_header amx lam test_shadow_stack
22# Some selftests require 32bit support enabled also on 64bit systems22# Some selftests require 32bit support enabled also on 64bit systems
23TARGETS_C_32BIT_NEEDED := ldt_gdt ptrace_syscall23TARGETS_C_32BIT_NEEDED := ldt_gdt ptrace_syscall
24 24
tools/testing/selftests/x86/test_shadow_stack.c+884−0
@@ -0,0 +1,884 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * This program test's basic kernel shadow stack support. It enables shadow
4 * stack manual via the arch_prctl(), instead of relying on glibc. It's
5 * Makefile doesn't compile with shadow stack support, so it doesn't rely on
6 * any particular glibc. As a result it can't do any operations that require
7 * special glibc shadow stack support (longjmp(), swapcontext(), etc). Just
8 * stick to the basics and hope the compiler doesn't do anything strange.
9 */
10
11#define _GNU_SOURCE
12
13#include <sys/syscall.h>
14#include <asm/mman.h>
15#include <sys/mman.h>
16#include <sys/stat.h>
17#include <sys/wait.h>
18#include <stdio.h>
19#include <stdlib.h>
20#include <fcntl.h>
21#include <unistd.h>
22#include <string.h>
23#include <errno.h>
24#include <stdbool.h>
25#include <x86intrin.h>
26#include <asm/prctl.h>
27#include <sys/prctl.h>
28#include <stdint.h>
29#include <signal.h>
30#include <pthread.h>
31#include <sys/ioctl.h>
32#include <linux/userfaultfd.h>
33#include <setjmp.h>
34#include <sys/ptrace.h>
35#include <sys/signal.h>
36#include <linux/elf.h>
37
38/*
39 * Define the ABI defines if needed, so people can run the tests
40 * without building the headers.
41 */
42#ifndef __NR_map_shadow_stack
43#define __NR_map_shadow_stack 452
44
45#define SHADOW_STACK_SET_TOKEN (1ULL << 0)
46
47#define ARCH_SHSTK_ENABLE 0x5001
48#define ARCH_SHSTK_DISABLE 0x5002
49#define ARCH_SHSTK_LOCK 0x5003
50#define ARCH_SHSTK_UNLOCK 0x5004
51#define ARCH_SHSTK_STATUS 0x5005
52
53#define ARCH_SHSTK_SHSTK (1ULL << 0)
54#define ARCH_SHSTK_WRSS (1ULL << 1)
55
56#define NT_X86_SHSTK 0x204
57#endif
58
59#define SS_SIZE 0x200000
60#define PAGE_SIZE 0x1000
61
62#if (__GNUC__ < 8) || (__GNUC__ == 8 && __GNUC_MINOR__ < 5)
63int main(int argc, char *argv[])
64{
65 printf("[SKIP]\tCompiler does not support CET.\n");
66 return 0;
67}
68#else
69void write_shstk(unsigned long *addr, unsigned long val)
70{
71 asm volatile("wrssq %[val], (%[addr])\n"
72 : "=m" (addr)
73 : [addr] "r" (addr), [val] "r" (val));
74}
75
76static inline unsigned long __attribute__((always_inline)) get_ssp(void)
77{
78 unsigned long ret = 0;
79
80 asm volatile("xor %0, %0; rdsspq %0" : "=r" (ret));
81 return ret;
82}
83
84/*
85 * For use in inline enablement of shadow stack.
86 *
87 * The program can't return from the point where shadow stack gets enabled
88 * because there will be no address on the shadow stack. So it can't use
89 * syscall() for enablement, since it is a function.
90 *
91 * Based on code from nolibc.h. Keep a copy here because this can't pull in all
92 * of nolibc.h.
93 */
94#define ARCH_PRCTL(arg1, arg2) \
95({ \
96 long _ret; \
97 register long _num asm("eax") = __NR_arch_prctl; \
98 register long _arg1 asm("rdi") = (long)(arg1); \
99 register long _arg2 asm("rsi") = (long)(arg2); \
100 \
101 asm volatile ( \
102 "syscall\n" \
103 : "=a"(_ret) \
104 : "r"(_arg1), "r"(_arg2), \
105 "0"(_num) \
106 : "rcx", "r11", "memory", "cc" \
107 ); \
108 _ret; \
109})
110
111void *create_shstk(void *addr)
112{
113 return (void *)syscall(__NR_map_shadow_stack, addr, SS_SIZE, SHADOW_STACK_SET_TOKEN);
114}
115
116void *create_normal_mem(void *addr)
117{
118 return mmap(addr, SS_SIZE, PROT_READ | PROT_WRITE,
119 MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
120}
121
122void free_shstk(void *shstk)
123{
124 munmap(shstk, SS_SIZE);
125}
126
127int reset_shstk(void *shstk)
128{
129 return madvise(shstk, SS_SIZE, MADV_DONTNEED);
130}
131
132void try_shstk(unsigned long new_ssp)
133{
134 unsigned long ssp;
135
136 printf("[INFO]\tnew_ssp = %lx, *new_ssp = %lx\n",
137 new_ssp, *((unsigned long *)new_ssp));
138
139 ssp = get_ssp();
140 printf("[INFO]\tchanging ssp from %lx to %lx\n", ssp, new_ssp);
141
142 asm volatile("rstorssp (%0)\n":: "r" (new_ssp));
143 asm volatile("saveprevssp");
144 printf("[INFO]\tssp is now %lx\n", get_ssp());
145
146 /* Switch back to original shadow stack */
147 ssp -= 8;
148 asm volatile("rstorssp (%0)\n":: "r" (ssp));
149 asm volatile("saveprevssp");
150}
151
152int test_shstk_pivot(void)
153{
154 void *shstk = create_shstk(0);
155
156 if (shstk == MAP_FAILED) {
157 printf("[FAIL]\tError creating shadow stack: %d\n", errno);
158 return 1;
159 }
160 try_shstk((unsigned long)shstk + SS_SIZE - 8);
161 free_shstk(shstk);
162
163 printf("[OK]\tShadow stack pivot\n");
164 return 0;
165}
166
167int test_shstk_faults(void)
168{
169 unsigned long *shstk = create_shstk(0);
170
171 /* Read shadow stack, test if it's zero to not get read optimized out */
172 if (*shstk != 0)
173 goto err;
174
175 /* Wrss memory that was already read. */
176 write_shstk(shstk, 1);
177 if (*shstk != 1)
178 goto err;
179
180 /* Page out memory, so we can wrss it again. */
181 if (reset_shstk((void *)shstk))
182 goto err;
183
184 write_shstk(shstk, 1);
185 if (*shstk != 1)
186 goto err;
187
188 printf("[OK]\tShadow stack faults\n");
189 return 0;
190
191err:
192 return 1;
193}
194
195unsigned long saved_ssp;
196unsigned long saved_ssp_val;
197volatile bool segv_triggered;
198
199void __attribute__((noinline)) violate_ss(void)
200{
201 saved_ssp = get_ssp();
202 saved_ssp_val = *(unsigned long *)saved_ssp;
203
204 /* Corrupt shadow stack */
205 printf("[INFO]\tCorrupting shadow stack\n");
206 write_shstk((void *)saved_ssp, 0);
207}
208
209void segv_handler(int signum, siginfo_t *si, void *uc)
210{
211 printf("[INFO]\tGenerated shadow stack violation successfully\n");
212
213 segv_triggered = true;
214
215 /* Fix shadow stack */
216 write_shstk((void *)saved_ssp, saved_ssp_val);
217}
218
219int test_shstk_violation(void)
220{
221 struct sigaction sa = {};
222
223 sa.sa_sigaction = segv_handler;
224 sa.sa_flags = SA_SIGINFO;
225 if (sigaction(SIGSEGV, &sa, NULL))
226 return 1;
227
228 segv_triggered = false;
229
230 /* Make sure segv_triggered is set before violate_ss() */
231 asm volatile("" : : : "memory");
232
233 violate_ss();
234
235 signal(SIGSEGV, SIG_DFL);
236
237 printf("[OK]\tShadow stack violation test\n");
238
239 return !segv_triggered;
240}
241
242/* Gup test state */
243#define MAGIC_VAL 0x12345678
244bool is_shstk_access;
245void *shstk_ptr;
246int fd;
247
248void reset_test_shstk(void *addr)
249{
250 if (shstk_ptr)
251 free_shstk(shstk_ptr);
252 shstk_ptr = create_shstk(addr);
253}
254
255void test_access_fix_handler(int signum, siginfo_t *si, void *uc)
256{
257 printf("[INFO]\tViolation from %s\n", is_shstk_access ? "shstk access" : "normal write");
258
259 segv_triggered = true;
260
261 /* Fix shadow stack */
262 if (is_shstk_access) {
263 reset_test_shstk(shstk_ptr);
264 return;
265 }
266
267 free_shstk(shstk_ptr);
268 create_normal_mem(shstk_ptr);
269}
270
271bool test_shstk_access(void *ptr)
272{
273 is_shstk_access = true;
274 segv_triggered = false;
275 write_shstk(ptr, MAGIC_VAL);
276
277 asm volatile("" : : : "memory");
278
279 return segv_triggered;
280}
281
282bool test_write_access(void *ptr)
283{
284 is_shstk_access = false;
285 segv_triggered = false;
286 *(unsigned long *)ptr = MAGIC_VAL;
287
288 asm volatile("" : : : "memory");
289
290 return segv_triggered;
291}
292
293bool gup_write(void *ptr)
294{
295 unsigned long val;
296
297 lseek(fd, (unsigned long)ptr, SEEK_SET);
298 if (write(fd, &val, sizeof(val)) < 0)
299 return 1;
300
301 return 0;
302}
303
304bool gup_read(void *ptr)
305{
306 unsigned long val;
307
308 lseek(fd, (unsigned long)ptr, SEEK_SET);
309 if (read(fd, &val, sizeof(val)) < 0)
310 return 1;
311
312 return 0;
313}
314
315int test_gup(void)
316{
317 struct sigaction sa = {};
318 int status;
319 pid_t pid;
320
321 sa.sa_sigaction = test_access_fix_handler;
322 sa.sa_flags = SA_SIGINFO;
323 if (sigaction(SIGSEGV, &sa, NULL))
324 return 1;
325
326 segv_triggered = false;
327
328 fd = open("/proc/self/mem", O_RDWR);
329 if (fd == -1)
330 return 1;
331
332 reset_test_shstk(0);
333 if (gup_read(shstk_ptr))
334 return 1;
335 if (test_shstk_access(shstk_ptr))
336 return 1;
337 printf("[INFO]\tGup read -> shstk access success\n");
338
339 reset_test_shstk(0);
340 if (gup_write(shstk_ptr))
341 return 1;
342 if (test_shstk_access(shstk_ptr))
343 return 1;
344 printf("[INFO]\tGup write -> shstk access success\n");
345
346 reset_test_shstk(0);
347 if (gup_read(shstk_ptr))
348 return 1;
349 if (!test_write_access(shstk_ptr))
350 return 1;
351 printf("[INFO]\tGup read -> write access success\n");
352
353 reset_test_shstk(0);
354 if (gup_write(shstk_ptr))
355 return 1;
356 if (!test_write_access(shstk_ptr))
357 return 1;
358 printf("[INFO]\tGup write -> write access success\n");
359
360 close(fd);
361
362 /* COW/gup test */
363 reset_test_shstk(0);
364 pid = fork();
365 if (!pid) {
366 fd = open("/proc/self/mem", O_RDWR);
367 if (fd == -1)
368 exit(1);
369
370 if (gup_write(shstk_ptr)) {
371 close(fd);
372 exit(1);
373 }
374 close(fd);
375 exit(0);
376 }
377 waitpid(pid, &status, 0);
378 if (WEXITSTATUS(status)) {
379 printf("[FAIL]\tWrite in child failed\n");
380 return 1;
381 }
382 if (*(unsigned long *)shstk_ptr == MAGIC_VAL) {
383 printf("[FAIL]\tWrite in child wrote through to shared memory\n");
384 return 1;
385 }
386
387 printf("[INFO]\tCow gup write -> write access success\n");
388
389 free_shstk(shstk_ptr);
390
391 signal(SIGSEGV, SIG_DFL);
392
393 printf("[OK]\tShadow gup test\n");
394
395 return 0;
396}
397
398int test_mprotect(void)
399{
400 struct sigaction sa = {};
401
402 sa.sa_sigaction = test_access_fix_handler;
403 sa.sa_flags = SA_SIGINFO;
404 if (sigaction(SIGSEGV, &sa, NULL))
405 return 1;
406
407 segv_triggered = false;
408
409 /* mprotect a shadow stack as read only */
410 reset_test_shstk(0);
411 if (mprotect(shstk_ptr, SS_SIZE, PROT_READ) < 0) {
412 printf("[FAIL]\tmprotect(PROT_READ) failed\n");
413 return 1;
414 }
415
416 /* try to wrss it and fail */
417 if (!test_shstk_access(shstk_ptr)) {
418 printf("[FAIL]\tShadow stack access to read-only memory succeeded\n");
419 return 1;
420 }
421
422 /*
423 * The shadow stack was reset above to resolve the fault, make the new one
424 * read-only.
425 */
426 if (mprotect(shstk_ptr, SS_SIZE, PROT_READ) < 0) {
427 printf("[FAIL]\tmprotect(PROT_READ) failed\n");
428 return 1;
429 }
430
431 /* then back to writable */
432 if (mprotect(shstk_ptr, SS_SIZE, PROT_WRITE | PROT_READ) < 0) {
433 printf("[FAIL]\tmprotect(PROT_WRITE) failed\n");
434 return 1;
435 }
436
437 /* then wrss to it and succeed */
438 if (test_shstk_access(shstk_ptr)) {
439 printf("[FAIL]\tShadow stack access to mprotect() writable memory failed\n");
440 return 1;
441 }
442
443 free_shstk(shstk_ptr);
444
445 signal(SIGSEGV, SIG_DFL);
446
447 printf("[OK]\tmprotect() test\n");
448
449 return 0;
450}
451
452char zero[4096];
453
454static void *uffd_thread(void *arg)
455{
456 struct uffdio_copy req;
457 int uffd = *(int *)arg;
458 struct uffd_msg msg;
459 int ret;
460
461 while (1) {
462 ret = read(uffd, &msg, sizeof(msg));
463 if (ret > 0)
464 break;
465 else if (errno == EAGAIN)
466 continue;
467 return (void *)1;
468 }
469
470 req.dst = msg.arg.pagefault.address;
471 req.src = (__u64)zero;
472 req.len = 4096;
473 req.mode = 0;
474
475 if (ioctl(uffd, UFFDIO_COPY, &req))
476 return (void *)1;
477
478 return (void *)0;
479}
480
481int test_userfaultfd(void)
482{
483 struct uffdio_register uffdio_register;
484 struct uffdio_api uffdio_api;
485 struct sigaction sa = {};
486 pthread_t thread;
487 void *res;
488 int uffd;
489
490 sa.sa_sigaction = test_access_fix_handler;
491 sa.sa_flags = SA_SIGINFO;
492 if (sigaction(SIGSEGV, &sa, NULL))
493 return 1;
494
495 uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
496 if (uffd < 0) {
497 printf("[SKIP]\tUserfaultfd unavailable.\n");
498 return 0;
499 }
500
501 reset_test_shstk(0);
502
503 uffdio_api.api = UFFD_API;
504 uffdio_api.features = 0;
505 if (ioctl(uffd, UFFDIO_API, &uffdio_api))
506 goto err;
507
508 uffdio_register.range.start = (__u64)shstk_ptr;
509 uffdio_register.range.len = 4096;
510 uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
511 if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
512 goto err;
513
514 if (pthread_create(&thread, NULL, &uffd_thread, &uffd))
515 goto err;
516
517 reset_shstk(shstk_ptr);
518 test_shstk_access(shstk_ptr);
519
520 if (pthread_join(thread, &res))
521 goto err;
522
523 if (test_shstk_access(shstk_ptr))
524 goto err;
525
526 free_shstk(shstk_ptr);
527
528 signal(SIGSEGV, SIG_DFL);
529
530 if (!res)
531 printf("[OK]\tUserfaultfd test\n");
532 return !!res;
533err:
534 free_shstk(shstk_ptr);
535 close(uffd);
536 signal(SIGSEGV, SIG_DFL);
537 return 1;
538}
539
540/* Simple linked list for keeping track of mappings in test_guard_gap() */
541struct node {
542 struct node *next;
543 void *mapping;
544};
545
546/*
547 * This tests whether mmap will place other mappings in a shadow stack's guard
548 * gap. The steps are:
549 * 1. Finds an empty place by mapping and unmapping something.
550 * 2. Map a shadow stack in the middle of the known empty area.
551 * 3. Map a bunch of PAGE_SIZE mappings. These will use the search down
552 * direction, filling any gaps until it encounters the shadow stack's
553 * guard gap.
554 * 4. When a mapping lands below the shadow stack from step 2, then all
555 * of the above gaps are filled. The search down algorithm will have
556 * looked at the shadow stack gaps.
557 * 5. See if it landed in the gap.
558 */
559int test_guard_gap(void)
560{
561 void *free_area, *shstk, *test_map = (void *)0xFFFFFFFFFFFFFFFF;
562 struct node *head = NULL, *cur;
563
564 free_area = mmap(0, SS_SIZE * 3, PROT_READ | PROT_WRITE,
565 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
566 munmap(free_area, SS_SIZE * 3);
567
568 shstk = create_shstk(free_area + SS_SIZE);
569 if (shstk == MAP_FAILED)
570 return 1;
571
572 while (test_map > shstk) {
573 test_map = mmap(0, PAGE_SIZE, PROT_READ | PROT_WRITE,
574 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
575 if (test_map == MAP_FAILED)
576 return 1;
577 cur = malloc(sizeof(*cur));
578 cur->mapping = test_map;
579
580 cur->next = head;
581 head = cur;
582 }
583
584 while (head) {
585 cur = head;
586 head = cur->next;
587 munmap(cur->mapping, PAGE_SIZE);
588 free(cur);
589 }
590
591 free_shstk(shstk);
592
593 if (shstk - test_map - PAGE_SIZE != PAGE_SIZE)
594 return 1;
595
596 printf("[OK]\tGuard gap test\n");
597
598 return 0;
599}
600
601/*
602 * Too complicated to pull it out of the 32 bit header, but also get the
603 * 64 bit one needed above. Just define a copy here.
604 */
605#define __NR_compat_sigaction 67
606
607/*
608 * Call 32 bit signal handler to get 32 bit signals ABI. Make sure
609 * to push the registers that will get clobbered.
610 */
611int sigaction32(int signum, const struct sigaction *restrict act,
612 struct sigaction *restrict oldact)
613{
614 register long syscall_reg asm("eax") = __NR_compat_sigaction;
615 register long signum_reg asm("ebx") = signum;
616 register long act_reg asm("ecx") = (long)act;
617 register long oldact_reg asm("edx") = (long)oldact;
618 int ret = 0;
619
620 asm volatile ("int $0x80;"
621 : "=a"(ret), "=m"(oldact)
622 : "r"(syscall_reg), "r"(signum_reg), "r"(act_reg),
623 "r"(oldact_reg)
624 : "r8", "r9", "r10", "r11"
625 );
626
627 return ret;
628}
629
630sigjmp_buf jmp_buffer;
631
632void segv_gp_handler(int signum, siginfo_t *si, void *uc)
633{
634 segv_triggered = true;
635
636 /*
637 * To work with old glibc, this can't rely on siglongjmp working with
638 * shadow stack enabled, so disable shadow stack before siglongjmp().
639 */
640 ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK);
641 siglongjmp(jmp_buffer, -1);
642}
643
644/*
645 * Transition to 32 bit mode and check that a #GP triggers a segfault.
646 */
647int test_32bit(void)
648{
649 struct sigaction sa = {};
650 struct sigaction *sa32;
651
652 /* Create sigaction in 32 bit address range */
653 sa32 = mmap(0, 4096, PROT_READ | PROT_WRITE,
654 MAP_32BIT | MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
655 sa32->sa_flags = SA_SIGINFO;
656
657 sa.sa_sigaction = segv_gp_handler;
658 sa.sa_flags = SA_SIGINFO;
659 if (sigaction(SIGSEGV, &sa, NULL))
660 return 1;
661
662
663 segv_triggered = false;
664
665 /* Make sure segv_triggered is set before triggering the #GP */
666 asm volatile("" : : : "memory");
667
668 /*
669 * Set handler to somewhere in 32 bit address space
670 */
671 sa32->sa_handler = (void *)sa32;
672 if (sigaction32(SIGUSR1, sa32, NULL))
673 return 1;
674
675 if (!sigsetjmp(jmp_buffer, 1))
676 raise(SIGUSR1);
677
678 if (segv_triggered)
679 printf("[OK]\t32 bit test\n");
680
681 return !segv_triggered;
682}
683
684void segv_handler_ptrace(int signum, siginfo_t *si, void *uc)
685{
686 /* The SSP adjustment caused a segfault. */
687 exit(0);
688}
689
690int test_ptrace(void)
691{
692 unsigned long saved_ssp, ssp = 0;
693 struct sigaction sa= {};
694 struct iovec iov;
695 int status;
696 int pid;
697
698 iov.iov_base = &ssp;
699 iov.iov_len = sizeof(ssp);
700
701 pid = fork();
702 if (!pid) {
703 ssp = get_ssp();
704
705 sa.sa_sigaction = segv_handler_ptrace;
706 sa.sa_flags = SA_SIGINFO;
707 if (sigaction(SIGSEGV, &sa, NULL))
708 return 1;
709
710 ptrace(PTRACE_TRACEME, NULL, NULL, NULL);
711 /*
712 * The parent will tweak the SSP and return from this function
713 * will #CP.
714 */
715 raise(SIGTRAP);
716
717 exit(1);
718 }
719
720 while (waitpid(pid, &status, 0) != -1 && WSTOPSIG(status) != SIGTRAP);
721
722 if (ptrace(PTRACE_GETREGSET, pid, NT_X86_SHSTK, &iov)) {
723 printf("[INFO]\tFailed to PTRACE_GETREGS\n");
724 goto out_kill;
725 }
726
727 if (!ssp) {
728 printf("[INFO]\tPtrace child SSP was 0\n");
729 goto out_kill;
730 }
731
732 saved_ssp = ssp;
733
734 iov.iov_len = 0;
735 if (!ptrace(PTRACE_SETREGSET, pid, NT_X86_SHSTK, &iov)) {
736 printf("[INFO]\tToo small size accepted via PTRACE_SETREGS\n");
737 goto out_kill;
738 }
739
740 iov.iov_len = sizeof(ssp) + 1;
741 if (!ptrace(PTRACE_SETREGSET, pid, NT_X86_SHSTK, &iov)) {
742 printf("[INFO]\tToo large size accepted via PTRACE_SETREGS\n");
743 goto out_kill;
744 }
745
746 ssp += 1;
747 if (!ptrace(PTRACE_SETREGSET, pid, NT_X86_SHSTK, &iov)) {
748 printf("[INFO]\tUnaligned SSP written via PTRACE_SETREGS\n");
749 goto out_kill;
750 }
751
752 ssp = 0xFFFFFFFFFFFF0000;
753 if (!ptrace(PTRACE_SETREGSET, pid, NT_X86_SHSTK, &iov)) {
754 printf("[INFO]\tKernel range SSP written via PTRACE_SETREGS\n");
755 goto out_kill;
756 }
757
758 /*
759 * Tweak the SSP so the child with #CP when it resumes and returns
760 * from raise()
761 */
762 ssp = saved_ssp + 8;
763 iov.iov_len = sizeof(ssp);
764 if (ptrace(PTRACE_SETREGSET, pid, NT_X86_SHSTK, &iov)) {
765 printf("[INFO]\tFailed to PTRACE_SETREGS\n");
766 goto out_kill;
767 }
768
769 if (ptrace(PTRACE_DETACH, pid, NULL, NULL)) {
770 printf("[INFO]\tFailed to PTRACE_DETACH\n");
771 goto out_kill;
772 }
773
774 waitpid(pid, &status, 0);
775 if (WEXITSTATUS(status))
776 return 1;
777
778 printf("[OK]\tPtrace test\n");
779 return 0;
780
781out_kill:
782 kill(pid, SIGKILL);
783 return 1;
784}
785
786int main(int argc, char *argv[])
787{
788 int ret = 0;
789
790 if (ARCH_PRCTL(ARCH_SHSTK_ENABLE, ARCH_SHSTK_SHSTK)) {
791 printf("[SKIP]\tCould not enable Shadow stack\n");
792 return 1;
793 }
794
795 if (ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK)) {
796 ret = 1;
797 printf("[FAIL]\tDisabling shadow stack failed\n");
798 }
799
800 if (ARCH_PRCTL(ARCH_SHSTK_ENABLE, ARCH_SHSTK_SHSTK)) {
801 printf("[SKIP]\tCould not re-enable Shadow stack\n");
802 return 1;
803 }
804
805 if (ARCH_PRCTL(ARCH_SHSTK_ENABLE, ARCH_SHSTK_WRSS)) {
806 printf("[SKIP]\tCould not enable WRSS\n");
807 ret = 1;
808 goto out;
809 }
810
811 /* Should have succeeded if here, but this is a test, so double check. */
812 if (!get_ssp()) {
813 printf("[FAIL]\tShadow stack disabled\n");
814 return 1;
815 }
816
817 if (test_shstk_pivot()) {
818 ret = 1;
819 printf("[FAIL]\tShadow stack pivot\n");
820 goto out;
821 }
822
823 if (test_shstk_faults()) {
824 ret = 1;
825 printf("[FAIL]\tShadow stack fault test\n");
826 goto out;
827 }
828
829 if (test_shstk_violation()) {
830 ret = 1;
831 printf("[FAIL]\tShadow stack violation test\n");
832 goto out;
833 }
834
835 if (test_gup()) {
836 ret = 1;
837 printf("[FAIL]\tShadow shadow stack gup\n");
838 goto out;
839 }
840
841 if (test_mprotect()) {
842 ret = 1;
843 printf("[FAIL]\tShadow shadow mprotect test\n");
844 goto out;
845 }
846
847 if (test_userfaultfd()) {
848 ret = 1;
849 printf("[FAIL]\tUserfaultfd test\n");
850 goto out;
851 }
852
853 if (test_guard_gap()) {
854 ret = 1;
855 printf("[FAIL]\tGuard gap test\n");
856 goto out;
857 }
858
859 if (test_ptrace()) {
860 ret = 1;
861 printf("[FAIL]\tptrace test\n");
862 }
863
864 if (test_32bit()) {
865 ret = 1;
866 printf("[FAIL]\t32 bit test\n");
867 goto out;
868 }
869
870 return ret;
871
872out:
873 /*
874 * Disable shadow stack before the function returns, or there will be a
875 * shadow stack violation.
876 */
877 if (ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK)) {
878 ret = 1;
879 printf("[FAIL]\tDisabling shadow stack failed\n");
880 }
881
882 return ret;
883}
884#endif

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t