History

Merge tag 'x86_shstk_for_6.6-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull x86 shadow stack support from Dave Hansen: "This is the long awaited x86 shadow stack support, part of Intel's Control-flow Enforcement Technology (CET). CET consists of two related security features: shadow stacks and indirect branch tracking. This series implements just the shadow stack part of this feature, and just for userspace. The main use case for shadow stack is providing protection against return oriented programming attacks. It works by maintaining a secondary (shadow) stack using a special memory type that has protections against modification. When executing a CALL instruction, the processor pushes the return address to both the normal stack and to the special permission shadow stack. Upon RET, the processor pops the shadow stack copy and compares it to the normal stack copy. For more information, refer to the links below for the earlier versions of this patch set" Link: https://lore.kernel.org/lkml/20220130211838.8382-1-rick.p.edgecombe@intel.com/ Link: https://lore.kernel.org/lkml/20230613001108.3040476-1-rick.p.edgecombe@intel.com/ * tag 'x86_shstk_for_6.6-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (47 commits) x86/shstk: Change order of __user in type x86/ibt: Convert IBT selftest to asm x86/shstk: Don't retry vm_munmap() on -EINTR x86/kbuild: Fix Documentation/ reference x86/shstk: Move arch detail comment out of core mm x86/shstk: Add ARCH_SHSTK_STATUS x86/shstk: Add ARCH_SHSTK_UNLOCK x86: Add PTRACE interface for shadow stack selftests/x86: Add shadow stack test x86/cpufeatures: Enable CET CR4 bit for shadow stack x86/shstk: Wire in shadow stack interface x86: Expose thread features in /proc/$PID/status x86/shstk: Support WRSS for userspace x86/shstk: Introduce map_shadow_stack syscall x86/shstk: Check that signal frame is shadow stack mem x86/shstk: Check that SSP is aligned on sigreturn x86/shstk: Handle signals for shadow stack x86/shstk: Introduce routines modifying shstk x86/shstk: Handle thread shadow stack x86/shstk: Add user-mode shadow stack support ...

Linus Torvalds committed 3 years ago

Commit df577212 parents b97d64c 1fe428d118 files · +2790 −308Browse files

Unified · Split

118 files
  1. Documentation/arch/x86/index.rst +1 −0
  2. Documentation/arch/x86/shstk.rst +179 −0
  3. Documentation/filesystems/proc.rst +1 −0
  4. Documentation/mm/arch_pgtable_helpers.rst +10 −2
  5. arch/Kconfig +8 −0
  6. arch/alpha/include/asm/pgtable.h +1 −1
  7. arch/arc/include/asm/hugepage.h +1 −1
  8. arch/arc/include/asm/pgtable-bits-arcv2.h +1 −1
  9. arch/arm/include/asm/pgtable-3level.h +1 −1
  10. arch/arm/include/asm/pgtable.h +1 −1
  11. arch/arm/kernel/signal.c +1 −1
  12. arch/arm64/include/asm/pgtable.h +2 −2
  13. arch/arm64/kernel/signal.c +1 −1
  14. arch/arm64/kernel/signal32.c +1 −1
  15. arch/arm64/mm/trans_pgd.c +2 −2
  16. arch/csky/include/asm/pgtable.h +1 −1
  17. arch/hexagon/include/asm/pgtable.h +1 −1
  18. arch/ia64/include/asm/pgtable.h +1 −1
  19. arch/loongarch/include/asm/pgtable.h +2 −2
  20. arch/m68k/include/asm/mcf_pgtable.h +1 −1
  21. arch/m68k/include/asm/motorola_pgtable.h +1 −1
  22. arch/m68k/include/asm/sun3_pgtable.h +1 −1
  23. arch/microblaze/include/asm/pgtable.h +1 −1
  24. arch/mips/include/asm/pgtable.h +3 −3
  25. arch/nios2/include/asm/pgtable.h +1 −1
  26. arch/openrisc/include/asm/pgtable.h +1 −1
  27. arch/parisc/include/asm/pgtable.h +1 −1
  28. arch/powerpc/include/asm/book3s/32/pgtable.h +1 −1
  29. arch/powerpc/include/asm/book3s/64/pgtable.h +2 −2
  30. arch/powerpc/include/asm/nohash/32/pgtable.h +2 −2
  31. arch/powerpc/include/asm/nohash/32/pte-8xx.h +2 −2
  32. arch/powerpc/include/asm/nohash/64/pgtable.h +1 −1
  33. arch/riscv/include/asm/pgtable.h +3 −3
  34. arch/s390/Kconfig +1 −0
  35. arch/s390/include/asm/hugetlb.h +1 −1
  36. arch/s390/include/asm/pgtable.h +2 −2
  37. arch/s390/mm/pageattr.c +2 −2
  38. arch/sh/include/asm/pgtable_32.h +2 −2
  39. arch/sparc/include/asm/pgtable_32.h +1 −1
  40. arch/sparc/include/asm/pgtable_64.h +3 −3
  41. arch/sparc/kernel/signal32.c +1 −1
  42. arch/sparc/kernel/signal_64.c +1 −1
  43. arch/um/include/asm/pgtable.h +1 −1
  44. arch/x86/Kconfig +24 −0
  45. arch/x86/Kconfig.assembler +5 −0
  46. arch/x86/entry/syscalls/syscall_64.tbl +1 −0
  47. arch/x86/include/asm/cpufeatures.h +2 −0
  48. arch/x86/include/asm/disabled-features.h +14 −2
  49. arch/x86/include/asm/fpu/api.h +9 −0
  50. arch/x86/include/asm/fpu/regset.h +4 −3
  51. arch/x86/include/asm/fpu/sched.h +2 −1
  52. arch/x86/include/asm/fpu/types.h +14 −2
  53. arch/x86/include/asm/fpu/xstate.h +4 −2
  54. arch/x86/include/asm/idtentry.h +1 −1
  55. arch/x86/include/asm/mmu_context.h +2 −0
  56. arch/x86/include/asm/pgtable.h +266 −36
  57. arch/x86/include/asm/pgtable_types.h +36 −8
  58. arch/x86/include/asm/processor.h +8 −0
  59. arch/x86/include/asm/shstk.h +38 −0
  60. arch/x86/include/asm/special_insns.h +13 −0
  61. arch/x86/include/asm/tlbflush.h +2 −1
  62. arch/x86/include/asm/trap_pf.h +2 −0
  63. arch/x86/include/asm/traps.h +14 −1
  64. arch/x86/include/uapi/asm/mman.h +4 −0
  65. arch/x86/include/uapi/asm/prctl.h +12 −0
  66. arch/x86/kernel/Makefile +5 −0
  67. arch/x86/kernel/cet.c +131 −0
  68. arch/x86/kernel/cpu/common.c +27 −8
  69. arch/x86/kernel/cpu/cpuid-deps.c +1 −0
  70. arch/x86/kernel/cpu/proc.c +23 −0
  71. arch/x86/kernel/fpu/core.c +53 −1
  72. arch/x86/kernel/fpu/regset.c +81 −0
  73. arch/x86/kernel/fpu/xstate.c +43 −47
  74. arch/x86/kernel/ibt_selftest.S +17 −0
  75. arch/x86/kernel/idt.c +1 −1
  76. arch/x86/kernel/process.c +20 −1
  77. arch/x86/kernel/process_64.c +8 −0
  78. arch/x86/kernel/ptrace.c +12 −0
  79. arch/x86/kernel/shstk.c +550 −0
  80. arch/x86/kernel/signal.c +1 −0
  81. arch/x86/kernel/signal_32.c +1 −1
  82. arch/x86/kernel/signal_64.c +7 −1
  83. arch/x86/kernel/sys_x86_64.c +5 −1
  84. arch/x86/kernel/traps.c +0 −87
  85. arch/x86/mm/fault.c +22 −0
  86. arch/x86/mm/pat/set_memory.c +2 −2
  87. arch/x86/mm/pgtable.c +40 −0
  88. arch/x86/xen/enlighten_pv.c +1 −1
  89. arch/x86/xen/mmu_pv.c +1 −1
  90. arch/x86/xen/xen-asm.S +1 −1
  91. arch/xtensa/include/asm/pgtable.h +1 −1
  92. fs/aio.c +1 −1
  93. fs/proc/array.c +6 −0
  94. fs/proc/task_mmu.c +3 −0
  95. include/asm-generic/hugetlb.h +1 −1
  96. include/linux/mm.h +39 −8
  97. include/linux/mman.h +4 −0
  98. include/linux/pgtable.h +28 −0
  99. include/linux/proc_fs.h +1 −0
  100. include/linux/syscalls.h +1 −0
  101. include/uapi/asm-generic/siginfo.h +2 −1
  102. include/uapi/linux/elf.h +2 −0
  103. ipc/shm.c +1 −1
  104. kernel/sys_ni.c +1 −0
  105. mm/debug_vm_pgtable.c +6 −6
  106. mm/gup.c +1 −1
  107. mm/huge_memory.c +6 −5
  108. mm/internal.h +2 −2
  109. mm/memory.c +3 −2
  110. mm/migrate.c +1 −1
  111. mm/migrate_device.c +1 −1
  112. mm/mmap.c +7 −7
  113. mm/mprotect.c +1 −1
  114. mm/nommu.c +2 −2
  115. mm/userfaultfd.c +1 −1
  116. mm/util.c +1 −1
  117. tools/testing/selftests/x86/Makefile +1 −1
  118. tools/testing/selftests/x86/test_shadow_stack.c +884 −0
Documentation/arch/x86/index.rst+1−0
@@ -22,6 +22,7 @@ x86-specific Documentation · 21 unmodified lines
2222 mtrr
2323 pat
2424 intel-hfi
25+ shstk
2526 iommu
2627 intel_txt
2728 amd-memory-encryption
Documentation/arch/x86/shstk.rst+179−0
@@ -0,0 +1,179 @@
1+.. SPDX-License-Identifier: GPL-2.0
2+
3+======================================================
4+Control-flow Enforcement Technology (CET) Shadow Stack
5+======================================================
6+
7+CET Background
8+==============
9+
10+Control-flow Enforcement Technology (CET) covers several related x86 processor
11+features that provide protection against control flow hijacking attacks. CET
12+can protect both applications and the kernel.
13+
14+CET introduces shadow stack and indirect branch tracking (IBT). A shadow stack
15+is a secondary stack allocated from memory which cannot be directly modified by
16+applications. When executing a CALL instruction, the processor pushes the
17+return address to both the normal stack and the shadow stack. Upon
18+function return, the processor pops the shadow stack copy and compares it
19+to the normal stack copy. If the two differ, the processor raises a
20+control-protection fault. IBT verifies indirect CALL/JMP targets are intended
21+as marked by the compiler with 'ENDBR' opcodes. Not all CPU's have both Shadow
22+Stack and Indirect Branch Tracking. Today in the 64-bit kernel, only userspace
23+shadow stack and kernel IBT are supported.
24+
25+Requirements to use Shadow Stack
26+================================
27+
28+To use userspace shadow stack you need HW that supports it, a kernel
29+configured with it and userspace libraries compiled with it.
30+
31+The kernel Kconfig option is X86_USER_SHADOW_STACK. When compiled in, shadow
32+stacks can be disabled at runtime with the kernel parameter: nousershstk.
33+
34+To build a user shadow stack enabled kernel, Binutils v2.29 or LLVM v6 or later
35+are required.
36+
37+At run time, /proc/cpuinfo shows CET features if the processor supports
38+CET. "user_shstk" means that userspace shadow stack is supported on the current
39+kernel and HW.
40+
41+Application Enabling
42+====================
43+
44+An application's CET capability is marked in its ELF note and can be verified
45+from readelf/llvm-readelf output::
46+
47+ readelf -n <application> | grep -a SHSTK
48+ properties: x86 feature: SHSTK
49+
50+The kernel does not process these applications markers directly. Applications
51+or loaders must enable CET features using the interface described in section 4.
52+Typically this would be done in dynamic loader or static runtime objects, as is
53+the case in GLIBC.
54+
55+Enabling arch_prctl()'s
56+=======================
57+
58+Elf features should be enabled by the loader using the below arch_prctl's. They
59+are only supported in 64 bit user applications. These operate on the features
60+on a per-thread basis. The enablement status is inherited on clone, so if the
61+feature is enabled on the first thread, it will propagate to all the thread's
62+in an app.
63+
64+arch_prctl(ARCH_SHSTK_ENABLE, unsigned long feature)
65+ Enable a single feature specified in 'feature'. Can only operate on
66+ one feature at a time.
67+
68+arch_prctl(ARCH_SHSTK_DISABLE, unsigned long feature)
69+ Disable a single feature specified in 'feature'. Can only operate on
70+ one feature at a time.
71+
72+arch_prctl(ARCH_SHSTK_LOCK, unsigned long features)
73+ Lock in features at their current enabled or disabled status. 'features'
74+ is a mask of all features to lock. All bits set are processed, unset bits
75+ are ignored. The mask is ORed with the existing value. So any feature bits
76+ set here cannot be enabled or disabled afterwards.
77+
78+arch_prctl(ARCH_SHSTK_UNLOCK, unsigned long features)
79+ Unlock features. 'features' is a mask of all features to unlock. All
80+ bits set are processed, unset bits are ignored. Only works via ptrace.
81+
82+arch_prctl(ARCH_SHSTK_STATUS, unsigned long addr)
83+ Copy the currently enabled features to the address passed in addr. The
84+ features are described using the bits passed into the others in
85+ 'features'.
86+
87+The return values are as follows. On success, return 0. On error, errno can
88+be::
89+
90+ -EPERM if any of the passed feature are locked.
91+ -ENOTSUPP if the feature is not supported by the hardware or
92+ kernel.
93+ -EINVAL arguments (non existing feature, etc)
94+ -EFAULT if could not copy information back to userspace
95+
96+The feature's bits supported are::
97+
98+ ARCH_SHSTK_SHSTK - Shadow stack
99+ ARCH_SHSTK_WRSS - WRSS
100+
101+Currently shadow stack and WRSS are supported via this interface. WRSS
102+can only be enabled with shadow stack, and is automatically disabled
103+if shadow stack is disabled.
104+
105+Proc Status
106+===========
107+To check if an application is actually running with shadow stack, the
108+user can read the /proc/$PID/status. It will report "wrss" or "shstk"
109+depending on what is enabled. The lines look like this::
110+
111+ x86_Thread_features: shstk wrss
112+ x86_Thread_features_locked: shstk wrss
113+
114+Implementation of the Shadow Stack
115+==================================
116+
117+Shadow Stack Size
118+-----------------
119+
120+A task's shadow stack is allocated from memory to a fixed size of
121+MIN(RLIMIT_STACK, 4 GB). In other words, the shadow stack is allocated to
122+the maximum size of the normal stack, but capped to 4 GB. In the case
123+of the clone3 syscall, there is a stack size passed in and shadow stack
124+uses this instead of the rlimit.
125+
126+Signal
127+------
128+
129+The main program and its signal handlers use the same shadow stack. Because
130+the shadow stack stores only return addresses, a large shadow stack covers
131+the condition that both the program stack and the signal alternate stack run
132+out.
133+
134+When a signal happens, the old pre-signal state is pushed on the stack. When
135+shadow stack is enabled, the shadow stack specific state is pushed onto the
136+shadow stack. Today this is only the old SSP (shadow stack pointer), pushed
137+in a special format with bit 63 set. On sigreturn this old SSP token is
138+verified and restored by the kernel. The kernel will also push the normal
139+restorer address to the shadow stack to help userspace avoid a shadow stack
140+violation on the sigreturn path that goes through the restorer.
141+
142+So the shadow stack signal frame format is as follows::
143+
144+ |1...old SSP| - Pointer to old pre-signal ssp in sigframe token format
145+ (bit 63 set to 1)
146+ | ...| - Other state may be added in the future
147+
148+
149+32 bit ABI signals are not supported in shadow stack processes. Linux prevents
150+32 bit execution while shadow stack is enabled by the allocating shadow stacks
151+outside of the 32 bit address space. When execution enters 32 bit mode, either
152+via far call or returning to userspace, a #GP is generated by the hardware
153+which, will be delivered to the process as a segfault. When transitioning to
154+userspace the register's state will be as if the userspace ip being returned to
155+caused the segfault.
156+
157+Fork
158+----
159+
160+The shadow stack's vma has VM_SHADOW_STACK flag set; its PTEs are required
161+to be read-only and dirty. When a shadow stack PTE is not RO and dirty, a
162+shadow access triggers a page fault with the shadow stack access bit set
163+in the page fault error code.
164+
165+When a task forks a child, its shadow stack PTEs are copied and both the
166+parent's and the child's shadow stack PTEs are cleared of the dirty bit.
167+Upon the next shadow stack access, the resulting shadow stack page fault
168+is handled by page copy/re-use.
169+
170+When a pthread child is created, the kernel allocates a new shadow stack
171+for the new thread. New shadow stack creation behaves like mmap() with respect
172+to ASLR behavior. Similarly, on thread exit the thread's shadow stack is
173+disabled.
174+
175+Exec
176+----
177+
178+On exec, shadow stack features are disabled by the kernel. At which point,
179+userspace can choose to re-enable, or lock them.
Documentation/filesystems/proc.rst+1−0
@@ -566,6 +566,7 @@ encoded manner. The codes are the following: · 565 unmodified lines
566566 mt arm64 MTE allocation tags are enabled
567567 um userfaultfd missing tracking
568568 uw userfaultfd wr-protect tracking
569+ ss shadow stack page
569570 == =======================================
570571
571572 Note that there is no guarantee that every flag and associated mnemonic will
Documentation/mm/arch_pgtable_helpers.rst+10−2
@@ -46,7 +46,11 @@ PTE Page Table Helpers · 45 unmodified lines
4646 +---------------------------+--------------------------------------------------+
4747 | pte_mkclean | Creates a clean PTE |
4848 +---------------------------+--------------------------------------------------+
49−| pte_mkwrite | Creates a writable PTE |
49+| pte_mkwrite | Creates a writable PTE of the type specified by |
50+| | the VMA. |
51++---------------------------+--------------------------------------------------+
52+| pte_mkwrite_novma | Creates a writable PTE, of the conventional type |
53+| | of writable. |
5054 +---------------------------+--------------------------------------------------+
5155 | pte_wrprotect | Creates a write protected PTE |
5256 +---------------------------+--------------------------------------------------+
@@ -118,7 +122,11 @@ PMD Page Table Helpers · 65 unmodified lines
118122 +---------------------------+--------------------------------------------------+
119123 | pmd_mkclean | Creates a clean PMD |
120124 +---------------------------+--------------------------------------------------+
121−| pmd_mkwrite | Creates a writable PMD |
125+| pmd_mkwrite | Creates a writable PMD of the type specified by |
126+| | the VMA. |
127++---------------------------+--------------------------------------------------+
128+| pmd_mkwrite_novma | Creates a writable PMD, of the conventional type |
129+| | of writable. |
122130 +---------------------------+--------------------------------------------------+
123131 | pmd_wrprotect | Creates a write protected PMD |
124132 +---------------------------+--------------------------------------------------+
arch/Kconfig+8−0
@@ -931,6 +931,14 @@ config HAVE_ARCH_HUGE_VMALLOC · 930 unmodified lines
931931 config ARCH_WANT_HUGE_PMD_SHARE
932932 bool
933933
934+# Archs that want to use pmd_mkwrite on kernel memory need it defined even
935+# if there are no userspace memory management features that use it
936+config ARCH_WANT_KERNEL_PMD_MKWRITE
937+ bool
938+
939+config ARCH_WANT_PMD_MKWRITE
940+ def_bool TRANSPARENT_HUGEPAGE || ARCH_WANT_KERNEL_PMD_MKWRITE
941+
934942 config HAVE_ARCH_SOFT_DIRTY
935943 bool
936944
arch/alpha/include/asm/pgtable.h+1−1
@@ -256,7 +256,7 @@ extern inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; · 255 unmodified lines
256256 extern inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) |= _PAGE_FOW; return pte; }
257257 extern inline pte_t pte_mkclean(pte_t pte) { pte_val(pte) &= ~(__DIRTY_BITS); return pte; }
258258 extern inline pte_t pte_mkold(pte_t pte) { pte_val(pte) &= ~(__ACCESS_BITS); return pte; }
259−extern inline pte_t pte_mkwrite(pte_t pte) { pte_val(pte) &= ~_PAGE_FOW; return pte; }
259+extern inline pte_t pte_mkwrite_novma(pte_t pte){ pte_val(pte) &= ~_PAGE_FOW; return pte; }
260260 extern inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= __DIRTY_BITS; return pte; }
261261 extern inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= __ACCESS_BITS; return pte; }
262262
arch/arc/include/asm/hugepage.h+1−1
@@ -21,7 +21,7 @@ static inline pmd_t pte_pmd(pte_t pte) · 20 unmodified lines
2121 }
2222
2323 #define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
24−#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))
24+#define pmd_mkwrite_novma(pmd) pte_pmd(pte_mkwrite_novma(pmd_pte(pmd)))
2525 #define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
2626 #define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
2727 #define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
arch/arc/include/asm/pgtable-bits-arcv2.h+1−1
@@ -87,7 +87,7 @@ · 86 unmodified lines
8787
8888 PTE_BIT_FUNC(mknotpresent, &= ~(_PAGE_PRESENT));
8989 PTE_BIT_FUNC(wrprotect, &= ~(_PAGE_WRITE));
90−PTE_BIT_FUNC(mkwrite, |= (_PAGE_WRITE));
90+PTE_BIT_FUNC(mkwrite_novma, |= (_PAGE_WRITE));
9191 PTE_BIT_FUNC(mkclean, &= ~(_PAGE_DIRTY));
9292 PTE_BIT_FUNC(mkdirty, |= (_PAGE_DIRTY));
9393 PTE_BIT_FUNC(mkold, &= ~(_PAGE_ACCESSED));
arch/arm/include/asm/pgtable-3level.h+1−1
@@ -202,7 +202,7 @@ static inline pmd_t pmd_##fn(pmd_t pmd) { pmd_val(pmd) op; return pmd; } · 201 unmodified lines
202202
203203 PMD_BIT_FUNC(wrprotect, |= L_PMD_SECT_RDONLY);
204204 PMD_BIT_FUNC(mkold, &= ~PMD_SECT_AF);
205−PMD_BIT_FUNC(mkwrite, &= ~L_PMD_SECT_RDONLY);
205+PMD_BIT_FUNC(mkwrite_novma, &= ~L_PMD_SECT_RDONLY);
206206 PMD_BIT_FUNC(mkdirty, |= L_PMD_SECT_DIRTY);
207207 PMD_BIT_FUNC(mkclean, &= ~L_PMD_SECT_DIRTY);
208208 PMD_BIT_FUNC(mkyoung, |= PMD_SECT_AF);
arch/arm/include/asm/pgtable.h+1−1
@@ -228,7 +228,7 @@ static inline pte_t pte_wrprotect(pte_t pte) · 227 unmodified lines
228228 return set_pte_bit(pte, __pgprot(L_PTE_RDONLY));
229229 }
230230
231−static inline pte_t pte_mkwrite(pte_t pte)
231+static inline pte_t pte_mkwrite_novma(pte_t pte)
232232 {
233233 return clear_pte_bit(pte, __pgprot(L_PTE_RDONLY));
234234 }
arch/arm/kernel/signal.c+1−1
@@ -682,7 +682,7 @@ asmlinkage void do_rseq_syscall(struct pt_regs *regs) · 681 unmodified lines
682682 */
683683 static_assert(NSIGILL == 11);
684684 static_assert(NSIGFPE == 15);
685−static_assert(NSIGSEGV == 9);
685+static_assert(NSIGSEGV == 10);
686686 static_assert(NSIGBUS == 5);
687687 static_assert(NSIGTRAP == 6);
688688 static_assert(NSIGCHLD == 6);
arch/arm64/include/asm/pgtable.h+2−2
@@ -181,7 +181,7 @@ static inline pmd_t set_pmd_bit(pmd_t pmd, pgprot_t prot) · 180 unmodified lines
181181 return pmd;
182182 }
183183
184−static inline pte_t pte_mkwrite(pte_t pte)
184+static inline pte_t pte_mkwrite_novma(pte_t pte)
185185 {
186186 pte = set_pte_bit(pte, __pgprot(PTE_WRITE));
187187 pte = clear_pte_bit(pte, __pgprot(PTE_RDONLY));
@@ -487,7 +487,7 @@ static inline int pmd_trans_huge(pmd_t pmd) · 299 unmodified lines
487487 #define pmd_cont(pmd) pte_cont(pmd_pte(pmd))
488488 #define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
489489 #define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
490−#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))
490+#define pmd_mkwrite_novma(pmd) pte_pmd(pte_mkwrite_novma(pmd_pte(pmd)))
491491 #define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
492492 #define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
493493 #define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
arch/arm64/kernel/signal.c+1−1
@@ -1344,7 +1344,7 @@ void __init minsigstksz_setup(void) · 1343 unmodified lines
13441344 */
13451345 static_assert(NSIGILL == 11);
13461346 static_assert(NSIGFPE == 15);
1347−static_assert(NSIGSEGV == 9);
1347+static_assert(NSIGSEGV == 10);
13481348 static_assert(NSIGBUS == 5);
13491349 static_assert(NSIGTRAP == 6);
13501350 static_assert(NSIGCHLD == 6);
arch/arm64/kernel/signal32.c+1−1
@@ -460,7 +460,7 @@ void compat_setup_restart_syscall(struct pt_regs *regs) · 459 unmodified lines
460460 */
461461 static_assert(NSIGILL == 11);
462462 static_assert(NSIGFPE == 15);
463−static_assert(NSIGSEGV == 9);
463+static_assert(NSIGSEGV == 10);
464464 static_assert(NSIGBUS == 5);
465465 static_assert(NSIGTRAP == 6);
466466 static_assert(NSIGCHLD == 6);
arch/arm64/mm/trans_pgd.c+2−2
@@ -41,7 +41,7 @@ static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) · 40 unmodified lines
4141 * read only (code, rodata). Clear the RDONLY bit from
4242 * the temporary mappings we use during restore.
4343 */
44− set_pte(dst_ptep, pte_mkwrite(pte));
44+ set_pte(dst_ptep, pte_mkwrite_novma(pte));
4545 } else if ((debug_pagealloc_enabled() ||
4646 is_kfence_address((void *)addr)) && !pte_none(pte)) {
4747 /*
@@ -55,7 +55,7 @@ static void _copy_pte(pte_t *dst_ptep, pte_t *src_ptep, unsigned long addr) · 7 unmodified lines
5555 */
5656 BUG_ON(!pfn_valid(pte_pfn(pte)));
5757
58− set_pte(dst_ptep, pte_mkpresent(pte_mkwrite(pte)));
58+ set_pte(dst_ptep, pte_mkpresent(pte_mkwrite_novma(pte)));
5959 }
6060 }
6161
arch/csky/include/asm/pgtable.h+1−1
@@ -176,7 +176,7 @@ static inline pte_t pte_mkold(pte_t pte) · 175 unmodified lines
176176 return pte;
177177 }
178178
179−static inline pte_t pte_mkwrite(pte_t pte)
179+static inline pte_t pte_mkwrite_novma(pte_t pte)
180180 {
181181 pte_val(pte) |= _PAGE_WRITE;
182182 if (pte_val(pte) & _PAGE_MODIFIED)
arch/hexagon/include/asm/pgtable.h+1−1
@@ -300,7 +300,7 @@ static inline pte_t pte_wrprotect(pte_t pte) · 299 unmodified lines
300300 }
301301
302302 /* pte_mkwrite - mark page as writable */
303−static inline pte_t pte_mkwrite(pte_t pte)
303+static inline pte_t pte_mkwrite_novma(pte_t pte)
304304 {
305305 pte_val(pte) |= _PAGE_WRITE;
306306 return pte;
arch/ia64/include/asm/pgtable.h+1−1
@@ -269,7 +269,7 @@ ia64_phys_addr_valid (unsigned long addr) · 268 unmodified lines
269269 * access rights:
270270 */
271271 #define pte_wrprotect(pte) (__pte(pte_val(pte) & ~_PAGE_AR_RW))
272−#define pte_mkwrite(pte) (__pte(pte_val(pte) | _PAGE_AR_RW))
272+#define pte_mkwrite_novma(pte) (__pte(pte_val(pte) | _PAGE_AR_RW))
273273 #define pte_mkold(pte) (__pte(pte_val(pte) & ~_PAGE_A))
274274 #define pte_mkyoung(pte) (__pte(pte_val(pte) | _PAGE_A))
275275 #define pte_mkclean(pte) (__pte(pte_val(pte) & ~_PAGE_D))
arch/loongarch/include/asm/pgtable.h+2−2
@@ -384,7 +384,7 @@ static inline pte_t pte_mkdirty(pte_t pte) · 383 unmodified lines
384384 return pte;
385385 }
386386
387−static inline pte_t pte_mkwrite(pte_t pte)
387+static inline pte_t pte_mkwrite_novma(pte_t pte)
388388 {
389389 pte_val(pte) |= _PAGE_WRITE;
390390 if (pte_val(pte) & _PAGE_MODIFIED)
@@ -493,7 +493,7 @@ static inline int pmd_write(pmd_t pmd) · 102 unmodified lines
493493 return !!(pmd_val(pmd) & _PAGE_WRITE);
494494 }
495495
496−static inline pmd_t pmd_mkwrite(pmd_t pmd)
496+static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
497497 {
498498 pmd_val(pmd) |= _PAGE_WRITE;
499499 if (pmd_val(pmd) & _PAGE_MODIFIED)
arch/m68k/include/asm/mcf_pgtable.h+1−1
@@ -210,7 +210,7 @@ static inline pte_t pte_mkold(pte_t pte) · 209 unmodified lines
210210 return pte;
211211 }
212212
213−static inline pte_t pte_mkwrite(pte_t pte)
213+static inline pte_t pte_mkwrite_novma(pte_t pte)
214214 {
215215 pte_val(pte) |= CF_PAGE_WRITABLE;
216216 return pte;
arch/m68k/include/asm/motorola_pgtable.h+1−1
@@ -156,7 +156,7 @@ static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED; · 155 unmodified lines
156156 static inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) |= _PAGE_RONLY; return pte; }
157157 static inline pte_t pte_mkclean(pte_t pte) { pte_val(pte) &= ~_PAGE_DIRTY; return pte; }
158158 static inline pte_t pte_mkold(pte_t pte) { pte_val(pte) &= ~_PAGE_ACCESSED; return pte; }
159−static inline pte_t pte_mkwrite(pte_t pte) { pte_val(pte) &= ~_PAGE_RONLY; return pte; }
159+static inline pte_t pte_mkwrite_novma(pte_t pte){ pte_val(pte) &= ~_PAGE_RONLY; return pte; }
160160 static inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= _PAGE_DIRTY; return pte; }
161161 static inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= _PAGE_ACCESSED; return pte; }
162162 static inline pte_t pte_mknocache(pte_t pte)
arch/m68k/include/asm/sun3_pgtable.h+1−1
@@ -144,7 +144,7 @@ static inline int pte_young(pte_t pte) { return pte_val(pte) & SUN3_PAGE_ACCESS · 143 unmodified lines
144144 static inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) &= ~SUN3_PAGE_WRITEABLE; return pte; }
145145 static inline pte_t pte_mkclean(pte_t pte) { pte_val(pte) &= ~SUN3_PAGE_MODIFIED; return pte; }
146146 static inline pte_t pte_mkold(pte_t pte) { pte_val(pte) &= ~SUN3_PAGE_ACCESSED; return pte; }
147−static inline pte_t pte_mkwrite(pte_t pte) { pte_val(pte) |= SUN3_PAGE_WRITEABLE; return pte; }
147+static inline pte_t pte_mkwrite_novma(pte_t pte){ pte_val(pte) |= SUN3_PAGE_WRITEABLE; return pte; }
148148 static inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= SUN3_PAGE_MODIFIED; return pte; }
149149 static inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= SUN3_PAGE_ACCESSED; return pte; }
150150 static inline pte_t pte_mknocache(pte_t pte) { pte_val(pte) |= SUN3_PAGE_NOCACHE; return pte; }
arch/microblaze/include/asm/pgtable.h+1−1
@@ -266,7 +266,7 @@ static inline pte_t pte_mkread(pte_t pte) \ · 265 unmodified lines
266266 { pte_val(pte) |= _PAGE_USER; return pte; }
267267 static inline pte_t pte_mkexec(pte_t pte) \
268268 { pte_val(pte) |= _PAGE_USER | _PAGE_EXEC; return pte; }
269−static inline pte_t pte_mkwrite(pte_t pte) \
269+static inline pte_t pte_mkwrite_novma(pte_t pte) \
270270 { pte_val(pte) |= _PAGE_RW; return pte; }
271271 static inline pte_t pte_mkdirty(pte_t pte) \
272272 { pte_val(pte) |= _PAGE_DIRTY; return pte; }
arch/mips/include/asm/pgtable.h+3−3
@@ -319,7 +319,7 @@ static inline pte_t pte_mkold(pte_t pte) · 318 unmodified lines
319319 return pte;
320320 }
321321
322−static inline pte_t pte_mkwrite(pte_t pte)
322+static inline pte_t pte_mkwrite_novma(pte_t pte)
323323 {
324324 pte.pte_low |= _PAGE_WRITE;
325325 if (pte.pte_low & _PAGE_MODIFIED) {
@@ -374,7 +374,7 @@ static inline pte_t pte_mkold(pte_t pte) · 48 unmodified lines
374374 return pte;
375375 }
376376
377−static inline pte_t pte_mkwrite(pte_t pte)
377+static inline pte_t pte_mkwrite_novma(pte_t pte)
378378 {
379379 pte_val(pte) |= _PAGE_WRITE;
380380 if (pte_val(pte) & _PAGE_MODIFIED)
@@ -646,7 +646,7 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd) · 265 unmodified lines
646646 return pmd;
647647 }
648648
649−static inline pmd_t pmd_mkwrite(pmd_t pmd)
649+static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
650650 {
651651 pmd_val(pmd) |= _PAGE_WRITE;
652652 if (pmd_val(pmd) & _PAGE_MODIFIED)
arch/nios2/include/asm/pgtable.h+1−1
@@ -129,7 +129,7 @@ static inline pte_t pte_mkold(pte_t pte) · 128 unmodified lines
129129 return pte;
130130 }
131131
132−static inline pte_t pte_mkwrite(pte_t pte)
132+static inline pte_t pte_mkwrite_novma(pte_t pte)
133133 {
134134 pte_val(pte) |= _PAGE_WRITE;
135135 return pte;
arch/openrisc/include/asm/pgtable.h+1−1
@@ -250,7 +250,7 @@ static inline pte_t pte_mkold(pte_t pte) · 249 unmodified lines
250250 return pte;
251251 }
252252
253−static inline pte_t pte_mkwrite(pte_t pte)
253+static inline pte_t pte_mkwrite_novma(pte_t pte)
254254 {
255255 pte_val(pte) |= _PAGE_WRITE;
256256 return pte;
arch/parisc/include/asm/pgtable.h+1−1
@@ -322,7 +322,7 @@ static inline pte_t pte_mkold(pte_t pte) { pte_val(pte) &= ~_PAGE_ACCESSED; retu · 321 unmodified lines
322322 static inline pte_t pte_wrprotect(pte_t pte) { pte_val(pte) &= ~_PAGE_WRITE; return pte; }
323323 static inline pte_t pte_mkdirty(pte_t pte) { pte_val(pte) |= _PAGE_DIRTY; return pte; }
324324 static inline pte_t pte_mkyoung(pte_t pte) { pte_val(pte) |= _PAGE_ACCESSED; return pte; }
325−static inline pte_t pte_mkwrite(pte_t pte) { pte_val(pte) |= _PAGE_WRITE; return pte; }
325+static inline pte_t pte_mkwrite_novma(pte_t pte) { pte_val(pte) |= _PAGE_WRITE; return pte; }
326326 static inline pte_t pte_mkspecial(pte_t pte) { pte_val(pte) |= _PAGE_SPECIAL; return pte; }
327327
328328 /*
arch/powerpc/include/asm/book3s/32/pgtable.h+1−1
@@ -493,7 +493,7 @@ static inline pte_t pte_mkpte(pte_t pte) · 492 unmodified lines
493493 return pte;
494494 }
495495
496−static inline pte_t pte_mkwrite(pte_t pte)
496+static inline pte_t pte_mkwrite_novma(pte_t pte)
497497 {
498498 return __pte(pte_val(pte) | _PAGE_RW);
499499 }
arch/powerpc/include/asm/book3s/64/pgtable.h+2−2
@@ -596,7 +596,7 @@ static inline pte_t pte_mkexec(pte_t pte) · 595 unmodified lines
596596 return __pte_raw(pte_raw(pte) | cpu_to_be64(_PAGE_EXEC));
597597 }
598598
599−static inline pte_t pte_mkwrite(pte_t pte)
599+static inline pte_t pte_mkwrite_novma(pte_t pte)
600600 {
601601 /*
602602 * write implies read, hence set both
@@ -1088,7 +1088,7 @@ static inline pte_t *pmdp_ptep(pmd_t *pmd) · 485 unmodified lines
10881088 #define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
10891089 #define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
10901090 #define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
1091−#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))
1091+#define pmd_mkwrite_novma(pmd) pte_pmd(pte_mkwrite_novma(pmd_pte(pmd)))
10921092
10931093 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
10941094 #define pmd_soft_dirty(pmd) pte_soft_dirty(pmd_pte(pmd))
arch/powerpc/include/asm/nohash/32/pgtable.h+2−2
@@ -170,8 +170,8 @@ void unmap_kernel_page(unsigned long va); · 169 unmodified lines
170170 #define pte_clear(mm, addr, ptep) \
171171 do { pte_update(mm, addr, ptep, ~0, 0, 0); } while (0)
172172
173−#ifndef pte_mkwrite
174−static inline pte_t pte_mkwrite(pte_t pte)
173+#ifndef pte_mkwrite_novma
174+static inline pte_t pte_mkwrite_novma(pte_t pte)
175175 {
176176 return __pte(pte_val(pte) | _PAGE_RW);
177177 }
arch/powerpc/include/asm/nohash/32/pte-8xx.h+2−2
@@ -101,12 +101,12 @@ static inline int pte_write(pte_t pte) · 100 unmodified lines
101101
102102 #define pte_write pte_write
103103
104−static inline pte_t pte_mkwrite(pte_t pte)
104+static inline pte_t pte_mkwrite_novma(pte_t pte)
105105 {
106106 return __pte(pte_val(pte) & ~_PAGE_RO);
107107 }
108108
109−#define pte_mkwrite pte_mkwrite
109+#define pte_mkwrite_novma pte_mkwrite_novma
110110
111111 static inline bool pte_user(pte_t pte)
112112 {
arch/powerpc/include/asm/nohash/64/pgtable.h+1−1
@@ -85,7 +85,7 @@ · 84 unmodified lines
8585 #ifndef __ASSEMBLY__
8686 /* pte_clear moved to later in this file */
8787
88−static inline pte_t pte_mkwrite(pte_t pte)
88+static inline pte_t pte_mkwrite_novma(pte_t pte)
8989 {
9090 return __pte(pte_val(pte) | _PAGE_RW);
9191 }
arch/riscv/include/asm/pgtable.h+3−3
@@ -380,7 +380,7 @@ static inline pte_t pte_wrprotect(pte_t pte) · 379 unmodified lines
380380
381381 /* static inline pte_t pte_mkread(pte_t pte) */
382382
383−static inline pte_t pte_mkwrite(pte_t pte)
383+static inline pte_t pte_mkwrite_novma(pte_t pte)
384384 {
385385 return __pte(pte_val(pte) | _PAGE_WRITE);
386386 }
@@ -677,9 +677,9 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) · 290 unmodified lines
677677 return pte_pmd(pte_mkyoung(pmd_pte(pmd)));
678678 }
679679
680−static inline pmd_t pmd_mkwrite(pmd_t pmd)
680+static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
681681 {
682− return pte_pmd(pte_mkwrite(pmd_pte(pmd)));
682+ return pte_pmd(pte_mkwrite_novma(pmd_pte(pmd)));
683683 }
684684
685685 static inline pmd_t pmd_wrprotect(pmd_t pmd)
arch/s390/Kconfig+1−0
@@ -127,6 +127,7 @@ config S390 · 126 unmodified lines
127127 select ARCH_WANTS_NO_INSTR
128128 select ARCH_WANT_DEFAULT_BPF_JIT
129129 select ARCH_WANT_IPC_PARSE_VERSION
130+ select ARCH_WANT_KERNEL_PMD_MKWRITE
130131 select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP
131132 select BUILDTIME_TABLE_SORT
132133 select CLONE_BACKWARDS2
arch/s390/include/asm/hugetlb.h+1−1
@@ -104,7 +104,7 @@ static inline int huge_pte_dirty(pte_t pte) · 103 unmodified lines
104104
105105 static inline pte_t huge_pte_mkwrite(pte_t pte)
106106 {
107− return pte_mkwrite(pte);
107+ return pte_mkwrite_novma(pte);
108108 }
109109
110110 static inline pte_t huge_pte_mkdirty(pte_t pte)
arch/s390/include/asm/pgtable.h+2−2
@@ -1001,7 +1001,7 @@ static inline pte_t pte_wrprotect(pte_t pte) · 1000 unmodified lines
10011001 return set_pte_bit(pte, __pgprot(_PAGE_PROTECT));
10021002 }
10031003
1004−static inline pte_t pte_mkwrite(pte_t pte)
1004+static inline pte_t pte_mkwrite_novma(pte_t pte)
10051005 {
10061006 pte = set_pte_bit(pte, __pgprot(_PAGE_WRITE));
10071007 if (pte_val(pte) & _PAGE_DIRTY)
@@ -1498,7 +1498,7 @@ static inline pmd_t pmd_wrprotect(pmd_t pmd) · 490 unmodified lines
14981498 return set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_PROTECT));
14991499 }
15001500
1501−static inline pmd_t pmd_mkwrite(pmd_t pmd)
1501+static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
15021502 {
15031503 pmd = set_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_WRITE));
15041504 if (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY)
arch/s390/mm/pageattr.c+2−2
@@ -98,7 +98,7 @@ static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, · 97 unmodified lines
9898 if (flags & SET_MEMORY_RO)
9999 new = pte_wrprotect(new);
100100 else if (flags & SET_MEMORY_RW)
101− new = pte_mkwrite(pte_mkdirty(new));
101+ new = pte_mkwrite_novma(pte_mkdirty(new));
102102 if (flags & SET_MEMORY_NX)
103103 new = set_pte_bit(new, __pgprot(_PAGE_NOEXEC));
104104 else if (flags & SET_MEMORY_X)
@@ -156,7 +156,7 @@ static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, · 51 unmodified lines
156156 if (flags & SET_MEMORY_RO)
157157 new = pmd_wrprotect(new);
158158 else if (flags & SET_MEMORY_RW)
159− new = pmd_mkwrite(pmd_mkdirty(new));
159+ new = pmd_mkwrite_novma(pmd_mkdirty(new));
160160 if (flags & SET_MEMORY_NX)
161161 new = set_pmd_bit(new, __pgprot(_SEGMENT_ENTRY_NOEXEC));
162162 else if (flags & SET_MEMORY_X)
arch/sh/include/asm/pgtable_32.h+2−2
@@ -358,11 +358,11 @@ static inline pte_t pte_##fn(pte_t pte) { pte.pte_##h op; return pte; } · 357 unmodified lines
358358 * kernel permissions), we attempt to couple them a bit more sanely here.
359359 */
360360 PTE_BIT_FUNC(high, wrprotect, &= ~(_PAGE_EXT_USER_WRITE | _PAGE_EXT_KERN_WRITE));
361−PTE_BIT_FUNC(high, mkwrite, |= _PAGE_EXT_USER_WRITE | _PAGE_EXT_KERN_WRITE);
361+PTE_BIT_FUNC(high, mkwrite_novma, |= _PAGE_EXT_USER_WRITE | _PAGE_EXT_KERN_WRITE);
362362 PTE_BIT_FUNC(high, mkhuge, |= _PAGE_SZHUGE);
363363 #else
364364 PTE_BIT_FUNC(low, wrprotect, &= ~_PAGE_RW);
365−PTE_BIT_FUNC(low, mkwrite, |= _PAGE_RW);
365+PTE_BIT_FUNC(low, mkwrite_novma, |= _PAGE_RW);
366366 PTE_BIT_FUNC(low, mkhuge, |= _PAGE_SZHUGE);
367367 #endif
368368
arch/sparc/include/asm/pgtable_32.h+1−1
@@ -239,7 +239,7 @@ static inline pte_t pte_mkold(pte_t pte) · 238 unmodified lines
239239 return __pte(pte_val(pte) & ~SRMMU_REF);
240240 }
241241
242−static inline pte_t pte_mkwrite(pte_t pte)
242+static inline pte_t pte_mkwrite_novma(pte_t pte)
243243 {
244244 return __pte(pte_val(pte) | SRMMU_WRITE);
245245 }
arch/sparc/include/asm/pgtable_64.h+3−3
@@ -518,7 +518,7 @@ static inline pte_t pte_mkclean(pte_t pte) · 517 unmodified lines
518518 return __pte(val);
519519 }
520520
521−static inline pte_t pte_mkwrite(pte_t pte)
521+static inline pte_t pte_mkwrite_novma(pte_t pte)
522522 {
523523 unsigned long val = pte_val(pte), mask;
524524
@@ -773,11 +773,11 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) · 248 unmodified lines
773773 return __pmd(pte_val(pte));
774774 }
775775
776−static inline pmd_t pmd_mkwrite(pmd_t pmd)
776+static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
777777 {
778778 pte_t pte = __pte(pmd_val(pmd));
779779
780− pte = pte_mkwrite(pte);
780+ pte = pte_mkwrite_novma(pte);
781781
782782 return __pmd(pte_val(pte));
783783 }
arch/sparc/kernel/signal32.c+1−1
@@ -753,7 +753,7 @@ out: · 752 unmodified lines
753753 */
754754 static_assert(NSIGILL == 11);
755755 static_assert(NSIGFPE == 15);
756−static_assert(NSIGSEGV == 9);
756+static_assert(NSIGSEGV == 10);
757757 static_assert(NSIGBUS == 5);
758758 static_assert(NSIGTRAP == 6);
759759 static_assert(NSIGCHLD == 6);
arch/sparc/kernel/signal_64.c+1−1
@@ -562,7 +562,7 @@ void do_notify_resume(struct pt_regs *regs, unsigned long orig_i0, unsigned long · 561 unmodified lines
562562 */
563563 static_assert(NSIGILL == 11);
564564 static_assert(NSIGFPE == 15);
565−static_assert(NSIGSEGV == 9);
565+static_assert(NSIGSEGV == 10);
566566 static_assert(NSIGBUS == 5);
567567 static_assert(NSIGTRAP == 6);
568568 static_assert(NSIGCHLD == 6);
arch/um/include/asm/pgtable.h+1−1
@@ -207,7 +207,7 @@ static inline pte_t pte_mkyoung(pte_t pte) · 206 unmodified lines
207207 return(pte);
208208 }
209209
210−static inline pte_t pte_mkwrite(pte_t pte)
210+static inline pte_t pte_mkwrite_novma(pte_t pte)
211211 {
212212 if (unlikely(pte_get_bits(pte, _PAGE_RW)))
213213 return pte;
arch/x86/Kconfig+24−0
@@ -1815,6 +1815,11 @@ config CC_HAS_IBT · 1814 unmodified lines
18151815 (CC_IS_CLANG && CLANG_VERSION >= 140000)) && \
18161816 $(as-instr,endbr64)
18171817
1818+config X86_CET
1819+ def_bool n
1820+ help
1821+ CET features configured (Shadow stack or IBT)
1822+
18181823 config X86_KERNEL_IBT
18191824 prompt "Indirect Branch Tracking"
18201825 def_bool y
@@ -1822,6 +1827,7 @@ config X86_KERNEL_IBT · 1 unmodified line
18221827 # https://github.com/llvm/llvm-project/commit/9d7001eba9c4cb311e03cd8cdc231f9e579f2d0f
18231828 depends on !LD_IS_LLD || LLD_VERSION >= 140000
18241829 select OBJTOOL
1830+ select X86_CET
18251831 help
18261832 Build the kernel with support for Indirect Branch Tracking, a
18271833 hardware support course-grain forward-edge Control Flow Integrity
@@ -1915,6 +1921,24 @@ config X86_SGX · 87 unmodified lines
19151921
19161922 If unsure, say N.
19171923
1924+config X86_USER_SHADOW_STACK
1925+ bool "X86 userspace shadow stack"
1926+ depends on AS_WRUSS
1927+ depends on X86_64
1928+ select ARCH_USES_HIGH_VMA_FLAGS
1929+ select X86_CET
1930+ help
1931+ Shadow stack protection is a hardware feature that detects function
1932+ return address corruption. This helps mitigate ROP attacks.
1933+ Applications must be enabled to use it, and old userspace does not
1934+ get protection "for free".
1935+
1936+ CPUs supporting shadow stacks were first released in 2020.
1937+
1938+ See Documentation/arch/x86/shstk.rst for more information.
1939+
1940+ If unsure, say N.
1941+
19181942 config EFI
19191943 bool "EFI runtime service support"
19201944 depends on ACPI
arch/x86/Kconfig.assembler+5−0
@@ -24,3 +24,8 @@ config AS_GFNI · 23 unmodified lines
2424 def_bool $(as-instr,vgf2p8mulb %xmm0$(comma)%xmm1$(comma)%xmm2)
2525 help
2626 Supported by binutils >= 2.30 and LLVM integrated assembler
27+
28+config AS_WRUSS
29+ def_bool $(as-instr,wrussq %rax$(comma)(%rbx))
30+ help
31+ Supported by binutils >= 2.31 and LLVM integrated assembler
arch/x86/entry/syscalls/syscall_64.tbl+1−0
@@ -374,6 +374,7 @@ · 373 unmodified lines
374374 450 common set_mempolicy_home_node sys_set_mempolicy_home_node
375375 451 common cachestat sys_cachestat
376376 452 common fchmodat2 sys_fchmodat2
377+453 64 map_shadow_stack sys_map_shadow_stack
377378
378379 #
379380 # Due to a historical design error, certain syscalls are numbered differently
arch/x86/include/asm/cpufeatures.h+2−0
@@ -307,6 +307,7 @@ · 306 unmodified lines
307307 #define X86_FEATURE_MSR_TSX_CTRL (11*32+20) /* "" MSR IA32_TSX_CTRL (Intel) implemented */
308308 #define X86_FEATURE_SMBA (11*32+21) /* "" Slow Memory Bandwidth Allocation */
309309 #define X86_FEATURE_BMEC (11*32+22) /* "" Bandwidth Monitoring Event Configuration */
310+#define X86_FEATURE_USER_SHSTK (11*32+23) /* Shadow stack support for user mode applications */
310311
311312 #define X86_FEATURE_SRSO (11*32+24) /* "" AMD BTB untrain RETs */
312313 #define X86_FEATURE_SRSO_ALIAS (11*32+25) /* "" AMD BTB untrain RETs through aliasing */
@@ -383,6 +384,7 @@ · 70 unmodified lines
383384 #define X86_FEATURE_OSPKE (16*32+ 4) /* OS Protection Keys Enable */
384385 #define X86_FEATURE_WAITPKG (16*32+ 5) /* UMONITOR/UMWAIT/TPAUSE Instructions */
385386 #define X86_FEATURE_AVX512_VBMI2 (16*32+ 6) /* Additional AVX512 Vector Bit Manipulation Instructions */
387+#define X86_FEATURE_SHSTK (16*32+ 7) /* "" Shadow stack */
386388 #define X86_FEATURE_GFNI (16*32+ 8) /* Galois Field New Instructions */
387389 #define X86_FEATURE_VAES (16*32+ 9) /* Vector AES */
388390 #define X86_FEATURE_VPCLMULQDQ (16*32+10) /* Carry-Less Multiplication Double Quadword */
arch/x86/include/asm/disabled-features.h+14−2
@@ -105,6 +105,18 @@ · 104 unmodified lines
105105 # define DISABLE_TDX_GUEST (1 << (X86_FEATURE_TDX_GUEST & 31))
106106 #endif
107107
108+#ifdef CONFIG_X86_USER_SHADOW_STACK
109+#define DISABLE_USER_SHSTK 0
110+#else
111+#define DISABLE_USER_SHSTK (1 << (X86_FEATURE_USER_SHSTK & 31))
112+#endif
113+
114+#ifdef CONFIG_X86_KERNEL_IBT
115+#define DISABLE_IBT 0
116+#else
117+#define DISABLE_IBT (1 << (X86_FEATURE_IBT & 31))
118+#endif
119+
108120 /*
109121 * Make sure to add features to the correct mask
110122 */
@@ -120,7 +132,7 @@ · 9 unmodified lines
120132 #define DISABLED_MASK9 (DISABLE_SGX)
121133 #define DISABLED_MASK10 0
122134 #define DISABLED_MASK11 (DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET| \
123− DISABLE_CALL_DEPTH_TRACKING)
135+ DISABLE_CALL_DEPTH_TRACKING|DISABLE_USER_SHSTK)
124136 #define DISABLED_MASK12 (DISABLE_LAM)
125137 #define DISABLED_MASK13 0
126138 #define DISABLED_MASK14 0
@@ -128,7 +140,7 @@ · 1 unmodified line
128140 #define DISABLED_MASK16 (DISABLE_PKU|DISABLE_OSPKE|DISABLE_LA57|DISABLE_UMIP| \
129141 DISABLE_ENQCMD)
130142 #define DISABLED_MASK17 0
131−#define DISABLED_MASK18 0
143+#define DISABLED_MASK18 (DISABLE_IBT)
132144 #define DISABLED_MASK19 0
133145 #define DISABLED_MASK20 0
134146 #define DISABLED_MASK_CHECK BUILD_BUG_ON_ZERO(NCAPINTS != 21)
arch/x86/include/asm/fpu/api.h+9−0
@@ -82,6 +82,15 @@ static inline void fpregs_unlock(void) · 81 unmodified lines
8282 preempt_enable();
8383 }
8484
85+/*
86+ * FPU state gets lazily restored before returning to userspace. So when in the
87+ * kernel, the valid FPU state may be kept in the buffer. This function will force
88+ * restore all the fpu state to the registers early if needed, and lock them from
89+ * being automatically saved/restored. Then FPU state can be modified safely in the
90+ * registers, before unlocking with fpregs_unlock().
91+ */
92+void fpregs_lock_and_load(void);
93+
8594 #ifdef CONFIG_X86_DEBUG_FPU
8695 extern void fpregs_assert_state_consistent(void);
8796 #else
arch/x86/include/asm/fpu/regset.h+4−3
@@ -7,11 +7,12 @@ · 6 unmodified lines
77
88 #include <linux/regset.h>
99
10−extern user_regset_active_fn regset_fpregs_active, regset_xregset_fpregs_active;
10+extern user_regset_active_fn regset_fpregs_active, regset_xregset_fpregs_active,
11+ ssp_active;
1112 extern user_regset_get2_fn fpregs_get, xfpregs_get, fpregs_soft_get,
12− xstateregs_get;
13+ xstateregs_get, ssp_get;
1314 extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set,
14− xstateregs_set;
15+ xstateregs_set, ssp_set;
1516
1617 /*
1718 * xstateregs_active == regset_fpregs_active. Please refer to the comment
arch/x86/include/asm/fpu/sched.h+2−1
@@ -11,7 +11,8 @@ · 10 unmodified lines
1111
1212 extern void save_fpregs_to_fpstate(struct fpu *fpu);
1313 extern void fpu__drop(struct fpu *fpu);
14−extern int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal);
14+extern int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal,
15+ unsigned long shstk_addr);
1516 extern void fpu_flush_thread(void);
1617
1718 /*
arch/x86/include/asm/fpu/types.h+14−2
@@ -115,8 +115,8 @@ enum xfeature { · 114 unmodified lines
115115 XFEATURE_PT_UNIMPLEMENTED_SO_FAR,
116116 XFEATURE_PKRU,
117117 XFEATURE_PASID,
118− XFEATURE_RSRVD_COMP_11,
119− XFEATURE_RSRVD_COMP_12,
118+ XFEATURE_CET_USER,
119+ XFEATURE_CET_KERNEL_UNUSED,
120120 XFEATURE_RSRVD_COMP_13,
121121 XFEATURE_RSRVD_COMP_14,
122122 XFEATURE_LBR,
@@ -138,6 +138,8 @@ enum xfeature { · 15 unmodified lines
138138 #define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR)
139139 #define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU)
140140 #define XFEATURE_MASK_PASID (1 << XFEATURE_PASID)
141+#define XFEATURE_MASK_CET_USER (1 << XFEATURE_CET_USER)
142+#define XFEATURE_MASK_CET_KERNEL (1 << XFEATURE_CET_KERNEL_UNUSED)
141143 #define XFEATURE_MASK_LBR (1 << XFEATURE_LBR)
142144 #define XFEATURE_MASK_XTILE_CFG (1 << XFEATURE_XTILE_CFG)
143145 #define XFEATURE_MASK_XTILE_DATA (1 << XFEATURE_XTILE_DATA)
@@ -252,6 +254,16 @@ struct pkru_state { · 108 unmodified lines
252254 u32 pad;
253255 } __packed;
254256
257+/*
258+ * State component 11 is Control-flow Enforcement user states
259+ */
260+struct cet_user_state {
261+ /* user control-flow settings */
262+ u64 user_cet;
263+ /* user shadow stack pointer */
264+ u64 user_ssp;
265+};
266+
255267 /*
256268 * State component 15: Architectural LBR configuration state.
257269 * The size of Arch LBR state depends on the number of LBRs (lbr_depth).
arch/x86/include/asm/fpu/xstate.h+4−2
@@ -50,7 +50,8 @@ · 49 unmodified lines
5050 #define XFEATURE_MASK_USER_DYNAMIC XFEATURE_MASK_XTILE_DATA
5151
5252 /* All currently supported supervisor features */
53−#define XFEATURE_MASK_SUPERVISOR_SUPPORTED (XFEATURE_MASK_PASID)
53+#define XFEATURE_MASK_SUPERVISOR_SUPPORTED (XFEATURE_MASK_PASID | \
54+ XFEATURE_MASK_CET_USER)
5455
5556 /*
5657 * A supervisor state component may not always contain valuable information,
@@ -77,7 +78,8 @@ · 20 unmodified lines
7778 * Unsupported supervisor features. When a supervisor feature in this mask is
7879 * supported in the future, move it to the supported supervisor feature mask.
7980 */
80−#define XFEATURE_MASK_SUPERVISOR_UNSUPPORTED (XFEATURE_MASK_PT)
81+#define XFEATURE_MASK_SUPERVISOR_UNSUPPORTED (XFEATURE_MASK_PT | \
82+ XFEATURE_MASK_CET_KERNEL)
8183
8284 /* All supervisor states including supported and unsupported states. */
8385 #define XFEATURE_MASK_SUPERVISOR_ALL (XFEATURE_MASK_SUPERVISOR_SUPPORTED | \
arch/x86/include/asm/idtentry.h+1−1
@@ -614,7 +614,7 @@ DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_DF, xenpv_exc_double_fault); · 613 unmodified lines
614614 #endif
615615
616616 /* #CP */
617−#ifdef CONFIG_X86_KERNEL_IBT
617+#ifdef CONFIG_X86_CET
618618 DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_CP, exc_control_protection);
619619 #endif
620620
arch/x86/include/asm/mmu_context.h+2−0
@@ -186,6 +186,8 @@ do { \ · 185 unmodified lines
186186 #else
187187 #define deactivate_mm(tsk, mm) \
188188 do { \
189+ if (!tsk->vfork_done) \
190+ shstk_free(tsk); \
189191 load_gs_index(0); \
190192 loadsegment(fs, 0); \
191193 } while (0)
arch/x86/include/asm/pgtable.h+266−36
@@ -125,9 +125,15 @@ extern pmdval_t early_pmd_flags; · 124 unmodified lines
125125 * The following only work if pte_present() is true.
126126 * Undefined behaviour if not..
127127 */
128−static inline int pte_dirty(pte_t pte)
128+static inline bool pte_dirty(pte_t pte)
129129 {
130− return pte_flags(pte) & _PAGE_DIRTY;
130+ return pte_flags(pte) & _PAGE_DIRTY_BITS;
131+}
132+
133+static inline bool pte_shstk(pte_t pte)
134+{
135+ return cpu_feature_enabled(X86_FEATURE_SHSTK) &&
136+ (pte_flags(pte) & (_PAGE_RW | _PAGE_DIRTY)) == _PAGE_DIRTY;
131137 }
132138
133139 static inline int pte_young(pte_t pte)
@@ -135,9 +141,16 @@ static inline int pte_young(pte_t pte) · 1 unmodified line
135141 return pte_flags(pte) & _PAGE_ACCESSED;
136142 }
137143
138−static inline int pmd_dirty(pmd_t pmd)
144+static inline bool pmd_dirty(pmd_t pmd)
145+{
146+ return pmd_flags(pmd) & _PAGE_DIRTY_BITS;
147+}
148+
149+static inline bool pmd_shstk(pmd_t pmd)
139150 {
140− return pmd_flags(pmd) & _PAGE_DIRTY;
151+ return cpu_feature_enabled(X86_FEATURE_SHSTK) &&
152+ (pmd_flags(pmd) & (_PAGE_RW | _PAGE_DIRTY | _PAGE_PSE)) ==
153+ (_PAGE_DIRTY | _PAGE_PSE);
141154 }
142155
143156 #define pmd_young pmd_young
@@ -146,9 +159,9 @@ static inline int pmd_young(pmd_t pmd) · 2 unmodified lines
146159 return pmd_flags(pmd) & _PAGE_ACCESSED;
147160 }
148161
149−static inline int pud_dirty(pud_t pud)
162+static inline bool pud_dirty(pud_t pud)
150163 {
151− return pud_flags(pud) & _PAGE_DIRTY;
164+ return pud_flags(pud) & _PAGE_DIRTY_BITS;
152165 }
153166
154167 static inline int pud_young(pud_t pud)
@@ -158,7 +171,27 @@ static inline int pud_young(pud_t pud) · 3 unmodified lines
158171
159172 static inline int pte_write(pte_t pte)
160173 {
161− return pte_flags(pte) & _PAGE_RW;
174+ /*
175+ * Shadow stack pages are logically writable, but do not have
176+ * _PAGE_RW. Check for them separately from _PAGE_RW itself.
177+ */
178+ return (pte_flags(pte) & _PAGE_RW) || pte_shstk(pte);
179+}
180+
181+#define pmd_write pmd_write
182+static inline int pmd_write(pmd_t pmd)
183+{
184+ /*
185+ * Shadow stack pages are logically writable, but do not have
186+ * _PAGE_RW. Check for them separately from _PAGE_RW itself.
187+ */
188+ return (pmd_flags(pmd) & _PAGE_RW) || pmd_shstk(pmd);
189+}
190+
191+#define pud_write pud_write
192+static inline int pud_write(pud_t pud)
193+{
194+ return pud_flags(pud) & _PAGE_RW;
162195 }
163196
164197 static inline int pte_huge(pte_t pte)
@@ -292,9 +325,63 @@ static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear) · 127 unmodified lines
292325 return native_make_pte(v & ~clear);
293326 }
294327
328+/*
329+ * Write protection operations can result in Dirty=1,Write=0 PTEs. But in the
330+ * case of X86_FEATURE_USER_SHSTK, these PTEs denote shadow stack memory. So
331+ * when creating dirty, write-protected memory, a software bit is used:
332+ * _PAGE_BIT_SAVED_DIRTY. The following functions take a PTE and transition the
333+ * Dirty bit to SavedDirty, and vice-vesra.
334+ *
335+ * This shifting is only done if needed. In the case of shifting
336+ * Dirty->SavedDirty, the condition is if the PTE is Write=0. In the case of
337+ * shifting SavedDirty->Dirty, the condition is Write=1.
338+ */
339+static inline pgprotval_t mksaveddirty_shift(pgprotval_t v)
340+{
341+ pgprotval_t cond = (~v >> _PAGE_BIT_RW) & 1;
342+
343+ v |= ((v >> _PAGE_BIT_DIRTY) & cond) << _PAGE_BIT_SAVED_DIRTY;
344+ v &= ~(cond << _PAGE_BIT_DIRTY);
345+
346+ return v;
347+}
348+
349+static inline pgprotval_t clear_saveddirty_shift(pgprotval_t v)
350+{
351+ pgprotval_t cond = (v >> _PAGE_BIT_RW) & 1;
352+
353+ v |= ((v >> _PAGE_BIT_SAVED_DIRTY) & cond) << _PAGE_BIT_DIRTY;
354+ v &= ~(cond << _PAGE_BIT_SAVED_DIRTY);
355+
356+ return v;
357+}
358+
359+static inline pte_t pte_mksaveddirty(pte_t pte)
360+{
361+ pteval_t v = native_pte_val(pte);
362+
363+ v = mksaveddirty_shift(v);
364+ return native_make_pte(v);
365+}
366+
367+static inline pte_t pte_clear_saveddirty(pte_t pte)
368+{
369+ pteval_t v = native_pte_val(pte);
370+
371+ v = clear_saveddirty_shift(v);
372+ return native_make_pte(v);
373+}
374+
295375 static inline pte_t pte_wrprotect(pte_t pte)
296376 {
297− return pte_clear_flags(pte, _PAGE_RW);
377+ pte = pte_clear_flags(pte, _PAGE_RW);
378+
379+ /*
380+ * Blindly clearing _PAGE_RW might accidentally create
381+ * a shadow stack PTE (Write=0,Dirty=1). Move the hardware
382+ * dirty value to the software bit, if present.
383+ */
384+ return pte_mksaveddirty(pte);
298385 }
299386
300387 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
@@ -332,7 +419,7 @@ static inline pte_t pte_clear_uffd_wp(pte_t pte) · 31 unmodified lines
332419
333420 static inline pte_t pte_mkclean(pte_t pte)
334421 {
335− return pte_clear_flags(pte, _PAGE_DIRTY);
422+ return pte_clear_flags(pte, _PAGE_DIRTY_BITS);
336423 }
337424
338425 static inline pte_t pte_mkold(pte_t pte)
@@ -347,7 +434,16 @@ static inline pte_t pte_mkexec(pte_t pte) · 8 unmodified lines
347434
348435 static inline pte_t pte_mkdirty(pte_t pte)
349436 {
350− return pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
437+ pte = pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
438+
439+ return pte_mksaveddirty(pte);
440+}
441+
442+static inline pte_t pte_mkwrite_shstk(pte_t pte)
443+{
444+ pte = pte_clear_flags(pte, _PAGE_RW);
445+
446+ return pte_set_flags(pte, _PAGE_DIRTY);
351447 }
352448
353449 static inline pte_t pte_mkyoung(pte_t pte)
@@ -355,11 +451,15 @@ static inline pte_t pte_mkyoung(pte_t pte) · 1 unmodified line
355451 return pte_set_flags(pte, _PAGE_ACCESSED);
356452 }
357453
358−static inline pte_t pte_mkwrite(pte_t pte)
454+static inline pte_t pte_mkwrite_novma(pte_t pte)
359455 {
360456 return pte_set_flags(pte, _PAGE_RW);
361457 }
362458
459+struct vm_area_struct;
460+pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma);
461+#define pte_mkwrite pte_mkwrite
462+
363463 static inline pte_t pte_mkhuge(pte_t pte)
364464 {
365465 return pte_set_flags(pte, _PAGE_PSE);
@@ -404,9 +504,34 @@ static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear) · 38 unmodified lines
404504 return native_make_pmd(v & ~clear);
405505 }
406506
507+/* See comments above mksaveddirty_shift() */
508+static inline pmd_t pmd_mksaveddirty(pmd_t pmd)
509+{
510+ pmdval_t v = native_pmd_val(pmd);
511+
512+ v = mksaveddirty_shift(v);
513+ return native_make_pmd(v);
514+}
515+
516+/* See comments above mksaveddirty_shift() */
517+static inline pmd_t pmd_clear_saveddirty(pmd_t pmd)
518+{
519+ pmdval_t v = native_pmd_val(pmd);
520+
521+ v = clear_saveddirty_shift(v);
522+ return native_make_pmd(v);
523+}
524+
407525 static inline pmd_t pmd_wrprotect(pmd_t pmd)
408526 {
409− return pmd_clear_flags(pmd, _PAGE_RW);
527+ pmd = pmd_clear_flags(pmd, _PAGE_RW);
528+
529+ /*
530+ * Blindly clearing _PAGE_RW might accidentally create
531+ * a shadow stack PMD (RW=0, Dirty=1). Move the hardware
532+ * dirty value to the software bit.
533+ */
534+ return pmd_mksaveddirty(pmd);
410535 }
411536
412537 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
@@ -433,12 +558,21 @@ static inline pmd_t pmd_mkold(pmd_t pmd) · 20 unmodified lines
433558
434559 static inline pmd_t pmd_mkclean(pmd_t pmd)
435560 {
436− return pmd_clear_flags(pmd, _PAGE_DIRTY);
561+ return pmd_clear_flags(pmd, _PAGE_DIRTY_BITS);
437562 }
438563
439564 static inline pmd_t pmd_mkdirty(pmd_t pmd)
440565 {
441− return pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
566+ pmd = pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
567+
568+ return pmd_mksaveddirty(pmd);
569+}
570+
571+static inline pmd_t pmd_mkwrite_shstk(pmd_t pmd)
572+{
573+ pmd = pmd_clear_flags(pmd, _PAGE_RW);
574+
575+ return pmd_set_flags(pmd, _PAGE_DIRTY);
442576 }
443577
444578 static inline pmd_t pmd_mkdevmap(pmd_t pmd)
@@ -456,11 +590,14 @@ static inline pmd_t pmd_mkyoung(pmd_t pmd) · 11 unmodified lines
456590 return pmd_set_flags(pmd, _PAGE_ACCESSED);
457591 }
458592
459−static inline pmd_t pmd_mkwrite(pmd_t pmd)
593+static inline pmd_t pmd_mkwrite_novma(pmd_t pmd)
460594 {
461595 return pmd_set_flags(pmd, _PAGE_RW);
462596 }
463597
598+pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
599+#define pmd_mkwrite pmd_mkwrite
600+
464601 static inline pud_t pud_set_flags(pud_t pud, pudval_t set)
465602 {
466603 pudval_t v = native_pud_val(pud);
@@ -475,6 +612,24 @@ static inline pud_t pud_clear_flags(pud_t pud, pudval_t clear) · 8 unmodified lines
475612 return native_make_pud(v & ~clear);
476613 }
477614
615+/* See comments above mksaveddirty_shift() */
616+static inline pud_t pud_mksaveddirty(pud_t pud)
617+{
618+ pudval_t v = native_pud_val(pud);
619+
620+ v = mksaveddirty_shift(v);
621+ return native_make_pud(v);
622+}
623+
624+/* See comments above mksaveddirty_shift() */
625+static inline pud_t pud_clear_saveddirty(pud_t pud)
626+{
627+ pudval_t v = native_pud_val(pud);
628+
629+ v = clear_saveddirty_shift(v);
630+ return native_make_pud(v);
631+}
632+
478633 static inline pud_t pud_mkold(pud_t pud)
479634 {
480635 return pud_clear_flags(pud, _PAGE_ACCESSED);
@@ -482,17 +637,26 @@ static inline pud_t pud_mkold(pud_t pud) · 1 unmodified line
482637
483638 static inline pud_t pud_mkclean(pud_t pud)
484639 {
485− return pud_clear_flags(pud, _PAGE_DIRTY);
640+ return pud_clear_flags(pud, _PAGE_DIRTY_BITS);
486641 }
487642
488643 static inline pud_t pud_wrprotect(pud_t pud)
489644 {
490− return pud_clear_flags(pud, _PAGE_RW);
645+ pud = pud_clear_flags(pud, _PAGE_RW);
646+
647+ /*
648+ * Blindly clearing _PAGE_RW might accidentally create
649+ * a shadow stack PUD (RW=0, Dirty=1). Move the hardware
650+ * dirty value to the software bit.
651+ */
652+ return pud_mksaveddirty(pud);
491653 }
492654
493655 static inline pud_t pud_mkdirty(pud_t pud)
494656 {
495− return pud_set_flags(pud, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
657+ pud = pud_set_flags(pud, _PAGE_DIRTY | _PAGE_SOFT_DIRTY);
658+
659+ return pud_mksaveddirty(pud);
496660 }
497661
498662 static inline pud_t pud_mkdevmap(pud_t pud)
@@ -512,7 +676,9 @@ static inline pud_t pud_mkyoung(pud_t pud) · 13 unmodified lines
512676
513677 static inline pud_t pud_mkwrite(pud_t pud)
514678 {
515− return pud_set_flags(pud, _PAGE_RW);
679+ pud = pud_set_flags(pud, _PAGE_RW);
680+
681+ return pud_clear_saveddirty(pud);
516682 }
517683
518684 #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY
@@ -629,6 +795,7 @@ static inline u64 flip_protnone_guard(u64 oldval, u64 val, u64 mask); · 110 unmodified lines
629795 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
630796 {
631797 pteval_t val = pte_val(pte), oldval = val;
798+ pte_t pte_result;
632799
633800 /*
634801 * Chop off the NX bit (if present), and add the NX portion of
@@ -637,17 +804,54 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) · 2 unmodified lines
637804 val &= _PAGE_CHG_MASK;
638805 val |= check_pgprot(newprot) & ~_PAGE_CHG_MASK;
639806 val = flip_protnone_guard(oldval, val, PTE_PFN_MASK);
640− return __pte(val);
807+
808+ pte_result = __pte(val);
809+
810+ /*
811+ * To avoid creating Write=0,Dirty=1 PTEs, pte_modify() needs to avoid:
812+ * 1. Marking Write=0 PTEs Dirty=1
813+ * 2. Marking Dirty=1 PTEs Write=0
814+ *
815+ * The first case cannot happen because the _PAGE_CHG_MASK will filter
816+ * out any Dirty bit passed in newprot. Handle the second case by
817+ * going through the mksaveddirty exercise. Only do this if the old
818+ * value was Write=1 to avoid doing this on Shadow Stack PTEs.
819+ */
820+ if (oldval & _PAGE_RW)
821+ pte_result = pte_mksaveddirty(pte_result);
822+ else
823+ pte_result = pte_clear_saveddirty(pte_result);
824+
825+ return pte_result;
641826 }
642827
643828 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
644829 {
645830 pmdval_t val = pmd_val(pmd), oldval = val;
831+ pmd_t pmd_result;
646832
647− val &= _HPAGE_CHG_MASK;
833+ val &= (_HPAGE_CHG_MASK & ~_PAGE_DIRTY);
648834 val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
649835 val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);
650− return __pmd(val);
836+
837+ pmd_result = __pmd(val);
838+
839+ /*
840+ * To avoid creating Write=0,Dirty=1 PMDs, pte_modify() needs to avoid:
841+ * 1. Marking Write=0 PMDs Dirty=1
842+ * 2. Marking Dirty=1 PMDs Write=0
843+ *
844+ * The first case cannot happen because the _PAGE_CHG_MASK will filter
845+ * out any Dirty bit passed in newprot. Handle the second case by
846+ * going through the mksaveddirty exercise. Only do this if the old
847+ * value was Write=1 to avoid doing this on Shadow Stack PTEs.
848+ */
849+ if (oldval & _PAGE_RW)
850+ pmd_result = pmd_mksaveddirty(pmd_result);
851+ else
852+ pmd_result = pmd_clear_saveddirty(pmd_result);
853+
854+ return pmd_result;
651855 }
652856
653857 /*
@@ -831,7 +1035,14 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) · 177 unmodified lines
8311035 * (Currently stuck as a macro because of indirect forward reference
8321036 * to linux/mm.h:page_to_nid())
8331037 */
834−#define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot))
1038+#define mk_pte(page, pgprot) \
1039+({ \
1040+ pgprot_t __pgprot = pgprot; \
1041+ \
1042+ WARN_ON_ONCE((pgprot_val(__pgprot) & (_PAGE_DIRTY | _PAGE_RW)) == \
1043+ _PAGE_DIRTY); \
1044+ pfn_pte(page_to_pfn(page), __pgprot); \
1045+})
8351046
8361047 static inline int pmd_bad(pmd_t pmd)
8371048 {
@@ -1090,7 +1301,17 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, · 252 unmodified lines
10901301 static inline void ptep_set_wrprotect(struct mm_struct *mm,
10911302 unsigned long addr, pte_t *ptep)
10921303 {
1093− clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte);
1304+ /*
1305+ * Avoid accidentally creating shadow stack PTEs
1306+ * (Write=0,Dirty=1). Use cmpxchg() to prevent races with
1307+ * the hardware setting Dirty=1.
1308+ */
1309+ pte_t old_pte, new_pte;
1310+
1311+ old_pte = READ_ONCE(*ptep);
1312+ do {
1313+ new_pte = pte_wrprotect(old_pte);
1314+ } while (!try_cmpxchg((long *)&ptep->pte, (long *)&old_pte, *(long *)&new_pte));
10941315 }
10951316
10961317 #define flush_tlb_fix_spurious_fault(vma, address, ptep) do { } while (0)
@@ -1116,12 +1337,6 @@ extern int pmdp_clear_flush_young(struct vm_area_struct *vma, · 19 unmodified lines
11161337 unsigned long address, pmd_t *pmdp);
11171338
11181339
1119−#define pmd_write pmd_write
1120−static inline int pmd_write(pmd_t pmd)
1121−{
1122− return pmd_flags(pmd) & _PAGE_RW;
1123−}
1124−
11251340 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
11261341 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long addr,
11271342 pmd_t *pmdp)
@@ -1148,13 +1363,17 @@ static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, · 20 unmodified lines
11481363 static inline void pmdp_set_wrprotect(struct mm_struct *mm,
11491364 unsigned long addr, pmd_t *pmdp)
11501365 {
1151− clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp);
1152−}
1366+ /*
1367+ * Avoid accidentally creating shadow stack PTEs
1368+ * (Write=0,Dirty=1). Use cmpxchg() to prevent races with
1369+ * the hardware setting Dirty=1.
1370+ */
1371+ pmd_t old_pmd, new_pmd;
11531372
1154−#define pud_write pud_write
1155−static inline int pud_write(pud_t pud)
1156−{
1157− return pud_flags(pud) & _PAGE_RW;
1373+ old_pmd = READ_ONCE(*pmdp);
1374+ do {
1375+ new_pmd = pmd_wrprotect(old_pmd);
1376+ } while (!try_cmpxchg((long *)pmdp, (long *)&old_pmd, *(long *)&new_pmd));
11581377 }
11591378
11601379 #ifndef pmdp_establish
@@ -1412,6 +1631,11 @@ static inline bool __pte_access_permitted(unsigned long pteval, bool write) · 251 unmodified lines
14121631 {
14131632 unsigned long need_pte_bits = _PAGE_PRESENT|_PAGE_USER;
14141633
1634+ /*
1635+ * Write=0,Dirty=1 PTEs are shadow stack, which the kernel
1636+ * shouldn't generally allow access to, but since they
1637+ * are already Write=0, the below logic covers both cases.
1638+ */
14151639 if (write)
14161640 need_pte_bits |= _PAGE_RW;
14171641
@@ -1453,6 +1677,12 @@ static inline bool arch_has_hw_pte_young(void) · 35 unmodified lines
14531677 return true;
14541678 }
14551679
1680+#define arch_check_zapped_pte arch_check_zapped_pte
1681+void arch_check_zapped_pte(struct vm_area_struct *vma, pte_t pte);
1682+
1683+#define arch_check_zapped_pmd arch_check_zapped_pmd
1684+void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd);
1685+
14561686 #ifdef CONFIG_XEN_PV
14571687 #define arch_has_hw_nonleaf_pmd_young arch_has_hw_nonleaf_pmd_young
14581688 static inline bool arch_has_hw_nonleaf_pmd_young(void)
arch/x86/include/asm/pgtable_types.h+36−8
@@ -21,7 +21,8 @@ · 20 unmodified lines
2121 #define _PAGE_BIT_SOFTW2 10 /* " */
2222 #define _PAGE_BIT_SOFTW3 11 /* " */
2323 #define _PAGE_BIT_PAT_LARGE 12 /* On 2MB or 1GB pages */
24−#define _PAGE_BIT_SOFTW4 58 /* available for programmer */
24+#define _PAGE_BIT_SOFTW4 57 /* available for programmer */
25+#define _PAGE_BIT_SOFTW5 58 /* available for programmer */
2526 #define _PAGE_BIT_PKEY_BIT0 59 /* Protection Keys, bit 1/4 */
2627 #define _PAGE_BIT_PKEY_BIT1 60 /* Protection Keys, bit 2/4 */
2728 #define _PAGE_BIT_PKEY_BIT2 61 /* Protection Keys, bit 3/4 */
@@ -34,6 +35,13 @@ · 6 unmodified lines
3435 #define _PAGE_BIT_SOFT_DIRTY _PAGE_BIT_SOFTW3 /* software dirty tracking */
3536 #define _PAGE_BIT_DEVMAP _PAGE_BIT_SOFTW4
3637
38+#ifdef CONFIG_X86_64
39+#define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW5 /* Saved Dirty bit */
40+#else
41+/* Shared with _PAGE_BIT_UFFD_WP which is not supported on 32 bit */
42+#define _PAGE_BIT_SAVED_DIRTY _PAGE_BIT_SOFTW2 /* Saved Dirty bit */
43+#endif
44+
3745 /* If _PAGE_BIT_PRESENT is clear, we use these: */
3846 /* - if the user mapped it with PROT_NONE; pte_present gives true */
3947 #define _PAGE_BIT_PROTNONE _PAGE_BIT_GLOBAL
@@ -117,6 +125,18 @@ · 77 unmodified lines
117125 #define _PAGE_SOFTW4 (_AT(pteval_t, 0))
118126 #endif
119127
128+/*
129+ * The hardware requires shadow stack to be Write=0,Dirty=1. However,
130+ * there are valid cases where the kernel might create read-only PTEs that
131+ * are dirty (e.g., fork(), mprotect(), uffd-wp(), soft-dirty tracking). In
132+ * this case, the _PAGE_SAVED_DIRTY bit is used instead of the HW-dirty bit,
133+ * to avoid creating a wrong "shadow stack" PTEs. Such PTEs have
134+ * (Write=0,SavedDirty=1,Dirty=0) set.
135+ */
136+#define _PAGE_SAVED_DIRTY (_AT(pteval_t, 1) << _PAGE_BIT_SAVED_DIRTY)
137+
138+#define _PAGE_DIRTY_BITS (_PAGE_DIRTY | _PAGE_SAVED_DIRTY)
139+
120140 #define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE)
121141
122142 /*
@@ -125,10 +145,10 @@ · 2 unmodified lines
125145 * instance, and is *not* included in this mask since
126146 * pte_modify() does modify it.
127147 */
128−#define _COMMON_PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \
129− _PAGE_SPECIAL | _PAGE_ACCESSED | _PAGE_DIRTY |\
130− _PAGE_SOFT_DIRTY | _PAGE_DEVMAP | _PAGE_ENC | \
131− _PAGE_UFFD_WP)
148+#define _COMMON_PAGE_CHG_MASK (PTE_PFN_MASK | _PAGE_PCD | _PAGE_PWT | \
149+ _PAGE_SPECIAL | _PAGE_ACCESSED | \
150+ _PAGE_DIRTY_BITS | _PAGE_SOFT_DIRTY | \
151+ _PAGE_DEVMAP | _PAGE_ENC | _PAGE_UFFD_WP)
132152 #define _PAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PAT)
133153 #define _HPAGE_CHG_MASK (_COMMON_PAGE_CHG_MASK | _PAGE_PSE | _PAGE_PAT_LARGE)
134154
@@ -189,14 +209,22 @@ enum page_cache_mode { · 54 unmodified lines
189209
190210 #define __PAGE_KERNEL (__PP|__RW| 0|___A|__NX|___D| 0|___G)
191211 #define __PAGE_KERNEL_EXEC (__PP|__RW| 0|___A| 0|___D| 0|___G)
212+
213+/*
214+ * Page tables needs to have Write=1 in order for any lower PTEs to be
215+ * writable. This includes shadow stack memory (Write=0, Dirty=1)
216+ */
192217 #define _KERNPG_TABLE_NOENC (__PP|__RW| 0|___A| 0|___D| 0| 0)
193218 #define _KERNPG_TABLE (__PP|__RW| 0|___A| 0|___D| 0| 0| _ENC)
194219 #define _PAGE_TABLE_NOENC (__PP|__RW|_USR|___A| 0|___D| 0| 0)
195220 #define _PAGE_TABLE (__PP|__RW|_USR|___A| 0|___D| 0| 0| _ENC)
196−#define __PAGE_KERNEL_RO (__PP| 0| 0|___A|__NX|___D| 0|___G)
197−#define __PAGE_KERNEL_ROX (__PP| 0| 0|___A| 0|___D| 0|___G)
221+
222+#define __PAGE_KERNEL_RO (__PP| 0| 0|___A|__NX| 0| 0|___G)
223+#define __PAGE_KERNEL_ROX (__PP| 0| 0|___A| 0| 0| 0|___G)
224+#define __PAGE_KERNEL (__PP|__RW| 0|___A|__NX|___D| 0|___G)
225+#define __PAGE_KERNEL_EXEC (__PP|__RW| 0|___A| 0|___D| 0|___G)
198226 #define __PAGE_KERNEL_NOCACHE (__PP|__RW| 0|___A|__NX|___D| 0|___G| __NC)
199−#define __PAGE_KERNEL_VVAR (__PP| 0|_USR|___A|__NX|___D| 0|___G)
227+#define __PAGE_KERNEL_VVAR (__PP| 0|_USR|___A|__NX| 0| 0|___G)
200228 #define __PAGE_KERNEL_LARGE (__PP|__RW| 0|___A|__NX|___D|_PSE|___G)
201229 #define __PAGE_KERNEL_LARGE_EXEC (__PP|__RW| 0|___A| 0|___D|_PSE|___G)
202230 #define __PAGE_KERNEL_WP (__PP|__RW| 0|___A|__NX|___D| 0|___G| __WP)
arch/x86/include/asm/processor.h+8−0
@@ -28,6 +28,7 @@ struct vm86; · 27 unmodified lines
2828 #include <asm/unwind_hints.h>
2929 #include <asm/vmxfeatures.h>
3030 #include <asm/vdso/processor.h>
31+#include <asm/shstk.h>
3132
3233 #include <linux/personality.h>
3334 #include <linux/cache.h>
@@ -474,6 +475,13 @@ struct thread_struct { · 440 unmodified lines
474475 */
475476 u32 pkru;
476477
478+#ifdef CONFIG_X86_USER_SHADOW_STACK
479+ unsigned long features;
480+ unsigned long features_locked;
481+
482+ struct thread_shstk shstk;
483+#endif
484+
477485 /* Floating point and extended processor state */
478486 struct fpu fpu;
479487 /*
arch/x86/include/asm/shstk.h+38−0
@@ -0,0 +1,38 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+#ifndef _ASM_X86_SHSTK_H
3+#define _ASM_X86_SHSTK_H
4+
5+#ifndef __ASSEMBLY__
6+#include <linux/types.h>
7+
8+struct task_struct;
9+struct ksignal;
10+
11+#ifdef CONFIG_X86_USER_SHADOW_STACK
12+struct thread_shstk {
13+ u64 base;
14+ u64 size;
15+};
16+
17+long shstk_prctl(struct task_struct *task, int option, unsigned long arg2);
18+void reset_thread_features(void);
19+unsigned long shstk_alloc_thread_stack(struct task_struct *p, unsigned long clone_flags,
20+ unsigned long stack_size);
21+void shstk_free(struct task_struct *p);
22+int setup_signal_shadow_stack(struct ksignal *ksig);
23+int restore_signal_shadow_stack(void);
24+#else
25+static inline long shstk_prctl(struct task_struct *task, int option,
26+ unsigned long arg2) { return -EINVAL; }
27+static inline void reset_thread_features(void) {}
28+static inline unsigned long shstk_alloc_thread_stack(struct task_struct *p,
29+ unsigned long clone_flags,
30+ unsigned long stack_size) { return 0; }
31+static inline void shstk_free(struct task_struct *p) {}
32+static inline int setup_signal_shadow_stack(struct ksignal *ksig) { return 0; }
33+static inline int restore_signal_shadow_stack(void) { return 0; }
34+#endif /* CONFIG_X86_USER_SHADOW_STACK */
35+
36+#endif /* __ASSEMBLY__ */
37+
38+#endif /* _ASM_X86_SHSTK_H */
arch/x86/include/asm/special_insns.h+13−0
@@ -202,6 +202,19 @@ static inline void clwb(volatile void *__p) · 201 unmodified lines
202202 : [pax] "a" (p));
203203 }
204204
205+#ifdef CONFIG_X86_USER_SHADOW_STACK
206+static inline int write_user_shstk_64(u64 __user *addr, u64 val)
207+{
208+ asm_volatile_goto("1: wrussq %[val], (%[addr])\n"
209+ _ASM_EXTABLE(1b, %l[fail])
210+ :: [addr] "r" (addr), [val] "r" (val)
211+ :: fail);
212+ return 0;
213+fail:
214+ return -EFAULT;
215+}
216+#endif /* CONFIG_X86_USER_SHADOW_STACK */
217+
205218 #define nop() asm volatile ("nop")
206219
207220 static inline void serialize(void)
arch/x86/include/asm/tlbflush.h+2−1
@@ -306,7 +306,8 @@ static inline bool pte_flags_need_flush(unsigned long oldflags, · 305 unmodified lines
306306 const pteval_t flush_on_clear = _PAGE_DIRTY | _PAGE_PRESENT |
307307 _PAGE_ACCESSED;
308308 const pteval_t software_flags = _PAGE_SOFTW1 | _PAGE_SOFTW2 |
309− _PAGE_SOFTW3 | _PAGE_SOFTW4;
309+ _PAGE_SOFTW3 | _PAGE_SOFTW4 |
310+ _PAGE_SAVED_DIRTY;
310311 const pteval_t flush_on_change = _PAGE_RW | _PAGE_USER | _PAGE_PWT |
311312 _PAGE_PCD | _PAGE_PSE | _PAGE_GLOBAL | _PAGE_PAT |
312313 _PAGE_PAT_LARGE | _PAGE_PKEY_BIT0 | _PAGE_PKEY_BIT1 |
arch/x86/include/asm/trap_pf.h+2−0
@@ -11,6 +11,7 @@ · 10 unmodified lines
1111 * bit 3 == 1: use of reserved bit detected
1212 * bit 4 == 1: fault was an instruction fetch
1313 * bit 5 == 1: protection keys block access
14+ * bit 6 == 1: shadow stack access fault
1415 * bit 15 == 1: SGX MMU page-fault
1516 */
1617 enum x86_pf_error_code {
@@ -20,6 +21,7 @@ enum x86_pf_error_code { · 3 unmodified lines
2021 X86_PF_RSVD = 1 << 3,
2122 X86_PF_INSTR = 1 << 4,
2223 X86_PF_PK = 1 << 5,
24+ X86_PF_SHSTK = 1 << 6,
2325 X86_PF_SGX = 1 << 15,
2426 };
2527
arch/x86/include/asm/traps.h+14−1
@@ -18,7 +18,8 @@ void __init trap_init(void); · 17 unmodified lines
1818 asmlinkage __visible noinstr struct pt_regs *vc_switch_off_ist(struct pt_regs *eregs);
1919 #endif
2020
21−extern bool ibt_selftest(void);
21+extern int ibt_selftest(void);
22+extern int ibt_selftest_noendbr(void);
2223
2324 #ifdef CONFIG_X86_F00F_BUG
2425 /* For handling the FOOF bug */
@@ -47,4 +48,16 @@ void __noreturn handle_stack_overflow(struct pt_regs *regs, · 22 unmodified lines
4748 struct stack_info *info);
4849 #endif
4950
51+static inline void cond_local_irq_enable(struct pt_regs *regs)
52+{
53+ if (regs->flags & X86_EFLAGS_IF)
54+ local_irq_enable();
55+}
56+
57+static inline void cond_local_irq_disable(struct pt_regs *regs)
58+{
59+ if (regs->flags & X86_EFLAGS_IF)
60+ local_irq_disable();
61+}
62+
5063 #endif /* _ASM_X86_TRAPS_H */
arch/x86/include/uapi/asm/mman.h+4−0
@@ -3,6 +3,7 @@ · 2 unmodified lines
33 #define _ASM_X86_MMAN_H
44
55 #define MAP_32BIT 0x40 /* only give out 32bit addresses */
6+#define MAP_ABOVE4G 0x80 /* only map above 4GB */
67
78 #ifdef CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS
89 #define arch_calc_vm_prot_bits(prot, key) ( \
@@ -12,6 +13,9 @@ · 3 unmodified lines
1213 ((key) & 0x8 ? VM_PKEY_BIT3 : 0))
1314 #endif
1415
16+/* Flags for map_shadow_stack(2) */
17+#define SHADOW_STACK_SET_TOKEN (1ULL << 0) /* Set up a restore token in the shadow stack */
18+
1519 #include <asm-generic/mman.h>
1620
1721 #endif /* _ASM_X86_MMAN_H */
arch/x86/include/uapi/asm/prctl.h+12−0
@@ -23,9 +23,21 @@ · 22 unmodified lines
2323 #define ARCH_MAP_VDSO_32 0x2002
2424 #define ARCH_MAP_VDSO_64 0x2003
2525
26+/* Don't use 0x3001-0x3004 because of old glibcs */
27+
2628 #define ARCH_GET_UNTAG_MASK 0x4001
2729 #define ARCH_ENABLE_TAGGED_ADDR 0x4002
2830 #define ARCH_GET_MAX_TAG_BITS 0x4003
2931 #define ARCH_FORCE_TAGGED_SVA 0x4004
3032
33+#define ARCH_SHSTK_ENABLE 0x5001
34+#define ARCH_SHSTK_DISABLE 0x5002
35+#define ARCH_SHSTK_LOCK 0x5003
36+#define ARCH_SHSTK_UNLOCK 0x5004
37+#define ARCH_SHSTK_STATUS 0x5005
38+
39+/* ARCH_SHSTK_ features bits */
40+#define ARCH_SHSTK_SHSTK (1ULL << 0)
41+#define ARCH_SHSTK_WRSS (1ULL << 1)
42+
3143 #endif /* _ASM_X86_PRCTL_H */
arch/x86/kernel/Makefile+5−0
@@ -48,6 +48,7 @@ obj-y += process_$(BITS).o signal.o signal_$(BITS).o · 47 unmodified lines
4848 obj-y += traps.o idt.o irq.o irq_$(BITS).o dumpstack_$(BITS).o
4949 obj-y += time.o ioport.o dumpstack.o nmi.o
5050 obj-$(CONFIG_MODIFY_LDT_SYSCALL) += ldt.o
51+obj-$(CONFIG_X86_KERNEL_IBT) += ibt_selftest.o
5152 obj-y += setup.o x86_init.o i8259.o irqinit.o
5253 obj-$(CONFIG_JUMP_LABEL) += jump_label.o
5354 obj-$(CONFIG_IRQ_WORK) += irq_work.o
@@ -144,6 +145,10 @@ obj-$(CONFIG_CFI_CLANG) += cfi.o · 90 unmodified lines
144145
145146 obj-$(CONFIG_CALL_THUNKS) += callthunks.o
146147
148+obj-$(CONFIG_X86_CET) += cet.o
149+
150+obj-$(CONFIG_X86_USER_SHADOW_STACK) += shstk.o
151+
147152 ###
148153 # 64 bit specific files
149154 ifeq ($(CONFIG_X86_64),y)
arch/x86/kernel/cet.c+131−0
@@ -0,0 +1,131 @@
1+// SPDX-License-Identifier: GPL-2.0
2+
3+#include <linux/ptrace.h>
4+#include <asm/bugs.h>
5+#include <asm/traps.h>
6+
7+enum cp_error_code {
8+ CP_EC = (1 << 15) - 1,
9+
10+ CP_RET = 1,
11+ CP_IRET = 2,
12+ CP_ENDBR = 3,
13+ CP_RSTRORSSP = 4,
14+ CP_SETSSBSY = 5,
15+
16+ CP_ENCL = 1 << 15,
17+};
18+
19+static const char cp_err[][10] = {
20+ [0] = "unknown",
21+ [1] = "near ret",
22+ [2] = "far/iret",
23+ [3] = "endbranch",
24+ [4] = "rstorssp",
25+ [5] = "setssbsy",
26+};
27+
28+static const char *cp_err_string(unsigned long error_code)
29+{
30+ unsigned int cpec = error_code & CP_EC;
31+
32+ if (cpec >= ARRAY_SIZE(cp_err))
33+ cpec = 0;
34+ return cp_err[cpec];
35+}
36+
37+static void do_unexpected_cp(struct pt_regs *regs, unsigned long error_code)
38+{
39+ WARN_ONCE(1, "Unexpected %s #CP, error_code: %s\n",
40+ user_mode(regs) ? "user mode" : "kernel mode",
41+ cp_err_string(error_code));
42+}
43+
44+static DEFINE_RATELIMIT_STATE(cpf_rate, DEFAULT_RATELIMIT_INTERVAL,
45+ DEFAULT_RATELIMIT_BURST);
46+
47+static void do_user_cp_fault(struct pt_regs *regs, unsigned long error_code)
48+{
49+ struct task_struct *tsk;
50+ unsigned long ssp;
51+
52+ /*
53+ * An exception was just taken from userspace. Since interrupts are disabled
54+ * here, no scheduling should have messed with the registers yet and they
55+ * will be whatever is live in userspace. So read the SSP before enabling
56+ * interrupts so locking the fpregs to do it later is not required.
57+ */
58+ rdmsrl(MSR_IA32_PL3_SSP, ssp);
59+
60+ cond_local_irq_enable(regs);
61+
62+ tsk = current;
63+ tsk->thread.error_code = error_code;
64+ tsk->thread.trap_nr = X86_TRAP_CP;
65+
66+ /* Ratelimit to prevent log spamming. */
67+ if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
68+ __ratelimit(&cpf_rate)) {
69+ pr_emerg("%s[%d] control protection ip:%lx sp:%lx ssp:%lx error:%lx(%s)%s",
70+ tsk->comm, task_pid_nr(tsk),
71+ regs->ip, regs->sp, ssp, error_code,
72+ cp_err_string(error_code),
73+ error_code & CP_ENCL ? " in enclave" : "");
74+ print_vma_addr(KERN_CONT " in ", regs->ip);
75+ pr_cont("\n");
76+ }
77+
78+ force_sig_fault(SIGSEGV, SEGV_CPERR, (void __user *)0);
79+ cond_local_irq_disable(regs);
80+}
81+
82+static __ro_after_init bool ibt_fatal = true;
83+
84+static void do_kernel_cp_fault(struct pt_regs *regs, unsigned long error_code)
85+{
86+ if ((error_code & CP_EC) != CP_ENDBR) {
87+ do_unexpected_cp(regs, error_code);
88+ return;
89+ }
90+
91+ if (unlikely(regs->ip == (unsigned long)&ibt_selftest_noendbr)) {
92+ regs->ax = 0;
93+ return;
94+ }
95+
96+ pr_err("Missing ENDBR: %pS\n", (void *)instruction_pointer(regs));
97+ if (!ibt_fatal) {
98+ printk(KERN_DEFAULT CUT_HERE);
99+ __warn(__FILE__, __LINE__, (void *)regs->ip, TAINT_WARN, regs, NULL);
100+ return;
101+ }
102+ BUG();
103+}
104+
105+static int __init ibt_setup(char *str)
106+{
107+ if (!strcmp(str, "off"))
108+ setup_clear_cpu_cap(X86_FEATURE_IBT);
109+
110+ if (!strcmp(str, "warn"))
111+ ibt_fatal = false;
112+
113+ return 1;
114+}
115+
116+__setup("ibt=", ibt_setup);
117+
118+DEFINE_IDTENTRY_ERRORCODE(exc_control_protection)
119+{
120+ if (user_mode(regs)) {
121+ if (cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
122+ do_user_cp_fault(regs, error_code);
123+ else
124+ do_unexpected_cp(regs, error_code);
125+ } else {
126+ if (cpu_feature_enabled(X86_FEATURE_IBT))
127+ do_kernel_cp_fault(regs, error_code);
128+ else
129+ do_unexpected_cp(regs, error_code);
130+ }
131+}
arch/x86/kernel/cpu/common.c+27−8
@@ -587,27 +587,43 @@ __noendbr void ibt_restore(u64 save) · 586 unmodified lines
587587
588588 static __always_inline void setup_cet(struct cpuinfo_x86 *c)
589589 {
590− u64 msr = CET_ENDBR_EN;
590+ bool user_shstk, kernel_ibt;
591591
592− if (!HAS_KERNEL_IBT ||
593− !cpu_feature_enabled(X86_FEATURE_IBT))
592+ if (!IS_ENABLED(CONFIG_X86_CET))
594593 return;
595594
596− wrmsrl(MSR_IA32_S_CET, msr);
595+ kernel_ibt = HAS_KERNEL_IBT && cpu_feature_enabled(X86_FEATURE_IBT);
596+ user_shstk = cpu_feature_enabled(X86_FEATURE_SHSTK) &&
597+ IS_ENABLED(CONFIG_X86_USER_SHADOW_STACK);
598+
599+ if (!kernel_ibt && !user_shstk)
600+ return;
601+
602+ if (user_shstk)
603+ set_cpu_cap(c, X86_FEATURE_USER_SHSTK);
604+
605+ if (kernel_ibt)
606+ wrmsrl(MSR_IA32_S_CET, CET_ENDBR_EN);
607+ else
608+ wrmsrl(MSR_IA32_S_CET, 0);
609+
597610 cr4_set_bits(X86_CR4_CET);
598611
599− if (!ibt_selftest()) {
612+ if (kernel_ibt && ibt_selftest()) {
600613 pr_err("IBT selftest: Failed!\n");
601614 wrmsrl(MSR_IA32_S_CET, 0);
602615 setup_clear_cpu_cap(X86_FEATURE_IBT);
603− return;
604616 }
605617 }
606618
607619 __noendbr void cet_disable(void)
608620 {
609− if (cpu_feature_enabled(X86_FEATURE_IBT))
610− wrmsrl(MSR_IA32_S_CET, 0);
621+ if (!(cpu_feature_enabled(X86_FEATURE_IBT) ||
622+ cpu_feature_enabled(X86_FEATURE_SHSTK)))
623+ return;
624+
625+ wrmsrl(MSR_IA32_S_CET, 0);
626+ wrmsrl(MSR_IA32_U_CET, 0);
611627 }
612628
613629 /*
@@ -1491,6 +1507,9 @@ static void __init cpu_parse_early_param(void) · 877 unmodified lines
14911507 if (cmdline_find_option_bool(boot_command_line, "noxsaves"))
14921508 setup_clear_cpu_cap(X86_FEATURE_XSAVES);
14931509
1510+ if (cmdline_find_option_bool(boot_command_line, "nousershstk"))
1511+ setup_clear_cpu_cap(X86_FEATURE_USER_SHSTK);
1512+
14941513 arglen = cmdline_find_option(boot_command_line, "clearcpuid", arg, sizeof(arg));
14951514 if (arglen <= 0)
14961515 return;
arch/x86/kernel/cpu/cpuid-deps.c+1−0
@@ -81,6 +81,7 @@ static const struct cpuid_dep cpuid_deps[] = { · 80 unmodified lines
8181 { X86_FEATURE_XFD, X86_FEATURE_XSAVES },
8282 { X86_FEATURE_XFD, X86_FEATURE_XGETBV1 },
8383 { X86_FEATURE_AMX_TILE, X86_FEATURE_XFD },
84+ { X86_FEATURE_SHSTK, X86_FEATURE_XSAVES },
8485 {}
8586 };
8687
arch/x86/kernel/cpu/proc.c+23−0
@@ -4,6 +4,8 @@ · 3 unmodified lines
44 #include <linux/string.h>
55 #include <linux/seq_file.h>
66 #include <linux/cpufreq.h>
7+#include <asm/prctl.h>
8+#include <linux/proc_fs.h>
79
810 #include "cpu.h"
911
@@ -175,3 +177,24 @@ const struct seq_operations cpuinfo_op = { · 165 unmodified lines
175177 .stop = c_stop,
176178 .show = show_cpuinfo,
177179 };
180+
181+#ifdef CONFIG_X86_USER_SHADOW_STACK
182+static void dump_x86_features(struct seq_file *m, unsigned long features)
183+{
184+ if (features & ARCH_SHSTK_SHSTK)
185+ seq_puts(m, "shstk ");
186+ if (features & ARCH_SHSTK_WRSS)
187+ seq_puts(m, "wrss ");
188+}
189+
190+void arch_proc_pid_thread_features(struct seq_file *m, struct task_struct *task)
191+{
192+ seq_puts(m, "x86_Thread_features:\t");
193+ dump_x86_features(m, task->thread.features);
194+ seq_putc(m, '\n');
195+
196+ seq_puts(m, "x86_Thread_features_locked:\t");
197+ dump_x86_features(m, task->thread.features_locked);
198+ seq_putc(m, '\n');
199+}
200+#endif /* CONFIG_X86_USER_SHADOW_STACK */
arch/x86/kernel/fpu/core.c+53−1
@@ -552,8 +552,36 @@ static inline void fpu_inherit_perms(struct fpu *dst_fpu) · 551 unmodified lines
552552 }
553553 }
554554
555+/* A passed ssp of zero will not cause any update */
556+static int update_fpu_shstk(struct task_struct *dst, unsigned long ssp)
557+{
558+#ifdef CONFIG_X86_USER_SHADOW_STACK
559+ struct cet_user_state *xstate;
560+
561+ /* If ssp update is not needed. */
562+ if (!ssp)
563+ return 0;
564+
565+ xstate = get_xsave_addr(&dst->thread.fpu.fpstate->regs.xsave,
566+ XFEATURE_CET_USER);
567+
568+ /*
569+ * If there is a non-zero ssp, then 'dst' must be configured with a shadow
570+ * stack and the fpu state should be up to date since it was just copied
571+ * from the parent in fpu_clone(). So there must be a valid non-init CET
572+ * state location in the buffer.
573+ */
574+ if (WARN_ON_ONCE(!xstate))
575+ return 1;
576+
577+ xstate->user_ssp = (u64)ssp;
578+#endif
579+ return 0;
580+}
581+
555582 /* Clone current's FPU state on fork */
556−int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal)
583+int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal,
584+ unsigned long ssp)
557585 {
558586 struct fpu *src_fpu = &current->thread.fpu;
559587 struct fpu *dst_fpu = &dst->thread.fpu;
@@ -613,6 +641,12 @@ int fpu_clone(struct task_struct *dst, unsigned long clone_flags, bool minimal) · 53 unmodified lines
613641 if (use_xsave())
614642 dst_fpu->fpstate->regs.xsave.header.xfeatures &= ~XFEATURE_MASK_PASID;
615643
644+ /*
645+ * Update shadow stack pointer, in case it changed during clone.
646+ */
647+ if (update_fpu_shstk(dst, ssp))
648+ return 1;
649+
616650 trace_x86_fpu_copy_src(src_fpu);
617651 trace_x86_fpu_copy_dst(dst_fpu);
618652
@@ -753,6 +787,24 @@ void switch_fpu_return(void) · 134 unmodified lines
753787 }
754788 EXPORT_SYMBOL_GPL(switch_fpu_return);
755789
790+void fpregs_lock_and_load(void)
791+{
792+ /*
793+ * fpregs_lock() only disables preemption (mostly). So modifying state
794+ * in an interrupt could screw up some in progress fpregs operation.
795+ * Warn about it.
796+ */
797+ WARN_ON_ONCE(!irq_fpu_usable());
798+ WARN_ON_ONCE(current->flags & PF_KTHREAD);
799+
800+ fpregs_lock();
801+
802+ fpregs_assert_state_consistent();
803+
804+ if (test_thread_flag(TIF_NEED_FPU_LOAD))
805+ fpregs_restore_userregs();
806+}
807+
756808 #ifdef CONFIG_X86_DEBUG_FPU
757809 /*
758810 * If current FPU state according to its tracking (loaded FPU context on this
arch/x86/kernel/fpu/regset.c+81−0
@@ -8,6 +8,7 @@ · 7 unmodified lines
88 #include <asm/fpu/api.h>
99 #include <asm/fpu/signal.h>
1010 #include <asm/fpu/regset.h>
11+#include <asm/prctl.h>
1112
1213 #include "context.h"
1314 #include "internal.h"
@@ -174,6 +175,86 @@ out: · 160 unmodified lines
174175 return ret;
175176 }
176177
178+#ifdef CONFIG_X86_USER_SHADOW_STACK
179+int ssp_active(struct task_struct *target, const struct user_regset *regset)
180+{
181+ if (target->thread.features & ARCH_SHSTK_SHSTK)
182+ return regset->n;
183+
184+ return 0;
185+}
186+
187+int ssp_get(struct task_struct *target, const struct user_regset *regset,
188+ struct membuf to)
189+{
190+ struct fpu *fpu = &target->thread.fpu;
191+ struct cet_user_state *cetregs;
192+
193+ if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
194+ return -ENODEV;
195+
196+ sync_fpstate(fpu);
197+ cetregs = get_xsave_addr(&fpu->fpstate->regs.xsave, XFEATURE_CET_USER);
198+ if (WARN_ON(!cetregs)) {
199+ /*
200+ * This shouldn't ever be NULL because shadow stack was
201+ * verified to be enabled above. This means
202+ * MSR_IA32_U_CET.CET_SHSTK_EN should be 1 and so
203+ * XFEATURE_CET_USER should not be in the init state.
204+ */
205+ return -ENODEV;
206+ }
207+
208+ return membuf_write(&to, (unsigned long *)&cetregs->user_ssp,
209+ sizeof(cetregs->user_ssp));
210+}
211+
212+int ssp_set(struct task_struct *target, const struct user_regset *regset,
213+ unsigned int pos, unsigned int count,
214+ const void *kbuf, const void __user *ubuf)
215+{
216+ struct fpu *fpu = &target->thread.fpu;
217+ struct xregs_state *xsave = &fpu->fpstate->regs.xsave;
218+ struct cet_user_state *cetregs;
219+ unsigned long user_ssp;
220+ int r;
221+
222+ if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) ||
223+ !ssp_active(target, regset))
224+ return -ENODEV;
225+
226+ if (pos != 0 || count != sizeof(user_ssp))
227+ return -EINVAL;
228+
229+ r = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &user_ssp, 0, -1);
230+ if (r)
231+ return r;
232+
233+ /*
234+ * Some kernel instructions (IRET, etc) can cause exceptions in the case
235+ * of disallowed CET register values. Just prevent invalid values.
236+ */
237+ if (user_ssp >= TASK_SIZE_MAX || !IS_ALIGNED(user_ssp, 8))
238+ return -EINVAL;
239+
240+ fpu_force_restore(fpu);
241+
242+ cetregs = get_xsave_addr(xsave, XFEATURE_CET_USER);
243+ if (WARN_ON(!cetregs)) {
244+ /*
245+ * This shouldn't ever be NULL because shadow stack was
246+ * verified to be enabled above. This means
247+ * MSR_IA32_U_CET.CET_SHSTK_EN should be 1 and so
248+ * XFEATURE_CET_USER should not be in the init state.
249+ */
250+ return -ENODEV;
251+ }
252+
253+ cetregs->user_ssp = user_ssp;
254+ return 0;
255+}
256+#endif /* CONFIG_X86_USER_SHADOW_STACK */
257+
177258 #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
178259
179260 /*
arch/x86/kernel/fpu/xstate.c+43−47
@@ -39,26 +39,26 @@ · 38 unmodified lines
3939 */
4040 static const char *xfeature_names[] =
4141 {
42− "x87 floating point registers" ,
43− "SSE registers" ,
44− "AVX registers" ,
45− "MPX bounds registers" ,
46− "MPX CSR" ,
47− "AVX-512 opmask" ,
48− "AVX-512 Hi256" ,
49− "AVX-512 ZMM_Hi256" ,
50− "Processor Trace (unused)" ,
42+ "x87 floating point registers",
43+ "SSE registers",
44+ "AVX registers",
45+ "MPX bounds registers",
46+ "MPX CSR",
47+ "AVX-512 opmask",
48+ "AVX-512 Hi256",
49+ "AVX-512 ZMM_Hi256",
50+ "Processor Trace (unused)",
5151 "Protection Keys User registers",
5252 "PASID state",
53− "unknown xstate feature" ,
54− "unknown xstate feature" ,
55− "unknown xstate feature" ,
56− "unknown xstate feature" ,
57− "unknown xstate feature" ,
58− "unknown xstate feature" ,
59− "AMX Tile config" ,
60− "AMX Tile data" ,
61− "unknown xstate feature" ,
53+ "Control-flow User registers",
54+ "Control-flow Kernel registers (unused)",
55+ "unknown xstate feature",
56+ "unknown xstate feature",
57+ "unknown xstate feature",
58+ "unknown xstate feature",
59+ "AMX Tile config",
60+ "AMX Tile data",
61+ "unknown xstate feature",
6262 };
6363
6464 static unsigned short xsave_cpuid_features[] __initdata = {
@@ -73,6 +73,7 @@ static unsigned short xsave_cpuid_features[] __initdata = { · 8 unmodified lines
7373 [XFEATURE_PT_UNIMPLEMENTED_SO_FAR] = X86_FEATURE_INTEL_PT,
7474 [XFEATURE_PKRU] = X86_FEATURE_PKU,
7575 [XFEATURE_PASID] = X86_FEATURE_ENQCMD,
76+ [XFEATURE_CET_USER] = X86_FEATURE_SHSTK,
7677 [XFEATURE_XTILE_CFG] = X86_FEATURE_AMX_TILE,
7778 [XFEATURE_XTILE_DATA] = X86_FEATURE_AMX_TILE,
7879 };
@@ -276,6 +277,7 @@ static void __init print_xstate_features(void) · 197 unmodified lines
276277 print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
277278 print_xstate_feature(XFEATURE_MASK_PKRU);
278279 print_xstate_feature(XFEATURE_MASK_PASID);
280+ print_xstate_feature(XFEATURE_MASK_CET_USER);
279281 print_xstate_feature(XFEATURE_MASK_XTILE_CFG);
280282 print_xstate_feature(XFEATURE_MASK_XTILE_DATA);
281283 }
@@ -344,6 +346,7 @@ static __init void os_xrstor_booting(struct xregs_state *xstate) · 62 unmodified lines
344346 XFEATURE_MASK_BNDREGS | \
345347 XFEATURE_MASK_BNDCSR | \
346348 XFEATURE_MASK_PASID | \
349+ XFEATURE_MASK_CET_USER | \
347350 XFEATURE_MASK_XTILE)
348351
349352 /*
@@ -446,14 +449,15 @@ static void __init __xstate_dump_leaves(void) · 96 unmodified lines
446449 } \
447450 } while (0)
448451
449−#define XCHECK_SZ(sz, nr, nr_macro, __struct) do { \
450− if ((nr == nr_macro) && \
451− WARN_ONCE(sz != sizeof(__struct), \
452− "%s: struct is %zu bytes, cpu state %d bytes\n", \
453− __stringify(nr_macro), sizeof(__struct), sz)) { \
452+#define XCHECK_SZ(sz, nr, __struct) ({ \
453+ if (WARN_ONCE(sz != sizeof(__struct), \
454+ "[%s]: struct is %zu bytes, cpu state %d bytes\n", \
455+ xfeature_names[nr], sizeof(__struct), sz)) { \
454456 __xstate_dump_leaves(); \
455457 } \
456−} while (0)
458+ true; \
459+})
460+
457461
458462 /**
459463 * check_xtile_data_against_struct - Check tile data state size.
@@ -527,36 +531,28 @@ static bool __init check_xstate_against_struct(int nr) · 67 unmodified lines
527531 * Ask the CPU for the size of the state.
528532 */
529533 int sz = xfeature_size(nr);
534+
530535 /*
531536 * Match each CPU state with the corresponding software
532537 * structure.
533538 */
534− XCHECK_SZ(sz, nr, XFEATURE_YMM, struct ymmh_struct);
535− XCHECK_SZ(sz, nr, XFEATURE_BNDREGS, struct mpx_bndreg_state);
536− XCHECK_SZ(sz, nr, XFEATURE_BNDCSR, struct mpx_bndcsr_state);
537− XCHECK_SZ(sz, nr, XFEATURE_OPMASK, struct avx_512_opmask_state);
538− XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state);
539− XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM, struct avx_512_hi16_state);
540− XCHECK_SZ(sz, nr, XFEATURE_PKRU, struct pkru_state);
541− XCHECK_SZ(sz, nr, XFEATURE_PASID, struct ia32_pasid_state);
542− XCHECK_SZ(sz, nr, XFEATURE_XTILE_CFG, struct xtile_cfg);
543−
544− /* The tile data size varies between implementations. */
545− if (nr == XFEATURE_XTILE_DATA)
546− check_xtile_data_against_struct(sz);
547−
548− /*
549− * Make *SURE* to add any feature numbers in below if
550− * there are "holes" in the xsave state component
551− * numbers.
552− */
553− if ((nr < XFEATURE_YMM) ||
554− (nr >= XFEATURE_MAX) ||
555− (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) ||
556− ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_RSRVD_COMP_16))) {
539+ switch (nr) {
540+ case XFEATURE_YMM: return XCHECK_SZ(sz, nr, struct ymmh_struct);
541+ case XFEATURE_BNDREGS: return XCHECK_SZ(sz, nr, struct mpx_bndreg_state);
542+ case XFEATURE_BNDCSR: return XCHECK_SZ(sz, nr, struct mpx_bndcsr_state);
543+ case XFEATURE_OPMASK: return XCHECK_SZ(sz, nr, struct avx_512_opmask_state);
544+ case XFEATURE_ZMM_Hi256: return XCHECK_SZ(sz, nr, struct avx_512_zmm_uppers_state);
545+ case XFEATURE_Hi16_ZMM: return XCHECK_SZ(sz, nr, struct avx_512_hi16_state);
546+ case XFEATURE_PKRU: return XCHECK_SZ(sz, nr, struct pkru_state);
547+ case XFEATURE_PASID: return XCHECK_SZ(sz, nr, struct ia32_pasid_state);
548+ case XFEATURE_XTILE_CFG: return XCHECK_SZ(sz, nr, struct xtile_cfg);
549+ case XFEATURE_CET_USER: return XCHECK_SZ(sz, nr, struct cet_user_state);
550+ case XFEATURE_XTILE_DATA: check_xtile_data_against_struct(sz); return true;
551+ default:
557552 XSTATE_WARN_ON(1, "No structure for xstate: %d\n", nr);
558553 return false;
559554 }
555+
560556 return true;
561557 }
562558
arch/x86/kernel/ibt_selftest.S+17−0
@@ -0,0 +1,17 @@
1+/* SPDX-License-Identifier: GPL-2.0 */
2+#include <linux/linkage.h>
3+#include <linux/objtool.h>
4+#include <asm/nospec-branch.h>
5+
6+SYM_CODE_START(ibt_selftest_noendbr)
7+ ANNOTATE_NOENDBR
8+ UNWIND_HINT_FUNC
9+ /* #CP handler sets %ax to 0 */
10+ RET
11+SYM_CODE_END(ibt_selftest_noendbr)
12+
13+SYM_FUNC_START(ibt_selftest)
14+ lea ibt_selftest_noendbr(%rip), %rax
15+ ANNOTATE_RETPOLINE_SAFE
16+ jmp *%rax
17+SYM_FUNC_END(ibt_selftest)
arch/x86/kernel/idt.c+1−1
@@ -107,7 +107,7 @@ static const __initconst struct idt_data def_idts[] = { · 106 unmodified lines
107107 ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE),
108108 #endif
109109
110−#ifdef CONFIG_X86_KERNEL_IBT
110+#ifdef CONFIG_X86_CET
111111 INTG(X86_TRAP_CP, asm_exc_control_protection),
112112 #endif
113113
arch/x86/kernel/process.c+20−1
@@ -51,6 +51,7 @@ · 50 unmodified lines
5151 #include <asm/unwind.h>
5252 #include <asm/tdx.h>
5353 #include <asm/mmu_context.h>
54+#include <asm/shstk.h>
5455
5556 #include "process.h"
5657
@@ -122,6 +123,7 @@ void exit_thread(struct task_struct *tsk) · 65 unmodified lines
122123
123124 free_vm86(t);
124125
126+ shstk_free(tsk);
125127 fpu__drop(fpu);
126128 }
127129
@@ -162,6 +164,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) · 34 unmodified lines
162164 struct inactive_task_frame *frame;
163165 struct fork_frame *fork_frame;
164166 struct pt_regs *childregs;
167+ unsigned long new_ssp;
165168 int ret = 0;
166169
167170 childregs = task_pt_regs(p);
@@ -199,7 +202,16 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) · 31 unmodified lines
199202 frame->flags = X86_EFLAGS_FIXED;
200203 #endif
201204
202− fpu_clone(p, clone_flags, args->fn);
205+ /*
206+ * Allocate a new shadow stack for thread if needed. If shadow stack,
207+ * is disabled, new_ssp will remain 0, and fpu_clone() will know not to
208+ * update it.
209+ */
210+ new_ssp = shstk_alloc_thread_stack(p, clone_flags, args->stack_size);
211+ if (IS_ERR_VALUE(new_ssp))
212+ return PTR_ERR((void *)new_ssp);
213+
214+ fpu_clone(p, clone_flags, args->fn, new_ssp);
203215
204216 /* Kernel thread ? */
205217 if (unlikely(p->flags & PF_KTHREAD)) {
@@ -245,6 +257,13 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args) · 39 unmodified lines
245257 if (!ret && unlikely(test_tsk_thread_flag(current, TIF_IO_BITMAP)))
246258 io_bitmap_share(p);
247259
260+ /*
261+ * If copy_thread() if failing, don't leak the shadow stack possibly
262+ * allocated in shstk_alloc_thread_stack() above.
263+ */
264+ if (ret)
265+ shstk_free(p);
266+
248267 return ret;
249268 }
250269
arch/x86/kernel/process_64.c+8−0
@@ -515,6 +515,8 @@ start_thread_common(struct pt_regs *regs, unsigned long new_ip, · 514 unmodified lines
515515 load_gs_index(__USER_DS);
516516 }
517517
518+ reset_thread_features();
519+
518520 loadsegment(fs, 0);
519521 loadsegment(es, _ds);
520522 loadsegment(ds, _ds);
@@ -894,6 +896,12 @@ long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2) · 373 unmodified lines
894896 else
895897 return put_user(LAM_U57_BITS, (unsigned long __user *)arg2);
896898 #endif
899+ case ARCH_SHSTK_ENABLE:
900+ case ARCH_SHSTK_DISABLE:
901+ case ARCH_SHSTK_LOCK:
902+ case ARCH_SHSTK_UNLOCK:
903+ case ARCH_SHSTK_STATUS:
904+ return shstk_prctl(task, option, arg2);
897905 default:
898906 ret = -EINVAL;
899907 break;
arch/x86/kernel/ptrace.c+12−0
@@ -58,6 +58,7 @@ enum x86_regset_64 { · 57 unmodified lines
5858 REGSET64_FP,
5959 REGSET64_IOPERM,
6060 REGSET64_XSTATE,
61+ REGSET64_SSP,
6162 };
6263
6364 #define REGSET_GENERAL \
@@ -1267,6 +1268,17 @@ static struct user_regset x86_64_regsets[] __ro_after_init = { · 1203 unmodified lines
12671268 .active = ioperm_active,
12681269 .regset_get = ioperm_get
12691270 },
1271+#ifdef CONFIG_X86_USER_SHADOW_STACK
1272+ [REGSET64_SSP] = {
1273+ .core_note_type = NT_X86_SHSTK,
1274+ .n = 1,
1275+ .size = sizeof(u64),
1276+ .align = sizeof(u64),
1277+ .active = ssp_active,
1278+ .regset_get = ssp_get,
1279+ .set = ssp_set
1280+ },
1281+#endif
12701282 };
12711283
12721284 static const struct user_regset_view user_x86_64_view = {
arch/x86/kernel/shstk.c+550−0
@@ -0,0 +1,550 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * shstk.c - Intel shadow stack support
4+ *
5+ * Copyright (c) 2021, Intel Corporation.
6+ * Yu-cheng Yu <yu-cheng.yu@intel.com>
7+ */
8+
9+#include <linux/sched.h>
10+#include <linux/bitops.h>
11+#include <linux/types.h>
12+#include <linux/mm.h>
13+#include <linux/mman.h>
14+#include <linux/slab.h>
15+#include <linux/uaccess.h>
16+#include <linux/sched/signal.h>
17+#include <linux/compat.h>
18+#include <linux/sizes.h>
19+#include <linux/user.h>
20+#include <linux/syscalls.h>
21+#include <asm/msr.h>
22+#include <asm/fpu/xstate.h>
23+#include <asm/fpu/types.h>
24+#include <asm/shstk.h>
25+#include <asm/special_insns.h>
26+#include <asm/fpu/api.h>
27+#include <asm/prctl.h>
28+
29+#define SS_FRAME_SIZE 8
30+
31+static bool features_enabled(unsigned long features)
32+{
33+ return current->thread.features & features;
34+}
35+
36+static void features_set(unsigned long features)
37+{
38+ current->thread.features |= features;
39+}
40+
41+static void features_clr(unsigned long features)
42+{
43+ current->thread.features &= ~features;
44+}
45+
46+/*
47+ * Create a restore token on the shadow stack. A token is always 8-byte
48+ * and aligned to 8.
49+ */
50+static int create_rstor_token(unsigned long ssp, unsigned long *token_addr)
51+{
52+ unsigned long addr;
53+
54+ /* Token must be aligned */
55+ if (!IS_ALIGNED(ssp, 8))
56+ return -EINVAL;
57+
58+ addr = ssp - SS_FRAME_SIZE;
59+
60+ /*
61+ * SSP is aligned, so reserved bits and mode bit are a zero, just mark
62+ * the token 64-bit.
63+ */
64+ ssp |= BIT(0);
65+
66+ if (write_user_shstk_64((u64 __user *)addr, (u64)ssp))
67+ return -EFAULT;
68+
69+ if (token_addr)
70+ *token_addr = addr;
71+
72+ return 0;
73+}
74+
75+/*
76+ * VM_SHADOW_STACK will have a guard page. This helps userspace protect
77+ * itself from attacks. The reasoning is as follows:
78+ *
79+ * The shadow stack pointer(SSP) is moved by CALL, RET, and INCSSPQ. The
80+ * INCSSP instruction can increment the shadow stack pointer. It is the
81+ * shadow stack analog of an instruction like:
82+ *
83+ * addq $0x80, %rsp
84+ *
85+ * However, there is one important difference between an ADD on %rsp
86+ * and INCSSP. In addition to modifying SSP, INCSSP also reads from the
87+ * memory of the first and last elements that were "popped". It can be
88+ * thought of as acting like this:
89+ *
90+ * READ_ONCE(ssp); // read+discard top element on stack
91+ * ssp += nr_to_pop * 8; // move the shadow stack
92+ * READ_ONCE(ssp-8); // read+discard last popped stack element
93+ *
94+ * The maximum distance INCSSP can move the SSP is 2040 bytes, before
95+ * it would read the memory. Therefore a single page gap will be enough
96+ * to prevent any operation from shifting the SSP to an adjacent stack,
97+ * since it would have to land in the gap at least once, causing a
98+ * fault.
99+ */
100+static unsigned long alloc_shstk(unsigned long addr, unsigned long size,
101+ unsigned long token_offset, bool set_res_tok)
102+{
103+ int flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_ABOVE4G;
104+ struct mm_struct *mm = current->mm;
105+ unsigned long mapped_addr, unused;
106+
107+ if (addr)
108+ flags |= MAP_FIXED_NOREPLACE;
109+
110+ mmap_write_lock(mm);
111+ mapped_addr = do_mmap(NULL, addr, size, PROT_READ, flags,
112+ VM_SHADOW_STACK | VM_WRITE, 0, &unused, NULL);
113+ mmap_write_unlock(mm);
114+
115+ if (!set_res_tok || IS_ERR_VALUE(mapped_addr))
116+ goto out;
117+
118+ if (create_rstor_token(mapped_addr + token_offset, NULL)) {
119+ vm_munmap(mapped_addr, size);
120+ return -EINVAL;
121+ }
122+
123+out:
124+ return mapped_addr;
125+}
126+
127+static unsigned long adjust_shstk_size(unsigned long size)
128+{
129+ if (size)
130+ return PAGE_ALIGN(size);
131+
132+ return PAGE_ALIGN(min_t(unsigned long long, rlimit(RLIMIT_STACK), SZ_4G));
133+}
134+
135+static void unmap_shadow_stack(u64 base, u64 size)
136+{
137+ int r;
138+
139+ r = vm_munmap(base, size);
140+
141+ /*
142+ * mmap_write_lock_killable() failed with -EINTR. This means
143+ * the process is about to die and have it's MM cleaned up.
144+ * This task shouldn't ever make it back to userspace. In this
145+ * case it is ok to leak a shadow stack, so just exit out.
146+ */
147+ if (r == -EINTR)
148+ return;
149+
150+ /*
151+ * For all other types of vm_munmap() failure, either the
152+ * system is out of memory or there is bug.
153+ */
154+ WARN_ON_ONCE(r);
155+}
156+
157+static int shstk_setup(void)
158+{
159+ struct thread_shstk *shstk = &current->thread.shstk;
160+ unsigned long addr, size;
161+
162+ /* Already enabled */
163+ if (features_enabled(ARCH_SHSTK_SHSTK))
164+ return 0;
165+
166+ /* Also not supported for 32 bit and x32 */
167+ if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) || in_32bit_syscall())
168+ return -EOPNOTSUPP;
169+
170+ size = adjust_shstk_size(0);
171+ addr = alloc_shstk(0, size, 0, false);
172+ if (IS_ERR_VALUE(addr))
173+ return PTR_ERR((void *)addr);
174+
175+ fpregs_lock_and_load();
176+ wrmsrl(MSR_IA32_PL3_SSP, addr + size);
177+ wrmsrl(MSR_IA32_U_CET, CET_SHSTK_EN);
178+ fpregs_unlock();
179+
180+ shstk->base = addr;
181+ shstk->size = size;
182+ features_set(ARCH_SHSTK_SHSTK);
183+
184+ return 0;
185+}
186+
187+void reset_thread_features(void)
188+{
189+ memset(&current->thread.shstk, 0, sizeof(struct thread_shstk));
190+ current->thread.features = 0;
191+ current->thread.features_locked = 0;
192+}
193+
194+unsigned long shstk_alloc_thread_stack(struct task_struct *tsk, unsigned long clone_flags,
195+ unsigned long stack_size)
196+{
197+ struct thread_shstk *shstk = &tsk->thread.shstk;
198+ unsigned long addr, size;
199+
200+ /*
201+ * If shadow stack is not enabled on the new thread, skip any
202+ * switch to a new shadow stack.
203+ */
204+ if (!features_enabled(ARCH_SHSTK_SHSTK))
205+ return 0;
206+
207+ /*
208+ * For CLONE_VM, except vfork, the child needs a separate shadow
209+ * stack.
210+ */
211+ if ((clone_flags & (CLONE_VFORK | CLONE_VM)) != CLONE_VM)
212+ return 0;
213+
214+ size = adjust_shstk_size(stack_size);
215+ addr = alloc_shstk(0, size, 0, false);
216+ if (IS_ERR_VALUE(addr))
217+ return addr;
218+
219+ shstk->base = addr;
220+ shstk->size = size;
221+
222+ return addr + size;
223+}
224+
225+static unsigned long get_user_shstk_addr(void)
226+{
227+ unsigned long long ssp;
228+
229+ fpregs_lock_and_load();
230+
231+ rdmsrl(MSR_IA32_PL3_SSP, ssp);
232+
233+ fpregs_unlock();
234+
235+ return ssp;
236+}
237+
238+#define SHSTK_DATA_BIT BIT(63)
239+
240+static int put_shstk_data(u64 __user *addr, u64 data)
241+{
242+ if (WARN_ON_ONCE(data & SHSTK_DATA_BIT))
243+ return -EINVAL;
244+
245+ /*
246+ * Mark the high bit so that the sigframe can't be processed as a
247+ * return address.
248+ */
249+ if (write_user_shstk_64(addr, data | SHSTK_DATA_BIT))
250+ return -EFAULT;
251+ return 0;
252+}
253+
254+static int get_shstk_data(unsigned long *data, unsigned long __user *addr)
255+{
256+ unsigned long ldata;
257+
258+ if (unlikely(get_user(ldata, addr)))
259+ return -EFAULT;
260+
261+ if (!(ldata & SHSTK_DATA_BIT))
262+ return -EINVAL;
263+
264+ *data = ldata & ~SHSTK_DATA_BIT;
265+
266+ return 0;
267+}
268+
269+static int shstk_push_sigframe(unsigned long *ssp)
270+{
271+ unsigned long target_ssp = *ssp;
272+
273+ /* Token must be aligned */
274+ if (!IS_ALIGNED(target_ssp, 8))
275+ return -EINVAL;
276+
277+ *ssp -= SS_FRAME_SIZE;
278+ if (put_shstk_data((void __user *)*ssp, target_ssp))
279+ return -EFAULT;
280+
281+ return 0;
282+}
283+
284+static int shstk_pop_sigframe(unsigned long *ssp)
285+{
286+ struct vm_area_struct *vma;
287+ unsigned long token_addr;
288+ bool need_to_check_vma;
289+ int err = 1;
290+
291+ /*
292+ * It is possible for the SSP to be off the end of a shadow stack by 4
293+ * or 8 bytes. If the shadow stack is at the start of a page or 4 bytes
294+ * before it, it might be this case, so check that the address being
295+ * read is actually shadow stack.
296+ */
297+ if (!IS_ALIGNED(*ssp, 8))
298+ return -EINVAL;
299+
300+ need_to_check_vma = PAGE_ALIGN(*ssp) == *ssp;
301+
302+ if (need_to_check_vma)
303+ mmap_read_lock_killable(current->mm);
304+
305+ err = get_shstk_data(&token_addr, (unsigned long __user *)*ssp);
306+ if (unlikely(err))
307+ goto out_err;
308+
309+ if (need_to_check_vma) {
310+ vma = find_vma(current->mm, *ssp);
311+ if (!vma || !(vma->vm_flags & VM_SHADOW_STACK)) {
312+ err = -EFAULT;
313+ goto out_err;
314+ }
315+
316+ mmap_read_unlock(current->mm);
317+ }
318+
319+ /* Restore SSP aligned? */
320+ if (unlikely(!IS_ALIGNED(token_addr, 8)))
321+ return -EINVAL;
322+
323+ /* SSP in userspace? */
324+ if (unlikely(token_addr >= TASK_SIZE_MAX))
325+ return -EINVAL;
326+
327+ *ssp = token_addr;
328+
329+ return 0;
330+out_err:
331+ if (need_to_check_vma)
332+ mmap_read_unlock(current->mm);
333+ return err;
334+}
335+
336+int setup_signal_shadow_stack(struct ksignal *ksig)
337+{
338+ void __user *restorer = ksig->ka.sa.sa_restorer;
339+ unsigned long ssp;
340+ int err;
341+
342+ if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) ||
343+ !features_enabled(ARCH_SHSTK_SHSTK))
344+ return 0;
345+
346+ if (!restorer)
347+ return -EINVAL;
348+
349+ ssp = get_user_shstk_addr();
350+ if (unlikely(!ssp))
351+ return -EINVAL;
352+
353+ err = shstk_push_sigframe(&ssp);
354+ if (unlikely(err))
355+ return err;
356+
357+ /* Push restorer address */
358+ ssp -= SS_FRAME_SIZE;
359+ err = write_user_shstk_64((u64 __user *)ssp, (u64)restorer);
360+ if (unlikely(err))
361+ return -EFAULT;
362+
363+ fpregs_lock_and_load();
364+ wrmsrl(MSR_IA32_PL3_SSP, ssp);
365+ fpregs_unlock();
366+
367+ return 0;
368+}
369+
370+int restore_signal_shadow_stack(void)
371+{
372+ unsigned long ssp;
373+ int err;
374+
375+ if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) ||
376+ !features_enabled(ARCH_SHSTK_SHSTK))
377+ return 0;
378+
379+ ssp = get_user_shstk_addr();
380+ if (unlikely(!ssp))
381+ return -EINVAL;
382+
383+ err = shstk_pop_sigframe(&ssp);
384+ if (unlikely(err))
385+ return err;
386+
387+ fpregs_lock_and_load();
388+ wrmsrl(MSR_IA32_PL3_SSP, ssp);
389+ fpregs_unlock();
390+
391+ return 0;
392+}
393+
394+void shstk_free(struct task_struct *tsk)
395+{
396+ struct thread_shstk *shstk = &tsk->thread.shstk;
397+
398+ if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK) ||
399+ !features_enabled(ARCH_SHSTK_SHSTK))
400+ return;
401+
402+ /*
403+ * When fork() with CLONE_VM fails, the child (tsk) already has a
404+ * shadow stack allocated, and exit_thread() calls this function to
405+ * free it. In this case the parent (current) and the child share
406+ * the same mm struct.
407+ */
408+ if (!tsk->mm || tsk->mm != current->mm)
409+ return;
410+
411+ unmap_shadow_stack(shstk->base, shstk->size);
412+}
413+
414+static int wrss_control(bool enable)
415+{
416+ u64 msrval;
417+
418+ if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
419+ return -EOPNOTSUPP;
420+
421+ /*
422+ * Only enable WRSS if shadow stack is enabled. If shadow stack is not
423+ * enabled, WRSS will already be disabled, so don't bother clearing it
424+ * when disabling.
425+ */
426+ if (!features_enabled(ARCH_SHSTK_SHSTK))
427+ return -EPERM;
428+
429+ /* Already enabled/disabled? */
430+ if (features_enabled(ARCH_SHSTK_WRSS) == enable)
431+ return 0;
432+
433+ fpregs_lock_and_load();
434+ rdmsrl(MSR_IA32_U_CET, msrval);
435+
436+ if (enable) {
437+ features_set(ARCH_SHSTK_WRSS);
438+ msrval |= CET_WRSS_EN;
439+ } else {
440+ features_clr(ARCH_SHSTK_WRSS);
441+ if (!(msrval & CET_WRSS_EN))
442+ goto unlock;
443+
444+ msrval &= ~CET_WRSS_EN;
445+ }
446+
447+ wrmsrl(MSR_IA32_U_CET, msrval);
448+
449+unlock:
450+ fpregs_unlock();
451+
452+ return 0;
453+}
454+
455+static int shstk_disable(void)
456+{
457+ if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
458+ return -EOPNOTSUPP;
459+
460+ /* Already disabled? */
461+ if (!features_enabled(ARCH_SHSTK_SHSTK))
462+ return 0;
463+
464+ fpregs_lock_and_load();
465+ /* Disable WRSS too when disabling shadow stack */
466+ wrmsrl(MSR_IA32_U_CET, 0);
467+ wrmsrl(MSR_IA32_PL3_SSP, 0);
468+ fpregs_unlock();
469+
470+ shstk_free(current);
471+ features_clr(ARCH_SHSTK_SHSTK | ARCH_SHSTK_WRSS);
472+
473+ return 0;
474+}
475+
476+SYSCALL_DEFINE3(map_shadow_stack, unsigned long, addr, unsigned long, size, unsigned int, flags)
477+{
478+ bool set_tok = flags & SHADOW_STACK_SET_TOKEN;
479+ unsigned long aligned_size;
480+
481+ if (!cpu_feature_enabled(X86_FEATURE_USER_SHSTK))
482+ return -EOPNOTSUPP;
483+
484+ if (flags & ~SHADOW_STACK_SET_TOKEN)
485+ return -EINVAL;
486+
487+ /* If there isn't space for a token */
488+ if (set_tok && size < 8)
489+ return -ENOSPC;
490+
491+ if (addr && addr < SZ_4G)
492+ return -ERANGE;
493+
494+ /*
495+ * An overflow would result in attempting to write the restore token
496+ * to the wrong location. Not catastrophic, but just return the right
497+ * error code and block it.
498+ */
499+ aligned_size = PAGE_ALIGN(size);
500+ if (aligned_size < size)
501+ return -EOVERFLOW;
502+
503+ return alloc_shstk(addr, aligned_size, size, set_tok);
504+}
505+
506+long shstk_prctl(struct task_struct *task, int option, unsigned long arg2)
507+{
508+ unsigned long features = arg2;
509+
510+ if (option == ARCH_SHSTK_STATUS) {
511+ return put_user(task->thread.features, (unsigned long __user *)arg2);
512+ }
513+
514+ if (option == ARCH_SHSTK_LOCK) {
515+ task->thread.features_locked |= features;
516+ return 0;
517+ }
518+
519+ /* Only allow via ptrace */
520+ if (task != current) {
521+ if (option == ARCH_SHSTK_UNLOCK && IS_ENABLED(CONFIG_CHECKPOINT_RESTORE)) {
522+ task->thread.features_locked &= ~features;
523+ return 0;
524+ }
525+ return -EINVAL;
526+ }
527+
528+ /* Do not allow to change locked features */
529+ if (features & task->thread.features_locked)
530+ return -EPERM;
531+
532+ /* Only support enabling/disabling one feature at a time. */
533+ if (hweight_long(features) > 1)
534+ return -EINVAL;
535+
536+ if (option == ARCH_SHSTK_DISABLE) {
537+ if (features & ARCH_SHSTK_WRSS)
538+ return wrss_control(false);
539+ if (features & ARCH_SHSTK_SHSTK)
540+ return shstk_disable();
541+ return -EINVAL;
542+ }
543+
544+ /* Handle ARCH_SHSTK_ENABLE */
545+ if (features & ARCH_SHSTK_SHSTK)
546+ return shstk_setup();
547+ if (features & ARCH_SHSTK_WRSS)
548+ return wrss_control(true);
549+ return -EINVAL;
550+}
arch/x86/kernel/signal.c+1−0
@@ -40,6 +40,7 @@ · 39 unmodified lines
4040 #include <asm/syscall.h>
4141 #include <asm/sigframe.h>
4242 #include <asm/signal.h>
43+#include <asm/shstk.h>
4344
4445 static inline int is_ia32_compat_frame(struct ksignal *ksig)
4546 {
arch/x86/kernel/signal_32.c+1−1
@@ -402,7 +402,7 @@ Efault: · 401 unmodified lines
402402 */
403403 static_assert(NSIGILL == 11);
404404 static_assert(NSIGFPE == 15);
405−static_assert(NSIGSEGV == 9);
405+static_assert(NSIGSEGV == 10);
406406 static_assert(NSIGBUS == 5);
407407 static_assert(NSIGTRAP == 6);
408408 static_assert(NSIGCHLD == 6);
arch/x86/kernel/signal_64.c+7−1
@@ -175,6 +175,9 @@ int x64_setup_rt_frame(struct ksignal *ksig, struct pt_regs *regs) · 174 unmodified lines
175175 frame = get_sigframe(ksig, regs, sizeof(struct rt_sigframe), &fp);
176176 uc_flags = frame_uc_flags(regs);
177177
178+ if (setup_signal_shadow_stack(ksig))
179+ return -EFAULT;
180+
178181 if (!user_access_begin(frame, sizeof(*frame)))
179182 return -EFAULT;
180183
@@ -260,6 +263,9 @@ SYSCALL_DEFINE0(rt_sigreturn) · 79 unmodified lines
260263 if (!restore_sigcontext(regs, &frame->uc.uc_mcontext, uc_flags))
261264 goto badframe;
262265
266+ if (restore_signal_shadow_stack())
267+ goto badframe;
268+
263269 if (restore_altstack(&frame->uc.uc_stack))
264270 goto badframe;
265271
@@ -403,7 +409,7 @@ void sigaction_compat_abi(struct k_sigaction *act, struct k_sigaction *oact) · 137 unmodified lines
403409 */
404410 static_assert(NSIGILL == 11);
405411 static_assert(NSIGFPE == 15);
406−static_assert(NSIGSEGV == 9);
412+static_assert(NSIGSEGV == 10);
407413 static_assert(NSIGBUS == 5);
408414 static_assert(NSIGTRAP == 6);
409415 static_assert(NSIGCHLD == 6);
arch/x86/kernel/sys_x86_64.c+5−1
@@ -193,7 +193,11 @@ get_unmapped_area: · 192 unmodified lines
193193
194194 info.flags = VM_UNMAPPED_AREA_TOPDOWN;
195195 info.length = len;
196− info.low_limit = PAGE_SIZE;
196+ if (!in_32bit_syscall() && (flags & MAP_ABOVE4G))
197+ info.low_limit = SZ_4G;
198+ else
199+ info.low_limit = PAGE_SIZE;
200+
197201 info.high_limit = get_mmap_base(0);
198202
199203 /*
arch/x86/kernel/traps.c+0−87
@@ -77,18 +77,6 @@ · 76 unmodified lines
7777
7878 DECLARE_BITMAP(system_vectors, NR_VECTORS);
7979
80−static inline void cond_local_irq_enable(struct pt_regs *regs)
81−{
82− if (regs->flags & X86_EFLAGS_IF)
83− local_irq_enable();
84−}
85−
86−static inline void cond_local_irq_disable(struct pt_regs *regs)
87−{
88− if (regs->flags & X86_EFLAGS_IF)
89− local_irq_disable();
90−}
91−
9280 __always_inline int is_valid_bugaddr(unsigned long addr)
9381 {
9482 if (addr < TASK_SIZE_MAX)
@@ -213,81 +201,6 @@ DEFINE_IDTENTRY(exc_overflow) · 118 unmodified lines
213201 do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL);
214202 }
215203
216−#ifdef CONFIG_X86_KERNEL_IBT
217−
218−static __ro_after_init bool ibt_fatal = true;
219−
220−extern void ibt_selftest_ip(void); /* code label defined in asm below */
221−
222−enum cp_error_code {
223− CP_EC = (1 << 15) - 1,
224−
225− CP_RET = 1,
226− CP_IRET = 2,
227− CP_ENDBR = 3,
228− CP_RSTRORSSP = 4,
229− CP_SETSSBSY = 5,
230−
231− CP_ENCL = 1 << 15,
232−};
233−
234−DEFINE_IDTENTRY_ERRORCODE(exc_control_protection)
235−{
236− if (!cpu_feature_enabled(X86_FEATURE_IBT)) {
237− pr_err("Unexpected #CP\n");
238− BUG();
239− }
240−
241− if (WARN_ON_ONCE(user_mode(regs) || (error_code & CP_EC) != CP_ENDBR))
242− return;
243−
244− if (unlikely(regs->ip == (unsigned long)&ibt_selftest_ip)) {
245− regs->ax = 0;
246− return;
247− }
248−
249− pr_err("Missing ENDBR: %pS\n", (void *)instruction_pointer(regs));
250− if (!ibt_fatal) {
251− printk(KERN_DEFAULT CUT_HERE);
252− __warn(__FILE__, __LINE__, (void *)regs->ip, TAINT_WARN, regs, NULL);
253− return;
254− }
255− BUG();
256−}
257−
258−/* Must be noinline to ensure uniqueness of ibt_selftest_ip. */
259−noinline bool ibt_selftest(void)
260−{
261− unsigned long ret;
262−
263− asm (" lea ibt_selftest_ip(%%rip), %%rax\n\t"
264− ANNOTATE_RETPOLINE_SAFE
265− " jmp *%%rax\n\t"
266− "ibt_selftest_ip:\n\t"
267− UNWIND_HINT_FUNC
268− ANNOTATE_NOENDBR
269− " nop\n\t"
270−
271− : "=a" (ret) : : "memory");
272−
273− return !ret;
274−}
275−
276−static int __init ibt_setup(char *str)
277−{
278− if (!strcmp(str, "off"))
279− setup_clear_cpu_cap(X86_FEATURE_IBT);
280−
281− if (!strcmp(str, "warn"))
282− ibt_fatal = false;
283−
284− return 1;
285−}
286−
287−__setup("ibt=", ibt_setup);
288−
289−#endif /* CONFIG_X86_KERNEL_IBT */
290−
291204 #ifdef CONFIG_X86_F00F_BUG
292205 void handle_invalid_op(struct pt_regs *regs)
293206 #else
arch/x86/mm/fault.c+22−0
@@ -1112,8 +1112,22 @@ access_error(unsigned long error_code, struct vm_area_struct *vma) · 1111 unmodified lines
11121112 (error_code & X86_PF_INSTR), foreign))
11131113 return 1;
11141114
1115+ /*
1116+ * Shadow stack accesses (PF_SHSTK=1) are only permitted to
1117+ * shadow stack VMAs. All other accesses result in an error.
1118+ */
1119+ if (error_code & X86_PF_SHSTK) {
1120+ if (unlikely(!(vma->vm_flags & VM_SHADOW_STACK)))
1121+ return 1;
1122+ if (unlikely(!(vma->vm_flags & VM_WRITE)))
1123+ return 1;
1124+ return 0;
1125+ }
1126+
11151127 if (error_code & X86_PF_WRITE) {
11161128 /* write, present and write, not present: */
1129+ if (unlikely(vma->vm_flags & VM_SHADOW_STACK))
1130+ return 1;
11171131 if (unlikely(!(vma->vm_flags & VM_WRITE)))
11181132 return 1;
11191133 return 0;
@@ -1305,6 +1319,14 @@ void do_user_addr_fault(struct pt_regs *regs, · 185 unmodified lines
13051319
13061320 perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
13071321
1322+ /*
1323+ * Read-only permissions can not be expressed in shadow stack PTEs.
1324+ * Treat all shadow stack accesses as WRITE faults. This ensures
1325+ * that the MM will prepare everything (e.g., break COW) such that
1326+ * maybe_mkwrite() can create a proper shadow stack PTE.
1327+ */
1328+ if (error_code & X86_PF_SHSTK)
1329+ flags |= FAULT_FLAG_WRITE;
13081330 if (error_code & X86_PF_WRITE)
13091331 flags |= FAULT_FLAG_WRITE;
13101332 if (error_code & X86_PF_INSTR)
arch/x86/mm/pat/set_memory.c+2−2
@@ -2074,12 +2074,12 @@ int set_memory_nx(unsigned long addr, int numpages) · 2073 unmodified lines
20742074
20752075 int set_memory_ro(unsigned long addr, int numpages)
20762076 {
2077− return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW), 0);
2077+ return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW | _PAGE_DIRTY), 0);
20782078 }
20792079
20802080 int set_memory_rox(unsigned long addr, int numpages)
20812081 {
2082− pgprot_t clr = __pgprot(_PAGE_RW);
2082+ pgprot_t clr = __pgprot(_PAGE_RW | _PAGE_DIRTY);
20832083
20842084 if (__supported_pte_mask & _PAGE_NX)
20852085 clr.pgprot |= _PAGE_NX;
arch/x86/mm/pgtable.c+40−0
@@ -881,3 +881,43 @@ int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) · 880 unmodified lines
881881
882882 #endif /* CONFIG_X86_64 */
883883 #endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */
884+
885+pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma)
886+{
887+ if (vma->vm_flags & VM_SHADOW_STACK)
888+ return pte_mkwrite_shstk(pte);
889+
890+ pte = pte_mkwrite_novma(pte);
891+
892+ return pte_clear_saveddirty(pte);
893+}
894+
895+pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
896+{
897+ if (vma->vm_flags & VM_SHADOW_STACK)
898+ return pmd_mkwrite_shstk(pmd);
899+
900+ pmd = pmd_mkwrite_novma(pmd);
901+
902+ return pmd_clear_saveddirty(pmd);
903+}
904+
905+void arch_check_zapped_pte(struct vm_area_struct *vma, pte_t pte)
906+{
907+ /*
908+ * Hardware before shadow stack can (rarely) set Dirty=1
909+ * on a Write=0 PTE. So the below condition
910+ * only indicates a software bug when shadow stack is
911+ * supported by the HW. This checking is covered in
912+ * pte_shstk().
913+ */
914+ VM_WARN_ON_ONCE(!(vma->vm_flags & VM_SHADOW_STACK) &&
915+ pte_shstk(pte));
916+}
917+
918+void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd)
919+{
920+ /* See note in arch_check_zapped_pte() */
921+ VM_WARN_ON_ONCE(!(vma->vm_flags & VM_SHADOW_STACK) &&
922+ pmd_shstk(pmd));
923+}
arch/x86/xen/enlighten_pv.c+1−1
@@ -694,7 +694,7 @@ static struct trap_array_entry trap_array[] = { · 693 unmodified lines
694694 TRAP_ENTRY(exc_coprocessor_error, false ),
695695 TRAP_ENTRY(exc_alignment_check, false ),
696696 TRAP_ENTRY(exc_simd_coprocessor_error, false ),
697−#ifdef CONFIG_X86_KERNEL_IBT
697+#ifdef CONFIG_X86_CET
698698 TRAP_ENTRY(exc_control_protection, false ),
699699 #endif
700700 };
arch/x86/xen/mmu_pv.c+1−1
@@ -166,7 +166,7 @@ void make_lowmem_page_readwrite(void *vaddr) · 165 unmodified lines
166166 if (pte == NULL)
167167 return; /* vaddr missing */
168168
169− ptev = pte_mkwrite(*pte);
169+ ptev = pte_mkwrite_novma(*pte);
170170
171171 if (HYPERVISOR_update_va_mapping(address, ptev, 0))
172172 BUG();
arch/x86/xen/xen-asm.S+1−1
@@ -148,7 +148,7 @@ xen_pv_trap asm_exc_page_fault · 147 unmodified lines
148148 xen_pv_trap asm_exc_spurious_interrupt_bug
149149 xen_pv_trap asm_exc_coprocessor_error
150150 xen_pv_trap asm_exc_alignment_check
151−#ifdef CONFIG_X86_KERNEL_IBT
151+#ifdef CONFIG_X86_CET
152152 xen_pv_trap asm_exc_control_protection
153153 #endif
154154 #ifdef CONFIG_X86_MCE
arch/xtensa/include/asm/pgtable.h+1−1
@@ -262,7 +262,7 @@ static inline pte_t pte_mkdirty(pte_t pte) · 261 unmodified lines
262262 { pte_val(pte) |= _PAGE_DIRTY; return pte; }
263263 static inline pte_t pte_mkyoung(pte_t pte)
264264 { pte_val(pte) |= _PAGE_ACCESSED; return pte; }
265−static inline pte_t pte_mkwrite(pte_t pte)
265+static inline pte_t pte_mkwrite_novma(pte_t pte)
266266 { pte_val(pte) |= _PAGE_WRITABLE; return pte; }
267267
268268 #define pgprot_noncached(prot) \
fs/aio.c+1−1
@@ -558,7 +558,7 @@ static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) · 557 unmodified lines
558558
559559 ctx->mmap_base = do_mmap(ctx->aio_ring_file, 0, ctx->mmap_size,
560560 PROT_READ | PROT_WRITE,
561− MAP_SHARED, 0, &unused, NULL);
561+ MAP_SHARED, 0, 0, &unused, NULL);
562562 mmap_write_unlock(mm);
563563 if (IS_ERR((void *)ctx->mmap_base)) {
564564 ctx->mmap_size = 0;
fs/proc/array.c+6−0
@@ -431,6 +431,11 @@ static inline void task_untag_mask(struct seq_file *m, struct mm_struct *mm) · 430 unmodified lines
431431 seq_printf(m, "untag_mask:\t%#lx\n", mm_untag_mask(mm));
432432 }
433433
434+__weak void arch_proc_pid_thread_features(struct seq_file *m,
435+ struct task_struct *task)
436+{
437+}
438+
434439 int proc_pid_status(struct seq_file *m, struct pid_namespace *ns,
435440 struct pid *pid, struct task_struct *task)
436441 {
@@ -455,6 +460,7 @@ int proc_pid_status(struct seq_file *m, struct pid_namespace *ns, · 18 unmodified lines
455460 task_cpus_allowed(m, task);
456461 cpuset_task_status_allowed(m, task);
457462 task_context_switch_counts(m, task);
463+ arch_proc_pid_thread_features(m, task);
458464 return 0;
459465 }
460466
fs/proc/task_mmu.c+3−0
@@ -692,6 +692,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma) · 691 unmodified lines
692692 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
693693 [ilog2(VM_UFFD_MINOR)] = "ui",
694694 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
695+#ifdef CONFIG_X86_USER_SHADOW_STACK
696+ [ilog2(VM_SHADOW_STACK)] = "ss",
697+#endif
695698 };
696699 size_t i;
697700
include/asm-generic/hugetlb.h+1−1
@@ -22,7 +22,7 @@ static inline unsigned long huge_pte_dirty(pte_t pte) · 21 unmodified lines
2222
2323 static inline pte_t huge_pte_mkwrite(pte_t pte)
2424 {
25− return pte_mkwrite(pte);
25+ return pte_mkwrite_novma(pte);
2626 }
2727
2828 #ifndef __HAVE_ARCH_HUGE_PTE_WRPROTECT
include/linux/mm.h+39−8
@@ -319,11 +319,13 @@ extern unsigned int kobjsize(const void *objp); · 318 unmodified lines
319319 #define VM_HIGH_ARCH_BIT_2 34 /* bit only usable on 64-bit architectures */
320320 #define VM_HIGH_ARCH_BIT_3 35 /* bit only usable on 64-bit architectures */
321321 #define VM_HIGH_ARCH_BIT_4 36 /* bit only usable on 64-bit architectures */
322+#define VM_HIGH_ARCH_BIT_5 37 /* bit only usable on 64-bit architectures */
322323 #define VM_HIGH_ARCH_0 BIT(VM_HIGH_ARCH_BIT_0)
323324 #define VM_HIGH_ARCH_1 BIT(VM_HIGH_ARCH_BIT_1)
324325 #define VM_HIGH_ARCH_2 BIT(VM_HIGH_ARCH_BIT_2)
325326 #define VM_HIGH_ARCH_3 BIT(VM_HIGH_ARCH_BIT_3)
326327 #define VM_HIGH_ARCH_4 BIT(VM_HIGH_ARCH_BIT_4)
328+#define VM_HIGH_ARCH_5 BIT(VM_HIGH_ARCH_BIT_5)
327329 #endif /* CONFIG_ARCH_USES_HIGH_VMA_FLAGS */
328330
329331 #ifdef CONFIG_ARCH_HAS_PKEYS
@@ -339,6 +341,21 @@ extern unsigned int kobjsize(const void *objp); · 9 unmodified lines
339341 #endif
340342 #endif /* CONFIG_ARCH_HAS_PKEYS */
341343
344+#ifdef CONFIG_X86_USER_SHADOW_STACK
345+/*
346+ * VM_SHADOW_STACK should not be set with VM_SHARED because of lack of
347+ * support core mm.
348+ *
349+ * These VMAs will get a single end guard page. This helps userspace protect
350+ * itself from attacks. A single page is enough for current shadow stack archs
351+ * (x86). See the comments near alloc_shstk() in arch/x86/kernel/shstk.c
352+ * for more details on the guard size.
353+ */
354+# define VM_SHADOW_STACK VM_HIGH_ARCH_5
355+#else
356+# define VM_SHADOW_STACK VM_NONE
357+#endif
358+
342359 #if defined(CONFIG_X86)
343360 # define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
344361 #elif defined(CONFIG_PPC)
@@ -370,7 +387,7 @@ extern unsigned int kobjsize(const void *objp); · 25 unmodified lines
370387 #endif
371388
372389 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR
373−# define VM_UFFD_MINOR_BIT 37
390+# define VM_UFFD_MINOR_BIT 38
374391 # define VM_UFFD_MINOR BIT(VM_UFFD_MINOR_BIT) /* UFFD minor faults */
375392 #else /* !CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */
376393 # define VM_UFFD_MINOR VM_NONE
@@ -397,6 +414,8 @@ extern unsigned int kobjsize(const void *objp); · 20 unmodified lines
397414 #define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
398415 #endif
399416
417+#define VM_STARTGAP_FLAGS (VM_GROWSDOWN | VM_SHADOW_STACK)
418+
400419 #ifdef CONFIG_STACK_GROWSUP
401420 #define VM_STACK VM_GROWSUP
402421 #define VM_STACK_EARLY VM_GROWSDOWN
@@ -1309,7 +1328,7 @@ static inline unsigned long thp_size(struct page *page) · 906 unmodified lines
13091328 static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
13101329 {
13111330 if (likely(vma->vm_flags & VM_WRITE))
1312− pte = pte_mkwrite(pte);
1331+ pte = pte_mkwrite(pte, vma);
13131332 return pte;
13141333 }
13151334
@@ -3265,7 +3284,8 @@ extern unsigned long mmap_region(struct file *file, unsigned long addr, · 1949 unmodified lines
32653284 struct list_head *uf);
32663285 extern unsigned long do_mmap(struct file *file, unsigned long addr,
32673286 unsigned long len, unsigned long prot, unsigned long flags,
3268− unsigned long pgoff, unsigned long *populate, struct list_head *uf);
3287+ vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate,
3288+ struct list_head *uf);
32693289 extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm,
32703290 unsigned long start, size_t len, struct list_head *uf,
32713291 bool unlock);
@@ -3353,15 +3373,26 @@ struct vm_area_struct *vma_lookup(struct mm_struct *mm, unsigned long addr) · 81 unmodified lines
33533373 return mtree_load(&mm->mm_mt, addr);
33543374 }
33553375
3376+static inline unsigned long stack_guard_start_gap(struct vm_area_struct *vma)
3377+{
3378+ if (vma->vm_flags & VM_GROWSDOWN)
3379+ return stack_guard_gap;
3380+
3381+ /* See reasoning around the VM_SHADOW_STACK definition */
3382+ if (vma->vm_flags & VM_SHADOW_STACK)
3383+ return PAGE_SIZE;
3384+
3385+ return 0;
3386+}
3387+
33563388 static inline unsigned long vm_start_gap(struct vm_area_struct *vma)
33573389 {
3390+ unsigned long gap = stack_guard_start_gap(vma);
33583391 unsigned long vm_start = vma->vm_start;
33593392
3360− if (vma->vm_flags & VM_GROWSDOWN) {
3361− vm_start -= stack_guard_gap;
3362− if (vm_start > vma->vm_start)
3363− vm_start = 0;
3364− }
3393+ vm_start -= gap;
3394+ if (vm_start > vma->vm_start)
3395+ vm_start = 0;
33653396 return vm_start;
33663397 }
33673398
include/linux/mman.h+4−0
@@ -15,6 +15,9 @@ · 14 unmodified lines
1515 #ifndef MAP_32BIT
1616 #define MAP_32BIT 0
1717 #endif
18+#ifndef MAP_ABOVE4G
19+#define MAP_ABOVE4G 0
20+#endif
1821 #ifndef MAP_HUGE_2MB
1922 #define MAP_HUGE_2MB 0
2023 #endif
@@ -50,6 +53,7 @@ · 29 unmodified lines
5053 | MAP_STACK \
5154 | MAP_HUGETLB \
5255 | MAP_32BIT \
56+ | MAP_ABOVE4G \
5357 | MAP_HUGE_2MB \
5458 | MAP_HUGE_1GB)
5559
include/linux/pgtable.h+28−0
@@ -371,6 +371,20 @@ static inline bool arch_has_hw_pte_young(void) · 370 unmodified lines
371371 }
372372 #endif
373373
374+#ifndef arch_check_zapped_pte
375+static inline void arch_check_zapped_pte(struct vm_area_struct *vma,
376+ pte_t pte)
377+{
378+}
379+#endif
380+
381+#ifndef arch_check_zapped_pmd
382+static inline void arch_check_zapped_pmd(struct vm_area_struct *vma,
383+ pmd_t pmd)
384+{
385+}
386+#endif
387+
374388 #ifndef __HAVE_ARCH_PTEP_GET_AND_CLEAR
375389 static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
376390 unsigned long address,
@@ -577,6 +591,20 @@ extern pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, · 200 unmodified lines
577591 pud_t *pudp);
578592 #endif
579593
594+#ifndef pte_mkwrite
595+static inline pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma)
596+{
597+ return pte_mkwrite_novma(pte);
598+}
599+#endif
600+
601+#if defined(CONFIG_ARCH_WANT_PMD_MKWRITE) && !defined(pmd_mkwrite)
602+static inline pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
603+{
604+ return pmd_mkwrite_novma(pmd);
605+}
606+#endif
607+
580608 #ifndef __HAVE_ARCH_PTEP_SET_WRPROTECT
581609 struct mm_struct;
582610 static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long address, pte_t *ptep)
include/linux/proc_fs.h+1−0
@@ -159,6 +159,7 @@ int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns, · 158 unmodified lines
159159 #endif /* CONFIG_PROC_PID_ARCH_STATUS */
160160
161161 void arch_report_meminfo(struct seq_file *m);
162+void arch_proc_pid_thread_features(struct seq_file *m, struct task_struct *task);
162163
163164 #else /* CONFIG_PROC_FS */
164165
include/linux/syscalls.h+1−0
@@ -939,6 +939,7 @@ asmlinkage long sys_set_mempolicy_home_node(unsigned long start, unsigned long l · 938 unmodified lines
939939 asmlinkage long sys_cachestat(unsigned int fd,
940940 struct cachestat_range __user *cstat_range,
941941 struct cachestat __user *cstat, unsigned int flags);
942+asmlinkage long sys_map_shadow_stack(unsigned long addr, unsigned long size, unsigned int flags);
942943
943944 /*
944945 * Architecture-specific system calls
include/uapi/asm-generic/siginfo.h+2−1
@@ -242,7 +242,8 @@ typedef struct siginfo { · 241 unmodified lines
242242 #define SEGV_ADIPERR 7 /* Precise MCD exception */
243243 #define SEGV_MTEAERR 8 /* Asynchronous ARM MTE error */
244244 #define SEGV_MTESERR 9 /* Synchronous ARM MTE exception */
245−#define NSIGSEGV 9
245+#define SEGV_CPERR 10 /* Control protection fault */
246+#define NSIGSEGV 10
246247
247248 /*
248249 * SIGBUS si_codes
include/uapi/linux/elf.h+2−0
@@ -409,6 +409,8 @@ typedef struct elf64_shdr { · 408 unmodified lines
409409 #define NT_386_TLS 0x200 /* i386 TLS slots (struct user_desc) */
410410 #define NT_386_IOPERM 0x201 /* x86 io permission bitmap (1=deny) */
411411 #define NT_X86_XSTATE 0x202 /* x86 extended state using xsave */
412+/* Old binutils treats 0x203 as a CET state */
413+#define NT_X86_SHSTK 0x204 /* x86 SHSTK state */
412414 #define NT_S390_HIGH_GPRS 0x300 /* s390 upper register halves */
413415 #define NT_S390_TIMER 0x301 /* s390 timer register */
414416 #define NT_S390_TODCMP 0x302 /* s390 TOD clock comparator register */
ipc/shm.c+1−1
@@ -1662,7 +1662,7 @@ long do_shmat(int shmid, char __user *shmaddr, int shmflg, · 1661 unmodified lines
16621662 goto invalid;
16631663 }
16641664
1665− addr = do_mmap(file, addr, size, prot, flags, 0, &populate, NULL);
1665+ addr = do_mmap(file, addr, size, prot, flags, 0, 0, &populate, NULL);
16661666 *raddr = addr;
16671667 err = 0;
16681668 if (IS_ERR_VALUE(addr))
kernel/sys_ni.c+1−0
@@ -274,6 +274,7 @@ COND_SYSCALL(vm86old); · 273 unmodified lines
274274 COND_SYSCALL(modify_ldt);
275275 COND_SYSCALL(vm86);
276276 COND_SYSCALL(kexec_file_load);
277+COND_SYSCALL(map_shadow_stack);
277278
278279 /* s390 */
279280 COND_SYSCALL(s390_pci_mmio_read);
mm/debug_vm_pgtable.c+6−6
@@ -109,10 +109,10 @@ static void __init pte_basic_tests(struct pgtable_debug_args *args, int idx) · 108 unmodified lines
109109 WARN_ON(!pte_same(pte, pte));
110110 WARN_ON(!pte_young(pte_mkyoung(pte_mkold(pte))));
111111 WARN_ON(!pte_dirty(pte_mkdirty(pte_mkclean(pte))));
112− WARN_ON(!pte_write(pte_mkwrite(pte_wrprotect(pte))));
112+ WARN_ON(!pte_write(pte_mkwrite(pte_wrprotect(pte), args->vma)));
113113 WARN_ON(pte_young(pte_mkold(pte_mkyoung(pte))));
114114 WARN_ON(pte_dirty(pte_mkclean(pte_mkdirty(pte))));
115− WARN_ON(pte_write(pte_wrprotect(pte_mkwrite(pte))));
115+ WARN_ON(pte_write(pte_wrprotect(pte_mkwrite(pte, args->vma))));
116116 WARN_ON(pte_dirty(pte_wrprotect(pte_mkclean(pte))));
117117 WARN_ON(!pte_dirty(pte_wrprotect(pte_mkdirty(pte))));
118118 }
@@ -156,7 +156,7 @@ static void __init pte_advanced_tests(struct pgtable_debug_args *args) · 37 unmodified lines
156156 pte = pte_mkclean(pte);
157157 set_pte_at(args->mm, args->vaddr, args->ptep, pte);
158158 flush_dcache_page(page);
159− pte = pte_mkwrite(pte);
159+ pte = pte_mkwrite(pte, args->vma);
160160 pte = pte_mkdirty(pte);
161161 ptep_set_access_flags(args->vma, args->vaddr, args->ptep, pte, 1);
162162 pte = ptep_get(args->ptep);
@@ -202,10 +202,10 @@ static void __init pmd_basic_tests(struct pgtable_debug_args *args, int idx) · 39 unmodified lines
202202 WARN_ON(!pmd_same(pmd, pmd));
203203 WARN_ON(!pmd_young(pmd_mkyoung(pmd_mkold(pmd))));
204204 WARN_ON(!pmd_dirty(pmd_mkdirty(pmd_mkclean(pmd))));
205− WARN_ON(!pmd_write(pmd_mkwrite(pmd_wrprotect(pmd))));
205+ WARN_ON(!pmd_write(pmd_mkwrite(pmd_wrprotect(pmd), args->vma)));
206206 WARN_ON(pmd_young(pmd_mkold(pmd_mkyoung(pmd))));
207207 WARN_ON(pmd_dirty(pmd_mkclean(pmd_mkdirty(pmd))));
208− WARN_ON(pmd_write(pmd_wrprotect(pmd_mkwrite(pmd))));
208+ WARN_ON(pmd_write(pmd_wrprotect(pmd_mkwrite(pmd, args->vma))));
209209 WARN_ON(pmd_dirty(pmd_wrprotect(pmd_mkclean(pmd))));
210210 WARN_ON(!pmd_dirty(pmd_wrprotect(pmd_mkdirty(pmd))));
211211 /*
@@ -256,7 +256,7 @@ static void __init pmd_advanced_tests(struct pgtable_debug_args *args) · 44 unmodified lines
256256 pmd = pmd_mkclean(pmd);
257257 set_pmd_at(args->mm, vaddr, args->pmdp, pmd);
258258 flush_dcache_page(page);
259− pmd = pmd_mkwrite(pmd);
259+ pmd = pmd_mkwrite(pmd, args->vma);
260260 pmd = pmd_mkdirty(pmd);
261261 pmdp_set_access_flags(args->vma, vaddr, args->pmdp, pmd, 1);
262262 pmd = READ_ONCE(*args->pmdp);
mm/gup.c+1−1
@@ -1051,7 +1051,7 @@ static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags) · 1050 unmodified lines
10511051 !writable_file_mapping_allowed(vma, gup_flags))
10521052 return -EFAULT;
10531053
1054− if (!(vm_flags & VM_WRITE)) {
1054+ if (!(vm_flags & VM_WRITE) || (vm_flags & VM_SHADOW_STACK)) {
10551055 if (!(gup_flags & FOLL_FORCE))
10561056 return -EFAULT;
10571057 /* hugetlb does not support FOLL_FORCE|FOLL_WRITE. */
mm/huge_memory.c+6−5
@@ -551,7 +551,7 @@ __setup("transparent_hugepage=", setup_transparent_hugepage); · 550 unmodified lines
551551 pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
552552 {
553553 if (likely(vma->vm_flags & VM_WRITE))
554− pmd = pmd_mkwrite(pmd);
554+ pmd = pmd_mkwrite(pmd, vma);
555555 return pmd;
556556 }
557557
@@ -1566,7 +1566,7 @@ out_map: · 1008 unmodified lines
15661566 pmd = pmd_modify(oldpmd, vma->vm_page_prot);
15671567 pmd = pmd_mkyoung(pmd);
15681568 if (writable)
1569− pmd = pmd_mkwrite(pmd);
1569+ pmd = pmd_mkwrite(pmd, vma);
15701570 set_pmd_at(vma->vm_mm, haddr, vmf->pmd, pmd);
15711571 update_mmu_cache_pmd(vma, vmf->address, vmf->pmd);
15721572 spin_unlock(vmf->ptl);
@@ -1675,6 +1675,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · 102 unmodified lines
16751675 */
16761676 orig_pmd = pmdp_huge_get_and_clear_full(vma, addr, pmd,
16771677 tlb->fullmm);
1678+ arch_check_zapped_pmd(vma, orig_pmd);
16781679 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
16791680 if (vma_is_special_huge(vma)) {
16801681 if (arch_needs_pgtable_deposit())
@@ -1919,7 +1920,7 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, · 238 unmodified lines
19191920 /* See change_pte_range(). */
19201921 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) &&
19211922 can_change_pmd_writable(vma, addr, entry))
1922− entry = pmd_mkwrite(entry);
1923+ entry = pmd_mkwrite(entry, vma);
19231924
19241925 ret = HPAGE_PMD_NR;
19251926 set_pmd_at(mm, addr, pmd, entry);
@@ -2233,7 +2234,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, · 307 unmodified lines
22332234 } else {
22342235 entry = mk_pte(page + i, READ_ONCE(vma->vm_page_prot));
22352236 if (write)
2236− entry = pte_mkwrite(entry);
2237+ entry = pte_mkwrite(entry, vma);
22372238 if (anon_exclusive)
22382239 SetPageAnonExclusive(page + i);
22392240 if (!young)
@@ -3265,7 +3266,7 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) · 1025 unmodified lines
32653266 if (pmd_swp_soft_dirty(*pvmw->pmd))
32663267 pmde = pmd_mksoft_dirty(pmde);
32673268 if (is_writable_migration_entry(entry))
3268− pmde = pmd_mkwrite(pmde);
3269+ pmde = pmd_mkwrite(pmde, vma);
32693270 if (pmd_swp_uffd_wp(*pvmw->pmd))
32703271 pmde = pmd_mkuffd_wp(pmde);
32713272 if (!is_migration_entry_young(entry))
mm/internal.h+2−2
@@ -556,14 +556,14 @@ static inline bool is_exec_mapping(vm_flags_t flags) · 555 unmodified lines
556556 }
557557
558558 /*
559− * Stack area - automatically grows in one direction
559+ * Stack area (including shadow stacks)
560560 *
561561 * VM_GROWSUP / VM_GROWSDOWN VMAs are always private anonymous:
562562 * do_mmap() forbids all other combinations.
563563 */
564564 static inline bool is_stack_mapping(vm_flags_t flags)
565565 {
566− return (flags & VM_STACK) == VM_STACK;
566+ return ((flags & VM_STACK) == VM_STACK) || (flags & VM_SHADOW_STACK);
567567 }
568568
569569 /*
mm/memory.c+3−2
@@ -1430,6 +1430,7 @@ static unsigned long zap_pte_range(struct mmu_gather *tlb, · 1429 unmodified lines
14301430 continue;
14311431 ptent = ptep_get_and_clear_full(mm, addr, pte,
14321432 tlb->fullmm);
1433+ arch_check_zapped_pte(vma, ptent);
14331434 tlb_remove_tlb_entry(tlb, pte, addr);
14341435 zap_install_uffd_wp_if_needed(vma, addr, pte, details,
14351436 ptent);
@@ -4124,7 +4125,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf) · 2688 unmodified lines
41244125 entry = mk_pte(&folio->page, vma->vm_page_prot);
41254126 entry = pte_sw_mkyoung(entry);
41264127 if (vma->vm_flags & VM_WRITE)
4127− entry = pte_mkwrite(pte_mkdirty(entry));
4128+ entry = pte_mkwrite(pte_mkdirty(entry), vma);
41284129
41294130 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
41304131 &vmf->ptl);
@@ -4842,7 +4843,7 @@ out_map: · 711 unmodified lines
48424843 pte = pte_modify(old_pte, vma->vm_page_prot);
48434844 pte = pte_mkyoung(pte);
48444845 if (writable)
4845− pte = pte_mkwrite(pte);
4846+ pte = pte_mkwrite(pte, vma);
48464847 ptep_modify_prot_commit(vma, vmf->address, vmf->pte, old_pte, pte);
48474848 update_mmu_cache_range(vmf, vma, vmf->address, vmf->pte, 1);
48484849 pte_unmap_unlock(vmf->pte, vmf->ptl);
mm/migrate.c+1−1
@@ -220,7 +220,7 @@ static bool remove_migration_pte(struct folio *folio, · 219 unmodified lines
220220 if (folio_test_dirty(folio) && is_migration_entry_dirty(entry))
221221 pte = pte_mkdirty(pte);
222222 if (is_writable_migration_entry(entry))
223− pte = pte_mkwrite(pte);
223+ pte = pte_mkwrite(pte, vma);
224224 else if (pte_swp_uffd_wp(old_pte))
225225 pte = pte_mkuffd_wp(pte);
226226
mm/migrate_device.c+1−1
@@ -624,7 +624,7 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate, · 623 unmodified lines
624624 }
625625 entry = mk_pte(page, vma->vm_page_prot);
626626 if (vma->vm_flags & VM_WRITE)
627− entry = pte_mkwrite(pte_mkdirty(entry));
627+ entry = pte_mkwrite(pte_mkdirty(entry), vma);
628628 }
629629
630630 ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
mm/mmap.c+7−7
@@ -1182,11 +1182,11 @@ static inline bool file_mmap_ok(struct file *file, struct inode *inode, · 1181 unmodified lines
11821182 */
11831183 unsigned long do_mmap(struct file *file, unsigned long addr,
11841184 unsigned long len, unsigned long prot,
1185− unsigned long flags, unsigned long pgoff,
1186− unsigned long *populate, struct list_head *uf)
1185+ unsigned long flags, vm_flags_t vm_flags,
1186+ unsigned long pgoff, unsigned long *populate,
1187+ struct list_head *uf)
11871188 {
11881189 struct mm_struct *mm = current->mm;
1189− vm_flags_t vm_flags;
11901190 int pkey = 0;
11911191
11921192 *populate = 0;
@@ -1246,7 +1246,7 @@ unsigned long do_mmap(struct file *file, unsigned long addr, · 53 unmodified lines
12461246 * to. we assume access permissions have been handled by the open
12471247 * of the memory object, so we don't do any here.
12481248 */
1249− vm_flags = calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
1249+ vm_flags |= calc_vm_prot_bits(prot, pkey) | calc_vm_flag_bits(flags) |
12501250 mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
12511251
12521252 if (flags & MAP_LOCKED)
@@ -1564,7 +1564,7 @@ retry: · 311 unmodified lines
15641564 gap = mas.index;
15651565 gap += (info->align_offset - gap) & info->align_mask;
15661566 tmp = mas_next(&mas, ULONG_MAX);
1567− if (tmp && (tmp->vm_flags & VM_GROWSDOWN)) { /* Avoid prev check if possible */
1567+ if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */
15681568 if (vm_start_gap(tmp) < gap + length - 1) {
15691569 low_limit = tmp->vm_end;
15701570 mas_reset(&mas);
@@ -1616,7 +1616,7 @@ retry: · 45 unmodified lines
16161616 gap -= (gap - info->align_offset) & info->align_mask;
16171617 gap_end = mas.last;
16181618 tmp = mas_next(&mas, ULONG_MAX);
1619− if (tmp && (tmp->vm_flags & VM_GROWSDOWN)) { /* Avoid prev check if possible */
1619+ if (tmp && (tmp->vm_flags & VM_STARTGAP_FLAGS)) { /* Avoid prev check if possible */
16201620 if (vm_start_gap(tmp) <= gap_end) {
16211621 high_limit = vm_start_gap(tmp);
16221622 mas_reset(&mas);
@@ -2998,7 +2998,7 @@ SYSCALL_DEFINE5(remap_file_pages, unsigned long, start, unsigned long, size, · 1375 unmodified lines
29982998
29992999 file = get_file(vma->vm_file);
30003000 ret = do_mmap(vma->vm_file, start, size,
3001− prot, flags, pgoff, &populate, NULL);
3001+ prot, flags, 0, pgoff, &populate, NULL);
30023002 fput(file);
30033003 out:
30043004 mmap_write_unlock(mm);
mm/mprotect.c+1−1
@@ -185,7 +185,7 @@ static long change_pte_range(struct mmu_gather *tlb, · 184 unmodified lines
185185 if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) &&
186186 !pte_write(ptent) &&
187187 can_change_pte_writable(vma, addr, ptent))
188− ptent = pte_mkwrite(ptent);
188+ ptent = pte_mkwrite(ptent, vma);
189189
190190 ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);
191191 if (pte_needs_flush(oldpte, ptent))
mm/nommu.c+2−2
@@ -1016,6 +1016,7 @@ unsigned long do_mmap(struct file *file, · 1015 unmodified lines
10161016 unsigned long len,
10171017 unsigned long prot,
10181018 unsigned long flags,
1019+ vm_flags_t vm_flags,
10191020 unsigned long pgoff,
10201021 unsigned long *populate,
10211022 struct list_head *uf)
@@ -1023,7 +1024,6 @@ unsigned long do_mmap(struct file *file, · 1 unmodified line
10231024 struct vm_area_struct *vma;
10241025 struct vm_region *region;
10251026 struct rb_node *rb;
1026− vm_flags_t vm_flags;
10271027 unsigned long capabilities, result;
10281028 int ret;
10291029 VMA_ITERATOR(vmi, current->mm, 0);
@@ -1043,7 +1043,7 @@ unsigned long do_mmap(struct file *file, · 13 unmodified lines
10431043
10441044 /* we've determined that we can make the mapping, now translate what we
10451045 * now know into VMA flags */
1046− vm_flags = determine_vm_flags(file, prot, flags, capabilities);
1046+ vm_flags |= determine_vm_flags(file, prot, flags, capabilities);
10471047
10481048
10491049 /* we're going to need to record the mapping */
mm/userfaultfd.c+1−1
@@ -86,7 +86,7 @@ int mfill_atomic_install_pte(pmd_t *dst_pmd, · 85 unmodified lines
8686 if (page_in_cache && !vm_shared)
8787 writable = false;
8888 if (writable)
89− _dst_pte = pte_mkwrite(_dst_pte);
89+ _dst_pte = pte_mkwrite(_dst_pte, dst_vma);
9090 if (flags & MFILL_ATOMIC_WP)
9191 _dst_pte = pte_mkuffd_wp(_dst_pte);
9292
mm/util.c+1−1
@@ -543,7 +543,7 @@ unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr, · 542 unmodified lines
543543 if (!ret) {
544544 if (mmap_write_lock_killable(mm))
545545 return -EINTR;
546− ret = do_mmap(file, addr, len, prot, flag, pgoff, &populate,
546+ ret = do_mmap(file, addr, len, prot, flag, 0, pgoff, &populate,
547547 &uf);
548548 mmap_write_unlock(mm);
549549 userfaultfd_unmap_complete(mm, &uf);
tools/testing/selftests/x86/Makefile+1−1
@@ -18,7 +18,7 @@ TARGETS_C_32BIT_ONLY := entry_from_vm86 test_syscall_vdso unwind_vdso \ · 17 unmodified lines
1818 test_FCMOV test_FCOMI test_FISTTP \
1919 vdso_restorer
2020 TARGETS_C_64BIT_ONLY := fsgsbase sysret_rip syscall_numbering \
21− corrupt_xstate_header amx lam
21+ corrupt_xstate_header amx lam test_shadow_stack
2222 # Some selftests require 32bit support enabled also on 64bit systems
2323 TARGETS_C_32BIT_NEEDED := ldt_gdt ptrace_syscall
2424
tools/testing/selftests/x86/test_shadow_stack.c+884−0
@@ -0,0 +1,884 @@
1+// SPDX-License-Identifier: GPL-2.0
2+/*
3+ * This program test's basic kernel shadow stack support. It enables shadow
4+ * stack manual via the arch_prctl(), instead of relying on glibc. It's
5+ * Makefile doesn't compile with shadow stack support, so it doesn't rely on
6+ * any particular glibc. As a result it can't do any operations that require
7+ * special glibc shadow stack support (longjmp(), swapcontext(), etc). Just
8+ * stick to the basics and hope the compiler doesn't do anything strange.
9+ */
10+
11+#define _GNU_SOURCE
12+
13+#include <sys/syscall.h>
14+#include <asm/mman.h>
15+#include <sys/mman.h>
16+#include <sys/stat.h>
17+#include <sys/wait.h>
18+#include <stdio.h>
19+#include <stdlib.h>
20+#include <fcntl.h>
21+#include <unistd.h>
22+#include <string.h>
23+#include <errno.h>
24+#include <stdbool.h>
25+#include <x86intrin.h>
26+#include <asm/prctl.h>
27+#include <sys/prctl.h>
28+#include <stdint.h>
29+#include <signal.h>
30+#include <pthread.h>
31+#include <sys/ioctl.h>
32+#include <linux/userfaultfd.h>
33+#include <setjmp.h>
34+#include <sys/ptrace.h>
35+#include <sys/signal.h>
36+#include <linux/elf.h>
37+
38+/*
39+ * Define the ABI defines if needed, so people can run the tests
40+ * without building the headers.
41+ */
42+#ifndef __NR_map_shadow_stack
43+#define __NR_map_shadow_stack 452
44+
45+#define SHADOW_STACK_SET_TOKEN (1ULL << 0)
46+
47+#define ARCH_SHSTK_ENABLE 0x5001
48+#define ARCH_SHSTK_DISABLE 0x5002
49+#define ARCH_SHSTK_LOCK 0x5003
50+#define ARCH_SHSTK_UNLOCK 0x5004
51+#define ARCH_SHSTK_STATUS 0x5005
52+
53+#define ARCH_SHSTK_SHSTK (1ULL << 0)
54+#define ARCH_SHSTK_WRSS (1ULL << 1)
55+
56+#define NT_X86_SHSTK 0x204
57+#endif
58+
59+#define SS_SIZE 0x200000
60+#define PAGE_SIZE 0x1000
61+
62+#if (__GNUC__ < 8) || (__GNUC__ == 8 && __GNUC_MINOR__ < 5)
63+int main(int argc, char *argv[])
64+{
65+ printf("[SKIP]\tCompiler does not support CET.\n");
66+ return 0;
67+}
68+#else
69+void write_shstk(unsigned long *addr, unsigned long val)
70+{
71+ asm volatile("wrssq %[val], (%[addr])\n"
72+ : "=m" (addr)
73+ : [addr] "r" (addr), [val] "r" (val));
74+}
75+
76+static inline unsigned long __attribute__((always_inline)) get_ssp(void)
77+{
78+ unsigned long ret = 0;
79+
80+ asm volatile("xor %0, %0; rdsspq %0" : "=r" (ret));
81+ return ret;
82+}
83+
84+/*
85+ * For use in inline enablement of shadow stack.
86+ *
87+ * The program can't return from the point where shadow stack gets enabled
88+ * because there will be no address on the shadow stack. So it can't use
89+ * syscall() for enablement, since it is a function.
90+ *
91+ * Based on code from nolibc.h. Keep a copy here because this can't pull in all
92+ * of nolibc.h.
93+ */
94+#define ARCH_PRCTL(arg1, arg2) \
95+({ \
96+ long _ret; \
97+ register long _num asm("eax") = __NR_arch_prctl; \
98+ register long _arg1 asm("rdi") = (long)(arg1); \
99+ register long _arg2 asm("rsi") = (long)(arg2); \
100+ \
101+ asm volatile ( \
102+ "syscall\n" \
103+ : "=a"(_ret) \
104+ : "r"(_arg1), "r"(_arg2), \
105+ "0"(_num) \
106+ : "rcx", "r11", "memory", "cc" \
107+ ); \
108+ _ret; \
109+})
110+
111+void *create_shstk(void *addr)
112+{
113+ return (void *)syscall(__NR_map_shadow_stack, addr, SS_SIZE, SHADOW_STACK_SET_TOKEN);
114+}
115+
116+void *create_normal_mem(void *addr)
117+{
118+ return mmap(addr, SS_SIZE, PROT_READ | PROT_WRITE,
119+ MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
120+}
121+
122+void free_shstk(void *shstk)
123+{
124+ munmap(shstk, SS_SIZE);
125+}
126+
127+int reset_shstk(void *shstk)
128+{
129+ return madvise(shstk, SS_SIZE, MADV_DONTNEED);
130+}
131+
132+void try_shstk(unsigned long new_ssp)
133+{
134+ unsigned long ssp;
135+
136+ printf("[INFO]\tnew_ssp = %lx, *new_ssp = %lx\n",
137+ new_ssp, *((unsigned long *)new_ssp));
138+
139+ ssp = get_ssp();
140+ printf("[INFO]\tchanging ssp from %lx to %lx\n", ssp, new_ssp);
141+
142+ asm volatile("rstorssp (%0)\n":: "r" (new_ssp));
143+ asm volatile("saveprevssp");
144+ printf("[INFO]\tssp is now %lx\n", get_ssp());
145+
146+ /* Switch back to original shadow stack */
147+ ssp -= 8;
148+ asm volatile("rstorssp (%0)\n":: "r" (ssp));
149+ asm volatile("saveprevssp");
150+}
151+
152+int test_shstk_pivot(void)
153+{
154+ void *shstk = create_shstk(0);
155+
156+ if (shstk == MAP_FAILED) {
157+ printf("[FAIL]\tError creating shadow stack: %d\n", errno);
158+ return 1;
159+ }
160+ try_shstk((unsigned long)shstk + SS_SIZE - 8);
161+ free_shstk(shstk);
162+
163+ printf("[OK]\tShadow stack pivot\n");
164+ return 0;
165+}
166+
167+int test_shstk_faults(void)
168+{
169+ unsigned long *shstk = create_shstk(0);
170+
171+ /* Read shadow stack, test if it's zero to not get read optimized out */
172+ if (*shstk != 0)
173+ goto err;
174+
175+ /* Wrss memory that was already read. */
176+ write_shstk(shstk, 1);
177+ if (*shstk != 1)
178+ goto err;
179+
180+ /* Page out memory, so we can wrss it again. */
181+ if (reset_shstk((void *)shstk))
182+ goto err;
183+
184+ write_shstk(shstk, 1);
185+ if (*shstk != 1)
186+ goto err;
187+
188+ printf("[OK]\tShadow stack faults\n");
189+ return 0;
190+
191+err:
192+ return 1;
193+}
194+
195+unsigned long saved_ssp;
196+unsigned long saved_ssp_val;
197+volatile bool segv_triggered;
198+
199+void __attribute__((noinline)) violate_ss(void)
200+{
201+ saved_ssp = get_ssp();
202+ saved_ssp_val = *(unsigned long *)saved_ssp;
203+
204+ /* Corrupt shadow stack */
205+ printf("[INFO]\tCorrupting shadow stack\n");
206+ write_shstk((void *)saved_ssp, 0);
207+}
208+
209+void segv_handler(int signum, siginfo_t *si, void *uc)
210+{
211+ printf("[INFO]\tGenerated shadow stack violation successfully\n");
212+
213+ segv_triggered = true;
214+
215+ /* Fix shadow stack */
216+ write_shstk((void *)saved_ssp, saved_ssp_val);
217+}
218+
219+int test_shstk_violation(void)
220+{
221+ struct sigaction sa = {};
222+
223+ sa.sa_sigaction = segv_handler;
224+ sa.sa_flags = SA_SIGINFO;
225+ if (sigaction(SIGSEGV, &sa, NULL))
226+ return 1;
227+
228+ segv_triggered = false;
229+
230+ /* Make sure segv_triggered is set before violate_ss() */
231+ asm volatile("" : : : "memory");
232+
233+ violate_ss();
234+
235+ signal(SIGSEGV, SIG_DFL);
236+
237+ printf("[OK]\tShadow stack violation test\n");
238+
239+ return !segv_triggered;
240+}
241+
242+/* Gup test state */
243+#define MAGIC_VAL 0x12345678
244+bool is_shstk_access;
245+void *shstk_ptr;
246+int fd;
247+
248+void reset_test_shstk(void *addr)
249+{
250+ if (shstk_ptr)
251+ free_shstk(shstk_ptr);
252+ shstk_ptr = create_shstk(addr);
253+}
254+
255+void test_access_fix_handler(int signum, siginfo_t *si, void *uc)
256+{
257+ printf("[INFO]\tViolation from %s\n", is_shstk_access ? "shstk access" : "normal write");
258+
259+ segv_triggered = true;
260+
261+ /* Fix shadow stack */
262+ if (is_shstk_access) {
263+ reset_test_shstk(shstk_ptr);
264+ return;
265+ }
266+
267+ free_shstk(shstk_ptr);
268+ create_normal_mem(shstk_ptr);
269+}
270+
271+bool test_shstk_access(void *ptr)
272+{
273+ is_shstk_access = true;
274+ segv_triggered = false;
275+ write_shstk(ptr, MAGIC_VAL);
276+
277+ asm volatile("" : : : "memory");
278+
279+ return segv_triggered;
280+}
281+
282+bool test_write_access(void *ptr)
283+{
284+ is_shstk_access = false;
285+ segv_triggered = false;
286+ *(unsigned long *)ptr = MAGIC_VAL;
287+
288+ asm volatile("" : : : "memory");
289+
290+ return segv_triggered;
291+}
292+
293+bool gup_write(void *ptr)
294+{
295+ unsigned long val;
296+
297+ lseek(fd, (unsigned long)ptr, SEEK_SET);
298+ if (write(fd, &val, sizeof(val)) < 0)
299+ return 1;
300+
301+ return 0;
302+}
303+
304+bool gup_read(void *ptr)
305+{
306+ unsigned long val;
307+
308+ lseek(fd, (unsigned long)ptr, SEEK_SET);
309+ if (read(fd, &val, sizeof(val)) < 0)
310+ return 1;
311+
312+ return 0;
313+}
314+
315+int test_gup(void)
316+{
317+ struct sigaction sa = {};
318+ int status;
319+ pid_t pid;
320+
321+ sa.sa_sigaction = test_access_fix_handler;
322+ sa.sa_flags = SA_SIGINFO;
323+ if (sigaction(SIGSEGV, &sa, NULL))
324+ return 1;
325+
326+ segv_triggered = false;
327+
328+ fd = open("/proc/self/mem", O_RDWR);
329+ if (fd == -1)
330+ return 1;
331+
332+ reset_test_shstk(0);
333+ if (gup_read(shstk_ptr))
334+ return 1;
335+ if (test_shstk_access(shstk_ptr))
336+ return 1;
337+ printf("[INFO]\tGup read -> shstk access success\n");
338+
339+ reset_test_shstk(0);
340+ if (gup_write(shstk_ptr))
341+ return 1;
342+ if (test_shstk_access(shstk_ptr))
343+ return 1;
344+ printf("[INFO]\tGup write -> shstk access success\n");
345+
346+ reset_test_shstk(0);
347+ if (gup_read(shstk_ptr))
348+ return 1;
349+ if (!test_write_access(shstk_ptr))
350+ return 1;
351+ printf("[INFO]\tGup read -> write access success\n");
352+
353+ reset_test_shstk(0);
354+ if (gup_write(shstk_ptr))
355+ return 1;
356+ if (!test_write_access(shstk_ptr))
357+ return 1;
358+ printf("[INFO]\tGup write -> write access success\n");
359+
360+ close(fd);
361+
362+ /* COW/gup test */
363+ reset_test_shstk(0);
364+ pid = fork();
365+ if (!pid) {
366+ fd = open("/proc/self/mem", O_RDWR);
367+ if (fd == -1)
368+ exit(1);
369+
370+ if (gup_write(shstk_ptr)) {
371+ close(fd);
372+ exit(1);
373+ }
374+ close(fd);
375+ exit(0);
376+ }
377+ waitpid(pid, &status, 0);
378+ if (WEXITSTATUS(status)) {
379+ printf("[FAIL]\tWrite in child failed\n");
380+ return 1;
381+ }
382+ if (*(unsigned long *)shstk_ptr == MAGIC_VAL) {
383+ printf("[FAIL]\tWrite in child wrote through to shared memory\n");
384+ return 1;
385+ }
386+
387+ printf("[INFO]\tCow gup write -> write access success\n");
388+
389+ free_shstk(shstk_ptr);
390+
391+ signal(SIGSEGV, SIG_DFL);
392+
393+ printf("[OK]\tShadow gup test\n");
394+
395+ return 0;
396+}
397+
398+int test_mprotect(void)
399+{
400+ struct sigaction sa = {};
401+
402+ sa.sa_sigaction = test_access_fix_handler;
403+ sa.sa_flags = SA_SIGINFO;
404+ if (sigaction(SIGSEGV, &sa, NULL))
405+ return 1;
406+
407+ segv_triggered = false;
408+
409+ /* mprotect a shadow stack as read only */
410+ reset_test_shstk(0);
411+ if (mprotect(shstk_ptr, SS_SIZE, PROT_READ) < 0) {
412+ printf("[FAIL]\tmprotect(PROT_READ) failed\n");
413+ return 1;
414+ }
415+
416+ /* try to wrss it and fail */
417+ if (!test_shstk_access(shstk_ptr)) {
418+ printf("[FAIL]\tShadow stack access to read-only memory succeeded\n");
419+ return 1;
420+ }
421+
422+ /*
423+ * The shadow stack was reset above to resolve the fault, make the new one
424+ * read-only.
425+ */
426+ if (mprotect(shstk_ptr, SS_SIZE, PROT_READ) < 0) {
427+ printf("[FAIL]\tmprotect(PROT_READ) failed\n");
428+ return 1;
429+ }
430+
431+ /* then back to writable */
432+ if (mprotect(shstk_ptr, SS_SIZE, PROT_WRITE | PROT_READ) < 0) {
433+ printf("[FAIL]\tmprotect(PROT_WRITE) failed\n");
434+ return 1;
435+ }
436+
437+ /* then wrss to it and succeed */
438+ if (test_shstk_access(shstk_ptr)) {
439+ printf("[FAIL]\tShadow stack access to mprotect() writable memory failed\n");
440+ return 1;
441+ }
442+
443+ free_shstk(shstk_ptr);
444+
445+ signal(SIGSEGV, SIG_DFL);
446+
447+ printf("[OK]\tmprotect() test\n");
448+
449+ return 0;
450+}
451+
452+char zero[4096];
453+
454+static void *uffd_thread(void *arg)
455+{
456+ struct uffdio_copy req;
457+ int uffd = *(int *)arg;
458+ struct uffd_msg msg;
459+ int ret;
460+
461+ while (1) {
462+ ret = read(uffd, &msg, sizeof(msg));
463+ if (ret > 0)
464+ break;
465+ else if (errno == EAGAIN)
466+ continue;
467+ return (void *)1;
468+ }
469+
470+ req.dst = msg.arg.pagefault.address;
471+ req.src = (__u64)zero;
472+ req.len = 4096;
473+ req.mode = 0;
474+
475+ if (ioctl(uffd, UFFDIO_COPY, &req))
476+ return (void *)1;
477+
478+ return (void *)0;
479+}
480+
481+int test_userfaultfd(void)
482+{
483+ struct uffdio_register uffdio_register;
484+ struct uffdio_api uffdio_api;
485+ struct sigaction sa = {};
486+ pthread_t thread;
487+ void *res;
488+ int uffd;
489+
490+ sa.sa_sigaction = test_access_fix_handler;
491+ sa.sa_flags = SA_SIGINFO;
492+ if (sigaction(SIGSEGV, &sa, NULL))
493+ return 1;
494+
495+ uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK);
496+ if (uffd < 0) {
497+ printf("[SKIP]\tUserfaultfd unavailable.\n");
498+ return 0;
499+ }
500+
501+ reset_test_shstk(0);
502+
503+ uffdio_api.api = UFFD_API;
504+ uffdio_api.features = 0;
505+ if (ioctl(uffd, UFFDIO_API, &uffdio_api))
506+ goto err;
507+
508+ uffdio_register.range.start = (__u64)shstk_ptr;
509+ uffdio_register.range.len = 4096;
510+ uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
511+ if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
512+ goto err;
513+
514+ if (pthread_create(&thread, NULL, &uffd_thread, &uffd))
515+ goto err;
516+
517+ reset_shstk(shstk_ptr);
518+ test_shstk_access(shstk_ptr);
519+
520+ if (pthread_join(thread, &res))
521+ goto err;
522+
523+ if (test_shstk_access(shstk_ptr))
524+ goto err;
525+
526+ free_shstk(shstk_ptr);
527+
528+ signal(SIGSEGV, SIG_DFL);
529+
530+ if (!res)
531+ printf("[OK]\tUserfaultfd test\n");
532+ return !!res;
533+err:
534+ free_shstk(shstk_ptr);
535+ close(uffd);
536+ signal(SIGSEGV, SIG_DFL);
537+ return 1;
538+}
539+
540+/* Simple linked list for keeping track of mappings in test_guard_gap() */
541+struct node {
542+ struct node *next;
543+ void *mapping;
544+};
545+
546+/*
547+ * This tests whether mmap will place other mappings in a shadow stack's guard
548+ * gap. The steps are:
549+ * 1. Finds an empty place by mapping and unmapping something.
550+ * 2. Map a shadow stack in the middle of the known empty area.
551+ * 3. Map a bunch of PAGE_SIZE mappings. These will use the search down
552+ * direction, filling any gaps until it encounters the shadow stack's
553+ * guard gap.
554+ * 4. When a mapping lands below the shadow stack from step 2, then all
555+ * of the above gaps are filled. The search down algorithm will have
556+ * looked at the shadow stack gaps.
557+ * 5. See if it landed in the gap.
558+ */
559+int test_guard_gap(void)
560+{
561+ void *free_area, *shstk, *test_map = (void *)0xFFFFFFFFFFFFFFFF;
562+ struct node *head = NULL, *cur;
563+
564+ free_area = mmap(0, SS_SIZE * 3, PROT_READ | PROT_WRITE,
565+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
566+ munmap(free_area, SS_SIZE * 3);
567+
568+ shstk = create_shstk(free_area + SS_SIZE);
569+ if (shstk == MAP_FAILED)
570+ return 1;
571+
572+ while (test_map > shstk) {
573+ test_map = mmap(0, PAGE_SIZE, PROT_READ | PROT_WRITE,
574+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
575+ if (test_map == MAP_FAILED)
576+ return 1;
577+ cur = malloc(sizeof(*cur));
578+ cur->mapping = test_map;
579+
580+ cur->next = head;
581+ head = cur;
582+ }
583+
584+ while (head) {
585+ cur = head;
586+ head = cur->next;
587+ munmap(cur->mapping, PAGE_SIZE);
588+ free(cur);
589+ }
590+
591+ free_shstk(shstk);
592+
593+ if (shstk - test_map - PAGE_SIZE != PAGE_SIZE)
594+ return 1;
595+
596+ printf("[OK]\tGuard gap test\n");
597+
598+ return 0;
599+}
600+
601+/*
602+ * Too complicated to pull it out of the 32 bit header, but also get the
603+ * 64 bit one needed above. Just define a copy here.
604+ */
605+#define __NR_compat_sigaction 67
606+
607+/*
608+ * Call 32 bit signal handler to get 32 bit signals ABI. Make sure
609+ * to push the registers that will get clobbered.
610+ */
611+int sigaction32(int signum, const struct sigaction *restrict act,
612+ struct sigaction *restrict oldact)
613+{
614+ register long syscall_reg asm("eax") = __NR_compat_sigaction;
615+ register long signum_reg asm("ebx") = signum;
616+ register long act_reg asm("ecx") = (long)act;
617+ register long oldact_reg asm("edx") = (long)oldact;
618+ int ret = 0;
619+
620+ asm volatile ("int $0x80;"
621+ : "=a"(ret), "=m"(oldact)
622+ : "r"(syscall_reg), "r"(signum_reg), "r"(act_reg),
623+ "r"(oldact_reg)
624+ : "r8", "r9", "r10", "r11"
625+ );
626+
627+ return ret;
628+}
629+
630+sigjmp_buf jmp_buffer;
631+
632+void segv_gp_handler(int signum, siginfo_t *si, void *uc)
633+{
634+ segv_triggered = true;
635+
636+ /*
637+ * To work with old glibc, this can't rely on siglongjmp working with
638+ * shadow stack enabled, so disable shadow stack before siglongjmp().
639+ */
640+ ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK);
641+ siglongjmp(jmp_buffer, -1);
642+}
643+
644+/*
645+ * Transition to 32 bit mode and check that a #GP triggers a segfault.
646+ */
647+int test_32bit(void)
648+{
649+ struct sigaction sa = {};
650+ struct sigaction *sa32;
651+
652+ /* Create sigaction in 32 bit address range */
653+ sa32 = mmap(0, 4096, PROT_READ | PROT_WRITE,
654+ MAP_32BIT | MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
655+ sa32->sa_flags = SA_SIGINFO;
656+
657+ sa.sa_sigaction = segv_gp_handler;
658+ sa.sa_flags = SA_SIGINFO;
659+ if (sigaction(SIGSEGV, &sa, NULL))
660+ return 1;
661+
662+
663+ segv_triggered = false;
664+
665+ /* Make sure segv_triggered is set before triggering the #GP */
666+ asm volatile("" : : : "memory");
667+
668+ /*
669+ * Set handler to somewhere in 32 bit address space
670+ */
671+ sa32->sa_handler = (void *)sa32;
672+ if (sigaction32(SIGUSR1, sa32, NULL))
673+ return 1;
674+
675+ if (!sigsetjmp(jmp_buffer, 1))
676+ raise(SIGUSR1);
677+
678+ if (segv_triggered)
679+ printf("[OK]\t32 bit test\n");
680+
681+ return !segv_triggered;
682+}
683+
684+void segv_handler_ptrace(int signum, siginfo_t *si, void *uc)
685+{
686+ /* The SSP adjustment caused a segfault. */
687+ exit(0);
688+}
689+
690+int test_ptrace(void)
691+{
692+ unsigned long saved_ssp, ssp = 0;
693+ struct sigaction sa= {};
694+ struct iovec iov;
695+ int status;
696+ int pid;
697+
698+ iov.iov_base = &ssp;
699+ iov.iov_len = sizeof(ssp);
700+
701+ pid = fork();
702+ if (!pid) {
703+ ssp = get_ssp();
704+
705+ sa.sa_sigaction = segv_handler_ptrace;
706+ sa.sa_flags = SA_SIGINFO;
707+ if (sigaction(SIGSEGV, &sa, NULL))
708+ return 1;
709+
710+ ptrace(PTRACE_TRACEME, NULL, NULL, NULL);
711+ /*
712+ * The parent will tweak the SSP and return from this function
713+ * will #CP.
714+ */
715+ raise(SIGTRAP);
716+
717+ exit(1);
718+ }
719+
720+ while (waitpid(pid, &status, 0) != -1 && WSTOPSIG(status) != SIGTRAP);
721+
722+ if (ptrace(PTRACE_GETREGSET, pid, NT_X86_SHSTK, &iov)) {
723+ printf("[INFO]\tFailed to PTRACE_GETREGS\n");
724+ goto out_kill;
725+ }
726+
727+ if (!ssp) {
728+ printf("[INFO]\tPtrace child SSP was 0\n");
729+ goto out_kill;
730+ }
731+
732+ saved_ssp = ssp;
733+
734+ iov.iov_len = 0;
735+ if (!ptrace(PTRACE_SETREGSET, pid, NT_X86_SHSTK, &iov)) {
736+ printf("[INFO]\tToo small size accepted via PTRACE_SETREGS\n");
737+ goto out_kill;
738+ }
739+
740+ iov.iov_len = sizeof(ssp) + 1;
741+ if (!ptrace(PTRACE_SETREGSET, pid, NT_X86_SHSTK, &iov)) {
742+ printf("[INFO]\tToo large size accepted via PTRACE_SETREGS\n");
743+ goto out_kill;
744+ }
745+
746+ ssp += 1;
747+ if (!ptrace(PTRACE_SETREGSET, pid, NT_X86_SHSTK, &iov)) {
748+ printf("[INFO]\tUnaligned SSP written via PTRACE_SETREGS\n");
749+ goto out_kill;
750+ }
751+
752+ ssp = 0xFFFFFFFFFFFF0000;
753+ if (!ptrace(PTRACE_SETREGSET, pid, NT_X86_SHSTK, &iov)) {
754+ printf("[INFO]\tKernel range SSP written via PTRACE_SETREGS\n");
755+ goto out_kill;
756+ }
757+
758+ /*
759+ * Tweak the SSP so the child with #CP when it resumes and returns
760+ * from raise()
761+ */
762+ ssp = saved_ssp + 8;
763+ iov.iov_len = sizeof(ssp);
764+ if (ptrace(PTRACE_SETREGSET, pid, NT_X86_SHSTK, &iov)) {
765+ printf("[INFO]\tFailed to PTRACE_SETREGS\n");
766+ goto out_kill;
767+ }
768+
769+ if (ptrace(PTRACE_DETACH, pid, NULL, NULL)) {
770+ printf("[INFO]\tFailed to PTRACE_DETACH\n");
771+ goto out_kill;
772+ }
773+
774+ waitpid(pid, &status, 0);
775+ if (WEXITSTATUS(status))
776+ return 1;
777+
778+ printf("[OK]\tPtrace test\n");
779+ return 0;
780+
781+out_kill:
782+ kill(pid, SIGKILL);
783+ return 1;
784+}
785+
786+int main(int argc, char *argv[])
787+{
788+ int ret = 0;
789+
790+ if (ARCH_PRCTL(ARCH_SHSTK_ENABLE, ARCH_SHSTK_SHSTK)) {
791+ printf("[SKIP]\tCould not enable Shadow stack\n");
792+ return 1;
793+ }
794+
795+ if (ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK)) {
796+ ret = 1;
797+ printf("[FAIL]\tDisabling shadow stack failed\n");
798+ }
799+
800+ if (ARCH_PRCTL(ARCH_SHSTK_ENABLE, ARCH_SHSTK_SHSTK)) {
801+ printf("[SKIP]\tCould not re-enable Shadow stack\n");
802+ return 1;
803+ }
804+
805+ if (ARCH_PRCTL(ARCH_SHSTK_ENABLE, ARCH_SHSTK_WRSS)) {
806+ printf("[SKIP]\tCould not enable WRSS\n");
807+ ret = 1;
808+ goto out;
809+ }
810+
811+ /* Should have succeeded if here, but this is a test, so double check. */
812+ if (!get_ssp()) {
813+ printf("[FAIL]\tShadow stack disabled\n");
814+ return 1;
815+ }
816+
817+ if (test_shstk_pivot()) {
818+ ret = 1;
819+ printf("[FAIL]\tShadow stack pivot\n");
820+ goto out;
821+ }
822+
823+ if (test_shstk_faults()) {
824+ ret = 1;
825+ printf("[FAIL]\tShadow stack fault test\n");
826+ goto out;
827+ }
828+
829+ if (test_shstk_violation()) {
830+ ret = 1;
831+ printf("[FAIL]\tShadow stack violation test\n");
832+ goto out;
833+ }
834+
835+ if (test_gup()) {
836+ ret = 1;
837+ printf("[FAIL]\tShadow shadow stack gup\n");
838+ goto out;
839+ }
840+
841+ if (test_mprotect()) {
842+ ret = 1;
843+ printf("[FAIL]\tShadow shadow mprotect test\n");
844+ goto out;
845+ }
846+
847+ if (test_userfaultfd()) {
848+ ret = 1;
849+ printf("[FAIL]\tUserfaultfd test\n");
850+ goto out;
851+ }
852+
853+ if (test_guard_gap()) {
854+ ret = 1;
855+ printf("[FAIL]\tGuard gap test\n");
856+ goto out;
857+ }
858+
859+ if (test_ptrace()) {
860+ ret = 1;
861+ printf("[FAIL]\tptrace test\n");
862+ }
863+
864+ if (test_32bit()) {
865+ ret = 1;
866+ printf("[FAIL]\t32 bit test\n");
867+ goto out;
868+ }
869+
870+ return ret;
871+
872+out:
873+ /*
874+ * Disable shadow stack before the function returns, or there will be a
875+ * shadow stack violation.
876+ */
877+ if (ARCH_PRCTL(ARCH_SHSTK_DISABLE, ARCH_SHSTK_SHSTK)) {
878+ ret = 1;
879+ printf("[FAIL]\tDisabling shadow stack failed\n");
880+ }
881+
882+ return ret;
883+}
884+#endif

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t