History

Merge branch 'for-next/scs' into for-next/core

Support for Clang's Shadow Call Stack in the kernel (Sami Tolvanen and Will Deacon) * for-next/scs: arm64: entry-ftrace.S: Update comment to indicate that x18 is live scs: Move DEFINE_SCS macro into core code scs: Remove references to asm/scs.h from core code scs: Move scs_overflow_check() out of architecture code arm64: scs: Use 'scs_sp' register alias for x18 scs: Move accounting into alloc/free functions arm64: scs: Store absolute SCS stack pointer value in thread_info efi/libstub: Disable Shadow Call Stack arm64: scs: Add shadow stacks for SDEI arm64: Implement Shadow Call Stack arm64: Disable SCS for hypervisor code arm64: vdso: Disable Shadow Call Stack arm64: efi: Restore register x18 if it was corrupted arm64: Preserve register x18 when CPU is suspended arm64: Reserve register x18 from general allocation with SCS scs: Disable when function graph tracing is enabled scs: Add support for stack usage debugging scs: Add page accounting for shadow call stack allocations scs: Add support for Clang's Shadow Call Stack (SCS)

Will Deacon committed 6 years ago

Commit 082af5e2 parents c350717 258c3d631 files · +406 −9Browse files

Unified · Split

31 files
  1. Makefile +6 −0
  2. arch/Kconfig +25 −0
  3. arch/arm64/Kconfig +5 −0
  4. arch/arm64/Makefile +4 −0
  5. arch/arm64/include/asm/kvm_hyp.h +1 −1
  6. arch/arm64/include/asm/scs.h +29 −0
  7. arch/arm64/include/asm/suspend.h +1 −1
  8. arch/arm64/include/asm/thread_info.h +13 −0
  9. arch/arm64/kernel/Makefile +1 −0
  10. arch/arm64/kernel/asm-offsets.c +4 −0
  11. arch/arm64/kernel/efi-rt-wrapper.S +10 −1
  12. arch/arm64/kernel/entry-ftrace.S +3 −2
  13. arch/arm64/kernel/entry.S +35 −3
  14. arch/arm64/kernel/head.S +6 −0
  15. arch/arm64/kernel/scs.c +16 −0
  16. arch/arm64/kernel/vdso/Makefile +1 −1
  17. arch/arm64/mm/proc.S +14 −0
  18. drivers/base/node.c +6 −0
  19. drivers/firmware/efi/libstub/Makefile +3 −0
  20. fs/proc/meminfo.c +4 −0
  21. include/linux/compiler-clang.h +4 −0
  22. include/linux/compiler_types.h +4 −0
  23. include/linux/mmzone.h +3 −0
  24. include/linux/scs.h +72 −0
  25. init/init_task.c +8 −0
  26. kernel/Makefile +1 −0
  27. kernel/fork.c +9 −0
  28. kernel/sched/core.c +5 −0
  29. kernel/scs.c +104 −0
  30. mm/page_alloc.c +6 −0
  31. mm/vmstat.c +3 −0
Makefile+6−0
@@ -866,6 +866,12 @@ ifdef CONFIG_LIVEPATCH · 865 unmodified lines
866KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)866KBUILD_CFLAGS += $(call cc-option, -flive-patching=inline-clone)
867endif867endif
868 868
869ifdef CONFIG_SHADOW_CALL_STACK
870CC_FLAGS_SCS := -fsanitize=shadow-call-stack
871KBUILD_CFLAGS += $(CC_FLAGS_SCS)
872export CC_FLAGS_SCS
873endif
874
869# arch Makefile may override CC so keep this after arch Makefile is included875# arch Makefile may override CC so keep this after arch Makefile is included
870NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)876NOSTDINC_FLAGS += -nostdinc -isystem $(shell $(CC) -print-file-name=include)
871 877
arch/Kconfig+25−0
@@ -533,6 +533,31 @@ config STACKPROTECTOR_STRONG · 532 unmodified lines
533 about 20% of all kernel functions, which increases the kernel code533 about 20% of all kernel functions, which increases the kernel code
534 size by about 2%.534 size by about 2%.
535 535
536config ARCH_SUPPORTS_SHADOW_CALL_STACK
537 bool
538 help
539 An architecture should select this if it supports Clang's Shadow
540 Call Stack and implements runtime support for shadow stack
541 switching.
542
543config SHADOW_CALL_STACK
544 bool "Clang Shadow Call Stack"
545 depends on CC_IS_CLANG && ARCH_SUPPORTS_SHADOW_CALL_STACK
546 depends on DYNAMIC_FTRACE_WITH_REGS || !FUNCTION_GRAPH_TRACER
547 help
548 This option enables Clang's Shadow Call Stack, which uses a
549 shadow stack to protect function return addresses from being
550 overwritten by an attacker. More information can be found in
551 Clang's documentation:
552
553 https://clang.llvm.org/docs/ShadowCallStack.html
554
555 Note that security guarantees in the kernel differ from the
556 ones documented for user space. The kernel must store addresses
557 of shadow stacks in memory, which means an attacker capable of
558 reading and writing arbitrary memory may be able to locate them
559 and hijack control flow by modifying the stacks.
560
536config HAVE_ARCH_WITHIN_STACK_FRAMES561config HAVE_ARCH_WITHIN_STACK_FRAMES
537 bool562 bool
538 help563 help
arch/arm64/Kconfig+5−0
@@ -68,6 +68,7 @@ config ARM64 · 67 unmodified lines
68 select ARCH_USE_QUEUED_SPINLOCKS68 select ARCH_USE_QUEUED_SPINLOCKS
69 select ARCH_USE_SYM_ANNOTATIONS69 select ARCH_USE_SYM_ANNOTATIONS
70 select ARCH_SUPPORTS_MEMORY_FAILURE70 select ARCH_SUPPORTS_MEMORY_FAILURE
71 select ARCH_SUPPORTS_SHADOW_CALL_STACK if CC_HAVE_SHADOW_CALL_STACK
71 select ARCH_SUPPORTS_ATOMIC_RMW72 select ARCH_SUPPORTS_ATOMIC_RMW
72 select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG)73 select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && (GCC_VERSION >= 50000 || CC_IS_CLANG)
73 select ARCH_SUPPORTS_NUMA_BALANCING74 select ARCH_SUPPORTS_NUMA_BALANCING
@@ -1026,6 +1027,10 @@ config ARCH_HAS_CACHE_LINE_SIZE · 952 unmodified lines
1026config ARCH_ENABLE_SPLIT_PMD_PTLOCK1027config ARCH_ENABLE_SPLIT_PMD_PTLOCK
1027 def_bool y if PGTABLE_LEVELS > 21028 def_bool y if PGTABLE_LEVELS > 2
1028 1029
1030# Supported by clang >= 7.0
1031config CC_HAVE_SHADOW_CALL_STACK
1032 def_bool $(cc-option, -fsanitize=shadow-call-stack -ffixed-x18)
1033
1029config SECCOMP1034config SECCOMP
1030 bool "Enable seccomp to safely compute untrusted bytecode"1035 bool "Enable seccomp to safely compute untrusted bytecode"
1031 ---help---1036 ---help---
arch/arm64/Makefile+4−0
@@ -87,6 +87,10 @@ endif · 86 unmodified lines
87 87
88KBUILD_CFLAGS += $(branch-prot-flags-y)88KBUILD_CFLAGS += $(branch-prot-flags-y)
89 89
90ifeq ($(CONFIG_SHADOW_CALL_STACK), y)
91KBUILD_CFLAGS += -ffixed-x18
92endif
93
90ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)94ifeq ($(CONFIG_CPU_BIG_ENDIAN), y)
91KBUILD_CPPFLAGS += -mbig-endian95KBUILD_CPPFLAGS += -mbig-endian
92CHECKFLAGS += -D__AARCH64EB__96CHECKFLAGS += -D__AARCH64EB__
arch/arm64/include/asm/kvm_hyp.h+1−1
@@ -12,7 +12,7 @@ · 11 unmodified lines
12#include <asm/alternative.h>12#include <asm/alternative.h>
13#include <asm/sysreg.h>13#include <asm/sysreg.h>
14 14
15#define __hyp_text __section(.hyp.text) notrace15#define __hyp_text __section(.hyp.text) notrace __noscs
16 16
17#define read_sysreg_elx(r,nvh,vh) \17#define read_sysreg_elx(r,nvh,vh) \
18 ({ \18 ({ \
arch/arm64/include/asm/scs.h+29−0
@@ -0,0 +1,29 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_SCS_H
3#define _ASM_SCS_H
4
5#ifdef __ASSEMBLY__
6
7#include <asm/asm-offsets.h>
8
9#ifdef CONFIG_SHADOW_CALL_STACK
10 scs_sp .req x18
11
12 .macro scs_load tsk, tmp
13 ldr scs_sp, [\tsk, #TSK_TI_SCS_SP]
14 .endm
15
16 .macro scs_save tsk, tmp
17 str scs_sp, [\tsk, #TSK_TI_SCS_SP]
18 .endm
19#else
20 .macro scs_load tsk, tmp
21 .endm
22
23 .macro scs_save tsk, tmp
24 .endm
25#endif /* CONFIG_SHADOW_CALL_STACK */
26
27#endif /* __ASSEMBLY __ */
28
29#endif /* _ASM_SCS_H */
arch/arm64/include/asm/suspend.h+1−1
@@ -2,7 +2,7 @@ · 1 unmodified line
2#ifndef __ASM_SUSPEND_H2#ifndef __ASM_SUSPEND_H
3#define __ASM_SUSPEND_H3#define __ASM_SUSPEND_H
4 4
5#define NR_CTX_REGS 125#define NR_CTX_REGS 13
6#define NR_CALLEE_SAVED_REGS 126#define NR_CALLEE_SAVED_REGS 12
7 7
8/*8/*
arch/arm64/include/asm/thread_info.h+13−0
@@ -41,6 +41,10 @@ struct thread_info { · 40 unmodified lines
41#endif41#endif
42 } preempt;42 } preempt;
43 };43 };
44#ifdef CONFIG_SHADOW_CALL_STACK
45 void *scs_base;
46 void *scs_sp;
47#endif
44};48};
45 49
46#define thread_saved_pc(tsk) \50#define thread_saved_pc(tsk) \
@@ -100,11 +104,20 @@ void arch_release_task_struct(struct task_struct *tsk); · 53 unmodified lines
100 _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \104 _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP | \
101 _TIF_SYSCALL_EMU)105 _TIF_SYSCALL_EMU)
102 106
107#ifdef CONFIG_SHADOW_CALL_STACK
108#define INIT_SCS \
109 .scs_base = init_shadow_call_stack, \
110 .scs_sp = init_shadow_call_stack,
111#else
112#define INIT_SCS
113#endif
114
103#define INIT_THREAD_INFO(tsk) \115#define INIT_THREAD_INFO(tsk) \
104{ \116{ \
105 .flags = _TIF_FOREIGN_FPSTATE, \117 .flags = _TIF_FOREIGN_FPSTATE, \
106 .preempt_count = INIT_PREEMPT_COUNT, \118 .preempt_count = INIT_PREEMPT_COUNT, \
107 .addr_limit = KERNEL_DS, \119 .addr_limit = KERNEL_DS, \
120 INIT_SCS \
108}121}
109 122
110#endif /* __ASM_THREAD_INFO_H */123#endif /* __ASM_THREAD_INFO_H */
arch/arm64/kernel/Makefile+1−0
@@ -63,6 +63,7 @@ obj-$(CONFIG_CRASH_CORE) += crash_core.o · 62 unmodified lines
63obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o63obj-$(CONFIG_ARM_SDE_INTERFACE) += sdei.o
64obj-$(CONFIG_ARM64_SSBD) += ssbd.o64obj-$(CONFIG_ARM64_SSBD) += ssbd.o
65obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o65obj-$(CONFIG_ARM64_PTR_AUTH) += pointer_auth.o
66obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
66 67
67obj-y += vdso/ probes/68obj-y += vdso/ probes/
68obj-$(CONFIG_COMPAT_VDSO) += vdso32/69obj-$(CONFIG_COMPAT_VDSO) += vdso32/
arch/arm64/kernel/asm-offsets.c+4−0
@@ -33,6 +33,10 @@ int main(void) · 32 unmodified lines
33 DEFINE(TSK_TI_ADDR_LIMIT, offsetof(struct task_struct, thread_info.addr_limit));33 DEFINE(TSK_TI_ADDR_LIMIT, offsetof(struct task_struct, thread_info.addr_limit));
34#ifdef CONFIG_ARM64_SW_TTBR0_PAN34#ifdef CONFIG_ARM64_SW_TTBR0_PAN
35 DEFINE(TSK_TI_TTBR0, offsetof(struct task_struct, thread_info.ttbr0));35 DEFINE(TSK_TI_TTBR0, offsetof(struct task_struct, thread_info.ttbr0));
36#endif
37#ifdef CONFIG_SHADOW_CALL_STACK
38 DEFINE(TSK_TI_SCS_BASE, offsetof(struct task_struct, thread_info.scs_base));
39 DEFINE(TSK_TI_SCS_SP, offsetof(struct task_struct, thread_info.scs_sp));
36#endif40#endif
37 DEFINE(TSK_STACK, offsetof(struct task_struct, stack));41 DEFINE(TSK_STACK, offsetof(struct task_struct, stack));
38#ifdef CONFIG_STACKPROTECTOR42#ifdef CONFIG_STACKPROTECTOR
arch/arm64/kernel/efi-rt-wrapper.S+10−1
@@ -34,5 +34,14 @@ SYM_FUNC_START(__efi_rt_asm_wrapper) · 33 unmodified lines
34 ldp x29, x30, [sp], #3234 ldp x29, x30, [sp], #32
35 b.ne 0f35 b.ne 0f
36 ret36 ret
370: b efi_handle_corrupted_x18 // tail call370:
38 /*
39 * With CONFIG_SHADOW_CALL_STACK, the kernel uses x18 to store a
40 * shadow stack pointer, which we need to restore before returning to
41 * potentially instrumented code. This is safe because the wrapper is
42 * called with preemption disabled and a separate shadow stack is used
43 * for interrupts.
44 */
45 mov x18, x2
46 b efi_handle_corrupted_x18 // tail call
38SYM_FUNC_END(__efi_rt_asm_wrapper)47SYM_FUNC_END(__efi_rt_asm_wrapper)
arch/arm64/kernel/entry-ftrace.S+3−2
@@ -23,8 +23,9 @@ · 22 unmodified lines
23 *23 *
24 * ... where <entry> is either ftrace_caller or ftrace_regs_caller.24 * ... where <entry> is either ftrace_caller or ftrace_regs_caller.
25 *25 *
26 * Each instrumented function follows the AAPCS, so here x0-x8 and x19-x30 are26 * Each instrumented function follows the AAPCS, so here x0-x8 and x18-x30 are
27 * live, and x9-x18 are safe to clobber.27 * live (x18 holds the Shadow Call Stack pointer), and x9-x17 are safe to
28 * clobber.
28 *29 *
29 * We save the callsite's context into a pt_regs before invoking any ftrace30 * We save the callsite's context into a pt_regs before invoking any ftrace
30 * callbacks. So that we can get a sensible backtrace, we create a stack record31 * callbacks. So that we can get a sensible backtrace, we create a stack record
arch/arm64/kernel/entry.S+35−3
@@ -23,6 +23,7 @@ · 22 unmodified lines
23#include <asm/mmu.h>23#include <asm/mmu.h>
24#include <asm/processor.h>24#include <asm/processor.h>
25#include <asm/ptrace.h>25#include <asm/ptrace.h>
26#include <asm/scs.h>
26#include <asm/thread_info.h>27#include <asm/thread_info.h>
27#include <asm/asm-uaccess.h>28#include <asm/asm-uaccess.h>
28#include <asm/unistd.h>29#include <asm/unistd.h>
@@ -179,6 +180,8 @@ alternative_cb_end · 150 unmodified lines
179 apply_ssbd 1, x22, x23180 apply_ssbd 1, x22, x23
180 181
181 ptrauth_keys_install_kernel tsk, x20, x22, x23182 ptrauth_keys_install_kernel tsk, x20, x22, x23
183
184 scs_load tsk, x20
182 .else185 .else
183 add x21, sp, #S_FRAME_SIZE186 add x21, sp, #S_FRAME_SIZE
184 get_current_task tsk187 get_current_task tsk
@@ -343,6 +346,8 @@ alternative_else_nop_endif · 158 unmodified lines
343 msr cntkctl_el1, x1346 msr cntkctl_el1, x1
3444:3474:
345#endif348#endif
349 scs_save tsk, x0
350
346 /* No kernel C function calls after this as user keys are set. */351 /* No kernel C function calls after this as user keys are set. */
347 ptrauth_keys_install_user tsk, x0, x1, x2352 ptrauth_keys_install_user tsk, x0, x1, x2
348 353
@@ -388,6 +393,9 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 · 39 unmodified lines
388 393
389 .macro irq_stack_entry394 .macro irq_stack_entry
390 mov x19, sp // preserve the original sp395 mov x19, sp // preserve the original sp
396#ifdef CONFIG_SHADOW_CALL_STACK
397 mov x24, scs_sp // preserve the original shadow stack
398#endif
391 399
392 /*400 /*
393 * Compare sp with the base of the task stack.401 * Compare sp with the base of the task stack.
@@ -405,15 +413,25 @@ alternative_insn eret, nop, ARM64_UNMAP_KERNEL_AT_EL0 · 11 unmodified lines
405 413
406 /* switch to the irq stack */414 /* switch to the irq stack */
407 mov sp, x26415 mov sp, x26
416
417#ifdef CONFIG_SHADOW_CALL_STACK
418 /* also switch to the irq shadow stack */
419 adr_this_cpu scs_sp, irq_shadow_call_stack, x26
420#endif
421
4089998:4229998:
409 .endm423 .endm
410 424
411 /*425 /*
412 * x19 should be preserved between irq_stack_entry and426 * The callee-saved regs (x19-x29) should be preserved between
413 * irq_stack_exit.427 * irq_stack_entry and irq_stack_exit, but note that kernel_entry
428 * uses x20-x23 to store data for later use.
414 */429 */
415 .macro irq_stack_exit430 .macro irq_stack_exit
416 mov sp, x19431 mov sp, x19
432#ifdef CONFIG_SHADOW_CALL_STACK
433 mov scs_sp, x24
434#endif
417 .endm435 .endm
418 436
419/* GPRs used by entry code */437/* GPRs used by entry code */
@@ -902,6 +920,8 @@ SYM_FUNC_START(cpu_switch_to) · 482 unmodified lines
902 mov sp, x9920 mov sp, x9
903 msr sp_el0, x1921 msr sp_el0, x1
904 ptrauth_keys_install_kernel x1, x8, x9, x10922 ptrauth_keys_install_kernel x1, x8, x9, x10
923 scs_save x0, x8
924 scs_load x1, x8
905 ret925 ret
906SYM_FUNC_END(cpu_switch_to)926SYM_FUNC_END(cpu_switch_to)
907NOKPROBE(cpu_switch_to)927NOKPROBE(cpu_switch_to)
@@ -1030,13 +1050,16 @@ SYM_CODE_START(__sdei_asm_handler) · 122 unmodified lines
1030 1050
1031 mov x19, x11051 mov x19, x1
1032 1052
1053#if defined(CONFIG_VMAP_STACK) || defined(CONFIG_SHADOW_CALL_STACK)
1054 ldrb w4, [x19, #SDEI_EVENT_PRIORITY]
1055#endif
1056
1033#ifdef CONFIG_VMAP_STACK1057#ifdef CONFIG_VMAP_STACK
1034 /*1058 /*
1035 * entry.S may have been using sp as a scratch register, find whether1059 * entry.S may have been using sp as a scratch register, find whether
1036 * this is a normal or critical event and switch to the appropriate1060 * this is a normal or critical event and switch to the appropriate
1037 * stack for this CPU.1061 * stack for this CPU.
1038 */1062 */
1039 ldrb w4, [x19, #SDEI_EVENT_PRIORITY]
1040 cbnz w4, 1f1063 cbnz w4, 1f
1041 ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x61064 ldr_this_cpu dst=x5, sym=sdei_stack_normal_ptr, tmp=x6
1042 b 2f1065 b 2f
@@ -1046,6 +1069,15 @@ SYM_CODE_START(__sdei_asm_handler) · 3 unmodified lines
1046 mov sp, x51069 mov sp, x5
1047#endif1070#endif
1048 1071
1072#ifdef CONFIG_SHADOW_CALL_STACK
1073 /* Use a separate shadow call stack for normal and critical events */
1074 cbnz w4, 3f
1075 adr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_normal, tmp=x6
1076 b 4f
10773: adr_this_cpu dst=scs_sp, sym=sdei_shadow_call_stack_critical, tmp=x6
10784:
1079#endif
1080
1049 /*1081 /*
1050 * We may have interrupted userspace, or a guest, or exit-from or1082 * We may have interrupted userspace, or a guest, or exit-from or
1051 * return-to either of these. We can't trust sp_el0, restore it.1083 * return-to either of these. We can't trust sp_el0, restore it.
arch/arm64/kernel/head.S+6−0
@@ -28,6 +28,7 @@ · 27 unmodified lines
28#include <asm/pgtable-hwdef.h>28#include <asm/pgtable-hwdef.h>
29#include <asm/pgtable.h>29#include <asm/pgtable.h>
30#include <asm/page.h>30#include <asm/page.h>
31#include <asm/scs.h>
31#include <asm/smp.h>32#include <asm/smp.h>
32#include <asm/sysreg.h>33#include <asm/sysreg.h>
33#include <asm/thread_info.h>34#include <asm/thread_info.h>
@@ -433,6 +434,10 @@ SYM_FUNC_START_LOCAL(__primary_switched) · 399 unmodified lines
433 stp xzr, x30, [sp, #-16]!434 stp xzr, x30, [sp, #-16]!
434 mov x29, sp435 mov x29, sp
435 436
437#ifdef CONFIG_SHADOW_CALL_STACK
438 adr_l scs_sp, init_shadow_call_stack // Set shadow call stack
439#endif
440
436 str_l x21, __fdt_pointer, x5 // Save FDT pointer441 str_l x21, __fdt_pointer, x5 // Save FDT pointer
437 442
438 ldr_l x4, kimage_vaddr // Save the offset between443 ldr_l x4, kimage_vaddr // Save the offset between
@@ -745,6 +750,7 @@ SYM_FUNC_START_LOCAL(__secondary_switched) · 306 unmodified lines
745 ldr x2, [x0, #CPU_BOOT_TASK]750 ldr x2, [x0, #CPU_BOOT_TASK]
746 cbz x2, __secondary_too_slow751 cbz x2, __secondary_too_slow
747 msr sp_el0, x2752 msr sp_el0, x2
753 scs_load x2, x3
748 mov x29, #0754 mov x29, #0
749 mov x30, #0755 mov x30, #0
750 756
arch/arm64/kernel/scs.c+16−0
@@ -0,0 +1,16 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shadow Call Stack support.
4 *
5 * Copyright (C) 2019 Google LLC
6 */
7
8#include <linux/percpu.h>
9#include <linux/scs.h>
10
11DEFINE_SCS(irq_shadow_call_stack);
12
13#ifdef CONFIG_ARM_SDE_INTERFACE
14DEFINE_SCS(sdei_shadow_call_stack_normal);
15DEFINE_SCS(sdei_shadow_call_stack_critical);
16#endif
arch/arm64/kernel/vdso/Makefile+1−1
@@ -29,7 +29,7 @@ ldflags-y := -shared -nostdlib -soname=linux-vdso.so.1 --hash-style=sysv \ · 28 unmodified lines
29ccflags-y := -fno-common -fno-builtin -fno-stack-protector -ffixed-x1829ccflags-y := -fno-common -fno-builtin -fno-stack-protector -ffixed-x18
30ccflags-y += -DDISABLE_BRANCH_PROFILING30ccflags-y += -DDISABLE_BRANCH_PROFILING
31 31
32CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os32CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS)
33KBUILD_CFLAGS += $(DISABLE_LTO)33KBUILD_CFLAGS += $(DISABLE_LTO)
34KASAN_SANITIZE := n34KASAN_SANITIZE := n
35UBSAN_SANITIZE := n35UBSAN_SANITIZE := n
arch/arm64/mm/proc.S+14−0
@@ -58,6 +58,8 @@ · 57 unmodified lines
58 * cpu_do_suspend - save CPU registers context58 * cpu_do_suspend - save CPU registers context
59 *59 *
60 * x0: virtual address of context pointer60 * x0: virtual address of context pointer
61 *
62 * This must be kept in sync with struct cpu_suspend_ctx in <asm/suspend.h>.
61 */63 */
62SYM_FUNC_START(cpu_do_suspend)64SYM_FUNC_START(cpu_do_suspend)
63 mrs x2, tpidr_el065 mrs x2, tpidr_el0
@@ -82,6 +84,11 @@ alternative_endif · 18 unmodified lines
82 stp x8, x9, [x0, #48]84 stp x8, x9, [x0, #48]
83 stp x10, x11, [x0, #64]85 stp x10, x11, [x0, #64]
84 stp x12, x13, [x0, #80]86 stp x12, x13, [x0, #80]
87 /*
88 * Save x18 as it may be used as a platform register, e.g. by shadow
89 * call stack.
90 */
91 str x18, [x0, #96]
85 ret92 ret
86SYM_FUNC_END(cpu_do_suspend)93SYM_FUNC_END(cpu_do_suspend)
87 94
@@ -98,6 +105,13 @@ SYM_FUNC_START(cpu_do_resume) · 10 unmodified lines
98 ldp x9, x10, [x0, #48]105 ldp x9, x10, [x0, #48]
99 ldp x11, x12, [x0, #64]106 ldp x11, x12, [x0, #64]
100 ldp x13, x14, [x0, #80]107 ldp x13, x14, [x0, #80]
108 /*
109 * Restore x18, as it may be used as a platform register, and clear
110 * the buffer to minimize the risk of exposure when used for shadow
111 * call stack.
112 */
113 ldr x18, [x0, #96]
114 str xzr, [x0, #96]
101 msr tpidr_el0, x2115 msr tpidr_el0, x2
102 msr tpidrro_el0, x3116 msr tpidrro_el0, x3
103 msr contextidr_el1, x4117 msr contextidr_el1, x4
drivers/base/node.c+6−0
@@ -415,6 +415,9 @@ static ssize_t node_read_meminfo(struct device *dev, · 414 unmodified lines
415 "Node %d AnonPages: %8lu kB\n"415 "Node %d AnonPages: %8lu kB\n"
416 "Node %d Shmem: %8lu kB\n"416 "Node %d Shmem: %8lu kB\n"
417 "Node %d KernelStack: %8lu kB\n"417 "Node %d KernelStack: %8lu kB\n"
418#ifdef CONFIG_SHADOW_CALL_STACK
419 "Node %d ShadowCallStack:%8lu kB\n"
420#endif
418 "Node %d PageTables: %8lu kB\n"421 "Node %d PageTables: %8lu kB\n"
419 "Node %d NFS_Unstable: %8lu kB\n"422 "Node %d NFS_Unstable: %8lu kB\n"
420 "Node %d Bounce: %8lu kB\n"423 "Node %d Bounce: %8lu kB\n"
@@ -438,6 +441,9 @@ static ssize_t node_read_meminfo(struct device *dev, · 17 unmodified lines
438 nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),441 nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
439 nid, K(i.sharedram),442 nid, K(i.sharedram),
440 nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),443 nid, sum_zone_node_page_state(nid, NR_KERNEL_STACK_KB),
444#ifdef CONFIG_SHADOW_CALL_STACK
445 nid, sum_zone_node_page_state(nid, NR_KERNEL_SCS_KB),
446#endif
441 nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),447 nid, K(sum_zone_node_page_state(nid, NR_PAGETABLE)),
442 nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)),448 nid, K(node_page_state(pgdat, NR_UNSTABLE_NFS)),
443 nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),449 nid, K(sum_zone_node_page_state(nid, NR_BOUNCE)),
drivers/firmware/efi/libstub/Makefile+3−0
@@ -32,6 +32,9 @@ KBUILD_CFLAGS := $(cflags-y) -DDISABLE_BRANCH_PROFILING \ · 31 unmodified lines
32 $(call cc-option,-fno-stack-protector) \32 $(call cc-option,-fno-stack-protector) \
33 -D__DISABLE_EXPORTS33 -D__DISABLE_EXPORTS
34 34
35# remove SCS flags from all objects in this directory
36KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS))
37
35GCOV_PROFILE := n38GCOV_PROFILE := n
36KASAN_SANITIZE := n39KASAN_SANITIZE := n
37UBSAN_SANITIZE := n40UBSAN_SANITIZE := n
fs/proc/meminfo.c+4−0
@@ -103,6 +103,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v) · 102 unmodified lines
103 show_val_kb(m, "SUnreclaim: ", sunreclaim);103 show_val_kb(m, "SUnreclaim: ", sunreclaim);
104 seq_printf(m, "KernelStack: %8lu kB\n",104 seq_printf(m, "KernelStack: %8lu kB\n",
105 global_zone_page_state(NR_KERNEL_STACK_KB));105 global_zone_page_state(NR_KERNEL_STACK_KB));
106#ifdef CONFIG_SHADOW_CALL_STACK
107 seq_printf(m, "ShadowCallStack:%8lu kB\n",
108 global_zone_page_state(NR_KERNEL_SCS_KB));
109#endif
106 show_val_kb(m, "PageTables: ",110 show_val_kb(m, "PageTables: ",
107 global_zone_page_state(NR_PAGETABLE));111 global_zone_page_state(NR_PAGETABLE));
108 112
include/linux/compiler-clang.h+4−0
@@ -42,3 +42,7 @@ · 41 unmodified lines
42 * compilers, like ICC.42 * compilers, like ICC.
43 */43 */
44#define barrier() __asm__ __volatile__("" : : : "memory")44#define barrier() __asm__ __volatile__("" : : : "memory")
45
46#if __has_feature(shadow_call_stack)
47# define __noscs __attribute__((__no_sanitize__("shadow-call-stack")))
48#endif
include/linux/compiler_types.h+4−0
@@ -193,6 +193,10 @@ struct ftrace_likely_data { · 192 unmodified lines
193# define randomized_struct_fields_end193# define randomized_struct_fields_end
194#endif194#endif
195 195
196#ifndef __noscs
197# define __noscs
198#endif
199
196#ifndef asm_volatile_goto200#ifndef asm_volatile_goto
197#define asm_volatile_goto(x...) asm goto(x)201#define asm_volatile_goto(x...) asm goto(x)
198#endif202#endif
include/linux/mmzone.h+3−0
@@ -156,6 +156,9 @@ enum zone_stat_item { · 155 unmodified lines
156 NR_MLOCK, /* mlock()ed pages found and moved off LRU */156 NR_MLOCK, /* mlock()ed pages found and moved off LRU */
157 NR_PAGETABLE, /* used for pagetables */157 NR_PAGETABLE, /* used for pagetables */
158 NR_KERNEL_STACK_KB, /* measured in KiB */158 NR_KERNEL_STACK_KB, /* measured in KiB */
159#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
160 NR_KERNEL_SCS_KB, /* measured in KiB */
161#endif
159 /* Second 128 byte cacheline */162 /* Second 128 byte cacheline */
160 NR_BOUNCE,163 NR_BOUNCE,
161#if IS_ENABLED(CONFIG_ZSMALLOC)164#if IS_ENABLED(CONFIG_ZSMALLOC)
include/linux/scs.h+72−0
@@ -0,0 +1,72 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Shadow Call Stack support.
4 *
5 * Copyright (C) 2019 Google LLC
6 */
7
8#ifndef _LINUX_SCS_H
9#define _LINUX_SCS_H
10
11#include <linux/gfp.h>
12#include <linux/poison.h>
13#include <linux/sched.h>
14#include <linux/sizes.h>
15
16#ifdef CONFIG_SHADOW_CALL_STACK
17
18/*
19 * In testing, 1 KiB shadow stack size (i.e. 128 stack frames on a 64-bit
20 * architecture) provided ~40% safety margin on stack usage while keeping
21 * memory allocation overhead reasonable.
22 */
23#define SCS_SIZE SZ_1K
24#define GFP_SCS (GFP_KERNEL | __GFP_ZERO)
25
26/* An illegal pointer value to mark the end of the shadow stack. */
27#define SCS_END_MAGIC (0x5f6UL + POISON_POINTER_DELTA)
28
29/* Allocate a static per-CPU shadow stack */
30#define DEFINE_SCS(name) \
31 DEFINE_PER_CPU(unsigned long [SCS_SIZE/sizeof(long)], name) \
32
33#define task_scs(tsk) (task_thread_info(tsk)->scs_base)
34#define task_scs_sp(tsk) (task_thread_info(tsk)->scs_sp)
35
36void scs_init(void);
37int scs_prepare(struct task_struct *tsk, int node);
38void scs_release(struct task_struct *tsk);
39
40static inline void scs_task_reset(struct task_struct *tsk)
41{
42 /*
43 * Reset the shadow stack to the base address in case the task
44 * is reused.
45 */
46 task_scs_sp(tsk) = task_scs(tsk);
47}
48
49static inline unsigned long *__scs_magic(void *s)
50{
51 return (unsigned long *)(s + SCS_SIZE) - 1;
52}
53
54static inline bool task_scs_end_corrupted(struct task_struct *tsk)
55{
56 unsigned long *magic = __scs_magic(task_scs(tsk));
57 unsigned long sz = task_scs_sp(tsk) - task_scs(tsk);
58
59 return sz >= SCS_SIZE - 1 || READ_ONCE_NOCHECK(*magic) != SCS_END_MAGIC;
60}
61
62#else /* CONFIG_SHADOW_CALL_STACK */
63
64static inline void scs_init(void) {}
65static inline void scs_task_reset(struct task_struct *tsk) {}
66static inline int scs_prepare(struct task_struct *tsk, int node) { return 0; }
67static inline void scs_release(struct task_struct *tsk) {}
68static inline bool task_scs_end_corrupted(struct task_struct *tsk) { return false; }
69
70#endif /* CONFIG_SHADOW_CALL_STACK */
71
72#endif /* _LINUX_SCS_H */
init/init_task.c+8−0
@@ -11,6 +11,7 @@ · 10 unmodified lines
11#include <linux/mm.h>11#include <linux/mm.h>
12#include <linux/audit.h>12#include <linux/audit.h>
13#include <linux/numa.h>13#include <linux/numa.h>
14#include <linux/scs.h>
14 15
15#include <asm/pgtable.h>16#include <asm/pgtable.h>
16#include <linux/uaccess.h>17#include <linux/uaccess.h>
@@ -50,6 +51,13 @@ static struct sighand_struct init_sighand = { · 33 unmodified lines
50 .signalfd_wqh = __WAIT_QUEUE_HEAD_INITIALIZER(init_sighand.signalfd_wqh),51 .signalfd_wqh = __WAIT_QUEUE_HEAD_INITIALIZER(init_sighand.signalfd_wqh),
51};52};
52 53
54#ifdef CONFIG_SHADOW_CALL_STACK
55unsigned long init_shadow_call_stack[SCS_SIZE / sizeof(long)]
56 __init_task_data = {
57 [(SCS_SIZE / sizeof(long)) - 1] = SCS_END_MAGIC
58};
59#endif
60
53/*61/*
54 * Set up the first task table, touch at your own risk!. Base=0,62 * Set up the first task table, touch at your own risk!. Base=0,
55 * limit=0x1fffff (=2MB)63 * limit=0x1fffff (=2MB)
kernel/Makefile+1−0
@@ -103,6 +103,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/ · 102 unmodified lines
103obj-$(CONFIG_IRQ_WORK) += irq_work.o103obj-$(CONFIG_IRQ_WORK) += irq_work.o
104obj-$(CONFIG_CPU_PM) += cpu_pm.o104obj-$(CONFIG_CPU_PM) += cpu_pm.o
105obj-$(CONFIG_BPF) += bpf/105obj-$(CONFIG_BPF) += bpf/
106obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o
106 107
107obj-$(CONFIG_PERF_EVENTS) += events/108obj-$(CONFIG_PERF_EVENTS) += events/
108 109
kernel/fork.c+9−0
@@ -94,6 +94,7 @@ · 93 unmodified lines
94#include <linux/thread_info.h>94#include <linux/thread_info.h>
95#include <linux/stackleak.h>95#include <linux/stackleak.h>
96#include <linux/kasan.h>96#include <linux/kasan.h>
97#include <linux/scs.h>
97 98
98#include <asm/pgtable.h>99#include <asm/pgtable.h>
99#include <asm/pgalloc.h>100#include <asm/pgalloc.h>
@@ -456,6 +457,8 @@ void put_task_stack(struct task_struct *tsk) · 356 unmodified lines
456 457
457void free_task(struct task_struct *tsk)458void free_task(struct task_struct *tsk)
458{459{
460 scs_release(tsk);
461
459#ifndef CONFIG_THREAD_INFO_IN_TASK462#ifndef CONFIG_THREAD_INFO_IN_TASK
460 /*463 /*
461 * The task is finally done with both the stack and thread_info,464 * The task is finally done with both the stack and thread_info,
@@ -840,6 +843,8 @@ void __init fork_init(void) · 378 unmodified lines
840 NULL, free_vm_stack_cache);843 NULL, free_vm_stack_cache);
841#endif844#endif
842 845
846 scs_init();
847
843 lockdep_init_task(&init_task);848 lockdep_init_task(&init_task);
844 uprobes_init();849 uprobes_init();
845}850}
@@ -899,6 +904,10 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node) · 53 unmodified lines
899 if (err)904 if (err)
900 goto free_stack;905 goto free_stack;
901 906
907 err = scs_prepare(tsk, node);
908 if (err)
909 goto free_stack;
910
902#ifdef CONFIG_SECCOMP911#ifdef CONFIG_SECCOMP
903 /*912 /*
904 * We must handle setting up seccomp filters once we're under913 * We must handle setting up seccomp filters once we're under
kernel/sched/core.c+5−0
@@ -11,6 +11,7 @@ · 10 unmodified lines
11#include <linux/nospec.h>11#include <linux/nospec.h>
12 12
13#include <linux/kcov.h>13#include <linux/kcov.h>
14#include <linux/scs.h>
14 15
15#include <asm/switch_to.h>16#include <asm/switch_to.h>
16#include <asm/tlb.h>17#include <asm/tlb.h>
@@ -3877,6 +3878,9 @@ static inline void schedule_debug(struct task_struct *prev, bool preempt) · 3860 unmodified lines
3877#ifdef CONFIG_SCHED_STACK_END_CHECK3878#ifdef CONFIG_SCHED_STACK_END_CHECK
3878 if (task_stack_end_corrupted(prev))3879 if (task_stack_end_corrupted(prev))
3879 panic("corrupted stack end detected inside scheduler\n");3880 panic("corrupted stack end detected inside scheduler\n");
3881
3882 if (task_scs_end_corrupted(prev))
3883 panic("corrupted shadow stack detected inside scheduler\n");
3880#endif3884#endif
3881 3885
3882#ifdef CONFIG_DEBUG_ATOMIC_SLEEP3886#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
@@ -6040,6 +6044,7 @@ void init_idle(struct task_struct *idle, int cpu) · 2157 unmodified lines
6040 idle->se.exec_start = sched_clock();6044 idle->se.exec_start = sched_clock();
6041 idle->flags |= PF_IDLE;6045 idle->flags |= PF_IDLE;
6042 6046
6047 scs_task_reset(idle);
6043 kasan_unpoison_task_stack(idle);6048 kasan_unpoison_task_stack(idle);
6044 6049
6045#ifdef CONFIG_SMP6050#ifdef CONFIG_SMP
kernel/scs.c+104−0
@@ -0,0 +1,104 @@
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Shadow Call Stack support.
4 *
5 * Copyright (C) 2019 Google LLC
6 */
7
8#include <linux/kasan.h>
9#include <linux/mm.h>
10#include <linux/scs.h>
11#include <linux/slab.h>
12#include <linux/vmstat.h>
13
14static struct kmem_cache *scs_cache;
15
16static void __scs_account(void *s, int account)
17{
18 struct page *scs_page = virt_to_page(s);
19
20 mod_zone_page_state(page_zone(scs_page), NR_KERNEL_SCS_KB,
21 account * (SCS_SIZE / SZ_1K));
22}
23
24static void *scs_alloc(int node)
25{
26 void *s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node);
27
28 if (!s)
29 return NULL;
30
31 *__scs_magic(s) = SCS_END_MAGIC;
32
33 /*
34 * Poison the allocation to catch unintentional accesses to
35 * the shadow stack when KASAN is enabled.
36 */
37 kasan_poison_object_data(scs_cache, s);
38 __scs_account(s, 1);
39 return s;
40}
41
42static void scs_free(void *s)
43{
44 __scs_account(s, -1);
45 kasan_unpoison_object_data(scs_cache, s);
46 kmem_cache_free(scs_cache, s);
47}
48
49void __init scs_init(void)
50{
51 scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL);
52}
53
54int scs_prepare(struct task_struct *tsk, int node)
55{
56 void *s = scs_alloc(node);
57
58 if (!s)
59 return -ENOMEM;
60
61 task_scs(tsk) = task_scs_sp(tsk) = s;
62 return 0;
63}
64
65static void scs_check_usage(struct task_struct *tsk)
66{
67 static unsigned long highest;
68
69 unsigned long *p, prev, curr = highest, used = 0;
70
71 if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE))
72 return;
73
74 for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) {
75 if (!READ_ONCE_NOCHECK(*p))
76 break;
77 used++;
78 }
79
80 while (used > curr) {
81 prev = cmpxchg_relaxed(&highest, curr, used);
82
83 if (prev == curr) {
84 pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
85 tsk->comm, task_pid_nr(tsk), used);
86 break;
87 }
88
89 curr = prev;
90 }
91}
92
93void scs_release(struct task_struct *tsk)
94{
95 void *s = task_scs(tsk);
96
97 if (!s)
98 return;
99
100 WARN(task_scs_end_corrupted(tsk),
101 "corrupted shadow stack detected when freeing task\n");
102 scs_check_usage(tsk);
103 scs_free(s);
104}
mm/page_alloc.c+6−0
@@ -5411,6 +5411,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) · 5410 unmodified lines
5411 " managed:%lukB"5411 " managed:%lukB"
5412 " mlocked:%lukB"5412 " mlocked:%lukB"
5413 " kernel_stack:%lukB"5413 " kernel_stack:%lukB"
5414#ifdef CONFIG_SHADOW_CALL_STACK
5415 " shadow_call_stack:%lukB"
5416#endif
5414 " pagetables:%lukB"5417 " pagetables:%lukB"
5415 " bounce:%lukB"5418 " bounce:%lukB"
5416 " free_pcp:%lukB"5419 " free_pcp:%lukB"
@@ -5433,6 +5436,9 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask) · 16 unmodified lines
5433 K(zone_managed_pages(zone)),5436 K(zone_managed_pages(zone)),
5434 K(zone_page_state(zone, NR_MLOCK)),5437 K(zone_page_state(zone, NR_MLOCK)),
5435 zone_page_state(zone, NR_KERNEL_STACK_KB),5438 zone_page_state(zone, NR_KERNEL_STACK_KB),
5439#ifdef CONFIG_SHADOW_CALL_STACK
5440 zone_page_state(zone, NR_KERNEL_SCS_KB),
5441#endif
5436 K(zone_page_state(zone, NR_PAGETABLE)),5442 K(zone_page_state(zone, NR_PAGETABLE)),
5437 K(zone_page_state(zone, NR_BOUNCE)),5443 K(zone_page_state(zone, NR_BOUNCE)),
5438 K(free_pcp),5444 K(free_pcp),
mm/vmstat.c+3−0
@@ -1119,6 +1119,9 @@ const char * const vmstat_text[] = { · 1118 unmodified lines
1119 "nr_mlock",1119 "nr_mlock",
1120 "nr_page_table_pages",1120 "nr_page_table_pages",
1121 "nr_kernel_stack",1121 "nr_kernel_stack",
1122#if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
1123 "nr_shadow_call_stack",
1124#endif
1122 "nr_bounce",1125 "nr_bounce",
1123#if IS_ENABLED(CONFIG_ZSMALLOC)1126#if IS_ENABLED(CONFIG_ZSMALLOC)
1124 "nr_zspages",1127 "nr_zspages",

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t