The kernel currently uses kmem_cache to allocate shadow call stacks, which means an overflows may not be immediately detected and can potentially result in another task's shadow stack to be overwritten. This change switches SCS to use virtually mapped shadow stacks for tasks, which increases shadow stack size to a full page and provides more robust overflow detection, similarly to VMAP_STACK.
Showing the changes to kernel (1 of 2 files). Show all
Unified · Split
| @@ -5,26 +5,49 @@ · 4 unmodified lines | ||
| 5 | 5 | * Copyright (C) 2019 Google LLC |
| 6 | 6 | */ |
| 7 | 7 | |
| 8 | +#include <linux/cpuhotplug.h> | |
| 8 | 9 | #include <linux/kasan.h> |
| 9 | 10 | #include <linux/mm.h> |
| 10 | 11 | #include <linux/scs.h> |
| 11 | −#include <linux/slab.h> | |
| 12 | +#include <linux/vmalloc.h> | |
| 12 | 13 | #include <linux/vmstat.h> |
| 13 | 14 | |
| 14 | −static struct kmem_cache *scs_cache; | |
| 15 | − | |
| 16 | 15 | static void __scs_account(void *s, int account) |
| 17 | 16 | { |
| 18 | − struct page *scs_page = virt_to_page(s); | |
| 17 | + struct page *scs_page = vmalloc_to_page(s); | |
| 19 | 18 | |
| 20 | 19 | mod_node_page_state(page_pgdat(scs_page), NR_KERNEL_SCS_KB, |
| 21 | 20 | account * (SCS_SIZE / SZ_1K)); |
| 22 | 21 | } |
| 23 | 22 | |
| 24 | −static void *scs_alloc(int node) | |
| 23 | +/* Matches NR_CACHED_STACKS for VMAP_STACK */ | |
| 24 | +#define NR_CACHED_SCS 2 | |
| 25 | +static DEFINE_PER_CPU(void *, scs_cache[NR_CACHED_SCS]); | |
| 26 | + | |
| 27 | +static void *__scs_alloc(int node) | |
| 25 | 28 | { |
| 26 | − void *s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node); | |
| 29 | + int i; | |
| 30 | + void *s; | |
| 31 | + | |
| 32 | + for (i = 0; i < NR_CACHED_SCS; i++) { | |
| 33 | + s = this_cpu_xchg(scs_cache[i], NULL); | |
| 34 | + if (s) { | |
| 35 | + kasan_unpoison_vmalloc(s, SCS_SIZE); | |
| 36 | + memset(s, 0, SCS_SIZE); | |
| 37 | + return s; | |
| 38 | + } | |
| 39 | + } | |
| 40 | + | |
| 41 | + return __vmalloc_node_range(SCS_SIZE, 1, VMALLOC_START, VMALLOC_END, | |
| 42 | + GFP_SCS, PAGE_KERNEL, 0, node, | |
| 43 | + __builtin_return_address(0)); | |
| 44 | +} | |
| 27 | 45 | |
| 46 | +void *scs_alloc(int node) | |
| 47 | +{ | |
| 48 | + void *s; | |
| 49 | + | |
| 50 | + s = __scs_alloc(node); | |
| 28 | 51 | if (!s) |
| 29 | 52 | return NULL; |
| 30 | 53 | |
| @@ -34,21 +57,47 @@ static void *scs_alloc(int node) · 3 unmodified lines | ||
| 34 | 57 | * Poison the allocation to catch unintentional accesses to |
| 35 | 58 | * the shadow stack when KASAN is enabled. |
| 36 | 59 | */ |
| 37 | − kasan_poison_object_data(scs_cache, s); | |
| 60 | + kasan_poison_vmalloc(s, SCS_SIZE); | |
| 38 | 61 | __scs_account(s, 1); |
| 39 | 62 | return s; |
| 40 | 63 | } |
| 41 | 64 | |
| 42 | −static void scs_free(void *s) | |
| 65 | +void scs_free(void *s) | |
| 43 | 66 | { |
| 67 | + int i; | |
| 68 | + | |
| 44 | 69 | __scs_account(s, -1); |
| 45 | − kasan_unpoison_object_data(scs_cache, s); | |
| 46 | − kmem_cache_free(scs_cache, s); | |
| 70 | + | |
| 71 | + /* | |
| 72 | + * We cannot sleep as this can be called in interrupt context, | |
| 73 | + * so use this_cpu_cmpxchg to update the cache, and vfree_atomic | |
| 74 | + * to free the stack. | |
| 75 | + */ | |
| 76 | + | |
| 77 | + for (i = 0; i < NR_CACHED_SCS; i++) | |
| 78 | + if (this_cpu_cmpxchg(scs_cache[i], 0, s) == NULL) | |
| 79 | + return; | |
| 80 | + | |
| 81 | + vfree_atomic(s); | |
| 82 | +} | |
| 83 | + | |
| 84 | +static int scs_cleanup(unsigned int cpu) | |
| 85 | +{ | |
| 86 | + int i; | |
| 87 | + void **cache = per_cpu_ptr(scs_cache, cpu); | |
| 88 | + | |
| 89 | + for (i = 0; i < NR_CACHED_SCS; i++) { | |
| 90 | + vfree(cache[i]); | |
| 91 | + cache[i] = NULL; | |
| 92 | + } | |
| 93 | + | |
| 94 | + return 0; | |
| 47 | 95 | } |
| 48 | 96 | |
| 49 | 97 | void __init scs_init(void) |
| 50 | 98 | { |
| 51 | − scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL); | |
| 99 | + cpuhp_setup_state(CPUHP_BP_PREPARE_DYN, "scs:scs_cache", NULL, | |
| 100 | + scs_cleanup); | |
| 52 | 101 | } |
| 53 | 102 | |
| 54 | 103 | int scs_prepare(struct task_struct *tsk, int node) |