History

kcsan: Add Kernel Concurrency Sanitizer infrastructure

Kernel Concurrency Sanitizer (KCSAN) is a dynamic data-race detector for kernel space. KCSAN is a sampling watchpoint-based data-race detector. See the included Documentation/dev-tools/kcsan.rst for more details. This patch adds basic infrastructure, but does not yet enable KCSAN for any architecture.

Marco Elver committed 7 years ago

Commit dfd402a1 parent 31f4f5b24 files · +2006 −9Browse files

Unified · Split

24 files
  1. MAINTAINERS +11 −0
  2. Makefile +2 −1
  3. include/linux/compiler-clang.h +9 −0
  4. include/linux/compiler-gcc.h +7 −0
  5. include/linux/compiler.h +29 −8
  6. include/linux/kcsan-checks.h +97 −0
  7. include/linux/kcsan.h +115 −0
  8. include/linux/sched.h +4 −0
  9. init/init_task.c +8 −0
  10. init/main.c +2 −0
  11. kernel/Makefile +1 −0
  12. kernel/kcsan/Makefile +11 −0
  13. kernel/kcsan/atomic.h +27 −0
  14. kernel/kcsan/core.c +626 −0
  15. kernel/kcsan/debugfs.c +275 −0
  16. kernel/kcsan/encoding.h +94 −0
  17. kernel/kcsan/kcsan.h +108 −0
  18. kernel/kcsan/report.c +320 −0
  19. kernel/kcsan/test.c +121 −0
  20. lib/Kconfig.debug +2 −0
  21. lib/Kconfig.kcsan +118 −0
  22. lib/Makefile +3 −0
  23. scripts/Makefile.kcsan +6 −0
  24. scripts/Makefile.lib +10 −0
MAINTAINERS+11−0
@@ -8848,6 +8848,17 @@ F: Documentation/kbuild/kconfig* · 8847 unmodified lines
8848F: scripts/kconfig/8848F: scripts/kconfig/
8849F: scripts/Kconfig.include8849F: scripts/Kconfig.include
8850 8850
8851KCSAN
8852M: Marco Elver <elver@google.com>
8853R: Dmitry Vyukov <dvyukov@google.com>
8854L: kasan-dev@googlegroups.com
8855S: Maintained
8856F: Documentation/dev-tools/kcsan.rst
8857F: include/linux/kcsan*.h
8858F: kernel/kcsan/
8859F: lib/Kconfig.kcsan
8860F: scripts/Makefile.kcsan
8861
8851KDUMP8862KDUMP
8852M: Dave Young <dyoung@redhat.com>8863M: Dave Young <dyoung@redhat.com>
8853M: Baoquan He <bhe@redhat.com>8864M: Baoquan He <bhe@redhat.com>
Makefile+2−1
@@ -478,7 +478,7 @@ export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE · 477 unmodified lines
478 478
479export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS479export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
480export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE480export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE
481export CFLAGS_KASAN CFLAGS_KASAN_NOSANITIZE CFLAGS_UBSAN481export CFLAGS_KASAN CFLAGS_KASAN_NOSANITIZE CFLAGS_UBSAN CFLAGS_KCSAN
482export KBUILD_AFLAGS AFLAGS_KERNEL AFLAGS_MODULE482export KBUILD_AFLAGS AFLAGS_KERNEL AFLAGS_MODULE
483export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_LDFLAGS_MODULE483export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_LDFLAGS_MODULE
484export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL484export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL
@@ -900,6 +900,7 @@ endif · 415 unmodified lines
900include scripts/Makefile.kasan900include scripts/Makefile.kasan
901include scripts/Makefile.extrawarn901include scripts/Makefile.extrawarn
902include scripts/Makefile.ubsan902include scripts/Makefile.ubsan
903include scripts/Makefile.kcsan
903 904
904# Add user supplied CPPFLAGS, AFLAGS and CFLAGS as the last assignments905# Add user supplied CPPFLAGS, AFLAGS and CFLAGS as the last assignments
905KBUILD_CPPFLAGS += $(KCPPFLAGS)906KBUILD_CPPFLAGS += $(KCPPFLAGS)
include/linux/compiler-clang.h+9−0
@@ -24,6 +24,15 @@ · 23 unmodified lines
24#define __no_sanitize_address24#define __no_sanitize_address
25#endif25#endif
26 26
27#if __has_feature(thread_sanitizer)
28/* emulate gcc's __SANITIZE_THREAD__ flag */
29#define __SANITIZE_THREAD__
30#define __no_sanitize_thread \
31 __attribute__((no_sanitize("thread")))
32#else
33#define __no_sanitize_thread
34#endif
35
27/*36/*
28 * Not all versions of clang implement the the type-generic versions37 * Not all versions of clang implement the the type-generic versions
29 * of the builtin overflow checkers. Fortunately, clang implements38 * of the builtin overflow checkers. Fortunately, clang implements
include/linux/compiler-gcc.h+7−0
@@ -145,6 +145,13 @@ · 144 unmodified lines
145#define __no_sanitize_address145#define __no_sanitize_address
146#endif146#endif
147 147
148#if defined(__SANITIZE_THREAD__) && __has_attribute(__no_sanitize_thread__)
149#define __no_sanitize_thread \
150 __attribute__((__noinline__)) __attribute__((no_sanitize_thread))
151#else
152#define __no_sanitize_thread
153#endif
154
148#if GCC_VERSION >= 50100155#if GCC_VERSION >= 50100
149#define COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW 1156#define COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW 1
150#endif157#endif
include/linux/compiler.h+29−8
@@ -178,6 +178,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 177 unmodified lines
178#endif178#endif
179 179
180#include <uapi/linux/types.h>180#include <uapi/linux/types.h>
181#include <linux/kcsan-checks.h>
181 182
182#define __READ_ONCE_SIZE \183#define __READ_ONCE_SIZE \
183({ \184({ \
@@ -193,12 +194,6 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 9 unmodified lines
193 } \194 } \
194})195})
195 196
196static __always_inline
197void __read_once_size(const volatile void *p, void *res, int size)
198{
199 __READ_ONCE_SIZE;
200}
201
202#ifdef CONFIG_KASAN197#ifdef CONFIG_KASAN
203/*198/*
204 * We can't declare function 'inline' because __no_sanitize_address confilcts199 * We can't declare function 'inline' because __no_sanitize_address confilcts
@@ -207,18 +202,44 @@ void __read_once_size(const volatile void *p, void *res, int size) · 2 unmodified lines
207 * '__maybe_unused' allows us to avoid defined-but-not-used warnings.202 * '__maybe_unused' allows us to avoid defined-but-not-used warnings.
208 */203 */
209# define __no_kasan_or_inline __no_sanitize_address notrace __maybe_unused204# define __no_kasan_or_inline __no_sanitize_address notrace __maybe_unused
205# define __no_sanitize_or_inline __no_kasan_or_inline
210#else206#else
211# define __no_kasan_or_inline __always_inline207# define __no_kasan_or_inline __always_inline
212#endif208#endif
213 209
214static __no_kasan_or_inline210#ifdef __SANITIZE_THREAD__
211/*
212 * Rely on __SANITIZE_THREAD__ instead of CONFIG_KCSAN, to avoid not inlining in
213 * compilation units where instrumentation is disabled.
214 */
215# define __no_kcsan_or_inline __no_sanitize_thread notrace __maybe_unused
216# define __no_sanitize_or_inline __no_kcsan_or_inline
217#else
218# define __no_kcsan_or_inline __always_inline
219#endif
220
221#ifndef __no_sanitize_or_inline
222#define __no_sanitize_or_inline __always_inline
223#endif
224
225static __no_kcsan_or_inline
226void __read_once_size(const volatile void *p, void *res, int size)
227{
228 kcsan_check_atomic_read(p, size);
229 __READ_ONCE_SIZE;
230}
231
232static __no_sanitize_or_inline
215void __read_once_size_nocheck(const volatile void *p, void *res, int size)233void __read_once_size_nocheck(const volatile void *p, void *res, int size)
216{234{
217 __READ_ONCE_SIZE;235 __READ_ONCE_SIZE;
218}236}
219 237
220static __always_inline void __write_once_size(volatile void *p, void *res, int size)238static __no_kcsan_or_inline
239void __write_once_size(volatile void *p, void *res, int size)
221{240{
241 kcsan_check_atomic_write(p, size);
242
222 switch (size) {243 switch (size) {
223 case 1: *(volatile __u8 *)p = *(__u8 *)res; break;244 case 1: *(volatile __u8 *)p = *(__u8 *)res; break;
224 case 2: *(volatile __u16 *)p = *(__u16 *)res; break;245 case 2: *(volatile __u16 *)p = *(__u16 *)res; break;
include/linux/kcsan-checks.h+97−0
@@ -0,0 +1,97 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2
3#ifndef _LINUX_KCSAN_CHECKS_H
4#define _LINUX_KCSAN_CHECKS_H
5
6#include <linux/types.h>
7
8/*
9 * Access type modifiers.
10 */
11#define KCSAN_ACCESS_WRITE 0x1
12#define KCSAN_ACCESS_ATOMIC 0x2
13
14/*
15 * __kcsan_*: Always calls into runtime when KCSAN is enabled. This may be used
16 * even in compilation units that selectively disable KCSAN, but must use KCSAN
17 * to validate access to an address. Never use these in header files!
18 */
19#ifdef CONFIG_KCSAN
20/**
21 * __kcsan_check_access - check generic access for data race
22 *
23 * @ptr address of access
24 * @size size of access
25 * @type access type modifier
26 */
27void __kcsan_check_access(const volatile void *ptr, size_t size, int type);
28
29#else
30static inline void __kcsan_check_access(const volatile void *ptr, size_t size,
31 int type) { }
32#endif
33
34/*
35 * kcsan_*: Only calls into runtime when the particular compilation unit has
36 * KCSAN instrumentation enabled. May be used in header files.
37 */
38#ifdef __SANITIZE_THREAD__
39#define kcsan_check_access __kcsan_check_access
40#else
41static inline void kcsan_check_access(const volatile void *ptr, size_t size,
42 int type) { }
43#endif
44
45/**
46 * __kcsan_check_read - check regular read access for data races
47 *
48 * @ptr address of access
49 * @size size of access
50 */
51#define __kcsan_check_read(ptr, size) __kcsan_check_access(ptr, size, 0)
52
53/**
54 * __kcsan_check_write - check regular write access for data races
55 *
56 * @ptr address of access
57 * @size size of access
58 */
59#define __kcsan_check_write(ptr, size) \
60 __kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
61
62/**
63 * kcsan_check_read - check regular read access for data races
64 *
65 * @ptr address of access
66 * @size size of access
67 */
68#define kcsan_check_read(ptr, size) kcsan_check_access(ptr, size, 0)
69
70/**
71 * kcsan_check_write - check regular write access for data races
72 *
73 * @ptr address of access
74 * @size size of access
75 */
76#define kcsan_check_write(ptr, size) \
77 kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
78
79/*
80 * Check for atomic accesses: if atomic access are not ignored, this simply
81 * aliases to kcsan_check_access, otherwise becomes a no-op.
82 */
83#ifdef CONFIG_KCSAN_IGNORE_ATOMICS
84#define kcsan_check_atomic_read(...) \
85 do { \
86 } while (0)
87#define kcsan_check_atomic_write(...) \
88 do { \
89 } while (0)
90#else
91#define kcsan_check_atomic_read(ptr, size) \
92 kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC)
93#define kcsan_check_atomic_write(ptr, size) \
94 kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE)
95#endif
96
97#endif /* _LINUX_KCSAN_CHECKS_H */
include/linux/kcsan.h+115−0
@@ -0,0 +1,115 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2
3#ifndef _LINUX_KCSAN_H
4#define _LINUX_KCSAN_H
5
6#include <linux/kcsan-checks.h>
7#include <linux/types.h>
8
9#ifdef CONFIG_KCSAN
10
11/*
12 * Context for each thread of execution: for tasks, this is stored in
13 * task_struct, and interrupts access internal per-CPU storage.
14 */
15struct kcsan_ctx {
16 int disable_count; /* disable counter */
17 int atomic_next; /* number of following atomic ops */
18
19 /*
20 * We distinguish between: (a) nestable atomic regions that may contain
21 * other nestable regions; and (b) flat atomic regions that do not keep
22 * track of nesting. Both (a) and (b) are entirely independent of each
23 * other, and a flat region may be started in a nestable region or
24 * vice-versa.
25 *
26 * This is required because, for example, in the annotations for
27 * seqlocks, we declare seqlock writer critical sections as (a) nestable
28 * atomic regions, but reader critical sections as (b) flat atomic
29 * regions, but have encountered cases where seqlock reader critical
30 * sections are contained within writer critical sections (the opposite
31 * may be possible, too).
32 *
33 * To support these cases, we independently track the depth of nesting
34 * for (a), and whether the leaf level is flat for (b).
35 */
36 int atomic_nest_count;
37 bool in_flat_atomic;
38};
39
40/**
41 * kcsan_init - initialize KCSAN runtime
42 */
43void kcsan_init(void);
44
45/**
46 * kcsan_disable_current - disable KCSAN for the current context
47 *
48 * Supports nesting.
49 */
50void kcsan_disable_current(void);
51
52/**
53 * kcsan_enable_current - re-enable KCSAN for the current context
54 *
55 * Supports nesting.
56 */
57void kcsan_enable_current(void);
58
59/**
60 * kcsan_nestable_atomic_begin - begin nestable atomic region
61 *
62 * Accesses within the atomic region may appear to race with other accesses but
63 * should be considered atomic.
64 */
65void kcsan_nestable_atomic_begin(void);
66
67/**
68 * kcsan_nestable_atomic_end - end nestable atomic region
69 */
70void kcsan_nestable_atomic_end(void);
71
72/**
73 * kcsan_flat_atomic_begin - begin flat atomic region
74 *
75 * Accesses within the atomic region may appear to race with other accesses but
76 * should be considered atomic.
77 */
78void kcsan_flat_atomic_begin(void);
79
80/**
81 * kcsan_flat_atomic_end - end flat atomic region
82 */
83void kcsan_flat_atomic_end(void);
84
85/**
86 * kcsan_atomic_next - consider following accesses as atomic
87 *
88 * Force treating the next n memory accesses for the current context as atomic
89 * operations.
90 *
91 * @n number of following memory accesses to treat as atomic.
92 */
93void kcsan_atomic_next(int n);
94
95#else /* CONFIG_KCSAN */
96
97static inline void kcsan_init(void) { }
98
99static inline void kcsan_disable_current(void) { }
100
101static inline void kcsan_enable_current(void) { }
102
103static inline void kcsan_nestable_atomic_begin(void) { }
104
105static inline void kcsan_nestable_atomic_end(void) { }
106
107static inline void kcsan_flat_atomic_begin(void) { }
108
109static inline void kcsan_flat_atomic_end(void) { }
110
111static inline void kcsan_atomic_next(int n) { }
112
113#endif /* CONFIG_KCSAN */
114
115#endif /* _LINUX_KCSAN_H */
include/linux/sched.h+4−0
@@ -31,6 +31,7 @@ · 30 unmodified lines
31#include <linux/task_io_accounting.h>31#include <linux/task_io_accounting.h>
32#include <linux/posix-timers.h>32#include <linux/posix-timers.h>
33#include <linux/rseq.h>33#include <linux/rseq.h>
34#include <linux/kcsan.h>
34 35
35/* task_struct member predeclarations (sorted alphabetically): */36/* task_struct member predeclarations (sorted alphabetically): */
36struct audit_context;37struct audit_context;
@@ -1172,6 +1173,9 @@ struct task_struct { · 1135 unmodified lines
1172#ifdef CONFIG_KASAN1173#ifdef CONFIG_KASAN
1173 unsigned int kasan_depth;1174 unsigned int kasan_depth;
1174#endif1175#endif
1176#ifdef CONFIG_KCSAN
1177 struct kcsan_ctx kcsan_ctx;
1178#endif
1175 1179
1176#ifdef CONFIG_FUNCTION_GRAPH_TRACER1180#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1177 /* Index of current stored address in ret_stack: */1181 /* Index of current stored address in ret_stack: */
init/init_task.c+8−0
@@ -161,6 +161,14 @@ struct task_struct init_task · 160 unmodified lines
161#ifdef CONFIG_KASAN161#ifdef CONFIG_KASAN
162 .kasan_depth = 1,162 .kasan_depth = 1,
163#endif163#endif
164#ifdef CONFIG_KCSAN
165 .kcsan_ctx = {
166 .disable_count = 0,
167 .atomic_next = 0,
168 .atomic_nest_count = 0,
169 .in_flat_atomic = false,
170 },
171#endif
164#ifdef CONFIG_TRACE_IRQFLAGS172#ifdef CONFIG_TRACE_IRQFLAGS
165 .softirqs_enabled = 1,173 .softirqs_enabled = 1,
166#endif174#endif
init/main.c+2−0
@@ -93,6 +93,7 @@ · 92 unmodified lines
93#include <linux/rodata_test.h>93#include <linux/rodata_test.h>
94#include <linux/jump_label.h>94#include <linux/jump_label.h>
95#include <linux/mem_encrypt.h>95#include <linux/mem_encrypt.h>
96#include <linux/kcsan.h>
96 97
97#include <asm/io.h>98#include <asm/io.h>
98#include <asm/bugs.h>99#include <asm/bugs.h>
@@ -779,6 +780,7 @@ asmlinkage __visible void __init start_kernel(void) · 680 unmodified lines
779 acpi_subsystem_init();780 acpi_subsystem_init();
780 arch_post_acpi_subsys_init();781 arch_post_acpi_subsys_init();
781 sfi_init_late();782 sfi_init_late();
783 kcsan_init();
782 784
783 /* Do the rest non-__init'ed, we're now alive */785 /* Do the rest non-__init'ed, we're now alive */
784 arch_call_rest_init();786 arch_call_rest_init();
kernel/Makefile+1−0
@@ -102,6 +102,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/ · 101 unmodified lines
102obj-$(CONFIG_IRQ_WORK) += irq_work.o102obj-$(CONFIG_IRQ_WORK) += irq_work.o
103obj-$(CONFIG_CPU_PM) += cpu_pm.o103obj-$(CONFIG_CPU_PM) += cpu_pm.o
104obj-$(CONFIG_BPF) += bpf/104obj-$(CONFIG_BPF) += bpf/
105obj-$(CONFIG_KCSAN) += kcsan/
105 106
106obj-$(CONFIG_PERF_EVENTS) += events/107obj-$(CONFIG_PERF_EVENTS) += events/
107 108
kernel/kcsan/Makefile+11−0
@@ -0,0 +1,11 @@
1# SPDX-License-Identifier: GPL-2.0
2KCSAN_SANITIZE := n
3KCOV_INSTRUMENT := n
4
5CFLAGS_REMOVE_core.o = $(CC_FLAGS_FTRACE)
6
7CFLAGS_core.o := $(call cc-option,-fno-conserve-stack,) \
8 $(call cc-option,-fno-stack-protector,)
9
10obj-y := core.o debugfs.o report.o
11obj-$(CONFIG_KCSAN_SELFTEST) += test.o
kernel/kcsan/atomic.h+27−0
@@ -0,0 +1,27 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2
3#ifndef _KERNEL_KCSAN_ATOMIC_H
4#define _KERNEL_KCSAN_ATOMIC_H
5
6#include <linux/jiffies.h>
7
8/*
9 * Helper that returns true if access to ptr should be considered as an atomic
10 * access, even though it is not explicitly atomic.
11 *
12 * List all volatile globals that have been observed in races, to suppress
13 * data race reports between accesses to these variables.
14 *
15 * For now, we assume that volatile accesses of globals are as strong as atomic
16 * accesses (READ_ONCE, WRITE_ONCE cast to volatile). The situation is still not
17 * entirely clear, as on some architectures (Alpha) READ_ONCE/WRITE_ONCE do more
18 * than cast to volatile. Eventually, we hope to be able to remove this
19 * function.
20 */
21static inline bool kcsan_is_atomic(const volatile void *ptr)
22{
23 /* only jiffies for now */
24 return ptr == &jiffies;
25}
26
27#endif /* _KERNEL_KCSAN_ATOMIC_H */
kernel/kcsan/core.c+626−0
@@ -0,0 +1,626 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/atomic.h>
4#include <linux/bug.h>
5#include <linux/delay.h>
6#include <linux/export.h>
7#include <linux/init.h>
8#include <linux/percpu.h>
9#include <linux/preempt.h>
10#include <linux/random.h>
11#include <linux/sched.h>
12#include <linux/uaccess.h>
13
14#include "atomic.h"
15#include "encoding.h"
16#include "kcsan.h"
17
18bool kcsan_enabled;
19
20/* Per-CPU kcsan_ctx for interrupts */
21static DEFINE_PER_CPU(struct kcsan_ctx, kcsan_cpu_ctx) = {
22 .disable_count = 0,
23 .atomic_next = 0,
24 .atomic_nest_count = 0,
25 .in_flat_atomic = false,
26};
27
28/*
29 * Helper macros to index into adjacent slots slots, starting from address slot
30 * itself, followed by the right and left slots.
31 *
32 * The purpose is 2-fold:
33 *
34 * 1. if during insertion the address slot is already occupied, check if
35 * any adjacent slots are free;
36 * 2. accesses that straddle a slot boundary due to size that exceeds a
37 * slot's range may check adjacent slots if any watchpoint matches.
38 *
39 * Note that accesses with very large size may still miss a watchpoint; however,
40 * given this should be rare, this is a reasonable trade-off to make, since this
41 * will avoid:
42 *
43 * 1. excessive contention between watchpoint checks and setup;
44 * 2. larger number of simultaneous watchpoints without sacrificing
45 * performance.
46 *
47 * Example: SLOT_IDX values for KCSAN_CHECK_ADJACENT=1, where i is [0, 1, 2]:
48 *
49 * slot=0: [ 1, 2, 0]
50 * slot=9: [10, 11, 9]
51 * slot=63: [64, 65, 63]
52 */
53#define NUM_SLOTS (1 + 2 * KCSAN_CHECK_ADJACENT)
54#define SLOT_IDX(slot, i) (slot + ((i + KCSAN_CHECK_ADJACENT) % NUM_SLOTS))
55
56/*
57 * SLOT_IDX_FAST is used in fast-path. Not first checking the address's primary
58 * slot (middle) is fine if we assume that data races occur rarely. The set of
59 * indices {SLOT_IDX(slot, i) | i in [0, NUM_SLOTS)} is equivalent to
60 * {SLOT_IDX_FAST(slot, i) | i in [0, NUM_SLOTS)}.
61 */
62#define SLOT_IDX_FAST(slot, i) (slot + i)
63
64/*
65 * Watchpoints, with each entry encoded as defined in encoding.h: in order to be
66 * able to safely update and access a watchpoint without introducing locking
67 * overhead, we encode each watchpoint as a single atomic long. The initial
68 * zero-initialized state matches INVALID_WATCHPOINT.
69 *
70 * Add NUM_SLOTS-1 entries to account for overflow; this helps avoid having to
71 * use more complicated SLOT_IDX_FAST calculation with modulo in fast-path.
72 */
73static atomic_long_t watchpoints[CONFIG_KCSAN_NUM_WATCHPOINTS + NUM_SLOTS - 1];
74
75/*
76 * Instructions to skip watching counter, used in should_watch(). We use a
77 * per-CPU counter to avoid excessive contention.
78 */
79static DEFINE_PER_CPU(long, kcsan_skip);
80
81static inline atomic_long_t *find_watchpoint(unsigned long addr, size_t size,
82 bool expect_write,
83 long *encoded_watchpoint)
84{
85 const int slot = watchpoint_slot(addr);
86 const unsigned long addr_masked = addr & WATCHPOINT_ADDR_MASK;
87 atomic_long_t *watchpoint;
88 unsigned long wp_addr_masked;
89 size_t wp_size;
90 bool is_write;
91 int i;
92
93 BUILD_BUG_ON(CONFIG_KCSAN_NUM_WATCHPOINTS < NUM_SLOTS);
94
95 for (i = 0; i < NUM_SLOTS; ++i) {
96 watchpoint = &watchpoints[SLOT_IDX_FAST(slot, i)];
97 *encoded_watchpoint = atomic_long_read(watchpoint);
98 if (!decode_watchpoint(*encoded_watchpoint, &wp_addr_masked,
99 &wp_size, &is_write))
100 continue;
101
102 if (expect_write && !is_write)
103 continue;
104
105 /* Check if the watchpoint matches the access. */
106 if (matching_access(wp_addr_masked, wp_size, addr_masked, size))
107 return watchpoint;
108 }
109
110 return NULL;
111}
112
113static inline atomic_long_t *insert_watchpoint(unsigned long addr, size_t size,
114 bool is_write)
115{
116 const int slot = watchpoint_slot(addr);
117 const long encoded_watchpoint = encode_watchpoint(addr, size, is_write);
118 atomic_long_t *watchpoint;
119 int i;
120
121 /* Check slot index logic, ensuring we stay within array bounds. */
122 BUILD_BUG_ON(SLOT_IDX(0, 0) != KCSAN_CHECK_ADJACENT);
123 BUILD_BUG_ON(SLOT_IDX(0, KCSAN_CHECK_ADJACENT + 1) != 0);
124 BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS - 1,
125 KCSAN_CHECK_ADJACENT) !=
126 ARRAY_SIZE(watchpoints) - 1);
127 BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS - 1,
128 KCSAN_CHECK_ADJACENT + 1) !=
129 ARRAY_SIZE(watchpoints) - NUM_SLOTS);
130
131 for (i = 0; i < NUM_SLOTS; ++i) {
132 long expect_val = INVALID_WATCHPOINT;
133
134 /* Try to acquire this slot. */
135 watchpoint = &watchpoints[SLOT_IDX(slot, i)];
136 if (atomic_long_try_cmpxchg_relaxed(watchpoint, &expect_val,
137 encoded_watchpoint))
138 return watchpoint;
139 }
140
141 return NULL;
142}
143
144/*
145 * Return true if watchpoint was successfully consumed, false otherwise.
146 *
147 * This may return false if:
148 *
149 * 1. another thread already consumed the watchpoint;
150 * 2. the thread that set up the watchpoint already removed it;
151 * 3. the watchpoint was removed and then re-used.
152 */
153static inline bool try_consume_watchpoint(atomic_long_t *watchpoint,
154 long encoded_watchpoint)
155{
156 return atomic_long_try_cmpxchg_relaxed(watchpoint, &encoded_watchpoint,
157 CONSUMED_WATCHPOINT);
158}
159
160/*
161 * Return true if watchpoint was not touched, false if consumed.
162 */
163static inline bool remove_watchpoint(atomic_long_t *watchpoint)
164{
165 return atomic_long_xchg_relaxed(watchpoint, INVALID_WATCHPOINT) !=
166 CONSUMED_WATCHPOINT;
167}
168
169static inline struct kcsan_ctx *get_ctx(void)
170{
171 /*
172 * In interrupt, use raw_cpu_ptr to avoid unnecessary checks, that would
173 * also result in calls that generate warnings in uaccess regions.
174 */
175 return in_task() ? &current->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx);
176}
177
178static inline bool is_atomic(const volatile void *ptr)
179{
180 struct kcsan_ctx *ctx = get_ctx();
181
182 if (unlikely(ctx->atomic_next > 0)) {
183 /*
184 * Because we do not have separate contexts for nested
185 * interrupts, in case atomic_next is set, we simply assume that
186 * the outer interrupt set atomic_next. In the worst case, we
187 * will conservatively consider operations as atomic. This is a
188 * reasonable trade-off to make, since this case should be
189 * extremely rare; however, even if extremely rare, it could
190 * lead to false positives otherwise.
191 */
192 if ((hardirq_count() >> HARDIRQ_SHIFT) < 2)
193 --ctx->atomic_next; /* in task, or outer interrupt */
194 return true;
195 }
196 if (unlikely(ctx->atomic_nest_count > 0 || ctx->in_flat_atomic))
197 return true;
198
199 return kcsan_is_atomic(ptr);
200}
201
202static inline bool should_watch(const volatile void *ptr, int type)
203{
204 /*
205 * Never set up watchpoints when memory operations are atomic.
206 *
207 * Need to check this first, before kcsan_skip check below: (1) atomics
208 * should not count towards skipped instructions, and (2) to actually
209 * decrement kcsan_atomic_next for consecutive instruction stream.
210 */
211 if ((type & KCSAN_ACCESS_ATOMIC) != 0 || is_atomic(ptr))
212 return false;
213
214 if (this_cpu_dec_return(kcsan_skip) >= 0)
215 return false;
216
217 /*
218 * NOTE: If we get here, kcsan_skip must always be reset in slow path
219 * via reset_kcsan_skip() to avoid underflow.
220 */
221
222 /* this operation should be watched */
223 return true;
224}
225
226static inline void reset_kcsan_skip(void)
227{
228 long skip_count = CONFIG_KCSAN_SKIP_WATCH -
229 (IS_ENABLED(CONFIG_KCSAN_SKIP_WATCH_RANDOMIZE) ?
230 prandom_u32_max(CONFIG_KCSAN_SKIP_WATCH) :
231 0);
232 this_cpu_write(kcsan_skip, skip_count);
233}
234
235static inline bool kcsan_is_enabled(void)
236{
237 return READ_ONCE(kcsan_enabled) && get_ctx()->disable_count == 0;
238}
239
240static inline unsigned int get_delay(void)
241{
242 unsigned int delay = in_task() ? CONFIG_KCSAN_UDELAY_TASK :
243 CONFIG_KCSAN_UDELAY_INTERRUPT;
244 return delay - (IS_ENABLED(CONFIG_KCSAN_DELAY_RANDOMIZE) ?
245 prandom_u32_max(delay) :
246 0);
247}
248
249/*
250 * Pull everything together: check_access() below contains the performance
251 * critical operations; the fast-path (including check_access) functions should
252 * all be inlinable by the instrumentation functions.
253 *
254 * The slow-path (kcsan_found_watchpoint, kcsan_setup_watchpoint) are
255 * non-inlinable -- note that, we prefix these with "kcsan_" to ensure they can
256 * be filtered from the stacktrace, as well as give them unique names for the
257 * UACCESS whitelist of objtool. Each function uses user_access_save/restore(),
258 * since they do not access any user memory, but instrumentation is still
259 * emitted in UACCESS regions.
260 */
261
262static noinline void kcsan_found_watchpoint(const volatile void *ptr,
263 size_t size, bool is_write,
264 atomic_long_t *watchpoint,
265 long encoded_watchpoint)
266{
267 unsigned long flags;
268 bool consumed;
269
270 if (!kcsan_is_enabled())
271 return;
272 /*
273 * Consume the watchpoint as soon as possible, to minimize the chances
274 * of !consumed. Consuming the watchpoint must always be guarded by
275 * kcsan_is_enabled() check, as otherwise we might erroneously
276 * triggering reports when disabled.
277 */
278 consumed = try_consume_watchpoint(watchpoint, encoded_watchpoint);
279
280 /* keep this after try_consume_watchpoint */
281 flags = user_access_save();
282
283 if (consumed) {
284 kcsan_report(ptr, size, is_write, true, raw_smp_processor_id(),
285 KCSAN_REPORT_CONSUMED_WATCHPOINT);
286 } else {
287 /*
288 * The other thread may not print any diagnostics, as it has
289 * already removed the watchpoint, or another thread consumed
290 * the watchpoint before this thread.
291 */
292 kcsan_counter_inc(KCSAN_COUNTER_REPORT_RACES);
293 }
294 kcsan_counter_inc(KCSAN_COUNTER_DATA_RACES);
295
296 user_access_restore(flags);
297}
298
299static noinline void kcsan_setup_watchpoint(const volatile void *ptr,
300 size_t size, bool is_write)
301{
302 atomic_long_t *watchpoint;
303 union {
304 u8 _1;
305 u16 _2;
306 u32 _4;
307 u64 _8;
308 } expect_value;
309 bool value_change = false;
310 unsigned long ua_flags = user_access_save();
311 unsigned long irq_flags;
312
313 /*
314 * Always reset kcsan_skip counter in slow-path to avoid underflow; see
315 * should_watch().
316 */
317 reset_kcsan_skip();
318
319 if (!kcsan_is_enabled())
320 goto out;
321
322 if (!check_encodable((unsigned long)ptr, size)) {
323 kcsan_counter_inc(KCSAN_COUNTER_UNENCODABLE_ACCESSES);
324 goto out;
325 }
326
327 /*
328 * Disable interrupts & preemptions to avoid another thread on the same
329 * CPU accessing memory locations for the set up watchpoint; this is to
330 * avoid reporting races to e.g. CPU-local data.
331 *
332 * An alternative would be adding the source CPU to the watchpoint
333 * encoding, and checking that watchpoint-CPU != this-CPU. There are
334 * several problems with this:
335 * 1. we should avoid stealing more bits from the watchpoint encoding
336 * as it would affect accuracy, as well as increase performance
337 * overhead in the fast-path;
338 * 2. if we are preempted, but there *is* a genuine data race, we
339 * would *not* report it -- since this is the common case (vs.
340 * CPU-local data accesses), it makes more sense (from a data race
341 * detection point of view) to simply disable preemptions to ensure
342 * as many tasks as possible run on other CPUs.
343 */
344 local_irq_save(irq_flags);
345
346 watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write);
347 if (watchpoint == NULL) {
348 /*
349 * Out of capacity: the size of `watchpoints`, and the frequency
350 * with which `should_watch()` returns true should be tweaked so
351 * that this case happens very rarely.
352 */
353 kcsan_counter_inc(KCSAN_COUNTER_NO_CAPACITY);
354 goto out_unlock;
355 }
356
357 kcsan_counter_inc(KCSAN_COUNTER_SETUP_WATCHPOINTS);
358 kcsan_counter_inc(KCSAN_COUNTER_USED_WATCHPOINTS);
359
360 /*
361 * Read the current value, to later check and infer a race if the data
362 * was modified via a non-instrumented access, e.g. from a device.
363 */
364 switch (size) {
365 case 1:
366 expect_value._1 = READ_ONCE(*(const u8 *)ptr);
367 break;
368 case 2:
369 expect_value._2 = READ_ONCE(*(const u16 *)ptr);
370 break;
371 case 4:
372 expect_value._4 = READ_ONCE(*(const u32 *)ptr);
373 break;
374 case 8:
375 expect_value._8 = READ_ONCE(*(const u64 *)ptr);
376 break;
377 default:
378 break; /* ignore; we do not diff the values */
379 }
380
381 if (IS_ENABLED(CONFIG_KCSAN_DEBUG)) {
382 kcsan_disable_current();
383 pr_err("KCSAN: watching %s, size: %zu, addr: %px [slot: %d, encoded: %lx]\n",
384 is_write ? "write" : "read", size, ptr,
385 watchpoint_slot((unsigned long)ptr),
386 encode_watchpoint((unsigned long)ptr, size, is_write));
387 kcsan_enable_current();
388 }
389
390 /*
391 * Delay this thread, to increase probability of observing a racy
392 * conflicting access.
393 */
394 udelay(get_delay());
395
396 /*
397 * Re-read value, and check if it is as expected; if not, we infer a
398 * racy access.
399 */
400 switch (size) {
401 case 1:
402 value_change = expect_value._1 != READ_ONCE(*(const u8 *)ptr);
403 break;
404 case 2:
405 value_change = expect_value._2 != READ_ONCE(*(const u16 *)ptr);
406 break;
407 case 4:
408 value_change = expect_value._4 != READ_ONCE(*(const u32 *)ptr);
409 break;
410 case 8:
411 value_change = expect_value._8 != READ_ONCE(*(const u64 *)ptr);
412 break;
413 default:
414 break; /* ignore; we do not diff the values */
415 }
416
417 /* Check if this access raced with another. */
418 if (!remove_watchpoint(watchpoint)) {
419 /*
420 * No need to increment 'data_races' counter, as the racing
421 * thread already did.
422 */
423 kcsan_report(ptr, size, is_write, size > 8 || value_change,
424 smp_processor_id(), KCSAN_REPORT_RACE_SIGNAL);
425 } else if (value_change) {
426 /* Inferring a race, since the value should not have changed. */
427 kcsan_counter_inc(KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN);
428 if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN))
429 kcsan_report(ptr, size, is_write, true,
430 smp_processor_id(),
431 KCSAN_REPORT_RACE_UNKNOWN_ORIGIN);
432 }
433
434 kcsan_counter_dec(KCSAN_COUNTER_USED_WATCHPOINTS);
435out_unlock:
436 local_irq_restore(irq_flags);
437out:
438 user_access_restore(ua_flags);
439}
440
441static __always_inline void check_access(const volatile void *ptr, size_t size,
442 int type)
443{
444 const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0;
445 atomic_long_t *watchpoint;
446 long encoded_watchpoint;
447
448 /*
449 * Avoid user_access_save in fast-path: find_watchpoint is safe without
450 * user_access_save, as the address that ptr points to is only used to
451 * check if a watchpoint exists; ptr is never dereferenced.
452 */
453 watchpoint = find_watchpoint((unsigned long)ptr, size, !is_write,
454 &encoded_watchpoint);
455 /*
456 * It is safe to check kcsan_is_enabled() after find_watchpoint in the
457 * slow-path, as long as no state changes that cause a data race to be
458 * detected and reported have occurred until kcsan_is_enabled() is
459 * checked.
460 */
461
462 if (unlikely(watchpoint != NULL))
463 kcsan_found_watchpoint(ptr, size, is_write, watchpoint,
464 encoded_watchpoint);
465 else if (unlikely(should_watch(ptr, type)))
466 kcsan_setup_watchpoint(ptr, size, is_write);
467}
468
469/* === Public interface ===================================================== */
470
471void __init kcsan_init(void)
472{
473 BUG_ON(!in_task());
474
475 kcsan_debugfs_init();
476
477 /*
478 * We are in the init task, and no other tasks should be running;
479 * WRITE_ONCE without memory barrier is sufficient.
480 */
481 if (IS_ENABLED(CONFIG_KCSAN_EARLY_ENABLE))
482 WRITE_ONCE(kcsan_enabled, true);
483}
484
485/* === Exported interface =================================================== */
486
487void kcsan_disable_current(void)
488{
489 ++get_ctx()->disable_count;
490}
491EXPORT_SYMBOL(kcsan_disable_current);
492
493void kcsan_enable_current(void)
494{
495 if (get_ctx()->disable_count-- == 0) {
496 /*
497 * Warn if kcsan_enable_current() calls are unbalanced with
498 * kcsan_disable_current() calls, which causes disable_count to
499 * become negative and should not happen.
500 */
501 kcsan_disable_current(); /* restore to 0, KCSAN still enabled */
502 kcsan_disable_current(); /* disable to generate warning */
503 WARN(1, "Unbalanced %s()", __func__);
504 kcsan_enable_current();
505 }
506}
507EXPORT_SYMBOL(kcsan_enable_current);
508
509void kcsan_nestable_atomic_begin(void)
510{
511 /*
512 * Do *not* check and warn if we are in a flat atomic region: nestable
513 * and flat atomic regions are independent from each other.
514 * See include/linux/kcsan.h: struct kcsan_ctx comments for more
515 * comments.
516 */
517
518 ++get_ctx()->atomic_nest_count;
519}
520EXPORT_SYMBOL(kcsan_nestable_atomic_begin);
521
522void kcsan_nestable_atomic_end(void)
523{
524 if (get_ctx()->atomic_nest_count-- == 0) {
525 /*
526 * Warn if kcsan_nestable_atomic_end() calls are unbalanced with
527 * kcsan_nestable_atomic_begin() calls, which causes
528 * atomic_nest_count to become negative and should not happen.
529 */
530 kcsan_nestable_atomic_begin(); /* restore to 0 */
531 kcsan_disable_current(); /* disable to generate warning */
532 WARN(1, "Unbalanced %s()", __func__);
533 kcsan_enable_current();
534 }
535}
536EXPORT_SYMBOL(kcsan_nestable_atomic_end);
537
538void kcsan_flat_atomic_begin(void)
539{
540 get_ctx()->in_flat_atomic = true;
541}
542EXPORT_SYMBOL(kcsan_flat_atomic_begin);
543
544void kcsan_flat_atomic_end(void)
545{
546 get_ctx()->in_flat_atomic = false;
547}
548EXPORT_SYMBOL(kcsan_flat_atomic_end);
549
550void kcsan_atomic_next(int n)
551{
552 get_ctx()->atomic_next = n;
553}
554EXPORT_SYMBOL(kcsan_atomic_next);
555
556void __kcsan_check_access(const volatile void *ptr, size_t size, int type)
557{
558 check_access(ptr, size, type);
559}
560EXPORT_SYMBOL(__kcsan_check_access);
561
562/*
563 * KCSAN uses the same instrumentation that is emitted by supported compilers
564 * for ThreadSanitizer (TSAN).
565 *
566 * When enabled, the compiler emits instrumentation calls (the functions
567 * prefixed with "__tsan" below) for all loads and stores that it generated;
568 * inline asm is not instrumented.
569 *
570 * Note that, not all supported compiler versions distinguish aligned/unaligned
571 * accesses, but e.g. recent versions of Clang do. We simply alias the unaligned
572 * version to the generic version, which can handle both.
573 */
574
575#define DEFINE_TSAN_READ_WRITE(size) \
576 void __tsan_read##size(void *ptr) \
577 { \
578 check_access(ptr, size, 0); \
579 } \
580 EXPORT_SYMBOL(__tsan_read##size); \
581 void __tsan_unaligned_read##size(void *ptr) \
582 __alias(__tsan_read##size); \
583 EXPORT_SYMBOL(__tsan_unaligned_read##size); \
584 void __tsan_write##size(void *ptr) \
585 { \
586 check_access(ptr, size, KCSAN_ACCESS_WRITE); \
587 } \
588 EXPORT_SYMBOL(__tsan_write##size); \
589 void __tsan_unaligned_write##size(void *ptr) \
590 __alias(__tsan_write##size); \
591 EXPORT_SYMBOL(__tsan_unaligned_write##size)
592
593DEFINE_TSAN_READ_WRITE(1);
594DEFINE_TSAN_READ_WRITE(2);
595DEFINE_TSAN_READ_WRITE(4);
596DEFINE_TSAN_READ_WRITE(8);
597DEFINE_TSAN_READ_WRITE(16);
598
599void __tsan_read_range(void *ptr, size_t size)
600{
601 check_access(ptr, size, 0);
602}
603EXPORT_SYMBOL(__tsan_read_range);
604
605void __tsan_write_range(void *ptr, size_t size)
606{
607 check_access(ptr, size, KCSAN_ACCESS_WRITE);
608}
609EXPORT_SYMBOL(__tsan_write_range);
610
611/*
612 * The below are not required by KCSAN, but can still be emitted by the
613 * compiler.
614 */
615void __tsan_func_entry(void *call_pc)
616{
617}
618EXPORT_SYMBOL(__tsan_func_entry);
619void __tsan_func_exit(void)
620{
621}
622EXPORT_SYMBOL(__tsan_func_exit);
623void __tsan_init(void)
624{
625}
626EXPORT_SYMBOL(__tsan_init);
kernel/kcsan/debugfs.c+275−0
@@ -0,0 +1,275 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/atomic.h>
4#include <linux/bsearch.h>
5#include <linux/bug.h>
6#include <linux/debugfs.h>
7#include <linux/init.h>
8#include <linux/kallsyms.h>
9#include <linux/seq_file.h>
10#include <linux/slab.h>
11#include <linux/sort.h>
12#include <linux/string.h>
13#include <linux/uaccess.h>
14
15#include "kcsan.h"
16
17/*
18 * Statistics counters.
19 */
20static atomic_long_t counters[KCSAN_COUNTER_COUNT];
21
22/*
23 * Addresses for filtering functions from reporting. This list can be used as a
24 * whitelist or blacklist.
25 */
26static struct {
27 unsigned long *addrs; /* array of addresses */
28 size_t size; /* current size */
29 int used; /* number of elements used */
30 bool sorted; /* if elements are sorted */
31 bool whitelist; /* if list is a blacklist or whitelist */
32} report_filterlist = {
33 .addrs = NULL,
34 .size = 8, /* small initial size */
35 .used = 0,
36 .sorted = false,
37 .whitelist = false, /* default is blacklist */
38};
39static DEFINE_SPINLOCK(report_filterlist_lock);
40
41static const char *counter_to_name(enum kcsan_counter_id id)
42{
43 switch (id) {
44 case KCSAN_COUNTER_USED_WATCHPOINTS:
45 return "used_watchpoints";
46 case KCSAN_COUNTER_SETUP_WATCHPOINTS:
47 return "setup_watchpoints";
48 case KCSAN_COUNTER_DATA_RACES:
49 return "data_races";
50 case KCSAN_COUNTER_NO_CAPACITY:
51 return "no_capacity";
52 case KCSAN_COUNTER_REPORT_RACES:
53 return "report_races";
54 case KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN:
55 return "races_unknown_origin";
56 case KCSAN_COUNTER_UNENCODABLE_ACCESSES:
57 return "unencodable_accesses";
58 case KCSAN_COUNTER_ENCODING_FALSE_POSITIVES:
59 return "encoding_false_positives";
60 case KCSAN_COUNTER_COUNT:
61 BUG();
62 }
63 return NULL;
64}
65
66void kcsan_counter_inc(enum kcsan_counter_id id)
67{
68 atomic_long_inc(&counters[id]);
69}
70
71void kcsan_counter_dec(enum kcsan_counter_id id)
72{
73 atomic_long_dec(&counters[id]);
74}
75
76/*
77 * The microbenchmark allows benchmarking KCSAN core runtime only. To run
78 * multiple threads, pipe 'microbench=<iters>' from multiple tasks into the
79 * debugfs file.
80 */
81static void microbenchmark(unsigned long iters)
82{
83 cycles_t cycles;
84
85 pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters);
86
87 cycles = get_cycles();
88 while (iters--) {
89 /*
90 * We can run this benchmark from multiple tasks; this address
91 * calculation increases likelyhood of some accesses overlapping
92 * (they still won't conflict because all are reads).
93 */
94 unsigned long addr =
95 iters % (CONFIG_KCSAN_NUM_WATCHPOINTS * PAGE_SIZE);
96 __kcsan_check_read((void *)addr, sizeof(long));
97 }
98 cycles = get_cycles() - cycles;
99
100 pr_info("KCSAN: %s end | cycles: %llu\n", __func__, cycles);
101}
102
103static int cmp_filterlist_addrs(const void *rhs, const void *lhs)
104{
105 const unsigned long a = *(const unsigned long *)rhs;
106 const unsigned long b = *(const unsigned long *)lhs;
107
108 return a < b ? -1 : a == b ? 0 : 1;
109}
110
111bool kcsan_skip_report_debugfs(unsigned long func_addr)
112{
113 unsigned long symbolsize, offset;
114 unsigned long flags;
115 bool ret = false;
116
117 if (!kallsyms_lookup_size_offset(func_addr, &symbolsize, &offset))
118 return false;
119 func_addr -= offset; /* get function start */
120
121 spin_lock_irqsave(&report_filterlist_lock, flags);
122 if (report_filterlist.used == 0)
123 goto out;
124
125 /* Sort array if it is unsorted, and then do a binary search. */
126 if (!report_filterlist.sorted) {
127 sort(report_filterlist.addrs, report_filterlist.used,
128 sizeof(unsigned long), cmp_filterlist_addrs, NULL);
129 report_filterlist.sorted = true;
130 }
131 ret = !!bsearch(&func_addr, report_filterlist.addrs,
132 report_filterlist.used, sizeof(unsigned long),
133 cmp_filterlist_addrs);
134 if (report_filterlist.whitelist)
135 ret = !ret;
136
137out:
138 spin_unlock_irqrestore(&report_filterlist_lock, flags);
139 return ret;
140}
141
142static void set_report_filterlist_whitelist(bool whitelist)
143{
144 unsigned long flags;
145
146 spin_lock_irqsave(&report_filterlist_lock, flags);
147 report_filterlist.whitelist = whitelist;
148 spin_unlock_irqrestore(&report_filterlist_lock, flags);
149}
150
151/* Returns 0 on success, error-code otherwise. */
152static ssize_t insert_report_filterlist(const char *func)
153{
154 unsigned long flags;
155 unsigned long addr = kallsyms_lookup_name(func);
156 ssize_t ret = 0;
157
158 if (!addr) {
159 pr_err("KCSAN: could not find function: '%s'\n", func);
160 return -ENOENT;
161 }
162
163 spin_lock_irqsave(&report_filterlist_lock, flags);
164
165 if (report_filterlist.addrs == NULL) {
166 /* initial allocation */
167 report_filterlist.addrs =
168 kmalloc_array(report_filterlist.size,
169 sizeof(unsigned long), GFP_KERNEL);
170 if (report_filterlist.addrs == NULL) {
171 ret = -ENOMEM;
172 goto out;
173 }
174 } else if (report_filterlist.used == report_filterlist.size) {
175 /* resize filterlist */
176 size_t new_size = report_filterlist.size * 2;
177 unsigned long *new_addrs =
178 krealloc(report_filterlist.addrs,
179 new_size * sizeof(unsigned long), GFP_KERNEL);
180
181 if (new_addrs == NULL) {
182 /* leave filterlist itself untouched */
183 ret = -ENOMEM;
184 goto out;
185 }
186
187 report_filterlist.size = new_size;
188 report_filterlist.addrs = new_addrs;
189 }
190
191 /* Note: deduplicating should be done in userspace. */
192 report_filterlist.addrs[report_filterlist.used++] =
193 kallsyms_lookup_name(func);
194 report_filterlist.sorted = false;
195
196out:
197 spin_unlock_irqrestore(&report_filterlist_lock, flags);
198 return ret;
199}
200
201static int show_info(struct seq_file *file, void *v)
202{
203 int i;
204 unsigned long flags;
205
206 /* show stats */
207 seq_printf(file, "enabled: %i\n", READ_ONCE(kcsan_enabled));
208 for (i = 0; i < KCSAN_COUNTER_COUNT; ++i)
209 seq_printf(file, "%s: %ld\n", counter_to_name(i),
210 atomic_long_read(&counters[i]));
211
212 /* show filter functions, and filter type */
213 spin_lock_irqsave(&report_filterlist_lock, flags);
214 seq_printf(file, "\n%s functions: %s\n",
215 report_filterlist.whitelist ? "whitelisted" : "blacklisted",
216 report_filterlist.used == 0 ? "none" : "");
217 for (i = 0; i < report_filterlist.used; ++i)
218 seq_printf(file, " %ps\n", (void *)report_filterlist.addrs[i]);
219 spin_unlock_irqrestore(&report_filterlist_lock, flags);
220
221 return 0;
222}
223
224static int debugfs_open(struct inode *inode, struct file *file)
225{
226 return single_open(file, show_info, NULL);
227}
228
229static ssize_t debugfs_write(struct file *file, const char __user *buf,
230 size_t count, loff_t *off)
231{
232 char kbuf[KSYM_NAME_LEN];
233 char *arg;
234 int read_len = count < (sizeof(kbuf) - 1) ? count : (sizeof(kbuf) - 1);
235
236 if (copy_from_user(kbuf, buf, read_len))
237 return -EFAULT;
238 kbuf[read_len] = '\0';
239 arg = strstrip(kbuf);
240
241 if (!strcmp(arg, "on")) {
242 WRITE_ONCE(kcsan_enabled, true);
243 } else if (!strcmp(arg, "off")) {
244 WRITE_ONCE(kcsan_enabled, false);
245 } else if (!strncmp(arg, "microbench=", sizeof("microbench=") - 1)) {
246 unsigned long iters;
247
248 if (kstrtoul(&arg[sizeof("microbench=") - 1], 0, &iters))
249 return -EINVAL;
250 microbenchmark(iters);
251 } else if (!strcmp(arg, "whitelist")) {
252 set_report_filterlist_whitelist(true);
253 } else if (!strcmp(arg, "blacklist")) {
254 set_report_filterlist_whitelist(false);
255 } else if (arg[0] == '!') {
256 ssize_t ret = insert_report_filterlist(&arg[1]);
257
258 if (ret < 0)
259 return ret;
260 } else {
261 return -EINVAL;
262 }
263
264 return count;
265}
266
267static const struct file_operations debugfs_ops = { .read = seq_read,
268 .open = debugfs_open,
269 .write = debugfs_write,
270 .release = single_release };
271
272void __init kcsan_debugfs_init(void)
273{
274 debugfs_create_file("kcsan", 0644, NULL, NULL, &debugfs_ops);
275}
kernel/kcsan/encoding.h+94−0
@@ -0,0 +1,94 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2
3#ifndef _KERNEL_KCSAN_ENCODING_H
4#define _KERNEL_KCSAN_ENCODING_H
5
6#include <linux/bits.h>
7#include <linux/log2.h>
8#include <linux/mm.h>
9
10#include "kcsan.h"
11
12#define SLOT_RANGE PAGE_SIZE
13#define INVALID_WATCHPOINT 0
14#define CONSUMED_WATCHPOINT 1
15
16/*
17 * The maximum useful size of accesses for which we set up watchpoints is the
18 * max range of slots we check on an access.
19 */
20#define MAX_ENCODABLE_SIZE (SLOT_RANGE * (1 + KCSAN_CHECK_ADJACENT))
21
22/*
23 * Number of bits we use to store size info.
24 */
25#define WATCHPOINT_SIZE_BITS bits_per(MAX_ENCODABLE_SIZE)
26/*
27 * This encoding for addresses discards the upper (1 for is-write + SIZE_BITS);
28 * however, most 64-bit architectures do not use the full 64-bit address space.
29 * Also, in order for a false positive to be observable 2 things need to happen:
30 *
31 * 1. different addresses but with the same encoded address race;
32 * 2. and both map onto the same watchpoint slots;
33 *
34 * Both these are assumed to be very unlikely. However, in case it still happens
35 * happens, the report logic will filter out the false positive (see report.c).
36 */
37#define WATCHPOINT_ADDR_BITS (BITS_PER_LONG - 1 - WATCHPOINT_SIZE_BITS)
38
39/*
40 * Masks to set/retrieve the encoded data.
41 */
42#define WATCHPOINT_WRITE_MASK BIT(BITS_PER_LONG - 1)
43#define WATCHPOINT_SIZE_MASK \
44 GENMASK(BITS_PER_LONG - 2, BITS_PER_LONG - 2 - WATCHPOINT_SIZE_BITS)
45#define WATCHPOINT_ADDR_MASK \
46 GENMASK(BITS_PER_LONG - 3 - WATCHPOINT_SIZE_BITS, 0)
47
48static inline bool check_encodable(unsigned long addr, size_t size)
49{
50 return size <= MAX_ENCODABLE_SIZE;
51}
52
53static inline long encode_watchpoint(unsigned long addr, size_t size,
54 bool is_write)
55{
56 return (long)((is_write ? WATCHPOINT_WRITE_MASK : 0) |
57 (size << WATCHPOINT_ADDR_BITS) |
58 (addr & WATCHPOINT_ADDR_MASK));
59}
60
61static inline bool decode_watchpoint(long watchpoint,
62 unsigned long *addr_masked, size_t *size,
63 bool *is_write)
64{
65 if (watchpoint == INVALID_WATCHPOINT ||
66 watchpoint == CONSUMED_WATCHPOINT)
67 return false;
68
69 *addr_masked = (unsigned long)watchpoint & WATCHPOINT_ADDR_MASK;
70 *size = ((unsigned long)watchpoint & WATCHPOINT_SIZE_MASK) >>
71 WATCHPOINT_ADDR_BITS;
72 *is_write = !!((unsigned long)watchpoint & WATCHPOINT_WRITE_MASK);
73
74 return true;
75}
76
77/*
78 * Return watchpoint slot for an address.
79 */
80static inline int watchpoint_slot(unsigned long addr)
81{
82 return (addr / PAGE_SIZE) % CONFIG_KCSAN_NUM_WATCHPOINTS;
83}
84
85static inline bool matching_access(unsigned long addr1, size_t size1,
86 unsigned long addr2, size_t size2)
87{
88 unsigned long end_range1 = addr1 + size1 - 1;
89 unsigned long end_range2 = addr2 + size2 - 1;
90
91 return addr1 <= end_range2 && addr2 <= end_range1;
92}
93
94#endif /* _KERNEL_KCSAN_ENCODING_H */
kernel/kcsan/kcsan.h+108−0
@@ -0,0 +1,108 @@
1/* SPDX-License-Identifier: GPL-2.0 */
2
3/*
4 * The Kernel Concurrency Sanitizer (KCSAN) infrastructure. For more info please
5 * see Documentation/dev-tools/kcsan.rst.
6 */
7
8#ifndef _KERNEL_KCSAN_KCSAN_H
9#define _KERNEL_KCSAN_KCSAN_H
10
11#include <linux/kcsan.h>
12
13/* The number of adjacent watchpoints to check. */
14#define KCSAN_CHECK_ADJACENT 1
15
16/*
17 * Globally enable and disable KCSAN.
18 */
19extern bool kcsan_enabled;
20
21/*
22 * Initialize debugfs file.
23 */
24void kcsan_debugfs_init(void);
25
26enum kcsan_counter_id {
27 /*
28 * Number of watchpoints currently in use.
29 */
30 KCSAN_COUNTER_USED_WATCHPOINTS,
31
32 /*
33 * Total number of watchpoints set up.
34 */
35 KCSAN_COUNTER_SETUP_WATCHPOINTS,
36
37 /*
38 * Total number of data races.
39 */
40 KCSAN_COUNTER_DATA_RACES,
41
42 /*
43 * Number of times no watchpoints were available.
44 */
45 KCSAN_COUNTER_NO_CAPACITY,
46
47 /*
48 * A thread checking a watchpoint raced with another checking thread;
49 * only one will be reported.
50 */
51 KCSAN_COUNTER_REPORT_RACES,
52
53 /*
54 * Observed data value change, but writer thread unknown.
55 */
56 KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN,
57
58 /*
59 * The access cannot be encoded to a valid watchpoint.
60 */
61 KCSAN_COUNTER_UNENCODABLE_ACCESSES,
62
63 /*
64 * Watchpoint encoding caused a watchpoint to fire on mismatching
65 * accesses.
66 */
67 KCSAN_COUNTER_ENCODING_FALSE_POSITIVES,
68
69 KCSAN_COUNTER_COUNT, /* number of counters */
70};
71
72/*
73 * Increment/decrement counter with given id; avoid calling these in fast-path.
74 */
75void kcsan_counter_inc(enum kcsan_counter_id id);
76void kcsan_counter_dec(enum kcsan_counter_id id);
77
78/*
79 * Returns true if data races in the function symbol that maps to func_addr
80 * (offsets are ignored) should *not* be reported.
81 */
82bool kcsan_skip_report_debugfs(unsigned long func_addr);
83
84enum kcsan_report_type {
85 /*
86 * The thread that set up the watchpoint and briefly stalled was
87 * signalled that another thread triggered the watchpoint.
88 */
89 KCSAN_REPORT_RACE_SIGNAL,
90
91 /*
92 * A thread found and consumed a matching watchpoint.
93 */
94 KCSAN_REPORT_CONSUMED_WATCHPOINT,
95
96 /*
97 * No other thread was observed to race with the access, but the data
98 * value before and after the stall differs.
99 */
100 KCSAN_REPORT_RACE_UNKNOWN_ORIGIN,
101};
102/*
103 * Print a race report from thread that encountered the race.
104 */
105void kcsan_report(const volatile void *ptr, size_t size, bool is_write,
106 bool value_change, int cpu_id, enum kcsan_report_type type);
107
108#endif /* _KERNEL_KCSAN_KCSAN_H */
kernel/kcsan/report.c+320−0
@@ -0,0 +1,320 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/kernel.h>
4#include <linux/preempt.h>
5#include <linux/printk.h>
6#include <linux/sched.h>
7#include <linux/spinlock.h>
8#include <linux/stacktrace.h>
9
10#include "kcsan.h"
11#include "encoding.h"
12
13/*
14 * Max. number of stack entries to show in the report.
15 */
16#define NUM_STACK_ENTRIES 64
17
18/*
19 * Other thread info: communicated from other racing thread to thread that set
20 * up the watchpoint, which then prints the complete report atomically. Only
21 * need one struct, as all threads should to be serialized regardless to print
22 * the reports, with reporting being in the slow-path.
23 */
24static struct {
25 const volatile void *ptr;
26 size_t size;
27 bool is_write;
28 int task_pid;
29 int cpu_id;
30 unsigned long stack_entries[NUM_STACK_ENTRIES];
31 int num_stack_entries;
32} other_info = { .ptr = NULL };
33
34/*
35 * This spinlock protects reporting and other_info, since other_info is usually
36 * required when reporting.
37 */
38static DEFINE_SPINLOCK(report_lock);
39
40/*
41 * Special rules to skip reporting.
42 */
43static bool skip_report(bool is_write, bool value_change,
44 unsigned long top_frame)
45{
46 if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY) && is_write &&
47 !value_change) {
48 /*
49 * The access is a write, but the data value did not change.
50 *
51 * We opt-out of this filter for certain functions at request of
52 * maintainers.
53 */
54 char buf[64];
55
56 snprintf(buf, sizeof(buf), "%ps", (void *)top_frame);
57 if (!strnstr(buf, "rcu_", sizeof(buf)) &&
58 !strnstr(buf, "_rcu", sizeof(buf)) &&
59 !strnstr(buf, "_srcu", sizeof(buf)))
60 return true;
61 }
62
63 return kcsan_skip_report_debugfs(top_frame);
64}
65
66static inline const char *get_access_type(bool is_write)
67{
68 return is_write ? "write" : "read";
69}
70
71/* Return thread description: in task or interrupt. */
72static const char *get_thread_desc(int task_id)
73{
74 if (task_id != -1) {
75 static char buf[32]; /* safe: protected by report_lock */
76
77 snprintf(buf, sizeof(buf), "task %i", task_id);
78 return buf;
79 }
80 return "interrupt";
81}
82
83/* Helper to skip KCSAN-related functions in stack-trace. */
84static int get_stack_skipnr(unsigned long stack_entries[], int num_entries)
85{
86 char buf[64];
87 int skip = 0;
88
89 for (; skip < num_entries; ++skip) {
90 snprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skip]);
91 if (!strnstr(buf, "csan_", sizeof(buf)) &&
92 !strnstr(buf, "tsan_", sizeof(buf)) &&
93 !strnstr(buf, "_once_size", sizeof(buf))) {
94 break;
95 }
96 }
97 return skip;
98}
99
100/* Compares symbolized strings of addr1 and addr2. */
101static int sym_strcmp(void *addr1, void *addr2)
102{
103 char buf1[64];
104 char buf2[64];
105
106 snprintf(buf1, sizeof(buf1), "%pS", addr1);
107 snprintf(buf2, sizeof(buf2), "%pS", addr2);
108 return strncmp(buf1, buf2, sizeof(buf1));
109}
110
111/*
112 * Returns true if a report was generated, false otherwise.
113 */
114static bool print_report(const volatile void *ptr, size_t size, bool is_write,
115 bool value_change, int cpu_id,
116 enum kcsan_report_type type)
117{
118 unsigned long stack_entries[NUM_STACK_ENTRIES] = { 0 };
119 int num_stack_entries =
120 stack_trace_save(stack_entries, NUM_STACK_ENTRIES, 1);
121 int skipnr = get_stack_skipnr(stack_entries, num_stack_entries);
122 int other_skipnr;
123
124 /*
125 * Must check report filter rules before starting to print.
126 */
127 if (skip_report(is_write, true, stack_entries[skipnr]))
128 return false;
129
130 if (type == KCSAN_REPORT_RACE_SIGNAL) {
131 other_skipnr = get_stack_skipnr(other_info.stack_entries,
132 other_info.num_stack_entries);
133
134 /* value_change is only known for the other thread */
135 if (skip_report(other_info.is_write, value_change,
136 other_info.stack_entries[other_skipnr]))
137 return false;
138 }
139
140 /* Print report header. */
141 pr_err("==================================================================\n");
142 switch (type) {
143 case KCSAN_REPORT_RACE_SIGNAL: {
144 void *this_fn = (void *)stack_entries[skipnr];
145 void *other_fn = (void *)other_info.stack_entries[other_skipnr];
146 int cmp;
147
148 /*
149 * Order functions lexographically for consistent bug titles.
150 * Do not print offset of functions to keep title short.
151 */
152 cmp = sym_strcmp(other_fn, this_fn);
153 pr_err("BUG: KCSAN: data-race in %ps / %ps\n",
154 cmp < 0 ? other_fn : this_fn,
155 cmp < 0 ? this_fn : other_fn);
156 } break;
157
158 case KCSAN_REPORT_RACE_UNKNOWN_ORIGIN:
159 pr_err("BUG: KCSAN: data-race in %pS\n",
160 (void *)stack_entries[skipnr]);
161 break;
162
163 default:
164 BUG();
165 }
166
167 pr_err("\n");
168
169 /* Print information about the racing accesses. */
170 switch (type) {
171 case KCSAN_REPORT_RACE_SIGNAL:
172 pr_err("%s to 0x%px of %zu bytes by %s on cpu %i:\n",
173 get_access_type(other_info.is_write), other_info.ptr,
174 other_info.size, get_thread_desc(other_info.task_pid),
175 other_info.cpu_id);
176
177 /* Print the other thread's stack trace. */
178 stack_trace_print(other_info.stack_entries + other_skipnr,
179 other_info.num_stack_entries - other_skipnr,
180 0);
181
182 pr_err("\n");
183 pr_err("%s to 0x%px of %zu bytes by %s on cpu %i:\n",
184 get_access_type(is_write), ptr, size,
185 get_thread_desc(in_task() ? task_pid_nr(current) : -1),
186 cpu_id);
187 break;
188
189 case KCSAN_REPORT_RACE_UNKNOWN_ORIGIN:
190 pr_err("race at unknown origin, with %s to 0x%px of %zu bytes by %s on cpu %i:\n",
191 get_access_type(is_write), ptr, size,
192 get_thread_desc(in_task() ? task_pid_nr(current) : -1),
193 cpu_id);
194 break;
195
196 default:
197 BUG();
198 }
199 /* Print stack trace of this thread. */
200 stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr,
201 0);
202
203 /* Print report footer. */
204 pr_err("\n");
205 pr_err("Reported by Kernel Concurrency Sanitizer on:\n");
206 dump_stack_print_info(KERN_DEFAULT);
207 pr_err("==================================================================\n");
208
209 return true;
210}
211
212static void release_report(unsigned long *flags, enum kcsan_report_type type)
213{
214 if (type == KCSAN_REPORT_RACE_SIGNAL)
215 other_info.ptr = NULL; /* mark for reuse */
216
217 spin_unlock_irqrestore(&report_lock, *flags);
218}
219
220/*
221 * Depending on the report type either sets other_info and returns false, or
222 * acquires the matching other_info and returns true. If other_info is not
223 * required for the report type, simply acquires report_lock and returns true.
224 */
225static bool prepare_report(unsigned long *flags, const volatile void *ptr,
226 size_t size, bool is_write, int cpu_id,
227 enum kcsan_report_type type)
228{
229 if (type != KCSAN_REPORT_CONSUMED_WATCHPOINT &&
230 type != KCSAN_REPORT_RACE_SIGNAL) {
231 /* other_info not required; just acquire report_lock */
232 spin_lock_irqsave(&report_lock, *flags);
233 return true;
234 }
235
236retry:
237 spin_lock_irqsave(&report_lock, *flags);
238
239 switch (type) {
240 case KCSAN_REPORT_CONSUMED_WATCHPOINT:
241 if (other_info.ptr != NULL)
242 break; /* still in use, retry */
243
244 other_info.ptr = ptr;
245 other_info.size = size;
246 other_info.is_write = is_write;
247 other_info.task_pid = in_task() ? task_pid_nr(current) : -1;
248 other_info.cpu_id = cpu_id;
249 other_info.num_stack_entries = stack_trace_save(
250 other_info.stack_entries, NUM_STACK_ENTRIES, 1);
251
252 spin_unlock_irqrestore(&report_lock, *flags);
253
254 /*
255 * The other thread will print the summary; other_info may now
256 * be consumed.
257 */
258 return false;
259
260 case KCSAN_REPORT_RACE_SIGNAL:
261 if (other_info.ptr == NULL)
262 break; /* no data available yet, retry */
263
264 /*
265 * First check if this is the other_info we are expecting, i.e.
266 * matches based on how watchpoint was encoded.
267 */
268 if (!matching_access((unsigned long)other_info.ptr &
269 WATCHPOINT_ADDR_MASK,
270 other_info.size,
271 (unsigned long)ptr & WATCHPOINT_ADDR_MASK,
272 size))
273 break; /* mismatching watchpoint, retry */
274
275 if (!matching_access((unsigned long)other_info.ptr,
276 other_info.size, (unsigned long)ptr,
277 size)) {
278 /*
279 * If the actual accesses to not match, this was a false
280 * positive due to watchpoint encoding.
281 */
282 kcsan_counter_inc(
283 KCSAN_COUNTER_ENCODING_FALSE_POSITIVES);
284
285 /* discard this other_info */
286 release_report(flags, KCSAN_REPORT_RACE_SIGNAL);
287 return false;
288 }
289
290 /*
291 * Matching & usable access in other_info: keep other_info_lock
292 * locked, as this thread consumes it to print the full report;
293 * unlocked in release_report.
294 */
295 return true;
296
297 default:
298 BUG();
299 }
300
301 spin_unlock_irqrestore(&report_lock, *flags);
302 goto retry;
303}
304
305void kcsan_report(const volatile void *ptr, size_t size, bool is_write,
306 bool value_change, int cpu_id, enum kcsan_report_type type)
307{
308 unsigned long flags = 0;
309
310 kcsan_disable_current();
311 if (prepare_report(&flags, ptr, size, is_write, cpu_id, type)) {
312 if (print_report(ptr, size, is_write, value_change, cpu_id,
313 type) &&
314 panic_on_warn)
315 panic("panic_on_warn set ...\n");
316
317 release_report(&flags, type);
318 }
319 kcsan_enable_current();
320}
kernel/kcsan/test.c+121−0
@@ -0,0 +1,121 @@
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/init.h>
4#include <linux/kernel.h>
5#include <linux/printk.h>
6#include <linux/random.h>
7#include <linux/types.h>
8
9#include "encoding.h"
10
11#define ITERS_PER_TEST 2000
12
13/* Test requirements. */
14static bool test_requires(void)
15{
16 /* random should be initialized for the below tests */
17 return prandom_u32() + prandom_u32() != 0;
18}
19
20/*
21 * Test watchpoint encode and decode: check that encoding some access's info,
22 * and then subsequent decode preserves the access's info.
23 */
24static bool test_encode_decode(void)
25{
26 int i;
27
28 for (i = 0; i < ITERS_PER_TEST; ++i) {
29 size_t size = prandom_u32_max(MAX_ENCODABLE_SIZE) + 1;
30 bool is_write = !!prandom_u32_max(2);
31 unsigned long addr;
32
33 prandom_bytes(&addr, sizeof(addr));
34 if (WARN_ON(!check_encodable(addr, size)))
35 return false;
36
37 /* encode and decode */
38 {
39 const long encoded_watchpoint =
40 encode_watchpoint(addr, size, is_write);
41 unsigned long verif_masked_addr;
42 size_t verif_size;
43 bool verif_is_write;
44
45 /* check special watchpoints */
46 if (WARN_ON(decode_watchpoint(
47 INVALID_WATCHPOINT, &verif_masked_addr,
48 &verif_size, &verif_is_write)))
49 return false;
50 if (WARN_ON(decode_watchpoint(
51 CONSUMED_WATCHPOINT, &verif_masked_addr,
52 &verif_size, &verif_is_write)))
53 return false;
54
55 /* check decoding watchpoint returns same data */
56 if (WARN_ON(!decode_watchpoint(
57 encoded_watchpoint, &verif_masked_addr,
58 &verif_size, &verif_is_write)))
59 return false;
60 if (WARN_ON(verif_masked_addr !=
61 (addr & WATCHPOINT_ADDR_MASK)))
62 goto fail;
63 if (WARN_ON(verif_size != size))
64 goto fail;
65 if (WARN_ON(is_write != verif_is_write))
66 goto fail;
67
68 continue;
69fail:
70 pr_err("%s fail: %s %zu bytes @ %lx -> encoded: %lx -> %s %zu bytes @ %lx\n",
71 __func__, is_write ? "write" : "read", size,
72 addr, encoded_watchpoint,
73 verif_is_write ? "write" : "read", verif_size,
74 verif_masked_addr);
75 return false;
76 }
77 }
78
79 return true;
80}
81
82/* Test access matching function. */
83static bool test_matching_access(void)
84{
85 if (WARN_ON(!matching_access(10, 1, 10, 1)))
86 return false;
87 if (WARN_ON(!matching_access(10, 2, 11, 1)))
88 return false;
89 if (WARN_ON(!matching_access(10, 1, 9, 2)))
90 return false;
91 if (WARN_ON(matching_access(10, 1, 11, 1)))
92 return false;
93 if (WARN_ON(matching_access(9, 1, 10, 1)))
94 return false;
95 return true;
96}
97
98static int __init kcsan_selftest(void)
99{
100 int passed = 0;
101 int total = 0;
102
103#define RUN_TEST(do_test) \
104 do { \
105 ++total; \
106 if (do_test()) \
107 ++passed; \
108 else \
109 pr_err("KCSAN selftest: " #do_test " failed"); \
110 } while (0)
111
112 RUN_TEST(test_requires);
113 RUN_TEST(test_encode_decode);
114 RUN_TEST(test_matching_access);
115
116 pr_info("KCSAN selftest: %d/%d tests passed\n", passed, total);
117 if (passed != total)
118 panic("KCSAN selftests failed");
119 return 0;
120}
121postcore_initcall(kcsan_selftest);
lib/Kconfig.debug+2−0
@@ -2086,6 +2086,8 @@ source "lib/Kconfig.kgdb" · 2085 unmodified lines
2086 2086
2087source "lib/Kconfig.ubsan"2087source "lib/Kconfig.ubsan"
2088 2088
2089source "lib/Kconfig.kcsan"
2090
2089config ARCH_HAS_DEVMEM_IS_ALLOWED2091config ARCH_HAS_DEVMEM_IS_ALLOWED
2090 bool2092 bool
2091 2093
lib/Kconfig.kcsan+118−0
@@ -0,0 +1,118 @@
1# SPDX-License-Identifier: GPL-2.0-only
2
3config HAVE_ARCH_KCSAN
4 bool
5
6menuconfig KCSAN
7 bool "KCSAN: watchpoint-based dynamic data race detector"
8 depends on HAVE_ARCH_KCSAN && !KASAN && STACKTRACE
9 default n
10 help
11 Kernel Concurrency Sanitizer is a dynamic data race detector, which
12 uses a watchpoint-based sampling approach to detect races. See
13 <file:Documentation/dev-tools/kcsan.rst> for more details.
14
15if KCSAN
16
17config KCSAN_DEBUG
18 bool "Debugging of KCSAN internals"
19 default n
20
21config KCSAN_SELFTEST
22 bool "Perform short selftests on boot"
23 default y
24 help
25 Run KCSAN selftests on boot. On test failure, causes kernel to panic.
26
27config KCSAN_EARLY_ENABLE
28 bool "Early enable during boot"
29 default y
30 help
31 If KCSAN should be enabled globally as soon as possible. KCSAN can
32 later be enabled/disabled via debugfs.
33
34config KCSAN_NUM_WATCHPOINTS
35 int "Number of available watchpoints"
36 default 64
37 help
38 Total number of available watchpoints. An address range maps into a
39 specific watchpoint slot as specified in kernel/kcsan/encoding.h.
40 Although larger number of watchpoints may not be usable due to
41 limited number of CPUs, a larger value helps to improve performance
42 due to reducing cache-line contention. The chosen default is a
43 conservative value; we should almost never observe "no_capacity"
44 events (see /sys/kernel/debug/kcsan).
45
46config KCSAN_UDELAY_TASK
47 int "Delay in microseconds (for tasks)"
48 default 80
49 help
50 For tasks, the microsecond delay after setting up a watchpoint.
51
52config KCSAN_UDELAY_INTERRUPT
53 int "Delay in microseconds (for interrupts)"
54 default 20
55 help
56 For interrupts, the microsecond delay after setting up a watchpoint.
57 Interrupts have tighter latency requirements, and their delay should
58 be lower than for tasks.
59
60config KCSAN_DELAY_RANDOMIZE
61 bool "Randomize above delays"
62 default y
63 help
64 If delays should be randomized, where the maximum is KCSAN_UDELAY_*.
65 If false, the chosen delays are always KCSAN_UDELAY_* defined above.
66
67config KCSAN_SKIP_WATCH
68 int "Skip instructions before setting up watchpoint"
69 default 4000
70 help
71 The number of per-CPU memory operations to skip, before another
72 watchpoint is set up, i.e. one in KCSAN_WATCH_SKIP per-CPU
73 memory operations are used to set up a watchpoint. A smaller value
74 results in more aggressive race detection, whereas a larger value
75 improves system performance at the cost of missing some races.
76
77config KCSAN_SKIP_WATCH_RANDOMIZE
78 bool "Randomize watchpoint instruction skip count"
79 default y
80 help
81 If instruction skip count should be randomized, where the maximum is
82 KCSAN_WATCH_SKIP. If false, the chosen value is always
83 KCSAN_WATCH_SKIP.
84
85# Note that, while some of the below options could be turned into boot
86# parameters, to optimize for the common use-case, we avoid this because: (a)
87# it would impact performance (and we want to avoid static branch for all
88# {READ,WRITE}_ONCE, atomic_*, bitops, etc.), and (b) complicate the design
89# without real benefit. The main purpose of the below options are for use in
90# fuzzer configs to control reported data races, and are not expected to be
91# switched frequently by a user.
92
93config KCSAN_REPORT_RACE_UNKNOWN_ORIGIN
94 bool "Report races of unknown origin"
95 default y
96 help
97 If KCSAN should report races where only one access is known, and the
98 conflicting access is of unknown origin. This type of race is
99 reported if it was only possible to infer a race due to a data value
100 change while an access is being delayed on a watchpoint.
101
102config KCSAN_REPORT_VALUE_CHANGE_ONLY
103 bool "Only report races where watcher observed a data value change"
104 default y
105 help
106 If enabled and a conflicting write is observed via watchpoint, but
107 the data value of the memory location was observed to remain
108 unchanged, do not report the data race.
109
110config KCSAN_IGNORE_ATOMICS
111 bool "Do not instrument marked atomic accesses"
112 default n
113 help
114 If enabled, never instruments marked atomic accesses. This results in
115 not reporting data races where one access is atomic and the other is
116 a plain access.
117
118endif # KCSAN
lib/Makefile+3−0
@@ -24,6 +24,9 @@ KASAN_SANITIZE_string.o := n · 23 unmodified lines
24CFLAGS_string.o := $(call cc-option, -fno-stack-protector)24CFLAGS_string.o := $(call cc-option, -fno-stack-protector)
25endif25endif
26 26
27# Used by KCSAN while enabled, avoid recursion.
28KCSAN_SANITIZE_random32.o := n
29
27lib-y := ctype.o string.o vsprintf.o cmdline.o \30lib-y := ctype.o string.o vsprintf.o cmdline.o \
28 rbtree.o radix-tree.o timerqueue.o xarray.o \31 rbtree.o radix-tree.o timerqueue.o xarray.o \
29 idr.o extable.o \32 idr.o extable.o \
scripts/Makefile.kcsan+6−0
@@ -0,0 +1,6 @@
1# SPDX-License-Identifier: GPL-2.0
2ifdef CONFIG_KCSAN
3
4CFLAGS_KCSAN := -fsanitize=thread
5
6endif # CONFIG_KCSAN
scripts/Makefile.lib+10−0
@@ -152,6 +152,16 @@ _c_flags += $(if $(patsubst n%,, \ · 151 unmodified lines
152 $(CFLAGS_KCOV))152 $(CFLAGS_KCOV))
153endif153endif
154 154
155#
156# Enable KCSAN flags except some files or directories we don't want to check
157# (depends on variables KCSAN_SANITIZE_obj.o, KCSAN_SANITIZE)
158#
159ifeq ($(CONFIG_KCSAN),y)
160_c_flags += $(if $(patsubst n%,, \
161 $(KCSAN_SANITIZE_$(basetarget).o)$(KCSAN_SANITIZE)y), \
162 $(CFLAGS_KCSAN))
163endif
164
155# $(srctree)/$(src) for including checkin headers from generated source files165# $(srctree)/$(src) for including checkin headers from generated source files
156# $(objtree)/$(obj) for including generated headers from checkin source files166# $(objtree)/$(obj) for including generated headers from checkin source files
157ifeq ($(KBUILD_EXTMOD),)167ifeq ($(KBUILD_EXTMOD),)