kcsan: Add Kernel Concurrency Sanitizer infrastructure

Kernel Concurrency Sanitizer (KCSAN) is a dynamic data-race detector for kernel space. KCSAN is a sampling watchpoint-based data-race detector. See the included Documentation/dev-tools/kcsan.rst for more details. This patch adds basic infrastructure, but does not yet enable KCSAN for any architecture.

Marco Elver committed 7 years ago

Commit dfd402a1 parent 31f4f5b24 files · +2006 −9Browse files

Unified · Split

24 files

MAINTAINERS +11 −0
Makefile +2 −1
include/linux/compiler-clang.h +9 −0
include/linux/compiler-gcc.h +7 −0
include/linux/compiler.h +29 −8
include/linux/kcsan-checks.h +97 −0
include/linux/kcsan.h +115 −0
include/linux/sched.h +4 −0
init/init_task.c +8 −0
init/main.c +2 −0
kernel/Makefile +1 −0
kernel/kcsan/Makefile +11 −0
kernel/kcsan/atomic.h +27 −0
kernel/kcsan/core.c +626 −0
kernel/kcsan/debugfs.c +275 −0
kernel/kcsan/encoding.h +94 −0
kernel/kcsan/kcsan.h +108 −0
kernel/kcsan/report.c +320 −0
kernel/kcsan/test.c +121 −0
lib/Kconfig.debug +2 −0
lib/Kconfig.kcsan +118 −0
lib/Makefile +3 −0
scripts/Makefile.kcsan +6 −0
scripts/Makefile.lib +10 −0

MAINTAINERS+11−0

@@ -8848,6 +8848,17 @@ F: Documentation/kbuild/kconfig* · 8847 unmodified lines
8848	8848	F: scripts/kconfig/
8849	8849	F: scripts/Kconfig.include
8850	8850
	8851	+KCSAN
	8852	+M: Marco Elver <elver@google.com>
	8853	+R: Dmitry Vyukov <dvyukov@google.com>
	8854	+L: kasan-dev@googlegroups.com
	8855	+S: Maintained
	8856	+F: Documentation/dev-tools/kcsan.rst
	8857	+F: include/linux/kcsan*.h
	8858	+F: kernel/kcsan/
	8859	+F: lib/Kconfig.kcsan
	8860	+F: scripts/Makefile.kcsan
	8861	+
8851	8862	KDUMP
8852	8863	M: Dave Young <dyoung@redhat.com>
8853	8864	M: Baoquan He <bhe@redhat.com>

Makefile+2−1

@@ -478,7 +478,7 @@ export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE · 477 unmodified lines
478	478
479	479	export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS
480	480	export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE
481		−export CFLAGS_KASAN CFLAGS_KASAN_NOSANITIZE CFLAGS_UBSAN
	481	+export CFLAGS_KASAN CFLAGS_KASAN_NOSANITIZE CFLAGS_UBSAN CFLAGS_KCSAN
482	482	export KBUILD_AFLAGS AFLAGS_KERNEL AFLAGS_MODULE
483	483	export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_LDFLAGS_MODULE
484	484	export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL
@@ -900,6 +900,7 @@ endif · 415 unmodified lines
900	900	include scripts/Makefile.kasan
901	901	include scripts/Makefile.extrawarn
902	902	include scripts/Makefile.ubsan
	903	+include scripts/Makefile.kcsan
903	904
904	905	# Add user supplied CPPFLAGS, AFLAGS and CFLAGS as the last assignments
905	906	KBUILD_CPPFLAGS += $(KCPPFLAGS)

include/linux/compiler-clang.h+9−0

@@ -24,6 +24,15 @@ · 23 unmodified lines
24	24	#define __no_sanitize_address
25	25	#endif
26	26
	27	+#if __has_feature(thread_sanitizer)
	28	+/* emulate gcc's __SANITIZE_THREAD__ flag */
	29	+#define __SANITIZE_THREAD__
	30	+#define __no_sanitize_thread \
	31	+ __attribute__((no_sanitize("thread")))
	32	+#else
	33	+#define __no_sanitize_thread
	34	+#endif
	35	+
27	36	/*
28	37	* Not all versions of clang implement the the type-generic versions
29	38	* of the builtin overflow checkers. Fortunately, clang implements

include/linux/compiler-gcc.h+7−0

@@ -145,6 +145,13 @@ · 144 unmodified lines
145	145	#define __no_sanitize_address
146	146	#endif
147	147
	148	+#if defined(__SANITIZE_THREAD__) && __has_attribute(__no_sanitize_thread__)
	149	+#define __no_sanitize_thread \
	150	+ __attribute__((__noinline__)) __attribute__((no_sanitize_thread))
	151	+#else
	152	+#define __no_sanitize_thread
	153	+#endif
	154	+
148	155	#if GCC_VERSION >= 50100
149	156	#define COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW 1
150	157	#endif

include/linux/compiler.h+29−8

@@ -178,6 +178,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 177 unmodified lines
178	178	#endif
179	179
180	180	#include <uapi/linux/types.h>
	181	+#include <linux/kcsan-checks.h>
181	182
182	183	#define __READ_ONCE_SIZE \
183	184	({ \
@@ -193,12 +194,6 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 9 unmodified lines
193	194	} \
194	195	})
195	196
196		−static __always_inline
197		−void __read_once_size(const volatile void p, void res, int size)
198		−{
199		− __READ_ONCE_SIZE;
200		−}
201		−
202	197	#ifdef CONFIG_KASAN
203	198	/*
204	199	* We can't declare function 'inline' because __no_sanitize_address confilcts
@@ -207,18 +202,44 @@ void __read_once_size(const volatile void p, void res, int size) · 2 unmodified lines
207	202	* '__maybe_unused' allows us to avoid defined-but-not-used warnings.
208	203	*/
209	204	# define __no_kasan_or_inline __no_sanitize_address notrace __maybe_unused
	205	+# define __no_sanitize_or_inline __no_kasan_or_inline
210	206	#else
211	207	# define __no_kasan_or_inline __always_inline
212	208	#endif
213	209
214		−static __no_kasan_or_inline
	210	+#ifdef __SANITIZE_THREAD__
	211	+/*
	212	+ * Rely on __SANITIZE_THREAD__ instead of CONFIG_KCSAN, to avoid not inlining in
	213	+ * compilation units where instrumentation is disabled.
	214	+ */
	215	+# define __no_kcsan_or_inline __no_sanitize_thread notrace __maybe_unused
	216	+# define __no_sanitize_or_inline __no_kcsan_or_inline
	217	+#else
	218	+# define __no_kcsan_or_inline __always_inline
	219	+#endif
	220	+
	221	+#ifndef __no_sanitize_or_inline
	222	+#define __no_sanitize_or_inline __always_inline
	223	+#endif
	224	+
	225	+static __no_kcsan_or_inline
	226	+void __read_once_size(const volatile void p, void res, int size)
	227	+{
	228	+ kcsan_check_atomic_read(p, size);
	229	+ __READ_ONCE_SIZE;
	230	+}
	231	+
	232	+static __no_sanitize_or_inline
215	233	void __read_once_size_nocheck(const volatile void p, void res, int size)
216	234	{
217	235	__READ_ONCE_SIZE;
218	236	}
219	237
220		−static __always_inline void __write_once_size(volatile void p, void res, int size)
	238	+static __no_kcsan_or_inline
	239	+void __write_once_size(volatile void p, void res, int size)
221	240	{
	241	+ kcsan_check_atomic_write(p, size);
	242	+
222	243	switch (size) {
223	244	case 1: (volatile __u8 )p = (__u8 )res; break;
224	245	case 2: (volatile __u16 )p = (__u16 )res; break;

include/linux/kcsan-checks.h+97−0

@@ -0,0 +1,97 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+
	3	+#ifndef _LINUX_KCSAN_CHECKS_H
	4	+#define _LINUX_KCSAN_CHECKS_H
	5	+
	6	+#include <linux/types.h>
	7	+
	8	+/*
	9	+ * Access type modifiers.
	10	+ */
	11	+#define KCSAN_ACCESS_WRITE 0x1
	12	+#define KCSAN_ACCESS_ATOMIC 0x2
	13	+
	14	+/*
	15	+ * __kcsan_*: Always calls into runtime when KCSAN is enabled. This may be used
	16	+ * even in compilation units that selectively disable KCSAN, but must use KCSAN
	17	+ * to validate access to an address. Never use these in header files!
	18	+ */
	19	+#ifdef CONFIG_KCSAN
	20	+/**
	21	+ * __kcsan_check_access - check generic access for data race
	22	+ *
	23	+ * @ptr address of access
	24	+ * @size size of access
	25	+ * @type access type modifier
	26	+ */
	27	+void __kcsan_check_access(const volatile void *ptr, size_t size, int type);
	28	+
	29	+#else
	30	+static inline void __kcsan_check_access(const volatile void *ptr, size_t size,
	31	+ int type) { }
	32	+#endif
	33	+
	34	+/*
	35	+ * kcsan_*: Only calls into runtime when the particular compilation unit has
	36	+ * KCSAN instrumentation enabled. May be used in header files.
	37	+ */
	38	+#ifdef __SANITIZE_THREAD__
	39	+#define kcsan_check_access __kcsan_check_access
	40	+#else
	41	+static inline void kcsan_check_access(const volatile void *ptr, size_t size,
	42	+ int type) { }
	43	+#endif
	44	+
	45	+/**
	46	+ * __kcsan_check_read - check regular read access for data races
	47	+ *
	48	+ * @ptr address of access
	49	+ * @size size of access
	50	+ */
	51	+#define __kcsan_check_read(ptr, size) __kcsan_check_access(ptr, size, 0)
	52	+
	53	+/**
	54	+ * __kcsan_check_write - check regular write access for data races
	55	+ *
	56	+ * @ptr address of access
	57	+ * @size size of access
	58	+ */
	59	+#define __kcsan_check_write(ptr, size) \
	60	+ __kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
	61	+
	62	+/**
	63	+ * kcsan_check_read - check regular read access for data races
	64	+ *
	65	+ * @ptr address of access
	66	+ * @size size of access
	67	+ */
	68	+#define kcsan_check_read(ptr, size) kcsan_check_access(ptr, size, 0)
	69	+
	70	+/**
	71	+ * kcsan_check_write - check regular write access for data races
	72	+ *
	73	+ * @ptr address of access
	74	+ * @size size of access
	75	+ */
	76	+#define kcsan_check_write(ptr, size) \
	77	+ kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
	78	+
	79	+/*
	80	+ * Check for atomic accesses: if atomic access are not ignored, this simply
	81	+ * aliases to kcsan_check_access, otherwise becomes a no-op.
	82	+ */
	83	+#ifdef CONFIG_KCSAN_IGNORE_ATOMICS
	84	+#define kcsan_check_atomic_read(...) \
	85	+ do { \
	86	+ } while (0)
	87	+#define kcsan_check_atomic_write(...) \
	88	+ do { \
	89	+ } while (0)
	90	+#else
	91	+#define kcsan_check_atomic_read(ptr, size) \
	92	+ kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC)
	93	+#define kcsan_check_atomic_write(ptr, size) \
	94	+ kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC \| KCSAN_ACCESS_WRITE)
	95	+#endif
	96	+
	97	+#endif /* _LINUX_KCSAN_CHECKS_H */

include/linux/kcsan.h+115−0

@@ -0,0 +1,115 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+
	3	+#ifndef _LINUX_KCSAN_H
	4	+#define _LINUX_KCSAN_H
	5	+
	6	+#include <linux/kcsan-checks.h>
	7	+#include <linux/types.h>
	8	+
	9	+#ifdef CONFIG_KCSAN
	10	+
	11	+/*
	12	+ * Context for each thread of execution: for tasks, this is stored in
	13	+ * task_struct, and interrupts access internal per-CPU storage.
	14	+ */
	15	+struct kcsan_ctx {
	16	+ int disable_count; /* disable counter */
	17	+ int atomic_next; /* number of following atomic ops */
	18	+
	19	+ /*
	20	+ * We distinguish between: (a) nestable atomic regions that may contain
	21	+ * other nestable regions; and (b) flat atomic regions that do not keep
	22	+ * track of nesting. Both (a) and (b) are entirely independent of each
	23	+ * other, and a flat region may be started in a nestable region or
	24	+ * vice-versa.
	25	+ *
	26	+ * This is required because, for example, in the annotations for
	27	+ * seqlocks, we declare seqlock writer critical sections as (a) nestable
	28	+ * atomic regions, but reader critical sections as (b) flat atomic
	29	+ * regions, but have encountered cases where seqlock reader critical
	30	+ * sections are contained within writer critical sections (the opposite
	31	+ * may be possible, too).
	32	+ *
	33	+ * To support these cases, we independently track the depth of nesting
	34	+ * for (a), and whether the leaf level is flat for (b).
	35	+ */
	36	+ int atomic_nest_count;
	37	+ bool in_flat_atomic;
	38	+};
	39	+
	40	+/**
	41	+ * kcsan_init - initialize KCSAN runtime
	42	+ */
	43	+void kcsan_init(void);
	44	+
	45	+/**
	46	+ * kcsan_disable_current - disable KCSAN for the current context
	47	+ *
	48	+ * Supports nesting.
	49	+ */
	50	+void kcsan_disable_current(void);
	51	+
	52	+/**
	53	+ * kcsan_enable_current - re-enable KCSAN for the current context
	54	+ *
	55	+ * Supports nesting.
	56	+ */
	57	+void kcsan_enable_current(void);
	58	+
	59	+/**
	60	+ * kcsan_nestable_atomic_begin - begin nestable atomic region
	61	+ *
	62	+ * Accesses within the atomic region may appear to race with other accesses but
	63	+ * should be considered atomic.
	64	+ */
	65	+void kcsan_nestable_atomic_begin(void);
	66	+
	67	+/**
	68	+ * kcsan_nestable_atomic_end - end nestable atomic region
	69	+ */
	70	+void kcsan_nestable_atomic_end(void);
	71	+
	72	+/**
	73	+ * kcsan_flat_atomic_begin - begin flat atomic region
	74	+ *
	75	+ * Accesses within the atomic region may appear to race with other accesses but
	76	+ * should be considered atomic.
	77	+ */
	78	+void kcsan_flat_atomic_begin(void);
	79	+
	80	+/**
	81	+ * kcsan_flat_atomic_end - end flat atomic region
	82	+ */
	83	+void kcsan_flat_atomic_end(void);
	84	+
	85	+/**
	86	+ * kcsan_atomic_next - consider following accesses as atomic
	87	+ *
	88	+ * Force treating the next n memory accesses for the current context as atomic
	89	+ * operations.
	90	+ *
	91	+ * @n number of following memory accesses to treat as atomic.
	92	+ */
	93	+void kcsan_atomic_next(int n);
	94	+
	95	+#else /* CONFIG_KCSAN */
	96	+
	97	+static inline void kcsan_init(void) { }
	98	+
	99	+static inline void kcsan_disable_current(void) { }
	100	+
	101	+static inline void kcsan_enable_current(void) { }
	102	+
	103	+static inline void kcsan_nestable_atomic_begin(void) { }
	104	+
	105	+static inline void kcsan_nestable_atomic_end(void) { }
	106	+
	107	+static inline void kcsan_flat_atomic_begin(void) { }
	108	+
	109	+static inline void kcsan_flat_atomic_end(void) { }
	110	+
	111	+static inline void kcsan_atomic_next(int n) { }
	112	+
	113	+#endif /* CONFIG_KCSAN */
	114	+
	115	+#endif /* _LINUX_KCSAN_H */

include/linux/sched.h+4−0

@@ -31,6 +31,7 @@ · 30 unmodified lines
31	31	#include <linux/task_io_accounting.h>
32	32	#include <linux/posix-timers.h>
33	33	#include <linux/rseq.h>
	34	+#include <linux/kcsan.h>
34	35
35	36	/* task_struct member predeclarations (sorted alphabetically): */
36	37	struct audit_context;
@@ -1172,6 +1173,9 @@ struct task_struct { · 1135 unmodified lines
1172	1173	#ifdef CONFIG_KASAN
1173	1174	unsigned int kasan_depth;
1174	1175	#endif
	1176	+#ifdef CONFIG_KCSAN
	1177	+ struct kcsan_ctx kcsan_ctx;
	1178	+#endif
1175	1179
1176	1180	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1177	1181	/* Index of current stored address in ret_stack: */

init/init_task.c+8−0

@@ -161,6 +161,14 @@ struct task_struct init_task · 160 unmodified lines
161	161	#ifdef CONFIG_KASAN
162	162	.kasan_depth = 1,
163	163	#endif
	164	+#ifdef CONFIG_KCSAN
	165	+ .kcsan_ctx = {
	166	+ .disable_count = 0,
	167	+ .atomic_next = 0,
	168	+ .atomic_nest_count = 0,
	169	+ .in_flat_atomic = false,
	170	+ },
	171	+#endif
164	172	#ifdef CONFIG_TRACE_IRQFLAGS
165	173	.softirqs_enabled = 1,
166	174	#endif

init/main.c+2−0

@@ -93,6 +93,7 @@ · 92 unmodified lines
93	93	#include <linux/rodata_test.h>
94	94	#include <linux/jump_label.h>
95	95	#include <linux/mem_encrypt.h>
	96	+#include <linux/kcsan.h>
96	97
97	98	#include <asm/io.h>
98	99	#include <asm/bugs.h>
@@ -779,6 +780,7 @@ asmlinkage __visible void __init start_kernel(void) · 680 unmodified lines
779	780	acpi_subsystem_init();
780	781	arch_post_acpi_subsys_init();
781	782	sfi_init_late();
	783	+ kcsan_init();
782	784
783	785	/* Do the rest non-__init'ed, we're now alive */
784	786	arch_call_rest_init();

kernel/Makefile+1−0

@@ -102,6 +102,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/ · 101 unmodified lines
102	102	obj-$(CONFIG_IRQ_WORK) += irq_work.o
103	103	obj-$(CONFIG_CPU_PM) += cpu_pm.o
104	104	obj-$(CONFIG_BPF) += bpf/
	105	+obj-$(CONFIG_KCSAN) += kcsan/
105	106
106	107	obj-$(CONFIG_PERF_EVENTS) += events/
107	108

kernel/kcsan/Makefile+11−0

@@ -0,0 +1,11 @@
	1	+# SPDX-License-Identifier: GPL-2.0
	2	+KCSAN_SANITIZE := n
	3	+KCOV_INSTRUMENT := n
	4	+
	5	+CFLAGS_REMOVE_core.o = $(CC_FLAGS_FTRACE)
	6	+
	7	+CFLAGS_core.o := $(call cc-option,-fno-conserve-stack,) \
	8	+ $(call cc-option,-fno-stack-protector,)
	9	+
	10	+obj-y := core.o debugfs.o report.o
	11	+obj-$(CONFIG_KCSAN_SELFTEST) += test.o

kernel/kcsan/atomic.h+27−0

@@ -0,0 +1,27 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+
	3	+#ifndef _KERNEL_KCSAN_ATOMIC_H
	4	+#define _KERNEL_KCSAN_ATOMIC_H
	5	+
	6	+#include <linux/jiffies.h>
	7	+
	8	+/*
	9	+ * Helper that returns true if access to ptr should be considered as an atomic
	10	+ * access, even though it is not explicitly atomic.
	11	+ *
	12	+ * List all volatile globals that have been observed in races, to suppress
	13	+ * data race reports between accesses to these variables.
	14	+ *
	15	+ * For now, we assume that volatile accesses of globals are as strong as atomic
	16	+ * accesses (READ_ONCE, WRITE_ONCE cast to volatile). The situation is still not
	17	+ * entirely clear, as on some architectures (Alpha) READ_ONCE/WRITE_ONCE do more
	18	+ * than cast to volatile. Eventually, we hope to be able to remove this
	19	+ * function.
	20	+ */
	21	+static inline bool kcsan_is_atomic(const volatile void *ptr)
	22	+{
	23	+ /* only jiffies for now */
	24	+ return ptr == &jiffies;
	25	+}
	26	+
	27	+#endif /* _KERNEL_KCSAN_ATOMIC_H */

kernel/kcsan/core.c+626−0

@@ -0,0 +1,626 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include <linux/atomic.h>
	4	+#include <linux/bug.h>
	5	+#include <linux/delay.h>
	6	+#include <linux/export.h>
	7	+#include <linux/init.h>
	8	+#include <linux/percpu.h>
	9	+#include <linux/preempt.h>
	10	+#include <linux/random.h>
	11	+#include <linux/sched.h>
	12	+#include <linux/uaccess.h>
	13	+
	14	+#include "atomic.h"
	15	+#include "encoding.h"
	16	+#include "kcsan.h"
	17	+
	18	+bool kcsan_enabled;
	19	+
	20	+/* Per-CPU kcsan_ctx for interrupts */
	21	+static DEFINE_PER_CPU(struct kcsan_ctx, kcsan_cpu_ctx) = {
	22	+ .disable_count = 0,
	23	+ .atomic_next = 0,
	24	+ .atomic_nest_count = 0,
	25	+ .in_flat_atomic = false,
	26	+};
	27	+
	28	+/*
	29	+ * Helper macros to index into adjacent slots slots, starting from address slot
	30	+ * itself, followed by the right and left slots.
	31	+ *
	32	+ * The purpose is 2-fold:
	33	+ *
	34	+ * 1. if during insertion the address slot is already occupied, check if
	35	+ * any adjacent slots are free;
	36	+ * 2. accesses that straddle a slot boundary due to size that exceeds a
	37	+ * slot's range may check adjacent slots if any watchpoint matches.
	38	+ *
	39	+ * Note that accesses with very large size may still miss a watchpoint; however,
	40	+ * given this should be rare, this is a reasonable trade-off to make, since this
	41	+ * will avoid:
	42	+ *
	43	+ * 1. excessive contention between watchpoint checks and setup;
	44	+ * 2. larger number of simultaneous watchpoints without sacrificing
	45	+ * performance.
	46	+ *
	47	+ * Example: SLOT_IDX values for KCSAN_CHECK_ADJACENT=1, where i is [0, 1, 2]:
	48	+ *
	49	+ * slot=0: [ 1, 2, 0]
	50	+ * slot=9: [10, 11, 9]
	51	+ * slot=63: [64, 65, 63]
	52	+ */
	53	+#define NUM_SLOTS (1 + 2 * KCSAN_CHECK_ADJACENT)
	54	+#define SLOT_IDX(slot, i) (slot + ((i + KCSAN_CHECK_ADJACENT) % NUM_SLOTS))
	55	+
	56	+/*
	57	+ * SLOT_IDX_FAST is used in fast-path. Not first checking the address's primary
	58	+ * slot (middle) is fine if we assume that data races occur rarely. The set of
	59	+ * indices {SLOT_IDX(slot, i) \| i in [0, NUM_SLOTS)} is equivalent to
	60	+ * {SLOT_IDX_FAST(slot, i) \| i in [0, NUM_SLOTS)}.
	61	+ */
	62	+#define SLOT_IDX_FAST(slot, i) (slot + i)
	63	+
	64	+/*
	65	+ * Watchpoints, with each entry encoded as defined in encoding.h: in order to be
	66	+ * able to safely update and access a watchpoint without introducing locking
	67	+ * overhead, we encode each watchpoint as a single atomic long. The initial
	68	+ * zero-initialized state matches INVALID_WATCHPOINT.
	69	+ *
	70	+ * Add NUM_SLOTS-1 entries to account for overflow; this helps avoid having to
	71	+ * use more complicated SLOT_IDX_FAST calculation with modulo in fast-path.
	72	+ */
	73	+static atomic_long_t watchpoints[CONFIG_KCSAN_NUM_WATCHPOINTS + NUM_SLOTS - 1];
	74	+
	75	+/*
	76	+ * Instructions to skip watching counter, used in should_watch(). We use a
	77	+ * per-CPU counter to avoid excessive contention.
	78	+ */
	79	+static DEFINE_PER_CPU(long, kcsan_skip);
	80	+
	81	+static inline atomic_long_t *find_watchpoint(unsigned long addr, size_t size,
	82	+ bool expect_write,
	83	+ long *encoded_watchpoint)
	84	+{
	85	+ const int slot = watchpoint_slot(addr);
	86	+ const unsigned long addr_masked = addr & WATCHPOINT_ADDR_MASK;
	87	+ atomic_long_t *watchpoint;
	88	+ unsigned long wp_addr_masked;
	89	+ size_t wp_size;
	90	+ bool is_write;
	91	+ int i;
	92	+
	93	+ BUILD_BUG_ON(CONFIG_KCSAN_NUM_WATCHPOINTS < NUM_SLOTS);
	94	+
	95	+ for (i = 0; i < NUM_SLOTS; ++i) {
	96	+ watchpoint = &watchpoints[SLOT_IDX_FAST(slot, i)];
	97	+ *encoded_watchpoint = atomic_long_read(watchpoint);
	98	+ if (!decode_watchpoint(*encoded_watchpoint, &wp_addr_masked,
	99	+ &wp_size, &is_write))
	100	+ continue;
	101	+
	102	+ if (expect_write && !is_write)
	103	+ continue;
	104	+
	105	+ /* Check if the watchpoint matches the access. */
	106	+ if (matching_access(wp_addr_masked, wp_size, addr_masked, size))
	107	+ return watchpoint;
	108	+ }
	109	+
	110	+ return NULL;
	111	+}
	112	+
	113	+static inline atomic_long_t *insert_watchpoint(unsigned long addr, size_t size,
	114	+ bool is_write)
	115	+{
	116	+ const int slot = watchpoint_slot(addr);
	117	+ const long encoded_watchpoint = encode_watchpoint(addr, size, is_write);
	118	+ atomic_long_t *watchpoint;
	119	+ int i;
	120	+
	121	+ /* Check slot index logic, ensuring we stay within array bounds. */
	122	+ BUILD_BUG_ON(SLOT_IDX(0, 0) != KCSAN_CHECK_ADJACENT);
	123	+ BUILD_BUG_ON(SLOT_IDX(0, KCSAN_CHECK_ADJACENT + 1) != 0);
	124	+ BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS - 1,
	125	+ KCSAN_CHECK_ADJACENT) !=
	126	+ ARRAY_SIZE(watchpoints) - 1);
	127	+ BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS - 1,
	128	+ KCSAN_CHECK_ADJACENT + 1) !=
	129	+ ARRAY_SIZE(watchpoints) - NUM_SLOTS);
	130	+
	131	+ for (i = 0; i < NUM_SLOTS; ++i) {
	132	+ long expect_val = INVALID_WATCHPOINT;
	133	+
	134	+ /* Try to acquire this slot. */
	135	+ watchpoint = &watchpoints[SLOT_IDX(slot, i)];
	136	+ if (atomic_long_try_cmpxchg_relaxed(watchpoint, &expect_val,
	137	+ encoded_watchpoint))
	138	+ return watchpoint;
	139	+ }
	140	+
	141	+ return NULL;
	142	+}
	143	+
	144	+/*
	145	+ * Return true if watchpoint was successfully consumed, false otherwise.
	146	+ *
	147	+ * This may return false if:
	148	+ *
	149	+ * 1. another thread already consumed the watchpoint;
	150	+ * 2. the thread that set up the watchpoint already removed it;
	151	+ * 3. the watchpoint was removed and then re-used.
	152	+ */
	153	+static inline bool try_consume_watchpoint(atomic_long_t *watchpoint,
	154	+ long encoded_watchpoint)
	155	+{
	156	+ return atomic_long_try_cmpxchg_relaxed(watchpoint, &encoded_watchpoint,
	157	+ CONSUMED_WATCHPOINT);
	158	+}
	159	+
	160	+/*
	161	+ * Return true if watchpoint was not touched, false if consumed.
	162	+ */
	163	+static inline bool remove_watchpoint(atomic_long_t *watchpoint)
	164	+{
	165	+ return atomic_long_xchg_relaxed(watchpoint, INVALID_WATCHPOINT) !=
	166	+ CONSUMED_WATCHPOINT;
	167	+}
	168	+
	169	+static inline struct kcsan_ctx *get_ctx(void)
	170	+{
	171	+ /*
	172	+ * In interrupt, use raw_cpu_ptr to avoid unnecessary checks, that would
	173	+ * also result in calls that generate warnings in uaccess regions.
	174	+ */
	175	+ return in_task() ? &current->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx);
	176	+}
	177	+
	178	+static inline bool is_atomic(const volatile void *ptr)
	179	+{
	180	+ struct kcsan_ctx *ctx = get_ctx();
	181	+
	182	+ if (unlikely(ctx->atomic_next > 0)) {
	183	+ /*
	184	+ * Because we do not have separate contexts for nested
	185	+ * interrupts, in case atomic_next is set, we simply assume that
	186	+ * the outer interrupt set atomic_next. In the worst case, we
	187	+ * will conservatively consider operations as atomic. This is a
	188	+ * reasonable trade-off to make, since this case should be
	189	+ * extremely rare; however, even if extremely rare, it could
	190	+ * lead to false positives otherwise.
	191	+ */
	192	+ if ((hardirq_count() >> HARDIRQ_SHIFT) < 2)
	193	+ --ctx->atomic_next; /* in task, or outer interrupt */
	194	+ return true;
	195	+ }
	196	+ if (unlikely(ctx->atomic_nest_count > 0 \|\| ctx->in_flat_atomic))
	197	+ return true;
	198	+
	199	+ return kcsan_is_atomic(ptr);
	200	+}
	201	+
	202	+static inline bool should_watch(const volatile void *ptr, int type)
	203	+{
	204	+ /*
	205	+ * Never set up watchpoints when memory operations are atomic.
	206	+ *
	207	+ * Need to check this first, before kcsan_skip check below: (1) atomics
	208	+ * should not count towards skipped instructions, and (2) to actually
	209	+ * decrement kcsan_atomic_next for consecutive instruction stream.
	210	+ */
	211	+ if ((type & KCSAN_ACCESS_ATOMIC) != 0 \|\| is_atomic(ptr))
	212	+ return false;
	213	+
	214	+ if (this_cpu_dec_return(kcsan_skip) >= 0)
	215	+ return false;
	216	+
	217	+ /*
	218	+ * NOTE: If we get here, kcsan_skip must always be reset in slow path
	219	+ * via reset_kcsan_skip() to avoid underflow.
	220	+ */
	221	+
	222	+ /* this operation should be watched */
	223	+ return true;
	224	+}
	225	+
	226	+static inline void reset_kcsan_skip(void)
	227	+{
	228	+ long skip_count = CONFIG_KCSAN_SKIP_WATCH -
	229	+ (IS_ENABLED(CONFIG_KCSAN_SKIP_WATCH_RANDOMIZE) ?
	230	+ prandom_u32_max(CONFIG_KCSAN_SKIP_WATCH) :
	231	+ 0);
	232	+ this_cpu_write(kcsan_skip, skip_count);
	233	+}
	234	+
	235	+static inline bool kcsan_is_enabled(void)
	236	+{
	237	+ return READ_ONCE(kcsan_enabled) && get_ctx()->disable_count == 0;
	238	+}
	239	+
	240	+static inline unsigned int get_delay(void)
	241	+{
	242	+ unsigned int delay = in_task() ? CONFIG_KCSAN_UDELAY_TASK :
	243	+ CONFIG_KCSAN_UDELAY_INTERRUPT;
	244	+ return delay - (IS_ENABLED(CONFIG_KCSAN_DELAY_RANDOMIZE) ?
	245	+ prandom_u32_max(delay) :
	246	+ 0);
	247	+}
	248	+
	249	+/*
	250	+ * Pull everything together: check_access() below contains the performance
	251	+ * critical operations; the fast-path (including check_access) functions should
	252	+ * all be inlinable by the instrumentation functions.
	253	+ *
	254	+ * The slow-path (kcsan_found_watchpoint, kcsan_setup_watchpoint) are
	255	+ * non-inlinable -- note that, we prefix these with "kcsan_" to ensure they can
	256	+ * be filtered from the stacktrace, as well as give them unique names for the
	257	+ * UACCESS whitelist of objtool. Each function uses user_access_save/restore(),
	258	+ * since they do not access any user memory, but instrumentation is still
	259	+ * emitted in UACCESS regions.
	260	+ */
	261	+
	262	+static noinline void kcsan_found_watchpoint(const volatile void *ptr,
	263	+ size_t size, bool is_write,
	264	+ atomic_long_t *watchpoint,
	265	+ long encoded_watchpoint)
	266	+{
	267	+ unsigned long flags;
	268	+ bool consumed;
	269	+
	270	+ if (!kcsan_is_enabled())
	271	+ return;
	272	+ /*
	273	+ * Consume the watchpoint as soon as possible, to minimize the chances
	274	+ * of !consumed. Consuming the watchpoint must always be guarded by
	275	+ * kcsan_is_enabled() check, as otherwise we might erroneously
	276	+ * triggering reports when disabled.
	277	+ */
	278	+ consumed = try_consume_watchpoint(watchpoint, encoded_watchpoint);
	279	+
	280	+ /* keep this after try_consume_watchpoint */
	281	+ flags = user_access_save();
	282	+
	283	+ if (consumed) {
	284	+ kcsan_report(ptr, size, is_write, true, raw_smp_processor_id(),
	285	+ KCSAN_REPORT_CONSUMED_WATCHPOINT);
	286	+ } else {
	287	+ /*
	288	+ * The other thread may not print any diagnostics, as it has
	289	+ * already removed the watchpoint, or another thread consumed
	290	+ * the watchpoint before this thread.
	291	+ */
	292	+ kcsan_counter_inc(KCSAN_COUNTER_REPORT_RACES);
	293	+ }
	294	+ kcsan_counter_inc(KCSAN_COUNTER_DATA_RACES);
	295	+
	296	+ user_access_restore(flags);
	297	+}
	298	+
	299	+static noinline void kcsan_setup_watchpoint(const volatile void *ptr,
	300	+ size_t size, bool is_write)
	301	+{
	302	+ atomic_long_t *watchpoint;
	303	+ union {
	304	+ u8 _1;
	305	+ u16 _2;
	306	+ u32 _4;
	307	+ u64 _8;
	308	+ } expect_value;
	309	+ bool value_change = false;
	310	+ unsigned long ua_flags = user_access_save();
	311	+ unsigned long irq_flags;
	312	+
	313	+ /*
	314	+ * Always reset kcsan_skip counter in slow-path to avoid underflow; see
	315	+ * should_watch().
	316	+ */
	317	+ reset_kcsan_skip();
	318	+
	319	+ if (!kcsan_is_enabled())
	320	+ goto out;
	321	+
	322	+ if (!check_encodable((unsigned long)ptr, size)) {
	323	+ kcsan_counter_inc(KCSAN_COUNTER_UNENCODABLE_ACCESSES);
	324	+ goto out;
	325	+ }
	326	+
	327	+ /*
	328	+ * Disable interrupts & preemptions to avoid another thread on the same
	329	+ * CPU accessing memory locations for the set up watchpoint; this is to
	330	+ * avoid reporting races to e.g. CPU-local data.
	331	+ *
	332	+ * An alternative would be adding the source CPU to the watchpoint
	333	+ * encoding, and checking that watchpoint-CPU != this-CPU. There are
	334	+ * several problems with this:
	335	+ * 1. we should avoid stealing more bits from the watchpoint encoding
	336	+ * as it would affect accuracy, as well as increase performance
	337	+ * overhead in the fast-path;
	338	+ * 2. if we are preempted, but there is a genuine data race, we
	339	+ * would not report it -- since this is the common case (vs.
	340	+ * CPU-local data accesses), it makes more sense (from a data race
	341	+ * detection point of view) to simply disable preemptions to ensure
	342	+ * as many tasks as possible run on other CPUs.
	343	+ */
	344	+ local_irq_save(irq_flags);
	345	+
	346	+ watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write);
	347	+ if (watchpoint == NULL) {
	348	+ /*
	349	+ * Out of capacity: the size of `watchpoints`, and the frequency
	350	+ * with which `should_watch()` returns true should be tweaked so
	351	+ * that this case happens very rarely.
	352	+ */
	353	+ kcsan_counter_inc(KCSAN_COUNTER_NO_CAPACITY);
	354	+ goto out_unlock;
	355	+ }
	356	+
	357	+ kcsan_counter_inc(KCSAN_COUNTER_SETUP_WATCHPOINTS);
	358	+ kcsan_counter_inc(KCSAN_COUNTER_USED_WATCHPOINTS);
	359	+
	360	+ /*
	361	+ * Read the current value, to later check and infer a race if the data
	362	+ * was modified via a non-instrumented access, e.g. from a device.
	363	+ */
	364	+ switch (size) {
	365	+ case 1:
	366	+ expect_value._1 = READ_ONCE((const u8 )ptr);
	367	+ break;
	368	+ case 2:
	369	+ expect_value._2 = READ_ONCE((const u16 )ptr);
	370	+ break;
	371	+ case 4:
	372	+ expect_value._4 = READ_ONCE((const u32 )ptr);
	373	+ break;
	374	+ case 8:
	375	+ expect_value._8 = READ_ONCE((const u64 )ptr);
	376	+ break;
	377	+ default:
	378	+ break; /* ignore; we do not diff the values */
	379	+ }
	380	+
	381	+ if (IS_ENABLED(CONFIG_KCSAN_DEBUG)) {
	382	+ kcsan_disable_current();
	383	+ pr_err("KCSAN: watching %s, size: %zu, addr: %px [slot: %d, encoded: %lx]\n",
	384	+ is_write ? "write" : "read", size, ptr,
	385	+ watchpoint_slot((unsigned long)ptr),
	386	+ encode_watchpoint((unsigned long)ptr, size, is_write));
	387	+ kcsan_enable_current();
	388	+ }
	389	+
	390	+ /*
	391	+ * Delay this thread, to increase probability of observing a racy
	392	+ * conflicting access.
	393	+ */
	394	+ udelay(get_delay());
	395	+
	396	+ /*
	397	+ * Re-read value, and check if it is as expected; if not, we infer a
	398	+ * racy access.
	399	+ */
	400	+ switch (size) {
	401	+ case 1:
	402	+ value_change = expect_value._1 != READ_ONCE((const u8 )ptr);
	403	+ break;
	404	+ case 2:
	405	+ value_change = expect_value._2 != READ_ONCE((const u16 )ptr);
	406	+ break;
	407	+ case 4:
	408	+ value_change = expect_value._4 != READ_ONCE((const u32 )ptr);
	409	+ break;
	410	+ case 8:
	411	+ value_change = expect_value._8 != READ_ONCE((const u64 )ptr);
	412	+ break;
	413	+ default:
	414	+ break; /* ignore; we do not diff the values */
	415	+ }
	416	+
	417	+ /* Check if this access raced with another. */
	418	+ if (!remove_watchpoint(watchpoint)) {
	419	+ /*
	420	+ * No need to increment 'data_races' counter, as the racing
	421	+ * thread already did.
	422	+ */
	423	+ kcsan_report(ptr, size, is_write, size > 8 \|\| value_change,
	424	+ smp_processor_id(), KCSAN_REPORT_RACE_SIGNAL);
	425	+ } else if (value_change) {
	426	+ /* Inferring a race, since the value should not have changed. */
	427	+ kcsan_counter_inc(KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN);
	428	+ if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN))
	429	+ kcsan_report(ptr, size, is_write, true,
	430	+ smp_processor_id(),
	431	+ KCSAN_REPORT_RACE_UNKNOWN_ORIGIN);
	432	+ }
	433	+
	434	+ kcsan_counter_dec(KCSAN_COUNTER_USED_WATCHPOINTS);
	435	+out_unlock:
	436	+ local_irq_restore(irq_flags);
	437	+out:
	438	+ user_access_restore(ua_flags);
	439	+}
	440	+
	441	+static __always_inline void check_access(const volatile void *ptr, size_t size,
	442	+ int type)
	443	+{
	444	+ const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0;
	445	+ atomic_long_t *watchpoint;
	446	+ long encoded_watchpoint;
	447	+
	448	+ /*
	449	+ * Avoid user_access_save in fast-path: find_watchpoint is safe without
	450	+ * user_access_save, as the address that ptr points to is only used to
	451	+ * check if a watchpoint exists; ptr is never dereferenced.
	452	+ */
	453	+ watchpoint = find_watchpoint((unsigned long)ptr, size, !is_write,
	454	+ &encoded_watchpoint);
	455	+ /*
	456	+ * It is safe to check kcsan_is_enabled() after find_watchpoint in the
	457	+ * slow-path, as long as no state changes that cause a data race to be
	458	+ * detected and reported have occurred until kcsan_is_enabled() is
	459	+ * checked.
	460	+ */
	461	+
	462	+ if (unlikely(watchpoint != NULL))
	463	+ kcsan_found_watchpoint(ptr, size, is_write, watchpoint,
	464	+ encoded_watchpoint);
	465	+ else if (unlikely(should_watch(ptr, type)))
	466	+ kcsan_setup_watchpoint(ptr, size, is_write);
	467	+}
	468	+
	469	+/* === Public interface ===================================================== */
	470	+
	471	+void __init kcsan_init(void)
	472	+{
	473	+ BUG_ON(!in_task());
	474	+
	475	+ kcsan_debugfs_init();
	476	+
	477	+ /*
	478	+ * We are in the init task, and no other tasks should be running;
	479	+ * WRITE_ONCE without memory barrier is sufficient.
	480	+ */
	481	+ if (IS_ENABLED(CONFIG_KCSAN_EARLY_ENABLE))
	482	+ WRITE_ONCE(kcsan_enabled, true);
	483	+}
	484	+
	485	+/* === Exported interface =================================================== */
	486	+
	487	+void kcsan_disable_current(void)
	488	+{
	489	+ ++get_ctx()->disable_count;
	490	+}
	491	+EXPORT_SYMBOL(kcsan_disable_current);
	492	+
	493	+void kcsan_enable_current(void)
	494	+{
	495	+ if (get_ctx()->disable_count-- == 0) {
	496	+ /*
	497	+ * Warn if kcsan_enable_current() calls are unbalanced with
	498	+ * kcsan_disable_current() calls, which causes disable_count to
	499	+ * become negative and should not happen.
	500	+ */
	501	+ kcsan_disable_current(); /* restore to 0, KCSAN still enabled */
	502	+ kcsan_disable_current(); /* disable to generate warning */
	503	+ WARN(1, "Unbalanced %s()", __func__);
	504	+ kcsan_enable_current();
	505	+ }
	506	+}
	507	+EXPORT_SYMBOL(kcsan_enable_current);
	508	+
	509	+void kcsan_nestable_atomic_begin(void)
	510	+{
	511	+ /*
	512	+ * Do not check and warn if we are in a flat atomic region: nestable
	513	+ * and flat atomic regions are independent from each other.
	514	+ * See include/linux/kcsan.h: struct kcsan_ctx comments for more
	515	+ * comments.
	516	+ */
	517	+
	518	+ ++get_ctx()->atomic_nest_count;
	519	+}
	520	+EXPORT_SYMBOL(kcsan_nestable_atomic_begin);
	521	+
	522	+void kcsan_nestable_atomic_end(void)
	523	+{
	524	+ if (get_ctx()->atomic_nest_count-- == 0) {
	525	+ /*
	526	+ * Warn if kcsan_nestable_atomic_end() calls are unbalanced with
	527	+ * kcsan_nestable_atomic_begin() calls, which causes
	528	+ * atomic_nest_count to become negative and should not happen.
	529	+ */
	530	+ kcsan_nestable_atomic_begin(); /* restore to 0 */
	531	+ kcsan_disable_current(); /* disable to generate warning */
	532	+ WARN(1, "Unbalanced %s()", __func__);
	533	+ kcsan_enable_current();
	534	+ }
	535	+}
	536	+EXPORT_SYMBOL(kcsan_nestable_atomic_end);
	537	+
	538	+void kcsan_flat_atomic_begin(void)
	539	+{
	540	+ get_ctx()->in_flat_atomic = true;
	541	+}
	542	+EXPORT_SYMBOL(kcsan_flat_atomic_begin);
	543	+
	544	+void kcsan_flat_atomic_end(void)
	545	+{
	546	+ get_ctx()->in_flat_atomic = false;
	547	+}
	548	+EXPORT_SYMBOL(kcsan_flat_atomic_end);
	549	+
	550	+void kcsan_atomic_next(int n)
	551	+{
	552	+ get_ctx()->atomic_next = n;
	553	+}
	554	+EXPORT_SYMBOL(kcsan_atomic_next);
	555	+
	556	+void __kcsan_check_access(const volatile void *ptr, size_t size, int type)
	557	+{
	558	+ check_access(ptr, size, type);
	559	+}
	560	+EXPORT_SYMBOL(__kcsan_check_access);
	561	+
	562	+/*
	563	+ * KCSAN uses the same instrumentation that is emitted by supported compilers
	564	+ * for ThreadSanitizer (TSAN).
	565	+ *
	566	+ * When enabled, the compiler emits instrumentation calls (the functions
	567	+ * prefixed with "__tsan" below) for all loads and stores that it generated;
	568	+ * inline asm is not instrumented.
	569	+ *
	570	+ * Note that, not all supported compiler versions distinguish aligned/unaligned
	571	+ * accesses, but e.g. recent versions of Clang do. We simply alias the unaligned
	572	+ * version to the generic version, which can handle both.
	573	+ */
	574	+
	575	+#define DEFINE_TSAN_READ_WRITE(size) \
	576	+ void __tsan_read##size(void *ptr) \
	577	+ { \
	578	+ check_access(ptr, size, 0); \
	579	+ } \
	580	+ EXPORT_SYMBOL(__tsan_read##size); \
	581	+ void __tsan_unaligned_read##size(void *ptr) \
	582	+ __alias(__tsan_read##size); \
	583	+ EXPORT_SYMBOL(__tsan_unaligned_read##size); \
	584	+ void __tsan_write##size(void *ptr) \
	585	+ { \
	586	+ check_access(ptr, size, KCSAN_ACCESS_WRITE); \
	587	+ } \
	588	+ EXPORT_SYMBOL(__tsan_write##size); \
	589	+ void __tsan_unaligned_write##size(void *ptr) \
	590	+ __alias(__tsan_write##size); \
	591	+ EXPORT_SYMBOL(__tsan_unaligned_write##size)
	592	+
	593	+DEFINE_TSAN_READ_WRITE(1);
	594	+DEFINE_TSAN_READ_WRITE(2);
	595	+DEFINE_TSAN_READ_WRITE(4);
	596	+DEFINE_TSAN_READ_WRITE(8);
	597	+DEFINE_TSAN_READ_WRITE(16);
	598	+
	599	+void __tsan_read_range(void *ptr, size_t size)
	600	+{
	601	+ check_access(ptr, size, 0);
	602	+}
	603	+EXPORT_SYMBOL(__tsan_read_range);
	604	+
	605	+void __tsan_write_range(void *ptr, size_t size)
	606	+{
	607	+ check_access(ptr, size, KCSAN_ACCESS_WRITE);
	608	+}
	609	+EXPORT_SYMBOL(__tsan_write_range);
	610	+
	611	+/*
	612	+ * The below are not required by KCSAN, but can still be emitted by the
	613	+ * compiler.
	614	+ */
	615	+void __tsan_func_entry(void *call_pc)
	616	+{
	617	+}
	618	+EXPORT_SYMBOL(__tsan_func_entry);
	619	+void __tsan_func_exit(void)
	620	+{
	621	+}
	622	+EXPORT_SYMBOL(__tsan_func_exit);
	623	+void __tsan_init(void)
	624	+{
	625	+}
	626	+EXPORT_SYMBOL(__tsan_init);

kernel/kcsan/debugfs.c+275−0

@@ -0,0 +1,275 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include <linux/atomic.h>
	4	+#include <linux/bsearch.h>
	5	+#include <linux/bug.h>
	6	+#include <linux/debugfs.h>
	7	+#include <linux/init.h>
	8	+#include <linux/kallsyms.h>
	9	+#include <linux/seq_file.h>
	10	+#include <linux/slab.h>
	11	+#include <linux/sort.h>
	12	+#include <linux/string.h>
	13	+#include <linux/uaccess.h>
	14	+
	15	+#include "kcsan.h"
	16	+
	17	+/*
	18	+ * Statistics counters.
	19	+ */
	20	+static atomic_long_t counters[KCSAN_COUNTER_COUNT];
	21	+
	22	+/*
	23	+ * Addresses for filtering functions from reporting. This list can be used as a
	24	+ * whitelist or blacklist.
	25	+ */
	26	+static struct {
	27	+ unsigned long addrs; / array of addresses */
	28	+ size_t size; /* current size */
	29	+ int used; /* number of elements used */
	30	+ bool sorted; /* if elements are sorted */
	31	+ bool whitelist; /* if list is a blacklist or whitelist */
	32	+} report_filterlist = {
	33	+ .addrs = NULL,
	34	+ .size = 8, /* small initial size */
	35	+ .used = 0,
	36	+ .sorted = false,
	37	+ .whitelist = false, /* default is blacklist */
	38	+};
	39	+static DEFINE_SPINLOCK(report_filterlist_lock);
	40	+
	41	+static const char *counter_to_name(enum kcsan_counter_id id)
	42	+{
	43	+ switch (id) {
	44	+ case KCSAN_COUNTER_USED_WATCHPOINTS:
	45	+ return "used_watchpoints";
	46	+ case KCSAN_COUNTER_SETUP_WATCHPOINTS:
	47	+ return "setup_watchpoints";
	48	+ case KCSAN_COUNTER_DATA_RACES:
	49	+ return "data_races";
	50	+ case KCSAN_COUNTER_NO_CAPACITY:
	51	+ return "no_capacity";
	52	+ case KCSAN_COUNTER_REPORT_RACES:
	53	+ return "report_races";
	54	+ case KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN:
	55	+ return "races_unknown_origin";
	56	+ case KCSAN_COUNTER_UNENCODABLE_ACCESSES:
	57	+ return "unencodable_accesses";
	58	+ case KCSAN_COUNTER_ENCODING_FALSE_POSITIVES:
	59	+ return "encoding_false_positives";
	60	+ case KCSAN_COUNTER_COUNT:
	61	+ BUG();
	62	+ }
	63	+ return NULL;
	64	+}
	65	+
	66	+void kcsan_counter_inc(enum kcsan_counter_id id)
	67	+{
	68	+ atomic_long_inc(&counters[id]);
	69	+}
	70	+
	71	+void kcsan_counter_dec(enum kcsan_counter_id id)
	72	+{
	73	+ atomic_long_dec(&counters[id]);
	74	+}
	75	+
	76	+/*
	77	+ * The microbenchmark allows benchmarking KCSAN core runtime only. To run
	78	+ * multiple threads, pipe 'microbench=<iters>' from multiple tasks into the
	79	+ * debugfs file.
	80	+ */
	81	+static void microbenchmark(unsigned long iters)
	82	+{
	83	+ cycles_t cycles;
	84	+
	85	+ pr_info("KCSAN: %s begin \| iters: %lu\n", __func__, iters);
	86	+
	87	+ cycles = get_cycles();
	88	+ while (iters--) {
	89	+ /*
	90	+ * We can run this benchmark from multiple tasks; this address
	91	+ * calculation increases likelyhood of some accesses overlapping
	92	+ * (they still won't conflict because all are reads).
	93	+ */
	94	+ unsigned long addr =
	95	+ iters % (CONFIG_KCSAN_NUM_WATCHPOINTS * PAGE_SIZE);
	96	+ __kcsan_check_read((void *)addr, sizeof(long));
	97	+ }
	98	+ cycles = get_cycles() - cycles;
	99	+
	100	+ pr_info("KCSAN: %s end \| cycles: %llu\n", __func__, cycles);
	101	+}
	102	+
	103	+static int cmp_filterlist_addrs(const void rhs, const void lhs)
	104	+{
	105	+ const unsigned long a = (const unsigned long )rhs;
	106	+ const unsigned long b = (const unsigned long )lhs;
	107	+
	108	+ return a < b ? -1 : a == b ? 0 : 1;
	109	+}
	110	+
	111	+bool kcsan_skip_report_debugfs(unsigned long func_addr)
	112	+{
	113	+ unsigned long symbolsize, offset;
	114	+ unsigned long flags;
	115	+ bool ret = false;
	116	+
	117	+ if (!kallsyms_lookup_size_offset(func_addr, &symbolsize, &offset))
	118	+ return false;
	119	+ func_addr -= offset; /* get function start */
	120	+
	121	+ spin_lock_irqsave(&report_filterlist_lock, flags);
	122	+ if (report_filterlist.used == 0)
	123	+ goto out;
	124	+
	125	+ /* Sort array if it is unsorted, and then do a binary search. */
	126	+ if (!report_filterlist.sorted) {
	127	+ sort(report_filterlist.addrs, report_filterlist.used,
	128	+ sizeof(unsigned long), cmp_filterlist_addrs, NULL);
	129	+ report_filterlist.sorted = true;
	130	+ }
	131	+ ret = !!bsearch(&func_addr, report_filterlist.addrs,
	132	+ report_filterlist.used, sizeof(unsigned long),
	133	+ cmp_filterlist_addrs);
	134	+ if (report_filterlist.whitelist)
	135	+ ret = !ret;
	136	+
	137	+out:
	138	+ spin_unlock_irqrestore(&report_filterlist_lock, flags);
	139	+ return ret;
	140	+}
	141	+
	142	+static void set_report_filterlist_whitelist(bool whitelist)
	143	+{
	144	+ unsigned long flags;
	145	+
	146	+ spin_lock_irqsave(&report_filterlist_lock, flags);
	147	+ report_filterlist.whitelist = whitelist;
	148	+ spin_unlock_irqrestore(&report_filterlist_lock, flags);
	149	+}
	150	+
	151	+/* Returns 0 on success, error-code otherwise. */
	152	+static ssize_t insert_report_filterlist(const char *func)
	153	+{
	154	+ unsigned long flags;
	155	+ unsigned long addr = kallsyms_lookup_name(func);
	156	+ ssize_t ret = 0;
	157	+
	158	+ if (!addr) {
	159	+ pr_err("KCSAN: could not find function: '%s'\n", func);
	160	+ return -ENOENT;
	161	+ }
	162	+
	163	+ spin_lock_irqsave(&report_filterlist_lock, flags);
	164	+
	165	+ if (report_filterlist.addrs == NULL) {
	166	+ /* initial allocation */
	167	+ report_filterlist.addrs =
	168	+ kmalloc_array(report_filterlist.size,
	169	+ sizeof(unsigned long), GFP_KERNEL);
	170	+ if (report_filterlist.addrs == NULL) {
	171	+ ret = -ENOMEM;
	172	+ goto out;
	173	+ }
	174	+ } else if (report_filterlist.used == report_filterlist.size) {
	175	+ /* resize filterlist */
	176	+ size_t new_size = report_filterlist.size * 2;
	177	+ unsigned long *new_addrs =
	178	+ krealloc(report_filterlist.addrs,
	179	+ new_size * sizeof(unsigned long), GFP_KERNEL);
	180	+
	181	+ if (new_addrs == NULL) {
	182	+ /* leave filterlist itself untouched */
	183	+ ret = -ENOMEM;
	184	+ goto out;
	185	+ }
	186	+
	187	+ report_filterlist.size = new_size;
	188	+ report_filterlist.addrs = new_addrs;
	189	+ }
	190	+
	191	+ /* Note: deduplicating should be done in userspace. */
	192	+ report_filterlist.addrs[report_filterlist.used++] =
	193	+ kallsyms_lookup_name(func);
	194	+ report_filterlist.sorted = false;
	195	+
	196	+out:
	197	+ spin_unlock_irqrestore(&report_filterlist_lock, flags);
	198	+ return ret;
	199	+}
	200	+
	201	+static int show_info(struct seq_file file, void v)
	202	+{
	203	+ int i;
	204	+ unsigned long flags;
	205	+
	206	+ /* show stats */
	207	+ seq_printf(file, "enabled: %i\n", READ_ONCE(kcsan_enabled));
	208	+ for (i = 0; i < KCSAN_COUNTER_COUNT; ++i)
	209	+ seq_printf(file, "%s: %ld\n", counter_to_name(i),
	210	+ atomic_long_read(&counters[i]));
	211	+
	212	+ /* show filter functions, and filter type */
	213	+ spin_lock_irqsave(&report_filterlist_lock, flags);
	214	+ seq_printf(file, "\n%s functions: %s\n",
	215	+ report_filterlist.whitelist ? "whitelisted" : "blacklisted",
	216	+ report_filterlist.used == 0 ? "none" : "");
	217	+ for (i = 0; i < report_filterlist.used; ++i)
	218	+ seq_printf(file, " %ps\n", (void *)report_filterlist.addrs[i]);
	219	+ spin_unlock_irqrestore(&report_filterlist_lock, flags);
	220	+
	221	+ return 0;
	222	+}
	223	+
	224	+static int debugfs_open(struct inode inode, struct file file)
	225	+{
	226	+ return single_open(file, show_info, NULL);
	227	+}
	228	+
	229	+static ssize_t debugfs_write(struct file file, const char __user buf,
	230	+ size_t count, loff_t *off)
	231	+{
	232	+ char kbuf[KSYM_NAME_LEN];
	233	+ char *arg;
	234	+ int read_len = count < (sizeof(kbuf) - 1) ? count : (sizeof(kbuf) - 1);
	235	+
	236	+ if (copy_from_user(kbuf, buf, read_len))
	237	+ return -EFAULT;
	238	+ kbuf[read_len] = '\0';
	239	+ arg = strstrip(kbuf);
	240	+
	241	+ if (!strcmp(arg, "on")) {
	242	+ WRITE_ONCE(kcsan_enabled, true);
	243	+ } else if (!strcmp(arg, "off")) {
	244	+ WRITE_ONCE(kcsan_enabled, false);
	245	+ } else if (!strncmp(arg, "microbench=", sizeof("microbench=") - 1)) {
	246	+ unsigned long iters;
	247	+
	248	+ if (kstrtoul(&arg[sizeof("microbench=") - 1], 0, &iters))
	249	+ return -EINVAL;
	250	+ microbenchmark(iters);
	251	+ } else if (!strcmp(arg, "whitelist")) {
	252	+ set_report_filterlist_whitelist(true);
	253	+ } else if (!strcmp(arg, "blacklist")) {
	254	+ set_report_filterlist_whitelist(false);
	255	+ } else if (arg[0] == '!') {
	256	+ ssize_t ret = insert_report_filterlist(&arg[1]);
	257	+
	258	+ if (ret < 0)
	259	+ return ret;
	260	+ } else {
	261	+ return -EINVAL;
	262	+ }
	263	+
	264	+ return count;
	265	+}
	266	+
	267	+static const struct file_operations debugfs_ops = { .read = seq_read,
	268	+ .open = debugfs_open,
	269	+ .write = debugfs_write,
	270	+ .release = single_release };
	271	+
	272	+void __init kcsan_debugfs_init(void)
	273	+{
	274	+ debugfs_create_file("kcsan", 0644, NULL, NULL, &debugfs_ops);
	275	+}

kernel/kcsan/encoding.h+94−0

@@ -0,0 +1,94 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+
	3	+#ifndef _KERNEL_KCSAN_ENCODING_H
	4	+#define _KERNEL_KCSAN_ENCODING_H
	5	+
	6	+#include <linux/bits.h>
	7	+#include <linux/log2.h>
	8	+#include <linux/mm.h>
	9	+
	10	+#include "kcsan.h"
	11	+
	12	+#define SLOT_RANGE PAGE_SIZE
	13	+#define INVALID_WATCHPOINT 0
	14	+#define CONSUMED_WATCHPOINT 1
	15	+
	16	+/*
	17	+ * The maximum useful size of accesses for which we set up watchpoints is the
	18	+ * max range of slots we check on an access.
	19	+ */
	20	+#define MAX_ENCODABLE_SIZE (SLOT_RANGE * (1 + KCSAN_CHECK_ADJACENT))
	21	+
	22	+/*
	23	+ * Number of bits we use to store size info.
	24	+ */
	25	+#define WATCHPOINT_SIZE_BITS bits_per(MAX_ENCODABLE_SIZE)
	26	+/*
	27	+ * This encoding for addresses discards the upper (1 for is-write + SIZE_BITS);
	28	+ * however, most 64-bit architectures do not use the full 64-bit address space.
	29	+ * Also, in order for a false positive to be observable 2 things need to happen:
	30	+ *
	31	+ * 1. different addresses but with the same encoded address race;
	32	+ * 2. and both map onto the same watchpoint slots;
	33	+ *
	34	+ * Both these are assumed to be very unlikely. However, in case it still happens
	35	+ * happens, the report logic will filter out the false positive (see report.c).
	36	+ */
	37	+#define WATCHPOINT_ADDR_BITS (BITS_PER_LONG - 1 - WATCHPOINT_SIZE_BITS)
	38	+
	39	+/*
	40	+ * Masks to set/retrieve the encoded data.
	41	+ */
	42	+#define WATCHPOINT_WRITE_MASK BIT(BITS_PER_LONG - 1)
	43	+#define WATCHPOINT_SIZE_MASK \
	44	+ GENMASK(BITS_PER_LONG - 2, BITS_PER_LONG - 2 - WATCHPOINT_SIZE_BITS)
	45	+#define WATCHPOINT_ADDR_MASK \
	46	+ GENMASK(BITS_PER_LONG - 3 - WATCHPOINT_SIZE_BITS, 0)
	47	+
	48	+static inline bool check_encodable(unsigned long addr, size_t size)
	49	+{
	50	+ return size <= MAX_ENCODABLE_SIZE;
	51	+}
	52	+
	53	+static inline long encode_watchpoint(unsigned long addr, size_t size,
	54	+ bool is_write)
	55	+{
	56	+ return (long)((is_write ? WATCHPOINT_WRITE_MASK : 0) \|
	57	+ (size << WATCHPOINT_ADDR_BITS) \|
	58	+ (addr & WATCHPOINT_ADDR_MASK));
	59	+}
	60	+
	61	+static inline bool decode_watchpoint(long watchpoint,
	62	+ unsigned long addr_masked, size_t size,
	63	+ bool *is_write)
	64	+{
	65	+ if (watchpoint == INVALID_WATCHPOINT \|\|
	66	+ watchpoint == CONSUMED_WATCHPOINT)
	67	+ return false;
	68	+
	69	+ *addr_masked = (unsigned long)watchpoint & WATCHPOINT_ADDR_MASK;
	70	+ *size = ((unsigned long)watchpoint & WATCHPOINT_SIZE_MASK) >>
	71	+ WATCHPOINT_ADDR_BITS;
	72	+ *is_write = !!((unsigned long)watchpoint & WATCHPOINT_WRITE_MASK);
	73	+
	74	+ return true;
	75	+}
	76	+
	77	+/*
	78	+ * Return watchpoint slot for an address.
	79	+ */
	80	+static inline int watchpoint_slot(unsigned long addr)
	81	+{
	82	+ return (addr / PAGE_SIZE) % CONFIG_KCSAN_NUM_WATCHPOINTS;
	83	+}
	84	+
	85	+static inline bool matching_access(unsigned long addr1, size_t size1,
	86	+ unsigned long addr2, size_t size2)
	87	+{
	88	+ unsigned long end_range1 = addr1 + size1 - 1;
	89	+ unsigned long end_range2 = addr2 + size2 - 1;
	90	+
	91	+ return addr1 <= end_range2 && addr2 <= end_range1;
	92	+}
	93	+
	94	+#endif /* _KERNEL_KCSAN_ENCODING_H */

kernel/kcsan/kcsan.h+108−0

@@ -0,0 +1,108 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+
	3	+/*
	4	+ * The Kernel Concurrency Sanitizer (KCSAN) infrastructure. For more info please
	5	+ * see Documentation/dev-tools/kcsan.rst.
	6	+ */
	7	+
	8	+#ifndef _KERNEL_KCSAN_KCSAN_H
	9	+#define _KERNEL_KCSAN_KCSAN_H
	10	+
	11	+#include <linux/kcsan.h>
	12	+
	13	+/* The number of adjacent watchpoints to check. */
	14	+#define KCSAN_CHECK_ADJACENT 1
	15	+
	16	+/*
	17	+ * Globally enable and disable KCSAN.
	18	+ */
	19	+extern bool kcsan_enabled;
	20	+
	21	+/*
	22	+ * Initialize debugfs file.
	23	+ */
	24	+void kcsan_debugfs_init(void);
	25	+
	26	+enum kcsan_counter_id {
	27	+ /*
	28	+ * Number of watchpoints currently in use.
	29	+ */
	30	+ KCSAN_COUNTER_USED_WATCHPOINTS,
	31	+
	32	+ /*
	33	+ * Total number of watchpoints set up.
	34	+ */
	35	+ KCSAN_COUNTER_SETUP_WATCHPOINTS,
	36	+
	37	+ /*
	38	+ * Total number of data races.
	39	+ */
	40	+ KCSAN_COUNTER_DATA_RACES,
	41	+
	42	+ /*
	43	+ * Number of times no watchpoints were available.
	44	+ */
	45	+ KCSAN_COUNTER_NO_CAPACITY,
	46	+
	47	+ /*
	48	+ * A thread checking a watchpoint raced with another checking thread;
	49	+ * only one will be reported.
	50	+ */
	51	+ KCSAN_COUNTER_REPORT_RACES,
	52	+
	53	+ /*
	54	+ * Observed data value change, but writer thread unknown.
	55	+ */
	56	+ KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN,
	57	+
	58	+ /*
	59	+ * The access cannot be encoded to a valid watchpoint.
	60	+ */
	61	+ KCSAN_COUNTER_UNENCODABLE_ACCESSES,
	62	+
	63	+ /*
	64	+ * Watchpoint encoding caused a watchpoint to fire on mismatching
	65	+ * accesses.
	66	+ */
	67	+ KCSAN_COUNTER_ENCODING_FALSE_POSITIVES,
	68	+
	69	+ KCSAN_COUNTER_COUNT, /* number of counters */
	70	+};
	71	+
	72	+/*
	73	+ * Increment/decrement counter with given id; avoid calling these in fast-path.
	74	+ */
	75	+void kcsan_counter_inc(enum kcsan_counter_id id);
	76	+void kcsan_counter_dec(enum kcsan_counter_id id);
	77	+
	78	+/*
	79	+ * Returns true if data races in the function symbol that maps to func_addr
	80	+ * (offsets are ignored) should not be reported.
	81	+ */
	82	+bool kcsan_skip_report_debugfs(unsigned long func_addr);
	83	+
	84	+enum kcsan_report_type {
	85	+ /*
	86	+ * The thread that set up the watchpoint and briefly stalled was
	87	+ * signalled that another thread triggered the watchpoint.
	88	+ */
	89	+ KCSAN_REPORT_RACE_SIGNAL,
	90	+
	91	+ /*
	92	+ * A thread found and consumed a matching watchpoint.
	93	+ */
	94	+ KCSAN_REPORT_CONSUMED_WATCHPOINT,
	95	+
	96	+ /*
	97	+ * No other thread was observed to race with the access, but the data
	98	+ * value before and after the stall differs.
	99	+ */
	100	+ KCSAN_REPORT_RACE_UNKNOWN_ORIGIN,
	101	+};
	102	+/*
	103	+ * Print a race report from thread that encountered the race.
	104	+ */
	105	+void kcsan_report(const volatile void *ptr, size_t size, bool is_write,
	106	+ bool value_change, int cpu_id, enum kcsan_report_type type);
	107	+
	108	+#endif /* _KERNEL_KCSAN_KCSAN_H */

kernel/kcsan/report.c+320−0

@@ -0,0 +1,320 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include <linux/kernel.h>
	4	+#include <linux/preempt.h>
	5	+#include <linux/printk.h>
	6	+#include <linux/sched.h>
	7	+#include <linux/spinlock.h>
	8	+#include <linux/stacktrace.h>
	9	+
	10	+#include "kcsan.h"
	11	+#include "encoding.h"
	12	+
	13	+/*
	14	+ * Max. number of stack entries to show in the report.
	15	+ */
	16	+#define NUM_STACK_ENTRIES 64
	17	+
	18	+/*
	19	+ * Other thread info: communicated from other racing thread to thread that set
	20	+ * up the watchpoint, which then prints the complete report atomically. Only
	21	+ * need one struct, as all threads should to be serialized regardless to print
	22	+ * the reports, with reporting being in the slow-path.
	23	+ */
	24	+static struct {
	25	+ const volatile void *ptr;
	26	+ size_t size;
	27	+ bool is_write;
	28	+ int task_pid;
	29	+ int cpu_id;
	30	+ unsigned long stack_entries[NUM_STACK_ENTRIES];
	31	+ int num_stack_entries;
	32	+} other_info = { .ptr = NULL };
	33	+
	34	+/*
	35	+ * This spinlock protects reporting and other_info, since other_info is usually
	36	+ * required when reporting.
	37	+ */
	38	+static DEFINE_SPINLOCK(report_lock);
	39	+
	40	+/*
	41	+ * Special rules to skip reporting.
	42	+ */
	43	+static bool skip_report(bool is_write, bool value_change,
	44	+ unsigned long top_frame)
	45	+{
	46	+ if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY) && is_write &&
	47	+ !value_change) {
	48	+ /*
	49	+ * The access is a write, but the data value did not change.
	50	+ *
	51	+ * We opt-out of this filter for certain functions at request of
	52	+ * maintainers.
	53	+ */
	54	+ char buf[64];
	55	+
	56	+ snprintf(buf, sizeof(buf), "%ps", (void *)top_frame);
	57	+ if (!strnstr(buf, "rcu_", sizeof(buf)) &&
	58	+ !strnstr(buf, "_rcu", sizeof(buf)) &&
	59	+ !strnstr(buf, "_srcu", sizeof(buf)))
	60	+ return true;
	61	+ }
	62	+
	63	+ return kcsan_skip_report_debugfs(top_frame);
	64	+}
	65	+
	66	+static inline const char *get_access_type(bool is_write)
	67	+{
	68	+ return is_write ? "write" : "read";
	69	+}
	70	+
	71	+/* Return thread description: in task or interrupt. */
	72	+static const char *get_thread_desc(int task_id)
	73	+{
	74	+ if (task_id != -1) {
	75	+ static char buf[32]; /* safe: protected by report_lock */
	76	+
	77	+ snprintf(buf, sizeof(buf), "task %i", task_id);
	78	+ return buf;
	79	+ }
	80	+ return "interrupt";
	81	+}
	82	+
	83	+/* Helper to skip KCSAN-related functions in stack-trace. */
	84	+static int get_stack_skipnr(unsigned long stack_entries[], int num_entries)
	85	+{
	86	+ char buf[64];
	87	+ int skip = 0;
	88	+
	89	+ for (; skip < num_entries; ++skip) {
	90	+ snprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skip]);
	91	+ if (!strnstr(buf, "csan_", sizeof(buf)) &&
	92	+ !strnstr(buf, "tsan_", sizeof(buf)) &&
	93	+ !strnstr(buf, "_once_size", sizeof(buf))) {
	94	+ break;
	95	+ }
	96	+ }
	97	+ return skip;
	98	+}
	99	+
	100	+/* Compares symbolized strings of addr1 and addr2. */
	101	+static int sym_strcmp(void addr1, void addr2)
	102	+{
	103	+ char buf1[64];
	104	+ char buf2[64];
	105	+
	106	+ snprintf(buf1, sizeof(buf1), "%pS", addr1);
	107	+ snprintf(buf2, sizeof(buf2), "%pS", addr2);
	108	+ return strncmp(buf1, buf2, sizeof(buf1));
	109	+}
	110	+
	111	+/*
	112	+ * Returns true if a report was generated, false otherwise.
	113	+ */
	114	+static bool print_report(const volatile void *ptr, size_t size, bool is_write,
	115	+ bool value_change, int cpu_id,
	116	+ enum kcsan_report_type type)
	117	+{
	118	+ unsigned long stack_entries[NUM_STACK_ENTRIES] = { 0 };
	119	+ int num_stack_entries =
	120	+ stack_trace_save(stack_entries, NUM_STACK_ENTRIES, 1);
	121	+ int skipnr = get_stack_skipnr(stack_entries, num_stack_entries);
	122	+ int other_skipnr;
	123	+
	124	+ /*
	125	+ * Must check report filter rules before starting to print.
	126	+ */
	127	+ if (skip_report(is_write, true, stack_entries[skipnr]))
	128	+ return false;
	129	+
	130	+ if (type == KCSAN_REPORT_RACE_SIGNAL) {
	131	+ other_skipnr = get_stack_skipnr(other_info.stack_entries,
	132	+ other_info.num_stack_entries);
	133	+
	134	+ /* value_change is only known for the other thread */
	135	+ if (skip_report(other_info.is_write, value_change,
	136	+ other_info.stack_entries[other_skipnr]))
	137	+ return false;
	138	+ }
	139	+
	140	+ /* Print report header. */
	141	+ pr_err("==================================================================\n");
	142	+ switch (type) {
	143	+ case KCSAN_REPORT_RACE_SIGNAL: {
	144	+ void this_fn = (void )stack_entries[skipnr];
	145	+ void other_fn = (void )other_info.stack_entries[other_skipnr];
	146	+ int cmp;
	147	+
	148	+ /*
	149	+ * Order functions lexographically for consistent bug titles.
	150	+ * Do not print offset of functions to keep title short.
	151	+ */
	152	+ cmp = sym_strcmp(other_fn, this_fn);
	153	+ pr_err("BUG: KCSAN: data-race in %ps / %ps\n",
	154	+ cmp < 0 ? other_fn : this_fn,
	155	+ cmp < 0 ? this_fn : other_fn);
	156	+ } break;
	157	+
	158	+ case KCSAN_REPORT_RACE_UNKNOWN_ORIGIN:
	159	+ pr_err("BUG: KCSAN: data-race in %pS\n",
	160	+ (void *)stack_entries[skipnr]);
	161	+ break;
	162	+
	163	+ default:
	164	+ BUG();
	165	+ }
	166	+
	167	+ pr_err("\n");
	168	+
	169	+ /* Print information about the racing accesses. */
	170	+ switch (type) {
	171	+ case KCSAN_REPORT_RACE_SIGNAL:
	172	+ pr_err("%s to 0x%px of %zu bytes by %s on cpu %i:\n",
	173	+ get_access_type(other_info.is_write), other_info.ptr,
	174	+ other_info.size, get_thread_desc(other_info.task_pid),
	175	+ other_info.cpu_id);
	176	+
	177	+ /* Print the other thread's stack trace. */
	178	+ stack_trace_print(other_info.stack_entries + other_skipnr,
	179	+ other_info.num_stack_entries - other_skipnr,
	180	+ 0);
	181	+
	182	+ pr_err("\n");
	183	+ pr_err("%s to 0x%px of %zu bytes by %s on cpu %i:\n",
	184	+ get_access_type(is_write), ptr, size,
	185	+ get_thread_desc(in_task() ? task_pid_nr(current) : -1),
	186	+ cpu_id);
	187	+ break;
	188	+
	189	+ case KCSAN_REPORT_RACE_UNKNOWN_ORIGIN:
	190	+ pr_err("race at unknown origin, with %s to 0x%px of %zu bytes by %s on cpu %i:\n",
	191	+ get_access_type(is_write), ptr, size,
	192	+ get_thread_desc(in_task() ? task_pid_nr(current) : -1),
	193	+ cpu_id);
	194	+ break;
	195	+
	196	+ default:
	197	+ BUG();
	198	+ }
	199	+ /* Print stack trace of this thread. */
	200	+ stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr,
	201	+ 0);
	202	+
	203	+ /* Print report footer. */
	204	+ pr_err("\n");
	205	+ pr_err("Reported by Kernel Concurrency Sanitizer on:\n");
	206	+ dump_stack_print_info(KERN_DEFAULT);
	207	+ pr_err("==================================================================\n");
	208	+
	209	+ return true;
	210	+}
	211	+
	212	+static void release_report(unsigned long *flags, enum kcsan_report_type type)
	213	+{
	214	+ if (type == KCSAN_REPORT_RACE_SIGNAL)
	215	+ other_info.ptr = NULL; /* mark for reuse */
	216	+
	217	+ spin_unlock_irqrestore(&report_lock, *flags);
	218	+}
	219	+
	220	+/*
	221	+ * Depending on the report type either sets other_info and returns false, or
	222	+ * acquires the matching other_info and returns true. If other_info is not
	223	+ * required for the report type, simply acquires report_lock and returns true.
	224	+ */
	225	+static bool prepare_report(unsigned long flags, const volatile void ptr,
	226	+ size_t size, bool is_write, int cpu_id,
	227	+ enum kcsan_report_type type)
	228	+{
	229	+ if (type != KCSAN_REPORT_CONSUMED_WATCHPOINT &&
	230	+ type != KCSAN_REPORT_RACE_SIGNAL) {
	231	+ /* other_info not required; just acquire report_lock */
	232	+ spin_lock_irqsave(&report_lock, *flags);
	233	+ return true;
	234	+ }
	235	+
	236	+retry:
	237	+ spin_lock_irqsave(&report_lock, *flags);
	238	+
	239	+ switch (type) {
	240	+ case KCSAN_REPORT_CONSUMED_WATCHPOINT:
	241	+ if (other_info.ptr != NULL)
	242	+ break; /* still in use, retry */
	243	+
	244	+ other_info.ptr = ptr;
	245	+ other_info.size = size;
	246	+ other_info.is_write = is_write;
	247	+ other_info.task_pid = in_task() ? task_pid_nr(current) : -1;
	248	+ other_info.cpu_id = cpu_id;
	249	+ other_info.num_stack_entries = stack_trace_save(
	250	+ other_info.stack_entries, NUM_STACK_ENTRIES, 1);
	251	+
	252	+ spin_unlock_irqrestore(&report_lock, *flags);
	253	+
	254	+ /*
	255	+ * The other thread will print the summary; other_info may now
	256	+ * be consumed.
	257	+ */
	258	+ return false;
	259	+
	260	+ case KCSAN_REPORT_RACE_SIGNAL:
	261	+ if (other_info.ptr == NULL)
	262	+ break; /* no data available yet, retry */
	263	+
	264	+ /*
	265	+ * First check if this is the other_info we are expecting, i.e.
	266	+ * matches based on how watchpoint was encoded.
	267	+ */
	268	+ if (!matching_access((unsigned long)other_info.ptr &
	269	+ WATCHPOINT_ADDR_MASK,
	270	+ other_info.size,
	271	+ (unsigned long)ptr & WATCHPOINT_ADDR_MASK,
	272	+ size))
	273	+ break; /* mismatching watchpoint, retry */
	274	+
	275	+ if (!matching_access((unsigned long)other_info.ptr,
	276	+ other_info.size, (unsigned long)ptr,
	277	+ size)) {
	278	+ /*
	279	+ * If the actual accesses to not match, this was a false
	280	+ * positive due to watchpoint encoding.
	281	+ */
	282	+ kcsan_counter_inc(
	283	+ KCSAN_COUNTER_ENCODING_FALSE_POSITIVES);
	284	+
	285	+ /* discard this other_info */
	286	+ release_report(flags, KCSAN_REPORT_RACE_SIGNAL);
	287	+ return false;
	288	+ }
	289	+
	290	+ /*
	291	+ * Matching & usable access in other_info: keep other_info_lock
	292	+ * locked, as this thread consumes it to print the full report;
	293	+ * unlocked in release_report.
	294	+ */
	295	+ return true;
	296	+
	297	+ default:
	298	+ BUG();
	299	+ }
	300	+
	301	+ spin_unlock_irqrestore(&report_lock, *flags);
	302	+ goto retry;
	303	+}
	304	+
	305	+void kcsan_report(const volatile void *ptr, size_t size, bool is_write,
	306	+ bool value_change, int cpu_id, enum kcsan_report_type type)
	307	+{
	308	+ unsigned long flags = 0;
	309	+
	310	+ kcsan_disable_current();
	311	+ if (prepare_report(&flags, ptr, size, is_write, cpu_id, type)) {
	312	+ if (print_report(ptr, size, is_write, value_change, cpu_id,
	313	+ type) &&
	314	+ panic_on_warn)
	315	+ panic("panic_on_warn set ...\n");
	316	+
	317	+ release_report(&flags, type);
	318	+ }
	319	+ kcsan_enable_current();
	320	+}

kernel/kcsan/test.c+121−0

@@ -0,0 +1,121 @@
	1	+// SPDX-License-Identifier: GPL-2.0
	2	+
	3	+#include <linux/init.h>
	4	+#include <linux/kernel.h>
	5	+#include <linux/printk.h>
	6	+#include <linux/random.h>
	7	+#include <linux/types.h>
	8	+
	9	+#include "encoding.h"
	10	+
	11	+#define ITERS_PER_TEST 2000
	12	+
	13	+/* Test requirements. */
	14	+static bool test_requires(void)
	15	+{
	16	+ /* random should be initialized for the below tests */
	17	+ return prandom_u32() + prandom_u32() != 0;
	18	+}
	19	+
	20	+/*
	21	+ * Test watchpoint encode and decode: check that encoding some access's info,
	22	+ * and then subsequent decode preserves the access's info.
	23	+ */
	24	+static bool test_encode_decode(void)
	25	+{
	26	+ int i;
	27	+
	28	+ for (i = 0; i < ITERS_PER_TEST; ++i) {
	29	+ size_t size = prandom_u32_max(MAX_ENCODABLE_SIZE) + 1;
	30	+ bool is_write = !!prandom_u32_max(2);
	31	+ unsigned long addr;
	32	+
	33	+ prandom_bytes(&addr, sizeof(addr));
	34	+ if (WARN_ON(!check_encodable(addr, size)))
	35	+ return false;
	36	+
	37	+ /* encode and decode */
	38	+ {
	39	+ const long encoded_watchpoint =
	40	+ encode_watchpoint(addr, size, is_write);
	41	+ unsigned long verif_masked_addr;
	42	+ size_t verif_size;
	43	+ bool verif_is_write;
	44	+
	45	+ /* check special watchpoints */
	46	+ if (WARN_ON(decode_watchpoint(
	47	+ INVALID_WATCHPOINT, &verif_masked_addr,
	48	+ &verif_size, &verif_is_write)))
	49	+ return false;
	50	+ if (WARN_ON(decode_watchpoint(
	51	+ CONSUMED_WATCHPOINT, &verif_masked_addr,
	52	+ &verif_size, &verif_is_write)))
	53	+ return false;
	54	+
	55	+ /* check decoding watchpoint returns same data */
	56	+ if (WARN_ON(!decode_watchpoint(
	57	+ encoded_watchpoint, &verif_masked_addr,
	58	+ &verif_size, &verif_is_write)))
	59	+ return false;
	60	+ if (WARN_ON(verif_masked_addr !=
	61	+ (addr & WATCHPOINT_ADDR_MASK)))
	62	+ goto fail;
	63	+ if (WARN_ON(verif_size != size))
	64	+ goto fail;
	65	+ if (WARN_ON(is_write != verif_is_write))
	66	+ goto fail;
	67	+
	68	+ continue;
	69	+fail:
	70	+ pr_err("%s fail: %s %zu bytes @ %lx -> encoded: %lx -> %s %zu bytes @ %lx\n",
	71	+ __func__, is_write ? "write" : "read", size,
	72	+ addr, encoded_watchpoint,
	73	+ verif_is_write ? "write" : "read", verif_size,
	74	+ verif_masked_addr);
	75	+ return false;
	76	+ }
	77	+ }
	78	+
	79	+ return true;
	80	+}
	81	+
	82	+/* Test access matching function. */
	83	+static bool test_matching_access(void)
	84	+{
	85	+ if (WARN_ON(!matching_access(10, 1, 10, 1)))
	86	+ return false;
	87	+ if (WARN_ON(!matching_access(10, 2, 11, 1)))
	88	+ return false;
	89	+ if (WARN_ON(!matching_access(10, 1, 9, 2)))
	90	+ return false;
	91	+ if (WARN_ON(matching_access(10, 1, 11, 1)))
	92	+ return false;
	93	+ if (WARN_ON(matching_access(9, 1, 10, 1)))
	94	+ return false;
	95	+ return true;
	96	+}
	97	+
	98	+static int __init kcsan_selftest(void)
	99	+{
	100	+ int passed = 0;
	101	+ int total = 0;
	102	+
	103	+#define RUN_TEST(do_test) \
	104	+ do { \
	105	+ ++total; \
	106	+ if (do_test()) \
	107	+ ++passed; \
	108	+ else \
	109	+ pr_err("KCSAN selftest: " #do_test " failed"); \
	110	+ } while (0)
	111	+
	112	+ RUN_TEST(test_requires);
	113	+ RUN_TEST(test_encode_decode);
	114	+ RUN_TEST(test_matching_access);
	115	+
	116	+ pr_info("KCSAN selftest: %d/%d tests passed\n", passed, total);
	117	+ if (passed != total)
	118	+ panic("KCSAN selftests failed");
	119	+ return 0;
	120	+}
	121	+postcore_initcall(kcsan_selftest);

lib/Kconfig.debug+2−0

@@ -2086,6 +2086,8 @@ source "lib/Kconfig.kgdb" · 2085 unmodified lines
2086	2086
2087	2087	source "lib/Kconfig.ubsan"
2088	2088
	2089	+source "lib/Kconfig.kcsan"
	2090	+
2089	2091	config ARCH_HAS_DEVMEM_IS_ALLOWED
2090	2092	bool
2091	2093

lib/Kconfig.kcsan+118−0

@@ -0,0 +1,118 @@
	1	+# SPDX-License-Identifier: GPL-2.0-only
	2	+
	3	+config HAVE_ARCH_KCSAN
	4	+ bool
	5	+
	6	+menuconfig KCSAN
	7	+ bool "KCSAN: watchpoint-based dynamic data race detector"
	8	+ depends on HAVE_ARCH_KCSAN && !KASAN && STACKTRACE
	9	+ default n
	10	+ help
	11	+ Kernel Concurrency Sanitizer is a dynamic data race detector, which
	12	+ uses a watchpoint-based sampling approach to detect races. See
	13	+ <file:Documentation/dev-tools/kcsan.rst> for more details.
	14	+
	15	+if KCSAN
	16	+
	17	+config KCSAN_DEBUG
	18	+ bool "Debugging of KCSAN internals"
	19	+ default n
	20	+
	21	+config KCSAN_SELFTEST
	22	+ bool "Perform short selftests on boot"
	23	+ default y
	24	+ help
	25	+ Run KCSAN selftests on boot. On test failure, causes kernel to panic.
	26	+
	27	+config KCSAN_EARLY_ENABLE
	28	+ bool "Early enable during boot"
	29	+ default y
	30	+ help
	31	+ If KCSAN should be enabled globally as soon as possible. KCSAN can
	32	+ later be enabled/disabled via debugfs.
	33	+
	34	+config KCSAN_NUM_WATCHPOINTS
	35	+ int "Number of available watchpoints"
	36	+ default 64
	37	+ help
	38	+ Total number of available watchpoints. An address range maps into a
	39	+ specific watchpoint slot as specified in kernel/kcsan/encoding.h.
	40	+ Although larger number of watchpoints may not be usable due to
	41	+ limited number of CPUs, a larger value helps to improve performance
	42	+ due to reducing cache-line contention. The chosen default is a
	43	+ conservative value; we should almost never observe "no_capacity"
	44	+ events (see /sys/kernel/debug/kcsan).
	45	+
	46	+config KCSAN_UDELAY_TASK
	47	+ int "Delay in microseconds (for tasks)"
	48	+ default 80
	49	+ help
	50	+ For tasks, the microsecond delay after setting up a watchpoint.
	51	+
	52	+config KCSAN_UDELAY_INTERRUPT
	53	+ int "Delay in microseconds (for interrupts)"
	54	+ default 20
	55	+ help
	56	+ For interrupts, the microsecond delay after setting up a watchpoint.
	57	+ Interrupts have tighter latency requirements, and their delay should
	58	+ be lower than for tasks.
	59	+
	60	+config KCSAN_DELAY_RANDOMIZE
	61	+ bool "Randomize above delays"
	62	+ default y
	63	+ help
	64	+ If delays should be randomized, where the maximum is KCSAN_UDELAY_*.
	65	+ If false, the chosen delays are always KCSAN_UDELAY_* defined above.
	66	+
	67	+config KCSAN_SKIP_WATCH
	68	+ int "Skip instructions before setting up watchpoint"
	69	+ default 4000
	70	+ help
	71	+ The number of per-CPU memory operations to skip, before another
	72	+ watchpoint is set up, i.e. one in KCSAN_WATCH_SKIP per-CPU
	73	+ memory operations are used to set up a watchpoint. A smaller value
	74	+ results in more aggressive race detection, whereas a larger value
	75	+ improves system performance at the cost of missing some races.
	76	+
	77	+config KCSAN_SKIP_WATCH_RANDOMIZE
	78	+ bool "Randomize watchpoint instruction skip count"
	79	+ default y
	80	+ help
	81	+ If instruction skip count should be randomized, where the maximum is
	82	+ KCSAN_WATCH_SKIP. If false, the chosen value is always
	83	+ KCSAN_WATCH_SKIP.
	84	+
	85	+# Note that, while some of the below options could be turned into boot
	86	+# parameters, to optimize for the common use-case, we avoid this because: (a)
	87	+# it would impact performance (and we want to avoid static branch for all
	88	+# {READ,WRITE}_ONCE, atomic_*, bitops, etc.), and (b) complicate the design
	89	+# without real benefit. The main purpose of the below options are for use in
	90	+# fuzzer configs to control reported data races, and are not expected to be
	91	+# switched frequently by a user.
	92	+
	93	+config KCSAN_REPORT_RACE_UNKNOWN_ORIGIN
	94	+ bool "Report races of unknown origin"
	95	+ default y
	96	+ help
	97	+ If KCSAN should report races where only one access is known, and the
	98	+ conflicting access is of unknown origin. This type of race is
	99	+ reported if it was only possible to infer a race due to a data value
	100	+ change while an access is being delayed on a watchpoint.
	101	+
	102	+config KCSAN_REPORT_VALUE_CHANGE_ONLY
	103	+ bool "Only report races where watcher observed a data value change"
	104	+ default y
	105	+ help
	106	+ If enabled and a conflicting write is observed via watchpoint, but
	107	+ the data value of the memory location was observed to remain
	108	+ unchanged, do not report the data race.
	109	+
	110	+config KCSAN_IGNORE_ATOMICS
	111	+ bool "Do not instrument marked atomic accesses"
	112	+ default n
	113	+ help
	114	+ If enabled, never instruments marked atomic accesses. This results in
	115	+ not reporting data races where one access is atomic and the other is
	116	+ a plain access.
	117	+
	118	+endif # KCSAN

lib/Makefile+3−0

@@ -24,6 +24,9 @@ KASAN_SANITIZE_string.o := n · 23 unmodified lines
24	24	CFLAGS_string.o := $(call cc-option, -fno-stack-protector)
25	25	endif
26	26
	27	+# Used by KCSAN while enabled, avoid recursion.
	28	+KCSAN_SANITIZE_random32.o := n
	29	+
27	30	lib-y := ctype.o string.o vsprintf.o cmdline.o \
28	31	rbtree.o radix-tree.o timerqueue.o xarray.o \
29	32	idr.o extable.o \

scripts/Makefile.kcsan+6−0

@@ -0,0 +1,6 @@
	1	+# SPDX-License-Identifier: GPL-2.0
	2	+ifdef CONFIG_KCSAN
	3	+
	4	+CFLAGS_KCSAN := -fsanitize=thread
	5	+
	6	+endif # CONFIG_KCSAN

scripts/Makefile.lib+10−0

@@ -152,6 +152,16 @@ _c_flags += $(if $(patsubst n%,, \ · 151 unmodified lines
152	152	$(CFLAGS_KCOV))
153	153	endif
154	154
	155	+#
	156	+# Enable KCSAN flags except some files or directories we don't want to check
	157	+# (depends on variables KCSAN_SANITIZE_obj.o, KCSAN_SANITIZE)
	158	+#
	159	+ifeq ($(CONFIG_KCSAN),y)
	160	+_c_flags += $(if $(patsubst n%,, \
	161	+ $(KCSAN_SANITIZE_$(basetarget).o)$(KCSAN_SANITIZE)y), \
	162	+ $(CFLAGS_KCSAN))
	163	+endif
	164	+
155	165	# $(srctree)/$(src) for including checkin headers from generated source files
156	166	# $(objtree)/$(obj) for including generated headers from checkin source files
157	167	ifeq ($(KBUILD_EXTMOD),)