kcsan: Add Kernel Concurrency Sanitizer infrastructure

Kernel Concurrency Sanitizer (KCSAN) is a dynamic data-race detector for kernel space. KCSAN is a sampling watchpoint-based data-race detector. See the included Documentation/dev-tools/kcsan.rst for more details. This patch adds basic infrastructure, but does not yet enable KCSAN for any architecture.

Marco Elver committed 7 years ago

Commit dfd402a1 parent 31f4f5b24 files · +2006 −9Browse files

Showing the changes to include/linux (6 of 24 files). Show all

Unified · Split

6 files

include/linux/compiler-clang.h +9 −0
include/linux/compiler-gcc.h +7 −0
include/linux/compiler.h +29 −8
include/linux/kcsan-checks.h +97 −0
include/linux/kcsan.h +115 −0
include/linux/sched.h +4 −0

include/linux/compiler-clang.h+9−0

@@ -24,6 +24,15 @@ · 23 unmodified lines
24	24	#define __no_sanitize_address
25	25	#endif
26	26
	27	+#if __has_feature(thread_sanitizer)
	28	+/* emulate gcc's __SANITIZE_THREAD__ flag */
	29	+#define __SANITIZE_THREAD__
	30	+#define __no_sanitize_thread \
	31	+ __attribute__((no_sanitize("thread")))
	32	+#else
	33	+#define __no_sanitize_thread
	34	+#endif
	35	+
27	36	/*
28	37	* Not all versions of clang implement the the type-generic versions
29	38	* of the builtin overflow checkers. Fortunately, clang implements

include/linux/compiler-gcc.h+7−0

@@ -145,6 +145,13 @@ · 144 unmodified lines
145	145	#define __no_sanitize_address
146	146	#endif
147	147
	148	+#if defined(__SANITIZE_THREAD__) && __has_attribute(__no_sanitize_thread__)
	149	+#define __no_sanitize_thread \
	150	+ __attribute__((__noinline__)) __attribute__((no_sanitize_thread))
	151	+#else
	152	+#define __no_sanitize_thread
	153	+#endif
	154	+
148	155	#if GCC_VERSION >= 50100
149	156	#define COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW 1
150	157	#endif

include/linux/compiler.h+29−8

@@ -178,6 +178,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 177 unmodified lines
178	178	#endif
179	179
180	180	#include <uapi/linux/types.h>
	181	+#include <linux/kcsan-checks.h>
181	182
182	183	#define __READ_ONCE_SIZE \
183	184	({ \
@@ -193,12 +194,6 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, · 9 unmodified lines
193	194	} \
194	195	})
195	196
196		−static __always_inline
197		−void __read_once_size(const volatile void p, void res, int size)
198		−{
199		− __READ_ONCE_SIZE;
200		−}
201		−
202	197	#ifdef CONFIG_KASAN
203	198	/*
204	199	* We can't declare function 'inline' because __no_sanitize_address confilcts
@@ -207,18 +202,44 @@ void __read_once_size(const volatile void p, void res, int size) · 2 unmodified lines
207	202	* '__maybe_unused' allows us to avoid defined-but-not-used warnings.
208	203	*/
209	204	# define __no_kasan_or_inline __no_sanitize_address notrace __maybe_unused
	205	+# define __no_sanitize_or_inline __no_kasan_or_inline
210	206	#else
211	207	# define __no_kasan_or_inline __always_inline
212	208	#endif
213	209
214		−static __no_kasan_or_inline
	210	+#ifdef __SANITIZE_THREAD__
	211	+/*
	212	+ * Rely on __SANITIZE_THREAD__ instead of CONFIG_KCSAN, to avoid not inlining in
	213	+ * compilation units where instrumentation is disabled.
	214	+ */
	215	+# define __no_kcsan_or_inline __no_sanitize_thread notrace __maybe_unused
	216	+# define __no_sanitize_or_inline __no_kcsan_or_inline
	217	+#else
	218	+# define __no_kcsan_or_inline __always_inline
	219	+#endif
	220	+
	221	+#ifndef __no_sanitize_or_inline
	222	+#define __no_sanitize_or_inline __always_inline
	223	+#endif
	224	+
	225	+static __no_kcsan_or_inline
	226	+void __read_once_size(const volatile void p, void res, int size)
	227	+{
	228	+ kcsan_check_atomic_read(p, size);
	229	+ __READ_ONCE_SIZE;
	230	+}
	231	+
	232	+static __no_sanitize_or_inline
215	233	void __read_once_size_nocheck(const volatile void p, void res, int size)
216	234	{
217	235	__READ_ONCE_SIZE;
218	236	}
219	237
220		−static __always_inline void __write_once_size(volatile void p, void res, int size)
	238	+static __no_kcsan_or_inline
	239	+void __write_once_size(volatile void p, void res, int size)
221	240	{
	241	+ kcsan_check_atomic_write(p, size);
	242	+
222	243	switch (size) {
223	244	case 1: (volatile __u8 )p = (__u8 )res; break;
224	245	case 2: (volatile __u16 )p = (__u16 )res; break;

include/linux/kcsan-checks.h+97−0

@@ -0,0 +1,97 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+
	3	+#ifndef _LINUX_KCSAN_CHECKS_H
	4	+#define _LINUX_KCSAN_CHECKS_H
	5	+
	6	+#include <linux/types.h>
	7	+
	8	+/*
	9	+ * Access type modifiers.
	10	+ */
	11	+#define KCSAN_ACCESS_WRITE 0x1
	12	+#define KCSAN_ACCESS_ATOMIC 0x2
	13	+
	14	+/*
	15	+ * __kcsan_*: Always calls into runtime when KCSAN is enabled. This may be used
	16	+ * even in compilation units that selectively disable KCSAN, but must use KCSAN
	17	+ * to validate access to an address. Never use these in header files!
	18	+ */
	19	+#ifdef CONFIG_KCSAN
	20	+/**
	21	+ * __kcsan_check_access - check generic access for data race
	22	+ *
	23	+ * @ptr address of access
	24	+ * @size size of access
	25	+ * @type access type modifier
	26	+ */
	27	+void __kcsan_check_access(const volatile void *ptr, size_t size, int type);
	28	+
	29	+#else
	30	+static inline void __kcsan_check_access(const volatile void *ptr, size_t size,
	31	+ int type) { }
	32	+#endif
	33	+
	34	+/*
	35	+ * kcsan_*: Only calls into runtime when the particular compilation unit has
	36	+ * KCSAN instrumentation enabled. May be used in header files.
	37	+ */
	38	+#ifdef __SANITIZE_THREAD__
	39	+#define kcsan_check_access __kcsan_check_access
	40	+#else
	41	+static inline void kcsan_check_access(const volatile void *ptr, size_t size,
	42	+ int type) { }
	43	+#endif
	44	+
	45	+/**
	46	+ * __kcsan_check_read - check regular read access for data races
	47	+ *
	48	+ * @ptr address of access
	49	+ * @size size of access
	50	+ */
	51	+#define __kcsan_check_read(ptr, size) __kcsan_check_access(ptr, size, 0)
	52	+
	53	+/**
	54	+ * __kcsan_check_write - check regular write access for data races
	55	+ *
	56	+ * @ptr address of access
	57	+ * @size size of access
	58	+ */
	59	+#define __kcsan_check_write(ptr, size) \
	60	+ __kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
	61	+
	62	+/**
	63	+ * kcsan_check_read - check regular read access for data races
	64	+ *
	65	+ * @ptr address of access
	66	+ * @size size of access
	67	+ */
	68	+#define kcsan_check_read(ptr, size) kcsan_check_access(ptr, size, 0)
	69	+
	70	+/**
	71	+ * kcsan_check_write - check regular write access for data races
	72	+ *
	73	+ * @ptr address of access
	74	+ * @size size of access
	75	+ */
	76	+#define kcsan_check_write(ptr, size) \
	77	+ kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
	78	+
	79	+/*
	80	+ * Check for atomic accesses: if atomic access are not ignored, this simply
	81	+ * aliases to kcsan_check_access, otherwise becomes a no-op.
	82	+ */
	83	+#ifdef CONFIG_KCSAN_IGNORE_ATOMICS
	84	+#define kcsan_check_atomic_read(...) \
	85	+ do { \
	86	+ } while (0)
	87	+#define kcsan_check_atomic_write(...) \
	88	+ do { \
	89	+ } while (0)
	90	+#else
	91	+#define kcsan_check_atomic_read(ptr, size) \
	92	+ kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC)
	93	+#define kcsan_check_atomic_write(ptr, size) \
	94	+ kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC \| KCSAN_ACCESS_WRITE)
	95	+#endif
	96	+
	97	+#endif /* _LINUX_KCSAN_CHECKS_H */

include/linux/kcsan.h+115−0

@@ -0,0 +1,115 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+
	3	+#ifndef _LINUX_KCSAN_H
	4	+#define _LINUX_KCSAN_H
	5	+
	6	+#include <linux/kcsan-checks.h>
	7	+#include <linux/types.h>
	8	+
	9	+#ifdef CONFIG_KCSAN
	10	+
	11	+/*
	12	+ * Context for each thread of execution: for tasks, this is stored in
	13	+ * task_struct, and interrupts access internal per-CPU storage.
	14	+ */
	15	+struct kcsan_ctx {
	16	+ int disable_count; /* disable counter */
	17	+ int atomic_next; /* number of following atomic ops */
	18	+
	19	+ /*
	20	+ * We distinguish between: (a) nestable atomic regions that may contain
	21	+ * other nestable regions; and (b) flat atomic regions that do not keep
	22	+ * track of nesting. Both (a) and (b) are entirely independent of each
	23	+ * other, and a flat region may be started in a nestable region or
	24	+ * vice-versa.
	25	+ *
	26	+ * This is required because, for example, in the annotations for
	27	+ * seqlocks, we declare seqlock writer critical sections as (a) nestable
	28	+ * atomic regions, but reader critical sections as (b) flat atomic
	29	+ * regions, but have encountered cases where seqlock reader critical
	30	+ * sections are contained within writer critical sections (the opposite
	31	+ * may be possible, too).
	32	+ *
	33	+ * To support these cases, we independently track the depth of nesting
	34	+ * for (a), and whether the leaf level is flat for (b).
	35	+ */
	36	+ int atomic_nest_count;
	37	+ bool in_flat_atomic;
	38	+};
	39	+
	40	+/**
	41	+ * kcsan_init - initialize KCSAN runtime
	42	+ */
	43	+void kcsan_init(void);
	44	+
	45	+/**
	46	+ * kcsan_disable_current - disable KCSAN for the current context
	47	+ *
	48	+ * Supports nesting.
	49	+ */
	50	+void kcsan_disable_current(void);
	51	+
	52	+/**
	53	+ * kcsan_enable_current - re-enable KCSAN for the current context
	54	+ *
	55	+ * Supports nesting.
	56	+ */
	57	+void kcsan_enable_current(void);
	58	+
	59	+/**
	60	+ * kcsan_nestable_atomic_begin - begin nestable atomic region
	61	+ *
	62	+ * Accesses within the atomic region may appear to race with other accesses but
	63	+ * should be considered atomic.
	64	+ */
	65	+void kcsan_nestable_atomic_begin(void);
	66	+
	67	+/**
	68	+ * kcsan_nestable_atomic_end - end nestable atomic region
	69	+ */
	70	+void kcsan_nestable_atomic_end(void);
	71	+
	72	+/**
	73	+ * kcsan_flat_atomic_begin - begin flat atomic region
	74	+ *
	75	+ * Accesses within the atomic region may appear to race with other accesses but
	76	+ * should be considered atomic.
	77	+ */
	78	+void kcsan_flat_atomic_begin(void);
	79	+
	80	+/**
	81	+ * kcsan_flat_atomic_end - end flat atomic region
	82	+ */
	83	+void kcsan_flat_atomic_end(void);
	84	+
	85	+/**
	86	+ * kcsan_atomic_next - consider following accesses as atomic
	87	+ *
	88	+ * Force treating the next n memory accesses for the current context as atomic
	89	+ * operations.
	90	+ *
	91	+ * @n number of following memory accesses to treat as atomic.
	92	+ */
	93	+void kcsan_atomic_next(int n);
	94	+
	95	+#else /* CONFIG_KCSAN */
	96	+
	97	+static inline void kcsan_init(void) { }
	98	+
	99	+static inline void kcsan_disable_current(void) { }
	100	+
	101	+static inline void kcsan_enable_current(void) { }
	102	+
	103	+static inline void kcsan_nestable_atomic_begin(void) { }
	104	+
	105	+static inline void kcsan_nestable_atomic_end(void) { }
	106	+
	107	+static inline void kcsan_flat_atomic_begin(void) { }
	108	+
	109	+static inline void kcsan_flat_atomic_end(void) { }
	110	+
	111	+static inline void kcsan_atomic_next(int n) { }
	112	+
	113	+#endif /* CONFIG_KCSAN */
	114	+
	115	+#endif /* _LINUX_KCSAN_H */

include/linux/sched.h+4−0

@@ -31,6 +31,7 @@ · 30 unmodified lines
31	31	#include <linux/task_io_accounting.h>
32	32	#include <linux/posix-timers.h>
33	33	#include <linux/rseq.h>
	34	+#include <linux/kcsan.h>
34	35
35	36	/* task_struct member predeclarations (sorted alphabetically): */
36	37	struct audit_context;
@@ -1172,6 +1173,9 @@ struct task_struct { · 1135 unmodified lines
1172	1173	#ifdef CONFIG_KASAN
1173	1174	unsigned int kasan_depth;
1174	1175	#endif
	1176	+#ifdef CONFIG_KCSAN
	1177	+ struct kcsan_ctx kcsan_ctx;
	1178	+#endif
1175	1179
1176	1180	#ifdef CONFIG_FUNCTION_GRAPH_TRACER
1177	1181	/* Index of current stored address in ret_stack: */

kcsan: Add Kernel Concurrency Sanitizer infrastructure

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code