kcsan: Add Kernel Concurrency Sanitizer infrastructure

Kernel Concurrency Sanitizer (KCSAN) is a dynamic data-race detector for kernel space. KCSAN is a sampling watchpoint-based data-race detector. See the included Documentation/dev-tools/kcsan.rst for more details. This patch adds basic infrastructure, but does not yet enable KCSAN for any architecture.

Marco Elver committed 7 years ago

Commit dfd402a1 parent 31f4f5b24 files · +2006 −9Browse files

Showing the changes to include/linux (6 of 24 files). Show all

Unified · Split

6 files

include/linux/compiler-clang.h +9 −0
include/linux/compiler-gcc.h +7 −0
include/linux/compiler.h +29 −8
include/linux/kcsan-checks.h +97 −0
include/linux/kcsan.h +115 −0
include/linux/sched.h +4 −0

include/linux/compiler-clang.h+9−0

@@ -24,6 +24,15 @@ · 23 unmodified lines
24	#define __no_sanitize_address	24	#define __no_sanitize_address
25	#endif	25	#endif
26		26
		27	#if __has_feature(thread_sanitizer)
		28	/* emulate gcc's __SANITIZE_THREAD__ flag */
		29	#define __SANITIZE_THREAD__
		30	#define __no_sanitize_thread \
		31	__attribute__((no_sanitize("thread")))
		32	#else
		33	#define __no_sanitize_thread
		34	#endif
		35
27	/*	36	/*
28	* Not all versions of clang implement the the type-generic versions	37	* Not all versions of clang implement the the type-generic versions
29	* of the builtin overflow checkers. Fortunately, clang implements	38	* of the builtin overflow checkers. Fortunately, clang implements

include/linux/compiler-gcc.h+7−0

@@ -145,6 +145,13 @@ · 144 unmodified lines
145	#define __no_sanitize_address	145	#define __no_sanitize_address
146	#endif	146	#endif
147		147
		148	#if defined(__SANITIZE_THREAD__) && __has_attribute(__no_sanitize_thread__)
		149	#define __no_sanitize_thread \
		150	__attribute__((__noinline__)) __attribute__((no_sanitize_thread))
		151	#else
		152	#define __no_sanitize_thread
		153	#endif
		154
148	#if GCC_VERSION >= 50100	155	#if GCC_VERSION >= 50100
149	#define COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW 1	156	#define COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW 1
150	#endif	157	#endif

include/linux/compiler.h+29−8

 #endif
 #include <uapi/linux/types.h>
+#include <linux/kcsan-checks.h>
 #define __READ_ONCE_SIZE						\
 ({									\
 	}								\
 })
-static __always_inline
-void __read_once_size(const volatile void *p, void *res, int size)
-{
-	__READ_ONCE_SIZE;
-}
 #ifdef CONFIG_KASAN
 /*
  * We can't declare function 'inline' because __no_sanitize_address confilcts
  * '__maybe_unused' allows us to avoid defined-but-not-used warnings.
  */
 # define __no_kasan_or_inline __no_sanitize_address notrace __maybe_unused
+# define __no_sanitize_or_inline __no_kasan_or_inline
 #else
 # define __no_kasan_or_inline __always_inline
 #endif
-static __no_kasan_or_inline
+#ifdef __SANITIZE_THREAD__
+/*
+ * Rely on __SANITIZE_THREAD__ instead of CONFIG_KCSAN, to avoid not inlining in
+ * compilation units where instrumentation is disabled.
+ */
+# define __no_kcsan_or_inline __no_sanitize_thread notrace __maybe_unused
+# define __no_sanitize_or_inline __no_kcsan_or_inline
+#else
+# define __no_kcsan_or_inline __always_inline
+#endif
+#ifndef __no_sanitize_or_inline
+#define __no_sanitize_or_inline __always_inline
+#endif
+static __no_kcsan_or_inline
+void __read_once_size(const volatile void *p, void *res, int size)
+{
+	kcsan_check_atomic_read(p, size);
+	__READ_ONCE_SIZE;
+}
+static __no_sanitize_or_inline
 void __read_once_size_nocheck(const volatile void *p, void *res, int size)
 {
 	__READ_ONCE_SIZE;
 }
-static __always_inline void __write_once_size(volatile void *p, void *res, int size)
+static __no_kcsan_or_inline
+void __write_once_size(volatile void *p, void *res, int size)
 {
+	kcsan_check_atomic_write(p, size);
 	switch (size) {
 	case 1: *(volatile __u8 *)p = *(__u8 *)res; break;
 	case 2: *(volatile __u16 *)p = *(__u16 *)res; break;

include/linux/kcsan-checks.h+97−0

@@ -0,0 +1,97 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	#ifndef _LINUX_KCSAN_CHECKS_H
	4	#define _LINUX_KCSAN_CHECKS_H
	5
	6	#include <linux/types.h>
	7
	8	/*
	9	* Access type modifiers.
	10	*/
	11	#define KCSAN_ACCESS_WRITE 0x1
	12	#define KCSAN_ACCESS_ATOMIC 0x2
	13
	14	/*
	15	* __kcsan_*: Always calls into runtime when KCSAN is enabled. This may be used
	16	* even in compilation units that selectively disable KCSAN, but must use KCSAN
	17	* to validate access to an address. Never use these in header files!
	18	*/
	19	#ifdef CONFIG_KCSAN
	20	/**
	21	* __kcsan_check_access - check generic access for data race
	22	*
	23	* @ptr address of access
	24	* @size size of access
	25	* @type access type modifier
	26	*/
	27	void __kcsan_check_access(const volatile void *ptr, size_t size, int type);
	28
	29	#else
	30	static inline void __kcsan_check_access(const volatile void *ptr, size_t size,
	31	int type) { }
	32	#endif
	33
	34	/*
	35	* kcsan_*: Only calls into runtime when the particular compilation unit has
	36	* KCSAN instrumentation enabled. May be used in header files.
	37	*/
	38	#ifdef __SANITIZE_THREAD__
	39	#define kcsan_check_access __kcsan_check_access
	40	#else
	41	static inline void kcsan_check_access(const volatile void *ptr, size_t size,
	42	int type) { }
	43	#endif
	44
	45	/**
	46	* __kcsan_check_read - check regular read access for data races
	47	*
	48	* @ptr address of access
	49	* @size size of access
	50	*/
	51	#define __kcsan_check_read(ptr, size) __kcsan_check_access(ptr, size, 0)
	52
	53	/**
	54	* __kcsan_check_write - check regular write access for data races
	55	*
	56	* @ptr address of access
	57	* @size size of access
	58	*/
	59	#define __kcsan_check_write(ptr, size) \
	60	__kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
	61
	62	/**
	63	* kcsan_check_read - check regular read access for data races
	64	*
	65	* @ptr address of access
	66	* @size size of access
	67	*/
	68	#define kcsan_check_read(ptr, size) kcsan_check_access(ptr, size, 0)
	69
	70	/**
	71	* kcsan_check_write - check regular write access for data races
	72	*
	73	* @ptr address of access
	74	* @size size of access
	75	*/
	76	#define kcsan_check_write(ptr, size) \
	77	kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE)
	78
	79	/*
	80	* Check for atomic accesses: if atomic access are not ignored, this simply
	81	* aliases to kcsan_check_access, otherwise becomes a no-op.
	82	*/
	83	#ifdef CONFIG_KCSAN_IGNORE_ATOMICS
	84	#define kcsan_check_atomic_read(...) \
	85	do { \
	86	} while (0)
	87	#define kcsan_check_atomic_write(...) \
	88	do { \
	89	} while (0)
	90	#else
	91	#define kcsan_check_atomic_read(ptr, size) \
	92	kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC)
	93	#define kcsan_check_atomic_write(ptr, size) \
	94	kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC \| KCSAN_ACCESS_WRITE)
	95	#endif
	96
	97	#endif /* _LINUX_KCSAN_CHECKS_H */

include/linux/kcsan.h+115−0

@@ -0,0 +1,115 @@
	1	/* SPDX-License-Identifier: GPL-2.0 */
	2
	3	#ifndef _LINUX_KCSAN_H
	4	#define _LINUX_KCSAN_H
	5
	6	#include <linux/kcsan-checks.h>
	7	#include <linux/types.h>
	8
	9	#ifdef CONFIG_KCSAN
	10
	11	/*
	12	* Context for each thread of execution: for tasks, this is stored in
	13	* task_struct, and interrupts access internal per-CPU storage.
	14	*/
	15	struct kcsan_ctx {
	16	int disable_count; /* disable counter */
	17	int atomic_next; /* number of following atomic ops */
	18
	19	/*
	20	* We distinguish between: (a) nestable atomic regions that may contain
	21	* other nestable regions; and (b) flat atomic regions that do not keep
	22	* track of nesting. Both (a) and (b) are entirely independent of each
	23	* other, and a flat region may be started in a nestable region or
	24	* vice-versa.
	25	*
	26	* This is required because, for example, in the annotations for
	27	* seqlocks, we declare seqlock writer critical sections as (a) nestable
	28	* atomic regions, but reader critical sections as (b) flat atomic
	29	* regions, but have encountered cases where seqlock reader critical
	30	* sections are contained within writer critical sections (the opposite
	31	* may be possible, too).
	32	*
	33	* To support these cases, we independently track the depth of nesting
	34	* for (a), and whether the leaf level is flat for (b).
	35	*/
	36	int atomic_nest_count;
	37	bool in_flat_atomic;
	38	};
	39
	40	/**
	41	* kcsan_init - initialize KCSAN runtime
	42	*/
	43	void kcsan_init(void);
	44
	45	/**
	46	* kcsan_disable_current - disable KCSAN for the current context
	47	*
	48	* Supports nesting.
	49	*/
	50	void kcsan_disable_current(void);
	51
	52	/**
	53	* kcsan_enable_current - re-enable KCSAN for the current context
	54	*
	55	* Supports nesting.
	56	*/
	57	void kcsan_enable_current(void);
	58
	59	/**
	60	* kcsan_nestable_atomic_begin - begin nestable atomic region
	61	*
	62	* Accesses within the atomic region may appear to race with other accesses but
	63	* should be considered atomic.
	64	*/
	65	void kcsan_nestable_atomic_begin(void);
	66
	67	/**
	68	* kcsan_nestable_atomic_end - end nestable atomic region
	69	*/
	70	void kcsan_nestable_atomic_end(void);
	71
	72	/**
	73	* kcsan_flat_atomic_begin - begin flat atomic region
	74	*
	75	* Accesses within the atomic region may appear to race with other accesses but
	76	* should be considered atomic.
	77	*/
	78	void kcsan_flat_atomic_begin(void);
	79
	80	/**
	81	* kcsan_flat_atomic_end - end flat atomic region
	82	*/
	83	void kcsan_flat_atomic_end(void);
	84
	85	/**
	86	* kcsan_atomic_next - consider following accesses as atomic
	87	*
	88	* Force treating the next n memory accesses for the current context as atomic
	89	* operations.
	90	*
	91	* @n number of following memory accesses to treat as atomic.
	92	*/
	93	void kcsan_atomic_next(int n);
	94
	95	#else /* CONFIG_KCSAN */
	96
	97	static inline void kcsan_init(void) { }
	98
	99	static inline void kcsan_disable_current(void) { }
	100
	101	static inline void kcsan_enable_current(void) { }
	102
	103	static inline void kcsan_nestable_atomic_begin(void) { }
	104
	105	static inline void kcsan_nestable_atomic_end(void) { }
	106
	107	static inline void kcsan_flat_atomic_begin(void) { }
	108
	109	static inline void kcsan_flat_atomic_end(void) { }
	110
	111	static inline void kcsan_atomic_next(int n) { }
	112
	113	#endif /* CONFIG_KCSAN */
	114
	115	#endif /* _LINUX_KCSAN_H */

include/linux/sched.h+4−0

 #include <linux/task_io_accounting.h>
 #include <linux/posix-timers.h>
 #include <linux/rseq.h>
+#include <linux/kcsan.h>
 /* task_struct member predeclarations (sorted alphabetically): */
 struct audit_context;
 #ifdef CONFIG_KASAN
 	unsigned int			kasan_depth;
 #endif
+#ifdef CONFIG_KCSAN
+	struct kcsan_ctx		kcsan_ctx;
+#endif
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	/* Index of current stored address in ret_stack: */

kcsan: Add Kernel Concurrency Sanitizer infrastructure

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code