kcsan: Add core support for a subset of weak memory modeling

Add support for modeling a subset of weak memory, which will enable detection of a subset of data races due to missing memory barriers. KCSAN's approach to detecting missing memory barriers is based on modeling access reordering, and enabled if `CONFIG_KCSAN_WEAK_MEMORY=y`, which depends on `CONFIG_KCSAN_STRICT=y`. The feature can be enabled or disabled at boot and runtime via the `kcsan.weak_memory` boot parameter. Each memory access for which a watchpoint is set up, is also selected for simulated reordering within the scope of its function (at most 1 in-flight access). We are limited to modeling the effects of "buffering" (delaying the access), since the runtime cannot "prefetch" accesses (therefore no acquire modeling). Once an access has been selected for reordering, it is checked along every other access until the end of the function scope. If an appropriate memory barrier is encountered, the access will no longer be considered for reordering. When the result of a memory operation should be ordered by a barrier, KCSAN can then detect data races where the conflict only occurs as a result of a missing barrier due to reordering accesses.

Marco Elver committed 5 years ago

Commit 69562e41 parent 9756f646 files · +235 −19Browse files

Unified · Split

6 files

include/linux/kcsan-checks.h +9 −1
include/linux/kcsan.h +9 −1
include/linux/sched.h +3 −0
kernel/kcsan/core.c +187 −15
lib/Kconfig.kcsan +20 −0
scripts/Makefile.kcsan +7 −2

include/linux/kcsan-checks.h+9−1

@@ -99,7 +99,15 @@ void kcsan_set_access_mask(unsigned long mask); · 98 unmodified lines
99		99
100	/* Scoped access information. */	100	/* Scoped access information. */
101	struct kcsan_scoped_access {	101	struct kcsan_scoped_access {
102	struct list_head list;	102	union {
		103	struct list_head list; /* scoped_accesses list */
		104	/*
		105	* Not an entry in scoped_accesses list; stack depth from where
		106	* the access was initialized.
		107	*/
		108	int stack_depth;
		109	};
		110
103	/* Access information. */	111	/* Access information. */
104	const volatile void *ptr;	112	const volatile void *ptr;
105	size_t size;	113	size_t size;

include/linux/kcsan.h+9−1

 	 */
 	unsigned long access_mask;
-	/* List of scoped accesses. */
+	/* List of scoped accesses; likely to be empty. */
 	struct list_head scoped_accesses;
+#ifdef CONFIG_KCSAN_WEAK_MEMORY
+	/*
+	 * Scoped access for modeling access reordering to detect missing memory
+	 * barriers; only keep 1 to keep fast-path complexity manageable.
+	 */
+	struct kcsan_scoped_access reorder_access;
+#endif
 };
 /**

include/linux/sched.h+3−0

 #ifdef CONFIG_TRACE_IRQFLAGS
 	struct irqtrace_events		kcsan_save_irqtrace;
 #endif
+#ifdef CONFIG_KCSAN_WEAK_MEMORY
+	int				kcsan_stack_depth;
+#endif
 #endif
 #if IS_ENABLED(CONFIG_KUNIT)

kernel/kcsan/core.c+187−15

@@ -40,6 +40,13 @@ module_param_named(udelay_interrupt, kcsan_udelay_interrupt, uint, 0644); · 39 unmodified lines
40	module_param_named(skip_watch, kcsan_skip_watch, long, 0644);	40	module_param_named(skip_watch, kcsan_skip_watch, long, 0644);
41	module_param_named(interrupt_watcher, kcsan_interrupt_watcher, bool, 0444);	41	module_param_named(interrupt_watcher, kcsan_interrupt_watcher, bool, 0444);
42		42
		43	#ifdef CONFIG_KCSAN_WEAK_MEMORY
		44	static bool kcsan_weak_memory = true;
		45	module_param_named(weak_memory, kcsan_weak_memory, bool, 0644);
		46	#else
		47	#define kcsan_weak_memory false
		48	#endif
		49
43	bool kcsan_enabled;	50	bool kcsan_enabled;
44		51
45	/* Per-CPU kcsan_ctx for interrupts */	52	/* Per-CPU kcsan_ctx for interrupts */
@@ -351,6 +358,67 @@ void kcsan_restore_irqtrace(struct task_struct *task) · 305 unmodified lines
351	#endif	358	#endif
352	}	359	}
353		360
		361	static __always_inline int get_kcsan_stack_depth(void)
		362	{
		363	#ifdef CONFIG_KCSAN_WEAK_MEMORY
		364	return current->kcsan_stack_depth;
		365	#else
		366	BUILD_BUG();
		367	return 0;
		368	#endif
		369	}
		370
		371	static __always_inline void add_kcsan_stack_depth(int val)
		372	{
		373	#ifdef CONFIG_KCSAN_WEAK_MEMORY
		374	current->kcsan_stack_depth += val;
		375	#else
		376	BUILD_BUG();
		377	#endif
		378	}
		379
		380	static __always_inline struct kcsan_scoped_access get_reorder_access(struct kcsan_ctx ctx)
		381	{
		382	#ifdef CONFIG_KCSAN_WEAK_MEMORY
		383	return ctx->disable_scoped ? NULL : &ctx->reorder_access;
		384	#else
		385	return NULL;
		386	#endif
		387	}
		388
		389	static __always_inline bool
		390	find_reorder_access(struct kcsan_ctx ctx, const volatile void ptr, size_t size,
		391	int type, unsigned long ip)
		392	{
		393	struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx);
		394
		395	if (!reorder_access)
		396	return false;
		397
		398	/*
		399	* Note: If accesses are repeated while reorder_access is identical,
		400	* never matches the new access, because !(type & KCSAN_ACCESS_SCOPED).
		401	*/
		402	return reorder_access->ptr == ptr && reorder_access->size == size &&
		403	reorder_access->type == type && reorder_access->ip == ip;
		404	}
		405
		406	static inline void
		407	set_reorder_access(struct kcsan_ctx ctx, const volatile void ptr, size_t size,
		408	int type, unsigned long ip)
		409	{
		410	struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx);
		411
		412	if (!reorder_access \|\| !kcsan_weak_memory)
		413	return;
		414
		415	reorder_access->ptr = ptr;
		416	reorder_access->size = size;
		417	reorder_access->type = type \| KCSAN_ACCESS_SCOPED;
		418	reorder_access->ip = ip;
		419	reorder_access->stack_depth = get_kcsan_stack_depth();
		420	}
		421
354	/*	422	/*
355	* Pull everything together: check_access() below contains the performance	423	* Pull everything together: check_access() below contains the performance
356	* critical operations; the fast-path (including check_access) functions should	424	* critical operations; the fast-path (including check_access) functions should
@@ -389,8 +457,10 @@ static noinline void kcsan_found_watchpoint(const volatile void *ptr, · 32 unmodified lines
389	* The access_mask check relies on value-change comparison. To avoid	457	* The access_mask check relies on value-change comparison. To avoid
390	* reporting a race where e.g. the writer set up the watchpoint, but the	458	* reporting a race where e.g. the writer set up the watchpoint, but the
391	* reader has access_mask!=0, we have to ignore the found watchpoint.	459	* reader has access_mask!=0, we have to ignore the found watchpoint.
		460	*
		461	* reorder_access is never created from an access with access_mask set.
392	*/	462	*/
393	if (ctx->access_mask)	463	if (ctx->access_mask && !find_reorder_access(ctx, ptr, size, type, ip))
394	return;	464	return;
395		465
396	/*	466	/*
@@ -440,11 +510,13 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 43 unmodified lines
440	const bool is_assert = (type & KCSAN_ACCESS_ASSERT) != 0;	510	const bool is_assert = (type & KCSAN_ACCESS_ASSERT) != 0;
441	atomic_long_t *watchpoint;	511	atomic_long_t *watchpoint;
442	u64 old, new, diff;	512	u64 old, new, diff;
443	unsigned long access_mask;
444	enum kcsan_value_change value_change = KCSAN_VALUE_CHANGE_MAYBE;	513	enum kcsan_value_change value_change = KCSAN_VALUE_CHANGE_MAYBE;
		514	bool interrupt_watcher = kcsan_interrupt_watcher;
445	unsigned long ua_flags = user_access_save();	515	unsigned long ua_flags = user_access_save();
446	struct kcsan_ctx *ctx = get_ctx();	516	struct kcsan_ctx *ctx = get_ctx();
		517	unsigned long access_mask = ctx->access_mask;
447	unsigned long irq_flags = 0;	518	unsigned long irq_flags = 0;
		519	bool is_reorder_access;
448		520
449	/*	521	/*
450	* Always reset kcsan_skip counter in slow-path to avoid underflow; see	522	* Always reset kcsan_skip counter in slow-path to avoid underflow; see
@@ -467,6 +539,17 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 16 unmodified lines
467	goto out;	539	goto out;
468	}	540	}
469		541
		542	/*
		543	* The local CPU cannot observe reordering of its own accesses, and
		544	* therefore we need to take care of 2 cases to avoid false positives:
		545	*
		546	* 1. Races of the reordered access with interrupts. To avoid, if
		547	* the current access is reorder_access, disable interrupts.
		548	* 2. Avoid races of scoped accesses from nested interrupts (below).
		549	*/
		550	is_reorder_access = find_reorder_access(ctx, ptr, size, type, ip);
		551	if (is_reorder_access)
		552	interrupt_watcher = false;
470	/*	553	/*
471	* Avoid races of scoped accesses from nested interrupts (or scheduler).	554	* Avoid races of scoped accesses from nested interrupts (or scheduler).
472	* Assume setting up a watchpoint for a non-scoped (normal) access that	555	* Assume setting up a watchpoint for a non-scoped (normal) access that
@@ -482,7 +565,7 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 9 unmodified lines
482	* information is lost if dirtied by KCSAN.	565	* information is lost if dirtied by KCSAN.
483	*/	566	*/
484	kcsan_save_irqtrace(current);	567	kcsan_save_irqtrace(current);
485	if (!kcsan_interrupt_watcher)	568	if (!interrupt_watcher)
486	local_irq_save(irq_flags);	569	local_irq_save(irq_flags);
487		570
488	watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write);	571	watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write);
@@ -503,7 +586,7 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 14 unmodified lines
503	* Read the current value, to later check and infer a race if the data	586	* Read the current value, to later check and infer a race if the data
504	* was modified via a non-instrumented access, e.g. from a device.	587	* was modified via a non-instrumented access, e.g. from a device.
505	*/	588	*/
506	old = read_instrumented_memory(ptr, size);	589	old = is_reorder_access ? 0 : read_instrumented_memory(ptr, size);
507		590
508	/*	591	/*
509	* Delay this thread, to increase probability of observing a racy	592	* Delay this thread, to increase probability of observing a racy
@@ -515,8 +598,17 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 5 unmodified lines
515	* Re-read value, and check if it is as expected; if not, we infer a	598	* Re-read value, and check if it is as expected; if not, we infer a
516	* racy access.	599	* racy access.
517	*/	600	*/
518	access_mask = ctx->access_mask;	601	if (!is_reorder_access) {
519	new = read_instrumented_memory(ptr, size);	602	new = read_instrumented_memory(ptr, size);
		603	} else {
		604	/*
		605	* Reordered accesses cannot be used for value change detection,
		606	* because the memory location may no longer be accessible and
		607	* could result in a fault.
		608	*/
		609	new = 0;
		610	access_mask = 0;
		611	}
520		612
521	diff = old ^ new;	613	diff = old ^ new;
522	if (access_mask)	614	if (access_mask)
@@ -585,11 +677,20 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 62 unmodified lines
585	*/	677	*/
586	remove_watchpoint(watchpoint);	678	remove_watchpoint(watchpoint);
587	atomic_long_dec(&kcsan_counters[KCSAN_COUNTER_USED_WATCHPOINTS]);	679	atomic_long_dec(&kcsan_counters[KCSAN_COUNTER_USED_WATCHPOINTS]);
		680
588	out_unlock:	681	out_unlock:
589	if (!kcsan_interrupt_watcher)	682	if (!interrupt_watcher)
590	local_irq_restore(irq_flags);	683	local_irq_restore(irq_flags);
591	kcsan_restore_irqtrace(current);	684	kcsan_restore_irqtrace(current);
592	ctx->disable_scoped--;	685	ctx->disable_scoped--;
		686
		687	/*
		688	* Reordered accesses cannot be used for value change detection,
		689	* therefore never consider for reordering if access_mask is set.
		690	* ASSERT_EXCLUSIVE are not real accesses, ignore them as well.
		691	*/
		692	if (!access_mask && !is_assert)
		693	set_reorder_access(ctx, ptr, size, type, ip);
593	out:	694	out:
594	user_access_restore(ua_flags);	695	user_access_restore(ua_flags);
595	}	696	}
@@ -597,7 +698,6 @@ out: · 1 unmodified line
597	static __always_inline void	698	static __always_inline void
598	check_access(const volatile void *ptr, size_t size, int type, unsigned long ip)	699	check_access(const volatile void *ptr, size_t size, int type, unsigned long ip)
599	{	700	{
600	const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0;
601	atomic_long_t *watchpoint;	701	atomic_long_t *watchpoint;
602	long encoded_watchpoint;	702	long encoded_watchpoint;
603		703
@@ -608,12 +708,14 @@ check_access(const volatile void *ptr, size_t size, int type, unsigned long ip) · 4 unmodified lines
608	if (unlikely(size == 0))	708	if (unlikely(size == 0))
609	return;	709	return;
610		710
		711	again:
611	/*	712	/*
612	* Avoid user_access_save in fast-path: find_watchpoint is safe without	713	* Avoid user_access_save in fast-path: find_watchpoint is safe without
613	* user_access_save, as the address that ptr points to is only used to	714	* user_access_save, as the address that ptr points to is only used to
614	* check if a watchpoint exists; ptr is never dereferenced.	715	* check if a watchpoint exists; ptr is never dereferenced.
615	*/	716	*/
616	watchpoint = find_watchpoint((unsigned long)ptr, size, !is_write,	717	watchpoint = find_watchpoint((unsigned long)ptr, size,
		718	!(type & KCSAN_ACCESS_WRITE),
617	&encoded_watchpoint);	719	&encoded_watchpoint);
618	/*	720	/*
619	* It is safe to check kcsan_is_enabled() after find_watchpoint in the	721	* It is safe to check kcsan_is_enabled() after find_watchpoint in the
@@ -627,9 +729,42 @@ check_access(const volatile void *ptr, size_t size, int type, unsigned long ip) · 7 unmodified lines
627	else {	729	else {
628	struct kcsan_ctx ctx = get_ctx(); / Call only once in fast-path. */	730	struct kcsan_ctx ctx = get_ctx(); / Call only once in fast-path. */
629		731
630	if (unlikely(should_watch(ctx, ptr, size, type)))	732	if (unlikely(should_watch(ctx, ptr, size, type))) {
631	kcsan_setup_watchpoint(ptr, size, type, ip);	733	kcsan_setup_watchpoint(ptr, size, type, ip);
632	else if (unlikely(ctx->scoped_accesses.prev))	734	return;
		735	}
		736
		737	if (!(type & KCSAN_ACCESS_SCOPED)) {
		738	struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx);
		739
		740	if (reorder_access) {
		741	/*
		742	* reorder_access check: simulates reordering of
		743	* the access after subsequent operations.
		744	*/
		745	ptr = reorder_access->ptr;
		746	type = reorder_access->type;
		747	ip = reorder_access->ip;
		748	/*
		749	* Upon a nested interrupt, this context's
		750	* reorder_access can be modified (shared ctx).
		751	* We know that upon return, reorder_access is
		752	* always invalidated by setting size to 0 via
		753	* __tsan_func_exit(). Therefore we must read
		754	* and check size after the other fields.
		755	*/
		756	barrier();
		757	size = READ_ONCE(reorder_access->size);
		758	if (size)
		759	goto again;
		760	}
		761	}
		762
		763	/*
		764	* Always checked last, right before returning from runtime;
		765	* if reorder_access is valid, checked after it was checked.
		766	*/
		767	if (unlikely(ctx->scoped_accesses.prev))
633	kcsan_check_scoped_accesses();	768	kcsan_check_scoped_accesses();
634	}	769	}
635	}	770	}
@@ -916,19 +1051,56 @@ DEFINE_TSAN_VOLATILE_READ_WRITE(8); · 280 unmodified lines
916	DEFINE_TSAN_VOLATILE_READ_WRITE(16);	1051	DEFINE_TSAN_VOLATILE_READ_WRITE(16);
917		1052
918	/*	1053	/*
919	* The below are not required by KCSAN, but can still be emitted by the	1054	* Function entry and exit are used to determine the validty of reorder_access.
920	* compiler.	1055	* Reordering of the access ends at the end of the function scope where the
		1056	* access happened. This is done for two reasons:
		1057	*
		1058	* 1. Artificially limits the scope where missing barriers are detected.
		1059	* This minimizes false positives due to uninstrumented functions that
		1060	* contain the required barriers but were missed.
		1061	*
		1062	* 2. Simplifies generating the stack trace of the access.
921	*/	1063	*/
922	void __tsan_func_entry(void *call_pc);	1064	void __tsan_func_entry(void *call_pc);
923	void __tsan_func_entry(void *call_pc)	1065	noinline void __tsan_func_entry(void *call_pc)
924	{	1066	{
		1067	if (!IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY))
		1068	return;
		1069
		1070	add_kcsan_stack_depth(1);
925	}	1071	}
926	EXPORT_SYMBOL(__tsan_func_entry);	1072	EXPORT_SYMBOL(__tsan_func_entry);
		1073
927	void __tsan_func_exit(void);	1074	void __tsan_func_exit(void);
928	void __tsan_func_exit(void)	1075	noinline void __tsan_func_exit(void)
929	{	1076	{
		1077	struct kcsan_scoped_access *reorder_access;
		1078
		1079	if (!IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY))
		1080	return;
		1081
		1082	reorder_access = get_reorder_access(get_ctx());
		1083	if (!reorder_access)
		1084	goto out;
		1085
		1086	if (get_kcsan_stack_depth() <= reorder_access->stack_depth) {
		1087	/*
		1088	* Access check to catch cases where write without a barrier
		1089	* (supposed release) was last access in function: because
		1090	* instrumentation is inserted before the real access, a data
		1091	* race due to the write giving up a c-s would only be caught if
		1092	* we do the conflicting access after.
		1093	*/
		1094	check_access(reorder_access->ptr, reorder_access->size,
		1095	reorder_access->type, reorder_access->ip);
		1096	reorder_access->size = 0;
		1097	reorder_access->stack_depth = INT_MIN;
		1098	}
		1099	out:
		1100	add_kcsan_stack_depth(-1);
930	}	1101	}
931	EXPORT_SYMBOL(__tsan_func_exit);	1102	EXPORT_SYMBOL(__tsan_func_exit);
		1103
932	void __tsan_init(void);	1104	void __tsan_init(void);
933	void __tsan_init(void)	1105	void __tsan_init(void)
934	{	1106	{

lib/Kconfig.kcsan+20−0

@@ -191,6 +191,26 @@ config KCSAN_STRICT · 190 unmodified lines
191	closely aligns with the rules defined by the Linux-kernel memory	191	closely aligns with the rules defined by the Linux-kernel memory
192	consistency model (LKMM).	192	consistency model (LKMM).
193		193
		194	config KCSAN_WEAK_MEMORY
		195	bool "Enable weak memory modeling to detect missing memory barriers"
		196	default y
		197	depends on KCSAN_STRICT
		198	# We can either let objtool nop __tsan_func_{entry,exit}() and builtin
		199	# atomics instrumentation in .noinstr.text, or use a compiler that can
		200	# implement __no_kcsan to really remove all instrumentation.
		201	depends on STACK_VALIDATION \|\| CC_IS_GCC
		202	help
		203	Enable support for modeling a subset of weak memory, which allows
		204	detecting a subset of data races due to missing memory barriers.
		205
		206	Depends on KCSAN_STRICT, because the options strenghtening certain
		207	plain accesses by default (depending on !KCSAN_STRICT) reduce the
		208	ability to detect any data races invoving reordered accesses, in
		209	particular reordered writes.
		210
		211	Weak memory modeling relies on additional instrumentation and may
		212	affect performance.
		213
194	config KCSAN_REPORT_VALUE_CHANGE_ONLY	214	config KCSAN_REPORT_VALUE_CHANGE_ONLY
195	bool "Only report races where watcher observed a data value change"	215	bool "Only report races where watcher observed a data value change"
196	default y	216	default y

scripts/Makefile.kcsan+7−2

 # Keep most options here optional, to allow enabling more compilers if absence
 # of some options does not break KCSAN nor causes false positive reports.
-export CFLAGS_KCSAN := -fsanitize=thread \
+kcsan-cflags := -fsanitize=thread -fno-optimize-sibling-calls \
-	$(call cc-option,$(call cc-param,tsan-instrument-func-entry-exit=0) -fno-optimize-sibling-calls) \
 	$(call cc-option,$(call cc-param,tsan-compound-read-before-write=1),$(call cc-option,$(call cc-param,tsan-instrument-read-before-write=1))) \
 	$(call cc-param,tsan-distinguish-volatile=1)
+ifndef CONFIG_KCSAN_WEAK_MEMORY
+kcsan-cflags += $(call cc-option,$(call cc-param,tsan-instrument-func-entry-exit=0))
+endif
+export CFLAGS_KCSAN := $(kcsan-cflags)