History

kcsan: Add core support for a subset of weak memory modeling

Add support for modeling a subset of weak memory, which will enable detection of a subset of data races due to missing memory barriers. KCSAN's approach to detecting missing memory barriers is based on modeling access reordering, and enabled if `CONFIG_KCSAN_WEAK_MEMORY=y`, which depends on `CONFIG_KCSAN_STRICT=y`. The feature can be enabled or disabled at boot and runtime via the `kcsan.weak_memory` boot parameter. Each memory access for which a watchpoint is set up, is also selected for simulated reordering within the scope of its function (at most 1 in-flight access). We are limited to modeling the effects of "buffering" (delaying the access), since the runtime cannot "prefetch" accesses (therefore no acquire modeling). Once an access has been selected for reordering, it is checked along every other access until the end of the function scope. If an appropriate memory barrier is encountered, the access will no longer be considered for reordering. When the result of a memory operation should be ordered by a barrier, KCSAN can then detect data races where the conflict only occurs as a result of a missing barrier due to reordering accesses.

Marco Elver committed 5 years ago

Commit 69562e41 parent 9756f646 files · +235 −19Browse files

Unified · Split

6 files
  1. include/linux/kcsan-checks.h +9 −1
  2. include/linux/kcsan.h +9 −1
  3. include/linux/sched.h +3 −0
  4. kernel/kcsan/core.c +187 −15
  5. lib/Kconfig.kcsan +20 −0
  6. scripts/Makefile.kcsan +7 −2
include/linux/kcsan-checks.h+9−1
@@ -99,7 +99,15 @@ void kcsan_set_access_mask(unsigned long mask); · 98 unmodified lines
9999
100100 /* Scoped access information. */
101101 struct kcsan_scoped_access {
102− struct list_head list;
102+ union {
103+ struct list_head list; /* scoped_accesses list */
104+ /*
105+ * Not an entry in scoped_accesses list; stack depth from where
106+ * the access was initialized.
107+ */
108+ int stack_depth;
109+ };
110+
103111 /* Access information. */
104112 const volatile void *ptr;
105113 size_t size;
include/linux/kcsan.h+9−1
@@ -49,8 +49,16 @@ struct kcsan_ctx { · 48 unmodified lines
4949 */
5050 unsigned long access_mask;
5151
52− /* List of scoped accesses. */
52+ /* List of scoped accesses; likely to be empty. */
5353 struct list_head scoped_accesses;
54+
55+#ifdef CONFIG_KCSAN_WEAK_MEMORY
56+ /*
57+ * Scoped access for modeling access reordering to detect missing memory
58+ * barriers; only keep 1 to keep fast-path complexity manageable.
59+ */
60+ struct kcsan_scoped_access reorder_access;
61+#endif
5462 };
5563
5664 /**
include/linux/sched.h+3−0
@@ -1339,6 +1339,9 @@ struct task_struct { · 1338 unmodified lines
13391339 #ifdef CONFIG_TRACE_IRQFLAGS
13401340 struct irqtrace_events kcsan_save_irqtrace;
13411341 #endif
1342+#ifdef CONFIG_KCSAN_WEAK_MEMORY
1343+ int kcsan_stack_depth;
1344+#endif
13421345 #endif
13431346
13441347 #if IS_ENABLED(CONFIG_KUNIT)
kernel/kcsan/core.c+187−15
@@ -40,6 +40,13 @@ module_param_named(udelay_interrupt, kcsan_udelay_interrupt, uint, 0644); · 39 unmodified lines
4040 module_param_named(skip_watch, kcsan_skip_watch, long, 0644);
4141 module_param_named(interrupt_watcher, kcsan_interrupt_watcher, bool, 0444);
4242
43+#ifdef CONFIG_KCSAN_WEAK_MEMORY
44+static bool kcsan_weak_memory = true;
45+module_param_named(weak_memory, kcsan_weak_memory, bool, 0644);
46+#else
47+#define kcsan_weak_memory false
48+#endif
49+
4350 bool kcsan_enabled;
4451
4552 /* Per-CPU kcsan_ctx for interrupts */
@@ -351,6 +358,67 @@ void kcsan_restore_irqtrace(struct task_struct *task) · 305 unmodified lines
351358 #endif
352359 }
353360
361+static __always_inline int get_kcsan_stack_depth(void)
362+{
363+#ifdef CONFIG_KCSAN_WEAK_MEMORY
364+ return current->kcsan_stack_depth;
365+#else
366+ BUILD_BUG();
367+ return 0;
368+#endif
369+}
370+
371+static __always_inline void add_kcsan_stack_depth(int val)
372+{
373+#ifdef CONFIG_KCSAN_WEAK_MEMORY
374+ current->kcsan_stack_depth += val;
375+#else
376+ BUILD_BUG();
377+#endif
378+}
379+
380+static __always_inline struct kcsan_scoped_access *get_reorder_access(struct kcsan_ctx *ctx)
381+{
382+#ifdef CONFIG_KCSAN_WEAK_MEMORY
383+ return ctx->disable_scoped ? NULL : &ctx->reorder_access;
384+#else
385+ return NULL;
386+#endif
387+}
388+
389+static __always_inline bool
390+find_reorder_access(struct kcsan_ctx *ctx, const volatile void *ptr, size_t size,
391+ int type, unsigned long ip)
392+{
393+ struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx);
394+
395+ if (!reorder_access)
396+ return false;
397+
398+ /*
399+ * Note: If accesses are repeated while reorder_access is identical,
400+ * never matches the new access, because !(type & KCSAN_ACCESS_SCOPED).
401+ */
402+ return reorder_access->ptr == ptr && reorder_access->size == size &&
403+ reorder_access->type == type && reorder_access->ip == ip;
404+}
405+
406+static inline void
407+set_reorder_access(struct kcsan_ctx *ctx, const volatile void *ptr, size_t size,
408+ int type, unsigned long ip)
409+{
410+ struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx);
411+
412+ if (!reorder_access || !kcsan_weak_memory)
413+ return;
414+
415+ reorder_access->ptr = ptr;
416+ reorder_access->size = size;
417+ reorder_access->type = type | KCSAN_ACCESS_SCOPED;
418+ reorder_access->ip = ip;
419+ reorder_access->stack_depth = get_kcsan_stack_depth();
420+}
421+
354422 /*
355423 * Pull everything together: check_access() below contains the performance
356424 * critical operations; the fast-path (including check_access) functions should
@@ -389,8 +457,10 @@ static noinline void kcsan_found_watchpoint(const volatile void *ptr, · 32 unmodified lines
389457 * The access_mask check relies on value-change comparison. To avoid
390458 * reporting a race where e.g. the writer set up the watchpoint, but the
391459 * reader has access_mask!=0, we have to ignore the found watchpoint.
460+ *
461+ * reorder_access is never created from an access with access_mask set.
392462 */
393− if (ctx->access_mask)
463+ if (ctx->access_mask && !find_reorder_access(ctx, ptr, size, type, ip))
394464 return;
395465
396466 /*
@@ -440,11 +510,13 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 43 unmodified lines
440510 const bool is_assert = (type & KCSAN_ACCESS_ASSERT) != 0;
441511 atomic_long_t *watchpoint;
442512 u64 old, new, diff;
443− unsigned long access_mask;
444513 enum kcsan_value_change value_change = KCSAN_VALUE_CHANGE_MAYBE;
514+ bool interrupt_watcher = kcsan_interrupt_watcher;
445515 unsigned long ua_flags = user_access_save();
446516 struct kcsan_ctx *ctx = get_ctx();
517+ unsigned long access_mask = ctx->access_mask;
447518 unsigned long irq_flags = 0;
519+ bool is_reorder_access;
448520
449521 /*
450522 * Always reset kcsan_skip counter in slow-path to avoid underflow; see
@@ -467,6 +539,17 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 16 unmodified lines
467539 goto out;
468540 }
469541
542+ /*
543+ * The local CPU cannot observe reordering of its own accesses, and
544+ * therefore we need to take care of 2 cases to avoid false positives:
545+ *
546+ * 1. Races of the reordered access with interrupts. To avoid, if
547+ * the current access is reorder_access, disable interrupts.
548+ * 2. Avoid races of scoped accesses from nested interrupts (below).
549+ */
550+ is_reorder_access = find_reorder_access(ctx, ptr, size, type, ip);
551+ if (is_reorder_access)
552+ interrupt_watcher = false;
470553 /*
471554 * Avoid races of scoped accesses from nested interrupts (or scheduler).
472555 * Assume setting up a watchpoint for a non-scoped (normal) access that
@@ -482,7 +565,7 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 9 unmodified lines
482565 * information is lost if dirtied by KCSAN.
483566 */
484567 kcsan_save_irqtrace(current);
485− if (!kcsan_interrupt_watcher)
568+ if (!interrupt_watcher)
486569 local_irq_save(irq_flags);
487570
488571 watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write);
@@ -503,7 +586,7 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 14 unmodified lines
503586 * Read the current value, to later check and infer a race if the data
504587 * was modified via a non-instrumented access, e.g. from a device.
505588 */
506− old = read_instrumented_memory(ptr, size);
589+ old = is_reorder_access ? 0 : read_instrumented_memory(ptr, size);
507590
508591 /*
509592 * Delay this thread, to increase probability of observing a racy
@@ -515,8 +598,17 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 5 unmodified lines
515598 * Re-read value, and check if it is as expected; if not, we infer a
516599 * racy access.
517600 */
518− access_mask = ctx->access_mask;
519− new = read_instrumented_memory(ptr, size);
601+ if (!is_reorder_access) {
602+ new = read_instrumented_memory(ptr, size);
603+ } else {
604+ /*
605+ * Reordered accesses cannot be used for value change detection,
606+ * because the memory location may no longer be accessible and
607+ * could result in a fault.
608+ */
609+ new = 0;
610+ access_mask = 0;
611+ }
520612
521613 diff = old ^ new;
522614 if (access_mask)
@@ -585,11 +677,20 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 62 unmodified lines
585677 */
586678 remove_watchpoint(watchpoint);
587679 atomic_long_dec(&kcsan_counters[KCSAN_COUNTER_USED_WATCHPOINTS]);
680+
588681 out_unlock:
589− if (!kcsan_interrupt_watcher)
682+ if (!interrupt_watcher)
590683 local_irq_restore(irq_flags);
591684 kcsan_restore_irqtrace(current);
592685 ctx->disable_scoped--;
686+
687+ /*
688+ * Reordered accesses cannot be used for value change detection,
689+ * therefore never consider for reordering if access_mask is set.
690+ * ASSERT_EXCLUSIVE are not real accesses, ignore them as well.
691+ */
692+ if (!access_mask && !is_assert)
693+ set_reorder_access(ctx, ptr, size, type, ip);
593694 out:
594695 user_access_restore(ua_flags);
595696 }
@@ -597,7 +698,6 @@ out: · 1 unmodified line
597698 static __always_inline void
598699 check_access(const volatile void *ptr, size_t size, int type, unsigned long ip)
599700 {
600− const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0;
601701 atomic_long_t *watchpoint;
602702 long encoded_watchpoint;
603703
@@ -608,12 +708,14 @@ check_access(const volatile void *ptr, size_t size, int type, unsigned long ip) · 4 unmodified lines
608708 if (unlikely(size == 0))
609709 return;
610710
711+again:
611712 /*
612713 * Avoid user_access_save in fast-path: find_watchpoint is safe without
613714 * user_access_save, as the address that ptr points to is only used to
614715 * check if a watchpoint exists; ptr is never dereferenced.
615716 */
616− watchpoint = find_watchpoint((unsigned long)ptr, size, !is_write,
717+ watchpoint = find_watchpoint((unsigned long)ptr, size,
718+ !(type & KCSAN_ACCESS_WRITE),
617719 &encoded_watchpoint);
618720 /*
619721 * It is safe to check kcsan_is_enabled() after find_watchpoint in the
@@ -627,9 +729,42 @@ check_access(const volatile void *ptr, size_t size, int type, unsigned long ip) · 7 unmodified lines
627729 else {
628730 struct kcsan_ctx *ctx = get_ctx(); /* Call only once in fast-path. */
629731
630− if (unlikely(should_watch(ctx, ptr, size, type)))
732+ if (unlikely(should_watch(ctx, ptr, size, type))) {
631733 kcsan_setup_watchpoint(ptr, size, type, ip);
632− else if (unlikely(ctx->scoped_accesses.prev))
734+ return;
735+ }
736+
737+ if (!(type & KCSAN_ACCESS_SCOPED)) {
738+ struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx);
739+
740+ if (reorder_access) {
741+ /*
742+ * reorder_access check: simulates reordering of
743+ * the access after subsequent operations.
744+ */
745+ ptr = reorder_access->ptr;
746+ type = reorder_access->type;
747+ ip = reorder_access->ip;
748+ /*
749+ * Upon a nested interrupt, this context's
750+ * reorder_access can be modified (shared ctx).
751+ * We know that upon return, reorder_access is
752+ * always invalidated by setting size to 0 via
753+ * __tsan_func_exit(). Therefore we must read
754+ * and check size after the other fields.
755+ */
756+ barrier();
757+ size = READ_ONCE(reorder_access->size);
758+ if (size)
759+ goto again;
760+ }
761+ }
762+
763+ /*
764+ * Always checked last, right before returning from runtime;
765+ * if reorder_access is valid, checked after it was checked.
766+ */
767+ if (unlikely(ctx->scoped_accesses.prev))
633768 kcsan_check_scoped_accesses();
634769 }
635770 }
@@ -916,19 +1051,56 @@ DEFINE_TSAN_VOLATILE_READ_WRITE(8); · 280 unmodified lines
9161051 DEFINE_TSAN_VOLATILE_READ_WRITE(16);
9171052
9181053 /*
919− * The below are not required by KCSAN, but can still be emitted by the
920− * compiler.
1054+ * Function entry and exit are used to determine the validty of reorder_access.
1055+ * Reordering of the access ends at the end of the function scope where the
1056+ * access happened. This is done for two reasons:
1057+ *
1058+ * 1. Artificially limits the scope where missing barriers are detected.
1059+ * This minimizes false positives due to uninstrumented functions that
1060+ * contain the required barriers but were missed.
1061+ *
1062+ * 2. Simplifies generating the stack trace of the access.
9211063 */
9221064 void __tsan_func_entry(void *call_pc);
923−void __tsan_func_entry(void *call_pc)
1065+noinline void __tsan_func_entry(void *call_pc)
9241066 {
1067+ if (!IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY))
1068+ return;
1069+
1070+ add_kcsan_stack_depth(1);
9251071 }
9261072 EXPORT_SYMBOL(__tsan_func_entry);
1073+
9271074 void __tsan_func_exit(void);
928−void __tsan_func_exit(void)
1075+noinline void __tsan_func_exit(void)
9291076 {
1077+ struct kcsan_scoped_access *reorder_access;
1078+
1079+ if (!IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY))
1080+ return;
1081+
1082+ reorder_access = get_reorder_access(get_ctx());
1083+ if (!reorder_access)
1084+ goto out;
1085+
1086+ if (get_kcsan_stack_depth() <= reorder_access->stack_depth) {
1087+ /*
1088+ * Access check to catch cases where write without a barrier
1089+ * (supposed release) was last access in function: because
1090+ * instrumentation is inserted before the real access, a data
1091+ * race due to the write giving up a c-s would only be caught if
1092+ * we do the conflicting access after.
1093+ */
1094+ check_access(reorder_access->ptr, reorder_access->size,
1095+ reorder_access->type, reorder_access->ip);
1096+ reorder_access->size = 0;
1097+ reorder_access->stack_depth = INT_MIN;
1098+ }
1099+out:
1100+ add_kcsan_stack_depth(-1);
9301101 }
9311102 EXPORT_SYMBOL(__tsan_func_exit);
1103+
9321104 void __tsan_init(void);
9331105 void __tsan_init(void)
9341106 {
lib/Kconfig.kcsan+20−0
@@ -191,6 +191,26 @@ config KCSAN_STRICT · 190 unmodified lines
191191 closely aligns with the rules defined by the Linux-kernel memory
192192 consistency model (LKMM).
193193
194+config KCSAN_WEAK_MEMORY
195+ bool "Enable weak memory modeling to detect missing memory barriers"
196+ default y
197+ depends on KCSAN_STRICT
198+ # We can either let objtool nop __tsan_func_{entry,exit}() and builtin
199+ # atomics instrumentation in .noinstr.text, or use a compiler that can
200+ # implement __no_kcsan to really remove all instrumentation.
201+ depends on STACK_VALIDATION || CC_IS_GCC
202+ help
203+ Enable support for modeling a subset of weak memory, which allows
204+ detecting a subset of data races due to missing memory barriers.
205+
206+ Depends on KCSAN_STRICT, because the options strenghtening certain
207+ plain accesses by default (depending on !KCSAN_STRICT) reduce the
208+ ability to detect any data races invoving reordered accesses, in
209+ particular reordered writes.
210+
211+ Weak memory modeling relies on additional instrumentation and may
212+ affect performance.
213+
194214 config KCSAN_REPORT_VALUE_CHANGE_ONLY
195215 bool "Only report races where watcher observed a data value change"
196216 default y
scripts/Makefile.kcsan+7−2
@@ -9,7 +9,12 @@ endif · 8 unmodified lines
99
1010 # Keep most options here optional, to allow enabling more compilers if absence
1111 # of some options does not break KCSAN nor causes false positive reports.
12−export CFLAGS_KCSAN := -fsanitize=thread \
13− $(call cc-option,$(call cc-param,tsan-instrument-func-entry-exit=0) -fno-optimize-sibling-calls) \
12+kcsan-cflags := -fsanitize=thread -fno-optimize-sibling-calls \
1413 $(call cc-option,$(call cc-param,tsan-compound-read-before-write=1),$(call cc-option,$(call cc-param,tsan-instrument-read-before-write=1))) \
1514 $(call cc-param,tsan-distinguish-volatile=1)
15+
16+ifndef CONFIG_KCSAN_WEAK_MEMORY
17+kcsan-cflags += $(call cc-option,$(call cc-param,tsan-instrument-func-entry-exit=0))
18+endif
19+
20+export CFLAGS_KCSAN := $(kcsan-cflags)