History

kcsan: Add current->state to implicitly atomic accesses

Add volatile current->state to list of implicitly atomic accesses. This is in preparation to eventually enable KCSAN on kernel/sched (which currently still has KCSAN_SANITIZE := n). Since accesses that match the special check in atomic.h are rare, it makes more sense to move this check to the slow-path, avoiding the additional compare in the fast-path. With the microbenchmark, a speedup of ~6% is measured.

Marco Elver committed 6 years ago

Commit 44656d31 parent 2402d0e3 files · +40 −30Browse files

Showing the changes to kernel (3 of 3 files). Show all

Unified · Split

3 files
  1. kernel/kcsan/atomic.h +7 −14
  2. kernel/kcsan/core.c +15 −7
  3. kernel/kcsan/debugfs.c +18 −9
kernel/kcsan/atomic.h+7−14
@@ -4,24 +4,17 @@ · 3 unmodified lines
4#define _KERNEL_KCSAN_ATOMIC_H4#define _KERNEL_KCSAN_ATOMIC_H
5 5
6#include <linux/jiffies.h>6#include <linux/jiffies.h>
7#include <linux/sched.h>
7 8
8/*9/*
9 * Helper that returns true if access to @ptr should be considered an atomic10 * Special rules for certain memory where concurrent conflicting accesses are
10 * access, even though it is not explicitly atomic.11 * common, however, the current convention is to not mark them; returns true if
11 *12 * access to @ptr should be considered atomic. Called from slow-path.
12 * List all volatile globals that have been observed in races, to suppress
13 * data race reports between accesses to these variables.
14 *
15 * For now, we assume that volatile accesses of globals are as strong as atomic
16 * accesses (READ_ONCE, WRITE_ONCE cast to volatile). The situation is still not
17 * entirely clear, as on some architectures (Alpha) READ_ONCE/WRITE_ONCE do more
18 * than cast to volatile. Eventually, we hope to be able to remove this
19 * function.
20 */13 */
21static __always_inline bool kcsan_is_atomic(const volatile void *ptr)14static bool kcsan_is_atomic_special(const volatile void *ptr)
22{15{
23 /* only jiffies for now */16 /* volatile globals that have been observed in data races. */
24 return ptr == &jiffies;17 return ptr == &jiffies || ptr == &current->state;
25}18}
26 19
27#endif /* _KERNEL_KCSAN_ATOMIC_H */20#endif /* _KERNEL_KCSAN_ATOMIC_H */
kernel/kcsan/core.c+15−7
@@ -188,12 +188,13 @@ static __always_inline struct kcsan_ctx *get_ctx(void) · 187 unmodified lines
188 return in_task() ? &current->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx);188 return in_task() ? &current->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx);
189}189}
190 190
191/* Rules for generic atomic accesses. Called from fast-path. */
191static __always_inline bool192static __always_inline bool
192is_atomic(const volatile void *ptr, size_t size, int type)193is_atomic(const volatile void *ptr, size_t size, int type)
193{194{
194 struct kcsan_ctx *ctx;195 struct kcsan_ctx *ctx;
195 196
196 if ((type & KCSAN_ACCESS_ATOMIC) != 0)197 if (type & KCSAN_ACCESS_ATOMIC)
197 return true;198 return true;
198 199
199 /*200 /*
@@ -201,16 +202,16 @@ is_atomic(const volatile void *ptr, size_t size, int type) · 1 unmodified line
201 * as atomic. This allows using them also in atomic regions, such as202 * as atomic. This allows using them also in atomic regions, such as
202 * seqlocks, without implicitly changing their semantics.203 * seqlocks, without implicitly changing their semantics.
203 */204 */
204 if ((type & KCSAN_ACCESS_ASSERT) != 0)205 if (type & KCSAN_ACCESS_ASSERT)
205 return false;206 return false;
206 207
207 if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC) &&208 if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC) &&
208 (type & KCSAN_ACCESS_WRITE) != 0 && size <= sizeof(long) &&209 (type & KCSAN_ACCESS_WRITE) && size <= sizeof(long) &&
209 IS_ALIGNED((unsigned long)ptr, size))210 IS_ALIGNED((unsigned long)ptr, size))
210 return true; /* Assume aligned writes up to word size are atomic. */211 return true; /* Assume aligned writes up to word size are atomic. */
211 212
212 ctx = get_ctx();213 ctx = get_ctx();
213 if (unlikely(ctx->atomic_next > 0)) {214 if (ctx->atomic_next > 0) {
214 /*215 /*
215 * Because we do not have separate contexts for nested216 * Because we do not have separate contexts for nested
216 * interrupts, in case atomic_next is set, we simply assume that217 * interrupts, in case atomic_next is set, we simply assume that
@@ -224,10 +225,8 @@ is_atomic(const volatile void *ptr, size_t size, int type) · 7 unmodified lines
224 --ctx->atomic_next; /* in task, or outer interrupt */225 --ctx->atomic_next; /* in task, or outer interrupt */
225 return true;226 return true;
226 }227 }
227 if (unlikely(ctx->atomic_nest_count > 0 || ctx->in_flat_atomic))
228 return true;
229 228
230 return kcsan_is_atomic(ptr);229 return ctx->atomic_nest_count > 0 || ctx->in_flat_atomic;
231}230}
232 231
233static __always_inline bool232static __always_inline bool
@@ -367,6 +366,15 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type) · 133 unmodified lines
367 if (!kcsan_is_enabled())366 if (!kcsan_is_enabled())
368 goto out;367 goto out;
369 368
369 /*
370 * Special atomic rules: unlikely to be true, so we check them here in
371 * the slow-path, and not in the fast-path in is_atomic(). Call after
372 * kcsan_is_enabled(), as we may access memory that is not yet
373 * initialized during early boot.
374 */
375 if (!is_assert && kcsan_is_atomic_special(ptr))
376 goto out;
377
370 if (!check_encodable((unsigned long)ptr, size)) {378 if (!check_encodable((unsigned long)ptr, size)) {
371 kcsan_counter_inc(KCSAN_COUNTER_UNENCODABLE_ACCESSES);379 kcsan_counter_inc(KCSAN_COUNTER_UNENCODABLE_ACCESSES);
372 goto out;380 goto out;
kernel/kcsan/debugfs.c+18−9
@@ -74,25 +74,34 @@ void kcsan_counter_dec(enum kcsan_counter_id id) · 73 unmodified lines
74 */74 */
75static noinline void microbenchmark(unsigned long iters)75static noinline void microbenchmark(unsigned long iters)
76{76{
77 const struct kcsan_ctx ctx_save = current->kcsan_ctx;
78 const bool was_enabled = READ_ONCE(kcsan_enabled);
77 cycles_t cycles;79 cycles_t cycles;
78 80
81 /* We may have been called from an atomic region; reset context. */
82 memset(&current->kcsan_ctx, 0, sizeof(current->kcsan_ctx));
83 /*
84 * Disable to benchmark fast-path for all accesses, and (expected
85 * negligible) call into slow-path, but never set up watchpoints.
86 */
87 WRITE_ONCE(kcsan_enabled, false);
88
79 pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters);89 pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters);
80 90
81 cycles = get_cycles();91 cycles = get_cycles();
82 while (iters--) {92 while (iters--) {
83 /*93 unsigned long addr = iters & ((PAGE_SIZE << 8) - 1);
84 * We can run this benchmark from multiple tasks; this address94 int type = !(iters & 0x7f) ? KCSAN_ACCESS_ATOMIC :
85 * calculation increases likelyhood of some accesses95 (!(iters & 0xf) ? KCSAN_ACCESS_WRITE : 0);
86 * overlapping. Make the access type an atomic read, to never96 __kcsan_check_access((void *)addr, sizeof(long), type);
87 * set up watchpoints and test the fast-path only.
88 */
89 unsigned long addr =
90 iters % (CONFIG_KCSAN_NUM_WATCHPOINTS * PAGE_SIZE);
91 __kcsan_check_access((void *)addr, sizeof(long), KCSAN_ACCESS_ATOMIC);
92 }97 }
93 cycles = get_cycles() - cycles;98 cycles = get_cycles() - cycles;
94 99
95 pr_info("KCSAN: %s end | cycles: %llu\n", __func__, cycles);100 pr_info("KCSAN: %s end | cycles: %llu\n", __func__, cycles);
101
102 WRITE_ONCE(kcsan_enabled, was_enabled);
103 /* restore context */
104 current->kcsan_ctx = ctx_save;
96}105}
97 106
98/*107/*

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t