Add volatile current->state to list of implicitly atomic accesses. This is in preparation to eventually enable KCSAN on kernel/sched (which currently still has KCSAN_SANITIZE := n). Since accesses that match the special check in atomic.h are rare, it makes more sense to move this check to the slow-path, avoiding the additional compare in the fast-path. With the microbenchmark, a speedup of ~6% is measured.
Showing the changes to kernel (3 of 3 files). Show all
Unified · Split
| @@ -4,24 +4,17 @@ · 3 unmodified lines | ||
| 4 | 4 | #define _KERNEL_KCSAN_ATOMIC_H |
| 5 | 5 | |
| 6 | 6 | #include <linux/jiffies.h> |
| 7 | +#include <linux/sched.h> | |
| 7 | 8 | |
| 8 | 9 | /* |
| 9 | − * Helper that returns true if access to @ptr should be considered an atomic | |
| 10 | − * access, even though it is not explicitly atomic. | |
| 11 | − * | |
| 12 | − * List all volatile globals that have been observed in races, to suppress | |
| 13 | − * data race reports between accesses to these variables. | |
| 14 | − * | |
| 15 | − * For now, we assume that volatile accesses of globals are as strong as atomic | |
| 16 | − * accesses (READ_ONCE, WRITE_ONCE cast to volatile). The situation is still not | |
| 17 | − * entirely clear, as on some architectures (Alpha) READ_ONCE/WRITE_ONCE do more | |
| 18 | − * than cast to volatile. Eventually, we hope to be able to remove this | |
| 19 | − * function. | |
| 10 | + * Special rules for certain memory where concurrent conflicting accesses are | |
| 11 | + * common, however, the current convention is to not mark them; returns true if | |
| 12 | + * access to @ptr should be considered atomic. Called from slow-path. | |
| 20 | 13 | */ |
| 21 | −static __always_inline bool kcsan_is_atomic(const volatile void *ptr) | |
| 14 | +static bool kcsan_is_atomic_special(const volatile void *ptr) | |
| 22 | 15 | { |
| 23 | − /* only jiffies for now */ | |
| 24 | − return ptr == &jiffies; | |
| 16 | + /* volatile globals that have been observed in data races. */ | |
| 17 | + return ptr == &jiffies || ptr == ¤t->state; | |
| 25 | 18 | } |
| 26 | 19 | |
| 27 | 20 | #endif /* _KERNEL_KCSAN_ATOMIC_H */ |
| @@ -188,12 +188,13 @@ static __always_inline struct kcsan_ctx *get_ctx(void) · 187 unmodified lines | ||
| 188 | 188 | return in_task() ? ¤t->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx); |
| 189 | 189 | } |
| 190 | 190 | |
| 191 | +/* Rules for generic atomic accesses. Called from fast-path. */ | |
| 191 | 192 | static __always_inline bool |
| 192 | 193 | is_atomic(const volatile void *ptr, size_t size, int type) |
| 193 | 194 | { |
| 194 | 195 | struct kcsan_ctx *ctx; |
| 195 | 196 | |
| 196 | − if ((type & KCSAN_ACCESS_ATOMIC) != 0) | |
| 197 | + if (type & KCSAN_ACCESS_ATOMIC) | |
| 197 | 198 | return true; |
| 198 | 199 | |
| 199 | 200 | /* |
| @@ -201,16 +202,16 @@ is_atomic(const volatile void *ptr, size_t size, int type) · 1 unmodified line | ||
| 201 | 202 | * as atomic. This allows using them also in atomic regions, such as |
| 202 | 203 | * seqlocks, without implicitly changing their semantics. |
| 203 | 204 | */ |
| 204 | − if ((type & KCSAN_ACCESS_ASSERT) != 0) | |
| 205 | + if (type & KCSAN_ACCESS_ASSERT) | |
| 205 | 206 | return false; |
| 206 | 207 | |
| 207 | 208 | if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC) && |
| 208 | − (type & KCSAN_ACCESS_WRITE) != 0 && size <= sizeof(long) && | |
| 209 | + (type & KCSAN_ACCESS_WRITE) && size <= sizeof(long) && | |
| 209 | 210 | IS_ALIGNED((unsigned long)ptr, size)) |
| 210 | 211 | return true; /* Assume aligned writes up to word size are atomic. */ |
| 211 | 212 | |
| 212 | 213 | ctx = get_ctx(); |
| 213 | − if (unlikely(ctx->atomic_next > 0)) { | |
| 214 | + if (ctx->atomic_next > 0) { | |
| 214 | 215 | /* |
| 215 | 216 | * Because we do not have separate contexts for nested |
| 216 | 217 | * interrupts, in case atomic_next is set, we simply assume that |
| @@ -224,10 +225,8 @@ is_atomic(const volatile void *ptr, size_t size, int type) · 7 unmodified lines | ||
| 224 | 225 | --ctx->atomic_next; /* in task, or outer interrupt */ |
| 225 | 226 | return true; |
| 226 | 227 | } |
| 227 | − if (unlikely(ctx->atomic_nest_count > 0 || ctx->in_flat_atomic)) | |
| 228 | − return true; | |
| 229 | 228 | |
| 230 | − return kcsan_is_atomic(ptr); | |
| 229 | + return ctx->atomic_nest_count > 0 || ctx->in_flat_atomic; | |
| 231 | 230 | } |
| 232 | 231 | |
| 233 | 232 | static __always_inline bool |
| @@ -367,6 +366,15 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type) · 133 unmodified lines | ||
| 367 | 366 | if (!kcsan_is_enabled()) |
| 368 | 367 | goto out; |
| 369 | 368 | |
| 369 | + /* | |
| 370 | + * Special atomic rules: unlikely to be true, so we check them here in | |
| 371 | + * the slow-path, and not in the fast-path in is_atomic(). Call after | |
| 372 | + * kcsan_is_enabled(), as we may access memory that is not yet | |
| 373 | + * initialized during early boot. | |
| 374 | + */ | |
| 375 | + if (!is_assert && kcsan_is_atomic_special(ptr)) | |
| 376 | + goto out; | |
| 377 | + | |
| 370 | 378 | if (!check_encodable((unsigned long)ptr, size)) { |
| 371 | 379 | kcsan_counter_inc(KCSAN_COUNTER_UNENCODABLE_ACCESSES); |
| 372 | 380 | goto out; |
| @@ -74,25 +74,34 @@ void kcsan_counter_dec(enum kcsan_counter_id id) · 73 unmodified lines | ||
| 74 | 74 | */ |
| 75 | 75 | static noinline void microbenchmark(unsigned long iters) |
| 76 | 76 | { |
| 77 | + const struct kcsan_ctx ctx_save = current->kcsan_ctx; | |
| 78 | + const bool was_enabled = READ_ONCE(kcsan_enabled); | |
| 77 | 79 | cycles_t cycles; |
| 78 | 80 | |
| 81 | + /* We may have been called from an atomic region; reset context. */ | |
| 82 | + memset(¤t->kcsan_ctx, 0, sizeof(current->kcsan_ctx)); | |
| 83 | + /* | |
| 84 | + * Disable to benchmark fast-path for all accesses, and (expected | |
| 85 | + * negligible) call into slow-path, but never set up watchpoints. | |
| 86 | + */ | |
| 87 | + WRITE_ONCE(kcsan_enabled, false); | |
| 88 | + | |
| 79 | 89 | pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters); |
| 80 | 90 | |
| 81 | 91 | cycles = get_cycles(); |
| 82 | 92 | while (iters--) { |
| 83 | − /* | |
| 84 | − * We can run this benchmark from multiple tasks; this address | |
| 85 | − * calculation increases likelyhood of some accesses | |
| 86 | − * overlapping. Make the access type an atomic read, to never | |
| 87 | − * set up watchpoints and test the fast-path only. | |
| 88 | − */ | |
| 89 | − unsigned long addr = | |
| 90 | − iters % (CONFIG_KCSAN_NUM_WATCHPOINTS * PAGE_SIZE); | |
| 91 | − __kcsan_check_access((void *)addr, sizeof(long), KCSAN_ACCESS_ATOMIC); | |
| 93 | + unsigned long addr = iters & ((PAGE_SIZE << 8) - 1); | |
| 94 | + int type = !(iters & 0x7f) ? KCSAN_ACCESS_ATOMIC : | |
| 95 | + (!(iters & 0xf) ? KCSAN_ACCESS_WRITE : 0); | |
| 96 | + __kcsan_check_access((void *)addr, sizeof(long), type); | |
| 92 | 97 | } |
| 93 | 98 | cycles = get_cycles() - cycles; |
| 94 | 99 | |
| 95 | 100 | pr_info("KCSAN: %s end | cycles: %llu\n", __func__, cycles); |
| 101 | + | |
| 102 | + WRITE_ONCE(kcsan_enabled, was_enabled); | |
| 103 | + /* restore context */ | |
| 104 | + current->kcsan_ctx = ctx_save; | |
| 96 | 105 | } |
| 97 | 106 | |
| 98 | 107 | /* |