Add volatile current->state to list of implicitly atomic accesses. This is in preparation to eventually enable KCSAN on kernel/sched (which currently still has KCSAN_SANITIZE := n). Since accesses that match the special check in atomic.h are rare, it makes more sense to move this check to the slow-path, avoiding the additional compare in the fast-path. With the microbenchmark, a speedup of ~6% is measured.
Unified · Split
| @@ -4,24 +4,17 @@ · 3 unmodified lines | |||
| 4 | #define _KERNEL_KCSAN_ATOMIC_H | 4 | #define _KERNEL_KCSAN_ATOMIC_H |
| 5 | 5 | ||
| 6 | #include <linux/jiffies.h> | 6 | #include <linux/jiffies.h> |
| 7 | #include <linux/sched.h> | ||
| 7 | 8 | ||
| 8 | /* | 9 | /* |
| 9 | * Helper that returns true if access to @ptr should be considered an atomic | 10 | * Special rules for certain memory where concurrent conflicting accesses are |
| 10 | * access, even though it is not explicitly atomic. | 11 | * common, however, the current convention is to not mark them; returns true if |
| 11 | * | 12 | * access to @ptr should be considered atomic. Called from slow-path. |
| 12 | * List all volatile globals that have been observed in races, to suppress | ||
| 13 | * data race reports between accesses to these variables. | ||
| 14 | * | ||
| 15 | * For now, we assume that volatile accesses of globals are as strong as atomic | ||
| 16 | * accesses (READ_ONCE, WRITE_ONCE cast to volatile). The situation is still not | ||
| 17 | * entirely clear, as on some architectures (Alpha) READ_ONCE/WRITE_ONCE do more | ||
| 18 | * than cast to volatile. Eventually, we hope to be able to remove this | ||
| 19 | * function. | ||
| 20 | */ | 13 | */ |
| 21 | static __always_inline bool kcsan_is_atomic(const volatile void *ptr) | 14 | static bool kcsan_is_atomic_special(const volatile void *ptr) |
| 22 | { | 15 | { |
| 23 | /* only jiffies for now */ | 16 | /* volatile globals that have been observed in data races. */ |
| 24 | return ptr == &jiffies; | 17 | return ptr == &jiffies || ptr == ¤t->state; |
| 25 | } | 18 | } |
| 26 | 19 | ||
| 27 | #endif /* _KERNEL_KCSAN_ATOMIC_H */ | 20 | #endif /* _KERNEL_KCSAN_ATOMIC_H */ |
| @@ -188,12 +188,13 @@ static __always_inline struct kcsan_ctx *get_ctx(void) · 187 unmodified lines | |||
| 188 | return in_task() ? ¤t->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx); | 188 | return in_task() ? ¤t->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx); |
| 189 | } | 189 | } |
| 190 | 190 | ||
| 191 | /* Rules for generic atomic accesses. Called from fast-path. */ | ||
| 191 | static __always_inline bool | 192 | static __always_inline bool |
| 192 | is_atomic(const volatile void *ptr, size_t size, int type) | 193 | is_atomic(const volatile void *ptr, size_t size, int type) |
| 193 | { | 194 | { |
| 194 | struct kcsan_ctx *ctx; | 195 | struct kcsan_ctx *ctx; |
| 195 | 196 | ||
| 196 | if ((type & KCSAN_ACCESS_ATOMIC) != 0) | 197 | if (type & KCSAN_ACCESS_ATOMIC) |
| 197 | return true; | 198 | return true; |
| 198 | 199 | ||
| 199 | /* | 200 | /* |
| @@ -201,16 +202,16 @@ is_atomic(const volatile void *ptr, size_t size, int type) · 1 unmodified line | |||
| 201 | * as atomic. This allows using them also in atomic regions, such as | 202 | * as atomic. This allows using them also in atomic regions, such as |
| 202 | * seqlocks, without implicitly changing their semantics. | 203 | * seqlocks, without implicitly changing their semantics. |
| 203 | */ | 204 | */ |
| 204 | if ((type & KCSAN_ACCESS_ASSERT) != 0) | 205 | if (type & KCSAN_ACCESS_ASSERT) |
| 205 | return false; | 206 | return false; |
| 206 | 207 | ||
| 207 | if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC) && | 208 | if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC) && |
| 208 | (type & KCSAN_ACCESS_WRITE) != 0 && size <= sizeof(long) && | 209 | (type & KCSAN_ACCESS_WRITE) && size <= sizeof(long) && |
| 209 | IS_ALIGNED((unsigned long)ptr, size)) | 210 | IS_ALIGNED((unsigned long)ptr, size)) |
| 210 | return true; /* Assume aligned writes up to word size are atomic. */ | 211 | return true; /* Assume aligned writes up to word size are atomic. */ |
| 211 | 212 | ||
| 212 | ctx = get_ctx(); | 213 | ctx = get_ctx(); |
| 213 | if (unlikely(ctx->atomic_next > 0)) { | 214 | if (ctx->atomic_next > 0) { |
| 214 | /* | 215 | /* |
| 215 | * Because we do not have separate contexts for nested | 216 | * Because we do not have separate contexts for nested |
| 216 | * interrupts, in case atomic_next is set, we simply assume that | 217 | * interrupts, in case atomic_next is set, we simply assume that |
| @@ -224,10 +225,8 @@ is_atomic(const volatile void *ptr, size_t size, int type) · 7 unmodified lines | |||
| 224 | --ctx->atomic_next; /* in task, or outer interrupt */ | 225 | --ctx->atomic_next; /* in task, or outer interrupt */ |
| 225 | return true; | 226 | return true; |
| 226 | } | 227 | } |
| 227 | if (unlikely(ctx->atomic_nest_count > 0 || ctx->in_flat_atomic)) | ||
| 228 | return true; | ||
| 229 | 228 | ||
| 230 | return kcsan_is_atomic(ptr); | 229 | return ctx->atomic_nest_count > 0 || ctx->in_flat_atomic; |
| 231 | } | 230 | } |
| 232 | 231 | ||
| 233 | static __always_inline bool | 232 | static __always_inline bool |
| @@ -367,6 +366,15 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type) · 133 unmodified lines | |||
| 367 | if (!kcsan_is_enabled()) | 366 | if (!kcsan_is_enabled()) |
| 368 | goto out; | 367 | goto out; |
| 369 | 368 | ||
| 369 | /* | ||
| 370 | * Special atomic rules: unlikely to be true, so we check them here in | ||
| 371 | * the slow-path, and not in the fast-path in is_atomic(). Call after | ||
| 372 | * kcsan_is_enabled(), as we may access memory that is not yet | ||
| 373 | * initialized during early boot. | ||
| 374 | */ | ||
| 375 | if (!is_assert && kcsan_is_atomic_special(ptr)) | ||
| 376 | goto out; | ||
| 377 | |||
| 370 | if (!check_encodable((unsigned long)ptr, size)) { | 378 | if (!check_encodable((unsigned long)ptr, size)) { |
| 371 | kcsan_counter_inc(KCSAN_COUNTER_UNENCODABLE_ACCESSES); | 379 | kcsan_counter_inc(KCSAN_COUNTER_UNENCODABLE_ACCESSES); |
| 372 | goto out; | 380 | goto out; |
| @@ -74,25 +74,34 @@ void kcsan_counter_dec(enum kcsan_counter_id id) · 73 unmodified lines | |||
| 74 | */ | 74 | */ |
| 75 | static noinline void microbenchmark(unsigned long iters) | 75 | static noinline void microbenchmark(unsigned long iters) |
| 76 | { | 76 | { |
| 77 | const struct kcsan_ctx ctx_save = current->kcsan_ctx; | ||
| 78 | const bool was_enabled = READ_ONCE(kcsan_enabled); | ||
| 77 | cycles_t cycles; | 79 | cycles_t cycles; |
| 78 | 80 | ||
| 81 | /* We may have been called from an atomic region; reset context. */ | ||
| 82 | memset(¤t->kcsan_ctx, 0, sizeof(current->kcsan_ctx)); | ||
| 83 | /* | ||
| 84 | * Disable to benchmark fast-path for all accesses, and (expected | ||
| 85 | * negligible) call into slow-path, but never set up watchpoints. | ||
| 86 | */ | ||
| 87 | WRITE_ONCE(kcsan_enabled, false); | ||
| 88 | |||
| 79 | pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters); | 89 | pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters); |
| 80 | 90 | ||
| 81 | cycles = get_cycles(); | 91 | cycles = get_cycles(); |
| 82 | while (iters--) { | 92 | while (iters--) { |
| 83 | /* | 93 | unsigned long addr = iters & ((PAGE_SIZE << 8) - 1); |
| 84 | * We can run this benchmark from multiple tasks; this address | 94 | int type = !(iters & 0x7f) ? KCSAN_ACCESS_ATOMIC : |
| 85 | * calculation increases likelyhood of some accesses | 95 | (!(iters & 0xf) ? KCSAN_ACCESS_WRITE : 0); |
| 86 | * overlapping. Make the access type an atomic read, to never | 96 | __kcsan_check_access((void *)addr, sizeof(long), type); |
| 87 | * set up watchpoints and test the fast-path only. | ||
| 88 | */ | ||
| 89 | unsigned long addr = | ||
| 90 | iters % (CONFIG_KCSAN_NUM_WATCHPOINTS * PAGE_SIZE); | ||
| 91 | __kcsan_check_access((void *)addr, sizeof(long), KCSAN_ACCESS_ATOMIC); | ||
| 92 | } | 97 | } |
| 93 | cycles = get_cycles() - cycles; | 98 | cycles = get_cycles() - cycles; |
| 94 | 99 | ||
| 95 | pr_info("KCSAN: %s end | cycles: %llu\n", __func__, cycles); | 100 | pr_info("KCSAN: %s end | cycles: %llu\n", __func__, cycles); |
| 101 | |||
| 102 | WRITE_ONCE(kcsan_enabled, was_enabled); | ||
| 103 | /* restore context */ | ||
| 104 | current->kcsan_ctx = ctx_save; | ||
| 96 | } | 105 | } |
| 97 | 106 | ||
| 98 | /* | 107 | /* |