Add support for modeling a subset of weak memory, which will enable detection of a subset of data races due to missing memory barriers. KCSAN's approach to detecting missing memory barriers is based on modeling access reordering, and enabled if `CONFIG_KCSAN_WEAK_MEMORY=y`, which depends on `CONFIG_KCSAN_STRICT=y`. The feature can be enabled or disabled at boot and runtime via the `kcsan.weak_memory` boot parameter. Each memory access for which a watchpoint is set up, is also selected for simulated reordering within the scope of its function (at most 1 in-flight access). We are limited to modeling the effects of "buffering" (delaying the access), since the runtime cannot "prefetch" accesses (therefore no acquire modeling). Once an access has been selected for reordering, it is checked along every other access until the end of the function scope. If an appropriate memory barrier is encountered, the access will no longer be considered for reordering. When the result of a memory operation should be ordered by a barrier, KCSAN can then detect data races where the conflict only occurs as a result of a missing barrier due to reordering accesses.
Showing the changes to kernel (1 of 6 files). Show all
Unified · Split
| @@ -40,6 +40,13 @@ module_param_named(udelay_interrupt, kcsan_udelay_interrupt, uint, 0644); · 39 unmodified lines | |||
| 40 | module_param_named(skip_watch, kcsan_skip_watch, long, 0644); | 40 | module_param_named(skip_watch, kcsan_skip_watch, long, 0644); |
| 41 | module_param_named(interrupt_watcher, kcsan_interrupt_watcher, bool, 0444); | 41 | module_param_named(interrupt_watcher, kcsan_interrupt_watcher, bool, 0444); |
| 42 | 42 | ||
| 43 | #ifdef CONFIG_KCSAN_WEAK_MEMORY | ||
| 44 | static bool kcsan_weak_memory = true; | ||
| 45 | module_param_named(weak_memory, kcsan_weak_memory, bool, 0644); | ||
| 46 | #else | ||
| 47 | #define kcsan_weak_memory false | ||
| 48 | #endif | ||
| 49 | |||
| 43 | bool kcsan_enabled; | 50 | bool kcsan_enabled; |
| 44 | 51 | ||
| 45 | /* Per-CPU kcsan_ctx for interrupts */ | 52 | /* Per-CPU kcsan_ctx for interrupts */ |
| @@ -351,6 +358,67 @@ void kcsan_restore_irqtrace(struct task_struct *task) · 305 unmodified lines | |||
| 351 | #endif | 358 | #endif |
| 352 | } | 359 | } |
| 353 | 360 | ||
| 361 | static __always_inline int get_kcsan_stack_depth(void) | ||
| 362 | { | ||
| 363 | #ifdef CONFIG_KCSAN_WEAK_MEMORY | ||
| 364 | return current->kcsan_stack_depth; | ||
| 365 | #else | ||
| 366 | BUILD_BUG(); | ||
| 367 | return 0; | ||
| 368 | #endif | ||
| 369 | } | ||
| 370 | |||
| 371 | static __always_inline void add_kcsan_stack_depth(int val) | ||
| 372 | { | ||
| 373 | #ifdef CONFIG_KCSAN_WEAK_MEMORY | ||
| 374 | current->kcsan_stack_depth += val; | ||
| 375 | #else | ||
| 376 | BUILD_BUG(); | ||
| 377 | #endif | ||
| 378 | } | ||
| 379 | |||
| 380 | static __always_inline struct kcsan_scoped_access *get_reorder_access(struct kcsan_ctx *ctx) | ||
| 381 | { | ||
| 382 | #ifdef CONFIG_KCSAN_WEAK_MEMORY | ||
| 383 | return ctx->disable_scoped ? NULL : &ctx->reorder_access; | ||
| 384 | #else | ||
| 385 | return NULL; | ||
| 386 | #endif | ||
| 387 | } | ||
| 388 | |||
| 389 | static __always_inline bool | ||
| 390 | find_reorder_access(struct kcsan_ctx *ctx, const volatile void *ptr, size_t size, | ||
| 391 | int type, unsigned long ip) | ||
| 392 | { | ||
| 393 | struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx); | ||
| 394 | |||
| 395 | if (!reorder_access) | ||
| 396 | return false; | ||
| 397 | |||
| 398 | /* | ||
| 399 | * Note: If accesses are repeated while reorder_access is identical, | ||
| 400 | * never matches the new access, because !(type & KCSAN_ACCESS_SCOPED). | ||
| 401 | */ | ||
| 402 | return reorder_access->ptr == ptr && reorder_access->size == size && | ||
| 403 | reorder_access->type == type && reorder_access->ip == ip; | ||
| 404 | } | ||
| 405 | |||
| 406 | static inline void | ||
| 407 | set_reorder_access(struct kcsan_ctx *ctx, const volatile void *ptr, size_t size, | ||
| 408 | int type, unsigned long ip) | ||
| 409 | { | ||
| 410 | struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx); | ||
| 411 | |||
| 412 | if (!reorder_access || !kcsan_weak_memory) | ||
| 413 | return; | ||
| 414 | |||
| 415 | reorder_access->ptr = ptr; | ||
| 416 | reorder_access->size = size; | ||
| 417 | reorder_access->type = type | KCSAN_ACCESS_SCOPED; | ||
| 418 | reorder_access->ip = ip; | ||
| 419 | reorder_access->stack_depth = get_kcsan_stack_depth(); | ||
| 420 | } | ||
| 421 | |||
| 354 | /* | 422 | /* |
| 355 | * Pull everything together: check_access() below contains the performance | 423 | * Pull everything together: check_access() below contains the performance |
| 356 | * critical operations; the fast-path (including check_access) functions should | 424 | * critical operations; the fast-path (including check_access) functions should |
| @@ -389,8 +457,10 @@ static noinline void kcsan_found_watchpoint(const volatile void *ptr, · 32 unmodified lines | |||
| 389 | * The access_mask check relies on value-change comparison. To avoid | 457 | * The access_mask check relies on value-change comparison. To avoid |
| 390 | * reporting a race where e.g. the writer set up the watchpoint, but the | 458 | * reporting a race where e.g. the writer set up the watchpoint, but the |
| 391 | * reader has access_mask!=0, we have to ignore the found watchpoint. | 459 | * reader has access_mask!=0, we have to ignore the found watchpoint. |
| 460 | * | ||
| 461 | * reorder_access is never created from an access with access_mask set. | ||
| 392 | */ | 462 | */ |
| 393 | if (ctx->access_mask) | 463 | if (ctx->access_mask && !find_reorder_access(ctx, ptr, size, type, ip)) |
| 394 | return; | 464 | return; |
| 395 | 465 | ||
| 396 | /* | 466 | /* |
| @@ -440,11 +510,13 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 43 unmodified lines | |||
| 440 | const bool is_assert = (type & KCSAN_ACCESS_ASSERT) != 0; | 510 | const bool is_assert = (type & KCSAN_ACCESS_ASSERT) != 0; |
| 441 | atomic_long_t *watchpoint; | 511 | atomic_long_t *watchpoint; |
| 442 | u64 old, new, diff; | 512 | u64 old, new, diff; |
| 443 | unsigned long access_mask; | ||
| 444 | enum kcsan_value_change value_change = KCSAN_VALUE_CHANGE_MAYBE; | 513 | enum kcsan_value_change value_change = KCSAN_VALUE_CHANGE_MAYBE; |
| 514 | bool interrupt_watcher = kcsan_interrupt_watcher; | ||
| 445 | unsigned long ua_flags = user_access_save(); | 515 | unsigned long ua_flags = user_access_save(); |
| 446 | struct kcsan_ctx *ctx = get_ctx(); | 516 | struct kcsan_ctx *ctx = get_ctx(); |
| 517 | unsigned long access_mask = ctx->access_mask; | ||
| 447 | unsigned long irq_flags = 0; | 518 | unsigned long irq_flags = 0; |
| 519 | bool is_reorder_access; | ||
| 448 | 520 | ||
| 449 | /* | 521 | /* |
| 450 | * Always reset kcsan_skip counter in slow-path to avoid underflow; see | 522 | * Always reset kcsan_skip counter in slow-path to avoid underflow; see |
| @@ -467,6 +539,17 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 16 unmodified lines | |||
| 467 | goto out; | 539 | goto out; |
| 468 | } | 540 | } |
| 469 | 541 | ||
| 542 | /* | ||
| 543 | * The local CPU cannot observe reordering of its own accesses, and | ||
| 544 | * therefore we need to take care of 2 cases to avoid false positives: | ||
| 545 | * | ||
| 546 | * 1. Races of the reordered access with interrupts. To avoid, if | ||
| 547 | * the current access is reorder_access, disable interrupts. | ||
| 548 | * 2. Avoid races of scoped accesses from nested interrupts (below). | ||
| 549 | */ | ||
| 550 | is_reorder_access = find_reorder_access(ctx, ptr, size, type, ip); | ||
| 551 | if (is_reorder_access) | ||
| 552 | interrupt_watcher = false; | ||
| 470 | /* | 553 | /* |
| 471 | * Avoid races of scoped accesses from nested interrupts (or scheduler). | 554 | * Avoid races of scoped accesses from nested interrupts (or scheduler). |
| 472 | * Assume setting up a watchpoint for a non-scoped (normal) access that | 555 | * Assume setting up a watchpoint for a non-scoped (normal) access that |
| @@ -482,7 +565,7 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 9 unmodified lines | |||
| 482 | * information is lost if dirtied by KCSAN. | 565 | * information is lost if dirtied by KCSAN. |
| 483 | */ | 566 | */ |
| 484 | kcsan_save_irqtrace(current); | 567 | kcsan_save_irqtrace(current); |
| 485 | if (!kcsan_interrupt_watcher) | 568 | if (!interrupt_watcher) |
| 486 | local_irq_save(irq_flags); | 569 | local_irq_save(irq_flags); |
| 487 | 570 | ||
| 488 | watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write); | 571 | watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write); |
| @@ -503,7 +586,7 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 14 unmodified lines | |||
| 503 | * Read the current value, to later check and infer a race if the data | 586 | * Read the current value, to later check and infer a race if the data |
| 504 | * was modified via a non-instrumented access, e.g. from a device. | 587 | * was modified via a non-instrumented access, e.g. from a device. |
| 505 | */ | 588 | */ |
| 506 | old = read_instrumented_memory(ptr, size); | 589 | old = is_reorder_access ? 0 : read_instrumented_memory(ptr, size); |
| 507 | 590 | ||
| 508 | /* | 591 | /* |
| 509 | * Delay this thread, to increase probability of observing a racy | 592 | * Delay this thread, to increase probability of observing a racy |
| @@ -515,8 +598,17 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 5 unmodified lines | |||
| 515 | * Re-read value, and check if it is as expected; if not, we infer a | 598 | * Re-read value, and check if it is as expected; if not, we infer a |
| 516 | * racy access. | 599 | * racy access. |
| 517 | */ | 600 | */ |
| 518 | access_mask = ctx->access_mask; | 601 | if (!is_reorder_access) { |
| 519 | new = read_instrumented_memory(ptr, size); | 602 | new = read_instrumented_memory(ptr, size); |
| 603 | } else { | ||
| 604 | /* | ||
| 605 | * Reordered accesses cannot be used for value change detection, | ||
| 606 | * because the memory location may no longer be accessible and | ||
| 607 | * could result in a fault. | ||
| 608 | */ | ||
| 609 | new = 0; | ||
| 610 | access_mask = 0; | ||
| 611 | } | ||
| 520 | 612 | ||
| 521 | diff = old ^ new; | 613 | diff = old ^ new; |
| 522 | if (access_mask) | 614 | if (access_mask) |
| @@ -585,11 +677,20 @@ kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type, unsigned · 62 unmodified lines | |||
| 585 | */ | 677 | */ |
| 586 | remove_watchpoint(watchpoint); | 678 | remove_watchpoint(watchpoint); |
| 587 | atomic_long_dec(&kcsan_counters[KCSAN_COUNTER_USED_WATCHPOINTS]); | 679 | atomic_long_dec(&kcsan_counters[KCSAN_COUNTER_USED_WATCHPOINTS]); |
| 680 | |||
| 588 | out_unlock: | 681 | out_unlock: |
| 589 | if (!kcsan_interrupt_watcher) | 682 | if (!interrupt_watcher) |
| 590 | local_irq_restore(irq_flags); | 683 | local_irq_restore(irq_flags); |
| 591 | kcsan_restore_irqtrace(current); | 684 | kcsan_restore_irqtrace(current); |
| 592 | ctx->disable_scoped--; | 685 | ctx->disable_scoped--; |
| 686 | |||
| 687 | /* | ||
| 688 | * Reordered accesses cannot be used for value change detection, | ||
| 689 | * therefore never consider for reordering if access_mask is set. | ||
| 690 | * ASSERT_EXCLUSIVE are not real accesses, ignore them as well. | ||
| 691 | */ | ||
| 692 | if (!access_mask && !is_assert) | ||
| 693 | set_reorder_access(ctx, ptr, size, type, ip); | ||
| 593 | out: | 694 | out: |
| 594 | user_access_restore(ua_flags); | 695 | user_access_restore(ua_flags); |
| 595 | } | 696 | } |
| @@ -597,7 +698,6 @@ out: · 1 unmodified line | |||
| 597 | static __always_inline void | 698 | static __always_inline void |
| 598 | check_access(const volatile void *ptr, size_t size, int type, unsigned long ip) | 699 | check_access(const volatile void *ptr, size_t size, int type, unsigned long ip) |
| 599 | { | 700 | { |
| 600 | const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0; | ||
| 601 | atomic_long_t *watchpoint; | 701 | atomic_long_t *watchpoint; |
| 602 | long encoded_watchpoint; | 702 | long encoded_watchpoint; |
| 603 | 703 | ||
| @@ -608,12 +708,14 @@ check_access(const volatile void *ptr, size_t size, int type, unsigned long ip) · 4 unmodified lines | |||
| 608 | if (unlikely(size == 0)) | 708 | if (unlikely(size == 0)) |
| 609 | return; | 709 | return; |
| 610 | 710 | ||
| 711 | again: | ||
| 611 | /* | 712 | /* |
| 612 | * Avoid user_access_save in fast-path: find_watchpoint is safe without | 713 | * Avoid user_access_save in fast-path: find_watchpoint is safe without |
| 613 | * user_access_save, as the address that ptr points to is only used to | 714 | * user_access_save, as the address that ptr points to is only used to |
| 614 | * check if a watchpoint exists; ptr is never dereferenced. | 715 | * check if a watchpoint exists; ptr is never dereferenced. |
| 615 | */ | 716 | */ |
| 616 | watchpoint = find_watchpoint((unsigned long)ptr, size, !is_write, | 717 | watchpoint = find_watchpoint((unsigned long)ptr, size, |
| 718 | !(type & KCSAN_ACCESS_WRITE), | ||
| 617 | &encoded_watchpoint); | 719 | &encoded_watchpoint); |
| 618 | /* | 720 | /* |
| 619 | * It is safe to check kcsan_is_enabled() after find_watchpoint in the | 721 | * It is safe to check kcsan_is_enabled() after find_watchpoint in the |
| @@ -627,9 +729,42 @@ check_access(const volatile void *ptr, size_t size, int type, unsigned long ip) · 7 unmodified lines | |||
| 627 | else { | 729 | else { |
| 628 | struct kcsan_ctx *ctx = get_ctx(); /* Call only once in fast-path. */ | 730 | struct kcsan_ctx *ctx = get_ctx(); /* Call only once in fast-path. */ |
| 629 | 731 | ||
| 630 | if (unlikely(should_watch(ctx, ptr, size, type))) | 732 | if (unlikely(should_watch(ctx, ptr, size, type))) { |
| 631 | kcsan_setup_watchpoint(ptr, size, type, ip); | 733 | kcsan_setup_watchpoint(ptr, size, type, ip); |
| 632 | else if (unlikely(ctx->scoped_accesses.prev)) | 734 | return; |
| 735 | } | ||
| 736 | |||
| 737 | if (!(type & KCSAN_ACCESS_SCOPED)) { | ||
| 738 | struct kcsan_scoped_access *reorder_access = get_reorder_access(ctx); | ||
| 739 | |||
| 740 | if (reorder_access) { | ||
| 741 | /* | ||
| 742 | * reorder_access check: simulates reordering of | ||
| 743 | * the access after subsequent operations. | ||
| 744 | */ | ||
| 745 | ptr = reorder_access->ptr; | ||
| 746 | type = reorder_access->type; | ||
| 747 | ip = reorder_access->ip; | ||
| 748 | /* | ||
| 749 | * Upon a nested interrupt, this context's | ||
| 750 | * reorder_access can be modified (shared ctx). | ||
| 751 | * We know that upon return, reorder_access is | ||
| 752 | * always invalidated by setting size to 0 via | ||
| 753 | * __tsan_func_exit(). Therefore we must read | ||
| 754 | * and check size after the other fields. | ||
| 755 | */ | ||
| 756 | barrier(); | ||
| 757 | size = READ_ONCE(reorder_access->size); | ||
| 758 | if (size) | ||
| 759 | goto again; | ||
| 760 | } | ||
| 761 | } | ||
| 762 | |||
| 763 | /* | ||
| 764 | * Always checked last, right before returning from runtime; | ||
| 765 | * if reorder_access is valid, checked after it was checked. | ||
| 766 | */ | ||
| 767 | if (unlikely(ctx->scoped_accesses.prev)) | ||
| 633 | kcsan_check_scoped_accesses(); | 768 | kcsan_check_scoped_accesses(); |
| 634 | } | 769 | } |
| 635 | } | 770 | } |
| @@ -916,19 +1051,56 @@ DEFINE_TSAN_VOLATILE_READ_WRITE(8); · 280 unmodified lines | |||
| 916 | DEFINE_TSAN_VOLATILE_READ_WRITE(16); | 1051 | DEFINE_TSAN_VOLATILE_READ_WRITE(16); |
| 917 | 1052 | ||
| 918 | /* | 1053 | /* |
| 919 | * The below are not required by KCSAN, but can still be emitted by the | 1054 | * Function entry and exit are used to determine the validty of reorder_access. |
| 920 | * compiler. | 1055 | * Reordering of the access ends at the end of the function scope where the |
| 1056 | * access happened. This is done for two reasons: | ||
| 1057 | * | ||
| 1058 | * 1. Artificially limits the scope where missing barriers are detected. | ||
| 1059 | * This minimizes false positives due to uninstrumented functions that | ||
| 1060 | * contain the required barriers but were missed. | ||
| 1061 | * | ||
| 1062 | * 2. Simplifies generating the stack trace of the access. | ||
| 921 | */ | 1063 | */ |
| 922 | void __tsan_func_entry(void *call_pc); | 1064 | void __tsan_func_entry(void *call_pc); |
| 923 | void __tsan_func_entry(void *call_pc) | 1065 | noinline void __tsan_func_entry(void *call_pc) |
| 924 | { | 1066 | { |
| 1067 | if (!IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY)) | ||
| 1068 | return; | ||
| 1069 | |||
| 1070 | add_kcsan_stack_depth(1); | ||
| 925 | } | 1071 | } |
| 926 | EXPORT_SYMBOL(__tsan_func_entry); | 1072 | EXPORT_SYMBOL(__tsan_func_entry); |
| 1073 | |||
| 927 | void __tsan_func_exit(void); | 1074 | void __tsan_func_exit(void); |
| 928 | void __tsan_func_exit(void) | 1075 | noinline void __tsan_func_exit(void) |
| 929 | { | 1076 | { |
| 1077 | struct kcsan_scoped_access *reorder_access; | ||
| 1078 | |||
| 1079 | if (!IS_ENABLED(CONFIG_KCSAN_WEAK_MEMORY)) | ||
| 1080 | return; | ||
| 1081 | |||
| 1082 | reorder_access = get_reorder_access(get_ctx()); | ||
| 1083 | if (!reorder_access) | ||
| 1084 | goto out; | ||
| 1085 | |||
| 1086 | if (get_kcsan_stack_depth() <= reorder_access->stack_depth) { | ||
| 1087 | /* | ||
| 1088 | * Access check to catch cases where write without a barrier | ||
| 1089 | * (supposed release) was last access in function: because | ||
| 1090 | * instrumentation is inserted before the real access, a data | ||
| 1091 | * race due to the write giving up a c-s would only be caught if | ||
| 1092 | * we do the conflicting access after. | ||
| 1093 | */ | ||
| 1094 | check_access(reorder_access->ptr, reorder_access->size, | ||
| 1095 | reorder_access->type, reorder_access->ip); | ||
| 1096 | reorder_access->size = 0; | ||
| 1097 | reorder_access->stack_depth = INT_MIN; | ||
| 1098 | } | ||
| 1099 | out: | ||
| 1100 | add_kcsan_stack_depth(-1); | ||
| 930 | } | 1101 | } |
| 931 | EXPORT_SYMBOL(__tsan_func_exit); | 1102 | EXPORT_SYMBOL(__tsan_func_exit); |
| 1103 | |||
| 932 | void __tsan_init(void); | 1104 | void __tsan_init(void); |
| 933 | void __tsan_init(void) | 1105 | void __tsan_init(void) |
| 934 | { | 1106 | { |