Although the rdp->rcu_cpu_has_work field is accessed only by the corresponding CPU, it can be accessed by both interrupt handlers via invoke_rcu_core_kthread() and at task level via rcu_cpu_kthread(). This means that we need this_cpu_read() rather than __this_cpu_read(), this_cpu_write() rather than __this_cpu_write(), and READ_ONCE() rather than plain C-language loads. The exception is the boot-time rcu_spawn_core_kthreads(), which cannot race with kthreads that have not yet been spawned. This commit therefore makes it so. KCSAN located this issue.
Showing the changes to kernel (1 of 1 file). Show all
Unified · Split
| @@ -2671,7 +2671,7 @@ static void rcu_do_batch(struct rcu_data *rdp) · 2670 unmodified lines | |||
| 2671 | // reporting, so check time limits for them. | 2671 | // reporting, so check time limits for them. |
| 2672 | if (rdp->rcu_cpu_kthread_status == RCU_KTHREAD_RUNNING && | 2672 | if (rdp->rcu_cpu_kthread_status == RCU_KTHREAD_RUNNING && |
| 2673 | rcu_do_batch_check_time(count, tlimit, jlimit_check, jlimit)) { | 2673 | rcu_do_batch_check_time(count, tlimit, jlimit_check, jlimit)) { |
| 2674 | rdp->rcu_cpu_has_work = 1; | 2674 | WRITE_ONCE(rdp->rcu_cpu_has_work, 1); |
| 2675 | break; | 2675 | break; |
| 2676 | } | 2676 | } |
| 2677 | } | 2677 | } |
| @@ -2931,7 +2931,7 @@ static void invoke_rcu_core_kthread(void) · 253 unmodified lines | |||
| 2931 | unsigned long flags; | 2931 | unsigned long flags; |
| 2932 | 2932 | ||
| 2933 | local_irq_save(flags); | 2933 | local_irq_save(flags); |
| 2934 | __this_cpu_write(rcu_data.rcu_cpu_has_work, 1); | 2934 | this_cpu_write(rcu_data.rcu_cpu_has_work, 1); |
| 2935 | t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task); | 2935 | t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task); |
| 2936 | if (t != NULL && t != current) | 2936 | if (t != NULL && t != current) |
| 2937 | rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status)); | 2937 | rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status)); |
| @@ -2958,7 +2958,7 @@ static void rcu_cpu_kthread_park(unsigned int cpu) · 20 unmodified lines | |||
| 2958 | 2958 | ||
| 2959 | static int rcu_cpu_kthread_should_run(unsigned int cpu) | 2959 | static int rcu_cpu_kthread_should_run(unsigned int cpu) |
| 2960 | { | 2960 | { |
| 2961 | return __this_cpu_read(rcu_data.rcu_cpu_has_work); | 2961 | return this_cpu_read(rcu_data.rcu_cpu_has_work); |
| 2962 | } | 2962 | } |
| 2963 | 2963 | ||
| 2964 | /* | 2964 | /* |
| @@ -2979,7 +2979,7 @@ static void rcu_cpu_kthread(unsigned int cpu) · 14 unmodified lines | |||
| 2979 | local_bh_disable(); | 2979 | local_bh_disable(); |
| 2980 | *statusp = RCU_KTHREAD_RUNNING; | 2980 | *statusp = RCU_KTHREAD_RUNNING; |
| 2981 | local_irq_disable(); | 2981 | local_irq_disable(); |
| 2982 | work = *workp; | 2982 | work = READ_ONCE(*workp); |
| 2983 | WRITE_ONCE(*workp, 0); | 2983 | WRITE_ONCE(*workp, 0); |
| 2984 | local_irq_enable(); | 2984 | local_irq_enable(); |
| 2985 | if (work) | 2985 | if (work) |