rcu: Mark accesses to rdp->rcu_cpu_has_work

Although the rdp->rcu_cpu_has_work field is accessed only by the corresponding CPU, it can be accessed by both interrupt handlers via invoke_rcu_core_kthread() and at task level via rcu_cpu_kthread(). This means that we need this_cpu_read() rather than __this_cpu_read(), this_cpu_write() rather than __this_cpu_write(), and READ_ONCE() rather than plain C-language loads. The exception is the boot-time rcu_spawn_core_kthreads(), which cannot race with kthreads that have not yet been spawned. This commit therefore makes it so. KCSAN located this issue.

Paul E. McKenney committed 3 months ago

Commit 2aaee3f1 parent 545b3ac1 file · +4 −4Browse files

Showing the changes to kernel (1 of 1 file). Show all

Unified · Split

kernel/rcu/tree.c+4−4

@@ -2671,7 +2671,7 @@ static void rcu_do_batch(struct rcu_data *rdp) · 2670 unmodified lines
2671	2671	// reporting, so check time limits for them.
2672	2672	if (rdp->rcu_cpu_kthread_status == RCU_KTHREAD_RUNNING &&
2673	2673	rcu_do_batch_check_time(count, tlimit, jlimit_check, jlimit)) {
2674		− rdp->rcu_cpu_has_work = 1;
	2674	+ WRITE_ONCE(rdp->rcu_cpu_has_work, 1);
2675	2675	break;
2676	2676	}
2677	2677	}
@@ -2931,7 +2931,7 @@ static void invoke_rcu_core_kthread(void) · 253 unmodified lines
2931	2931	unsigned long flags;
2932	2932
2933	2933	local_irq_save(flags);
2934		− __this_cpu_write(rcu_data.rcu_cpu_has_work, 1);
	2934	+ this_cpu_write(rcu_data.rcu_cpu_has_work, 1);
2935	2935	t = __this_cpu_read(rcu_data.rcu_cpu_kthread_task);
2936	2936	if (t != NULL && t != current)
2937	2937	rcu_wake_cond(t, __this_cpu_read(rcu_data.rcu_cpu_kthread_status));
@@ -2958,7 +2958,7 @@ static void rcu_cpu_kthread_park(unsigned int cpu) · 20 unmodified lines
2958	2958
2959	2959	static int rcu_cpu_kthread_should_run(unsigned int cpu)
2960	2960	{
2961		− return __this_cpu_read(rcu_data.rcu_cpu_has_work);
	2961	+ return this_cpu_read(rcu_data.rcu_cpu_has_work);
2962	2962	}
2963	2963
2964	2964	/*
@@ -2979,7 +2979,7 @@ static void rcu_cpu_kthread(unsigned int cpu) · 14 unmodified lines
2979	2979	local_bh_disable();
2980	2980	*statusp = RCU_KTHREAD_RUNNING;
2981	2981	local_irq_disable();
2982		− work = *workp;
	2982	+ work = READ_ONCE(*workp);
2983	2983	WRITE_ONCE(*workp, 0);
2984	2984	local_irq_enable();
2985	2985	if (work)