rcu-tasks: Fix computation of CPU-to-list shift counts

The ->percpu_enqueue_shift field is used to map from the running CPU number to the index of the corresponding callback list. This mapping can change at runtime in response to varying callback load, resulting in varying levels of contention on the callback-list locks. Unfortunately, the initial value of this field is correct only if the system happens to have a power-of-two number of CPUs, otherwise the callbacks from the high-numbered CPUs can be placed into the callback list indexed by 1 (rather than 0), and those index-1 callbacks will be ignored. This can result in soft lockups and hangs. This commit therefore corrects this mapping, adding one to this shift count as needed for systems having odd numbers of CPUs.

Paul E. McKenney committed 4 years ago

Commit da123011 parent e7833621 file · +8 −4Browse files

Unified · Split

kernel/rcu/tasks.h+8−4

@@ -123,7 +123,7 @@ static struct rcu_tasks rt_name = \ · 122 unmodified lines
123	123	.call_func = call, \
124	124	.rtpcpu = &rt_name ## __percpu, \
125	125	.name = n, \
126		− .percpu_enqueue_shift = ilog2(CONFIG_NR_CPUS), \
	126	+ .percpu_enqueue_shift = ilog2(CONFIG_NR_CPUS) + 1, \
127	127	.percpu_enqueue_lim = 1, \
128	128	.percpu_dequeue_lim = 1, \
129	129	.barrier_q_mutex = __MUTEX_INITIALIZER(rt_name.barrier_q_mutex), \
@@ -216,6 +216,7 @@ static void cblist_init_generic(struct rcu_tasks *rtp) · 86 unmodified lines
216	216	int cpu;
217	217	unsigned long flags;
218	218	int lim;
	219	+ int shift;
219	220
220	221	raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
221	222	if (rcu_task_enqueue_lim < 0) {
@@ -229,7 +230,10 @@ static void cblist_init_generic(struct rcu_tasks *rtp) · 7 unmodified lines
229	230
230	231	if (lim > nr_cpu_ids)
231	232	lim = nr_cpu_ids;
232		− WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids / lim));
	233	+ shift = ilog2(nr_cpu_ids / lim);
	234	+ if (((nr_cpu_ids - 1) >> shift) >= lim)
	235	+ shift++;
	236	+ WRITE_ONCE(rtp->percpu_enqueue_shift, shift);
233	237	WRITE_ONCE(rtp->percpu_dequeue_lim, lim);
234	238	smp_store_release(&rtp->percpu_enqueue_lim, lim);
235	239	for_each_possible_cpu(cpu) {
@@ -298,7 +302,7 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, · 62 unmodified lines
298	302	if (unlikely(needadjust)) {
299	303	raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
300	304	if (rtp->percpu_enqueue_lim != nr_cpu_ids) {
301		− WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids));
	305	+ WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids) + 1);
302	306	WRITE_ONCE(rtp->percpu_dequeue_lim, nr_cpu_ids);
303	307	smp_store_release(&rtp->percpu_enqueue_lim, nr_cpu_ids);
304	308	pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name);
@@ -413,7 +417,7 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) · 108 unmodified lines
413	417	if (rcu_task_cb_adjust && ncbs <= rcu_task_collapse_lim) {
414	418	raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
415	419	if (rtp->percpu_enqueue_lim > 1) {
416		− WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids));
	420	+ WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids) + 1);
417	421	smp_store_release(&rtp->percpu_enqueue_lim, 1);
418	422	rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu();
419	423	pr_info("Starting switch %s to CPU-0 callback queuing.\n", rtp->name);