History

rcu-tasks: Fix computation of CPU-to-list shift counts

The ->percpu_enqueue_shift field is used to map from the running CPU number to the index of the corresponding callback list. This mapping can change at runtime in response to varying callback load, resulting in varying levels of contention on the callback-list locks. Unfortunately, the initial value of this field is correct only if the system happens to have a power-of-two number of CPUs, otherwise the callbacks from the high-numbered CPUs can be placed into the callback list indexed by 1 (rather than 0), and those index-1 callbacks will be ignored. This can result in soft lockups and hangs. This commit therefore corrects this mapping, adding one to this shift count as needed for systems having odd numbers of CPUs.

Paul E. McKenney committed 4 years ago

Commit da123011 parent e7833621 file · +8 −4Browse files

Unified · Split

kernel/rcu/tasks.h+8−4
@@ -123,7 +123,7 @@ static struct rcu_tasks rt_name = \ · 122 unmodified lines
123 .call_func = call, \123 .call_func = call, \
124 .rtpcpu = &rt_name ## __percpu, \124 .rtpcpu = &rt_name ## __percpu, \
125 .name = n, \125 .name = n, \
126 .percpu_enqueue_shift = ilog2(CONFIG_NR_CPUS), \126 .percpu_enqueue_shift = ilog2(CONFIG_NR_CPUS) + 1, \
127 .percpu_enqueue_lim = 1, \127 .percpu_enqueue_lim = 1, \
128 .percpu_dequeue_lim = 1, \128 .percpu_dequeue_lim = 1, \
129 .barrier_q_mutex = __MUTEX_INITIALIZER(rt_name.barrier_q_mutex), \129 .barrier_q_mutex = __MUTEX_INITIALIZER(rt_name.barrier_q_mutex), \
@@ -216,6 +216,7 @@ static void cblist_init_generic(struct rcu_tasks *rtp) · 86 unmodified lines
216 int cpu;216 int cpu;
217 unsigned long flags;217 unsigned long flags;
218 int lim;218 int lim;
219 int shift;
219 220
220 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);221 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
221 if (rcu_task_enqueue_lim < 0) {222 if (rcu_task_enqueue_lim < 0) {
@@ -229,7 +230,10 @@ static void cblist_init_generic(struct rcu_tasks *rtp) · 7 unmodified lines
229 230
230 if (lim > nr_cpu_ids)231 if (lim > nr_cpu_ids)
231 lim = nr_cpu_ids;232 lim = nr_cpu_ids;
232 WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids / lim));233 shift = ilog2(nr_cpu_ids / lim);
234 if (((nr_cpu_ids - 1) >> shift) >= lim)
235 shift++;
236 WRITE_ONCE(rtp->percpu_enqueue_shift, shift);
233 WRITE_ONCE(rtp->percpu_dequeue_lim, lim);237 WRITE_ONCE(rtp->percpu_dequeue_lim, lim);
234 smp_store_release(&rtp->percpu_enqueue_lim, lim);238 smp_store_release(&rtp->percpu_enqueue_lim, lim);
235 for_each_possible_cpu(cpu) {239 for_each_possible_cpu(cpu) {
@@ -298,7 +302,7 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, · 62 unmodified lines
298 if (unlikely(needadjust)) {302 if (unlikely(needadjust)) {
299 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);303 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
300 if (rtp->percpu_enqueue_lim != nr_cpu_ids) {304 if (rtp->percpu_enqueue_lim != nr_cpu_ids) {
301 WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids));305 WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids) + 1);
302 WRITE_ONCE(rtp->percpu_dequeue_lim, nr_cpu_ids);306 WRITE_ONCE(rtp->percpu_dequeue_lim, nr_cpu_ids);
303 smp_store_release(&rtp->percpu_enqueue_lim, nr_cpu_ids);307 smp_store_release(&rtp->percpu_enqueue_lim, nr_cpu_ids);
304 pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name);308 pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name);
@@ -413,7 +417,7 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) · 108 unmodified lines
413 if (rcu_task_cb_adjust && ncbs <= rcu_task_collapse_lim) {417 if (rcu_task_cb_adjust && ncbs <= rcu_task_collapse_lim) {
414 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);418 raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
415 if (rtp->percpu_enqueue_lim > 1) {419 if (rtp->percpu_enqueue_lim > 1) {
416 WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids));420 WRITE_ONCE(rtp->percpu_enqueue_shift, ilog2(nr_cpu_ids) + 1);
417 smp_store_release(&rtp->percpu_enqueue_lim, 1);421 smp_store_release(&rtp->percpu_enqueue_lim, 1);
418 rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu();422 rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu();
419 pr_info("Starting switch %s to CPU-0 callback queuing.\n", rtp->name);423 pr_info("Starting switch %s to CPU-0 callback queuing.\n", rtp->name);