History

rcu: Refactor rcu_barrier() empty-list handling

This commit saves a few lines by checking first for an empty callback list. If the callback list is empty, then that CPU is taken care of, regardless of its online or nocb state. Also simplify tracing accordingly and fold a few lines together.

Paul E. McKenney committed 4 years ago

Commit 0cabb471 parent 82980b12 files · +12 −22Browse files

Unified · Split

2 files
  1. include/trace/events/rcu.h +4 −5
  2. kernel/rcu/tree.c +8 −17
include/trace/events/rcu.h+4−5
@@ -794,16 +794,15 @@ TRACE_EVENT_RCU(rcu_torture_read, · 793 unmodified lines
794 * Tracepoint for rcu_barrier() execution. The string "s" describes794 * Tracepoint for rcu_barrier() execution. The string "s" describes
795 * the rcu_barrier phase:795 * the rcu_barrier phase:
796 * "Begin": rcu_barrier() started.796 * "Begin": rcu_barrier() started.
797 * "CB": An rcu_barrier_callback() invoked a callback, not the last.
797 * "EarlyExit": rcu_barrier() piggybacked, thus early exit.798 * "EarlyExit": rcu_barrier() piggybacked, thus early exit.
798 * "Inc1": rcu_barrier() piggyback check counter incremented.799 * "Inc1": rcu_barrier() piggyback check counter incremented.
799 * "OfflineNoCBQ": rcu_barrier() found offline no-CBs CPU with callbacks.800 * "Inc2": rcu_barrier() piggyback check counter incremented.
800 * "OnlineQ": rcu_barrier() found online CPU with callbacks.
801 * "OnlineNQ": rcu_barrier() found online CPU, no callbacks.
802 * "IRQ": An rcu_barrier_callback() callback posted on remote CPU.801 * "IRQ": An rcu_barrier_callback() callback posted on remote CPU.
803 * "IRQNQ": An rcu_barrier_callback() callback found no callbacks.802 * "IRQNQ": An rcu_barrier_callback() callback found no callbacks.
804 * "CB": An rcu_barrier_callback() invoked a callback, not the last.
805 * "LastCB": An rcu_barrier_callback() invoked the last callback.803 * "LastCB": An rcu_barrier_callback() invoked the last callback.
806 * "Inc2": rcu_barrier() piggyback check counter incremented.804 * "NQ": rcu_barrier() found a CPU with no callbacks.
805 * "OnlineQ": rcu_barrier() found online CPU with callbacks.
807 * The "cpu" argument is the CPU or -1 if meaningless, the "cnt" argument806 * The "cpu" argument is the CPU or -1 if meaningless, the "cnt" argument
808 * is the count of remaining callbacks, and "done" is the piggybacking count.807 * is the count of remaining callbacks, and "done" is the piggybacking count.
809 */808 */
kernel/rcu/tree.c+8−17
@@ -4030,8 +4030,7 @@ void rcu_barrier(void) · 4029 unmodified lines
4030 4030
4031 /* Did someone else do our work for us? */4031 /* Did someone else do our work for us? */
4032 if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {4032 if (rcu_seq_done(&rcu_state.barrier_sequence, s)) {
4033 rcu_barrier_trace(TPS("EarlyExit"), -1,4033 rcu_barrier_trace(TPS("EarlyExit"), -1, rcu_state.barrier_sequence);
4034 rcu_state.barrier_sequence);
4035 smp_mb(); /* caller's subsequent code after above check. */4034 smp_mb(); /* caller's subsequent code after above check. */
4036 mutex_unlock(&rcu_state.barrier_mutex);4035 mutex_unlock(&rcu_state.barrier_mutex);
4037 return;4036 return;
@@ -4059,26 +4058,18 @@ void rcu_barrier(void) · 21 unmodified lines
4059 */4058 */
4060 for_each_possible_cpu(cpu) {4059 for_each_possible_cpu(cpu) {
4061 rdp = per_cpu_ptr(&rcu_data, cpu);4060 rdp = per_cpu_ptr(&rcu_data, cpu);
4062 if (cpu_is_offline(cpu) &&4061 if (!rcu_segcblist_n_cbs(&rdp->cblist)) {
4063 !rcu_rdp_is_offloaded(rdp))4062 rcu_barrier_trace(TPS("NQ"), cpu, rcu_state.barrier_sequence);
4064 continue;4063 continue;
4065 if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) {4064 }
4066 rcu_barrier_trace(TPS("OnlineQ"), cpu,4065 if (cpu_online(cpu)) {
4067 rcu_state.barrier_sequence);4066 rcu_barrier_trace(TPS("OnlineQ"), cpu, rcu_state.barrier_sequence);
4068 smp_call_function_single(cpu, rcu_barrier_func, (void *)cpu, 1);4067 smp_call_function_single(cpu, rcu_barrier_func, (void *)cpu, 1);
4069 } else if (rcu_segcblist_n_cbs(&rdp->cblist) &&4068 } else {
4070 cpu_is_offline(cpu)) {4069 rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu, rcu_state.barrier_sequence);
4071 rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu,
4072 rcu_state.barrier_sequence);
4073 local_irq_disable();4070 local_irq_disable();
4074 rcu_barrier_func((void *)cpu);4071 rcu_barrier_func((void *)cpu);
4075 local_irq_enable();4072 local_irq_enable();
4076 } else if (cpu_is_offline(cpu)) {
4077 rcu_barrier_trace(TPS("OfflineNoCBNoQ"), cpu,
4078 rcu_state.barrier_sequence);
4079 } else {
4080 rcu_barrier_trace(TPS("OnlineNQ"), cpu,
4081 rcu_state.barrier_sequence);
4082 }4073 }
4083 }4074 }
4084 cpus_read_unlock();4075 cpus_read_unlock();