History

rcu: use softirq instead of kthreads except when RCU_BOOST=y

This patch #ifdefs RCU kthreads out of the kernel unless RCU_BOOST=y, thus eliminating context-switch overhead if RCU priority boosting has not been configured.

Paul E. McKenney committed 15 years ago

Commit a46e0891 parent 09223374 files · +99 −41Browse files

Unified · Split

4 files
  1. kernel/rcutree.c +44 −15
  2. kernel/rcutree.h +6 −2
  3. kernel/rcutree_plugin.h +26 −15
  4. kernel/rcutree_trace.c +23 −9
kernel/rcutree.c+44−15
@@ -87,6 +87,8 @@ static struct rcu_state *rcu_state; · 86 unmodified lines
8787 int rcu_scheduler_active __read_mostly;
8888 EXPORT_SYMBOL_GPL(rcu_scheduler_active);
8989
90+#ifdef CONFIG_RCU_BOOST
91+
9092 /*
9193 * Control variables for per-CPU and per-rcu_node kthreads. These
9294 * handle all flavors of RCU.
@@ -98,9 +100,11 @@ DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); · 5 unmodified lines
98100 DEFINE_PER_CPU(char, rcu_cpu_has_work);
99101 static char rcu_kthreads_spawnable;
100102
103+#endif /* #ifdef CONFIG_RCU_BOOST */
104+
101105 static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
102−static void invoke_rcu_cpu_kthread(void);
103−static void __invoke_rcu_cpu_kthread(void);
106+static void invoke_rcu_core(void);
107+static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
104108
105109 #define RCU_KTHREAD_PRIO 1 /* RT priority for per-CPU kthreads. */
106110
@@ -1089,6 +1093,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) · 982 unmodified lines
10891093 int need_report = 0;
10901094 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
10911095 struct rcu_node *rnp;
1096+#ifdef CONFIG_RCU_BOOST
10921097 struct task_struct *t;
10931098
10941099 /* Stop the CPU's kthread. */
@@ -1097,6 +1102,7 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) · 2 unmodified lines
10971102 per_cpu(rcu_cpu_kthread_task, cpu) = NULL;
10981103 kthread_stop(t);
10991104 }
1105+#endif /* #ifdef CONFIG_RCU_BOOST */
11001106
11011107 /* Exclude any attempts to start a new grace period. */
11021108 raw_spin_lock_irqsave(&rsp->onofflock, flags);
@@ -1232,7 +1238,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) · 129 unmodified lines
12321238
12331239 /* Re-raise the RCU softirq if there are callbacks remaining. */
12341240 if (cpu_has_callbacks_ready_to_invoke(rdp))
1235− invoke_rcu_cpu_kthread();
1241+ invoke_rcu_core();
12361242 }
12371243
12381244 /*
@@ -1278,7 +1284,7 @@ void rcu_check_callbacks(int cpu, int user) · 39 unmodified lines
12781284 }
12791285 rcu_preempt_check_callbacks(cpu);
12801286 if (rcu_pending(cpu))
1281− invoke_rcu_cpu_kthread();
1287+ invoke_rcu_core();
12821288 }
12831289
12841290 #ifdef CONFIG_SMP
@@ -1444,9 +1450,11 @@ __rcu_process_callbacks(struct rcu_state *rsp, struct rcu_data *rdp) · 159 unmodified lines
14441450
14451451 /* If there are callbacks ready, invoke them. */
14461452 if (cpu_has_callbacks_ready_to_invoke(rdp))
1447− __invoke_rcu_cpu_kthread();
1453+ invoke_rcu_callbacks(rsp, rdp);
14481454 }
14491455
1456+#ifdef CONFIG_RCU_BOOST
1457+
14501458 static void rcu_kthread_do_work(void)
14511459 {
14521460 rcu_do_batch(&rcu_sched_state, &__get_cpu_var(rcu_sched_data));
@@ -1454,6 +1462,8 @@ static void rcu_kthread_do_work(void) · 1 unmodified line
14541462 rcu_preempt_do_callbacks();
14551463 }
14561464
1465+#endif /* #ifdef CONFIG_RCU_BOOST */
1466+
14571467 /*
14581468 * Do softirq processing for the current CPU.
14591469 */
@@ -1474,25 +1484,22 @@ static void rcu_process_callbacks(struct softirq_action *unused) · 14 unmodified lines
14741484 * the current CPU with interrupts disabled, the rcu_cpu_kthread_task
14751485 * cannot disappear out from under us.
14761486 */
1477−static void __invoke_rcu_cpu_kthread(void)
1487+static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
14781488 {
1479− unsigned long flags;
1480−
1481− local_irq_save(flags);
1482− __this_cpu_write(rcu_cpu_has_work, 1);
1483− if (__this_cpu_read(rcu_cpu_kthread_task) == NULL) {
1484− local_irq_restore(flags);
1489+ if (likely(!rsp->boost)) {
1490+ rcu_do_batch(rsp, rdp);
14851491 return;
14861492 }
1487− wake_up_process(__this_cpu_read(rcu_cpu_kthread_task));
1488− local_irq_restore(flags);
1493+ invoke_rcu_callbacks_kthread();
14891494 }
14901495
1491−static void invoke_rcu_cpu_kthread(void)
1496+static void invoke_rcu_core(void)
14921497 {
14931498 raise_softirq(RCU_SOFTIRQ);
14941499 }
14951500
1501+#ifdef CONFIG_RCU_BOOST
1502+
14961503 /*
14971504 * Wake up the specified per-rcu_node-structure kthread.
14981505 * Because the per-rcu_node kthreads are immortal, we don't need
@@ -1818,6 +1825,18 @@ static int __init rcu_spawn_kthreads(void) · 319 unmodified lines
18181825 }
18191826 early_initcall(rcu_spawn_kthreads);
18201827
1828+#else /* #ifdef CONFIG_RCU_BOOST */
1829+
1830+static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu)
1831+{
1832+}
1833+
1834+static void rcu_cpu_kthread_setrt(int cpu, int to_rt)
1835+{
1836+}
1837+
1838+#endif /* #else #ifdef CONFIG_RCU_BOOST */
1839+
18211840 static void
18221841 __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
18231842 struct rcu_state *rsp)
@@ -2224,6 +2243,8 @@ static void __cpuinit rcu_prepare_cpu(int cpu) · 400 unmodified lines
22242243 rcu_preempt_init_percpu_data(cpu);
22252244 }
22262245
2246+#ifdef CONFIG_RCU_BOOST
2247+
22272248 static void __cpuinit rcu_prepare_kthreads(int cpu)
22282249 {
22292250 struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu);
@@ -2237,6 +2258,14 @@ static void __cpuinit rcu_prepare_kthreads(int cpu) · 7 unmodified lines
22372258 }
22382259 }
22392260
2261+#else /* #ifdef CONFIG_RCU_BOOST */
2262+
2263+static void __cpuinit rcu_prepare_kthreads(int cpu)
2264+{
2265+}
2266+
2267+#endif /* #else #ifdef CONFIG_RCU_BOOST */
2268+
22402269 /*
22412270 * Handle CPU online/offline notification events.
22422271 */
kernel/rcutree.h+6−2
@@ -369,6 +369,7 @@ struct rcu_state { · 368 unmodified lines
369369 /* period because */
370370 /* force_quiescent_state() */
371371 /* was running. */
372+ u8 boost; /* Subject to priority boost. */
372373 unsigned long gpnum; /* Current gp number. */
373374 unsigned long completed; /* # of last completed gp. */
374375
@@ -439,7 +440,6 @@ static void rcu_preempt_offline_cpu(int cpu); · 64 unmodified lines
439440 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
440441 static void rcu_preempt_check_callbacks(int cpu);
441442 static void rcu_preempt_process_callbacks(void);
442−static void rcu_preempt_do_callbacks(void);
443443 void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
444444 #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_TREE_PREEMPT_RCU)
445445 static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp);
@@ -451,11 +451,15 @@ static void rcu_preempt_send_cbs_to_online(void); · 5 unmodified lines
451451 static void __init __rcu_init_preempt(void);
452452 static void rcu_needs_cpu_flush(void);
453453 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);
454+static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);
455+static void invoke_rcu_callbacks_kthread(void);
456+#ifdef CONFIG_RCU_BOOST
457+static void rcu_preempt_do_callbacks(void);
454458 static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp,
455459 cpumask_var_t cm);
456−static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);
457460 static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
458461 struct rcu_node *rnp,
459462 int rnp_index);
463+#endif /* #ifdef CONFIG_RCU_BOOST */
460464
461465 #endif /* #ifndef RCU_TREE_NONCORE */
kernel/rcutree_plugin.h+26−15
@@ -602,11 +602,15 @@ static void rcu_preempt_process_callbacks(void) · 601 unmodified lines
602602 &__get_cpu_var(rcu_preempt_data));
603603 }
604604
605+#ifdef CONFIG_RCU_BOOST
606+
605607 static void rcu_preempt_do_callbacks(void)
606608 {
607609 rcu_do_batch(&rcu_preempt_state, &__get_cpu_var(rcu_preempt_data));
608610 }
609611
612+#endif /* #ifdef CONFIG_RCU_BOOST */
613+
610614 /*
611615 * Queue a preemptible-RCU callback for invocation after a grace period.
612616 */
@@ -1002,10 +1006,6 @@ static void rcu_preempt_process_callbacks(void) · 389 unmodified lines
10021006 {
10031007 }
10041008
1005−static void rcu_preempt_do_callbacks(void)
1006−{
1007−}
1008−
10091009 /*
10101010 * Wait for an rcu-preempt grace period, but make it happen quickly.
10111011 * But because preemptible RCU does not exist, map to rcu-sched.
@@ -1257,6 +1257,23 @@ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) · 245 unmodified lines
12571257 }
12581258 }
12591259
1260+/*
1261+ * Wake up the per-CPU kthread to invoke RCU callbacks.
1262+ */
1263+static void invoke_rcu_callbacks_kthread(void)
1264+{
1265+ unsigned long flags;
1266+
1267+ local_irq_save(flags);
1268+ __this_cpu_write(rcu_cpu_has_work, 1);
1269+ if (__this_cpu_read(rcu_cpu_kthread_task) == NULL) {
1270+ local_irq_restore(flags);
1271+ return;
1272+ }
1273+ wake_up_process(__this_cpu_read(rcu_cpu_kthread_task));
1274+ local_irq_restore(flags);
1275+}
1276+
12601277 /*
12611278 * Set the affinity of the boost kthread. The CPU-hotplug locks are
12621279 * held, so no one should be messing with the existence of the boost
@@ -1297,6 +1314,7 @@ static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp, · 34 unmodified lines
12971314
12981315 if (&rcu_preempt_state != rsp)
12991316 return 0;
1317+ rsp->boost = 1;
13001318 if (rnp->boost_kthread_task != NULL)
13011319 return 0;
13021320 t = kthread_create(rcu_boost_kthread, (void *)rnp,
@@ -1319,22 +1337,15 @@ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) · 16 unmodified lines
13191337 raw_spin_unlock_irqrestore(&rnp->lock, flags);
13201338 }
13211339
1322−static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp,
1323− cpumask_var_t cm)
1340+static void invoke_rcu_callbacks_kthread(void)
13241341 {
1342+ WARN_ON_ONCE(1);
13251343 }
13261344
13271345 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp)
13281346 {
13291347 }
13301348
1331−static int __cpuinit rcu_spawn_one_boost_kthread(struct rcu_state *rsp,
1332− struct rcu_node *rnp,
1333− int rnp_index)
1334−{
1335− return 0;
1336−}
1337−
13381349 #endif /* #else #ifdef CONFIG_RCU_BOOST */
13391350
13401351 #ifndef CONFIG_SMP
@@ -1509,7 +1520,7 @@ static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff); · 168 unmodified lines
15091520 *
15101521 * Because it is not legal to invoke rcu_process_callbacks() with irqs
15111522 * disabled, we do one pass of force_quiescent_state(), then do a
1512− * invoke_rcu_cpu_kthread() to cause rcu_process_callbacks() to be invoked
1523+ * invoke_rcu_core() to cause rcu_process_callbacks() to be invoked
15131524 * later. The per-cpu rcu_dyntick_drain variable controls the sequencing.
15141525 */
15151526 int rcu_needs_cpu(int cpu)
@@ -1560,7 +1571,7 @@ int rcu_needs_cpu(int cpu) · 44 unmodified lines
15601571
15611572 /* If RCU callbacks are still pending, RCU still needs this CPU. */
15621573 if (c)
1563− invoke_rcu_cpu_kthread();
1574+ invoke_rcu_core();
15641575 return c;
15651576 }
15661577
kernel/rcutree_trace.c+23−9
@@ -46,6 +46,8 @@ · 45 unmodified lines
4646 #define RCU_TREE_NONCORE
4747 #include "rcutree.h"
4848
49+#ifdef CONFIG_RCU_BOOST
50+
4951 DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
5052 DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_cpu);
5153 DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
@@ -58,6 +60,8 @@ static char convert_kthread_status(unsigned int kthread_status) · 6 unmodified lines
5860 return "SRWOY"[kthread_status];
5961 }
6062
63+#endif /* #ifdef CONFIG_RCU_BOOST */
64+
6165 static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp)
6266 {
6367 if (!rdp->beenonline)
@@ -76,7 +80,7 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp) · 12 unmodified lines
7680 rdp->dynticks_fqs);
7781 #endif /* #ifdef CONFIG_NO_HZ */
7882 seq_printf(m, " of=%lu ri=%lu", rdp->offline_fqs, rdp->resched_ipi);
79− seq_printf(m, " ql=%ld qs=%c%c%c%c kt=%d/%c/%d ktl=%x b=%ld",
83+ seq_printf(m, " ql=%ld qs=%c%c%c%c",
8084 rdp->qlen,
8185 ".N"[rdp->nxttail[RCU_NEXT_READY_TAIL] !=
8286 rdp->nxttail[RCU_NEXT_TAIL]],
@@ -84,13 +88,16 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp) · 1 unmodified line
8488 rdp->nxttail[RCU_NEXT_READY_TAIL]],
8589 ".W"[rdp->nxttail[RCU_DONE_TAIL] !=
8690 rdp->nxttail[RCU_WAIT_TAIL]],
87− ".D"[&rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL]],
91+ ".D"[&rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL]]);
92+#ifdef CONFIG_RCU_BOOST
93+ seq_printf(m, " kt=%d/%c/%d ktl=%x",
8894 per_cpu(rcu_cpu_has_work, rdp->cpu),
8995 convert_kthread_status(per_cpu(rcu_cpu_kthread_status,
9096 rdp->cpu)),
9197 per_cpu(rcu_cpu_kthread_cpu, rdp->cpu),
92− per_cpu(rcu_cpu_kthread_loops, rdp->cpu) & 0xffff,
93− rdp->blimit);
98+ per_cpu(rcu_cpu_kthread_loops, rdp->cpu) & 0xffff);
99+#endif /* #ifdef CONFIG_RCU_BOOST */
100+ seq_printf(m, " b=%ld", rdp->blimit);
94101 seq_printf(m, " ci=%lu co=%lu ca=%lu\n",
95102 rdp->n_cbs_invoked, rdp->n_cbs_orphaned, rdp->n_cbs_adopted);
96103 }
@@ -147,18 +154,21 @@ static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp) · 50 unmodified lines
147154 rdp->dynticks_fqs);
148155 #endif /* #ifdef CONFIG_NO_HZ */
149156 seq_printf(m, ",%lu,%lu", rdp->offline_fqs, rdp->resched_ipi);
150− seq_printf(m, ",%ld,\"%c%c%c%c\",%d,\"%c\",%ld", rdp->qlen,
157+ seq_printf(m, ",%ld,\"%c%c%c%c\"", rdp->qlen,
151158 ".N"[rdp->nxttail[RCU_NEXT_READY_TAIL] !=
152159 rdp->nxttail[RCU_NEXT_TAIL]],
153160 ".R"[rdp->nxttail[RCU_WAIT_TAIL] !=
154161 rdp->nxttail[RCU_NEXT_READY_TAIL]],
155162 ".W"[rdp->nxttail[RCU_DONE_TAIL] !=
156163 rdp->nxttail[RCU_WAIT_TAIL]],
157− ".D"[&rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL]],
164+ ".D"[&rdp->nxtlist != rdp->nxttail[RCU_DONE_TAIL]]);
165+#ifdef CONFIG_RCU_BOOST
166+ seq_printf(m, ",%d,\"%c\"",
158167 per_cpu(rcu_cpu_has_work, rdp->cpu),
159168 convert_kthread_status(per_cpu(rcu_cpu_kthread_status,
160− rdp->cpu)),
161− rdp->blimit);
169+ rdp->cpu)));
170+#endif /* #ifdef CONFIG_RCU_BOOST */
171+ seq_printf(m, ",%ld", rdp->blimit);
162172 seq_printf(m, ",%lu,%lu,%lu\n",
163173 rdp->n_cbs_invoked, rdp->n_cbs_orphaned, rdp->n_cbs_adopted);
164174 }
@@ -169,7 +179,11 @@ static int show_rcudata_csv(struct seq_file *m, void *unused) · 4 unmodified lines
169179 #ifdef CONFIG_NO_HZ
170180 seq_puts(m, "\"dt\",\"dt nesting\",\"dt NMI nesting\",\"df\",");
171181 #endif /* #ifdef CONFIG_NO_HZ */
172− seq_puts(m, "\"of\",\"ri\",\"ql\",\"b\",\"ci\",\"co\",\"ca\"\n");
182+ seq_puts(m, "\"of\",\"ri\",\"ql\",\"qs\"");
183+#ifdef CONFIG_RCU_BOOST
184+ seq_puts(m, "\"kt\",\"ktl\"");
185+#endif /* #ifdef CONFIG_RCU_BOOST */
186+ seq_puts(m, ",\"b\",\"ci\",\"co\",\"ca\"\n");
173187 #ifdef CONFIG_TREE_PREEMPT_RCU
174188 seq_puts(m, "\"rcu_preempt:\"\n");
175189 PRINT_RCU_DATA(rcu_preempt_data, print_one_rcu_data_csv, m);

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t