Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: (24 commits) rcu: remove all rcu head initializations, except on_stack initializations rcu head introduce rcu head init on stack Debugobjects transition check rcu: fix build bug in RCU_FAST_NO_HZ builds rcu: RCU_FAST_NO_HZ must check RCU dyntick state rcu: make SRCU usable in modules rcu: improve the RCU CPU-stall warning documentation rcu: reduce the number of spurious RCU_SOFTIRQ invocations rcu: permit discontiguous cpu_possible_mask CPU numbering rcu: improve RCU CPU stall-warning messages rcu: print boot-time console messages if RCU configs out of ordinary rcu: disable CPU stall warnings upon panic rcu: enable CPU_STALL_VERBOSE by default rcu: slim down rcutiny by removing rcu_scheduler_active and friends rcu: refactor RCU's context-switch handling rcu: rename rcutiny rcu_ctrlblk to rcu_sched_ctrlblk rcu: shrink rcutiny by making synchronize_rcu_bh() be inline rcu: fix now-bogus rcu_scheduler_active comments. rcu: Fix bogus CONFIG_PROVE_LOCKING in comments to reflect reality. rcu: ignore offline CPUs in last non-dyntick-idle CPU check ...

Linus Torvalds committed 16 years ago

Commit f262af32 parents 1014cfe 72d5a9f21 files · +470 −143Browse files

Showing the changes to kernel (11 of 21 files). Show all

Unified · Split

11 files

kernel/lockdep.c +3 −0
kernel/rcupdate.c +0 −19
kernel/rcutiny.c +22 −13
kernel/rcutiny_plugin.h +39 −0
kernel/rcutorture.c +2 −0
kernel/rcutree.c +96 −35
kernel/rcutree.h +2 −0
kernel/rcutree_plugin.h +61 −8
kernel/rcutree_trace.c +3 −1
kernel/sched.c +1 −1
kernel/softirq.c +1 −1

kernel/lockdep.c+3−0

@@ -3806,8 +3806,11 @@ void lockdep_rcu_dereference(const char *file, const int line) · 3805 unmodified lines
3806	3806	{
3807	3807	struct task_struct *curr = current;
3808	3808
	3809	+#ifndef CONFIG_PROVE_RCU_REPEATEDLY
3809	3810	if (!debug_locks_off())
3810	3811	return;
	3812	+#endif /* #ifdef CONFIG_PROVE_RCU_REPEATEDLY */
	3813	+ /* Note: the following can be executed concurrently, so be careful. */
3811	3814	printk("\n===================================================\n");
3812	3815	printk( "[ INFO: suspicious rcu_dereference_check() usage. ]\n");
3813	3816	printk( "---------------------------------------------------\n");

kernel/rcupdate.c+0−19

@@ -44,7 +44,6 @@ · 43 unmodified lines
44	44	#include <linux/cpu.h>
45	45	#include <linux/mutex.h>
46	46	#include <linux/module.h>
47		−#include <linux/kernel_stat.h>
48	47	#include <linux/hardirq.h>
49	48
50	49	#ifdef CONFIG_DEBUG_LOCK_ALLOC
@@ -64,9 +63,6 @@ struct lockdep_map rcu_sched_lock_map = · 13 unmodified lines
64	63	EXPORT_SYMBOL_GPL(rcu_sched_lock_map);
65	64	#endif
66	65
67		−int rcu_scheduler_active __read_mostly;
68		−EXPORT_SYMBOL_GPL(rcu_scheduler_active);
69		−
70	66	#ifdef CONFIG_DEBUG_LOCK_ALLOC
71	67
72	68	int debug_lockdep_rcu_enabled(void)
@@ -96,21 +92,6 @@ EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held); · 23 unmodified lines
96	92
97	93	#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
98	94
99		−/*
100		− * This function is invoked towards the end of the scheduler's initialization
101		− * process. Before this is called, the idle task might contain
102		− * RCU read-side critical sections (during which time, this idle
103		− * task is booting the system). After this function is called, the
104		− * idle tasks are prohibited from containing RCU read-side critical
105		− * sections.
106		− */
107		−void rcu_scheduler_starting(void)
108		−{
109		− WARN_ON(num_online_cpus() != 1);
110		− WARN_ON(nr_context_switches() > 0);
111		− rcu_scheduler_active = 1;
112		−}
113		−
114	95	/*
115	96	* Awaken the corresponding synchronize_rcu() instance now that a
116	97	* grace period has elapsed.

kernel/rcutiny.c+22−13

@@ -44,9 +44,9 @@ struct rcu_ctrlblk { · 43 unmodified lines
44	44	};
45	45
46	46	/* Definition for rcupdate control block. */
47		−static struct rcu_ctrlblk rcu_ctrlblk = {
48		− .donetail = &rcu_ctrlblk.rcucblist,
49		− .curtail = &rcu_ctrlblk.rcucblist,
	47	+static struct rcu_ctrlblk rcu_sched_ctrlblk = {
	48	+ .donetail = &rcu_sched_ctrlblk.rcucblist,
	49	+ .curtail = &rcu_sched_ctrlblk.rcucblist,
50	50	};
51	51
52	52	static struct rcu_ctrlblk rcu_bh_ctrlblk = {
@@ -54,6 +54,11 @@ static struct rcu_ctrlblk rcu_bh_ctrlblk = { · 1 unmodified line
54	54	.curtail = &rcu_bh_ctrlblk.rcucblist,
55	55	};
56	56
	57	+#ifdef CONFIG_DEBUG_LOCK_ALLOC
	58	+int rcu_scheduler_active __read_mostly;
	59	+EXPORT_SYMBOL_GPL(rcu_scheduler_active);
	60	+#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
	61	+
57	62	#ifdef CONFIG_NO_HZ
58	63
59	64	static long rcu_dynticks_nesting = 1;
@@ -108,7 +113,8 @@ static int rcu_qsctr_help(struct rcu_ctrlblk *rcp) · 48 unmodified lines
108	113	*/
109	114	void rcu_sched_qs(int cpu)
110	115	{
111		− if (rcu_qsctr_help(&rcu_ctrlblk) + rcu_qsctr_help(&rcu_bh_ctrlblk))
	116	+ if (rcu_qsctr_help(&rcu_sched_ctrlblk) +
	117	+ rcu_qsctr_help(&rcu_bh_ctrlblk))
112	118	raise_softirq(RCU_SOFTIRQ);
113	119	}
114	120
@@ -173,7 +179,7 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp) · 58 unmodified lines
173	179	*/
174	180	static void rcu_process_callbacks(struct softirq_action *unused)
175	181	{
176		− __rcu_process_callbacks(&rcu_ctrlblk);
	182	+ __rcu_process_callbacks(&rcu_sched_ctrlblk);
177	183	__rcu_process_callbacks(&rcu_bh_ctrlblk);
178	184	}
179	185
@@ -187,7 +193,8 @@ static void rcu_process_callbacks(struct softirq_action *unused) · 7 unmodified lines
187	193	*
188	194	* Cool, huh? (Due to Josh Triplett.)
189	195	*
190		− * But we want to make this a static inline later.
	196	+ * But we want to make this a static inline later. The cond_resched()
	197	+ * currently makes this problematic.
191	198	*/
192	199	void synchronize_sched(void)
193	200	{
@@ -195,12 +202,6 @@ void synchronize_sched(void) · 1 unmodified line
195	202	}
196	203	EXPORT_SYMBOL_GPL(synchronize_sched);
197	204
198		−void synchronize_rcu_bh(void)
199		−{
200		− synchronize_sched();
201		−}
202		−EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
203		−
204	205	/*
205	206	* Helper function for call_rcu() and call_rcu_bh().
206	207	*/
@@ -226,7 +227,7 @@ static void __call_rcu(struct rcu_head *head, · 19 unmodified lines
226	227	*/
227	228	void call_rcu(struct rcu_head head, void (func)(struct rcu_head *rcu))
228	229	{
229		− __call_rcu(head, func, &rcu_ctrlblk);
	230	+ __call_rcu(head, func, &rcu_sched_ctrlblk);
230	231	}
231	232	EXPORT_SYMBOL_GPL(call_rcu);
232	233
@@ -244,11 +245,13 @@ void rcu_barrier(void) · 11 unmodified lines
244	245	{
245	246	struct rcu_synchronize rcu;
246	247
	248	+ init_rcu_head_on_stack(&rcu.head);
247	249	init_completion(&rcu.completion);
248	250	/* Will wake me after RCU finished. */
249	251	call_rcu(&rcu.head, wakeme_after_rcu);
250	252	/* Wait for it. */
251	253	wait_for_completion(&rcu.completion);
	254	+ destroy_rcu_head_on_stack(&rcu.head);
252	255	}
253	256	EXPORT_SYMBOL_GPL(rcu_barrier);
254	257
@@ -256,11 +259,13 @@ void rcu_barrier_bh(void) · 1 unmodified line
256	259	{
257	260	struct rcu_synchronize rcu;
258	261
	262	+ init_rcu_head_on_stack(&rcu.head);
259	263	init_completion(&rcu.completion);
260	264	/* Will wake me after RCU finished. */
261	265	call_rcu_bh(&rcu.head, wakeme_after_rcu);
262	266	/* Wait for it. */
263	267	wait_for_completion(&rcu.completion);
	268	+ destroy_rcu_head_on_stack(&rcu.head);
264	269	}
265	270	EXPORT_SYMBOL_GPL(rcu_barrier_bh);
266	271
@@ -268,11 +273,13 @@ void rcu_barrier_sched(void) · 1 unmodified line
268	273	{
269	274	struct rcu_synchronize rcu;
270	275
	276	+ init_rcu_head_on_stack(&rcu.head);
271	277	init_completion(&rcu.completion);
272	278	/* Will wake me after RCU finished. */
273	279	call_rcu_sched(&rcu.head, wakeme_after_rcu);
274	280	/* Wait for it. */
275	281	wait_for_completion(&rcu.completion);
	282	+ destroy_rcu_head_on_stack(&rcu.head);
276	283	}
277	284	EXPORT_SYMBOL_GPL(rcu_barrier_sched);
278	285
@@ -280,3 +287,5 @@ void __init rcu_init(void) · 1 unmodified line
280	287	{
281	288	open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
282	289	}
	290	+
	291	+#include "rcutiny_plugin.h"

kernel/rcutiny_plugin.h+39−0

@@ -0,0 +1,39 @@
	1	+/*
	2	+ * Read-Copy Update mechanism for mutual exclusion (tree-based version)
	3	+ * Internal non-public definitions that provide either classic
	4	+ * or preemptable semantics.
	5	+ *
	6	+ * This program is free software; you can redistribute it and/or modify
	7	+ * it under the terms of the GNU General Public License as published by
	8	+ * the Free Software Foundation; either version 2 of the License, or
	9	+ * (at your option) any later version.
	10	+ *
	11	+ * This program is distributed in the hope that it will be useful,
	12	+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	14	+ * GNU General Public License for more details.
	15	+ *
	16	+ * You should have received a copy of the GNU General Public License
	17	+ * along with this program; if not, write to the Free Software
	18	+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	19	+ *
	20	+ * Copyright IBM Corporation, 2009
	21	+ *
	22	+ * Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
	23	+ */
	24	+
	25	+#ifdef CONFIG_DEBUG_LOCK_ALLOC
	26	+
	27	+#include <linux/kernel_stat.h>
	28	+
	29	+/*
	30	+ * During boot, we forgive RCU lockdep issues. After this function is
	31	+ * invoked, we start taking RCU lockdep issues seriously.
	32	+ */
	33	+void rcu_scheduler_starting(void)
	34	+{
	35	+ WARN_ON(nr_context_switches() > 0);
	36	+ rcu_scheduler_active = 1;
	37	+}
	38	+
	39	+#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */

kernel/rcutorture.c+2−0

@@ -464,9 +464,11 @@ static void rcu_bh_torture_synchronize(void) · 463 unmodified lines
464	464	{
465	465	struct rcu_bh_torture_synchronize rcu;
466	466
	467	+ init_rcu_head_on_stack(&rcu.head);
467	468	init_completion(&rcu.completion);
468	469	call_rcu_bh(&rcu.head, rcu_bh_torture_wakeme_after_cb);
469	470	wait_for_completion(&rcu.completion);
	471	+ destroy_rcu_head_on_stack(&rcu.head);
470	472	}
471	473
472	474	static struct rcu_torture_ops rcu_bh_ops = {

kernel/rcutree.c+96−35

@@ -46,6 +46,7 @@ · 45 unmodified lines
46	46	#include <linux/cpu.h>
47	47	#include <linux/mutex.h>
48	48	#include <linux/time.h>
	49	+#include <linux/kernel_stat.h>
49	50
50	51	#include "rcutree.h"
51	52
@@ -53,8 +54,8 @@ · 1 unmodified line
53	54
54	55	static struct lock_class_key rcu_node_class[NUM_RCU_LVLS];
55	56
56		−#define RCU_STATE_INITIALIZER(name) { \
57		− .level = { &name.node[0] }, \
	57	+#define RCU_STATE_INITIALIZER(structname) { \
	58	+ .level = { &structname.node[0] }, \
58	59	.levelcnt = { \
59	60	NUM_RCU_LVL_0, /* root of hierarchy. */ \
60	61	NUM_RCU_LVL_1, \
@@ -65,13 +66,14 @@ static struct lock_class_key rcu_node_class[NUM_RCU_LVLS]; · 4 unmodified lines
65	66	.signaled = RCU_GP_IDLE, \
66	67	.gpnum = -300, \
67	68	.completed = -300, \
68		− .onofflock = __RAW_SPIN_LOCK_UNLOCKED(&name.onofflock), \
	69	+ .onofflock = __RAW_SPIN_LOCK_UNLOCKED(&structname.onofflock), \
69	70	.orphan_cbs_list = NULL, \
70		− .orphan_cbs_tail = &name.orphan_cbs_list, \
	71	+ .orphan_cbs_tail = &structname.orphan_cbs_list, \
71	72	.orphan_qlen = 0, \
72		− .fqslock = __RAW_SPIN_LOCK_UNLOCKED(&name.fqslock), \
	73	+ .fqslock = __RAW_SPIN_LOCK_UNLOCKED(&structname.fqslock), \
73	74	.n_force_qs = 0, \
74	75	.n_force_qs_ngp = 0, \
	76	+ .name = #structname, \
75	77	}
76	78
77	79	struct rcu_state rcu_sched_state = RCU_STATE_INITIALIZER(rcu_sched_state);
@@ -80,6 +82,9 @@ DEFINE_PER_CPU(struct rcu_data, rcu_sched_data); · 2 unmodified lines
80	82	struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state);
81	83	DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);
82	84
	85	+int rcu_scheduler_active __read_mostly;
	86	+EXPORT_SYMBOL_GPL(rcu_scheduler_active);
	87	+
83	88	/*
84	89	* Return true if an RCU grace period is in progress. The ACCESS_ONCE()s
85	90	* permit this function to be invoked without holding the root rcu_node
@@ -97,25 +102,32 @@ static int rcu_gp_in_progress(struct rcu_state *rsp) · 11 unmodified lines
97	102	*/
98	103	void rcu_sched_qs(int cpu)
99	104	{
100		− struct rcu_data *rdp;
	105	+ struct rcu_data *rdp = &per_cpu(rcu_sched_data, cpu);
101	106
102		− rdp = &per_cpu(rcu_sched_data, cpu);
103	107	rdp->passed_quiesc_completed = rdp->gpnum - 1;
104	108	barrier();
105	109	rdp->passed_quiesc = 1;
106		− rcu_preempt_note_context_switch(cpu);
107	110	}
108	111
109	112	void rcu_bh_qs(int cpu)
110	113	{
111		− struct rcu_data *rdp;
	114	+ struct rcu_data *rdp = &per_cpu(rcu_bh_data, cpu);
112	115
113		− rdp = &per_cpu(rcu_bh_data, cpu);
114	116	rdp->passed_quiesc_completed = rdp->gpnum - 1;
115	117	barrier();
116	118	rdp->passed_quiesc = 1;
117	119	}
118	120
	121	+/*
	122	+ * Note a context switch. This is a quiescent state for RCU-sched,
	123	+ * and requires special handling for preemptible RCU.
	124	+ */
	125	+void rcu_note_context_switch(int cpu)
	126	+{
	127	+ rcu_sched_qs(cpu);
	128	+ rcu_preempt_note_context_switch(cpu);
	129	+}
	130	+
119	131	#ifdef CONFIG_NO_HZ
120	132	DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
121	133	.dynticks_nesting = 1,
@@ -438,6 +450,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) · 316 unmodified lines
438	450
439	451	#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
440	452
	453	+int rcu_cpu_stall_panicking __read_mostly;
	454	+
441	455	static void record_gp_stall_check_time(struct rcu_state *rsp)
442	456	{
443	457	rsp->gp_start = jiffies;
@@ -470,7 +484,8 @@ static void print_other_cpu_stall(struct rcu_state *rsp) · 26 unmodified lines
470	484
471	485	/* OK, time to rat on our buddy... */
472	486
473		− printk(KERN_ERR "INFO: RCU detected CPU stalls:");
	487	+ printk(KERN_ERR "INFO: %s detected stalls on CPUs/tasks: {",
	488	+ rsp->name);
474	489	rcu_for_each_leaf_node(rsp, rnp) {
475	490	raw_spin_lock_irqsave(&rnp->lock, flags);
476	491	rcu_print_task_stall(rnp);
@@ -481,7 +496,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp) · 4 unmodified lines
481	496	if (rnp->qsmask & (1UL << cpu))
482	497	printk(" %d", rnp->grplo + cpu);
483	498	}
484		− printk(" (detected by %d, t=%ld jiffies)\n",
	499	+ printk("} (detected by %d, t=%ld jiffies)\n",
485	500	smp_processor_id(), (long)(jiffies - rsp->gp_start));
486	501	trigger_all_cpu_backtrace();
487	502
@@ -497,8 +512,8 @@ static void print_cpu_stall(struct rcu_state *rsp) · 9 unmodified lines
497	512	unsigned long flags;
498	513	struct rcu_node *rnp = rcu_get_root(rsp);
499	514
500		− printk(KERN_ERR "INFO: RCU detected CPU %d stall (t=%lu jiffies)\n",
501		− smp_processor_id(), jiffies - rsp->gp_start);
	515	+ printk(KERN_ERR "INFO: %s detected stall on CPU %d (t=%lu jiffies)\n",
	516	+ rsp->name, smp_processor_id(), jiffies - rsp->gp_start);
502	517	trigger_all_cpu_backtrace();
503	518
504	519	raw_spin_lock_irqsave(&rnp->lock, flags);
@@ -515,6 +530,8 @@ static void check_cpu_stall(struct rcu_state rsp, struct rcu_data rdp) · 10 unmodified lines
515	530	long delta;
516	531	struct rcu_node *rnp;
517	532
	533	+ if (rcu_cpu_stall_panicking)
	534	+ return;
518	535	delta = jiffies - rsp->jiffies_stall;
519	536	rnp = rdp->mynode;
520	537	if ((rnp->qsmask & rdp->grpmask) && delta >= 0) {
@@ -529,6 +546,21 @@ static void check_cpu_stall(struct rcu_state rsp, struct rcu_data rdp) · 8 unmodified lines
529	546	}
530	547	}
531	548
	549	+static int rcu_panic(struct notifier_block this, unsigned long ev, void ptr)
	550	+{
	551	+ rcu_cpu_stall_panicking = 1;
	552	+ return NOTIFY_DONE;
	553	+}
	554	+
	555	+static struct notifier_block rcu_panic_block = {
	556	+ .notifier_call = rcu_panic,
	557	+};
	558	+
	559	+static void __init check_cpu_stall_init(void)
	560	+{
	561	+ atomic_notifier_chain_register(&panic_notifier_list, &rcu_panic_block);
	562	+}
	563	+
532	564	#else /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
533	565
534	566	static void record_gp_stall_check_time(struct rcu_state *rsp)
@@ -539,6 +571,10 @@ static void check_cpu_stall(struct rcu_state rsp, struct rcu_data rdp) · 4 unmodified lines
539	571	{
540	572	}
541	573
	574	+static void __init check_cpu_stall_init(void)
	575	+{
	576	+}
	577	+
542	578	#endif /* #else #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
543	579
544	580	/*
@@ -1125,8 +1161,6 @@ static void rcu_do_batch(struct rcu_state rsp, struct rcu_data rdp) · 580 unmodified lines
1125	1161	*/
1126	1162	void rcu_check_callbacks(int cpu, int user)
1127	1163	{
1128		− if (!rcu_pending(cpu))
1129		− return; /* if nothing for RCU to do. */
1130	1164	if (user \|\|
1131	1165	(idle_cpu(cpu) && rcu_scheduler_active &&
1132	1166	!in_softirq() && hardirq_count() <= (1 << HARDIRQ_SHIFT))) {
@@ -1158,7 +1192,8 @@ void rcu_check_callbacks(int cpu, int user) · 25 unmodified lines
1158	1192	rcu_bh_qs(cpu);
1159	1193	}
1160	1194	rcu_preempt_check_callbacks(cpu);
1161		− raise_softirq(RCU_SOFTIRQ);
	1195	+ if (rcu_pending(cpu))
	1196	+ raise_softirq(RCU_SOFTIRQ);
1162	1197	}
1163	1198
1164	1199	#ifdef CONFIG_SMP
@@ -1236,11 +1271,11 @@ static void force_quiescent_state(struct rcu_state *rsp, int relaxed) · 71 unmodified lines
1236	1271	break; /* grace period idle or initializing, ignore. */
1237	1272
1238	1273	case RCU_SAVE_DYNTICK:
1239		−
1240		− raw_spin_unlock(&rnp->lock); /* irqs remain disabled */
1241	1274	if (RCU_SIGNAL_INIT != RCU_SAVE_DYNTICK)
1242	1275	break; /* So gcc recognizes the dead code. */
1243	1276
	1277	+ raw_spin_unlock(&rnp->lock); /* irqs remain disabled */
	1278	+
1244	1279	/* Record dyntick-idle state. */
1245	1280	force_qs_rnp(rsp, dyntick_save_progress_counter);
1246	1281	raw_spin_lock(&rnp->lock); /* irqs already disabled */
@@ -1449,11 +1484,13 @@ void synchronize_sched(void) · 202 unmodified lines
1449	1484	if (rcu_blocking_is_gp())
1450	1485	return;
1451	1486
	1487	+ init_rcu_head_on_stack(&rcu.head);
1452	1488	init_completion(&rcu.completion);
1453	1489	/* Will wake me after RCU finished. */
1454	1490	call_rcu_sched(&rcu.head, wakeme_after_rcu);
1455	1491	/* Wait for it. */
1456	1492	wait_for_completion(&rcu.completion);
	1493	+ destroy_rcu_head_on_stack(&rcu.head);
1457	1494	}
1458	1495	EXPORT_SYMBOL_GPL(synchronize_sched);
1459	1496
@@ -1473,11 +1510,13 @@ void synchronize_rcu_bh(void) · 13 unmodified lines
1473	1510	if (rcu_blocking_is_gp())
1474	1511	return;
1475	1512
	1513	+ init_rcu_head_on_stack(&rcu.head);
1476	1514	init_completion(&rcu.completion);
1477	1515	/* Will wake me after RCU finished. */
1478	1516	call_rcu_bh(&rcu.head, wakeme_after_rcu);
1479	1517	/* Wait for it. */
1480	1518	wait_for_completion(&rcu.completion);
	1519	+ destroy_rcu_head_on_stack(&rcu.head);
1481	1520	}
1482	1521	EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
1483	1522
@@ -1498,8 +1537,20 @@ static int __rcu_pending(struct rcu_state rsp, struct rcu_data rdp) · 14 unmodified lines
1498	1537	check_cpu_stall(rsp, rdp);
1499	1538
1500	1539	/* Is the RCU core waiting for a quiescent state from this CPU? */
1501		− if (rdp->qs_pending) {
	1540	+ if (rdp->qs_pending && !rdp->passed_quiesc) {
	1541	+
	1542	+ /*
	1543	+ * If force_quiescent_state() coming soon and this CPU
	1544	+ * needs a quiescent state, and this is either RCU-sched
	1545	+ * or RCU-bh, force a local reschedule.
	1546	+ */
1502	1547	rdp->n_rp_qs_pending++;
	1548	+ if (!rdp->preemptable &&
	1549	+ ULONG_CMP_LT(ACCESS_ONCE(rsp->jiffies_force_qs) - 1,
	1550	+ jiffies))
	1551	+ set_need_resched();
	1552	+ } else if (rdp->qs_pending && rdp->passed_quiesc) {
	1553	+ rdp->n_rp_report_qs++;
1503	1554	return 1;
1504	1555	}
1505	1556
@@ -1766,6 +1817,21 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self, · 260 unmodified lines
1766	1817	return NOTIFY_OK;
1767	1818	}
1768	1819
	1820	+/*
	1821	+ * This function is invoked towards the end of the scheduler's initialization
	1822	+ * process. Before this is called, the idle task might contain
	1823	+ * RCU read-side critical sections (during which time, this idle
	1824	+ * task is booting the system). After this function is called, the
	1825	+ * idle tasks are prohibited from containing RCU read-side critical
	1826	+ * sections. This function also enables RCU lockdep checking.
	1827	+ */
	1828	+void rcu_scheduler_starting(void)
	1829	+{
	1830	+ WARN_ON(num_online_cpus() != 1);
	1831	+ WARN_ON(nr_context_switches() > 0);
	1832	+ rcu_scheduler_active = 1;
	1833	+}
	1834	+
1769	1835	/*
1770	1836	* Compute the per-level fanout, either using the exact fanout specified
1771	1837	* or balancing the tree, depending on CONFIG_RCU_FANOUT_EXACT.
@@ -1849,6 +1915,14 @@ static void __init rcu_init_one(struct rcu_state *rsp) · 77 unmodified lines
1849	1915	INIT_LIST_HEAD(&rnp->blocked_tasks[3]);
1850	1916	}
1851	1917	}
	1918	+
	1919	+ rnp = rsp->level[NUM_RCU_LVLS - 1];
	1920	+ for_each_possible_cpu(i) {
	1921	+ while (i > rnp->grphi)
	1922	+ rnp++;
	1923	+ rsp->rda[i]->mynode = rnp;
	1924	+ rcu_boot_init_percpu_data(i, rsp);
	1925	+ }
1852	1926	}
1853	1927
1854	1928	/*
@@ -1859,19 +1933,11 @@ static void __init rcu_init_one(struct rcu_state *rsp) · 4 unmodified lines
1859	1933	#define RCU_INIT_FLAVOR(rsp, rcu_data) \
1860	1934	do { \
1861	1935	int i; \
1862		− int j; \
1863		− struct rcu_node *rnp; \
1864	1936	\
1865		− rcu_init_one(rsp); \
1866		− rnp = (rsp)->level[NUM_RCU_LVLS - 1]; \
1867		− j = 0; \
1868	1937	for_each_possible_cpu(i) { \
1869		− if (i > rnp[j].grphi) \
1870		− j++; \
1871		− per_cpu(rcu_data, i).mynode = &rnp[j]; \
1872	1938	(rsp)->rda[i] = &per_cpu(rcu_data, i); \
1873		− rcu_boot_init_percpu_data(i, rsp); \
1874	1939	} \
	1940	+ rcu_init_one(rsp); \
1875	1941	} while (0)
1876	1942
1877	1943	void __init rcu_init(void)
@@ -1879,12 +1945,6 @@ void __init rcu_init(void) · 1 unmodified line
1879	1945	int cpu;
1880	1946
1881	1947	rcu_bootup_announce();
1882		−#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
1883		− printk(KERN_INFO "RCU-based detection of stalled CPUs is enabled.\n");
1884		−#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
1885		−#if NUM_RCU_LVL_4 != 0
1886		− printk(KERN_INFO "Experimental four-level hierarchy is enabled.\n");
1887		−#endif /* #if NUM_RCU_LVL_4 != 0 */
1888	1948	RCU_INIT_FLAVOR(&rcu_sched_state, rcu_sched_data);
1889	1949	RCU_INIT_FLAVOR(&rcu_bh_state, rcu_bh_data);
1890	1950	__rcu_init_preempt();
@@ -1898,6 +1958,7 @@ void __init rcu_init(void) · 7 unmodified lines
1898	1958	cpu_notifier(rcu_cpu_notify, 0);
1899	1959	for_each_online_cpu(cpu)
1900	1960	rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)cpu);
	1961	+ check_cpu_stall_init();
1901	1962	}
1902	1963
1903	1964	#include "rcutree_plugin.h"

kernel/rcutree.h+2−0

@@ -223,6 +223,7 @@ struct rcu_data { · 222 unmodified lines
223	223	/* 5) __rcu_pending() statistics. */
224	224	unsigned long n_rcu_pending; /* rcu_pending() calls since boot. */
225	225	unsigned long n_rp_qs_pending;
	226	+ unsigned long n_rp_report_qs;
226	227	unsigned long n_rp_cb_ready;
227	228	unsigned long n_rp_cpu_needs_gp;
228	229	unsigned long n_rp_gp_completed;
@@ -326,6 +327,7 @@ struct rcu_state { · 97 unmodified lines
326	327	unsigned long jiffies_stall; /* Time at which to check */
327	328	/* for CPU stalls. */
328	329	#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
	330	+ char name; / Name of structure. */
329	331	};
330	332
331	333	/* Return values for rcu_preempt_offline_tasks(). */

kernel/rcutree_plugin.h+61−8

@@ -26,6 +26,45 @@ · 25 unmodified lines
26	26
27	27	#include <linux/delay.h>
28	28
	29	+/*
	30	+ * Check the RCU kernel configuration parameters and print informative
	31	+ * messages about anything out of the ordinary. If you like #ifdef, you
	32	+ * will love this function.
	33	+ */
	34	+static void __init rcu_bootup_announce_oddness(void)
	35	+{
	36	+#ifdef CONFIG_RCU_TRACE
	37	+ printk(KERN_INFO "\tRCU debugfs-based tracing is enabled.\n");
	38	+#endif
	39	+#if (defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 64) \|\| (!defined(CONFIG_64BIT) && CONFIG_RCU_FANOUT != 32)
	40	+ printk(KERN_INFO "\tCONFIG_RCU_FANOUT set to non-default value of %d\n",
	41	+ CONFIG_RCU_FANOUT);
	42	+#endif
	43	+#ifdef CONFIG_RCU_FANOUT_EXACT
	44	+ printk(KERN_INFO "\tHierarchical RCU autobalancing is disabled.\n");
	45	+#endif
	46	+#ifdef CONFIG_RCU_FAST_NO_HZ
	47	+ printk(KERN_INFO
	48	+ "\tRCU dyntick-idle grace-period acceleration is enabled.\n");
	49	+#endif
	50	+#ifdef CONFIG_PROVE_RCU
	51	+ printk(KERN_INFO "\tRCU lockdep checking is enabled.\n");
	52	+#endif
	53	+#ifdef CONFIG_RCU_TORTURE_TEST_RUNNABLE
	54	+ printk(KERN_INFO "\tRCU torture testing starts during boot.\n");
	55	+#endif
	56	+#ifndef CONFIG_RCU_CPU_STALL_DETECTOR
	57	+ printk(KERN_INFO
	58	+ "\tRCU-based detection of stalled CPUs is disabled.\n");
	59	+#endif
	60	+#ifndef CONFIG_RCU_CPU_STALL_VERBOSE
	61	+ printk(KERN_INFO "\tVerbose stalled-CPUs detection is disabled.\n");
	62	+#endif
	63	+#if NUM_RCU_LVL_4 != 0
	64	+ printk(KERN_INFO "\tExperimental four-level hierarchy is enabled.\n");
	65	+#endif
	66	+}
	67	+
29	68	#ifdef CONFIG_TREE_PREEMPT_RCU
30	69
31	70	struct rcu_state rcu_preempt_state = RCU_STATE_INITIALIZER(rcu_preempt_state);
@@ -38,8 +77,8 @@ static int rcu_preempted_readers_exp(struct rcu_node *rnp); · 6 unmodified lines
38	77	*/
39	78	static void __init rcu_bootup_announce(void)
40	79	{
41		− printk(KERN_INFO
42		− "Experimental preemptable hierarchical RCU implementation.\n");
	80	+ printk(KERN_INFO "Preemptable hierarchical RCU implementation.\n");
	81	+ rcu_bootup_announce_oddness();
43	82	}
44	83
45	84	/*
@@ -75,13 +114,19 @@ EXPORT_SYMBOL_GPL(rcu_force_quiescent_state); · 29 unmodified lines
75	114	* that this just means that the task currently running on the CPU is
76	115	* not in a quiescent state. There might be any number of tasks blocked
77	116	* while in an RCU read-side critical section.
	117	+ *
	118	+ * Unlike the other rcu_*_qs() functions, callers to this function
	119	+ * must disable irqs in order to protect the assignment to
	120	+ * ->rcu_read_unlock_special.
78	121	*/
79	122	static void rcu_preempt_qs(int cpu)
80	123	{
81	124	struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
	125	+
82	126	rdp->passed_quiesc_completed = rdp->gpnum - 1;
83	127	barrier();
84	128	rdp->passed_quiesc = 1;
	129	+ current->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
85	130	}
86	131
87	132	/*
@@ -144,9 +189,8 @@ static void rcu_preempt_note_context_switch(int cpu) · 56 unmodified lines
144	189	* grace period, then the fact that the task has been enqueued
145	190	* means that we continue to block the current grace period.
146	191	*/
147		− rcu_preempt_qs(cpu);
148	192	local_irq_save(flags);
149		− t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
	193	+ rcu_preempt_qs(cpu);
150	194	local_irq_restore(flags);
151	195	}
152	196
@@ -236,7 +280,6 @@ static void rcu_read_unlock_special(struct task_struct *t) · 83 unmodified lines
236	280	*/
237	281	special = t->rcu_read_unlock_special;
238	282	if (special & RCU_READ_UNLOCK_NEED_QS) {
239		− t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
240	283	rcu_preempt_qs(smp_processor_id());
241	284	}
242	285
@@ -473,7 +516,6 @@ static void rcu_preempt_check_callbacks(int cpu) · 230 unmodified lines
473	516	struct task_struct *t = current;
474	517
475	518	if (t->rcu_read_lock_nesting == 0) {
476		− t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_NEED_QS;
477	519	rcu_preempt_qs(cpu);
478	520	return;
479	521	}
@@ -515,11 +557,13 @@ void synchronize_rcu(void) · 35 unmodified lines
515	557	if (!rcu_scheduler_active)
516	558	return;
517	559
	560	+ init_rcu_head_on_stack(&rcu.head);
518	561	init_completion(&rcu.completion);
519	562	/* Will wake me after RCU finished. */
520	563	call_rcu(&rcu.head, wakeme_after_rcu);
521	564	/* Wait for it. */
522	565	wait_for_completion(&rcu.completion);
	566	+ destroy_rcu_head_on_stack(&rcu.head);
523	567	}
524	568	EXPORT_SYMBOL_GPL(synchronize_rcu);
525	569
@@ -754,6 +798,7 @@ void exit_rcu(void) · 228 unmodified lines
754	798	static void __init rcu_bootup_announce(void)
755	799	{
756	800	printk(KERN_INFO "Hierarchical RCU implementation.\n");
	801	+ rcu_bootup_announce_oddness();
757	802	}
758	803
759	804	/*
@@ -1008,6 +1053,8 @@ static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff); · 248 unmodified lines
1008	1053	int rcu_needs_cpu(int cpu)
1009	1054	{
1010	1055	int c = 0;
	1056	+ int snap;
	1057	+ int snap_nmi;
1011	1058	int thatcpu;
1012	1059
1013	1060	/* Check for being in the holdoff period. */
@@ -1015,12 +1062,18 @@ int rcu_needs_cpu(int cpu) · 1 unmodified line
1015	1062	return rcu_needs_cpu_quick_check(cpu);
1016	1063
1017	1064	/* Don't bother unless we are the last non-dyntick-idle CPU. */
1018		− for_each_cpu_not(thatcpu, nohz_cpu_mask)
1019		− if (thatcpu != cpu) {
	1065	+ for_each_online_cpu(thatcpu) {
	1066	+ if (thatcpu == cpu)
	1067	+ continue;
	1068	+ snap = per_cpu(rcu_dynticks, thatcpu).dynticks;
	1069	+ snap_nmi = per_cpu(rcu_dynticks, thatcpu).dynticks_nmi;
	1070	+ smp_mb(); /* Order sampling of snap with end of grace period. */
	1071	+ if (((snap & 0x1) != 0) \|\| ((snap_nmi & 0x1) != 0)) {
1020	1072	per_cpu(rcu_dyntick_drain, cpu) = 0;
1021	1073	per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1;
1022	1074	return rcu_needs_cpu_quick_check(cpu);
1023	1075	}
	1076	+ }
1024	1077
1025	1078	/* Check and update the rcu_dyntick_drain sequencing. */
1026	1079	if (per_cpu(rcu_dyntick_drain, cpu) <= 0) {

kernel/rcutree_trace.c+3−1

@@ -241,11 +241,13 @@ static const struct file_operations rcugp_fops = { · 240 unmodified lines
241	241	static void print_one_rcu_pending(struct seq_file m, struct rcu_data rdp)
242	242	{
243	243	seq_printf(m, "%3d%cnp=%ld "
244		− "qsp=%ld cbr=%ld cng=%ld gpc=%ld gps=%ld nf=%ld nn=%ld\n",
	244	+ "qsp=%ld rpq=%ld cbr=%ld cng=%ld "
	245	+ "gpc=%ld gps=%ld nf=%ld nn=%ld\n",
245	246	rdp->cpu,
246	247	cpu_is_offline(rdp->cpu) ? '!' : ' ',
247	248	rdp->n_rcu_pending,
248	249	rdp->n_rp_qs_pending,
	250	+ rdp->n_rp_report_qs,
249	251	rdp->n_rp_cb_ready,
250	252	rdp->n_rp_cpu_needs_gp,
251	253	rdp->n_rp_gp_completed,

kernel/sched.c+1−1

@@ -3706,7 +3706,7 @@ need_resched: · 3705 unmodified lines
3706	3706	preempt_disable();
3707	3707	cpu = smp_processor_id();
3708	3708	rq = cpu_rq(cpu);
3709		− rcu_sched_qs(cpu);
	3709	+ rcu_note_context_switch(cpu);
3710	3710	prev = rq->curr;
3711	3711	switch_count = &prev->nivcsw;
3712	3712

kernel/softirq.c+1−1

@@ -716,7 +716,7 @@ static int run_ksoftirqd(void * __bind_cpu) · 715 unmodified lines
716	716	preempt_enable_no_resched();
717	717	cond_resched();
718	718	preempt_disable();
719		− rcu_sched_qs((long)__bind_cpu);
	719	+ rcu_note_context_switch((long)__bind_cpu);
720	720	}
721	721	preempt_enable();
722	722	set_current_state(TASK_INTERRUPTIBLE);

Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code