History

Merge branch 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull RCU changes for v3.4 from Ingo Molnar. The major features of this series are: - making RCU more aggressive about entering dyntick-idle mode in order to improve energy efficiency - converting a few more call_rcu()s to kfree_rcu()s - applying a number of rcutree fixes and cleanups to rcutiny - removing CONFIG_SMP #ifdefs from treercu - allowing RCU CPU stall times to be set via sysfs - adding CPU-stall capability to rcutorture - adding more RCU-abuse diagnostics - updating documentation - fixing yet more issues located by the still-ongoing top-to-bottom inspection of RCU, this time with a special focus on the CPU-hotplug code path. * 'core-rcu-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (48 commits) rcu: Stop spurious warnings from synchronize_sched_expedited rcu: Hold off RCU_FAST_NO_HZ after timer posted rcu: Eliminate softirq-mediated RCU_FAST_NO_HZ idle-entry loop rcu: Add RCU_NONIDLE() for idle-loop RCU read-side critical sections rcu: Allow nesting of rcu_idle_enter() and rcu_idle_exit() rcu: Remove redundant check for rcu_head misalignment PTR_ERR should be called before its argument is cleared. rcu: Convert WARN_ON_ONCE() in rcu_lock_acquire() to lockdep rcu: Trace only after NULL-pointer check rcu: Call out dangers of expedited RCU primitives rcu: Rework detection of use of RCU by offline CPUs lockdep: Add CPU-idle/offline warning to lockdep-RCU splat rcu: No interrupt disabling for rcu_prepare_for_idle() rcu: Move synchronize_sched_expedited() to rcutree.c rcu: Check for illegal use of RCU from offlined CPUs rcu: Update stall-warning documentation rcu: Add CPU-stall capability to rcutorture rcu: Make documentation give more realistic rcutorture duration rcutorture: Permit holding off CPU-hotplug operations during boot rcu: Print scheduling-clock information on RCU CPU stall-warning messages ...

Linus Torvalds committed 14 years ago

Commit 5928a2b2 parents 5ed59af bdd443129 files · +2965 −642Browse files

Showing the changes to kernel (11 of 29 files). Show all

Unified · Split

11 files
  1. kernel/lockdep.c +7 −1
  2. kernel/rcu.h +23 −3
  3. kernel/rcupdate.c +5 −0
  4. kernel/rcutiny.c +20 −6
  5. kernel/rcutiny_plugin.h +60 −17
  6. kernel/rcutorture.c +86 −5
  7. kernel/rcutree.c +371 −136
  8. kernel/rcutree.h +16 −11
  9. kernel/rcutree_plugin.h +276 −174
  10. kernel/rcutree_trace.c +6 −6
  11. kernel/srcu.c +23 −10
kernel/lockdep.c+7−1
@@ -4176,7 +4176,13 @@ void lockdep_rcu_suspicious(const char *file, const int line, const char *s) · 4175 unmodified lines
41764176 printk("-------------------------------\n");
41774177 printk("%s:%d %s!\n", file, line, s);
41784178 printk("\nother info that might help us debug this:\n\n");
4179− printk("\nrcu_scheduler_active = %d, debug_locks = %d\n", rcu_scheduler_active, debug_locks);
4179+ printk("\n%srcu_scheduler_active = %d, debug_locks = %d\n",
4180+ !rcu_lockdep_current_cpu_online()
4181+ ? "RCU used illegally from offline CPU!\n"
4182+ : rcu_is_cpu_idle()
4183+ ? "RCU used illegally from idle CPU!\n"
4184+ : "",
4185+ rcu_scheduler_active, debug_locks);
41804186
41814187 /*
41824188 * If a CPU is in the RCU-free window in idle (ie: in the section
kernel/rcu.h+23−3
@@ -33,8 +33,27 @@ · 32 unmodified lines
3333 * Process-level increment to ->dynticks_nesting field. This allows for
3434 * architectures that use half-interrupts and half-exceptions from
3535 * process context.
36+ *
37+ * DYNTICK_TASK_NEST_MASK defines a field of width DYNTICK_TASK_NEST_WIDTH
38+ * that counts the number of process-based reasons why RCU cannot
39+ * consider the corresponding CPU to be idle, and DYNTICK_TASK_NEST_VALUE
40+ * is the value used to increment or decrement this field.
41+ *
42+ * The rest of the bits could in principle be used to count interrupts,
43+ * but this would mean that a negative-one value in the interrupt
44+ * field could incorrectly zero out the DYNTICK_TASK_NEST_MASK field.
45+ * We therefore provide a two-bit guard field defined by DYNTICK_TASK_MASK
46+ * that is set to DYNTICK_TASK_FLAG upon initial exit from idle.
47+ * The DYNTICK_TASK_EXIT_IDLE value is thus the combined value used upon
48+ * initial exit from idle.
3649 */
37−#define DYNTICK_TASK_NESTING (LLONG_MAX / 2 - 1)
50+#define DYNTICK_TASK_NEST_WIDTH 7
51+#define DYNTICK_TASK_NEST_VALUE ((LLONG_MAX >> DYNTICK_TASK_NEST_WIDTH) + 1)
52+#define DYNTICK_TASK_NEST_MASK (LLONG_MAX - DYNTICK_TASK_NEST_VALUE + 1)
53+#define DYNTICK_TASK_FLAG ((DYNTICK_TASK_NEST_VALUE / 8) * 2)
54+#define DYNTICK_TASK_MASK ((DYNTICK_TASK_NEST_VALUE / 8) * 3)
55+#define DYNTICK_TASK_EXIT_IDLE (DYNTICK_TASK_NEST_VALUE + \
56+ DYNTICK_TASK_FLAG)
3857
3958 /*
4059 * debug_rcu_head_queue()/debug_rcu_head_unqueue() are used internally
@@ -50,7 +69,6 @@ extern struct debug_obj_descr rcuhead_debug_descr; · 9 unmodified lines
5069
5170 static inline void debug_rcu_head_queue(struct rcu_head *head)
5271 {
53− WARN_ON_ONCE((unsigned long)head & 0x3);
5472 debug_object_activate(head, &rcuhead_debug_descr);
5573 debug_object_active_state(head, &rcuhead_debug_descr,
5674 STATE_RCU_HEAD_READY,
@@ -76,16 +94,18 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head) · 19 unmodified lines
7694
7795 extern void kfree(const void *);
7896
79−static inline void __rcu_reclaim(char *rn, struct rcu_head *head)
97+static inline bool __rcu_reclaim(char *rn, struct rcu_head *head)
8098 {
8199 unsigned long offset = (unsigned long)head->func;
82100
83101 if (__is_kfree_rcu_offset(offset)) {
84102 RCU_TRACE(trace_rcu_invoke_kfree_callback(rn, head, offset));
85103 kfree((void *)head - offset);
104+ return 1;
86105 } else {
87106 RCU_TRACE(trace_rcu_invoke_callback(rn, head));
88107 head->func(head);
108+ return 0;
89109 }
90110 }
91111
kernel/rcupdate.c+5−0
@@ -88,6 +88,9 @@ EXPORT_SYMBOL_GPL(debug_lockdep_rcu_enabled); · 87 unmodified lines
8888 * section.
8989 *
9090 * Check debug_lockdep_rcu_enabled() to prevent false positives during boot.
91+ *
92+ * Note that rcu_read_lock() is disallowed if the CPU is either idle or
93+ * offline from an RCU perspective, so check for those as well.
9194 */
9295 int rcu_read_lock_bh_held(void)
9396 {
@@ -95,6 +98,8 @@ int rcu_read_lock_bh_held(void) · 1 unmodified line
9598 return 1;
9699 if (rcu_is_cpu_idle())
97100 return 0;
101+ if (!rcu_lockdep_current_cpu_online())
102+ return 0;
98103 return in_softirq() || irqs_disabled();
99104 }
100105 EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held);
kernel/rcutiny.c+20−6
@@ -53,7 +53,7 @@ static void __call_rcu(struct rcu_head *head, · 52 unmodified lines
5353
5454 #include "rcutiny_plugin.h"
5555
56−static long long rcu_dynticks_nesting = DYNTICK_TASK_NESTING;
56+static long long rcu_dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
5757
5858 /* Common code for rcu_idle_enter() and rcu_irq_exit(), see kernel/rcutree.c. */
5959 static void rcu_idle_enter_common(long long oldval)
@@ -88,10 +88,16 @@ void rcu_idle_enter(void) · 28 unmodified lines
8888
8989 local_irq_save(flags);
9090 oldval = rcu_dynticks_nesting;
91− rcu_dynticks_nesting = 0;
91+ WARN_ON_ONCE((rcu_dynticks_nesting & DYNTICK_TASK_NEST_MASK) == 0);
92+ if ((rcu_dynticks_nesting & DYNTICK_TASK_NEST_MASK) ==
93+ DYNTICK_TASK_NEST_VALUE)
94+ rcu_dynticks_nesting = 0;
95+ else
96+ rcu_dynticks_nesting -= DYNTICK_TASK_NEST_VALUE;
9297 rcu_idle_enter_common(oldval);
9398 local_irq_restore(flags);
9499 }
100+EXPORT_SYMBOL_GPL(rcu_idle_enter);
95101
96102 /*
97103 * Exit an interrupt handler towards idle.
@@ -140,11 +146,15 @@ void rcu_idle_exit(void) · 42 unmodified lines
140146
141147 local_irq_save(flags);
142148 oldval = rcu_dynticks_nesting;
143− WARN_ON_ONCE(oldval != 0);
144− rcu_dynticks_nesting = DYNTICK_TASK_NESTING;
149+ WARN_ON_ONCE(rcu_dynticks_nesting < 0);
150+ if (rcu_dynticks_nesting & DYNTICK_TASK_NEST_MASK)
151+ rcu_dynticks_nesting += DYNTICK_TASK_NEST_VALUE;
152+ else
153+ rcu_dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
145154 rcu_idle_exit_common(oldval);
146155 local_irq_restore(flags);
147156 }
157+EXPORT_SYMBOL_GPL(rcu_idle_exit);
148158
149159 /*
150160 * Enter an interrupt handler, moving away from idle.
@@ -258,7 +268,7 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp) · 107 unmodified lines
258268
259269 /* If no RCU callbacks ready to invoke, just return. */
260270 if (&rcp->rcucblist == rcp->donetail) {
261− RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, -1));
271+ RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, 0, -1));
262272 RCU_TRACE(trace_rcu_batch_end(rcp->name, 0,
263273 ACCESS_ONCE(rcp->rcucblist),
264274 need_resched(),
@@ -269,7 +279,7 @@ static void __rcu_process_callbacks(struct rcu_ctrlblk *rcp) · 4 unmodified lines
269279
270280 /* Move the ready-to-invoke callbacks to a local list. */
271281 local_irq_save(flags);
272− RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, -1));
282+ RCU_TRACE(trace_rcu_batch_start(rcp->name, 0, rcp->qlen, -1));
273283 list = rcp->rcucblist;
274284 rcp->rcucblist = *rcp->donetail;
275285 *rcp->donetail = NULL;
@@ -319,6 +329,10 @@ static void rcu_process_callbacks(struct softirq_action *unused) · 43 unmodified lines
319329 */
320330 void synchronize_sched(void)
321331 {
332+ rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) &&
333+ !lock_is_held(&rcu_lock_map) &&
334+ !lock_is_held(&rcu_sched_lock_map),
335+ "Illegal synchronize_sched() in RCU read-side critical section");
322336 cond_resched();
323337 }
324338 EXPORT_SYMBOL_GPL(synchronize_sched);
kernel/rcutiny_plugin.h+60−17
@@ -132,6 +132,7 @@ static struct rcu_preempt_ctrlblk rcu_preempt_ctrlblk = { · 131 unmodified lines
132132 RCU_TRACE(.rcb.name = "rcu_preempt")
133133 };
134134
135+static void rcu_read_unlock_special(struct task_struct *t);
135136 static int rcu_preempted_readers_exp(void);
136137 static void rcu_report_exp_done(void);
137138
@@ -146,6 +147,16 @@ static int rcu_cpu_blocking_cur_gp(void) · 8 unmodified lines
146147 /*
147148 * Check for a running RCU reader. Because there is only one CPU,
148149 * there can be but one running RCU reader at a time. ;-)
150+ *
151+ * Returns zero if there are no running readers. Returns a positive
152+ * number if there is at least one reader within its RCU read-side
153+ * critical section. Returns a negative number if an outermost reader
154+ * is in the midst of exiting from its RCU read-side critical section
155+ *
156+ * Returns zero if there are no running readers. Returns a positive
157+ * number if there is at least one reader within its RCU read-side
158+ * critical section. Returns a negative number if an outermost reader
159+ * is in the midst of exiting from its RCU read-side critical section.
149160 */
150161 static int rcu_preempt_running_reader(void)
151162 {
@@ -307,7 +318,6 @@ static int rcu_boost(void) · 155 unmodified lines
307318 t = container_of(tb, struct task_struct, rcu_node_entry);
308319 rt_mutex_init_proxy_locked(&mtx, t);
309320 t->rcu_boost_mutex = &mtx;
310− t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BOOSTED;
311321 raw_local_irq_restore(flags);
312322 rt_mutex_lock(&mtx);
313323 rt_mutex_unlock(&mtx); /* Keep lockdep happy. */
@@ -475,7 +485,7 @@ void rcu_preempt_note_context_switch(void) · 161 unmodified lines
475485 unsigned long flags;
476486
477487 local_irq_save(flags); /* must exclude scheduler_tick(). */
478− if (rcu_preempt_running_reader() &&
488+ if (rcu_preempt_running_reader() > 0 &&
479489 (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
480490
481491 /* Possibly blocking in an RCU read-side critical section. */
@@ -494,6 +504,13 @@ void rcu_preempt_note_context_switch(void) · 12 unmodified lines
494504 list_add(&t->rcu_node_entry, &rcu_preempt_ctrlblk.blkd_tasks);
495505 if (rcu_cpu_blocking_cur_gp())
496506 rcu_preempt_ctrlblk.gp_tasks = &t->rcu_node_entry;
507+ } else if (rcu_preempt_running_reader() < 0 &&
508+ t->rcu_read_unlock_special) {
509+ /*
510+ * Complete exit from RCU read-side critical section on
511+ * behalf of preempted instance of __rcu_read_unlock().
512+ */
513+ rcu_read_unlock_special(t);
497514 }
498515
499516 /*
@@ -526,12 +543,15 @@ EXPORT_SYMBOL_GPL(__rcu_read_lock); · 26 unmodified lines
526543 * notify RCU core processing or task having blocked during the RCU
527544 * read-side critical section.
528545 */
529−static void rcu_read_unlock_special(struct task_struct *t)
546+static noinline void rcu_read_unlock_special(struct task_struct *t)
530547 {
531548 int empty;
532549 int empty_exp;
533550 unsigned long flags;
534551 struct list_head *np;
552+#ifdef CONFIG_RCU_BOOST
553+ struct rt_mutex *rbmp = NULL;
554+#endif /* #ifdef CONFIG_RCU_BOOST */
535555 int special;
536556
537557 /*
@@ -552,7 +572,7 @@ static void rcu_read_unlock_special(struct task_struct *t) · 14 unmodified lines
552572 rcu_preempt_cpu_qs();
553573
554574 /* Hardware IRQ handlers cannot block. */
555− if (in_irq()) {
575+ if (in_irq() || in_serving_softirq()) {
556576 local_irq_restore(flags);
557577 return;
558578 }
@@ -597,10 +617,10 @@ static void rcu_read_unlock_special(struct task_struct *t) · 38 unmodified lines
597617 }
598618 #ifdef CONFIG_RCU_BOOST
599619 /* Unboost self if was boosted. */
600− if (special & RCU_READ_UNLOCK_BOOSTED) {
601− t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BOOSTED;
602− rt_mutex_unlock(t->rcu_boost_mutex);
620+ if (t->rcu_boost_mutex != NULL) {
621+ rbmp = t->rcu_boost_mutex;
603622 t->rcu_boost_mutex = NULL;
623+ rt_mutex_unlock(rbmp);
604624 }
605625 #endif /* #ifdef CONFIG_RCU_BOOST */
606626 local_irq_restore(flags);
@@ -618,13 +638,22 @@ void __rcu_read_unlock(void) · 11 unmodified lines
618638 struct task_struct *t = current;
619639
620640 barrier(); /* needed if we ever invoke rcu_read_unlock in rcutiny.c */
621− --t->rcu_read_lock_nesting;
622− barrier(); /* decrement before load of ->rcu_read_unlock_special */
623− if (t->rcu_read_lock_nesting == 0 &&
624− unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
625− rcu_read_unlock_special(t);
641+ if (t->rcu_read_lock_nesting != 1)
642+ --t->rcu_read_lock_nesting;
643+ else {
644+ t->rcu_read_lock_nesting = INT_MIN;
645+ barrier(); /* assign before ->rcu_read_unlock_special load */
646+ if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
647+ rcu_read_unlock_special(t);
648+ barrier(); /* ->rcu_read_unlock_special load before assign */
649+ t->rcu_read_lock_nesting = 0;
650+ }
626651 #ifdef CONFIG_PROVE_LOCKING
627− WARN_ON_ONCE(t->rcu_read_lock_nesting < 0);
652+ {
653+ int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
654+
655+ WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
656+ }
628657 #endif /* #ifdef CONFIG_PROVE_LOCKING */
629658 }
630659 EXPORT_SYMBOL_GPL(__rcu_read_unlock);
@@ -649,7 +678,7 @@ static void rcu_preempt_check_callbacks(void) · 18 unmodified lines
649678 invoke_rcu_callbacks();
650679 if (rcu_preempt_gp_in_progress() &&
651680 rcu_cpu_blocking_cur_gp() &&
652− rcu_preempt_running_reader())
681+ rcu_preempt_running_reader() > 0)
653682 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
654683 }
655684
@@ -706,6 +735,11 @@ EXPORT_SYMBOL_GPL(call_rcu); · 50 unmodified lines
706735 */
707736 void synchronize_rcu(void)
708737 {
738+ rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) &&
739+ !lock_is_held(&rcu_lock_map) &&
740+ !lock_is_held(&rcu_sched_lock_map),
741+ "Illegal synchronize_rcu() in RCU read-side critical section");
742+
709743 #ifdef CONFIG_DEBUG_LOCK_ALLOC
710744 if (!rcu_scheduler_active)
711745 return;
@@ -882,7 +916,8 @@ static void rcu_preempt_process_callbacks(void) · 170 unmodified lines
882916 static void invoke_rcu_callbacks(void)
883917 {
884918 have_rcu_kthread_work = 1;
885− wake_up(&rcu_kthread_wq);
919+ if (rcu_kthread_task != NULL)
920+ wake_up(&rcu_kthread_wq);
886921 }
887922
888923 #ifdef CONFIG_RCU_TRACE
@@ -943,12 +978,16 @@ early_initcall(rcu_spawn_kthreads); · 54 unmodified lines
943978
944979 #else /* #ifdef CONFIG_RCU_BOOST */
945980
981+/* Hold off callback invocation until early_initcall() time. */
982+static int rcu_scheduler_fully_active __read_mostly;
983+
946984 /*
947985 * Start up softirq processing of callbacks.
948986 */
949987 void invoke_rcu_callbacks(void)
950988 {
951− raise_softirq(RCU_SOFTIRQ);
989+ if (rcu_scheduler_fully_active)
990+ raise_softirq(RCU_SOFTIRQ);
952991 }
953992
954993 #ifdef CONFIG_RCU_TRACE
@@ -963,10 +1002,14 @@ static bool rcu_is_callbacks_kthread(void) · 8 unmodified lines
9631002
9641003 #endif /* #ifdef CONFIG_RCU_TRACE */
9651004
966−void rcu_init(void)
1005+static int __init rcu_scheduler_really_started(void)
9671006 {
1007+ rcu_scheduler_fully_active = 1;
9681008 open_softirq(RCU_SOFTIRQ, rcu_process_callbacks);
1009+ raise_softirq(RCU_SOFTIRQ); /* Invoke any callbacks from early boot. */
1010+ return 0;
9691011 }
1012+early_initcall(rcu_scheduler_really_started);
9701013
9711014 #endif /* #else #ifdef CONFIG_RCU_BOOST */
9721015
kernel/rcutorture.c+86−5
@@ -65,7 +65,10 @@ static int fqs_duration; /* Duration of bursts (us), 0 to disable. */ · 64 unmodified lines
6565 static int fqs_holdoff; /* Hold time within burst (us). */
6666 static int fqs_stutter = 3; /* Wait time between bursts (s). */
6767 static int onoff_interval; /* Wait time between CPU hotplugs, 0=disable. */
68+static int onoff_holdoff; /* Seconds after boot before CPU hotplugs. */
6869 static int shutdown_secs; /* Shutdown time (s). <=0 for no shutdown. */
70+static int stall_cpu; /* CPU-stall duration (s). 0 for no stall. */
71+static int stall_cpu_holdoff = 10; /* Time to wait until stall (s). */
6972 static int test_boost = 1; /* Test RCU prio boost: 0=no, 1=maybe, 2=yes. */
7073 static int test_boost_interval = 7; /* Interval between boost tests, seconds. */
7174 static int test_boost_duration = 4; /* Duration of each boost test, seconds. */
@@ -95,8 +98,14 @@ module_param(fqs_stutter, int, 0444); · 23 unmodified lines
9598 MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");
9699 module_param(onoff_interval, int, 0444);
97100 MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable");
101+module_param(onoff_holdoff, int, 0444);
102+MODULE_PARM_DESC(onoff_holdoff, "Time after boot before CPU hotplugs (s)");
98103 module_param(shutdown_secs, int, 0444);
99104 MODULE_PARM_DESC(shutdown_secs, "Shutdown time (s), zero to disable.");
105+module_param(stall_cpu, int, 0444);
106+MODULE_PARM_DESC(stall_cpu, "Stall duration (s), zero to disable.");
107+module_param(stall_cpu_holdoff, int, 0444);
108+MODULE_PARM_DESC(stall_cpu_holdoff, "Time to wait before starting stall (s).");
100109 module_param(test_boost, int, 0444);
101110 MODULE_PARM_DESC(test_boost, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");
102111 module_param(test_boost_interval, int, 0444);
@@ -129,6 +138,7 @@ static struct task_struct *shutdown_task; · 26 unmodified lines
129138 #ifdef CONFIG_HOTPLUG_CPU
130139 static struct task_struct *onoff_task;
131140 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
141+static struct task_struct *stall_task;
132142
133143 #define RCU_TORTURE_PIPE_LEN 10
134144
@@ -990,12 +1000,12 @@ static void rcu_torture_timer(unsigned long unused) · 855 unmodified lines
9901000 rcu_read_lock_bh_held() ||
9911001 rcu_read_lock_sched_held() ||
9921002 srcu_read_lock_held(&srcu_ctl));
993− do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu);
9941003 if (p == NULL) {
9951004 /* Leave because rcu_torture_writer is not yet underway */
9961005 cur_ops->readunlock(idx);
9971006 return;
9981007 }
1008+ do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu);
9991009 if (p->rtort_mbtest == 0)
10001010 atomic_inc(&n_rcu_torture_mberror);
10011011 spin_lock(&rand_lock);
@@ -1053,13 +1063,13 @@ rcu_torture_reader(void *arg) · 51 unmodified lines
10531063 rcu_read_lock_bh_held() ||
10541064 rcu_read_lock_sched_held() ||
10551065 srcu_read_lock_held(&srcu_ctl));
1056− do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu);
10571066 if (p == NULL) {
10581067 /* Wait for rcu_torture_writer to get underway */
10591068 cur_ops->readunlock(idx);
10601069 schedule_timeout_interruptible(HZ);
10611070 continue;
10621071 }
1072+ do_trace_rcu_torture_read(cur_ops->name, &p->rtort_rcu);
10631073 if (p->rtort_mbtest == 0)
10641074 atomic_inc(&n_rcu_torture_mberror);
10651075 cur_ops->read_delay(&rand);
@@ -1300,13 +1310,13 @@ rcu_torture_print_module_parms(struct rcu_torture_ops *cur_ops, char *tag) · 234 unmodified lines
13001310 "fqs_duration=%d fqs_holdoff=%d fqs_stutter=%d "
13011311 "test_boost=%d/%d test_boost_interval=%d "
13021312 "test_boost_duration=%d shutdown_secs=%d "
1303− "onoff_interval=%d\n",
1313+ "onoff_interval=%d onoff_holdoff=%d\n",
13041314 torture_type, tag, nrealreaders, nfakewriters,
13051315 stat_interval, verbose, test_no_idle_hz, shuffle_interval,
13061316 stutter, irqreader, fqs_duration, fqs_holdoff, fqs_stutter,
13071317 test_boost, cur_ops->can_boost,
13081318 test_boost_interval, test_boost_duration, shutdown_secs,
1309− onoff_interval);
1319+ onoff_interval, onoff_holdoff);
13101320 }
13111321
13121322 static struct notifier_block rcutorture_shutdown_nb = {
@@ -1410,6 +1420,11 @@ rcu_torture_onoff(void *arg) · 97 unmodified lines
14101420 for_each_online_cpu(cpu)
14111421 maxcpu = cpu;
14121422 WARN_ON(maxcpu < 0);
1423+ if (onoff_holdoff > 0) {
1424+ VERBOSE_PRINTK_STRING("rcu_torture_onoff begin holdoff");
1425+ schedule_timeout_interruptible(onoff_holdoff * HZ);
1426+ VERBOSE_PRINTK_STRING("rcu_torture_onoff end holdoff");
1427+ }
14131428 while (!kthread_should_stop()) {
14141429 cpu = (rcu_random(&rand) >> 4) % (maxcpu + 1);
14151430 if (cpu_online(cpu) && cpu_is_hotpluggable(cpu)) {
@@ -1450,12 +1465,15 @@ rcu_torture_onoff(void *arg) · 34 unmodified lines
14501465 static int __cpuinit
14511466 rcu_torture_onoff_init(void)
14521467 {
1468+ int ret;
1469+
14531470 if (onoff_interval <= 0)
14541471 return 0;
14551472 onoff_task = kthread_run(rcu_torture_onoff, NULL, "rcu_torture_onoff");
14561473 if (IS_ERR(onoff_task)) {
1474+ ret = PTR_ERR(onoff_task);
14571475 onoff_task = NULL;
1458− return PTR_ERR(onoff_task);
1476+ return ret;
14591477 }
14601478 return 0;
14611479 }
@@ -1481,6 +1499,63 @@ static void rcu_torture_onoff_cleanup(void) · 19 unmodified lines
14811499
14821500 #endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
14831501
1502+/*
1503+ * CPU-stall kthread. It waits as specified by stall_cpu_holdoff, then
1504+ * induces a CPU stall for the time specified by stall_cpu.
1505+ */
1506+static int __cpuinit rcu_torture_stall(void *args)
1507+{
1508+ unsigned long stop_at;
1509+
1510+ VERBOSE_PRINTK_STRING("rcu_torture_stall task started");
1511+ if (stall_cpu_holdoff > 0) {
1512+ VERBOSE_PRINTK_STRING("rcu_torture_stall begin holdoff");
1513+ schedule_timeout_interruptible(stall_cpu_holdoff * HZ);
1514+ VERBOSE_PRINTK_STRING("rcu_torture_stall end holdoff");
1515+ }
1516+ if (!kthread_should_stop()) {
1517+ stop_at = get_seconds() + stall_cpu;
1518+ /* RCU CPU stall is expected behavior in following code. */
1519+ printk(KERN_ALERT "rcu_torture_stall start.\n");
1520+ rcu_read_lock();
1521+ preempt_disable();
1522+ while (ULONG_CMP_LT(get_seconds(), stop_at))
1523+ continue; /* Induce RCU CPU stall warning. */
1524+ preempt_enable();
1525+ rcu_read_unlock();
1526+ printk(KERN_ALERT "rcu_torture_stall end.\n");
1527+ }
1528+ rcutorture_shutdown_absorb("rcu_torture_stall");
1529+ while (!kthread_should_stop())
1530+ schedule_timeout_interruptible(10 * HZ);
1531+ return 0;
1532+}
1533+
1534+/* Spawn CPU-stall kthread, if stall_cpu specified. */
1535+static int __init rcu_torture_stall_init(void)
1536+{
1537+ int ret;
1538+
1539+ if (stall_cpu <= 0)
1540+ return 0;
1541+ stall_task = kthread_run(rcu_torture_stall, NULL, "rcu_torture_stall");
1542+ if (IS_ERR(stall_task)) {
1543+ ret = PTR_ERR(stall_task);
1544+ stall_task = NULL;
1545+ return ret;
1546+ }
1547+ return 0;
1548+}
1549+
1550+/* Clean up after the CPU-stall kthread, if one was spawned. */
1551+static void rcu_torture_stall_cleanup(void)
1552+{
1553+ if (stall_task == NULL)
1554+ return;
1555+ VERBOSE_PRINTK_STRING("Stopping rcu_torture_stall_task.");
1556+ kthread_stop(stall_task);
1557+}
1558+
14841559 static int rcutorture_cpu_notify(struct notifier_block *self,
14851560 unsigned long action, void *hcpu)
14861561 {
@@ -1523,6 +1598,7 @@ rcu_torture_cleanup(void) · 36 unmodified lines
15231598 fullstop = FULLSTOP_RMMOD;
15241599 mutex_unlock(&fullstop_mutex);
15251600 unregister_reboot_notifier(&rcutorture_shutdown_nb);
1601+ rcu_torture_stall_cleanup();
15261602 if (stutter_task) {
15271603 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stutter task");
15281604 kthread_stop(stutter_task);
@@ -1602,6 +1678,10 @@ rcu_torture_cleanup(void) · 73 unmodified lines
16021678 cur_ops->cleanup();
16031679 if (atomic_read(&n_rcu_torture_error))
16041680 rcu_torture_print_module_parms(cur_ops, "End of test: FAILURE");
1681+ else if (n_online_successes != n_online_attempts ||
1682+ n_offline_successes != n_offline_attempts)
1683+ rcu_torture_print_module_parms(cur_ops,
1684+ "End of test: RCU_HOTPLUG");
16051685 else
16061686 rcu_torture_print_module_parms(cur_ops, "End of test: SUCCESS");
16071687 }
@@ -1819,6 +1899,7 @@ rcu_torture_init(void) · 211 unmodified lines
18191899 }
18201900 rcu_torture_onoff_init();
18211901 register_reboot_notifier(&rcutorture_shutdown_nb);
1902+ rcu_torture_stall_init();
18221903 rcutorture_record_test_transition();
18231904 mutex_unlock(&fullstop_mutex);
18241905 return 0;
kernel/rcutree.c+371−136
@@ -50,6 +50,8 @@ · 49 unmodified lines
5050 #include <linux/wait.h>
5151 #include <linux/kthread.h>
5252 #include <linux/prefetch.h>
53+#include <linux/delay.h>
54+#include <linux/stop_machine.h>
5355
5456 #include "rcutree.h"
5557 #include <trace/events/rcu.h>
@@ -196,7 +198,7 @@ void rcu_note_context_switch(int cpu) · 140 unmodified lines
196198 EXPORT_SYMBOL_GPL(rcu_note_context_switch);
197199
198200 DEFINE_PER_CPU(struct rcu_dynticks, rcu_dynticks) = {
199− .dynticks_nesting = DYNTICK_TASK_NESTING,
201+ .dynticks_nesting = DYNTICK_TASK_EXIT_IDLE,
200202 .dynticks = ATOMIC_INIT(1),
201203 };
202204
@@ -208,8 +210,11 @@ module_param(blimit, int, 0); · 5 unmodified lines
208210 module_param(qhimark, int, 0);
209211 module_param(qlowmark, int, 0);
210212
211−int rcu_cpu_stall_suppress __read_mostly;
213+int rcu_cpu_stall_suppress __read_mostly; /* 1 = suppress stall warnings. */
214+int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
215+
212216 module_param(rcu_cpu_stall_suppress, int, 0644);
217+module_param(rcu_cpu_stall_timeout, int, 0644);
213218
214219 static void force_quiescent_state(struct rcu_state *rsp, int relaxed);
215220 static int rcu_pending(int cpu);
@@ -301,8 +306,6 @@ static struct rcu_node *rcu_get_root(struct rcu_state *rsp) · 85 unmodified lines
301306 return &rsp->node[0];
302307 }
303308
304−#ifdef CONFIG_SMP
305−
306309 /*
307310 * If the specified CPU is offline, tell the caller that it is in
308311 * a quiescent state. Otherwise, whack it with a reschedule IPI.
@@ -317,30 +320,21 @@ static struct rcu_node *rcu_get_root(struct rcu_state *rsp) · 8 unmodified lines
317320 static int rcu_implicit_offline_qs(struct rcu_data *rdp)
318321 {
319322 /*
320− * If the CPU is offline, it is in a quiescent state. We can
321− * trust its state not to change because interrupts are disabled.
323+ * If the CPU is offline for more than a jiffy, it is in a quiescent
324+ * state. We can trust its state not to change because interrupts
325+ * are disabled. The reason for the jiffy's worth of slack is to
326+ * handle CPUs initializing on the way up and finding their way
327+ * to the idle loop on the way down.
322328 */
323− if (cpu_is_offline(rdp->cpu)) {
329+ if (cpu_is_offline(rdp->cpu) &&
330+ ULONG_CMP_LT(rdp->rsp->gp_start + 2, jiffies)) {
324331 trace_rcu_fqs(rdp->rsp->name, rdp->gpnum, rdp->cpu, "ofl");
325332 rdp->offline_fqs++;
326333 return 1;
327334 }
328−
329− /*
330− * The CPU is online, so send it a reschedule IPI. This forces
331− * it through the scheduler, and (inefficiently) also handles cases
332− * where idle loops fail to inform RCU about the CPU being idle.
333− */
334− if (rdp->cpu != smp_processor_id())
335− smp_send_reschedule(rdp->cpu);
336− else
337− set_need_resched();
338− rdp->resched_ipi++;
339335 return 0;
340336 }
341337
342−#endif /* #ifdef CONFIG_SMP */
343−
344338 /*
345339 * rcu_idle_enter_common - inform RCU that current CPU is moving towards idle
346340 *
@@ -366,6 +360,17 @@ static void rcu_idle_enter_common(struct rcu_dynticks *rdtp, long long oldval) · 19 unmodified lines
366360 atomic_inc(&rdtp->dynticks);
367361 smp_mb__after_atomic_inc(); /* Force ordering with next sojourn. */
368362 WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
363+
364+ /*
365+ * The idle task is not permitted to enter the idle loop while
366+ * in an RCU read-side critical section.
367+ */
368+ rcu_lockdep_assert(!lock_is_held(&rcu_lock_map),
369+ "Illegal idle entry in RCU read-side critical section.");
370+ rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map),
371+ "Illegal idle entry in RCU-bh read-side critical section.");
372+ rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map),
373+ "Illegal idle entry in RCU-sched read-side critical section.");
369374 }
370375
371376 /**
@@ -389,10 +394,15 @@ void rcu_idle_enter(void) · 17 unmodified lines
389394 local_irq_save(flags);
390395 rdtp = &__get_cpu_var(rcu_dynticks);
391396 oldval = rdtp->dynticks_nesting;
392− rdtp->dynticks_nesting = 0;
397+ WARN_ON_ONCE((oldval & DYNTICK_TASK_NEST_MASK) == 0);
398+ if ((oldval & DYNTICK_TASK_NEST_MASK) == DYNTICK_TASK_NEST_VALUE)
399+ rdtp->dynticks_nesting = 0;
400+ else
401+ rdtp->dynticks_nesting -= DYNTICK_TASK_NEST_VALUE;
393402 rcu_idle_enter_common(rdtp, oldval);
394403 local_irq_restore(flags);
395404 }
405+EXPORT_SYMBOL_GPL(rcu_idle_enter);
396406
397407 /**
398408 * rcu_irq_exit - inform RCU that current CPU is exiting irq towards idle
@@ -462,7 +472,7 @@ static void rcu_idle_exit_common(struct rcu_dynticks *rdtp, long long oldval) · 63 unmodified lines
462472 * Exit idle mode, in other words, -enter- the mode in which RCU
463473 * read-side critical sections can occur.
464474 *
465− * We crowbar the ->dynticks_nesting field to DYNTICK_TASK_NESTING to
475+ * We crowbar the ->dynticks_nesting field to DYNTICK_TASK_NEST to
466476 * allow for the possibility of usermode upcalls messing up our count
467477 * of interrupt nesting level during the busy period that is just
468478 * now starting.
@@ -476,11 +486,15 @@ void rcu_idle_exit(void) · 7 unmodified lines
476486 local_irq_save(flags);
477487 rdtp = &__get_cpu_var(rcu_dynticks);
478488 oldval = rdtp->dynticks_nesting;
479− WARN_ON_ONCE(oldval != 0);
480− rdtp->dynticks_nesting = DYNTICK_TASK_NESTING;
489+ WARN_ON_ONCE(oldval < 0);
490+ if (oldval & DYNTICK_TASK_NEST_MASK)
491+ rdtp->dynticks_nesting += DYNTICK_TASK_NEST_VALUE;
492+ else
493+ rdtp->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
481494 rcu_idle_exit_common(rdtp, oldval);
482495 local_irq_restore(flags);
483496 }
497+EXPORT_SYMBOL_GPL(rcu_idle_exit);
484498
485499 /**
486500 * rcu_irq_enter - inform RCU that current CPU is entering irq away from idle
@@ -581,6 +595,49 @@ int rcu_is_cpu_idle(void) · 94 unmodified lines
581595 }
582596 EXPORT_SYMBOL(rcu_is_cpu_idle);
583597
598+#ifdef CONFIG_HOTPLUG_CPU
599+
600+/*
601+ * Is the current CPU online? Disable preemption to avoid false positives
602+ * that could otherwise happen due to the current CPU number being sampled,
603+ * this task being preempted, its old CPU being taken offline, resuming
604+ * on some other CPU, then determining that its old CPU is now offline.
605+ * It is OK to use RCU on an offline processor during initial boot, hence
606+ * the check for rcu_scheduler_fully_active. Note also that it is OK
607+ * for a CPU coming online to use RCU for one jiffy prior to marking itself
608+ * online in the cpu_online_mask. Similarly, it is OK for a CPU going
609+ * offline to continue to use RCU for one jiffy after marking itself
610+ * offline in the cpu_online_mask. This leniency is necessary given the
611+ * non-atomic nature of the online and offline processing, for example,
612+ * the fact that a CPU enters the scheduler after completing the CPU_DYING
613+ * notifiers.
614+ *
615+ * This is also why RCU internally marks CPUs online during the
616+ * CPU_UP_PREPARE phase and offline during the CPU_DEAD phase.
617+ *
618+ * Disable checking if in an NMI handler because we cannot safely report
619+ * errors from NMI handlers anyway.
620+ */
621+bool rcu_lockdep_current_cpu_online(void)
622+{
623+ struct rcu_data *rdp;
624+ struct rcu_node *rnp;
625+ bool ret;
626+
627+ if (in_nmi())
628+ return 1;
629+ preempt_disable();
630+ rdp = &__get_cpu_var(rcu_sched_data);
631+ rnp = rdp->mynode;
632+ ret = (rdp->grpmask & rnp->qsmaskinit) ||
633+ !rcu_scheduler_fully_active;
634+ preempt_enable();
635+ return ret;
636+}
637+EXPORT_SYMBOL_GPL(rcu_lockdep_current_cpu_online);
638+
639+#endif /* #ifdef CONFIG_HOTPLUG_CPU */
640+
584641 #endif /* #ifdef CONFIG_PROVE_RCU */
585642
586643 /**
@@ -595,8 +652,6 @@ int rcu_is_cpu_rrupt_from_idle(void) · 8 unmodified lines
595652 return __get_cpu_var(rcu_dynticks).dynticks_nesting <= 1;
596653 }
597654
598−#ifdef CONFIG_SMP
599−
600655 /*
601656 * Snapshot the specified CPU's dynticks counter so that we can later
602657 * credit them with an implicit quiescent state. Return 1 if this CPU
@@ -640,12 +695,28 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) · 37 unmodified lines
640695 return rcu_implicit_offline_qs(rdp);
641696 }
642697
643−#endif /* #ifdef CONFIG_SMP */
698+static int jiffies_till_stall_check(void)
699+{
700+ int till_stall_check = ACCESS_ONCE(rcu_cpu_stall_timeout);
701+
702+ /*
703+ * Limit check must be consistent with the Kconfig limits
704+ * for CONFIG_RCU_CPU_STALL_TIMEOUT.
705+ */
706+ if (till_stall_check < 3) {
707+ ACCESS_ONCE(rcu_cpu_stall_timeout) = 3;
708+ till_stall_check = 3;
709+ } else if (till_stall_check > 300) {
710+ ACCESS_ONCE(rcu_cpu_stall_timeout) = 300;
711+ till_stall_check = 300;
712+ }
713+ return till_stall_check * HZ + RCU_STALL_DELAY_DELTA;
714+}
644715
645716 static void record_gp_stall_check_time(struct rcu_state *rsp)
646717 {
647718 rsp->gp_start = jiffies;
648− rsp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_CHECK;
719+ rsp->jiffies_stall = jiffies + jiffies_till_stall_check();
649720 }
650721
651722 static void print_other_cpu_stall(struct rcu_state *rsp)
@@ -664,13 +735,7 @@ static void print_other_cpu_stall(struct rcu_state *rsp) · 12 unmodified lines
664735 raw_spin_unlock_irqrestore(&rnp->lock, flags);
665736 return;
666737 }
667− rsp->jiffies_stall = jiffies + RCU_SECONDS_TILL_STALL_RECHECK;
668−
669− /*
670− * Now rat on any tasks that got kicked up to the root rcu_node
671− * due to CPU offlining.
672− */
673− ndetected = rcu_print_task_stall(rnp);
738+ rsp->jiffies_stall = jiffies + 3 * jiffies_till_stall_check() + 3;
674739 raw_spin_unlock_irqrestore(&rnp->lock, flags);
675740
676741 /*
@@ -678,8 +743,9 @@ static void print_other_cpu_stall(struct rcu_state *rsp) · 1 unmodified line
678743 * See Documentation/RCU/stallwarn.txt for info on how to debug
679744 * RCU CPU stall warnings.
680745 */
681− printk(KERN_ERR "INFO: %s detected stalls on CPUs/tasks: {",
746+ printk(KERN_ERR "INFO: %s detected stalls on CPUs/tasks:",
682747 rsp->name);
748+ print_cpu_stall_info_begin();
683749 rcu_for_each_leaf_node(rsp, rnp) {
684750 raw_spin_lock_irqsave(&rnp->lock, flags);
685751 ndetected += rcu_print_task_stall(rnp);
@@ -688,11 +754,22 @@ static void print_other_cpu_stall(struct rcu_state *rsp) · 2 unmodified lines
688754 continue;
689755 for (cpu = 0; cpu <= rnp->grphi - rnp->grplo; cpu++)
690756 if (rnp->qsmask & (1UL << cpu)) {
691− printk(" %d", rnp->grplo + cpu);
757+ print_cpu_stall_info(rsp, rnp->grplo + cpu);
692758 ndetected++;
693759 }
694760 }
695− printk("} (detected by %d, t=%ld jiffies)\n",
761+
762+ /*
763+ * Now rat on any tasks that got kicked up to the root rcu_node
764+ * due to CPU offlining.
765+ */
766+ rnp = rcu_get_root(rsp);
767+ raw_spin_lock_irqsave(&rnp->lock, flags);
768+ ndetected = rcu_print_task_stall(rnp);
769+ raw_spin_unlock_irqrestore(&rnp->lock, flags);
770+
771+ print_cpu_stall_info_end();
772+ printk(KERN_CONT "(detected by %d, t=%ld jiffies)\n",
696773 smp_processor_id(), (long)(jiffies - rsp->gp_start));
697774 if (ndetected == 0)
698775 printk(KERN_ERR "INFO: Stall ended before state dump start\n");
@@ -716,15 +793,18 @@ static void print_cpu_stall(struct rcu_state *rsp) · 17 unmodified lines
716793 * See Documentation/RCU/stallwarn.txt for info on how to debug
717794 * RCU CPU stall warnings.
718795 */
719− printk(KERN_ERR "INFO: %s detected stall on CPU %d (t=%lu jiffies)\n",
720− rsp->name, smp_processor_id(), jiffies - rsp->gp_start);
796+ printk(KERN_ERR "INFO: %s self-detected stall on CPU", rsp->name);
797+ print_cpu_stall_info_begin();
798+ print_cpu_stall_info(rsp, smp_processor_id());
799+ print_cpu_stall_info_end();
800+ printk(KERN_CONT " (t=%lu jiffies)\n", jiffies - rsp->gp_start);
721801 if (!trigger_all_cpu_backtrace())
722802 dump_stack();
723803
724804 raw_spin_lock_irqsave(&rnp->lock, flags);
725805 if (ULONG_CMP_GE(jiffies, rsp->jiffies_stall))
726− rsp->jiffies_stall =
727− jiffies + RCU_SECONDS_TILL_STALL_RECHECK;
806+ rsp->jiffies_stall = jiffies +
807+ 3 * jiffies_till_stall_check() + 3;
728808 raw_spin_unlock_irqrestore(&rnp->lock, flags);
729809
730810 set_need_resched(); /* kick ourselves to get things going. */
@@ -807,6 +887,7 @@ static void __note_new_gpnum(struct rcu_state *rsp, struct rcu_node *rnp, struct · 76 unmodified lines
807887 rdp->passed_quiesce = 0;
808888 } else
809889 rdp->qs_pending = 0;
890+ zero_cpu_stall_ticks(rdp);
810891 }
811892 }
812893
@@ -943,6 +1024,10 @@ rcu_start_gp_per_cpu(struct rcu_state *rsp, struct rcu_node *rnp, struct rcu_dat · 130 unmodified lines
9431024 * in preparation for detecting the next grace period. The caller must hold
9441025 * the root node's ->lock, which is released before return. Hard irqs must
9451026 * be disabled.
1027+ *
1028+ * Note that it is legal for a dying CPU (which is marked as offline) to
1029+ * invoke this function. This can happen when the dying CPU reports its
1030+ * quiescent state.
9461031 */
9471032 static void
9481033 rcu_start_gp(struct rcu_state *rsp, unsigned long flags)
@@ -980,26 +1065,8 @@ rcu_start_gp(struct rcu_state *rsp, unsigned long flags) · 31 unmodified lines
9801065 rsp->fqs_state = RCU_GP_INIT; /* Hold off force_quiescent_state. */
9811066 rsp->jiffies_force_qs = jiffies + RCU_JIFFIES_TILL_FORCE_QS;
9821067 record_gp_stall_check_time(rsp);
983−
984− /* Special-case the common single-level case. */
985− if (NUM_RCU_NODES == 1) {
986− rcu_preempt_check_blocked_tasks(rnp);
987− rnp->qsmask = rnp->qsmaskinit;
988− rnp->gpnum = rsp->gpnum;
989− rnp->completed = rsp->completed;
990− rsp->fqs_state = RCU_SIGNAL_INIT; /* force_quiescent_state OK */
991− rcu_start_gp_per_cpu(rsp, rnp, rdp);
992− rcu_preempt_boost_start_gp(rnp);
993− trace_rcu_grace_period_init(rsp->name, rnp->gpnum,
994− rnp->level, rnp->grplo,
995− rnp->grphi, rnp->qsmask);
996− raw_spin_unlock_irqrestore(&rnp->lock, flags);
997− return;
998− }
999−
10001068 raw_spin_unlock(&rnp->lock); /* leave irqs disabled. */
10011069
1002−
10031070 /* Exclude any concurrent CPU-hotplug operations. */
10041071 raw_spin_lock(&rsp->onofflock); /* irqs already disabled. */
10051072
@@ -1245,53 +1312,115 @@ rcu_check_quiescent_state(struct rcu_state *rsp, struct rcu_data *rdp) · 239 unmodified lines
12451312
12461313 /*
12471314 * Move a dying CPU's RCU callbacks to online CPU's callback list.
1248− * Synchronization is not required because this function executes
1249− * in stop_machine() context.
1315+ * Also record a quiescent state for this CPU for the current grace period.
1316+ * Synchronization and interrupt disabling are not required because
1317+ * this function executes in stop_machine() context. Therefore, cleanup
1318+ * operations that might block must be done later from the CPU_DEAD
1319+ * notifier.
1320+ *
1321+ * Note that the outgoing CPU's bit has already been cleared in the
1322+ * cpu_online_mask. This allows us to randomly pick a callback
1323+ * destination from the bits set in that mask.
12501324 */
1251−static void rcu_send_cbs_to_online(struct rcu_state *rsp)
1325+static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
12521326 {
12531327 int i;
1254− /* current DYING CPU is cleared in the cpu_online_mask */
1328+ unsigned long mask;
12551329 int receive_cpu = cpumask_any(cpu_online_mask);
12561330 struct rcu_data *rdp = this_cpu_ptr(rsp->rda);
12571331 struct rcu_data *receive_rdp = per_cpu_ptr(rsp->rda, receive_cpu);
1332+ RCU_TRACE(struct rcu_node *rnp = rdp->mynode); /* For dying CPU. */
1333+
1334+ /* First, adjust the counts. */
1335+ if (rdp->nxtlist != NULL) {
1336+ receive_rdp->qlen_lazy += rdp->qlen_lazy;
1337+ receive_rdp->qlen += rdp->qlen;
1338+ rdp->qlen_lazy = 0;
1339+ rdp->qlen = 0;
1340+ }
12581341
1259− if (rdp->nxtlist == NULL)
1260− return; /* irqs disabled, so comparison is stable. */
1342+ /*
1343+ * Next, move ready-to-invoke callbacks to be invoked on some
1344+ * other CPU. These will not be required to pass through another
1345+ * grace period: They are done, regardless of CPU.
1346+ */
1347+ if (rdp->nxtlist != NULL &&
1348+ rdp->nxttail[RCU_DONE_TAIL] != &rdp->nxtlist) {
1349+ struct rcu_head *oldhead;
1350+ struct rcu_head **oldtail;
1351+ struct rcu_head **newtail;
1352+
1353+ oldhead = rdp->nxtlist;
1354+ oldtail = receive_rdp->nxttail[RCU_DONE_TAIL];
1355+ rdp->nxtlist = *rdp->nxttail[RCU_DONE_TAIL];
1356+ *rdp->nxttail[RCU_DONE_TAIL] = *oldtail;
1357+ *receive_rdp->nxttail[RCU_DONE_TAIL] = oldhead;
1358+ newtail = rdp->nxttail[RCU_DONE_TAIL];
1359+ for (i = RCU_DONE_TAIL; i < RCU_NEXT_SIZE; i++) {
1360+ if (receive_rdp->nxttail[i] == oldtail)
1361+ receive_rdp->nxttail[i] = newtail;
1362+ if (rdp->nxttail[i] == newtail)
1363+ rdp->nxttail[i] = &rdp->nxtlist;
1364+ }
1365+ }
12611366
1262− *receive_rdp->nxttail[RCU_NEXT_TAIL] = rdp->nxtlist;
1263− receive_rdp->nxttail[RCU_NEXT_TAIL] = rdp->nxttail[RCU_NEXT_TAIL];
1264− receive_rdp->qlen += rdp->qlen;
1265− receive_rdp->n_cbs_adopted += rdp->qlen;
1266− rdp->n_cbs_orphaned += rdp->qlen;
1367+ /*
1368+ * Finally, put the rest of the callbacks at the end of the list.
1369+ * The ones that made it partway through get to start over: We
1370+ * cannot assume that grace periods are synchronized across CPUs.
1371+ * (We could splice RCU_WAIT_TAIL into RCU_NEXT_READY_TAIL, but
1372+ * this does not seem compelling. Not yet, anyway.)
1373+ */
1374+ if (rdp->nxtlist != NULL) {
1375+ *receive_rdp->nxttail[RCU_NEXT_TAIL] = rdp->nxtlist;
1376+ receive_rdp->nxttail[RCU_NEXT_TAIL] =
1377+ rdp->nxttail[RCU_NEXT_TAIL];
1378+ receive_rdp->n_cbs_adopted += rdp->qlen;
1379+ rdp->n_cbs_orphaned += rdp->qlen;
1380+
1381+ rdp->nxtlist = NULL;
1382+ for (i = 0; i < RCU_NEXT_SIZE; i++)
1383+ rdp->nxttail[i] = &rdp->nxtlist;
1384+ }
12671385
1268− rdp->nxtlist = NULL;
1269− for (i = 0; i < RCU_NEXT_SIZE; i++)
1270− rdp->nxttail[i] = &rdp->nxtlist;
1271− rdp->qlen = 0;
1386+ /*
1387+ * Record a quiescent state for the dying CPU. This is safe
1388+ * only because we have already cleared out the callbacks.
1389+ * (Otherwise, the RCU core might try to schedule the invocation
1390+ * of callbacks on this now-offline CPU, which would be bad.)
1391+ */
1392+ mask = rdp->grpmask; /* rnp->grplo is constant. */
1393+ trace_rcu_grace_period(rsp->name,
1394+ rnp->gpnum + 1 - !!(rnp->qsmask & mask),
1395+ "cpuofl");
1396+ rcu_report_qs_rdp(smp_processor_id(), rsp, rdp, rsp->gpnum);
1397+ /* Note that rcu_report_qs_rdp() might call trace_rcu_grace_period(). */
12721398 }
12731399
12741400 /*
1275− * Remove the outgoing CPU from the bitmasks in the rcu_node hierarchy
1276− * and move all callbacks from the outgoing CPU to the current one.
1401+ * The CPU has been completely removed, and some other CPU is reporting
1402+ * this fact from process context. Do the remainder of the cleanup.
12771403 * There can only be one CPU hotplug operation at a time, so no other
12781404 * CPU can be attempting to update rcu_cpu_kthread_task.
12791405 */
1280−static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp)
1406+static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
12811407 {
12821408 unsigned long flags;
12831409 unsigned long mask;
12841410 int need_report = 0;
12851411 struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
1286− struct rcu_node *rnp;
1412+ struct rcu_node *rnp = rdp->mynode; /* Outgoing CPU's rnp. */
12871413
1414+ /* Adjust any no-longer-needed kthreads. */
12881415 rcu_stop_cpu_kthread(cpu);
1416+ rcu_node_kthread_setaffinity(rnp, -1);
1417+
1418+ /* Remove the dying CPU from the bitmasks in the rcu_node hierarchy. */
12891419
12901420 /* Exclude any attempts to start a new grace period. */
12911421 raw_spin_lock_irqsave(&rsp->onofflock, flags);
12921422
12931423 /* Remove the outgoing CPU from the masks in the rcu_node hierarchy. */
1294− rnp = rdp->mynode; /* this is the outgoing CPU's rnp. */
12951424 mask = rdp->grpmask; /* rnp->grplo is constant. */
12961425 do {
12971426 raw_spin_lock(&rnp->lock); /* irqs already disabled. */
@@ -1299,20 +1428,11 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) · 1 unmodified line
12991428 if (rnp->qsmaskinit != 0) {
13001429 if (rnp != rdp->mynode)
13011430 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
1302− else
1303− trace_rcu_grace_period(rsp->name,
1304− rnp->gpnum + 1 -
1305− !!(rnp->qsmask & mask),
1306− "cpuofl");
13071431 break;
13081432 }
1309− if (rnp == rdp->mynode) {
1310− trace_rcu_grace_period(rsp->name,
1311− rnp->gpnum + 1 -
1312− !!(rnp->qsmask & mask),
1313− "cpuofl");
1433+ if (rnp == rdp->mynode)
13141434 need_report = rcu_preempt_offline_tasks(rsp, rnp, rdp);
1315− } else
1435+ else
13161436 raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
13171437 mask = rnp->grpmask;
13181438 rnp = rnp->parent;
@@ -1332,29 +1452,15 @@ static void __rcu_offline_cpu(int cpu, struct rcu_state *rsp) · 13 unmodified lines
13321452 raw_spin_unlock_irqrestore(&rnp->lock, flags);
13331453 if (need_report & RCU_OFL_TASKS_EXP_GP)
13341454 rcu_report_exp_rnp(rsp, rnp, true);
1335− rcu_node_kthread_setaffinity(rnp, -1);
1336−}
1337−
1338−/*
1339− * Remove the specified CPU from the RCU hierarchy and move any pending
1340− * callbacks that it might have to the current CPU. This code assumes
1341− * that at least one CPU in the system will remain running at all times.
1342− * Any attempt to offline -all- CPUs is likely to strand RCU callbacks.
1343− */
1344−static void rcu_offline_cpu(int cpu)
1345−{
1346− __rcu_offline_cpu(cpu, &rcu_sched_state);
1347− __rcu_offline_cpu(cpu, &rcu_bh_state);
1348− rcu_preempt_offline_cpu(cpu);
13491455 }
13501456
13511457 #else /* #ifdef CONFIG_HOTPLUG_CPU */
13521458
1353−static void rcu_send_cbs_to_online(struct rcu_state *rsp)
1459+static void rcu_cleanup_dying_cpu(struct rcu_state *rsp)
13541460 {
13551461 }
13561462
1357−static void rcu_offline_cpu(int cpu)
1463+static void rcu_cleanup_dead_cpu(int cpu, struct rcu_state *rsp)
13581464 {
13591465 }
13601466
@@ -1368,11 +1474,11 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) · 7 unmodified lines
13681474 {
13691475 unsigned long flags;
13701476 struct rcu_head *next, *list, **tail;
1371− int bl, count;
1477+ int bl, count, count_lazy;
13721478
13731479 /* If no callbacks are ready, just return.*/
13741480 if (!cpu_has_callbacks_ready_to_invoke(rdp)) {
1375− trace_rcu_batch_start(rsp->name, 0, 0);
1481+ trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, 0);
13761482 trace_rcu_batch_end(rsp->name, 0, !!ACCESS_ONCE(rdp->nxtlist),
13771483 need_resched(), is_idle_task(current),
13781484 rcu_is_callbacks_kthread());
@@ -1384,8 +1490,9 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) · 5 unmodified lines
13841490 * races with call_rcu() from interrupt handlers.
13851491 */
13861492 local_irq_save(flags);
1493+ WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
13871494 bl = rdp->blimit;
1388− trace_rcu_batch_start(rsp->name, rdp->qlen, bl);
1495+ trace_rcu_batch_start(rsp->name, rdp->qlen_lazy, rdp->qlen, bl);
13891496 list = rdp->nxtlist;
13901497 rdp->nxtlist = *rdp->nxttail[RCU_DONE_TAIL];
13911498 *rdp->nxttail[RCU_DONE_TAIL] = NULL;
@@ -1396,12 +1503,13 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) · 4 unmodified lines
13961503 local_irq_restore(flags);
13971504
13981505 /* Invoke callbacks. */
1399− count = 0;
1506+ count = count_lazy = 0;
14001507 while (list) {
14011508 next = list->next;
14021509 prefetch(next);
14031510 debug_rcu_head_unqueue(list);
1404− __rcu_reclaim(rsp->name, list);
1511+ if (__rcu_reclaim(rsp->name, list))
1512+ count_lazy++;
14051513 list = next;
14061514 /* Stop only if limit reached and CPU has something to do. */
14071515 if (++count >= bl &&
@@ -1416,6 +1524,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) · 8 unmodified lines
14161524 rcu_is_callbacks_kthread());
14171525
14181526 /* Update count, and requeue any remaining callbacks. */
1527+ rdp->qlen_lazy -= count_lazy;
14191528 rdp->qlen -= count;
14201529 rdp->n_cbs_invoked += count;
14211530 if (list != NULL) {
@@ -1458,6 +1567,7 @@ static void rcu_do_batch(struct rcu_state *rsp, struct rcu_data *rdp) · 36 unmodified lines
14581567 void rcu_check_callbacks(int cpu, int user)
14591568 {
14601569 trace_rcu_utilization("Start scheduler-tick");
1570+ increment_cpu_stall_ticks();
14611571 if (user || rcu_is_cpu_rrupt_from_idle()) {
14621572
14631573 /*
@@ -1492,8 +1602,6 @@ void rcu_check_callbacks(int cpu, int user) · 28 unmodified lines
14921602 trace_rcu_utilization("End scheduler-tick");
14931603 }
14941604
1495−#ifdef CONFIG_SMP
1496−
14971605 /*
14981606 * Scan the leaf rcu_node structures, processing dyntick state for any that
14991607 * have not yet encountered a quiescent state, using the function specified.
@@ -1616,15 +1724,6 @@ unlock_fqs_ret: · 116 unmodified lines
16161724 trace_rcu_utilization("End fqs");
16171725 }
16181726
1619−#else /* #ifdef CONFIG_SMP */
1620−
1621−static void force_quiescent_state(struct rcu_state *rsp, int relaxed)
1622−{
1623− set_need_resched();
1624−}
1625−
1626−#endif /* #else #ifdef CONFIG_SMP */
1627−
16281727 /*
16291728 * This does the RCU core processing work for the specified rcu_state
16301729 * and rcu_data structures. This may be called only from the CPU to
@@ -1702,11 +1801,12 @@ static void invoke_rcu_core(void) · 71 unmodified lines
17021801
17031802 static void
17041803 __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
1705− struct rcu_state *rsp)
1804+ struct rcu_state *rsp, bool lazy)
17061805 {
17071806 unsigned long flags;
17081807 struct rcu_data *rdp;
17091808
1809+ WARN_ON_ONCE((unsigned long)head & 0x3); /* Misaligned rcu_head! */
17101810 debug_rcu_head_queue(head);
17111811 head->func = func;
17121812 head->next = NULL;
@@ -1720,18 +1820,21 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), · 7 unmodified lines
17201820 * a quiescent state betweentimes.
17211821 */
17221822 local_irq_save(flags);
1823+ WARN_ON_ONCE(cpu_is_offline(smp_processor_id()));
17231824 rdp = this_cpu_ptr(rsp->rda);
17241825
17251826 /* Add the callback to our list. */
17261827 *rdp->nxttail[RCU_NEXT_TAIL] = head;
17271828 rdp->nxttail[RCU_NEXT_TAIL] = &head->next;
17281829 rdp->qlen++;
1830+ if (lazy)
1831+ rdp->qlen_lazy++;
17291832
17301833 if (__is_kfree_rcu_offset((unsigned long)func))
17311834 trace_rcu_kfree_callback(rsp->name, head, (unsigned long)func,
1732− rdp->qlen);
1835+ rdp->qlen_lazy, rdp->qlen);
17331836 else
1734− trace_rcu_callback(rsp->name, head, rdp->qlen);
1837+ trace_rcu_callback(rsp->name, head, rdp->qlen_lazy, rdp->qlen);
17351838
17361839 /* If interrupts were disabled, don't dive into RCU core. */
17371840 if (irqs_disabled_flags(flags)) {
@@ -1778,16 +1881,16 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu), · 40 unmodified lines
17781881 */
17791882 void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
17801883 {
1781− __call_rcu(head, func, &rcu_sched_state);
1884+ __call_rcu(head, func, &rcu_sched_state, 0);
17821885 }
17831886 EXPORT_SYMBOL_GPL(call_rcu_sched);
17841887
17851888 /*
1786− * Queue an RCU for invocation after a quicker grace period.
1889+ * Queue an RCU callback for invocation after a quicker grace period.
17871890 */
17881891 void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
17891892 {
1790− __call_rcu(head, func, &rcu_bh_state);
1893+ __call_rcu(head, func, &rcu_bh_state, 0);
17911894 }
17921895 EXPORT_SYMBOL_GPL(call_rcu_bh);
17931896
@@ -1816,6 +1919,10 @@ EXPORT_SYMBOL_GPL(call_rcu_bh); · 22 unmodified lines
18161919 */
18171920 void synchronize_sched(void)
18181921 {
1922+ rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) &&
1923+ !lock_is_held(&rcu_lock_map) &&
1924+ !lock_is_held(&rcu_sched_lock_map),
1925+ "Illegal synchronize_sched() in RCU-sched read-side critical section");
18191926 if (rcu_blocking_is_gp())
18201927 return;
18211928 wait_rcu_gp(call_rcu_sched);
@@ -1833,12 +1940,137 @@ EXPORT_SYMBOL_GPL(synchronize_sched); · 11 unmodified lines
18331940 */
18341941 void synchronize_rcu_bh(void)
18351942 {
1943+ rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) &&
1944+ !lock_is_held(&rcu_lock_map) &&
1945+ !lock_is_held(&rcu_sched_lock_map),
1946+ "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
18361947 if (rcu_blocking_is_gp())
18371948 return;
18381949 wait_rcu_gp(call_rcu_bh);
18391950 }
18401951 EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
18411952
1953+static atomic_t sync_sched_expedited_started = ATOMIC_INIT(0);
1954+static atomic_t sync_sched_expedited_done = ATOMIC_INIT(0);
1955+
1956+static int synchronize_sched_expedited_cpu_stop(void *data)
1957+{
1958+ /*
1959+ * There must be a full memory barrier on each affected CPU
1960+ * between the time that try_stop_cpus() is called and the
1961+ * time that it returns.
1962+ *
1963+ * In the current initial implementation of cpu_stop, the
1964+ * above condition is already met when the control reaches
1965+ * this point and the following smp_mb() is not strictly
1966+ * necessary. Do smp_mb() anyway for documentation and
1967+ * robustness against future implementation changes.
1968+ */
1969+ smp_mb(); /* See above comment block. */
1970+ return 0;
1971+}
1972+
1973+/**
1974+ * synchronize_sched_expedited - Brute-force RCU-sched grace period
1975+ *
1976+ * Wait for an RCU-sched grace period to elapse, but use a "big hammer"
1977+ * approach to force the grace period to end quickly. This consumes
1978+ * significant time on all CPUs and is unfriendly to real-time workloads,
1979+ * so is thus not recommended for any sort of common-case code. In fact,
1980+ * if you are using synchronize_sched_expedited() in a loop, please
1981+ * restructure your code to batch your updates, and then use a single
1982+ * synchronize_sched() instead.
1983+ *
1984+ * Note that it is illegal to call this function while holding any lock
1985+ * that is acquired by a CPU-hotplug notifier. And yes, it is also illegal
1986+ * to call this function from a CPU-hotplug notifier. Failing to observe
1987+ * these restriction will result in deadlock.
1988+ *
1989+ * This implementation can be thought of as an application of ticket
1990+ * locking to RCU, with sync_sched_expedited_started and
1991+ * sync_sched_expedited_done taking on the roles of the halves
1992+ * of the ticket-lock word. Each task atomically increments
1993+ * sync_sched_expedited_started upon entry, snapshotting the old value,
1994+ * then attempts to stop all the CPUs. If this succeeds, then each
1995+ * CPU will have executed a context switch, resulting in an RCU-sched
1996+ * grace period. We are then done, so we use atomic_cmpxchg() to
1997+ * update sync_sched_expedited_done to match our snapshot -- but
1998+ * only if someone else has not already advanced past our snapshot.
1999+ *
2000+ * On the other hand, if try_stop_cpus() fails, we check the value
2001+ * of sync_sched_expedited_done. If it has advanced past our
2002+ * initial snapshot, then someone else must have forced a grace period
2003+ * some time after we took our snapshot. In this case, our work is
2004+ * done for us, and we can simply return. Otherwise, we try again,
2005+ * but keep our initial snapshot for purposes of checking for someone
2006+ * doing our work for us.
2007+ *
2008+ * If we fail too many times in a row, we fall back to synchronize_sched().
2009+ */
2010+void synchronize_sched_expedited(void)
2011+{
2012+ int firstsnap, s, snap, trycount = 0;
2013+
2014+ /* Note that atomic_inc_return() implies full memory barrier. */
2015+ firstsnap = snap = atomic_inc_return(&sync_sched_expedited_started);
2016+ get_online_cpus();
2017+ WARN_ON_ONCE(cpu_is_offline(raw_smp_processor_id()));
2018+
2019+ /*
2020+ * Each pass through the following loop attempts to force a
2021+ * context switch on each CPU.
2022+ */
2023+ while (try_stop_cpus(cpu_online_mask,
2024+ synchronize_sched_expedited_cpu_stop,
2025+ NULL) == -EAGAIN) {
2026+ put_online_cpus();
2027+
2028+ /* No joy, try again later. Or just synchronize_sched(). */
2029+ if (trycount++ < 10)
2030+ udelay(trycount * num_online_cpus());
2031+ else {
2032+ synchronize_sched();
2033+ return;
2034+ }
2035+
2036+ /* Check to see if someone else did our work for us. */
2037+ s = atomic_read(&sync_sched_expedited_done);
2038+ if (UINT_CMP_GE((unsigned)s, (unsigned)firstsnap)) {
2039+ smp_mb(); /* ensure test happens before caller kfree */
2040+ return;
2041+ }
2042+
2043+ /*
2044+ * Refetching sync_sched_expedited_started allows later
2045+ * callers to piggyback on our grace period. We subtract
2046+ * 1 to get the same token that the last incrementer got.
2047+ * We retry after they started, so our grace period works
2048+ * for them, and they started after our first try, so their
2049+ * grace period works for us.
2050+ */
2051+ get_online_cpus();
2052+ snap = atomic_read(&sync_sched_expedited_started);
2053+ smp_mb(); /* ensure read is before try_stop_cpus(). */
2054+ }
2055+
2056+ /*
2057+ * Everyone up to our most recent fetch is covered by our grace
2058+ * period. Update the counter, but only if our work is still
2059+ * relevant -- which it won't be if someone who started later
2060+ * than we did beat us to the punch.
2061+ */
2062+ do {
2063+ s = atomic_read(&sync_sched_expedited_done);
2064+ if (UINT_CMP_GE((unsigned)s, (unsigned)snap)) {
2065+ smp_mb(); /* ensure test happens before caller kfree */
2066+ break;
2067+ }
2068+ } while (atomic_cmpxchg(&sync_sched_expedited_done, s, snap) != s);
2069+
2070+ put_online_cpus();
2071+}
2072+EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
2073+
18422074 /*
18432075 * Check to see if there is any immediate RCU-related work to be done
18442076 * by the current CPU, for the specified type of RCU, returning 1 if so.
@@ -1932,7 +2164,7 @@ static int rcu_cpu_has_callbacks(int cpu) · 87 unmodified lines
19322164 /* RCU callbacks either ready or pending? */
19332165 return per_cpu(rcu_sched_data, cpu).nxtlist ||
19342166 per_cpu(rcu_bh_data, cpu).nxtlist ||
1935− rcu_preempt_needs_cpu(cpu);
2167+ rcu_preempt_cpu_has_callbacks(cpu);
19362168 }
19372169
19382170 static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL};
@@ -2027,9 +2259,10 @@ rcu_boot_init_percpu_data(int cpu, struct rcu_state *rsp) · 88 unmodified lines
20272259 rdp->nxtlist = NULL;
20282260 for (i = 0; i < RCU_NEXT_SIZE; i++)
20292261 rdp->nxttail[i] = &rdp->nxtlist;
2262+ rdp->qlen_lazy = 0;
20302263 rdp->qlen = 0;
20312264 rdp->dynticks = &per_cpu(rcu_dynticks, cpu);
2032− WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_NESTING);
2265+ WARN_ON_ONCE(rdp->dynticks->dynticks_nesting != DYNTICK_TASK_EXIT_IDLE);
20332266 WARN_ON_ONCE(atomic_read(&rdp->dynticks->dynticks) != 1);
20342267 rdp->cpu = cpu;
20352268 rdp->rsp = rsp;
@@ -2057,7 +2290,7 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp, int preemptible) · 21 unmodified lines
20572290 rdp->qlen_last_fqs_check = 0;
20582291 rdp->n_force_qs_snap = rsp->n_force_qs;
20592292 rdp->blimit = blimit;
2060− rdp->dynticks->dynticks_nesting = DYNTICK_TASK_NESTING;
2293+ rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
20612294 atomic_set(&rdp->dynticks->dynticks,
20622295 (atomic_read(&rdp->dynticks->dynticks) & ~0x1) + 1);
20632296 rcu_prepare_for_idle_init(cpu);
@@ -2139,16 +2372,18 @@ static int __cpuinit rcu_cpu_notify(struct notifier_block *self, · 75 unmodified lines
21392372 * touch any data without introducing corruption. We send the
21402373 * dying CPU's callbacks to an arbitrarily chosen online CPU.
21412374 */
2142− rcu_send_cbs_to_online(&rcu_bh_state);
2143− rcu_send_cbs_to_online(&rcu_sched_state);
2144− rcu_preempt_send_cbs_to_online();
2375+ rcu_cleanup_dying_cpu(&rcu_bh_state);
2376+ rcu_cleanup_dying_cpu(&rcu_sched_state);
2377+ rcu_preempt_cleanup_dying_cpu();
21452378 rcu_cleanup_after_idle(cpu);
21462379 break;
21472380 case CPU_DEAD:
21482381 case CPU_DEAD_FROZEN:
21492382 case CPU_UP_CANCELED:
21502383 case CPU_UP_CANCELED_FROZEN:
2151− rcu_offline_cpu(cpu);
2384+ rcu_cleanup_dead_cpu(cpu, &rcu_bh_state);
2385+ rcu_cleanup_dead_cpu(cpu, &rcu_sched_state);
2386+ rcu_preempt_cleanup_dead_cpu(cpu);
21522387 break;
21532388 default:
21542389 break;
kernel/rcutree.h+16−11
@@ -239,6 +239,12 @@ struct rcu_data { · 238 unmodified lines
239239 bool preemptible; /* Preemptible RCU? */
240240 struct rcu_node *mynode; /* This CPU's leaf of hierarchy */
241241 unsigned long grpmask; /* Mask to apply to leaf qsmask. */
242+#ifdef CONFIG_RCU_CPU_STALL_INFO
243+ unsigned long ticks_this_gp; /* The number of scheduling-clock */
244+ /* ticks this CPU has handled */
245+ /* during and after the last grace */
246+ /* period it is aware of. */
247+#endif /* #ifdef CONFIG_RCU_CPU_STALL_INFO */
242248
243249 /* 2) batch handling */
244250 /*
@@ -265,7 +271,8 @@ struct rcu_data { · 20 unmodified lines
265271 */
266272 struct rcu_head *nxtlist;
267273 struct rcu_head **nxttail[RCU_NEXT_SIZE];
268− long qlen; /* # of queued callbacks */
274+ long qlen_lazy; /* # of lazy queued callbacks */
275+ long qlen; /* # of queued callbacks, incl lazy */
269276 long qlen_last_fqs_check;
270277 /* qlen at last check for QS forcing */
271278 unsigned long n_cbs_invoked; /* count of RCU cbs invoked. */
@@ -282,7 +289,6 @@ struct rcu_data { · 10 unmodified lines
282289 /* 4) reasons this CPU needed to be kicked by force_quiescent_state */
283290 unsigned long dynticks_fqs; /* Kicked due to dynticks idle. */
284291 unsigned long offline_fqs; /* Kicked due to being offline. */
285− unsigned long resched_ipi; /* Sent a resched IPI. */
286292
287293 /* 5) __rcu_pending() statistics. */
288294 unsigned long n_rcu_pending; /* rcu_pending() calls since boot. */
@@ -313,12 +319,6 @@ struct rcu_data { · 24 unmodified lines
313319 #else
314320 #define RCU_STALL_DELAY_DELTA 0
315321 #endif
316−
317−#define RCU_SECONDS_TILL_STALL_CHECK (CONFIG_RCU_CPU_STALL_TIMEOUT * HZ + \
318− RCU_STALL_DELAY_DELTA)
319− /* for rsp->jiffies_stall */
320−#define RCU_SECONDS_TILL_STALL_RECHECK (3 * RCU_SECONDS_TILL_STALL_CHECK + 30)
321− /* for rsp->jiffies_stall */
322322 #define RCU_STALL_RAT_DELAY 2 /* Allow other CPUs time */
323323 /* to take at least one */
324324 /* scheduling clock irq */
@@ -438,8 +438,8 @@ static void rcu_preempt_check_blocked_tasks(struct rcu_node *rnp); · 113 unmodified lines
438438 static int rcu_preempt_offline_tasks(struct rcu_state *rsp,
439439 struct rcu_node *rnp,
440440 struct rcu_data *rdp);
441−static void rcu_preempt_offline_cpu(int cpu);
442441 #endif /* #ifdef CONFIG_HOTPLUG_CPU */
442+static void rcu_preempt_cleanup_dead_cpu(int cpu);
443443 static void rcu_preempt_check_callbacks(int cpu);
444444 static void rcu_preempt_process_callbacks(void);
445445 void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu));
@@ -448,9 +448,9 @@ static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp, · 2 unmodified lines
448448 bool wake);
449449 #endif /* #if defined(CONFIG_HOTPLUG_CPU) || defined(CONFIG_TREE_PREEMPT_RCU) */
450450 static int rcu_preempt_pending(int cpu);
451−static int rcu_preempt_needs_cpu(int cpu);
451+static int rcu_preempt_cpu_has_callbacks(int cpu);
452452 static void __cpuinit rcu_preempt_init_percpu_data(int cpu);
453−static void rcu_preempt_send_cbs_to_online(void);
453+static void rcu_preempt_cleanup_dying_cpu(void);
454454 static void __init __rcu_init_preempt(void);
455455 static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags);
456456 static void rcu_preempt_boost_start_gp(struct rcu_node *rnp);
@@ -471,5 +471,10 @@ static void __cpuinit rcu_prepare_kthreads(int cpu); · 14 unmodified lines
471471 static void rcu_prepare_for_idle_init(int cpu);
472472 static void rcu_cleanup_after_idle(int cpu);
473473 static void rcu_prepare_for_idle(int cpu);
474+static void print_cpu_stall_info_begin(void);
475+static void print_cpu_stall_info(struct rcu_state *rsp, int cpu);
476+static void print_cpu_stall_info_end(void);
477+static void zero_cpu_stall_ticks(struct rcu_data *rdp);
478+static void increment_cpu_stall_ticks(void);
474479
475480 #endif /* #ifndef RCU_TREE_NONCORE */
kernel/rcutree_plugin.h+276−174
@@ -25,7 +25,6 @@ · 24 unmodified lines
2525 */
2626
2727 #include <linux/delay.h>
28−#include <linux/stop_machine.h>
2928
3029 #define RCU_KTHREAD_PRIO 1
3130
@@ -63,7 +62,10 @@ static void __init rcu_bootup_announce_oddness(void) · 31 unmodified lines
6362 printk(KERN_INFO "\tRCU torture testing starts during boot.\n");
6463 #endif
6564 #if defined(CONFIG_TREE_PREEMPT_RCU) && !defined(CONFIG_RCU_CPU_STALL_VERBOSE)
66− printk(KERN_INFO "\tVerbose stalled-CPUs detection is disabled.\n");
65+ printk(KERN_INFO "\tDump stacks of tasks blocking RCU-preempt GP.\n");
66+#endif
67+#if defined(CONFIG_RCU_CPU_STALL_INFO)
68+ printk(KERN_INFO "\tAdditional per-CPU info printed with stalls.\n");
6769 #endif
6870 #if NUM_RCU_LVL_4 != 0
6971 printk(KERN_INFO "\tExperimental four-level hierarchy is enabled.\n");
@@ -490,6 +492,31 @@ static void rcu_print_detail_task_stall(struct rcu_state *rsp) · 420 unmodified lines
490492
491493 #endif /* #else #ifdef CONFIG_RCU_CPU_STALL_VERBOSE */
492494
495+#ifdef CONFIG_RCU_CPU_STALL_INFO
496+
497+static void rcu_print_task_stall_begin(struct rcu_node *rnp)
498+{
499+ printk(KERN_ERR "\tTasks blocked on level-%d rcu_node (CPUs %d-%d):",
500+ rnp->level, rnp->grplo, rnp->grphi);
501+}
502+
503+static void rcu_print_task_stall_end(void)
504+{
505+ printk(KERN_CONT "\n");
506+}
507+
508+#else /* #ifdef CONFIG_RCU_CPU_STALL_INFO */
509+
510+static void rcu_print_task_stall_begin(struct rcu_node *rnp)
511+{
512+}
513+
514+static void rcu_print_task_stall_end(void)
515+{
516+}
517+
518+#endif /* #else #ifdef CONFIG_RCU_CPU_STALL_INFO */
519+
493520 /*
494521 * Scan the current list of tasks blocked within RCU read-side critical
495522 * sections, printing out the tid of each.
@@ -501,12 +528,14 @@ static int rcu_print_task_stall(struct rcu_node *rnp) · 5 unmodified lines
501528
502529 if (!rcu_preempt_blocked_readers_cgp(rnp))
503530 return 0;
531+ rcu_print_task_stall_begin(rnp);
504532 t = list_entry(rnp->gp_tasks,
505533 struct task_struct, rcu_node_entry);
506534 list_for_each_entry_continue(t, &rnp->blkd_tasks, rcu_node_entry) {
507− printk(" P%d", t->pid);
535+ printk(KERN_CONT " P%d", t->pid);
508536 ndetected++;
509537 }
538+ rcu_print_task_stall_end();
510539 return ndetected;
511540 }
512541
@@ -581,7 +610,7 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp, · 68 unmodified lines
581610 * absolutely necessary, but this is a good performance/complexity
582611 * tradeoff.
583612 */
584− if (rcu_preempt_blocked_readers_cgp(rnp))
613+ if (rcu_preempt_blocked_readers_cgp(rnp) && rnp->qsmask == 0)
585614 retval |= RCU_OFL_TASKS_NORM_GP;
586615 if (rcu_preempted_readers_exp(rnp))
587616 retval |= RCU_OFL_TASKS_EXP_GP;
@@ -618,16 +647,16 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp, · 30 unmodified lines
618647 return retval;
619648 }
620649
650+#endif /* #ifdef CONFIG_HOTPLUG_CPU */
651+
621652 /*
622653 * Do CPU-offline processing for preemptible RCU.
623654 */
624−static void rcu_preempt_offline_cpu(int cpu)
655+static void rcu_preempt_cleanup_dead_cpu(int cpu)
625656 {
626− __rcu_offline_cpu(cpu, &rcu_preempt_state);
657+ rcu_cleanup_dead_cpu(cpu, &rcu_preempt_state);
627658 }
628659
629−#endif /* #ifdef CONFIG_HOTPLUG_CPU */
630−
631660 /*
632661 * Check for a quiescent state from the current CPU. When a task blocks,
633662 * the task is recorded in the corresponding CPU's rcu_node structure,
@@ -671,10 +700,24 @@ static void rcu_preempt_do_callbacks(void) · 37 unmodified lines
671700 */
672701 void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu))
673702 {
674− __call_rcu(head, func, &rcu_preempt_state);
703+ __call_rcu(head, func, &rcu_preempt_state, 0);
675704 }
676705 EXPORT_SYMBOL_GPL(call_rcu);
677706
707+/*
708+ * Queue an RCU callback for lazy invocation after a grace period.
709+ * This will likely be later named something like "call_rcu_lazy()",
710+ * but this change will require some way of tagging the lazy RCU
711+ * callbacks in the list of pending callbacks. Until then, this
712+ * function may only be called from __kfree_rcu().
713+ */
714+void kfree_call_rcu(struct rcu_head *head,
715+ void (*func)(struct rcu_head *rcu))
716+{
717+ __call_rcu(head, func, &rcu_preempt_state, 1);
718+}
719+EXPORT_SYMBOL_GPL(kfree_call_rcu);
720+
678721 /**
679722 * synchronize_rcu - wait until a grace period has elapsed.
680723 *
@@ -688,6 +731,10 @@ EXPORT_SYMBOL_GPL(call_rcu); · 7 unmodified lines
688731 */
689732 void synchronize_rcu(void)
690733 {
734+ rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map) &&
735+ !lock_is_held(&rcu_lock_map) &&
736+ !lock_is_held(&rcu_sched_lock_map),
737+ "Illegal synchronize_rcu() in RCU read-side critical section");
691738 if (!rcu_scheduler_active)
692739 return;
693740 wait_rcu_gp(call_rcu);
@@ -788,10 +835,22 @@ sync_rcu_preempt_exp_init(struct rcu_state *rsp, struct rcu_node *rnp) · 94 unmodified lines
788835 rcu_report_exp_rnp(rsp, rnp, false); /* Don't wake self. */
789836 }
790837
791−/*
792− * Wait for an rcu-preempt grace period, but expedite it. The basic idea
793− * is to invoke synchronize_sched_expedited() to push all the tasks to
794− * the ->blkd_tasks lists and wait for this list to drain.
838+/**
839+ * synchronize_rcu_expedited - Brute-force RCU grace period
840+ *
841+ * Wait for an RCU-preempt grace period, but expedite it. The basic
842+ * idea is to invoke synchronize_sched_expedited() to push all the tasks to
843+ * the ->blkd_tasks lists and wait for this list to drain. This consumes
844+ * significant time on all CPUs and is unfriendly to real-time workloads,
845+ * so is thus not recommended for any sort of common-case code.
846+ * In fact, if you are using synchronize_rcu_expedited() in a loop,
847+ * please restructure your code to batch your updates, and then Use a
848+ * single synchronize_rcu() instead.
849+ *
850+ * Note that it is illegal to call this function while holding any lock
851+ * that is acquired by a CPU-hotplug notifier. And yes, it is also illegal
852+ * to call this function from a CPU-hotplug notifier. Failing to observe
853+ * these restriction will result in deadlock.
795854 */
796855 void synchronize_rcu_expedited(void)
797856 {
@@ -869,9 +928,9 @@ static int rcu_preempt_pending(int cpu) · 71 unmodified lines
869928 }
870929
871930 /*
872− * Does preemptible RCU need the CPU to stay out of dynticks mode?
931+ * Does preemptible RCU have callbacks on this CPU?
873932 */
874−static int rcu_preempt_needs_cpu(int cpu)
933+static int rcu_preempt_cpu_has_callbacks(int cpu)
875934 {
876935 return !!per_cpu(rcu_preempt_data, cpu).nxtlist;
877936 }
@@ -894,11 +953,12 @@ static void __cpuinit rcu_preempt_init_percpu_data(int cpu) · 16 unmodified lines
894953 }
895954
896955 /*
897− * Move preemptible RCU's callbacks from dying CPU to other online CPU.
956+ * Move preemptible RCU's callbacks from dying CPU to other online CPU
957+ * and record a quiescent state.
898958 */
899−static void rcu_preempt_send_cbs_to_online(void)
959+static void rcu_preempt_cleanup_dying_cpu(void)
900960 {
901− rcu_send_cbs_to_online(&rcu_preempt_state);
961+ rcu_cleanup_dying_cpu(&rcu_preempt_state);
902962 }
903963
904964 /*
@@ -1034,16 +1094,16 @@ static int rcu_preempt_offline_tasks(struct rcu_state *rsp, · 129 unmodified lines
10341094 return 0;
10351095 }
10361096
1097+#endif /* #ifdef CONFIG_HOTPLUG_CPU */
1098+
10371099 /*
10381100 * Because preemptible RCU does not exist, it never needs CPU-offline
10391101 * processing.
10401102 */
1041−static void rcu_preempt_offline_cpu(int cpu)
1103+static void rcu_preempt_cleanup_dead_cpu(int cpu)
10421104 {
10431105 }
10441106
1045−#endif /* #ifdef CONFIG_HOTPLUG_CPU */
1046−
10471107 /*
10481108 * Because preemptible RCU does not exist, it never has any callbacks
10491109 * to check.
@@ -1060,6 +1120,22 @@ static void rcu_preempt_process_callbacks(void) · 10 unmodified lines
10601120 {
10611121 }
10621122
1123+/*
1124+ * Queue an RCU callback for lazy invocation after a grace period.
1125+ * This will likely be later named something like "call_rcu_lazy()",
1126+ * but this change will require some way of tagging the lazy RCU
1127+ * callbacks in the list of pending callbacks. Until then, this
1128+ * function may only be called from __kfree_rcu().
1129+ *
1130+ * Because there is no preemptible RCU, we use RCU-sched instead.
1131+ */
1132+void kfree_call_rcu(struct rcu_head *head,
1133+ void (*func)(struct rcu_head *rcu))
1134+{
1135+ __call_rcu(head, func, &rcu_sched_state, 1);
1136+}
1137+EXPORT_SYMBOL_GPL(kfree_call_rcu);
1138+
10631139 /*
10641140 * Wait for an rcu-preempt grace period, but make it happen quickly.
10651141 * But because preemptible RCU does not exist, map to rcu-sched.
@@ -1093,9 +1169,9 @@ static int rcu_preempt_pending(int cpu) · 27 unmodified lines
10931169 }
10941170
10951171 /*
1096− * Because preemptible RCU does not exist, it never needs any CPU.
1172+ * Because preemptible RCU does not exist, it never has callbacks
10971173 */
1098−static int rcu_preempt_needs_cpu(int cpu)
1174+static int rcu_preempt_cpu_has_callbacks(int cpu)
10991175 {
11001176 return 0;
11011177 }
@@ -1119,9 +1195,9 @@ static void __cpuinit rcu_preempt_init_percpu_data(int cpu) · 17 unmodified lines
11191195 }
11201196
11211197 /*
1122− * Because there is no preemptible RCU, there are no callbacks to move.
1198+ * Because there is no preemptible RCU, there is no cleanup to do.
11231199 */
1124−static void rcu_preempt_send_cbs_to_online(void)
1200+static void rcu_preempt_cleanup_dying_cpu(void)
11251201 {
11261202 }
11271203
@@ -1823,132 +1899,6 @@ static void __cpuinit rcu_prepare_kthreads(int cpu) · 695 unmodified lines
18231899
18241900 #endif /* #else #ifdef CONFIG_RCU_BOOST */
18251901
1826−#ifndef CONFIG_SMP
1827−
1828−void synchronize_sched_expedited(void)
1829−{
1830− cond_resched();
1831−}
1832−EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
1833−
1834−#else /* #ifndef CONFIG_SMP */
1835−
1836−static atomic_t sync_sched_expedited_started = ATOMIC_INIT(0);
1837−static atomic_t sync_sched_expedited_done = ATOMIC_INIT(0);
1838−
1839−static int synchronize_sched_expedited_cpu_stop(void *data)
1840−{
1841− /*
1842− * There must be a full memory barrier on each affected CPU
1843− * between the time that try_stop_cpus() is called and the
1844− * time that it returns.
1845− *
1846− * In the current initial implementation of cpu_stop, the
1847− * above condition is already met when the control reaches
1848− * this point and the following smp_mb() is not strictly
1849− * necessary. Do smp_mb() anyway for documentation and
1850− * robustness against future implementation changes.
1851− */
1852− smp_mb(); /* See above comment block. */
1853− return 0;
1854−}
1855−
1856−/*
1857− * Wait for an rcu-sched grace period to elapse, but use "big hammer"
1858− * approach to force grace period to end quickly. This consumes
1859− * significant time on all CPUs, and is thus not recommended for
1860− * any sort of common-case code.
1861− *
1862− * Note that it is illegal to call this function while holding any
1863− * lock that is acquired by a CPU-hotplug notifier. Failing to
1864− * observe this restriction will result in deadlock.
1865− *
1866− * This implementation can be thought of as an application of ticket
1867− * locking to RCU, with sync_sched_expedited_started and
1868− * sync_sched_expedited_done taking on the roles of the halves
1869− * of the ticket-lock word. Each task atomically increments
1870− * sync_sched_expedited_started upon entry, snapshotting the old value,
1871− * then attempts to stop all the CPUs. If this succeeds, then each
1872− * CPU will have executed a context switch, resulting in an RCU-sched
1873− * grace period. We are then done, so we use atomic_cmpxchg() to
1874− * update sync_sched_expedited_done to match our snapshot -- but
1875− * only if someone else has not already advanced past our snapshot.
1876− *
1877− * On the other hand, if try_stop_cpus() fails, we check the value
1878− * of sync_sched_expedited_done. If it has advanced past our
1879− * initial snapshot, then someone else must have forced a grace period
1880− * some time after we took our snapshot. In this case, our work is
1881− * done for us, and we can simply return. Otherwise, we try again,
1882− * but keep our initial snapshot for purposes of checking for someone
1883− * doing our work for us.
1884− *
1885− * If we fail too many times in a row, we fall back to synchronize_sched().
1886− */
1887−void synchronize_sched_expedited(void)
1888−{
1889− int firstsnap, s, snap, trycount = 0;
1890−
1891− /* Note that atomic_inc_return() implies full memory barrier. */
1892− firstsnap = snap = atomic_inc_return(&sync_sched_expedited_started);
1893− get_online_cpus();
1894−
1895− /*
1896− * Each pass through the following loop attempts to force a
1897− * context switch on each CPU.
1898− */
1899− while (try_stop_cpus(cpu_online_mask,
1900− synchronize_sched_expedited_cpu_stop,
1901− NULL) == -EAGAIN) {
1902− put_online_cpus();
1903−
1904− /* No joy, try again later. Or just synchronize_sched(). */
1905− if (trycount++ < 10)
1906− udelay(trycount * num_online_cpus());
1907− else {
1908− synchronize_sched();
1909− return;
1910− }
1911−
1912− /* Check to see if someone else did our work for us. */
1913− s = atomic_read(&sync_sched_expedited_done);
1914− if (UINT_CMP_GE((unsigned)s, (unsigned)firstsnap)) {
1915− smp_mb(); /* ensure test happens before caller kfree */
1916− return;
1917− }
1918−
1919− /*
1920− * Refetching sync_sched_expedited_started allows later
1921− * callers to piggyback on our grace period. We subtract
1922− * 1 to get the same token that the last incrementer got.
1923− * We retry after they started, so our grace period works
1924− * for them, and they started after our first try, so their
1925− * grace period works for us.
1926− */
1927− get_online_cpus();
1928− snap = atomic_read(&sync_sched_expedited_started);
1929− smp_mb(); /* ensure read is before try_stop_cpus(). */
1930− }
1931−
1932− /*
1933− * Everyone up to our most recent fetch is covered by our grace
1934− * period. Update the counter, but only if our work is still
1935− * relevant -- which it won't be if someone who started later
1936− * than we did beat us to the punch.
1937− */
1938− do {
1939− s = atomic_read(&sync_sched_expedited_done);
1940− if (UINT_CMP_GE((unsigned)s, (unsigned)snap)) {
1941− smp_mb(); /* ensure test happens before caller kfree */
1942− break;
1943− }
1944− } while (atomic_cmpxchg(&sync_sched_expedited_done, s, snap) != s);
1945−
1946− put_online_cpus();
1947−}
1948−EXPORT_SYMBOL_GPL(synchronize_sched_expedited);
1949−
1950−#endif /* #else #ifndef CONFIG_SMP */
1951−
19521902 #if !defined(CONFIG_RCU_FAST_NO_HZ)
19531903
19541904 /*
@@ -1981,7 +1931,7 @@ static void rcu_cleanup_after_idle(int cpu) · 26 unmodified lines
19811931 }
19821932
19831933 /*
1984− * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=y,
1934+ * Do the idle-entry grace-period work, which, because CONFIG_RCU_FAST_NO_HZ=n,
19851935 * is nothing.
19861936 */
19871937 static void rcu_prepare_for_idle(int cpu)
@@ -2015,6 +1965,9 @@ static void rcu_prepare_for_idle(int cpu) · 27 unmodified lines
20151965 * number, be warned: Setting RCU_IDLE_GP_DELAY too high can hang your
20161966 * system. And if you are -that- concerned about energy efficiency,
20171967 * just power the system down and be done with it!
1968+ * RCU_IDLE_LAZY_GP_DELAY gives the number of jiffies that a CPU is
1969+ * permitted to sleep in dyntick-idle mode with only lazy RCU
1970+ * callbacks pending. Setting this too high can OOM your system.
20181971 *
20191972 * The values below work well in practice. If future workloads require
20201973 * adjustment, they can be converted into kernel config parameters, though
@@ -2023,11 +1976,13 @@ static void rcu_prepare_for_idle(int cpu) · 2 unmodified lines
20231976 #define RCU_IDLE_FLUSHES 5 /* Number of dyntick-idle tries. */
20241977 #define RCU_IDLE_OPT_FLUSHES 3 /* Optional dyntick-idle tries. */
20251978 #define RCU_IDLE_GP_DELAY 6 /* Roughly one grace period. */
1979+#define RCU_IDLE_LAZY_GP_DELAY (6 * HZ) /* Roughly six seconds. */
20261980
20271981 static DEFINE_PER_CPU(int, rcu_dyntick_drain);
20281982 static DEFINE_PER_CPU(unsigned long, rcu_dyntick_holdoff);
20291983 static DEFINE_PER_CPU(struct hrtimer, rcu_idle_gp_timer);
2030−static ktime_t rcu_idle_gp_wait;
1984+static ktime_t rcu_idle_gp_wait; /* If some non-lazy callbacks. */
1985+static ktime_t rcu_idle_lazy_gp_wait; /* If only lazy callbacks. */
20311986
20321987 /*
20331988 * Allow the CPU to enter dyntick-idle mode if either: (1) There are no
@@ -2047,6 +2002,48 @@ int rcu_needs_cpu(int cpu) · 13 unmodified lines
20472002 return per_cpu(rcu_dyntick_holdoff, cpu) == jiffies;
20482003 }
20492004
2005+/*
2006+ * Does the specified flavor of RCU have non-lazy callbacks pending on
2007+ * the specified CPU? Both RCU flavor and CPU are specified by the
2008+ * rcu_data structure.
2009+ */
2010+static bool __rcu_cpu_has_nonlazy_callbacks(struct rcu_data *rdp)
2011+{
2012+ return rdp->qlen != rdp->qlen_lazy;
2013+}
2014+
2015+#ifdef CONFIG_TREE_PREEMPT_RCU
2016+
2017+/*
2018+ * Are there non-lazy RCU-preempt callbacks? (There cannot be if there
2019+ * is no RCU-preempt in the kernel.)
2020+ */
2021+static bool rcu_preempt_cpu_has_nonlazy_callbacks(int cpu)
2022+{
2023+ struct rcu_data *rdp = &per_cpu(rcu_preempt_data, cpu);
2024+
2025+ return __rcu_cpu_has_nonlazy_callbacks(rdp);
2026+}
2027+
2028+#else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
2029+
2030+static bool rcu_preempt_cpu_has_nonlazy_callbacks(int cpu)
2031+{
2032+ return 0;
2033+}
2034+
2035+#endif /* else #ifdef CONFIG_TREE_PREEMPT_RCU */
2036+
2037+/*
2038+ * Does any flavor of RCU have non-lazy callbacks on the specified CPU?
2039+ */
2040+static bool rcu_cpu_has_nonlazy_callbacks(int cpu)
2041+{
2042+ return __rcu_cpu_has_nonlazy_callbacks(&per_cpu(rcu_sched_data, cpu)) ||
2043+ __rcu_cpu_has_nonlazy_callbacks(&per_cpu(rcu_bh_data, cpu)) ||
2044+ rcu_preempt_cpu_has_nonlazy_callbacks(cpu);
2045+}
2046+
20502047 /*
20512048 * Timer handler used to force CPU to start pushing its remaining RCU
20522049 * callbacks in the case where it entered dyntick-idle mode with callbacks
@@ -2074,6 +2071,8 @@ static void rcu_prepare_for_idle_init(int cpu) · 21 unmodified lines
20742071 unsigned int upj = jiffies_to_usecs(RCU_IDLE_GP_DELAY);
20752072
20762073 rcu_idle_gp_wait = ns_to_ktime(upj * (u64)1000);
2074+ upj = jiffies_to_usecs(RCU_IDLE_LAZY_GP_DELAY);
2075+ rcu_idle_lazy_gp_wait = ns_to_ktime(upj * (u64)1000);
20772076 firsttime = 0;
20782077 }
20792078 }
@@ -2109,10 +2108,6 @@ static void rcu_cleanup_after_idle(int cpu) · 29 unmodified lines
21092108 */
21102109 static void rcu_prepare_for_idle(int cpu)
21112110 {
2112− unsigned long flags;
2113−
2114− local_irq_save(flags);
2115−
21162111 /*
21172112 * If there are no callbacks on this CPU, enter dyntick-idle mode.
21182113 * Also reset state to avoid prejudicing later attempts.
@@ -2120,7 +2115,6 @@ static void rcu_prepare_for_idle(int cpu) · 1 unmodified line
21202115 if (!rcu_cpu_has_callbacks(cpu)) {
21212116 per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1;
21222117 per_cpu(rcu_dyntick_drain, cpu) = 0;
2123− local_irq_restore(flags);
21242118 trace_rcu_prep_idle("No callbacks");
21252119 return;
21262120 }
@@ -2130,7 +2124,6 @@ static void rcu_prepare_for_idle(int cpu) · 3 unmodified lines
21302124 * refrained from disabling the scheduling-clock tick.
21312125 */
21322126 if (per_cpu(rcu_dyntick_holdoff, cpu) == jiffies) {
2133− local_irq_restore(flags);
21342127 trace_rcu_prep_idle("In holdoff");
21352128 return;
21362129 }
@@ -2140,18 +2133,22 @@ static void rcu_prepare_for_idle(int cpu) · 3 unmodified lines
21402133 /* First time through, initialize the counter. */
21412134 per_cpu(rcu_dyntick_drain, cpu) = RCU_IDLE_FLUSHES;
21422135 } else if (per_cpu(rcu_dyntick_drain, cpu) <= RCU_IDLE_OPT_FLUSHES &&
2143− !rcu_pending(cpu)) {
2136+ !rcu_pending(cpu) &&
2137+ !local_softirq_pending()) {
21442138 /* Can we go dyntick-idle despite still having callbacks? */
21452139 trace_rcu_prep_idle("Dyntick with callbacks");
21462140 per_cpu(rcu_dyntick_drain, cpu) = 0;
2147− per_cpu(rcu_dyntick_holdoff, cpu) = jiffies - 1;
2148− hrtimer_start(&per_cpu(rcu_idle_gp_timer, cpu),
2149− rcu_idle_gp_wait, HRTIMER_MODE_REL);
2141+ per_cpu(rcu_dyntick_holdoff, cpu) = jiffies;
2142+ if (rcu_cpu_has_nonlazy_callbacks(cpu))
2143+ hrtimer_start(&per_cpu(rcu_idle_gp_timer, cpu),
2144+ rcu_idle_gp_wait, HRTIMER_MODE_REL);
2145+ else
2146+ hrtimer_start(&per_cpu(rcu_idle_gp_timer, cpu),
2147+ rcu_idle_lazy_gp_wait, HRTIMER_MODE_REL);
21502148 return; /* Nothing more to do immediately. */
21512149 } else if (--per_cpu(rcu_dyntick_drain, cpu) <= 0) {
21522150 /* We have hit the limit, so time to give up. */
21532151 per_cpu(rcu_dyntick_holdoff, cpu) = jiffies;
2154− local_irq_restore(flags);
21552152 trace_rcu_prep_idle("Begin holdoff");
21562153 invoke_rcu_core(); /* Force the CPU out of dyntick-idle. */
21572154 return;
@@ -2163,23 +2160,17 @@ static void rcu_prepare_for_idle(int cpu) · 5 unmodified lines
21632160 */
21642161 #ifdef CONFIG_TREE_PREEMPT_RCU
21652162 if (per_cpu(rcu_preempt_data, cpu).nxtlist) {
2166− local_irq_restore(flags);
21672163 rcu_preempt_qs(cpu);
21682164 force_quiescent_state(&rcu_preempt_state, 0);
2169− local_irq_save(flags);
21702165 }
21712166 #endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
21722167 if (per_cpu(rcu_sched_data, cpu).nxtlist) {
2173− local_irq_restore(flags);
21742168 rcu_sched_qs(cpu);
21752169 force_quiescent_state(&rcu_sched_state, 0);
2176− local_irq_save(flags);
21772170 }
21782171 if (per_cpu(rcu_bh_data, cpu).nxtlist) {
2179− local_irq_restore(flags);
21802172 rcu_bh_qs(cpu);
21812173 force_quiescent_state(&rcu_bh_state, 0);
2182− local_irq_save(flags);
21832174 }
21842175
21852176 /*
@@ -2187,13 +2178,124 @@ static void rcu_prepare_for_idle(int cpu) · 1 unmodified line
21872178 * So try forcing the callbacks through the grace period.
21882179 */
21892180 if (rcu_cpu_has_callbacks(cpu)) {
2190− local_irq_restore(flags);
21912181 trace_rcu_prep_idle("More callbacks");
21922182 invoke_rcu_core();
2193− } else {
2194− local_irq_restore(flags);
2183+ } else
21952184 trace_rcu_prep_idle("Callbacks drained");
2196− }
21972185 }
21982186
21992187 #endif /* #else #if !defined(CONFIG_RCU_FAST_NO_HZ) */
2188+
2189+#ifdef CONFIG_RCU_CPU_STALL_INFO
2190+
2191+#ifdef CONFIG_RCU_FAST_NO_HZ
2192+
2193+static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
2194+{
2195+ struct hrtimer *hrtp = &per_cpu(rcu_idle_gp_timer, cpu);
2196+
2197+ sprintf(cp, "drain=%d %c timer=%lld",
2198+ per_cpu(rcu_dyntick_drain, cpu),
2199+ per_cpu(rcu_dyntick_holdoff, cpu) == jiffies ? 'H' : '.',
2200+ hrtimer_active(hrtp)
2201+ ? ktime_to_us(hrtimer_get_remaining(hrtp))
2202+ : -1);
2203+}
2204+
2205+#else /* #ifdef CONFIG_RCU_FAST_NO_HZ */
2206+
2207+static void print_cpu_stall_fast_no_hz(char *cp, int cpu)
2208+{
2209+}
2210+
2211+#endif /* #else #ifdef CONFIG_RCU_FAST_NO_HZ */
2212+
2213+/* Initiate the stall-info list. */
2214+static void print_cpu_stall_info_begin(void)
2215+{
2216+ printk(KERN_CONT "\n");
2217+}
2218+
2219+/*
2220+ * Print out diagnostic information for the specified stalled CPU.
2221+ *
2222+ * If the specified CPU is aware of the current RCU grace period
2223+ * (flavor specified by rsp), then print the number of scheduling
2224+ * clock interrupts the CPU has taken during the time that it has
2225+ * been aware. Otherwise, print the number of RCU grace periods
2226+ * that this CPU is ignorant of, for example, "1" if the CPU was
2227+ * aware of the previous grace period.
2228+ *
2229+ * Also print out idle and (if CONFIG_RCU_FAST_NO_HZ) idle-entry info.
2230+ */
2231+static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
2232+{
2233+ char fast_no_hz[72];
2234+ struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
2235+ struct rcu_dynticks *rdtp = rdp->dynticks;
2236+ char *ticks_title;
2237+ unsigned long ticks_value;
2238+
2239+ if (rsp->gpnum == rdp->gpnum) {
2240+ ticks_title = "ticks this GP";
2241+ ticks_value = rdp->ticks_this_gp;
2242+ } else {
2243+ ticks_title = "GPs behind";
2244+ ticks_value = rsp->gpnum - rdp->gpnum;
2245+ }
2246+ print_cpu_stall_fast_no_hz(fast_no_hz, cpu);
2247+ printk(KERN_ERR "\t%d: (%lu %s) idle=%03x/%llx/%d %s\n",
2248+ cpu, ticks_value, ticks_title,
2249+ atomic_read(&rdtp->dynticks) & 0xfff,
2250+ rdtp->dynticks_nesting, rdtp->dynticks_nmi_nesting,
2251+ fast_no_hz);
2252+}
2253+
2254+/* Terminate the stall-info list. */
2255+static void print_cpu_stall_info_end(void)
2256+{
2257+ printk(KERN_ERR "\t");
2258+}
2259+
2260+/* Zero ->ticks_this_gp for all flavors of RCU. */
2261+static void zero_cpu_stall_ticks(struct rcu_data *rdp)
2262+{
2263+ rdp->ticks_this_gp = 0;
2264+}
2265+
2266+/* Increment ->ticks_this_gp for all flavors of RCU. */
2267+static void increment_cpu_stall_ticks(void)
2268+{
2269+ __get_cpu_var(rcu_sched_data).ticks_this_gp++;
2270+ __get_cpu_var(rcu_bh_data).ticks_this_gp++;
2271+#ifdef CONFIG_TREE_PREEMPT_RCU
2272+ __get_cpu_var(rcu_preempt_data).ticks_this_gp++;
2273+#endif /* #ifdef CONFIG_TREE_PREEMPT_RCU */
2274+}
2275+
2276+#else /* #ifdef CONFIG_RCU_CPU_STALL_INFO */
2277+
2278+static void print_cpu_stall_info_begin(void)
2279+{
2280+ printk(KERN_CONT " {");
2281+}
2282+
2283+static void print_cpu_stall_info(struct rcu_state *rsp, int cpu)
2284+{
2285+ printk(KERN_CONT " %d", cpu);
2286+}
2287+
2288+static void print_cpu_stall_info_end(void)
2289+{
2290+ printk(KERN_CONT "} ");
2291+}
2292+
2293+static void zero_cpu_stall_ticks(struct rcu_data *rdp)
2294+{
2295+}
2296+
2297+static void increment_cpu_stall_ticks(void)
2298+{
2299+}
2300+
2301+#endif /* #else #ifdef CONFIG_RCU_CPU_STALL_INFO */
kernel/rcutree_trace.c+6−6
@@ -72,9 +72,9 @@ static void print_one_rcu_data(struct seq_file *m, struct rcu_data *rdp) · 71 unmodified lines
7272 rdp->dynticks->dynticks_nesting,
7373 rdp->dynticks->dynticks_nmi_nesting,
7474 rdp->dynticks_fqs);
75− seq_printf(m, " of=%lu ri=%lu", rdp->offline_fqs, rdp->resched_ipi);
76− seq_printf(m, " ql=%ld qs=%c%c%c%c",
77− rdp->qlen,
75+ seq_printf(m, " of=%lu", rdp->offline_fqs);
76+ seq_printf(m, " ql=%ld/%ld qs=%c%c%c%c",
77+ rdp->qlen_lazy, rdp->qlen,
7878 ".N"[rdp->nxttail[RCU_NEXT_READY_TAIL] !=
7979 rdp->nxttail[RCU_NEXT_TAIL]],
8080 ".R"[rdp->nxttail[RCU_WAIT_TAIL] !=
@@ -144,8 +144,8 @@ static void print_one_rcu_data_csv(struct seq_file *m, struct rcu_data *rdp) · 63 unmodified lines
144144 rdp->dynticks->dynticks_nesting,
145145 rdp->dynticks->dynticks_nmi_nesting,
146146 rdp->dynticks_fqs);
147− seq_printf(m, ",%lu,%lu", rdp->offline_fqs, rdp->resched_ipi);
148− seq_printf(m, ",%ld,\"%c%c%c%c\"", rdp->qlen,
147+ seq_printf(m, ",%lu", rdp->offline_fqs);
148+ seq_printf(m, ",%ld,%ld,\"%c%c%c%c\"", rdp->qlen_lazy, rdp->qlen,
149149 ".N"[rdp->nxttail[RCU_NEXT_READY_TAIL] !=
150150 rdp->nxttail[RCU_NEXT_TAIL]],
151151 ".R"[rdp->nxttail[RCU_WAIT_TAIL] !=
@@ -168,7 +168,7 @@ static int show_rcudata_csv(struct seq_file *m, void *unused) · 16 unmodified lines
168168 {
169169 seq_puts(m, "\"CPU\",\"Online?\",\"c\",\"g\",\"pq\",\"pgp\",\"pq\",");
170170 seq_puts(m, "\"dt\",\"dt nesting\",\"dt NMI nesting\",\"df\",");
171− seq_puts(m, "\"of\",\"ri\",\"ql\",\"qs\"");
171+ seq_puts(m, "\"of\",\"qll\",\"ql\",\"qs\"");
172172 #ifdef CONFIG_RCU_BOOST
173173 seq_puts(m, "\"kt\",\"ktl\"");
174174 #endif /* #ifdef CONFIG_RCU_BOOST */
kernel/srcu.c+23−10
@@ -172,6 +172,12 @@ static void __synchronize_srcu(struct srcu_struct *sp, void (*sync_func)(void)) · 171 unmodified lines
172172 {
173173 int idx;
174174
175+ rcu_lockdep_assert(!lock_is_held(&sp->dep_map) &&
176+ !lock_is_held(&rcu_bh_lock_map) &&
177+ !lock_is_held(&rcu_lock_map) &&
178+ !lock_is_held(&rcu_sched_lock_map),
179+ "Illegal synchronize_srcu() in same-type SRCU (or RCU) read-side critical section");
180+
175181 idx = sp->completed;
176182 mutex_lock(&sp->mutex);
177183
@@ -280,19 +286,26 @@ void synchronize_srcu(struct srcu_struct *sp) · 102 unmodified lines
280286 EXPORT_SYMBOL_GPL(synchronize_srcu);
281287
282288 /**
283− * synchronize_srcu_expedited - like synchronize_srcu, but less patient
289+ * synchronize_srcu_expedited - Brute-force SRCU grace period
284290 * @sp: srcu_struct with which to synchronize.
285291 *
286− * Flip the completed counter, and wait for the old count to drain to zero.
287− * As with classic RCU, the updater must use some separate means of
288− * synchronizing concurrent updates. Can block; must be called from
289− * process context.
292+ * Wait for an SRCU grace period to elapse, but use a "big hammer"
293+ * approach to force the grace period to end quickly. This consumes
294+ * significant time on all CPUs and is unfriendly to real-time workloads,
295+ * so is thus not recommended for any sort of common-case code. In fact,
296+ * if you are using synchronize_srcu_expedited() in a loop, please
297+ * restructure your code to batch your updates, and then use a single
298+ * synchronize_srcu() instead.
290299 *
291− * Note that it is illegal to call synchronize_srcu_expedited()
292− * from the corresponding SRCU read-side critical section; doing so
293− * will result in deadlock. However, it is perfectly legal to call
294− * synchronize_srcu_expedited() on one srcu_struct from some other
295− * srcu_struct's read-side critical section.
300+ * Note that it is illegal to call this function while holding any lock
301+ * that is acquired by a CPU-hotplug notifier. And yes, it is also illegal
302+ * to call this function from a CPU-hotplug notifier. Failing to observe
303+ * these restriction will result in deadlock. It is also illegal to call
304+ * synchronize_srcu_expedited() from the corresponding SRCU read-side
305+ * critical section; doing so will result in deadlock. However, it is
306+ * perfectly legal to call synchronize_srcu_expedited() on one srcu_struct
307+ * from some other srcu_struct's read-side critical section, as long as
308+ * the resulting graph of srcu_structs is acyclic.
296309 */
297310 void synchronize_srcu_expedited(struct srcu_struct *sp)
298311 {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t