History

Merge branch 'core-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'core-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: signal: align __lock_task_sighand() irq disabling and RCU softirq,rcu: Inform RCU of irq_exit() activity sched: Add irq_{enter,exit}() to scheduler_ipi() rcu: protect __rcu_read_unlock() against scheduler-using irq handlers rcu: Streamline code produced by __rcu_read_unlock() rcu: Fix RCU_BOOST race handling current->rcu_read_unlock_special rcu: decrease rcu_report_exp_rnp coupling with scheduler

Linus Torvalds committed 15 years ago

Commit cf6ace12 parents acc11ea d1e9ae45 files · +103 −28Browse files

Unified · Split

5 files
  1. include/linux/sched.h +3 −0
  2. kernel/rcutree_plugin.h +39 −14
  3. kernel/sched.c +38 −6
  4. kernel/signal.c +13 −6
  5. kernel/softirq.c +10 −2
include/linux/sched.h+3−0
@@ -1260,6 +1260,9 @@ struct task_struct { · 1259 unmodified lines
12601260 #ifdef CONFIG_PREEMPT_RCU
12611261 int rcu_read_lock_nesting;
12621262 char rcu_read_unlock_special;
1263+#if defined(CONFIG_RCU_BOOST) && defined(CONFIG_TREE_PREEMPT_RCU)
1264+ int rcu_boosted;
1265+#endif /* #if defined(CONFIG_RCU_BOOST) && defined(CONFIG_TREE_PREEMPT_RCU) */
12631266 struct list_head rcu_node_entry;
12641267 #endif /* #ifdef CONFIG_PREEMPT_RCU */
12651268 #ifdef CONFIG_TREE_PREEMPT_RCU
kernel/rcutree_plugin.h+39−14
@@ -68,6 +68,7 @@ struct rcu_state rcu_preempt_state = RCU_STATE_INITIALIZER(rcu_preempt_state); · 67 unmodified lines
6868 DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
6969 static struct rcu_state *rcu_state = &rcu_preempt_state;
7070
71+static void rcu_read_unlock_special(struct task_struct *t);
7172 static int rcu_preempted_readers_exp(struct rcu_node *rnp);
7273
7374 /*
@@ -147,7 +148,7 @@ static void rcu_preempt_note_context_switch(int cpu) · 73 unmodified lines
147148 struct rcu_data *rdp;
148149 struct rcu_node *rnp;
149150
150− if (t->rcu_read_lock_nesting &&
151+ if (t->rcu_read_lock_nesting > 0 &&
151152 (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
152153
153154 /* Possibly blocking in an RCU read-side critical section. */
@@ -190,6 +191,14 @@ static void rcu_preempt_note_context_switch(int cpu) · 36 unmodified lines
190191 rnp->gp_tasks = &t->rcu_node_entry;
191192 }
192193 raw_spin_unlock_irqrestore(&rnp->lock, flags);
194+ } else if (t->rcu_read_lock_nesting < 0 &&
195+ t->rcu_read_unlock_special) {
196+
197+ /*
198+ * Complete exit from RCU read-side critical section on
199+ * behalf of preempted instance of __rcu_read_unlock().
200+ */
201+ rcu_read_unlock_special(t);
193202 }
194203
195204 /*
@@ -284,7 +293,7 @@ static struct list_head *rcu_next_node_entry(struct task_struct *t, · 88 unmodified lines
284293 * notify RCU core processing or task having blocked during the RCU
285294 * read-side critical section.
286295 */
287−static void rcu_read_unlock_special(struct task_struct *t)
296+static noinline void rcu_read_unlock_special(struct task_struct *t)
288297 {
289298 int empty;
290299 int empty_exp;
@@ -309,7 +318,7 @@ static void rcu_read_unlock_special(struct task_struct *t) · 18 unmodified lines
309318 }
310319
311320 /* Hardware IRQ handlers cannot block. */
312− if (in_irq()) {
321+ if (in_irq() || in_serving_softirq()) {
313322 local_irq_restore(flags);
314323 return;
315324 }
@@ -342,6 +351,11 @@ static void rcu_read_unlock_special(struct task_struct *t) · 26 unmodified lines
342351 #ifdef CONFIG_RCU_BOOST
343352 if (&t->rcu_node_entry == rnp->boost_tasks)
344353 rnp->boost_tasks = np;
354+ /* Snapshot and clear ->rcu_boosted with rcu_node lock held. */
355+ if (t->rcu_boosted) {
356+ special |= RCU_READ_UNLOCK_BOOSTED;
357+ t->rcu_boosted = 0;
358+ }
345359 #endif /* #ifdef CONFIG_RCU_BOOST */
346360 t->rcu_blocked_node = NULL;
347361
@@ -358,7 +372,6 @@ static void rcu_read_unlock_special(struct task_struct *t) · 10 unmodified lines
358372 #ifdef CONFIG_RCU_BOOST
359373 /* Unboost if we were boosted. */
360374 if (special & RCU_READ_UNLOCK_BOOSTED) {
361− t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BOOSTED;
362375 rt_mutex_unlock(t->rcu_boost_mutex);
363376 t->rcu_boost_mutex = NULL;
364377 }
@@ -387,13 +400,22 @@ void __rcu_read_unlock(void) · 22 unmodified lines
387400 struct task_struct *t = current;
388401
389402 barrier(); /* needed if we ever invoke rcu_read_unlock in rcutree.c */
390− --t->rcu_read_lock_nesting;
391− barrier(); /* decrement before load of ->rcu_read_unlock_special */
392− if (t->rcu_read_lock_nesting == 0 &&
393− unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
394− rcu_read_unlock_special(t);
403+ if (t->rcu_read_lock_nesting != 1)
404+ --t->rcu_read_lock_nesting;
405+ else {
406+ t->rcu_read_lock_nesting = INT_MIN;
407+ barrier(); /* assign before ->rcu_read_unlock_special load */
408+ if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
409+ rcu_read_unlock_special(t);
410+ barrier(); /* ->rcu_read_unlock_special load before assign */
411+ t->rcu_read_lock_nesting = 0;
412+ }
395413 #ifdef CONFIG_PROVE_LOCKING
396− WARN_ON_ONCE(ACCESS_ONCE(t->rcu_read_lock_nesting) < 0);
414+ {
415+ int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
416+
417+ WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
418+ }
397419 #endif /* #ifdef CONFIG_PROVE_LOCKING */
398420 }
399421 EXPORT_SYMBOL_GPL(__rcu_read_unlock);
@@ -589,7 +611,8 @@ static void rcu_preempt_check_callbacks(int cpu) · 189 unmodified lines
589611 rcu_preempt_qs(cpu);
590612 return;
591613 }
592− if (per_cpu(rcu_preempt_data, cpu).qs_pending)
614+ if (t->rcu_read_lock_nesting > 0 &&
615+ per_cpu(rcu_preempt_data, cpu).qs_pending)
593616 t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
594617 }
595618
@@ -695,9 +718,12 @@ static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp) · 99 unmodified lines
695718
696719 raw_spin_lock_irqsave(&rnp->lock, flags);
697720 for (;;) {
698− if (!sync_rcu_preempt_exp_done(rnp))
721+ if (!sync_rcu_preempt_exp_done(rnp)) {
722+ raw_spin_unlock_irqrestore(&rnp->lock, flags);
699723 break;
724+ }
700725 if (rnp->parent == NULL) {
726+ raw_spin_unlock_irqrestore(&rnp->lock, flags);
701727 wake_up(&sync_rcu_preempt_exp_wq);
702728 break;
703729 }
@@ -707,7 +733,6 @@ static void rcu_report_exp_rnp(struct rcu_state *rsp, struct rcu_node *rnp) · 3 unmodified lines
707733 raw_spin_lock(&rnp->lock); /* irqs already disabled */
708734 rnp->expmask &= ~mask;
709735 }
710− raw_spin_unlock_irqrestore(&rnp->lock, flags);
711736 }
712737
713738 /*
@@ -1174,7 +1199,7 @@ static int rcu_boost(struct rcu_node *rnp) · 460 unmodified lines
11741199 t = container_of(tb, struct task_struct, rcu_node_entry);
11751200 rt_mutex_init_proxy_locked(&mtx, t);
11761201 t->rcu_boost_mutex = &mtx;
1177− t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BOOSTED;
1202+ t->rcu_boosted = 1;
11781203 raw_spin_unlock_irqrestore(&rnp->lock, flags);
11791204 rt_mutex_lock(&mtx); /* Side effect: boosts task t's priority. */
11801205 rt_mutex_unlock(&mtx); /* Keep lockdep happy. */
kernel/sched.c+38−6
@@ -2544,13 +2544,9 @@ static int ttwu_remote(struct task_struct *p, int wake_flags) · 2543 unmodified lines
25442544 }
25452545
25462546 #ifdef CONFIG_SMP
2547−static void sched_ttwu_pending(void)
2547+static void sched_ttwu_do_pending(struct task_struct *list)
25482548 {
25492549 struct rq *rq = this_rq();
2550− struct task_struct *list = xchg(&rq->wake_list, NULL);
2551−
2552− if (!list)
2553− return;
25542550
25552551 raw_spin_lock(&rq->lock);
25562552
@@ -2563,9 +2559,45 @@ static void sched_ttwu_pending(void) · 6 unmodified lines
25632559 raw_spin_unlock(&rq->lock);
25642560 }
25652561
2562+#ifdef CONFIG_HOTPLUG_CPU
2563+
2564+static void sched_ttwu_pending(void)
2565+{
2566+ struct rq *rq = this_rq();
2567+ struct task_struct *list = xchg(&rq->wake_list, NULL);
2568+
2569+ if (!list)
2570+ return;
2571+
2572+ sched_ttwu_do_pending(list);
2573+}
2574+
2575+#endif /* CONFIG_HOTPLUG_CPU */
2576+
25662577 void scheduler_ipi(void)
25672578 {
2568− sched_ttwu_pending();
2579+ struct rq *rq = this_rq();
2580+ struct task_struct *list = xchg(&rq->wake_list, NULL);
2581+
2582+ if (!list)
2583+ return;
2584+
2585+ /*
2586+ * Not all reschedule IPI handlers call irq_enter/irq_exit, since
2587+ * traditionally all their work was done from the interrupt return
2588+ * path. Now that we actually do some work, we need to make sure
2589+ * we do call them.
2590+ *
2591+ * Some archs already do call them, luckily irq_enter/exit nest
2592+ * properly.
2593+ *
2594+ * Arguably we should visit all archs and update all handlers,
2595+ * however a fair share of IPIs are still resched only so this would
2596+ * somewhat pessimize the simple resched case.
2597+ */
2598+ irq_enter();
2599+ sched_ttwu_do_pending(list);
2600+ irq_exit();
25692601 }
25702602
25712603 static void ttwu_queue_remote(struct task_struct *p, int cpu)
kernel/signal.c+13−6
@@ -1178,18 +1178,25 @@ struct sighand_struct *__lock_task_sighand(struct task_struct *tsk, · 1177 unmodified lines
11781178 {
11791179 struct sighand_struct *sighand;
11801180
1181− rcu_read_lock();
11821181 for (;;) {
1182+ local_irq_save(*flags);
1183+ rcu_read_lock();
11831184 sighand = rcu_dereference(tsk->sighand);
1184− if (unlikely(sighand == NULL))
1185+ if (unlikely(sighand == NULL)) {
1186+ rcu_read_unlock();
1187+ local_irq_restore(*flags);
11851188 break;
1189+ }
11861190
1187− spin_lock_irqsave(&sighand->siglock, *flags);
1188− if (likely(sighand == tsk->sighand))
1191+ spin_lock(&sighand->siglock);
1192+ if (likely(sighand == tsk->sighand)) {
1193+ rcu_read_unlock();
11891194 break;
1190− spin_unlock_irqrestore(&sighand->siglock, *flags);
1195+ }
1196+ spin_unlock(&sighand->siglock);
1197+ rcu_read_unlock();
1198+ local_irq_restore(*flags);
11911199 }
1192− rcu_read_unlock();
11931200
11941201 return sighand;
11951202 }
kernel/softirq.c+10−2
@@ -315,16 +315,24 @@ static inline void invoke_softirq(void) · 314 unmodified lines
315315 {
316316 if (!force_irqthreads)
317317 __do_softirq();
318− else
318+ else {
319+ __local_bh_disable((unsigned long)__builtin_return_address(0),
320+ SOFTIRQ_OFFSET);
319321 wakeup_softirqd();
322+ __local_bh_enable(SOFTIRQ_OFFSET);
323+ }
320324 }
321325 #else
322326 static inline void invoke_softirq(void)
323327 {
324328 if (!force_irqthreads)
325329 do_softirq();
326− else
330+ else {
331+ __local_bh_disable((unsigned long)__builtin_return_address(0),
332+ SOFTIRQ_OFFSET);
327333 wakeup_softirqd();
334+ __local_bh_enable(SOFTIRQ_OFFSET);
335+ }
328336 }
329337 #endif
330338