Merge branch 'core-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

* 'core-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: signal: align __lock_task_sighand() irq disabling and RCU softirq,rcu: Inform RCU of irq_exit() activity sched: Add irq_{enter,exit}() to scheduler_ipi() rcu: protect __rcu_read_unlock() against scheduler-using irq handlers rcu: Streamline code produced by __rcu_read_unlock() rcu: Fix RCU_BOOST race handling current->rcu_read_unlock_special rcu: decrease rcu_report_exp_rnp coupling with scheduler

Linus Torvalds committed 15 years ago

Commit cf6ace12 parents acc11ea d1e9ae45 files · +103 −28Browse files

Unified · Split

5 files

include/linux/sched.h +3 −0
kernel/rcutree_plugin.h +39 −14
kernel/sched.c +38 −6
kernel/signal.c +13 −6
kernel/softirq.c +10 −2

include/linux/sched.h+3−0

 #ifdef CONFIG_PREEMPT_RCU
 	int rcu_read_lock_nesting;
 	char rcu_read_unlock_special;
+#if defined(CONFIG_RCU_BOOST) && defined(CONFIG_TREE_PREEMPT_RCU)
+	int rcu_boosted;
+#endif /* #if defined(CONFIG_RCU_BOOST) && defined(CONFIG_TREE_PREEMPT_RCU) */
 	struct list_head rcu_node_entry;
 #endif /* #ifdef CONFIG_PREEMPT_RCU */
 #ifdef CONFIG_TREE_PREEMPT_RCU

kernel/rcutree_plugin.h+39−14

 DEFINE_PER_CPU(struct rcu_data, rcu_preempt_data);
 static struct rcu_state *rcu_state = &rcu_preempt_state;
+static void rcu_read_unlock_special(struct task_struct *t);
 static int rcu_preempted_readers_exp(struct rcu_node *rnp);
 /*
 	struct rcu_data *rdp;
 	struct rcu_node *rnp;
-	if (t->rcu_read_lock_nesting &&
+	if (t->rcu_read_lock_nesting > 0 &&
 	    (t->rcu_read_unlock_special & RCU_READ_UNLOCK_BLOCKED) == 0) {
 		/* Possibly blocking in an RCU read-side critical section. */
 				rnp->gp_tasks = &t->rcu_node_entry;
 		}
 		raw_spin_unlock_irqrestore(&rnp->lock, flags);
+	} else if (t->rcu_read_lock_nesting < 0 &&
+		   t->rcu_read_unlock_special) {
+		/*
+		 * Complete exit from RCU read-side critical section on
+		 * behalf of preempted instance of __rcu_read_unlock().
+		 */
+		rcu_read_unlock_special(t);
 	}
 	/*
  * notify RCU core processing or task having blocked during the RCU
  * read-side critical section.
  */
-static void rcu_read_unlock_special(struct task_struct *t)
+static noinline void rcu_read_unlock_special(struct task_struct *t)
 {
 	int empty;
 	int empty_exp;
 	}
 	/* Hardware IRQ handlers cannot block. */
-	if (in_irq()) {
+	if (in_irq() || in_serving_softirq()) {
 		local_irq_restore(flags);
 		return;
 	}
 #ifdef CONFIG_RCU_BOOST
 		if (&t->rcu_node_entry == rnp->boost_tasks)
 			rnp->boost_tasks = np;
+		/* Snapshot and clear ->rcu_boosted with rcu_node lock held. */
+		if (t->rcu_boosted) {
+			special |= RCU_READ_UNLOCK_BOOSTED;
+			t->rcu_boosted = 0;
+		}
 #endif /* #ifdef CONFIG_RCU_BOOST */
 		t->rcu_blocked_node = NULL;
 #ifdef CONFIG_RCU_BOOST
 		/* Unboost if we were boosted. */
 		if (special & RCU_READ_UNLOCK_BOOSTED) {
-			t->rcu_read_unlock_special &= ~RCU_READ_UNLOCK_BOOSTED;
 			rt_mutex_unlock(t->rcu_boost_mutex);
 			t->rcu_boost_mutex = NULL;
 		}
 	struct task_struct *t = current;
 	barrier();  /* needed if we ever invoke rcu_read_unlock in rcutree.c */
-	--t->rcu_read_lock_nesting;
+	if (t->rcu_read_lock_nesting != 1)
-	barrier();  /* decrement before load of ->rcu_read_unlock_special */
+		--t->rcu_read_lock_nesting;
-	if (t->rcu_read_lock_nesting == 0 &&
+	else {
-	    unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
+		t->rcu_read_lock_nesting = INT_MIN;
-		rcu_read_unlock_special(t);
+		barrier();  /* assign before ->rcu_read_unlock_special load */
+		if (unlikely(ACCESS_ONCE(t->rcu_read_unlock_special)))
+			rcu_read_unlock_special(t);
+		barrier();  /* ->rcu_read_unlock_special load before assign */
+		t->rcu_read_lock_nesting = 0;
+	}
 #ifdef CONFIG_PROVE_LOCKING
-	WARN_ON_ONCE(ACCESS_ONCE(t->rcu_read_lock_nesting) < 0);
+	{
+		int rrln = ACCESS_ONCE(t->rcu_read_lock_nesting);
+		WARN_ON_ONCE(rrln < 0 && rrln > INT_MIN / 2);
+	}
 #endif /* #ifdef CONFIG_PROVE_LOCKING */
 }
 EXPORT_SYMBOL_GPL(__rcu_read_unlock);
 		rcu_preempt_qs(cpu);
 		return;
 	}
-	if (per_cpu(rcu_preempt_data, cpu).qs_pending)
+	if (t->rcu_read_lock_nesting > 0 &&
+	    per_cpu(rcu_preempt_data, cpu).qs_pending)
 		t->rcu_read_unlock_special |= RCU_READ_UNLOCK_NEED_QS;
 }
 	raw_spin_lock_irqsave(&rnp->lock, flags);
 	for (;;) {
-		if (!sync_rcu_preempt_exp_done(rnp))
+		if (!sync_rcu_preempt_exp_done(rnp)) {
+			raw_spin_unlock_irqrestore(&rnp->lock, flags);
 			break;
+		}
 		if (rnp->parent == NULL) {
+			raw_spin_unlock_irqrestore(&rnp->lock, flags);
 			wake_up(&sync_rcu_preempt_exp_wq);
 			break;
 		}
 		raw_spin_lock(&rnp->lock); /* irqs already disabled */
 		rnp->expmask &= ~mask;
 	}
-	raw_spin_unlock_irqrestore(&rnp->lock, flags);
 }
 /*
 	t = container_of(tb, struct task_struct, rcu_node_entry);
 	rt_mutex_init_proxy_locked(&mtx, t);
 	t->rcu_boost_mutex = &mtx;
-	t->rcu_read_unlock_special |= RCU_READ_UNLOCK_BOOSTED;
+	t->rcu_boosted = 1;
 	raw_spin_unlock_irqrestore(&rnp->lock, flags);
 	rt_mutex_lock(&mtx);  /* Side effect: boosts task t's priority. */
 	rt_mutex_unlock(&mtx);  /* Keep lockdep happy. */

kernel/sched.c+38−6

@@ -2544,13 +2544,9 @@ static int ttwu_remote(struct task_struct *p, int wake_flags) · 2543 unmodified lines
2544	}	2544	}
2545		2545
2546	#ifdef CONFIG_SMP	2546	#ifdef CONFIG_SMP
2547	static void sched_ttwu_pending(void)	2547	static void sched_ttwu_do_pending(struct task_struct *list)
2548	{	2548	{
2549	struct rq *rq = this_rq();	2549	struct rq *rq = this_rq();
2550	struct task_struct *list = xchg(&rq->wake_list, NULL);
2551
2552	if (!list)
2553	return;
2554		2550
2555	raw_spin_lock(&rq->lock);	2551	raw_spin_lock(&rq->lock);
2556		2552
@@ -2563,9 +2559,45 @@ static void sched_ttwu_pending(void) · 6 unmodified lines
2563	raw_spin_unlock(&rq->lock);	2559	raw_spin_unlock(&rq->lock);
2564	}	2560	}
2565		2561
		2562	#ifdef CONFIG_HOTPLUG_CPU
		2563
		2564	static void sched_ttwu_pending(void)
		2565	{
		2566	struct rq *rq = this_rq();
		2567	struct task_struct *list = xchg(&rq->wake_list, NULL);
		2568
		2569	if (!list)
		2570	return;
		2571
		2572	sched_ttwu_do_pending(list);
		2573	}
		2574
		2575	#endif /* CONFIG_HOTPLUG_CPU */
		2576
2566	void scheduler_ipi(void)	2577	void scheduler_ipi(void)
2567	{	2578	{
2568	sched_ttwu_pending();	2579	struct rq *rq = this_rq();
		2580	struct task_struct *list = xchg(&rq->wake_list, NULL);
		2581
		2582	if (!list)
		2583	return;
		2584
		2585	/*
		2586	* Not all reschedule IPI handlers call irq_enter/irq_exit, since
		2587	* traditionally all their work was done from the interrupt return
		2588	* path. Now that we actually do some work, we need to make sure
		2589	* we do call them.
		2590	*
		2591	* Some archs already do call them, luckily irq_enter/exit nest
		2592	* properly.
		2593	*
		2594	* Arguably we should visit all archs and update all handlers,
		2595	* however a fair share of IPIs are still resched only so this would
		2596	* somewhat pessimize the simple resched case.
		2597	*/
		2598	irq_enter();
		2599	sched_ttwu_do_pending(list);
		2600	irq_exit();
2569	}	2601	}
2570		2602
2571	static void ttwu_queue_remote(struct task_struct *p, int cpu)	2603	static void ttwu_queue_remote(struct task_struct *p, int cpu)

kernel/signal.c+13−6

 {
 	struct sighand_struct *sighand;
-	rcu_read_lock();
 	for (;;) {
+		local_irq_save(*flags);
+		rcu_read_lock();
 		sighand = rcu_dereference(tsk->sighand);
-		if (unlikely(sighand == NULL))
+		if (unlikely(sighand == NULL)) {
+			rcu_read_unlock();
+			local_irq_restore(*flags);
 			break;
+		}
-		spin_lock_irqsave(&sighand->siglock, *flags);
+		spin_lock(&sighand->siglock);
-		if (likely(sighand == tsk->sighand))
+		if (likely(sighand == tsk->sighand)) {
+			rcu_read_unlock();
 			break;
-		spin_unlock_irqrestore(&sighand->siglock, *flags);
+		}
+		spin_unlock(&sighand->siglock);
+		rcu_read_unlock();
+		local_irq_restore(*flags);
 	}
-	rcu_read_unlock();
 	return sighand;
 }

kernel/softirq.c+10−2

 {
 	if (!force_irqthreads)
 		__do_softirq();
-	else
+	else {
+		__local_bh_disable((unsigned long)__builtin_return_address(0),
+				SOFTIRQ_OFFSET);
 		wakeup_softirqd();
+		__local_bh_enable(SOFTIRQ_OFFSET);
+	}
 }
 #else
 static inline void invoke_softirq(void)
 {
 	if (!force_irqthreads)
 		do_softirq();
-	else
+	else {
+		__local_bh_disable((unsigned long)__builtin_return_address(0),
+				SOFTIRQ_OFFSET);
 		wakeup_softirqd();
+		__local_bh_enable(SOFTIRQ_OFFSET);
+	}
 }
 #endif