Merge branch 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking fix from Thomas Gleixner: "A single commit, which makes the rtmutex.wait_lock an irq safe lock. This prevents a potential deadlock which can be triggered by the rcu boosting code from rcu_read_unlock()" * 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: rtmutex: Make wait_lock irq safe

Linus Torvalds committed 10 years ago

Commit bbfb2392 parents 30e4c9a b4abf912 files · +81 −72Browse files

Showing the changes to kernel (2 of 2 files). Show all

Unified · Split

2 files

kernel/futex.c +9 −9
kernel/locking/rtmutex.c +72 −63

kernel/futex.c+9−9

@@ -1191,7 +1191,7 @@ static int wake_futex_pi(u32 __user uaddr, u32 uval, struct futex_q this, · 1190 unmodified lines
1191	if (pi_state->owner != current)	1191	if (pi_state->owner != current)
1192	return -EINVAL;	1192	return -EINVAL;
1193		1193
1194	raw_spin_lock(&pi_state->pi_mutex.wait_lock);	1194	raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
1195	new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);	1195	new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);
1196		1196
1197	/*	1197	/*
@@ -1217,22 +1217,22 @@ static int wake_futex_pi(u32 __user uaddr, u32 uval, struct futex_q this, · 19 unmodified lines
1217	else if (curval != uval)	1217	else if (curval != uval)
1218	ret = -EINVAL;	1218	ret = -EINVAL;
1219	if (ret) {	1219	if (ret) {
1220	raw_spin_unlock(&pi_state->pi_mutex.wait_lock);	1220	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1221	return ret;	1221	return ret;
1222	}	1222	}
1223		1223
1224	raw_spin_lock_irq(&pi_state->owner->pi_lock);	1224	raw_spin_lock(&pi_state->owner->pi_lock);
1225	WARN_ON(list_empty(&pi_state->list));	1225	WARN_ON(list_empty(&pi_state->list));
1226	list_del_init(&pi_state->list);	1226	list_del_init(&pi_state->list);
1227	raw_spin_unlock_irq(&pi_state->owner->pi_lock);	1227	raw_spin_unlock(&pi_state->owner->pi_lock);
1228		1228
1229	raw_spin_lock_irq(&new_owner->pi_lock);	1229	raw_spin_lock(&new_owner->pi_lock);
1230	WARN_ON(!list_empty(&pi_state->list));	1230	WARN_ON(!list_empty(&pi_state->list));
1231	list_add(&pi_state->list, &new_owner->pi_state_list);	1231	list_add(&pi_state->list, &new_owner->pi_state_list);
1232	pi_state->owner = new_owner;	1232	pi_state->owner = new_owner;
1233	raw_spin_unlock_irq(&new_owner->pi_lock);	1233	raw_spin_unlock(&new_owner->pi_lock);
1234		1234
1235	raw_spin_unlock(&pi_state->pi_mutex.wait_lock);	1235	raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1236		1236
1237	deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);	1237	deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);
1238		1238
@@ -2127,11 +2127,11 @@ static int fixup_owner(u32 __user uaddr, struct futex_q q, int locked) · 888 unmodified lines
2127	* we returned due to timeout or signal without taking the	2127	* we returned due to timeout or signal without taking the
2128	* rt_mutex. Too late.	2128	* rt_mutex. Too late.
2129	*/	2129	*/
2130	raw_spin_lock(&q->pi_state->pi_mutex.wait_lock);	2130	raw_spin_lock_irq(&q->pi_state->pi_mutex.wait_lock);
2131	owner = rt_mutex_owner(&q->pi_state->pi_mutex);	2131	owner = rt_mutex_owner(&q->pi_state->pi_mutex);
2132	if (!owner)	2132	if (!owner)
2133	owner = rt_mutex_next_owner(&q->pi_state->pi_mutex);	2133	owner = rt_mutex_next_owner(&q->pi_state->pi_mutex);
2134	raw_spin_unlock(&q->pi_state->pi_mutex.wait_lock);	2134	raw_spin_unlock_irq(&q->pi_state->pi_mutex.wait_lock);
2135	ret = fixup_pi_state_owner(uaddr, q, owner);	2135	ret = fixup_pi_state_owner(uaddr, q, owner);
2136	goto out;	2136	goto out;
2137	}	2137	}

kernel/locking/rtmutex.c+72−63

  * 2) Drop lock->wait_lock
  * 3) Try to unlock the lock with cmpxchg
  */
-static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock)
+static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
+					unsigned long flags)
 	__releases(lock->wait_lock)
 {
 	struct task_struct *owner = rt_mutex_owner(lock);
 	clear_rt_mutex_waiters(lock);
-	raw_spin_unlock(&lock->wait_lock);
+	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
 	/*
 	 * If a new waiter comes in between the unlock and the cmpxchg
 	 * we have two situations:
 /*
  * Simple slow path only version: lock->owner is protected by lock->wait_lock.
  */
-static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock)
+static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
+					unsigned long flags)
 	__releases(lock->wait_lock)
 {
 	lock->owner = NULL;
-	raw_spin_unlock(&lock->wait_lock);
+	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
 	return true;
 }
 #endif
 	int ret = 0, depth = 0;
 	struct rt_mutex *lock;
 	bool detect_deadlock;
-	unsigned long flags;
 	bool requeue = true;
 	detect_deadlock = rt_mutex_cond_detect_deadlock(orig_waiter, chwalk);
 	/*
 	 * [1] Task cannot go away as we did a get_task() before !
 	 */
-	raw_spin_lock_irqsave(&task->pi_lock, flags);
+	raw_spin_lock_irq(&task->pi_lock);
 	/*
 	 * [2] Get the waiter on which @task is blocked on.
 	 * operations.
 	 */
 	if (!raw_spin_trylock(&lock->wait_lock)) {
-		raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+		raw_spin_unlock_irq(&task->pi_lock);
 		cpu_relax();
 		goto retry;
 	}
 		/*
 		 * No requeue[7] here. Just release @task [8]
 		 */
-		raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+		raw_spin_unlock(&task->pi_lock);
 		put_task_struct(task);
 		/*
 		 * If there is no owner of the lock, end of chain.
 		 */
 		if (!rt_mutex_owner(lock)) {
-			raw_spin_unlock(&lock->wait_lock);
+			raw_spin_unlock_irq(&lock->wait_lock);
 			return 0;
 		}
 		/* [10] Grab the next task, i.e. owner of @lock */
 		task = rt_mutex_owner(lock);
 		get_task_struct(task);
-		raw_spin_lock_irqsave(&task->pi_lock, flags);
+		raw_spin_lock(&task->pi_lock);
 		/*
 		 * No requeue [11] here. We just do deadlock detection.
 		top_waiter = rt_mutex_top_waiter(lock);
 		/* [13] Drop locks */
-		raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+		raw_spin_unlock(&task->pi_lock);
-		raw_spin_unlock(&lock->wait_lock);
+		raw_spin_unlock_irq(&lock->wait_lock);
 		/* If owner is not blocked, end of chain. */
 		if (!next_lock)
 	rt_mutex_enqueue(lock, waiter);
 	/* [8] Release the task */
-	raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+	raw_spin_unlock(&task->pi_lock);
 	put_task_struct(task);
 	/*
 		 */
 		if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))
 			wake_up_process(rt_mutex_top_waiter(lock)->task);
-		raw_spin_unlock(&lock->wait_lock);
+		raw_spin_unlock_irq(&lock->wait_lock);
 		return 0;
 	}
 	/* [10] Grab the next task, i.e. the owner of @lock */
 	task = rt_mutex_owner(lock);
 	get_task_struct(task);
-	raw_spin_lock_irqsave(&task->pi_lock, flags);
+	raw_spin_lock(&task->pi_lock);
 	/* [11] requeue the pi waiters if necessary */
 	if (waiter == rt_mutex_top_waiter(lock)) {
 	top_waiter = rt_mutex_top_waiter(lock);
 	/* [13] Drop the locks */
-	raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+	raw_spin_unlock(&task->pi_lock);
-	raw_spin_unlock(&lock->wait_lock);
+	raw_spin_unlock_irq(&lock->wait_lock);
 	/*
 	 * Make the actual exit decisions [12], based on the stored
 	goto again;
  out_unlock_pi:
-	raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+	raw_spin_unlock_irq(&task->pi_lock);
  out_put_task:
 	put_task_struct(task);
 /*
  * Try to take an rt-mutex
  *
- * Must be called with lock->wait_lock held.
+ * Must be called with lock->wait_lock held and interrupts disabled
  *
  * @lock:   The lock to be acquired.
  * @task:   The task which wants to acquire the lock
 static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
 				struct rt_mutex_waiter *waiter)
 {
-	unsigned long flags;
 	/*
 	 * Before testing whether we can acquire @lock, we set the
 	 * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all
 	 * case, but conditionals are more expensive than a redundant
 	 * store.
 	 */
-	raw_spin_lock_irqsave(&task->pi_lock, flags);
+	raw_spin_lock(&task->pi_lock);
 	task->pi_blocked_on = NULL;
 	/*
 	 * Finish the lock acquisition. @task is the new owner. If
 	 */
 	if (rt_mutex_has_waiters(lock))
 		rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));
-	raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+	raw_spin_unlock(&task->pi_lock);
 takeit:
 	/* We got the lock. */
  *
  * Prepare waiter and propagate pi chain
  *
- * This must be called with lock->wait_lock held.
+ * This must be called with lock->wait_lock held and interrupts disabled
  */
 static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
 				   struct rt_mutex_waiter *waiter,
 	struct rt_mutex_waiter *top_waiter = waiter;
 	struct rt_mutex *next_lock;
 	int chain_walk = 0, res;
-	unsigned long flags;
 	/*
 	 * Early deadlock detection. We really don't want the task to
 	if (owner == task)
 		return -EDEADLK;
-	raw_spin_lock_irqsave(&task->pi_lock, flags);
+	raw_spin_lock(&task->pi_lock);
 	__rt_mutex_adjust_prio(task);
 	waiter->task = task;
 	waiter->lock = lock;
 	task->pi_blocked_on = waiter;
-	raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+	raw_spin_unlock(&task->pi_lock);
 	if (!owner)
 		return 0;
-	raw_spin_lock_irqsave(&owner->pi_lock, flags);
+	raw_spin_lock(&owner->pi_lock);
 	if (waiter == rt_mutex_top_waiter(lock)) {
 		rt_mutex_dequeue_pi(owner, top_waiter);
 		rt_mutex_enqueue_pi(owner, waiter);
 	/* Store the lock on which owner is blocked or NULL */
 	next_lock = task_blocked_on_lock(owner);
-	raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
+	raw_spin_unlock(&owner->pi_lock);
 	/*
 	 * Even if full deadlock detection is on, if the owner is not
 	 * blocked itself, we can avoid finding this out in the chain
 	 */
 	get_task_struct(owner);
-	raw_spin_unlock(&lock->wait_lock);
+	raw_spin_unlock_irq(&lock->wait_lock);
 	res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,
 					 next_lock, waiter, task);
-	raw_spin_lock(&lock->wait_lock);
+	raw_spin_lock_irq(&lock->wait_lock);
 	return res;
 }
  * Remove the top waiter from the current tasks pi waiter tree and
  * queue it up.
  *
- * Called with lock->wait_lock held.
+ * Called with lock->wait_lock held and interrupts disabled.
  */
 static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
 				    struct rt_mutex *lock)
 {
 	struct rt_mutex_waiter *waiter;
-	unsigned long flags;
-	raw_spin_lock_irqsave(&current->pi_lock, flags);
+	raw_spin_lock(&current->pi_lock);
 	waiter = rt_mutex_top_waiter(lock);
 	 */
 	lock->owner = (void *) RT_MUTEX_HAS_WAITERS;
-	raw_spin_unlock_irqrestore(&current->pi_lock, flags);
+	raw_spin_unlock(&current->pi_lock);
 	wake_q_add(wake_q, waiter->task);
 }
 /*
  * Remove a waiter from a lock and give up
  *
- * Must be called with lock->wait_lock held and
+ * Must be called with lock->wait_lock held and interrupts disabled. I must
  * have just failed to try_to_take_rt_mutex().
  */
 static void remove_waiter(struct rt_mutex *lock,
 	bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
 	struct task_struct *owner = rt_mutex_owner(lock);
 	struct rt_mutex *next_lock;
-	unsigned long flags;
-	raw_spin_lock_irqsave(&current->pi_lock, flags);
+	raw_spin_lock(&current->pi_lock);
 	rt_mutex_dequeue(lock, waiter);
 	current->pi_blocked_on = NULL;
-	raw_spin_unlock_irqrestore(&current->pi_lock, flags);
+	raw_spin_unlock(&current->pi_lock);
 	/*
 	 * Only update priority if the waiter was the highest priority
 	if (!owner || !is_top_waiter)
 		return;
-	raw_spin_lock_irqsave(&owner->pi_lock, flags);
+	raw_spin_lock(&owner->pi_lock);
 	rt_mutex_dequeue_pi(owner, waiter);
 	/* Store the lock on which owner is blocked or NULL */
 	next_lock = task_blocked_on_lock(owner);
-	raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
+	raw_spin_unlock(&owner->pi_lock);
 	/*
 	 * Don't walk the chain, if the owner task is not blocked
 	/* gets dropped in rt_mutex_adjust_prio_chain()! */
 	get_task_struct(owner);
-	raw_spin_unlock(&lock->wait_lock);
+	raw_spin_unlock_irq(&lock->wait_lock);
 	rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,
 				   next_lock, NULL, current);
-	raw_spin_lock(&lock->wait_lock);
+	raw_spin_lock_irq(&lock->wait_lock);
 }
 /*
  * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
  * @lock:		 the rt_mutex to take
  * @state:		 the state the task should block in (TASK_INTERRUPTIBLE
- * 			 or TASK_UNINTERRUPTIBLE)
+ *			 or TASK_UNINTERRUPTIBLE)
  * @timeout:		 the pre-initialized and started timer, or NULL for none
  * @waiter:		 the pre-initialized rt_mutex_waiter
  *
- * lock->wait_lock must be held by the caller.
+ * Must be called with lock->wait_lock held and interrupts disabled
  */
 static int __sched
 __rt_mutex_slowlock(struct rt_mutex *lock, int state,
 				break;
 		}
-		raw_spin_unlock(&lock->wait_lock);
+		raw_spin_unlock_irq(&lock->wait_lock);
 		debug_rt_mutex_print_deadlock(waiter);
 		schedule();
-		raw_spin_lock(&lock->wait_lock);
+		raw_spin_lock_irq(&lock->wait_lock);
 		set_current_state(state);
 	}
 		  enum rtmutex_chainwalk chwalk)
 {
 	struct rt_mutex_waiter waiter;
+	unsigned long flags;
 	int ret = 0;
 	debug_rt_mutex_init_waiter(&waiter);
 	RB_CLEAR_NODE(&waiter.pi_tree_entry);
 	RB_CLEAR_NODE(&waiter.tree_entry);
-	raw_spin_lock(&lock->wait_lock);
+	/*
+	 * Technically we could use raw_spin_[un]lock_irq() here, but this can
+	 * be called in early boot if the cmpxchg() fast path is disabled
+	 * (debug, no architecture support). In this case we will acquire the
+	 * rtmutex with lock->wait_lock held. But we cannot unconditionally
+	 * enable interrupts in that early boot case. So we need to use the
+	 * irqsave/restore variants.
+	 */
+	raw_spin_lock_irqsave(&lock->wait_lock, flags);
 	/* Try to acquire the lock again: */
 	if (try_to_take_rt_mutex(lock, current, NULL)) {
-		raw_spin_unlock(&lock->wait_lock);
+		raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
 		return 0;
 	}
 	 */
 	fixup_rt_mutex_waiters(lock);
-	raw_spin_unlock(&lock->wait_lock);
+	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
 	/* Remove pending timer: */
 	if (unlikely(timeout))
  */
 static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)
 {
+	unsigned long flags;
 	int ret;
 	/*
 		return 0;
 	/*
-	 * The mutex has currently no owner. Lock the wait lock and
+	 * The mutex has currently no owner. Lock the wait lock and try to
-	 * try to acquire the lock.
+	 * acquire the lock. We use irqsave here to support early boot calls.
 	 */
-	raw_spin_lock(&lock->wait_lock);
+	raw_spin_lock_irqsave(&lock->wait_lock, flags);
 	ret = try_to_take_rt_mutex(lock, current, NULL);
 	 */
 	fixup_rt_mutex_waiters(lock);
-	raw_spin_unlock(&lock->wait_lock);
+	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
 	return ret;
 }
 static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock,
 					struct wake_q_head *wake_q)
 {
-	raw_spin_lock(&lock->wait_lock);
+	unsigned long flags;
+	/* irqsave required to support early boot calls */
+	raw_spin_lock_irqsave(&lock->wait_lock, flags);
 	debug_rt_mutex_unlock(lock);
 	 */
 	while (!rt_mutex_has_waiters(lock)) {
 		/* Drops lock->wait_lock ! */
-		if (unlock_rt_mutex_safe(lock) == true)
+		if (unlock_rt_mutex_safe(lock, flags) == true)
 			return false;
 		/* Relock the rtmutex and try again */
-		raw_spin_lock(&lock->wait_lock);
+		raw_spin_lock_irqsave(&lock->wait_lock, flags);
 	}
 	/*
 	 */
 	mark_wakeup_next_waiter(wake_q, lock);
-	raw_spin_unlock(&lock->wait_lock);
+	raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
 	/* check PI boosting */
 	return true;
 {
 	int ret;
-	raw_spin_lock(&lock->wait_lock);
+	raw_spin_lock_irq(&lock->wait_lock);
 	if (try_to_take_rt_mutex(lock, task, NULL)) {
-		raw_spin_unlock(&lock->wait_lock);
+		raw_spin_unlock_irq(&lock->wait_lock);
 		return 1;
 	}
 	if (unlikely(ret))
 		remove_waiter(lock, waiter);
-	raw_spin_unlock(&lock->wait_lock);
+	raw_spin_unlock_irq(&lock->wait_lock);
 	debug_rt_mutex_print_deadlock(waiter);
 {
 	int ret;
-	raw_spin_lock(&lock->wait_lock);
+	raw_spin_lock_irq(&lock->wait_lock);
 	set_current_state(TASK_INTERRUPTIBLE);
 	 */
 	fixup_rt_mutex_waiters(lock);
-	raw_spin_unlock(&lock->wait_lock);
+	raw_spin_unlock_irq(&lock->wait_lock);
 	return ret;
 }