History

Merge branch 'locking-urgent-for-linus.patch' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull rtmutex fixes from Thomas Gleixner: "Another three patches to make the rtmutex code more robust. That's the last urgent fallout from the big futex/rtmutex investigation" * 'locking-urgent-for-linus.patch' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: rtmutex: Plug slow unlock race rtmutex: Detect changes in the pi lock chain rtmutex: Handle deadlock detection smarter

Linus Torvalds committed 12 years ago

Commit 7b08d612 parents 7a8e9c8 27e35713 files · +218 −35Browse files

Unified · Split

3 files
  1. kernel/locking/rtmutex-debug.h +5 −0
  2. kernel/locking/rtmutex.c +208 −35
  3. kernel/locking/rtmutex.h +5 −0
kernel/locking/rtmutex-debug.h+5−0
@@ -31,3 +31,8 @@ static inline int debug_rt_mutex_detect_deadlock(struct rt_mutex_waiter *waiter, · 30 unmodified lines
3131 {
3232 return (waiter != NULL);
3333 }
34+
35+static inline void rt_mutex_print_deadlock(struct rt_mutex_waiter *w)
36+{
37+ debug_rt_mutex_print_deadlock(w);
38+}
kernel/locking/rtmutex.c+208−35
@@ -83,6 +83,47 @@ static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) · 82 unmodified lines
8383 owner = *p;
8484 } while (cmpxchg(p, owner, owner | RT_MUTEX_HAS_WAITERS) != owner);
8585 }
86+
87+/*
88+ * Safe fastpath aware unlock:
89+ * 1) Clear the waiters bit
90+ * 2) Drop lock->wait_lock
91+ * 3) Try to unlock the lock with cmpxchg
92+ */
93+static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock)
94+ __releases(lock->wait_lock)
95+{
96+ struct task_struct *owner = rt_mutex_owner(lock);
97+
98+ clear_rt_mutex_waiters(lock);
99+ raw_spin_unlock(&lock->wait_lock);
100+ /*
101+ * If a new waiter comes in between the unlock and the cmpxchg
102+ * we have two situations:
103+ *
104+ * unlock(wait_lock);
105+ * lock(wait_lock);
106+ * cmpxchg(p, owner, 0) == owner
107+ * mark_rt_mutex_waiters(lock);
108+ * acquire(lock);
109+ * or:
110+ *
111+ * unlock(wait_lock);
112+ * lock(wait_lock);
113+ * mark_rt_mutex_waiters(lock);
114+ *
115+ * cmpxchg(p, owner, 0) != owner
116+ * enqueue_waiter();
117+ * unlock(wait_lock);
118+ * lock(wait_lock);
119+ * wake waiter();
120+ * unlock(wait_lock);
121+ * lock(wait_lock);
122+ * acquire(lock);
123+ */
124+ return rt_mutex_cmpxchg(lock, owner, NULL);
125+}
126+
86127 #else
87128 # define rt_mutex_cmpxchg(l,c,n) (0)
88129 static inline void mark_rt_mutex_waiters(struct rt_mutex *lock)
@@ -90,6 +131,17 @@ static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) · 1 unmodified line
90131 lock->owner = (struct task_struct *)
91132 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);
92133 }
134+
135+/*
136+ * Simple slow path only version: lock->owner is protected by lock->wait_lock.
137+ */
138+static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock)
139+ __releases(lock->wait_lock)
140+{
141+ lock->owner = NULL;
142+ raw_spin_unlock(&lock->wait_lock);
143+ return true;
144+}
93145 #endif
94146
95147 static inline int
@@ -260,27 +312,36 @@ static void rt_mutex_adjust_prio(struct task_struct *task) · 164 unmodified lines
260312 */
261313 int max_lock_depth = 1024;
262314
315+static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
316+{
317+ return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL;
318+}
319+
263320 /*
264321 * Adjust the priority chain. Also used for deadlock detection.
265322 * Decreases task's usage by one - may thus free the task.
266323 *
267− * @task: the task owning the mutex (owner) for which a chain walk is probably
268− * needed
324+ * @task: the task owning the mutex (owner) for which a chain walk is
325+ * probably needed
269326 * @deadlock_detect: do we have to carry out deadlock detection?
270− * @orig_lock: the mutex (can be NULL if we are walking the chain to recheck
271− * things for a task that has just got its priority adjusted, and
272− * is waiting on a mutex)
327+ * @orig_lock: the mutex (can be NULL if we are walking the chain to recheck
328+ * things for a task that has just got its priority adjusted, and
329+ * is waiting on a mutex)
330+ * @next_lock: the mutex on which the owner of @orig_lock was blocked before
331+ * we dropped its pi_lock. Is never dereferenced, only used for
332+ * comparison to detect lock chain changes.
273333 * @orig_waiter: rt_mutex_waiter struct for the task that has just donated
274− * its priority to the mutex owner (can be NULL in the case
275− * depicted above or if the top waiter is gone away and we are
276− * actually deboosting the owner)
277− * @top_task: the current top waiter
334+ * its priority to the mutex owner (can be NULL in the case
335+ * depicted above or if the top waiter is gone away and we are
336+ * actually deboosting the owner)
337+ * @top_task: the current top waiter
278338 *
279339 * Returns 0 or -EDEADLK.
280340 */
281341 static int rt_mutex_adjust_prio_chain(struct task_struct *task,
282342 int deadlock_detect,
283343 struct rt_mutex *orig_lock,
344+ struct rt_mutex *next_lock,
284345 struct rt_mutex_waiter *orig_waiter,
285346 struct task_struct *top_task)
286347 {
@@ -314,7 +375,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 27 unmodified lines
314375 }
315376 put_task_struct(task);
316377
317− return deadlock_detect ? -EDEADLK : 0;
378+ return -EDEADLK;
318379 }
319380 retry:
320381 /*
@@ -338,6 +399,18 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 17 unmodified lines
338399 if (orig_waiter && !rt_mutex_owner(orig_lock))
339400 goto out_unlock_pi;
340401
402+ /*
403+ * We dropped all locks after taking a refcount on @task, so
404+ * the task might have moved on in the lock chain or even left
405+ * the chain completely and blocks now on an unrelated lock or
406+ * on @orig_lock.
407+ *
408+ * We stored the lock on which @task was blocked in @next_lock,
409+ * so we can detect the chain change.
410+ */
411+ if (next_lock != waiter->lock)
412+ goto out_unlock_pi;
413+
341414 /*
342415 * Drop out, when the task has no waiters. Note,
343416 * top_waiter can be NULL, when we are in the deboosting
@@ -377,7 +450,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 33 unmodified lines
377450 if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
378451 debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock);
379452 raw_spin_unlock(&lock->wait_lock);
380− ret = deadlock_detect ? -EDEADLK : 0;
453+ ret = -EDEADLK;
381454 goto out_unlock_pi;
382455 }
383456
@@ -422,11 +495,26 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 38 unmodified lines
422495 __rt_mutex_adjust_prio(task);
423496 }
424497
498+ /*
499+ * Check whether the task which owns the current lock is pi
500+ * blocked itself. If yes we store a pointer to the lock for
501+ * the lock chain change detection above. After we dropped
502+ * task->pi_lock next_lock cannot be dereferenced anymore.
503+ */
504+ next_lock = task_blocked_on_lock(task);
505+
425506 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
426507
427508 top_waiter = rt_mutex_top_waiter(lock);
428509 raw_spin_unlock(&lock->wait_lock);
429510
511+ /*
512+ * We reached the end of the lock chain. Stop right here. No
513+ * point to go back just to figure that out.
514+ */
515+ if (!next_lock)
516+ goto out_put_task;
517+
430518 if (!detect_deadlock && waiter != top_waiter)
431519 goto out_put_task;
432520
@@ -536,8 +624,9 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 103 unmodified lines
536624 {
537625 struct task_struct *owner = rt_mutex_owner(lock);
538626 struct rt_mutex_waiter *top_waiter = waiter;
539− unsigned long flags;
627+ struct rt_mutex *next_lock;
540628 int chain_walk = 0, res;
629+ unsigned long flags;
541630
542631 /*
543632 * Early deadlock detection. We really don't want the task to
@@ -548,7 +637,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 4 unmodified lines
548637 * which is wrong, as the other waiter is not in a deadlock
549638 * situation.
550639 */
551− if (detect_deadlock && owner == task)
640+ if (owner == task)
552641 return -EDEADLK;
553642
554643 raw_spin_lock_irqsave(&task->pi_lock, flags);
@@ -569,20 +658,28 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 14 unmodified lines
569658 if (!owner)
570659 return 0;
571660
661+ raw_spin_lock_irqsave(&owner->pi_lock, flags);
572662 if (waiter == rt_mutex_top_waiter(lock)) {
573− raw_spin_lock_irqsave(&owner->pi_lock, flags);
574663 rt_mutex_dequeue_pi(owner, top_waiter);
575664 rt_mutex_enqueue_pi(owner, waiter);
576665
577666 __rt_mutex_adjust_prio(owner);
578667 if (owner->pi_blocked_on)
579668 chain_walk = 1;
580− raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
581− }
582− else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock))
669+ } else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock)) {
583670 chain_walk = 1;
671+ }
584672
585− if (!chain_walk)
673+ /* Store the lock on which owner is blocked or NULL */
674+ next_lock = task_blocked_on_lock(owner);
675+
676+ raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
677+ /*
678+ * Even if full deadlock detection is on, if the owner is not
679+ * blocked itself, we can avoid finding this out in the chain
680+ * walk.
681+ */
682+ if (!chain_walk || !next_lock)
586683 return 0;
587684
588685 /*
@@ -594,8 +691,8 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 5 unmodified lines
594691
595692 raw_spin_unlock(&lock->wait_lock);
596693
597− res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock, waiter,
598− task);
694+ res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock,
695+ next_lock, waiter, task);
599696
600697 raw_spin_lock(&lock->wait_lock);
601698
@@ -605,7 +702,8 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 3 unmodified lines
605702 /*
606703 * Wake up the next waiter on the lock.
607704 *
608− * Remove the top waiter from the current tasks waiter list and wake it up.
705+ * Remove the top waiter from the current tasks pi waiter list and
706+ * wake it up.
609707 *
610708 * Called with lock->wait_lock held.
611709 */
@@ -626,10 +724,23 @@ static void wakeup_next_waiter(struct rt_mutex *lock) · 14 unmodified lines
626724 */
627725 rt_mutex_dequeue_pi(current, waiter);
628726
629− rt_mutex_set_owner(lock, NULL);
727+ /*
728+ * As we are waking up the top waiter, and the waiter stays
729+ * queued on the lock until it gets the lock, this lock
730+ * obviously has waiters. Just set the bit here and this has
731+ * the added benefit of forcing all new tasks into the
732+ * slow path making sure no task of lower priority than
733+ * the top waiter can steal this lock.
734+ */
735+ lock->owner = (void *) RT_MUTEX_HAS_WAITERS;
630736
631737 raw_spin_unlock_irqrestore(&current->pi_lock, flags);
632738
739+ /*
740+ * It's safe to dereference waiter as it cannot go away as
741+ * long as we hold lock->wait_lock. The waiter task needs to
742+ * acquire it in order to dequeue the waiter.
743+ */
633744 wake_up_process(waiter->task);
634745 }
635746
@@ -644,8 +755,8 @@ static void remove_waiter(struct rt_mutex *lock, · 8 unmodified lines
644755 {
645756 int first = (waiter == rt_mutex_top_waiter(lock));
646757 struct task_struct *owner = rt_mutex_owner(lock);
758+ struct rt_mutex *next_lock = NULL;
647759 unsigned long flags;
648− int chain_walk = 0;
649760
650761 raw_spin_lock_irqsave(&current->pi_lock, flags);
651762 rt_mutex_dequeue(lock, waiter);
@@ -669,13 +780,13 @@ static void remove_waiter(struct rt_mutex *lock, · 17 unmodified lines
669780 }
670781 __rt_mutex_adjust_prio(owner);
671782
672− if (owner->pi_blocked_on)
673− chain_walk = 1;
783+ /* Store the lock on which owner is blocked or NULL */
784+ next_lock = task_blocked_on_lock(owner);
674785
675786 raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
676787 }
677788
678− if (!chain_walk)
789+ if (!next_lock)
679790 return;
680791
681792 /* gets dropped in rt_mutex_adjust_prio_chain()! */
@@ -683,7 +794,7 @@ static void remove_waiter(struct rt_mutex *lock, · 1 unmodified line
683794
684795 raw_spin_unlock(&lock->wait_lock);
685796
686− rt_mutex_adjust_prio_chain(owner, 0, lock, NULL, current);
797+ rt_mutex_adjust_prio_chain(owner, 0, lock, next_lock, NULL, current);
687798
688799 raw_spin_lock(&lock->wait_lock);
689800 }
@@ -696,6 +807,7 @@ static void remove_waiter(struct rt_mutex *lock, · 6 unmodified lines
696807 void rt_mutex_adjust_pi(struct task_struct *task)
697808 {
698809 struct rt_mutex_waiter *waiter;
810+ struct rt_mutex *next_lock;
699811 unsigned long flags;
700812
701813 raw_spin_lock_irqsave(&task->pi_lock, flags);
@@ -706,12 +818,13 @@ void rt_mutex_adjust_pi(struct task_struct *task) · 4 unmodified lines
706818 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
707819 return;
708820 }
709−
821+ next_lock = waiter->lock;
710822 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
711823
712824 /* gets dropped in rt_mutex_adjust_prio_chain()! */
713825 get_task_struct(task);
714− rt_mutex_adjust_prio_chain(task, 0, NULL, NULL, task);
826+
827+ rt_mutex_adjust_prio_chain(task, 0, NULL, next_lock, NULL, task);
715828 }
716829
717830 /**
@@ -763,6 +876,26 @@ __rt_mutex_slowlock(struct rt_mutex *lock, int state, · 45 unmodified lines
763876 return ret;
764877 }
765878
879+static void rt_mutex_handle_deadlock(int res, int detect_deadlock,
880+ struct rt_mutex_waiter *w)
881+{
882+ /*
883+ * If the result is not -EDEADLOCK or the caller requested
884+ * deadlock detection, nothing to do here.
885+ */
886+ if (res != -EDEADLOCK || detect_deadlock)
887+ return;
888+
889+ /*
890+ * Yell lowdly and stop the task right here.
891+ */
892+ rt_mutex_print_deadlock(w);
893+ while (1) {
894+ set_current_state(TASK_INTERRUPTIBLE);
895+ schedule();
896+ }
897+}
898+
766899 /*
767900 * Slow path lock function:
768901 */
@@ -802,8 +935,10 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, · 33 unmodified lines
802935
803936 set_current_state(TASK_RUNNING);
804937
805− if (unlikely(ret))
938+ if (unlikely(ret)) {
806939 remove_waiter(lock, &waiter);
940+ rt_mutex_handle_deadlock(ret, detect_deadlock, &waiter);
941+ }
807942
808943 /*
809944 * try_to_take_rt_mutex() sets the waiter bit
@@ -859,12 +994,49 @@ rt_mutex_slowunlock(struct rt_mutex *lock) · 49 unmodified lines
859994
860995 rt_mutex_deadlock_account_unlock(current);
861996
862− if (!rt_mutex_has_waiters(lock)) {
863− lock->owner = NULL;
864− raw_spin_unlock(&lock->wait_lock);
865− return;
997+ /*
998+ * We must be careful here if the fast path is enabled. If we
999+ * have no waiters queued we cannot set owner to NULL here
1000+ * because of:
1001+ *
1002+ * foo->lock->owner = NULL;
1003+ * rtmutex_lock(foo->lock); <- fast path
1004+ * free = atomic_dec_and_test(foo->refcnt);
1005+ * rtmutex_unlock(foo->lock); <- fast path
1006+ * if (free)
1007+ * kfree(foo);
1008+ * raw_spin_unlock(foo->lock->wait_lock);
1009+ *
1010+ * So for the fastpath enabled kernel:
1011+ *
1012+ * Nothing can set the waiters bit as long as we hold
1013+ * lock->wait_lock. So we do the following sequence:
1014+ *
1015+ * owner = rt_mutex_owner(lock);
1016+ * clear_rt_mutex_waiters(lock);
1017+ * raw_spin_unlock(&lock->wait_lock);
1018+ * if (cmpxchg(&lock->owner, owner, 0) == owner)
1019+ * return;
1020+ * goto retry;
1021+ *
1022+ * The fastpath disabled variant is simple as all access to
1023+ * lock->owner is serialized by lock->wait_lock:
1024+ *
1025+ * lock->owner = NULL;
1026+ * raw_spin_unlock(&lock->wait_lock);
1027+ */
1028+ while (!rt_mutex_has_waiters(lock)) {
1029+ /* Drops lock->wait_lock ! */
1030+ if (unlock_rt_mutex_safe(lock) == true)
1031+ return;
1032+ /* Relock the rtmutex and try again */
1033+ raw_spin_lock(&lock->wait_lock);
8661034 }
8671035
1036+ /*
1037+ * The wakeup next waiter path does not suffer from the above
1038+ * race. See the comments there.
1039+ */
8681040 wakeup_next_waiter(lock);
8691041
8701042 raw_spin_unlock(&lock->wait_lock);
@@ -1112,7 +1284,8 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock, · 241 unmodified lines
11121284 return 1;
11131285 }
11141286
1115− ret = task_blocks_on_rt_mutex(lock, waiter, task, detect_deadlock);
1287+ /* We enforce deadlock detection for futexes */
1288+ ret = task_blocks_on_rt_mutex(lock, waiter, task, 1);
11161289
11171290 if (ret && !rt_mutex_owner(lock)) {
11181291 /*
kernel/locking/rtmutex.h+5−0
@@ -24,3 +24,8 @@ · 23 unmodified lines
2424 #define debug_rt_mutex_print_deadlock(w) do { } while (0)
2525 #define debug_rt_mutex_detect_deadlock(w,d) (d)
2626 #define debug_rt_mutex_reset_waiter(w) do { } while (0)
27+
28+static inline void rt_mutex_print_deadlock(struct rt_mutex_waiter *w)
29+{
30+ WARN(1, "rtmutex deadlock detected\n");
31+}