History

Merge branch 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull locking fix from Thomas Gleixner: "A single commit, which makes the rtmutex.wait_lock an irq safe lock. This prevents a potential deadlock which can be triggered by the rcu boosting code from rcu_read_unlock()" * 'locking-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: rtmutex: Make wait_lock irq safe

Linus Torvalds committed 10 years ago

Commit bbfb2392 parents 30e4c9a b4abf912 files · +81 −72Browse files

Unified · Split

2 files
  1. kernel/futex.c +9 −9
  2. kernel/locking/rtmutex.c +72 −63
kernel/futex.c+9−9
@@ -1191,7 +1191,7 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this, · 1190 unmodified lines
11911191 if (pi_state->owner != current)
11921192 return -EINVAL;
11931193
1194− raw_spin_lock(&pi_state->pi_mutex.wait_lock);
1194+ raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
11951195 new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);
11961196
11971197 /*
@@ -1217,22 +1217,22 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this, · 19 unmodified lines
12171217 else if (curval != uval)
12181218 ret = -EINVAL;
12191219 if (ret) {
1220− raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
1220+ raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
12211221 return ret;
12221222 }
12231223
1224− raw_spin_lock_irq(&pi_state->owner->pi_lock);
1224+ raw_spin_lock(&pi_state->owner->pi_lock);
12251225 WARN_ON(list_empty(&pi_state->list));
12261226 list_del_init(&pi_state->list);
1227− raw_spin_unlock_irq(&pi_state->owner->pi_lock);
1227+ raw_spin_unlock(&pi_state->owner->pi_lock);
12281228
1229− raw_spin_lock_irq(&new_owner->pi_lock);
1229+ raw_spin_lock(&new_owner->pi_lock);
12301230 WARN_ON(!list_empty(&pi_state->list));
12311231 list_add(&pi_state->list, &new_owner->pi_state_list);
12321232 pi_state->owner = new_owner;
1233− raw_spin_unlock_irq(&new_owner->pi_lock);
1233+ raw_spin_unlock(&new_owner->pi_lock);
12341234
1235− raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
1235+ raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
12361236
12371237 deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);
12381238
@@ -2127,11 +2127,11 @@ static int fixup_owner(u32 __user *uaddr, struct futex_q *q, int locked) · 888 unmodified lines
21272127 * we returned due to timeout or signal without taking the
21282128 * rt_mutex. Too late.
21292129 */
2130− raw_spin_lock(&q->pi_state->pi_mutex.wait_lock);
2130+ raw_spin_lock_irq(&q->pi_state->pi_mutex.wait_lock);
21312131 owner = rt_mutex_owner(&q->pi_state->pi_mutex);
21322132 if (!owner)
21332133 owner = rt_mutex_next_owner(&q->pi_state->pi_mutex);
2134− raw_spin_unlock(&q->pi_state->pi_mutex.wait_lock);
2134+ raw_spin_unlock_irq(&q->pi_state->pi_mutex.wait_lock);
21352135 ret = fixup_pi_state_owner(uaddr, q, owner);
21362136 goto out;
21372137 }
kernel/locking/rtmutex.c+72−63
@@ -99,13 +99,14 @@ static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) · 98 unmodified lines
9999 * 2) Drop lock->wait_lock
100100 * 3) Try to unlock the lock with cmpxchg
101101 */
102−static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock)
102+static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
103+ unsigned long flags)
103104 __releases(lock->wait_lock)
104105 {
105106 struct task_struct *owner = rt_mutex_owner(lock);
106107
107108 clear_rt_mutex_waiters(lock);
108− raw_spin_unlock(&lock->wait_lock);
109+ raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
109110 /*
110111 * If a new waiter comes in between the unlock and the cmpxchg
111112 * we have two situations:
@@ -147,11 +148,12 @@ static inline void mark_rt_mutex_waiters(struct rt_mutex *lock) · 35 unmodified lines
147148 /*
148149 * Simple slow path only version: lock->owner is protected by lock->wait_lock.
149150 */
150−static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock)
151+static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock,
152+ unsigned long flags)
151153 __releases(lock->wait_lock)
152154 {
153155 lock->owner = NULL;
154− raw_spin_unlock(&lock->wait_lock);
156+ raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
155157 return true;
156158 }
157159 #endif
@@ -433,7 +435,6 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 275 unmodified lines
433435 int ret = 0, depth = 0;
434436 struct rt_mutex *lock;
435437 bool detect_deadlock;
436− unsigned long flags;
437438 bool requeue = true;
438439
439440 detect_deadlock = rt_mutex_cond_detect_deadlock(orig_waiter, chwalk);
@@ -476,7 +477,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 36 unmodified lines
476477 /*
477478 * [1] Task cannot go away as we did a get_task() before !
478479 */
479− raw_spin_lock_irqsave(&task->pi_lock, flags);
480+ raw_spin_lock_irq(&task->pi_lock);
480481
481482 /*
482483 * [2] Get the waiter on which @task is blocked on.
@@ -560,7 +561,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 77 unmodified lines
560561 * operations.
561562 */
562563 if (!raw_spin_trylock(&lock->wait_lock)) {
563− raw_spin_unlock_irqrestore(&task->pi_lock, flags);
564+ raw_spin_unlock_irq(&task->pi_lock);
564565 cpu_relax();
565566 goto retry;
566567 }
@@ -591,7 +592,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 24 unmodified lines
591592 /*
592593 * No requeue[7] here. Just release @task [8]
593594 */
594− raw_spin_unlock_irqrestore(&task->pi_lock, flags);
595+ raw_spin_unlock(&task->pi_lock);
595596 put_task_struct(task);
596597
597598 /*
@@ -599,14 +600,14 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 1 unmodified line
599600 * If there is no owner of the lock, end of chain.
600601 */
601602 if (!rt_mutex_owner(lock)) {
602− raw_spin_unlock(&lock->wait_lock);
603+ raw_spin_unlock_irq(&lock->wait_lock);
603604 return 0;
604605 }
605606
606607 /* [10] Grab the next task, i.e. owner of @lock */
607608 task = rt_mutex_owner(lock);
608609 get_task_struct(task);
609− raw_spin_lock_irqsave(&task->pi_lock, flags);
610+ raw_spin_lock(&task->pi_lock);
610611
611612 /*
612613 * No requeue [11] here. We just do deadlock detection.
@@ -621,8 +622,8 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 8 unmodified lines
621622 top_waiter = rt_mutex_top_waiter(lock);
622623
623624 /* [13] Drop locks */
624− raw_spin_unlock_irqrestore(&task->pi_lock, flags);
625− raw_spin_unlock(&lock->wait_lock);
625+ raw_spin_unlock(&task->pi_lock);
626+ raw_spin_unlock_irq(&lock->wait_lock);
626627
627628 /* If owner is not blocked, end of chain. */
628629 if (!next_lock)
@@ -643,7 +644,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 14 unmodified lines
643644 rt_mutex_enqueue(lock, waiter);
644645
645646 /* [8] Release the task */
646− raw_spin_unlock_irqrestore(&task->pi_lock, flags);
647+ raw_spin_unlock(&task->pi_lock);
647648 put_task_struct(task);
648649
649650 /*
@@ -661,14 +662,14 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 11 unmodified lines
661662 */
662663 if (prerequeue_top_waiter != rt_mutex_top_waiter(lock))
663664 wake_up_process(rt_mutex_top_waiter(lock)->task);
664− raw_spin_unlock(&lock->wait_lock);
665+ raw_spin_unlock_irq(&lock->wait_lock);
665666 return 0;
666667 }
667668
668669 /* [10] Grab the next task, i.e. the owner of @lock */
669670 task = rt_mutex_owner(lock);
670671 get_task_struct(task);
671− raw_spin_lock_irqsave(&task->pi_lock, flags);
672+ raw_spin_lock(&task->pi_lock);
672673
673674 /* [11] requeue the pi waiters if necessary */
674675 if (waiter == rt_mutex_top_waiter(lock)) {
@@ -722,8 +723,8 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 47 unmodified lines
722723 top_waiter = rt_mutex_top_waiter(lock);
723724
724725 /* [13] Drop the locks */
725− raw_spin_unlock_irqrestore(&task->pi_lock, flags);
726− raw_spin_unlock(&lock->wait_lock);
726+ raw_spin_unlock(&task->pi_lock);
727+ raw_spin_unlock_irq(&lock->wait_lock);
727728
728729 /*
729730 * Make the actual exit decisions [12], based on the stored
@@ -746,7 +747,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 16 unmodified lines
746747 goto again;
747748
748749 out_unlock_pi:
749− raw_spin_unlock_irqrestore(&task->pi_lock, flags);
750+ raw_spin_unlock_irq(&task->pi_lock);
750751 out_put_task:
751752 put_task_struct(task);
752753
@@ -756,7 +757,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 3 unmodified lines
756757 /*
757758 * Try to take an rt-mutex
758759 *
759− * Must be called with lock->wait_lock held.
760+ * Must be called with lock->wait_lock held and interrupts disabled
760761 *
761762 * @lock: The lock to be acquired.
762763 * @task: The task which wants to acquire the lock
@@ -766,8 +767,6 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task, · 3 unmodified lines
766767 static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
767768 struct rt_mutex_waiter *waiter)
768769 {
769− unsigned long flags;
770−
771770 /*
772771 * Before testing whether we can acquire @lock, we set the
773772 * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all
@@ -852,7 +851,7 @@ static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, · 78 unmodified lines
852851 * case, but conditionals are more expensive than a redundant
853852 * store.
854853 */
855− raw_spin_lock_irqsave(&task->pi_lock, flags);
854+ raw_spin_lock(&task->pi_lock);
856855 task->pi_blocked_on = NULL;
857856 /*
858857 * Finish the lock acquisition. @task is the new owner. If
@@ -861,7 +860,7 @@ static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, · 2 unmodified lines
861860 */
862861 if (rt_mutex_has_waiters(lock))
863862 rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));
864− raw_spin_unlock_irqrestore(&task->pi_lock, flags);
863+ raw_spin_unlock(&task->pi_lock);
865864
866865 takeit:
867866 /* We got the lock. */
@@ -883,7 +882,7 @@ takeit: · 15 unmodified lines
883882 *
884883 * Prepare waiter and propagate pi chain
885884 *
886− * This must be called with lock->wait_lock held.
885+ * This must be called with lock->wait_lock held and interrupts disabled
887886 */
888887 static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
889888 struct rt_mutex_waiter *waiter,
@@ -894,7 +893,6 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 4 unmodified lines
894893 struct rt_mutex_waiter *top_waiter = waiter;
895894 struct rt_mutex *next_lock;
896895 int chain_walk = 0, res;
897− unsigned long flags;
898896
899897 /*
900898 * Early deadlock detection. We really don't want the task to
@@ -908,7 +906,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 7 unmodified lines
908906 if (owner == task)
909907 return -EDEADLK;
910908
911− raw_spin_lock_irqsave(&task->pi_lock, flags);
909+ raw_spin_lock(&task->pi_lock);
912910 __rt_mutex_adjust_prio(task);
913911 waiter->task = task;
914912 waiter->lock = lock;
@@ -921,12 +919,12 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 6 unmodified lines
921919
922920 task->pi_blocked_on = waiter;
923921
924− raw_spin_unlock_irqrestore(&task->pi_lock, flags);
922+ raw_spin_unlock(&task->pi_lock);
925923
926924 if (!owner)
927925 return 0;
928926
929− raw_spin_lock_irqsave(&owner->pi_lock, flags);
927+ raw_spin_lock(&owner->pi_lock);
930928 if (waiter == rt_mutex_top_waiter(lock)) {
931929 rt_mutex_dequeue_pi(owner, top_waiter);
932930 rt_mutex_enqueue_pi(owner, waiter);
@@ -941,7 +939,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 8 unmodified lines
941939 /* Store the lock on which owner is blocked or NULL */
942940 next_lock = task_blocked_on_lock(owner);
943941
944− raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
942+ raw_spin_unlock(&owner->pi_lock);
945943 /*
946944 * Even if full deadlock detection is on, if the owner is not
947945 * blocked itself, we can avoid finding this out in the chain
@@ -957,12 +955,12 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 9 unmodified lines
957955 */
958956 get_task_struct(owner);
959957
960− raw_spin_unlock(&lock->wait_lock);
958+ raw_spin_unlock_irq(&lock->wait_lock);
961959
962960 res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,
963961 next_lock, waiter, task);
964962
965− raw_spin_lock(&lock->wait_lock);
963+ raw_spin_lock_irq(&lock->wait_lock);
966964
967965 return res;
968966 }
@@ -971,15 +969,14 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock, · 2 unmodified lines
971969 * Remove the top waiter from the current tasks pi waiter tree and
972970 * queue it up.
973971 *
974− * Called with lock->wait_lock held.
972+ * Called with lock->wait_lock held and interrupts disabled.
975973 */
976974 static void mark_wakeup_next_waiter(struct wake_q_head *wake_q,
977975 struct rt_mutex *lock)
978976 {
979977 struct rt_mutex_waiter *waiter;
980− unsigned long flags;
981978
982− raw_spin_lock_irqsave(&current->pi_lock, flags);
979+ raw_spin_lock(&current->pi_lock);
983980
984981 waiter = rt_mutex_top_waiter(lock);
985982
@@ -1001,7 +998,7 @@ static void mark_wakeup_next_waiter(struct wake_q_head *wake_q, · 15 unmodified lines
1001998 */
1002999 lock->owner = (void *) RT_MUTEX_HAS_WAITERS;
10031000
1004− raw_spin_unlock_irqrestore(&current->pi_lock, flags);
1001+ raw_spin_unlock(&current->pi_lock);
10051002
10061003 wake_q_add(wake_q, waiter->task);
10071004 }
@@ -1009,7 +1006,7 @@ static void mark_wakeup_next_waiter(struct wake_q_head *wake_q, · 1 unmodified line
10091006 /*
10101007 * Remove a waiter from a lock and give up
10111008 *
1012− * Must be called with lock->wait_lock held and
1009+ * Must be called with lock->wait_lock held and interrupts disabled. I must
10131010 * have just failed to try_to_take_rt_mutex().
10141011 */
10151012 static void remove_waiter(struct rt_mutex *lock,
@@ -1018,12 +1015,11 @@ static void remove_waiter(struct rt_mutex *lock, · 2 unmodified lines
10181015 bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
10191016 struct task_struct *owner = rt_mutex_owner(lock);
10201017 struct rt_mutex *next_lock;
1021− unsigned long flags;
10221018
1023− raw_spin_lock_irqsave(&current->pi_lock, flags);
1019+ raw_spin_lock(&current->pi_lock);
10241020 rt_mutex_dequeue(lock, waiter);
10251021 current->pi_blocked_on = NULL;
1026− raw_spin_unlock_irqrestore(&current->pi_lock, flags);
1022+ raw_spin_unlock(&current->pi_lock);
10271023
10281024 /*
10291025 * Only update priority if the waiter was the highest priority
@@ -1032,7 +1028,7 @@ static void remove_waiter(struct rt_mutex *lock, · 2 unmodified lines
10321028 if (!owner || !is_top_waiter)
10331029 return;
10341030
1035− raw_spin_lock_irqsave(&owner->pi_lock, flags);
1031+ raw_spin_lock(&owner->pi_lock);
10361032
10371033 rt_mutex_dequeue_pi(owner, waiter);
10381034
@@ -1044,7 +1040,7 @@ static void remove_waiter(struct rt_mutex *lock, · 5 unmodified lines
10441040 /* Store the lock on which owner is blocked or NULL */
10451041 next_lock = task_blocked_on_lock(owner);
10461042
1047− raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
1043+ raw_spin_unlock(&owner->pi_lock);
10481044
10491045 /*
10501046 * Don't walk the chain, if the owner task is not blocked
@@ -1056,12 +1052,12 @@ static void remove_waiter(struct rt_mutex *lock, · 5 unmodified lines
10561052 /* gets dropped in rt_mutex_adjust_prio_chain()! */
10571053 get_task_struct(owner);
10581054
1059− raw_spin_unlock(&lock->wait_lock);
1055+ raw_spin_unlock_irq(&lock->wait_lock);
10601056
10611057 rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,
10621058 next_lock, NULL, current);
10631059
1064− raw_spin_lock(&lock->wait_lock);
1060+ raw_spin_lock_irq(&lock->wait_lock);
10651061 }
10661062
10671063 /*
@@ -1097,11 +1093,11 @@ void rt_mutex_adjust_pi(struct task_struct *task) · 29 unmodified lines
10971093 * __rt_mutex_slowlock() - Perform the wait-wake-try-to-take loop
10981094 * @lock: the rt_mutex to take
10991095 * @state: the state the task should block in (TASK_INTERRUPTIBLE
1100− * or TASK_UNINTERRUPTIBLE)
1096+ * or TASK_UNINTERRUPTIBLE)
11011097 * @timeout: the pre-initialized and started timer, or NULL for none
11021098 * @waiter: the pre-initialized rt_mutex_waiter
11031099 *
1104− * lock->wait_lock must be held by the caller.
1100+ * Must be called with lock->wait_lock held and interrupts disabled
11051101 */
11061102 static int __sched
11071103 __rt_mutex_slowlock(struct rt_mutex *lock, int state,
@@ -1129,13 +1125,13 @@ __rt_mutex_slowlock(struct rt_mutex *lock, int state, · 21 unmodified lines
11291125 break;
11301126 }
11311127
1132− raw_spin_unlock(&lock->wait_lock);
1128+ raw_spin_unlock_irq(&lock->wait_lock);
11331129
11341130 debug_rt_mutex_print_deadlock(waiter);
11351131
11361132 schedule();
11371133
1138− raw_spin_lock(&lock->wait_lock);
1134+ raw_spin_lock_irq(&lock->wait_lock);
11391135 set_current_state(state);
11401136 }
11411137
@@ -1172,17 +1168,26 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, · 30 unmodified lines
11721168 enum rtmutex_chainwalk chwalk)
11731169 {
11741170 struct rt_mutex_waiter waiter;
1171+ unsigned long flags;
11751172 int ret = 0;
11761173
11771174 debug_rt_mutex_init_waiter(&waiter);
11781175 RB_CLEAR_NODE(&waiter.pi_tree_entry);
11791176 RB_CLEAR_NODE(&waiter.tree_entry);
11801177
1181− raw_spin_lock(&lock->wait_lock);
1178+ /*
1179+ * Technically we could use raw_spin_[un]lock_irq() here, but this can
1180+ * be called in early boot if the cmpxchg() fast path is disabled
1181+ * (debug, no architecture support). In this case we will acquire the
1182+ * rtmutex with lock->wait_lock held. But we cannot unconditionally
1183+ * enable interrupts in that early boot case. So we need to use the
1184+ * irqsave/restore variants.
1185+ */
1186+ raw_spin_lock_irqsave(&lock->wait_lock, flags);
11821187
11831188 /* Try to acquire the lock again: */
11841189 if (try_to_take_rt_mutex(lock, current, NULL)) {
1185− raw_spin_unlock(&lock->wait_lock);
1190+ raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
11861191 return 0;
11871192 }
11881193
@@ -1211,7 +1216,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, · 22 unmodified lines
12111216 */
12121217 fixup_rt_mutex_waiters(lock);
12131218
1214− raw_spin_unlock(&lock->wait_lock);
1219+ raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
12151220
12161221 /* Remove pending timer: */
12171222 if (unlikely(timeout))
@@ -1227,6 +1232,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state, · 9 unmodified lines
12271232 */
12281233 static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)
12291234 {
1235+ unsigned long flags;
12301236 int ret;
12311237
12321238 /*
@@ -1238,10 +1244,10 @@ static inline int rt_mutex_slowtrylock(struct rt_mutex *lock) · 5 unmodified lines
12381244 return 0;
12391245
12401246 /*
1241− * The mutex has currently no owner. Lock the wait lock and
1242− * try to acquire the lock.
1247+ * The mutex has currently no owner. Lock the wait lock and try to
1248+ * acquire the lock. We use irqsave here to support early boot calls.
12431249 */
1244− raw_spin_lock(&lock->wait_lock);
1250+ raw_spin_lock_irqsave(&lock->wait_lock, flags);
12451251
12461252 ret = try_to_take_rt_mutex(lock, current, NULL);
12471253
@@ -1251,7 +1257,7 @@ static inline int rt_mutex_slowtrylock(struct rt_mutex *lock) · 3 unmodified lines
12511257 */
12521258 fixup_rt_mutex_waiters(lock);
12531259
1254− raw_spin_unlock(&lock->wait_lock);
1260+ raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
12551261
12561262 return ret;
12571263 }
@@ -1263,7 +1269,10 @@ static inline int rt_mutex_slowtrylock(struct rt_mutex *lock) · 5 unmodified lines
12631269 static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock,
12641270 struct wake_q_head *wake_q)
12651271 {
1266− raw_spin_lock(&lock->wait_lock);
1272+ unsigned long flags;
1273+
1274+ /* irqsave required to support early boot calls */
1275+ raw_spin_lock_irqsave(&lock->wait_lock, flags);
12671276
12681277 debug_rt_mutex_unlock(lock);
12691278
@@ -1302,10 +1311,10 @@ static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock, · 32 unmodified lines
13021311 */
13031312 while (!rt_mutex_has_waiters(lock)) {
13041313 /* Drops lock->wait_lock ! */
1305− if (unlock_rt_mutex_safe(lock) == true)
1314+ if (unlock_rt_mutex_safe(lock, flags) == true)
13061315 return false;
13071316 /* Relock the rtmutex and try again */
1308− raw_spin_lock(&lock->wait_lock);
1317+ raw_spin_lock_irqsave(&lock->wait_lock, flags);
13091318 }
13101319
13111320 /*
@@ -1316,7 +1325,7 @@ static bool __sched rt_mutex_slowunlock(struct rt_mutex *lock, · 4 unmodified lines
13161325 */
13171326 mark_wakeup_next_waiter(wake_q, lock);
13181327
1319− raw_spin_unlock(&lock->wait_lock);
1328+ raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
13201329
13211330 /* check PI boosting */
13221331 return true;
@@ -1596,10 +1605,10 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock, · 273 unmodified lines
15961605 {
15971606 int ret;
15981607
1599− raw_spin_lock(&lock->wait_lock);
1608+ raw_spin_lock_irq(&lock->wait_lock);
16001609
16011610 if (try_to_take_rt_mutex(lock, task, NULL)) {
1602− raw_spin_unlock(&lock->wait_lock);
1611+ raw_spin_unlock_irq(&lock->wait_lock);
16031612 return 1;
16041613 }
16051614
@@ -1620,7 +1629,7 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock, · 14 unmodified lines
16201629 if (unlikely(ret))
16211630 remove_waiter(lock, waiter);
16221631
1623− raw_spin_unlock(&lock->wait_lock);
1632+ raw_spin_unlock_irq(&lock->wait_lock);
16241633
16251634 debug_rt_mutex_print_deadlock(waiter);
16261635
@@ -1668,7 +1677,7 @@ int rt_mutex_finish_proxy_lock(struct rt_mutex *lock, · 41 unmodified lines
16681677 {
16691678 int ret;
16701679
1671− raw_spin_lock(&lock->wait_lock);
1680+ raw_spin_lock_irq(&lock->wait_lock);
16721681
16731682 set_current_state(TASK_INTERRUPTIBLE);
16741683
@@ -1684,7 +1693,7 @@ int rt_mutex_finish_proxy_lock(struct rt_mutex *lock, · 9 unmodified lines
16841693 */
16851694 fixup_rt_mutex_waiters(lock);
16861695
1687− raw_spin_unlock(&lock->wait_lock);
1696+ raw_spin_unlock_irq(&lock->wait_lock);
16881697
16891698 return ret;
16901699 }