History

sched: Add blocked_donor link to task for smarter mutex handoffs

Add link to the task this task is proxying for, and use it so the mutex owner can do an intelligent hand-off of the mutex to the task that the owner is running on behalf. [jstultz: This patch was split out from larger proxy patch] Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Juri Lelli <juri.lelli@redhat.com> Signed-off-by: Valentin Schneider <valentin.schneider@arm.com> Signed-off-by: Connor O'Brien <connoro@google.com> Signed-off-by: John Stultz <jstultz@google.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://patch.msgid.link/20260512025635.2840817-8-jstultz@google.com

Peter Zijlstra committed 6 months ago

Commit 1628b251 parent 4c2a2045 files · +75 −8Browse files

Unified · Split

5 files
  1. include/linux/sched.h +7 −0
  2. init/init_task.c +1 −0
  3. kernel/fork.c +1 −0
  4. kernel/locking/mutex.c +53 −7
  5. kernel/sched/core.c +13 −1
include/linux/sched.h+7−0
@@ -1250,6 +1250,13 @@ struct task_struct { · 1249 unmodified lines
12501250 struct mutex *blocked_on; /* lock we're blocked on */
12511251 raw_spinlock_t blocked_lock;
12521252
1253+ /*
1254+ * The task that is boosting this task; a back link for the current
1255+ * donor stack. Set in schedule() -> find_proxy_task() and only stable
1256+ * under preempt_disable().
1257+ */
1258+ struct task_struct *blocked_donor;
1259+
12531260 #ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
12541261 /*
12551262 * Encoded lock address causing task block (lower 2 bits = type from
init/init_task.c+1−0
@@ -200,6 +200,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = { · 199 unmodified lines
200200 .mems_allowed_seq = SEQCNT_SPINLOCK_ZERO(init_task.mems_allowed_seq,
201201 &init_task.alloc_lock),
202202 #endif
203+ .blocked_donor = NULL,
203204 #ifdef CONFIG_RT_MUTEXES
204205 .pi_waiters = RB_ROOT_CACHED,
205206 .pi_top_task = NULL,
kernel/fork.c+1−0
@@ -2224,6 +2224,7 @@ __latent_entropy struct task_struct *copy_process( · 2223 unmodified lines
22242224 lockdep_init_task(p);
22252225
22262226 p->blocked_on = NULL; /* not blocked yet */
2227+ p->blocked_donor = NULL; /* nobody is boosting p yet */
22272228
22282229 #ifdef CONFIG_BCACHE
22292230 p->sequential_io = 0;
kernel/locking/mutex.c+53−7
@@ -981,15 +981,22 @@ EXPORT_SYMBOL_GPL(ww_mutex_lock_interruptible); · 980 unmodified lines
981981 static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
982982 __releases(lock)
983983 {
984− struct task_struct *next = NULL;
984+ struct task_struct *donor, *next = NULL;
985985 struct mutex_waiter *waiter;
986− DEFINE_WAKE_Q(wake_q);
987986 unsigned long owner;
988987 unsigned long flags;
989988
990989 mutex_release(&lock->dep_map, ip);
991990 __release(lock);
992991
992+ /*
993+ * Ensures the proxy donor stack is stable across unlock and handoff.
994+ * Specifically, it avoids the case where current->blocked_donor is
995+ * NULL when it is inspected while doing the unlock, but a preemption
996+ * before taking the wake_lock would make it set and a hand-off is
997+ * missed.
998+ */
999+ guard(preempt)();
9931000 /*
9941001 * Release the lock before (potentially) taking the spinlock such that
9951002 * other contenders can get on with things ASAP.
@@ -1002,6 +1009,12 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne · 6 unmodified lines
10021009 MUTEX_WARN_ON(__owner_task(owner) != current);
10031010 MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
10041011
1012+ if (sched_proxy_exec() && current->blocked_donor) {
1013+ /* force handoff if we have a blocked_donor */
1014+ owner = MUTEX_FLAG_HANDOFF;
1015+ break;
1016+ }
1017+
10051018 if (owner & MUTEX_FLAG_HANDOFF)
10061019 break;
10071020
@@ -1014,20 +1027,53 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne · 6 unmodified lines
10141027 }
10151028
10161029 raw_spin_lock_irqsave(&lock->wait_lock, flags);
1030+ raw_spin_lock(&current->blocked_lock);
10171031 debug_mutex_unlock(lock);
1032+
1033+ if (sched_proxy_exec()) {
1034+ /*
1035+ * If we have a task boosting current, and that task was boosting
1036+ * current through this lock, hand the lock to that task, as that
1037+ * is the highest waiter, as selected by the scheduling function.
1038+ */
1039+ donor = current->blocked_donor;
1040+ if (donor) {
1041+ struct mutex *next_lock;
1042+
1043+ raw_spin_lock_nested(&donor->blocked_lock, SINGLE_DEPTH_NESTING);
1044+ next_lock = __get_task_blocked_on(donor);
1045+ if (next_lock == lock) {
1046+ next = get_task_struct(donor);
1047+ __set_task_blocked_on_waking(donor, next_lock);
1048+ current->blocked_donor = NULL;
1049+ }
1050+ raw_spin_unlock(&donor->blocked_lock);
1051+ }
1052+ }
1053+
1054+ /*
1055+ * Failing that, pick first on the wait list.
1056+ */
10181057 waiter = lock->first_waiter;
1019− if (waiter) {
1020− next = waiter->task;
1058+ if (!next && waiter) {
1059+ next = get_task_struct(waiter->task);
10211060
1061+ raw_spin_lock_nested(&next->blocked_lock, SINGLE_DEPTH_NESTING);
10221062 debug_mutex_wake_waiter(lock, waiter);
1023− set_task_blocked_on_waking(next, lock);
1024− wake_q_add(&wake_q, next);
1063+ __set_task_blocked_on_waking(next, lock);
1064+ raw_spin_unlock(&next->blocked_lock);
1065+
10251066 }
10261067
10271068 if (owner & MUTEX_FLAG_HANDOFF)
10281069 __mutex_handoff(lock, next);
10291070
1030− raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);
1071+ raw_spin_unlock(&current->blocked_lock);
1072+ raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1073+ if (next) {
1074+ wake_up_process(next);
1075+ put_task_struct(next);
1076+ }
10311077 }
10321078
10331079 #ifndef CONFIG_DEBUG_LOCK_ALLOC
kernel/sched/core.c+13−1
@@ -6827,7 +6827,17 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf, · 6826 unmodified lines
68276827 * Find runnable lock owner to proxy for mutex blocked donor
68286828 *
68296829 * Follow the blocked-on relation:
6830− * task->blocked_on -> mutex->owner -> task...
6830+ *
6831+ * ,-> task
6832+ * | | blocked-on
6833+ * | v
6834+ * blocked_donor | mutex
6835+ * | | owner
6836+ * | v
6837+ * `-- task
6838+ *
6839+ * and set the blocked_donor relation, this latter is used by the mutex
6840+ * code to find which (blocked) task to hand-off to.
68316841 *
68326842 * Lock order:
68336843 *
@@ -6969,6 +6979,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) · 135 unmodified lines
69696979 * rq, therefore holding @rq->lock is sufficient to
69706980 * guarantee its existence, as per ttwu_remote().
69716981 */
6982+ owner->blocked_donor = p;
69726983 }
69736984 WARN_ON_ONCE(owner && !owner->on_rq);
69746985 return owner;
@@ -7125,6 +7136,7 @@ pick_again: · 150 unmodified lines
71257136 clear_task_blocked_on(prev, NULL);
71267137
71277138 rq_set_donor(rq, next);
7139+ next->blocked_donor = NULL;
71287140 if (unlikely(next->is_blocked && next->blocked_on)) {
71297141 next = find_proxy_task(rq, next, &rf);
71307142 if (!next) {