History

sched: Add blocked_donor link to task for smarter mutex handoffs

Add link to the task this task is proxying for, and use it so the mutex owner can do an intelligent hand-off of the mutex to the task that the owner is running on behalf. [jstultz: This patch was split out from larger proxy patch] Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Juri Lelli <juri.lelli@redhat.com> Signed-off-by: Valentin Schneider <valentin.schneider@arm.com> Signed-off-by: Connor O'Brien <connoro@google.com> Signed-off-by: John Stultz <jstultz@google.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://patch.msgid.link/20260512025635.2840817-8-jstultz@google.com

Peter Zijlstra committed 6 months ago

Commit 1628b251 parent 4c2a2045 files · +75 −8Browse files

Unified · Split

5 files
  1. include/linux/sched.h +7 −0
  2. init/init_task.c +1 −0
  3. kernel/fork.c +1 −0
  4. kernel/locking/mutex.c +53 −7
  5. kernel/sched/core.c +13 −1
include/linux/sched.h+7−0
@@ -1250,6 +1250,13 @@ struct task_struct { · 1249 unmodified lines
1250 struct mutex *blocked_on; /* lock we're blocked on */1250 struct mutex *blocked_on; /* lock we're blocked on */
1251 raw_spinlock_t blocked_lock;1251 raw_spinlock_t blocked_lock;
1252 1252
1253 /*
1254 * The task that is boosting this task; a back link for the current
1255 * donor stack. Set in schedule() -> find_proxy_task() and only stable
1256 * under preempt_disable().
1257 */
1258 struct task_struct *blocked_donor;
1259
1253#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER1260#ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER
1254 /*1261 /*
1255 * Encoded lock address causing task block (lower 2 bits = type from1262 * Encoded lock address causing task block (lower 2 bits = type from
init/init_task.c+1−0
@@ -200,6 +200,7 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = { · 199 unmodified lines
200 .mems_allowed_seq = SEQCNT_SPINLOCK_ZERO(init_task.mems_allowed_seq,200 .mems_allowed_seq = SEQCNT_SPINLOCK_ZERO(init_task.mems_allowed_seq,
201 &init_task.alloc_lock),201 &init_task.alloc_lock),
202#endif202#endif
203 .blocked_donor = NULL,
203#ifdef CONFIG_RT_MUTEXES204#ifdef CONFIG_RT_MUTEXES
204 .pi_waiters = RB_ROOT_CACHED,205 .pi_waiters = RB_ROOT_CACHED,
205 .pi_top_task = NULL,206 .pi_top_task = NULL,
kernel/fork.c+1−0
@@ -2224,6 +2224,7 @@ __latent_entropy struct task_struct *copy_process( · 2223 unmodified lines
2224 lockdep_init_task(p);2224 lockdep_init_task(p);
2225 2225
2226 p->blocked_on = NULL; /* not blocked yet */2226 p->blocked_on = NULL; /* not blocked yet */
2227 p->blocked_donor = NULL; /* nobody is boosting p yet */
2227 2228
2228#ifdef CONFIG_BCACHE2229#ifdef CONFIG_BCACHE
2229 p->sequential_io = 0;2230 p->sequential_io = 0;
kernel/locking/mutex.c+53−7
@@ -981,15 +981,22 @@ EXPORT_SYMBOL_GPL(ww_mutex_lock_interruptible); · 980 unmodified lines
981static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)981static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigned long ip)
982 __releases(lock)982 __releases(lock)
983{983{
984 struct task_struct *next = NULL;984 struct task_struct *donor, *next = NULL;
985 struct mutex_waiter *waiter;985 struct mutex_waiter *waiter;
986 DEFINE_WAKE_Q(wake_q);
987 unsigned long owner;986 unsigned long owner;
988 unsigned long flags;987 unsigned long flags;
989 988
990 mutex_release(&lock->dep_map, ip);989 mutex_release(&lock->dep_map, ip);
991 __release(lock);990 __release(lock);
992 991
992 /*
993 * Ensures the proxy donor stack is stable across unlock and handoff.
994 * Specifically, it avoids the case where current->blocked_donor is
995 * NULL when it is inspected while doing the unlock, but a preemption
996 * before taking the wake_lock would make it set and a hand-off is
997 * missed.
998 */
999 guard(preempt)();
993 /*1000 /*
994 * Release the lock before (potentially) taking the spinlock such that1001 * Release the lock before (potentially) taking the spinlock such that
995 * other contenders can get on with things ASAP.1002 * other contenders can get on with things ASAP.
@@ -1002,6 +1009,12 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne · 6 unmodified lines
1002 MUTEX_WARN_ON(__owner_task(owner) != current);1009 MUTEX_WARN_ON(__owner_task(owner) != current);
1003 MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);1010 MUTEX_WARN_ON(owner & MUTEX_FLAG_PICKUP);
1004 1011
1012 if (sched_proxy_exec() && current->blocked_donor) {
1013 /* force handoff if we have a blocked_donor */
1014 owner = MUTEX_FLAG_HANDOFF;
1015 break;
1016 }
1017
1005 if (owner & MUTEX_FLAG_HANDOFF)1018 if (owner & MUTEX_FLAG_HANDOFF)
1006 break;1019 break;
1007 1020
@@ -1014,20 +1027,53 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne · 6 unmodified lines
1014 }1027 }
1015 1028
1016 raw_spin_lock_irqsave(&lock->wait_lock, flags);1029 raw_spin_lock_irqsave(&lock->wait_lock, flags);
1030 raw_spin_lock(&current->blocked_lock);
1017 debug_mutex_unlock(lock);1031 debug_mutex_unlock(lock);
1032
1033 if (sched_proxy_exec()) {
1034 /*
1035 * If we have a task boosting current, and that task was boosting
1036 * current through this lock, hand the lock to that task, as that
1037 * is the highest waiter, as selected by the scheduling function.
1038 */
1039 donor = current->blocked_donor;
1040 if (donor) {
1041 struct mutex *next_lock;
1042
1043 raw_spin_lock_nested(&donor->blocked_lock, SINGLE_DEPTH_NESTING);
1044 next_lock = __get_task_blocked_on(donor);
1045 if (next_lock == lock) {
1046 next = get_task_struct(donor);
1047 __set_task_blocked_on_waking(donor, next_lock);
1048 current->blocked_donor = NULL;
1049 }
1050 raw_spin_unlock(&donor->blocked_lock);
1051 }
1052 }
1053
1054 /*
1055 * Failing that, pick first on the wait list.
1056 */
1018 waiter = lock->first_waiter;1057 waiter = lock->first_waiter;
1019 if (waiter) {1058 if (!next && waiter) {
1020 next = waiter->task;1059 next = get_task_struct(waiter->task);
1021 1060
1061 raw_spin_lock_nested(&next->blocked_lock, SINGLE_DEPTH_NESTING);
1022 debug_mutex_wake_waiter(lock, waiter);1062 debug_mutex_wake_waiter(lock, waiter);
1023 set_task_blocked_on_waking(next, lock);1063 __set_task_blocked_on_waking(next, lock);
1024 wake_q_add(&wake_q, next);1064 raw_spin_unlock(&next->blocked_lock);
1065
1025 }1066 }
1026 1067
1027 if (owner & MUTEX_FLAG_HANDOFF)1068 if (owner & MUTEX_FLAG_HANDOFF)
1028 __mutex_handoff(lock, next);1069 __mutex_handoff(lock, next);
1029 1070
1030 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q);1071 raw_spin_unlock(&current->blocked_lock);
1072 raw_spin_unlock_irqrestore(&lock->wait_lock, flags);
1073 if (next) {
1074 wake_up_process(next);
1075 put_task_struct(next);
1076 }
1031}1077}
1032 1078
1033#ifndef CONFIG_DEBUG_LOCK_ALLOC1079#ifndef CONFIG_DEBUG_LOCK_ALLOC
kernel/sched/core.c+13−1
@@ -6827,7 +6827,17 @@ static void proxy_migrate_task(struct rq *rq, struct rq_flags *rf, · 6826 unmodified lines
6827 * Find runnable lock owner to proxy for mutex blocked donor6827 * Find runnable lock owner to proxy for mutex blocked donor
6828 *6828 *
6829 * Follow the blocked-on relation:6829 * Follow the blocked-on relation:
6830 * task->blocked_on -> mutex->owner -> task...6830 *
6831 * ,-> task
6832 * | | blocked-on
6833 * | v
6834 * blocked_donor | mutex
6835 * | | owner
6836 * | v
6837 * `-- task
6838 *
6839 * and set the blocked_donor relation, this latter is used by the mutex
6840 * code to find which (blocked) task to hand-off to.
6831 *6841 *
6832 * Lock order:6842 * Lock order:
6833 *6843 *
@@ -6969,6 +6979,7 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) · 135 unmodified lines
6969 * rq, therefore holding @rq->lock is sufficient to6979 * rq, therefore holding @rq->lock is sufficient to
6970 * guarantee its existence, as per ttwu_remote().6980 * guarantee its existence, as per ttwu_remote().
6971 */6981 */
6982 owner->blocked_donor = p;
6972 }6983 }
6973 WARN_ON_ONCE(owner && !owner->on_rq);6984 WARN_ON_ONCE(owner && !owner->on_rq);
6974 return owner;6985 return owner;
@@ -7125,6 +7136,7 @@ pick_again: · 150 unmodified lines
7125 clear_task_blocked_on(prev, NULL);7136 clear_task_blocked_on(prev, NULL);
7126 7137
7127 rq_set_donor(rq, next);7138 rq_set_donor(rq, next);
7139 next->blocked_donor = NULL;
7128 if (unlikely(next->is_blocked && next->blocked_on)) {7140 if (unlikely(next->is_blocked && next->blocked_on)) {
7129 next = find_proxy_task(rq, next, &rf);7141 next = find_proxy_task(rq, next, &rf);
7130 if (!next) {7142 if (!next) {