History

sched/locking: Add special p->blocked_on==PROXY_WAKING value for proxy return-migration

As we add functionality to proxy execution, we may migrate a donor task to a runqueue where it can't run due to cpu affinity. Thus, we must be careful to ensure we return-migrate the task back to a cpu in its cpumask when it becomes unblocked. Peter helpfully provided the following example with pictures: "Suppose we have a ww_mutex cycle: ,-+-* Mutex-1 <-. Task-A ---' | | ,-- Task-B `-> Mutex-2 *-+-' Where Task-A holds Mutex-1 and tries to acquire Mutex-2, and where Task-B holds Mutex-2 and tries to acquire Mutex-1. Then the blocked_on->owner chain will go in circles. Task-A -> Mutex-2 ^ | | v Mutex-1 <- Task-B We need two things: - find_proxy_task() to stop iterating the circle; - the woken task to 'unblock' and run, such that it can back-off and re-try the transaction. Now, the current code [without this patch] does: __clear_task_blocked_on(); wake_q_add(); And surely clearing ->blocked_on is sufficient to break the cycle. Suppose it is Task-B that is made to back-off, then we have: Task-A -> Mutex-2 -> Task-B (no further blocked_on) and it would attempt to run Task-B. Or worse, it could directly pick Task-B and run it, without ever getting into find_proxy_task(). Now, here is a problem because Task-B might not be runnable on the CPU it is currently on; and because !task_is_blocked() we don't get into the proxy paths, so nobody is going to fix this up. Ideally we would have dequeued Task-B alongside of clearing ->blocked_on, but alas, [the lock ordering prevents us from getting the task_rq_lock() and] spoils things." Thus we need more than just a binary concept of the task being blocked on a mutex or not. So allow setting blocked_on to PROXY_WAKING as a special value which specifies the task is no longer blocked, but needs to be evaluated for return migration *before* it can be run. This will then be used in a later patch to handle proxy return-migration.

John Stultz committed 7 months ago

Commit 2d762261 parent 56f4b244 files · +74 −11Browse files

Unified · Split

4 files
  1. include/linux/sched.h +49 −2
  2. kernel/locking/mutex.c +1 −1
  3. kernel/locking/ww_mutex.h +8 −8
  4. kernel/sched/core.c +16 −0
include/linux/sched.h+49−2
@@ -2180,10 +2180,20 @@ extern int __cond_resched_rwlock_write(rwlock_t *lock) __must_hold(lock); · 2179 unmodified lines
2180})2180})
2181 2181
2182#ifndef CONFIG_PREEMPT_RT2182#ifndef CONFIG_PREEMPT_RT
2183
2184/*
2185 * With proxy exec, if a task has been proxy-migrated, it may be a donor
2186 * on a cpu that it can't actually run on. Thus we need a special state
2187 * to denote that the task is being woken, but that it needs to be
2188 * evaluated for return-migration before it is run. So if the task is
2189 * blocked_on PROXY_WAKING, return migrate it before running it.
2190 */
2191#define PROXY_WAKING ((struct mutex *)(-1L))
2192
2183static inline struct mutex *__get_task_blocked_on(struct task_struct *p)2193static inline struct mutex *__get_task_blocked_on(struct task_struct *p)
2184{2194{
2185 lockdep_assert_held_once(&p->blocked_lock);2195 lockdep_assert_held_once(&p->blocked_lock);
2186 return p->blocked_on;2196 return p->blocked_on == PROXY_WAKING ? NULL : p->blocked_on;
2187}2197}
2188 2198
2189static inline void __set_task_blocked_on(struct task_struct *p, struct mutex *m)2199static inline void __set_task_blocked_on(struct task_struct *p, struct mutex *m)
@@ -2211,7 +2221,7 @@ static inline void __clear_task_blocked_on(struct task_struct *p, struct mutex * · 21 unmodified lines
2211 * blocked_on relationships, but make sure we are not2221 * blocked_on relationships, but make sure we are not
2212 * clearing the relationship with a different lock.2222 * clearing the relationship with a different lock.
2213 */2223 */
2214 WARN_ON_ONCE(m && p->blocked_on && p->blocked_on != m);2224 WARN_ON_ONCE(m && p->blocked_on && p->blocked_on != m && p->blocked_on != PROXY_WAKING);
2215 p->blocked_on = NULL;2225 p->blocked_on = NULL;
2216}2226}
2217 2227
@@ -2220,6 +2230,35 @@ static inline void clear_task_blocked_on(struct task_struct *p, struct mutex *m) · 2 unmodified lines
2220 guard(raw_spinlock_irqsave)(&p->blocked_lock);2230 guard(raw_spinlock_irqsave)(&p->blocked_lock);
2221 __clear_task_blocked_on(p, m);2231 __clear_task_blocked_on(p, m);
2222}2232}
2233
2234static inline void __set_task_blocked_on_waking(struct task_struct *p, struct mutex *m)
2235{
2236 /* Currently we serialize blocked_on under the task::blocked_lock */
2237 lockdep_assert_held_once(&p->blocked_lock);
2238
2239 if (!sched_proxy_exec()) {
2240 __clear_task_blocked_on(p, m);
2241 return;
2242 }
2243
2244 /* Don't set PROXY_WAKING if blocked_on was already cleared */
2245 if (!p->blocked_on)
2246 return;
2247 /*
2248 * There may be cases where we set PROXY_WAKING on tasks that were
2249 * already set to waking, but make sure we are not changing
2250 * the relationship with a different lock.
2251 */
2252 WARN_ON_ONCE(m && p->blocked_on != m && p->blocked_on != PROXY_WAKING);
2253 p->blocked_on = PROXY_WAKING;
2254}
2255
2256static inline void set_task_blocked_on_waking(struct task_struct *p, struct mutex *m)
2257{
2258 guard(raw_spinlock_irqsave)(&p->blocked_lock);
2259 __set_task_blocked_on_waking(p, m);
2260}
2261
2223#else2262#else
2224static inline void __clear_task_blocked_on(struct task_struct *p, struct rt_mutex *m)2263static inline void __clear_task_blocked_on(struct task_struct *p, struct rt_mutex *m)
2225{2264{
@@ -2228,6 +2267,14 @@ static inline void __clear_task_blocked_on(struct task_struct *p, struct rt_mute · 2 unmodified lines
2228static inline void clear_task_blocked_on(struct task_struct *p, struct rt_mutex *m)2267static inline void clear_task_blocked_on(struct task_struct *p, struct rt_mutex *m)
2229{2268{
2230}2269}
2270
2271static inline void __set_task_blocked_on_waking(struct task_struct *p, struct rt_mutex *m)
2272{
2273}
2274
2275static inline void set_task_blocked_on_waking(struct task_struct *p, struct rt_mutex *m)
2276{
2277}
2231#endif /* !CONFIG_PREEMPT_RT */2278#endif /* !CONFIG_PREEMPT_RT */
2232 2279
2233static __always_inline bool need_resched(void)2280static __always_inline bool need_resched(void)
kernel/locking/mutex.c+1−1
@@ -983,7 +983,7 @@ static noinline void __sched __mutex_unlock_slowpath(struct mutex *lock, unsigne · 982 unmodified lines
983 next = waiter->task;983 next = waiter->task;
984 984
985 debug_mutex_wake_waiter(lock, waiter);985 debug_mutex_wake_waiter(lock, waiter);
986 clear_task_blocked_on(next, lock);986 set_task_blocked_on_waking(next, lock);
987 wake_q_add(&wake_q, next);987 wake_q_add(&wake_q, next);
988 }988 }
989 989
kernel/locking/ww_mutex.h+8−8
@@ -285,11 +285,11 @@ __ww_mutex_die(struct MUTEX *lock, struct MUTEX_WAITER *waiter, · 284 unmodified lines
285 debug_mutex_wake_waiter(lock, waiter);285 debug_mutex_wake_waiter(lock, waiter);
286#endif286#endif
287 /*287 /*
288 * When waking up the task to die, be sure to clear the288 * When waking up the task to die, be sure to set the
289 * blocked_on pointer. Otherwise we can see circular289 * blocked_on to PROXY_WAKING. Otherwise we can see
290 * blocked_on relationships that can't resolve.290 * circular blocked_on relationships that can't resolve.
291 */291 */
292 clear_task_blocked_on(waiter->task, lock);292 set_task_blocked_on_waking(waiter->task, lock);
293 wake_q_add(wake_q, waiter->task);293 wake_q_add(wake_q, waiter->task);
294 }294 }
295 295
@@ -339,15 +339,15 @@ static bool __ww_mutex_wound(struct MUTEX *lock, · 43 unmodified lines
339 */339 */
340 if (owner != current) {340 if (owner != current) {
341 /*341 /*
342 * When waking up the task to wound, be sure to clear the342 * When waking up the task to wound, be sure to set the
343 * blocked_on pointer. Otherwise we can see circular343 * blocked_on to PROXY_WAKING. Otherwise we can see
344 * blocked_on relationships that can't resolve.344 * circular blocked_on relationships that can't resolve.
345 *345 *
346 * NOTE: We pass NULL here instead of lock, because we346 * NOTE: We pass NULL here instead of lock, because we
347 * are waking the mutex owner, who may be currently347 * are waking the mutex owner, who may be currently
348 * blocked on a different mutex.348 * blocked on a different mutex.
349 */349 */
350 clear_task_blocked_on(owner, NULL);350 set_task_blocked_on_waking(owner, NULL);
351 wake_q_add(wake_q, owner);351 wake_q_add(wake_q, owner);
352 }352 }
353 return true;353 return true;
kernel/sched/core.c+16−0
@@ -4239,6 +4239,13 @@ int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) · 4238 unmodified lines
4239 ttwu_queue(p, cpu, wake_flags);4239 ttwu_queue(p, cpu, wake_flags);
4240 }4240 }
4241out:4241out:
4242 /*
4243 * For now, if we've been woken up, clear the task->blocked_on
4244 * regardless if it was set to a mutex or PROXY_WAKING so the
4245 * task can run. We will need to be more careful later when
4246 * properly handling proxy migration
4247 */
4248 clear_task_blocked_on(p, NULL);
4242 if (success)4249 if (success)
4243 ttwu_stat(p, task_cpu(p), wake_flags);4250 ttwu_stat(p, task_cpu(p), wake_flags);
4244 4251
@@ -6600,6 +6607,10 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) · 2355 unmodified lines
6600 6607
6601 /* Follow blocked_on chain. */6608 /* Follow blocked_on chain. */
6602 for (p = donor; (mutex = p->blocked_on); p = owner) {6609 for (p = donor; (mutex = p->blocked_on); p = owner) {
6610 /* if its PROXY_WAKING, resched_idle so ttwu can complete */
6611 if (mutex == PROXY_WAKING)
6612 return proxy_resched_idle(rq);
6613
6603 /*6614 /*
6604 * By taking mutex->wait_lock we hold off concurrent mutex_unlock()6615 * By taking mutex->wait_lock we hold off concurrent mutex_unlock()
6605 * and ensure @owner sticks around.6616 * and ensure @owner sticks around.
@@ -6620,6 +6631,11 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) · 14 unmodified lines
6620 6631
6621 owner = __mutex_owner(mutex);6632 owner = __mutex_owner(mutex);
6622 if (!owner) {6633 if (!owner) {
6634 /*
6635 * If there is no owner, clear blocked_on
6636 * and return p so it can run and try to
6637 * acquire the lock
6638 */
6623 __clear_task_blocked_on(p, mutex);6639 __clear_task_blocked_on(p, mutex);
6624 return p;6640 return p;
6625 }6641 }