scx_local_or_reject_dsq() authorizes a local-DSQ insert against the caps of the scheduler doing the insert. On the consume/dispatch paths that is the scheduler running balance_one(), passed down through scx_consume_dispatch_q() and move_local_task_to_local_dsq(), so the check is correct. The remote-move path loses it. move_remote_task_to_local_dsq() re-activates @p on the destination rq through enqueue_task_scx(), which reconstructs the scheduler from the task, i.e. @p's owner. When an ancestor places a descendant's task - e.g. draining a bypassed sub-scheduler - the owner is a sub-scheduler of the placer, so authorizing against the owner checks a narrower cap set and can spuriously reject a task the placer is entitled to run. Carry the placing scheduler across the activate_task() boundary the same way enq_flags already are, via a per-rq field set only for the duration of the re-activation, and have scx_local_or_reject_dsq() authorize against it. The placer's caps are a superset of the owner's, so this admits what the placer may run and keeps rejecting what it may not. v2: Document @sch in move_remote_task_to_local_dsq()'s kerneldoc. (Andrea)
Showing the changes to kernel (3 of 3 files). Show all
Unified · Split
| @@ -2268,6 +2268,7 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, · 2267 unmodified lines | ||
| 2268 | 2268 | |
| 2269 | 2269 | /** |
| 2270 | 2270 | * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ |
| 2271 | + * @sch: scheduler placing @p | |
| 2271 | 2272 | * @p: task to move |
| 2272 | 2273 | * @enq_flags: %SCX_ENQ_* |
| 2273 | 2274 | * @src_rq: rq to move the task from, locked on entry, released on return |
| @@ -2275,7 +2276,8 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, · 1 unmodified line | ||
| 2275 | 2276 | * |
| 2276 | 2277 | * Move @p which is currently on @src_rq to @dst_rq's local DSQ. |
| 2277 | 2278 | */ |
| 2278 | −static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, | |
| 2279 | +static void move_remote_task_to_local_dsq(struct scx_sched *sch, | |
| 2280 | + struct task_struct *p, u64 enq_flags, | |
| 2279 | 2281 | struct rq *src_rq, struct rq *dst_rq) |
| 2280 | 2282 | { |
| 2281 | 2283 | lockdep_assert_rq_held(src_rq); |
| @@ -2291,15 +2293,19 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, · 9 unmodified lines | ||
| 2291 | 2293 | switch_rq_lock(src_rq, dst_rq); |
| 2292 | 2294 | |
| 2293 | 2295 | /* |
| 2294 | − * We want to pass scx-specific enq_flags but activate_task() will | |
| 2295 | − * truncate the upper 32 bit. As we own @rq, we can pass them through | |
| 2296 | − * @rq->scx.remote_activate_enq_flags instead. | |
| 2296 | + * activate_task() below truncates enq_flags to 32 bits and re-derives | |
| 2297 | + * @p's owner, dropping our scx flags and the placing @sch. We own @rq, | |
| 2298 | + * so stash both across the call. The enqueue reads them back, keeping | |
| 2299 | + * the scx flags and checking caps against the placer, not the owner. | |
| 2297 | 2300 | */ |
| 2298 | 2301 | WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)); |
| 2299 | − WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags); | |
| 2302 | + WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags || | |
| 2303 | + dst_rq->scx.remote_activate_sch); | |
| 2300 | 2304 | dst_rq->scx.remote_activate_enq_flags = enq_flags; |
| 2305 | + dst_rq->scx.remote_activate_sch = sch; | |
| 2301 | 2306 | activate_task(dst_rq, p, 0); |
| 2302 | 2307 | dst_rq->scx.remote_activate_enq_flags = 0; |
| 2308 | + dst_rq->scx.remote_activate_sch = NULL; | |
| 2303 | 2309 | } |
| 2304 | 2310 | |
| 2305 | 2311 | /* |
| @@ -2431,12 +2437,12 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, · 125 unmodified lines | ||
| 2431 | 2437 | !WARN_ON_ONCE(src_rq != task_rq(p)); |
| 2432 | 2438 | } |
| 2433 | 2439 | |
| 2434 | −static bool consume_remote_task(struct rq *this_rq, | |
| 2440 | +static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq, | |
| 2435 | 2441 | struct task_struct *p, u64 enq_flags, |
| 2436 | 2442 | struct scx_dispatch_q *dsq, struct rq *src_rq) |
| 2437 | 2443 | { |
| 2438 | 2444 | if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) { |
| 2439 | − move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq); | |
| 2445 | + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq); | |
| 2440 | 2446 | return true; |
| 2441 | 2447 | } else { |
| 2442 | 2448 | switch_rq_lock(src_rq, this_rq); |
| @@ -2497,8 +2503,7 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, · 54 unmodified lines | ||
| 2497 | 2503 | raw_spin_unlock(&src_dsq->lock); |
| 2498 | 2504 | } else { |
| 2499 | 2505 | raw_spin_unlock(&src_dsq->lock); |
| 2500 | − move_remote_task_to_local_dsq(p, enq_flags, | |
| 2501 | − src_rq, dst_rq); | |
| 2506 | + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); | |
| 2502 | 2507 | } |
| 2503 | 2508 | } else { |
| 2504 | 2509 | /* |
| @@ -2551,7 +2556,7 @@ retry: · 46 unmodified lines | ||
| 2551 | 2556 | } |
| 2552 | 2557 | |
| 2553 | 2558 | if (task_can_run_on_remote_rq(sch, p, rq, false)) { |
| 2554 | − if (likely(consume_remote_task(rq, p, enq_flags, dsq, task_rq))) | |
| 2559 | + if (likely(consume_remote_task(sch, rq, p, enq_flags, dsq, task_rq))) | |
| 2555 | 2560 | return true; |
| 2556 | 2561 | goto retry; |
| 2557 | 2562 | } |
| @@ -2644,8 +2649,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, · 86 unmodified lines | ||
| 2644 | 2649 | scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)), |
| 2645 | 2650 | p, enq_flags | SCX_ENQ_GDSQ_FALLBACK); |
| 2646 | 2651 | } else { |
| 2647 | − move_remote_task_to_local_dsq(p, enq_flags, | |
| 2648 | − src_rq, dst_rq); | |
| 2652 | + move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq); | |
| 2649 | 2653 | /* task has been moved to dst_rq, which is now locked */ |
| 2650 | 2654 | locked_rq = dst_rq; |
| 2651 | 2655 | } |
| @@ -237,12 +237,17 @@ struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq · 236 unmodified lines | ||
| 237 | 237 | struct task_struct *p, u64 *enq_flags) |
| 238 | 238 | { |
| 239 | 239 | s32 cid = __scx_cpu_to_cid(cpu_of(rq)); |
| 240 | + struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch; | |
| 240 | 241 | u64 needed = scx_caps_for_enq(*enq_flags); |
| 241 | 242 | u64 missing; |
| 242 | 243 | |
| 244 | + /* | |
| 245 | + * On a remote activation the scheduling sched (@asch) differs from | |
| 246 | + * @p's owner (@sch). Check caps against the scheduling sched. | |
| 247 | + */ | |
| 243 | 248 | if (*enq_flags & SCX_ENQ_PREEMPT) |
| 244 | − needed |= scx_caps_for_preempt(sch, rq); | |
| 245 | − missing = scx_missing_caps(sch, cpu_of(rq), needed); | |
| 249 | + needed |= scx_caps_for_preempt(asch, rq); | |
| 250 | + missing = scx_missing_caps(asch, cpu_of(rq), needed); | |
| 246 | 251 | |
| 247 | 252 | /* requirements met */ |
| 248 | 253 | if (likely(!missing)) |
| @@ -800,8 +800,9 @@ struct scx_rq { · 799 unmodified lines | ||
| 800 | 800 | struct list_head runnable_list; /* runnable tasks on this rq */ |
| 801 | 801 | struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */ |
| 802 | 802 | unsigned long ops_qseq; |
| 803 | − /* see move_remote_task_to_local_dsq() */ | |
| 803 | + /* both stashed across the activate_task() in move_remote_task_to_local_dsq() */ | |
| 804 | 804 | u64 remote_activate_enq_flags; |
| 805 | + struct scx_sched *remote_activate_sch; | |
| 805 | 806 | u32 nr_running; |
| 806 | 807 | u32 cpuperf_target; /* [0, SCHED_CAPACITY_SCALE] */ |
| 807 | 808 | bool in_select_cpu; |