History

sched_ext: Authorize remote-move inserts against the placing scheduler

scx_local_or_reject_dsq() authorizes a local-DSQ insert against the caps of the scheduler doing the insert. On the consume/dispatch paths that is the scheduler running balance_one(), passed down through scx_consume_dispatch_q() and move_local_task_to_local_dsq(), so the check is correct. The remote-move path loses it. move_remote_task_to_local_dsq() re-activates @p on the destination rq through enqueue_task_scx(), which reconstructs the scheduler from the task, i.e. @p's owner. When an ancestor places a descendant's task - e.g. draining a bypassed sub-scheduler - the owner is a sub-scheduler of the placer, so authorizing against the owner checks a narrower cap set and can spuriously reject a task the placer is entitled to run. Carry the placing scheduler across the activate_task() boundary the same way enq_flags already are, via a per-rq field set only for the duration of the re-activation, and have scx_local_or_reject_dsq() authorize against it. The placer's caps are a superset of the owner's, so this admits what the placer may run and keeps rejecting what it may not. v2: Document @sch in move_remote_task_to_local_dsq()'s kerneldoc. (Andrea)

Tejun Heo committed 4 months ago

Commit f2c9f511 parent 6ea3be33 files · +25 −15Browse files

Unified · Split

3 files
  1. kernel/sched/ext/ext.c +16 −12
  2. kernel/sched/ext/sub.c +7 −2
  3. kernel/sched/sched.h +2 −1
kernel/sched/ext/ext.c+16−12
@@ -2268,6 +2268,7 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, · 2267 unmodified lines
22682268
22692269 /**
22702270 * move_remote_task_to_local_dsq - Move a task from a foreign rq to a local DSQ
2271+ * @sch: scheduler placing @p
22712272 * @p: task to move
22722273 * @enq_flags: %SCX_ENQ_*
22732274 * @src_rq: rq to move the task from, locked on entry, released on return
@@ -2275,7 +2276,8 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch, · 1 unmodified line
22752276 *
22762277 * Move @p which is currently on @src_rq to @dst_rq's local DSQ.
22772278 */
2278−static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
2279+static void move_remote_task_to_local_dsq(struct scx_sched *sch,
2280+ struct task_struct *p, u64 enq_flags,
22792281 struct rq *src_rq, struct rq *dst_rq)
22802282 {
22812283 lockdep_assert_rq_held(src_rq);
@@ -2291,15 +2293,19 @@ static void move_remote_task_to_local_dsq(struct task_struct *p, u64 enq_flags, · 9 unmodified lines
22912293 switch_rq_lock(src_rq, dst_rq);
22922294
22932295 /*
2294− * We want to pass scx-specific enq_flags but activate_task() will
2295− * truncate the upper 32 bit. As we own @rq, we can pass them through
2296− * @rq->scx.remote_activate_enq_flags instead.
2296+ * activate_task() below truncates enq_flags to 32 bits and re-derives
2297+ * @p's owner, dropping our scx flags and the placing @sch. We own @rq,
2298+ * so stash both across the call. The enqueue reads them back, keeping
2299+ * the scx flags and checking caps against the placer, not the owner.
22972300 */
22982301 WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr));
2299− WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags);
2302+ WARN_ON_ONCE(dst_rq->scx.remote_activate_enq_flags ||
2303+ dst_rq->scx.remote_activate_sch);
23002304 dst_rq->scx.remote_activate_enq_flags = enq_flags;
2305+ dst_rq->scx.remote_activate_sch = sch;
23012306 activate_task(dst_rq, p, 0);
23022307 dst_rq->scx.remote_activate_enq_flags = 0;
2308+ dst_rq->scx.remote_activate_sch = NULL;
23032309 }
23042310
23052311 /*
@@ -2431,12 +2437,12 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p, · 125 unmodified lines
24312437 !WARN_ON_ONCE(src_rq != task_rq(p));
24322438 }
24332439
2434−static bool consume_remote_task(struct rq *this_rq,
2440+static bool consume_remote_task(struct scx_sched *sch, struct rq *this_rq,
24352441 struct task_struct *p, u64 enq_flags,
24362442 struct scx_dispatch_q *dsq, struct rq *src_rq)
24372443 {
24382444 if (unlink_dsq_and_switch_rq_lock(p, dsq, this_rq, src_rq)) {
2439− move_remote_task_to_local_dsq(p, enq_flags, src_rq, this_rq);
2445+ move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, this_rq);
24402446 return true;
24412447 } else {
24422448 switch_rq_lock(src_rq, this_rq);
@@ -2497,8 +2503,7 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch, · 54 unmodified lines
24972503 raw_spin_unlock(&src_dsq->lock);
24982504 } else {
24992505 raw_spin_unlock(&src_dsq->lock);
2500− move_remote_task_to_local_dsq(p, enq_flags,
2501− src_rq, dst_rq);
2506+ move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq);
25022507 }
25032508 } else {
25042509 /*
@@ -2551,7 +2556,7 @@ retry: · 46 unmodified lines
25512556 }
25522557
25532558 if (task_can_run_on_remote_rq(sch, p, rq, false)) {
2554− if (likely(consume_remote_task(rq, p, enq_flags, dsq, task_rq)))
2559+ if (likely(consume_remote_task(sch, rq, p, enq_flags, dsq, task_rq)))
25552560 return true;
25562561 goto retry;
25572562 }
@@ -2644,8 +2649,7 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq, · 86 unmodified lines
26442649 scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)),
26452650 p, enq_flags | SCX_ENQ_GDSQ_FALLBACK);
26462651 } else {
2647− move_remote_task_to_local_dsq(p, enq_flags,
2648− src_rq, dst_rq);
2652+ move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq);
26492653 /* task has been moved to dst_rq, which is now locked */
26502654 locked_rq = dst_rq;
26512655 }
kernel/sched/ext/sub.c+7−2
@@ -237,12 +237,17 @@ struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq · 236 unmodified lines
237237 struct task_struct *p, u64 *enq_flags)
238238 {
239239 s32 cid = __scx_cpu_to_cid(cpu_of(rq));
240+ struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;
240241 u64 needed = scx_caps_for_enq(*enq_flags);
241242 u64 missing;
242243
244+ /*
245+ * On a remote activation the scheduling sched (@asch) differs from
246+ * @p's owner (@sch). Check caps against the scheduling sched.
247+ */
243248 if (*enq_flags & SCX_ENQ_PREEMPT)
244− needed |= scx_caps_for_preempt(sch, rq);
245− missing = scx_missing_caps(sch, cpu_of(rq), needed);
249+ needed |= scx_caps_for_preempt(asch, rq);
250+ missing = scx_missing_caps(asch, cpu_of(rq), needed);
246251
247252 /* requirements met */
248253 if (likely(!missing))
kernel/sched/sched.h+2−1
@@ -800,8 +800,9 @@ struct scx_rq { · 799 unmodified lines
800800 struct list_head runnable_list; /* runnable tasks on this rq */
801801 struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */
802802 unsigned long ops_qseq;
803− /* see move_remote_task_to_local_dsq() */
803+ /* both stashed across the activate_task() in move_remote_task_to_local_dsq() */
804804 u64 remote_activate_enq_flags;
805+ struct scx_sched *remote_activate_sch;
805806 u32 nr_running;
806807 u32 cpuperf_target; /* [0, SCHED_CAPACITY_SCALE] */
807808 bool in_select_cpu;