History

sched_ext: Add the scx_has_subs static key and gate sub-sched hot paths

With CONFIG_EXT_SUB_SCHED=y but no sub-scheduler attached - the common case - hot paths still pay for sub-sched bookkeeping. Gate it behind __scx_has_subs, a static key counting live sub-schedulers, so that a root-only system stops paying. Most conversions are simple skip-if-no-sub tests. scx_idle_notify() is special - it's a hierarchy walk, so give it a fast path which notifies the root directly using the same tests as the walk. A pending SCX_RQ_SUB_IDLE_RENOTIFY can be ignored as no sub can be owed one and the caller clears the flag either way.

Tejun Heo committed 4 months ago

Commit 8946dbd1 parent 6ba3bd65 files · +65 −4Browse files

Unified · Split

5 files
  1. kernel/sched/ext/ext.c +4 −0
  2. kernel/sched/ext/idle.c +10 −0
  3. kernel/sched/ext/internal.h +17 −2
  4. kernel/sched/ext/sub.c +19 −2
  5. kernel/sched/ext/sub.h +15 −0
kernel/sched/ext/ext.c+4−0
@@ -4920,6 +4920,10 @@ static void scx_sched_free_rcu_work(struct work_struct *work) · 4919 unmodified lines
49204920 scx_arena_pool_destroy(sch);
49214921 if (sch->arena_map)
49224922 bpf_map_put(sch->arena_map);
4923+
4924+ /* @sch is completely inactive by now */
4925+ scx_dec_has_subs(sch);
4926+
49234927 kfree(sch);
49244928 }
49254929
kernel/sched/ext/idle.c+10−0
@@ -746,6 +746,16 @@ static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_ · 745 unmodified lines
746746
747747 lockdep_assert_rq_held(rq);
748748
749+ /* with no sub-sched, only the root can be owed a notification */
750+ if (!scx_has_subs()) {
751+ struct scx_sched *sch = scx_root;
752+
753+ if ((do_notify || root_renotify) &&
754+ SCX_HAS_OP(sch, update_idle) && !scx_bypassing(sch, cpu))
755+ SCX_CALL_OP(sch, update_idle, rq, cid, idle);
756+ return;
757+ }
758+
749759 pos = scx_next_descendant_pre(NULL, scx_root);
750760 while (pos) {
751761 bool forced = false;
kernel/sched/ext/internal.h+17−2
@@ -2110,7 +2110,7 @@ do { \ · 2109 unmodified lines
21102110 */
21112111 #define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \
21122112 do { \
2113− WARN_ON_ONCE((sch) != scx_task_sched_rcu(task)); \
2113+ WARN_ON_ONCE(scx_has_subs() && (sch) != scx_task_sched_rcu(task)); \
21142114 __SCX_CALL_OP_TASK((sch), ops, op, locked_rq, task, ##args); \
21152115 } while (0)
21162116
@@ -2125,7 +2125,7 @@ do { \ · 8 unmodified lines
21252125 #define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \
21262126 ({ \
21272127 __typeof__((sch)->ops.op(task, ##args)) __ret; \
2128− WARN_ON_ONCE((sch) != scx_task_sched_rcu(task)); \
2128+ WARN_ON_ONCE(scx_has_subs() && (sch) != scx_task_sched_rcu(task)); \
21292129 WARN_ON_ONCE(current->scx.kf_tasks[0]); \
21302130 current->scx.kf_tasks[0] = task; \
21312131 __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \
@@ -2165,6 +2165,19 @@ static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu) · 33 unmodified lines
21652165 }
21662166
21672167 #ifdef CONFIG_EXT_SUB_SCHED
2168+DECLARE_STATIC_KEY_FALSE(__scx_has_subs);
2169+
2170+/**
2171+ * scx_has_subs - Whether any sub-scheduler exists
2172+ *
2173+ * Gates the sub-sched portions of hot paths so that a root-only system doesn't
2174+ * pay for them. See scx_sub_enable_workfn() and scx_sched_free_rcu_work().
2175+ */
2176+static inline bool scx_has_subs(void)
2177+{
2178+ return static_branch_unlikely(&__scx_has_subs);
2179+}
2180+
21682181 /**
21692182 * scx_task_sched - Find scx_sched scheduling a task
21702183 * @p: task of interest
@@ -2259,6 +2272,8 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch) · 88 unmodified lines
22592272 return NULL;
22602273 }
22612274 #else /* CONFIG_EXT_SUB_SCHED */
2275+static inline bool scx_has_subs(void) { return false; }
2276+
22622277 static inline struct scx_sched *scx_task_sched(const struct task_struct *p)
22632278 {
22642279 return rcu_dereference_protected(scx_root,
kernel/sched/ext/sub.c+19−2
@@ -22,6 +22,12 @@ · 21 unmodified lines
2222
2323 #ifdef CONFIG_EXT_SUB_SCHED
2424
25+/*
26+ * On while any sub-scheduler exists so that a root-only system doesn't pay for
27+ * the sub-sched portions of hot paths. See scx_has_subs().
28+ */
29+DEFINE_STATIC_KEY_FALSE(__scx_has_subs);
30+
2531 /**
2632 * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk
2733 * @pos: current position
@@ -236,6 +242,9 @@ void scx_init_root_caps(struct scx_sched *sch) · 208 unmodified lines
236242 struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq,
237243 struct task_struct *p, u64 *enq_flags)
238244 {
245+ if (!scx_has_subs())
246+ return &rq->scx.local_dsq;
247+
239248 s32 cid = __scx_cpu_to_cid(cpu_of(rq));
240249 struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;
241250 u64 needed = scx_caps_for_enq(*enq_flags);
@@ -314,7 +323,7 @@ void scx_reenq_reject(struct rq *rq) · 72 unmodified lines
314323
315324 lockdep_assert_rq_held(rq);
316325
317− if (list_empty(&rq->scx.reject_dsq.list))
326+ if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list))
318327 return;
319328
320329 /*
@@ -497,7 +506,7 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev) · 176 unmodified lines
497506
498507 lockdep_assert_rq_held(rq);
499508
500− if (likely(llist_empty(&rq->scx.ecaps_to_sync)))
509+ if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync)))
501510 return;
502511
503512 /*
@@ -1016,10 +1025,18 @@ void scx_sub_enable_workfn(struct kthread_work *work) · 512 unmodified lines
10161025 kobject_get(&parent->kobj);
10171026 raw_spin_unlock_irq(&scx_sched_lock);
10181027
1028+ /*
1029+ * Flip the hot-path gates before ops->priv is published - the sub's
1030+ * programs can e.g. kick cpus from that point on. The matching dec is
1031+ * at the end of scx_sched_free_rcu_work().
1032+ */
1033+ static_branch_inc(&__scx_has_subs);
1034+
10191035 /* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */
10201036 sch = scx_alloc_and_add_sched(cmd, cgrp, parent);
10211037 kobject_put(&parent->kobj);
10221038 if (IS_ERR(sch)) {
1039+ static_branch_dec(&__scx_has_subs);
10231040 ret = PTR_ERR(sch);
10241041 goto out_unlock;
10251042 }
kernel/sched/ext/sub.h+15−0
@@ -44,6 +44,13 @@ static inline const char *sch_cgrp_path(struct scx_sched *sch) · 43 unmodified lines
4444 return sch->cgrp_path;
4545 }
4646
47+/* a dying sub's hot-path influence ends in scx_sched_free_rcu_work() */
48+static inline void scx_dec_has_subs(struct scx_sched *sch)
49+{
50+ if (sch->level)
51+ static_branch_dec(&__scx_has_subs);
52+}
53+
4754 #else /* CONFIG_EXT_SUB_SCHED */
4855
4956 static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
@@ -66,6 +73,7 @@ static inline void scx_discard_stale_ecaps_syncs(void) {} · 16 unmodified lines
6673 static inline struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; }
6774 static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; }
6875 static inline void scx_reenq_reject(struct rq *rq) {}
76+static inline void scx_dec_has_subs(struct scx_sched *sch) {}
6977
7078 #endif /* CONFIG_EXT_SUB_SCHED */
7179
@@ -96,6 +104,10 @@ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) · 24 unmodified lines
96104 {
97105 u64 ecaps;
98106
107+ /* no sub-scheds, no missing caps */
108+ if (!scx_has_subs())
109+ return 0;
110+
99111 /* root holds every cap on every cpu */
100112 if (!sch->level)
101113 return 0;
@@ -156,6 +168,9 @@ static inline u64 scx_caps_implied(u64 cap) · 54 unmodified lines
156168 /* may @p keep running on @rq's cpu? requires baseline cpu access */
157169 static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p)
158170 {
171+ if (!scx_has_subs())
172+ return true;
173+
159174 /* a migration-disabled task is let in without caps, keep it likewise */
160175 if (unlikely(is_migration_disabled(p)))
161176 return true;