sched_ext: Add the scx_has_subs static key and gate sub-sched hot paths

With CONFIG_EXT_SUB_SCHED=y but no sub-scheduler attached - the common case - hot paths still pay for sub-sched bookkeeping. Gate it behind __scx_has_subs, a static key counting live sub-schedulers, so that a root-only system stops paying. Most conversions are simple skip-if-no-sub tests. scx_idle_notify() is special - it's a hierarchy walk, so give it a fast path which notifies the root directly using the same tests as the walk. A pending SCX_RQ_SUB_IDLE_RENOTIFY can be ignored as no sub can be owed one and the caller clears the flag either way.

Tejun Heo committed 4 months ago

Commit 8946dbd1 parent 6ba3bd65 files · +65 −4Browse files

Showing the changes to kernel (5 of 5 files). Show all

Unified · Split

5 files

kernel/sched/ext/ext.c +4 −0
kernel/sched/ext/idle.c +10 −0
kernel/sched/ext/internal.h +17 −2
kernel/sched/ext/sub.c +19 −2
kernel/sched/ext/sub.h +15 −0

kernel/sched/ext/ext.c+4−0

@@ -4920,6 +4920,10 @@ static void scx_sched_free_rcu_work(struct work_struct *work) · 4919 unmodified lines
4920	scx_arena_pool_destroy(sch);	4920	scx_arena_pool_destroy(sch);
4921	if (sch->arena_map)	4921	if (sch->arena_map)
4922	bpf_map_put(sch->arena_map);	4922	bpf_map_put(sch->arena_map);
		4923
		4924	/* @sch is completely inactive by now */
		4925	scx_dec_has_subs(sch);
		4926
4923	kfree(sch);	4927	kfree(sch);
4924	}	4928	}
4925		4929

kernel/sched/ext/idle.c+10−0

@@ -746,6 +746,16 @@ static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_ · 745 unmodified lines
746		746
747	lockdep_assert_rq_held(rq);	747	lockdep_assert_rq_held(rq);
748		748
		749	/* with no sub-sched, only the root can be owed a notification */
		750	if (!scx_has_subs()) {
		751	struct scx_sched *sch = scx_root;
		752
		753	if ((do_notify \|\| root_renotify) &&
		754	SCX_HAS_OP(sch, update_idle) && !scx_bypassing(sch, cpu))
		755	SCX_CALL_OP(sch, update_idle, rq, cid, idle);
		756	return;
		757	}
		758
749	pos = scx_next_descendant_pre(NULL, scx_root);	759	pos = scx_next_descendant_pre(NULL, scx_root);
750	while (pos) {	760	while (pos) {
751	bool forced = false;	761	bool forced = false;

kernel/sched/ext/internal.h+17−2

  */
 #define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...)			\
 do {										\
-	WARN_ON_ONCE((sch) != scx_task_sched_rcu(task));			\
+	WARN_ON_ONCE(scx_has_subs() && (sch) != scx_task_sched_rcu(task));	\
 	__SCX_CALL_OP_TASK((sch), ops, op, locked_rq, task, ##args);		\
 } while (0)
 #define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...)			\
 ({										\
 	__typeof__((sch)->ops.op(task, ##args)) __ret;				\
-	WARN_ON_ONCE((sch) != scx_task_sched_rcu(task));			\
+	WARN_ON_ONCE(scx_has_subs() && (sch) != scx_task_sched_rcu(task));	\
 	WARN_ON_ONCE(current->scx.kf_tasks[0]);					\
 	current->scx.kf_tasks[0] = task;					\
 	__ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args);		\
 }
 #ifdef CONFIG_EXT_SUB_SCHED
+DECLARE_STATIC_KEY_FALSE(__scx_has_subs);
+/**
+ * scx_has_subs - Whether any sub-scheduler exists
+ *
+ * Gates the sub-sched portions of hot paths so that a root-only system doesn't
+ * pay for them. See scx_sub_enable_workfn() and scx_sched_free_rcu_work().
+ */
+static inline bool scx_has_subs(void)
+{
+	return static_branch_unlikely(&__scx_has_subs);
+}
 /**
  * scx_task_sched - Find scx_sched scheduling a task
  * @p: task of interest
 		return NULL;
 }
 #else	/* CONFIG_EXT_SUB_SCHED */
+static inline bool scx_has_subs(void) { return false; }
 static inline struct scx_sched *scx_task_sched(const struct task_struct *p)
 {
 	return rcu_dereference_protected(scx_root,

kernel/sched/ext/sub.c+19−2

 #ifdef CONFIG_EXT_SUB_SCHED
+/*
+ * On while any sub-scheduler exists so that a root-only system doesn't pay for
+ * the sub-sched portions of hot paths. See scx_has_subs().
+ */
+DEFINE_STATIC_KEY_FALSE(__scx_has_subs);
 /**
  * scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk
  * @pos: current position
 struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq,
 					       struct task_struct *p, u64 *enq_flags)
 {
+	if (!scx_has_subs())
+		return &rq->scx.local_dsq;
 	s32 cid = __scx_cpu_to_cid(cpu_of(rq));
 	struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;
 	u64 needed = scx_caps_for_enq(*enq_flags);
 	lockdep_assert_rq_held(rq);
-	if (list_empty(&rq->scx.reject_dsq.list))
+	if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list))
 		return;
 	/*
 	lockdep_assert_rq_held(rq);
-	if (likely(llist_empty(&rq->scx.ecaps_to_sync)))
+	if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync)))
 		return;
 	/*
 	kobject_get(&parent->kobj);
 	raw_spin_unlock_irq(&scx_sched_lock);
+	/*
+	 * Flip the hot-path gates before ops->priv is published - the sub's
+	 * programs can e.g. kick cpus from that point on. The matching dec is
+	 * at the end of scx_sched_free_rcu_work().
+	 */
+	static_branch_inc(&__scx_has_subs);
 	/* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */
 	sch = scx_alloc_and_add_sched(cmd, cgrp, parent);
 	kobject_put(&parent->kobj);
 	if (IS_ERR(sch)) {
+		static_branch_dec(&__scx_has_subs);
 		ret = PTR_ERR(sch);
 		goto out_unlock;
 	}

kernel/sched/ext/sub.h+15−0

 	return sch->cgrp_path;
 }
+/* a dying sub's hot-path influence ends in scx_sched_free_rcu_work() */
+static inline void scx_dec_has_subs(struct scx_sched *sch)
+{
+	if (sch->level)
+		static_branch_dec(&__scx_has_subs);
+}
 #else	/* CONFIG_EXT_SUB_SCHED */
 static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
 static inline struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; }
 static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; }
 static inline void scx_reenq_reject(struct rq *rq) {}
+static inline void scx_dec_has_subs(struct scx_sched *sch) {}
 #endif	/* CONFIG_EXT_SUB_SCHED */
 {
 	u64 ecaps;
+	/* no sub-scheds, no missing caps */
+	if (!scx_has_subs())
+		return 0;
 	/* root holds every cap on every cpu */
 	if (!sch->level)
 		return 0;
 /* may @p keep running on @rq's cpu? requires baseline cpu access */
 static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p)
 {
+	if (!scx_has_subs())
+		return true;
 	/* a migration-disabled task is let in without caps, keep it likewise */
 	if (unlikely(is_migration_disabled(p)))
 		return true;