sched_ext: Add per-shard scx_sched storage scaffolding

Add struct scx_pshard and sch->pshard[] indexed by shard_idx, each entry allocated on its shard's NUMA node from scx_shard_node[si]. The struct starts empty (one dummy field). Follow-up patches will grow it as shard-local state lands. Only cid-type schedulers with an arena pool get pshards. Allocation happens after ops.init_cids() returns so any scx_bpf_cid_override() it issues has finalized scx_nr_cid_shards and scx_shard_node[]. sch->nr_pshards records the array size for the async RCU free path, which may run after a later scheduler's scx_cid_init() has rewritten the global. v3: Build and publish pshard[] fully-formed here rather than a later patch. v2: Free the partially-allocated pshard array on alloc failure. (sashiko AI)

Tejun Heo committed 4 months ago

Commit 8dba3bb1 parent 80e6ada4 files · +84 −0Browse files

Unified · Split

4 files

kernel/sched/ext/ext.c +8 −0
kernel/sched/ext/internal.h +18 −0
kernel/sched/ext/sub.c +54 −0
kernel/sched/ext/sub.h +4 −0

kernel/sched/ext/ext.c+8−0

 		free_pnode(sch->pnode[node]);
 	kfree(sch->pnode);
+	scx_free_pshards(sch);
 	rhashtable_walk_enter(&sch->dsq_hash, &rht_iter);
 	do {
 		rhashtable_walk_start(&rht_iter);
 		goto err_disable;
 	}
+	ret = scx_alloc_pshards(sch);
+	if (ret) {
+		cpus_read_unlock();
+		goto err_disable;
+	}
 	if (sch->ops.init) {
 		ret = SCX_CALL_OP_RET(sch, init, NULL);
 		if (ret) {

kernel/sched/ext/internal.h+18−0

 	struct scx_dispatch_q	global_dsq;
 };
+#ifdef CONFIG_EXT_SUB_SCHED
+struct scx_pshard {
+	int			_dummy;		/* until the first real field lands */
+};
+#endif
 struct scx_sched {
 	/*
 	 * cpu-form and cid-form ops share field offsets up to .priv (verified
 	 */
 	struct rhashtable	dsq_hash;
 	struct scx_sched_pnode	**pnode;
+#ifdef CONFIG_EXT_SUB_SCHED
+	struct scx_pshard	**pshard;	/* indexed by shard_idx */
+#endif
 	struct scx_sched_pcpu __percpu *pcpu;
 	u64			slice_dfl;
 	u32			dsp_max_batch;
 	s32			level;
+#ifdef CONFIG_EXT_SUB_SCHED
+	/*
+	 * pshard[] size captured at enable for the async RCU free path -
+	 * scx_nr_cid_shards may be rewritten by a later scx_cid_init() before
+	 * free runs. While sch is active, use the global.
+	 */
+	u32			nr_pshards;
+#endif
 	/*
 	 * Updates to the following warned bitfields can race causing RMW issues
 	 * but it doesn't really matter.

kernel/sched/ext/sub.c+54−0

@@ -82,6 +82,60 @@ void set_cgroup_sched(struct cgroup cgrp, struct scx_sched sch) · 81 unmodified lines
82	rcu_assign_pointer(pos->scx_sched, sch);	82	rcu_assign_pointer(pos->scx_sched, sch);
83	}	83	}
84		84
		85	static void free_pshard(struct scx_pshard *pshard)
		86	{
		87	kfree(pshard);
		88	}
		89
		90	void scx_free_pshards(struct scx_sched *sch)
		91	{
		92	s32 si;
		93
		94	if (!sch->pshard)
		95	return;
		96	for (si = 0; si < sch->nr_pshards; si++)
		97	free_pshard(sch->pshard[si]);
		98	kfree(sch->pshard);
		99	}
		100
		101	static struct scx_pshard alloc_pshard(struct scx_sched sch, s32 shard_idx, s32 node)
		102	{
		103	return kzalloc_node(sizeof(struct scx_pshard), GFP_KERNEL, node);
		104	}
		105
		106	s32 scx_alloc_pshards(struct scx_sched *sch)
		107	{
		108	struct scx_pshard **pshard;
		109	s32 si;
		110
		111	if (!sch->is_cid_type \|\| !sch->arena_pool)
		112	return 0;
		113
		114	pshard = kzalloc_objs(pshard[0], scx_nr_cid_shards, GFP_KERNEL);
		115	if (!pshard)
		116	return -ENOMEM;
		117
		118	for (si = 0; si < scx_nr_cid_shards; si++) {
		119	pshard[si] = alloc_pshard(sch, si, scx_shard_node[si]);
		120	if (!pshard[si]) {
		121	while (--si >= 0)
		122	free_pshard(pshard[si]);
		123	kfree(pshard);
		124	return -ENOMEM;
		125	}
		126	}
		127
		128	sch->nr_pshards = scx_nr_cid_shards;
		129	/*
		130	* Publish only after every entry is built so a reader observing
		131	* @sch->pshard never sees a partially-filled array. Pair the store
		132	* with a barrier and READ_ONCE() on the read side.
		133	*/
		134	smp_wmb();
		135	WRITE_ONCE(sch->pshard, pshard);
		136	return 0;
		137	}
		138
85	static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq);	139	static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq);
86		140
87	void drain_descendants(struct scx_sched *sch)	141	void drain_descendants(struct scx_sched *sch)

kernel/sched/ext/sub.h+4−0

 void scx_sub_disable(struct scx_sched *sch);
 void scx_sub_enable_workfn(struct kthread_work *work);
 bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux);
+void scx_free_pshards(struct scx_sched *sch);
+s32 scx_alloc_pshards(struct scx_sched *sch);
 static inline const char *sch_cgrp_path(struct scx_sched *sch)
 {
 static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {}
 static inline void drain_descendants(struct scx_sched *sch) { }
 static inline void scx_sub_disable(struct scx_sched *sch) { }
+static inline void scx_free_pshards(struct scx_sched *sch) {}
+static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; }
 #endif	/* CONFIG_EXT_SUB_SCHED */