History

sched_ext: Add per-shard scx_sched storage scaffolding

Add struct scx_pshard and sch->pshard[] indexed by shard_idx, each entry allocated on its shard's NUMA node from scx_shard_node[si]. The struct starts empty (one dummy field). Follow-up patches will grow it as shard-local state lands. Only cid-type schedulers with an arena pool get pshards. Allocation happens after ops.init_cids() returns so any scx_bpf_cid_override() it issues has finalized scx_nr_cid_shards and scx_shard_node[]. sch->nr_pshards records the array size for the async RCU free path, which may run after a later scheduler's scx_cid_init() has rewritten the global. v3: Build and publish pshard[] fully-formed here rather than a later patch. v2: Free the partially-allocated pshard array on alloc failure. (sashiko AI)

Tejun Heo committed 4 months ago

Commit 8dba3bb1 parent 80e6ada4 files · +84 −0Browse files

Unified · Split

4 files
  1. kernel/sched/ext/ext.c +8 −0
  2. kernel/sched/ext/internal.h +18 −0
  3. kernel/sched/ext/sub.c +54 −0
  4. kernel/sched/ext/sub.h +4 −0
kernel/sched/ext/ext.c+8−0
@@ -4681,6 +4681,8 @@ static void scx_sched_free_rcu_work(struct work_struct *work) · 4680 unmodified lines
46814681 free_pnode(sch->pnode[node]);
46824682 kfree(sch->pnode);
46834683
4684+ scx_free_pshards(sch);
4685+
46844686 rhashtable_walk_enter(&sch->dsq_hash, &rht_iter);
46854687 do {
46864688 rhashtable_walk_start(&rht_iter);
@@ -6763,6 +6765,12 @@ static void scx_root_enable_workfn(struct kthread_work *work) · 2076 unmodified lines
67636765 goto err_disable;
67646766 }
67656767
6768+ ret = scx_alloc_pshards(sch);
6769+ if (ret) {
6770+ cpus_read_unlock();
6771+ goto err_disable;
6772+ }
6773+
67666774 if (sch->ops.init) {
67676775 ret = SCX_CALL_OP_RET(sch, init, NULL);
67686776 if (ret) {
kernel/sched/ext/internal.h+18−0
@@ -1183,6 +1183,12 @@ struct scx_sched_pnode { · 1182 unmodified lines
11831183 struct scx_dispatch_q global_dsq;
11841184 };
11851185
1186+#ifdef CONFIG_EXT_SUB_SCHED
1187+struct scx_pshard {
1188+ int _dummy; /* until the first real field lands */
1189+};
1190+#endif
1191+
11861192 struct scx_sched {
11871193 /*
11881194 * cpu-form and cid-form ops share field offsets up to .priv (verified
@@ -1230,6 +1236,9 @@ struct scx_sched { · 41 unmodified lines
12301236 */
12311237 struct rhashtable dsq_hash;
12321238 struct scx_sched_pnode **pnode;
1239+#ifdef CONFIG_EXT_SUB_SCHED
1240+ struct scx_pshard **pshard; /* indexed by shard_idx */
1241+#endif
12331242 struct scx_sched_pcpu __percpu *pcpu;
12341243
12351244 u64 slice_dfl;
@@ -1245,6 +1254,15 @@ struct scx_sched { · 9 unmodified lines
12451254 u32 dsp_max_batch;
12461255 s32 level;
12471256
1257+#ifdef CONFIG_EXT_SUB_SCHED
1258+ /*
1259+ * pshard[] size captured at enable for the async RCU free path -
1260+ * scx_nr_cid_shards may be rewritten by a later scx_cid_init() before
1261+ * free runs. While sch is active, use the global.
1262+ */
1263+ u32 nr_pshards;
1264+#endif
1265+
12481266 /*
12491267 * Updates to the following warned bitfields can race causing RMW issues
12501268 * but it doesn't really matter.
kernel/sched/ext/sub.c+54−0
@@ -82,6 +82,60 @@ void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) · 81 unmodified lines
8282 rcu_assign_pointer(pos->scx_sched, sch);
8383 }
8484
85+static void free_pshard(struct scx_pshard *pshard)
86+{
87+ kfree(pshard);
88+}
89+
90+void scx_free_pshards(struct scx_sched *sch)
91+{
92+ s32 si;
93+
94+ if (!sch->pshard)
95+ return;
96+ for (si = 0; si < sch->nr_pshards; si++)
97+ free_pshard(sch->pshard[si]);
98+ kfree(sch->pshard);
99+}
100+
101+static struct scx_pshard *alloc_pshard(struct scx_sched *sch, s32 shard_idx, s32 node)
102+{
103+ return kzalloc_node(sizeof(struct scx_pshard), GFP_KERNEL, node);
104+}
105+
106+s32 scx_alloc_pshards(struct scx_sched *sch)
107+{
108+ struct scx_pshard **pshard;
109+ s32 si;
110+
111+ if (!sch->is_cid_type || !sch->arena_pool)
112+ return 0;
113+
114+ pshard = kzalloc_objs(pshard[0], scx_nr_cid_shards, GFP_KERNEL);
115+ if (!pshard)
116+ return -ENOMEM;
117+
118+ for (si = 0; si < scx_nr_cid_shards; si++) {
119+ pshard[si] = alloc_pshard(sch, si, scx_shard_node[si]);
120+ if (!pshard[si]) {
121+ while (--si >= 0)
122+ free_pshard(pshard[si]);
123+ kfree(pshard);
124+ return -ENOMEM;
125+ }
126+ }
127+
128+ sch->nr_pshards = scx_nr_cid_shards;
129+ /*
130+ * Publish only after every entry is built so a reader observing
131+ * @sch->pshard never sees a partially-filled array. Pair the store
132+ * with a barrier and READ_ONCE() on the read side.
133+ */
134+ smp_wmb();
135+ WRITE_ONCE(sch->pshard, pshard);
136+ return 0;
137+}
138+
85139 static DECLARE_WAIT_QUEUE_HEAD(scx_unlink_waitq);
86140
87141 void drain_descendants(struct scx_sched *sch)
kernel/sched/ext/sub.h+4−0
@@ -24,6 +24,8 @@ void drain_descendants(struct scx_sched *sch); · 23 unmodified lines
2424 void scx_sub_disable(struct scx_sched *sch);
2525 void scx_sub_enable_workfn(struct kthread_work *work);
2626 bool scx_bpf_sub_dispatch(u64 cgroup_id, const struct bpf_prog_aux *aux);
27+void scx_free_pshards(struct scx_sched *sch);
28+s32 scx_alloc_pshards(struct scx_sched *sch);
2729
2830 static inline const char *sch_cgrp_path(struct scx_sched *sch)
2931 {
@@ -39,6 +41,8 @@ static inline const char *sch_cgrp_path(struct scx_sched *sch) { return "/"; } · 9 unmodified lines
3941 static inline void set_cgroup_sched(struct cgroup *cgrp, struct scx_sched *sch) {}
4042 static inline void drain_descendants(struct scx_sched *sch) { }
4143 static inline void scx_sub_disable(struct scx_sched *sch) { }
44+static inline void scx_free_pshards(struct scx_sched *sch) {}
45+static inline s32 scx_alloc_pshards(struct scx_sched *sch) { return 0; }
4246
4347 #endif /* CONFIG_EXT_SUB_SCHED */
4448