sched_ext: Convert ops.set_cmask() to arena-resident cmask

ops_cid.set_cmask() expects a cmask. The kernel couldn't write into the arena, so it translated cpumask -> cmask in kernel memory and passed the result as a trusted pointer. The BPF cmask helpers all operate on arena cmasks though, so the BPF side had to word-by-word probe-read the kernel cmask into an arena cmask via cmask_copy_from_kernel() before any helper could touch it. It works, but is clumsy. With direct kernel-side arena access now in place, build the cmask in the arena. The kernel writes to it through the kern_va side of the dual mapping. BPF directly dereferences it via an __arena pointer like any other arena struct.

Tejun Heo committed 5 months ago

Commit abdc2511 parent 9eca0875 files · +75 −80Browse files

Showing the changes to kernel (3 of 5 files). Show all

Unified · Split

3 files

kernel/sched/ext.c +63 −5
kernel/sched/ext_cid.c +1 −19
kernel/sched/ext_internal.h +8 −2

kernel/sched/ext.c+63−5

@@ -621,11 +621,16 @@ static inline void scx_call_op_set_cpumask(struct scx_sched sch, struct rq rq, · 620 unmodified lines
621	621	update_locked_rq(rq);
622	622
623	623	if (scx_is_cid_type()) {
624		− struct scx_cmask *cmask = this_cpu_ptr(scx_set_cmask_scratch);
625		−
626		− lockdep_assert_irqs_disabled();
627		− scx_cpumask_to_cmask(cpumask, cmask);
628		− sch->ops_cid.set_cmask(task, cmask);
	624	+ struct scx_cmask kern_va = this_cpu_ptr(sch->set_cmask_scratch);
	625	+ unsigned long uaddr = (unsigned long)kern_va -
	626	+ bpf_arena_map_kern_vm_start(sch->arena_map);
	627	+ /*
	628	+ * Build the per-CPU arena cmask and hand BPF the uaddr. Caller
	629	+ * holds the rq lock with IRQs disabled, which makes us the sole
	630	+ * user of the scratch area.
	631	+ */
	632	+ scx_cpumask_to_cmask(cpumask, kern_va);
	633	+ sch->ops_cid.set_cmask(task, (struct scx_cmask *)uaddr);
629	634	} else {
630	635	sch->ops.set_cpumask(task, cpumask);
631	636	}
@@ -4949,6 +4954,48 @@ static const struct attribute_group scx_global_attr_group = { · 4317 unmodified lines
4949	4954	static void free_pnode(struct scx_sched_pnode *pnode);
4950	4955	static void free_exit_info(struct scx_exit_info *ei);
4951	4956
	4957	+static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch)
	4958	+{
	4959	+ size_t size = struct_size_t(struct scx_cmask, bits,
	4960	+ SCX_CMASK_NR_WORDS(num_possible_cpus()));
	4961	+ int cpu;
	4962	+
	4963	+ if (!sch->is_cid_type \|\| !sch->arena_pool)
	4964	+ return 0;
	4965	+
	4966	+ sch->set_cmask_scratch = alloc_percpu(struct scx_cmask *);
	4967	+ if (!sch->set_cmask_scratch)
	4968	+ return -ENOMEM;
	4969	+
	4970	+ for_each_possible_cpu(cpu) {
	4971	+ struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu);
	4972	+
	4973	+ *slot = scx_arena_alloc(sch, size);
	4974	+ if (!*slot)
	4975	+ return -ENOMEM;
	4976	+ scx_cmask_init(*slot, 0, num_possible_cpus());
	4977	+ }
	4978	+ return 0;
	4979	+}
	4980	+
	4981	+static void scx_set_cmask_scratch_free(struct scx_sched *sch)
	4982	+{
	4983	+ size_t size = struct_size_t(struct scx_cmask, bits,
	4984	+ SCX_CMASK_NR_WORDS(num_possible_cpus()));
	4985	+ int cpu;
	4986	+
	4987	+ if (!sch->set_cmask_scratch)
	4988	+ return;
	4989	+
	4990	+ for_each_possible_cpu(cpu) {
	4991	+ struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu);
	4992	+
	4993	+ scx_arena_free(sch, *slot, size);
	4994	+ }
	4995	+ free_percpu(sch->set_cmask_scratch);
	4996	+ sch->set_cmask_scratch = NULL;
	4997	+}
	4998	+
4952	4999	static void scx_sched_free_rcu_work(struct work_struct *work)
4953	5000	{
4954	5001	struct rcu_work *rcu_work = to_rcu_work(work);
@@ -5003,6 +5050,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) · 48 unmodified lines
5003	5050
5004	5051	rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);
5005	5052	free_exit_info(sch->exit_info);
	5053	+ scx_set_cmask_scratch_free(sch);
5006	5054	scx_arena_pool_destroy(sch);
5007	5055	if (sch->arena_map)
5008	5056	bpf_map_put(sch->arena_map);
@@ -7162,6 +7210,12 @@ static void scx_root_enable_workfn(struct kthread_work *work) · 2153 unmodified lines
7162	7210	goto err_disable;
7163	7211	}
7164	7212
	7213	+ ret = scx_set_cmask_scratch_alloc(sch);
	7214	+ if (ret) {
	7215	+ cpus_read_unlock();
	7216	+ goto err_disable;
	7217	+ }
	7218	+
7165	7219	for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++)
7166	7220	if (((void (**)(void))ops)[i])
7167	7221	set_bit(i, sch->has_op);
@@ -7484,6 +7538,10 @@ static void scx_sub_enable_workfn(struct kthread_work *work) · 316 unmodified lines
7484	7538	if (ret)
7485	7539	goto err_disable;
7486	7540
	7541	+ ret = scx_set_cmask_scratch_alloc(sch);
	7542	+ if (ret)
	7543	+ goto err_disable;
	7544	+
7487	7545	if (validate_ops(sch, ops))
7488	7546	goto err_disable;
7489	7547

kernel/sched/ext_cid.c+1−19

@@ -7,14 +7,6 @@ · 6 unmodified lines
7	7	*/
8	8	#include <linux/cacheinfo.h>
9	9
10		−/*
11		− * Per-cpu scratch cmask used by scx_call_op_set_cpumask() to synthesize a
12		− * cmask from a cpumask. Allocated alongside the cid arrays on first enable
13		− * and never freed. Sized to the full cid space. Caller holds rq lock so
14		− * this_cpu_ptr is safe.
15		− */
16		−struct scx_cmask __percpu *scx_set_cmask_scratch;
17		−
18	10	/*
19	11	* cid tables.
20	12	*
@@ -54,8 +46,6 @@ static s32 scx_cid_arrays_alloc(void) · 33 unmodified lines
54	46	u32 npossible = num_possible_cpus();
55	47	s16 cid_to_cpu, cpu_to_cid;
56	48	struct scx_cid_topo *cid_topo;
57		− struct scx_cmask __percpu *set_cmask_scratch;
58		− s32 cpu;
59	49
60	50	if (scx_cid_to_cpu_tbl)
61	51	return 0;
@@ -63,25 +53,17 @@ static s32 scx_cid_arrays_alloc(void) · 1 unmodified line
63	53	cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL);
64	54	cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL);
65	55	cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL);
66		− set_cmask_scratch = __alloc_percpu(struct_size(set_cmask_scratch, bits,
67		− SCX_CMASK_NR_WORDS(npossible)),
68		− sizeof(u64));
69	56
70		− if (!cid_to_cpu \|\| !cpu_to_cid \|\| !cid_topo \|\| !set_cmask_scratch) {
	57	+ if (!cid_to_cpu \|\| !cpu_to_cid \|\| !cid_topo) {
71	58	kfree(cid_to_cpu);
72	59	kfree(cpu_to_cid);
73	60	kfree(cid_topo);
74		− free_percpu(set_cmask_scratch);
75	61	return -ENOMEM;
76	62	}
77	63
78	64	WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu);
79	65	WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid);
80	66	WRITE_ONCE(scx_cid_topo, cid_topo);
81		− for_each_possible_cpu(cpu)
82		− scx_cmask_init(per_cpu_ptr(set_cmask_scratch, cpu),
83		− 0, npossible);
84		− WRITE_ONCE(scx_set_cmask_scratch, set_cmask_scratch);
85	67	return 0;
86	68	}
87	69

kernel/sched/ext_internal.h+8−2

@@ -1124,6 +1124,14 @@ struct scx_sched { · 1123 unmodified lines
1124	1124	struct bpf_map *arena_map;
1125	1125	struct gen_pool *arena_pool;
1126	1126
	1127	+ /*
	1128	+ * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask
	1129	+ * to ops_cid.set_cmask(). The kernel writes through the stored kern_va;
	1130	+ * the BPF-arena uaddr handed to BPF is recovered by subtracting the
	1131	+ * arena's kern_vm_start.
	1132	+ */
	1133	+ struct scx_cmask * __percpu *set_cmask_scratch;
	1134	+
1127	1135	DECLARE_BITMAP(has_op, SCX_OPI_END);
1128	1136
1129	1137	/*
@@ -1480,8 +1488,6 @@ enum scx_ops_state { · 350 unmodified lines
1480	1488	extern struct scx_sched __rcu *scx_root;
1481	1489	DECLARE_PER_CPU(struct rq *, scx_locked_rq_state);
1482	1490
1483		−extern struct scx_cmask __percpu *scx_set_cmask_scratch;
1484		−
1485	1491	/*
1486	1492	* True when the currently loaded scheduler hierarchy is cid-form. All scheds
1487	1493	* in a hierarchy share one form, so this single key tells callsites which

sched_ext: Convert ops.set_cmask() to arena-resident cmask

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code