sched_ext: Build the set_cmask scratch from trusted geometry

scx_call_op_set_cpumask() builds a per-cpu cmask in the set_cmask scratch, which lives in BPF-writable arena. A scheduler can corrupt the scratch's inline header (base, nr_cids, alloc_words) from another cpu, so sizing and indexing the write from it risks an out-of-bounds write. Drive the build from kernel-known geometry instead. scx_cmask_ref_init_kern() imposes base and nr_cids rather than reading them, and scx_cmask_ref_from_cpumask() fills the scratch from the ref. Neither reads the header back.

Tejun Heo committed 4 months ago

Commit 33ffb561 parent cc7c2543 files · +81 −4Browse files

Unified · Split

3 files

kernel/sched/ext/cid.c +70 −0
kernel/sched/ext/cid.h +4 −0
kernel/sched/ext/ext.c +7 −4

kernel/sched/ext/cid.c+70−0

@@ -951,6 +951,38 @@ int scx_cmask_ref_init(struct scx_sched sch, const struct scx_cmask src, · 950 unmodified lines
951	return 0;	951	return 0;
952	}	952	}
953		953
		954	/**
		955	* scx_cmask_ref_init_kern - Bind a scx_cmask_ref to a kernel-owned cmask
		956	* @sch: scheduler the cmask belongs to
		957	* @m: kernel address of the target cmask, storage sized for @nr_cids at @base
		958	* @base: first cid of the active range
		959	* @nr_cids: active range length
		960	* @ref: output ref
		961	*
		962	* Like scx_cmask_ref_init() but the geometry is supplied by the caller, not
		963	* read from @m's header, so a concurrent BPF write to the header can't steer
		964	* later sizing or offsets. Rewrite the header from the trusted geometry and
		965	* bind @ref to it.
		966	*/
		967	void scx_cmask_ref_init_kern(struct scx_sched sch, struct scx_cmask m,
		968	u32 base, u32 nr_cids, struct scx_cmask_ref *ref)
		969	{
		970	WRITE_ONCE(m->base, base);
		971	WRITE_ONCE(m->nr_cids, nr_cids);
		972	WRITE_ONCE(m->alloc_words, SCX_CMASK_NR_WORDS(nr_cids));
		973
		974	ref->sch = sch;
		975	ref->src = m;
		976	ref->base = base;
		977	ref->nr_cids = nr_cids;
		978
		979	ref->shard_first = scx_cid_to_shard[base];
		980	if (likely(nr_cids))
		981	ref->shard_end = scx_cid_to_shard[base + nr_cids - 1] + 1;
		982	else
		983	ref->shard_end = ref->shard_first;
		984	}
		985
954	/**	986	/**
955	* scx_cmask_ref_shard - Read one shard from @ref into @out	987	* scx_cmask_ref_shard - Read one shard from @ref into @out
956	* @ref: validated ref	988	* @ref: validated ref
@@ -1032,6 +1064,44 @@ void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask · 75 unmodified lines
1032	src->bits, src->base, src->nr_cids, CMASK_OP2_REF_COPY);	1064	src->bits, src->base, src->nr_cids, CMASK_OP2_REF_COPY);
1033	}	1065	}
1034		1066
		1067	/**
		1068	* scx_cmask_ref_from_cpumask - Populate @ref's arena cmask from a cpumask
		1069	* @ref: kern-bound ref, see scx_cmask_ref_init_kern()
		1070	* @cpumask: cpus to translate into cids
		1071	*
		1072	* Write @ref's active range one word at a time, setting each cid's bit when
		1073	* its cpu is in @cpumask. Offsets and length come from @ref's trusted geometry
		1074	* and stores use WRITE_ONCE since BPF may read concurrently, so the arena
		1075	* header is never read.
		1076	*/
		1077	void scx_cmask_ref_from_cpumask(const struct scx_cmask_ref *ref,
		1078	const struct cpumask *cpumask)
		1079	{
		1080	struct scx_cmask *m = ref->src;
		1081	u32 base = ref->base, nr_cids = ref->nr_cids;
		1082	u32 wi, nr_words;
		1083
		1084	if (!nr_cids)
		1085	return;
		1086
		1087	nr_words = (base + nr_cids - 1) / 64 - base / 64 + 1;
		1088	for (wi = 0; wi < nr_words; wi++) {
		1089	u32 word_first_cid = (base / 64 + wi) * 64;
		1090	u64 word = 0;
		1091	u32 bit;
		1092
		1093	for (bit = 0; bit < 64; bit++) {
		1094	u32 cid = word_first_cid + bit;
		1095
		1096	if (cid < base \|\| cid >= base + nr_cids)
		1097	continue;
		1098	if (cpumask_test_cpu(__scx_cid_to_cpu(cid), cpumask))
		1099	word \|= BIT_U64(bit);
		1100	}
		1101	WRITE_ONCE(m->bits[wi], word);
		1102	}
		1103	}
		1104
1035	int scx_cid_kfunc_init(void)	1105	int scx_cid_kfunc_init(void)
1036	{	1106	{
1037	return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init_cids) ?:	1107	return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init_cids) ?:

kernel/sched/ext/cid.h+4−0

 int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src,
 		       struct scx_cmask_ref *ref);
+void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m,
+			     u32 base, u32 nr_cids, struct scx_cmask_ref *ref);
 void scx_cmask_ref_shard(const struct scx_cmask_ref *ref, s32 shard_idx,
 			 struct scx_cmask *out);
+void scx_cmask_ref_from_cpumask(const struct scx_cmask_ref *ref,
+				const struct cpumask *cpumask);
 void scx_cmask_ref_or(const struct scx_cmask_ref *ref, const struct scx_cmask *src);
 void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask *src);

kernel/sched/ext/ext.c+7−4

 {
 	if (scx_is_cid_type()) {
 		struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch);
+		struct scx_cmask_ref ref;
 		/*
-		 * Build the per-CPU arena cmask and hand BPF its arena address.
+		 * Build the per-cpu arena cmask from kernel geometry via @ref,
-		 * Caller holds the rq lock with IRQs disabled, which makes us
+		 * never reading its BPF-writable header, and hand BPF the arena
-		 * the sole user of the scratch area.
+		 * address. The rq lock makes this cpu the sole kernel writer.
 		 */
-		scx_cpumask_to_cmask(cpumask, kern_va);
+		scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref);
+		scx_cmask_ref_from_cpumask(&ref, cpumask);
 		SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task,
 				     scx_kaddr_to_arena(sch, kern_va));
 	} else {