History

sched_ext: Build the set_cmask scratch from trusted geometry

scx_call_op_set_cpumask() builds a per-cpu cmask in the set_cmask scratch, which lives in BPF-writable arena. A scheduler can corrupt the scratch's inline header (base, nr_cids, alloc_words) from another cpu, so sizing and indexing the write from it risks an out-of-bounds write. Drive the build from kernel-known geometry instead. scx_cmask_ref_init_kern() imposes base and nr_cids rather than reading them, and scx_cmask_ref_from_cpumask() fills the scratch from the ref. Neither reads the header back.

Tejun Heo committed 4 months ago

Commit 33ffb561 parent cc7c2543 files · +81 −4Browse files

Unified · Split

3 files
  1. kernel/sched/ext/cid.c +70 −0
  2. kernel/sched/ext/cid.h +4 −0
  3. kernel/sched/ext/ext.c +7 −4
kernel/sched/ext/cid.c+70−0
@@ -951,6 +951,38 @@ int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src, · 950 unmodified lines
951951 return 0;
952952 }
953953
954+/**
955+ * scx_cmask_ref_init_kern - Bind a scx_cmask_ref to a kernel-owned cmask
956+ * @sch: scheduler the cmask belongs to
957+ * @m: kernel address of the target cmask, storage sized for @nr_cids at @base
958+ * @base: first cid of the active range
959+ * @nr_cids: active range length
960+ * @ref: output ref
961+ *
962+ * Like scx_cmask_ref_init() but the geometry is supplied by the caller, not
963+ * read from @m's header, so a concurrent BPF write to the header can't steer
964+ * later sizing or offsets. Rewrite the header from the trusted geometry and
965+ * bind @ref to it.
966+ */
967+void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m,
968+ u32 base, u32 nr_cids, struct scx_cmask_ref *ref)
969+{
970+ WRITE_ONCE(m->base, base);
971+ WRITE_ONCE(m->nr_cids, nr_cids);
972+ WRITE_ONCE(m->alloc_words, SCX_CMASK_NR_WORDS(nr_cids));
973+
974+ ref->sch = sch;
975+ ref->src = m;
976+ ref->base = base;
977+ ref->nr_cids = nr_cids;
978+
979+ ref->shard_first = scx_cid_to_shard[base];
980+ if (likely(nr_cids))
981+ ref->shard_end = scx_cid_to_shard[base + nr_cids - 1] + 1;
982+ else
983+ ref->shard_end = ref->shard_first;
984+}
985+
954986 /**
955987 * scx_cmask_ref_shard - Read one shard from @ref into @out
956988 * @ref: validated ref
@@ -1032,6 +1064,44 @@ void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask · 75 unmodified lines
10321064 src->bits, src->base, src->nr_cids, CMASK_OP2_REF_COPY);
10331065 }
10341066
1067+/**
1068+ * scx_cmask_ref_from_cpumask - Populate @ref's arena cmask from a cpumask
1069+ * @ref: kern-bound ref, see scx_cmask_ref_init_kern()
1070+ * @cpumask: cpus to translate into cids
1071+ *
1072+ * Write @ref's active range one word at a time, setting each cid's bit when
1073+ * its cpu is in @cpumask. Offsets and length come from @ref's trusted geometry
1074+ * and stores use WRITE_ONCE since BPF may read concurrently, so the arena
1075+ * header is never read.
1076+ */
1077+void scx_cmask_ref_from_cpumask(const struct scx_cmask_ref *ref,
1078+ const struct cpumask *cpumask)
1079+{
1080+ struct scx_cmask *m = ref->src;
1081+ u32 base = ref->base, nr_cids = ref->nr_cids;
1082+ u32 wi, nr_words;
1083+
1084+ if (!nr_cids)
1085+ return;
1086+
1087+ nr_words = (base + nr_cids - 1) / 64 - base / 64 + 1;
1088+ for (wi = 0; wi < nr_words; wi++) {
1089+ u32 word_first_cid = (base / 64 + wi) * 64;
1090+ u64 word = 0;
1091+ u32 bit;
1092+
1093+ for (bit = 0; bit < 64; bit++) {
1094+ u32 cid = word_first_cid + bit;
1095+
1096+ if (cid < base || cid >= base + nr_cids)
1097+ continue;
1098+ if (cpumask_test_cpu(__scx_cid_to_cpu(cid), cpumask))
1099+ word |= BIT_U64(bit);
1100+ }
1101+ WRITE_ONCE(m->bits[wi], word);
1102+ }
1103+}
1104+
10351105 int scx_cid_kfunc_init(void)
10361106 {
10371107 return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &scx_kfunc_set_init_cids) ?:
kernel/sched/ext/cid.h+4−0
@@ -295,8 +295,12 @@ static inline s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid) · 294 unmodified lines
295295
296296 int scx_cmask_ref_init(struct scx_sched *sch, const struct scx_cmask *src,
297297 struct scx_cmask_ref *ref);
298+void scx_cmask_ref_init_kern(struct scx_sched *sch, struct scx_cmask *m,
299+ u32 base, u32 nr_cids, struct scx_cmask_ref *ref);
298300 void scx_cmask_ref_shard(const struct scx_cmask_ref *ref, s32 shard_idx,
299301 struct scx_cmask *out);
302+void scx_cmask_ref_from_cpumask(const struct scx_cmask_ref *ref,
303+ const struct cpumask *cpumask);
300304 void scx_cmask_ref_or(const struct scx_cmask_ref *ref, const struct scx_cmask *src);
301305 void scx_cmask_ref_copy(const struct scx_cmask_ref *ref, const struct scx_cmask *src);
302306
kernel/sched/ext/ext.c+7−4
@@ -424,12 +424,15 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, · 423 unmodified lines
424424 {
425425 if (scx_is_cid_type()) {
426426 struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch);
427+ struct scx_cmask_ref ref;
428+
427429 /*
428− * Build the per-CPU arena cmask and hand BPF its arena address.
429− * Caller holds the rq lock with IRQs disabled, which makes us
430− * the sole user of the scratch area.
430+ * Build the per-cpu arena cmask from kernel geometry via @ref,
431+ * never reading its BPF-writable header, and hand BPF the arena
432+ * address. The rq lock makes this cpu the sole kernel writer.
431433 */
432− scx_cpumask_to_cmask(cpumask, kern_va);
434+ scx_cmask_ref_init_kern(sch, kern_va, 0, num_possible_cpus(), &ref);
435+ scx_cmask_ref_from_cpumask(&ref, cpumask);
433436 SCX_CALL_CID_OP_TASK(sch, set_cmask, rq, task,
434437 scx_kaddr_to_arena(sch, kern_va));
435438 } else {