History

sched_ext: Convert ops.set_cmask() to arena-resident cmask

ops_cid.set_cmask() expects a cmask. The kernel couldn't write into the arena, so it translated cpumask -> cmask in kernel memory and passed the result as a trusted pointer. The BPF cmask helpers all operate on arena cmasks though, so the BPF side had to word-by-word probe-read the kernel cmask into an arena cmask via cmask_copy_from_kernel() before any helper could touch it. It works, but is clumsy. With direct kernel-side arena access now in place, build the cmask in the arena. The kernel writes to it through the kern_va side of the dual mapping. BPF directly dereferences it via an __arena pointer like any other arena struct.

Tejun Heo committed 5 months ago

Commit abdc2511 parent 9eca0875 files · +75 −80Browse files

Showing the changes to kernel (3 of 5 files). Show all

Unified · Split

3 files
  1. kernel/sched/ext.c +63 −5
  2. kernel/sched/ext_cid.c +1 −19
  3. kernel/sched/ext_internal.h +8 −2
kernel/sched/ext.c+63−5
@@ -621,11 +621,16 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, · 620 unmodified lines
621 update_locked_rq(rq);621 update_locked_rq(rq);
622 622
623 if (scx_is_cid_type()) {623 if (scx_is_cid_type()) {
624 struct scx_cmask *cmask = this_cpu_ptr(scx_set_cmask_scratch);624 struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch);
625 625 unsigned long uaddr = (unsigned long)kern_va -
626 lockdep_assert_irqs_disabled();626 bpf_arena_map_kern_vm_start(sch->arena_map);
627 scx_cpumask_to_cmask(cpumask, cmask);627 /*
628 sch->ops_cid.set_cmask(task, cmask);628 * Build the per-CPU arena cmask and hand BPF the uaddr. Caller
629 * holds the rq lock with IRQs disabled, which makes us the sole
630 * user of the scratch area.
631 */
632 scx_cpumask_to_cmask(cpumask, kern_va);
633 sch->ops_cid.set_cmask(task, (struct scx_cmask *)uaddr);
629 } else {634 } else {
630 sch->ops.set_cpumask(task, cpumask);635 sch->ops.set_cpumask(task, cpumask);
631 }636 }
@@ -4949,6 +4954,48 @@ static const struct attribute_group scx_global_attr_group = { · 4317 unmodified lines
4949static void free_pnode(struct scx_sched_pnode *pnode);4954static void free_pnode(struct scx_sched_pnode *pnode);
4950static void free_exit_info(struct scx_exit_info *ei);4955static void free_exit_info(struct scx_exit_info *ei);
4951 4956
4957static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch)
4958{
4959 size_t size = struct_size_t(struct scx_cmask, bits,
4960 SCX_CMASK_NR_WORDS(num_possible_cpus()));
4961 int cpu;
4962
4963 if (!sch->is_cid_type || !sch->arena_pool)
4964 return 0;
4965
4966 sch->set_cmask_scratch = alloc_percpu(struct scx_cmask *);
4967 if (!sch->set_cmask_scratch)
4968 return -ENOMEM;
4969
4970 for_each_possible_cpu(cpu) {
4971 struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu);
4972
4973 *slot = scx_arena_alloc(sch, size);
4974 if (!*slot)
4975 return -ENOMEM;
4976 scx_cmask_init(*slot, 0, num_possible_cpus());
4977 }
4978 return 0;
4979}
4980
4981static void scx_set_cmask_scratch_free(struct scx_sched *sch)
4982{
4983 size_t size = struct_size_t(struct scx_cmask, bits,
4984 SCX_CMASK_NR_WORDS(num_possible_cpus()));
4985 int cpu;
4986
4987 if (!sch->set_cmask_scratch)
4988 return;
4989
4990 for_each_possible_cpu(cpu) {
4991 struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu);
4992
4993 scx_arena_free(sch, *slot, size);
4994 }
4995 free_percpu(sch->set_cmask_scratch);
4996 sch->set_cmask_scratch = NULL;
4997}
4998
4952static void scx_sched_free_rcu_work(struct work_struct *work)4999static void scx_sched_free_rcu_work(struct work_struct *work)
4953{5000{
4954 struct rcu_work *rcu_work = to_rcu_work(work);5001 struct rcu_work *rcu_work = to_rcu_work(work);
@@ -5003,6 +5050,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) · 48 unmodified lines
5003 5050
5004 rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);5051 rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL);
5005 free_exit_info(sch->exit_info);5052 free_exit_info(sch->exit_info);
5053 scx_set_cmask_scratch_free(sch);
5006 scx_arena_pool_destroy(sch);5054 scx_arena_pool_destroy(sch);
5007 if (sch->arena_map)5055 if (sch->arena_map)
5008 bpf_map_put(sch->arena_map);5056 bpf_map_put(sch->arena_map);
@@ -7162,6 +7210,12 @@ static void scx_root_enable_workfn(struct kthread_work *work) · 2153 unmodified lines
7162 goto err_disable;7210 goto err_disable;
7163 }7211 }
7164 7212
7213 ret = scx_set_cmask_scratch_alloc(sch);
7214 if (ret) {
7215 cpus_read_unlock();
7216 goto err_disable;
7217 }
7218
7165 for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++)7219 for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++)
7166 if (((void (**)(void))ops)[i])7220 if (((void (**)(void))ops)[i])
7167 set_bit(i, sch->has_op);7221 set_bit(i, sch->has_op);
@@ -7484,6 +7538,10 @@ static void scx_sub_enable_workfn(struct kthread_work *work) · 316 unmodified lines
7484 if (ret)7538 if (ret)
7485 goto err_disable;7539 goto err_disable;
7486 7540
7541 ret = scx_set_cmask_scratch_alloc(sch);
7542 if (ret)
7543 goto err_disable;
7544
7487 if (validate_ops(sch, ops))7545 if (validate_ops(sch, ops))
7488 goto err_disable;7546 goto err_disable;
7489 7547
kernel/sched/ext_cid.c+1−19
@@ -7,14 +7,6 @@ · 6 unmodified lines
7 */7 */
8#include <linux/cacheinfo.h>8#include <linux/cacheinfo.h>
9 9
10/*
11 * Per-cpu scratch cmask used by scx_call_op_set_cpumask() to synthesize a
12 * cmask from a cpumask. Allocated alongside the cid arrays on first enable
13 * and never freed. Sized to the full cid space. Caller holds rq lock so
14 * this_cpu_ptr is safe.
15 */
16struct scx_cmask __percpu *scx_set_cmask_scratch;
17
18/*10/*
19 * cid tables.11 * cid tables.
20 *12 *
@@ -54,8 +46,6 @@ static s32 scx_cid_arrays_alloc(void) · 33 unmodified lines
54 u32 npossible = num_possible_cpus();46 u32 npossible = num_possible_cpus();
55 s16 *cid_to_cpu, *cpu_to_cid;47 s16 *cid_to_cpu, *cpu_to_cid;
56 struct scx_cid_topo *cid_topo;48 struct scx_cid_topo *cid_topo;
57 struct scx_cmask __percpu *set_cmask_scratch;
58 s32 cpu;
59 49
60 if (scx_cid_to_cpu_tbl)50 if (scx_cid_to_cpu_tbl)
61 return 0;51 return 0;
@@ -63,25 +53,17 @@ static s32 scx_cid_arrays_alloc(void) · 1 unmodified line
63 cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL);53 cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL);
64 cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL);54 cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL);
65 cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL);55 cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL);
66 set_cmask_scratch = __alloc_percpu(struct_size(set_cmask_scratch, bits,
67 SCX_CMASK_NR_WORDS(npossible)),
68 sizeof(u64));
69 56
70 if (!cid_to_cpu || !cpu_to_cid || !cid_topo || !set_cmask_scratch) {57 if (!cid_to_cpu || !cpu_to_cid || !cid_topo) {
71 kfree(cid_to_cpu);58 kfree(cid_to_cpu);
72 kfree(cpu_to_cid);59 kfree(cpu_to_cid);
73 kfree(cid_topo);60 kfree(cid_topo);
74 free_percpu(set_cmask_scratch);
75 return -ENOMEM;61 return -ENOMEM;
76 }62 }
77 63
78 WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu);64 WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu);
79 WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid);65 WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid);
80 WRITE_ONCE(scx_cid_topo, cid_topo);66 WRITE_ONCE(scx_cid_topo, cid_topo);
81 for_each_possible_cpu(cpu)
82 scx_cmask_init(per_cpu_ptr(set_cmask_scratch, cpu),
83 0, npossible);
84 WRITE_ONCE(scx_set_cmask_scratch, set_cmask_scratch);
85 return 0;67 return 0;
86}68}
87 69
kernel/sched/ext_internal.h+8−2
@@ -1124,6 +1124,14 @@ struct scx_sched { · 1123 unmodified lines
1124 struct bpf_map *arena_map;1124 struct bpf_map *arena_map;
1125 struct gen_pool *arena_pool;1125 struct gen_pool *arena_pool;
1126 1126
1127 /*
1128 * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask
1129 * to ops_cid.set_cmask(). The kernel writes through the stored kern_va;
1130 * the BPF-arena uaddr handed to BPF is recovered by subtracting the
1131 * arena's kern_vm_start.
1132 */
1133 struct scx_cmask * __percpu *set_cmask_scratch;
1134
1127 DECLARE_BITMAP(has_op, SCX_OPI_END);1135 DECLARE_BITMAP(has_op, SCX_OPI_END);
1128 1136
1129 /*1137 /*
@@ -1480,8 +1488,6 @@ enum scx_ops_state { · 350 unmodified lines
1480extern struct scx_sched __rcu *scx_root;1488extern struct scx_sched __rcu *scx_root;
1481DECLARE_PER_CPU(struct rq *, scx_locked_rq_state);1489DECLARE_PER_CPU(struct rq *, scx_locked_rq_state);
1482 1490
1483extern struct scx_cmask __percpu *scx_set_cmask_scratch;
1484
1485/*1491/*
1486 * True when the currently loaded scheduler hierarchy is cid-form. All scheds1492 * True when the currently loaded scheduler hierarchy is cid-form. All scheds
1487 * in a hierarchy share one form, so this single key tells callsites which1493 * in a hierarchy share one form, so this single key tells callsites which

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t