ops_cid.set_cmask() expects a cmask. The kernel couldn't write into the arena, so it translated cpumask -> cmask in kernel memory and passed the result as a trusted pointer. The BPF cmask helpers all operate on arena cmasks though, so the BPF side had to word-by-word probe-read the kernel cmask into an arena cmask via cmask_copy_from_kernel() before any helper could touch it. It works, but is clumsy. With direct kernel-side arena access now in place, build the cmask in the arena. The kernel writes to it through the kern_va side of the dual mapping. BPF directly dereferences it via an __arena pointer like any other arena struct.
Unified · Split
| @@ -621,11 +621,16 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, · 620 unmodified lines | |||
| 621 | update_locked_rq(rq); | 621 | update_locked_rq(rq); |
| 622 | 622 | ||
| 623 | if (scx_is_cid_type()) { | 623 | if (scx_is_cid_type()) { |
| 624 | struct scx_cmask *cmask = this_cpu_ptr(scx_set_cmask_scratch); | 624 | struct scx_cmask *kern_va = *this_cpu_ptr(sch->set_cmask_scratch); |
| 625 | 625 | unsigned long uaddr = (unsigned long)kern_va - | |
| 626 | lockdep_assert_irqs_disabled(); | 626 | bpf_arena_map_kern_vm_start(sch->arena_map); |
| 627 | scx_cpumask_to_cmask(cpumask, cmask); | 627 | /* |
| 628 | sch->ops_cid.set_cmask(task, cmask); | 628 | * Build the per-CPU arena cmask and hand BPF the uaddr. Caller |
| 629 | * holds the rq lock with IRQs disabled, which makes us the sole | ||
| 630 | * user of the scratch area. | ||
| 631 | */ | ||
| 632 | scx_cpumask_to_cmask(cpumask, kern_va); | ||
| 633 | sch->ops_cid.set_cmask(task, (struct scx_cmask *)uaddr); | ||
| 629 | } else { | 634 | } else { |
| 630 | sch->ops.set_cpumask(task, cpumask); | 635 | sch->ops.set_cpumask(task, cpumask); |
| 631 | } | 636 | } |
| @@ -4949,6 +4954,48 @@ static const struct attribute_group scx_global_attr_group = { · 4317 unmodified lines | |||
| 4949 | static void free_pnode(struct scx_sched_pnode *pnode); | 4954 | static void free_pnode(struct scx_sched_pnode *pnode); |
| 4950 | static void free_exit_info(struct scx_exit_info *ei); | 4955 | static void free_exit_info(struct scx_exit_info *ei); |
| 4951 | 4956 | ||
| 4957 | static s32 scx_set_cmask_scratch_alloc(struct scx_sched *sch) | ||
| 4958 | { | ||
| 4959 | size_t size = struct_size_t(struct scx_cmask, bits, | ||
| 4960 | SCX_CMASK_NR_WORDS(num_possible_cpus())); | ||
| 4961 | int cpu; | ||
| 4962 | |||
| 4963 | if (!sch->is_cid_type || !sch->arena_pool) | ||
| 4964 | return 0; | ||
| 4965 | |||
| 4966 | sch->set_cmask_scratch = alloc_percpu(struct scx_cmask *); | ||
| 4967 | if (!sch->set_cmask_scratch) | ||
| 4968 | return -ENOMEM; | ||
| 4969 | |||
| 4970 | for_each_possible_cpu(cpu) { | ||
| 4971 | struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); | ||
| 4972 | |||
| 4973 | *slot = scx_arena_alloc(sch, size); | ||
| 4974 | if (!*slot) | ||
| 4975 | return -ENOMEM; | ||
| 4976 | scx_cmask_init(*slot, 0, num_possible_cpus()); | ||
| 4977 | } | ||
| 4978 | return 0; | ||
| 4979 | } | ||
| 4980 | |||
| 4981 | static void scx_set_cmask_scratch_free(struct scx_sched *sch) | ||
| 4982 | { | ||
| 4983 | size_t size = struct_size_t(struct scx_cmask, bits, | ||
| 4984 | SCX_CMASK_NR_WORDS(num_possible_cpus())); | ||
| 4985 | int cpu; | ||
| 4986 | |||
| 4987 | if (!sch->set_cmask_scratch) | ||
| 4988 | return; | ||
| 4989 | |||
| 4990 | for_each_possible_cpu(cpu) { | ||
| 4991 | struct scx_cmask **slot = per_cpu_ptr(sch->set_cmask_scratch, cpu); | ||
| 4992 | |||
| 4993 | scx_arena_free(sch, *slot, size); | ||
| 4994 | } | ||
| 4995 | free_percpu(sch->set_cmask_scratch); | ||
| 4996 | sch->set_cmask_scratch = NULL; | ||
| 4997 | } | ||
| 4998 | |||
| 4952 | static void scx_sched_free_rcu_work(struct work_struct *work) | 4999 | static void scx_sched_free_rcu_work(struct work_struct *work) |
| 4953 | { | 5000 | { |
| 4954 | struct rcu_work *rcu_work = to_rcu_work(work); | 5001 | struct rcu_work *rcu_work = to_rcu_work(work); |
| @@ -5003,6 +5050,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work) · 48 unmodified lines | |||
| 5003 | 5050 | ||
| 5004 | rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); | 5051 | rhashtable_free_and_destroy(&sch->dsq_hash, NULL, NULL); |
| 5005 | free_exit_info(sch->exit_info); | 5052 | free_exit_info(sch->exit_info); |
| 5053 | scx_set_cmask_scratch_free(sch); | ||
| 5006 | scx_arena_pool_destroy(sch); | 5054 | scx_arena_pool_destroy(sch); |
| 5007 | if (sch->arena_map) | 5055 | if (sch->arena_map) |
| 5008 | bpf_map_put(sch->arena_map); | 5056 | bpf_map_put(sch->arena_map); |
| @@ -7162,6 +7210,12 @@ static void scx_root_enable_workfn(struct kthread_work *work) · 2153 unmodified lines | |||
| 7162 | goto err_disable; | 7210 | goto err_disable; |
| 7163 | } | 7211 | } |
| 7164 | 7212 | ||
| 7213 | ret = scx_set_cmask_scratch_alloc(sch); | ||
| 7214 | if (ret) { | ||
| 7215 | cpus_read_unlock(); | ||
| 7216 | goto err_disable; | ||
| 7217 | } | ||
| 7218 | |||
| 7165 | for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) | 7219 | for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) |
| 7166 | if (((void (**)(void))ops)[i]) | 7220 | if (((void (**)(void))ops)[i]) |
| 7167 | set_bit(i, sch->has_op); | 7221 | set_bit(i, sch->has_op); |
| @@ -7484,6 +7538,10 @@ static void scx_sub_enable_workfn(struct kthread_work *work) · 316 unmodified lines | |||
| 7484 | if (ret) | 7538 | if (ret) |
| 7485 | goto err_disable; | 7539 | goto err_disable; |
| 7486 | 7540 | ||
| 7541 | ret = scx_set_cmask_scratch_alloc(sch); | ||
| 7542 | if (ret) | ||
| 7543 | goto err_disable; | ||
| 7544 | |||
| 7487 | if (validate_ops(sch, ops)) | 7545 | if (validate_ops(sch, ops)) |
| 7488 | goto err_disable; | 7546 | goto err_disable; |
| 7489 | 7547 | ||
| @@ -7,14 +7,6 @@ · 6 unmodified lines | |||
| 7 | */ | 7 | */ |
| 8 | #include <linux/cacheinfo.h> | 8 | #include <linux/cacheinfo.h> |
| 9 | 9 | ||
| 10 | /* | ||
| 11 | * Per-cpu scratch cmask used by scx_call_op_set_cpumask() to synthesize a | ||
| 12 | * cmask from a cpumask. Allocated alongside the cid arrays on first enable | ||
| 13 | * and never freed. Sized to the full cid space. Caller holds rq lock so | ||
| 14 | * this_cpu_ptr is safe. | ||
| 15 | */ | ||
| 16 | struct scx_cmask __percpu *scx_set_cmask_scratch; | ||
| 17 | |||
| 18 | /* | 10 | /* |
| 19 | * cid tables. | 11 | * cid tables. |
| 20 | * | 12 | * |
| @@ -54,8 +46,6 @@ static s32 scx_cid_arrays_alloc(void) · 33 unmodified lines | |||
| 54 | u32 npossible = num_possible_cpus(); | 46 | u32 npossible = num_possible_cpus(); |
| 55 | s16 *cid_to_cpu, *cpu_to_cid; | 47 | s16 *cid_to_cpu, *cpu_to_cid; |
| 56 | struct scx_cid_topo *cid_topo; | 48 | struct scx_cid_topo *cid_topo; |
| 57 | struct scx_cmask __percpu *set_cmask_scratch; | ||
| 58 | s32 cpu; | ||
| 59 | 49 | ||
| 60 | if (scx_cid_to_cpu_tbl) | 50 | if (scx_cid_to_cpu_tbl) |
| 61 | return 0; | 51 | return 0; |
| @@ -63,25 +53,17 @@ static s32 scx_cid_arrays_alloc(void) · 1 unmodified line | |||
| 63 | cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL); | 53 | cid_to_cpu = kzalloc_objs(*scx_cid_to_cpu_tbl, npossible, GFP_KERNEL); |
| 64 | cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL); | 54 | cpu_to_cid = kzalloc_objs(*scx_cpu_to_cid_tbl, nr_cpu_ids, GFP_KERNEL); |
| 65 | cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL); | 55 | cid_topo = kmalloc_objs(*scx_cid_topo, npossible, GFP_KERNEL); |
| 66 | set_cmask_scratch = __alloc_percpu(struct_size(set_cmask_scratch, bits, | ||
| 67 | SCX_CMASK_NR_WORDS(npossible)), | ||
| 68 | sizeof(u64)); | ||
| 69 | 56 | ||
| 70 | if (!cid_to_cpu || !cpu_to_cid || !cid_topo || !set_cmask_scratch) { | 57 | if (!cid_to_cpu || !cpu_to_cid || !cid_topo) { |
| 71 | kfree(cid_to_cpu); | 58 | kfree(cid_to_cpu); |
| 72 | kfree(cpu_to_cid); | 59 | kfree(cpu_to_cid); |
| 73 | kfree(cid_topo); | 60 | kfree(cid_topo); |
| 74 | free_percpu(set_cmask_scratch); | ||
| 75 | return -ENOMEM; | 61 | return -ENOMEM; |
| 76 | } | 62 | } |
| 77 | 63 | ||
| 78 | WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu); | 64 | WRITE_ONCE(scx_cid_to_cpu_tbl, cid_to_cpu); |
| 79 | WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid); | 65 | WRITE_ONCE(scx_cpu_to_cid_tbl, cpu_to_cid); |
| 80 | WRITE_ONCE(scx_cid_topo, cid_topo); | 66 | WRITE_ONCE(scx_cid_topo, cid_topo); |
| 81 | for_each_possible_cpu(cpu) | ||
| 82 | scx_cmask_init(per_cpu_ptr(set_cmask_scratch, cpu), | ||
| 83 | 0, npossible); | ||
| 84 | WRITE_ONCE(scx_set_cmask_scratch, set_cmask_scratch); | ||
| 85 | return 0; | 67 | return 0; |
| 86 | } | 68 | } |
| 87 | 69 | ||
| @@ -1124,6 +1124,14 @@ struct scx_sched { · 1123 unmodified lines | |||
| 1124 | struct bpf_map *arena_map; | 1124 | struct bpf_map *arena_map; |
| 1125 | struct gen_pool *arena_pool; | 1125 | struct gen_pool *arena_pool; |
| 1126 | 1126 | ||
| 1127 | /* | ||
| 1128 | * Per-CPU arena cmask used by scx_call_op_set_cpumask() to hand a cmask | ||
| 1129 | * to ops_cid.set_cmask(). The kernel writes through the stored kern_va; | ||
| 1130 | * the BPF-arena uaddr handed to BPF is recovered by subtracting the | ||
| 1131 | * arena's kern_vm_start. | ||
| 1132 | */ | ||
| 1133 | struct scx_cmask * __percpu *set_cmask_scratch; | ||
| 1134 | |||
| 1127 | DECLARE_BITMAP(has_op, SCX_OPI_END); | 1135 | DECLARE_BITMAP(has_op, SCX_OPI_END); |
| 1128 | 1136 | ||
| 1129 | /* | 1137 | /* |
| @@ -1480,8 +1488,6 @@ enum scx_ops_state { · 350 unmodified lines | |||
| 1480 | extern struct scx_sched __rcu *scx_root; | 1488 | extern struct scx_sched __rcu *scx_root; |
| 1481 | DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); | 1489 | DECLARE_PER_CPU(struct rq *, scx_locked_rq_state); |
| 1482 | 1490 | ||
| 1483 | extern struct scx_cmask __percpu *scx_set_cmask_scratch; | ||
| 1484 | |||
| 1485 | /* | 1491 | /* |
| 1486 | * True when the currently loaded scheduler hierarchy is cid-form. All scheds | 1492 | * True when the currently loaded scheduler hierarchy is cid-form. All scheds |
| 1487 | * in a hierarchy share one form, so this single key tells callsites which | 1493 | * in a hierarchy share one form, so this single key tells callsites which |
| @@ -675,56 +675,4 @@ static __always_inline void cmask_from_cpumask(struct scx_cmask __arena *m, · 674 unmodified lines | |||
| 675 | } | 675 | } |
| 676 | } | 676 | } |
| 677 | 677 | ||
| 678 | /** | ||
| 679 | * cmask_copy_from_kernel - probe-read a kernel cmask into an arena cmask | ||
| 680 | * @dst: arena cmask to fill; must have @dst->base == 0 and be sized for @src. | ||
| 681 | * @src: kernel-memory cmask (e.g. ops.set_cmask() arg); @src->base must be 0. | ||
| 682 | * | ||
| 683 | * Word-for-word copy; @src and @dst must share base 0 alignment. Triggers | ||
| 684 | * scx_bpf_error() on probe failure or precondition violation. | ||
| 685 | */ | ||
| 686 | static __always_inline void cmask_copy_from_kernel(struct scx_cmask __arena *dst, | ||
| 687 | const struct scx_cmask *src) | ||
| 688 | { | ||
| 689 | u32 base = 0, nr_cids = 0, nr_words, wi; | ||
| 690 | |||
| 691 | if (dst->base != 0) { | ||
| 692 | scx_bpf_error("cmask_copy_from_kernel requires dst->base == 0"); | ||
| 693 | return; | ||
| 694 | } | ||
| 695 | |||
| 696 | if (bpf_probe_read_kernel(&base, sizeof(base), &src->base)) { | ||
| 697 | scx_bpf_error("probe-read cmask->base failed"); | ||
| 698 | return; | ||
| 699 | } | ||
| 700 | if (base != 0) { | ||
| 701 | scx_bpf_error("cmask_copy_from_kernel requires src->base == 0"); | ||
| 702 | return; | ||
| 703 | } | ||
| 704 | |||
| 705 | if (bpf_probe_read_kernel(&nr_cids, sizeof(nr_cids), &src->nr_cids)) { | ||
| 706 | scx_bpf_error("probe-read cmask->nr_cids failed"); | ||
| 707 | return; | ||
| 708 | } | ||
| 709 | |||
| 710 | if (nr_cids > dst->nr_cids) { | ||
| 711 | scx_bpf_error("src cmask nr_cids=%u exceeds dst nr_cids=%u", | ||
| 712 | nr_cids, dst->nr_cids); | ||
| 713 | return; | ||
| 714 | } | ||
| 715 | |||
| 716 | nr_words = CMASK_NR_WORDS(nr_cids); | ||
| 717 | cmask_zero(dst); | ||
| 718 | bpf_for(wi, 0, CMASK_MAX_WORDS) { | ||
| 719 | u64 word = 0; | ||
| 720 | if (wi >= nr_words) | ||
| 721 | break; | ||
| 722 | if (bpf_probe_read_kernel(&word, sizeof(u64), &src->bits[wi])) { | ||
| 723 | scx_bpf_error("probe-read cmask->bits[%u] failed", wi); | ||
| 724 | return; | ||
| 725 | } | ||
| 726 | dst->bits[wi] = word; | ||
| 727 | } | ||
| 728 | } | ||
| 729 | |||
| 730 | #endif /* __SCX_CID_BPF_H */ | 678 | #endif /* __SCX_CID_BPF_H */ |
| @@ -919,14 +919,15 @@ void BPF_STRUCT_OPS(qmap_update_idle, s32 cid, bool idle) · 918 unmodified lines | |||
| 919 | } | 919 | } |
| 920 | 920 | ||
| 921 | void BPF_STRUCT_OPS(qmap_set_cmask, struct task_struct *p, | 921 | void BPF_STRUCT_OPS(qmap_set_cmask, struct task_struct *p, |
| 922 | const struct scx_cmask *cmask) | 922 | const struct scx_cmask *cmask_in) |
| 923 | { | 923 | { |
| 924 | struct scx_cmask __arena *cmask = (struct scx_cmask __arena *)(long)cmask_in; | ||
| 924 | task_ctx_t *taskc; | 925 | task_ctx_t *taskc; |
| 925 | 926 | ||
| 926 | taskc = lookup_task_ctx(p); | 927 | taskc = lookup_task_ctx(p); |
| 927 | if (!taskc) | 928 | if (!taskc) |
| 928 | return; | 929 | return; |
| 929 | cmask_copy_from_kernel(&taskc->cpus_allowed, cmask); | 930 | cmask_copy(&taskc->cpus_allowed, cmask); |
| 930 | } | 931 | } |
| 931 | 932 | ||
| 932 | struct monitor_timer { | 933 | struct monitor_timer { |