/* SPDX-License-Identifier: GPL-2.0 *//* * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst * * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. * Copyright (c) 2022 Tejun Heo <tj@kernel.org> * Copyright (c) 2022 David Vernet <dvernet@meta.com> */#include <linux/btf_ids.h>#include "ext_idle.h" /* * NOTE: sched_ext is in the process of growing multiple scheduler support and * scx_root usage is in a transitional state. Naked dereferences are safe if the * caller is one of the tasks attached to SCX and explicit RCU dereference is * necessary otherwise. Naked scx_root dereferences trigger sparse warnings but * are used as temporary markers to indicate that the dereferences need to be * updated to point to the associated scheduler instances rather than scx_root. */static struct scx_sched __rcu *scx_root; /* * During exit, a task may schedule after losing its PIDs. When disabling the * BPF scheduler, we need to be able to iterate tasks in every state to * guarantee system safety. Maintain a dedicated task list which contains every * task between its fork and eventual free. */static DEFINE_RAW_SPINLOCK(scx_tasks_lock);static LIST_HEAD(scx_tasks); /* ops enable/disable */static DEFINE_MUTEX(scx_enable_mutex);DEFINE_STATIC_KEY_FALSE(__scx_enabled);DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem);static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED);static int scx_bypass_depth;static cpumask_var_t scx_bypass_lb_donee_cpumask;static cpumask_var_t scx_bypass_lb_resched_cpumask;static bool scx_aborting;static bool scx_init_task_enabled;static bool scx_switching_all;DEFINE_STATIC_KEY_FALSE(__scx_switched_all); /* * Tracks whether scx_enable() called scx_bypass(true). Used to balance bypass * depth on enable failure. Will be removed when bypass depth is moved into the * sched instance. */static bool scx_bypassed_for_enable; static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0);static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); /* * A monotically increasing sequence number that is incremented every time a * scheduler is enabled. This can be used by to check if any custom sched_ext * scheduler has ever been used in the system. */static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0); /* * The maximum amount of time in jiffies that a task may be runnable without * being scheduled on a CPU. If this timeout is exceeded, it will trigger * scx_error(). */static unsigned long scx_watchdog_timeout; /* * The last time the delayed work was run. This delayed work relies on * ksoftirqd being able to run to service timer interrupts, so it's possible * that this work itself could get wedged. To account for this, we check that * it's not stalled in the timer tick, and trigger an error if it is. */static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; static struct delayed_work scx_watchdog_work; /* * For %SCX_KICK_WAIT: Each CPU has a pointer to an array of kick_sync sequence * numbers. The arrays are allocated with kvzalloc() as size can exceed percpu * allocator limits on large machines. O(nr_cpu_ids^2) allocation, allocated * lazily when enabling and freed when disabling to avoid waste when sched_ext * isn't active. */struct scx_kick_syncs { struct rcu_head rcu; unsigned long syncs[];}; static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs); /* * Direct dispatch marker. * * Non-NULL values are used for direct dispatch from enqueue path. A valid * pointer points to the task currently being enqueued. An ERR_PTR value is used * to indicate that direct dispatch has already happened. */static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); static const struct rhashtable_params dsq_hash_params = { .key_len = sizeof_field(struct scx_dispatch_q, id), .key_offset = offsetof(struct scx_dispatch_q, id), .head_offset = offsetof(struct scx_dispatch_q, hash_node),}; static LLIST_HEAD(dsqs_to_free); /* dispatch buf */struct scx_dsp_buf_ent { struct task_struct *task; unsigned long qseq; u64 dsq_id; u64 enq_flags;}; static u32 scx_dsp_max_batch; struct scx_dsp_ctx { struct rq *rq; u32 cursor; u32 nr_tasks; struct scx_dsp_buf_ent buf[];}; static struct scx_dsp_ctx __percpu *scx_dsp_ctx; /* string formatting from BPF */struct scx_bstr_buf { u64 data[MAX_BPRINTF_VARARGS]; char line[SCX_EXIT_MSG_LEN];}; static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock);static struct scx_bstr_buf scx_exit_bstr_buf; /* ops debug dump */struct scx_dump_data { s32 cpu; bool first; s32 cursor; struct seq_buf *s; const char *prefix; struct scx_bstr_buf buf;}; static struct scx_dump_data scx_dump_data = { .cpu = -1,}; /* /sys/kernel/sched_ext interface */static struct kset *scx_kset; /* * Parameters that can be adjusted through /sys/module/sched_ext/parameters. * There usually is no reason to modify these as normal scheduler operation * shouldn't be affected by them. The knobs are primarily for debugging. */static u64 scx_slice_dfl = SCX_SLICE_DFL;static unsigned int scx_slice_bypass_us = SCX_SLICE_BYPASS / NSEC_PER_USEC;static unsigned int scx_bypass_lb_intv_us = SCX_BYPASS_LB_DFL_INTV_US; static int set_slice_us(const char *val, const struct kernel_param *kp){ return param_set_uint_minmax(val, kp, 100, 100 * USEC_PER_MSEC);} static const struct kernel_param_ops slice_us_param_ops = { .set = set_slice_us, .get = param_get_uint,}; static int set_bypass_lb_intv_us(const char *val, const struct kernel_param *kp){ return param_set_uint_minmax(val, kp, 0, 10 * USEC_PER_SEC);} static const struct kernel_param_ops bypass_lb_intv_us_param_ops = { .set = set_bypass_lb_intv_us, .get = param_get_uint,}; #undef MODULE_PARAM_PREFIX#define MODULE_PARAM_PREFIX "sched_ext." module_param_cb(slice_bypass_us, &slice_us_param_ops, &scx_slice_bypass_us, 0600);MODULE_PARM_DESC(slice_bypass_us, "bypass slice in microseconds, applied on [un]load (100us to 100ms)");module_param_cb(bypass_lb_intv_us, &bypass_lb_intv_us_param_ops, &scx_bypass_lb_intv_us, 0600);MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microseconds (0 (disable) to 10s)"); #undef MODULE_PARAM_PREFIX #define CREATE_TRACE_POINTS#include <trace/events/sched_ext.h> static void process_ddsp_deferred_locals(struct rq *rq);static bool task_dead_and_done(struct task_struct *p);static u32 reenq_local(struct rq *rq);static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags);static bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, s64 exit_code, const char *fmt, va_list args); static __printf(4, 5) bool scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, s64 exit_code, const char *fmt, ...){ va_list args; bool ret; va_start(args, fmt); ret = scx_vexit(sch, kind, exit_code, fmt, args); va_end(args); return ret;} #define scx_error(sch, fmt, args...) scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args)#define scx_verror(sch, fmt, args) scx_vexit((sch), SCX_EXIT_ERROR, 0, fmt, args) #define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) static long jiffies_delta_msecs(unsigned long at, unsigned long now){ if (time_after(at, now)) return jiffies_to_msecs(at - now); else return -(long)jiffies_to_msecs(now - at);} /* if the highest set bit is N, return a mask with bits [N+1, 31] set */static u32 higher_bits(u32 flags){ return ~((1 << fls(flags)) - 1);} /* return the mask with only the highest bit set */static u32 highest_bit(u32 flags){ int bit = fls(flags); return ((u64)1 << bit) >> 1;} static bool u32_before(u32 a, u32 b){ return (s32)(a - b) < 0;} static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, struct task_struct *p){ return sch->global_dsqs[cpu_to_node(task_cpu(p))];} static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id){ return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params);} static const struct sched_class *scx_setscheduler_class(struct task_struct *p){ if (p->sched_class == &stop_sched_class) return &stop_sched_class; return __setscheduler_class(p->policy, p->prio);} /* * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check * whether it's running from an allowed context. * * @mask is constant, always inline to cull the mask calculations. */static __always_inline void scx_kf_allow(u32 mask){ /* nesting is allowed only in increasing scx_kf_mask order */ WARN_ONCE((mask | higher_bits(mask)) & current->scx.kf_mask, "invalid nesting current->scx.kf_mask=0x%x mask=0x%x\n", current->scx.kf_mask, mask); current->scx.kf_mask |= mask; barrier();} static void scx_kf_disallow(u32 mask){ barrier(); current->scx.kf_mask &= ~mask;} /* * Track the rq currently locked. * * This allows kfuncs to safely operate on rq from any scx ops callback, * knowing which rq is already locked. */DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); static inline void update_locked_rq(struct rq *rq){ /* * Check whether @rq is actually locked. This can help expose bugs * or incorrect assumptions about the context in which a kfunc or * callback is executed. */ if (rq) lockdep_assert_rq_held(rq); __this_cpu_write(scx_locked_rq_state, rq);} #define SCX_CALL_OP(sch, mask, op, rq, args...) \do { \ if (rq) \ update_locked_rq(rq); \ if (mask) { \ scx_kf_allow(mask); \ (sch)->ops.op(args); \ scx_kf_disallow(mask); \ } else { \ (sch)->ops.op(args); \ } \ if (rq) \ update_locked_rq(NULL); \} while (0) #define SCX_CALL_OP_RET(sch, mask, op, rq, args...) \({ \ __typeof__((sch)->ops.op(args)) __ret; \ \ if (rq) \ update_locked_rq(rq); \ if (mask) { \ scx_kf_allow(mask); \ __ret = (sch)->ops.op(args); \ scx_kf_disallow(mask); \ } else { \ __ret = (sch)->ops.op(args); \ } \ if (rq) \ update_locked_rq(NULL); \ __ret; \}) /* * Some kfuncs are allowed only on the tasks that are subjects of the * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such * restrictions, the following SCX_CALL_OP_*() variants should be used when * invoking scx_ops operations that take task arguments. These can only be used * for non-nesting operations due to the way the tasks are tracked. * * kfuncs which can only operate on such tasks can in turn use * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on * the specific task. */#define SCX_CALL_OP_TASK(sch, mask, op, rq, task, args...) \do { \ BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ current->scx.kf_tasks[0] = task; \ SCX_CALL_OP((sch), mask, op, rq, task, ##args); \ current->scx.kf_tasks[0] = NULL; \} while (0) #define SCX_CALL_OP_TASK_RET(sch, mask, op, rq, task, args...) \({ \ __typeof__((sch)->ops.op(task, ##args)) __ret; \ BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ current->scx.kf_tasks[0] = task; \ __ret = SCX_CALL_OP_RET((sch), mask, op, rq, task, ##args); \ current->scx.kf_tasks[0] = NULL; \ __ret; \}) #define SCX_CALL_OP_2TASKS_RET(sch, mask, op, rq, task0, task1, args...) \({ \ __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ current->scx.kf_tasks[0] = task0; \ current->scx.kf_tasks[1] = task1; \ __ret = SCX_CALL_OP_RET((sch), mask, op, rq, task0, task1, ##args); \ current->scx.kf_tasks[0] = NULL; \ current->scx.kf_tasks[1] = NULL; \ __ret; \}) /* @mask is constant, always inline to cull unnecessary branches */static __always_inline bool scx_kf_allowed(struct scx_sched *sch, u32 mask){ if (unlikely(!(current->scx.kf_mask & mask))) { scx_error(sch, "kfunc with mask 0x%x called from an operation only allowing 0x%x", mask, current->scx.kf_mask); return false; } /* * Enforce nesting boundaries. e.g. A kfunc which can be called from * DISPATCH must not be called if we're running DEQUEUE which is nested * inside ops.dispatch(). We don't need to check boundaries for any * blocking kfuncs as the verifier ensures they're only called from * sleepable progs. */ if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && (current->scx.kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { scx_error(sch, "cpu_release kfunc called from a nested operation"); return false; } if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && (current->scx.kf_mask & higher_bits(SCX_KF_DISPATCH)))) { scx_error(sch, "dispatch kfunc called from a nested operation"); return false; } return true;} /* see SCX_CALL_OP_TASK() */static __always_inline bool scx_kf_allowed_on_arg_tasks(struct scx_sched *sch, u32 mask, struct task_struct *p){ if (!scx_kf_allowed(sch, mask)) return false; if (unlikely((p != current->scx.kf_tasks[0] && p != current->scx.kf_tasks[1]))) { scx_error(sch, "called on a task not being operated on"); return false; } return true;} /** * nldsq_next_task - Iterate to the next task in a non-local DSQ * @dsq: user dsq being iterated * @cur: current position, %NULL to start iteration * @rev: walk backwards * * Returns %NULL when iteration is finished. */static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, struct task_struct *cur, bool rev){ struct list_head *list_node; struct scx_dsq_list_node *dsq_lnode; lockdep_assert_held(&dsq->lock); if (cur) list_node = &cur->scx.dsq_list.node; else list_node = &dsq->list; /* find the next task, need to skip BPF iteration cursors */ do { if (rev) list_node = list_node->prev; else list_node = list_node->next; if (list_node == &dsq->list) return NULL; dsq_lnode = container_of(list_node, struct scx_dsq_list_node, node); } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR); return container_of(dsq_lnode, struct task_struct, scx.dsq_list);} #define nldsq_for_each_task(p, dsq) \ for ((p) = nldsq_next_task((dsq), NULL, false); (p); \ (p) = nldsq_next_task((dsq), (p), false)) /* * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse] * dispatch order. BPF-visible iterator is opaque and larger to allow future * changes without breaking backward compatibility. Can be used with * bpf_for_each(). See bpf_iter_scx_dsq_*(). */enum scx_dsq_iter_flags { /* iterate in the reverse dispatch order */ SCX_DSQ_ITER_REV = 1U << 16, __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | __SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME,}; struct bpf_iter_scx_dsq_kern { struct scx_dsq_list_node cursor; struct scx_dispatch_q *dsq; u64 slice; u64 vtime;} __attribute__((aligned(8)));Lines 1–500 of 7542. Load more
/* SPDX-License-Identifier: GPL-2.0 */ /* * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst * * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. * Copyright (c) 2022 Tejun Heo <tj@kernel.org> * Copyright (c) 2022 David Vernet <dvernet@meta.com> */ #include <linux/btf_ids.h> #include "ext_idle.h" /* * NOTE: sched_ext is in the process of growing multiple scheduler support and * scx_root usage is in a transitional state. Naked dereferences are safe if the * caller is one of the tasks attached to SCX and explicit RCU dereference is * necessary otherwise. Naked scx_root dereferences trigger sparse warnings but * are used as temporary markers to indicate that the dereferences need to be * updated to point to the associated scheduler instances rather than scx_root. */ static struct scx_sched __rcu *scx_root; /* * During exit, a task may schedule after losing its PIDs. When disabling the * BPF scheduler, we need to be able to iterate tasks in every state to * guarantee system safety. Maintain a dedicated task list which contains every * task between its fork and eventual free. */ static DEFINE_RAW_SPINLOCK(scx_tasks_lock); static LIST_HEAD(scx_tasks); /* ops enable/disable */ static DEFINE_MUTEX(scx_enable_mutex); DEFINE_STATIC_KEY_FALSE(__scx_enabled); DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem); static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED); static int scx_bypass_depth; static cpumask_var_t scx_bypass_lb_donee_cpumask; static cpumask_var_t scx_bypass_lb_resched_cpumask; static bool scx_aborting; static bool scx_init_task_enabled; static bool scx_switching_all; DEFINE_STATIC_KEY_FALSE(__scx_switched_all); /* * Tracks whether scx_enable() called scx_bypass(true). Used to balance bypass * depth on enable failure. Will be removed when bypass depth is moved into the * sched instance. */ static bool scx_bypassed_for_enable; static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0); static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0); /* * A monotically increasing sequence number that is incremented every time a * scheduler is enabled. This can be used by to check if any custom sched_ext * scheduler has ever been used in the system. */ static atomic_long_t scx_enable_seq = ATOMIC_LONG_INIT(0); /* * The maximum amount of time in jiffies that a task may be runnable without * being scheduled on a CPU. If this timeout is exceeded, it will trigger * scx_error(). */ static unsigned long scx_watchdog_timeout; /* * The last time the delayed work was run. This delayed work relies on * ksoftirqd being able to run to service timer interrupts, so it's possible * that this work itself could get wedged. To account for this, we check that * it's not stalled in the timer tick, and trigger an error if it is. */ static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES; static struct delayed_work scx_watchdog_work; /* * For %SCX_KICK_WAIT: Each CPU has a pointer to an array of kick_sync sequence * numbers. The arrays are allocated with kvzalloc() as size can exceed percpu * allocator limits on large machines. O(nr_cpu_ids^2) allocation, allocated * lazily when enabling and freed when disabling to avoid waste when sched_ext * isn't active. */ struct scx_kick_syncs { struct rcu_head rcu; unsigned long syncs[]; }; static DEFINE_PER_CPU(struct scx_kick_syncs __rcu *, scx_kick_syncs); /* * Direct dispatch marker. * * Non-NULL values are used for direct dispatch from enqueue path. A valid * pointer points to the task currently being enqueued. An ERR_PTR value is used * to indicate that direct dispatch has already happened. */ static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task); static const struct rhashtable_params dsq_hash_params = { .key_len = sizeof_field(struct scx_dispatch_q, id), .key_offset = offsetof(struct scx_dispatch_q, id), .head_offset = offsetof(struct scx_dispatch_q, hash_node), }; static LLIST_HEAD(dsqs_to_free); /* dispatch buf */ struct scx_dsp_buf_ent { struct task_struct *task; unsigned long qseq; u64 dsq_id; u64 enq_flags; }; static u32 scx_dsp_max_batch; struct scx_dsp_ctx { struct rq *rq; u32 cursor; u32 nr_tasks; struct scx_dsp_buf_ent buf[]; }; static struct scx_dsp_ctx __percpu *scx_dsp_ctx; /* string formatting from BPF */ struct scx_bstr_buf { u64 data[MAX_BPRINTF_VARARGS]; char line[SCX_EXIT_MSG_LEN]; }; static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock); static struct scx_bstr_buf scx_exit_bstr_buf; /* ops debug dump */ struct scx_dump_data { s32 cpu; bool first; s32 cursor; struct seq_buf *s; const char *prefix; struct scx_bstr_buf buf; }; static struct scx_dump_data scx_dump_data = { .cpu = -1, }; /* /sys/kernel/sched_ext interface */ static struct kset *scx_kset; /* * Parameters that can be adjusted through /sys/module/sched_ext/parameters. * There usually is no reason to modify these as normal scheduler operation * shouldn't be affected by them. The knobs are primarily for debugging. */ static u64 scx_slice_dfl = SCX_SLICE_DFL; static unsigned int scx_slice_bypass_us = SCX_SLICE_BYPASS / NSEC_PER_USEC; static unsigned int scx_bypass_lb_intv_us = SCX_BYPASS_LB_DFL_INTV_US; static int set_slice_us(const char *val, const struct kernel_param *kp) { return param_set_uint_minmax(val, kp, 100, 100 * USEC_PER_MSEC); } static const struct kernel_param_ops slice_us_param_ops = { .set = set_slice_us, .get = param_get_uint, }; static int set_bypass_lb_intv_us(const char *val, const struct kernel_param *kp) { return param_set_uint_minmax(val, kp, 0, 10 * USEC_PER_SEC); } static const struct kernel_param_ops bypass_lb_intv_us_param_ops = { .set = set_bypass_lb_intv_us, .get = param_get_uint, }; #undef MODULE_PARAM_PREFIX #define MODULE_PARAM_PREFIX "sched_ext." module_param_cb(slice_bypass_us, &slice_us_param_ops, &scx_slice_bypass_us, 0600); MODULE_PARM_DESC(slice_bypass_us, "bypass slice in microseconds, applied on [un]load (100us to 100ms)"); module_param_cb(bypass_lb_intv_us, &bypass_lb_intv_us_param_ops, &scx_bypass_lb_intv_us, 0600); MODULE_PARM_DESC(bypass_lb_intv_us, "bypass load balance interval in microseconds (0 (disable) to 10s)"); #undef MODULE_PARAM_PREFIX #define CREATE_TRACE_POINTS #include <trace/events/sched_ext.h> static void process_ddsp_deferred_locals(struct rq *rq); static bool task_dead_and_done(struct task_struct *p); static u32 reenq_local(struct rq *rq); static void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags); static bool scx_vexit(struct scx_sched *sch, enum scx_exit_kind kind, s64 exit_code, const char *fmt, va_list args); static __printf(4, 5) bool scx_exit(struct scx_sched *sch, enum scx_exit_kind kind, s64 exit_code, const char *fmt, ...) { va_list args; bool ret; va_start(args, fmt); ret = scx_vexit(sch, kind, exit_code, fmt, args); va_end(args); return ret; } #define scx_error(sch, fmt, args...) scx_exit((sch), SCX_EXIT_ERROR, 0, fmt, ##args) #define scx_verror(sch, fmt, args) scx_vexit((sch), SCX_EXIT_ERROR, 0, fmt, args) #define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op) static long jiffies_delta_msecs(unsigned long at, unsigned long now) { if (time_after(at, now)) return jiffies_to_msecs(at - now); else return -(long)jiffies_to_msecs(now - at); } /* if the highest set bit is N, return a mask with bits [N+1, 31] set */ static u32 higher_bits(u32 flags) { return ~((1 << fls(flags)) - 1); } /* return the mask with only the highest bit set */ static u32 highest_bit(u32 flags) { int bit = fls(flags); return ((u64)1 << bit) >> 1; } static bool u32_before(u32 a, u32 b) { return (s32)(a - b) < 0; } static struct scx_dispatch_q *find_global_dsq(struct scx_sched *sch, struct task_struct *p) { return sch->global_dsqs[cpu_to_node(task_cpu(p))]; } static struct scx_dispatch_q *find_user_dsq(struct scx_sched *sch, u64 dsq_id) { return rhashtable_lookup(&sch->dsq_hash, &dsq_id, dsq_hash_params); } static const struct sched_class *scx_setscheduler_class(struct task_struct *p) { if (p->sched_class == &stop_sched_class) return &stop_sched_class; return __setscheduler_class(p->policy, p->prio); } /* * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check * whether it's running from an allowed context. * * @mask is constant, always inline to cull the mask calculations. */ static __always_inline void scx_kf_allow(u32 mask) { /* nesting is allowed only in increasing scx_kf_mask order */ WARN_ONCE((mask | higher_bits(mask)) & current->scx.kf_mask, "invalid nesting current->scx.kf_mask=0x%x mask=0x%x\n", current->scx.kf_mask, mask); current->scx.kf_mask |= mask; barrier(); } static void scx_kf_disallow(u32 mask) { barrier(); current->scx.kf_mask &= ~mask; } /* * Track the rq currently locked. * * This allows kfuncs to safely operate on rq from any scx ops callback, * knowing which rq is already locked. */ DEFINE_PER_CPU(struct rq *, scx_locked_rq_state); static inline void update_locked_rq(struct rq *rq) { /* * Check whether @rq is actually locked. This can help expose bugs * or incorrect assumptions about the context in which a kfunc or * callback is executed. */ if (rq) lockdep_assert_rq_held(rq); __this_cpu_write(scx_locked_rq_state, rq); } #define SCX_CALL_OP(sch, mask, op, rq, args...) \ do { \ if (rq) \ update_locked_rq(rq); \ if (mask) { \ scx_kf_allow(mask); \ (sch)->ops.op(args); \ scx_kf_disallow(mask); \ } else { \ (sch)->ops.op(args); \ } \ if (rq) \ update_locked_rq(NULL); \ } while (0) #define SCX_CALL_OP_RET(sch, mask, op, rq, args...) \ ({ \ __typeof__((sch)->ops.op(args)) __ret; \ \ if (rq) \ update_locked_rq(rq); \ if (mask) { \ scx_kf_allow(mask); \ __ret = (sch)->ops.op(args); \ scx_kf_disallow(mask); \ } else { \ __ret = (sch)->ops.op(args); \ } \ if (rq) \ update_locked_rq(NULL); \ __ret; \ }) /* * Some kfuncs are allowed only on the tasks that are subjects of the * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such * restrictions, the following SCX_CALL_OP_*() variants should be used when * invoking scx_ops operations that take task arguments. These can only be used * for non-nesting operations due to the way the tasks are tracked. * * kfuncs which can only operate on such tasks can in turn use * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on * the specific task. */ #define SCX_CALL_OP_TASK(sch, mask, op, rq, task, args...) \ do { \ BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ current->scx.kf_tasks[0] = task; \ SCX_CALL_OP((sch), mask, op, rq, task, ##args); \ current->scx.kf_tasks[0] = NULL; \ } while (0) #define SCX_CALL_OP_TASK_RET(sch, mask, op, rq, task, args...) \ ({ \ __typeof__((sch)->ops.op(task, ##args)) __ret; \ BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ current->scx.kf_tasks[0] = task; \ __ret = SCX_CALL_OP_RET((sch), mask, op, rq, task, ##args); \ current->scx.kf_tasks[0] = NULL; \ __ret; \ }) #define SCX_CALL_OP_2TASKS_RET(sch, mask, op, rq, task0, task1, args...) \ ({ \ __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \ BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \ current->scx.kf_tasks[0] = task0; \ current->scx.kf_tasks[1] = task1; \ __ret = SCX_CALL_OP_RET((sch), mask, op, rq, task0, task1, ##args); \ current->scx.kf_tasks[0] = NULL; \ current->scx.kf_tasks[1] = NULL; \ __ret; \ }) /* @mask is constant, always inline to cull unnecessary branches */ static __always_inline bool scx_kf_allowed(struct scx_sched *sch, u32 mask) { if (unlikely(!(current->scx.kf_mask & mask))) { scx_error(sch, "kfunc with mask 0x%x called from an operation only allowing 0x%x", mask, current->scx.kf_mask); return false; } /* * Enforce nesting boundaries. e.g. A kfunc which can be called from * DISPATCH must not be called if we're running DEQUEUE which is nested * inside ops.dispatch(). We don't need to check boundaries for any * blocking kfuncs as the verifier ensures they're only called from * sleepable progs. */ if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE && (current->scx.kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) { scx_error(sch, "cpu_release kfunc called from a nested operation"); return false; } if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH && (current->scx.kf_mask & higher_bits(SCX_KF_DISPATCH)))) { scx_error(sch, "dispatch kfunc called from a nested operation"); return false; } return true; } /* see SCX_CALL_OP_TASK() */ static __always_inline bool scx_kf_allowed_on_arg_tasks(struct scx_sched *sch, u32 mask, struct task_struct *p) { if (!scx_kf_allowed(sch, mask)) return false; if (unlikely((p != current->scx.kf_tasks[0] && p != current->scx.kf_tasks[1]))) { scx_error(sch, "called on a task not being operated on"); return false; } return true; } /** * nldsq_next_task - Iterate to the next task in a non-local DSQ * @dsq: user dsq being iterated * @cur: current position, %NULL to start iteration * @rev: walk backwards * * Returns %NULL when iteration is finished. */ static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq, struct task_struct *cur, bool rev) { struct list_head *list_node; struct scx_dsq_list_node *dsq_lnode; lockdep_assert_held(&dsq->lock); if (cur) list_node = &cur->scx.dsq_list.node; else list_node = &dsq->list; /* find the next task, need to skip BPF iteration cursors */ do { if (rev) list_node = list_node->prev; else list_node = list_node->next; if (list_node == &dsq->list) return NULL; dsq_lnode = container_of(list_node, struct scx_dsq_list_node, node); } while (dsq_lnode->flags & SCX_DSQ_LNODE_ITER_CURSOR); return container_of(dsq_lnode, struct task_struct, scx.dsq_list); } #define nldsq_for_each_task(p, dsq) \ for ((p) = nldsq_next_task((dsq), NULL, false); (p); \ (p) = nldsq_next_task((dsq), (p), false)) /* * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse] * dispatch order. BPF-visible iterator is opaque and larger to allow future * changes without breaking backward compatibility. Can be used with * bpf_for_each(). See bpf_iter_scx_dsq_*(). */ enum scx_dsq_iter_flags { /* iterate in the reverse dispatch order */ SCX_DSQ_ITER_REV = 1U << 16, __SCX_DSQ_ITER_HAS_SLICE = 1U << 30, __SCX_DSQ_ITER_HAS_VTIME = 1U << 31, __SCX_DSQ_ITER_USER_FLAGS = SCX_DSQ_ITER_REV, __SCX_DSQ_ITER_ALL_FLAGS = __SCX_DSQ_ITER_USER_FLAGS | __SCX_DSQ_ITER_HAS_SLICE | __SCX_DSQ_ITER_HAS_VTIME, }; struct bpf_iter_scx_dsq_kern { struct scx_dsq_list_node cursor; struct scx_dispatch_q *dsq; u64 slice; u64 vtime; } __attribute__((aligned(8)));