/* SPDX-License-Identifier: GPL-2.0 *//* * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst * * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. * Copyright (c) 2022 Tejun Heo <tj@kernel.org> * Copyright (c) 2022 David Vernet <dvernet@meta.com> */#include <linux/btf_ids.h>#include "ext_idle.h" #define SCX_OP_IDX(op)		(offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) enum scx_consts {	SCX_DSP_DFL_MAX_BATCH		= 32,	SCX_DSP_MAX_LOOPS		= 32,	SCX_WATCHDOG_MAX_TIMEOUT	= 30 * HZ, 	SCX_EXIT_BT_LEN			= 64,	SCX_EXIT_MSG_LEN		= 1024,	SCX_EXIT_DUMP_DFL_LEN		= 32768, 	SCX_CPUPERF_ONE			= SCHED_CAPACITY_SCALE, 	/*	 * Iterating all tasks may take a while. Periodically drop	 * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls.	 */	SCX_TASK_ITER_BATCH		= 32,}; enum scx_exit_kind {	SCX_EXIT_NONE,	SCX_EXIT_DONE, 	SCX_EXIT_UNREG = 64,	/* user-space initiated unregistration */	SCX_EXIT_UNREG_BPF,	/* BPF-initiated unregistration */	SCX_EXIT_UNREG_KERN,	/* kernel-initiated unregistration */	SCX_EXIT_SYSRQ,		/* requested by 'S' sysrq */ 	SCX_EXIT_ERROR = 1024,	/* runtime error, error msg contains details */	SCX_EXIT_ERROR_BPF,	/* ERROR but triggered through scx_bpf_error() */	SCX_EXIT_ERROR_STALL,	/* watchdog detected stalled runnable tasks */}; /* * An exit code can be specified when exiting with scx_bpf_exit() or scx_exit(), * corresponding to exit_kind UNREG_BPF and UNREG_KERN respectively. The codes * are 64bit of the format: * *   Bits: [63  ..  48 47   ..  32 31 .. 0] *         [ SYS ACT ] [ SYS RSN ] [ USR  ] * *   SYS ACT: System-defined exit actions *   SYS RSN: System-defined exit reasons *   USR    : User-defined exit codes and reasons * * Using the above, users may communicate intention and context by ORing system * actions and/or system reasons with a user-defined exit code. */enum scx_exit_code {	/* Reasons */	SCX_ECODE_RSN_HOTPLUG	= 1LLU << 32, 	/* Actions */	SCX_ECODE_ACT_RESTART	= 1LLU << 48,}; /* * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is * being disabled. */struct scx_exit_info {	/* %SCX_EXIT_* - broad category of the exit reason */	enum scx_exit_kind	kind; 	/* exit code if gracefully exiting */	s64			exit_code; 	/* textual representation of the above */	const char		*reason; 	/* backtrace if exiting due to an error */	unsigned long		*bt;	u32			bt_len; 	/* informational message */	char			*msg; 	/* debug dump */	char			*dump;}; /* sched_ext_ops.flags */enum scx_ops_flags {	/*	 * Keep built-in idle tracking even if ops.update_idle() is implemented.	 */	SCX_OPS_KEEP_BUILTIN_IDLE	= 1LLU << 0, 	/*	 * By default, if there are no other task to run on the CPU, ext core	 * keeps running the current task even after its slice expires. If this	 * flag is specified, such tasks are passed to ops.enqueue() with	 * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info.	 */	SCX_OPS_ENQ_LAST		= 1LLU << 1, 	/*	 * An exiting task may schedule after PF_EXITING is set. In such cases,	 * bpf_task_from_pid() may not be able to find the task and if the BPF	 * scheduler depends on pid lookup for dispatching, the task will be	 * lost leading to various issues including RCU grace period stalls.	 *	 * To mask this problem, by default, unhashed tasks are automatically	 * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't	 * depend on pid lookups and wants to handle these tasks directly, the	 * following flag can be used.	 */	SCX_OPS_ENQ_EXITING		= 1LLU << 2, 	/*	 * If set, only tasks with policy set to SCHED_EXT are attached to	 * sched_ext. If clear, SCHED_NORMAL tasks are also included.	 */	SCX_OPS_SWITCH_PARTIAL		= 1LLU << 3, 	/*	 * A migration disabled task can only execute on its current CPU. By	 * default, such tasks are automatically put on the CPU's local DSQ with	 * the default slice on enqueue. If this ops flag is set, they also go	 * through ops.enqueue().	 *	 * A migration disabled task never invokes ops.select_cpu() as it can	 * only select the current CPU. Also, p->cpus_ptr will only contain its	 * current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr	 * and thus may disagree with cpumask_weight(p->cpus_ptr).	 */	SCX_OPS_ENQ_MIGRATION_DISABLED	= 1LLU << 4, 	/*	 * Queued wakeup (ttwu_queue) is a wakeup optimization that invokes	 * ops.enqueue() on the ops.select_cpu() selected or the wakee's	 * previous CPU via IPI (inter-processor interrupt) to reduce cacheline	 * transfers. When this optimization is enabled, ops.select_cpu() is	 * skipped in some cases (when racing against the wakee switching out).	 * As the BPF scheduler may depend on ops.select_cpu() being invoked	 * during wakeups, queued wakeup is disabled by default.	 *	 * If this ops flag is set, queued wakeup optimization is enabled and	 * the BPF scheduler must be able to handle ops.enqueue() invoked on the	 * wakee's CPU without preceding ops.select_cpu() even for tasks which	 * may be executed on multiple CPUs.	 */	SCX_OPS_ALLOW_QUEUED_WAKEUP	= 1LLU << 5, 	/*	 * If set, enable per-node idle cpumasks. If clear, use a single global	 * flat idle cpumask.	 */	SCX_OPS_BUILTIN_IDLE_PER_NODE	= 1LLU << 6, 	/*	 * CPU cgroup support flags	 */	SCX_OPS_HAS_CGROUP_WEIGHT	= 1LLU << 16,	/* DEPRECATED, will be removed on 6.18 */ 	SCX_OPS_ALL_FLAGS		= SCX_OPS_KEEP_BUILTIN_IDLE |					  SCX_OPS_ENQ_LAST |					  SCX_OPS_ENQ_EXITING |					  SCX_OPS_ENQ_MIGRATION_DISABLED |					  SCX_OPS_ALLOW_QUEUED_WAKEUP |					  SCX_OPS_SWITCH_PARTIAL |					  SCX_OPS_BUILTIN_IDLE_PER_NODE |					  SCX_OPS_HAS_CGROUP_WEIGHT, 	/* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */	__SCX_OPS_INTERNAL_MASK		= 0xffLLU << 56, 	SCX_OPS_HAS_CPU_PREEMPT		= 1LLU << 56,}; /* argument container for ops.init_task() */struct scx_init_task_args {	/*	 * Set if ops.init_task() is being invoked on the fork path, as opposed	 * to the scheduler transition path.	 */	bool			fork;#ifdef CONFIG_EXT_GROUP_SCHED	/* the cgroup the task is joining */	struct cgroup		*cgroup;#endif}; /* argument container for ops.exit_task() */struct scx_exit_task_args {	/* Whether the task exited before running on sched_ext. */	bool cancelled;}; /* argument container for ops->cgroup_init() */struct scx_cgroup_init_args {	/* the weight of the cgroup [1..10000] */	u32			weight;}; enum scx_cpu_preempt_reason {	/* next task is being scheduled by &sched_class_rt */	SCX_CPU_PREEMPT_RT,	/* next task is being scheduled by &sched_class_dl */	SCX_CPU_PREEMPT_DL,	/* next task is being scheduled by &sched_class_stop */	SCX_CPU_PREEMPT_STOP,	/* unknown reason for SCX being preempted */	SCX_CPU_PREEMPT_UNKNOWN,}; /* * Argument container for ops->cpu_acquire(). Currently empty, but may be * expanded in the future. */struct scx_cpu_acquire_args {}; /* argument container for ops->cpu_release() */struct scx_cpu_release_args {	/* the reason the CPU was preempted */	enum scx_cpu_preempt_reason reason; 	/* the task that's going to be scheduled on the CPU */	struct task_struct	*task;}; /* * Informational context provided to dump operations. */struct scx_dump_ctx {	enum scx_exit_kind	kind;	s64			exit_code;	const char		*reason;	u64			at_ns;	u64			at_jiffies;}; /** * struct sched_ext_ops - Operation table for BPF scheduler implementation * * A BPF scheduler can implement an arbitrary scheduling policy by * implementing and loading operations in this table. Note that a userland * scheduling policy can also be implemented using the BPF scheduler * as a shim layer. */struct sched_ext_ops {	/**	 * @select_cpu: Pick the target CPU for a task which is being woken up	 * @p: task being woken up	 * @prev_cpu: the cpu @p was on before sleeping	 * @wake_flags: SCX_WAKE_*	 *	 * Decision made here isn't final. @p may be moved to any CPU while it	 * is getting dispatched for execution later. However, as @p is not on	 * the rq at this point, getting the eventual execution CPU right here	 * saves a small bit of overhead down the line.	 *	 * If an idle CPU is returned, the CPU is kicked and will try to	 * dispatch. While an explicit custom mechanism can be added,	 * select_cpu() serves as the default way to wake up idle CPUs.	 *	 * @p may be inserted into a DSQ directly by calling	 * scx_bpf_dsq_insert(). If so, the ops.enqueue() will be skipped.	 * Directly inserting into %SCX_DSQ_LOCAL will put @p in the local DSQ	 * of the CPU returned by this operation.	 *	 * Note that select_cpu() is never called for tasks that can only run	 * on a single CPU or tasks with migration disabled, as they don't have	 * the option to select a different CPU. See select_task_rq() for	 * details.	 */	s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); 	/**	 * @enqueue: Enqueue a task on the BPF scheduler	 * @p: task being enqueued	 * @enq_flags: %SCX_ENQ_*	 *	 * @p is ready to run. Insert directly into a DSQ by calling	 * scx_bpf_dsq_insert() or enqueue on the BPF scheduler. If not directly	 * inserted, the bpf scheduler owns @p and if it fails to dispatch @p,	 * the task will stall.	 *	 * If @p was inserted into a DSQ from ops.select_cpu(), this callback is	 * skipped.	 */	void (*enqueue)(struct task_struct *p, u64 enq_flags); 	/**	 * @dequeue: Remove a task from the BPF scheduler	 * @p: task being dequeued	 * @deq_flags: %SCX_DEQ_*	 *	 * Remove @p from the BPF scheduler. This is usually called to isolate	 * the task while updating its scheduling properties (e.g. priority).	 *	 * The ext core keeps track of whether the BPF side owns a given task or	 * not and can gracefully ignore spurious dispatches from BPF side,	 * which makes it safe to not implement this method. However, depending	 * on the scheduling logic, this can lead to confusing behaviors - e.g.	 * scheduling position not being updated across a priority change.	 */	void (*dequeue)(struct task_struct *p, u64 deq_flags); 	/**	 * @dispatch: Dispatch tasks from the BPF scheduler and/or user DSQs	 * @cpu: CPU to dispatch tasks for	 * @prev: previous task being switched out	 *	 * Called when a CPU's local dsq is empty. The operation should dispatch	 * one or more tasks from the BPF scheduler into the DSQs using	 * scx_bpf_dsq_insert() and/or move from user DSQs into the local DSQ	 * using scx_bpf_dsq_move_to_local().	 *	 * The maximum number of times scx_bpf_dsq_insert() can be called	 * without an intervening scx_bpf_dsq_move_to_local() is specified by	 * ops.dispatch_max_batch. See the comments on top of the two functions	 * for more details.	 *	 * When not %NULL, @prev is an SCX task with its slice depleted. If	 * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in	 * @prev->scx.flags, it is not enqueued yet and will be enqueued after	 * ops.dispatch() returns. To keep executing @prev, return without	 * dispatching or moving any tasks. Also see %SCX_OPS_ENQ_LAST.	 */	void (*dispatch)(s32 cpu, struct task_struct *prev); 	/**	 * @tick: Periodic tick	 * @p: task running currently	 *	 * This operation is called every 1/HZ seconds on CPUs which are	 * executing an SCX task. Setting @p->scx.slice to 0 will trigger an	 * immediate dispatch cycle on the CPU.	 */	void (*tick)(struct task_struct *p); 	/**	 * @runnable: A task is becoming runnable on its associated CPU	 * @p: task becoming runnable	 * @enq_flags: %SCX_ENQ_*	 *	 * This and the following three functions can be used to track a task's	 * execution state transitions. A task becomes ->runnable() on a CPU,	 * and then goes through one or more ->running() and ->stopping() pairs	 * as it runs on the CPU, and eventually becomes ->quiescent() when it's	 * done running on the CPU.	 *	 * @p is becoming runnable on the CPU because it's	 *	 * - waking up (%SCX_ENQ_WAKEUP)	 * - being moved from another CPU	 * - being restored after temporarily taken off the queue for an	 *   attribute change.	 *	 * This and ->enqueue() are related but not coupled. This operation	 * notifies @p's state transition and may not be followed by ->enqueue()	 * e.g. when @p is being dispatched to a remote CPU, or when @p is	 * being enqueued on a CPU experiencing a hotplug event. Likewise, a	 * task may be ->enqueue()'d without being preceded by this operation	 * e.g. after exhausting its slice.	 */	void (*runnable)(struct task_struct *p, u64 enq_flags); 	/**	 * @running: A task is starting to run on its associated CPU	 * @p: task starting to run	 *	 * Note that this callback may be called from a CPU other than the	 * one the task is going to run on. This can happen when a task	 * property is changed (i.e., affinity), since scx_next_task_scx(),	 * which triggers this callback, may run on a CPU different from	 * the task's assigned CPU.	 *	 * Therefore, always use scx_bpf_task_cpu(@p) to determine the	 * target CPU the task is going to use.	 *	 * See ->runnable() for explanation on the task state notifiers.	 */	void (*running)(struct task_struct *p); 	/**	 * @stopping: A task is stopping execution	 * @p: task stopping to run	 * @runnable: is task @p still runnable?	 *	 * Note that this callback may be called from a CPU other than the	 * one the task was running on. This can happen when a task	 * property is changed (i.e., affinity), since dequeue_task_scx(),	 * which triggers this callback, may run on a CPU different from	 * the task's assigned CPU.	 *	 * Therefore, always use scx_bpf_task_cpu(@p) to retrieve the CPU	 * the task was running on.	 *	 * See ->runnable() for explanation on the task state notifiers. If	 * !@runnable, ->quiescent() will be invoked after this operation	 * returns.	 */	void (*stopping)(struct task_struct *p, bool runnable); 	/**	 * @quiescent: A task is becoming not runnable on its associated CPU	 * @p: task becoming not runnable	 * @deq_flags: %SCX_DEQ_*	 *	 * See ->runnable() for explanation on the task state notifiers.	 *	 * @p is becoming quiescent on the CPU because it's	 *	 * - sleeping (%SCX_DEQ_SLEEP)	 * - being moved to another CPU	 * - being temporarily taken off the queue for an attribute change	 *   (%SCX_DEQ_SAVE)	 *	 * This and ->dequeue() are related but not coupled. This operation	 * notifies @p's state transition and may not be preceded by ->dequeue()	 * e.g. when @p is being dispatched to a remote CPU.	 */	void (*quiescent)(struct task_struct *p, u64 deq_flags); 	/**	 * @yield: Yield CPU	 * @from: yielding task	 * @to: optional yield target task	 *	 * If @to is NULL, @from is yielding the CPU to other runnable tasks.	 * The BPF scheduler should ensure that other available tasks are	 * dispatched before the yielding task. Return value is ignored in this	 * case.	 *	 * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf	 * scheduler can implement the request, return %true; otherwise, %false.	 */	bool (*yield)(struct task_struct *from, struct task_struct *to); 	/**	 * @core_sched_before: Task ordering for core-sched	 * @a: task A	 * @b: task B	 *	 * Used by core-sched to determine the ordering between two tasks. See	 * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on	 * core-sched.	 *	 * Both @a and @b are runnable and may or may not currently be queued on	 * the BPF scheduler. Should return %true if @a should run before @b.	 * %false if there's no required ordering or @b should run before @a.	 *	 * If not specified, the default is ordering them according to when they	 * became runnable.	 */	bool (*core_sched_before)(struct task_struct *a, struct task_struct *b); 	/**	 * @set_weight: Set task weight	 * @p: task to set weight for	 * @weight: new weight [1..10000]	 *	 * Update @p's weight to @weight.	 */	void (*set_weight)(struct task_struct *p, u32 weight); 	/**	 * @set_cpumask: Set CPU affinity	 * @p: task to set CPU affinity for	 * @cpumask: cpumask of cpus that @p can run on	 *	 * Update @p's CPU affinity to @cpumask.	 */	void (*set_cpumask)(struct task_struct *p,			    const struct cpumask *cpumask); 	/**	 * @update_idle: Update the idle state of a CPU	 * @cpu: CPU to update the idle state for	 * @idle: whether entering or exiting the idle state	 *	 * This operation is called when @rq's CPU goes or leaves the idle	 * state. By default, implementing this operation disables the built-in	 * idle CPU tracking and the following helpers become unavailable:	 *	 * - scx_bpf_select_cpu_dfl()	 * - scx_bpf_select_cpu_and()	 * - scx_bpf_test_and_clear_cpu_idle()	 * - scx_bpf_pick_idle_cpu()	 *	 * The user also must implement ops.select_cpu() as the default	 * implementation relies on scx_bpf_select_cpu_dfl().	 *	 * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle	 * tracking.	 */

Lines 1–500 of 7682. Load more