/* SPDX-License-Identifier: GPL-2.0 *//* * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst * * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. * Copyright (c) 2022 Tejun Heo <tj@kernel.org> * Copyright (c) 2022 David Vernet <dvernet@meta.com> */#define SCX_OP_IDX(op)		(offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void))) enum scx_consts {	SCX_DSP_DFL_MAX_BATCH		= 32,	SCX_DSP_MAX_LOOPS		= 32,	SCX_WATCHDOG_MAX_TIMEOUT	= 30 * HZ, 	SCX_EXIT_BT_LEN			= 64,	SCX_EXIT_MSG_LEN		= 1024,	SCX_EXIT_DUMP_DFL_LEN		= 32768, 	SCX_CPUPERF_ONE			= SCHED_CAPACITY_SCALE, 	/*	 * Iterating all tasks may take a while. Periodically drop	 * scx_tasks_lock to avoid causing e.g. CSD and RCU stalls.	 */	SCX_OPS_TASK_ITER_BATCH		= 32,}; enum scx_exit_kind {	SCX_EXIT_NONE,	SCX_EXIT_DONE, 	SCX_EXIT_UNREG = 64,	/* user-space initiated unregistration */	SCX_EXIT_UNREG_BPF,	/* BPF-initiated unregistration */	SCX_EXIT_UNREG_KERN,	/* kernel-initiated unregistration */	SCX_EXIT_SYSRQ,		/* requested by 'S' sysrq */ 	SCX_EXIT_ERROR = 1024,	/* runtime error, error msg contains details */	SCX_EXIT_ERROR_BPF,	/* ERROR but triggered through scx_bpf_error() */	SCX_EXIT_ERROR_STALL,	/* watchdog detected stalled runnable tasks */}; /* * An exit code can be specified when exiting with scx_bpf_exit() or * scx_ops_exit(), corresponding to exit_kind UNREG_BPF and UNREG_KERN * respectively. The codes are 64bit of the format: * *   Bits: [63  ..  48 47   ..  32 31 .. 0] *         [ SYS ACT ] [ SYS RSN ] [ USR  ] * *   SYS ACT: System-defined exit actions *   SYS RSN: System-defined exit reasons *   USR    : User-defined exit codes and reasons * * Using the above, users may communicate intention and context by ORing system * actions and/or system reasons with a user-defined exit code. */enum scx_exit_code {	/* Reasons */	SCX_ECODE_RSN_HOTPLUG	= 1LLU << 32, 	/* Actions */	SCX_ECODE_ACT_RESTART	= 1LLU << 48,}; /* * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is * being disabled. */struct scx_exit_info {	/* %SCX_EXIT_* - broad category of the exit reason */	enum scx_exit_kind	kind; 	/* exit code if gracefully exiting */	s64			exit_code; 	/* textual representation of the above */	const char		*reason; 	/* backtrace if exiting due to an error */	unsigned long		*bt;	u32			bt_len; 	/* informational message */	char			*msg; 	/* debug dump */	char			*dump;}; /* sched_ext_ops.flags */enum scx_ops_flags {	/*	 * Keep built-in idle tracking even if ops.update_idle() is implemented.	 */	SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0, 	/*	 * By default, if there are no other task to run on the CPU, ext core	 * keeps running the current task even after its slice expires. If this	 * flag is specified, such tasks are passed to ops.enqueue() with	 * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info.	 */	SCX_OPS_ENQ_LAST	= 1LLU << 1, 	/*	 * An exiting task may schedule after PF_EXITING is set. In such cases,	 * bpf_task_from_pid() may not be able to find the task and if the BPF	 * scheduler depends on pid lookup for dispatching, the task will be	 * lost leading to various issues including RCU grace period stalls.	 *	 * To mask this problem, by default, unhashed tasks are automatically	 * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't	 * depend on pid lookups and wants to handle these tasks directly, the	 * following flag can be used.	 */	SCX_OPS_ENQ_EXITING	= 1LLU << 2, 	/*	 * If set, only tasks with policy set to SCHED_EXT are attached to	 * sched_ext. If clear, SCHED_NORMAL tasks are also included.	 */	SCX_OPS_SWITCH_PARTIAL	= 1LLU << 3, 	/*	 * CPU cgroup support flags	 */	SCX_OPS_HAS_CGROUP_WEIGHT = 1LLU << 16,	/* cpu.weight */ 	SCX_OPS_ALL_FLAGS	= SCX_OPS_KEEP_BUILTIN_IDLE |				  SCX_OPS_ENQ_LAST |				  SCX_OPS_ENQ_EXITING |				  SCX_OPS_SWITCH_PARTIAL |				  SCX_OPS_HAS_CGROUP_WEIGHT,}; /* argument container for ops.init_task() */struct scx_init_task_args {	/*	 * Set if ops.init_task() is being invoked on the fork path, as opposed	 * to the scheduler transition path.	 */	bool			fork;#ifdef CONFIG_EXT_GROUP_SCHED	/* the cgroup the task is joining */	struct cgroup		*cgroup;#endif}; /* argument container for ops.exit_task() */struct scx_exit_task_args {	/* Whether the task exited before running on sched_ext. */	bool cancelled;}; /* argument container for ops->cgroup_init() */struct scx_cgroup_init_args {	/* the weight of the cgroup [1..10000] */	u32			weight;}; enum scx_cpu_preempt_reason {	/* next task is being scheduled by &sched_class_rt */	SCX_CPU_PREEMPT_RT,	/* next task is being scheduled by &sched_class_dl */	SCX_CPU_PREEMPT_DL,	/* next task is being scheduled by &sched_class_stop */	SCX_CPU_PREEMPT_STOP,	/* unknown reason for SCX being preempted */	SCX_CPU_PREEMPT_UNKNOWN,}; /* * Argument container for ops->cpu_acquire(). Currently empty, but may be * expanded in the future. */struct scx_cpu_acquire_args {}; /* argument container for ops->cpu_release() */struct scx_cpu_release_args {	/* the reason the CPU was preempted */	enum scx_cpu_preempt_reason reason; 	/* the task that's going to be scheduled on the CPU */	struct task_struct	*task;}; /* * Informational context provided to dump operations. */struct scx_dump_ctx {	enum scx_exit_kind	kind;	s64			exit_code;	const char		*reason;	u64			at_ns;	u64			at_jiffies;}; /** * struct sched_ext_ops - Operation table for BPF scheduler implementation * * A BPF scheduler can implement an arbitrary scheduling policy by * implementing and loading operations in this table. Note that a userland * scheduling policy can also be implemented using the BPF scheduler * as a shim layer. */struct sched_ext_ops {	/**	 * select_cpu - Pick the target CPU for a task which is being woken up	 * @p: task being woken up	 * @prev_cpu: the cpu @p was on before sleeping	 * @wake_flags: SCX_WAKE_*	 *	 * Decision made here isn't final. @p may be moved to any CPU while it	 * is getting dispatched for execution later. However, as @p is not on	 * the rq at this point, getting the eventual execution CPU right here	 * saves a small bit of overhead down the line.	 *	 * If an idle CPU is returned, the CPU is kicked and will try to	 * dispatch. While an explicit custom mechanism can be added,	 * select_cpu() serves as the default way to wake up idle CPUs.	 *	 * @p may be inserted into a DSQ directly by calling	 * scx_bpf_dsq_insert(). If so, the ops.enqueue() will be skipped.	 * Directly inserting into %SCX_DSQ_LOCAL will put @p in the local DSQ	 * of the CPU returned by this operation.	 *	 * Note that select_cpu() is never called for tasks that can only run	 * on a single CPU or tasks with migration disabled, as they don't have	 * the option to select a different CPU. See select_task_rq() for	 * details.	 */	s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); 	/**	 * enqueue - Enqueue a task on the BPF scheduler	 * @p: task being enqueued	 * @enq_flags: %SCX_ENQ_*	 *	 * @p is ready to run. Insert directly into a DSQ by calling	 * scx_bpf_dsq_insert() or enqueue on the BPF scheduler. If not directly	 * inserted, the bpf scheduler owns @p and if it fails to dispatch @p,	 * the task will stall.	 *	 * If @p was inserted into a DSQ from ops.select_cpu(), this callback is	 * skipped.	 */	void (*enqueue)(struct task_struct *p, u64 enq_flags); 	/**	 * dequeue - Remove a task from the BPF scheduler	 * @p: task being dequeued	 * @deq_flags: %SCX_DEQ_*	 *	 * Remove @p from the BPF scheduler. This is usually called to isolate	 * the task while updating its scheduling properties (e.g. priority).	 *	 * The ext core keeps track of whether the BPF side owns a given task or	 * not and can gracefully ignore spurious dispatches from BPF side,	 * which makes it safe to not implement this method. However, depending	 * on the scheduling logic, this can lead to confusing behaviors - e.g.	 * scheduling position not being updated across a priority change.	 */	void (*dequeue)(struct task_struct *p, u64 deq_flags); 	/**	 * dispatch - Dispatch tasks from the BPF scheduler and/or user DSQs	 * @cpu: CPU to dispatch tasks for	 * @prev: previous task being switched out	 *	 * Called when a CPU's local dsq is empty. The operation should dispatch	 * one or more tasks from the BPF scheduler into the DSQs using	 * scx_bpf_dsq_insert() and/or move from user DSQs into the local DSQ	 * using scx_bpf_dsq_move_to_local().	 *	 * The maximum number of times scx_bpf_dsq_insert() can be called	 * without an intervening scx_bpf_dsq_move_to_local() is specified by	 * ops.dispatch_max_batch. See the comments on top of the two functions	 * for more details.	 *	 * When not %NULL, @prev is an SCX task with its slice depleted. If	 * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in	 * @prev->scx.flags, it is not enqueued yet and will be enqueued after	 * ops.dispatch() returns. To keep executing @prev, return without	 * dispatching or moving any tasks. Also see %SCX_OPS_ENQ_LAST.	 */	void (*dispatch)(s32 cpu, struct task_struct *prev); 	/**	 * tick - Periodic tick	 * @p: task running currently	 *	 * This operation is called every 1/HZ seconds on CPUs which are	 * executing an SCX task. Setting @p->scx.slice to 0 will trigger an	 * immediate dispatch cycle on the CPU.	 */	void (*tick)(struct task_struct *p); 	/**	 * runnable - A task is becoming runnable on its associated CPU	 * @p: task becoming runnable	 * @enq_flags: %SCX_ENQ_*	 *	 * This and the following three functions can be used to track a task's	 * execution state transitions. A task becomes ->runnable() on a CPU,	 * and then goes through one or more ->running() and ->stopping() pairs	 * as it runs on the CPU, and eventually becomes ->quiescent() when it's	 * done running on the CPU.	 *	 * @p is becoming runnable on the CPU because it's	 *	 * - waking up (%SCX_ENQ_WAKEUP)	 * - being moved from another CPU	 * - being restored after temporarily taken off the queue for an	 *   attribute change.	 *	 * This and ->enqueue() are related but not coupled. This operation	 * notifies @p's state transition and may not be followed by ->enqueue()	 * e.g. when @p is being dispatched to a remote CPU, or when @p is	 * being enqueued on a CPU experiencing a hotplug event. Likewise, a	 * task may be ->enqueue()'d without being preceded by this operation	 * e.g. after exhausting its slice.	 */	void (*runnable)(struct task_struct *p, u64 enq_flags); 	/**	 * running - A task is starting to run on its associated CPU	 * @p: task starting to run	 *	 * See ->runnable() for explanation on the task state notifiers.	 */	void (*running)(struct task_struct *p); 	/**	 * stopping - A task is stopping execution	 * @p: task stopping to run	 * @runnable: is task @p still runnable?	 *	 * See ->runnable() for explanation on the task state notifiers. If	 * !@runnable, ->quiescent() will be invoked after this operation	 * returns.	 */	void (*stopping)(struct task_struct *p, bool runnable); 	/**	 * quiescent - A task is becoming not runnable on its associated CPU	 * @p: task becoming not runnable	 * @deq_flags: %SCX_DEQ_*	 *	 * See ->runnable() for explanation on the task state notifiers.	 *	 * @p is becoming quiescent on the CPU because it's	 *	 * - sleeping (%SCX_DEQ_SLEEP)	 * - being moved to another CPU	 * - being temporarily taken off the queue for an attribute change	 *   (%SCX_DEQ_SAVE)	 *	 * This and ->dequeue() are related but not coupled. This operation	 * notifies @p's state transition and may not be preceded by ->dequeue()	 * e.g. when @p is being dispatched to a remote CPU.	 */	void (*quiescent)(struct task_struct *p, u64 deq_flags); 	/**	 * yield - Yield CPU	 * @from: yielding task	 * @to: optional yield target task	 *	 * If @to is NULL, @from is yielding the CPU to other runnable tasks.	 * The BPF scheduler should ensure that other available tasks are	 * dispatched before the yielding task. Return value is ignored in this	 * case.	 *	 * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf	 * scheduler can implement the request, return %true; otherwise, %false.	 */	bool (*yield)(struct task_struct *from, struct task_struct *to); 	/**	 * core_sched_before - Task ordering for core-sched	 * @a: task A	 * @b: task B	 *	 * Used by core-sched to determine the ordering between two tasks. See	 * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on	 * core-sched.	 *	 * Both @a and @b are runnable and may or may not currently be queued on	 * the BPF scheduler. Should return %true if @a should run before @b.	 * %false if there's no required ordering or @b should run before @a.	 *	 * If not specified, the default is ordering them according to when they	 * became runnable.	 */	bool (*core_sched_before)(struct task_struct *a, struct task_struct *b); 	/**	 * set_weight - Set task weight	 * @p: task to set weight for	 * @weight: new weight [1..10000]	 *	 * Update @p's weight to @weight.	 */	void (*set_weight)(struct task_struct *p, u32 weight); 	/**	 * set_cpumask - Set CPU affinity	 * @p: task to set CPU affinity for	 * @cpumask: cpumask of cpus that @p can run on	 *	 * Update @p's CPU affinity to @cpumask.	 */	void (*set_cpumask)(struct task_struct *p,			    const struct cpumask *cpumask); 	/**	 * update_idle - Update the idle state of a CPU	 * @cpu: CPU to udpate the idle state for	 * @idle: whether entering or exiting the idle state	 *	 * This operation is called when @rq's CPU goes or leaves the idle	 * state. By default, implementing this operation disables the built-in	 * idle CPU tracking and the following helpers become unavailable:	 *	 * - scx_bpf_select_cpu_dfl()	 * - scx_bpf_test_and_clear_cpu_idle()	 * - scx_bpf_pick_idle_cpu()	 *	 * The user also must implement ops.select_cpu() as the default	 * implementation relies on scx_bpf_select_cpu_dfl().	 *	 * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle	 * tracking.	 */	void (*update_idle)(s32 cpu, bool idle); 	/**	 * cpu_acquire - A CPU is becoming available to the BPF scheduler	 * @cpu: The CPU being acquired by the BPF scheduler.	 * @args: Acquire arguments, see the struct definition.	 *	 * A CPU that was previously released from the BPF scheduler is now once	 * again under its control.	 */	void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args); 	/**	 * cpu_release - A CPU is taken away from the BPF scheduler	 * @cpu: The CPU being released by the BPF scheduler.	 * @args: Release arguments, see the struct definition.	 *	 * The specified CPU is no longer under the control of the BPF	 * scheduler. This could be because it was preempted by a higher	 * priority sched_class, though there may be other reasons as well. The	 * caller should consult @args->reason to determine the cause.	 */	void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args); 	/**	 * init_task - Initialize a task to run in a BPF scheduler	 * @p: task to initialize for BPF scheduling	 * @args: init arguments, see the struct definition	 *	 * Either we're loading a BPF scheduler or a new task is being forked.	 * Initialize @p for BPF scheduling. This operation may block and can	 * be used for allocations, and is called exactly once for a task.	 *	 * Return 0 for success, -errno for failure. An error return while	 * loading will abort loading of the BPF scheduler. During a fork, it	 * will abort that specific fork.	 */	s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args); 	/**	 * exit_task - Exit a previously-running task from the system	 * @p: task to exit	 *	 * @p is exiting or the BPF scheduler is being unloaded. Perform any	 * necessary cleanup for @p.	 */	void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args); 	/**	 * enable - Enable BPF scheduling for a task	 * @p: task to enable BPF scheduling for	 *	 * Enable @p for BPF scheduling. enable() is called on @p any time it	 * enters SCX, and is always paired with a matching disable().	 */	void (*enable)(struct task_struct *p); 	/**	 * disable - Disable BPF scheduling for a task	 * @p: task to disable BPF scheduling for	 *	 * @p is exiting, leaving SCX or the BPF scheduler is being unloaded.	 * Disable BPF scheduling for @p. A disable() call is always matched	 * with a prior enable() call.	 */

Lines 1–500 of 7694. Load more