/* SPDX-License-Identifier: GPL-2.0 *//* * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst * * Copyright (c) 2025 Meta Platforms, Inc. and affiliates. * Copyright (c) 2025 Tejun Heo <tj@kernel.org> */#ifndef _KERNEL_SCHED_EXT_INTERNAL_H#define _KERNEL_SCHED_EXT_INTERNAL_H #include "../sched.h"#include "types.h" #include <trace/events/sched_ext.h> /** * scx_add_event - Increase an event counter for 'name' by 'cnt' * @sch: scx_sched to account events for * @name: an event name defined in struct scx_event_stats * @cnt: the number of the event occurred * * This can be used when preemption is not disabled. */#define scx_add_event(sch, name, cnt) do {					\	this_cpu_add((sch)->pcpu->event_stats.name, (cnt));			\	trace_sched_ext_event(#name, (cnt));					\} while(0) /** * __scx_add_event - Increase an event counter for 'name' by 'cnt' * @sch: scx_sched to account events for * @name: an event name defined in struct scx_event_stats * @cnt: the number of the event occurred * * This should be used only when preemption is disabled. */#define __scx_add_event(sch, name, cnt) do {					\	__this_cpu_add((sch)->pcpu->event_stats.name, (cnt));			\	trace_sched_ext_event(#name, cnt);					\} while(0) #define SCX_OP_IDX(op)		(offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void)))#define SCX_MOFF_IDX(moff)	((moff) / sizeof(void (*)(void))) enum scx_exit_kind {	SCX_EXIT_NONE,	SCX_EXIT_DONE, 	SCX_EXIT_UNREG = 64,	/* user-space initiated unregistration */	SCX_EXIT_UNREG_BPF,	/* BPF-initiated unregistration */	SCX_EXIT_UNREG_KERN,	/* kernel-initiated unregistration */	SCX_EXIT_SYSRQ,		/* requested by 'S' sysrq */	SCX_EXIT_PARENT,	/* parent exiting */	SCX_EXIT_PARENT_KILL,	/* killed by parent scheduler */ 	SCX_EXIT_ERROR = 1024,	/* runtime error, error msg contains details */	SCX_EXIT_ERROR_BPF,	/* ERROR but triggered through scx_bpf_error() */	SCX_EXIT_ERROR_STALL,	/* watchdog detected stalled runnable tasks */	SCX_EXIT_ERROR_REENQ,	/* task hit reenqueue limit without running */	SCX_EXIT_ERROR_RESCUE,	/* ejected for overloading rescue execution */}; /* * An exit code can be specified when exiting with scx_bpf_exit() or scx_exit(), * corresponding to exit_kind UNREG_BPF and UNREG_KERN respectively. The codes * are 64bit of the format: * *   Bits: [63  ..  48 47   ..  32 31 .. 0] *         [ SYS ACT ] [ SYS RSN ] [ USR  ] * *   SYS ACT: System-defined exit actions *   SYS RSN: System-defined exit reasons *   USR    : User-defined exit codes and reasons * * Using the above, users may communicate intention and context by ORing system * actions and/or system reasons with a user-defined exit code. */enum scx_exit_code {	/* Reasons */	SCX_ECODE_RSN_HOTPLUG	= 1LLU << 32,	SCX_ECODE_RSN_CGROUP_OFFLINE = 2LLU << 32, 	/* Actions */	SCX_ECODE_ACT_RESTART	= 1LLU << 48,}; enum scx_exit_flags {	/*	 * ops.exit() may be called even if the loading failed before ops.init()	 * finishes successfully. This is because ops.exit() allows rich exit	 * info communication. The following flag indicates whether ops.init()	 * finished successfully.	 */	SCX_EFLAG_INITIALIZED   = 1LLU << 0,}; /* * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is * being disabled. */struct scx_exit_info {	/* %SCX_EXIT_* - broad category of the exit reason */	enum scx_exit_kind	kind; 	/*	 * CPU that initiated the exit, valid once @kind has been set.	 * Negative if the exit path didn't identify a CPU.	 */	s32			exit_cpu; 	/* exit code if gracefully exiting */	s64			exit_code; 	/* %SCX_EFLAG_* */	u64			flags; 	/* textual representation of the above */	const char		*reason; 	/* backtrace if exiting due to an error */	unsigned long		*bt;	u32			bt_len; 	/* informational message */	char			*msg; 	/* debug dump */	char			*dump;}; /* sched_ext_ops.flags */enum scx_ops_flags {	/*	 * Keep built-in idle tracking even if ops.update_idle() is implemented.	 */	SCX_OPS_KEEP_BUILTIN_IDLE	= 1LLU << 0, 	/*	 * By default, if there are no other task to run on the CPU, ext core	 * keeps running the current task even after its slice expires. If this	 * flag is specified, such tasks are passed to ops.enqueue() with	 * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info.	 */	SCX_OPS_ENQ_LAST		= 1LLU << 1, 	/*	 * An exiting task may schedule after PF_EXITING is set. In such cases,	 * bpf_task_from_pid() may not be able to find the task and if the BPF	 * scheduler depends on pid lookup for dispatching, the task will be	 * lost leading to various issues including RCU grace period stalls.	 *	 * To mask this problem, by default, unhashed tasks are automatically	 * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't	 * depend on pid lookups and wants to handle these tasks directly, the	 * following flag can be used. With %SCX_OPS_TID_TO_TASK,	 * scx_bpf_tid_to_task() can find exiting tasks reliably.	 */	SCX_OPS_ENQ_EXITING		= 1LLU << 2, 	/*	 * If set, only tasks with policy set to SCHED_EXT are attached to	 * sched_ext. If clear, SCHED_NORMAL tasks are also included.	 */	SCX_OPS_SWITCH_PARTIAL		= 1LLU << 3, 	/*	 * A migration disabled task can only execute on its current CPU. By	 * default, such tasks are automatically put on the CPU's local DSQ with	 * the default slice on enqueue. If this ops flag is set, they also go	 * through ops.enqueue().	 *	 * A migration disabled task never invokes ops.select_cpu() as it can	 * only select the current CPU. Also, p->cpus_ptr will only contain its	 * current CPU while p->nr_cpus_allowed keeps tracking p->user_cpus_ptr	 * and thus may disagree with cpumask_weight(p->cpus_ptr).	 */	SCX_OPS_ENQ_MIGRATION_DISABLED	= 1LLU << 4, 	/*	 * Queued wakeup (ttwu_queue) is a wakeup optimization that invokes	 * ops.enqueue() on the ops.select_cpu() selected or the wakee's	 * previous CPU via IPI (inter-processor interrupt) to reduce cacheline	 * transfers. When this optimization is enabled, ops.select_cpu() is	 * skipped in some cases (when racing against the wakee switching out).	 * As the BPF scheduler may depend on ops.select_cpu() being invoked	 * during wakeups, queued wakeup is disabled by default.	 *	 * If this ops flag is set, queued wakeup optimization is enabled and	 * the BPF scheduler must be able to handle ops.enqueue() invoked on the	 * wakee's CPU without preceding ops.select_cpu() even for tasks which	 * may be executed on multiple CPUs.	 */	SCX_OPS_ALLOW_QUEUED_WAKEUP	= 1LLU << 5, 	/*	 * If set, enable per-node idle cpumasks. If clear, use a single global	 * flat idle cpumask.	 */	SCX_OPS_BUILTIN_IDLE_PER_NODE	= 1LLU << 6, 	/*	 * If set, %SCX_ENQ_IMMED is assumed to be set on all local DSQ	 * enqueues.	 */	SCX_OPS_ALWAYS_ENQ_IMMED	= 1LLU << 7, 	/*	 * Maintain a mapping from p->scx.tid to task_struct so the BPF	 * scheduler can recover task pointers from stored tids via	 * scx_bpf_tid_to_task().	 *	 * Only the root scheduler turns this on. A sub-sched may set the flag	 * to declare a dependency on the lookup; if the root scheduler hasn't	 * enabled it, attaching the sub-sched is rejected.	 */	SCX_OPS_TID_TO_TASK		= 1LLU << 8, 	SCX_OPS_ALL_FLAGS		= SCX_OPS_KEEP_BUILTIN_IDLE |					  SCX_OPS_ENQ_LAST |					  SCX_OPS_ENQ_EXITING |					  SCX_OPS_ENQ_MIGRATION_DISABLED |					  SCX_OPS_ALLOW_QUEUED_WAKEUP |					  SCX_OPS_SWITCH_PARTIAL |					  SCX_OPS_BUILTIN_IDLE_PER_NODE |					  SCX_OPS_ALWAYS_ENQ_IMMED |					  SCX_OPS_TID_TO_TASK, 	/* high 8 bits are internal, don't include in SCX_OPS_ALL_FLAGS */	__SCX_OPS_INTERNAL_MASK		= 0xffLLU << 56, 	SCX_OPS_HAS_CPU_PREEMPT		= 1LLU << 56,}; /* argument container for ops.init_task() */struct scx_init_task_args {	/*	 * Set if ops.init_task() is being invoked on the fork path, as opposed	 * to the scheduler transition path.	 */	bool			fork;#ifdef CONFIG_EXT_GROUP_SCHED	/* the cgroup the task is joining */	struct cgroup		*cgroup;#endif}; /* argument container for ops.exit_task() */struct scx_exit_task_args {	/* Whether the task exited before running on sched_ext. */	bool cancelled;}; /* argument container for ops.cgroup_init() */struct scx_cgroup_init_args {	/* the weight of the cgroup [1..10000] */	u32			weight; 	/* bandwidth control parameters from cpu.max and cpu.max.burst */	u64			bw_period_us;	u64			bw_quota_us;	u64			bw_burst_us;}; enum scx_cpu_preempt_reason {	/* next task is being scheduled by &sched_class_rt */	SCX_CPU_PREEMPT_RT,	/* next task is being scheduled by &sched_class_dl */	SCX_CPU_PREEMPT_DL,	/* next task is being scheduled by &sched_class_stop */	SCX_CPU_PREEMPT_STOP,	/* unknown reason for SCX being preempted */	SCX_CPU_PREEMPT_UNKNOWN,}; /* * Argument container for ops.cpu_acquire(). Currently empty, but may be * expanded in the future. */struct scx_cpu_acquire_args {}; /* argument container for ops.cpu_release() */struct scx_cpu_release_args {	/* the reason the CPU was preempted */	enum scx_cpu_preempt_reason reason; 	/* the task that's going to be scheduled on the CPU */	struct task_struct	*task;}; /* informational context provided to dump operations */struct scx_dump_ctx {	enum scx_exit_kind	kind;	s64			exit_code;	const char		*reason;	u64			at_ns;	u64			at_jiffies;}; /* argument container for ops.sub_attach() */struct scx_sub_attach_args {	struct sched_ext_ops	*ops;	char			*cgroup_path;}; /* argument container for ops.sub_detach() */struct scx_sub_detach_args {	struct sched_ext_ops	*ops;	char			*cgroup_path;}; /** * struct sched_ext_ops - Operation table for BPF scheduler implementation * * A BPF scheduler can implement an arbitrary scheduling policy by * implementing and loading operations in this table. Note that a userland * scheduling policy can also be implemented using the BPF scheduler * as a shim layer. */struct sched_ext_ops {	/**	 * @select_cpu: Pick the target CPU for a task which is being woken up	 * @p: task being woken up	 * @prev_cpu: the cpu @p was on before sleeping	 * @wake_flags: SCX_WAKE_*	 *	 * Decision made here isn't final. @p may be moved to any CPU while it	 * is getting dispatched for execution later. However, as @p is not on	 * the rq at this point, getting the eventual execution CPU right here	 * saves a small bit of overhead down the line.	 *	 * If an idle CPU is returned, the CPU is kicked and will try to	 * dispatch. While an explicit custom mechanism can be added,	 * select_cpu() serves as the default way to wake up idle CPUs.	 *	 * @p may be inserted into a DSQ directly by calling	 * scx_bpf_dsq_insert(). If so, the ops.enqueue() will be skipped.	 * Directly inserting into %SCX_DSQ_LOCAL will put @p in the local DSQ	 * of the CPU returned by this operation.	 *	 * Note that select_cpu() is never called for tasks that can only run	 * on a single CPU or tasks with migration disabled, as they don't have	 * the option to select a different CPU. See select_task_rq() for	 * details.	 */	s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags); 	/**	 * @enqueue: Enqueue a task on the BPF scheduler	 * @p: task being enqueued	 * @enq_flags: %SCX_ENQ_*	 *	 * @p is ready to run. Insert directly into a DSQ by calling	 * scx_bpf_dsq_insert() or enqueue on the BPF scheduler. If not directly	 * inserted, the bpf scheduler owns @p and if it fails to dispatch @p,	 * the task will stall.	 *	 * If @p was inserted into a DSQ from ops.select_cpu(), this callback is	 * skipped.	 */	void (*enqueue)(struct task_struct *p, u64 enq_flags); 	/**	 * @dequeue: Remove a task from the BPF scheduler	 * @p: task being dequeued	 * @deq_flags: %SCX_DEQ_*	 *	 * Remove @p from the BPF scheduler. This is usually called to isolate	 * the task while updating its scheduling properties (e.g. priority).	 *	 * The ext core keeps track of whether the BPF side owns a given task or	 * not and can gracefully ignore spurious dispatches from BPF side,	 * which makes it safe to not implement this method. However, depending	 * on the scheduling logic, this can lead to confusing behaviors - e.g.	 * scheduling position not being updated across a priority change.	 */	void (*dequeue)(struct task_struct *p, u64 deq_flags); 	/**	 * @dispatch: Dispatch tasks from the BPF scheduler and/or user DSQs	 * @cpu: CPU to dispatch tasks for	 * @prev: previous task being switched out	 *	 * Called when a CPU's local dsq is empty. The operation should dispatch	 * one or more tasks from the BPF scheduler into the DSQs using	 * scx_bpf_dsq_insert() and/or move from user DSQs into the local DSQ	 * using scx_bpf_dsq_move_to_local().	 *	 * The maximum number of times scx_bpf_dsq_insert() can be called	 * without an intervening scx_bpf_dsq_move_to_local() is specified by	 * ops.dispatch_max_batch. See the comments on top of the two functions	 * for more details.	 *	 * When not %NULL, @prev is an SCX task with its slice depleted. If	 * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in	 * @prev->scx.flags, it is not enqueued yet and will be enqueued after	 * ops.dispatch() returns. To keep executing @prev, return without	 * dispatching or moving any tasks. Also see %SCX_OPS_ENQ_LAST.	 */	void (*dispatch)(s32 cpu, struct task_struct *prev); 	/**	 * @tick: Periodic tick	 * @p: task running currently	 *	 * This operation is called every 1/HZ seconds on CPUs which are	 * executing an SCX task. Setting a slice of 0 for @p with	 * scx_bpf_task_set_slice() will trigger an immediate dispatch cycle on	 * the CPU.	 */	void (*tick)(struct task_struct *p); 	/**	 * @runnable: A task is becoming runnable on its associated CPU	 * @p: task becoming runnable	 * @enq_flags: %SCX_ENQ_*	 *	 * This and the following three functions can be used to track a task's	 * execution state transitions. A task becomes ->runnable() on a CPU,	 * and then goes through one or more ->running() and ->stopping() pairs	 * as it runs on the CPU, and eventually becomes ->quiescent() when it's	 * done running on the CPU.	 *	 * @p is becoming runnable on the CPU because it's	 *	 * - waking up (%SCX_ENQ_WAKEUP)	 * - being moved from another CPU	 * - being restored after temporarily taken off the queue for an	 *   attribute change.	 *	 * This and ->enqueue() are related but not coupled. This operation	 * notifies @p's state transition and may not be followed by ->enqueue()	 * e.g. when @p is being dispatched to a remote CPU, or when @p is	 * being enqueued on a CPU experiencing a hotplug event. Likewise, a	 * task may be ->enqueue()'d without being preceded by this operation	 * e.g. after exhausting its slice.	 */	void (*runnable)(struct task_struct *p, u64 enq_flags); 	/**	 * @running: A task is starting to run on its associated CPU	 * @p: task starting to run	 *	 * Note that this callback may be called from a CPU other than the	 * one the task is going to run on. This can happen when a task	 * property is changed (i.e., affinity), since set_next_task_scx(),	 * which triggers this callback, may run on a CPU different from	 * the task's assigned CPU.	 *	 * Therefore, always use scx_bpf_task_cpu(@p) to determine the	 * target CPU the task is going to use.	 *	 * See ->runnable() for explanation on the task state notifiers.	 */	void (*running)(struct task_struct *p); 	/**	 * @stopping: A task is stopping execution	 * @p: task stopping to run	 * @runnable: is task @p still runnable?	 *	 * Note that this callback may be called from a CPU other than the	 * one the task was running on. This can happen when a task	 * property is changed (i.e., affinity), since dequeue_task_scx(),	 * which triggers this callback, may run on a CPU different from	 * the task's assigned CPU.	 *	 * Therefore, always use scx_bpf_task_cpu(@p) to retrieve the CPU	 * the task was running on.	 *	 * See ->runnable() for explanation on the task state notifiers. If	 * !@runnable, ->quiescent() will be invoked after this operation	 * returns.	 */	void (*stopping)(struct task_struct *p, bool runnable); 	/**	 * @quiescent: A task is becoming not runnable on its associated CPU	 * @p: task becoming not runnable	 * @deq_flags: %SCX_DEQ_*	 *	 * See ->runnable() for explanation on the task state notifiers.	 *	 * @p is becoming quiescent on the CPU because it's	 *	 * - sleeping (%SCX_DEQ_SLEEP)	 * - being moved to another CPU	 * - being temporarily taken off the queue for an attribute change	 *   (%SCX_DEQ_SCHED_CHANGE)	 *	 * This and ->dequeue() are related but not coupled. This operation	 * notifies @p's state transition and may not be preceded by ->dequeue()	 * e.g. when @p is being dispatched to a remote CPU.	 */	void (*quiescent)(struct task_struct *p, u64 deq_flags); 	/**	 * @yield: Yield CPU	 * @from: yielding task	 * @to: optional yield target task	 *

Lines 1–500 of 2486. Load more