/* SPDX-License-Identifier: GPL-2.0 *//* * Scheduler internal types and methods: */#ifndef _KERNEL_SCHED_SCHED_H#define _KERNEL_SCHED_SCHED_H #include <linux/prandom.h>#include <linux/sched/affinity.h>#include <linux/sched/autogroup.h>#include <linux/sched/cpufreq.h>#include <linux/sched/deadline.h>#include <linux/sched.h>#include <linux/sched/loadavg.h>#include <linux/sched/mm.h>#include <linux/sched/rseq_api.h>#include <linux/sched/signal.h>#include <linux/sched/smt.h>#include <linux/sched/stat.h>#include <linux/sched/sysctl.h>#include <linux/sched/task_flags.h>#include <linux/sched/task.h>#include <linux/sched/topology.h>#include <linux/atomic.h>#include <linux/bitmap.h>#include <linux/bug.h>#include <linux/capability.h>#include <linux/cgroup_api.h>#include <linux/cgroup.h>#include <linux/context_tracking.h>#include <linux/cpufreq.h>#include <linux/cpumask_api.h>#include <linux/cpuset.h>#include <linux/ctype.h>#include <linux/file.h>#include <linux/fs_api.h>#include <linux/hrtimer_api.h>#include <linux/interrupt.h>#include <linux/irq_work.h>#include <linux/jiffies.h>#include <linux/kref_api.h>#include <linux/kthread.h>#include <linux/ktime_api.h>#include <linux/lockdep_api.h>#include <linux/lockdep.h>#include <linux/memblock.h>#include <linux/memcontrol.h>#include <linux/minmax.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/mutex_api.h>#include <linux/plist.h>#include <linux/poll.h>#include <linux/proc_fs.h>#include <linux/profile.h>#include <linux/psi.h>#include <linux/rcupdate.h>#include <linux/seq_file.h>#include <linux/seqlock.h>#include <linux/softirq.h>#include <linux/spinlock_api.h>#include <linux/static_key.h>#include <linux/stop_machine.h>#include <linux/syscalls_api.h>#include <linux/syscalls.h>#include <linux/tick.h>#include <linux/topology.h>#include <linux/types.h>#include <linux/u64_stats_sync_api.h>#include <linux/uaccess.h>#include <linux/vmstat.h>#include <linux/wait_api.h>#include <linux/wait_bit.h>#include <linux/workqueue_api.h>#include <linux/delayacct.h>#include <linux/mmu_context.h> #include <trace/events/power.h>#include <trace/events/sched.h> #include "../workqueue_internal.h" struct rq;struct cfs_rq;struct rt_rq;struct sched_group;struct cpuidle_state; #if defined(CONFIG_PARAVIRT) && !defined(CONFIG_HAVE_PV_STEAL_CLOCK_GEN)# include <asm/paravirt.h>#endif #include <asm/barrier.h> #include "cpupri.h"#include "cpudeadline.h" /* task_struct::on_rq states: */#define TASK_ON_RQ_QUEUED 1#define TASK_ON_RQ_MIGRATING 2 extern __read_mostly int scheduler_running; extern unsigned long calc_load_update;extern atomic_long_t calc_load_tasks; extern void calc_global_load_tick(struct rq *this_rq);extern long calc_load_fold_active(struct rq *this_rq, long adjust); extern void call_trace_sched_update_nr_running(struct rq *rq, int count); extern int sysctl_sched_rt_period;extern int sysctl_sched_rt_runtime;extern int sched_rr_timeslice; /* * Asymmetric CPU capacity bits */struct asym_cap_data { struct list_head link; struct rcu_head rcu; unsigned long capacity; unsigned long cpus[];}; extern struct list_head asym_cap_list; #define cpu_capacity_span(asym_data) to_cpumask((asym_data)->cpus) /* * Helpers for converting nanosecond timing to jiffy resolution */#define NS_TO_JIFFIES(time) ((unsigned long)(time) / (NSEC_PER_SEC/HZ)) /* * Increase resolution of nice-level calculations for 64-bit architectures. * The extra resolution improves shares distribution and load balancing of * low-weight task groups (eg. nice +19 on an autogroup), deeper task-group * hierarchies, especially on larger systems. This is not a user-visible change * and does not change the user-interface for setting shares/weights. * * We increase resolution only if we have enough bits to allow this increased * resolution (i.e. 64-bit). The costs for increasing resolution when 32-bit * are pretty high and the returns do not justify the increased costs. * * Really only required when CONFIG_FAIR_GROUP_SCHED=y is also set, but to * increase coverage and consistency always enable it on 64-bit platforms. */#ifdef CONFIG_64BIT# define NICE_0_LOAD_SHIFT (SCHED_FIXEDPOINT_SHIFT + SCHED_FIXEDPOINT_SHIFT)# define scale_load(w) ((w) << SCHED_FIXEDPOINT_SHIFT)# define scale_load_down(w) \({ \ unsigned long __w = (w); \ \ if (__w) \ __w = max(2UL, __w >> SCHED_FIXEDPOINT_SHIFT); \ __w; \})#else# define NICE_0_LOAD_SHIFT (SCHED_FIXEDPOINT_SHIFT)# define scale_load(w) (w)# define scale_load_down(w) (w)#endif /* * Task weight (visible to users) and its load (invisible to users) have * independent resolution, but they should be well calibrated. We use * scale_load() and scale_load_down(w) to convert between them. The * following must be true: * * scale_load(sched_prio_to_weight[NICE_TO_PRIO(0)-MAX_RT_PRIO]) == NICE_0_LOAD * */#define NICE_0_LOAD (1L << NICE_0_LOAD_SHIFT) /* * Single value that decides SCHED_DEADLINE internal math precision. * 10 -> just above 1us * 9 -> just above 0.5us */#define DL_SCALE 10 /* * Single value that denotes runtime == period, ie unlimited time. */#define RUNTIME_INF ((u64)~0ULL) static inline int idle_policy(int policy){ return policy == SCHED_IDLE;} static inline int normal_policy(int policy){#ifdef CONFIG_SCHED_CLASS_EXT if (policy == SCHED_EXT) return true;#endif return policy == SCHED_NORMAL;} static inline int fair_policy(int policy){ return normal_policy(policy) || policy == SCHED_BATCH;} static inline int rt_policy(int policy){ return policy == SCHED_FIFO || policy == SCHED_RR;} static inline int dl_policy(int policy){ return policy == SCHED_DEADLINE;} static inline bool valid_policy(int policy){ return idle_policy(policy) || fair_policy(policy) || rt_policy(policy) || dl_policy(policy);} static inline int task_has_idle_policy(struct task_struct *p){ return idle_policy(p->policy);} static inline int task_has_rt_policy(struct task_struct *p){ return rt_policy(p->policy);} static inline int task_has_dl_policy(struct task_struct *p){ return dl_policy(p->policy);} #define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT) static inline void update_avg(u64 *avg, u64 sample){ s64 diff = sample - *avg; *avg += diff / 8;} /* * Shifting a value by an exponent greater *or equal* to the size of said value * is UB; cap at size-1. */#define shr_bound(val, shift) \ (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) /* * cgroup weight knobs should use the common MIN, DFL and MAX values which are * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it * maps pretty well onto the shares value used by scheduler and the round-trip * conversions preserve the original value over the entire range. */static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight){ return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL);} static inline unsigned long sched_weight_to_cgroup(unsigned long weight){ return clamp_t(unsigned long, DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX);} /* * !! For sched_setattr_nocheck() (kernel) only !! * * This is actually gross. :( * * It is used to make schedutil kworker(s) higher priority than SCHED_DEADLINE * tasks, but still be able to sleep. We need this on platforms that cannot * atomically change clock frequency. Remove once fast switching will be * available on such platforms. * * SUGOV stands for SchedUtil GOVernor. */#define SCHED_FLAG_SUGOV 0x10000000 #define SCHED_DL_FLAGS (SCHED_FLAG_RECLAIM | SCHED_FLAG_DL_OVERRUN | SCHED_FLAG_SUGOV) static inline bool dl_entity_is_special(const struct sched_dl_entity *dl_se){#ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL return unlikely(dl_se->flags & SCHED_FLAG_SUGOV);#else return false;#endif} /* * Tells if entity @a should preempt entity @b. */static inline bool dl_entity_preempt(const struct sched_dl_entity *a, const struct sched_dl_entity *b){ return dl_entity_is_special(a) || dl_time_before(a->deadline, b->deadline);} /* * This is the priority-queue data structure of the RT scheduling class: */struct rt_prio_array { DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */ struct list_head queue[MAX_RT_PRIO];}; struct rt_bandwidth { /* nests inside the rq lock: */ raw_spinlock_t rt_runtime_lock; ktime_t rt_period; u64 rt_runtime; struct hrtimer rt_period_timer; unsigned int rt_period_active;}; static inline int dl_bandwidth_enabled(void){ return sysctl_sched_rt_runtime >= 0;} /* * To keep the bandwidth of -deadline tasks under control * we need some place where: * - store the maximum -deadline bandwidth of each cpu; * - cache the fraction of bandwidth that is currently allocated in * each root domain; * * This is all done in the data structure below. It is similar to the * one used for RT-throttling (rt_bandwidth), with the main difference * that, since here we are only interested in admission control, we * do not decrease any runtime while the group "executes", neither we * need a timer to replenish it. * * With respect to SMP, bandwidth is given on a per root domain basis, * meaning that: * - bw (< 100%) is the deadline bandwidth of each CPU; * - total_bw is the currently allocated bandwidth in each root domain; */struct dl_bw { raw_spinlock_t lock; u64 bw; u64 total_bw;}; extern void init_dl_bw(struct dl_bw *dl_b);extern int sched_dl_global_validate(void);extern void sched_dl_do_global(void);extern int sched_dl_overflow(struct task_struct *p, int policy, const struct sched_attr *attr);extern void __setparam_dl(struct task_struct *p, const struct sched_attr *attr);extern void __getparam_dl(struct task_struct *p, struct sched_attr *attr, unsigned int flags);extern bool __checkparam_dl(const struct sched_attr *attr);extern bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr);extern int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);extern int dl_bw_deactivate(int cpu);extern s64 dl_scaled_delta_exec(struct rq *rq, struct sched_dl_entity *dl_se, s64 delta_exec);/* * SCHED_DEADLINE supports servers (nested scheduling) with the following * interface: * * dl_se::rq -- runqueue we belong to. * * dl_se::server_pick() -- nested pick_next_task(); we yield the period if this * returns NULL. * * dl_server_update() -- called from update_curr_common(), propagates runtime * to the server. * * dl_server_start() -- start the server when it has tasks; it will stop * automatically when there are no more tasks, per * dl_se::server_pick() returning NULL. * * dl_server_stop() -- (force) stop the server; use when updating * parameters. * * dl_server_init() -- initializes the server. * * When started the dl_server will (per dl_defer) schedule a timer for its * zero-laxity point -- that is, unlike regular EDF tasks which run ASAP, a * server will run at the very end of its period. * * This is done such that any runtime from the target class can be accounted * against the server -- through dl_server_update() above -- such that when it * becomes time to run, it might already be out of runtime and get deferred * until the next period. In this case dl_server_timer() will alternate * between defer and replenish but never actually enqueue the server. * * Only when the target class does not manage to exhaust the server's runtime * (there's actualy starvation in the given period), will the dl_server get on * the runqueue. Once queued it will pick tasks from the target class and run * them until either its runtime is exhaused, at which point its back to * dl_server_timer, or until there are no more tasks to run, at which point * the dl_server stops itself. * * By stopping at this point the dl_server retains bandwidth, which, if a new * task wakes up imminently (starting the server again), can be used -- * subject to CBS wakeup rules -- without having to wait for the next period. * * Additionally, because of the dl_defer behaviour the start/stop behaviour is * naturally thottled to once per period, avoiding high context switch * workloads from spamming the hrtimer program/cancel paths. */extern void dl_server_update_idle(struct sched_dl_entity *dl_se, s64 delta_exec);extern void dl_server_update(struct sched_dl_entity *dl_se, s64 delta_exec);extern void dl_server_start(struct sched_dl_entity *dl_se);extern void dl_server_stop(struct sched_dl_entity *dl_se);extern void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq, dl_server_pick_f pick_task);extern void sched_init_dl_servers(void); extern void fair_server_init(struct rq *rq);extern void ext_server_init(struct rq *rq);extern void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq);extern int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 period, bool init);extern int dl_server_attach_bw(struct sched_dl_entity *dl_se);extern void dl_server_detach_bw(struct sched_dl_entity *dl_se);extern int dl_server_swap_bw(struct sched_dl_entity *detach_se, struct sched_dl_entity *attach_se); static inline bool dl_server_active(struct sched_dl_entity *dl_se){ return dl_se->dl_server_active;} #ifdef CONFIG_CGROUP_SCHED extern struct list_head task_groups; #ifdef CONFIG_GROUP_SCHED_BANDWIDTHextern const u64 max_bw_quota_period_us; /* * default period for group bandwidth. * default: 0.1s, units: microseconds */static inline u64 default_bw_period_us(void){ return 100000ULL;}#endif /* CONFIG_GROUP_SCHED_BANDWIDTH */ struct cfs_bandwidth {#ifdef CONFIG_CFS_BANDWIDTH raw_spinlock_t lock; ktime_t period; u64 quota; u64 runtime; u64 burst; u64 runtime_snap; s64 hierarchical_quota; u8 idle; u8 period_active; u8 slack_started; struct hrtimer period_timer; struct hrtimer slack_timer; struct list_head throttled_cfs_rq; /* Statistics: */ int nr_periods; int nr_throttled; int nr_burst; u64 throttled_time; u64 burst_time;#endif /* CONFIG_CFS_BANDWIDTH */}; /* Task group related information */struct task_group { struct cgroup_subsys_state css; #ifdef CONFIG_GROUP_SCHED_WEIGHT /* A positive value indicates that this is a SCHED_IDLE group. */ int idle;#endif #ifdef CONFIG_FAIR_GROUP_SCHED /* runqueue "owned" by this group on each CPU */ struct cfs_rq __percpu *cfs_rq; unsigned long shares; /* * load_avg can be heavily contended at clock tick time, so put * it in its own cache-line separated from the fields above which * will also be accessed at each tick. */ atomic_long_t load_avg ____cacheline_aligned; atomic_long_t runnable_avg; #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_RT_GROUP_SCHEDLines 1–500 of 4255. Load more
/* SPDX-License-Identifier: GPL-2.0 */ /* * Scheduler internal types and methods: */ #ifndef _KERNEL_SCHED_SCHED_H #define _KERNEL_SCHED_SCHED_H #include <linux/prandom.h> #include <linux/sched/affinity.h> #include <linux/sched/autogroup.h> #include <linux/sched/cpufreq.h> #include <linux/sched/deadline.h> #include <linux/sched.h> #include <linux/sched/loadavg.h> #include <linux/sched/mm.h> #include <linux/sched/rseq_api.h> #include <linux/sched/signal.h> #include <linux/sched/smt.h> #include <linux/sched/stat.h> #include <linux/sched/sysctl.h> #include <linux/sched/task_flags.h> #include <linux/sched/task.h> #include <linux/sched/topology.h> #include <linux/atomic.h> #include <linux/bitmap.h> #include <linux/bug.h> #include <linux/capability.h> #include <linux/cgroup_api.h> #include <linux/cgroup.h> #include <linux/context_tracking.h> #include <linux/cpufreq.h> #include <linux/cpumask_api.h> #include <linux/cpuset.h> #include <linux/ctype.h> #include <linux/file.h> #include <linux/fs_api.h> #include <linux/hrtimer_api.h> #include <linux/interrupt.h> #include <linux/irq_work.h> #include <linux/jiffies.h> #include <linux/kref_api.h> #include <linux/kthread.h> #include <linux/ktime_api.h> #include <linux/lockdep_api.h> #include <linux/lockdep.h> #include <linux/memblock.h> #include <linux/memcontrol.h> #include <linux/minmax.h> #include <linux/mm.h> #include <linux/module.h> #include <linux/mutex_api.h> #include <linux/plist.h> #include <linux/poll.h> #include <linux/proc_fs.h> #include <linux/profile.h> #include <linux/psi.h> #include <linux/rcupdate.h> #include <linux/seq_file.h> #include <linux/seqlock.h> #include <linux/softirq.h> #include <linux/spinlock_api.h> #include <linux/static_key.h> #include <linux/stop_machine.h> #include <linux/syscalls_api.h> #include <linux/syscalls.h> #include <linux/tick.h> #include <linux/topology.h> #include <linux/types.h> #include <linux/u64_stats_sync_api.h> #include <linux/uaccess.h> #include <linux/vmstat.h> #include <linux/wait_api.h> #include <linux/wait_bit.h> #include <linux/workqueue_api.h> #include <linux/delayacct.h> #include <linux/mmu_context.h> #include <trace/events/power.h> #include <trace/events/sched.h> #include "../workqueue_internal.h" struct rq; struct cfs_rq; struct rt_rq; struct sched_group; struct cpuidle_state; #if defined(CONFIG_PARAVIRT) && !defined(CONFIG_HAVE_PV_STEAL_CLOCK_GEN) # include <asm/paravirt.h> #endif #include <asm/barrier.h> #include "cpupri.h" #include "cpudeadline.h" /* task_struct::on_rq states: */ #define TASK_ON_RQ_QUEUED 1 #define TASK_ON_RQ_MIGRATING 2 extern __read_mostly int scheduler_running; extern unsigned long calc_load_update; extern atomic_long_t calc_load_tasks; extern void calc_global_load_tick(struct rq *this_rq); extern long calc_load_fold_active(struct rq *this_rq, long adjust); extern void call_trace_sched_update_nr_running(struct rq *rq, int count); extern int sysctl_sched_rt_period; extern int sysctl_sched_rt_runtime; extern int sched_rr_timeslice; /* * Asymmetric CPU capacity bits */ struct asym_cap_data { struct list_head link; struct rcu_head rcu; unsigned long capacity; unsigned long cpus[]; }; extern struct list_head asym_cap_list; #define cpu_capacity_span(asym_data) to_cpumask((asym_data)->cpus) /* * Helpers for converting nanosecond timing to jiffy resolution */ #define NS_TO_JIFFIES(time) ((unsigned long)(time) / (NSEC_PER_SEC/HZ)) /* * Increase resolution of nice-level calculations for 64-bit architectures. * The extra resolution improves shares distribution and load balancing of * low-weight task groups (eg. nice +19 on an autogroup), deeper task-group * hierarchies, especially on larger systems. This is not a user-visible change * and does not change the user-interface for setting shares/weights. * * We increase resolution only if we have enough bits to allow this increased * resolution (i.e. 64-bit). The costs for increasing resolution when 32-bit * are pretty high and the returns do not justify the increased costs. * * Really only required when CONFIG_FAIR_GROUP_SCHED=y is also set, but to * increase coverage and consistency always enable it on 64-bit platforms. */ #ifdef CONFIG_64BIT # define NICE_0_LOAD_SHIFT (SCHED_FIXEDPOINT_SHIFT + SCHED_FIXEDPOINT_SHIFT) # define scale_load(w) ((w) << SCHED_FIXEDPOINT_SHIFT) # define scale_load_down(w) \ ({ \ unsigned long __w = (w); \ \ if (__w) \ __w = max(2UL, __w >> SCHED_FIXEDPOINT_SHIFT); \ __w; \ }) #else # define NICE_0_LOAD_SHIFT (SCHED_FIXEDPOINT_SHIFT) # define scale_load(w) (w) # define scale_load_down(w) (w) #endif /* * Task weight (visible to users) and its load (invisible to users) have * independent resolution, but they should be well calibrated. We use * scale_load() and scale_load_down(w) to convert between them. The * following must be true: * * scale_load(sched_prio_to_weight[NICE_TO_PRIO(0)-MAX_RT_PRIO]) == NICE_0_LOAD * */ #define NICE_0_LOAD (1L << NICE_0_LOAD_SHIFT) /* * Single value that decides SCHED_DEADLINE internal math precision. * 10 -> just above 1us * 9 -> just above 0.5us */ #define DL_SCALE 10 /* * Single value that denotes runtime == period, ie unlimited time. */ #define RUNTIME_INF ((u64)~0ULL) static inline int idle_policy(int policy) { return policy == SCHED_IDLE; } static inline int normal_policy(int policy) { #ifdef CONFIG_SCHED_CLASS_EXT if (policy == SCHED_EXT) return true; #endif return policy == SCHED_NORMAL; } static inline int fair_policy(int policy) { return normal_policy(policy) || policy == SCHED_BATCH; } static inline int rt_policy(int policy) { return policy == SCHED_FIFO || policy == SCHED_RR; } static inline int dl_policy(int policy) { return policy == SCHED_DEADLINE; } static inline bool valid_policy(int policy) { return idle_policy(policy) || fair_policy(policy) || rt_policy(policy) || dl_policy(policy); } static inline int task_has_idle_policy(struct task_struct *p) { return idle_policy(p->policy); } static inline int task_has_rt_policy(struct task_struct *p) { return rt_policy(p->policy); } static inline int task_has_dl_policy(struct task_struct *p) { return dl_policy(p->policy); } #define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT) static inline void update_avg(u64 *avg, u64 sample) { s64 diff = sample - *avg; *avg += diff / 8; } /* * Shifting a value by an exponent greater *or equal* to the size of said value * is UB; cap at size-1. */ #define shr_bound(val, shift) \ (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1)) /* * cgroup weight knobs should use the common MIN, DFL and MAX values which are * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it * maps pretty well onto the shares value used by scheduler and the round-trip * conversions preserve the original value over the entire range. */ static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight) { return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL); } static inline unsigned long sched_weight_to_cgroup(unsigned long weight) { return clamp_t(unsigned long, DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024), CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX); } /* * !! For sched_setattr_nocheck() (kernel) only !! * * This is actually gross. :( * * It is used to make schedutil kworker(s) higher priority than SCHED_DEADLINE * tasks, but still be able to sleep. We need this on platforms that cannot * atomically change clock frequency. Remove once fast switching will be * available on such platforms. * * SUGOV stands for SchedUtil GOVernor. */ #define SCHED_FLAG_SUGOV 0x10000000 #define SCHED_DL_FLAGS (SCHED_FLAG_RECLAIM | SCHED_FLAG_DL_OVERRUN | SCHED_FLAG_SUGOV) static inline bool dl_entity_is_special(const struct sched_dl_entity *dl_se) { #ifdef CONFIG_CPU_FREQ_GOV_SCHEDUTIL return unlikely(dl_se->flags & SCHED_FLAG_SUGOV); #else return false; #endif } /* * Tells if entity @a should preempt entity @b. */ static inline bool dl_entity_preempt(const struct sched_dl_entity *a, const struct sched_dl_entity *b) { return dl_entity_is_special(a) || dl_time_before(a->deadline, b->deadline); } /* * This is the priority-queue data structure of the RT scheduling class: */ struct rt_prio_array { DECLARE_BITMAP(bitmap, MAX_RT_PRIO+1); /* include 1 bit for delimiter */ struct list_head queue[MAX_RT_PRIO]; }; struct rt_bandwidth { /* nests inside the rq lock: */ raw_spinlock_t rt_runtime_lock; ktime_t rt_period; u64 rt_runtime; struct hrtimer rt_period_timer; unsigned int rt_period_active; }; static inline int dl_bandwidth_enabled(void) { return sysctl_sched_rt_runtime >= 0; } /* * To keep the bandwidth of -deadline tasks under control * we need some place where: * - store the maximum -deadline bandwidth of each cpu; * - cache the fraction of bandwidth that is currently allocated in * each root domain; * * This is all done in the data structure below. It is similar to the * one used for RT-throttling (rt_bandwidth), with the main difference * that, since here we are only interested in admission control, we * do not decrease any runtime while the group "executes", neither we * need a timer to replenish it. * * With respect to SMP, bandwidth is given on a per root domain basis, * meaning that: * - bw (< 100%) is the deadline bandwidth of each CPU; * - total_bw is the currently allocated bandwidth in each root domain; */ struct dl_bw { raw_spinlock_t lock; u64 bw; u64 total_bw; }; extern void init_dl_bw(struct dl_bw *dl_b); extern int sched_dl_global_validate(void); extern void sched_dl_do_global(void); extern int sched_dl_overflow(struct task_struct *p, int policy, const struct sched_attr *attr); extern void __setparam_dl(struct task_struct *p, const struct sched_attr *attr); extern void __getparam_dl(struct task_struct *p, struct sched_attr *attr, unsigned int flags); extern bool __checkparam_dl(const struct sched_attr *attr); extern bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr); extern int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial); extern int dl_bw_deactivate(int cpu); extern s64 dl_scaled_delta_exec(struct rq *rq, struct sched_dl_entity *dl_se, s64 delta_exec); /* * SCHED_DEADLINE supports servers (nested scheduling) with the following * interface: * * dl_se::rq -- runqueue we belong to. * * dl_se::server_pick() -- nested pick_next_task(); we yield the period if this * returns NULL. * * dl_server_update() -- called from update_curr_common(), propagates runtime * to the server. * * dl_server_start() -- start the server when it has tasks; it will stop * automatically when there are no more tasks, per * dl_se::server_pick() returning NULL. * * dl_server_stop() -- (force) stop the server; use when updating * parameters. * * dl_server_init() -- initializes the server. * * When started the dl_server will (per dl_defer) schedule a timer for its * zero-laxity point -- that is, unlike regular EDF tasks which run ASAP, a * server will run at the very end of its period. * * This is done such that any runtime from the target class can be accounted * against the server -- through dl_server_update() above -- such that when it * becomes time to run, it might already be out of runtime and get deferred * until the next period. In this case dl_server_timer() will alternate * between defer and replenish but never actually enqueue the server. * * Only when the target class does not manage to exhaust the server's runtime * (there's actualy starvation in the given period), will the dl_server get on * the runqueue. Once queued it will pick tasks from the target class and run * them until either its runtime is exhaused, at which point its back to * dl_server_timer, or until there are no more tasks to run, at which point * the dl_server stops itself. * * By stopping at this point the dl_server retains bandwidth, which, if a new * task wakes up imminently (starting the server again), can be used -- * subject to CBS wakeup rules -- without having to wait for the next period. * * Additionally, because of the dl_defer behaviour the start/stop behaviour is * naturally thottled to once per period, avoiding high context switch * workloads from spamming the hrtimer program/cancel paths. */ extern void dl_server_update_idle(struct sched_dl_entity *dl_se, s64 delta_exec); extern void dl_server_update(struct sched_dl_entity *dl_se, s64 delta_exec); extern void dl_server_start(struct sched_dl_entity *dl_se); extern void dl_server_stop(struct sched_dl_entity *dl_se); extern void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq, dl_server_pick_f pick_task); extern void sched_init_dl_servers(void); extern void fair_server_init(struct rq *rq); extern void ext_server_init(struct rq *rq); extern void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq); extern int dl_server_apply_params(struct sched_dl_entity *dl_se, u64 runtime, u64 period, bool init); extern int dl_server_attach_bw(struct sched_dl_entity *dl_se); extern void dl_server_detach_bw(struct sched_dl_entity *dl_se); extern int dl_server_swap_bw(struct sched_dl_entity *detach_se, struct sched_dl_entity *attach_se); static inline bool dl_server_active(struct sched_dl_entity *dl_se) { return dl_se->dl_server_active; } #ifdef CONFIG_CGROUP_SCHED extern struct list_head task_groups; #ifdef CONFIG_GROUP_SCHED_BANDWIDTH extern const u64 max_bw_quota_period_us; /* * default period for group bandwidth. * default: 0.1s, units: microseconds */ static inline u64 default_bw_period_us(void) { return 100000ULL; } #endif /* CONFIG_GROUP_SCHED_BANDWIDTH */ struct cfs_bandwidth { #ifdef CONFIG_CFS_BANDWIDTH raw_spinlock_t lock; ktime_t period; u64 quota; u64 runtime; u64 burst; u64 runtime_snap; s64 hierarchical_quota; u8 idle; u8 period_active; u8 slack_started; struct hrtimer period_timer; struct hrtimer slack_timer; struct list_head throttled_cfs_rq; /* Statistics: */ int nr_periods; int nr_throttled; int nr_burst; u64 throttled_time; u64 burst_time; #endif /* CONFIG_CFS_BANDWIDTH */ }; /* Task group related information */ struct task_group { struct cgroup_subsys_state css; #ifdef CONFIG_GROUP_SCHED_WEIGHT /* A positive value indicates that this is a SCHED_IDLE group. */ int idle; #endif #ifdef CONFIG_FAIR_GROUP_SCHED /* runqueue "owned" by this group on each CPU */ struct cfs_rq __percpu *cfs_rq; unsigned long shares; /* * load_avg can be heavily contended at clock tick time, so put * it in its own cache-line separated from the fields above which * will also be accessed at each tick. */ atomic_long_t load_avg ____cacheline_aligned; atomic_long_t runnable_avg; #endif /* CONFIG_FAIR_GROUP_SCHED */ #ifdef CONFIG_RT_GROUP_SCHED