// SPDX-License-Identifier: GPL-2.0/* * Deadline Scheduling Class (SCHED_DEADLINE) * * Earliest Deadline First (EDF) + Constant Bandwidth Server (CBS). * * Tasks that periodically executes their instances for less than their * runtime won't miss any of their deadlines. * Tasks that are not periodic or sporadic or that tries to execute more * than their reserved bandwidth will be slowed down (and may potentially * miss some of their deadlines), and won't affect any other task. * * Copyright (C) 2012 Dario Faggioli <raistlin@linux.it>, * Juri Lelli <juri.lelli@gmail.com>, * Michael Trimarchi <michael@amarulasolutions.com>, * Fabio Checconi <fchecconi@gmail.com> */ #include <linux/cpuset.h>#include <linux/sched/clock.h>#include <linux/sched/deadline.h>#include <uapi/linux/sched/types.h>#include "sched.h"#include "pelt.h" /* * Default limits for DL period; on the top end we guard against small util * tasks still getting ridiculously long effective runtimes, on the bottom end we * guard against timer DoS. */static unsigned int sysctl_sched_dl_period_max = 1 << 22; /* ~4 seconds */static unsigned int sysctl_sched_dl_period_min = 100; /* 100 us */#ifdef CONFIG_SYSCTLstatic const struct ctl_table sched_dl_sysctls[] = { { .procname = "sched_deadline_period_max_us", .data = &sysctl_sched_dl_period_max, .maxlen = sizeof(unsigned int), .mode = 0644, .proc_handler = proc_douintvec_minmax, .extra1 = (void *)&sysctl_sched_dl_period_min, }, { .procname = "sched_deadline_period_min_us", .data = &sysctl_sched_dl_period_min, .maxlen = sizeof(unsigned int), .mode = 0644, .proc_handler = proc_douintvec_minmax, .extra2 = (void *)&sysctl_sched_dl_period_max, },}; static int __init sched_dl_sysctl_init(void){ register_sysctl_init("kernel", sched_dl_sysctls); return 0;}late_initcall(sched_dl_sysctl_init);#endif /* CONFIG_SYSCTL */ static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq){ return container_of(dl_rq, struct rq, dl);} static inline struct rq *rq_of_dl_se(struct sched_dl_entity *dl_se){ struct rq *rq = dl_se->rq; if (!dl_server(dl_se)) rq = task_rq(dl_task_of(dl_se)); return rq;} static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se){ return &rq_of_dl_se(dl_se)->dl;} static inline int on_dl_rq(struct sched_dl_entity *dl_se){ return !RB_EMPTY_NODE(&dl_se->rb_node);} #ifdef CONFIG_RT_MUTEXESstatic inline struct sched_dl_entity *pi_of(struct sched_dl_entity *dl_se){ return dl_se->pi_se;} static inline bool is_dl_boosted(struct sched_dl_entity *dl_se){ return pi_of(dl_se) != dl_se;}#else /* !CONFIG_RT_MUTEXES: */static inline struct sched_dl_entity *pi_of(struct sched_dl_entity *dl_se){ return dl_se;} static inline bool is_dl_boosted(struct sched_dl_entity *dl_se){ return false;}#endif /* !CONFIG_RT_MUTEXES */ static inline u8 dl_get_type(struct sched_dl_entity *dl_se, struct rq *rq){ if (!dl_server(dl_se)) return DL_TASK; if (dl_se == &rq->fair_server) return DL_SERVER_FAIR;#ifdef CONFIG_SCHED_CLASS_EXT if (dl_se == &rq->ext_server) return DL_SERVER_EXT;#endif return DL_OTHER;} static inline struct dl_bw *dl_bw_of(int i){ RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(), "sched RCU must be held"); return &cpu_rq(i)->rd->dl_bw;} static inline int dl_bw_cpus(int i){ struct root_domain *rd = cpu_rq(i)->rd; RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(), "sched RCU must be held"); return cpumask_weight_and(rd->span, cpu_active_mask);} static inline unsigned long __dl_bw_capacity(const struct cpumask *mask){ unsigned long cap = 0; int i; for_each_cpu_and(i, mask, cpu_active_mask) cap += arch_scale_cpu_capacity(i); return cap;} /* * XXX Fix: If 'rq->rd == def_root_domain' perform AC against capacity * of the CPU the task is running on rather rd's \Sum CPU capacity. */static inline unsigned long dl_bw_capacity(int i){ if (!sched_asym_cpucap_active() && arch_scale_cpu_capacity(i) == SCHED_CAPACITY_SCALE) { return dl_bw_cpus(i) << SCHED_CAPACITY_SHIFT; } else { RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(), "sched RCU must be held"); return __dl_bw_capacity(cpu_rq(i)->rd->span); }} bool dl_bw_visited(int cpu, u64 cookie){ struct root_domain *rd = cpu_rq(cpu)->rd; if (rd->visit_cookie == cookie) return true; rd->visit_cookie = cookie; return false;} static inlinevoid __dl_update(struct dl_bw *dl_b, s64 bw){ struct root_domain *rd = container_of(dl_b, struct root_domain, dl_bw); int i; RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(), "sched RCU must be held"); for_each_cpu_and(i, rd->span, cpu_active_mask) { struct rq *rq = cpu_rq(i); rq->dl.extra_bw += bw; }} static inlinevoid __dl_sub(struct dl_bw *dl_b, u64 tsk_bw, int cpus){ dl_b->total_bw -= tsk_bw; __dl_update(dl_b, (s32)tsk_bw / cpus);} static inlinevoid __dl_add(struct dl_bw *dl_b, u64 tsk_bw, int cpus){ dl_b->total_bw += tsk_bw; __dl_update(dl_b, -((s32)tsk_bw / cpus));} static inline bool__dl_overflow(struct dl_bw *dl_b, unsigned long cap, u64 old_bw, u64 new_bw){ return dl_b->bw != -1 && cap_scale(dl_b->bw, cap) < dl_b->total_bw - old_bw + new_bw;} static inlinevoid __add_running_bw(u64 dl_bw, struct dl_rq *dl_rq){ u64 old = dl_rq->running_bw; lockdep_assert_rq_held(rq_of_dl_rq(dl_rq)); dl_rq->running_bw += dl_bw; WARN_ON_ONCE(dl_rq->running_bw < old); /* overflow */ WARN_ON_ONCE(dl_rq->running_bw > dl_rq->this_bw); /* kick cpufreq (see the comment in kernel/sched/sched.h). */ cpufreq_update_util(rq_of_dl_rq(dl_rq), 0);} static inlinevoid __sub_running_bw(u64 dl_bw, struct dl_rq *dl_rq){ u64 old = dl_rq->running_bw; lockdep_assert_rq_held(rq_of_dl_rq(dl_rq)); dl_rq->running_bw -= dl_bw; WARN_ON_ONCE(dl_rq->running_bw > old); /* underflow */ if (dl_rq->running_bw > old) dl_rq->running_bw = 0; /* kick cpufreq (see the comment in kernel/sched/sched.h). */ cpufreq_update_util(rq_of_dl_rq(dl_rq), 0);} static inlinevoid __add_rq_bw(u64 dl_bw, struct dl_rq *dl_rq){ u64 old = dl_rq->this_bw; lockdep_assert_rq_held(rq_of_dl_rq(dl_rq)); dl_rq->this_bw += dl_bw; WARN_ON_ONCE(dl_rq->this_bw < old); /* overflow */} static inlinevoid __sub_rq_bw(u64 dl_bw, struct dl_rq *dl_rq){ u64 old = dl_rq->this_bw; lockdep_assert_rq_held(rq_of_dl_rq(dl_rq)); dl_rq->this_bw -= dl_bw; WARN_ON_ONCE(dl_rq->this_bw > old); /* underflow */ if (dl_rq->this_bw > old) dl_rq->this_bw = 0; WARN_ON_ONCE(dl_rq->running_bw > dl_rq->this_bw);} static inlinevoid add_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq){ if (!dl_entity_is_special(dl_se)) __add_rq_bw(dl_se->dl_bw, dl_rq);} static inlinevoid sub_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq){ if (!dl_entity_is_special(dl_se)) __sub_rq_bw(dl_se->dl_bw, dl_rq);} static inlinevoid add_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq){ if (!dl_entity_is_special(dl_se)) __add_running_bw(dl_se->dl_bw, dl_rq);} static inlinevoid sub_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq){ if (!dl_entity_is_special(dl_se)) __sub_running_bw(dl_se->dl_bw, dl_rq);} static void dl_rq_change_utilization(struct rq *rq, struct sched_dl_entity *dl_se, u64 new_bw){ if (dl_se->dl_non_contending) { sub_running_bw(dl_se, &rq->dl); dl_se->dl_non_contending = 0; /* * If the timer handler is currently running and the * timer cannot be canceled, inactive_task_timer() * will see that dl_not_contending is not set, and * will not touch the rq's active utilization, * so we are still safe. */ if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1) { if (!dl_server(dl_se)) put_task_struct(dl_task_of(dl_se)); } } __sub_rq_bw(dl_se->dl_bw, &rq->dl); __add_rq_bw(new_bw, &rq->dl);} static __always_inlinevoid cancel_dl_timer(struct sched_dl_entity *dl_se, struct hrtimer *timer){ /* * If the timer callback was running (hrtimer_try_to_cancel == -1), * it will eventually call put_task_struct(). */ if (hrtimer_try_to_cancel(timer) == 1 && !dl_server(dl_se)) put_task_struct(dl_task_of(dl_se));} static __always_inlinevoid cancel_replenish_timer(struct sched_dl_entity *dl_se){ cancel_dl_timer(dl_se, &dl_se->dl_timer);} static __always_inlinevoid cancel_inactive_timer(struct sched_dl_entity *dl_se){ cancel_dl_timer(dl_se, &dl_se->inactive_timer);} static void dl_change_utilization(struct task_struct *p, u64 new_bw){ WARN_ON_ONCE(p->dl.flags & SCHED_FLAG_SUGOV); if (task_on_rq_queued(p)) return; dl_rq_change_utilization(task_rq(p), &p->dl, new_bw);} static void __dl_clear_params(struct sched_dl_entity *dl_se); /* * The utilization of a task cannot be immediately removed from * the rq active utilization (running_bw) when the task blocks. * Instead, we have to wait for the so called "0-lag time". * * If a task blocks before the "0-lag time", a timer (the inactive * timer) is armed, and running_bw is decreased when the timer * fires. * * If the task wakes up again before the inactive timer fires, * the timer is canceled, whereas if the task wakes up after the * inactive timer fired (and running_bw has been decreased) the * task's utilization has to be added to running_bw again. * A flag in the deadline scheduling entity (dl_non_contending) * is used to avoid race conditions between the inactive timer handler * and task wakeups. * * The following diagram shows how running_bw is updated. A task is * "ACTIVE" when its utilization contributes to running_bw; an * "ACTIVE contending" task is in the TASK_RUNNING state, while an * "ACTIVE non contending" task is a blocked task for which the "0-lag time" * has not passed yet. An "INACTIVE" task is a task for which the "0-lag" * time already passed, which does not contribute to running_bw anymore. * +------------------+ * wakeup | ACTIVE | * +------------------>+ contending | * | add_running_bw | | * | +----+------+------+ * | | ^ * | dequeue | | * +--------+-------+ | | * | | t >= 0-lag | | wakeup * | INACTIVE |<---------------+ | * | | sub_running_bw | | * +--------+-------+ | | * ^ | | * | t < 0-lag | | * | | | * | V | * | +----+------+------+ * | sub_running_bw | ACTIVE | * +-------------------+ | * inactive timer | non contending | * fired +------------------+ * * The task_non_contending() function is invoked when a task * blocks, and checks if the 0-lag time already passed or * not (in the first case, it directly updates running_bw; * in the second case, it arms the inactive timer). * * The task_contending() function is invoked when a task wakes * up, and checks if the task is still in the "ACTIVE non contending" * state or not (in the second case, it updates running_bw). */static void task_non_contending(struct sched_dl_entity *dl_se, bool dl_task){ struct hrtimer *timer = &dl_se->inactive_timer; struct rq *rq = rq_of_dl_se(dl_se); struct dl_rq *dl_rq = &rq->dl; s64 zerolag_time; /* * If this is a non-deadline task that has been boosted, * do nothing */ if (dl_se->dl_runtime == 0) return; if (dl_entity_is_special(dl_se)) return; WARN_ON(dl_se->dl_non_contending); zerolag_time = dl_se->deadline - div64_long((dl_se->runtime * dl_se->dl_period), dl_se->dl_runtime); /* * Using relative times instead of the absolute "0-lag time" * allows to simplify the code */ zerolag_time -= rq_clock(rq); /* * If the "0-lag time" already passed, decrease the active * utilization now, instead of starting a timer */ if ((zerolag_time < 0) || hrtimer_active(&dl_se->inactive_timer)) { if (dl_server(dl_se)) { sub_running_bw(dl_se, dl_rq); } else { struct task_struct *p = dl_task_of(dl_se); if (dl_task) sub_running_bw(dl_se, dl_rq); if (!dl_task || READ_ONCE(p->__state) == TASK_DEAD) { struct dl_bw *dl_b = dl_bw_of(task_cpu(p)); if (READ_ONCE(p->__state) == TASK_DEAD) sub_rq_bw(dl_se, &rq->dl); raw_spin_lock(&dl_b->lock); __dl_sub(dl_b, dl_se->dl_bw, dl_bw_cpus(task_cpu(p))); raw_spin_unlock(&dl_b->lock); __dl_clear_params(dl_se); } } return; } dl_se->dl_non_contending = 1; if (!dl_server(dl_se)) get_task_struct(dl_task_of(dl_se)); hrtimer_start(timer, ns_to_ktime(zerolag_time), HRTIMER_MODE_REL_HARD);} static void task_contending(struct sched_dl_entity *dl_se, int flags){ struct dl_rq *dl_rq = dl_rq_of_se(dl_se); /* * If this is a non-deadline task that has been boosted, * do nothing */ if (dl_se->dl_runtime == 0) return; if (flags & ENQUEUE_MIGRATED) add_rq_bw(dl_se, dl_rq); if (dl_se->dl_non_contending) { dl_se->dl_non_contending = 0; /* * If the timer handler is currently running and the * timer cannot be canceled, inactive_task_timer() * will see that dl_not_contending is not set, and * will not touch the rq's active utilization, * so we are still safe. */ cancel_inactive_timer(dl_se); } else { /* * Since "dl_non_contending" is not set, the * task's utilization has already been removed from * active utilization (either when the task blocked, * when the "inactive timer" fired). * So, add it back. */ add_running_bw(dl_se, dl_rq); }}Lines 1–500 of 4098. Load more
// SPDX-License-Identifier: GPL-2.0 /* * Deadline Scheduling Class (SCHED_DEADLINE) * * Earliest Deadline First (EDF) + Constant Bandwidth Server (CBS). * * Tasks that periodically executes their instances for less than their * runtime won't miss any of their deadlines. * Tasks that are not periodic or sporadic or that tries to execute more * than their reserved bandwidth will be slowed down (and may potentially * miss some of their deadlines), and won't affect any other task. * * Copyright (C) 2012 Dario Faggioli <raistlin@linux.it>, * Juri Lelli <juri.lelli@gmail.com>, * Michael Trimarchi <michael@amarulasolutions.com>, * Fabio Checconi <fchecconi@gmail.com> */ #include <linux/cpuset.h> #include <linux/sched/clock.h> #include <linux/sched/deadline.h> #include <uapi/linux/sched/types.h> #include "sched.h" #include "pelt.h" /* * Default limits for DL period; on the top end we guard against small util * tasks still getting ridiculously long effective runtimes, on the bottom end we * guard against timer DoS. */ static unsigned int sysctl_sched_dl_period_max = 1 << 22; /* ~4 seconds */ static unsigned int sysctl_sched_dl_period_min = 100; /* 100 us */ #ifdef CONFIG_SYSCTL static const struct ctl_table sched_dl_sysctls[] = { { .procname = "sched_deadline_period_max_us", .data = &sysctl_sched_dl_period_max, .maxlen = sizeof(unsigned int), .mode = 0644, .proc_handler = proc_douintvec_minmax, .extra1 = (void *)&sysctl_sched_dl_period_min, }, { .procname = "sched_deadline_period_min_us", .data = &sysctl_sched_dl_period_min, .maxlen = sizeof(unsigned int), .mode = 0644, .proc_handler = proc_douintvec_minmax, .extra2 = (void *)&sysctl_sched_dl_period_max, }, }; static int __init sched_dl_sysctl_init(void) { register_sysctl_init("kernel", sched_dl_sysctls); return 0; } late_initcall(sched_dl_sysctl_init); #endif /* CONFIG_SYSCTL */ static inline struct rq *rq_of_dl_rq(struct dl_rq *dl_rq) { return container_of(dl_rq, struct rq, dl); } static inline struct rq *rq_of_dl_se(struct sched_dl_entity *dl_se) { struct rq *rq = dl_se->rq; if (!dl_server(dl_se)) rq = task_rq(dl_task_of(dl_se)); return rq; } static inline struct dl_rq *dl_rq_of_se(struct sched_dl_entity *dl_se) { return &rq_of_dl_se(dl_se)->dl; } static inline int on_dl_rq(struct sched_dl_entity *dl_se) { return !RB_EMPTY_NODE(&dl_se->rb_node); } #ifdef CONFIG_RT_MUTEXES static inline struct sched_dl_entity *pi_of(struct sched_dl_entity *dl_se) { return dl_se->pi_se; } static inline bool is_dl_boosted(struct sched_dl_entity *dl_se) { return pi_of(dl_se) != dl_se; } #else /* !CONFIG_RT_MUTEXES: */ static inline struct sched_dl_entity *pi_of(struct sched_dl_entity *dl_se) { return dl_se; } static inline bool is_dl_boosted(struct sched_dl_entity *dl_se) { return false; } #endif /* !CONFIG_RT_MUTEXES */ static inline u8 dl_get_type(struct sched_dl_entity *dl_se, struct rq *rq) { if (!dl_server(dl_se)) return DL_TASK; if (dl_se == &rq->fair_server) return DL_SERVER_FAIR; #ifdef CONFIG_SCHED_CLASS_EXT if (dl_se == &rq->ext_server) return DL_SERVER_EXT; #endif return DL_OTHER; } static inline struct dl_bw *dl_bw_of(int i) { RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(), "sched RCU must be held"); return &cpu_rq(i)->rd->dl_bw; } static inline int dl_bw_cpus(int i) { struct root_domain *rd = cpu_rq(i)->rd; RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(), "sched RCU must be held"); return cpumask_weight_and(rd->span, cpu_active_mask); } static inline unsigned long __dl_bw_capacity(const struct cpumask *mask) { unsigned long cap = 0; int i; for_each_cpu_and(i, mask, cpu_active_mask) cap += arch_scale_cpu_capacity(i); return cap; } /* * XXX Fix: If 'rq->rd == def_root_domain' perform AC against capacity * of the CPU the task is running on rather rd's \Sum CPU capacity. */ static inline unsigned long dl_bw_capacity(int i) { if (!sched_asym_cpucap_active() && arch_scale_cpu_capacity(i) == SCHED_CAPACITY_SCALE) { return dl_bw_cpus(i) << SCHED_CAPACITY_SHIFT; } else { RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(), "sched RCU must be held"); return __dl_bw_capacity(cpu_rq(i)->rd->span); } } bool dl_bw_visited(int cpu, u64 cookie) { struct root_domain *rd = cpu_rq(cpu)->rd; if (rd->visit_cookie == cookie) return true; rd->visit_cookie = cookie; return false; } static inline void __dl_update(struct dl_bw *dl_b, s64 bw) { struct root_domain *rd = container_of(dl_b, struct root_domain, dl_bw); int i; RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held(), "sched RCU must be held"); for_each_cpu_and(i, rd->span, cpu_active_mask) { struct rq *rq = cpu_rq(i); rq->dl.extra_bw += bw; } } static inline void __dl_sub(struct dl_bw *dl_b, u64 tsk_bw, int cpus) { dl_b->total_bw -= tsk_bw; __dl_update(dl_b, (s32)tsk_bw / cpus); } static inline void __dl_add(struct dl_bw *dl_b, u64 tsk_bw, int cpus) { dl_b->total_bw += tsk_bw; __dl_update(dl_b, -((s32)tsk_bw / cpus)); } static inline bool __dl_overflow(struct dl_bw *dl_b, unsigned long cap, u64 old_bw, u64 new_bw) { return dl_b->bw != -1 && cap_scale(dl_b->bw, cap) < dl_b->total_bw - old_bw + new_bw; } static inline void __add_running_bw(u64 dl_bw, struct dl_rq *dl_rq) { u64 old = dl_rq->running_bw; lockdep_assert_rq_held(rq_of_dl_rq(dl_rq)); dl_rq->running_bw += dl_bw; WARN_ON_ONCE(dl_rq->running_bw < old); /* overflow */ WARN_ON_ONCE(dl_rq->running_bw > dl_rq->this_bw); /* kick cpufreq (see the comment in kernel/sched/sched.h). */ cpufreq_update_util(rq_of_dl_rq(dl_rq), 0); } static inline void __sub_running_bw(u64 dl_bw, struct dl_rq *dl_rq) { u64 old = dl_rq->running_bw; lockdep_assert_rq_held(rq_of_dl_rq(dl_rq)); dl_rq->running_bw -= dl_bw; WARN_ON_ONCE(dl_rq->running_bw > old); /* underflow */ if (dl_rq->running_bw > old) dl_rq->running_bw = 0; /* kick cpufreq (see the comment in kernel/sched/sched.h). */ cpufreq_update_util(rq_of_dl_rq(dl_rq), 0); } static inline void __add_rq_bw(u64 dl_bw, struct dl_rq *dl_rq) { u64 old = dl_rq->this_bw; lockdep_assert_rq_held(rq_of_dl_rq(dl_rq)); dl_rq->this_bw += dl_bw; WARN_ON_ONCE(dl_rq->this_bw < old); /* overflow */ } static inline void __sub_rq_bw(u64 dl_bw, struct dl_rq *dl_rq) { u64 old = dl_rq->this_bw; lockdep_assert_rq_held(rq_of_dl_rq(dl_rq)); dl_rq->this_bw -= dl_bw; WARN_ON_ONCE(dl_rq->this_bw > old); /* underflow */ if (dl_rq->this_bw > old) dl_rq->this_bw = 0; WARN_ON_ONCE(dl_rq->running_bw > dl_rq->this_bw); } static inline void add_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) { if (!dl_entity_is_special(dl_se)) __add_rq_bw(dl_se->dl_bw, dl_rq); } static inline void sub_rq_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) { if (!dl_entity_is_special(dl_se)) __sub_rq_bw(dl_se->dl_bw, dl_rq); } static inline void add_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) { if (!dl_entity_is_special(dl_se)) __add_running_bw(dl_se->dl_bw, dl_rq); } static inline void sub_running_bw(struct sched_dl_entity *dl_se, struct dl_rq *dl_rq) { if (!dl_entity_is_special(dl_se)) __sub_running_bw(dl_se->dl_bw, dl_rq); } static void dl_rq_change_utilization(struct rq *rq, struct sched_dl_entity *dl_se, u64 new_bw) { if (dl_se->dl_non_contending) { sub_running_bw(dl_se, &rq->dl); dl_se->dl_non_contending = 0; /* * If the timer handler is currently running and the * timer cannot be canceled, inactive_task_timer() * will see that dl_not_contending is not set, and * will not touch the rq's active utilization, * so we are still safe. */ if (hrtimer_try_to_cancel(&dl_se->inactive_timer) == 1) { if (!dl_server(dl_se)) put_task_struct(dl_task_of(dl_se)); } } __sub_rq_bw(dl_se->dl_bw, &rq->dl); __add_rq_bw(new_bw, &rq->dl); } static __always_inline void cancel_dl_timer(struct sched_dl_entity *dl_se, struct hrtimer *timer) { /* * If the timer callback was running (hrtimer_try_to_cancel == -1), * it will eventually call put_task_struct(). */ if (hrtimer_try_to_cancel(timer) == 1 && !dl_server(dl_se)) put_task_struct(dl_task_of(dl_se)); } static __always_inline void cancel_replenish_timer(struct sched_dl_entity *dl_se) { cancel_dl_timer(dl_se, &dl_se->dl_timer); } static __always_inline void cancel_inactive_timer(struct sched_dl_entity *dl_se) { cancel_dl_timer(dl_se, &dl_se->inactive_timer); } static void dl_change_utilization(struct task_struct *p, u64 new_bw) { WARN_ON_ONCE(p->dl.flags & SCHED_FLAG_SUGOV); if (task_on_rq_queued(p)) return; dl_rq_change_utilization(task_rq(p), &p->dl, new_bw); } static void __dl_clear_params(struct sched_dl_entity *dl_se); /* * The utilization of a task cannot be immediately removed from * the rq active utilization (running_bw) when the task blocks. * Instead, we have to wait for the so called "0-lag time". * * If a task blocks before the "0-lag time", a timer (the inactive * timer) is armed, and running_bw is decreased when the timer * fires. * * If the task wakes up again before the inactive timer fires, * the timer is canceled, whereas if the task wakes up after the * inactive timer fired (and running_bw has been decreased) the * task's utilization has to be added to running_bw again. * A flag in the deadline scheduling entity (dl_non_contending) * is used to avoid race conditions between the inactive timer handler * and task wakeups. * * The following diagram shows how running_bw is updated. A task is * "ACTIVE" when its utilization contributes to running_bw; an * "ACTIVE contending" task is in the TASK_RUNNING state, while an * "ACTIVE non contending" task is a blocked task for which the "0-lag time" * has not passed yet. An "INACTIVE" task is a task for which the "0-lag" * time already passed, which does not contribute to running_bw anymore. * +------------------+ * wakeup | ACTIVE | * +------------------>+ contending | * | add_running_bw | | * | +----+------+------+ * | | ^ * | dequeue | | * +--------+-------+ | | * | | t >= 0-lag | | wakeup * | INACTIVE |<---------------+ | * | | sub_running_bw | | * +--------+-------+ | | * ^ | | * | t < 0-lag | | * | | | * | V | * | +----+------+------+ * | sub_running_bw | ACTIVE | * +-------------------+ | * inactive timer | non contending | * fired +------------------+ * * The task_non_contending() function is invoked when a task * blocks, and checks if the 0-lag time already passed or * not (in the first case, it directly updates running_bw; * in the second case, it arms the inactive timer). * * The task_contending() function is invoked when a task wakes * up, and checks if the task is still in the "ACTIVE non contending" * state or not (in the second case, it updates running_bw). */ static void task_non_contending(struct sched_dl_entity *dl_se, bool dl_task) { struct hrtimer *timer = &dl_se->inactive_timer; struct rq *rq = rq_of_dl_se(dl_se); struct dl_rq *dl_rq = &rq->dl; s64 zerolag_time; /* * If this is a non-deadline task that has been boosted, * do nothing */ if (dl_se->dl_runtime == 0) return; if (dl_entity_is_special(dl_se)) return; WARN_ON(dl_se->dl_non_contending); zerolag_time = dl_se->deadline - div64_long((dl_se->runtime * dl_se->dl_period), dl_se->dl_runtime); /* * Using relative times instead of the absolute "0-lag time" * allows to simplify the code */ zerolag_time -= rq_clock(rq); /* * If the "0-lag time" already passed, decrease the active * utilization now, instead of starting a timer */ if ((zerolag_time < 0) || hrtimer_active(&dl_se->inactive_timer)) { if (dl_server(dl_se)) { sub_running_bw(dl_se, dl_rq); } else { struct task_struct *p = dl_task_of(dl_se); if (dl_task) sub_running_bw(dl_se, dl_rq); if (!dl_task || READ_ONCE(p->__state) == TASK_DEAD) { struct dl_bw *dl_b = dl_bw_of(task_cpu(p)); if (READ_ONCE(p->__state) == TASK_DEAD) sub_rq_bw(dl_se, &rq->dl); raw_spin_lock(&dl_b->lock); __dl_sub(dl_b, dl_se->dl_bw, dl_bw_cpus(task_cpu(p))); raw_spin_unlock(&dl_b->lock); __dl_clear_params(dl_se); } } return; } dl_se->dl_non_contending = 1; if (!dl_server(dl_se)) get_task_struct(dl_task_of(dl_se)); hrtimer_start(timer, ns_to_ktime(zerolag_time), HRTIMER_MODE_REL_HARD); } static void task_contending(struct sched_dl_entity *dl_se, int flags) { struct dl_rq *dl_rq = dl_rq_of_se(dl_se); /* * If this is a non-deadline task that has been boosted, * do nothing */ if (dl_se->dl_runtime == 0) return; if (flags & ENQUEUE_MIGRATED) add_rq_bw(dl_se, dl_rq); if (dl_se->dl_non_contending) { dl_se->dl_non_contending = 0; /* * If the timer handler is currently running and the * timer cannot be canceled, inactive_task_timer() * will see that dl_not_contending is not set, and * will not touch the rq's active utilization, * so we are still safe. */ cancel_inactive_timer(dl_se); } else { /* * Since "dl_non_contending" is not set, the * task's utilization has already been removed from * active utilization (either when the task blocked, * when the "inactive timer" fired). * So, add it back. */ add_running_bw(dl_se, dl_rq); } }