History

sched/deadline: Add bandwidth management for SCHED_DEADLINE tasks

In order of deadline scheduling to be effective and useful, it is important that some method of having the allocation of the available CPU bandwidth to tasks and task groups under control. This is usually called "admission control" and if it is not performed at all, no guarantee can be given on the actual scheduling of the -deadline tasks. Since when RT-throttling has been introduced each task group have a bandwidth associated to itself, calculated as a certain amount of runtime over a period. Moreover, to make it possible to manipulate such bandwidth, readable/writable controls have been added to both procfs (for system wide settings) and cgroupfs (for per-group settings). Therefore, the same interface is being used for controlling the bandwidth distrubution to -deadline tasks and task groups, i.e., new controls but with similar names, equivalent meaning and with the same usage paradigm are added. However, more discussion is needed in order to figure out how we want to manage SCHED_DEADLINE bandwidth at the task group level. Therefore, this patch adds a less sophisticated, but actually very sensible, mechanism to ensure that a certain utilization cap is not overcome per each root_domain (the single rq for !SMP configurations). Another main difference between deadline bandwidth management and RT-throttling is that -deadline tasks have bandwidth on their own (while -rt ones doesn't!), and thus we don't need an higher level throttling mechanism to enforce the desired bandwidth. This patch, therefore: - adds system wide deadline bandwidth management by means of: * /proc/sys/kernel/sched_dl_runtime_us, * /proc/sys/kernel/sched_dl_period_us, that determine (i.e., runtime / period) the total bandwidth available on each CPU of each root_domain for -deadline tasks; - couples the RT and deadline bandwidth management, i.e., enforces that the sum of how much bandwidth is being devoted to -rt -deadline tasks to stay below 100%. This means that, for a root_domain comprising M CPUs, -deadline tasks can be created until the sum of their bandwidths stay below: M * (sched_dl_runtime_us / sched_dl_period_us) It is also possible to disable this bandwidth management logic, and be thus free of oversubscribing the system up to any arbitrary level.

Dario Faggioli committed 13 years ago

Commit 332ac171 parent 2d3d8916 files · +555 −36Browse files

Unified · Split

6 files
  1. include/linux/sched.h +1 −0
  2. include/linux/sched/sysctl.h +13 −0
  3. kernel/sched/core.c +414 −27
  4. kernel/sched/deadline.c +40 −6
  5. kernel/sched/sched.h +73 −3
  6. kernel/sysctl.c +14 −0
include/linux/sched.h+1−0
@@ -1104,6 +1104,7 @@ struct sched_dl_entity { · 1103 unmodified lines
11041104 u64 dl_runtime; /* maximum runtime for each instance */
11051105 u64 dl_deadline; /* relative deadline of each instance */
11061106 u64 dl_period; /* separation of two instances (period) */
1107+ u64 dl_bw; /* dl_runtime / dl_deadline */
11071108
11081109 /*
11091110 * Actual scheduling parameters. Initialized with the values above,
include/linux/sched/sysctl.h+13−0
@@ -81,6 +81,15 @@ static inline unsigned int get_sysctl_timer_migration(void) · 80 unmodified lines
8181 extern unsigned int sysctl_sched_rt_period;
8282 extern int sysctl_sched_rt_runtime;
8383
84+/*
85+ * control SCHED_DEADLINE reservations:
86+ *
87+ * /proc/sys/kernel/sched_dl_period_us
88+ * /proc/sys/kernel/sched_dl_runtime_us
89+ */
90+extern unsigned int sysctl_sched_dl_period;
91+extern int sysctl_sched_dl_runtime;
92+
8493 #ifdef CONFIG_CFS_BANDWIDTH
8594 extern unsigned int sysctl_sched_cfs_bandwidth_slice;
8695 #endif
@@ -99,4 +108,8 @@ extern int sched_rt_handler(struct ctl_table *table, int write, · 12 unmodified lines
99108 void __user *buffer, size_t *lenp,
100109 loff_t *ppos);
101110
111+int sched_dl_handler(struct ctl_table *table, int write,
112+ void __user *buffer, size_t *lenp,
113+ loff_t *ppos);
114+
102115 #endif /* _SCHED_SYSCTL_H */
kernel/sched/core.c+414−27
@@ -296,6 +296,15 @@ __read_mostly int scheduler_running; · 295 unmodified lines
296296 */
297297 int sysctl_sched_rt_runtime = 950000;
298298
299+/*
300+ * Maximum bandwidth available for all -deadline tasks and groups
301+ * (if group scheduling is configured) on each CPU.
302+ *
303+ * default: 5%
304+ */
305+unsigned int sysctl_sched_dl_period = 1000000;
306+int sysctl_sched_dl_runtime = 50000;
307+
299308
300309
301310 /*
@@ -1856,6 +1865,111 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p) · 1554 unmodified lines
18561865 return 0;
18571866 }
18581867
1868+unsigned long to_ratio(u64 period, u64 runtime)
1869+{
1870+ if (runtime == RUNTIME_INF)
1871+ return 1ULL << 20;
1872+
1873+ /*
1874+ * Doing this here saves a lot of checks in all
1875+ * the calling paths, and returning zero seems
1876+ * safe for them anyway.
1877+ */
1878+ if (period == 0)
1879+ return 0;
1880+
1881+ return div64_u64(runtime << 20, period);
1882+}
1883+
1884+#ifdef CONFIG_SMP
1885+inline struct dl_bw *dl_bw_of(int i)
1886+{
1887+ return &cpu_rq(i)->rd->dl_bw;
1888+}
1889+
1890+static inline int __dl_span_weight(struct rq *rq)
1891+{
1892+ return cpumask_weight(rq->rd->span);
1893+}
1894+#else
1895+inline struct dl_bw *dl_bw_of(int i)
1896+{
1897+ return &cpu_rq(i)->dl.dl_bw;
1898+}
1899+
1900+static inline int __dl_span_weight(struct rq *rq)
1901+{
1902+ return 1;
1903+}
1904+#endif
1905+
1906+static inline
1907+void __dl_clear(struct dl_bw *dl_b, u64 tsk_bw)
1908+{
1909+ dl_b->total_bw -= tsk_bw;
1910+}
1911+
1912+static inline
1913+void __dl_add(struct dl_bw *dl_b, u64 tsk_bw)
1914+{
1915+ dl_b->total_bw += tsk_bw;
1916+}
1917+
1918+static inline
1919+bool __dl_overflow(struct dl_bw *dl_b, int cpus, u64 old_bw, u64 new_bw)
1920+{
1921+ return dl_b->bw != -1 &&
1922+ dl_b->bw * cpus < dl_b->total_bw - old_bw + new_bw;
1923+}
1924+
1925+/*
1926+ * We must be sure that accepting a new task (or allowing changing the
1927+ * parameters of an existing one) is consistent with the bandwidth
1928+ * constraints. If yes, this function also accordingly updates the currently
1929+ * allocated bandwidth to reflect the new situation.
1930+ *
1931+ * This function is called while holding p's rq->lock.
1932+ */
1933+static int dl_overflow(struct task_struct *p, int policy,
1934+ const struct sched_attr *attr)
1935+{
1936+
1937+ struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
1938+ u64 period = attr->sched_period;
1939+ u64 runtime = attr->sched_runtime;
1940+ u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0;
1941+ int cpus = __dl_span_weight(task_rq(p));
1942+ int err = -1;
1943+
1944+ if (new_bw == p->dl.dl_bw)
1945+ return 0;
1946+
1947+ /*
1948+ * Either if a task, enters, leave, or stays -deadline but changes
1949+ * its parameters, we may need to update accordingly the total
1950+ * allocated bandwidth of the container.
1951+ */
1952+ raw_spin_lock(&dl_b->lock);
1953+ if (dl_policy(policy) && !task_has_dl_policy(p) &&
1954+ !__dl_overflow(dl_b, cpus, 0, new_bw)) {
1955+ __dl_add(dl_b, new_bw);
1956+ err = 0;
1957+ } else if (dl_policy(policy) && task_has_dl_policy(p) &&
1958+ !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) {
1959+ __dl_clear(dl_b, p->dl.dl_bw);
1960+ __dl_add(dl_b, new_bw);
1961+ err = 0;
1962+ } else if (!dl_policy(policy) && task_has_dl_policy(p)) {
1963+ __dl_clear(dl_b, p->dl.dl_bw);
1964+ err = 0;
1965+ }
1966+ raw_spin_unlock(&dl_b->lock);
1967+
1968+ return err;
1969+}
1970+
1971+extern void init_dl_bw(struct dl_bw *dl_b);
1972+
18591973 /*
18601974 * wake_up_new_task - wake up a newly created task for the first time.
18611975 *
@@ -3053,6 +3167,7 @@ __setparam_dl(struct task_struct *p, const struct sched_attr *attr) · 1191 unmodified lines
30533167 dl_se->dl_deadline = attr->sched_deadline;
30543168 dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline;
30553169 dl_se->flags = attr->sched_flags;
3170+ dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime);
30563171 dl_se->dl_throttled = 0;
30573172 dl_se->dl_new = 1;
30583173 }
@@ -3101,7 +3216,9 @@ __getparam_dl(struct task_struct *p, struct sched_attr *attr) · 42 unmodified lines
31013216 * This function validates the new parameters of a -deadline task.
31023217 * We ask for the deadline not being zero, and greater or equal
31033218 * than the runtime, as well as the period of being zero or
3104− * greater than deadline.
3219+ * greater than deadline. Furthermore, we have to be sure that
3220+ * user parameters are above the internal resolution (1us); we
3221+ * check sched_runtime only since it is always the smaller one.
31053222 */
31063223 static bool
31073224 __checkparam_dl(const struct sched_attr *attr)
@@ -3109,7 +3226,8 @@ __checkparam_dl(const struct sched_attr *attr) · 1 unmodified line
31093226 return attr && attr->sched_deadline != 0 &&
31103227 (attr->sched_period == 0 ||
31113228 (s64)(attr->sched_period - attr->sched_deadline) >= 0) &&
3112− (s64)(attr->sched_deadline - attr->sched_runtime ) >= 0;
3229+ (s64)(attr->sched_deadline - attr->sched_runtime ) >= 0 &&
3230+ attr->sched_runtime >= (2 << (DL_SCALE - 1));
31133231 }
31143232
31153233 /*
@@ -3250,8 +3368,8 @@ recheck: · 134 unmodified lines
32503368 }
32513369 change:
32523370
3253−#ifdef CONFIG_RT_GROUP_SCHED
32543371 if (user) {
3372+#ifdef CONFIG_RT_GROUP_SCHED
32553373 /*
32563374 * Do not allow realtime tasks into groups that have no runtime
32573375 * assigned.
@@ -3262,8 +3380,33 @@ change: · 4 unmodified lines
32623380 task_rq_unlock(rq, p, &flags);
32633381 return -EPERM;
32643382 }
3265− }
32663383 #endif
3384+#ifdef CONFIG_SMP
3385+ if (dl_bandwidth_enabled() && dl_policy(policy)) {
3386+ cpumask_t *span = rq->rd->span;
3387+ cpumask_t act_affinity;
3388+
3389+ /*
3390+ * cpus_allowed mask is statically initialized with
3391+ * CPU_MASK_ALL, span is instead dynamic. Here we
3392+ * compute the "dynamic" affinity of a task.
3393+ */
3394+ cpumask_and(&act_affinity, &p->cpus_allowed,
3395+ cpu_active_mask);
3396+
3397+ /*
3398+ * Don't allow tasks with an affinity mask smaller than
3399+ * the entire root_domain to become SCHED_DEADLINE. We
3400+ * will also fail if there's no bandwidth available.
3401+ */
3402+ if (!cpumask_equal(&act_affinity, span) ||
3403+ rq->rd->dl_bw.bw == 0) {
3404+ task_rq_unlock(rq, p, &flags);
3405+ return -EPERM;
3406+ }
3407+ }
3408+#endif
3409+ }
32673410
32683411 /* recheck policy now with rq lock held */
32693412 if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) {
@@ -3271,6 +3414,18 @@ change: · 1 unmodified line
32713414 task_rq_unlock(rq, p, &flags);
32723415 goto recheck;
32733416 }
3417+
3418+ /*
3419+ * If setscheduling to SCHED_DEADLINE (or changing the parameters
3420+ * of a SCHED_DEADLINE task) we need to check if enough bandwidth
3421+ * is available.
3422+ */
3423+ if ((dl_policy(policy) || dl_task(p)) &&
3424+ dl_overflow(p, policy, attr)) {
3425+ task_rq_unlock(rq, p, &flags);
3426+ return -EBUSY;
3427+ }
3428+
32743429 on_rq = p->on_rq;
32753430 running = task_current(rq, p);
32763431 if (on_rq)
@@ -3705,6 +3860,24 @@ long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) · 428 unmodified lines
37053860 if (retval)
37063861 goto out_unlock;
37073862
3863+ /*
3864+ * Since bandwidth control happens on root_domain basis,
3865+ * if admission test is enabled, we only admit -deadline
3866+ * tasks allowed to run on all the CPUs in the task's
3867+ * root_domain.
3868+ */
3869+#ifdef CONFIG_SMP
3870+ if (task_has_dl_policy(p)) {
3871+ const struct cpumask *span = task_rq(p)->rd->span;
3872+
3873+ if (dl_bandwidth_enabled() &&
3874+ !cpumask_equal(in_mask, span)) {
3875+ retval = -EBUSY;
3876+ goto out_unlock;
3877+ }
3878+ }
3879+#endif
3880+
37083881 cpuset_cpus_allowed(p, cpus_allowed);
37093882 cpumask_and(new_mask, in_mask, cpus_allowed);
37103883 again:
@@ -4358,6 +4531,42 @@ out: · 647 unmodified lines
43584531 }
43594532 EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
43604533
4534+/*
4535+ * When dealing with a -deadline task, we have to check if moving it to
4536+ * a new CPU is possible or not. In fact, this is only true iff there
4537+ * is enough bandwidth available on such CPU, otherwise we want the
4538+ * whole migration progedure to fail over.
4539+ */
4540+static inline
4541+bool set_task_cpu_dl(struct task_struct *p, unsigned int cpu)
4542+{
4543+ struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
4544+ struct dl_bw *cpu_b = dl_bw_of(cpu);
4545+ int ret = 1;
4546+ u64 bw;
4547+
4548+ if (dl_b == cpu_b)
4549+ return 1;
4550+
4551+ raw_spin_lock(&dl_b->lock);
4552+ raw_spin_lock(&cpu_b->lock);
4553+
4554+ bw = cpu_b->bw * cpumask_weight(cpu_rq(cpu)->rd->span);
4555+ if (dl_bandwidth_enabled() &&
4556+ bw < cpu_b->total_bw + p->dl.dl_bw) {
4557+ ret = 0;
4558+ goto unlock;
4559+ }
4560+ dl_b->total_bw -= p->dl.dl_bw;
4561+ cpu_b->total_bw += p->dl.dl_bw;
4562+
4563+unlock:
4564+ raw_spin_unlock(&cpu_b->lock);
4565+ raw_spin_unlock(&dl_b->lock);
4566+
4567+ return ret;
4568+}
4569+
43614570 /*
43624571 * Move (not current) task off this cpu, onto dest cpu. We're doing
43634572 * this because either it can't run here any more (set_cpus_allowed()
@@ -4389,6 +4598,13 @@ static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) · 25 unmodified lines
43894598 if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p)))
43904599 goto fail;
43914600
4601+ /*
4602+ * If p is -deadline, proceed only if there is enough
4603+ * bandwidth available on dest_cpu
4604+ */
4605+ if (unlikely(dl_task(p)) && !set_task_cpu_dl(p, dest_cpu))
4606+ goto fail;
4607+
43924608 /*
43934609 * If we're not on a rq, the next wake-up will ensure we're
43944610 * placed properly.
@@ -5128,6 +5344,8 @@ static int init_rootdomain(struct root_domain *rd) · 733 unmodified lines
51285344 if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
51295345 goto free_dlo_mask;
51305346
5347+ init_dl_bw(&rd->dl_bw);
5348+
51315349 if (cpupri_init(&rd->cpupri) != 0)
51325350 goto free_rto_mask;
51335351 return 0;
@@ -6557,13 +6775,15 @@ void __init sched_init(void) · 1423 unmodified lines
65576775 #endif /* CONFIG_CPUMASK_OFFSTACK */
65586776 }
65596777
6778+ init_rt_bandwidth(&def_rt_bandwidth,
6779+ global_rt_period(), global_rt_runtime());
6780+ init_dl_bandwidth(&def_dl_bandwidth,
6781+ global_dl_period(), global_dl_runtime());
6782+
65606783 #ifdef CONFIG_SMP
65616784 init_defrootdomain();
65626785 #endif
65636786
6564− init_rt_bandwidth(&def_rt_bandwidth,
6565− global_rt_period(), global_rt_runtime());
6566−
65676787 #ifdef CONFIG_RT_GROUP_SCHED
65686788 init_rt_bandwidth(&root_task_group.rt_bandwidth,
65696789 global_rt_period(), global_rt_runtime());
@@ -6966,16 +7186,6 @@ void sched_move_task(struct task_struct *tsk) · 396 unmodified lines
69667186 }
69677187 #endif /* CONFIG_CGROUP_SCHED */
69687188
6969−#if defined(CONFIG_RT_GROUP_SCHED) || defined(CONFIG_CFS_BANDWIDTH)
6970−static unsigned long to_ratio(u64 period, u64 runtime)
6971−{
6972− if (runtime == RUNTIME_INF)
6973− return 1ULL << 20;
6974−
6975− return div64_u64(runtime << 20, period);
6976−}
6977−#endif
6978−
69797189 #ifdef CONFIG_RT_GROUP_SCHED
69807190 /*
69817191 * Ensure that the real time constraints are schedulable.
@@ -7149,10 +7359,48 @@ static long sched_group_rt_period(struct task_group *tg) · 167 unmodified lines
71497359 do_div(rt_period_us, NSEC_PER_USEC);
71507360 return rt_period_us;
71517361 }
7362+#endif /* CONFIG_RT_GROUP_SCHED */
71527363
7364+/*
7365+ * Coupling of -rt and -deadline bandwidth.
7366+ *
7367+ * Here we check if the new -rt bandwidth value is consistent
7368+ * with the system settings for the bandwidth available
7369+ * to -deadline tasks.
7370+ *
7371+ * IOW, we want to enforce that
7372+ *
7373+ * rt_bandwidth + dl_bandwidth <= 100%
7374+ *
7375+ * is always true.
7376+ */
7377+static bool __sched_rt_dl_global_constraints(u64 rt_bw)
7378+{
7379+ unsigned long flags;
7380+ u64 dl_bw;
7381+ bool ret;
7382+
7383+ raw_spin_lock_irqsave(&def_dl_bandwidth.dl_runtime_lock, flags);
7384+ if (global_rt_runtime() == RUNTIME_INF ||
7385+ global_dl_runtime() == RUNTIME_INF) {
7386+ ret = true;
7387+ goto unlock;
7388+ }
7389+
7390+ dl_bw = to_ratio(def_dl_bandwidth.dl_period,
7391+ def_dl_bandwidth.dl_runtime);
7392+
7393+ ret = rt_bw + dl_bw <= to_ratio(RUNTIME_INF, RUNTIME_INF);
7394+unlock:
7395+ raw_spin_unlock_irqrestore(&def_dl_bandwidth.dl_runtime_lock, flags);
7396+
7397+ return ret;
7398+}
7399+
7400+#ifdef CONFIG_RT_GROUP_SCHED
71537401 static int sched_rt_global_constraints(void)
71547402 {
7155− u64 runtime, period;
7403+ u64 runtime, period, bw;
71567404 int ret = 0;
71577405
71587406 if (sysctl_sched_rt_period <= 0)
@@ -7167,6 +7415,10 @@ static int sched_rt_global_constraints(void) · 8 unmodified lines
71677415 if (runtime > period && runtime != RUNTIME_INF)
71687416 return -EINVAL;
71697417
7418+ bw = to_ratio(period, runtime);
7419+ if (!__sched_rt_dl_global_constraints(bw))
7420+ return -EINVAL;
7421+
71707422 mutex_lock(&rt_constraints_mutex);
71717423 read_lock(&tasklist_lock);
71727424 ret = __rt_schedulable(NULL, 0, 0);
@@ -7189,19 +7441,19 @@ static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk) · 16 unmodified lines
71897441 static int sched_rt_global_constraints(void)
71907442 {
71917443 unsigned long flags;
7192− int i;
7444+ int i, ret = 0;
7445+ u64 bw;
71937446
71947447 if (sysctl_sched_rt_period <= 0)
71957448 return -EINVAL;
71967449
7197− /*
7198− * There's always some RT tasks in the root group
7199− * -- migration, kstopmachine etc..
7200− */
7201− if (sysctl_sched_rt_runtime == 0)
7202− return -EBUSY;
7203−
72047450 raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags);
7451+ bw = to_ratio(global_rt_period(), global_rt_runtime());
7452+ if (!__sched_rt_dl_global_constraints(bw)) {
7453+ ret = -EINVAL;
7454+ goto unlock;
7455+ }
7456+
72057457 for_each_possible_cpu(i) {
72067458 struct rt_rq *rt_rq = &cpu_rq(i)->rt;
72077459
@@ -7209,12 +7461,93 @@ static int sched_rt_global_constraints(void) · 1 unmodified line
72097461 rt_rq->rt_runtime = global_rt_runtime();
72107462 raw_spin_unlock(&rt_rq->rt_runtime_lock);
72117463 }
7464+unlock:
72127465 raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags);
72137466
7214− return 0;
7467+ return ret;
72157468 }
72167469 #endif /* CONFIG_RT_GROUP_SCHED */
72177470
7471+/*
7472+ * Coupling of -dl and -rt bandwidth.
7473+ *
7474+ * Here we check, while setting the system wide bandwidth available
7475+ * for -dl tasks and groups, if the new values are consistent with
7476+ * the system settings for the bandwidth available to -rt entities.
7477+ *
7478+ * IOW, we want to enforce that
7479+ *
7480+ * rt_bandwidth + dl_bandwidth <= 100%
7481+ *
7482+ * is always true.
7483+ */
7484+static bool __sched_dl_rt_global_constraints(u64 dl_bw)
7485+{
7486+ u64 rt_bw;
7487+ bool ret;
7488+
7489+ raw_spin_lock(&def_rt_bandwidth.rt_runtime_lock);
7490+ if (global_dl_runtime() == RUNTIME_INF ||
7491+ global_rt_runtime() == RUNTIME_INF) {
7492+ ret = true;
7493+ goto unlock;
7494+ }
7495+
7496+ rt_bw = to_ratio(ktime_to_ns(def_rt_bandwidth.rt_period),
7497+ def_rt_bandwidth.rt_runtime);
7498+
7499+ ret = rt_bw + dl_bw <= to_ratio(RUNTIME_INF, RUNTIME_INF);
7500+unlock:
7501+ raw_spin_unlock(&def_rt_bandwidth.rt_runtime_lock);
7502+
7503+ return ret;
7504+}
7505+
7506+static bool __sched_dl_global_constraints(u64 runtime, u64 period)
7507+{
7508+ if (!period || (runtime != RUNTIME_INF && runtime > period))
7509+ return -EINVAL;
7510+
7511+ return 0;
7512+}
7513+
7514+static int sched_dl_global_constraints(void)
7515+{
7516+ u64 runtime = global_dl_runtime();
7517+ u64 period = global_dl_period();
7518+ u64 new_bw = to_ratio(period, runtime);
7519+ int ret, i;
7520+
7521+ ret = __sched_dl_global_constraints(runtime, period);
7522+ if (ret)
7523+ return ret;
7524+
7525+ if (!__sched_dl_rt_global_constraints(new_bw))
7526+ return -EINVAL;
7527+
7528+ /*
7529+ * Here we want to check the bandwidth not being set to some
7530+ * value smaller than the currently allocated bandwidth in
7531+ * any of the root_domains.
7532+ *
7533+ * FIXME: Cycling on all the CPUs is overdoing, but simpler than
7534+ * cycling on root_domains... Discussion on different/better
7535+ * solutions is welcome!
7536+ */
7537+ for_each_possible_cpu(i) {
7538+ struct dl_bw *dl_b = dl_bw_of(i);
7539+
7540+ raw_spin_lock(&dl_b->lock);
7541+ if (new_bw < dl_b->total_bw) {
7542+ raw_spin_unlock(&dl_b->lock);
7543+ return -EBUSY;
7544+ }
7545+ raw_spin_unlock(&dl_b->lock);
7546+ }
7547+
7548+ return 0;
7549+}
7550+
72187551 int sched_rr_handler(struct ctl_table *table, int write,
72197552 void __user *buffer, size_t *lenp,
72207553 loff_t *ppos)
@@ -7264,6 +7597,60 @@ int sched_rt_handler(struct ctl_table *table, int write, · 43 unmodified lines
72647597 return ret;
72657598 }
72667599
7600+int sched_dl_handler(struct ctl_table *table, int write,
7601+ void __user *buffer, size_t *lenp,
7602+ loff_t *ppos)
7603+{
7604+ int ret;
7605+ int old_period, old_runtime;
7606+ static DEFINE_MUTEX(mutex);
7607+ unsigned long flags;
7608+
7609+ mutex_lock(&mutex);
7610+ old_period = sysctl_sched_dl_period;
7611+ old_runtime = sysctl_sched_dl_runtime;
7612+
7613+ ret = proc_dointvec(table, write, buffer, lenp, ppos);
7614+
7615+ if (!ret && write) {
7616+ raw_spin_lock_irqsave(&def_dl_bandwidth.dl_runtime_lock,
7617+ flags);
7618+
7619+ ret = sched_dl_global_constraints();
7620+ if (ret) {
7621+ sysctl_sched_dl_period = old_period;
7622+ sysctl_sched_dl_runtime = old_runtime;
7623+ } else {
7624+ u64 new_bw;
7625+ int i;
7626+
7627+ def_dl_bandwidth.dl_period = global_dl_period();
7628+ def_dl_bandwidth.dl_runtime = global_dl_runtime();
7629+ if (global_dl_runtime() == RUNTIME_INF)
7630+ new_bw = -1;
7631+ else
7632+ new_bw = to_ratio(global_dl_period(),
7633+ global_dl_runtime());
7634+ /*
7635+ * FIXME: As above...
7636+ */
7637+ for_each_possible_cpu(i) {
7638+ struct dl_bw *dl_b = dl_bw_of(i);
7639+
7640+ raw_spin_lock(&dl_b->lock);
7641+ dl_b->bw = new_bw;
7642+ raw_spin_unlock(&dl_b->lock);
7643+ }
7644+ }
7645+
7646+ raw_spin_unlock_irqrestore(&def_dl_bandwidth.dl_runtime_lock,
7647+ flags);
7648+ }
7649+ mutex_unlock(&mutex);
7650+
7651+ return ret;
7652+}
7653+
72677654 #ifdef CONFIG_CGROUP_SCHED
72687655
72697656 static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
kernel/sched/deadline.c+40−6
@@ -16,6 +16,8 @@ · 15 unmodified lines
1616 */
1717 #include "sched.h"
1818
19+struct dl_bandwidth def_dl_bandwidth;
20+
1921 static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se)
2022 {
2123 return container_of(dl_se, struct task_struct, dl);
@@ -46,6 +48,27 @@ static inline int is_leftmost(struct task_struct *p, struct dl_rq *dl_rq) · 24 unmodified lines
4648 return dl_rq->rb_leftmost == &dl_se->rb_node;
4749 }
4850
51+void init_dl_bandwidth(struct dl_bandwidth *dl_b, u64 period, u64 runtime)
52+{
53+ raw_spin_lock_init(&dl_b->dl_runtime_lock);
54+ dl_b->dl_period = period;
55+ dl_b->dl_runtime = runtime;
56+}
57+
58+extern unsigned long to_ratio(u64 period, u64 runtime);
59+
60+void init_dl_bw(struct dl_bw *dl_b)
61+{
62+ raw_spin_lock_init(&dl_b->lock);
63+ raw_spin_lock(&def_dl_bandwidth.dl_runtime_lock);
64+ if (global_dl_runtime() == RUNTIME_INF)
65+ dl_b->bw = -1;
66+ else
67+ dl_b->bw = to_ratio(global_dl_period(), global_dl_runtime());
68+ raw_spin_unlock(&def_dl_bandwidth.dl_runtime_lock);
69+ dl_b->total_bw = 0;
70+}
71+
4972 void init_dl_rq(struct dl_rq *dl_rq, struct rq *rq)
5073 {
5174 dl_rq->rb_root = RB_ROOT;
@@ -57,6 +80,8 @@ void init_dl_rq(struct dl_rq *dl_rq, struct rq *rq) · 5 unmodified lines
5780 dl_rq->dl_nr_migratory = 0;
5881 dl_rq->overloaded = 0;
5982 dl_rq->pushable_dl_tasks_root = RB_ROOT;
83+#else
84+ init_dl_bw(&dl_rq->dl_bw);
6085 #endif
6186 }
6287
@@ -359,8 +384,9 @@ static bool dl_entity_overflow(struct sched_dl_entity *dl_se, · 296 unmodified lines
359384 * of anything below microseconds resolution is actually fiction
360385 * (but still we want to give the user that illusion >;).
361386 */
362− left = (pi_se->dl_period >> 10) * (dl_se->runtime >> 10);
363− right = ((dl_se->deadline - t) >> 10) * (pi_se->dl_runtime >> 10);
387+ left = (pi_se->dl_period >> DL_SCALE) * (dl_se->runtime >> DL_SCALE);
388+ right = ((dl_se->deadline - t) >> DL_SCALE) *
389+ (pi_se->dl_runtime >> DL_SCALE);
364390
365391 return dl_time_before(right, left);
366392 }
@@ -911,8 +937,8 @@ static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p, · 544 unmodified lines
911937 * In the unlikely case current and p have the same deadline
912938 * let us try to decide what's the best thing to do...
913939 */
914− if ((s64)(p->dl.deadline - rq->curr->dl.deadline) == 0 &&
915− !need_resched())
940+ if ((p->dl.deadline == rq->curr->dl.deadline) &&
941+ !test_tsk_need_resched(rq->curr))
916942 check_preempt_equal_dl(rq, p);
917943 #endif /* CONFIG_SMP */
918944 }
@@ -1000,6 +1026,14 @@ static void task_fork_dl(struct task_struct *p) · 81 unmodified lines
10001026 static void task_dead_dl(struct task_struct *p)
10011027 {
10021028 struct hrtimer *timer = &p->dl.dl_timer;
1029+ struct dl_bw *dl_b = dl_bw_of(task_cpu(p));
1030+
1031+ /*
1032+ * Since we are TASK_DEAD we won't slip out of the domain!
1033+ */
1034+ raw_spin_lock_irq(&dl_b->lock);
1035+ dl_b->total_bw -= p->dl.dl_bw;
1036+ raw_spin_unlock_irq(&dl_b->lock);
10031037
10041038 hrtimer_cancel(timer);
10051039 }
@@ -1226,7 +1260,7 @@ static struct task_struct *pick_next_pushable_dl_task(struct rq *rq) · 220 unmodified lines
12261260 BUG_ON(task_current(rq, p));
12271261 BUG_ON(p->nr_cpus_allowed <= 1);
12281262
1229− BUG_ON(!p->se.on_rq);
1263+ BUG_ON(!p->on_rq);
12301264 BUG_ON(!dl_task(p));
12311265
12321266 return p;
@@ -1373,7 +1407,7 @@ static int pull_dl_task(struct rq *this_rq) · 140 unmodified lines
13731407 dl_time_before(p->dl.deadline,
13741408 this_rq->dl.earliest_dl.curr))) {
13751409 WARN_ON(p == src_rq->curr);
1376− WARN_ON(!p->se.on_rq);
1410+ WARN_ON(!p->on_rq);
13771411
13781412 /*
13791413 * Then we pull iff p has actually an earlier
kernel/sched/sched.h+73−3
@@ -73,6 +73,13 @@ extern void update_cpu_load_active(struct rq *this_rq); · 72 unmodified lines
7373 #define NICE_0_LOAD SCHED_LOAD_SCALE
7474 #define NICE_0_SHIFT SCHED_LOAD_SHIFT
7575
76+/*
77+ * Single value that decides SCHED_DEADLINE internal math precision.
78+ * 10 -> just above 1us
79+ * 9 -> just above 0.5us
80+ */
81+#define DL_SCALE (10)
82+
7683 /*
7784 * These are the 'tuning knobs' of the scheduler:
7885 */
@@ -107,7 +114,7 @@ static inline int task_has_dl_policy(struct task_struct *p) · 28 unmodified lines
107114 return dl_policy(p->policy);
108115 }
109116
110−static inline int dl_time_before(u64 a, u64 b)
117+static inline bool dl_time_before(u64 a, u64 b)
111118 {
112119 return (s64)(a - b) < 0;
113120 }
@@ -115,8 +122,8 @@ static inline int dl_time_before(u64 a, u64 b) · 1 unmodified line
115122 /*
116123 * Tells if entity @a should preempt entity @b.
117124 */
118−static inline
119−int dl_entity_preempt(struct sched_dl_entity *a, struct sched_dl_entity *b)
125+static inline bool
126+dl_entity_preempt(struct sched_dl_entity *a, struct sched_dl_entity *b)
120127 {
121128 return dl_time_before(a->deadline, b->deadline);
122129 }
@@ -136,6 +143,50 @@ struct rt_bandwidth { · 13 unmodified lines
136143 u64 rt_runtime;
137144 struct hrtimer rt_period_timer;
138145 };
146+/*
147+ * To keep the bandwidth of -deadline tasks and groups under control
148+ * we need some place where:
149+ * - store the maximum -deadline bandwidth of the system (the group);
150+ * - cache the fraction of that bandwidth that is currently allocated.
151+ *
152+ * This is all done in the data structure below. It is similar to the
153+ * one used for RT-throttling (rt_bandwidth), with the main difference
154+ * that, since here we are only interested in admission control, we
155+ * do not decrease any runtime while the group "executes", neither we
156+ * need a timer to replenish it.
157+ *
158+ * With respect to SMP, the bandwidth is given on a per-CPU basis,
159+ * meaning that:
160+ * - dl_bw (< 100%) is the bandwidth of the system (group) on each CPU;
161+ * - dl_total_bw array contains, in the i-eth element, the currently
162+ * allocated bandwidth on the i-eth CPU.
163+ * Moreover, groups consume bandwidth on each CPU, while tasks only
164+ * consume bandwidth on the CPU they're running on.
165+ * Finally, dl_total_bw_cpu is used to cache the index of dl_total_bw
166+ * that will be shown the next time the proc or cgroup controls will
167+ * be red. It on its turn can be changed by writing on its own
168+ * control.
169+ */
170+struct dl_bandwidth {
171+ raw_spinlock_t dl_runtime_lock;
172+ u64 dl_runtime;
173+ u64 dl_period;
174+};
175+
176+static inline int dl_bandwidth_enabled(void)
177+{
178+ return sysctl_sched_dl_runtime >= 0;
179+}
180+
181+extern struct dl_bw *dl_bw_of(int i);
182+
183+struct dl_bw {
184+ raw_spinlock_t lock;
185+ u64 bw, total_bw;
186+};
187+
188+static inline u64 global_dl_period(void);
189+static inline u64 global_dl_runtime(void);
139190
140191 extern struct mutex sched_domains_mutex;
141192
@@ -423,6 +474,8 @@ struct dl_rq { · 281 unmodified lines
423474 */
424475 struct rb_root pushable_dl_tasks_root;
425476 struct rb_node *pushable_dl_tasks_leftmost;
477+#else
478+ struct dl_bw dl_bw;
426479 #endif
427480 };
428481
@@ -449,6 +502,7 @@ struct root_domain { · 20 unmodified lines
449502 */
450503 cpumask_var_t dlo_mask;
451504 atomic_t dlo_count;
505+ struct dl_bw dl_bw;
452506
453507 /*
454508 * The "RT overload" flag: it gets set if a CPU has more than
@@ -897,7 +951,18 @@ static inline u64 global_rt_runtime(void) · 442 unmodified lines
897951 return (u64)sysctl_sched_rt_runtime * NSEC_PER_USEC;
898952 }
899953
954+static inline u64 global_dl_period(void)
955+{
956+ return (u64)sysctl_sched_dl_period * NSEC_PER_USEC;
957+}
958+
959+static inline u64 global_dl_runtime(void)
960+{
961+ if (sysctl_sched_dl_runtime < 0)
962+ return RUNTIME_INF;
900963
964+ return (u64)sysctl_sched_dl_runtime * NSEC_PER_USEC;
965+}
901966
902967 static inline int task_current(struct rq *rq, struct task_struct *p)
903968 {
@@ -1145,6 +1210,7 @@ extern void update_max_interval(void); · 241 unmodified lines
11451210 extern void init_sched_dl_class(void);
11461211 extern void init_sched_rt_class(void);
11471212 extern void init_sched_fair_class(void);
1213+extern void init_sched_dl_class(void);
11481214
11491215 extern void resched_task(struct task_struct *p);
11501216 extern void resched_cpu(int cpu);
@@ -1152,8 +1218,12 @@ extern void resched_cpu(int cpu); · 1 unmodified line
11521218 extern struct rt_bandwidth def_rt_bandwidth;
11531219 extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime);
11541220
1221+extern struct dl_bandwidth def_dl_bandwidth;
1222+extern void init_dl_bandwidth(struct dl_bandwidth *dl_b, u64 period, u64 runtime);
11551223 extern void init_dl_task_timer(struct sched_dl_entity *dl_se);
11561224
1225+unsigned long to_ratio(u64 period, u64 runtime);
1226+
11571227 extern void update_idle_cpu_load(struct rq *this_rq);
11581228
11591229 extern void init_task_runnable_average(struct task_struct *p);
kernel/sysctl.c+14−0
@@ -414,6 +414,20 @@ static struct ctl_table kern_table[] = { · 413 unmodified lines
414414 .mode = 0644,
415415 .proc_handler = sched_rr_handler,
416416 },
417+ {
418+ .procname = "sched_dl_period_us",
419+ .data = &sysctl_sched_dl_period,
420+ .maxlen = sizeof(unsigned int),
421+ .mode = 0644,
422+ .proc_handler = sched_dl_handler,
423+ },
424+ {
425+ .procname = "sched_dl_runtime_us",
426+ .data = &sysctl_sched_dl_runtime,
427+ .maxlen = sizeof(int),
428+ .mode = 0644,
429+ .proc_handler = sched_dl_handler,
430+ },
417431 #ifdef CONFIG_SCHED_AUTOGROUP
418432 {
419433 .procname = "sched_autogroup_enabled",