History

Merge tag 'sched_urgent_for_v5.16_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler fixes from Borislav Petkov: - Avoid touching ~100 config files in order to be able to select the preemption model - clear cluster CPU masks too, on the CPU unplug path - prevent use-after-free in cfs - Prevent a race condition when updating CPU cache domains - Factor out common shared part of smp_prepare_cpus() into a common helper which can be called by both baremetal and Xen, in order to fix a booting of Xen PV guests * tag 'sched_urgent_for_v5.16_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: preempt: Restore preemption model selection configs arch_topology: Fix missing clear cluster_cpumask in remove_cpu_topology() sched/fair: Prevent dead task groups from regaining cfs_rq's sched/core: Mitigate race cpus_share_cache()/update_top_cache_domain() x86/smp: Factor out parts of native_smp_prepare_cpus()

Linus Torvalds committed 5 years ago

Commit fc661f22 parents f7018be a8b769113 files · +96 −59Browse files

Showing the changes to kernel (6 of 13 files). Show all

Unified · Split

6 files
  1. kernel/Kconfig.preempt +21 −21
  2. kernel/sched/autogroup.c +1 −1
  3. kernel/sched/core.c +41 −12
  4. kernel/sched/fair.c +2 −2
  5. kernel/sched/rt.c +9 −3
  6. kernel/sched/sched.h +2 −1
kernel/Kconfig.preempt+21−21
@@ -1,12 +1,23 @@
1# SPDX-License-Identifier: GPL-2.0-only1# SPDX-License-Identifier: GPL-2.0-only
2 2
3config PREEMPT_NONE_BUILD
4 bool
5
6config PREEMPT_VOLUNTARY_BUILD
7 bool
8
9config PREEMPT_BUILD
10 bool
11 select PREEMPTION
12 select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK
13
3choice14choice
4 prompt "Preemption Model"15 prompt "Preemption Model"
5 default PREEMPT_NONE_BEHAVIOUR16 default PREEMPT_NONE
6 17
7config PREEMPT_NONE_BEHAVIOUR18config PREEMPT_NONE
8 bool "No Forced Preemption (Server)"19 bool "No Forced Preemption (Server)"
9 select PREEMPT_NONE if !PREEMPT_DYNAMIC20 select PREEMPT_NONE_BUILD if !PREEMPT_DYNAMIC
10 help21 help
11 This is the traditional Linux preemption model, geared towards22 This is the traditional Linux preemption model, geared towards
12 throughput. It will still provide good latencies most of the23 throughput. It will still provide good latencies most of the
@@ -18,10 +29,10 @@ config PREEMPT_NONE_BEHAVIOUR · 5 unmodified lines
18 raw processing power of the kernel, irrespective of scheduling29 raw processing power of the kernel, irrespective of scheduling
19 latencies.30 latencies.
20 31
21config PREEMPT_VOLUNTARY_BEHAVIOUR32config PREEMPT_VOLUNTARY
22 bool "Voluntary Kernel Preemption (Desktop)"33 bool "Voluntary Kernel Preemption (Desktop)"
23 depends on !ARCH_NO_PREEMPT34 depends on !ARCH_NO_PREEMPT
24 select PREEMPT_VOLUNTARY if !PREEMPT_DYNAMIC35 select PREEMPT_VOLUNTARY_BUILD if !PREEMPT_DYNAMIC
25 help36 help
26 This option reduces the latency of the kernel by adding more37 This option reduces the latency of the kernel by adding more
27 "explicit preemption points" to the kernel code. These new38 "explicit preemption points" to the kernel code. These new
@@ -37,10 +48,10 @@ config PREEMPT_VOLUNTARY_BEHAVIOUR · 9 unmodified lines
37 48
38 Select this if you are building a kernel for a desktop system.49 Select this if you are building a kernel for a desktop system.
39 50
40config PREEMPT_BEHAVIOUR51config PREEMPT
41 bool "Preemptible Kernel (Low-Latency Desktop)"52 bool "Preemptible Kernel (Low-Latency Desktop)"
42 depends on !ARCH_NO_PREEMPT53 depends on !ARCH_NO_PREEMPT
43 select PREEMPT54 select PREEMPT_BUILD
44 help55 help
45 This option reduces the latency of the kernel by making56 This option reduces the latency of the kernel by making
46 all kernel code (that is not executing in a critical section)57 all kernel code (that is not executing in a critical section)
@@ -58,7 +69,7 @@ config PREEMPT_BEHAVIOUR · 11 unmodified lines
58 69
59config PREEMPT_RT70config PREEMPT_RT
60 bool "Fully Preemptible Kernel (Real-Time)"71 bool "Fully Preemptible Kernel (Real-Time)"
61 depends on EXPERT && ARCH_SUPPORTS_RT && !PREEMPT_DYNAMIC72 depends on EXPERT && ARCH_SUPPORTS_RT
62 select PREEMPTION73 select PREEMPTION
63 help74 help
64 This option turns the kernel into a real-time kernel by replacing75 This option turns the kernel into a real-time kernel by replacing
@@ -75,17 +86,6 @@ config PREEMPT_RT · 10 unmodified lines
75 86
76endchoice87endchoice
77 88
78config PREEMPT_NONE
79 bool
80
81config PREEMPT_VOLUNTARY
82 bool
83
84config PREEMPT
85 bool
86 select PREEMPTION
87 select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK
88
89config PREEMPT_COUNT89config PREEMPT_COUNT
90 bool90 bool
91 91
@@ -95,8 +95,8 @@ config PREEMPTION · 3 unmodified lines
95 95
96config PREEMPT_DYNAMIC96config PREEMPT_DYNAMIC
97 bool "Preemption behaviour defined on boot"97 bool "Preemption behaviour defined on boot"
98 depends on HAVE_PREEMPT_DYNAMIC98 depends on HAVE_PREEMPT_DYNAMIC && !PREEMPT_RT
99 select PREEMPT99 select PREEMPT_BUILD
100 default y100 default y
101 help101 help
102 This option allows to define the preemption model on the kernel102 This option allows to define the preemption model on the kernel
kernel/sched/autogroup.c+1−1
@@ -31,7 +31,7 @@ static inline void autogroup_destroy(struct kref *kref) · 30 unmodified lines
31 ag->tg->rt_se = NULL;31 ag->tg->rt_se = NULL;
32 ag->tg->rt_rq = NULL;32 ag->tg->rt_rq = NULL;
33#endif33#endif
34 sched_offline_group(ag->tg);34 sched_release_group(ag->tg);
35 sched_destroy_group(ag->tg);35 sched_destroy_group(ag->tg);
36}36}
37 37
kernel/sched/core.c+41−12
@@ -3726,6 +3726,9 @@ out: · 3725 unmodified lines
3726 3726
3727bool cpus_share_cache(int this_cpu, int that_cpu)3727bool cpus_share_cache(int this_cpu, int that_cpu)
3728{3728{
3729 if (this_cpu == that_cpu)
3730 return true;
3731
3729 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);3732 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
3730}3733}
3731 3734
@@ -6625,13 +6628,13 @@ __setup("preempt=", setup_preempt_mode); · 2893 unmodified lines
6625static void __init preempt_dynamic_init(void)6628static void __init preempt_dynamic_init(void)
6626{6629{
6627 if (preempt_dynamic_mode == preempt_dynamic_undefined) {6630 if (preempt_dynamic_mode == preempt_dynamic_undefined) {
6628 if (IS_ENABLED(CONFIG_PREEMPT_NONE_BEHAVIOUR)) {6631 if (IS_ENABLED(CONFIG_PREEMPT_NONE)) {
6629 sched_dynamic_update(preempt_dynamic_none);6632 sched_dynamic_update(preempt_dynamic_none);
6630 } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY_BEHAVIOUR)) {6633 } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) {
6631 sched_dynamic_update(preempt_dynamic_voluntary);6634 sched_dynamic_update(preempt_dynamic_voluntary);
6632 } else {6635 } else {
6633 /* Default static call setting, nothing to do */6636 /* Default static call setting, nothing to do */
6634 WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT_BEHAVIOUR));6637 WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT));
6635 preempt_dynamic_mode = preempt_dynamic_full;6638 preempt_dynamic_mode = preempt_dynamic_full;
6636 pr_info("Dynamic Preempt: full\n");6639 pr_info("Dynamic Preempt: full\n");
6637 }6640 }
@@ -9716,6 +9719,22 @@ static void sched_free_group(struct task_group *tg) · 3078 unmodified lines
9716 kmem_cache_free(task_group_cache, tg);9719 kmem_cache_free(task_group_cache, tg);
9717}9720}
9718 9721
9722static void sched_free_group_rcu(struct rcu_head *rcu)
9723{
9724 sched_free_group(container_of(rcu, struct task_group, rcu));
9725}
9726
9727static void sched_unregister_group(struct task_group *tg)
9728{
9729 unregister_fair_sched_group(tg);
9730 unregister_rt_sched_group(tg);
9731 /*
9732 * We have to wait for yet another RCU grace period to expire, as
9733 * print_cfs_stats() might run concurrently.
9734 */
9735 call_rcu(&tg->rcu, sched_free_group_rcu);
9736}
9737
9719/* allocate runqueue etc for a new task group */9738/* allocate runqueue etc for a new task group */
9720struct task_group *sched_create_group(struct task_group *parent)9739struct task_group *sched_create_group(struct task_group *parent)
9721{9740{
@@ -9759,25 +9778,35 @@ void sched_online_group(struct task_group *tg, struct task_group *parent) · 37 unmodified lines
9759}9778}
9760 9779
9761/* rcu callback to free various structures associated with a task group */9780/* rcu callback to free various structures associated with a task group */
9762static void sched_free_group_rcu(struct rcu_head *rhp)9781static void sched_unregister_group_rcu(struct rcu_head *rhp)
9763{9782{
9764 /* Now it should be safe to free those cfs_rqs: */9783 /* Now it should be safe to free those cfs_rqs: */
9765 sched_free_group(container_of(rhp, struct task_group, rcu));9784 sched_unregister_group(container_of(rhp, struct task_group, rcu));
9766}9785}
9767 9786
9768void sched_destroy_group(struct task_group *tg)9787void sched_destroy_group(struct task_group *tg)
9769{9788{
9770 /* Wait for possible concurrent references to cfs_rqs complete: */9789 /* Wait for possible concurrent references to cfs_rqs complete: */
9771 call_rcu(&tg->rcu, sched_free_group_rcu);9790 call_rcu(&tg->rcu, sched_unregister_group_rcu);
9772}9791}
9773 9792
9774void sched_offline_group(struct task_group *tg)9793void sched_release_group(struct task_group *tg)
9775{9794{
9776 unsigned long flags;9795 unsigned long flags;
9777 9796
9778 /* End participation in shares distribution: */9797 /*
9779 unregister_fair_sched_group(tg);9798 * Unlink first, to avoid walk_tg_tree_from() from finding us (via
9780 9799 * sched_cfs_period_timer()).
9800 *
9801 * For this to be effective, we have to wait for all pending users of
9802 * this task group to leave their RCU critical section to ensure no new
9803 * user will see our dying task group any more. Specifically ensure
9804 * that tg_unthrottle_up() won't add decayed cfs_rq's to it.
9805 *
9806 * We therefore defer calling unregister_fair_sched_group() to
9807 * sched_unregister_group() which is guarantied to get called only after the
9808 * current RCU grace period has expired.
9809 */
9781 spin_lock_irqsave(&task_group_lock, flags);9810 spin_lock_irqsave(&task_group_lock, flags);
9782 list_del_rcu(&tg->list);9811 list_del_rcu(&tg->list);
9783 list_del_rcu(&tg->siblings);9812 list_del_rcu(&tg->siblings);
@@ -9896,7 +9925,7 @@ static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) · 112 unmodified lines
9896{9925{
9897 struct task_group *tg = css_tg(css);9926 struct task_group *tg = css_tg(css);
9898 9927
9899 sched_offline_group(tg);9928 sched_release_group(tg);
9900}9929}
9901 9930
9902static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)9931static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
@@ -9906,7 +9935,7 @@ static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) · 3 unmodified lines
9906 /*9935 /*
9907 * Relies on the RCU grace period between css_released() and this.9936 * Relies on the RCU grace period between css_released() and this.
9908 */9937 */
9909 sched_free_group(tg);9938 sched_unregister_group(tg);
9910}9939}
9911 9940
9912/*9941/*
kernel/sched/fair.c+2−2
@@ -11456,8 +11456,6 @@ void free_fair_sched_group(struct task_group *tg) · 11455 unmodified lines
11456{11456{
11457 int i;11457 int i;
11458 11458
11459 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
11460
11461 for_each_possible_cpu(i) {11459 for_each_possible_cpu(i) {
11462 if (tg->cfs_rq)11460 if (tg->cfs_rq)
11463 kfree(tg->cfs_rq[i]);11461 kfree(tg->cfs_rq[i]);
@@ -11534,6 +11532,8 @@ void unregister_fair_sched_group(struct task_group *tg) · 70 unmodified lines
11534 struct rq *rq;11532 struct rq *rq;
11535 int cpu;11533 int cpu;
11536 11534
11535 destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
11536
11537 for_each_possible_cpu(cpu) {11537 for_each_possible_cpu(cpu) {
11538 if (tg->se[cpu])11538 if (tg->se[cpu])
11539 remove_entity_load_avg(tg->se[cpu]);11539 remove_entity_load_avg(tg->se[cpu]);
kernel/sched/rt.c+9−3
@@ -137,13 +137,17 @@ static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) · 136 unmodified lines
137 return rt_rq->rq;137 return rt_rq->rq;
138}138}
139 139
140void free_rt_sched_group(struct task_group *tg)140void unregister_rt_sched_group(struct task_group *tg)
141{141{
142 int i;
143
144 if (tg->rt_se)142 if (tg->rt_se)
145 destroy_rt_bandwidth(&tg->rt_bandwidth);143 destroy_rt_bandwidth(&tg->rt_bandwidth);
146 144
145}
146
147void free_rt_sched_group(struct task_group *tg)
148{
149 int i;
150
147 for_each_possible_cpu(i) {151 for_each_possible_cpu(i) {
148 if (tg->rt_rq)152 if (tg->rt_rq)
149 kfree(tg->rt_rq[i]);153 kfree(tg->rt_rq[i]);
@@ -250,6 +254,8 @@ static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) · 100 unmodified lines
250 return &rq->rt;254 return &rq->rt;
251}255}
252 256
257void unregister_rt_sched_group(struct task_group *tg) { }
258
253void free_rt_sched_group(struct task_group *tg) { }259void free_rt_sched_group(struct task_group *tg) { }
254 260
255int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)261int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
kernel/sched/sched.h+2−1
@@ -488,6 +488,7 @@ extern void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b); · 487 unmodified lines
488extern void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b);488extern void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b);
489extern void unthrottle_cfs_rq(struct cfs_rq *cfs_rq);489extern void unthrottle_cfs_rq(struct cfs_rq *cfs_rq);
490 490
491extern void unregister_rt_sched_group(struct task_group *tg);
491extern void free_rt_sched_group(struct task_group *tg);492extern void free_rt_sched_group(struct task_group *tg);
492extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent);493extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent);
493extern void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,494extern void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
@@ -503,7 +504,7 @@ extern struct task_group *sched_create_group(struct task_group *parent); · 9 unmodified lines
503extern void sched_online_group(struct task_group *tg,504extern void sched_online_group(struct task_group *tg,
504 struct task_group *parent);505 struct task_group *parent);
505extern void sched_destroy_group(struct task_group *tg);506extern void sched_destroy_group(struct task_group *tg);
506extern void sched_offline_group(struct task_group *tg);507extern void sched_release_group(struct task_group *tg);
507 508
508extern void sched_move_task(struct task_struct *tsk);509extern void sched_move_task(struct task_struct *tsk);
509 510