History

Merge tag 'sched_urgent_for_v5.16_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler fixes from Borislav Petkov: - Avoid touching ~100 config files in order to be able to select the preemption model - clear cluster CPU masks too, on the CPU unplug path - prevent use-after-free in cfs - Prevent a race condition when updating CPU cache domains - Factor out common shared part of smp_prepare_cpus() into a common helper which can be called by both baremetal and Xen, in order to fix a booting of Xen PV guests * tag 'sched_urgent_for_v5.16_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: preempt: Restore preemption model selection configs arch_topology: Fix missing clear cluster_cpumask in remove_cpu_topology() sched/fair: Prevent dead task groups from regaining cfs_rq's sched/core: Mitigate race cpus_share_cache()/update_top_cache_domain() x86/smp: Factor out parts of native_smp_prepare_cpus()

Linus Torvalds committed 5 years ago

Commit fc661f22 parents f7018be a8b769113 files · +96 −59Browse files

Showing the changes to kernel (6 of 13 files). Show all

Unified · Split

6 files
  1. kernel/Kconfig.preempt +21 −21
  2. kernel/sched/autogroup.c +1 −1
  3. kernel/sched/core.c +41 −12
  4. kernel/sched/fair.c +2 −2
  5. kernel/sched/rt.c +9 −3
  6. kernel/sched/sched.h +2 −1
kernel/Kconfig.preempt+21−21
@@ -1,12 +1,23 @@
11 # SPDX-License-Identifier: GPL-2.0-only
22
3+config PREEMPT_NONE_BUILD
4+ bool
5+
6+config PREEMPT_VOLUNTARY_BUILD
7+ bool
8+
9+config PREEMPT_BUILD
10+ bool
11+ select PREEMPTION
12+ select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK
13+
314 choice
415 prompt "Preemption Model"
5− default PREEMPT_NONE_BEHAVIOUR
16+ default PREEMPT_NONE
617
7−config PREEMPT_NONE_BEHAVIOUR
18+config PREEMPT_NONE
819 bool "No Forced Preemption (Server)"
9− select PREEMPT_NONE if !PREEMPT_DYNAMIC
20+ select PREEMPT_NONE_BUILD if !PREEMPT_DYNAMIC
1021 help
1122 This is the traditional Linux preemption model, geared towards
1223 throughput. It will still provide good latencies most of the
@@ -18,10 +29,10 @@ config PREEMPT_NONE_BEHAVIOUR · 5 unmodified lines
1829 raw processing power of the kernel, irrespective of scheduling
1930 latencies.
2031
21−config PREEMPT_VOLUNTARY_BEHAVIOUR
32+config PREEMPT_VOLUNTARY
2233 bool "Voluntary Kernel Preemption (Desktop)"
2334 depends on !ARCH_NO_PREEMPT
24− select PREEMPT_VOLUNTARY if !PREEMPT_DYNAMIC
35+ select PREEMPT_VOLUNTARY_BUILD if !PREEMPT_DYNAMIC
2536 help
2637 This option reduces the latency of the kernel by adding more
2738 "explicit preemption points" to the kernel code. These new
@@ -37,10 +48,10 @@ config PREEMPT_VOLUNTARY_BEHAVIOUR · 9 unmodified lines
3748
3849 Select this if you are building a kernel for a desktop system.
3950
40−config PREEMPT_BEHAVIOUR
51+config PREEMPT
4152 bool "Preemptible Kernel (Low-Latency Desktop)"
4253 depends on !ARCH_NO_PREEMPT
43− select PREEMPT
54+ select PREEMPT_BUILD
4455 help
4556 This option reduces the latency of the kernel by making
4657 all kernel code (that is not executing in a critical section)
@@ -58,7 +69,7 @@ config PREEMPT_BEHAVIOUR · 11 unmodified lines
5869
5970 config PREEMPT_RT
6071 bool "Fully Preemptible Kernel (Real-Time)"
61− depends on EXPERT && ARCH_SUPPORTS_RT && !PREEMPT_DYNAMIC
72+ depends on EXPERT && ARCH_SUPPORTS_RT
6273 select PREEMPTION
6374 help
6475 This option turns the kernel into a real-time kernel by replacing
@@ -75,17 +86,6 @@ config PREEMPT_RT · 10 unmodified lines
7586
7687 endchoice
7788
78−config PREEMPT_NONE
79− bool
80−
81−config PREEMPT_VOLUNTARY
82− bool
83−
84−config PREEMPT
85− bool
86− select PREEMPTION
87− select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK
88−
8989 config PREEMPT_COUNT
9090 bool
9191
@@ -95,8 +95,8 @@ config PREEMPTION · 3 unmodified lines
9595
9696 config PREEMPT_DYNAMIC
9797 bool "Preemption behaviour defined on boot"
98− depends on HAVE_PREEMPT_DYNAMIC
99− select PREEMPT
98+ depends on HAVE_PREEMPT_DYNAMIC && !PREEMPT_RT
99+ select PREEMPT_BUILD
100100 default y
101101 help
102102 This option allows to define the preemption model on the kernel
kernel/sched/autogroup.c+1−1
@@ -31,7 +31,7 @@ static inline void autogroup_destroy(struct kref *kref) · 30 unmodified lines
3131 ag->tg->rt_se = NULL;
3232 ag->tg->rt_rq = NULL;
3333 #endif
34− sched_offline_group(ag->tg);
34+ sched_release_group(ag->tg);
3535 sched_destroy_group(ag->tg);
3636 }
3737
kernel/sched/core.c+41−12
@@ -3726,6 +3726,9 @@ out: · 3725 unmodified lines
37263726
37273727 bool cpus_share_cache(int this_cpu, int that_cpu)
37283728 {
3729+ if (this_cpu == that_cpu)
3730+ return true;
3731+
37293732 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
37303733 }
37313734
@@ -6625,13 +6628,13 @@ __setup("preempt=", setup_preempt_mode); · 2893 unmodified lines
66256628 static void __init preempt_dynamic_init(void)
66266629 {
66276630 if (preempt_dynamic_mode == preempt_dynamic_undefined) {
6628− if (IS_ENABLED(CONFIG_PREEMPT_NONE_BEHAVIOUR)) {
6631+ if (IS_ENABLED(CONFIG_PREEMPT_NONE)) {
66296632 sched_dynamic_update(preempt_dynamic_none);
6630− } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY_BEHAVIOUR)) {
6633+ } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) {
66316634 sched_dynamic_update(preempt_dynamic_voluntary);
66326635 } else {
66336636 /* Default static call setting, nothing to do */
6634− WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT_BEHAVIOUR));
6637+ WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT));
66356638 preempt_dynamic_mode = preempt_dynamic_full;
66366639 pr_info("Dynamic Preempt: full\n");
66376640 }
@@ -9716,6 +9719,22 @@ static void sched_free_group(struct task_group *tg) · 3078 unmodified lines
97169719 kmem_cache_free(task_group_cache, tg);
97179720 }
97189721
9722+static void sched_free_group_rcu(struct rcu_head *rcu)
9723+{
9724+ sched_free_group(container_of(rcu, struct task_group, rcu));
9725+}
9726+
9727+static void sched_unregister_group(struct task_group *tg)
9728+{
9729+ unregister_fair_sched_group(tg);
9730+ unregister_rt_sched_group(tg);
9731+ /*
9732+ * We have to wait for yet another RCU grace period to expire, as
9733+ * print_cfs_stats() might run concurrently.
9734+ */
9735+ call_rcu(&tg->rcu, sched_free_group_rcu);
9736+}
9737+
97199738 /* allocate runqueue etc for a new task group */
97209739 struct task_group *sched_create_group(struct task_group *parent)
97219740 {
@@ -9759,25 +9778,35 @@ void sched_online_group(struct task_group *tg, struct task_group *parent) · 37 unmodified lines
97599778 }
97609779
97619780 /* rcu callback to free various structures associated with a task group */
9762−static void sched_free_group_rcu(struct rcu_head *rhp)
9781+static void sched_unregister_group_rcu(struct rcu_head *rhp)
97639782 {
97649783 /* Now it should be safe to free those cfs_rqs: */
9765− sched_free_group(container_of(rhp, struct task_group, rcu));
9784+ sched_unregister_group(container_of(rhp, struct task_group, rcu));
97669785 }
97679786
97689787 void sched_destroy_group(struct task_group *tg)
97699788 {
97709789 /* Wait for possible concurrent references to cfs_rqs complete: */
9771− call_rcu(&tg->rcu, sched_free_group_rcu);
9790+ call_rcu(&tg->rcu, sched_unregister_group_rcu);
97729791 }
97739792
9774−void sched_offline_group(struct task_group *tg)
9793+void sched_release_group(struct task_group *tg)
97759794 {
97769795 unsigned long flags;
97779796
9778− /* End participation in shares distribution: */
9779− unregister_fair_sched_group(tg);
9780−
9797+ /*
9798+ * Unlink first, to avoid walk_tg_tree_from() from finding us (via
9799+ * sched_cfs_period_timer()).
9800+ *
9801+ * For this to be effective, we have to wait for all pending users of
9802+ * this task group to leave their RCU critical section to ensure no new
9803+ * user will see our dying task group any more. Specifically ensure
9804+ * that tg_unthrottle_up() won't add decayed cfs_rq's to it.
9805+ *
9806+ * We therefore defer calling unregister_fair_sched_group() to
9807+ * sched_unregister_group() which is guarantied to get called only after the
9808+ * current RCU grace period has expired.
9809+ */
97819810 spin_lock_irqsave(&task_group_lock, flags);
97829811 list_del_rcu(&tg->list);
97839812 list_del_rcu(&tg->siblings);
@@ -9896,7 +9925,7 @@ static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) · 112 unmodified lines
98969925 {
98979926 struct task_group *tg = css_tg(css);
98989927
9899− sched_offline_group(tg);
9928+ sched_release_group(tg);
99009929 }
99019930
99029931 static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
@@ -9906,7 +9935,7 @@ static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) · 3 unmodified lines
99069935 /*
99079936 * Relies on the RCU grace period between css_released() and this.
99089937 */
9909− sched_free_group(tg);
9938+ sched_unregister_group(tg);
99109939 }
99119940
99129941 /*
kernel/sched/fair.c+2−2
@@ -11456,8 +11456,6 @@ void free_fair_sched_group(struct task_group *tg) · 11455 unmodified lines
1145611456 {
1145711457 int i;
1145811458
11459− destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
11460−
1146111459 for_each_possible_cpu(i) {
1146211460 if (tg->cfs_rq)
1146311461 kfree(tg->cfs_rq[i]);
@@ -11534,6 +11532,8 @@ void unregister_fair_sched_group(struct task_group *tg) · 70 unmodified lines
1153411532 struct rq *rq;
1153511533 int cpu;
1153611534
11535+ destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
11536+
1153711537 for_each_possible_cpu(cpu) {
1153811538 if (tg->se[cpu])
1153911539 remove_entity_load_avg(tg->se[cpu]);
kernel/sched/rt.c+9−3
@@ -137,13 +137,17 @@ static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) · 136 unmodified lines
137137 return rt_rq->rq;
138138 }
139139
140−void free_rt_sched_group(struct task_group *tg)
140+void unregister_rt_sched_group(struct task_group *tg)
141141 {
142− int i;
143−
144142 if (tg->rt_se)
145143 destroy_rt_bandwidth(&tg->rt_bandwidth);
146144
145+}
146+
147+void free_rt_sched_group(struct task_group *tg)
148+{
149+ int i;
150+
147151 for_each_possible_cpu(i) {
148152 if (tg->rt_rq)
149153 kfree(tg->rt_rq[i]);
@@ -250,6 +254,8 @@ static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) · 100 unmodified lines
250254 return &rq->rt;
251255 }
252256
257+void unregister_rt_sched_group(struct task_group *tg) { }
258+
253259 void free_rt_sched_group(struct task_group *tg) { }
254260
255261 int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
kernel/sched/sched.h+2−1
@@ -488,6 +488,7 @@ extern void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b); · 487 unmodified lines
488488 extern void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b);
489489 extern void unthrottle_cfs_rq(struct cfs_rq *cfs_rq);
490490
491+extern void unregister_rt_sched_group(struct task_group *tg);
491492 extern void free_rt_sched_group(struct task_group *tg);
492493 extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent);
493494 extern void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
@@ -503,7 +504,7 @@ extern struct task_group *sched_create_group(struct task_group *parent); · 9 unmodified lines
503504 extern void sched_online_group(struct task_group *tg,
504505 struct task_group *parent);
505506 extern void sched_destroy_group(struct task_group *tg);
506−extern void sched_offline_group(struct task_group *tg);
507+extern void sched_release_group(struct task_group *tg);
507508
508509 extern void sched_move_task(struct task_struct *tsk);
509510