History

Merge tag 'sched_urgent_for_v5.16_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler fixes from Borislav Petkov: - Avoid touching ~100 config files in order to be able to select the preemption model - clear cluster CPU masks too, on the CPU unplug path - prevent use-after-free in cfs - Prevent a race condition when updating CPU cache domains - Factor out common shared part of smp_prepare_cpus() into a common helper which can be called by both baremetal and Xen, in order to fix a booting of Xen PV guests * tag 'sched_urgent_for_v5.16_rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: preempt: Restore preemption model selection configs arch_topology: Fix missing clear cluster_cpumask in remove_cpu_topology() sched/fair: Prevent dead task groups from regaining cfs_rq's sched/core: Mitigate race cpus_share_cache()/update_top_cache_domain() x86/smp: Factor out parts of native_smp_prepare_cpus()

Linus Torvalds committed 5 years ago

Commit fc661f22 parents f7018be a8b769113 files · +96 −59Browse files

Unified · Split

13 files
  1. arch/x86/include/asm/smp.h +1 −0
  2. arch/x86/kernel/smpboot.c +12 −6
  3. arch/x86/xen/smp_pv.c +2 −10
  4. drivers/base/arch_topology.c +2 −0
  5. include/linux/kernel.h +1 −1
  6. include/linux/vermagic.h +1 −1
  7. init/Makefile +1 −1
  8. kernel/Kconfig.preempt +21 −21
  9. kernel/sched/autogroup.c +1 −1
  10. kernel/sched/core.c +41 −12
  11. kernel/sched/fair.c +2 −2
  12. kernel/sched/rt.c +9 −3
  13. kernel/sched/sched.h +2 −1
arch/x86/include/asm/smp.h+1−0
@@ -126,6 +126,7 @@ static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask) · 125 unmodified lines
126126
127127 void cpu_disable_common(void);
128128 void native_smp_prepare_boot_cpu(void);
129+void smp_prepare_cpus_common(void);
129130 void native_smp_prepare_cpus(unsigned int max_cpus);
130131 void calculate_max_logical_packages(void);
131132 void native_smp_cpus_done(unsigned int max_cpus);
arch/x86/kernel/smpboot.c+12−6
@@ -1350,12 +1350,7 @@ static void __init smp_get_logical_apicid(void) · 1349 unmodified lines
13501350 cpu0_logical_apicid = GET_APIC_LOGICAL_ID(apic_read(APIC_LDR));
13511351 }
13521352
1353−/*
1354− * Prepare for SMP bootup.
1355− * @max_cpus: configured maximum number of CPUs, It is a legacy parameter
1356− * for common interface support.
1357− */
1358−void __init native_smp_prepare_cpus(unsigned int max_cpus)
1353+void __init smp_prepare_cpus_common(void)
13591354 {
13601355 unsigned int i;
13611356
@@ -1386,6 +1381,17 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus) · 24 unmodified lines
13861381 set_sched_topology(x86_topology);
13871382
13881383 set_cpu_sibling_map(0);
1384+}
1385+
1386+/*
1387+ * Prepare for SMP bootup.
1388+ * @max_cpus: configured maximum number of CPUs, It is a legacy parameter
1389+ * for common interface support.
1390+ */
1391+void __init native_smp_prepare_cpus(unsigned int max_cpus)
1392+{
1393+ smp_prepare_cpus_common();
1394+
13891395 init_freq_invariance(false, false);
13901396 smp_sanity_check();
13911397
arch/x86/xen/smp_pv.c+2−10
@@ -225,7 +225,6 @@ static void __init xen_pv_smp_prepare_boot_cpu(void) · 224 unmodified lines
225225 static void __init xen_pv_smp_prepare_cpus(unsigned int max_cpus)
226226 {
227227 unsigned cpu;
228− unsigned int i;
229228
230229 if (skip_ioapic_setup) {
231230 char *m = (max_cpus == 0) ?
@@ -238,16 +237,9 @@ static void __init xen_pv_smp_prepare_cpus(unsigned int max_cpus) · 6 unmodified lines
238237 }
239238 xen_init_lock_cpu(0);
240239
241− smp_store_boot_cpu_info();
242− cpu_data(0).x86_max_cores = 1;
240+ smp_prepare_cpus_common();
243241
244− for_each_possible_cpu(i) {
245− zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
246− zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
247− zalloc_cpumask_var(&per_cpu(cpu_die_map, i), GFP_KERNEL);
248− zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
249− }
250− set_cpu_sibling_map(0);
242+ cpu_data(0).x86_max_cores = 1;
251243
252244 speculative_store_bypass_ht_init();
253245
drivers/base/arch_topology.c+2−0
@@ -677,6 +677,8 @@ void remove_cpu_topology(unsigned int cpu) · 676 unmodified lines
677677 cpumask_clear_cpu(cpu, topology_core_cpumask(sibling));
678678 for_each_cpu(sibling, topology_sibling_cpumask(cpu))
679679 cpumask_clear_cpu(cpu, topology_sibling_cpumask(sibling));
680+ for_each_cpu(sibling, topology_cluster_cpumask(cpu))
681+ cpumask_clear_cpu(cpu, topology_cluster_cpumask(sibling));
680682 for_each_cpu(sibling, topology_llc_cpumask(cpu))
681683 cpumask_clear_cpu(cpu, topology_llc_cpumask(sibling));
682684
include/linux/kernel.h+1−1
@@ -85,7 +85,7 @@ · 84 unmodified lines
8585 struct completion;
8686 struct user;
8787
88−#ifdef CONFIG_PREEMPT_VOLUNTARY
88+#ifdef CONFIG_PREEMPT_VOLUNTARY_BUILD
8989
9090 extern int __cond_resched(void);
9191 # define might_resched() __cond_resched()
include/linux/vermagic.h+1−1
@@ -15,7 +15,7 @@ · 14 unmodified lines
1515 #else
1616 #define MODULE_VERMAGIC_SMP ""
1717 #endif
18−#ifdef CONFIG_PREEMPT
18+#ifdef CONFIG_PREEMPT_BUILD
1919 #define MODULE_VERMAGIC_PREEMPT "preempt "
2020 #elif defined(CONFIG_PREEMPT_RT)
2121 #define MODULE_VERMAGIC_PREEMPT "preempt_rt "
init/Makefile+1−1
@@ -30,7 +30,7 @@ $(obj)/version.o: include/generated/compile.h · 29 unmodified lines
3030 quiet_cmd_compile.h = CHK $@
3131 cmd_compile.h = \
3232 $(CONFIG_SHELL) $(srctree)/scripts/mkcompile_h $@ \
33− "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" \
33+ "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT_BUILD)" \
3434 "$(CONFIG_PREEMPT_RT)" $(CONFIG_CC_VERSION_TEXT) "$(LD)"
3535
3636 include/generated/compile.h: FORCE
kernel/Kconfig.preempt+21−21
@@ -1,12 +1,23 @@
11 # SPDX-License-Identifier: GPL-2.0-only
22
3+config PREEMPT_NONE_BUILD
4+ bool
5+
6+config PREEMPT_VOLUNTARY_BUILD
7+ bool
8+
9+config PREEMPT_BUILD
10+ bool
11+ select PREEMPTION
12+ select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK
13+
314 choice
415 prompt "Preemption Model"
5− default PREEMPT_NONE_BEHAVIOUR
16+ default PREEMPT_NONE
617
7−config PREEMPT_NONE_BEHAVIOUR
18+config PREEMPT_NONE
819 bool "No Forced Preemption (Server)"
9− select PREEMPT_NONE if !PREEMPT_DYNAMIC
20+ select PREEMPT_NONE_BUILD if !PREEMPT_DYNAMIC
1021 help
1122 This is the traditional Linux preemption model, geared towards
1223 throughput. It will still provide good latencies most of the
@@ -18,10 +29,10 @@ config PREEMPT_NONE_BEHAVIOUR · 5 unmodified lines
1829 raw processing power of the kernel, irrespective of scheduling
1930 latencies.
2031
21−config PREEMPT_VOLUNTARY_BEHAVIOUR
32+config PREEMPT_VOLUNTARY
2233 bool "Voluntary Kernel Preemption (Desktop)"
2334 depends on !ARCH_NO_PREEMPT
24− select PREEMPT_VOLUNTARY if !PREEMPT_DYNAMIC
35+ select PREEMPT_VOLUNTARY_BUILD if !PREEMPT_DYNAMIC
2536 help
2637 This option reduces the latency of the kernel by adding more
2738 "explicit preemption points" to the kernel code. These new
@@ -37,10 +48,10 @@ config PREEMPT_VOLUNTARY_BEHAVIOUR · 9 unmodified lines
3748
3849 Select this if you are building a kernel for a desktop system.
3950
40−config PREEMPT_BEHAVIOUR
51+config PREEMPT
4152 bool "Preemptible Kernel (Low-Latency Desktop)"
4253 depends on !ARCH_NO_PREEMPT
43− select PREEMPT
54+ select PREEMPT_BUILD
4455 help
4556 This option reduces the latency of the kernel by making
4657 all kernel code (that is not executing in a critical section)
@@ -58,7 +69,7 @@ config PREEMPT_BEHAVIOUR · 11 unmodified lines
5869
5970 config PREEMPT_RT
6071 bool "Fully Preemptible Kernel (Real-Time)"
61− depends on EXPERT && ARCH_SUPPORTS_RT && !PREEMPT_DYNAMIC
72+ depends on EXPERT && ARCH_SUPPORTS_RT
6273 select PREEMPTION
6374 help
6475 This option turns the kernel into a real-time kernel by replacing
@@ -75,17 +86,6 @@ config PREEMPT_RT · 10 unmodified lines
7586
7687 endchoice
7788
78−config PREEMPT_NONE
79− bool
80−
81−config PREEMPT_VOLUNTARY
82− bool
83−
84−config PREEMPT
85− bool
86− select PREEMPTION
87− select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK
88−
8989 config PREEMPT_COUNT
9090 bool
9191
@@ -95,8 +95,8 @@ config PREEMPTION · 3 unmodified lines
9595
9696 config PREEMPT_DYNAMIC
9797 bool "Preemption behaviour defined on boot"
98− depends on HAVE_PREEMPT_DYNAMIC
99− select PREEMPT
98+ depends on HAVE_PREEMPT_DYNAMIC && !PREEMPT_RT
99+ select PREEMPT_BUILD
100100 default y
101101 help
102102 This option allows to define the preemption model on the kernel
kernel/sched/autogroup.c+1−1
@@ -31,7 +31,7 @@ static inline void autogroup_destroy(struct kref *kref) · 30 unmodified lines
3131 ag->tg->rt_se = NULL;
3232 ag->tg->rt_rq = NULL;
3333 #endif
34− sched_offline_group(ag->tg);
34+ sched_release_group(ag->tg);
3535 sched_destroy_group(ag->tg);
3636 }
3737
kernel/sched/core.c+41−12
@@ -3726,6 +3726,9 @@ out: · 3725 unmodified lines
37263726
37273727 bool cpus_share_cache(int this_cpu, int that_cpu)
37283728 {
3729+ if (this_cpu == that_cpu)
3730+ return true;
3731+
37293732 return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
37303733 }
37313734
@@ -6625,13 +6628,13 @@ __setup("preempt=", setup_preempt_mode); · 2893 unmodified lines
66256628 static void __init preempt_dynamic_init(void)
66266629 {
66276630 if (preempt_dynamic_mode == preempt_dynamic_undefined) {
6628− if (IS_ENABLED(CONFIG_PREEMPT_NONE_BEHAVIOUR)) {
6631+ if (IS_ENABLED(CONFIG_PREEMPT_NONE)) {
66296632 sched_dynamic_update(preempt_dynamic_none);
6630− } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY_BEHAVIOUR)) {
6633+ } else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) {
66316634 sched_dynamic_update(preempt_dynamic_voluntary);
66326635 } else {
66336636 /* Default static call setting, nothing to do */
6634− WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT_BEHAVIOUR));
6637+ WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT));
66356638 preempt_dynamic_mode = preempt_dynamic_full;
66366639 pr_info("Dynamic Preempt: full\n");
66376640 }
@@ -9716,6 +9719,22 @@ static void sched_free_group(struct task_group *tg) · 3078 unmodified lines
97169719 kmem_cache_free(task_group_cache, tg);
97179720 }
97189721
9722+static void sched_free_group_rcu(struct rcu_head *rcu)
9723+{
9724+ sched_free_group(container_of(rcu, struct task_group, rcu));
9725+}
9726+
9727+static void sched_unregister_group(struct task_group *tg)
9728+{
9729+ unregister_fair_sched_group(tg);
9730+ unregister_rt_sched_group(tg);
9731+ /*
9732+ * We have to wait for yet another RCU grace period to expire, as
9733+ * print_cfs_stats() might run concurrently.
9734+ */
9735+ call_rcu(&tg->rcu, sched_free_group_rcu);
9736+}
9737+
97199738 /* allocate runqueue etc for a new task group */
97209739 struct task_group *sched_create_group(struct task_group *parent)
97219740 {
@@ -9759,25 +9778,35 @@ void sched_online_group(struct task_group *tg, struct task_group *parent) · 37 unmodified lines
97599778 }
97609779
97619780 /* rcu callback to free various structures associated with a task group */
9762−static void sched_free_group_rcu(struct rcu_head *rhp)
9781+static void sched_unregister_group_rcu(struct rcu_head *rhp)
97639782 {
97649783 /* Now it should be safe to free those cfs_rqs: */
9765− sched_free_group(container_of(rhp, struct task_group, rcu));
9784+ sched_unregister_group(container_of(rhp, struct task_group, rcu));
97669785 }
97679786
97689787 void sched_destroy_group(struct task_group *tg)
97699788 {
97709789 /* Wait for possible concurrent references to cfs_rqs complete: */
9771− call_rcu(&tg->rcu, sched_free_group_rcu);
9790+ call_rcu(&tg->rcu, sched_unregister_group_rcu);
97729791 }
97739792
9774−void sched_offline_group(struct task_group *tg)
9793+void sched_release_group(struct task_group *tg)
97759794 {
97769795 unsigned long flags;
97779796
9778− /* End participation in shares distribution: */
9779− unregister_fair_sched_group(tg);
9780−
9797+ /*
9798+ * Unlink first, to avoid walk_tg_tree_from() from finding us (via
9799+ * sched_cfs_period_timer()).
9800+ *
9801+ * For this to be effective, we have to wait for all pending users of
9802+ * this task group to leave their RCU critical section to ensure no new
9803+ * user will see our dying task group any more. Specifically ensure
9804+ * that tg_unthrottle_up() won't add decayed cfs_rq's to it.
9805+ *
9806+ * We therefore defer calling unregister_fair_sched_group() to
9807+ * sched_unregister_group() which is guarantied to get called only after the
9808+ * current RCU grace period has expired.
9809+ */
97819810 spin_lock_irqsave(&task_group_lock, flags);
97829811 list_del_rcu(&tg->list);
97839812 list_del_rcu(&tg->siblings);
@@ -9896,7 +9925,7 @@ static void cpu_cgroup_css_released(struct cgroup_subsys_state *css) · 112 unmodified lines
98969925 {
98979926 struct task_group *tg = css_tg(css);
98989927
9899− sched_offline_group(tg);
9928+ sched_release_group(tg);
99009929 }
99019930
99029931 static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
@@ -9906,7 +9935,7 @@ static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) · 3 unmodified lines
99069935 /*
99079936 * Relies on the RCU grace period between css_released() and this.
99089937 */
9909− sched_free_group(tg);
9938+ sched_unregister_group(tg);
99109939 }
99119940
99129941 /*
kernel/sched/fair.c+2−2
@@ -11456,8 +11456,6 @@ void free_fair_sched_group(struct task_group *tg) · 11455 unmodified lines
1145611456 {
1145711457 int i;
1145811458
11459− destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
11460−
1146111459 for_each_possible_cpu(i) {
1146211460 if (tg->cfs_rq)
1146311461 kfree(tg->cfs_rq[i]);
@@ -11534,6 +11532,8 @@ void unregister_fair_sched_group(struct task_group *tg) · 70 unmodified lines
1153411532 struct rq *rq;
1153511533 int cpu;
1153611534
11535+ destroy_cfs_bandwidth(tg_cfs_bandwidth(tg));
11536+
1153711537 for_each_possible_cpu(cpu) {
1153811538 if (tg->se[cpu])
1153911539 remove_entity_load_avg(tg->se[cpu]);
kernel/sched/rt.c+9−3
@@ -137,13 +137,17 @@ static inline struct rq *rq_of_rt_se(struct sched_rt_entity *rt_se) · 136 unmodified lines
137137 return rt_rq->rq;
138138 }
139139
140−void free_rt_sched_group(struct task_group *tg)
140+void unregister_rt_sched_group(struct task_group *tg)
141141 {
142− int i;
143−
144142 if (tg->rt_se)
145143 destroy_rt_bandwidth(&tg->rt_bandwidth);
146144
145+}
146+
147+void free_rt_sched_group(struct task_group *tg)
148+{
149+ int i;
150+
147151 for_each_possible_cpu(i) {
148152 if (tg->rt_rq)
149153 kfree(tg->rt_rq[i]);
@@ -250,6 +254,8 @@ static inline struct rt_rq *rt_rq_of_se(struct sched_rt_entity *rt_se) · 100 unmodified lines
250254 return &rq->rt;
251255 }
252256
257+void unregister_rt_sched_group(struct task_group *tg) { }
258+
253259 void free_rt_sched_group(struct task_group *tg) { }
254260
255261 int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent)
kernel/sched/sched.h+2−1
@@ -488,6 +488,7 @@ extern void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b); · 487 unmodified lines
488488 extern void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b);
489489 extern void unthrottle_cfs_rq(struct cfs_rq *cfs_rq);
490490
491+extern void unregister_rt_sched_group(struct task_group *tg);
491492 extern void free_rt_sched_group(struct task_group *tg);
492493 extern int alloc_rt_sched_group(struct task_group *tg, struct task_group *parent);
493494 extern void init_tg_rt_entry(struct task_group *tg, struct rt_rq *rt_rq,
@@ -503,7 +504,7 @@ extern struct task_group *sched_create_group(struct task_group *parent); · 9 unmodified lines
503504 extern void sched_online_group(struct task_group *tg,
504505 struct task_group *parent);
505506 extern void sched_destroy_group(struct task_group *tg);
506−extern void sched_offline_group(struct task_group *tg);
507+extern void sched_release_group(struct task_group *tg);
507508
508509 extern void sched_move_task(struct task_struct *tsk);
509510