History

sched: Simplify ifdeffery around cpu_smt_mask

Now, that cpu_smt_mask is defined as cpumask_of(cpu) for CONFIG_SCHED_SMT=n, it is possible to get rid of the ifdeffery. Effectively, - This makes sched_smt_present is defined always - cpumask_weight(cpumask_of(cpu)) == 1. So sched_smt_present_inc/dec will never enable the sched_smt_present. Which is expected. - Paths that were compile-time eliminated become runtime guarded using static keys. - Defines set_idle_cores, test_idle_cores, etc which could likely benefit the CONFIG_SCHED_SMT=n systems to use the same optimizations within the LLC at wakeups. - This will expose sched_smt_present symbol for CONFIG_SCHED_SMT=n. Likely not a concern. - There is a bloat of code CONFIG_SCHED_SMT=n. (NR_CPUS=2048) add/remove: 24/18 grow/shrink: 26/28 up/down: 6396/-3188 (3208) Total: Before=30629880, After=30633088, chg +0.01% - No code bloat for CONFIG_SCHED_SMT=y, which is expected. - Add comments around stop_core_cpuslocked on why ifdefs are not removed. - This leaves the remaining uses of CONFIG_SCHED_SMT mainly for topology building bits which has a policy based decision.

Shrikanth Hegde committed 6 months ago

Commit 5bc6ab21 parent 815c5cb8 files · +5 −63Browse files

Unified · Split

8 files
  1. include/linux/sched/smt.h +0 −4
  2. kernel/sched/core.c +0 −6
  3. kernel/sched/ext_idle.c +0 −6
  4. kernel/sched/fair.c +0 −35
  5. kernel/sched/sched.h +0 −6
  6. kernel/sched/topology.c +0 −2
  7. kernel/stop_machine.c +5 −0
  8. kernel/workqueue.c +0 −4
include/linux/sched/smt.h+0−4
@@ -4,16 +4,12 @@ · 3 unmodified lines
4 4
5#include <linux/static_key.h>5#include <linux/static_key.h>
6 6
7#ifdef CONFIG_SCHED_SMT
8extern struct static_key_false sched_smt_present;7extern struct static_key_false sched_smt_present;
9 8
10static __always_inline bool sched_smt_active(void)9static __always_inline bool sched_smt_active(void)
11{10{
12 return static_branch_likely(&sched_smt_present);11 return static_branch_likely(&sched_smt_present);
13}12}
14#else
15static __always_inline bool sched_smt_active(void) { return false; }
16#endif
17 13
18void arch_smt_update(void);14void arch_smt_update(void);
19 15
kernel/sched/core.c+0−6
@@ -8612,18 +8612,14 @@ static void cpuset_cpu_inactive(unsigned int cpu) · 8611 unmodified lines
8612 8612
8613static inline void sched_smt_present_inc(int cpu)8613static inline void sched_smt_present_inc(int cpu)
8614{8614{
8615#ifdef CONFIG_SCHED_SMT
8616 if (cpumask_weight(cpu_smt_mask(cpu)) == 2)8615 if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
8617 static_branch_inc_cpuslocked(&sched_smt_present);8616 static_branch_inc_cpuslocked(&sched_smt_present);
8618#endif
8619}8617}
8620 8618
8621static inline void sched_smt_present_dec(int cpu)8619static inline void sched_smt_present_dec(int cpu)
8622{8620{
8623#ifdef CONFIG_SCHED_SMT
8624 if (cpumask_weight(cpu_smt_mask(cpu)) == 2)8621 if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
8625 static_branch_dec_cpuslocked(&sched_smt_present);8622 static_branch_dec_cpuslocked(&sched_smt_present);
8626#endif
8627}8623}
8628 8624
8629int sched_cpu_activate(unsigned int cpu)8625int sched_cpu_activate(unsigned int cpu)
@@ -8711,9 +8707,7 @@ int sched_cpu_deactivate(unsigned int cpu) · 81 unmodified lines
8711 */8707 */
8712 sched_smt_present_dec(cpu);8708 sched_smt_present_dec(cpu);
8713 8709
8714#ifdef CONFIG_SCHED_SMT
8715 sched_core_cpu_deactivate(cpu);8710 sched_core_cpu_deactivate(cpu);
8716#endif
8717 8711
8718 if (!sched_smp_initialized)8712 if (!sched_smp_initialized)
8719 return 0;8713 return 0;
kernel/sched/ext_idle.c+0−6
@@ -79,7 +79,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu) · 78 unmodified lines
79 int node = scx_cpu_node_if_enabled(cpu);79 int node = scx_cpu_node_if_enabled(cpu);
80 struct cpumask *idle_cpus = idle_cpumask(node)->cpu;80 struct cpumask *idle_cpus = idle_cpumask(node)->cpu;
81 81
82#ifdef CONFIG_SCHED_SMT
83 /*82 /*
84 * SMT mask should be cleared whether we can claim @cpu or not. The SMT83 * SMT mask should be cleared whether we can claim @cpu or not. The SMT
85 * cluster is not wholly idle either way. This also prevents84 * cluster is not wholly idle either way. This also prevents
@@ -104,7 +103,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu) · 18 unmodified lines
104 else if (cpumask_test_cpu(cpu, idle_smts))103 else if (cpumask_test_cpu(cpu, idle_smts))
105 __cpumask_clear_cpu(cpu, idle_smts);104 __cpumask_clear_cpu(cpu, idle_smts);
106 }105 }
107#endif
108 106
109 return cpumask_test_and_clear_cpu(cpu, idle_cpus);107 return cpumask_test_and_clear_cpu(cpu, idle_cpus);
110}108}
@@ -622,7 +620,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, · 511 unmodified lines
622 goto out_unlock;620 goto out_unlock;
623 }621 }
624 622
625#ifdef CONFIG_SCHED_SMT
626 /*623 /*
627 * Use @prev_cpu's sibling if it's idle.624 * Use @prev_cpu's sibling if it's idle.
628 */625 */
@@ -634,7 +631,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, · 5 unmodified lines
634 goto out_unlock;631 goto out_unlock;
635 }632 }
636 }633 }
637#endif
638 634
639 /*635 /*
640 * Search for any idle CPU in the same LLC domain.636 * Search for any idle CPU in the same LLC domain.
@@ -714,7 +710,6 @@ static void update_builtin_idle(int cpu, bool idle) · 73 unmodified lines
714 710
715 assign_cpu(cpu, idle_cpus, idle);711 assign_cpu(cpu, idle_cpus, idle);
716 712
717#ifdef CONFIG_SCHED_SMT
718 if (sched_smt_active()) {713 if (sched_smt_active()) {
719 const struct cpumask *smt = cpu_smt_mask(cpu);714 const struct cpumask *smt = cpu_smt_mask(cpu);
720 struct cpumask *idle_smts = idle_cpumask(node)->smt;715 struct cpumask *idle_smts = idle_cpumask(node)->smt;
@@ -731,7 +726,6 @@ static void update_builtin_idle(int cpu, bool idle) · 10 unmodified lines
731 cpumask_andnot(idle_smts, idle_smts, smt);726 cpumask_andnot(idle_smts, idle_smts, smt);
732 }727 }
733 }728 }
734#endif
735}729}
736 730
737/*731/*
kernel/sched/fair.c+0−35
@@ -1555,7 +1555,6 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) · 1554 unmodified lines
1555 1555
1556static inline bool is_core_idle(int cpu)1556static inline bool is_core_idle(int cpu)
1557{1557{
1558#ifdef CONFIG_SCHED_SMT
1559 int sibling;1558 int sibling;
1560 1559
1561 for_each_cpu(sibling, cpu_smt_mask(cpu)) {1560 for_each_cpu(sibling, cpu_smt_mask(cpu)) {
@@ -1565,7 +1564,6 @@ static inline bool is_core_idle(int cpu) · 3 unmodified lines
1565 if (!idle_cpu(sibling))1564 if (!idle_cpu(sibling))
1566 return false;1565 return false;
1567 }1566 }
1568#endif
1569 1567
1570 return true;1568 return true;
1571}1569}
@@ -2248,7 +2246,6 @@ numa_type numa_classify(unsigned int imbalance_pct, · 676 unmodified lines
2248 return node_fully_busy;2246 return node_fully_busy;
2249}2247}
2250 2248
2251#ifdef CONFIG_SCHED_SMT
2252/* Forward declarations of select_idle_sibling helpers */2249/* Forward declarations of select_idle_sibling helpers */
2253static inline bool test_idle_cores(int cpu);2250static inline bool test_idle_cores(int cpu);
2254static inline int numa_idle_core(int idle_core, int cpu)2251static inline int numa_idle_core(int idle_core, int cpu)
@@ -2266,12 +2263,6 @@ static inline int numa_idle_core(int idle_core, int cpu) · 11 unmodified lines
2266 2263
2267 return idle_core;2264 return idle_core;
2268}2265}
2269#else /* !CONFIG_SCHED_SMT: */
2270static inline int numa_idle_core(int idle_core, int cpu)
2271{
2272 return idle_core;
2273}
2274#endif /* !CONFIG_SCHED_SMT */
2275 2266
2276/*2267/*
2277 * Gather all necessary information to make NUMA balancing placement2268 * Gather all necessary information to make NUMA balancing placement
@@ -7778,7 +7769,6 @@ static inline int __select_idle_cpu(int cpu, struct task_struct *p) · 5500 unmodified lines
7778 return -1;7769 return -1;
7779}7770}
7780 7771
7781#ifdef CONFIG_SCHED_SMT
7782DEFINE_STATIC_KEY_FALSE(sched_smt_present);7772DEFINE_STATIC_KEY_FALSE(sched_smt_present);
7783EXPORT_SYMBOL_GPL(sched_smt_present);7773EXPORT_SYMBOL_GPL(sched_smt_present);
7784 7774
@@ -7888,29 +7878,6 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t · 103 unmodified lines
7888 return -1;7878 return -1;
7889}7879}
7890 7880
7891#else /* !CONFIG_SCHED_SMT: */
7892
7893static inline void set_idle_cores(int cpu, int val)
7894{
7895}
7896
7897static inline bool test_idle_cores(int cpu)
7898{
7899 return false;
7900}
7901
7902static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
7903{
7904 return __select_idle_cpu(core, p);
7905}
7906
7907static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
7908{
7909 return -1;
7910}
7911
7912#endif /* !CONFIG_SCHED_SMT */
7913
7914/*7881/*
7915 * Scan the LLC domain for idle CPUs; this is dynamically regulated by7882 * Scan the LLC domain for idle CPUs; this is dynamically regulated by
7916 * comparing the average scan cost (tracked in sd->avg_scan_cost) against the7883 * comparing the average scan cost (tracked in sd->avg_scan_cost) against the
@@ -12002,9 +11969,7 @@ static int should_we_balance(struct lb_env *env) · 4085 unmodified lines
12002 * idle has been found, then its not needed to check other11969 * idle has been found, then its not needed to check other
12003 * SMT siblings for idleness:11970 * SMT siblings for idleness:
12004 */11971 */
12005#ifdef CONFIG_SCHED_SMT
12006 cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu));11972 cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu));
12007#endif
12008 continue;11973 continue;
12009 }11974 }
12010 11975
kernel/sched/sched.h+0−6
@@ -1667,7 +1667,6 @@ do { \ · 1666 unmodified lines
1667 flags = _raw_spin_rq_lock_irqsave(rq); \1667 flags = _raw_spin_rq_lock_irqsave(rq); \
1668} while (0)1668} while (0)
1669 1669
1670#ifdef CONFIG_SCHED_SMT
1671extern void __update_idle_core(struct rq *rq);1670extern void __update_idle_core(struct rq *rq);
1672 1671
1673static inline void update_idle_core(struct rq *rq)1672static inline void update_idle_core(struct rq *rq)
@@ -1676,12 +1675,7 @@ static inline void update_idle_core(struct rq *rq) · 2 unmodified lines
1676 __update_idle_core(rq);1675 __update_idle_core(rq);
1677}1676}
1678 1677
1679#else /* !CONFIG_SCHED_SMT: */
1680static inline void update_idle_core(struct rq *rq) { }
1681#endif /* !CONFIG_SCHED_SMT */
1682
1683#ifdef CONFIG_FAIR_GROUP_SCHED1678#ifdef CONFIG_FAIR_GROUP_SCHED
1684
1685static inline struct task_struct *task_of(struct sched_entity *se)1679static inline struct task_struct *task_of(struct sched_entity *se)
1686{1680{
1687 WARN_ON_ONCE(!entity_is_task(se));1681 WARN_ON_ONCE(!entity_is_task(se));
kernel/sched/topology.c+0−2
@@ -1310,9 +1310,7 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) · 1309 unmodified lines
1310 cpumask_copy(mask, sched_group_span(sg));1310 cpumask_copy(mask, sched_group_span(sg));
1311 for_each_cpu(cpu, mask) {1311 for_each_cpu(cpu, mask) {
1312 cores++;1312 cores++;
1313#ifdef CONFIG_SCHED_SMT
1314 cpumask_andnot(mask, mask, cpu_smt_mask(cpu));1313 cpumask_andnot(mask, mask, cpu_smt_mask(cpu));
1315#endif
1316 }1314 }
1317 sg->cores = cores;1315 sg->cores = cores;
1318 1316
kernel/stop_machine.c+5−0
@@ -633,6 +633,11 @@ int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus) · 632 unmodified lines
633EXPORT_SYMBOL_GPL(stop_machine);633EXPORT_SYMBOL_GPL(stop_machine);
634 634
635#ifdef CONFIG_SCHED_SMT635#ifdef CONFIG_SCHED_SMT
636/*
637 * INTEL_IFS is the only user of this API. That selftest can
638 * only be compiled if SMP=y. On x86 it selects SCHED_SMT.
639 * Keep the ifdefs for now.
640 */
636int stop_core_cpuslocked(unsigned int cpu, cpu_stop_fn_t fn, void *data)641int stop_core_cpuslocked(unsigned int cpu, cpu_stop_fn_t fn, void *data)
637{642{
638 const struct cpumask *smt_mask = cpu_smt_mask(cpu);643 const struct cpumask *smt_mask = cpu_smt_mask(cpu);
kernel/workqueue.c+0−4
@@ -8187,11 +8187,7 @@ static bool __init cpus_dont_share(int cpu0, int cpu1) · 8186 unmodified lines
8187 8187
8188static bool __init cpus_share_smt(int cpu0, int cpu1)8188static bool __init cpus_share_smt(int cpu0, int cpu1)
8189{8189{
8190#ifdef CONFIG_SCHED_SMT
8191 return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1));8190 return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1));
8192#else
8193 return false;
8194#endif
8195}8191}
8196 8192
8197static bool __init cpus_share_numa(int cpu0, int cpu1)8193static bool __init cpus_share_numa(int cpu0, int cpu1)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t