Now, that cpu_smt_mask is defined as cpumask_of(cpu) for CONFIG_SCHED_SMT=n, it is possible to get rid of the ifdeffery. Effectively, - This makes sched_smt_present is defined always - cpumask_weight(cpumask_of(cpu)) == 1. So sched_smt_present_inc/dec will never enable the sched_smt_present. Which is expected. - Paths that were compile-time eliminated become runtime guarded using static keys. - Defines set_idle_cores, test_idle_cores, etc which could likely benefit the CONFIG_SCHED_SMT=n systems to use the same optimizations within the LLC at wakeups. - This will expose sched_smt_present symbol for CONFIG_SCHED_SMT=n. Likely not a concern. - There is a bloat of code CONFIG_SCHED_SMT=n. (NR_CPUS=2048) add/remove: 24/18 grow/shrink: 26/28 up/down: 6396/-3188 (3208) Total: Before=30629880, After=30633088, chg +0.01% - No code bloat for CONFIG_SCHED_SMT=y, which is expected. - Add comments around stop_core_cpuslocked on why ifdefs are not removed. - This leaves the remaining uses of CONFIG_SCHED_SMT mainly for topology building bits which has a policy based decision.
Unified · Split
| @@ -4,16 +4,12 @@ · 3 unmodified lines | ||
| 4 | 4 | |
| 5 | 5 | #include <linux/static_key.h> |
| 6 | 6 | |
| 7 | −#ifdef CONFIG_SCHED_SMT | |
| 8 | 7 | extern struct static_key_false sched_smt_present; |
| 9 | 8 | |
| 10 | 9 | static __always_inline bool sched_smt_active(void) |
| 11 | 10 | { |
| 12 | 11 | return static_branch_likely(&sched_smt_present); |
| 13 | 12 | } |
| 14 | −#else | |
| 15 | −static __always_inline bool sched_smt_active(void) { return false; } | |
| 16 | −#endif | |
| 17 | 13 | |
| 18 | 14 | void arch_smt_update(void); |
| 19 | 15 | |
| @@ -8612,18 +8612,14 @@ static void cpuset_cpu_inactive(unsigned int cpu) · 8611 unmodified lines | ||
| 8612 | 8612 | |
| 8613 | 8613 | static inline void sched_smt_present_inc(int cpu) |
| 8614 | 8614 | { |
| 8615 | −#ifdef CONFIG_SCHED_SMT | |
| 8616 | 8615 | if (cpumask_weight(cpu_smt_mask(cpu)) == 2) |
| 8617 | 8616 | static_branch_inc_cpuslocked(&sched_smt_present); |
| 8618 | −#endif | |
| 8619 | 8617 | } |
| 8620 | 8618 | |
| 8621 | 8619 | static inline void sched_smt_present_dec(int cpu) |
| 8622 | 8620 | { |
| 8623 | −#ifdef CONFIG_SCHED_SMT | |
| 8624 | 8621 | if (cpumask_weight(cpu_smt_mask(cpu)) == 2) |
| 8625 | 8622 | static_branch_dec_cpuslocked(&sched_smt_present); |
| 8626 | −#endif | |
| 8627 | 8623 | } |
| 8628 | 8624 | |
| 8629 | 8625 | int sched_cpu_activate(unsigned int cpu) |
| @@ -8711,9 +8707,7 @@ int sched_cpu_deactivate(unsigned int cpu) · 81 unmodified lines | ||
| 8711 | 8707 | */ |
| 8712 | 8708 | sched_smt_present_dec(cpu); |
| 8713 | 8709 | |
| 8714 | −#ifdef CONFIG_SCHED_SMT | |
| 8715 | 8710 | sched_core_cpu_deactivate(cpu); |
| 8716 | −#endif | |
| 8717 | 8711 | |
| 8718 | 8712 | if (!sched_smp_initialized) |
| 8719 | 8713 | return 0; |
| @@ -79,7 +79,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu) · 78 unmodified lines | ||
| 79 | 79 | int node = scx_cpu_node_if_enabled(cpu); |
| 80 | 80 | struct cpumask *idle_cpus = idle_cpumask(node)->cpu; |
| 81 | 81 | |
| 82 | −#ifdef CONFIG_SCHED_SMT | |
| 83 | 82 | /* |
| 84 | 83 | * SMT mask should be cleared whether we can claim @cpu or not. The SMT |
| 85 | 84 | * cluster is not wholly idle either way. This also prevents |
| @@ -104,7 +103,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu) · 18 unmodified lines | ||
| 104 | 103 | else if (cpumask_test_cpu(cpu, idle_smts)) |
| 105 | 104 | __cpumask_clear_cpu(cpu, idle_smts); |
| 106 | 105 | } |
| 107 | −#endif | |
| 108 | 106 | |
| 109 | 107 | return cpumask_test_and_clear_cpu(cpu, idle_cpus); |
| 110 | 108 | } |
| @@ -622,7 +620,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, · 511 unmodified lines | ||
| 622 | 620 | goto out_unlock; |
| 623 | 621 | } |
| 624 | 622 | |
| 625 | −#ifdef CONFIG_SCHED_SMT | |
| 626 | 623 | /* |
| 627 | 624 | * Use @prev_cpu's sibling if it's idle. |
| 628 | 625 | */ |
| @@ -634,7 +631,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, · 5 unmodified lines | ||
| 634 | 631 | goto out_unlock; |
| 635 | 632 | } |
| 636 | 633 | } |
| 637 | −#endif | |
| 638 | 634 | |
| 639 | 635 | /* |
| 640 | 636 | * Search for any idle CPU in the same LLC domain. |
| @@ -714,7 +710,6 @@ static void update_builtin_idle(int cpu, bool idle) · 73 unmodified lines | ||
| 714 | 710 | |
| 715 | 711 | assign_cpu(cpu, idle_cpus, idle); |
| 716 | 712 | |
| 717 | −#ifdef CONFIG_SCHED_SMT | |
| 718 | 713 | if (sched_smt_active()) { |
| 719 | 714 | const struct cpumask *smt = cpu_smt_mask(cpu); |
| 720 | 715 | struct cpumask *idle_smts = idle_cpumask(node)->smt; |
| @@ -731,7 +726,6 @@ static void update_builtin_idle(int cpu, bool idle) · 10 unmodified lines | ||
| 731 | 726 | cpumask_andnot(idle_smts, idle_smts, smt); |
| 732 | 727 | } |
| 733 | 728 | } |
| 734 | −#endif | |
| 735 | 729 | } |
| 736 | 730 | |
| 737 | 731 | /* |
| @@ -1555,7 +1555,6 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) · 1554 unmodified lines | ||
| 1555 | 1555 | |
| 1556 | 1556 | static inline bool is_core_idle(int cpu) |
| 1557 | 1557 | { |
| 1558 | −#ifdef CONFIG_SCHED_SMT | |
| 1559 | 1558 | int sibling; |
| 1560 | 1559 | |
| 1561 | 1560 | for_each_cpu(sibling, cpu_smt_mask(cpu)) { |
| @@ -1565,7 +1564,6 @@ static inline bool is_core_idle(int cpu) · 3 unmodified lines | ||
| 1565 | 1564 | if (!idle_cpu(sibling)) |
| 1566 | 1565 | return false; |
| 1567 | 1566 | } |
| 1568 | −#endif | |
| 1569 | 1567 | |
| 1570 | 1568 | return true; |
| 1571 | 1569 | } |
| @@ -2248,7 +2246,6 @@ numa_type numa_classify(unsigned int imbalance_pct, · 676 unmodified lines | ||
| 2248 | 2246 | return node_fully_busy; |
| 2249 | 2247 | } |
| 2250 | 2248 | |
| 2251 | −#ifdef CONFIG_SCHED_SMT | |
| 2252 | 2249 | /* Forward declarations of select_idle_sibling helpers */ |
| 2253 | 2250 | static inline bool test_idle_cores(int cpu); |
| 2254 | 2251 | static inline int numa_idle_core(int idle_core, int cpu) |
| @@ -2266,12 +2263,6 @@ static inline int numa_idle_core(int idle_core, int cpu) · 11 unmodified lines | ||
| 2266 | 2263 | |
| 2267 | 2264 | return idle_core; |
| 2268 | 2265 | } |
| 2269 | −#else /* !CONFIG_SCHED_SMT: */ | |
| 2270 | −static inline int numa_idle_core(int idle_core, int cpu) | |
| 2271 | −{ | |
| 2272 | − return idle_core; | |
| 2273 | −} | |
| 2274 | −#endif /* !CONFIG_SCHED_SMT */ | |
| 2275 | 2266 | |
| 2276 | 2267 | /* |
| 2277 | 2268 | * Gather all necessary information to make NUMA balancing placement |
| @@ -7778,7 +7769,6 @@ static inline int __select_idle_cpu(int cpu, struct task_struct *p) · 5500 unmodified lines | ||
| 7778 | 7769 | return -1; |
| 7779 | 7770 | } |
| 7780 | 7771 | |
| 7781 | −#ifdef CONFIG_SCHED_SMT | |
| 7782 | 7772 | DEFINE_STATIC_KEY_FALSE(sched_smt_present); |
| 7783 | 7773 | EXPORT_SYMBOL_GPL(sched_smt_present); |
| 7784 | 7774 | |
| @@ -7888,29 +7878,6 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t · 103 unmodified lines | ||
| 7888 | 7878 | return -1; |
| 7889 | 7879 | } |
| 7890 | 7880 | |
| 7891 | −#else /* !CONFIG_SCHED_SMT: */ | |
| 7892 | − | |
| 7893 | −static inline void set_idle_cores(int cpu, int val) | |
| 7894 | −{ | |
| 7895 | −} | |
| 7896 | − | |
| 7897 | −static inline bool test_idle_cores(int cpu) | |
| 7898 | −{ | |
| 7899 | − return false; | |
| 7900 | −} | |
| 7901 | − | |
| 7902 | −static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) | |
| 7903 | −{ | |
| 7904 | − return __select_idle_cpu(core, p); | |
| 7905 | −} | |
| 7906 | − | |
| 7907 | −static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) | |
| 7908 | −{ | |
| 7909 | − return -1; | |
| 7910 | −} | |
| 7911 | − | |
| 7912 | −#endif /* !CONFIG_SCHED_SMT */ | |
| 7913 | − | |
| 7914 | 7881 | /* |
| 7915 | 7882 | * Scan the LLC domain for idle CPUs; this is dynamically regulated by |
| 7916 | 7883 | * comparing the average scan cost (tracked in sd->avg_scan_cost) against the |
| @@ -12002,9 +11969,7 @@ static int should_we_balance(struct lb_env *env) · 4085 unmodified lines | ||
| 12002 | 11969 | * idle has been found, then its not needed to check other |
| 12003 | 11970 | * SMT siblings for idleness: |
| 12004 | 11971 | */ |
| 12005 | −#ifdef CONFIG_SCHED_SMT | |
| 12006 | 11972 | cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu)); |
| 12007 | −#endif | |
| 12008 | 11973 | continue; |
| 12009 | 11974 | } |
| 12010 | 11975 | |
| @@ -1667,7 +1667,6 @@ do { \ · 1666 unmodified lines | ||
| 1667 | 1667 | flags = _raw_spin_rq_lock_irqsave(rq); \ |
| 1668 | 1668 | } while (0) |
| 1669 | 1669 | |
| 1670 | −#ifdef CONFIG_SCHED_SMT | |
| 1671 | 1670 | extern void __update_idle_core(struct rq *rq); |
| 1672 | 1671 | |
| 1673 | 1672 | static inline void update_idle_core(struct rq *rq) |
| @@ -1676,12 +1675,7 @@ static inline void update_idle_core(struct rq *rq) · 2 unmodified lines | ||
| 1676 | 1675 | __update_idle_core(rq); |
| 1677 | 1676 | } |
| 1678 | 1677 | |
| 1679 | −#else /* !CONFIG_SCHED_SMT: */ | |
| 1680 | −static inline void update_idle_core(struct rq *rq) { } | |
| 1681 | −#endif /* !CONFIG_SCHED_SMT */ | |
| 1682 | − | |
| 1683 | 1678 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 1684 | − | |
| 1685 | 1679 | static inline struct task_struct *task_of(struct sched_entity *se) |
| 1686 | 1680 | { |
| 1687 | 1681 | WARN_ON_ONCE(!entity_is_task(se)); |
| @@ -1310,9 +1310,7 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) · 1309 unmodified lines | ||
| 1310 | 1310 | cpumask_copy(mask, sched_group_span(sg)); |
| 1311 | 1311 | for_each_cpu(cpu, mask) { |
| 1312 | 1312 | cores++; |
| 1313 | −#ifdef CONFIG_SCHED_SMT | |
| 1314 | 1313 | cpumask_andnot(mask, mask, cpu_smt_mask(cpu)); |
| 1315 | −#endif | |
| 1316 | 1314 | } |
| 1317 | 1315 | sg->cores = cores; |
| 1318 | 1316 | |
| @@ -633,6 +633,11 @@ int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus) · 632 unmodified lines | ||
| 633 | 633 | EXPORT_SYMBOL_GPL(stop_machine); |
| 634 | 634 | |
| 635 | 635 | #ifdef CONFIG_SCHED_SMT |
| 636 | +/* | |
| 637 | + * INTEL_IFS is the only user of this API. That selftest can | |
| 638 | + * only be compiled if SMP=y. On x86 it selects SCHED_SMT. | |
| 639 | + * Keep the ifdefs for now. | |
| 640 | + */ | |
| 636 | 641 | int stop_core_cpuslocked(unsigned int cpu, cpu_stop_fn_t fn, void *data) |
| 637 | 642 | { |
| 638 | 643 | const struct cpumask *smt_mask = cpu_smt_mask(cpu); |
| @@ -8187,11 +8187,7 @@ static bool __init cpus_dont_share(int cpu0, int cpu1) · 8186 unmodified lines | ||
| 8187 | 8187 | |
| 8188 | 8188 | static bool __init cpus_share_smt(int cpu0, int cpu1) |
| 8189 | 8189 | { |
| 8190 | −#ifdef CONFIG_SCHED_SMT | |
| 8191 | 8190 | return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1)); |
| 8192 | −#else | |
| 8193 | − return false; | |
| 8194 | −#endif | |
| 8195 | 8191 | } |
| 8196 | 8192 | |
| 8197 | 8193 | static bool __init cpus_share_numa(int cpu0, int cpu1) |