Now, that cpu_smt_mask is defined as cpumask_of(cpu) for CONFIG_SCHED_SMT=n, it is possible to get rid of the ifdeffery. Effectively, - This makes sched_smt_present is defined always - cpumask_weight(cpumask_of(cpu)) == 1. So sched_smt_present_inc/dec will never enable the sched_smt_present. Which is expected. - Paths that were compile-time eliminated become runtime guarded using static keys. - Defines set_idle_cores, test_idle_cores, etc which could likely benefit the CONFIG_SCHED_SMT=n systems to use the same optimizations within the LLC at wakeups. - This will expose sched_smt_present symbol for CONFIG_SCHED_SMT=n. Likely not a concern. - There is a bloat of code CONFIG_SCHED_SMT=n. (NR_CPUS=2048) add/remove: 24/18 grow/shrink: 26/28 up/down: 6396/-3188 (3208) Total: Before=30629880, After=30633088, chg +0.01% - No code bloat for CONFIG_SCHED_SMT=y, which is expected. - Add comments around stop_core_cpuslocked on why ifdefs are not removed. - This leaves the remaining uses of CONFIG_SCHED_SMT mainly for topology building bits which has a policy based decision.
Unified · Split
| @@ -4,16 +4,12 @@ · 3 unmodified lines | |||
| 4 | 4 | ||
| 5 | #include <linux/static_key.h> | 5 | #include <linux/static_key.h> |
| 6 | 6 | ||
| 7 | #ifdef CONFIG_SCHED_SMT | ||
| 8 | extern struct static_key_false sched_smt_present; | 7 | extern struct static_key_false sched_smt_present; |
| 9 | 8 | ||
| 10 | static __always_inline bool sched_smt_active(void) | 9 | static __always_inline bool sched_smt_active(void) |
| 11 | { | 10 | { |
| 12 | return static_branch_likely(&sched_smt_present); | 11 | return static_branch_likely(&sched_smt_present); |
| 13 | } | 12 | } |
| 14 | #else | ||
| 15 | static __always_inline bool sched_smt_active(void) { return false; } | ||
| 16 | #endif | ||
| 17 | 13 | ||
| 18 | void arch_smt_update(void); | 14 | void arch_smt_update(void); |
| 19 | 15 | ||
| @@ -8612,18 +8612,14 @@ static void cpuset_cpu_inactive(unsigned int cpu) · 8611 unmodified lines | |||
| 8612 | 8612 | ||
| 8613 | static inline void sched_smt_present_inc(int cpu) | 8613 | static inline void sched_smt_present_inc(int cpu) |
| 8614 | { | 8614 | { |
| 8615 | #ifdef CONFIG_SCHED_SMT | ||
| 8616 | if (cpumask_weight(cpu_smt_mask(cpu)) == 2) | 8615 | if (cpumask_weight(cpu_smt_mask(cpu)) == 2) |
| 8617 | static_branch_inc_cpuslocked(&sched_smt_present); | 8616 | static_branch_inc_cpuslocked(&sched_smt_present); |
| 8618 | #endif | ||
| 8619 | } | 8617 | } |
| 8620 | 8618 | ||
| 8621 | static inline void sched_smt_present_dec(int cpu) | 8619 | static inline void sched_smt_present_dec(int cpu) |
| 8622 | { | 8620 | { |
| 8623 | #ifdef CONFIG_SCHED_SMT | ||
| 8624 | if (cpumask_weight(cpu_smt_mask(cpu)) == 2) | 8621 | if (cpumask_weight(cpu_smt_mask(cpu)) == 2) |
| 8625 | static_branch_dec_cpuslocked(&sched_smt_present); | 8622 | static_branch_dec_cpuslocked(&sched_smt_present); |
| 8626 | #endif | ||
| 8627 | } | 8623 | } |
| 8628 | 8624 | ||
| 8629 | int sched_cpu_activate(unsigned int cpu) | 8625 | int sched_cpu_activate(unsigned int cpu) |
| @@ -8711,9 +8707,7 @@ int sched_cpu_deactivate(unsigned int cpu) · 81 unmodified lines | |||
| 8711 | */ | 8707 | */ |
| 8712 | sched_smt_present_dec(cpu); | 8708 | sched_smt_present_dec(cpu); |
| 8713 | 8709 | ||
| 8714 | #ifdef CONFIG_SCHED_SMT | ||
| 8715 | sched_core_cpu_deactivate(cpu); | 8710 | sched_core_cpu_deactivate(cpu); |
| 8716 | #endif | ||
| 8717 | 8711 | ||
| 8718 | if (!sched_smp_initialized) | 8712 | if (!sched_smp_initialized) |
| 8719 | return 0; | 8713 | return 0; |
| @@ -79,7 +79,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu) · 78 unmodified lines | |||
| 79 | int node = scx_cpu_node_if_enabled(cpu); | 79 | int node = scx_cpu_node_if_enabled(cpu); |
| 80 | struct cpumask *idle_cpus = idle_cpumask(node)->cpu; | 80 | struct cpumask *idle_cpus = idle_cpumask(node)->cpu; |
| 81 | 81 | ||
| 82 | #ifdef CONFIG_SCHED_SMT | ||
| 83 | /* | 82 | /* |
| 84 | * SMT mask should be cleared whether we can claim @cpu or not. The SMT | 83 | * SMT mask should be cleared whether we can claim @cpu or not. The SMT |
| 85 | * cluster is not wholly idle either way. This also prevents | 84 | * cluster is not wholly idle either way. This also prevents |
| @@ -104,7 +103,6 @@ static bool scx_idle_test_and_clear_cpu(int cpu) · 18 unmodified lines | |||
| 104 | else if (cpumask_test_cpu(cpu, idle_smts)) | 103 | else if (cpumask_test_cpu(cpu, idle_smts)) |
| 105 | __cpumask_clear_cpu(cpu, idle_smts); | 104 | __cpumask_clear_cpu(cpu, idle_smts); |
| 106 | } | 105 | } |
| 107 | #endif | ||
| 108 | 106 | ||
| 109 | return cpumask_test_and_clear_cpu(cpu, idle_cpus); | 107 | return cpumask_test_and_clear_cpu(cpu, idle_cpus); |
| 110 | } | 108 | } |
| @@ -622,7 +620,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, · 511 unmodified lines | |||
| 622 | goto out_unlock; | 620 | goto out_unlock; |
| 623 | } | 621 | } |
| 624 | 622 | ||
| 625 | #ifdef CONFIG_SCHED_SMT | ||
| 626 | /* | 623 | /* |
| 627 | * Use @prev_cpu's sibling if it's idle. | 624 | * Use @prev_cpu's sibling if it's idle. |
| 628 | */ | 625 | */ |
| @@ -634,7 +631,6 @@ s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, · 5 unmodified lines | |||
| 634 | goto out_unlock; | 631 | goto out_unlock; |
| 635 | } | 632 | } |
| 636 | } | 633 | } |
| 637 | #endif | ||
| 638 | 634 | ||
| 639 | /* | 635 | /* |
| 640 | * Search for any idle CPU in the same LLC domain. | 636 | * Search for any idle CPU in the same LLC domain. |
| @@ -714,7 +710,6 @@ static void update_builtin_idle(int cpu, bool idle) · 73 unmodified lines | |||
| 714 | 710 | ||
| 715 | assign_cpu(cpu, idle_cpus, idle); | 711 | assign_cpu(cpu, idle_cpus, idle); |
| 716 | 712 | ||
| 717 | #ifdef CONFIG_SCHED_SMT | ||
| 718 | if (sched_smt_active()) { | 713 | if (sched_smt_active()) { |
| 719 | const struct cpumask *smt = cpu_smt_mask(cpu); | 714 | const struct cpumask *smt = cpu_smt_mask(cpu); |
| 720 | struct cpumask *idle_smts = idle_cpumask(node)->smt; | 715 | struct cpumask *idle_smts = idle_cpumask(node)->smt; |
| @@ -731,7 +726,6 @@ static void update_builtin_idle(int cpu, bool idle) · 10 unmodified lines | |||
| 731 | cpumask_andnot(idle_smts, idle_smts, smt); | 726 | cpumask_andnot(idle_smts, idle_smts, smt); |
| 732 | } | 727 | } |
| 733 | } | 728 | } |
| 734 | #endif | ||
| 735 | } | 729 | } |
| 736 | 730 | ||
| 737 | /* | 731 | /* |
| @@ -1555,7 +1555,6 @@ update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se) · 1554 unmodified lines | |||
| 1555 | 1555 | ||
| 1556 | static inline bool is_core_idle(int cpu) | 1556 | static inline bool is_core_idle(int cpu) |
| 1557 | { | 1557 | { |
| 1558 | #ifdef CONFIG_SCHED_SMT | ||
| 1559 | int sibling; | 1558 | int sibling; |
| 1560 | 1559 | ||
| 1561 | for_each_cpu(sibling, cpu_smt_mask(cpu)) { | 1560 | for_each_cpu(sibling, cpu_smt_mask(cpu)) { |
| @@ -1565,7 +1564,6 @@ static inline bool is_core_idle(int cpu) · 3 unmodified lines | |||
| 1565 | if (!idle_cpu(sibling)) | 1564 | if (!idle_cpu(sibling)) |
| 1566 | return false; | 1565 | return false; |
| 1567 | } | 1566 | } |
| 1568 | #endif | ||
| 1569 | 1567 | ||
| 1570 | return true; | 1568 | return true; |
| 1571 | } | 1569 | } |
| @@ -2248,7 +2246,6 @@ numa_type numa_classify(unsigned int imbalance_pct, · 676 unmodified lines | |||
| 2248 | return node_fully_busy; | 2246 | return node_fully_busy; |
| 2249 | } | 2247 | } |
| 2250 | 2248 | ||
| 2251 | #ifdef CONFIG_SCHED_SMT | ||
| 2252 | /* Forward declarations of select_idle_sibling helpers */ | 2249 | /* Forward declarations of select_idle_sibling helpers */ |
| 2253 | static inline bool test_idle_cores(int cpu); | 2250 | static inline bool test_idle_cores(int cpu); |
| 2254 | static inline int numa_idle_core(int idle_core, int cpu) | 2251 | static inline int numa_idle_core(int idle_core, int cpu) |
| @@ -2266,12 +2263,6 @@ static inline int numa_idle_core(int idle_core, int cpu) · 11 unmodified lines | |||
| 2266 | 2263 | ||
| 2267 | return idle_core; | 2264 | return idle_core; |
| 2268 | } | 2265 | } |
| 2269 | #else /* !CONFIG_SCHED_SMT: */ | ||
| 2270 | static inline int numa_idle_core(int idle_core, int cpu) | ||
| 2271 | { | ||
| 2272 | return idle_core; | ||
| 2273 | } | ||
| 2274 | #endif /* !CONFIG_SCHED_SMT */ | ||
| 2275 | 2266 | ||
| 2276 | /* | 2267 | /* |
| 2277 | * Gather all necessary information to make NUMA balancing placement | 2268 | * Gather all necessary information to make NUMA balancing placement |
| @@ -7778,7 +7769,6 @@ static inline int __select_idle_cpu(int cpu, struct task_struct *p) · 5500 unmodified lines | |||
| 7778 | return -1; | 7769 | return -1; |
| 7779 | } | 7770 | } |
| 7780 | 7771 | ||
| 7781 | #ifdef CONFIG_SCHED_SMT | ||
| 7782 | DEFINE_STATIC_KEY_FALSE(sched_smt_present); | 7772 | DEFINE_STATIC_KEY_FALSE(sched_smt_present); |
| 7783 | EXPORT_SYMBOL_GPL(sched_smt_present); | 7773 | EXPORT_SYMBOL_GPL(sched_smt_present); |
| 7784 | 7774 | ||
| @@ -7888,29 +7878,6 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t · 103 unmodified lines | |||
| 7888 | return -1; | 7878 | return -1; |
| 7889 | } | 7879 | } |
| 7890 | 7880 | ||
| 7891 | #else /* !CONFIG_SCHED_SMT: */ | ||
| 7892 | |||
| 7893 | static inline void set_idle_cores(int cpu, int val) | ||
| 7894 | { | ||
| 7895 | } | ||
| 7896 | |||
| 7897 | static inline bool test_idle_cores(int cpu) | ||
| 7898 | { | ||
| 7899 | return false; | ||
| 7900 | } | ||
| 7901 | |||
| 7902 | static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu) | ||
| 7903 | { | ||
| 7904 | return __select_idle_cpu(core, p); | ||
| 7905 | } | ||
| 7906 | |||
| 7907 | static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target) | ||
| 7908 | { | ||
| 7909 | return -1; | ||
| 7910 | } | ||
| 7911 | |||
| 7912 | #endif /* !CONFIG_SCHED_SMT */ | ||
| 7913 | |||
| 7914 | /* | 7881 | /* |
| 7915 | * Scan the LLC domain for idle CPUs; this is dynamically regulated by | 7882 | * Scan the LLC domain for idle CPUs; this is dynamically regulated by |
| 7916 | * comparing the average scan cost (tracked in sd->avg_scan_cost) against the | 7883 | * comparing the average scan cost (tracked in sd->avg_scan_cost) against the |
| @@ -12002,9 +11969,7 @@ static int should_we_balance(struct lb_env *env) · 4085 unmodified lines | |||
| 12002 | * idle has been found, then its not needed to check other | 11969 | * idle has been found, then its not needed to check other |
| 12003 | * SMT siblings for idleness: | 11970 | * SMT siblings for idleness: |
| 12004 | */ | 11971 | */ |
| 12005 | #ifdef CONFIG_SCHED_SMT | ||
| 12006 | cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu)); | 11972 | cpumask_andnot(swb_cpus, swb_cpus, cpu_smt_mask(cpu)); |
| 12007 | #endif | ||
| 12008 | continue; | 11973 | continue; |
| 12009 | } | 11974 | } |
| 12010 | 11975 | ||
| @@ -1667,7 +1667,6 @@ do { \ · 1666 unmodified lines | |||
| 1667 | flags = _raw_spin_rq_lock_irqsave(rq); \ | 1667 | flags = _raw_spin_rq_lock_irqsave(rq); \ |
| 1668 | } while (0) | 1668 | } while (0) |
| 1669 | 1669 | ||
| 1670 | #ifdef CONFIG_SCHED_SMT | ||
| 1671 | extern void __update_idle_core(struct rq *rq); | 1670 | extern void __update_idle_core(struct rq *rq); |
| 1672 | 1671 | ||
| 1673 | static inline void update_idle_core(struct rq *rq) | 1672 | static inline void update_idle_core(struct rq *rq) |
| @@ -1676,12 +1675,7 @@ static inline void update_idle_core(struct rq *rq) · 2 unmodified lines | |||
| 1676 | __update_idle_core(rq); | 1675 | __update_idle_core(rq); |
| 1677 | } | 1676 | } |
| 1678 | 1677 | ||
| 1679 | #else /* !CONFIG_SCHED_SMT: */ | ||
| 1680 | static inline void update_idle_core(struct rq *rq) { } | ||
| 1681 | #endif /* !CONFIG_SCHED_SMT */ | ||
| 1682 | |||
| 1683 | #ifdef CONFIG_FAIR_GROUP_SCHED | 1678 | #ifdef CONFIG_FAIR_GROUP_SCHED |
| 1684 | |||
| 1685 | static inline struct task_struct *task_of(struct sched_entity *se) | 1679 | static inline struct task_struct *task_of(struct sched_entity *se) |
| 1686 | { | 1680 | { |
| 1687 | WARN_ON_ONCE(!entity_is_task(se)); | 1681 | WARN_ON_ONCE(!entity_is_task(se)); |
| @@ -1310,9 +1310,7 @@ static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) · 1309 unmodified lines | |||
| 1310 | cpumask_copy(mask, sched_group_span(sg)); | 1310 | cpumask_copy(mask, sched_group_span(sg)); |
| 1311 | for_each_cpu(cpu, mask) { | 1311 | for_each_cpu(cpu, mask) { |
| 1312 | cores++; | 1312 | cores++; |
| 1313 | #ifdef CONFIG_SCHED_SMT | ||
| 1314 | cpumask_andnot(mask, mask, cpu_smt_mask(cpu)); | 1313 | cpumask_andnot(mask, mask, cpu_smt_mask(cpu)); |
| 1315 | #endif | ||
| 1316 | } | 1314 | } |
| 1317 | sg->cores = cores; | 1315 | sg->cores = cores; |
| 1318 | 1316 | ||
| @@ -633,6 +633,11 @@ int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus) · 632 unmodified lines | |||
| 633 | EXPORT_SYMBOL_GPL(stop_machine); | 633 | EXPORT_SYMBOL_GPL(stop_machine); |
| 634 | 634 | ||
| 635 | #ifdef CONFIG_SCHED_SMT | 635 | #ifdef CONFIG_SCHED_SMT |
| 636 | /* | ||
| 637 | * INTEL_IFS is the only user of this API. That selftest can | ||
| 638 | * only be compiled if SMP=y. On x86 it selects SCHED_SMT. | ||
| 639 | * Keep the ifdefs for now. | ||
| 640 | */ | ||
| 636 | int stop_core_cpuslocked(unsigned int cpu, cpu_stop_fn_t fn, void *data) | 641 | int stop_core_cpuslocked(unsigned int cpu, cpu_stop_fn_t fn, void *data) |
| 637 | { | 642 | { |
| 638 | const struct cpumask *smt_mask = cpu_smt_mask(cpu); | 643 | const struct cpumask *smt_mask = cpu_smt_mask(cpu); |
| @@ -8187,11 +8187,7 @@ static bool __init cpus_dont_share(int cpu0, int cpu1) · 8186 unmodified lines | |||
| 8187 | 8187 | ||
| 8188 | static bool __init cpus_share_smt(int cpu0, int cpu1) | 8188 | static bool __init cpus_share_smt(int cpu0, int cpu1) |
| 8189 | { | 8189 | { |
| 8190 | #ifdef CONFIG_SCHED_SMT | ||
| 8191 | return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1)); | 8190 | return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1)); |
| 8192 | #else | ||
| 8193 | return false; | ||
| 8194 | #endif | ||
| 8195 | } | 8191 | } |
| 8196 | 8192 | ||
| 8197 | static bool __init cpus_share_numa(int cpu0, int cpu1) | 8193 | static bool __init cpus_share_numa(int cpu0, int cpu1) |