Provide a debugfs directory llc_balancing, and a knob named "enabled" under it to allow the user to turn off and on the cache aware scheduling at runtime.
Unified · Split
| @@ -210,6 +210,46 @@ static const struct file_operations sched_scaling_fops = { · 209 unmodified lines | ||
| 210 | 210 | .release = single_release, |
| 211 | 211 | }; |
| 212 | 212 | |
| 213 | +#ifdef CONFIG_SCHED_CACHE | |
| 214 | +static ssize_t | |
| 215 | +sched_cache_enable_write(struct file *filp, const char __user *ubuf, | |
| 216 | + size_t cnt, loff_t *ppos) | |
| 217 | +{ | |
| 218 | + bool val; | |
| 219 | + int ret; | |
| 220 | + | |
| 221 | + ret = kstrtobool_from_user(ubuf, cnt, &val); | |
| 222 | + if (ret) | |
| 223 | + return ret; | |
| 224 | + | |
| 225 | + sysctl_sched_cache_user = val; | |
| 226 | + | |
| 227 | + sched_cache_active_set_unlocked(); | |
| 228 | + | |
| 229 | + return cnt; | |
| 230 | +} | |
| 231 | + | |
| 232 | +static int sched_cache_enable_show(struct seq_file *m, void *v) | |
| 233 | +{ | |
| 234 | + seq_printf(m, "%d\n", sysctl_sched_cache_user); | |
| 235 | + return 0; | |
| 236 | +} | |
| 237 | + | |
| 238 | +static int sched_cache_enable_open(struct inode *inode, | |
| 239 | + struct file *filp) | |
| 240 | +{ | |
| 241 | + return single_open(filp, sched_cache_enable_show, NULL); | |
| 242 | +} | |
| 243 | + | |
| 244 | +static const struct file_operations sched_cache_enable_fops = { | |
| 245 | + .open = sched_cache_enable_open, | |
| 246 | + .write = sched_cache_enable_write, | |
| 247 | + .read = seq_read, | |
| 248 | + .llseek = seq_lseek, | |
| 249 | + .release = single_release, | |
| 250 | +}; | |
| 251 | +#endif | |
| 252 | + | |
| 213 | 253 | #ifdef CONFIG_PREEMPT_DYNAMIC |
| 214 | 254 | |
| 215 | 255 | static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf, |
| @@ -593,7 +633,7 @@ static void debugfs_ext_server_init(void) · 377 unmodified lines | ||
| 593 | 633 | |
| 594 | 634 | static __init int sched_init_debug(void) |
| 595 | 635 | { |
| 596 | − struct dentry __maybe_unused *numa; | |
| 636 | + struct dentry __maybe_unused *numa, *llc; | |
| 597 | 637 | |
| 598 | 638 | debugfs_sched = debugfs_create_dir("sched", NULL); |
| 599 | 639 | |
| @@ -626,6 +666,12 @@ static __init int sched_init_debug(void) · 26 unmodified lines | ||
| 626 | 666 | debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold); |
| 627 | 667 | #endif /* CONFIG_NUMA_BALANCING */ |
| 628 | 668 | |
| 669 | +#ifdef CONFIG_SCHED_CACHE | |
| 670 | + llc = debugfs_create_dir("llc_balancing", debugfs_sched); | |
| 671 | + debugfs_create_file("enabled", 0644, llc, NULL, | |
| 672 | + &sched_cache_enable_fops); | |
| 673 | +#endif | |
| 674 | + | |
| 629 | 675 | debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); |
| 630 | 676 | |
| 631 | 677 | debugfs_fair_server_init(); |
| @@ -4070,11 +4070,16 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct · 4069 unmodified lines | ||
| 4070 | 4070 | |
| 4071 | 4071 | #ifdef CONFIG_SCHED_CACHE |
| 4072 | 4072 | DECLARE_STATIC_KEY_FALSE(sched_cache_present); |
| 4073 | +DECLARE_STATIC_KEY_FALSE(sched_cache_active); | |
| 4074 | +extern int sysctl_sched_cache_user; | |
| 4073 | 4075 | |
| 4074 | 4076 | static inline bool sched_cache_enabled(void) |
| 4075 | 4077 | { |
| 4076 | − return static_branch_unlikely(&sched_cache_present); | |
| 4078 | + return static_branch_unlikely(&sched_cache_active); | |
| 4077 | 4079 | } |
| 4080 | + | |
| 4081 | +extern void sched_cache_active_set_unlocked(void); | |
| 4082 | + | |
| 4078 | 4083 | #endif |
| 4079 | 4084 | |
| 4080 | 4085 | void sched_domains_free_llc_id(int cpu); |
| @@ -821,7 +821,16 @@ enum s_alloc { · 820 unmodified lines | ||
| 821 | 821 | }; |
| 822 | 822 | |
| 823 | 823 | #ifdef CONFIG_SCHED_CACHE |
| 824 | +/* hardware support for cache aware scheduling */ | |
| 824 | 825 | DEFINE_STATIC_KEY_FALSE(sched_cache_present); |
| 826 | +/* | |
| 827 | + * Indicator of whether cache aware scheduling | |
| 828 | + * is active, used by the scheduler. | |
| 829 | + */ | |
| 830 | +DEFINE_STATIC_KEY_FALSE(sched_cache_active); | |
| 831 | +/* user wants cache aware scheduling [0 or 1] */ | |
| 832 | +int sysctl_sched_cache_user = 1; | |
| 833 | + | |
| 825 | 834 | static bool alloc_sd_llc(const struct cpumask *cpu_map, |
| 826 | 835 | struct s_data *d) |
| 827 | 836 | { |
| @@ -856,6 +865,60 @@ err: · 28 unmodified lines | ||
| 856 | 865 | |
| 857 | 866 | return false; |
| 858 | 867 | } |
| 868 | + | |
| 869 | +static void _sched_cache_active_set(bool enable, bool locked) | |
| 870 | +{ | |
| 871 | + if (enable) { | |
| 872 | + if (locked) | |
| 873 | + static_branch_enable_cpuslocked(&sched_cache_active); | |
| 874 | + else | |
| 875 | + static_branch_enable(&sched_cache_active); | |
| 876 | + } else { | |
| 877 | + if (locked) | |
| 878 | + static_branch_disable_cpuslocked(&sched_cache_active); | |
| 879 | + else | |
| 880 | + static_branch_disable(&sched_cache_active); | |
| 881 | + } | |
| 882 | +} | |
| 883 | + | |
| 884 | +/* | |
| 885 | + * Enable/disable cache aware scheduling according to | |
| 886 | + * user input and the presence of hardware support. | |
| 887 | + */ | |
| 888 | +static void sched_cache_active_set(bool locked) | |
| 889 | +{ | |
| 890 | + /* hardware does not support */ | |
| 891 | + if (!static_branch_likely(&sched_cache_present)) { | |
| 892 | + _sched_cache_active_set(false, locked); | |
| 893 | + return; | |
| 894 | + } | |
| 895 | + | |
| 896 | + /* | |
| 897 | + * user wants it or not ? | |
| 898 | + * TBD: read before writing the static key. | |
| 899 | + * It is not in the critical path, leave as-is | |
| 900 | + * for now. | |
| 901 | + */ | |
| 902 | + if (sysctl_sched_cache_user) { | |
| 903 | + _sched_cache_active_set(true, locked); | |
| 904 | + if (sched_debug()) | |
| 905 | + pr_info("%s: enabling cache aware scheduling\n", __func__); | |
| 906 | + } else { | |
| 907 | + _sched_cache_active_set(false, locked); | |
| 908 | + if (sched_debug()) | |
| 909 | + pr_info("%s: disabling cache aware scheduling\n", __func__); | |
| 910 | + } | |
| 911 | +} | |
| 912 | + | |
| 913 | +static void sched_cache_active_set_locked(void) | |
| 914 | +{ | |
| 915 | + return sched_cache_active_set(true); | |
| 916 | +} | |
| 917 | + | |
| 918 | +void sched_cache_active_set_unlocked(void) | |
| 919 | +{ | |
| 920 | + return sched_cache_active_set(false); | |
| 921 | +} | |
| 859 | 922 | #else |
| 860 | 923 | static bool alloc_sd_llc(const struct cpumask *cpu_map, |
| 861 | 924 | struct s_data *d) |
| @@ -2926,6 +2989,8 @@ error: · 2064 unmodified lines | ||
| 2926 | 2989 | static_branch_enable_cpuslocked(&sched_cache_present); |
| 2927 | 2990 | else |
| 2928 | 2991 | static_branch_disable_cpuslocked(&sched_cache_present); |
| 2992 | + | |
| 2993 | + sched_cache_active_set_locked(); | |
| 2929 | 2994 | #endif |
| 2930 | 2995 | __free_domain_allocs(&d, alloc_state, cpu_map); |
| 2931 | 2996 | |