Provide a debugfs directory llc_balancing, and a knob named "enabled" under it to allow the user to turn off and on the cache aware scheduling at runtime.
Showing the changes to kernel (3 of 3 files). Show all
Unified · Split
| @@ -210,6 +210,46 @@ static const struct file_operations sched_scaling_fops = { · 209 unmodified lines | |||
| 210 | .release = single_release, | 210 | .release = single_release, |
| 211 | }; | 211 | }; |
| 212 | 212 | ||
| 213 | #ifdef CONFIG_SCHED_CACHE | ||
| 214 | static ssize_t | ||
| 215 | sched_cache_enable_write(struct file *filp, const char __user *ubuf, | ||
| 216 | size_t cnt, loff_t *ppos) | ||
| 217 | { | ||
| 218 | bool val; | ||
| 219 | int ret; | ||
| 220 | |||
| 221 | ret = kstrtobool_from_user(ubuf, cnt, &val); | ||
| 222 | if (ret) | ||
| 223 | return ret; | ||
| 224 | |||
| 225 | sysctl_sched_cache_user = val; | ||
| 226 | |||
| 227 | sched_cache_active_set_unlocked(); | ||
| 228 | |||
| 229 | return cnt; | ||
| 230 | } | ||
| 231 | |||
| 232 | static int sched_cache_enable_show(struct seq_file *m, void *v) | ||
| 233 | { | ||
| 234 | seq_printf(m, "%d\n", sysctl_sched_cache_user); | ||
| 235 | return 0; | ||
| 236 | } | ||
| 237 | |||
| 238 | static int sched_cache_enable_open(struct inode *inode, | ||
| 239 | struct file *filp) | ||
| 240 | { | ||
| 241 | return single_open(filp, sched_cache_enable_show, NULL); | ||
| 242 | } | ||
| 243 | |||
| 244 | static const struct file_operations sched_cache_enable_fops = { | ||
| 245 | .open = sched_cache_enable_open, | ||
| 246 | .write = sched_cache_enable_write, | ||
| 247 | .read = seq_read, | ||
| 248 | .llseek = seq_lseek, | ||
| 249 | .release = single_release, | ||
| 250 | }; | ||
| 251 | #endif | ||
| 252 | |||
| 213 | #ifdef CONFIG_PREEMPT_DYNAMIC | 253 | #ifdef CONFIG_PREEMPT_DYNAMIC |
| 214 | 254 | ||
| 215 | static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf, | 255 | static ssize_t sched_dynamic_write(struct file *filp, const char __user *ubuf, |
| @@ -593,7 +633,7 @@ static void debugfs_ext_server_init(void) · 377 unmodified lines | |||
| 593 | 633 | ||
| 594 | static __init int sched_init_debug(void) | 634 | static __init int sched_init_debug(void) |
| 595 | { | 635 | { |
| 596 | struct dentry __maybe_unused *numa; | 636 | struct dentry __maybe_unused *numa, *llc; |
| 597 | 637 | ||
| 598 | debugfs_sched = debugfs_create_dir("sched", NULL); | 638 | debugfs_sched = debugfs_create_dir("sched", NULL); |
| 599 | 639 | ||
| @@ -626,6 +666,12 @@ static __init int sched_init_debug(void) · 26 unmodified lines | |||
| 626 | debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold); | 666 | debugfs_create_u32("hot_threshold_ms", 0644, numa, &sysctl_numa_balancing_hot_threshold); |
| 627 | #endif /* CONFIG_NUMA_BALANCING */ | 667 | #endif /* CONFIG_NUMA_BALANCING */ |
| 628 | 668 | ||
| 669 | #ifdef CONFIG_SCHED_CACHE | ||
| 670 | llc = debugfs_create_dir("llc_balancing", debugfs_sched); | ||
| 671 | debugfs_create_file("enabled", 0644, llc, NULL, | ||
| 672 | &sched_cache_enable_fops); | ||
| 673 | #endif | ||
| 674 | |||
| 629 | debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); | 675 | debugfs_create_file("debug", 0444, debugfs_sched, NULL, &sched_debug_fops); |
| 630 | 676 | ||
| 631 | debugfs_fair_server_init(); | 677 | debugfs_fair_server_init(); |
| @@ -4070,11 +4070,16 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct · 4069 unmodified lines | |||
| 4070 | 4070 | ||
| 4071 | #ifdef CONFIG_SCHED_CACHE | 4071 | #ifdef CONFIG_SCHED_CACHE |
| 4072 | DECLARE_STATIC_KEY_FALSE(sched_cache_present); | 4072 | DECLARE_STATIC_KEY_FALSE(sched_cache_present); |
| 4073 | DECLARE_STATIC_KEY_FALSE(sched_cache_active); | ||
| 4074 | extern int sysctl_sched_cache_user; | ||
| 4073 | 4075 | ||
| 4074 | static inline bool sched_cache_enabled(void) | 4076 | static inline bool sched_cache_enabled(void) |
| 4075 | { | 4077 | { |
| 4076 | return static_branch_unlikely(&sched_cache_present); | 4078 | return static_branch_unlikely(&sched_cache_active); |
| 4077 | } | 4079 | } |
| 4080 | |||
| 4081 | extern void sched_cache_active_set_unlocked(void); | ||
| 4082 | |||
| 4078 | #endif | 4083 | #endif |
| 4079 | 4084 | ||
| 4080 | void sched_domains_free_llc_id(int cpu); | 4085 | void sched_domains_free_llc_id(int cpu); |
| @@ -821,7 +821,16 @@ enum s_alloc { · 820 unmodified lines | |||
| 821 | }; | 821 | }; |
| 822 | 822 | ||
| 823 | #ifdef CONFIG_SCHED_CACHE | 823 | #ifdef CONFIG_SCHED_CACHE |
| 824 | /* hardware support for cache aware scheduling */ | ||
| 824 | DEFINE_STATIC_KEY_FALSE(sched_cache_present); | 825 | DEFINE_STATIC_KEY_FALSE(sched_cache_present); |
| 826 | /* | ||
| 827 | * Indicator of whether cache aware scheduling | ||
| 828 | * is active, used by the scheduler. | ||
| 829 | */ | ||
| 830 | DEFINE_STATIC_KEY_FALSE(sched_cache_active); | ||
| 831 | /* user wants cache aware scheduling [0 or 1] */ | ||
| 832 | int sysctl_sched_cache_user = 1; | ||
| 833 | |||
| 825 | static bool alloc_sd_llc(const struct cpumask *cpu_map, | 834 | static bool alloc_sd_llc(const struct cpumask *cpu_map, |
| 826 | struct s_data *d) | 835 | struct s_data *d) |
| 827 | { | 836 | { |
| @@ -856,6 +865,60 @@ err: · 28 unmodified lines | |||
| 856 | 865 | ||
| 857 | return false; | 866 | return false; |
| 858 | } | 867 | } |
| 868 | |||
| 869 | static void _sched_cache_active_set(bool enable, bool locked) | ||
| 870 | { | ||
| 871 | if (enable) { | ||
| 872 | if (locked) | ||
| 873 | static_branch_enable_cpuslocked(&sched_cache_active); | ||
| 874 | else | ||
| 875 | static_branch_enable(&sched_cache_active); | ||
| 876 | } else { | ||
| 877 | if (locked) | ||
| 878 | static_branch_disable_cpuslocked(&sched_cache_active); | ||
| 879 | else | ||
| 880 | static_branch_disable(&sched_cache_active); | ||
| 881 | } | ||
| 882 | } | ||
| 883 | |||
| 884 | /* | ||
| 885 | * Enable/disable cache aware scheduling according to | ||
| 886 | * user input and the presence of hardware support. | ||
| 887 | */ | ||
| 888 | static void sched_cache_active_set(bool locked) | ||
| 889 | { | ||
| 890 | /* hardware does not support */ | ||
| 891 | if (!static_branch_likely(&sched_cache_present)) { | ||
| 892 | _sched_cache_active_set(false, locked); | ||
| 893 | return; | ||
| 894 | } | ||
| 895 | |||
| 896 | /* | ||
| 897 | * user wants it or not ? | ||
| 898 | * TBD: read before writing the static key. | ||
| 899 | * It is not in the critical path, leave as-is | ||
| 900 | * for now. | ||
| 901 | */ | ||
| 902 | if (sysctl_sched_cache_user) { | ||
| 903 | _sched_cache_active_set(true, locked); | ||
| 904 | if (sched_debug()) | ||
| 905 | pr_info("%s: enabling cache aware scheduling\n", __func__); | ||
| 906 | } else { | ||
| 907 | _sched_cache_active_set(false, locked); | ||
| 908 | if (sched_debug()) | ||
| 909 | pr_info("%s: disabling cache aware scheduling\n", __func__); | ||
| 910 | } | ||
| 911 | } | ||
| 912 | |||
| 913 | static void sched_cache_active_set_locked(void) | ||
| 914 | { | ||
| 915 | return sched_cache_active_set(true); | ||
| 916 | } | ||
| 917 | |||
| 918 | void sched_cache_active_set_unlocked(void) | ||
| 919 | { | ||
| 920 | return sched_cache_active_set(false); | ||
| 921 | } | ||
| 859 | #else | 922 | #else |
| 860 | static bool alloc_sd_llc(const struct cpumask *cpu_map, | 923 | static bool alloc_sd_llc(const struct cpumask *cpu_map, |
| 861 | struct s_data *d) | 924 | struct s_data *d) |
| @@ -2926,6 +2989,8 @@ error: · 2064 unmodified lines | |||
| 2926 | static_branch_enable_cpuslocked(&sched_cache_present); | 2989 | static_branch_enable_cpuslocked(&sched_cache_present); |
| 2927 | else | 2990 | else |
| 2928 | static_branch_disable_cpuslocked(&sched_cache_present); | 2991 | static_branch_disable_cpuslocked(&sched_cache_present); |
| 2992 | |||
| 2993 | sched_cache_active_set_locked(); | ||
| 2929 | #endif | 2994 | #endif |
| 2930 | __free_domain_allocs(&d, alloc_state, cpu_map); | 2995 | __free_domain_allocs(&d, alloc_state, cpu_map); |
| 2931 | 2996 | ||