Pull workqueue updates from Tejun Heo: - New default WQ_AFFN_CACHE_SHARD affinity scope subdivides LLCs into smaller shards to improve scalability on machines with many CPUs per LLC - Misc: - system_dfl_long_wq for long unbound works - devm_alloc_workqueue() for device-managed allocation - sysfs exposure for ordered workqueues and the EFI workqueue - removal of HK_TYPE_WQ from wq_unbound_cpumask - various small fixes * tag 'wq-for-7.1' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq: (21 commits) workqueue: validate cpumask_first() result in llc_populate_cpu_shard_id() workqueue: use NR_STD_WORKER_POOLS instead of hardcoded value workqueue: avoid unguarded 64-bit division docs: workqueue: document WQ_AFFN_CACHE_SHARD affinity scope workqueue: add test_workqueue benchmark module tools/workqueue: add CACHE_SHARD support to wq_dump.py workqueue: set WQ_AFFN_CACHE_SHARD as the default affinity scope workqueue: add WQ_AFFN_CACHE_SHARD affinity scope workqueue: fix typo in WQ_AFFN_SMT comment workqueue: Remove HK_TYPE_WQ from affecting wq_unbound_cpumask workqueue: unlink pwqs from wq->pwqs list in alloc_and_link_pwqs() error path workqueue: Remove NULL wq WARN in __queue_delayed_work() workqueue: fix parse_affn_scope() prefix matching bug workqueue: devres: Add device-managed allocate workqueue workqueue: Add system_dfl_long_wq for long unbound works tools/workqueue/wq_dump.py: add NODE prefix to all node columns tools/workqueue/wq_dump.py: fix column alignment in node_nr/max_active section tools/workqueue/wq_dump.py: remove backslash separator from node_nr/max_active header efi: Allow to expose the workqueue via sysfs workqueue: Allow to expose ordered workqueues via sysfs ...
Showing the changes to kernel/workqueue.c (1 of 10 files). Show all
Unified · Split
| @@ -41,6 +41,7 @@ · 40 unmodified lines | |||
| 41 | #include <linux/mempolicy.h> | 41 | #include <linux/mempolicy.h> |
| 42 | #include <linux/freezer.h> | 42 | #include <linux/freezer.h> |
| 43 | #include <linux/debug_locks.h> | 43 | #include <linux/debug_locks.h> |
| 44 | #include <linux/device/devres.h> | ||
| 44 | #include <linux/lockdep.h> | 45 | #include <linux/lockdep.h> |
| 45 | #include <linux/idr.h> | 46 | #include <linux/idr.h> |
| 46 | #include <linux/jhash.h> | 47 | #include <linux/jhash.h> |
| @@ -130,6 +131,14 @@ enum wq_internal_consts { · 83 unmodified lines | |||
| 130 | WORKER_ID_LEN = 10 + WQ_NAME_LEN, /* "kworker/R-" + WQ_NAME_LEN */ | 131 | WORKER_ID_LEN = 10 + WQ_NAME_LEN, /* "kworker/R-" + WQ_NAME_LEN */ |
| 131 | }; | 132 | }; |
| 132 | 133 | ||
| 134 | /* Layout of shards within one LLC pod */ | ||
| 135 | struct llc_shard_layout { | ||
| 136 | int nr_large_shards; /* number of large shards (cores_per_shard + 1) */ | ||
| 137 | int cores_per_shard; /* base number of cores per default shard */ | ||
| 138 | int nr_shards; /* total number of shards */ | ||
| 139 | /* nr_default shards = (nr_shards - nr_large_shards) */ | ||
| 140 | }; | ||
| 141 | |||
| 133 | /* | 142 | /* |
| 134 | * We don't want to trap softirq for too long. See MAX_SOFTIRQ_TIME and | 143 | * We don't want to trap softirq for too long. See MAX_SOFTIRQ_TIME and |
| 135 | * MAX_SOFTIRQ_RESTART in kernel/softirq.c. These are macros because | 144 | * MAX_SOFTIRQ_RESTART in kernel/softirq.c. These are macros because |
| @@ -404,11 +413,12 @@ struct work_offq_data { · 268 unmodified lines | |||
| 404 | u32 flags; | 413 | u32 flags; |
| 405 | }; | 414 | }; |
| 406 | 415 | ||
| 407 | static const char *wq_affn_names[WQ_AFFN_NR_TYPES] = { | 416 | static const char * const wq_affn_names[WQ_AFFN_NR_TYPES] = { |
| 408 | [WQ_AFFN_DFL] = "default", | 417 | [WQ_AFFN_DFL] = "default", |
| 409 | [WQ_AFFN_CPU] = "cpu", | 418 | [WQ_AFFN_CPU] = "cpu", |
| 410 | [WQ_AFFN_SMT] = "smt", | 419 | [WQ_AFFN_SMT] = "smt", |
| 411 | [WQ_AFFN_CACHE] = "cache", | 420 | [WQ_AFFN_CACHE] = "cache", |
| 421 | [WQ_AFFN_CACHE_SHARD] = "cache_shard", | ||
| 412 | [WQ_AFFN_NUMA] = "numa", | 422 | [WQ_AFFN_NUMA] = "numa", |
| 413 | [WQ_AFFN_SYSTEM] = "system", | 423 | [WQ_AFFN_SYSTEM] = "system", |
| 414 | }; | 424 | }; |
| @@ -431,13 +441,16 @@ module_param_named(cpu_intensive_warning_thresh, wq_cpu_intensive_warning_thresh · 16 unmodified lines | |||
| 431 | static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT); | 441 | static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT); |
| 432 | module_param_named(power_efficient, wq_power_efficient, bool, 0444); | 442 | module_param_named(power_efficient, wq_power_efficient, bool, 0444); |
| 433 | 443 | ||
| 444 | static unsigned int wq_cache_shard_size = 8; | ||
| 445 | module_param_named(cache_shard_size, wq_cache_shard_size, uint, 0444); | ||
| 446 | |||
| 434 | static bool wq_online; /* can kworkers be created yet? */ | 447 | static bool wq_online; /* can kworkers be created yet? */ |
| 435 | static bool wq_topo_initialized __read_mostly = false; | 448 | static bool wq_topo_initialized __read_mostly = false; |
| 436 | 449 | ||
| 437 | static struct kmem_cache *pwq_cache; | 450 | static struct kmem_cache *pwq_cache; |
| 438 | 451 | ||
| 439 | static struct wq_pod_type wq_pod_types[WQ_AFFN_NR_TYPES]; | 452 | static struct wq_pod_type wq_pod_types[WQ_AFFN_NR_TYPES]; |
| 440 | static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE; | 453 | static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE_SHARD; |
| 441 | 454 | ||
| 442 | /* buf for wq_update_unbound_pod_attrs(), protected by CPU hotplug exclusion */ | 455 | /* buf for wq_update_unbound_pod_attrs(), protected by CPU hotplug exclusion */ |
| 443 | static struct workqueue_attrs *unbound_wq_update_pwq_attrs_buf; | 456 | static struct workqueue_attrs *unbound_wq_update_pwq_attrs_buf; |
| @@ -530,6 +543,8 @@ struct workqueue_struct *system_bh_wq; · 86 unmodified lines | |||
| 530 | EXPORT_SYMBOL_GPL(system_bh_wq); | 543 | EXPORT_SYMBOL_GPL(system_bh_wq); |
| 531 | struct workqueue_struct *system_bh_highpri_wq; | 544 | struct workqueue_struct *system_bh_highpri_wq; |
| 532 | EXPORT_SYMBOL_GPL(system_bh_highpri_wq); | 545 | EXPORT_SYMBOL_GPL(system_bh_highpri_wq); |
| 546 | struct workqueue_struct *system_dfl_long_wq __ro_after_init; | ||
| 547 | EXPORT_SYMBOL_GPL(system_dfl_long_wq); | ||
| 533 | 548 | ||
| 534 | static int worker_thread(void *__worker); | 549 | static int worker_thread(void *__worker); |
| 535 | static void workqueue_sysfs_unregister(struct workqueue_struct *wq); | 550 | static void workqueue_sysfs_unregister(struct workqueue_struct *wq); |
| @@ -2519,7 +2534,6 @@ static void __queue_delayed_work(int cpu, struct workqueue_struct *wq, · 1983 unmodified lines | |||
| 2519 | struct timer_list *timer = &dwork->timer; | 2534 | struct timer_list *timer = &dwork->timer; |
| 2520 | struct work_struct *work = &dwork->work; | 2535 | struct work_struct *work = &dwork->work; |
| 2521 | 2536 | ||
| 2522 | WARN_ON_ONCE(!wq); | ||
| 2523 | WARN_ON_ONCE(timer->function != delayed_work_timer_fn); | 2537 | WARN_ON_ONCE(timer->function != delayed_work_timer_fn); |
| 2524 | WARN_ON_ONCE(timer_pending(timer)); | 2538 | WARN_ON_ONCE(timer_pending(timer)); |
| 2525 | WARN_ON_ONCE(!list_empty(&work->entry)); | 2539 | WARN_ON_ONCE(!list_empty(&work->entry)); |
| @@ -5635,8 +5649,16 @@ enomem: · 3109 unmodified lines | |||
| 5635 | for_each_possible_cpu(cpu) { | 5649 | for_each_possible_cpu(cpu) { |
| 5636 | struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu); | 5650 | struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu); |
| 5637 | 5651 | ||
| 5638 | if (pwq) | 5652 | if (pwq) { |
| 5653 | /* | ||
| 5654 | * Unlink pwq from wq->pwqs since link_pwq() | ||
| 5655 | * may have already added it. wq->mutex is not | ||
| 5656 | * needed as the wq has not been published yet. | ||
| 5657 | */ | ||
| 5658 | if (!list_empty(&pwq->pwqs_node)) | ||
| 5659 | list_del_rcu(&pwq->pwqs_node); | ||
| 5639 | kmem_cache_free(pwq_cache, pwq); | 5660 | kmem_cache_free(pwq_cache, pwq); |
| 5661 | } | ||
| 5640 | } | 5662 | } |
| 5641 | free_percpu(wq->cpu_pwq); | 5663 | free_percpu(wq->cpu_pwq); |
| 5642 | wq->cpu_pwq = NULL; | 5664 | wq->cpu_pwq = NULL; |
| @@ -5904,6 +5926,33 @@ struct workqueue_struct *alloc_workqueue_noprof(const char *fmt, · 261 unmodified lines | |||
| 5904 | } | 5926 | } |
| 5905 | EXPORT_SYMBOL_GPL(alloc_workqueue_noprof); | 5927 | EXPORT_SYMBOL_GPL(alloc_workqueue_noprof); |
| 5906 | 5928 | ||
| 5929 | static void devm_workqueue_release(void *res) | ||
| 5930 | { | ||
| 5931 | destroy_workqueue(res); | ||
| 5932 | } | ||
| 5933 | |||
| 5934 | __printf(2, 5) struct workqueue_struct * | ||
| 5935 | devm_alloc_workqueue(struct device *dev, const char *fmt, unsigned int flags, | ||
| 5936 | int max_active, ...) | ||
| 5937 | { | ||
| 5938 | struct workqueue_struct *wq; | ||
| 5939 | va_list args; | ||
| 5940 | int ret; | ||
| 5941 | |||
| 5942 | va_start(args, max_active); | ||
| 5943 | wq = alloc_workqueue(fmt, flags, max_active, args); | ||
| 5944 | va_end(args); | ||
| 5945 | if (!wq) | ||
| 5946 | return NULL; | ||
| 5947 | |||
| 5948 | ret = devm_add_action_or_reset(dev, devm_workqueue_release, wq); | ||
| 5949 | if (ret) | ||
| 5950 | return NULL; | ||
| 5951 | |||
| 5952 | return wq; | ||
| 5953 | } | ||
| 5954 | EXPORT_SYMBOL_GPL(devm_alloc_workqueue); | ||
| 5955 | |||
| 5907 | #ifdef CONFIG_LOCKDEP | 5956 | #ifdef CONFIG_LOCKDEP |
| 5908 | __printf(1, 5) | 5957 | __printf(1, 5) |
| 5909 | struct workqueue_struct * | 5958 | struct workqueue_struct * |
| @@ -7059,7 +7108,7 @@ int workqueue_unbound_housekeeping_update(const struct cpumask *hk) · 1149 unmodified lines | |||
| 7059 | /* | 7108 | /* |
| 7060 | * If the operation fails, it will fall back to | 7109 | * If the operation fails, it will fall back to |
| 7061 | * wq_requested_unbound_cpumask which is initially set to | 7110 | * wq_requested_unbound_cpumask which is initially set to |
| 7062 | * (HK_TYPE_WQ ∩ HK_TYPE_DOMAIN) house keeping mask and rewritten | 7111 | * HK_TYPE_DOMAIN house keeping mask and rewritten |
| 7063 | * by any subsequent write to workqueue/cpumask sysfs file. | 7112 | * by any subsequent write to workqueue/cpumask sysfs file. |
| 7064 | */ | 7113 | */ |
| 7065 | if (!cpumask_and(cpumask, wq_requested_unbound_cpumask, hk)) | 7114 | if (!cpumask_and(cpumask, wq_requested_unbound_cpumask, hk)) |
| @@ -7078,13 +7127,7 @@ int workqueue_unbound_housekeeping_update(const struct cpumask *hk) · 12 unmodified lines | |||
| 7078 | 7127 | ||
| 7079 | static int parse_affn_scope(const char *val) | 7128 | static int parse_affn_scope(const char *val) |
| 7080 | { | 7129 | { |
| 7081 | int i; | 7130 | return sysfs_match_string(wq_affn_names, val); |
| 7082 | |||
| 7083 | for (i = 0; i < ARRAY_SIZE(wq_affn_names); i++) { | ||
| 7084 | if (!strncasecmp(val, wq_affn_names[i], strlen(wq_affn_names[i]))) | ||
| 7085 | return i; | ||
| 7086 | } | ||
| 7087 | return -EINVAL; | ||
| 7088 | } | 7131 | } |
| 7089 | 7132 | ||
| 7090 | static int wq_affn_dfl_set(const char *val, const struct kernel_param *kp) | 7133 | static int wq_affn_dfl_set(const char *val, const struct kernel_param *kp) |
| @@ -7191,7 +7234,26 @@ static struct attribute *wq_sysfs_attrs[] = { · 100 unmodified lines | |||
| 7191 | &dev_attr_max_active.attr, | 7234 | &dev_attr_max_active.attr, |
| 7192 | NULL, | 7235 | NULL, |
| 7193 | }; | 7236 | }; |
| 7194 | ATTRIBUTE_GROUPS(wq_sysfs); | 7237 | |
| 7238 | static umode_t wq_sysfs_is_visible(struct kobject *kobj, struct attribute *a, int n) | ||
| 7239 | { | ||
| 7240 | struct device *dev = kobj_to_dev(kobj); | ||
| 7241 | struct workqueue_struct *wq = dev_to_wq(dev); | ||
| 7242 | |||
| 7243 | /* | ||
| 7244 | * Adjusting max_active breaks ordering guarantee. Changing it has no | ||
| 7245 | * effect on BH worker. Limit max_active to RO in such case. | ||
| 7246 | */ | ||
| 7247 | if (wq->flags & (WQ_BH | __WQ_ORDERED)) | ||
| 7248 | return 0444; | ||
| 7249 | return a->mode; | ||
| 7250 | } | ||
| 7251 | |||
| 7252 | static const struct attribute_group wq_sysfs_group = { | ||
| 7253 | .is_visible = wq_sysfs_is_visible, | ||
| 7254 | .attrs = wq_sysfs_attrs, | ||
| 7255 | }; | ||
| 7256 | __ATTRIBUTE_GROUPS(wq_sysfs); | ||
| 7195 | 7257 | ||
| 7196 | static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr, | 7258 | static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr, |
| 7197 | char *buf) | 7259 | char *buf) |
| @@ -7494,13 +7556,6 @@ int workqueue_sysfs_register(struct workqueue_struct *wq) · 296 unmodified lines | |||
| 7494 | struct wq_device *wq_dev; | 7556 | struct wq_device *wq_dev; |
| 7495 | int ret; | 7557 | int ret; |
| 7496 | 7558 | ||
| 7497 | /* | ||
| 7498 | * Adjusting max_active breaks ordering guarantee. Disallow exposing | ||
| 7499 | * ordered workqueues. | ||
| 7500 | */ | ||
| 7501 | if (WARN_ON(wq->flags & __WQ_ORDERED)) | ||
| 7502 | return -EINVAL; | ||
| 7503 | |||
| 7504 | wq->wq_dev = wq_dev = kzalloc_obj(*wq_dev); | 7559 | wq->wq_dev = wq_dev = kzalloc_obj(*wq_dev); |
| 7505 | if (!wq_dev) | 7560 | if (!wq_dev) |
| 7506 | return -ENOMEM; | 7561 | return -ENOMEM; |
| @@ -7877,8 +7932,8 @@ void __init workqueue_init_early(void) · 370 unmodified lines | |||
| 7877 | { | 7932 | { |
| 7878 | struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_SYSTEM]; | 7933 | struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_SYSTEM]; |
| 7879 | int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL }; | 7934 | int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL }; |
| 7880 | void (*irq_work_fns[2])(struct irq_work *) = { bh_pool_kick_normal, | 7935 | void (*irq_work_fns[NR_STD_WORKER_POOLS])(struct irq_work *) = |
| 7881 | bh_pool_kick_highpri }; | 7936 | { bh_pool_kick_normal, bh_pool_kick_highpri }; |
| 7882 | int i, cpu; | 7937 | int i, cpu; |
| 7883 | 7938 | ||
| 7884 | BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long)); | 7939 | BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long)); |
| @@ -7890,7 +7945,6 @@ void __init workqueue_init_early(void) · 5 unmodified lines | |||
| 7890 | 7945 | ||
| 7891 | cpumask_copy(wq_online_cpumask, cpu_online_mask); | 7946 | cpumask_copy(wq_online_cpumask, cpu_online_mask); |
| 7892 | cpumask_copy(wq_unbound_cpumask, cpu_possible_mask); | 7947 | cpumask_copy(wq_unbound_cpumask, cpu_possible_mask); |
| 7893 | restrict_unbound_cpumask("HK_TYPE_WQ", housekeeping_cpumask(HK_TYPE_WQ)); | ||
| 7894 | restrict_unbound_cpumask("HK_TYPE_DOMAIN", housekeeping_cpumask(HK_TYPE_DOMAIN)); | 7948 | restrict_unbound_cpumask("HK_TYPE_DOMAIN", housekeeping_cpumask(HK_TYPE_DOMAIN)); |
| 7895 | if (!cpumask_empty(&wq_cmdline_cpumask)) | 7949 | if (!cpumask_empty(&wq_cmdline_cpumask)) |
| 7896 | restrict_unbound_cpumask("workqueue.unbound_cpus", &wq_cmdline_cpumask); | 7950 | restrict_unbound_cpumask("workqueue.unbound_cpus", &wq_cmdline_cpumask); |
| @@ -7974,11 +8028,12 @@ void __init workqueue_init_early(void) · 77 unmodified lines | |||
| 7974 | system_bh_wq = alloc_workqueue("events_bh", WQ_BH | WQ_PERCPU, 0); | 8028 | system_bh_wq = alloc_workqueue("events_bh", WQ_BH | WQ_PERCPU, 0); |
| 7975 | system_bh_highpri_wq = alloc_workqueue("events_bh_highpri", | 8029 | system_bh_highpri_wq = alloc_workqueue("events_bh_highpri", |
| 7976 | WQ_BH | WQ_HIGHPRI | WQ_PERCPU, 0); | 8030 | WQ_BH | WQ_HIGHPRI | WQ_PERCPU, 0); |
| 8031 | system_dfl_long_wq = alloc_workqueue("events_dfl_long", WQ_UNBOUND, WQ_MAX_ACTIVE); | ||
| 7977 | BUG_ON(!system_wq || !system_percpu_wq|| !system_highpri_wq || !system_long_wq || | 8032 | BUG_ON(!system_wq || !system_percpu_wq|| !system_highpri_wq || !system_long_wq || |
| 7978 | !system_unbound_wq || !system_freezable_wq || !system_dfl_wq || | 8033 | !system_unbound_wq || !system_freezable_wq || !system_dfl_wq || |
| 7979 | !system_power_efficient_wq || | 8034 | !system_power_efficient_wq || |
| 7980 | !system_freezable_power_efficient_wq || | 8035 | !system_freezable_power_efficient_wq || |
| 7981 | !system_bh_wq || !system_bh_highpri_wq); | 8036 | !system_bh_wq || !system_bh_highpri_wq || !system_dfl_long_wq); |
| 7982 | } | 8037 | } |
| 7983 | 8038 | ||
| 7984 | static void __init wq_cpu_intensive_thresh_init(void) | 8039 | static void __init wq_cpu_intensive_thresh_init(void) |
| @@ -8144,6 +8199,186 @@ static bool __init cpus_share_numa(int cpu0, int cpu1) · 159 unmodified lines | |||
| 8144 | return cpu_to_node(cpu0) == cpu_to_node(cpu1); | 8199 | return cpu_to_node(cpu0) == cpu_to_node(cpu1); |
| 8145 | } | 8200 | } |
| 8146 | 8201 | ||
| 8202 | /* Maps each CPU to its shard index within the LLC pod it belongs to */ | ||
| 8203 | static int cpu_shard_id[NR_CPUS] __initdata; | ||
| 8204 | |||
| 8205 | /** | ||
| 8206 | * llc_count_cores - count distinct cores (SMT groups) within an LLC pod | ||
| 8207 | * @pod_cpus: the cpumask of CPUs in the LLC pod | ||
| 8208 | * @smt_pods: the SMT pod type, used to identify sibling groups | ||
| 8209 | * | ||
| 8210 | * A core is represented by the lowest-numbered CPU in its SMT group. Returns | ||
| 8211 | * the number of distinct cores found in @pod_cpus. | ||
| 8212 | */ | ||
| 8213 | static int __init llc_count_cores(const struct cpumask *pod_cpus, | ||
| 8214 | struct wq_pod_type *smt_pods) | ||
| 8215 | { | ||
| 8216 | const struct cpumask *sibling_cpus; | ||
| 8217 | int nr_cores = 0, c; | ||
| 8218 | |||
| 8219 | /* | ||
| 8220 | * Count distinct cores by only counting the first CPU in each | ||
| 8221 | * SMT sibling group. | ||
| 8222 | */ | ||
| 8223 | for_each_cpu(c, pod_cpus) { | ||
| 8224 | sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]]; | ||
| 8225 | if (cpumask_first(sibling_cpus) == c) | ||
| 8226 | nr_cores++; | ||
| 8227 | } | ||
| 8228 | |||
| 8229 | return nr_cores; | ||
| 8230 | } | ||
| 8231 | |||
| 8232 | /* | ||
| 8233 | * llc_shard_size - number of cores in a given shard | ||
| 8234 | * | ||
| 8235 | * Cores are spread as evenly as possible. The first @nr_large_shards shards are | ||
| 8236 | * "large shards" with (cores_per_shard + 1) cores; the rest are "default | ||
| 8237 | * shards" with cores_per_shard cores. | ||
| 8238 | */ | ||
| 8239 | static int __init llc_shard_size(int shard_id, int cores_per_shard, int nr_large_shards) | ||
| 8240 | { | ||
| 8241 | /* The first @nr_large_shards shards are large shards */ | ||
| 8242 | if (shard_id < nr_large_shards) | ||
| 8243 | return cores_per_shard + 1; | ||
| 8244 | |||
| 8245 | /* The remaining shards are default shards */ | ||
| 8246 | return cores_per_shard; | ||
| 8247 | } | ||
| 8248 | |||
| 8249 | /* | ||
| 8250 | * llc_calc_shard_layout - compute the shard layout for an LLC pod | ||
| 8251 | * @nr_cores: number of distinct cores in the LLC pod | ||
| 8252 | * | ||
| 8253 | * Chooses the number of shards that keeps average shard size closest to | ||
| 8254 | * wq_cache_shard_size. Returns a struct describing the total number of shards, | ||
| 8255 | * the base size of each, and how many are large shards. | ||
| 8256 | */ | ||
| 8257 | static struct llc_shard_layout __init llc_calc_shard_layout(int nr_cores) | ||
| 8258 | { | ||
| 8259 | struct llc_shard_layout layout; | ||
| 8260 | |||
| 8261 | /* Ensure at least one shard; pick the count closest to the target size */ | ||
| 8262 | layout.nr_shards = max(1, DIV_ROUND_CLOSEST(nr_cores, wq_cache_shard_size)); | ||
| 8263 | layout.cores_per_shard = nr_cores / layout.nr_shards; | ||
| 8264 | layout.nr_large_shards = nr_cores % layout.nr_shards; | ||
| 8265 | |||
| 8266 | return layout; | ||
| 8267 | } | ||
| 8268 | |||
| 8269 | /* | ||
| 8270 | * llc_shard_is_full - check whether a shard has reached its core capacity | ||
| 8271 | * @cores_in_shard: number of cores already assigned to this shard | ||
| 8272 | * @shard_id: index of the shard being checked | ||
| 8273 | * @layout: the shard layout computed by llc_calc_shard_layout() | ||
| 8274 | * | ||
| 8275 | * Returns true if @cores_in_shard equals the expected size for @shard_id. | ||
| 8276 | */ | ||
| 8277 | static bool __init llc_shard_is_full(int cores_in_shard, int shard_id, | ||
| 8278 | const struct llc_shard_layout *layout) | ||
| 8279 | { | ||
| 8280 | return cores_in_shard == llc_shard_size(shard_id, layout->cores_per_shard, | ||
| 8281 | layout->nr_large_shards); | ||
| 8282 | } | ||
| 8283 | |||
| 8284 | /** | ||
| 8285 | * llc_populate_cpu_shard_id - populate cpu_shard_id[] for each CPU in an LLC pod | ||
| 8286 | * @pod_cpus: the cpumask of CPUs in the LLC pod | ||
| 8287 | * @smt_pods: the SMT pod type, used to identify sibling groups | ||
| 8288 | * @nr_cores: number of distinct cores in @pod_cpus (from llc_count_cores()) | ||
| 8289 | * | ||
| 8290 | * Walks @pod_cpus in order. At each SMT group leader, advances to the next | ||
| 8291 | * shard once the current shard is full. Results are written to cpu_shard_id[]. | ||
| 8292 | */ | ||
| 8293 | static void __init llc_populate_cpu_shard_id(const struct cpumask *pod_cpus, | ||
| 8294 | struct wq_pod_type *smt_pods, | ||
| 8295 | int nr_cores) | ||
| 8296 | { | ||
| 8297 | struct llc_shard_layout layout = llc_calc_shard_layout(nr_cores); | ||
| 8298 | const struct cpumask *sibling_cpus; | ||
| 8299 | /* Count the number of cores in the current shard_id */ | ||
| 8300 | int cores_in_shard = 0; | ||
| 8301 | unsigned int leader; | ||
| 8302 | /* This is a cursor for the shards. Go from zero to nr_shards - 1*/ | ||
| 8303 | int shard_id = 0; | ||
| 8304 | int c; | ||
| 8305 | |||
| 8306 | /* Iterate at every CPU for a given LLC pod, and assign it a shard */ | ||
| 8307 | for_each_cpu(c, pod_cpus) { | ||
| 8308 | sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]]; | ||
| 8309 | if (cpumask_first(sibling_cpus) == c) { | ||
| 8310 | /* This is the CPU leader for the siblings */ | ||
| 8311 | if (llc_shard_is_full(cores_in_shard, shard_id, &layout)) { | ||
| 8312 | shard_id++; | ||
| 8313 | cores_in_shard = 0; | ||
| 8314 | } | ||
| 8315 | cores_in_shard++; | ||
| 8316 | cpu_shard_id[c] = shard_id; | ||
| 8317 | } else { | ||
| 8318 | /* | ||
| 8319 | * The siblings' shard MUST be the same as the leader. | ||
| 8320 | * never split threads in the same core. | ||
| 8321 | */ | ||
| 8322 | leader = cpumask_first(sibling_cpus); | ||
| 8323 | |||
| 8324 | /* | ||
| 8325 | * This check silences a Warray-bounds warning on UP | ||
| 8326 | * configs where NR_CPUS=1 makes cpu_shard_id[] | ||
| 8327 | * a single-element array, and the compiler can't | ||
| 8328 | * prove the index is always 0. | ||
| 8329 | */ | ||
| 8330 | if (WARN_ON_ONCE(leader >= nr_cpu_ids)) | ||
| 8331 | continue; | ||
| 8332 | cpu_shard_id[c] = cpu_shard_id[leader]; | ||
| 8333 | } | ||
| 8334 | } | ||
| 8335 | |||
| 8336 | WARN_ON_ONCE(shard_id != (layout.nr_shards - 1)); | ||
| 8337 | } | ||
| 8338 | |||
| 8339 | /** | ||
| 8340 | * precompute_cache_shard_ids - assign each CPU its shard index within its LLC | ||
| 8341 | * | ||
| 8342 | * Iterates over all LLC pods. For each pod, counts distinct cores then assigns | ||
| 8343 | * shard indices to all CPUs in the pod. Must be called after WQ_AFFN_CACHE and | ||
| 8344 | * WQ_AFFN_SMT have been initialized. | ||
| 8345 | */ | ||
| 8346 | static void __init precompute_cache_shard_ids(void) | ||
| 8347 | { | ||
| 8348 | struct wq_pod_type *llc_pods = &wq_pod_types[WQ_AFFN_CACHE]; | ||
| 8349 | struct wq_pod_type *smt_pods = &wq_pod_types[WQ_AFFN_SMT]; | ||
| 8350 | const struct cpumask *cpus_sharing_llc; | ||
| 8351 | int nr_cores; | ||
| 8352 | int pod; | ||
| 8353 | |||
| 8354 | if (!wq_cache_shard_size) { | ||
| 8355 | pr_warn("workqueue: cache_shard_size must be > 0, setting to 1\n"); | ||
| 8356 | wq_cache_shard_size = 1; | ||
| 8357 | } | ||
| 8358 | |||
| 8359 | for (pod = 0; pod < llc_pods->nr_pods; pod++) { | ||
| 8360 | cpus_sharing_llc = llc_pods->pod_cpus[pod]; | ||
| 8361 | |||
| 8362 | /* Number of cores in this given LLC */ | ||
| 8363 | nr_cores = llc_count_cores(cpus_sharing_llc, smt_pods); | ||
| 8364 | llc_populate_cpu_shard_id(cpus_sharing_llc, smt_pods, nr_cores); | ||
| 8365 | } | ||
| 8366 | } | ||
| 8367 | |||
| 8368 | /* | ||
| 8369 | * cpus_share_cache_shard - test whether two CPUs belong to the same cache shard | ||
| 8370 | * | ||
| 8371 | * Two CPUs share a cache shard if they are in the same LLC and have the same | ||
| 8372 | * shard index. Used as the pod affinity callback for WQ_AFFN_CACHE_SHARD. | ||
| 8373 | */ | ||
| 8374 | static bool __init cpus_share_cache_shard(int cpu0, int cpu1) | ||
| 8375 | { | ||
| 8376 | if (!cpus_share_cache(cpu0, cpu1)) | ||
| 8377 | return false; | ||
| 8378 | |||
| 8379 | return cpu_shard_id[cpu0] == cpu_shard_id[cpu1]; | ||
| 8380 | } | ||
| 8381 | |||
| 8147 | /** | 8382 | /** |
| 8148 | * workqueue_init_topology - initialize CPU pods for unbound workqueues | 8383 | * workqueue_init_topology - initialize CPU pods for unbound workqueues |
| 8149 | * | 8384 | * |
| @@ -8159,6 +8394,8 @@ void __init workqueue_init_topology(void) · 9 unmodified lines | |||
| 8159 | init_pod_type(&wq_pod_types[WQ_AFFN_CPU], cpus_dont_share); | 8394 | init_pod_type(&wq_pod_types[WQ_AFFN_CPU], cpus_dont_share); |
| 8160 | init_pod_type(&wq_pod_types[WQ_AFFN_SMT], cpus_share_smt); | 8395 | init_pod_type(&wq_pod_types[WQ_AFFN_SMT], cpus_share_smt); |
| 8161 | init_pod_type(&wq_pod_types[WQ_AFFN_CACHE], cpus_share_cache); | 8396 | init_pod_type(&wq_pod_types[WQ_AFFN_CACHE], cpus_share_cache); |
| 8397 | precompute_cache_shard_ids(); | ||
| 8398 | init_pod_type(&wq_pod_types[WQ_AFFN_CACHE_SHARD], cpus_share_cache_shard); | ||
| 8162 | init_pod_type(&wq_pod_types[WQ_AFFN_NUMA], cpus_share_numa); | 8399 | init_pod_type(&wq_pod_types[WQ_AFFN_NUMA], cpus_share_numa); |
| 8163 | 8400 | ||
| 8164 | wq_topo_initialized = true; | 8401 | wq_topo_initialized = true; |