Pull workqueue updates from Tejun Heo: - New default WQ_AFFN_CACHE_SHARD affinity scope subdivides LLCs into smaller shards to improve scalability on machines with many CPUs per LLC - Misc: - system_dfl_long_wq for long unbound works - devm_alloc_workqueue() for device-managed allocation - sysfs exposure for ordered workqueues and the EFI workqueue - removal of HK_TYPE_WQ from wq_unbound_cpumask - various small fixes * tag 'wq-for-7.1' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/wq: (21 commits) workqueue: validate cpumask_first() result in llc_populate_cpu_shard_id() workqueue: use NR_STD_WORKER_POOLS instead of hardcoded value workqueue: avoid unguarded 64-bit division docs: workqueue: document WQ_AFFN_CACHE_SHARD affinity scope workqueue: add test_workqueue benchmark module tools/workqueue: add CACHE_SHARD support to wq_dump.py workqueue: set WQ_AFFN_CACHE_SHARD as the default affinity scope workqueue: add WQ_AFFN_CACHE_SHARD affinity scope workqueue: fix typo in WQ_AFFN_SMT comment workqueue: Remove HK_TYPE_WQ from affecting wq_unbound_cpumask workqueue: unlink pwqs from wq->pwqs list in alloc_and_link_pwqs() error path workqueue: Remove NULL wq WARN in __queue_delayed_work() workqueue: fix parse_affn_scope() prefix matching bug workqueue: devres: Add device-managed allocate workqueue workqueue: Add system_dfl_long_wq for long unbound works tools/workqueue/wq_dump.py: add NODE prefix to all node columns tools/workqueue/wq_dump.py: fix column alignment in node_nr/max_active section tools/workqueue/wq_dump.py: remove backslash separator from node_nr/max_active header efi: Allow to expose the workqueue via sysfs workqueue: Allow to expose ordered workqueues via sysfs ...
Showing the changes to kernel/workqueue.c (1 of 10 files). Show all
Unified · Split
| @@ -41,6 +41,7 @@ · 40 unmodified lines | ||
| 41 | 41 | #include <linux/mempolicy.h> |
| 42 | 42 | #include <linux/freezer.h> |
| 43 | 43 | #include <linux/debug_locks.h> |
| 44 | +#include <linux/device/devres.h> | |
| 44 | 45 | #include <linux/lockdep.h> |
| 45 | 46 | #include <linux/idr.h> |
| 46 | 47 | #include <linux/jhash.h> |
| @@ -130,6 +131,14 @@ enum wq_internal_consts { · 83 unmodified lines | ||
| 130 | 131 | WORKER_ID_LEN = 10 + WQ_NAME_LEN, /* "kworker/R-" + WQ_NAME_LEN */ |
| 131 | 132 | }; |
| 132 | 133 | |
| 134 | +/* Layout of shards within one LLC pod */ | |
| 135 | +struct llc_shard_layout { | |
| 136 | + int nr_large_shards; /* number of large shards (cores_per_shard + 1) */ | |
| 137 | + int cores_per_shard; /* base number of cores per default shard */ | |
| 138 | + int nr_shards; /* total number of shards */ | |
| 139 | + /* nr_default shards = (nr_shards - nr_large_shards) */ | |
| 140 | +}; | |
| 141 | + | |
| 133 | 142 | /* |
| 134 | 143 | * We don't want to trap softirq for too long. See MAX_SOFTIRQ_TIME and |
| 135 | 144 | * MAX_SOFTIRQ_RESTART in kernel/softirq.c. These are macros because |
| @@ -404,11 +413,12 @@ struct work_offq_data { · 268 unmodified lines | ||
| 404 | 413 | u32 flags; |
| 405 | 414 | }; |
| 406 | 415 | |
| 407 | −static const char *wq_affn_names[WQ_AFFN_NR_TYPES] = { | |
| 416 | +static const char * const wq_affn_names[WQ_AFFN_NR_TYPES] = { | |
| 408 | 417 | [WQ_AFFN_DFL] = "default", |
| 409 | 418 | [WQ_AFFN_CPU] = "cpu", |
| 410 | 419 | [WQ_AFFN_SMT] = "smt", |
| 411 | 420 | [WQ_AFFN_CACHE] = "cache", |
| 421 | + [WQ_AFFN_CACHE_SHARD] = "cache_shard", | |
| 412 | 422 | [WQ_AFFN_NUMA] = "numa", |
| 413 | 423 | [WQ_AFFN_SYSTEM] = "system", |
| 414 | 424 | }; |
| @@ -431,13 +441,16 @@ module_param_named(cpu_intensive_warning_thresh, wq_cpu_intensive_warning_thresh · 16 unmodified lines | ||
| 431 | 441 | static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT); |
| 432 | 442 | module_param_named(power_efficient, wq_power_efficient, bool, 0444); |
| 433 | 443 | |
| 444 | +static unsigned int wq_cache_shard_size = 8; | |
| 445 | +module_param_named(cache_shard_size, wq_cache_shard_size, uint, 0444); | |
| 446 | + | |
| 434 | 447 | static bool wq_online; /* can kworkers be created yet? */ |
| 435 | 448 | static bool wq_topo_initialized __read_mostly = false; |
| 436 | 449 | |
| 437 | 450 | static struct kmem_cache *pwq_cache; |
| 438 | 451 | |
| 439 | 452 | static struct wq_pod_type wq_pod_types[WQ_AFFN_NR_TYPES]; |
| 440 | −static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE; | |
| 453 | +static enum wq_affn_scope wq_affn_dfl = WQ_AFFN_CACHE_SHARD; | |
| 441 | 454 | |
| 442 | 455 | /* buf for wq_update_unbound_pod_attrs(), protected by CPU hotplug exclusion */ |
| 443 | 456 | static struct workqueue_attrs *unbound_wq_update_pwq_attrs_buf; |
| @@ -530,6 +543,8 @@ struct workqueue_struct *system_bh_wq; · 86 unmodified lines | ||
| 530 | 543 | EXPORT_SYMBOL_GPL(system_bh_wq); |
| 531 | 544 | struct workqueue_struct *system_bh_highpri_wq; |
| 532 | 545 | EXPORT_SYMBOL_GPL(system_bh_highpri_wq); |
| 546 | +struct workqueue_struct *system_dfl_long_wq __ro_after_init; | |
| 547 | +EXPORT_SYMBOL_GPL(system_dfl_long_wq); | |
| 533 | 548 | |
| 534 | 549 | static int worker_thread(void *__worker); |
| 535 | 550 | static void workqueue_sysfs_unregister(struct workqueue_struct *wq); |
| @@ -2519,7 +2534,6 @@ static void __queue_delayed_work(int cpu, struct workqueue_struct *wq, · 1983 unmodified lines | ||
| 2519 | 2534 | struct timer_list *timer = &dwork->timer; |
| 2520 | 2535 | struct work_struct *work = &dwork->work; |
| 2521 | 2536 | |
| 2522 | − WARN_ON_ONCE(!wq); | |
| 2523 | 2537 | WARN_ON_ONCE(timer->function != delayed_work_timer_fn); |
| 2524 | 2538 | WARN_ON_ONCE(timer_pending(timer)); |
| 2525 | 2539 | WARN_ON_ONCE(!list_empty(&work->entry)); |
| @@ -5635,8 +5649,16 @@ enomem: · 3109 unmodified lines | ||
| 5635 | 5649 | for_each_possible_cpu(cpu) { |
| 5636 | 5650 | struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu); |
| 5637 | 5651 | |
| 5638 | − if (pwq) | |
| 5652 | + if (pwq) { | |
| 5653 | + /* | |
| 5654 | + * Unlink pwq from wq->pwqs since link_pwq() | |
| 5655 | + * may have already added it. wq->mutex is not | |
| 5656 | + * needed as the wq has not been published yet. | |
| 5657 | + */ | |
| 5658 | + if (!list_empty(&pwq->pwqs_node)) | |
| 5659 | + list_del_rcu(&pwq->pwqs_node); | |
| 5639 | 5660 | kmem_cache_free(pwq_cache, pwq); |
| 5661 | + } | |
| 5640 | 5662 | } |
| 5641 | 5663 | free_percpu(wq->cpu_pwq); |
| 5642 | 5664 | wq->cpu_pwq = NULL; |
| @@ -5904,6 +5926,33 @@ struct workqueue_struct *alloc_workqueue_noprof(const char *fmt, · 261 unmodified lines | ||
| 5904 | 5926 | } |
| 5905 | 5927 | EXPORT_SYMBOL_GPL(alloc_workqueue_noprof); |
| 5906 | 5928 | |
| 5929 | +static void devm_workqueue_release(void *res) | |
| 5930 | +{ | |
| 5931 | + destroy_workqueue(res); | |
| 5932 | +} | |
| 5933 | + | |
| 5934 | +__printf(2, 5) struct workqueue_struct * | |
| 5935 | +devm_alloc_workqueue(struct device *dev, const char *fmt, unsigned int flags, | |
| 5936 | + int max_active, ...) | |
| 5937 | +{ | |
| 5938 | + struct workqueue_struct *wq; | |
| 5939 | + va_list args; | |
| 5940 | + int ret; | |
| 5941 | + | |
| 5942 | + va_start(args, max_active); | |
| 5943 | + wq = alloc_workqueue(fmt, flags, max_active, args); | |
| 5944 | + va_end(args); | |
| 5945 | + if (!wq) | |
| 5946 | + return NULL; | |
| 5947 | + | |
| 5948 | + ret = devm_add_action_or_reset(dev, devm_workqueue_release, wq); | |
| 5949 | + if (ret) | |
| 5950 | + return NULL; | |
| 5951 | + | |
| 5952 | + return wq; | |
| 5953 | +} | |
| 5954 | +EXPORT_SYMBOL_GPL(devm_alloc_workqueue); | |
| 5955 | + | |
| 5907 | 5956 | #ifdef CONFIG_LOCKDEP |
| 5908 | 5957 | __printf(1, 5) |
| 5909 | 5958 | struct workqueue_struct * |
| @@ -7059,7 +7108,7 @@ int workqueue_unbound_housekeeping_update(const struct cpumask *hk) · 1149 unmodified lines | ||
| 7059 | 7108 | /* |
| 7060 | 7109 | * If the operation fails, it will fall back to |
| 7061 | 7110 | * wq_requested_unbound_cpumask which is initially set to |
| 7062 | − * (HK_TYPE_WQ ∩ HK_TYPE_DOMAIN) house keeping mask and rewritten | |
| 7111 | + * HK_TYPE_DOMAIN house keeping mask and rewritten | |
| 7063 | 7112 | * by any subsequent write to workqueue/cpumask sysfs file. |
| 7064 | 7113 | */ |
| 7065 | 7114 | if (!cpumask_and(cpumask, wq_requested_unbound_cpumask, hk)) |
| @@ -7078,13 +7127,7 @@ int workqueue_unbound_housekeeping_update(const struct cpumask *hk) · 12 unmodified lines | ||
| 7078 | 7127 | |
| 7079 | 7128 | static int parse_affn_scope(const char *val) |
| 7080 | 7129 | { |
| 7081 | − int i; | |
| 7082 | − | |
| 7083 | − for (i = 0; i < ARRAY_SIZE(wq_affn_names); i++) { | |
| 7084 | − if (!strncasecmp(val, wq_affn_names[i], strlen(wq_affn_names[i]))) | |
| 7085 | − return i; | |
| 7086 | − } | |
| 7087 | − return -EINVAL; | |
| 7130 | + return sysfs_match_string(wq_affn_names, val); | |
| 7088 | 7131 | } |
| 7089 | 7132 | |
| 7090 | 7133 | static int wq_affn_dfl_set(const char *val, const struct kernel_param *kp) |
| @@ -7191,7 +7234,26 @@ static struct attribute *wq_sysfs_attrs[] = { · 100 unmodified lines | ||
| 7191 | 7234 | &dev_attr_max_active.attr, |
| 7192 | 7235 | NULL, |
| 7193 | 7236 | }; |
| 7194 | −ATTRIBUTE_GROUPS(wq_sysfs); | |
| 7237 | + | |
| 7238 | +static umode_t wq_sysfs_is_visible(struct kobject *kobj, struct attribute *a, int n) | |
| 7239 | +{ | |
| 7240 | + struct device *dev = kobj_to_dev(kobj); | |
| 7241 | + struct workqueue_struct *wq = dev_to_wq(dev); | |
| 7242 | + | |
| 7243 | + /* | |
| 7244 | + * Adjusting max_active breaks ordering guarantee. Changing it has no | |
| 7245 | + * effect on BH worker. Limit max_active to RO in such case. | |
| 7246 | + */ | |
| 7247 | + if (wq->flags & (WQ_BH | __WQ_ORDERED)) | |
| 7248 | + return 0444; | |
| 7249 | + return a->mode; | |
| 7250 | +} | |
| 7251 | + | |
| 7252 | +static const struct attribute_group wq_sysfs_group = { | |
| 7253 | + .is_visible = wq_sysfs_is_visible, | |
| 7254 | + .attrs = wq_sysfs_attrs, | |
| 7255 | +}; | |
| 7256 | +__ATTRIBUTE_GROUPS(wq_sysfs); | |
| 7195 | 7257 | |
| 7196 | 7258 | static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr, |
| 7197 | 7259 | char *buf) |
| @@ -7494,13 +7556,6 @@ int workqueue_sysfs_register(struct workqueue_struct *wq) · 296 unmodified lines | ||
| 7494 | 7556 | struct wq_device *wq_dev; |
| 7495 | 7557 | int ret; |
| 7496 | 7558 | |
| 7497 | − /* | |
| 7498 | − * Adjusting max_active breaks ordering guarantee. Disallow exposing | |
| 7499 | − * ordered workqueues. | |
| 7500 | − */ | |
| 7501 | − if (WARN_ON(wq->flags & __WQ_ORDERED)) | |
| 7502 | − return -EINVAL; | |
| 7503 | − | |
| 7504 | 7559 | wq->wq_dev = wq_dev = kzalloc_obj(*wq_dev); |
| 7505 | 7560 | if (!wq_dev) |
| 7506 | 7561 | return -ENOMEM; |
| @@ -7877,8 +7932,8 @@ void __init workqueue_init_early(void) · 370 unmodified lines | ||
| 7877 | 7932 | { |
| 7878 | 7933 | struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_SYSTEM]; |
| 7879 | 7934 | int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL }; |
| 7880 | − void (*irq_work_fns[2])(struct irq_work *) = { bh_pool_kick_normal, | |
| 7881 | − bh_pool_kick_highpri }; | |
| 7935 | + void (*irq_work_fns[NR_STD_WORKER_POOLS])(struct irq_work *) = | |
| 7936 | + { bh_pool_kick_normal, bh_pool_kick_highpri }; | |
| 7882 | 7937 | int i, cpu; |
| 7883 | 7938 | |
| 7884 | 7939 | BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long)); |
| @@ -7890,7 +7945,6 @@ void __init workqueue_init_early(void) · 5 unmodified lines | ||
| 7890 | 7945 | |
| 7891 | 7946 | cpumask_copy(wq_online_cpumask, cpu_online_mask); |
| 7892 | 7947 | cpumask_copy(wq_unbound_cpumask, cpu_possible_mask); |
| 7893 | − restrict_unbound_cpumask("HK_TYPE_WQ", housekeeping_cpumask(HK_TYPE_WQ)); | |
| 7894 | 7948 | restrict_unbound_cpumask("HK_TYPE_DOMAIN", housekeeping_cpumask(HK_TYPE_DOMAIN)); |
| 7895 | 7949 | if (!cpumask_empty(&wq_cmdline_cpumask)) |
| 7896 | 7950 | restrict_unbound_cpumask("workqueue.unbound_cpus", &wq_cmdline_cpumask); |
| @@ -7974,11 +8028,12 @@ void __init workqueue_init_early(void) · 77 unmodified lines | ||
| 7974 | 8028 | system_bh_wq = alloc_workqueue("events_bh", WQ_BH | WQ_PERCPU, 0); |
| 7975 | 8029 | system_bh_highpri_wq = alloc_workqueue("events_bh_highpri", |
| 7976 | 8030 | WQ_BH | WQ_HIGHPRI | WQ_PERCPU, 0); |
| 8031 | + system_dfl_long_wq = alloc_workqueue("events_dfl_long", WQ_UNBOUND, WQ_MAX_ACTIVE); | |
| 7977 | 8032 | BUG_ON(!system_wq || !system_percpu_wq|| !system_highpri_wq || !system_long_wq || |
| 7978 | 8033 | !system_unbound_wq || !system_freezable_wq || !system_dfl_wq || |
| 7979 | 8034 | !system_power_efficient_wq || |
| 7980 | 8035 | !system_freezable_power_efficient_wq || |
| 7981 | − !system_bh_wq || !system_bh_highpri_wq); | |
| 8036 | + !system_bh_wq || !system_bh_highpri_wq || !system_dfl_long_wq); | |
| 7982 | 8037 | } |
| 7983 | 8038 | |
| 7984 | 8039 | static void __init wq_cpu_intensive_thresh_init(void) |
| @@ -8144,6 +8199,186 @@ static bool __init cpus_share_numa(int cpu0, int cpu1) · 159 unmodified lines | ||
| 8144 | 8199 | return cpu_to_node(cpu0) == cpu_to_node(cpu1); |
| 8145 | 8200 | } |
| 8146 | 8201 | |
| 8202 | +/* Maps each CPU to its shard index within the LLC pod it belongs to */ | |
| 8203 | +static int cpu_shard_id[NR_CPUS] __initdata; | |
| 8204 | + | |
| 8205 | +/** | |
| 8206 | + * llc_count_cores - count distinct cores (SMT groups) within an LLC pod | |
| 8207 | + * @pod_cpus: the cpumask of CPUs in the LLC pod | |
| 8208 | + * @smt_pods: the SMT pod type, used to identify sibling groups | |
| 8209 | + * | |
| 8210 | + * A core is represented by the lowest-numbered CPU in its SMT group. Returns | |
| 8211 | + * the number of distinct cores found in @pod_cpus. | |
| 8212 | + */ | |
| 8213 | +static int __init llc_count_cores(const struct cpumask *pod_cpus, | |
| 8214 | + struct wq_pod_type *smt_pods) | |
| 8215 | +{ | |
| 8216 | + const struct cpumask *sibling_cpus; | |
| 8217 | + int nr_cores = 0, c; | |
| 8218 | + | |
| 8219 | + /* | |
| 8220 | + * Count distinct cores by only counting the first CPU in each | |
| 8221 | + * SMT sibling group. | |
| 8222 | + */ | |
| 8223 | + for_each_cpu(c, pod_cpus) { | |
| 8224 | + sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]]; | |
| 8225 | + if (cpumask_first(sibling_cpus) == c) | |
| 8226 | + nr_cores++; | |
| 8227 | + } | |
| 8228 | + | |
| 8229 | + return nr_cores; | |
| 8230 | +} | |
| 8231 | + | |
| 8232 | +/* | |
| 8233 | + * llc_shard_size - number of cores in a given shard | |
| 8234 | + * | |
| 8235 | + * Cores are spread as evenly as possible. The first @nr_large_shards shards are | |
| 8236 | + * "large shards" with (cores_per_shard + 1) cores; the rest are "default | |
| 8237 | + * shards" with cores_per_shard cores. | |
| 8238 | + */ | |
| 8239 | +static int __init llc_shard_size(int shard_id, int cores_per_shard, int nr_large_shards) | |
| 8240 | +{ | |
| 8241 | + /* The first @nr_large_shards shards are large shards */ | |
| 8242 | + if (shard_id < nr_large_shards) | |
| 8243 | + return cores_per_shard + 1; | |
| 8244 | + | |
| 8245 | + /* The remaining shards are default shards */ | |
| 8246 | + return cores_per_shard; | |
| 8247 | +} | |
| 8248 | + | |
| 8249 | +/* | |
| 8250 | + * llc_calc_shard_layout - compute the shard layout for an LLC pod | |
| 8251 | + * @nr_cores: number of distinct cores in the LLC pod | |
| 8252 | + * | |
| 8253 | + * Chooses the number of shards that keeps average shard size closest to | |
| 8254 | + * wq_cache_shard_size. Returns a struct describing the total number of shards, | |
| 8255 | + * the base size of each, and how many are large shards. | |
| 8256 | + */ | |
| 8257 | +static struct llc_shard_layout __init llc_calc_shard_layout(int nr_cores) | |
| 8258 | +{ | |
| 8259 | + struct llc_shard_layout layout; | |
| 8260 | + | |
| 8261 | + /* Ensure at least one shard; pick the count closest to the target size */ | |
| 8262 | + layout.nr_shards = max(1, DIV_ROUND_CLOSEST(nr_cores, wq_cache_shard_size)); | |
| 8263 | + layout.cores_per_shard = nr_cores / layout.nr_shards; | |
| 8264 | + layout.nr_large_shards = nr_cores % layout.nr_shards; | |
| 8265 | + | |
| 8266 | + return layout; | |
| 8267 | +} | |
| 8268 | + | |
| 8269 | +/* | |
| 8270 | + * llc_shard_is_full - check whether a shard has reached its core capacity | |
| 8271 | + * @cores_in_shard: number of cores already assigned to this shard | |
| 8272 | + * @shard_id: index of the shard being checked | |
| 8273 | + * @layout: the shard layout computed by llc_calc_shard_layout() | |
| 8274 | + * | |
| 8275 | + * Returns true if @cores_in_shard equals the expected size for @shard_id. | |
| 8276 | + */ | |
| 8277 | +static bool __init llc_shard_is_full(int cores_in_shard, int shard_id, | |
| 8278 | + const struct llc_shard_layout *layout) | |
| 8279 | +{ | |
| 8280 | + return cores_in_shard == llc_shard_size(shard_id, layout->cores_per_shard, | |
| 8281 | + layout->nr_large_shards); | |
| 8282 | +} | |
| 8283 | + | |
| 8284 | +/** | |
| 8285 | + * llc_populate_cpu_shard_id - populate cpu_shard_id[] for each CPU in an LLC pod | |
| 8286 | + * @pod_cpus: the cpumask of CPUs in the LLC pod | |
| 8287 | + * @smt_pods: the SMT pod type, used to identify sibling groups | |
| 8288 | + * @nr_cores: number of distinct cores in @pod_cpus (from llc_count_cores()) | |
| 8289 | + * | |
| 8290 | + * Walks @pod_cpus in order. At each SMT group leader, advances to the next | |
| 8291 | + * shard once the current shard is full. Results are written to cpu_shard_id[]. | |
| 8292 | + */ | |
| 8293 | +static void __init llc_populate_cpu_shard_id(const struct cpumask *pod_cpus, | |
| 8294 | + struct wq_pod_type *smt_pods, | |
| 8295 | + int nr_cores) | |
| 8296 | +{ | |
| 8297 | + struct llc_shard_layout layout = llc_calc_shard_layout(nr_cores); | |
| 8298 | + const struct cpumask *sibling_cpus; | |
| 8299 | + /* Count the number of cores in the current shard_id */ | |
| 8300 | + int cores_in_shard = 0; | |
| 8301 | + unsigned int leader; | |
| 8302 | + /* This is a cursor for the shards. Go from zero to nr_shards - 1*/ | |
| 8303 | + int shard_id = 0; | |
| 8304 | + int c; | |
| 8305 | + | |
| 8306 | + /* Iterate at every CPU for a given LLC pod, and assign it a shard */ | |
| 8307 | + for_each_cpu(c, pod_cpus) { | |
| 8308 | + sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]]; | |
| 8309 | + if (cpumask_first(sibling_cpus) == c) { | |
| 8310 | + /* This is the CPU leader for the siblings */ | |
| 8311 | + if (llc_shard_is_full(cores_in_shard, shard_id, &layout)) { | |
| 8312 | + shard_id++; | |
| 8313 | + cores_in_shard = 0; | |
| 8314 | + } | |
| 8315 | + cores_in_shard++; | |
| 8316 | + cpu_shard_id[c] = shard_id; | |
| 8317 | + } else { | |
| 8318 | + /* | |
| 8319 | + * The siblings' shard MUST be the same as the leader. | |
| 8320 | + * never split threads in the same core. | |
| 8321 | + */ | |
| 8322 | + leader = cpumask_first(sibling_cpus); | |
| 8323 | + | |
| 8324 | + /* | |
| 8325 | + * This check silences a Warray-bounds warning on UP | |
| 8326 | + * configs where NR_CPUS=1 makes cpu_shard_id[] | |
| 8327 | + * a single-element array, and the compiler can't | |
| 8328 | + * prove the index is always 0. | |
| 8329 | + */ | |
| 8330 | + if (WARN_ON_ONCE(leader >= nr_cpu_ids)) | |
| 8331 | + continue; | |
| 8332 | + cpu_shard_id[c] = cpu_shard_id[leader]; | |
| 8333 | + } | |
| 8334 | + } | |
| 8335 | + | |
| 8336 | + WARN_ON_ONCE(shard_id != (layout.nr_shards - 1)); | |
| 8337 | +} | |
| 8338 | + | |
| 8339 | +/** | |
| 8340 | + * precompute_cache_shard_ids - assign each CPU its shard index within its LLC | |
| 8341 | + * | |
| 8342 | + * Iterates over all LLC pods. For each pod, counts distinct cores then assigns | |
| 8343 | + * shard indices to all CPUs in the pod. Must be called after WQ_AFFN_CACHE and | |
| 8344 | + * WQ_AFFN_SMT have been initialized. | |
| 8345 | + */ | |
| 8346 | +static void __init precompute_cache_shard_ids(void) | |
| 8347 | +{ | |
| 8348 | + struct wq_pod_type *llc_pods = &wq_pod_types[WQ_AFFN_CACHE]; | |
| 8349 | + struct wq_pod_type *smt_pods = &wq_pod_types[WQ_AFFN_SMT]; | |
| 8350 | + const struct cpumask *cpus_sharing_llc; | |
| 8351 | + int nr_cores; | |
| 8352 | + int pod; | |
| 8353 | + | |
| 8354 | + if (!wq_cache_shard_size) { | |
| 8355 | + pr_warn("workqueue: cache_shard_size must be > 0, setting to 1\n"); | |
| 8356 | + wq_cache_shard_size = 1; | |
| 8357 | + } | |
| 8358 | + | |
| 8359 | + for (pod = 0; pod < llc_pods->nr_pods; pod++) { | |
| 8360 | + cpus_sharing_llc = llc_pods->pod_cpus[pod]; | |
| 8361 | + | |
| 8362 | + /* Number of cores in this given LLC */ | |
| 8363 | + nr_cores = llc_count_cores(cpus_sharing_llc, smt_pods); | |
| 8364 | + llc_populate_cpu_shard_id(cpus_sharing_llc, smt_pods, nr_cores); | |
| 8365 | + } | |
| 8366 | +} | |
| 8367 | + | |
| 8368 | +/* | |
| 8369 | + * cpus_share_cache_shard - test whether two CPUs belong to the same cache shard | |
| 8370 | + * | |
| 8371 | + * Two CPUs share a cache shard if they are in the same LLC and have the same | |
| 8372 | + * shard index. Used as the pod affinity callback for WQ_AFFN_CACHE_SHARD. | |
| 8373 | + */ | |
| 8374 | +static bool __init cpus_share_cache_shard(int cpu0, int cpu1) | |
| 8375 | +{ | |
| 8376 | + if (!cpus_share_cache(cpu0, cpu1)) | |
| 8377 | + return false; | |
| 8378 | + | |
| 8379 | + return cpu_shard_id[cpu0] == cpu_shard_id[cpu1]; | |
| 8380 | +} | |
| 8381 | + | |
| 8147 | 8382 | /** |
| 8148 | 8383 | * workqueue_init_topology - initialize CPU pods for unbound workqueues |
| 8149 | 8384 | * |
| @@ -8159,6 +8394,8 @@ void __init workqueue_init_topology(void) · 9 unmodified lines | ||
| 8159 | 8394 | init_pod_type(&wq_pod_types[WQ_AFFN_CPU], cpus_dont_share); |
| 8160 | 8395 | init_pod_type(&wq_pod_types[WQ_AFFN_SMT], cpus_share_smt); |
| 8161 | 8396 | init_pod_type(&wq_pod_types[WQ_AFFN_CACHE], cpus_share_cache); |
| 8397 | + precompute_cache_shard_ids(); | |
| 8398 | + init_pod_type(&wq_pod_types[WQ_AFFN_CACHE_SHARD], cpus_share_cache_shard); | |
| 8162 | 8399 | init_pod_type(&wq_pod_types[WQ_AFFN_NUMA], cpus_share_numa); |
| 8163 | 8400 | |
| 8164 | 8401 | wq_topo_initialized = true; |