* If nohz_full is enabled, set power efficient workqueue as unbound.	 * This allows workqueue items to be moved to HK CPUs.	 */	if (housekeeping_enabled(HK_TYPE_TICK))		wq_power_efficient = true; 	/* initialize WQ_AFFN_SYSTEM pods */	pt->pod_cpus = kzalloc_objs(pt->pod_cpus[0], 1);	pt->pod_node = kzalloc_objs(pt->pod_node[0], 1);	pt->cpu_pod = kzalloc_objs(pt->cpu_pod[0], nr_cpu_ids);	BUG_ON(!pt->pod_cpus || !pt->pod_node || !pt->cpu_pod); 	BUG_ON(!zalloc_cpumask_var_node(&pt->pod_cpus[0], GFP_KERNEL, NUMA_NO_NODE)); 	pt->nr_pods = 1;	cpumask_copy(pt->pod_cpus[0], cpu_possible_mask);	pt->pod_node[0] = NUMA_NO_NODE;	pt->cpu_pod[0] = 0; 	/* initialize BH and CPU pools */	for_each_possible_cpu(cpu) {		struct worker_pool *pool; 		i = 0;		for_each_bh_worker_pool(pool, cpu) {			init_cpu_worker_pool(pool, cpu, std_nice[i]);			pool->flags |= POOL_BH;			init_irq_work(bh_pool_irq_work(pool), irq_work_fns[i]);			i++;		} 		i = 0;		for_each_cpu_worker_pool(pool, cpu)			init_cpu_worker_pool(pool, cpu, std_nice[i++]);	} 	/* create default unbound and ordered wq attrs */	for (i = 0; i < NR_STD_WORKER_POOLS; i++) {		struct workqueue_attrs *attrs; 		BUG_ON(!(attrs = alloc_workqueue_attrs()));		attrs->nice = std_nice[i];		unbound_std_wq_attrs[i] = attrs; 		/*		 * An ordered wq should have only one pwq as ordering is		 * guaranteed by max_active which is enforced by pwqs.		 */		BUG_ON(!(attrs = alloc_workqueue_attrs()));		attrs->nice = std_nice[i];		attrs->ordered = true;		ordered_wq_attrs[i] = attrs;	} 	system_wq = alloc_workqueue("events", WQ_PERCPU | __WQ_DEPRECATED, 0);	system_percpu_wq = alloc_workqueue("events", WQ_PERCPU, 0);	system_highpri_wq = alloc_workqueue("events_highpri",					    WQ_HIGHPRI | WQ_PERCPU, 0);	system_long_wq = alloc_workqueue("events_long", WQ_PERCPU, 0);	system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND | __WQ_DEPRECATED, WQ_MAX_ACTIVE);	system_dfl_wq = alloc_workqueue("events_unbound", WQ_UNBOUND, WQ_MAX_ACTIVE);	system_freezable_wq = alloc_workqueue("events_freezable",					      WQ_FREEZABLE | WQ_PERCPU, 0);	system_power_efficient_wq = alloc_workqueue("events_power_efficient",					      WQ_POWER_EFFICIENT | WQ_PERCPU, 0);	system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_pwr_efficient",					      WQ_FREEZABLE | WQ_POWER_EFFICIENT | WQ_PERCPU, 0);	system_bh_wq = alloc_workqueue("events_bh", WQ_BH | WQ_PERCPU, 0);	system_bh_highpri_wq = alloc_workqueue("events_bh_highpri",					       WQ_BH | WQ_HIGHPRI | WQ_PERCPU, 0);	system_dfl_long_wq = alloc_workqueue("events_dfl_long", WQ_UNBOUND, WQ_MAX_ACTIVE);	BUG_ON(!system_wq || !system_percpu_wq|| !system_highpri_wq || !system_long_wq ||	       !system_unbound_wq || !system_freezable_wq || !system_dfl_wq ||	       !system_power_efficient_wq ||	       !system_freezable_power_efficient_wq ||	       !system_bh_wq || !system_bh_highpri_wq || !system_dfl_long_wq);} static void __init wq_cpu_intensive_thresh_init(void){	unsigned long thresh;	unsigned long bogo; 	pwq_release_worker = kthread_run_worker(0, "pool_workqueue_release");	BUG_ON(IS_ERR(pwq_release_worker)); 	/* if the user set it to a specific value, keep it */	if (wq_cpu_intensive_thresh_us != ULONG_MAX)		return; 	/*	 * The default of 10ms is derived from the fact that most modern (as of	 * 2023) processors can do a lot in 10ms and that it's just below what	 * most consider human-perceivable. However, the kernel also runs on a	 * lot slower CPUs including microcontrollers where the threshold is way	 * too low.	 *	 * Let's scale up the threshold upto 1 second if BogoMips is below 4000.	 * This is by no means accurate but it doesn't have to be. The mechanism	 * is still useful even when the threshold is fully scaled up. Also, as	 * the reports would usually be applicable to everyone, some machines	 * operating on longer thresholds won't significantly diminish their	 * usefulness.	 */	thresh = 10 * USEC_PER_MSEC; 	/* see init/calibrate.c for lpj -> BogoMIPS calculation */	bogo = max_t(unsigned long, loops_per_jiffy / 500000 * HZ, 1);	if (bogo < 4000)		thresh = min_t(unsigned long, thresh * 4000 / bogo, USEC_PER_SEC); 	pr_debug("wq_cpu_intensive_thresh: lpj=%lu BogoMIPS=%lu thresh_us=%lu\n",		 loops_per_jiffy, bogo, thresh); 	wq_cpu_intensive_thresh_us = thresh;} /** * workqueue_init - bring workqueue subsystem fully online * * This is the second step of three-staged workqueue subsystem initialization * and invoked as soon as kthreads can be created and scheduled. Workqueues have * been created and work items queued on them, but there are no kworkers * executing the work items yet. Populate the worker pools with the initial * workers and enable future kworker creations. */void __init workqueue_init(void){	struct workqueue_struct *wq;	struct worker_pool *pool;	int cpu, bkt; 	wq_cpu_intensive_thresh_init(); 	mutex_lock(&wq_pool_mutex); 	/*	 * Per-cpu pools created earlier could be missing node hint. Fix them	 * up. Also, create a rescuer for workqueues that requested it.	 */	for_each_possible_cpu(cpu) {		for_each_bh_worker_pool(pool, cpu)			pool->node = cpu_to_node(cpu);		for_each_cpu_worker_pool(pool, cpu)			pool->node = cpu_to_node(cpu);	} 	list_for_each_entry(wq, &workqueues, list) {		WARN(init_rescuer(wq),		     "workqueue: failed to create early rescuer for %s",		     wq->name);	} 	mutex_unlock(&wq_pool_mutex); 	/*	 * Create the initial workers. A BH pool has one pseudo worker that	 * represents the shared BH execution context and thus doesn't get	 * affected by hotplug events. Create the BH pseudo workers for all	 * possible CPUs here.	 */	for_each_possible_cpu(cpu)		for_each_bh_worker_pool(pool, cpu)			BUG_ON(!create_worker(pool)); 	for_each_online_cpu(cpu) {		for_each_cpu_worker_pool(pool, cpu) {			pool->flags &= ~POOL_DISASSOCIATED;			BUG_ON(!create_worker(pool));		}	} 	hash_for_each(unbound_pool_hash, bkt, pool, hash_node)		BUG_ON(!create_worker(pool)); 	wq_online = true;	wq_watchdog_init();} /* * Initialize @pt by first initializing @pt->cpu_pod[] with pod IDs according to * @cpu_shares_pod(). Each subset of CPUs that share a pod is assigned a unique * and consecutive pod ID. The rest of @pt is initialized accordingly. */static void __init init_pod_type(struct wq_pod_type *pt,				 bool (*cpus_share_pod)(int, int)){	int cur, pre, cpu, pod; 	pt->nr_pods = 0; 	/* init @pt->cpu_pod[] according to @cpus_share_pod() */	pt->cpu_pod = kzalloc_objs(pt->cpu_pod[0], nr_cpu_ids);	BUG_ON(!pt->cpu_pod); 	for_each_possible_cpu(cur) {		for_each_possible_cpu(pre) {			if (pre >= cur) {				pt->cpu_pod[cur] = pt->nr_pods++;				break;			}			if (cpus_share_pod(cur, pre)) {				pt->cpu_pod[cur] = pt->cpu_pod[pre];				break;			}		}	} 	/* init the rest to match @pt->cpu_pod[] */	pt->pod_cpus = kzalloc_objs(pt->pod_cpus[0], pt->nr_pods);	pt->pod_node = kzalloc_objs(pt->pod_node[0], pt->nr_pods);	BUG_ON(!pt->pod_cpus || !pt->pod_node); 	for (pod = 0; pod < pt->nr_pods; pod++)		BUG_ON(!zalloc_cpumask_var(&pt->pod_cpus[pod], GFP_KERNEL)); 	for_each_possible_cpu(cpu) {		cpumask_set_cpu(cpu, pt->pod_cpus[pt->cpu_pod[cpu]]);		pt->pod_node[pt->cpu_pod[cpu]] = cpu_to_node(cpu);	}} static bool __init cpus_dont_share(int cpu0, int cpu1){	return false;} static bool __init cpus_share_smt(int cpu0, int cpu1){	return cpumask_test_cpu(cpu0, cpu_smt_mask(cpu1));} static bool __init cpus_share_numa(int cpu0, int cpu1){	return cpu_to_node(cpu0) == cpu_to_node(cpu1);} /* Maps each CPU to its shard index within the LLC pod it belongs to */static int cpu_shard_id[NR_CPUS] __initdata; /** * llc_count_cores - count distinct cores (SMT groups) within an LLC pod * @pod_cpus:  the cpumask of CPUs in the LLC pod * @smt_pods:  the SMT pod type, used to identify sibling groups * * A core is represented by the lowest-numbered CPU in its SMT group. Returns * the number of distinct cores found in @pod_cpus. */static int __init llc_count_cores(const struct cpumask *pod_cpus,				  struct wq_pod_type *smt_pods){	const struct cpumask *sibling_cpus;	int nr_cores = 0, c; 	/*	 * Count distinct cores by only counting the first CPU in each	 * SMT sibling group.	 */	for_each_cpu(c, pod_cpus) {		sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]];		if (cpumask_first(sibling_cpus) == c)			nr_cores++;	} 	return nr_cores;} /* * llc_shard_size - number of cores in a given shard * * Cores are spread as evenly as possible. The first @nr_large_shards shards are * "large shards" with (cores_per_shard + 1) cores; the rest are "default * shards" with cores_per_shard cores. */static int __init llc_shard_size(int shard_id, int cores_per_shard, int nr_large_shards){	/* The first @nr_large_shards shards are large shards */	if (shard_id < nr_large_shards)		return cores_per_shard + 1; 	/* The remaining shards are default shards */	return cores_per_shard;} /* * llc_calc_shard_layout - compute the shard layout for an LLC pod * @nr_cores:  number of distinct cores in the LLC pod * * Chooses the number of shards that keeps average shard size closest to * wq_cache_shard_size. Returns a struct describing the total number of shards, * the base size of each, and how many are large shards. */static struct llc_shard_layout __init llc_calc_shard_layout(int nr_cores){	struct llc_shard_layout layout; 	/* Ensure at least one shard; pick the count closest to the target size */	layout.nr_shards = max(1, DIV_ROUND_CLOSEST(nr_cores, wq_cache_shard_size));	layout.cores_per_shard = nr_cores / layout.nr_shards;	layout.nr_large_shards = nr_cores % layout.nr_shards; 	return layout;} /* * llc_shard_is_full - check whether a shard has reached its core capacity * @cores_in_shard: number of cores already assigned to this shard * @shard_id:       index of the shard being checked * @layout:         the shard layout computed by llc_calc_shard_layout() * * Returns true if @cores_in_shard equals the expected size for @shard_id. */static bool __init llc_shard_is_full(int cores_in_shard, int shard_id,				     const struct llc_shard_layout *layout){	return cores_in_shard == llc_shard_size(shard_id, layout->cores_per_shard,						layout->nr_large_shards);} /** * llc_populate_cpu_shard_id - populate cpu_shard_id[] for each CPU in an LLC pod * @pod_cpus:  the cpumask of CPUs in the LLC pod * @smt_pods:  the SMT pod type, used to identify sibling groups * @nr_cores:  number of distinct cores in @pod_cpus (from llc_count_cores()) * * Walks @pod_cpus in order. At each SMT group leader, advances to the next * shard once the current shard is full. Results are written to cpu_shard_id[]. */static void __init llc_populate_cpu_shard_id(const struct cpumask *pod_cpus,					     struct wq_pod_type *smt_pods,					     int nr_cores){	struct llc_shard_layout layout = llc_calc_shard_layout(nr_cores);	const struct cpumask *sibling_cpus;	/* Count the number of cores in the current shard_id */	int cores_in_shard = 0;	unsigned int leader;	/* This is a cursor for the shards. Go from zero to nr_shards - 1*/	int shard_id = 0;	int c; 	/* Iterate at every CPU for a given LLC pod, and assign it a shard */	for_each_cpu(c, pod_cpus) {		sibling_cpus = smt_pods->pod_cpus[smt_pods->cpu_pod[c]];		if (cpumask_first(sibling_cpus) == c) {			/* This is the CPU leader for the siblings */			if (llc_shard_is_full(cores_in_shard, shard_id, &layout)) {				shard_id++;				cores_in_shard = 0;			}			cores_in_shard++;			cpu_shard_id[c] = shard_id;		} else {			/*			 * The siblings' shard MUST be the same as the leader.			 * never split threads in the same core.			 */			leader = cpumask_first(sibling_cpus); 			/*			 * This check silences a Warray-bounds warning on UP			 * configs where NR_CPUS=1 makes cpu_shard_id[]			 * a single-element array, and the compiler can't			 * prove the index is always 0.			 */			if (WARN_ON_ONCE(leader >= nr_cpu_ids))				continue;			cpu_shard_id[c] = cpu_shard_id[leader];		}	} 	WARN_ON_ONCE(shard_id != (layout.nr_shards - 1));} /** * precompute_cache_shard_ids - assign each CPU its shard index within its LLC * * Iterates over all LLC pods. For each pod, counts distinct cores then assigns * shard indices to all CPUs in the pod. Must be called after WQ_AFFN_CACHE and * WQ_AFFN_SMT have been initialized. */static void __init precompute_cache_shard_ids(void){	struct wq_pod_type *llc_pods = &wq_pod_types[WQ_AFFN_CACHE];	struct wq_pod_type *smt_pods = &wq_pod_types[WQ_AFFN_SMT];	const struct cpumask *cpus_sharing_llc;	int nr_cores;	int pod; 	if (!wq_cache_shard_size) {		pr_warn("workqueue: cache_shard_size must be > 0, setting to 1\n");		wq_cache_shard_size = 1;	} 	for (pod = 0; pod < llc_pods->nr_pods; pod++) {		cpus_sharing_llc = llc_pods->pod_cpus[pod]; 		/* Number of cores in this given LLC */		nr_cores = llc_count_cores(cpus_sharing_llc, smt_pods);		llc_populate_cpu_shard_id(cpus_sharing_llc, smt_pods, nr_cores);	}} /* * cpus_share_cache_shard - test whether two CPUs belong to the same cache shard * * Two CPUs share a cache shard if they are in the same LLC and have the same * shard index. Used as the pod affinity callback for WQ_AFFN_CACHE_SHARD. */static bool __init cpus_share_cache_shard(int cpu0, int cpu1){	if (!cpus_share_cache(cpu0, cpu1))		return false; 	return cpu_shard_id[cpu0] == cpu_shard_id[cpu1];} /** * workqueue_init_topology - initialize CPU pods for unbound workqueues * * This is the third step of three-staged workqueue subsystem initialization and * invoked after SMP and topology information are fully initialized. It * initializes the unbound CPU pods accordingly. */void __init workqueue_init_topology(void){	struct workqueue_struct *wq;	int cpu; 	init_pod_type(&wq_pod_types[WQ_AFFN_CPU], cpus_dont_share);	init_pod_type(&wq_pod_types[WQ_AFFN_SMT], cpus_share_smt);	init_pod_type(&wq_pod_types[WQ_AFFN_CACHE], cpus_share_cache);	precompute_cache_shard_ids();	init_pod_type(&wq_pod_types[WQ_AFFN_CACHE_SHARD], cpus_share_cache_shard);	init_pod_type(&wq_pod_types[WQ_AFFN_NUMA], cpus_share_numa); 	wq_topo_initialized = true; 	mutex_lock(&wq_pool_mutex); 	/*	 * Workqueues allocated earlier would have all CPUs sharing the default	 * worker pool. Explicitly call unbound_wq_update_pwq() on all workqueue	 * and CPU combinations to apply per-pod sharing.	 */	list_for_each_entry(wq, &workqueues, list) {		for_each_online_cpu(cpu)			unbound_wq_update_pwq(wq, cpu);		if (wq->flags & WQ_UNBOUND) {			mutex_lock(&wq->mutex);			wq_update_node_max_active(wq, -1);			mutex_unlock(&wq->mutex);		}	} 	mutex_unlock(&wq_pool_mutex);} void __warn_flushing_systemwide_wq(void){	pr_warn("WARNING: Flushing system-wide workqueues will be prohibited in near future.\n");	dump_stack();}EXPORT_SYMBOL(__warn_flushing_systemwide_wq); static int __init workqueue_unbound_cpus_setup(char *str){	if (cpulist_parse(str, &wq_cmdline_cpumask) < 0) {		cpumask_clear(&wq_cmdline_cpumask);		pr_warn("workqueue.unbound_cpus: incorrect CPU range, using default\n");	} 	return 1;}__setup("workqueue.unbound_cpus=", workqueue_unbound_cpus_setup);

Lines 8001–8475 of 8475. Previous lines