apply_wqattrs_cleanup(ctx); 	return 0;} /** * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue * @wq: the target workqueue * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs() * * Apply @attrs to an unbound workqueue @wq. Unless disabled, this function maps * a separate pwq to each CPU pod with possibles CPUs in @attrs->cpumask so that * work items are affine to the pod it was issued on. Older pwqs are released as * in-flight work items finish. Note that a work item which repeatedly requeues * itself back-to-back will stay on its current pwq. * * Performs GFP_KERNEL allocations. * * Return: 0 on success and -errno on failure. */int apply_workqueue_attrs(struct workqueue_struct *wq,			  const struct workqueue_attrs *attrs){	int ret; 	mutex_lock(&wq_pool_mutex);	ret = apply_workqueue_attrs_locked(wq, attrs);	mutex_unlock(&wq_pool_mutex); 	return ret;} /** * unbound_wq_update_pwq - update a pwq slot for CPU hot[un]plug * @wq: the target workqueue * @cpu: the CPU to update the pwq slot for * * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and * %CPU_DOWN_FAILED.  @cpu is in the same pod of the CPU being hot[un]plugged. * * * If pod affinity can't be adjusted due to memory allocation failure, it falls * back to @wq->dfl_pwq which may not be optimal but is always correct. * * Note that when the last allowed CPU of a pod goes offline for a workqueue * with a cpumask spanning multiple pods, the workers which were already * executing the work items for the workqueue will lose their CPU affinity and * may execute on any CPU. This is similar to how per-cpu workqueues behave on * CPU_DOWN. If a workqueue user wants strict affinity, it's the user's * responsibility to flush the work item from CPU_DOWN_PREPARE. */static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu){	struct pool_workqueue *old_pwq = NULL, *pwq;	struct workqueue_attrs *target_attrs; 	lockdep_assert_held(&wq_pool_mutex); 	if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered)		return; 	/*	 * We don't wanna alloc/free wq_attrs for each wq for each CPU.	 * Let's use a preallocated one.  The following buf is protected by	 * CPU hotplug exclusion.	 */	target_attrs = unbound_wq_update_pwq_attrs_buf; 	copy_workqueue_attrs(target_attrs, wq->unbound_attrs);	wqattrs_actualize_cpumask(target_attrs, wq_unbound_cpumask); 	/* nothing to do if the target cpumask matches the current pwq */	wq_calc_pod_cpumask(target_attrs, cpu);	if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))		return; 	/* create a new pwq */	pwq = alloc_unbound_pwq(wq, target_attrs);	if (!pwq) {		pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n",			wq->name);		goto use_dfl_pwq;	} 	/* Install the new pwq. */	mutex_lock(&wq->mutex);	old_pwq = install_unbound_pwq(wq, cpu, pwq);	goto out_unlock; use_dfl_pwq:	mutex_lock(&wq->mutex);	pwq = unbound_pwq(wq, -1);	raw_spin_lock_irq(&pwq->pool->lock);	get_pwq(pwq);	raw_spin_unlock_irq(&pwq->pool->lock);	old_pwq = install_unbound_pwq(wq, cpu, pwq);out_unlock:	mutex_unlock(&wq->mutex);	put_pwq_unlocked(old_pwq);} static int alloc_and_link_pwqs(struct workqueue_struct *wq){	bool highpri = wq->flags & WQ_HIGHPRI;	int cpu, ret; 	lockdep_assert_held(&wq_pool_mutex); 	wq->cpu_pwq = alloc_percpu(struct pool_workqueue *);	if (!wq->cpu_pwq)		goto enomem; 	if (!(wq->flags & WQ_UNBOUND)) {		struct worker_pool __percpu *pools; 		if (wq->flags & WQ_BH)			pools = bh_worker_pools;		else			pools = cpu_worker_pools; 		for_each_possible_cpu(cpu) {			struct pool_workqueue **pwq_p;			struct worker_pool *pool; 			pool = &(per_cpu_ptr(pools, cpu)[highpri]);			pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu); 			*pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL,						       pool->node);			if (!*pwq_p)				goto enomem; 			init_pwq(*pwq_p, wq, pool); 			mutex_lock(&wq->mutex);			link_pwq(*pwq_p);			mutex_unlock(&wq->mutex);		}		return 0;	} 	if (wq->flags & __WQ_ORDERED) {		struct pool_workqueue *dfl_pwq; 		ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]);		/* there should only be single pwq for ordering guarantee */		dfl_pwq = rcu_access_pointer(wq->dfl_pwq);		WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||			      wq->pwqs.prev != &dfl_pwq->pwqs_node),		     "ordering guarantee broken for workqueue %s\n", wq->name);	} else {		ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]);	} 	if (ret)		goto enomem;	return 0; enomem:	if (wq->cpu_pwq) {		for_each_possible_cpu(cpu) {			struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu); 			if (pwq) {				/*				 * Unlink pwq from wq->pwqs since link_pwq()				 * may have already added it. wq->mutex is not				 * needed as the wq has not been published yet.				 */				if (!list_empty(&pwq->pwqs_node))					list_del_rcu(&pwq->pwqs_node);				kmem_cache_free(pwq_cache, pwq);			}		}		free_percpu(wq->cpu_pwq);		wq->cpu_pwq = NULL;	}	return -ENOMEM;} static int wq_clamp_max_active(int max_active, unsigned int flags,			       const char *name){	if (max_active < 1 || max_active > WQ_MAX_ACTIVE)		pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",			max_active, name, 1, WQ_MAX_ACTIVE); 	return clamp_val(max_active, 1, WQ_MAX_ACTIVE);} /* * Workqueues which may be used during memory reclaim should have a rescuer * to guarantee forward progress. */static int init_rescuer(struct workqueue_struct *wq){	struct worker *rescuer;	char id_buf[WORKER_ID_LEN];	int ret; 	lockdep_assert_held(&wq_pool_mutex); 	if (!(wq->flags & WQ_MEM_RECLAIM))		return 0; 	rescuer = alloc_worker(NUMA_NO_NODE);	if (!rescuer) {		pr_err("workqueue: Failed to allocate a rescuer for wq \"%s\"\n",		       wq->name);		return -ENOMEM;	} 	rescuer->rescue_wq = wq;	format_worker_id(id_buf, sizeof(id_buf), rescuer, NULL); 	rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", id_buf);	if (IS_ERR(rescuer->task)) {		ret = PTR_ERR(rescuer->task);		pr_err("workqueue: Failed to create a rescuer kthread for wq \"%s\": %pe",		       wq->name, ERR_PTR(ret));		kfree(rescuer);		return ret;	} 	wq->rescuer = rescuer; 	/* initial cpumask is consistent with the detached rescuer and unbind_worker() */	if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask))		kthread_bind_mask(rescuer->task, wq_unbound_cpumask);	else		kthread_bind_mask(rescuer->task, cpu_possible_mask); 	wake_up_process(rescuer->task); 	return 0;} /** * wq_adjust_max_active - update a wq's max_active to the current setting * @wq: target workqueue * * If @wq isn't freezing, set @wq->max_active to the saved_max_active and * activate inactive work items accordingly. If @wq is freezing, clear * @wq->max_active to zero. */static void wq_adjust_max_active(struct workqueue_struct *wq){	bool activated;	int new_max, new_min; 	lockdep_assert_held(&wq->mutex); 	if ((wq->flags & WQ_FREEZABLE) && workqueue_freezing) {		new_max = 0;		new_min = 0;	} else {		new_max = wq->saved_max_active;		new_min = wq->saved_min_active;	} 	if (wq->max_active == new_max && wq->min_active == new_min)		return; 	/*	 * Update @wq->max/min_active and then kick inactive work items if more	 * active work items are allowed. This doesn't break work item ordering	 * because new work items are always queued behind existing inactive	 * work items if there are any.	 */	WRITE_ONCE(wq->max_active, new_max);	WRITE_ONCE(wq->min_active, new_min); 	if (wq->flags & WQ_UNBOUND)		wq_update_node_max_active(wq, -1); 	if (new_max == 0)		return; 	/*	 * Round-robin through pwq's activating the first inactive work item	 * until max_active is filled.	 */	do {		struct pool_workqueue *pwq; 		activated = false;		for_each_pwq(pwq, wq) {			unsigned long irq_flags; 			/* can be called during early boot w/ irq disabled */			raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);			if (pwq_activate_first_inactive(pwq, true)) {				activated = true;				kick_pool(pwq->pool);			}			raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);		}	} while (activated);} __printf(1, 0)static struct workqueue_struct *__alloc_workqueue(const char *fmt,						  unsigned int flags,						  int max_active, va_list args){	struct workqueue_struct *wq;	size_t wq_size;	int name_len; 	if (flags & WQ_BH) {		if (WARN_ON_ONCE(flags & ~__WQ_BH_ALLOWS))			return NULL;		if (WARN_ON_ONCE(max_active))			return NULL;	} 	/* see the comment above the definition of WQ_POWER_EFFICIENT */	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)		flags = (flags & ~WQ_PERCPU) | WQ_UNBOUND; 	/* allocate wq and format name */	if (flags & WQ_UNBOUND)		wq_size = struct_size(wq, node_nr_active, nr_node_ids + 1);	else		wq_size = sizeof(*wq); 	wq = kzalloc_noprof(wq_size, GFP_KERNEL);	if (!wq)		return NULL; 	if (flags & WQ_UNBOUND) {		wq->unbound_attrs = alloc_workqueue_attrs_noprof();		if (!wq->unbound_attrs)			goto err_free_wq;	} 	name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args); 	if (name_len >= WQ_NAME_LEN)		pr_warn_once("workqueue: name exceeds WQ_NAME_LEN. Truncating to: %s\n",			     wq->name); 	/*	 * One among WQ_PERCPU and WQ_UNBOUND must be set, but not both.	 * - If neither is set, default to WQ_PERCPU	 * - If both are set, default to WQ_UNBOUND	 *	 * This code can be removed after workqueue are unbound by default	 */	if (unlikely(!(flags & (WQ_UNBOUND | WQ_PERCPU)))) {		WARN_ONCE(1, "workqueue: %s is using neither WQ_PERCPU or WQ_UNBOUND. "			  "Setting WQ_PERCPU.\n", wq->name);		flags |= WQ_PERCPU;	} else if (unlikely((flags & WQ_PERCPU) && (flags & WQ_UNBOUND))) {		WARN_ONCE(1, "workqueue: %s uses both WQ_PERCPU and WQ_UNBOUND. "			  "Dropped WQ_PERCPU, keeping WQ_UNBOUND.\n", wq->name);		flags &= ~WQ_PERCPU;	} 	if (flags & WQ_BH) {		/*		 * BH workqueues always share a single execution context per CPU		 * and don't impose any max_active limit.		 */		max_active = INT_MAX;	} else {		max_active = max_active ?: WQ_DFL_ACTIVE;		max_active = wq_clamp_max_active(max_active, flags, wq->name);	} 	/* init wq */	wq->flags = flags;	wq->max_active = max_active;	wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE);	wq->saved_max_active = wq->max_active;	wq->saved_min_active = wq->min_active;	mutex_init(&wq->mutex);	atomic_set(&wq->nr_pwqs_to_flush, 0);	INIT_LIST_HEAD(&wq->pwqs);	INIT_LIST_HEAD(&wq->flusher_queue);	INIT_LIST_HEAD(&wq->flusher_overflow);	INIT_LIST_HEAD(&wq->maydays); 	INIT_LIST_HEAD(&wq->list); 	if (flags & WQ_UNBOUND) {		if (alloc_node_nr_active(wq->node_nr_active) < 0)			goto err_free_wq;	} 	/*	 * wq_pool_mutex protects the workqueues list, allocations of PWQs,	 * and the global freeze state.	 */	mutex_lock(&wq_pool_mutex); 	if (alloc_and_link_pwqs(wq) < 0)		goto err_unlock_free_node_nr_active; 	mutex_lock(&wq->mutex);	wq_adjust_max_active(wq);	mutex_unlock(&wq->mutex); 	list_add_tail_rcu(&wq->list, &workqueues); 	if (wq_online && init_rescuer(wq) < 0)		goto err_unlock_destroy; 	mutex_unlock(&wq_pool_mutex); 	if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))		goto err_destroy; 	return wq; err_unlock_free_node_nr_active:	mutex_unlock(&wq_pool_mutex);	/*	 * Failed alloc_and_link_pwqs() may leave pending pwq->release_work,	 * flushing the pwq_release_worker ensures that the pwq_release_workfn()	 * completes before calling kfree(wq).	 */	if (wq->flags & WQ_UNBOUND) {		kthread_flush_worker(pwq_release_worker);		free_node_nr_active(wq->node_nr_active);	}err_free_wq:	free_workqueue_attrs(wq->unbound_attrs);	kfree(wq);	return NULL;err_unlock_destroy:	mutex_unlock(&wq_pool_mutex);err_destroy:	destroy_workqueue(wq);	return NULL;} __printf(1, 0)static struct workqueue_struct *alloc_workqueue_va(const char *fmt,						   unsigned int flags,						   int max_active,						   va_list args){	struct workqueue_struct *wq; 	wq = __alloc_workqueue(fmt, flags, max_active, args);	if (wq)		wq_init_lockdep(wq); 	return wq;} __printf(1, 4)struct workqueue_struct *alloc_workqueue_noprof(const char *fmt,						unsigned int flags,						int max_active, ...){	struct workqueue_struct *wq;	va_list args; 	va_start(args, max_active);	wq = alloc_workqueue_va(fmt, flags, max_active, args);	va_end(args); 	return wq;}EXPORT_SYMBOL_GPL(alloc_workqueue_noprof); static void devm_workqueue_release(void *res){	destroy_workqueue(res);} __printf(2, 5) struct workqueue_struct *devm_alloc_workqueue_noprof(struct device *dev, const char *fmt,			    unsigned int flags, int max_active, ...){	struct workqueue_struct *wq;	va_list args;	int ret; 	va_start(args, max_active);	wq = alloc_workqueue_va(fmt, flags, max_active, args);	va_end(args);	if (!wq)		return NULL; 	ret = devm_add_action_or_reset(dev, devm_workqueue_release, wq);	if (ret)		return NULL; 	return wq;}EXPORT_SYMBOL_GPL(devm_alloc_workqueue_noprof); #ifdef CONFIG_LOCKDEP__printf(1, 5)struct workqueue_struct *alloc_workqueue_lockdep_map(const char *fmt, unsigned int flags,			    int max_active, struct lockdep_map *lockdep_map, ...)

Lines 5501–6000 of 8475. Previous lines Load more