struct wq_node_nr_active *nna;	int node; 	for_each_node(node) {		nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, node);		if (!nna)			goto err_free;		init_node_nr_active(nna);		nna_ar[node] = nna;	} 	/* [nr_node_ids] is used as the fallback */	nna = kzalloc_node(sizeof(*nna), GFP_KERNEL, NUMA_NO_NODE);	if (!nna)		goto err_free;	init_node_nr_active(nna);	nna_ar[nr_node_ids] = nna; 	return 0; err_free:	free_node_nr_active(nna_ar);	return -ENOMEM;} static void rcu_free_wq(struct rcu_head *rcu){	struct workqueue_struct *wq =		container_of(rcu, struct workqueue_struct, rcu); 	if (wq->flags & WQ_UNBOUND)		free_node_nr_active(wq->node_nr_active); 	wq_free_lockdep(wq);	free_percpu(wq->cpu_pwq);	free_workqueue_attrs(wq->unbound_attrs);	kfree(wq);} static void rcu_free_pool(struct rcu_head *rcu){	struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu); 	ida_destroy(&pool->worker_ida);	free_workqueue_attrs(pool->attrs);	kfree(pool);} /** * put_unbound_pool - put a worker_pool * @pool: worker_pool to put * * Put @pool.  If its refcnt reaches zero, it gets destroyed in RCU * safe manner.  get_unbound_pool() calls this function on its failure path * and this function should be able to release pools which went through, * successfully or not, init_worker_pool(). * * Should be called with wq_pool_mutex held. */static void put_unbound_pool(struct worker_pool *pool){	struct worker *worker;	LIST_HEAD(cull_list); 	lockdep_assert_held(&wq_pool_mutex); 	if (--pool->refcnt)		return; 	/* sanity checks */	if (WARN_ON(!(pool->cpu < 0)) ||	    WARN_ON(!list_empty(&pool->worklist)))		return; 	/* release id and unhash */	if (pool->id >= 0)		idr_remove(&worker_pool_idr, pool->id);	hash_del(&pool->hash_node); 	/*	 * Become the manager and destroy all workers.  This prevents	 * @pool's workers from blocking on attach_mutex.  We're the last	 * manager and @pool gets freed with the flag set.	 *	 * Having a concurrent manager is quite unlikely to happen as we can	 * only get here with	 *   pwq->refcnt == pool->refcnt == 0	 * which implies no work queued to the pool, which implies no worker can	 * become the manager. However a worker could have taken the role of	 * manager before the refcnts dropped to 0, since maybe_create_worker()	 * drops pool->lock	 */	while (true) {		rcuwait_wait_event(&manager_wait,				   !(pool->flags & POOL_MANAGER_ACTIVE),				   TASK_UNINTERRUPTIBLE); 		mutex_lock(&wq_pool_attach_mutex);		raw_spin_lock_irq(&pool->lock);		if (!(pool->flags & POOL_MANAGER_ACTIVE)) {			pool->flags |= POOL_MANAGER_ACTIVE;			break;		}		raw_spin_unlock_irq(&pool->lock);		mutex_unlock(&wq_pool_attach_mutex);	} 	while ((worker = first_idle_worker(pool)))		set_worker_dying(worker, &cull_list);	WARN_ON(pool->nr_workers || pool->nr_idle);	raw_spin_unlock_irq(&pool->lock); 	detach_dying_workers(&cull_list); 	mutex_unlock(&wq_pool_attach_mutex); 	reap_dying_workers(&cull_list); 	/* shut down the timers */	timer_delete_sync(&pool->idle_timer);	cancel_work_sync(&pool->idle_cull_work);	timer_delete_sync(&pool->mayday_timer); 	/* RCU protected to allow dereferences from get_work_pool() */	call_rcu(&pool->rcu, rcu_free_pool);} /** * get_unbound_pool - get a worker_pool with the specified attributes * @attrs: the attributes of the worker_pool to get * * Obtain a worker_pool which has the same attributes as @attrs, bump the * reference count and return it.  If there already is a matching * worker_pool, it will be used; otherwise, this function attempts to * create a new one. * * Should be called with wq_pool_mutex held. * * Return: On success, a worker_pool with the same attributes as @attrs. * On failure, %NULL. */static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs){	struct wq_pod_type *pt = &wq_pod_types[WQ_AFFN_NUMA];	u32 hash = wqattrs_hash(attrs);	struct worker_pool *pool;	int pod, node = NUMA_NO_NODE; 	lockdep_assert_held(&wq_pool_mutex); 	/* do we already have a matching pool? */	hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {		if (wqattrs_equal(pool->attrs, attrs)) {			pool->refcnt++;			return pool;		}	} 	/* If __pod_cpumask is contained inside a NUMA pod, that's our node */	for (pod = 0; pod < pt->nr_pods; pod++) {		if (cpumask_subset(attrs->__pod_cpumask, pt->pod_cpus[pod])) {			node = pt->pod_node[pod];			break;		}	} 	/* nope, create a new one */	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, node);	if (!pool || init_worker_pool(pool) < 0)		goto fail; 	pool->node = node;	copy_workqueue_attrs(pool->attrs, attrs);	wqattrs_clear_for_pool(pool->attrs); 	if (worker_pool_assign_id(pool) < 0)		goto fail; 	/* create and start the initial worker */	if (wq_online && !create_worker(pool))		goto fail; 	/* install */	hash_add(unbound_pool_hash, &pool->hash_node, hash); 	return pool;fail:	if (pool)		put_unbound_pool(pool);	return NULL;} /* * Scheduled on pwq_release_worker by put_pwq() when an unbound pwq hits zero * refcnt and needs to be destroyed. */static void pwq_release_workfn(struct kthread_work *work){	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,						  release_work);	struct workqueue_struct *wq = pwq->wq;	struct worker_pool *pool = pwq->pool;	bool is_last = false; 	/*	 * When @pwq is not linked, it doesn't hold any reference to the	 * @wq, and @wq is invalid to access.	 */	if (!list_empty(&pwq->pwqs_node)) {		mutex_lock(&wq->mutex);		list_del_rcu(&pwq->pwqs_node);		is_last = list_empty(&wq->pwqs); 		/*		 * For ordered workqueue with a plugged dfl_pwq, restart it now.		 */		if (!is_last && (wq->flags & __WQ_ORDERED))			unplug_oldest_pwq(wq); 		mutex_unlock(&wq->mutex);	} 	if (wq->flags & WQ_UNBOUND) {		mutex_lock(&wq_pool_mutex);		put_unbound_pool(pool);		mutex_unlock(&wq_pool_mutex);	} 	if (!list_empty(&pwq->pending_node)) {		struct wq_node_nr_active *nna =			wq_node_nr_active(pwq->wq, pwq->pool->node); 		raw_spin_lock_irq(&nna->lock);		list_del_init(&pwq->pending_node);		raw_spin_unlock_irq(&nna->lock);	} 	kfree_rcu(pwq, rcu); 	/*	 * If we're the last pwq going away, @wq is already dead and no one	 * is gonna access it anymore.  Schedule RCU free.	 */	if (is_last) {		wq_unregister_lockdep(wq);		call_rcu(&wq->rcu, rcu_free_wq);	}} /* initialize newly allocated @pwq which is associated with @wq and @pool */static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,		     struct worker_pool *pool){	BUG_ON((unsigned long)pwq & ~WORK_STRUCT_PWQ_MASK); 	memset(pwq, 0, sizeof(*pwq)); 	pwq->pool = pool;	pwq->wq = wq;	pwq->flush_color = -1;	pwq->refcnt = 1;	INIT_LIST_HEAD(&pwq->inactive_works);	INIT_LIST_HEAD(&pwq->pending_node);	INIT_LIST_HEAD(&pwq->pwqs_node);	INIT_LIST_HEAD(&pwq->mayday_node);	kthread_init_work(&pwq->release_work, pwq_release_workfn); 	/*	 * Set the dummy cursor work with valid function and get_work_pwq().	 *	 * The cursor work should only be in the pwq->pool->worklist, and	 * should not be treated as a processable work item.	 *	 * WORK_STRUCT_PENDING and WORK_STRUCT_INACTIVE just make it less	 * surprise for kernel debugging tools and reviewers.	 */	INIT_WORK(&pwq->mayday_cursor, mayday_cursor_func);	atomic_long_set(&pwq->mayday_cursor.data, (unsigned long)pwq |			WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | WORK_STRUCT_INACTIVE);} /* sync @pwq with the current state of its associated wq and link it */static void link_pwq(struct pool_workqueue *pwq){	struct workqueue_struct *wq = pwq->wq; 	lockdep_assert_held(&wq->mutex); 	/* may be called multiple times, ignore if already linked */	if (!list_empty(&pwq->pwqs_node))		return; 	/* set the matching work_color */	pwq->work_color = wq->work_color; 	/* link in @pwq */	list_add_tail_rcu(&pwq->pwqs_node, &wq->pwqs);} /* obtain a pool matching @attr and create a pwq associating the pool and @wq */static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,					const struct workqueue_attrs *attrs){	struct worker_pool *pool;	struct pool_workqueue *pwq; 	lockdep_assert_held(&wq_pool_mutex); 	pool = get_unbound_pool(attrs);	if (!pool)		return NULL; 	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);	if (!pwq) {		put_unbound_pool(pool);		return NULL;	} 	init_pwq(pwq, wq, pool);	return pwq;} /** * wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod * @attrs: the wq_attrs of the default pwq of the target workqueue * @cpu: the target CPU * * Calculate the cpumask a workqueue with @attrs should use on @pod. * The result is stored in @attrs->__pod_cpumask. * * If pod affinity is not enabled, @attrs->cpumask is always used. If enabled * and @pod has online CPUs requested by @attrs, the returned cpumask is the * intersection of the possible CPUs of @pod and @attrs->cpumask. * * The caller is responsible for ensuring that the cpumask of @pod stays stable. */static void wq_calc_pod_cpumask(struct workqueue_attrs *attrs, int cpu){	const struct wq_pod_type *pt = wqattrs_pod_type(attrs);	int pod = pt->cpu_pod[cpu]; 	/* calculate possible CPUs in @pod that @attrs wants */	cpumask_and(attrs->__pod_cpumask, pt->pod_cpus[pod], attrs->cpumask);	/* does @pod have any online CPUs @attrs wants? */	if (!cpumask_intersects(attrs->__pod_cpumask, wq_online_cpumask)) {		cpumask_copy(attrs->__pod_cpumask, attrs->cpumask);		return;	}} /* install @pwq into @wq and return the old pwq, @cpu < 0 for dfl_pwq */static struct pool_workqueue *install_unbound_pwq(struct workqueue_struct *wq,					int cpu, struct pool_workqueue *pwq){	struct pool_workqueue __rcu **slot = unbound_pwq_slot(wq, cpu);	struct pool_workqueue *old_pwq; 	lockdep_assert_held(&wq_pool_mutex);	lockdep_assert_held(&wq->mutex); 	/* link_pwq() can handle duplicate calls */	link_pwq(pwq); 	old_pwq = rcu_access_pointer(*slot);	rcu_assign_pointer(*slot, pwq);	return old_pwq;} /* context to store the prepared attrs & pwqs before applying */struct apply_wqattrs_ctx {	struct workqueue_struct	*wq;		/* target workqueue */	struct workqueue_attrs	*attrs;		/* attrs to apply */	struct list_head	list;		/* queued for batching commit */	struct pool_workqueue	*dfl_pwq;	struct pool_workqueue	*pwq_tbl[];}; /* free the resources after success or abort */static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx){	if (ctx) {		int cpu; 		for_each_possible_cpu(cpu)			put_pwq_unlocked(ctx->pwq_tbl[cpu]);		put_pwq_unlocked(ctx->dfl_pwq); 		free_workqueue_attrs(ctx->attrs); 		kfree(ctx);	}} /* allocate the attrs and pwqs for later installation */static struct apply_wqattrs_ctx *apply_wqattrs_prepare(struct workqueue_struct *wq,		      const struct workqueue_attrs *attrs,		      const cpumask_var_t unbound_cpumask){	struct apply_wqattrs_ctx *ctx;	struct workqueue_attrs *new_attrs;	int cpu; 	lockdep_assert_held(&wq_pool_mutex); 	if (WARN_ON(attrs->affn_scope < 0 ||		    attrs->affn_scope >= WQ_AFFN_NR_TYPES))		return ERR_PTR(-EINVAL); 	ctx = kzalloc_flex(*ctx, pwq_tbl, nr_cpu_ids); 	new_attrs = alloc_workqueue_attrs();	if (!ctx || !new_attrs)		goto out_free; 	/*	 * If something goes wrong during CPU up/down, we'll fall back to	 * the default pwq covering whole @attrs->cpumask.  Always create	 * it even if we don't use it immediately.	 */	copy_workqueue_attrs(new_attrs, attrs);	wqattrs_actualize_cpumask(new_attrs, unbound_cpumask);	cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);	ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);	if (!ctx->dfl_pwq)		goto out_free; 	for_each_possible_cpu(cpu) {		if (new_attrs->ordered) {			ctx->dfl_pwq->refcnt++;			ctx->pwq_tbl[cpu] = ctx->dfl_pwq;		} else {			wq_calc_pod_cpumask(new_attrs, cpu);			ctx->pwq_tbl[cpu] = alloc_unbound_pwq(wq, new_attrs);			if (!ctx->pwq_tbl[cpu])				goto out_free;		}	} 	/* save the user configured attrs and sanitize it. */	copy_workqueue_attrs(new_attrs, attrs);	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);	cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);	ctx->attrs = new_attrs; 	/*	 * For initialized ordered workqueues, there should only be one pwq	 * (dfl_pwq). Set the plugged flag of ctx->dfl_pwq to suspend execution	 * of newly queued work items until execution of older work items in	 * the old pwq's have completed.	 */	if ((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs))		ctx->dfl_pwq->plugged = true; 	ctx->wq = wq;	return ctx; out_free:	free_workqueue_attrs(new_attrs);	apply_wqattrs_cleanup(ctx);	return ERR_PTR(-ENOMEM);} /* set attrs and install prepared pwqs, @ctx points to old pwqs on return */static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx){	int cpu; 	/* all pwqs have been created successfully, let's install'em */	mutex_lock(&ctx->wq->mutex); 	copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs); 	/* save the previous pwqs and install the new ones */	for_each_possible_cpu(cpu)		ctx->pwq_tbl[cpu] = install_unbound_pwq(ctx->wq, cpu,							ctx->pwq_tbl[cpu]);	ctx->dfl_pwq = install_unbound_pwq(ctx->wq, -1, ctx->dfl_pwq); 	/* update node_nr_active->max */	wq_update_node_max_active(ctx->wq, -1); 	mutex_unlock(&ctx->wq->mutex);} static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,					const struct workqueue_attrs *attrs){	struct apply_wqattrs_ctx *ctx; 	/* only unbound workqueues can change attributes */	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))		return -EINVAL; 	ctx = apply_wqattrs_prepare(wq, attrs, wq_unbound_cpumask);	if (IS_ERR(ctx))		return PTR_ERR(ctx); 	/* the ctx has been prepared successfully, let's commit it */	apply_wqattrs_commit(ctx);

Lines 5001–5500 of 8475. Previous lines Load more