5816 lines · 159.4 KB · C
RawDownload
 	mutex_lock(&wq_pool_mutex);	put_unbound_pool(pool);	mutex_unlock(&wq_pool_mutex); 	call_rcu_sched(&pwq->rcu, rcu_free_pwq); 	/*	 * If we're the last pwq going away, @wq is already dead and no one	 * is gonna access it anymore.  Schedule RCU free.	 */	if (is_last)		call_rcu_sched(&wq->rcu, rcu_free_wq);} /** * pwq_adjust_max_active - update a pwq's max_active to the current setting * @pwq: target pool_workqueue * * If @pwq isn't freezing, set @pwq->max_active to the associated * workqueue's saved_max_active and activate delayed work items * accordingly.  If @pwq is freezing, clear @pwq->max_active to zero. */static void pwq_adjust_max_active(struct pool_workqueue *pwq){	struct workqueue_struct *wq = pwq->wq;	bool freezable = wq->flags & WQ_FREEZABLE;	unsigned long flags; 	/* for @wq->saved_max_active */	lockdep_assert_held(&wq->mutex); 	/* fast exit for non-freezable wqs */	if (!freezable && pwq->max_active == wq->saved_max_active)		return; 	/* this function can be called during early boot w/ irq disabled */	spin_lock_irqsave(&pwq->pool->lock, flags); 	/*	 * During [un]freezing, the caller is responsible for ensuring that	 * this function is called at least once after @workqueue_freezing	 * is updated and visible.	 */	if (!freezable || !workqueue_freezing) {		pwq->max_active = wq->saved_max_active; 		while (!list_empty(&pwq->delayed_works) &&		       pwq->nr_active < pwq->max_active)			pwq_activate_first_delayed(pwq); 		/*		 * Need to kick a worker after thawed or an unbound wq's		 * max_active is bumped.  It's a slow path.  Do it always.		 */		wake_up_worker(pwq->pool);	} else {		pwq->max_active = 0;	} 	spin_unlock_irqrestore(&pwq->pool->lock, flags);} /* initialize newly alloced @pwq which is associated with @wq and @pool */static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,		     struct worker_pool *pool){	BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK); 	memset(pwq, 0, sizeof(*pwq)); 	pwq->pool = pool;	pwq->wq = wq;	pwq->flush_color = -1;	pwq->refcnt = 1;	INIT_LIST_HEAD(&pwq->delayed_works);	INIT_LIST_HEAD(&pwq->pwqs_node);	INIT_LIST_HEAD(&pwq->mayday_node);	INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);} /* sync @pwq with the current state of its associated wq and link it */static void link_pwq(struct pool_workqueue *pwq){	struct workqueue_struct *wq = pwq->wq; 	lockdep_assert_held(&wq->mutex); 	/* may be called multiple times, ignore if already linked */	if (!list_empty(&pwq->pwqs_node))		return; 	/* set the matching work_color */	pwq->work_color = wq->work_color; 	/* sync max_active to the current setting */	pwq_adjust_max_active(pwq); 	/* link in @pwq */	list_add_rcu(&pwq->pwqs_node, &wq->pwqs);} /* obtain a pool matching @attr and create a pwq associating the pool and @wq */static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,					const struct workqueue_attrs *attrs){	struct worker_pool *pool;	struct pool_workqueue *pwq; 	lockdep_assert_held(&wq_pool_mutex); 	pool = get_unbound_pool(attrs);	if (!pool)		return NULL; 	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);	if (!pwq) {		put_unbound_pool(pool);		return NULL;	} 	init_pwq(pwq, wq, pool);	return pwq;} /** * wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node * @attrs: the wq_attrs of the default pwq of the target workqueue * @node: the target NUMA node * @cpu_going_down: if >= 0, the CPU to consider as offline * @cpumask: outarg, the resulting cpumask * * Calculate the cpumask a workqueue with @attrs should use on @node.  If * @cpu_going_down is >= 0, that cpu is considered offline during * calculation.  The result is stored in @cpumask. * * If NUMA affinity is not enabled, @attrs->cpumask is always used.  If * enabled and @node has online CPUs requested by @attrs, the returned * cpumask is the intersection of the possible CPUs of @node and * @attrs->cpumask. * * The caller is responsible for ensuring that the cpumask of @node stays * stable. * * Return: %true if the resulting @cpumask is different from @attrs->cpumask, * %false if equal. */static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,				 int cpu_going_down, cpumask_t *cpumask){	if (!wq_numa_enabled || attrs->no_numa)		goto use_dfl; 	/* does @node have any online CPUs @attrs wants? */	cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);	if (cpu_going_down >= 0)		cpumask_clear_cpu(cpu_going_down, cpumask); 	if (cpumask_empty(cpumask))		goto use_dfl; 	/* yeap, return possible CPUs in @node that @attrs wants */	cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]); 	if (cpumask_empty(cpumask)) {		pr_warn_once("WARNING: workqueue cpumask: online intersect > "				"possible intersect\n");		return false;	} 	return !cpumask_equal(cpumask, attrs->cpumask); use_dfl:	cpumask_copy(cpumask, attrs->cpumask);	return false;} /* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,						   int node,						   struct pool_workqueue *pwq){	struct pool_workqueue *old_pwq; 	lockdep_assert_held(&wq_pool_mutex);	lockdep_assert_held(&wq->mutex); 	/* link_pwq() can handle duplicate calls */	link_pwq(pwq); 	old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);	rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);	return old_pwq;} /* context to store the prepared attrs & pwqs before applying */struct apply_wqattrs_ctx {	struct workqueue_struct	*wq;		/* target workqueue */	struct workqueue_attrs	*attrs;		/* attrs to apply */	struct list_head	list;		/* queued for batching commit */	struct pool_workqueue	*dfl_pwq;	struct pool_workqueue	*pwq_tbl[];}; /* free the resources after success or abort */static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx){	if (ctx) {		int node; 		for_each_node(node)			put_pwq_unlocked(ctx->pwq_tbl[node]);		put_pwq_unlocked(ctx->dfl_pwq); 		free_workqueue_attrs(ctx->attrs); 		kfree(ctx);	}} /* allocate the attrs and pwqs for later installation */static struct apply_wqattrs_ctx *apply_wqattrs_prepare(struct workqueue_struct *wq,		      const struct workqueue_attrs *attrs){	struct apply_wqattrs_ctx *ctx;	struct workqueue_attrs *new_attrs, *tmp_attrs;	int node; 	lockdep_assert_held(&wq_pool_mutex); 	ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_node_ids), GFP_KERNEL); 	new_attrs = alloc_workqueue_attrs(GFP_KERNEL);	tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);	if (!ctx || !new_attrs || !tmp_attrs)		goto out_free; 	/*	 * Calculate the attrs of the default pwq.	 * If the user configured cpumask doesn't overlap with the	 * wq_unbound_cpumask, we fallback to the wq_unbound_cpumask.	 */	copy_workqueue_attrs(new_attrs, attrs);	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, wq_unbound_cpumask);	if (unlikely(cpumask_empty(new_attrs->cpumask)))		cpumask_copy(new_attrs->cpumask, wq_unbound_cpumask); 	/*	 * We may create multiple pwqs with differing cpumasks.  Make a	 * copy of @new_attrs which will be modified and used to obtain	 * pools.	 */	copy_workqueue_attrs(tmp_attrs, new_attrs); 	/*	 * If something goes wrong during CPU up/down, we'll fall back to	 * the default pwq covering whole @attrs->cpumask.  Always create	 * it even if we don't use it immediately.	 */	ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);	if (!ctx->dfl_pwq)		goto out_free; 	for_each_node(node) {		if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) {			ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);			if (!ctx->pwq_tbl[node])				goto out_free;		} else {			ctx->dfl_pwq->refcnt++;			ctx->pwq_tbl[node] = ctx->dfl_pwq;		}	} 	/* save the user configured attrs and sanitize it. */	copy_workqueue_attrs(new_attrs, attrs);	cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);	ctx->attrs = new_attrs; 	ctx->wq = wq;	free_workqueue_attrs(tmp_attrs);	return ctx; out_free:	free_workqueue_attrs(tmp_attrs);	free_workqueue_attrs(new_attrs);	apply_wqattrs_cleanup(ctx);	return NULL;} /* set attrs and install prepared pwqs, @ctx points to old pwqs on return */static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx){	int node; 	/* all pwqs have been created successfully, let's install'em */	mutex_lock(&ctx->wq->mutex); 	copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs); 	/* save the previous pwq and install the new one */	for_each_node(node)		ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node,							  ctx->pwq_tbl[node]); 	/* @dfl_pwq might not have been used, ensure it's linked */	link_pwq(ctx->dfl_pwq);	swap(ctx->wq->dfl_pwq, ctx->dfl_pwq); 	mutex_unlock(&ctx->wq->mutex);} static void apply_wqattrs_lock(void){	/* CPUs should stay stable across pwq creations and installations */	get_online_cpus();	mutex_lock(&wq_pool_mutex);} static void apply_wqattrs_unlock(void){	mutex_unlock(&wq_pool_mutex);	put_online_cpus();} static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,					const struct workqueue_attrs *attrs){	struct apply_wqattrs_ctx *ctx; 	/* only unbound workqueues can change attributes */	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))		return -EINVAL; 	/* creating multiple pwqs breaks ordering guarantee */	if (!list_empty(&wq->pwqs)) {		if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))			return -EINVAL; 		wq->flags &= ~__WQ_ORDERED;	} 	ctx = apply_wqattrs_prepare(wq, attrs);	if (!ctx)		return -ENOMEM; 	/* the ctx has been prepared successfully, let's commit it */	apply_wqattrs_commit(ctx);	apply_wqattrs_cleanup(ctx); 	return 0;} /** * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue * @wq: the target workqueue * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs() * * Apply @attrs to an unbound workqueue @wq.  Unless disabled, on NUMA * machines, this function maps a separate pwq to each NUMA node with * possibles CPUs in @attrs->cpumask so that work items are affine to the * NUMA node it was issued on.  Older pwqs are released as in-flight work * items finish.  Note that a work item which repeatedly requeues itself * back-to-back will stay on its current pwq. * * Performs GFP_KERNEL allocations. * * Return: 0 on success and -errno on failure. */int apply_workqueue_attrs(struct workqueue_struct *wq,			  const struct workqueue_attrs *attrs){	int ret; 	apply_wqattrs_lock();	ret = apply_workqueue_attrs_locked(wq, attrs);	apply_wqattrs_unlock(); 	return ret;}EXPORT_SYMBOL_GPL(apply_workqueue_attrs); /** * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug * @wq: the target workqueue * @cpu: the CPU coming up or going down * @online: whether @cpu is coming up or going down * * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and * %CPU_DOWN_FAILED.  @cpu is being hot[un]plugged, update NUMA affinity of * @wq accordingly. * * If NUMA affinity can't be adjusted due to memory allocation failure, it * falls back to @wq->dfl_pwq which may not be optimal but is always * correct. * * Note that when the last allowed CPU of a NUMA node goes offline for a * workqueue with a cpumask spanning multiple nodes, the workers which were * already executing the work items for the workqueue will lose their CPU * affinity and may execute on any CPU.  This is similar to how per-cpu * workqueues behave on CPU_DOWN.  If a workqueue user wants strict * affinity, it's the user's responsibility to flush the work item from * CPU_DOWN_PREPARE. */static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,				   bool online){	int node = cpu_to_node(cpu);	int cpu_off = online ? -1 : cpu;	struct pool_workqueue *old_pwq = NULL, *pwq;	struct workqueue_attrs *target_attrs;	cpumask_t *cpumask; 	lockdep_assert_held(&wq_pool_mutex); 	if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND) ||	    wq->unbound_attrs->no_numa)		return; 	/*	 * We don't wanna alloc/free wq_attrs for each wq for each CPU.	 * Let's use a preallocated one.  The following buf is protected by	 * CPU hotplug exclusion.	 */	target_attrs = wq_update_unbound_numa_attrs_buf;	cpumask = target_attrs->cpumask; 	copy_workqueue_attrs(target_attrs, wq->unbound_attrs);	pwq = unbound_pwq_by_node(wq, node); 	/*	 * Let's determine what needs to be done.  If the target cpumask is	 * different from the default pwq's, we need to compare it to @pwq's	 * and create a new one if they don't match.  If the target cpumask	 * equals the default pwq's, the default pwq should be used.	 */	if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) {		if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))			return;	} else {		goto use_dfl_pwq;	} 	/* create a new pwq */	pwq = alloc_unbound_pwq(wq, target_attrs);	if (!pwq) {		pr_warn("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",			wq->name);		goto use_dfl_pwq;	} 	/* Install the new pwq. */	mutex_lock(&wq->mutex);	old_pwq = numa_pwq_tbl_install(wq, node, pwq);	goto out_unlock; use_dfl_pwq:	mutex_lock(&wq->mutex);	spin_lock_irq(&wq->dfl_pwq->pool->lock);	get_pwq(wq->dfl_pwq);	spin_unlock_irq(&wq->dfl_pwq->pool->lock);	old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);out_unlock:	mutex_unlock(&wq->mutex);	put_pwq_unlocked(old_pwq);} static int alloc_and_link_pwqs(struct workqueue_struct *wq){	bool highpri = wq->flags & WQ_HIGHPRI;	int cpu, ret; 	if (!(wq->flags & WQ_UNBOUND)) {		wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);		if (!wq->cpu_pwqs)			return -ENOMEM; 		for_each_possible_cpu(cpu) {			struct pool_workqueue *pwq =				per_cpu_ptr(wq->cpu_pwqs, cpu);			struct worker_pool *cpu_pools =				per_cpu(cpu_worker_pools, cpu); 			init_pwq(pwq, wq, &cpu_pools[highpri]); 			mutex_lock(&wq->mutex);			link_pwq(pwq);			mutex_unlock(&wq->mutex);		}		return 0;	} else if (wq->flags & __WQ_ORDERED) {		ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);		/* there should only be single pwq for ordering guarantee */		WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||			      wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),		     "ordering guarantee broken for workqueue %s\n", wq->name);		return ret;	} else {		return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);

Lines 3501–4000 of 5816. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t