for_each_pool_worker(worker, pool) {			kthread_set_per_cpu(worker->task, -1);			WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, wq_unbound_cpumask) < 0);		} 		mutex_unlock(&wq_pool_attach_mutex);	}} /** * rebind_workers - rebind all workers of a pool to the associated CPU * @pool: pool of interest * * @pool->cpu is coming online.  Rebind all workers to the CPU. */static void rebind_workers(struct worker_pool *pool){	struct worker *worker; 	lockdep_assert_held(&wq_pool_attach_mutex); 	/*	 * Restore CPU affinity of all workers.  As all idle workers should	 * be on the run-queue of the associated CPU before any local	 * wake-ups for concurrency management happen, restore CPU affinity	 * of all workers first and then clear UNBOUND.  As we're called	 * from CPU_ONLINE, the following shouldn't fail.	 */	for_each_pool_worker(worker, pool) {		kthread_set_per_cpu(worker->task, pool->cpu);		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,						  pool->attrs->cpumask) < 0);	} 	raw_spin_lock_irq(&pool->lock); 	pool->flags &= ~POOL_DISASSOCIATED; 	for_each_pool_worker(worker, pool) {		unsigned int worker_flags = worker->flags; 		/*		 * We want to clear UNBOUND but can't directly call		 * worker_clr_flags() or adjust nr_running.  Atomically		 * replace UNBOUND with another NOT_RUNNING flag REBOUND.		 * @worker will clear REBOUND using worker_clr_flags() when		 * it initiates the next execution cycle thus restoring		 * concurrency management.  Note that when or whether		 * @worker clears REBOUND doesn't affect correctness.		 *		 * WRITE_ONCE() is necessary because @worker->flags may be		 * tested without holding any lock in		 * wq_worker_running().  Without it, NOT_RUNNING test may		 * fail incorrectly leading to premature concurrency		 * management operations.		 */		WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));		worker_flags |= WORKER_REBOUND;		worker_flags &= ~WORKER_UNBOUND;		WRITE_ONCE(worker->flags, worker_flags);	} 	raw_spin_unlock_irq(&pool->lock);} /** * restore_unbound_workers_cpumask - restore cpumask of unbound workers * @pool: unbound pool of interest * @cpu: the CPU which is coming up * * An unbound pool may end up with a cpumask which doesn't have any online * CPUs.  When a worker of such pool get scheduled, the scheduler resets * its cpus_allowed.  If @cpu is in @pool's cpumask which didn't have any * online CPU before, cpus_allowed of all its workers should be restored. */static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu){	static cpumask_t cpumask;	struct worker *worker; 	lockdep_assert_held(&wq_pool_attach_mutex); 	/* is @cpu allowed for @pool? */	if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))		return; 	cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask); 	/* as we're called from CPU_ONLINE, the following shouldn't fail */	for_each_pool_worker(worker, pool)		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);} int workqueue_prepare_cpu(unsigned int cpu){	struct worker_pool *pool; 	for_each_cpu_worker_pool(pool, cpu) {		if (pool->nr_workers)			continue;		if (!create_worker(pool))			return -ENOMEM;	}	return 0;} int workqueue_online_cpu(unsigned int cpu){	struct worker_pool *pool;	struct workqueue_struct *wq;	int pi; 	mutex_lock(&wq_pool_mutex); 	for_each_pool(pool, pi) {		mutex_lock(&wq_pool_attach_mutex); 		if (pool->cpu == cpu)			rebind_workers(pool);		else if (pool->cpu < 0)			restore_unbound_workers_cpumask(pool, cpu); 		mutex_unlock(&wq_pool_attach_mutex);	} 	/* update NUMA affinity of unbound workqueues */	list_for_each_entry(wq, &workqueues, list)		wq_update_unbound_numa(wq, cpu, true); 	mutex_unlock(&wq_pool_mutex);	return 0;} int workqueue_offline_cpu(unsigned int cpu){	struct workqueue_struct *wq; 	/* unbinding per-cpu workers should happen on the local CPU */	if (WARN_ON(cpu != smp_processor_id()))		return -1; 	unbind_workers(cpu); 	/* update NUMA affinity of unbound workqueues */	mutex_lock(&wq_pool_mutex);	list_for_each_entry(wq, &workqueues, list)		wq_update_unbound_numa(wq, cpu, false);	mutex_unlock(&wq_pool_mutex); 	return 0;} struct work_for_cpu {	struct work_struct work;	long (*fn)(void *);	void *arg;	long ret;}; static void work_for_cpu_fn(struct work_struct *work){	struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work); 	wfc->ret = wfc->fn(wfc->arg);} /** * work_on_cpu - run a function in thread context on a particular cpu * @cpu: the cpu to run on * @fn: the function to run * @arg: the function arg * * It is up to the caller to ensure that the cpu doesn't go offline. * The caller must not hold any locks which would prevent @fn from completing. * * Return: The value @fn returns. */long work_on_cpu(int cpu, long (*fn)(void *), void *arg){	struct work_for_cpu wfc = { .fn = fn, .arg = arg }; 	INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);	schedule_work_on(cpu, &wfc.work);	flush_work(&wfc.work);	destroy_work_on_stack(&wfc.work);	return wfc.ret;}EXPORT_SYMBOL_GPL(work_on_cpu); /** * work_on_cpu_safe - run a function in thread context on a particular cpu * @cpu: the cpu to run on * @fn:  the function to run * @arg: the function argument * * Disables CPU hotplug and calls work_on_cpu(). The caller must not hold * any locks which would prevent @fn from completing. * * Return: The value @fn returns. */long work_on_cpu_safe(int cpu, long (*fn)(void *), void *arg){	long ret = -ENODEV; 	cpus_read_lock();	if (cpu_online(cpu))		ret = work_on_cpu(cpu, fn, arg);	cpus_read_unlock();	return ret;}EXPORT_SYMBOL_GPL(work_on_cpu_safe);#endif /* CONFIG_SMP */ #ifdef CONFIG_FREEZER /** * freeze_workqueues_begin - begin freezing workqueues * * Start freezing workqueues.  After this function returns, all freezable * workqueues will queue new works to their inactive_works list instead of * pool->worklist. * * CONTEXT: * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's. */void freeze_workqueues_begin(void){	struct workqueue_struct *wq;	struct pool_workqueue *pwq; 	mutex_lock(&wq_pool_mutex); 	WARN_ON_ONCE(workqueue_freezing);	workqueue_freezing = true; 	list_for_each_entry(wq, &workqueues, list) {		mutex_lock(&wq->mutex);		for_each_pwq(pwq, wq)			pwq_adjust_max_active(pwq);		mutex_unlock(&wq->mutex);	} 	mutex_unlock(&wq_pool_mutex);} /** * freeze_workqueues_busy - are freezable workqueues still busy? * * Check whether freezing is complete.  This function must be called * between freeze_workqueues_begin() and thaw_workqueues(). * * CONTEXT: * Grabs and releases wq_pool_mutex. * * Return: * %true if some freezable workqueues are still busy.  %false if freezing * is complete. */bool freeze_workqueues_busy(void){	bool busy = false;	struct workqueue_struct *wq;	struct pool_workqueue *pwq; 	mutex_lock(&wq_pool_mutex); 	WARN_ON_ONCE(!workqueue_freezing); 	list_for_each_entry(wq, &workqueues, list) {		if (!(wq->flags & WQ_FREEZABLE))			continue;		/*		 * nr_active is monotonically decreasing.  It's safe		 * to peek without lock.		 */		rcu_read_lock();		for_each_pwq(pwq, wq) {			WARN_ON_ONCE(pwq->nr_active < 0);			if (pwq->nr_active) {				busy = true;				rcu_read_unlock();				goto out_unlock;			}		}		rcu_read_unlock();	}out_unlock:	mutex_unlock(&wq_pool_mutex);	return busy;} /** * thaw_workqueues - thaw workqueues * * Thaw workqueues.  Normal queueing is restored and all collected * frozen works are transferred to their respective pool worklists. * * CONTEXT: * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's. */void thaw_workqueues(void){	struct workqueue_struct *wq;	struct pool_workqueue *pwq; 	mutex_lock(&wq_pool_mutex); 	if (!workqueue_freezing)		goto out_unlock; 	workqueue_freezing = false; 	/* restore max_active and repopulate worklist */	list_for_each_entry(wq, &workqueues, list) {		mutex_lock(&wq->mutex);		for_each_pwq(pwq, wq)			pwq_adjust_max_active(pwq);		mutex_unlock(&wq->mutex);	} out_unlock:	mutex_unlock(&wq_pool_mutex);}#endif /* CONFIG_FREEZER */ static int workqueue_apply_unbound_cpumask(void){	LIST_HEAD(ctxs);	int ret = 0;	struct workqueue_struct *wq;	struct apply_wqattrs_ctx *ctx, *n; 	lockdep_assert_held(&wq_pool_mutex); 	list_for_each_entry(wq, &workqueues, list) {		if (!(wq->flags & WQ_UNBOUND))			continue;		/* creating multiple pwqs breaks ordering guarantee */		if (wq->flags & __WQ_ORDERED)			continue; 		ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);		if (!ctx) {			ret = -ENOMEM;			break;		} 		list_add_tail(&ctx->list, &ctxs);	} 	list_for_each_entry_safe(ctx, n, &ctxs, list) {		if (!ret)			apply_wqattrs_commit(ctx);		apply_wqattrs_cleanup(ctx);	} 	return ret;} /** *  workqueue_set_unbound_cpumask - Set the low-level unbound cpumask *  @cpumask: the cpumask to set * *  The low-level workqueues cpumask is a global cpumask that limits *  the affinity of all unbound workqueues.  This function check the @cpumask *  and apply it to all unbound workqueues and updates all pwqs of them. * *  Return:	0	- Success *  		-EINVAL	- Invalid @cpumask *  		-ENOMEM	- Failed to allocate memory for attrs or pwqs. */int workqueue_set_unbound_cpumask(cpumask_var_t cpumask){	int ret = -EINVAL;	cpumask_var_t saved_cpumask; 	/*	 * Not excluding isolated cpus on purpose.	 * If the user wishes to include them, we allow that.	 */	cpumask_and(cpumask, cpumask, cpu_possible_mask);	if (!cpumask_empty(cpumask)) {		apply_wqattrs_lock();		if (cpumask_equal(cpumask, wq_unbound_cpumask)) {			ret = 0;			goto out_unlock;		} 		if (!zalloc_cpumask_var(&saved_cpumask, GFP_KERNEL)) {			ret = -ENOMEM;			goto out_unlock;		} 		/* save the old wq_unbound_cpumask. */		cpumask_copy(saved_cpumask, wq_unbound_cpumask); 		/* update wq_unbound_cpumask at first and apply it to wqs. */		cpumask_copy(wq_unbound_cpumask, cpumask);		ret = workqueue_apply_unbound_cpumask(); 		/* restore the wq_unbound_cpumask when failed. */		if (ret < 0)			cpumask_copy(wq_unbound_cpumask, saved_cpumask); 		free_cpumask_var(saved_cpumask);out_unlock:		apply_wqattrs_unlock();	} 	return ret;} #ifdef CONFIG_SYSFS/* * Workqueues with WQ_SYSFS flag set is visible to userland via * /sys/bus/workqueue/devices/WQ_NAME.  All visible workqueues have the * following attributes. * *  per_cpu	RO bool	: whether the workqueue is per-cpu or unbound *  max_active	RW int	: maximum number of in-flight work items * * Unbound workqueues have the following extra attributes. * *  pool_ids	RO int	: the associated pool IDs for each node *  nice	RW int	: nice value of the workers *  cpumask	RW mask	: bitmask of allowed CPUs for the workers *  numa	RW bool	: whether enable NUMA affinity */struct wq_device {	struct workqueue_struct		*wq;	struct device			dev;}; static struct workqueue_struct *dev_to_wq(struct device *dev){	struct wq_device *wq_dev = container_of(dev, struct wq_device, dev); 	return wq_dev->wq;} static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,			    char *buf){	struct workqueue_struct *wq = dev_to_wq(dev); 	return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));}static DEVICE_ATTR_RO(per_cpu); static ssize_t max_active_show(struct device *dev,			       struct device_attribute *attr, char *buf){	struct workqueue_struct *wq = dev_to_wq(dev); 	return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);} static ssize_t max_active_store(struct device *dev,				struct device_attribute *attr, const char *buf,				size_t count){	struct workqueue_struct *wq = dev_to_wq(dev);	int val; 	if (sscanf(buf, "%d", &val) != 1 || val <= 0)		return -EINVAL; 	workqueue_set_max_active(wq, val);	return count;}static DEVICE_ATTR_RW(max_active); static struct attribute *wq_sysfs_attrs[] = {	&dev_attr_per_cpu.attr,	&dev_attr_max_active.attr,	NULL,};ATTRIBUTE_GROUPS(wq_sysfs); static ssize_t wq_pool_ids_show(struct device *dev,				struct device_attribute *attr, char *buf){	struct workqueue_struct *wq = dev_to_wq(dev);	const char *delim = "";	int node, written = 0; 	cpus_read_lock();	rcu_read_lock();	for_each_node(node) {		written += scnprintf(buf + written, PAGE_SIZE - written,				     "%s%d:%d", delim, node,				     unbound_pwq_by_node(wq, node)->pool->id);		delim = " ";	}	written += scnprintf(buf + written, PAGE_SIZE - written, "\n");	rcu_read_unlock();	cpus_read_unlock(); 	return written;

Lines 5001–5500 of 6121. Previous lines Load more