5816 lines · 159.4 KB · C
RawDownload
	 * isn't executing.	 */	if (wq_online)		__flush_work(work, true); 	clear_work_data(work); 	/*	 * Paired with prepare_to_wait() above so that either	 * waitqueue_active() is visible here or !work_is_canceling() is	 * visible there.	 */	smp_mb();	if (waitqueue_active(&cancel_waitq))		__wake_up(&cancel_waitq, TASK_NORMAL, 1, work); 	return ret;} /** * cancel_work_sync - cancel a work and wait for it to finish * @work: the work to cancel * * Cancel @work and wait for its execution to finish.  This function * can be used even if the work re-queues itself or migrates to * another workqueue.  On return from this function, @work is * guaranteed to be not pending or executing on any CPU. * * cancel_work_sync(&delayed_work->work) must not be used for * delayed_work's.  Use cancel_delayed_work_sync() instead. * * The caller must ensure that the workqueue on which @work was last * queued can't be destroyed before this function returns. * * Return: * %true if @work was pending, %false otherwise. */bool cancel_work_sync(struct work_struct *work){	return __cancel_work_timer(work, false);}EXPORT_SYMBOL_GPL(cancel_work_sync); /** * flush_delayed_work - wait for a dwork to finish executing the last queueing * @dwork: the delayed work to flush * * Delayed timer is cancelled and the pending work is queued for * immediate execution.  Like flush_work(), this function only * considers the last queueing instance of @dwork. * * Return: * %true if flush_work() waited for the work to finish execution, * %false if it was already idle. */bool flush_delayed_work(struct delayed_work *dwork){	local_irq_disable();	if (del_timer_sync(&dwork->timer))		__queue_work(dwork->cpu, dwork->wq, &dwork->work);	local_irq_enable();	return flush_work(&dwork->work);}EXPORT_SYMBOL(flush_delayed_work); /** * flush_rcu_work - wait for a rwork to finish executing the last queueing * @rwork: the rcu work to flush * * Return: * %true if flush_rcu_work() waited for the work to finish execution, * %false if it was already idle. */bool flush_rcu_work(struct rcu_work *rwork){	if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {		rcu_barrier();		flush_work(&rwork->work);		return true;	} else {		return flush_work(&rwork->work);	}}EXPORT_SYMBOL(flush_rcu_work); static bool __cancel_work(struct work_struct *work, bool is_dwork){	unsigned long flags;	int ret; 	do {		ret = try_to_grab_pending(work, is_dwork, &flags);	} while (unlikely(ret == -EAGAIN)); 	if (unlikely(ret < 0))		return false; 	set_work_pool_and_clear_pending(work, get_work_pool_id(work));	local_irq_restore(flags);	return ret;} /** * cancel_delayed_work - cancel a delayed work * @dwork: delayed_work to cancel * * Kill off a pending delayed_work. * * Return: %true if @dwork was pending and canceled; %false if it wasn't * pending. * * Note: * The work callback function may still be running on return, unless * it returns %true and the work doesn't re-arm itself.  Explicitly flush or * use cancel_delayed_work_sync() to wait on it. * * This function is safe to call from any context including IRQ handler. */bool cancel_delayed_work(struct delayed_work *dwork){	return __cancel_work(&dwork->work, true);}EXPORT_SYMBOL(cancel_delayed_work); /** * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish * @dwork: the delayed work cancel * * This is cancel_work_sync() for delayed works. * * Return: * %true if @dwork was pending, %false otherwise. */bool cancel_delayed_work_sync(struct delayed_work *dwork){	return __cancel_work_timer(&dwork->work, true);}EXPORT_SYMBOL(cancel_delayed_work_sync); /** * schedule_on_each_cpu - execute a function synchronously on each online CPU * @func: the function to call * * schedule_on_each_cpu() executes @func on each online CPU using the * system workqueue and blocks until all CPUs have completed. * schedule_on_each_cpu() is very slow. * * Return: * 0 on success, -errno on failure. */int schedule_on_each_cpu(work_func_t func){	int cpu;	struct work_struct __percpu *works; 	works = alloc_percpu(struct work_struct);	if (!works)		return -ENOMEM; 	get_online_cpus(); 	for_each_online_cpu(cpu) {		struct work_struct *work = per_cpu_ptr(works, cpu); 		INIT_WORK(work, func);		schedule_work_on(cpu, work);	} 	for_each_online_cpu(cpu)		flush_work(per_cpu_ptr(works, cpu)); 	put_online_cpus();	free_percpu(works);	return 0;} /** * execute_in_process_context - reliably execute the routine with user context * @fn:		the function to execute * @ew:		guaranteed storage for the execute work structure (must *		be available when the work executes) * * Executes the function immediately if process context is available, * otherwise schedules the function for delayed execution. * * Return:	0 - function was executed *		1 - function was scheduled for execution */int execute_in_process_context(work_func_t fn, struct execute_work *ew){	if (!in_interrupt()) {		fn(&ew->work);		return 0;	} 	INIT_WORK(&ew->work, fn);	schedule_work(&ew->work); 	return 1;}EXPORT_SYMBOL_GPL(execute_in_process_context); /** * free_workqueue_attrs - free a workqueue_attrs * @attrs: workqueue_attrs to free * * Undo alloc_workqueue_attrs(). */void free_workqueue_attrs(struct workqueue_attrs *attrs){	if (attrs) {		free_cpumask_var(attrs->cpumask);		kfree(attrs);	}} /** * alloc_workqueue_attrs - allocate a workqueue_attrs * @gfp_mask: allocation mask to use * * Allocate a new workqueue_attrs, initialize with default settings and * return it. * * Return: The allocated new workqueue_attr on success. %NULL on failure. */struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask){	struct workqueue_attrs *attrs; 	attrs = kzalloc(sizeof(*attrs), gfp_mask);	if (!attrs)		goto fail;	if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))		goto fail; 	cpumask_copy(attrs->cpumask, cpu_possible_mask);	return attrs;fail:	free_workqueue_attrs(attrs);	return NULL;} static void copy_workqueue_attrs(struct workqueue_attrs *to,				 const struct workqueue_attrs *from){	to->nice = from->nice;	cpumask_copy(to->cpumask, from->cpumask);	/*	 * Unlike hash and equality test, this function doesn't ignore	 * ->no_numa as it is used for both pool and wq attrs.  Instead,	 * get_unbound_pool() explicitly clears ->no_numa after copying.	 */	to->no_numa = from->no_numa;} /* hash value of the content of @attr */static u32 wqattrs_hash(const struct workqueue_attrs *attrs){	u32 hash = 0; 	hash = jhash_1word(attrs->nice, hash);	hash = jhash(cpumask_bits(attrs->cpumask),		     BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);	return hash;} /* content equality test */static bool wqattrs_equal(const struct workqueue_attrs *a,			  const struct workqueue_attrs *b){	if (a->nice != b->nice)		return false;	if (!cpumask_equal(a->cpumask, b->cpumask))		return false;	return true;} /** * init_worker_pool - initialize a newly zalloc'd worker_pool * @pool: worker_pool to initialize * * Initialize a newly zalloc'd @pool.  It also allocates @pool->attrs. * * Return: 0 on success, -errno on failure.  Even on failure, all fields * inside @pool proper are initialized and put_unbound_pool() can be called * on @pool safely to release it. */static int init_worker_pool(struct worker_pool *pool){	spin_lock_init(&pool->lock);	pool->id = -1;	pool->cpu = -1;	pool->node = NUMA_NO_NODE;	pool->flags |= POOL_DISASSOCIATED;	pool->watchdog_ts = jiffies;	INIT_LIST_HEAD(&pool->worklist);	INIT_LIST_HEAD(&pool->idle_list);	hash_init(pool->busy_hash); 	timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE); 	timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0); 	INIT_LIST_HEAD(&pool->workers); 	ida_init(&pool->worker_ida);	INIT_HLIST_NODE(&pool->hash_node);	pool->refcnt = 1; 	/* shouldn't fail above this point */	pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);	if (!pool->attrs)		return -ENOMEM;	return 0;} static void rcu_free_wq(struct rcu_head *rcu){	struct workqueue_struct *wq =		container_of(rcu, struct workqueue_struct, rcu); 	if (!(wq->flags & WQ_UNBOUND))		free_percpu(wq->cpu_pwqs);	else		free_workqueue_attrs(wq->unbound_attrs); 	kfree(wq->rescuer);	kfree(wq);} static void rcu_free_pool(struct rcu_head *rcu){	struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu); 	ida_destroy(&pool->worker_ida);	free_workqueue_attrs(pool->attrs);	kfree(pool);} /** * put_unbound_pool - put a worker_pool * @pool: worker_pool to put * * Put @pool.  If its refcnt reaches zero, it gets destroyed in sched-RCU * safe manner.  get_unbound_pool() calls this function on its failure path * and this function should be able to release pools which went through, * successfully or not, init_worker_pool(). * * Should be called with wq_pool_mutex held. */static void put_unbound_pool(struct worker_pool *pool){	DECLARE_COMPLETION_ONSTACK(detach_completion);	struct worker *worker; 	lockdep_assert_held(&wq_pool_mutex); 	if (--pool->refcnt)		return; 	/* sanity checks */	if (WARN_ON(!(pool->cpu < 0)) ||	    WARN_ON(!list_empty(&pool->worklist)))		return; 	/* release id and unhash */	if (pool->id >= 0)		idr_remove(&worker_pool_idr, pool->id);	hash_del(&pool->hash_node); 	/*	 * Become the manager and destroy all workers.  This prevents	 * @pool's workers from blocking on attach_mutex.  We're the last	 * manager and @pool gets freed with the flag set.	 */	spin_lock_irq(&pool->lock);	wait_event_lock_irq(wq_manager_wait,			    !(pool->flags & POOL_MANAGER_ACTIVE), pool->lock);	pool->flags |= POOL_MANAGER_ACTIVE; 	while ((worker = first_idle_worker(pool)))		destroy_worker(worker);	WARN_ON(pool->nr_workers || pool->nr_idle);	spin_unlock_irq(&pool->lock); 	mutex_lock(&wq_pool_attach_mutex);	if (!list_empty(&pool->workers))		pool->detach_completion = &detach_completion;	mutex_unlock(&wq_pool_attach_mutex); 	if (pool->detach_completion)		wait_for_completion(pool->detach_completion); 	/* shut down the timers */	del_timer_sync(&pool->idle_timer);	del_timer_sync(&pool->mayday_timer); 	/* sched-RCU protected to allow dereferences from get_work_pool() */	call_rcu_sched(&pool->rcu, rcu_free_pool);} /** * get_unbound_pool - get a worker_pool with the specified attributes * @attrs: the attributes of the worker_pool to get * * Obtain a worker_pool which has the same attributes as @attrs, bump the * reference count and return it.  If there already is a matching * worker_pool, it will be used; otherwise, this function attempts to * create a new one. * * Should be called with wq_pool_mutex held. * * Return: On success, a worker_pool with the same attributes as @attrs. * On failure, %NULL. */static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs){	u32 hash = wqattrs_hash(attrs);	struct worker_pool *pool;	int node;	int target_node = NUMA_NO_NODE; 	lockdep_assert_held(&wq_pool_mutex); 	/* do we already have a matching pool? */	hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {		if (wqattrs_equal(pool->attrs, attrs)) {			pool->refcnt++;			return pool;		}	} 	/* if cpumask is contained inside a NUMA node, we belong to that node */	if (wq_numa_enabled) {		for_each_node(node) {			if (cpumask_subset(attrs->cpumask,					   wq_numa_possible_cpumask[node])) {				target_node = node;				break;			}		}	} 	/* nope, create a new one */	pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, target_node);	if (!pool || init_worker_pool(pool) < 0)		goto fail; 	lockdep_set_subclass(&pool->lock, 1);	/* see put_pwq() */	copy_workqueue_attrs(pool->attrs, attrs);	pool->node = target_node; 	/*	 * no_numa isn't a worker_pool attribute, always clear it.  See	 * 'struct workqueue_attrs' comments for detail.	 */	pool->attrs->no_numa = false; 	if (worker_pool_assign_id(pool) < 0)		goto fail; 	/* create and start the initial worker */	if (wq_online && !create_worker(pool))		goto fail; 	/* install */	hash_add(unbound_pool_hash, &pool->hash_node, hash); 	return pool;fail:	if (pool)		put_unbound_pool(pool);	return NULL;} static void rcu_free_pwq(struct rcu_head *rcu){	kmem_cache_free(pwq_cache,			container_of(rcu, struct pool_workqueue, rcu));} /* * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt * and needs to be destroyed. */static void pwq_unbound_release_workfn(struct work_struct *work){	struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,						  unbound_release_work);	struct workqueue_struct *wq = pwq->wq;	struct worker_pool *pool = pwq->pool;	bool is_last; 	if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))		return; 	mutex_lock(&wq->mutex);	list_del_rcu(&pwq->pwqs_node);	is_last = list_empty(&wq->pwqs);	mutex_unlock(&wq->mutex);

Lines 3001–3500 of 5816. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t