} 	check_flush_dependency(pwq->wq, work); 	insert_wq_barrier(pwq, barr, work, worker);	raw_spin_unlock_irq(&pool->lock); 	/*	 * Force a lock recursion deadlock when using flush_work() inside a	 * single-threaded or rescuer equipped workqueue.	 *	 * For single threaded workqueues the deadlock happens when the work	 * is after the work issuing the flush_work(). For rescuer equipped	 * workqueues the deadlock happens when the rescuer stalls, blocking	 * forward progress.	 */	if (!from_cancel &&	    (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)) {		lock_map_acquire(&pwq->wq->lockdep_map);		lock_map_release(&pwq->wq->lockdep_map);	}	rcu_read_unlock();	return true;already_gone:	raw_spin_unlock_irq(&pool->lock);	rcu_read_unlock();	return false;} static bool __flush_work(struct work_struct *work, bool from_cancel){	struct wq_barrier barr; 	if (WARN_ON(!wq_online))		return false; 	if (WARN_ON(!work->func))		return false; 	if (!from_cancel) {		lock_map_acquire(&work->lockdep_map);		lock_map_release(&work->lockdep_map);	} 	if (start_flush_work(work, &barr, from_cancel)) {		wait_for_completion(&barr.done);		destroy_work_on_stack(&barr.work);		return true;	} else {		return false;	}} /** * flush_work - wait for a work to finish executing the last queueing instance * @work: the work to flush * * Wait until @work has finished execution.  @work is guaranteed to be idle * on return if it hasn't been requeued since flush started. * * Return: * %true if flush_work() waited for the work to finish execution, * %false if it was already idle. */bool flush_work(struct work_struct *work){	return __flush_work(work, false);}EXPORT_SYMBOL_GPL(flush_work); struct cwt_wait {	wait_queue_entry_t		wait;	struct work_struct	*work;}; static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *key){	struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait); 	if (cwait->work != key)		return 0;	return autoremove_wake_function(wait, mode, sync, key);} static bool __cancel_work_timer(struct work_struct *work, bool is_dwork){	static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq);	unsigned long flags;	int ret; 	do {		ret = try_to_grab_pending(work, is_dwork, &flags);		/*		 * If someone else is already canceling, wait for it to		 * finish.  flush_work() doesn't work for PREEMPT_NONE		 * because we may get scheduled between @work's completion		 * and the other canceling task resuming and clearing		 * CANCELING - flush_work() will return false immediately		 * as @work is no longer busy, try_to_grab_pending() will		 * return -ENOENT as @work is still being canceled and the		 * other canceling task won't be able to clear CANCELING as		 * we're hogging the CPU.		 *		 * Let's wait for completion using a waitqueue.  As this		 * may lead to the thundering herd problem, use a custom		 * wake function which matches @work along with exclusive		 * wait and wakeup.		 */		if (unlikely(ret == -ENOENT)) {			struct cwt_wait cwait; 			init_wait(&cwait.wait);			cwait.wait.func = cwt_wakefn;			cwait.work = work; 			prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait,						  TASK_UNINTERRUPTIBLE);			if (work_is_canceling(work))				schedule();			finish_wait(&cancel_waitq, &cwait.wait);		}	} while (unlikely(ret < 0)); 	/* tell other tasks trying to grab @work to back off */	mark_work_canceling(work);	local_irq_restore(flags); 	/*	 * This allows canceling during early boot.  We know that @work	 * isn't executing.	 */	if (wq_online)		__flush_work(work, true); 	clear_work_data(work); 	/*	 * Paired with prepare_to_wait() above so that either	 * waitqueue_active() is visible here or !work_is_canceling() is	 * visible there.	 */	smp_mb();	if (waitqueue_active(&cancel_waitq))		__wake_up(&cancel_waitq, TASK_NORMAL, 1, work); 	return ret;} /** * cancel_work_sync - cancel a work and wait for it to finish * @work: the work to cancel * * Cancel @work and wait for its execution to finish.  This function * can be used even if the work re-queues itself or migrates to * another workqueue.  On return from this function, @work is * guaranteed to be not pending or executing on any CPU. * * cancel_work_sync(&delayed_work->work) must not be used for * delayed_work's.  Use cancel_delayed_work_sync() instead. * * The caller must ensure that the workqueue on which @work was last * queued can't be destroyed before this function returns. * * Return: * %true if @work was pending, %false otherwise. */bool cancel_work_sync(struct work_struct *work){	return __cancel_work_timer(work, false);}EXPORT_SYMBOL_GPL(cancel_work_sync); /** * flush_delayed_work - wait for a dwork to finish executing the last queueing * @dwork: the delayed work to flush * * Delayed timer is cancelled and the pending work is queued for * immediate execution.  Like flush_work(), this function only * considers the last queueing instance of @dwork. * * Return: * %true if flush_work() waited for the work to finish execution, * %false if it was already idle. */bool flush_delayed_work(struct delayed_work *dwork){	local_irq_disable();	if (del_timer_sync(&dwork->timer))		__queue_work(dwork->cpu, dwork->wq, &dwork->work);	local_irq_enable();	return flush_work(&dwork->work);}EXPORT_SYMBOL(flush_delayed_work); /** * flush_rcu_work - wait for a rwork to finish executing the last queueing * @rwork: the rcu work to flush * * Return: * %true if flush_rcu_work() waited for the work to finish execution, * %false if it was already idle. */bool flush_rcu_work(struct rcu_work *rwork){	if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {		rcu_barrier();		flush_work(&rwork->work);		return true;	} else {		return flush_work(&rwork->work);	}}EXPORT_SYMBOL(flush_rcu_work); static bool __cancel_work(struct work_struct *work, bool is_dwork){	unsigned long flags;	int ret; 	do {		ret = try_to_grab_pending(work, is_dwork, &flags);	} while (unlikely(ret == -EAGAIN)); 	if (unlikely(ret < 0))		return false; 	set_work_pool_and_clear_pending(work, get_work_pool_id(work));	local_irq_restore(flags);	return ret;} /** * cancel_delayed_work - cancel a delayed work * @dwork: delayed_work to cancel * * Kill off a pending delayed_work. * * Return: %true if @dwork was pending and canceled; %false if it wasn't * pending. * * Note: * The work callback function may still be running on return, unless * it returns %true and the work doesn't re-arm itself.  Explicitly flush or * use cancel_delayed_work_sync() to wait on it. * * This function is safe to call from any context including IRQ handler. */bool cancel_delayed_work(struct delayed_work *dwork){	return __cancel_work(&dwork->work, true);}EXPORT_SYMBOL(cancel_delayed_work); /** * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish * @dwork: the delayed work cancel * * This is cancel_work_sync() for delayed works. * * Return: * %true if @dwork was pending, %false otherwise. */bool cancel_delayed_work_sync(struct delayed_work *dwork){	return __cancel_work_timer(&dwork->work, true);}EXPORT_SYMBOL(cancel_delayed_work_sync); /** * schedule_on_each_cpu - execute a function synchronously on each online CPU * @func: the function to call * * schedule_on_each_cpu() executes @func on each online CPU using the * system workqueue and blocks until all CPUs have completed. * schedule_on_each_cpu() is very slow. * * Return: * 0 on success, -errno on failure. */int schedule_on_each_cpu(work_func_t func){	int cpu;	struct work_struct __percpu *works; 	works = alloc_percpu(struct work_struct);	if (!works)		return -ENOMEM; 	get_online_cpus(); 	for_each_online_cpu(cpu) {		struct work_struct *work = per_cpu_ptr(works, cpu); 		INIT_WORK(work, func);		schedule_work_on(cpu, work);	} 	for_each_online_cpu(cpu)		flush_work(per_cpu_ptr(works, cpu)); 	put_online_cpus();	free_percpu(works);	return 0;} /** * execute_in_process_context - reliably execute the routine with user context * @fn:		the function to execute * @ew:		guaranteed storage for the execute work structure (must *		be available when the work executes) * * Executes the function immediately if process context is available, * otherwise schedules the function for delayed execution. * * Return:	0 - function was executed *		1 - function was scheduled for execution */int execute_in_process_context(work_func_t fn, struct execute_work *ew){	if (!in_interrupt()) {		fn(&ew->work);		return 0;	} 	INIT_WORK(&ew->work, fn);	schedule_work(&ew->work); 	return 1;}EXPORT_SYMBOL_GPL(execute_in_process_context); /** * free_workqueue_attrs - free a workqueue_attrs * @attrs: workqueue_attrs to free * * Undo alloc_workqueue_attrs(). */void free_workqueue_attrs(struct workqueue_attrs *attrs){	if (attrs) {		free_cpumask_var(attrs->cpumask);		kfree(attrs);	}} /** * alloc_workqueue_attrs - allocate a workqueue_attrs * * Allocate a new workqueue_attrs, initialize with default settings and * return it. * * Return: The allocated new workqueue_attr on success. %NULL on failure. */struct workqueue_attrs *alloc_workqueue_attrs(void){	struct workqueue_attrs *attrs; 	attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);	if (!attrs)		goto fail;	if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))		goto fail; 	cpumask_copy(attrs->cpumask, cpu_possible_mask);	return attrs;fail:	free_workqueue_attrs(attrs);	return NULL;} static void copy_workqueue_attrs(struct workqueue_attrs *to,				 const struct workqueue_attrs *from){	to->nice = from->nice;	cpumask_copy(to->cpumask, from->cpumask);	/*	 * Unlike hash and equality test, this function doesn't ignore	 * ->no_numa as it is used for both pool and wq attrs.  Instead,	 * get_unbound_pool() explicitly clears ->no_numa after copying.	 */	to->no_numa = from->no_numa;} /* hash value of the content of @attr */static u32 wqattrs_hash(const struct workqueue_attrs *attrs){	u32 hash = 0; 	hash = jhash_1word(attrs->nice, hash);	hash = jhash(cpumask_bits(attrs->cpumask),		     BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);	return hash;} /* content equality test */static bool wqattrs_equal(const struct workqueue_attrs *a,			  const struct workqueue_attrs *b){	if (a->nice != b->nice)		return false;	if (!cpumask_equal(a->cpumask, b->cpumask))		return false;	return true;} /** * init_worker_pool - initialize a newly zalloc'd worker_pool * @pool: worker_pool to initialize * * Initialize a newly zalloc'd @pool.  It also allocates @pool->attrs. * * Return: 0 on success, -errno on failure.  Even on failure, all fields * inside @pool proper are initialized and put_unbound_pool() can be called * on @pool safely to release it. */static int init_worker_pool(struct worker_pool *pool){	raw_spin_lock_init(&pool->lock);	pool->id = -1;	pool->cpu = -1;	pool->node = NUMA_NO_NODE;	pool->flags |= POOL_DISASSOCIATED;	pool->watchdog_ts = jiffies;	INIT_LIST_HEAD(&pool->worklist);	INIT_LIST_HEAD(&pool->idle_list);	hash_init(pool->busy_hash); 	timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE); 	timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0); 	INIT_LIST_HEAD(&pool->workers); 	ida_init(&pool->worker_ida);	INIT_HLIST_NODE(&pool->hash_node);	pool->refcnt = 1; 	/* shouldn't fail above this point */	pool->attrs = alloc_workqueue_attrs();	if (!pool->attrs)		return -ENOMEM;	return 0;} #ifdef CONFIG_LOCKDEPstatic void wq_init_lockdep(struct workqueue_struct *wq){	char *lock_name; 	lockdep_register_key(&wq->key);	lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);	if (!lock_name)		lock_name = wq->name; 	wq->lock_name = lock_name;	lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0);} static void wq_unregister_lockdep(struct workqueue_struct *wq){	lockdep_unregister_key(&wq->key);} static void wq_free_lockdep(struct workqueue_struct *wq){	if (wq->lock_name != wq->name)		kfree(wq->lock_name);}#elsestatic void wq_init_lockdep(struct workqueue_struct *wq){} static void wq_unregister_lockdep(struct workqueue_struct *wq){} static void wq_free_lockdep(struct workqueue_struct *wq){}#endif static void rcu_free_wq(struct rcu_head *rcu){	struct workqueue_struct *wq =		container_of(rcu, struct workqueue_struct, rcu); 	wq_free_lockdep(wq); 	if (!(wq->flags & WQ_UNBOUND))		free_percpu(wq->cpu_pwqs);	else		free_workqueue_attrs(wq->unbound_attrs); 	kfree(wq);} static void rcu_free_pool(struct rcu_head *rcu){	struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);

Lines 3001–3500 of 6039. Previous lines Load more