* the right NUMA node. If no node is requested or the requested node is * offline then we just fall back to standard queue_work behavior. * * Currently the "random" CPU ends up being the first available CPU in the * intersection of cpu_online_mask and the cpumask of the node, unless we * are running on the node. In that case we just use the current CPU. * * Return: %false if @work was already on a queue, %true otherwise. */bool queue_work_node(int node, struct workqueue_struct *wq,		     struct work_struct *work){	unsigned long irq_flags;	bool ret = false; 	/*	 * This current implementation is specific to unbound workqueues.	 * Specifically we only return the first available CPU for a given	 * node instead of cycling through individual CPUs within the node.	 *	 * If this is used with a per-cpu workqueue then the logic in	 * workqueue_select_cpu_near would need to be updated to allow for	 * some round robin type logic.	 */	WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)); 	local_irq_save(irq_flags); 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&	    !clear_pending_if_disabled(work)) {		int cpu = select_numa_node_cpu(node); 		__queue_work(cpu, wq, work);		ret = true;	} 	local_irq_restore(irq_flags);	return ret;}EXPORT_SYMBOL_GPL(queue_work_node); void delayed_work_timer_fn(struct timer_list *t){	struct delayed_work *dwork = timer_container_of(dwork, t, timer); 	/* should have been called from irqsafe timer with irq already off */	__queue_work(dwork->cpu, dwork->wq, &dwork->work);}EXPORT_SYMBOL(delayed_work_timer_fn); static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,				struct delayed_work *dwork, unsigned long delay){	struct timer_list *timer = &dwork->timer;	struct work_struct *work = &dwork->work; 	WARN_ON_ONCE(timer->function != delayed_work_timer_fn);	WARN_ON_ONCE(timer_pending(timer));	WARN_ON_ONCE(!list_empty(&work->entry)); 	/*	 * If @delay is 0, queue @dwork->work immediately.  This is for	 * both optimization and correctness.  The earliest @timer can	 * expire is on the closest next tick and delayed_work users depend	 * on that there's no such delay when @delay is 0.	 */	if (!delay) {		__queue_work(cpu, wq, &dwork->work);		return;	} 	WARN_ON_ONCE(cpu != WORK_CPU_UNBOUND && !cpu_online(cpu));	dwork->wq = wq;	dwork->cpu = cpu;	timer->expires = jiffies + delay; 	if (housekeeping_enabled(HK_TYPE_TIMER)) {		/* If the current cpu is a housekeeping cpu, use it. */		cpu = smp_processor_id();		if (!housekeeping_test_cpu(cpu, HK_TYPE_TIMER))			cpu = housekeeping_any_cpu(HK_TYPE_TIMER);		add_timer_on(timer, cpu);	} else {		if (likely(cpu == WORK_CPU_UNBOUND))			add_timer_global(timer);		else			add_timer_on(timer, cpu);	}} /** * queue_delayed_work_on - queue work on specific CPU after delay * @cpu: CPU number to execute work on * @wq: workqueue to use * @dwork: work to queue * @delay: number of jiffies to wait before queueing * * We queue the delayed_work to a specific CPU, for non-zero delays the * caller must ensure it is online and can't go away. Callers that fail * to ensure this, may get @dwork->timer queued to an offlined CPU and * this will prevent queueing of @dwork->work unless the offlined CPU * becomes online again. * * Return: %false if @work was already on a queue, %true otherwise.  If * @delay is zero and @dwork is idle, it will be scheduled for immediate * execution. */bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,			   struct delayed_work *dwork, unsigned long delay){	struct work_struct *work = &dwork->work;	bool ret = false;	unsigned long irq_flags; 	/* read the comment in __queue_work() */	local_irq_save(irq_flags); 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&	    !clear_pending_if_disabled(work)) {		__queue_delayed_work(cpu, wq, dwork, delay);		ret = true;	} 	local_irq_restore(irq_flags);	return ret;}EXPORT_SYMBOL(queue_delayed_work_on); /** * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU * @cpu: CPU number to execute work on * @wq: workqueue to use * @dwork: work to queue * @delay: number of jiffies to wait before queueing * * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise, * modify @dwork's timer so that it expires after @delay.  If @delay is * zero, @work is guaranteed to be scheduled immediately regardless of its * current state. * * Return: %false if @dwork was idle and queued, %true if @dwork was * pending and its timer was modified. * * This function is safe to call from any context including IRQ handler. * See try_to_grab_pending() for details. */bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,			 struct delayed_work *dwork, unsigned long delay){	unsigned long irq_flags;	bool ret; 	ret = work_grab_pending(&dwork->work, WORK_CANCEL_DELAYED, &irq_flags); 	if (!clear_pending_if_disabled(&dwork->work))		__queue_delayed_work(cpu, wq, dwork, delay); 	local_irq_restore(irq_flags);	return ret;}EXPORT_SYMBOL_GPL(mod_delayed_work_on); static void rcu_work_rcufn(struct rcu_head *rcu){	struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu); 	/* read the comment in __queue_work() */	local_irq_disable();	__queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);	local_irq_enable();} /** * queue_rcu_work - queue work after a RCU grace period * @wq: workqueue to use * @rwork: work to queue * * Return: %false if @rwork was already pending, %true otherwise.  Note * that a full RCU grace period is guaranteed only after a %true return. * While @rwork is guaranteed to be executed after a %false return, the * execution may happen before a full RCU grace period has passed. */bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork){	struct work_struct *work = &rwork->work; 	/*	 * rcu_work can't be canceled or disabled. Warn if the user reached	 * inside @rwork and disabled the inner work.	 */	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&	    !WARN_ON_ONCE(clear_pending_if_disabled(work))) {		rwork->wq = wq;		call_rcu_hurry(&rwork->rcu, rcu_work_rcufn);		return true;	} 	return false;}EXPORT_SYMBOL(queue_rcu_work); static struct worker *alloc_worker(int node){	struct worker *worker; 	worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);	if (worker) {		INIT_LIST_HEAD(&worker->entry);		INIT_LIST_HEAD(&worker->scheduled);		INIT_LIST_HEAD(&worker->node);		/* on creation a worker is in !idle && prep state */		worker->flags = WORKER_PREP;	}	return worker;} static cpumask_t *pool_allowed_cpus(struct worker_pool *pool){	if (pool->cpu < 0 && pool->attrs->affn_strict)		return pool->attrs->__pod_cpumask;	else		return pool->attrs->cpumask;} /** * worker_attach_to_pool() - attach a worker to a pool * @worker: worker to be attached * @pool: the target pool * * Attach @worker to @pool.  Once attached, the %WORKER_UNBOUND flag and * cpu-binding of @worker are kept coordinated with the pool across * cpu-[un]hotplugs. */static void worker_attach_to_pool(struct worker *worker,				  struct worker_pool *pool){	mutex_lock(&wq_pool_attach_mutex); 	/*	 * The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains stable	 * across this function. See the comments above the flag definition for	 * details. BH workers are, while per-CPU, always DISASSOCIATED.	 */	if (pool->flags & POOL_DISASSOCIATED) {		worker->flags |= WORKER_UNBOUND;	} else {		WARN_ON_ONCE(pool->flags & POOL_BH);		kthread_set_per_cpu(worker->task, pool->cpu);	} 	if (worker->rescue_wq)		set_cpus_allowed_ptr(worker->task, pool_allowed_cpus(pool)); 	list_add_tail(&worker->node, &pool->workers);	worker->pool = pool; 	mutex_unlock(&wq_pool_attach_mutex);} static void unbind_worker(struct worker *worker){	lockdep_assert_held(&wq_pool_attach_mutex); 	kthread_set_per_cpu(worker->task, -1);	if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask))		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, wq_unbound_cpumask) < 0);	else		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0);}  static void detach_worker(struct worker *worker){	lockdep_assert_held(&wq_pool_attach_mutex); 	unbind_worker(worker);	list_del(&worker->node);} /** * worker_detach_from_pool() - detach a worker from its pool * @worker: worker which is attached to its pool * * Undo the attaching which had been done in worker_attach_to_pool().  The * caller worker shouldn't access to the pool after detached except it has * other reference to the pool. */static void worker_detach_from_pool(struct worker *worker){	struct worker_pool *pool = worker->pool; 	/* there is one permanent BH worker per CPU which should never detach */	WARN_ON_ONCE(pool->flags & POOL_BH); 	mutex_lock(&wq_pool_attach_mutex);	detach_worker(worker);	worker->pool = NULL;	mutex_unlock(&wq_pool_attach_mutex); 	/* clear leftover flags without pool->lock after it is detached */	worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);} static int format_worker_id(char *buf, size_t size, struct worker *worker,			    struct worker_pool *pool){	if (worker->rescue_wq)		return scnprintf(buf, size, "kworker/R-%s",				 worker->rescue_wq->name); 	if (pool) {		if (pool->cpu >= 0)			return scnprintf(buf, size, "kworker/%d:%d%s",					 pool->cpu, worker->id,					 pool->attrs->nice < 0  ? "H" : "");		else			return scnprintf(buf, size, "kworker/u%d:%d",					 pool->id, worker->id);	} else {		return scnprintf(buf, size, "kworker/dying");	}} /** * create_worker - create a new workqueue worker * @pool: pool the new worker will belong to * * Create and start a new worker which is attached to @pool. * * CONTEXT: * Might sleep.  Does GFP_KERNEL allocations. * * Return: * Pointer to the newly created worker. */static struct worker *create_worker(struct worker_pool *pool){	struct worker *worker;	int id; 	/* ID is needed to determine kthread name */	id = ida_alloc(&pool->worker_ida, GFP_KERNEL);	if (id < 0) {		pr_err_once("workqueue: Failed to allocate a worker ID: %pe\n",			    ERR_PTR(id));		return NULL;	} 	worker = alloc_worker(pool->node);	if (!worker) {		pr_err_once("workqueue: Failed to allocate a worker\n");		goto fail;	} 	worker->id = id; 	if (!(pool->flags & POOL_BH)) {		char id_buf[WORKER_ID_LEN]; 		format_worker_id(id_buf, sizeof(id_buf), worker, pool);		worker->task = kthread_create_on_node(worker_thread, worker,						      pool->node, "%s", id_buf);		if (IS_ERR(worker->task)) {			if (PTR_ERR(worker->task) == -EINTR) {				pr_err("workqueue: Interrupted when creating a worker thread \"%s\"\n",				       id_buf);			} else {				pr_err_once("workqueue: Failed to create a worker thread: %pe",					    worker->task);			}			goto fail;		} 		set_user_nice(worker->task, pool->attrs->nice);		kthread_bind_mask(worker->task, pool_allowed_cpus(pool));	} 	/* successful, attach the worker to the pool */	worker_attach_to_pool(worker, pool); 	/* start the newly created worker */	raw_spin_lock_irq(&pool->lock); 	worker->pool->nr_workers++;	worker_enter_idle(worker); 	/*	 * @worker is waiting on a completion in kthread() and will trigger hung	 * check if not woken up soon. As kick_pool() is noop if @pool is empty,	 * wake it up explicitly.	 */	if (worker->task)		wake_up_process(worker->task); 	raw_spin_unlock_irq(&pool->lock); 	return worker; fail:	ida_free(&pool->worker_ida, id);	kfree(worker);	return NULL;} static void detach_dying_workers(struct list_head *cull_list){	struct worker *worker; 	list_for_each_entry(worker, cull_list, entry)		detach_worker(worker);} static void reap_dying_workers(struct list_head *cull_list){	struct worker *worker, *tmp; 	list_for_each_entry_safe(worker, tmp, cull_list, entry) {		list_del_init(&worker->entry);		kthread_stop_put(worker->task);		kfree(worker);	}} /** * set_worker_dying - Tag a worker for destruction * @worker: worker to be destroyed * @list: transfer worker away from its pool->idle_list and into list * * Tag @worker for destruction and adjust @pool stats accordingly.  The worker * should be idle. * * CONTEXT: * raw_spin_lock_irq(pool->lock). */static void set_worker_dying(struct worker *worker, struct list_head *list){	struct worker_pool *pool = worker->pool; 	lockdep_assert_held(&pool->lock);	lockdep_assert_held(&wq_pool_attach_mutex); 	/* sanity check frenzy */	if (WARN_ON(worker->current_work) ||	    WARN_ON(!list_empty(&worker->scheduled)) ||	    WARN_ON(!(worker->flags & WORKER_IDLE)))		return; 	pool->nr_workers--;	pool->nr_idle--; 	worker->flags |= WORKER_DIE; 	list_move(&worker->entry, list); 	/* get an extra task struct reference for later kthread_stop_put() */	get_task_struct(worker->task);} /** * idle_worker_timeout - check if some idle workers can now be deleted. * @t: The pool's idle_timer that just expired * * The timer is armed in worker_enter_idle(). Note that it isn't disarmed in * worker_leave_idle(), as a worker flicking between idle and active while its * pool is at the too_many_workers() tipping point would cause too much timer * housekeeping overhead. Since IDLE_WORKER_TIMEOUT is long enough, we just let * it expire and re-evaluate things from there. */static void idle_worker_timeout(struct timer_list *t){	struct worker_pool *pool = timer_container_of(pool, t, idle_timer);	bool do_cull = false; 	if (work_pending(&pool->idle_cull_work))		return; 	raw_spin_lock_irq(&pool->lock); 	if (too_many_workers(pool)) {		struct worker *worker;		unsigned long expires; 		/* idle_list is kept in LIFO order, check the last one */		worker = list_last_entry(&pool->idle_list, struct worker, entry);		expires = worker->last_active + IDLE_WORKER_TIMEOUT;		do_cull = !time_before(jiffies, expires); 		if (!do_cull)			mod_timer(&pool->idle_timer, expires);	}	raw_spin_unlock_irq(&pool->lock); 	if (do_cull)		queue_work(system_dfl_wq, &pool->idle_cull_work);} /** * idle_cull_fn - cull workers that have been idle for too long. * @work: the pool's work for handling these idle workers *

Lines 2501–3000 of 8475. Previous lines Load more