.name		= "work_struct",	.debug_hint	= work_debug_hint,	.is_static_object = work_is_static_object,	.fixup_init	= work_fixup_init,	.fixup_free	= work_fixup_free,}; static inline void debug_work_activate(struct work_struct *work){	debug_object_activate(work, &work_debug_descr);} static inline void debug_work_deactivate(struct work_struct *work){	debug_object_deactivate(work, &work_debug_descr);} void __init_work(struct work_struct *work, int onstack){	if (onstack)		debug_object_init_on_stack(work, &work_debug_descr);	else		debug_object_init(work, &work_debug_descr);}EXPORT_SYMBOL_GPL(__init_work); void destroy_work_on_stack(struct work_struct *work){	debug_object_free(work, &work_debug_descr);}EXPORT_SYMBOL_GPL(destroy_work_on_stack); void destroy_delayed_work_on_stack(struct delayed_work *work){	destroy_timer_on_stack(&work->timer);	debug_object_free(&work->work, &work_debug_descr);}EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack); #elsestatic inline void debug_work_activate(struct work_struct *work) { }static inline void debug_work_deactivate(struct work_struct *work) { }#endif /** * worker_pool_assign_id - allocate ID and assign it to @pool * @pool: the pool pointer of interest * * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned * successfully, -errno on failure. */static int worker_pool_assign_id(struct worker_pool *pool){	int ret; 	lockdep_assert_held(&wq_pool_mutex); 	ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,			GFP_KERNEL);	if (ret >= 0) {		pool->id = ret;		return 0;	}	return ret;} /** * unbound_pwq_by_node - return the unbound pool_workqueue for the given node * @wq: the target workqueue * @node: the node ID * * This must be called with any of wq_pool_mutex, wq->mutex or RCU * read locked. * If the pwq needs to be used beyond the locking in effect, the caller is * responsible for guaranteeing that the pwq stays online. * * Return: The unbound pool_workqueue for @node. */static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,						  int node){	assert_rcu_or_wq_mutex_or_pool_mutex(wq); 	/*	 * XXX: @node can be NUMA_NO_NODE if CPU goes offline while a	 * delayed item is pending.  The plan is to keep CPU -> NODE	 * mapping valid and stable across CPU on/offlines.  Once that	 * happens, this workaround can be removed.	 */	if (unlikely(node == NUMA_NO_NODE))		return wq->dfl_pwq; 	return rcu_dereference_raw(wq->numa_pwq_tbl[node]);} static unsigned int work_color_to_flags(int color){	return color << WORK_STRUCT_COLOR_SHIFT;} static int get_work_color(unsigned long work_data){	return (work_data >> WORK_STRUCT_COLOR_SHIFT) &		((1 << WORK_STRUCT_COLOR_BITS) - 1);} static int work_next_color(int color){	return (color + 1) % WORK_NR_COLORS;} /* * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data * contain the pointer to the queued pwq.  Once execution starts, the flag * is cleared and the high bits contain OFFQ flags and pool ID. * * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling() * and clear_work_data() can be used to set the pwq, pool or clear * work->data.  These functions should only be called while the work is * owned - ie. while the PENDING bit is set. * * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq * corresponding to a work.  Pool is available once the work has been * queued anywhere after initialization until it is sync canceled.  pwq is * available only while the work item is queued. * * %WORK_OFFQ_CANCELING is used to mark a work item which is being * canceled.  While being canceled, a work item may have its PENDING set * but stay off timer and worklist for arbitrarily long and nobody should * try to steal the PENDING bit. */static inline void set_work_data(struct work_struct *work, unsigned long data,				 unsigned long flags){	WARN_ON_ONCE(!work_pending(work));	atomic_long_set(&work->data, data | flags | work_static(work));} static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,			 unsigned long extra_flags){	set_work_data(work, (unsigned long)pwq,		      WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);} static void set_work_pool_and_keep_pending(struct work_struct *work,					   int pool_id){	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,		      WORK_STRUCT_PENDING);} static void set_work_pool_and_clear_pending(struct work_struct *work,					    int pool_id){	/*	 * The following wmb is paired with the implied mb in	 * test_and_set_bit(PENDING) and ensures all updates to @work made	 * here are visible to and precede any updates by the next PENDING	 * owner.	 */	smp_wmb();	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);	/*	 * The following mb guarantees that previous clear of a PENDING bit	 * will not be reordered with any speculative LOADS or STORES from	 * work->current_func, which is executed afterwards.  This possible	 * reordering can lead to a missed execution on attempt to queue	 * the same @work.  E.g. consider this case:	 *	 *   CPU#0                         CPU#1	 *   ----------------------------  --------------------------------	 *	 * 1  STORE event_indicated	 * 2  queue_work_on() {	 * 3    test_and_set_bit(PENDING)	 * 4 }                             set_..._and_clear_pending() {	 * 5                                 set_work_data() # clear bit	 * 6                                 smp_mb()	 * 7                               work->current_func() {	 * 8				      LOAD event_indicated	 *				   }	 *	 * Without an explicit full barrier speculative LOAD on line 8 can	 * be executed before CPU#0 does STORE on line 1.  If that happens,	 * CPU#0 observes the PENDING bit is still set and new execution of	 * a @work is not queued in a hope, that CPU#1 will eventually	 * finish the queued @work.  Meanwhile CPU#1 does not see	 * event_indicated is set, because speculative LOAD was executed	 * before actual STORE.	 */	smp_mb();} static void clear_work_data(struct work_struct *work){	smp_wmb();	/* see set_work_pool_and_clear_pending() */	set_work_data(work, WORK_STRUCT_NO_POOL, 0);} static struct pool_workqueue *get_work_pwq(struct work_struct *work){	unsigned long data = atomic_long_read(&work->data); 	if (data & WORK_STRUCT_PWQ)		return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);	else		return NULL;} /** * get_work_pool - return the worker_pool a given work was associated with * @work: the work item of interest * * Pools are created and destroyed under wq_pool_mutex, and allows read * access under RCU read lock.  As such, this function should be * called under wq_pool_mutex or inside of a rcu_read_lock() region. * * All fields of the returned pool are accessible as long as the above * mentioned locking is in effect.  If the returned pool needs to be used * beyond the critical section, the caller is responsible for ensuring the * returned pool is and stays online. * * Return: The worker_pool @work was last associated with.  %NULL if none. */static struct worker_pool *get_work_pool(struct work_struct *work){	unsigned long data = atomic_long_read(&work->data);	int pool_id; 	assert_rcu_or_pool_mutex(); 	if (data & WORK_STRUCT_PWQ)		return ((struct pool_workqueue *)			(data & WORK_STRUCT_WQ_DATA_MASK))->pool; 	pool_id = data >> WORK_OFFQ_POOL_SHIFT;	if (pool_id == WORK_OFFQ_POOL_NONE)		return NULL; 	return idr_find(&worker_pool_idr, pool_id);} /** * get_work_pool_id - return the worker pool ID a given work is associated with * @work: the work item of interest * * Return: The worker_pool ID @work was last associated with. * %WORK_OFFQ_POOL_NONE if none. */static int get_work_pool_id(struct work_struct *work){	unsigned long data = atomic_long_read(&work->data); 	if (data & WORK_STRUCT_PWQ)		return ((struct pool_workqueue *)			(data & WORK_STRUCT_WQ_DATA_MASK))->pool->id; 	return data >> WORK_OFFQ_POOL_SHIFT;} static void mark_work_canceling(struct work_struct *work){	unsigned long pool_id = get_work_pool_id(work); 	pool_id <<= WORK_OFFQ_POOL_SHIFT;	set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);} static bool work_is_canceling(struct work_struct *work){	unsigned long data = atomic_long_read(&work->data); 	return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);} /* * Policy functions.  These define the policies on how the global worker * pools are managed.  Unless noted otherwise, these functions assume that * they're being called with pool->lock held. */ static bool __need_more_worker(struct worker_pool *pool){	return !pool->nr_running;} /* * Need to wake up a worker?  Called from anything but currently * running workers. * * Note that, because unbound workers never contribute to nr_running, this * function will always return %true for unbound pools as long as the * worklist isn't empty. */static bool need_more_worker(struct worker_pool *pool){	return !list_empty(&pool->worklist) && __need_more_worker(pool);} /* Can I start working?  Called from busy but !running workers. */static bool may_start_working(struct worker_pool *pool){	return pool->nr_idle;} /* Do I need to keep working?  Called from currently running workers. */static bool keep_working(struct worker_pool *pool){	return !list_empty(&pool->worklist) && (pool->nr_running <= 1);} /* Do we need a new worker?  Called from manager. */static bool need_to_create_worker(struct worker_pool *pool){	return need_more_worker(pool) && !may_start_working(pool);} /* Do we have too many workers and should some go away? */static bool too_many_workers(struct worker_pool *pool){	bool managing = pool->flags & POOL_MANAGER_ACTIVE;	int nr_idle = pool->nr_idle + managing; /* manager is considered idle */	int nr_busy = pool->nr_workers - nr_idle; 	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;} /* * Wake up functions. */ /* Return the first idle worker.  Called with pool->lock held. */static struct worker *first_idle_worker(struct worker_pool *pool){	if (unlikely(list_empty(&pool->idle_list)))		return NULL; 	return list_first_entry(&pool->idle_list, struct worker, entry);} /** * wake_up_worker - wake up an idle worker * @pool: worker pool to wake worker from * * Wake up the first idle worker of @pool. * * CONTEXT: * raw_spin_lock_irq(pool->lock). */static void wake_up_worker(struct worker_pool *pool){	struct worker *worker = first_idle_worker(pool); 	if (likely(worker))		wake_up_process(worker->task);} /** * wq_worker_running - a worker is running again * @task: task waking up * * This function is called when a worker returns from schedule() */void wq_worker_running(struct task_struct *task){	struct worker *worker = kthread_data(task); 	if (!worker->sleeping)		return; 	/*	 * If preempted by unbind_workers() between the WORKER_NOT_RUNNING check	 * and the nr_running increment below, we may ruin the nr_running reset	 * and leave with an unexpected pool->nr_running == 1 on the newly unbound	 * pool. Protect against such race.	 */	preempt_disable();	if (!(worker->flags & WORKER_NOT_RUNNING))		worker->pool->nr_running++;	preempt_enable();	worker->sleeping = 0;} /** * wq_worker_sleeping - a worker is going to sleep * @task: task going to sleep * * This function is called from schedule() when a busy worker is * going to sleep. */void wq_worker_sleeping(struct task_struct *task){	struct worker *worker = kthread_data(task);	struct worker_pool *pool; 	/*	 * Rescuers, which may not have all the fields set up like normal	 * workers, also reach here, let's not access anything before	 * checking NOT_RUNNING.	 */	if (worker->flags & WORKER_NOT_RUNNING)		return; 	pool = worker->pool; 	/* Return if preempted before wq_worker_running() was reached */	if (worker->sleeping)		return; 	worker->sleeping = 1;	raw_spin_lock_irq(&pool->lock); 	/*	 * Recheck in case unbind_workers() preempted us. We don't	 * want to decrement nr_running after the worker is unbound	 * and nr_running has been reset.	 */	if (worker->flags & WORKER_NOT_RUNNING) {		raw_spin_unlock_irq(&pool->lock);		return;	} 	pool->nr_running--;	if (need_more_worker(pool))		wake_up_worker(pool);	raw_spin_unlock_irq(&pool->lock);} /** * wq_worker_last_func - retrieve worker's last work function * @task: Task to retrieve last work function of. * * Determine the last function a worker executed. This is called from * the scheduler to get a worker's last known identity. * * CONTEXT: * raw_spin_lock_irq(rq->lock) * * This function is called during schedule() when a kworker is going * to sleep. It's used by psi to identify aggregation workers during * dequeuing, to allow periodic aggregation to shut-off when that * worker is the last task in the system or cgroup to go to sleep. * * As this function doesn't involve any workqueue-related locking, it * only returns stable values when called from inside the scheduler's * queuing and dequeuing paths, when @task, which must be a kworker, * is guaranteed to not be processing any works. * * Return: * The last work function %current executed as a worker, NULL if it * hasn't executed any work yet. */work_func_t wq_worker_last_func(struct task_struct *task){	struct worker *worker = kthread_data(task); 	return worker->last_func;} /** * worker_set_flags - set worker flags and adjust nr_running accordingly * @worker: self * @flags: flags to set * * Set @flags in @worker->flags and adjust nr_running accordingly. * * CONTEXT: * raw_spin_lock_irq(pool->lock) */static inline void worker_set_flags(struct worker *worker, unsigned int flags){	struct worker_pool *pool = worker->pool; 	WARN_ON_ONCE(worker->task != current); 	/* If transitioning into NOT_RUNNING, adjust nr_running. */	if ((flags & WORKER_NOT_RUNNING) &&	    !(worker->flags & WORKER_NOT_RUNNING)) {		pool->nr_running--;	} 	worker->flags |= flags;} /** * worker_clr_flags - clear worker flags and adjust nr_running accordingly * @worker: self * @flags: flags to clear * * Clear @flags in @worker->flags and adjust nr_running accordingly. * * CONTEXT: * raw_spin_lock_irq(pool->lock) */static inline void worker_clr_flags(struct worker *worker, unsigned int flags){	struct worker_pool *pool = worker->pool;	unsigned int oflags = worker->flags; 

Lines 501–1000 of 6121. Previous lines Load more