debug_object_init_on_stack(work, &work_debug_descr);	else		debug_object_init(work, &work_debug_descr);}EXPORT_SYMBOL_GPL(__init_work); void destroy_work_on_stack(struct work_struct *work){	debug_object_free(work, &work_debug_descr);}EXPORT_SYMBOL_GPL(destroy_work_on_stack); void destroy_delayed_work_on_stack(struct delayed_work *work){	destroy_timer_on_stack(&work->timer);	debug_object_free(&work->work, &work_debug_descr);}EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack); #elsestatic inline void debug_work_activate(struct work_struct *work) { }static inline void debug_work_deactivate(struct work_struct *work) { }#endif /** * worker_pool_assign_id - allocate ID and assing it to @pool * @pool: the pool pointer of interest * * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned * successfully, -errno on failure. */static int worker_pool_assign_id(struct worker_pool *pool){	int ret; 	lockdep_assert_held(&wq_pool_mutex); 	ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,			GFP_KERNEL);	if (ret >= 0) {		pool->id = ret;		return 0;	}	return ret;} /** * unbound_pwq_by_node - return the unbound pool_workqueue for the given node * @wq: the target workqueue * @node: the node ID * * This must be called with any of wq_pool_mutex, wq->mutex or RCU * read locked. * If the pwq needs to be used beyond the locking in effect, the caller is * responsible for guaranteeing that the pwq stays online. * * Return: The unbound pool_workqueue for @node. */static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,						  int node){	assert_rcu_or_wq_mutex_or_pool_mutex(wq); 	/*	 * XXX: @node can be NUMA_NO_NODE if CPU goes offline while a	 * delayed item is pending.  The plan is to keep CPU -> NODE	 * mapping valid and stable across CPU on/offlines.  Once that	 * happens, this workaround can be removed.	 */	if (unlikely(node == NUMA_NO_NODE))		return wq->dfl_pwq; 	return rcu_dereference_raw(wq->numa_pwq_tbl[node]);} static unsigned int work_color_to_flags(int color){	return color << WORK_STRUCT_COLOR_SHIFT;} static int get_work_color(struct work_struct *work){	return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &		((1 << WORK_STRUCT_COLOR_BITS) - 1);} static int work_next_color(int color){	return (color + 1) % WORK_NR_COLORS;} /* * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data * contain the pointer to the queued pwq.  Once execution starts, the flag * is cleared and the high bits contain OFFQ flags and pool ID. * * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling() * and clear_work_data() can be used to set the pwq, pool or clear * work->data.  These functions should only be called while the work is * owned - ie. while the PENDING bit is set. * * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq * corresponding to a work.  Pool is available once the work has been * queued anywhere after initialization until it is sync canceled.  pwq is * available only while the work item is queued. * * %WORK_OFFQ_CANCELING is used to mark a work item which is being * canceled.  While being canceled, a work item may have its PENDING set * but stay off timer and worklist for arbitrarily long and nobody should * try to steal the PENDING bit. */static inline void set_work_data(struct work_struct *work, unsigned long data,				 unsigned long flags){	WARN_ON_ONCE(!work_pending(work));	atomic_long_set(&work->data, data | flags | work_static(work));} static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,			 unsigned long extra_flags){	set_work_data(work, (unsigned long)pwq,		      WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);} static void set_work_pool_and_keep_pending(struct work_struct *work,					   int pool_id){	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,		      WORK_STRUCT_PENDING);} static void set_work_pool_and_clear_pending(struct work_struct *work,					    int pool_id){	/*	 * The following wmb is paired with the implied mb in	 * test_and_set_bit(PENDING) and ensures all updates to @work made	 * here are visible to and precede any updates by the next PENDING	 * owner.	 */	smp_wmb();	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);	/*	 * The following mb guarantees that previous clear of a PENDING bit	 * will not be reordered with any speculative LOADS or STORES from	 * work->current_func, which is executed afterwards.  This possible	 * reordering can lead to a missed execution on attempt to queue	 * the same @work.  E.g. consider this case:	 *	 *   CPU#0                         CPU#1	 *   ----------------------------  --------------------------------	 *	 * 1  STORE event_indicated	 * 2  queue_work_on() {	 * 3    test_and_set_bit(PENDING)	 * 4 }                             set_..._and_clear_pending() {	 * 5                                 set_work_data() # clear bit	 * 6                                 smp_mb()	 * 7                               work->current_func() {	 * 8				      LOAD event_indicated	 *				   }	 *	 * Without an explicit full barrier speculative LOAD on line 8 can	 * be executed before CPU#0 does STORE on line 1.  If that happens,	 * CPU#0 observes the PENDING bit is still set and new execution of	 * a @work is not queued in a hope, that CPU#1 will eventually	 * finish the queued @work.  Meanwhile CPU#1 does not see	 * event_indicated is set, because speculative LOAD was executed	 * before actual STORE.	 */	smp_mb();} static void clear_work_data(struct work_struct *work){	smp_wmb();	/* see set_work_pool_and_clear_pending() */	set_work_data(work, WORK_STRUCT_NO_POOL, 0);} static struct pool_workqueue *get_work_pwq(struct work_struct *work){	unsigned long data = atomic_long_read(&work->data); 	if (data & WORK_STRUCT_PWQ)		return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);	else		return NULL;} /** * get_work_pool - return the worker_pool a given work was associated with * @work: the work item of interest * * Pools are created and destroyed under wq_pool_mutex, and allows read * access under RCU read lock.  As such, this function should be * called under wq_pool_mutex or inside of a rcu_read_lock() region. * * All fields of the returned pool are accessible as long as the above * mentioned locking is in effect.  If the returned pool needs to be used * beyond the critical section, the caller is responsible for ensuring the * returned pool is and stays online. * * Return: The worker_pool @work was last associated with.  %NULL if none. */static struct worker_pool *get_work_pool(struct work_struct *work){	unsigned long data = atomic_long_read(&work->data);	int pool_id; 	assert_rcu_or_pool_mutex(); 	if (data & WORK_STRUCT_PWQ)		return ((struct pool_workqueue *)			(data & WORK_STRUCT_WQ_DATA_MASK))->pool; 	pool_id = data >> WORK_OFFQ_POOL_SHIFT;	if (pool_id == WORK_OFFQ_POOL_NONE)		return NULL; 	return idr_find(&worker_pool_idr, pool_id);} /** * get_work_pool_id - return the worker pool ID a given work is associated with * @work: the work item of interest * * Return: The worker_pool ID @work was last associated with. * %WORK_OFFQ_POOL_NONE if none. */static int get_work_pool_id(struct work_struct *work){	unsigned long data = atomic_long_read(&work->data); 	if (data & WORK_STRUCT_PWQ)		return ((struct pool_workqueue *)			(data & WORK_STRUCT_WQ_DATA_MASK))->pool->id; 	return data >> WORK_OFFQ_POOL_SHIFT;} static void mark_work_canceling(struct work_struct *work){	unsigned long pool_id = get_work_pool_id(work); 	pool_id <<= WORK_OFFQ_POOL_SHIFT;	set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);} static bool work_is_canceling(struct work_struct *work){	unsigned long data = atomic_long_read(&work->data); 	return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);} /* * Policy functions.  These define the policies on how the global worker * pools are managed.  Unless noted otherwise, these functions assume that * they're being called with pool->lock held. */ static bool __need_more_worker(struct worker_pool *pool){	return !atomic_read(&pool->nr_running);} /* * Need to wake up a worker?  Called from anything but currently * running workers. * * Note that, because unbound workers never contribute to nr_running, this * function will always return %true for unbound pools as long as the * worklist isn't empty. */static bool need_more_worker(struct worker_pool *pool){	return !list_empty(&pool->worklist) && __need_more_worker(pool);} /* Can I start working?  Called from busy but !running workers. */static bool may_start_working(struct worker_pool *pool){	return pool->nr_idle;} /* Do I need to keep working?  Called from currently running workers. */static bool keep_working(struct worker_pool *pool){	return !list_empty(&pool->worklist) &&		atomic_read(&pool->nr_running) <= 1;} /* Do we need a new worker?  Called from manager. */static bool need_to_create_worker(struct worker_pool *pool){	return need_more_worker(pool) && !may_start_working(pool);} /* Do we have too many workers and should some go away? */static bool too_many_workers(struct worker_pool *pool){	bool managing = pool->flags & POOL_MANAGER_ACTIVE;	int nr_idle = pool->nr_idle + managing; /* manager is considered idle */	int nr_busy = pool->nr_workers - nr_idle; 	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;} /* * Wake up functions. */ /* Return the first idle worker.  Safe with preemption disabled */static struct worker *first_idle_worker(struct worker_pool *pool){	if (unlikely(list_empty(&pool->idle_list)))		return NULL; 	return list_first_entry(&pool->idle_list, struct worker, entry);} /** * wake_up_worker - wake up an idle worker * @pool: worker pool to wake worker from * * Wake up the first idle worker of @pool. * * CONTEXT: * raw_spin_lock_irq(pool->lock). */static void wake_up_worker(struct worker_pool *pool){	struct worker *worker = first_idle_worker(pool); 	if (likely(worker))		wake_up_process(worker->task);} /** * wq_worker_running - a worker is running again * @task: task waking up * * This function is called when a worker returns from schedule() */void wq_worker_running(struct task_struct *task){	struct worker *worker = kthread_data(task); 	if (!worker->sleeping)		return;	if (!(worker->flags & WORKER_NOT_RUNNING))		atomic_inc(&worker->pool->nr_running);	worker->sleeping = 0;} /** * wq_worker_sleeping - a worker is going to sleep * @task: task going to sleep * * This function is called from schedule() when a busy worker is * going to sleep. Preemption needs to be disabled to protect ->sleeping * assignment. */void wq_worker_sleeping(struct task_struct *task){	struct worker *next, *worker = kthread_data(task);	struct worker_pool *pool; 	/*	 * Rescuers, which may not have all the fields set up like normal	 * workers, also reach here, let's not access anything before	 * checking NOT_RUNNING.	 */	if (worker->flags & WORKER_NOT_RUNNING)		return; 	pool = worker->pool; 	/* Return if preempted before wq_worker_running() was reached */	if (worker->sleeping)		return; 	worker->sleeping = 1;	raw_spin_lock_irq(&pool->lock); 	/*	 * The counterpart of the following dec_and_test, implied mb,	 * worklist not empty test sequence is in insert_work().	 * Please read comment there.	 *	 * NOT_RUNNING is clear.  This means that we're bound to and	 * running on the local cpu w/ rq lock held and preemption	 * disabled, which in turn means that none else could be	 * manipulating idle_list, so dereferencing idle_list without pool	 * lock is safe.	 */	if (atomic_dec_and_test(&pool->nr_running) &&	    !list_empty(&pool->worklist)) {		next = first_idle_worker(pool);		if (next)			wake_up_process(next->task);	}	raw_spin_unlock_irq(&pool->lock);} /** * wq_worker_last_func - retrieve worker's last work function * @task: Task to retrieve last work function of. * * Determine the last function a worker executed. This is called from * the scheduler to get a worker's last known identity. * * CONTEXT: * raw_spin_lock_irq(rq->lock) * * This function is called during schedule() when a kworker is going * to sleep. It's used by psi to identify aggregation workers during * dequeuing, to allow periodic aggregation to shut-off when that * worker is the last task in the system or cgroup to go to sleep. * * As this function doesn't involve any workqueue-related locking, it * only returns stable values when called from inside the scheduler's * queuing and dequeuing paths, when @task, which must be a kworker, * is guaranteed to not be processing any works. * * Return: * The last work function %current executed as a worker, NULL if it * hasn't executed any work yet. */work_func_t wq_worker_last_func(struct task_struct *task){	struct worker *worker = kthread_data(task); 	return worker->last_func;} /** * worker_set_flags - set worker flags and adjust nr_running accordingly * @worker: self * @flags: flags to set * * Set @flags in @worker->flags and adjust nr_running accordingly. * * CONTEXT: * raw_spin_lock_irq(pool->lock) */static inline void worker_set_flags(struct worker *worker, unsigned int flags){	struct worker_pool *pool = worker->pool; 	WARN_ON_ONCE(worker->task != current); 	/* If transitioning into NOT_RUNNING, adjust nr_running. */	if ((flags & WORKER_NOT_RUNNING) &&	    !(worker->flags & WORKER_NOT_RUNNING)) {		atomic_dec(&pool->nr_running);	} 	worker->flags |= flags;} /** * worker_clr_flags - clear worker flags and adjust nr_running accordingly * @worker: self * @flags: flags to clear * * Clear @flags in @worker->flags and adjust nr_running accordingly. * * CONTEXT: * raw_spin_lock_irq(pool->lock) */static inline void worker_clr_flags(struct worker *worker, unsigned int flags){	struct worker_pool *pool = worker->pool;	unsigned int oflags = worker->flags; 	WARN_ON_ONCE(worker->task != current); 	worker->flags &= ~flags; 	/*	 * If transitioning out of NOT_RUNNING, increment nr_running.  Note	 * that the nested NOT_RUNNING is not a noop.  NOT_RUNNING is mask	 * of multiple flags, not a single flag.	 */	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))		if (!(worker->flags & WORKER_NOT_RUNNING))			atomic_inc(&pool->nr_running);} /** * find_worker_executing_work - find worker which is executing a work * @pool: pool of interest * @work: work to find worker for * * Find a worker which is executing @work on @pool by searching * @pool->busy_hash which is keyed by the address of @work.  For a worker * to match, its current execution should match the address of @work and * its work function.  This is to avoid unwanted dependency between

Lines 501–1000 of 6039. Previous lines Load more