{	debug_object_free(work, &work_debug_descr);}EXPORT_SYMBOL_GPL(destroy_work_on_stack); void destroy_delayed_work_on_stack(struct delayed_work *work){	destroy_timer_on_stack(&work->timer);	debug_object_free(&work->work, &work_debug_descr);}EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack); #elsestatic inline void debug_work_activate(struct work_struct *work) { }static inline void debug_work_deactivate(struct work_struct *work) { }#endif /** * worker_pool_assign_id - allocate ID and assing it to @pool * @pool: the pool pointer of interest * * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned * successfully, -errno on failure. */static int worker_pool_assign_id(struct worker_pool *pool){	int ret; 	lockdep_assert_held(&wq_pool_mutex); 	ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,			GFP_KERNEL);	if (ret >= 0) {		pool->id = ret;		return 0;	}	return ret;} /** * unbound_pwq_by_node - return the unbound pool_workqueue for the given node * @wq: the target workqueue * @node: the node ID * * This must be called either with pwq_lock held or sched RCU read locked. * If the pwq needs to be used beyond the locking in effect, the caller is * responsible for guaranteeing that the pwq stays online. * * Return: The unbound pool_workqueue for @node. */static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,						  int node){	assert_rcu_or_wq_mutex(wq);	return rcu_dereference_raw(wq->numa_pwq_tbl[node]);} static unsigned int work_color_to_flags(int color){	return color << WORK_STRUCT_COLOR_SHIFT;} static int get_work_color(struct work_struct *work){	return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &		((1 << WORK_STRUCT_COLOR_BITS) - 1);} static int work_next_color(int color){	return (color + 1) % WORK_NR_COLORS;} /* * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data * contain the pointer to the queued pwq.  Once execution starts, the flag * is cleared and the high bits contain OFFQ flags and pool ID. * * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling() * and clear_work_data() can be used to set the pwq, pool or clear * work->data.  These functions should only be called while the work is * owned - ie. while the PENDING bit is set. * * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq * corresponding to a work.  Pool is available once the work has been * queued anywhere after initialization until it is sync canceled.  pwq is * available only while the work item is queued. * * %WORK_OFFQ_CANCELING is used to mark a work item which is being * canceled.  While being canceled, a work item may have its PENDING set * but stay off timer and worklist for arbitrarily long and nobody should * try to steal the PENDING bit. */static inline void set_work_data(struct work_struct *work, unsigned long data,				 unsigned long flags){	WARN_ON_ONCE(!work_pending(work));	atomic_long_set(&work->data, data | flags | work_static(work));} static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,			 unsigned long extra_flags){	set_work_data(work, (unsigned long)pwq,		      WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);} static void set_work_pool_and_keep_pending(struct work_struct *work,					   int pool_id){	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,		      WORK_STRUCT_PENDING);} static void set_work_pool_and_clear_pending(struct work_struct *work,					    int pool_id){	/*	 * The following wmb is paired with the implied mb in	 * test_and_set_bit(PENDING) and ensures all updates to @work made	 * here are visible to and precede any updates by the next PENDING	 * owner.	 */	smp_wmb();	set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);} static void clear_work_data(struct work_struct *work){	smp_wmb();	/* see set_work_pool_and_clear_pending() */	set_work_data(work, WORK_STRUCT_NO_POOL, 0);} static struct pool_workqueue *get_work_pwq(struct work_struct *work){	unsigned long data = atomic_long_read(&work->data); 	if (data & WORK_STRUCT_PWQ)		return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);	else		return NULL;} /** * get_work_pool - return the worker_pool a given work was associated with * @work: the work item of interest * * Pools are created and destroyed under wq_pool_mutex, and allows read * access under sched-RCU read lock.  As such, this function should be * called under wq_pool_mutex or with preemption disabled. * * All fields of the returned pool are accessible as long as the above * mentioned locking is in effect.  If the returned pool needs to be used * beyond the critical section, the caller is responsible for ensuring the * returned pool is and stays online. * * Return: The worker_pool @work was last associated with.  %NULL if none. */static struct worker_pool *get_work_pool(struct work_struct *work){	unsigned long data = atomic_long_read(&work->data);	int pool_id; 	assert_rcu_or_pool_mutex(); 	if (data & WORK_STRUCT_PWQ)		return ((struct pool_workqueue *)			(data & WORK_STRUCT_WQ_DATA_MASK))->pool; 	pool_id = data >> WORK_OFFQ_POOL_SHIFT;	if (pool_id == WORK_OFFQ_POOL_NONE)		return NULL; 	return idr_find(&worker_pool_idr, pool_id);} /** * get_work_pool_id - return the worker pool ID a given work is associated with * @work: the work item of interest * * Return: The worker_pool ID @work was last associated with. * %WORK_OFFQ_POOL_NONE if none. */static int get_work_pool_id(struct work_struct *work){	unsigned long data = atomic_long_read(&work->data); 	if (data & WORK_STRUCT_PWQ)		return ((struct pool_workqueue *)			(data & WORK_STRUCT_WQ_DATA_MASK))->pool->id; 	return data >> WORK_OFFQ_POOL_SHIFT;} static void mark_work_canceling(struct work_struct *work){	unsigned long pool_id = get_work_pool_id(work); 	pool_id <<= WORK_OFFQ_POOL_SHIFT;	set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);} static bool work_is_canceling(struct work_struct *work){	unsigned long data = atomic_long_read(&work->data); 	return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);} /* * Policy functions.  These define the policies on how the global worker * pools are managed.  Unless noted otherwise, these functions assume that * they're being called with pool->lock held. */ static bool __need_more_worker(struct worker_pool *pool){	return !atomic_read(&pool->nr_running);} /* * Need to wake up a worker?  Called from anything but currently * running workers. * * Note that, because unbound workers never contribute to nr_running, this * function will always return %true for unbound pools as long as the * worklist isn't empty. */static bool need_more_worker(struct worker_pool *pool){	return !list_empty(&pool->worklist) && __need_more_worker(pool);} /* Can I start working?  Called from busy but !running workers. */static bool may_start_working(struct worker_pool *pool){	return pool->nr_idle;} /* Do I need to keep working?  Called from currently running workers. */static bool keep_working(struct worker_pool *pool){	return !list_empty(&pool->worklist) &&		atomic_read(&pool->nr_running) <= 1;} /* Do we need a new worker?  Called from manager. */static bool need_to_create_worker(struct worker_pool *pool){	return need_more_worker(pool) && !may_start_working(pool);} /* Do we have too many workers and should some go away? */static bool too_many_workers(struct worker_pool *pool){	bool managing = mutex_is_locked(&pool->manager_arb);	int nr_idle = pool->nr_idle + managing; /* manager is considered idle */	int nr_busy = pool->nr_workers - nr_idle; 	return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;} /* * Wake up functions. */ /* Return the first idle worker.  Safe with preemption disabled */static struct worker *first_idle_worker(struct worker_pool *pool){	if (unlikely(list_empty(&pool->idle_list)))		return NULL; 	return list_first_entry(&pool->idle_list, struct worker, entry);} /** * wake_up_worker - wake up an idle worker * @pool: worker pool to wake worker from * * Wake up the first idle worker of @pool. * * CONTEXT: * spin_lock_irq(pool->lock). */static void wake_up_worker(struct worker_pool *pool){	struct worker *worker = first_idle_worker(pool); 	if (likely(worker))		wake_up_process(worker->task);} /** * wq_worker_waking_up - a worker is waking up * @task: task waking up * @cpu: CPU @task is waking up to * * This function is called during try_to_wake_up() when a worker is * being awoken. * * CONTEXT: * spin_lock_irq(rq->lock) */void wq_worker_waking_up(struct task_struct *task, int cpu){	struct worker *worker = kthread_data(task); 	if (!(worker->flags & WORKER_NOT_RUNNING)) {		WARN_ON_ONCE(worker->pool->cpu != cpu);		atomic_inc(&worker->pool->nr_running);	}} /** * wq_worker_sleeping - a worker is going to sleep * @task: task going to sleep * @cpu: CPU in question, must be the current CPU number * * This function is called during schedule() when a busy worker is * going to sleep.  Worker on the same cpu can be woken up by * returning pointer to its task. * * CONTEXT: * spin_lock_irq(rq->lock) * * Return: * Worker task on @cpu to wake up, %NULL if none. */struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu){	struct worker *worker = kthread_data(task), *to_wakeup = NULL;	struct worker_pool *pool; 	/*	 * Rescuers, which may not have all the fields set up like normal	 * workers, also reach here, let's not access anything before	 * checking NOT_RUNNING.	 */	if (worker->flags & WORKER_NOT_RUNNING)		return NULL; 	pool = worker->pool; 	/* this can only happen on the local cpu */	if (WARN_ON_ONCE(cpu != raw_smp_processor_id() || pool->cpu != cpu))		return NULL; 	/*	 * The counterpart of the following dec_and_test, implied mb,	 * worklist not empty test sequence is in insert_work().	 * Please read comment there.	 *	 * NOT_RUNNING is clear.  This means that we're bound to and	 * running on the local cpu w/ rq lock held and preemption	 * disabled, which in turn means that none else could be	 * manipulating idle_list, so dereferencing idle_list without pool	 * lock is safe.	 */	if (atomic_dec_and_test(&pool->nr_running) &&	    !list_empty(&pool->worklist))		to_wakeup = first_idle_worker(pool);	return to_wakeup ? to_wakeup->task : NULL;} /** * worker_set_flags - set worker flags and adjust nr_running accordingly * @worker: self * @flags: flags to set * * Set @flags in @worker->flags and adjust nr_running accordingly. * * CONTEXT: * spin_lock_irq(pool->lock) */static inline void worker_set_flags(struct worker *worker, unsigned int flags){	struct worker_pool *pool = worker->pool; 	WARN_ON_ONCE(worker->task != current); 	/* If transitioning into NOT_RUNNING, adjust nr_running. */	if ((flags & WORKER_NOT_RUNNING) &&	    !(worker->flags & WORKER_NOT_RUNNING)) {		atomic_dec(&pool->nr_running);	} 	worker->flags |= flags;} /** * worker_clr_flags - clear worker flags and adjust nr_running accordingly * @worker: self * @flags: flags to clear * * Clear @flags in @worker->flags and adjust nr_running accordingly. * * CONTEXT: * spin_lock_irq(pool->lock) */static inline void worker_clr_flags(struct worker *worker, unsigned int flags){	struct worker_pool *pool = worker->pool;	unsigned int oflags = worker->flags; 	WARN_ON_ONCE(worker->task != current); 	worker->flags &= ~flags; 	/*	 * If transitioning out of NOT_RUNNING, increment nr_running.  Note	 * that the nested NOT_RUNNING is not a noop.  NOT_RUNNING is mask	 * of multiple flags, not a single flag.	 */	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))		if (!(worker->flags & WORKER_NOT_RUNNING))			atomic_inc(&pool->nr_running);} /** * find_worker_executing_work - find worker which is executing a work * @pool: pool of interest * @work: work to find worker for * * Find a worker which is executing @work on @pool by searching * @pool->busy_hash which is keyed by the address of @work.  For a worker * to match, its current execution should match the address of @work and * its work function.  This is to avoid unwanted dependency between * unrelated work executions through a work item being recycled while still * being executed. * * This is a bit tricky.  A work item may be freed once its execution * starts and nothing prevents the freed area from being recycled for * another work item.  If the same work item address ends up being reused * before the original execution finishes, workqueue will identify the * recycled work item as currently executing and make it wait until the * current execution finishes, introducing an unwanted dependency. * * This function checks the work item address and work function to avoid * false positives.  Note that this isn't complete as one may construct a * work function which can introduce dependency onto itself through a * recycled work item.  Well, if somebody wants to shoot oneself in the * foot that badly, there's only so much we can do, and if such deadlock * actually occurs, it should be easy to locate the culprit work function. * * CONTEXT: * spin_lock_irq(pool->lock). * * Return: * Pointer to worker which is executing @work if found, %NULL * otherwise. */static struct worker *find_worker_executing_work(struct worker_pool *pool,						 struct work_struct *work){	struct worker *worker; 	hash_for_each_possible(pool->busy_hash, worker, hentry,			       (unsigned long)work)		if (worker->current_work == work &&		    worker->current_func == work->func)			return worker; 	return NULL;} /** * move_linked_works - move linked works to a list * @work: start of series of works to be scheduled * @head: target list to append @work to * @nextp: out paramter for nested worklist walking * * Schedule linked works starting from @work to @head.  Work series to * be scheduled starts at @work and includes any consecutive work with * WORK_STRUCT_LINKED set in its predecessor. * * If @nextp is not NULL, it's updated to point to the next work of * the last scheduled work.  This allows move_linked_works() to be * nested inside outer list_for_each_entry_safe(). * * CONTEXT: * spin_lock_irq(pool->lock). */static void move_linked_works(struct work_struct *work, struct list_head *head,			      struct work_struct **nextp){	struct work_struct *n; 	/*	 * Linked worklist will always end before the end of the list,	 * use NULL for list head.	 */	list_for_each_entry_safe_from(work, n, NULL, entry) {		list_move_tail(&work->entry, head);		if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))			break;	} 	/*	 * If we're already inside safe list traversal and have moved	 * multiple works to the scheduled queue, the next position

Lines 501–1000 of 4902. Previous lines Load more