worker->flags |= flags;} /** * worker_clr_flags - clear worker flags and adjust nr_running accordingly * @worker: self * @flags: flags to clear * * Clear @flags in @worker->flags and adjust nr_running accordingly. */static inline void worker_clr_flags(struct worker *worker, unsigned int flags){	struct worker_pool *pool = worker->pool;	unsigned int oflags = worker->flags; 	lockdep_assert_held(&pool->lock); 	worker->flags &= ~flags; 	/*	 * If transitioning out of NOT_RUNNING, increment nr_running.  Note	 * that the nested NOT_RUNNING is not a noop.  NOT_RUNNING is mask	 * of multiple flags, not a single flag.	 */	if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))		if (!(worker->flags & WORKER_NOT_RUNNING))			pool->nr_running++;} /* Return the first idle worker.  Called with pool->lock held. */static struct worker *first_idle_worker(struct worker_pool *pool){	if (unlikely(list_empty(&pool->idle_list)))		return NULL; 	return list_first_entry(&pool->idle_list, struct worker, entry);} /** * worker_enter_idle - enter idle state * @worker: worker which is entering idle state * * @worker is entering idle state.  Update stats and idle timer if * necessary. * * LOCKING: * raw_spin_lock_irq(pool->lock). */static void worker_enter_idle(struct worker *worker){	struct worker_pool *pool = worker->pool; 	if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||	    WARN_ON_ONCE(!list_empty(&worker->entry) &&			 (worker->hentry.next || worker->hentry.pprev)))		return; 	/* can't use worker_set_flags(), also called from create_worker() */	worker->flags |= WORKER_IDLE;	pool->nr_idle++;	worker->last_active = jiffies; 	/* idle_list is LIFO */	list_add(&worker->entry, &pool->idle_list); 	if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))		mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); 	/* Sanity check nr_running. */	WARN_ON_ONCE(pool->nr_workers == pool->nr_idle && pool->nr_running);} /** * worker_leave_idle - leave idle state * @worker: worker which is leaving idle state * * @worker is leaving idle state.  Update stats. * * LOCKING: * raw_spin_lock_irq(pool->lock). */static void worker_leave_idle(struct worker *worker){	struct worker_pool *pool = worker->pool; 	if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))		return;	worker_clr_flags(worker, WORKER_IDLE);	pool->nr_idle--;	list_del_init(&worker->entry);} /** * find_worker_executing_work - find worker which is executing a work * @pool: pool of interest * @work: work to find worker for * * Find a worker which is executing @work on @pool by searching * @pool->busy_hash which is keyed by the address of @work.  For a worker * to match, its current execution should match the address of @work and * its work function.  This is to avoid unwanted dependency between * unrelated work executions through a work item being recycled while still * being executed. * * This is a bit tricky.  A work item may be freed once its execution * starts and nothing prevents the freed area from being recycled for * another work item.  If the same work item address ends up being reused * before the original execution finishes, workqueue will identify the * recycled work item as currently executing and make it wait until the * current execution finishes, introducing an unwanted dependency. * * This function checks the work item address and work function to avoid * false positives.  Note that this isn't complete as one may construct a * work function which can introduce dependency onto itself through a * recycled work item.  Well, if somebody wants to shoot oneself in the * foot that badly, there's only so much we can do, and if such deadlock * actually occurs, it should be easy to locate the culprit work function. * * CONTEXT: * raw_spin_lock_irq(pool->lock). * * Return: * Pointer to worker which is executing @work if found, %NULL * otherwise. */static struct worker *find_worker_executing_work(struct worker_pool *pool,						 struct work_struct *work){	struct worker *worker; 	hash_for_each_possible(pool->busy_hash, worker, hentry,			       (unsigned long)work)		if (worker->current_work == work &&		    worker->current_func == work->func)			return worker; 	return NULL;} static void mayday_cursor_func(struct work_struct *work){	/* should not be processed, only for marking position */	BUG();} /** * move_linked_works - move linked works to a list * @work: start of series of works to be scheduled * @head: target list to append @work to * @nextp: out parameter for nested worklist walking * * Schedule linked works starting from @work to @head. Work series to be * scheduled starts at @work and includes any consecutive work with * WORK_STRUCT_LINKED set in its predecessor. See assign_work() for details on * @nextp. * * CONTEXT: * raw_spin_lock_irq(pool->lock). */static void move_linked_works(struct work_struct *work, struct list_head *head,			      struct work_struct **nextp){	struct work_struct *n; 	/*	 * Linked worklist will always end before the end of the list,	 * use NULL for list head.	 */	list_for_each_entry_safe_from(work, n, NULL, entry) {		list_move_tail(&work->entry, head);		if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))			break;	} 	/*	 * If we're already inside safe list traversal and have moved	 * multiple works to the scheduled queue, the next position	 * needs to be updated.	 */	if (nextp)		*nextp = n;} /** * assign_work - assign a work item and its linked work items to a worker * @work: work to assign * @worker: worker to assign to * @nextp: out parameter for nested worklist walking * * Assign @work and its linked work items to @worker. If @work is already being * executed by another worker in the same pool, it'll be punted there. * * If @nextp is not NULL, it's updated to point to the next work of the last * scheduled work. This allows assign_work() to be nested inside * list_for_each_entry_safe(). * * Returns %true if @work was successfully assigned to @worker. %false if @work * was punted to another worker already executing it. */static bool assign_work(struct work_struct *work, struct worker *worker,			struct work_struct **nextp){	struct worker_pool *pool = worker->pool;	struct worker *collision; 	lockdep_assert_held(&pool->lock); 	/* The cursor work should not be processed */	if (unlikely(work->func == mayday_cursor_func)) {		/* only worker_thread() can possibly take this branch */		WARN_ON_ONCE(worker->rescue_wq);		if (nextp)			*nextp = list_next_entry(work, entry);		list_del_init(&work->entry);		return false;	} 	/*	 * A single work shouldn't be executed concurrently by multiple workers.	 * __queue_work() ensures that @work doesn't jump to a different pool	 * while still running in the previous pool. Here, we should ensure that	 * @work is not executed concurrently by multiple workers from the same	 * pool. Check whether anyone is already processing the work. If so,	 * defer the work to the currently executing one.	 */	collision = find_worker_executing_work(pool, work);	if (unlikely(collision)) {		move_linked_works(work, &collision->scheduled, nextp);		return false;	} 	move_linked_works(work, &worker->scheduled, nextp);	return true;} static struct irq_work *bh_pool_irq_work(struct worker_pool *pool){	int high = pool->attrs->nice == HIGHPRI_NICE_LEVEL ? 1 : 0; 	return &per_cpu(bh_pool_irq_works, pool->cpu)[high];} static void kick_bh_pool(struct worker_pool *pool){#ifdef CONFIG_SMP	/* see drain_dead_softirq_workfn() for BH_DRAINING */	if (unlikely(pool->cpu != smp_processor_id() &&		     !(pool->flags & POOL_BH_DRAINING))) {		irq_work_queue_on(bh_pool_irq_work(pool), pool->cpu);		return;	}#endif	if (pool->attrs->nice == HIGHPRI_NICE_LEVEL)		raise_softirq_irqoff(HI_SOFTIRQ);	else		raise_softirq_irqoff(TASKLET_SOFTIRQ);} /** * kick_pool - wake up an idle worker if necessary * @pool: pool to kick * * @pool may have pending work items. Wake up worker if necessary. Returns * whether a worker was woken up. */static bool kick_pool(struct worker_pool *pool){	struct worker *worker = first_idle_worker(pool);	struct task_struct *p; 	lockdep_assert_held(&pool->lock); 	if (!need_more_worker(pool) || !worker)		return false; 	if (pool->flags & POOL_BH) {		kick_bh_pool(pool);		return true;	} 	p = worker->task; #ifdef CONFIG_SMP	/*	 * Idle @worker is about to execute @work and waking up provides an	 * opportunity to migrate @worker at a lower cost by setting the task's	 * wake_cpu field. Let's see if we want to move @worker to improve	 * execution locality.	 *	 * We're waking the worker that went idle the latest and there's some	 * chance that @worker is marked idle but hasn't gone off CPU yet. If	 * so, setting the wake_cpu won't do anything. As this is a best-effort	 * optimization and the race window is narrow, let's leave as-is for	 * now. If this becomes pronounced, we can skip over workers which are	 * still on cpu when picking an idle worker.	 *	 * If @pool has non-strict affinity, @worker might have ended up outside	 * its affinity scope. Repatriate.	 */	if (!pool->attrs->affn_strict &&	    !cpumask_test_cpu(p->wake_cpu, pool->attrs->__pod_cpumask)) {		struct work_struct *work = list_first_entry(&pool->worklist,						struct work_struct, entry);		int wake_cpu = cpumask_any_and_distribute(pool->attrs->__pod_cpumask,							  cpu_online_mask);		if (wake_cpu < nr_cpu_ids) {			p->wake_cpu = wake_cpu;			get_work_pwq(work)->stats[PWQ_STAT_REPATRIATED]++;		}	}#endif	wake_up_process(p);	return true;} #ifdef CONFIG_WQ_CPU_INTENSIVE_REPORT /* * Concurrency-managed per-cpu work items that hog CPU for longer than * wq_cpu_intensive_thresh_us trigger the automatic CPU_INTENSIVE mechanism, * which prevents them from stalling other concurrency-managed work items. If a * work function keeps triggering this mechanism, it's likely that the work item * should be using an unbound workqueue instead. * * wq_cpu_intensive_report() tracks work functions which trigger such conditions * and report them so that they can be examined and converted to use unbound * workqueues as appropriate. To avoid flooding the console, each violating work * function is tracked and reported with exponential backoff. */#define WCI_MAX_ENTS 128 struct wci_ent {	work_func_t		func;	atomic64_t		cnt;	struct hlist_node	hash_node;}; static struct wci_ent wci_ents[WCI_MAX_ENTS];static int wci_nr_ents;static DEFINE_RAW_SPINLOCK(wci_lock);static DEFINE_HASHTABLE(wci_hash, ilog2(WCI_MAX_ENTS)); static struct wci_ent *wci_find_ent(work_func_t func){	struct wci_ent *ent; 	hash_for_each_possible_rcu(wci_hash, ent, hash_node,				   (unsigned long)func) {		if (ent->func == func)			return ent;	}	return NULL;} static void wq_cpu_intensive_report(work_func_t func){	struct wci_ent *ent; restart:	ent = wci_find_ent(func);	if (ent) {		u64 cnt; 		/*		 * Start reporting from the warning_thresh and back off		 * exponentially.		 */		cnt = atomic64_inc_return_relaxed(&ent->cnt);		if (wq_cpu_intensive_warning_thresh &&		    cnt >= wq_cpu_intensive_warning_thresh &&		    is_power_of_2(cnt + 1 - wq_cpu_intensive_warning_thresh))			printk_deferred(KERN_WARNING "workqueue: %ps hogged CPU for >%luus %llu times, consider switching to WQ_UNBOUND\n",					ent->func, wq_cpu_intensive_thresh_us,					atomic64_read(&ent->cnt));		return;	} 	/*	 * @func is a new violation. Allocate a new entry for it. If wcn_ents[]	 * is exhausted, something went really wrong and we probably made enough	 * noise already.	 */	if (wci_nr_ents >= WCI_MAX_ENTS)		return; 	raw_spin_lock(&wci_lock); 	if (wci_nr_ents >= WCI_MAX_ENTS) {		raw_spin_unlock(&wci_lock);		return;	} 	if (wci_find_ent(func)) {		raw_spin_unlock(&wci_lock);		goto restart;	} 	ent = &wci_ents[wci_nr_ents++];	ent->func = func;	atomic64_set(&ent->cnt, 0);	hash_add_rcu(wci_hash, &ent->hash_node, (unsigned long)func); 	raw_spin_unlock(&wci_lock); 	goto restart;} #else	/* CONFIG_WQ_CPU_INTENSIVE_REPORT */static void wq_cpu_intensive_report(work_func_t func) {}#endif	/* CONFIG_WQ_CPU_INTENSIVE_REPORT */ /** * wq_worker_running - a worker is running again * @task: task waking up * * This function is called when a worker returns from schedule() */void wq_worker_running(struct task_struct *task){	struct worker *worker = kthread_data(task); 	if (!READ_ONCE(worker->sleeping))		return; 	/*	 * If preempted by unbind_workers() between the WORKER_NOT_RUNNING check	 * and the nr_running increment below, we may ruin the nr_running reset	 * and leave with an unexpected pool->nr_running == 1 on the newly unbound	 * pool. Protect against such race.	 */	preempt_disable();	if (!(worker->flags & WORKER_NOT_RUNNING))		worker->pool->nr_running++;	preempt_enable(); 	/*	 * CPU intensive auto-detection cares about how long a work item hogged	 * CPU without sleeping. Reset the starting timestamp on wakeup.	 */	worker->current_at = worker->task->se.sum_exec_runtime; 	WRITE_ONCE(worker->sleeping, 0);} /** * wq_worker_sleeping - a worker is going to sleep * @task: task going to sleep * * This function is called from schedule() when a busy worker is * going to sleep. */void wq_worker_sleeping(struct task_struct *task){	struct worker *worker = kthread_data(task);	struct worker_pool *pool; 	/*	 * Rescuers, which may not have all the fields set up like normal	 * workers, also reach here, let's not access anything before	 * checking NOT_RUNNING.	 */	if (worker->flags & WORKER_NOT_RUNNING)		return; 	pool = worker->pool; 	/* Return if preempted before wq_worker_running() was reached */	if (READ_ONCE(worker->sleeping))		return; 	WRITE_ONCE(worker->sleeping, 1);	raw_spin_lock_irq(&pool->lock); 	/*	 * Recheck in case unbind_workers() preempted us. We don't	 * want to decrement nr_running after the worker is unbound	 * and nr_running has been reset.	 */	if (worker->flags & WORKER_NOT_RUNNING) {		raw_spin_unlock_irq(&pool->lock);		return;	} 	pool->nr_running--;	if (kick_pool(pool))		worker->current_pwq->stats[PWQ_STAT_CM_WAKEUP]++; 	raw_spin_unlock_irq(&pool->lock);} /** * wq_worker_tick - a scheduler tick occurred while a kworker is running * @task: task currently running * * Called from sched_tick(). We're in the IRQ context and the current * worker's fields which follow the 'K' locking rule can be accessed safely. */void wq_worker_tick(struct task_struct *task){

Lines 1001–1500 of 8475. Previous lines Load more