5816 lines · 159.4 KB · C
RawDownload
		pr_info("    pending:");		list_for_each_entry(work, &pool->worklist, entry) {			if (get_work_pwq(work) != pwq)				continue; 			pr_cont_work(comma, work);			comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);		}		pr_cont("\n");	} 	if (!list_empty(&pwq->delayed_works)) {		bool comma = false; 		pr_info("    delayed:");		list_for_each_entry(work, &pwq->delayed_works, entry) {			pr_cont_work(comma, work);			comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);		}		pr_cont("\n");	}} /** * show_workqueue_state - dump workqueue state * * Called from a sysrq handler or try_to_freeze_tasks() and prints out * all busy workqueues and pools. */void show_workqueue_state(void){	struct workqueue_struct *wq;	struct worker_pool *pool;	unsigned long flags;	int pi; 	rcu_read_lock_sched(); 	pr_info("Showing busy workqueues and worker pools:\n"); 	list_for_each_entry_rcu(wq, &workqueues, list) {		struct pool_workqueue *pwq;		bool idle = true; 		for_each_pwq(pwq, wq) {			if (pwq->nr_active || !list_empty(&pwq->delayed_works)) {				idle = false;				break;			}		}		if (idle)			continue; 		pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags); 		for_each_pwq(pwq, wq) {			spin_lock_irqsave(&pwq->pool->lock, flags);			if (pwq->nr_active || !list_empty(&pwq->delayed_works))				show_pwq(pwq);			spin_unlock_irqrestore(&pwq->pool->lock, flags);			/*			 * We could be printing a lot from atomic context, e.g.			 * sysrq-t -> show_workqueue_state(). Avoid triggering			 * hard lockup.			 */			touch_nmi_watchdog();		}	} 	for_each_pool(pool, pi) {		struct worker *worker;		bool first = true; 		spin_lock_irqsave(&pool->lock, flags);		if (pool->nr_workers == pool->nr_idle)			goto next_pool; 		pr_info("pool %d:", pool->id);		pr_cont_pool_info(pool);		pr_cont(" hung=%us workers=%d",			jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000,			pool->nr_workers);		if (pool->manager)			pr_cont(" manager: %d",				task_pid_nr(pool->manager->task));		list_for_each_entry(worker, &pool->idle_list, entry) {			pr_cont(" %s%d", first ? "idle: " : "",				task_pid_nr(worker->task));			first = false;		}		pr_cont("\n");	next_pool:		spin_unlock_irqrestore(&pool->lock, flags);		/*		 * We could be printing a lot from atomic context, e.g.		 * sysrq-t -> show_workqueue_state(). Avoid triggering		 * hard lockup.		 */		touch_nmi_watchdog();	} 	rcu_read_unlock_sched();} /* used to show worker information through /proc/PID/{comm,stat,status} */void wq_worker_comm(char *buf, size_t size, struct task_struct *task){	int off; 	/* always show the actual comm */	off = strscpy(buf, task->comm, size);	if (off < 0)		return; 	/* stabilize PF_WQ_WORKER and worker pool association */	mutex_lock(&wq_pool_attach_mutex); 	if (task->flags & PF_WQ_WORKER) {		struct worker *worker = kthread_data(task);		struct worker_pool *pool = worker->pool; 		if (pool) {			spin_lock_irq(&pool->lock);			/*			 * ->desc tracks information (wq name or			 * set_worker_desc()) for the latest execution.  If			 * current, prepend '+', otherwise '-'.			 */			if (worker->desc[0] != '\0') {				if (worker->current_work)					scnprintf(buf + off, size - off, "+%s",						  worker->desc);				else					scnprintf(buf + off, size - off, "-%s",						  worker->desc);			}			spin_unlock_irq(&pool->lock);		}	} 	mutex_unlock(&wq_pool_attach_mutex);} #ifdef CONFIG_SMP /* * CPU hotplug. * * There are two challenges in supporting CPU hotplug.  Firstly, there * are a lot of assumptions on strong associations among work, pwq and * pool which make migrating pending and scheduled works very * difficult to implement without impacting hot paths.  Secondly, * worker pools serve mix of short, long and very long running works making * blocked draining impractical. * * This is solved by allowing the pools to be disassociated from the CPU * running as an unbound one and allowing it to be reattached later if the * cpu comes back online. */ static void unbind_workers(int cpu){	struct worker_pool *pool;	struct worker *worker; 	for_each_cpu_worker_pool(pool, cpu) {		mutex_lock(&wq_pool_attach_mutex);		spin_lock_irq(&pool->lock); 		/*		 * We've blocked all attach/detach operations. Make all workers		 * unbound and set DISASSOCIATED.  Before this, all workers		 * except for the ones which are still executing works from		 * before the last CPU down must be on the cpu.  After		 * this, they may become diasporas.		 */		for_each_pool_worker(worker, pool)			worker->flags |= WORKER_UNBOUND; 		pool->flags |= POOL_DISASSOCIATED; 		spin_unlock_irq(&pool->lock);		mutex_unlock(&wq_pool_attach_mutex); 		/*		 * Call schedule() so that we cross rq->lock and thus can		 * guarantee sched callbacks see the %WORKER_UNBOUND flag.		 * This is necessary as scheduler callbacks may be invoked		 * from other cpus.		 */		schedule(); 		/*		 * Sched callbacks are disabled now.  Zap nr_running.		 * After this, nr_running stays zero and need_more_worker()		 * and keep_working() are always true as long as the		 * worklist is not empty.  This pool now behaves as an		 * unbound (in terms of concurrency management) pool which		 * are served by workers tied to the pool.		 */		atomic_set(&pool->nr_running, 0); 		/*		 * With concurrency management just turned off, a busy		 * worker blocking could lead to lengthy stalls.  Kick off		 * unbound chain execution of currently pending work items.		 */		spin_lock_irq(&pool->lock);		wake_up_worker(pool);		spin_unlock_irq(&pool->lock);	}} /** * rebind_workers - rebind all workers of a pool to the associated CPU * @pool: pool of interest * * @pool->cpu is coming online.  Rebind all workers to the CPU. */static void rebind_workers(struct worker_pool *pool){	struct worker *worker; 	lockdep_assert_held(&wq_pool_attach_mutex); 	/*	 * Restore CPU affinity of all workers.  As all idle workers should	 * be on the run-queue of the associated CPU before any local	 * wake-ups for concurrency management happen, restore CPU affinity	 * of all workers first and then clear UNBOUND.  As we're called	 * from CPU_ONLINE, the following shouldn't fail.	 */	for_each_pool_worker(worker, pool)		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,						  pool->attrs->cpumask) < 0); 	spin_lock_irq(&pool->lock); 	pool->flags &= ~POOL_DISASSOCIATED; 	for_each_pool_worker(worker, pool) {		unsigned int worker_flags = worker->flags; 		/*		 * A bound idle worker should actually be on the runqueue		 * of the associated CPU for local wake-ups targeting it to		 * work.  Kick all idle workers so that they migrate to the		 * associated CPU.  Doing this in the same loop as		 * replacing UNBOUND with REBOUND is safe as no worker will		 * be bound before @pool->lock is released.		 */		if (worker_flags & WORKER_IDLE)			wake_up_process(worker->task); 		/*		 * We want to clear UNBOUND but can't directly call		 * worker_clr_flags() or adjust nr_running.  Atomically		 * replace UNBOUND with another NOT_RUNNING flag REBOUND.		 * @worker will clear REBOUND using worker_clr_flags() when		 * it initiates the next execution cycle thus restoring		 * concurrency management.  Note that when or whether		 * @worker clears REBOUND doesn't affect correctness.		 *		 * WRITE_ONCE() is necessary because @worker->flags may be		 * tested without holding any lock in		 * wq_worker_waking_up().  Without it, NOT_RUNNING test may		 * fail incorrectly leading to premature concurrency		 * management operations.		 */		WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));		worker_flags |= WORKER_REBOUND;		worker_flags &= ~WORKER_UNBOUND;		WRITE_ONCE(worker->flags, worker_flags);	} 	spin_unlock_irq(&pool->lock);} /** * restore_unbound_workers_cpumask - restore cpumask of unbound workers * @pool: unbound pool of interest * @cpu: the CPU which is coming up * * An unbound pool may end up with a cpumask which doesn't have any online * CPUs.  When a worker of such pool get scheduled, the scheduler resets * its cpus_allowed.  If @cpu is in @pool's cpumask which didn't have any * online CPU before, cpus_allowed of all its workers should be restored. */static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu){	static cpumask_t cpumask;	struct worker *worker; 	lockdep_assert_held(&wq_pool_attach_mutex); 	/* is @cpu allowed for @pool? */	if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))		return; 	cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask); 	/* as we're called from CPU_ONLINE, the following shouldn't fail */	for_each_pool_worker(worker, pool)		WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);} int workqueue_prepare_cpu(unsigned int cpu){	struct worker_pool *pool; 	for_each_cpu_worker_pool(pool, cpu) {		if (pool->nr_workers)			continue;		if (!create_worker(pool))			return -ENOMEM;	}	return 0;} int workqueue_online_cpu(unsigned int cpu){	struct worker_pool *pool;	struct workqueue_struct *wq;	int pi; 	mutex_lock(&wq_pool_mutex); 	for_each_pool(pool, pi) {		mutex_lock(&wq_pool_attach_mutex); 		if (pool->cpu == cpu)			rebind_workers(pool);		else if (pool->cpu < 0)			restore_unbound_workers_cpumask(pool, cpu); 		mutex_unlock(&wq_pool_attach_mutex);	} 	/* update NUMA affinity of unbound workqueues */	list_for_each_entry(wq, &workqueues, list)		wq_update_unbound_numa(wq, cpu, true); 	mutex_unlock(&wq_pool_mutex);	return 0;} int workqueue_offline_cpu(unsigned int cpu){	struct workqueue_struct *wq; 	/* unbinding per-cpu workers should happen on the local CPU */	if (WARN_ON(cpu != smp_processor_id()))		return -1; 	unbind_workers(cpu); 	/* update NUMA affinity of unbound workqueues */	mutex_lock(&wq_pool_mutex);	list_for_each_entry(wq, &workqueues, list)		wq_update_unbound_numa(wq, cpu, false);	mutex_unlock(&wq_pool_mutex); 	return 0;} struct work_for_cpu {	struct work_struct work;	long (*fn)(void *);	void *arg;	long ret;}; static void work_for_cpu_fn(struct work_struct *work){	struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work); 	wfc->ret = wfc->fn(wfc->arg);} /** * work_on_cpu - run a function in thread context on a particular cpu * @cpu: the cpu to run on * @fn: the function to run * @arg: the function arg * * It is up to the caller to ensure that the cpu doesn't go offline. * The caller must not hold any locks which would prevent @fn from completing. * * Return: The value @fn returns. */long work_on_cpu(int cpu, long (*fn)(void *), void *arg){	struct work_for_cpu wfc = { .fn = fn, .arg = arg }; 	INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);	schedule_work_on(cpu, &wfc.work);	flush_work(&wfc.work);	destroy_work_on_stack(&wfc.work);	return wfc.ret;}EXPORT_SYMBOL_GPL(work_on_cpu); /** * work_on_cpu_safe - run a function in thread context on a particular cpu * @cpu: the cpu to run on * @fn:  the function to run * @arg: the function argument * * Disables CPU hotplug and calls work_on_cpu(). The caller must not hold * any locks which would prevent @fn from completing. * * Return: The value @fn returns. */long work_on_cpu_safe(int cpu, long (*fn)(void *), void *arg){	long ret = -ENODEV; 	get_online_cpus();	if (cpu_online(cpu))		ret = work_on_cpu(cpu, fn, arg);	put_online_cpus();	return ret;}EXPORT_SYMBOL_GPL(work_on_cpu_safe);#endif /* CONFIG_SMP */ #ifdef CONFIG_FREEZER /** * freeze_workqueues_begin - begin freezing workqueues * * Start freezing workqueues.  After this function returns, all freezable * workqueues will queue new works to their delayed_works list instead of * pool->worklist. * * CONTEXT: * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's. */void freeze_workqueues_begin(void){	struct workqueue_struct *wq;	struct pool_workqueue *pwq; 	mutex_lock(&wq_pool_mutex); 	WARN_ON_ONCE(workqueue_freezing);	workqueue_freezing = true; 	list_for_each_entry(wq, &workqueues, list) {		mutex_lock(&wq->mutex);		for_each_pwq(pwq, wq)			pwq_adjust_max_active(pwq);		mutex_unlock(&wq->mutex);	} 	mutex_unlock(&wq_pool_mutex);} /** * freeze_workqueues_busy - are freezable workqueues still busy? * * Check whether freezing is complete.  This function must be called * between freeze_workqueues_begin() and thaw_workqueues(). * * CONTEXT: * Grabs and releases wq_pool_mutex. * * Return: * %true if some freezable workqueues are still busy.  %false if freezing * is complete. */bool freeze_workqueues_busy(void){	bool busy = false;	struct workqueue_struct *wq;	struct pool_workqueue *pwq; 	mutex_lock(&wq_pool_mutex); 	WARN_ON_ONCE(!workqueue_freezing); 	list_for_each_entry(wq, &workqueues, list) {		if (!(wq->flags & WQ_FREEZABLE))			continue;		/*		 * nr_active is monotonically decreasing.  It's safe		 * to peek without lock.		 */		rcu_read_lock_sched();		for_each_pwq(pwq, wq) {			WARN_ON_ONCE(pwq->nr_active < 0);			if (pwq->nr_active) {				busy = true;				rcu_read_unlock_sched();				goto out_unlock;			}		}		rcu_read_unlock_sched();	}out_unlock:

Lines 4501–5000 of 5816. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t