struct worker *worker = kthread_data(task);	struct pool_workqueue *pwq = worker->current_pwq;	struct worker_pool *pool = worker->pool; 	if (!pwq)		return; 	pwq->stats[PWQ_STAT_CPU_TIME] += TICK_USEC; 	if (!wq_cpu_intensive_thresh_us)		return; 	/*	 * If the current worker is concurrency managed and hogged the CPU for	 * longer than wq_cpu_intensive_thresh_us, it's automatically marked	 * CPU_INTENSIVE to avoid stalling other concurrency-managed work items.	 *	 * Set @worker->sleeping means that @worker is in the process of	 * switching out voluntarily and won't be contributing to	 * @pool->nr_running until it wakes up. As wq_worker_sleeping() also	 * decrements ->nr_running, setting CPU_INTENSIVE here can lead to	 * double decrements. The task is releasing the CPU anyway. Let's skip.	 * We probably want to make this prettier in the future.	 */	if ((worker->flags & WORKER_NOT_RUNNING) || READ_ONCE(worker->sleeping) ||	    worker->task->se.sum_exec_runtime - worker->current_at <	    wq_cpu_intensive_thresh_us * NSEC_PER_USEC)		return; 	raw_spin_lock(&pool->lock); 	worker_set_flags(worker, WORKER_CPU_INTENSIVE);	wq_cpu_intensive_report(worker->current_func);	pwq->stats[PWQ_STAT_CPU_INTENSIVE]++; 	if (kick_pool(pool))		pwq->stats[PWQ_STAT_CM_WAKEUP]++; 	raw_spin_unlock(&pool->lock);} /** * wq_worker_last_func - retrieve worker's last work function * @task: Task to retrieve last work function of. * * Determine the last function a worker executed. This is called from * the scheduler to get a worker's last known identity. * * CONTEXT: * raw_spin_lock_irq(rq->lock) * * This function is called during schedule() when a kworker is going * to sleep. It's used by psi to identify aggregation workers during * dequeuing, to allow periodic aggregation to shut-off when that * worker is the last task in the system or cgroup to go to sleep. * * As this function doesn't involve any workqueue-related locking, it * only returns stable values when called from inside the scheduler's * queuing and dequeuing paths, when @task, which must be a kworker, * is guaranteed to not be processing any works. * * Return: * The last work function %current executed as a worker, NULL if it * hasn't executed any work yet. */work_func_t wq_worker_last_func(struct task_struct *task){	struct worker *worker = kthread_data(task); 	return worker->last_func;} /** * wq_node_nr_active - Determine wq_node_nr_active to use * @wq: workqueue of interest * @node: NUMA node, can be %NUMA_NO_NODE * * Determine wq_node_nr_active to use for @wq on @node. Returns: * * - %NULL for per-cpu workqueues as they don't need to use shared nr_active. * * - node_nr_active[nr_node_ids] if @node is %NUMA_NO_NODE. * * - Otherwise, node_nr_active[@node]. */static struct wq_node_nr_active *wq_node_nr_active(struct workqueue_struct *wq,						   int node){	if (!(wq->flags & WQ_UNBOUND))		return NULL; 	if (node == NUMA_NO_NODE)		node = nr_node_ids; 	return wq->node_nr_active[node];} /** * wq_update_node_max_active - Update per-node max_actives to use * @wq: workqueue to update * @off_cpu: CPU that's going down, -1 if a CPU is not going down * * Update @wq->node_nr_active[]->max. @wq must be unbound. max_active is * distributed among nodes according to the proportions of numbers of online * cpus. The result is always between @wq->min_active and max_active. */static void wq_update_node_max_active(struct workqueue_struct *wq, int off_cpu){	struct cpumask *effective = unbound_effective_cpumask(wq);	int min_active = READ_ONCE(wq->min_active);	int max_active = READ_ONCE(wq->max_active);	int total_cpus, node; 	lockdep_assert_held(&wq->mutex); 	if (!wq_topo_initialized)		return; 	if (off_cpu >= 0 && !cpumask_test_cpu(off_cpu, effective))		off_cpu = -1; 	total_cpus = cpumask_weight_and(effective, cpu_online_mask);	if (off_cpu >= 0)		total_cpus--; 	/* If all CPUs of the wq get offline, use the default values */	if (unlikely(!total_cpus)) {		for_each_node(node)			wq_node_nr_active(wq, node)->max = min_active; 		wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;		return;	} 	for_each_node(node) {		int node_cpus; 		node_cpus = cpumask_weight_and(effective, cpumask_of_node(node));		if (off_cpu >= 0 && cpu_to_node(off_cpu) == node)			node_cpus--; 		wq_node_nr_active(wq, node)->max =			clamp(DIV_ROUND_UP(max_active * node_cpus, total_cpus),			      min_active, max_active);	} 	wq_node_nr_active(wq, NUMA_NO_NODE)->max = max_active;} /** * get_pwq - get an extra reference on the specified pool_workqueue * @pwq: pool_workqueue to get * * Obtain an extra reference on @pwq.  The caller should guarantee that * @pwq has positive refcnt and be holding the matching pool->lock. */static void get_pwq(struct pool_workqueue *pwq){	lockdep_assert_held(&pwq->pool->lock);	WARN_ON_ONCE(pwq->refcnt <= 0);	pwq->refcnt++;} /** * put_pwq - put a pool_workqueue reference * @pwq: pool_workqueue to put * * Drop a reference of @pwq.  If its refcnt reaches zero, schedule its * destruction.  The caller should be holding the matching pool->lock. */static void put_pwq(struct pool_workqueue *pwq){	lockdep_assert_held(&pwq->pool->lock);	if (likely(--pwq->refcnt))		return;	/*	 * @pwq can't be released under pool->lock, bounce to a dedicated	 * kthread_worker to avoid A-A deadlocks.	 */	kthread_queue_work(pwq_release_worker, &pwq->release_work);} /** * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock * @pwq: pool_workqueue to put (can be %NULL) * * put_pwq() with locking.  This function also allows %NULL @pwq. */static void put_pwq_unlocked(struct pool_workqueue *pwq){	if (pwq) {		/*		 * As both pwqs and pools are RCU protected, the		 * following lock operations are safe.		 */		raw_spin_lock_irq(&pwq->pool->lock);		put_pwq(pwq);		raw_spin_unlock_irq(&pwq->pool->lock);	}} static bool pwq_is_empty(struct pool_workqueue *pwq){	return !pwq->nr_active && list_empty(&pwq->inactive_works);} static void __pwq_activate_work(struct pool_workqueue *pwq,				struct work_struct *work){	unsigned long *wdb = work_data_bits(work); 	WARN_ON_ONCE(!(*wdb & WORK_STRUCT_INACTIVE));	trace_workqueue_activate_work(work);	if (list_empty(&pwq->pool->worklist))		pwq->pool->last_progress_ts = jiffies;	move_linked_works(work, &pwq->pool->worklist, NULL);	__clear_bit(WORK_STRUCT_INACTIVE_BIT, wdb);} static bool tryinc_node_nr_active(struct wq_node_nr_active *nna){	int max = READ_ONCE(nna->max);	int old = atomic_read(&nna->nr); 	do {		if (old >= max)			return false;	} while (!atomic_try_cmpxchg_relaxed(&nna->nr, &old, old + 1)); 	return true;} /** * pwq_tryinc_nr_active - Try to increment nr_active for a pwq * @pwq: pool_workqueue of interest * @fill: max_active may have increased, try to increase concurrency level * * Try to increment nr_active for @pwq. Returns %true if an nr_active count is * successfully obtained. %false otherwise. */static bool pwq_tryinc_nr_active(struct pool_workqueue *pwq, bool fill){	struct workqueue_struct *wq = pwq->wq;	struct worker_pool *pool = pwq->pool;	struct wq_node_nr_active *nna = wq_node_nr_active(wq, pool->node);	bool obtained = false; 	lockdep_assert_held(&pool->lock); 	if (!nna) {		/* BH or per-cpu workqueue, pwq->nr_active is sufficient */		obtained = pwq->nr_active < READ_ONCE(wq->max_active);		goto out;	} 	if (unlikely(pwq->plugged))		return false; 	/*	 * Unbound workqueue uses per-node shared nr_active $nna. If @pwq is	 * already waiting on $nna, pwq_dec_nr_active() will maintain the	 * concurrency level. Don't jump the line.	 *	 * We need to ignore the pending test after max_active has increased as	 * pwq_dec_nr_active() can only maintain the concurrency level but not	 * increase it. This is indicated by @fill.	 */	if (!list_empty(&pwq->pending_node) && likely(!fill))		goto out; 	obtained = tryinc_node_nr_active(nna);	if (obtained)		goto out; 	/*	 * Lockless acquisition failed. Lock, add ourself to $nna->pending_pwqs	 * and try again. The smp_mb() is paired with the implied memory barrier	 * of atomic_dec_return() in pwq_dec_nr_active() to ensure that either	 * we see the decremented $nna->nr or they see non-empty	 * $nna->pending_pwqs.	 */	raw_spin_lock(&nna->lock); 	if (list_empty(&pwq->pending_node))		list_add_tail(&pwq->pending_node, &nna->pending_pwqs);	else if (likely(!fill))		goto out_unlock; 	smp_mb(); 	obtained = tryinc_node_nr_active(nna); 	/*	 * If @fill, @pwq might have already been pending. Being spuriously	 * pending in cold paths doesn't affect anything. Let's leave it be.	 */	if (obtained && likely(!fill))		list_del_init(&pwq->pending_node); out_unlock:	raw_spin_unlock(&nna->lock);out:	if (obtained)		pwq->nr_active++;	return obtained;} /** * pwq_activate_first_inactive - Activate the first inactive work item on a pwq * @pwq: pool_workqueue of interest * @fill: max_active may have increased, try to increase concurrency level * * Activate the first inactive work item of @pwq if available and allowed by * max_active limit. * * Returns %true if an inactive work item has been activated. %false if no * inactive work item is found or max_active limit is reached. */static bool pwq_activate_first_inactive(struct pool_workqueue *pwq, bool fill){	struct work_struct *work =		list_first_entry_or_null(&pwq->inactive_works,					 struct work_struct, entry); 	if (work && pwq_tryinc_nr_active(pwq, fill)) {		__pwq_activate_work(pwq, work);		return true;	} else {		return false;	}} /** * unplug_oldest_pwq - unplug the oldest pool_workqueue * @wq: workqueue_struct where its oldest pwq is to be unplugged * * This function should only be called for ordered workqueues where only the * oldest pwq is unplugged, the others are plugged to suspend execution to * ensure proper work item ordering:: * *    dfl_pwq --------------+     [P] - plugged *                          | *                          v *    pwqs -> A -> B [P] -> C [P] (newest) *            |    |        | *            1    3        5 *            |    |        | *            2    4        6 * * When the oldest pwq is drained and removed, this function should be called * to unplug the next oldest one to start its work item execution. Note that * pwq's are linked into wq->pwqs with the oldest first, so the first one in * the list is the oldest. */static void unplug_oldest_pwq(struct workqueue_struct *wq){	struct pool_workqueue *pwq; 	lockdep_assert_held(&wq->mutex); 	/* Caller should make sure that pwqs isn't empty before calling */	pwq = list_first_entry_or_null(&wq->pwqs, struct pool_workqueue,				       pwqs_node);	raw_spin_lock_irq(&pwq->pool->lock);	if (pwq->plugged) {		pwq->plugged = false;		if (pwq_activate_first_inactive(pwq, true)) {			/*			 * While plugged, queueing skips activation which			 * includes bumping the nr_active count and adding the			 * pwq to nna->pending_pwqs if the count can't be			 * obtained. We need to restore both for the pwq being			 * unplugged. The first call activates the first			 * inactive work item and the second, if there are more			 * inactive, puts the pwq on pending_pwqs.			 */			pwq_activate_first_inactive(pwq, false); 			kick_pool(pwq->pool);		}	}	raw_spin_unlock_irq(&pwq->pool->lock);} /** * node_activate_pending_pwq - Activate a pending pwq on a wq_node_nr_active * @nna: wq_node_nr_active to activate a pending pwq for * @caller_pool: worker_pool the caller is locking * * Activate a pwq in @nna->pending_pwqs. Called with @caller_pool locked. * @caller_pool may be unlocked and relocked to lock other worker_pools. */static void node_activate_pending_pwq(struct wq_node_nr_active *nna,				      struct worker_pool *caller_pool){	struct worker_pool *locked_pool = caller_pool;	struct pool_workqueue *pwq;	struct work_struct *work; 	lockdep_assert_held(&caller_pool->lock); 	raw_spin_lock(&nna->lock);retry:	pwq = list_first_entry_or_null(&nna->pending_pwqs,				       struct pool_workqueue, pending_node);	if (!pwq)		goto out_unlock; 	/*	 * If @pwq is for a different pool than @locked_pool, we need to lock	 * @pwq->pool->lock. Let's trylock first. If unsuccessful, do the unlock	 * / lock dance. For that, we also need to release @nna->lock as it's	 * nested inside pool locks.	 */	if (pwq->pool != locked_pool) {		raw_spin_unlock(&locked_pool->lock);		locked_pool = pwq->pool;		if (!raw_spin_trylock(&locked_pool->lock)) {			raw_spin_unlock(&nna->lock);			raw_spin_lock(&locked_pool->lock);			raw_spin_lock(&nna->lock);			goto retry;		}	} 	/*	 * $pwq may not have any inactive work items due to e.g. cancellations.	 * Drop it from pending_pwqs and see if there's another one.	 */	work = list_first_entry_or_null(&pwq->inactive_works,					struct work_struct, entry);	if (!work) {		list_del_init(&pwq->pending_node);		goto retry;	} 	/*	 * Acquire an nr_active count and activate the inactive work item. If	 * $pwq still has inactive work items, rotate it to the end of the	 * pending_pwqs so that we round-robin through them. This means that	 * inactive work items are not activated in queueing order which is fine	 * given that there has never been any ordering across different pwqs.	 */	if (likely(tryinc_node_nr_active(nna))) {		pwq->nr_active++;		__pwq_activate_work(pwq, work); 		if (list_empty(&pwq->inactive_works))			list_del_init(&pwq->pending_node);		else			list_move_tail(&pwq->pending_node, &nna->pending_pwqs); 		/* if activating a foreign pool, make sure it's running */		if (pwq->pool != caller_pool)			kick_pool(pwq->pool);	} out_unlock:	raw_spin_unlock(&nna->lock);	if (locked_pool != caller_pool) {		raw_spin_unlock(&locked_pool->lock);		raw_spin_lock(&caller_pool->lock);	}} /** * pwq_dec_nr_active - Retire an active count * @pwq: pool_workqueue of interest * * Decrement @pwq's nr_active and try to activate the first inactive work item. * For unbound workqueues, this function may temporarily drop @pwq->pool->lock. */static void pwq_dec_nr_active(struct pool_workqueue *pwq){	struct worker_pool *pool = pwq->pool;	struct wq_node_nr_active *nna = wq_node_nr_active(pwq->wq, pool->node); 	lockdep_assert_held(&pool->lock); 	/*	 * @pwq->nr_active should be decremented for both percpu and unbound	 * workqueues.	 */	pwq->nr_active--; 	/*	 * For a percpu workqueue, it's simple. Just need to kick the first	 * inactive work item on @pwq itself.	 */	if (!nna) {		pwq_activate_first_inactive(pwq, false);		return;	} 	/*	 * If @pwq is for an unbound workqueue, it's more complicated because	 * multiple pwqs and pools may be sharing the nr_active count. When a	 * pwq needs to wait for an nr_active count, it puts itself on	 * $nna->pending_pwqs. The following atomic_dec_return()'s implied	 * memory barrier is paired with smp_mb() in pwq_tryinc_nr_active() to

Lines 1501–2000 of 8475. Previous lines Load more