* guarantee that either we see non-empty pending_pwqs or they see	 * decremented $nna->nr.	 *	 * $nna->max may change as CPUs come online/offline and @pwq->wq's	 * max_active gets updated. However, it is guaranteed to be equal to or	 * larger than @pwq->wq->min_active which is above zero unless freezing.	 * This maintains the forward progress guarantee.	 */	if (atomic_dec_return(&nna->nr) >= READ_ONCE(nna->max))		return; 	if (!list_empty(&nna->pending_pwqs))		node_activate_pending_pwq(nna, pool);} /** * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight * @pwq: pwq of interest * @work_data: work_data of work which left the queue * * A work either has completed or is removed from pending queue, * decrement nr_in_flight of its pwq and handle workqueue flushing. * * NOTE: * For unbound workqueues, this function may temporarily drop @pwq->pool->lock * and thus should be called after all other state updates for the in-flight * work item is complete. * * CONTEXT: * raw_spin_lock_irq(pool->lock). */static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data){	int color = get_work_color(work_data); 	if (!(work_data & WORK_STRUCT_INACTIVE))		pwq_dec_nr_active(pwq); 	pwq->nr_in_flight[color]--; 	/* is flush in progress and are we at the flushing tip? */	if (likely(pwq->flush_color != color))		goto out_put; 	/* are there still in-flight works? */	if (pwq->nr_in_flight[color])		goto out_put; 	/* this pwq is done, clear flush_color */	pwq->flush_color = -1; 	/*	 * If this was the last pwq, wake up the first flusher.  It	 * will handle the rest.	 */	if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))		complete(&pwq->wq->first_flusher->done);out_put:	put_pwq(pwq);} /** * try_to_grab_pending - steal work item from worklist and disable irq * @work: work item to steal * @cflags: %WORK_CANCEL_ flags * @irq_flags: place to store irq state * * Try to grab PENDING bit of @work.  This function can handle @work in any * stable state - idle, on timer or on worklist. * * Return: * *  ========	================================================================ *  1		if @work was pending and we successfully stole PENDING *  0		if @work was idle and we claimed PENDING *  -EAGAIN	if PENDING couldn't be grabbed at the moment, safe to busy-retry *  ========	================================================================ * * Note: * On >= 0 return, the caller owns @work's PENDING bit.  To avoid getting * interrupted while holding PENDING and @work off queue, irq must be * disabled on entry.  This, combined with delayed_work->timer being * irqsafe, ensures that we return -EAGAIN for finite short period of time. * * On successful return, >= 0, irq is disabled and the caller is * responsible for releasing it using local_irq_restore(*@irq_flags). * * This function is safe to call from any context including IRQ handler. */static int try_to_grab_pending(struct work_struct *work, u32 cflags,			       unsigned long *irq_flags){	struct worker_pool *pool;	struct pool_workqueue *pwq; 	local_irq_save(*irq_flags); 	/* try to steal the timer if it exists */	if (cflags & WORK_CANCEL_DELAYED) {		struct delayed_work *dwork = to_delayed_work(work); 		/*		 * dwork->timer is irqsafe.  If timer_delete() fails, it's		 * guaranteed that the timer is not queued anywhere and not		 * running on the local CPU.		 */		if (likely(timer_delete(&dwork->timer)))			return 1;	} 	/* try to claim PENDING the normal way */	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))		return 0; 	rcu_read_lock();	/*	 * The queueing is in progress, or it is already queued. Try to	 * steal it from ->worklist without clearing WORK_STRUCT_PENDING.	 */	pool = get_work_pool(work);	if (!pool)		goto fail; 	raw_spin_lock(&pool->lock);	/*	 * work->data is guaranteed to point to pwq only while the work	 * item is queued on pwq->wq, and both updating work->data to point	 * to pwq on queueing and to pool on dequeueing are done under	 * pwq->pool->lock.  This in turn guarantees that, if work->data	 * points to pwq which is associated with a locked pool, the work	 * item is currently queued on that pool.	 */	pwq = get_work_pwq(work);	if (pwq && pwq->pool == pool) {		unsigned long work_data = *work_data_bits(work); 		debug_work_deactivate(work); 		/*		 * A cancelable inactive work item must be in the		 * pwq->inactive_works since a queued barrier can't be		 * canceled (see the comments in insert_wq_barrier()).		 *		 * An inactive work item cannot be deleted directly because		 * it might have linked barrier work items which, if left		 * on the inactive_works list, will confuse pwq->nr_active		 * management later on and cause stall.  Move the linked		 * barrier work items to the worklist when deleting the grabbed		 * item. Also keep WORK_STRUCT_INACTIVE in work_data, so that		 * it doesn't participate in nr_active management in later		 * pwq_dec_nr_in_flight().		 */		if (work_data & WORK_STRUCT_INACTIVE)			move_linked_works(work, &pwq->pool->worklist, NULL); 		list_del_init(&work->entry); 		/*		 * work->data points to pwq iff queued. Let's point to pool. As		 * this destroys work->data needed by the next step, stash it.		 */		set_work_pool_and_keep_pending(work, pool->id,					       pool_offq_flags(pool)); 		/* must be the last step, see the function comment */		pwq_dec_nr_in_flight(pwq, work_data); 		raw_spin_unlock(&pool->lock);		rcu_read_unlock();		return 1;	}	raw_spin_unlock(&pool->lock);fail:	rcu_read_unlock();	local_irq_restore(*irq_flags);	return -EAGAIN;} /** * work_grab_pending - steal work item from worklist and disable irq * @work: work item to steal * @cflags: %WORK_CANCEL_ flags * @irq_flags: place to store IRQ state * * Grab PENDING bit of @work. @work can be in any stable state - idle, on timer * or on worklist. * * Can be called from any context. IRQ is disabled on return with IRQ state * stored in *@irq_flags. The caller is responsible for re-enabling it using * local_irq_restore(). * * Returns %true if @work was pending. %false if idle. */static bool work_grab_pending(struct work_struct *work, u32 cflags,			      unsigned long *irq_flags){	int ret; 	while (true) {		ret = try_to_grab_pending(work, cflags, irq_flags);		if (ret >= 0)			return ret;		cpu_relax();	}} /** * insert_work - insert a work into a pool * @pwq: pwq @work belongs to * @work: work to insert * @head: insertion point * @extra_flags: extra WORK_STRUCT_* flags to set * * Insert @work which belongs to @pwq after @head.  @extra_flags is or'd to * work_struct flags. * * CONTEXT: * raw_spin_lock_irq(pool->lock). */static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,			struct list_head *head, unsigned int extra_flags){	debug_work_activate(work); 	/* record the work call stack in order to print it in KASAN reports */	kasan_record_aux_stack(work); 	/* we own @work, set data and link */	set_work_pwq(work, pwq, extra_flags);	list_add_tail(&work->entry, head);	get_pwq(pwq);} /* * Test whether @work is being queued from another work executing on the * same workqueue. */static bool is_chained_work(struct workqueue_struct *wq){	struct worker *worker; 	worker = current_wq_worker();	/*	 * Return %true iff I'm a worker executing a work item on @wq.  If	 * I'm @worker, it's safe to dereference it without locking.	 */	return worker && worker->current_pwq->wq == wq;} /* * When queueing an unbound work item to a wq, prefer local CPU if allowed * by wq_unbound_cpumask.  Otherwise, round robin among the allowed ones to * avoid perturbing sensitive tasks. */static int wq_select_unbound_cpu(int cpu){	int new_cpu; 	if (likely(!wq_debug_force_rr_cpu)) {		if (cpumask_test_cpu(cpu, wq_unbound_cpumask))			return cpu;	} else {		pr_warn_once("workqueue: round-robin CPU selection forced, expect performance impact\n");	} 	new_cpu = __this_cpu_read(wq_rr_cpu_last);	new_cpu = cpumask_next_and_wrap(new_cpu, wq_unbound_cpumask, cpu_online_mask);	if (unlikely(new_cpu >= nr_cpu_ids))		return cpu;	__this_cpu_write(wq_rr_cpu_last, new_cpu); 	return new_cpu;} static void __queue_work(int cpu, struct workqueue_struct *wq,			 struct work_struct *work){	struct pool_workqueue *pwq;	struct worker_pool *last_pool, *pool;	unsigned int work_flags;	unsigned int req_cpu = cpu; 	/*	 * NOTE: Check whether the used workqueue is deprecated and warn	 */	if (unlikely(wq->flags & __WQ_DEPRECATED))		pr_warn_once("workqueue: work func %ps enqueued on deprecated workqueue. "			"Use system_{percpu|dfl}_wq instead.\n",			work->func); 	/*	 * While a work item is PENDING && off queue, a task trying to	 * steal the PENDING will busy-loop waiting for it to either get	 * queued or lose PENDING.  Grabbing PENDING and queueing should	 * happen with IRQ disabled.	 */	lockdep_assert_irqs_disabled(); 	/*	 * For a draining wq, only works from the same workqueue are	 * allowed. The __WQ_DESTROYING helps to spot the issue that	 * queues a new work item to a wq after destroy_workqueue(wq).	 */	if (unlikely(wq->flags & (__WQ_DESTROYING | __WQ_DRAINING) &&		     WARN_ONCE(!is_chained_work(wq), "workqueue: cannot queue %ps on wq %s\n",			       work->func, wq->name))) {		struct work_offq_data offqd; 		/*		 * State on entry: PENDING is set, work is off-queue (no		 * insert_work() has run).		 *		 * Returning without clearing PENDING would leave the work		 * in a weird state (PENDING=1, PWQ=0, entry empty)		 */		work_offqd_unpack(&offqd, *work_data_bits(work));		set_work_pool_and_clear_pending(work, offqd.pool_id,						work_offqd_pack_flags(&offqd));		return;	}	rcu_read_lock();retry:	/* pwq which will be used unless @work is executing elsewhere */	if (req_cpu == WORK_CPU_UNBOUND) {		if (wq->flags & WQ_UNBOUND)			cpu = wq_select_unbound_cpu(raw_smp_processor_id());		else			cpu = raw_smp_processor_id();	} 	pwq = rcu_dereference(*per_cpu_ptr(wq->cpu_pwq, cpu));	pool = pwq->pool; 	/*	 * If @work was previously on a different pool, it might still be	 * running there, in which case the work needs to be queued on that	 * pool to guarantee non-reentrancy.	 *	 * For ordered workqueue, work items must be queued on the newest pwq	 * for accurate order management.  Guaranteed order also guarantees	 * non-reentrancy.  See the comments above unplug_oldest_pwq().	 */	last_pool = get_work_pool(work);	if (last_pool && last_pool != pool && !(wq->flags & __WQ_ORDERED)) {		struct worker *worker; 		raw_spin_lock(&last_pool->lock); 		worker = find_worker_executing_work(last_pool, work); 		if (worker && worker->current_pwq->wq == wq) {			pwq = worker->current_pwq;			pool = pwq->pool;			WARN_ON_ONCE(pool != last_pool);		} else {			/* meh... not running there, queue here */			raw_spin_unlock(&last_pool->lock);			raw_spin_lock(&pool->lock);		}	} else {		raw_spin_lock(&pool->lock);	} 	/*	 * pwq is determined and locked. For unbound pools, we could have raced	 * with pwq release and it could already be dead. If its refcnt is zero,	 * repeat pwq selection. Note that unbound pwqs never die without	 * another pwq replacing it in cpu_pwq or while work items are executing	 * on it, so the retrying is guaranteed to make forward-progress.	 */	if (unlikely(!pwq->refcnt)) {		if (wq->flags & WQ_UNBOUND) {			raw_spin_unlock(&pool->lock);			cpu_relax();			goto retry;		}		/* oops */		WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",			  wq->name, cpu);	} 	/* pwq determined, queue */	trace_workqueue_queue_work(req_cpu, pwq, work); 	if (WARN_ON(!list_empty(&work->entry)))		goto out; 	pwq->nr_in_flight[pwq->work_color]++;	work_flags = work_color_to_flags(pwq->work_color); 	/*	 * Limit the number of concurrently active work items to max_active.	 * @work must also queue behind existing inactive work items to maintain	 * ordering when max_active changes. See wq_adjust_max_active().	 */	if (list_empty(&pwq->inactive_works) && pwq_tryinc_nr_active(pwq, false)) {		if (list_empty(&pool->worklist))			pool->last_progress_ts = jiffies; 		trace_workqueue_activate_work(work);		insert_work(pwq, work, &pool->worklist, work_flags);		kick_pool(pool);	} else {		work_flags |= WORK_STRUCT_INACTIVE;		insert_work(pwq, work, &pwq->inactive_works, work_flags);	} out:	raw_spin_unlock(&pool->lock);	rcu_read_unlock();} static bool clear_pending_if_disabled(struct work_struct *work){	unsigned long data = *work_data_bits(work);	struct work_offq_data offqd; 	if (likely((data & WORK_STRUCT_PWQ) ||		   !(data & WORK_OFFQ_DISABLE_MASK)))		return false; 	work_offqd_unpack(&offqd, data);	set_work_pool_and_clear_pending(work, offqd.pool_id,					work_offqd_pack_flags(&offqd));	return true;} /** * queue_work_on - queue work on specific cpu * @cpu: CPU number to execute work on * @wq: workqueue to use * @work: work to queue * * We queue the work to a specific CPU, the caller must ensure it * can't go away.  Callers that fail to ensure that the specified * CPU cannot go away will execute on a randomly chosen CPU. * But note well that callers specifying a CPU that never has been * online will get a splat. * * Return: %false if @work was already on a queue, %true otherwise. */bool queue_work_on(int cpu, struct workqueue_struct *wq,		   struct work_struct *work){	bool ret = false;	unsigned long irq_flags; 	local_irq_save(irq_flags); 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)) &&	    !clear_pending_if_disabled(work)) {		__queue_work(cpu, wq, work);		ret = true;	} 	local_irq_restore(irq_flags);	return ret;}EXPORT_SYMBOL(queue_work_on); /** * select_numa_node_cpu - Select a CPU based on NUMA node * @node: NUMA node ID that we want to select a CPU from * * This function will attempt to find a "random" cpu available on a given * node. If there are no CPUs available on the given node it will return * WORK_CPU_UNBOUND indicating that we should just schedule to any * available CPU if we need to schedule this work. */static int select_numa_node_cpu(int node){	int cpu; 	/* Delay binding to CPU if node is not valid or online */	if (node < 0 || node >= MAX_NUMNODES || !node_online(node))		return WORK_CPU_UNBOUND; 	/* Use local node/cpu if we are already there */	cpu = raw_smp_processor_id();	if (node == cpu_to_node(cpu))		return cpu; 	/* Use "random" otherwise know as "first" online CPU of node */	cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask); 	/* If CPU is valid return that, otherwise just defer */	return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;} /** * queue_work_node - queue work on a "random" cpu for a given NUMA node * @node: NUMA node that we are targeting the work for * @wq: workqueue to use * @work: work to queue * * We queue the work to a "random" CPU within a given NUMA node. The basic * idea here is to provide a way to somehow associate work with a given * NUMA node. * * This function will only make a best effort attempt at getting this onto

Lines 2001–2500 of 8475. Previous lines Load more