5816 lines · 159.4 KB · C
RawDownload
}EXPORT_SYMBOL(delayed_work_timer_fn); static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,				struct delayed_work *dwork, unsigned long delay){	struct timer_list *timer = &dwork->timer;	struct work_struct *work = &dwork->work; 	WARN_ON_ONCE(!wq);	WARN_ON_ONCE(timer->function != delayed_work_timer_fn);	WARN_ON_ONCE(timer_pending(timer));	WARN_ON_ONCE(!list_empty(&work->entry)); 	/*	 * If @delay is 0, queue @dwork->work immediately.  This is for	 * both optimization and correctness.  The earliest @timer can	 * expire is on the closest next tick and delayed_work users depend	 * on that there's no such delay when @delay is 0.	 */	if (!delay) {		__queue_work(cpu, wq, &dwork->work);		return;	} 	dwork->wq = wq;	dwork->cpu = cpu;	timer->expires = jiffies + delay; 	if (unlikely(cpu != WORK_CPU_UNBOUND))		add_timer_on(timer, cpu);	else		add_timer(timer);} /** * queue_delayed_work_on - queue work on specific CPU after delay * @cpu: CPU number to execute work on * @wq: workqueue to use * @dwork: work to queue * @delay: number of jiffies to wait before queueing * * Return: %false if @work was already on a queue, %true otherwise.  If * @delay is zero and @dwork is idle, it will be scheduled for immediate * execution. */bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,			   struct delayed_work *dwork, unsigned long delay){	struct work_struct *work = &dwork->work;	bool ret = false;	unsigned long flags; 	/* read the comment in __queue_work() */	local_irq_save(flags); 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {		__queue_delayed_work(cpu, wq, dwork, delay);		ret = true;	} 	local_irq_restore(flags);	return ret;}EXPORT_SYMBOL(queue_delayed_work_on); /** * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU * @cpu: CPU number to execute work on * @wq: workqueue to use * @dwork: work to queue * @delay: number of jiffies to wait before queueing * * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise, * modify @dwork's timer so that it expires after @delay.  If @delay is * zero, @work is guaranteed to be scheduled immediately regardless of its * current state. * * Return: %false if @dwork was idle and queued, %true if @dwork was * pending and its timer was modified. * * This function is safe to call from any context including IRQ handler. * See try_to_grab_pending() for details. */bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,			 struct delayed_work *dwork, unsigned long delay){	unsigned long flags;	int ret; 	do {		ret = try_to_grab_pending(&dwork->work, true, &flags);	} while (unlikely(ret == -EAGAIN)); 	if (likely(ret >= 0)) {		__queue_delayed_work(cpu, wq, dwork, delay);		local_irq_restore(flags);	} 	/* -ENOENT from try_to_grab_pending() becomes %true */	return ret;}EXPORT_SYMBOL_GPL(mod_delayed_work_on); static void rcu_work_rcufn(struct rcu_head *rcu){	struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu); 	/* read the comment in __queue_work() */	local_irq_disable();	__queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);	local_irq_enable();} /** * queue_rcu_work - queue work after a RCU grace period * @wq: workqueue to use * @rwork: work to queue * * Return: %false if @rwork was already pending, %true otherwise.  Note * that a full RCU grace period is guaranteed only after a %true return. * While @rwork is guarnateed to be executed after a %false return, the * execution may happen before a full RCU grace period has passed. */bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork){	struct work_struct *work = &rwork->work; 	if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {		rwork->wq = wq;		call_rcu(&rwork->rcu, rcu_work_rcufn);		return true;	} 	return false;}EXPORT_SYMBOL(queue_rcu_work); /** * worker_enter_idle - enter idle state * @worker: worker which is entering idle state * * @worker is entering idle state.  Update stats and idle timer if * necessary. * * LOCKING: * spin_lock_irq(pool->lock). */static void worker_enter_idle(struct worker *worker){	struct worker_pool *pool = worker->pool; 	if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||	    WARN_ON_ONCE(!list_empty(&worker->entry) &&			 (worker->hentry.next || worker->hentry.pprev)))		return; 	/* can't use worker_set_flags(), also called from create_worker() */	worker->flags |= WORKER_IDLE;	pool->nr_idle++;	worker->last_active = jiffies; 	/* idle_list is LIFO */	list_add(&worker->entry, &pool->idle_list); 	if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))		mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); 	/*	 * Sanity check nr_running.  Because unbind_workers() releases	 * pool->lock between setting %WORKER_UNBOUND and zapping	 * nr_running, the warning may trigger spuriously.  Check iff	 * unbind is not in progress.	 */	WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&		     pool->nr_workers == pool->nr_idle &&		     atomic_read(&pool->nr_running));} /** * worker_leave_idle - leave idle state * @worker: worker which is leaving idle state * * @worker is leaving idle state.  Update stats. * * LOCKING: * spin_lock_irq(pool->lock). */static void worker_leave_idle(struct worker *worker){	struct worker_pool *pool = worker->pool; 	if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))		return;	worker_clr_flags(worker, WORKER_IDLE);	pool->nr_idle--;	list_del_init(&worker->entry);} static struct worker *alloc_worker(int node){	struct worker *worker; 	worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);	if (worker) {		INIT_LIST_HEAD(&worker->entry);		INIT_LIST_HEAD(&worker->scheduled);		INIT_LIST_HEAD(&worker->node);		/* on creation a worker is in !idle && prep state */		worker->flags = WORKER_PREP;	}	return worker;} /** * worker_attach_to_pool() - attach a worker to a pool * @worker: worker to be attached * @pool: the target pool * * Attach @worker to @pool.  Once attached, the %WORKER_UNBOUND flag and * cpu-binding of @worker are kept coordinated with the pool across * cpu-[un]hotplugs. */static void worker_attach_to_pool(struct worker *worker,				   struct worker_pool *pool){	mutex_lock(&wq_pool_attach_mutex); 	/*	 * set_cpus_allowed_ptr() will fail if the cpumask doesn't have any	 * online CPUs.  It'll be re-applied when any of the CPUs come up.	 */	set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask); 	/*	 * The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains	 * stable across this function.  See the comments above the flag	 * definition for details.	 */	if (pool->flags & POOL_DISASSOCIATED)		worker->flags |= WORKER_UNBOUND; 	list_add_tail(&worker->node, &pool->workers);	worker->pool = pool; 	mutex_unlock(&wq_pool_attach_mutex);} /** * worker_detach_from_pool() - detach a worker from its pool * @worker: worker which is attached to its pool * * Undo the attaching which had been done in worker_attach_to_pool().  The * caller worker shouldn't access to the pool after detached except it has * other reference to the pool. */static void worker_detach_from_pool(struct worker *worker){	struct worker_pool *pool = worker->pool;	struct completion *detach_completion = NULL; 	mutex_lock(&wq_pool_attach_mutex); 	list_del(&worker->node);	worker->pool = NULL; 	if (list_empty(&pool->workers))		detach_completion = pool->detach_completion;	mutex_unlock(&wq_pool_attach_mutex); 	/* clear leftover flags without pool->lock after it is detached */	worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND); 	if (detach_completion)		complete(detach_completion);} /** * create_worker - create a new workqueue worker * @pool: pool the new worker will belong to * * Create and start a new worker which is attached to @pool. * * CONTEXT: * Might sleep.  Does GFP_KERNEL allocations. * * Return: * Pointer to the newly created worker. */static struct worker *create_worker(struct worker_pool *pool){	struct worker *worker = NULL;	int id = -1;	char id_buf[16]; 	/* ID is needed to determine kthread name */	id = ida_simple_get(&pool->worker_ida, 0, 0, GFP_KERNEL);	if (id < 0)		goto fail; 	worker = alloc_worker(pool->node);	if (!worker)		goto fail; 	worker->id = id; 	if (pool->cpu >= 0)		snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,			 pool->attrs->nice < 0  ? "H" : "");	else		snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id); 	worker->task = kthread_create_on_node(worker_thread, worker, pool->node,					      "kworker/%s", id_buf);	if (IS_ERR(worker->task))		goto fail; 	set_user_nice(worker->task, pool->attrs->nice);	kthread_bind_mask(worker->task, pool->attrs->cpumask); 	/* successful, attach the worker to the pool */	worker_attach_to_pool(worker, pool); 	/* start the newly created worker */	spin_lock_irq(&pool->lock);	worker->pool->nr_workers++;	worker_enter_idle(worker);	wake_up_process(worker->task);	spin_unlock_irq(&pool->lock); 	return worker; fail:	if (id >= 0)		ida_simple_remove(&pool->worker_ida, id);	kfree(worker);	return NULL;} /** * destroy_worker - destroy a workqueue worker * @worker: worker to be destroyed * * Destroy @worker and adjust @pool stats accordingly.  The worker should * be idle. * * CONTEXT: * spin_lock_irq(pool->lock). */static void destroy_worker(struct worker *worker){	struct worker_pool *pool = worker->pool; 	lockdep_assert_held(&pool->lock); 	/* sanity check frenzy */	if (WARN_ON(worker->current_work) ||	    WARN_ON(!list_empty(&worker->scheduled)) ||	    WARN_ON(!(worker->flags & WORKER_IDLE)))		return; 	pool->nr_workers--;	pool->nr_idle--; 	list_del_init(&worker->entry);	worker->flags |= WORKER_DIE;	wake_up_process(worker->task);} static void idle_worker_timeout(struct timer_list *t){	struct worker_pool *pool = from_timer(pool, t, idle_timer); 	spin_lock_irq(&pool->lock); 	while (too_many_workers(pool)) {		struct worker *worker;		unsigned long expires; 		/* idle_list is kept in LIFO order, check the last one */		worker = list_entry(pool->idle_list.prev, struct worker, entry);		expires = worker->last_active + IDLE_WORKER_TIMEOUT; 		if (time_before(jiffies, expires)) {			mod_timer(&pool->idle_timer, expires);			break;		} 		destroy_worker(worker);	} 	spin_unlock_irq(&pool->lock);} static void send_mayday(struct work_struct *work){	struct pool_workqueue *pwq = get_work_pwq(work);	struct workqueue_struct *wq = pwq->wq; 	lockdep_assert_held(&wq_mayday_lock); 	if (!wq->rescuer)		return; 	/* mayday mayday mayday */	if (list_empty(&pwq->mayday_node)) {		/*		 * If @pwq is for an unbound wq, its base ref may be put at		 * any time due to an attribute change.  Pin @pwq until the		 * rescuer is done with it.		 */		get_pwq(pwq);		list_add_tail(&pwq->mayday_node, &wq->maydays);		wake_up_process(wq->rescuer->task);	}} static void pool_mayday_timeout(struct timer_list *t){	struct worker_pool *pool = from_timer(pool, t, mayday_timer);	struct work_struct *work; 	spin_lock_irq(&pool->lock);	spin_lock(&wq_mayday_lock);		/* for wq->maydays */ 	if (need_to_create_worker(pool)) {		/*		 * We've been trying to create a new worker but		 * haven't been successful.  We might be hitting an		 * allocation deadlock.  Send distress signals to		 * rescuers.		 */		list_for_each_entry(work, &pool->worklist, entry)			send_mayday(work);	} 	spin_unlock(&wq_mayday_lock);	spin_unlock_irq(&pool->lock); 	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);} /** * maybe_create_worker - create a new worker if necessary * @pool: pool to create a new worker for * * Create a new worker for @pool if necessary.  @pool is guaranteed to * have at least one idle worker on return from this function.  If * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is * sent to all rescuers with works scheduled on @pool to resolve * possible allocation deadlock. * * On return, need_to_create_worker() is guaranteed to be %false and * may_start_working() %true. * * LOCKING: * spin_lock_irq(pool->lock) which may be released and regrabbed * multiple times.  Does GFP_KERNEL allocations.  Called only from * manager. */static void maybe_create_worker(struct worker_pool *pool)__releases(&pool->lock)__acquires(&pool->lock){restart:	spin_unlock_irq(&pool->lock); 	/* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */	mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT); 	while (true) {		if (create_worker(pool) || !need_to_create_worker(pool))			break; 		schedule_timeout_interruptible(CREATE_COOLDOWN); 		if (!need_to_create_worker(pool))			break;	} 	del_timer_sync(&pool->mayday_timer);	spin_lock_irq(&pool->lock);	/*	 * This is necessary even after a new worker was just successfully	 * created as @pool->lock was dropped and the new worker might have	 * already become busy.	 */	if (need_to_create_worker(pool))		goto restart;} /** * manage_workers - manage worker pool * @worker: self * * Assume the manager role and manage the worker pool @worker belongs * to.  At any given time, there can be only zero or one manager per * pool.  The exclusion is handled automatically by this function. * * The caller can safely start processing works on false return.  On

Lines 1501–2000 of 5816. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t