atomic_set(&wq->nr_cwqs_to_flush, 0);	INIT_LIST_HEAD(&wq->flusher_queue);	INIT_LIST_HEAD(&wq->flusher_overflow); 	lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);	INIT_LIST_HEAD(&wq->list); 	if (alloc_cwqs(wq) < 0)		goto err; 	for_each_cwq_cpu(cpu, wq) {		struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);		struct global_cwq *gcwq = get_gcwq(cpu); 		BUG_ON((unsigned long)cwq & WORK_STRUCT_FLAG_MASK);		cwq->gcwq = gcwq;		cwq->wq = wq;		cwq->flush_color = -1;		cwq->max_active = max_active;		INIT_LIST_HEAD(&cwq->delayed_works);	} 	if (flags & WQ_RESCUER) {		struct worker *rescuer; 		if (!alloc_mayday_mask(&wq->mayday_mask, GFP_KERNEL))			goto err; 		wq->rescuer = rescuer = alloc_worker();		if (!rescuer)			goto err; 		rescuer->task = kthread_create(rescuer_thread, wq, "%s",					       wq->name);		if (IS_ERR(rescuer->task))			goto err; 		rescuer->task->flags |= PF_THREAD_BOUND;		wake_up_process(rescuer->task);	} 	/*	 * workqueue_lock protects global freeze state and workqueues	 * list.  Grab it, set max_active accordingly and add the new	 * workqueue to workqueues list.	 */	spin_lock(&workqueue_lock); 	if (workqueue_freezing && wq->flags & WQ_FREEZABLE)		for_each_cwq_cpu(cpu, wq)			get_cwq(cpu, wq)->max_active = 0; 	list_add(&wq->list, &workqueues); 	spin_unlock(&workqueue_lock); 	return wq;err:	if (wq) {		free_cwqs(wq);		free_mayday_mask(wq->mayday_mask);		kfree(wq->rescuer);		kfree(wq);	}	return NULL;}EXPORT_SYMBOL_GPL(__alloc_workqueue_key); /** * destroy_workqueue - safely terminate a workqueue * @wq: target workqueue * * Safely destroy a workqueue. All work currently pending will be done first. */void destroy_workqueue(struct workqueue_struct *wq){	unsigned int cpu; 	/* drain it before proceeding with destruction */	drain_workqueue(wq); 	/*	 * wq list is used to freeze wq, remove from list after	 * flushing is complete in case freeze races us.	 */	spin_lock(&workqueue_lock);	list_del(&wq->list);	spin_unlock(&workqueue_lock); 	/* sanity check */	for_each_cwq_cpu(cpu, wq) {		struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq);		int i; 		for (i = 0; i < WORK_NR_COLORS; i++)			BUG_ON(cwq->nr_in_flight[i]);		BUG_ON(cwq->nr_active);		BUG_ON(!list_empty(&cwq->delayed_works));	} 	if (wq->flags & WQ_RESCUER) {		kthread_stop(wq->rescuer->task);		free_mayday_mask(wq->mayday_mask);		kfree(wq->rescuer);	} 	free_cwqs(wq);	kfree(wq);}EXPORT_SYMBOL_GPL(destroy_workqueue); /** * workqueue_set_max_active - adjust max_active of a workqueue * @wq: target workqueue * @max_active: new max_active value. * * Set max_active of @wq to @max_active. * * CONTEXT: * Don't call from IRQ context. */void workqueue_set_max_active(struct workqueue_struct *wq, int max_active){	unsigned int cpu; 	max_active = wq_clamp_max_active(max_active, wq->flags, wq->name); 	spin_lock(&workqueue_lock); 	wq->saved_max_active = max_active; 	for_each_cwq_cpu(cpu, wq) {		struct global_cwq *gcwq = get_gcwq(cpu); 		spin_lock_irq(&gcwq->lock); 		if (!(wq->flags & WQ_FREEZABLE) ||		    !(gcwq->flags & GCWQ_FREEZING))			get_cwq(gcwq->cpu, wq)->max_active = max_active; 		spin_unlock_irq(&gcwq->lock);	} 	spin_unlock(&workqueue_lock);}EXPORT_SYMBOL_GPL(workqueue_set_max_active); /** * workqueue_congested - test whether a workqueue is congested * @cpu: CPU in question * @wq: target workqueue * * Test whether @wq's cpu workqueue for @cpu is congested.  There is * no synchronization around this function and the test result is * unreliable and only useful as advisory hints or for debugging. * * RETURNS: * %true if congested, %false otherwise. */bool workqueue_congested(unsigned int cpu, struct workqueue_struct *wq){	struct cpu_workqueue_struct *cwq = get_cwq(cpu, wq); 	return !list_empty(&cwq->delayed_works);}EXPORT_SYMBOL_GPL(workqueue_congested); /** * work_cpu - return the last known associated cpu for @work * @work: the work of interest * * RETURNS: * CPU number if @work was ever queued.  WORK_CPU_NONE otherwise. */unsigned int work_cpu(struct work_struct *work){	struct global_cwq *gcwq = get_work_gcwq(work); 	return gcwq ? gcwq->cpu : WORK_CPU_NONE;}EXPORT_SYMBOL_GPL(work_cpu); /** * work_busy - test whether a work is currently pending or running * @work: the work to be tested * * Test whether @work is currently pending or running.  There is no * synchronization around this function and the test result is * unreliable and only useful as advisory hints or for debugging. * Especially for reentrant wqs, the pending state might hide the * running state. * * RETURNS: * OR'd bitmask of WORK_BUSY_* bits. */unsigned int work_busy(struct work_struct *work){	struct global_cwq *gcwq = get_work_gcwq(work);	unsigned long flags;	unsigned int ret = 0; 	if (!gcwq)		return false; 	spin_lock_irqsave(&gcwq->lock, flags); 	if (work_pending(work))		ret |= WORK_BUSY_PENDING;	if (find_worker_executing_work(gcwq, work))		ret |= WORK_BUSY_RUNNING; 	spin_unlock_irqrestore(&gcwq->lock, flags); 	return ret;}EXPORT_SYMBOL_GPL(work_busy); /* * CPU hotplug. * * There are two challenges in supporting CPU hotplug.  Firstly, there * are a lot of assumptions on strong associations among work, cwq and * gcwq which make migrating pending and scheduled works very * difficult to implement without impacting hot paths.  Secondly, * gcwqs serve mix of short, long and very long running works making * blocked draining impractical. * * This is solved by allowing a gcwq to be detached from CPU, running * it with unbound (rogue) workers and allowing it to be reattached * later if the cpu comes back online.  A separate thread is created * to govern a gcwq in such state and is called the trustee of the * gcwq. * * Trustee states and their descriptions. * * START	Command state used on startup.  On CPU_DOWN_PREPARE, a *		new trustee is started with this state. * * IN_CHARGE	Once started, trustee will enter this state after *		assuming the manager role and making all existing *		workers rogue.  DOWN_PREPARE waits for trustee to *		enter this state.  After reaching IN_CHARGE, trustee *		tries to execute the pending worklist until it's empty *		and the state is set to BUTCHER, or the state is set *		to RELEASE. * * BUTCHER	Command state which is set by the cpu callback after *		the cpu has went down.  Once this state is set trustee *		knows that there will be no new works on the worklist *		and once the worklist is empty it can proceed to *		killing idle workers. * * RELEASE	Command state which is set by the cpu callback if the *		cpu down has been canceled or it has come online *		again.  After recognizing this state, trustee stops *		trying to drain or butcher and clears ROGUE, rebinds *		all remaining workers back to the cpu and releases *		manager role. * * DONE		Trustee will enter this state after BUTCHER or RELEASE *		is complete. * *          trustee                 CPU                draining *         took over                down               complete * START -----------> IN_CHARGE -----------> BUTCHER -----------> DONE *                        |                     |                  ^ *                        | CPU is back online  v   return workers | *                         ----------------> RELEASE -------------- */ /** * trustee_wait_event_timeout - timed event wait for trustee * @cond: condition to wait for * @timeout: timeout in jiffies * * wait_event_timeout() for trustee to use.  Handles locking and * checks for RELEASE request. * * CONTEXT: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times.  To be used by trustee. * * RETURNS: * Positive indicating left time if @cond is satisfied, 0 if timed * out, -1 if canceled. */#define trustee_wait_event_timeout(cond, timeout) ({			\	long __ret = (timeout);						\	while (!((cond) || (gcwq->trustee_state == TRUSTEE_RELEASE)) &&	\	       __ret) {							\		spin_unlock_irq(&gcwq->lock);				\		__wait_event_timeout(gcwq->trustee_wait, (cond) ||	\			(gcwq->trustee_state == TRUSTEE_RELEASE),	\			__ret);						\		spin_lock_irq(&gcwq->lock);				\	}								\	gcwq->trustee_state == TRUSTEE_RELEASE ? -1 : (__ret);		\}) /** * trustee_wait_event - event wait for trustee * @cond: condition to wait for * * wait_event() for trustee to use.  Automatically handles locking and * checks for CANCEL request. * * CONTEXT: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times.  To be used by trustee. * * RETURNS: * 0 if @cond is satisfied, -1 if canceled. */#define trustee_wait_event(cond) ({					\	long __ret1;							\	__ret1 = trustee_wait_event_timeout(cond, MAX_SCHEDULE_TIMEOUT);\	__ret1 < 0 ? -1 : 0;						\}) static int __cpuinit trustee_thread(void *__gcwq){	struct global_cwq *gcwq = __gcwq;	struct worker *worker;	struct work_struct *work;	struct hlist_node *pos;	long rc;	int i; 	BUG_ON(gcwq->cpu != smp_processor_id()); 	spin_lock_irq(&gcwq->lock);	/*	 * Claim the manager position and make all workers rogue.	 * Trustee must be bound to the target cpu and can't be	 * cancelled.	 */	BUG_ON(gcwq->cpu != smp_processor_id());	rc = trustee_wait_event(!(gcwq->flags & GCWQ_MANAGING_WORKERS));	BUG_ON(rc < 0); 	gcwq->flags |= GCWQ_MANAGING_WORKERS; 	list_for_each_entry(worker, &gcwq->idle_list, entry)		worker->flags |= WORKER_ROGUE; 	for_each_busy_worker(worker, i, pos, gcwq)		worker->flags |= WORKER_ROGUE; 	/*	 * Call schedule() so that we cross rq->lock and thus can	 * guarantee sched callbacks see the rogue flag.  This is	 * necessary as scheduler callbacks may be invoked from other	 * cpus.	 */	spin_unlock_irq(&gcwq->lock);	schedule();	spin_lock_irq(&gcwq->lock); 	/*	 * Sched callbacks are disabled now.  Zap nr_running.  After	 * this, nr_running stays zero and need_more_worker() and	 * keep_working() are always true as long as the worklist is	 * not empty.	 */	atomic_set(get_gcwq_nr_running(gcwq->cpu), 0); 	spin_unlock_irq(&gcwq->lock);	del_timer_sync(&gcwq->idle_timer);	spin_lock_irq(&gcwq->lock); 	/*	 * We're now in charge.  Notify and proceed to drain.  We need	 * to keep the gcwq running during the whole CPU down	 * procedure as other cpu hotunplug callbacks may need to	 * flush currently running tasks.	 */	gcwq->trustee_state = TRUSTEE_IN_CHARGE;	wake_up_all(&gcwq->trustee_wait); 	/*	 * The original cpu is in the process of dying and may go away	 * anytime now.  When that happens, we and all workers would	 * be migrated to other cpus.  Try draining any left work.  We	 * want to get it over with ASAP - spam rescuers, wake up as	 * many idlers as necessary and create new ones till the	 * worklist is empty.  Note that if the gcwq is frozen, there	 * may be frozen works in freezable cwqs.  Don't declare	 * completion while frozen.	 */	while (gcwq->nr_workers != gcwq->nr_idle ||	       gcwq->flags & GCWQ_FREEZING ||	       gcwq->trustee_state == TRUSTEE_IN_CHARGE) {		int nr_works = 0; 		list_for_each_entry(work, &gcwq->worklist, entry) {			send_mayday(work);			nr_works++;		} 		list_for_each_entry(worker, &gcwq->idle_list, entry) {			if (!nr_works--)				break;			wake_up_process(worker->task);		} 		if (need_to_create_worker(gcwq)) {			spin_unlock_irq(&gcwq->lock);			worker = create_worker(gcwq, false);			spin_lock_irq(&gcwq->lock);			if (worker) {				worker->flags |= WORKER_ROGUE;				start_worker(worker);			}		} 		/* give a breather */		if (trustee_wait_event_timeout(false, TRUSTEE_COOLDOWN) < 0)			break;	} 	/*	 * Either all works have been scheduled and cpu is down, or	 * cpu down has already been canceled.  Wait for and butcher	 * all workers till we're canceled.	 */	do {		rc = trustee_wait_event(!list_empty(&gcwq->idle_list));		while (!list_empty(&gcwq->idle_list))			destroy_worker(list_first_entry(&gcwq->idle_list,							struct worker, entry));	} while (gcwq->nr_workers && rc >= 0); 	/*	 * At this point, either draining has completed and no worker	 * is left, or cpu down has been canceled or the cpu is being	 * brought back up.  There shouldn't be any idle one left.	 * Tell the remaining busy ones to rebind once it finishes the	 * currently scheduled works by scheduling the rebind_work.	 */	WARN_ON(!list_empty(&gcwq->idle_list)); 	for_each_busy_worker(worker, i, pos, gcwq) {		struct work_struct *rebind_work = &worker->rebind_work; 		/*		 * Rebind_work may race with future cpu hotplug		 * operations.  Use a separate flag to mark that		 * rebinding is scheduled.		 */		worker->flags |= WORKER_REBIND;		worker->flags &= ~WORKER_ROGUE; 		/* queue rebind_work, wq doesn't matter, use the default one */		if (test_and_set_bit(WORK_STRUCT_PENDING_BIT,				     work_data_bits(rebind_work)))			continue; 		debug_work_activate(rebind_work);		insert_work(get_cwq(gcwq->cpu, system_wq), rebind_work,			    worker->scheduled.next,			    work_color_to_flags(WORK_NO_COLOR));	} 	/* relinquish manager role */	gcwq->flags &= ~GCWQ_MANAGING_WORKERS; 	/* notify completion */	gcwq->trustee = NULL;	gcwq->trustee_state = TRUSTEE_DONE;	wake_up_all(&gcwq->trustee_wait);	spin_unlock_irq(&gcwq->lock);	return 0;} /** * wait_trustee_state - wait for trustee to enter the specified state * @gcwq: gcwq the trustee of interest belongs to * @state: target state to wait for * * Wait for the trustee to reach @state.  DONE is already matched. * * CONTEXT: * spin_lock_irq(gcwq->lock) which may be released and regrabbed * multiple times.  To be used by cpu_callback. */static void __cpuinit wait_trustee_state(struct global_cwq *gcwq, int state)__releases(&gcwq->lock)__acquires(&gcwq->lock){	if (!(gcwq->trustee_state == state ||	      gcwq->trustee_state == TRUSTEE_DONE)) {		spin_unlock_irq(&gcwq->lock);		__wait_event(gcwq->trustee_wait,			     gcwq->trustee_state == state ||			     gcwq->trustee_state == TRUSTEE_DONE);		spin_lock_irq(&gcwq->lock);	}} static int __devinit workqueue_cpu_callback(struct notifier_block *nfb,

Lines 3001–3500 of 3842. Previous lines Load more