8171 lines · 229.8 KB · C
RawDownload
 * * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and * %CPU_DOWN_FAILED.  @cpu is in the same pod of the CPU being hot[un]plugged. * * * If pod affinity can't be adjusted due to memory allocation failure, it falls * back to @wq->dfl_pwq which may not be optimal but is always correct. * * Note that when the last allowed CPU of a pod goes offline for a workqueue * with a cpumask spanning multiple pods, the workers which were already * executing the work items for the workqueue will lose their CPU affinity and * may execute on any CPU. This is similar to how per-cpu workqueues behave on * CPU_DOWN. If a workqueue user wants strict affinity, it's the user's * responsibility to flush the work item from CPU_DOWN_PREPARE. */static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu){	struct pool_workqueue *old_pwq = NULL, *pwq;	struct workqueue_attrs *target_attrs; 	lockdep_assert_held(&wq_pool_mutex); 	if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered)		return; 	/*	 * We don't wanna alloc/free wq_attrs for each wq for each CPU.	 * Let's use a preallocated one.  The following buf is protected by	 * CPU hotplug exclusion.	 */	target_attrs = unbound_wq_update_pwq_attrs_buf; 	copy_workqueue_attrs(target_attrs, wq->unbound_attrs);	wqattrs_actualize_cpumask(target_attrs, wq_unbound_cpumask); 	/* nothing to do if the target cpumask matches the current pwq */	wq_calc_pod_cpumask(target_attrs, cpu);	if (wqattrs_equal(target_attrs, unbound_pwq(wq, cpu)->pool->attrs))		return; 	/* create a new pwq */	pwq = alloc_unbound_pwq(wq, target_attrs);	if (!pwq) {		pr_warn("workqueue: allocation failed while updating CPU pod affinity of \"%s\"\n",			wq->name);		goto use_dfl_pwq;	} 	/* Install the new pwq. */	mutex_lock(&wq->mutex);	old_pwq = install_unbound_pwq(wq, cpu, pwq);	goto out_unlock; use_dfl_pwq:	mutex_lock(&wq->mutex);	pwq = unbound_pwq(wq, -1);	raw_spin_lock_irq(&pwq->pool->lock);	get_pwq(pwq);	raw_spin_unlock_irq(&pwq->pool->lock);	old_pwq = install_unbound_pwq(wq, cpu, pwq);out_unlock:	mutex_unlock(&wq->mutex);	put_pwq_unlocked(old_pwq);} static int alloc_and_link_pwqs(struct workqueue_struct *wq){	bool highpri = wq->flags & WQ_HIGHPRI;	int cpu, ret; 	lockdep_assert_held(&wq_pool_mutex); 	wq->cpu_pwq = alloc_percpu(struct pool_workqueue *);	if (!wq->cpu_pwq)		goto enomem; 	if (!(wq->flags & WQ_UNBOUND)) {		struct worker_pool __percpu *pools; 		if (wq->flags & WQ_BH)			pools = bh_worker_pools;		else			pools = cpu_worker_pools; 		for_each_possible_cpu(cpu) {			struct pool_workqueue **pwq_p;			struct worker_pool *pool; 			pool = &(per_cpu_ptr(pools, cpu)[highpri]);			pwq_p = per_cpu_ptr(wq->cpu_pwq, cpu); 			*pwq_p = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL,						       pool->node);			if (!*pwq_p)				goto enomem; 			init_pwq(*pwq_p, wq, pool); 			mutex_lock(&wq->mutex);			link_pwq(*pwq_p);			mutex_unlock(&wq->mutex);		}		return 0;	} 	if (wq->flags & __WQ_ORDERED) {		struct pool_workqueue *dfl_pwq; 		ret = apply_workqueue_attrs_locked(wq, ordered_wq_attrs[highpri]);		/* there should only be single pwq for ordering guarantee */		dfl_pwq = rcu_access_pointer(wq->dfl_pwq);		WARN(!ret && (wq->pwqs.next != &dfl_pwq->pwqs_node ||			      wq->pwqs.prev != &dfl_pwq->pwqs_node),		     "ordering guarantee broken for workqueue %s\n", wq->name);	} else {		ret = apply_workqueue_attrs_locked(wq, unbound_std_wq_attrs[highpri]);	} 	return ret; enomem:	if (wq->cpu_pwq) {		for_each_possible_cpu(cpu) {			struct pool_workqueue *pwq = *per_cpu_ptr(wq->cpu_pwq, cpu); 			if (pwq)				kmem_cache_free(pwq_cache, pwq);		}		free_percpu(wq->cpu_pwq);		wq->cpu_pwq = NULL;	}	return -ENOMEM;} static int wq_clamp_max_active(int max_active, unsigned int flags,			       const char *name){	if (max_active < 1 || max_active > WQ_MAX_ACTIVE)		pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",			max_active, name, 1, WQ_MAX_ACTIVE); 	return clamp_val(max_active, 1, WQ_MAX_ACTIVE);} /* * Workqueues which may be used during memory reclaim should have a rescuer * to guarantee forward progress. */static int init_rescuer(struct workqueue_struct *wq){	struct worker *rescuer;	char id_buf[WORKER_ID_LEN];	int ret; 	lockdep_assert_held(&wq_pool_mutex); 	if (!(wq->flags & WQ_MEM_RECLAIM))		return 0; 	rescuer = alloc_worker(NUMA_NO_NODE);	if (!rescuer) {		pr_err("workqueue: Failed to allocate a rescuer for wq \"%s\"\n",		       wq->name);		return -ENOMEM;	} 	rescuer->rescue_wq = wq;	format_worker_id(id_buf, sizeof(id_buf), rescuer, NULL); 	rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", id_buf);	if (IS_ERR(rescuer->task)) {		ret = PTR_ERR(rescuer->task);		pr_err("workqueue: Failed to create a rescuer kthread for wq \"%s\": %pe",		       wq->name, ERR_PTR(ret));		kfree(rescuer);		return ret;	} 	wq->rescuer = rescuer; 	/* initial cpumask is consistent with the detached rescuer and unbind_worker() */	if (cpumask_intersects(wq_unbound_cpumask, cpu_active_mask))		kthread_bind_mask(rescuer->task, wq_unbound_cpumask);	else		kthread_bind_mask(rescuer->task, cpu_possible_mask); 	wake_up_process(rescuer->task); 	return 0;} /** * wq_adjust_max_active - update a wq's max_active to the current setting * @wq: target workqueue * * If @wq isn't freezing, set @wq->max_active to the saved_max_active and * activate inactive work items accordingly. If @wq is freezing, clear * @wq->max_active to zero. */static void wq_adjust_max_active(struct workqueue_struct *wq){	bool activated;	int new_max, new_min; 	lockdep_assert_held(&wq->mutex); 	if ((wq->flags & WQ_FREEZABLE) && workqueue_freezing) {		new_max = 0;		new_min = 0;	} else {		new_max = wq->saved_max_active;		new_min = wq->saved_min_active;	} 	if (wq->max_active == new_max && wq->min_active == new_min)		return; 	/*	 * Update @wq->max/min_active and then kick inactive work items if more	 * active work items are allowed. This doesn't break work item ordering	 * because new work items are always queued behind existing inactive	 * work items if there are any.	 */	WRITE_ONCE(wq->max_active, new_max);	WRITE_ONCE(wq->min_active, new_min); 	if (wq->flags & WQ_UNBOUND)		wq_update_node_max_active(wq, -1); 	if (new_max == 0)		return; 	/*	 * Round-robin through pwq's activating the first inactive work item	 * until max_active is filled.	 */	do {		struct pool_workqueue *pwq; 		activated = false;		for_each_pwq(pwq, wq) {			unsigned long irq_flags; 			/* can be called during early boot w/ irq disabled */			raw_spin_lock_irqsave(&pwq->pool->lock, irq_flags);			if (pwq_activate_first_inactive(pwq, true)) {				activated = true;				kick_pool(pwq->pool);			}			raw_spin_unlock_irqrestore(&pwq->pool->lock, irq_flags);		}	} while (activated);} __printf(1, 0)static struct workqueue_struct *__alloc_workqueue(const char *fmt,						  unsigned int flags,						  int max_active, va_list args){	struct workqueue_struct *wq;	size_t wq_size;	int name_len; 	if (flags & WQ_BH) {		if (WARN_ON_ONCE(flags & ~__WQ_BH_ALLOWS))			return NULL;		if (WARN_ON_ONCE(max_active))			return NULL;	} 	/* see the comment above the definition of WQ_POWER_EFFICIENT */	if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)		flags |= WQ_UNBOUND; 	/* allocate wq and format name */	if (flags & WQ_UNBOUND)		wq_size = struct_size(wq, node_nr_active, nr_node_ids + 1);	else		wq_size = sizeof(*wq); 	wq = kzalloc_noprof(wq_size, GFP_KERNEL);	if (!wq)		return NULL; 	if (flags & WQ_UNBOUND) {		wq->unbound_attrs = alloc_workqueue_attrs_noprof();		if (!wq->unbound_attrs)			goto err_free_wq;	} 	name_len = vsnprintf(wq->name, sizeof(wq->name), fmt, args); 	if (name_len >= WQ_NAME_LEN)		pr_warn_once("workqueue: name exceeds WQ_NAME_LEN. Truncating to: %s\n",			     wq->name); 	if (flags & WQ_BH) {		/*		 * BH workqueues always share a single execution context per CPU		 * and don't impose any max_active limit.		 */		max_active = INT_MAX;	} else {		max_active = max_active ?: WQ_DFL_ACTIVE;		max_active = wq_clamp_max_active(max_active, flags, wq->name);	} 	/* init wq */	wq->flags = flags;	wq->max_active = max_active;	wq->min_active = min(max_active, WQ_DFL_MIN_ACTIVE);	wq->saved_max_active = wq->max_active;	wq->saved_min_active = wq->min_active;	mutex_init(&wq->mutex);	atomic_set(&wq->nr_pwqs_to_flush, 0);	INIT_LIST_HEAD(&wq->pwqs);	INIT_LIST_HEAD(&wq->flusher_queue);	INIT_LIST_HEAD(&wq->flusher_overflow);	INIT_LIST_HEAD(&wq->maydays); 	INIT_LIST_HEAD(&wq->list); 	if (flags & WQ_UNBOUND) {		if (alloc_node_nr_active(wq->node_nr_active) < 0)			goto err_free_wq;	} 	/*	 * wq_pool_mutex protects the workqueues list, allocations of PWQs,	 * and the global freeze state.	 */	apply_wqattrs_lock(); 	if (alloc_and_link_pwqs(wq) < 0)		goto err_unlock_free_node_nr_active; 	mutex_lock(&wq->mutex);	wq_adjust_max_active(wq);	mutex_unlock(&wq->mutex); 	list_add_tail_rcu(&wq->list, &workqueues); 	if (wq_online && init_rescuer(wq) < 0)		goto err_unlock_destroy; 	apply_wqattrs_unlock(); 	if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))		goto err_destroy; 	return wq; err_unlock_free_node_nr_active:	apply_wqattrs_unlock();	/*	 * Failed alloc_and_link_pwqs() may leave pending pwq->release_work,	 * flushing the pwq_release_worker ensures that the pwq_release_workfn()	 * completes before calling kfree(wq).	 */	if (wq->flags & WQ_UNBOUND) {		kthread_flush_worker(pwq_release_worker);		free_node_nr_active(wq->node_nr_active);	}err_free_wq:	free_workqueue_attrs(wq->unbound_attrs);	kfree(wq);	return NULL;err_unlock_destroy:	apply_wqattrs_unlock();err_destroy:	destroy_workqueue(wq);	return NULL;} __printf(1, 4)struct workqueue_struct *alloc_workqueue_noprof(const char *fmt,						unsigned int flags,						int max_active, ...){	struct workqueue_struct *wq;	va_list args; 	va_start(args, max_active);	wq = __alloc_workqueue(fmt, flags, max_active, args);	va_end(args);	if (!wq)		return NULL; 	wq_init_lockdep(wq); 	return wq;}EXPORT_SYMBOL_GPL(alloc_workqueue_noprof); #ifdef CONFIG_LOCKDEP__printf(1, 5)struct workqueue_struct *alloc_workqueue_lockdep_map(const char *fmt, unsigned int flags,			    int max_active, struct lockdep_map *lockdep_map, ...){	struct workqueue_struct *wq;	va_list args; 	va_start(args, lockdep_map);	wq = __alloc_workqueue(fmt, flags, max_active, args);	va_end(args);	if (!wq)		return NULL; 	wq->lockdep_map = lockdep_map; 	return wq;}EXPORT_SYMBOL_GPL(alloc_workqueue_lockdep_map);#endif static bool pwq_busy(struct pool_workqueue *pwq){	int i; 	for (i = 0; i < WORK_NR_COLORS; i++)		if (pwq->nr_in_flight[i])			return true; 	if ((pwq != rcu_access_pointer(pwq->wq->dfl_pwq)) && (pwq->refcnt > 1))		return true;	if (!pwq_is_empty(pwq))		return true; 	return false;} /** * destroy_workqueue - safely terminate a workqueue * @wq: target workqueue * * Safely destroy a workqueue. All work currently pending will be done first. * * This function does NOT guarantee that non-pending work that has been * submitted with queue_delayed_work() and similar functions will be done * before destroying the workqueue. The fundamental problem is that, currently, * the workqueue has no way of accessing non-pending delayed_work. delayed_work * is only linked on the timer-side. All delayed_work must, therefore, be * canceled before calling this function. * * TODO: It would be better if the problem described above wouldn't exist and * destroy_workqueue() would cleanly cancel all pending and non-pending * delayed_work. */void destroy_workqueue(struct workqueue_struct *wq){	struct pool_workqueue *pwq;	int cpu; 	/*	 * Remove it from sysfs first so that sanity check failure doesn't	 * lead to sysfs name conflicts.	 */	workqueue_sysfs_unregister(wq); 	/* mark the workqueue destruction is in progress */	mutex_lock(&wq->mutex);	wq->flags |= __WQ_DESTROYING;	mutex_unlock(&wq->mutex); 	/* drain it before proceeding with destruction */	drain_workqueue(wq); 	/* kill rescuer, if sanity checks fail, leave it w/o rescuer */	if (wq->rescuer) {		/* rescuer will empty maydays list before exiting */		kthread_stop(wq->rescuer->task);		kfree(wq->rescuer);		wq->rescuer = NULL;	} 	/*	 * Sanity checks - grab all the locks so that we wait for all	 * in-flight operations which may do put_pwq().	 */	mutex_lock(&wq_pool_mutex);	mutex_lock(&wq->mutex);	for_each_pwq(pwq, wq) {		raw_spin_lock_irq(&pwq->pool->lock);		if (WARN_ON(pwq_busy(pwq))) {			pr_warn("%s: %s has the following busy pwq\n",				__func__, wq->name);			show_pwq(pwq);			raw_spin_unlock_irq(&pwq->pool->lock);			mutex_unlock(&wq->mutex);			mutex_unlock(&wq_pool_mutex);			show_one_workqueue(wq);			return;		}		raw_spin_unlock_irq(&pwq->pool->lock);	}	mutex_unlock(&wq->mutex); 	/*	 * wq list is used to freeze wq, remove from list after

Lines 5501–6000 of 8171. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t