1118 lines · 27.1 KB · C
RawDownload
 	/* Wait for all the helpers to finish. */	wait_for_completion(&ps.completion); 	destroy_work_on_stack(&my_work.pw_work);	padata_works_free(&works);} /* Initialize all percpu queues used by serial workers */static void padata_init_squeues(struct parallel_data *pd){	int cpu;	struct padata_serial_queue *squeue; 	for_each_cpu(cpu, pd->cpumask.cbcpu) {		squeue = per_cpu_ptr(pd->squeue, cpu);		squeue->pd = pd;		INIT_LIST_HEAD(&squeue->serial.list);		spin_lock_init(&squeue->serial.lock);		INIT_WORK(&squeue->work, padata_serial_worker);	}} /* Initialize per-CPU reorder lists */static void padata_init_reorder_list(struct parallel_data *pd){	int cpu;	struct padata_list *list; 	for_each_cpu(cpu, pd->cpumask.pcpu) {		list = per_cpu_ptr(pd->reorder_list, cpu);		INIT_LIST_HEAD(&list->list);		spin_lock_init(&list->lock);	}} /* Allocate and initialize the internal cpumask dependend resources. */static struct parallel_data *padata_alloc_pd(struct padata_shell *ps,					     int offlining_cpu){	struct padata_instance *pinst = ps->pinst;	struct parallel_data *pd; 	pd = kzalloc_obj(struct parallel_data);	if (!pd)		goto err; 	pd->reorder_list = alloc_percpu(struct padata_list);	if (!pd->reorder_list)		goto err_free_pd; 	pd->squeue = alloc_percpu(struct padata_serial_queue);	if (!pd->squeue)		goto err_free_reorder_list; 	pd->ps = ps; 	if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL))		goto err_free_squeue;	if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL))		goto err_free_pcpu; 	cpumask_and(pd->cpumask.pcpu, pinst->cpumask.pcpu, cpu_online_mask);	cpumask_and(pd->cpumask.cbcpu, pinst->cpumask.cbcpu, cpu_online_mask);	if (offlining_cpu >= 0) {		__cpumask_clear_cpu(offlining_cpu, pd->cpumask.pcpu);		__cpumask_clear_cpu(offlining_cpu, pd->cpumask.cbcpu);	} 	padata_init_reorder_list(pd);	padata_init_squeues(pd);	pd->seq_nr = -1;	refcount_set(&pd->refcnt, 1);	pd->cpu = cpumask_first(pd->cpumask.pcpu); 	return pd; err_free_pcpu:	free_cpumask_var(pd->cpumask.pcpu);err_free_squeue:	free_percpu(pd->squeue);err_free_reorder_list:	free_percpu(pd->reorder_list);err_free_pd:	kfree(pd);err:	return NULL;} static void padata_free_pd(struct parallel_data *pd){	free_cpumask_var(pd->cpumask.pcpu);	free_cpumask_var(pd->cpumask.cbcpu);	free_percpu(pd->reorder_list);	free_percpu(pd->squeue);	kfree(pd);} static void __padata_start(struct padata_instance *pinst){	pinst->flags |= PADATA_INIT;} static void __padata_stop(struct padata_instance *pinst){	if (!(pinst->flags & PADATA_INIT))		return; 	pinst->flags &= ~PADATA_INIT; 	synchronize_rcu();} /* Replace the internal control structure with a new one. */static int padata_replace_one(struct padata_shell *ps, int offlining_cpu){	struct parallel_data *pd_new; 	pd_new = padata_alloc_pd(ps, offlining_cpu);	if (!pd_new)		return -ENOMEM; 	ps->opd = rcu_dereference_protected(ps->pd, 1);	rcu_assign_pointer(ps->pd, pd_new); 	return 0;} static int padata_replace(struct padata_instance *pinst, int offlining_cpu){	struct padata_shell *ps;	int err = 0; 	pinst->flags |= PADATA_RESET; 	list_for_each_entry(ps, &pinst->pslist, list) {		err = padata_replace_one(ps, offlining_cpu);		if (err)			break;	} 	synchronize_rcu(); 	list_for_each_entry_continue_reverse(ps, &pinst->pslist, list)		padata_put_pd(ps->opd); 	pinst->flags &= ~PADATA_RESET; 	return err;} /* If cpumask contains no active cpu, we mark the instance as invalid. */static bool padata_validate_cpumask(struct padata_instance *pinst,				    const struct cpumask *cpumask,				    int offlining_cpu){	cpumask_copy(pinst->validate_cpumask, cpu_online_mask); 	/*	 * @offlining_cpu is still in cpu_online_mask, so remove it here for	 * validation.  Using a sub-CPUHP_TEARDOWN_CPU hotplug state where	 * @offlining_cpu wouldn't be in the online mask doesn't work because	 * padata_cpu_offline() can fail but such a state doesn't allow failure.	 */	if (offlining_cpu >= 0)		__cpumask_clear_cpu(offlining_cpu, pinst->validate_cpumask); 	if (!cpumask_intersects(cpumask, pinst->validate_cpumask)) {		pinst->flags |= PADATA_INVALID;		return false;	} 	pinst->flags &= ~PADATA_INVALID;	return true;} static int __padata_set_cpumasks(struct padata_instance *pinst,				 cpumask_var_t pcpumask,				 cpumask_var_t cbcpumask){	int valid;	int err; 	valid = padata_validate_cpumask(pinst, pcpumask, -1);	if (!valid) {		__padata_stop(pinst);		goto out_replace;	} 	valid = padata_validate_cpumask(pinst, cbcpumask, -1);	if (!valid)		__padata_stop(pinst); out_replace:	cpumask_copy(pinst->cpumask.pcpu, pcpumask);	cpumask_copy(pinst->cpumask.cbcpu, cbcpumask); 	err = padata_setup_cpumasks(pinst) ?: padata_replace(pinst, -1); 	if (valid)		__padata_start(pinst); 	return err;} /** * padata_set_cpumask - Sets specified by @cpumask_type cpumask to the value *                      equivalent to @cpumask. * @pinst: padata instance * @cpumask_type: PADATA_CPU_SERIAL or PADATA_CPU_PARALLEL corresponding *                to parallel and serial cpumasks respectively. * @cpumask: the cpumask to use * * Return: 0 on success or negative error code */int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,		       cpumask_var_t cpumask){	struct cpumask *serial_mask, *parallel_mask;	int err = -EINVAL; 	cpus_read_lock();	mutex_lock(&pinst->lock); 	switch (cpumask_type) {	case PADATA_CPU_PARALLEL:		serial_mask = pinst->cpumask.cbcpu;		parallel_mask = cpumask;		break;	case PADATA_CPU_SERIAL:		parallel_mask = pinst->cpumask.pcpu;		serial_mask = cpumask;		break;	default:		 goto out;	} 	err =  __padata_set_cpumasks(pinst, parallel_mask, serial_mask); out:	mutex_unlock(&pinst->lock);	cpus_read_unlock(); 	return err;}EXPORT_SYMBOL(padata_set_cpumask); #ifdef CONFIG_HOTPLUG_CPU static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu){	return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) ||		cpumask_test_cpu(cpu, pinst->cpumask.cbcpu);} static int padata_cpu_online(unsigned int cpu, struct hlist_node *node){	struct padata_instance *pinst;	int ret; 	pinst = hlist_entry_safe(node, struct padata_instance, cpuhp_node);	if (!pinst_has_cpu(pinst, cpu))		return 0; 	mutex_lock(&pinst->lock); 	ret = padata_replace(pinst, -1); 	if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu, -1) &&	    padata_validate_cpumask(pinst, pinst->cpumask.cbcpu, -1))		__padata_start(pinst); 	mutex_unlock(&pinst->lock);	return ret;} static int padata_cpu_offline(unsigned int cpu, struct hlist_node *node){	struct padata_instance *pinst;	int ret; 	pinst = hlist_entry_safe(node, struct padata_instance, cpuhp_node);	if (!pinst_has_cpu(pinst, cpu))		return 0; 	mutex_lock(&pinst->lock); 	if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu, cpu) ||	    !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu, cpu))		__padata_stop(pinst); 	ret = padata_replace(pinst, cpu); 	mutex_unlock(&pinst->lock);	return ret;} static enum cpuhp_state hp_online;#endif static void __padata_free(struct padata_instance *pinst){#ifdef CONFIG_HOTPLUG_CPU	cpuhp_state_remove_instance_nocalls(hp_online, &pinst->cpuhp_node);#endif 	WARN_ON(!list_empty(&pinst->pslist)); 	free_cpumask_var(pinst->cpumask.pcpu);	free_cpumask_var(pinst->cpumask.cbcpu);	free_cpumask_var(pinst->validate_cpumask);	destroy_workqueue(pinst->serial_wq);	destroy_workqueue(pinst->parallel_wq);	kfree(pinst);} #define kobj2pinst(_kobj)					\	container_of(_kobj, struct padata_instance, kobj)#define attr2pentry(_attr)					\	container_of_const(_attr, struct padata_sysfs_entry, attr) static void padata_sysfs_release(struct kobject *kobj){	struct padata_instance *pinst = kobj2pinst(kobj);	__padata_free(pinst);} struct padata_sysfs_entry {	struct attribute attr;	ssize_t (*show)(struct padata_instance *, const struct attribute *, char *);	ssize_t (*store)(struct padata_instance *, const struct attribute *,			 const char *, size_t);}; static ssize_t show_cpumask(struct padata_instance *pinst,			    const struct attribute *attr,  char *buf){	struct cpumask *cpumask;	ssize_t len; 	mutex_lock(&pinst->lock);	if (!strcmp(attr->name, "serial_cpumask"))		cpumask = pinst->cpumask.cbcpu;	else		cpumask = pinst->cpumask.pcpu; 	len = snprintf(buf, PAGE_SIZE, "%*pb\n",		       nr_cpu_ids, cpumask_bits(cpumask));	mutex_unlock(&pinst->lock);	return len < PAGE_SIZE ? len : -EINVAL;} static ssize_t store_cpumask(struct padata_instance *pinst,			     const struct attribute *attr,			     const char *buf, size_t count){	cpumask_var_t new_cpumask;	ssize_t ret;	int mask_type; 	if (!alloc_cpumask_var(&new_cpumask, GFP_KERNEL))		return -ENOMEM; 	ret = bitmap_parse(buf, count, cpumask_bits(new_cpumask),			   nr_cpumask_bits);	if (ret < 0)		goto out; 	mask_type = !strcmp(attr->name, "serial_cpumask") ?		PADATA_CPU_SERIAL : PADATA_CPU_PARALLEL;	ret = padata_set_cpumask(pinst, mask_type, new_cpumask);	if (!ret)		ret = count; out:	free_cpumask_var(new_cpumask);	return ret;} #define PADATA_ATTR_RW(_name, _show_name, _store_name)		\	static const struct padata_sysfs_entry _name##_attr =	\		__ATTR(_name, 0644, _show_name, _store_name)#define PADATA_ATTR_RO(_name, _show_name)			\	static const struct padata_sysfs_entry _name##_attr =	\		__ATTR(_name, 0400, _show_name, NULL) PADATA_ATTR_RW(serial_cpumask, show_cpumask, store_cpumask);PADATA_ATTR_RW(parallel_cpumask, show_cpumask, store_cpumask); /* * Padata sysfs provides the following objects: * serial_cpumask   [RW] - cpumask for serial workers * parallel_cpumask [RW] - cpumask for parallel workers */static const struct attribute *const padata_default_attrs[] = {	&serial_cpumask_attr.attr,	&parallel_cpumask_attr.attr,	NULL,};ATTRIBUTE_GROUPS(padata_default); static ssize_t padata_sysfs_show(struct kobject *kobj,				 struct attribute *attr, char *buf){	const struct padata_sysfs_entry *pentry;	struct padata_instance *pinst;	ssize_t ret = -EIO; 	pinst = kobj2pinst(kobj);	pentry = attr2pentry(attr);	if (pentry->show)		ret = pentry->show(pinst, attr, buf); 	return ret;} static ssize_t padata_sysfs_store(struct kobject *kobj, struct attribute *attr,				  const char *buf, size_t count){	const struct padata_sysfs_entry *pentry;	struct padata_instance *pinst;	ssize_t ret = -EIO; 	pinst = kobj2pinst(kobj);	pentry = attr2pentry(attr);	if (pentry->store)		ret = pentry->store(pinst, attr, buf, count); 	return ret;} static const struct sysfs_ops padata_sysfs_ops = {	.show = padata_sysfs_show,	.store = padata_sysfs_store,}; static const struct kobj_type padata_attr_type = {	.sysfs_ops = &padata_sysfs_ops,	.default_groups = padata_default_groups,	.release = padata_sysfs_release,}; /** * padata_alloc - allocate and initialize a padata instance * @name: used to identify the instance * * Return: new instance on success, NULL on error */struct padata_instance *padata_alloc(const char *name){	struct padata_instance *pinst; 	pinst = kzalloc_obj(struct padata_instance);	if (!pinst)		goto err; 	pinst->parallel_wq = alloc_workqueue("%s_parallel", WQ_UNBOUND, 0,					     name);	if (!pinst->parallel_wq)		goto err_free_inst; 	cpus_read_lock(); 	pinst->serial_wq = alloc_workqueue("%s_serial",					   WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE | WQ_PERCPU,					   1, name);	if (!pinst->serial_wq)		goto err_put_cpus; 	if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL))		goto err_free_serial_wq;	if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL))		goto err_free_p_mask;	if (!alloc_cpumask_var(&pinst->validate_cpumask, GFP_KERNEL))		goto err_free_cb_mask; 	INIT_LIST_HEAD(&pinst->pslist); 	cpumask_copy(pinst->cpumask.pcpu, cpu_possible_mask);	cpumask_copy(pinst->cpumask.cbcpu, cpu_possible_mask); 	if (padata_setup_cpumasks(pinst))		goto err_free_v_mask; 	__padata_start(pinst); 	kobject_init(&pinst->kobj, &padata_attr_type);	mutex_init(&pinst->lock); #ifdef CONFIG_HOTPLUG_CPU	cpuhp_state_add_instance_nocalls_cpuslocked(hp_online,						    &pinst->cpuhp_node);#endif 	cpus_read_unlock(); 	return pinst; err_free_v_mask:	free_cpumask_var(pinst->validate_cpumask);

Lines 501–1000 of 1118. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t