/* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. * GPL v2 and any later version. */#include <linux/cpu.h>#include <linux/err.h>#include <linux/kthread.h>#include <linux/module.h>#include <linux/sched.h>#include <linux/stop_machine.h>#include <linux/syscalls.h>#include <linux/interrupt.h> #include <asm/atomic.h>#include <asm/uaccess.h> /* This controls the threads on each CPU. */enum stopmachine_state {	/* Dummy starting state for thread. */	STOPMACHINE_NONE,	/* Awaiting everyone to be scheduled. */	STOPMACHINE_PREPARE,	/* Disable interrupts. */	STOPMACHINE_DISABLE_IRQ,	/* Run the function */	STOPMACHINE_RUN,	/* Exit */	STOPMACHINE_EXIT,};static enum stopmachine_state state; struct stop_machine_data {	int (*fn)(void *);	void *data;	int fnret;}; /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */static unsigned int num_threads;static atomic_t thread_ack;static DEFINE_MUTEX(lock);/* setup_lock protects refcount, stop_machine_wq and stop_machine_work. */static DEFINE_MUTEX(setup_lock);/* Users of stop_machine. */static int refcount;static struct workqueue_struct *stop_machine_wq;static struct stop_machine_data active, idle;static const struct cpumask *active_cpus;static void *stop_machine_work; static void set_state(enum stopmachine_state newstate){	/* Reset ack counter. */	atomic_set(&thread_ack, num_threads);	smp_wmb();	state = newstate;} /* Last one to ack a state moves to the next state. */static void ack_state(void){	if (atomic_dec_and_test(&thread_ack))		set_state(state + 1);} /* This is the actual function which stops the CPU. It runs * in the context of a dedicated stopmachine workqueue. */static void stop_cpu(struct work_struct *unused){	enum stopmachine_state curstate = STOPMACHINE_NONE;	struct stop_machine_data *smdata = &idle;	int cpu = smp_processor_id();	int err; 	if (!active_cpus) {		if (cpu == cpumask_first(cpu_online_mask))			smdata = &active;	} else {		if (cpumask_test_cpu(cpu, active_cpus))			smdata = &active;	}	/* Simple state machine */	do {		/* Chill out and ensure we re-read stopmachine_state. */		cpu_relax();		if (state != curstate) {			curstate = state;			switch (curstate) {			case STOPMACHINE_DISABLE_IRQ:				local_irq_disable();				hard_irq_disable();				break;			case STOPMACHINE_RUN:				/* On multiple CPUs only a single error code				 * is needed to tell that something failed. */				err = smdata->fn(smdata->data);				if (err)					smdata->fnret = err;				break;			default:				break;			}			ack_state();		}	} while (curstate != STOPMACHINE_EXIT); 	local_irq_enable();} /* Callback for CPUs which aren't supposed to do anything. */static int chill(void *unused){	return 0;} int stop_machine_create(void){	mutex_lock(&setup_lock);	if (refcount)		goto done;	stop_machine_wq = create_rt_workqueue("kstop");	if (!stop_machine_wq)		goto err_out;	stop_machine_work = alloc_percpu(struct work_struct);	if (!stop_machine_work)		goto err_out;done:	refcount++;	mutex_unlock(&setup_lock);	return 0; err_out:	if (stop_machine_wq)		destroy_workqueue(stop_machine_wq);	mutex_unlock(&setup_lock);	return -ENOMEM;}EXPORT_SYMBOL_GPL(stop_machine_create); void stop_machine_destroy(void){	mutex_lock(&setup_lock);	refcount--;	if (refcount)		goto done;	destroy_workqueue(stop_machine_wq);	free_percpu(stop_machine_work);done:	mutex_unlock(&setup_lock);}EXPORT_SYMBOL_GPL(stop_machine_destroy); int __stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus){	struct work_struct *sm_work;	int i, ret; 	/* Set up initial state. */	mutex_lock(&lock);	num_threads = num_online_cpus();	active_cpus = cpus;	active.fn = fn;	active.data = data;	active.fnret = 0;	idle.fn = chill;	idle.data = NULL; 	set_state(STOPMACHINE_PREPARE); 	/* Schedule the stop_cpu work on all cpus: hold this CPU so one	 * doesn't hit this CPU until we're ready. */	get_cpu();	for_each_online_cpu(i) {		sm_work = per_cpu_ptr(stop_machine_work, i);		INIT_WORK(sm_work, stop_cpu);		queue_work_on(i, stop_machine_wq, sm_work);	}	/* This will release the thread on our CPU. */	put_cpu();	flush_workqueue(stop_machine_wq);	ret = active.fnret;	mutex_unlock(&lock);	return ret;} int stop_machine(int (*fn)(void *), void *data, const struct cpumask *cpus){	int ret; 	ret = stop_machine_create();	if (ret)		return ret;	/* No CPUs can come up or down during this. */	get_online_cpus();	ret = __stop_machine(fn, data, cpus);	put_online_cpus();	stop_machine_destroy();	return ret;}EXPORT_SYMBOL_GPL(stop_machine);