/* * Read-Copy Update module-based torture test facility * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * * Copyright (C) IBM Corporation, 2005, 2006 * * Authors: Paul E. McKenney <paulmck@us.ibm.com> *	  Josh Triplett <josh@freedesktop.org> * * See also:  Documentation/RCU/torture.txt */#include <linux/types.h>#include <linux/kernel.h>#include <linux/init.h>#include <linux/module.h>#include <linux/kthread.h>#include <linux/err.h>#include <linux/spinlock.h>#include <linux/smp.h>#include <linux/rcupdate.h>#include <linux/interrupt.h>#include <linux/sched.h>#include <linux/atomic.h>#include <linux/bitops.h>#include <linux/completion.h>#include <linux/moduleparam.h>#include <linux/percpu.h>#include <linux/notifier.h>#include <linux/reboot.h>#include <linux/freezer.h>#include <linux/cpu.h>#include <linux/delay.h>#include <linux/stat.h>#include <linux/srcu.h>#include <linux/slab.h>#include <asm/byteorder.h> MODULE_LICENSE("GPL");MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com> and "	      "Josh Triplett <josh@freedesktop.org>"); static int nreaders = -1;	/* # reader threads, defaults to 2*ncpus */static int nfakewriters = 4;	/* # fake writer threads */static int stat_interval;	/* Interval between stats, in seconds. */				/*  Defaults to "only at end of test". */static bool verbose;		/* Print more debug info. */static bool test_no_idle_hz;	/* Test RCU's support for tickless idle CPUs. */static int shuffle_interval = 3; /* Interval between shuffles (in sec)*/static int stutter = 5;		/* Start/stop testing interval (in sec) */static int irqreader = 1;	/* RCU readers from irq (timers). */static int fqs_duration;	/* Duration of bursts (us), 0 to disable. */static int fqs_holdoff;		/* Hold time within burst (us). */static int fqs_stutter = 3;	/* Wait time between bursts (s). */static int n_barrier_cbs;	/* Number of callbacks to test RCU barriers. */static int onoff_interval;	/* Wait time between CPU hotplugs, 0=disable. */static int onoff_holdoff;	/* Seconds after boot before CPU hotplugs. */static int shutdown_secs;	/* Shutdown time (s).  <=0 for no shutdown. */static int stall_cpu;		/* CPU-stall duration (s).  0 for no stall. */static int stall_cpu_holdoff = 10; /* Time to wait until stall (s).  */static int test_boost = 1;	/* Test RCU prio boost: 0=no, 1=maybe, 2=yes. */static int test_boost_interval = 7; /* Interval between boost tests, seconds. */static int test_boost_duration = 4; /* Duration of each boost test, seconds. */static char *torture_type = "rcu"; /* What RCU implementation to torture. */ module_param(nreaders, int, 0444);MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");module_param(nfakewriters, int, 0444);MODULE_PARM_DESC(nfakewriters, "Number of RCU fake writer threads");module_param(stat_interval, int, 0644);MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");module_param(verbose, bool, 0444);MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");module_param(test_no_idle_hz, bool, 0444);MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");module_param(shuffle_interval, int, 0444);MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");module_param(stutter, int, 0444);MODULE_PARM_DESC(stutter, "Number of seconds to run/halt test");module_param(irqreader, int, 0444);MODULE_PARM_DESC(irqreader, "Allow RCU readers from irq handlers");module_param(fqs_duration, int, 0444);MODULE_PARM_DESC(fqs_duration, "Duration of fqs bursts (us)");module_param(fqs_holdoff, int, 0444);MODULE_PARM_DESC(fqs_holdoff, "Holdoff time within fqs bursts (us)");module_param(fqs_stutter, int, 0444);MODULE_PARM_DESC(fqs_stutter, "Wait time between fqs bursts (s)");module_param(n_barrier_cbs, int, 0444);MODULE_PARM_DESC(n_barrier_cbs, "# of callbacks/kthreads for barrier testing");module_param(onoff_interval, int, 0444);MODULE_PARM_DESC(onoff_interval, "Time between CPU hotplugs (s), 0=disable");module_param(onoff_holdoff, int, 0444);MODULE_PARM_DESC(onoff_holdoff, "Time after boot before CPU hotplugs (s)");module_param(shutdown_secs, int, 0444);MODULE_PARM_DESC(shutdown_secs, "Shutdown time (s), zero to disable.");module_param(stall_cpu, int, 0444);MODULE_PARM_DESC(stall_cpu, "Stall duration (s), zero to disable.");module_param(stall_cpu_holdoff, int, 0444);MODULE_PARM_DESC(stall_cpu_holdoff, "Time to wait before starting stall (s).");module_param(test_boost, int, 0444);MODULE_PARM_DESC(test_boost, "Test RCU prio boost: 0=no, 1=maybe, 2=yes.");module_param(test_boost_interval, int, 0444);MODULE_PARM_DESC(test_boost_interval, "Interval between boost tests, seconds.");module_param(test_boost_duration, int, 0444);MODULE_PARM_DESC(test_boost_duration, "Duration of each boost test, seconds.");module_param(torture_type, charp, 0444);MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)"); #define TORTURE_FLAG "-torture:"#define PRINTK_STRING(s) \	do { printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)#define VERBOSE_PRINTK_STRING(s) \	do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)#define VERBOSE_PRINTK_ERRSTRING(s) \	do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0) static char printk_buf[4096]; static int nrealreaders;static struct task_struct *writer_task;static struct task_struct **fakewriter_tasks;static struct task_struct **reader_tasks;static struct task_struct *stats_task;static struct task_struct *shuffler_task;static struct task_struct *stutter_task;static struct task_struct *fqs_task;static struct task_struct *boost_tasks[NR_CPUS];static struct task_struct *shutdown_task;#ifdef CONFIG_HOTPLUG_CPUstatic struct task_struct *onoff_task;#endif /* #ifdef CONFIG_HOTPLUG_CPU */static struct task_struct *stall_task;static struct task_struct **barrier_cbs_tasks;static struct task_struct *barrier_task; #define RCU_TORTURE_PIPE_LEN 10 struct rcu_torture {	struct rcu_head rtort_rcu;	int rtort_pipe_count;	struct list_head rtort_free;	int rtort_mbtest;}; static LIST_HEAD(rcu_torture_freelist);static struct rcu_torture __rcu *rcu_torture_current;static unsigned long rcu_torture_current_version;static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];static DEFINE_SPINLOCK(rcu_torture_lock);static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =	{ 0 };static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =	{ 0 };static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];static atomic_t n_rcu_torture_alloc;static atomic_t n_rcu_torture_alloc_fail;static atomic_t n_rcu_torture_free;static atomic_t n_rcu_torture_mberror;static atomic_t n_rcu_torture_error;static long n_rcu_torture_barrier_error;static long n_rcu_torture_boost_ktrerror;static long n_rcu_torture_boost_rterror;static long n_rcu_torture_boost_failure;static long n_rcu_torture_boosts;static long n_rcu_torture_timers;static long n_offline_attempts;static long n_offline_successes;static long n_online_attempts;static long n_online_successes;static long n_barrier_attempts;static long n_barrier_successes;static struct list_head rcu_torture_removed;static cpumask_var_t shuffle_tmp_mask; static int stutter_pause_test; #if defined(MODULE) || defined(CONFIG_RCU_TORTURE_TEST_RUNNABLE)#define RCUTORTURE_RUNNABLE_INIT 1#else#define RCUTORTURE_RUNNABLE_INIT 0#endifint rcutorture_runnable = RCUTORTURE_RUNNABLE_INIT;module_param(rcutorture_runnable, int, 0444);MODULE_PARM_DESC(rcutorture_runnable, "Start rcutorture at boot"); #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU)#define rcu_can_boost() 1#else /* #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */#define rcu_can_boost() 0#endif /* #else #if defined(CONFIG_RCU_BOOST) && !defined(CONFIG_HOTPLUG_CPU) */ static unsigned long shutdown_time;	/* jiffies to system shutdown. */static unsigned long boost_starttime;	/* jiffies of next boost test start. */DEFINE_MUTEX(boost_mutex);		/* protect setting boost_starttime */					/*  and boost task create/destroy. */static atomic_t barrier_cbs_count;	/* Barrier callbacks registered. */static atomic_t barrier_cbs_invoked;	/* Barrier callbacks invoked. */static wait_queue_head_t *barrier_cbs_wq; /* Coordinate barrier testing. */static DECLARE_WAIT_QUEUE_HEAD(barrier_wq); /* Mediate rmmod and system shutdown.  Concurrent rmmod & shutdown illegal! */ #define FULLSTOP_DONTSTOP 0	/* Normal operation. */#define FULLSTOP_SHUTDOWN 1	/* System shutdown with rcutorture running. */#define FULLSTOP_RMMOD    2	/* Normal rmmod of rcutorture. */static int fullstop = FULLSTOP_RMMOD;/* * Protect fullstop transitions and spawning of kthreads. */static DEFINE_MUTEX(fullstop_mutex); /* Forward reference. */static void rcu_torture_cleanup(void); /* * Detect and respond to a system shutdown. */static intrcutorture_shutdown_notify(struct notifier_block *unused1,			   unsigned long unused2, void *unused3){	mutex_lock(&fullstop_mutex);	if (fullstop == FULLSTOP_DONTSTOP)		fullstop = FULLSTOP_SHUTDOWN;	else		printk(KERN_WARNING /* but going down anyway, so... */		       "Concurrent 'rmmod rcutorture' and shutdown illegal!\n");	mutex_unlock(&fullstop_mutex);	return NOTIFY_DONE;} /* * Absorb kthreads into a kernel function that won't return, so that * they won't ever access module text or data again. */static void rcutorture_shutdown_absorb(char *title){	if (ACCESS_ONCE(fullstop) == FULLSTOP_SHUTDOWN) {		printk(KERN_NOTICE		       "rcutorture thread %s parking due to system shutdown\n",		       title);		schedule_timeout_uninterruptible(MAX_SCHEDULE_TIMEOUT);	}} /* * Allocate an element from the rcu_tortures pool. */static struct rcu_torture *rcu_torture_alloc(void){	struct list_head *p; 	spin_lock_bh(&rcu_torture_lock);	if (list_empty(&rcu_torture_freelist)) {		atomic_inc(&n_rcu_torture_alloc_fail);		spin_unlock_bh(&rcu_torture_lock);		return NULL;	}	atomic_inc(&n_rcu_torture_alloc);	p = rcu_torture_freelist.next;	list_del_init(p);	spin_unlock_bh(&rcu_torture_lock);	return container_of(p, struct rcu_torture, rtort_free);} /* * Free an element to the rcu_tortures pool. */static voidrcu_torture_free(struct rcu_torture *p){	atomic_inc(&n_rcu_torture_free);	spin_lock_bh(&rcu_torture_lock);	list_add_tail(&p->rtort_free, &rcu_torture_freelist);	spin_unlock_bh(&rcu_torture_lock);} struct rcu_random_state {	unsigned long rrs_state;	long rrs_count;}; #define RCU_RANDOM_MULT 39916801  /* prime */#define RCU_RANDOM_ADD	479001701 /* prime */#define RCU_RANDOM_REFRESH 10000 #define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 } /* * Crude but fast random-number generator.  Uses a linear congruential * generator, with occasional help from cpu_clock(). */static unsigned longrcu_random(struct rcu_random_state *rrsp){	if (--rrsp->rrs_count < 0) {		rrsp->rrs_state += (unsigned long)local_clock();		rrsp->rrs_count = RCU_RANDOM_REFRESH;	}	rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;	return swahw32(rrsp->rrs_state);} static voidrcu_stutter_wait(char *title){	while (stutter_pause_test || !rcutorture_runnable) {		if (rcutorture_runnable)			schedule_timeout_interruptible(1);		else			schedule_timeout_interruptible(round_jiffies_relative(HZ));		rcutorture_shutdown_absorb(title);	}} /* * Operations vector for selecting different types of tests. */ struct rcu_torture_ops {	void (*init)(void);	void (*cleanup)(void);	int (*readlock)(void);	void (*read_delay)(struct rcu_random_state *rrsp);	void (*readunlock)(int idx);	int (*completed)(void);	void (*deferred_free)(struct rcu_torture *p);	void (*sync)(void);	void (*call)(struct rcu_head *head, void (*func)(struct rcu_head *rcu));	void (*cb_barrier)(void);	void (*fqs)(void);	int (*stats)(char *page);	int irq_capable;	int can_boost;	char *name;}; static struct rcu_torture_ops *cur_ops; /* * Definitions for rcu torture testing. */ static int rcu_torture_read_lock(void) __acquires(RCU){	rcu_read_lock();	return 0;} static void rcu_read_delay(struct rcu_random_state *rrsp){	const unsigned long shortdelay_us = 200;	const unsigned long longdelay_ms = 50; 	/* We want a short delay sometimes to make a reader delay the grace	 * period, and we want a long delay occasionally to trigger	 * force_quiescent_state. */ 	if (!(rcu_random(rrsp) % (nrealreaders * 2000 * longdelay_ms)))		mdelay(longdelay_ms);	if (!(rcu_random(rrsp) % (nrealreaders * 2 * shortdelay_us)))		udelay(shortdelay_us);#ifdef CONFIG_PREEMPT	if (!preempt_count() && !(rcu_random(rrsp) % (nrealreaders * 20000)))		preempt_schedule();  /* No QS if preempt_disable() in effect */#endif} static void rcu_torture_read_unlock(int idx) __releases(RCU){	rcu_read_unlock();} static int rcu_torture_completed(void){	return rcu_batches_completed();} static voidrcu_torture_cb(struct rcu_head *p){	int i;	struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu); 	if (fullstop != FULLSTOP_DONTSTOP) {		/* Test is ending, just drop callbacks on the floor. */		/* The next initialization will pick up the pieces. */		return;	}	i = rp->rtort_pipe_count;	if (i > RCU_TORTURE_PIPE_LEN)		i = RCU_TORTURE_PIPE_LEN;	atomic_inc(&rcu_torture_wcount[i]);	if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {		rp->rtort_mbtest = 0;		rcu_torture_free(rp);	} else		cur_ops->deferred_free(rp);} static int rcu_no_completed(void){	return 0;} static void rcu_torture_deferred_free(struct rcu_torture *p){	call_rcu(&p->rtort_rcu, rcu_torture_cb);} static struct rcu_torture_ops rcu_ops = {	.init		= NULL,	.cleanup	= NULL,	.readlock	= rcu_torture_read_lock,	.read_delay	= rcu_read_delay,	.readunlock	= rcu_torture_read_unlock,	.completed	= rcu_torture_completed,	.deferred_free	= rcu_torture_deferred_free,	.sync		= synchronize_rcu,	.call		= call_rcu,	.cb_barrier	= rcu_barrier,	.fqs		= rcu_force_quiescent_state,	.stats		= NULL,	.irq_capable	= 1,	.can_boost	= rcu_can_boost(),	.name		= "rcu"}; static void rcu_sync_torture_deferred_free(struct rcu_torture *p){	int i;	struct rcu_torture *rp;	struct rcu_torture *rp1; 	cur_ops->sync();	list_add(&p->rtort_free, &rcu_torture_removed);	list_for_each_entry_safe(rp, rp1, &rcu_torture_removed, rtort_free) {		i = rp->rtort_pipe_count;		if (i > RCU_TORTURE_PIPE_LEN)			i = RCU_TORTURE_PIPE_LEN;		atomic_inc(&rcu_torture_wcount[i]);		if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {			rp->rtort_mbtest = 0;			list_del(&rp->rtort_free);			rcu_torture_free(rp);		}	}} static void rcu_sync_torture_init(void){	INIT_LIST_HEAD(&rcu_torture_removed);} static struct rcu_torture_ops rcu_sync_ops = {	.init		= rcu_sync_torture_init,	.cleanup	= NULL,	.readlock	= rcu_torture_read_lock,	.read_delay	= rcu_read_delay,	.readunlock	= rcu_torture_read_unlock,	.completed	= rcu_torture_completed,	.deferred_free	= rcu_sync_torture_deferred_free,	.sync		= synchronize_rcu,	.call		= NULL,	.cb_barrier	= NULL,	.fqs		= rcu_force_quiescent_state,	.stats		= NULL,	.irq_capable	= 1,	.can_boost	= rcu_can_boost(),	.name		= "rcu_sync"}; static struct rcu_torture_ops rcu_expedited_ops = {	.init		= rcu_sync_torture_init,	.cleanup	= NULL,	.readlock	= rcu_torture_read_lock,	.read_delay	= rcu_read_delay,  /* just reuse rcu's version. */	.readunlock	= rcu_torture_read_unlock,	.completed	= rcu_no_completed,	.deferred_free	= rcu_sync_torture_deferred_free,	.sync		= synchronize_rcu_expedited,	.call		= NULL,	.cb_barrier	= NULL,	.fqs		= rcu_force_quiescent_state,	.stats		= NULL,	.irq_capable	= 1,	.can_boost	= rcu_can_boost(),

Lines 1–500 of 2141. Load more