// SPDX-License-Identifier: GPL-2.0+//// Scalability test comparing RCU vs other mechanisms// for acquiring references on objects.//// Copyright (C) Google, 2020.//// Author: Joel Fernandes <joel@joelfernandes.org> #define pr_fmt(fmt) fmt #include <linux/atomic.h>#include <linux/bitops.h>#include <linux/completion.h>#include <linux/cpu.h>#include <linux/delay.h>#include <linux/err.h>#include <linux/init.h>#include <linux/interrupt.h>#include <linux/kthread.h>#include <linux/kernel.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/moduleparam.h>#include <linux/notifier.h>#include <linux/percpu.h>#include <linux/rcupdate.h>#include <linux/rcupdate_trace.h>#include <linux/reboot.h>#include <linux/sched.h>#include <linux/seq_buf.h>#include <linux/spinlock.h>#include <linux/smp.h>#include <linux/stat.h>#include <linux/srcu.h>#include <linux/slab.h>#include <linux/torture.h>#include <linux/types.h> #include "rcu.h" #define SCALE_FLAG "-ref-scale: " #define SCALEOUT(s, x...) \	pr_alert("%s" SCALE_FLAG s, scale_type, ## x) #define VERBOSE_SCALEOUT(s, x...) \	do { \		if (verbose) \			pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \	} while (0) static atomic_t verbose_batch_ctr; #define VERBOSE_SCALEOUT_BATCH(s, x...)							\do {											\	if (verbose &&									\	    (verbose_batched <= 0 ||							\	     !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) {		\		schedule_timeout_uninterruptible(1);					\		pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x);			\	}										\} while (0) #define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x) MODULE_DESCRIPTION("Scalability test for object reference mechanisms");MODULE_LICENSE("GPL");MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>"); static char *scale_type = "rcu";module_param(scale_type, charp, 0444);MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock."); torture_param(int, verbose, 0, "Enable verbose debugging printk()s");torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s"); // Number of seconds to extend warm-up and cool-down for multiple guest OSestorture_param(long, guest_os_delay, 0,	      "Number of seconds to extend warm-up/cool-down for multiple guest OSes.");// Wait until there are multiple CPUs before starting test.torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,	      "Holdoff time before test start (s)");// Number of typesafe_lookup structures, that is, the degree of concurrency.torture_param(long, lookup_instances, 0, "Number of typesafe_lookup structures.");// Number of loops per experiment, all readers execute operations concurrently.torture_param(long, loops, 10000, "Number of loops per experiment.");// Number of readers, with -1 defaulting to about 75% of the CPUs.torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");// Number of runs.torture_param(int, nruns, 30, "Number of experiments to run.");// Reader delay in nanoseconds, 0 for no delay.torture_param(int, readdelay, 0, "Read-side delay in nanoseconds."); #ifdef MODULE# define REFSCALE_SHUTDOWN 0#else# define REFSCALE_SHUTDOWN 1#endif torture_param(bool, shutdown, REFSCALE_SHUTDOWN,	      "Shutdown at end of scalability tests."); struct reader_task {	struct task_struct *task;	int start_reader;	wait_queue_head_t wq;	u64 last_duration_ns;}; static struct task_struct *shutdown_task;static wait_queue_head_t shutdown_wq; static struct task_struct *main_task;static wait_queue_head_t main_wq;static int shutdown_start; static struct reader_task *reader_tasks; // Number of readers that are part of the current experiment.static atomic_t nreaders_exp; // Use to wait for all threads to start.static atomic_t n_init;static atomic_t n_started;static atomic_t n_warmedup;static atomic_t n_cooleddown; // Track which experiment is currently running.static int exp_idx; // Operations vector for selecting different types of tests.struct ref_scale_ops {	bool (*init)(void);	void (*cleanup)(void);	void (*readsection)(const int nloops);	void (*delaysection)(const int nloops, const int udl, const int ndl);	const char *name;}; static const struct ref_scale_ops *cur_ops; static void un_delay(const int udl, const int ndl){	if (udl)		udelay(udl);	if (ndl)		ndelay(ndl);} static void ref_rcu_read_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		rcu_read_lock();		rcu_read_unlock();	}} static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		rcu_read_lock();		un_delay(udl, ndl);		rcu_read_unlock();	}} static bool rcu_sync_scale_init(void){	return true;} static const struct ref_scale_ops rcu_ops = {	.init		= rcu_sync_scale_init,	.readsection	= ref_rcu_read_section,	.delaysection	= ref_rcu_delay_section,	.name		= "rcu"}; // Definitions for SRCU ref scale testing.DEFINE_STATIC_SRCU(srcu_refctl_scale);static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale; static void srcu_ref_scale_read_section(const int nloops){	int i;	int idx; 	for (i = nloops; i >= 0; i--) {		idx = srcu_read_lock(srcu_ctlp);		srcu_read_unlock(srcu_ctlp, idx);	}} static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl){	int i;	int idx; 	for (i = nloops; i >= 0; i--) {		idx = srcu_read_lock(srcu_ctlp);		un_delay(udl, ndl);		srcu_read_unlock(srcu_ctlp, idx);	}} static const struct ref_scale_ops srcu_ops = {	.init		= rcu_sync_scale_init,	.readsection	= srcu_ref_scale_read_section,	.delaysection	= srcu_ref_scale_delay_section,	.name		= "srcu"}; static void srcu_lite_ref_scale_read_section(const int nloops){	int i;	int idx; 	for (i = nloops; i >= 0; i--) {		idx = srcu_read_lock_lite(srcu_ctlp);		srcu_read_unlock_lite(srcu_ctlp, idx);	}} static void srcu_lite_ref_scale_delay_section(const int nloops, const int udl, const int ndl){	int i;	int idx; 	for (i = nloops; i >= 0; i--) {		idx = srcu_read_lock_lite(srcu_ctlp);		un_delay(udl, ndl);		srcu_read_unlock_lite(srcu_ctlp, idx);	}} static const struct ref_scale_ops srcu_lite_ops = {	.init		= rcu_sync_scale_init,	.readsection	= srcu_lite_ref_scale_read_section,	.delaysection	= srcu_lite_ref_scale_delay_section,	.name		= "srcu-lite"}; #ifdef CONFIG_TASKS_RCU // Definitions for RCU Tasks ref scale testing: Empty read markers.// These definitions also work for RCU Rude readers.static void rcu_tasks_ref_scale_read_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--)		continue;} static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--)		un_delay(udl, ndl);} static const struct ref_scale_ops rcu_tasks_ops = {	.init		= rcu_sync_scale_init,	.readsection	= rcu_tasks_ref_scale_read_section,	.delaysection	= rcu_tasks_ref_scale_delay_section,	.name		= "rcu-tasks"}; #define RCU_TASKS_OPS &rcu_tasks_ops, #else // #ifdef CONFIG_TASKS_RCU #define RCU_TASKS_OPS #endif // #else // #ifdef CONFIG_TASKS_RCU #ifdef CONFIG_TASKS_TRACE_RCU // Definitions for RCU Tasks Trace ref scale testing.static void rcu_trace_ref_scale_read_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		rcu_read_lock_trace();		rcu_read_unlock_trace();	}} static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		rcu_read_lock_trace();		un_delay(udl, ndl);		rcu_read_unlock_trace();	}} static const struct ref_scale_ops rcu_trace_ops = {	.init		= rcu_sync_scale_init,	.readsection	= rcu_trace_ref_scale_read_section,	.delaysection	= rcu_trace_ref_scale_delay_section,	.name		= "rcu-trace"}; #define RCU_TRACE_OPS &rcu_trace_ops, #else // #ifdef CONFIG_TASKS_TRACE_RCU #define RCU_TRACE_OPS #endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU // Definitions for reference countstatic atomic_t refcnt; static void ref_refcnt_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		atomic_inc(&refcnt);		atomic_dec(&refcnt);	}} static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		atomic_inc(&refcnt);		un_delay(udl, ndl);		atomic_dec(&refcnt);	}} static const struct ref_scale_ops refcnt_ops = {	.init		= rcu_sync_scale_init,	.readsection	= ref_refcnt_section,	.delaysection	= ref_refcnt_delay_section,	.name		= "refcnt"}; // Definitions for rwlockstatic rwlock_t test_rwlock; static bool ref_rwlock_init(void){	rwlock_init(&test_rwlock);	return true;} static void ref_rwlock_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		read_lock(&test_rwlock);		read_unlock(&test_rwlock);	}} static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		read_lock(&test_rwlock);		un_delay(udl, ndl);		read_unlock(&test_rwlock);	}} static const struct ref_scale_ops rwlock_ops = {	.init		= ref_rwlock_init,	.readsection	= ref_rwlock_section,	.delaysection	= ref_rwlock_delay_section,	.name		= "rwlock"}; // Definitions for rwsemstatic struct rw_semaphore test_rwsem; static bool ref_rwsem_init(void){	init_rwsem(&test_rwsem);	return true;} static void ref_rwsem_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		down_read(&test_rwsem);		up_read(&test_rwsem);	}} static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		down_read(&test_rwsem);		un_delay(udl, ndl);		up_read(&test_rwsem);	}} static const struct ref_scale_ops rwsem_ops = {	.init		= ref_rwsem_init,	.readsection	= ref_rwsem_section,	.delaysection	= ref_rwsem_delay_section,	.name		= "rwsem"}; // Definitions for global spinlockstatic DEFINE_RAW_SPINLOCK(test_lock); static void ref_lock_section(const int nloops){	int i; 	preempt_disable();	for (i = nloops; i >= 0; i--) {		raw_spin_lock(&test_lock);		raw_spin_unlock(&test_lock);	}	preempt_enable();} static void ref_lock_delay_section(const int nloops, const int udl, const int ndl){	int i; 	preempt_disable();	for (i = nloops; i >= 0; i--) {		raw_spin_lock(&test_lock);		un_delay(udl, ndl);		raw_spin_unlock(&test_lock);	}	preempt_enable();} static const struct ref_scale_ops lock_ops = {	.readsection	= ref_lock_section,	.delaysection	= ref_lock_delay_section,	.name		= "lock"}; // Definitions for global irq-save spinlock static void ref_lock_irq_section(const int nloops){	unsigned long flags;	int i; 	preempt_disable();	for (i = nloops; i >= 0; i--) {		raw_spin_lock_irqsave(&test_lock, flags);		raw_spin_unlock_irqrestore(&test_lock, flags);	}	preempt_enable();} static void ref_lock_irq_delay_section(const int nloops, const int udl, const int ndl){	unsigned long flags;	int i; 	preempt_disable();	for (i = nloops; i >= 0; i--) {		raw_spin_lock_irqsave(&test_lock, flags);		un_delay(udl, ndl);		raw_spin_unlock_irqrestore(&test_lock, flags);	}	preempt_enable();} static const struct ref_scale_ops lock_irq_ops = {	.readsection	= ref_lock_irq_section,	.delaysection	= ref_lock_irq_delay_section,	.name		= "lock-irq"}; // Definitions acquire-release.static DEFINE_PER_CPU(unsigned long, test_acqrel); static void ref_acqrel_section(const int nloops){

Lines 1–500 of 1221. Load more