// SPDX-License-Identifier: GPL-2.0+//// Scalability test comparing RCU vs other mechanisms// for acquiring references on objects.//// Copyright (C) Google, 2020.//// Author: Joel Fernandes <joel@joelfernandes.org> #define pr_fmt(fmt) fmt #include <linux/atomic.h>#include <linux/bitops.h>#include <linux/completion.h>#include <linux/cpu.h>#include <linux/delay.h>#include <linux/err.h>#include <linux/init.h>#include <linux/interrupt.h>#include <linux/kthread.h>#include <linux/kernel.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/moduleparam.h>#include <linux/notifier.h>#include <linux/percpu.h>#include <linux/rcupdate.h>#include <linux/rcupdate_trace.h>#include <linux/reboot.h>#include <linux/sched.h>#include <linux/seq_buf.h>#include <linux/spinlock.h>#include <linux/smp.h>#include <linux/stat.h>#include <linux/srcu.h>#include <linux/slab.h>#include <linux/torture.h>#include <linux/types.h>#include <linux/sched/clock.h> #include "rcu.h" #define SCALE_FLAG "-ref-scale: " #define SCALEOUT(s, x...) \	pr_alert("%s" SCALE_FLAG s, scale_type, ## x) #define VERBOSE_SCALEOUT(s, x...) \	do { \		if (verbose) \			pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x); \	} while (0) static atomic_t verbose_batch_ctr; #define VERBOSE_SCALEOUT_BATCH(s, x...)							\do {											\	if (verbose &&									\	    (verbose_batched <= 0 ||							\	     !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) {		\		schedule_timeout_uninterruptible(1);					\		pr_alert("%s" SCALE_FLAG s "\n", scale_type, ## x);			\	}										\} while (0) #define SCALEOUT_ERRSTRING(s, x...) pr_alert("%s" SCALE_FLAG "!!! " s "\n", scale_type, ## x) MODULE_DESCRIPTION("Scalability test for object reference mechanisms");MODULE_LICENSE("GPL");MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>"); static char *scale_type = "rcu";module_param(scale_type, charp, 0444);MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock."); torture_param(int, verbose, 0, "Enable verbose debugging printk()s");torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s"); // Number of seconds to extend warm-up and cool-down for multiple guest OSestorture_param(long, guest_os_delay, 0,	      "Number of seconds to extend warm-up/cool-down for multiple guest OSes.");// Wait until there are multiple CPUs before starting test.torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,	      "Holdoff time before test start (s)");// Number of typesafe_lookup structures, that is, the degree of concurrency.torture_param(long, lookup_instances, 0, "Number of typesafe_lookup structures.");// Number of loops per experiment, all readers execute operations concurrently.torture_param(int, loops, 10000, "Number of loops per experiment.");// Number of readers, with -1 defaulting to about 75% of the CPUs.torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");// Number of runs.torture_param(int, nruns, 30, "Number of experiments to run.");// Reader delay in nanoseconds, 0 for no delay.torture_param(int, readdelay, 0, "Read-side delay in nanoseconds.");// Maximum shutdown delay in seconds, or zero for no shutdown.torture_param(int, shutdown_secs, !IS_MODULE(CONFIG_REPRO_TEST) * 300,	      "Shutdown at end of scalability tests or at specified timeout (s)."); struct reader_task {	struct task_struct *task;	int start_reader;	wait_queue_head_t wq;	u64 last_duration_ns;}; static struct task_struct *main_task;static wait_queue_head_t main_wq; static struct reader_task *reader_tasks; // Number of readers that are part of the current experiment.static atomic_t nreaders_exp; // Use to wait for all threads to start.static atomic_t n_init;static atomic_t n_started;static atomic_t n_warmedup;static atomic_t n_cooleddown; // Track which experiment is currently running.static int exp_idx; // Operations vector for selecting different types of tests.struct ref_scale_ops {	bool (*init)(void);	void (*cleanup)(void);	void (*readsection)(const int nloops);	void (*delaysection)(const int nloops, const int udl, const int ndl);	bool enable_irqs;	const char *name;}; static const struct ref_scale_ops *cur_ops; static void un_delay(const int udl, const int ndl){	if (udl)		udelay(udl);	if (ndl)		ndelay(ndl);} static void ref_rcu_read_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		rcu_read_lock();		rcu_read_unlock();	}} static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		rcu_read_lock();		un_delay(udl, ndl);		rcu_read_unlock();	}} static bool rcu_sync_scale_init(void){	return true;} static const struct ref_scale_ops rcu_ops = {	.init		= rcu_sync_scale_init,	.readsection	= ref_rcu_read_section,	.delaysection	= ref_rcu_delay_section,	.name		= "rcu"}; // Definitions for SRCU ref scale testing.DEFINE_STATIC_SRCU(srcu_refctl_scale);DEFINE_STATIC_SRCU_FAST(srcu_fast_refctl_scale);DEFINE_STATIC_SRCU_FAST_UPDOWN(srcu_fast_updown_refctl_scale);static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale; static void srcu_ref_scale_read_section(const int nloops){	int i;	int idx; 	for (i = nloops; i >= 0; i--) {		idx = srcu_read_lock(srcu_ctlp);		srcu_read_unlock(srcu_ctlp, idx);	}} static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl){	int i;	int idx; 	for (i = nloops; i >= 0; i--) {		idx = srcu_read_lock(srcu_ctlp);		un_delay(udl, ndl);		srcu_read_unlock(srcu_ctlp, idx);	}} static const struct ref_scale_ops srcu_ops = {	.init		= rcu_sync_scale_init,	.readsection	= srcu_ref_scale_read_section,	.delaysection	= srcu_ref_scale_delay_section,	.name		= "srcu"}; static bool srcu_fast_sync_scale_init(void){	srcu_ctlp = &srcu_fast_refctl_scale;	return true;} static void srcu_fast_ref_scale_read_section(const int nloops){	int i;	struct srcu_ctr __percpu *scp; 	for (i = nloops; i >= 0; i--) {		scp = srcu_read_lock_fast(srcu_ctlp);		srcu_read_unlock_fast(srcu_ctlp, scp);	}} static void srcu_fast_ref_scale_delay_section(const int nloops, const int udl, const int ndl){	int i;	struct srcu_ctr __percpu *scp; 	for (i = nloops; i >= 0; i--) {		scp = srcu_read_lock_fast(srcu_ctlp);		un_delay(udl, ndl);		srcu_read_unlock_fast(srcu_ctlp, scp);	}} static const struct ref_scale_ops srcu_fast_ops = {	.init		= srcu_fast_sync_scale_init,	.readsection	= srcu_fast_ref_scale_read_section,	.delaysection	= srcu_fast_ref_scale_delay_section,	.name		= "srcu-fast"}; static bool srcu_fast_updown_sync_scale_init(void){	srcu_ctlp = &srcu_fast_updown_refctl_scale;	return true;} static void srcu_fast_updown_ref_scale_read_section(const int nloops){	int i;	struct srcu_ctr __percpu *scp; 	for (i = nloops; i >= 0; i--) {		scp = srcu_read_lock_fast_updown(srcu_ctlp);		srcu_read_unlock_fast_updown(srcu_ctlp, scp);	}} static void srcu_fast_updown_ref_scale_delay_section(const int nloops, const int udl, const int ndl){	int i;	struct srcu_ctr __percpu *scp; 	for (i = nloops; i >= 0; i--) {		scp = srcu_read_lock_fast_updown(srcu_ctlp);		un_delay(udl, ndl);		srcu_read_unlock_fast_updown(srcu_ctlp, scp);	}} static const struct ref_scale_ops srcu_fast_updown_ops = {	.init		= srcu_fast_updown_sync_scale_init,	.readsection	= srcu_fast_updown_ref_scale_read_section,	.delaysection	= srcu_fast_updown_ref_scale_delay_section,	.name		= "srcu-fast-updown"}; #ifdef CONFIG_TASKS_RCU // Definitions for RCU Tasks ref scale testing: Empty read markers.// These definitions also work for RCU Rude readers.static void rcu_tasks_ref_scale_read_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--)		continue;} static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--)		un_delay(udl, ndl);} static const struct ref_scale_ops rcu_tasks_ops = {	.init		= rcu_sync_scale_init,	.readsection	= rcu_tasks_ref_scale_read_section,	.delaysection	= rcu_tasks_ref_scale_delay_section,	.name		= "rcu-tasks"}; #define RCU_TASKS_OPS &rcu_tasks_ops, #else // #ifdef CONFIG_TASKS_RCU #define RCU_TASKS_OPS #endif // #else // #ifdef CONFIG_TASKS_RCU #ifdef CONFIG_TASKS_TRACE_RCU // Definitions for RCU Tasks Trace ref scale testing.static void rcu_trace_ref_scale_read_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		rcu_read_lock_trace();		rcu_read_unlock_trace();	}} static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		rcu_read_lock_trace();		un_delay(udl, ndl);		rcu_read_unlock_trace();	}} static const struct ref_scale_ops rcu_trace_ops = {	.init		= rcu_sync_scale_init,	.readsection	= rcu_trace_ref_scale_read_section,	.delaysection	= rcu_trace_ref_scale_delay_section,	.name		= "rcu-trace"}; #define RCU_TRACE_OPS &rcu_trace_ops, #else // #ifdef CONFIG_TASKS_TRACE_RCU #define RCU_TRACE_OPS #endif // #else // #ifdef CONFIG_TASKS_TRACE_RCU // Definitions for reference countstatic atomic_t refcnt; // Definitions acquire-release.static DEFINE_PER_CPU(unsigned long, test_acqrel); static void ref_refcnt_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		atomic_inc(&refcnt);		atomic_dec(&refcnt);	}} static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		atomic_inc(&refcnt);		un_delay(udl, ndl);		atomic_dec(&refcnt);	}} static const struct ref_scale_ops refcnt_ops = {	.init		= rcu_sync_scale_init,	.readsection	= ref_refcnt_section,	.delaysection	= ref_refcnt_delay_section,	.name		= "refcnt"}; static void ref_percpuinc_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		this_cpu_inc(test_acqrel);		this_cpu_dec(test_acqrel);	}} static void ref_percpuinc_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		this_cpu_inc(test_acqrel);		un_delay(udl, ndl);		this_cpu_dec(test_acqrel);	}} static const struct ref_scale_ops percpuinc_ops = {	.init		= rcu_sync_scale_init,	.readsection	= ref_percpuinc_section,	.delaysection	= ref_percpuinc_delay_section,	.name		= "percpuinc"}; // Note that this can lose counts in preemptible kernels.static void ref_incpercpu_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		unsigned long *tap = this_cpu_ptr(&test_acqrel); 		WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);		WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);	}} static void ref_incpercpu_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		unsigned long *tap = this_cpu_ptr(&test_acqrel); 		WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);		un_delay(udl, ndl);		WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);	}} static const struct ref_scale_ops incpercpu_ops = {	.init		= rcu_sync_scale_init,	.readsection	= ref_incpercpu_section,	.delaysection	= ref_incpercpu_delay_section,	.name		= "incpercpu"}; static void ref_incpercpupreempt_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		unsigned long *tap; 		preempt_disable();		tap = this_cpu_ptr(&test_acqrel);		WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);		WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);		preempt_enable();	}} static void ref_incpercpupreempt_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		unsigned long *tap; 		preempt_disable();		tap = this_cpu_ptr(&test_acqrel);		WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);		un_delay(udl, ndl);		WRITE_ONCE(*tap, READ_ONCE(*tap) - 1);		preempt_enable();	}} static const struct ref_scale_ops incpercpupreempt_ops = {	.init		= rcu_sync_scale_init,	.readsection	= ref_incpercpupreempt_section,	.delaysection	= ref_incpercpupreempt_delay_section,	.name		= "incpercpupreempt"}; static void ref_incpercpubh_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		unsigned long *tap; 		local_bh_disable();		tap = this_cpu_ptr(&test_acqrel);		WRITE_ONCE(*tap, READ_ONCE(*tap) + 1);

Lines 1–500 of 1592. Load more