// SPDX-License-Identifier: GPL-2.0+//// Scalability test comparing RCU vs other mechanisms// for acquiring references on objects.//// Copyright (C) Google, 2020.//// Author: Joel Fernandes <joel@joelfernandes.org> #define pr_fmt(fmt) fmt #include <linux/atomic.h>#include <linux/bitops.h>#include <linux/completion.h>#include <linux/cpu.h>#include <linux/delay.h>#include <linux/err.h>#include <linux/init.h>#include <linux/interrupt.h>#include <linux/kthread.h>#include <linux/kernel.h>#include <linux/mm.h>#include <linux/module.h>#include <linux/moduleparam.h>#include <linux/notifier.h>#include <linux/percpu.h>#include <linux/rcupdate.h>#include <linux/rcupdate_trace.h>#include <linux/reboot.h>#include <linux/sched.h>#include <linux/spinlock.h>#include <linux/smp.h>#include <linux/stat.h>#include <linux/srcu.h>#include <linux/slab.h>#include <linux/torture.h>#include <linux/types.h> #include "rcu.h" #define SCALE_FLAG "-ref-scale: " #define SCALEOUT(s, x...) \	pr_alert("%s" SCALE_FLAG s, scale_type, ## x) #define VERBOSE_SCALEOUT(s, x...) \	do { if (verbose) pr_alert("%s" SCALE_FLAG s, scale_type, ## x); } while (0) static atomic_t verbose_batch_ctr; #define VERBOSE_SCALEOUT_BATCH(s, x...)							\do {											\	if (verbose &&									\	    (verbose_batched <= 0 ||							\	     !(atomic_inc_return(&verbose_batch_ctr) % verbose_batched))) {		\		schedule_timeout_uninterruptible(1);					\		pr_alert("%s" SCALE_FLAG s, scale_type, ## x);				\	}										\} while (0) #define VERBOSE_SCALEOUT_ERRSTRING(s, x...) \	do { if (verbose) pr_alert("%s" SCALE_FLAG "!!! " s, scale_type, ## x); } while (0) MODULE_LICENSE("GPL");MODULE_AUTHOR("Joel Fernandes (Google) <joel@joelfernandes.org>"); static char *scale_type = "rcu";module_param(scale_type, charp, 0444);MODULE_PARM_DESC(scale_type, "Type of test (rcu, srcu, refcnt, rwsem, rwlock."); torture_param(int, verbose, 0, "Enable verbose debugging printk()s");torture_param(int, verbose_batched, 0, "Batch verbose debugging printk()s"); // Wait until there are multiple CPUs before starting test.torture_param(int, holdoff, IS_BUILTIN(CONFIG_RCU_REF_SCALE_TEST) ? 10 : 0,	      "Holdoff time before test start (s)");// Number of loops per experiment, all readers execute operations concurrently.torture_param(long, loops, 10000, "Number of loops per experiment.");// Number of readers, with -1 defaulting to about 75% of the CPUs.torture_param(int, nreaders, -1, "Number of readers, -1 for 75% of CPUs.");// Number of runs.torture_param(int, nruns, 30, "Number of experiments to run.");// Reader delay in nanoseconds, 0 for no delay.torture_param(int, readdelay, 0, "Read-side delay in nanoseconds."); #ifdef MODULE# define REFSCALE_SHUTDOWN 0#else# define REFSCALE_SHUTDOWN 1#endif torture_param(bool, shutdown, REFSCALE_SHUTDOWN,	      "Shutdown at end of scalability tests."); struct reader_task {	struct task_struct *task;	int start_reader;	wait_queue_head_t wq;	u64 last_duration_ns;}; static struct task_struct *shutdown_task;static wait_queue_head_t shutdown_wq; static struct task_struct *main_task;static wait_queue_head_t main_wq;static int shutdown_start; static struct reader_task *reader_tasks; // Number of readers that are part of the current experiment.static atomic_t nreaders_exp; // Use to wait for all threads to start.static atomic_t n_init;static atomic_t n_started;static atomic_t n_warmedup;static atomic_t n_cooleddown; // Track which experiment is currently running.static int exp_idx; // Operations vector for selecting different types of tests.struct ref_scale_ops {	void (*init)(void);	void (*cleanup)(void);	void (*readsection)(const int nloops);	void (*delaysection)(const int nloops, const int udl, const int ndl);	const char *name;}; static struct ref_scale_ops *cur_ops; static void un_delay(const int udl, const int ndl){	if (udl)		udelay(udl);	if (ndl)		ndelay(ndl);} static void ref_rcu_read_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		rcu_read_lock();		rcu_read_unlock();	}} static void ref_rcu_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		rcu_read_lock();		un_delay(udl, ndl);		rcu_read_unlock();	}} static void rcu_sync_scale_init(void){} static struct ref_scale_ops rcu_ops = {	.init		= rcu_sync_scale_init,	.readsection	= ref_rcu_read_section,	.delaysection	= ref_rcu_delay_section,	.name		= "rcu"}; // Definitions for SRCU ref scale testing.DEFINE_STATIC_SRCU(srcu_refctl_scale);static struct srcu_struct *srcu_ctlp = &srcu_refctl_scale; static void srcu_ref_scale_read_section(const int nloops){	int i;	int idx; 	for (i = nloops; i >= 0; i--) {		idx = srcu_read_lock(srcu_ctlp);		srcu_read_unlock(srcu_ctlp, idx);	}} static void srcu_ref_scale_delay_section(const int nloops, const int udl, const int ndl){	int i;	int idx; 	for (i = nloops; i >= 0; i--) {		idx = srcu_read_lock(srcu_ctlp);		un_delay(udl, ndl);		srcu_read_unlock(srcu_ctlp, idx);	}} static struct ref_scale_ops srcu_ops = {	.init		= rcu_sync_scale_init,	.readsection	= srcu_ref_scale_read_section,	.delaysection	= srcu_ref_scale_delay_section,	.name		= "srcu"}; // Definitions for RCU Tasks ref scale testing: Empty read markers.// These definitions also work for RCU Rude readers.static void rcu_tasks_ref_scale_read_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--)		continue;} static void rcu_tasks_ref_scale_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--)		un_delay(udl, ndl);} static struct ref_scale_ops rcu_tasks_ops = {	.init		= rcu_sync_scale_init,	.readsection	= rcu_tasks_ref_scale_read_section,	.delaysection	= rcu_tasks_ref_scale_delay_section,	.name		= "rcu-tasks"}; // Definitions for RCU Tasks Trace ref scale testing.static void rcu_trace_ref_scale_read_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		rcu_read_lock_trace();		rcu_read_unlock_trace();	}} static void rcu_trace_ref_scale_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		rcu_read_lock_trace();		un_delay(udl, ndl);		rcu_read_unlock_trace();	}} static struct ref_scale_ops rcu_trace_ops = {	.init		= rcu_sync_scale_init,	.readsection	= rcu_trace_ref_scale_read_section,	.delaysection	= rcu_trace_ref_scale_delay_section,	.name		= "rcu-trace"}; // Definitions for reference countstatic atomic_t refcnt; static void ref_refcnt_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		atomic_inc(&refcnt);		atomic_dec(&refcnt);	}} static void ref_refcnt_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		atomic_inc(&refcnt);		un_delay(udl, ndl);		atomic_dec(&refcnt);	}} static struct ref_scale_ops refcnt_ops = {	.init		= rcu_sync_scale_init,	.readsection	= ref_refcnt_section,	.delaysection	= ref_refcnt_delay_section,	.name		= "refcnt"}; // Definitions for rwlockstatic rwlock_t test_rwlock; static void ref_rwlock_init(void){	rwlock_init(&test_rwlock);} static void ref_rwlock_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		read_lock(&test_rwlock);		read_unlock(&test_rwlock);	}} static void ref_rwlock_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		read_lock(&test_rwlock);		un_delay(udl, ndl);		read_unlock(&test_rwlock);	}} static struct ref_scale_ops rwlock_ops = {	.init		= ref_rwlock_init,	.readsection	= ref_rwlock_section,	.delaysection	= ref_rwlock_delay_section,	.name		= "rwlock"}; // Definitions for rwsemstatic struct rw_semaphore test_rwsem; static void ref_rwsem_init(void){	init_rwsem(&test_rwsem);} static void ref_rwsem_section(const int nloops){	int i; 	for (i = nloops; i >= 0; i--) {		down_read(&test_rwsem);		up_read(&test_rwsem);	}} static void ref_rwsem_delay_section(const int nloops, const int udl, const int ndl){	int i; 	for (i = nloops; i >= 0; i--) {		down_read(&test_rwsem);		un_delay(udl, ndl);		up_read(&test_rwsem);	}} static struct ref_scale_ops rwsem_ops = {	.init		= ref_rwsem_init,	.readsection	= ref_rwsem_section,	.delaysection	= ref_rwsem_delay_section,	.name		= "rwsem"}; static void rcu_scale_one_reader(void){	if (readdelay <= 0)		cur_ops->readsection(loops);	else		cur_ops->delaysection(loops, readdelay / 1000, readdelay % 1000);} // Reader kthread.  Repeatedly does empty RCU read-side// critical section, minimizing update-side interference.static intref_scale_reader(void *arg){	unsigned long flags;	long me = (long)arg;	struct reader_task *rt = &(reader_tasks[me]);	u64 start;	s64 duration; 	VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: task started", me);	set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));	set_user_nice(current, MAX_NICE);	atomic_inc(&n_init);	if (holdoff)		schedule_timeout_interruptible(holdoff * HZ);repeat:	VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: waiting to start next experiment on cpu %d", me, smp_processor_id()); 	// Wait for signal that this reader can start.	wait_event(rt->wq, (atomic_read(&nreaders_exp) && smp_load_acquire(&rt->start_reader)) ||			   torture_must_stop()); 	if (torture_must_stop())		goto end; 	// Make sure that the CPU is affinitized appropriately during testing.	WARN_ON_ONCE(smp_processor_id() != me); 	WRITE_ONCE(rt->start_reader, 0);	if (!atomic_dec_return(&n_started))		while (atomic_read_acquire(&n_started))			cpu_relax(); 	VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d started", me, exp_idx);  	// To reduce noise, do an initial cache-warming invocation, check	// in, and then keep warming until everyone has checked in.	rcu_scale_one_reader();	if (!atomic_dec_return(&n_warmedup))		while (atomic_read_acquire(&n_warmedup))			rcu_scale_one_reader();	// Also keep interrupts disabled.  This also has the effect	// of preventing entries into slow path for rcu_read_unlock().	local_irq_save(flags);	start = ktime_get_mono_fast_ns(); 	rcu_scale_one_reader(); 	duration = ktime_get_mono_fast_ns() - start;	local_irq_restore(flags); 	rt->last_duration_ns = WARN_ON_ONCE(duration < 0) ? 0 : duration;	// To reduce runtime-skew noise, do maintain-load invocations until	// everyone is done.	if (!atomic_dec_return(&n_cooleddown))		while (atomic_read_acquire(&n_cooleddown))			rcu_scale_one_reader(); 	if (atomic_dec_and_test(&nreaders_exp))		wake_up(&main_wq); 	VERBOSE_SCALEOUT_BATCH("ref_scale_reader %ld: experiment %d ended, (readers remaining=%d)",				me, exp_idx, atomic_read(&nreaders_exp)); 	if (!torture_must_stop())		goto repeat;end:	torture_kthread_stopping("ref_scale_reader");	return 0;} static void reset_readers(void){	int i;	struct reader_task *rt; 	for (i = 0; i < nreaders; i++) {		rt = &(reader_tasks[i]); 		rt->last_duration_ns = 0;	}} // Print the results of each reader and return the sum of all their durations.static u64 process_durations(int n){	int i;	struct reader_task *rt;	char buf1[64];	char *buf;	u64 sum = 0; 	buf = kmalloc(128 + nreaders * 32, GFP_KERNEL);	if (!buf)		return 0;	buf[0] = 0;	sprintf(buf, "Experiment #%d (Format: <THREAD-NUM>:<Total loop time in ns>)",		exp_idx); 	for (i = 0; i < n && !torture_must_stop(); i++) {		rt = &(reader_tasks[i]);		sprintf(buf1, "%d: %llu\t", i, rt->last_duration_ns); 		if (i % 5 == 0)			strcat(buf, "\n");		strcat(buf, buf1); 		sum += rt->last_duration_ns;	}	strcat(buf, "\n"); 	SCALEOUT("%s\n", buf); 	kfree(buf);	return sum;} // The main_func is the main orchestrator, it performs a bunch of// experiments.  For every experiment, it orders all the readers// involved to start and waits for them to finish the experiment. It// then reads their timestamps and starts the next experiment. Each// experiment progresses from 1 concurrent reader to N of them at which// point all the timestamps are printed.static int main_func(void *arg){

Lines 1–500 of 741. Load more