Merge branches 'doc.2020.02.27a', 'fixes.2020.03.21a', 'kfree_rcu.2020.02.20a', 'locktorture.2020.02.20a', 'ovld.2020.02.20a', 'rcu-tasks.2020.02.20a', 'srcu.2020.02.20a' and 'torture.2020.02.20a' into HEAD

doc.2020.02.27a: Documentation updates. fixes.2020.03.21a: Miscellaneous fixes. kfree_rcu.2020.02.20a: Updates to kfree_rcu(). locktorture.2020.02.20a: Lock torture-test updates. ovld.2020.02.20a: Updates to callback-overload handling. rcu-tasks.2020.02.20a: RCU-tasks updates. srcu.2020.02.20a: SRCU updates. torture.2020.02.20a: Torture-test updates.

Paul E. McKenney committed 6 years ago

Commit aa93ec68 parents 8149b5c 127e298 6137079 28e09a2 b692dc4 90ba11b 7104260 a14493529 files · +644 −203Browse files

Unified · Split

29 files

Documentation/admin-guide/kernel-parameters.txt +19 −0
fs/nfs/dir.c +1 −1
include/linux/rculist.h +2 −2
include/linux/rcutiny.h +1 −0
include/linux/rcutree.h +1 −0
include/linux/timer.h +1 −1
include/trace/events/rcu.h +29 −0
kernel/locking/locktorture.c +8 −7
kernel/locking/rtmutex.c +1 −1
kernel/rcu/Makefile +4 −0
kernel/rcu/rcu.h +21 −2
kernel/rcu/rcu_segcblist.c +1 −3
kernel/rcu/rcuperf.c +12 −2
kernel/rcu/rcutorture.c +50 −17
kernel/rcu/srcutree.c +10 −8
kernel/rcu/tree.c +339 −113
kernel/rcu/tree.h +4 −0
kernel/rcu/tree_exp.h +9 −4
kernel/rcu/tree_plugin.h +16 −9
kernel/rcu/tree_stall.h +23 −18
kernel/rcu/update.c +23 −5
kernel/time/timer.c +4 −3
kernel/torture.c +25 −4
tools/testing/selftests/rcutorture/bin/functions.sh +1 −1
tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh +2 −0
tools/testing/selftests/rcutorture/bin/kvm-recheck.sh +16 −1
tools/testing/selftests/rcutorture/bin/kvm.sh +1 −1
tools/testing/selftests/rcutorture/configs/rcu/CFcommon +2 −0
tools/testing/selftests/rcutorture/configs/rcu/TREE10 +18 −0

Documentation/admin-guide/kernel-parameters.txt+19−0

@@ -3980,6 +3980,15 @@ · 3979 unmodified lines
3980	Set threshold of queued RCU callbacks below which	3980	Set threshold of queued RCU callbacks below which
3981	batch limiting is re-enabled.	3981	batch limiting is re-enabled.
3982		3982
		3983	rcutree.qovld= [KNL]
		3984	Set threshold of queued RCU callbacks beyond which
		3985	RCU's force-quiescent-state scan will aggressively
		3986	enlist help from cond_resched() and sched IPIs to
		3987	help CPUs more quickly reach quiescent states.
		3988	Set to less than zero to make this be set based
		3989	on rcutree.qhimark at boot time and to zero to
		3990	disable more aggressive help enlistment.
		3991
3983	rcutree.rcu_idle_gp_delay= [KNL]	3992	rcutree.rcu_idle_gp_delay= [KNL]
3984	Set wakeup interval for idle CPUs that have	3993	Set wakeup interval for idle CPUs that have
3985	RCU callbacks (RCU_FAST_NO_HZ=y).	3994	RCU callbacks (RCU_FAST_NO_HZ=y).
@@ -4195,6 +4204,12 @@ · 209 unmodified lines
4195	rcupdate.rcu_cpu_stall_suppress= [KNL]	4204	rcupdate.rcu_cpu_stall_suppress= [KNL]
4196	Suppress RCU CPU stall warning messages.	4205	Suppress RCU CPU stall warning messages.
4197		4206
		4207	rcupdate.rcu_cpu_stall_suppress_at_boot= [KNL]
		4208	Suppress RCU CPU stall warning messages and
		4209	rcutorture writer stall warnings that occur
		4210	during early boot, that is, during the time
		4211	before the init task is spawned.
		4212
4198	rcupdate.rcu_cpu_stall_timeout= [KNL]	4213	rcupdate.rcu_cpu_stall_timeout= [KNL]
4199	Set timeout for RCU CPU stall warning messages.	4214	Set timeout for RCU CPU stall warning messages.
4200		4215
@@ -4867,6 +4882,10 @@ · 666 unmodified lines
4867	topology updates sent by the hypervisor to this	4882	topology updates sent by the hypervisor to this
4868	LPAR.	4883	LPAR.
4869		4884
		4885	torture.disable_onoff_at_boot= [KNL]
		4886	Prevent the CPU-hotplug component of torturing
		4887	until after init has spawned.
		4888
4870	tp720= [HW,PS2]	4889	tp720= [HW,PS2]
4871		4890
4872	tpm_suspend_pcr=[HW,TPM]	4891	tpm_suspend_pcr=[HW,TPM]

fs/nfs/dir.c+1−1

@@ -2383,7 +2383,7 @@ static int nfs_access_get_cached_rcu(struct inode inode, const struct cred cre · 2382 unmodified lines
2383	rcu_read_lock();	2383	rcu_read_lock();
2384	if (nfsi->cache_validity & NFS_INO_INVALID_ACCESS)	2384	if (nfsi->cache_validity & NFS_INO_INVALID_ACCESS)
2385	goto out;	2385	goto out;
2386	lh = rcu_dereference(nfsi->access_cache_entry_lru.prev);	2386	lh = rcu_dereference(list_tail_rcu(&nfsi->access_cache_entry_lru));
2387	cache = list_entry(lh, struct nfs_access_entry, lru);	2387	cache = list_entry(lh, struct nfs_access_entry, lru);
2388	if (lh == &nfsi->access_cache_entry_lru \|\|	2388	if (lh == &nfsi->access_cache_entry_lru \|\|
2389	cred_fscmp(cred, cache->cred) != 0)	2389	cred_fscmp(cred, cache->cred) != 0)

include/linux/rculist.h+2−2

@@ -60,9 +60,9 @@ static inline void INIT_LIST_HEAD_RCU(struct list_head *list) · 59 unmodified lines
60	#define __list_check_rcu(dummy, cond, extra...) \	60	#define __list_check_rcu(dummy, cond, extra...) \
61	({ \	61	({ \
62	check_arg_count_one(extra); \	62	check_arg_count_one(extra); \
63	RCU_LOCKDEP_WARN(!cond && !rcu_read_lock_any_held(), \	63	RCU_LOCKDEP_WARN(!(cond) && !rcu_read_lock_any_held(), \
64	"RCU-list traversed in non-reader section!"); \	64	"RCU-list traversed in non-reader section!"); \
65	})	65	})
66	#else	66	#else
67	#define __list_check_rcu(dummy, cond, extra...) \	67	#define __list_check_rcu(dummy, cond, extra...) \
68	({ check_arg_count_one(extra); })	68	({ check_arg_count_one(extra); })

include/linux/rcutiny.h+1−0

@@ -83,6 +83,7 @@ void rcu_scheduler_starting(void); · 82 unmodified lines
83	static inline void rcu_scheduler_starting(void) { }	83	static inline void rcu_scheduler_starting(void) { }
84	#endif /* #else #ifndef CONFIG_SRCU */	84	#endif /* #else #ifndef CONFIG_SRCU */
85	static inline void rcu_end_inkernel_boot(void) { }	85	static inline void rcu_end_inkernel_boot(void) { }
		86	static inline bool rcu_inkernel_boot_has_ended(void) { return true; }
86	static inline bool rcu_is_watching(void) { return true; }	87	static inline bool rcu_is_watching(void) { return true; }
87	static inline void rcu_momentary_dyntick_idle(void) { }	88	static inline void rcu_momentary_dyntick_idle(void) { }
88	static inline void kfree_rcu_scheduler_running(void) { }	89	static inline void kfree_rcu_scheduler_running(void) { }

include/linux/rcutree.h+1−0

@@ -54,6 +54,7 @@ void exit_rcu(void); · 53 unmodified lines
54	void rcu_scheduler_starting(void);	54	void rcu_scheduler_starting(void);
55	extern int rcu_scheduler_active __read_mostly;	55	extern int rcu_scheduler_active __read_mostly;
56	void rcu_end_inkernel_boot(void);	56	void rcu_end_inkernel_boot(void);
		57	bool rcu_inkernel_boot_has_ended(void);
57	bool rcu_is_watching(void);	58	bool rcu_is_watching(void);
58	#ifndef CONFIG_PREEMPTION	59	#ifndef CONFIG_PREEMPTION
59	void rcu_all_qs(void);	60	void rcu_all_qs(void);

include/linux/timer.h+1−1

@@ -164,7 +164,7 @@ static inline void destroy_timer_on_stack(struct timer_list *timer) { } · 163 unmodified lines
164	*/	164	*/
165	static inline int timer_pending(const struct timer_list * timer)	165	static inline int timer_pending(const struct timer_list * timer)
166	{	166	{
167	return timer->entry.pprev != NULL;	167	return !hlist_unhashed_lockless(&timer->entry);
168	}	168	}
169		169
170	extern void add_timer_on(struct timer_list *timer, int cpu);	170	extern void add_timer_on(struct timer_list *timer, int cpu);

include/trace/events/rcu.h+29−0

@@ -623,6 +623,34 @@ TRACE_EVENT_RCU(rcu_invoke_kfree_callback, · 622 unmodified lines
623	__entry->rcuname, __entry->rhp, __entry->offset)	623	__entry->rcuname, __entry->rhp, __entry->offset)
624	);	624	);
625		625
		626	/*
		627	* Tracepoint for the invocation of a single RCU callback of the special
		628	* kfree_bulk() form. The first argument is the RCU flavor, the second
		629	* argument is a number of elements in array to free, the third is an
		630	* address of the array holding nr_records entries.
		631	*/
		632	TRACE_EVENT_RCU(rcu_invoke_kfree_bulk_callback,
		633
		634	TP_PROTO(const char rcuname, unsigned long nr_records, void *p),
		635
		636	TP_ARGS(rcuname, nr_records, p),
		637
		638	TP_STRUCT__entry(
		639	__field(const char *, rcuname)
		640	__field(unsigned long, nr_records)
		641	__field(void **, p)
		642	),
		643
		644	TP_fast_assign(
		645	__entry->rcuname = rcuname;
		646	__entry->nr_records = nr_records;
		647	__entry->p = p;
		648	),
		649
		650	TP_printk("%s bulk=0x%p nr_records=%lu",
		651	__entry->rcuname, __entry->p, __entry->nr_records)
		652	);
		653
626	/*	654	/*
627	* Tracepoint for exiting rcu_do_batch after RCU callbacks have been	655	* Tracepoint for exiting rcu_do_batch after RCU callbacks have been
628	* invoked. The first argument is the name of the RCU flavor,	656	* invoked. The first argument is the name of the RCU flavor,
@@ -712,6 +740,7 @@ TRACE_EVENT_RCU(rcu_torture_read, · 83 unmodified lines
712	* "Begin": rcu_barrier() started.	740	* "Begin": rcu_barrier() started.
713	* "EarlyExit": rcu_barrier() piggybacked, thus early exit.	741	* "EarlyExit": rcu_barrier() piggybacked, thus early exit.
714	* "Inc1": rcu_barrier() piggyback check counter incremented.	742	* "Inc1": rcu_barrier() piggyback check counter incremented.
		743	* "OfflineNoCBQ": rcu_barrier() found offline no-CBs CPU with callbacks.
715	* "OnlineQ": rcu_barrier() found online CPU with callbacks.	744	* "OnlineQ": rcu_barrier() found online CPU with callbacks.
716	* "OnlineNQ": rcu_barrier() found online CPU, no callbacks.	745	* "OnlineNQ": rcu_barrier() found online CPU, no callbacks.
717	* "IRQ": An rcu_barrier_callback() callback posted on remote CPU.	746	* "IRQ": An rcu_barrier_callback() callback posted on remote CPU.

kernel/locking/locktorture.c+8−7

 static int lock_torture_writer(void *arg)
 {
 	struct lock_stress_stats *lwsp = arg;
-	static DEFINE_TORTURE_RANDOM(rand);
+	DEFINE_TORTURE_RANDOM(rand);
 	VERBOSE_TOROUT_STRING("lock_torture_writer task started");
 	set_user_nice(current, MAX_NICE);
 static int lock_torture_reader(void *arg)
 {
 	struct lock_stress_stats *lrsp = arg;
-	static DEFINE_TORTURE_RANDOM(rand);
+	DEFINE_TORTURE_RANDOM(rand);
 	VERBOSE_TOROUT_STRING("lock_torture_reader task started");
 	set_user_nice(current, MAX_NICE);
 		if (statp[i].n_lock_fail)
 			fail = true;
 		sum += statp[i].n_lock_acquired;
-		if (max < statp[i].n_lock_fail)
+		if (max < statp[i].n_lock_acquired)
-			max = statp[i].n_lock_fail;
+			max = statp[i].n_lock_acquired;
-		if (min > statp[i].n_lock_fail)
+		if (min > statp[i].n_lock_acquired)
-			min = statp[i].n_lock_fail;
+			min = statp[i].n_lock_acquired;
 	}
 	page += sprintf(page,
 			"%s:  Total: %lld  Max/Min: %ld/%ld %s  Fail: %d %s\n",
 			write ? "Writes" : "Reads ",
-			sum, max, min, max / 2 > min ? "???" : "",
+			sum, max, min,
+			!onoff_interval && max / 2 > min ? "???" : "",
 			fail, fail ? "!!!" : "");
 	if (fail)
 		atomic_inc(&cxt.n_lock_torture_errors);

kernel/locking/rtmutex.c+1−1

@@ -57,7 +57,7 @@ rt_mutex_set_owner(struct rt_mutex lock, struct task_struct owner) · 56 unmodified lines
57	if (rt_mutex_has_waiters(lock))	57	if (rt_mutex_has_waiters(lock))
58	val \|= RT_MUTEX_HAS_WAITERS;	58	val \|= RT_MUTEX_HAS_WAITERS;
59		59
60	lock->owner = (struct task_struct *)val;	60	WRITE_ONCE(lock->owner, (struct task_struct *)val);
61	}	61	}
62		62
63	static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)	63	static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)

kernel/rcu/Makefile+4−0

 # and is generally not a function of system call inputs.
 KCOV_INSTRUMENT := n
+ifeq ($(CONFIG_KCSAN),y)
+KBUILD_CFLAGS += -g -fno-omit-frame-pointer
+endif
 obj-y += update.o sync.o
 obj-$(CONFIG_TREE_SRCU) += srcutree.o
 obj-$(CONFIG_TINY_SRCU) += srcutiny.o

kernel/rcu/rcu.h+21−2

 }
 #endif	/* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
+extern int rcu_cpu_stall_suppress_at_boot;
+static inline bool rcu_stall_is_suppressed_at_boot(void)
+{
+	return rcu_cpu_stall_suppress_at_boot && !rcu_inkernel_boot_has_ended();
+}
 #ifdef CONFIG_RCU_STALL_COMMON
 extern int rcu_cpu_stall_ftrace_dump;
 extern int rcu_cpu_stall_timeout;
 int rcu_jiffies_till_stall_check(void);
+static inline bool rcu_stall_is_suppressed(void)
+{
+	return rcu_stall_is_suppressed_at_boot() || rcu_cpu_stall_suppress;
+}
 #define rcu_ftrace_dump_stall_suppress() \
 do { \
 	if (!rcu_cpu_stall_suppress) \
 } while (0)
 #else /* #endif #ifdef CONFIG_RCU_STALL_COMMON */
+static inline bool rcu_stall_is_suppressed(void)
+{
+	return rcu_stall_is_suppressed_at_boot();
+}
 #define rcu_ftrace_dump_stall_suppress()
 #define rcu_ftrace_dump_stall_unsuppress()
 #endif /* #ifdef CONFIG_RCU_STALL_COMMON */
  * Iterate over all possible CPUs in a leaf RCU node.
  */
 #define for_each_leaf_node_possible_cpu(rnp, cpu) \
-	for ((cpu) = cpumask_next((rnp)->grplo - 1, cpu_possible_mask); \
+	for (WARN_ON_ONCE(!rcu_is_leaf_node(rnp)), \
+	     (cpu) = cpumask_next((rnp)->grplo - 1, cpu_possible_mask); \
 	     (cpu) <= rnp->grphi; \
 	     (cpu) = cpumask_next((cpu), cpu_possible_mask))
 #define rcu_find_next_bit(rnp, cpu, mask) \
 	((rnp)->grplo + find_next_bit(&(mask), BITS_PER_LONG, (cpu)))
 #define for_each_leaf_node_cpu_mask(rnp, cpu, mask) \
-	for ((cpu) = rcu_find_next_bit((rnp), 0, (mask)); \
+	for (WARN_ON_ONCE(!rcu_is_leaf_node(rnp)), \
+	     (cpu) = rcu_find_next_bit((rnp), 0, (mask)); \
 	     (cpu) <= rnp->grphi; \
 	     (cpu) = rcu_find_next_bit((rnp), (cpu) + 1 - (rnp->grplo), (mask)))

kernel/rcu/rcu_segcblist.c+1−3

 bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp)
 {
 	return rcu_segcblist_is_enabled(rsclp) &&
-	       &rsclp->head != rsclp->tails[RCU_DONE_TAIL];
+	       &rsclp->head != READ_ONCE(rsclp->tails[RCU_DONE_TAIL]);
 }
 /*
 		return; /* Nothing to do. */
 	WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head);
 	WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail);
-	rclp->head = NULL;
-	rclp->tail = &rclp->head;
 }
 /*

kernel/rcu/rcuperf.c+12−2

 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/init.h>
+#include <linux/mm.h>
 #include <linux/module.h>
 #include <linux/kthread.h>
 #include <linux/err.h>
 	long me = (long)arg;
 	struct kfree_obj *alloc_ptr;
 	u64 start_time, end_time;
+	long long mem_begin, mem_during = 0;
 	VERBOSE_PERFOUT_STRING("kfree_perf_thread task started");
 	set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
 	}
 	do {
+		if (!mem_during) {
+			mem_during = mem_begin = si_mem_available();
+		} else if (loop % (kfree_loops / 4) == 0) {
+			mem_during = (mem_during + si_mem_available()) / 2;
+		}
 		for (i = 0; i < kfree_alloc_num; i++) {
 			alloc_ptr = kmalloc(sizeof(struct kfree_obj), GFP_KERNEL);
 			if (!alloc_ptr)
 		else
 			b_rcu_gp_test_finished = cur_ops->get_gp_seq();
-		pr_alert("Total time taken by all kfree'ers: %llu ns, loops: %d, batches: %ld\n",
+		pr_alert("Total time taken by all kfree'ers: %llu ns, loops: %d, batches: %ld, memory footprint: %lldMB\n",
 		       (unsigned long long)(end_time - start_time), kfree_loops,
-		       rcuperf_seq_diff(b_rcu_gp_test_finished, b_rcu_gp_test_started));
+		       rcuperf_seq_diff(b_rcu_gp_test_finished, b_rcu_gp_test_started),
+		       (mem_begin - mem_during) >> (20 - PAGE_SHIFT));
 		if (shutdown) {
 			smp_mb(); /* Assign before wake. */
 			wake_up(&shutdown_wq);

kernel/rcu/rcutorture.c+50−17

 	 * period, and we want a long delay occasionally to trigger
 	 * force_quiescent_state. */
-	if (!rcu_fwd_cb_nodelay &&
+	if (!READ_ONCE(rcu_fwd_cb_nodelay) &&
 	    !(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) {
 		started = cur_ops->get_gp_seq();
 		ts = rcu_trace_clock_local();
 {
 	int i;
-	i = rp->rtort_pipe_count;
+	i = READ_ONCE(rp->rtort_pipe_count);
 	if (i > RCU_TORTURE_PIPE_LEN)
 		i = RCU_TORTURE_PIPE_LEN;
 	atomic_inc(&rcu_torture_wcount[i]);
-	if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
+	WRITE_ONCE(rp->rtort_pipe_count, i + 1);
+	if (rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 		rp->rtort_mbtest = 0;
 		return true;
 	}
 			if (i > RCU_TORTURE_PIPE_LEN)
 				i = RCU_TORTURE_PIPE_LEN;
 			atomic_inc(&rcu_torture_wcount[i]);
-			old_rp->rtort_pipe_count++;
+			WRITE_ONCE(old_rp->rtort_pipe_count,
+				   old_rp->rtort_pipe_count + 1);
 			switch (synctype[torture_random(&rand) % nsynctypes]) {
 			case RTWS_DEF_FREE:
 				rcu_torture_writer_state = RTWS_DEF_FREE;
 		if (stutter_wait("rcu_torture_writer") &&
 		    !READ_ONCE(rcu_fwd_cb_nodelay) &&
 		    !cur_ops->slow_gps &&
-		    !torture_must_stop())
+		    !torture_must_stop() &&
+		    rcu_inkernel_boot_has_ended())
 			for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++)
 				if (list_empty(&rcu_tortures[i].rtort_free) &&
 				    rcu_access_pointer(rcu_torture_current) !=
 		atomic_inc(&n_rcu_torture_mberror);
 	rtrsp = rcutorture_loop_extend(&readstate, trsp, rtrsp);
 	preempt_disable();
-	pipe_count = p->rtort_pipe_count;
+	pipe_count = READ_ONCE(p->rtort_pipe_count);
 	if (pipe_count > RCU_TORTURE_PIPE_LEN) {
 		/* Should not happen, but... */
 		pipe_count = RCU_TORTURE_PIPE_LEN;
 	int i;
 	long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
 	long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
+	struct rcu_torture *rtcp;
 	static unsigned long rtcv_snap = ULONG_MAX;
 	static bool splatted;
 	struct task_struct *wtp;
 	for_each_possible_cpu(cpu) {
 		for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
-			pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
+			pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]);
-			batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
+			batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]);
 		}
 	}
 	for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
 	}
 	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
+	rtcp = rcu_access_pointer(rcu_torture_current);
 	pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
-		rcu_torture_current,
+		rtcp,
-		rcu_torture_current ? "ver" : "VER",
+		rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER",
 		rcu_torture_current_version,
 		list_empty(&rcu_torture_freelist),
 		atomic_read(&n_rcu_torture_alloc),
 	if (cur_ops->stats)
 		cur_ops->stats();
 	if (rtcv_snap == rcu_torture_current_version &&
-	    rcu_torture_current != NULL) {
+	    rcu_access_pointer(rcu_torture_current) &&
+	    !rcu_stall_is_suppressed()) {
 		int __maybe_unused flags = 0;
 		unsigned long __maybe_unused gp_seq = 0;
 		schedule_timeout_interruptible(fwd_progress_holdoff * HZ);
 		WRITE_ONCE(rcu_fwd_emergency_stop, false);
 		register_oom_notifier(&rcutorture_oom_nb);
-		rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);
+		if (!IS_ENABLED(CONFIG_TINY_RCU) ||
-		rcu_torture_fwd_prog_cr(rfp);
+		    rcu_inkernel_boot_has_ended())
+			rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);
+		if (rcu_inkernel_boot_has_ended())
+			rcu_torture_fwd_prog_cr(rfp);
 		unregister_oom_notifier(&rcutorture_oom_nb);
 		/* Avoid slow periods, better to test when busy. */
 	atomic_inc(&barrier_cbs_invoked);
 }
+/* IPI handler to get callback posted on desired CPU, if online. */
+static void rcu_torture_barrier1cb(void *rcu_void)
+{
+	struct rcu_head *rhp = rcu_void;
+	cur_ops->call(rhp, rcu_torture_barrier_cbf);
+}
 /* kthread function to register callbacks used to test RCU barriers. */
 static int rcu_torture_barrier_cbs(void *arg)
 {
 		 * The above smp_load_acquire() ensures barrier_phase load
 		 * is ordered before the following ->call().
 		 */
-		local_irq_disable(); /* Just to test no-irq call_rcu(). */
+		if (smp_call_function_single(myid, rcu_torture_barrier1cb,
-		cur_ops->call(&rcu, rcu_torture_barrier_cbf);
+					     &rcu, 1)) {
-		local_irq_enable();
+			// IPI failed, so use direct call from current CPU.
+			cur_ops->call(&rcu, rcu_torture_barrier_cbf);
+		}
 		if (atomic_dec_and_test(&barrier_cbs_count))
 			wake_up(&barrier_wq);
 	} while (!torture_must_stop());
 			pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n",
 			       atomic_read(&barrier_cbs_invoked),
 			       n_barrier_cbs);
-			WARN_ON_ONCE(1);
+			WARN_ON(1);
+			// Wait manually for the remaining callbacks
+			i = 0;
+			do {
+				if (WARN_ON(i++ > HZ))
+					i = INT_MIN;
+				schedule_timeout_interruptible(1);
+				cur_ops->cb_barrier();
+			} while (atomic_read(&barrier_cbs_invoked) !=
+				 n_barrier_cbs &&
+				 !torture_must_stop());
+			smp_mb(); // Can't trust ordering if broken.
+			if (!torture_must_stop())
+				pr_err("Recovered: barrier_cbs_invoked = %d\n",
+				       atomic_read(&barrier_cbs_invoked));
 		} else {
 			n_barrier_successes++;
 		}

kernel/rcu/srcutree.c+10−8

  * Copyright (C) IBM Corporation, 2006
  * Copyright (C) Fujitsu, 2012
  *
- * Author: Paul McKenney <paulmck@linux.ibm.com>
+ * Authors: Paul McKenney <paulmck@linux.ibm.com>
  *	   Lai Jiangshan <laijs@cn.fujitsu.com>
  *
  * For detailed explanation of Read-Copy Update mechanism see -
 	spin_unlock_rcu_node(sdp);  /* Interrupts remain disabled. */
 	smp_mb(); /* Order prior store to ->srcu_gp_seq_needed vs. GP start. */
 	rcu_seq_start(&ssp->srcu_gp_seq);
-	state = rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq));
+	state = rcu_seq_state(ssp->srcu_gp_seq);
 	WARN_ON_ONCE(state != SRCU_STATE_SCAN1);
 }
 	rcu_seq_end(&ssp->srcu_gp_seq);
 	gpseq = rcu_seq_current(&ssp->srcu_gp_seq);
 	if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, gpseq))
-		ssp->srcu_gp_seq_needed_exp = gpseq;
+		WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, gpseq);
 	spin_unlock_irq_rcu_node(ssp);
 	mutex_unlock(&ssp->srcu_gp_mutex);
 	/* A new grace period can start at this point.  But only one. */
 		snp->srcu_have_cbs[idx] = gpseq;
 		rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);
 		if (ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, gpseq))
-			snp->srcu_gp_seq_needed_exp = gpseq;
+			WRITE_ONCE(snp->srcu_gp_seq_needed_exp, gpseq);
 		mask = snp->srcu_data_have_cbs[idx];
 		snp->srcu_data_have_cbs[idx] = 0;
 		spin_unlock_irq_rcu_node(snp);
 	}
 	spin_lock_irqsave_rcu_node(ssp, flags);
 	if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))
-		ssp->srcu_gp_seq_needed_exp = s;
+		WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s);
 	spin_unlock_irqrestore_rcu_node(ssp, flags);
 }
 		if (snp == sdp->mynode)
 			snp->srcu_data_have_cbs[idx] |= sdp->grpmask;
 		if (!do_norm && ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, s))
-			snp->srcu_gp_seq_needed_exp = s;
+			WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);
 		spin_unlock_irqrestore_rcu_node(snp, flags);
 	}
 		smp_store_release(&ssp->srcu_gp_seq_needed, s); /*^^^*/
 	}
 	if (!do_norm && ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))
-		ssp->srcu_gp_seq_needed_exp = s;
+		WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s);
 	/* If grace period not already done and none in progress, start it. */
 	if (!rcu_seq_done(&ssp->srcu_gp_seq, s) &&
  */
 unsigned long srcu_batches_completed(struct srcu_struct *ssp)
 {
-	return ssp->srcu_idx;
+	return READ_ONCE(ssp->srcu_idx);
 }
 EXPORT_SYMBOL_GPL(srcu_batches_completed);
 			return; /* readers present, retry later. */
 		}
 		srcu_flip(ssp);
+		spin_lock_irq_rcu_node(ssp);
 		rcu_seq_set_state(&ssp->srcu_gp_seq, SRCU_STATE_SCAN2);
+		spin_unlock_irq_rcu_node(ssp);
 	}
 	if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN2) {

kernel/rcu/tree.c+339−113

@@ -1,12 +1,12 @@
1	// SPDX-License-Identifier: GPL-2.0+	1	// SPDX-License-Identifier: GPL-2.0+
2	/*	2	/*
3	* Read-Copy Update mechanism for mutual exclusion	3	* Read-Copy Update mechanism for mutual exclusion (tree-based version)
4	*	4	*
5	* Copyright IBM Corporation, 2008	5	* Copyright IBM Corporation, 2008
6	*	6	*
7	* Authors: Dipankar Sarma <dipankar@in.ibm.com>	7	* Authors: Dipankar Sarma <dipankar@in.ibm.com>
8	* Manfred Spraul <manfred@colorfullife.com>	8	* Manfred Spraul <manfred@colorfullife.com>
9	* Paul E. McKenney <paulmck@linux.ibm.com> Hierarchical version	9	* Paul E. McKenney <paulmck@linux.ibm.com>
10	*	10	*
11	* Based on the original work by Paul McKenney <paulmck@linux.ibm.com>	11	* Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
12	* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.	12	* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
@@ -150,6 +150,7 @@ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu) · 137 unmodified lines
150	static void invoke_rcu_core(void);	150	static void invoke_rcu_core(void);
151	static void rcu_report_exp_rdp(struct rcu_data *rdp);	151	static void rcu_report_exp_rdp(struct rcu_data *rdp);
152	static void sync_sched_exp_online_cleanup(int cpu);	152	static void sync_sched_exp_online_cleanup(int cpu);
		153	static void check_cb_ovld_locked(struct rcu_data rdp, struct rcu_node rnp);
153		154
154	/* rcuc/rcub kthread realtime priority */	155	/* rcuc/rcub kthread realtime priority */
155	static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;	156	static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
@@ -342,14 +343,17 @@ bool rcu_eqs_special_set(int cpu) · 186 unmodified lines
342	{	343	{
343	int old;	344	int old;
344	int new;	345	int new;
		346	int new_old;
345	struct rcu_data *rdp = &per_cpu(rcu_data, cpu);	347	struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
346		348
		349	new_old = atomic_read(&rdp->dynticks);
347	do {	350	do {
348	old = atomic_read(&rdp->dynticks);	351	old = new_old;
349	if (old & RCU_DYNTICK_CTRL_CTR)	352	if (old & RCU_DYNTICK_CTRL_CTR)
350	return false;	353	return false;
351	new = old \| RCU_DYNTICK_CTRL_MASK;	354	new = old \| RCU_DYNTICK_CTRL_MASK;
352	} while (atomic_cmpxchg(&rdp->dynticks, old, new) != old);	355	new_old = atomic_cmpxchg(&rdp->dynticks, old, new);
		356	} while (new_old != old);
353	return true;	357	return true;
354	}	358	}
355		359
@@ -410,10 +414,15 @@ static long blimit = DEFAULT_RCU_BLIMIT; · 54 unmodified lines
410	static long qhimark = DEFAULT_RCU_QHIMARK;	414	static long qhimark = DEFAULT_RCU_QHIMARK;
411	#define DEFAULT_RCU_QLOMARK 100 /* Once only this many pending, use blimit. */	415	#define DEFAULT_RCU_QLOMARK 100 /* Once only this many pending, use blimit. */
412	static long qlowmark = DEFAULT_RCU_QLOMARK;	416	static long qlowmark = DEFAULT_RCU_QLOMARK;
		417	#define DEFAULT_RCU_QOVLD_MULT 2
		418	#define DEFAULT_RCU_QOVLD (DEFAULT_RCU_QOVLD_MULT * DEFAULT_RCU_QHIMARK)
		419	static long qovld = DEFAULT_RCU_QOVLD; /* If this many pending, hammer QS. */
		420	static long qovld_calc = -1; /* No pre-initialization lock acquisitions! */
413		421
414	module_param(blimit, long, 0444);	422	module_param(blimit, long, 0444);
415	module_param(qhimark, long, 0444);	423	module_param(qhimark, long, 0444);
416	module_param(qlowmark, long, 0444);	424	module_param(qlowmark, long, 0444);
		425	module_param(qovld, long, 0444);
417		426
418	static ulong jiffies_till_first_fqs = ULONG_MAX;	427	static ulong jiffies_till_first_fqs = ULONG_MAX;
419	static ulong jiffies_till_next_fqs = ULONG_MAX;	428	static ulong jiffies_till_next_fqs = ULONG_MAX;
@@ -818,11 +827,12 @@ static __always_inline void rcu_nmi_enter_common(bool irq) · 398 unmodified lines
818	incby = 1;	827	incby = 1;
819	} else if (tick_nohz_full_cpu(rdp->cpu) &&	828	} else if (tick_nohz_full_cpu(rdp->cpu) &&
820	rdp->dynticks_nmi_nesting == DYNTICK_IRQ_NONIDLE &&	829	rdp->dynticks_nmi_nesting == DYNTICK_IRQ_NONIDLE &&
821	READ_ONCE(rdp->rcu_urgent_qs) && !rdp->rcu_forced_tick) {	830	READ_ONCE(rdp->rcu_urgent_qs) &&
		831	!READ_ONCE(rdp->rcu_forced_tick)) {
822	raw_spin_lock_rcu_node(rdp->mynode);	832	raw_spin_lock_rcu_node(rdp->mynode);
823	// Recheck under lock.	833	// Recheck under lock.
824	if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {	834	if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {
825	rdp->rcu_forced_tick = true;	835	WRITE_ONCE(rdp->rcu_forced_tick, true);
826	tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);	836	tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
827	}	837	}
828	raw_spin_unlock_rcu_node(rdp->mynode);	838	raw_spin_unlock_rcu_node(rdp->mynode);
@@ -899,7 +909,7 @@ static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp) · 70 unmodified lines
899	WRITE_ONCE(rdp->rcu_need_heavy_qs, false);	909	WRITE_ONCE(rdp->rcu_need_heavy_qs, false);
900	if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) {	910	if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) {
901	tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU);	911	tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
902	rdp->rcu_forced_tick = false;	912	WRITE_ONCE(rdp->rcu_forced_tick, false);
903	}	913	}
904	}	914	}
905		915
@@ -1072,7 +1082,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) · 166 unmodified lines
1072	rnhqp = &per_cpu(rcu_data.rcu_need_heavy_qs, rdp->cpu);	1082	rnhqp = &per_cpu(rcu_data.rcu_need_heavy_qs, rdp->cpu);
1073	if (!READ_ONCE(*rnhqp) &&	1083	if (!READ_ONCE(*rnhqp) &&
1074	(time_after(jiffies, rcu_state.gp_start + jtsq * 2) \|\|	1084	(time_after(jiffies, rcu_state.gp_start + jtsq * 2) \|\|
1075	time_after(jiffies, rcu_state.jiffies_resched))) {	1085	time_after(jiffies, rcu_state.jiffies_resched) \|\|
		1086	rcu_state.cbovld)) {
1076	WRITE_ONCE(*rnhqp, true);	1087	WRITE_ONCE(*rnhqp, true);
1077	/* Store rcu_need_heavy_qs before rcu_urgent_qs. */	1088	/* Store rcu_need_heavy_qs before rcu_urgent_qs. */
1078	smp_store_release(ruqp, true);	1089	smp_store_release(ruqp, true);
@@ -1089,8 +1100,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) · 10 unmodified lines
1089	* So hit them over the head with the resched_cpu() hammer!	1100	* So hit them over the head with the resched_cpu() hammer!
1090	*/	1101	*/
1091	if (tick_nohz_full_cpu(rdp->cpu) &&	1102	if (tick_nohz_full_cpu(rdp->cpu) &&
1092	time_after(jiffies,	1103	(time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) \|\|
1093	READ_ONCE(rdp->last_fqs_resched) + jtsq * 3)) {	1104	rcu_state.cbovld)) {
1094	WRITE_ONCE(*ruqp, true);	1105	WRITE_ONCE(*ruqp, true);
1095	resched_cpu(rdp->cpu);	1106	resched_cpu(rdp->cpu);
1096	WRITE_ONCE(rdp->last_fqs_resched, jiffies);	1107	WRITE_ONCE(rdp->last_fqs_resched, jiffies);
@@ -1126,8 +1137,9 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) · 29 unmodified lines
1126	static void trace_rcu_this_gp(struct rcu_node rnp, struct rcu_data rdp,	1137	static void trace_rcu_this_gp(struct rcu_node rnp, struct rcu_data rdp,
1127	unsigned long gp_seq_req, const char *s)	1138	unsigned long gp_seq_req, const char *s)
1128	{	1139	{
1129	trace_rcu_future_grace_period(rcu_state.name, rnp->gp_seq, gp_seq_req,	1140	trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
1130	rnp->level, rnp->grplo, rnp->grphi, s);	1141	gp_seq_req, rnp->level,
		1142	rnp->grplo, rnp->grphi, s);
1131	}	1143	}
1132		1144
1133	/*	1145	/*
@@ -1174,7 +1186,7 @@ static bool rcu_start_this_gp(struct rcu_node rnp_start, struct rcu_data rdp, · 40 unmodified lines
1174	TPS("Prestarted"));	1186	TPS("Prestarted"));
1175	goto unlock_out;	1187	goto unlock_out;
1176	}	1188	}
1177	rnp->gp_seq_needed = gp_seq_req;	1189	WRITE_ONCE(rnp->gp_seq_needed, gp_seq_req);
1178	if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {	1190	if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {
1179	/*	1191	/*
1180	* We just marked the leaf or internal node, and a	1192	* We just marked the leaf or internal node, and a
@@ -1199,18 +1211,18 @@ static bool rcu_start_this_gp(struct rcu_node rnp_start, struct rcu_data rdp, · 18 unmodified lines
1199	}	1211	}
1200	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot"));	1212	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot"));
1201	WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags \| RCU_GP_FLAG_INIT);	1213	WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags \| RCU_GP_FLAG_INIT);
1202	rcu_state.gp_req_activity = jiffies;	1214	WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
1203	if (!rcu_state.gp_kthread) {	1215	if (!READ_ONCE(rcu_state.gp_kthread)) {
1204	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread"));	1216	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread"));
1205	goto unlock_out;	1217	goto unlock_out;
1206	}	1218	}
1207	trace_rcu_grace_period(rcu_state.name, READ_ONCE(rcu_state.gp_seq), TPS("newreq"));	1219	trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("newreq"));
1208	ret = true; /* Caller must wake GP kthread. */	1220	ret = true; /* Caller must wake GP kthread. */
1209	unlock_out:	1221	unlock_out:
1210	/* Push furthest requested GP to leaf node and rcu_data structure. */	1222	/* Push furthest requested GP to leaf node and rcu_data structure. */
1211	if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) {	1223	if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) {
1212	rnp_start->gp_seq_needed = rnp->gp_seq_needed;	1224	WRITE_ONCE(rnp_start->gp_seq_needed, rnp->gp_seq_needed);
1213	rdp->gp_seq_needed = rnp->gp_seq_needed;	1225	WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
1214	}	1226	}
1215	if (rnp != rnp_start)	1227	if (rnp != rnp_start)
1216	raw_spin_unlock_rcu_node(rnp);	1228	raw_spin_unlock_rcu_node(rnp);
@@ -1235,12 +1247,13 @@ static bool rcu_future_gp_cleanup(struct rcu_node *rnp) · 18 unmodified lines
1235	}	1247	}
1236		1248
1237	/*	1249	/*
1238	* Awaken the grace-period kthread. Don't do a self-awaken (unless in	1250	* Awaken the grace-period kthread. Don't do a self-awaken (unless in an
1239	* an interrupt or softirq handler), and don't bother awakening when there	1251	* interrupt or softirq handler, in which case we just might immediately
1240	* is nothing for the grace-period kthread to do (as in several CPUs raced	1252	* sleep upon return, resulting in a grace-period hang), and don't bother
1241	* to awaken, and we lost), and finally don't try to awaken a kthread that	1253	* awakening when there is nothing for the grace-period kthread to do
1242	* has not yet been created. If all those checks are passed, track some	1254	* (as in several CPUs raced to awaken, we lost), and finally don't try
1243	* debug information and awaken.	1255	* to awaken a kthread that has not yet been created. If all those checks
		1256	* are passed, track some debug information and awaken.
1244	*	1257	*
1245	* So why do the self-wakeup when in an interrupt or softirq handler	1258	* So why do the self-wakeup when in an interrupt or softirq handler
1246	* in the grace-period kthread's context? Because the kthread might have	1259	* in the grace-period kthread's context? Because the kthread might have
@@ -1250,10 +1263,10 @@ static bool rcu_future_gp_cleanup(struct rcu_node *rnp) · 3 unmodified lines
1250	*/	1263	*/
1251	static void rcu_gp_kthread_wake(void)	1264	static void rcu_gp_kthread_wake(void)
1252	{	1265	{
1253	if ((current == rcu_state.gp_kthread &&	1266	struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
1254	!in_irq() && !in_serving_softirq()) \|\|	1267
1255	!READ_ONCE(rcu_state.gp_flags) \|\|	1268	if ((current == t && !in_irq() && !in_serving_softirq()) \|\|
1256	!rcu_state.gp_kthread)	1269	!READ_ONCE(rcu_state.gp_flags) \|\| !t)
1257	return;	1270	return;
1258	WRITE_ONCE(rcu_state.gp_wake_time, jiffies);	1271	WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
1259	WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));	1272	WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
@@ -1321,7 +1334,7 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, · 61 unmodified lines
1321		1334
1322	rcu_lockdep_assert_cblist_protected(rdp);	1335	rcu_lockdep_assert_cblist_protected(rdp);
1323	c = rcu_seq_snap(&rcu_state.gp_seq);	1336	c = rcu_seq_snap(&rcu_state.gp_seq);
1324	if (!rdp->gpwrap && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {	1337	if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
1325	/* Old request still live, so mark recent callbacks. */	1338	/* Old request still live, so mark recent callbacks. */
1326	(void)rcu_segcblist_accelerate(&rdp->cblist, c);	1339	(void)rcu_segcblist_accelerate(&rdp->cblist, c);
1327	return;	1340	return;
@@ -1386,7 +1399,7 @@ static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp, · 58 unmodified lines
1386	static bool __note_gp_changes(struct rcu_node rnp, struct rcu_data rdp)	1399	static bool __note_gp_changes(struct rcu_node rnp, struct rcu_data rdp)
1387	{	1400	{
1388	bool ret = false;	1401	bool ret = false;
1389	bool need_gp;	1402	bool need_qs;
1390	const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&	1403	const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
1391	rcu_segcblist_is_offloaded(&rdp->cblist);	1404	rcu_segcblist_is_offloaded(&rdp->cblist);
1392		1405
@@ -1400,10 +1413,13 @@ static bool __note_gp_changes(struct rcu_node rnp, struct rcu_data rdp) · 7 unmodified lines
1400	unlikely(READ_ONCE(rdp->gpwrap))) {	1413	unlikely(READ_ONCE(rdp->gpwrap))) {
1401	if (!offloaded)	1414	if (!offloaded)
1402	ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */	1415	ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */
		1416	rdp->core_needs_qs = false;
1403	trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));	1417	trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));
1404	} else {	1418	} else {
1405	if (!offloaded)	1419	if (!offloaded)
1406	ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */	1420	ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */
		1421	if (rdp->core_needs_qs)
		1422	rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask);
1407	}	1423	}
1408		1424
1409	/* Now handle the beginnings of any new-to-this-CPU grace periods. */	1425	/* Now handle the beginnings of any new-to-this-CPU grace periods. */
@@ -1415,14 +1431,14 @@ static bool __note_gp_changes(struct rcu_node rnp, struct rcu_data rdp) · 5 unmodified lines
1415	* go looking for one.	1431	* go looking for one.
1416	*/	1432	*/
1417	trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));	1433	trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));
1418	need_gp = !!(rnp->qsmask & rdp->grpmask);	1434	need_qs = !!(rnp->qsmask & rdp->grpmask);
1419	rdp->cpu_no_qs.b.norm = need_gp;	1435	rdp->cpu_no_qs.b.norm = need_qs;
1420	rdp->core_needs_qs = need_gp;	1436	rdp->core_needs_qs = need_qs;
1421	zero_cpu_stall_ticks(rdp);	1437	zero_cpu_stall_ticks(rdp);
1422	}	1438	}
1423	rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */	1439	rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */
1424	if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) \|\| rdp->gpwrap)	1440	if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) \|\| rdp->gpwrap)
1425	rdp->gp_seq_needed = rnp->gp_seq_needed;	1441	WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
1426	WRITE_ONCE(rdp->gpwrap, false);	1442	WRITE_ONCE(rdp->gpwrap, false);
1427	rcu_gpnum_ovf(rnp, rdp);	1443	rcu_gpnum_ovf(rnp, rdp);
1428	return ret;	1444	return ret;
@@ -1651,8 +1667,7 @@ static void rcu_gp_fqs_loop(void) · 222 unmodified lines
1651	WRITE_ONCE(rcu_state.jiffies_kick_kthreads,	1667	WRITE_ONCE(rcu_state.jiffies_kick_kthreads,
1652	jiffies + (j ? 3 * j : 2));	1668	jiffies + (j ? 3 * j : 2));
1653	}	1669	}
1654	trace_rcu_grace_period(rcu_state.name,	1670	trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1655	READ_ONCE(rcu_state.gp_seq),
1656	TPS("fqswait"));	1671	TPS("fqswait"));
1657	rcu_state.gp_state = RCU_GP_WAIT_FQS;	1672	rcu_state.gp_state = RCU_GP_WAIT_FQS;
1658	ret = swait_event_idle_timeout_exclusive(	1673	ret = swait_event_idle_timeout_exclusive(
@@ -1666,13 +1681,11 @@ static void rcu_gp_fqs_loop(void) · 7 unmodified lines
1666	/* If time for quiescent-state forcing, do it. */	1681	/* If time for quiescent-state forcing, do it. */
1667	if (ULONG_CMP_GE(jiffies, rcu_state.jiffies_force_qs) \|\|	1682	if (ULONG_CMP_GE(jiffies, rcu_state.jiffies_force_qs) \|\|
1668	(gf & RCU_GP_FLAG_FQS)) {	1683	(gf & RCU_GP_FLAG_FQS)) {
1669	trace_rcu_grace_period(rcu_state.name,	1684	trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1670	READ_ONCE(rcu_state.gp_seq),
1671	TPS("fqsstart"));	1685	TPS("fqsstart"));
1672	rcu_gp_fqs(first_gp_fqs);	1686	rcu_gp_fqs(first_gp_fqs);
1673	first_gp_fqs = false;	1687	first_gp_fqs = false;
1674	trace_rcu_grace_period(rcu_state.name,	1688	trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1675	READ_ONCE(rcu_state.gp_seq),
1676	TPS("fqsend"));	1689	TPS("fqsend"));
1677	cond_resched_tasks_rcu_qs();	1690	cond_resched_tasks_rcu_qs();
1678	WRITE_ONCE(rcu_state.gp_activity, jiffies);	1691	WRITE_ONCE(rcu_state.gp_activity, jiffies);
@@ -1683,8 +1696,7 @@ static void rcu_gp_fqs_loop(void) · 4 unmodified lines
1683	cond_resched_tasks_rcu_qs();	1696	cond_resched_tasks_rcu_qs();
1684	WRITE_ONCE(rcu_state.gp_activity, jiffies);	1697	WRITE_ONCE(rcu_state.gp_activity, jiffies);
1685	WARN_ON(signal_pending(current));	1698	WARN_ON(signal_pending(current));
1686	trace_rcu_grace_period(rcu_state.name,	1699	trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1687	READ_ONCE(rcu_state.gp_seq),
1688	TPS("fqswaitsig"));	1700	TPS("fqswaitsig"));
1689	ret = 1; /* Keep old FQS timing. */	1701	ret = 1; /* Keep old FQS timing. */
1690	j = jiffies;	1702	j = jiffies;
@@ -1701,8 +1713,9 @@ static void rcu_gp_fqs_loop(void) · 10 unmodified lines
1701	*/	1713	*/
1702	static void rcu_gp_cleanup(void)	1714	static void rcu_gp_cleanup(void)
1703	{	1715	{
1704	unsigned long gp_duration;	1716	int cpu;
1705	bool needgp = false;	1717	bool needgp = false;
		1718	unsigned long gp_duration;
1706	unsigned long new_gp_seq;	1719	unsigned long new_gp_seq;
1707	bool offloaded;	1720	bool offloaded;
1708	struct rcu_data *rdp;	1721	struct rcu_data *rdp;
@@ -1748,6 +1761,12 @@ static void rcu_gp_cleanup(void) · 39 unmodified lines
1748	needgp = __note_gp_changes(rnp, rdp) \|\| needgp;	1761	needgp = __note_gp_changes(rnp, rdp) \|\| needgp;
1749	/* smp_mb() provided by prior unlock-lock pair. */	1762	/* smp_mb() provided by prior unlock-lock pair. */
1750	needgp = rcu_future_gp_cleanup(rnp) \|\| needgp;	1763	needgp = rcu_future_gp_cleanup(rnp) \|\| needgp;
		1764	// Reset overload indication for CPUs no longer overloaded
		1765	if (rcu_is_leaf_node(rnp))
		1766	for_each_leaf_node_cpu_mask(rnp, cpu, rnp->cbovldmask) {
		1767	rdp = per_cpu_ptr(&rcu_data, cpu);
		1768	check_cb_ovld_locked(rdp, rnp);
		1769	}
1751	sq = rcu_nocb_gp_get(rnp);	1770	sq = rcu_nocb_gp_get(rnp);
1752	raw_spin_unlock_irq_rcu_node(rnp);	1771	raw_spin_unlock_irq_rcu_node(rnp);
1753	rcu_nocb_gp_cleanup(sq);	1772	rcu_nocb_gp_cleanup(sq);
@@ -1774,9 +1793,9 @@ static void rcu_gp_cleanup(void) · 20 unmodified lines
1774	rcu_segcblist_is_offloaded(&rdp->cblist);	1793	rcu_segcblist_is_offloaded(&rdp->cblist);
1775	if ((offloaded \|\| !rcu_accelerate_cbs(rnp, rdp)) && needgp) {	1794	if ((offloaded \|\| !rcu_accelerate_cbs(rnp, rdp)) && needgp) {
1776	WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);	1795	WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);
1777	rcu_state.gp_req_activity = jiffies;	1796	WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
1778	trace_rcu_grace_period(rcu_state.name,	1797	trace_rcu_grace_period(rcu_state.name,
1779	READ_ONCE(rcu_state.gp_seq),	1798	rcu_state.gp_seq,
1780	TPS("newreq"));	1799	TPS("newreq"));
1781	} else {	1800	} else {
1782	WRITE_ONCE(rcu_state.gp_flags,	1801	WRITE_ONCE(rcu_state.gp_flags,
@@ -1795,8 +1814,7 @@ static int __noreturn rcu_gp_kthread(void *unused) · 12 unmodified lines
1795		1814
1796	/* Handle grace-period start. */	1815	/* Handle grace-period start. */
1797	for (;;) {	1816	for (;;) {
1798	trace_rcu_grace_period(rcu_state.name,	1817	trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1799	READ_ONCE(rcu_state.gp_seq),
1800	TPS("reqwait"));	1818	TPS("reqwait"));
1801	rcu_state.gp_state = RCU_GP_WAIT_GPS;	1819	rcu_state.gp_state = RCU_GP_WAIT_GPS;
1802	swait_event_idle_exclusive(rcu_state.gp_wq,	1820	swait_event_idle_exclusive(rcu_state.gp_wq,
@@ -1809,8 +1827,7 @@ static int __noreturn rcu_gp_kthread(void *unused) · 6 unmodified lines
1809	cond_resched_tasks_rcu_qs();	1827	cond_resched_tasks_rcu_qs();
1810	WRITE_ONCE(rcu_state.gp_activity, jiffies);	1828	WRITE_ONCE(rcu_state.gp_activity, jiffies);
1811	WARN_ON(signal_pending(current));	1829	WARN_ON(signal_pending(current));
1812	trace_rcu_grace_period(rcu_state.name,	1830	trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1813	READ_ONCE(rcu_state.gp_seq),
1814	TPS("reqwaitsig"));	1831	TPS("reqwaitsig"));
1815	}	1832	}
1816		1833
@@ -1881,7 +1898,7 @@ static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp, · 64 unmodified lines
1881	WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */	1898	WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */
1882	WARN_ON_ONCE(!rcu_is_leaf_node(rnp) &&	1899	WARN_ON_ONCE(!rcu_is_leaf_node(rnp) &&
1883	rcu_preempt_blocked_readers_cgp(rnp));	1900	rcu_preempt_blocked_readers_cgp(rnp));
1884	rnp->qsmask &= ~mask;	1901	WRITE_ONCE(rnp->qsmask, rnp->qsmask & ~mask);
1885	trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,	1902	trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,
1886	mask, rnp->qsmask, rnp->level,	1903	mask, rnp->qsmask, rnp->level,
1887	rnp->grplo, rnp->grphi,	1904	rnp->grplo, rnp->grphi,
@@ -1904,7 +1921,7 @@ static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp, · 16 unmodified lines
1904	rnp_c = rnp;	1921	rnp_c = rnp;
1905	rnp = rnp->parent;	1922	rnp = rnp->parent;
1906	raw_spin_lock_irqsave_rcu_node(rnp, flags);	1923	raw_spin_lock_irqsave_rcu_node(rnp, flags);
1907	oldmask = rnp_c->qsmask;	1924	oldmask = READ_ONCE(rnp_c->qsmask);
1908	}	1925	}
1909		1926
1910	/*	1927	/*
@@ -1987,6 +2004,8 @@ rcu_report_qs_rdp(int cpu, struct rcu_data *rdp) · 76 unmodified lines
1987	return;	2004	return;
1988	}	2005	}
1989	mask = rdp->grpmask;	2006	mask = rdp->grpmask;
		2007	if (rdp->cpu == smp_processor_id())
		2008	rdp->core_needs_qs = false;
1990	if ((rnp->qsmask & mask) == 0) {	2009	if ((rnp->qsmask & mask) == 0) {
1991	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);	2010	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1992	} else {	2011	} else {
@@ -2052,7 +2071,7 @@ int rcutree_dying_cpu(unsigned int cpu) · 59 unmodified lines
2052	return 0;	2071	return 0;
2053		2072
2054	blkd = !!(rnp->qsmask & rdp->grpmask);	2073	blkd = !!(rnp->qsmask & rdp->grpmask);
2055	trace_rcu_grace_period(rcu_state.name, rnp->gp_seq,	2074	trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
2056	blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));	2075	blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));
2057	return 0;	2076	return 0;
2058	}	2077	}
@@ -2294,10 +2313,13 @@ static void force_qs_rnp(int (f)(struct rcu_data rdp)) · 235 unmodified lines
2294	struct rcu_data *rdp;	2313	struct rcu_data *rdp;
2295	struct rcu_node *rnp;	2314	struct rcu_node *rnp;
2296		2315
		2316	rcu_state.cbovld = rcu_state.cbovldnext;
		2317	rcu_state.cbovldnext = false;
2297	rcu_for_each_leaf_node(rnp) {	2318	rcu_for_each_leaf_node(rnp) {
2298	cond_resched_tasks_rcu_qs();	2319	cond_resched_tasks_rcu_qs();
2299	mask = 0;	2320	mask = 0;
2300	raw_spin_lock_irqsave_rcu_node(rnp, flags);	2321	raw_spin_lock_irqsave_rcu_node(rnp, flags);
		2322	rcu_state.cbovldnext \|= !!rnp->cbovldmask;
2301	if (rnp->qsmask == 0) {	2323	if (rnp->qsmask == 0) {
2302	if (!IS_ENABLED(CONFIG_PREEMPT_RCU) \|\|	2324	if (!IS_ENABLED(CONFIG_PREEMPT_RCU) \|\|
2303	rcu_preempt_blocked_readers_cgp(rnp)) {	2325	rcu_preempt_blocked_readers_cgp(rnp)) {
@@ -2579,11 +2601,48 @@ static void rcu_leak_callback(struct rcu_head *rhp) · 275 unmodified lines
2579	}	2601	}
2580		2602
2581	/*	2603	/*
2582	* Helper function for call_rcu() and friends. The cpu argument will	2604	* Check and if necessary update the leaf rcu_node structure's
2583	* normally be -1, indicating "currently running CPU". It may specify	2605	* ->cbovldmask bit corresponding to the current CPU based on that CPU's
2584	* a CPU only if that CPU is a no-CBs CPU. Currently, only rcu_barrier()	2606	* number of queued RCU callbacks. The caller must hold the leaf rcu_node
2585	* is expected to specify a CPU.	2607	* structure's ->lock.
2586	*/	2608	*/
		2609	static void check_cb_ovld_locked(struct rcu_data rdp, struct rcu_node rnp)
		2610	{
		2611	raw_lockdep_assert_held_rcu_node(rnp);
		2612	if (qovld_calc <= 0)
		2613	return; // Early boot and wildcard value set.
		2614	if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc)
		2615	WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask \| rdp->grpmask);
		2616	else
		2617	WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask & ~rdp->grpmask);
		2618	}
		2619
		2620	/*
		2621	* Check and if necessary update the leaf rcu_node structure's
		2622	* ->cbovldmask bit corresponding to the current CPU based on that CPU's
		2623	* number of queued RCU callbacks. No locks need be held, but the
		2624	* caller must have disabled interrupts.
		2625	*
		2626	* Note that this function ignores the possibility that there are a lot
		2627	* of callbacks all of which have already seen the end of their respective
		2628	* grace periods. This omission is due to the need for no-CBs CPUs to
		2629	* be holding ->nocb_lock to do this check, which is too heavy for a
		2630	* common-case operation.
		2631	*/
		2632	static void check_cb_ovld(struct rcu_data *rdp)
		2633	{
		2634	struct rcu_node *const rnp = rdp->mynode;
		2635
		2636	if (qovld_calc <= 0 \|\|
		2637	((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) ==
		2638	!!(READ_ONCE(rnp->cbovldmask) & rdp->grpmask)))
		2639	return; // Early boot wildcard value or already set correctly.
		2640	raw_spin_lock_rcu_node(rnp);
		2641	check_cb_ovld_locked(rdp, rnp);
		2642	raw_spin_unlock_rcu_node(rnp);
		2643	}
		2644
		2645	/* Helper function for call_rcu() and friends. */
2587	static void	2646	static void
2588	__call_rcu(struct rcu_head *head, rcu_callback_t func)	2647	__call_rcu(struct rcu_head *head, rcu_callback_t func)
2589	{	2648	{
@@ -2621,9 +2680,10 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func) · 31 unmodified lines
2621	rcu_segcblist_init(&rdp->cblist);	2680	rcu_segcblist_init(&rdp->cblist);
2622	}	2681	}
2623		2682
		2683	check_cb_ovld(rdp);
2624	if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags))	2684	if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags))
2625	return; // Enqueued onto ->nocb_bypass, so just leave.	2685	return; // Enqueued onto ->nocb_bypass, so just leave.
2626	/* If we get here, rcu_nocb_try_bypass() acquired ->nocb_lock. */	2686	// If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock.
2627	rcu_segcblist_enqueue(&rdp->cblist, head);	2687	rcu_segcblist_enqueue(&rdp->cblist, head);
2628	if (__is_kfree_rcu_offset((unsigned long)func))	2688	if (__is_kfree_rcu_offset((unsigned long)func))
2629	trace_rcu_kfree_callback(rcu_state.name, head,	2689	trace_rcu_kfree_callback(rcu_state.name, head,
@@ -2689,22 +2749,47 @@ EXPORT_SYMBOL_GPL(call_rcu); · 59 unmodified lines
2689	#define KFREE_DRAIN_JIFFIES (HZ / 50)	2749	#define KFREE_DRAIN_JIFFIES (HZ / 50)
2690	#define KFREE_N_BATCHES 2	2750	#define KFREE_N_BATCHES 2
2691		2751
		2752	/*
		2753	* This macro defines how many entries the "records" array
		2754	* will contain. It is based on the fact that the size of
		2755	* kfree_rcu_bulk_data structure becomes exactly one page.
		2756	*/
		2757	#define KFREE_BULK_MAX_ENTR ((PAGE_SIZE / sizeof(void *)) - 3)
		2758
		2759	/**
		2760	* struct kfree_rcu_bulk_data - single block to store kfree_rcu() pointers
		2761	* @nr_records: Number of active pointers in the array
		2762	* @records: Array of the kfree_rcu() pointers
		2763	* @next: Next bulk object in the block chain
		2764	* @head_free_debug: For debug, when CONFIG_DEBUG_OBJECTS_RCU_HEAD is set
		2765	*/
		2766	struct kfree_rcu_bulk_data {
		2767	unsigned long nr_records;
		2768	void *records[KFREE_BULK_MAX_ENTR];
		2769	struct kfree_rcu_bulk_data *next;
		2770	struct rcu_head *head_free_debug;
		2771	};
		2772
2692	/**	2773	/**
2693	* struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests	2774	* struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests
2694	* @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period	2775	* @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period
2695	* @head_free: List of kfree_rcu() objects waiting for a grace period	2776	* @head_free: List of kfree_rcu() objects waiting for a grace period
		2777	* @bhead_free: Bulk-List of kfree_rcu() objects waiting for a grace period
2696	* @krcp: Pointer to @kfree_rcu_cpu structure	2778	* @krcp: Pointer to @kfree_rcu_cpu structure
2697	*/	2779	*/
2698		2780
2699	struct kfree_rcu_cpu_work {	2781	struct kfree_rcu_cpu_work {
2700	struct rcu_work rcu_work;	2782	struct rcu_work rcu_work;
2701	struct rcu_head *head_free;	2783	struct rcu_head *head_free;
		2784	struct kfree_rcu_bulk_data *bhead_free;
2702	struct kfree_rcu_cpu *krcp;	2785	struct kfree_rcu_cpu *krcp;
2703	};	2786	};
2704		2787
2705	/**	2788	/**
2706	* struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period	2789	* struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period
2707	* @head: List of kfree_rcu() objects not yet waiting for a grace period	2790	* @head: List of kfree_rcu() objects not yet waiting for a grace period
		2791	* @bhead: Bulk-List of kfree_rcu() objects not yet waiting for a grace period
		2792	* @bcached: Keeps at most one object for later reuse when build chain blocks
2708	* @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period	2793	* @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period
2709	* @lock: Synchronize access to this structure	2794	* @lock: Synchronize access to this structure
2710	* @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES	2795	* @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES
@@ -2718,6 +2803,8 @@ struct kfree_rcu_cpu_work { · 7 unmodified lines
2718	*/	2803	*/
2719	struct kfree_rcu_cpu {	2804	struct kfree_rcu_cpu {
2720	struct rcu_head *head;	2805	struct rcu_head *head;
		2806	struct kfree_rcu_bulk_data *bhead;
		2807	struct kfree_rcu_bulk_data *bcached;
2721	struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES];	2808	struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES];
2722	spinlock_t lock;	2809	spinlock_t lock;
2723	struct delayed_work monitor_work;	2810	struct delayed_work monitor_work;
@@ -2727,14 +2814,24 @@ struct kfree_rcu_cpu { · 3 unmodified lines
2727		2814
2728	static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc);	2815	static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc);
2729		2816
		2817	static __always_inline void
		2818	debug_rcu_head_unqueue_bulk(struct rcu_head *head)
		2819	{
		2820	#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
		2821	for (; head; head = head->next)
		2822	debug_rcu_head_unqueue(head);
		2823	#endif
		2824	}
		2825
2730	/*	2826	/*
2731	* This function is invoked in workqueue context after a grace period.	2827	* This function is invoked in workqueue context after a grace period.
2732	* It frees all the objects queued on ->head_free.	2828	* It frees all the objects queued on ->bhead_free or ->head_free.
2733	*/	2829	*/
2734	static void kfree_rcu_work(struct work_struct *work)	2830	static void kfree_rcu_work(struct work_struct *work)
2735	{	2831	{
2736	unsigned long flags;	2832	unsigned long flags;
2737	struct rcu_head head, next;	2833	struct rcu_head head, next;
		2834	struct kfree_rcu_bulk_data bhead, bnext;
2738	struct kfree_rcu_cpu *krcp;	2835	struct kfree_rcu_cpu *krcp;
2739	struct kfree_rcu_cpu_work *krwp;	2836	struct kfree_rcu_cpu_work *krwp;
2740		2837
@@ -2744,22 +2841,44 @@ static void kfree_rcu_work(struct work_struct *work) · 3 unmodified lines
2744	spin_lock_irqsave(&krcp->lock, flags);	2841	spin_lock_irqsave(&krcp->lock, flags);
2745	head = krwp->head_free;	2842	head = krwp->head_free;
2746	krwp->head_free = NULL;	2843	krwp->head_free = NULL;
		2844	bhead = krwp->bhead_free;
		2845	krwp->bhead_free = NULL;
2747	spin_unlock_irqrestore(&krcp->lock, flags);	2846	spin_unlock_irqrestore(&krcp->lock, flags);
2748		2847
2749	// List "head" is now private, so traverse locklessly.	2848	/* "bhead" is now private, so traverse locklessly. */
		2849	for (; bhead; bhead = bnext) {
		2850	bnext = bhead->next;
		2851
		2852	debug_rcu_head_unqueue_bulk(bhead->head_free_debug);
		2853
		2854	rcu_lock_acquire(&rcu_callback_map);
		2855	trace_rcu_invoke_kfree_bulk_callback(rcu_state.name,
		2856	bhead->nr_records, bhead->records);
		2857
		2858	kfree_bulk(bhead->nr_records, bhead->records);
		2859	rcu_lock_release(&rcu_callback_map);
		2860
		2861	if (cmpxchg(&krcp->bcached, NULL, bhead))
		2862	free_page((unsigned long) bhead);
		2863
		2864	cond_resched_tasks_rcu_qs();
		2865	}
		2866
		2867	/*
		2868	* Emergency case only. It can happen under low memory
		2869	* condition when an allocation gets failed, so the "bulk"
		2870	* path can not be temporary maintained.
		2871	*/
2750	for (; head; head = next) {	2872	for (; head; head = next) {
2751	unsigned long offset = (unsigned long)head->func;	2873	unsigned long offset = (unsigned long)head->func;
2752		2874
2753	next = head->next;	2875	next = head->next;
2754	// Potentially optimize with kfree_bulk in future.
2755	debug_rcu_head_unqueue(head);	2876	debug_rcu_head_unqueue(head);
2756	rcu_lock_acquire(&rcu_callback_map);	2877	rcu_lock_acquire(&rcu_callback_map);
2757	trace_rcu_invoke_kfree_callback(rcu_state.name, head, offset);	2878	trace_rcu_invoke_kfree_callback(rcu_state.name, head, offset);
2758		2879
2759	if (!WARN_ON_ONCE(!__is_kfree_rcu_offset(offset))) {	2880	if (!WARN_ON_ONCE(!__is_kfree_rcu_offset(offset)))
2760	/* Could be optimized with kfree_bulk() in future. */
2761	kfree((void *)head - offset);	2881	kfree((void *)head - offset);
2762	}
2763		2882
2764	rcu_lock_release(&rcu_callback_map);	2883	rcu_lock_release(&rcu_callback_map);
2765	cond_resched_tasks_rcu_qs();	2884	cond_resched_tasks_rcu_qs();
@@ -2774,26 +2893,48 @@ static void kfree_rcu_work(struct work_struct *work) · 8 unmodified lines
2774	*/	2893	*/
2775	static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp)	2894	static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp)
2776	{	2895	{
		2896	struct kfree_rcu_cpu_work *krwp;
		2897	bool queued = false;
2777	int i;	2898	int i;
2778	struct kfree_rcu_cpu_work *krwp = NULL;
2779		2899
2780	lockdep_assert_held(&krcp->lock);	2900	lockdep_assert_held(&krcp->lock);
2781	for (i = 0; i < KFREE_N_BATCHES; i++)
2782	if (!krcp->krw_arr[i].head_free) {
2783	krwp = &(krcp->krw_arr[i]);
2784	break;
2785	}
2786		2901
2787	// If a previous RCU batch is in progress, we cannot immediately	2902	for (i = 0; i < KFREE_N_BATCHES; i++) {
2788	// queue another one, so return false to tell caller to retry.	2903	krwp = &(krcp->krw_arr[i]);
2789	if (!krwp)
2790	return false;
2791		2904
2792	krwp->head_free = krcp->head;	2905	/*
2793	krcp->head = NULL;	2906	* Try to detach bhead or head and attach it over any
2794	INIT_RCU_WORK(&krwp->rcu_work, kfree_rcu_work);	2907	* available corresponding free channel. It can be that
2795	queue_rcu_work(system_wq, &krwp->rcu_work);	2908	* a previous RCU batch is in progress, it means that
2796	return true;	2909	* immediately to queue another one is not possible so
		2910	* return false to tell caller to retry.
		2911	*/
		2912	if ((krcp->bhead && !krwp->bhead_free) \|\|
		2913	(krcp->head && !krwp->head_free)) {
		2914	/* Channel 1. */
		2915	if (!krwp->bhead_free) {
		2916	krwp->bhead_free = krcp->bhead;
		2917	krcp->bhead = NULL;
		2918	}
		2919
		2920	/* Channel 2. */
		2921	if (!krwp->head_free) {
		2922	krwp->head_free = krcp->head;
		2923	krcp->head = NULL;
		2924	}
		2925
		2926	/*
		2927	* One work is per one batch, so there are two "free channels",
		2928	* "bhead_free" and "head_free" the batch can handle. It can be
		2929	* that the work is in the pending state when two channels have
		2930	* been detached following each other, one by one.
		2931	*/
		2932	queue_rcu_work(system_wq, &krwp->rcu_work);
		2933	queued = true;
		2934	}
		2935	}
		2936
		2937	return queued;
2797	}	2938	}
2798		2939
2799	static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu *krcp,	2940	static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu *krcp,
@@ -2830,19 +2971,65 @@ static void kfree_rcu_monitor(struct work_struct *work) · 30 unmodified lines
2830	spin_unlock_irqrestore(&krcp->lock, flags);	2971	spin_unlock_irqrestore(&krcp->lock, flags);
2831	}	2972	}
2832		2973
		2974	static inline bool
		2975	kfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp,
		2976	struct rcu_head *head, rcu_callback_t func)
		2977	{
		2978	struct kfree_rcu_bulk_data *bnode;
		2979
		2980	if (unlikely(!krcp->initialized))
		2981	return false;
		2982
		2983	lockdep_assert_held(&krcp->lock);
		2984
		2985	/* Check if a new block is required. */
		2986	if (!krcp->bhead \|\|
		2987	krcp->bhead->nr_records == KFREE_BULK_MAX_ENTR) {
		2988	bnode = xchg(&krcp->bcached, NULL);
		2989	if (!bnode) {
		2990	WARN_ON_ONCE(sizeof(struct kfree_rcu_bulk_data) > PAGE_SIZE);
		2991
		2992	bnode = (struct kfree_rcu_bulk_data *)
		2993	__get_free_page(GFP_NOWAIT \| __GFP_NOWARN);
		2994	}
		2995
		2996	/* Switch to emergency path. */
		2997	if (unlikely(!bnode))
		2998	return false;
		2999
		3000	/* Initialize the new block. */
		3001	bnode->nr_records = 0;
		3002	bnode->next = krcp->bhead;
		3003	bnode->head_free_debug = NULL;
		3004
		3005	/* Attach it to the head. */
		3006	krcp->bhead = bnode;
		3007	}
		3008
		3009	#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
		3010	head->func = func;
		3011	head->next = krcp->bhead->head_free_debug;
		3012	krcp->bhead->head_free_debug = head;
		3013	#endif
		3014
		3015	/* Finally insert. */
		3016	krcp->bhead->records[krcp->bhead->nr_records++] =
		3017	(void *) head - (unsigned long) func;
		3018
		3019	return true;
		3020	}
		3021
2833	/*	3022	/*
2834	* Queue a request for lazy invocation of kfree() after a grace period.	3023	* Queue a request for lazy invocation of kfree_bulk()/kfree() after a grace
		3024	* period. Please note there are two paths are maintained, one is the main one
		3025	* that uses kfree_bulk() interface and second one is emergency one, that is
		3026	* used only when the main path can not be maintained temporary, due to memory
		3027	* pressure.
2835	*	3028	*
2836	* Each kfree_call_rcu() request is added to a batch. The batch will be drained	3029	* Each kfree_call_rcu() request is added to a batch. The batch will be drained
2837	* every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch	3030	* every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch will
2838	* will be kfree'd in workqueue context. This allows us to:	3031	* be free'd in workqueue context. This allows us to: batch requests together to
2839	*	3032	* reduce the number of grace periods during heavy kfree_rcu() load.
2840	* 1. Batch requests together to reduce the number of grace periods during
2841	* heavy kfree_rcu() load.
2842	*
2843	* 2. It makes it possible to use kfree_bulk() on a large number of
2844	* kfree_rcu() requests thus reducing cache misses and the per-object
2845	* overhead of kfree().
2846	*/	3033	*/
2847	void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)	3034	void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
2848	{	3035	{
@@ -2861,9 +3048,16 @@ void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func) · 12 unmodified lines
2861	__func__, head);	3048	__func__, head);
2862	goto unlock_return;	3049	goto unlock_return;
2863	}	3050	}
2864	head->func = func;	3051
2865	head->next = krcp->head;	3052	/*
2866	krcp->head = head;	3053	* Under high memory pressure GFP_NOWAIT can fail,
		3054	* in that case the emergency path is maintained.
		3055	*/
		3056	if (unlikely(!kfree_call_rcu_add_ptr_to_bulk(krcp, head, func))) {
		3057	head->func = func;
		3058	head->next = krcp->head;
		3059	krcp->head = head;
		3060	}
2867		3061
2868	// Set timer to drain after KFREE_DRAIN_JIFFIES.	3062	// Set timer to drain after KFREE_DRAIN_JIFFIES.
2869	if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&	3063	if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
@@ -3075,24 +3269,32 @@ static void rcu_barrier_trace(const char *s, int cpu, unsigned long done) · 205 unmodified lines
3075	/*	3269	/*
3076	* RCU callback function for rcu_barrier(). If we are last, wake	3270	* RCU callback function for rcu_barrier(). If we are last, wake
3077	* up the task executing rcu_barrier().	3271	* up the task executing rcu_barrier().
		3272	*
		3273	* Note that the value of rcu_state.barrier_sequence must be captured
		3274	* before the atomic_dec_and_test(). Otherwise, if this CPU is not last,
		3275	* other CPUs might count the value down to zero before this CPU gets
		3276	* around to invoking rcu_barrier_trace(), which might result in bogus
		3277	* data from the next instance of rcu_barrier().
3078	*/	3278	*/
3079	static void rcu_barrier_callback(struct rcu_head *rhp)	3279	static void rcu_barrier_callback(struct rcu_head *rhp)
3080	{	3280	{
		3281	unsigned long __maybe_unused s = rcu_state.barrier_sequence;
		3282
3081	if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {	3283	if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {
3082	rcu_barrier_trace(TPS("LastCB"), -1,	3284	rcu_barrier_trace(TPS("LastCB"), -1, s);
3083	rcu_state.barrier_sequence);
3084	complete(&rcu_state.barrier_completion);	3285	complete(&rcu_state.barrier_completion);
3085	} else {	3286	} else {
3086	rcu_barrier_trace(TPS("CB"), -1, rcu_state.barrier_sequence);	3287	rcu_barrier_trace(TPS("CB"), -1, s);
3087	}	3288	}
3088	}	3289	}
3089		3290
3090	/*	3291	/*
3091	* Called with preemption disabled, and from cross-cpu IRQ context.	3292	* Called with preemption disabled, and from cross-cpu IRQ context.
3092	*/	3293	*/
3093	static void rcu_barrier_func(void *unused)	3294	static void rcu_barrier_func(void *cpu_in)
3094	{	3295	{
3095	struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);	3296	uintptr_t cpu = (uintptr_t)cpu_in;
		3297	struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3096		3298
3097	rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);	3299	rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);
3098	rdp->barrier_head.func = rcu_barrier_callback;	3300	rdp->barrier_head.func = rcu_barrier_callback;
@@ -3119,7 +3321,7 @@ static void rcu_barrier_func(void *unused) · 20 unmodified lines
3119	*/	3321	*/
3120	void rcu_barrier(void)	3322	void rcu_barrier(void)
3121	{	3323	{
3122	int cpu;	3324	uintptr_t cpu;
3123	struct rcu_data *rdp;	3325	struct rcu_data *rdp;
3124	unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);	3326	unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);
3125		3327
@@ -3142,13 +3344,14 @@ void rcu_barrier(void) · 16 unmodified lines
3142	rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);	3344	rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);
3143		3345
3144	/*	3346	/*
3145	* Initialize the count to one rather than to zero in order to	3347	* Initialize the count to two rather than to zero in order
3146	* avoid a too-soon return to zero in case of a short grace period	3348	* to avoid a too-soon return to zero in case of an immediate
3147	* (or preemption of this task). Exclude CPU-hotplug operations	3349	* invocation of the just-enqueued callback (or preemption of
3148	* to ensure that no offline CPU has callbacks queued.	3350	* this task). Exclude CPU-hotplug operations to ensure that no
		3351	* offline non-offloaded CPU has callbacks queued.
3149	*/	3352	*/
3150	init_completion(&rcu_state.barrier_completion);	3353	init_completion(&rcu_state.barrier_completion);
3151	atomic_set(&rcu_state.barrier_cpu_count, 1);	3354	atomic_set(&rcu_state.barrier_cpu_count, 2);
3152	get_online_cpus();	3355	get_online_cpus();
3153		3356
3154	/*	3357	/*
@@ -3158,13 +3361,23 @@ void rcu_barrier(void) · 3 unmodified lines
3158	*/	3361	*/
3159	for_each_possible_cpu(cpu) {	3362	for_each_possible_cpu(cpu) {
3160	rdp = per_cpu_ptr(&rcu_data, cpu);	3363	rdp = per_cpu_ptr(&rcu_data, cpu);
3161	if (!cpu_online(cpu) &&	3364	if (cpu_is_offline(cpu) &&
3162	!rcu_segcblist_is_offloaded(&rdp->cblist))	3365	!rcu_segcblist_is_offloaded(&rdp->cblist))
3163	continue;	3366	continue;
3164	if (rcu_segcblist_n_cbs(&rdp->cblist)) {	3367	if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) {
3165	rcu_barrier_trace(TPS("OnlineQ"), cpu,	3368	rcu_barrier_trace(TPS("OnlineQ"), cpu,
3166	rcu_state.barrier_sequence);	3369	rcu_state.barrier_sequence);
3167	smp_call_function_single(cpu, rcu_barrier_func, NULL, 1);	3370	smp_call_function_single(cpu, rcu_barrier_func, (void *)cpu, 1);
		3371	} else if (rcu_segcblist_n_cbs(&rdp->cblist) &&
		3372	cpu_is_offline(cpu)) {
		3373	rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu,
		3374	rcu_state.barrier_sequence);
		3375	local_irq_disable();
		3376	rcu_barrier_func((void *)cpu);
		3377	local_irq_enable();
		3378	} else if (cpu_is_offline(cpu)) {
		3379	rcu_barrier_trace(TPS("OfflineNoCBNoQ"), cpu,
		3380	rcu_state.barrier_sequence);
3168	} else {	3381	} else {
3169	rcu_barrier_trace(TPS("OnlineNQ"), cpu,	3382	rcu_barrier_trace(TPS("OnlineNQ"), cpu,
3170	rcu_state.barrier_sequence);	3383	rcu_state.barrier_sequence);
@@ -3176,7 +3389,7 @@ void rcu_barrier(void) · 5 unmodified lines
3176	* Now that we have an rcu_barrier_callback() callback on each	3389	* Now that we have an rcu_barrier_callback() callback on each
3177	* CPU, and thus each counted, remove the initial count.	3390	* CPU, and thus each counted, remove the initial count.
3178	*/	3391	*/
3179	if (atomic_dec_and_test(&rcu_state.barrier_cpu_count))	3392	if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count))
3180	complete(&rcu_state.barrier_completion);	3393	complete(&rcu_state.barrier_completion);
3181		3394
3182	/* Wait for all rcu_barrier_callback() callbacks to be invoked. */	3395	/* Wait for all rcu_barrier_callback() callbacks to be invoked. */
@@ -3275,12 +3488,12 @@ int rcutree_prepare_cpu(unsigned int cpu) · 92 unmodified lines
3275	rnp = rdp->mynode;	3488	rnp = rdp->mynode;
3276	raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */	3489	raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
3277	rdp->beenonline = true; /* We have now been online. */	3490	rdp->beenonline = true; /* We have now been online. */
3278	rdp->gp_seq = rnp->gp_seq;	3491	rdp->gp_seq = READ_ONCE(rnp->gp_seq);
3279	rdp->gp_seq_needed = rnp->gp_seq;	3492	rdp->gp_seq_needed = rdp->gp_seq;
3280	rdp->cpu_no_qs.b.norm = true;	3493	rdp->cpu_no_qs.b.norm = true;
3281	rdp->core_needs_qs = false;	3494	rdp->core_needs_qs = false;
3282	rdp->rcu_iw_pending = false;	3495	rdp->rcu_iw_pending = false;
3283	rdp->rcu_iw_gp_seq = rnp->gp_seq - 1;	3496	rdp->rcu_iw_gp_seq = rdp->gp_seq - 1;
3284	trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));	3497	trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
3285	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);	3498	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3286	rcu_prepare_kthreads(cpu);	3499	rcu_prepare_kthreads(cpu);
@@ -3378,7 +3591,7 @@ void rcu_cpu_starting(unsigned int cpu) · 91 unmodified lines
3378	rnp = rdp->mynode;	3591	rnp = rdp->mynode;
3379	mask = rdp->grpmask;	3592	mask = rdp->grpmask;
3380	raw_spin_lock_irqsave_rcu_node(rnp, flags);	3593	raw_spin_lock_irqsave_rcu_node(rnp, flags);
3381	rnp->qsmaskinitnext \|= mask;	3594	WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext \| mask);
3382	oldmask = rnp->expmaskinitnext;	3595	oldmask = rnp->expmaskinitnext;
3383	rnp->expmaskinitnext \|= mask;	3596	rnp->expmaskinitnext \|= mask;
3384	oldmask ^= rnp->expmaskinitnext;	3597	oldmask ^= rnp->expmaskinitnext;
@@ -3431,7 +3644,7 @@ void rcu_report_dead(unsigned int cpu) · 46 unmodified lines
3431	rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);	3644	rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
3432	raw_spin_lock_irqsave_rcu_node(rnp, flags);	3645	raw_spin_lock_irqsave_rcu_node(rnp, flags);
3433	}	3646	}
3434	rnp->qsmaskinitnext &= ~mask;	3647	WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext & ~mask);
3435	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);	3648	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3436	raw_spin_unlock(&rcu_state.ofl_lock);	3649	raw_spin_unlock(&rcu_state.ofl_lock);
3437		3650
@@ -3545,7 +3758,10 @@ static int __init rcu_spawn_gp_kthread(void) · 107 unmodified lines
3545	}	3758	}
3546	rnp = rcu_get_root();	3759	rnp = rcu_get_root();
3547	raw_spin_lock_irqsave_rcu_node(rnp, flags);	3760	raw_spin_lock_irqsave_rcu_node(rnp, flags);
3548	rcu_state.gp_kthread = t;	3761	WRITE_ONCE(rcu_state.gp_activity, jiffies);
		3762	WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
		3763	// Reset .gp_activity and .gp_req_activity before setting .gp_kthread.
		3764	smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */
3549	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);	3765	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3550	wake_up_process(t);	3766	wake_up_process(t);
3551	rcu_spawn_nocb_kthreads();	3767	rcu_spawn_nocb_kthreads();
@@ -3769,8 +3985,11 @@ static void __init kfree_rcu_batch_init(void) · 217 unmodified lines
3769	struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);	3985	struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
3770		3986
3771	spin_lock_init(&krcp->lock);	3987	spin_lock_init(&krcp->lock);
3772	for (i = 0; i < KFREE_N_BATCHES; i++)	3988	for (i = 0; i < KFREE_N_BATCHES; i++) {
		3989	INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
3773	krcp->krw_arr[i].krcp = krcp;	3990	krcp->krw_arr[i].krcp = krcp;
		3991	}
		3992
3774	INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);	3993	INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
3775	krcp->initialized = true;	3994	krcp->initialized = true;
3776	}	3995	}
@@ -3809,6 +4028,13 @@ void __init rcu_init(void) · 32 unmodified lines
3809	rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);	4028	rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);
3810	WARN_ON(!rcu_par_gp_wq);	4029	WARN_ON(!rcu_par_gp_wq);
3811	srcu_init();	4030	srcu_init();
		4031
		4032	/* Fill in default value for rcutree.qovld boot parameter. */
		4033	/* -After- the rcu_node ->lock fields are initialized! */
		4034	if (qovld < 0)
		4035	qovld_calc = DEFAULT_RCU_QOVLD_MULT * qhimark;
		4036	else
		4037	qovld_calc = qovld;
3812	}	4038	}
3813		4039
3814	#include "tree_stall.h"	4040	#include "tree_stall.h"

kernel/rcu/tree.h+4−0

 				/* Online CPUs for next expedited GP. */
 				/*  Any CPU that has ever been online will */
 				/*  have its bit set. */
+	unsigned long cbovldmask;
+				/* CPUs experiencing callback overload. */
 	unsigned long ffmask;	/* Fully functional CPUs. */
 	unsigned long grpmask;	/* Mask to apply to parent qsmask. */
 				/*  Only one bit will be set in this mask. */
 	atomic_t expedited_need_qs;		/* # CPUs left to check in. */
 	struct swait_queue_head expedited_wq;	/* Wait for check-ins. */
 	int ncpus_snap;				/* # CPUs seen last time. */
+	u8 cbovld;				/* Callback overload now? */
+	u8 cbovldnext;				/* ^        ^  next time? */
 	unsigned long jiffies_force_qs;		/* Time at which to invoke */
 						/*  force_quiescent_state(). */

kernel/rcu/tree_exp.h+9−4

 				   sync_exp_work_done(s));
 			return true;
 		}
-		rnp->exp_seq_rq = s; /* Followers can wait on us. */
+		WRITE_ONCE(rnp->exp_seq_rq, s); /* Followers can wait on us. */
 		spin_unlock(&rnp->exp_lock);
 		trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level,
 					  rnp->grplo, rnp->grphi, TPS("nxtlvl"));
 static void synchronize_rcu_expedited_wait(void)
 {
 	int cpu;
+	unsigned long j;
 	unsigned long jiffies_stall;
 	unsigned long jiffies_start;
 	unsigned long mask;
 	trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("startwait"));
 	jiffies_stall = rcu_jiffies_till_stall_check();
 	jiffies_start = jiffies;
-	if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
+	if (tick_nohz_full_enabled() && rcu_inkernel_boot_has_ended()) {
 		if (synchronize_rcu_expedited_wait_once(1))
 			return;
 		rcu_for_each_leaf_node(rnp) {
 				tick_dep_set_cpu(cpu, TICK_DEP_BIT_RCU_EXP);
 			}
 		}
+		j = READ_ONCE(jiffies_till_first_fqs);
+		if (synchronize_rcu_expedited_wait_once(j + HZ))
+			return;
+		WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT));
 	}
 	for (;;) {
 		if (synchronize_rcu_expedited_wait_once(jiffies_stall))
 			return;
-		if (rcu_cpu_stall_suppress)
+		if (rcu_stall_is_suppressed())
 			continue;
 		panic_on_rcu_stall();
 		pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {",
 			spin_lock(&rnp->exp_lock);
 			/* Recheck, avoid hang in case someone just arrived. */
 			if (ULONG_CMP_LT(rnp->exp_seq_rq, s))
-				rnp->exp_seq_rq = s;
+				WRITE_ONCE(rnp->exp_seq_rq, s);
 			spin_unlock(&rnp->exp_lock);
 		}
 		smp_mb(); /* All above changes before wakeup. */

kernel/rcu/tree_plugin.h+16−9

 		pr_info("\tBoot-time adjustment of callback high-water mark to %ld.\n", qhimark);
 	if (qlowmark != DEFAULT_RCU_QLOMARK)
 		pr_info("\tBoot-time adjustment of callback low-water mark to %ld.\n", qlowmark);
+	if (qovld != DEFAULT_RCU_QOVLD)
+		pr_info("\tBoot-time adjustment of callback overload level to %ld.\n", qovld);
 	if (jiffies_till_first_fqs != ULONG_MAX)
 		pr_info("\tBoot-time adjustment of first FQS scan delay to %ld jiffies.\n", jiffies_till_first_fqs);
 	if (jiffies_till_next_fqs != ULONG_MAX)
 	raw_lockdep_assert_held_rcu_node(rnp);
 	pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
 		__func__, rnp->grplo, rnp->grphi, rnp->level,
-		(long)rnp->gp_seq, (long)rnp->completedqs);
+		(long)READ_ONCE(rnp->gp_seq), (long)rnp->completedqs);
 	for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent)
 		pr_info("%s: %d:%d ->qsmask %#lx ->qsmaskinit %#lx ->qsmaskinitnext %#lx\n",
 			__func__, rnp1->grplo, rnp1->grphi, rnp1->qsmask, rnp1->qsmaskinit, rnp1->qsmaskinitnext);
 	trace_rcu_utilization(TPS("Start boost kthread@init"));
 	for (;;) {
-		rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
+		WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_WAITING);
 		trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
 		rcu_wait(rnp->boost_tasks || rnp->exp_tasks);
 		trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
-		rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
+		WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_RUNNING);
 		more2boost = rcu_boost(rnp);
 		if (more2boost)
 			spincnt++;
 		else
 			spincnt = 0;
 		if (spincnt > 10) {
-			rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;
+			WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_YIELDING);
 			trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
 			schedule_timeout_interruptible(2);
 			trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
 	    (rnp->gp_tasks != NULL &&
 	     rnp->boost_tasks == NULL &&
 	     rnp->qsmask == 0 &&
-	     ULONG_CMP_GE(jiffies, rnp->boost_time))) {
+	     (ULONG_CMP_GE(jiffies, rnp->boost_time) || rcu_state.cbovld))) {
 		if (rnp->exp_tasks == NULL)
 			rnp->boost_tasks = rnp->gp_tasks;
 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
 		rcu_wake_cond(rnp->boost_kthread_task,
-			      rnp->boost_kthread_status);
+			      READ_ONCE(rnp->boost_kthread_status));
 	} else {
 		raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
 	}
  * flag the contention.
  */
 static void rcu_nocb_bypass_lock(struct rcu_data *rdp)
+	__acquires(&rdp->nocb_bypass_lock)
 {
 	lockdep_assert_irqs_disabled();
 	if (raw_spin_trylock(&rdp->nocb_bypass_lock))
  * Release the specified rcu_data structure's ->nocb_bypass_lock.
  */
 static void rcu_nocb_bypass_unlock(struct rcu_data *rdp)
+	__releases(&rdp->nocb_bypass_lock)
 {
 	lockdep_assert_irqs_disabled();
 	raw_spin_unlock(&rdp->nocb_bypass_lock);
 static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)
 {
 	lockdep_assert_irqs_disabled();
-	if (rcu_segcblist_is_offloaded(&rdp->cblist) &&
+	if (rcu_segcblist_is_offloaded(&rdp->cblist))
-	    cpu_online(rdp->cpu))
 		lockdep_assert_held(&rdp->nocb_lock);
 }
 	struct rcu_data *rdp;
 	struct rcu_node *rnp;
 	unsigned long wait_gp_seq = 0; // Suppress "use uninitialized" warning.
+	bool wasempty = false;
 	/*
 	 * Each pass through the following loop checks for CBs and for the
 		     rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) {
 			raw_spin_lock_rcu_node(rnp); /* irqs disabled. */
 			needwake_gp = rcu_advance_cbs(rnp, rdp);
+			wasempty = rcu_segcblist_restempty(&rdp->cblist,
+							   RCU_NEXT_READY_TAIL);
 			raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */
 		}
 		// Need to wait on some grace period?
-		WARN_ON_ONCE(!rcu_segcblist_restempty(&rdp->cblist,
+		WARN_ON_ONCE(wasempty &&
+			     !rcu_segcblist_restempty(&rdp->cblist,
 						      RCU_NEXT_READY_TAIL));
 		if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) {
 			if (!needwait_gp ||

kernel/rcu/tree_stall.h+23−18

 	unsigned long j = jiffies;
 	unsigned long j1;
-	rcu_state.gp_start = j;
+	WRITE_ONCE(rcu_state.gp_start, j);
 	j1 = rcu_jiffies_till_stall_check();
 	/* Record ->gp_start before ->jiffies_stall. */
 	smp_store_release(&rcu_state.jiffies_stall, j + j1); /* ^^^ */
 	/* Kick and suppress, if so configured. */
 	rcu_stall_kick_kthreads();
-	if (rcu_cpu_stall_suppress)
+	if (rcu_stall_is_suppressed())
 		return;
 	/*
 	/* Kick and suppress, if so configured. */
 	rcu_stall_kick_kthreads();
-	if (rcu_cpu_stall_suppress)
+	if (rcu_stall_is_suppressed())
 		return;
 	/*
 	unsigned long js;
 	struct rcu_node *rnp;
-	if ((rcu_cpu_stall_suppress && !rcu_kick_kthreads) ||
+	if ((rcu_stall_is_suppressed() && !rcu_kick_kthreads) ||
 	    !rcu_gp_in_progress())
 		return;
 	rcu_stall_kick_kthreads();
 	unsigned long jw;
 	struct rcu_data *rdp;
 	struct rcu_node *rnp;
+	struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
 	j = jiffies;
 	ja = j - READ_ONCE(rcu_state.gp_activity);
 	jw = j - READ_ONCE(rcu_state.gp_wake_time);
 	pr_info("%s: wait state: %s(%d) ->state: %#lx delta ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_flags %#x\n",
 		rcu_state.name, gp_state_getname(rcu_state.gp_state),
-		rcu_state.gp_state,
+		rcu_state.gp_state, t ? t->state : 0x1ffffL,
-		rcu_state.gp_kthread ? rcu_state.gp_kthread->state : 0x1ffffL,
 		ja, jr, jw, (long)READ_ONCE(rcu_state.gp_wake_seq),
 		(long)READ_ONCE(rcu_state.gp_seq),
 		(long)READ_ONCE(rcu_get_root()->gp_seq_needed),
 		READ_ONCE(rcu_state.gp_flags));
 	rcu_for_each_node_breadth_first(rnp) {
-		if (ULONG_CMP_GE(rcu_state.gp_seq, rnp->gp_seq_needed))
+		if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq),
+				 READ_ONCE(rnp->gp_seq_needed)))
 			continue;
 		pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld\n",
-			rnp->grplo, rnp->grphi, (long)rnp->gp_seq,
+			rnp->grplo, rnp->grphi, (long)READ_ONCE(rnp->gp_seq),
-			(long)rnp->gp_seq_needed);
+			(long)READ_ONCE(rnp->gp_seq_needed));
 		if (!rcu_is_leaf_node(rnp))
 			continue;
 		for_each_leaf_node_possible_cpu(rnp, cpu) {
 			rdp = per_cpu_ptr(&rcu_data, cpu);
-			if (rdp->gpwrap ||
+			if (READ_ONCE(rdp->gpwrap) ||
-			    ULONG_CMP_GE(rcu_state.gp_seq,
+			    ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq),
-					 rdp->gp_seq_needed))
+					 READ_ONCE(rdp->gp_seq_needed)))
 				continue;
 			pr_info("\tcpu %d ->gp_seq_needed %ld\n",
-				cpu, (long)rdp->gp_seq_needed);
+				cpu, (long)READ_ONCE(rdp->gp_seq_needed));
 		}
 	}
 	for_each_possible_cpu(cpu) {
 	static atomic_t warned = ATOMIC_INIT(0);
 	if (!IS_ENABLED(CONFIG_PROVE_RCU) || rcu_gp_in_progress() ||
-	    ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed))
+	    ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
+			 READ_ONCE(rnp_root->gp_seq_needed)) ||
+	    !smp_load_acquire(&rcu_state.gp_kthread)) // Get stable kthread.
 		return;
 	j = jiffies; /* Expensive access, and in common case don't get here. */
 	if (time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
 	raw_spin_lock_irqsave_rcu_node(rnp, flags);
 	j = jiffies;
 	if (rcu_gp_in_progress() ||
-	    ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) ||
+	    ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
+			 READ_ONCE(rnp_root->gp_seq_needed)) ||
 	    time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
 	    time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
 	    atomic_read(&warned)) {
 		raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
 	j = jiffies;
 	if (rcu_gp_in_progress() ||
-	    ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) ||
+	    ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
-	    time_before(j, rcu_state.gp_req_activity + gpssdelay) ||
+			 READ_ONCE(rnp_root->gp_seq_needed)) ||
-	    time_before(j, rcu_state.gp_activity + gpssdelay) ||
+	    time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) ||
+	    time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) ||
 	    atomic_xchg(&warned, 1)) {
 		if (rnp_root != rnp)
 			/* irqs remain disabled. */

kernel/rcu/update.c+23−5

 }
 EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);
+static bool rcu_boot_ended __read_mostly;
 /*
  * Inform RCU of the end of the in-kernel boot sequence.
  */
 	rcu_unexpedite_gp();
 	if (rcu_normal_after_boot)
 		WRITE_ONCE(rcu_normal, 1);
+	rcu_boot_ended = 1;
 }
+/*
+ * Let rcutorture know when it is OK to turn it up to eleven.
+ */
+bool rcu_inkernel_boot_has_ended(void)
+{
+	return rcu_boot_ended;
+}
+EXPORT_SYMBOL_GPL(rcu_inkernel_boot_has_ended);
 #endif /* #ifndef CONFIG_TINY_RCU */
 /*
 #ifdef CONFIG_RCU_STALL_COMMON
 int rcu_cpu_stall_ftrace_dump __read_mostly;
 module_param(rcu_cpu_stall_ftrace_dump, int, 0644);
-int rcu_cpu_stall_suppress __read_mostly; /* 1 = suppress stall warnings. */
+int rcu_cpu_stall_suppress __read_mostly; // !0 = suppress stall warnings.
 EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress);
 module_param(rcu_cpu_stall_suppress, int, 0644);
 int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
 module_param(rcu_cpu_stall_timeout, int, 0644);
 #endif /* #ifdef CONFIG_RCU_STALL_COMMON */
+// Suppress boot-time RCU CPU stall warnings and rcutorture writer stall
+// warnings.  Also used by rcutorture even if stall warnings are excluded.
+int rcu_cpu_stall_suppress_at_boot __read_mostly; // !0 = suppress boot stalls.
+EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress_at_boot);
+module_param(rcu_cpu_stall_suppress_at_boot, int, 0444);
 #ifdef CONFIG_TASKS_RCU
 /*
 	rhp->func = func;
 	raw_spin_lock_irqsave(&rcu_tasks_cbs_lock, flags);
 	needwake = !rcu_tasks_cbs_head;
-	*rcu_tasks_cbs_tail = rhp;
+	WRITE_ONCE(*rcu_tasks_cbs_tail, rhp);
 	rcu_tasks_cbs_tail = &rhp->next;
 	raw_spin_unlock_irqrestore(&rcu_tasks_cbs_lock, flags);
 	/* We can't create the thread unless interrupts are enabled. */
 		/* If there were none, wait a bit and start over. */
 		if (!list) {
 			wait_event_interruptible(rcu_tasks_cbs_wq,
-						 rcu_tasks_cbs_head);
+						 READ_ONCE(rcu_tasks_cbs_head));
 			if (!rcu_tasks_cbs_head) {
 				WARN_ON(signal_pending(current));
 				schedule_timeout_interruptible(HZ/10);
 core_initcall(rcu_spawn_tasks_kthread);
 /* Do the srcu_read_lock() for the above synchronize_srcu().  */
-void exit_tasks_rcu_start(void)
+void exit_tasks_rcu_start(void) __acquires(&tasks_rcu_exit_srcu)
 {
 	preempt_disable();
 	current->rcu_tasks_idx = __srcu_read_lock(&tasks_rcu_exit_srcu);
 }
 /* Do the srcu_read_unlock() for the above synchronize_srcu().  */
-void exit_tasks_rcu_finish(void)
+void exit_tasks_rcu_finish(void) __releases(&tasks_rcu_exit_srcu)
 {
 	preempt_disable();
 	__srcu_read_unlock(&tasks_rcu_exit_srcu, current->rcu_tasks_idx);

kernel/time/timer.c+4−3

 #define MOD_TIMER_PENDING_ONLY		0x01
 #define MOD_TIMER_REDUCE		0x02
+#define MOD_TIMER_NOTPENDING		0x04
 static inline int
 __mod_timer(struct timer_list *timer, unsigned long expires, unsigned int options)
 	 * the timer is re-modified to have the same timeout or ends up in the
 	 * same array bucket then just return:
 	 */
-	if (timer_pending(timer)) {
+	if (!(options & MOD_TIMER_NOTPENDING) && timer_pending(timer)) {
 		/*
 		 * The downside of this optimization is that it can result in
 		 * larger granularity than you would get from adding a new
 void add_timer(struct timer_list *timer)
 {
 	BUG_ON(timer_pending(timer));
-	mod_timer(timer, timer->expires);
+	__mod_timer(timer, timer->expires, MOD_TIMER_NOTPENDING);
 }
 EXPORT_SYMBOL(add_timer);
 	timer.task = current;
 	timer_setup_on_stack(&timer.timer, process_timeout, 0);
-	__mod_timer(&timer.timer, expire, 0);
+	__mod_timer(&timer.timer, expire, MOD_TIMER_NOTPENDING);
 	schedule();
 	del_singleshot_timer_sync(&timer.timer);

kernel/torture.c+25−4

 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
+static bool disable_onoff_at_boot;
+module_param(disable_onoff_at_boot, bool, 0444);
 static char *torture_type;
 static int verbose;
 {
 	unsigned long delta;
 	int ret;
+	char *s;
 	unsigned long starttime;
 	if (!cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
 	(*n_offl_attempts)++;
 	ret = cpu_down(cpu);
 	if (ret) {
+		s = "";
+		if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
+			// PCI probe frequently disables hotplug during boot.
+			(*n_offl_attempts)--;
+			s = " (-EBUSY forgiven during boot)";
+		}
 		if (verbose)
 			pr_alert("%s" TORTURE_FLAG
-				 "torture_onoff task: offline %d failed: errno %d\n",
+				 "torture_onoff task: offline %d failed%s: errno %d\n",
-				 torture_type, cpu, ret);
+				 torture_type, cpu, s, ret);
 	} else {
 		if (verbose > 1)
 			pr_alert("%s" TORTURE_FLAG
 {
 	unsigned long delta;
 	int ret;
+	char *s;
 	unsigned long starttime;
 	if (cpu_online(cpu) || !cpu_is_hotpluggable(cpu))
 	(*n_onl_attempts)++;
 	ret = cpu_up(cpu);
 	if (ret) {
+		s = "";
+		if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
+			// PCI probe frequently disables hotplug during boot.
+			(*n_onl_attempts)--;
+			s = " (-EBUSY forgiven during boot)";
+		}
 		if (verbose)
 			pr_alert("%s" TORTURE_FLAG
-				 "torture_onoff task: online %d failed: errno %d\n",
+				 "torture_onoff task: online %d failed%s: errno %d\n",
-				 torture_type, cpu, ret);
+				 torture_type, cpu, s, ret);
 	} else {
 		if (verbose > 1)
 			pr_alert("%s" TORTURE_FLAG
 		VERBOSE_TOROUT_STRING("torture_onoff end holdoff");
 	}
 	while (!torture_must_stop()) {
+		if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) {
+			schedule_timeout_interruptible(HZ / 10);
+			continue;
+		}
 		cpu = (torture_random(&rand) >> 4) % (maxcpu + 1);
 		if (!torture_offline(cpu,
 				     &n_offline_attempts, &n_offline_successes,

tools/testing/selftests/rcutorture/bin/functions.sh+1−1

@@ -12,7 +12,7 @@ · 11 unmodified lines
12	# Returns 1 if the specified boot-parameter string tells rcutorture to	12	# Returns 1 if the specified boot-parameter string tells rcutorture to
13	# test CPU-hotplug operations.	13	# test CPU-hotplug operations.
14	bootparam_hotplug_cpu () {	14	bootparam_hotplug_cpu () {
15	echo "$1" \| grep -q "rcutorture\.onoff_"	15	echo "$1" \| grep -q "torture\.onoff_"
16	}	16	}
17		17
18	# checkarg --argname argtype $# arg mustmatch cannotmatch	18	# checkarg --argname argtype $# arg mustmatch cannotmatch

tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh+2−0

 rundir="${1}"
 if test -z "$rundir" -o ! -d "$rundir"
 then
+	echo Directory "$rundir" not found.
 	echo Usage: $0 directory
+	exit 1
 fi
 editor=${EDITOR-vi}

tools/testing/selftests/rcutorture/bin/kvm-recheck.sh+16−1

@@ -13,6 +13,9 @@ · 12 unmodified lines
13	#	13	#
14	# Authors: Paul E. McKenney <paulmck@linux.ibm.com>	14	# Authors: Paul E. McKenney <paulmck@linux.ibm.com>
15		15
		16	T=/tmp/kvm-recheck.sh.$$
		17	trap 'rm -f $T' 0 2
		18
16	PATH=`pwd`/tools/testing/selftests/rcutorture/bin:$PATH; export PATH	19	PATH=`pwd`/tools/testing/selftests/rcutorture/bin:$PATH; export PATH
17	. functions.sh	20	. functions.sh
18	for rd in "$@"	21	for rd in "$@"
@@ -68,4 +71,16 @@ do · 49 unmodified lines
68	fi	71	fi
69	done	72	done
70	done	73	done
71	EDITOR=echo kvm-find-errors.sh "${@: -1}" > /dev/null 2>&1	74	EDITOR=echo kvm-find-errors.sh "${@: -1}" > $T 2>&1
		75	ret=$?
		76	builderrors="`tr ' ' '\012' < $T \| grep -c '/Make.out.diags'`"
		77	if test "$builderrors" -gt 0
		78	then
		79	echo $builderrors runs with build errors.
		80	fi
		81	runerrors="`tr ' ' '\012' < $T \| grep -c '/console.log.diags'`"
		82	if test "$runerrors" -gt 0
		83	then
		84	echo $runerrors runs with runtime errors.
		85	fi
		86	exit $ret

tools/testing/selftests/rcutorture/bin/kvm.sh+1−1

@@ -39,7 +39,7 @@ TORTURE_TRUST_MAKE="" · 38 unmodified lines
39	resdir=""	39	resdir=""
40	configs=""	40	configs=""
41	cpus=0	41	cpus=0
42	ds=`date +%Y.%m.%d-%H:%M:%S`	42	ds=`date +%Y.%m.%d-%H.%M.%S`
43	jitter="-1"	43	jitter="-1"
44		44
45	usage () {	45	usage () {

tools/testing/selftests/rcutorture/configs/rcu/CFcommon+2−0

@@ -3,3 +3,5 @@ CONFIG_PRINTK_TIME=y · 2 unmodified lines
3	CONFIG_HYPERVISOR_GUEST=y	3	CONFIG_HYPERVISOR_GUEST=y
4	CONFIG_PARAVIRT=y	4	CONFIG_PARAVIRT=y
5	CONFIG_KVM_GUEST=y	5	CONFIG_KVM_GUEST=y
		6	CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=n
		7	CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY=n

tools/testing/selftests/rcutorture/configs/rcu/TREE10+18−0

@@ -0,0 +1,18 @@
	1	CONFIG_SMP=y
	2	CONFIG_NR_CPUS=100
	3	CONFIG_PREEMPT_NONE=y
	4	CONFIG_PREEMPT_VOLUNTARY=n
	5	CONFIG_PREEMPT=n
	6	#CHECK#CONFIG_TREE_RCU=y
	7	CONFIG_HZ_PERIODIC=n
	8	CONFIG_NO_HZ_IDLE=y
	9	CONFIG_NO_HZ_FULL=n
	10	CONFIG_RCU_FAST_NO_HZ=n
	11	CONFIG_RCU_TRACE=n
	12	CONFIG_RCU_NOCB_CPU=n
	13	CONFIG_DEBUG_LOCK_ALLOC=n
	14	CONFIG_PROVE_LOCKING=n
	15	#CHECK#CONFIG_PROVE_RCU=n
	16	CONFIG_DEBUG_OBJECTS=n
	17	CONFIG_DEBUG_OBJECTS_RCU_HEAD=n
	18	CONFIG_RCU_EXPERT=n