Merge branches 'doc.2020.02.27a', 'fixes.2020.03.21a', 'kfree_rcu.2020.02.20a', 'locktorture.2020.02.20a', 'ovld.2020.02.20a', 'rcu-tasks.2020.02.20a', 'srcu.2020.02.20a' and 'torture.2020.02.20a' into HEAD

doc.2020.02.27a: Documentation updates. fixes.2020.03.21a: Miscellaneous fixes. kfree_rcu.2020.02.20a: Updates to kfree_rcu(). locktorture.2020.02.20a: Lock torture-test updates. ovld.2020.02.20a: Updates to callback-overload handling. rcu-tasks.2020.02.20a: RCU-tasks updates. srcu.2020.02.20a: SRCU updates. torture.2020.02.20a: Torture-test updates.

Paul E. McKenney committed 6 years ago

Commit aa93ec68 parents 8149b5c 127e298 6137079 28e09a2 b692dc4 90ba11b 7104260 a14493529 files · +644 −203Browse files

Unified · Split

29 files

Documentation/admin-guide/kernel-parameters.txt +19 −0
fs/nfs/dir.c +1 −1
include/linux/rculist.h +2 −2
include/linux/rcutiny.h +1 −0
include/linux/rcutree.h +1 −0
include/linux/timer.h +1 −1
include/trace/events/rcu.h +29 −0
kernel/locking/locktorture.c +8 −7
kernel/locking/rtmutex.c +1 −1
kernel/rcu/Makefile +4 −0
kernel/rcu/rcu.h +21 −2
kernel/rcu/rcu_segcblist.c +1 −3
kernel/rcu/rcuperf.c +12 −2
kernel/rcu/rcutorture.c +50 −17
kernel/rcu/srcutree.c +10 −8
kernel/rcu/tree.c +339 −113
kernel/rcu/tree.h +4 −0
kernel/rcu/tree_exp.h +9 −4
kernel/rcu/tree_plugin.h +16 −9
kernel/rcu/tree_stall.h +23 −18
kernel/rcu/update.c +23 −5
kernel/time/timer.c +4 −3
kernel/torture.c +25 −4
tools/testing/selftests/rcutorture/bin/functions.sh +1 −1
tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh +2 −0
tools/testing/selftests/rcutorture/bin/kvm-recheck.sh +16 −1
tools/testing/selftests/rcutorture/bin/kvm.sh +1 −1
tools/testing/selftests/rcutorture/configs/rcu/CFcommon +2 −0
tools/testing/selftests/rcutorture/configs/rcu/TREE10 +18 −0

Documentation/admin-guide/kernel-parameters.txt+19−0

@@ -3980,6 +3980,15 @@ · 3979 unmodified lines
3980	3980	Set threshold of queued RCU callbacks below which
3981	3981	batch limiting is re-enabled.
3982	3982
	3983	+ rcutree.qovld= [KNL]
	3984	+ Set threshold of queued RCU callbacks beyond which
	3985	+ RCU's force-quiescent-state scan will aggressively
	3986	+ enlist help from cond_resched() and sched IPIs to
	3987	+ help CPUs more quickly reach quiescent states.
	3988	+ Set to less than zero to make this be set based
	3989	+ on rcutree.qhimark at boot time and to zero to
	3990	+ disable more aggressive help enlistment.
	3991	+
3983	3992	rcutree.rcu_idle_gp_delay= [KNL]
3984	3993	Set wakeup interval for idle CPUs that have
3985	3994	RCU callbacks (RCU_FAST_NO_HZ=y).
@@ -4195,6 +4204,12 @@ · 209 unmodified lines
4195	4204	rcupdate.rcu_cpu_stall_suppress= [KNL]
4196	4205	Suppress RCU CPU stall warning messages.
4197	4206
	4207	+ rcupdate.rcu_cpu_stall_suppress_at_boot= [KNL]
	4208	+ Suppress RCU CPU stall warning messages and
	4209	+ rcutorture writer stall warnings that occur
	4210	+ during early boot, that is, during the time
	4211	+ before the init task is spawned.
	4212	+
4198	4213	rcupdate.rcu_cpu_stall_timeout= [KNL]
4199	4214	Set timeout for RCU CPU stall warning messages.
4200	4215
@@ -4867,6 +4882,10 @@ · 666 unmodified lines
4867	4882	topology updates sent by the hypervisor to this
4868	4883	LPAR.
4869	4884
	4885	+ torture.disable_onoff_at_boot= [KNL]
	4886	+ Prevent the CPU-hotplug component of torturing
	4887	+ until after init has spawned.
	4888	+
4870	4889	tp720= [HW,PS2]
4871	4890
4872	4891	tpm_suspend_pcr=[HW,TPM]

fs/nfs/dir.c+1−1

@@ -2383,7 +2383,7 @@ static int nfs_access_get_cached_rcu(struct inode inode, const struct cred cre · 2382 unmodified lines
2383	2383	rcu_read_lock();
2384	2384	if (nfsi->cache_validity & NFS_INO_INVALID_ACCESS)
2385	2385	goto out;
2386		− lh = rcu_dereference(nfsi->access_cache_entry_lru.prev);
	2386	+ lh = rcu_dereference(list_tail_rcu(&nfsi->access_cache_entry_lru));
2387	2387	cache = list_entry(lh, struct nfs_access_entry, lru);
2388	2388	if (lh == &nfsi->access_cache_entry_lru \|\|
2389	2389	cred_fscmp(cred, cache->cred) != 0)

include/linux/rculist.h+2−2

@@ -60,9 +60,9 @@ static inline void INIT_LIST_HEAD_RCU(struct list_head *list) · 59 unmodified lines
60	60	#define __list_check_rcu(dummy, cond, extra...) \
61	61	({ \
62	62	check_arg_count_one(extra); \
63		− RCU_LOCKDEP_WARN(!cond && !rcu_read_lock_any_held(), \
	63	+ RCU_LOCKDEP_WARN(!(cond) && !rcu_read_lock_any_held(), \
64	64	"RCU-list traversed in non-reader section!"); \
65		− })
	65	+ })
66	66	#else
67	67	#define __list_check_rcu(dummy, cond, extra...) \
68	68	({ check_arg_count_one(extra); })

include/linux/rcutiny.h+1−0

@@ -83,6 +83,7 @@ void rcu_scheduler_starting(void); · 82 unmodified lines
83	83	static inline void rcu_scheduler_starting(void) { }
84	84	#endif /* #else #ifndef CONFIG_SRCU */
85	85	static inline void rcu_end_inkernel_boot(void) { }
	86	+static inline bool rcu_inkernel_boot_has_ended(void) { return true; }
86	87	static inline bool rcu_is_watching(void) { return true; }
87	88	static inline void rcu_momentary_dyntick_idle(void) { }
88	89	static inline void kfree_rcu_scheduler_running(void) { }

include/linux/rcutree.h+1−0

@@ -54,6 +54,7 @@ void exit_rcu(void); · 53 unmodified lines
54	54	void rcu_scheduler_starting(void);
55	55	extern int rcu_scheduler_active __read_mostly;
56	56	void rcu_end_inkernel_boot(void);
	57	+bool rcu_inkernel_boot_has_ended(void);
57	58	bool rcu_is_watching(void);
58	59	#ifndef CONFIG_PREEMPTION
59	60	void rcu_all_qs(void);

include/linux/timer.h+1−1

@@ -164,7 +164,7 @@ static inline void destroy_timer_on_stack(struct timer_list *timer) { } · 163 unmodified lines
164	164	*/
165	165	static inline int timer_pending(const struct timer_list * timer)
166	166	{
167		− return timer->entry.pprev != NULL;
	167	+ return !hlist_unhashed_lockless(&timer->entry);
168	168	}
169	169
170	170	extern void add_timer_on(struct timer_list *timer, int cpu);

include/trace/events/rcu.h+29−0

@@ -623,6 +623,34 @@ TRACE_EVENT_RCU(rcu_invoke_kfree_callback, · 622 unmodified lines
623	623	__entry->rcuname, __entry->rhp, __entry->offset)
624	624	);
625	625
	626	+/*
	627	+ * Tracepoint for the invocation of a single RCU callback of the special
	628	+ * kfree_bulk() form. The first argument is the RCU flavor, the second
	629	+ * argument is a number of elements in array to free, the third is an
	630	+ * address of the array holding nr_records entries.
	631	+ */
	632	+TRACE_EVENT_RCU(rcu_invoke_kfree_bulk_callback,
	633	+
	634	+ TP_PROTO(const char rcuname, unsigned long nr_records, void *p),
	635	+
	636	+ TP_ARGS(rcuname, nr_records, p),
	637	+
	638	+ TP_STRUCT__entry(
	639	+ __field(const char *, rcuname)
	640	+ __field(unsigned long, nr_records)
	641	+ __field(void **, p)
	642	+ ),
	643	+
	644	+ TP_fast_assign(
	645	+ __entry->rcuname = rcuname;
	646	+ __entry->nr_records = nr_records;
	647	+ __entry->p = p;
	648	+ ),
	649	+
	650	+ TP_printk("%s bulk=0x%p nr_records=%lu",
	651	+ __entry->rcuname, __entry->p, __entry->nr_records)
	652	+);
	653	+
626	654	/*
627	655	* Tracepoint for exiting rcu_do_batch after RCU callbacks have been
628	656	* invoked. The first argument is the name of the RCU flavor,
@@ -712,6 +740,7 @@ TRACE_EVENT_RCU(rcu_torture_read, · 83 unmodified lines
712	740	* "Begin": rcu_barrier() started.
713	741	* "EarlyExit": rcu_barrier() piggybacked, thus early exit.
714	742	* "Inc1": rcu_barrier() piggyback check counter incremented.
	743	+ * "OfflineNoCBQ": rcu_barrier() found offline no-CBs CPU with callbacks.
715	744	* "OnlineQ": rcu_barrier() found online CPU with callbacks.
716	745	* "OnlineNQ": rcu_barrier() found online CPU, no callbacks.
717	746	* "IRQ": An rcu_barrier_callback() callback posted on remote CPU.

kernel/locking/locktorture.c+8−7

@@ -618,7 +618,7 @@ static struct lock_torture_ops percpu_rwsem_lock_ops = { · 617 unmodified lines
618	618	static int lock_torture_writer(void *arg)
619	619	{
620	620	struct lock_stress_stats *lwsp = arg;
621		− static DEFINE_TORTURE_RANDOM(rand);
	621	+ DEFINE_TORTURE_RANDOM(rand);
622	622
623	623	VERBOSE_TOROUT_STRING("lock_torture_writer task started");
624	624	set_user_nice(current, MAX_NICE);
@@ -655,7 +655,7 @@ static int lock_torture_writer(void *arg) · 30 unmodified lines
655	655	static int lock_torture_reader(void *arg)
656	656	{
657	657	struct lock_stress_stats *lrsp = arg;
658		− static DEFINE_TORTURE_RANDOM(rand);
	658	+ DEFINE_TORTURE_RANDOM(rand);
659	659
660	660	VERBOSE_TOROUT_STRING("lock_torture_reader task started");
661	661	set_user_nice(current, MAX_NICE);
@@ -696,15 +696,16 @@ static void __torture_print_stats(char *page, · 34 unmodified lines
696	696	if (statp[i].n_lock_fail)
697	697	fail = true;
698	698	sum += statp[i].n_lock_acquired;
699		− if (max < statp[i].n_lock_fail)
700		− max = statp[i].n_lock_fail;
701		− if (min > statp[i].n_lock_fail)
702		− min = statp[i].n_lock_fail;
	699	+ if (max < statp[i].n_lock_acquired)
	700	+ max = statp[i].n_lock_acquired;
	701	+ if (min > statp[i].n_lock_acquired)
	702	+ min = statp[i].n_lock_acquired;
703	703	}
704	704	page += sprintf(page,
705	705	"%s: Total: %lld Max/Min: %ld/%ld %s Fail: %d %s\n",
706	706	write ? "Writes" : "Reads ",
707		− sum, max, min, max / 2 > min ? "???" : "",
	707	+ sum, max, min,
	708	+ !onoff_interval && max / 2 > min ? "???" : "",
708	709	fail, fail ? "!!!" : "");
709	710	if (fail)
710	711	atomic_inc(&cxt.n_lock_torture_errors);

kernel/locking/rtmutex.c+1−1

@@ -57,7 +57,7 @@ rt_mutex_set_owner(struct rt_mutex lock, struct task_struct owner) · 56 unmodified lines
57	57	if (rt_mutex_has_waiters(lock))
58	58	val \|= RT_MUTEX_HAS_WAITERS;
59	59
60		− lock->owner = (struct task_struct *)val;
	60	+ WRITE_ONCE(lock->owner, (struct task_struct *)val);
61	61	}
62	62
63	63	static inline void clear_rt_mutex_waiters(struct rt_mutex *lock)

kernel/rcu/Makefile+4−0

@@ -3,6 +3,10 @@ · 2 unmodified lines
3	3	# and is generally not a function of system call inputs.
4	4	KCOV_INSTRUMENT := n
5	5
	6	+ifeq ($(CONFIG_KCSAN),y)
	7	+KBUILD_CFLAGS += -g -fno-omit-frame-pointer
	8	+endif
	9	+
6	10	obj-y += update.o sync.o
7	11	obj-$(CONFIG_TREE_SRCU) += srcutree.o
8	12	obj-$(CONFIG_TINY_SRCU) += srcutiny.o

kernel/rcu/rcu.h+21−2

@@ -198,6 +198,13 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head) · 197 unmodified lines
198	198	}
199	199	#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
200	200
	201	+extern int rcu_cpu_stall_suppress_at_boot;
	202	+
	203	+static inline bool rcu_stall_is_suppressed_at_boot(void)
	204	+{
	205	+ return rcu_cpu_stall_suppress_at_boot && !rcu_inkernel_boot_has_ended();
	206	+}
	207	+
201	208	#ifdef CONFIG_RCU_STALL_COMMON
202	209
203	210	extern int rcu_cpu_stall_ftrace_dump;
@@ -205,6 +212,11 @@ extern int rcu_cpu_stall_suppress; · 1 unmodified line
205	212	extern int rcu_cpu_stall_timeout;
206	213	int rcu_jiffies_till_stall_check(void);
207	214
	215	+static inline bool rcu_stall_is_suppressed(void)
	216	+{
	217	+ return rcu_stall_is_suppressed_at_boot() \|\| rcu_cpu_stall_suppress;
	218	+}
	219	+
208	220	#define rcu_ftrace_dump_stall_suppress() \
209	221	do { \
210	222	if (!rcu_cpu_stall_suppress) \
@@ -218,6 +230,11 @@ do { \ · 7 unmodified lines
218	230	} while (0)
219	231
220	232	#else /* #endif #ifdef CONFIG_RCU_STALL_COMMON */
	233	+
	234	+static inline bool rcu_stall_is_suppressed(void)
	235	+{
	236	+ return rcu_stall_is_suppressed_at_boot();
	237	+}
221	238	#define rcu_ftrace_dump_stall_suppress()
222	239	#define rcu_ftrace_dump_stall_unsuppress()
223	240	#endif /* #ifdef CONFIG_RCU_STALL_COMMON */
@@ -325,7 +342,8 @@ static inline void rcu_init_levelspread(int levelspread, const int levelcnt) · 101 unmodified lines
325	342	* Iterate over all possible CPUs in a leaf RCU node.
326	343	*/
327	344	#define for_each_leaf_node_possible_cpu(rnp, cpu) \
328		− for ((cpu) = cpumask_next((rnp)->grplo - 1, cpu_possible_mask); \
	345	+ for (WARN_ON_ONCE(!rcu_is_leaf_node(rnp)), \
	346	+ (cpu) = cpumask_next((rnp)->grplo - 1, cpu_possible_mask); \
329	347	(cpu) <= rnp->grphi; \
330	348	(cpu) = cpumask_next((cpu), cpu_possible_mask))
331	349
@@ -335,7 +353,8 @@ static inline void rcu_init_levelspread(int levelspread, const int levelcnt) · 3 unmodified lines
335	353	#define rcu_find_next_bit(rnp, cpu, mask) \
336	354	((rnp)->grplo + find_next_bit(&(mask), BITS_PER_LONG, (cpu)))
337	355	#define for_each_leaf_node_cpu_mask(rnp, cpu, mask) \
338		− for ((cpu) = rcu_find_next_bit((rnp), 0, (mask)); \
	356	+ for (WARN_ON_ONCE(!rcu_is_leaf_node(rnp)), \
	357	+ (cpu) = rcu_find_next_bit((rnp), 0, (mask)); \
339	358	(cpu) <= rnp->grphi; \
340	359	(cpu) = rcu_find_next_bit((rnp), (cpu) + 1 - (rnp->grplo), (mask)))
341	360

kernel/rcu/rcu_segcblist.c+1−3

@@ -182,7 +182,7 @@ void rcu_segcblist_offload(struct rcu_segcblist *rsclp) · 181 unmodified lines
182	182	bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp)
183	183	{
184	184	return rcu_segcblist_is_enabled(rsclp) &&
185		− &rsclp->head != rsclp->tails[RCU_DONE_TAIL];
	185	+ &rsclp->head != READ_ONCE(rsclp->tails[RCU_DONE_TAIL]);
186	186	}
187	187
188	188	/*
@@ -381,8 +381,6 @@ void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp, · 192 unmodified lines
381	381	return; /* Nothing to do. */
382	382	WRITE_ONCE(*rsclp->tails[RCU_NEXT_TAIL], rclp->head);
383	383	WRITE_ONCE(rsclp->tails[RCU_NEXT_TAIL], rclp->tail);
384		− rclp->head = NULL;
385		− rclp->tail = &rclp->head;
386	384	}
387	385
388	386	/*

kernel/rcu/rcuperf.c+12−2

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	12	#include <linux/types.h>
13	13	#include <linux/kernel.h>
14	14	#include <linux/init.h>
	15	+#include <linux/mm.h>
15	16	#include <linux/module.h>
16	17	#include <linux/kthread.h>
17	18	#include <linux/err.h>
@@ -611,6 +612,7 @@ kfree_perf_thread(void *arg) · 593 unmodified lines
611	612	long me = (long)arg;
612	613	struct kfree_obj *alloc_ptr;
613	614	u64 start_time, end_time;
	615	+ long long mem_begin, mem_during = 0;
614	616
615	617	VERBOSE_PERFOUT_STRING("kfree_perf_thread task started");
616	618	set_cpus_allowed_ptr(current, cpumask_of(me % nr_cpu_ids));
@@ -626,6 +628,12 @@ kfree_perf_thread(void *arg) · 9 unmodified lines
626	628	}
627	629
628	630	do {
	631	+ if (!mem_during) {
	632	+ mem_during = mem_begin = si_mem_available();
	633	+ } else if (loop % (kfree_loops / 4) == 0) {
	634	+ mem_during = (mem_during + si_mem_available()) / 2;
	635	+ }
	636	+
629	637	for (i = 0; i < kfree_alloc_num; i++) {
630	638	alloc_ptr = kmalloc(sizeof(struct kfree_obj), GFP_KERNEL);
631	639	if (!alloc_ptr)
@@ -645,9 +653,11 @@ kfree_perf_thread(void *arg) · 13 unmodified lines
645	653	else
646	654	b_rcu_gp_test_finished = cur_ops->get_gp_seq();
647	655
648		− pr_alert("Total time taken by all kfree'ers: %llu ns, loops: %d, batches: %ld\n",
	656	+ pr_alert("Total time taken by all kfree'ers: %llu ns, loops: %d, batches: %ld, memory footprint: %lldMB\n",
649	657	(unsigned long long)(end_time - start_time), kfree_loops,
650		− rcuperf_seq_diff(b_rcu_gp_test_finished, b_rcu_gp_test_started));
	658	+ rcuperf_seq_diff(b_rcu_gp_test_finished, b_rcu_gp_test_started),
	659	+ (mem_begin - mem_during) >> (20 - PAGE_SHIFT));
	660	+
651	661	if (shutdown) {
652	662	smp_mb(); /* Assign before wake. */
653	663	wake_up(&shutdown_wq);

kernel/rcu/rcutorture.c+50−17

@@ -339,7 +339,7 @@ rcu_read_delay(struct torture_random_state rrsp, struct rt_read_seg rtrsp) · 338 unmodified lines
339	339	* period, and we want a long delay occasionally to trigger
340	340	* force_quiescent_state. */
341	341
342		− if (!rcu_fwd_cb_nodelay &&
	342	+ if (!READ_ONCE(rcu_fwd_cb_nodelay) &&
343	343	!(torture_random(rrsp) % (nrealreaders * 2000 * longdelay_ms))) {
344	344	started = cur_ops->get_gp_seq();
345	345	ts = rcu_trace_clock_local();
@@ -375,11 +375,12 @@ rcu_torture_pipe_update_one(struct rcu_torture *rp) · 29 unmodified lines
375	375	{
376	376	int i;
377	377
378		− i = rp->rtort_pipe_count;
	378	+ i = READ_ONCE(rp->rtort_pipe_count);
379	379	if (i > RCU_TORTURE_PIPE_LEN)
380	380	i = RCU_TORTURE_PIPE_LEN;
381	381	atomic_inc(&rcu_torture_wcount[i]);
382		− if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
	382	+ WRITE_ONCE(rp->rtort_pipe_count, i + 1);
	383	+ if (rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
383	384	rp->rtort_mbtest = 0;
384	385	return true;
385	386	}
@@ -1015,7 +1016,8 @@ rcu_torture_writer(void *arg) · 629 unmodified lines
1015	1016	if (i > RCU_TORTURE_PIPE_LEN)
1016	1017	i = RCU_TORTURE_PIPE_LEN;
1017	1018	atomic_inc(&rcu_torture_wcount[i]);
1018		− old_rp->rtort_pipe_count++;
	1019	+ WRITE_ONCE(old_rp->rtort_pipe_count,
	1020	+ old_rp->rtort_pipe_count + 1);
1019	1021	switch (synctype[torture_random(&rand) % nsynctypes]) {
1020	1022	case RTWS_DEF_FREE:
1021	1023	rcu_torture_writer_state = RTWS_DEF_FREE;
@@ -1067,7 +1069,8 @@ rcu_torture_writer(void *arg) · 45 unmodified lines
1067	1069	if (stutter_wait("rcu_torture_writer") &&
1068	1070	!READ_ONCE(rcu_fwd_cb_nodelay) &&
1069	1071	!cur_ops->slow_gps &&
1070		− !torture_must_stop())
	1072	+ !torture_must_stop() &&
	1073	+ rcu_inkernel_boot_has_ended())
1071	1074	for (i = 0; i < ARRAY_SIZE(rcu_tortures); i++)
1072	1075	if (list_empty(&rcu_tortures[i].rtort_free) &&
1073	1076	rcu_access_pointer(rcu_torture_current) !=
@@ -1290,7 +1293,7 @@ static bool rcu_torture_one_read(struct torture_random_state *trsp) · 216 unmodified lines
1290	1293	atomic_inc(&n_rcu_torture_mberror);
1291	1294	rtrsp = rcutorture_loop_extend(&readstate, trsp, rtrsp);
1292	1295	preempt_disable();
1293		− pipe_count = p->rtort_pipe_count;
	1296	+ pipe_count = READ_ONCE(p->rtort_pipe_count);
1294	1297	if (pipe_count > RCU_TORTURE_PIPE_LEN) {
1295	1298	/* Should not happen, but... */
1296	1299	pipe_count = RCU_TORTURE_PIPE_LEN;
@@ -1404,14 +1407,15 @@ rcu_torture_stats_print(void) · 107 unmodified lines
1404	1407	int i;
1405	1408	long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
1406	1409	long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
	1410	+ struct rcu_torture *rtcp;
1407	1411	static unsigned long rtcv_snap = ULONG_MAX;
1408	1412	static bool splatted;
1409	1413	struct task_struct *wtp;
1410	1414
1411	1415	for_each_possible_cpu(cpu) {
1412	1416	for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
1413		− pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
1414		− batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
	1417	+ pipesummary[i] += READ_ONCE(per_cpu(rcu_torture_count, cpu)[i]);
	1418	+ batchsummary[i] += READ_ONCE(per_cpu(rcu_torture_batch, cpu)[i]);
1415	1419	}
1416	1420	}
1417	1421	for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
@@ -1420,9 +1424,10 @@ rcu_torture_stats_print(void) · 2 unmodified lines
1420	1424	}
1421	1425
1422	1426	pr_alert("%s%s ", torture_type, TORTURE_FLAG);
	1427	+ rtcp = rcu_access_pointer(rcu_torture_current);
1423	1428	pr_cont("rtc: %p %s: %lu tfle: %d rta: %d rtaf: %d rtf: %d ",
1424		− rcu_torture_current,
1425		− rcu_torture_current ? "ver" : "VER",
	1429	+ rtcp,
	1430	+ rtcp && !rcu_stall_is_suppressed_at_boot() ? "ver" : "VER",
1426	1431	rcu_torture_current_version,
1427	1432	list_empty(&rcu_torture_freelist),
1428	1433	atomic_read(&n_rcu_torture_alloc),
@@ -1478,7 +1483,8 @@ rcu_torture_stats_print(void) · 49 unmodified lines
1478	1483	if (cur_ops->stats)
1479	1484	cur_ops->stats();
1480	1485	if (rtcv_snap == rcu_torture_current_version &&
1481		− rcu_torture_current != NULL) {
	1486	+ rcu_access_pointer(rcu_torture_current) &&
	1487	+ !rcu_stall_is_suppressed()) {
1482	1488	int __maybe_unused flags = 0;
1483	1489	unsigned long __maybe_unused gp_seq = 0;
1484	1490
@@ -1993,8 +1999,11 @@ static int rcu_torture_fwd_prog(void *args) · 508 unmodified lines
1993	1999	schedule_timeout_interruptible(fwd_progress_holdoff * HZ);
1994	2000	WRITE_ONCE(rcu_fwd_emergency_stop, false);
1995	2001	register_oom_notifier(&rcutorture_oom_nb);
1996		− rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);
1997		− rcu_torture_fwd_prog_cr(rfp);
	2002	+ if (!IS_ENABLED(CONFIG_TINY_RCU) \|\|
	2003	+ rcu_inkernel_boot_has_ended())
	2004	+ rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries);
	2005	+ if (rcu_inkernel_boot_has_ended())
	2006	+ rcu_torture_fwd_prog_cr(rfp);
1998	2007	unregister_oom_notifier(&rcutorture_oom_nb);
1999	2008
2000	2009	/* Avoid slow periods, better to test when busy. */
@@ -2044,6 +2053,14 @@ static void rcu_torture_barrier_cbf(struct rcu_head *rcu) · 43 unmodified lines
2044	2053	atomic_inc(&barrier_cbs_invoked);
2045	2054	}
2046	2055
	2056	+/* IPI handler to get callback posted on desired CPU, if online. */
	2057	+static void rcu_torture_barrier1cb(void *rcu_void)
	2058	+{
	2059	+ struct rcu_head *rhp = rcu_void;
	2060	+
	2061	+ cur_ops->call(rhp, rcu_torture_barrier_cbf);
	2062	+}
	2063	+
2047	2064	/* kthread function to register callbacks used to test RCU barriers. */
2048	2065	static int rcu_torture_barrier_cbs(void *arg)
2049	2066	{
@@ -2067,9 +2084,11 @@ static int rcu_torture_barrier_cbs(void *arg) · 17 unmodified lines
2067	2084	* The above smp_load_acquire() ensures barrier_phase load
2068	2085	* is ordered before the following ->call().
2069	2086	*/
2070		− local_irq_disable(); /* Just to test no-irq call_rcu(). */
2071		− cur_ops->call(&rcu, rcu_torture_barrier_cbf);
2072		− local_irq_enable();
	2087	+ if (smp_call_function_single(myid, rcu_torture_barrier1cb,
	2088	+ &rcu, 1)) {
	2089	+ // IPI failed, so use direct call from current CPU.
	2090	+ cur_ops->call(&rcu, rcu_torture_barrier_cbf);
	2091	+ }
2073	2092	if (atomic_dec_and_test(&barrier_cbs_count))
2074	2093	wake_up(&barrier_wq);
2075	2094	} while (!torture_must_stop());
@@ -2105,7 +2124,21 @@ static int rcu_torture_barrier(void *arg) · 29 unmodified lines
2105	2124	pr_err("barrier_cbs_invoked = %d, n_barrier_cbs = %d\n",
2106	2125	atomic_read(&barrier_cbs_invoked),
2107	2126	n_barrier_cbs);
2108		− WARN_ON_ONCE(1);
	2127	+ WARN_ON(1);
	2128	+ // Wait manually for the remaining callbacks
	2129	+ i = 0;
	2130	+ do {
	2131	+ if (WARN_ON(i++ > HZ))
	2132	+ i = INT_MIN;
	2133	+ schedule_timeout_interruptible(1);
	2134	+ cur_ops->cb_barrier();
	2135	+ } while (atomic_read(&barrier_cbs_invoked) !=
	2136	+ n_barrier_cbs &&
	2137	+ !torture_must_stop());
	2138	+ smp_mb(); // Can't trust ordering if broken.
	2139	+ if (!torture_must_stop())
	2140	+ pr_err("Recovered: barrier_cbs_invoked = %d\n",
	2141	+ atomic_read(&barrier_cbs_invoked));
2109	2142	} else {
2110	2143	n_barrier_successes++;
2111	2144	}

kernel/rcu/srcutree.c+10−8

@@ -5,7 +5,7 @@ · 4 unmodified lines
5	5	* Copyright (C) IBM Corporation, 2006
6	6	* Copyright (C) Fujitsu, 2012
7	7	*
8		− * Author: Paul McKenney <paulmck@linux.ibm.com>
	8	+ * Authors: Paul McKenney <paulmck@linux.ibm.com>
9	9	* Lai Jiangshan <laijs@cn.fujitsu.com>
10	10	*
11	11	* For detailed explanation of Read-Copy Update mechanism see -
@@ -450,7 +450,7 @@ static void srcu_gp_start(struct srcu_struct *ssp) · 438 unmodified lines
450	450	spin_unlock_rcu_node(sdp); /* Interrupts remain disabled. */
451	451	smp_mb(); /* Order prior store to ->srcu_gp_seq_needed vs. GP start. */
452	452	rcu_seq_start(&ssp->srcu_gp_seq);
453		− state = rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq));
	453	+ state = rcu_seq_state(ssp->srcu_gp_seq);
454	454	WARN_ON_ONCE(state != SRCU_STATE_SCAN1);
455	455	}
456	456
@@ -534,7 +534,7 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 77 unmodified lines
534	534	rcu_seq_end(&ssp->srcu_gp_seq);
535	535	gpseq = rcu_seq_current(&ssp->srcu_gp_seq);
536	536	if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, gpseq))
537		− ssp->srcu_gp_seq_needed_exp = gpseq;
	537	+ WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, gpseq);
538	538	spin_unlock_irq_rcu_node(ssp);
539	539	mutex_unlock(&ssp->srcu_gp_mutex);
540	540	/* A new grace period can start at this point. But only one. */
@@ -550,7 +550,7 @@ static void srcu_gp_end(struct srcu_struct *ssp) · 9 unmodified lines
550	550	snp->srcu_have_cbs[idx] = gpseq;
551	551	rcu_seq_set_state(&snp->srcu_have_cbs[idx], 1);
552	552	if (ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, gpseq))
553		− snp->srcu_gp_seq_needed_exp = gpseq;
	553	+ WRITE_ONCE(snp->srcu_gp_seq_needed_exp, gpseq);
554	554	mask = snp->srcu_data_have_cbs[idx];
555	555	snp->srcu_data_have_cbs[idx] = 0;
556	556	spin_unlock_irq_rcu_node(snp);
@@ -614,7 +614,7 @@ static void srcu_funnel_exp_start(struct srcu_struct ssp, struct srcu_node snp · 57 unmodified lines
614	614	}
615	615	spin_lock_irqsave_rcu_node(ssp, flags);
616	616	if (ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))
617		− ssp->srcu_gp_seq_needed_exp = s;
	617	+ WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s);
618	618	spin_unlock_irqrestore_rcu_node(ssp, flags);
619	619	}
620	620
@@ -660,7 +660,7 @@ static void srcu_funnel_gp_start(struct srcu_struct ssp, struct srcu_data sdp, · 39 unmodified lines
660	660	if (snp == sdp->mynode)
661	661	snp->srcu_data_have_cbs[idx] \|= sdp->grpmask;
662	662	if (!do_norm && ULONG_CMP_LT(snp->srcu_gp_seq_needed_exp, s))
663		− snp->srcu_gp_seq_needed_exp = s;
	663	+ WRITE_ONCE(snp->srcu_gp_seq_needed_exp, s);
664	664	spin_unlock_irqrestore_rcu_node(snp, flags);
665	665	}
666	666
@@ -674,7 +674,7 @@ static void srcu_funnel_gp_start(struct srcu_struct ssp, struct srcu_data sdp, · 7 unmodified lines
674	674	smp_store_release(&ssp->srcu_gp_seq_needed, s); /^^^/
675	675	}
676	676	if (!do_norm && ULONG_CMP_LT(ssp->srcu_gp_seq_needed_exp, s))
677		− ssp->srcu_gp_seq_needed_exp = s;
	677	+ WRITE_ONCE(ssp->srcu_gp_seq_needed_exp, s);
678	678
679	679	/* If grace period not already done and none in progress, start it. */
680	680	if (!rcu_seq_done(&ssp->srcu_gp_seq, s) &&
@@ -1079,7 +1079,7 @@ EXPORT_SYMBOL_GPL(srcu_barrier); · 398 unmodified lines
1079	1079	*/
1080	1080	unsigned long srcu_batches_completed(struct srcu_struct *ssp)
1081	1081	{
1082		− return ssp->srcu_idx;
	1082	+ return READ_ONCE(ssp->srcu_idx);
1083	1083	}
1084	1084	EXPORT_SYMBOL_GPL(srcu_batches_completed);
1085	1085
@@ -1130,7 +1130,9 @@ static void srcu_advance_state(struct srcu_struct *ssp) · 44 unmodified lines
1130	1130	return; /* readers present, retry later. */
1131	1131	}
1132	1132	srcu_flip(ssp);
	1133	+ spin_lock_irq_rcu_node(ssp);
1133	1134	rcu_seq_set_state(&ssp->srcu_gp_seq, SRCU_STATE_SCAN2);
	1135	+ spin_unlock_irq_rcu_node(ssp);
1134	1136	}
1135	1137
1136	1138	if (rcu_seq_state(READ_ONCE(ssp->srcu_gp_seq)) == SRCU_STATE_SCAN2) {

kernel/rcu/tree.c+339−113

@@ -1,12 +1,12 @@
1	1	// SPDX-License-Identifier: GPL-2.0+
2	2	/*
3		− * Read-Copy Update mechanism for mutual exclusion
	3	+ * Read-Copy Update mechanism for mutual exclusion (tree-based version)
4	4	*
5	5	* Copyright IBM Corporation, 2008
6	6	*
7	7	* Authors: Dipankar Sarma <dipankar@in.ibm.com>
8	8	* Manfred Spraul <manfred@colorfullife.com>
9		− * Paul E. McKenney <paulmck@linux.ibm.com> Hierarchical version
	9	+ * Paul E. McKenney <paulmck@linux.ibm.com>
10	10	*
11	11	* Based on the original work by Paul McKenney <paulmck@linux.ibm.com>
12	12	* and inputs from Rusty Russell, Andrea Arcangeli and Andi Kleen.
@@ -150,6 +150,7 @@ static void rcu_boost_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu) · 137 unmodified lines
150	150	static void invoke_rcu_core(void);
151	151	static void rcu_report_exp_rdp(struct rcu_data *rdp);
152	152	static void sync_sched_exp_online_cleanup(int cpu);
	153	+static void check_cb_ovld_locked(struct rcu_data rdp, struct rcu_node rnp);
153	154
154	155	/* rcuc/rcub kthread realtime priority */
155	156	static int kthread_prio = IS_ENABLED(CONFIG_RCU_BOOST) ? 1 : 0;
@@ -342,14 +343,17 @@ bool rcu_eqs_special_set(int cpu) · 186 unmodified lines
342	343	{
343	344	int old;
344	345	int new;
	346	+ int new_old;
345	347	struct rcu_data *rdp = &per_cpu(rcu_data, cpu);
346	348
	349	+ new_old = atomic_read(&rdp->dynticks);
347	350	do {
348		− old = atomic_read(&rdp->dynticks);
	351	+ old = new_old;
349	352	if (old & RCU_DYNTICK_CTRL_CTR)
350	353	return false;
351	354	new = old \| RCU_DYNTICK_CTRL_MASK;
352		− } while (atomic_cmpxchg(&rdp->dynticks, old, new) != old);
	355	+ new_old = atomic_cmpxchg(&rdp->dynticks, old, new);
	356	+ } while (new_old != old);
353	357	return true;
354	358	}
355	359
@@ -410,10 +414,15 @@ static long blimit = DEFAULT_RCU_BLIMIT; · 54 unmodified lines
410	414	static long qhimark = DEFAULT_RCU_QHIMARK;
411	415	#define DEFAULT_RCU_QLOMARK 100 /* Once only this many pending, use blimit. */
412	416	static long qlowmark = DEFAULT_RCU_QLOMARK;
	417	+#define DEFAULT_RCU_QOVLD_MULT 2
	418	+#define DEFAULT_RCU_QOVLD (DEFAULT_RCU_QOVLD_MULT * DEFAULT_RCU_QHIMARK)
	419	+static long qovld = DEFAULT_RCU_QOVLD; /* If this many pending, hammer QS. */
	420	+static long qovld_calc = -1; /* No pre-initialization lock acquisitions! */
413	421
414	422	module_param(blimit, long, 0444);
415	423	module_param(qhimark, long, 0444);
416	424	module_param(qlowmark, long, 0444);
	425	+module_param(qovld, long, 0444);
417	426
418	427	static ulong jiffies_till_first_fqs = ULONG_MAX;
419	428	static ulong jiffies_till_next_fqs = ULONG_MAX;
@@ -818,11 +827,12 @@ static __always_inline void rcu_nmi_enter_common(bool irq) · 398 unmodified lines
818	827	incby = 1;
819	828	} else if (tick_nohz_full_cpu(rdp->cpu) &&
820	829	rdp->dynticks_nmi_nesting == DYNTICK_IRQ_NONIDLE &&
821		− READ_ONCE(rdp->rcu_urgent_qs) && !rdp->rcu_forced_tick) {
	830	+ READ_ONCE(rdp->rcu_urgent_qs) &&
	831	+ !READ_ONCE(rdp->rcu_forced_tick)) {
822	832	raw_spin_lock_rcu_node(rdp->mynode);
823	833	// Recheck under lock.
824	834	if (rdp->rcu_urgent_qs && !rdp->rcu_forced_tick) {
825		− rdp->rcu_forced_tick = true;
	835	+ WRITE_ONCE(rdp->rcu_forced_tick, true);
826	836	tick_dep_set_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
827	837	}
828	838	raw_spin_unlock_rcu_node(rdp->mynode);
@@ -899,7 +909,7 @@ static void rcu_disable_urgency_upon_qs(struct rcu_data *rdp) · 70 unmodified lines
899	909	WRITE_ONCE(rdp->rcu_need_heavy_qs, false);
900	910	if (tick_nohz_full_cpu(rdp->cpu) && rdp->rcu_forced_tick) {
901	911	tick_dep_clear_cpu(rdp->cpu, TICK_DEP_BIT_RCU);
902		− rdp->rcu_forced_tick = false;
	912	+ WRITE_ONCE(rdp->rcu_forced_tick, false);
903	913	}
904	914	}
905	915
@@ -1072,7 +1082,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) · 166 unmodified lines
1072	1082	rnhqp = &per_cpu(rcu_data.rcu_need_heavy_qs, rdp->cpu);
1073	1083	if (!READ_ONCE(*rnhqp) &&
1074	1084	(time_after(jiffies, rcu_state.gp_start + jtsq * 2) \|\|
1075		− time_after(jiffies, rcu_state.jiffies_resched))) {
	1085	+ time_after(jiffies, rcu_state.jiffies_resched) \|\|
	1086	+ rcu_state.cbovld)) {
1076	1087	WRITE_ONCE(*rnhqp, true);
1077	1088	/* Store rcu_need_heavy_qs before rcu_urgent_qs. */
1078	1089	smp_store_release(ruqp, true);
@@ -1089,8 +1100,8 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) · 10 unmodified lines
1089	1100	* So hit them over the head with the resched_cpu() hammer!
1090	1101	*/
1091	1102	if (tick_nohz_full_cpu(rdp->cpu) &&
1092		− time_after(jiffies,
1093		− READ_ONCE(rdp->last_fqs_resched) + jtsq * 3)) {
	1103	+ (time_after(jiffies, READ_ONCE(rdp->last_fqs_resched) + jtsq * 3) \|\|
	1104	+ rcu_state.cbovld)) {
1094	1105	WRITE_ONCE(*ruqp, true);
1095	1106	resched_cpu(rdp->cpu);
1096	1107	WRITE_ONCE(rdp->last_fqs_resched, jiffies);
@@ -1126,8 +1137,9 @@ static int rcu_implicit_dynticks_qs(struct rcu_data *rdp) · 29 unmodified lines
1126	1137	static void trace_rcu_this_gp(struct rcu_node rnp, struct rcu_data rdp,
1127	1138	unsigned long gp_seq_req, const char *s)
1128	1139	{
1129		− trace_rcu_future_grace_period(rcu_state.name, rnp->gp_seq, gp_seq_req,
1130		− rnp->level, rnp->grplo, rnp->grphi, s);
	1140	+ trace_rcu_future_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
	1141	+ gp_seq_req, rnp->level,
	1142	+ rnp->grplo, rnp->grphi, s);
1131	1143	}
1132	1144
1133	1145	/*
@@ -1174,7 +1186,7 @@ static bool rcu_start_this_gp(struct rcu_node rnp_start, struct rcu_data rdp, · 40 unmodified lines
1174	1186	TPS("Prestarted"));
1175	1187	goto unlock_out;
1176	1188	}
1177		− rnp->gp_seq_needed = gp_seq_req;
	1189	+ WRITE_ONCE(rnp->gp_seq_needed, gp_seq_req);
1178	1190	if (rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) {
1179	1191	/*
1180	1192	* We just marked the leaf or internal node, and a
@@ -1199,18 +1211,18 @@ static bool rcu_start_this_gp(struct rcu_node rnp_start, struct rcu_data rdp, · 18 unmodified lines
1199	1211	}
1200	1212	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("Startedroot"));
1201	1213	WRITE_ONCE(rcu_state.gp_flags, rcu_state.gp_flags \| RCU_GP_FLAG_INIT);
1202		− rcu_state.gp_req_activity = jiffies;
1203		− if (!rcu_state.gp_kthread) {
	1214	+ WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
	1215	+ if (!READ_ONCE(rcu_state.gp_kthread)) {
1204	1216	trace_rcu_this_gp(rnp, rdp, gp_seq_req, TPS("NoGPkthread"));
1205	1217	goto unlock_out;
1206	1218	}
1207		− trace_rcu_grace_period(rcu_state.name, READ_ONCE(rcu_state.gp_seq), TPS("newreq"));
	1219	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq, TPS("newreq"));
1208	1220	ret = true; /* Caller must wake GP kthread. */
1209	1221	unlock_out:
1210	1222	/* Push furthest requested GP to leaf node and rcu_data structure. */
1211	1223	if (ULONG_CMP_LT(gp_seq_req, rnp->gp_seq_needed)) {
1212		− rnp_start->gp_seq_needed = rnp->gp_seq_needed;
1213		− rdp->gp_seq_needed = rnp->gp_seq_needed;
	1224	+ WRITE_ONCE(rnp_start->gp_seq_needed, rnp->gp_seq_needed);
	1225	+ WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
1214	1226	}
1215	1227	if (rnp != rnp_start)
1216	1228	raw_spin_unlock_rcu_node(rnp);
@@ -1235,12 +1247,13 @@ static bool rcu_future_gp_cleanup(struct rcu_node *rnp) · 18 unmodified lines
1235	1247	}
1236	1248
1237	1249	/*
1238		− * Awaken the grace-period kthread. Don't do a self-awaken (unless in
1239		− * an interrupt or softirq handler), and don't bother awakening when there
1240		− * is nothing for the grace-period kthread to do (as in several CPUs raced
1241		− * to awaken, and we lost), and finally don't try to awaken a kthread that
1242		− * has not yet been created. If all those checks are passed, track some
1243		− * debug information and awaken.
	1250	+ * Awaken the grace-period kthread. Don't do a self-awaken (unless in an
	1251	+ * interrupt or softirq handler, in which case we just might immediately
	1252	+ * sleep upon return, resulting in a grace-period hang), and don't bother
	1253	+ * awakening when there is nothing for the grace-period kthread to do
	1254	+ * (as in several CPUs raced to awaken, we lost), and finally don't try
	1255	+ * to awaken a kthread that has not yet been created. If all those checks
	1256	+ * are passed, track some debug information and awaken.
1244	1257	*
1245	1258	* So why do the self-wakeup when in an interrupt or softirq handler
1246	1259	* in the grace-period kthread's context? Because the kthread might have
@@ -1250,10 +1263,10 @@ static bool rcu_future_gp_cleanup(struct rcu_node *rnp) · 3 unmodified lines
1250	1263	*/
1251	1264	static void rcu_gp_kthread_wake(void)
1252	1265	{
1253		− if ((current == rcu_state.gp_kthread &&
1254		− !in_irq() && !in_serving_softirq()) \|\|
1255		− !READ_ONCE(rcu_state.gp_flags) \|\|
1256		− !rcu_state.gp_kthread)
	1266	+ struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
	1267	+
	1268	+ if ((current == t && !in_irq() && !in_serving_softirq()) \|\|
	1269	+ !READ_ONCE(rcu_state.gp_flags) \|\| !t)
1257	1270	return;
1258	1271	WRITE_ONCE(rcu_state.gp_wake_time, jiffies);
1259	1272	WRITE_ONCE(rcu_state.gp_wake_seq, READ_ONCE(rcu_state.gp_seq));
@@ -1321,7 +1334,7 @@ static void rcu_accelerate_cbs_unlocked(struct rcu_node *rnp, · 61 unmodified lines
1321	1334
1322	1335	rcu_lockdep_assert_cblist_protected(rdp);
1323	1336	c = rcu_seq_snap(&rcu_state.gp_seq);
1324		− if (!rdp->gpwrap && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
	1337	+ if (!READ_ONCE(rdp->gpwrap) && ULONG_CMP_GE(rdp->gp_seq_needed, c)) {
1325	1338	/* Old request still live, so mark recent callbacks. */
1326	1339	(void)rcu_segcblist_accelerate(&rdp->cblist, c);
1327	1340	return;
@@ -1386,7 +1399,7 @@ static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp, · 58 unmodified lines
1386	1399	static bool __note_gp_changes(struct rcu_node rnp, struct rcu_data rdp)
1387	1400	{
1388	1401	bool ret = false;
1389		− bool need_gp;
	1402	+ bool need_qs;
1390	1403	const bool offloaded = IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
1391	1404	rcu_segcblist_is_offloaded(&rdp->cblist);
1392	1405
@@ -1400,10 +1413,13 @@ static bool __note_gp_changes(struct rcu_node rnp, struct rcu_data rdp) · 7 unmodified lines
1400	1413	unlikely(READ_ONCE(rdp->gpwrap))) {
1401	1414	if (!offloaded)
1402	1415	ret = rcu_advance_cbs(rnp, rdp); /* Advance CBs. */
	1416	+ rdp->core_needs_qs = false;
1403	1417	trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuend"));
1404	1418	} else {
1405	1419	if (!offloaded)
1406	1420	ret = rcu_accelerate_cbs(rnp, rdp); /* Recent CBs. */
	1421	+ if (rdp->core_needs_qs)
	1422	+ rdp->core_needs_qs = !!(rnp->qsmask & rdp->grpmask);
1407	1423	}
1408	1424
1409	1425	/* Now handle the beginnings of any new-to-this-CPU grace periods. */
@@ -1415,14 +1431,14 @@ static bool __note_gp_changes(struct rcu_node rnp, struct rcu_data rdp) · 5 unmodified lines
1415	1431	* go looking for one.
1416	1432	*/
1417	1433	trace_rcu_grace_period(rcu_state.name, rnp->gp_seq, TPS("cpustart"));
1418		− need_gp = !!(rnp->qsmask & rdp->grpmask);
1419		− rdp->cpu_no_qs.b.norm = need_gp;
1420		− rdp->core_needs_qs = need_gp;
	1434	+ need_qs = !!(rnp->qsmask & rdp->grpmask);
	1435	+ rdp->cpu_no_qs.b.norm = need_qs;
	1436	+ rdp->core_needs_qs = need_qs;
1421	1437	zero_cpu_stall_ticks(rdp);
1422	1438	}
1423	1439	rdp->gp_seq = rnp->gp_seq; /* Remember new grace-period state. */
1424	1440	if (ULONG_CMP_LT(rdp->gp_seq_needed, rnp->gp_seq_needed) \|\| rdp->gpwrap)
1425		− rdp->gp_seq_needed = rnp->gp_seq_needed;
	1441	+ WRITE_ONCE(rdp->gp_seq_needed, rnp->gp_seq_needed);
1426	1442	WRITE_ONCE(rdp->gpwrap, false);
1427	1443	rcu_gpnum_ovf(rnp, rdp);
1428	1444	return ret;
@@ -1651,8 +1667,7 @@ static void rcu_gp_fqs_loop(void) · 222 unmodified lines
1651	1667	WRITE_ONCE(rcu_state.jiffies_kick_kthreads,
1652	1668	jiffies + (j ? 3 * j : 2));
1653	1669	}
1654		− trace_rcu_grace_period(rcu_state.name,
1655		− READ_ONCE(rcu_state.gp_seq),
	1670	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1656	1671	TPS("fqswait"));
1657	1672	rcu_state.gp_state = RCU_GP_WAIT_FQS;
1658	1673	ret = swait_event_idle_timeout_exclusive(
@@ -1666,13 +1681,11 @@ static void rcu_gp_fqs_loop(void) · 7 unmodified lines
1666	1681	/* If time for quiescent-state forcing, do it. */
1667	1682	if (ULONG_CMP_GE(jiffies, rcu_state.jiffies_force_qs) \|\|
1668	1683	(gf & RCU_GP_FLAG_FQS)) {
1669		− trace_rcu_grace_period(rcu_state.name,
1670		− READ_ONCE(rcu_state.gp_seq),
	1684	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1671	1685	TPS("fqsstart"));
1672	1686	rcu_gp_fqs(first_gp_fqs);
1673	1687	first_gp_fqs = false;
1674		− trace_rcu_grace_period(rcu_state.name,
1675		− READ_ONCE(rcu_state.gp_seq),
	1688	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1676	1689	TPS("fqsend"));
1677	1690	cond_resched_tasks_rcu_qs();
1678	1691	WRITE_ONCE(rcu_state.gp_activity, jiffies);
@@ -1683,8 +1696,7 @@ static void rcu_gp_fqs_loop(void) · 4 unmodified lines
1683	1696	cond_resched_tasks_rcu_qs();
1684	1697	WRITE_ONCE(rcu_state.gp_activity, jiffies);
1685	1698	WARN_ON(signal_pending(current));
1686		− trace_rcu_grace_period(rcu_state.name,
1687		− READ_ONCE(rcu_state.gp_seq),
	1699	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1688	1700	TPS("fqswaitsig"));
1689	1701	ret = 1; /* Keep old FQS timing. */
1690	1702	j = jiffies;
@@ -1701,8 +1713,9 @@ static void rcu_gp_fqs_loop(void) · 10 unmodified lines
1701	1713	*/
1702	1714	static void rcu_gp_cleanup(void)
1703	1715	{
1704		− unsigned long gp_duration;
	1716	+ int cpu;
1705	1717	bool needgp = false;
	1718	+ unsigned long gp_duration;
1706	1719	unsigned long new_gp_seq;
1707	1720	bool offloaded;
1708	1721	struct rcu_data *rdp;
@@ -1748,6 +1761,12 @@ static void rcu_gp_cleanup(void) · 39 unmodified lines
1748	1761	needgp = __note_gp_changes(rnp, rdp) \|\| needgp;
1749	1762	/* smp_mb() provided by prior unlock-lock pair. */
1750	1763	needgp = rcu_future_gp_cleanup(rnp) \|\| needgp;
	1764	+ // Reset overload indication for CPUs no longer overloaded
	1765	+ if (rcu_is_leaf_node(rnp))
	1766	+ for_each_leaf_node_cpu_mask(rnp, cpu, rnp->cbovldmask) {
	1767	+ rdp = per_cpu_ptr(&rcu_data, cpu);
	1768	+ check_cb_ovld_locked(rdp, rnp);
	1769	+ }
1751	1770	sq = rcu_nocb_gp_get(rnp);
1752	1771	raw_spin_unlock_irq_rcu_node(rnp);
1753	1772	rcu_nocb_gp_cleanup(sq);
@@ -1774,9 +1793,9 @@ static void rcu_gp_cleanup(void) · 20 unmodified lines
1774	1793	rcu_segcblist_is_offloaded(&rdp->cblist);
1775	1794	if ((offloaded \|\| !rcu_accelerate_cbs(rnp, rdp)) && needgp) {
1776	1795	WRITE_ONCE(rcu_state.gp_flags, RCU_GP_FLAG_INIT);
1777		− rcu_state.gp_req_activity = jiffies;
	1796	+ WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
1778	1797	trace_rcu_grace_period(rcu_state.name,
1779		− READ_ONCE(rcu_state.gp_seq),
	1798	+ rcu_state.gp_seq,
1780	1799	TPS("newreq"));
1781	1800	} else {
1782	1801	WRITE_ONCE(rcu_state.gp_flags,
@@ -1795,8 +1814,7 @@ static int __noreturn rcu_gp_kthread(void *unused) · 12 unmodified lines
1795	1814
1796	1815	/* Handle grace-period start. */
1797	1816	for (;;) {
1798		− trace_rcu_grace_period(rcu_state.name,
1799		− READ_ONCE(rcu_state.gp_seq),
	1817	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1800	1818	TPS("reqwait"));
1801	1819	rcu_state.gp_state = RCU_GP_WAIT_GPS;
1802	1820	swait_event_idle_exclusive(rcu_state.gp_wq,
@@ -1809,8 +1827,7 @@ static int __noreturn rcu_gp_kthread(void *unused) · 6 unmodified lines
1809	1827	cond_resched_tasks_rcu_qs();
1810	1828	WRITE_ONCE(rcu_state.gp_activity, jiffies);
1811	1829	WARN_ON(signal_pending(current));
1812		− trace_rcu_grace_period(rcu_state.name,
1813		− READ_ONCE(rcu_state.gp_seq),
	1830	+ trace_rcu_grace_period(rcu_state.name, rcu_state.gp_seq,
1814	1831	TPS("reqwaitsig"));
1815	1832	}
1816	1833
@@ -1881,7 +1898,7 @@ static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp, · 64 unmodified lines
1881	1898	WARN_ON_ONCE(oldmask); /* Any child must be all zeroed! */
1882	1899	WARN_ON_ONCE(!rcu_is_leaf_node(rnp) &&
1883	1900	rcu_preempt_blocked_readers_cgp(rnp));
1884		− rnp->qsmask &= ~mask;
	1901	+ WRITE_ONCE(rnp->qsmask, rnp->qsmask & ~mask);
1885	1902	trace_rcu_quiescent_state_report(rcu_state.name, rnp->gp_seq,
1886	1903	mask, rnp->qsmask, rnp->level,
1887	1904	rnp->grplo, rnp->grphi,
@@ -1904,7 +1921,7 @@ static void rcu_report_qs_rnp(unsigned long mask, struct rcu_node *rnp, · 16 unmodified lines
1904	1921	rnp_c = rnp;
1905	1922	rnp = rnp->parent;
1906	1923	raw_spin_lock_irqsave_rcu_node(rnp, flags);
1907		− oldmask = rnp_c->qsmask;
	1924	+ oldmask = READ_ONCE(rnp_c->qsmask);
1908	1925	}
1909	1926
1910	1927	/*
@@ -1987,6 +2004,8 @@ rcu_report_qs_rdp(int cpu, struct rcu_data *rdp) · 76 unmodified lines
1987	2004	return;
1988	2005	}
1989	2006	mask = rdp->grpmask;
	2007	+ if (rdp->cpu == smp_processor_id())
	2008	+ rdp->core_needs_qs = false;
1990	2009	if ((rnp->qsmask & mask) == 0) {
1991	2010	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1992	2011	} else {
@@ -2052,7 +2071,7 @@ int rcutree_dying_cpu(unsigned int cpu) · 59 unmodified lines
2052	2071	return 0;
2053	2072
2054	2073	blkd = !!(rnp->qsmask & rdp->grpmask);
2055		− trace_rcu_grace_period(rcu_state.name, rnp->gp_seq,
	2074	+ trace_rcu_grace_period(rcu_state.name, READ_ONCE(rnp->gp_seq),
2056	2075	blkd ? TPS("cpuofl") : TPS("cpuofl-bgp"));
2057	2076	return 0;
2058	2077	}
@@ -2294,10 +2313,13 @@ static void force_qs_rnp(int (f)(struct rcu_data rdp)) · 235 unmodified lines
2294	2313	struct rcu_data *rdp;
2295	2314	struct rcu_node *rnp;
2296	2315
	2316	+ rcu_state.cbovld = rcu_state.cbovldnext;
	2317	+ rcu_state.cbovldnext = false;
2297	2318	rcu_for_each_leaf_node(rnp) {
2298	2319	cond_resched_tasks_rcu_qs();
2299	2320	mask = 0;
2300	2321	raw_spin_lock_irqsave_rcu_node(rnp, flags);
	2322	+ rcu_state.cbovldnext \|= !!rnp->cbovldmask;
2301	2323	if (rnp->qsmask == 0) {
2302	2324	if (!IS_ENABLED(CONFIG_PREEMPT_RCU) \|\|
2303	2325	rcu_preempt_blocked_readers_cgp(rnp)) {
@@ -2579,11 +2601,48 @@ static void rcu_leak_callback(struct rcu_head *rhp) · 275 unmodified lines
2579	2601	}
2580	2602
2581	2603	/*
2582		− * Helper function for call_rcu() and friends. The cpu argument will
2583		− * normally be -1, indicating "currently running CPU". It may specify
2584		− * a CPU only if that CPU is a no-CBs CPU. Currently, only rcu_barrier()
2585		− * is expected to specify a CPU.
	2604	+ * Check and if necessary update the leaf rcu_node structure's
	2605	+ * ->cbovldmask bit corresponding to the current CPU based on that CPU's
	2606	+ * number of queued RCU callbacks. The caller must hold the leaf rcu_node
	2607	+ * structure's ->lock.
2586	2608	*/
	2609	+static void check_cb_ovld_locked(struct rcu_data rdp, struct rcu_node rnp)
	2610	+{
	2611	+ raw_lockdep_assert_held_rcu_node(rnp);
	2612	+ if (qovld_calc <= 0)
	2613	+ return; // Early boot and wildcard value set.
	2614	+ if (rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc)
	2615	+ WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask \| rdp->grpmask);
	2616	+ else
	2617	+ WRITE_ONCE(rnp->cbovldmask, rnp->cbovldmask & ~rdp->grpmask);
	2618	+}
	2619	+
	2620	+/*
	2621	+ * Check and if necessary update the leaf rcu_node structure's
	2622	+ * ->cbovldmask bit corresponding to the current CPU based on that CPU's
	2623	+ * number of queued RCU callbacks. No locks need be held, but the
	2624	+ * caller must have disabled interrupts.
	2625	+ *
	2626	+ * Note that this function ignores the possibility that there are a lot
	2627	+ * of callbacks all of which have already seen the end of their respective
	2628	+ * grace periods. This omission is due to the need for no-CBs CPUs to
	2629	+ * be holding ->nocb_lock to do this check, which is too heavy for a
	2630	+ * common-case operation.
	2631	+ */
	2632	+static void check_cb_ovld(struct rcu_data *rdp)
	2633	+{
	2634	+ struct rcu_node *const rnp = rdp->mynode;
	2635	+
	2636	+ if (qovld_calc <= 0 \|\|
	2637	+ ((rcu_segcblist_n_cbs(&rdp->cblist) >= qovld_calc) ==
	2638	+ !!(READ_ONCE(rnp->cbovldmask) & rdp->grpmask)))
	2639	+ return; // Early boot wildcard value or already set correctly.
	2640	+ raw_spin_lock_rcu_node(rnp);
	2641	+ check_cb_ovld_locked(rdp, rnp);
	2642	+ raw_spin_unlock_rcu_node(rnp);
	2643	+}
	2644	+
	2645	+/* Helper function for call_rcu() and friends. */
2587	2646	static void
2588	2647	__call_rcu(struct rcu_head *head, rcu_callback_t func)
2589	2648	{
@@ -2621,9 +2680,10 @@ __call_rcu(struct rcu_head *head, rcu_callback_t func) · 31 unmodified lines
2621	2680	rcu_segcblist_init(&rdp->cblist);
2622	2681	}
2623	2682
	2683	+ check_cb_ovld(rdp);
2624	2684	if (rcu_nocb_try_bypass(rdp, head, &was_alldone, flags))
2625	2685	return; // Enqueued onto ->nocb_bypass, so just leave.
2626		− /* If we get here, rcu_nocb_try_bypass() acquired ->nocb_lock. */
	2686	+ // If no-CBs CPU gets here, rcu_nocb_try_bypass() acquired ->nocb_lock.
2627	2687	rcu_segcblist_enqueue(&rdp->cblist, head);
2628	2688	if (__is_kfree_rcu_offset((unsigned long)func))
2629	2689	trace_rcu_kfree_callback(rcu_state.name, head,
@@ -2689,22 +2749,47 @@ EXPORT_SYMBOL_GPL(call_rcu); · 59 unmodified lines
2689	2749	#define KFREE_DRAIN_JIFFIES (HZ / 50)
2690	2750	#define KFREE_N_BATCHES 2
2691	2751
	2752	+/*
	2753	+ * This macro defines how many entries the "records" array
	2754	+ * will contain. It is based on the fact that the size of
	2755	+ * kfree_rcu_bulk_data structure becomes exactly one page.
	2756	+ */
	2757	+#define KFREE_BULK_MAX_ENTR ((PAGE_SIZE / sizeof(void *)) - 3)
	2758	+
	2759	+/**
	2760	+ * struct kfree_rcu_bulk_data - single block to store kfree_rcu() pointers
	2761	+ * @nr_records: Number of active pointers in the array
	2762	+ * @records: Array of the kfree_rcu() pointers
	2763	+ * @next: Next bulk object in the block chain
	2764	+ * @head_free_debug: For debug, when CONFIG_DEBUG_OBJECTS_RCU_HEAD is set
	2765	+ */
	2766	+struct kfree_rcu_bulk_data {
	2767	+ unsigned long nr_records;
	2768	+ void *records[KFREE_BULK_MAX_ENTR];
	2769	+ struct kfree_rcu_bulk_data *next;
	2770	+ struct rcu_head *head_free_debug;
	2771	+};
	2772	+
2692	2773	/**
2693	2774	* struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests
2694	2775	* @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period
2695	2776	* @head_free: List of kfree_rcu() objects waiting for a grace period
	2777	+ * @bhead_free: Bulk-List of kfree_rcu() objects waiting for a grace period
2696	2778	* @krcp: Pointer to @kfree_rcu_cpu structure
2697	2779	*/
2698	2780
2699	2781	struct kfree_rcu_cpu_work {
2700	2782	struct rcu_work rcu_work;
2701	2783	struct rcu_head *head_free;
	2784	+ struct kfree_rcu_bulk_data *bhead_free;
2702	2785	struct kfree_rcu_cpu *krcp;
2703	2786	};
2704	2787
2705	2788	/**
2706	2789	* struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period
2707	2790	* @head: List of kfree_rcu() objects not yet waiting for a grace period
	2791	+ * @bhead: Bulk-List of kfree_rcu() objects not yet waiting for a grace period
	2792	+ * @bcached: Keeps at most one object for later reuse when build chain blocks
2708	2793	* @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period
2709	2794	* @lock: Synchronize access to this structure
2710	2795	* @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES
@@ -2718,6 +2803,8 @@ struct kfree_rcu_cpu_work { · 7 unmodified lines
2718	2803	*/
2719	2804	struct kfree_rcu_cpu {
2720	2805	struct rcu_head *head;
	2806	+ struct kfree_rcu_bulk_data *bhead;
	2807	+ struct kfree_rcu_bulk_data *bcached;
2721	2808	struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES];
2722	2809	spinlock_t lock;
2723	2810	struct delayed_work monitor_work;
@@ -2727,14 +2814,24 @@ struct kfree_rcu_cpu { · 3 unmodified lines
2727	2814
2728	2815	static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc);
2729	2816
	2817	+static __always_inline void
	2818	+debug_rcu_head_unqueue_bulk(struct rcu_head *head)
	2819	+{
	2820	+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
	2821	+ for (; head; head = head->next)
	2822	+ debug_rcu_head_unqueue(head);
	2823	+#endif
	2824	+}
	2825	+
2730	2826	/*
2731	2827	* This function is invoked in workqueue context after a grace period.
2732		− * It frees all the objects queued on ->head_free.
	2828	+ * It frees all the objects queued on ->bhead_free or ->head_free.
2733	2829	*/
2734	2830	static void kfree_rcu_work(struct work_struct *work)
2735	2831	{
2736	2832	unsigned long flags;
2737	2833	struct rcu_head head, next;
	2834	+ struct kfree_rcu_bulk_data bhead, bnext;
2738	2835	struct kfree_rcu_cpu *krcp;
2739	2836	struct kfree_rcu_cpu_work *krwp;
2740	2837
@@ -2744,22 +2841,44 @@ static void kfree_rcu_work(struct work_struct *work) · 3 unmodified lines
2744	2841	spin_lock_irqsave(&krcp->lock, flags);
2745	2842	head = krwp->head_free;
2746	2843	krwp->head_free = NULL;
	2844	+ bhead = krwp->bhead_free;
	2845	+ krwp->bhead_free = NULL;
2747	2846	spin_unlock_irqrestore(&krcp->lock, flags);
2748	2847
2749		− // List "head" is now private, so traverse locklessly.
	2848	+ /* "bhead" is now private, so traverse locklessly. */
	2849	+ for (; bhead; bhead = bnext) {
	2850	+ bnext = bhead->next;
	2851	+
	2852	+ debug_rcu_head_unqueue_bulk(bhead->head_free_debug);
	2853	+
	2854	+ rcu_lock_acquire(&rcu_callback_map);
	2855	+ trace_rcu_invoke_kfree_bulk_callback(rcu_state.name,
	2856	+ bhead->nr_records, bhead->records);
	2857	+
	2858	+ kfree_bulk(bhead->nr_records, bhead->records);
	2859	+ rcu_lock_release(&rcu_callback_map);
	2860	+
	2861	+ if (cmpxchg(&krcp->bcached, NULL, bhead))
	2862	+ free_page((unsigned long) bhead);
	2863	+
	2864	+ cond_resched_tasks_rcu_qs();
	2865	+ }
	2866	+
	2867	+ /*
	2868	+ * Emergency case only. It can happen under low memory
	2869	+ * condition when an allocation gets failed, so the "bulk"
	2870	+ * path can not be temporary maintained.
	2871	+ */
2750	2872	for (; head; head = next) {
2751	2873	unsigned long offset = (unsigned long)head->func;
2752	2874
2753	2875	next = head->next;
2754		− // Potentially optimize with kfree_bulk in future.
2755	2876	debug_rcu_head_unqueue(head);
2756	2877	rcu_lock_acquire(&rcu_callback_map);
2757	2878	trace_rcu_invoke_kfree_callback(rcu_state.name, head, offset);
2758	2879
2759		− if (!WARN_ON_ONCE(!__is_kfree_rcu_offset(offset))) {
2760		− /* Could be optimized with kfree_bulk() in future. */
	2880	+ if (!WARN_ON_ONCE(!__is_kfree_rcu_offset(offset)))
2761	2881	kfree((void *)head - offset);
2762		− }
2763	2882
2764	2883	rcu_lock_release(&rcu_callback_map);
2765	2884	cond_resched_tasks_rcu_qs();
@@ -2774,26 +2893,48 @@ static void kfree_rcu_work(struct work_struct *work) · 8 unmodified lines
2774	2893	*/
2775	2894	static inline bool queue_kfree_rcu_work(struct kfree_rcu_cpu *krcp)
2776	2895	{
	2896	+ struct kfree_rcu_cpu_work *krwp;
	2897	+ bool queued = false;
2777	2898	int i;
2778		− struct kfree_rcu_cpu_work *krwp = NULL;
2779	2899
2780	2900	lockdep_assert_held(&krcp->lock);
2781		− for (i = 0; i < KFREE_N_BATCHES; i++)
2782		− if (!krcp->krw_arr[i].head_free) {
2783		− krwp = &(krcp->krw_arr[i]);
2784		− break;
2785		− }
2786	2901
2787		− // If a previous RCU batch is in progress, we cannot immediately
2788		− // queue another one, so return false to tell caller to retry.
2789		− if (!krwp)
2790		− return false;
	2902	+ for (i = 0; i < KFREE_N_BATCHES; i++) {
	2903	+ krwp = &(krcp->krw_arr[i]);
2791	2904
2792		− krwp->head_free = krcp->head;
2793		− krcp->head = NULL;
2794		− INIT_RCU_WORK(&krwp->rcu_work, kfree_rcu_work);
2795		− queue_rcu_work(system_wq, &krwp->rcu_work);
2796		− return true;
	2905	+ /*
	2906	+ * Try to detach bhead or head and attach it over any
	2907	+ * available corresponding free channel. It can be that
	2908	+ * a previous RCU batch is in progress, it means that
	2909	+ * immediately to queue another one is not possible so
	2910	+ * return false to tell caller to retry.
	2911	+ */
	2912	+ if ((krcp->bhead && !krwp->bhead_free) \|\|
	2913	+ (krcp->head && !krwp->head_free)) {
	2914	+ /* Channel 1. */
	2915	+ if (!krwp->bhead_free) {
	2916	+ krwp->bhead_free = krcp->bhead;
	2917	+ krcp->bhead = NULL;
	2918	+ }
	2919	+
	2920	+ /* Channel 2. */
	2921	+ if (!krwp->head_free) {
	2922	+ krwp->head_free = krcp->head;
	2923	+ krcp->head = NULL;
	2924	+ }
	2925	+
	2926	+ /*
	2927	+ * One work is per one batch, so there are two "free channels",
	2928	+ * "bhead_free" and "head_free" the batch can handle. It can be
	2929	+ * that the work is in the pending state when two channels have
	2930	+ * been detached following each other, one by one.
	2931	+ */
	2932	+ queue_rcu_work(system_wq, &krwp->rcu_work);
	2933	+ queued = true;
	2934	+ }
	2935	+ }
	2936	+
	2937	+ return queued;
2797	2938	}
2798	2939
2799	2940	static inline void kfree_rcu_drain_unlock(struct kfree_rcu_cpu *krcp,
@@ -2830,19 +2971,65 @@ static void kfree_rcu_monitor(struct work_struct *work) · 30 unmodified lines
2830	2971	spin_unlock_irqrestore(&krcp->lock, flags);
2831	2972	}
2832	2973
	2974	+static inline bool
	2975	+kfree_call_rcu_add_ptr_to_bulk(struct kfree_rcu_cpu *krcp,
	2976	+ struct rcu_head *head, rcu_callback_t func)
	2977	+{
	2978	+ struct kfree_rcu_bulk_data *bnode;
	2979	+
	2980	+ if (unlikely(!krcp->initialized))
	2981	+ return false;
	2982	+
	2983	+ lockdep_assert_held(&krcp->lock);
	2984	+
	2985	+ /* Check if a new block is required. */
	2986	+ if (!krcp->bhead \|\|
	2987	+ krcp->bhead->nr_records == KFREE_BULK_MAX_ENTR) {
	2988	+ bnode = xchg(&krcp->bcached, NULL);
	2989	+ if (!bnode) {
	2990	+ WARN_ON_ONCE(sizeof(struct kfree_rcu_bulk_data) > PAGE_SIZE);
	2991	+
	2992	+ bnode = (struct kfree_rcu_bulk_data *)
	2993	+ __get_free_page(GFP_NOWAIT \| __GFP_NOWARN);
	2994	+ }
	2995	+
	2996	+ /* Switch to emergency path. */
	2997	+ if (unlikely(!bnode))
	2998	+ return false;
	2999	+
	3000	+ /* Initialize the new block. */
	3001	+ bnode->nr_records = 0;
	3002	+ bnode->next = krcp->bhead;
	3003	+ bnode->head_free_debug = NULL;
	3004	+
	3005	+ /* Attach it to the head. */
	3006	+ krcp->bhead = bnode;
	3007	+ }
	3008	+
	3009	+#ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
	3010	+ head->func = func;
	3011	+ head->next = krcp->bhead->head_free_debug;
	3012	+ krcp->bhead->head_free_debug = head;
	3013	+#endif
	3014	+
	3015	+ /* Finally insert. */
	3016	+ krcp->bhead->records[krcp->bhead->nr_records++] =
	3017	+ (void *) head - (unsigned long) func;
	3018	+
	3019	+ return true;
	3020	+}
	3021	+
2833	3022	/*
2834		− * Queue a request for lazy invocation of kfree() after a grace period.
	3023	+ * Queue a request for lazy invocation of kfree_bulk()/kfree() after a grace
	3024	+ * period. Please note there are two paths are maintained, one is the main one
	3025	+ * that uses kfree_bulk() interface and second one is emergency one, that is
	3026	+ * used only when the main path can not be maintained temporary, due to memory
	3027	+ * pressure.
2835	3028	*
2836	3029	* Each kfree_call_rcu() request is added to a batch. The batch will be drained
2837		− * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch
2838		− * will be kfree'd in workqueue context. This allows us to:
2839		− *
2840		− * 1. Batch requests together to reduce the number of grace periods during
2841		− * heavy kfree_rcu() load.
2842		− *
2843		− * 2. It makes it possible to use kfree_bulk() on a large number of
2844		− * kfree_rcu() requests thus reducing cache misses and the per-object
2845		− * overhead of kfree().
	3030	+ * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch will
	3031	+ * be free'd in workqueue context. This allows us to: batch requests together to
	3032	+ * reduce the number of grace periods during heavy kfree_rcu() load.
2846	3033	*/
2847	3034	void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func)
2848	3035	{
@@ -2861,9 +3048,16 @@ void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func) · 12 unmodified lines
2861	3048	__func__, head);
2862	3049	goto unlock_return;
2863	3050	}
2864		− head->func = func;
2865		− head->next = krcp->head;
2866		− krcp->head = head;
	3051	+
	3052	+ /*
	3053	+ * Under high memory pressure GFP_NOWAIT can fail,
	3054	+ * in that case the emergency path is maintained.
	3055	+ */
	3056	+ if (unlikely(!kfree_call_rcu_add_ptr_to_bulk(krcp, head, func))) {
	3057	+ head->func = func;
	3058	+ head->next = krcp->head;
	3059	+ krcp->head = head;
	3060	+ }
2867	3061
2868	3062	// Set timer to drain after KFREE_DRAIN_JIFFIES.
2869	3063	if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
@@ -3075,24 +3269,32 @@ static void rcu_barrier_trace(const char *s, int cpu, unsigned long done) · 205 unmodified lines
3075	3269	/*
3076	3270	* RCU callback function for rcu_barrier(). If we are last, wake
3077	3271	* up the task executing rcu_barrier().
	3272	+ *
	3273	+ * Note that the value of rcu_state.barrier_sequence must be captured
	3274	+ * before the atomic_dec_and_test(). Otherwise, if this CPU is not last,
	3275	+ * other CPUs might count the value down to zero before this CPU gets
	3276	+ * around to invoking rcu_barrier_trace(), which might result in bogus
	3277	+ * data from the next instance of rcu_barrier().
3078	3278	*/
3079	3279	static void rcu_barrier_callback(struct rcu_head *rhp)
3080	3280	{
	3281	+ unsigned long __maybe_unused s = rcu_state.barrier_sequence;
	3282	+
3081	3283	if (atomic_dec_and_test(&rcu_state.barrier_cpu_count)) {
3082		− rcu_barrier_trace(TPS("LastCB"), -1,
3083		− rcu_state.barrier_sequence);
	3284	+ rcu_barrier_trace(TPS("LastCB"), -1, s);
3084	3285	complete(&rcu_state.barrier_completion);
3085	3286	} else {
3086		− rcu_barrier_trace(TPS("CB"), -1, rcu_state.barrier_sequence);
	3287	+ rcu_barrier_trace(TPS("CB"), -1, s);
3087	3288	}
3088	3289	}
3089	3290
3090	3291	/*
3091	3292	* Called with preemption disabled, and from cross-cpu IRQ context.
3092	3293	*/
3093		−static void rcu_barrier_func(void *unused)
	3294	+static void rcu_barrier_func(void *cpu_in)
3094	3295	{
3095		− struct rcu_data *rdp = raw_cpu_ptr(&rcu_data);
	3296	+ uintptr_t cpu = (uintptr_t)cpu_in;
	3297	+ struct rcu_data *rdp = per_cpu_ptr(&rcu_data, cpu);
3096	3298
3097	3299	rcu_barrier_trace(TPS("IRQ"), -1, rcu_state.barrier_sequence);
3098	3300	rdp->barrier_head.func = rcu_barrier_callback;
@@ -3119,7 +3321,7 @@ static void rcu_barrier_func(void *unused) · 20 unmodified lines
3119	3321	*/
3120	3322	void rcu_barrier(void)
3121	3323	{
3122		− int cpu;
	3324	+ uintptr_t cpu;
3123	3325	struct rcu_data *rdp;
3124	3326	unsigned long s = rcu_seq_snap(&rcu_state.barrier_sequence);
3125	3327
@@ -3142,13 +3344,14 @@ void rcu_barrier(void) · 16 unmodified lines
3142	3344	rcu_barrier_trace(TPS("Inc1"), -1, rcu_state.barrier_sequence);
3143	3345
3144	3346	/*
3145		− * Initialize the count to one rather than to zero in order to
3146		− * avoid a too-soon return to zero in case of a short grace period
3147		− * (or preemption of this task). Exclude CPU-hotplug operations
3148		− * to ensure that no offline CPU has callbacks queued.
	3347	+ * Initialize the count to two rather than to zero in order
	3348	+ * to avoid a too-soon return to zero in case of an immediate
	3349	+ * invocation of the just-enqueued callback (or preemption of
	3350	+ * this task). Exclude CPU-hotplug operations to ensure that no
	3351	+ * offline non-offloaded CPU has callbacks queued.
3149	3352	*/
3150	3353	init_completion(&rcu_state.barrier_completion);
3151		− atomic_set(&rcu_state.barrier_cpu_count, 1);
	3354	+ atomic_set(&rcu_state.barrier_cpu_count, 2);
3152	3355	get_online_cpus();
3153	3356
3154	3357	/*
@@ -3158,13 +3361,23 @@ void rcu_barrier(void) · 3 unmodified lines
3158	3361	*/
3159	3362	for_each_possible_cpu(cpu) {
3160	3363	rdp = per_cpu_ptr(&rcu_data, cpu);
3161		− if (!cpu_online(cpu) &&
	3364	+ if (cpu_is_offline(cpu) &&
3162	3365	!rcu_segcblist_is_offloaded(&rdp->cblist))
3163	3366	continue;
3164		− if (rcu_segcblist_n_cbs(&rdp->cblist)) {
	3367	+ if (rcu_segcblist_n_cbs(&rdp->cblist) && cpu_online(cpu)) {
3165	3368	rcu_barrier_trace(TPS("OnlineQ"), cpu,
3166	3369	rcu_state.barrier_sequence);
3167		− smp_call_function_single(cpu, rcu_barrier_func, NULL, 1);
	3370	+ smp_call_function_single(cpu, rcu_barrier_func, (void *)cpu, 1);
	3371	+ } else if (rcu_segcblist_n_cbs(&rdp->cblist) &&
	3372	+ cpu_is_offline(cpu)) {
	3373	+ rcu_barrier_trace(TPS("OfflineNoCBQ"), cpu,
	3374	+ rcu_state.barrier_sequence);
	3375	+ local_irq_disable();
	3376	+ rcu_barrier_func((void *)cpu);
	3377	+ local_irq_enable();
	3378	+ } else if (cpu_is_offline(cpu)) {
	3379	+ rcu_barrier_trace(TPS("OfflineNoCBNoQ"), cpu,
	3380	+ rcu_state.barrier_sequence);
3168	3381	} else {
3169	3382	rcu_barrier_trace(TPS("OnlineNQ"), cpu,
3170	3383	rcu_state.barrier_sequence);
@@ -3176,7 +3389,7 @@ void rcu_barrier(void) · 5 unmodified lines
3176	3389	* Now that we have an rcu_barrier_callback() callback on each
3177	3390	* CPU, and thus each counted, remove the initial count.
3178	3391	*/
3179		− if (atomic_dec_and_test(&rcu_state.barrier_cpu_count))
	3392	+ if (atomic_sub_and_test(2, &rcu_state.barrier_cpu_count))
3180	3393	complete(&rcu_state.barrier_completion);
3181	3394
3182	3395	/* Wait for all rcu_barrier_callback() callbacks to be invoked. */
@@ -3275,12 +3488,12 @@ int rcutree_prepare_cpu(unsigned int cpu) · 92 unmodified lines
3275	3488	rnp = rdp->mynode;
3276	3489	raw_spin_lock_rcu_node(rnp); /* irqs already disabled. */
3277	3490	rdp->beenonline = true; /* We have now been online. */
3278		− rdp->gp_seq = rnp->gp_seq;
3279		− rdp->gp_seq_needed = rnp->gp_seq;
	3491	+ rdp->gp_seq = READ_ONCE(rnp->gp_seq);
	3492	+ rdp->gp_seq_needed = rdp->gp_seq;
3280	3493	rdp->cpu_no_qs.b.norm = true;
3281	3494	rdp->core_needs_qs = false;
3282	3495	rdp->rcu_iw_pending = false;
3283		− rdp->rcu_iw_gp_seq = rnp->gp_seq - 1;
	3496	+ rdp->rcu_iw_gp_seq = rdp->gp_seq - 1;
3284	3497	trace_rcu_grace_period(rcu_state.name, rdp->gp_seq, TPS("cpuonl"));
3285	3498	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3286	3499	rcu_prepare_kthreads(cpu);
@@ -3378,7 +3591,7 @@ void rcu_cpu_starting(unsigned int cpu) · 91 unmodified lines
3378	3591	rnp = rdp->mynode;
3379	3592	mask = rdp->grpmask;
3380	3593	raw_spin_lock_irqsave_rcu_node(rnp, flags);
3381		− rnp->qsmaskinitnext \|= mask;
	3594	+ WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext \| mask);
3382	3595	oldmask = rnp->expmaskinitnext;
3383	3596	rnp->expmaskinitnext \|= mask;
3384	3597	oldmask ^= rnp->expmaskinitnext;
@@ -3431,7 +3644,7 @@ void rcu_report_dead(unsigned int cpu) · 46 unmodified lines
3431	3644	rcu_report_qs_rnp(mask, rnp, rnp->gp_seq, flags);
3432	3645	raw_spin_lock_irqsave_rcu_node(rnp, flags);
3433	3646	}
3434		− rnp->qsmaskinitnext &= ~mask;
	3647	+ WRITE_ONCE(rnp->qsmaskinitnext, rnp->qsmaskinitnext & ~mask);
3435	3648	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3436	3649	raw_spin_unlock(&rcu_state.ofl_lock);
3437	3650
@@ -3545,7 +3758,10 @@ static int __init rcu_spawn_gp_kthread(void) · 107 unmodified lines
3545	3758	}
3546	3759	rnp = rcu_get_root();
3547	3760	raw_spin_lock_irqsave_rcu_node(rnp, flags);
3548		− rcu_state.gp_kthread = t;
	3761	+ WRITE_ONCE(rcu_state.gp_activity, jiffies);
	3762	+ WRITE_ONCE(rcu_state.gp_req_activity, jiffies);
	3763	+ // Reset .gp_activity and .gp_req_activity before setting .gp_kthread.
	3764	+ smp_store_release(&rcu_state.gp_kthread, t); /* ^^^ */
3549	3765	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
3550	3766	wake_up_process(t);
3551	3767	rcu_spawn_nocb_kthreads();
@@ -3769,8 +3985,11 @@ static void __init kfree_rcu_batch_init(void) · 217 unmodified lines
3769	3985	struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
3770	3986
3771	3987	spin_lock_init(&krcp->lock);
3772		− for (i = 0; i < KFREE_N_BATCHES; i++)
	3988	+ for (i = 0; i < KFREE_N_BATCHES; i++) {
	3989	+ INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
3773	3990	krcp->krw_arr[i].krcp = krcp;
	3991	+ }
	3992	+
3774	3993	INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
3775	3994	krcp->initialized = true;
3776	3995	}
@@ -3809,6 +4028,13 @@ void __init rcu_init(void) · 32 unmodified lines
3809	4028	rcu_par_gp_wq = alloc_workqueue("rcu_par_gp", WQ_MEM_RECLAIM, 0);
3810	4029	WARN_ON(!rcu_par_gp_wq);
3811	4030	srcu_init();
	4031	+
	4032	+ /* Fill in default value for rcutree.qovld boot parameter. */
	4033	+ /* -After- the rcu_node ->lock fields are initialized! */
	4034	+ if (qovld < 0)
	4035	+ qovld_calc = DEFAULT_RCU_QOVLD_MULT * qhimark;
	4036	+ else
	4037	+ qovld_calc = qovld;
3812	4038	}
3813	4039
3814	4040	#include "tree_stall.h"

kernel/rcu/tree.h+4−0

@@ -68,6 +68,8 @@ struct rcu_node { · 67 unmodified lines
68	68	/* Online CPUs for next expedited GP. */
69	69	/* Any CPU that has ever been online will */
70	70	/* have its bit set. */
	71	+ unsigned long cbovldmask;
	72	+ /* CPUs experiencing callback overload. */
71	73	unsigned long ffmask; /* Fully functional CPUs. */
72	74	unsigned long grpmask; /* Mask to apply to parent qsmask. */
73	75	/* Only one bit will be set in this mask. */
@@ -321,6 +323,8 @@ struct rcu_state { · 247 unmodified lines
321	323	atomic_t expedited_need_qs; /* # CPUs left to check in. */
322	324	struct swait_queue_head expedited_wq; /* Wait for check-ins. */
323	325	int ncpus_snap; /* # CPUs seen last time. */
	326	+ u8 cbovld; /* Callback overload now? */
	327	+ u8 cbovldnext; /* ^ ^ next time? */
324	328
325	329	unsigned long jiffies_force_qs; /* Time at which to invoke */
326	330	/* force_quiescent_state(). */

kernel/rcu/tree_exp.h+9−4

@@ -314,7 +314,7 @@ static bool exp_funnel_lock(unsigned long s) · 313 unmodified lines
314	314	sync_exp_work_done(s));
315	315	return true;
316	316	}
317		− rnp->exp_seq_rq = s; /* Followers can wait on us. */
	317	+ WRITE_ONCE(rnp->exp_seq_rq, s); /* Followers can wait on us. */
318	318	spin_unlock(&rnp->exp_lock);
319	319	trace_rcu_exp_funnel_lock(rcu_state.name, rnp->level,
320	320	rnp->grplo, rnp->grphi, TPS("nxtlvl"));
@@ -485,6 +485,7 @@ static bool synchronize_rcu_expedited_wait_once(long tlimit) · 164 unmodified lines
485	485	static void synchronize_rcu_expedited_wait(void)
486	486	{
487	487	int cpu;
	488	+ unsigned long j;
488	489	unsigned long jiffies_stall;
489	490	unsigned long jiffies_start;
490	491	unsigned long mask;
@@ -496,7 +497,7 @@ static void synchronize_rcu_expedited_wait(void) · 5 unmodified lines
496	497	trace_rcu_exp_grace_period(rcu_state.name, rcu_exp_gp_seq_endval(), TPS("startwait"));
497	498	jiffies_stall = rcu_jiffies_till_stall_check();
498	499	jiffies_start = jiffies;
499		− if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
	500	+ if (tick_nohz_full_enabled() && rcu_inkernel_boot_has_ended()) {
500	501	if (synchronize_rcu_expedited_wait_once(1))
501	502	return;
502	503	rcu_for_each_leaf_node(rnp) {
@@ -508,12 +509,16 @@ static void synchronize_rcu_expedited_wait(void) · 5 unmodified lines
508	509	tick_dep_set_cpu(cpu, TICK_DEP_BIT_RCU_EXP);
509	510	}
510	511	}
	512	+ j = READ_ONCE(jiffies_till_first_fqs);
	513	+ if (synchronize_rcu_expedited_wait_once(j + HZ))
	514	+ return;
	515	+ WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT));
511	516	}
512	517
513	518	for (;;) {
514	519	if (synchronize_rcu_expedited_wait_once(jiffies_stall))
515	520	return;
516		− if (rcu_cpu_stall_suppress)
	521	+ if (rcu_stall_is_suppressed())
517	522	continue;
518	523	panic_on_rcu_stall();
519	524	pr_err("INFO: %s detected expedited stalls on CPUs/tasks: {",
@@ -589,7 +594,7 @@ static void rcu_exp_wait_wake(unsigned long s) · 69 unmodified lines
589	594	spin_lock(&rnp->exp_lock);
590	595	/* Recheck, avoid hang in case someone just arrived. */
591	596	if (ULONG_CMP_LT(rnp->exp_seq_rq, s))
592		− rnp->exp_seq_rq = s;
	597	+ WRITE_ONCE(rnp->exp_seq_rq, s);
593	598	spin_unlock(&rnp->exp_lock);
594	599	}
595	600	smp_mb(); /* All above changes before wakeup. */

kernel/rcu/tree_plugin.h+16−9

@@ -56,6 +56,8 @@ static void __init rcu_bootup_announce_oddness(void) · 55 unmodified lines
56	56	pr_info("\tBoot-time adjustment of callback high-water mark to %ld.\n", qhimark);
57	57	if (qlowmark != DEFAULT_RCU_QLOMARK)
58	58	pr_info("\tBoot-time adjustment of callback low-water mark to %ld.\n", qlowmark);
	59	+ if (qovld != DEFAULT_RCU_QOVLD)
	60	+ pr_info("\tBoot-time adjustment of callback overload level to %ld.\n", qovld);
59	61	if (jiffies_till_first_fqs != ULONG_MAX)
60	62	pr_info("\tBoot-time adjustment of first FQS scan delay to %ld jiffies.\n", jiffies_till_first_fqs);
61	63	if (jiffies_till_next_fqs != ULONG_MAX)
@@ -753,7 +755,7 @@ dump_blkd_tasks(struct rcu_node *rnp, int ncheck) · 691 unmodified lines
753	755	raw_lockdep_assert_held_rcu_node(rnp);
754	756	pr_info("%s: grp: %d-%d level: %d ->gp_seq %ld ->completedqs %ld\n",
755	757	__func__, rnp->grplo, rnp->grphi, rnp->level,
756		− (long)rnp->gp_seq, (long)rnp->completedqs);
	758	+ (long)READ_ONCE(rnp->gp_seq), (long)rnp->completedqs);
757	759	for (rnp1 = rnp; rnp1; rnp1 = rnp1->parent)
758	760	pr_info("%s: %d:%d ->qsmask %#lx ->qsmaskinit %#lx ->qsmaskinitnext %#lx\n",
759	761	__func__, rnp1->grplo, rnp1->grphi, rnp1->qsmask, rnp1->qsmaskinit, rnp1->qsmaskinitnext);
@@ -1032,18 +1034,18 @@ static int rcu_boost_kthread(void *arg) · 272 unmodified lines
1032	1034
1033	1035	trace_rcu_utilization(TPS("Start boost kthread@init"));
1034	1036	for (;;) {
1035		− rnp->boost_kthread_status = RCU_KTHREAD_WAITING;
	1037	+ WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_WAITING);
1036	1038	trace_rcu_utilization(TPS("End boost kthread@rcu_wait"));
1037	1039	rcu_wait(rnp->boost_tasks \|\| rnp->exp_tasks);
1038	1040	trace_rcu_utilization(TPS("Start boost kthread@rcu_wait"));
1039		− rnp->boost_kthread_status = RCU_KTHREAD_RUNNING;
	1041	+ WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_RUNNING);
1040	1042	more2boost = rcu_boost(rnp);
1041	1043	if (more2boost)
1042	1044	spincnt++;
1043	1045	else
1044	1046	spincnt = 0;
1045	1047	if (spincnt > 10) {
1046		− rnp->boost_kthread_status = RCU_KTHREAD_YIELDING;
	1048	+ WRITE_ONCE(rnp->boost_kthread_status, RCU_KTHREAD_YIELDING);
1047	1049	trace_rcu_utilization(TPS("End boost kthread@rcu_yield"));
1048	1050	schedule_timeout_interruptible(2);
1049	1051	trace_rcu_utilization(TPS("Start boost kthread@rcu_yield"));
@@ -1077,12 +1079,12 @@ static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags) · 27 unmodified lines
1077	1079	(rnp->gp_tasks != NULL &&
1078	1080	rnp->boost_tasks == NULL &&
1079	1081	rnp->qsmask == 0 &&
1080		− ULONG_CMP_GE(jiffies, rnp->boost_time))) {
	1082	+ (ULONG_CMP_GE(jiffies, rnp->boost_time) \|\| rcu_state.cbovld))) {
1081	1083	if (rnp->exp_tasks == NULL)
1082	1084	rnp->boost_tasks = rnp->gp_tasks;
1083	1085	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1084	1086	rcu_wake_cond(rnp->boost_kthread_task,
1085		− rnp->boost_kthread_status);
	1087	+ READ_ONCE(rnp->boost_kthread_status));
1086	1088	} else {
1087	1089	raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
1088	1090	}
@@ -1486,6 +1488,7 @@ module_param(nocb_nobypass_lim_per_jiffy, int, 0); · 397 unmodified lines
1486	1488	* flag the contention.
1487	1489	*/
1488	1490	static void rcu_nocb_bypass_lock(struct rcu_data *rdp)
	1491	+ __acquires(&rdp->nocb_bypass_lock)
1489	1492	{
1490	1493	lockdep_assert_irqs_disabled();
1491	1494	if (raw_spin_trylock(&rdp->nocb_bypass_lock))
@@ -1529,6 +1532,7 @@ static bool rcu_nocb_bypass_trylock(struct rcu_data *rdp) · 37 unmodified lines
1529	1532	* Release the specified rcu_data structure's ->nocb_bypass_lock.
1530	1533	*/
1531	1534	static void rcu_nocb_bypass_unlock(struct rcu_data *rdp)
	1535	+ __releases(&rdp->nocb_bypass_lock)
1532	1536	{
1533	1537	lockdep_assert_irqs_disabled();
1534	1538	raw_spin_unlock(&rdp->nocb_bypass_lock);
@@ -1577,8 +1581,7 @@ static void rcu_nocb_unlock_irqrestore(struct rcu_data *rdp, · 42 unmodified lines
1577	1581	static void rcu_lockdep_assert_cblist_protected(struct rcu_data *rdp)
1578	1582	{
1579	1583	lockdep_assert_irqs_disabled();
1580		− if (rcu_segcblist_is_offloaded(&rdp->cblist) &&
1581		− cpu_online(rdp->cpu))
	1584	+ if (rcu_segcblist_is_offloaded(&rdp->cblist))
1582	1585	lockdep_assert_held(&rdp->nocb_lock);
1583	1586	}
1584	1587
@@ -1930,6 +1933,7 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) · 345 unmodified lines
1930	1933	struct rcu_data *rdp;
1931	1934	struct rcu_node *rnp;
1932	1935	unsigned long wait_gp_seq = 0; // Suppress "use uninitialized" warning.
	1936	+ bool wasempty = false;
1933	1937
1934	1938	/*
1935	1939	* Each pass through the following loop checks for CBs and for the
@@ -1969,10 +1973,13 @@ static void nocb_gp_wait(struct rcu_data *my_rdp) · 33 unmodified lines
1969	1973	rcu_seq_done(&rnp->gp_seq, cur_gp_seq))) {
1970	1974	raw_spin_lock_rcu_node(rnp); /* irqs disabled. */
1971	1975	needwake_gp = rcu_advance_cbs(rnp, rdp);
	1976	+ wasempty = rcu_segcblist_restempty(&rdp->cblist,
	1977	+ RCU_NEXT_READY_TAIL);
1972	1978	raw_spin_unlock_rcu_node(rnp); /* irqs disabled. */
1973	1979	}
1974	1980	// Need to wait on some grace period?
1975		− WARN_ON_ONCE(!rcu_segcblist_restempty(&rdp->cblist,
	1981	+ WARN_ON_ONCE(wasempty &&
	1982	+ !rcu_segcblist_restempty(&rdp->cblist,
1976	1983	RCU_NEXT_READY_TAIL));
1977	1984	if (rcu_segcblist_nextgp(&rdp->cblist, &cur_gp_seq)) {
1978	1985	if (!needwait_gp \|\|

kernel/rcu/tree_stall.h+23−18

@@ -102,7 +102,7 @@ static void record_gp_stall_check_time(void) · 101 unmodified lines
102	102	unsigned long j = jiffies;
103	103	unsigned long j1;
104	104
105		− rcu_state.gp_start = j;
	105	+ WRITE_ONCE(rcu_state.gp_start, j);
106	106	j1 = rcu_jiffies_till_stall_check();
107	107	/* Record ->gp_start before ->jiffies_stall. */
108	108	smp_store_release(&rcu_state.jiffies_stall, j + j1); /* ^^^ */
@@ -383,7 +383,7 @@ static void print_other_cpu_stall(unsigned long gp_seq) · 274 unmodified lines
383	383
384	384	/* Kick and suppress, if so configured. */
385	385	rcu_stall_kick_kthreads();
386		− if (rcu_cpu_stall_suppress)
	386	+ if (rcu_stall_is_suppressed())
387	387	return;
388	388
389	389	/*
@@ -452,7 +452,7 @@ static void print_cpu_stall(void) · 62 unmodified lines
452	452
453	453	/* Kick and suppress, if so configured. */
454	454	rcu_stall_kick_kthreads();
455		− if (rcu_cpu_stall_suppress)
	455	+ if (rcu_stall_is_suppressed())
456	456	return;
457	457
458	458	/*
@@ -504,7 +504,7 @@ static void check_cpu_stall(struct rcu_data *rdp) · 45 unmodified lines
504	504	unsigned long js;
505	505	struct rcu_node *rnp;
506	506
507		− if ((rcu_cpu_stall_suppress && !rcu_kick_kthreads) \|\|
	507	+ if ((rcu_stall_is_suppressed() && !rcu_kick_kthreads) \|\|
508	508	!rcu_gp_in_progress())
509	509	return;
510	510	rcu_stall_kick_kthreads();
@@ -578,6 +578,7 @@ void show_rcu_gp_kthreads(void) · 67 unmodified lines
578	578	unsigned long jw;
579	579	struct rcu_data *rdp;
580	580	struct rcu_node *rnp;
	581	+ struct task_struct *t = READ_ONCE(rcu_state.gp_kthread);
581	582
582	583	j = jiffies;
583	584	ja = j - READ_ONCE(rcu_state.gp_activity);
@@ -585,28 +586,28 @@ void show_rcu_gp_kthreads(void) · 1 unmodified line
585	586	jw = j - READ_ONCE(rcu_state.gp_wake_time);
586	587	pr_info("%s: wait state: %s(%d) ->state: %#lx delta ->gp_activity %lu ->gp_req_activity %lu ->gp_wake_time %lu ->gp_wake_seq %ld ->gp_seq %ld ->gp_seq_needed %ld ->gp_flags %#x\n",
587	588	rcu_state.name, gp_state_getname(rcu_state.gp_state),
588		− rcu_state.gp_state,
589		− rcu_state.gp_kthread ? rcu_state.gp_kthread->state : 0x1ffffL,
	589	+ rcu_state.gp_state, t ? t->state : 0x1ffffL,
590	590	ja, jr, jw, (long)READ_ONCE(rcu_state.gp_wake_seq),
591	591	(long)READ_ONCE(rcu_state.gp_seq),
592	592	(long)READ_ONCE(rcu_get_root()->gp_seq_needed),
593	593	READ_ONCE(rcu_state.gp_flags));
594	594	rcu_for_each_node_breadth_first(rnp) {
595		− if (ULONG_CMP_GE(rcu_state.gp_seq, rnp->gp_seq_needed))
	595	+ if (ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq),
	596	+ READ_ONCE(rnp->gp_seq_needed)))
596	597	continue;
597	598	pr_info("\trcu_node %d:%d ->gp_seq %ld ->gp_seq_needed %ld\n",
598		− rnp->grplo, rnp->grphi, (long)rnp->gp_seq,
599		− (long)rnp->gp_seq_needed);
	599	+ rnp->grplo, rnp->grphi, (long)READ_ONCE(rnp->gp_seq),
	600	+ (long)READ_ONCE(rnp->gp_seq_needed));
600	601	if (!rcu_is_leaf_node(rnp))
601	602	continue;
602	603	for_each_leaf_node_possible_cpu(rnp, cpu) {
603	604	rdp = per_cpu_ptr(&rcu_data, cpu);
604		− if (rdp->gpwrap \|\|
605		− ULONG_CMP_GE(rcu_state.gp_seq,
606		− rdp->gp_seq_needed))
	605	+ if (READ_ONCE(rdp->gpwrap) \|\|
	606	+ ULONG_CMP_GE(READ_ONCE(rcu_state.gp_seq),
	607	+ READ_ONCE(rdp->gp_seq_needed)))
607	608	continue;
608	609	pr_info("\tcpu %d ->gp_seq_needed %ld\n",
609		− cpu, (long)rdp->gp_seq_needed);
	610	+ cpu, (long)READ_ONCE(rdp->gp_seq_needed));
610	611	}
611	612	}
612	613	for_each_possible_cpu(cpu) {
@@ -631,7 +632,9 @@ static void rcu_check_gp_start_stall(struct rcu_node rnp, struct rcu_data rdp, · 18 unmodified lines
631	632	static atomic_t warned = ATOMIC_INIT(0);
632	633
633	634	if (!IS_ENABLED(CONFIG_PROVE_RCU) \|\| rcu_gp_in_progress() \|\|
634		− ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed))
	635	+ ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
	636	+ READ_ONCE(rnp_root->gp_seq_needed)) \|\|
	637	+ !smp_load_acquire(&rcu_state.gp_kthread)) // Get stable kthread.
635	638	return;
636	639	j = jiffies; /* Expensive access, and in common case don't get here. */
637	640	if (time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) \|\|
@@ -642,7 +645,8 @@ static void rcu_check_gp_start_stall(struct rcu_node rnp, struct rcu_data rdp, · 4 unmodified lines
642	645	raw_spin_lock_irqsave_rcu_node(rnp, flags);
643	646	j = jiffies;
644	647	if (rcu_gp_in_progress() \|\|
645		− ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) \|\|
	648	+ ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
	649	+ READ_ONCE(rnp_root->gp_seq_needed)) \|\|
646	650	time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) \|\|
647	651	time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) \|\|
648	652	atomic_read(&warned)) {
@@ -655,9 +659,10 @@ static void rcu_check_gp_start_stall(struct rcu_node rnp, struct rcu_data rdp, · 6 unmodified lines
655	659	raw_spin_lock_rcu_node(rnp_root); /* irqs already disabled. */
656	660	j = jiffies;
657	661	if (rcu_gp_in_progress() \|\|
658		− ULONG_CMP_GE(rnp_root->gp_seq, rnp_root->gp_seq_needed) \|\|
659		− time_before(j, rcu_state.gp_req_activity + gpssdelay) \|\|
660		− time_before(j, rcu_state.gp_activity + gpssdelay) \|\|
	662	+ ULONG_CMP_GE(READ_ONCE(rnp_root->gp_seq),
	663	+ READ_ONCE(rnp_root->gp_seq_needed)) \|\|
	664	+ time_before(j, READ_ONCE(rcu_state.gp_req_activity) + gpssdelay) \|\|
	665	+ time_before(j, READ_ONCE(rcu_state.gp_activity) + gpssdelay) \|\|
661	666	atomic_xchg(&warned, 1)) {
662	667	if (rnp_root != rnp)
663	668	/* irqs remain disabled. */

kernel/rcu/update.c+23−5

@@ -183,6 +183,8 @@ void rcu_unexpedite_gp(void) · 182 unmodified lines
183	183	}
184	184	EXPORT_SYMBOL_GPL(rcu_unexpedite_gp);
185	185
	186	+static bool rcu_boot_ended __read_mostly;
	187	+
186	188	/*
187	189	* Inform RCU of the end of the in-kernel boot sequence.
188	190	*/
@@ -191,8 +193,18 @@ void rcu_end_inkernel_boot(void) · 2 unmodified lines
191	193	rcu_unexpedite_gp();
192	194	if (rcu_normal_after_boot)
193	195	WRITE_ONCE(rcu_normal, 1);
	196	+ rcu_boot_ended = 1;
194	197	}
195	198
	199	+/*
	200	+ * Let rcutorture know when it is OK to turn it up to eleven.
	201	+ */
	202	+bool rcu_inkernel_boot_has_ended(void)
	203	+{
	204	+ return rcu_boot_ended;
	205	+}
	206	+EXPORT_SYMBOL_GPL(rcu_inkernel_boot_has_ended);
	207	+
196	208	#endif /* #ifndef CONFIG_TINY_RCU */
197	209
198	210	/*
@@ -464,13 +476,19 @@ EXPORT_SYMBOL_GPL(rcutorture_sched_setaffinity); · 265 unmodified lines
464	476	#ifdef CONFIG_RCU_STALL_COMMON
465	477	int rcu_cpu_stall_ftrace_dump __read_mostly;
466	478	module_param(rcu_cpu_stall_ftrace_dump, int, 0644);
467		−int rcu_cpu_stall_suppress __read_mostly; /* 1 = suppress stall warnings. */
	479	+int rcu_cpu_stall_suppress __read_mostly; // !0 = suppress stall warnings.
468	480	EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress);
469	481	module_param(rcu_cpu_stall_suppress, int, 0644);
470	482	int rcu_cpu_stall_timeout __read_mostly = CONFIG_RCU_CPU_STALL_TIMEOUT;
471	483	module_param(rcu_cpu_stall_timeout, int, 0644);
472	484	#endif /* #ifdef CONFIG_RCU_STALL_COMMON */
473	485
	486	+// Suppress boot-time RCU CPU stall warnings and rcutorture writer stall
	487	+// warnings. Also used by rcutorture even if stall warnings are excluded.
	488	+int rcu_cpu_stall_suppress_at_boot __read_mostly; // !0 = suppress boot stalls.
	489	+EXPORT_SYMBOL_GPL(rcu_cpu_stall_suppress_at_boot);
	490	+module_param(rcu_cpu_stall_suppress_at_boot, int, 0444);
	491	+
474	492	#ifdef CONFIG_TASKS_RCU
475	493
476	494	/*
@@ -528,7 +546,7 @@ void call_rcu_tasks(struct rcu_head *rhp, rcu_callback_t func) · 51 unmodified lines
528	546	rhp->func = func;
529	547	raw_spin_lock_irqsave(&rcu_tasks_cbs_lock, flags);
530	548	needwake = !rcu_tasks_cbs_head;
531		− *rcu_tasks_cbs_tail = rhp;
	549	+ WRITE_ONCE(*rcu_tasks_cbs_tail, rhp);
532	550	rcu_tasks_cbs_tail = &rhp->next;
533	551	raw_spin_unlock_irqrestore(&rcu_tasks_cbs_lock, flags);
534	552	/* We can't create the thread unless interrupts are enabled. */
@@ -658,7 +676,7 @@ static int __noreturn rcu_tasks_kthread(void *arg) · 123 unmodified lines
658	676	/* If there were none, wait a bit and start over. */
659	677	if (!list) {
660	678	wait_event_interruptible(rcu_tasks_cbs_wq,
661		− rcu_tasks_cbs_head);
	679	+ READ_ONCE(rcu_tasks_cbs_head));
662	680	if (!rcu_tasks_cbs_head) {
663	681	WARN_ON(signal_pending(current));
664	682	schedule_timeout_interruptible(HZ/10);
@@ -801,7 +819,7 @@ static int __init rcu_spawn_tasks_kthread(void) · 136 unmodified lines
801	819	core_initcall(rcu_spawn_tasks_kthread);
802	820
803	821	/* Do the srcu_read_lock() for the above synchronize_srcu(). */
804		−void exit_tasks_rcu_start(void)
	822	+void exit_tasks_rcu_start(void) __acquires(&tasks_rcu_exit_srcu)
805	823	{
806	824	preempt_disable();
807	825	current->rcu_tasks_idx = __srcu_read_lock(&tasks_rcu_exit_srcu);
@@ -809,7 +827,7 @@ void exit_tasks_rcu_start(void) · 1 unmodified line
809	827	}
810	828
811	829	/* Do the srcu_read_unlock() for the above synchronize_srcu(). */
812		−void exit_tasks_rcu_finish(void)
	830	+void exit_tasks_rcu_finish(void) __releases(&tasks_rcu_exit_srcu)
813	831	{
814	832	preempt_disable();
815	833	__srcu_read_unlock(&tasks_rcu_exit_srcu, current->rcu_tasks_idx);

kernel/time/timer.c+4−3

@@ -944,6 +944,7 @@ static struct timer_base lock_timer_base(struct timer_list timer, · 943 unmodified lines
944	944
945	945	#define MOD_TIMER_PENDING_ONLY 0x01
946	946	#define MOD_TIMER_REDUCE 0x02
	947	+#define MOD_TIMER_NOTPENDING 0x04
947	948
948	949	static inline int
949	950	__mod_timer(struct timer_list *timer, unsigned long expires, unsigned int options)
@@ -960,7 +961,7 @@ __mod_timer(struct timer_list *timer, unsigned long expires, unsigned int option · 10 unmodified lines
960	961	* the timer is re-modified to have the same timeout or ends up in the
961	962	* same array bucket then just return:
962	963	*/
963		− if (timer_pending(timer)) {
	964	+ if (!(options & MOD_TIMER_NOTPENDING) && timer_pending(timer)) {
964	965	/*
965	966	* The downside of this optimization is that it can result in
966	967	* larger granularity than you would get from adding a new
@@ -1133,7 +1134,7 @@ EXPORT_SYMBOL(timer_reduce); · 166 unmodified lines
1133	1134	void add_timer(struct timer_list *timer)
1134	1135	{
1135	1136	BUG_ON(timer_pending(timer));
1136		− mod_timer(timer, timer->expires);
	1137	+ __mod_timer(timer, timer->expires, MOD_TIMER_NOTPENDING);
1137	1138	}
1138	1139	EXPORT_SYMBOL(add_timer);
1139	1140
@@ -1891,7 +1892,7 @@ signed long __sched schedule_timeout(signed long timeout) · 751 unmodified lines
1891	1892
1892	1893	timer.task = current;
1893	1894	timer_setup_on_stack(&timer.timer, process_timeout, 0);
1894		− __mod_timer(&timer.timer, expire, 0);
	1895	+ __mod_timer(&timer.timer, expire, MOD_TIMER_NOTPENDING);
1895	1896	schedule();
1896	1897	del_singleshot_timer_sync(&timer.timer);
1897	1898

kernel/torture.c+25−4

@@ -42,6 +42,9 @@ · 41 unmodified lines
42	42	MODULE_LICENSE("GPL");
43	43	MODULE_AUTHOR("Paul E. McKenney <paulmck@linux.ibm.com>");
44	44
	45	+static bool disable_onoff_at_boot;
	46	+module_param(disable_onoff_at_boot, bool, 0444);
	47	+
45	48	static char *torture_type;
46	49	static int verbose;
47	50
@@ -84,6 +87,7 @@ bool torture_offline(int cpu, long n_offl_attempts, long n_offl_successes, · 36 unmodified lines
84	87	{
85	88	unsigned long delta;
86	89	int ret;
	90	+ char *s;
87	91	unsigned long starttime;
88	92
89	93	if (!cpu_online(cpu) \|\| !cpu_is_hotpluggable(cpu))
@@ -99,10 +103,16 @@ bool torture_offline(int cpu, long n_offl_attempts, long n_offl_successes, · 9 unmodified lines
99	103	(*n_offl_attempts)++;
100	104	ret = cpu_down(cpu);
101	105	if (ret) {
	106	+ s = "";
	107	+ if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
	108	+ // PCI probe frequently disables hotplug during boot.
	109	+ (*n_offl_attempts)--;
	110	+ s = " (-EBUSY forgiven during boot)";
	111	+ }
102	112	if (verbose)
103	113	pr_alert("%s" TORTURE_FLAG
104		− "torture_onoff task: offline %d failed: errno %d\n",
105		− torture_type, cpu, ret);
	114	+ "torture_onoff task: offline %d failed%s: errno %d\n",
	115	+ torture_type, cpu, s, ret);
106	116	} else {
107	117	if (verbose > 1)
108	118	pr_alert("%s" TORTURE_FLAG
@@ -137,6 +147,7 @@ bool torture_online(int cpu, long n_onl_attempts, long n_onl_successes, · 28 unmodified lines
137	147	{
138	148	unsigned long delta;
139	149	int ret;
	150	+ char *s;
140	151	unsigned long starttime;
141	152
142	153	if (cpu_online(cpu) \|\| !cpu_is_hotpluggable(cpu))
@@ -150,10 +161,16 @@ bool torture_online(int cpu, long n_onl_attempts, long n_onl_successes, · 7 unmodified lines
150	161	(*n_onl_attempts)++;
151	162	ret = cpu_up(cpu);
152	163	if (ret) {
	164	+ s = "";
	165	+ if (!rcu_inkernel_boot_has_ended() && ret == -EBUSY) {
	166	+ // PCI probe frequently disables hotplug during boot.
	167	+ (*n_onl_attempts)--;
	168	+ s = " (-EBUSY forgiven during boot)";
	169	+ }
153	170	if (verbose)
154	171	pr_alert("%s" TORTURE_FLAG
155		− "torture_onoff task: online %d failed: errno %d\n",
156		− torture_type, cpu, ret);
	172	+ "torture_onoff task: online %d failed%s: errno %d\n",
	173	+ torture_type, cpu, s, ret);
157	174	} else {
158	175	if (verbose > 1)
159	176	pr_alert("%s" TORTURE_FLAG
@@ -215,6 +232,10 @@ torture_onoff(void *arg) · 55 unmodified lines
215	232	VERBOSE_TOROUT_STRING("torture_onoff end holdoff");
216	233	}
217	234	while (!torture_must_stop()) {
	235	+ if (disable_onoff_at_boot && !rcu_inkernel_boot_has_ended()) {
	236	+ schedule_timeout_interruptible(HZ / 10);
	237	+ continue;
	238	+ }
218	239	cpu = (torture_random(&rand) >> 4) % (maxcpu + 1);
219	240	if (!torture_offline(cpu,
220	241	&n_offline_attempts, &n_offline_successes,

tools/testing/selftests/rcutorture/bin/functions.sh+1−1

@@ -12,7 +12,7 @@ · 11 unmodified lines
12	12	# Returns 1 if the specified boot-parameter string tells rcutorture to
13	13	# test CPU-hotplug operations.
14	14	bootparam_hotplug_cpu () {
15		− echo "$1" \| grep -q "rcutorture\.onoff_"
	15	+ echo "$1" \| grep -q "torture\.onoff_"
16	16	}
17	17
18	18	# checkarg --argname argtype $# arg mustmatch cannotmatch

tools/testing/selftests/rcutorture/bin/kvm-find-errors.sh+2−0

@@ -20,7 +20,9 @@ · 19 unmodified lines
20	20	rundir="${1}"
21	21	if test -z "$rundir" -o ! -d "$rundir"
22	22	then
	23	+ echo Directory "$rundir" not found.
23	24	echo Usage: $0 directory
	25	+ exit 1
24	26	fi
25	27	editor=${EDITOR-vi}
26	28

tools/testing/selftests/rcutorture/bin/kvm-recheck.sh+16−1

@@ -13,6 +13,9 @@ · 12 unmodified lines
13	13	#
14	14	# Authors: Paul E. McKenney <paulmck@linux.ibm.com>
15	15
	16	+T=/tmp/kvm-recheck.sh.$$
	17	+trap 'rm -f $T' 0 2
	18	+
16	19	PATH=`pwd`/tools/testing/selftests/rcutorture/bin:$PATH; export PATH
17	20	. functions.sh
18	21	for rd in "$@"
@@ -68,4 +71,16 @@ do · 49 unmodified lines
68	71	fi
69	72	done
70	73	done
71		−EDITOR=echo kvm-find-errors.sh "${@: -1}" > /dev/null 2>&1
	74	+EDITOR=echo kvm-find-errors.sh "${@: -1}" > $T 2>&1
	75	+ret=$?
	76	+builderrors="`tr ' ' '\012' < $T \| grep -c '/Make.out.diags'`"
	77	+if test "$builderrors" -gt 0
	78	+then
	79	+ echo $builderrors runs with build errors.
	80	+fi
	81	+runerrors="`tr ' ' '\012' < $T \| grep -c '/console.log.diags'`"
	82	+if test "$runerrors" -gt 0
	83	+then
	84	+ echo $runerrors runs with runtime errors.
	85	+fi
	86	+exit $ret

tools/testing/selftests/rcutorture/bin/kvm.sh+1−1

@@ -39,7 +39,7 @@ TORTURE_TRUST_MAKE="" · 38 unmodified lines
39	39	resdir=""
40	40	configs=""
41	41	cpus=0
42		−ds=`date +%Y.%m.%d-%H:%M:%S`
	42	+ds=`date +%Y.%m.%d-%H.%M.%S`
43	43	jitter="-1"
44	44
45	45	usage () {

tools/testing/selftests/rcutorture/configs/rcu/CFcommon+2−0

@@ -3,3 +3,5 @@ CONFIG_PRINTK_TIME=y · 2 unmodified lines
3	3	CONFIG_HYPERVISOR_GUEST=y
4	4	CONFIG_PARAVIRT=y
5	5	CONFIG_KVM_GUEST=y
	6	+CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=n
	7	+CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY=n

tools/testing/selftests/rcutorture/configs/rcu/TREE10+18−0

@@ -0,0 +1,18 @@
	1	+CONFIG_SMP=y
	2	+CONFIG_NR_CPUS=100
	3	+CONFIG_PREEMPT_NONE=y
	4	+CONFIG_PREEMPT_VOLUNTARY=n
	5	+CONFIG_PREEMPT=n
	6	+#CHECK#CONFIG_TREE_RCU=y
	7	+CONFIG_HZ_PERIODIC=n
	8	+CONFIG_NO_HZ_IDLE=y
	9	+CONFIG_NO_HZ_FULL=n
	10	+CONFIG_RCU_FAST_NO_HZ=n
	11	+CONFIG_RCU_TRACE=n
	12	+CONFIG_RCU_NOCB_CPU=n
	13	+CONFIG_DEBUG_LOCK_ALLOC=n
	14	+CONFIG_PROVE_LOCKING=n
	15	+#CHECK#CONFIG_PROVE_RCU=n
	16	+CONFIG_DEBUG_OBJECTS=n
	17	+CONFIG_DEBUG_OBJECTS_RCU_HEAD=n
	18	+CONFIG_RCU_EXPERT=n