Merge branches 'fixes.2020.04.27a', 'kfree_rcu.2020.04.27a', 'rcu-tasks.2020.04.27a', 'stall.2020.04.27a' and 'torture.2020.05.07a' into HEAD

fixes.2020.04.27a: Miscellaneous fixes. kfree_rcu.2020.04.27a: Changes related to kfree_rcu(). rcu-tasks.2020.04.27a: Addition of new RCU-tasks flavors. stall.2020.04.27a: RCU CPU stall-warning updates. torture.2020.05.07a: Torture-test updates.

Paul E. McKenney committed 6 years ago

Commit f736e0f5 parents e2f3ccf 6be7436 e5a971d 33b2b93 3c80b4040 files · +2054 −564Browse files

Showing the changes to include/linux (8 of 40 files). Show all

Unified · Split

8 files

include/linux/rcupdate.h +43 −10
include/linux/rcupdate_trace.h +88 −0
include/linux/rcupdate_wait.h +19 −0
include/linux/rcutiny.h +2 −1
include/linux/rcutree.h +1 −0
include/linux/sched.h +9 −1
include/linux/torture.h +1 −1
include/linux/wait.h +2 −0

include/linux/rcupdate.h+43−10

@@ -37,6 +37,7 @@ · 36 unmodified lines
37	37	/* Exported common interfaces */
38	38	void call_rcu(struct rcu_head *head, rcu_callback_t func);
39	39	void rcu_barrier_tasks(void);
	40	+void rcu_barrier_tasks_rude(void);
40	41	void synchronize_rcu(void);
41	42
42	43	#ifdef CONFIG_PREEMPT_RCU
@@ -129,25 +130,57 @@ static inline void rcu_init_nohz(void) { } · 86 unmodified lines
129	130	* Note a quasi-voluntary context switch for RCU-tasks's benefit.
130	131	* This is a macro rather than an inline function to avoid #include hell.
131	132	*/
132		−#ifdef CONFIG_TASKS_RCU
133		−#define rcu_tasks_qs(t) \
134		− do { \
135		− if (READ_ONCE((t)->rcu_tasks_holdout)) \
136		− WRITE_ONCE((t)->rcu_tasks_holdout, false); \
	133	+#ifdef CONFIG_TASKS_RCU_GENERIC
	134	+
	135	+# ifdef CONFIG_TASKS_RCU
	136	+# define rcu_tasks_classic_qs(t, preempt) \
	137	+ do { \
	138	+ if (!(preempt) && READ_ONCE((t)->rcu_tasks_holdout)) \
	139	+ WRITE_ONCE((t)->rcu_tasks_holdout, false); \
137	140	} while (0)
138		−#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t)
139	141	void call_rcu_tasks(struct rcu_head *head, rcu_callback_t func);
140	142	void synchronize_rcu_tasks(void);
	143	+# else
	144	+# define rcu_tasks_classic_qs(t, preempt) do { } while (0)
	145	+# define call_rcu_tasks call_rcu
	146	+# define synchronize_rcu_tasks synchronize_rcu
	147	+# endif
	148	+
	149	+# ifdef CONFIG_TASKS_RCU_TRACE
	150	+# define rcu_tasks_trace_qs(t) \
	151	+ do { \
	152	+ if (!likely(READ_ONCE((t)->trc_reader_checked)) && \
	153	+ !unlikely(READ_ONCE((t)->trc_reader_nesting))) { \
	154	+ smp_store_release(&(t)->trc_reader_checked, true); \
	155	+ smp_mb(); /* Readers partitioned by store. */ \
	156	+ } \
	157	+ } while (0)
	158	+# else
	159	+# define rcu_tasks_trace_qs(t) do { } while (0)
	160	+# endif
	161	+
	162	+#define rcu_tasks_qs(t, preempt) \
	163	+do { \
	164	+ rcu_tasks_classic_qs((t), (preempt)); \
	165	+ rcu_tasks_trace_qs((t)); \
	166	+} while (0)
	167	+
	168	+# ifdef CONFIG_TASKS_RUDE_RCU
	169	+void call_rcu_tasks_rude(struct rcu_head *head, rcu_callback_t func);
	170	+void synchronize_rcu_tasks_rude(void);
	171	+# endif
	172	+
	173	+#define rcu_note_voluntary_context_switch(t) rcu_tasks_qs(t, false)
141	174	void exit_tasks_rcu_start(void);
142	175	void exit_tasks_rcu_finish(void);
143		−#else /* #ifdef CONFIG_TASKS_RCU */
144		−#define rcu_tasks_qs(t) do { } while (0)
	176	+#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
	177	+#define rcu_tasks_qs(t, preempt) do { } while (0)
145	178	#define rcu_note_voluntary_context_switch(t) do { } while (0)
146	179	#define call_rcu_tasks call_rcu
147	180	#define synchronize_rcu_tasks synchronize_rcu
148	181	static inline void exit_tasks_rcu_start(void) { }
149	182	static inline void exit_tasks_rcu_finish(void) { }
150		−#endif /* #else #ifdef CONFIG_TASKS_RCU */
	183	+#endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
151	184
152	185	/**
153	186	* cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU
@@ -158,7 +191,7 @@ static inline void exit_tasks_rcu_finish(void) { } · 4 unmodified lines
158	191	*/
159	192	#define cond_resched_tasks_rcu_qs() \
160	193	do { \
161		− rcu_tasks_qs(current); \
	194	+ rcu_tasks_qs(current, false); \
162	195	cond_resched(); \
163	196	} while (0)
164	197

include/linux/rcupdate_trace.h+88−0

@@ -0,0 +1,88 @@
	1	+/* SPDX-License-Identifier: GPL-2.0+ */
	2	+/*
	3	+ * Read-Copy Update mechanism for mutual exclusion, adapted for tracing.
	4	+ *
	5	+ * Copyright (C) 2020 Paul E. McKenney.
	6	+ */
	7	+
	8	+#ifndef __LINUX_RCUPDATE_TRACE_H
	9	+#define __LINUX_RCUPDATE_TRACE_H
	10	+
	11	+#include <linux/sched.h>
	12	+#include <linux/rcupdate.h>
	13	+
	14	+#ifdef CONFIG_DEBUG_LOCK_ALLOC
	15	+
	16	+extern struct lockdep_map rcu_trace_lock_map;
	17	+
	18	+static inline int rcu_read_lock_trace_held(void)
	19	+{
	20	+ return lock_is_held(&rcu_trace_lock_map);
	21	+}
	22	+
	23	+#else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
	24	+
	25	+static inline int rcu_read_lock_trace_held(void)
	26	+{
	27	+ return 1;
	28	+}
	29	+
	30	+#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
	31	+
	32	+#ifdef CONFIG_TASKS_TRACE_RCU
	33	+
	34	+void rcu_read_unlock_trace_special(struct task_struct *t, int nesting);
	35	+
	36	+/**
	37	+ * rcu_read_lock_trace - mark beginning of RCU-trace read-side critical section
	38	+ *
	39	+ * When synchronize_rcu_trace() is invoked by one task, then that task
	40	+ * is guaranteed to block until all other tasks exit their read-side
	41	+ * critical sections. Similarly, if call_rcu_trace() is invoked on one
	42	+ * task while other tasks are within RCU read-side critical sections,
	43	+ * invocation of the corresponding RCU callback is deferred until after
	44	+ * the all the other tasks exit their critical sections.
	45	+ *
	46	+ * For more details, please see the documentation for rcu_read_lock().
	47	+ */
	48	+static inline void rcu_read_lock_trace(void)
	49	+{
	50	+ struct task_struct *t = current;
	51	+
	52	+ WRITE_ONCE(t->trc_reader_nesting, READ_ONCE(t->trc_reader_nesting) + 1);
	53	+ if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) &&
	54	+ t->trc_reader_special.b.need_mb)
	55	+ smp_mb(); // Pairs with update-side barriers
	56	+ rcu_lock_acquire(&rcu_trace_lock_map);
	57	+}
	58	+
	59	+/**
	60	+ * rcu_read_unlock_trace - mark end of RCU-trace read-side critical section
	61	+ *
	62	+ * Pairs with a preceding call to rcu_read_lock_trace(), and nesting is
	63	+ * allowed. Invoking a rcu_read_unlock_trace() when there is no matching
	64	+ * rcu_read_lock_trace() is verboten, and will result in lockdep complaints.
	65	+ *
	66	+ * For more details, please see the documentation for rcu_read_unlock().
	67	+ */
	68	+static inline void rcu_read_unlock_trace(void)
	69	+{
	70	+ int nesting;
	71	+ struct task_struct *t = current;
	72	+
	73	+ rcu_lock_release(&rcu_trace_lock_map);
	74	+ nesting = READ_ONCE(t->trc_reader_nesting) - 1;
	75	+ if (likely(!READ_ONCE(t->trc_reader_special.s)) \|\| nesting) {
	76	+ WRITE_ONCE(t->trc_reader_nesting, nesting);
	77	+ return; // We assume shallow reader nesting.
	78	+ }
	79	+ rcu_read_unlock_trace_special(t, nesting);
	80	+}
	81	+
	82	+void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func);
	83	+void synchronize_rcu_tasks_trace(void);
	84	+void rcu_barrier_tasks_trace(void);
	85	+
	86	+#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
	87	+
	88	+#endif /* __LINUX_RCUPDATE_TRACE_H */

include/linux/rcupdate_wait.h+19−0

@@ -31,4 +31,23 @@ do { \ · 30 unmodified lines
31	31
32	32	#define wait_rcu_gp(...) _wait_rcu_gp(false, __VA_ARGS__)
33	33
	34	+/**
	35	+ * synchronize_rcu_mult - Wait concurrently for multiple grace periods
	36	+ * @...: List of call_rcu() functions for different grace periods to wait on
	37	+ *
	38	+ * This macro waits concurrently for multiple types of RCU grace periods.
	39	+ * For example, synchronize_rcu_mult(call_rcu, call_rcu_tasks) would wait
	40	+ * on concurrent RCU and RCU-tasks grace periods. Waiting on a given SRCU
	41	+ * domain requires you to write a wrapper function for that SRCU domain's
	42	+ * call_srcu() function, with this wrapper supplying the pointer to the
	43	+ * corresponding srcu_struct.
	44	+ *
	45	+ * The first argument tells Tiny RCU's _wait_rcu_gp() not to
	46	+ * bother waiting for RCU. The reason for this is because anywhere
	47	+ * synchronize_rcu_mult() can be called is automatically already a full
	48	+ * grace period.
	49	+ */
	50	+#define synchronize_rcu_mult(...) \
	51	+ _wait_rcu_gp(IS_ENABLED(CONFIG_TINY_RCU), __VA_ARGS__)
	52	+
34	53	#endif /* _LINUX_SCHED_RCUPDATE_WAIT_H */

include/linux/rcutiny.h+2−1

@@ -49,7 +49,7 @@ static inline void rcu_softirq_qs(void) · 48 unmodified lines
49	49	#define rcu_note_context_switch(preempt) \
50	50	do { \
51	51	rcu_qs(); \
52		− rcu_tasks_qs(current); \
	52	+ rcu_tasks_qs(current, (preempt)); \
53	53	} while (0)
54	54
55	55	static inline int rcu_needs_cpu(u64 basemono, u64 *nextevt)
@@ -87,6 +87,7 @@ static inline bool rcu_inkernel_boot_has_ended(void) { return true; } · 31 unmodified lines
87	87	static inline bool rcu_is_watching(void) { return true; }
88	88	static inline void rcu_momentary_dyntick_idle(void) { }
89	89	static inline void kfree_rcu_scheduler_running(void) { }
	90	+static inline bool rcu_gp_might_be_stalled(void) { return false; }
90	91
91	92	/* Avoid RCU read-side critical sections leaking across. */
92	93	static inline void rcu_all_qs(void) { barrier(); }

include/linux/rcutree.h+1−0

@@ -39,6 +39,7 @@ void rcu_barrier(void); · 38 unmodified lines
39	39	bool rcu_eqs_special_set(int cpu);
40	40	void rcu_momentary_dyntick_idle(void);
41	41	void kfree_rcu_scheduler_running(void);
	42	+bool rcu_gp_might_be_stalled(void);
42	43	unsigned long get_state_synchronize_rcu(void);
43	44	void cond_synchronize_rcu(unsigned long oldstate);
44	45

include/linux/sched.h+9−1

@@ -613,7 +613,7 @@ union rcu_special { · 612 unmodified lines
613	613	u8 blocked;
614	614	u8 need_qs;
615	615	u8 exp_hint; /* Hint for performance. */
616		− u8 deferred_qs;
	616	+ u8 need_mb; /* Readers need smp_mb(). */
617	617	} b; /* Bits. */
618	618	u32 s; /* Set of bits. */
619	619	};
@@ -724,6 +724,14 @@ struct task_struct { · 104 unmodified lines
724	724	struct list_head rcu_tasks_holdout_list;
725	725	#endif /* #ifdef CONFIG_TASKS_RCU */
726	726
	727	+#ifdef CONFIG_TASKS_TRACE_RCU
	728	+ int trc_reader_nesting;
	729	+ int trc_ipi_to_cpu;
	730	+ union rcu_special trc_reader_special;
	731	+ bool trc_reader_checked;
	732	+ struct list_head trc_holdout_list;
	733	+#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
	734	+
727	735	struct sched_info sched_info;
728	736
729	737	struct list_head tasks;

include/linux/torture.h+1−1

@@ -89,7 +89,7 @@ void _torture_stop_kthread(char m, struct task_struct *tp); · 88 unmodified lines
89	89	#ifdef CONFIG_PREEMPTION
90	90	#define torture_preempt_schedule() preempt_schedule()
91	91	#else
92		−#define torture_preempt_schedule()
	92	+#define torture_preempt_schedule() do { } while (0)
93	93	#endif
94	94
95	95	#endif /* __LINUX_TORTURE_H */

include/linux/wait.h+2−0

@@ -1149,4 +1149,6 @@ int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, i · 1148 unmodified lines
1149	1149	(wait)->flags = 0; \
1150	1150	} while (0)
1151	1151
	1152	+bool try_invoke_on_locked_down_task(struct task_struct p, bool (func)(struct task_struct t, void arg), void *arg);
	1153	+
1152	1154	#endif /* _LINUX_WAIT_H */

Merge branches 'fixes.2020.04.27a', 'kfree_rcu.2020.04.27a', 'rcu-tasks.2020.04.27a', 'stall.2020.04.27a' and 'torture.2020.05.07a' into HEAD

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code