Merge branch 'rcu/next' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu into core/rcu

Pull the v3.5 RCU tree from Paul E. McKenney: 1) A set of improvements and fixes to the RCU_FAST_NO_HZ feature (with more on the way for 3.6). Posted to LKML: https://lkml.org/lkml/2012/4/23/324 (commits 1-3 and 5), https://lkml.org/lkml/2012/4/16/611 (commit 4), https://lkml.org/lkml/2012/4/30/390 (commit 6), and https://lkml.org/lkml/2012/5/4/410 (commit 7, combined with the other commits for the convenience of the tester). 2) Changes to make rcu_barrier() avoid disrupting execution of CPUs that have no RCU callbacks. Posted to LKML: https://lkml.org/lkml/2012/4/23/322. 3) A couple of commits that improve the efficiency of the interaction between preemptible RCU and the scheduler, these two being all that survived an abortive attempt to allow preemptible RCU's __rcu_read_lock() to be inlined. The full set was posted to LKML at https://lkml.org/lkml/2012/4/14/143, and the first and third patches of that set remain. 4) Lai Jiangshan's algorithmic implementation of SRCU, which includes call_srcu() and srcu_barrier(). A major feature of this new implementation is that synchronize_srcu() no longer disturbs the execution of other CPUs. This work is based on earlier implementations by Peter Zijlstra and Paul E. McKenney. Posted to LKML: https://lkml.org/lkml/2012/2/22/82. 5) A number of miscellaneous bug fixes and improvements which were posted to LKML at: https://lkml.org/lkml/2012/4/23/353 with subsequent updates posted to LKML.

Ingo Molnar committed 14 years ago

Commit 2d84e022 parents 9ff00d5 dc36be423 files · +1358 −353Browse files

Showing the changes to include (7 of 23 files). Show all

Unified · Split

7 files

include/linux/rculist.h +35 −5
include/linux/rcupdate.h +20 −0
include/linux/rcutiny.h +0 −11
include/linux/rcutree.h +0 −19
include/linux/sched.h +10 −0
include/linux/srcu.h +39 −9
include/trace/events/rcu.h +2 −0

include/linux/rculist.h+35−5

@@ -30,6 +30,7 @@ · 29 unmodified lines
30	30	* This is only for internal list manipulation where we know
31	31	* the prev/next entries already!
32	32	*/
	33	+#ifndef CONFIG_DEBUG_LIST
33	34	static inline void __list_add_rcu(struct list_head *new,
34	35	struct list_head prev, struct list_head next)
35	36	{
@@ -38,6 +39,10 @@ static inline void __list_add_rcu(struct list_head *new, · 2 unmodified lines
38	39	rcu_assign_pointer(list_next_rcu(prev), new);
39	40	next->prev = new;
40	41	}
	42	+#else
	43	+extern void __list_add_rcu(struct list_head *new,
	44	+ struct list_head prev, struct list_head next);
	45	+#endif
41	46
42	47	/**
43	48	* list_add_rcu - add a new entry to rcu-protected list
@@ -108,7 +113,7 @@ static inline void list_add_tail_rcu(struct list_head *new, · 64 unmodified lines
108	113	*/
109	114	static inline void list_del_rcu(struct list_head *entry)
110	115	{
111		− __list_del(entry->prev, entry->next);
	116	+ __list_del_entry(entry);
112	117	entry->prev = LIST_POISON2;
113	118	}
114	119
@@ -228,18 +233,43 @@ static inline void list_splice_init_rcu(struct list_head *list, · 113 unmodified lines
228	233	})
229	234
230	235	/**
231		− * list_first_entry_rcu - get the first element from a list
	236	+ * Where are list_empty_rcu() and list_first_entry_rcu()?
	237	+ *
	238	+ * Implementing those functions following their counterparts list_empty() and
	239	+ * list_first_entry() is not advisable because they lead to subtle race
	240	+ * conditions as the following snippet shows:
	241	+ *
	242	+ * if (!list_empty_rcu(mylist)) {
	243	+ * struct foo *bar = list_first_entry_rcu(mylist, struct foo, list_member);
	244	+ * do_something(bar);
	245	+ * }
	246	+ *
	247	+ * The list may not be empty when list_empty_rcu checks it, but it may be when
	248	+ * list_first_entry_rcu rereads the ->next pointer.
	249	+ *
	250	+ * Rereading the ->next pointer is not a problem for list_empty() and
	251	+ * list_first_entry() because they would be protected by a lock that blocks
	252	+ * writers.
	253	+ *
	254	+ * See list_first_or_null_rcu for an alternative.
	255	+ */
	256	+
	257	+/**
	258	+ * list_first_or_null_rcu - get the first element from a list
232	259	* @ptr: the list head to take the element from.
233	260	* @type: the type of the struct this is embedded in.
234	261	* @member: the name of the list_struct within the struct.
235	262	*
236		− * Note, that list is expected to be not empty.
	263	+ * Note that if the list is empty, it returns NULL.
237	264	*
238	265	* This primitive may safely run concurrently with the _rcu list-mutation
239	266	* primitives such as list_add_rcu() as long as it's guarded by rcu_read_lock().
240	267	*/
241		−#define list_first_entry_rcu(ptr, type, member) \
242		− list_entry_rcu((ptr)->next, type, member)
	268	+#define list_first_or_null_rcu(ptr, type, member) \
	269	+ ({struct list_head *__ptr = (ptr); \
	270	+ struct list_head __rcu *__next = list_next_rcu(__ptr); \
	271	+ likely(__ptr != __next) ? container_of(__next, type, member) : NULL; \
	272	+ })
243	273
244	274	/**
245	275	* list_for_each_entry_rcu - iterate over rcu list of given type

include/linux/rcupdate.h+20−0

@@ -184,12 +184,14 @@ static inline int rcu_preempt_depth(void) · 183 unmodified lines
184	184	/* Internal to kernel */
185	185	extern void rcu_sched_qs(int cpu);
186	186	extern void rcu_bh_qs(int cpu);
	187	+extern void rcu_preempt_note_context_switch(void);
187	188	extern void rcu_check_callbacks(int cpu, int user);
188	189	struct notifier_block;
189	190	extern void rcu_idle_enter(void);
190	191	extern void rcu_idle_exit(void);
191	192	extern void rcu_irq_enter(void);
192	193	extern void rcu_irq_exit(void);
	194	+extern void exit_rcu(void);
193	195
194	196	/**
195	197	* RCU_NONIDLE - Indicate idle-loop code that needs RCU readers
@@ -922,6 +924,21 @@ void __kfree_rcu(struct rcu_head *head, unsigned long offset) · 726 unmodified lines
922	924	kfree_call_rcu(head, (rcu_callback)offset);
923	925	}
924	926
	927	+/*
	928	+ * Does the specified offset indicate that the corresponding rcu_head
	929	+ * structure can be handled by kfree_rcu()?
	930	+ */
	931	+#define __is_kfree_rcu_offset(offset) ((offset) < 4096)
	932	+
	933	+/*
	934	+ * Helper macro for kfree_rcu() to prevent argument-expansion eyestrain.
	935	+ */
	936	+#define __kfree_rcu(head, offset) \
	937	+ do { \
	938	+ BUILD_BUG_ON(!__is_kfree_rcu_offset(offset)); \
	939	+ call_rcu(head, (void ()(struct rcu_head ))(unsigned long)(offset)); \
	940	+ } while (0)
	941	+
925	942	/**
926	943	* kfree_rcu() - kfree an object after a grace period.
927	944	* @ptr: pointer to kfree
@@ -944,6 +961,9 @@ void __kfree_rcu(struct rcu_head *head, unsigned long offset) · 16 unmodified lines
944	961	*
945	962	* Note that the allowable offset might decrease in the future, for example,
946	963	* to allow something like kmem_cache_free_rcu().
	964	+ *
	965	+ * The BUILD_BUG_ON check must not involve any function calls, hence the
	966	+ * checks are done in macros here.
947	967	*/
948	968	#define kfree_rcu(ptr, rcu_head) \
949	969	__kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))

include/linux/rcutiny.h+0−11

@@ -87,14 +87,6 @@ static inline void kfree_call_rcu(struct rcu_head *head, · 86 unmodified lines
87	87
88	88	#ifdef CONFIG_TINY_RCU
89	89
90		−static inline void rcu_preempt_note_context_switch(void)
91		−{
92		−}
93		−
94		−static inline void exit_rcu(void)
95		−{
96		−}
97		−
98	90	static inline int rcu_needs_cpu(int cpu)
99	91	{
100	92	return 0;
@@ -102,8 +94,6 @@ static inline int rcu_needs_cpu(int cpu) · 1 unmodified line
102	94
103	95	#else /* #ifdef CONFIG_TINY_RCU */
104	96
105		−void rcu_preempt_note_context_switch(void);
106		−extern void exit_rcu(void);
107	97	int rcu_preempt_needs_cpu(void);
108	98
109	99	static inline int rcu_needs_cpu(int cpu)
@@ -116,7 +106,6 @@ static inline int rcu_needs_cpu(int cpu) · 6 unmodified lines
116	106	static inline void rcu_note_context_switch(int cpu)
117	107	{
118	108	rcu_sched_qs(cpu);
119		− rcu_preempt_note_context_switch();
120	109	}
121	110
122	111	/*

include/linux/rcutree.h+0−19

@@ -45,18 +45,6 @@ static inline void rcu_virt_note_context_switch(int cpu) · 44 unmodified lines
45	45	rcu_note_context_switch(cpu);
46	46	}
47	47
48		−#ifdef CONFIG_TREE_PREEMPT_RCU
49		−
50		−extern void exit_rcu(void);
51		−
52		−#else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
53		−
54		−static inline void exit_rcu(void)
55		−{
56		−}
57		−
58		−#endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
59		−
60	48	extern void synchronize_rcu_bh(void);
61	49	extern void synchronize_sched_expedited(void);
62	50	extern void synchronize_rcu_expedited(void);
@@ -98,13 +86,6 @@ extern void rcu_force_quiescent_state(void); · 35 unmodified lines
98	86	extern void rcu_bh_force_quiescent_state(void);
99	87	extern void rcu_sched_force_quiescent_state(void);
100	88
101		−/* A context switch is a grace period for RCU-sched and RCU-bh. */
102		−static inline int rcu_blocking_is_gp(void)
103		−{
104		− might_sleep(); /* Check for RCU read-side critical section. */
105		− return num_online_cpus() == 1;
106		−}
107		−
108	89	extern void rcu_scheduler_starting(void);
109	90	extern int rcu_scheduler_active __read_mostly;
110	91

include/linux/sched.h+10−0

@@ -1905,12 +1905,22 @@ static inline void rcu_copy_process(struct task_struct *p) · 1904 unmodified lines
1905	1905	INIT_LIST_HEAD(&p->rcu_node_entry);
1906	1906	}
1907	1907
	1908	+static inline void rcu_switch_from(struct task_struct *prev)
	1909	+{
	1910	+ if (prev->rcu_read_lock_nesting != 0)
	1911	+ rcu_preempt_note_context_switch();
	1912	+}
	1913	+
1908	1914	#else
1909	1915
1910	1916	static inline void rcu_copy_process(struct task_struct *p)
1911	1917	{
1912	1918	}
1913	1919
	1920	+static inline void rcu_switch_from(struct task_struct *prev)
	1921	+{
	1922	+}
	1923	+
1914	1924	#endif
1915	1925
1916	1926	#ifdef CONFIG_SMP

include/linux/srcu.h+39−9

@@ -29,26 +29,35 @@ · 28 unmodified lines
29	29
30	30	#include <linux/mutex.h>
31	31	#include <linux/rcupdate.h>
	32	+#include <linux/workqueue.h>
32	33
33	34	struct srcu_struct_array {
34		− int c[2];
	35	+ unsigned long c[2];
	36	+ unsigned long seq[2];
	37	+};
	38	+
	39	+struct rcu_batch {
	40	+ struct rcu_head head, *tail;
35	41	};
36	42
37	43	struct srcu_struct {
38		− int completed;
	44	+ unsigned completed;
39	45	struct srcu_struct_array __percpu *per_cpu_ref;
40		− struct mutex mutex;
	46	+ spinlock_t queue_lock; /* protect ->batch_queue, ->running */
	47	+ bool running;
	48	+ /* callbacks just queued */
	49	+ struct rcu_batch batch_queue;
	50	+ /* callbacks try to do the first check_zero */
	51	+ struct rcu_batch batch_check0;
	52	+ /* callbacks done with the first check_zero and the flip */
	53	+ struct rcu_batch batch_check1;
	54	+ struct rcu_batch batch_done;
	55	+ struct delayed_work work;
41	56	#ifdef CONFIG_DEBUG_LOCK_ALLOC
42	57	struct lockdep_map dep_map;
43	58	#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
44	59	};
45	60
46		−#ifndef CONFIG_PREEMPT
47		−#define srcu_barrier() barrier()
48		−#else /* #ifndef CONFIG_PREEMPT */
49		−#define srcu_barrier()
50		−#endif /* #else #ifndef CONFIG_PREEMPT */
51		−
52	61	#ifdef CONFIG_DEBUG_LOCK_ALLOC
53	62
54	63	int __init_srcu_struct(struct srcu_struct sp, const char name,
@@ -67,12 +76,33 @@ int init_srcu_struct(struct srcu_struct *sp); · 12 unmodified lines
67	76
68	77	#endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */
69	78
	79	+/**
	80	+ * call_srcu() - Queue a callback for invocation after an SRCU grace period
	81	+ * @sp: srcu_struct in queue the callback
	82	+ * @head: structure to be used for queueing the SRCU callback.
	83	+ * @func: function to be invoked after the SRCU grace period
	84	+ *
	85	+ * The callback function will be invoked some time after a full SRCU
	86	+ * grace period elapses, in other words after all pre-existing SRCU
	87	+ * read-side critical sections have completed. However, the callback
	88	+ * function might well execute concurrently with other SRCU read-side
	89	+ * critical sections that started after call_srcu() was invoked. SRCU
	90	+ * read-side critical sections are delimited by srcu_read_lock() and
	91	+ * srcu_read_unlock(), and may be nested.
	92	+ *
	93	+ * The callback will be invoked from process context, but must nevertheless
	94	+ * be fast and must not block.
	95	+ */
	96	+void call_srcu(struct srcu_struct sp, struct rcu_head head,
	97	+ void (func)(struct rcu_head head));
	98	+
70	99	void cleanup_srcu_struct(struct srcu_struct *sp);
71	100	int __srcu_read_lock(struct srcu_struct *sp) __acquires(sp);
72	101	void __srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp);
73	102	void synchronize_srcu(struct srcu_struct *sp);
74	103	void synchronize_srcu_expedited(struct srcu_struct *sp);
75	104	long srcu_batches_completed(struct srcu_struct *sp);
	105	+void srcu_barrier(struct srcu_struct *sp);
76	106
77	107	#ifdef CONFIG_DEBUG_LOCK_ALLOC
78	108

include/trace/events/rcu.h+2−0

@@ -292,6 +292,8 @@ TRACE_EVENT(rcu_dyntick, · 291 unmodified lines
292	292	* "More callbacks": Still more callbacks, try again to clear them out.
293	293	* "Callbacks drained": All callbacks processed, off to dyntick idle!
294	294	* "Timer": Timer fired to cause CPU to continue processing callbacks.
	295	+ * "Demigrate": Timer fired on wrong CPU, woke up correct CPU.
	296	+ * "Cleanup after idle": Idle exited, timer canceled.
295	297	*/
296	298	TRACE_EVENT(rcu_prep_idle,
297	299