Merge branch 'for-mingo' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu into core/rcu

Pull RCU updates from Paul E. McKenney: - Expedited grace-period updates - kfree_rcu() updates - RCU list updates - Preemptible RCU updates - Torture-test updates - Miscellaneous fixes - Documentation updates

Ingo Molnar committed 6 years ago

Commit f8a4bb62 parents 4703d91 0e2473846 files · +1476 −851Browse files

Showing the changes to include/linux (9 of 46 files). Show all

Unified · Split

9 files

include/linux/list.h +112 −24
include/linux/list_nulls.h +26 −4
include/linux/rcu_segcblist.h +0 −2
include/linux/rculist.h +24 −14
include/linux/rculist_nulls.h +14 −6
include/linux/rcupdate.h +8 −20
include/linux/rcutiny.h +1 −0
include/linux/rcutree.h +1 −0
include/linux/tick.h +4 −1

include/linux/list.h+112−24

@@ -23,6 +23,13 @@ · 22 unmodified lines
23	23	#define LIST_HEAD(name) \
24	24	struct list_head name = LIST_HEAD_INIT(name)
25	25
	26	+/**
	27	+ * INIT_LIST_HEAD - Initialize a list_head structure
	28	+ * @list: list_head structure to be initialized.
	29	+ *
	30	+ * Initializes the list_head to point to itself. If it is a list header,
	31	+ * the result is an empty list.
	32	+ */
26	33	static inline void INIT_LIST_HEAD(struct list_head *list)
27	34	{
28	35	WRITE_ONCE(list->next, list);
@@ -120,12 +127,6 @@ static inline void __list_del_clearprev(struct list_head *entry) · 91 unmodified lines
120	127	entry->prev = NULL;
121	128	}
122	129
123		−/**
124		− * list_del - deletes entry from list.
125		− * @entry: the element to delete from the list.
126		− * Note: list_empty() on entry does not return true after this, the entry is
127		− * in an undefined state.
128		− */
129	130	static inline void __list_del_entry(struct list_head *entry)
130	131	{
131	132	if (!__list_del_entry_valid(entry))
@@ -134,6 +135,12 @@ static inline void __list_del_entry(struct list_head *entry) · 2 unmodified lines
134	135	__list_del(entry->prev, entry->next);
135	136	}
136	137
	138	+/**
	139	+ * list_del - deletes entry from list.
	140	+ * @entry: the element to delete from the list.
	141	+ * Note: list_empty() on entry does not return true after this, the entry is
	142	+ * in an undefined state.
	143	+ */
137	144	static inline void list_del(struct list_head *entry)
138	145	{
139	146	__list_del_entry(entry);
@@ -157,8 +164,15 @@ static inline void list_replace(struct list_head *old, · 17 unmodified lines
157	164	new->prev->next = new;
158	165	}
159	166
	167	+/**
	168	+ * list_replace_init - replace old entry by new one and initialize the old one
	169	+ * @old : the element to be replaced
	170	+ * @new : the new element to insert
	171	+ *
	172	+ * If @old was empty, it will be overwritten.
	173	+ */
160	174	static inline void list_replace_init(struct list_head *old,
161		− struct list_head *new)
	175	+ struct list_head *new)
162	176	{
163	177	list_replace(old, new);
164	178	INIT_LIST_HEAD(old);
@@ -744,11 +758,36 @@ static inline void INIT_HLIST_NODE(struct hlist_node *h) · 579 unmodified lines
744	758	h->pprev = NULL;
745	759	}
746	760
	761	+/**
	762	+ * hlist_unhashed - Has node been removed from list and reinitialized?
	763	+ * @h: Node to be checked
	764	+ *
	765	+ * Not that not all removal functions will leave a node in unhashed
	766	+ * state. For example, hlist_nulls_del_init_rcu() does leave the
	767	+ * node in unhashed state, but hlist_nulls_del() does not.
	768	+ */
747	769	static inline int hlist_unhashed(const struct hlist_node *h)
748	770	{
749	771	return !h->pprev;
750	772	}
751	773
	774	+/**
	775	+ * hlist_unhashed_lockless - Version of hlist_unhashed for lockless use
	776	+ * @h: Node to be checked
	777	+ *
	778	+ * This variant of hlist_unhashed() must be used in lockless contexts
	779	+ * to avoid potential load-tearing. The READ_ONCE() is paired with the
	780	+ * various WRITE_ONCE() in hlist helpers that are defined below.
	781	+ */
	782	+static inline int hlist_unhashed_lockless(const struct hlist_node *h)
	783	+{
	784	+ return !READ_ONCE(h->pprev);
	785	+}
	786	+
	787	+/**
	788	+ * hlist_empty - Is the specified hlist_head structure an empty hlist?
	789	+ * @h: Structure to check.
	790	+ */
752	791	static inline int hlist_empty(const struct hlist_head *h)
753	792	{
754	793	return !READ_ONCE(h->first);
@@ -761,9 +800,16 @@ static inline void __hlist_del(struct hlist_node *n) · 6 unmodified lines
761	800
762	801	WRITE_ONCE(*pprev, next);
763	802	if (next)
764		− next->pprev = pprev;
	803	+ WRITE_ONCE(next->pprev, pprev);
765	804	}
766	805
	806	+/**
	807	+ * hlist_del - Delete the specified hlist_node from its list
	808	+ * @n: Node to delete.
	809	+ *
	810	+ * Note that this function leaves the node in hashed state. Use
	811	+ * hlist_del_init() or similar instead to unhash @n.
	812	+ */
767	813	static inline void hlist_del(struct hlist_node *n)
768	814	{
769	815	__hlist_del(n);
@@ -771,6 +817,12 @@ static inline void hlist_del(struct hlist_node *n) · 1 unmodified line
771	817	n->pprev = LIST_POISON2;
772	818	}
773	819
	820	+/**
	821	+ * hlist_del_init - Delete the specified hlist_node from its list and initialize
	822	+ * @n: Node to delete.
	823	+ *
	824	+ * Note that this function leaves the node in unhashed state.
	825	+ */
774	826	static inline void hlist_del_init(struct hlist_node *n)
775	827	{
776	828	if (!hlist_unhashed(n)) {
@@ -779,51 +831,83 @@ static inline void hlist_del_init(struct hlist_node *n) · 2 unmodified lines
779	831	}
780	832	}
781	833
	834	+/**
	835	+ * hlist_add_head - add a new entry at the beginning of the hlist
	836	+ * @n: new entry to be added
	837	+ * @h: hlist head to add it after
	838	+ *
	839	+ * Insert a new entry after the specified head.
	840	+ * This is good for implementing stacks.
	841	+ */
782	842	static inline void hlist_add_head(struct hlist_node n, struct hlist_head h)
783	843	{
784	844	struct hlist_node *first = h->first;
785		− n->next = first;
	845	+ WRITE_ONCE(n->next, first);
786	846	if (first)
787		− first->pprev = &n->next;
	847	+ WRITE_ONCE(first->pprev, &n->next);
788	848	WRITE_ONCE(h->first, n);
789		− n->pprev = &h->first;
	849	+ WRITE_ONCE(n->pprev, &h->first);
790	850	}
791	851
792		−/* next must be != NULL */
	852	+/**
	853	+ * hlist_add_before - add a new entry before the one specified
	854	+ * @n: new entry to be added
	855	+ * @next: hlist node to add it before, which must be non-NULL
	856	+ */
793	857	static inline void hlist_add_before(struct hlist_node *n,
794		− struct hlist_node *next)
	858	+ struct hlist_node *next)
795	859	{
796		− n->pprev = next->pprev;
797		− n->next = next;
798		− next->pprev = &n->next;
	860	+ WRITE_ONCE(n->pprev, next->pprev);
	861	+ WRITE_ONCE(n->next, next);
	862	+ WRITE_ONCE(next->pprev, &n->next);
799	863	WRITE_ONCE(*(n->pprev), n);
800	864	}
801	865
	866	+/**
	867	+ * hlist_add_behing - add a new entry after the one specified
	868	+ * @n: new entry to be added
	869	+ * @prev: hlist node to add it after, which must be non-NULL
	870	+ */
802	871	static inline void hlist_add_behind(struct hlist_node *n,
803	872	struct hlist_node *prev)
804	873	{
805		− n->next = prev->next;
806		− prev->next = n;
807		− n->pprev = &prev->next;
	874	+ WRITE_ONCE(n->next, prev->next);
	875	+ WRITE_ONCE(prev->next, n);
	876	+ WRITE_ONCE(n->pprev, &prev->next);
808	877
809	878	if (n->next)
810		− n->next->pprev = &n->next;
	879	+ WRITE_ONCE(n->next->pprev, &n->next);
811	880	}
812	881
813		−/* after that we'll appear to be on some hlist and hlist_del will work */
	882	+/**
	883	+ * hlist_add_fake - create a fake hlist consisting of a single headless node
	884	+ * @n: Node to make a fake list out of
	885	+ *
	886	+ * This makes @n appear to be its own predecessor on a headless hlist.
	887	+ * The point of this is to allow things like hlist_del() to work correctly
	888	+ * in cases where there is no list.
	889	+ */
814	890	static inline void hlist_add_fake(struct hlist_node *n)
815	891	{
816	892	n->pprev = &n->next;
817	893	}
818	894
	895	+/**
	896	+ * hlist_fake: Is this node a fake hlist?
	897	+ * @h: Node to check for being a self-referential fake hlist.
	898	+ */
819	899	static inline bool hlist_fake(struct hlist_node *h)
820	900	{
821	901	return h->pprev == &h->next;
822	902	}
823	903
824		−/*
	904	+/**
	905	+ * hlist_is_singular_node - is node the only element of the specified hlist?
	906	+ * @n: Node to check for singularity.
	907	+ * @h: Header for potentially singular list.
	908	+ *
825	909	* Check whether the node is the only node of the head without
826		− * accessing head:
	910	+ * accessing head, thus avoiding unnecessary cache misses.
827	911	*/
828	912	static inline bool
829	913	hlist_is_singular_node(struct hlist_node n, struct hlist_head h)
@@ -831,7 +915,11 @@ hlist_is_singular_node(struct hlist_node n, struct hlist_head h) · 1 unmodified line
831	915	return !n->next && n->pprev == &h->first;
832	916	}
833	917
834		−/*
	918	+/**
	919	+ * hlist_move_list - Move an hlist
	920	+ * @old: hlist_head for old list.
	921	+ * @new: hlist_head for new list.
	922	+ *
835	923	* Move a list from one list head to another. Fixup the pprev
836	924	* reference of the first entry if it exists.
837	925	*/

include/linux/list_nulls.h+26−4

@@ -56,11 +56,33 @@ static inline unsigned long get_nulls_value(const struct hlist_nulls_node *ptr) · 55 unmodified lines
56	56	return ((unsigned long)ptr) >> 1;
57	57	}
58	58
	59	+/**
	60	+ * hlist_nulls_unhashed - Has node been removed and reinitialized?
	61	+ * @h: Node to be checked
	62	+ *
	63	+ * Not that not all removal functions will leave a node in unhashed state.
	64	+ * For example, hlist_del_init_rcu() leaves the node in unhashed state,
	65	+ * but hlist_nulls_del() does not.
	66	+ */
59	67	static inline int hlist_nulls_unhashed(const struct hlist_nulls_node *h)
60	68	{
61	69	return !h->pprev;
62	70	}
63	71
	72	+/**
	73	+ * hlist_nulls_unhashed_lockless - Has node been removed and reinitialized?
	74	+ * @h: Node to be checked
	75	+ *
	76	+ * Not that not all removal functions will leave a node in unhashed state.
	77	+ * For example, hlist_del_init_rcu() leaves the node in unhashed state,
	78	+ * but hlist_nulls_del() does not. Unlike hlist_nulls_unhashed(), this
	79	+ * function may be used locklessly.
	80	+ */
	81	+static inline int hlist_nulls_unhashed_lockless(const struct hlist_nulls_node *h)
	82	+{
	83	+ return !READ_ONCE(h->pprev);
	84	+}
	85	+
64	86	static inline int hlist_nulls_empty(const struct hlist_nulls_head *h)
65	87	{
66	88	return is_a_nulls(READ_ONCE(h->first));
@@ -72,10 +94,10 @@ static inline void hlist_nulls_add_head(struct hlist_nulls_node *n, · 5 unmodified lines
72	94	struct hlist_nulls_node *first = h->first;
73	95
74	96	n->next = first;
75		− n->pprev = &h->first;
	97	+ WRITE_ONCE(n->pprev, &h->first);
76	98	h->first = n;
77	99	if (!is_a_nulls(first))
78		− first->pprev = &n->next;
	100	+ WRITE_ONCE(first->pprev, &n->next);
79	101	}
80	102
81	103	static inline void __hlist_nulls_del(struct hlist_nulls_node *n)
@@ -85,13 +107,13 @@ static inline void __hlist_nulls_del(struct hlist_nulls_node *n) · 3 unmodified lines
85	107
86	108	WRITE_ONCE(*pprev, next);
87	109	if (!is_a_nulls(next))
88		− next->pprev = pprev;
	110	+ WRITE_ONCE(next->pprev, pprev);
89	111	}
90	112
91	113	static inline void hlist_nulls_del(struct hlist_nulls_node *n)
92	114	{
93	115	__hlist_nulls_del(n);
94		− n->pprev = LIST_POISON2;
	116	+ WRITE_ONCE(n->pprev, LIST_POISON2);
95	117	}
96	118
97	119	/**

include/linux/rcu_segcblist.h+0−2

@@ -22,7 +22,6 @@ struct rcu_cblist { · 21 unmodified lines
22	22	struct rcu_head *head;
23	23	struct rcu_head **tail;
24	24	long len;
25		− long len_lazy;
26	25	};
27	26
28	27	#define RCU_CBLIST_INITIALIZER(n) { .head = NULL, .tail = &n.head }
@@ -73,7 +72,6 @@ struct rcu_segcblist { · 44 unmodified lines
73	72	#else
74	73	long len;
75	74	#endif
76		− long len_lazy;
77	75	u8 enabled;
78	76	u8 offloaded;
79	77	};

include/linux/rculist.h+24−14

@@ -40,6 +40,16 @@ static inline void INIT_LIST_HEAD_RCU(struct list_head *list) · 39 unmodified lines
40	40	*/
41	41	#define list_next_rcu(list) (((struct list_head __rcu *)(&(list)->next)))
42	42
	43	+/**
	44	+ * list_tail_rcu - returns the prev pointer of the head of the list
	45	+ * @head: the head of the list
	46	+ *
	47	+ * Note: This should only be used with the list header, and even then
	48	+ * only if list_del() and similar primitives are not also used on the
	49	+ * list header.
	50	+ */
	51	+#define list_tail_rcu(head) (((struct list_head __rcu *)(&(head)->prev)))
	52	+
43	53	/*
44	54	* Check during list traversal that we are within an RCU reader
45	55	*/
@@ -173,7 +183,7 @@ static inline void hlist_del_init_rcu(struct hlist_node *n) · 127 unmodified lines
173	183	{
174	184	if (!hlist_unhashed(n)) {
175	185	__hlist_del(n);
176		− n->pprev = NULL;
	186	+ WRITE_ONCE(n->pprev, NULL);
177	187	}
178	188	}
179	189
@@ -361,7 +371,7 @@ static inline void list_splice_tail_init_rcu(struct list_head *list, · 181 unmodified lines
361	371	* @pos: the type * to use as a loop cursor.
362	372	* @head: the head for your list.
363	373	* @member: the name of the list_head within the struct.
364		− * @cond: optional lockdep expression if called from non-RCU protection.
	374	+ * @cond...: optional lockdep expression if called from non-RCU protection.
365	375	*
366	376	* This list-traversal primitive may safely run concurrently with
367	377	* the _rcu list-mutation primitives such as list_add_rcu()
@@ -473,7 +483,7 @@ static inline void list_splice_tail_init_rcu(struct list_head *list, · 105 unmodified lines
473	483	static inline void hlist_del_rcu(struct hlist_node *n)
474	484	{
475	485	__hlist_del(n);
476		− n->pprev = LIST_POISON2;
	486	+ WRITE_ONCE(n->pprev, LIST_POISON2);
477	487	}
478	488
479	489	/**
@@ -489,11 +499,11 @@ static inline void hlist_replace_rcu(struct hlist_node *old, · 9 unmodified lines
489	499	struct hlist_node *next = old->next;
490	500
491	501	new->next = next;
492		− new->pprev = old->pprev;
	502	+ WRITE_ONCE(new->pprev, old->pprev);
493	503	rcu_assign_pointer((struct hlist_node __rcu *)new->pprev, new);
494	504	if (next)
495		− new->next->pprev = &new->next;
496		− old->pprev = LIST_POISON2;
	505	+ WRITE_ONCE(new->next->pprev, &new->next);
	506	+ WRITE_ONCE(old->pprev, LIST_POISON2);
497	507	}
498	508
499	509	/*
@@ -528,10 +538,10 @@ static inline void hlist_add_head_rcu(struct hlist_node *n, · 28 unmodified lines
528	538	struct hlist_node *first = h->first;
529	539
530	540	n->next = first;
531		− n->pprev = &h->first;
	541	+ WRITE_ONCE(n->pprev, &h->first);
532	542	rcu_assign_pointer(hlist_first_rcu(h), n);
533	543	if (first)
534		− first->pprev = &n->next;
	544	+ WRITE_ONCE(first->pprev, &n->next);
535	545	}
536	546
537	547	/**
@@ -564,7 +574,7 @@ static inline void hlist_add_tail_rcu(struct hlist_node *n, · 26 unmodified lines
564	574
565	575	if (last) {
566	576	n->next = last->next;
567		− n->pprev = &last->next;
	577	+ WRITE_ONCE(n->pprev, &last->next);
568	578	rcu_assign_pointer(hlist_next_rcu(last), n);
569	579	} else {
570	580	hlist_add_head_rcu(n, h);
@@ -592,10 +602,10 @@ static inline void hlist_add_tail_rcu(struct hlist_node *n, · 21 unmodified lines
592	602	static inline void hlist_add_before_rcu(struct hlist_node *n,
593	603	struct hlist_node *next)
594	604	{
595		− n->pprev = next->pprev;
	605	+ WRITE_ONCE(n->pprev, next->pprev);
596	606	n->next = next;
597	607	rcu_assign_pointer(hlist_pprev_rcu(n), n);
598		− next->pprev = &n->next;
	608	+ WRITE_ONCE(next->pprev, &n->next);
599	609	}
600	610
601	611	/**
@@ -620,10 +630,10 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n, · 18 unmodified lines
620	630	struct hlist_node *prev)
621	631	{
622	632	n->next = prev->next;
623		− n->pprev = &prev->next;
	633	+ WRITE_ONCE(n->pprev, &prev->next);
624	634	rcu_assign_pointer(hlist_next_rcu(prev), n);
625	635	if (n->next)
626		− n->next->pprev = &n->next;
	636	+ WRITE_ONCE(n->next->pprev, &n->next);
627	637	}
628	638
629	639	#define __hlist_for_each_rcu(pos, head) \
@@ -636,7 +646,7 @@ static inline void hlist_add_behind_rcu(struct hlist_node *n, · 6 unmodified lines
636	646	* @pos: the type * to use as a loop cursor.
637	647	* @head: the head for your list.
638	648	* @member: the name of the hlist_node within the struct.
639		− * @cond: optional lockdep expression if called from non-RCU protection.
	649	+ * @cond...: optional lockdep expression if called from non-RCU protection.
640	650	*
641	651	* This list-traversal primitive may safely run concurrently with
642	652	* the _rcu list-mutation primitives such as hlist_add_head_rcu()

include/linux/rculist_nulls.h+14−6

@@ -34,13 +34,21 @@ static inline void hlist_nulls_del_init_rcu(struct hlist_nulls_node *n) · 33 unmodified lines
34	34	{
35	35	if (!hlist_nulls_unhashed(n)) {
36	36	__hlist_nulls_del(n);
37		− n->pprev = NULL;
	37	+ WRITE_ONCE(n->pprev, NULL);
38	38	}
39	39	}
40	40
	41	+/**
	42	+ * hlist_nulls_first_rcu - returns the first element of the hash list.
	43	+ * @head: the head of the list.
	44	+ */
41	45	#define hlist_nulls_first_rcu(head) \
42	46	(((struct hlist_nulls_node __rcu __force *)&(head)->first))
43	47
	48	+/**
	49	+ * hlist_nulls_next_rcu - returns the element of the list after @node.
	50	+ * @node: element of the list.
	51	+ */
44	52	#define hlist_nulls_next_rcu(node) \
45	53	(((struct hlist_nulls_node __rcu __force *)&(node)->next))
46	54
@@ -66,7 +74,7 @@ static inline void hlist_nulls_del_init_rcu(struct hlist_nulls_node *n) · 19 unmodified lines
66	74	static inline void hlist_nulls_del_rcu(struct hlist_nulls_node *n)
67	75	{
68	76	__hlist_nulls_del(n);
69		− n->pprev = LIST_POISON2;
	77	+ WRITE_ONCE(n->pprev, LIST_POISON2);
70	78	}
71	79
72	80	/**
@@ -94,10 +102,10 @@ static inline void hlist_nulls_add_head_rcu(struct hlist_nulls_node *n, · 21 unmodified lines
94	102	struct hlist_nulls_node *first = h->first;
95	103
96	104	n->next = first;
97		− n->pprev = &h->first;
	105	+ WRITE_ONCE(n->pprev, &h->first);
98	106	rcu_assign_pointer(hlist_nulls_first_rcu(h), n);
99	107	if (!is_a_nulls(first))
100		− first->pprev = &n->next;
	108	+ WRITE_ONCE(first->pprev, &n->next);
101	109	}
102	110
103	111	/**
@@ -141,7 +149,7 @@ static inline void hlist_nulls_add_tail_rcu(struct hlist_nulls_node *n, · 37 unmodified lines
141	149	* hlist_nulls_for_each_entry_rcu - iterate over rcu list of given type
142	150	* @tpos: the type * to use as a loop cursor.
143	151	* @pos: the &struct hlist_nulls_node to use as a loop cursor.
144		− * @head: the head for your list.
	152	+ * @head: the head of the list.
145	153	* @member: the name of the hlist_nulls_node within the struct.
146	154	*
147	155	* The barrier() is needed to make sure compiler doesn't cache first element [1],
@@ -161,7 +169,7 @@ static inline void hlist_nulls_add_tail_rcu(struct hlist_nulls_node *n, · 13 unmodified lines
161	169	* iterate over list of given type safe against removal of list entry
162	170	* @tpos: the type * to use as a loop cursor.
163	171	* @pos: the &struct hlist_nulls_node to use as a loop cursor.
164		− * @head: the head for your list.
	172	+ * @head: the head of the list.
165	173	* @member: the name of the hlist_nulls_node within the struct.
166	174	*/
167	175	#define hlist_nulls_for_each_entry_safe(tpos, pos, head, member) \

include/linux/rcupdate.h+8−20

@@ -154,7 +154,7 @@ static inline void exit_tasks_rcu_finish(void) { } · 153 unmodified lines
154	154	*
155	155	* This macro resembles cond_resched(), except that it is defined to
156	156	* report potential quiescent states to RCU-tasks even if the cond_resched()
157		− * machinery were to be shut off, as some advocate for PREEMPT kernels.
	157	+ * machinery were to be shut off, as some advocate for PREEMPTION kernels.
158	158	*/
159	159	#define cond_resched_tasks_rcu_qs() \
160	160	do { \
@@ -167,7 +167,7 @@ do { \ · 6 unmodified lines
167	167	* TREE_RCU and rcu_barrier_() primitives in TINY_RCU.
168	168	*/
169	169
170		−#if defined(CONFIG_TREE_RCU) \|\| defined(CONFIG_PREEMPT_RCU)
	170	+#if defined(CONFIG_TREE_RCU)
171	171	#include <linux/rcutree.h>
172	172	#elif defined(CONFIG_TINY_RCU)
173	173	#include <linux/rcutiny.h>
@@ -400,22 +400,6 @@ do { \ · 226 unmodified lines
400	400	__tmp; \
401	401	})
402	402
403		−/**
404		− * rcu_swap_protected() - swap an RCU and a regular pointer
405		− * @rcu_ptr: RCU pointer
406		− * @ptr: regular pointer
407		− * @c: the conditions under which the dereference will take place
408		− *
409		− * Perform swap(@rcu_ptr, @ptr) where @rcu_ptr is an RCU-annotated pointer and
410		− * @c is the argument that is passed to the rcu_dereference_protected() call
411		− * used to read that pointer.
412		− */
413		−#define rcu_swap_protected(rcu_ptr, ptr, c) do { \
414		− typeof(ptr) __tmp = rcu_dereference_protected((rcu_ptr), (c)); \
415		− rcu_assign_pointer((rcu_ptr), (ptr)); \
416		− (ptr) = __tmp; \
417		−} while (0)
418		−
419	403	/**
420	404	* rcu_access_pointer() - fetch RCU pointer with no dereferencing
421	405	* @p: The pointer to read
@@ -598,10 +582,10 @@ do { \ · 176 unmodified lines
598	582	*
599	583	* You can avoid reading and understanding the next paragraph by
600	584	* following this rule: don't put anything in an rcu_read_lock() RCU
601		− * read-side critical section that would block in a !PREEMPT kernel.
	585	+ * read-side critical section that would block in a !PREEMPTION kernel.
602	586	* But if you want the full story, read on!
603	587	*
604		− * In non-preemptible RCU implementations (TREE_RCU and TINY_RCU),
	588	+ * In non-preemptible RCU implementations (pure TREE_RCU and TINY_RCU),
605	589	* it is illegal to block while in an RCU read-side critical section.
606	590	* In preemptible RCU implementations (PREEMPT_RCU) in CONFIG_PREEMPTION
607	591	* kernel builds, RCU read-side critical sections may be preempted,
@@ -912,4 +896,8 @@ rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f) · 304 unmodified lines
912	896	return false;
913	897	}
914	898
	899	+/* kernel/ksysfs.c definitions */
	900	+extern int rcu_expedited;
	901	+extern int rcu_normal;
	902	+
915	903	#endif /* __LINUX_RCUPDATE_H */

include/linux/rcutiny.h+1−0

@@ -85,6 +85,7 @@ static inline void rcu_scheduler_starting(void) { } · 84 unmodified lines
85	85	static inline void rcu_end_inkernel_boot(void) { }
86	86	static inline bool rcu_is_watching(void) { return true; }
87	87	static inline void rcu_momentary_dyntick_idle(void) { }
	88	+static inline void kfree_rcu_scheduler_running(void) { }
88	89
89	90	/* Avoid RCU read-side critical sections leaking across. */
90	91	static inline void rcu_all_qs(void) { barrier(); }

include/linux/rcutree.h+1−0

@@ -38,6 +38,7 @@ void kfree_call_rcu(struct rcu_head *head, rcu_callback_t func); · 37 unmodified lines
38	38	void rcu_barrier(void);
39	39	bool rcu_eqs_special_set(int cpu);
40	40	void rcu_momentary_dyntick_idle(void);
	41	+void kfree_rcu_scheduler_running(void);
41	42	unsigned long get_state_synchronize_rcu(void);
42	43	void cond_synchronize_rcu(unsigned long oldstate);
43	44

include/linux/tick.h+4−1

@@ -109,8 +109,10 @@ enum tick_dep_bits { · 108 unmodified lines
109	109	TICK_DEP_BIT_PERF_EVENTS = 1,
110	110	TICK_DEP_BIT_SCHED = 2,
111	111	TICK_DEP_BIT_CLOCK_UNSTABLE = 3,
112		− TICK_DEP_BIT_RCU = 4
	112	+ TICK_DEP_BIT_RCU = 4,
	113	+ TICK_DEP_BIT_RCU_EXP = 5
113	114	};
	115	+#define TICK_DEP_BIT_MAX TICK_DEP_BIT_RCU_EXP
114	116
115	117	#define TICK_DEP_MASK_NONE 0
116	118	#define TICK_DEP_MASK_POSIX_TIMER (1 << TICK_DEP_BIT_POSIX_TIMER)
@@ -118,6 +120,7 @@ enum tick_dep_bits { · 1 unmodified line
118	120	#define TICK_DEP_MASK_SCHED (1 << TICK_DEP_BIT_SCHED)
119	121	#define TICK_DEP_MASK_CLOCK_UNSTABLE (1 << TICK_DEP_BIT_CLOCK_UNSTABLE)
120	122	#define TICK_DEP_MASK_RCU (1 << TICK_DEP_BIT_RCU)
	123	+#define TICK_DEP_MASK_RCU_EXP (1 << TICK_DEP_BIT_RCU_EXP)
121	124
122	125	#ifdef CONFIG_NO_HZ_COMMON
123	126	extern bool tick_nohz_enabled;