History

Merge branches 'doc.2021.11.30c', 'exp.2021.12.07a', 'fastnohz.2021.11.30c', 'fixes.2021.11.30c', 'nocb.2021.12.09a', 'nolibc.2021.11.30c', 'tasks.2021.12.09a', 'torture.2021.12.07a' and 'torturescript.2021.11.30c' into HEAD

doc.2021.11.30c: Documentation updates. exp.2021.12.07a: Expedited-grace-period fixes. fastnohz.2021.11.30c: Remove CONFIG_RCU_FAST_NO_HZ. fixes.2021.11.30c: Miscellaneous fixes. nocb.2021.12.09a: No-CB CPU updates. nolibc.2021.11.30c: Tiny in-kernel library updates. tasks.2021.12.09a: RCU-tasks updates, including update-side scalability. torture.2021.12.07a: Torture-test in-kernel module updates. torturescript.2021.11.30c: Torture-test scripting updates.

Paul E. McKenney committed 4 years ago

Commit f80fe669 parents 5861dad 81f6d49 bc849e9 1f8da40 10d4703 b0fe9de fd796e4 53b541f 90b21bc51 files · +1089 −719Browse files

Showing the changes to include (5 of 51 files). Show all

Unified · Split

5 files
  1. include/linux/rcu_segcblist.h +37 −14
  2. include/linux/rcupdate.h +28 −22
  3. include/linux/rcutiny.h +1 −1
  4. include/linux/srcu.h +2 −1
  5. include/linux/torture.h +2 −7
include/linux/rcu_segcblist.h+37−14
@@ -69,7 +69,7 @@ struct rcu_cblist { · 68 unmodified lines
6969 *
7070 *
7171 * ----------------------------------------------------------------------------
72− * | SEGCBLIST_SOFTIRQ_ONLY |
72+ * | SEGCBLIST_RCU_CORE |
7373 * | |
7474 * | Callbacks processed by rcu_core() from softirqs or local |
7575 * | rcuc kthread, without holding nocb_lock. |
@@ -77,7 +77,7 @@ struct rcu_cblist { · 1 unmodified line
7777 * |
7878 * v
7979 * ----------------------------------------------------------------------------
80− * | SEGCBLIST_OFFLOADED |
80+ * | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED |
8181 * | |
8282 * | Callbacks processed by rcu_core() from softirqs or local |
8383 * | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads, |
@@ -89,7 +89,9 @@ struct rcu_cblist { · 5 unmodified lines
8989 * | |
9090 * v v
9191 * --------------------------------------- ----------------------------------|
92− * | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | |
92+ * | SEGCBLIST_RCU_CORE | | | SEGCBLIST_RCU_CORE | |
93+ * | SEGCBLIST_LOCKING | | | SEGCBLIST_LOCKING | |
94+ * | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | |
9395 * | SEGCBLIST_KTHREAD_CB | | SEGCBLIST_KTHREAD_GP |
9496 * | | | |
9597 * | | | |
@@ -104,9 +106,10 @@ struct rcu_cblist { · 8 unmodified lines
104106 * |
105107 * v
106108 * |--------------------------------------------------------------------------|
107− * | SEGCBLIST_OFFLOADED | |
108− * | SEGCBLIST_KTHREAD_CB | |
109− * | SEGCBLIST_KTHREAD_GP |
109+ * | SEGCBLIST_LOCKING | |
110+ * | SEGCBLIST_OFFLOADED | |
111+ * | SEGCBLIST_KTHREAD_GP | |
112+ * | SEGCBLIST_KTHREAD_CB |
110113 * | |
111114 * | Kthreads handle callbacks holding nocb_lock, local rcu_core() stops |
112115 * | handling callbacks. Enable bypass queueing. |
@@ -120,7 +123,8 @@ struct rcu_cblist { · 7 unmodified lines
120123 *
121124 *
122125 * |--------------------------------------------------------------------------|
123− * | SEGCBLIST_OFFLOADED | |
126+ * | SEGCBLIST_LOCKING | |
127+ * | SEGCBLIST_OFFLOADED | |
124128 * | SEGCBLIST_KTHREAD_CB | |
125129 * | SEGCBLIST_KTHREAD_GP |
126130 * | |
@@ -130,6 +134,22 @@ struct rcu_cblist { · 3 unmodified lines
130134 * |
131135 * v
132136 * |--------------------------------------------------------------------------|
137+ * | SEGCBLIST_RCU_CORE | |
138+ * | SEGCBLIST_LOCKING | |
139+ * | SEGCBLIST_OFFLOADED | |
140+ * | SEGCBLIST_KTHREAD_CB | |
141+ * | SEGCBLIST_KTHREAD_GP |
142+ * | |
143+ * | CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core() |
144+ * | handles callbacks concurrently. Bypass enqueue is enabled. |
145+ * | Invoke RCU core so we make sure not to preempt it in the middle with |
146+ * | leaving some urgent work unattended within a jiffy. |
147+ * ----------------------------------------------------------------------------
148+ * |
149+ * v
150+ * |--------------------------------------------------------------------------|
151+ * | SEGCBLIST_RCU_CORE | |
152+ * | SEGCBLIST_LOCKING | |
133153 * | SEGCBLIST_KTHREAD_CB | |
134154 * | SEGCBLIST_KTHREAD_GP |
135155 * | |
@@ -143,7 +163,9 @@ struct rcu_cblist { · 7 unmodified lines
143163 * | |
144164 * v v
145165 * ---------------------------------------------------------------------------|
146− * | |
166+ * | | |
167+ * | SEGCBLIST_RCU_CORE | | SEGCBLIST_RCU_CORE | |
168+ * | SEGCBLIST_LOCKING | | SEGCBLIST_LOCKING | |
147169 * | SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP |
148170 * | | |
149171 * | GP kthread woke up and | CB kthread woke up and |
@@ -159,7 +181,7 @@ struct rcu_cblist { · 9 unmodified lines
159181 * |
160182 * v
161183 * ----------------------------------------------------------------------------
162− * | 0 |
184+ * | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING |
163185 * | |
164186 * | Callbacks processed by rcu_core() from softirqs or local |
165187 * | rcuc kthread, while holding nocb_lock. Forbid nocb_timer to be armed. |
@@ -168,17 +190,18 @@ struct rcu_cblist { · 2 unmodified lines
168190 * |
169191 * v
170192 * ----------------------------------------------------------------------------
171− * | SEGCBLIST_SOFTIRQ_ONLY |
193+ * | SEGCBLIST_RCU_CORE |
172194 * | |
173195 * | Callbacks processed by rcu_core() from softirqs or local |
174196 * | rcuc kthread, without holding nocb_lock. |
175197 * ----------------------------------------------------------------------------
176198 */
177199 #define SEGCBLIST_ENABLED BIT(0)
178−#define SEGCBLIST_SOFTIRQ_ONLY BIT(1)
179−#define SEGCBLIST_KTHREAD_CB BIT(2)
180−#define SEGCBLIST_KTHREAD_GP BIT(3)
181−#define SEGCBLIST_OFFLOADED BIT(4)
200+#define SEGCBLIST_RCU_CORE BIT(1)
201+#define SEGCBLIST_LOCKING BIT(2)
202+#define SEGCBLIST_KTHREAD_CB BIT(3)
203+#define SEGCBLIST_KTHREAD_GP BIT(4)
204+#define SEGCBLIST_OFFLOADED BIT(5)
182205
183206 struct rcu_segcblist {
184207 struct rcu_head *head;
include/linux/rcupdate.h+28−22
@@ -364,6 +364,12 @@ static inline void rcu_preempt_sleep_check(void) { } · 363 unmodified lines
364364 #define rcu_check_sparse(p, space)
365365 #endif /* #else #ifdef __CHECKER__ */
366366
367+#define __unrcu_pointer(p, local) \
368+({ \
369+ typeof(*p) *local = (typeof(*p) *__force)(p); \
370+ rcu_check_sparse(p, __rcu); \
371+ ((typeof(*p) __force __kernel *)(local)); \
372+})
367373 /**
368374 * unrcu_pointer - mark a pointer as not being RCU protected
369375 * @p: pointer needing to lose its __rcu property
@@ -371,39 +377,35 @@ static inline void rcu_preempt_sleep_check(void) { } · 1 unmodified line
371377 * Converts @p from an __rcu pointer to a __kernel pointer.
372378 * This allows an __rcu pointer to be used with xchg() and friends.
373379 */
374−#define unrcu_pointer(p) \
375−({ \
376− typeof(*p) *_________p1 = (typeof(*p) *__force)(p); \
377− rcu_check_sparse(p, __rcu); \
378− ((typeof(*p) __force __kernel *)(_________p1)); \
379−})
380+#define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu))
380381
381−#define __rcu_access_pointer(p, space) \
382+#define __rcu_access_pointer(p, local, space) \
382383 ({ \
383− typeof(*p) *_________p1 = (typeof(*p) *__force)READ_ONCE(p); \
384+ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \
384385 rcu_check_sparse(p, space); \
385− ((typeof(*p) __force __kernel *)(_________p1)); \
386+ ((typeof(*p) __force __kernel *)(local)); \
386387 })
387−#define __rcu_dereference_check(p, c, space) \
388+#define __rcu_dereference_check(p, local, c, space) \
388389 ({ \
389390 /* Dependency order vs. p above. */ \
390− typeof(*p) *________p1 = (typeof(*p) *__force)READ_ONCE(p); \
391+ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \
391392 RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \
392393 rcu_check_sparse(p, space); \
393− ((typeof(*p) __force __kernel *)(________p1)); \
394+ ((typeof(*p) __force __kernel *)(local)); \
394395 })
395−#define __rcu_dereference_protected(p, c, space) \
396+#define __rcu_dereference_protected(p, local, c, space) \
396397 ({ \
397398 RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \
398399 rcu_check_sparse(p, space); \
399400 ((typeof(*p) __force __kernel *)(p)); \
400401 })
401−#define rcu_dereference_raw(p) \
402+#define __rcu_dereference_raw(p, local) \
402403 ({ \
403404 /* Dependency order vs. p above. */ \
404− typeof(p) ________p1 = READ_ONCE(p); \
405− ((typeof(*p) __force __kernel *)(________p1)); \
405+ typeof(p) local = READ_ONCE(p); \
406+ ((typeof(*p) __force __kernel *)(local)); \
406407 })
408+#define rcu_dereference_raw(p) __rcu_dereference_raw(p, __UNIQUE_ID(rcu))
407409
408410 /**
409411 * RCU_INITIALIZER() - statically initialize an RCU-protected global variable
@@ -490,7 +492,7 @@ do { \ · 80 unmodified lines
490492 * when tearing down multi-linked structures after a grace period
491493 * has elapsed.
492494 */
493−#define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu)
495+#define rcu_access_pointer(p) __rcu_access_pointer((p), __UNIQUE_ID(rcu), __rcu)
494496
495497 /**
496498 * rcu_dereference_check() - rcu_dereference with debug checking
@@ -526,7 +528,8 @@ do { \ · 29 unmodified lines
526528 * annotated as __rcu.
527529 */
528530 #define rcu_dereference_check(p, c) \
529− __rcu_dereference_check((p), (c) || rcu_read_lock_held(), __rcu)
531+ __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
532+ (c) || rcu_read_lock_held(), __rcu)
530533
531534 /**
532535 * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
@@ -541,7 +544,8 @@ do { \ · 8 unmodified lines
541544 * rcu_read_lock() but also rcu_read_lock_bh() into account.
542545 */
543546 #define rcu_dereference_bh_check(p, c) \
544− __rcu_dereference_check((p), (c) || rcu_read_lock_bh_held(), __rcu)
547+ __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
548+ (c) || rcu_read_lock_bh_held(), __rcu)
545549
546550 /**
547551 * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
@@ -556,7 +560,8 @@ do { \ · 8 unmodified lines
556560 * only rcu_read_lock() but also rcu_read_lock_sched() into account.
557561 */
558562 #define rcu_dereference_sched_check(p, c) \
559− __rcu_dereference_check((p), (c) || rcu_read_lock_sched_held(), \
563+ __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
564+ (c) || rcu_read_lock_sched_held(), \
560565 __rcu)
561566
562567 /*
@@ -566,7 +571,8 @@ do { \ · 3 unmodified lines
566571 * The no-tracing version of rcu_dereference_raw() must not call
567572 * rcu_read_lock_held().
568573 */
569−#define rcu_dereference_raw_check(p) __rcu_dereference_check((p), 1, __rcu)
574+#define rcu_dereference_raw_check(p) \
575+ __rcu_dereference_check((p), __UNIQUE_ID(rcu), 1, __rcu)
570576
571577 /**
572578 * rcu_dereference_protected() - fetch RCU pointer when updates prevented
@@ -585,7 +591,7 @@ do { \ · 12 unmodified lines
585591 * but very ugly failures.
586592 */
587593 #define rcu_dereference_protected(p, c) \
588− __rcu_dereference_protected((p), (c), __rcu)
594+ __rcu_dereference_protected((p), __UNIQUE_ID(rcu), (c), __rcu)
589595
590596
591597 /**
include/linux/rcutiny.h+1−1
@@ -85,7 +85,7 @@ static inline void rcu_irq_enter_irqson(void) { } · 84 unmodified lines
8585 static inline void rcu_irq_exit(void) { }
8686 static inline void rcu_irq_exit_check_preempt(void) { }
8787 #define rcu_is_idle_cpu(cpu) \
88− (is_idle_task(current) && !in_nmi() && !in_irq() && !in_serving_softirq())
88+ (is_idle_task(current) && !in_nmi() && !in_hardirq() && !in_serving_softirq())
8989 static inline void exit_rcu(void) { }
9090 static inline bool rcu_preempt_need_deferred_qs(struct task_struct *t)
9191 {
include/linux/srcu.h+2−1
@@ -117,7 +117,8 @@ static inline int srcu_read_lock_held(const struct srcu_struct *ssp) · 116 unmodified lines
117117 * lockdep_is_held() calls.
118118 */
119119 #define srcu_dereference_check(p, ssp, c) \
120− __rcu_dereference_check((p), (c) || srcu_read_lock_held(ssp), __rcu)
120+ __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
121+ (c) || srcu_read_lock_held(ssp), __rcu)
121122
122123 /**
123124 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing
include/linux/torture.h+2−7
@@ -38,13 +38,8 @@ do { \ · 37 unmodified lines
3838 pr_alert("%s" TORTURE_FLAG " %s\n", torture_type, s); \
3939 } \
4040 } while (0)
41−#define VERBOSE_TOROUT_ERRSTRING(s) \
42−do { \
43− if (verbose) { \
44− verbose_torout_sleep(); \
45− pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s); \
46− } \
47−} while (0)
41+#define TOROUT_ERRSTRING(s) \
42+ pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s)
4843 void verbose_torout_sleep(void);
4944
5045 #define torture_init_error(firsterr) \

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t