History

Merge branches 'doc.2021.11.30c', 'exp.2021.12.07a', 'fastnohz.2021.11.30c', 'fixes.2021.11.30c', 'nocb.2021.12.09a', 'nolibc.2021.11.30c', 'tasks.2021.12.09a', 'torture.2021.12.07a' and 'torturescript.2021.11.30c' into HEAD

doc.2021.11.30c: Documentation updates. exp.2021.12.07a: Expedited-grace-period fixes. fastnohz.2021.11.30c: Remove CONFIG_RCU_FAST_NO_HZ. fixes.2021.11.30c: Miscellaneous fixes. nocb.2021.12.09a: No-CB CPU updates. nolibc.2021.11.30c: Tiny in-kernel library updates. tasks.2021.12.09a: RCU-tasks updates, including update-side scalability. torture.2021.12.07a: Torture-test in-kernel module updates. torturescript.2021.11.30c: Torture-test scripting updates.

Paul E. McKenney committed 4 years ago

Commit f80fe669 parents 5861dad 81f6d49 bc849e9 1f8da40 10d4703 b0fe9de fd796e4 53b541f 90b21bc51 files · +1089 −719Browse files

Showing the changes to include (5 of 51 files). Show all

Unified · Split

5 files
  1. include/linux/rcu_segcblist.h +37 −14
  2. include/linux/rcupdate.h +28 −22
  3. include/linux/rcutiny.h +1 −1
  4. include/linux/srcu.h +2 −1
  5. include/linux/torture.h +2 −7
include/linux/rcu_segcblist.h+37−14
@@ -69,7 +69,7 @@ struct rcu_cblist { · 68 unmodified lines
69 *69 *
70 *70 *
71 * ----------------------------------------------------------------------------71 * ----------------------------------------------------------------------------
72 * | SEGCBLIST_SOFTIRQ_ONLY |72 * | SEGCBLIST_RCU_CORE |
73 * | |73 * | |
74 * | Callbacks processed by rcu_core() from softirqs or local |74 * | Callbacks processed by rcu_core() from softirqs or local |
75 * | rcuc kthread, without holding nocb_lock. |75 * | rcuc kthread, without holding nocb_lock. |
@@ -77,7 +77,7 @@ struct rcu_cblist { · 1 unmodified line
77 * |77 * |
78 * v78 * v
79 * ----------------------------------------------------------------------------79 * ----------------------------------------------------------------------------
80 * | SEGCBLIST_OFFLOADED |80 * | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED |
81 * | |81 * | |
82 * | Callbacks processed by rcu_core() from softirqs or local |82 * | Callbacks processed by rcu_core() from softirqs or local |
83 * | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads, |83 * | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads, |
@@ -89,7 +89,9 @@ struct rcu_cblist { · 5 unmodified lines
89 * | |89 * | |
90 * v v90 * v v
91 * --------------------------------------- ----------------------------------|91 * --------------------------------------- ----------------------------------|
92 * | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | |92 * | SEGCBLIST_RCU_CORE | | | SEGCBLIST_RCU_CORE | |
93 * | SEGCBLIST_LOCKING | | | SEGCBLIST_LOCKING | |
94 * | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | |
93 * | SEGCBLIST_KTHREAD_CB | | SEGCBLIST_KTHREAD_GP |95 * | SEGCBLIST_KTHREAD_CB | | SEGCBLIST_KTHREAD_GP |
94 * | | | |96 * | | | |
95 * | | | |97 * | | | |
@@ -104,9 +106,10 @@ struct rcu_cblist { · 8 unmodified lines
104 * |106 * |
105 * v107 * v
106 * |--------------------------------------------------------------------------|108 * |--------------------------------------------------------------------------|
107 * | SEGCBLIST_OFFLOADED | |109 * | SEGCBLIST_LOCKING | |
108 * | SEGCBLIST_KTHREAD_CB | |110 * | SEGCBLIST_OFFLOADED | |
109 * | SEGCBLIST_KTHREAD_GP |111 * | SEGCBLIST_KTHREAD_GP | |
112 * | SEGCBLIST_KTHREAD_CB |
110 * | |113 * | |
111 * | Kthreads handle callbacks holding nocb_lock, local rcu_core() stops |114 * | Kthreads handle callbacks holding nocb_lock, local rcu_core() stops |
112 * | handling callbacks. Enable bypass queueing. |115 * | handling callbacks. Enable bypass queueing. |
@@ -120,7 +123,8 @@ struct rcu_cblist { · 7 unmodified lines
120 *123 *
121 *124 *
122 * |--------------------------------------------------------------------------|125 * |--------------------------------------------------------------------------|
123 * | SEGCBLIST_OFFLOADED | |126 * | SEGCBLIST_LOCKING | |
127 * | SEGCBLIST_OFFLOADED | |
124 * | SEGCBLIST_KTHREAD_CB | |128 * | SEGCBLIST_KTHREAD_CB | |
125 * | SEGCBLIST_KTHREAD_GP |129 * | SEGCBLIST_KTHREAD_GP |
126 * | |130 * | |
@@ -130,6 +134,22 @@ struct rcu_cblist { · 3 unmodified lines
130 * |134 * |
131 * v135 * v
132 * |--------------------------------------------------------------------------|136 * |--------------------------------------------------------------------------|
137 * | SEGCBLIST_RCU_CORE | |
138 * | SEGCBLIST_LOCKING | |
139 * | SEGCBLIST_OFFLOADED | |
140 * | SEGCBLIST_KTHREAD_CB | |
141 * | SEGCBLIST_KTHREAD_GP |
142 * | |
143 * | CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core() |
144 * | handles callbacks concurrently. Bypass enqueue is enabled. |
145 * | Invoke RCU core so we make sure not to preempt it in the middle with |
146 * | leaving some urgent work unattended within a jiffy. |
147 * ----------------------------------------------------------------------------
148 * |
149 * v
150 * |--------------------------------------------------------------------------|
151 * | SEGCBLIST_RCU_CORE | |
152 * | SEGCBLIST_LOCKING | |
133 * | SEGCBLIST_KTHREAD_CB | |153 * | SEGCBLIST_KTHREAD_CB | |
134 * | SEGCBLIST_KTHREAD_GP |154 * | SEGCBLIST_KTHREAD_GP |
135 * | |155 * | |
@@ -143,7 +163,9 @@ struct rcu_cblist { · 7 unmodified lines
143 * | |163 * | |
144 * v v164 * v v
145 * ---------------------------------------------------------------------------|165 * ---------------------------------------------------------------------------|
146 * | |166 * | | |
167 * | SEGCBLIST_RCU_CORE | | SEGCBLIST_RCU_CORE | |
168 * | SEGCBLIST_LOCKING | | SEGCBLIST_LOCKING | |
147 * | SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP |169 * | SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP |
148 * | | |170 * | | |
149 * | GP kthread woke up and | CB kthread woke up and |171 * | GP kthread woke up and | CB kthread woke up and |
@@ -159,7 +181,7 @@ struct rcu_cblist { · 9 unmodified lines
159 * |181 * |
160 * v182 * v
161 * ----------------------------------------------------------------------------183 * ----------------------------------------------------------------------------
162 * | 0 |184 * | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING |
163 * | |185 * | |
164 * | Callbacks processed by rcu_core() from softirqs or local |186 * | Callbacks processed by rcu_core() from softirqs or local |
165 * | rcuc kthread, while holding nocb_lock. Forbid nocb_timer to be armed. |187 * | rcuc kthread, while holding nocb_lock. Forbid nocb_timer to be armed. |
@@ -168,17 +190,18 @@ struct rcu_cblist { · 2 unmodified lines
168 * |190 * |
169 * v191 * v
170 * ----------------------------------------------------------------------------192 * ----------------------------------------------------------------------------
171 * | SEGCBLIST_SOFTIRQ_ONLY |193 * | SEGCBLIST_RCU_CORE |
172 * | |194 * | |
173 * | Callbacks processed by rcu_core() from softirqs or local |195 * | Callbacks processed by rcu_core() from softirqs or local |
174 * | rcuc kthread, without holding nocb_lock. |196 * | rcuc kthread, without holding nocb_lock. |
175 * ----------------------------------------------------------------------------197 * ----------------------------------------------------------------------------
176 */198 */
177#define SEGCBLIST_ENABLED BIT(0)199#define SEGCBLIST_ENABLED BIT(0)
178#define SEGCBLIST_SOFTIRQ_ONLY BIT(1)200#define SEGCBLIST_RCU_CORE BIT(1)
179#define SEGCBLIST_KTHREAD_CB BIT(2)201#define SEGCBLIST_LOCKING BIT(2)
180#define SEGCBLIST_KTHREAD_GP BIT(3)202#define SEGCBLIST_KTHREAD_CB BIT(3)
181#define SEGCBLIST_OFFLOADED BIT(4)203#define SEGCBLIST_KTHREAD_GP BIT(4)
204#define SEGCBLIST_OFFLOADED BIT(5)
182 205
183struct rcu_segcblist {206struct rcu_segcblist {
184 struct rcu_head *head;207 struct rcu_head *head;
include/linux/rcupdate.h+28−22
@@ -364,6 +364,12 @@ static inline void rcu_preempt_sleep_check(void) { } · 363 unmodified lines
364#define rcu_check_sparse(p, space)364#define rcu_check_sparse(p, space)
365#endif /* #else #ifdef __CHECKER__ */365#endif /* #else #ifdef __CHECKER__ */
366 366
367#define __unrcu_pointer(p, local) \
368({ \
369 typeof(*p) *local = (typeof(*p) *__force)(p); \
370 rcu_check_sparse(p, __rcu); \
371 ((typeof(*p) __force __kernel *)(local)); \
372})
367/**373/**
368 * unrcu_pointer - mark a pointer as not being RCU protected374 * unrcu_pointer - mark a pointer as not being RCU protected
369 * @p: pointer needing to lose its __rcu property375 * @p: pointer needing to lose its __rcu property
@@ -371,39 +377,35 @@ static inline void rcu_preempt_sleep_check(void) { } · 1 unmodified line
371 * Converts @p from an __rcu pointer to a __kernel pointer.377 * Converts @p from an __rcu pointer to a __kernel pointer.
372 * This allows an __rcu pointer to be used with xchg() and friends.378 * This allows an __rcu pointer to be used with xchg() and friends.
373 */379 */
374#define unrcu_pointer(p) \380#define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu))
375({ \
376 typeof(*p) *_________p1 = (typeof(*p) *__force)(p); \
377 rcu_check_sparse(p, __rcu); \
378 ((typeof(*p) __force __kernel *)(_________p1)); \
379})
380 381
381#define __rcu_access_pointer(p, space) \382#define __rcu_access_pointer(p, local, space) \
382({ \383({ \
383 typeof(*p) *_________p1 = (typeof(*p) *__force)READ_ONCE(p); \384 typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \
384 rcu_check_sparse(p, space); \385 rcu_check_sparse(p, space); \
385 ((typeof(*p) __force __kernel *)(_________p1)); \386 ((typeof(*p) __force __kernel *)(local)); \
386})387})
387#define __rcu_dereference_check(p, c, space) \388#define __rcu_dereference_check(p, local, c, space) \
388({ \389({ \
389 /* Dependency order vs. p above. */ \390 /* Dependency order vs. p above. */ \
390 typeof(*p) *________p1 = (typeof(*p) *__force)READ_ONCE(p); \391 typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \
391 RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \392 RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \
392 rcu_check_sparse(p, space); \393 rcu_check_sparse(p, space); \
393 ((typeof(*p) __force __kernel *)(________p1)); \394 ((typeof(*p) __force __kernel *)(local)); \
394})395})
395#define __rcu_dereference_protected(p, c, space) \396#define __rcu_dereference_protected(p, local, c, space) \
396({ \397({ \
397 RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \398 RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \
398 rcu_check_sparse(p, space); \399 rcu_check_sparse(p, space); \
399 ((typeof(*p) __force __kernel *)(p)); \400 ((typeof(*p) __force __kernel *)(p)); \
400})401})
401#define rcu_dereference_raw(p) \402#define __rcu_dereference_raw(p, local) \
402({ \403({ \
403 /* Dependency order vs. p above. */ \404 /* Dependency order vs. p above. */ \
404 typeof(p) ________p1 = READ_ONCE(p); \405 typeof(p) local = READ_ONCE(p); \
405 ((typeof(*p) __force __kernel *)(________p1)); \406 ((typeof(*p) __force __kernel *)(local)); \
406})407})
408#define rcu_dereference_raw(p) __rcu_dereference_raw(p, __UNIQUE_ID(rcu))
407 409
408/**410/**
409 * RCU_INITIALIZER() - statically initialize an RCU-protected global variable411 * RCU_INITIALIZER() - statically initialize an RCU-protected global variable
@@ -490,7 +492,7 @@ do { \ · 80 unmodified lines
490 * when tearing down multi-linked structures after a grace period492 * when tearing down multi-linked structures after a grace period
491 * has elapsed.493 * has elapsed.
492 */494 */
493#define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu)495#define rcu_access_pointer(p) __rcu_access_pointer((p), __UNIQUE_ID(rcu), __rcu)
494 496
495/**497/**
496 * rcu_dereference_check() - rcu_dereference with debug checking498 * rcu_dereference_check() - rcu_dereference with debug checking
@@ -526,7 +528,8 @@ do { \ · 29 unmodified lines
526 * annotated as __rcu.528 * annotated as __rcu.
527 */529 */
528#define rcu_dereference_check(p, c) \530#define rcu_dereference_check(p, c) \
529 __rcu_dereference_check((p), (c) || rcu_read_lock_held(), __rcu)531 __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
532 (c) || rcu_read_lock_held(), __rcu)
530 533
531/**534/**
532 * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking535 * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking
@@ -541,7 +544,8 @@ do { \ · 8 unmodified lines
541 * rcu_read_lock() but also rcu_read_lock_bh() into account.544 * rcu_read_lock() but also rcu_read_lock_bh() into account.
542 */545 */
543#define rcu_dereference_bh_check(p, c) \546#define rcu_dereference_bh_check(p, c) \
544 __rcu_dereference_check((p), (c) || rcu_read_lock_bh_held(), __rcu)547 __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
548 (c) || rcu_read_lock_bh_held(), __rcu)
545 549
546/**550/**
547 * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking551 * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking
@@ -556,7 +560,8 @@ do { \ · 8 unmodified lines
556 * only rcu_read_lock() but also rcu_read_lock_sched() into account.560 * only rcu_read_lock() but also rcu_read_lock_sched() into account.
557 */561 */
558#define rcu_dereference_sched_check(p, c) \562#define rcu_dereference_sched_check(p, c) \
559 __rcu_dereference_check((p), (c) || rcu_read_lock_sched_held(), \563 __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
564 (c) || rcu_read_lock_sched_held(), \
560 __rcu)565 __rcu)
561 566
562/*567/*
@@ -566,7 +571,8 @@ do { \ · 3 unmodified lines
566 * The no-tracing version of rcu_dereference_raw() must not call571 * The no-tracing version of rcu_dereference_raw() must not call
567 * rcu_read_lock_held().572 * rcu_read_lock_held().
568 */573 */
569#define rcu_dereference_raw_check(p) __rcu_dereference_check((p), 1, __rcu)574#define rcu_dereference_raw_check(p) \
575 __rcu_dereference_check((p), __UNIQUE_ID(rcu), 1, __rcu)
570 576
571/**577/**
572 * rcu_dereference_protected() - fetch RCU pointer when updates prevented578 * rcu_dereference_protected() - fetch RCU pointer when updates prevented
@@ -585,7 +591,7 @@ do { \ · 12 unmodified lines
585 * but very ugly failures.591 * but very ugly failures.
586 */592 */
587#define rcu_dereference_protected(p, c) \593#define rcu_dereference_protected(p, c) \
588 __rcu_dereference_protected((p), (c), __rcu)594 __rcu_dereference_protected((p), __UNIQUE_ID(rcu), (c), __rcu)
589 595
590 596
591/**597/**
include/linux/rcutiny.h+1−1
@@ -85,7 +85,7 @@ static inline void rcu_irq_enter_irqson(void) { } · 84 unmodified lines
85static inline void rcu_irq_exit(void) { }85static inline void rcu_irq_exit(void) { }
86static inline void rcu_irq_exit_check_preempt(void) { }86static inline void rcu_irq_exit_check_preempt(void) { }
87#define rcu_is_idle_cpu(cpu) \87#define rcu_is_idle_cpu(cpu) \
88 (is_idle_task(current) && !in_nmi() && !in_irq() && !in_serving_softirq())88 (is_idle_task(current) && !in_nmi() && !in_hardirq() && !in_serving_softirq())
89static inline void exit_rcu(void) { }89static inline void exit_rcu(void) { }
90static inline bool rcu_preempt_need_deferred_qs(struct task_struct *t)90static inline bool rcu_preempt_need_deferred_qs(struct task_struct *t)
91{91{
include/linux/srcu.h+2−1
@@ -117,7 +117,8 @@ static inline int srcu_read_lock_held(const struct srcu_struct *ssp) · 116 unmodified lines
117 * lockdep_is_held() calls.117 * lockdep_is_held() calls.
118 */118 */
119#define srcu_dereference_check(p, ssp, c) \119#define srcu_dereference_check(p, ssp, c) \
120 __rcu_dereference_check((p), (c) || srcu_read_lock_held(ssp), __rcu)120 __rcu_dereference_check((p), __UNIQUE_ID(rcu), \
121 (c) || srcu_read_lock_held(ssp), __rcu)
121 122
122/**123/**
123 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing124 * srcu_dereference - fetch SRCU-protected pointer for later dereferencing
include/linux/torture.h+2−7
@@ -38,13 +38,8 @@ do { \ · 37 unmodified lines
38 pr_alert("%s" TORTURE_FLAG " %s\n", torture_type, s); \38 pr_alert("%s" TORTURE_FLAG " %s\n", torture_type, s); \
39 } \39 } \
40} while (0)40} while (0)
41#define VERBOSE_TOROUT_ERRSTRING(s) \41#define TOROUT_ERRSTRING(s) \
42do { \42 pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s)
43 if (verbose) { \
44 verbose_torout_sleep(); \
45 pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s); \
46 } \
47} while (0)
48void verbose_torout_sleep(void);43void verbose_torout_sleep(void);
49 44
50#define torture_init_error(firsterr) \45#define torture_init_error(firsterr) \

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t