doc.2021.11.30c: Documentation updates. exp.2021.12.07a: Expedited-grace-period fixes. fastnohz.2021.11.30c: Remove CONFIG_RCU_FAST_NO_HZ. fixes.2021.11.30c: Miscellaneous fixes. nocb.2021.12.09a: No-CB CPU updates. nolibc.2021.11.30c: Tiny in-kernel library updates. tasks.2021.12.09a: RCU-tasks updates, including update-side scalability. torture.2021.12.07a: Torture-test in-kernel module updates. torturescript.2021.11.30c: Torture-test scripting updates.
f80fe669 parents 5861dad 81f6d49 bc849e9 1f8da40 10d4703 b0fe9de fd796e4 53b541f 90b21bc51 files · +1089 −719Browse filesShowing the changes to include (5 of 51 files). Show all
Unified · Split
| @@ -69,7 +69,7 @@ struct rcu_cblist { · 68 unmodified lines | ||
| 69 | 69 | * |
| 70 | 70 | * |
| 71 | 71 | * ---------------------------------------------------------------------------- |
| 72 | − * | SEGCBLIST_SOFTIRQ_ONLY | | |
| 72 | + * | SEGCBLIST_RCU_CORE | | |
| 73 | 73 | * | | |
| 74 | 74 | * | Callbacks processed by rcu_core() from softirqs or local | |
| 75 | 75 | * | rcuc kthread, without holding nocb_lock. | |
| @@ -77,7 +77,7 @@ struct rcu_cblist { · 1 unmodified line | ||
| 77 | 77 | * | |
| 78 | 78 | * v |
| 79 | 79 | * ---------------------------------------------------------------------------- |
| 80 | − * | SEGCBLIST_OFFLOADED | | |
| 80 | + * | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED | | |
| 81 | 81 | * | | |
| 82 | 82 | * | Callbacks processed by rcu_core() from softirqs or local | |
| 83 | 83 | * | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads, | |
| @@ -89,7 +89,9 @@ struct rcu_cblist { · 5 unmodified lines | ||
| 89 | 89 | * | | |
| 90 | 90 | * v v |
| 91 | 91 | * --------------------------------------- ----------------------------------| |
| 92 | − * | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | | | |
| 92 | + * | SEGCBLIST_RCU_CORE | | | SEGCBLIST_RCU_CORE | | | |
| 93 | + * | SEGCBLIST_LOCKING | | | SEGCBLIST_LOCKING | | | |
| 94 | + * | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | | | |
| 93 | 95 | * | SEGCBLIST_KTHREAD_CB | | SEGCBLIST_KTHREAD_GP | |
| 94 | 96 | * | | | | |
| 95 | 97 | * | | | | |
| @@ -104,9 +106,10 @@ struct rcu_cblist { · 8 unmodified lines | ||
| 104 | 106 | * | |
| 105 | 107 | * v |
| 106 | 108 | * |--------------------------------------------------------------------------| |
| 107 | − * | SEGCBLIST_OFFLOADED | | | |
| 108 | − * | SEGCBLIST_KTHREAD_CB | | | |
| 109 | − * | SEGCBLIST_KTHREAD_GP | | |
| 109 | + * | SEGCBLIST_LOCKING | | | |
| 110 | + * | SEGCBLIST_OFFLOADED | | | |
| 111 | + * | SEGCBLIST_KTHREAD_GP | | | |
| 112 | + * | SEGCBLIST_KTHREAD_CB | | |
| 110 | 113 | * | | |
| 111 | 114 | * | Kthreads handle callbacks holding nocb_lock, local rcu_core() stops | |
| 112 | 115 | * | handling callbacks. Enable bypass queueing. | |
| @@ -120,7 +123,8 @@ struct rcu_cblist { · 7 unmodified lines | ||
| 120 | 123 | * |
| 121 | 124 | * |
| 122 | 125 | * |--------------------------------------------------------------------------| |
| 123 | − * | SEGCBLIST_OFFLOADED | | | |
| 126 | + * | SEGCBLIST_LOCKING | | | |
| 127 | + * | SEGCBLIST_OFFLOADED | | | |
| 124 | 128 | * | SEGCBLIST_KTHREAD_CB | | |
| 125 | 129 | * | SEGCBLIST_KTHREAD_GP | |
| 126 | 130 | * | | |
| @@ -130,6 +134,22 @@ struct rcu_cblist { · 3 unmodified lines | ||
| 130 | 134 | * | |
| 131 | 135 | * v |
| 132 | 136 | * |--------------------------------------------------------------------------| |
| 137 | + * | SEGCBLIST_RCU_CORE | | | |
| 138 | + * | SEGCBLIST_LOCKING | | | |
| 139 | + * | SEGCBLIST_OFFLOADED | | | |
| 140 | + * | SEGCBLIST_KTHREAD_CB | | | |
| 141 | + * | SEGCBLIST_KTHREAD_GP | | |
| 142 | + * | | | |
| 143 | + * | CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core() | | |
| 144 | + * | handles callbacks concurrently. Bypass enqueue is enabled. | | |
| 145 | + * | Invoke RCU core so we make sure not to preempt it in the middle with | | |
| 146 | + * | leaving some urgent work unattended within a jiffy. | | |
| 147 | + * ---------------------------------------------------------------------------- | |
| 148 | + * | | |
| 149 | + * v | |
| 150 | + * |--------------------------------------------------------------------------| | |
| 151 | + * | SEGCBLIST_RCU_CORE | | | |
| 152 | + * | SEGCBLIST_LOCKING | | | |
| 133 | 153 | * | SEGCBLIST_KTHREAD_CB | | |
| 134 | 154 | * | SEGCBLIST_KTHREAD_GP | |
| 135 | 155 | * | | |
| @@ -143,7 +163,9 @@ struct rcu_cblist { · 7 unmodified lines | ||
| 143 | 163 | * | | |
| 144 | 164 | * v v |
| 145 | 165 | * ---------------------------------------------------------------------------| |
| 146 | − * | | | |
| 166 | + * | | | | |
| 167 | + * | SEGCBLIST_RCU_CORE | | SEGCBLIST_RCU_CORE | | | |
| 168 | + * | SEGCBLIST_LOCKING | | SEGCBLIST_LOCKING | | | |
| 147 | 169 | * | SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP | |
| 148 | 170 | * | | | |
| 149 | 171 | * | GP kthread woke up and | CB kthread woke up and | |
| @@ -159,7 +181,7 @@ struct rcu_cblist { · 9 unmodified lines | ||
| 159 | 181 | * | |
| 160 | 182 | * v |
| 161 | 183 | * ---------------------------------------------------------------------------- |
| 162 | − * | 0 | | |
| 184 | + * | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | | |
| 163 | 185 | * | | |
| 164 | 186 | * | Callbacks processed by rcu_core() from softirqs or local | |
| 165 | 187 | * | rcuc kthread, while holding nocb_lock. Forbid nocb_timer to be armed. | |
| @@ -168,17 +190,18 @@ struct rcu_cblist { · 2 unmodified lines | ||
| 168 | 190 | * | |
| 169 | 191 | * v |
| 170 | 192 | * ---------------------------------------------------------------------------- |
| 171 | − * | SEGCBLIST_SOFTIRQ_ONLY | | |
| 193 | + * | SEGCBLIST_RCU_CORE | | |
| 172 | 194 | * | | |
| 173 | 195 | * | Callbacks processed by rcu_core() from softirqs or local | |
| 174 | 196 | * | rcuc kthread, without holding nocb_lock. | |
| 175 | 197 | * ---------------------------------------------------------------------------- |
| 176 | 198 | */ |
| 177 | 199 | #define SEGCBLIST_ENABLED BIT(0) |
| 178 | −#define SEGCBLIST_SOFTIRQ_ONLY BIT(1) | |
| 179 | −#define SEGCBLIST_KTHREAD_CB BIT(2) | |
| 180 | −#define SEGCBLIST_KTHREAD_GP BIT(3) | |
| 181 | −#define SEGCBLIST_OFFLOADED BIT(4) | |
| 200 | +#define SEGCBLIST_RCU_CORE BIT(1) | |
| 201 | +#define SEGCBLIST_LOCKING BIT(2) | |
| 202 | +#define SEGCBLIST_KTHREAD_CB BIT(3) | |
| 203 | +#define SEGCBLIST_KTHREAD_GP BIT(4) | |
| 204 | +#define SEGCBLIST_OFFLOADED BIT(5) | |
| 182 | 205 | |
| 183 | 206 | struct rcu_segcblist { |
| 184 | 207 | struct rcu_head *head; |
| @@ -364,6 +364,12 @@ static inline void rcu_preempt_sleep_check(void) { } · 363 unmodified lines | ||
| 364 | 364 | #define rcu_check_sparse(p, space) |
| 365 | 365 | #endif /* #else #ifdef __CHECKER__ */ |
| 366 | 366 | |
| 367 | +#define __unrcu_pointer(p, local) \ | |
| 368 | +({ \ | |
| 369 | + typeof(*p) *local = (typeof(*p) *__force)(p); \ | |
| 370 | + rcu_check_sparse(p, __rcu); \ | |
| 371 | + ((typeof(*p) __force __kernel *)(local)); \ | |
| 372 | +}) | |
| 367 | 373 | /** |
| 368 | 374 | * unrcu_pointer - mark a pointer as not being RCU protected |
| 369 | 375 | * @p: pointer needing to lose its __rcu property |
| @@ -371,39 +377,35 @@ static inline void rcu_preempt_sleep_check(void) { } · 1 unmodified line | ||
| 371 | 377 | * Converts @p from an __rcu pointer to a __kernel pointer. |
| 372 | 378 | * This allows an __rcu pointer to be used with xchg() and friends. |
| 373 | 379 | */ |
| 374 | −#define unrcu_pointer(p) \ | |
| 375 | −({ \ | |
| 376 | − typeof(*p) *_________p1 = (typeof(*p) *__force)(p); \ | |
| 377 | − rcu_check_sparse(p, __rcu); \ | |
| 378 | − ((typeof(*p) __force __kernel *)(_________p1)); \ | |
| 379 | −}) | |
| 380 | +#define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu)) | |
| 380 | 381 | |
| 381 | −#define __rcu_access_pointer(p, space) \ | |
| 382 | +#define __rcu_access_pointer(p, local, space) \ | |
| 382 | 383 | ({ \ |
| 383 | − typeof(*p) *_________p1 = (typeof(*p) *__force)READ_ONCE(p); \ | |
| 384 | + typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ | |
| 384 | 385 | rcu_check_sparse(p, space); \ |
| 385 | − ((typeof(*p) __force __kernel *)(_________p1)); \ | |
| 386 | + ((typeof(*p) __force __kernel *)(local)); \ | |
| 386 | 387 | }) |
| 387 | −#define __rcu_dereference_check(p, c, space) \ | |
| 388 | +#define __rcu_dereference_check(p, local, c, space) \ | |
| 388 | 389 | ({ \ |
| 389 | 390 | /* Dependency order vs. p above. */ \ |
| 390 | − typeof(*p) *________p1 = (typeof(*p) *__force)READ_ONCE(p); \ | |
| 391 | + typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ | |
| 391 | 392 | RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \ |
| 392 | 393 | rcu_check_sparse(p, space); \ |
| 393 | − ((typeof(*p) __force __kernel *)(________p1)); \ | |
| 394 | + ((typeof(*p) __force __kernel *)(local)); \ | |
| 394 | 395 | }) |
| 395 | −#define __rcu_dereference_protected(p, c, space) \ | |
| 396 | +#define __rcu_dereference_protected(p, local, c, space) \ | |
| 396 | 397 | ({ \ |
| 397 | 398 | RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \ |
| 398 | 399 | rcu_check_sparse(p, space); \ |
| 399 | 400 | ((typeof(*p) __force __kernel *)(p)); \ |
| 400 | 401 | }) |
| 401 | −#define rcu_dereference_raw(p) \ | |
| 402 | +#define __rcu_dereference_raw(p, local) \ | |
| 402 | 403 | ({ \ |
| 403 | 404 | /* Dependency order vs. p above. */ \ |
| 404 | − typeof(p) ________p1 = READ_ONCE(p); \ | |
| 405 | − ((typeof(*p) __force __kernel *)(________p1)); \ | |
| 405 | + typeof(p) local = READ_ONCE(p); \ | |
| 406 | + ((typeof(*p) __force __kernel *)(local)); \ | |
| 406 | 407 | }) |
| 408 | +#define rcu_dereference_raw(p) __rcu_dereference_raw(p, __UNIQUE_ID(rcu)) | |
| 407 | 409 | |
| 408 | 410 | /** |
| 409 | 411 | * RCU_INITIALIZER() - statically initialize an RCU-protected global variable |
| @@ -490,7 +492,7 @@ do { \ · 80 unmodified lines | ||
| 490 | 492 | * when tearing down multi-linked structures after a grace period |
| 491 | 493 | * has elapsed. |
| 492 | 494 | */ |
| 493 | −#define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu) | |
| 495 | +#define rcu_access_pointer(p) __rcu_access_pointer((p), __UNIQUE_ID(rcu), __rcu) | |
| 494 | 496 | |
| 495 | 497 | /** |
| 496 | 498 | * rcu_dereference_check() - rcu_dereference with debug checking |
| @@ -526,7 +528,8 @@ do { \ · 29 unmodified lines | ||
| 526 | 528 | * annotated as __rcu. |
| 527 | 529 | */ |
| 528 | 530 | #define rcu_dereference_check(p, c) \ |
| 529 | − __rcu_dereference_check((p), (c) || rcu_read_lock_held(), __rcu) | |
| 531 | + __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ | |
| 532 | + (c) || rcu_read_lock_held(), __rcu) | |
| 530 | 533 | |
| 531 | 534 | /** |
| 532 | 535 | * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking |
| @@ -541,7 +544,8 @@ do { \ · 8 unmodified lines | ||
| 541 | 544 | * rcu_read_lock() but also rcu_read_lock_bh() into account. |
| 542 | 545 | */ |
| 543 | 546 | #define rcu_dereference_bh_check(p, c) \ |
| 544 | − __rcu_dereference_check((p), (c) || rcu_read_lock_bh_held(), __rcu) | |
| 547 | + __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ | |
| 548 | + (c) || rcu_read_lock_bh_held(), __rcu) | |
| 545 | 549 | |
| 546 | 550 | /** |
| 547 | 551 | * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking |
| @@ -556,7 +560,8 @@ do { \ · 8 unmodified lines | ||
| 556 | 560 | * only rcu_read_lock() but also rcu_read_lock_sched() into account. |
| 557 | 561 | */ |
| 558 | 562 | #define rcu_dereference_sched_check(p, c) \ |
| 559 | − __rcu_dereference_check((p), (c) || rcu_read_lock_sched_held(), \ | |
| 563 | + __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ | |
| 564 | + (c) || rcu_read_lock_sched_held(), \ | |
| 560 | 565 | __rcu) |
| 561 | 566 | |
| 562 | 567 | /* |
| @@ -566,7 +571,8 @@ do { \ · 3 unmodified lines | ||
| 566 | 571 | * The no-tracing version of rcu_dereference_raw() must not call |
| 567 | 572 | * rcu_read_lock_held(). |
| 568 | 573 | */ |
| 569 | −#define rcu_dereference_raw_check(p) __rcu_dereference_check((p), 1, __rcu) | |
| 574 | +#define rcu_dereference_raw_check(p) \ | |
| 575 | + __rcu_dereference_check((p), __UNIQUE_ID(rcu), 1, __rcu) | |
| 570 | 576 | |
| 571 | 577 | /** |
| 572 | 578 | * rcu_dereference_protected() - fetch RCU pointer when updates prevented |
| @@ -585,7 +591,7 @@ do { \ · 12 unmodified lines | ||
| 585 | 591 | * but very ugly failures. |
| 586 | 592 | */ |
| 587 | 593 | #define rcu_dereference_protected(p, c) \ |
| 588 | − __rcu_dereference_protected((p), (c), __rcu) | |
| 594 | + __rcu_dereference_protected((p), __UNIQUE_ID(rcu), (c), __rcu) | |
| 589 | 595 | |
| 590 | 596 | |
| 591 | 597 | /** |
| @@ -85,7 +85,7 @@ static inline void rcu_irq_enter_irqson(void) { } · 84 unmodified lines | ||
| 85 | 85 | static inline void rcu_irq_exit(void) { } |
| 86 | 86 | static inline void rcu_irq_exit_check_preempt(void) { } |
| 87 | 87 | #define rcu_is_idle_cpu(cpu) \ |
| 88 | − (is_idle_task(current) && !in_nmi() && !in_irq() && !in_serving_softirq()) | |
| 88 | + (is_idle_task(current) && !in_nmi() && !in_hardirq() && !in_serving_softirq()) | |
| 89 | 89 | static inline void exit_rcu(void) { } |
| 90 | 90 | static inline bool rcu_preempt_need_deferred_qs(struct task_struct *t) |
| 91 | 91 | { |
| @@ -117,7 +117,8 @@ static inline int srcu_read_lock_held(const struct srcu_struct *ssp) · 116 unmodified lines | ||
| 117 | 117 | * lockdep_is_held() calls. |
| 118 | 118 | */ |
| 119 | 119 | #define srcu_dereference_check(p, ssp, c) \ |
| 120 | − __rcu_dereference_check((p), (c) || srcu_read_lock_held(ssp), __rcu) | |
| 120 | + __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ | |
| 121 | + (c) || srcu_read_lock_held(ssp), __rcu) | |
| 121 | 122 | |
| 122 | 123 | /** |
| 123 | 124 | * srcu_dereference - fetch SRCU-protected pointer for later dereferencing |
| @@ -38,13 +38,8 @@ do { \ · 37 unmodified lines | ||
| 38 | 38 | pr_alert("%s" TORTURE_FLAG " %s\n", torture_type, s); \ |
| 39 | 39 | } \ |
| 40 | 40 | } while (0) |
| 41 | −#define VERBOSE_TOROUT_ERRSTRING(s) \ | |
| 42 | −do { \ | |
| 43 | − if (verbose) { \ | |
| 44 | − verbose_torout_sleep(); \ | |
| 45 | − pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s); \ | |
| 46 | − } \ | |
| 47 | −} while (0) | |
| 41 | +#define TOROUT_ERRSTRING(s) \ | |
| 42 | + pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s) | |
| 48 | 43 | void verbose_torout_sleep(void); |
| 49 | 44 | |
| 50 | 45 | #define torture_init_error(firsterr) \ |