doc.2021.11.30c: Documentation updates. exp.2021.12.07a: Expedited-grace-period fixes. fastnohz.2021.11.30c: Remove CONFIG_RCU_FAST_NO_HZ. fixes.2021.11.30c: Miscellaneous fixes. nocb.2021.12.09a: No-CB CPU updates. nolibc.2021.11.30c: Tiny in-kernel library updates. tasks.2021.12.09a: RCU-tasks updates, including update-side scalability. torture.2021.12.07a: Torture-test in-kernel module updates. torturescript.2021.11.30c: Torture-test scripting updates.
f80fe669 parents 5861dad 81f6d49 bc849e9 1f8da40 10d4703 b0fe9de fd796e4 53b541f 90b21bc51 files · +1089 −719Browse filesShowing the changes to include (5 of 51 files). Show all
Unified · Split
| @@ -69,7 +69,7 @@ struct rcu_cblist { · 68 unmodified lines | |||
| 69 | * | 69 | * |
| 70 | * | 70 | * |
| 71 | * ---------------------------------------------------------------------------- | 71 | * ---------------------------------------------------------------------------- |
| 72 | * | SEGCBLIST_SOFTIRQ_ONLY | | 72 | * | SEGCBLIST_RCU_CORE | |
| 73 | * | | | 73 | * | | |
| 74 | * | Callbacks processed by rcu_core() from softirqs or local | | 74 | * | Callbacks processed by rcu_core() from softirqs or local | |
| 75 | * | rcuc kthread, without holding nocb_lock. | | 75 | * | rcuc kthread, without holding nocb_lock. | |
| @@ -77,7 +77,7 @@ struct rcu_cblist { · 1 unmodified line | |||
| 77 | * | | 77 | * | |
| 78 | * v | 78 | * v |
| 79 | * ---------------------------------------------------------------------------- | 79 | * ---------------------------------------------------------------------------- |
| 80 | * | SEGCBLIST_OFFLOADED | | 80 | * | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | SEGCBLIST_OFFLOADED | |
| 81 | * | | | 81 | * | | |
| 82 | * | Callbacks processed by rcu_core() from softirqs or local | | 82 | * | Callbacks processed by rcu_core() from softirqs or local | |
| 83 | * | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads, | | 83 | * | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads, | |
| @@ -89,7 +89,9 @@ struct rcu_cblist { · 5 unmodified lines | |||
| 89 | * | | | 89 | * | | |
| 90 | * v v | 90 | * v v |
| 91 | * --------------------------------------- ----------------------------------| | 91 | * --------------------------------------- ----------------------------------| |
| 92 | * | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | | | 92 | * | SEGCBLIST_RCU_CORE | | | SEGCBLIST_RCU_CORE | | |
| 93 | * | SEGCBLIST_LOCKING | | | SEGCBLIST_LOCKING | | | ||
| 94 | * | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | | | ||
| 93 | * | SEGCBLIST_KTHREAD_CB | | SEGCBLIST_KTHREAD_GP | | 95 | * | SEGCBLIST_KTHREAD_CB | | SEGCBLIST_KTHREAD_GP | |
| 94 | * | | | | | 96 | * | | | | |
| 95 | * | | | | | 97 | * | | | | |
| @@ -104,9 +106,10 @@ struct rcu_cblist { · 8 unmodified lines | |||
| 104 | * | | 106 | * | |
| 105 | * v | 107 | * v |
| 106 | * |--------------------------------------------------------------------------| | 108 | * |--------------------------------------------------------------------------| |
| 107 | * | SEGCBLIST_OFFLOADED | | | 109 | * | SEGCBLIST_LOCKING | | |
| 108 | * | SEGCBLIST_KTHREAD_CB | | | 110 | * | SEGCBLIST_OFFLOADED | | |
| 109 | * | SEGCBLIST_KTHREAD_GP | | 111 | * | SEGCBLIST_KTHREAD_GP | | |
| 112 | * | SEGCBLIST_KTHREAD_CB | | ||
| 110 | * | | | 113 | * | | |
| 111 | * | Kthreads handle callbacks holding nocb_lock, local rcu_core() stops | | 114 | * | Kthreads handle callbacks holding nocb_lock, local rcu_core() stops | |
| 112 | * | handling callbacks. Enable bypass queueing. | | 115 | * | handling callbacks. Enable bypass queueing. | |
| @@ -120,7 +123,8 @@ struct rcu_cblist { · 7 unmodified lines | |||
| 120 | * | 123 | * |
| 121 | * | 124 | * |
| 122 | * |--------------------------------------------------------------------------| | 125 | * |--------------------------------------------------------------------------| |
| 123 | * | SEGCBLIST_OFFLOADED | | | 126 | * | SEGCBLIST_LOCKING | | |
| 127 | * | SEGCBLIST_OFFLOADED | | | ||
| 124 | * | SEGCBLIST_KTHREAD_CB | | | 128 | * | SEGCBLIST_KTHREAD_CB | | |
| 125 | * | SEGCBLIST_KTHREAD_GP | | 129 | * | SEGCBLIST_KTHREAD_GP | |
| 126 | * | | | 130 | * | | |
| @@ -130,6 +134,22 @@ struct rcu_cblist { · 3 unmodified lines | |||
| 130 | * | | 134 | * | |
| 131 | * v | 135 | * v |
| 132 | * |--------------------------------------------------------------------------| | 136 | * |--------------------------------------------------------------------------| |
| 137 | * | SEGCBLIST_RCU_CORE | | | ||
| 138 | * | SEGCBLIST_LOCKING | | | ||
| 139 | * | SEGCBLIST_OFFLOADED | | | ||
| 140 | * | SEGCBLIST_KTHREAD_CB | | | ||
| 141 | * | SEGCBLIST_KTHREAD_GP | | ||
| 142 | * | | | ||
| 143 | * | CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core() | | ||
| 144 | * | handles callbacks concurrently. Bypass enqueue is enabled. | | ||
| 145 | * | Invoke RCU core so we make sure not to preempt it in the middle with | | ||
| 146 | * | leaving some urgent work unattended within a jiffy. | | ||
| 147 | * ---------------------------------------------------------------------------- | ||
| 148 | * | | ||
| 149 | * v | ||
| 150 | * |--------------------------------------------------------------------------| | ||
| 151 | * | SEGCBLIST_RCU_CORE | | | ||
| 152 | * | SEGCBLIST_LOCKING | | | ||
| 133 | * | SEGCBLIST_KTHREAD_CB | | | 153 | * | SEGCBLIST_KTHREAD_CB | | |
| 134 | * | SEGCBLIST_KTHREAD_GP | | 154 | * | SEGCBLIST_KTHREAD_GP | |
| 135 | * | | | 155 | * | | |
| @@ -143,7 +163,9 @@ struct rcu_cblist { · 7 unmodified lines | |||
| 143 | * | | | 163 | * | | |
| 144 | * v v | 164 | * v v |
| 145 | * ---------------------------------------------------------------------------| | 165 | * ---------------------------------------------------------------------------| |
| 146 | * | | | 166 | * | | | |
| 167 | * | SEGCBLIST_RCU_CORE | | SEGCBLIST_RCU_CORE | | | ||
| 168 | * | SEGCBLIST_LOCKING | | SEGCBLIST_LOCKING | | | ||
| 147 | * | SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP | | 169 | * | SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP | |
| 148 | * | | | | 170 | * | | | |
| 149 | * | GP kthread woke up and | CB kthread woke up and | | 171 | * | GP kthread woke up and | CB kthread woke up and | |
| @@ -159,7 +181,7 @@ struct rcu_cblist { · 9 unmodified lines | |||
| 159 | * | | 181 | * | |
| 160 | * v | 182 | * v |
| 161 | * ---------------------------------------------------------------------------- | 183 | * ---------------------------------------------------------------------------- |
| 162 | * | 0 | | 184 | * | SEGCBLIST_RCU_CORE | SEGCBLIST_LOCKING | |
| 163 | * | | | 185 | * | | |
| 164 | * | Callbacks processed by rcu_core() from softirqs or local | | 186 | * | Callbacks processed by rcu_core() from softirqs or local | |
| 165 | * | rcuc kthread, while holding nocb_lock. Forbid nocb_timer to be armed. | | 187 | * | rcuc kthread, while holding nocb_lock. Forbid nocb_timer to be armed. | |
| @@ -168,17 +190,18 @@ struct rcu_cblist { · 2 unmodified lines | |||
| 168 | * | | 190 | * | |
| 169 | * v | 191 | * v |
| 170 | * ---------------------------------------------------------------------------- | 192 | * ---------------------------------------------------------------------------- |
| 171 | * | SEGCBLIST_SOFTIRQ_ONLY | | 193 | * | SEGCBLIST_RCU_CORE | |
| 172 | * | | | 194 | * | | |
| 173 | * | Callbacks processed by rcu_core() from softirqs or local | | 195 | * | Callbacks processed by rcu_core() from softirqs or local | |
| 174 | * | rcuc kthread, without holding nocb_lock. | | 196 | * | rcuc kthread, without holding nocb_lock. | |
| 175 | * ---------------------------------------------------------------------------- | 197 | * ---------------------------------------------------------------------------- |
| 176 | */ | 198 | */ |
| 177 | #define SEGCBLIST_ENABLED BIT(0) | 199 | #define SEGCBLIST_ENABLED BIT(0) |
| 178 | #define SEGCBLIST_SOFTIRQ_ONLY BIT(1) | 200 | #define SEGCBLIST_RCU_CORE BIT(1) |
| 179 | #define SEGCBLIST_KTHREAD_CB BIT(2) | 201 | #define SEGCBLIST_LOCKING BIT(2) |
| 180 | #define SEGCBLIST_KTHREAD_GP BIT(3) | 202 | #define SEGCBLIST_KTHREAD_CB BIT(3) |
| 181 | #define SEGCBLIST_OFFLOADED BIT(4) | 203 | #define SEGCBLIST_KTHREAD_GP BIT(4) |
| 204 | #define SEGCBLIST_OFFLOADED BIT(5) | ||
| 182 | 205 | ||
| 183 | struct rcu_segcblist { | 206 | struct rcu_segcblist { |
| 184 | struct rcu_head *head; | 207 | struct rcu_head *head; |
| @@ -364,6 +364,12 @@ static inline void rcu_preempt_sleep_check(void) { } · 363 unmodified lines | |||
| 364 | #define rcu_check_sparse(p, space) | 364 | #define rcu_check_sparse(p, space) |
| 365 | #endif /* #else #ifdef __CHECKER__ */ | 365 | #endif /* #else #ifdef __CHECKER__ */ |
| 366 | 366 | ||
| 367 | #define __unrcu_pointer(p, local) \ | ||
| 368 | ({ \ | ||
| 369 | typeof(*p) *local = (typeof(*p) *__force)(p); \ | ||
| 370 | rcu_check_sparse(p, __rcu); \ | ||
| 371 | ((typeof(*p) __force __kernel *)(local)); \ | ||
| 372 | }) | ||
| 367 | /** | 373 | /** |
| 368 | * unrcu_pointer - mark a pointer as not being RCU protected | 374 | * unrcu_pointer - mark a pointer as not being RCU protected |
| 369 | * @p: pointer needing to lose its __rcu property | 375 | * @p: pointer needing to lose its __rcu property |
| @@ -371,39 +377,35 @@ static inline void rcu_preempt_sleep_check(void) { } · 1 unmodified line | |||
| 371 | * Converts @p from an __rcu pointer to a __kernel pointer. | 377 | * Converts @p from an __rcu pointer to a __kernel pointer. |
| 372 | * This allows an __rcu pointer to be used with xchg() and friends. | 378 | * This allows an __rcu pointer to be used with xchg() and friends. |
| 373 | */ | 379 | */ |
| 374 | #define unrcu_pointer(p) \ | 380 | #define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu)) |
| 375 | ({ \ | ||
| 376 | typeof(*p) *_________p1 = (typeof(*p) *__force)(p); \ | ||
| 377 | rcu_check_sparse(p, __rcu); \ | ||
| 378 | ((typeof(*p) __force __kernel *)(_________p1)); \ | ||
| 379 | }) | ||
| 380 | 381 | ||
| 381 | #define __rcu_access_pointer(p, space) \ | 382 | #define __rcu_access_pointer(p, local, space) \ |
| 382 | ({ \ | 383 | ({ \ |
| 383 | typeof(*p) *_________p1 = (typeof(*p) *__force)READ_ONCE(p); \ | 384 | typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ |
| 384 | rcu_check_sparse(p, space); \ | 385 | rcu_check_sparse(p, space); \ |
| 385 | ((typeof(*p) __force __kernel *)(_________p1)); \ | 386 | ((typeof(*p) __force __kernel *)(local)); \ |
| 386 | }) | 387 | }) |
| 387 | #define __rcu_dereference_check(p, c, space) \ | 388 | #define __rcu_dereference_check(p, local, c, space) \ |
| 388 | ({ \ | 389 | ({ \ |
| 389 | /* Dependency order vs. p above. */ \ | 390 | /* Dependency order vs. p above. */ \ |
| 390 | typeof(*p) *________p1 = (typeof(*p) *__force)READ_ONCE(p); \ | 391 | typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ |
| 391 | RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \ | 392 | RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \ |
| 392 | rcu_check_sparse(p, space); \ | 393 | rcu_check_sparse(p, space); \ |
| 393 | ((typeof(*p) __force __kernel *)(________p1)); \ | 394 | ((typeof(*p) __force __kernel *)(local)); \ |
| 394 | }) | 395 | }) |
| 395 | #define __rcu_dereference_protected(p, c, space) \ | 396 | #define __rcu_dereference_protected(p, local, c, space) \ |
| 396 | ({ \ | 397 | ({ \ |
| 397 | RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \ | 398 | RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \ |
| 398 | rcu_check_sparse(p, space); \ | 399 | rcu_check_sparse(p, space); \ |
| 399 | ((typeof(*p) __force __kernel *)(p)); \ | 400 | ((typeof(*p) __force __kernel *)(p)); \ |
| 400 | }) | 401 | }) |
| 401 | #define rcu_dereference_raw(p) \ | 402 | #define __rcu_dereference_raw(p, local) \ |
| 402 | ({ \ | 403 | ({ \ |
| 403 | /* Dependency order vs. p above. */ \ | 404 | /* Dependency order vs. p above. */ \ |
| 404 | typeof(p) ________p1 = READ_ONCE(p); \ | 405 | typeof(p) local = READ_ONCE(p); \ |
| 405 | ((typeof(*p) __force __kernel *)(________p1)); \ | 406 | ((typeof(*p) __force __kernel *)(local)); \ |
| 406 | }) | 407 | }) |
| 408 | #define rcu_dereference_raw(p) __rcu_dereference_raw(p, __UNIQUE_ID(rcu)) | ||
| 407 | 409 | ||
| 408 | /** | 410 | /** |
| 409 | * RCU_INITIALIZER() - statically initialize an RCU-protected global variable | 411 | * RCU_INITIALIZER() - statically initialize an RCU-protected global variable |
| @@ -490,7 +492,7 @@ do { \ · 80 unmodified lines | |||
| 490 | * when tearing down multi-linked structures after a grace period | 492 | * when tearing down multi-linked structures after a grace period |
| 491 | * has elapsed. | 493 | * has elapsed. |
| 492 | */ | 494 | */ |
| 493 | #define rcu_access_pointer(p) __rcu_access_pointer((p), __rcu) | 495 | #define rcu_access_pointer(p) __rcu_access_pointer((p), __UNIQUE_ID(rcu), __rcu) |
| 494 | 496 | ||
| 495 | /** | 497 | /** |
| 496 | * rcu_dereference_check() - rcu_dereference with debug checking | 498 | * rcu_dereference_check() - rcu_dereference with debug checking |
| @@ -526,7 +528,8 @@ do { \ · 29 unmodified lines | |||
| 526 | * annotated as __rcu. | 528 | * annotated as __rcu. |
| 527 | */ | 529 | */ |
| 528 | #define rcu_dereference_check(p, c) \ | 530 | #define rcu_dereference_check(p, c) \ |
| 529 | __rcu_dereference_check((p), (c) || rcu_read_lock_held(), __rcu) | 531 | __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ |
| 532 | (c) || rcu_read_lock_held(), __rcu) | ||
| 530 | 533 | ||
| 531 | /** | 534 | /** |
| 532 | * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking | 535 | * rcu_dereference_bh_check() - rcu_dereference_bh with debug checking |
| @@ -541,7 +544,8 @@ do { \ · 8 unmodified lines | |||
| 541 | * rcu_read_lock() but also rcu_read_lock_bh() into account. | 544 | * rcu_read_lock() but also rcu_read_lock_bh() into account. |
| 542 | */ | 545 | */ |
| 543 | #define rcu_dereference_bh_check(p, c) \ | 546 | #define rcu_dereference_bh_check(p, c) \ |
| 544 | __rcu_dereference_check((p), (c) || rcu_read_lock_bh_held(), __rcu) | 547 | __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ |
| 548 | (c) || rcu_read_lock_bh_held(), __rcu) | ||
| 545 | 549 | ||
| 546 | /** | 550 | /** |
| 547 | * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking | 551 | * rcu_dereference_sched_check() - rcu_dereference_sched with debug checking |
| @@ -556,7 +560,8 @@ do { \ · 8 unmodified lines | |||
| 556 | * only rcu_read_lock() but also rcu_read_lock_sched() into account. | 560 | * only rcu_read_lock() but also rcu_read_lock_sched() into account. |
| 557 | */ | 561 | */ |
| 558 | #define rcu_dereference_sched_check(p, c) \ | 562 | #define rcu_dereference_sched_check(p, c) \ |
| 559 | __rcu_dereference_check((p), (c) || rcu_read_lock_sched_held(), \ | 563 | __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ |
| 564 | (c) || rcu_read_lock_sched_held(), \ | ||
| 560 | __rcu) | 565 | __rcu) |
| 561 | 566 | ||
| 562 | /* | 567 | /* |
| @@ -566,7 +571,8 @@ do { \ · 3 unmodified lines | |||
| 566 | * The no-tracing version of rcu_dereference_raw() must not call | 571 | * The no-tracing version of rcu_dereference_raw() must not call |
| 567 | * rcu_read_lock_held(). | 572 | * rcu_read_lock_held(). |
| 568 | */ | 573 | */ |
| 569 | #define rcu_dereference_raw_check(p) __rcu_dereference_check((p), 1, __rcu) | 574 | #define rcu_dereference_raw_check(p) \ |
| 575 | __rcu_dereference_check((p), __UNIQUE_ID(rcu), 1, __rcu) | ||
| 570 | 576 | ||
| 571 | /** | 577 | /** |
| 572 | * rcu_dereference_protected() - fetch RCU pointer when updates prevented | 578 | * rcu_dereference_protected() - fetch RCU pointer when updates prevented |
| @@ -585,7 +591,7 @@ do { \ · 12 unmodified lines | |||
| 585 | * but very ugly failures. | 591 | * but very ugly failures. |
| 586 | */ | 592 | */ |
| 587 | #define rcu_dereference_protected(p, c) \ | 593 | #define rcu_dereference_protected(p, c) \ |
| 588 | __rcu_dereference_protected((p), (c), __rcu) | 594 | __rcu_dereference_protected((p), __UNIQUE_ID(rcu), (c), __rcu) |
| 589 | 595 | ||
| 590 | 596 | ||
| 591 | /** | 597 | /** |
| @@ -85,7 +85,7 @@ static inline void rcu_irq_enter_irqson(void) { } · 84 unmodified lines | |||
| 85 | static inline void rcu_irq_exit(void) { } | 85 | static inline void rcu_irq_exit(void) { } |
| 86 | static inline void rcu_irq_exit_check_preempt(void) { } | 86 | static inline void rcu_irq_exit_check_preempt(void) { } |
| 87 | #define rcu_is_idle_cpu(cpu) \ | 87 | #define rcu_is_idle_cpu(cpu) \ |
| 88 | (is_idle_task(current) && !in_nmi() && !in_irq() && !in_serving_softirq()) | 88 | (is_idle_task(current) && !in_nmi() && !in_hardirq() && !in_serving_softirq()) |
| 89 | static inline void exit_rcu(void) { } | 89 | static inline void exit_rcu(void) { } |
| 90 | static inline bool rcu_preempt_need_deferred_qs(struct task_struct *t) | 90 | static inline bool rcu_preempt_need_deferred_qs(struct task_struct *t) |
| 91 | { | 91 | { |
| @@ -117,7 +117,8 @@ static inline int srcu_read_lock_held(const struct srcu_struct *ssp) · 116 unmodified lines | |||
| 117 | * lockdep_is_held() calls. | 117 | * lockdep_is_held() calls. |
| 118 | */ | 118 | */ |
| 119 | #define srcu_dereference_check(p, ssp, c) \ | 119 | #define srcu_dereference_check(p, ssp, c) \ |
| 120 | __rcu_dereference_check((p), (c) || srcu_read_lock_held(ssp), __rcu) | 120 | __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ |
| 121 | (c) || srcu_read_lock_held(ssp), __rcu) | ||
| 121 | 122 | ||
| 122 | /** | 123 | /** |
| 123 | * srcu_dereference - fetch SRCU-protected pointer for later dereferencing | 124 | * srcu_dereference - fetch SRCU-protected pointer for later dereferencing |
| @@ -38,13 +38,8 @@ do { \ · 37 unmodified lines | |||
| 38 | pr_alert("%s" TORTURE_FLAG " %s\n", torture_type, s); \ | 38 | pr_alert("%s" TORTURE_FLAG " %s\n", torture_type, s); \ |
| 39 | } \ | 39 | } \ |
| 40 | } while (0) | 40 | } while (0) |
| 41 | #define VERBOSE_TOROUT_ERRSTRING(s) \ | 41 | #define TOROUT_ERRSTRING(s) \ |
| 42 | do { \ | 42 | pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s) |
| 43 | if (verbose) { \ | ||
| 44 | verbose_torout_sleep(); \ | ||
| 45 | pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s); \ | ||
| 46 | } \ | ||
| 47 | } while (0) | ||
| 48 | void verbose_torout_sleep(void); | 43 | void verbose_torout_sleep(void); |
| 49 | 44 | ||
| 50 | #define torture_init_error(firsterr) \ | 45 | #define torture_init_error(firsterr) \ |