rcutorture.2026.05.24: Torture-test updates misc.2026.05.24: Miscellaneous RCU updates
Unified · Split
| @@ -5862,13 +5862,13 @@ Kernel parameters · 5861 unmodified lines | |||
| 5862 | use a call_rcu[_hurry]() path. Please note, this is for a | 5862 | use a call_rcu[_hurry]() path. Please note, this is for a |
| 5863 | normal grace period. | 5863 | normal grace period. |
| 5864 | 5864 | ||
| 5865 | How to enable it: | 5865 | How to disable it: |
| 5866 | 5866 | ||
| 5867 | echo 1 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp | 5867 | echo 0 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp |
| 5868 | or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=1" | 5868 | or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=0" |
| 5869 | 5869 | ||
| 5870 | Default is 1 if num_possible_cpus() <= 16 and it is not explicitly | 5870 | Default is 1 if it is not explicitly disabled by the boot parameter |
| 5871 | disabled by the boot parameter passing 0. | 5871 | passing 0. |
| 5872 | 5872 | ||
| 5873 | rcuscale.gp_async= [KNL] | 5873 | rcuscale.gp_async= [KNL] |
| 5874 | Measure performance of asynchronous | 5874 | Measure performance of asynchronous |
| @@ -592,11 +592,13 @@ context_unsafe( \ · 591 unmodified lines | |||
| 592 | * lockdep checks for being in an RCU read-side critical section. This is | 592 | * lockdep checks for being in an RCU read-side critical section. This is |
| 593 | * useful when the value of this pointer is accessed, but the pointer is | 593 | * useful when the value of this pointer is accessed, but the pointer is |
| 594 | * not dereferenced, for example, when testing an RCU-protected pointer | 594 | * not dereferenced, for example, when testing an RCU-protected pointer |
| 595 | * against NULL. Although rcu_access_pointer() may also be used in cases | 595 | * against NULL. Within an RCU read-side critical section, there is little |
| 596 | * where update-side locks prevent the value of the pointer from changing, | 596 | * reason to use rcu_access_pointer(). Although rcu_access_pointer() may |
| 597 | * you should instead use rcu_dereference_protected() for this use case. | 597 | * also be used in cases where update-side locks prevent the value of the |
| 598 | * Within an RCU read-side critical section, there is little reason to | 598 | * pointer from changing, you should instead use rcu_dereference_protected() |
| 599 | * use rcu_access_pointer(). | 599 | * for this use case. It is also permissible to use rcu_access_pointer() |
| 600 | * within lockless updaters to obtain the old value for an atomic operation, | ||
| 601 | * for example, for cmpxchg(). | ||
| 600 | * | 602 | * |
| 601 | * It is usually best to test the rcu_access_pointer() return value | 603 | * It is usually best to test the rcu_access_pointer() return value |
| 602 | * directly in order to avoid accidental dereferences being introduced | 604 | * directly in order to avoid accidental dereferences being introduced |
| @@ -397,7 +397,7 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_ · 396 unmodified lines | |||
| 397 | * | 397 | * |
| 398 | * The same srcu_struct may be used concurrently by srcu_down_read_fast() | 398 | * The same srcu_struct may be used concurrently by srcu_down_read_fast() |
| 399 | * and srcu_read_lock_fast(). However, the same definition/initialization | 399 | * and srcu_read_lock_fast(). However, the same definition/initialization |
| 400 | * requirements called out for srcu_read_lock_safe() apply. | 400 | * requirements called out for srcu_read_lock_fast_updown() apply. |
| 401 | */ | 401 | */ |
| 402 | static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires_shared(ssp) | 402 | static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires_shared(ssp) |
| 403 | { | 403 | { |
| @@ -373,7 +373,8 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, · 372 unmodified lines | |||
| 373 | // Queuing callbacks before initialization not yet supported. | 373 | // Queuing callbacks before initialization not yet supported. |
| 374 | if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist))) | 374 | if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist))) |
| 375 | rcu_segcblist_init(&rtpcp->cblist); | 375 | rcu_segcblist_init(&rtpcp->cblist); |
| 376 | needwake = (func == wakeme_after_rcu) || | 376 | needwake = (!havekthread && rcu_segcblist_empty(&rtpcp->cblist)) || |
| 377 | (func == wakeme_after_rcu) || | ||
| 377 | (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim); | 378 | (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim); |
| 378 | if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) { | 379 | if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) { |
| 379 | if (rtp->lazy_jiffies) | 380 | if (rtp->lazy_jiffies) |
| @@ -492,7 +492,7 @@ static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param · 491 unmodified lines | |||
| 492 | int ret = kstrtoul(val, 0, &j); | 492 | int ret = kstrtoul(val, 0, &j); |
| 493 | 493 | ||
| 494 | if (!ret) { | 494 | if (!ret) { |
| 495 | WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : (j ?: 1)); | 495 | WRITE_ONCE(*(ulong *)kp->arg, clamp_val(j, 1, HZ)); |
| 496 | adjust_jiffies_till_sched_qs(); | 496 | adjust_jiffies_till_sched_qs(); |
| 497 | } | 497 | } |
| 498 | return ret; | 498 | return ret; |
| @@ -1632,17 +1632,21 @@ static void rcu_sr_put_wait_head(struct llist_node *node) · 1133 unmodified lines | |||
| 1632 | atomic_set_release(&sr_wn->inuse, 0); | 1632 | atomic_set_release(&sr_wn->inuse, 0); |
| 1633 | } | 1633 | } |
| 1634 | 1634 | ||
| 1635 | /* Enable rcu_normal_wake_from_gp automatically on small systems. */ | 1635 | static int rcu_normal_wake_from_gp = 1; |
| 1636 | #define WAKE_FROM_GP_CPU_THRESHOLD 16 | ||
| 1637 | |||
| 1638 | static int rcu_normal_wake_from_gp = -1; | ||
| 1639 | module_param(rcu_normal_wake_from_gp, int, 0644); | 1636 | module_param(rcu_normal_wake_from_gp, int, 0644); |
| 1640 | static struct workqueue_struct *sync_wq; | 1637 | static struct workqueue_struct *sync_wq; |
| 1641 | 1638 | ||
| 1639 | #define RCU_SR_NORMAL_LATCH_THR 64 | ||
| 1640 | |||
| 1641 | /* Number of in-flight synchronize_rcu() calls queued on srs_next. */ | ||
| 1642 | static atomic_long_t rcu_sr_normal_count; | ||
| 1643 | static int rcu_sr_normal_latched; /* 0/1 */ | ||
| 1644 | |||
| 1642 | static void rcu_sr_normal_complete(struct llist_node *node) | 1645 | static void rcu_sr_normal_complete(struct llist_node *node) |
| 1643 | { | 1646 | { |
| 1644 | struct rcu_synchronize *rs = container_of( | 1647 | struct rcu_synchronize *rs = container_of( |
| 1645 | (struct rcu_head *) node, struct rcu_synchronize, head); | 1648 | (struct rcu_head *) node, struct rcu_synchronize, head); |
| 1649 | long nr; | ||
| 1646 | 1650 | ||
| 1647 | WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && | 1651 | WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && |
| 1648 | !poll_state_synchronize_rcu_full(&rs->oldstate), | 1652 | !poll_state_synchronize_rcu_full(&rs->oldstate), |
| @@ -1650,6 +1654,15 @@ static void rcu_sr_normal_complete(struct llist_node *node) · 1 unmodified line | |||
| 1650 | 1654 | ||
| 1651 | /* Finally. */ | 1655 | /* Finally. */ |
| 1652 | complete(&rs->completion); | 1656 | complete(&rs->completion); |
| 1657 | nr = atomic_long_dec_return(&rcu_sr_normal_count); | ||
| 1658 | WARN_ON_ONCE(nr < 0); | ||
| 1659 | |||
| 1660 | /* | ||
| 1661 | * Unlatch: switch back to normal path when fully | ||
| 1662 | * drained and if it has been latched. | ||
| 1663 | */ | ||
| 1664 | if (nr == 0) | ||
| 1665 | (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 1, 0); | ||
| 1653 | } | 1666 | } |
| 1654 | 1667 | ||
| 1655 | static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work) | 1668 | static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work) |
| @@ -1795,6 +1808,24 @@ static bool rcu_sr_normal_gp_init(void) · 139 unmodified lines | |||
| 1795 | 1808 | ||
| 1796 | static void rcu_sr_normal_add_req(struct rcu_synchronize *rs) | 1809 | static void rcu_sr_normal_add_req(struct rcu_synchronize *rs) |
| 1797 | { | 1810 | { |
| 1811 | /* | ||
| 1812 | * Increment before publish to avoid a complete | ||
| 1813 | * vs enqueue race on latch. | ||
| 1814 | */ | ||
| 1815 | long nr = atomic_long_inc_return(&rcu_sr_normal_count); | ||
| 1816 | |||
| 1817 | /* | ||
| 1818 | * Latch when threshold is reached. Checking for an exact match | ||
| 1819 | * restricts cmpxchg() to a single context. | ||
| 1820 | * | ||
| 1821 | * This latch is intentionally relaxed and best-effort. Concurrent | ||
| 1822 | * set/clear can race and temporarily lose the latch, which is OK | ||
| 1823 | * because it only selects between the fast and fallback paths. | ||
| 1824 | */ | ||
| 1825 | if (nr == RCU_SR_NORMAL_LATCH_THR) | ||
| 1826 | (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 0, 1); | ||
| 1827 | |||
| 1828 | /* Publish for the GP kthread/worker. */ | ||
| 1798 | llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next); | 1829 | llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next); |
| 1799 | } | 1830 | } |
| 1800 | 1831 | ||
| @@ -2584,7 +2615,7 @@ static void rcu_do_batch(struct rcu_data *rdp) · 783 unmodified lines | |||
| 2584 | const long npj = NSEC_PER_SEC / HZ; | 2615 | const long npj = NSEC_PER_SEC / HZ; |
| 2585 | long rrn = READ_ONCE(rcu_resched_ns); | 2616 | long rrn = READ_ONCE(rcu_resched_ns); |
| 2586 | 2617 | ||
| 2587 | rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn; | 2618 | rrn = clamp(rrn, NSEC_PER_MSEC, NSEC_PER_SEC); |
| 2588 | tlimit = local_clock() + rrn; | 2619 | tlimit = local_clock() + rrn; |
| 2589 | jlimit = jiffies + (rrn + npj + 1) / npj; | 2620 | jlimit = jiffies + (rrn + npj + 1) / npj; |
| 2590 | jlimit_check = true; | 2621 | jlimit_check = true; |
| @@ -3278,14 +3309,15 @@ static void synchronize_rcu_normal(void) · 687 unmodified lines | |||
| 3278 | { | 3309 | { |
| 3279 | struct rcu_synchronize rs; | 3310 | struct rcu_synchronize rs; |
| 3280 | 3311 | ||
| 3312 | init_rcu_head_on_stack(&rs.head); | ||
| 3281 | trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request")); | 3313 | trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request")); |
| 3282 | 3314 | ||
| 3283 | if (READ_ONCE(rcu_normal_wake_from_gp) < 1) { | 3315 | if (READ_ONCE(rcu_normal_wake_from_gp) < 1 || |
| 3316 | READ_ONCE(rcu_sr_normal_latched)) { | ||
| 3284 | wait_rcu_gp(call_rcu_hurry); | 3317 | wait_rcu_gp(call_rcu_hurry); |
| 3285 | goto trace_complete_out; | 3318 | goto trace_complete_out; |
| 3286 | } | 3319 | } |
| 3287 | 3320 | ||
| 3288 | init_rcu_head_on_stack(&rs.head); | ||
| 3289 | init_completion(&rs.completion); | 3321 | init_completion(&rs.completion); |
| 3290 | 3322 | ||
| 3291 | /* | 3323 | /* |
| @@ -3302,10 +3334,10 @@ static void synchronize_rcu_normal(void) · 10 unmodified lines | |||
| 3302 | 3334 | ||
| 3303 | /* Now we can wait. */ | 3335 | /* Now we can wait. */ |
| 3304 | wait_for_completion(&rs.completion); | 3336 | wait_for_completion(&rs.completion); |
| 3305 | destroy_rcu_head_on_stack(&rs.head); | ||
| 3306 | 3337 | ||
| 3307 | trace_complete_out: | 3338 | trace_complete_out: |
| 3308 | trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete")); | 3339 | trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete")); |
| 3340 | destroy_rcu_head_on_stack(&rs.head); | ||
| 3309 | } | 3341 | } |
| 3310 | 3342 | ||
| 3311 | /** | 3343 | /** |
| @@ -4904,12 +4936,6 @@ void __init rcu_init(void) · 1592 unmodified lines | |||
| 4904 | sync_wq = alloc_workqueue("sync_wq", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); | 4936 | sync_wq = alloc_workqueue("sync_wq", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); |
| 4905 | WARN_ON(!sync_wq); | 4937 | WARN_ON(!sync_wq); |
| 4906 | 4938 | ||
| 4907 | /* Respect if explicitly disabled via a boot parameter. */ | ||
| 4908 | if (rcu_normal_wake_from_gp < 0) { | ||
| 4909 | if (num_possible_cpus() <= WAKE_FROM_GP_CPU_THRESHOLD) | ||
| 4910 | rcu_normal_wake_from_gp = 1; | ||
| 4911 | } | ||
| 4912 | |||
| 4913 | /* Fill in default value for rcutree.qovld boot parameter. */ | 4939 | /* Fill in default value for rcutree.qovld boot parameter. */ |
| 4914 | /* -After- the rcu_node ->lock fields are initialized! */ | 4940 | /* -After- the rcu_node ->lock fields are initialized! */ |
| 4915 | if (qovld < 0) | 4941 | if (qovld < 0) |
| @@ -655,7 +655,7 @@ static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) · 654 unmodified lines | |||
| 655 | * No-CBs GP kthreads come here to wait for additional callbacks to show up | 655 | * No-CBs GP kthreads come here to wait for additional callbacks to show up |
| 656 | * or for grace periods to end. | 656 | * or for grace periods to end. |
| 657 | */ | 657 | */ |
| 658 | static void nocb_gp_wait(struct rcu_data *my_rdp) | 658 | static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) |
| 659 | { | 659 | { |
| 660 | bool bypass = false; | 660 | bool bypass = false; |
| 661 | int __maybe_unused cpu = my_rdp->cpu; | 661 | int __maybe_unused cpu = my_rdp->cpu; |
| @@ -865,8 +865,6 @@ our %deprecated_apis = ( · 864 unmodified lines | |||
| 865 | "DEFINE_IDR" => "DEFINE_XARRAY", | 865 | "DEFINE_IDR" => "DEFINE_XARRAY", |
| 866 | "idr_init" => "xa_init", | 866 | "idr_init" => "xa_init", |
| 867 | "idr_init_base" => "xa_init_flags", | 867 | "idr_init_base" => "xa_init_flags", |
| 868 | "rcu_read_lock_trace" => "rcu_read_lock_tasks_trace", | ||
| 869 | "rcu_read_unlock_trace" => "rcu_read_unlock_tasks_trace", | ||
| 870 | ); | 868 | ); |
| 871 | 869 | ||
| 872 | #Create a search pattern for all these strings to speed up a loop below | 870 | #Create a search pattern for all these strings to speed up a loop below |
| @@ -7596,12 +7594,15 @@ sub process { · 6723 unmodified lines | |||
| 7596 | 7594 | ||
| 7597 | # Complain about RCU Tasks Trace used outside of BPF (and of course, RCU). | 7595 | # Complain about RCU Tasks Trace used outside of BPF (and of course, RCU). |
| 7598 | our $rcu_trace_funcs = qr{(?x: | 7596 | our $rcu_trace_funcs = qr{(?x: |
| 7597 | rcu_read_lock_tasks_trace | | ||
| 7599 | rcu_read_lock_trace | | 7598 | rcu_read_lock_trace | |
| 7600 | rcu_read_lock_trace_held | | 7599 | rcu_read_lock_trace_held | |
| 7601 | rcu_read_unlock_trace | | 7600 | rcu_read_unlock_trace | |
| 7601 | rcu_read_unlock_tasks_trace | | ||
| 7602 | call_rcu_tasks_trace | | 7602 | call_rcu_tasks_trace | |
| 7603 | synchronize_rcu_tasks_trace | | 7603 | synchronize_rcu_tasks_trace | |
| 7604 | rcu_barrier_tasks_trace | | 7604 | rcu_barrier_tasks_trace | |
| 7605 | rcu_tasks_trace_expedite_current | | ||
| 7605 | rcu_request_urgent_qs_task | 7606 | rcu_request_urgent_qs_task |
| 7606 | )}; | 7607 | )}; |
| 7607 | our $rcu_trace_paths = qr{(?x: | 7608 | our $rcu_trace_paths = qr{(?x: |