rcutorture.2026.05.24: Torture-test updates misc.2026.05.24: Miscellaneous RCU updates
Unified · Split
| @@ -5862,13 +5862,13 @@ Kernel parameters · 5861 unmodified lines | ||
| 5862 | 5862 | use a call_rcu[_hurry]() path. Please note, this is for a |
| 5863 | 5863 | normal grace period. |
| 5864 | 5864 | |
| 5865 | − How to enable it: | |
| 5865 | + How to disable it: | |
| 5866 | 5866 | |
| 5867 | − echo 1 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp | |
| 5868 | − or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=1" | |
| 5867 | + echo 0 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp | |
| 5868 | + or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=0" | |
| 5869 | 5869 | |
| 5870 | − Default is 1 if num_possible_cpus() <= 16 and it is not explicitly | |
| 5871 | − disabled by the boot parameter passing 0. | |
| 5870 | + Default is 1 if it is not explicitly disabled by the boot parameter | |
| 5871 | + passing 0. | |
| 5872 | 5872 | |
| 5873 | 5873 | rcuscale.gp_async= [KNL] |
| 5874 | 5874 | Measure performance of asynchronous |
| @@ -592,11 +592,13 @@ context_unsafe( \ · 591 unmodified lines | ||
| 592 | 592 | * lockdep checks for being in an RCU read-side critical section. This is |
| 593 | 593 | * useful when the value of this pointer is accessed, but the pointer is |
| 594 | 594 | * not dereferenced, for example, when testing an RCU-protected pointer |
| 595 | − * against NULL. Although rcu_access_pointer() may also be used in cases | |
| 596 | − * where update-side locks prevent the value of the pointer from changing, | |
| 597 | − * you should instead use rcu_dereference_protected() for this use case. | |
| 598 | − * Within an RCU read-side critical section, there is little reason to | |
| 599 | − * use rcu_access_pointer(). | |
| 595 | + * against NULL. Within an RCU read-side critical section, there is little | |
| 596 | + * reason to use rcu_access_pointer(). Although rcu_access_pointer() may | |
| 597 | + * also be used in cases where update-side locks prevent the value of the | |
| 598 | + * pointer from changing, you should instead use rcu_dereference_protected() | |
| 599 | + * for this use case. It is also permissible to use rcu_access_pointer() | |
| 600 | + * within lockless updaters to obtain the old value for an atomic operation, | |
| 601 | + * for example, for cmpxchg(). | |
| 600 | 602 | * |
| 601 | 603 | * It is usually best to test the rcu_access_pointer() return value |
| 602 | 604 | * directly in order to avoid accidental dereferences being introduced |
| @@ -397,7 +397,7 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_ · 396 unmodified lines | ||
| 397 | 397 | * |
| 398 | 398 | * The same srcu_struct may be used concurrently by srcu_down_read_fast() |
| 399 | 399 | * and srcu_read_lock_fast(). However, the same definition/initialization |
| 400 | − * requirements called out for srcu_read_lock_safe() apply. | |
| 400 | + * requirements called out for srcu_read_lock_fast_updown() apply. | |
| 401 | 401 | */ |
| 402 | 402 | static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires_shared(ssp) |
| 403 | 403 | { |
| @@ -373,7 +373,8 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, · 372 unmodified lines | ||
| 373 | 373 | // Queuing callbacks before initialization not yet supported. |
| 374 | 374 | if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist))) |
| 375 | 375 | rcu_segcblist_init(&rtpcp->cblist); |
| 376 | − needwake = (func == wakeme_after_rcu) || | |
| 376 | + needwake = (!havekthread && rcu_segcblist_empty(&rtpcp->cblist)) || | |
| 377 | + (func == wakeme_after_rcu) || | |
| 377 | 378 | (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim); |
| 378 | 379 | if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) { |
| 379 | 380 | if (rtp->lazy_jiffies) |
| @@ -492,7 +492,7 @@ static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param · 491 unmodified lines | ||
| 492 | 492 | int ret = kstrtoul(val, 0, &j); |
| 493 | 493 | |
| 494 | 494 | if (!ret) { |
| 495 | − WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : (j ?: 1)); | |
| 495 | + WRITE_ONCE(*(ulong *)kp->arg, clamp_val(j, 1, HZ)); | |
| 496 | 496 | adjust_jiffies_till_sched_qs(); |
| 497 | 497 | } |
| 498 | 498 | return ret; |
| @@ -1632,17 +1632,21 @@ static void rcu_sr_put_wait_head(struct llist_node *node) · 1133 unmodified lines | ||
| 1632 | 1632 | atomic_set_release(&sr_wn->inuse, 0); |
| 1633 | 1633 | } |
| 1634 | 1634 | |
| 1635 | −/* Enable rcu_normal_wake_from_gp automatically on small systems. */ | |
| 1636 | −#define WAKE_FROM_GP_CPU_THRESHOLD 16 | |
| 1637 | − | |
| 1638 | −static int rcu_normal_wake_from_gp = -1; | |
| 1635 | +static int rcu_normal_wake_from_gp = 1; | |
| 1639 | 1636 | module_param(rcu_normal_wake_from_gp, int, 0644); |
| 1640 | 1637 | static struct workqueue_struct *sync_wq; |
| 1641 | 1638 | |
| 1639 | +#define RCU_SR_NORMAL_LATCH_THR 64 | |
| 1640 | + | |
| 1641 | +/* Number of in-flight synchronize_rcu() calls queued on srs_next. */ | |
| 1642 | +static atomic_long_t rcu_sr_normal_count; | |
| 1643 | +static int rcu_sr_normal_latched; /* 0/1 */ | |
| 1644 | + | |
| 1642 | 1645 | static void rcu_sr_normal_complete(struct llist_node *node) |
| 1643 | 1646 | { |
| 1644 | 1647 | struct rcu_synchronize *rs = container_of( |
| 1645 | 1648 | (struct rcu_head *) node, struct rcu_synchronize, head); |
| 1649 | + long nr; | |
| 1646 | 1650 | |
| 1647 | 1651 | WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && |
| 1648 | 1652 | !poll_state_synchronize_rcu_full(&rs->oldstate), |
| @@ -1650,6 +1654,15 @@ static void rcu_sr_normal_complete(struct llist_node *node) · 1 unmodified line | ||
| 1650 | 1654 | |
| 1651 | 1655 | /* Finally. */ |
| 1652 | 1656 | complete(&rs->completion); |
| 1657 | + nr = atomic_long_dec_return(&rcu_sr_normal_count); | |
| 1658 | + WARN_ON_ONCE(nr < 0); | |
| 1659 | + | |
| 1660 | + /* | |
| 1661 | + * Unlatch: switch back to normal path when fully | |
| 1662 | + * drained and if it has been latched. | |
| 1663 | + */ | |
| 1664 | + if (nr == 0) | |
| 1665 | + (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 1, 0); | |
| 1653 | 1666 | } |
| 1654 | 1667 | |
| 1655 | 1668 | static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work) |
| @@ -1795,6 +1808,24 @@ static bool rcu_sr_normal_gp_init(void) · 139 unmodified lines | ||
| 1795 | 1808 | |
| 1796 | 1809 | static void rcu_sr_normal_add_req(struct rcu_synchronize *rs) |
| 1797 | 1810 | { |
| 1811 | + /* | |
| 1812 | + * Increment before publish to avoid a complete | |
| 1813 | + * vs enqueue race on latch. | |
| 1814 | + */ | |
| 1815 | + long nr = atomic_long_inc_return(&rcu_sr_normal_count); | |
| 1816 | + | |
| 1817 | + /* | |
| 1818 | + * Latch when threshold is reached. Checking for an exact match | |
| 1819 | + * restricts cmpxchg() to a single context. | |
| 1820 | + * | |
| 1821 | + * This latch is intentionally relaxed and best-effort. Concurrent | |
| 1822 | + * set/clear can race and temporarily lose the latch, which is OK | |
| 1823 | + * because it only selects between the fast and fallback paths. | |
| 1824 | + */ | |
| 1825 | + if (nr == RCU_SR_NORMAL_LATCH_THR) | |
| 1826 | + (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 0, 1); | |
| 1827 | + | |
| 1828 | + /* Publish for the GP kthread/worker. */ | |
| 1798 | 1829 | llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next); |
| 1799 | 1830 | } |
| 1800 | 1831 | |
| @@ -2584,7 +2615,7 @@ static void rcu_do_batch(struct rcu_data *rdp) · 783 unmodified lines | ||
| 2584 | 2615 | const long npj = NSEC_PER_SEC / HZ; |
| 2585 | 2616 | long rrn = READ_ONCE(rcu_resched_ns); |
| 2586 | 2617 | |
| 2587 | − rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn; | |
| 2618 | + rrn = clamp(rrn, NSEC_PER_MSEC, NSEC_PER_SEC); | |
| 2588 | 2619 | tlimit = local_clock() + rrn; |
| 2589 | 2620 | jlimit = jiffies + (rrn + npj + 1) / npj; |
| 2590 | 2621 | jlimit_check = true; |
| @@ -3278,14 +3309,15 @@ static void synchronize_rcu_normal(void) · 687 unmodified lines | ||
| 3278 | 3309 | { |
| 3279 | 3310 | struct rcu_synchronize rs; |
| 3280 | 3311 | |
| 3312 | + init_rcu_head_on_stack(&rs.head); | |
| 3281 | 3313 | trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request")); |
| 3282 | 3314 | |
| 3283 | − if (READ_ONCE(rcu_normal_wake_from_gp) < 1) { | |
| 3315 | + if (READ_ONCE(rcu_normal_wake_from_gp) < 1 || | |
| 3316 | + READ_ONCE(rcu_sr_normal_latched)) { | |
| 3284 | 3317 | wait_rcu_gp(call_rcu_hurry); |
| 3285 | 3318 | goto trace_complete_out; |
| 3286 | 3319 | } |
| 3287 | 3320 | |
| 3288 | − init_rcu_head_on_stack(&rs.head); | |
| 3289 | 3321 | init_completion(&rs.completion); |
| 3290 | 3322 | |
| 3291 | 3323 | /* |
| @@ -3302,10 +3334,10 @@ static void synchronize_rcu_normal(void) · 10 unmodified lines | ||
| 3302 | 3334 | |
| 3303 | 3335 | /* Now we can wait. */ |
| 3304 | 3336 | wait_for_completion(&rs.completion); |
| 3305 | − destroy_rcu_head_on_stack(&rs.head); | |
| 3306 | 3337 | |
| 3307 | 3338 | trace_complete_out: |
| 3308 | 3339 | trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete")); |
| 3340 | + destroy_rcu_head_on_stack(&rs.head); | |
| 3309 | 3341 | } |
| 3310 | 3342 | |
| 3311 | 3343 | /** |
| @@ -4904,12 +4936,6 @@ void __init rcu_init(void) · 1592 unmodified lines | ||
| 4904 | 4936 | sync_wq = alloc_workqueue("sync_wq", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); |
| 4905 | 4937 | WARN_ON(!sync_wq); |
| 4906 | 4938 | |
| 4907 | − /* Respect if explicitly disabled via a boot parameter. */ | |
| 4908 | − if (rcu_normal_wake_from_gp < 0) { | |
| 4909 | − if (num_possible_cpus() <= WAKE_FROM_GP_CPU_THRESHOLD) | |
| 4910 | − rcu_normal_wake_from_gp = 1; | |
| 4911 | − } | |
| 4912 | − | |
| 4913 | 4939 | /* Fill in default value for rcutree.qovld boot parameter. */ |
| 4914 | 4940 | /* -After- the rcu_node ->lock fields are initialized! */ |
| 4915 | 4941 | if (qovld < 0) |
| @@ -655,7 +655,7 @@ static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) · 654 unmodified lines | ||
| 655 | 655 | * No-CBs GP kthreads come here to wait for additional callbacks to show up |
| 656 | 656 | * or for grace periods to end. |
| 657 | 657 | */ |
| 658 | −static void nocb_gp_wait(struct rcu_data *my_rdp) | |
| 658 | +static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp) | |
| 659 | 659 | { |
| 660 | 660 | bool bypass = false; |
| 661 | 661 | int __maybe_unused cpu = my_rdp->cpu; |
| @@ -865,8 +865,6 @@ our %deprecated_apis = ( · 864 unmodified lines | ||
| 865 | 865 | "DEFINE_IDR" => "DEFINE_XARRAY", |
| 866 | 866 | "idr_init" => "xa_init", |
| 867 | 867 | "idr_init_base" => "xa_init_flags", |
| 868 | − "rcu_read_lock_trace" => "rcu_read_lock_tasks_trace", | |
| 869 | − "rcu_read_unlock_trace" => "rcu_read_unlock_tasks_trace", | |
| 870 | 868 | ); |
| 871 | 869 | |
| 872 | 870 | #Create a search pattern for all these strings to speed up a loop below |
| @@ -7596,12 +7594,15 @@ sub process { · 6723 unmodified lines | ||
| 7596 | 7594 | |
| 7597 | 7595 | # Complain about RCU Tasks Trace used outside of BPF (and of course, RCU). |
| 7598 | 7596 | our $rcu_trace_funcs = qr{(?x: |
| 7597 | + rcu_read_lock_tasks_trace | | |
| 7599 | 7598 | rcu_read_lock_trace | |
| 7600 | 7599 | rcu_read_lock_trace_held | |
| 7601 | 7600 | rcu_read_unlock_trace | |
| 7601 | + rcu_read_unlock_tasks_trace | | |
| 7602 | 7602 | call_rcu_tasks_trace | |
| 7603 | 7603 | synchronize_rcu_tasks_trace | |
| 7604 | 7604 | rcu_barrier_tasks_trace | |
| 7605 | + rcu_tasks_trace_expedite_current | | |
| 7605 | 7606 | rcu_request_urgent_qs_task |
| 7606 | 7607 | )}; |
| 7607 | 7608 | our $rcu_trace_paths = qr{(?x: |