History

Merge branches 'rcutorture.2026.05.24' and 'misc.2026.05.24' into rcu-merge.2026.05.24

rcutorture.2026.05.24: Torture-test updates misc.2026.05.24: Miscellaneous RCU updates

Uladzislau Rezki (Sony) committed 5 months ago

Commit e853c1b2 parents 956d852 00266887 files · +60 −30Browse files

Showing the changes to kernel (3 of 7 files). Show all

Unified · Split

3 files
  1. kernel/rcu/tasks.h +2 −1
  2. kernel/rcu/tree.c +41 −15
  3. kernel/rcu/tree_nocb.h +1 −1
kernel/rcu/tasks.h+2−1
@@ -373,7 +373,8 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, · 372 unmodified lines
373 // Queuing callbacks before initialization not yet supported.373 // Queuing callbacks before initialization not yet supported.
374 if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist)))374 if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist)))
375 rcu_segcblist_init(&rtpcp->cblist);375 rcu_segcblist_init(&rtpcp->cblist);
376 needwake = (func == wakeme_after_rcu) ||376 needwake = (!havekthread && rcu_segcblist_empty(&rtpcp->cblist)) ||
377 (func == wakeme_after_rcu) ||
377 (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim);378 (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim);
378 if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) {379 if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) {
379 if (rtp->lazy_jiffies)380 if (rtp->lazy_jiffies)
kernel/rcu/tree.c+41−15
@@ -492,7 +492,7 @@ static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param · 491 unmodified lines
492 int ret = kstrtoul(val, 0, &j);492 int ret = kstrtoul(val, 0, &j);
493 493
494 if (!ret) {494 if (!ret) {
495 WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : (j ?: 1));495 WRITE_ONCE(*(ulong *)kp->arg, clamp_val(j, 1, HZ));
496 adjust_jiffies_till_sched_qs();496 adjust_jiffies_till_sched_qs();
497 }497 }
498 return ret;498 return ret;
@@ -1632,17 +1632,21 @@ static void rcu_sr_put_wait_head(struct llist_node *node) · 1133 unmodified lines
1632 atomic_set_release(&sr_wn->inuse, 0);1632 atomic_set_release(&sr_wn->inuse, 0);
1633}1633}
1634 1634
1635/* Enable rcu_normal_wake_from_gp automatically on small systems. */1635static int rcu_normal_wake_from_gp = 1;
1636#define WAKE_FROM_GP_CPU_THRESHOLD 16
1637
1638static int rcu_normal_wake_from_gp = -1;
1639module_param(rcu_normal_wake_from_gp, int, 0644);1636module_param(rcu_normal_wake_from_gp, int, 0644);
1640static struct workqueue_struct *sync_wq;1637static struct workqueue_struct *sync_wq;
1641 1638
1639#define RCU_SR_NORMAL_LATCH_THR 64
1640
1641/* Number of in-flight synchronize_rcu() calls queued on srs_next. */
1642static atomic_long_t rcu_sr_normal_count;
1643static int rcu_sr_normal_latched; /* 0/1 */
1644
1642static void rcu_sr_normal_complete(struct llist_node *node)1645static void rcu_sr_normal_complete(struct llist_node *node)
1643{1646{
1644 struct rcu_synchronize *rs = container_of(1647 struct rcu_synchronize *rs = container_of(
1645 (struct rcu_head *) node, struct rcu_synchronize, head);1648 (struct rcu_head *) node, struct rcu_synchronize, head);
1649 long nr;
1646 1650
1647 WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) &&1651 WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) &&
1648 !poll_state_synchronize_rcu_full(&rs->oldstate),1652 !poll_state_synchronize_rcu_full(&rs->oldstate),
@@ -1650,6 +1654,15 @@ static void rcu_sr_normal_complete(struct llist_node *node) · 1 unmodified line
1650 1654
1651 /* Finally. */1655 /* Finally. */
1652 complete(&rs->completion);1656 complete(&rs->completion);
1657 nr = atomic_long_dec_return(&rcu_sr_normal_count);
1658 WARN_ON_ONCE(nr < 0);
1659
1660 /*
1661 * Unlatch: switch back to normal path when fully
1662 * drained and if it has been latched.
1663 */
1664 if (nr == 0)
1665 (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 1, 0);
1653}1666}
1654 1667
1655static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work)1668static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work)
@@ -1795,6 +1808,24 @@ static bool rcu_sr_normal_gp_init(void) · 139 unmodified lines
1795 1808
1796static void rcu_sr_normal_add_req(struct rcu_synchronize *rs)1809static void rcu_sr_normal_add_req(struct rcu_synchronize *rs)
1797{1810{
1811 /*
1812 * Increment before publish to avoid a complete
1813 * vs enqueue race on latch.
1814 */
1815 long nr = atomic_long_inc_return(&rcu_sr_normal_count);
1816
1817 /*
1818 * Latch when threshold is reached. Checking for an exact match
1819 * restricts cmpxchg() to a single context.
1820 *
1821 * This latch is intentionally relaxed and best-effort. Concurrent
1822 * set/clear can race and temporarily lose the latch, which is OK
1823 * because it only selects between the fast and fallback paths.
1824 */
1825 if (nr == RCU_SR_NORMAL_LATCH_THR)
1826 (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 0, 1);
1827
1828 /* Publish for the GP kthread/worker. */
1798 llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next);1829 llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next);
1799}1830}
1800 1831
@@ -2584,7 +2615,7 @@ static void rcu_do_batch(struct rcu_data *rdp) · 783 unmodified lines
2584 const long npj = NSEC_PER_SEC / HZ;2615 const long npj = NSEC_PER_SEC / HZ;
2585 long rrn = READ_ONCE(rcu_resched_ns);2616 long rrn = READ_ONCE(rcu_resched_ns);
2586 2617
2587 rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;2618 rrn = clamp(rrn, NSEC_PER_MSEC, NSEC_PER_SEC);
2588 tlimit = local_clock() + rrn;2619 tlimit = local_clock() + rrn;
2589 jlimit = jiffies + (rrn + npj + 1) / npj;2620 jlimit = jiffies + (rrn + npj + 1) / npj;
2590 jlimit_check = true;2621 jlimit_check = true;
@@ -3278,14 +3309,15 @@ static void synchronize_rcu_normal(void) · 687 unmodified lines
3278{3309{
3279 struct rcu_synchronize rs;3310 struct rcu_synchronize rs;
3280 3311
3312 init_rcu_head_on_stack(&rs.head);
3281 trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request"));3313 trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request"));
3282 3314
3283 if (READ_ONCE(rcu_normal_wake_from_gp) < 1) {3315 if (READ_ONCE(rcu_normal_wake_from_gp) < 1 ||
3316 READ_ONCE(rcu_sr_normal_latched)) {
3284 wait_rcu_gp(call_rcu_hurry);3317 wait_rcu_gp(call_rcu_hurry);
3285 goto trace_complete_out;3318 goto trace_complete_out;
3286 }3319 }
3287 3320
3288 init_rcu_head_on_stack(&rs.head);
3289 init_completion(&rs.completion);3321 init_completion(&rs.completion);
3290 3322
3291 /*3323 /*
@@ -3302,10 +3334,10 @@ static void synchronize_rcu_normal(void) · 10 unmodified lines
3302 3334
3303 /* Now we can wait. */3335 /* Now we can wait. */
3304 wait_for_completion(&rs.completion);3336 wait_for_completion(&rs.completion);
3305 destroy_rcu_head_on_stack(&rs.head);
3306 3337
3307trace_complete_out:3338trace_complete_out:
3308 trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete"));3339 trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete"));
3340 destroy_rcu_head_on_stack(&rs.head);
3309}3341}
3310 3342
3311/**3343/**
@@ -4904,12 +4936,6 @@ void __init rcu_init(void) · 1592 unmodified lines
4904 sync_wq = alloc_workqueue("sync_wq", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);4936 sync_wq = alloc_workqueue("sync_wq", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
4905 WARN_ON(!sync_wq);4937 WARN_ON(!sync_wq);
4906 4938
4907 /* Respect if explicitly disabled via a boot parameter. */
4908 if (rcu_normal_wake_from_gp < 0) {
4909 if (num_possible_cpus() <= WAKE_FROM_GP_CPU_THRESHOLD)
4910 rcu_normal_wake_from_gp = 1;
4911 }
4912
4913 /* Fill in default value for rcutree.qovld boot parameter. */4939 /* Fill in default value for rcutree.qovld boot parameter. */
4914 /* -After- the rcu_node ->lock fields are initialized! */4940 /* -After- the rcu_node ->lock fields are initialized! */
4915 if (qovld < 0)4941 if (qovld < 0)
kernel/rcu/tree_nocb.h+1−1
@@ -655,7 +655,7 @@ static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) · 654 unmodified lines
655 * No-CBs GP kthreads come here to wait for additional callbacks to show up655 * No-CBs GP kthreads come here to wait for additional callbacks to show up
656 * or for grace periods to end.656 * or for grace periods to end.
657 */657 */
658static void nocb_gp_wait(struct rcu_data *my_rdp)658static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp)
659{659{
660 bool bypass = false;660 bool bypass = false;
661 int __maybe_unused cpu = my_rdp->cpu;661 int __maybe_unused cpu = my_rdp->cpu;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t