History

Merge branches 'rcutorture.2026.05.24' and 'misc.2026.05.24' into rcu-merge.2026.05.24

rcutorture.2026.05.24: Torture-test updates misc.2026.05.24: Miscellaneous RCU updates

Uladzislau Rezki (Sony) committed 5 months ago

Commit e853c1b2 parents 956d852 00266887 files · +60 −30Browse files

Showing the changes to kernel (3 of 7 files). Show all

Unified · Split

3 files
  1. kernel/rcu/tasks.h +2 −1
  2. kernel/rcu/tree.c +41 −15
  3. kernel/rcu/tree_nocb.h +1 −1
kernel/rcu/tasks.h+2−1
@@ -373,7 +373,8 @@ static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func, · 372 unmodified lines
373373 // Queuing callbacks before initialization not yet supported.
374374 if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist)))
375375 rcu_segcblist_init(&rtpcp->cblist);
376− needwake = (func == wakeme_after_rcu) ||
376+ needwake = (!havekthread && rcu_segcblist_empty(&rtpcp->cblist)) ||
377+ (func == wakeme_after_rcu) ||
377378 (rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim);
378379 if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) {
379380 if (rtp->lazy_jiffies)
kernel/rcu/tree.c+41−15
@@ -492,7 +492,7 @@ static int param_set_next_fqs_jiffies(const char *val, const struct kernel_param · 491 unmodified lines
492492 int ret = kstrtoul(val, 0, &j);
493493
494494 if (!ret) {
495− WRITE_ONCE(*(ulong *)kp->arg, (j > HZ) ? HZ : (j ?: 1));
495+ WRITE_ONCE(*(ulong *)kp->arg, clamp_val(j, 1, HZ));
496496 adjust_jiffies_till_sched_qs();
497497 }
498498 return ret;
@@ -1632,17 +1632,21 @@ static void rcu_sr_put_wait_head(struct llist_node *node) · 1133 unmodified lines
16321632 atomic_set_release(&sr_wn->inuse, 0);
16331633 }
16341634
1635−/* Enable rcu_normal_wake_from_gp automatically on small systems. */
1636−#define WAKE_FROM_GP_CPU_THRESHOLD 16
1637−
1638−static int rcu_normal_wake_from_gp = -1;
1635+static int rcu_normal_wake_from_gp = 1;
16391636 module_param(rcu_normal_wake_from_gp, int, 0644);
16401637 static struct workqueue_struct *sync_wq;
16411638
1639+#define RCU_SR_NORMAL_LATCH_THR 64
1640+
1641+/* Number of in-flight synchronize_rcu() calls queued on srs_next. */
1642+static atomic_long_t rcu_sr_normal_count;
1643+static int rcu_sr_normal_latched; /* 0/1 */
1644+
16421645 static void rcu_sr_normal_complete(struct llist_node *node)
16431646 {
16441647 struct rcu_synchronize *rs = container_of(
16451648 (struct rcu_head *) node, struct rcu_synchronize, head);
1649+ long nr;
16461650
16471651 WARN_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) &&
16481652 !poll_state_synchronize_rcu_full(&rs->oldstate),
@@ -1650,6 +1654,15 @@ static void rcu_sr_normal_complete(struct llist_node *node) · 1 unmodified line
16501654
16511655 /* Finally. */
16521656 complete(&rs->completion);
1657+ nr = atomic_long_dec_return(&rcu_sr_normal_count);
1658+ WARN_ON_ONCE(nr < 0);
1659+
1660+ /*
1661+ * Unlatch: switch back to normal path when fully
1662+ * drained and if it has been latched.
1663+ */
1664+ if (nr == 0)
1665+ (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 1, 0);
16531666 }
16541667
16551668 static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work)
@@ -1795,6 +1808,24 @@ static bool rcu_sr_normal_gp_init(void) · 139 unmodified lines
17951808
17961809 static void rcu_sr_normal_add_req(struct rcu_synchronize *rs)
17971810 {
1811+ /*
1812+ * Increment before publish to avoid a complete
1813+ * vs enqueue race on latch.
1814+ */
1815+ long nr = atomic_long_inc_return(&rcu_sr_normal_count);
1816+
1817+ /*
1818+ * Latch when threshold is reached. Checking for an exact match
1819+ * restricts cmpxchg() to a single context.
1820+ *
1821+ * This latch is intentionally relaxed and best-effort. Concurrent
1822+ * set/clear can race and temporarily lose the latch, which is OK
1823+ * because it only selects between the fast and fallback paths.
1824+ */
1825+ if (nr == RCU_SR_NORMAL_LATCH_THR)
1826+ (void)cmpxchg_relaxed(&rcu_sr_normal_latched, 0, 1);
1827+
1828+ /* Publish for the GP kthread/worker. */
17981829 llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next);
17991830 }
18001831
@@ -2584,7 +2615,7 @@ static void rcu_do_batch(struct rcu_data *rdp) · 783 unmodified lines
25842615 const long npj = NSEC_PER_SEC / HZ;
25852616 long rrn = READ_ONCE(rcu_resched_ns);
25862617
2587− rrn = rrn < NSEC_PER_MSEC ? NSEC_PER_MSEC : rrn > NSEC_PER_SEC ? NSEC_PER_SEC : rrn;
2618+ rrn = clamp(rrn, NSEC_PER_MSEC, NSEC_PER_SEC);
25882619 tlimit = local_clock() + rrn;
25892620 jlimit = jiffies + (rrn + npj + 1) / npj;
25902621 jlimit_check = true;
@@ -3278,14 +3309,15 @@ static void synchronize_rcu_normal(void) · 687 unmodified lines
32783309 {
32793310 struct rcu_synchronize rs;
32803311
3312+ init_rcu_head_on_stack(&rs.head);
32813313 trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("request"));
32823314
3283− if (READ_ONCE(rcu_normal_wake_from_gp) < 1) {
3315+ if (READ_ONCE(rcu_normal_wake_from_gp) < 1 ||
3316+ READ_ONCE(rcu_sr_normal_latched)) {
32843317 wait_rcu_gp(call_rcu_hurry);
32853318 goto trace_complete_out;
32863319 }
32873320
3288− init_rcu_head_on_stack(&rs.head);
32893321 init_completion(&rs.completion);
32903322
32913323 /*
@@ -3302,10 +3334,10 @@ static void synchronize_rcu_normal(void) · 10 unmodified lines
33023334
33033335 /* Now we can wait. */
33043336 wait_for_completion(&rs.completion);
3305− destroy_rcu_head_on_stack(&rs.head);
33063337
33073338 trace_complete_out:
33083339 trace_rcu_sr_normal(rcu_state.name, &rs.head, TPS("complete"));
3340+ destroy_rcu_head_on_stack(&rs.head);
33093341 }
33103342
33113343 /**
@@ -4904,12 +4936,6 @@ void __init rcu_init(void) · 1592 unmodified lines
49044936 sync_wq = alloc_workqueue("sync_wq", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
49054937 WARN_ON(!sync_wq);
49064938
4907− /* Respect if explicitly disabled via a boot parameter. */
4908− if (rcu_normal_wake_from_gp < 0) {
4909− if (num_possible_cpus() <= WAKE_FROM_GP_CPU_THRESHOLD)
4910− rcu_normal_wake_from_gp = 1;
4911− }
4912−
49134939 /* Fill in default value for rcutree.qovld boot parameter. */
49144940 /* -After- the rcu_node ->lock fields are initialized! */
49154941 if (qovld < 0)
kernel/rcu/tree_nocb.h+1−1
@@ -655,7 +655,7 @@ static void nocb_gp_sleep(struct rcu_data *my_rdp, int cpu) · 654 unmodified lines
655655 * No-CBs GP kthreads come here to wait for additional callbacks to show up
656656 * or for grace periods to end.
657657 */
658−static void nocb_gp_wait(struct rcu_data *my_rdp)
658+static noinline_for_stack void nocb_gp_wait(struct rcu_data *my_rdp)
659659 {
660660 bool bypass = false;
661661 int __maybe_unused cpu = my_rdp->cpu;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t