The RCU tasks flavors of RCU now need concurrent callback flooding to test their ability to switch between single-queue mode and per-CPU queue mode, but their lack of heavy-duty forward-progress features rules out the use of rcutorture's current callback-flooding code. This commit therefore provides the ability to limit the intensity of the callback floods using a new ->cbflood_max field in the rcu_operations structure. When this field is zero, there is no limit, otherwise, each callback-flood kthread allocates at most ->cbflood_max callbacks.
Unified · Split
| @@ -348,6 +348,7 @@ struct rcu_torture_ops { · 347 unmodified lines | |||
| 348 | void (*gp_kthread_dbg)(void); | 348 | void (*gp_kthread_dbg)(void); |
| 349 | bool (*check_boost_failed)(unsigned long gp_state, int *cpup); | 349 | bool (*check_boost_failed)(unsigned long gp_state, int *cpup); |
| 350 | int (*stall_dur)(void); | 350 | int (*stall_dur)(void); |
| 351 | long cbflood_max; | ||
| 351 | int irq_capable; | 352 | int irq_capable; |
| 352 | int can_boost; | 353 | int can_boost; |
| 353 | int extendables; | 354 | int extendables; |
| @@ -841,6 +842,7 @@ static struct rcu_torture_ops tasks_rude_ops = { · 487 unmodified lines | |||
| 841 | .call = call_rcu_tasks_rude, | 842 | .call = call_rcu_tasks_rude, |
| 842 | .cb_barrier = rcu_barrier_tasks_rude, | 843 | .cb_barrier = rcu_barrier_tasks_rude, |
| 843 | .gp_kthread_dbg = show_rcu_tasks_rude_gp_kthread, | 844 | .gp_kthread_dbg = show_rcu_tasks_rude_gp_kthread, |
| 845 | .cbflood_max = 50000, | ||
| 844 | .fqs = NULL, | 846 | .fqs = NULL, |
| 845 | .stats = NULL, | 847 | .stats = NULL, |
| 846 | .irq_capable = 1, | 848 | .irq_capable = 1, |
| @@ -881,6 +883,7 @@ static struct rcu_torture_ops tasks_tracing_ops = { · 34 unmodified lines | |||
| 881 | .call = call_rcu_tasks_trace, | 883 | .call = call_rcu_tasks_trace, |
| 882 | .cb_barrier = rcu_barrier_tasks_trace, | 884 | .cb_barrier = rcu_barrier_tasks_trace, |
| 883 | .gp_kthread_dbg = show_rcu_tasks_trace_gp_kthread, | 885 | .gp_kthread_dbg = show_rcu_tasks_trace_gp_kthread, |
| 886 | .cbflood_max = 50000, | ||
| 884 | .fqs = NULL, | 887 | .fqs = NULL, |
| 885 | .stats = NULL, | 888 | .stats = NULL, |
| 886 | .irq_capable = 1, | 889 | .irq_capable = 1, |
| @@ -2387,7 +2390,7 @@ static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) · 1500 unmodified lines | |||
| 2387 | rfp->rcu_fwd_cb_head = rfcpn; | 2390 | rfp->rcu_fwd_cb_head = rfcpn; |
| 2388 | n_launders++; | 2391 | n_launders++; |
| 2389 | n_launders_sa++; | 2392 | n_launders_sa++; |
| 2390 | } else { | 2393 | } else if (!cur_ops->cbflood_max || cur_ops->cbflood_max > n_max_cbs) { |
| 2391 | rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL); | 2394 | rfcp = kmalloc(sizeof(*rfcp), GFP_KERNEL); |
| 2392 | if (WARN_ON_ONCE(!rfcp)) { | 2395 | if (WARN_ON_ONCE(!rfcp)) { |
| 2393 | schedule_timeout_interruptible(1); | 2396 | schedule_timeout_interruptible(1); |
| @@ -2397,8 +2400,11 @@ static void rcu_torture_fwd_prog_cr(struct rcu_fwd *rfp) · 3 unmodified lines | |||
| 2397 | n_launders_sa = 0; | 2400 | n_launders_sa = 0; |
| 2398 | rfcp->rfc_gps = 0; | 2401 | rfcp->rfc_gps = 0; |
| 2399 | rfcp->rfc_rfp = rfp; | 2402 | rfcp->rfc_rfp = rfp; |
| 2403 | } else { | ||
| 2404 | rfcp = NULL; | ||
| 2400 | } | 2405 | } |
| 2401 | cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr); | 2406 | if (rfcp) |
| 2407 | cur_ops->call(&rfcp->rh, rcu_torture_fwd_cb_cr); | ||
| 2402 | rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs); | 2408 | rcu_torture_fwd_prog_cond_resched(n_launders + n_max_cbs); |
| 2403 | if (tick_nohz_full_enabled()) { | 2409 | if (tick_nohz_full_enabled()) { |
| 2404 | local_irq_save(flags); | 2410 | local_irq_save(flags); |
| @@ -2506,8 +2512,10 @@ static int rcu_torture_fwd_prog(void *args) · 101 unmodified lines | |||
| 2506 | pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); | 2512 | pr_alert("%s: Starting forward-progress test %d\n", __func__, rfp->rcu_fwd_id); |
| 2507 | if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id) | 2513 | if (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id) |
| 2508 | rcu_torture_fwd_prog_cr(rfp); | 2514 | rcu_torture_fwd_prog_cr(rfp); |
| 2509 | if (!IS_ENABLED(CONFIG_TINY_RCU) || | 2515 | if ((cur_ops->stall_dur && cur_ops->stall_dur() > 0) && |
| 2510 | (rcu_inkernel_boot_has_ended() && torture_num_online_cpus() > rfp->rcu_fwd_id)) | 2516 | (!IS_ENABLED(CONFIG_TINY_RCU) || |
| 2517 | (rcu_inkernel_boot_has_ended() && | ||
| 2518 | torture_num_online_cpus() > rfp->rcu_fwd_id))) | ||
| 2511 | rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries); | 2519 | rcu_torture_fwd_prog_nr(rfp, &tested, &tested_tries); |
| 2512 | 2520 | ||
| 2513 | /* Avoid slow periods, better to test when busy. */ | 2521 | /* Avoid slow periods, better to test when busy. */ |
| @@ -2539,7 +2547,8 @@ static int __init rcu_torture_fwd_prog_init(void) · 25 unmodified lines | |||
| 2539 | fwd_progress = nr_cpu_ids; | 2547 | fwd_progress = nr_cpu_ids; |
| 2540 | } | 2548 | } |
| 2541 | if ((!cur_ops->sync && !cur_ops->call) || | 2549 | if ((!cur_ops->sync && !cur_ops->call) || |
| 2542 | !cur_ops->stall_dur || cur_ops->stall_dur() <= 0 || cur_ops == &rcu_busted_ops) { | 2550 | (!cur_ops->cbflood_max && (!cur_ops->stall_dur || cur_ops->stall_dur() <= 0)) || |
| 2551 | cur_ops == &rcu_busted_ops) { | ||
| 2543 | VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test"); | 2552 | VERBOSE_TOROUT_STRING("rcu_torture_fwd_prog_init: Disabled, unsupported by RCU flavor under test"); |
| 2544 | fwd_progress = 0; | 2553 | fwd_progress = 0; |
| 2545 | return 0; | 2554 | return 0; |