The RCU Tasks Trace grace periods are too slow, as in 40x slower than those of RCU Tasks. This is due to my having assumed a one-second grace period was OK, and thus not having optimized any further. This commit provides the first step in this optimization process, namely by allowing the task_list scan backoff interval to be specified on a per-flavor basis, and then speeding up the scans for RCU Tasks Trace. However, kernels built with CONFIG_TASKS_TRACE_RCU_READ_MB=y continue to use the old slower backoff, consistent with that Kconfig option's goal of reducing IPIs.
Showing the changes to kernel (1 of 1 file). Show all
Unified · Split
| @@ -28,6 +28,7 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp); · 27 unmodified lines | |||
| 28 | * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. | 28 | * @kthread_ptr: This flavor's grace-period/callback-invocation kthread. |
| 29 | * @gp_func: This flavor's grace-period-wait function. | 29 | * @gp_func: This flavor's grace-period-wait function. |
| 30 | * @gp_state: Grace period's most recent state transition (debugging). | 30 | * @gp_state: Grace period's most recent state transition (debugging). |
| 31 | * @init_fract: Initial backoff sleep interval. | ||
| 31 | * @gp_jiffies: Time of last @gp_state transition. | 32 | * @gp_jiffies: Time of last @gp_state transition. |
| 32 | * @gp_start: Most recent grace-period start in jiffies. | 33 | * @gp_start: Most recent grace-period start in jiffies. |
| 33 | * @n_gps: Number of grace periods completed since boot. | 34 | * @n_gps: Number of grace periods completed since boot. |
| @@ -48,6 +49,7 @@ struct rcu_tasks { · 14 unmodified lines | |||
| 48 | struct wait_queue_head cbs_wq; | 49 | struct wait_queue_head cbs_wq; |
| 49 | raw_spinlock_t cbs_lock; | 50 | raw_spinlock_t cbs_lock; |
| 50 | int gp_state; | 51 | int gp_state; |
| 52 | int init_fract; | ||
| 51 | unsigned long gp_jiffies; | 53 | unsigned long gp_jiffies; |
| 52 | unsigned long gp_start; | 54 | unsigned long gp_start; |
| 53 | unsigned long n_gps; | 55 | unsigned long n_gps; |
| @@ -329,8 +331,10 @@ static void rcu_tasks_wait_gp(struct rcu_tasks *rtp) · 275 unmodified lines | |||
| 329 | */ | 331 | */ |
| 330 | lastreport = jiffies; | 332 | lastreport = jiffies; |
| 331 | 333 | ||
| 332 | /* Start off with HZ/10 wait and slowly back off to 1 HZ wait. */ | 334 | // Start off with initial wait and slowly back off to 1 HZ wait. |
| 333 | fract = 10; | 335 | fract = rtp->init_fract; |
| 336 | if (fract > HZ) | ||
| 337 | fract = HZ; | ||
| 334 | 338 | ||
| 335 | for (;;) { | 339 | for (;;) { |
| 336 | bool firstreport; | 340 | bool firstreport; |
| @@ -553,6 +557,7 @@ EXPORT_SYMBOL_GPL(rcu_barrier_tasks); · 216 unmodified lines | |||
| 553 | 557 | ||
| 554 | static int __init rcu_spawn_tasks_kthread(void) | 558 | static int __init rcu_spawn_tasks_kthread(void) |
| 555 | { | 559 | { |
| 560 | rcu_tasks.init_fract = 10; | ||
| 556 | rcu_tasks.pregp_func = rcu_tasks_pregp_step; | 561 | rcu_tasks.pregp_func = rcu_tasks_pregp_step; |
| 557 | rcu_tasks.pertask_func = rcu_tasks_pertask; | 562 | rcu_tasks.pertask_func = rcu_tasks_pertask; |
| 558 | rcu_tasks.postscan_func = rcu_tasks_postscan; | 563 | rcu_tasks.postscan_func = rcu_tasks_postscan; |
| @@ -1163,6 +1168,13 @@ EXPORT_SYMBOL_GPL(rcu_barrier_tasks_trace); · 604 unmodified lines | |||
| 1163 | 1168 | ||
| 1164 | static int __init rcu_spawn_tasks_trace_kthread(void) | 1169 | static int __init rcu_spawn_tasks_trace_kthread(void) |
| 1165 | { | 1170 | { |
| 1171 | if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) { | ||
| 1172 | rcu_tasks_trace.init_fract = 10; | ||
| 1173 | } else { | ||
| 1174 | rcu_tasks_trace.init_fract = HZ / 5; | ||
| 1175 | if (rcu_tasks_trace.init_fract <= 0) | ||
| 1176 | rcu_tasks_trace.init_fract = 1; | ||
| 1177 | } | ||
| 1166 | rcu_tasks_trace.pregp_func = rcu_tasks_trace_pregp_step; | 1178 | rcu_tasks_trace.pregp_func = rcu_tasks_trace_pregp_step; |
| 1167 | rcu_tasks_trace.pertask_func = rcu_tasks_trace_pertask; | 1179 | rcu_tasks_trace.pertask_func = rcu_tasks_trace_pertask; |
| 1168 | rcu_tasks_trace.postscan_func = rcu_tasks_trace_postscan; | 1180 | rcu_tasks_trace.postscan_func = rcu_tasks_trace_postscan; |