If there is a flood of callbacks, it is necessary to put multiple CPUs to work invoking those callbacks. This commit therefore uses a workqueue-flooding approach to parallelize RCU Tasks callback execution.
Showing the changes to kernel (1 of 1 file). Show all
Unified · Split
| @@ -24,10 +24,14 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp); · 23 unmodified lines | |||
| 24 | * struct rcu_tasks_percpu - Per-CPU component of definition for a Tasks-RCU-like mechanism. | 24 | * struct rcu_tasks_percpu - Per-CPU component of definition for a Tasks-RCU-like mechanism. |
| 25 | * @cblist: Callback list. | 25 | * @cblist: Callback list. |
| 26 | * @lock: Lock protecting per-CPU callback list. | 26 | * @lock: Lock protecting per-CPU callback list. |
| 27 | * @rtp_work: Work queue for invoking callbacks. | ||
| 27 | */ | 28 | */ |
| 28 | struct rcu_tasks_percpu { | 29 | struct rcu_tasks_percpu { |
| 29 | struct rcu_segcblist cblist; | 30 | struct rcu_segcblist cblist; |
| 30 | raw_spinlock_t __private lock; | 31 | raw_spinlock_t __private lock; |
| 32 | struct work_struct rtp_work; | ||
| 33 | int cpu; | ||
| 34 | struct rcu_tasks *rtpp; | ||
| 31 | }; | 35 | }; |
| 32 | 36 | ||
| 33 | /** | 37 | /** |
| @@ -146,6 +150,8 @@ static const char * const rcu_tasks_gp_state_names[] = { · 112 unmodified lines | |||
| 146 | // | 150 | // |
| 147 | // Generic code. | 151 | // Generic code. |
| 148 | 152 | ||
| 153 | static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp); | ||
| 154 | |||
| 149 | /* Record grace-period phase and time. */ | 155 | /* Record grace-period phase and time. */ |
| 150 | static void set_tasks_gp_state(struct rcu_tasks *rtp, int newstate) | 156 | static void set_tasks_gp_state(struct rcu_tasks *rtp, int newstate) |
| 151 | { | 157 | { |
| @@ -185,6 +191,9 @@ static void cblist_init_generic(struct rcu_tasks *rtp) · 33 unmodified lines | |||
| 185 | raw_spin_lock_rcu_node(rtpcp); // irqs already disabled. | 191 | raw_spin_lock_rcu_node(rtpcp); // irqs already disabled. |
| 186 | if (rcu_segcblist_empty(&rtpcp->cblist)) | 192 | if (rcu_segcblist_empty(&rtpcp->cblist)) |
| 187 | rcu_segcblist_init(&rtpcp->cblist); | 193 | rcu_segcblist_init(&rtpcp->cblist); |
| 194 | INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq); | ||
| 195 | rtpcp->cpu = cpu; | ||
| 196 | rtpcp->rtpp = rtp; | ||
| 188 | raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled. | 197 | raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled. |
| 189 | } | 198 | } |
| 190 | raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); | 199 | raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); |
| @@ -256,35 +265,55 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) · 65 unmodified lines | |||
| 256 | } | 265 | } |
| 257 | 266 | ||
| 258 | // Advance callbacks and invoke any that are ready. | 267 | // Advance callbacks and invoke any that are ready. |
| 259 | static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp) | 268 | static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu *rtpcp) |
| 260 | { | 269 | { |
| 261 | int cpu; | 270 | int cpu; |
| 271 | int cpunext; | ||
| 262 | unsigned long flags; | 272 | unsigned long flags; |
| 263 | int len; | 273 | int len; |
| 264 | struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl); | ||
| 265 | struct rcu_head *rhp; | 274 | struct rcu_head *rhp; |
| 266 | 275 | struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl); | |
| 267 | for (cpu = 0; cpu < rtp->percpu_enqueue_lim; cpu++) { | 276 | struct rcu_tasks_percpu *rtpcp_next; |
| 268 | struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); | 277 | |
| 269 | 278 | cpu = rtpcp->cpu; | |
| 270 | if (rcu_segcblist_empty(&rtpcp->cblist)) | 279 | cpunext = cpu * 2 + 1; |
| 271 | continue; | 280 | if (cpunext < rtp->percpu_enqueue_lim) { |
| 272 | raw_spin_lock_irqsave_rcu_node(rtpcp, flags); | 281 | rtpcp_next = per_cpu_ptr(rtp->rtpcpu, cpunext); |
| 273 | rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); | 282 | queue_work_on(cpunext, system_wq, &rtpcp_next->rtp_work); |
| 274 | rcu_segcblist_extract_done_cbs(&rtpcp->cblist, &rcl); | 283 | cpunext++; |
| 275 | raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); | 284 | if (cpunext < rtp->percpu_enqueue_lim) { |
| 276 | len = rcl.len; | 285 | rtpcp_next = per_cpu_ptr(rtp->rtpcpu, cpunext); |
| 277 | for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = rcu_cblist_dequeue(&rcl)) { | 286 | queue_work_on(cpunext, system_wq, &rtpcp_next->rtp_work); |
| 278 | local_bh_disable(); | ||
| 279 | rhp->func(rhp); | ||
| 280 | local_bh_enable(); | ||
| 281 | cond_resched(); | ||
| 282 | } | 287 | } |
| 283 | raw_spin_lock_irqsave_rcu_node(rtpcp, flags); | ||
| 284 | rcu_segcblist_add_len(&rtpcp->cblist, -len); | ||
| 285 | (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); | ||
| 286 | raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); | ||
| 287 | } | 288 | } |
| 289 | |||
| 290 | if (rcu_segcblist_empty(&rtpcp->cblist)) | ||
| 291 | return; | ||
| 292 | raw_spin_lock_irqsave_rcu_node(rtpcp, flags); | ||
| 293 | rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); | ||
| 294 | rcu_segcblist_extract_done_cbs(&rtpcp->cblist, &rcl); | ||
| 295 | raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); | ||
| 296 | len = rcl.len; | ||
| 297 | for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = rcu_cblist_dequeue(&rcl)) { | ||
| 298 | local_bh_disable(); | ||
| 299 | rhp->func(rhp); | ||
| 300 | local_bh_enable(); | ||
| 301 | cond_resched(); | ||
| 302 | } | ||
| 303 | raw_spin_lock_irqsave_rcu_node(rtpcp, flags); | ||
| 304 | rcu_segcblist_add_len(&rtpcp->cblist, -len); | ||
| 305 | (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); | ||
| 306 | raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); | ||
| 307 | } | ||
| 308 | |||
| 309 | // Workqueue flood to advance callbacks and invoke any that are ready. | ||
| 310 | static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp) | ||
| 311 | { | ||
| 312 | struct rcu_tasks *rtp; | ||
| 313 | struct rcu_tasks_percpu *rtpcp = container_of(wp, struct rcu_tasks_percpu, rtp_work); | ||
| 314 | |||
| 315 | rtp = rtpcp->rtpp; | ||
| 316 | rcu_tasks_invoke_cbs(rtp, rtpcp); | ||
| 288 | } | 317 | } |
| 289 | 318 | ||
| 290 | /* RCU-tasks kthread that detects grace periods and invokes callbacks. */ | 319 | /* RCU-tasks kthread that detects grace periods and invokes callbacks. */ |
| @@ -320,7 +349,7 @@ static int __noreturn rcu_tasks_kthread(void *arg) · 29 unmodified lines | |||
| 320 | 349 | ||
| 321 | /* Invoke callbacks. */ | 350 | /* Invoke callbacks. */ |
| 322 | set_tasks_gp_state(rtp, RTGS_INVOKE_CBS); | 351 | set_tasks_gp_state(rtp, RTGS_INVOKE_CBS); |
| 323 | rcu_tasks_invoke_cbs(rtp); | 352 | rcu_tasks_invoke_cbs(rtp, per_cpu_ptr(rtp->rtpcpu, 0)); |
| 324 | 353 | ||
| 325 | /* Paranoid sleep to keep this from entering a tight loop */ | 354 | /* Paranoid sleep to keep this from entering a tight loop */ |
| 326 | schedule_timeout_idle(rtp->gp_sleep); | 355 | schedule_timeout_idle(rtp->gp_sleep); |