If there is a flood of callbacks, it is necessary to put multiple CPUs to work invoking those callbacks. This commit therefore uses a workqueue-flooding approach to parallelize RCU Tasks callback execution.
Showing the changes to kernel (1 of 1 file). Show all
Unified · Split
| @@ -24,10 +24,14 @@ typedef void (*postgp_func_t)(struct rcu_tasks *rtp); · 23 unmodified lines | ||
| 24 | 24 | * struct rcu_tasks_percpu - Per-CPU component of definition for a Tasks-RCU-like mechanism. |
| 25 | 25 | * @cblist: Callback list. |
| 26 | 26 | * @lock: Lock protecting per-CPU callback list. |
| 27 | + * @rtp_work: Work queue for invoking callbacks. | |
| 27 | 28 | */ |
| 28 | 29 | struct rcu_tasks_percpu { |
| 29 | 30 | struct rcu_segcblist cblist; |
| 30 | 31 | raw_spinlock_t __private lock; |
| 32 | + struct work_struct rtp_work; | |
| 33 | + int cpu; | |
| 34 | + struct rcu_tasks *rtpp; | |
| 31 | 35 | }; |
| 32 | 36 | |
| 33 | 37 | /** |
| @@ -146,6 +150,8 @@ static const char * const rcu_tasks_gp_state_names[] = { · 112 unmodified lines | ||
| 146 | 150 | // |
| 147 | 151 | // Generic code. |
| 148 | 152 | |
| 153 | +static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp); | |
| 154 | + | |
| 149 | 155 | /* Record grace-period phase and time. */ |
| 150 | 156 | static void set_tasks_gp_state(struct rcu_tasks *rtp, int newstate) |
| 151 | 157 | { |
| @@ -185,6 +191,9 @@ static void cblist_init_generic(struct rcu_tasks *rtp) · 33 unmodified lines | ||
| 185 | 191 | raw_spin_lock_rcu_node(rtpcp); // irqs already disabled. |
| 186 | 192 | if (rcu_segcblist_empty(&rtpcp->cblist)) |
| 187 | 193 | rcu_segcblist_init(&rtpcp->cblist); |
| 194 | + INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq); | |
| 195 | + rtpcp->cpu = cpu; | |
| 196 | + rtpcp->rtpp = rtp; | |
| 188 | 197 | raw_spin_unlock_rcu_node(rtpcp); // irqs remain disabled. |
| 189 | 198 | } |
| 190 | 199 | raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags); |
| @@ -256,35 +265,55 @@ static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp) · 65 unmodified lines | ||
| 256 | 265 | } |
| 257 | 266 | |
| 258 | 267 | // Advance callbacks and invoke any that are ready. |
| 259 | −static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp) | |
| 268 | +static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu *rtpcp) | |
| 260 | 269 | { |
| 261 | 270 | int cpu; |
| 271 | + int cpunext; | |
| 262 | 272 | unsigned long flags; |
| 263 | 273 | int len; |
| 264 | − struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl); | |
| 265 | 274 | struct rcu_head *rhp; |
| 266 | − | |
| 267 | − for (cpu = 0; cpu < rtp->percpu_enqueue_lim; cpu++) { | |
| 268 | − struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu); | |
| 269 | − | |
| 270 | − if (rcu_segcblist_empty(&rtpcp->cblist)) | |
| 271 | − continue; | |
| 272 | − raw_spin_lock_irqsave_rcu_node(rtpcp, flags); | |
| 273 | − rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); | |
| 274 | − rcu_segcblist_extract_done_cbs(&rtpcp->cblist, &rcl); | |
| 275 | − raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); | |
| 276 | − len = rcl.len; | |
| 277 | − for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = rcu_cblist_dequeue(&rcl)) { | |
| 278 | − local_bh_disable(); | |
| 279 | − rhp->func(rhp); | |
| 280 | − local_bh_enable(); | |
| 281 | − cond_resched(); | |
| 275 | + struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl); | |
| 276 | + struct rcu_tasks_percpu *rtpcp_next; | |
| 277 | + | |
| 278 | + cpu = rtpcp->cpu; | |
| 279 | + cpunext = cpu * 2 + 1; | |
| 280 | + if (cpunext < rtp->percpu_enqueue_lim) { | |
| 281 | + rtpcp_next = per_cpu_ptr(rtp->rtpcpu, cpunext); | |
| 282 | + queue_work_on(cpunext, system_wq, &rtpcp_next->rtp_work); | |
| 283 | + cpunext++; | |
| 284 | + if (cpunext < rtp->percpu_enqueue_lim) { | |
| 285 | + rtpcp_next = per_cpu_ptr(rtp->rtpcpu, cpunext); | |
| 286 | + queue_work_on(cpunext, system_wq, &rtpcp_next->rtp_work); | |
| 282 | 287 | } |
| 283 | − raw_spin_lock_irqsave_rcu_node(rtpcp, flags); | |
| 284 | − rcu_segcblist_add_len(&rtpcp->cblist, -len); | |
| 285 | − (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); | |
| 286 | − raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); | |
| 287 | 288 | } |
| 289 | + | |
| 290 | + if (rcu_segcblist_empty(&rtpcp->cblist)) | |
| 291 | + return; | |
| 292 | + raw_spin_lock_irqsave_rcu_node(rtpcp, flags); | |
| 293 | + rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq)); | |
| 294 | + rcu_segcblist_extract_done_cbs(&rtpcp->cblist, &rcl); | |
| 295 | + raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); | |
| 296 | + len = rcl.len; | |
| 297 | + for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = rcu_cblist_dequeue(&rcl)) { | |
| 298 | + local_bh_disable(); | |
| 299 | + rhp->func(rhp); | |
| 300 | + local_bh_enable(); | |
| 301 | + cond_resched(); | |
| 302 | + } | |
| 303 | + raw_spin_lock_irqsave_rcu_node(rtpcp, flags); | |
| 304 | + rcu_segcblist_add_len(&rtpcp->cblist, -len); | |
| 305 | + (void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq)); | |
| 306 | + raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags); | |
| 307 | +} | |
| 308 | + | |
| 309 | +// Workqueue flood to advance callbacks and invoke any that are ready. | |
| 310 | +static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp) | |
| 311 | +{ | |
| 312 | + struct rcu_tasks *rtp; | |
| 313 | + struct rcu_tasks_percpu *rtpcp = container_of(wp, struct rcu_tasks_percpu, rtp_work); | |
| 314 | + | |
| 315 | + rtp = rtpcp->rtpp; | |
| 316 | + rcu_tasks_invoke_cbs(rtp, rtpcp); | |
| 288 | 317 | } |
| 289 | 318 | |
| 290 | 319 | /* RCU-tasks kthread that detects grace periods and invokes callbacks. */ |
| @@ -320,7 +349,7 @@ static int __noreturn rcu_tasks_kthread(void *arg) · 29 unmodified lines | ||
| 320 | 349 | |
| 321 | 350 | /* Invoke callbacks. */ |
| 322 | 351 | set_tasks_gp_state(rtp, RTGS_INVOKE_CBS); |
| 323 | − rcu_tasks_invoke_cbs(rtp); | |
| 352 | + rcu_tasks_invoke_cbs(rtp, per_cpu_ptr(rtp->rtpcpu, 0)); | |
| 324 | 353 | |
| 325 | 354 | /* Paranoid sleep to keep this from entering a tight loop */ |
| 326 | 355 | schedule_timeout_idle(rtp->gp_sleep); |