RCU's current CPU-offline code path dumps all of the outgoing CPU's callbacks onto the RCU_NEXT_TAIL portion of the surviving CPU's callback list. This means that all the ready-to-invoke callbacks from the outgoing CPU must wait for another full RCU grace period. This was just fine when CPU-hotplug events were rare, but there is increasing evidence that users are planning to make increasing use of CPU hotplug. Therefore, this commit changes the callback-dumping procedure so that callbacks that are ready to invoke are moved to the RCU_DONE_TAIL portion of the surviving CPU's callback list. This avoids running these callbacks through a second unnecessary grace period.
Showing the changes to kernel (1 of 1 file). Show all
Unified · Split
| @@ -1270,24 +1270,64 @@ static void rcu_cleanup_dying_cpu(struct rcu_state *rsp) · 1269 unmodified lines | ||
| 1270 | 1270 | struct rcu_data *receive_rdp = per_cpu_ptr(rsp->rda, receive_cpu); |
| 1271 | 1271 | struct rcu_node *rnp = rdp->mynode; /* For dying CPU. */ |
| 1272 | 1272 | |
| 1273 | − /* Move callbacks to some other CPU. */ | |
| 1273 | + /* First, adjust the counts. */ | |
| 1274 | + if (rdp->nxtlist != NULL) { | |
| 1275 | + receive_rdp->qlen_lazy += rdp->qlen_lazy; | |
| 1276 | + receive_rdp->qlen += rdp->qlen; | |
| 1277 | + rdp->qlen_lazy = 0; | |
| 1278 | + rdp->qlen = 0; | |
| 1279 | + } | |
| 1280 | + | |
| 1281 | + /* | |
| 1282 | + * Next, move ready-to-invoke callbacks to be invoked on some | |
| 1283 | + * other CPU. These will not be required to pass through another | |
| 1284 | + * grace period: They are done, regardless of CPU. | |
| 1285 | + */ | |
| 1286 | + if (rdp->nxtlist != NULL && | |
| 1287 | + rdp->nxttail[RCU_DONE_TAIL] != &rdp->nxtlist) { | |
| 1288 | + struct rcu_head *oldhead; | |
| 1289 | + struct rcu_head **oldtail; | |
| 1290 | + struct rcu_head **newtail; | |
| 1291 | + | |
| 1292 | + oldhead = rdp->nxtlist; | |
| 1293 | + oldtail = receive_rdp->nxttail[RCU_DONE_TAIL]; | |
| 1294 | + rdp->nxtlist = *rdp->nxttail[RCU_DONE_TAIL]; | |
| 1295 | + *rdp->nxttail[RCU_DONE_TAIL] = *oldtail; | |
| 1296 | + *receive_rdp->nxttail[RCU_DONE_TAIL] = oldhead; | |
| 1297 | + newtail = rdp->nxttail[RCU_DONE_TAIL]; | |
| 1298 | + for (i = RCU_DONE_TAIL; i < RCU_NEXT_SIZE; i++) { | |
| 1299 | + if (receive_rdp->nxttail[i] == oldtail) | |
| 1300 | + receive_rdp->nxttail[i] = newtail; | |
| 1301 | + if (rdp->nxttail[i] == newtail) | |
| 1302 | + rdp->nxttail[i] = &rdp->nxtlist; | |
| 1303 | + } | |
| 1304 | + } | |
| 1305 | + | |
| 1306 | + /* | |
| 1307 | + * Finally, put the rest of the callbacks at the end of the list. | |
| 1308 | + * The ones that made it partway through get to start over: We | |
| 1309 | + * cannot assume that grace periods are synchronized across CPUs. | |
| 1310 | + * (We could splice RCU_WAIT_TAIL into RCU_NEXT_READY_TAIL, but | |
| 1311 | + * this does not seem compelling. Not yet, anyway.) | |
| 1312 | + */ | |
| 1274 | 1313 | if (rdp->nxtlist != NULL) { |
| 1275 | 1314 | *receive_rdp->nxttail[RCU_NEXT_TAIL] = rdp->nxtlist; |
| 1276 | 1315 | receive_rdp->nxttail[RCU_NEXT_TAIL] = |
| 1277 | 1316 | rdp->nxttail[RCU_NEXT_TAIL]; |
| 1278 | − receive_rdp->qlen_lazy += rdp->qlen_lazy; | |
| 1279 | − receive_rdp->qlen += rdp->qlen; | |
| 1280 | 1317 | receive_rdp->n_cbs_adopted += rdp->qlen; |
| 1281 | 1318 | rdp->n_cbs_orphaned += rdp->qlen; |
| 1282 | 1319 | |
| 1283 | 1320 | rdp->nxtlist = NULL; |
| 1284 | 1321 | for (i = 0; i < RCU_NEXT_SIZE; i++) |
| 1285 | 1322 | rdp->nxttail[i] = &rdp->nxtlist; |
| 1286 | − rdp->qlen_lazy = 0; | |
| 1287 | − rdp->qlen = 0; | |
| 1288 | 1323 | } |
| 1289 | 1324 | |
| 1290 | − /* Record a quiescent state for the dying CPU. */ | |
| 1325 | + /* | |
| 1326 | + * Record a quiescent state for the dying CPU. This is safe | |
| 1327 | + * only because we have already cleared out the callbacks. | |
| 1328 | + * (Otherwise, the RCU core might try to schedule the invocation | |
| 1329 | + * of callbacks on this now-offline CPU, which would be bad.) | |
| 1330 | + */ | |
| 1291 | 1331 | mask = rdp->grpmask; /* rnp->grplo is constant. */ |
| 1292 | 1332 | trace_rcu_grace_period(rsp->name, |
| 1293 | 1333 | rnp->gpnum + 1 - !!(rnp->qsmask & mask), |