a5b98009f16d ("sched/psi: fix race between file release and pressure write") made pressure_write() hold cgroup_mutex across psi_trigger_create(), which forks the psimon kthread for the first rtpoll trigger. As kthread creation depends on the whole fork path, the commit inadvertently created a lot of unwanted locking dependencies from cgroup_mutex. sched_ext got hit by one: its enable path blocks forks and then grabs cgroup_mutex, so a pressure write racing a scheduler enable deadlocks, with every other fork piling up behind. Fix it by splitting trigger creation so that the worker is forked with cgroup_mutex dropped and the kernfs active reference left broken. The latter matters because rmdir and cgroup.pressure writes drain active references under cgroup_mutex. Publishing the trigger last keeps error reporting synchronous and preserves the of->priv lifetime rules. The trigger registered in the first stage pins the group's rtpoll machinery across the unlocked window, leaving only creation races to resolve. The catch-up poll on installation covers scheduling attempts dropped while there was no worker. v2: Retagged sched/psi (was cgroup).
Showing the changes to kernel (2 of 3 files). Show all
Unified · Split
| @@ -3996,6 +3996,7 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, · 3995 unmodified lines | ||
| 3996 | 3996 | struct psi_trigger *new; |
| 3997 | 3997 | struct cgroup *cgrp; |
| 3998 | 3998 | struct psi_group *psi; |
| 3999 | + bool need_rtpoll_worker; | |
| 3999 | 4000 | ssize_t ret = 0; |
| 4000 | 4001 | |
| 4001 | 4002 | cgrp = cgroup_kn_lock_live(of->kn, false); |
| @@ -4015,12 +4016,32 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, · 13 unmodified lines | ||
| 4015 | 4016 | } |
| 4016 | 4017 | |
| 4017 | 4018 | psi = cgroup_psi(cgrp); |
| 4018 | − new = psi_trigger_create(psi, buf, res, of->file, of); | |
| 4019 | + new = psi_trigger_create(psi, buf, res, of->file, of, | |
| 4020 | + &need_rtpoll_worker); | |
| 4019 | 4021 | if (IS_ERR(new)) { |
| 4020 | 4022 | ret = PTR_ERR(new); |
| 4021 | 4023 | goto out_unlock; |
| 4022 | 4024 | } |
| 4023 | 4025 | |
| 4026 | + /* | |
| 4027 | + * The worker fork must run with neither cgroup_mutex nor the file's | |
| 4028 | + * kernfs active reference held. The latter is broken since | |
| 4029 | + * cgroup_kn_lock_live(). @of->priv may be released while unlocked, so | |
| 4030 | + * recheck before publishing @new. | |
| 4031 | + */ | |
| 4032 | + if (need_rtpoll_worker) { | |
| 4033 | + cgroup_unlock(); | |
| 4034 | + ret = psi_trigger_create_rtpoll_worker(psi); | |
| 4035 | + cgroup_lock(); | |
| 4036 | + | |
| 4037 | + if (!ret && !of->priv) | |
| 4038 | + ret = -ENODEV; | |
| 4039 | + if (ret) { | |
| 4040 | + psi_trigger_destroy(new); | |
| 4041 | + goto out_unlock; | |
| 4042 | + } | |
| 4043 | + } | |
| 4044 | + | |
| 4024 | 4045 | smp_store_release(&ctx->psi.trigger, new); |
| 4025 | 4046 | |
| 4026 | 4047 | out_unlock: |
| @@ -1292,9 +1292,44 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) · 1291 unmodified lines | ||
| 1292 | 1292 | return 0; |
| 1293 | 1293 | } |
| 1294 | 1294 | |
| 1295 | +/* | |
| 1296 | + * Create @group's rtpoll worker after psi_trigger_create() reported the need | |
| 1297 | + * for one. kthread creation depends on the whole fork path and we don't want | |
| 1298 | + * all of that nested inside cgroup_mutex, so the caller must drop it and any | |
| 1299 | + * other lock that forks can wait behind. If two callers race, the loser stops | |
| 1300 | + * its never-woken kthread. | |
| 1301 | + */ | |
| 1302 | +int psi_trigger_create_rtpoll_worker(struct psi_group *group) | |
| 1303 | +{ | |
| 1304 | + struct task_struct *task; | |
| 1305 | + | |
| 1306 | + task = kthread_create(psi_rtpoll_worker, group, "psimon"); | |
| 1307 | + if (IS_ERR(task)) | |
| 1308 | + return PTR_ERR(task); | |
| 1309 | + | |
| 1310 | + scoped_guard(mutex, &group->rtpoll_trigger_lock) { | |
| 1311 | + if (!rcu_access_pointer(group->rtpoll_task)) { | |
| 1312 | + atomic_set(&group->rtpoll_wakeup, 0); | |
| 1313 | + wake_up_process(task); | |
| 1314 | + rcu_assign_pointer(group->rtpoll_task, task); | |
| 1315 | + | |
| 1316 | + /* | |
| 1317 | + * Poll once to catch up on scheduling attempts dropped | |
| 1318 | + * while there was no rtpoll worker. | |
| 1319 | + */ | |
| 1320 | + psi_schedule_rtpoll_work(group, 1, true); | |
| 1321 | + return 0; | |
| 1322 | + } | |
| 1323 | + } | |
| 1324 | + | |
| 1325 | + kthread_stop(task); | |
| 1326 | + return 0; | |
| 1327 | +} | |
| 1328 | + | |
| 1295 | 1329 | struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, |
| 1296 | 1330 | enum psi_res res, struct file *file, |
| 1297 | − struct kernfs_open_file *of) | |
| 1331 | + struct kernfs_open_file *of, | |
| 1332 | + bool *need_rtpoll_worker) | |
| 1298 | 1333 | { |
| 1299 | 1334 | struct psi_trigger *t; |
| 1300 | 1335 | enum psi_states state; |
| @@ -1302,6 +1337,8 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, · 1 unmodified line | ||
| 1302 | 1337 | bool privileged; |
| 1303 | 1338 | u32 window_us; |
| 1304 | 1339 | |
| 1340 | + *need_rtpoll_worker = false; | |
| 1341 | + | |
| 1305 | 1342 | if (static_branch_likely(&psi_disabled)) |
| 1306 | 1343 | return ERR_PTR(-EOPNOTSUPP); |
| 1307 | 1344 | |
| @@ -1362,26 +1399,14 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, · 54 unmodified lines | ||
| 1362 | 1399 | if (privileged) { |
| 1363 | 1400 | mutex_lock(&group->rtpoll_trigger_lock); |
| 1364 | 1401 | |
| 1365 | − if (!rcu_access_pointer(group->rtpoll_task)) { | |
| 1366 | − struct task_struct *task; | |
| 1367 | − | |
| 1368 | − task = kthread_create(psi_rtpoll_worker, group, "psimon"); | |
| 1369 | − if (IS_ERR(task)) { | |
| 1370 | − kfree(t); | |
| 1371 | − mutex_unlock(&group->rtpoll_trigger_lock); | |
| 1372 | − return ERR_CAST(task); | |
| 1373 | − } | |
| 1374 | − atomic_set(&group->rtpoll_wakeup, 0); | |
| 1375 | − wake_up_process(task); | |
| 1376 | − rcu_assign_pointer(group->rtpoll_task, task); | |
| 1377 | − } | |
| 1378 | − | |
| 1379 | 1402 | list_add(&t->node, &group->rtpoll_triggers); |
| 1380 | 1403 | group->rtpoll_min_period = min(group->rtpoll_min_period, |
| 1381 | 1404 | div_u64(t->win.size, UPDATES_PER_WINDOW)); |
| 1382 | 1405 | group->rtpoll_nr_triggers[t->state]++; |
| 1383 | 1406 | group->rtpoll_states |= (1 << t->state); |
| 1384 | 1407 | |
| 1408 | + *need_rtpoll_worker = !rcu_access_pointer(group->rtpoll_task); | |
| 1409 | + | |
| 1385 | 1410 | mutex_unlock(&group->rtpoll_trigger_lock); |
| 1386 | 1411 | } else { |
| 1387 | 1412 | mutex_lock(&group->avgs_lock); |
| @@ -1541,6 +1566,8 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, · 153 unmodified lines | ||
| 1541 | 1566 | size_t buf_size; |
| 1542 | 1567 | struct seq_file *seq; |
| 1543 | 1568 | struct psi_trigger *new; |
| 1569 | + bool need_rtpoll_worker; | |
| 1570 | + int ret; | |
| 1544 | 1571 | |
| 1545 | 1572 | if (static_branch_likely(&psi_disabled)) |
| 1546 | 1573 | return -EOPNOTSUPP; |
| @@ -1565,12 +1592,22 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, · 18 unmodified lines | ||
| 1565 | 1592 | return -EBUSY; |
| 1566 | 1593 | } |
| 1567 | 1594 | |
| 1568 | − new = psi_trigger_create(&psi_system, buf, res, file, NULL); | |
| 1595 | + new = psi_trigger_create(&psi_system, buf, res, file, NULL, | |
| 1596 | + &need_rtpoll_worker); | |
| 1569 | 1597 | if (IS_ERR(new)) { |
| 1570 | 1598 | mutex_unlock(&seq->lock); |
| 1571 | 1599 | return PTR_ERR(new); |
| 1572 | 1600 | } |
| 1573 | 1601 | |
| 1602 | + if (need_rtpoll_worker) { | |
| 1603 | + ret = psi_trigger_create_rtpoll_worker(&psi_system); | |
| 1604 | + if (ret) { | |
| 1605 | + psi_trigger_destroy(new); | |
| 1606 | + mutex_unlock(&seq->lock); | |
| 1607 | + return ret; | |
| 1608 | + } | |
| 1609 | + } | |
| 1610 | + | |
| 1574 | 1611 | smp_store_release(&seq->private, new); |
| 1575 | 1612 | mutex_unlock(&seq->lock); |
| 1576 | 1613 | |