sched/psi: Create the psimon kthread outside of cgroup_mutex

a5b98009f16d ("sched/psi: fix race between file release and pressure write") made pressure_write() hold cgroup_mutex across psi_trigger_create(), which forks the psimon kthread for the first rtpoll trigger. As kthread creation depends on the whole fork path, the commit inadvertently created a lot of unwanted locking dependencies from cgroup_mutex. sched_ext got hit by one: its enable path blocks forks and then grabs cgroup_mutex, so a pressure write racing a scheduler enable deadlocks, with every other fork piling up behind. Fix it by splitting trigger creation so that the worker is forked with cgroup_mutex dropped and the kernfs active reference left broken. The latter matters because rmdir and cgroup.pressure writes drain active references under cgroup_mutex. Publishing the trigger last keeps error reporting synchronous and preserves the of->priv lifetime rules. The trigger registered in the first stage pins the group's rtpoll machinery across the unlocked window, leaving only creation races to resolve. The catch-up poll on installation covers scheduling attempts dropped while there was no worker. v2: Retagged sched/psi (was cgroup).

Tejun Heo committed 4 months ago

Commit fadeedd1 parent 97fef603 files · +78 −18Browse files

Unified · Split

3 files

include/linux/psi.h +3 −1
kernel/cgroup/cgroup.c +22 −1
kernel/sched/psi.c +53 −16

include/linux/psi.h+3−1

@@ -25,7 +25,9 @@ void psi_memstall_leave(unsigned long *flags); · 24 unmodified lines
25	25	int psi_show(struct seq_file s, struct psi_group group, enum psi_res res);
26	26	struct psi_trigger psi_trigger_create(struct psi_group group, char *buf,
27	27	enum psi_res res, struct file *file,
28		− struct kernfs_open_file *of);
	28	+ struct kernfs_open_file *of,
	29	+ bool *need_rtpoll_worker);
	30	+int psi_trigger_create_rtpoll_worker(struct psi_group *group);
29	31	void psi_trigger_destroy(struct psi_trigger *t);
30	32
31	33	__poll_t psi_trigger_poll(void *trigger_ptr, struct file file,

kernel/cgroup/cgroup.c+22−1

@@ -3996,6 +3996,7 @@ static ssize_t pressure_write(struct kernfs_open_file of, char buf, · 3995 unmodified lines
3996	3996	struct psi_trigger *new;
3997	3997	struct cgroup *cgrp;
3998	3998	struct psi_group *psi;
	3999	+ bool need_rtpoll_worker;
3999	4000	ssize_t ret = 0;
4000	4001
4001	4002	cgrp = cgroup_kn_lock_live(of->kn, false);
@@ -4015,12 +4016,32 @@ static ssize_t pressure_write(struct kernfs_open_file of, char buf, · 13 unmodified lines
4015	4016	}
4016	4017
4017	4018	psi = cgroup_psi(cgrp);
4018		− new = psi_trigger_create(psi, buf, res, of->file, of);
	4019	+ new = psi_trigger_create(psi, buf, res, of->file, of,
	4020	+ &need_rtpoll_worker);
4019	4021	if (IS_ERR(new)) {
4020	4022	ret = PTR_ERR(new);
4021	4023	goto out_unlock;
4022	4024	}
4023	4025
	4026	+ /*
	4027	+ * The worker fork must run with neither cgroup_mutex nor the file's
	4028	+ * kernfs active reference held. The latter is broken since
	4029	+ * cgroup_kn_lock_live(). @of->priv may be released while unlocked, so
	4030	+ * recheck before publishing @new.
	4031	+ */
	4032	+ if (need_rtpoll_worker) {
	4033	+ cgroup_unlock();
	4034	+ ret = psi_trigger_create_rtpoll_worker(psi);
	4035	+ cgroup_lock();
	4036	+
	4037	+ if (!ret && !of->priv)
	4038	+ ret = -ENODEV;
	4039	+ if (ret) {
	4040	+ psi_trigger_destroy(new);
	4041	+ goto out_unlock;
	4042	+ }
	4043	+ }
	4044	+
4024	4045	smp_store_release(&ctx->psi.trigger, new);
4025	4046
4026	4047	out_unlock:

kernel/sched/psi.c+53−16

@@ -1292,9 +1292,44 @@ int psi_show(struct seq_file m, struct psi_group group, enum psi_res res) · 1291 unmodified lines
1292	1292	return 0;
1293	1293	}
1294	1294
	1295	+/*
	1296	+ * Create @group's rtpoll worker after psi_trigger_create() reported the need
	1297	+ * for one. kthread creation depends on the whole fork path and we don't want
	1298	+ * all of that nested inside cgroup_mutex, so the caller must drop it and any
	1299	+ * other lock that forks can wait behind. If two callers race, the loser stops
	1300	+ * its never-woken kthread.
	1301	+ */
	1302	+int psi_trigger_create_rtpoll_worker(struct psi_group *group)
	1303	+{
	1304	+ struct task_struct *task;
	1305	+
	1306	+ task = kthread_create(psi_rtpoll_worker, group, "psimon");
	1307	+ if (IS_ERR(task))
	1308	+ return PTR_ERR(task);
	1309	+
	1310	+ scoped_guard(mutex, &group->rtpoll_trigger_lock) {
	1311	+ if (!rcu_access_pointer(group->rtpoll_task)) {
	1312	+ atomic_set(&group->rtpoll_wakeup, 0);
	1313	+ wake_up_process(task);
	1314	+ rcu_assign_pointer(group->rtpoll_task, task);
	1315	+
	1316	+ /*
	1317	+ * Poll once to catch up on scheduling attempts dropped
	1318	+ * while there was no rtpoll worker.
	1319	+ */
	1320	+ psi_schedule_rtpoll_work(group, 1, true);
	1321	+ return 0;
	1322	+ }
	1323	+ }
	1324	+
	1325	+ kthread_stop(task);
	1326	+ return 0;
	1327	+}
	1328	+
1295	1329	struct psi_trigger psi_trigger_create(struct psi_group group, char *buf,
1296	1330	enum psi_res res, struct file *file,
1297		− struct kernfs_open_file *of)
	1331	+ struct kernfs_open_file *of,
	1332	+ bool *need_rtpoll_worker)
1298	1333	{
1299	1334	struct psi_trigger *t;
1300	1335	enum psi_states state;
@@ -1302,6 +1337,8 @@ struct psi_trigger psi_trigger_create(struct psi_group group, char *buf, · 1 unmodified line
1302	1337	bool privileged;
1303	1338	u32 window_us;
1304	1339
	1340	+ *need_rtpoll_worker = false;
	1341	+
1305	1342	if (static_branch_likely(&psi_disabled))
1306	1343	return ERR_PTR(-EOPNOTSUPP);
1307	1344
@@ -1362,26 +1399,14 @@ struct psi_trigger psi_trigger_create(struct psi_group group, char *buf, · 54 unmodified lines
1362	1399	if (privileged) {
1363	1400	mutex_lock(&group->rtpoll_trigger_lock);
1364	1401
1365		− if (!rcu_access_pointer(group->rtpoll_task)) {
1366		− struct task_struct *task;
1367		−
1368		− task = kthread_create(psi_rtpoll_worker, group, "psimon");
1369		− if (IS_ERR(task)) {
1370		− kfree(t);
1371		− mutex_unlock(&group->rtpoll_trigger_lock);
1372		− return ERR_CAST(task);
1373		− }
1374		− atomic_set(&group->rtpoll_wakeup, 0);
1375		− wake_up_process(task);
1376		− rcu_assign_pointer(group->rtpoll_task, task);
1377		− }
1378		−
1379	1402	list_add(&t->node, &group->rtpoll_triggers);
1380	1403	group->rtpoll_min_period = min(group->rtpoll_min_period,
1381	1404	div_u64(t->win.size, UPDATES_PER_WINDOW));
1382	1405	group->rtpoll_nr_triggers[t->state]++;
1383	1406	group->rtpoll_states \|= (1 << t->state);
1384	1407
	1408	+ *need_rtpoll_worker = !rcu_access_pointer(group->rtpoll_task);
	1409	+
1385	1410	mutex_unlock(&group->rtpoll_trigger_lock);
1386	1411	} else {
1387	1412	mutex_lock(&group->avgs_lock);
@@ -1541,6 +1566,8 @@ static ssize_t psi_write(struct file file, const char __user user_buf, · 153 unmodified lines
1541	1566	size_t buf_size;
1542	1567	struct seq_file *seq;
1543	1568	struct psi_trigger *new;
	1569	+ bool need_rtpoll_worker;
	1570	+ int ret;
1544	1571
1545	1572	if (static_branch_likely(&psi_disabled))
1546	1573	return -EOPNOTSUPP;
@@ -1565,12 +1592,22 @@ static ssize_t psi_write(struct file file, const char __user user_buf, · 18 unmodified lines
1565	1592	return -EBUSY;
1566	1593	}
1567	1594
1568		− new = psi_trigger_create(&psi_system, buf, res, file, NULL);
	1595	+ new = psi_trigger_create(&psi_system, buf, res, file, NULL,
	1596	+ &need_rtpoll_worker);
1569	1597	if (IS_ERR(new)) {
1570	1598	mutex_unlock(&seq->lock);
1571	1599	return PTR_ERR(new);
1572	1600	}
1573	1601
	1602	+ if (need_rtpoll_worker) {
	1603	+ ret = psi_trigger_create_rtpoll_worker(&psi_system);
	1604	+ if (ret) {
	1605	+ psi_trigger_destroy(new);
	1606	+ mutex_unlock(&seq->lock);
	1607	+ return ret;
	1608	+ }
	1609	+ }
	1610	+
1574	1611	smp_store_release(&seq->private, new);
1575	1612	mutex_unlock(&seq->lock);
1576	1613