Merge tag 'sched-psi-2022-10-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull PSI updates from Ingo Molnar: - Various performance optimizations, resulting in a 4%-9% speedup in the mmtests/config-scheduler-perfpipe micro-benchmark. - New interface to turn PSI on/off on a per cgroup level. * tag 'sched-psi-2022-10-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched/psi: Per-cgroup PSI accounting disable/re-enable interface sched/psi: Cache parent psi_group to speed up group iteration sched/psi: Consolidate cgroup_psi() sched/psi: Add PSI_IRQ to track IRQ/SOFTIRQ pressure sched/psi: Remove NR_ONCPU task accounting sched/psi: Optimize task switch inside shared cgroups again sched/psi: Move private helpers to sched/stats.h sched/psi: Save percpu memory when !psi_cgroups_enabled sched/psi: Don't create cgroup PSI files when psi_disabled sched/psi: Fix periodic aggregation shut off

Linus Torvalds committed 4 years ago

Commit bd9a3db2 parents 1df046a 34f26a19 files · +362 −103Browse files

Unified · Split

9 files

Documentation/admin-guide/cgroup-v2.rst +23 −0
include/linux/cgroup-defs.h +3 −0
include/linux/cgroup.h +0 −5
include/linux/psi.h +8 −4
include/linux/psi_types.h +20 −11
kernel/cgroup/cgroup.c +95 −9
kernel/sched/core.c +1 −0
kernel/sched/psi.c +206 −74
kernel/sched/stats.h +6 −0

Documentation/admin-guide/cgroup-v2.rst+23−0

@@ -976,6 +976,29 @@ All cgroup core files are prefixed with "cgroup." · 975 unmodified lines
976	976	killing cgroups is a process directed operation, i.e. it affects
977	977	the whole thread-group.
978	978
	979	+ cgroup.pressure
	980	+ A read-write single value file that allowed values are "0" and "1".
	981	+ The default is "1".
	982	+
	983	+ Writing "0" to the file will disable the cgroup PSI accounting.
	984	+ Writing "1" to the file will re-enable the cgroup PSI accounting.
	985	+
	986	+ This control attribute is not hierarchical, so disable or enable PSI
	987	+ accounting in a cgroup does not affect PSI accounting in descendants
	988	+ and doesn't need pass enablement via ancestors from root.
	989	+
	990	+ The reason this control attribute exists is that PSI accounts stalls for
	991	+ each cgroup separately and aggregates it at each level of the hierarchy.
	992	+ This may cause non-negligible overhead for some workloads when under
	993	+ deep level of the hierarchy, in which case this control attribute can
	994	+ be used to disable PSI accounting in the non-leaf cgroups.
	995	+
	996	+ irq.pressure
	997	+ A read-write nested-keyed file.
	998	+
	999	+ Shows pressure stall information for IRQ/SOFTIRQ. See
	1000	+ :ref:`Documentation/accounting/psi.rst <psi>` for details.
	1001	+
979	1002	Controllers
980	1003	===========
981	1004

include/linux/cgroup-defs.h+3−0

@@ -428,6 +428,9 @@ struct cgroup { · 427 unmodified lines
428	428	struct cgroup_file procs_file; /* handle for "cgroup.procs" */
429	429	struct cgroup_file events_file; /* handle for "cgroup.events" */
430	430
	431	+ /* handles for "{cpu,memory,io,irq}.pressure" */
	432	+ struct cgroup_file psi_files[NR_PSI_RESOURCES];
	433	+
431	434	/*
432	435	* The bitmask of subsystems enabled on the child cgroups.
433	436	* ->subtree_control is the one configured through

include/linux/cgroup.h+0−5

@@ -682,11 +682,6 @@ static inline void pr_cont_cgroup_path(struct cgroup *cgrp) · 681 unmodified lines
682	682	pr_cont_kernfs_path(cgrp->kn);
683	683	}
684	684
685		−static inline struct psi_group cgroup_psi(struct cgroup cgrp)
686		−{
687		− return cgrp->psi;
688		−}
689		−
690	685	bool cgroup_psi_enabled(void);
691	686
692	687	static inline void cgroup_init_kthreadd(void)

include/linux/psi.h+8−4

@@ -7,6 +7,7 @@ · 6 unmodified lines
7	7	#include <linux/sched.h>
8	8	#include <linux/poll.h>
9	9	#include <linux/cgroup-defs.h>
	10	+#include <linux/cgroup.h>
10	11
11	12	struct seq_file;
12	13	struct css_set;
@@ -18,10 +19,6 @@ extern struct psi_group psi_system; · 5 unmodified lines
18	19
19	20	void psi_init(void);
20	21
21		−void psi_task_change(struct task_struct *task, int clear, int set);
22		−void psi_task_switch(struct task_struct prev, struct task_struct next,
23		− bool sleep);
24		−
25	22	void psi_memstall_enter(unsigned long *flags);
26	23	void psi_memstall_leave(unsigned long *flags);
27	24
@@ -34,9 +31,15 @@ __poll_t psi_trigger_poll(void *trigger_ptr, struct file file, · 6 unmodified lines
34	31	poll_table *wait);
35	32
36	33	#ifdef CONFIG_CGROUPS
	34	+static inline struct psi_group cgroup_psi(struct cgroup cgrp)
	35	+{
	36	+ return cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;
	37	+}
	38	+
37	39	int psi_cgroup_alloc(struct cgroup *cgrp);
38	40	void psi_cgroup_free(struct cgroup *cgrp);
39	41	void cgroup_move_task(struct task_struct p, struct css_set to);
	42	+void psi_cgroup_restart(struct psi_group *group);
40	43	#endif
41	44
42	45	#else /* CONFIG_PSI */
@@ -58,6 +61,7 @@ static inline void cgroup_move_task(struct task_struct p, struct css_set to) · 15 unmodified lines
58	61	{
59	62	rcu_assign_pointer(p->cgroups, to);
60	63	}
	64	+static inline void psi_cgroup_restart(struct psi_group *group) {}
61	65	#endif
62	66
63	67	#endif /* CONFIG_PSI */

include/linux/psi_types.h+20−11

@@ -15,13 +15,6 @@ enum psi_task_count { · 14 unmodified lines
15	15	NR_IOWAIT,
16	16	NR_MEMSTALL,
17	17	NR_RUNNING,
18		− /*
19		− * This can't have values other than 0 or 1 and could be
20		− * implemented as a bit flag. But for now we still have room
21		− * in the first cacheline of psi_group_cpu, and this way we
22		− * don't have to special case any state tracking for it.
23		− */
24		− NR_ONCPU,
25	18	/*
26	19	* For IO and CPU stalls the presence of running/oncpu tasks
27	20	* in the domain means a partial rather than a full stall.
@@ -32,22 +25,27 @@ enum psi_task_count { · 4 unmodified lines
32	25	* threads and memstall ones.
33	26	*/
34	27	NR_MEMSTALL_RUNNING,
35		− NR_PSI_TASK_COUNTS = 5,
	28	+ NR_PSI_TASK_COUNTS = 4,
36	29	};
37	30
38	31	/* Task state bitmasks */
39	32	#define TSK_IOWAIT (1 << NR_IOWAIT)
40	33	#define TSK_MEMSTALL (1 << NR_MEMSTALL)
41	34	#define TSK_RUNNING (1 << NR_RUNNING)
42		−#define TSK_ONCPU (1 << NR_ONCPU)
43	35	#define TSK_MEMSTALL_RUNNING (1 << NR_MEMSTALL_RUNNING)
44	36
	37	+/* Only one task can be scheduled, no corresponding task count */
	38	+#define TSK_ONCPU (1 << NR_PSI_TASK_COUNTS)
	39	+
45	40	/* Resources that workloads could be stalled on */
46	41	enum psi_res {
47	42	PSI_IO,
48	43	PSI_MEM,
49	44	PSI_CPU,
50		− NR_PSI_RESOURCES = 3,
	45	+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
	46	+ PSI_IRQ,
	47	+#endif
	48	+ NR_PSI_RESOURCES,
51	49	};
52	50
53	51	/*
@@ -63,11 +61,17 @@ enum psi_states { · 9 unmodified lines
63	61	PSI_MEM_FULL,
64	62	PSI_CPU_SOME,
65	63	PSI_CPU_FULL,
	64	+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
	65	+ PSI_IRQ_FULL,
	66	+#endif
66	67	/* Only per-CPU, to weigh the CPU in the global average: */
67	68	PSI_NONIDLE,
68		− NR_PSI_STATES = 7,
	69	+ NR_PSI_STATES,
69	70	};
70	71
	72	+/* Use one bit in the state mask to track TSK_ONCPU */
	73	+#define PSI_ONCPU (1 << NR_PSI_STATES)
	74	+
71	75	enum psi_aggregators {
72	76	PSI_AVGS = 0,
73	77	PSI_POLL,
@@ -147,6 +151,9 @@ struct psi_trigger { · 73 unmodified lines
147	151	};
148	152
149	153	struct psi_group {
	154	+ struct psi_group *parent;
	155	+ bool enabled;
	156	+
150	157	/* Protects data used by the aggregator */
151	158	struct mutex avgs_lock;
152	159
@@ -188,6 +195,8 @@ struct psi_group { · 35 unmodified lines
188	195
189	196	#else /* CONFIG_PSI */
190	197
	198	+#define NR_PSI_RESOURCES 0
	199	+
191	200	struct psi_group { };
192	201
193	202	#endif /* CONFIG_PSI */

kernel/cgroup/cgroup.c+95−9

@@ -3698,27 +3698,27 @@ static int cpu_stat_show(struct seq_file seq, void v) · 3697 unmodified lines
3698	3698	static int cgroup_io_pressure_show(struct seq_file seq, void v)
3699	3699	{
3700	3700	struct cgroup *cgrp = seq_css(seq)->cgroup;
3701		− struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;
	3701	+ struct psi_group *psi = cgroup_psi(cgrp);
3702	3702
3703	3703	return psi_show(seq, psi, PSI_IO);
3704	3704	}
3705	3705	static int cgroup_memory_pressure_show(struct seq_file seq, void v)
3706	3706	{
3707	3707	struct cgroup *cgrp = seq_css(seq)->cgroup;
3708		− struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;
	3708	+ struct psi_group *psi = cgroup_psi(cgrp);
3709	3709
3710	3710	return psi_show(seq, psi, PSI_MEM);
3711	3711	}
3712	3712	static int cgroup_cpu_pressure_show(struct seq_file seq, void v)
3713	3713	{
3714	3714	struct cgroup *cgrp = seq_css(seq)->cgroup;
3715		− struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;
	3715	+ struct psi_group *psi = cgroup_psi(cgrp);
3716	3716
3717	3717	return psi_show(seq, psi, PSI_CPU);
3718	3718	}
3719	3719
3720		−static ssize_t cgroup_pressure_write(struct kernfs_open_file of, char buf,
3721		− size_t nbytes, enum psi_res res)
	3720	+static ssize_t pressure_write(struct kernfs_open_file of, char buf,
	3721	+ size_t nbytes, enum psi_res res)
3722	3722	{
3723	3723	struct cgroup_file_ctx *ctx = of->priv;
3724	3724	struct psi_trigger *new;
@@ -3738,7 +3738,7 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file of, char buf, · 13 unmodified lines
3738	3738	return -EBUSY;
3739	3739	}
3740	3740
3741		− psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;
	3741	+ psi = cgroup_psi(cgrp);
3742	3742	new = psi_trigger_create(psi, buf, res);
3743	3743	if (IS_ERR(new)) {
3744	3744	cgroup_put(cgrp);
@@ -3755,21 +3755,86 @@ static ssize_t cgroup_io_pressure_write(struct kernfs_open_file *of, · 10 unmodified lines
3755	3755	char *buf, size_t nbytes,
3756	3756	loff_t off)
3757	3757	{
3758		− return cgroup_pressure_write(of, buf, nbytes, PSI_IO);
	3758	+ return pressure_write(of, buf, nbytes, PSI_IO);
3759	3759	}
3760	3760
3761	3761	static ssize_t cgroup_memory_pressure_write(struct kernfs_open_file *of,
3762	3762	char *buf, size_t nbytes,
3763	3763	loff_t off)
3764	3764	{
3765		− return cgroup_pressure_write(of, buf, nbytes, PSI_MEM);
	3765	+ return pressure_write(of, buf, nbytes, PSI_MEM);
3766	3766	}
3767	3767
3768	3768	static ssize_t cgroup_cpu_pressure_write(struct kernfs_open_file *of,
3769	3769	char *buf, size_t nbytes,
3770	3770	loff_t off)
3771	3771	{
3772		− return cgroup_pressure_write(of, buf, nbytes, PSI_CPU);
	3772	+ return pressure_write(of, buf, nbytes, PSI_CPU);
	3773	+}
	3774	+
	3775	+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
	3776	+static int cgroup_irq_pressure_show(struct seq_file seq, void v)
	3777	+{
	3778	+ struct cgroup *cgrp = seq_css(seq)->cgroup;
	3779	+ struct psi_group *psi = cgroup_psi(cgrp);
	3780	+
	3781	+ return psi_show(seq, psi, PSI_IRQ);
	3782	+}
	3783	+
	3784	+static ssize_t cgroup_irq_pressure_write(struct kernfs_open_file *of,
	3785	+ char *buf, size_t nbytes,
	3786	+ loff_t off)
	3787	+{
	3788	+ return pressure_write(of, buf, nbytes, PSI_IRQ);
	3789	+}
	3790	+#endif
	3791	+
	3792	+static int cgroup_pressure_show(struct seq_file seq, void v)
	3793	+{
	3794	+ struct cgroup *cgrp = seq_css(seq)->cgroup;
	3795	+ struct psi_group *psi = cgroup_psi(cgrp);
	3796	+
	3797	+ seq_printf(seq, "%d\n", psi->enabled);
	3798	+
	3799	+ return 0;
	3800	+}
	3801	+
	3802	+static ssize_t cgroup_pressure_write(struct kernfs_open_file *of,
	3803	+ char *buf, size_t nbytes,
	3804	+ loff_t off)
	3805	+{
	3806	+ ssize_t ret;
	3807	+ int enable;
	3808	+ struct cgroup *cgrp;
	3809	+ struct psi_group *psi;
	3810	+
	3811	+ ret = kstrtoint(strstrip(buf), 0, &enable);
	3812	+ if (ret)
	3813	+ return ret;
	3814	+
	3815	+ if (enable < 0 \|\| enable > 1)
	3816	+ return -ERANGE;
	3817	+
	3818	+ cgrp = cgroup_kn_lock_live(of->kn, false);
	3819	+ if (!cgrp)
	3820	+ return -ENOENT;
	3821	+
	3822	+ psi = cgroup_psi(cgrp);
	3823	+ if (psi->enabled != enable) {
	3824	+ int i;
	3825	+
	3826	+ /* show or hide {cpu,memory,io,irq}.pressure files */
	3827	+ for (i = 0; i < NR_PSI_RESOURCES; i++)
	3828	+ cgroup_file_show(&cgrp->psi_files[i], enable);
	3829	+
	3830	+ psi->enabled = enable;
	3831	+ if (enable)
	3832	+ psi_cgroup_restart(psi);
	3833	+ }
	3834	+
	3835	+ cgroup_kn_unlock(of->kn);
	3836	+
	3837	+ return nbytes;
3773	3838	}
3774	3839
3775	3840	static __poll_t cgroup_pressure_poll(struct kernfs_open_file *of,
@@ -3789,6 +3854,9 @@ static void cgroup_pressure_release(struct kernfs_open_file *of) · 13 unmodified lines
3789	3854
3790	3855	bool cgroup_psi_enabled(void)
3791	3856	{
	3857	+ if (static_branch_likely(&psi_disabled))
	3858	+ return false;
	3859	+
3792	3860	return (cgroup_feature_disable_mask & (1 << OPT_FEATURE_PRESSURE)) == 0;
3793	3861	}
3794	3862
@@ -5175,6 +5243,7 @@ static struct cftype cgroup_psi_files[] = { · 1380 unmodified lines
5175	5243	#ifdef CONFIG_PSI
5176	5244	{
5177	5245	.name = "io.pressure",
	5246	+ .file_offset = offsetof(struct cgroup, psi_files[PSI_IO]),
5178	5247	.seq_show = cgroup_io_pressure_show,
5179	5248	.write = cgroup_io_pressure_write,
5180	5249	.poll = cgroup_pressure_poll,
@@ -5182,6 +5251,7 @@ static struct cftype cgroup_psi_files[] = { · 1 unmodified line
5182	5251	},
5183	5252	{
5184	5253	.name = "memory.pressure",
	5254	+ .file_offset = offsetof(struct cgroup, psi_files[PSI_MEM]),
5185	5255	.seq_show = cgroup_memory_pressure_show,
5186	5256	.write = cgroup_memory_pressure_write,
5187	5257	.poll = cgroup_pressure_poll,
@@ -5189,11 +5259,27 @@ static struct cftype cgroup_psi_files[] = { · 1 unmodified line
5189	5259	},
5190	5260	{
5191	5261	.name = "cpu.pressure",
	5262	+ .file_offset = offsetof(struct cgroup, psi_files[PSI_CPU]),
5192	5263	.seq_show = cgroup_cpu_pressure_show,
5193	5264	.write = cgroup_cpu_pressure_write,
5194	5265	.poll = cgroup_pressure_poll,
5195	5266	.release = cgroup_pressure_release,
5196	5267	},
	5268	+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
	5269	+ {
	5270	+ .name = "irq.pressure",
	5271	+ .file_offset = offsetof(struct cgroup, psi_files[PSI_IRQ]),
	5272	+ .seq_show = cgroup_irq_pressure_show,
	5273	+ .write = cgroup_irq_pressure_write,
	5274	+ .poll = cgroup_pressure_poll,
	5275	+ .release = cgroup_pressure_release,
	5276	+ },
	5277	+#endif
	5278	+ {
	5279	+ .name = "cgroup.pressure",
	5280	+ .seq_show = cgroup_pressure_show,
	5281	+ .write = cgroup_pressure_write,
	5282	+ },
5197	5283	#endif /* CONFIG_PSI */
5198	5284	{ } /* terminate */
5199	5285	};

kernel/sched/core.c+1−0

@@ -701,6 +701,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) · 700 unmodified lines
701	701
702	702	rq->prev_irq_time += irq_delta;
703	703	delta -= irq_delta;
	704	+ psi_account_irqtime(rq->curr, irq_delta);
704	705	#endif
705	706	#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
706	707	if (static_key_false((&paravirt_steal_rq_enabled))) {

kernel/sched/psi.c+206−74

@@ -181,6 +181,7 @@ static void group_init(struct psi_group *group) · 180 unmodified lines
181	181	{
182	182	int cpu;
183	183
	184	+ group->enabled = true;
184	185	for_each_possible_cpu(cpu)
185	186	seqcount_init(&per_cpu_ptr(group->pcpu, cpu)->seq);
186	187	group->avg_last_update = sched_clock();
@@ -201,6 +202,7 @@ void __init psi_init(void) · 14 unmodified lines
201	202	{
202	203	if (!psi_enable) {
203	204	static_branch_enable(&psi_disabled);
	205	+ static_branch_disable(&psi_cgroups_enabled);
204	206	return;
205	207	}
206	208
@@ -211,7 +213,7 @@ void __init psi_init(void) · 4 unmodified lines
211	213	group_init(&psi_system);
212	214	}
213	215
214		−static bool test_state(unsigned int *tasks, enum psi_states state)
	216	+static bool test_state(unsigned int *tasks, enum psi_states state, bool oncpu)
215	217	{
216	218	switch (state) {
217	219	case PSI_IO_SOME:
@@ -224,9 +226,9 @@ static bool test_state(unsigned int *tasks, enum psi_states state) · 6 unmodified lines
224	226	return unlikely(tasks[NR_MEMSTALL] &&
225	227	tasks[NR_RUNNING] == tasks[NR_MEMSTALL_RUNNING]);
226	228	case PSI_CPU_SOME:
227		− return unlikely(tasks[NR_RUNNING] > tasks[NR_ONCPU]);
	229	+ return unlikely(tasks[NR_RUNNING] > oncpu);
228	230	case PSI_CPU_FULL:
229		− return unlikely(tasks[NR_RUNNING] && !tasks[NR_ONCPU]);
	231	+ return unlikely(tasks[NR_RUNNING] && !oncpu);
230	232	case PSI_NONIDLE:
231	233	return tasks[NR_IOWAIT] \|\| tasks[NR_MEMSTALL] \|\|
232	234	tasks[NR_RUNNING];
@@ -688,35 +690,53 @@ static void psi_group_change(struct psi_group *group, int cpu, · 455 unmodified lines
688	690	bool wake_clock)
689	691	{
690	692	struct psi_group_cpu *groupc;
691		− u32 state_mask = 0;
692	693	unsigned int t, m;
693	694	enum psi_states s;
	695	+ u32 state_mask;
694	696
695	697	groupc = per_cpu_ptr(group->pcpu, cpu);
696	698
697	699	/*
698		− * First we assess the aggregate resource states this CPU's
699		− * tasks have been in since the last change, and account any
700		− * SOME and FULL time these may have resulted in.
701		− *
702		− * Then we update the task counts according to the state
	700	+ * First we update the task counts according to the state
703	701	* change requested through the @clear and @set bits.
	702	+ *
	703	+ * Then if the cgroup PSI stats accounting enabled, we
	704	+ * assess the aggregate resource states this CPU's tasks
	705	+ * have been in since the last change, and account any
	706	+ * SOME and FULL time these may have resulted in.
704	707	*/
705	708	write_seqcount_begin(&groupc->seq);
706	709
707		− record_times(groupc, now);
	710	+ /*
	711	+ * Start with TSK_ONCPU, which doesn't have a corresponding
	712	+ * task count - it's just a boolean flag directly encoded in
	713	+ * the state mask. Clear, set, or carry the current state if
	714	+ * no changes are requested.
	715	+ */
	716	+ if (unlikely(clear & TSK_ONCPU)) {
	717	+ state_mask = 0;
	718	+ clear &= ~TSK_ONCPU;
	719	+ } else if (unlikely(set & TSK_ONCPU)) {
	720	+ state_mask = PSI_ONCPU;
	721	+ set &= ~TSK_ONCPU;
	722	+ } else {
	723	+ state_mask = groupc->state_mask & PSI_ONCPU;
	724	+ }
708	725
	726	+ /*
	727	+ * The rest of the state mask is calculated based on the task
	728	+ * counts. Update those first, then construct the mask.
	729	+ */
709	730	for (t = 0, m = clear; m; m &= ~(1 << t), t++) {
710	731	if (!(m & (1 << t)))
711	732	continue;
712	733	if (groupc->tasks[t]) {
713	734	groupc->tasks[t]--;
714	735	} else if (!psi_bug) {
715		− printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u %u] clear=%x set=%x\n",
	736	+ printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u] clear=%x set=%x\n",
716	737	cpu, t, groupc->tasks[0],
717	738	groupc->tasks[1], groupc->tasks[2],
718		− groupc->tasks[3], groupc->tasks[4],
719		− clear, set);
	739	+ groupc->tasks[3], clear, set);
720	740	psi_bug = 1;
721	741	}
722	742	}
@@ -725,9 +745,25 @@ static void psi_group_change(struct psi_group *group, int cpu, · 2 unmodified lines
725	745	if (set & (1 << t))
726	746	groupc->tasks[t]++;
727	747
728		− /* Calculate state mask representing active states */
	748	+ if (!group->enabled) {
	749	+ /*
	750	+ * On the first group change after disabling PSI, conclude
	751	+ * the current state and flush its time. This is unlikely
	752	+ * to matter to the user, but aggregation (get_recent_times)
	753	+ * may have already incorporated the live state into times_prev;
	754	+ * avoid a delta sample underflow when PSI is later re-enabled.
	755	+ */
	756	+ if (unlikely(groupc->state_mask & (1 << PSI_NONIDLE)))
	757	+ record_times(groupc, now);
	758	+
	759	+ groupc->state_mask = state_mask;
	760	+
	761	+ write_seqcount_end(&groupc->seq);
	762	+ return;
	763	+ }
	764	+
729	765	for (s = 0; s < NR_PSI_STATES; s++) {
730		− if (test_state(groupc->tasks, s))
	766	+ if (test_state(groupc->tasks, s, state_mask & PSI_ONCPU))
731	767	state_mask \|= (1 << s);
732	768	}
733	769
@@ -739,9 +775,11 @@ static void psi_group_change(struct psi_group *group, int cpu, · 5 unmodified lines
739	775	* task in a cgroup is in_memstall, the corresponding groupc
740	776	* on that cpu is in PSI_MEM_FULL state.
741	777	*/
742		− if (unlikely(groupc->tasks[NR_ONCPU] && cpu_curr(cpu)->in_memstall))
	778	+ if (unlikely((state_mask & PSI_ONCPU) && cpu_curr(cpu)->in_memstall))
743	779	state_mask \|= (1 << PSI_MEM_FULL);
744	780
	781	+ record_times(groupc, now);
	782	+
745	783	groupc->state_mask = state_mask;
746	784
747	785	write_seqcount_end(&groupc->seq);
@@ -753,27 +791,12 @@ static void psi_group_change(struct psi_group *group, int cpu, · 5 unmodified lines
753	791	schedule_delayed_work(&group->avgs_work, PSI_FREQ);
754	792	}
755	793
756		−static struct psi_group iterate_groups(struct task_struct task, void **iter)
	794	+static inline struct psi_group task_psi_group(struct task_struct task)
757	795	{
758		− if (*iter == &psi_system)
759		− return NULL;
760		−
761	796	#ifdef CONFIG_CGROUPS
762		− if (static_branch_likely(&psi_cgroups_enabled)) {
763		− struct cgroup *cgroup = NULL;
764		−
765		− if (!*iter)
766		− cgroup = task->cgroups->dfl_cgrp;
767		− else
768		− cgroup = cgroup_parent(*iter);
769		−
770		− if (cgroup && cgroup_parent(cgroup)) {
771		− *iter = cgroup;
772		− return cgroup_psi(cgroup);
773		− }
774		− }
	797	+ if (static_branch_likely(&psi_cgroups_enabled))
	798	+ return cgroup_psi(task_dfl_cgroup(task));
775	799	#endif
776		− *iter = &psi_system;
777	800	return &psi_system;
778	801	}
779	802
@@ -796,8 +819,6 @@ void psi_task_change(struct task_struct *task, int clear, int set) · 16 unmodified lines
796	819	{
797	820	int cpu = task_cpu(task);
798	821	struct psi_group *group;
799		− bool wake_clock = true;
800		− void *iter = NULL;
801	822	u64 now;
802	823
803	824	if (!task->pid)
@@ -806,19 +827,11 @@ void psi_task_change(struct task_struct *task, int clear, int set) · 2 unmodified lines
806	827	psi_flags_change(task, clear, set);
807	828
808	829	now = cpu_clock(cpu);
809		− /*
810		− * Periodic aggregation shuts off if there is a period of no
811		− * task changes, so we wake it back up if necessary. However,
812		− * don't do this if the task change is the aggregation worker
813		− * itself going to sleep, or we'll ping-pong forever.
814		− */
815		− if (unlikely((clear & TSK_RUNNING) &&
816		− (task->flags & PF_WQ_WORKER) &&
817		− wq_worker_last_func(task) == psi_avgs_work))
818		− wake_clock = false;
819	830
820		− while ((group = iterate_groups(task, &iter)))
821		− psi_group_change(group, cpu, clear, set, now, wake_clock);
	831	+ group = task_psi_group(task);
	832	+ do {
	833	+ psi_group_change(group, cpu, clear, set, now, true);
	834	+ } while ((group = group->parent));
822	835	}
823	836
824	837	void psi_task_switch(struct task_struct prev, struct task_struct next,
@@ -826,34 +839,30 @@ void psi_task_switch(struct task_struct prev, struct task_struct next, · 1 unmodified line
826	839	{
827	840	struct psi_group group, common = NULL;
828	841	int cpu = task_cpu(prev);
829		− void *iter;
830	842	u64 now = cpu_clock(cpu);
831	843
832	844	if (next->pid) {
833		− bool identical_state;
834		−
835	845	psi_flags_change(next, 0, TSK_ONCPU);
836	846	/*
837		− * When switching between tasks that have an identical
838		− * runtime state, the cgroup that contains both tasks
839		− * we reach the first common ancestor. Iterate @next's
840		− * ancestors only until we encounter @prev's ONCPU.
	847	+ * Set TSK_ONCPU on @next's cgroups. If @next shares any
	848	+ * ancestors with @prev, those will already have @prev's
	849	+ * TSK_ONCPU bit set, and we can stop the iteration there.
841	850	*/
842		− identical_state = prev->psi_flags == next->psi_flags;
843		− iter = NULL;
844		− while ((group = iterate_groups(next, &iter))) {
845		− if (identical_state &&
846		− per_cpu_ptr(group->pcpu, cpu)->tasks[NR_ONCPU]) {
	851	+ group = task_psi_group(next);
	852	+ do {
	853	+ if (per_cpu_ptr(group->pcpu, cpu)->state_mask &
	854	+ PSI_ONCPU) {
847	855	common = group;
848	856	break;
849	857	}
850	858
851	859	psi_group_change(group, cpu, 0, TSK_ONCPU, now, true);
852		− }
	860	+ } while ((group = group->parent));
853	861	}
854	862
855	863	if (prev->pid) {
856	864	int clear = TSK_ONCPU, set = 0;
	865	+ bool wake_clock = true;
857	866
858	867	/*
859	868	* When we're going to sleep, psi_dequeue() lets us
@@ -867,26 +876,74 @@ void psi_task_switch(struct task_struct prev, struct task_struct next, · 7 unmodified lines
867	876	clear \|= TSK_MEMSTALL_RUNNING;
868	877	if (prev->in_iowait)
869	878	set \|= TSK_IOWAIT;
	879	+
	880	+ /*
	881	+ * Periodic aggregation shuts off if there is a period of no
	882	+ * task changes, so we wake it back up if necessary. However,
	883	+ * don't do this if the task change is the aggregation worker
	884	+ * itself going to sleep, or we'll ping-pong forever.
	885	+ */
	886	+ if (unlikely((prev->flags & PF_WQ_WORKER) &&
	887	+ wq_worker_last_func(prev) == psi_avgs_work))
	888	+ wake_clock = false;
870	889	}
871	890
872	891	psi_flags_change(prev, clear, set);
873	892
874		− iter = NULL;
875		− while ((group = iterate_groups(prev, &iter)) && group != common)
876		− psi_group_change(group, cpu, clear, set, now, true);
	893	+ group = task_psi_group(prev);
	894	+ do {
	895	+ if (group == common)
	896	+ break;
	897	+ psi_group_change(group, cpu, clear, set, now, wake_clock);
	898	+ } while ((group = group->parent));
877	899
878	900	/*
879		− * TSK_ONCPU is handled up to the common ancestor. If we're tasked
880		− * with dequeuing too, finish that for the rest of the hierarchy.
	901	+ * TSK_ONCPU is handled up to the common ancestor. If there are
	902	+ * any other differences between the two tasks (e.g. prev goes
	903	+ * to sleep, or only one task is memstall), finish propagating
	904	+ * those differences all the way up to the root.
881	905	*/
882		− if (sleep) {
	906	+ if ((prev->psi_flags ^ next->psi_flags) & ~TSK_ONCPU) {
883	907	clear &= ~TSK_ONCPU;
884		− for (; group; group = iterate_groups(prev, &iter))
885		− psi_group_change(group, cpu, clear, set, now, true);
	908	+ for (; group; group = group->parent)
	909	+ psi_group_change(group, cpu, clear, set, now, wake_clock);
886	910	}
887	911	}
888	912	}
889	913
	914	+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
	915	+void psi_account_irqtime(struct task_struct *task, u32 delta)
	916	+{
	917	+ int cpu = task_cpu(task);
	918	+ struct psi_group *group;
	919	+ struct psi_group_cpu *groupc;
	920	+ u64 now;
	921	+
	922	+ if (!task->pid)
	923	+ return;
	924	+
	925	+ now = cpu_clock(cpu);
	926	+
	927	+ group = task_psi_group(task);
	928	+ do {
	929	+ if (!group->enabled)
	930	+ continue;
	931	+
	932	+ groupc = per_cpu_ptr(group->pcpu, cpu);
	933	+
	934	+ write_seqcount_begin(&groupc->seq);
	935	+
	936	+ record_times(groupc, now);
	937	+ groupc->times[PSI_IRQ_FULL] += delta;
	938	+
	939	+ write_seqcount_end(&groupc->seq);
	940	+
	941	+ if (group->poll_states & (1 << PSI_IRQ_FULL))
	942	+ psi_schedule_poll_work(group, 1);
	943	+ } while ((group = group->parent));
	944	+}
	945	+#endif
	946	+
890	947	/**
891	948	* psi_memstall_enter - mark the beginning of a memory stall section
892	949	* @flags: flags to handle nested sections
@@ -952,7 +1009,7 @@ EXPORT_SYMBOL_GPL(psi_memstall_leave); · 59 unmodified lines
952	1009	#ifdef CONFIG_CGROUPS
953	1010	int psi_cgroup_alloc(struct cgroup *cgroup)
954	1011	{
955		− if (static_branch_likely(&psi_disabled))
	1012	+ if (!static_branch_likely(&psi_cgroups_enabled))
956	1013	return 0;
957	1014
958	1015	cgroup->psi = kzalloc(sizeof(struct psi_group), GFP_KERNEL);
@@ -965,12 +1022,13 @@ int psi_cgroup_alloc(struct cgroup *cgroup) · 6 unmodified lines
965	1022	return -ENOMEM;
966	1023	}
967	1024	group_init(cgroup->psi);
	1025	+ cgroup->psi->parent = cgroup_psi(cgroup_parent(cgroup));
968	1026	return 0;
969	1027	}
970	1028
971	1029	void psi_cgroup_free(struct cgroup *cgroup)
972	1030	{
973		− if (static_branch_likely(&psi_disabled))
	1031	+ if (!static_branch_likely(&psi_cgroups_enabled))
974	1032	return;
975	1033
976	1034	cancel_delayed_work_sync(&cgroup->psi->avgs_work);
@@ -998,7 +1056,7 @@ void cgroup_move_task(struct task_struct task, struct css_set to) · 21 unmodified lines
998	1056	struct rq_flags rf;
999	1057	struct rq *rq;
1000	1058
1001		− if (static_branch_likely(&psi_disabled)) {
	1059	+ if (!static_branch_likely(&psi_cgroups_enabled)) {
1002	1060	/*
1003	1061	* Lame to do this here, but the scheduler cannot be locked
1004	1062	* from the outside, so we move cgroups from inside sched/.
@@ -1046,10 +1104,45 @@ void cgroup_move_task(struct task_struct task, struct css_set to) · 41 unmodified lines
1046	1104
1047	1105	task_rq_unlock(rq, task, &rf);
1048	1106	}
	1107	+
	1108	+void psi_cgroup_restart(struct psi_group *group)
	1109	+{
	1110	+ int cpu;
	1111	+
	1112	+ /*
	1113	+ * After we disable psi_group->enabled, we don't actually
	1114	+ * stop percpu tasks accounting in each psi_group_cpu,
	1115	+ * instead only stop test_state() loop, record_times()
	1116	+ * and averaging worker, see psi_group_change() for details.
	1117	+ *
	1118	+ * When disable cgroup PSI, this function has nothing to sync
	1119	+ * since cgroup pressure files are hidden and percpu psi_group_cpu
	1120	+ * would see !psi_group->enabled and only do task accounting.
	1121	+ *
	1122	+ * When re-enable cgroup PSI, this function use psi_group_change()
	1123	+ * to get correct state mask from test_state() loop on tasks[],
	1124	+ * and restart groupc->state_start from now, use .clear = .set = 0
	1125	+ * here since no task status really changed.
	1126	+ */
	1127	+ if (!group->enabled)
	1128	+ return;
	1129	+
	1130	+ for_each_possible_cpu(cpu) {
	1131	+ struct rq *rq = cpu_rq(cpu);
	1132	+ struct rq_flags rf;
	1133	+ u64 now;
	1134	+
	1135	+ rq_lock_irq(rq, &rf);
	1136	+ now = cpu_clock(cpu);
	1137	+ psi_group_change(group, cpu, 0, 0, now, true);
	1138	+ rq_unlock_irq(rq, &rf);
	1139	+ }
	1140	+}
1049	1141	#endif /* CONFIG_CGROUPS */
1050	1142
1051	1143	int psi_show(struct seq_file m, struct psi_group group, enum psi_res res)
1052	1144	{
	1145	+ bool only_full = false;
1053	1146	int full;
1054	1147	u64 now;
1055	1148
@@ -1064,7 +1157,11 @@ int psi_show(struct seq_file m, struct psi_group group, enum psi_res res) · 8 unmodified lines
1064	1157	group->avg_next_update = update_averages(group, now);
1065	1158	mutex_unlock(&group->avgs_lock);
1066	1159
1067		− for (full = 0; full < 2; full++) {
	1160	+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
	1161	+ only_full = res == PSI_IRQ;
	1162	+#endif
	1163	+
	1164	+ for (full = 0; full < 2 - only_full; full++) {
1068	1165	unsigned long avg[3] = { 0, };
1069	1166	u64 total = 0;
1070	1167	int w;
@@ -1078,7 +1175,7 @@ int psi_show(struct seq_file m, struct psi_group group, enum psi_res res) · 7 unmodified lines
1078	1175	}
1079	1176
1080	1177	seq_printf(m, "%s avg10=%lu.%02lu avg60=%lu.%02lu avg300=%lu.%02lu total=%llu\n",
1081		− full ? "full" : "some",
	1178	+ full \|\| only_full ? "full" : "some",
1082	1179	LOAD_INT(avg[0]), LOAD_FRAC(avg[0]),
1083	1180	LOAD_INT(avg[1]), LOAD_FRAC(avg[1]),
1084	1181	LOAD_INT(avg[2]), LOAD_FRAC(avg[2]),
@@ -1106,6 +1203,11 @@ struct psi_trigger psi_trigger_create(struct psi_group group, · 21 unmodified lines
1106	1203	else
1107	1204	return ERR_PTR(-EINVAL);
1108	1205
	1206	+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
	1207	+ if (res == PSI_IRQ && --state != PSI_IRQ_FULL)
	1208	+ return ERR_PTR(-EINVAL);
	1209	+#endif
	1210	+
1109	1211	if (state >= PSI_NONIDLE)
1110	1212	return ERR_PTR(-EINVAL);
1111	1213
@@ -1390,6 +1492,33 @@ static const struct proc_ops psi_cpu_proc_ops = { · 278 unmodified lines
1390	1492	.proc_release = psi_fop_release,
1391	1493	};
1392	1494
	1495	+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
	1496	+static int psi_irq_show(struct seq_file m, void v)
	1497	+{
	1498	+ return psi_show(m, &psi_system, PSI_IRQ);
	1499	+}
	1500	+
	1501	+static int psi_irq_open(struct inode inode, struct file file)
	1502	+{
	1503	+ return psi_open(file, psi_irq_show);
	1504	+}
	1505	+
	1506	+static ssize_t psi_irq_write(struct file file, const char __user user_buf,
	1507	+ size_t nbytes, loff_t *ppos)
	1508	+{
	1509	+ return psi_write(file, user_buf, nbytes, PSI_IRQ);
	1510	+}
	1511	+
	1512	+static const struct proc_ops psi_irq_proc_ops = {
	1513	+ .proc_open = psi_irq_open,
	1514	+ .proc_read = seq_read,
	1515	+ .proc_lseek = seq_lseek,
	1516	+ .proc_write = psi_irq_write,
	1517	+ .proc_poll = psi_fop_poll,
	1518	+ .proc_release = psi_fop_release,
	1519	+};
	1520	+#endif
	1521	+
1393	1522	static int __init psi_proc_init(void)
1394	1523	{
1395	1524	if (psi_enable) {
@@ -1397,6 +1526,9 @@ static int __init psi_proc_init(void) · 1 unmodified line
1397	1526	proc_create("pressure/io", 0666, NULL, &psi_io_proc_ops);
1398	1527	proc_create("pressure/memory", 0666, NULL, &psi_memory_proc_ops);
1399	1528	proc_create("pressure/cpu", 0666, NULL, &psi_cpu_proc_ops);
	1529	+#ifdef CONFIG_IRQ_TIME_ACCOUNTING
	1530	+ proc_create("pressure/irq", 0666, NULL, &psi_irq_proc_ops);
	1531	+#endif
1400	1532	}
1401	1533	return 0;
1402	1534	}

kernel/sched/stats.h+6−0

@@ -107,6 +107,11 @@ __schedstats_from_se(struct sched_entity *se) · 106 unmodified lines
107	107	}
108	108
109	109	#ifdef CONFIG_PSI
	110	+void psi_task_change(struct task_struct *task, int clear, int set);
	111	+void psi_task_switch(struct task_struct prev, struct task_struct next,
	112	+ bool sleep);
	113	+void psi_account_irqtime(struct task_struct *task, u32 delta);
	114	+
110	115	/*
111	116	* PSI tracks state that persists across sleeps, such as iowaits and
112	117	* memory stalls. As a result, it has to distinguish between sleeps,
@@ -201,6 +206,7 @@ static inline void psi_ttwu_dequeue(struct task_struct *p) {} · 88 unmodified lines
201	206	static inline void psi_sched_switch(struct task_struct *prev,
202	207	struct task_struct *next,
203	208	bool sleep) {}
	209	+static inline void psi_account_irqtime(struct task_struct *task, u32 delta) {}
204	210	#endif /* CONFIG_PSI */
205	211
206	212	#ifdef CONFIG_SCHED_INFO