History

Merge tag 'sched-psi-2022-10-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull PSI updates from Ingo Molnar: - Various performance optimizations, resulting in a 4%-9% speedup in the mmtests/config-scheduler-perfpipe micro-benchmark. - New interface to turn PSI on/off on a per cgroup level. * tag 'sched-psi-2022-10-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched/psi: Per-cgroup PSI accounting disable/re-enable interface sched/psi: Cache parent psi_group to speed up group iteration sched/psi: Consolidate cgroup_psi() sched/psi: Add PSI_IRQ to track IRQ/SOFTIRQ pressure sched/psi: Remove NR_ONCPU task accounting sched/psi: Optimize task switch inside shared cgroups again sched/psi: Move private helpers to sched/stats.h sched/psi: Save percpu memory when !psi_cgroups_enabled sched/psi: Don't create cgroup PSI files when psi_disabled sched/psi: Fix periodic aggregation shut off

Linus Torvalds committed 4 years ago

Commit bd9a3db2 parents 1df046a 34f26a19 files · +362 −103Browse files

Unified · Split

9 files
  1. Documentation/admin-guide/cgroup-v2.rst +23 −0
  2. include/linux/cgroup-defs.h +3 −0
  3. include/linux/cgroup.h +0 −5
  4. include/linux/psi.h +8 −4
  5. include/linux/psi_types.h +20 −11
  6. kernel/cgroup/cgroup.c +95 −9
  7. kernel/sched/core.c +1 −0
  8. kernel/sched/psi.c +206 −74
  9. kernel/sched/stats.h +6 −0
Documentation/admin-guide/cgroup-v2.rst+23−0
@@ -976,6 +976,29 @@ All cgroup core files are prefixed with "cgroup." · 975 unmodified lines
976 killing cgroups is a process directed operation, i.e. it affects976 killing cgroups is a process directed operation, i.e. it affects
977 the whole thread-group.977 the whole thread-group.
978 978
979 cgroup.pressure
980 A read-write single value file that allowed values are "0" and "1".
981 The default is "1".
982
983 Writing "0" to the file will disable the cgroup PSI accounting.
984 Writing "1" to the file will re-enable the cgroup PSI accounting.
985
986 This control attribute is not hierarchical, so disable or enable PSI
987 accounting in a cgroup does not affect PSI accounting in descendants
988 and doesn't need pass enablement via ancestors from root.
989
990 The reason this control attribute exists is that PSI accounts stalls for
991 each cgroup separately and aggregates it at each level of the hierarchy.
992 This may cause non-negligible overhead for some workloads when under
993 deep level of the hierarchy, in which case this control attribute can
994 be used to disable PSI accounting in the non-leaf cgroups.
995
996 irq.pressure
997 A read-write nested-keyed file.
998
999 Shows pressure stall information for IRQ/SOFTIRQ. See
1000 :ref:`Documentation/accounting/psi.rst <psi>` for details.
1001
979Controllers1002Controllers
980===========1003===========
981 1004
include/linux/cgroup-defs.h+3−0
@@ -428,6 +428,9 @@ struct cgroup { · 427 unmodified lines
428 struct cgroup_file procs_file; /* handle for "cgroup.procs" */428 struct cgroup_file procs_file; /* handle for "cgroup.procs" */
429 struct cgroup_file events_file; /* handle for "cgroup.events" */429 struct cgroup_file events_file; /* handle for "cgroup.events" */
430 430
431 /* handles for "{cpu,memory,io,irq}.pressure" */
432 struct cgroup_file psi_files[NR_PSI_RESOURCES];
433
431 /*434 /*
432 * The bitmask of subsystems enabled on the child cgroups.435 * The bitmask of subsystems enabled on the child cgroups.
433 * ->subtree_control is the one configured through436 * ->subtree_control is the one configured through
include/linux/cgroup.h+0−5
@@ -682,11 +682,6 @@ static inline void pr_cont_cgroup_path(struct cgroup *cgrp) · 681 unmodified lines
682 pr_cont_kernfs_path(cgrp->kn);682 pr_cont_kernfs_path(cgrp->kn);
683}683}
684 684
685static inline struct psi_group *cgroup_psi(struct cgroup *cgrp)
686{
687 return cgrp->psi;
688}
689
690bool cgroup_psi_enabled(void);685bool cgroup_psi_enabled(void);
691 686
692static inline void cgroup_init_kthreadd(void)687static inline void cgroup_init_kthreadd(void)
include/linux/psi.h+8−4
@@ -7,6 +7,7 @@ · 6 unmodified lines
7#include <linux/sched.h>7#include <linux/sched.h>
8#include <linux/poll.h>8#include <linux/poll.h>
9#include <linux/cgroup-defs.h>9#include <linux/cgroup-defs.h>
10#include <linux/cgroup.h>
10 11
11struct seq_file;12struct seq_file;
12struct css_set;13struct css_set;
@@ -18,10 +19,6 @@ extern struct psi_group psi_system; · 5 unmodified lines
18 19
19void psi_init(void);20void psi_init(void);
20 21
21void psi_task_change(struct task_struct *task, int clear, int set);
22void psi_task_switch(struct task_struct *prev, struct task_struct *next,
23 bool sleep);
24
25void psi_memstall_enter(unsigned long *flags);22void psi_memstall_enter(unsigned long *flags);
26void psi_memstall_leave(unsigned long *flags);23void psi_memstall_leave(unsigned long *flags);
27 24
@@ -34,9 +31,15 @@ __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file, · 6 unmodified lines
34 poll_table *wait);31 poll_table *wait);
35 32
36#ifdef CONFIG_CGROUPS33#ifdef CONFIG_CGROUPS
34static inline struct psi_group *cgroup_psi(struct cgroup *cgrp)
35{
36 return cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;
37}
38
37int psi_cgroup_alloc(struct cgroup *cgrp);39int psi_cgroup_alloc(struct cgroup *cgrp);
38void psi_cgroup_free(struct cgroup *cgrp);40void psi_cgroup_free(struct cgroup *cgrp);
39void cgroup_move_task(struct task_struct *p, struct css_set *to);41void cgroup_move_task(struct task_struct *p, struct css_set *to);
42void psi_cgroup_restart(struct psi_group *group);
40#endif43#endif
41 44
42#else /* CONFIG_PSI */45#else /* CONFIG_PSI */
@@ -58,6 +61,7 @@ static inline void cgroup_move_task(struct task_struct *p, struct css_set *to) · 15 unmodified lines
58{61{
59 rcu_assign_pointer(p->cgroups, to);62 rcu_assign_pointer(p->cgroups, to);
60}63}
64static inline void psi_cgroup_restart(struct psi_group *group) {}
61#endif65#endif
62 66
63#endif /* CONFIG_PSI */67#endif /* CONFIG_PSI */
include/linux/psi_types.h+20−11
@@ -15,13 +15,6 @@ enum psi_task_count { · 14 unmodified lines
15 NR_IOWAIT,15 NR_IOWAIT,
16 NR_MEMSTALL,16 NR_MEMSTALL,
17 NR_RUNNING,17 NR_RUNNING,
18 /*
19 * This can't have values other than 0 or 1 and could be
20 * implemented as a bit flag. But for now we still have room
21 * in the first cacheline of psi_group_cpu, and this way we
22 * don't have to special case any state tracking for it.
23 */
24 NR_ONCPU,
25 /*18 /*
26 * For IO and CPU stalls the presence of running/oncpu tasks19 * For IO and CPU stalls the presence of running/oncpu tasks
27 * in the domain means a partial rather than a full stall.20 * in the domain means a partial rather than a full stall.
@@ -32,22 +25,27 @@ enum psi_task_count { · 4 unmodified lines
32 * threads and memstall ones.25 * threads and memstall ones.
33 */26 */
34 NR_MEMSTALL_RUNNING,27 NR_MEMSTALL_RUNNING,
35 NR_PSI_TASK_COUNTS = 5,28 NR_PSI_TASK_COUNTS = 4,
36};29};
37 30
38/* Task state bitmasks */31/* Task state bitmasks */
39#define TSK_IOWAIT (1 << NR_IOWAIT)32#define TSK_IOWAIT (1 << NR_IOWAIT)
40#define TSK_MEMSTALL (1 << NR_MEMSTALL)33#define TSK_MEMSTALL (1 << NR_MEMSTALL)
41#define TSK_RUNNING (1 << NR_RUNNING)34#define TSK_RUNNING (1 << NR_RUNNING)
42#define TSK_ONCPU (1 << NR_ONCPU)
43#define TSK_MEMSTALL_RUNNING (1 << NR_MEMSTALL_RUNNING)35#define TSK_MEMSTALL_RUNNING (1 << NR_MEMSTALL_RUNNING)
44 36
37/* Only one task can be scheduled, no corresponding task count */
38#define TSK_ONCPU (1 << NR_PSI_TASK_COUNTS)
39
45/* Resources that workloads could be stalled on */40/* Resources that workloads could be stalled on */
46enum psi_res {41enum psi_res {
47 PSI_IO,42 PSI_IO,
48 PSI_MEM,43 PSI_MEM,
49 PSI_CPU,44 PSI_CPU,
50 NR_PSI_RESOURCES = 3,45#ifdef CONFIG_IRQ_TIME_ACCOUNTING
46 PSI_IRQ,
47#endif
48 NR_PSI_RESOURCES,
51};49};
52 50
53/*51/*
@@ -63,11 +61,17 @@ enum psi_states { · 9 unmodified lines
63 PSI_MEM_FULL,61 PSI_MEM_FULL,
64 PSI_CPU_SOME,62 PSI_CPU_SOME,
65 PSI_CPU_FULL,63 PSI_CPU_FULL,
64#ifdef CONFIG_IRQ_TIME_ACCOUNTING
65 PSI_IRQ_FULL,
66#endif
66 /* Only per-CPU, to weigh the CPU in the global average: */67 /* Only per-CPU, to weigh the CPU in the global average: */
67 PSI_NONIDLE,68 PSI_NONIDLE,
68 NR_PSI_STATES = 7,69 NR_PSI_STATES,
69};70};
70 71
72/* Use one bit in the state mask to track TSK_ONCPU */
73#define PSI_ONCPU (1 << NR_PSI_STATES)
74
71enum psi_aggregators {75enum psi_aggregators {
72 PSI_AVGS = 0,76 PSI_AVGS = 0,
73 PSI_POLL,77 PSI_POLL,
@@ -147,6 +151,9 @@ struct psi_trigger { · 73 unmodified lines
147};151};
148 152
149struct psi_group {153struct psi_group {
154 struct psi_group *parent;
155 bool enabled;
156
150 /* Protects data used by the aggregator */157 /* Protects data used by the aggregator */
151 struct mutex avgs_lock;158 struct mutex avgs_lock;
152 159
@@ -188,6 +195,8 @@ struct psi_group { · 35 unmodified lines
188 195
189#else /* CONFIG_PSI */196#else /* CONFIG_PSI */
190 197
198#define NR_PSI_RESOURCES 0
199
191struct psi_group { };200struct psi_group { };
192 201
193#endif /* CONFIG_PSI */202#endif /* CONFIG_PSI */
kernel/cgroup/cgroup.c+95−9
@@ -3698,27 +3698,27 @@ static int cpu_stat_show(struct seq_file *seq, void *v) · 3697 unmodified lines
3698static int cgroup_io_pressure_show(struct seq_file *seq, void *v)3698static int cgroup_io_pressure_show(struct seq_file *seq, void *v)
3699{3699{
3700 struct cgroup *cgrp = seq_css(seq)->cgroup;3700 struct cgroup *cgrp = seq_css(seq)->cgroup;
3701 struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;3701 struct psi_group *psi = cgroup_psi(cgrp);
3702 3702
3703 return psi_show(seq, psi, PSI_IO);3703 return psi_show(seq, psi, PSI_IO);
3704}3704}
3705static int cgroup_memory_pressure_show(struct seq_file *seq, void *v)3705static int cgroup_memory_pressure_show(struct seq_file *seq, void *v)
3706{3706{
3707 struct cgroup *cgrp = seq_css(seq)->cgroup;3707 struct cgroup *cgrp = seq_css(seq)->cgroup;
3708 struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;3708 struct psi_group *psi = cgroup_psi(cgrp);
3709 3709
3710 return psi_show(seq, psi, PSI_MEM);3710 return psi_show(seq, psi, PSI_MEM);
3711}3711}
3712static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v)3712static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v)
3713{3713{
3714 struct cgroup *cgrp = seq_css(seq)->cgroup;3714 struct cgroup *cgrp = seq_css(seq)->cgroup;
3715 struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;3715 struct psi_group *psi = cgroup_psi(cgrp);
3716 3716
3717 return psi_show(seq, psi, PSI_CPU);3717 return psi_show(seq, psi, PSI_CPU);
3718}3718}
3719 3719
3720static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf,3720static ssize_t pressure_write(struct kernfs_open_file *of, char *buf,
3721 size_t nbytes, enum psi_res res)3721 size_t nbytes, enum psi_res res)
3722{3722{
3723 struct cgroup_file_ctx *ctx = of->priv;3723 struct cgroup_file_ctx *ctx = of->priv;
3724 struct psi_trigger *new;3724 struct psi_trigger *new;
@@ -3738,7 +3738,7 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf, · 13 unmodified lines
3738 return -EBUSY;3738 return -EBUSY;
3739 }3739 }
3740 3740
3741 psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;3741 psi = cgroup_psi(cgrp);
3742 new = psi_trigger_create(psi, buf, res);3742 new = psi_trigger_create(psi, buf, res);
3743 if (IS_ERR(new)) {3743 if (IS_ERR(new)) {
3744 cgroup_put(cgrp);3744 cgroup_put(cgrp);
@@ -3755,21 +3755,86 @@ static ssize_t cgroup_io_pressure_write(struct kernfs_open_file *of, · 10 unmodified lines
3755 char *buf, size_t nbytes,3755 char *buf, size_t nbytes,
3756 loff_t off)3756 loff_t off)
3757{3757{
3758 return cgroup_pressure_write(of, buf, nbytes, PSI_IO);3758 return pressure_write(of, buf, nbytes, PSI_IO);
3759}3759}
3760 3760
3761static ssize_t cgroup_memory_pressure_write(struct kernfs_open_file *of,3761static ssize_t cgroup_memory_pressure_write(struct kernfs_open_file *of,
3762 char *buf, size_t nbytes,3762 char *buf, size_t nbytes,
3763 loff_t off)3763 loff_t off)
3764{3764{
3765 return cgroup_pressure_write(of, buf, nbytes, PSI_MEM);3765 return pressure_write(of, buf, nbytes, PSI_MEM);
3766}3766}
3767 3767
3768static ssize_t cgroup_cpu_pressure_write(struct kernfs_open_file *of,3768static ssize_t cgroup_cpu_pressure_write(struct kernfs_open_file *of,
3769 char *buf, size_t nbytes,3769 char *buf, size_t nbytes,
3770 loff_t off)3770 loff_t off)
3771{3771{
3772 return cgroup_pressure_write(of, buf, nbytes, PSI_CPU);3772 return pressure_write(of, buf, nbytes, PSI_CPU);
3773}
3774
3775#ifdef CONFIG_IRQ_TIME_ACCOUNTING
3776static int cgroup_irq_pressure_show(struct seq_file *seq, void *v)
3777{
3778 struct cgroup *cgrp = seq_css(seq)->cgroup;
3779 struct psi_group *psi = cgroup_psi(cgrp);
3780
3781 return psi_show(seq, psi, PSI_IRQ);
3782}
3783
3784static ssize_t cgroup_irq_pressure_write(struct kernfs_open_file *of,
3785 char *buf, size_t nbytes,
3786 loff_t off)
3787{
3788 return pressure_write(of, buf, nbytes, PSI_IRQ);
3789}
3790#endif
3791
3792static int cgroup_pressure_show(struct seq_file *seq, void *v)
3793{
3794 struct cgroup *cgrp = seq_css(seq)->cgroup;
3795 struct psi_group *psi = cgroup_psi(cgrp);
3796
3797 seq_printf(seq, "%d\n", psi->enabled);
3798
3799 return 0;
3800}
3801
3802static ssize_t cgroup_pressure_write(struct kernfs_open_file *of,
3803 char *buf, size_t nbytes,
3804 loff_t off)
3805{
3806 ssize_t ret;
3807 int enable;
3808 struct cgroup *cgrp;
3809 struct psi_group *psi;
3810
3811 ret = kstrtoint(strstrip(buf), 0, &enable);
3812 if (ret)
3813 return ret;
3814
3815 if (enable < 0 || enable > 1)
3816 return -ERANGE;
3817
3818 cgrp = cgroup_kn_lock_live(of->kn, false);
3819 if (!cgrp)
3820 return -ENOENT;
3821
3822 psi = cgroup_psi(cgrp);
3823 if (psi->enabled != enable) {
3824 int i;
3825
3826 /* show or hide {cpu,memory,io,irq}.pressure files */
3827 for (i = 0; i < NR_PSI_RESOURCES; i++)
3828 cgroup_file_show(&cgrp->psi_files[i], enable);
3829
3830 psi->enabled = enable;
3831 if (enable)
3832 psi_cgroup_restart(psi);
3833 }
3834
3835 cgroup_kn_unlock(of->kn);
3836
3837 return nbytes;
3773}3838}
3774 3839
3775static __poll_t cgroup_pressure_poll(struct kernfs_open_file *of,3840static __poll_t cgroup_pressure_poll(struct kernfs_open_file *of,
@@ -3789,6 +3854,9 @@ static void cgroup_pressure_release(struct kernfs_open_file *of) · 13 unmodified lines
3789 3854
3790bool cgroup_psi_enabled(void)3855bool cgroup_psi_enabled(void)
3791{3856{
3857 if (static_branch_likely(&psi_disabled))
3858 return false;
3859
3792 return (cgroup_feature_disable_mask & (1 << OPT_FEATURE_PRESSURE)) == 0;3860 return (cgroup_feature_disable_mask & (1 << OPT_FEATURE_PRESSURE)) == 0;
3793}3861}
3794 3862
@@ -5175,6 +5243,7 @@ static struct cftype cgroup_psi_files[] = { · 1380 unmodified lines
5175#ifdef CONFIG_PSI5243#ifdef CONFIG_PSI
5176 {5244 {
5177 .name = "io.pressure",5245 .name = "io.pressure",
5246 .file_offset = offsetof(struct cgroup, psi_files[PSI_IO]),
5178 .seq_show = cgroup_io_pressure_show,5247 .seq_show = cgroup_io_pressure_show,
5179 .write = cgroup_io_pressure_write,5248 .write = cgroup_io_pressure_write,
5180 .poll = cgroup_pressure_poll,5249 .poll = cgroup_pressure_poll,
@@ -5182,6 +5251,7 @@ static struct cftype cgroup_psi_files[] = { · 1 unmodified line
5182 },5251 },
5183 {5252 {
5184 .name = "memory.pressure",5253 .name = "memory.pressure",
5254 .file_offset = offsetof(struct cgroup, psi_files[PSI_MEM]),
5185 .seq_show = cgroup_memory_pressure_show,5255 .seq_show = cgroup_memory_pressure_show,
5186 .write = cgroup_memory_pressure_write,5256 .write = cgroup_memory_pressure_write,
5187 .poll = cgroup_pressure_poll,5257 .poll = cgroup_pressure_poll,
@@ -5189,11 +5259,27 @@ static struct cftype cgroup_psi_files[] = { · 1 unmodified line
5189 },5259 },
5190 {5260 {
5191 .name = "cpu.pressure",5261 .name = "cpu.pressure",
5262 .file_offset = offsetof(struct cgroup, psi_files[PSI_CPU]),
5192 .seq_show = cgroup_cpu_pressure_show,5263 .seq_show = cgroup_cpu_pressure_show,
5193 .write = cgroup_cpu_pressure_write,5264 .write = cgroup_cpu_pressure_write,
5194 .poll = cgroup_pressure_poll,5265 .poll = cgroup_pressure_poll,
5195 .release = cgroup_pressure_release,5266 .release = cgroup_pressure_release,
5196 },5267 },
5268#ifdef CONFIG_IRQ_TIME_ACCOUNTING
5269 {
5270 .name = "irq.pressure",
5271 .file_offset = offsetof(struct cgroup, psi_files[PSI_IRQ]),
5272 .seq_show = cgroup_irq_pressure_show,
5273 .write = cgroup_irq_pressure_write,
5274 .poll = cgroup_pressure_poll,
5275 .release = cgroup_pressure_release,
5276 },
5277#endif
5278 {
5279 .name = "cgroup.pressure",
5280 .seq_show = cgroup_pressure_show,
5281 .write = cgroup_pressure_write,
5282 },
5197#endif /* CONFIG_PSI */5283#endif /* CONFIG_PSI */
5198 { } /* terminate */5284 { } /* terminate */
5199};5285};
kernel/sched/core.c+1−0
@@ -701,6 +701,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) · 700 unmodified lines
701 701
702 rq->prev_irq_time += irq_delta;702 rq->prev_irq_time += irq_delta;
703 delta -= irq_delta;703 delta -= irq_delta;
704 psi_account_irqtime(rq->curr, irq_delta);
704#endif705#endif
705#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING706#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
706 if (static_key_false((&paravirt_steal_rq_enabled))) {707 if (static_key_false((&paravirt_steal_rq_enabled))) {
kernel/sched/psi.c+206−74
@@ -181,6 +181,7 @@ static void group_init(struct psi_group *group) · 180 unmodified lines
181{181{
182 int cpu;182 int cpu;
183 183
184 group->enabled = true;
184 for_each_possible_cpu(cpu)185 for_each_possible_cpu(cpu)
185 seqcount_init(&per_cpu_ptr(group->pcpu, cpu)->seq);186 seqcount_init(&per_cpu_ptr(group->pcpu, cpu)->seq);
186 group->avg_last_update = sched_clock();187 group->avg_last_update = sched_clock();
@@ -201,6 +202,7 @@ void __init psi_init(void) · 14 unmodified lines
201{202{
202 if (!psi_enable) {203 if (!psi_enable) {
203 static_branch_enable(&psi_disabled);204 static_branch_enable(&psi_disabled);
205 static_branch_disable(&psi_cgroups_enabled);
204 return;206 return;
205 }207 }
206 208
@@ -211,7 +213,7 @@ void __init psi_init(void) · 4 unmodified lines
211 group_init(&psi_system);213 group_init(&psi_system);
212}214}
213 215
214static bool test_state(unsigned int *tasks, enum psi_states state)216static bool test_state(unsigned int *tasks, enum psi_states state, bool oncpu)
215{217{
216 switch (state) {218 switch (state) {
217 case PSI_IO_SOME:219 case PSI_IO_SOME:
@@ -224,9 +226,9 @@ static bool test_state(unsigned int *tasks, enum psi_states state) · 6 unmodified lines
224 return unlikely(tasks[NR_MEMSTALL] &&226 return unlikely(tasks[NR_MEMSTALL] &&
225 tasks[NR_RUNNING] == tasks[NR_MEMSTALL_RUNNING]);227 tasks[NR_RUNNING] == tasks[NR_MEMSTALL_RUNNING]);
226 case PSI_CPU_SOME:228 case PSI_CPU_SOME:
227 return unlikely(tasks[NR_RUNNING] > tasks[NR_ONCPU]);229 return unlikely(tasks[NR_RUNNING] > oncpu);
228 case PSI_CPU_FULL:230 case PSI_CPU_FULL:
229 return unlikely(tasks[NR_RUNNING] && !tasks[NR_ONCPU]);231 return unlikely(tasks[NR_RUNNING] && !oncpu);
230 case PSI_NONIDLE:232 case PSI_NONIDLE:
231 return tasks[NR_IOWAIT] || tasks[NR_MEMSTALL] ||233 return tasks[NR_IOWAIT] || tasks[NR_MEMSTALL] ||
232 tasks[NR_RUNNING];234 tasks[NR_RUNNING];
@@ -688,35 +690,53 @@ static void psi_group_change(struct psi_group *group, int cpu, · 455 unmodified lines
688 bool wake_clock)690 bool wake_clock)
689{691{
690 struct psi_group_cpu *groupc;692 struct psi_group_cpu *groupc;
691 u32 state_mask = 0;
692 unsigned int t, m;693 unsigned int t, m;
693 enum psi_states s;694 enum psi_states s;
695 u32 state_mask;
694 696
695 groupc = per_cpu_ptr(group->pcpu, cpu);697 groupc = per_cpu_ptr(group->pcpu, cpu);
696 698
697 /*699 /*
698 * First we assess the aggregate resource states this CPU's700 * First we update the task counts according to the state
699 * tasks have been in since the last change, and account any
700 * SOME and FULL time these may have resulted in.
701 *
702 * Then we update the task counts according to the state
703 * change requested through the @clear and @set bits.701 * change requested through the @clear and @set bits.
702 *
703 * Then if the cgroup PSI stats accounting enabled, we
704 * assess the aggregate resource states this CPU's tasks
705 * have been in since the last change, and account any
706 * SOME and FULL time these may have resulted in.
704 */707 */
705 write_seqcount_begin(&groupc->seq);708 write_seqcount_begin(&groupc->seq);
706 709
707 record_times(groupc, now);710 /*
711 * Start with TSK_ONCPU, which doesn't have a corresponding
712 * task count - it's just a boolean flag directly encoded in
713 * the state mask. Clear, set, or carry the current state if
714 * no changes are requested.
715 */
716 if (unlikely(clear & TSK_ONCPU)) {
717 state_mask = 0;
718 clear &= ~TSK_ONCPU;
719 } else if (unlikely(set & TSK_ONCPU)) {
720 state_mask = PSI_ONCPU;
721 set &= ~TSK_ONCPU;
722 } else {
723 state_mask = groupc->state_mask & PSI_ONCPU;
724 }
708 725
726 /*
727 * The rest of the state mask is calculated based on the task
728 * counts. Update those first, then construct the mask.
729 */
709 for (t = 0, m = clear; m; m &= ~(1 << t), t++) {730 for (t = 0, m = clear; m; m &= ~(1 << t), t++) {
710 if (!(m & (1 << t)))731 if (!(m & (1 << t)))
711 continue;732 continue;
712 if (groupc->tasks[t]) {733 if (groupc->tasks[t]) {
713 groupc->tasks[t]--;734 groupc->tasks[t]--;
714 } else if (!psi_bug) {735 } else if (!psi_bug) {
715 printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u %u] clear=%x set=%x\n",736 printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u] clear=%x set=%x\n",
716 cpu, t, groupc->tasks[0],737 cpu, t, groupc->tasks[0],
717 groupc->tasks[1], groupc->tasks[2],738 groupc->tasks[1], groupc->tasks[2],
718 groupc->tasks[3], groupc->tasks[4],739 groupc->tasks[3], clear, set);
719 clear, set);
720 psi_bug = 1;740 psi_bug = 1;
721 }741 }
722 }742 }
@@ -725,9 +745,25 @@ static void psi_group_change(struct psi_group *group, int cpu, · 2 unmodified lines
725 if (set & (1 << t))745 if (set & (1 << t))
726 groupc->tasks[t]++;746 groupc->tasks[t]++;
727 747
728 /* Calculate state mask representing active states */748 if (!group->enabled) {
749 /*
750 * On the first group change after disabling PSI, conclude
751 * the current state and flush its time. This is unlikely
752 * to matter to the user, but aggregation (get_recent_times)
753 * may have already incorporated the live state into times_prev;
754 * avoid a delta sample underflow when PSI is later re-enabled.
755 */
756 if (unlikely(groupc->state_mask & (1 << PSI_NONIDLE)))
757 record_times(groupc, now);
758
759 groupc->state_mask = state_mask;
760
761 write_seqcount_end(&groupc->seq);
762 return;
763 }
764
729 for (s = 0; s < NR_PSI_STATES; s++) {765 for (s = 0; s < NR_PSI_STATES; s++) {
730 if (test_state(groupc->tasks, s))766 if (test_state(groupc->tasks, s, state_mask & PSI_ONCPU))
731 state_mask |= (1 << s);767 state_mask |= (1 << s);
732 }768 }
733 769
@@ -739,9 +775,11 @@ static void psi_group_change(struct psi_group *group, int cpu, · 5 unmodified lines
739 * task in a cgroup is in_memstall, the corresponding groupc775 * task in a cgroup is in_memstall, the corresponding groupc
740 * on that cpu is in PSI_MEM_FULL state.776 * on that cpu is in PSI_MEM_FULL state.
741 */777 */
742 if (unlikely(groupc->tasks[NR_ONCPU] && cpu_curr(cpu)->in_memstall))778 if (unlikely((state_mask & PSI_ONCPU) && cpu_curr(cpu)->in_memstall))
743 state_mask |= (1 << PSI_MEM_FULL);779 state_mask |= (1 << PSI_MEM_FULL);
744 780
781 record_times(groupc, now);
782
745 groupc->state_mask = state_mask;783 groupc->state_mask = state_mask;
746 784
747 write_seqcount_end(&groupc->seq);785 write_seqcount_end(&groupc->seq);
@@ -753,27 +791,12 @@ static void psi_group_change(struct psi_group *group, int cpu, · 5 unmodified lines
753 schedule_delayed_work(&group->avgs_work, PSI_FREQ);791 schedule_delayed_work(&group->avgs_work, PSI_FREQ);
754}792}
755 793
756static struct psi_group *iterate_groups(struct task_struct *task, void **iter)794static inline struct psi_group *task_psi_group(struct task_struct *task)
757{795{
758 if (*iter == &psi_system)
759 return NULL;
760
761#ifdef CONFIG_CGROUPS796#ifdef CONFIG_CGROUPS
762 if (static_branch_likely(&psi_cgroups_enabled)) {797 if (static_branch_likely(&psi_cgroups_enabled))
763 struct cgroup *cgroup = NULL;798 return cgroup_psi(task_dfl_cgroup(task));
764
765 if (!*iter)
766 cgroup = task->cgroups->dfl_cgrp;
767 else
768 cgroup = cgroup_parent(*iter);
769
770 if (cgroup && cgroup_parent(cgroup)) {
771 *iter = cgroup;
772 return cgroup_psi(cgroup);
773 }
774 }
775#endif799#endif
776 *iter = &psi_system;
777 return &psi_system;800 return &psi_system;
778}801}
779 802
@@ -796,8 +819,6 @@ void psi_task_change(struct task_struct *task, int clear, int set) · 16 unmodified lines
796{819{
797 int cpu = task_cpu(task);820 int cpu = task_cpu(task);
798 struct psi_group *group;821 struct psi_group *group;
799 bool wake_clock = true;
800 void *iter = NULL;
801 u64 now;822 u64 now;
802 823
803 if (!task->pid)824 if (!task->pid)
@@ -806,19 +827,11 @@ void psi_task_change(struct task_struct *task, int clear, int set) · 2 unmodified lines
806 psi_flags_change(task, clear, set);827 psi_flags_change(task, clear, set);
807 828
808 now = cpu_clock(cpu);829 now = cpu_clock(cpu);
809 /*
810 * Periodic aggregation shuts off if there is a period of no
811 * task changes, so we wake it back up if necessary. However,
812 * don't do this if the task change is the aggregation worker
813 * itself going to sleep, or we'll ping-pong forever.
814 */
815 if (unlikely((clear & TSK_RUNNING) &&
816 (task->flags & PF_WQ_WORKER) &&
817 wq_worker_last_func(task) == psi_avgs_work))
818 wake_clock = false;
819 830
820 while ((group = iterate_groups(task, &iter)))831 group = task_psi_group(task);
821 psi_group_change(group, cpu, clear, set, now, wake_clock);832 do {
833 psi_group_change(group, cpu, clear, set, now, true);
834 } while ((group = group->parent));
822}835}
823 836
824void psi_task_switch(struct task_struct *prev, struct task_struct *next,837void psi_task_switch(struct task_struct *prev, struct task_struct *next,
@@ -826,34 +839,30 @@ void psi_task_switch(struct task_struct *prev, struct task_struct *next, · 1 unmodified line
826{839{
827 struct psi_group *group, *common = NULL;840 struct psi_group *group, *common = NULL;
828 int cpu = task_cpu(prev);841 int cpu = task_cpu(prev);
829 void *iter;
830 u64 now = cpu_clock(cpu);842 u64 now = cpu_clock(cpu);
831 843
832 if (next->pid) {844 if (next->pid) {
833 bool identical_state;
834
835 psi_flags_change(next, 0, TSK_ONCPU);845 psi_flags_change(next, 0, TSK_ONCPU);
836 /*846 /*
837 * When switching between tasks that have an identical847 * Set TSK_ONCPU on @next's cgroups. If @next shares any
838 * runtime state, the cgroup that contains both tasks848 * ancestors with @prev, those will already have @prev's
839 * we reach the first common ancestor. Iterate @next's849 * TSK_ONCPU bit set, and we can stop the iteration there.
840 * ancestors only until we encounter @prev's ONCPU.
841 */850 */
842 identical_state = prev->psi_flags == next->psi_flags;851 group = task_psi_group(next);
843 iter = NULL;852 do {
844 while ((group = iterate_groups(next, &iter))) {853 if (per_cpu_ptr(group->pcpu, cpu)->state_mask &
845 if (identical_state &&854 PSI_ONCPU) {
846 per_cpu_ptr(group->pcpu, cpu)->tasks[NR_ONCPU]) {
847 common = group;855 common = group;
848 break;856 break;
849 }857 }
850 858
851 psi_group_change(group, cpu, 0, TSK_ONCPU, now, true);859 psi_group_change(group, cpu, 0, TSK_ONCPU, now, true);
852 }860 } while ((group = group->parent));
853 }861 }
854 862
855 if (prev->pid) {863 if (prev->pid) {
856 int clear = TSK_ONCPU, set = 0;864 int clear = TSK_ONCPU, set = 0;
865 bool wake_clock = true;
857 866
858 /*867 /*
859 * When we're going to sleep, psi_dequeue() lets us868 * When we're going to sleep, psi_dequeue() lets us
@@ -867,26 +876,74 @@ void psi_task_switch(struct task_struct *prev, struct task_struct *next, · 7 unmodified lines
867 clear |= TSK_MEMSTALL_RUNNING;876 clear |= TSK_MEMSTALL_RUNNING;
868 if (prev->in_iowait)877 if (prev->in_iowait)
869 set |= TSK_IOWAIT;878 set |= TSK_IOWAIT;
879
880 /*
881 * Periodic aggregation shuts off if there is a period of no
882 * task changes, so we wake it back up if necessary. However,
883 * don't do this if the task change is the aggregation worker
884 * itself going to sleep, or we'll ping-pong forever.
885 */
886 if (unlikely((prev->flags & PF_WQ_WORKER) &&
887 wq_worker_last_func(prev) == psi_avgs_work))
888 wake_clock = false;
870 }889 }
871 890
872 psi_flags_change(prev, clear, set);891 psi_flags_change(prev, clear, set);
873 892
874 iter = NULL;893 group = task_psi_group(prev);
875 while ((group = iterate_groups(prev, &iter)) && group != common)894 do {
876 psi_group_change(group, cpu, clear, set, now, true);895 if (group == common)
896 break;
897 psi_group_change(group, cpu, clear, set, now, wake_clock);
898 } while ((group = group->parent));
877 899
878 /*900 /*
879 * TSK_ONCPU is handled up to the common ancestor. If we're tasked901 * TSK_ONCPU is handled up to the common ancestor. If there are
880 * with dequeuing too, finish that for the rest of the hierarchy.902 * any other differences between the two tasks (e.g. prev goes
903 * to sleep, or only one task is memstall), finish propagating
904 * those differences all the way up to the root.
881 */905 */
882 if (sleep) {906 if ((prev->psi_flags ^ next->psi_flags) & ~TSK_ONCPU) {
883 clear &= ~TSK_ONCPU;907 clear &= ~TSK_ONCPU;
884 for (; group; group = iterate_groups(prev, &iter))908 for (; group; group = group->parent)
885 psi_group_change(group, cpu, clear, set, now, true);909 psi_group_change(group, cpu, clear, set, now, wake_clock);
886 }910 }
887 }911 }
888}912}
889 913
914#ifdef CONFIG_IRQ_TIME_ACCOUNTING
915void psi_account_irqtime(struct task_struct *task, u32 delta)
916{
917 int cpu = task_cpu(task);
918 struct psi_group *group;
919 struct psi_group_cpu *groupc;
920 u64 now;
921
922 if (!task->pid)
923 return;
924
925 now = cpu_clock(cpu);
926
927 group = task_psi_group(task);
928 do {
929 if (!group->enabled)
930 continue;
931
932 groupc = per_cpu_ptr(group->pcpu, cpu);
933
934 write_seqcount_begin(&groupc->seq);
935
936 record_times(groupc, now);
937 groupc->times[PSI_IRQ_FULL] += delta;
938
939 write_seqcount_end(&groupc->seq);
940
941 if (group->poll_states & (1 << PSI_IRQ_FULL))
942 psi_schedule_poll_work(group, 1);
943 } while ((group = group->parent));
944}
945#endif
946
890/**947/**
891 * psi_memstall_enter - mark the beginning of a memory stall section948 * psi_memstall_enter - mark the beginning of a memory stall section
892 * @flags: flags to handle nested sections949 * @flags: flags to handle nested sections
@@ -952,7 +1009,7 @@ EXPORT_SYMBOL_GPL(psi_memstall_leave); · 59 unmodified lines
952#ifdef CONFIG_CGROUPS1009#ifdef CONFIG_CGROUPS
953int psi_cgroup_alloc(struct cgroup *cgroup)1010int psi_cgroup_alloc(struct cgroup *cgroup)
954{1011{
955 if (static_branch_likely(&psi_disabled))1012 if (!static_branch_likely(&psi_cgroups_enabled))
956 return 0;1013 return 0;
957 1014
958 cgroup->psi = kzalloc(sizeof(struct psi_group), GFP_KERNEL);1015 cgroup->psi = kzalloc(sizeof(struct psi_group), GFP_KERNEL);
@@ -965,12 +1022,13 @@ int psi_cgroup_alloc(struct cgroup *cgroup) · 6 unmodified lines
965 return -ENOMEM;1022 return -ENOMEM;
966 }1023 }
967 group_init(cgroup->psi);1024 group_init(cgroup->psi);
1025 cgroup->psi->parent = cgroup_psi(cgroup_parent(cgroup));
968 return 0;1026 return 0;
969}1027}
970 1028
971void psi_cgroup_free(struct cgroup *cgroup)1029void psi_cgroup_free(struct cgroup *cgroup)
972{1030{
973 if (static_branch_likely(&psi_disabled))1031 if (!static_branch_likely(&psi_cgroups_enabled))
974 return;1032 return;
975 1033
976 cancel_delayed_work_sync(&cgroup->psi->avgs_work);1034 cancel_delayed_work_sync(&cgroup->psi->avgs_work);
@@ -998,7 +1056,7 @@ void cgroup_move_task(struct task_struct *task, struct css_set *to) · 21 unmodified lines
998 struct rq_flags rf;1056 struct rq_flags rf;
999 struct rq *rq;1057 struct rq *rq;
1000 1058
1001 if (static_branch_likely(&psi_disabled)) {1059 if (!static_branch_likely(&psi_cgroups_enabled)) {
1002 /*1060 /*
1003 * Lame to do this here, but the scheduler cannot be locked1061 * Lame to do this here, but the scheduler cannot be locked
1004 * from the outside, so we move cgroups from inside sched/.1062 * from the outside, so we move cgroups from inside sched/.
@@ -1046,10 +1104,45 @@ void cgroup_move_task(struct task_struct *task, struct css_set *to) · 41 unmodified lines
1046 1104
1047 task_rq_unlock(rq, task, &rf);1105 task_rq_unlock(rq, task, &rf);
1048}1106}
1107
1108void psi_cgroup_restart(struct psi_group *group)
1109{
1110 int cpu;
1111
1112 /*
1113 * After we disable psi_group->enabled, we don't actually
1114 * stop percpu tasks accounting in each psi_group_cpu,
1115 * instead only stop test_state() loop, record_times()
1116 * and averaging worker, see psi_group_change() for details.
1117 *
1118 * When disable cgroup PSI, this function has nothing to sync
1119 * since cgroup pressure files are hidden and percpu psi_group_cpu
1120 * would see !psi_group->enabled and only do task accounting.
1121 *
1122 * When re-enable cgroup PSI, this function use psi_group_change()
1123 * to get correct state mask from test_state() loop on tasks[],
1124 * and restart groupc->state_start from now, use .clear = .set = 0
1125 * here since no task status really changed.
1126 */
1127 if (!group->enabled)
1128 return;
1129
1130 for_each_possible_cpu(cpu) {
1131 struct rq *rq = cpu_rq(cpu);
1132 struct rq_flags rf;
1133 u64 now;
1134
1135 rq_lock_irq(rq, &rf);
1136 now = cpu_clock(cpu);
1137 psi_group_change(group, cpu, 0, 0, now, true);
1138 rq_unlock_irq(rq, &rf);
1139 }
1140}
1049#endif /* CONFIG_CGROUPS */1141#endif /* CONFIG_CGROUPS */
1050 1142
1051int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res)1143int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res)
1052{1144{
1145 bool only_full = false;
1053 int full;1146 int full;
1054 u64 now;1147 u64 now;
1055 1148
@@ -1064,7 +1157,11 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) · 8 unmodified lines
1064 group->avg_next_update = update_averages(group, now);1157 group->avg_next_update = update_averages(group, now);
1065 mutex_unlock(&group->avgs_lock);1158 mutex_unlock(&group->avgs_lock);
1066 1159
1067 for (full = 0; full < 2; full++) {1160#ifdef CONFIG_IRQ_TIME_ACCOUNTING
1161 only_full = res == PSI_IRQ;
1162#endif
1163
1164 for (full = 0; full < 2 - only_full; full++) {
1068 unsigned long avg[3] = { 0, };1165 unsigned long avg[3] = { 0, };
1069 u64 total = 0;1166 u64 total = 0;
1070 int w;1167 int w;
@@ -1078,7 +1175,7 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) · 7 unmodified lines
1078 }1175 }
1079 1176
1080 seq_printf(m, "%s avg10=%lu.%02lu avg60=%lu.%02lu avg300=%lu.%02lu total=%llu\n",1177 seq_printf(m, "%s avg10=%lu.%02lu avg60=%lu.%02lu avg300=%lu.%02lu total=%llu\n",
1081 full ? "full" : "some",1178 full || only_full ? "full" : "some",
1082 LOAD_INT(avg[0]), LOAD_FRAC(avg[0]),1179 LOAD_INT(avg[0]), LOAD_FRAC(avg[0]),
1083 LOAD_INT(avg[1]), LOAD_FRAC(avg[1]),1180 LOAD_INT(avg[1]), LOAD_FRAC(avg[1]),
1084 LOAD_INT(avg[2]), LOAD_FRAC(avg[2]),1181 LOAD_INT(avg[2]), LOAD_FRAC(avg[2]),
@@ -1106,6 +1203,11 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, · 21 unmodified lines
1106 else1203 else
1107 return ERR_PTR(-EINVAL);1204 return ERR_PTR(-EINVAL);
1108 1205
1206#ifdef CONFIG_IRQ_TIME_ACCOUNTING
1207 if (res == PSI_IRQ && --state != PSI_IRQ_FULL)
1208 return ERR_PTR(-EINVAL);
1209#endif
1210
1109 if (state >= PSI_NONIDLE)1211 if (state >= PSI_NONIDLE)
1110 return ERR_PTR(-EINVAL);1212 return ERR_PTR(-EINVAL);
1111 1213
@@ -1390,6 +1492,33 @@ static const struct proc_ops psi_cpu_proc_ops = { · 278 unmodified lines
1390 .proc_release = psi_fop_release,1492 .proc_release = psi_fop_release,
1391};1493};
1392 1494
1495#ifdef CONFIG_IRQ_TIME_ACCOUNTING
1496static int psi_irq_show(struct seq_file *m, void *v)
1497{
1498 return psi_show(m, &psi_system, PSI_IRQ);
1499}
1500
1501static int psi_irq_open(struct inode *inode, struct file *file)
1502{
1503 return psi_open(file, psi_irq_show);
1504}
1505
1506static ssize_t psi_irq_write(struct file *file, const char __user *user_buf,
1507 size_t nbytes, loff_t *ppos)
1508{
1509 return psi_write(file, user_buf, nbytes, PSI_IRQ);
1510}
1511
1512static const struct proc_ops psi_irq_proc_ops = {
1513 .proc_open = psi_irq_open,
1514 .proc_read = seq_read,
1515 .proc_lseek = seq_lseek,
1516 .proc_write = psi_irq_write,
1517 .proc_poll = psi_fop_poll,
1518 .proc_release = psi_fop_release,
1519};
1520#endif
1521
1393static int __init psi_proc_init(void)1522static int __init psi_proc_init(void)
1394{1523{
1395 if (psi_enable) {1524 if (psi_enable) {
@@ -1397,6 +1526,9 @@ static int __init psi_proc_init(void) · 1 unmodified line
1397 proc_create("pressure/io", 0666, NULL, &psi_io_proc_ops);1526 proc_create("pressure/io", 0666, NULL, &psi_io_proc_ops);
1398 proc_create("pressure/memory", 0666, NULL, &psi_memory_proc_ops);1527 proc_create("pressure/memory", 0666, NULL, &psi_memory_proc_ops);
1399 proc_create("pressure/cpu", 0666, NULL, &psi_cpu_proc_ops);1528 proc_create("pressure/cpu", 0666, NULL, &psi_cpu_proc_ops);
1529#ifdef CONFIG_IRQ_TIME_ACCOUNTING
1530 proc_create("pressure/irq", 0666, NULL, &psi_irq_proc_ops);
1531#endif
1400 }1532 }
1401 return 0;1533 return 0;
1402}1534}
kernel/sched/stats.h+6−0
@@ -107,6 +107,11 @@ __schedstats_from_se(struct sched_entity *se) · 106 unmodified lines
107}107}
108 108
109#ifdef CONFIG_PSI109#ifdef CONFIG_PSI
110void psi_task_change(struct task_struct *task, int clear, int set);
111void psi_task_switch(struct task_struct *prev, struct task_struct *next,
112 bool sleep);
113void psi_account_irqtime(struct task_struct *task, u32 delta);
114
110/*115/*
111 * PSI tracks state that persists across sleeps, such as iowaits and116 * PSI tracks state that persists across sleeps, such as iowaits and
112 * memory stalls. As a result, it has to distinguish between sleeps,117 * memory stalls. As a result, it has to distinguish between sleeps,
@@ -201,6 +206,7 @@ static inline void psi_ttwu_dequeue(struct task_struct *p) {} · 88 unmodified lines
201static inline void psi_sched_switch(struct task_struct *prev,206static inline void psi_sched_switch(struct task_struct *prev,
202 struct task_struct *next,207 struct task_struct *next,
203 bool sleep) {}208 bool sleep) {}
209static inline void psi_account_irqtime(struct task_struct *task, u32 delta) {}
204#endif /* CONFIG_PSI */210#endif /* CONFIG_PSI */
205 211
206#ifdef CONFIG_SCHED_INFO212#ifdef CONFIG_SCHED_INFO