History

Merge tag 'sched-psi-2022-10-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull PSI updates from Ingo Molnar: - Various performance optimizations, resulting in a 4%-9% speedup in the mmtests/config-scheduler-perfpipe micro-benchmark. - New interface to turn PSI on/off on a per cgroup level. * tag 'sched-psi-2022-10-14' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched/psi: Per-cgroup PSI accounting disable/re-enable interface sched/psi: Cache parent psi_group to speed up group iteration sched/psi: Consolidate cgroup_psi() sched/psi: Add PSI_IRQ to track IRQ/SOFTIRQ pressure sched/psi: Remove NR_ONCPU task accounting sched/psi: Optimize task switch inside shared cgroups again sched/psi: Move private helpers to sched/stats.h sched/psi: Save percpu memory when !psi_cgroups_enabled sched/psi: Don't create cgroup PSI files when psi_disabled sched/psi: Fix periodic aggregation shut off

Linus Torvalds committed 4 years ago

Commit bd9a3db2 parents 1df046a 34f26a19 files · +362 −103Browse files

Showing the changes to kernel (4 of 9 files). Show all

Unified · Split

4 files
  1. kernel/cgroup/cgroup.c +95 −9
  2. kernel/sched/core.c +1 −0
  3. kernel/sched/psi.c +206 −74
  4. kernel/sched/stats.h +6 −0
kernel/cgroup/cgroup.c+95−9
@@ -3698,27 +3698,27 @@ static int cpu_stat_show(struct seq_file *seq, void *v) · 3697 unmodified lines
3698static int cgroup_io_pressure_show(struct seq_file *seq, void *v)3698static int cgroup_io_pressure_show(struct seq_file *seq, void *v)
3699{3699{
3700 struct cgroup *cgrp = seq_css(seq)->cgroup;3700 struct cgroup *cgrp = seq_css(seq)->cgroup;
3701 struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;3701 struct psi_group *psi = cgroup_psi(cgrp);
3702 3702
3703 return psi_show(seq, psi, PSI_IO);3703 return psi_show(seq, psi, PSI_IO);
3704}3704}
3705static int cgroup_memory_pressure_show(struct seq_file *seq, void *v)3705static int cgroup_memory_pressure_show(struct seq_file *seq, void *v)
3706{3706{
3707 struct cgroup *cgrp = seq_css(seq)->cgroup;3707 struct cgroup *cgrp = seq_css(seq)->cgroup;
3708 struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;3708 struct psi_group *psi = cgroup_psi(cgrp);
3709 3709
3710 return psi_show(seq, psi, PSI_MEM);3710 return psi_show(seq, psi, PSI_MEM);
3711}3711}
3712static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v)3712static int cgroup_cpu_pressure_show(struct seq_file *seq, void *v)
3713{3713{
3714 struct cgroup *cgrp = seq_css(seq)->cgroup;3714 struct cgroup *cgrp = seq_css(seq)->cgroup;
3715 struct psi_group *psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;3715 struct psi_group *psi = cgroup_psi(cgrp);
3716 3716
3717 return psi_show(seq, psi, PSI_CPU);3717 return psi_show(seq, psi, PSI_CPU);
3718}3718}
3719 3719
3720static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf,3720static ssize_t pressure_write(struct kernfs_open_file *of, char *buf,
3721 size_t nbytes, enum psi_res res)3721 size_t nbytes, enum psi_res res)
3722{3722{
3723 struct cgroup_file_ctx *ctx = of->priv;3723 struct cgroup_file_ctx *ctx = of->priv;
3724 struct psi_trigger *new;3724 struct psi_trigger *new;
@@ -3738,7 +3738,7 @@ static ssize_t cgroup_pressure_write(struct kernfs_open_file *of, char *buf, · 13 unmodified lines
3738 return -EBUSY;3738 return -EBUSY;
3739 }3739 }
3740 3740
3741 psi = cgroup_ino(cgrp) == 1 ? &psi_system : cgrp->psi;3741 psi = cgroup_psi(cgrp);
3742 new = psi_trigger_create(psi, buf, res);3742 new = psi_trigger_create(psi, buf, res);
3743 if (IS_ERR(new)) {3743 if (IS_ERR(new)) {
3744 cgroup_put(cgrp);3744 cgroup_put(cgrp);
@@ -3755,21 +3755,86 @@ static ssize_t cgroup_io_pressure_write(struct kernfs_open_file *of, · 10 unmodified lines
3755 char *buf, size_t nbytes,3755 char *buf, size_t nbytes,
3756 loff_t off)3756 loff_t off)
3757{3757{
3758 return cgroup_pressure_write(of, buf, nbytes, PSI_IO);3758 return pressure_write(of, buf, nbytes, PSI_IO);
3759}3759}
3760 3760
3761static ssize_t cgroup_memory_pressure_write(struct kernfs_open_file *of,3761static ssize_t cgroup_memory_pressure_write(struct kernfs_open_file *of,
3762 char *buf, size_t nbytes,3762 char *buf, size_t nbytes,
3763 loff_t off)3763 loff_t off)
3764{3764{
3765 return cgroup_pressure_write(of, buf, nbytes, PSI_MEM);3765 return pressure_write(of, buf, nbytes, PSI_MEM);
3766}3766}
3767 3767
3768static ssize_t cgroup_cpu_pressure_write(struct kernfs_open_file *of,3768static ssize_t cgroup_cpu_pressure_write(struct kernfs_open_file *of,
3769 char *buf, size_t nbytes,3769 char *buf, size_t nbytes,
3770 loff_t off)3770 loff_t off)
3771{3771{
3772 return cgroup_pressure_write(of, buf, nbytes, PSI_CPU);3772 return pressure_write(of, buf, nbytes, PSI_CPU);
3773}
3774
3775#ifdef CONFIG_IRQ_TIME_ACCOUNTING
3776static int cgroup_irq_pressure_show(struct seq_file *seq, void *v)
3777{
3778 struct cgroup *cgrp = seq_css(seq)->cgroup;
3779 struct psi_group *psi = cgroup_psi(cgrp);
3780
3781 return psi_show(seq, psi, PSI_IRQ);
3782}
3783
3784static ssize_t cgroup_irq_pressure_write(struct kernfs_open_file *of,
3785 char *buf, size_t nbytes,
3786 loff_t off)
3787{
3788 return pressure_write(of, buf, nbytes, PSI_IRQ);
3789}
3790#endif
3791
3792static int cgroup_pressure_show(struct seq_file *seq, void *v)
3793{
3794 struct cgroup *cgrp = seq_css(seq)->cgroup;
3795 struct psi_group *psi = cgroup_psi(cgrp);
3796
3797 seq_printf(seq, "%d\n", psi->enabled);
3798
3799 return 0;
3800}
3801
3802static ssize_t cgroup_pressure_write(struct kernfs_open_file *of,
3803 char *buf, size_t nbytes,
3804 loff_t off)
3805{
3806 ssize_t ret;
3807 int enable;
3808 struct cgroup *cgrp;
3809 struct psi_group *psi;
3810
3811 ret = kstrtoint(strstrip(buf), 0, &enable);
3812 if (ret)
3813 return ret;
3814
3815 if (enable < 0 || enable > 1)
3816 return -ERANGE;
3817
3818 cgrp = cgroup_kn_lock_live(of->kn, false);
3819 if (!cgrp)
3820 return -ENOENT;
3821
3822 psi = cgroup_psi(cgrp);
3823 if (psi->enabled != enable) {
3824 int i;
3825
3826 /* show or hide {cpu,memory,io,irq}.pressure files */
3827 for (i = 0; i < NR_PSI_RESOURCES; i++)
3828 cgroup_file_show(&cgrp->psi_files[i], enable);
3829
3830 psi->enabled = enable;
3831 if (enable)
3832 psi_cgroup_restart(psi);
3833 }
3834
3835 cgroup_kn_unlock(of->kn);
3836
3837 return nbytes;
3773}3838}
3774 3839
3775static __poll_t cgroup_pressure_poll(struct kernfs_open_file *of,3840static __poll_t cgroup_pressure_poll(struct kernfs_open_file *of,
@@ -3789,6 +3854,9 @@ static void cgroup_pressure_release(struct kernfs_open_file *of) · 13 unmodified lines
3789 3854
3790bool cgroup_psi_enabled(void)3855bool cgroup_psi_enabled(void)
3791{3856{
3857 if (static_branch_likely(&psi_disabled))
3858 return false;
3859
3792 return (cgroup_feature_disable_mask & (1 << OPT_FEATURE_PRESSURE)) == 0;3860 return (cgroup_feature_disable_mask & (1 << OPT_FEATURE_PRESSURE)) == 0;
3793}3861}
3794 3862
@@ -5175,6 +5243,7 @@ static struct cftype cgroup_psi_files[] = { · 1380 unmodified lines
5175#ifdef CONFIG_PSI5243#ifdef CONFIG_PSI
5176 {5244 {
5177 .name = "io.pressure",5245 .name = "io.pressure",
5246 .file_offset = offsetof(struct cgroup, psi_files[PSI_IO]),
5178 .seq_show = cgroup_io_pressure_show,5247 .seq_show = cgroup_io_pressure_show,
5179 .write = cgroup_io_pressure_write,5248 .write = cgroup_io_pressure_write,
5180 .poll = cgroup_pressure_poll,5249 .poll = cgroup_pressure_poll,
@@ -5182,6 +5251,7 @@ static struct cftype cgroup_psi_files[] = { · 1 unmodified line
5182 },5251 },
5183 {5252 {
5184 .name = "memory.pressure",5253 .name = "memory.pressure",
5254 .file_offset = offsetof(struct cgroup, psi_files[PSI_MEM]),
5185 .seq_show = cgroup_memory_pressure_show,5255 .seq_show = cgroup_memory_pressure_show,
5186 .write = cgroup_memory_pressure_write,5256 .write = cgroup_memory_pressure_write,
5187 .poll = cgroup_pressure_poll,5257 .poll = cgroup_pressure_poll,
@@ -5189,11 +5259,27 @@ static struct cftype cgroup_psi_files[] = { · 1 unmodified line
5189 },5259 },
5190 {5260 {
5191 .name = "cpu.pressure",5261 .name = "cpu.pressure",
5262 .file_offset = offsetof(struct cgroup, psi_files[PSI_CPU]),
5192 .seq_show = cgroup_cpu_pressure_show,5263 .seq_show = cgroup_cpu_pressure_show,
5193 .write = cgroup_cpu_pressure_write,5264 .write = cgroup_cpu_pressure_write,
5194 .poll = cgroup_pressure_poll,5265 .poll = cgroup_pressure_poll,
5195 .release = cgroup_pressure_release,5266 .release = cgroup_pressure_release,
5196 },5267 },
5268#ifdef CONFIG_IRQ_TIME_ACCOUNTING
5269 {
5270 .name = "irq.pressure",
5271 .file_offset = offsetof(struct cgroup, psi_files[PSI_IRQ]),
5272 .seq_show = cgroup_irq_pressure_show,
5273 .write = cgroup_irq_pressure_write,
5274 .poll = cgroup_pressure_poll,
5275 .release = cgroup_pressure_release,
5276 },
5277#endif
5278 {
5279 .name = "cgroup.pressure",
5280 .seq_show = cgroup_pressure_show,
5281 .write = cgroup_pressure_write,
5282 },
5197#endif /* CONFIG_PSI */5283#endif /* CONFIG_PSI */
5198 { } /* terminate */5284 { } /* terminate */
5199};5285};
kernel/sched/core.c+1−0
@@ -701,6 +701,7 @@ static void update_rq_clock_task(struct rq *rq, s64 delta) · 700 unmodified lines
701 701
702 rq->prev_irq_time += irq_delta;702 rq->prev_irq_time += irq_delta;
703 delta -= irq_delta;703 delta -= irq_delta;
704 psi_account_irqtime(rq->curr, irq_delta);
704#endif705#endif
705#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING706#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
706 if (static_key_false((&paravirt_steal_rq_enabled))) {707 if (static_key_false((&paravirt_steal_rq_enabled))) {
kernel/sched/psi.c+206−74
@@ -181,6 +181,7 @@ static void group_init(struct psi_group *group) · 180 unmodified lines
181{181{
182 int cpu;182 int cpu;
183 183
184 group->enabled = true;
184 for_each_possible_cpu(cpu)185 for_each_possible_cpu(cpu)
185 seqcount_init(&per_cpu_ptr(group->pcpu, cpu)->seq);186 seqcount_init(&per_cpu_ptr(group->pcpu, cpu)->seq);
186 group->avg_last_update = sched_clock();187 group->avg_last_update = sched_clock();
@@ -201,6 +202,7 @@ void __init psi_init(void) · 14 unmodified lines
201{202{
202 if (!psi_enable) {203 if (!psi_enable) {
203 static_branch_enable(&psi_disabled);204 static_branch_enable(&psi_disabled);
205 static_branch_disable(&psi_cgroups_enabled);
204 return;206 return;
205 }207 }
206 208
@@ -211,7 +213,7 @@ void __init psi_init(void) · 4 unmodified lines
211 group_init(&psi_system);213 group_init(&psi_system);
212}214}
213 215
214static bool test_state(unsigned int *tasks, enum psi_states state)216static bool test_state(unsigned int *tasks, enum psi_states state, bool oncpu)
215{217{
216 switch (state) {218 switch (state) {
217 case PSI_IO_SOME:219 case PSI_IO_SOME:
@@ -224,9 +226,9 @@ static bool test_state(unsigned int *tasks, enum psi_states state) · 6 unmodified lines
224 return unlikely(tasks[NR_MEMSTALL] &&226 return unlikely(tasks[NR_MEMSTALL] &&
225 tasks[NR_RUNNING] == tasks[NR_MEMSTALL_RUNNING]);227 tasks[NR_RUNNING] == tasks[NR_MEMSTALL_RUNNING]);
226 case PSI_CPU_SOME:228 case PSI_CPU_SOME:
227 return unlikely(tasks[NR_RUNNING] > tasks[NR_ONCPU]);229 return unlikely(tasks[NR_RUNNING] > oncpu);
228 case PSI_CPU_FULL:230 case PSI_CPU_FULL:
229 return unlikely(tasks[NR_RUNNING] && !tasks[NR_ONCPU]);231 return unlikely(tasks[NR_RUNNING] && !oncpu);
230 case PSI_NONIDLE:232 case PSI_NONIDLE:
231 return tasks[NR_IOWAIT] || tasks[NR_MEMSTALL] ||233 return tasks[NR_IOWAIT] || tasks[NR_MEMSTALL] ||
232 tasks[NR_RUNNING];234 tasks[NR_RUNNING];
@@ -688,35 +690,53 @@ static void psi_group_change(struct psi_group *group, int cpu, · 455 unmodified lines
688 bool wake_clock)690 bool wake_clock)
689{691{
690 struct psi_group_cpu *groupc;692 struct psi_group_cpu *groupc;
691 u32 state_mask = 0;
692 unsigned int t, m;693 unsigned int t, m;
693 enum psi_states s;694 enum psi_states s;
695 u32 state_mask;
694 696
695 groupc = per_cpu_ptr(group->pcpu, cpu);697 groupc = per_cpu_ptr(group->pcpu, cpu);
696 698
697 /*699 /*
698 * First we assess the aggregate resource states this CPU's700 * First we update the task counts according to the state
699 * tasks have been in since the last change, and account any
700 * SOME and FULL time these may have resulted in.
701 *
702 * Then we update the task counts according to the state
703 * change requested through the @clear and @set bits.701 * change requested through the @clear and @set bits.
702 *
703 * Then if the cgroup PSI stats accounting enabled, we
704 * assess the aggregate resource states this CPU's tasks
705 * have been in since the last change, and account any
706 * SOME and FULL time these may have resulted in.
704 */707 */
705 write_seqcount_begin(&groupc->seq);708 write_seqcount_begin(&groupc->seq);
706 709
707 record_times(groupc, now);710 /*
711 * Start with TSK_ONCPU, which doesn't have a corresponding
712 * task count - it's just a boolean flag directly encoded in
713 * the state mask. Clear, set, or carry the current state if
714 * no changes are requested.
715 */
716 if (unlikely(clear & TSK_ONCPU)) {
717 state_mask = 0;
718 clear &= ~TSK_ONCPU;
719 } else if (unlikely(set & TSK_ONCPU)) {
720 state_mask = PSI_ONCPU;
721 set &= ~TSK_ONCPU;
722 } else {
723 state_mask = groupc->state_mask & PSI_ONCPU;
724 }
708 725
726 /*
727 * The rest of the state mask is calculated based on the task
728 * counts. Update those first, then construct the mask.
729 */
709 for (t = 0, m = clear; m; m &= ~(1 << t), t++) {730 for (t = 0, m = clear; m; m &= ~(1 << t), t++) {
710 if (!(m & (1 << t)))731 if (!(m & (1 << t)))
711 continue;732 continue;
712 if (groupc->tasks[t]) {733 if (groupc->tasks[t]) {
713 groupc->tasks[t]--;734 groupc->tasks[t]--;
714 } else if (!psi_bug) {735 } else if (!psi_bug) {
715 printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u %u] clear=%x set=%x\n",736 printk_deferred(KERN_ERR "psi: task underflow! cpu=%d t=%d tasks=[%u %u %u %u] clear=%x set=%x\n",
716 cpu, t, groupc->tasks[0],737 cpu, t, groupc->tasks[0],
717 groupc->tasks[1], groupc->tasks[2],738 groupc->tasks[1], groupc->tasks[2],
718 groupc->tasks[3], groupc->tasks[4],739 groupc->tasks[3], clear, set);
719 clear, set);
720 psi_bug = 1;740 psi_bug = 1;
721 }741 }
722 }742 }
@@ -725,9 +745,25 @@ static void psi_group_change(struct psi_group *group, int cpu, · 2 unmodified lines
725 if (set & (1 << t))745 if (set & (1 << t))
726 groupc->tasks[t]++;746 groupc->tasks[t]++;
727 747
728 /* Calculate state mask representing active states */748 if (!group->enabled) {
749 /*
750 * On the first group change after disabling PSI, conclude
751 * the current state and flush its time. This is unlikely
752 * to matter to the user, but aggregation (get_recent_times)
753 * may have already incorporated the live state into times_prev;
754 * avoid a delta sample underflow when PSI is later re-enabled.
755 */
756 if (unlikely(groupc->state_mask & (1 << PSI_NONIDLE)))
757 record_times(groupc, now);
758
759 groupc->state_mask = state_mask;
760
761 write_seqcount_end(&groupc->seq);
762 return;
763 }
764
729 for (s = 0; s < NR_PSI_STATES; s++) {765 for (s = 0; s < NR_PSI_STATES; s++) {
730 if (test_state(groupc->tasks, s))766 if (test_state(groupc->tasks, s, state_mask & PSI_ONCPU))
731 state_mask |= (1 << s);767 state_mask |= (1 << s);
732 }768 }
733 769
@@ -739,9 +775,11 @@ static void psi_group_change(struct psi_group *group, int cpu, · 5 unmodified lines
739 * task in a cgroup is in_memstall, the corresponding groupc775 * task in a cgroup is in_memstall, the corresponding groupc
740 * on that cpu is in PSI_MEM_FULL state.776 * on that cpu is in PSI_MEM_FULL state.
741 */777 */
742 if (unlikely(groupc->tasks[NR_ONCPU] && cpu_curr(cpu)->in_memstall))778 if (unlikely((state_mask & PSI_ONCPU) && cpu_curr(cpu)->in_memstall))
743 state_mask |= (1 << PSI_MEM_FULL);779 state_mask |= (1 << PSI_MEM_FULL);
744 780
781 record_times(groupc, now);
782
745 groupc->state_mask = state_mask;783 groupc->state_mask = state_mask;
746 784
747 write_seqcount_end(&groupc->seq);785 write_seqcount_end(&groupc->seq);
@@ -753,27 +791,12 @@ static void psi_group_change(struct psi_group *group, int cpu, · 5 unmodified lines
753 schedule_delayed_work(&group->avgs_work, PSI_FREQ);791 schedule_delayed_work(&group->avgs_work, PSI_FREQ);
754}792}
755 793
756static struct psi_group *iterate_groups(struct task_struct *task, void **iter)794static inline struct psi_group *task_psi_group(struct task_struct *task)
757{795{
758 if (*iter == &psi_system)
759 return NULL;
760
761#ifdef CONFIG_CGROUPS796#ifdef CONFIG_CGROUPS
762 if (static_branch_likely(&psi_cgroups_enabled)) {797 if (static_branch_likely(&psi_cgroups_enabled))
763 struct cgroup *cgroup = NULL;798 return cgroup_psi(task_dfl_cgroup(task));
764
765 if (!*iter)
766 cgroup = task->cgroups->dfl_cgrp;
767 else
768 cgroup = cgroup_parent(*iter);
769
770 if (cgroup && cgroup_parent(cgroup)) {
771 *iter = cgroup;
772 return cgroup_psi(cgroup);
773 }
774 }
775#endif799#endif
776 *iter = &psi_system;
777 return &psi_system;800 return &psi_system;
778}801}
779 802
@@ -796,8 +819,6 @@ void psi_task_change(struct task_struct *task, int clear, int set) · 16 unmodified lines
796{819{
797 int cpu = task_cpu(task);820 int cpu = task_cpu(task);
798 struct psi_group *group;821 struct psi_group *group;
799 bool wake_clock = true;
800 void *iter = NULL;
801 u64 now;822 u64 now;
802 823
803 if (!task->pid)824 if (!task->pid)
@@ -806,19 +827,11 @@ void psi_task_change(struct task_struct *task, int clear, int set) · 2 unmodified lines
806 psi_flags_change(task, clear, set);827 psi_flags_change(task, clear, set);
807 828
808 now = cpu_clock(cpu);829 now = cpu_clock(cpu);
809 /*
810 * Periodic aggregation shuts off if there is a period of no
811 * task changes, so we wake it back up if necessary. However,
812 * don't do this if the task change is the aggregation worker
813 * itself going to sleep, or we'll ping-pong forever.
814 */
815 if (unlikely((clear & TSK_RUNNING) &&
816 (task->flags & PF_WQ_WORKER) &&
817 wq_worker_last_func(task) == psi_avgs_work))
818 wake_clock = false;
819 830
820 while ((group = iterate_groups(task, &iter)))831 group = task_psi_group(task);
821 psi_group_change(group, cpu, clear, set, now, wake_clock);832 do {
833 psi_group_change(group, cpu, clear, set, now, true);
834 } while ((group = group->parent));
822}835}
823 836
824void psi_task_switch(struct task_struct *prev, struct task_struct *next,837void psi_task_switch(struct task_struct *prev, struct task_struct *next,
@@ -826,34 +839,30 @@ void psi_task_switch(struct task_struct *prev, struct task_struct *next, · 1 unmodified line
826{839{
827 struct psi_group *group, *common = NULL;840 struct psi_group *group, *common = NULL;
828 int cpu = task_cpu(prev);841 int cpu = task_cpu(prev);
829 void *iter;
830 u64 now = cpu_clock(cpu);842 u64 now = cpu_clock(cpu);
831 843
832 if (next->pid) {844 if (next->pid) {
833 bool identical_state;
834
835 psi_flags_change(next, 0, TSK_ONCPU);845 psi_flags_change(next, 0, TSK_ONCPU);
836 /*846 /*
837 * When switching between tasks that have an identical847 * Set TSK_ONCPU on @next's cgroups. If @next shares any
838 * runtime state, the cgroup that contains both tasks848 * ancestors with @prev, those will already have @prev's
839 * we reach the first common ancestor. Iterate @next's849 * TSK_ONCPU bit set, and we can stop the iteration there.
840 * ancestors only until we encounter @prev's ONCPU.
841 */850 */
842 identical_state = prev->psi_flags == next->psi_flags;851 group = task_psi_group(next);
843 iter = NULL;852 do {
844 while ((group = iterate_groups(next, &iter))) {853 if (per_cpu_ptr(group->pcpu, cpu)->state_mask &
845 if (identical_state &&854 PSI_ONCPU) {
846 per_cpu_ptr(group->pcpu, cpu)->tasks[NR_ONCPU]) {
847 common = group;855 common = group;
848 break;856 break;
849 }857 }
850 858
851 psi_group_change(group, cpu, 0, TSK_ONCPU, now, true);859 psi_group_change(group, cpu, 0, TSK_ONCPU, now, true);
852 }860 } while ((group = group->parent));
853 }861 }
854 862
855 if (prev->pid) {863 if (prev->pid) {
856 int clear = TSK_ONCPU, set = 0;864 int clear = TSK_ONCPU, set = 0;
865 bool wake_clock = true;
857 866
858 /*867 /*
859 * When we're going to sleep, psi_dequeue() lets us868 * When we're going to sleep, psi_dequeue() lets us
@@ -867,26 +876,74 @@ void psi_task_switch(struct task_struct *prev, struct task_struct *next, · 7 unmodified lines
867 clear |= TSK_MEMSTALL_RUNNING;876 clear |= TSK_MEMSTALL_RUNNING;
868 if (prev->in_iowait)877 if (prev->in_iowait)
869 set |= TSK_IOWAIT;878 set |= TSK_IOWAIT;
879
880 /*
881 * Periodic aggregation shuts off if there is a period of no
882 * task changes, so we wake it back up if necessary. However,
883 * don't do this if the task change is the aggregation worker
884 * itself going to sleep, or we'll ping-pong forever.
885 */
886 if (unlikely((prev->flags & PF_WQ_WORKER) &&
887 wq_worker_last_func(prev) == psi_avgs_work))
888 wake_clock = false;
870 }889 }
871 890
872 psi_flags_change(prev, clear, set);891 psi_flags_change(prev, clear, set);
873 892
874 iter = NULL;893 group = task_psi_group(prev);
875 while ((group = iterate_groups(prev, &iter)) && group != common)894 do {
876 psi_group_change(group, cpu, clear, set, now, true);895 if (group == common)
896 break;
897 psi_group_change(group, cpu, clear, set, now, wake_clock);
898 } while ((group = group->parent));
877 899
878 /*900 /*
879 * TSK_ONCPU is handled up to the common ancestor. If we're tasked901 * TSK_ONCPU is handled up to the common ancestor. If there are
880 * with dequeuing too, finish that for the rest of the hierarchy.902 * any other differences between the two tasks (e.g. prev goes
903 * to sleep, or only one task is memstall), finish propagating
904 * those differences all the way up to the root.
881 */905 */
882 if (sleep) {906 if ((prev->psi_flags ^ next->psi_flags) & ~TSK_ONCPU) {
883 clear &= ~TSK_ONCPU;907 clear &= ~TSK_ONCPU;
884 for (; group; group = iterate_groups(prev, &iter))908 for (; group; group = group->parent)
885 psi_group_change(group, cpu, clear, set, now, true);909 psi_group_change(group, cpu, clear, set, now, wake_clock);
886 }910 }
887 }911 }
888}912}
889 913
914#ifdef CONFIG_IRQ_TIME_ACCOUNTING
915void psi_account_irqtime(struct task_struct *task, u32 delta)
916{
917 int cpu = task_cpu(task);
918 struct psi_group *group;
919 struct psi_group_cpu *groupc;
920 u64 now;
921
922 if (!task->pid)
923 return;
924
925 now = cpu_clock(cpu);
926
927 group = task_psi_group(task);
928 do {
929 if (!group->enabled)
930 continue;
931
932 groupc = per_cpu_ptr(group->pcpu, cpu);
933
934 write_seqcount_begin(&groupc->seq);
935
936 record_times(groupc, now);
937 groupc->times[PSI_IRQ_FULL] += delta;
938
939 write_seqcount_end(&groupc->seq);
940
941 if (group->poll_states & (1 << PSI_IRQ_FULL))
942 psi_schedule_poll_work(group, 1);
943 } while ((group = group->parent));
944}
945#endif
946
890/**947/**
891 * psi_memstall_enter - mark the beginning of a memory stall section948 * psi_memstall_enter - mark the beginning of a memory stall section
892 * @flags: flags to handle nested sections949 * @flags: flags to handle nested sections
@@ -952,7 +1009,7 @@ EXPORT_SYMBOL_GPL(psi_memstall_leave); · 59 unmodified lines
952#ifdef CONFIG_CGROUPS1009#ifdef CONFIG_CGROUPS
953int psi_cgroup_alloc(struct cgroup *cgroup)1010int psi_cgroup_alloc(struct cgroup *cgroup)
954{1011{
955 if (static_branch_likely(&psi_disabled))1012 if (!static_branch_likely(&psi_cgroups_enabled))
956 return 0;1013 return 0;
957 1014
958 cgroup->psi = kzalloc(sizeof(struct psi_group), GFP_KERNEL);1015 cgroup->psi = kzalloc(sizeof(struct psi_group), GFP_KERNEL);
@@ -965,12 +1022,13 @@ int psi_cgroup_alloc(struct cgroup *cgroup) · 6 unmodified lines
965 return -ENOMEM;1022 return -ENOMEM;
966 }1023 }
967 group_init(cgroup->psi);1024 group_init(cgroup->psi);
1025 cgroup->psi->parent = cgroup_psi(cgroup_parent(cgroup));
968 return 0;1026 return 0;
969}1027}
970 1028
971void psi_cgroup_free(struct cgroup *cgroup)1029void psi_cgroup_free(struct cgroup *cgroup)
972{1030{
973 if (static_branch_likely(&psi_disabled))1031 if (!static_branch_likely(&psi_cgroups_enabled))
974 return;1032 return;
975 1033
976 cancel_delayed_work_sync(&cgroup->psi->avgs_work);1034 cancel_delayed_work_sync(&cgroup->psi->avgs_work);
@@ -998,7 +1056,7 @@ void cgroup_move_task(struct task_struct *task, struct css_set *to) · 21 unmodified lines
998 struct rq_flags rf;1056 struct rq_flags rf;
999 struct rq *rq;1057 struct rq *rq;
1000 1058
1001 if (static_branch_likely(&psi_disabled)) {1059 if (!static_branch_likely(&psi_cgroups_enabled)) {
1002 /*1060 /*
1003 * Lame to do this here, but the scheduler cannot be locked1061 * Lame to do this here, but the scheduler cannot be locked
1004 * from the outside, so we move cgroups from inside sched/.1062 * from the outside, so we move cgroups from inside sched/.
@@ -1046,10 +1104,45 @@ void cgroup_move_task(struct task_struct *task, struct css_set *to) · 41 unmodified lines
1046 1104
1047 task_rq_unlock(rq, task, &rf);1105 task_rq_unlock(rq, task, &rf);
1048}1106}
1107
1108void psi_cgroup_restart(struct psi_group *group)
1109{
1110 int cpu;
1111
1112 /*
1113 * After we disable psi_group->enabled, we don't actually
1114 * stop percpu tasks accounting in each psi_group_cpu,
1115 * instead only stop test_state() loop, record_times()
1116 * and averaging worker, see psi_group_change() for details.
1117 *
1118 * When disable cgroup PSI, this function has nothing to sync
1119 * since cgroup pressure files are hidden and percpu psi_group_cpu
1120 * would see !psi_group->enabled and only do task accounting.
1121 *
1122 * When re-enable cgroup PSI, this function use psi_group_change()
1123 * to get correct state mask from test_state() loop on tasks[],
1124 * and restart groupc->state_start from now, use .clear = .set = 0
1125 * here since no task status really changed.
1126 */
1127 if (!group->enabled)
1128 return;
1129
1130 for_each_possible_cpu(cpu) {
1131 struct rq *rq = cpu_rq(cpu);
1132 struct rq_flags rf;
1133 u64 now;
1134
1135 rq_lock_irq(rq, &rf);
1136 now = cpu_clock(cpu);
1137 psi_group_change(group, cpu, 0, 0, now, true);
1138 rq_unlock_irq(rq, &rf);
1139 }
1140}
1049#endif /* CONFIG_CGROUPS */1141#endif /* CONFIG_CGROUPS */
1050 1142
1051int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res)1143int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res)
1052{1144{
1145 bool only_full = false;
1053 int full;1146 int full;
1054 u64 now;1147 u64 now;
1055 1148
@@ -1064,7 +1157,11 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) · 8 unmodified lines
1064 group->avg_next_update = update_averages(group, now);1157 group->avg_next_update = update_averages(group, now);
1065 mutex_unlock(&group->avgs_lock);1158 mutex_unlock(&group->avgs_lock);
1066 1159
1067 for (full = 0; full < 2; full++) {1160#ifdef CONFIG_IRQ_TIME_ACCOUNTING
1161 only_full = res == PSI_IRQ;
1162#endif
1163
1164 for (full = 0; full < 2 - only_full; full++) {
1068 unsigned long avg[3] = { 0, };1165 unsigned long avg[3] = { 0, };
1069 u64 total = 0;1166 u64 total = 0;
1070 int w;1167 int w;
@@ -1078,7 +1175,7 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) · 7 unmodified lines
1078 }1175 }
1079 1176
1080 seq_printf(m, "%s avg10=%lu.%02lu avg60=%lu.%02lu avg300=%lu.%02lu total=%llu\n",1177 seq_printf(m, "%s avg10=%lu.%02lu avg60=%lu.%02lu avg300=%lu.%02lu total=%llu\n",
1081 full ? "full" : "some",1178 full || only_full ? "full" : "some",
1082 LOAD_INT(avg[0]), LOAD_FRAC(avg[0]),1179 LOAD_INT(avg[0]), LOAD_FRAC(avg[0]),
1083 LOAD_INT(avg[1]), LOAD_FRAC(avg[1]),1180 LOAD_INT(avg[1]), LOAD_FRAC(avg[1]),
1084 LOAD_INT(avg[2]), LOAD_FRAC(avg[2]),1181 LOAD_INT(avg[2]), LOAD_FRAC(avg[2]),
@@ -1106,6 +1203,11 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, · 21 unmodified lines
1106 else1203 else
1107 return ERR_PTR(-EINVAL);1204 return ERR_PTR(-EINVAL);
1108 1205
1206#ifdef CONFIG_IRQ_TIME_ACCOUNTING
1207 if (res == PSI_IRQ && --state != PSI_IRQ_FULL)
1208 return ERR_PTR(-EINVAL);
1209#endif
1210
1109 if (state >= PSI_NONIDLE)1211 if (state >= PSI_NONIDLE)
1110 return ERR_PTR(-EINVAL);1212 return ERR_PTR(-EINVAL);
1111 1213
@@ -1390,6 +1492,33 @@ static const struct proc_ops psi_cpu_proc_ops = { · 278 unmodified lines
1390 .proc_release = psi_fop_release,1492 .proc_release = psi_fop_release,
1391};1493};
1392 1494
1495#ifdef CONFIG_IRQ_TIME_ACCOUNTING
1496static int psi_irq_show(struct seq_file *m, void *v)
1497{
1498 return psi_show(m, &psi_system, PSI_IRQ);
1499}
1500
1501static int psi_irq_open(struct inode *inode, struct file *file)
1502{
1503 return psi_open(file, psi_irq_show);
1504}
1505
1506static ssize_t psi_irq_write(struct file *file, const char __user *user_buf,
1507 size_t nbytes, loff_t *ppos)
1508{
1509 return psi_write(file, user_buf, nbytes, PSI_IRQ);
1510}
1511
1512static const struct proc_ops psi_irq_proc_ops = {
1513 .proc_open = psi_irq_open,
1514 .proc_read = seq_read,
1515 .proc_lseek = seq_lseek,
1516 .proc_write = psi_irq_write,
1517 .proc_poll = psi_fop_poll,
1518 .proc_release = psi_fop_release,
1519};
1520#endif
1521
1393static int __init psi_proc_init(void)1522static int __init psi_proc_init(void)
1394{1523{
1395 if (psi_enable) {1524 if (psi_enable) {
@@ -1397,6 +1526,9 @@ static int __init psi_proc_init(void) · 1 unmodified line
1397 proc_create("pressure/io", 0666, NULL, &psi_io_proc_ops);1526 proc_create("pressure/io", 0666, NULL, &psi_io_proc_ops);
1398 proc_create("pressure/memory", 0666, NULL, &psi_memory_proc_ops);1527 proc_create("pressure/memory", 0666, NULL, &psi_memory_proc_ops);
1399 proc_create("pressure/cpu", 0666, NULL, &psi_cpu_proc_ops);1528 proc_create("pressure/cpu", 0666, NULL, &psi_cpu_proc_ops);
1529#ifdef CONFIG_IRQ_TIME_ACCOUNTING
1530 proc_create("pressure/irq", 0666, NULL, &psi_irq_proc_ops);
1531#endif
1400 }1532 }
1401 return 0;1533 return 0;
1402}1534}
kernel/sched/stats.h+6−0
@@ -107,6 +107,11 @@ __schedstats_from_se(struct sched_entity *se) · 106 unmodified lines
107}107}
108 108
109#ifdef CONFIG_PSI109#ifdef CONFIG_PSI
110void psi_task_change(struct task_struct *task, int clear, int set);
111void psi_task_switch(struct task_struct *prev, struct task_struct *next,
112 bool sleep);
113void psi_account_irqtime(struct task_struct *task, u32 delta);
114
110/*115/*
111 * PSI tracks state that persists across sleeps, such as iowaits and116 * PSI tracks state that persists across sleeps, such as iowaits and
112 * memory stalls. As a result, it has to distinguish between sleeps,117 * memory stalls. As a result, it has to distinguish between sleeps,
@@ -201,6 +206,7 @@ static inline void psi_ttwu_dequeue(struct task_struct *p) {} · 88 unmodified lines
201static inline void psi_sched_switch(struct task_struct *prev,206static inline void psi_sched_switch(struct task_struct *prev,
202 struct task_struct *next,207 struct task_struct *next,
203 bool sleep) {}208 bool sleep) {}
209static inline void psi_account_irqtime(struct task_struct *task, u32 delta) {}
204#endif /* CONFIG_PSI */210#endif /* CONFIG_PSI */
205 211
206#ifdef CONFIG_SCHED_INFO212#ifdef CONFIG_SCHED_INFO