History

Merge tag 'sched-urgent-2024-08-04' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler fixes from Thomas Gleixner: - When stime is larger than rtime due to accounting imprecision, then utime = rtime - stime becomes negative. As this is unsigned math, the result becomes a huge positive number. Cure it by resetting stime to rtime in that case, so utime becomes 0. - Restore consistent state when sched_cpu_deactivate() fails. When offlining a CPU fails in sched_cpu_deactivate() after the SMT present counter has been decremented, then the function aborts but fails to increment the SMT present counter and leaves it imbalanced. Consecutive operations cause it to underflow. Add the missing fixup for the error path. For SMT accounting the runqueue needs to marked online again in the error exit path to restore consistent state. * tag 'sched-urgent-2024-08-04' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched/core: Fix unbalance set_rq_online/offline() in sched_cpu_deactivate() sched/core: Introduce sched_set_rq_on/offline() helper sched/smt: Fix unbalance sched_smt_present dec/inc sched/smt: Introduce sched_smt_present_inc/dec() helper sched/cputime: Fix mul_u64_u64_div_u64() precision for cputime

Linus Torvalds committed 2 years ago

Commit 6cc82dc2 parents 1ddeb0e fe7a11c2 files · +53 −21Browse files

Unified · Split

2 files
  1. kernel/sched/core.c +47 −21
  2. kernel/sched/cputime.c +6 −0
kernel/sched/core.c+47−21
@@ -7845,6 +7845,30 @@ void set_rq_offline(struct rq *rq) · 7844 unmodified lines
78457845 }
78467846 }
78477847
7848+static inline void sched_set_rq_online(struct rq *rq, int cpu)
7849+{
7850+ struct rq_flags rf;
7851+
7852+ rq_lock_irqsave(rq, &rf);
7853+ if (rq->rd) {
7854+ BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7855+ set_rq_online(rq);
7856+ }
7857+ rq_unlock_irqrestore(rq, &rf);
7858+}
7859+
7860+static inline void sched_set_rq_offline(struct rq *rq, int cpu)
7861+{
7862+ struct rq_flags rf;
7863+
7864+ rq_lock_irqsave(rq, &rf);
7865+ if (rq->rd) {
7866+ BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7867+ set_rq_offline(rq);
7868+ }
7869+ rq_unlock_irqrestore(rq, &rf);
7870+}
7871+
78487872 /*
78497873 * used to mark begin/end of suspend/resume:
78507874 */
@@ -7895,10 +7919,25 @@ static int cpuset_cpu_inactive(unsigned int cpu) · 44 unmodified lines
78957919 return 0;
78967920 }
78977921
7922+static inline void sched_smt_present_inc(int cpu)
7923+{
7924+#ifdef CONFIG_SCHED_SMT
7925+ if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
7926+ static_branch_inc_cpuslocked(&sched_smt_present);
7927+#endif
7928+}
7929+
7930+static inline void sched_smt_present_dec(int cpu)
7931+{
7932+#ifdef CONFIG_SCHED_SMT
7933+ if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
7934+ static_branch_dec_cpuslocked(&sched_smt_present);
7935+#endif
7936+}
7937+
78987938 int sched_cpu_activate(unsigned int cpu)
78997939 {
79007940 struct rq *rq = cpu_rq(cpu);
7901− struct rq_flags rf;
79027941
79037942 /*
79047943 * Clear the balance_push callback and prepare to schedule
@@ -7906,13 +7945,10 @@ int sched_cpu_activate(unsigned int cpu) · 1 unmodified line
79067945 */
79077946 balance_push_set(cpu, false);
79087947
7909−#ifdef CONFIG_SCHED_SMT
79107948 /*
79117949 * When going up, increment the number of cores with SMT present.
79127950 */
7913− if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
7914− static_branch_inc_cpuslocked(&sched_smt_present);
7915−#endif
7951+ sched_smt_present_inc(cpu);
79167952 set_cpu_active(cpu, true);
79177953
79187954 if (sched_smp_initialized) {
@@ -7930,12 +7966,7 @@ int sched_cpu_activate(unsigned int cpu) · 11 unmodified lines
79307966 * 2) At runtime, if cpuset_cpu_active() fails to rebuild the
79317967 * domains.
79327968 */
7933− rq_lock_irqsave(rq, &rf);
7934− if (rq->rd) {
7935− BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7936− set_rq_online(rq);
7937− }
7938− rq_unlock_irqrestore(rq, &rf);
7969+ sched_set_rq_online(rq, cpu);
79397970
79407971 return 0;
79417972 }
@@ -7943,7 +7974,6 @@ int sched_cpu_activate(unsigned int cpu) · 1 unmodified line
79437974 int sched_cpu_deactivate(unsigned int cpu)
79447975 {
79457976 struct rq *rq = cpu_rq(cpu);
7946− struct rq_flags rf;
79477977 int ret;
79487978
79497979 /*
@@ -7974,20 +8004,14 @@ int sched_cpu_deactivate(unsigned int cpu) · 24 unmodified lines
79748004 */
79758005 synchronize_rcu();
79768006
7977− rq_lock_irqsave(rq, &rf);
7978− if (rq->rd) {
7979− BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
7980− set_rq_offline(rq);
7981− }
7982− rq_unlock_irqrestore(rq, &rf);
8007+ sched_set_rq_offline(rq, cpu);
79838008
7984−#ifdef CONFIG_SCHED_SMT
79858009 /*
79868010 * When going down, decrement the number of cores with SMT present.
79878011 */
7988− if (cpumask_weight(cpu_smt_mask(cpu)) == 2)
7989− static_branch_dec_cpuslocked(&sched_smt_present);
8012+ sched_smt_present_dec(cpu);
79908013
8014+#ifdef CONFIG_SCHED_SMT
79918015 sched_core_cpu_deactivate(cpu);
79928016 #endif
79938017
@@ -7997,6 +8021,8 @@ int sched_cpu_deactivate(unsigned int cpu) · 3 unmodified lines
79978021 sched_update_numa(cpu, false);
79988022 ret = cpuset_cpu_inactive(cpu);
79998023 if (ret) {
8024+ sched_smt_present_inc(cpu);
8025+ sched_set_rq_online(rq, cpu);
80008026 balance_push_set(cpu, false);
80018027 set_cpu_active(cpu, true);
80028028 sched_update_numa(cpu, true);
kernel/sched/cputime.c+6−0
@@ -582,6 +582,12 @@ void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev, · 581 unmodified lines
582582 }
583583
584584 stime = mul_u64_u64_div_u64(stime, rtime, stime + utime);
585+ /*
586+ * Because mul_u64_u64_div_u64() can approximate on some
587+ * achitectures; enforce the constraint that: a*b/(b+c) <= a.
588+ */
589+ if (unlikely(stime > rtime))
590+ stime = rtime;
585591
586592 update:
587593 /*