Pull scheduler fixes from Thomas Gleixner: - When stime is larger than rtime due to accounting imprecision, then utime = rtime - stime becomes negative. As this is unsigned math, the result becomes a huge positive number. Cure it by resetting stime to rtime in that case, so utime becomes 0. - Restore consistent state when sched_cpu_deactivate() fails. When offlining a CPU fails in sched_cpu_deactivate() after the SMT present counter has been decremented, then the function aborts but fails to increment the SMT present counter and leaves it imbalanced. Consecutive operations cause it to underflow. Add the missing fixup for the error path. For SMT accounting the runqueue needs to marked online again in the error exit path to restore consistent state. * tag 'sched-urgent-2024-08-04' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched/core: Fix unbalance set_rq_online/offline() in sched_cpu_deactivate() sched/core: Introduce sched_set_rq_on/offline() helper sched/smt: Fix unbalance sched_smt_present dec/inc sched/smt: Introduce sched_smt_present_inc/dec() helper sched/cputime: Fix mul_u64_u64_div_u64() precision for cputime
Unified · Split
| @@ -7845,6 +7845,30 @@ void set_rq_offline(struct rq *rq) · 7844 unmodified lines | |||
| 7845 | } | 7845 | } |
| 7846 | } | 7846 | } |
| 7847 | 7847 | ||
| 7848 | static inline void sched_set_rq_online(struct rq *rq, int cpu) | ||
| 7849 | { | ||
| 7850 | struct rq_flags rf; | ||
| 7851 | |||
| 7852 | rq_lock_irqsave(rq, &rf); | ||
| 7853 | if (rq->rd) { | ||
| 7854 | BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); | ||
| 7855 | set_rq_online(rq); | ||
| 7856 | } | ||
| 7857 | rq_unlock_irqrestore(rq, &rf); | ||
| 7858 | } | ||
| 7859 | |||
| 7860 | static inline void sched_set_rq_offline(struct rq *rq, int cpu) | ||
| 7861 | { | ||
| 7862 | struct rq_flags rf; | ||
| 7863 | |||
| 7864 | rq_lock_irqsave(rq, &rf); | ||
| 7865 | if (rq->rd) { | ||
| 7866 | BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); | ||
| 7867 | set_rq_offline(rq); | ||
| 7868 | } | ||
| 7869 | rq_unlock_irqrestore(rq, &rf); | ||
| 7870 | } | ||
| 7871 | |||
| 7848 | /* | 7872 | /* |
| 7849 | * used to mark begin/end of suspend/resume: | 7873 | * used to mark begin/end of suspend/resume: |
| 7850 | */ | 7874 | */ |
| @@ -7895,10 +7919,25 @@ static int cpuset_cpu_inactive(unsigned int cpu) · 44 unmodified lines | |||
| 7895 | return 0; | 7919 | return 0; |
| 7896 | } | 7920 | } |
| 7897 | 7921 | ||
| 7922 | static inline void sched_smt_present_inc(int cpu) | ||
| 7923 | { | ||
| 7924 | #ifdef CONFIG_SCHED_SMT | ||
| 7925 | if (cpumask_weight(cpu_smt_mask(cpu)) == 2) | ||
| 7926 | static_branch_inc_cpuslocked(&sched_smt_present); | ||
| 7927 | #endif | ||
| 7928 | } | ||
| 7929 | |||
| 7930 | static inline void sched_smt_present_dec(int cpu) | ||
| 7931 | { | ||
| 7932 | #ifdef CONFIG_SCHED_SMT | ||
| 7933 | if (cpumask_weight(cpu_smt_mask(cpu)) == 2) | ||
| 7934 | static_branch_dec_cpuslocked(&sched_smt_present); | ||
| 7935 | #endif | ||
| 7936 | } | ||
| 7937 | |||
| 7898 | int sched_cpu_activate(unsigned int cpu) | 7938 | int sched_cpu_activate(unsigned int cpu) |
| 7899 | { | 7939 | { |
| 7900 | struct rq *rq = cpu_rq(cpu); | 7940 | struct rq *rq = cpu_rq(cpu); |
| 7901 | struct rq_flags rf; | ||
| 7902 | 7941 | ||
| 7903 | /* | 7942 | /* |
| 7904 | * Clear the balance_push callback and prepare to schedule | 7943 | * Clear the balance_push callback and prepare to schedule |
| @@ -7906,13 +7945,10 @@ int sched_cpu_activate(unsigned int cpu) · 1 unmodified line | |||
| 7906 | */ | 7945 | */ |
| 7907 | balance_push_set(cpu, false); | 7946 | balance_push_set(cpu, false); |
| 7908 | 7947 | ||
| 7909 | #ifdef CONFIG_SCHED_SMT | ||
| 7910 | /* | 7948 | /* |
| 7911 | * When going up, increment the number of cores with SMT present. | 7949 | * When going up, increment the number of cores with SMT present. |
| 7912 | */ | 7950 | */ |
| 7913 | if (cpumask_weight(cpu_smt_mask(cpu)) == 2) | 7951 | sched_smt_present_inc(cpu); |
| 7914 | static_branch_inc_cpuslocked(&sched_smt_present); | ||
| 7915 | #endif | ||
| 7916 | set_cpu_active(cpu, true); | 7952 | set_cpu_active(cpu, true); |
| 7917 | 7953 | ||
| 7918 | if (sched_smp_initialized) { | 7954 | if (sched_smp_initialized) { |
| @@ -7930,12 +7966,7 @@ int sched_cpu_activate(unsigned int cpu) · 11 unmodified lines | |||
| 7930 | * 2) At runtime, if cpuset_cpu_active() fails to rebuild the | 7966 | * 2) At runtime, if cpuset_cpu_active() fails to rebuild the |
| 7931 | * domains. | 7967 | * domains. |
| 7932 | */ | 7968 | */ |
| 7933 | rq_lock_irqsave(rq, &rf); | 7969 | sched_set_rq_online(rq, cpu); |
| 7934 | if (rq->rd) { | ||
| 7935 | BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); | ||
| 7936 | set_rq_online(rq); | ||
| 7937 | } | ||
| 7938 | rq_unlock_irqrestore(rq, &rf); | ||
| 7939 | 7970 | ||
| 7940 | return 0; | 7971 | return 0; |
| 7941 | } | 7972 | } |
| @@ -7943,7 +7974,6 @@ int sched_cpu_activate(unsigned int cpu) · 1 unmodified line | |||
| 7943 | int sched_cpu_deactivate(unsigned int cpu) | 7974 | int sched_cpu_deactivate(unsigned int cpu) |
| 7944 | { | 7975 | { |
| 7945 | struct rq *rq = cpu_rq(cpu); | 7976 | struct rq *rq = cpu_rq(cpu); |
| 7946 | struct rq_flags rf; | ||
| 7947 | int ret; | 7977 | int ret; |
| 7948 | 7978 | ||
| 7949 | /* | 7979 | /* |
| @@ -7974,20 +8004,14 @@ int sched_cpu_deactivate(unsigned int cpu) · 24 unmodified lines | |||
| 7974 | */ | 8004 | */ |
| 7975 | synchronize_rcu(); | 8005 | synchronize_rcu(); |
| 7976 | 8006 | ||
| 7977 | rq_lock_irqsave(rq, &rf); | 8007 | sched_set_rq_offline(rq, cpu); |
| 7978 | if (rq->rd) { | ||
| 7979 | BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); | ||
| 7980 | set_rq_offline(rq); | ||
| 7981 | } | ||
| 7982 | rq_unlock_irqrestore(rq, &rf); | ||
| 7983 | 8008 | ||
| 7984 | #ifdef CONFIG_SCHED_SMT | ||
| 7985 | /* | 8009 | /* |
| 7986 | * When going down, decrement the number of cores with SMT present. | 8010 | * When going down, decrement the number of cores with SMT present. |
| 7987 | */ | 8011 | */ |
| 7988 | if (cpumask_weight(cpu_smt_mask(cpu)) == 2) | 8012 | sched_smt_present_dec(cpu); |
| 7989 | static_branch_dec_cpuslocked(&sched_smt_present); | ||
| 7990 | 8013 | ||
| 8014 | #ifdef CONFIG_SCHED_SMT | ||
| 7991 | sched_core_cpu_deactivate(cpu); | 8015 | sched_core_cpu_deactivate(cpu); |
| 7992 | #endif | 8016 | #endif |
| 7993 | 8017 | ||
| @@ -7997,6 +8021,8 @@ int sched_cpu_deactivate(unsigned int cpu) · 3 unmodified lines | |||
| 7997 | sched_update_numa(cpu, false); | 8021 | sched_update_numa(cpu, false); |
| 7998 | ret = cpuset_cpu_inactive(cpu); | 8022 | ret = cpuset_cpu_inactive(cpu); |
| 7999 | if (ret) { | 8023 | if (ret) { |
| 8024 | sched_smt_present_inc(cpu); | ||
| 8025 | sched_set_rq_online(rq, cpu); | ||
| 8000 | balance_push_set(cpu, false); | 8026 | balance_push_set(cpu, false); |
| 8001 | set_cpu_active(cpu, true); | 8027 | set_cpu_active(cpu, true); |
| 8002 | sched_update_numa(cpu, true); | 8028 | sched_update_numa(cpu, true); |
| @@ -582,6 +582,12 @@ void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev, · 581 unmodified lines | |||
| 582 | } | 582 | } |
| 583 | 583 | ||
| 584 | stime = mul_u64_u64_div_u64(stime, rtime, stime + utime); | 584 | stime = mul_u64_u64_div_u64(stime, rtime, stime + utime); |
| 585 | /* | ||
| 586 | * Because mul_u64_u64_div_u64() can approximate on some | ||
| 587 | * achitectures; enforce the constraint that: a*b/(b+c) <= a. | ||
| 588 | */ | ||
| 589 | if (unlikely(stime > rtime)) | ||
| 590 | stime = rtime; | ||
| 585 | 591 | ||
| 586 | update: | 592 | update: |
| 587 | /* | 593 | /* |