History

sched/pelt: Remove unused runnable load average

Now that runnable_load_avg is no more used, we can remove it to make space for a new signal.

Vincent Guittot committed 6 years ago

Commit 0dacee11 parent fb86f5b6 files · +43 −171Browse files

Unified · Split

6 files
  1. include/linux/sched.h +1 −4
  2. kernel/sched/core.c +0 −2
  3. kernel/sched/debug.c +0 −8
  4. kernel/sched/fair.c +16 −114
  5. kernel/sched/pelt.c +24 −38
  6. kernel/sched/sched.h +2 −5
include/linux/sched.h+1−4
@@ -357,7 +357,7 @@ struct util_est { · 356 unmodified lines
357 357
358/*358/*
359 * The load_avg/util_avg accumulates an infinite geometric series359 * The load_avg/util_avg accumulates an infinite geometric series
360 * (see __update_load_avg() in kernel/sched/fair.c).360 * (see __update_load_avg_cfs_rq() in kernel/sched/pelt.c).
361 *361 *
362 * [load_avg definition]362 * [load_avg definition]
363 *363 *
@@ -401,11 +401,9 @@ struct util_est { · 37 unmodified lines
401struct sched_avg {401struct sched_avg {
402 u64 last_update_time;402 u64 last_update_time;
403 u64 load_sum;403 u64 load_sum;
404 u64 runnable_load_sum;
405 u32 util_sum;404 u32 util_sum;
406 u32 period_contrib;405 u32 period_contrib;
407 unsigned long load_avg;406 unsigned long load_avg;
408 unsigned long runnable_load_avg;
409 unsigned long util_avg;407 unsigned long util_avg;
410 struct util_est util_est;408 struct util_est util_est;
411} ____cacheline_aligned;409} ____cacheline_aligned;
@@ -449,7 +447,6 @@ struct sched_statistics { · 37 unmodified lines
449struct sched_entity {447struct sched_entity {
450 /* For load-balancing: */448 /* For load-balancing: */
451 struct load_weight load;449 struct load_weight load;
452 unsigned long runnable_weight;
453 struct rb_node run_node;450 struct rb_node run_node;
454 struct list_head group_node;451 struct list_head group_node;
455 unsigned int on_rq;452 unsigned int on_rq;
kernel/sched/core.c+0−2
@@ -761,7 +761,6 @@ static void set_load_weight(struct task_struct *p, bool update_load) · 760 unmodified lines
761 if (task_has_idle_policy(p)) {761 if (task_has_idle_policy(p)) {
762 load->weight = scale_load(WEIGHT_IDLEPRIO);762 load->weight = scale_load(WEIGHT_IDLEPRIO);
763 load->inv_weight = WMULT_IDLEPRIO;763 load->inv_weight = WMULT_IDLEPRIO;
764 p->se.runnable_weight = load->weight;
765 return;764 return;
766 }765 }
767 766
@@ -774,7 +773,6 @@ static void set_load_weight(struct task_struct *p, bool update_load) · 6 unmodified lines
774 } else {773 } else {
775 load->weight = scale_load(sched_prio_to_weight[prio]);774 load->weight = scale_load(sched_prio_to_weight[prio]);
776 load->inv_weight = sched_prio_to_wmult[prio];775 load->inv_weight = sched_prio_to_wmult[prio];
777 p->se.runnable_weight = load->weight;
778 }776 }
779}777}
780 778
kernel/sched/debug.c+0−8
@@ -402,11 +402,9 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group · 401 unmodified lines
402 }402 }
403 403
404 P(se->load.weight);404 P(se->load.weight);
405 P(se->runnable_weight);
406#ifdef CONFIG_SMP405#ifdef CONFIG_SMP
407 P(se->avg.load_avg);406 P(se->avg.load_avg);
408 P(se->avg.util_avg);407 P(se->avg.util_avg);
409 P(se->avg.runnable_load_avg);
410#endif408#endif
411 409
412#undef PN_SCHEDSTAT410#undef PN_SCHEDSTAT
@@ -524,11 +522,8 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) · 111 unmodified lines
524 SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running);522 SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running);
525 SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight);523 SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight);
526#ifdef CONFIG_SMP524#ifdef CONFIG_SMP
527 SEQ_printf(m, " .%-30s: %ld\n", "runnable_weight", cfs_rq->runnable_weight);
528 SEQ_printf(m, " .%-30s: %lu\n", "load_avg",525 SEQ_printf(m, " .%-30s: %lu\n", "load_avg",
529 cfs_rq->avg.load_avg);526 cfs_rq->avg.load_avg);
530 SEQ_printf(m, " .%-30s: %lu\n", "runnable_load_avg",
531 cfs_rq->avg.runnable_load_avg);
532 SEQ_printf(m, " .%-30s: %lu\n", "util_avg",527 SEQ_printf(m, " .%-30s: %lu\n", "util_avg",
533 cfs_rq->avg.util_avg);528 cfs_rq->avg.util_avg);
534 SEQ_printf(m, " .%-30s: %u\n", "util_est_enqueued",529 SEQ_printf(m, " .%-30s: %u\n", "util_est_enqueued",
@@ -947,13 +942,10 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns, · 412 unmodified lines
947 "nr_involuntary_switches", (long long)p->nivcsw);942 "nr_involuntary_switches", (long long)p->nivcsw);
948 943
949 P(se.load.weight);944 P(se.load.weight);
950 P(se.runnable_weight);
951#ifdef CONFIG_SMP945#ifdef CONFIG_SMP
952 P(se.avg.load_sum);946 P(se.avg.load_sum);
953 P(se.avg.runnable_load_sum);
954 P(se.avg.util_sum);947 P(se.avg.util_sum);
955 P(se.avg.load_avg);948 P(se.avg.load_avg);
956 P(se.avg.runnable_load_avg);
957 P(se.avg.util_avg);949 P(se.avg.util_avg);
958 P(se.avg.last_update_time);950 P(se.avg.last_update_time);
959 P(se.avg.util_est.ewma);951 P(se.avg.util_est.ewma);
kernel/sched/fair.c+16−114
@@ -741,9 +741,7 @@ void init_entity_runnable_average(struct sched_entity *se) · 740 unmodified lines
741 * nothing has been attached to the task group yet.741 * nothing has been attached to the task group yet.
742 */742 */
743 if (entity_is_task(se))743 if (entity_is_task(se))
744 sa->runnable_load_avg = sa->load_avg = scale_load_down(se->load.weight);744 sa->load_avg = scale_load_down(se->load.weight);
745
746 se->runnable_weight = se->load.weight;
747 745
748 /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */746 /* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */
749}747}
@@ -2898,25 +2896,6 @@ account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se) · 2148 unmodified lines
2898} while (0)2896} while (0)
2899 2897
2900#ifdef CONFIG_SMP2898#ifdef CONFIG_SMP
2901static inline void
2902enqueue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
2903{
2904 cfs_rq->runnable_weight += se->runnable_weight;
2905
2906 cfs_rq->avg.runnable_load_avg += se->avg.runnable_load_avg;
2907 cfs_rq->avg.runnable_load_sum += se_runnable(se) * se->avg.runnable_load_sum;
2908}
2909
2910static inline void
2911dequeue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
2912{
2913 cfs_rq->runnable_weight -= se->runnable_weight;
2914
2915 sub_positive(&cfs_rq->avg.runnable_load_avg, se->avg.runnable_load_avg);
2916 sub_positive(&cfs_rq->avg.runnable_load_sum,
2917 se_runnable(se) * se->avg.runnable_load_sum);
2918}
2919
2920static inline void2899static inline void
2921enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)2900enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
2922{2901{
@@ -2932,28 +2911,22 @@ dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) · 9 unmodified lines
2932}2911}
2933#else2912#else
2934static inline void2913static inline void
2935enqueue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
2936static inline void
2937dequeue_runnable_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
2938static inline void
2939enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }2914enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
2940static inline void2915static inline void
2941dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }2916dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
2942#endif2917#endif
2943 2918
2944static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,2919static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
2945 unsigned long weight, unsigned long runnable)2920 unsigned long weight)
2946{2921{
2947 if (se->on_rq) {2922 if (se->on_rq) {
2948 /* commit outstanding execution time */2923 /* commit outstanding execution time */
2949 if (cfs_rq->curr == se)2924 if (cfs_rq->curr == se)
2950 update_curr(cfs_rq);2925 update_curr(cfs_rq);
2951 account_entity_dequeue(cfs_rq, se);2926 account_entity_dequeue(cfs_rq, se);
2952 dequeue_runnable_load_avg(cfs_rq, se);
2953 }2927 }
2954 dequeue_load_avg(cfs_rq, se);2928 dequeue_load_avg(cfs_rq, se);
2955 2929
2956 se->runnable_weight = runnable;
2957 update_load_set(&se->load, weight);2930 update_load_set(&se->load, weight);
2958 2931
2959#ifdef CONFIG_SMP2932#ifdef CONFIG_SMP
@@ -2961,16 +2934,13 @@ static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, · 1 unmodified line
2961 u32 divider = LOAD_AVG_MAX - 1024 + se->avg.period_contrib;2934 u32 divider = LOAD_AVG_MAX - 1024 + se->avg.period_contrib;
2962 2935
2963 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider);2936 se->avg.load_avg = div_u64(se_weight(se) * se->avg.load_sum, divider);
2964 se->avg.runnable_load_avg =
2965 div_u64(se_runnable(se) * se->avg.runnable_load_sum, divider);
2966 } while (0);2937 } while (0);
2967#endif2938#endif
2968 2939
2969 enqueue_load_avg(cfs_rq, se);2940 enqueue_load_avg(cfs_rq, se);
2970 if (se->on_rq) {2941 if (se->on_rq)
2971 account_entity_enqueue(cfs_rq, se);2942 account_entity_enqueue(cfs_rq, se);
2972 enqueue_runnable_load_avg(cfs_rq, se);2943
2973 }
2974}2944}
2975 2945
2976void reweight_task(struct task_struct *p, int prio)2946void reweight_task(struct task_struct *p, int prio)
@@ -2980,7 +2950,7 @@ void reweight_task(struct task_struct *p, int prio) · 3 unmodified lines
2980 struct load_weight *load = &se->load;2950 struct load_weight *load = &se->load;
2981 unsigned long weight = scale_load(sched_prio_to_weight[prio]);2951 unsigned long weight = scale_load(sched_prio_to_weight[prio]);
2982 2952
2983 reweight_entity(cfs_rq, se, weight, weight);2953 reweight_entity(cfs_rq, se, weight);
2984 load->inv_weight = sched_prio_to_wmult[prio];2954 load->inv_weight = sched_prio_to_wmult[prio];
2985}2955}
2986 2956
@@ -3092,50 +3062,6 @@ static long calc_group_shares(struct cfs_rq *cfs_rq) · 105 unmodified lines
3092 */3062 */
3093 return clamp_t(long, shares, MIN_SHARES, tg_shares);3063 return clamp_t(long, shares, MIN_SHARES, tg_shares);
3094}3064}
3095
3096/*
3097 * This calculates the effective runnable weight for a group entity based on
3098 * the group entity weight calculated above.
3099 *
3100 * Because of the above approximation (2), our group entity weight is
3101 * an load_avg based ratio (3). This means that it includes blocked load and
3102 * does not represent the runnable weight.
3103 *
3104 * Approximate the group entity's runnable weight per ratio from the group
3105 * runqueue:
3106 *
3107 * grq->avg.runnable_load_avg
3108 * ge->runnable_weight = ge->load.weight * -------------------------- (7)
3109 * grq->avg.load_avg
3110 *
3111 * However, analogous to above, since the avg numbers are slow, this leads to
3112 * transients in the from-idle case. Instead we use:
3113 *
3114 * ge->runnable_weight = ge->load.weight *
3115 *
3116 * max(grq->avg.runnable_load_avg, grq->runnable_weight)
3117 * ----------------------------------------------------- (8)
3118 * max(grq->avg.load_avg, grq->load.weight)
3119 *
3120 * Where these max() serve both to use the 'instant' values to fix the slow
3121 * from-idle and avoid the /0 on to-idle, similar to (6).
3122 */
3123static long calc_group_runnable(struct cfs_rq *cfs_rq, long shares)
3124{
3125 long runnable, load_avg;
3126
3127 load_avg = max(cfs_rq->avg.load_avg,
3128 scale_load_down(cfs_rq->load.weight));
3129
3130 runnable = max(cfs_rq->avg.runnable_load_avg,
3131 scale_load_down(cfs_rq->runnable_weight));
3132
3133 runnable *= shares;
3134 if (load_avg)
3135 runnable /= load_avg;
3136
3137 return clamp_t(long, runnable, MIN_SHARES, shares);
3138}
3139#endif /* CONFIG_SMP */3065#endif /* CONFIG_SMP */
3140 3066
3141static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);3067static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
@@ -3147,7 +3073,7 @@ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq); · 5 unmodified lines
3147static void update_cfs_group(struct sched_entity *se)3073static void update_cfs_group(struct sched_entity *se)
3148{3074{
3149 struct cfs_rq *gcfs_rq = group_cfs_rq(se);3075 struct cfs_rq *gcfs_rq = group_cfs_rq(se);
3150 long shares, runnable;3076 long shares;
3151 3077
3152 if (!gcfs_rq)3078 if (!gcfs_rq)
3153 return;3079 return;
@@ -3156,16 +3082,15 @@ static void update_cfs_group(struct sched_entity *se) · 2 unmodified lines
3156 return;3082 return;
3157 3083
3158#ifndef CONFIG_SMP3084#ifndef CONFIG_SMP
3159 runnable = shares = READ_ONCE(gcfs_rq->tg->shares);3085 shares = READ_ONCE(gcfs_rq->tg->shares);
3160 3086
3161 if (likely(se->load.weight == shares))3087 if (likely(se->load.weight == shares))
3162 return;3088 return;
3163#else3089#else
3164 shares = calc_group_shares(gcfs_rq);3090 shares = calc_group_shares(gcfs_rq);
3165 runnable = calc_group_runnable(gcfs_rq, shares);
3166#endif3091#endif
3167 3092
3168 reweight_entity(cfs_rq_of(se), se, shares, runnable);3093 reweight_entity(cfs_rq_of(se), se, shares);
3169}3094}
3170 3095
3171#else /* CONFIG_FAIR_GROUP_SCHED */3096#else /* CONFIG_FAIR_GROUP_SCHED */
@@ -3290,11 +3215,11 @@ void set_task_rq_fair(struct sched_entity *se, · 118 unmodified lines
3290 * _IFF_ we look at the pure running and runnable sums. Because they3215 * _IFF_ we look at the pure running and runnable sums. Because they
3291 * represent the very same entity, just at different points in the hierarchy.3216 * represent the very same entity, just at different points in the hierarchy.
3292 *3217 *
3293 * Per the above update_tg_cfs_util() is trivial and simply copies the running3218 * Per the above update_tg_cfs_util() is trivial * and simply copies the
3294 * sum over (but still wrong, because the group entity and group rq do not have3219 * running sum over (but still wrong, because the group entity and group rq do
3295 * their PELT windows aligned).3220 * not have their PELT windows aligned).
3296 *3221 *
3297 * However, update_tg_cfs_runnable() is more complex. So we have:3222 * However, update_tg_cfs_load() is more complex. So we have:
3298 *3223 *
3299 * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2)3224 * ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2)
3300 *3225 *
@@ -3375,11 +3300,11 @@ update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq · 74 unmodified lines
3375}3300}
3376 3301
3377static inline void3302static inline void
3378update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)3303update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
3379{3304{
3380 long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum;3305 long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum;
3381 unsigned long runnable_load_avg, load_avg;3306 unsigned long load_avg;
3382 u64 runnable_load_sum, load_sum = 0;3307 u64 load_sum = 0;
3383 s64 delta_sum;3308 s64 delta_sum;
3384 3309
3385 if (!runnable_sum)3310 if (!runnable_sum)
@@ -3427,20 +3352,6 @@ update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cf · 41 unmodified lines
3427 se->avg.load_avg = load_avg;3352 se->avg.load_avg = load_avg;
3428 add_positive(&cfs_rq->avg.load_avg, delta_avg);3353 add_positive(&cfs_rq->avg.load_avg, delta_avg);
3429 add_positive(&cfs_rq->avg.load_sum, delta_sum);3354 add_positive(&cfs_rq->avg.load_sum, delta_sum);
3430
3431 runnable_load_sum = (s64)se_runnable(se) * runnable_sum;
3432 runnable_load_avg = div_s64(runnable_load_sum, LOAD_AVG_MAX);
3433
3434 if (se->on_rq) {
3435 delta_sum = runnable_load_sum -
3436 se_weight(se) * se->avg.runnable_load_sum;
3437 delta_avg = runnable_load_avg - se->avg.runnable_load_avg;
3438 add_positive(&cfs_rq->avg.runnable_load_avg, delta_avg);
3439 add_positive(&cfs_rq->avg.runnable_load_sum, delta_sum);
3440 }
3441
3442 se->avg.runnable_load_sum = runnable_sum;
3443 se->avg.runnable_load_avg = runnable_load_avg;
3444}3355}
3445 3356
3446static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum)3357static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum)
@@ -3468,7 +3379,7 @@ static inline int propagate_entity_load_avg(struct sched_entity *se) · 21 unmodified lines
3468 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum);3379 add_tg_cfs_propagate(cfs_rq, gcfs_rq->prop_runnable_sum);
3469 3380
3470 update_tg_cfs_util(cfs_rq, se, gcfs_rq);3381 update_tg_cfs_util(cfs_rq, se, gcfs_rq);
3471 update_tg_cfs_runnable(cfs_rq, se, gcfs_rq);3382 update_tg_cfs_load(cfs_rq, se, gcfs_rq);
3472 3383
3473 trace_pelt_cfs_tp(cfs_rq);3384 trace_pelt_cfs_tp(cfs_rq);
3474 trace_pelt_se_tp(se);3385 trace_pelt_se_tp(se);
@@ -3612,8 +3523,6 @@ static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s · 137 unmodified lines
3612 div_u64(se->avg.load_avg * se->avg.load_sum, se_weight(se));3523 div_u64(se->avg.load_avg * se->avg.load_sum, se_weight(se));
3613 }3524 }
3614 3525
3615 se->avg.runnable_load_sum = se->avg.load_sum;
3616
3617 enqueue_load_avg(cfs_rq, se);3526 enqueue_load_avg(cfs_rq, se);
3618 cfs_rq->avg.util_avg += se->avg.util_avg;3527 cfs_rq->avg.util_avg += se->avg.util_avg;
3619 cfs_rq->avg.util_sum += se->avg.util_sum;3528 cfs_rq->avg.util_sum += se->avg.util_sum;
@@ -4074,14 +3983,12 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) · 454 unmodified lines
4074 /*3983 /*
4075 * When enqueuing a sched_entity, we must:3984 * When enqueuing a sched_entity, we must:
4076 * - Update loads to have both entity and cfs_rq synced with now.3985 * - Update loads to have both entity and cfs_rq synced with now.
4077 * - Add its load to cfs_rq->runnable_avg
4078 * - For group_entity, update its weight to reflect the new share of3986 * - For group_entity, update its weight to reflect the new share of
4079 * its group cfs_rq3987 * its group cfs_rq
4080 * - Add its new weight to cfs_rq->load.weight3988 * - Add its new weight to cfs_rq->load.weight
4081 */3989 */
4082 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH);3990 update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH);
4083 update_cfs_group(se);3991 update_cfs_group(se);
4084 enqueue_runnable_load_avg(cfs_rq, se);
4085 account_entity_enqueue(cfs_rq, se);3992 account_entity_enqueue(cfs_rq, se);
4086 3993
4087 if (flags & ENQUEUE_WAKEUP)3994 if (flags & ENQUEUE_WAKEUP)
@@ -4158,13 +4065,11 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) · 70 unmodified lines
4158 /*4065 /*
4159 * When dequeuing a sched_entity, we must:4066 * When dequeuing a sched_entity, we must:
4160 * - Update loads to have both entity and cfs_rq synced with now.4067 * - Update loads to have both entity and cfs_rq synced with now.
4161 * - Subtract its load from the cfs_rq->runnable_avg.
4162 * - Subtract its previous weight from cfs_rq->load.weight.4068 * - Subtract its previous weight from cfs_rq->load.weight.
4163 * - For group entity, update its weight to reflect the new share4069 * - For group entity, update its weight to reflect the new share
4164 * of its group cfs_rq.4070 * of its group cfs_rq.
4165 */4071 */
4166 update_load_avg(cfs_rq, se, UPDATE_TG);4072 update_load_avg(cfs_rq, se, UPDATE_TG);
4167 dequeue_runnable_load_avg(cfs_rq, se);
4168 4073
4169 update_stats_dequeue(cfs_rq, se, flags);4074 update_stats_dequeue(cfs_rq, se, flags);
4170 4075
@@ -7649,9 +7554,6 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) · 3478 unmodified lines
7649 if (cfs_rq->avg.util_sum)7554 if (cfs_rq->avg.util_sum)
7650 return false;7555 return false;
7651 7556
7652 if (cfs_rq->avg.runnable_load_sum)
7653 return false;
7654
7655 return true;7557 return true;
7656}7558}
7657 7559
kernel/sched/pelt.c+24−38
@@ -108,7 +108,7 @@ static u32 __accumulate_pelt_segments(u64 periods, u32 d1, u32 d3) · 107 unmodified lines
108 */108 */
109static __always_inline u32109static __always_inline u32
110accumulate_sum(u64 delta, struct sched_avg *sa,110accumulate_sum(u64 delta, struct sched_avg *sa,
111 unsigned long load, unsigned long runnable, int running)111 unsigned long load, int running)
112{112{
113 u32 contrib = (u32)delta; /* p == 0 -> delta < 1024 */113 u32 contrib = (u32)delta; /* p == 0 -> delta < 1024 */
114 u64 periods;114 u64 periods;
@@ -121,8 +121,6 @@ accumulate_sum(u64 delta, struct sched_avg *sa, · 6 unmodified lines
121 */121 */
122 if (periods) {122 if (periods) {
123 sa->load_sum = decay_load(sa->load_sum, periods);123 sa->load_sum = decay_load(sa->load_sum, periods);
124 sa->runnable_load_sum =
125 decay_load(sa->runnable_load_sum, periods);
126 sa->util_sum = decay_load((u64)(sa->util_sum), periods);124 sa->util_sum = decay_load((u64)(sa->util_sum), periods);
127 125
128 /*126 /*
@@ -148,8 +146,6 @@ accumulate_sum(u64 delta, struct sched_avg *sa, · 19 unmodified lines
148 146
149 if (load)147 if (load)
150 sa->load_sum += load * contrib;148 sa->load_sum += load * contrib;
151 if (runnable)
152 sa->runnable_load_sum += runnable * contrib;
153 if (running)149 if (running)
154 sa->util_sum += contrib << SCHED_CAPACITY_SHIFT;150 sa->util_sum += contrib << SCHED_CAPACITY_SHIFT;
155 151
@@ -186,7 +182,7 @@ accumulate_sum(u64 delta, struct sched_avg *sa, · 30 unmodified lines
186 */182 */
187static __always_inline int183static __always_inline int
188___update_load_sum(u64 now, struct sched_avg *sa,184___update_load_sum(u64 now, struct sched_avg *sa,
189 unsigned long load, unsigned long runnable, int running)185 unsigned long load, int running)
190{186{
191 u64 delta;187 u64 delta;
192 188
@@ -222,7 +218,7 @@ ___update_load_sum(u64 now, struct sched_avg *sa, · 29 unmodified lines
222 * Also see the comment in accumulate_sum().218 * Also see the comment in accumulate_sum().
223 */219 */
224 if (!load)220 if (!load)
225 runnable = running = 0;221 running = 0;
226 222
227 /*223 /*
228 * Now we know we crossed measurement unit boundaries. The *_avg224 * Now we know we crossed measurement unit boundaries. The *_avg
@@ -231,14 +227,14 @@ ___update_load_sum(u64 now, struct sched_avg *sa, · 2 unmodified lines
231 * Step 1: accumulate *_sum since last_update_time. If we haven't227 * Step 1: accumulate *_sum since last_update_time. If we haven't
232 * crossed period boundaries, finish.228 * crossed period boundaries, finish.
233 */229 */
234 if (!accumulate_sum(delta, sa, load, runnable, running))230 if (!accumulate_sum(delta, sa, load, running))
235 return 0;231 return 0;
236 232
237 return 1;233 return 1;
238}234}
239 235
240static __always_inline void236static __always_inline void
241___update_load_avg(struct sched_avg *sa, unsigned long load, unsigned long runnable)237___update_load_avg(struct sched_avg *sa, unsigned long load)
242{238{
243 u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib;239 u32 divider = LOAD_AVG_MAX - 1024 + sa->period_contrib;
244 240
@@ -246,7 +242,6 @@ ___update_load_avg(struct sched_avg *sa, unsigned long load, unsigned long runna · 1 unmodified line
246 * Step 2: update *_avg.242 * Step 2: update *_avg.
247 */243 */
248 sa->load_avg = div_u64(load * sa->load_sum, divider);244 sa->load_avg = div_u64(load * sa->load_sum, divider);
249 sa->runnable_load_avg = div_u64(runnable * sa->runnable_load_sum, divider);
250 WRITE_ONCE(sa->util_avg, sa->util_sum / divider);245 WRITE_ONCE(sa->util_avg, sa->util_sum / divider);
251}246}
252 247
@@ -254,17 +249,13 @@ ___update_load_avg(struct sched_avg *sa, unsigned long load, unsigned long runna · 1 unmodified line
254 * sched_entity:249 * sched_entity:
255 *250 *
256 * task:251 * task:
257 * se_runnable() == se_weight()252 * se_weight() = se->load.weight
258 *253 *
259 * group: [ see update_cfs_group() ]254 * group: [ see update_cfs_group() ]
260 * se_weight() = tg->weight * grq->load_avg / tg->load_avg255 * se_weight() = tg->weight * grq->load_avg / tg->load_avg
261 * se_runnable() = se_weight(se) * grq->runnable_load_avg / grq->load_avg
262 *256 *
263 * load_sum := runnable_sum257 * load_sum := runnable
264 * load_avg = se_weight(se) * runnable_avg258 * load_avg = se_weight(se) * load_sum
265 *
266 * runnable_load_sum := runnable_sum
267 * runnable_load_avg = se_runnable(se) * runnable_avg
268 *259 *
269 * XXX collapse load_sum and runnable_load_sum260 * XXX collapse load_sum and runnable_load_sum
270 *261 *
@@ -272,15 +263,12 @@ ___update_load_avg(struct sched_avg *sa, unsigned long load, unsigned long runna · 1 unmodified line
272 *263 *
273 * load_sum = \Sum se_weight(se) * se->avg.load_sum264 * load_sum = \Sum se_weight(se) * se->avg.load_sum
274 * load_avg = \Sum se->avg.load_avg265 * load_avg = \Sum se->avg.load_avg
275 *
276 * runnable_load_sum = \Sum se_runnable(se) * se->avg.runnable_load_sum
277 * runnable_load_avg = \Sum se->avg.runable_load_avg
278 */266 */
279 267
280int __update_load_avg_blocked_se(u64 now, struct sched_entity *se)268int __update_load_avg_blocked_se(u64 now, struct sched_entity *se)
281{269{
282 if (___update_load_sum(now, &se->avg, 0, 0, 0)) {270 if (___update_load_sum(now, &se->avg, 0, 0)) {
283 ___update_load_avg(&se->avg, se_weight(se), se_runnable(se));271 ___update_load_avg(&se->avg, se_weight(se));
284 trace_pelt_se_tp(se);272 trace_pelt_se_tp(se);
285 return 1;273 return 1;
286 }274 }
@@ -290,10 +278,9 @@ int __update_load_avg_blocked_se(u64 now, struct sched_entity *se) · 3 unmodified lines
290 278
291int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se)279int __update_load_avg_se(u64 now, struct cfs_rq *cfs_rq, struct sched_entity *se)
292{280{
293 if (___update_load_sum(now, &se->avg, !!se->on_rq, !!se->on_rq,281 if (___update_load_sum(now, &se->avg, !!se->on_rq, cfs_rq->curr == se)) {
294 cfs_rq->curr == se)) {
295 282
296 ___update_load_avg(&se->avg, se_weight(se), se_runnable(se));283 ___update_load_avg(&se->avg, se_weight(se));
297 cfs_se_util_change(&se->avg);284 cfs_se_util_change(&se->avg);
298 trace_pelt_se_tp(se);285 trace_pelt_se_tp(se);
299 return 1;286 return 1;
@@ -306,10 +293,9 @@ int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq) · 6 unmodified lines
306{293{
307 if (___update_load_sum(now, &cfs_rq->avg,294 if (___update_load_sum(now, &cfs_rq->avg,
308 scale_load_down(cfs_rq->load.weight),295 scale_load_down(cfs_rq->load.weight),
309 scale_load_down(cfs_rq->runnable_weight),
310 cfs_rq->curr != NULL)) {296 cfs_rq->curr != NULL)) {
311 297
312 ___update_load_avg(&cfs_rq->avg, 1, 1);298 ___update_load_avg(&cfs_rq->avg, 1);
313 trace_pelt_cfs_tp(cfs_rq);299 trace_pelt_cfs_tp(cfs_rq);
314 return 1;300 return 1;
315 }301 }
@@ -322,20 +308,19 @@ int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq) · 6 unmodified lines
322 *308 *
323 * util_sum = \Sum se->avg.util_sum but se->avg.util_sum is not tracked309 * util_sum = \Sum se->avg.util_sum but se->avg.util_sum is not tracked
324 * util_sum = cpu_scale * load_sum310 * util_sum = cpu_scale * load_sum
325 * runnable_load_sum = load_sum311 * runnable_sum = util_sum
326 *312 *
327 * load_avg and runnable_load_avg are not supported and meaningless.313 * load_avg is not supported and meaningless.
328 *314 *
329 */315 */
330 316
331int update_rt_rq_load_avg(u64 now, struct rq *rq, int running)317int update_rt_rq_load_avg(u64 now, struct rq *rq, int running)
332{318{
333 if (___update_load_sum(now, &rq->avg_rt,319 if (___update_load_sum(now, &rq->avg_rt,
334 running,
335 running,320 running,
336 running)) {321 running)) {
337 322
338 ___update_load_avg(&rq->avg_rt, 1, 1);323 ___update_load_avg(&rq->avg_rt, 1);
339 trace_pelt_rt_tp(rq);324 trace_pelt_rt_tp(rq);
340 return 1;325 return 1;
341 }326 }
@@ -348,18 +333,19 @@ int update_rt_rq_load_avg(u64 now, struct rq *rq, int running) · 6 unmodified lines
348 *333 *
349 * util_sum = \Sum se->avg.util_sum but se->avg.util_sum is not tracked334 * util_sum = \Sum se->avg.util_sum but se->avg.util_sum is not tracked
350 * util_sum = cpu_scale * load_sum335 * util_sum = cpu_scale * load_sum
351 * runnable_load_sum = load_sum336 * runnable_sum = util_sum
337 *
338 * load_avg is not supported and meaningless.
352 *339 *
353 */340 */
354 341
355int update_dl_rq_load_avg(u64 now, struct rq *rq, int running)342int update_dl_rq_load_avg(u64 now, struct rq *rq, int running)
356{343{
357 if (___update_load_sum(now, &rq->avg_dl,344 if (___update_load_sum(now, &rq->avg_dl,
358 running,
359 running,345 running,
360 running)) {346 running)) {
361 347
362 ___update_load_avg(&rq->avg_dl, 1, 1);348 ___update_load_avg(&rq->avg_dl, 1);
363 trace_pelt_dl_tp(rq);349 trace_pelt_dl_tp(rq);
364 return 1;350 return 1;
365 }351 }
@@ -373,7 +359,9 @@ int update_dl_rq_load_avg(u64 now, struct rq *rq, int running) · 7 unmodified lines
373 *359 *
374 * util_sum = \Sum se->avg.util_sum but se->avg.util_sum is not tracked360 * util_sum = \Sum se->avg.util_sum but se->avg.util_sum is not tracked
375 * util_sum = cpu_scale * load_sum361 * util_sum = cpu_scale * load_sum
376 * runnable_load_sum = load_sum362 * runnable_sum = util_sum
363 *
364 * load_avg is not supported and meaningless.
377 *365 *
378 */366 */
379 367
@@ -401,16 +389,14 @@ int update_irq_load_avg(struct rq *rq, u64 running) · 21 unmodified lines
401 * rq->clock += delta with delta >= running389 * rq->clock += delta with delta >= running
402 */390 */
403 ret = ___update_load_sum(rq->clock - running, &rq->avg_irq,391 ret = ___update_load_sum(rq->clock - running, &rq->avg_irq,
404 0,
405 0,392 0,
406 0);393 0);
407 ret += ___update_load_sum(rq->clock, &rq->avg_irq,394 ret += ___update_load_sum(rq->clock, &rq->avg_irq,
408 1,
409 1,395 1,
410 1);396 1);
411 397
412 if (ret) {398 if (ret) {
413 ___update_load_avg(&rq->avg_irq, 1, 1);399 ___update_load_avg(&rq->avg_irq, 1);
414 trace_pelt_irq_tp(rq);400 trace_pelt_irq_tp(rq);
415 }401 }
416 402
kernel/sched/sched.h+2−5
@@ -489,7 +489,6 @@ struct cfs_bandwidth { }; · 488 unmodified lines
489/* CFS-related fields in a runqueue */489/* CFS-related fields in a runqueue */
490struct cfs_rq {490struct cfs_rq {
491 struct load_weight load;491 struct load_weight load;
492 unsigned long runnable_weight;
493 unsigned int nr_running;492 unsigned int nr_running;
494 unsigned int h_nr_running; /* SCHED_{NORMAL,BATCH,IDLE} */493 unsigned int h_nr_running; /* SCHED_{NORMAL,BATCH,IDLE} */
495 unsigned int idle_h_nr_running; /* SCHED_IDLE */494 unsigned int idle_h_nr_running; /* SCHED_IDLE */
@@ -688,8 +687,10 @@ struct dl_rq { · 192 unmodified lines
688#ifdef CONFIG_FAIR_GROUP_SCHED687#ifdef CONFIG_FAIR_GROUP_SCHED
689/* An entity is a task if it doesn't "own" a runqueue */688/* An entity is a task if it doesn't "own" a runqueue */
690#define entity_is_task(se) (!se->my_q)689#define entity_is_task(se) (!se->my_q)
690
691#else691#else
692#define entity_is_task(se) 1692#define entity_is_task(se) 1
693
693#endif694#endif
694 695
695#ifdef CONFIG_SMP696#ifdef CONFIG_SMP
@@ -701,10 +702,6 @@ static inline long se_weight(struct sched_entity *se) · 5 unmodified lines
701 return scale_load_down(se->load.weight);702 return scale_load_down(se->load.weight);
702}703}
703 704
704static inline long se_runnable(struct sched_entity *se)
705{
706 return scale_load_down(se->runnable_weight);
707}
708 705
709static inline bool sched_asym_prefer(int a, int b)706static inline bool sched_asym_prefer(int a, int b)
710{707{