History

Merge tag 'sched_urgent_for_v6.13_rc3-p2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler fixes from Borislav Petkov: - Prevent incorrect dequeueing of the deadline dlserver helper task and fix its time accounting - Properly track the CFS runqueue runnable stats - Check the total number of all queued tasks in a sched fair's runqueue hierarchy before deciding to stop the tick - Fix the scheduling of the task that got woken last (NEXT_BUDDY) by preventing those from being delayed * tag 'sched_urgent_for_v6.13_rc3-p2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched/dlserver: Fix dlserver time accounting sched/dlserver: Fix dlserver double enqueue sched/eevdf: More PELT vs DELAYED_DEQUEUE sched/fair: Fix sched_can_stop_tick() for fair tasks sched/fair: Fix NEXT_BUDDY

Linus Torvalds committed 1 year ago

Commit acd855a2 parents 81576a9 c7f7e9c7 files · +84 −22Browse files

Unified · Split

7 files
  1. include/linux/sched.h +7 −0
  2. kernel/sched/core.c +1 −1
  3. kernel/sched/deadline.c +6 −2
  4. kernel/sched/debug.c +1 −0
  5. kernel/sched/fair.c +57 −16
  6. kernel/sched/pelt.c +1 −1
  7. kernel/sched/sched.h +11 −2
include/linux/sched.h+7−0
@@ -656,6 +656,12 @@ struct sched_dl_entity { · 655 unmodified lines
656 * @dl_defer_armed tells if the deferrable server is waiting656 * @dl_defer_armed tells if the deferrable server is waiting
657 * for the replenishment timer to activate it.657 * for the replenishment timer to activate it.
658 *658 *
659 * @dl_server_active tells if the dlserver is active(started).
660 * dlserver is started on first cfs enqueue on an idle runqueue
661 * and is stopped when a dequeue results in 0 cfs tasks on the
662 * runqueue. In other words, dlserver is active only when cpu's
663 * runqueue has atleast one cfs task.
664 *
659 * @dl_defer_running tells if the deferrable server is actually665 * @dl_defer_running tells if the deferrable server is actually
660 * running, skipping the defer phase.666 * running, skipping the defer phase.
661 */667 */
@@ -664,6 +670,7 @@ struct sched_dl_entity { · 2 unmodified lines
664 unsigned int dl_non_contending : 1;670 unsigned int dl_non_contending : 1;
665 unsigned int dl_overrun : 1;671 unsigned int dl_overrun : 1;
666 unsigned int dl_server : 1;672 unsigned int dl_server : 1;
673 unsigned int dl_server_active : 1;
667 unsigned int dl_defer : 1;674 unsigned int dl_defer : 1;
668 unsigned int dl_defer_armed : 1;675 unsigned int dl_defer_armed : 1;
669 unsigned int dl_defer_running : 1;676 unsigned int dl_defer_running : 1;
kernel/sched/core.c+1−1
@@ -1341,7 +1341,7 @@ bool sched_can_stop_tick(struct rq *rq) · 1340 unmodified lines
1341 if (scx_enabled() && !scx_can_stop_tick(rq))1341 if (scx_enabled() && !scx_can_stop_tick(rq))
1342 return false;1342 return false;
1343 1343
1344 if (rq->cfs.nr_running > 1)1344 if (rq->cfs.h_nr_running > 1)
1345 return false;1345 return false;
1346 1346
1347 /*1347 /*
kernel/sched/deadline.c+6−2
@@ -1647,6 +1647,7 @@ void dl_server_start(struct sched_dl_entity *dl_se) · 1646 unmodified lines
1647 if (!dl_se->dl_runtime)1647 if (!dl_se->dl_runtime)
1648 return;1648 return;
1649 1649
1650 dl_se->dl_server_active = 1;
1650 enqueue_dl_entity(dl_se, ENQUEUE_WAKEUP);1651 enqueue_dl_entity(dl_se, ENQUEUE_WAKEUP);
1651 if (!dl_task(dl_se->rq->curr) || dl_entity_preempt(dl_se, &rq->curr->dl))1652 if (!dl_task(dl_se->rq->curr) || dl_entity_preempt(dl_se, &rq->curr->dl))
1652 resched_curr(dl_se->rq);1653 resched_curr(dl_se->rq);
@@ -1661,6 +1662,7 @@ void dl_server_stop(struct sched_dl_entity *dl_se) · 8 unmodified lines
1661 hrtimer_try_to_cancel(&dl_se->dl_timer);1662 hrtimer_try_to_cancel(&dl_se->dl_timer);
1662 dl_se->dl_defer_armed = 0;1663 dl_se->dl_defer_armed = 0;
1663 dl_se->dl_throttled = 0;1664 dl_se->dl_throttled = 0;
1665 dl_se->dl_server_active = 0;
1664}1666}
1665 1667
1666void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq,1668void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq,
@@ -2421,8 +2423,10 @@ again: · 754 unmodified lines
2421 if (dl_server(dl_se)) {2423 if (dl_server(dl_se)) {
2422 p = dl_se->server_pick_task(dl_se);2424 p = dl_se->server_pick_task(dl_se);
2423 if (!p) {2425 if (!p) {
2424 dl_se->dl_yielded = 1;2426 if (dl_server_active(dl_se)) {
2425 update_curr_dl_se(rq, dl_se, 0);2427 dl_se->dl_yielded = 1;
2428 update_curr_dl_se(rq, dl_se, 0);
2429 }
2426 goto again;2430 goto again;
2427 }2431 }
2428 rq->dl_server = dl_se;2432 rq->dl_server = dl_se;
kernel/sched/debug.c+1−0
@@ -845,6 +845,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) · 844 unmodified lines
845 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread", SPLIT_NS(spread));845 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread", SPLIT_NS(spread));
846 SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running);846 SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running);
847 SEQ_printf(m, " .%-30s: %d\n", "h_nr_running", cfs_rq->h_nr_running);847 SEQ_printf(m, " .%-30s: %d\n", "h_nr_running", cfs_rq->h_nr_running);
848 SEQ_printf(m, " .%-30s: %d\n", "h_nr_delayed", cfs_rq->h_nr_delayed);
848 SEQ_printf(m, " .%-30s: %d\n", "idle_nr_running",849 SEQ_printf(m, " .%-30s: %d\n", "idle_nr_running",
849 cfs_rq->idle_nr_running);850 cfs_rq->idle_nr_running);
850 SEQ_printf(m, " .%-30s: %d\n", "idle_h_nr_running",851 SEQ_printf(m, " .%-30s: %d\n", "idle_h_nr_running",
kernel/sched/fair.c+57−16
@@ -1159,8 +1159,6 @@ static inline void update_curr_task(struct task_struct *p, s64 delta_exec) · 1158 unmodified lines
1159 trace_sched_stat_runtime(p, delta_exec);1159 trace_sched_stat_runtime(p, delta_exec);
1160 account_group_exec_runtime(p, delta_exec);1160 account_group_exec_runtime(p, delta_exec);
1161 cgroup_account_cputime(p, delta_exec);1161 cgroup_account_cputime(p, delta_exec);
1162 if (p->dl_server)
1163 dl_server_update(p->dl_server, delta_exec);
1164}1162}
1165 1163
1166static inline bool did_preempt_short(struct cfs_rq *cfs_rq, struct sched_entity *curr)1164static inline bool did_preempt_short(struct cfs_rq *cfs_rq, struct sched_entity *curr)
@@ -1237,11 +1235,16 @@ static void update_curr(struct cfs_rq *cfs_rq) · 70 unmodified lines
1237 update_curr_task(p, delta_exec);1235 update_curr_task(p, delta_exec);
1238 1236
1239 /*1237 /*
1240 * Any fair task that runs outside of fair_server should1238 * If the fair_server is active, we need to account for the
1241 * account against fair_server such that it can account for1239 * fair_server time whether or not the task is running on
1242 * this time and possibly avoid running this period.1240 * behalf of fair_server or not:
1241 * - If the task is running on behalf of fair_server, we need
1242 * to limit its time based on the assigned runtime.
1243 * - Fair task that runs outside of fair_server should account
1244 * against fair_server such that it can account for this time
1245 * and possibly avoid running this period.
1243 */1246 */
1244 if (p->dl_server != &rq->fair_server)1247 if (dl_server_active(&rq->fair_server))
1245 dl_server_update(&rq->fair_server, delta_exec);1248 dl_server_update(&rq->fair_server, delta_exec);
1246 }1249 }
1247 1250
@@ -5471,9 +5474,33 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) · 4223 unmodified lines
5471 5474
5472static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);5475static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
5473 5476
5474static inline void finish_delayed_dequeue_entity(struct sched_entity *se)5477static void set_delayed(struct sched_entity *se)
5478{
5479 se->sched_delayed = 1;
5480 for_each_sched_entity(se) {
5481 struct cfs_rq *cfs_rq = cfs_rq_of(se);
5482
5483 cfs_rq->h_nr_delayed++;
5484 if (cfs_rq_throttled(cfs_rq))
5485 break;
5486 }
5487}
5488
5489static void clear_delayed(struct sched_entity *se)
5475{5490{
5476 se->sched_delayed = 0;5491 se->sched_delayed = 0;
5492 for_each_sched_entity(se) {
5493 struct cfs_rq *cfs_rq = cfs_rq_of(se);
5494
5495 cfs_rq->h_nr_delayed--;
5496 if (cfs_rq_throttled(cfs_rq))
5497 break;
5498 }
5499}
5500
5501static inline void finish_delayed_dequeue_entity(struct sched_entity *se)
5502{
5503 clear_delayed(se);
5477 if (sched_feat(DELAY_ZERO) && se->vlag > 0)5504 if (sched_feat(DELAY_ZERO) && se->vlag > 0)
5478 se->vlag = 0;5505 se->vlag = 0;
5479}5506}
@@ -5484,6 +5511,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) · 4 unmodified lines
5484 bool sleep = flags & DEQUEUE_SLEEP;5511 bool sleep = flags & DEQUEUE_SLEEP;
5485 5512
5486 update_curr(cfs_rq);5513 update_curr(cfs_rq);
5514 clear_buddies(cfs_rq, se);
5487 5515
5488 if (flags & DEQUEUE_DELAYED) {5516 if (flags & DEQUEUE_DELAYED) {
5489 SCHED_WARN_ON(!se->sched_delayed);5517 SCHED_WARN_ON(!se->sched_delayed);
@@ -5500,10 +5528,8 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) · 10 unmodified lines
5500 5528
5501 if (sched_feat(DELAY_DEQUEUE) && delay &&5529 if (sched_feat(DELAY_DEQUEUE) && delay &&
5502 !entity_eligible(cfs_rq, se)) {5530 !entity_eligible(cfs_rq, se)) {
5503 if (cfs_rq->next == se)
5504 cfs_rq->next = NULL;
5505 update_load_avg(cfs_rq, se, 0);5531 update_load_avg(cfs_rq, se, 0);
5506 se->sched_delayed = 1;5532 set_delayed(se);
5507 return false;5533 return false;
5508 }5534 }
5509 }5535 }
@@ -5526,8 +5552,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) · 16 unmodified lines
5526 5552
5527 update_stats_dequeue_fair(cfs_rq, se, flags);5553 update_stats_dequeue_fair(cfs_rq, se, flags);
5528 5554
5529 clear_buddies(cfs_rq, se);
5530
5531 update_entity_lag(cfs_rq, se);5555 update_entity_lag(cfs_rq, se);
5532 if (sched_feat(PLACE_REL_DEADLINE) && !sleep) {5556 if (sched_feat(PLACE_REL_DEADLINE) && !sleep) {
5533 se->deadline -= se->vruntime;5557 se->deadline -= se->vruntime;
@@ -5917,7 +5941,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 383 unmodified lines
5917 struct rq *rq = rq_of(cfs_rq);5941 struct rq *rq = rq_of(cfs_rq);
5918 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);5942 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
5919 struct sched_entity *se;5943 struct sched_entity *se;
5920 long task_delta, idle_task_delta, dequeue = 1;5944 long task_delta, idle_task_delta, delayed_delta, dequeue = 1;
5921 long rq_h_nr_running = rq->cfs.h_nr_running;5945 long rq_h_nr_running = rq->cfs.h_nr_running;
5922 5946
5923 raw_spin_lock(&cfs_b->lock);5947 raw_spin_lock(&cfs_b->lock);
@@ -5950,6 +5974,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 26 unmodified lines
5950 5974
5951 task_delta = cfs_rq->h_nr_running;5975 task_delta = cfs_rq->h_nr_running;
5952 idle_task_delta = cfs_rq->idle_h_nr_running;5976 idle_task_delta = cfs_rq->idle_h_nr_running;
5977 delayed_delta = cfs_rq->h_nr_delayed;
5953 for_each_sched_entity(se) {5978 for_each_sched_entity(se) {
5954 struct cfs_rq *qcfs_rq = cfs_rq_of(se);5979 struct cfs_rq *qcfs_rq = cfs_rq_of(se);
5955 int flags;5980 int flags;
@@ -5973,6 +5998,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 17 unmodified lines
5973 5998
5974 qcfs_rq->h_nr_running -= task_delta;5999 qcfs_rq->h_nr_running -= task_delta;
5975 qcfs_rq->idle_h_nr_running -= idle_task_delta;6000 qcfs_rq->idle_h_nr_running -= idle_task_delta;
6001 qcfs_rq->h_nr_delayed -= delayed_delta;
5976 6002
5977 if (qcfs_rq->load.weight) {6003 if (qcfs_rq->load.weight) {
5978 /* Avoid re-evaluating load for this entity: */6004 /* Avoid re-evaluating load for this entity: */
@@ -5995,6 +6021,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 16 unmodified lines
5995 6021
5996 qcfs_rq->h_nr_running -= task_delta;6022 qcfs_rq->h_nr_running -= task_delta;
5997 qcfs_rq->idle_h_nr_running -= idle_task_delta;6023 qcfs_rq->idle_h_nr_running -= idle_task_delta;
6024 qcfs_rq->h_nr_delayed -= delayed_delta;
5998 }6025 }
5999 6026
6000 /* At this point se is NULL and we are at root level*/6027 /* At this point se is NULL and we are at root level*/
@@ -6020,7 +6047,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 19 unmodified lines
6020 struct rq *rq = rq_of(cfs_rq);6047 struct rq *rq = rq_of(cfs_rq);
6021 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);6048 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
6022 struct sched_entity *se;6049 struct sched_entity *se;
6023 long task_delta, idle_task_delta;6050 long task_delta, idle_task_delta, delayed_delta;
6024 long rq_h_nr_running = rq->cfs.h_nr_running;6051 long rq_h_nr_running = rq->cfs.h_nr_running;
6025 6052
6026 se = cfs_rq->tg->se[cpu_of(rq)];6053 se = cfs_rq->tg->se[cpu_of(rq)];
@@ -6056,6 +6083,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 29 unmodified lines
6056 6083
6057 task_delta = cfs_rq->h_nr_running;6084 task_delta = cfs_rq->h_nr_running;
6058 idle_task_delta = cfs_rq->idle_h_nr_running;6085 idle_task_delta = cfs_rq->idle_h_nr_running;
6086 delayed_delta = cfs_rq->h_nr_delayed;
6059 for_each_sched_entity(se) {6087 for_each_sched_entity(se) {
6060 struct cfs_rq *qcfs_rq = cfs_rq_of(se);6088 struct cfs_rq *qcfs_rq = cfs_rq_of(se);
6061 6089
@@ -6073,6 +6101,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 11 unmodified lines
6073 6101
6074 qcfs_rq->h_nr_running += task_delta;6102 qcfs_rq->h_nr_running += task_delta;
6075 qcfs_rq->idle_h_nr_running += idle_task_delta;6103 qcfs_rq->idle_h_nr_running += idle_task_delta;
6104 qcfs_rq->h_nr_delayed += delayed_delta;
6076 6105
6077 /* end evaluation on encountering a throttled cfs_rq */6106 /* end evaluation on encountering a throttled cfs_rq */
6078 if (cfs_rq_throttled(qcfs_rq))6107 if (cfs_rq_throttled(qcfs_rq))
@@ -6090,6 +6119,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 11 unmodified lines
6090 6119
6091 qcfs_rq->h_nr_running += task_delta;6120 qcfs_rq->h_nr_running += task_delta;
6092 qcfs_rq->idle_h_nr_running += idle_task_delta;6121 qcfs_rq->idle_h_nr_running += idle_task_delta;
6122 qcfs_rq->h_nr_delayed += delayed_delta;
6093 6123
6094 /* end evaluation on encountering a throttled cfs_rq */6124 /* end evaluation on encountering a throttled cfs_rq */
6095 if (cfs_rq_throttled(qcfs_rq))6125 if (cfs_rq_throttled(qcfs_rq))
@@ -6943,7 +6973,7 @@ requeue_delayed_entity(struct sched_entity *se) · 847 unmodified lines
6943 }6973 }
6944 6974
6945 update_load_avg(cfs_rq, se, 0);6975 update_load_avg(cfs_rq, se, 0);
6946 se->sched_delayed = 0;6976 clear_delayed(se);
6947}6977}
6948 6978
6949/*6979/*
@@ -6957,6 +6987,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) · 7 unmodified lines
6957 struct cfs_rq *cfs_rq;6987 struct cfs_rq *cfs_rq;
6958 struct sched_entity *se = &p->se;6988 struct sched_entity *se = &p->se;
6959 int idle_h_nr_running = task_has_idle_policy(p);6989 int idle_h_nr_running = task_has_idle_policy(p);
6990 int h_nr_delayed = 0;
6960 int task_new = !(flags & ENQUEUE_WAKEUP);6991 int task_new = !(flags & ENQUEUE_WAKEUP);
6961 int rq_h_nr_running = rq->cfs.h_nr_running;6992 int rq_h_nr_running = rq->cfs.h_nr_running;
6962 u64 slice = 0;6993 u64 slice = 0;
@@ -6983,6 +7014,9 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) · 20 unmodified lines
6983 if (p->in_iowait)7014 if (p->in_iowait)
6984 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);7015 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
6985 7016
7017 if (task_new)
7018 h_nr_delayed = !!se->sched_delayed;
7019
6986 for_each_sched_entity(se) {7020 for_each_sched_entity(se) {
6987 if (se->on_rq) {7021 if (se->on_rq) {
6988 if (se->sched_delayed)7022 if (se->sched_delayed)
@@ -7005,6 +7039,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) · 16 unmodified lines
7005 7039
7006 cfs_rq->h_nr_running++;7040 cfs_rq->h_nr_running++;
7007 cfs_rq->idle_h_nr_running += idle_h_nr_running;7041 cfs_rq->idle_h_nr_running += idle_h_nr_running;
7042 cfs_rq->h_nr_delayed += h_nr_delayed;
7008 7043
7009 if (cfs_rq_is_idle(cfs_rq))7044 if (cfs_rq_is_idle(cfs_rq))
7010 idle_h_nr_running = 1;7045 idle_h_nr_running = 1;
@@ -7028,6 +7063,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) · 17 unmodified lines
7028 7063
7029 cfs_rq->h_nr_running++;7064 cfs_rq->h_nr_running++;
7030 cfs_rq->idle_h_nr_running += idle_h_nr_running;7065 cfs_rq->idle_h_nr_running += idle_h_nr_running;
7066 cfs_rq->h_nr_delayed += h_nr_delayed;
7031 7067
7032 if (cfs_rq_is_idle(cfs_rq))7068 if (cfs_rq_is_idle(cfs_rq))
7033 idle_h_nr_running = 1;7069 idle_h_nr_running = 1;
@@ -7090,6 +7126,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) · 56 unmodified lines
7090 struct task_struct *p = NULL;7126 struct task_struct *p = NULL;
7091 int idle_h_nr_running = 0;7127 int idle_h_nr_running = 0;
7092 int h_nr_running = 0;7128 int h_nr_running = 0;
7129 int h_nr_delayed = 0;
7093 struct cfs_rq *cfs_rq;7130 struct cfs_rq *cfs_rq;
7094 u64 slice = 0;7131 u64 slice = 0;
7095 7132
@@ -7097,6 +7134,8 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) · 1 unmodified line
7097 p = task_of(se);7134 p = task_of(se);
7098 h_nr_running = 1;7135 h_nr_running = 1;
7099 idle_h_nr_running = task_has_idle_policy(p);7136 idle_h_nr_running = task_has_idle_policy(p);
7137 if (!task_sleep && !task_delayed)
7138 h_nr_delayed = !!se->sched_delayed;
7100 } else {7139 } else {
7101 cfs_rq = group_cfs_rq(se);7140 cfs_rq = group_cfs_rq(se);
7102 slice = cfs_rq_min_slice(cfs_rq);7141 slice = cfs_rq_min_slice(cfs_rq);
@@ -7114,6 +7153,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) · 11 unmodified lines
7114 7153
7115 cfs_rq->h_nr_running -= h_nr_running;7154 cfs_rq->h_nr_running -= h_nr_running;
7116 cfs_rq->idle_h_nr_running -= idle_h_nr_running;7155 cfs_rq->idle_h_nr_running -= idle_h_nr_running;
7156 cfs_rq->h_nr_delayed -= h_nr_delayed;
7117 7157
7118 if (cfs_rq_is_idle(cfs_rq))7158 if (cfs_rq_is_idle(cfs_rq))
7119 idle_h_nr_running = h_nr_running;7159 idle_h_nr_running = h_nr_running;
@@ -7152,6 +7192,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) · 32 unmodified lines
7152 7192
7153 cfs_rq->h_nr_running -= h_nr_running;7193 cfs_rq->h_nr_running -= h_nr_running;
7154 cfs_rq->idle_h_nr_running -= idle_h_nr_running;7194 cfs_rq->idle_h_nr_running -= idle_h_nr_running;
7195 cfs_rq->h_nr_delayed -= h_nr_delayed;
7155 7196
7156 if (cfs_rq_is_idle(cfs_rq))7197 if (cfs_rq_is_idle(cfs_rq))
7157 idle_h_nr_running = h_nr_running;7198 idle_h_nr_running = h_nr_running;
@@ -8780,7 +8821,7 @@ static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int · 1622 unmodified lines
8780 if (unlikely(throttled_hierarchy(cfs_rq_of(pse))))8821 if (unlikely(throttled_hierarchy(cfs_rq_of(pse))))
8781 return;8822 return;
8782 8823
8783 if (sched_feat(NEXT_BUDDY) && !(wake_flags & WF_FORK)) {8824 if (sched_feat(NEXT_BUDDY) && !(wake_flags & WF_FORK) && !pse->sched_delayed) {
8784 set_next_buddy(pse);8825 set_next_buddy(pse);
8785 }8826 }
8786 8827
kernel/sched/pelt.c+1−1
@@ -321,7 +321,7 @@ int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq) · 320 unmodified lines
321{321{
322 if (___update_load_sum(now, &cfs_rq->avg,322 if (___update_load_sum(now, &cfs_rq->avg,
323 scale_load_down(cfs_rq->load.weight),323 scale_load_down(cfs_rq->load.weight),
324 cfs_rq->h_nr_running,324 cfs_rq->h_nr_running - cfs_rq->h_nr_delayed,
325 cfs_rq->curr != NULL)) {325 cfs_rq->curr != NULL)) {
326 326
327 ___update_load_avg(&cfs_rq->avg, 1);327 ___update_load_avg(&cfs_rq->avg, 1);
kernel/sched/sched.h+11−2
@@ -398,6 +398,11 @@ extern void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq · 397 unmodified lines
398extern int dl_server_apply_params(struct sched_dl_entity *dl_se,398extern int dl_server_apply_params(struct sched_dl_entity *dl_se,
399 u64 runtime, u64 period, bool init);399 u64 runtime, u64 period, bool init);
400 400
401static inline bool dl_server_active(struct sched_dl_entity *dl_se)
402{
403 return dl_se->dl_server_active;
404}
405
401#ifdef CONFIG_CGROUP_SCHED406#ifdef CONFIG_CGROUP_SCHED
402 407
403extern struct list_head task_groups;408extern struct list_head task_groups;
@@ -649,6 +654,7 @@ struct cfs_rq { · 245 unmodified lines
649 unsigned int h_nr_running; /* SCHED_{NORMAL,BATCH,IDLE} */654 unsigned int h_nr_running; /* SCHED_{NORMAL,BATCH,IDLE} */
650 unsigned int idle_nr_running; /* SCHED_IDLE */655 unsigned int idle_nr_running; /* SCHED_IDLE */
651 unsigned int idle_h_nr_running; /* SCHED_IDLE */656 unsigned int idle_h_nr_running; /* SCHED_IDLE */
657 unsigned int h_nr_delayed;
652 658
653 s64 avg_vruntime;659 s64 avg_vruntime;
654 u64 avg_load;660 u64 avg_load;
@@ -898,8 +904,11 @@ struct dl_rq { · 243 unmodified lines
898 904
899static inline void se_update_runnable(struct sched_entity *se)905static inline void se_update_runnable(struct sched_entity *se)
900{906{
901 if (!entity_is_task(se))907 if (!entity_is_task(se)) {
902 se->runnable_weight = se->my_q->h_nr_running;908 struct cfs_rq *cfs_rq = se->my_q;
909
910 se->runnable_weight = cfs_rq->h_nr_running - cfs_rq->h_nr_delayed;
911 }
903}912}
904 913
905static inline long se_runnable(struct sched_entity *se)914static inline long se_runnable(struct sched_entity *se)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t