History

Merge tag 'sched_urgent_for_v6.13_rc3-p2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull scheduler fixes from Borislav Petkov: - Prevent incorrect dequeueing of the deadline dlserver helper task and fix its time accounting - Properly track the CFS runqueue runnable stats - Check the total number of all queued tasks in a sched fair's runqueue hierarchy before deciding to stop the tick - Fix the scheduling of the task that got woken last (NEXT_BUDDY) by preventing those from being delayed * tag 'sched_urgent_for_v6.13_rc3-p2' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: sched/dlserver: Fix dlserver time accounting sched/dlserver: Fix dlserver double enqueue sched/eevdf: More PELT vs DELAYED_DEQUEUE sched/fair: Fix sched_can_stop_tick() for fair tasks sched/fair: Fix NEXT_BUDDY

Linus Torvalds committed 1 year ago

Commit acd855a2 parents 81576a9 c7f7e9c7 files · +84 −22Browse files

Unified · Split

7 files
  1. include/linux/sched.h +7 −0
  2. kernel/sched/core.c +1 −1
  3. kernel/sched/deadline.c +6 −2
  4. kernel/sched/debug.c +1 −0
  5. kernel/sched/fair.c +57 −16
  6. kernel/sched/pelt.c +1 −1
  7. kernel/sched/sched.h +11 −2
include/linux/sched.h+7−0
@@ -656,6 +656,12 @@ struct sched_dl_entity { · 655 unmodified lines
656656 * @dl_defer_armed tells if the deferrable server is waiting
657657 * for the replenishment timer to activate it.
658658 *
659+ * @dl_server_active tells if the dlserver is active(started).
660+ * dlserver is started on first cfs enqueue on an idle runqueue
661+ * and is stopped when a dequeue results in 0 cfs tasks on the
662+ * runqueue. In other words, dlserver is active only when cpu's
663+ * runqueue has atleast one cfs task.
664+ *
659665 * @dl_defer_running tells if the deferrable server is actually
660666 * running, skipping the defer phase.
661667 */
@@ -664,6 +670,7 @@ struct sched_dl_entity { · 2 unmodified lines
664670 unsigned int dl_non_contending : 1;
665671 unsigned int dl_overrun : 1;
666672 unsigned int dl_server : 1;
673+ unsigned int dl_server_active : 1;
667674 unsigned int dl_defer : 1;
668675 unsigned int dl_defer_armed : 1;
669676 unsigned int dl_defer_running : 1;
kernel/sched/core.c+1−1
@@ -1341,7 +1341,7 @@ bool sched_can_stop_tick(struct rq *rq) · 1340 unmodified lines
13411341 if (scx_enabled() && !scx_can_stop_tick(rq))
13421342 return false;
13431343
1344− if (rq->cfs.nr_running > 1)
1344+ if (rq->cfs.h_nr_running > 1)
13451345 return false;
13461346
13471347 /*
kernel/sched/deadline.c+6−2
@@ -1647,6 +1647,7 @@ void dl_server_start(struct sched_dl_entity *dl_se) · 1646 unmodified lines
16471647 if (!dl_se->dl_runtime)
16481648 return;
16491649
1650+ dl_se->dl_server_active = 1;
16501651 enqueue_dl_entity(dl_se, ENQUEUE_WAKEUP);
16511652 if (!dl_task(dl_se->rq->curr) || dl_entity_preempt(dl_se, &rq->curr->dl))
16521653 resched_curr(dl_se->rq);
@@ -1661,6 +1662,7 @@ void dl_server_stop(struct sched_dl_entity *dl_se) · 8 unmodified lines
16611662 hrtimer_try_to_cancel(&dl_se->dl_timer);
16621663 dl_se->dl_defer_armed = 0;
16631664 dl_se->dl_throttled = 0;
1665+ dl_se->dl_server_active = 0;
16641666 }
16651667
16661668 void dl_server_init(struct sched_dl_entity *dl_se, struct rq *rq,
@@ -2421,8 +2423,10 @@ again: · 754 unmodified lines
24212423 if (dl_server(dl_se)) {
24222424 p = dl_se->server_pick_task(dl_se);
24232425 if (!p) {
2424− dl_se->dl_yielded = 1;
2425− update_curr_dl_se(rq, dl_se, 0);
2426+ if (dl_server_active(dl_se)) {
2427+ dl_se->dl_yielded = 1;
2428+ update_curr_dl_se(rq, dl_se, 0);
2429+ }
24262430 goto again;
24272431 }
24282432 rq->dl_server = dl_se;
kernel/sched/debug.c+1−0
@@ -845,6 +845,7 @@ void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) · 844 unmodified lines
845845 SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread", SPLIT_NS(spread));
846846 SEQ_printf(m, " .%-30s: %d\n", "nr_running", cfs_rq->nr_running);
847847 SEQ_printf(m, " .%-30s: %d\n", "h_nr_running", cfs_rq->h_nr_running);
848+ SEQ_printf(m, " .%-30s: %d\n", "h_nr_delayed", cfs_rq->h_nr_delayed);
848849 SEQ_printf(m, " .%-30s: %d\n", "idle_nr_running",
849850 cfs_rq->idle_nr_running);
850851 SEQ_printf(m, " .%-30s: %d\n", "idle_h_nr_running",
kernel/sched/fair.c+57−16
@@ -1159,8 +1159,6 @@ static inline void update_curr_task(struct task_struct *p, s64 delta_exec) · 1158 unmodified lines
11591159 trace_sched_stat_runtime(p, delta_exec);
11601160 account_group_exec_runtime(p, delta_exec);
11611161 cgroup_account_cputime(p, delta_exec);
1162− if (p->dl_server)
1163− dl_server_update(p->dl_server, delta_exec);
11641162 }
11651163
11661164 static inline bool did_preempt_short(struct cfs_rq *cfs_rq, struct sched_entity *curr)
@@ -1237,11 +1235,16 @@ static void update_curr(struct cfs_rq *cfs_rq) · 70 unmodified lines
12371235 update_curr_task(p, delta_exec);
12381236
12391237 /*
1240− * Any fair task that runs outside of fair_server should
1241− * account against fair_server such that it can account for
1242− * this time and possibly avoid running this period.
1238+ * If the fair_server is active, we need to account for the
1239+ * fair_server time whether or not the task is running on
1240+ * behalf of fair_server or not:
1241+ * - If the task is running on behalf of fair_server, we need
1242+ * to limit its time based on the assigned runtime.
1243+ * - Fair task that runs outside of fair_server should account
1244+ * against fair_server such that it can account for this time
1245+ * and possibly avoid running this period.
12431246 */
1244− if (p->dl_server != &rq->fair_server)
1247+ if (dl_server_active(&rq->fair_server))
12451248 dl_server_update(&rq->fair_server, delta_exec);
12461249 }
12471250
@@ -5471,9 +5474,33 @@ static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se) · 4223 unmodified lines
54715474
54725475 static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
54735476
5474−static inline void finish_delayed_dequeue_entity(struct sched_entity *se)
5477+static void set_delayed(struct sched_entity *se)
5478+{
5479+ se->sched_delayed = 1;
5480+ for_each_sched_entity(se) {
5481+ struct cfs_rq *cfs_rq = cfs_rq_of(se);
5482+
5483+ cfs_rq->h_nr_delayed++;
5484+ if (cfs_rq_throttled(cfs_rq))
5485+ break;
5486+ }
5487+}
5488+
5489+static void clear_delayed(struct sched_entity *se)
54755490 {
54765491 se->sched_delayed = 0;
5492+ for_each_sched_entity(se) {
5493+ struct cfs_rq *cfs_rq = cfs_rq_of(se);
5494+
5495+ cfs_rq->h_nr_delayed--;
5496+ if (cfs_rq_throttled(cfs_rq))
5497+ break;
5498+ }
5499+}
5500+
5501+static inline void finish_delayed_dequeue_entity(struct sched_entity *se)
5502+{
5503+ clear_delayed(se);
54775504 if (sched_feat(DELAY_ZERO) && se->vlag > 0)
54785505 se->vlag = 0;
54795506 }
@@ -5484,6 +5511,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) · 4 unmodified lines
54845511 bool sleep = flags & DEQUEUE_SLEEP;
54855512
54865513 update_curr(cfs_rq);
5514+ clear_buddies(cfs_rq, se);
54875515
54885516 if (flags & DEQUEUE_DELAYED) {
54895517 SCHED_WARN_ON(!se->sched_delayed);
@@ -5500,10 +5528,8 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) · 10 unmodified lines
55005528
55015529 if (sched_feat(DELAY_DEQUEUE) && delay &&
55025530 !entity_eligible(cfs_rq, se)) {
5503− if (cfs_rq->next == se)
5504− cfs_rq->next = NULL;
55055531 update_load_avg(cfs_rq, se, 0);
5506− se->sched_delayed = 1;
5532+ set_delayed(se);
55075533 return false;
55085534 }
55095535 }
@@ -5526,8 +5552,6 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) · 16 unmodified lines
55265552
55275553 update_stats_dequeue_fair(cfs_rq, se, flags);
55285554
5529− clear_buddies(cfs_rq, se);
5530−
55315555 update_entity_lag(cfs_rq, se);
55325556 if (sched_feat(PLACE_REL_DEADLINE) && !sleep) {
55335557 se->deadline -= se->vruntime;
@@ -5917,7 +5941,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 383 unmodified lines
59175941 struct rq *rq = rq_of(cfs_rq);
59185942 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
59195943 struct sched_entity *se;
5920− long task_delta, idle_task_delta, dequeue = 1;
5944+ long task_delta, idle_task_delta, delayed_delta, dequeue = 1;
59215945 long rq_h_nr_running = rq->cfs.h_nr_running;
59225946
59235947 raw_spin_lock(&cfs_b->lock);
@@ -5950,6 +5974,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 26 unmodified lines
59505974
59515975 task_delta = cfs_rq->h_nr_running;
59525976 idle_task_delta = cfs_rq->idle_h_nr_running;
5977+ delayed_delta = cfs_rq->h_nr_delayed;
59535978 for_each_sched_entity(se) {
59545979 struct cfs_rq *qcfs_rq = cfs_rq_of(se);
59555980 int flags;
@@ -5973,6 +5998,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 17 unmodified lines
59735998
59745999 qcfs_rq->h_nr_running -= task_delta;
59756000 qcfs_rq->idle_h_nr_running -= idle_task_delta;
6001+ qcfs_rq->h_nr_delayed -= delayed_delta;
59766002
59776003 if (qcfs_rq->load.weight) {
59786004 /* Avoid re-evaluating load for this entity: */
@@ -5995,6 +6021,7 @@ static bool throttle_cfs_rq(struct cfs_rq *cfs_rq) · 16 unmodified lines
59956021
59966022 qcfs_rq->h_nr_running -= task_delta;
59976023 qcfs_rq->idle_h_nr_running -= idle_task_delta;
6024+ qcfs_rq->h_nr_delayed -= delayed_delta;
59986025 }
59996026
60006027 /* At this point se is NULL and we are at root level*/
@@ -6020,7 +6047,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 19 unmodified lines
60206047 struct rq *rq = rq_of(cfs_rq);
60216048 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
60226049 struct sched_entity *se;
6023− long task_delta, idle_task_delta;
6050+ long task_delta, idle_task_delta, delayed_delta;
60246051 long rq_h_nr_running = rq->cfs.h_nr_running;
60256052
60266053 se = cfs_rq->tg->se[cpu_of(rq)];
@@ -6056,6 +6083,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 29 unmodified lines
60566083
60576084 task_delta = cfs_rq->h_nr_running;
60586085 idle_task_delta = cfs_rq->idle_h_nr_running;
6086+ delayed_delta = cfs_rq->h_nr_delayed;
60596087 for_each_sched_entity(se) {
60606088 struct cfs_rq *qcfs_rq = cfs_rq_of(se);
60616089
@@ -6073,6 +6101,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 11 unmodified lines
60736101
60746102 qcfs_rq->h_nr_running += task_delta;
60756103 qcfs_rq->idle_h_nr_running += idle_task_delta;
6104+ qcfs_rq->h_nr_delayed += delayed_delta;
60766105
60776106 /* end evaluation on encountering a throttled cfs_rq */
60786107 if (cfs_rq_throttled(qcfs_rq))
@@ -6090,6 +6119,7 @@ void unthrottle_cfs_rq(struct cfs_rq *cfs_rq) · 11 unmodified lines
60906119
60916120 qcfs_rq->h_nr_running += task_delta;
60926121 qcfs_rq->idle_h_nr_running += idle_task_delta;
6122+ qcfs_rq->h_nr_delayed += delayed_delta;
60936123
60946124 /* end evaluation on encountering a throttled cfs_rq */
60956125 if (cfs_rq_throttled(qcfs_rq))
@@ -6943,7 +6973,7 @@ requeue_delayed_entity(struct sched_entity *se) · 847 unmodified lines
69436973 }
69446974
69456975 update_load_avg(cfs_rq, se, 0);
6946− se->sched_delayed = 0;
6976+ clear_delayed(se);
69476977 }
69486978
69496979 /*
@@ -6957,6 +6987,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) · 7 unmodified lines
69576987 struct cfs_rq *cfs_rq;
69586988 struct sched_entity *se = &p->se;
69596989 int idle_h_nr_running = task_has_idle_policy(p);
6990+ int h_nr_delayed = 0;
69606991 int task_new = !(flags & ENQUEUE_WAKEUP);
69616992 int rq_h_nr_running = rq->cfs.h_nr_running;
69626993 u64 slice = 0;
@@ -6983,6 +7014,9 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) · 20 unmodified lines
69837014 if (p->in_iowait)
69847015 cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
69857016
7017+ if (task_new)
7018+ h_nr_delayed = !!se->sched_delayed;
7019+
69867020 for_each_sched_entity(se) {
69877021 if (se->on_rq) {
69887022 if (se->sched_delayed)
@@ -7005,6 +7039,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) · 16 unmodified lines
70057039
70067040 cfs_rq->h_nr_running++;
70077041 cfs_rq->idle_h_nr_running += idle_h_nr_running;
7042+ cfs_rq->h_nr_delayed += h_nr_delayed;
70087043
70097044 if (cfs_rq_is_idle(cfs_rq))
70107045 idle_h_nr_running = 1;
@@ -7028,6 +7063,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) · 17 unmodified lines
70287063
70297064 cfs_rq->h_nr_running++;
70307065 cfs_rq->idle_h_nr_running += idle_h_nr_running;
7066+ cfs_rq->h_nr_delayed += h_nr_delayed;
70317067
70327068 if (cfs_rq_is_idle(cfs_rq))
70337069 idle_h_nr_running = 1;
@@ -7090,6 +7126,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) · 56 unmodified lines
70907126 struct task_struct *p = NULL;
70917127 int idle_h_nr_running = 0;
70927128 int h_nr_running = 0;
7129+ int h_nr_delayed = 0;
70937130 struct cfs_rq *cfs_rq;
70947131 u64 slice = 0;
70957132
@@ -7097,6 +7134,8 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) · 1 unmodified line
70977134 p = task_of(se);
70987135 h_nr_running = 1;
70997136 idle_h_nr_running = task_has_idle_policy(p);
7137+ if (!task_sleep && !task_delayed)
7138+ h_nr_delayed = !!se->sched_delayed;
71007139 } else {
71017140 cfs_rq = group_cfs_rq(se);
71027141 slice = cfs_rq_min_slice(cfs_rq);
@@ -7114,6 +7153,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) · 11 unmodified lines
71147153
71157154 cfs_rq->h_nr_running -= h_nr_running;
71167155 cfs_rq->idle_h_nr_running -= idle_h_nr_running;
7156+ cfs_rq->h_nr_delayed -= h_nr_delayed;
71177157
71187158 if (cfs_rq_is_idle(cfs_rq))
71197159 idle_h_nr_running = h_nr_running;
@@ -7152,6 +7192,7 @@ static int dequeue_entities(struct rq *rq, struct sched_entity *se, int flags) · 32 unmodified lines
71527192
71537193 cfs_rq->h_nr_running -= h_nr_running;
71547194 cfs_rq->idle_h_nr_running -= idle_h_nr_running;
7195+ cfs_rq->h_nr_delayed -= h_nr_delayed;
71557196
71567197 if (cfs_rq_is_idle(cfs_rq))
71577198 idle_h_nr_running = h_nr_running;
@@ -8780,7 +8821,7 @@ static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int · 1622 unmodified lines
87808821 if (unlikely(throttled_hierarchy(cfs_rq_of(pse))))
87818822 return;
87828823
8783− if (sched_feat(NEXT_BUDDY) && !(wake_flags & WF_FORK)) {
8824+ if (sched_feat(NEXT_BUDDY) && !(wake_flags & WF_FORK) && !pse->sched_delayed) {
87848825 set_next_buddy(pse);
87858826 }
87868827
kernel/sched/pelt.c+1−1
@@ -321,7 +321,7 @@ int __update_load_avg_cfs_rq(u64 now, struct cfs_rq *cfs_rq) · 320 unmodified lines
321321 {
322322 if (___update_load_sum(now, &cfs_rq->avg,
323323 scale_load_down(cfs_rq->load.weight),
324− cfs_rq->h_nr_running,
324+ cfs_rq->h_nr_running - cfs_rq->h_nr_delayed,
325325 cfs_rq->curr != NULL)) {
326326
327327 ___update_load_avg(&cfs_rq->avg, 1);
kernel/sched/sched.h+11−2
@@ -398,6 +398,11 @@ extern void __dl_server_attach_root(struct sched_dl_entity *dl_se, struct rq *rq · 397 unmodified lines
398398 extern int dl_server_apply_params(struct sched_dl_entity *dl_se,
399399 u64 runtime, u64 period, bool init);
400400
401+static inline bool dl_server_active(struct sched_dl_entity *dl_se)
402+{
403+ return dl_se->dl_server_active;
404+}
405+
401406 #ifdef CONFIG_CGROUP_SCHED
402407
403408 extern struct list_head task_groups;
@@ -649,6 +654,7 @@ struct cfs_rq { · 245 unmodified lines
649654 unsigned int h_nr_running; /* SCHED_{NORMAL,BATCH,IDLE} */
650655 unsigned int idle_nr_running; /* SCHED_IDLE */
651656 unsigned int idle_h_nr_running; /* SCHED_IDLE */
657+ unsigned int h_nr_delayed;
652658
653659 s64 avg_vruntime;
654660 u64 avg_load;
@@ -898,8 +904,11 @@ struct dl_rq { · 243 unmodified lines
898904
899905 static inline void se_update_runnable(struct sched_entity *se)
900906 {
901− if (!entity_is_task(se))
902− se->runnable_weight = se->my_q->h_nr_running;
907+ if (!entity_is_task(se)) {
908+ struct cfs_rq *cfs_rq = se->my_q;
909+
910+ se->runnable_weight = cfs_rq->h_nr_running - cfs_rq->h_nr_delayed;
911+ }
903912 }
904913
905914 static inline long se_runnable(struct sched_entity *se)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t