The default core-sched ordering runs the longest waiting task first by comparing p->scx.core_sched_at stamps. The stamp is maintained under two rules. touch_core_sched() stamps when a task starts waiting for a CPU and when its slice runs out. If the scheduler implements ops.core_sched_before(), touch_core_sched_dispatch() re-stamps on every dispatch. A comparison can see one stamp taken under each rule, which isn't a meaningful ordering. The dispatch rule also buys little - it only aligns bypass-mode comparisons with the local DSQ order. Multiple schedulers make the mixed comparisons more common. Wait time is what p->scx.runnable_at already tracks for the stall watchdog. Delete core_sched_at with both touch functions and compare runnable_at in the scx_prio_less() fallback. runnable_at is refreshed only on enqueue and goes stale while a task keeps occupying its CPU. Instead of re-stamping, order a running task after every waiting task as it is the most recently serviced.
Showing the changes to kernel (1 of 2 files). Show all
Unified · Split
| @@ -1155,53 +1155,6 @@ void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq, · 1154 unmodified lines | |||
| 1155 | schedule_deferred(rq); | 1155 | schedule_deferred(rq); |
| 1156 | } | 1156 | } |
| 1157 | 1157 | ||
| 1158 | /** | ||
| 1159 | * touch_core_sched - Update timestamp used for core-sched task ordering | ||
| 1160 | * @rq: rq to read clock from, must be locked | ||
| 1161 | * @p: task to update the timestamp for | ||
| 1162 | * | ||
| 1163 | * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to | ||
| 1164 | * implement global or local-DSQ FIFO ordering for core-sched. Should be called | ||
| 1165 | * when a task becomes runnable and its turn on the CPU ends (e.g. slice | ||
| 1166 | * exhaustion). | ||
| 1167 | */ | ||
| 1168 | static void touch_core_sched(struct rq *rq, struct task_struct *p) | ||
| 1169 | { | ||
| 1170 | lockdep_assert_rq_held(rq); | ||
| 1171 | |||
| 1172 | #ifdef CONFIG_SCHED_CORE | ||
| 1173 | /* | ||
| 1174 | * It's okay to update the timestamp spuriously. Use | ||
| 1175 | * sched_core_disabled() which is cheaper than enabled(). | ||
| 1176 | * | ||
| 1177 | * As this is used to determine ordering between tasks of sibling CPUs, | ||
| 1178 | * it may be better to use per-core dispatch sequence instead. | ||
| 1179 | */ | ||
| 1180 | if (!sched_core_disabled()) | ||
| 1181 | p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq)); | ||
| 1182 | #endif | ||
| 1183 | } | ||
| 1184 | |||
| 1185 | /** | ||
| 1186 | * touch_core_sched_dispatch - Update core-sched timestamp on dispatch | ||
| 1187 | * @rq: rq to read clock from, must be locked | ||
| 1188 | * @p: task being dispatched | ||
| 1189 | * | ||
| 1190 | * If the BPF scheduler implements custom core-sched ordering via | ||
| 1191 | * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO | ||
| 1192 | * ordering within each local DSQ. This function is called from dispatch paths | ||
| 1193 | * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect. | ||
| 1194 | */ | ||
| 1195 | static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p) | ||
| 1196 | { | ||
| 1197 | lockdep_assert_rq_held(rq); | ||
| 1198 | |||
| 1199 | #ifdef CONFIG_SCHED_CORE | ||
| 1200 | if (unlikely(SCX_HAS_OP(scx_root, core_sched_before))) | ||
| 1201 | touch_core_sched(rq, p); | ||
| 1202 | #endif | ||
| 1203 | } | ||
| 1204 | |||
| 1205 | /* | 1158 | /* |
| 1206 | * p->scx.slice_oob packs an out-of-band slice request into one atomic64. A zero | 1159 | * p->scx.slice_oob packs an out-of-band slice request into one atomic64. A zero |
| 1207 | * word means no request. Otherwise the fields are: | 1160 | * word means no request. Otherwise the fields are: |
| @@ -1446,11 +1399,8 @@ static void update_curr_scx(struct rq *rq) · 238 unmodified lines | |||
| 1446 | if (unlikely(delta_exec <= 0)) | 1399 | if (unlikely(delta_exec <= 0)) |
| 1447 | return; | 1400 | return; |
| 1448 | 1401 | ||
| 1449 | if (curr->scx.slice != SCX_SLICE_INF) { | 1402 | if (curr->scx.slice != SCX_SLICE_INF) |
| 1450 | curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); | 1403 | curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec); |
| 1451 | if (!curr->scx.slice) | ||
| 1452 | touch_core_sched(rq, curr); | ||
| 1453 | } | ||
| 1454 | 1404 | ||
| 1455 | if (unlikely(curr == scx_rescuee(rq))) | 1405 | if (unlikely(curr == scx_rescuee(rq))) |
| 1456 | scx_rescue_charge(rq, delta_exec); | 1406 | scx_rescue_charge(rq, delta_exec); |
| @@ -1963,8 +1913,6 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p, · 506 unmodified lines | |||
| 1963 | find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); | 1913 | find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p)); |
| 1964 | u64 ddsp_enq_flags, slice, vtime; | 1914 | u64 ddsp_enq_flags, slice, vtime; |
| 1965 | 1915 | ||
| 1966 | touch_core_sched_dispatch(rq, p); | ||
| 1967 | |||
| 1968 | p->scx.ddsp_enq_flags |= enq_flags; | 1916 | p->scx.ddsp_enq_flags |= enq_flags; |
| 1969 | 1917 | ||
| 1970 | /* | 1918 | /* |
| @@ -2143,12 +2091,6 @@ bypass: · 172 unmodified lines | |||
| 2143 | goto enqueue; | 2091 | goto enqueue; |
| 2144 | 2092 | ||
| 2145 | enqueue: | 2093 | enqueue: |
| 2146 | /* | ||
| 2147 | * For task-ordering, slice refill must be treated as implying the end | ||
| 2148 | * of the current slice. Otherwise, the longer @p stays on the CPU, the | ||
| 2149 | * higher priority it becomes from scx_prio_less()'s POV. | ||
| 2150 | */ | ||
| 2151 | touch_core_sched(rq, p); | ||
| 2152 | refill_task_slice_dfl(sch, p); | 2094 | refill_task_slice_dfl(sch, p); |
| 2153 | clear_direct_dispatch(p); | 2095 | clear_direct_dispatch(p); |
| 2154 | scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); | 2096 | scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags); |
| @@ -2226,9 +2168,6 @@ static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int core_enq_ · 71 unmodified lines | |||
| 2226 | if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) | 2168 | if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p)) |
| 2227 | SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags); | 2169 | SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags); |
| 2228 | 2170 | ||
| 2229 | if (enq_flags & SCX_ENQ_WAKEUP) | ||
| 2230 | touch_core_sched(rq, p); | ||
| 2231 | |||
| 2232 | /* Start dl_server if this is the first task being enqueued */ | 2171 | /* Start dl_server if this is the first task being enqueued */ |
| 2233 | if (rq->scx.nr_running == 1) | 2172 | if (rq->scx.nr_running == 1) |
| 2234 | dl_server_start(&rq->ext_server); | 2173 | dl_server_start(&rq->ext_server); |
| @@ -2886,7 +2825,6 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq, struct task_st · 651 unmodified lines | |||
| 2886 | struct scx_dispatch_q *dsq; | 2825 | struct scx_dispatch_q *dsq; |
| 2887 | unsigned long opss; | 2826 | unsigned long opss; |
| 2888 | 2827 | ||
| 2889 | touch_core_sched_dispatch(rq, p); | ||
| 2890 | retry: | 2828 | retry: |
| 2891 | /* | 2829 | /* |
| 2892 | * No need for _acquire here. @p is accessed only after a successful | 2830 | * No need for _acquire here. @p is accessed only after a successful |
| @@ -3521,13 +3459,10 @@ void ext_server_init(struct rq *rq) · 628 unmodified lines | |||
| 3521 | * usual sched_class'es and needs to find out the expected task ordering. For | 3459 | * usual sched_class'es and needs to find out the expected task ordering. For |
| 3522 | * SCX, core-sched calls this function to interrogate the task ordering. | 3460 | * SCX, core-sched calls this function to interrogate the task ordering. |
| 3523 | * | 3461 | * |
| 3524 | * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used | 3462 | * Unless overridden by ops.core_sched_before(), the default task ordering runs |
| 3525 | * to implement the default task ordering. The older the timestamp, the higher | 3463 | * the task which has been waiting longer first. A running task counts as the |
| 3526 | * priority the task - the global FIFO ordering matching the default scheduling | 3464 | * most recently serviced and orders after every waiting task. Waiting tasks are |
| 3527 | * behavior. | 3465 | * compared by @p->scx.runnable_at. |
| 3528 | * | ||
| 3529 | * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to | ||
| 3530 | * implement FIFO ordering within each local DSQ. See pick_task_scx(). | ||
| 3531 | * | 3466 | * |
| 3532 | * Return: %true if @a should run after @b. | 3467 | * Return: %true if @a should run after @b. |
| 3533 | */ | 3468 | */ |
| @@ -3536,6 +3471,7 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, · 2 unmodified lines | |||
| 3536 | { | 3471 | { |
| 3537 | struct scx_sched *sch_a = scx_task_sched(a); | 3472 | struct scx_sched *sch_a = scx_task_sched(a); |
| 3538 | struct scx_sched *sch_b = scx_task_sched(b); | 3473 | struct scx_sched *sch_b = scx_task_sched(b); |
| 3474 | bool a_running, b_running; | ||
| 3539 | 3475 | ||
| 3540 | /* | 3476 | /* |
| 3541 | * scx_prio_less() returns whether @a should run after @b while | 3477 | * scx_prio_less() returns whether @a should run after @b while |
| @@ -3552,8 +3488,19 @@ bool scx_prio_less(const struct task_struct *a, const struct task_struct *b, · 10 unmodified lines | |||
| 3552 | task_rq(a), | 3488 | task_rq(a), |
| 3553 | (struct task_struct *)b, | 3489 | (struct task_struct *)b, |
| 3554 | (struct task_struct *)a); | 3490 | (struct task_struct *)a); |
| 3555 | else | 3491 | |
| 3556 | return time_after64(a->scx.core_sched_at, b->scx.core_sched_at); | 3492 | /* |
| 3493 | * runnable_at is refreshed only on enqueue, so a task which keeps | ||
| 3494 | * occupying its CPU carries a stale stamp. A running task is the most | ||
| 3495 | * recently serviced whatever its stamp says. Order it after every | ||
| 3496 | * waiting task. | ||
| 3497 | */ | ||
| 3498 | a_running = a->on_cpu; | ||
| 3499 | b_running = b->on_cpu; | ||
| 3500 | if (a_running != b_running) | ||
| 3501 | return a_running; | ||
| 3502 | |||
| 3503 | return time_after(a->scx.runnable_at, b->scx.runnable_at); | ||
| 3557 | } | 3504 | } |
| 3558 | #endif /* CONFIG_SCHED_CORE */ | 3505 | #endif /* CONFIG_SCHED_CORE */ |
| 3559 | 3506 | ||
| @@ -3824,15 +3771,13 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued) · 264 unmodified lines | |||
| 3824 | update_curr_scx(rq); | 3771 | update_curr_scx(rq); |
| 3825 | 3772 | ||
| 3826 | /* | 3773 | /* |
| 3827 | * While disabling, always resched and refresh core-sched timestamp as | 3774 | * While disabling, always resched as we can't trust the slice |
| 3828 | * we can't trust the slice management or ops.core_sched_before(). | 3775 | * management. |
| 3829 | */ | 3776 | */ |
| 3830 | if (scx_bypassing(sch, cpu_of(rq))) { | 3777 | if (scx_bypassing(sch, cpu_of(rq))) |
| 3831 | scx_set_task_slice(curr, 0); | 3778 | scx_set_task_slice(curr, 0); |
| 3832 | touch_core_sched(rq, curr); | 3779 | else if (SCX_HAS_OP(sch, tick)) |
| 3833 | } else if (SCX_HAS_OP(sch, tick)) { | ||
| 3834 | SCX_CALL_OP_TASK(sch, tick, rq, curr); | 3780 | SCX_CALL_OP_TASK(sch, tick, rq, curr); |
| 3835 | } | ||
| 3836 | 3781 | ||
| 3837 | if (!curr->scx.slice) | 3782 | if (!curr->scx.slice) |
| 3838 | resched_curr(rq); | 3783 | resched_curr(rq); |
| @@ -6085,14 +6030,14 @@ static void unbypass_renotify_idle(struct rq *rq, struct scx_sched *pos, · 2246 unmodified lines | |||
| 6085 | * | 6030 | * |
| 6086 | * - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice | 6031 | * - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice |
| 6087 | * can't be trusted. Whenever a tick triggers, the running task is rotated to | 6032 | * can't be trusted. Whenever a tick triggers, the running task is rotated to |
| 6088 | * the tail of the queue with core_sched_at touched. | 6033 | * the tail of the queue. |
| 6089 | * | 6034 | * |
| 6090 | * - pick_next_task() suppresses zero slice warning. | 6035 | * - pick_next_task() suppresses zero slice warning. |
| 6091 | * | 6036 | * |
| 6092 | * - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM | 6037 | * - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM |
| 6093 | * operations. | 6038 | * operations. |
| 6094 | * | 6039 | * |
| 6095 | * - scx_prio_less() reverts to the default core_sched_at order. | 6040 | * - scx_prio_less() reverts to the default runnable_at order. |
| 6096 | */ | 6041 | */ |
| 6097 | void scx_bypass(struct scx_sched *sch, bool bypass) | 6042 | void scx_bypass(struct scx_sched *sch, bool bypass) |
| 6098 | { | 6043 | { |