sched_ext: Use runnable_at for the default core-sched task ordering

The default core-sched ordering runs the longest waiting task first by comparing p->scx.core_sched_at stamps. The stamp is maintained under two rules. touch_core_sched() stamps when a task starts waiting for a CPU and when its slice runs out. If the scheduler implements ops.core_sched_before(), touch_core_sched_dispatch() re-stamps on every dispatch. A comparison can see one stamp taken under each rule, which isn't a meaningful ordering. The dispatch rule also buys little - it only aligns bypass-mode comparisons with the local DSQ order. Multiple schedulers make the mixed comparisons more common. Wait time is what p->scx.runnable_at already tracks for the stall watchdog. Delete core_sched_at with both touch functions and compare runnable_at in the scx_prio_less() fallback. runnable_at is refreshed only on enqueue and goes stale while a task keeps occupying its CPU. Instead of re-stamping, order a running task after every waiting task as it is the most recently serviced.

Tejun Heo committed 3 months ago

Commit 0ec5dd01 parent f7b6d122 files · +25 −83Browse files

Showing the changes to kernel (1 of 2 files). Show all

Unified · Split

kernel/sched/ext/ext.c+25−80

@@ -1155,53 +1155,6 @@ void schedule_dsq_reenq(struct scx_sched sch, struct scx_dispatch_q dsq, · 1154 unmodified lines
1155	1155	schedule_deferred(rq);
1156	1156	}
1157	1157
1158		−/**
1159		− * touch_core_sched - Update timestamp used for core-sched task ordering
1160		− * @rq: rq to read clock from, must be locked
1161		− * @p: task to update the timestamp for
1162		− *
1163		− * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to
1164		− * implement global or local-DSQ FIFO ordering for core-sched. Should be called
1165		− * when a task becomes runnable and its turn on the CPU ends (e.g. slice
1166		− * exhaustion).
1167		− */
1168		−static void touch_core_sched(struct rq rq, struct task_struct p)
1169		−{
1170		− lockdep_assert_rq_held(rq);
1171		−
1172		−#ifdef CONFIG_SCHED_CORE
1173		− /*
1174		− * It's okay to update the timestamp spuriously. Use
1175		− * sched_core_disabled() which is cheaper than enabled().
1176		− *
1177		− * As this is used to determine ordering between tasks of sibling CPUs,
1178		− * it may be better to use per-core dispatch sequence instead.
1179		− */
1180		− if (!sched_core_disabled())
1181		− p->scx.core_sched_at = sched_clock_cpu(cpu_of(rq));
1182		−#endif
1183		−}
1184		−
1185		−/**
1186		− * touch_core_sched_dispatch - Update core-sched timestamp on dispatch
1187		− * @rq: rq to read clock from, must be locked
1188		− * @p: task being dispatched
1189		− *
1190		− * If the BPF scheduler implements custom core-sched ordering via
1191		− * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO
1192		− * ordering within each local DSQ. This function is called from dispatch paths
1193		− * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect.
1194		− */
1195		−static void touch_core_sched_dispatch(struct rq rq, struct task_struct p)
1196		−{
1197		− lockdep_assert_rq_held(rq);
1198		−
1199		−#ifdef CONFIG_SCHED_CORE
1200		− if (unlikely(SCX_HAS_OP(scx_root, core_sched_before)))
1201		− touch_core_sched(rq, p);
1202		−#endif
1203		−}
1204		−
1205	1158	/*
1206	1159	* p->scx.slice_oob packs an out-of-band slice request into one atomic64. A zero
1207	1160	* word means no request. Otherwise the fields are:
@@ -1446,11 +1399,8 @@ static void update_curr_scx(struct rq *rq) · 238 unmodified lines
1446	1399	if (unlikely(delta_exec <= 0))
1447	1400	return;
1448	1401
1449		− if (curr->scx.slice != SCX_SLICE_INF) {
	1402	+ if (curr->scx.slice != SCX_SLICE_INF)
1450	1403	curr->scx.slice -= min_t(u64, curr->scx.slice, delta_exec);
1451		− if (!curr->scx.slice)
1452		− touch_core_sched(rq, curr);
1453		− }
1454	1404
1455	1405	if (unlikely(curr == scx_rescuee(rq)))
1456	1406	scx_rescue_charge(rq, delta_exec);
@@ -1963,8 +1913,6 @@ static void direct_dispatch(struct scx_sched sch, struct task_struct p, · 506 unmodified lines
1963	1913	find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p));
1964	1914	u64 ddsp_enq_flags, slice, vtime;
1965	1915
1966		− touch_core_sched_dispatch(rq, p);
1967		−
1968	1916	p->scx.ddsp_enq_flags \|= enq_flags;
1969	1917
1970	1918	/*
@@ -2143,12 +2091,6 @@ bypass: · 172 unmodified lines
2143	2091	goto enqueue;
2144	2092
2145	2093	enqueue:
2146		− /*
2147		− * For task-ordering, slice refill must be treated as implying the end
2148		− * of the current slice. Otherwise, the longer @p stays on the CPU, the
2149		− * higher priority it becomes from scx_prio_less()'s POV.
2150		− */
2151		− touch_core_sched(rq, p);
2152	2094	refill_task_slice_dfl(sch, p);
2153	2095	clear_direct_dispatch(p);
2154	2096	scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags);
@@ -2226,9 +2168,6 @@ static void enqueue_task_scx(struct rq rq, struct task_struct p, int core_enq_ · 71 unmodified lines
2226	2168	if (SCX_HAS_OP(sch, runnable) && !task_on_rq_migrating(p))
2227	2169	SCX_CALL_OP_TASK(sch, runnable, rq, p, enq_flags);
2228	2170
2229		− if (enq_flags & SCX_ENQ_WAKEUP)
2230		− touch_core_sched(rq, p);
2231		−
2232	2171	/* Start dl_server if this is the first task being enqueued */
2233	2172	if (rq->scx.nr_running == 1)
2234	2173	dl_server_start(&rq->ext_server);
@@ -2886,7 +2825,6 @@ static void finish_dispatch(struct scx_sched sch, struct rq rq, struct task_st · 651 unmodified lines
2886	2825	struct scx_dispatch_q *dsq;
2887	2826	unsigned long opss;
2888	2827
2889		− touch_core_sched_dispatch(rq, p);
2890	2828	retry:
2891	2829	/*
2892	2830	* No need for _acquire here. @p is accessed only after a successful
@@ -3521,13 +3459,10 @@ void ext_server_init(struct rq *rq) · 628 unmodified lines
3521	3459	* usual sched_class'es and needs to find out the expected task ordering. For
3522	3460	* SCX, core-sched calls this function to interrogate the task ordering.
3523	3461	*
3524		− * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used
3525		− * to implement the default task ordering. The older the timestamp, the higher
3526		− * priority the task - the global FIFO ordering matching the default scheduling
3527		− * behavior.
3528		− *
3529		− * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to
3530		− * implement FIFO ordering within each local DSQ. See pick_task_scx().
	3462	+ * Unless overridden by ops.core_sched_before(), the default task ordering runs
	3463	+ * the task which has been waiting longer first. A running task counts as the
	3464	+ * most recently serviced and orders after every waiting task. Waiting tasks are
	3465	+ * compared by @p->scx.runnable_at.
3531	3466	*
3532	3467	* Return: %true if @a should run after @b.
3533	3468	*/
@@ -3536,6 +3471,7 @@ bool scx_prio_less(const struct task_struct a, const struct task_struct b, · 2 unmodified lines
3536	3471	{
3537	3472	struct scx_sched *sch_a = scx_task_sched(a);
3538	3473	struct scx_sched *sch_b = scx_task_sched(b);
	3474	+ bool a_running, b_running;
3539	3475
3540	3476	/*
3541	3477	* scx_prio_less() returns whether @a should run after @b while
@@ -3552,8 +3488,19 @@ bool scx_prio_less(const struct task_struct a, const struct task_struct b, · 10 unmodified lines
3552	3488	task_rq(a),
3553	3489	(struct task_struct *)b,
3554	3490	(struct task_struct *)a);
3555		− else
3556		− return time_after64(a->scx.core_sched_at, b->scx.core_sched_at);
	3491	+
	3492	+ /*
	3493	+ * runnable_at is refreshed only on enqueue, so a task which keeps
	3494	+ * occupying its CPU carries a stale stamp. A running task is the most
	3495	+ * recently serviced whatever its stamp says. Order it after every
	3496	+ * waiting task.
	3497	+ */
	3498	+ a_running = a->on_cpu;
	3499	+ b_running = b->on_cpu;
	3500	+ if (a_running != b_running)
	3501	+ return a_running;
	3502	+
	3503	+ return time_after(a->scx.runnable_at, b->scx.runnable_at);
3557	3504	}
3558	3505	#endif /* CONFIG_SCHED_CORE */
3559	3506
@@ -3824,15 +3771,13 @@ static void task_tick_scx(struct rq rq, struct task_struct curr, int queued) · 264 unmodified lines
3824	3771	update_curr_scx(rq);
3825	3772
3826	3773	/*
3827		− * While disabling, always resched and refresh core-sched timestamp as
3828		− * we can't trust the slice management or ops.core_sched_before().
	3774	+ * While disabling, always resched as we can't trust the slice
	3775	+ * management.
3829	3776	*/
3830		− if (scx_bypassing(sch, cpu_of(rq))) {
	3777	+ if (scx_bypassing(sch, cpu_of(rq)))
3831	3778	scx_set_task_slice(curr, 0);
3832		− touch_core_sched(rq, curr);
3833		− } else if (SCX_HAS_OP(sch, tick)) {
	3779	+ else if (SCX_HAS_OP(sch, tick))
3834	3780	SCX_CALL_OP_TASK(sch, tick, rq, curr);
3835		− }
3836	3781
3837	3782	if (!curr->scx.slice)
3838	3783	resched_curr(rq);
@@ -6085,14 +6030,14 @@ static void unbypass_renotify_idle(struct rq rq, struct scx_sched pos, · 2246 unmodified lines
6085	6030	*
6086	6031	* - dispatch_one() does not report %SCX_DSP_PREV on non-zero slice as slice
6087	6032	* can't be trusted. Whenever a tick triggers, the running task is rotated to
6088		− * the tail of the queue with core_sched_at touched.
	6033	+ * the tail of the queue.
6089	6034	*
6090	6035	* - pick_next_task() suppresses zero slice warning.
6091	6036	*
6092	6037	* - scx_kick_cpu() is disabled to avoid irq_work malfunction during PM
6093	6038	* operations.
6094	6039	*
6095		− * - scx_prio_less() reverts to the default core_sched_at order.
	6040	+ * - scx_prio_less() reverts to the default runnable_at order.
6096	6041	*/
6097	6042	void scx_bypass(struct scx_sched *sch, bool bypass)
6098	6043	{

sched_ext: Use runnable_at for the default core-sched task ordering

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code