Merge tag 'sched_ext-for-7.1-rc1-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext

Pull sched_ext fixes from Tejun Heo: "The merge window pulled in the cgroup sub-scheduler infrastructure, and new AI reviews are accelerating bug reporting and fixing - hence the larger than usual fixes batch: - Use-after-frees during scheduler load/unload: - The disable path could free the BPF scheduler while deferred irq_work / kthread work was still in flight - cgroup setter callbacks read the active scheduler outside the rwsem that synchronizes against teardown Fix both, and reuse the disable drain in the enable error paths so the BPF JIT page can't be freed under live callbacks. - Several BPF op invocations didn't tell the framework which runqueue was already locked, so helper kfuncs that re-acquire the runqueue by CPU could deadlock on the held lock Fix the affected callsites, including recursive parent-into-child dispatch. - The hardlockup notifier ran from NMI but eventually took a non-NMI-safe lock. Bounce it through irq_work. - A handful of bugs in the new sub-scheduler hierarchy: - helper kfuncs hard-coded the root instead of resolving the caller's scheduler - the enable error path tried to disable per-task state that had never been initialized, and leaked cpus_read_lock on the way out - a sysfs object was leaked on every load/unload - the dispatch fast-path used the root scheduler instead of the task's - a couple of CONFIG #ifdef guards were misclassified - Verifier-time hardening: BPF programs of unrelated struct_ops types (e.g. tcp_congestion_ops) could call sched_ext kfuncs - a semantic bug and, once sub-sched was enabled, a KASAN out-of-bounds read. Now rejected at load. Plus a few NULL and cross-task argument checks on sched_ext kfuncs, and a selftest covering the new deny. - rhashtable (Herbert): restore the insecure_elasticity toggle and bounce the deferred-resize kick through irq_work to break a lock-order cycle observable from raw-spinlock callers. sched_ext's scheduler-instance hash is the first user of both. - The bypass-mode load balancer used file-scope cpumasks; with multiple scheduler instances now possible, those raced. Move to per-instance cpumasks, plus a follow-up to skip tasks whose recorded CPU is stale relative to the new owning runqueue. - Smaller fixes: - a dispatch queue's first-task tracking misbehaved when a parked iterator cursor sat in the list - the runqueue's next-class wasn't promoted on local-queue enqueue, leaving an SCX task behind RT in edge cases - the reference qmap scheduler stopped erroring on legitimate cross-scheduler task-storage misses" * tag 'sched_ext-for-7.1-rc1-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext: (26 commits) sched_ext: Fix scx_flush_disable_work() UAF race sched_ext: Call wakeup_preempt() in local_dsq_post_enq() sched_ext: Release cpus_read_lock on scx_link_sched() failure in root enable sched_ext: Reject NULL-sch callers in scx_bpf_task_set_slice/dsq_vtime sched_ext: Refuse cross-task select_cpu_from_kfunc calls sched_ext: Align cgroup #ifdef guards with SUB_SCHED vs GROUP_SCHED sched_ext: Make bypass LB cpumasks per-scheduler sched_ext: Pass held rq to SCX_CALL_OP() for core_sched_before sched_ext: Pass held rq to SCX_CALL_OP() for dump_cpu/dump_task sched_ext: Save and restore scx_locked_rq across SCX_CALL_OP sched_ext: Use dsq->first_task instead of list_empty() in dispatch_enqueue() FIFO-tail sched_ext: Resolve caller's scheduler in scx_bpf_destroy_dsq() / scx_bpf_dsq_nr_queued() sched_ext: Read scx_root under scx_cgroup_ops_rwsem in cgroup setters sched_ext: Don't disable tasks in scx_sub_enable_workfn() abort path sched_ext: Skip tasks with stale task_rq in bypass_lb_cpu() sched_ext: Guard scx_dsq_move() against NULL kit->dsq after failed iter_new sched_ext: Unregister sub_kset on scheduler disable sched_ext: Defer scx_hardlockup() out of NMI sched_ext: sync disable_irq_work in bpf_scx_unreg() sched_ext: Fix local_dsq_post_enq() to use task's scheduler in sub-sched ...

Linus Torvalds committed 6 months ago

Commit 664f0f62 parents dca922e d99f7a311 files · +436 −150Browse files

Showing the changes to kernel (4 of 11 files). Show all

Unified · Split

4 files

kernel/sched/ext.c +276 −122
kernel/sched/ext_idle.c +18 −2
kernel/sched/ext_idle.h +1 −0
kernel/sched/ext_internal.h +2 −0

kernel/sched/ext.c+276−122

@@ -32,6 +32,7 @@ static const struct rhashtable_params scx_sched_hash_params = { · 31 unmodified lines
32	.key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id),	32	.key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id),
33	.key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id),	33	.key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id),
34	.head_offset = offsetof(struct scx_sched, hash_node),	34	.head_offset = offsetof(struct scx_sched, hash_node),
		35	.insecure_elasticity = true, /* inserted under scx_sched_lock */
35	};	36	};
36		37
37	static struct rhashtable scx_sched_hash;	38	static struct rhashtable scx_sched_hash;
@@ -52,8 +53,6 @@ DEFINE_STATIC_KEY_FALSE(__scx_enabled); · 14 unmodified lines
52	DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem);	53	DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem);
53	static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED);	54	static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED);
54	static DEFINE_RAW_SPINLOCK(scx_bypass_lock);	55	static DEFINE_RAW_SPINLOCK(scx_bypass_lock);
55	static cpumask_var_t scx_bypass_lb_donee_cpumask;
56	static cpumask_var_t scx_bypass_lb_resched_cpumask;
57	static bool scx_init_task_enabled;	56	static bool scx_init_task_enabled;
58	static bool scx_switching_all;	57	static bool scx_switching_all;
59	DEFINE_STATIC_KEY_FALSE(__scx_switched_all);	58	DEFINE_STATIC_KEY_FALSE(__scx_switched_all);
@@ -469,24 +468,35 @@ static inline void update_locked_rq(struct rq *rq) · 409 unmodified lines
469	__this_cpu_write(scx_locked_rq_state, rq);	468	__this_cpu_write(scx_locked_rq_state, rq);
470	}	469	}
471		470
472	#define SCX_CALL_OP(sch, op, rq, args...) \	471	/*
		472	* SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not
		473	* clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit.
		474	*/
		475	#define SCX_CALL_OP(sch, op, locked_rq, args...) \
473	do { \	476	do { \
474	if (rq) \	477	struct rq *__prev_locked_rq; \
475	update_locked_rq(rq); \	478	\
		479	if (locked_rq) { \
		480	__prev_locked_rq = scx_locked_rq(); \
		481	update_locked_rq(locked_rq); \
		482	} \
476	(sch)->ops.op(args); \	483	(sch)->ops.op(args); \
477	if (rq) \	484	if (locked_rq) \
478	update_locked_rq(NULL); \	485	update_locked_rq(__prev_locked_rq); \
479	} while (0)	486	} while (0)
480		487
481	#define SCX_CALL_OP_RET(sch, op, rq, args...) \	488	#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \
482	({ \	489	({ \
		490	struct rq *__prev_locked_rq; \
483	__typeof__((sch)->ops.op(args)) __ret; \	491	__typeof__((sch)->ops.op(args)) __ret; \
484	\	492	\
485	if (rq) \	493	if (locked_rq) { \
486	update_locked_rq(rq); \	494	__prev_locked_rq = scx_locked_rq(); \
		495	update_locked_rq(locked_rq); \
		496	} \
487	__ret = (sch)->ops.op(args); \	497	__ret = (sch)->ops.op(args); \
488	if (rq) \	498	if (locked_rq) \
489	update_locked_rq(NULL); \	499	update_locked_rq(__prev_locked_rq); \
490	__ret; \	500	__ret; \
491	})	501	})
492		502
@@ -498,39 +508,39 @@ do { \ · 5 unmodified lines
498	* those subject tasks.	508	* those subject tasks.
499	*	509	*
500	* Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held -	510	* Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held -
501	* either via the @rq argument here, or (for ops.select_cpu()) via @p's pi_lock	511	* either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's
502	* held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu. So if	512	* pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu.
503	* kf_tasks[] is set, @p's scheduler-protected fields are stable.	513	* So if kf_tasks[] is set, @p's scheduler-protected fields are stable.
504	*	514	*
505	* kf_tasks[] can not stack, so task-based SCX ops must not nest. The	515	* kf_tasks[] can not stack, so task-based SCX ops must not nest. The
506	* WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants	516	* WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants
507	* while a previous one is still in progress.	517	* while a previous one is still in progress.
508	*/	518	*/
509	#define SCX_CALL_OP_TASK(sch, op, rq, task, args...) \	519	#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \
510	do { \	520	do { \
511	WARN_ON_ONCE(current->scx.kf_tasks[0]); \	521	WARN_ON_ONCE(current->scx.kf_tasks[0]); \
512	current->scx.kf_tasks[0] = task; \	522	current->scx.kf_tasks[0] = task; \
513	SCX_CALL_OP((sch), op, rq, task, ##args); \	523	SCX_CALL_OP((sch), op, locked_rq, task, ##args); \
514	current->scx.kf_tasks[0] = NULL; \	524	current->scx.kf_tasks[0] = NULL; \
515	} while (0)	525	} while (0)
516		526
517	#define SCX_CALL_OP_TASK_RET(sch, op, rq, task, args...) \	527	#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \
518	({ \	528	({ \
519	__typeof__((sch)->ops.op(task, ##args)) __ret; \	529	__typeof__((sch)->ops.op(task, ##args)) __ret; \
520	WARN_ON_ONCE(current->scx.kf_tasks[0]); \	530	WARN_ON_ONCE(current->scx.kf_tasks[0]); \
521	current->scx.kf_tasks[0] = task; \	531	current->scx.kf_tasks[0] = task; \
522	__ret = SCX_CALL_OP_RET((sch), op, rq, task, ##args); \	532	__ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \
523	current->scx.kf_tasks[0] = NULL; \	533	current->scx.kf_tasks[0] = NULL; \
524	__ret; \	534	__ret; \
525	})	535	})
526		536
527	#define SCX_CALL_OP_2TASKS_RET(sch, op, rq, task0, task1, args...) \	537	#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \
528	({ \	538	({ \
529	__typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \	539	__typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \
530	WARN_ON_ONCE(current->scx.kf_tasks[0]); \	540	WARN_ON_ONCE(current->scx.kf_tasks[0]); \
531	current->scx.kf_tasks[0] = task0; \	541	current->scx.kf_tasks[0] = task0; \
532	current->scx.kf_tasks[1] = task1; \	542	current->scx.kf_tasks[1] = task1; \
533	__ret = SCX_CALL_OP_RET((sch), op, rq, task0, task1, ##args); \	543	__ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \
534	current->scx.kf_tasks[0] = NULL; \	544	current->scx.kf_tasks[0] = NULL; \
535	current->scx.kf_tasks[1] = NULL; \	545	current->scx.kf_tasks[1] = NULL; \
536	__ret; \	546	__ret; \
@@ -1388,18 +1398,55 @@ static void call_task_dequeue(struct scx_sched sch, struct rq rq, · 851 unmodified lines
1388	p->scx.flags &= ~SCX_TASK_IN_CUSTODY;	1398	p->scx.flags &= ~SCX_TASK_IN_CUSTODY;
1389	}	1399	}
1390		1400
1391	static void local_dsq_post_enq(struct scx_dispatch_q dsq, struct task_struct p,	1401	static void local_dsq_post_enq(struct scx_sched sch, struct scx_dispatch_q dsq,
1392	u64 enq_flags)	1402	struct task_struct *p, u64 enq_flags)
1393	{	1403	{
1394	struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);	1404	struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
1395	bool preempt = false;
1396		1405
1397	call_task_dequeue(scx_root, rq, p, 0);	1406	call_task_dequeue(sch, rq, p, 0);
		1407
		1408	/*
		1409	* Note that @rq's lock may be dropped between this enqueue and @p
		1410	* actually getting on CPU. This gives higher-class tasks (e.g. RT)
		1411	* an opportunity to wake up on @rq and prevent @p from running.
		1412	* Here are some concrete examples:
		1413	*
		1414	* Example 1:
		1415	*
		1416	* We dispatch two tasks from a single ops.dispatch():
		1417	* - First, a local task to this CPU's local DSQ;
		1418	* - Second, a local/remote task to a remote CPU's local DSQ.
		1419	* We must drop the local rq lock in order to finish the second
		1420	* dispatch. In that time, an RT task can wake up on the local rq.
		1421	*
		1422	* Example 2:
		1423	*
		1424	* We dispatch a local/remote task to a remote CPU's local DSQ.
		1425	* We must drop the remote rq lock before the dispatched task can run,
		1426	* which gives an RT task an opportunity to wake up on the remote rq.
		1427	*
		1428	* Both examples work the same if we replace dispatching with moving
		1429	* the tasks from a user-created DSQ.
		1430	*
		1431	* We must detect these wakeups so that we can re-enqueue IMMED tasks
		1432	* from @rq's local DSQ. scx_wakeup_preempt() serves exactly this
		1433	* purpose, but for it to be invoked, we must ensure that we bump
		1434	* @rq->next_class to &ext_sched_class if it's currently idle.
		1435	*
		1436	* wakeup_preempt() does the bumping, and since we only invoke it if
		1437	* @rq->next_class is below &ext_sched_class, it will also
		1438	* resched_curr(rq).
		1439	*/
		1440	if (sched_class_above(p->sched_class, rq->next_class))
		1441	wakeup_preempt(rq, p, 0);
1398		1442
1399	/*	1443	/*
1400	* If @rq is in balance, the CPU is already vacant and looking for the	1444	* If @rq is in balance, the CPU is already vacant and looking for the
1401	* next task to run. No need to preempt or trigger resched after moving	1445	* next task to run. No need to preempt or trigger resched after moving
1402	* @p into its local DSQ.	1446	* @p into its local DSQ.
		1447	* Note that the wakeup_preempt() above may have already triggered
		1448	* a resched if @rq->next_class was idle. It's harmless, since
		1449	* need_resched is cleared immediately after task pick.
1403	*/	1450	*/
1404	if (rq->scx.flags & SCX_RQ_IN_BALANCE)	1451	if (rq->scx.flags & SCX_RQ_IN_BALANCE)
1405	return;	1452	return;
@@ -1407,11 +1454,8 @@ static void local_dsq_post_enq(struct scx_dispatch_q dsq, struct task_struct p · 1 unmodified line
1407	if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&	1454	if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
1408	rq->curr->sched_class == &ext_sched_class) {	1455	rq->curr->sched_class == &ext_sched_class) {
1409	rq->curr->scx.slice = 0;	1456	rq->curr->scx.slice = 0;
1410	preempt = true;
1411	}
1412
1413	if (preempt \|\| sched_class_above(&ext_sched_class, rq->curr->sched_class))
1414	resched_curr(rq);	1457	resched_curr(rq);
		1458	}
1415	}	1459	}
1416		1460
1417	static void dispatch_enqueue(struct scx_sched sch, struct rq rq,	1461	static void dispatch_enqueue(struct scx_sched sch, struct rq rq,
@@ -1494,11 +1538,13 @@ static void dispatch_enqueue(struct scx_sched sch, struct rq rq, · 76 unmodified lines
1494	if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN))	1538	if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN))
1495	rcu_assign_pointer(dsq->first_task, p);	1539	rcu_assign_pointer(dsq->first_task, p);
1496	} else {	1540	} else {
1497	bool was_empty;	1541	/*
1498		1542	* dsq->list can contain parked BPF iterator cursors, so
1499	was_empty = list_empty(&dsq->list);	1543	* list_empty() here isn't a reliable proxy for "no real
		1544	* task in the DSQ". Test dsq->first_task directly.
		1545	*/
1500	list_add_tail(&p->scx.dsq_list.node, &dsq->list);	1546	list_add_tail(&p->scx.dsq_list.node, &dsq->list);
1501	if (was_empty && !(dsq->id & SCX_DSQ_FLAG_BUILTIN))	1547	if (!dsq->first_task && !(dsq->id & SCX_DSQ_FLAG_BUILTIN))
1502	rcu_assign_pointer(dsq->first_task, p);	1548	rcu_assign_pointer(dsq->first_task, p);
1503	}	1549	}
1504	}	1550	}
@@ -1518,7 +1564,7 @@ static void dispatch_enqueue(struct scx_sched sch, struct rq rq, · 13 unmodified lines
1518	* concurrently in a non-atomic way.	1564	* concurrently in a non-atomic way.
1519	*/	1565	*/
1520	if (is_local) {	1566	if (is_local) {
1521	local_dsq_post_enq(dsq, p, enq_flags);	1567	local_dsq_post_enq(sch, dsq, p, enq_flags);
1522	} else {	1568	} else {
1523	/*	1569	/*
1524	* Task on global/bypass DSQ: leave custody, task on	1570	* Task on global/bypass DSQ: leave custody, task on
@@ -2129,7 +2175,8 @@ static void wakeup_preempt_scx(struct rq rq, struct task_struct p, int wake_fl · 604 unmodified lines
2129	schedule_reenq_local(rq, 0);	2175	schedule_reenq_local(rq, 0);
2130	}	2176	}
2131		2177
2132	static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags,	2178	static void move_local_task_to_local_dsq(struct scx_sched *sch,
		2179	struct task_struct *p, u64 enq_flags,
2133	struct scx_dispatch_q *src_dsq,	2180	struct scx_dispatch_q *src_dsq,
2134	struct rq *dst_rq)	2181	struct rq *dst_rq)
2135	{	2182	{
@@ -2149,7 +2196,7 @@ static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags, · 13 unmodified lines
2149	dsq_inc_nr(dst_dsq, p, enq_flags);	2196	dsq_inc_nr(dst_dsq, p, enq_flags);
2150	p->scx.dsq = dst_dsq;	2197	p->scx.dsq = dst_dsq;
2151		2198
2152	local_dsq_post_enq(dst_dsq, p, enq_flags);	2199	local_dsq_post_enq(sch, dst_dsq, p, enq_flags);
2153	}	2200	}
2154		2201
2155	/**	2202	/**
@@ -2370,7 +2417,7 @@ static struct rq move_task_between_dsqs(struct scx_sched sch, · 214 unmodified lines
2370	/* @p is going from a non-local DSQ to a local DSQ */	2417	/* @p is going from a non-local DSQ to a local DSQ */
2371	if (src_rq == dst_rq) {	2418	if (src_rq == dst_rq) {
2372	task_unlink_from_dsq(p, src_dsq);	2419	task_unlink_from_dsq(p, src_dsq);
2373	move_local_task_to_local_dsq(p, enq_flags,	2420	move_local_task_to_local_dsq(sch, p, enq_flags,
2374	src_dsq, dst_rq);	2421	src_dsq, dst_rq);
2375	raw_spin_unlock(&src_dsq->lock);	2422	raw_spin_unlock(&src_dsq->lock);
2376	} else {	2423	} else {
@@ -2423,7 +2470,7 @@ retry: · 46 unmodified lines
2423		2470
2424	if (rq == task_rq) {	2471	if (rq == task_rq) {
2425	task_unlink_from_dsq(p, dsq);	2472	task_unlink_from_dsq(p, dsq);
2426	move_local_task_to_local_dsq(p, enq_flags, dsq, rq);	2473	move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq);
2427	raw_spin_unlock(&dsq->lock);	2474	raw_spin_unlock(&dsq->lock);
2428	return true;	2475	return true;
2429	}	2476	}
@@ -3183,7 +3230,7 @@ bool scx_prio_less(const struct task_struct a, const struct task_struct b, · 753 unmodified lines
3183	if (sch_a == sch_b && SCX_HAS_OP(sch_a, core_sched_before) &&	3230	if (sch_a == sch_b && SCX_HAS_OP(sch_a, core_sched_before) &&
3184	!scx_bypassing(sch_a, task_cpu(a)))	3231	!scx_bypassing(sch_a, task_cpu(a)))
3185	return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before,	3232	return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before,
3186	NULL,	3233	task_rq(a),
3187	(struct task_struct *)a,	3234	(struct task_struct *)a,
3188	(struct task_struct *)b);	3235	(struct task_struct *)b);
3189	else	3236	else
@@ -3631,6 +3678,22 @@ static void __scx_disable_and_exit_task(struct scx_sched *sch, · 441 unmodified lines
3631	SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args);	3678	SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args);
3632	}	3679	}
3633		3680
		3681	/*
		3682	* Undo a completed __scx_init_task(sch, p, false) when scx_enable_task() never
		3683	* ran. The task state has not been transitioned, so this mirrors the
		3684	* SCX_TASK_INIT branch in __scx_disable_and_exit_task().
		3685	*/
		3686	static void scx_sub_init_cancel_task(struct scx_sched sch, struct task_struct p)
		3687	{
		3688	struct scx_exit_task_args args = { .cancelled = true };
		3689
		3690	lockdep_assert_held(&p->pi_lock);
		3691	lockdep_assert_rq_held(task_rq(p));
		3692
		3693	if (SCX_HAS_OP(sch, exit_task))
		3694	SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args);
		3695	}
		3696
3634	static void scx_disable_and_exit_task(struct scx_sched *sch,	3697	static void scx_disable_and_exit_task(struct scx_sched *sch,
3635	struct task_struct *p)	3698	struct task_struct *p)
3636	{	3699	{
@@ -3639,11 +3702,12 @@ static void scx_disable_and_exit_task(struct scx_sched *sch, · 2 unmodified lines
3639	/*	3702	/*
3640	* If set, @p exited between __scx_init_task() and scx_enable_task() in	3703	* If set, @p exited between __scx_init_task() and scx_enable_task() in
3641	* scx_sub_enable() and is initialized for both the associated sched and	3704	* scx_sub_enable() and is initialized for both the associated sched and
3642	* its parent. Disable and exit for the child too.	3705	* its parent. Exit for the child too - scx_enable_task() never ran for
		3706	* it, so undo only init_task.
3643	*/	3707	*/
3644	if ((p->scx.flags & SCX_TASK_SUB_INIT) &&	3708	if (p->scx.flags & SCX_TASK_SUB_INIT) {
3645	!WARN_ON_ONCE(!scx_enabling_sub_sched)) {	3709	if (!WARN_ON_ONCE(!scx_enabling_sub_sched))
3646	__scx_disable_and_exit_task(scx_enabling_sub_sched, p);	3710	scx_sub_init_cancel_task(scx_enabling_sub_sched, p);
3647	p->scx.flags &= ~SCX_TASK_SUB_INIT;	3711	p->scx.flags &= ~SCX_TASK_SUB_INIT;
3648	}	3712	}
3649		3713
@@ -4324,9 +4388,10 @@ void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) · 674 unmodified lines
4324		4388
4325	void scx_group_set_weight(struct task_group *tg, unsigned long weight)	4389	void scx_group_set_weight(struct task_group *tg, unsigned long weight)
4326	{	4390	{
4327	struct scx_sched *sch = scx_root;	4391	struct scx_sched *sch;
4328		4392
4329	percpu_down_read(&scx_cgroup_ops_rwsem);	4393	percpu_down_read(&scx_cgroup_ops_rwsem);
		4394	sch = scx_root;
4330		4395
4331	if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) &&	4396	if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) &&
4332	tg->scx.weight != weight)	4397	tg->scx.weight != weight)
@@ -4339,9 +4404,10 @@ void scx_group_set_weight(struct task_group *tg, unsigned long weight) · 6 unmodified lines
4339		4404
4340	void scx_group_set_idle(struct task_group *tg, bool idle)	4405	void scx_group_set_idle(struct task_group *tg, bool idle)
4341	{	4406	{
4342	struct scx_sched *sch = scx_root;	4407	struct scx_sched *sch;
4343		4408
4344	percpu_down_read(&scx_cgroup_ops_rwsem);	4409	percpu_down_read(&scx_cgroup_ops_rwsem);
		4410	sch = scx_root;
4345		4411
4346	if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle))	4412	if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle))
4347	SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle);	4413	SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle);
@@ -4355,9 +4421,10 @@ void scx_group_set_idle(struct task_group *tg, bool idle) · 7 unmodified lines
4355	void scx_group_set_bandwidth(struct task_group *tg,	4421	void scx_group_set_bandwidth(struct task_group *tg,
4356	u64 period_us, u64 quota_us, u64 burst_us)	4422	u64 period_us, u64 quota_us, u64 burst_us)
4357	{	4423	{
4358	struct scx_sched *sch = scx_root;	4424	struct scx_sched *sch;
4359		4425
4360	percpu_down_read(&scx_cgroup_ops_rwsem);	4426	percpu_down_read(&scx_cgroup_ops_rwsem);
		4427	sch = scx_root;
4361		4428
4362	if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) &&	4429	if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) &&
4363	(tg->scx.bw_period_us != period_us \|\|	4430	(tg->scx.bw_period_us != period_us \|\|
@@ -4380,21 +4447,6 @@ static struct cgroup *root_cgroup(void) · 16 unmodified lines
4380	return &cgrp_dfl_root.cgrp;	4447	return &cgrp_dfl_root.cgrp;
4381	}	4448	}
4382		4449
4383	static struct cgroup sch_cgroup(struct scx_sched sch)
4384	{
4385	return sch->cgrp;
4386	}
4387
4388	/* for each descendant of @cgrp including self, set ->scx_sched to @sch */
4389	static void set_cgroup_sched(struct cgroup cgrp, struct scx_sched sch)
4390	{
4391	struct cgroup *pos;
4392	struct cgroup_subsys_state *css;
4393
4394	cgroup_for_each_live_descendant_pre(pos, css, cgrp)
4395	rcu_assign_pointer(pos->scx_sched, sch);
4396	}
4397
4398	static void scx_cgroup_lock(void)	4450	static void scx_cgroup_lock(void)
4399	{	4451	{
4400	#ifdef CONFIG_EXT_GROUP_SCHED	4452	#ifdef CONFIG_EXT_GROUP_SCHED
@@ -4412,12 +4464,30 @@ static void scx_cgroup_unlock(void) · 11 unmodified lines
4412	}	4464	}
4413	#else /* CONFIG_EXT_GROUP_SCHED \|\| CONFIG_EXT_SUB_SCHED */	4465	#else /* CONFIG_EXT_GROUP_SCHED \|\| CONFIG_EXT_SUB_SCHED */
4414	static struct cgroup *root_cgroup(void) { return NULL; }	4466	static struct cgroup *root_cgroup(void) { return NULL; }
4415	static struct cgroup sch_cgroup(struct scx_sched sch) { return NULL; }
4416	static void set_cgroup_sched(struct cgroup cgrp, struct scx_sched sch) {}
4417	static void scx_cgroup_lock(void) {}	4467	static void scx_cgroup_lock(void) {}
4418	static void scx_cgroup_unlock(void) {}	4468	static void scx_cgroup_unlock(void) {}
4419	#endif /* CONFIG_EXT_GROUP_SCHED \|\| CONFIG_EXT_SUB_SCHED */	4469	#endif /* CONFIG_EXT_GROUP_SCHED \|\| CONFIG_EXT_SUB_SCHED */
4420		4470
		4471	#ifdef CONFIG_EXT_SUB_SCHED
		4472	static struct cgroup sch_cgroup(struct scx_sched sch)
		4473	{
		4474	return sch->cgrp;
		4475	}
		4476
		4477	/* for each descendant of @cgrp including self, set ->scx_sched to @sch */
		4478	static void set_cgroup_sched(struct cgroup cgrp, struct scx_sched sch)
		4479	{
		4480	struct cgroup *pos;
		4481	struct cgroup_subsys_state *css;
		4482
		4483	cgroup_for_each_live_descendant_pre(pos, css, cgrp)
		4484	rcu_assign_pointer(pos->scx_sched, sch);
		4485	}
		4486	#else /* CONFIG_EXT_SUB_SCHED */
		4487	static struct cgroup sch_cgroup(struct scx_sched sch) { return NULL; }
		4488	static void set_cgroup_sched(struct cgroup cgrp, struct scx_sched sch) {}
		4489	#endif /* CONFIG_EXT_SUB_SCHED */
		4490
4421	/*	4491	/*
4422	* Omitted operations:	4492	* Omitted operations:
4423	*	4493	*
@@ -4712,6 +4782,8 @@ static void scx_sched_free_rcu_work(struct work_struct *work) · 288 unmodified lines
4712	irq_work_sync(&sch->disable_irq_work);	4782	irq_work_sync(&sch->disable_irq_work);
4713	kthread_destroy_worker(sch->helper);	4783	kthread_destroy_worker(sch->helper);
4714	timer_shutdown_sync(&sch->bypass_lb_timer);	4784	timer_shutdown_sync(&sch->bypass_lb_timer);
		4785	free_cpumask_var(sch->bypass_lb_donee_cpumask);
		4786	free_cpumask_var(sch->bypass_lb_resched_cpumask);
4715		4787
4716	#ifdef CONFIG_EXT_SUB_SCHED	4788	#ifdef CONFIG_EXT_SUB_SCHED
4717	kfree(sch->cgrp_path);	4789	kfree(sch->cgrp_path);
@@ -4938,6 +5010,25 @@ void scx_softlockup(u32 dur_s) · 220 unmodified lines
4938	smp_processor_id(), dur_s);	5010	smp_processor_id(), dur_s);
4939	}	5011	}
4940		5012
		5013	/*
		5014	* scx_hardlockup() runs from NMI and eventually calls scx_claim_exit(),
		5015	* which takes scx_sched_lock. scx_sched_lock isn't NMI-safe and grabbing
		5016	* it from NMI context can lead to deadlocks. Defer via irq_work; the
		5017	* disable path runs off irq_work anyway.
		5018	*/
		5019	static atomic_t scx_hardlockup_cpu = ATOMIC_INIT(-1);
		5020
		5021	static void scx_hardlockup_irq_workfn(struct irq_work *work)
		5022	{
		5023	int cpu = atomic_xchg(&scx_hardlockup_cpu, -1);
		5024
		5025	if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu))
		5026	printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n",
		5027	cpu);
		5028	}
		5029
		5030	static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn);
		5031
4941	/**	5032	/**
4942	* scx_hardlockup - sched_ext hardlockup handler	5033	* scx_hardlockup - sched_ext hardlockup handler
4943	*	5034	*
@@ -4946,17 +5037,19 @@ void scx_softlockup(u32 dur_s) · 2 unmodified lines
4946	* Try kicking out the current scheduler in an attempt to recover the system to	5037	* Try kicking out the current scheduler in an attempt to recover the system to
4947	* a good state before taking more drastic actions.	5038	* a good state before taking more drastic actions.
4948	*	5039	*
4949	* Returns %true if sched_ext is enabled and abort was initiated, which may	5040	* Queues an irq_work; the handle_lockup() call happens in IRQ context (see
4950	* resolve the reported hardlockup. %false if sched_ext is not enabled or	5041	* scx_hardlockup_irq_workfn).
4951	* someone else already initiated abort.	5042	*
		5043	* Returns %true if sched_ext is enabled and the work was queued, %false
		5044	* otherwise.
4952	*/	5045	*/
4953	bool scx_hardlockup(int cpu)	5046	bool scx_hardlockup(int cpu)
4954	{	5047	{
4955	if (!handle_lockup("hard lockup - CPU %d", cpu))	5048	if (!rcu_access_pointer(scx_root))
4956	return false;	5049	return false;
4957		5050
4958	printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n",	5051	atomic_cmpxchg(&scx_hardlockup_cpu, -1, cpu);
4959	cpu);	5052	irq_work_queue(&scx_hardlockup_irq_work);
4960	return true;	5053	return true;
4961	}	5054	}
4962		5055
@@ -5000,6 +5093,15 @@ resume: · 37 unmodified lines
5000	if (cpumask_empty(donee_mask))	5093	if (cpumask_empty(donee_mask))
5001	break;	5094	break;
5002		5095
		5096	/*
		5097	* If an earlier pass placed @p on @donor_dsq from a different
		5098	* CPU and the donee hasn't consumed it yet, @p is still on the
		5099	* previous CPU and task_rq(@p) != @donor_rq. @p can't be moved
		5100	* without its rq locked. Skip.
		5101	*/
		5102	if (task_rq(p) != donor_rq)
		5103	continue;
		5104
5003	donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr);	5105	donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr);
5004	if (donee >= nr_cpu_ids)	5106	if (donee >= nr_cpu_ids)
5005	continue;	5107	continue;
@@ -5058,8 +5160,8 @@ resume: · 52 unmodified lines
5058	static void bypass_lb_node(struct scx_sched *sch, int node)	5160	static void bypass_lb_node(struct scx_sched *sch, int node)
5059	{	5161	{
5060	const struct cpumask *node_mask = cpumask_of_node(node);	5162	const struct cpumask *node_mask = cpumask_of_node(node);
5061	struct cpumask *donee_mask = scx_bypass_lb_donee_cpumask;	5163	struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask;
5062	struct cpumask *resched_mask = scx_bypass_lb_resched_cpumask;	5164	struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask;
5063	u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0;	5165	u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0;
5064	u32 nr_target, nr_donor_target;	5166	u32 nr_target, nr_donor_target;
5065	u32 before_min = U32_MAX, before_max = 0;	5167	u32 before_min = U32_MAX, before_max = 0;
@@ -5698,6 +5800,8 @@ static void scx_sub_disable(struct scx_sched *sch) · 632 unmodified lines
5698		5800
5699	if (sch->ops.exit)	5801	if (sch->ops.exit)
5700	SCX_CALL_OP(sch, exit, NULL, sch->exit_info);	5802	SCX_CALL_OP(sch, exit, NULL, sch->exit_info);
		5803	if (sch->sub_kset)
		5804	kset_unregister(sch->sub_kset);
5701	kobject_del(&sch->kobj);	5805	kobject_del(&sch->kobj);
5702	}	5806	}
5703	#else /* CONFIG_EXT_SUB_SCHED */	5807	#else /* CONFIG_EXT_SUB_SCHED */
@@ -5829,6 +5933,10 @@ static void scx_root_disable(struct scx_sched *sch) · 125 unmodified lines
5829	* could observe an object of the same name still in the hierarchy when	5933	* could observe an object of the same name still in the hierarchy when
5830	* the next scheduler is loaded.	5934	* the next scheduler is loaded.
5831	*/	5935	*/
		5936	#ifdef CONFIG_EXT_SUB_SCHED
		5937	if (sch->sub_kset)
		5938	kset_unregister(sch->sub_kset);
		5939	#endif
5832	kobject_del(&sch->kobj);	5940	kobject_del(&sch->kobj);
5833		5941
5834	free_kick_syncs();	5942	free_kick_syncs();
@@ -5921,6 +6029,25 @@ static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind) · 86 unmodified lines
5921	irq_work_queue(&sch->disable_irq_work);	6029	irq_work_queue(&sch->disable_irq_work);
5922	}	6030	}
5923		6031
		6032	/**
		6033	* scx_flush_disable_work - flush the disable work and wait for it to finish
		6034	* @sch: the scheduler
		6035	*
		6036	* sch->disable_work might still not queued, causing kthread_flush_work()
		6037	* as a noop. Syncing the irq_work first is required to guarantee the
		6038	* kthread work has been queued before waiting for it.
		6039	*/
		6040	static void scx_flush_disable_work(struct scx_sched *sch)
		6041	{
		6042	int kind;
		6043
		6044	do {
		6045	irq_work_sync(&sch->disable_irq_work);
		6046	kthread_flush_work(&sch->disable_work);
		6047	kind = atomic_read(&sch->exit_kind);
		6048	} while (kind != SCX_EXIT_NONE && kind != SCX_EXIT_DONE);
		6049	}
		6050
5924	static void dump_newline(struct seq_buf *s)	6051	static void dump_newline(struct seq_buf *s)
5925	{	6052	{
5926	trace_sched_ext_dump("");	6053	trace_sched_ext_dump("");
@@ -6032,9 +6159,8 @@ static void ops_dump_exit(void) · 105 unmodified lines
6032	scx_dump_data.cpu = -1;	6159	scx_dump_data.cpu = -1;
6033	}	6160	}
6034		6161
6035	static void scx_dump_task(struct scx_sched *sch,	6162	static void scx_dump_task(struct scx_sched sch, struct seq_buf s, struct scx_dump_ctx *dctx,
6036	struct seq_buf s, struct scx_dump_ctx dctx,	6163	struct rq rq, struct task_struct p, char marker)
6037	struct task_struct *p, char marker)
6038	{	6164	{
6039	static unsigned long bt[SCX_EXIT_BT_LEN];	6165	static unsigned long bt[SCX_EXIT_BT_LEN];
6040	struct scx_sched *task_sch = scx_task_sched(p);	6166	struct scx_sched *task_sch = scx_task_sched(p);
@@ -6075,7 +6201,7 @@ static void scx_dump_task(struct scx_sched *sch, · 34 unmodified lines
6075		6201
6076	if (SCX_HAS_OP(sch, dump_task)) {	6202	if (SCX_HAS_OP(sch, dump_task)) {
6077	ops_dump_init(s, " ");	6203	ops_dump_init(s, " ");
6078	SCX_CALL_OP(sch, dump_task, NULL, dctx, p);	6204	SCX_CALL_OP(sch, dump_task, rq, dctx, p);
6079	ops_dump_exit();	6205	ops_dump_exit();
6080	}	6206	}
6081		6207
@@ -6199,8 +6325,7 @@ static void scx_dump_state(struct scx_sched sch, struct scx_exit_info ei, · 117 unmodified lines
6199	used = seq_buf_used(&ns);	6325	used = seq_buf_used(&ns);
6200	if (SCX_HAS_OP(sch, dump_cpu)) {	6326	if (SCX_HAS_OP(sch, dump_cpu)) {
6201	ops_dump_init(&ns, " ");	6327	ops_dump_init(&ns, " ");
6202	SCX_CALL_OP(sch, dump_cpu, NULL,	6328	SCX_CALL_OP(sch, dump_cpu, rq, &dctx, cpu, idle);
6203	&dctx, cpu, idle);
6204	ops_dump_exit();	6329	ops_dump_exit();
6205	}	6330	}
6206		6331
@@ -6223,11 +6348,11 @@ static void scx_dump_state(struct scx_sched sch, struct scx_exit_info ei, · 16 unmodified lines
6223		6348
6224	if (rq->curr->sched_class == &ext_sched_class &&	6349	if (rq->curr->sched_class == &ext_sched_class &&
6225	(dump_all_tasks \|\| scx_task_on_sched(sch, rq->curr)))	6350	(dump_all_tasks \|\| scx_task_on_sched(sch, rq->curr)))
6226	scx_dump_task(sch, &s, &dctx, rq->curr, '*');	6351	scx_dump_task(sch, &s, &dctx, rq, rq->curr, '*');
6227		6352
6228	list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)	6353	list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)
6229	if (dump_all_tasks \|\| scx_task_on_sched(sch, p))	6354	if (dump_all_tasks \|\| scx_task_on_sched(sch, p))
6230	scx_dump_task(sch, &s, &dctx, p, ' ');	6355	scx_dump_task(sch, &s, &dctx, rq, p, ' ');
6231	next:	6356	next:
6232	rq_unlock_irqrestore(rq, &rf);	6357	rq_unlock_irqrestore(rq, &rf);
6233	}	6358	}
@@ -6437,6 +6562,15 @@ static struct scx_sched scx_alloc_and_add_sched(struct sched_ext_ops ops, · 203 unmodified lines
6437	init_irq_work(&sch->disable_irq_work, scx_disable_irq_workfn);	6562	init_irq_work(&sch->disable_irq_work, scx_disable_irq_workfn);
6438	kthread_init_work(&sch->disable_work, scx_disable_workfn);	6563	kthread_init_work(&sch->disable_work, scx_disable_workfn);
6439	timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0);	6564	timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0);
		6565
		6566	if (!alloc_cpumask_var(&sch->bypass_lb_donee_cpumask, GFP_KERNEL)) {
		6567	ret = -ENOMEM;
		6568	goto err_stop_helper;
		6569	}
		6570	if (!alloc_cpumask_var(&sch->bypass_lb_resched_cpumask, GFP_KERNEL)) {
		6571	ret = -ENOMEM;
		6572	goto err_free_lb_cpumask;
		6573	}
6440	sch->ops = *ops;	6574	sch->ops = *ops;
6441	rcu_assign_pointer(ops->priv, sch);	6575	rcu_assign_pointer(ops->priv, sch);
6442		6576
@@ -6446,14 +6580,14 @@ static struct scx_sched scx_alloc_and_add_sched(struct sched_ext_ops ops, · 3 unmodified lines
6446	char *buf = kzalloc(PATH_MAX, GFP_KERNEL);	6580	char *buf = kzalloc(PATH_MAX, GFP_KERNEL);
6447	if (!buf) {	6581	if (!buf) {
6448	ret = -ENOMEM;	6582	ret = -ENOMEM;
6449	goto err_stop_helper;	6583	goto err_free_lb_resched;
6450	}	6584	}
6451	cgroup_path(cgrp, buf, PATH_MAX);	6585	cgroup_path(cgrp, buf, PATH_MAX);
6452	sch->cgrp_path = kstrdup(buf, GFP_KERNEL);	6586	sch->cgrp_path = kstrdup(buf, GFP_KERNEL);
6453	kfree(buf);	6587	kfree(buf);
6454	if (!sch->cgrp_path) {	6588	if (!sch->cgrp_path) {
6455	ret = -ENOMEM;	6589	ret = -ENOMEM;
6456	goto err_stop_helper;	6590	goto err_free_lb_resched;
6457	}	6591	}
6458		6592
6459	sch->cgrp = cgrp;	6593	sch->cgrp = cgrp;
@@ -6488,10 +6622,12 @@ static struct scx_sched scx_alloc_and_add_sched(struct sched_ext_ops ops, · 28 unmodified lines
6488	#endif /* CONFIG_EXT_SUB_SCHED */	6622	#endif /* CONFIG_EXT_SUB_SCHED */
6489	return sch;	6623	return sch;
6490		6624
6491	#ifdef CONFIG_EXT_SUB_SCHED	6625	err_free_lb_resched:
		6626	free_cpumask_var(sch->bypass_lb_resched_cpumask);
		6627	err_free_lb_cpumask:
		6628	free_cpumask_var(sch->bypass_lb_donee_cpumask);
6492	err_stop_helper:	6629	err_stop_helper:
6493	kthread_destroy_worker(sch->helper);	6630	kthread_destroy_worker(sch->helper);
6494	#endif
6495	err_free_pcpu:	6631	err_free_pcpu:
6496	for_each_possible_cpu(cpu) {	6632	for_each_possible_cpu(cpu) {
6497	if (cpu == bypass_fail_cpu)	6633	if (cpu == bypass_fail_cpu)
@@ -6510,7 +6646,7 @@ err_free_ei: · 12 unmodified lines
6510	err_free_sch:	6646	err_free_sch:
6511	kfree(sch);	6647	kfree(sch);
6512	err_put_cgrp:	6648	err_put_cgrp:
6513	#if defined(CONFIG_EXT_GROUP_SCHED) \|\| defined(CONFIG_EXT_SUB_SCHED)	6649	#ifdef CONFIG_EXT_SUB_SCHED
6514	cgroup_put(cgrp);	6650	cgroup_put(cgrp);
6515	#endif	6651	#endif
6516	return ERR_PTR(ret);	6652	return ERR_PTR(ret);
@@ -6601,7 +6737,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) · 84 unmodified lines
6601	if (ret)	6737	if (ret)
6602	goto err_unlock;	6738	goto err_unlock;
6603		6739
6604	#if defined(CONFIG_EXT_GROUP_SCHED) \|\| defined(CONFIG_EXT_SUB_SCHED)	6740	#ifdef CONFIG_EXT_SUB_SCHED
6605	cgroup_get(cgrp);	6741	cgroup_get(cgrp);
6606	#endif	6742	#endif
6607	sch = scx_alloc_and_add_sched(ops, cgrp, NULL);	6743	sch = scx_alloc_and_add_sched(ops, cgrp, NULL);
@@ -6639,8 +6775,10 @@ static void scx_root_enable_workfn(struct kthread_work *work) · 31 unmodified lines
6639	rcu_assign_pointer(scx_root, sch);	6775	rcu_assign_pointer(scx_root, sch);
6640		6776
6641	ret = scx_link_sched(sch);	6777	ret = scx_link_sched(sch);
6642	if (ret)	6778	if (ret) {
		6779	cpus_read_unlock();
6643	goto err_disable;	6780	goto err_disable;
		6781	}
6644		6782
6645	scx_idle_enable(ops);	6783	scx_idle_enable(ops);
6646		6784
@@ -6821,7 +6959,7 @@ err_disable: · 174 unmodified lines
6821	* completion. sch's base reference will be put by bpf_scx_unreg().	6959	* completion. sch's base reference will be put by bpf_scx_unreg().
6822	*/	6960	*/
6823	scx_error(sch, "scx_root_enable() failed (%d)", ret);	6961	scx_error(sch, "scx_root_enable() failed (%d)", ret);
6824	kthread_flush_work(&sch->disable_work);	6962	scx_flush_disable_work(sch);
6825	cmd->ret = 0;	6963	cmd->ret = 0;
6826	}	6964	}
6827		6965
@@ -7072,23 +7210,30 @@ out_unlock: · 244 unmodified lines
7072	abort:	7210	abort:
7073	put_task_struct(p);	7211	put_task_struct(p);
7074	scx_task_iter_stop(&sti);	7212	scx_task_iter_stop(&sti);
7075	scx_enabling_sub_sched = NULL;
7076		7213
		7214	/*
		7215	* Undo __scx_init_task() for tasks we marked. scx_enable_task() never
		7216	* ran for @sch on them, so calling scx_disable_task() here would invoke
		7217	* ops.disable() without a matching ops.enable(). scx_enabling_sub_sched
		7218	* must stay set until SUB_INIT is cleared from every marked task -
		7219	* scx_disable_and_exit_task() reads it when a task exits concurrently.
		7220	*/
7077	scx_task_iter_start(&sti, sch->cgrp);	7221	scx_task_iter_start(&sti, sch->cgrp);
7078	while ((p = scx_task_iter_next_locked(&sti))) {	7222	while ((p = scx_task_iter_next_locked(&sti))) {
7079	if (p->scx.flags & SCX_TASK_SUB_INIT) {	7223	if (p->scx.flags & SCX_TASK_SUB_INIT) {
7080	__scx_disable_and_exit_task(sch, p);	7224	scx_sub_init_cancel_task(sch, p);
7081	p->scx.flags &= ~SCX_TASK_SUB_INIT;	7225	p->scx.flags &= ~SCX_TASK_SUB_INIT;
7082	}	7226	}
7083	}	7227	}
7084	scx_task_iter_stop(&sti);	7228	scx_task_iter_stop(&sti);
		7229	scx_enabling_sub_sched = NULL;
7085	err_unlock_and_disable:	7230	err_unlock_and_disable:
7086	/* we'll soon enter disable path, keep bypass on */	7231	/* we'll soon enter disable path, keep bypass on */
7087	scx_cgroup_unlock();	7232	scx_cgroup_unlock();
7088	percpu_up_write(&scx_fork_rwsem);	7233	percpu_up_write(&scx_fork_rwsem);
7089	err_disable:	7234	err_disable:
7090	mutex_unlock(&scx_enable_mutex);	7235	mutex_unlock(&scx_enable_mutex);
7091	kthread_flush_work(&sch->disable_work);	7236	scx_flush_disable_work(sch);
7092	cmd->ret = 0;	7237	cmd->ret = 0;
7093	}	7238	}
7094		7239
@@ -7349,7 +7494,7 @@ static void bpf_scx_unreg(void kdata, struct bpf_link link) · 254 unmodified lines
7349	struct scx_sched *sch = rcu_dereference_protected(ops->priv, true);	7494	struct scx_sched *sch = rcu_dereference_protected(ops->priv, true);
7350		7495
7351	scx_disable(sch, SCX_EXIT_UNREG);	7496	scx_disable(sch, SCX_EXIT_UNREG);
7352	kthread_flush_work(&sch->disable_work);	7497	scx_flush_disable_work(sch);
7353	RCU_INIT_POINTER(ops->priv, NULL);	7498	RCU_INIT_POINTER(ops->priv, NULL);
7354	kobject_put(&sch->kobj);	7499	kobject_put(&sch->kobj);
7355	}	7500	}
@@ -8033,12 +8178,22 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, · 677 unmodified lines
8033	struct task_struct *p, u64 dsq_id, u64 enq_flags)	8178	struct task_struct *p, u64 dsq_id, u64 enq_flags)
8034	{	8179	{
8035	struct scx_dispatch_q src_dsq = kit->dsq, dst_dsq;	8180	struct scx_dispatch_q src_dsq = kit->dsq, dst_dsq;
8036	struct scx_sched *sch = src_dsq->sched;	8181	struct scx_sched *sch;
8037	struct rq this_rq, src_rq, *locked_rq;	8182	struct rq this_rq, src_rq, *locked_rq;
8038	bool dispatched = false;	8183	bool dispatched = false;
8039	bool in_balance;	8184	bool in_balance;
8040	unsigned long flags;	8185	unsigned long flags;
8041		8186
		8187	/*
		8188	* The verifier considers an iterator slot initialized on any
		8189	* KF_ITER_NEW return, so a BPF program may legally reach here after
		8190	* bpf_iter_scx_dsq_new() failed and left @kit->dsq NULL.
		8191	*/
		8192	if (unlikely(!src_dsq))
		8193	return false;
		8194
		8195	sch = src_dsq->sched;
		8196
8042	if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags))	8197	if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags))
8043	return false;	8198	return false;
8044		8199
@@ -8526,7 +8681,7 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, · 481 unmodified lines
8526		8681
8527	guard(rcu)();	8682	guard(rcu)();
8528	sch = scx_prog_sched(aux);	8683	sch = scx_prog_sched(aux);
8529	if (unlikely(!scx_task_on_sched(sch, p)))	8684	if (unlikely(!sch \|\| !scx_task_on_sched(sch, p)))
8530	return false;	8685	return false;
8531		8686
8532	p->scx.slice = slice;	8687	p->scx.slice = slice;
@@ -8549,7 +8704,7 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, · 16 unmodified lines
8549		8704
8550	guard(rcu)();	8705	guard(rcu)();
8551	sch = scx_prog_sched(aux);	8706	sch = scx_prog_sched(aux);
8552	if (unlikely(!scx_task_on_sched(sch, p)))	8707	if (unlikely(!sch \|\| !scx_task_on_sched(sch, p)))
8553	return false;	8708	return false;
8554		8709
8555	p->scx.dsq_vtime = vtime;	8710	p->scx.dsq_vtime = vtime;
@@ -8633,11 +8788,12 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux · 77 unmodified lines
8633	/**	8788	/**
8634	* scx_bpf_dsq_nr_queued - Return the number of queued tasks	8789	* scx_bpf_dsq_nr_queued - Return the number of queued tasks
8635	* @dsq_id: id of the DSQ	8790	* @dsq_id: id of the DSQ
		8791	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
8636	*	8792	*
8637	* Return the number of tasks in the DSQ matching @dsq_id. If not found,	8793	* Return the number of tasks in the DSQ matching @dsq_id. If not found,
8638	* -%ENOENT is returned.	8794	* -%ENOENT is returned.
8639	*/	8795	*/
8640	__bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id)	8796	__bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux)
8641	{	8797	{
8642	struct scx_sched *sch;	8798	struct scx_sched *sch;
8643	struct scx_dispatch_q *dsq;	8799	struct scx_dispatch_q *dsq;
@@ -8645,7 +8801,7 @@ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id) · 1 unmodified line
8645		8801
8646	preempt_disable();	8802	preempt_disable();
8647		8803
8648	sch = rcu_dereference_sched(scx_root);	8804	sch = scx_prog_sched(aux);
8649	if (unlikely(!sch)) {	8805	if (unlikely(!sch)) {
8650	ret = -ENODEV;	8806	ret = -ENODEV;
8651	goto out;	8807	goto out;
@@ -8677,21 +8833,21 @@ out: · 25 unmodified lines
8677	/**	8833	/**
8678	* scx_bpf_destroy_dsq - Destroy a custom DSQ	8834	* scx_bpf_destroy_dsq - Destroy a custom DSQ
8679	* @dsq_id: DSQ to destroy	8835	* @dsq_id: DSQ to destroy
		8836	* @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
8680	*	8837	*
8681	* Destroy the custom DSQ identified by @dsq_id. Only DSQs created with	8838	* Destroy the custom DSQ identified by @dsq_id. Only DSQs created with
8682	* scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is	8839	* scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is
8683	* empty and no further tasks are dispatched to it. Ignored if called on a DSQ	8840	* empty and no further tasks are dispatched to it. Ignored if called on a DSQ
8684	* which doesn't exist. Can be called from any online scx_ops operations.	8841	* which doesn't exist. Can be called from any online scx_ops operations.
8685	*/	8842	*/
8686	__bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id)	8843	__bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux)
8687	{	8844	{
8688	struct scx_sched *sch;	8845	struct scx_sched *sch;
8689		8846
8690	rcu_read_lock();	8847	guard(rcu)();
8691	sch = rcu_dereference(scx_root);	8848	sch = scx_prog_sched(aux);
8692	if (sch)	8849	if (sch)
8693	destroy_dsq(sch, dsq_id);	8850	destroy_dsq(sch, dsq_id);
8694	rcu_read_unlock();
8695	}	8851	}
8696		8852
8697	/**	8853	/**
@@ -9445,8 +9601,8 @@ BTF_KFUNCS_START(scx_kfunc_ids_any) · 747 unmodified lines
9445	BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS \| KF_RCU);	9601	BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS \| KF_RCU);
9446	BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS \| KF_RCU);	9602	BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS \| KF_RCU);
9447	BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)	9603	BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)
9448	BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued)	9604	BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS)
9449	BTF_ID_FLAGS(func, scx_bpf_destroy_dsq)	9605	BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS)
9450	BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS \| KF_RCU_PROTECTED \| KF_RET_NULL)	9606	BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS \| KF_RCU_PROTECTED \| KF_RET_NULL)
9451	BTF_ID_FLAGS(func, scx_bpf_dsq_reenq, KF_IMPLICIT_ARGS)	9607	BTF_ID_FLAGS(func, scx_bpf_dsq_reenq, KF_IMPLICIT_ARGS)
9452	BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2, KF_IMPLICIT_ARGS)	9608	BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2, KF_IMPLICIT_ARGS)
@@ -9479,6 +9635,7 @@ BTF_KFUNCS_END(scx_kfunc_ids_any) · 26 unmodified lines
9479	static const struct btf_kfunc_id_set scx_kfunc_set_any = {	9635	static const struct btf_kfunc_id_set scx_kfunc_set_any = {
9480	.owner = THIS_MODULE,	9636	.owner = THIS_MODULE,
9481	.set = &scx_kfunc_ids_any,	9637	.set = &scx_kfunc_ids_any,
		9638	.filter = scx_kfunc_context_filter,
9482	};	9639	};
9483		9640
9484	/*	9641	/*
@@ -9526,13 +9683,12 @@ static const u32 scx_kf_allow_flags[] = { · 41 unmodified lines
9526	};	9683	};
9527		9684
9528	/*	9685	/*
9529	* Verifier-time filter for context-sensitive SCX kfuncs. Registered via the	9686	* Verifier-time filter for SCX kfuncs. Registered via the .filter field on
9530	* .filter field on each per-group btf_kfunc_id_set. The BPF core invokes this	9687	* each per-group btf_kfunc_id_set. The BPF core invokes this for every kfunc
9531	* for every kfunc call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or	9688	* call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or
9532	* BPF_PROG_TYPE_SYSCALL), regardless of which set originally introduced the	9689	* BPF_PROG_TYPE_SYSCALL), regardless of which set originally introduced the
9533	* kfunc - so the filter must short-circuit on kfuncs it doesn't govern (e.g.	9690	* kfunc - so the filter must short-circuit on kfuncs it doesn't govern by
9534	* scx_kfunc_ids_any) by falling through to "allow" when none of the	9691	* falling through to "allow" when none of the SCX sets contain the kfunc.
9535	* context-sensitive sets contain the kfunc.
9536	*/	9692	*/
9537	int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)	9693	int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)
9538	{	9694	{
@@ -9541,18 +9697,21 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) · 2 unmodified lines
9541	bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id);	9697	bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id);
9542	bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id);	9698	bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id);
9543	bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id);	9699	bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id);
		9700	bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id);
		9701	bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id);
9544	u32 moff, flags;	9702	u32 moff, flags;
9545		9703
9546	/* Not a context-sensitive kfunc (e.g. from scx_kfunc_ids_any) - allow. */	9704	/* Not an SCX kfunc - allow. */
9547	if (!(in_unlocked \|\| in_select_cpu \|\| in_enqueue \|\| in_dispatch \|\| in_cpu_release))	9705	if (!(in_unlocked \|\| in_select_cpu \|\| in_enqueue \|\| in_dispatch \|\|
		9706	in_cpu_release \|\| in_idle \|\| in_any))
9548	return 0;	9707	return 0;
9549		9708
9550	/* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */	9709	/* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */
9551	if (prog->type == BPF_PROG_TYPE_SYSCALL)	9710	if (prog->type == BPF_PROG_TYPE_SYSCALL)
9552	return (in_unlocked \|\| in_select_cpu) ? 0 : -EACCES;	9711	return (in_unlocked \|\| in_select_cpu \|\| in_idle \|\| in_any) ? 0 : -EACCES;
9553		9712
9554	if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)	9713	if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
9555	return -EACCES;	9714	return (in_any \|\| in_idle) ? 0 : -EACCES;
9556		9715
9557	/*	9716	/*
9558	* add_subprog_and_kfunc() collects all kfunc calls, including dead code	9717	* add_subprog_and_kfunc() collects all kfunc calls, including dead code
@@ -9565,14 +9724,15 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) · 6 unmodified lines
9565	return 0;	9724	return 0;
9566		9725
9567	/*	9726	/*
9568	* Non-SCX struct_ops: only unlocked kfuncs are safe. The other	9727	* Non-SCX struct_ops: SCX kfuncs are not permitted.
9569	* context-sensitive kfuncs assume the rq lock is held by the SCX
9570	* dispatch path, which doesn't apply to other struct_ops users.
9571	*/	9728	*/
9572	if (prog->aux->st_ops != &bpf_sched_ext_ops)	9729	if (prog->aux->st_ops != &bpf_sched_ext_ops)
9573	return in_unlocked ? 0 : -EACCES;	9730	return -EACCES;
9574		9731
9575	/* SCX struct_ops: check the per-op allow list. */	9732	/* SCX struct_ops: check the per-op allow list. */
		9733	if (in_any \|\| in_idle)
		9734	return 0;
		9735
9576	moff = prog->aux->attach_st_ops_member_off;	9736	moff = prog->aux->attach_st_ops_member_off;
9577	flags = scx_kf_allow_flags[SCX_MOFF_IDX(moff)];	9737	flags = scx_kf_allow_flags[SCX_MOFF_IDX(moff)];
9578		9738
@@ -9656,12 +9816,6 @@ static int __init scx_init(void) · 77 unmodified lines
9656	return ret;	9816	return ret;
9657	}	9817	}
9658		9818
9659	if (!alloc_cpumask_var(&scx_bypass_lb_donee_cpumask, GFP_KERNEL) \|\|
9660	!alloc_cpumask_var(&scx_bypass_lb_resched_cpumask, GFP_KERNEL)) {
9661	pr_err("sched_ext: Failed to allocate cpumasks\n");
9662	return -ENOMEM;
9663	}
9664
9665	return 0;	9819	return 0;
9666	}	9820	}
9667	__initcall(scx_init);	9821	__initcall(scx_init);

kernel/sched/ext_idle.c+18−2

 	 * Accessing p->cpus_ptr / p->nr_cpus_allowed needs either @p's rq
 	 * lock or @p's pi_lock. Three cases:
 	 *
-	 *  - inside ops.select_cpu(): try_to_wake_up() holds @p's pi_lock.
+	 *  - inside ops.select_cpu(): try_to_wake_up() holds the wake-up
+	 *    task's pi_lock; the wake-up task is recorded in kf_tasks[0]
+	 *    by SCX_CALL_OP_TASK_RET().
 	 *  - other rq-locked SCX op: scx_locked_rq() points at the held rq.
 	 *  - truly unlocked (UNLOCKED ops, SYSCALL, non-SCX struct_ops):
 	 *    nothing held, take pi_lock ourselves.
+	 *
+	 * In the first two cases, BPF schedulers may pass an arbitrary task
+	 * that the held lock doesn't cover. Refuse those.
 	 */
 	if (this_rq()->scx.in_select_cpu) {
+		if (!scx_kf_arg_task_ok(sch, p))
+			return -EINVAL;
 		lockdep_assert_held(&p->pi_lock);
-	} else if (!scx_locked_rq()) {
+	} else if (scx_locked_rq()) {
+		if (task_rq(p) != scx_locked_rq())
+			goto cross_task;
+	} else {
 		raw_spin_lock_irqsave(&p->pi_lock, irq_flags);
 		we_locked = true;
 	}
 		raw_spin_unlock_irqrestore(&p->pi_lock, irq_flags);
 	return cpu;
+cross_task:
+	scx_error(sch, "select_cpu kfunc called cross-task on %s[%d]",
+		  p->comm, p->pid);
+	return -EINVAL;
 }
 /**
 static const struct btf_kfunc_id_set scx_kfunc_set_idle = {
 	.owner			= THIS_MODULE,
 	.set			= &scx_kfunc_ids_idle,
+	.filter			= scx_kfunc_context_filter,
 };
 /*

kernel/sched/ext_idle.h+1−0

@@ -12,6 +12,7 @@ · 11 unmodified lines
12		12
13	struct sched_ext_ops;	13	struct sched_ext_ops;
14		14
		15	extern struct btf_id_set8 scx_kfunc_ids_idle;
15	extern struct btf_id_set8 scx_kfunc_ids_select_cpu;	16	extern struct btf_id_set8 scx_kfunc_ids_select_cpu;
16		17
17	void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops);	18	void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops);

kernel/sched/ext_internal.h+2−0

@@ -1075,6 +1075,8 @@ struct scx_sched { · 1074 unmodified lines
1075	struct irq_work disable_irq_work;	1075	struct irq_work disable_irq_work;
1076	struct kthread_work disable_work;	1076	struct kthread_work disable_work;
1077	struct timer_list bypass_lb_timer;	1077	struct timer_list bypass_lb_timer;
		1078	cpumask_var_t bypass_lb_donee_cpumask;
		1079	cpumask_var_t bypass_lb_resched_cpumask;
1078	struct rcu_work rcu_work;	1080	struct rcu_work rcu_work;
1079		1081
1080	/* all ancestors including self */	1082	/* all ancestors including self */