Merge tag 'sched_ext-for-7.1-rc1-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext

Pull sched_ext fixes from Tejun Heo: "The merge window pulled in the cgroup sub-scheduler infrastructure, and new AI reviews are accelerating bug reporting and fixing - hence the larger than usual fixes batch: - Use-after-frees during scheduler load/unload: - The disable path could free the BPF scheduler while deferred irq_work / kthread work was still in flight - cgroup setter callbacks read the active scheduler outside the rwsem that synchronizes against teardown Fix both, and reuse the disable drain in the enable error paths so the BPF JIT page can't be freed under live callbacks. - Several BPF op invocations didn't tell the framework which runqueue was already locked, so helper kfuncs that re-acquire the runqueue by CPU could deadlock on the held lock Fix the affected callsites, including recursive parent-into-child dispatch. - The hardlockup notifier ran from NMI but eventually took a non-NMI-safe lock. Bounce it through irq_work. - A handful of bugs in the new sub-scheduler hierarchy: - helper kfuncs hard-coded the root instead of resolving the caller's scheduler - the enable error path tried to disable per-task state that had never been initialized, and leaked cpus_read_lock on the way out - a sysfs object was leaked on every load/unload - the dispatch fast-path used the root scheduler instead of the task's - a couple of CONFIG #ifdef guards were misclassified - Verifier-time hardening: BPF programs of unrelated struct_ops types (e.g. tcp_congestion_ops) could call sched_ext kfuncs - a semantic bug and, once sub-sched was enabled, a KASAN out-of-bounds read. Now rejected at load. Plus a few NULL and cross-task argument checks on sched_ext kfuncs, and a selftest covering the new deny. - rhashtable (Herbert): restore the insecure_elasticity toggle and bounce the deferred-resize kick through irq_work to break a lock-order cycle observable from raw-spinlock callers. sched_ext's scheduler-instance hash is the first user of both. - The bypass-mode load balancer used file-scope cpumasks; with multiple scheduler instances now possible, those raced. Move to per-instance cpumasks, plus a follow-up to skip tasks whose recorded CPU is stale relative to the new owning runqueue. - Smaller fixes: - a dispatch queue's first-task tracking misbehaved when a parked iterator cursor sat in the list - the runqueue's next-class wasn't promoted on local-queue enqueue, leaving an SCX task behind RT in edge cases - the reference qmap scheduler stopped erroring on legitimate cross-scheduler task-storage misses" * tag 'sched_ext-for-7.1-rc1-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext: (26 commits) sched_ext: Fix scx_flush_disable_work() UAF race sched_ext: Call wakeup_preempt() in local_dsq_post_enq() sched_ext: Release cpus_read_lock on scx_link_sched() failure in root enable sched_ext: Reject NULL-sch callers in scx_bpf_task_set_slice/dsq_vtime sched_ext: Refuse cross-task select_cpu_from_kfunc calls sched_ext: Align cgroup #ifdef guards with SUB_SCHED vs GROUP_SCHED sched_ext: Make bypass LB cpumasks per-scheduler sched_ext: Pass held rq to SCX_CALL_OP() for core_sched_before sched_ext: Pass held rq to SCX_CALL_OP() for dump_cpu/dump_task sched_ext: Save and restore scx_locked_rq across SCX_CALL_OP sched_ext: Use dsq->first_task instead of list_empty() in dispatch_enqueue() FIFO-tail sched_ext: Resolve caller's scheduler in scx_bpf_destroy_dsq() / scx_bpf_dsq_nr_queued() sched_ext: Read scx_root under scx_cgroup_ops_rwsem in cgroup setters sched_ext: Don't disable tasks in scx_sub_enable_workfn() abort path sched_ext: Skip tasks with stale task_rq in bypass_lb_cpu() sched_ext: Guard scx_dsq_move() against NULL kit->dsq after failed iter_new sched_ext: Unregister sub_kset on scheduler disable sched_ext: Defer scx_hardlockup() out of NMI sched_ext: sync disable_irq_work in bpf_scx_unreg() sched_ext: Fix local_dsq_post_enq() to use task's scheduler in sub-sched ...

Linus Torvalds committed 6 months ago

Commit 664f0f62 parents dca922e d99f7a311 files · +436 −150Browse files

Unified · Split

11 files

include/linux/rhashtable-types.h +5 −0
include/linux/rhashtable.h +5 −3
kernel/sched/ext.c +276 −122
kernel/sched/ext_idle.c +18 −2
kernel/sched/ext_idle.h +1 −0
kernel/sched/ext_internal.h +2 −0
lib/rhashtable.c +31 −5
tools/sched_ext/scx_qmap.bpf.c +6 −18
tools/testing/selftests/sched_ext/Makefile +1 −0
tools/testing/selftests/sched_ext/non_scx_kfunc_deny.bpf.c +44 −0
tools/testing/selftests/sched_ext/non_scx_kfunc_deny.c +47 −0

include/linux/rhashtable-types.h+5−0

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	12	#include <linux/alloc_tag.h>
13	13	#include <linux/atomic.h>
14	14	#include <linux/compiler.h>
	15	+#include <linux/irq_work_types.h>
15	16	#include <linux/mutex.h>
16	17	#include <linux/workqueue_types.h>
17	18
@@ -49,6 +50,7 @@ typedef int (rht_obj_cmpfn_t)(struct rhashtable_compare_arg arg, · 31 unmodified lines
49	50	* @head_offset: Offset of rhash_head in struct to be hashed
50	51	* @max_size: Maximum size while expanding
51	52	* @min_size: Minimum size while shrinking
	53	+ * @insecure_elasticity: Set to true to disable chain length checks
52	54	* @automatic_shrinking: Enable automatic shrinking of tables
53	55	* @hashfn: Hash function (default: jhash2 if !(key_len % 4), or jhash)
54	56	* @obj_hashfn: Function to hash object
@@ -61,6 +63,7 @@ struct rhashtable_params { · 6 unmodified lines
61	63	u16 head_offset;
62	64	unsigned int max_size;
63	65	u16 min_size;
	66	+ bool insecure_elasticity;
64	67	bool automatic_shrinking;
65	68	rht_hashfn_t hashfn;
66	69	rht_obj_hashfn_t obj_hashfn;
@@ -75,6 +78,7 @@ struct rhashtable_params { · 8 unmodified lines
75	78	* @p: Configuration parameters
76	79	* @rhlist: True if this is an rhltable
77	80	* @run_work: Deferred worker to expand/shrink asynchronously
	81	+ * @run_irq_work: Bounces the @run_work kick through hard IRQ context.
78	82	* @mutex: Mutex to protect current/future table swapping
79	83	* @lock: Spin lock to protect walker list
80	84	* @nelems: Number of elements in table
@@ -86,6 +90,7 @@ struct rhashtable { · 5 unmodified lines
86	90	struct rhashtable_params p;
87	91	bool rhlist;
88	92	struct work_struct run_work;
	93	+ struct irq_work run_irq_work;
89	94	struct mutex mutex;
90	95	spinlock_t lock;
91	96	atomic_t nelems;

include/linux/rhashtable.h+5−3

@@ -20,6 +20,7 @@ · 19 unmodified lines
20	20
21	21	#include <linux/err.h>
22	22	#include <linux/errno.h>
	23	+#include <linux/irq_work.h>
23	24	#include <linux/jhash.h>
24	25	#include <linux/list_nulls.h>
25	26	#include <linux/workqueue.h>
@@ -821,14 +822,15 @@ slow_path: · 795 unmodified lines
821	822	goto out;
822	823	}
823	824
824		− if (elasticity <= 0)
	825	+ if (elasticity <= 0 && !params.insecure_elasticity)
825	826	goto slow_path;
826	827
827	828	data = ERR_PTR(-E2BIG);
828	829	if (unlikely(rht_grow_above_max(ht, tbl)))
829	830	goto out_unlock;
830	831
831		− if (unlikely(rht_grow_above_100(ht, tbl)))
	832	+ if (unlikely(rht_grow_above_100(ht, tbl)) &&
	833	+ !params.insecure_elasticity)
832	834	goto slow_path;
833	835
834	836	/* Inserting at head of list makes unlocking free. */
@@ -846,7 +848,7 @@ slow_path: · 11 unmodified lines
846	848	rht_assign_unlock(tbl, bkt, obj, flags);
847	849
848	850	if (rht_grow_above_75(ht, tbl))
849		− schedule_work(&ht->run_work);
	851	+ irq_work_queue(&ht->run_irq_work);
850	852
851	853	data = NULL;
852	854	out:

kernel/sched/ext.c+276−122

@@ -32,6 +32,7 @@ static const struct rhashtable_params scx_sched_hash_params = { · 31 unmodified lines
32	32	.key_len = sizeof_field(struct scx_sched, ops.sub_cgroup_id),
33	33	.key_offset = offsetof(struct scx_sched, ops.sub_cgroup_id),
34	34	.head_offset = offsetof(struct scx_sched, hash_node),
	35	+ .insecure_elasticity = true, /* inserted under scx_sched_lock */
35	36	};
36	37
37	38	static struct rhashtable scx_sched_hash;
@@ -52,8 +53,6 @@ DEFINE_STATIC_KEY_FALSE(__scx_enabled); · 14 unmodified lines
52	53	DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem);
53	54	static atomic_t scx_enable_state_var = ATOMIC_INIT(SCX_DISABLED);
54	55	static DEFINE_RAW_SPINLOCK(scx_bypass_lock);
55		−static cpumask_var_t scx_bypass_lb_donee_cpumask;
56		−static cpumask_var_t scx_bypass_lb_resched_cpumask;
57	56	static bool scx_init_task_enabled;
58	57	static bool scx_switching_all;
59	58	DEFINE_STATIC_KEY_FALSE(__scx_switched_all);
@@ -469,24 +468,35 @@ static inline void update_locked_rq(struct rq *rq) · 409 unmodified lines
469	468	__this_cpu_write(scx_locked_rq_state, rq);
470	469	}
471	470
472		−#define SCX_CALL_OP(sch, op, rq, args...) \
	471	+/*
	472	+ * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not
	473	+ * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit.
	474	+ */
	475	+#define SCX_CALL_OP(sch, op, locked_rq, args...) \
473	476	do { \
474		− if (rq) \
475		− update_locked_rq(rq); \
	477	+ struct rq *__prev_locked_rq; \
	478	+ \
	479	+ if (locked_rq) { \
	480	+ __prev_locked_rq = scx_locked_rq(); \
	481	+ update_locked_rq(locked_rq); \
	482	+ } \
476	483	(sch)->ops.op(args); \
477		− if (rq) \
478		− update_locked_rq(NULL); \
	484	+ if (locked_rq) \
	485	+ update_locked_rq(__prev_locked_rq); \
479	486	} while (0)
480	487
481		−#define SCX_CALL_OP_RET(sch, op, rq, args...) \
	488	+#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \
482	489	({ \
	490	+ struct rq *__prev_locked_rq; \
483	491	__typeof__((sch)->ops.op(args)) __ret; \
484	492	\
485		− if (rq) \
486		− update_locked_rq(rq); \
	493	+ if (locked_rq) { \
	494	+ __prev_locked_rq = scx_locked_rq(); \
	495	+ update_locked_rq(locked_rq); \
	496	+ } \
487	497	__ret = (sch)->ops.op(args); \
488		− if (rq) \
489		− update_locked_rq(NULL); \
	498	+ if (locked_rq) \
	499	+ update_locked_rq(__prev_locked_rq); \
490	500	__ret; \
491	501	})
492	502
@@ -498,39 +508,39 @@ do { \ · 5 unmodified lines
498	508	* those subject tasks.
499	509	*
500	510	* Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held -
501		− * either via the @rq argument here, or (for ops.select_cpu()) via @p's pi_lock
502		− * held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu. So if
503		− * kf_tasks[] is set, @p's scheduler-protected fields are stable.
	511	+ * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's
	512	+ * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu.
	513	+ * So if kf_tasks[] is set, @p's scheduler-protected fields are stable.
504	514	*
505	515	* kf_tasks[] can not stack, so task-based SCX ops must not nest. The
506	516	* WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants
507	517	* while a previous one is still in progress.
508	518	*/
509		−#define SCX_CALL_OP_TASK(sch, op, rq, task, args...) \
	519	+#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \
510	520	do { \
511	521	WARN_ON_ONCE(current->scx.kf_tasks[0]); \
512	522	current->scx.kf_tasks[0] = task; \
513		− SCX_CALL_OP((sch), op, rq, task, ##args); \
	523	+ SCX_CALL_OP((sch), op, locked_rq, task, ##args); \
514	524	current->scx.kf_tasks[0] = NULL; \
515	525	} while (0)
516	526
517		−#define SCX_CALL_OP_TASK_RET(sch, op, rq, task, args...) \
	527	+#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \
518	528	({ \
519	529	__typeof__((sch)->ops.op(task, ##args)) __ret; \
520	530	WARN_ON_ONCE(current->scx.kf_tasks[0]); \
521	531	current->scx.kf_tasks[0] = task; \
522		− __ret = SCX_CALL_OP_RET((sch), op, rq, task, ##args); \
	532	+ __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \
523	533	current->scx.kf_tasks[0] = NULL; \
524	534	__ret; \
525	535	})
526	536
527		−#define SCX_CALL_OP_2TASKS_RET(sch, op, rq, task0, task1, args...) \
	537	+#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \
528	538	({ \
529	539	__typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \
530	540	WARN_ON_ONCE(current->scx.kf_tasks[0]); \
531	541	current->scx.kf_tasks[0] = task0; \
532	542	current->scx.kf_tasks[1] = task1; \
533		− __ret = SCX_CALL_OP_RET((sch), op, rq, task0, task1, ##args); \
	543	+ __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \
534	544	current->scx.kf_tasks[0] = NULL; \
535	545	current->scx.kf_tasks[1] = NULL; \
536	546	__ret; \
@@ -1388,18 +1398,55 @@ static void call_task_dequeue(struct scx_sched sch, struct rq rq, · 851 unmodified lines
1388	1398	p->scx.flags &= ~SCX_TASK_IN_CUSTODY;
1389	1399	}
1390	1400
1391		−static void local_dsq_post_enq(struct scx_dispatch_q dsq, struct task_struct p,
1392		− u64 enq_flags)
	1401	+static void local_dsq_post_enq(struct scx_sched sch, struct scx_dispatch_q dsq,
	1402	+ struct task_struct *p, u64 enq_flags)
1393	1403	{
1394	1404	struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
1395		− bool preempt = false;
1396	1405
1397		− call_task_dequeue(scx_root, rq, p, 0);
	1406	+ call_task_dequeue(sch, rq, p, 0);
	1407	+
	1408	+ /*
	1409	+ * Note that @rq's lock may be dropped between this enqueue and @p
	1410	+ * actually getting on CPU. This gives higher-class tasks (e.g. RT)
	1411	+ * an opportunity to wake up on @rq and prevent @p from running.
	1412	+ * Here are some concrete examples:
	1413	+ *
	1414	+ * Example 1:
	1415	+ *
	1416	+ * We dispatch two tasks from a single ops.dispatch():
	1417	+ * - First, a local task to this CPU's local DSQ;
	1418	+ * - Second, a local/remote task to a remote CPU's local DSQ.
	1419	+ * We must drop the local rq lock in order to finish the second
	1420	+ * dispatch. In that time, an RT task can wake up on the local rq.
	1421	+ *
	1422	+ * Example 2:
	1423	+ *
	1424	+ * We dispatch a local/remote task to a remote CPU's local DSQ.
	1425	+ * We must drop the remote rq lock before the dispatched task can run,
	1426	+ * which gives an RT task an opportunity to wake up on the remote rq.
	1427	+ *
	1428	+ * Both examples work the same if we replace dispatching with moving
	1429	+ * the tasks from a user-created DSQ.
	1430	+ *
	1431	+ * We must detect these wakeups so that we can re-enqueue IMMED tasks
	1432	+ * from @rq's local DSQ. scx_wakeup_preempt() serves exactly this
	1433	+ * purpose, but for it to be invoked, we must ensure that we bump
	1434	+ * @rq->next_class to &ext_sched_class if it's currently idle.
	1435	+ *
	1436	+ * wakeup_preempt() does the bumping, and since we only invoke it if
	1437	+ * @rq->next_class is below &ext_sched_class, it will also
	1438	+ * resched_curr(rq).
	1439	+ */
	1440	+ if (sched_class_above(p->sched_class, rq->next_class))
	1441	+ wakeup_preempt(rq, p, 0);
1398	1442
1399	1443	/*
1400	1444	* If @rq is in balance, the CPU is already vacant and looking for the
1401	1445	* next task to run. No need to preempt or trigger resched after moving
1402	1446	* @p into its local DSQ.
	1447	+ * Note that the wakeup_preempt() above may have already triggered
	1448	+ * a resched if @rq->next_class was idle. It's harmless, since
	1449	+ * need_resched is cleared immediately after task pick.
1403	1450	*/
1404	1451	if (rq->scx.flags & SCX_RQ_IN_BALANCE)
1405	1452	return;
@@ -1407,11 +1454,8 @@ static void local_dsq_post_enq(struct scx_dispatch_q dsq, struct task_struct p · 1 unmodified line
1407	1454	if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
1408	1455	rq->curr->sched_class == &ext_sched_class) {
1409	1456	rq->curr->scx.slice = 0;
1410		− preempt = true;
1411		− }
1412		−
1413		− if (preempt \|\| sched_class_above(&ext_sched_class, rq->curr->sched_class))
1414	1457	resched_curr(rq);
	1458	+ }
1415	1459	}
1416	1460
1417	1461	static void dispatch_enqueue(struct scx_sched sch, struct rq rq,
@@ -1494,11 +1538,13 @@ static void dispatch_enqueue(struct scx_sched sch, struct rq rq, · 76 unmodified lines
1494	1538	if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN))
1495	1539	rcu_assign_pointer(dsq->first_task, p);
1496	1540	} else {
1497		− bool was_empty;
1498		−
1499		− was_empty = list_empty(&dsq->list);
	1541	+ /*
	1542	+ * dsq->list can contain parked BPF iterator cursors, so
	1543	+ * list_empty() here isn't a reliable proxy for "no real
	1544	+ * task in the DSQ". Test dsq->first_task directly.
	1545	+ */
1500	1546	list_add_tail(&p->scx.dsq_list.node, &dsq->list);
1501		− if (was_empty && !(dsq->id & SCX_DSQ_FLAG_BUILTIN))
	1547	+ if (!dsq->first_task && !(dsq->id & SCX_DSQ_FLAG_BUILTIN))
1502	1548	rcu_assign_pointer(dsq->first_task, p);
1503	1549	}
1504	1550	}
@@ -1518,7 +1564,7 @@ static void dispatch_enqueue(struct scx_sched sch, struct rq rq, · 13 unmodified lines
1518	1564	* concurrently in a non-atomic way.
1519	1565	*/
1520	1566	if (is_local) {
1521		− local_dsq_post_enq(dsq, p, enq_flags);
	1567	+ local_dsq_post_enq(sch, dsq, p, enq_flags);
1522	1568	} else {
1523	1569	/*
1524	1570	* Task on global/bypass DSQ: leave custody, task on
@@ -2129,7 +2175,8 @@ static void wakeup_preempt_scx(struct rq rq, struct task_struct p, int wake_fl · 604 unmodified lines
2129	2175	schedule_reenq_local(rq, 0);
2130	2176	}
2131	2177
2132		−static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags,
	2178	+static void move_local_task_to_local_dsq(struct scx_sched *sch,
	2179	+ struct task_struct *p, u64 enq_flags,
2133	2180	struct scx_dispatch_q *src_dsq,
2134	2181	struct rq *dst_rq)
2135	2182	{
@@ -2149,7 +2196,7 @@ static void move_local_task_to_local_dsq(struct task_struct *p, u64 enq_flags, · 13 unmodified lines
2149	2196	dsq_inc_nr(dst_dsq, p, enq_flags);
2150	2197	p->scx.dsq = dst_dsq;
2151	2198
2152		− local_dsq_post_enq(dst_dsq, p, enq_flags);
	2199	+ local_dsq_post_enq(sch, dst_dsq, p, enq_flags);
2153	2200	}
2154	2201
2155	2202	/**
@@ -2370,7 +2417,7 @@ static struct rq move_task_between_dsqs(struct scx_sched sch, · 214 unmodified lines
2370	2417	/* @p is going from a non-local DSQ to a local DSQ */
2371	2418	if (src_rq == dst_rq) {
2372	2419	task_unlink_from_dsq(p, src_dsq);
2373		− move_local_task_to_local_dsq(p, enq_flags,
	2420	+ move_local_task_to_local_dsq(sch, p, enq_flags,
2374	2421	src_dsq, dst_rq);
2375	2422	raw_spin_unlock(&src_dsq->lock);
2376	2423	} else {
@@ -2423,7 +2470,7 @@ retry: · 46 unmodified lines
2423	2470
2424	2471	if (rq == task_rq) {
2425	2472	task_unlink_from_dsq(p, dsq);
2426		− move_local_task_to_local_dsq(p, enq_flags, dsq, rq);
	2473	+ move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq);
2427	2474	raw_spin_unlock(&dsq->lock);
2428	2475	return true;
2429	2476	}
@@ -3183,7 +3230,7 @@ bool scx_prio_less(const struct task_struct a, const struct task_struct b, · 753 unmodified lines
3183	3230	if (sch_a == sch_b && SCX_HAS_OP(sch_a, core_sched_before) &&
3184	3231	!scx_bypassing(sch_a, task_cpu(a)))
3185	3232	return SCX_CALL_OP_2TASKS_RET(sch_a, core_sched_before,
3186		− NULL,
	3233	+ task_rq(a),
3187	3234	(struct task_struct *)a,
3188	3235	(struct task_struct *)b);
3189	3236	else
@@ -3631,6 +3678,22 @@ static void __scx_disable_and_exit_task(struct scx_sched *sch, · 441 unmodified lines
3631	3678	SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args);
3632	3679	}
3633	3680
	3681	+/*
	3682	+ * Undo a completed __scx_init_task(sch, p, false) when scx_enable_task() never
	3683	+ * ran. The task state has not been transitioned, so this mirrors the
	3684	+ * SCX_TASK_INIT branch in __scx_disable_and_exit_task().
	3685	+ */
	3686	+static void scx_sub_init_cancel_task(struct scx_sched sch, struct task_struct p)
	3687	+{
	3688	+ struct scx_exit_task_args args = { .cancelled = true };
	3689	+
	3690	+ lockdep_assert_held(&p->pi_lock);
	3691	+ lockdep_assert_rq_held(task_rq(p));
	3692	+
	3693	+ if (SCX_HAS_OP(sch, exit_task))
	3694	+ SCX_CALL_OP_TASK(sch, exit_task, task_rq(p), p, &args);
	3695	+}
	3696	+
3634	3697	static void scx_disable_and_exit_task(struct scx_sched *sch,
3635	3698	struct task_struct *p)
3636	3699	{
@@ -3639,11 +3702,12 @@ static void scx_disable_and_exit_task(struct scx_sched *sch, · 2 unmodified lines
3639	3702	/*
3640	3703	* If set, @p exited between __scx_init_task() and scx_enable_task() in
3641	3704	* scx_sub_enable() and is initialized for both the associated sched and
3642		− * its parent. Disable and exit for the child too.
	3705	+ * its parent. Exit for the child too - scx_enable_task() never ran for
	3706	+ * it, so undo only init_task.
3643	3707	*/
3644		− if ((p->scx.flags & SCX_TASK_SUB_INIT) &&
3645		− !WARN_ON_ONCE(!scx_enabling_sub_sched)) {
3646		− __scx_disable_and_exit_task(scx_enabling_sub_sched, p);
	3708	+ if (p->scx.flags & SCX_TASK_SUB_INIT) {
	3709	+ if (!WARN_ON_ONCE(!scx_enabling_sub_sched))
	3710	+ scx_sub_init_cancel_task(scx_enabling_sub_sched, p);
3647	3711	p->scx.flags &= ~SCX_TASK_SUB_INIT;
3648	3712	}
3649	3713
@@ -4324,9 +4388,10 @@ void scx_cgroup_cancel_attach(struct cgroup_taskset *tset) · 674 unmodified lines
4324	4388
4325	4389	void scx_group_set_weight(struct task_group *tg, unsigned long weight)
4326	4390	{
4327		− struct scx_sched *sch = scx_root;
	4391	+ struct scx_sched *sch;
4328	4392
4329	4393	percpu_down_read(&scx_cgroup_ops_rwsem);
	4394	+ sch = scx_root;
4330	4395
4331	4396	if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_weight) &&
4332	4397	tg->scx.weight != weight)
@@ -4339,9 +4404,10 @@ void scx_group_set_weight(struct task_group *tg, unsigned long weight) · 6 unmodified lines
4339	4404
4340	4405	void scx_group_set_idle(struct task_group *tg, bool idle)
4341	4406	{
4342		− struct scx_sched *sch = scx_root;
	4407	+ struct scx_sched *sch;
4343	4408
4344	4409	percpu_down_read(&scx_cgroup_ops_rwsem);
	4410	+ sch = scx_root;
4345	4411
4346	4412	if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_idle))
4347	4413	SCX_CALL_OP(sch, cgroup_set_idle, NULL, tg_cgrp(tg), idle);
@@ -4355,9 +4421,10 @@ void scx_group_set_idle(struct task_group *tg, bool idle) · 7 unmodified lines
4355	4421	void scx_group_set_bandwidth(struct task_group *tg,
4356	4422	u64 period_us, u64 quota_us, u64 burst_us)
4357	4423	{
4358		− struct scx_sched *sch = scx_root;
	4424	+ struct scx_sched *sch;
4359	4425
4360	4426	percpu_down_read(&scx_cgroup_ops_rwsem);
	4427	+ sch = scx_root;
4361	4428
4362	4429	if (scx_cgroup_enabled && SCX_HAS_OP(sch, cgroup_set_bandwidth) &&
4363	4430	(tg->scx.bw_period_us != period_us \|\|
@@ -4380,21 +4447,6 @@ static struct cgroup *root_cgroup(void) · 16 unmodified lines
4380	4447	return &cgrp_dfl_root.cgrp;
4381	4448	}
4382	4449
4383		−static struct cgroup sch_cgroup(struct scx_sched sch)
4384		−{
4385		− return sch->cgrp;
4386		−}
4387		−
4388		−/* for each descendant of @cgrp including self, set ->scx_sched to @sch */
4389		−static void set_cgroup_sched(struct cgroup cgrp, struct scx_sched sch)
4390		−{
4391		− struct cgroup *pos;
4392		− struct cgroup_subsys_state *css;
4393		−
4394		− cgroup_for_each_live_descendant_pre(pos, css, cgrp)
4395		− rcu_assign_pointer(pos->scx_sched, sch);
4396		−}
4397		−
4398	4450	static void scx_cgroup_lock(void)
4399	4451	{
4400	4452	#ifdef CONFIG_EXT_GROUP_SCHED
@@ -4412,12 +4464,30 @@ static void scx_cgroup_unlock(void) · 11 unmodified lines
4412	4464	}
4413	4465	#else /* CONFIG_EXT_GROUP_SCHED \|\| CONFIG_EXT_SUB_SCHED */
4414	4466	static struct cgroup *root_cgroup(void) { return NULL; }
4415		−static struct cgroup sch_cgroup(struct scx_sched sch) { return NULL; }
4416		−static void set_cgroup_sched(struct cgroup cgrp, struct scx_sched sch) {}
4417	4467	static void scx_cgroup_lock(void) {}
4418	4468	static void scx_cgroup_unlock(void) {}
4419	4469	#endif /* CONFIG_EXT_GROUP_SCHED \|\| CONFIG_EXT_SUB_SCHED */
4420	4470
	4471	+#ifdef CONFIG_EXT_SUB_SCHED
	4472	+static struct cgroup sch_cgroup(struct scx_sched sch)
	4473	+{
	4474	+ return sch->cgrp;
	4475	+}
	4476	+
	4477	+/* for each descendant of @cgrp including self, set ->scx_sched to @sch */
	4478	+static void set_cgroup_sched(struct cgroup cgrp, struct scx_sched sch)
	4479	+{
	4480	+ struct cgroup *pos;
	4481	+ struct cgroup_subsys_state *css;
	4482	+
	4483	+ cgroup_for_each_live_descendant_pre(pos, css, cgrp)
	4484	+ rcu_assign_pointer(pos->scx_sched, sch);
	4485	+}
	4486	+#else /* CONFIG_EXT_SUB_SCHED */
	4487	+static struct cgroup sch_cgroup(struct scx_sched sch) { return NULL; }
	4488	+static void set_cgroup_sched(struct cgroup cgrp, struct scx_sched sch) {}
	4489	+#endif /* CONFIG_EXT_SUB_SCHED */
	4490	+
4421	4491	/*
4422	4492	* Omitted operations:
4423	4493	*
@@ -4712,6 +4782,8 @@ static void scx_sched_free_rcu_work(struct work_struct *work) · 288 unmodified lines
4712	4782	irq_work_sync(&sch->disable_irq_work);
4713	4783	kthread_destroy_worker(sch->helper);
4714	4784	timer_shutdown_sync(&sch->bypass_lb_timer);
	4785	+ free_cpumask_var(sch->bypass_lb_donee_cpumask);
	4786	+ free_cpumask_var(sch->bypass_lb_resched_cpumask);
4715	4787
4716	4788	#ifdef CONFIG_EXT_SUB_SCHED
4717	4789	kfree(sch->cgrp_path);
@@ -4938,6 +5010,25 @@ void scx_softlockup(u32 dur_s) · 220 unmodified lines
4938	5010	smp_processor_id(), dur_s);
4939	5011	}
4940	5012
	5013	+/*
	5014	+ * scx_hardlockup() runs from NMI and eventually calls scx_claim_exit(),
	5015	+ * which takes scx_sched_lock. scx_sched_lock isn't NMI-safe and grabbing
	5016	+ * it from NMI context can lead to deadlocks. Defer via irq_work; the
	5017	+ * disable path runs off irq_work anyway.
	5018	+ */
	5019	+static atomic_t scx_hardlockup_cpu = ATOMIC_INIT(-1);
	5020	+
	5021	+static void scx_hardlockup_irq_workfn(struct irq_work *work)
	5022	+{
	5023	+ int cpu = atomic_xchg(&scx_hardlockup_cpu, -1);
	5024	+
	5025	+ if (cpu >= 0 && handle_lockup("hard lockup - CPU %d", cpu))
	5026	+ printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n",
	5027	+ cpu);
	5028	+}
	5029	+
	5030	+static DEFINE_IRQ_WORK(scx_hardlockup_irq_work, scx_hardlockup_irq_workfn);
	5031	+
4941	5032	/**
4942	5033	* scx_hardlockup - sched_ext hardlockup handler
4943	5034	*
@@ -4946,17 +5037,19 @@ void scx_softlockup(u32 dur_s) · 2 unmodified lines
4946	5037	* Try kicking out the current scheduler in an attempt to recover the system to
4947	5038	* a good state before taking more drastic actions.
4948	5039	*
4949		− * Returns %true if sched_ext is enabled and abort was initiated, which may
4950		− * resolve the reported hardlockup. %false if sched_ext is not enabled or
4951		− * someone else already initiated abort.
	5040	+ * Queues an irq_work; the handle_lockup() call happens in IRQ context (see
	5041	+ * scx_hardlockup_irq_workfn).
	5042	+ *
	5043	+ * Returns %true if sched_ext is enabled and the work was queued, %false
	5044	+ * otherwise.
4952	5045	*/
4953	5046	bool scx_hardlockup(int cpu)
4954	5047	{
4955		− if (!handle_lockup("hard lockup - CPU %d", cpu))
	5048	+ if (!rcu_access_pointer(scx_root))
4956	5049	return false;
4957	5050
4958		− printk_deferred(KERN_ERR "sched_ext: Hard lockup - CPU %d, disabling BPF scheduler\n",
4959		− cpu);
	5051	+ atomic_cmpxchg(&scx_hardlockup_cpu, -1, cpu);
	5052	+ irq_work_queue(&scx_hardlockup_irq_work);
4960	5053	return true;
4961	5054	}
4962	5055
@@ -5000,6 +5093,15 @@ resume: · 37 unmodified lines
5000	5093	if (cpumask_empty(donee_mask))
5001	5094	break;
5002	5095
	5096	+ /*
	5097	+ * If an earlier pass placed @p on @donor_dsq from a different
	5098	+ * CPU and the donee hasn't consumed it yet, @p is still on the
	5099	+ * previous CPU and task_rq(@p) != @donor_rq. @p can't be moved
	5100	+ * without its rq locked. Skip.
	5101	+ */
	5102	+ if (task_rq(p) != donor_rq)
	5103	+ continue;
	5104	+
5003	5105	donee = cpumask_any_and_distribute(donee_mask, p->cpus_ptr);
5004	5106	if (donee >= nr_cpu_ids)
5005	5107	continue;
@@ -5058,8 +5160,8 @@ resume: · 52 unmodified lines
5058	5160	static void bypass_lb_node(struct scx_sched *sch, int node)
5059	5161	{
5060	5162	const struct cpumask *node_mask = cpumask_of_node(node);
5061		− struct cpumask *donee_mask = scx_bypass_lb_donee_cpumask;
5062		− struct cpumask *resched_mask = scx_bypass_lb_resched_cpumask;
	5163	+ struct cpumask *donee_mask = sch->bypass_lb_donee_cpumask;
	5164	+ struct cpumask *resched_mask = sch->bypass_lb_resched_cpumask;
5063	5165	u32 nr_tasks = 0, nr_cpus = 0, nr_balanced = 0;
5064	5166	u32 nr_target, nr_donor_target;
5065	5167	u32 before_min = U32_MAX, before_max = 0;
@@ -5698,6 +5800,8 @@ static void scx_sub_disable(struct scx_sched *sch) · 632 unmodified lines
5698	5800
5699	5801	if (sch->ops.exit)
5700	5802	SCX_CALL_OP(sch, exit, NULL, sch->exit_info);
	5803	+ if (sch->sub_kset)
	5804	+ kset_unregister(sch->sub_kset);
5701	5805	kobject_del(&sch->kobj);
5702	5806	}
5703	5807	#else /* CONFIG_EXT_SUB_SCHED */
@@ -5829,6 +5933,10 @@ static void scx_root_disable(struct scx_sched *sch) · 125 unmodified lines
5829	5933	* could observe an object of the same name still in the hierarchy when
5830	5934	* the next scheduler is loaded.
5831	5935	*/
	5936	+#ifdef CONFIG_EXT_SUB_SCHED
	5937	+ if (sch->sub_kset)
	5938	+ kset_unregister(sch->sub_kset);
	5939	+#endif
5832	5940	kobject_del(&sch->kobj);
5833	5941
5834	5942	free_kick_syncs();
@@ -5921,6 +6029,25 @@ static void scx_disable(struct scx_sched *sch, enum scx_exit_kind kind) · 86 unmodified lines
5921	6029	irq_work_queue(&sch->disable_irq_work);
5922	6030	}
5923	6031
	6032	+/**
	6033	+ * scx_flush_disable_work - flush the disable work and wait for it to finish
	6034	+ * @sch: the scheduler
	6035	+ *
	6036	+ * sch->disable_work might still not queued, causing kthread_flush_work()
	6037	+ * as a noop. Syncing the irq_work first is required to guarantee the
	6038	+ * kthread work has been queued before waiting for it.
	6039	+ */
	6040	+static void scx_flush_disable_work(struct scx_sched *sch)
	6041	+{
	6042	+ int kind;
	6043	+
	6044	+ do {
	6045	+ irq_work_sync(&sch->disable_irq_work);
	6046	+ kthread_flush_work(&sch->disable_work);
	6047	+ kind = atomic_read(&sch->exit_kind);
	6048	+ } while (kind != SCX_EXIT_NONE && kind != SCX_EXIT_DONE);
	6049	+}
	6050	+
5924	6051	static void dump_newline(struct seq_buf *s)
5925	6052	{
5926	6053	trace_sched_ext_dump("");
@@ -6032,9 +6159,8 @@ static void ops_dump_exit(void) · 105 unmodified lines
6032	6159	scx_dump_data.cpu = -1;
6033	6160	}
6034	6161
6035		−static void scx_dump_task(struct scx_sched *sch,
6036		− struct seq_buf s, struct scx_dump_ctx dctx,
6037		− struct task_struct *p, char marker)
	6162	+static void scx_dump_task(struct scx_sched sch, struct seq_buf s, struct scx_dump_ctx *dctx,
	6163	+ struct rq rq, struct task_struct p, char marker)
6038	6164	{
6039	6165	static unsigned long bt[SCX_EXIT_BT_LEN];
6040	6166	struct scx_sched *task_sch = scx_task_sched(p);
@@ -6075,7 +6201,7 @@ static void scx_dump_task(struct scx_sched *sch, · 34 unmodified lines
6075	6201
6076	6202	if (SCX_HAS_OP(sch, dump_task)) {
6077	6203	ops_dump_init(s, " ");
6078		− SCX_CALL_OP(sch, dump_task, NULL, dctx, p);
	6204	+ SCX_CALL_OP(sch, dump_task, rq, dctx, p);
6079	6205	ops_dump_exit();
6080	6206	}
6081	6207
@@ -6199,8 +6325,7 @@ static void scx_dump_state(struct scx_sched sch, struct scx_exit_info ei, · 117 unmodified lines
6199	6325	used = seq_buf_used(&ns);
6200	6326	if (SCX_HAS_OP(sch, dump_cpu)) {
6201	6327	ops_dump_init(&ns, " ");
6202		− SCX_CALL_OP(sch, dump_cpu, NULL,
6203		− &dctx, cpu, idle);
	6328	+ SCX_CALL_OP(sch, dump_cpu, rq, &dctx, cpu, idle);
6204	6329	ops_dump_exit();
6205	6330	}
6206	6331
@@ -6223,11 +6348,11 @@ static void scx_dump_state(struct scx_sched sch, struct scx_exit_info ei, · 16 unmodified lines
6223	6348
6224	6349	if (rq->curr->sched_class == &ext_sched_class &&
6225	6350	(dump_all_tasks \|\| scx_task_on_sched(sch, rq->curr)))
6226		− scx_dump_task(sch, &s, &dctx, rq->curr, '*');
	6351	+ scx_dump_task(sch, &s, &dctx, rq, rq->curr, '*');
6227	6352
6228	6353	list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)
6229	6354	if (dump_all_tasks \|\| scx_task_on_sched(sch, p))
6230		− scx_dump_task(sch, &s, &dctx, p, ' ');
	6355	+ scx_dump_task(sch, &s, &dctx, rq, p, ' ');
6231	6356	next:
6232	6357	rq_unlock_irqrestore(rq, &rf);
6233	6358	}
@@ -6437,6 +6562,15 @@ static struct scx_sched scx_alloc_and_add_sched(struct sched_ext_ops ops, · 203 unmodified lines
6437	6562	init_irq_work(&sch->disable_irq_work, scx_disable_irq_workfn);
6438	6563	kthread_init_work(&sch->disable_work, scx_disable_workfn);
6439	6564	timer_setup(&sch->bypass_lb_timer, scx_bypass_lb_timerfn, 0);
	6565	+
	6566	+ if (!alloc_cpumask_var(&sch->bypass_lb_donee_cpumask, GFP_KERNEL)) {
	6567	+ ret = -ENOMEM;
	6568	+ goto err_stop_helper;
	6569	+ }
	6570	+ if (!alloc_cpumask_var(&sch->bypass_lb_resched_cpumask, GFP_KERNEL)) {
	6571	+ ret = -ENOMEM;
	6572	+ goto err_free_lb_cpumask;
	6573	+ }
6440	6574	sch->ops = *ops;
6441	6575	rcu_assign_pointer(ops->priv, sch);
6442	6576
@@ -6446,14 +6580,14 @@ static struct scx_sched scx_alloc_and_add_sched(struct sched_ext_ops ops, · 3 unmodified lines
6446	6580	char *buf = kzalloc(PATH_MAX, GFP_KERNEL);
6447	6581	if (!buf) {
6448	6582	ret = -ENOMEM;
6449		− goto err_stop_helper;
	6583	+ goto err_free_lb_resched;
6450	6584	}
6451	6585	cgroup_path(cgrp, buf, PATH_MAX);
6452	6586	sch->cgrp_path = kstrdup(buf, GFP_KERNEL);
6453	6587	kfree(buf);
6454	6588	if (!sch->cgrp_path) {
6455	6589	ret = -ENOMEM;
6456		− goto err_stop_helper;
	6590	+ goto err_free_lb_resched;
6457	6591	}
6458	6592
6459	6593	sch->cgrp = cgrp;
@@ -6488,10 +6622,12 @@ static struct scx_sched scx_alloc_and_add_sched(struct sched_ext_ops ops, · 28 unmodified lines
6488	6622	#endif /* CONFIG_EXT_SUB_SCHED */
6489	6623	return sch;
6490	6624
6491		−#ifdef CONFIG_EXT_SUB_SCHED
	6625	+err_free_lb_resched:
	6626	+ free_cpumask_var(sch->bypass_lb_resched_cpumask);
	6627	+err_free_lb_cpumask:
	6628	+ free_cpumask_var(sch->bypass_lb_donee_cpumask);
6492	6629	err_stop_helper:
6493	6630	kthread_destroy_worker(sch->helper);
6494		−#endif
6495	6631	err_free_pcpu:
6496	6632	for_each_possible_cpu(cpu) {
6497	6633	if (cpu == bypass_fail_cpu)
@@ -6510,7 +6646,7 @@ err_free_ei: · 12 unmodified lines
6510	6646	err_free_sch:
6511	6647	kfree(sch);
6512	6648	err_put_cgrp:
6513		−#if defined(CONFIG_EXT_GROUP_SCHED) \|\| defined(CONFIG_EXT_SUB_SCHED)
	6649	+#ifdef CONFIG_EXT_SUB_SCHED
6514	6650	cgroup_put(cgrp);
6515	6651	#endif
6516	6652	return ERR_PTR(ret);
@@ -6601,7 +6737,7 @@ static void scx_root_enable_workfn(struct kthread_work *work) · 84 unmodified lines
6601	6737	if (ret)
6602	6738	goto err_unlock;
6603	6739
6604		−#if defined(CONFIG_EXT_GROUP_SCHED) \|\| defined(CONFIG_EXT_SUB_SCHED)
	6740	+#ifdef CONFIG_EXT_SUB_SCHED
6605	6741	cgroup_get(cgrp);
6606	6742	#endif
6607	6743	sch = scx_alloc_and_add_sched(ops, cgrp, NULL);
@@ -6639,8 +6775,10 @@ static void scx_root_enable_workfn(struct kthread_work *work) · 31 unmodified lines
6639	6775	rcu_assign_pointer(scx_root, sch);
6640	6776
6641	6777	ret = scx_link_sched(sch);
6642		− if (ret)
	6778	+ if (ret) {
	6779	+ cpus_read_unlock();
6643	6780	goto err_disable;
	6781	+ }
6644	6782
6645	6783	scx_idle_enable(ops);
6646	6784
@@ -6821,7 +6959,7 @@ err_disable: · 174 unmodified lines
6821	6959	* completion. sch's base reference will be put by bpf_scx_unreg().
6822	6960	*/
6823	6961	scx_error(sch, "scx_root_enable() failed (%d)", ret);
6824		− kthread_flush_work(&sch->disable_work);
	6962	+ scx_flush_disable_work(sch);
6825	6963	cmd->ret = 0;
6826	6964	}
6827	6965
@@ -7072,23 +7210,30 @@ out_unlock: · 244 unmodified lines
7072	7210	abort:
7073	7211	put_task_struct(p);
7074	7212	scx_task_iter_stop(&sti);
7075		− scx_enabling_sub_sched = NULL;
7076	7213
	7214	+ /*
	7215	+ * Undo __scx_init_task() for tasks we marked. scx_enable_task() never
	7216	+ * ran for @sch on them, so calling scx_disable_task() here would invoke
	7217	+ * ops.disable() without a matching ops.enable(). scx_enabling_sub_sched
	7218	+ * must stay set until SUB_INIT is cleared from every marked task -
	7219	+ * scx_disable_and_exit_task() reads it when a task exits concurrently.
	7220	+ */
7077	7221	scx_task_iter_start(&sti, sch->cgrp);
7078	7222	while ((p = scx_task_iter_next_locked(&sti))) {
7079	7223	if (p->scx.flags & SCX_TASK_SUB_INIT) {
7080		− __scx_disable_and_exit_task(sch, p);
	7224	+ scx_sub_init_cancel_task(sch, p);
7081	7225	p->scx.flags &= ~SCX_TASK_SUB_INIT;
7082	7226	}
7083	7227	}
7084	7228	scx_task_iter_stop(&sti);
	7229	+ scx_enabling_sub_sched = NULL;
7085	7230	err_unlock_and_disable:
7086	7231	/* we'll soon enter disable path, keep bypass on */
7087	7232	scx_cgroup_unlock();
7088	7233	percpu_up_write(&scx_fork_rwsem);
7089	7234	err_disable:
7090	7235	mutex_unlock(&scx_enable_mutex);
7091		− kthread_flush_work(&sch->disable_work);
	7236	+ scx_flush_disable_work(sch);
7092	7237	cmd->ret = 0;
7093	7238	}
7094	7239
@@ -7349,7 +7494,7 @@ static void bpf_scx_unreg(void kdata, struct bpf_link link) · 254 unmodified lines
7349	7494	struct scx_sched *sch = rcu_dereference_protected(ops->priv, true);
7350	7495
7351	7496	scx_disable(sch, SCX_EXIT_UNREG);
7352		− kthread_flush_work(&sch->disable_work);
	7497	+ scx_flush_disable_work(sch);
7353	7498	RCU_INIT_POINTER(ops->priv, NULL);
7354	7499	kobject_put(&sch->kobj);
7355	7500	}
@@ -8033,12 +8178,22 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit, · 677 unmodified lines
8033	8178	struct task_struct *p, u64 dsq_id, u64 enq_flags)
8034	8179	{
8035	8180	struct scx_dispatch_q src_dsq = kit->dsq, dst_dsq;
8036		− struct scx_sched *sch = src_dsq->sched;
	8181	+ struct scx_sched *sch;
8037	8182	struct rq this_rq, src_rq, *locked_rq;
8038	8183	bool dispatched = false;
8039	8184	bool in_balance;
8040	8185	unsigned long flags;
8041	8186
	8187	+ /*
	8188	+ * The verifier considers an iterator slot initialized on any
	8189	+ * KF_ITER_NEW return, so a BPF program may legally reach here after
	8190	+ * bpf_iter_scx_dsq_new() failed and left @kit->dsq NULL.
	8191	+ */
	8192	+ if (unlikely(!src_dsq))
	8193	+ return false;
	8194	+
	8195	+ sch = src_dsq->sched;
	8196	+
8042	8197	if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags))
8043	8198	return false;
8044	8199
@@ -8526,7 +8681,7 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice, · 481 unmodified lines
8526	8681
8527	8682	guard(rcu)();
8528	8683	sch = scx_prog_sched(aux);
8529		− if (unlikely(!scx_task_on_sched(sch, p)))
	8684	+ if (unlikely(!sch \|\| !scx_task_on_sched(sch, p)))
8530	8685	return false;
8531	8686
8532	8687	p->scx.slice = slice;
@@ -8549,7 +8704,7 @@ __bpf_kfunc bool scx_bpf_task_set_dsq_vtime(struct task_struct *p, u64 vtime, · 16 unmodified lines
8549	8704
8550	8705	guard(rcu)();
8551	8706	sch = scx_prog_sched(aux);
8552		− if (unlikely(!scx_task_on_sched(sch, p)))
	8707	+ if (unlikely(!sch \|\| !scx_task_on_sched(sch, p)))
8553	8708	return false;
8554	8709
8555	8710	p->scx.dsq_vtime = vtime;
@@ -8633,11 +8788,12 @@ __bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags, const struct bpf_prog_aux · 77 unmodified lines
8633	8788	/**
8634	8789	* scx_bpf_dsq_nr_queued - Return the number of queued tasks
8635	8790	* @dsq_id: id of the DSQ
	8791	+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
8636	8792	*
8637	8793	* Return the number of tasks in the DSQ matching @dsq_id. If not found,
8638	8794	* -%ENOENT is returned.
8639	8795	*/
8640		−__bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id)
	8796	+__bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id, const struct bpf_prog_aux *aux)
8641	8797	{
8642	8798	struct scx_sched *sch;
8643	8799	struct scx_dispatch_q *dsq;
@@ -8645,7 +8801,7 @@ __bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id) · 1 unmodified line
8645	8801
8646	8802	preempt_disable();
8647	8803
8648		− sch = rcu_dereference_sched(scx_root);
	8804	+ sch = scx_prog_sched(aux);
8649	8805	if (unlikely(!sch)) {
8650	8806	ret = -ENODEV;
8651	8807	goto out;
@@ -8677,21 +8833,21 @@ out: · 25 unmodified lines
8677	8833	/**
8678	8834	* scx_bpf_destroy_dsq - Destroy a custom DSQ
8679	8835	* @dsq_id: DSQ to destroy
	8836	+ * @aux: implicit BPF argument to access bpf_prog_aux hidden from BPF progs
8680	8837	*
8681	8838	* Destroy the custom DSQ identified by @dsq_id. Only DSQs created with
8682	8839	* scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is
8683	8840	* empty and no further tasks are dispatched to it. Ignored if called on a DSQ
8684	8841	* which doesn't exist. Can be called from any online scx_ops operations.
8685	8842	*/
8686		−__bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id)
	8843	+__bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id, const struct bpf_prog_aux *aux)
8687	8844	{
8688	8845	struct scx_sched *sch;
8689	8846
8690		− rcu_read_lock();
8691		− sch = rcu_dereference(scx_root);
	8847	+ guard(rcu)();
	8848	+ sch = scx_prog_sched(aux);
8692	8849	if (sch)
8693	8850	destroy_dsq(sch, dsq_id);
8694		− rcu_read_unlock();
8695	8851	}
8696	8852
8697	8853	/**
@@ -9445,8 +9601,8 @@ BTF_KFUNCS_START(scx_kfunc_ids_any) · 747 unmodified lines
9445	9601	BTF_ID_FLAGS(func, scx_bpf_task_set_slice, KF_IMPLICIT_ARGS \| KF_RCU);
9446	9602	BTF_ID_FLAGS(func, scx_bpf_task_set_dsq_vtime, KF_IMPLICIT_ARGS \| KF_RCU);
9447	9603	BTF_ID_FLAGS(func, scx_bpf_kick_cpu, KF_IMPLICIT_ARGS)
9448		−BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued)
9449		−BTF_ID_FLAGS(func, scx_bpf_destroy_dsq)
	9604	+BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued, KF_IMPLICIT_ARGS)
	9605	+BTF_ID_FLAGS(func, scx_bpf_destroy_dsq, KF_IMPLICIT_ARGS)
9450	9606	BTF_ID_FLAGS(func, scx_bpf_dsq_peek, KF_IMPLICIT_ARGS \| KF_RCU_PROTECTED \| KF_RET_NULL)
9451	9607	BTF_ID_FLAGS(func, scx_bpf_dsq_reenq, KF_IMPLICIT_ARGS)
9452	9608	BTF_ID_FLAGS(func, scx_bpf_reenqueue_local___v2, KF_IMPLICIT_ARGS)
@@ -9479,6 +9635,7 @@ BTF_KFUNCS_END(scx_kfunc_ids_any) · 26 unmodified lines
9479	9635	static const struct btf_kfunc_id_set scx_kfunc_set_any = {
9480	9636	.owner = THIS_MODULE,
9481	9637	.set = &scx_kfunc_ids_any,
	9638	+ .filter = scx_kfunc_context_filter,
9482	9639	};
9483	9640
9484	9641	/*
@@ -9526,13 +9683,12 @@ static const u32 scx_kf_allow_flags[] = { · 41 unmodified lines
9526	9683	};
9527	9684
9528	9685	/*
9529		− * Verifier-time filter for context-sensitive SCX kfuncs. Registered via the
9530		− * .filter field on each per-group btf_kfunc_id_set. The BPF core invokes this
9531		− * for every kfunc call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or
	9686	+ * Verifier-time filter for SCX kfuncs. Registered via the .filter field on
	9687	+ * each per-group btf_kfunc_id_set. The BPF core invokes this for every kfunc
	9688	+ * call in the registered hook (BPF_PROG_TYPE_STRUCT_OPS or
9532	9689	* BPF_PROG_TYPE_SYSCALL), regardless of which set originally introduced the
9533		− * kfunc - so the filter must short-circuit on kfuncs it doesn't govern (e.g.
9534		− * scx_kfunc_ids_any) by falling through to "allow" when none of the
9535		− * context-sensitive sets contain the kfunc.
	9690	+ * kfunc - so the filter must short-circuit on kfuncs it doesn't govern by
	9691	+ * falling through to "allow" when none of the SCX sets contain the kfunc.
9536	9692	*/
9537	9693	int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id)
9538	9694	{
@@ -9541,18 +9697,21 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) · 2 unmodified lines
9541	9697	bool in_enqueue = btf_id_set8_contains(&scx_kfunc_ids_enqueue_dispatch, kfunc_id);
9542	9698	bool in_dispatch = btf_id_set8_contains(&scx_kfunc_ids_dispatch, kfunc_id);
9543	9699	bool in_cpu_release = btf_id_set8_contains(&scx_kfunc_ids_cpu_release, kfunc_id);
	9700	+ bool in_idle = btf_id_set8_contains(&scx_kfunc_ids_idle, kfunc_id);
	9701	+ bool in_any = btf_id_set8_contains(&scx_kfunc_ids_any, kfunc_id);
9544	9702	u32 moff, flags;
9545	9703
9546		− /* Not a context-sensitive kfunc (e.g. from scx_kfunc_ids_any) - allow. */
9547		− if (!(in_unlocked \|\| in_select_cpu \|\| in_enqueue \|\| in_dispatch \|\| in_cpu_release))
	9704	+ /* Not an SCX kfunc - allow. */
	9705	+ if (!(in_unlocked \|\| in_select_cpu \|\| in_enqueue \|\| in_dispatch \|\|
	9706	+ in_cpu_release \|\| in_idle \|\| in_any))
9548	9707	return 0;
9549	9708
9550	9709	/* SYSCALL progs (e.g. BPF test_run()) may call unlocked and select_cpu kfuncs. */
9551	9710	if (prog->type == BPF_PROG_TYPE_SYSCALL)
9552		− return (in_unlocked \|\| in_select_cpu) ? 0 : -EACCES;
	9711	+ return (in_unlocked \|\| in_select_cpu \|\| in_idle \|\| in_any) ? 0 : -EACCES;
9553	9712
9554	9713	if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)
9555		− return -EACCES;
	9714	+ return (in_any \|\| in_idle) ? 0 : -EACCES;
9556	9715
9557	9716	/*
9558	9717	* add_subprog_and_kfunc() collects all kfunc calls, including dead code
@@ -9565,14 +9724,15 @@ int scx_kfunc_context_filter(const struct bpf_prog *prog, u32 kfunc_id) · 6 unmodified lines
9565	9724	return 0;
9566	9725
9567	9726	/*
9568		− * Non-SCX struct_ops: only unlocked kfuncs are safe. The other
9569		− * context-sensitive kfuncs assume the rq lock is held by the SCX
9570		− * dispatch path, which doesn't apply to other struct_ops users.
	9727	+ * Non-SCX struct_ops: SCX kfuncs are not permitted.
9571	9728	*/
9572	9729	if (prog->aux->st_ops != &bpf_sched_ext_ops)
9573		− return in_unlocked ? 0 : -EACCES;
	9730	+ return -EACCES;
9574	9731
9575	9732	/* SCX struct_ops: check the per-op allow list. */
	9733	+ if (in_any \|\| in_idle)
	9734	+ return 0;
	9735	+
9576	9736	moff = prog->aux->attach_st_ops_member_off;
9577	9737	flags = scx_kf_allow_flags[SCX_MOFF_IDX(moff)];
9578	9738
@@ -9656,12 +9816,6 @@ static int __init scx_init(void) · 77 unmodified lines
9656	9816	return ret;
9657	9817	}
9658	9818
9659		− if (!alloc_cpumask_var(&scx_bypass_lb_donee_cpumask, GFP_KERNEL) \|\|
9660		− !alloc_cpumask_var(&scx_bypass_lb_resched_cpumask, GFP_KERNEL)) {
9661		− pr_err("sched_ext: Failed to allocate cpumasks\n");
9662		− return -ENOMEM;
9663		− }
9664		−
9665	9819	return 0;
9666	9820	}
9667	9821	__initcall(scx_init);

kernel/sched/ext_idle.c+18−2

@@ -927,14 +927,24 @@ static s32 select_cpu_from_kfunc(struct scx_sched sch, struct task_struct p, · 926 unmodified lines
927	927	* Accessing p->cpus_ptr / p->nr_cpus_allowed needs either @p's rq
928	928	* lock or @p's pi_lock. Three cases:
929	929	*
930		− * - inside ops.select_cpu(): try_to_wake_up() holds @p's pi_lock.
	930	+ * - inside ops.select_cpu(): try_to_wake_up() holds the wake-up
	931	+ * task's pi_lock; the wake-up task is recorded in kf_tasks[0]
	932	+ * by SCX_CALL_OP_TASK_RET().
931	933	* - other rq-locked SCX op: scx_locked_rq() points at the held rq.
932	934	* - truly unlocked (UNLOCKED ops, SYSCALL, non-SCX struct_ops):
933	935	* nothing held, take pi_lock ourselves.
	936	+ *
	937	+ * In the first two cases, BPF schedulers may pass an arbitrary task
	938	+ * that the held lock doesn't cover. Refuse those.
934	939	*/
935	940	if (this_rq()->scx.in_select_cpu) {
	941	+ if (!scx_kf_arg_task_ok(sch, p))
	942	+ return -EINVAL;
936	943	lockdep_assert_held(&p->pi_lock);
937		− } else if (!scx_locked_rq()) {
	944	+ } else if (scx_locked_rq()) {
	945	+ if (task_rq(p) != scx_locked_rq())
	946	+ goto cross_task;
	947	+ } else {
938	948	raw_spin_lock_irqsave(&p->pi_lock, irq_flags);
939	949	we_locked = true;
940	950	}
@@ -960,6 +970,11 @@ static s32 select_cpu_from_kfunc(struct scx_sched sch, struct task_struct p, · 19 unmodified lines
960	970	raw_spin_unlock_irqrestore(&p->pi_lock, irq_flags);
961	971
962	972	return cpu;
	973	+
	974	+cross_task:
	975	+ scx_error(sch, "select_cpu kfunc called cross-task on %s[%d]",
	976	+ p->comm, p->pid);
	977	+ return -EINVAL;
963	978	}
964	979
965	980	/**
@@ -1467,6 +1482,7 @@ BTF_KFUNCS_END(scx_kfunc_ids_idle) · 501 unmodified lines
1467	1482	static const struct btf_kfunc_id_set scx_kfunc_set_idle = {
1468	1483	.owner = THIS_MODULE,
1469	1484	.set = &scx_kfunc_ids_idle,
	1485	+ .filter = scx_kfunc_context_filter,
1470	1486	};
1471	1487
1472	1488	/*

kernel/sched/ext_idle.h+1−0

@@ -12,6 +12,7 @@ · 11 unmodified lines
12	12
13	13	struct sched_ext_ops;
14	14
	15	+extern struct btf_id_set8 scx_kfunc_ids_idle;
15	16	extern struct btf_id_set8 scx_kfunc_ids_select_cpu;
16	17
17	18	void scx_idle_update_selcpu_topology(struct sched_ext_ops *ops);

kernel/sched/ext_internal.h+2−0

@@ -1075,6 +1075,8 @@ struct scx_sched { · 1074 unmodified lines
1075	1075	struct irq_work disable_irq_work;
1076	1076	struct kthread_work disable_work;
1077	1077	struct timer_list bypass_lb_timer;
	1078	+ cpumask_var_t bypass_lb_donee_cpumask;
	1079	+ cpumask_var_t bypass_lb_resched_cpumask;
1078	1080	struct rcu_work rcu_work;
1079	1081
1080	1082	/* all ancestors including self */

lib/rhashtable.c+31−5

@@ -441,10 +441,33 @@ static void rht_deferred_worker(struct work_struct *work) · 440 unmodified lines
441	441
442	442	mutex_unlock(&ht->mutex);
443	443
	444	+ /*
	445	+ * Re-arm via @run_work, not @run_irq_work.
	446	+ * rhashtable_free_and_destroy() drains async work as irq_work_sync()
	447	+ * followed by cancel_work_sync(). If this site queued irq_work while
	448	+ * cancel_work_sync() was waiting for us, irq_work_sync() would already
	449	+ * have returned and the stale irq_work could fire post-teardown.
	450	+ * cancel_work_sync() natively handles self-requeue on @run_work.
	451	+ */
444	452	if (err)
445	453	schedule_work(&ht->run_work);
446	454	}
447	455
	456	+/*
	457	+ * Insert-path callers can run under a raw spinlock (e.g. an insecure_elasticity
	458	+ * user). Calling schedule_work() under that lock records caller_lock ->
	459	+ * pool->lock -> pi_lock -> rq->__lock, closing a locking cycle if any of
	460	+ * these is acquired in the reverse direction elsewhere. Bounce through
	461	+ * irq_work so the schedule_work() runs with the caller's lock no longer held.
	462	+ */
	463	+static void rht_deferred_irq_work(struct irq_work *irq_work)
	464	+{
	465	+ struct rhashtable *ht = container_of(irq_work, struct rhashtable,
	466	+ run_irq_work);
	467	+
	468	+ schedule_work(&ht->run_work);
	469	+}
	470	+
448	471	static int rhashtable_insert_rehash(struct rhashtable *ht,
449	472	struct bucket_table *tbl)
450	473	{
@@ -477,7 +500,7 @@ static int rhashtable_insert_rehash(struct rhashtable *ht, · 26 unmodified lines
477	500	if (err == -EEXIST)
478	501	err = 0;
479	502	} else
480		− schedule_work(&ht->run_work);
	503	+ irq_work_queue(&ht->run_irq_work);
481	504
482	505	return err;
483	506
@@ -488,7 +511,7 @@ fail: · 4 unmodified lines
488	511
489	512	/* Schedule async rehash to retry allocation in process context. */
490	513	if (err == -ENOMEM)
491		− schedule_work(&ht->run_work);
	514	+ irq_work_queue(&ht->run_irq_work);
492	515
493	516	return err;
494	517	}
@@ -538,7 +561,7 @@ static void rhashtable_lookup_one(struct rhashtable ht, · 43 unmodified lines
538	561	return NULL;
539	562	}
540	563
541		− if (elasticity <= 0)
	564	+ if (elasticity <= 0 && !ht->p.insecure_elasticity)
542	565	return ERR_PTR(-EAGAIN);
543	566
544	567	return ERR_PTR(-ENOENT);
@@ -568,7 +591,8 @@ static struct bucket_table *rhashtable_insert_one( · 23 unmodified lines
568	591	if (unlikely(rht_grow_above_max(ht, tbl)))
569	592	return ERR_PTR(-E2BIG);
570	593
571		− if (unlikely(rht_grow_above_100(ht, tbl)))
	594	+ if (unlikely(rht_grow_above_100(ht, tbl)) &&
	595	+ !ht->p.insecure_elasticity)
572	596	return ERR_PTR(-EAGAIN);
573	597
574	598	head = rht_ptr(bkt, tbl, hash);
@@ -629,7 +653,7 @@ static void rhashtable_try_insert(struct rhashtable ht, const void *key, · 54 unmodified lines
629	653	rht_unlock(tbl, bkt, flags);
630	654
631	655	if (inserted && rht_grow_above_75(ht, tbl))
632		− schedule_work(&ht->run_work);
	656	+ irq_work_queue(&ht->run_irq_work);
633	657	}
634	658	} while (!IS_ERR_OR_NULL(new_tbl));
635	659
@@ -1084,6 +1108,7 @@ int rhashtable_init_noprof(struct rhashtable *ht, · 448 unmodified lines
1084	1108	RCU_INIT_POINTER(ht->tbl, tbl);
1085	1109
1086	1110	INIT_WORK(&ht->run_work, rht_deferred_worker);
	1111	+ init_irq_work(&ht->run_irq_work, rht_deferred_irq_work);
1087	1112
1088	1113	return 0;
1089	1114	}
@@ -1149,6 +1174,7 @@ void rhashtable_free_and_destroy(struct rhashtable *ht, · 59 unmodified lines
1149	1174	struct bucket_table tbl, next_tbl;
1150	1175	unsigned int i;
1151	1176
	1177	+ irq_work_sync(&ht->run_irq_work);
1152	1178	cancel_work_sync(&ht->run_work);
1153	1179
1154	1180	mutex_lock(&ht->mutex);

tools/sched_ext/scx_qmap.bpf.c+6−18

@@ -159,13 +159,7 @@ static s32 pick_direct_dispatch_cpu(struct task_struct *p, s32 prev_cpu) · 158 unmodified lines
159	159
160	160	static struct task_ctx lookup_task_ctx(struct task_struct p)
161	161	{
162		− struct task_ctx *tctx;
163		−
164		− if (!(tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0))) {
165		− scx_bpf_error("task_ctx lookup failed");
166		− return NULL;
167		− }
168		− return tctx;
	162	+ return bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
169	163	}
170	164
171	165	s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p,
@@ -175,7 +169,7 @@ s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p, · 3 unmodified lines
175	169	s32 cpu;
176	170
177	171	if (!(tctx = lookup_task_ctx(p)))
178		− return -ESRCH;
	172	+ return prev_cpu;
179	173
180	174	if (p->scx.weight < 2 && !(p->flags & PF_KTHREAD))
181	175	return prev_cpu;
@@ -540,13 +534,9 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev) · 358 unmodified lines
540	534	*/
541	535	if (prev) {
542	536	tctx = bpf_task_storage_get(&task_ctx_stor, prev, 0, 0);
543		− if (!tctx) {
544		− scx_bpf_error("task_ctx lookup failed");
545		− return;
546		− }
547		−
548		− tctx->core_sched_seq =
549		− core_sched_tail_seqs[weight_to_idx(prev->scx.weight)]++;
	537	+ if (tctx)
	538	+ tctx->core_sched_seq =
	539	+ core_sched_tail_seqs[weight_to_idx(prev->scx.weight)]++;
550	540	}
551	541	}
552	542
@@ -584,10 +574,8 @@ static s64 task_qdist(struct task_struct *p) · 31 unmodified lines
584	574	s64 qdist;
585	575
586	576	tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
587		− if (!tctx) {
588		− scx_bpf_error("task_ctx lookup failed");
	577	+ if (!tctx)
589	578	return 0;
590		− }
591	579
592	580	qdist = tctx->core_sched_seq - core_sched_head_seqs[idx];
593	581

tools/testing/selftests/sched_ext/Makefile+1−0

@@ -175,6 +175,7 @@ auto-test-targets := \ · 174 unmodified lines
175	175	maximal \
176	176	maybe_null \
177	177	minimal \
	178	+ non_scx_kfunc_deny \
178	179	numa \
179	180	allowed_cpus \
180	181	peek_dsq \

tools/testing/selftests/sched_ext/non_scx_kfunc_deny.bpf.c+44−0

@@ -0,0 +1,44 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/*
	3	+ * Verify that context-sensitive SCX kfuncs (even "unlocked" ones) are
	4	+ * restricted to only SCX struct_ops programs. Non-SCX struct_ops programs,
	5	+ * such as TCP congestion control programs, should be rejected by the BPF
	6	+ * verifier when attempting to call these kfuncs.
	7	+ *
	8	+ * Copyright (C) 2026 Ching-Chun (Jim) Huang <jserv@ccns.ncku.edu.tw>
	9	+ * Copyright (C) 2026 Cheng-Yang Chou <yphbchou0911@gmail.com>
	10	+ */
	11	+
	12	+#include <vmlinux.h>
	13	+#include <bpf/bpf_helpers.h>
	14	+#include <bpf/bpf_tracing.h>
	15	+
	16	+/* SCX kfunc from scx_kfunc_ids_any set */
	17	+void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym;
	18	+
	19	+SEC("struct_ops/ssthresh")
	20	+__u32 BPF_PROG(tcp_ca_ssthresh, struct sock *sk)
	21	+{
	22	+ /*
	23	+ * This call should be rejected by the verifier because this is a
	24	+ * TCP congestion control program (non-SCX struct_ops).
	25	+ */
	26	+ scx_bpf_kick_cpu(0, 0);
	27	+ return 2;
	28	+}
	29	+
	30	+SEC("struct_ops/cong_avoid")
	31	+void BPF_PROG(tcp_ca_cong_avoid, struct sock *sk, __u32 ack, __u32 acked) {}
	32	+
	33	+SEC("struct_ops/undo_cwnd")
	34	+__u32 BPF_PROG(tcp_ca_undo_cwnd, struct sock *sk) { return 2; }
	35	+
	36	+SEC(".struct_ops")
	37	+struct tcp_congestion_ops tcp_non_scx_ca = {
	38	+ .ssthresh = (void *)tcp_ca_ssthresh,
	39	+ .cong_avoid = (void *)tcp_ca_cong_avoid,
	40	+ .undo_cwnd = (void *)tcp_ca_undo_cwnd,
	41	+ .name = "tcp_kfunc_deny",
	42	+};
	43	+
	44	+char _license[] SEC("license") = "GPL";

tools/testing/selftests/sched_ext/non_scx_kfunc_deny.c+47−0

@@ -0,0 +1,47 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
	2	+/*
	3	+ * Verify that context-sensitive SCX kfuncs (even "unlocked" ones) are
	4	+ * restricted to only SCX struct_ops programs. Non-SCX struct_ops programs,
	5	+ * such as TCP congestion control programs, should be rejected by the BPF
	6	+ * verifier when attempting to call these kfuncs.
	7	+ *
	8	+ * Copyright (C) 2026 Ching-Chun (Jim) Huang <jserv@ccns.ncku.edu.tw>
	9	+ * Copyright (C) 2026 Cheng-Yang Chou <yphbchou0911@gmail.com>
	10	+ */
	11	+
	12	+#include <bpf/bpf.h>
	13	+#include <scx/common.h>
	14	+#include <unistd.h>
	15	+#include <errno.h>
	16	+#include <stdio.h>
	17	+#include "non_scx_kfunc_deny.bpf.skel.h"
	18	+#include "scx_test.h"
	19	+
	20	+static enum scx_test_status run(void *ctx)
	21	+{
	22	+ struct non_scx_kfunc_deny *skel;
	23	+ int err;
	24	+
	25	+ skel = non_scx_kfunc_deny__open();
	26	+ if (!skel) {
	27	+ SCX_ERR("Failed to open skel");
	28	+ return SCX_TEST_FAIL;
	29	+ }
	30	+
	31	+ err = non_scx_kfunc_deny__load(skel);
	32	+ non_scx_kfunc_deny__destroy(skel);
	33	+
	34	+ if (err == 0) {
	35	+ SCX_ERR("non-SCX BPF program loaded when it should have been rejected");
	36	+ return SCX_TEST_FAIL;
	37	+ }
	38	+
	39	+ return SCX_TEST_PASS;
	40	+}
	41	+
	42	+struct scx_test non_scx_kfunc_deny = {
	43	+ .name = "non_scx_kfunc_deny",
	44	+ .description = "Verify that non-SCX struct_ops programs cannot call SCX kfuncs",
	45	+ .run = run,
	46	+};
	47	+REGISTER_SCX_TEST(&non_scx_kfunc_deny)