sched_ext: Move shared helpers from ext.c into internal.h and cid.h

idle.c and cid.c are included into build_policy.c together with ext.c and use helpers that ext.c defines. Because the helpers live in ext.c, the two files can not parse as standalone units and clangd reports errors in them. Move the helpers to the headers they belong to. The op-dispatch macros and helpers plus scx_parent() to internal.h, and scx_cpu_arg()/scx_cpu_ret() to cid.h. No functional change. idle.c and cid.c now parse clean standalone.

Tejun Heo committed 4 months ago

Commit 4437ad11 parent 3cd1f763 files · +142 −141Browse files

Showing the changes to kernel (3 of 3 files). Show all

Unified · Split

3 files

kernel/sched/ext/cid.h +21 −0
kernel/sched/ext/ext.c +0 −141
kernel/sched/ext/internal.h +121 −0

kernel/sched/ext/cid.h+21−0

@@ -270,4 +270,25 @@ static inline u32 scx_cmask_nr_used_words(const struct scx_cmask *m) · 269 unmodified lines
270	270	__w && ((cid) = __bs + __wi * 64 + __ffs64(__w), true); \
271	271	__w &= __w - 1)
272	272
	273	+/*
	274	+ * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form
	275	+ * schedulers it resolves to the matching cid; for cpu-form it passes @cpu
	276	+ * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op
	277	+ * (currently only ops.select_cpu); it validates the BPF-supplied cid and
	278	+ * triggers scx_error() on @sch if invalid.
	279	+ */
	280	+static inline s32 scx_cpu_arg(s32 cpu)
	281	+{
	282	+ if (scx_is_cid_type())
	283	+ return __scx_cpu_to_cid(cpu);
	284	+ return cpu;
	285	+}
	286	+
	287	+static inline s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid)
	288	+{
	289	+ if (cpu_or_cid < 0 \|\| !scx_is_cid_type())
	290	+ return cpu_or_cid;
	291	+ return scx_cid_to_cpu(sch, cpu_or_cid);
	292	+}
	293	+
273	294	#endif /* _KERNEL_SCHED_EXT_CID_H */

kernel/sched/ext/ext.c+0−141

@@ -259,8 +259,6 @@ __printf(5, 6) bool __scx_exit(struct scx_sched *sch, · 258 unmodified lines
259	259	return ret;
260	260	}
261	261
262		−#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op)
263		−
264	262	static long jiffies_delta_msecs(unsigned long at, unsigned long now)
265	263	{
266	264	if (time_after(at, now))
@@ -275,20 +273,6 @@ static bool u32_before(u32 a, u32 b) · 8 unmodified lines
275	273	}
276	274
277	275	#ifdef CONFIG_EXT_SUB_SCHED
278		−/**
279		− * scx_parent - Find the parent sched
280		− * @sch: sched to find the parent of
281		− *
282		− * Returns the parent scheduler or %NULL if @sch is root.
283		− */
284		−static struct scx_sched scx_parent(struct scx_sched sch)
285		−{
286		− if (sch->level)
287		− return sch->ancestors[sch->level - 1];
288		− else
289		− return NULL;
290		−}
291		−
292	276	/**
293	277	* scx_next_descendant_pre - find the next descendant for pre-order walk
294	278	* @pos: the current position (%NULL to initiate traversal)
@@ -336,7 +320,6 @@ static void scx_set_task_sched(struct task_struct p, struct scx_sched sch) · 41 unmodified lines
336	320	rcu_assign_pointer(p->scx.sched, sch);
337	321	}
338	322	#else /* CONFIG_EXT_SUB_SCHED */
339		−static inline struct scx_sched scx_parent(struct scx_sched sch) { return NULL; }
340	323	static inline struct scx_sched scx_next_descendant_pre(struct scx_sched pos, struct scx_sched *root) { return pos ? NULL : root; }
341	324	static inline void scx_set_task_sched(struct task_struct p, struct scx_sched sch) {}
342	325	#endif /* CONFIG_EXT_SUB_SCHED */
@@ -496,123 +479,12 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) · 153 unmodified lines
496	479	*/
497	480	DEFINE_PER_CPU(struct rq *, scx_locked_rq_state);
498	481
499		−static inline void update_locked_rq(struct rq *rq)
500		−{
501		− /*
502		− * Check whether @rq is actually locked. This can help expose bugs
503		− * or incorrect assumptions about the context in which a kfunc or
504		− * callback is executed.
505		− */
506		− if (rq)
507		− lockdep_assert_rq_held(rq);
508		− __this_cpu_write(scx_locked_rq_state, rq);
509		−}
510		−
511		−/*
512		− * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not
513		− * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit.
514		− */
515		−#define SCX_CALL_OP(sch, op, locked_rq, args...) \
516		−do { \
517		− struct rq *__prev_locked_rq; \
518		− \
519		− if (locked_rq) { \
520		− __prev_locked_rq = scx_locked_rq(); \
521		− update_locked_rq(locked_rq); \
522		− } \
523		− (sch)->ops.op(args); \
524		− if (locked_rq) \
525		− update_locked_rq(__prev_locked_rq); \
526		−} while (0)
527		−
528	482	/*
529	483	* Flipped on enable per sch->is_cid_type. Declared in internal.h so
530	484	* subsystem inlines can read it.
531	485	*/
532	486	DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type);
533	487
534		−/*
535		− * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form
536		− * schedulers it resolves to the matching cid; for cpu-form it passes @cpu
537		− * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op
538		− * (currently only ops.select_cpu); it validates the BPF-supplied cid and
539		− * triggers scx_error() on @sch if invalid.
540		− */
541		−static s32 scx_cpu_arg(s32 cpu)
542		−{
543		− if (scx_is_cid_type())
544		− return __scx_cpu_to_cid(cpu);
545		− return cpu;
546		−}
547		−
548		−static s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid)
549		−{
550		− if (cpu_or_cid < 0 \|\| !scx_is_cid_type())
551		− return cpu_or_cid;
552		− return scx_cid_to_cpu(sch, cpu_or_cid);
553		−}
554		−
555		−#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \
556		−({ \
557		− struct rq *__prev_locked_rq; \
558		− __typeof__((sch)->ops.op(args)) __ret; \
559		− \
560		− if (locked_rq) { \
561		− __prev_locked_rq = scx_locked_rq(); \
562		− update_locked_rq(locked_rq); \
563		− } \
564		− __ret = (sch)->ops.op(args); \
565		− if (locked_rq) \
566		− update_locked_rq(__prev_locked_rq); \
567		− __ret; \
568		−})
569		−
570		−/*
571		− * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments
572		− * and records them in current->scx.kf_tasks[] for the duration of the call. A
573		− * kfunc invoked from inside such an op can then use
574		− * scx_kf_arg_task_ok() to verify that its task argument is one of
575		− * those subject tasks.
576		− *
577		− * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held -
578		− * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's
579		− * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu.
580		− * So if kf_tasks[] is set, @p's scheduler-protected fields are stable.
581		− *
582		− * kf_tasks[] can not stack, so task-based SCX ops must not nest. The
583		− * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants
584		− * while a previous one is still in progress.
585		− */
586		−#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \
587		−do { \
588		− WARN_ON_ONCE(current->scx.kf_tasks[0]); \
589		− current->scx.kf_tasks[0] = task; \
590		− SCX_CALL_OP((sch), op, locked_rq, task, ##args); \
591		− current->scx.kf_tasks[0] = NULL; \
592		−} while (0)
593		−
594		−#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \
595		−({ \
596		− __typeof__((sch)->ops.op(task, ##args)) __ret; \
597		− WARN_ON_ONCE(current->scx.kf_tasks[0]); \
598		− current->scx.kf_tasks[0] = task; \
599		− __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \
600		− current->scx.kf_tasks[0] = NULL; \
601		− __ret; \
602		−})
603		−
604		−#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \
605		−({ \
606		− __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \
607		− WARN_ON_ONCE(current->scx.kf_tasks[0]); \
608		− current->scx.kf_tasks[0] = task0; \
609		− current->scx.kf_tasks[1] = task1; \
610		− __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \
611		− current->scx.kf_tasks[0] = NULL; \
612		− current->scx.kf_tasks[1] = NULL; \
613		− __ret; \
614		−})
615		−
616	488	/**
617	489	* scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task
618	490	* @sch: scx_sched being invoked
@@ -651,19 +523,6 @@ static inline void scx_call_op_set_cpumask(struct scx_sched sch, struct rq rq, · 32 unmodified lines
651	523	current->scx.kf_tasks[0] = NULL;
652	524	}
653	525
654		−/* see SCX_CALL_OP_TASK() */
655		−static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch,
656		− struct task_struct *p)
657		−{
658		− if (unlikely((p != current->scx.kf_tasks[0] &&
659		− p != current->scx.kf_tasks[1]))) {
660		− scx_error(sch, "called on a task not being operated on");
661		− return false;
662		− }
663		−
664		− return true;
665		−}
666		−
667	526	enum scx_dsq_iter_flags {
668	527	/* iterate in the reverse dispatch order */
669	528	SCX_DSQ_ITER_REV = 1U << 16,

kernel/sched/ext/internal.h+121−0

@@ -1553,6 +1553,111 @@ static inline struct rq *scx_locked_rq(void) · 1552 unmodified lines
1553	1553	return __this_cpu_read(scx_locked_rq_state);
1554	1554	}
1555	1555
	1556	+static inline void update_locked_rq(struct rq *rq)
	1557	+{
	1558	+ /*
	1559	+ * Check whether @rq is actually locked. This can help expose bugs
	1560	+ * or incorrect assumptions about the context in which a kfunc or
	1561	+ * callback is executed.
	1562	+ */
	1563	+ if (rq)
	1564	+ lockdep_assert_rq_held(rq);
	1565	+ __this_cpu_write(scx_locked_rq_state, rq);
	1566	+}
	1567	+
	1568	+#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op)
	1569	+
	1570	+/*
	1571	+ * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not
	1572	+ * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit.
	1573	+ */
	1574	+#define SCX_CALL_OP(sch, op, locked_rq, args...) \
	1575	+do { \
	1576	+ struct rq *__prev_locked_rq; \
	1577	+ \
	1578	+ if (locked_rq) { \
	1579	+ __prev_locked_rq = scx_locked_rq(); \
	1580	+ update_locked_rq(locked_rq); \
	1581	+ } \
	1582	+ (sch)->ops.op(args); \
	1583	+ if (locked_rq) \
	1584	+ update_locked_rq(__prev_locked_rq); \
	1585	+} while (0)
	1586	+
	1587	+#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \
	1588	+({ \
	1589	+ struct rq *__prev_locked_rq; \
	1590	+ __typeof__((sch)->ops.op(args)) __ret; \
	1591	+ \
	1592	+ if (locked_rq) { \
	1593	+ __prev_locked_rq = scx_locked_rq(); \
	1594	+ update_locked_rq(locked_rq); \
	1595	+ } \
	1596	+ __ret = (sch)->ops.op(args); \
	1597	+ if (locked_rq) \
	1598	+ update_locked_rq(__prev_locked_rq); \
	1599	+ __ret; \
	1600	+})
	1601	+
	1602	+/*
	1603	+ * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments
	1604	+ * and records them in current->scx.kf_tasks[] for the duration of the call. A
	1605	+ * kfunc invoked from inside such an op can then use
	1606	+ * scx_kf_arg_task_ok() to verify that its task argument is one of
	1607	+ * those subject tasks.
	1608	+ *
	1609	+ * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held -
	1610	+ * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's
	1611	+ * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu.
	1612	+ * So if kf_tasks[] is set, @p's scheduler-protected fields are stable.
	1613	+ *
	1614	+ * kf_tasks[] can not stack, so task-based SCX ops must not nest. The
	1615	+ * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants
	1616	+ * while a previous one is still in progress.
	1617	+ */
	1618	+#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \
	1619	+do { \
	1620	+ WARN_ON_ONCE(current->scx.kf_tasks[0]); \
	1621	+ current->scx.kf_tasks[0] = task; \
	1622	+ SCX_CALL_OP((sch), op, locked_rq, task, ##args); \
	1623	+ current->scx.kf_tasks[0] = NULL; \
	1624	+} while (0)
	1625	+
	1626	+#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \
	1627	+({ \
	1628	+ __typeof__((sch)->ops.op(task, ##args)) __ret; \
	1629	+ WARN_ON_ONCE(current->scx.kf_tasks[0]); \
	1630	+ current->scx.kf_tasks[0] = task; \
	1631	+ __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \
	1632	+ current->scx.kf_tasks[0] = NULL; \
	1633	+ __ret; \
	1634	+})
	1635	+
	1636	+#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \
	1637	+({ \
	1638	+ __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \
	1639	+ WARN_ON_ONCE(current->scx.kf_tasks[0]); \
	1640	+ current->scx.kf_tasks[0] = task0; \
	1641	+ current->scx.kf_tasks[1] = task1; \
	1642	+ __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \
	1643	+ current->scx.kf_tasks[0] = NULL; \
	1644	+ current->scx.kf_tasks[1] = NULL; \
	1645	+ __ret; \
	1646	+})
	1647	+
	1648	+/* see SCX_CALL_OP_TASK() */
	1649	+static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch,
	1650	+ struct task_struct *p)
	1651	+{
	1652	+ if (unlikely((p != current->scx.kf_tasks[0] &&
	1653	+ p != current->scx.kf_tasks[1]))) {
	1654	+ scx_error(sch, "called on a task not being operated on");
	1655	+ return false;
	1656	+ }
	1657	+
	1658	+ return true;
	1659	+}
	1660	+
1556	1661	static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu)
1557	1662	{
1558	1663	return unlikely(per_cpu_ptr(sch->pcpu, cpu)->flags &
@@ -1633,6 +1738,20 @@ static inline struct scx_sched scx_prog_sched(const struct bpf_prog_aux aux) · 74 unmodified lines
1633	1738
1634	1739	return NULL;
1635	1740	}
	1741	+
	1742	+/**
	1743	+ * scx_parent - Find the parent sched
	1744	+ * @sch: sched to find the parent of
	1745	+ *
	1746	+ * Returns the parent scheduler or %NULL if @sch is root.
	1747	+ */
	1748	+static inline struct scx_sched scx_parent(struct scx_sched sch)
	1749	+{
	1750	+ if (sch->level)
	1751	+ return sch->ancestors[sch->level - 1];
	1752	+ else
	1753	+ return NULL;
	1754	+}
1636	1755	#else /* CONFIG_EXT_SUB_SCHED */
1637	1756	static inline struct scx_sched scx_task_sched(const struct task_struct p)
1638	1757	{
@@ -1656,6 +1775,8 @@ static inline struct scx_sched scx_prog_sched(const struct bpf_prog_aux aux) · 17 unmodified lines
1656	1775	{
1657	1776	return rcu_dereference_all(scx_root);
1658	1777	}
	1778	+
	1779	+static inline struct scx_sched scx_parent(struct scx_sched sch) { return NULL; }
1659	1780	#endif /* CONFIG_EXT_SUB_SCHED */
1660	1781
1661	1782	#endif /* _KERNEL_SCHED_EXT_INTERNAL_H */