History

sched_ext: Move shared helpers from ext.c into internal.h and cid.h

idle.c and cid.c are included into build_policy.c together with ext.c and use helpers that ext.c defines. Because the helpers live in ext.c, the two files can not parse as standalone units and clangd reports errors in them. Move the helpers to the headers they belong to. The op-dispatch macros and helpers plus scx_parent() to internal.h, and scx_cpu_arg()/scx_cpu_ret() to cid.h. No functional change. idle.c and cid.c now parse clean standalone.

Tejun Heo committed 4 months ago

Commit 4437ad11 parent 3cd1f763 files · +142 −141Browse files

Unified · Split

3 files
  1. kernel/sched/ext/cid.h +21 −0
  2. kernel/sched/ext/ext.c +0 −141
  3. kernel/sched/ext/internal.h +121 −0
kernel/sched/ext/cid.h+21−0
@@ -270,4 +270,25 @@ static inline u32 scx_cmask_nr_used_words(const struct scx_cmask *m) · 269 unmodified lines
270 __w && ((cid) = __bs + __wi * 64 + __ffs64(__w), true); \270 __w && ((cid) = __bs + __wi * 64 + __ffs64(__w), true); \
271 __w &= __w - 1)271 __w &= __w - 1)
272 272
273/*
274 * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form
275 * schedulers it resolves to the matching cid; for cpu-form it passes @cpu
276 * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op
277 * (currently only ops.select_cpu); it validates the BPF-supplied cid and
278 * triggers scx_error() on @sch if invalid.
279 */
280static inline s32 scx_cpu_arg(s32 cpu)
281{
282 if (scx_is_cid_type())
283 return __scx_cpu_to_cid(cpu);
284 return cpu;
285}
286
287static inline s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid)
288{
289 if (cpu_or_cid < 0 || !scx_is_cid_type())
290 return cpu_or_cid;
291 return scx_cid_to_cpu(sch, cpu_or_cid);
292}
293
273#endif /* _KERNEL_SCHED_EXT_CID_H */294#endif /* _KERNEL_SCHED_EXT_CID_H */
kernel/sched/ext/ext.c+0−141
@@ -259,8 +259,6 @@ __printf(5, 6) bool __scx_exit(struct scx_sched *sch, · 258 unmodified lines
259 return ret;259 return ret;
260}260}
261 261
262#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op)
263
264static long jiffies_delta_msecs(unsigned long at, unsigned long now)262static long jiffies_delta_msecs(unsigned long at, unsigned long now)
265{263{
266 if (time_after(at, now))264 if (time_after(at, now))
@@ -275,20 +273,6 @@ static bool u32_before(u32 a, u32 b) · 8 unmodified lines
275}273}
276 274
277#ifdef CONFIG_EXT_SUB_SCHED275#ifdef CONFIG_EXT_SUB_SCHED
278/**
279 * scx_parent - Find the parent sched
280 * @sch: sched to find the parent of
281 *
282 * Returns the parent scheduler or %NULL if @sch is root.
283 */
284static struct scx_sched *scx_parent(struct scx_sched *sch)
285{
286 if (sch->level)
287 return sch->ancestors[sch->level - 1];
288 else
289 return NULL;
290}
291
292/**276/**
293 * scx_next_descendant_pre - find the next descendant for pre-order walk277 * scx_next_descendant_pre - find the next descendant for pre-order walk
294 * @pos: the current position (%NULL to initiate traversal)278 * @pos: the current position (%NULL to initiate traversal)
@@ -336,7 +320,6 @@ static void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) · 41 unmodified lines
336 rcu_assign_pointer(p->scx.sched, sch);320 rcu_assign_pointer(p->scx.sched, sch);
337}321}
338#else /* CONFIG_EXT_SUB_SCHED */322#else /* CONFIG_EXT_SUB_SCHED */
339static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; }
340static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }323static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
341static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {}324static inline void scx_set_task_sched(struct task_struct *p, struct scx_sched *sch) {}
342#endif /* CONFIG_EXT_SUB_SCHED */325#endif /* CONFIG_EXT_SUB_SCHED */
@@ -496,123 +479,12 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags) · 153 unmodified lines
496 */479 */
497DEFINE_PER_CPU(struct rq *, scx_locked_rq_state);480DEFINE_PER_CPU(struct rq *, scx_locked_rq_state);
498 481
499static inline void update_locked_rq(struct rq *rq)
500{
501 /*
502 * Check whether @rq is actually locked. This can help expose bugs
503 * or incorrect assumptions about the context in which a kfunc or
504 * callback is executed.
505 */
506 if (rq)
507 lockdep_assert_rq_held(rq);
508 __this_cpu_write(scx_locked_rq_state, rq);
509}
510
511/*
512 * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not
513 * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit.
514 */
515#define SCX_CALL_OP(sch, op, locked_rq, args...) \
516do { \
517 struct rq *__prev_locked_rq; \
518 \
519 if (locked_rq) { \
520 __prev_locked_rq = scx_locked_rq(); \
521 update_locked_rq(locked_rq); \
522 } \
523 (sch)->ops.op(args); \
524 if (locked_rq) \
525 update_locked_rq(__prev_locked_rq); \
526} while (0)
527
528/*482/*
529 * Flipped on enable per sch->is_cid_type. Declared in internal.h so483 * Flipped on enable per sch->is_cid_type. Declared in internal.h so
530 * subsystem inlines can read it.484 * subsystem inlines can read it.
531 */485 */
532DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type);486DEFINE_STATIC_KEY_FALSE(__scx_is_cid_type);
533 487
534/*
535 * scx_cpu_arg() wraps a cpu arg being handed to an SCX op. For cid-form
536 * schedulers it resolves to the matching cid; for cpu-form it passes @cpu
537 * through. scx_cpu_ret() is the inverse for a cpu/cid returned from an op
538 * (currently only ops.select_cpu); it validates the BPF-supplied cid and
539 * triggers scx_error() on @sch if invalid.
540 */
541static s32 scx_cpu_arg(s32 cpu)
542{
543 if (scx_is_cid_type())
544 return __scx_cpu_to_cid(cpu);
545 return cpu;
546}
547
548static s32 scx_cpu_ret(struct scx_sched *sch, s32 cpu_or_cid)
549{
550 if (cpu_or_cid < 0 || !scx_is_cid_type())
551 return cpu_or_cid;
552 return scx_cid_to_cpu(sch, cpu_or_cid);
553}
554
555#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \
556({ \
557 struct rq *__prev_locked_rq; \
558 __typeof__((sch)->ops.op(args)) __ret; \
559 \
560 if (locked_rq) { \
561 __prev_locked_rq = scx_locked_rq(); \
562 update_locked_rq(locked_rq); \
563 } \
564 __ret = (sch)->ops.op(args); \
565 if (locked_rq) \
566 update_locked_rq(__prev_locked_rq); \
567 __ret; \
568})
569
570/*
571 * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments
572 * and records them in current->scx.kf_tasks[] for the duration of the call. A
573 * kfunc invoked from inside such an op can then use
574 * scx_kf_arg_task_ok() to verify that its task argument is one of
575 * those subject tasks.
576 *
577 * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held -
578 * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's
579 * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu.
580 * So if kf_tasks[] is set, @p's scheduler-protected fields are stable.
581 *
582 * kf_tasks[] can not stack, so task-based SCX ops must not nest. The
583 * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants
584 * while a previous one is still in progress.
585 */
586#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \
587do { \
588 WARN_ON_ONCE(current->scx.kf_tasks[0]); \
589 current->scx.kf_tasks[0] = task; \
590 SCX_CALL_OP((sch), op, locked_rq, task, ##args); \
591 current->scx.kf_tasks[0] = NULL; \
592} while (0)
593
594#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \
595({ \
596 __typeof__((sch)->ops.op(task, ##args)) __ret; \
597 WARN_ON_ONCE(current->scx.kf_tasks[0]); \
598 current->scx.kf_tasks[0] = task; \
599 __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \
600 current->scx.kf_tasks[0] = NULL; \
601 __ret; \
602})
603
604#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \
605({ \
606 __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \
607 WARN_ON_ONCE(current->scx.kf_tasks[0]); \
608 current->scx.kf_tasks[0] = task0; \
609 current->scx.kf_tasks[1] = task1; \
610 __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \
611 current->scx.kf_tasks[0] = NULL; \
612 current->scx.kf_tasks[1] = NULL; \
613 __ret; \
614})
615
616/**488/**
617 * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task489 * scx_call_op_set_cpumask - invoke ops.set_cpumask / ops_cid.set_cmask for @task
618 * @sch: scx_sched being invoked490 * @sch: scx_sched being invoked
@@ -651,19 +523,6 @@ static inline void scx_call_op_set_cpumask(struct scx_sched *sch, struct rq *rq, · 32 unmodified lines
651 current->scx.kf_tasks[0] = NULL;523 current->scx.kf_tasks[0] = NULL;
652}524}
653 525
654/* see SCX_CALL_OP_TASK() */
655static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch,
656 struct task_struct *p)
657{
658 if (unlikely((p != current->scx.kf_tasks[0] &&
659 p != current->scx.kf_tasks[1]))) {
660 scx_error(sch, "called on a task not being operated on");
661 return false;
662 }
663
664 return true;
665}
666
667enum scx_dsq_iter_flags {526enum scx_dsq_iter_flags {
668 /* iterate in the reverse dispatch order */527 /* iterate in the reverse dispatch order */
669 SCX_DSQ_ITER_REV = 1U << 16,528 SCX_DSQ_ITER_REV = 1U << 16,
kernel/sched/ext/internal.h+121−0
@@ -1553,6 +1553,111 @@ static inline struct rq *scx_locked_rq(void) · 1552 unmodified lines
1553 return __this_cpu_read(scx_locked_rq_state);1553 return __this_cpu_read(scx_locked_rq_state);
1554}1554}
1555 1555
1556static inline void update_locked_rq(struct rq *rq)
1557{
1558 /*
1559 * Check whether @rq is actually locked. This can help expose bugs
1560 * or incorrect assumptions about the context in which a kfunc or
1561 * callback is executed.
1562 */
1563 if (rq)
1564 lockdep_assert_rq_held(rq);
1565 __this_cpu_write(scx_locked_rq_state, rq);
1566}
1567
1568#define SCX_HAS_OP(sch, op) test_bit(SCX_OP_IDX(op), (sch)->has_op)
1569
1570/*
1571 * SCX ops can recurse via scx_bpf_sub_dispatch() - the inner call must not
1572 * clobber the outer's scx_locked_rq_state. Save it on entry, restore on exit.
1573 */
1574#define SCX_CALL_OP(sch, op, locked_rq, args...) \
1575do { \
1576 struct rq *__prev_locked_rq; \
1577 \
1578 if (locked_rq) { \
1579 __prev_locked_rq = scx_locked_rq(); \
1580 update_locked_rq(locked_rq); \
1581 } \
1582 (sch)->ops.op(args); \
1583 if (locked_rq) \
1584 update_locked_rq(__prev_locked_rq); \
1585} while (0)
1586
1587#define SCX_CALL_OP_RET(sch, op, locked_rq, args...) \
1588({ \
1589 struct rq *__prev_locked_rq; \
1590 __typeof__((sch)->ops.op(args)) __ret; \
1591 \
1592 if (locked_rq) { \
1593 __prev_locked_rq = scx_locked_rq(); \
1594 update_locked_rq(locked_rq); \
1595 } \
1596 __ret = (sch)->ops.op(args); \
1597 if (locked_rq) \
1598 update_locked_rq(__prev_locked_rq); \
1599 __ret; \
1600})
1601
1602/*
1603 * SCX_CALL_OP_TASK*() invokes an SCX op that takes one or two task arguments
1604 * and records them in current->scx.kf_tasks[] for the duration of the call. A
1605 * kfunc invoked from inside such an op can then use
1606 * scx_kf_arg_task_ok() to verify that its task argument is one of
1607 * those subject tasks.
1608 *
1609 * Every SCX_CALL_OP_TASK*() call site invokes its op with @p's rq lock held -
1610 * either via the @locked_rq argument here, or (for ops.select_cpu()) via @p's
1611 * pi_lock held by try_to_wake_up() with rq tracking via scx_rq.in_select_cpu.
1612 * So if kf_tasks[] is set, @p's scheduler-protected fields are stable.
1613 *
1614 * kf_tasks[] can not stack, so task-based SCX ops must not nest. The
1615 * WARN_ON_ONCE() in each macro catches a re-entry of any of the three variants
1616 * while a previous one is still in progress.
1617 */
1618#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \
1619do { \
1620 WARN_ON_ONCE(current->scx.kf_tasks[0]); \
1621 current->scx.kf_tasks[0] = task; \
1622 SCX_CALL_OP((sch), op, locked_rq, task, ##args); \
1623 current->scx.kf_tasks[0] = NULL; \
1624} while (0)
1625
1626#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \
1627({ \
1628 __typeof__((sch)->ops.op(task, ##args)) __ret; \
1629 WARN_ON_ONCE(current->scx.kf_tasks[0]); \
1630 current->scx.kf_tasks[0] = task; \
1631 __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \
1632 current->scx.kf_tasks[0] = NULL; \
1633 __ret; \
1634})
1635
1636#define SCX_CALL_OP_2TASKS_RET(sch, op, locked_rq, task0, task1, args...) \
1637({ \
1638 __typeof__((sch)->ops.op(task0, task1, ##args)) __ret; \
1639 WARN_ON_ONCE(current->scx.kf_tasks[0]); \
1640 current->scx.kf_tasks[0] = task0; \
1641 current->scx.kf_tasks[1] = task1; \
1642 __ret = SCX_CALL_OP_RET((sch), op, locked_rq, task0, task1, ##args); \
1643 current->scx.kf_tasks[0] = NULL; \
1644 current->scx.kf_tasks[1] = NULL; \
1645 __ret; \
1646})
1647
1648/* see SCX_CALL_OP_TASK() */
1649static __always_inline bool scx_kf_arg_task_ok(struct scx_sched *sch,
1650 struct task_struct *p)
1651{
1652 if (unlikely((p != current->scx.kf_tasks[0] &&
1653 p != current->scx.kf_tasks[1]))) {
1654 scx_error(sch, "called on a task not being operated on");
1655 return false;
1656 }
1657
1658 return true;
1659}
1660
1556static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu)1661static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu)
1557{1662{
1558 return unlikely(per_cpu_ptr(sch->pcpu, cpu)->flags &1663 return unlikely(per_cpu_ptr(sch->pcpu, cpu)->flags &
@@ -1633,6 +1738,20 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) · 74 unmodified lines
1633 1738
1634 return NULL;1739 return NULL;
1635}1740}
1741
1742/**
1743 * scx_parent - Find the parent sched
1744 * @sch: sched to find the parent of
1745 *
1746 * Returns the parent scheduler or %NULL if @sch is root.
1747 */
1748static inline struct scx_sched *scx_parent(struct scx_sched *sch)
1749{
1750 if (sch->level)
1751 return sch->ancestors[sch->level - 1];
1752 else
1753 return NULL;
1754}
1636#else /* CONFIG_EXT_SUB_SCHED */1755#else /* CONFIG_EXT_SUB_SCHED */
1637static inline struct scx_sched *scx_task_sched(const struct task_struct *p)1756static inline struct scx_sched *scx_task_sched(const struct task_struct *p)
1638{1757{
@@ -1656,6 +1775,8 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux) · 17 unmodified lines
1656{1775{
1657 return rcu_dereference_all(scx_root);1776 return rcu_dereference_all(scx_root);
1658}1777}
1778
1779static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; }
1659#endif /* CONFIG_EXT_SUB_SCHED */1780#endif /* CONFIG_EXT_SUB_SCHED */
1660 1781
1661#endif /* _KERNEL_SCHED_EXT_INTERNAL_H */1782#endif /* _KERNEL_SCHED_EXT_INTERNAL_H */