Factor out scx_rehome_task() and scx_punt_task() from the sub-disable re-home loop and scx_fail_parent(). The upcoming cgroup migration re-homing also needs scx_rehome_task(). No functional changes.
Unified · Split
| @@ -758,6 +758,57 @@ void drain_descendants(struct scx_sched *sch) · 757 unmodified lines | ||
| 758 | 758 | wait_event(scx_unlink_waitq, list_empty(&sch->children)); |
| 759 | 759 | } |
| 760 | 760 | |
| 761 | +/** | |
| 762 | + * scx_rehome_task - Move a task to a sched it has been initialized for | |
| 763 | + * @to: sched taking over @p, @p's init on it already complete | |
| 764 | + * @p: task to re-home | |
| 765 | + * | |
| 766 | + * Exit @p from its current sched and switch it over to @to, overriding the | |
| 767 | + * state to %SCX_TASK_READY to account for the already completed init. A task | |
| 768 | + * on a non-ext class, possible under an %SCX_OPS_SWITCH_PARTIAL root, stays | |
| 769 | + * %READY and is enabled by switching_to_scx() if it switches over. | |
| 770 | + */ | |
| 771 | +static void scx_rehome_task(struct scx_sched *to, struct task_struct *p) | |
| 772 | +{ | |
| 773 | + lockdep_assert_held(&p->pi_lock); | |
| 774 | + lockdep_assert_rq_held(task_rq(p)); | |
| 775 | + | |
| 776 | + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { | |
| 777 | + scx_disable_and_exit_task(scx_task_sched(p), p); | |
| 778 | + scx_set_task_state(p, SCX_TASK_INIT_BEGIN); | |
| 779 | + scx_set_task_state(p, SCX_TASK_INIT); | |
| 780 | + scx_set_task_sched(p, to); | |
| 781 | + scx_set_task_state(p, SCX_TASK_READY); | |
| 782 | + if (p->sched_class == &ext_sched_class) | |
| 783 | + scx_enable_task(to, p); | |
| 784 | + } | |
| 785 | +} | |
| 786 | + | |
| 787 | +/** | |
| 788 | + * scx_punt_task - Hand a task to a failed sched without initialization | |
| 789 | + * @to: failed and bypassed sched taking custody of @p | |
| 790 | + * @p: task to punt | |
| 791 | + * | |
| 792 | + * Take @p off its current sched and put it on @to at %SCX_TASK_NONE. @to is | |
| 793 | + * dying and its teardown will re-home @p properly. | |
| 794 | + * | |
| 795 | + * Used when @to must take over @p but failed to initialize it. Bypass keeps | |
| 796 | + * scheduling decisions away from @to but @p can still trigger its task ops, | |
| 797 | + * which may confuse the BPF side. @to is dying anyway. The exit paths skip | |
| 798 | + * %NONE tasks (see __scx_disable_and_exit_task() and switched_from_scx()). | |
| 799 | + */ | |
| 800 | +static void scx_punt_task(struct scx_sched *to, struct task_struct *p) | |
| 801 | +{ | |
| 802 | + lockdep_assert_held(&p->pi_lock); | |
| 803 | + lockdep_assert_rq_held(task_rq(p)); | |
| 804 | + WARN_ON_ONCE(!READ_ONCE(to->bypass_depth)); | |
| 805 | + | |
| 806 | + scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { | |
| 807 | + scx_disable_and_exit_task(scx_task_sched(p), p); | |
| 808 | + scx_set_task_sched(p, to); | |
| 809 | + } | |
| 810 | +} | |
| 811 | + | |
| 761 | 812 | static void scx_fail_parent(struct scx_sched *sch, |
| 762 | 813 | struct task_struct *failed, s32 fail_code) |
| 763 | 814 | { |
| @@ -769,9 +820,9 @@ static void scx_fail_parent(struct scx_sched *sch, · 5 unmodified lines | ||
| 769 | 820 | fail_code, failed->comm, failed->pid); |
| 770 | 821 | |
| 771 | 822 | /* |
| 772 | − * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into | |
| 773 | − * it. This may cause downstream failures on the BPF side but $parent is | |
| 774 | − * dying anyway. | |
| 823 | + * Once $parent is bypassed, tasks can be punted into it. This may | |
| 824 | + * cause downstream failures on the BPF side but $parent is dying | |
| 825 | + * anyway. | |
| 775 | 826 | */ |
| 776 | 827 | scx_bypass(parent, true); |
| 777 | 828 | |
| @@ -780,10 +831,7 @@ static void scx_fail_parent(struct scx_sched *sch, · 2 unmodified lines | ||
| 780 | 831 | if (scx_task_on_sched(parent, p)) |
| 781 | 832 | continue; |
| 782 | 833 | |
| 783 | − scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { | |
| 784 | − scx_disable_and_exit_task(sch, p); | |
| 785 | − scx_set_task_sched(p, parent); | |
| 786 | − } | |
| 834 | + scx_punt_task(parent, p); | |
| 787 | 835 | } |
| 788 | 836 | scx_task_iter_stop(&sti); |
| 789 | 837 | } |
| @@ -881,27 +929,7 @@ void scx_sub_disable(struct scx_sched *sch) · 91 unmodified lines | ||
| 881 | 929 | continue; |
| 882 | 930 | } |
| 883 | 931 | |
| 884 | − scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { | |
| 885 | − /* | |
| 886 | − * $p is initialized for $parent and still attached to | |
| 887 | − * @sch. Disable and exit for @sch, switch over to | |
| 888 | − * $parent and override the state to READY to account | |
| 889 | − * for $p having already been initialized. | |
| 890 | − */ | |
| 891 | − scx_disable_and_exit_task(sch, p); | |
| 892 | − scx_set_task_state(p, SCX_TASK_INIT_BEGIN); | |
| 893 | − scx_set_task_state(p, SCX_TASK_INIT); | |
| 894 | − scx_set_task_sched(p, parent); | |
| 895 | − scx_set_task_state(p, SCX_TASK_READY); | |
| 896 | − | |
| 897 | − /* | |
| 898 | − * A task on a non-ext class, possible under an | |
| 899 | − * %SCX_OPS_SWITCH_PARTIAL root, stays READY and is | |
| 900 | − * enabled by switching_to_scx() if it switches over. | |
| 901 | − */ | |
| 902 | − if (p->sched_class == &ext_sched_class) | |
| 903 | − scx_enable_task(parent, p); | |
| 904 | − } | |
| 932 | + scx_rehome_task(parent, p); | |
| 905 | 933 | |
| 906 | 934 | task_rq_unlock(rq, p, &rf); |
| 907 | 935 | put_task_struct(p); |