Factor out scx_rehome_task() and scx_punt_task() from the sub-disable re-home loop and scx_fail_parent(). The upcoming cgroup migration re-homing also needs scx_rehome_task(). No functional changes.
Unified · Split
| @@ -758,6 +758,57 @@ void drain_descendants(struct scx_sched *sch) · 757 unmodified lines | |||
| 758 | wait_event(scx_unlink_waitq, list_empty(&sch->children)); | 758 | wait_event(scx_unlink_waitq, list_empty(&sch->children)); |
| 759 | } | 759 | } |
| 760 | 760 | ||
| 761 | /** | ||
| 762 | * scx_rehome_task - Move a task to a sched it has been initialized for | ||
| 763 | * @to: sched taking over @p, @p's init on it already complete | ||
| 764 | * @p: task to re-home | ||
| 765 | * | ||
| 766 | * Exit @p from its current sched and switch it over to @to, overriding the | ||
| 767 | * state to %SCX_TASK_READY to account for the already completed init. A task | ||
| 768 | * on a non-ext class, possible under an %SCX_OPS_SWITCH_PARTIAL root, stays | ||
| 769 | * %READY and is enabled by switching_to_scx() if it switches over. | ||
| 770 | */ | ||
| 771 | static void scx_rehome_task(struct scx_sched *to, struct task_struct *p) | ||
| 772 | { | ||
| 773 | lockdep_assert_held(&p->pi_lock); | ||
| 774 | lockdep_assert_rq_held(task_rq(p)); | ||
| 775 | |||
| 776 | scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { | ||
| 777 | scx_disable_and_exit_task(scx_task_sched(p), p); | ||
| 778 | scx_set_task_state(p, SCX_TASK_INIT_BEGIN); | ||
| 779 | scx_set_task_state(p, SCX_TASK_INIT); | ||
| 780 | scx_set_task_sched(p, to); | ||
| 781 | scx_set_task_state(p, SCX_TASK_READY); | ||
| 782 | if (p->sched_class == &ext_sched_class) | ||
| 783 | scx_enable_task(to, p); | ||
| 784 | } | ||
| 785 | } | ||
| 786 | |||
| 787 | /** | ||
| 788 | * scx_punt_task - Hand a task to a failed sched without initialization | ||
| 789 | * @to: failed and bypassed sched taking custody of @p | ||
| 790 | * @p: task to punt | ||
| 791 | * | ||
| 792 | * Take @p off its current sched and put it on @to at %SCX_TASK_NONE. @to is | ||
| 793 | * dying and its teardown will re-home @p properly. | ||
| 794 | * | ||
| 795 | * Used when @to must take over @p but failed to initialize it. Bypass keeps | ||
| 796 | * scheduling decisions away from @to but @p can still trigger its task ops, | ||
| 797 | * which may confuse the BPF side. @to is dying anyway. The exit paths skip | ||
| 798 | * %NONE tasks (see __scx_disable_and_exit_task() and switched_from_scx()). | ||
| 799 | */ | ||
| 800 | static void scx_punt_task(struct scx_sched *to, struct task_struct *p) | ||
| 801 | { | ||
| 802 | lockdep_assert_held(&p->pi_lock); | ||
| 803 | lockdep_assert_rq_held(task_rq(p)); | ||
| 804 | WARN_ON_ONCE(!READ_ONCE(to->bypass_depth)); | ||
| 805 | |||
| 806 | scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { | ||
| 807 | scx_disable_and_exit_task(scx_task_sched(p), p); | ||
| 808 | scx_set_task_sched(p, to); | ||
| 809 | } | ||
| 810 | } | ||
| 811 | |||
| 761 | static void scx_fail_parent(struct scx_sched *sch, | 812 | static void scx_fail_parent(struct scx_sched *sch, |
| 762 | struct task_struct *failed, s32 fail_code) | 813 | struct task_struct *failed, s32 fail_code) |
| 763 | { | 814 | { |
| @@ -769,9 +820,9 @@ static void scx_fail_parent(struct scx_sched *sch, · 5 unmodified lines | |||
| 769 | fail_code, failed->comm, failed->pid); | 820 | fail_code, failed->comm, failed->pid); |
| 770 | 821 | ||
| 771 | /* | 822 | /* |
| 772 | * Once $parent is bypassed, it's safe to put SCX_TASK_NONE tasks into | 823 | * Once $parent is bypassed, tasks can be punted into it. This may |
| 773 | * it. This may cause downstream failures on the BPF side but $parent is | 824 | * cause downstream failures on the BPF side but $parent is dying |
| 774 | * dying anyway. | 825 | * anyway. |
| 775 | */ | 826 | */ |
| 776 | scx_bypass(parent, true); | 827 | scx_bypass(parent, true); |
| 777 | 828 | ||
| @@ -780,10 +831,7 @@ static void scx_fail_parent(struct scx_sched *sch, · 2 unmodified lines | |||
| 780 | if (scx_task_on_sched(parent, p)) | 831 | if (scx_task_on_sched(parent, p)) |
| 781 | continue; | 832 | continue; |
| 782 | 833 | ||
| 783 | scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { | 834 | scx_punt_task(parent, p); |
| 784 | scx_disable_and_exit_task(sch, p); | ||
| 785 | scx_set_task_sched(p, parent); | ||
| 786 | } | ||
| 787 | } | 835 | } |
| 788 | scx_task_iter_stop(&sti); | 836 | scx_task_iter_stop(&sti); |
| 789 | } | 837 | } |
| @@ -881,27 +929,7 @@ void scx_sub_disable(struct scx_sched *sch) · 91 unmodified lines | |||
| 881 | continue; | 929 | continue; |
| 882 | } | 930 | } |
| 883 | 931 | ||
| 884 | scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) { | 932 | scx_rehome_task(parent, p); |
| 885 | /* | ||
| 886 | * $p is initialized for $parent and still attached to | ||
| 887 | * @sch. Disable and exit for @sch, switch over to | ||
| 888 | * $parent and override the state to READY to account | ||
| 889 | * for $p having already been initialized. | ||
| 890 | */ | ||
| 891 | scx_disable_and_exit_task(sch, p); | ||
| 892 | scx_set_task_state(p, SCX_TASK_INIT_BEGIN); | ||
| 893 | scx_set_task_state(p, SCX_TASK_INIT); | ||
| 894 | scx_set_task_sched(p, parent); | ||
| 895 | scx_set_task_state(p, SCX_TASK_READY); | ||
| 896 | |||
| 897 | /* | ||
| 898 | * A task on a non-ext class, possible under an | ||
| 899 | * %SCX_OPS_SWITCH_PARTIAL root, stays READY and is | ||
| 900 | * enabled by switching_to_scx() if it switches over. | ||
| 901 | */ | ||
| 902 | if (p->sched_class == &ext_sched_class) | ||
| 903 | scx_enable_task(parent, p); | ||
| 904 | } | ||
| 905 | 933 | ||
| 906 | task_rq_unlock(rq, p, &rf); | 934 | task_rq_unlock(rq, p, &rf); |
| 907 | put_task_struct(p); | 935 | put_task_struct(p); |