History

sched: Make class_schedulers avoid pushing current, and get rid of proxy_tag_curr()

With proxy-execution, the scheduler selects the donor, but for blocked donors, we end up running the lock owner. This caused some complexity, because the class schedulers make sure to remove the task they pick from their pushable task lists, which prevents the donor from being migrated, but there wasn't then anything to prevent rq->curr from being migrated if rq->curr != rq->donor. This was sort of hacked around by calling proxy_tag_curr() on the rq->curr task if we were running something other then the donor. proxy_tag_curr() did a dequeue/enqueue pair on the rq->curr task, allowing the class schedulers to remove it from their pushable list. The dequeue/enqueue pair was wasteful, and additonally K Prateek highlighted that we didn't properly undo things when we stopped proxying, leaving the lock owner off the pushable list. After some alternative approaches were considered, Peter suggested just having the RT/DL classes just avoid migrating when task_on_cpu(). So rework pick_next_pushable_dl_task() and the rt pick_next_pushable_task() functions so that they skip over the first pushable task if it is on_cpu. Then just drop all of the proxy_tag_curr() logic.

John Stultz committed 7 months ago

Commit e0ca8991 parent 98539143 files · +28 −29Browse files

Unified · Split

3 files
  1. kernel/sched/core.c +0 −24
  2. kernel/sched/deadline.c +16 −2
  3. kernel/sched/rt.c +12 −3
kernel/sched/core.c+0−24
@@ -6702,23 +6702,6 @@ find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf) · 6701 unmodified lines
6702}6702}
6703#endif /* SCHED_PROXY_EXEC */6703#endif /* SCHED_PROXY_EXEC */
6704 6704
6705static inline void proxy_tag_curr(struct rq *rq, struct task_struct *owner)
6706{
6707 if (!sched_proxy_exec())
6708 return;
6709 /*
6710 * pick_next_task() calls set_next_task() on the chosen task
6711 * at some point, which ensures it is not push/pullable.
6712 * However, the chosen/donor task *and* the mutex owner form an
6713 * atomic pair wrt push/pull.
6714 *
6715 * Make sure owner we run is not pushable. Unfortunately we can
6716 * only deal with that by means of a dequeue/enqueue cycle. :-/
6717 */
6718 dequeue_task(rq, owner, DEQUEUE_NOCLOCK | DEQUEUE_SAVE);
6719 enqueue_task(rq, owner, ENQUEUE_NOCLOCK | ENQUEUE_RESTORE);
6720}
6721
6722/*6705/*
6723 * __schedule() is the main scheduler function.6706 * __schedule() is the main scheduler function.
6724 *6707 *
@@ -6871,9 +6854,6 @@ keep_resched: · 146 unmodified lines
6871 */6854 */
6872 RCU_INIT_POINTER(rq->curr, next);6855 RCU_INIT_POINTER(rq->curr, next);
6873 6856
6874 if (!task_current_donor(rq, next))
6875 proxy_tag_curr(rq, next);
6876
6877 /*6857 /*
6878 * The membarrier system call requires each architecture6858 * The membarrier system call requires each architecture
6879 * to have a full memory barrier after updating6859 * to have a full memory barrier after updating
@@ -6907,10 +6887,6 @@ keep_resched: · 27 unmodified lines
6907 /* Also unlocks the rq: */6887 /* Also unlocks the rq: */
6908 rq = context_switch(rq, prev, next, &rf);6888 rq = context_switch(rq, prev, next, &rf);
6909 } else {6889 } else {
6910 /* In case next was already curr but just got blocked_donor */
6911 if (!task_current_donor(rq, next))
6912 proxy_tag_curr(rq, next);
6913
6914 rq_unpin_lock(rq, &rf);6890 rq_unpin_lock(rq, &rf);
6915 __balance_callbacks(rq, NULL);6891 __balance_callbacks(rq, NULL);
6916 raw_spin_rq_unlock_irq(rq);6892 raw_spin_rq_unlock_irq(rq);
kernel/sched/deadline.c+16−2
@@ -2805,12 +2805,26 @@ static int find_later_rq(struct task_struct *task) · 2804 unmodified lines
2805 2805
2806static struct task_struct *pick_next_pushable_dl_task(struct rq *rq)2806static struct task_struct *pick_next_pushable_dl_task(struct rq *rq)
2807{2807{
2808 struct task_struct *p;2808 struct task_struct *i, *p = NULL;
2809 struct rb_node *next_node;
2809 2810
2810 if (!has_pushable_dl_tasks(rq))2811 if (!has_pushable_dl_tasks(rq))
2811 return NULL;2812 return NULL;
2812 2813
2813 p = __node_2_pdl(rb_first_cached(&rq->dl.pushable_dl_tasks_root));2814 next_node = rb_first_cached(&rq->dl.pushable_dl_tasks_root);
2815 while (next_node) {
2816 i = __node_2_pdl(next_node);
2817 /* make sure task isn't on_cpu (possible with proxy-exec) */
2818 if (!task_on_cpu(rq, i)) {
2819 p = i;
2820 break;
2821 }
2822
2823 next_node = rb_next(next_node);
2824 }
2825
2826 if (!p)
2827 return NULL;
2814 2828
2815 WARN_ON_ONCE(rq->cpu != task_cpu(p));2829 WARN_ON_ONCE(rq->cpu != task_cpu(p));
2816 WARN_ON_ONCE(task_current(rq, p));2830 WARN_ON_ONCE(task_current(rq, p));
kernel/sched/rt.c+12−3
@@ -1858,13 +1858,22 @@ static int find_lowest_rq(struct task_struct *task) · 1857 unmodified lines
1858 1858
1859static struct task_struct *pick_next_pushable_task(struct rq *rq)1859static struct task_struct *pick_next_pushable_task(struct rq *rq)
1860{1860{
1861 struct task_struct *p;1861 struct plist_head *head = &rq->rt.pushable_tasks;
1862 struct task_struct *i, *p = NULL;
1862 1863
1863 if (!has_pushable_tasks(rq))1864 if (!has_pushable_tasks(rq))
1864 return NULL;1865 return NULL;
1865 1866
1866 p = plist_first_entry(&rq->rt.pushable_tasks,1867 plist_for_each_entry(i, head, pushable_tasks) {
1867 struct task_struct, pushable_tasks);1868 /* make sure task isn't on_cpu (possible with proxy-exec) */
1869 if (!task_on_cpu(rq, i)) {
1870 p = i;
1871 break;
1872 }
1873 }
1874
1875 if (!p)
1876 return NULL;
1868 1877
1869 BUG_ON(rq->cpu != task_cpu(p));1878 BUG_ON(rq->cpu != task_cpu(p));
1870 BUG_ON(task_current(rq, p));1879 BUG_ON(task_current(rq, p));