sched: Make class_schedulers avoid pushing current, and get rid of proxy_tag_curr()

With proxy-execution, the scheduler selects the donor, but for blocked donors, we end up running the lock owner. This caused some complexity, because the class schedulers make sure to remove the task they pick from their pushable task lists, which prevents the donor from being migrated, but there wasn't then anything to prevent rq->curr from being migrated if rq->curr != rq->donor. This was sort of hacked around by calling proxy_tag_curr() on the rq->curr task if we were running something other then the donor. proxy_tag_curr() did a dequeue/enqueue pair on the rq->curr task, allowing the class schedulers to remove it from their pushable list. The dequeue/enqueue pair was wasteful, and additonally K Prateek highlighted that we didn't properly undo things when we stopped proxying, leaving the lock owner off the pushable list. After some alternative approaches were considered, Peter suggested just having the RT/DL classes just avoid migrating when task_on_cpu(). So rework pick_next_pushable_dl_task() and the rt pick_next_pushable_task() functions so that they skip over the first pushable task if it is on_cpu. Then just drop all of the proxy_tag_curr() logic.

John Stultz committed 7 months ago

Commit e0ca8991 parent 98539143 files · +28 −29Browse files

Unified · Split

3 files

kernel/sched/core.c +0 −24
kernel/sched/deadline.c +16 −2
kernel/sched/rt.c +12 −3

kernel/sched/core.c+0−24

@@ -6702,23 +6702,6 @@ find_proxy_task(struct rq rq, struct task_struct donor, struct rq_flags *rf) · 6701 unmodified lines
6702	6702	}
6703	6703	#endif /* SCHED_PROXY_EXEC */
6704	6704
6705		−static inline void proxy_tag_curr(struct rq rq, struct task_struct owner)
6706		−{
6707		− if (!sched_proxy_exec())
6708		− return;
6709		− /*
6710		− * pick_next_task() calls set_next_task() on the chosen task
6711		− * at some point, which ensures it is not push/pullable.
6712		− * However, the chosen/donor task and the mutex owner form an
6713		− * atomic pair wrt push/pull.
6714		− *
6715		− * Make sure owner we run is not pushable. Unfortunately we can
6716		− * only deal with that by means of a dequeue/enqueue cycle. :-/
6717		− */
6718		− dequeue_task(rq, owner, DEQUEUE_NOCLOCK \| DEQUEUE_SAVE);
6719		− enqueue_task(rq, owner, ENQUEUE_NOCLOCK \| ENQUEUE_RESTORE);
6720		−}
6721		−
6722	6705	/*
6723	6706	* __schedule() is the main scheduler function.
6724	6707	*
@@ -6871,9 +6854,6 @@ keep_resched: · 146 unmodified lines
6871	6854	*/
6872	6855	RCU_INIT_POINTER(rq->curr, next);
6873	6856
6874		− if (!task_current_donor(rq, next))
6875		− proxy_tag_curr(rq, next);
6876		−
6877	6857	/*
6878	6858	* The membarrier system call requires each architecture
6879	6859	* to have a full memory barrier after updating
@@ -6907,10 +6887,6 @@ keep_resched: · 27 unmodified lines
6907	6887	/* Also unlocks the rq: */
6908	6888	rq = context_switch(rq, prev, next, &rf);
6909	6889	} else {
6910		− /* In case next was already curr but just got blocked_donor */
6911		− if (!task_current_donor(rq, next))
6912		− proxy_tag_curr(rq, next);
6913		−
6914	6890	rq_unpin_lock(rq, &rf);
6915	6891	__balance_callbacks(rq, NULL);
6916	6892	raw_spin_rq_unlock_irq(rq);

kernel/sched/deadline.c+16−2

@@ -2805,12 +2805,26 @@ static int find_later_rq(struct task_struct *task) · 2804 unmodified lines
2805	2805
2806	2806	static struct task_struct pick_next_pushable_dl_task(struct rq rq)
2807	2807	{
2808		− struct task_struct *p;
	2808	+ struct task_struct i, p = NULL;
	2809	+ struct rb_node *next_node;
2809	2810
2810	2811	if (!has_pushable_dl_tasks(rq))
2811	2812	return NULL;
2812	2813
2813		− p = __node_2_pdl(rb_first_cached(&rq->dl.pushable_dl_tasks_root));
	2814	+ next_node = rb_first_cached(&rq->dl.pushable_dl_tasks_root);
	2815	+ while (next_node) {
	2816	+ i = __node_2_pdl(next_node);
	2817	+ /* make sure task isn't on_cpu (possible with proxy-exec) */
	2818	+ if (!task_on_cpu(rq, i)) {
	2819	+ p = i;
	2820	+ break;
	2821	+ }
	2822	+
	2823	+ next_node = rb_next(next_node);
	2824	+ }
	2825	+
	2826	+ if (!p)
	2827	+ return NULL;
2814	2828
2815	2829	WARN_ON_ONCE(rq->cpu != task_cpu(p));
2816	2830	WARN_ON_ONCE(task_current(rq, p));

kernel/sched/rt.c+12−3

@@ -1858,13 +1858,22 @@ static int find_lowest_rq(struct task_struct *task) · 1857 unmodified lines
1858	1858
1859	1859	static struct task_struct pick_next_pushable_task(struct rq rq)
1860	1860	{
1861		− struct task_struct *p;
	1861	+ struct plist_head *head = &rq->rt.pushable_tasks;
	1862	+ struct task_struct i, p = NULL;
1862	1863
1863	1864	if (!has_pushable_tasks(rq))
1864	1865	return NULL;
1865	1866
1866		− p = plist_first_entry(&rq->rt.pushable_tasks,
1867		− struct task_struct, pushable_tasks);
	1867	+ plist_for_each_entry(i, head, pushable_tasks) {
	1868	+ /* make sure task isn't on_cpu (possible with proxy-exec) */
	1869	+ if (!task_on_cpu(rq, i)) {
	1870	+ p = i;
	1871	+ break;
	1872	+ }
	1873	+ }
	1874	+
	1875	+ if (!p)
	1876	+ return NULL;
1868	1877
1869	1878	BUG_ON(rq->cpu != task_cpu(p));
1870	1879	BUG_ON(task_current(rq, p));