sched: Fix balance_callback()

The intent of balance_callback() has always been to delay executing balancing operations until the end of the current rq->lock section. This is because balance operations must often drop rq->lock, and that isn't safe in general. However, as noted by Scott, there were a few holes in that scheme; balance_callback() was called after rq->lock was dropped, which means another CPU can interleave and touch the callback list. Rework code to call the balance callbacks before dropping rq->lock where possible, and otherwise splice the balance list onto a local stack. This guarantees that the balance list must be empty when we take rq->lock. IOW, we'll only ever run our own balance callbacks.

Peter Zijlstra committed 6 years ago

Commit 565790d1 parent a8b62fd2 files · +78 −44Browse files

Unified · Split

2 files

kernel/sched/core.c +75 −44
kernel/sched/sched.h +3 −0

kernel/sched/core.c+75−44

@@ -3485,6 +3485,69 @@ static inline void finish_task(struct task_struct *prev) · 3484 unmodified lines
3485	3485	#endif
3486	3486	}
3487	3487
	3488	+#ifdef CONFIG_SMP
	3489	+
	3490	+static void do_balance_callbacks(struct rq rq, struct callback_head head)
	3491	+{
	3492	+ void (func)(struct rq rq);
	3493	+ struct callback_head *next;
	3494	+
	3495	+ lockdep_assert_held(&rq->lock);
	3496	+
	3497	+ while (head) {
	3498	+ func = (void ()(struct rq ))head->func;
	3499	+ next = head->next;
	3500	+ head->next = NULL;
	3501	+ head = next;
	3502	+
	3503	+ func(rq);
	3504	+ }
	3505	+}
	3506	+
	3507	+static inline struct callback_head splice_balance_callbacks(struct rq rq)
	3508	+{
	3509	+ struct callback_head *head = rq->balance_callback;
	3510	+
	3511	+ lockdep_assert_held(&rq->lock);
	3512	+ if (head)
	3513	+ rq->balance_callback = NULL;
	3514	+
	3515	+ return head;
	3516	+}
	3517	+
	3518	+static void __balance_callbacks(struct rq *rq)
	3519	+{
	3520	+ do_balance_callbacks(rq, splice_balance_callbacks(rq));
	3521	+}
	3522	+
	3523	+static inline void balance_callbacks(struct rq rq, struct callback_head head)
	3524	+{
	3525	+ unsigned long flags;
	3526	+
	3527	+ if (unlikely(head)) {
	3528	+ raw_spin_lock_irqsave(&rq->lock, flags);
	3529	+ do_balance_callbacks(rq, head);
	3530	+ raw_spin_unlock_irqrestore(&rq->lock, flags);
	3531	+ }
	3532	+}
	3533	+
	3534	+#else
	3535	+
	3536	+static inline void __balance_callbacks(struct rq *rq)
	3537	+{
	3538	+}
	3539	+
	3540	+static inline struct callback_head splice_balance_callbacks(struct rq rq)
	3541	+{
	3542	+ return NULL;
	3543	+}
	3544	+
	3545	+static inline void balance_callbacks(struct rq rq, struct callback_head head)
	3546	+{
	3547	+}
	3548	+
	3549	+#endif
	3550	+
3488	3551	static inline void
3489	3552	prepare_lock_switch(struct rq rq, struct task_struct next, struct rq_flags *rf)
3490	3553	{
@@ -3510,6 +3573,7 @@ static inline void finish_lock_switch(struct rq *rq) · 19 unmodified lines
3510	3573	* prev into current:
3511	3574	*/
3512	3575	spin_acquire(&rq->lock.dep_map, 0, 0, _THIS_IP_);
	3576	+ __balance_callbacks(rq);
3513	3577	raw_spin_unlock_irq(&rq->lock);
3514	3578	}
3515	3579
@@ -3651,43 +3715,6 @@ static struct rq finish_task_switch(struct task_struct prev) · 135 unmodified lines
3651	3715	return rq;
3652	3716	}
3653	3717
3654		−#ifdef CONFIG_SMP
3655		−
3656		−/* rq->lock is NOT held, but preemption is disabled */
3657		−static void __balance_callback(struct rq *rq)
3658		−{
3659		− struct callback_head head, next;
3660		− void (func)(struct rq rq);
3661		− unsigned long flags;
3662		−
3663		− raw_spin_lock_irqsave(&rq->lock, flags);
3664		− head = rq->balance_callback;
3665		− rq->balance_callback = NULL;
3666		− while (head) {
3667		− func = (void ()(struct rq ))head->func;
3668		− next = head->next;
3669		− head->next = NULL;
3670		− head = next;
3671		−
3672		− func(rq);
3673		− }
3674		− raw_spin_unlock_irqrestore(&rq->lock, flags);
3675		−}
3676		−
3677		−static inline void balance_callback(struct rq *rq)
3678		−{
3679		− if (unlikely(rq->balance_callback))
3680		− __balance_callback(rq);
3681		−}
3682		−
3683		−#else
3684		−
3685		−static inline void balance_callback(struct rq *rq)
3686		−{
3687		−}
3688		−
3689		−#endif
3690		−
3691	3718	/**
3692	3719	* schedule_tail - first thing a freshly forked thread must call.
3693	3720	* @prev: the thread we just switched away from.
@@ -3707,7 +3734,6 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev) · 13 unmodified lines
3707	3734	*/
3708	3735
3709	3736	rq = finish_task_switch(prev);
3710		− balance_callback(rq);
3711	3737	preempt_enable();
3712	3738
3713	3739	if (current->set_child_tid)
@@ -4523,10 +4549,11 @@ static void __sched notrace __schedule(bool preempt) · 809 unmodified lines
4523	4549	rq = context_switch(rq, prev, next, &rf);
4524	4550	} else {
4525	4551	rq->clock_update_flags &= ~(RQCF_ACT_SKIP\|RQCF_REQ_SKIP);
4526		− rq_unlock_irq(rq, &rf);
4527		− }
4528	4552
4529		− balance_callback(rq);
	4553	+ rq_unpin_lock(rq, &rf);
	4554	+ __balance_callbacks(rq);
	4555	+ raw_spin_unlock_irq(&rq->lock);
	4556	+ }
4530	4557	}
4531	4558
4532	4559	void __noreturn do_task_dead(void)
@@ -4937,9 +4964,11 @@ void rt_mutex_setprio(struct task_struct p, struct task_struct pi_task) · 404 unmodified lines
4937	4964	out_unlock:
4938	4965	/* Avoid rq from going away on us: */
4939	4966	preempt_disable();
4940		− __task_rq_unlock(rq, &rf);
4941	4967
4942		− balance_callback(rq);
	4968	+ rq_unpin_lock(rq, &rf);
	4969	+ __balance_callbacks(rq);
	4970	+ raw_spin_unlock(&rq->lock);
	4971	+
4943	4972	preempt_enable();
4944	4973	}
4945	4974	#else
@@ -5213,6 +5242,7 @@ static int __sched_setscheduler(struct task_struct *p, · 267 unmodified lines
5213	5242	int retval, oldprio, oldpolicy = -1, queued, running;
5214	5243	int new_effective_prio, policy = attr->sched_policy;
5215	5244	const struct sched_class *prev_class;
	5245	+ struct callback_head *head;
5216	5246	struct rq_flags rf;
5217	5247	int reset_on_fork;
5218	5248	int queue_flags = DEQUEUE_SAVE \| DEQUEUE_MOVE \| DEQUEUE_NOCLOCK;
@@ -5451,6 +5481,7 @@ change: · 232 unmodified lines
5451	5481
5452	5482	/* Avoid rq from going away on us: */
5453	5483	preempt_disable();
	5484	+ head = splice_balance_callbacks(rq);
5454	5485	task_rq_unlock(rq, p, &rf);
5455	5486
5456	5487	if (pi) {
@@ -5459,7 +5490,7 @@ change: · 2 unmodified lines
5459	5490	}
5460	5491
5461	5492	/* Run balance callbacks after we've adjusted the PI chain: */
5462		− balance_callback(rq);
	5493	+ balance_callbacks(rq, head);
5463	5494	preempt_enable();
5464	5495
5465	5496	return 0;

kernel/sched/sched.h+3−0

@@ -1221,6 +1221,9 @@ static inline void rq_pin_lock(struct rq rq, struct rq_flags rf) · 1220 unmodified lines
1221	1221	rq->clock_update_flags &= (RQCF_REQ_SKIP\|RQCF_ACT_SKIP);
1222	1222	rf->clock_update_flags = 0;
1223	1223	#endif
	1224	+#ifdef CONFIG_SMP
	1225	+ SCHED_WARN_ON(rq->balance_callback);
	1226	+#endif
1224	1227	}
1225	1228
1226	1229	static inline void rq_unpin_lock(struct rq rq, struct rq_flags rf)

sched: Fix balance_callback()

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code