History

sched/fair: Eliminate bandwidth race between throttling and distribution

There is a race window in which an entity begins throttling before quota is added to the pool, but does not finish throttling until after we have finished with distribute_cfs_runtime(). This entity is not observed by distribute_cfs_runtime() because it was not on the throttled list at the time that distribution was running. This race manifests as rare period-length statlls for such entities. Rather than heavy-weight the synchronization with the progress of distribution, we can fix this by aborting throttling if bandwidth has become available. Otherwise, we immediately add the entity to the throttled list so that it can be observed by a subsequent distribution. Additionally, we can remove the case of adding the throttled entity to the head of the throttled list, and simply always add to the tail. Thanks to 26a8b12747c97, distribute_cfs_runtime() no longer holds onto its own pool of runtime. This means that if we do hit the !assign and distribute_running case, we know that distribution is about to end.

Paul Turner committed 6 years ago

Commit e98fa021 parent f080d931 file · +47 −32Browse files

Unified · Split

kernel/sched/fair.c+47−32
@@ -4588,16 +4588,16 @@ static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) · 4587 unmodified lines
45884588 }
45894589
45904590 /* returns 0 on failure to allocate runtime */
4591−static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4591+static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b,
4592+ struct cfs_rq *cfs_rq, u64 target_runtime)
45924593 {
4593− struct task_group *tg = cfs_rq->tg;
4594− struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
4595− u64 amount = 0, min_amount;
4594+ u64 min_amount, amount = 0;
4595+
4596+ lockdep_assert_held(&cfs_b->lock);
45964597
45974598 /* note: this is a positive sum as runtime_remaining <= 0 */
4598− min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining;
4599+ min_amount = target_runtime - cfs_rq->runtime_remaining;
45994600
4600− raw_spin_lock(&cfs_b->lock);
46014601 if (cfs_b->quota == RUNTIME_INF)
46024602 amount = min_amount;
46034603 else {
@@ -4609,13 +4609,25 @@ static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) · 5 unmodified lines
46094609 cfs_b->idle = 0;
46104610 }
46114611 }
4612− raw_spin_unlock(&cfs_b->lock);
46134612
46144613 cfs_rq->runtime_remaining += amount;
46154614
46164615 return cfs_rq->runtime_remaining > 0;
46174616 }
46184617
4618+/* returns 0 on failure to allocate runtime */
4619+static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4620+{
4621+ struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4622+ int ret;
4623+
4624+ raw_spin_lock(&cfs_b->lock);
4625+ ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice());
4626+ raw_spin_unlock(&cfs_b->lock);
4627+
4628+ return ret;
4629+}
4630+
46194631 static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
46204632 {
46214633 /* dock delta_exec before expiring quota (as it could span periods) */
@@ -4704,13 +4716,33 @@ static int tg_throttle_down(struct task_group *tg, void *data) · 82 unmodified lines
47044716 return 0;
47054717 }
47064718
4707−static void throttle_cfs_rq(struct cfs_rq *cfs_rq)
4719+static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
47084720 {
47094721 struct rq *rq = rq_of(cfs_rq);
47104722 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
47114723 struct sched_entity *se;
47124724 long task_delta, idle_task_delta, dequeue = 1;
4713− bool empty;
4725+
4726+ raw_spin_lock(&cfs_b->lock);
4727+ /* This will start the period timer if necessary */
4728+ if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) {
4729+ /*
4730+ * We have raced with bandwidth becoming available, and if we
4731+ * actually throttled the timer might not unthrottle us for an
4732+ * entire period. We additionally needed to make sure that any
4733+ * subsequent check_cfs_rq_runtime calls agree not to throttle
4734+ * us, as we may commit to do cfs put_prev+pick_next, so we ask
4735+ * for 1ns of runtime rather than just check cfs_b.
4736+ */
4737+ dequeue = 0;
4738+ } else {
4739+ list_add_tail_rcu(&cfs_rq->throttled_list,
4740+ &cfs_b->throttled_cfs_rq);
4741+ }
4742+ raw_spin_unlock(&cfs_b->lock);
4743+
4744+ if (!dequeue)
4745+ return false; /* Throttle no longer required. */
47144746
47154747 se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
47164748
@@ -4744,29 +4776,13 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) · 27 unmodified lines
47444776 if (!se)
47454777 sub_nr_running(rq, task_delta);
47464778
4747− cfs_rq->throttled = 1;
4748− cfs_rq->throttled_clock = rq_clock(rq);
4749− raw_spin_lock(&cfs_b->lock);
4750− empty = list_empty(&cfs_b->throttled_cfs_rq);
4751−
4752− /*
4753− * Add to the _head_ of the list, so that an already-started
4754− * distribute_cfs_runtime will not see us. If disribute_cfs_runtime is
4755− * not running add to the tail so that later runqueues don't get starved.
4756− */
4757− if (cfs_b->distribute_running)
4758− list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
4759− else
4760− list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
4761−
47624779 /*
4763− * If we're the first throttled task, make sure the bandwidth
4764− * timer is running.
4780+ * Note: distribution will already see us throttled via the
4781+ * throttled-list. rq->lock protects completion.
47654782 */
4766− if (empty)
4767− start_cfs_bandwidth(cfs_b);
4768−
4769− raw_spin_unlock(&cfs_b->lock);
4783+ cfs_rq->throttled = 1;
4784+ cfs_rq->throttled_clock = rq_clock(rq);
4785+ return true;
47704786 }
47714787
47724788 void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
@@ -5121,8 +5137,7 @@ static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) · 348 unmodified lines
51215137 if (cfs_rq_throttled(cfs_rq))
51225138 return true;
51235139
5124− throttle_cfs_rq(cfs_rq);
5125− return true;
5140+ return throttle_cfs_rq(cfs_rq);
51265141 }
51275142
51285143 static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t