History

sched/fair: Eliminate bandwidth race between throttling and distribution

There is a race window in which an entity begins throttling before quota is added to the pool, but does not finish throttling until after we have finished with distribute_cfs_runtime(). This entity is not observed by distribute_cfs_runtime() because it was not on the throttled list at the time that distribution was running. This race manifests as rare period-length statlls for such entities. Rather than heavy-weight the synchronization with the progress of distribution, we can fix this by aborting throttling if bandwidth has become available. Otherwise, we immediately add the entity to the throttled list so that it can be observed by a subsequent distribution. Additionally, we can remove the case of adding the throttled entity to the head of the throttled list, and simply always add to the tail. Thanks to 26a8b12747c97, distribute_cfs_runtime() no longer holds onto its own pool of runtime. This means that if we do hit the !assign and distribute_running case, we know that distribution is about to end.

Paul Turner committed 6 years ago

Commit e98fa021 parent f080d931 file · +47 −32Browse files

Unified · Split

kernel/sched/fair.c+47−32
@@ -4588,16 +4588,16 @@ static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg) · 4587 unmodified lines
4588}4588}
4589 4589
4590/* returns 0 on failure to allocate runtime */4590/* returns 0 on failure to allocate runtime */
4591static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)4591static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b,
4592 struct cfs_rq *cfs_rq, u64 target_runtime)
4592{4593{
4593 struct task_group *tg = cfs_rq->tg;4594 u64 min_amount, amount = 0;
4594 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);4595
4595 u64 amount = 0, min_amount;4596 lockdep_assert_held(&cfs_b->lock);
4596 4597
4597 /* note: this is a positive sum as runtime_remaining <= 0 */4598 /* note: this is a positive sum as runtime_remaining <= 0 */
4598 min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining;4599 min_amount = target_runtime - cfs_rq->runtime_remaining;
4599 4600
4600 raw_spin_lock(&cfs_b->lock);
4601 if (cfs_b->quota == RUNTIME_INF)4601 if (cfs_b->quota == RUNTIME_INF)
4602 amount = min_amount;4602 amount = min_amount;
4603 else {4603 else {
@@ -4609,13 +4609,25 @@ static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq) · 5 unmodified lines
4609 cfs_b->idle = 0;4609 cfs_b->idle = 0;
4610 }4610 }
4611 }4611 }
4612 raw_spin_unlock(&cfs_b->lock);
4613 4612
4614 cfs_rq->runtime_remaining += amount;4613 cfs_rq->runtime_remaining += amount;
4615 4614
4616 return cfs_rq->runtime_remaining > 0;4615 return cfs_rq->runtime_remaining > 0;
4617}4616}
4618 4617
4618/* returns 0 on failure to allocate runtime */
4619static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
4620{
4621 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4622 int ret;
4623
4624 raw_spin_lock(&cfs_b->lock);
4625 ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, sched_cfs_bandwidth_slice());
4626 raw_spin_unlock(&cfs_b->lock);
4627
4628 return ret;
4629}
4630
4619static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)4631static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
4620{4632{
4621 /* dock delta_exec before expiring quota (as it could span periods) */4633 /* dock delta_exec before expiring quota (as it could span periods) */
@@ -4704,13 +4716,33 @@ static int tg_throttle_down(struct task_group *tg, void *data) · 82 unmodified lines
4704 return 0;4716 return 0;
4705}4717}
4706 4718
4707static void throttle_cfs_rq(struct cfs_rq *cfs_rq)4719static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
4708{4720{
4709 struct rq *rq = rq_of(cfs_rq);4721 struct rq *rq = rq_of(cfs_rq);
4710 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);4722 struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(cfs_rq->tg);
4711 struct sched_entity *se;4723 struct sched_entity *se;
4712 long task_delta, idle_task_delta, dequeue = 1;4724 long task_delta, idle_task_delta, dequeue = 1;
4713 bool empty;4725
4726 raw_spin_lock(&cfs_b->lock);
4727 /* This will start the period timer if necessary */
4728 if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, 1)) {
4729 /*
4730 * We have raced with bandwidth becoming available, and if we
4731 * actually throttled the timer might not unthrottle us for an
4732 * entire period. We additionally needed to make sure that any
4733 * subsequent check_cfs_rq_runtime calls agree not to throttle
4734 * us, as we may commit to do cfs put_prev+pick_next, so we ask
4735 * for 1ns of runtime rather than just check cfs_b.
4736 */
4737 dequeue = 0;
4738 } else {
4739 list_add_tail_rcu(&cfs_rq->throttled_list,
4740 &cfs_b->throttled_cfs_rq);
4741 }
4742 raw_spin_unlock(&cfs_b->lock);
4743
4744 if (!dequeue)
4745 return false; /* Throttle no longer required. */
4714 4746
4715 se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];4747 se = cfs_rq->tg->se[cpu_of(rq_of(cfs_rq))];
4716 4748
@@ -4744,29 +4776,13 @@ static void throttle_cfs_rq(struct cfs_rq *cfs_rq) · 27 unmodified lines
4744 if (!se)4776 if (!se)
4745 sub_nr_running(rq, task_delta);4777 sub_nr_running(rq, task_delta);
4746 4778
4747 cfs_rq->throttled = 1;
4748 cfs_rq->throttled_clock = rq_clock(rq);
4749 raw_spin_lock(&cfs_b->lock);
4750 empty = list_empty(&cfs_b->throttled_cfs_rq);
4751
4752 /*
4753 * Add to the _head_ of the list, so that an already-started
4754 * distribute_cfs_runtime will not see us. If disribute_cfs_runtime is
4755 * not running add to the tail so that later runqueues don't get starved.
4756 */
4757 if (cfs_b->distribute_running)
4758 list_add_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
4759 else
4760 list_add_tail_rcu(&cfs_rq->throttled_list, &cfs_b->throttled_cfs_rq);
4761
4762 /*4779 /*
4763 * If we're the first throttled task, make sure the bandwidth4780 * Note: distribution will already see us throttled via the
4764 * timer is running.4781 * throttled-list. rq->lock protects completion.
4765 */4782 */
4766 if (empty)4783 cfs_rq->throttled = 1;
4767 start_cfs_bandwidth(cfs_b);4784 cfs_rq->throttled_clock = rq_clock(rq);
4768 4785 return true;
4769 raw_spin_unlock(&cfs_b->lock);
4770}4786}
4771 4787
4772void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)4788void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
@@ -5121,8 +5137,7 @@ static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) · 348 unmodified lines
5121 if (cfs_rq_throttled(cfs_rq))5137 if (cfs_rq_throttled(cfs_rq))
5122 return true;5138 return true;
5123 5139
5124 throttle_cfs_rq(cfs_rq);5140 return throttle_cfs_rq(cfs_rq);
5125 return true;
5126}5141}
5127 5142
5128static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)5143static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t