When a cfs_rq sleeps and returns its quota, we delay for 5ms before waking any throttled cfs_rqs to coalesce with other cfs_rqs going to sleep, as this has to be done outside of the rq lock we hold. The current code waits for 5ms without any sleeps, instead of waiting for 5ms from the first sleep, which can delay the unthrottle more than we want. Switch this around so that we can't push this forward forever. This requires an extra flag rather than using hrtimer_active, since we need to start a new timer if the current one is in the process of finishing.
Unified · Split
| @@ -4729,6 +4729,11 @@ static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b) · 4728 unmodified lines | |||
| 4729 | if (runtime_refresh_within(cfs_b, min_left)) | 4729 | if (runtime_refresh_within(cfs_b, min_left)) |
| 4730 | return; | 4730 | return; |
| 4731 | 4731 | ||
| 4732 | /* don't push forwards an existing deferred unthrottle */ | ||
| 4733 | if (cfs_b->slack_started) | ||
| 4734 | return; | ||
| 4735 | cfs_b->slack_started = true; | ||
| 4736 | |||
| 4732 | hrtimer_start(&cfs_b->slack_timer, | 4737 | hrtimer_start(&cfs_b->slack_timer, |
| 4733 | ns_to_ktime(cfs_bandwidth_slack_period), | 4738 | ns_to_ktime(cfs_bandwidth_slack_period), |
| 4734 | HRTIMER_MODE_REL); | 4739 | HRTIMER_MODE_REL); |
| @@ -4782,6 +4787,7 @@ static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b) · 47 unmodified lines | |||
| 4782 | 4787 | ||
| 4783 | /* confirm we're still not at a refresh boundary */ | 4788 | /* confirm we're still not at a refresh boundary */ |
| 4784 | raw_spin_lock_irqsave(&cfs_b->lock, flags); | 4789 | raw_spin_lock_irqsave(&cfs_b->lock, flags); |
| 4790 | cfs_b->slack_started = false; | ||
| 4785 | if (cfs_b->distribute_running) { | 4791 | if (cfs_b->distribute_running) { |
| 4786 | raw_spin_unlock_irqrestore(&cfs_b->lock, flags); | 4792 | raw_spin_unlock_irqrestore(&cfs_b->lock, flags); |
| 4787 | return; | 4793 | return; |
| @@ -4945,6 +4951,7 @@ void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b) · 157 unmodified lines | |||
| 4945 | hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); | 4951 | hrtimer_init(&cfs_b->slack_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); |
| 4946 | cfs_b->slack_timer.function = sched_cfs_slack_timer; | 4952 | cfs_b->slack_timer.function = sched_cfs_slack_timer; |
| 4947 | cfs_b->distribute_running = 0; | 4953 | cfs_b->distribute_running = 0; |
| 4954 | cfs_b->slack_started = false; | ||
| 4948 | } | 4955 | } |
| 4949 | 4956 | ||
| 4950 | static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) | 4957 | static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) |
| @@ -338,8 +338,10 @@ struct cfs_bandwidth { · 337 unmodified lines | |||
| 338 | u64 runtime_expires; | 338 | u64 runtime_expires; |
| 339 | int expires_seq; | 339 | int expires_seq; |
| 340 | 340 | ||
| 341 | short idle; | 341 | u8 idle; |
| 342 | short period_active; | 342 | u8 period_active; |
| 343 | u8 distribute_running; | ||
| 344 | u8 slack_started; | ||
| 343 | struct hrtimer period_timer; | 345 | struct hrtimer period_timer; |
| 344 | struct hrtimer slack_timer; | 346 | struct hrtimer slack_timer; |
| 345 | struct list_head throttled_cfs_rq; | 347 | struct list_head throttled_cfs_rq; |
| @@ -348,8 +350,6 @@ struct cfs_bandwidth { · 2 unmodified lines | |||
| 348 | int nr_periods; | 350 | int nr_periods; |
| 349 | int nr_throttled; | 351 | int nr_throttled; |
| 350 | u64 throttled_time; | 352 | u64 throttled_time; |
| 351 | |||
| 352 | bool distribute_running; | ||
| 353 | #endif | 353 | #endif |
| 354 | }; | 354 | }; |
| 355 | 355 | ||