History

timers/nohz: Update NOHZ load in remote tick

The way loadavg is tracked during nohz only pays attention to the load upon entering nohz. This can be particularly noticeable if full nohz is entered while non-idle, and then the cpu goes idle and stays that way for a long time. Use the remote tick to ensure that full nohz cpus report their deltas within a reasonable time. [ swood: Added changelog and removed recheck of stopped tick. ]

Peter Zijlstra (Intel) committed 6 years ago

Commit ebc0f831 parent 488603b3 files · +28 −11Browse files

Unified · Split

3 files
  1. include/linux/sched/nohz.h +2 −0
  2. kernel/sched/core.c +3 −1
  3. kernel/sched/loadavg.c +23 −10
include/linux/sched/nohz.h+2−0
@@ -15,9 +15,11 @@ static inline void nohz_balance_enter_idle(int cpu) { } · 14 unmodified lines
15 15
16#ifdef CONFIG_NO_HZ_COMMON16#ifdef CONFIG_NO_HZ_COMMON
17void calc_load_nohz_start(void);17void calc_load_nohz_start(void);
18void calc_load_nohz_remote(struct rq *rq);
18void calc_load_nohz_stop(void);19void calc_load_nohz_stop(void);
19#else20#else
20static inline void calc_load_nohz_start(void) { }21static inline void calc_load_nohz_start(void) { }
22static inline void calc_load_nohz_remote(struct rq *rq) { }
21static inline void calc_load_nohz_stop(void) { }23static inline void calc_load_nohz_stop(void) { }
22#endif /* CONFIG_NO_HZ_COMMON */24#endif /* CONFIG_NO_HZ_COMMON */
23 25
kernel/sched/core.c+3−1
@@ -3677,6 +3677,7 @@ static void sched_tick_remote(struct work_struct *work) · 3676 unmodified lines
3677 if (cpu_is_offline(cpu))3677 if (cpu_is_offline(cpu))
3678 goto out_unlock;3678 goto out_unlock;
3679 3679
3680 curr = rq->curr;
3680 update_rq_clock(rq);3681 update_rq_clock(rq);
3681 3682
3682 if (!is_idle_task(curr)) {3683 if (!is_idle_task(curr)) {
@@ -3689,10 +3690,11 @@ static void sched_tick_remote(struct work_struct *work) · 6 unmodified lines
3689 }3690 }
3690 curr->sched_class->task_tick(rq, curr, 0);3691 curr->sched_class->task_tick(rq, curr, 0);
3691 3692
3693 calc_load_nohz_remote(rq);
3692out_unlock:3694out_unlock:
3693 rq_unlock_irq(rq, &rf);3695 rq_unlock_irq(rq, &rf);
3694
3695out_requeue:3696out_requeue:
3697
3696 /*3698 /*
3697 * Run the remote tick once per second (1Hz). This arbitrary3699 * Run the remote tick once per second (1Hz). This arbitrary
3698 * frequency is large enough to avoid overload but short enough3700 * frequency is large enough to avoid overload but short enough
kernel/sched/loadavg.c+23−10
@@ -231,16 +231,11 @@ static inline int calc_load_read_idx(void) · 230 unmodified lines
231 return calc_load_idx & 1;231 return calc_load_idx & 1;
232}232}
233 233
234void calc_load_nohz_start(void)234static void calc_load_nohz_fold(struct rq *rq)
235{235{
236 struct rq *this_rq = this_rq();
237 long delta;236 long delta;
238 237
239 /*238 delta = calc_load_fold_active(rq, 0);
240 * We're going into NO_HZ mode, if there's any pending delta, fold it
241 * into the pending NO_HZ delta.
242 */
243 delta = calc_load_fold_active(this_rq, 0);
244 if (delta) {239 if (delta) {
245 int idx = calc_load_write_idx();240 int idx = calc_load_write_idx();
246 241
@@ -248,6 +243,24 @@ void calc_load_nohz_start(void) · 1 unmodified line
248 }243 }
249}244}
250 245
246void calc_load_nohz_start(void)
247{
248 /*
249 * We're going into NO_HZ mode, if there's any pending delta, fold it
250 * into the pending NO_HZ delta.
251 */
252 calc_load_nohz_fold(this_rq());
253}
254
255/*
256 * Keep track of the load for NOHZ_FULL, must be called between
257 * calc_load_nohz_{start,stop}().
258 */
259void calc_load_nohz_remote(struct rq *rq)
260{
261 calc_load_nohz_fold(rq);
262}
263
251void calc_load_nohz_stop(void)264void calc_load_nohz_stop(void)
252{265{
253 struct rq *this_rq = this_rq();266 struct rq *this_rq = this_rq();
@@ -268,7 +281,7 @@ void calc_load_nohz_stop(void) · 14 unmodified lines
268 this_rq->calc_load_update += LOAD_FREQ;281 this_rq->calc_load_update += LOAD_FREQ;
269}282}
270 283
271static long calc_load_nohz_fold(void)284static long calc_load_nohz_read(void)
272{285{
273 int idx = calc_load_read_idx();286 int idx = calc_load_read_idx();
274 long delta = 0;287 long delta = 0;
@@ -323,7 +336,7 @@ static void calc_global_nohz(void) · 48 unmodified lines
323}336}
324#else /* !CONFIG_NO_HZ_COMMON */337#else /* !CONFIG_NO_HZ_COMMON */
325 338
326static inline long calc_load_nohz_fold(void) { return 0; }339static inline long calc_load_nohz_read(void) { return 0; }
327static inline void calc_global_nohz(void) { }340static inline void calc_global_nohz(void) { }
328 341
329#endif /* CONFIG_NO_HZ_COMMON */342#endif /* CONFIG_NO_HZ_COMMON */
@@ -346,7 +359,7 @@ void calc_global_load(unsigned long ticks) · 16 unmodified lines
346 /*359 /*
347 * Fold the 'old' NO_HZ-delta to include all NO_HZ CPUs.360 * Fold the 'old' NO_HZ-delta to include all NO_HZ CPUs.
348 */361 */
349 delta = calc_load_nohz_fold();362 delta = calc_load_nohz_read();
350 if (delta)363 if (delta)
351 atomic_long_add(delta, &calc_load_tasks);364 atomic_long_add(delta, &calc_load_tasks);
352 365

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t