timers/nohz: Update NOHZ load in remote tick

The way loadavg is tracked during nohz only pays attention to the load upon entering nohz. This can be particularly noticeable if full nohz is entered while non-idle, and then the cpu goes idle and stays that way for a long time. Use the remote tick to ensure that full nohz cpus report their deltas within a reasonable time. [ swood: Added changelog and removed recheck of stopped tick. ]

Peter Zijlstra (Intel) committed 6 years ago

Commit ebc0f831 parent 488603b3 files · +28 −11Browse files

Unified · Split

3 files

include/linux/sched/nohz.h +2 −0
kernel/sched/core.c +3 −1
kernel/sched/loadavg.c +23 −10

include/linux/sched/nohz.h+2−0

@@ -15,9 +15,11 @@ static inline void nohz_balance_enter_idle(int cpu) { } · 14 unmodified lines
15	15
16	16	#ifdef CONFIG_NO_HZ_COMMON
17	17	void calc_load_nohz_start(void);
	18	+void calc_load_nohz_remote(struct rq *rq);
18	19	void calc_load_nohz_stop(void);
19	20	#else
20	21	static inline void calc_load_nohz_start(void) { }
	22	+static inline void calc_load_nohz_remote(struct rq *rq) { }
21	23	static inline void calc_load_nohz_stop(void) { }
22	24	#endif /* CONFIG_NO_HZ_COMMON */
23	25

kernel/sched/core.c+3−1

@@ -3677,6 +3677,7 @@ static void sched_tick_remote(struct work_struct *work) · 3676 unmodified lines
3677	3677	if (cpu_is_offline(cpu))
3678	3678	goto out_unlock;
3679	3679
	3680	+ curr = rq->curr;
3680	3681	update_rq_clock(rq);
3681	3682
3682	3683	if (!is_idle_task(curr)) {
@@ -3689,10 +3690,11 @@ static void sched_tick_remote(struct work_struct *work) · 6 unmodified lines
3689	3690	}
3690	3691	curr->sched_class->task_tick(rq, curr, 0);
3691	3692
	3693	+ calc_load_nohz_remote(rq);
3692	3694	out_unlock:
3693	3695	rq_unlock_irq(rq, &rf);
3694		−
3695	3696	out_requeue:
	3697	+
3696	3698	/*
3697	3699	* Run the remote tick once per second (1Hz). This arbitrary
3698	3700	* frequency is large enough to avoid overload but short enough

kernel/sched/loadavg.c+23−10

@@ -231,16 +231,11 @@ static inline int calc_load_read_idx(void) · 230 unmodified lines
231	231	return calc_load_idx & 1;
232	232	}
233	233
234		−void calc_load_nohz_start(void)
	234	+static void calc_load_nohz_fold(struct rq *rq)
235	235	{
236		− struct rq *this_rq = this_rq();
237	236	long delta;
238	237
239		− /*
240		− * We're going into NO_HZ mode, if there's any pending delta, fold it
241		− * into the pending NO_HZ delta.
242		− */
243		− delta = calc_load_fold_active(this_rq, 0);
	238	+ delta = calc_load_fold_active(rq, 0);
244	239	if (delta) {
245	240	int idx = calc_load_write_idx();
246	241
@@ -248,6 +243,24 @@ void calc_load_nohz_start(void) · 1 unmodified line
248	243	}
249	244	}
250	245
	246	+void calc_load_nohz_start(void)
	247	+{
	248	+ /*
	249	+ * We're going into NO_HZ mode, if there's any pending delta, fold it
	250	+ * into the pending NO_HZ delta.
	251	+ */
	252	+ calc_load_nohz_fold(this_rq());
	253	+}
	254	+
	255	+/*
	256	+ * Keep track of the load for NOHZ_FULL, must be called between
	257	+ * calc_load_nohz_{start,stop}().
	258	+ */
	259	+void calc_load_nohz_remote(struct rq *rq)
	260	+{
	261	+ calc_load_nohz_fold(rq);
	262	+}
	263	+
251	264	void calc_load_nohz_stop(void)
252	265	{
253	266	struct rq *this_rq = this_rq();
@@ -268,7 +281,7 @@ void calc_load_nohz_stop(void) · 14 unmodified lines
268	281	this_rq->calc_load_update += LOAD_FREQ;
269	282	}
270	283
271		−static long calc_load_nohz_fold(void)
	284	+static long calc_load_nohz_read(void)
272	285	{
273	286	int idx = calc_load_read_idx();
274	287	long delta = 0;
@@ -323,7 +336,7 @@ static void calc_global_nohz(void) · 48 unmodified lines
323	336	}
324	337	#else /* !CONFIG_NO_HZ_COMMON */
325	338
326		−static inline long calc_load_nohz_fold(void) { return 0; }
	339	+static inline long calc_load_nohz_read(void) { return 0; }
327	340	static inline void calc_global_nohz(void) { }
328	341
329	342	#endif /* CONFIG_NO_HZ_COMMON */
@@ -346,7 +359,7 @@ void calc_global_load(unsigned long ticks) · 16 unmodified lines
346	359	/*
347	360	* Fold the 'old' NO_HZ-delta to include all NO_HZ CPUs.
348	361	*/
349		− delta = calc_load_nohz_fold();
	362	+ delta = calc_load_nohz_read();
350	363	if (delta)
351	364	atomic_long_add(delta, &calc_load_tasks);
352	365

timers/nohz: Update NOHZ load in remote tick

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code