History

sched clock: revert various sched_clock() changes

Found an interactivity problem on a quad core test-system - simple CPU loops would occasionally delay the system un an unacceptable way. After much debugging with Peter Zijlstra it turned out that the problem is caused by the string of sched_clock() changes - they caused the CPU clock to jump backwards a bit - which confuses the scheduler arithmetics. (which is unsigned for performance reasons) So revert: # c300ba2: sched_clock: and multiplier for TSC to gtod drift # c0c8773: sched_clock: only update deltas with local reads. # af52a90: sched_clock: stop maximum check on NO HZ # f7cce27: sched_clock: widen the max and min time This solves the interactivity problems.

Ingo Molnar committed 18 years ago

Commit e4e4e531 parent 39675e83 files · +14 −114Browse files

Unified · Split

3 files
  1. include/linux/sched.h +1 −16
  2. kernel/sched_clock.c +13 −96
  3. kernel/time/tick-sched.c +0 −2
include/linux/sched.h+1−16
@@ -1572,28 +1572,13 @@ static inline void sched_clock_idle_sleep_event(void) · 1571 unmodified lines
15721572 static inline void sched_clock_idle_wakeup_event(u64 delta_ns)
15731573 {
15741574 }
1575−
1576−#ifdef CONFIG_NO_HZ
1577−static inline void sched_clock_tick_stop(int cpu)
1578−{
1579−}
1580−
1581−static inline void sched_clock_tick_start(int cpu)
1582−{
1583−}
1584−#endif
1585−
1586−#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
1575+#else
15871576 extern void sched_clock_init(void);
15881577 extern u64 sched_clock_cpu(int cpu);
15891578 extern void sched_clock_tick(void);
15901579 extern void sched_clock_idle_sleep_event(void);
15911580 extern void sched_clock_idle_wakeup_event(u64 delta_ns);
1592−#ifdef CONFIG_NO_HZ
1593−extern void sched_clock_tick_stop(int cpu);
1594−extern void sched_clock_tick_start(int cpu);
15951581 #endif
1596−#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
15971582
15981583 /*
15991584 * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
kernel/sched_clock.c+13−96
@@ -44,11 +44,6 @@ unsigned long long __attribute__((weak)) sched_clock(void) · 43 unmodified lines
4444
4545 #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
4646
47−#define MULTI_SHIFT 15
48−/* Max is double, Min is 1/2 */
49−#define MAX_MULTI (2LL << MULTI_SHIFT)
50−#define MIN_MULTI (1LL << (MULTI_SHIFT-1))
51−
5247 struct sched_clock_data {
5348 /*
5449 * Raw spinlock - this is a special case: this might be called
@@ -62,10 +57,6 @@ struct sched_clock_data { · 7 unmodified lines
6257 u64 tick_raw;
6358 u64 tick_gtod;
6459 u64 clock;
65− s64 multi;
66−#ifdef CONFIG_NO_HZ
67− int check_max;
68−#endif
6960 };
7061
7162 static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
@@ -97,53 +88,18 @@ void sched_clock_init(void) · 25 unmodified lines
9788 scd->tick_raw = 0;
9889 scd->tick_gtod = ktime_now;
9990 scd->clock = ktime_now;
100− scd->multi = 1 << MULTI_SHIFT;
101−#ifdef CONFIG_NO_HZ
102− scd->check_max = 1;
103−#endif
10491 }
10592
10693 sched_clock_running = 1;
10794 }
10895
109−#ifdef CONFIG_NO_HZ
110−/*
111− * The dynamic ticks makes the delta jiffies inaccurate. This
112− * prevents us from checking the maximum time update.
113− * Disable the maximum check during stopped ticks.
114− */
115−void sched_clock_tick_stop(int cpu)
116−{
117− struct sched_clock_data *scd = cpu_sdc(cpu);
118−
119− scd->check_max = 0;
120−}
121−
122−void sched_clock_tick_start(int cpu)
123−{
124− struct sched_clock_data *scd = cpu_sdc(cpu);
125−
126− scd->check_max = 1;
127−}
128−
129−static int check_max(struct sched_clock_data *scd)
130−{
131− return scd->check_max;
132−}
133−#else
134−static int check_max(struct sched_clock_data *scd)
135−{
136− return 1;
137−}
138−#endif /* CONFIG_NO_HZ */
139−
14096 /*
14197 * update the percpu scd from the raw @now value
14298 *
14399 * - filter out backward motion
144100 * - use jiffies to generate a min,max window to clip the raw values
145101 */
146−static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *time)
102+static void __update_sched_clock(struct sched_clock_data *scd, u64 now)
147103 {
148104 unsigned long now_jiffies = jiffies;
149105 long delta_jiffies = now_jiffies - scd->tick_jiffies;
@@ -152,31 +108,16 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim · 2 unmodified lines
152108 s64 delta = now - scd->prev_raw;
153109
154110 WARN_ON_ONCE(!irqs_disabled());
155−
156− /*
157− * At schedule tick the clock can be just under the gtod. We don't
158− * want to push it too prematurely.
159− */
160− min_clock = scd->tick_gtod + (delta_jiffies * TICK_NSEC);
161− if (min_clock > TICK_NSEC)
162− min_clock -= TICK_NSEC / 2;
111+ min_clock = scd->tick_gtod + delta_jiffies * TICK_NSEC;
163112
164113 if (unlikely(delta < 0)) {
165114 clock++;
166115 goto out;
167116 }
168117
169− /*
170− * The clock must stay within a jiffie of the gtod.
171− * But since we may be at the start of a jiffy or the end of one
172− * we add another jiffy buffer.
173− */
174− max_clock = scd->tick_gtod + (2 + delta_jiffies) * TICK_NSEC;
175−
176− delta *= scd->multi;
177− delta >>= MULTI_SHIFT;
118+ max_clock = min_clock + TICK_NSEC;
178119
179− if (unlikely(clock + delta > max_clock) && check_max(scd)) {
120+ if (unlikely(clock + delta > max_clock)) {
180121 if (clock < max_clock)
181122 clock = max_clock;
182123 else
@@ -189,12 +130,9 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim · 6 unmodified lines
189130 if (unlikely(clock < min_clock))
190131 clock = min_clock;
191132
192− if (time)
193− *time = clock;
194− else {
195− scd->prev_raw = now;
196− scd->clock = clock;
197− }
133+ scd->prev_raw = now;
134+ scd->tick_jiffies = now_jiffies;
135+ scd->clock = clock;
198136 }
199137
200138 static void lock_double_clock(struct sched_clock_data *data1,
@@ -238,26 +176,21 @@ u64 sched_clock_cpu(int cpu) · 37 unmodified lines
238176 now -= scd->tick_gtod;
239177
240178 __raw_spin_unlock(&my_scd->lock);
241−
242− __update_sched_clock(scd, now, &clock);
243−
244− __raw_spin_unlock(&scd->lock);
245−
246179 } else {
247180 __raw_spin_lock(&scd->lock);
248− __update_sched_clock(scd, now, NULL);
249− clock = scd->clock;
250− __raw_spin_unlock(&scd->lock);
251181 }
252182
183+ __update_sched_clock(scd, now);
184+ clock = scd->clock;
185+
186+ __raw_spin_unlock(&scd->lock);
187+
253188 return clock;
254189 }
255190
256191 void sched_clock_tick(void)
257192 {
258193 struct sched_clock_data *scd = this_scd();
259− unsigned long now_jiffies = jiffies;
260− s64 mult, delta_gtod, delta_raw;
261194 u64 now, now_gtod;
262195
263196 if (unlikely(!sched_clock_running))
@@ -269,29 +202,14 @@ void sched_clock_tick(void) · 5 unmodified lines
269202 now = sched_clock();
270203
271204 __raw_spin_lock(&scd->lock);
272− __update_sched_clock(scd, now, NULL);
205+ __update_sched_clock(scd, now);
273206 /*
274207 * update tick_gtod after __update_sched_clock() because that will
275208 * already observe 1 new jiffy; adding a new tick_gtod to that would
276209 * increase the clock 2 jiffies.
277210 */
278− delta_gtod = now_gtod - scd->tick_gtod;
279− delta_raw = now - scd->tick_raw;
280−
281− if ((long)delta_raw > 0) {
282− mult = delta_gtod << MULTI_SHIFT;
283− do_div(mult, delta_raw);
284− scd->multi = mult;
285− if (scd->multi > MAX_MULTI)
286− scd->multi = MAX_MULTI;
287− else if (scd->multi < MIN_MULTI)
288− scd->multi = MIN_MULTI;
289− } else
290− scd->multi = 1 << MULTI_SHIFT;
291−
292211 scd->tick_raw = now;
293212 scd->tick_gtod = now_gtod;
294− scd->tick_jiffies = now_jiffies;
295213 __raw_spin_unlock(&scd->lock);
296214 }
297215
@@ -321,7 +239,6 @@ void sched_clock_idle_wakeup_event(u64 delta_ns) · 23 unmodified lines
321239 __raw_spin_lock(&scd->lock);
322240 scd->prev_raw = now;
323241 scd->clock += delta_ns;
324− scd->multi = 1 << MULTI_SHIFT;
325242 __raw_spin_unlock(&scd->lock);
326243
327244 touch_softlockup_watchdog();
kernel/time/tick-sched.c+0−2
@@ -289,7 +289,6 @@ void tick_nohz_stop_sched_tick(int inidle) · 288 unmodified lines
289289 ts->tick_stopped = 1;
290290 ts->idle_jiffies = last_jiffies;
291291 rcu_enter_nohz();
292− sched_clock_tick_stop(cpu);
293292 }
294293
295294 /*
@@ -392,7 +391,6 @@ void tick_nohz_restart_sched_tick(void) · 96 unmodified lines
392391 select_nohz_load_balancer(0);
393392 now = ktime_get();
394393 tick_do_update_jiffies64(now);
395− sched_clock_tick_start(cpu);
396394 cpu_clear(cpu, nohz_cpu_mask);
397395
398396 /*