History

sched clock: revert various sched_clock() changes

Found an interactivity problem on a quad core test-system - simple CPU loops would occasionally delay the system un an unacceptable way. After much debugging with Peter Zijlstra it turned out that the problem is caused by the string of sched_clock() changes - they caused the CPU clock to jump backwards a bit - which confuses the scheduler arithmetics. (which is unsigned for performance reasons) So revert: # c300ba2: sched_clock: and multiplier for TSC to gtod drift # c0c8773: sched_clock: only update deltas with local reads. # af52a90: sched_clock: stop maximum check on NO HZ # f7cce27: sched_clock: widen the max and min time This solves the interactivity problems.

Ingo Molnar committed 18 years ago

Commit e4e4e531 parent 39675e83 files · +14 −114Browse files

Unified · Split

3 files
  1. include/linux/sched.h +1 −16
  2. kernel/sched_clock.c +13 −96
  3. kernel/time/tick-sched.c +0 −2
include/linux/sched.h+1−16
@@ -1572,28 +1572,13 @@ static inline void sched_clock_idle_sleep_event(void) · 1571 unmodified lines
1572static inline void sched_clock_idle_wakeup_event(u64 delta_ns)1572static inline void sched_clock_idle_wakeup_event(u64 delta_ns)
1573{1573{
1574}1574}
1575 1575#else
1576#ifdef CONFIG_NO_HZ
1577static inline void sched_clock_tick_stop(int cpu)
1578{
1579}
1580
1581static inline void sched_clock_tick_start(int cpu)
1582{
1583}
1584#endif
1585
1586#else /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
1587extern void sched_clock_init(void);1576extern void sched_clock_init(void);
1588extern u64 sched_clock_cpu(int cpu);1577extern u64 sched_clock_cpu(int cpu);
1589extern void sched_clock_tick(void);1578extern void sched_clock_tick(void);
1590extern void sched_clock_idle_sleep_event(void);1579extern void sched_clock_idle_sleep_event(void);
1591extern void sched_clock_idle_wakeup_event(u64 delta_ns);1580extern void sched_clock_idle_wakeup_event(u64 delta_ns);
1592#ifdef CONFIG_NO_HZ
1593extern void sched_clock_tick_stop(int cpu);
1594extern void sched_clock_tick_start(int cpu);
1595#endif1581#endif
1596#endif /* CONFIG_HAVE_UNSTABLE_SCHED_CLOCK */
1597 1582
1598/*1583/*
1599 * For kernel-internal use: high-speed (but slightly incorrect) per-cpu1584 * For kernel-internal use: high-speed (but slightly incorrect) per-cpu
kernel/sched_clock.c+13−96
@@ -44,11 +44,6 @@ unsigned long long __attribute__((weak)) sched_clock(void) · 43 unmodified lines
44 44
45#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK45#ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
46 46
47#define MULTI_SHIFT 15
48/* Max is double, Min is 1/2 */
49#define MAX_MULTI (2LL << MULTI_SHIFT)
50#define MIN_MULTI (1LL << (MULTI_SHIFT-1))
51
52struct sched_clock_data {47struct sched_clock_data {
53 /*48 /*
54 * Raw spinlock - this is a special case: this might be called49 * Raw spinlock - this is a special case: this might be called
@@ -62,10 +57,6 @@ struct sched_clock_data { · 7 unmodified lines
62 u64 tick_raw;57 u64 tick_raw;
63 u64 tick_gtod;58 u64 tick_gtod;
64 u64 clock;59 u64 clock;
65 s64 multi;
66#ifdef CONFIG_NO_HZ
67 int check_max;
68#endif
69};60};
70 61
71static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);62static DEFINE_PER_CPU_SHARED_ALIGNED(struct sched_clock_data, sched_clock_data);
@@ -97,53 +88,18 @@ void sched_clock_init(void) · 25 unmodified lines
97 scd->tick_raw = 0;88 scd->tick_raw = 0;
98 scd->tick_gtod = ktime_now;89 scd->tick_gtod = ktime_now;
99 scd->clock = ktime_now;90 scd->clock = ktime_now;
100 scd->multi = 1 << MULTI_SHIFT;
101#ifdef CONFIG_NO_HZ
102 scd->check_max = 1;
103#endif
104 }91 }
105 92
106 sched_clock_running = 1;93 sched_clock_running = 1;
107}94}
108 95
109#ifdef CONFIG_NO_HZ
110/*
111 * The dynamic ticks makes the delta jiffies inaccurate. This
112 * prevents us from checking the maximum time update.
113 * Disable the maximum check during stopped ticks.
114 */
115void sched_clock_tick_stop(int cpu)
116{
117 struct sched_clock_data *scd = cpu_sdc(cpu);
118
119 scd->check_max = 0;
120}
121
122void sched_clock_tick_start(int cpu)
123{
124 struct sched_clock_data *scd = cpu_sdc(cpu);
125
126 scd->check_max = 1;
127}
128
129static int check_max(struct sched_clock_data *scd)
130{
131 return scd->check_max;
132}
133#else
134static int check_max(struct sched_clock_data *scd)
135{
136 return 1;
137}
138#endif /* CONFIG_NO_HZ */
139
140/*96/*
141 * update the percpu scd from the raw @now value97 * update the percpu scd from the raw @now value
142 *98 *
143 * - filter out backward motion99 * - filter out backward motion
144 * - use jiffies to generate a min,max window to clip the raw values100 * - use jiffies to generate a min,max window to clip the raw values
145 */101 */
146static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *time)102static void __update_sched_clock(struct sched_clock_data *scd, u64 now)
147{103{
148 unsigned long now_jiffies = jiffies;104 unsigned long now_jiffies = jiffies;
149 long delta_jiffies = now_jiffies - scd->tick_jiffies;105 long delta_jiffies = now_jiffies - scd->tick_jiffies;
@@ -152,31 +108,16 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim · 2 unmodified lines
152 s64 delta = now - scd->prev_raw;108 s64 delta = now - scd->prev_raw;
153 109
154 WARN_ON_ONCE(!irqs_disabled());110 WARN_ON_ONCE(!irqs_disabled());
155 111 min_clock = scd->tick_gtod + delta_jiffies * TICK_NSEC;
156 /*
157 * At schedule tick the clock can be just under the gtod. We don't
158 * want to push it too prematurely.
159 */
160 min_clock = scd->tick_gtod + (delta_jiffies * TICK_NSEC);
161 if (min_clock > TICK_NSEC)
162 min_clock -= TICK_NSEC / 2;
163 112
164 if (unlikely(delta < 0)) {113 if (unlikely(delta < 0)) {
165 clock++;114 clock++;
166 goto out;115 goto out;
167 }116 }
168 117
169 /*118 max_clock = min_clock + TICK_NSEC;
170 * The clock must stay within a jiffie of the gtod.
171 * But since we may be at the start of a jiffy or the end of one
172 * we add another jiffy buffer.
173 */
174 max_clock = scd->tick_gtod + (2 + delta_jiffies) * TICK_NSEC;
175
176 delta *= scd->multi;
177 delta >>= MULTI_SHIFT;
178 119
179 if (unlikely(clock + delta > max_clock) && check_max(scd)) {120 if (unlikely(clock + delta > max_clock)) {
180 if (clock < max_clock)121 if (clock < max_clock)
181 clock = max_clock;122 clock = max_clock;
182 else123 else
@@ -189,12 +130,9 @@ static void __update_sched_clock(struct sched_clock_data *scd, u64 now, u64 *tim · 6 unmodified lines
189 if (unlikely(clock < min_clock))130 if (unlikely(clock < min_clock))
190 clock = min_clock;131 clock = min_clock;
191 132
192 if (time)133 scd->prev_raw = now;
193 *time = clock;134 scd->tick_jiffies = now_jiffies;
194 else {135 scd->clock = clock;
195 scd->prev_raw = now;
196 scd->clock = clock;
197 }
198}136}
199 137
200static void lock_double_clock(struct sched_clock_data *data1,138static void lock_double_clock(struct sched_clock_data *data1,
@@ -238,26 +176,21 @@ u64 sched_clock_cpu(int cpu) · 37 unmodified lines
238 now -= scd->tick_gtod;176 now -= scd->tick_gtod;
239 177
240 __raw_spin_unlock(&my_scd->lock);178 __raw_spin_unlock(&my_scd->lock);
241
242 __update_sched_clock(scd, now, &clock);
243
244 __raw_spin_unlock(&scd->lock);
245
246 } else {179 } else {
247 __raw_spin_lock(&scd->lock);180 __raw_spin_lock(&scd->lock);
248 __update_sched_clock(scd, now, NULL);
249 clock = scd->clock;
250 __raw_spin_unlock(&scd->lock);
251 }181 }
252 182
183 __update_sched_clock(scd, now);
184 clock = scd->clock;
185
186 __raw_spin_unlock(&scd->lock);
187
253 return clock;188 return clock;
254}189}
255 190
256void sched_clock_tick(void)191void sched_clock_tick(void)
257{192{
258 struct sched_clock_data *scd = this_scd();193 struct sched_clock_data *scd = this_scd();
259 unsigned long now_jiffies = jiffies;
260 s64 mult, delta_gtod, delta_raw;
261 u64 now, now_gtod;194 u64 now, now_gtod;
262 195
263 if (unlikely(!sched_clock_running))196 if (unlikely(!sched_clock_running))
@@ -269,29 +202,14 @@ void sched_clock_tick(void) · 5 unmodified lines
269 now = sched_clock();202 now = sched_clock();
270 203
271 __raw_spin_lock(&scd->lock);204 __raw_spin_lock(&scd->lock);
272 __update_sched_clock(scd, now, NULL);205 __update_sched_clock(scd, now);
273 /*206 /*
274 * update tick_gtod after __update_sched_clock() because that will207 * update tick_gtod after __update_sched_clock() because that will
275 * already observe 1 new jiffy; adding a new tick_gtod to that would208 * already observe 1 new jiffy; adding a new tick_gtod to that would
276 * increase the clock 2 jiffies.209 * increase the clock 2 jiffies.
277 */210 */
278 delta_gtod = now_gtod - scd->tick_gtod;
279 delta_raw = now - scd->tick_raw;
280
281 if ((long)delta_raw > 0) {
282 mult = delta_gtod << MULTI_SHIFT;
283 do_div(mult, delta_raw);
284 scd->multi = mult;
285 if (scd->multi > MAX_MULTI)
286 scd->multi = MAX_MULTI;
287 else if (scd->multi < MIN_MULTI)
288 scd->multi = MIN_MULTI;
289 } else
290 scd->multi = 1 << MULTI_SHIFT;
291
292 scd->tick_raw = now;211 scd->tick_raw = now;
293 scd->tick_gtod = now_gtod;212 scd->tick_gtod = now_gtod;
294 scd->tick_jiffies = now_jiffies;
295 __raw_spin_unlock(&scd->lock);213 __raw_spin_unlock(&scd->lock);
296}214}
297 215
@@ -321,7 +239,6 @@ void sched_clock_idle_wakeup_event(u64 delta_ns) · 23 unmodified lines
321 __raw_spin_lock(&scd->lock);239 __raw_spin_lock(&scd->lock);
322 scd->prev_raw = now;240 scd->prev_raw = now;
323 scd->clock += delta_ns;241 scd->clock += delta_ns;
324 scd->multi = 1 << MULTI_SHIFT;
325 __raw_spin_unlock(&scd->lock);242 __raw_spin_unlock(&scd->lock);
326 243
327 touch_softlockup_watchdog();244 touch_softlockup_watchdog();
kernel/time/tick-sched.c+0−2
@@ -289,7 +289,6 @@ void tick_nohz_stop_sched_tick(int inidle) · 288 unmodified lines
289 ts->tick_stopped = 1;289 ts->tick_stopped = 1;
290 ts->idle_jiffies = last_jiffies;290 ts->idle_jiffies = last_jiffies;
291 rcu_enter_nohz();291 rcu_enter_nohz();
292 sched_clock_tick_stop(cpu);
293 }292 }
294 293
295 /*294 /*
@@ -392,7 +391,6 @@ void tick_nohz_restart_sched_tick(void) · 96 unmodified lines
392 select_nohz_load_balancer(0);391 select_nohz_load_balancer(0);
393 now = ktime_get();392 now = ktime_get();
394 tick_do_update_jiffies64(now);393 tick_do_update_jiffies64(now);
395 sched_clock_tick_start(cpu);
396 cpu_clear(cpu, nohz_cpu_mask);394 cpu_clear(cpu, nohz_cpu_mask);
397 395
398 /*396 /*