931 lines · 23.3 KB · C
RawDownload
/* *  linux/kernel/time/tick-sched.c * *  Copyright(C) 2005-2006, Thomas Gleixner <tglx@linutronix.de> *  Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar *  Copyright(C) 2006-2007  Timesys Corp., Thomas Gleixner * *  No idle tick implementation for low and high resolution timers * *  Started by: Thomas Gleixner and Ingo Molnar * *  Distribute under GPLv2. */#include <linux/cpu.h>#include <linux/err.h>#include <linux/hrtimer.h>#include <linux/interrupt.h>#include <linux/kernel_stat.h>#include <linux/percpu.h>#include <linux/profile.h>#include <linux/sched.h>#include <linux/module.h> #include <asm/irq_regs.h> #include "tick-internal.h" /* * Per cpu nohz control structure */static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched); /* * The time, when the last jiffy update happened. Protected by xtime_lock. */static ktime_t last_jiffies_update; struct tick_sched *tick_get_tick_sched(int cpu){	return &per_cpu(tick_cpu_sched, cpu);} /* * Must be called with interrupts disabled ! */static void tick_do_update_jiffies64(ktime_t now){	unsigned long ticks = 0;	ktime_t delta; 	/*	 * Do a quick check without holding xtime_lock:	 */	delta = ktime_sub(now, last_jiffies_update);	if (delta.tv64 < tick_period.tv64)		return; 	/* Reevalute with xtime_lock held */	write_seqlock(&xtime_lock); 	delta = ktime_sub(now, last_jiffies_update);	if (delta.tv64 >= tick_period.tv64) { 		delta = ktime_sub(delta, tick_period);		last_jiffies_update = ktime_add(last_jiffies_update,						tick_period); 		/* Slow path for long timeouts */		if (unlikely(delta.tv64 >= tick_period.tv64)) {			s64 incr = ktime_to_ns(tick_period); 			ticks = ktime_divns(delta, incr); 			last_jiffies_update = ktime_add_ns(last_jiffies_update,							   incr * ticks);		}		do_timer(++ticks); 		/* Keep the tick_next_period variable up to date */		tick_next_period = ktime_add(last_jiffies_update, tick_period);	}	write_sequnlock(&xtime_lock);} /* * Initialize and return retrieve the jiffies update. */static ktime_t tick_init_jiffy_update(void){	ktime_t period; 	write_seqlock(&xtime_lock);	/* Did we start the jiffies update yet ? */	if (last_jiffies_update.tv64 == 0)		last_jiffies_update = tick_next_period;	period = last_jiffies_update;	write_sequnlock(&xtime_lock);	return period;} /* * NOHZ - aka dynamic tick functionality */#ifdef CONFIG_NO_HZ/* * NO HZ enabled ? */static int tick_nohz_enabled __read_mostly  = 1; /* * Enable / Disable tickless mode */static int __init setup_tick_nohz(char *str){	if (!strcmp(str, "off"))		tick_nohz_enabled = 0;	else if (!strcmp(str, "on"))		tick_nohz_enabled = 1;	else		return 0;	return 1;} __setup("nohz=", setup_tick_nohz); /** * tick_nohz_update_jiffies - update jiffies when idle was interrupted * * Called from interrupt entry when the CPU was idle * * In case the sched_tick was stopped on this CPU, we have to check if jiffies * must be updated. Otherwise an interrupt handler could use a stale jiffy * value. We do this unconditionally on any cpu, as we don't know whether the * cpu, which has the update task assigned is in a long sleep. */static void tick_nohz_update_jiffies(ktime_t now){	int cpu = smp_processor_id();	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);	unsigned long flags; 	ts->idle_waketime = now; 	local_irq_save(flags);	tick_do_update_jiffies64(now);	local_irq_restore(flags); 	touch_softlockup_watchdog();} /* * Updates the per cpu time idle statistics counters */static voidupdate_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time){	ktime_t delta; 	if (ts->idle_active) {		delta = ktime_sub(now, ts->idle_entrytime);		if (nr_iowait_cpu(cpu) > 0)			ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);		else			ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);		ts->idle_entrytime = now;	} 	if (last_update_time)		*last_update_time = ktime_to_us(now); } static void tick_nohz_stop_idle(int cpu, ktime_t now){	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu); 	update_ts_time_stats(cpu, ts, now, NULL);	ts->idle_active = 0; 	sched_clock_idle_wakeup_event(0);} static ktime_t tick_nohz_start_idle(int cpu, struct tick_sched *ts){	ktime_t now = ktime_get(); 	ts->idle_entrytime = now;	ts->idle_active = 1;	sched_clock_idle_sleep_event();	return now;} /** * get_cpu_idle_time_us - get the total idle time of a cpu * @cpu: CPU number to query * @last_update_time: variable to store update time in. Do not update * counters if NULL. * * Return the cummulative idle time (since boot) for a given * CPU, in microseconds. * * This time is measured via accounting rather than sampling, * and is as accurate as ktime_get() is. * * This function returns -1 if NOHZ is not enabled. */u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time){	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);	ktime_t now, idle; 	if (!tick_nohz_enabled)		return -1; 	now = ktime_get();	if (last_update_time) {		update_ts_time_stats(cpu, ts, now, last_update_time);		idle = ts->idle_sleeptime;	} else {		if (ts->idle_active && !nr_iowait_cpu(cpu)) {			ktime_t delta = ktime_sub(now, ts->idle_entrytime); 			idle = ktime_add(ts->idle_sleeptime, delta);		} else {			idle = ts->idle_sleeptime;		}	} 	return ktime_to_us(idle); }EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); /** * get_cpu_iowait_time_us - get the total iowait time of a cpu * @cpu: CPU number to query * @last_update_time: variable to store update time in. Do not update * counters if NULL. * * Return the cummulative iowait time (since boot) for a given * CPU, in microseconds. * * This time is measured via accounting rather than sampling, * and is as accurate as ktime_get() is. * * This function returns -1 if NOHZ is not enabled. */u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time){	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);	ktime_t now, iowait; 	if (!tick_nohz_enabled)		return -1; 	now = ktime_get();	if (last_update_time) {		update_ts_time_stats(cpu, ts, now, last_update_time);		iowait = ts->iowait_sleeptime;	} else {		if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {			ktime_t delta = ktime_sub(now, ts->idle_entrytime); 			iowait = ktime_add(ts->iowait_sleeptime, delta);		} else {			iowait = ts->iowait_sleeptime;		}	} 	return ktime_to_us(iowait);}EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); static void tick_nohz_stop_sched_tick(struct tick_sched *ts){	unsigned long seq, last_jiffies, next_jiffies, delta_jiffies;	ktime_t last_update, expires, now;	struct clock_event_device *dev = __get_cpu_var(tick_cpu_device).evtdev;	u64 time_delta;	int cpu; 	cpu = smp_processor_id();	ts = &per_cpu(tick_cpu_sched, cpu); 	now = tick_nohz_start_idle(cpu, ts); 	/*	 * If this cpu is offline and it is the one which updates	 * jiffies, then give up the assignment and let it be taken by	 * the cpu which runs the tick timer next. If we don't drop	 * this here the jiffies might be stale and do_timer() never	 * invoked.	 */	if (unlikely(!cpu_online(cpu))) {		if (cpu == tick_do_timer_cpu)			tick_do_timer_cpu = TICK_DO_TIMER_NONE;	} 	if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))		return; 	if (need_resched())		return; 	if (unlikely(local_softirq_pending() && cpu_online(cpu))) {		static int ratelimit; 		if (ratelimit < 10) {			printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n",			       (unsigned int) local_softirq_pending());			ratelimit++;		}		return;	} 	ts->idle_calls++;	/* Read jiffies and the time when jiffies were updated last */	do {		seq = read_seqbegin(&xtime_lock);		last_update = last_jiffies_update;		last_jiffies = jiffies;		time_delta = timekeeping_max_deferment();	} while (read_seqretry(&xtime_lock, seq)); 	if (rcu_needs_cpu(cpu) || printk_needs_cpu(cpu) ||	    arch_needs_cpu(cpu)) {		next_jiffies = last_jiffies + 1;		delta_jiffies = 1;	} else {		/* Get the next timer wheel timer */		next_jiffies = get_next_timer_interrupt(last_jiffies);		delta_jiffies = next_jiffies - last_jiffies;	}	/*	 * Do not stop the tick, if we are only one off	 * or if the cpu is required for rcu	 */	if (!ts->tick_stopped && delta_jiffies == 1)		goto out; 	/* Schedule the tick, if we are at least one jiffie off */	if ((long)delta_jiffies >= 1) { 		/*		 * If this cpu is the one which updates jiffies, then		 * give up the assignment and let it be taken by the		 * cpu which runs the tick timer next, which might be		 * this cpu as well. If we don't drop this here the		 * jiffies might be stale and do_timer() never		 * invoked. Keep track of the fact that it was the one		 * which had the do_timer() duty last. If this cpu is		 * the one which had the do_timer() duty last, we		 * limit the sleep time to the timekeeping		 * max_deferement value which we retrieved		 * above. Otherwise we can sleep as long as we want.		 */		if (cpu == tick_do_timer_cpu) {			tick_do_timer_cpu = TICK_DO_TIMER_NONE;			ts->do_timer_last = 1;		} else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {			time_delta = KTIME_MAX;			ts->do_timer_last = 0;		} else if (!ts->do_timer_last) {			time_delta = KTIME_MAX;		} 		/*		 * calculate the expiry time for the next timer wheel		 * timer. delta_jiffies >= NEXT_TIMER_MAX_DELTA signals		 * that there is no timer pending or at least extremely		 * far into the future (12 days for HZ=1000). In this		 * case we set the expiry to the end of time.		 */		if (likely(delta_jiffies < NEXT_TIMER_MAX_DELTA)) {			/*			 * Calculate the time delta for the next timer event.			 * If the time delta exceeds the maximum time delta			 * permitted by the current clocksource then adjust			 * the time delta accordingly to ensure the			 * clocksource does not wrap.			 */			time_delta = min_t(u64, time_delta,					   tick_period.tv64 * delta_jiffies);		} 		if (time_delta < KTIME_MAX)			expires = ktime_add_ns(last_update, time_delta);		else			expires.tv64 = KTIME_MAX; 		/* Skip reprogram of event if its not changed */		if (ts->tick_stopped && ktime_equal(expires, dev->next_event))			goto out; 		/*		 * nohz_stop_sched_tick can be called several times before		 * the nohz_restart_sched_tick is called. This happens when		 * interrupts arrive which do not cause a reschedule. In the		 * first call we save the current tick time, so we can restart		 * the scheduler tick in nohz_restart_sched_tick.		 */		if (!ts->tick_stopped) {			select_nohz_load_balancer(1); 			ts->idle_tick = hrtimer_get_expires(&ts->sched_timer);			ts->tick_stopped = 1;			ts->idle_jiffies = last_jiffies;		} 		ts->idle_sleeps++; 		/* Mark expires */		ts->idle_expires = expires; 		/*		 * If the expiration time == KTIME_MAX, then		 * in this case we simply stop the tick timer.		 */		 if (unlikely(expires.tv64 == KTIME_MAX)) {			if (ts->nohz_mode == NOHZ_MODE_HIGHRES)				hrtimer_cancel(&ts->sched_timer);			goto out;		} 		if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {			hrtimer_start(&ts->sched_timer, expires,				      HRTIMER_MODE_ABS_PINNED);			/* Check, if the timer was already in the past */			if (hrtimer_active(&ts->sched_timer))				goto out;		} else if (!tick_program_event(expires, 0))				goto out;		/*		 * We are past the event already. So we crossed a		 * jiffie boundary. Update jiffies and raise the		 * softirq.		 */		tick_do_update_jiffies64(ktime_get());	}	raise_softirq_irqoff(TIMER_SOFTIRQ);out:	ts->next_jiffies = next_jiffies;	ts->last_jiffies = last_jiffies;	ts->sleep_length = ktime_sub(dev->next_event, now);} /** * tick_nohz_idle_enter - stop the idle tick from the idle task * * When the next event is more than a tick into the future, stop the idle tick * Called when we start the idle loop. * * The arch is responsible of calling: * * - rcu_idle_enter() after its last use of RCU before the CPU is put *  to sleep. * - rcu_idle_exit() before the first use of RCU after the CPU is woken up. */void tick_nohz_idle_enter(void){	struct tick_sched *ts; 	WARN_ON_ONCE(irqs_disabled()); 	/* 	 * Update the idle state in the scheduler domain hierarchy 	 * when tick_nohz_stop_sched_tick() is called from the idle loop. 	 * State will be updated to busy during the first busy tick after 	 * exiting idle. 	 */	set_cpu_sd_state_idle(); 	local_irq_disable(); 	ts = &__get_cpu_var(tick_cpu_sched);	/*	 * set ts->inidle unconditionally. even if the system did not	 * switch to nohz mode the cpu frequency governers rely on the	 * update of the idle time accounting in tick_nohz_start_idle().	 */	ts->inidle = 1;	tick_nohz_stop_sched_tick(ts); 	local_irq_enable();} /** * tick_nohz_irq_exit - update next tick event from interrupt exit * * When an interrupt fires while we are idle and it doesn't cause * a reschedule, it may still add, modify or delete a timer, enqueue * an RCU callback, etc... * So we need to re-calculate and reprogram the next tick event. */void tick_nohz_irq_exit(void){	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched); 	if (!ts->inidle)		return;

Lines 1–500 of 931.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t