hrtimer_hres_enabled = 1;	else		return 0;	return 1;} __setup("highres=", setup_hrtimer_hres); /* * hrtimer_high_res_enabled - query, if the highres mode is enabled */static inline int hrtimer_is_hres_enabled(void){	return hrtimer_hres_enabled;} /* * Is the high resolution mode active ? */static inline int hrtimer_hres_active(void){	return __this_cpu_read(hrtimer_bases.hres_active);} /* * Reprogram the event source with checking both queues for the * next event * Called with interrupts disabled and base->lock held */static voidhrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal){	int i;	struct hrtimer_clock_base *base = cpu_base->clock_base;	ktime_t expires, expires_next; 	expires_next.tv64 = KTIME_MAX; 	for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) {		struct hrtimer *timer;		struct timerqueue_node *next; 		next = timerqueue_getnext(&base->active);		if (!next)			continue;		timer = container_of(next, struct hrtimer, node); 		expires = ktime_sub(hrtimer_get_expires(timer), base->offset);		/*		 * clock_was_set() has changed base->offset so the		 * result might be negative. Fix it up to prevent a		 * false positive in clockevents_program_event()		 */		if (expires.tv64 < 0)			expires.tv64 = 0;		if (expires.tv64 < expires_next.tv64)			expires_next = expires;	} 	if (skip_equal && expires_next.tv64 == cpu_base->expires_next.tv64)		return; 	cpu_base->expires_next.tv64 = expires_next.tv64; 	if (cpu_base->expires_next.tv64 != KTIME_MAX)		tick_program_event(cpu_base->expires_next, 1);} /* * Shared reprogramming for clock_realtime and clock_monotonic * * When a timer is enqueued and expires earlier than the already enqueued * timers, we have to check, whether it expires earlier than the timer for * which the clock event device was armed. * * Called with interrupts disabled and base->cpu_base.lock held */static int hrtimer_reprogram(struct hrtimer *timer,			     struct hrtimer_clock_base *base){	struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);	ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);	int res; 	WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0); 	/*	 * When the callback is running, we do not reprogram the clock event	 * device. The timer callback is either running on a different CPU or	 * the callback is executed in the hrtimer_interrupt context. The	 * reprogramming is handled either by the softirq, which called the	 * callback or at the end of the hrtimer_interrupt.	 */	if (hrtimer_callback_running(timer))		return 0; 	/*	 * CLOCK_REALTIME timer might be requested with an absolute	 * expiry time which is less than base->offset. Nothing wrong	 * about that, just avoid to call into the tick code, which	 * has now objections against negative expiry values.	 */	if (expires.tv64 < 0)		return -ETIME; 	if (expires.tv64 >= cpu_base->expires_next.tv64)		return 0; 	/*	 * If a hang was detected in the last timer interrupt then we	 * do not schedule a timer which is earlier than the expiry	 * which we enforced in the hang detection. We want the system	 * to make progress.	 */	if (cpu_base->hang_detected)		return 0; 	/*	 * Clockevents returns -ETIME, when the event was in the past.	 */	res = tick_program_event(expires, 0);	if (!IS_ERR_VALUE(res))		cpu_base->expires_next = expires;	return res;} /* * Initialize the high resolution related parts of cpu_base */static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base){	base->expires_next.tv64 = KTIME_MAX;	base->hres_active = 0;} /* * When High resolution timers are active, try to reprogram. Note, that in case * the state has HRTIMER_STATE_CALLBACK set, no reprogramming and no expiry * check happens. The timer gets enqueued into the rbtree. The reprogramming * and expiry check is done in the hrtimer_interrupt or in the softirq. */static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,					    struct hrtimer_clock_base *base,					    int wakeup){	if (base->cpu_base->hres_active && hrtimer_reprogram(timer, base)) {		if (wakeup) {			raw_spin_unlock(&base->cpu_base->lock);			raise_softirq_irqoff(HRTIMER_SOFTIRQ);			raw_spin_lock(&base->cpu_base->lock);		} else			__raise_softirq_irqoff(HRTIMER_SOFTIRQ); 		return 1;	} 	return 0;} /* * Retrigger next event is called after clock was set * * Called with interrupts disabled via on_each_cpu() */static void retrigger_next_event(void *arg){	struct hrtimer_cpu_base *base = &__get_cpu_var(hrtimer_bases);	struct timespec realtime_offset, xtim, wtm, sleep; 	if (!hrtimer_hres_active())		return; 	/* Optimized out for !HIGH_RES */	get_xtime_and_monotonic_and_sleep_offset(&xtim, &wtm, &sleep);	set_normalized_timespec(&realtime_offset, -wtm.tv_sec, -wtm.tv_nsec); 	/* Adjust CLOCK_REALTIME offset */	raw_spin_lock(&base->lock);	base->clock_base[HRTIMER_BASE_REALTIME].offset =		timespec_to_ktime(realtime_offset);	base->clock_base[HRTIMER_BASE_BOOTTIME].offset =		timespec_to_ktime(sleep); 	hrtimer_force_reprogram(base, 0);	raw_spin_unlock(&base->lock);} /* * Switch to high resolution mode */static int hrtimer_switch_to_hres(void){	int i, cpu = smp_processor_id();	struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu);	unsigned long flags; 	if (base->hres_active)		return 1; 	local_irq_save(flags); 	if (tick_init_highres()) {		local_irq_restore(flags);		printk(KERN_WARNING "Could not switch to high resolution "				    "mode on CPU %d\n", cpu);		return 0;	}	base->hres_active = 1;	for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++)		base->clock_base[i].resolution = KTIME_HIGH_RES; 	tick_setup_sched_timer(); 	/* "Retrigger" the interrupt to get things going */	retrigger_next_event(NULL);	local_irq_restore(flags);	return 1;} #else static inline int hrtimer_hres_active(void) { return 0; }static inline int hrtimer_is_hres_enabled(void) { return 0; }static inline int hrtimer_switch_to_hres(void) { return 0; }static inline voidhrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { }static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,					    struct hrtimer_clock_base *base,					    int wakeup){	return 0;}static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { }static inline void retrigger_next_event(void *arg) { } #endif /* CONFIG_HIGH_RES_TIMERS */ /* * Clock realtime was set * * Change the offset of the realtime clock vs. the monotonic * clock. * * We might have to reprogram the high resolution timer interrupt. On * SMP we call the architecture specific code to retrigger _all_ high * resolution timer interrupts. On UP we just disable interrupts and * call the high resolution interrupt code. */void clock_was_set(void){#ifdef CONFIG_HIGH_RES_TIMERS	/* Retrigger the CPU local events everywhere */	on_each_cpu(retrigger_next_event, NULL, 1);#endif	timerfd_clock_was_set();} /* * During resume we might have to reprogram the high resolution timer * interrupt (on the local CPU): */void hrtimers_resume(void){	WARN_ONCE(!irqs_disabled(),		  KERN_INFO "hrtimers_resume() called with IRQs enabled!"); 	retrigger_next_event(NULL);	timerfd_clock_was_set();} static inline void timer_stats_hrtimer_set_start_info(struct hrtimer *timer){#ifdef CONFIG_TIMER_STATS	if (timer->start_site)		return;	timer->start_site = __builtin_return_address(0);	memcpy(timer->start_comm, current->comm, TASK_COMM_LEN);	timer->start_pid = current->pid;#endif} static inline void timer_stats_hrtimer_clear_start_info(struct hrtimer *timer){#ifdef CONFIG_TIMER_STATS	timer->start_site = NULL;#endif} static inline void timer_stats_account_hrtimer(struct hrtimer *timer){#ifdef CONFIG_TIMER_STATS	if (likely(!timer_stats_active))		return;	timer_stats_update_stats(timer, timer->start_pid, timer->start_site,				 timer->function, timer->start_comm, 0);#endif} /* * Counterpart to lock_hrtimer_base above: */static inlinevoid unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags){	raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);} /** * hrtimer_forward - forward the timer expiry * @timer:	hrtimer to forward * @now:	forward past this time * @interval:	the interval to forward * * Forward the timer expiry so it will expire in the future. * Returns the number of overruns. */u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval){	u64 orun = 1;	ktime_t delta; 	delta = ktime_sub(now, hrtimer_get_expires(timer)); 	if (delta.tv64 < 0)		return 0; 	if (interval.tv64 < timer->base->resolution.tv64)		interval.tv64 = timer->base->resolution.tv64; 	if (unlikely(delta.tv64 >= interval.tv64)) {		s64 incr = ktime_to_ns(interval); 		orun = ktime_divns(delta, incr);		hrtimer_add_expires_ns(timer, incr * orun);		if (hrtimer_get_expires_tv64(timer) > now.tv64)			return orun;		/*		 * This (and the ktime_add() below) is the		 * correction for exact:		 */		orun++;	}	hrtimer_add_expires(timer, interval); 	return orun;}EXPORT_SYMBOL_GPL(hrtimer_forward); /* * enqueue_hrtimer - internal function to (re)start a timer * * The timer is inserted in expiry order. Insertion into the * red black tree is O(log(n)). Must hold the base lock. * * Returns 1 when the new timer is the leftmost timer in the tree. */static int enqueue_hrtimer(struct hrtimer *timer,			   struct hrtimer_clock_base *base){	debug_activate(timer); 	timerqueue_add(&base->active, &timer->node);	base->cpu_base->active_bases |= 1 << base->index; 	/*	 * HRTIMER_STATE_ENQUEUED is or'ed to the current state to preserve the	 * state of a possibly running callback.	 */	timer->state |= HRTIMER_STATE_ENQUEUED; 	return (&timer->node == base->active.next);} /* * __remove_hrtimer - internal function to remove a timer * * Caller must hold the base lock. * * High resolution timer mode reprograms the clock event device when the * timer is the one which expires next. The caller can disable this by setting * reprogram to zero. This is useful, when the context does a reprogramming * anyway (e.g. timer interrupt) */static void __remove_hrtimer(struct hrtimer *timer,			     struct hrtimer_clock_base *base,			     unsigned long newstate, int reprogram){	struct timerqueue_node *next_timer;	if (!(timer->state & HRTIMER_STATE_ENQUEUED))		goto out; 	next_timer = timerqueue_getnext(&base->active);	timerqueue_del(&base->active, &timer->node);	if (&timer->node == next_timer) {#ifdef CONFIG_HIGH_RES_TIMERS		/* Reprogram the clock event device. if enabled */		if (reprogram && hrtimer_hres_active()) {			ktime_t expires; 			expires = ktime_sub(hrtimer_get_expires(timer),					    base->offset);			if (base->cpu_base->expires_next.tv64 == expires.tv64)				hrtimer_force_reprogram(base->cpu_base, 1);		}#endif	}	if (!timerqueue_getnext(&base->active))		base->cpu_base->active_bases &= ~(1 << base->index);out:	timer->state = newstate;} /* * remove hrtimer, called with base lock held */static inline intremove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base){	if (hrtimer_is_queued(timer)) {		unsigned long state;		int reprogram; 		/*		 * Remove the timer and force reprogramming when high		 * resolution mode is active and the timer is on the current		 * CPU. If we remove a timer on another CPU, reprogramming is		 * skipped. The interrupt event on this CPU is fired and		 * reprogramming happens in the interrupt handler. This is a		 * rare case and less expensive than a smp call.		 */		debug_deactivate(timer);		timer_stats_hrtimer_clear_start_info(timer);		reprogram = base->cpu_base == &__get_cpu_var(hrtimer_bases);		/*		 * We must preserve the CALLBACK state flag here,		 * otherwise we could move the timer base in		 * switch_hrtimer_base.		 */		state = timer->state & HRTIMER_STATE_CALLBACK;		__remove_hrtimer(timer, base, state, reprogram);		return 1;	}	return 0;} int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,		unsigned long delta_ns, const enum hrtimer_mode mode,		int wakeup){	struct hrtimer_clock_base *base, *new_base;	unsigned long flags;	int ret, leftmost; 	base = lock_hrtimer_base(timer, &flags); 	/* Remove an active timer from the queue: */	ret = remove_hrtimer(timer, base); 	/* Switch the timer base, if necessary: */	new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED); 	if (mode & HRTIMER_MODE_REL) {		tim = ktime_add_safe(tim, new_base->get_time());		/*		 * CONFIG_TIME_LOW_RES is a temporary way for architectures		 * to signal that they simply return xtime in		 * do_gettimeoffset(). In this case we want to round up by		 * resolution when starting a relative timer, to avoid short		 * timeouts. This will go away with the GTOD framework.		 */#ifdef CONFIG_TIME_LOW_RES		tim = ktime_add_safe(tim, base->resolution);#endif	} 	hrtimer_set_expires_range_ns(timer, tim, delta_ns); 	timer_stats_hrtimer_set_start_info(timer); 	leftmost = enqueue_hrtimer(timer, new_base); 	/*	 * Only allow reprogramming if the new base is on this CPU.	 * (it might still be on another CPU if the timer was pending)	 *	 * XXX send_remote_softirq() ?	 */	if (leftmost && new_base->cpu_base == &__get_cpu_var(hrtimer_bases))		hrtimer_enqueue_reprogram(timer, new_base, wakeup); 	unlock_hrtimer_base(timer, &flags); 	return ret;} /** * hrtimer_start_range_ns - (re)start an hrtimer on the current CPU * @timer:	the timer to be added * @tim:	expiry time * @delta_ns:	"slack" range for the timer

Lines 501–1000 of 1861. Previous lines Load more