2580 lines · 77.2 KB · C
RawDownload
	 */	if (time_before_eq(nextevt, basej + 1)) {		/* If we missed a tick already, force 0 delta */		if (time_before(nextevt, basej))			nextevt = basej;		tevt->local = basem + (u64)(nextevt - basej) * TICK_NSEC; 		/*		 * This is required for the remote check only but it doesn't		 * hurt, when it is done for both call sites:		 *		 * * The remote callers will only take care of the global timers		 *   as local timers will be handled by CPU itself. When not		 *   updating tevt->global with the already missed first global		 *   timer, it is possible that it will be missed completely.		 *		 * * The local callers will ignore the tevt->global anyway, when		 *   nextevt is max. one tick away.		 */		if (!local_first)			tevt->global = tevt->local;		return nextevt;	} 	/*	 * Update tevt.* values:	 *	 * If the local queue expires first, then the global event can be	 * ignored. If the global queue is empty, nothing to do either.	 */	if (!local_first && base_global->timers_pending)		tevt->global = basem + (u64)(nextevt_global - basej) * TICK_NSEC; 	if (base_local->timers_pending)		tevt->local = basem + (u64)(nextevt_local - basej) * TICK_NSEC; 	return nextevt;} # ifdef CONFIG_SMP/** * fetch_next_timer_interrupt_remote() - Store next timers into @tevt * @basej:	base time jiffies * @basem:	base time clock monotonic * @tevt:	Pointer to the storage for the expiry values * @cpu:	Remote CPU * * Stores the next pending local and global timer expiry values in the * struct pointed to by @tevt. If a queue is empty the corresponding * field is set to KTIME_MAX. If local event expires before global * event, global event is set to KTIME_MAX as well. * * Caller needs to make sure timer base locks are held (use * timer_lock_remote_bases() for this purpose). */void fetch_next_timer_interrupt_remote(unsigned long basej, u64 basem,				       struct timer_events *tevt,				       unsigned int cpu){	struct timer_base *base_local, *base_global; 	/* Preset local / global events */	tevt->local = tevt->global = KTIME_MAX; 	base_local = per_cpu_ptr(&timer_bases[BASE_LOCAL], cpu);	base_global = per_cpu_ptr(&timer_bases[BASE_GLOBAL], cpu); 	lockdep_assert_held(&base_local->lock);	lockdep_assert_held(&base_global->lock); 	fetch_next_timer_interrupt(basej, basem, base_local, base_global, tevt);} /** * timer_unlock_remote_bases - unlock timer bases of cpu * @cpu:	Remote CPU * * Unlocks the remote timer bases. */void timer_unlock_remote_bases(unsigned int cpu)	__releases(timer_bases[BASE_LOCAL]->lock)	__releases(timer_bases[BASE_GLOBAL]->lock){	struct timer_base *base_local, *base_global; 	base_local = per_cpu_ptr(&timer_bases[BASE_LOCAL], cpu);	base_global = per_cpu_ptr(&timer_bases[BASE_GLOBAL], cpu); 	raw_spin_unlock(&base_global->lock);	raw_spin_unlock(&base_local->lock);} /** * timer_lock_remote_bases - lock timer bases of cpu * @cpu:	Remote CPU * * Locks the remote timer bases. */void timer_lock_remote_bases(unsigned int cpu)	__acquires(timer_bases[BASE_LOCAL]->lock)	__acquires(timer_bases[BASE_GLOBAL]->lock){	struct timer_base *base_local, *base_global; 	base_local = per_cpu_ptr(&timer_bases[BASE_LOCAL], cpu);	base_global = per_cpu_ptr(&timer_bases[BASE_GLOBAL], cpu); 	lockdep_assert_irqs_disabled(); 	raw_spin_lock(&base_local->lock);	raw_spin_lock_nested(&base_global->lock, SINGLE_DEPTH_NESTING);} /** * timer_base_is_idle() - Return whether timer base is set idle * * Returns value of local timer base is_idle value. */bool timer_base_is_idle(void){	return __this_cpu_read(timer_bases[BASE_LOCAL].is_idle);} static void __run_timer_base(struct timer_base *base); /** * timer_expire_remote() - expire global timers of cpu * @cpu:	Remote CPU * * Expire timers of global base of remote CPU. */void timer_expire_remote(unsigned int cpu){	struct timer_base *base = per_cpu_ptr(&timer_bases[BASE_GLOBAL], cpu); 	__run_timer_base(base);} static void timer_use_tmigr(unsigned long basej, u64 basem,			    unsigned long *nextevt, bool *tick_stop_path,			    bool timer_base_idle, struct timer_events *tevt){	u64 next_tmigr; 	if (timer_base_idle)		next_tmigr = tmigr_cpu_new_timer(tevt->global);	else if (tick_stop_path)		next_tmigr = tmigr_cpu_deactivate(tevt->global);	else		next_tmigr = tmigr_quick_check(tevt->global); 	/*	 * If the CPU is the last going idle in timer migration hierarchy, make	 * sure the CPU will wake up in time to handle remote timers.	 * next_tmigr == KTIME_MAX if other CPUs are still active.	 */	if (next_tmigr < tevt->local) {		u64 tmp; 		/* If we missed a tick already, force 0 delta */		if (next_tmigr < basem)			next_tmigr = basem; 		tmp = div_u64(next_tmigr - basem, TICK_NSEC); 		*nextevt = basej + (unsigned long)tmp;		tevt->local = next_tmigr;	}}# elsestatic void timer_use_tmigr(unsigned long basej, u64 basem,			    unsigned long *nextevt, bool *tick_stop_path,			    bool timer_base_idle, struct timer_events *tevt){	/*	 * Make sure first event is written into tevt->local to not miss a	 * timer on !SMP systems.	 */	tevt->local = min_t(u64, tevt->local, tevt->global);}# endif /* CONFIG_SMP */ static inline u64 __get_next_timer_interrupt(unsigned long basej, u64 basem,					     bool *idle){	struct timer_events tevt = { .local = KTIME_MAX, .global = KTIME_MAX };	struct timer_base *base_local, *base_global;	unsigned long nextevt;	bool idle_is_possible; 	/*	 * When the CPU is offline, the tick is cancelled and nothing is supposed	 * to try to stop it.	 */	if (WARN_ON_ONCE(cpu_is_offline(smp_processor_id()))) {		if (idle)			*idle = true;		return tevt.local;	} 	base_local = this_cpu_ptr(&timer_bases[BASE_LOCAL]);	base_global = this_cpu_ptr(&timer_bases[BASE_GLOBAL]); 	raw_spin_lock(&base_local->lock);	raw_spin_lock_nested(&base_global->lock, SINGLE_DEPTH_NESTING); 	nextevt = fetch_next_timer_interrupt(basej, basem, base_local,					     base_global, &tevt); 	/*	 * If the next event is only one jiffy ahead there is no need to call	 * timer migration hierarchy related functions. The value for the next	 * global timer in @tevt struct equals then KTIME_MAX. This is also	 * true, when the timer base is idle.	 *	 * The proper timer migration hierarchy function depends on the callsite	 * and whether timer base is idle or not. @nextevt will be updated when	 * this CPU needs to handle the first timer migration hierarchy	 * event. See timer_use_tmigr() for detailed information.	 */	idle_is_possible = time_after(nextevt, basej + 1);	if (idle_is_possible)		timer_use_tmigr(basej, basem, &nextevt, idle,				base_local->is_idle, &tevt); 	/*	 * We have a fresh next event. Check whether we can forward the	 * base.	 */	__forward_timer_base(base_local, basej);	__forward_timer_base(base_global, basej); 	/*	 * Set base->is_idle only when caller is timer_base_try_to_set_idle()	 */	if (idle) {		/*		 * Bases are idle if the next event is more than a tick		 * away. Caution: @nextevt could have changed by enqueueing a		 * global timer into timer migration hierarchy. Therefore a new		 * check is required here.		 *		 * If the base is marked idle then any timer add operation must		 * forward the base clk itself to keep granularity small. This		 * idle logic is only maintained for the BASE_LOCAL and		 * BASE_GLOBAL base, deferrable timers may still see large		 * granularity skew (by design).		 */		if (!base_local->is_idle && time_after(nextevt, basej + 1)) {			base_local->is_idle = true;			/*			 * Global timers queued locally while running in a task			 * in nohz_full mode need a self-IPI to kick reprogramming			 * in IRQ tail.			 */			if (tick_nohz_full_cpu(base_local->cpu))				base_global->is_idle = true;			trace_timer_base_idle(true, base_local->cpu);		}		*idle = base_local->is_idle; 		/*		 * When timer base is not set idle, undo the effect of		 * tmigr_cpu_deactivate() to prevent inconsistent states - active		 * timer base but inactive timer migration hierarchy.		 *		 * When timer base was already marked idle, nothing will be		 * changed here.		 */		if (!base_local->is_idle && idle_is_possible)			tmigr_cpu_activate();	} 	raw_spin_unlock(&base_global->lock);	raw_spin_unlock(&base_local->lock); 	return cmp_next_hrtimer_event(basem, tevt.local);} /** * get_next_timer_interrupt() - return the time (clock mono) of the next timer * @basej:	base time jiffies * @basem:	base time clock monotonic * * Returns the tick aligned clock monotonic time of the next pending timer or * KTIME_MAX if no timer is pending. If timer of global base was queued into * timer migration hierarchy, first global timer is not taken into account. If * it was the last CPU of timer migration hierarchy going idle, first global * event is taken into account. */u64 get_next_timer_interrupt(unsigned long basej, u64 basem){	return __get_next_timer_interrupt(basej, basem, NULL);} /** * timer_base_try_to_set_idle() - Try to set the idle state of the timer bases * @basej:	base time jiffies * @basem:	base time clock monotonic * @idle:	pointer to store the value of timer_base->is_idle on return; *		*idle contains the information whether tick was already stopped * * Returns the tick aligned clock monotonic time of the next pending timer or * KTIME_MAX if no timer is pending. When tick was already stopped KTIME_MAX is * returned as well. */u64 timer_base_try_to_set_idle(unsigned long basej, u64 basem, bool *idle){	if (*idle)		return KTIME_MAX; 	return __get_next_timer_interrupt(basej, basem, idle);} /** * timer_clear_idle - Clear the idle state of the timer base * * Called with interrupts disabled */void timer_clear_idle(void){	int this_cpu = smp_processor_id();	/*	 * We do this unlocked. The worst outcome is a remote pinned timer	 * enqueue sending a pointless IPI, but taking the lock would just	 * make the window for sending the IPI a few instructions smaller	 * for the cost of taking the lock in the exit from idle	 * path. Required for BASE_LOCAL only.	 */	__this_cpu_write(timer_bases[BASE_LOCAL].is_idle, false);	if (tick_nohz_full_cpu(this_cpu))		__this_cpu_write(timer_bases[BASE_GLOBAL].is_idle, false);	trace_timer_base_idle(false, this_cpu); 	/* Activate without holding the timer_base->lock */	tmigr_cpu_activate();}#endif /** * __run_timers - run all expired timers (if any) on this CPU. * @base: the timer vector to be processed. */static inline void __run_timers(struct timer_base *base){	struct hlist_head heads[LVL_DEPTH];	int levels; 	lockdep_assert_held(&base->lock); 	if (base->running_timer)		return; 	while (time_after_eq(jiffies, base->clk) &&	       time_after_eq(jiffies, base->next_expiry)) {		levels = collect_expired_timers(base, heads);		/*		 * The two possible reasons for not finding any expired		 * timer at this clk are that all matching timers have been		 * dequeued or no timer has been queued since		 * base::next_expiry was set to base::clk +		 * TIMER_NEXT_MAX_DELTA.		 */		WARN_ON_ONCE(!levels && !base->next_expiry_recalc			     && base->timers_pending);		/*		 * While executing timers, base->clk is set 1 offset ahead of		 * jiffies to avoid endless requeuing to current jiffies.		 */		base->clk++;		timer_recalc_next_expiry(base); 		while (levels--)			expire_timers(base, heads + levels);	}} static void __run_timer_base(struct timer_base *base){	/* Can race against a remote CPU updating next_expiry under the lock */	if (time_before(jiffies, READ_ONCE(base->next_expiry)))		return; 	timer_base_lock_expiry(base);	raw_spin_lock_irq(&base->lock);	__run_timers(base);	raw_spin_unlock_irq(&base->lock);	timer_base_unlock_expiry(base);} static void run_timer_base(int index){	struct timer_base *base = this_cpu_ptr(&timer_bases[index]); 	__run_timer_base(base);} /* * This function runs timers and the timer-tq in bottom half context. */static __latent_entropy void run_timer_softirq(void){	run_timer_base(BASE_LOCAL);	if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) {		run_timer_base(BASE_GLOBAL);		run_timer_base(BASE_DEF); 		if (is_timers_nohz_active())			tmigr_handle_remote();	}} /* * Called by the local, per-CPU timer interrupt on SMP. */static void run_local_timers(void){	struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_LOCAL]); 	hrtimer_run_queues(); 	for (int i = 0; i < NR_BASES; i++, base++) {		/*		 * Raise the softirq only if required.		 *		 * timer_base::next_expiry can be written by a remote CPU while		 * holding the lock. If this write happens at the same time than		 * the lockless local read, sanity checker could complain about		 * data corruption.		 *		 * There are two possible situations where		 * timer_base::next_expiry is written by a remote CPU:		 *		 * 1. Remote CPU expires global timers of this CPU and updates		 * timer_base::next_expiry of BASE_GLOBAL afterwards in		 * next_timer_interrupt() or timer_recalc_next_expiry(). The		 * worst outcome is a superfluous raise of the timer softirq		 * when the not yet updated value is read.		 *		 * 2. A new first pinned timer is enqueued by a remote CPU		 * and therefore timer_base::next_expiry of BASE_LOCAL is		 * updated. When this update is missed, this isn't a		 * problem, as an IPI is executed nevertheless when the CPU		 * was idle before. When the CPU wasn't idle but the update		 * is missed, then the timer would expire one jiffy late -		 * bad luck.		 *		 * Those unlikely corner cases where the worst outcome is only a		 * one jiffy delay or a superfluous raise of the softirq are		 * not that expensive as doing the check always while holding		 * the lock.		 *		 * Possible remote writers are using WRITE_ONCE(). Local reader		 * uses therefore READ_ONCE().		 */		if (time_after_eq(jiffies, READ_ONCE(base->next_expiry)) ||		    (i == BASE_DEF && tmigr_requires_handle_remote())) {			raise_timer_softirq(TIMER_SOFTIRQ);			return;		}	}} /* * Called from the timer interrupt handler to charge one tick to the current * process.  user_tick is 1 if the tick is user time, 0 for system. */void update_process_times(int user_tick){	struct task_struct *p = current; 	/* Note: this timer irq context must be accounted for as well. */	account_process_tick(p, user_tick);	run_local_timers();	rcu_sched_clock_irq(user_tick);#ifdef CONFIG_IRQ_WORK	if (in_hardirq())		irq_work_tick();#endif	sched_tick();	if (IS_ENABLED(CONFIG_POSIX_TIMERS))		run_posix_cpu_timers();} #ifdef CONFIG_HOTPLUG_CPUstatic void migrate_timer_list(struct timer_base *new_base, struct hlist_head *head){	struct timer_list *timer;	int cpu = new_base->cpu; 	while (!hlist_empty(head)) {		timer = hlist_entry(head->first, struct timer_list, entry);		detach_timer(timer, false);		timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu;		internal_add_timer(new_base, timer);	}} int timers_prepare_cpu(unsigned int cpu){

Lines 2001–2500 of 2580. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t