2580 lines · 77.2 KB · C
RawDownload
{	spin_lock(&base->expiry_lock);} static inline void timer_base_unlock_expiry(struct timer_base *base){	spin_unlock(&base->expiry_lock);} /* * The counterpart to del_timer_wait_running(). * * If there is a waiter for base->expiry_lock, then it was waiting for the * timer callback to finish. Drop expiry_lock and reacquire it. That allows * the waiter to acquire the lock and make progress. */static void timer_sync_wait_running(struct timer_base *base)	__releases(&base->lock) __releases(&base->expiry_lock)	__acquires(&base->expiry_lock) __acquires(&base->lock){	if (atomic_read(&base->timer_waiters)) {		raw_spin_unlock_irq(&base->lock);		spin_unlock(&base->expiry_lock);		spin_lock(&base->expiry_lock);		raw_spin_lock_irq(&base->lock);	}} /* * This function is called on PREEMPT_RT kernels when the fast path * deletion of a timer failed because the timer callback function was * running. * * This prevents priority inversion, if the softirq thread on a remote CPU * got preempted, and it prevents a life lock when the task which tries to * delete a timer preempted the softirq thread running the timer callback * function. */static void del_timer_wait_running(struct timer_list *timer){	u32 tf; 	tf = READ_ONCE(timer->flags);	if (!(tf & (TIMER_MIGRATING | TIMER_IRQSAFE))) {		struct timer_base *base = get_timer_base(tf); 		/*		 * Mark the base as contended and grab the expiry lock,		 * which is held by the softirq across the timer		 * callback. Drop the lock immediately so the softirq can		 * expire the next timer. In theory the timer could already		 * be running again, but that's more than unlikely and just		 * causes another wait loop.		 */		atomic_inc(&base->timer_waiters);		spin_lock_bh(&base->expiry_lock);		atomic_dec(&base->timer_waiters);		spin_unlock_bh(&base->expiry_lock);	}}#elsestatic inline void timer_base_init_expiry_lock(struct timer_base *base) { }static inline void timer_base_lock_expiry(struct timer_base *base) { }static inline void timer_base_unlock_expiry(struct timer_base *base) { }static inline void timer_sync_wait_running(struct timer_base *base) { }static inline void del_timer_wait_running(struct timer_list *timer) { }#endif /** * __timer_delete_sync - Internal function: Deactivate a timer and wait *			 for the handler to finish. * @timer:	The timer to be deactivated * @shutdown:	If true, @timer->function will be set to NULL under the *		timer base lock which prevents rearming of @timer * * If @shutdown is not set the timer can be rearmed later. If the timer can * be rearmed concurrently, i.e. after dropping the base lock then the * return value is meaningless. * * If @shutdown is set then @timer->function is set to NULL under timer * base lock which prevents rearming of the timer. Any attempt to rearm * a shutdown timer is silently ignored. * * If the timer should be reused after shutdown it has to be initialized * again. * * Return: * * %0	- The timer was not pending * * %1	- The timer was pending and deactivated */static int __timer_delete_sync(struct timer_list *timer, bool shutdown){	int ret; #ifdef CONFIG_LOCKDEP	unsigned long flags; 	/*	 * If lockdep gives a backtrace here, please reference	 * the synchronization rules above.	 */	local_irq_save(flags);	lock_map_acquire(&timer->lockdep_map);	lock_map_release(&timer->lockdep_map);	local_irq_restore(flags);#endif	/*	 * don't use it in hardirq context, because it	 * could lead to deadlock.	 */	WARN_ON(in_hardirq() && !(timer->flags & TIMER_IRQSAFE)); 	/*	 * Must be able to sleep on PREEMPT_RT because of the slowpath in	 * del_timer_wait_running().	 */	if (IS_ENABLED(CONFIG_PREEMPT_RT) && !(timer->flags & TIMER_IRQSAFE))		lockdep_assert_preemption_enabled(); 	do {		ret = __try_to_del_timer_sync(timer, shutdown); 		if (unlikely(ret < 0)) {			del_timer_wait_running(timer);			cpu_relax();		}	} while (ret < 0); 	return ret;} /** * timer_delete_sync - Deactivate a timer and wait for the handler to finish. * @timer:	The timer to be deactivated * * Synchronization rules: Callers must prevent restarting of the timer, * otherwise this function is meaningless. It must not be called from * interrupt contexts unless the timer is an irqsafe one. The caller must * not hold locks which would prevent completion of the timer's callback * function. The timer's handler must not call add_timer_on(). Upon exit * the timer is not queued and the handler is not running on any CPU. * * For !irqsafe timers, the caller must not hold locks that are held in * interrupt context. Even if the lock has nothing to do with the timer in * question.  Here's why:: * *    CPU0                             CPU1 *    ----                             ---- *                                     <SOFTIRQ> *                                       call_timer_fn(); *                                       base->running_timer = mytimer; *    spin_lock_irq(somelock); *                                     <IRQ> *                                        spin_lock(somelock); *    timer_delete_sync(mytimer); *    while (base->running_timer == mytimer); * * Now timer_delete_sync() will never return and never release somelock. * The interrupt on the other CPU is waiting to grab somelock but it has * interrupted the softirq that CPU0 is waiting to finish. * * This function cannot guarantee that the timer is not rearmed again by * some concurrent or preempting code, right after it dropped the base * lock. If there is the possibility of a concurrent rearm then the return * value of the function is meaningless. * * If such a guarantee is needed, e.g. for teardown situations then use * timer_shutdown_sync() instead. * * Return: * * %0	- The timer was not pending * * %1	- The timer was pending and deactivated */int timer_delete_sync(struct timer_list *timer){	return __timer_delete_sync(timer, false);}EXPORT_SYMBOL(timer_delete_sync); /** * timer_shutdown_sync - Shutdown a timer and prevent rearming * @timer: The timer to be shutdown * * When the function returns it is guaranteed that: *   - @timer is not queued *   - The callback function of @timer is not running *   - @timer cannot be enqueued again. Any attempt to rearm *     @timer is silently ignored. * * See timer_delete_sync() for synchronization rules. * * This function is useful for final teardown of an infrastructure where * the timer is subject to a circular dependency problem. * * A common pattern for this is a timer and a workqueue where the timer can * schedule work and work can arm the timer. On shutdown the workqueue must * be destroyed and the timer must be prevented from rearming. Unless the * code has conditionals like 'if (mything->in_shutdown)' to prevent that * there is no way to get this correct with timer_delete_sync(). * * timer_shutdown_sync() is solving the problem. The correct ordering of * calls in this case is: * *	timer_shutdown_sync(&mything->timer); *	workqueue_destroy(&mything->workqueue); * * After this 'mything' can be safely freed. * * This obviously implies that the timer is not required to be functional * for the rest of the shutdown operation. * * Return: * * %0 - The timer was not pending * * %1 - The timer was pending */int timer_shutdown_sync(struct timer_list *timer){	return __timer_delete_sync(timer, true);}EXPORT_SYMBOL_GPL(timer_shutdown_sync); static void call_timer_fn(struct timer_list *timer,			  void (*fn)(struct timer_list *),			  unsigned long baseclk){	int count = preempt_count(); #ifdef CONFIG_LOCKDEP	/*	 * It is permissible to free the timer from inside the	 * function that is called from it, this we need to take into	 * account for lockdep too. To avoid bogus "held lock freed"	 * warnings as well as problems when looking into	 * timer->lockdep_map, make a copy and use that here.	 */	struct lockdep_map lockdep_map; 	lockdep_copy_map(&lockdep_map, &timer->lockdep_map);#endif	/*	 * Couple the lock chain with the lock chain at	 * timer_delete_sync() by acquiring the lock_map around the fn()	 * call here and in timer_delete_sync().	 */	lock_map_acquire(&lockdep_map); 	trace_timer_expire_entry(timer, baseclk);	fn(timer);	trace_timer_expire_exit(timer); 	lock_map_release(&lockdep_map); 	if (count != preempt_count()) {		WARN_ONCE(1, "timer: %pS preempt leak: %08x -> %08x\n",			  fn, count, preempt_count());		/*		 * Restore the preempt count. That gives us a decent		 * chance to survive and extract information. If the		 * callback kept a lock held, bad luck, but not worse		 * than the BUG() we had.		 */		preempt_count_set(count);	}} static void expire_timers(struct timer_base *base, struct hlist_head *head){	/*	 * This value is required only for tracing. base->clk was	 * incremented directly before expire_timers was called. But expiry	 * is related to the old base->clk value.	 */	unsigned long baseclk = base->clk - 1; 	while (!hlist_empty(head)) {		struct timer_list *timer;		void (*fn)(struct timer_list *); 		timer = hlist_entry(head->first, struct timer_list, entry); 		base->running_timer = timer;		detach_timer(timer, true); 		fn = timer->function; 		if (WARN_ON_ONCE(!fn)) {			/* Should never happen. Emphasis on should! */			base->running_timer = NULL;			continue;		} 		if (timer->flags & TIMER_IRQSAFE) {			raw_spin_unlock(&base->lock);			call_timer_fn(timer, fn, baseclk);			raw_spin_lock(&base->lock);			base->running_timer = NULL;		} else {			raw_spin_unlock_irq(&base->lock);			call_timer_fn(timer, fn, baseclk);			raw_spin_lock_irq(&base->lock);			base->running_timer = NULL;			timer_sync_wait_running(base);		}	}} static int collect_expired_timers(struct timer_base *base,				  struct hlist_head *heads){	unsigned long clk = base->clk = base->next_expiry;	struct hlist_head *vec;	int i, levels = 0;	unsigned int idx; 	for (i = 0; i < LVL_DEPTH; i++) {		idx = (clk & LVL_MASK) + i * LVL_SIZE; 		if (__test_and_clear_bit(idx, base->pending_map)) {			vec = base->vectors + idx;			hlist_move_list(vec, heads++);			levels++;		}		/* Is it time to look at the next level? */		if (clk & LVL_CLK_MASK)			break;		/* Shift clock for the next level granularity */		clk >>= LVL_CLK_SHIFT;	}	return levels;} /* * Find the next pending bucket of a level. Search from level start (@offset) * + @clk upwards and if nothing there, search from start of the level * (@offset) up to @offset + clk. */static int next_pending_bucket(struct timer_base *base, unsigned offset,			       unsigned clk){	unsigned pos, start = offset + clk;	unsigned end = offset + LVL_SIZE; 	pos = find_next_bit(base->pending_map, end, start);	if (pos < end)		return pos - start; 	pos = find_next_bit(base->pending_map, start, offset);	return pos < start ? pos + LVL_SIZE - start : -1;} /* * Search the first expiring timer in the various clock levels. Caller must * hold base->lock. * * Store next expiry time in base->next_expiry. */static void timer_recalc_next_expiry(struct timer_base *base){	unsigned long clk, next, adj;	unsigned lvl, offset = 0; 	next = base->clk + TIMER_NEXT_MAX_DELTA;	clk = base->clk;	for (lvl = 0; lvl < LVL_DEPTH; lvl++, offset += LVL_SIZE) {		int pos = next_pending_bucket(base, offset, clk & LVL_MASK);		unsigned long lvl_clk = clk & LVL_CLK_MASK; 		if (pos >= 0) {			unsigned long tmp = clk + (unsigned long) pos; 			tmp <<= LVL_SHIFT(lvl);			if (time_before(tmp, next))				next = tmp; 			/*			 * If the next expiration happens before we reach			 * the next level, no need to check further.			 */			if (pos <= ((LVL_CLK_DIV - lvl_clk) & LVL_CLK_MASK))				break;		}		/*		 * Clock for the next level. If the current level clock lower		 * bits are zero, we look at the next level as is. If not we		 * need to advance it by one because that's going to be the		 * next expiring bucket in that level. base->clk is the next		 * expiring jiffy. So in case of:		 *		 * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0		 *  0    0    0    0    0    0		 *		 * we have to look at all levels @index 0. With		 *		 * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0		 *  0    0    0    0    0    2		 *		 * LVL0 has the next expiring bucket @index 2. The upper		 * levels have the next expiring bucket @index 1.		 *		 * In case that the propagation wraps the next level the same		 * rules apply:		 *		 * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0		 *  0    0    0    0    F    2		 *		 * So after looking at LVL0 we get:		 *		 * LVL5 LVL4 LVL3 LVL2 LVL1		 *  0    0    0    1    0		 *		 * So no propagation from LVL1 to LVL2 because that happened		 * with the add already, but then we need to propagate further		 * from LVL2 to LVL3.		 *		 * So the simple check whether the lower bits of the current		 * level are 0 or not is sufficient for all cases.		 */		adj = lvl_clk ? 1 : 0;		clk >>= LVL_CLK_SHIFT;		clk += adj;	} 	WRITE_ONCE(base->next_expiry, next);	base->next_expiry_recalc = false;	base->timers_pending = !(next == base->clk + TIMER_NEXT_MAX_DELTA);} #ifdef CONFIG_NO_HZ_COMMON/* * Check, if the next hrtimer event is before the next timer wheel * event: */static u64 cmp_next_hrtimer_event(u64 basem, u64 expires){	u64 nextevt = ktime_to_ns(hrtimer_get_next_event()); 	/*	 * If high resolution timers are enabled	 * hrtimer_get_next_event() returns KTIME_MAX.	 */	if (expires <= nextevt)		return expires; 	/*	 * If the next timer is already expired, return the tick base	 * time so the tick is fired immediately.	 */	if (nextevt <= basem)		return basem; 	/*	 * Round up to the next jiffy. High resolution timers are	 * off, so the hrtimers are expired in the tick and we need to	 * make sure that this tick really expires the timer to avoid	 * a ping pong of the nohz stop code.	 *	 * Use DIV_ROUND_UP_ULL to prevent gcc calling __divdi3	 */	return DIV_ROUND_UP_ULL(nextevt, TICK_NSEC) * TICK_NSEC;} static unsigned long next_timer_interrupt(struct timer_base *base,					  unsigned long basej){	if (base->next_expiry_recalc)		timer_recalc_next_expiry(base); 	/*	 * Move next_expiry for the empty base into the future to prevent an	 * unnecessary raise of the timer softirq when the next_expiry value	 * will be reached even if there is no timer pending.	 *	 * This update is also required to make timer_base::next_expiry values	 * easy comparable to find out which base holds the first pending timer.	 */	if (!base->timers_pending)		WRITE_ONCE(base->next_expiry, basej + TIMER_NEXT_MAX_DELTA); 	return base->next_expiry;} static unsigned long fetch_next_timer_interrupt(unsigned long basej, u64 basem,						struct timer_base *base_local,						struct timer_base *base_global,						struct timer_events *tevt){	unsigned long nextevt, nextevt_local, nextevt_global;	bool local_first; 	nextevt_local = next_timer_interrupt(base_local, basej);	nextevt_global = next_timer_interrupt(base_global, basej); 	local_first = time_before_eq(nextevt_local, nextevt_global); 	nextevt = local_first ? nextevt_local : nextevt_global; 	/*	 * If the @nextevt is at max. one tick away, use @nextevt and store	 * it in the local expiry value. The next global event is irrelevant in	 * this case and can be left as KTIME_MAX.

Lines 1501–2000 of 2580. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t