2076 lines · 60.1 KB · C
RawDownload
		} 		clk = base->clk;		idx = calc_wheel_index(expires, clk, &bucket_expiry); 		/*		 * Retrieve and compare the array index of the pending		 * timer. If it matches set the expiry to the new value so a		 * subsequent call will exit in the expires check above.		 */		if (idx == timer_get_idx(timer)) {			if (!(options & MOD_TIMER_REDUCE))				timer->expires = expires;			else if (time_after(timer->expires, expires))				timer->expires = expires;			ret = 1;			goto out_unlock;		}	} else {		base = lock_timer_base(timer, &flags);		forward_timer_base(base);	} 	ret = detach_if_pending(timer, base, false);	if (!ret && (options & MOD_TIMER_PENDING_ONLY))		goto out_unlock; 	new_base = get_target_base(base, timer->flags); 	if (base != new_base) {		/*		 * We are trying to schedule the timer on the new base.		 * However we can't change timer's base while it is running,		 * otherwise del_timer_sync() can't detect that the timer's		 * handler yet has not finished. This also guarantees that the		 * timer is serialized wrt itself.		 */		if (likely(base->running_timer != timer)) {			/* See the comment in lock_timer_base() */			timer->flags |= TIMER_MIGRATING; 			raw_spin_unlock(&base->lock);			base = new_base;			raw_spin_lock(&base->lock);			WRITE_ONCE(timer->flags,				   (timer->flags & ~TIMER_BASEMASK) | base->cpu);			forward_timer_base(base);		}	} 	debug_timer_activate(timer); 	timer->expires = expires;	/*	 * If 'idx' was calculated above and the base time did not advance	 * between calculating 'idx' and possibly switching the base, only	 * enqueue_timer() is required. Otherwise we need to (re)calculate	 * the wheel index via internal_add_timer().	 */	if (idx != UINT_MAX && clk == base->clk)		enqueue_timer(base, timer, idx, bucket_expiry);	else		internal_add_timer(base, timer); out_unlock:	raw_spin_unlock_irqrestore(&base->lock, flags); 	return ret;} /** * mod_timer_pending - modify a pending timer's timeout * @timer: the pending timer to be modified * @expires: new timeout in jiffies * * mod_timer_pending() is the same for pending timers as mod_timer(), * but will not re-activate and modify already deleted timers. * * It is useful for unserialized use of timers. */int mod_timer_pending(struct timer_list *timer, unsigned long expires){	return __mod_timer(timer, expires, MOD_TIMER_PENDING_ONLY);}EXPORT_SYMBOL(mod_timer_pending); /** * mod_timer - modify a timer's timeout * @timer: the timer to be modified * @expires: new timeout in jiffies * * mod_timer() is a more efficient way to update the expire field of an * active timer (if the timer is inactive it will be activated) * * mod_timer(timer, expires) is equivalent to: * *     del_timer(timer); timer->expires = expires; add_timer(timer); * * Note that if there are multiple unserialized concurrent users of the * same timer, then mod_timer() is the only safe way to modify the timeout, * since add_timer() cannot modify an already running timer. * * The function returns whether it has modified a pending timer or not. * (ie. mod_timer() of an inactive timer returns 0, mod_timer() of an * active timer returns 1.) */int mod_timer(struct timer_list *timer, unsigned long expires){	return __mod_timer(timer, expires, 0);}EXPORT_SYMBOL(mod_timer); /** * timer_reduce - Modify a timer's timeout if it would reduce the timeout * @timer:	The timer to be modified * @expires:	New timeout in jiffies * * timer_reduce() is very similar to mod_timer(), except that it will only * modify a running timer if that would reduce the expiration time (it will * start a timer that isn't running). */int timer_reduce(struct timer_list *timer, unsigned long expires){	return __mod_timer(timer, expires, MOD_TIMER_REDUCE);}EXPORT_SYMBOL(timer_reduce); /** * add_timer - start a timer * @timer: the timer to be added * * The kernel will do a ->function(@timer) callback from the * timer interrupt at the ->expires point in the future. The * current time is 'jiffies'. * * The timer's ->expires, ->function fields must be set prior calling this * function. * * Timers with an ->expires field in the past will be executed in the next * timer tick. */void add_timer(struct timer_list *timer){	BUG_ON(timer_pending(timer));	__mod_timer(timer, timer->expires, MOD_TIMER_NOTPENDING);}EXPORT_SYMBOL(add_timer); /** * add_timer_on - start a timer on a particular CPU * @timer: the timer to be added * @cpu: the CPU to start it on * * This is not very scalable on SMP. Double adds are not possible. */void add_timer_on(struct timer_list *timer, int cpu){	struct timer_base *new_base, *base;	unsigned long flags; 	BUG_ON(timer_pending(timer) || !timer->function); 	new_base = get_timer_cpu_base(timer->flags, cpu); 	/*	 * If @timer was on a different CPU, it should be migrated with the	 * old base locked to prevent other operations proceeding with the	 * wrong base locked.  See lock_timer_base().	 */	base = lock_timer_base(timer, &flags);	if (base != new_base) {		timer->flags |= TIMER_MIGRATING; 		raw_spin_unlock(&base->lock);		base = new_base;		raw_spin_lock(&base->lock);		WRITE_ONCE(timer->flags,			   (timer->flags & ~TIMER_BASEMASK) | cpu);	}	forward_timer_base(base); 	debug_timer_activate(timer);	internal_add_timer(base, timer);	raw_spin_unlock_irqrestore(&base->lock, flags);}EXPORT_SYMBOL_GPL(add_timer_on); /** * del_timer - deactivate a timer. * @timer: the timer to be deactivated * * del_timer() deactivates a timer - this works on both active and inactive * timers. * * The function returns whether it has deactivated a pending timer or not. * (ie. del_timer() of an inactive timer returns 0, del_timer() of an * active timer returns 1.) */int del_timer(struct timer_list *timer){	struct timer_base *base;	unsigned long flags;	int ret = 0; 	debug_assert_init(timer); 	if (timer_pending(timer)) {		base = lock_timer_base(timer, &flags);		ret = detach_if_pending(timer, base, true);		raw_spin_unlock_irqrestore(&base->lock, flags);	} 	return ret;}EXPORT_SYMBOL(del_timer); /** * try_to_del_timer_sync - Try to deactivate a timer * @timer: timer to delete * * This function tries to deactivate a timer. Upon successful (ret >= 0) * exit the timer is not queued and the handler is not running on any CPU. */int try_to_del_timer_sync(struct timer_list *timer){	struct timer_base *base;	unsigned long flags;	int ret = -1; 	debug_assert_init(timer); 	base = lock_timer_base(timer, &flags); 	if (base->running_timer != timer)		ret = detach_if_pending(timer, base, true); 	raw_spin_unlock_irqrestore(&base->lock, flags); 	return ret;}EXPORT_SYMBOL(try_to_del_timer_sync); #ifdef CONFIG_PREEMPT_RTstatic __init void timer_base_init_expiry_lock(struct timer_base *base){	spin_lock_init(&base->expiry_lock);} static inline void timer_base_lock_expiry(struct timer_base *base){	spin_lock(&base->expiry_lock);} static inline void timer_base_unlock_expiry(struct timer_base *base){	spin_unlock(&base->expiry_lock);} /* * The counterpart to del_timer_wait_running(). * * If there is a waiter for base->expiry_lock, then it was waiting for the * timer callback to finish. Drop expiry_lock and reaquire it. That allows * the waiter to acquire the lock and make progress. */static void timer_sync_wait_running(struct timer_base *base){	if (atomic_read(&base->timer_waiters)) {		spin_unlock(&base->expiry_lock);		spin_lock(&base->expiry_lock);	}} /* * This function is called on PREEMPT_RT kernels when the fast path * deletion of a timer failed because the timer callback function was * running. * * This prevents priority inversion, if the softirq thread on a remote CPU * got preempted, and it prevents a life lock when the task which tries to * delete a timer preempted the softirq thread running the timer callback * function. */static void del_timer_wait_running(struct timer_list *timer){	u32 tf; 	tf = READ_ONCE(timer->flags);	if (!(tf & TIMER_MIGRATING)) {		struct timer_base *base = get_timer_base(tf); 		/*		 * Mark the base as contended and grab the expiry lock,		 * which is held by the softirq across the timer		 * callback. Drop the lock immediately so the softirq can		 * expire the next timer. In theory the timer could already		 * be running again, but that's more than unlikely and just		 * causes another wait loop.		 */		atomic_inc(&base->timer_waiters);		spin_lock_bh(&base->expiry_lock);		atomic_dec(&base->timer_waiters);		spin_unlock_bh(&base->expiry_lock);	}}#elsestatic inline void timer_base_init_expiry_lock(struct timer_base *base) { }static inline void timer_base_lock_expiry(struct timer_base *base) { }static inline void timer_base_unlock_expiry(struct timer_base *base) { }static inline void timer_sync_wait_running(struct timer_base *base) { }static inline void del_timer_wait_running(struct timer_list *timer) { }#endif #if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)/** * del_timer_sync - deactivate a timer and wait for the handler to finish. * @timer: the timer to be deactivated * * This function only differs from del_timer() on SMP: besides deactivating * the timer it also makes sure the handler has finished executing on other * CPUs. * * Synchronization rules: Callers must prevent restarting of the timer, * otherwise this function is meaningless. It must not be called from * interrupt contexts unless the timer is an irqsafe one. The caller must * not hold locks which would prevent completion of the timer's * handler. The timer's handler must not call add_timer_on(). Upon exit the * timer is not queued and the handler is not running on any CPU. * * Note: For !irqsafe timers, you must not hold locks that are held in *   interrupt context while calling this function. Even if the lock has *   nothing to do with the timer in question.  Here's why:: * *    CPU0                             CPU1 *    ----                             ---- *                                     <SOFTIRQ> *                                       call_timer_fn(); *                                       base->running_timer = mytimer; *    spin_lock_irq(somelock); *                                     <IRQ> *                                        spin_lock(somelock); *    del_timer_sync(mytimer); *    while (base->running_timer == mytimer); * * Now del_timer_sync() will never return and never release somelock. * The interrupt on the other CPU is waiting to grab somelock but * it has interrupted the softirq that CPU0 is waiting to finish. * * The function returns whether it has deactivated a pending timer or not. */int del_timer_sync(struct timer_list *timer){	int ret; #ifdef CONFIG_LOCKDEP	unsigned long flags; 	/*	 * If lockdep gives a backtrace here, please reference	 * the synchronization rules above.	 */	local_irq_save(flags);	lock_map_acquire(&timer->lockdep_map);	lock_map_release(&timer->lockdep_map);	local_irq_restore(flags);#endif	/*	 * don't use it in hardirq context, because it	 * could lead to deadlock.	 */	WARN_ON(in_irq() && !(timer->flags & TIMER_IRQSAFE)); 	do {		ret = try_to_del_timer_sync(timer); 		if (unlikely(ret < 0)) {			del_timer_wait_running(timer);			cpu_relax();		}	} while (ret < 0); 	return ret;}EXPORT_SYMBOL(del_timer_sync);#endif static void call_timer_fn(struct timer_list *timer,			  void (*fn)(struct timer_list *),			  unsigned long baseclk){	int count = preempt_count(); #ifdef CONFIG_LOCKDEP	/*	 * It is permissible to free the timer from inside the	 * function that is called from it, this we need to take into	 * account for lockdep too. To avoid bogus "held lock freed"	 * warnings as well as problems when looking into	 * timer->lockdep_map, make a copy and use that here.	 */	struct lockdep_map lockdep_map; 	lockdep_copy_map(&lockdep_map, &timer->lockdep_map);#endif	/*	 * Couple the lock chain with the lock chain at	 * del_timer_sync() by acquiring the lock_map around the fn()	 * call here and in del_timer_sync().	 */	lock_map_acquire(&lockdep_map); 	trace_timer_expire_entry(timer, baseclk);	fn(timer);	trace_timer_expire_exit(timer); 	lock_map_release(&lockdep_map); 	if (count != preempt_count()) {		WARN_ONCE(1, "timer: %pS preempt leak: %08x -> %08x\n",			  fn, count, preempt_count());		/*		 * Restore the preempt count. That gives us a decent		 * chance to survive and extract information. If the		 * callback kept a lock held, bad luck, but not worse		 * than the BUG() we had.		 */		preempt_count_set(count);	}} static void expire_timers(struct timer_base *base, struct hlist_head *head){	/*	 * This value is required only for tracing. base->clk was	 * incremented directly before expire_timers was called. But expiry	 * is related to the old base->clk value.	 */	unsigned long baseclk = base->clk - 1; 	while (!hlist_empty(head)) {		struct timer_list *timer;		void (*fn)(struct timer_list *); 		timer = hlist_entry(head->first, struct timer_list, entry); 		base->running_timer = timer;		detach_timer(timer, true); 		fn = timer->function; 		if (timer->flags & TIMER_IRQSAFE) {			raw_spin_unlock(&base->lock);			call_timer_fn(timer, fn, baseclk);			base->running_timer = NULL;			raw_spin_lock(&base->lock);		} else {			raw_spin_unlock_irq(&base->lock);			call_timer_fn(timer, fn, baseclk);			base->running_timer = NULL;			timer_sync_wait_running(base);			raw_spin_lock_irq(&base->lock);		}	}} static int collect_expired_timers(struct timer_base *base,				  struct hlist_head *heads){	unsigned long clk = base->clk = base->next_expiry;	struct hlist_head *vec;	int i, levels = 0;	unsigned int idx; 	for (i = 0; i < LVL_DEPTH; i++) {		idx = (clk & LVL_MASK) + i * LVL_SIZE; 		if (__test_and_clear_bit(idx, base->pending_map)) {			vec = base->vectors + idx;			hlist_move_list(vec, heads++);			levels++;		}		/* Is it time to look at the next level? */		if (clk & LVL_CLK_MASK)			break;		/* Shift clock for the next level granularity */		clk >>= LVL_CLK_SHIFT;	}	return levels;} /* * Find the next pending bucket of a level. Search from level start (@offset) * + @clk upwards and if nothing there, search from start of the level * (@offset) up to @offset + clk. */static int next_pending_bucket(struct timer_base *base, unsigned offset,			       unsigned clk){	unsigned pos, start = offset + clk;	unsigned end = offset + LVL_SIZE;

Lines 1001–1500 of 2076. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t