2076 lines · 60.1 KB · C
RawDownload
 	pos = find_next_bit(base->pending_map, end, start);	if (pos < end)		return pos - start; 	pos = find_next_bit(base->pending_map, start, offset);	return pos < start ? pos + LVL_SIZE - start : -1;} /* * Search the first expiring timer in the various clock levels. Caller must * hold base->lock. */static unsigned long __next_timer_interrupt(struct timer_base *base){	unsigned long clk, next, adj;	unsigned lvl, offset = 0; 	next = base->clk + NEXT_TIMER_MAX_DELTA;	clk = base->clk;	for (lvl = 0; lvl < LVL_DEPTH; lvl++, offset += LVL_SIZE) {		int pos = next_pending_bucket(base, offset, clk & LVL_MASK);		unsigned long lvl_clk = clk & LVL_CLK_MASK; 		if (pos >= 0) {			unsigned long tmp = clk + (unsigned long) pos; 			tmp <<= LVL_SHIFT(lvl);			if (time_before(tmp, next))				next = tmp; 			/*			 * If the next expiration happens before we reach			 * the next level, no need to check further.			 */			if (pos <= ((LVL_CLK_DIV - lvl_clk) & LVL_CLK_MASK))				break;		}		/*		 * Clock for the next level. If the current level clock lower		 * bits are zero, we look at the next level as is. If not we		 * need to advance it by one because that's going to be the		 * next expiring bucket in that level. base->clk is the next		 * expiring jiffie. So in case of:		 *		 * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0		 *  0    0    0    0    0    0		 *		 * we have to look at all levels @index 0. With		 *		 * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0		 *  0    0    0    0    0    2		 *		 * LVL0 has the next expiring bucket @index 2. The upper		 * levels have the next expiring bucket @index 1.		 *		 * In case that the propagation wraps the next level the same		 * rules apply:		 *		 * LVL5 LVL4 LVL3 LVL2 LVL1 LVL0		 *  0    0    0    0    F    2		 *		 * So after looking at LVL0 we get:		 *		 * LVL5 LVL4 LVL3 LVL2 LVL1		 *  0    0    0    1    0		 *		 * So no propagation from LVL1 to LVL2 because that happened		 * with the add already, but then we need to propagate further		 * from LVL2 to LVL3.		 *		 * So the simple check whether the lower bits of the current		 * level are 0 or not is sufficient for all cases.		 */		adj = lvl_clk ? 1 : 0;		clk >>= LVL_CLK_SHIFT;		clk += adj;	} 	base->next_expiry_recalc = false; 	return next;} #ifdef CONFIG_NO_HZ_COMMON/* * Check, if the next hrtimer event is before the next timer wheel * event: */static u64 cmp_next_hrtimer_event(u64 basem, u64 expires){	u64 nextevt = hrtimer_get_next_event(); 	/*	 * If high resolution timers are enabled	 * hrtimer_get_next_event() returns KTIME_MAX.	 */	if (expires <= nextevt)		return expires; 	/*	 * If the next timer is already expired, return the tick base	 * time so the tick is fired immediately.	 */	if (nextevt <= basem)		return basem; 	/*	 * Round up to the next jiffie. High resolution timers are	 * off, so the hrtimers are expired in the tick and we need to	 * make sure that this tick really expires the timer to avoid	 * a ping pong of the nohz stop code.	 *	 * Use DIV_ROUND_UP_ULL to prevent gcc calling __divdi3	 */	return DIV_ROUND_UP_ULL(nextevt, TICK_NSEC) * TICK_NSEC;} /** * get_next_timer_interrupt - return the time (clock mono) of the next timer * @basej:	base time jiffies * @basem:	base time clock monotonic * * Returns the tick aligned clock monotonic time of the next pending * timer or KTIME_MAX if no timer is pending. */u64 get_next_timer_interrupt(unsigned long basej, u64 basem){	struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]);	u64 expires = KTIME_MAX;	unsigned long nextevt;	bool is_max_delta; 	/*	 * Pretend that there is no timer pending if the cpu is offline.	 * Possible pending timers will be migrated later to an active cpu.	 */	if (cpu_is_offline(smp_processor_id()))		return expires; 	raw_spin_lock(&base->lock);	if (base->next_expiry_recalc)		base->next_expiry = __next_timer_interrupt(base);	nextevt = base->next_expiry;	is_max_delta = (nextevt == base->clk + NEXT_TIMER_MAX_DELTA); 	/*	 * We have a fresh next event. Check whether we can forward the	 * base. We can only do that when @basej is past base->clk	 * otherwise we might rewind base->clk.	 */	if (time_after(basej, base->clk)) {		if (time_after(nextevt, basej))			base->clk = basej;		else if (time_after(nextevt, base->clk))			base->clk = nextevt;	} 	if (time_before_eq(nextevt, basej)) {		expires = basem;		base->is_idle = false;	} else {		if (!is_max_delta)			expires = basem + (u64)(nextevt - basej) * TICK_NSEC;		/*		 * If we expect to sleep more than a tick, mark the base idle.		 * Also the tick is stopped so any added timer must forward		 * the base clk itself to keep granularity small. This idle		 * logic is only maintained for the BASE_STD base, deferrable		 * timers may still see large granularity skew (by design).		 */		if ((expires - basem) > TICK_NSEC)			base->is_idle = true;	}	raw_spin_unlock(&base->lock); 	return cmp_next_hrtimer_event(basem, expires);} /** * timer_clear_idle - Clear the idle state of the timer base * * Called with interrupts disabled */void timer_clear_idle(void){	struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]); 	/*	 * We do this unlocked. The worst outcome is a remote enqueue sending	 * a pointless IPI, but taking the lock would just make the window for	 * sending the IPI a few instructions smaller for the cost of taking	 * the lock in the exit from idle path.	 */	base->is_idle = false;}#endif /* * Called from the timer interrupt handler to charge one tick to the current * process.  user_tick is 1 if the tick is user time, 0 for system. */void update_process_times(int user_tick){	struct task_struct *p = current; 	/* Note: this timer irq context must be accounted for as well. */	account_process_tick(p, user_tick);	run_local_timers();	rcu_sched_clock_irq(user_tick);#ifdef CONFIG_IRQ_WORK	if (in_irq())		irq_work_tick();#endif	scheduler_tick();	if (IS_ENABLED(CONFIG_POSIX_TIMERS))		run_posix_cpu_timers(); 	/* The current CPU might make use of net randoms without receiving IRQs	 * to renew them often enough. Let's update the net_rand_state from a	 * non-constant value that's not affine to the number of calls to make	 * sure it's updated when there's some activity (we don't care in idle).	 */	this_cpu_add(net_rand_state.s1, rol32(jiffies, 24) + user_tick);} /** * __run_timers - run all expired timers (if any) on this CPU. * @base: the timer vector to be processed. */static inline void __run_timers(struct timer_base *base){	struct hlist_head heads[LVL_DEPTH];	int levels; 	if (time_before(jiffies, base->next_expiry))		return; 	timer_base_lock_expiry(base);	raw_spin_lock_irq(&base->lock); 	while (time_after_eq(jiffies, base->clk) &&	       time_after_eq(jiffies, base->next_expiry)) {		levels = collect_expired_timers(base, heads);		/*		 * The only possible reason for not finding any expired		 * timer at this clk is that all matching timers have been		 * dequeued.		 */		WARN_ON_ONCE(!levels && !base->next_expiry_recalc);		base->clk++;		base->next_expiry = __next_timer_interrupt(base); 		while (levels--)			expire_timers(base, heads + levels);	}	raw_spin_unlock_irq(&base->lock);	timer_base_unlock_expiry(base);} /* * This function runs timers and the timer-tq in bottom half context. */static __latent_entropy void run_timer_softirq(struct softirq_action *h){	struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]); 	__run_timers(base);	if (IS_ENABLED(CONFIG_NO_HZ_COMMON))		__run_timers(this_cpu_ptr(&timer_bases[BASE_DEF]));} /* * Called by the local, per-CPU timer interrupt on SMP. */void run_local_timers(void){	struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]); 	hrtimer_run_queues();	/* Raise the softirq only if required. */	if (time_before(jiffies, base->next_expiry)) {		if (!IS_ENABLED(CONFIG_NO_HZ_COMMON))			return;		/* CPU is awake, so check the deferrable base. */		base++;		if (time_before(jiffies, base->next_expiry))			return;	}	raise_softirq(TIMER_SOFTIRQ);} /* * Since schedule_timeout()'s timer is defined on the stack, it must store * the target task on the stack as well. */struct process_timer {	struct timer_list timer;	struct task_struct *task;}; static void process_timeout(struct timer_list *t){	struct process_timer *timeout = from_timer(timeout, t, timer); 	wake_up_process(timeout->task);} /** * schedule_timeout - sleep until timeout * @timeout: timeout value in jiffies * * Make the current task sleep until @timeout jiffies have elapsed. * The function behavior depends on the current task state * (see also set_current_state() description): * * %TASK_RUNNING - the scheduler is called, but the task does not sleep * at all. That happens because sched_submit_work() does nothing for * tasks in %TASK_RUNNING state. * * %TASK_UNINTERRUPTIBLE - at least @timeout jiffies are guaranteed to * pass before the routine returns unless the current task is explicitly * woken up, (e.g. by wake_up_process()). * * %TASK_INTERRUPTIBLE - the routine may return early if a signal is * delivered to the current task or the current task is explicitly woken * up. * * The current task state is guaranteed to be %TASK_RUNNING when this * routine returns. * * Specifying a @timeout value of %MAX_SCHEDULE_TIMEOUT will schedule * the CPU away without a bound on the timeout. In this case the return * value will be %MAX_SCHEDULE_TIMEOUT. * * Returns 0 when the timer has expired otherwise the remaining time in * jiffies will be returned. In all cases the return value is guaranteed * to be non-negative. */signed long __sched schedule_timeout(signed long timeout){	struct process_timer timer;	unsigned long expire; 	switch (timeout)	{	case MAX_SCHEDULE_TIMEOUT:		/*		 * These two special cases are useful to be comfortable		 * in the caller. Nothing more. We could take		 * MAX_SCHEDULE_TIMEOUT from one of the negative value		 * but I' d like to return a valid offset (>=0) to allow		 * the caller to do everything it want with the retval.		 */		schedule();		goto out;	default:		/*		 * Another bit of PARANOID. Note that the retval will be		 * 0 since no piece of kernel is supposed to do a check		 * for a negative retval of schedule_timeout() (since it		 * should never happens anyway). You just have the printk()		 * that will tell you if something is gone wrong and where.		 */		if (timeout < 0) {			printk(KERN_ERR "schedule_timeout: wrong timeout "				"value %lx\n", timeout);			dump_stack();			current->state = TASK_RUNNING;			goto out;		}	} 	expire = timeout + jiffies; 	timer.task = current;	timer_setup_on_stack(&timer.timer, process_timeout, 0);	__mod_timer(&timer.timer, expire, MOD_TIMER_NOTPENDING);	schedule();	del_singleshot_timer_sync(&timer.timer); 	/* Remove the timer from the object tracker */	destroy_timer_on_stack(&timer.timer); 	timeout = expire - jiffies;  out:	return timeout < 0 ? 0 : timeout;}EXPORT_SYMBOL(schedule_timeout); /* * We can use __set_current_state() here because schedule_timeout() calls * schedule() unconditionally. */signed long __sched schedule_timeout_interruptible(signed long timeout){	__set_current_state(TASK_INTERRUPTIBLE);	return schedule_timeout(timeout);}EXPORT_SYMBOL(schedule_timeout_interruptible); signed long __sched schedule_timeout_killable(signed long timeout){	__set_current_state(TASK_KILLABLE);	return schedule_timeout(timeout);}EXPORT_SYMBOL(schedule_timeout_killable); signed long __sched schedule_timeout_uninterruptible(signed long timeout){	__set_current_state(TASK_UNINTERRUPTIBLE);	return schedule_timeout(timeout);}EXPORT_SYMBOL(schedule_timeout_uninterruptible); /* * Like schedule_timeout_uninterruptible(), except this task will not contribute * to load average. */signed long __sched schedule_timeout_idle(signed long timeout){	__set_current_state(TASK_IDLE);	return schedule_timeout(timeout);}EXPORT_SYMBOL(schedule_timeout_idle); #ifdef CONFIG_HOTPLUG_CPUstatic void migrate_timer_list(struct timer_base *new_base, struct hlist_head *head){	struct timer_list *timer;	int cpu = new_base->cpu; 	while (!hlist_empty(head)) {		timer = hlist_entry(head->first, struct timer_list, entry);		detach_timer(timer, false);		timer->flags = (timer->flags & ~TIMER_BASEMASK) | cpu;		internal_add_timer(new_base, timer);	}} int timers_prepare_cpu(unsigned int cpu){	struct timer_base *base;	int b; 	for (b = 0; b < NR_BASES; b++) {		base = per_cpu_ptr(&timer_bases[b], cpu);		base->clk = jiffies;		base->next_expiry = base->clk + NEXT_TIMER_MAX_DELTA;		base->is_idle = false;	}	return 0;} int timers_dead_cpu(unsigned int cpu){	struct timer_base *old_base;	struct timer_base *new_base;	int b, i; 	BUG_ON(cpu_online(cpu)); 	for (b = 0; b < NR_BASES; b++) {		old_base = per_cpu_ptr(&timer_bases[b], cpu);		new_base = get_cpu_ptr(&timer_bases[b]);		/*		 * The caller is globally serialized and nobody else		 * takes two locks at once, deadlock is not possible.		 */		raw_spin_lock_irq(&new_base->lock);		raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING); 		/*		 * The current CPUs base clock might be stale. Update it		 * before moving the timers over.		 */		forward_timer_base(new_base); 		BUG_ON(old_base->running_timer); 		for (i = 0; i < WHEEL_SIZE; i++)			migrate_timer_list(new_base, old_base->vectors + i); 		raw_spin_unlock(&old_base->lock);		raw_spin_unlock_irq(&new_base->lock);		put_cpu_ptr(&timer_bases);	}	return 0;} #endif /* CONFIG_HOTPLUG_CPU */ static void __init init_timer_cpu(int cpu){	struct timer_base *base;	int i; 	for (i = 0; i < NR_BASES; i++) {		base = per_cpu_ptr(&timer_bases[i], cpu);

Lines 1501–2000 of 2076. Previous lines

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t