for (;;) { int ret = hrtimer_try_to_cancel(timer); if (ret >= 0) return ret; cpu_relax(); }}EXPORT_SYMBOL_GPL(hrtimer_cancel); /** * hrtimer_get_remaining - get remaining time for the timer * * @timer: the timer to read */ktime_t hrtimer_get_remaining(const struct hrtimer *timer){ struct hrtimer_base *base; unsigned long flags; ktime_t rem; base = lock_hrtimer_base(timer, &flags); rem = ktime_sub(timer->expires, timer->base->get_time()); unlock_hrtimer_base(timer, &flags); return rem;}EXPORT_SYMBOL_GPL(hrtimer_get_remaining); #ifdef CONFIG_NO_IDLE_HZ/** * hrtimer_get_next_event - get the time until next expiry event * * Returns the delta to the next expiry event or KTIME_MAX if no timer * is pending. */ktime_t hrtimer_get_next_event(void){ struct hrtimer_base *base = __get_cpu_var(hrtimer_bases); ktime_t delta, mindelta = { .tv64 = KTIME_MAX }; unsigned long flags; int i; for (i = 0; i < MAX_HRTIMER_BASES; i++, base++) { struct hrtimer *timer; spin_lock_irqsave(&base->lock, flags); if (!base->first) { spin_unlock_irqrestore(&base->lock, flags); continue; } timer = rb_entry(base->first, struct hrtimer, node); delta.tv64 = timer->expires.tv64; spin_unlock_irqrestore(&base->lock, flags); delta = ktime_sub(delta, base->get_time()); if (delta.tv64 < mindelta.tv64) mindelta.tv64 = delta.tv64; } if (mindelta.tv64 < 0) mindelta.tv64 = 0; return mindelta;}#endif /** * hrtimer_init - initialize a timer to the given clock * * @timer: the timer to be initialized * @clock_id: the clock to be used * @mode: timer mode abs/rel */void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, enum hrtimer_mode mode){ struct hrtimer_base *bases; memset(timer, 0, sizeof(struct hrtimer)); bases = per_cpu(hrtimer_bases, raw_smp_processor_id()); if (clock_id == CLOCK_REALTIME && mode != HRTIMER_ABS) clock_id = CLOCK_MONOTONIC; timer->base = &bases[clock_id]; rb_set_parent(&timer->node, &timer->node);}EXPORT_SYMBOL_GPL(hrtimer_init); /** * hrtimer_get_res - get the timer resolution for a clock * * @which_clock: which clock to query * @tp: pointer to timespec variable to store the resolution * * Store the resolution of the clock selected by which_clock in the * variable pointed to by tp. */int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp){ struct hrtimer_base *bases; bases = per_cpu(hrtimer_bases, raw_smp_processor_id()); *tp = ktime_to_timespec(bases[which_clock].resolution); return 0;}EXPORT_SYMBOL_GPL(hrtimer_get_res); /* * Expire the per base hrtimer-queue: */static inline void run_hrtimer_queue(struct hrtimer_base *base){ struct rb_node *node; if (!base->first) return; if (base->get_softirq_time) base->softirq_time = base->get_softirq_time(); spin_lock_irq(&base->lock); while ((node = base->first)) { struct hrtimer *timer; int (*fn)(struct hrtimer *); int restart; timer = rb_entry(node, struct hrtimer, node); if (base->softirq_time.tv64 <= timer->expires.tv64) break; fn = timer->function; set_curr_timer(base, timer); __remove_hrtimer(timer, base); spin_unlock_irq(&base->lock); restart = fn(timer); spin_lock_irq(&base->lock); if (restart != HRTIMER_NORESTART) { BUG_ON(hrtimer_active(timer)); enqueue_hrtimer(timer, base); } } set_curr_timer(base, NULL); spin_unlock_irq(&base->lock);} /* * Called from timer softirq every jiffy, expire hrtimers: */void hrtimer_run_queues(void){ struct hrtimer_base *base = __get_cpu_var(hrtimer_bases); int i; hrtimer_get_softirq_time(base); for (i = 0; i < MAX_HRTIMER_BASES; i++) run_hrtimer_queue(&base[i]);} /* * Sleep related functions: */static int hrtimer_wakeup(struct hrtimer *timer){ struct hrtimer_sleeper *t = container_of(timer, struct hrtimer_sleeper, timer); struct task_struct *task = t->task; t->task = NULL; if (task) wake_up_process(task); return HRTIMER_NORESTART;} void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, task_t *task){ sl->timer.function = hrtimer_wakeup; sl->task = task;} static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode){ hrtimer_init_sleeper(t, current); do { set_current_state(TASK_INTERRUPTIBLE); hrtimer_start(&t->timer, t->timer.expires, mode); schedule(); hrtimer_cancel(&t->timer); mode = HRTIMER_ABS; } while (t->task && !signal_pending(current)); return t->task == NULL;} static long __sched nanosleep_restart(struct restart_block *restart){ struct hrtimer_sleeper t; struct timespec __user *rmtp; struct timespec tu; ktime_t time; restart->fn = do_no_restart_syscall; hrtimer_init(&t.timer, restart->arg3, HRTIMER_ABS); t.timer.expires.tv64 = ((u64)restart->arg1 << 32) | (u64) restart->arg0; if (do_nanosleep(&t, HRTIMER_ABS)) return 0; rmtp = (struct timespec __user *) restart->arg2; if (rmtp) { time = ktime_sub(t.timer.expires, t.timer.base->get_time()); if (time.tv64 <= 0) return 0; tu = ktime_to_timespec(time); if (copy_to_user(rmtp, &tu, sizeof(tu))) return -EFAULT; } restart->fn = nanosleep_restart; /* The other values in restart are already filled in */ return -ERESTART_RESTARTBLOCK;} long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, const enum hrtimer_mode mode, const clockid_t clockid){ struct restart_block *restart; struct hrtimer_sleeper t; struct timespec tu; ktime_t rem; hrtimer_init(&t.timer, clockid, mode); t.timer.expires = timespec_to_ktime(*rqtp); if (do_nanosleep(&t, mode)) return 0; /* Absolute timers do not update the rmtp value and restart: */ if (mode == HRTIMER_ABS) return -ERESTARTNOHAND; if (rmtp) { rem = ktime_sub(t.timer.expires, t.timer.base->get_time()); if (rem.tv64 <= 0) return 0; tu = ktime_to_timespec(rem); if (copy_to_user(rmtp, &tu, sizeof(tu))) return -EFAULT; } restart = ¤t_thread_info()->restart_block; restart->fn = nanosleep_restart; restart->arg0 = t.timer.expires.tv64 & 0xFFFFFFFF; restart->arg1 = t.timer.expires.tv64 >> 32; restart->arg2 = (unsigned long) rmtp; restart->arg3 = (unsigned long) t.timer.base->index; return -ERESTART_RESTARTBLOCK;} asmlinkage longsys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp){ struct timespec tu; if (copy_from_user(&tu, rqtp, sizeof(tu))) return -EFAULT; if (!timespec_valid(&tu)) return -EINVAL; return hrtimer_nanosleep(&tu, rmtp, HRTIMER_REL, CLOCK_MONOTONIC);} /* * Functions related to boot-time initialization: */static void __devinit init_hrtimers_cpu(int cpu){ struct hrtimer_base *base = per_cpu(hrtimer_bases, cpu); int i; for (i = 0; i < MAX_HRTIMER_BASES; i++, base++) spin_lock_init(&base->lock);} #ifdef CONFIG_HOTPLUG_CPU static void migrate_hrtimer_list(struct hrtimer_base *old_base, struct hrtimer_base *new_base){ struct hrtimer *timer; struct rb_node *node; while ((node = rb_first(&old_base->active))) { timer = rb_entry(node, struct hrtimer, node); __remove_hrtimer(timer, old_base); timer->base = new_base; enqueue_hrtimer(timer, new_base); }} static void migrate_hrtimers(int cpu){ struct hrtimer_base *old_base, *new_base; int i; BUG_ON(cpu_online(cpu)); old_base = per_cpu(hrtimer_bases, cpu); new_base = get_cpu_var(hrtimer_bases); local_irq_disable(); for (i = 0; i < MAX_HRTIMER_BASES; i++) { spin_lock(&new_base->lock); spin_lock(&old_base->lock); BUG_ON(old_base->curr_timer); migrate_hrtimer_list(old_base, new_base); spin_unlock(&old_base->lock); spin_unlock(&new_base->lock); old_base++; new_base++; } local_irq_enable(); put_cpu_var(hrtimer_bases);}#endif /* CONFIG_HOTPLUG_CPU */ static int hrtimer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu){ long cpu = (long)hcpu; switch (action) { case CPU_UP_PREPARE: init_hrtimers_cpu(cpu); break; #ifdef CONFIG_HOTPLUG_CPU case CPU_DEAD: migrate_hrtimers(cpu); break;#endif default: break; } return NOTIFY_OK;} static struct notifier_block hrtimers_nb = { .notifier_call = hrtimer_cpu_notify,}; void __init hrtimers_init(void){ hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE, (void *)(long)smp_processor_id()); register_cpu_notifier(&hrtimers_nb);} Lines 501–879 of 879. Previous lines
for (;;) { int ret = hrtimer_try_to_cancel(timer); if (ret >= 0) return ret; cpu_relax(); } } EXPORT_SYMBOL_GPL(hrtimer_cancel); /** * hrtimer_get_remaining - get remaining time for the timer * * @timer: the timer to read */ ktime_t hrtimer_get_remaining(const struct hrtimer *timer) { struct hrtimer_base *base; unsigned long flags; ktime_t rem; base = lock_hrtimer_base(timer, &flags); rem = ktime_sub(timer->expires, timer->base->get_time()); unlock_hrtimer_base(timer, &flags); return rem; } EXPORT_SYMBOL_GPL(hrtimer_get_remaining); #ifdef CONFIG_NO_IDLE_HZ /** * hrtimer_get_next_event - get the time until next expiry event * * Returns the delta to the next expiry event or KTIME_MAX if no timer * is pending. */ ktime_t hrtimer_get_next_event(void) { struct hrtimer_base *base = __get_cpu_var(hrtimer_bases); ktime_t delta, mindelta = { .tv64 = KTIME_MAX }; unsigned long flags; int i; for (i = 0; i < MAX_HRTIMER_BASES; i++, base++) { struct hrtimer *timer; spin_lock_irqsave(&base->lock, flags); if (!base->first) { spin_unlock_irqrestore(&base->lock, flags); continue; } timer = rb_entry(base->first, struct hrtimer, node); delta.tv64 = timer->expires.tv64; spin_unlock_irqrestore(&base->lock, flags); delta = ktime_sub(delta, base->get_time()); if (delta.tv64 < mindelta.tv64) mindelta.tv64 = delta.tv64; } if (mindelta.tv64 < 0) mindelta.tv64 = 0; return mindelta; } #endif /** * hrtimer_init - initialize a timer to the given clock * * @timer: the timer to be initialized * @clock_id: the clock to be used * @mode: timer mode abs/rel */ void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, enum hrtimer_mode mode) { struct hrtimer_base *bases; memset(timer, 0, sizeof(struct hrtimer)); bases = per_cpu(hrtimer_bases, raw_smp_processor_id()); if (clock_id == CLOCK_REALTIME && mode != HRTIMER_ABS) clock_id = CLOCK_MONOTONIC; timer->base = &bases[clock_id]; rb_set_parent(&timer->node, &timer->node); } EXPORT_SYMBOL_GPL(hrtimer_init); /** * hrtimer_get_res - get the timer resolution for a clock * * @which_clock: which clock to query * @tp: pointer to timespec variable to store the resolution * * Store the resolution of the clock selected by which_clock in the * variable pointed to by tp. */ int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp) { struct hrtimer_base *bases; bases = per_cpu(hrtimer_bases, raw_smp_processor_id()); *tp = ktime_to_timespec(bases[which_clock].resolution); return 0; } EXPORT_SYMBOL_GPL(hrtimer_get_res); /* * Expire the per base hrtimer-queue: */ static inline void run_hrtimer_queue(struct hrtimer_base *base) { struct rb_node *node; if (!base->first) return; if (base->get_softirq_time) base->softirq_time = base->get_softirq_time(); spin_lock_irq(&base->lock); while ((node = base->first)) { struct hrtimer *timer; int (*fn)(struct hrtimer *); int restart; timer = rb_entry(node, struct hrtimer, node); if (base->softirq_time.tv64 <= timer->expires.tv64) break; fn = timer->function; set_curr_timer(base, timer); __remove_hrtimer(timer, base); spin_unlock_irq(&base->lock); restart = fn(timer); spin_lock_irq(&base->lock); if (restart != HRTIMER_NORESTART) { BUG_ON(hrtimer_active(timer)); enqueue_hrtimer(timer, base); } } set_curr_timer(base, NULL); spin_unlock_irq(&base->lock); } /* * Called from timer softirq every jiffy, expire hrtimers: */ void hrtimer_run_queues(void) { struct hrtimer_base *base = __get_cpu_var(hrtimer_bases); int i; hrtimer_get_softirq_time(base); for (i = 0; i < MAX_HRTIMER_BASES; i++) run_hrtimer_queue(&base[i]); } /* * Sleep related functions: */ static int hrtimer_wakeup(struct hrtimer *timer) { struct hrtimer_sleeper *t = container_of(timer, struct hrtimer_sleeper, timer); struct task_struct *task = t->task; t->task = NULL; if (task) wake_up_process(task); return HRTIMER_NORESTART; } void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, task_t *task) { sl->timer.function = hrtimer_wakeup; sl->task = task; } static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) { hrtimer_init_sleeper(t, current); do { set_current_state(TASK_INTERRUPTIBLE); hrtimer_start(&t->timer, t->timer.expires, mode); schedule(); hrtimer_cancel(&t->timer); mode = HRTIMER_ABS; } while (t->task && !signal_pending(current)); return t->task == NULL; } static long __sched nanosleep_restart(struct restart_block *restart) { struct hrtimer_sleeper t; struct timespec __user *rmtp; struct timespec tu; ktime_t time; restart->fn = do_no_restart_syscall; hrtimer_init(&t.timer, restart->arg3, HRTIMER_ABS); t.timer.expires.tv64 = ((u64)restart->arg1 << 32) | (u64) restart->arg0; if (do_nanosleep(&t, HRTIMER_ABS)) return 0; rmtp = (struct timespec __user *) restart->arg2; if (rmtp) { time = ktime_sub(t.timer.expires, t.timer.base->get_time()); if (time.tv64 <= 0) return 0; tu = ktime_to_timespec(time); if (copy_to_user(rmtp, &tu, sizeof(tu))) return -EFAULT; } restart->fn = nanosleep_restart; /* The other values in restart are already filled in */ return -ERESTART_RESTARTBLOCK; } long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, const enum hrtimer_mode mode, const clockid_t clockid) { struct restart_block *restart; struct hrtimer_sleeper t; struct timespec tu; ktime_t rem; hrtimer_init(&t.timer, clockid, mode); t.timer.expires = timespec_to_ktime(*rqtp); if (do_nanosleep(&t, mode)) return 0; /* Absolute timers do not update the rmtp value and restart: */ if (mode == HRTIMER_ABS) return -ERESTARTNOHAND; if (rmtp) { rem = ktime_sub(t.timer.expires, t.timer.base->get_time()); if (rem.tv64 <= 0) return 0; tu = ktime_to_timespec(rem); if (copy_to_user(rmtp, &tu, sizeof(tu))) return -EFAULT; } restart = ¤t_thread_info()->restart_block; restart->fn = nanosleep_restart; restart->arg0 = t.timer.expires.tv64 & 0xFFFFFFFF; restart->arg1 = t.timer.expires.tv64 >> 32; restart->arg2 = (unsigned long) rmtp; restart->arg3 = (unsigned long) t.timer.base->index; return -ERESTART_RESTARTBLOCK; } asmlinkage long sys_nanosleep(struct timespec __user *rqtp, struct timespec __user *rmtp) { struct timespec tu; if (copy_from_user(&tu, rqtp, sizeof(tu))) return -EFAULT; if (!timespec_valid(&tu)) return -EINVAL; return hrtimer_nanosleep(&tu, rmtp, HRTIMER_REL, CLOCK_MONOTONIC); } /* * Functions related to boot-time initialization: */ static void __devinit init_hrtimers_cpu(int cpu) { struct hrtimer_base *base = per_cpu(hrtimer_bases, cpu); int i; for (i = 0; i < MAX_HRTIMER_BASES; i++, base++) spin_lock_init(&base->lock); } #ifdef CONFIG_HOTPLUG_CPU static void migrate_hrtimer_list(struct hrtimer_base *old_base, struct hrtimer_base *new_base) { struct hrtimer *timer; struct rb_node *node; while ((node = rb_first(&old_base->active))) { timer = rb_entry(node, struct hrtimer, node); __remove_hrtimer(timer, old_base); timer->base = new_base; enqueue_hrtimer(timer, new_base); } } static void migrate_hrtimers(int cpu) { struct hrtimer_base *old_base, *new_base; int i; BUG_ON(cpu_online(cpu)); old_base = per_cpu(hrtimer_bases, cpu); new_base = get_cpu_var(hrtimer_bases); local_irq_disable(); for (i = 0; i < MAX_HRTIMER_BASES; i++) { spin_lock(&new_base->lock); spin_lock(&old_base->lock); BUG_ON(old_base->curr_timer); migrate_hrtimer_list(old_base, new_base); spin_unlock(&old_base->lock); spin_unlock(&new_base->lock); old_base++; new_base++; } local_irq_enable(); put_cpu_var(hrtimer_bases); } #endif /* CONFIG_HOTPLUG_CPU */ static int hrtimer_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) { long cpu = (long)hcpu; switch (action) { case CPU_UP_PREPARE: init_hrtimers_cpu(cpu); break; #ifdef CONFIG_HOTPLUG_CPU case CPU_DEAD: migrate_hrtimers(cpu); break; #endif default: break; } return NOTIFY_OK; } static struct notifier_block hrtimers_nb = { .notifier_call = hrtimer_cpu_notify, }; void __init hrtimers_init(void) { hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE, (void *)(long)smp_processor_id()); register_cpu_notifier(&hrtimers_nb); }