*/unsigned long round_jiffies_up_relative(unsigned long j){	return __round_jiffies_up_relative(j, raw_smp_processor_id());}EXPORT_SYMBOL_GPL(round_jiffies_up_relative);  static inline unsigned int timer_get_idx(struct timer_list *timer){	return (timer->flags & TIMER_ARRAYMASK) >> TIMER_ARRAYSHIFT;} static inline void timer_set_idx(struct timer_list *timer, unsigned int idx){	timer->flags = (timer->flags & ~TIMER_ARRAYMASK) |			idx << TIMER_ARRAYSHIFT;} /* * Helper function to calculate the array index for a given expiry * time. */static inline unsigned calc_index(unsigned long expires, unsigned lvl,				  unsigned long *bucket_expiry){ 	/*	 * The timer wheel has to guarantee that a timer does not fire	 * early. Early expiry can happen due to:	 * - Timer is armed at the edge of a tick	 * - Truncation of the expiry time in the outer wheel levels	 *	 * Round up with level granularity to prevent this.	 */	expires = (expires >> LVL_SHIFT(lvl)) + 1;	*bucket_expiry = expires << LVL_SHIFT(lvl);	return LVL_OFFS(lvl) + (expires & LVL_MASK);} static int calc_wheel_index(unsigned long expires, unsigned long clk,			    unsigned long *bucket_expiry){	unsigned long delta = expires - clk;	unsigned int idx; 	if (delta < LVL_START(1)) {		idx = calc_index(expires, 0, bucket_expiry);	} else if (delta < LVL_START(2)) {		idx = calc_index(expires, 1, bucket_expiry);	} else if (delta < LVL_START(3)) {		idx = calc_index(expires, 2, bucket_expiry);	} else if (delta < LVL_START(4)) {		idx = calc_index(expires, 3, bucket_expiry);	} else if (delta < LVL_START(5)) {		idx = calc_index(expires, 4, bucket_expiry);	} else if (delta < LVL_START(6)) {		idx = calc_index(expires, 5, bucket_expiry);	} else if (delta < LVL_START(7)) {		idx = calc_index(expires, 6, bucket_expiry);	} else if (LVL_DEPTH > 8 && delta < LVL_START(8)) {		idx = calc_index(expires, 7, bucket_expiry);	} else if ((long) delta < 0) {		idx = clk & LVL_MASK;		*bucket_expiry = clk;	} else {		/*		 * Force expire obscene large timeouts to expire at the		 * capacity limit of the wheel.		 */		if (delta >= WHEEL_TIMEOUT_CUTOFF)			expires = clk + WHEEL_TIMEOUT_MAX; 		idx = calc_index(expires, LVL_DEPTH - 1, bucket_expiry);	}	return idx;} static voidtrigger_dyntick_cpu(struct timer_base *base, struct timer_list *timer){	/*	 * Deferrable timers do not prevent the CPU from entering dynticks and	 * are not taken into account on the idle/nohz_full path. An IPI when a	 * new deferrable timer is enqueued will wake up the remote CPU but	 * nothing will be done with the deferrable timer base. Therefore skip	 * the remote IPI for deferrable timers completely.	 */	if (!is_timers_nohz_active() || timer->flags & TIMER_DEFERRABLE)		return; 	/*	 * We might have to IPI the remote CPU if the base is idle and the	 * timer is pinned. If it is a non pinned timer, it is only queued	 * on the remote CPU, when timer was running during queueing. Then	 * everything is handled by remote CPU anyway. If the other CPU is	 * on the way to idle then it can't set base->is_idle as we hold	 * the base lock:	 */	if (base->is_idle) {		WARN_ON_ONCE(!(timer->flags & TIMER_PINNED ||			       tick_nohz_full_cpu(base->cpu)));		wake_up_nohz_cpu(base->cpu);	}} /* * Enqueue the timer into the hash bucket, mark it pending in * the bitmap, store the index in the timer flags then wake up * the target CPU if needed. */static void enqueue_timer(struct timer_base *base, struct timer_list *timer,			  unsigned int idx, unsigned long bucket_expiry){ 	hlist_add_head(&timer->entry, base->vectors + idx);	__set_bit(idx, base->pending_map);	timer_set_idx(timer, idx); 	trace_timer_start(timer, bucket_expiry); 	/*	 * Check whether this is the new first expiring timer. The	 * effective expiry time of the timer is required here	 * (bucket_expiry) instead of timer->expires.	 */	if (time_before(bucket_expiry, base->next_expiry)) {		/*		 * Set the next expiry time and kick the CPU so it		 * can reevaluate the wheel:		 */		WRITE_ONCE(base->next_expiry, bucket_expiry);		base->timers_pending = true;		base->next_expiry_recalc = false;		trigger_dyntick_cpu(base, timer);	}} static void internal_add_timer(struct timer_base *base, struct timer_list *timer){	unsigned long bucket_expiry;	unsigned int idx; 	idx = calc_wheel_index(timer->expires, base->clk, &bucket_expiry);	enqueue_timer(base, timer, idx, bucket_expiry);} #ifdef CONFIG_DEBUG_OBJECTS_TIMERS static const struct debug_obj_descr timer_debug_descr; struct timer_hint {	void	(*function)(struct timer_list *t);	long	offset;}; #define TIMER_HINT(fn, container, timr, hintfn)			\	{							\		.function = fn,					\		.offset	  = offsetof(container, hintfn) -	\			    offsetof(container, timr)		\	} static const struct timer_hint timer_hints[] = {	TIMER_HINT(delayed_work_timer_fn,		   struct delayed_work, timer, work.func),	TIMER_HINT(kthread_delayed_work_timer_fn,		   struct kthread_delayed_work, timer, work.func),}; static void *timer_debug_hint(void *addr){	struct timer_list *timer = addr;	int i; 	for (i = 0; i < ARRAY_SIZE(timer_hints); i++) {		if (timer_hints[i].function == timer->function) {			void (**fn)(void) = addr + timer_hints[i].offset; 			return *fn;		}	} 	return timer->function;} static bool timer_is_static_object(void *addr){	struct timer_list *timer = addr; 	return (timer->entry.pprev == NULL &&		timer->entry.next == TIMER_ENTRY_STATIC);} /* * timer_fixup_init is called when: * - an active object is initialized */static bool timer_fixup_init(void *addr, enum debug_obj_state state){	struct timer_list *timer = addr; 	switch (state) {	case ODEBUG_STATE_ACTIVE:		timer_delete_sync(timer);		debug_object_init(timer, &timer_debug_descr);		return true;	default:		return false;	}} /* Stub timer callback for improperly used timers. */static void stub_timer(struct timer_list *unused){	WARN_ON(1);} /* * timer_fixup_activate is called when: * - an active object is activated * - an unknown non-static object is activated */static bool timer_fixup_activate(void *addr, enum debug_obj_state state){	struct timer_list *timer = addr; 	switch (state) {	case ODEBUG_STATE_NOTAVAILABLE:		timer_setup(timer, stub_timer, 0);		return true; 	case ODEBUG_STATE_ACTIVE:		WARN_ON(1);		fallthrough;	default:		return false;	}} /* * timer_fixup_free is called when: * - an active object is freed */static bool timer_fixup_free(void *addr, enum debug_obj_state state){	struct timer_list *timer = addr; 	switch (state) {	case ODEBUG_STATE_ACTIVE:		timer_delete_sync(timer);		debug_object_free(timer, &timer_debug_descr);		return true;	default:		return false;	}} /* * timer_fixup_assert_init is called when: * - an untracked/uninit-ed object is found */static bool timer_fixup_assert_init(void *addr, enum debug_obj_state state){	struct timer_list *timer = addr; 	switch (state) {	case ODEBUG_STATE_NOTAVAILABLE:		timer_setup(timer, stub_timer, 0);		return true;	default:		return false;	}} static const struct debug_obj_descr timer_debug_descr = {	.name			= "timer_list",	.debug_hint		= timer_debug_hint,	.is_static_object	= timer_is_static_object,	.fixup_init		= timer_fixup_init,	.fixup_activate		= timer_fixup_activate,	.fixup_free		= timer_fixup_free,	.fixup_assert_init	= timer_fixup_assert_init,}; static inline void debug_timer_init(struct timer_list *timer){	debug_object_init(timer, &timer_debug_descr);} static inline void debug_timer_activate(struct timer_list *timer){	debug_object_activate(timer, &timer_debug_descr);} static inline void debug_timer_deactivate(struct timer_list *timer){	debug_object_deactivate(timer, &timer_debug_descr);} static inline void debug_timer_assert_init(struct timer_list *timer){	debug_object_assert_init(timer, &timer_debug_descr);} static void do_init_timer(struct timer_list *timer,			  void (*func)(struct timer_list *),			  unsigned int flags,			  const char *name, struct lock_class_key *key); void timer_init_key_on_stack(struct timer_list *timer,			     void (*func)(struct timer_list *),			     unsigned int flags,			     const char *name, struct lock_class_key *key){	debug_object_init_on_stack(timer, &timer_debug_descr);	do_init_timer(timer, func, flags, name, key);}EXPORT_SYMBOL_GPL(timer_init_key_on_stack); void timer_destroy_on_stack(struct timer_list *timer){	debug_object_free(timer, &timer_debug_descr);}EXPORT_SYMBOL_GPL(timer_destroy_on_stack); #elsestatic inline void debug_timer_init(struct timer_list *timer) { }static inline void debug_timer_activate(struct timer_list *timer) { }static inline void debug_timer_deactivate(struct timer_list *timer) { }static inline void debug_timer_assert_init(struct timer_list *timer) { }#endif static inline void debug_init(struct timer_list *timer){	debug_timer_init(timer);	trace_timer_init(timer);} static inline void debug_deactivate(struct timer_list *timer){	debug_timer_deactivate(timer);	trace_timer_cancel(timer);} static inline void debug_assert_init(struct timer_list *timer){	debug_timer_assert_init(timer);} static void do_init_timer(struct timer_list *timer,			  void (*func)(struct timer_list *),			  unsigned int flags,			  const char *name, struct lock_class_key *key){	timer->entry.pprev = NULL;	timer->function = func;	if (WARN_ON_ONCE(flags & ~TIMER_INIT_FLAGS))		flags &= TIMER_INIT_FLAGS;	timer->flags = flags | raw_smp_processor_id();	lockdep_init_map(&timer->lockdep_map, name, key, 0);} /** * timer_init_key - initialize a timer * @timer: the timer to be initialized * @func: timer callback function * @flags: timer flags * @name: name of the timer * @key: lockdep class key of the fake lock used for tracking timer *       sync lock dependencies * * timer_init_key() must be done to a timer prior to calling *any* of the * other timer functions. */void timer_init_key(struct timer_list *timer,		    void (*func)(struct timer_list *), unsigned int flags,		    const char *name, struct lock_class_key *key){	debug_init(timer);	do_init_timer(timer, func, flags, name, key);}EXPORT_SYMBOL(timer_init_key); static inline void detach_timer(struct timer_list *timer, bool clear_pending){	struct hlist_node *entry = &timer->entry; 	debug_deactivate(timer); 	__hlist_del(entry);	if (clear_pending)		entry->pprev = NULL;	entry->next = LIST_POISON2;} static int detach_if_pending(struct timer_list *timer, struct timer_base *base,			     bool clear_pending){	unsigned idx = timer_get_idx(timer); 	if (!timer_pending(timer))		return 0; 	if (hlist_is_singular_node(&timer->entry, base->vectors + idx)) {		__clear_bit(idx, base->pending_map);		base->next_expiry_recalc = true;	} 	detach_timer(timer, clear_pending);	return 1;} static inline struct timer_base *get_timer_cpu_base(u32 tflags, u32 cpu){	int index = tflags & TIMER_PINNED ? BASE_LOCAL : BASE_GLOBAL; 	/*	 * If the timer is deferrable and NO_HZ_COMMON is set then we need	 * to use the deferrable base.	 */	if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE))		index = BASE_DEF; 	return per_cpu_ptr(&timer_bases[index], cpu);} static inline struct timer_base *get_timer_this_cpu_base(u32 tflags){	int index = tflags & TIMER_PINNED ? BASE_LOCAL : BASE_GLOBAL; 	/*	 * If the timer is deferrable and NO_HZ_COMMON is set then we need	 * to use the deferrable base.	 */	if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE))		index = BASE_DEF; 	return this_cpu_ptr(&timer_bases[index]);} static inline struct timer_base *get_timer_base(u32 tflags){	return get_timer_cpu_base(tflags, tflags & TIMER_CPUMASK);} static inline void __forward_timer_base(struct timer_base *base,					unsigned long basej){	/*	 * Check whether we can forward the base. We can only do that when	 * @basej is past base->clk otherwise we might rewind base->clk.	 */	if (time_before_eq(basej, base->clk))		return; 	/*	 * If the next expiry value is > jiffies, then we fast forward to	 * jiffies otherwise we forward to the next expiry value.	 */	if (time_after(base->next_expiry, basej)) {		base->clk = basej;	} else {		if (WARN_ON_ONCE(time_before(base->next_expiry, base->clk)))			return;		base->clk = base->next_expiry;	} } static inline void forward_timer_base(struct timer_base *base){	__forward_timer_base(base, READ_ONCE(jiffies));} /* * We are using hashed locking: Holding per_cpu(timer_bases[x]).lock means * that all timers which are tied to this base are locked, and the base itself * is locked too. * * So __run_timers/migrate_timers can safely modify all timers which could * be found in the base->vectors array. * * When a timer is migrating then the TIMER_MIGRATING flag is set and we need * to wait until the migration is done. */static struct timer_base *lock_timer_base(struct timer_list *timer,					  unsigned long *flags)	__acquires(timer->base->lock){	for (;;) {		struct timer_base *base;		u32 tf; 		/*		 * We need to use READ_ONCE() here, otherwise the compiler		 * might re-read @tf between the check for TIMER_MIGRATING		 * and spin_lock().		 */		tf = READ_ONCE(timer->flags);

Lines 501–1000 of 2580. Previous lines Load more