Pull timer fixes from Thomas Gleixner: "A pile of fixes for long standing issues with the timer wheel and the NOHZ code: - Prevent timer base confusion accross the nohz switch, which can cause unlocked access and data corruption - Reinitialize the stale base clock on cpu hotplug to prevent subtle side effects including rollovers on 32bit - Prevent an interrupt storm when the timer softirq is already pending caused by tick_nohz_stop_sched_tick() - Move the timer start tracepoint to a place where it actually makes sense - Add documentation to timerqueue functions as they caused confusion several times now" * 'timers-urgent-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: timerqueue: Document return values of timerqueue_add/del() timers: Invoke timer_start_debug() where it makes sense nohz: Prevent a timer interrupt storm in tick_nohz_stop_sched_tick() timers: Reinitialize per cpu bases on hotplug timers: Use deferrable base independent of base::nohz_active
Unified · Split
| @@ -86,7 +86,7 @@ enum cpuhp_state { · 85 unmodified lines | ||
| 86 | 86 | CPUHP_MM_ZSWP_POOL_PREPARE, |
| 87 | 87 | CPUHP_KVM_PPC_BOOK3S_PREPARE, |
| 88 | 88 | CPUHP_ZCOMP_PREPARE, |
| 89 | − CPUHP_TIMERS_DEAD, | |
| 89 | + CPUHP_TIMERS_PREPARE, | |
| 90 | 90 | CPUHP_MIPS_SOC_PREPARE, |
| 91 | 91 | CPUHP_BP_PREPARE_DYN, |
| 92 | 92 | CPUHP_BP_PREPARE_DYN_END = CPUHP_BP_PREPARE_DYN + 20, |
| @@ -207,9 +207,11 @@ unsigned long round_jiffies_up(unsigned long j); · 206 unmodified lines | ||
| 207 | 207 | unsigned long round_jiffies_up_relative(unsigned long j); |
| 208 | 208 | |
| 209 | 209 | #ifdef CONFIG_HOTPLUG_CPU |
| 210 | +int timers_prepare_cpu(unsigned int cpu); | |
| 210 | 211 | int timers_dead_cpu(unsigned int cpu); |
| 211 | 212 | #else |
| 212 | −#define timers_dead_cpu NULL | |
| 213 | +#define timers_prepare_cpu NULL | |
| 214 | +#define timers_dead_cpu NULL | |
| 213 | 215 | #endif |
| 214 | 216 | |
| 215 | 217 | #endif |
| @@ -1277,9 +1277,9 @@ static struct cpuhp_step cpuhp_bp_states[] = { · 1276 unmodified lines | ||
| 1277 | 1277 | * before blk_mq_queue_reinit_notify() from notify_dead(), |
| 1278 | 1278 | * otherwise a RCU stall occurs. |
| 1279 | 1279 | */ |
| 1280 | − [CPUHP_TIMERS_DEAD] = { | |
| 1280 | + [CPUHP_TIMERS_PREPARE] = { | |
| 1281 | 1281 | .name = "timers:dead", |
| 1282 | − .startup.single = NULL, | |
| 1282 | + .startup.single = timers_prepare_cpu, | |
| 1283 | 1283 | .teardown.single = timers_dead_cpu, |
| 1284 | 1284 | }, |
| 1285 | 1285 | /* Kicks the plugged cpu into life */ |
| @@ -650,6 +650,11 @@ static void tick_nohz_restart(struct tick_sched *ts, ktime_t now) · 649 unmodified lines | ||
| 650 | 650 | ts->next_tick = 0; |
| 651 | 651 | } |
| 652 | 652 | |
| 653 | +static inline bool local_timer_softirq_pending(void) | |
| 654 | +{ | |
| 655 | + return local_softirq_pending() & TIMER_SOFTIRQ; | |
| 656 | +} | |
| 657 | + | |
| 653 | 658 | static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, |
| 654 | 659 | ktime_t now, int cpu) |
| 655 | 660 | { |
| @@ -666,8 +671,18 @@ static ktime_t tick_nohz_stop_sched_tick(struct tick_sched *ts, · 10 unmodified lines | ||
| 666 | 671 | } while (read_seqretry(&jiffies_lock, seq)); |
| 667 | 672 | ts->last_jiffies = basejiff; |
| 668 | 673 | |
| 669 | − if (rcu_needs_cpu(basemono, &next_rcu) || | |
| 670 | − arch_needs_cpu() || irq_work_needs_cpu()) { | |
| 674 | + /* | |
| 675 | + * Keep the periodic tick, when RCU, architecture or irq_work | |
| 676 | + * requests it. | |
| 677 | + * Aside of that check whether the local timer softirq is | |
| 678 | + * pending. If so its a bad idea to call get_next_timer_interrupt() | |
| 679 | + * because there is an already expired timer, so it will request | |
| 680 | + * immeditate expiry, which rearms the hardware timer with a | |
| 681 | + * minimal delta which brings us back to this place | |
| 682 | + * immediately. Lather, rinse and repeat... | |
| 683 | + */ | |
| 684 | + if (rcu_needs_cpu(basemono, &next_rcu) || arch_needs_cpu() || | |
| 685 | + irq_work_needs_cpu() || local_timer_softirq_pending()) { | |
| 671 | 686 | next_tick = basemono + TICK_NSEC; |
| 672 | 687 | } else { |
| 673 | 688 | /* |
| @@ -823,11 +823,10 @@ static inline struct timer_base *get_timer_cpu_base(u32 tflags, u32 cpu) · 822 unmodified lines | ||
| 823 | 823 | struct timer_base *base = per_cpu_ptr(&timer_bases[BASE_STD], cpu); |
| 824 | 824 | |
| 825 | 825 | /* |
| 826 | − * If the timer is deferrable and nohz is active then we need to use | |
| 827 | − * the deferrable base. | |
| 826 | + * If the timer is deferrable and NO_HZ_COMMON is set then we need | |
| 827 | + * to use the deferrable base. | |
| 828 | 828 | */ |
| 829 | − if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && base->nohz_active && | |
| 830 | − (tflags & TIMER_DEFERRABLE)) | |
| 829 | + if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE)) | |
| 831 | 830 | base = per_cpu_ptr(&timer_bases[BASE_DEF], cpu); |
| 832 | 831 | return base; |
| 833 | 832 | } |
| @@ -837,11 +836,10 @@ static inline struct timer_base *get_timer_this_cpu_base(u32 tflags) · 3 unmodified lines | ||
| 837 | 836 | struct timer_base *base = this_cpu_ptr(&timer_bases[BASE_STD]); |
| 838 | 837 | |
| 839 | 838 | /* |
| 840 | − * If the timer is deferrable and nohz is active then we need to use | |
| 841 | − * the deferrable base. | |
| 839 | + * If the timer is deferrable and NO_HZ_COMMON is set then we need | |
| 840 | + * to use the deferrable base. | |
| 842 | 841 | */ |
| 843 | − if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && base->nohz_active && | |
| 844 | − (tflags & TIMER_DEFERRABLE)) | |
| 842 | + if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && (tflags & TIMER_DEFERRABLE)) | |
| 845 | 843 | base = this_cpu_ptr(&timer_bases[BASE_DEF]); |
| 846 | 844 | return base; |
| 847 | 845 | } |
| @@ -1009,8 +1007,6 @@ __mod_timer(struct timer_list *timer, unsigned long expires, unsigned int option · 161 unmodified lines | ||
| 1009 | 1007 | if (!ret && (options & MOD_TIMER_PENDING_ONLY)) |
| 1010 | 1008 | goto out_unlock; |
| 1011 | 1009 | |
| 1012 | − debug_activate(timer, expires); | |
| 1013 | − | |
| 1014 | 1010 | new_base = get_target_base(base, timer->flags); |
| 1015 | 1011 | |
| 1016 | 1012 | if (base != new_base) { |
| @@ -1034,6 +1030,8 @@ __mod_timer(struct timer_list *timer, unsigned long expires, unsigned int option · 17 unmodified lines | ||
| 1034 | 1030 | } |
| 1035 | 1031 | } |
| 1036 | 1032 | |
| 1033 | + debug_activate(timer, expires); | |
| 1034 | + | |
| 1037 | 1035 | timer->expires = expires; |
| 1038 | 1036 | /* |
| 1039 | 1037 | * If 'idx' was calculated above and the base time did not advance |
| @@ -1684,7 +1682,7 @@ static __latent_entropy void run_timer_softirq(struct softirq_action *h) · 644 unmodified lines | ||
| 1684 | 1682 | base->must_forward_clk = false; |
| 1685 | 1683 | |
| 1686 | 1684 | __run_timers(base); |
| 1687 | − if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && base->nohz_active) | |
| 1685 | + if (IS_ENABLED(CONFIG_NO_HZ_COMMON)) | |
| 1688 | 1686 | __run_timers(this_cpu_ptr(&timer_bases[BASE_DEF])); |
| 1689 | 1687 | } |
| 1690 | 1688 | |
| @@ -1855,6 +1853,21 @@ static void migrate_timer_list(struct timer_base *new_base, struct hlist_head *h · 164 unmodified lines | ||
| 1855 | 1853 | } |
| 1856 | 1854 | } |
| 1857 | 1855 | |
| 1856 | +int timers_prepare_cpu(unsigned int cpu) | |
| 1857 | +{ | |
| 1858 | + struct timer_base *base; | |
| 1859 | + int b; | |
| 1860 | + | |
| 1861 | + for (b = 0; b < NR_BASES; b++) { | |
| 1862 | + base = per_cpu_ptr(&timer_bases[b], cpu); | |
| 1863 | + base->clk = jiffies; | |
| 1864 | + base->next_expiry = base->clk + NEXT_TIMER_MAX_DELTA; | |
| 1865 | + base->is_idle = false; | |
| 1866 | + base->must_forward_clk = true; | |
| 1867 | + } | |
| 1868 | + return 0; | |
| 1869 | +} | |
| 1870 | + | |
| 1858 | 1871 | int timers_dead_cpu(unsigned int cpu) |
| 1859 | 1872 | { |
| 1860 | 1873 | struct timer_base *old_base; |
| @@ -33,8 +33,9 @@ · 32 unmodified lines | ||
| 33 | 33 | * @head: head of timerqueue |
| 34 | 34 | * @node: timer node to be added |
| 35 | 35 | * |
| 36 | − * Adds the timer node to the timerqueue, sorted by the | |
| 37 | − * node's expires value. | |
| 36 | + * Adds the timer node to the timerqueue, sorted by the node's expires | |
| 37 | + * value. Returns true if the newly added timer is the first expiring timer in | |
| 38 | + * the queue. | |
| 38 | 39 | */ |
| 39 | 40 | bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node) |
| 40 | 41 | { |
| @@ -70,7 +71,8 @@ EXPORT_SYMBOL_GPL(timerqueue_add); · 29 unmodified lines | ||
| 70 | 71 | * @head: head of timerqueue |
| 71 | 72 | * @node: timer node to be removed |
| 72 | 73 | * |
| 73 | − * Removes the timer node from the timerqueue. | |
| 74 | + * Removes the timer node from the timerqueue. Returns true if the queue is | |
| 75 | + * not empty after the remove. | |
| 74 | 76 | */ |
| 75 | 77 | bool timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node) |
| 76 | 78 | { |