Checking for and processing RCU-nocb deferred wakeup upon user/guest entry is only relevant when nohz_full runs on the local CPU, otherwise the periodic tick should take care of it. Make sure we don't needlessly pollute these fast-paths as a -3% performance regression on a will-it-scale.per_process_ops has been reported so far.
Unified · Split
| @@ -3,6 +3,7 @@ · 2 unmodified lines | |||
| 3 | #define __LINUX_ENTRYKVM_H | 3 | #define __LINUX_ENTRYKVM_H |
| 4 | 4 | ||
| 5 | #include <linux/entry-common.h> | 5 | #include <linux/entry-common.h> |
| 6 | #include <linux/tick.h> | ||
| 6 | 7 | ||
| 7 | /* Transfer to guest mode work */ | 8 | /* Transfer to guest mode work */ |
| 8 | #ifdef CONFIG_KVM_XFER_TO_GUEST_WORK | 9 | #ifdef CONFIG_KVM_XFER_TO_GUEST_WORK |
| @@ -57,7 +58,7 @@ int xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu); · 48 unmodified lines | |||
| 57 | static inline void xfer_to_guest_mode_prepare(void) | 58 | static inline void xfer_to_guest_mode_prepare(void) |
| 58 | { | 59 | { |
| 59 | lockdep_assert_irqs_disabled(); | 60 | lockdep_assert_irqs_disabled(); |
| 60 | rcu_nocb_flush_deferred_wakeup(); | 61 | tick_nohz_user_enter_prepare(); |
| 61 | } | 62 | } |
| 62 | 63 | ||
| 63 | /** | 64 | /** |
| @@ -11,6 +11,7 @@ · 10 unmodified lines | |||
| 11 | #include <linux/context_tracking_state.h> | 11 | #include <linux/context_tracking_state.h> |
| 12 | #include <linux/cpumask.h> | 12 | #include <linux/cpumask.h> |
| 13 | #include <linux/sched.h> | 13 | #include <linux/sched.h> |
| 14 | #include <linux/rcupdate.h> | ||
| 14 | 15 | ||
| 15 | #ifdef CONFIG_GENERIC_CLOCKEVENTS | 16 | #ifdef CONFIG_GENERIC_CLOCKEVENTS |
| 16 | extern void __init tick_init(void); | 17 | extern void __init tick_init(void); |
| @@ -300,4 +301,10 @@ static inline void tick_nohz_task_switch(void) · 283 unmodified lines | |||
| 300 | __tick_nohz_task_switch(); | 301 | __tick_nohz_task_switch(); |
| 301 | } | 302 | } |
| 302 | 303 | ||
| 304 | static inline void tick_nohz_user_enter_prepare(void) | ||
| 305 | { | ||
| 306 | if (tick_nohz_full_cpu(smp_processor_id())) | ||
| 307 | rcu_nocb_flush_deferred_wakeup(); | ||
| 308 | } | ||
| 309 | |||
| 303 | #endif | 310 | #endif |
| @@ -5,6 +5,7 @@ · 4 unmodified lines | |||
| 5 | #include <linux/highmem.h> | 5 | #include <linux/highmem.h> |
| 6 | #include <linux/livepatch.h> | 6 | #include <linux/livepatch.h> |
| 7 | #include <linux/audit.h> | 7 | #include <linux/audit.h> |
| 8 | #include <linux/tick.h> | ||
| 8 | 9 | ||
| 9 | #include "common.h" | 10 | #include "common.h" |
| 10 | 11 | ||
| @@ -186,7 +187,7 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, · 175 unmodified lines | |||
| 186 | local_irq_disable_exit_to_user(); | 187 | local_irq_disable_exit_to_user(); |
| 187 | 188 | ||
| 188 | /* Check if any of the above work has queued a deferred wakeup */ | 189 | /* Check if any of the above work has queued a deferred wakeup */ |
| 189 | rcu_nocb_flush_deferred_wakeup(); | 190 | tick_nohz_user_enter_prepare(); |
| 190 | 191 | ||
| 191 | ti_work = READ_ONCE(current_thread_info()->flags); | 192 | ti_work = READ_ONCE(current_thread_info()->flags); |
| 192 | } | 193 | } |
| @@ -202,7 +203,7 @@ static void exit_to_user_mode_prepare(struct pt_regs *regs) · 9 unmodified lines | |||
| 202 | lockdep_assert_irqs_disabled(); | 203 | lockdep_assert_irqs_disabled(); |
| 203 | 204 | ||
| 204 | /* Flush pending rcuog wakeup before the last need_resched() check */ | 205 | /* Flush pending rcuog wakeup before the last need_resched() check */ |
| 205 | rcu_nocb_flush_deferred_wakeup(); | 206 | tick_nohz_user_enter_prepare(); |
| 206 | 207 | ||
| 207 | if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK)) | 208 | if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK)) |
| 208 | ti_work = exit_to_user_mode_loop(regs, ti_work); | 209 | ti_work = exit_to_user_mode_loop(regs, ti_work); |
| @@ -230,6 +230,7 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) · 229 unmodified lines | |||
| 230 | 230 | ||
| 231 | #ifdef CONFIG_NO_HZ_FULL | 231 | #ifdef CONFIG_NO_HZ_FULL |
| 232 | cpumask_var_t tick_nohz_full_mask; | 232 | cpumask_var_t tick_nohz_full_mask; |
| 233 | EXPORT_SYMBOL_GPL(tick_nohz_full_mask); | ||
| 233 | bool tick_nohz_full_running; | 234 | bool tick_nohz_full_running; |
| 234 | EXPORT_SYMBOL_GPL(tick_nohz_full_running); | 235 | EXPORT_SYMBOL_GPL(tick_nohz_full_running); |
| 235 | static atomic_t tick_dep_mask; | 236 | static atomic_t tick_dep_mask; |