Checking for and processing RCU-nocb deferred wakeup upon user/guest entry is only relevant when nohz_full runs on the local CPU, otherwise the periodic tick should take care of it. Make sure we don't needlessly pollute these fast-paths as a -3% performance regression on a will-it-scale.per_process_ops has been reported so far.
Unified · Split
| @@ -3,6 +3,7 @@ · 2 unmodified lines | ||
| 3 | 3 | #define __LINUX_ENTRYKVM_H |
| 4 | 4 | |
| 5 | 5 | #include <linux/entry-common.h> |
| 6 | +#include <linux/tick.h> | |
| 6 | 7 | |
| 7 | 8 | /* Transfer to guest mode work */ |
| 8 | 9 | #ifdef CONFIG_KVM_XFER_TO_GUEST_WORK |
| @@ -57,7 +58,7 @@ int xfer_to_guest_mode_handle_work(struct kvm_vcpu *vcpu); · 48 unmodified lines | ||
| 57 | 58 | static inline void xfer_to_guest_mode_prepare(void) |
| 58 | 59 | { |
| 59 | 60 | lockdep_assert_irqs_disabled(); |
| 60 | − rcu_nocb_flush_deferred_wakeup(); | |
| 61 | + tick_nohz_user_enter_prepare(); | |
| 61 | 62 | } |
| 62 | 63 | |
| 63 | 64 | /** |
| @@ -11,6 +11,7 @@ · 10 unmodified lines | ||
| 11 | 11 | #include <linux/context_tracking_state.h> |
| 12 | 12 | #include <linux/cpumask.h> |
| 13 | 13 | #include <linux/sched.h> |
| 14 | +#include <linux/rcupdate.h> | |
| 14 | 15 | |
| 15 | 16 | #ifdef CONFIG_GENERIC_CLOCKEVENTS |
| 16 | 17 | extern void __init tick_init(void); |
| @@ -300,4 +301,10 @@ static inline void tick_nohz_task_switch(void) · 283 unmodified lines | ||
| 300 | 301 | __tick_nohz_task_switch(); |
| 301 | 302 | } |
| 302 | 303 | |
| 304 | +static inline void tick_nohz_user_enter_prepare(void) | |
| 305 | +{ | |
| 306 | + if (tick_nohz_full_cpu(smp_processor_id())) | |
| 307 | + rcu_nocb_flush_deferred_wakeup(); | |
| 308 | +} | |
| 309 | + | |
| 303 | 310 | #endif |
| @@ -5,6 +5,7 @@ · 4 unmodified lines | ||
| 5 | 5 | #include <linux/highmem.h> |
| 6 | 6 | #include <linux/livepatch.h> |
| 7 | 7 | #include <linux/audit.h> |
| 8 | +#include <linux/tick.h> | |
| 8 | 9 | |
| 9 | 10 | #include "common.h" |
| 10 | 11 | |
| @@ -186,7 +187,7 @@ static unsigned long exit_to_user_mode_loop(struct pt_regs *regs, · 175 unmodified lines | ||
| 186 | 187 | local_irq_disable_exit_to_user(); |
| 187 | 188 | |
| 188 | 189 | /* Check if any of the above work has queued a deferred wakeup */ |
| 189 | − rcu_nocb_flush_deferred_wakeup(); | |
| 190 | + tick_nohz_user_enter_prepare(); | |
| 190 | 191 | |
| 191 | 192 | ti_work = READ_ONCE(current_thread_info()->flags); |
| 192 | 193 | } |
| @@ -202,7 +203,7 @@ static void exit_to_user_mode_prepare(struct pt_regs *regs) · 9 unmodified lines | ||
| 202 | 203 | lockdep_assert_irqs_disabled(); |
| 203 | 204 | |
| 204 | 205 | /* Flush pending rcuog wakeup before the last need_resched() check */ |
| 205 | − rcu_nocb_flush_deferred_wakeup(); | |
| 206 | + tick_nohz_user_enter_prepare(); | |
| 206 | 207 | |
| 207 | 208 | if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK)) |
| 208 | 209 | ti_work = exit_to_user_mode_loop(regs, ti_work); |
| @@ -230,6 +230,7 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) · 229 unmodified lines | ||
| 230 | 230 | |
| 231 | 231 | #ifdef CONFIG_NO_HZ_FULL |
| 232 | 232 | cpumask_var_t tick_nohz_full_mask; |
| 233 | +EXPORT_SYMBOL_GPL(tick_nohz_full_mask); | |
| 233 | 234 | bool tick_nohz_full_running; |
| 234 | 235 | EXPORT_SYMBOL_GPL(tick_nohz_full_running); |
| 235 | 236 | static atomic_t tick_dep_mask; |