Following the idle loop model, cleanly check for pending rcuog wakeup before the last rescheduling point upon resuming to guest mode. This way we can avoid to do it from rcu_user_enter() with the last resort self-IPI hack that enforces rescheduling.
Showing the changes to kernel (2 of 4 files). Show all
Unified · Split
| @@ -678,9 +678,10 @@ EXPORT_SYMBOL_GPL(rcu_idle_enter); · 677 unmodified lines | |||
| 678 | 678 | ||
| 679 | #ifdef CONFIG_NO_HZ_FULL | 679 | #ifdef CONFIG_NO_HZ_FULL |
| 680 | 680 | ||
| 681 | #if !defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK) | ||
| 681 | /* | 682 | /* |
| 682 | * An empty function that will trigger a reschedule on | 683 | * An empty function that will trigger a reschedule on |
| 683 | * IRQ tail once IRQs get re-enabled on userspace resume. | 684 | * IRQ tail once IRQs get re-enabled on userspace/guest resume. |
| 684 | */ | 685 | */ |
| 685 | static void late_wakeup_func(struct irq_work *work) | 686 | static void late_wakeup_func(struct irq_work *work) |
| 686 | { | 687 | { |
| @@ -689,6 +690,37 @@ static void late_wakeup_func(struct irq_work *work) · 2 unmodified lines | |||
| 689 | static DEFINE_PER_CPU(struct irq_work, late_wakeup_work) = | 690 | static DEFINE_PER_CPU(struct irq_work, late_wakeup_work) = |
| 690 | IRQ_WORK_INIT(late_wakeup_func); | 691 | IRQ_WORK_INIT(late_wakeup_func); |
| 691 | 692 | ||
| 693 | /* | ||
| 694 | * If either: | ||
| 695 | * | ||
| 696 | * 1) the task is about to enter in guest mode and $ARCH doesn't support KVM generic work | ||
| 697 | * 2) the task is about to enter in user mode and $ARCH doesn't support generic entry. | ||
| 698 | * | ||
| 699 | * In these cases the late RCU wake ups aren't supported in the resched loops and our | ||
| 700 | * last resort is to fire a local irq_work that will trigger a reschedule once IRQs | ||
| 701 | * get re-enabled again. | ||
| 702 | */ | ||
| 703 | noinstr static void rcu_irq_work_resched(void) | ||
| 704 | { | ||
| 705 | struct rcu_data *rdp = this_cpu_ptr(&rcu_data); | ||
| 706 | |||
| 707 | if (IS_ENABLED(CONFIG_GENERIC_ENTRY) && !(current->flags & PF_VCPU)) | ||
| 708 | return; | ||
| 709 | |||
| 710 | if (IS_ENABLED(CONFIG_KVM_XFER_TO_GUEST_WORK) && (current->flags & PF_VCPU)) | ||
| 711 | return; | ||
| 712 | |||
| 713 | instrumentation_begin(); | ||
| 714 | if (do_nocb_deferred_wakeup(rdp) && need_resched()) { | ||
| 715 | irq_work_queue(this_cpu_ptr(&late_wakeup_work)); | ||
| 716 | } | ||
| 717 | instrumentation_end(); | ||
| 718 | } | ||
| 719 | |||
| 720 | #else | ||
| 721 | static inline void rcu_irq_work_resched(void) { } | ||
| 722 | #endif | ||
| 723 | |||
| 692 | /** | 724 | /** |
| 693 | * rcu_user_enter - inform RCU that we are resuming userspace. | 725 | * rcu_user_enter - inform RCU that we are resuming userspace. |
| 694 | * | 726 | * |
| @@ -702,8 +734,6 @@ static DEFINE_PER_CPU(struct irq_work, late_wakeup_work) = · 7 unmodified lines | |||
| 702 | */ | 734 | */ |
| 703 | noinstr void rcu_user_enter(void) | 735 | noinstr void rcu_user_enter(void) |
| 704 | { | 736 | { |
| 705 | struct rcu_data *rdp = this_cpu_ptr(&rcu_data); | ||
| 706 | |||
| 707 | lockdep_assert_irqs_disabled(); | 737 | lockdep_assert_irqs_disabled(); |
| 708 | 738 | ||
| 709 | /* | 739 | /* |
| @@ -711,13 +741,7 @@ noinstr void rcu_user_enter(void) · 1 unmodified line | |||
| 711 | * rescheduling opportunity in the entry code. Trigger a self IPI | 741 | * rescheduling opportunity in the entry code. Trigger a self IPI |
| 712 | * that will fire and reschedule once we resume in user/guest mode. | 742 | * that will fire and reschedule once we resume in user/guest mode. |
| 713 | */ | 743 | */ |
| 714 | instrumentation_begin(); | 744 | rcu_irq_work_resched(); |
| 715 | if (!IS_ENABLED(CONFIG_GENERIC_ENTRY) || (current->flags & PF_VCPU)) { | ||
| 716 | if (do_nocb_deferred_wakeup(rdp) && need_resched()) | ||
| 717 | irq_work_queue(this_cpu_ptr(&late_wakeup_work)); | ||
| 718 | } | ||
| 719 | instrumentation_end(); | ||
| 720 | |||
| 721 | rcu_eqs_enter(true); | 745 | rcu_eqs_enter(true); |
| 722 | } | 746 | } |
| 723 | 747 | ||
| @@ -2197,6 +2197,7 @@ void rcu_nocb_flush_deferred_wakeup(void) · 2196 unmodified lines | |||
| 2197 | { | 2197 | { |
| 2198 | do_nocb_deferred_wakeup(this_cpu_ptr(&rcu_data)); | 2198 | do_nocb_deferred_wakeup(this_cpu_ptr(&rcu_data)); |
| 2199 | } | 2199 | } |
| 2200 | EXPORT_SYMBOL_GPL(rcu_nocb_flush_deferred_wakeup); | ||
| 2200 | 2201 | ||
| 2201 | void __init rcu_init_nohz(void) | 2202 | void __init rcu_init_nohz(void) |
| 2202 | { | 2203 | { |