| @@ -678,9 +678,10 @@ EXPORT_SYMBOL_GPL(rcu_idle_enter); · 677 unmodified lines |
| 678 | 678 | |
| 679 | 679 | #ifdef CONFIG_NO_HZ_FULL |
| 680 | 680 | |
| 681 | +#if !defined(CONFIG_GENERIC_ENTRY) || !defined(CONFIG_KVM_XFER_TO_GUEST_WORK) |
| 681 | 682 | /* |
| 682 | 683 | * An empty function that will trigger a reschedule on |
| 683 | | − * IRQ tail once IRQs get re-enabled on userspace resume. |
| 684 | + * IRQ tail once IRQs get re-enabled on userspace/guest resume. |
| 684 | 685 | */ |
| 685 | 686 | static void late_wakeup_func(struct irq_work *work) |
| 686 | 687 | { |
| @@ -689,6 +690,37 @@ static void late_wakeup_func(struct irq_work *work) · 2 unmodified lines |
| 689 | 690 | static DEFINE_PER_CPU(struct irq_work, late_wakeup_work) = |
| 690 | 691 | IRQ_WORK_INIT(late_wakeup_func); |
| 691 | 692 | |
| 693 | +/* |
| 694 | + * If either: |
| 695 | + * |
| 696 | + * 1) the task is about to enter in guest mode and $ARCH doesn't support KVM generic work |
| 697 | + * 2) the task is about to enter in user mode and $ARCH doesn't support generic entry. |
| 698 | + * |
| 699 | + * In these cases the late RCU wake ups aren't supported in the resched loops and our |
| 700 | + * last resort is to fire a local irq_work that will trigger a reschedule once IRQs |
| 701 | + * get re-enabled again. |
| 702 | + */ |
| 703 | +noinstr static void rcu_irq_work_resched(void) |
| 704 | +{ |
| 705 | + struct rcu_data *rdp = this_cpu_ptr(&rcu_data); |
| 706 | + |
| 707 | + if (IS_ENABLED(CONFIG_GENERIC_ENTRY) && !(current->flags & PF_VCPU)) |
| 708 | + return; |
| 709 | + |
| 710 | + if (IS_ENABLED(CONFIG_KVM_XFER_TO_GUEST_WORK) && (current->flags & PF_VCPU)) |
| 711 | + return; |
| 712 | + |
| 713 | + instrumentation_begin(); |
| 714 | + if (do_nocb_deferred_wakeup(rdp) && need_resched()) { |
| 715 | + irq_work_queue(this_cpu_ptr(&late_wakeup_work)); |
| 716 | + } |
| 717 | + instrumentation_end(); |
| 718 | +} |
| 719 | + |
| 720 | +#else |
| 721 | +static inline void rcu_irq_work_resched(void) { } |
| 722 | +#endif |
| 723 | + |
| 692 | 724 | /** |
| 693 | 725 | * rcu_user_enter - inform RCU that we are resuming userspace. |
| 694 | 726 | * |
| @@ -702,8 +734,6 @@ static DEFINE_PER_CPU(struct irq_work, late_wakeup_work) = · 7 unmodified lines |
| 702 | 734 | */ |
| 703 | 735 | noinstr void rcu_user_enter(void) |
| 704 | 736 | { |
| 705 | | − struct rcu_data *rdp = this_cpu_ptr(&rcu_data); |
| 706 | | − |
| 707 | 737 | lockdep_assert_irqs_disabled(); |
| 708 | 738 | |
| 709 | 739 | /* |
| @@ -711,13 +741,7 @@ noinstr void rcu_user_enter(void) · 1 unmodified line |
| 711 | 741 | * rescheduling opportunity in the entry code. Trigger a self IPI |
| 712 | 742 | * that will fire and reschedule once we resume in user/guest mode. |
| 713 | 743 | */ |
| 714 | | − instrumentation_begin(); |
| 715 | | − if (!IS_ENABLED(CONFIG_GENERIC_ENTRY) || (current->flags & PF_VCPU)) { |
| 716 | | − if (do_nocb_deferred_wakeup(rdp) && need_resched()) |
| 717 | | − irq_work_queue(this_cpu_ptr(&late_wakeup_work)); |
| 718 | | − } |
| 719 | | − instrumentation_end(); |
| 720 | | − |
| 744 | + rcu_irq_work_resched(); |
| 721 | 745 | rcu_eqs_enter(true); |
| 722 | 746 | } |
| 723 | 747 | |