History

kvm: Replace vcpu->swait with rcuwait

The use of any sort of waitqueue (simple or regular) for wait/waking vcpus has always been an overkill and semantically wrong. Because this is per-vcpu (which is blocked) there is only ever a single waiting vcpu, thus no need for any sort of queue. As such, make use of the rcuwait primitive, with the following considerations: - rcuwait already provides the proper barriers that serialize concurrent waiter and waker. - Task wakeup is done in rcu read critical region, with a stable task pointer. - Because there is no concurrency among waiters, we need not worry about rcuwait_wait_event() calls corrupting the wait->task. As a consequence, this saves the locking done in swait when modifying the queue. This also applies to per-vcore wait for powerpc kvm-hv. The x86 tscdeadline_latency test mentioned in 8577370fb0cb ("KVM: Use simple waitqueue for vcpu->wq") shows that, on avg, latency is reduced by around 15-20% with this change.

Davidlohr Bueso committed 6 years ago

Commit da4ad881 parent 191a43b11 files · +38 −43Browse files

Unified · Split

11 files
  1. arch/mips/kvm/mips.c +2 −4
  2. arch/powerpc/include/asm/kvm_book3s.h +1 −1
  3. arch/powerpc/include/asm/kvm_host.h +1 −1
  4. arch/powerpc/kvm/book3s_hv.c +10 −13
  5. arch/powerpc/kvm/powerpc.c +1 −1
  6. arch/x86/kvm/lapic.c +1 −1
  7. include/linux/kvm_host.h +5 −5
  8. virt/kvm/arm/arch_timer.c +2 −1
  9. virt/kvm/arm/arm.c +5 −4
  10. virt/kvm/async_pf.c +1 −2
  11. virt/kvm/kvm_main.c +9 −10
arch/mips/kvm/mips.c+2−4
@@ -283,8 +283,7 @@ static enum hrtimer_restart kvm_mips_comparecount_wakeup(struct hrtimer *timer) · 282 unmodified lines
283 kvm_mips_callbacks->queue_timer_int(vcpu);283 kvm_mips_callbacks->queue_timer_int(vcpu);
284 284
285 vcpu->arch.wait = 0;285 vcpu->arch.wait = 0;
286 if (swq_has_sleeper(&vcpu->wq))286 rcuwait_wake_up(&vcpu->wait);
287 swake_up_one(&vcpu->wq);
288 287
289 return kvm_mips_count_timeout(vcpu);288 return kvm_mips_count_timeout(vcpu);
290}289}
@@ -511,8 +510,7 @@ int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, · 220 unmodified lines
511 510
512 dvcpu->arch.wait = 0;511 dvcpu->arch.wait = 0;
513 512
514 if (swq_has_sleeper(&dvcpu->wq))513 rcuwait_wake_up(&dvcpu->wait);
515 swake_up_one(&dvcpu->wq);
516 514
517 return 0;515 return 0;
518}516}
arch/powerpc/include/asm/kvm_book3s.h+1−1
@@ -78,7 +78,7 @@ struct kvmppc_vcore { · 77 unmodified lines
78 struct kvm_vcpu *runnable_threads[MAX_SMT_THREADS];78 struct kvm_vcpu *runnable_threads[MAX_SMT_THREADS];
79 struct list_head preempt_list;79 struct list_head preempt_list;
80 spinlock_t lock;80 spinlock_t lock;
81 struct swait_queue_head wq;81 struct rcuwait wait;
82 spinlock_t stoltb_lock; /* protects stolen_tb and preempt_tb */82 spinlock_t stoltb_lock; /* protects stolen_tb and preempt_tb */
83 u64 stolen_tb;83 u64 stolen_tb;
84 u64 preempt_tb;84 u64 preempt_tb;
arch/powerpc/include/asm/kvm_host.h+1−1
@@ -751,7 +751,7 @@ struct kvm_vcpu_arch { · 750 unmodified lines
751 u8 irq_pending; /* Used by XIVE to signal pending guest irqs */751 u8 irq_pending; /* Used by XIVE to signal pending guest irqs */
752 u32 last_inst;752 u32 last_inst;
753 753
754 struct swait_queue_head *wqp;754 struct rcuwait *waitp;
755 struct kvmppc_vcore *vcore;755 struct kvmppc_vcore *vcore;
756 int ret;756 int ret;
757 int trap;757 int trap;
arch/powerpc/kvm/book3s_hv.c+10−13
@@ -230,13 +230,11 @@ static bool kvmppc_ipi_thread(int cpu) · 229 unmodified lines
230static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)230static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
231{231{
232 int cpu;232 int cpu;
233 struct swait_queue_head *wqp;233 struct rcuwait *waitp;
234 234
235 wqp = kvm_arch_vcpu_wq(vcpu);235 waitp = kvm_arch_vcpu_get_wait(vcpu);
236 if (swq_has_sleeper(wqp)) {236 if (rcuwait_wake_up(waitp))
237 swake_up_one(wqp);
238 ++vcpu->stat.halt_wakeup;237 ++vcpu->stat.halt_wakeup;
239 }
240 238
241 cpu = READ_ONCE(vcpu->arch.thread_cpu);239 cpu = READ_ONCE(vcpu->arch.thread_cpu);
242 if (cpu >= 0 && kvmppc_ipi_thread(cpu))240 if (cpu >= 0 && kvmppc_ipi_thread(cpu))
@@ -2125,7 +2123,7 @@ static struct kvmppc_vcore *kvmppc_vcore_create(struct kvm *kvm, int id) · 1882 unmodified lines
2125 2123
2126 spin_lock_init(&vcore->lock);2124 spin_lock_init(&vcore->lock);
2127 spin_lock_init(&vcore->stoltb_lock);2125 spin_lock_init(&vcore->stoltb_lock);
2128 init_swait_queue_head(&vcore->wq);2126 rcuwait_init(&vcore->wait);
2129 vcore->preempt_tb = TB_NIL;2127 vcore->preempt_tb = TB_NIL;
2130 vcore->lpcr = kvm->arch.lpcr;2128 vcore->lpcr = kvm->arch.lpcr;
2131 vcore->first_vcpuid = id;2129 vcore->first_vcpuid = id;
@@ -3784,7 +3782,6 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) · 1652 unmodified lines
3784 ktime_t cur, start_poll, start_wait;3782 ktime_t cur, start_poll, start_wait;
3785 int do_sleep = 1;3783 int do_sleep = 1;
3786 u64 block_ns;3784 u64 block_ns;
3787 DECLARE_SWAITQUEUE(wait);
3788 3785
3789 /* Poll for pending exceptions and ceded state */3786 /* Poll for pending exceptions and ceded state */
3790 cur = start_poll = ktime_get();3787 cur = start_poll = ktime_get();
@@ -3812,10 +3809,10 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) · 21 unmodified lines
3812 }3809 }
3813 }3810 }
3814 3811
3815 prepare_to_swait_exclusive(&vc->wq, &wait, TASK_INTERRUPTIBLE);3812 prepare_to_rcuwait(&vc->wait);
3816 3813 set_current_state(TASK_INTERRUPTIBLE);
3817 if (kvmppc_vcore_check_block(vc)) {3814 if (kvmppc_vcore_check_block(vc)) {
3818 finish_swait(&vc->wq, &wait);3815 finish_rcuwait(&vc->wait);
3819 do_sleep = 0;3816 do_sleep = 0;
3820 /* If we polled, count this as a successful poll */3817 /* If we polled, count this as a successful poll */
3821 if (vc->halt_poll_ns)3818 if (vc->halt_poll_ns)
@@ -3829,7 +3826,7 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) · 7 unmodified lines
3829 trace_kvmppc_vcore_blocked(vc, 0);3826 trace_kvmppc_vcore_blocked(vc, 0);
3830 spin_unlock(&vc->lock);3827 spin_unlock(&vc->lock);
3831 schedule();3828 schedule();
3832 finish_swait(&vc->wq, &wait);3829 finish_rcuwait(&vc->wait);
3833 spin_lock(&vc->lock);3830 spin_lock(&vc->lock);
3834 vc->vcore_state = VCORE_INACTIVE;3831 vc->vcore_state = VCORE_INACTIVE;
3835 trace_kvmppc_vcore_blocked(vc, 1);3832 trace_kvmppc_vcore_blocked(vc, 1);
@@ -3940,7 +3937,7 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) · 104 unmodified lines
3940 kvmppc_start_thread(vcpu, vc);3937 kvmppc_start_thread(vcpu, vc);
3941 trace_kvm_guest_enter(vcpu);3938 trace_kvm_guest_enter(vcpu);
3942 } else if (vc->vcore_state == VCORE_SLEEPING) {3939 } else if (vc->vcore_state == VCORE_SLEEPING) {
3943 swake_up_one(&vc->wq);3940 rcuwait_wake_up(&vc->wait);
3944 }3941 }
3945 3942
3946 }3943 }
@@ -4279,7 +4276,7 @@ static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu) · 332 unmodified lines
4279 }4276 }
4280 user_vrsave = mfspr(SPRN_VRSAVE);4277 user_vrsave = mfspr(SPRN_VRSAVE);
4281 4278
4282 vcpu->arch.wqp = &vcpu->arch.vcore->wq;4279 vcpu->arch.waitp = &vcpu->arch.vcore->wait;
4283 vcpu->arch.pgdir = kvm->mm->pgd;4280 vcpu->arch.pgdir = kvm->mm->pgd;
4284 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;4281 vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
4285 4282
arch/powerpc/kvm/powerpc.c+1−1
@@ -752,7 +752,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) · 751 unmodified lines
752 if (err)752 if (err)
753 goto out_vcpu_uninit;753 goto out_vcpu_uninit;
754 754
755 vcpu->arch.wqp = &vcpu->wq;755 vcpu->arch.waitp = &vcpu->wait;
756 kvmppc_create_vcpu_debugfs(vcpu, vcpu->vcpu_id);756 kvmppc_create_vcpu_debugfs(vcpu, vcpu->vcpu_id);
757 return 0;757 return 0;
758 758
arch/x86/kvm/lapic.c+1−1
@@ -1861,7 +1861,7 @@ void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) · 1860 unmodified lines
1861 /* If the preempt notifier has already run, it also called apic_timer_expired */1861 /* If the preempt notifier has already run, it also called apic_timer_expired */
1862 if (!apic->lapic_timer.hv_timer_in_use)1862 if (!apic->lapic_timer.hv_timer_in_use)
1863 goto out;1863 goto out;
1864 WARN_ON(swait_active(&vcpu->wq));1864 WARN_ON(rcuwait_active(&vcpu->wait));
1865 cancel_hv_timer(apic);1865 cancel_hv_timer(apic);
1866 apic_timer_expired(apic);1866 apic_timer_expired(apic);
1867 1867
include/linux/kvm_host.h+5−5
@@ -23,7 +23,7 @@ · 22 unmodified lines
23#include <linux/irqflags.h>23#include <linux/irqflags.h>
24#include <linux/context_tracking.h>24#include <linux/context_tracking.h>
25#include <linux/irqbypass.h>25#include <linux/irqbypass.h>
26#include <linux/swait.h>26#include <linux/rcuwait.h>
27#include <linux/refcount.h>27#include <linux/refcount.h>
28#include <linux/nospec.h>28#include <linux/nospec.h>
29#include <asm/signal.h>29#include <asm/signal.h>
@@ -277,7 +277,7 @@ struct kvm_vcpu { · 247 unmodified lines
277 struct mutex mutex;277 struct mutex mutex;
278 struct kvm_run *run;278 struct kvm_run *run;
279 279
280 struct swait_queue_head wq;280 struct rcuwait wait;
281 struct pid __rcu *pid;281 struct pid __rcu *pid;
282 int sigset_active;282 int sigset_active;
283 sigset_t sigset;283 sigset_t sigset;
@@ -960,12 +960,12 @@ static inline bool kvm_arch_has_assigned_device(struct kvm *kvm) · 676 unmodified lines
960}960}
961#endif961#endif
962 962
963static inline struct swait_queue_head *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu)963static inline struct rcuwait *kvm_arch_vcpu_get_wait(struct kvm_vcpu *vcpu)
964{964{
965#ifdef __KVM_HAVE_ARCH_WQP965#ifdef __KVM_HAVE_ARCH_WQP
966 return vcpu->arch.wqp;966 return vcpu->arch.waitp;
967#else967#else
968 return &vcpu->wq;968 return &vcpu->wait;
969#endif969#endif
970}970}
971 971
virt/kvm/arm/arch_timer.c+2−1
@@ -571,6 +571,7 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) · 570 unmodified lines
571{571{
572 struct arch_timer_cpu *timer = vcpu_timer(vcpu);572 struct arch_timer_cpu *timer = vcpu_timer(vcpu);
573 struct timer_map map;573 struct timer_map map;
574 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
574 575
575 if (unlikely(!timer->enabled))576 if (unlikely(!timer->enabled))
576 return;577 return;
@@ -593,7 +594,7 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) · 16 unmodified lines
593 if (map.emul_ptimer)594 if (map.emul_ptimer)
594 soft_timer_cancel(&map.emul_ptimer->hrtimer);595 soft_timer_cancel(&map.emul_ptimer->hrtimer);
595 596
596 if (swait_active(kvm_arch_vcpu_wq(vcpu)))597 if (rcuwait_active(wait))
597 kvm_timer_blocking(vcpu);598 kvm_timer_blocking(vcpu);
598 599
599 /*600 /*
virt/kvm/arm/arm.c+5−4
@@ -579,16 +579,17 @@ void kvm_arm_resume_guest(struct kvm *kvm) · 578 unmodified lines
579 579
580 kvm_for_each_vcpu(i, vcpu, kvm) {580 kvm_for_each_vcpu(i, vcpu, kvm) {
581 vcpu->arch.pause = false;581 vcpu->arch.pause = false;
582 swake_up_one(kvm_arch_vcpu_wq(vcpu));582 rcuwait_wake_up(kvm_arch_vcpu_get_wait(vcpu));
583 }583 }
584}584}
585 585
586static void vcpu_req_sleep(struct kvm_vcpu *vcpu)586static void vcpu_req_sleep(struct kvm_vcpu *vcpu)
587{587{
588 struct swait_queue_head *wq = kvm_arch_vcpu_wq(vcpu);588 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
589 589
590 swait_event_interruptible_exclusive(*wq, ((!vcpu->arch.power_off) &&590 rcuwait_wait_event(wait,
591 (!vcpu->arch.pause)));591 (!vcpu->arch.power_off) &&(!vcpu->arch.pause),
592 TASK_INTERRUPTIBLE);
592 593
593 if (vcpu->arch.power_off || vcpu->arch.pause) {594 if (vcpu->arch.power_off || vcpu->arch.pause) {
594 /* Awaken to handle a signal, request we sleep again later. */595 /* Awaken to handle a signal, request we sleep again later. */
virt/kvm/async_pf.c+1−2
@@ -80,8 +80,7 @@ static void async_pf_execute(struct work_struct *work) · 79 unmodified lines
80 80
81 trace_kvm_async_pf_completed(addr, cr2_or_gpa);81 trace_kvm_async_pf_completed(addr, cr2_or_gpa);
82 82
83 if (swq_has_sleeper(&vcpu->wq))83 rcuwait_wake_up(&vcpu->wait);
84 swake_up_one(&vcpu->wq);
85 84
86 mmput(mm);85 mmput(mm);
87 kvm_put_kvm(vcpu->kvm);86 kvm_put_kvm(vcpu->kvm);
virt/kvm/kvm_main.c+9−10
@@ -349,7 +349,7 @@ static void kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) · 348 unmodified lines
349 vcpu->kvm = kvm;349 vcpu->kvm = kvm;
350 vcpu->vcpu_id = id;350 vcpu->vcpu_id = id;
351 vcpu->pid = NULL;351 vcpu->pid = NULL;
352 init_swait_queue_head(&vcpu->wq);352 rcuwait_init(&vcpu->wait);
353 kvm_async_pf_vcpu_init(vcpu);353 kvm_async_pf_vcpu_init(vcpu);
354 354
355 vcpu->pre_pcpu = -1;355 vcpu->pre_pcpu = -1;
@@ -2678,7 +2678,6 @@ out: · 2322 unmodified lines
2678void kvm_vcpu_block(struct kvm_vcpu *vcpu)2678void kvm_vcpu_block(struct kvm_vcpu *vcpu)
2679{2679{
2680 ktime_t start, cur;2680 ktime_t start, cur;
2681 DECLARE_SWAITQUEUE(wait);
2682 bool waited = false;2681 bool waited = false;
2683 u64 block_ns;2682 u64 block_ns;
2684 2683
@@ -2704,8 +2703,9 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) · 19 unmodified lines
2704 } while (single_task_running() && ktime_before(cur, stop));2703 } while (single_task_running() && ktime_before(cur, stop));
2705 }2704 }
2706 2705
2706 prepare_to_rcuwait(&vcpu->wait);
2707 for (;;) {2707 for (;;) {
2708 prepare_to_swait_exclusive(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);2708 set_current_state(TASK_INTERRUPTIBLE);
2709 2709
2710 if (kvm_vcpu_check_block(vcpu) < 0)2710 if (kvm_vcpu_check_block(vcpu) < 0)
2711 break;2711 break;
@@ -2713,8 +2713,7 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) · 1 unmodified line
2713 waited = true;2713 waited = true;
2714 schedule();2714 schedule();
2715 }2715 }
2716 2716 finish_rcuwait(&vcpu->wait);
2717 finish_swait(&vcpu->wq, &wait);
2718 cur = ktime_get();2717 cur = ktime_get();
2719out:2718out:
2720 kvm_arch_vcpu_unblocking(vcpu);2719 kvm_arch_vcpu_unblocking(vcpu);
@@ -2746,11 +2745,10 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_block); · 25 unmodified lines
2746 2745
2747bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)2746bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
2748{2747{
2749 struct swait_queue_head *wqp;2748 struct rcuwait *waitp;
2750 2749
2751 wqp = kvm_arch_vcpu_wq(vcpu);2750 waitp = kvm_arch_vcpu_get_wait(vcpu);
2752 if (swq_has_sleeper(wqp)) {2751 if (rcuwait_wake_up(waitp)) {
2753 swake_up_one(wqp);
2754 WRITE_ONCE(vcpu->ready, true);2752 WRITE_ONCE(vcpu->ready, true);
2755 ++vcpu->stat.halt_wakeup;2753 ++vcpu->stat.halt_wakeup;
2756 return true;2754 return true;
@@ -2892,7 +2890,8 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode) · 135 unmodified lines
2892 continue;2890 continue;
2893 if (vcpu == me)2891 if (vcpu == me)
2894 continue;2892 continue;
2895 if (swait_active(&vcpu->wq) && !vcpu_dy_runnable(vcpu))2893 if (rcuwait_active(&vcpu->wait) &&
2894 !vcpu_dy_runnable(vcpu))
2896 continue;2895 continue;
2897 if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode &&2896 if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode &&
2898 !kvm_arch_vcpu_in_kernel(vcpu))2897 !kvm_arch_vcpu_in_kernel(vcpu))