kvm: Replace vcpu->swait with rcuwait

The use of any sort of waitqueue (simple or regular) for wait/waking vcpus has always been an overkill and semantically wrong. Because this is per-vcpu (which is blocked) there is only ever a single waiting vcpu, thus no need for any sort of queue. As such, make use of the rcuwait primitive, with the following considerations: - rcuwait already provides the proper barriers that serialize concurrent waiter and waker. - Task wakeup is done in rcu read critical region, with a stable task pointer. - Because there is no concurrency among waiters, we need not worry about rcuwait_wait_event() calls corrupting the wait->task. As a consequence, this saves the locking done in swait when modifying the queue. This also applies to per-vcore wait for powerpc kvm-hv. The x86 tscdeadline_latency test mentioned in 8577370fb0cb ("KVM: Use simple waitqueue for vcpu->wq") shows that, on avg, latency is reduced by around 15-20% with this change.

Davidlohr Bueso committed 6 years ago

Commit da4ad881 parent 191a43b11 files · +38 −43Browse files

Unified · Split

11 files

arch/mips/kvm/mips.c +2 −4
arch/powerpc/include/asm/kvm_book3s.h +1 −1
arch/powerpc/include/asm/kvm_host.h +1 −1
arch/powerpc/kvm/book3s_hv.c +10 −13
arch/powerpc/kvm/powerpc.c +1 −1
arch/x86/kvm/lapic.c +1 −1
include/linux/kvm_host.h +5 −5
virt/kvm/arm/arch_timer.c +2 −1
virt/kvm/arm/arm.c +5 −4
virt/kvm/async_pf.c +1 −2
virt/kvm/kvm_main.c +9 −10

arch/mips/kvm/mips.c+2−4

@@ -283,8 +283,7 @@ static enum hrtimer_restart kvm_mips_comparecount_wakeup(struct hrtimer *timer) · 282 unmodified lines
283	283	kvm_mips_callbacks->queue_timer_int(vcpu);
284	284
285	285	vcpu->arch.wait = 0;
286		− if (swq_has_sleeper(&vcpu->wq))
287		− swake_up_one(&vcpu->wq);
	286	+ rcuwait_wake_up(&vcpu->wait);
288	287
289	288	return kvm_mips_count_timeout(vcpu);
290	289	}
@@ -511,8 +510,7 @@ int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, · 220 unmodified lines
511	510
512	511	dvcpu->arch.wait = 0;
513	512
514		− if (swq_has_sleeper(&dvcpu->wq))
515		− swake_up_one(&dvcpu->wq);
	513	+ rcuwait_wake_up(&dvcpu->wait);
516	514
517	515	return 0;
518	516	}

arch/powerpc/include/asm/kvm_book3s.h+1−1

@@ -78,7 +78,7 @@ struct kvmppc_vcore { · 77 unmodified lines
78	78	struct kvm_vcpu *runnable_threads[MAX_SMT_THREADS];
79	79	struct list_head preempt_list;
80	80	spinlock_t lock;
81		− struct swait_queue_head wq;
	81	+ struct rcuwait wait;
82	82	spinlock_t stoltb_lock; /* protects stolen_tb and preempt_tb */
83	83	u64 stolen_tb;
84	84	u64 preempt_tb;

arch/powerpc/include/asm/kvm_host.h+1−1

@@ -751,7 +751,7 @@ struct kvm_vcpu_arch { · 750 unmodified lines
751	751	u8 irq_pending; /* Used by XIVE to signal pending guest irqs */
752	752	u32 last_inst;
753	753
754		− struct swait_queue_head *wqp;
	754	+ struct rcuwait *waitp;
755	755	struct kvmppc_vcore *vcore;
756	756	int ret;
757	757	int trap;

arch/powerpc/kvm/book3s_hv.c+10−13

@@ -230,13 +230,11 @@ static bool kvmppc_ipi_thread(int cpu) · 229 unmodified lines
230	230	static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu)
231	231	{
232	232	int cpu;
233		− struct swait_queue_head *wqp;
	233	+ struct rcuwait *waitp;
234	234
235		− wqp = kvm_arch_vcpu_wq(vcpu);
236		− if (swq_has_sleeper(wqp)) {
237		− swake_up_one(wqp);
	235	+ waitp = kvm_arch_vcpu_get_wait(vcpu);
	236	+ if (rcuwait_wake_up(waitp))
238	237	++vcpu->stat.halt_wakeup;
239		− }
240	238
241	239	cpu = READ_ONCE(vcpu->arch.thread_cpu);
242	240	if (cpu >= 0 && kvmppc_ipi_thread(cpu))
@@ -2125,7 +2123,7 @@ static struct kvmppc_vcore kvmppc_vcore_create(struct kvm kvm, int id) · 1882 unmodified lines
2125	2123
2126	2124	spin_lock_init(&vcore->lock);
2127	2125	spin_lock_init(&vcore->stoltb_lock);
2128		− init_swait_queue_head(&vcore->wq);
	2126	+ rcuwait_init(&vcore->wait);
2129	2127	vcore->preempt_tb = TB_NIL;
2130	2128	vcore->lpcr = kvm->arch.lpcr;
2131	2129	vcore->first_vcpuid = id;
@@ -3784,7 +3782,6 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) · 1652 unmodified lines
3784	3782	ktime_t cur, start_poll, start_wait;
3785	3783	int do_sleep = 1;
3786	3784	u64 block_ns;
3787		− DECLARE_SWAITQUEUE(wait);
3788	3785
3789	3786	/* Poll for pending exceptions and ceded state */
3790	3787	cur = start_poll = ktime_get();
@@ -3812,10 +3809,10 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) · 21 unmodified lines
3812	3809	}
3813	3810	}
3814	3811
3815		− prepare_to_swait_exclusive(&vc->wq, &wait, TASK_INTERRUPTIBLE);
3816		−
	3812	+ prepare_to_rcuwait(&vc->wait);
	3813	+ set_current_state(TASK_INTERRUPTIBLE);
3817	3814	if (kvmppc_vcore_check_block(vc)) {
3818		− finish_swait(&vc->wq, &wait);
	3815	+ finish_rcuwait(&vc->wait);
3819	3816	do_sleep = 0;
3820	3817	/* If we polled, count this as a successful poll */
3821	3818	if (vc->halt_poll_ns)
@@ -3829,7 +3826,7 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) · 7 unmodified lines
3829	3826	trace_kvmppc_vcore_blocked(vc, 0);
3830	3827	spin_unlock(&vc->lock);
3831	3828	schedule();
3832		− finish_swait(&vc->wq, &wait);
	3829	+ finish_rcuwait(&vc->wait);
3833	3830	spin_lock(&vc->lock);
3834	3831	vc->vcore_state = VCORE_INACTIVE;
3835	3832	trace_kvmppc_vcore_blocked(vc, 1);
@@ -3940,7 +3937,7 @@ static int kvmppc_run_vcpu(struct kvm_run kvm_run, struct kvm_vcpu vcpu) · 104 unmodified lines
3940	3937	kvmppc_start_thread(vcpu, vc);
3941	3938	trace_kvm_guest_enter(vcpu);
3942	3939	} else if (vc->vcore_state == VCORE_SLEEPING) {
3943		− swake_up_one(&vc->wq);
	3940	+ rcuwait_wake_up(&vc->wait);
3944	3941	}
3945	3942
3946	3943	}
@@ -4279,7 +4276,7 @@ static int kvmppc_vcpu_run_hv(struct kvm_run run, struct kvm_vcpu vcpu) · 332 unmodified lines
4279	4276	}
4280	4277	user_vrsave = mfspr(SPRN_VRSAVE);
4281	4278
4282		− vcpu->arch.wqp = &vcpu->arch.vcore->wq;
	4279	+ vcpu->arch.waitp = &vcpu->arch.vcore->wait;
4283	4280	vcpu->arch.pgdir = kvm->mm->pgd;
4284	4281	vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST;
4285	4282

arch/powerpc/kvm/powerpc.c+1−1

@@ -752,7 +752,7 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) · 751 unmodified lines
752	752	if (err)
753	753	goto out_vcpu_uninit;
754	754
755		− vcpu->arch.wqp = &vcpu->wq;
	755	+ vcpu->arch.waitp = &vcpu->wait;
756	756	kvmppc_create_vcpu_debugfs(vcpu, vcpu->vcpu_id);
757	757	return 0;
758	758

arch/x86/kvm/lapic.c+1−1

@@ -1861,7 +1861,7 @@ void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu) · 1860 unmodified lines
1861	1861	/* If the preempt notifier has already run, it also called apic_timer_expired */
1862	1862	if (!apic->lapic_timer.hv_timer_in_use)
1863	1863	goto out;
1864		− WARN_ON(swait_active(&vcpu->wq));
	1864	+ WARN_ON(rcuwait_active(&vcpu->wait));
1865	1865	cancel_hv_timer(apic);
1866	1866	apic_timer_expired(apic);
1867	1867

include/linux/kvm_host.h+5−5

@@ -23,7 +23,7 @@ · 22 unmodified lines
23	23	#include <linux/irqflags.h>
24	24	#include <linux/context_tracking.h>
25	25	#include <linux/irqbypass.h>
26		−#include <linux/swait.h>
	26	+#include <linux/rcuwait.h>
27	27	#include <linux/refcount.h>
28	28	#include <linux/nospec.h>
29	29	#include <asm/signal.h>
@@ -277,7 +277,7 @@ struct kvm_vcpu { · 247 unmodified lines
277	277	struct mutex mutex;
278	278	struct kvm_run *run;
279	279
280		− struct swait_queue_head wq;
	280	+ struct rcuwait wait;
281	281	struct pid __rcu *pid;
282	282	int sigset_active;
283	283	sigset_t sigset;
@@ -960,12 +960,12 @@ static inline bool kvm_arch_has_assigned_device(struct kvm *kvm) · 676 unmodified lines
960	960	}
961	961	#endif
962	962
963		−static inline struct swait_queue_head kvm_arch_vcpu_wq(struct kvm_vcpu vcpu)
	963	+static inline struct rcuwait kvm_arch_vcpu_get_wait(struct kvm_vcpu vcpu)
964	964	{
965	965	#ifdef __KVM_HAVE_ARCH_WQP
966		− return vcpu->arch.wqp;
	966	+ return vcpu->arch.waitp;
967	967	#else
968		− return &vcpu->wq;
	968	+ return &vcpu->wait;
969	969	#endif
970	970	}
971	971

virt/kvm/arm/arch_timer.c+2−1

@@ -571,6 +571,7 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) · 570 unmodified lines
571	571	{
572	572	struct arch_timer_cpu *timer = vcpu_timer(vcpu);
573	573	struct timer_map map;
	574	+ struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
574	575
575	576	if (unlikely(!timer->enabled))
576	577	return;
@@ -593,7 +594,7 @@ void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu) · 16 unmodified lines
593	594	if (map.emul_ptimer)
594	595	soft_timer_cancel(&map.emul_ptimer->hrtimer);
595	596
596		− if (swait_active(kvm_arch_vcpu_wq(vcpu)))
	597	+ if (rcuwait_active(wait))
597	598	kvm_timer_blocking(vcpu);
598	599
599	600	/*

virt/kvm/arm/arm.c+5−4

@@ -579,16 +579,17 @@ void kvm_arm_resume_guest(struct kvm *kvm) · 578 unmodified lines
579	579
580	580	kvm_for_each_vcpu(i, vcpu, kvm) {
581	581	vcpu->arch.pause = false;
582		− swake_up_one(kvm_arch_vcpu_wq(vcpu));
	582	+ rcuwait_wake_up(kvm_arch_vcpu_get_wait(vcpu));
583	583	}
584	584	}
585	585
586	586	static void vcpu_req_sleep(struct kvm_vcpu *vcpu)
587	587	{
588		− struct swait_queue_head *wq = kvm_arch_vcpu_wq(vcpu);
	588	+ struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu);
589	589
590		− swait_event_interruptible_exclusive(*wq, ((!vcpu->arch.power_off) &&
591		− (!vcpu->arch.pause)));
	590	+ rcuwait_wait_event(wait,
	591	+ (!vcpu->arch.power_off) &&(!vcpu->arch.pause),
	592	+ TASK_INTERRUPTIBLE);
592	593
593	594	if (vcpu->arch.power_off \|\| vcpu->arch.pause) {
594	595	/* Awaken to handle a signal, request we sleep again later. */

virt/kvm/async_pf.c+1−2

@@ -80,8 +80,7 @@ static void async_pf_execute(struct work_struct *work) · 79 unmodified lines
80	80
81	81	trace_kvm_async_pf_completed(addr, cr2_or_gpa);
82	82
83		− if (swq_has_sleeper(&vcpu->wq))
84		− swake_up_one(&vcpu->wq);
	83	+ rcuwait_wake_up(&vcpu->wait);
85	84
86	85	mmput(mm);
87	86	kvm_put_kvm(vcpu->kvm);

virt/kvm/kvm_main.c+9−10

@@ -349,7 +349,7 @@ static void kvm_vcpu_init(struct kvm_vcpu vcpu, struct kvm kvm, unsigned id) · 348 unmodified lines
349	349	vcpu->kvm = kvm;
350	350	vcpu->vcpu_id = id;
351	351	vcpu->pid = NULL;
352		− init_swait_queue_head(&vcpu->wq);
	352	+ rcuwait_init(&vcpu->wait);
353	353	kvm_async_pf_vcpu_init(vcpu);
354	354
355	355	vcpu->pre_pcpu = -1;
@@ -2678,7 +2678,6 @@ out: · 2322 unmodified lines
2678	2678	void kvm_vcpu_block(struct kvm_vcpu *vcpu)
2679	2679	{
2680	2680	ktime_t start, cur;
2681		− DECLARE_SWAITQUEUE(wait);
2682	2681	bool waited = false;
2683	2682	u64 block_ns;
2684	2683
@@ -2704,8 +2703,9 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) · 19 unmodified lines
2704	2703	} while (single_task_running() && ktime_before(cur, stop));
2705	2704	}
2706	2705
	2706	+ prepare_to_rcuwait(&vcpu->wait);
2707	2707	for (;;) {
2708		− prepare_to_swait_exclusive(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
	2708	+ set_current_state(TASK_INTERRUPTIBLE);
2709	2709
2710	2710	if (kvm_vcpu_check_block(vcpu) < 0)
2711	2711	break;
@@ -2713,8 +2713,7 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) · 1 unmodified line
2713	2713	waited = true;
2714	2714	schedule();
2715	2715	}
2716		−
2717		− finish_swait(&vcpu->wq, &wait);
	2716	+ finish_rcuwait(&vcpu->wait);
2718	2717	cur = ktime_get();
2719	2718	out:
2720	2719	kvm_arch_vcpu_unblocking(vcpu);
@@ -2746,11 +2745,10 @@ EXPORT_SYMBOL_GPL(kvm_vcpu_block); · 25 unmodified lines
2746	2745
2747	2746	bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu)
2748	2747	{
2749		− struct swait_queue_head *wqp;
	2748	+ struct rcuwait *waitp;
2750	2749
2751		− wqp = kvm_arch_vcpu_wq(vcpu);
2752		− if (swq_has_sleeper(wqp)) {
2753		− swake_up_one(wqp);
	2750	+ waitp = kvm_arch_vcpu_get_wait(vcpu);
	2751	+ if (rcuwait_wake_up(waitp)) {
2754	2752	WRITE_ONCE(vcpu->ready, true);
2755	2753	++vcpu->stat.halt_wakeup;
2756	2754	return true;
@@ -2892,7 +2890,8 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode) · 135 unmodified lines
2892	2890	continue;
2893	2891	if (vcpu == me)
2894	2892	continue;
2895		− if (swait_active(&vcpu->wq) && !vcpu_dy_runnable(vcpu))
	2893	+ if (rcuwait_active(&vcpu->wait) &&
	2894	+ !vcpu_dy_runnable(vcpu))
2896	2895	continue;
2897	2896	if (READ_ONCE(vcpu->preempted) && yield_to_kernel_mode &&
2898	2897	!kvm_arch_vcpu_in_kernel(vcpu))