The use of any sort of waitqueue (simple or regular) for wait/waking vcpus has always been an overkill and semantically wrong. Because this is per-vcpu (which is blocked) there is only ever a single waiting vcpu, thus no need for any sort of queue. As such, make use of the rcuwait primitive, with the following considerations: - rcuwait already provides the proper barriers that serialize concurrent waiter and waker. - Task wakeup is done in rcu read critical region, with a stable task pointer. - Because there is no concurrency among waiters, we need not worry about rcuwait_wait_event() calls corrupting the wait->task. As a consequence, this saves the locking done in swait when modifying the queue. This also applies to per-vcore wait for powerpc kvm-hv. The x86 tscdeadline_latency test mentioned in 8577370fb0cb ("KVM: Use simple waitqueue for vcpu->wq") shows that, on avg, latency is reduced by around 15-20% with this change.
Showing the changes to include/linux (1 of 11 files). Show all
Unified · Split
| @@ -23,7 +23,7 @@ · 22 unmodified lines | |||
| 23 | #include <linux/irqflags.h> | 23 | #include <linux/irqflags.h> |
| 24 | #include <linux/context_tracking.h> | 24 | #include <linux/context_tracking.h> |
| 25 | #include <linux/irqbypass.h> | 25 | #include <linux/irqbypass.h> |
| 26 | #include <linux/swait.h> | 26 | #include <linux/rcuwait.h> |
| 27 | #include <linux/refcount.h> | 27 | #include <linux/refcount.h> |
| 28 | #include <linux/nospec.h> | 28 | #include <linux/nospec.h> |
| 29 | #include <asm/signal.h> | 29 | #include <asm/signal.h> |
| @@ -277,7 +277,7 @@ struct kvm_vcpu { · 247 unmodified lines | |||
| 277 | struct mutex mutex; | 277 | struct mutex mutex; |
| 278 | struct kvm_run *run; | 278 | struct kvm_run *run; |
| 279 | 279 | ||
| 280 | struct swait_queue_head wq; | 280 | struct rcuwait wait; |
| 281 | struct pid __rcu *pid; | 281 | struct pid __rcu *pid; |
| 282 | int sigset_active; | 282 | int sigset_active; |
| 283 | sigset_t sigset; | 283 | sigset_t sigset; |
| @@ -960,12 +960,12 @@ static inline bool kvm_arch_has_assigned_device(struct kvm *kvm) · 676 unmodified lines | |||
| 960 | } | 960 | } |
| 961 | #endif | 961 | #endif |
| 962 | 962 | ||
| 963 | static inline struct swait_queue_head *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu) | 963 | static inline struct rcuwait *kvm_arch_vcpu_get_wait(struct kvm_vcpu *vcpu) |
| 964 | { | 964 | { |
| 965 | #ifdef __KVM_HAVE_ARCH_WQP | 965 | #ifdef __KVM_HAVE_ARCH_WQP |
| 966 | return vcpu->arch.wqp; | 966 | return vcpu->arch.waitp; |
| 967 | #else | 967 | #else |
| 968 | return &vcpu->wq; | 968 | return &vcpu->wait; |
| 969 | #endif | 969 | #endif |
| 970 | } | 970 | } |
| 971 | 971 | ||