The use of any sort of waitqueue (simple or regular) for wait/waking vcpus has always been an overkill and semantically wrong. Because this is per-vcpu (which is blocked) there is only ever a single waiting vcpu, thus no need for any sort of queue. As such, make use of the rcuwait primitive, with the following considerations: - rcuwait already provides the proper barriers that serialize concurrent waiter and waker. - Task wakeup is done in rcu read critical region, with a stable task pointer. - Because there is no concurrency among waiters, we need not worry about rcuwait_wait_event() calls corrupting the wait->task. As a consequence, this saves the locking done in swait when modifying the queue. This also applies to per-vcore wait for powerpc kvm-hv. The x86 tscdeadline_latency test mentioned in 8577370fb0cb ("KVM: Use simple waitqueue for vcpu->wq") shows that, on avg, latency is reduced by around 15-20% with this change.
Showing the changes to include/linux (1 of 11 files). Show all
Unified · Split
| @@ -23,7 +23,7 @@ · 22 unmodified lines | ||
| 23 | 23 | #include <linux/irqflags.h> |
| 24 | 24 | #include <linux/context_tracking.h> |
| 25 | 25 | #include <linux/irqbypass.h> |
| 26 | −#include <linux/swait.h> | |
| 26 | +#include <linux/rcuwait.h> | |
| 27 | 27 | #include <linux/refcount.h> |
| 28 | 28 | #include <linux/nospec.h> |
| 29 | 29 | #include <asm/signal.h> |
| @@ -277,7 +277,7 @@ struct kvm_vcpu { · 247 unmodified lines | ||
| 277 | 277 | struct mutex mutex; |
| 278 | 278 | struct kvm_run *run; |
| 279 | 279 | |
| 280 | − struct swait_queue_head wq; | |
| 280 | + struct rcuwait wait; | |
| 281 | 281 | struct pid __rcu *pid; |
| 282 | 282 | int sigset_active; |
| 283 | 283 | sigset_t sigset; |
| @@ -960,12 +960,12 @@ static inline bool kvm_arch_has_assigned_device(struct kvm *kvm) · 676 unmodified lines | ||
| 960 | 960 | } |
| 961 | 961 | #endif |
| 962 | 962 | |
| 963 | −static inline struct swait_queue_head *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu) | |
| 963 | +static inline struct rcuwait *kvm_arch_vcpu_get_wait(struct kvm_vcpu *vcpu) | |
| 964 | 964 | { |
| 965 | 965 | #ifdef __KVM_HAVE_ARCH_WQP |
| 966 | − return vcpu->arch.wqp; | |
| 966 | + return vcpu->arch.waitp; | |
| 967 | 967 | #else |
| 968 | − return &vcpu->wq; | |
| 968 | + return &vcpu->wait; | |
| 969 | 969 | #endif |
| 970 | 970 | } |
| 971 | 971 | |