Inspired by commit 9cac38dd5d (KVM/s390: Set preempted flag during vcpu wakeup and interrupt delivery), we want to also boost not just lock holders but also vCPUs that are delivering interrupts. Most smp_call_function_many calls are synchronous, so the IPI target vCPUs are also good yield candidates. This patch introduces vcpu->ready to boost vCPUs during wakeup and interrupt delivery time; unlike s390 we do not reuse vcpu->preempted so that voluntarily preempted vCPUs are taken into account by kvm_vcpu_on_spin, but vmx_vcpu_pi_put is not affected (VT-d PI handles voluntary preemption separately, in pi_pre_block). Testing on 80 HT 2 socket Xeon Skylake server, with 80 vCPUs VM 80GB RAM: ebizzy -M vanilla boosting improved 1VM 21443 23520 9% 2VM 2800 8000 180% 3VM 1800 3100 72% Testing on my Haswell desktop 8 HT, with 8 vCPUs VM 8GB RAM, two VMs, one running ebizzy -M, the other running 'stress --cpu 2': w/ boosting + w/o pv sched yield(vanilla) vanilla boosting improved 1570 4000 155% w/ boosting + w/ pv sched yield(vanilla) vanilla boosting improved 1844 5157 179% w/o boosting, perf top in VM: 72.33% [kernel] [k] smp_call_function_many 4.22% [kernel] [k] call_function_i 3.71% [kernel] [k] async_page_fault w/ boosting, perf top in VM: 38.43% [kernel] [k] smp_call_function_many 6.31% [kernel] [k] async_page_fault 6.13% libc-2.23.so [.] __memcpy_avx_unaligned 4.88% [kernel] [k] call_function_interrupt
Unified · Split
| @@ -1240,7 +1240,7 @@ void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu) · 1239 unmodified lines | |||
| 1240 | * The vcpu gave up the cpu voluntarily, mark it as a good | 1240 | * The vcpu gave up the cpu voluntarily, mark it as a good |
| 1241 | * yield-candidate. | 1241 | * yield-candidate. |
| 1242 | */ | 1242 | */ |
| 1243 | vcpu->preempted = true; | 1243 | vcpu->ready = true; |
| 1244 | swake_up_one(&vcpu->wq); | 1244 | swake_up_one(&vcpu->wq); |
| 1245 | vcpu->stat.halt_wakeup++; | 1245 | vcpu->stat.halt_wakeup++; |
| 1246 | } | 1246 | } |
| @@ -318,6 +318,7 @@ struct kvm_vcpu { · 317 unmodified lines | |||
| 318 | } spin_loop; | 318 | } spin_loop; |
| 319 | #endif | 319 | #endif |
| 320 | bool preempted; | 320 | bool preempted; |
| 321 | bool ready; | ||
| 321 | struct kvm_vcpu_arch arch; | 322 | struct kvm_vcpu_arch arch; |
| 322 | struct dentry *debugfs_dentry; | 323 | struct dentry *debugfs_dentry; |
| 323 | }; | 324 | }; |
| @@ -314,6 +314,7 @@ int kvm_vcpu_init(struct kvm_vcpu *vcpu, struct kvm *kvm, unsigned id) · 313 unmodified lines | |||
| 314 | kvm_vcpu_set_in_spin_loop(vcpu, false); | 314 | kvm_vcpu_set_in_spin_loop(vcpu, false); |
| 315 | kvm_vcpu_set_dy_eligible(vcpu, false); | 315 | kvm_vcpu_set_dy_eligible(vcpu, false); |
| 316 | vcpu->preempted = false; | 316 | vcpu->preempted = false; |
| 317 | vcpu->ready = false; | ||
| 317 | 318 | ||
| 318 | r = kvm_arch_vcpu_init(vcpu); | 319 | r = kvm_arch_vcpu_init(vcpu); |
| 319 | if (r < 0) | 320 | if (r < 0) |
| @@ -2387,6 +2388,7 @@ bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) · 2067 unmodified lines | |||
| 2387 | wqp = kvm_arch_vcpu_wq(vcpu); | 2388 | wqp = kvm_arch_vcpu_wq(vcpu); |
| 2388 | if (swq_has_sleeper(wqp)) { | 2389 | if (swq_has_sleeper(wqp)) { |
| 2389 | swake_up_one(wqp); | 2390 | swake_up_one(wqp); |
| 2391 | WRITE_ONCE(vcpu->ready, true); | ||
| 2390 | ++vcpu->stat.halt_wakeup; | 2392 | ++vcpu->stat.halt_wakeup; |
| 2391 | return true; | 2393 | return true; |
| 2392 | } | 2394 | } |
| @@ -2500,7 +2502,7 @@ void kvm_vcpu_on_spin(struct kvm_vcpu *me, bool yield_to_kernel_mode) · 107 unmodified lines | |||
| 2500 | continue; | 2502 | continue; |
| 2501 | } else if (pass && i > last_boosted_vcpu) | 2503 | } else if (pass && i > last_boosted_vcpu) |
| 2502 | break; | 2504 | break; |
| 2503 | if (!READ_ONCE(vcpu->preempted)) | 2505 | if (!READ_ONCE(vcpu->ready)) |
| 2504 | continue; | 2506 | continue; |
| 2505 | if (vcpu == me) | 2507 | if (vcpu == me) |
| 2506 | continue; | 2508 | continue; |
| @@ -4203,8 +4205,8 @@ static void kvm_sched_in(struct preempt_notifier *pn, int cpu) · 1696 unmodified lines | |||
| 4203 | { | 4205 | { |
| 4204 | struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); | 4206 | struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); |
| 4205 | 4207 | ||
| 4206 | if (vcpu->preempted) | 4208 | vcpu->preempted = false; |
| 4207 | vcpu->preempted = false; | 4209 | WRITE_ONCE(vcpu->ready, false); |
| 4208 | 4210 | ||
| 4209 | kvm_arch_sched_in(vcpu, cpu); | 4211 | kvm_arch_sched_in(vcpu, cpu); |
| 4210 | 4212 | ||
| @@ -4216,8 +4218,10 @@ static void kvm_sched_out(struct preempt_notifier *pn, · 5 unmodified lines | |||
| 4216 | { | 4218 | { |
| 4217 | struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); | 4219 | struct kvm_vcpu *vcpu = preempt_notifier_to_vcpu(pn); |
| 4218 | 4220 | ||
| 4219 | if (current->state == TASK_RUNNING) | 4221 | if (current->state == TASK_RUNNING) { |
| 4220 | vcpu->preempted = true; | 4222 | vcpu->preempted = true; |
| 4223 | WRITE_ONCE(vcpu->ready, true); | ||
| 4224 | } | ||
| 4221 | kvm_arch_vcpu_put(vcpu); | 4225 | kvm_arch_vcpu_put(vcpu); |
| 4222 | } | 4226 | } |
| 4223 | 4227 | ||