History

KVM: s390: pv: asynchronous destroy for reboot

Until now, destroying a protected guest was an entirely synchronous operation that could potentially take a very long time, depending on the size of the guest, due to the time needed to clean up the address space from protected pages. This patch implements an asynchronous destroy mechanism, that allows a protected guest to reboot significantly faster than previously. This is achieved by clearing the pages of the old guest in background. In case of reboot, the new guest will be able to run in the same address space almost immediately. The old protected guest is then only destroyed when all of its memory has been destroyed or otherwise made non protected. Two new PV commands are added for the KVM_S390_PV_COMMAND ioctl: KVM_PV_ASYNC_CLEANUP_PREPARE: set aside the current protected VM for later asynchronous teardown. The current KVM VM will then continue immediately as non-protected. If a protected VM had already been set aside for asynchronous teardown, but without starting the teardown process, this call will fail. There can be at most one VM set aside at any time. Once it is set aside, the protected VM only exists in the context of the Ultravisor, it is not associated with the KVM VM anymore. Its protected CPUs have already been destroyed, but not its memory. This command can be issued again immediately after starting KVM_PV_ASYNC_CLEANUP_PERFORM, without having to wait for completion. KVM_PV_ASYNC_CLEANUP_PERFORM: tears down the protected VM previously set aside using KVM_PV_ASYNC_CLEANUP_PREPARE. Ideally the KVM_PV_ASYNC_CLEANUP_PERFORM PV command should be issued by userspace from a separate thread. If a fatal signal is received (or if the process terminates naturally), the command will terminate immediately without completing. All protected VMs whose teardown was interrupted will be put in the need_cleanup list. The rest of the normal KVM teardown process will take care of properly cleaning up all remaining protected VMs, including the ones on the need_cleanup list.

Claudio Imbrenda committed 4 years ago

Commit fb491d51 parent 58635d65 files · +333 −18Browse files

Unified · Split

5 files
  1. arch/s390/include/asm/kvm_host.h +2 −0
  2. arch/s390/kvm/kvm-s390.c +41 −8
  3. arch/s390/kvm/kvm-s390.h +3 −0
  4. arch/s390/kvm/pv.c +285 −10
  5. include/uapi/linux/kvm.h +2 −0
arch/s390/include/asm/kvm_host.h+2−0
@@ -942,6 +942,8 @@ struct kvm_s390_pv { · 941 unmodified lines
942942 unsigned long stor_base;
943943 void *stor_var;
944944 bool dumping;
945+ void *set_aside;
946+ struct list_head need_cleanup;
945947 struct mmu_notifier mmu_notifier;
946948 };
947949
arch/s390/kvm/kvm-s390.c+41−8
@@ -209,6 +209,8 @@ unsigned int diag9c_forwarding_hz; · 208 unmodified lines
209209 module_param(diag9c_forwarding_hz, uint, 0644);
210210 MODULE_PARM_DESC(diag9c_forwarding_hz, "Maximum diag9c forwarding per second, 0 to turn off");
211211
212+static int async_destroy;
213+
212214 /*
213215 * For now we handle at most 16 double words as this is what the s390 base
214216 * kernel handles and stores in the prefix page. If we ever need to go beyond
@@ -2504,9 +2506,13 @@ static int kvm_s390_pv_dmp(struct kvm *kvm, struct kvm_pv_cmd *cmd, · 2289 unmodified lines
25042506
25052507 static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
25062508 {
2509+ const bool need_lock = (cmd->cmd != KVM_PV_ASYNC_CLEANUP_PERFORM);
2510+ void __user *argp = (void __user *)cmd->data;
25072511 int r = 0;
25082512 u16 dummy;
2509− void __user *argp = (void __user *)cmd->data;
2513+
2514+ if (need_lock)
2515+ mutex_lock(&kvm->lock);
25102516
25112517 switch (cmd->cmd) {
25122518 case KVM_PV_ENABLE: {
@@ -2540,6 +2546,31 @@ static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd) · 27 unmodified lines
25402546 set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
25412547 break;
25422548 }
2549+ case KVM_PV_ASYNC_CLEANUP_PREPARE:
2550+ r = -EINVAL;
2551+ if (!kvm_s390_pv_is_protected(kvm) || !async_destroy)
2552+ break;
2553+
2554+ r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2555+ /*
2556+ * If a CPU could not be destroyed, destroy VM will also fail.
2557+ * There is no point in trying to destroy it. Instead return
2558+ * the rc and rrc from the first CPU that failed destroying.
2559+ */
2560+ if (r)
2561+ break;
2562+ r = kvm_s390_pv_set_aside(kvm, &cmd->rc, &cmd->rrc);
2563+
2564+ /* no need to block service interrupts any more */
2565+ clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2566+ break;
2567+ case KVM_PV_ASYNC_CLEANUP_PERFORM:
2568+ r = -EINVAL;
2569+ if (!async_destroy)
2570+ break;
2571+ /* kvm->lock must not be held; this is asserted inside the function. */
2572+ r = kvm_s390_pv_deinit_aside_vm(kvm, &cmd->rc, &cmd->rrc);
2573+ break;
25432574 case KVM_PV_DISABLE: {
25442575 r = -EINVAL;
25452576 if (!kvm_s390_pv_is_protected(kvm))
@@ -2553,7 +2584,7 @@ static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd) · 7 unmodified lines
25532584 */
25542585 if (r)
25552586 break;
2556− r = kvm_s390_pv_deinit_vm(kvm, &cmd->rc, &cmd->rrc);
2587+ r = kvm_s390_pv_deinit_cleanup_all(kvm, &cmd->rc, &cmd->rrc);
25572588
25582589 /* no need to block service interrupts any more */
25592590 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
@@ -2703,6 +2734,9 @@ static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd) · 143 unmodified lines
27032734 default:
27042735 r = -ENOTTY;
27052736 }
2737+ if (need_lock)
2738+ mutex_unlock(&kvm->lock);
2739+
27062740 return r;
27072741 }
27082742
@@ -2907,9 +2941,8 @@ long kvm_arch_vm_ioctl(struct file *filp, · 198 unmodified lines
29072941 r = -EINVAL;
29082942 break;
29092943 }
2910− mutex_lock(&kvm->lock);
2944+ /* must be called without kvm->lock */
29112945 r = kvm_s390_handle_pv(kvm, &args);
2912− mutex_unlock(&kvm->lock);
29132946 if (copy_to_user(argp, &args, sizeof(args))) {
29142947 r = -EFAULT;
29152948 break;
@@ -3228,6 +3261,8 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) · 312 unmodified lines
32283261 kvm_s390_vsie_init(kvm);
32293262 if (use_gisa)
32303263 kvm_s390_gisa_init(kvm);
3264+ INIT_LIST_HEAD(&kvm->arch.pv.need_cleanup);
3265+ kvm->arch.pv.set_aside = NULL;
32313266 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
32323267
32333268 return 0;
@@ -3272,11 +3307,9 @@ void kvm_arch_destroy_vm(struct kvm *kvm) · 38 unmodified lines
32723307 /*
32733308 * We are already at the end of life and kvm->lock is not taken.
32743309 * This is ok as the file descriptor is closed by now and nobody
3275− * can mess with the pv state. To avoid lockdep_assert_held from
3276− * complaining we do not use kvm_s390_pv_is_protected.
3310+ * can mess with the pv state.
32773311 */
3278− if (kvm_s390_pv_get_handle(kvm))
3279− kvm_s390_pv_deinit_vm(kvm, &rc, &rrc);
3312+ kvm_s390_pv_deinit_cleanup_all(kvm, &rc, &rrc);
32803313 /*
32813314 * Remove the mmu notifier only when the whole KVM VM is torn down,
32823315 * and only if one was registered to begin with. If the VM is
arch/s390/kvm/kvm-s390.h+3−0
@@ -244,6 +244,9 @@ static inline u32 kvm_s390_get_gisa_desc(struct kvm *kvm) · 243 unmodified lines
244244 /* implemented in pv.c */
245245 int kvm_s390_pv_destroy_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc);
246246 int kvm_s390_pv_create_cpu(struct kvm_vcpu *vcpu, u16 *rc, u16 *rrc);
247+int kvm_s390_pv_set_aside(struct kvm *kvm, u16 *rc, u16 *rrc);
248+int kvm_s390_pv_deinit_aside_vm(struct kvm *kvm, u16 *rc, u16 *rrc);
249+int kvm_s390_pv_deinit_cleanup_all(struct kvm *kvm, u16 *rc, u16 *rrc);
247250 int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc);
248251 int kvm_s390_pv_init_vm(struct kvm *kvm, u16 *rc, u16 *rrc);
249252 int kvm_s390_pv_set_sec_parms(struct kvm *kvm, void *hdr, u64 length, u16 *rc,
arch/s390/kvm/pv.c+285−10
@@ -18,6 +18,29 @@ · 17 unmodified lines
1818 #include <linux/mmu_notifier.h>
1919 #include "kvm-s390.h"
2020
21+/**
22+ * struct pv_vm_to_be_destroyed - Represents a protected VM that needs to
23+ * be destroyed
24+ *
25+ * @list: list head for the list of leftover VMs
26+ * @old_gmap_table: the gmap table of the leftover protected VM
27+ * @handle: the handle of the leftover protected VM
28+ * @stor_var: pointer to the variable storage of the leftover protected VM
29+ * @stor_base: address of the base storage of the leftover protected VM
30+ *
31+ * Represents a protected VM that is still registered with the Ultravisor,
32+ * but which does not correspond any longer to an active KVM VM. It should
33+ * be destroyed at some point later, either asynchronously or when the
34+ * process terminates.
35+ */
36+struct pv_vm_to_be_destroyed {
37+ struct list_head list;
38+ unsigned long old_gmap_table;
39+ u64 handle;
40+ void *stor_var;
41+ unsigned long stor_base;
42+};
43+
2144 static void kvm_s390_clear_pv_state(struct kvm *kvm)
2245 {
2346 kvm->arch.pv.handle = 0;
@@ -161,23 +184,150 @@ out_err: · 137 unmodified lines
161184 return -ENOMEM;
162185 }
163186
164−/* this should not fail, but if it does, we must not free the donated memory */
165−int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
187+/**
188+ * kvm_s390_pv_dispose_one_leftover - Clean up one leftover protected VM.
189+ * @kvm: the KVM that was associated with this leftover protected VM
190+ * @leftover: details about the leftover protected VM that needs a clean up
191+ * @rc: the RC code of the Destroy Secure Configuration UVC
192+ * @rrc: the RRC code of the Destroy Secure Configuration UVC
193+ *
194+ * Destroy one leftover protected VM.
195+ * On success, kvm->mm->context.protected_count will be decremented atomically
196+ * and all other resources used by the VM will be freed.
197+ *
198+ * Return: 0 in case of success, otherwise 1
199+ */
200+static int kvm_s390_pv_dispose_one_leftover(struct kvm *kvm,
201+ struct pv_vm_to_be_destroyed *leftover,
202+ u16 *rc, u16 *rrc)
166203 {
167204 int cc;
168205
169− cc = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
170− UVC_CMD_DESTROY_SEC_CONF, rc, rrc);
171− WRITE_ONCE(kvm->arch.gmap->guest_handle, 0);
206+ cc = uv_cmd_nodata(leftover->handle, UVC_CMD_DESTROY_SEC_CONF, rc, rrc);
207+ KVM_UV_EVENT(kvm, 3, "PROTVIRT DESTROY LEFTOVER VM: rc %x rrc %x", *rc, *rrc);
208+ WARN_ONCE(cc, "protvirt destroy leftover vm failed rc %x rrc %x", *rc, *rrc);
209+ if (cc)
210+ return cc;
172211 /*
173− * if the mm still has a mapping, make all its pages accessible
174− * before destroying the guest
212+ * Intentionally leak unusable memory. If the UVC fails, the memory
213+ * used for the VM and its metadata is permanently unusable.
214+ * This can only happen in case of a serious KVM or hardware bug; it
215+ * is not expected to happen in normal operation.
175216 */
176− if (mmget_not_zero(kvm->mm)) {
177− s390_uv_destroy_range(kvm->mm, 0, TASK_SIZE);
178− mmput(kvm->mm);
217+ free_pages(leftover->stor_base, get_order(uv_info.guest_base_stor_len));
218+ free_pages(leftover->old_gmap_table, CRST_ALLOC_ORDER);
219+ vfree(leftover->stor_var);
220+ atomic_dec(&kvm->mm->context.protected_count);
221+ return 0;
222+}
223+
224+/**
225+ * kvm_s390_destroy_lower_2g - Destroy the first 2GB of protected guest memory.
226+ * @kvm: the VM whose memory is to be cleared.
227+ *
228+ * Destroy the first 2GB of guest memory, to avoid prefix issues after reboot.
229+ * The CPUs of the protected VM need to be destroyed beforehand.
230+ */
231+static void kvm_s390_destroy_lower_2g(struct kvm *kvm)
232+{
233+ const unsigned long pages_2g = SZ_2G / PAGE_SIZE;
234+ struct kvm_memory_slot *slot;
235+ unsigned long len;
236+ int srcu_idx;
237+
238+ srcu_idx = srcu_read_lock(&kvm->srcu);
239+
240+ /* Take the memslot containing guest absolute address 0 */
241+ slot = gfn_to_memslot(kvm, 0);
242+ /* Clear all slots or parts thereof that are below 2GB */
243+ while (slot && slot->base_gfn < pages_2g) {
244+ len = min_t(u64, slot->npages, pages_2g - slot->base_gfn) * PAGE_SIZE;
245+ s390_uv_destroy_range(kvm->mm, slot->userspace_addr, slot->userspace_addr + len);
246+ /* Take the next memslot */
247+ slot = gfn_to_memslot(kvm, slot->base_gfn + slot->npages);
248+ }
249+
250+ srcu_read_unlock(&kvm->srcu, srcu_idx);
251+}
252+
253+/**
254+ * kvm_s390_pv_set_aside - Set aside a protected VM for later teardown.
255+ * @kvm: the VM
256+ * @rc: return value for the RC field of the UVCB
257+ * @rrc: return value for the RRC field of the UVCB
258+ *
259+ * Set aside the protected VM for a subsequent teardown. The VM will be able
260+ * to continue immediately as a non-secure VM, and the information needed to
261+ * properly tear down the protected VM is set aside. If another protected VM
262+ * was already set aside without starting its teardown, this function will
263+ * fail.
264+ * The CPUs of the protected VM need to be destroyed beforehand.
265+ *
266+ * Context: kvm->lock needs to be held
267+ *
268+ * Return: 0 in case of success, -EINVAL if another protected VM was already set
269+ * aside, -ENOMEM if the system ran out of memory.
270+ */
271+int kvm_s390_pv_set_aside(struct kvm *kvm, u16 *rc, u16 *rrc)
272+{
273+ struct pv_vm_to_be_destroyed *priv;
274+
275+ lockdep_assert_held(&kvm->lock);
276+ /*
277+ * If another protected VM was already prepared for teardown, refuse.
278+ * A normal deinitialization has to be performed instead.
279+ */
280+ if (kvm->arch.pv.set_aside)
281+ return -EINVAL;
282+ priv = kzalloc(sizeof(*priv), GFP_KERNEL);
283+ if (!priv)
284+ return -ENOMEM;
285+
286+ priv->stor_var = kvm->arch.pv.stor_var;
287+ priv->stor_base = kvm->arch.pv.stor_base;
288+ priv->handle = kvm_s390_pv_get_handle(kvm);
289+ priv->old_gmap_table = (unsigned long)kvm->arch.gmap->table;
290+ WRITE_ONCE(kvm->arch.gmap->guest_handle, 0);
291+ if (s390_replace_asce(kvm->arch.gmap)) {
292+ kfree(priv);
293+ return -ENOMEM;
179294 }
180295
296+ kvm_s390_destroy_lower_2g(kvm);
297+ kvm_s390_clear_pv_state(kvm);
298+ kvm->arch.pv.set_aside = priv;
299+
300+ *rc = UVC_RC_EXECUTED;
301+ *rrc = 42;
302+ return 0;
303+}
304+
305+/**
306+ * kvm_s390_pv_deinit_vm - Deinitialize the current protected VM
307+ * @kvm: the KVM whose protected VM needs to be deinitialized
308+ * @rc: the RC code of the UVC
309+ * @rrc: the RRC code of the UVC
310+ *
311+ * Deinitialize the current protected VM. This function will destroy and
312+ * cleanup the current protected VM, but it will not cleanup the guest
313+ * memory. This function should only be called when the protected VM has
314+ * just been created and therefore does not have any guest memory, or when
315+ * the caller cleans up the guest memory separately.
316+ *
317+ * This function should not fail, but if it does, the donated memory must
318+ * not be freed.
319+ *
320+ * Context: kvm->lock needs to be held
321+ *
322+ * Return: 0 in case of success, otherwise -EIO
323+ */
324+int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
325+{
326+ int cc;
327+
328+ cc = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
329+ UVC_CMD_DESTROY_SEC_CONF, rc, rrc);
330+ WRITE_ONCE(kvm->arch.gmap->guest_handle, 0);
181331 if (!cc) {
182332 atomic_dec(&kvm->mm->context.protected_count);
183333 kvm_s390_pv_dealloc_vm(kvm);
@@ -191,6 +341,131 @@ int kvm_s390_pv_deinit_vm(struct kvm *kvm, u16 *rc, u16 *rrc) · 7 unmodified lines
191341 return cc ? -EIO : 0;
192342 }
193343
344+/**
345+ * kvm_s390_pv_deinit_cleanup_all - Clean up all protected VMs associated
346+ * with a specific KVM.
347+ * @kvm: the KVM to be cleaned up
348+ * @rc: the RC code of the first failing UVC
349+ * @rrc: the RRC code of the first failing UVC
350+ *
351+ * This function will clean up all protected VMs associated with a KVM.
352+ * This includes the active one, the one prepared for deinitialization with
353+ * kvm_s390_pv_set_aside, and any still pending in the need_cleanup list.
354+ *
355+ * Context: kvm->lock needs to be held unless being called from
356+ * kvm_arch_destroy_vm.
357+ *
358+ * Return: 0 if all VMs are successfully cleaned up, otherwise -EIO
359+ */
360+int kvm_s390_pv_deinit_cleanup_all(struct kvm *kvm, u16 *rc, u16 *rrc)
361+{
362+ struct pv_vm_to_be_destroyed *cur;
363+ bool need_zap = false;
364+ u16 _rc, _rrc;
365+ int cc = 0;
366+
367+ /* Make sure the counter does not reach 0 before calling s390_uv_destroy_range */
368+ atomic_inc(&kvm->mm->context.protected_count);
369+
370+ *rc = 1;
371+ /* If the current VM is protected, destroy it */
372+ if (kvm_s390_pv_get_handle(kvm)) {
373+ cc = kvm_s390_pv_deinit_vm(kvm, rc, rrc);
374+ need_zap = true;
375+ }
376+
377+ /* If a previous protected VM was set aside, put it in the need_cleanup list */
378+ if (kvm->arch.pv.set_aside) {
379+ list_add(kvm->arch.pv.set_aside, &kvm->arch.pv.need_cleanup);
380+ kvm->arch.pv.set_aside = NULL;
381+ }
382+
383+ /* Cleanup all protected VMs in the need_cleanup list */
384+ while (!list_empty(&kvm->arch.pv.need_cleanup)) {
385+ cur = list_first_entry(&kvm->arch.pv.need_cleanup, typeof(*cur), list);
386+ need_zap = true;
387+ if (kvm_s390_pv_dispose_one_leftover(kvm, cur, &_rc, &_rrc)) {
388+ cc = 1;
389+ /*
390+ * Only return the first error rc and rrc, so make
391+ * sure it is not overwritten. All destroys will
392+ * additionally be reported via KVM_UV_EVENT().
393+ */
394+ if (*rc == UVC_RC_EXECUTED) {
395+ *rc = _rc;
396+ *rrc = _rrc;
397+ }
398+ }
399+ list_del(&cur->list);
400+ kfree(cur);
401+ }
402+
403+ /*
404+ * If the mm still has a mapping, try to mark all its pages as
405+ * accessible. The counter should not reach zero before this
406+ * cleanup has been performed.
407+ */
408+ if (need_zap && mmget_not_zero(kvm->mm)) {
409+ s390_uv_destroy_range(kvm->mm, 0, TASK_SIZE);
410+ mmput(kvm->mm);
411+ }
412+
413+ /* Now the counter can safely reach 0 */
414+ atomic_dec(&kvm->mm->context.protected_count);
415+ return cc ? -EIO : 0;
416+}
417+
418+/**
419+ * kvm_s390_pv_deinit_aside_vm - Teardown a previously set aside protected VM.
420+ * @kvm: the VM previously associated with the protected VM
421+ * @rc: return value for the RC field of the UVCB
422+ * @rrc: return value for the RRC field of the UVCB
423+ *
424+ * Tear down the protected VM that had been previously prepared for teardown
425+ * using kvm_s390_pv_set_aside_vm. Ideally this should be called by
426+ * userspace asynchronously from a separate thread.
427+ *
428+ * Context: kvm->lock must not be held.
429+ *
430+ * Return: 0 in case of success, -EINVAL if no protected VM had been
431+ * prepared for asynchronous teardowm, -EIO in case of other errors.
432+ */
433+int kvm_s390_pv_deinit_aside_vm(struct kvm *kvm, u16 *rc, u16 *rrc)
434+{
435+ struct pv_vm_to_be_destroyed *p;
436+ int ret = 0;
437+
438+ lockdep_assert_not_held(&kvm->lock);
439+ mutex_lock(&kvm->lock);
440+ p = kvm->arch.pv.set_aside;
441+ kvm->arch.pv.set_aside = NULL;
442+ mutex_unlock(&kvm->lock);
443+ if (!p)
444+ return -EINVAL;
445+
446+ /* When a fatal signal is received, stop immediately */
447+ if (s390_uv_destroy_range_interruptible(kvm->mm, 0, TASK_SIZE_MAX))
448+ goto done;
449+ if (kvm_s390_pv_dispose_one_leftover(kvm, p, rc, rrc))
450+ ret = -EIO;
451+ kfree(p);
452+ p = NULL;
453+done:
454+ /*
455+ * p is not NULL if we aborted because of a fatal signal, in which
456+ * case queue the leftover for later cleanup.
457+ */
458+ if (p) {
459+ mutex_lock(&kvm->lock);
460+ list_add(&p->list, &kvm->arch.pv.need_cleanup);
461+ mutex_unlock(&kvm->lock);
462+ /* Did not finish, but pretend things went well */
463+ *rc = UVC_RC_EXECUTED;
464+ *rrc = 42;
465+ }
466+ return ret;
467+}
468+
194469 static void kvm_s390_pv_mmu_notifier_release(struct mmu_notifier *subscription,
195470 struct mm_struct *mm)
196471 {
include/uapi/linux/kvm.h+2−0
@@ -1740,6 +1740,8 @@ enum pv_cmd_id { · 1739 unmodified lines
17401740 KVM_PV_UNSHARE_ALL,
17411741 KVM_PV_INFO,
17421742 KVM_PV_DUMP,
1743+ KVM_PV_ASYNC_CLEANUP_PREPARE,
1744+ KVM_PV_ASYNC_CLEANUP_PERFORM,
17431745 };
17441746
17451747 struct kvm_pv_cmd {