lael-labs/linux

Documentation/virtual/kvm/api.txt+16−0

@@ -4391,6 +4391,22 @@ all such vmexits. · 4390 unmodified lines
4391		4391
4392	Do not enable KVM_FEATURE_PV_UNHALT if you disable HLT exits.	4392	Do not enable KVM_FEATURE_PV_UNHALT if you disable HLT exits.
4393		4393
		4394	7.14 KVM_CAP_S390_HPAGE_1M
		4395
		4396	Architectures: s390
		4397	Parameters: none
		4398	Returns: 0 on success, -EINVAL if hpage module parameter was not set
		4399	or cmma is enabled
		4400
		4401	With this capability the KVM support for memory backing with 1m pages
		4402	through hugetlbfs can be enabled for a VM. After the capability is
		4403	enabled, cmma can't be enabled anymore and pfmfi and the storage key
		4404	interpretation are disabled. If cmma has already been enabled or the
		4405	hpage module parameter is not set to 1, -EINVAL is returned.
		4406
		4407	While it is generally possible to create a huge page backed VM without
		4408	this capability, the VM will not be able to run.
		4409
4394	8. Other capabilities.	4410	8. Other capabilities.
4395	----------------------	4411	----------------------
4396		4412

arch/s390/include/asm/gmap.h+10−0

 #ifndef _ASM_S390_GMAP_H
 #define _ASM_S390_GMAP_H
+/* Generic bits for GMAP notification on DAT table entry changes. */
+#define GMAP_NOTIFY_SHADOW	0x2
+#define GMAP_NOTIFY_MPROT	0x1
+/* Status bits only for huge segment entries */
+#define _SEGMENT_ENTRY_GMAP_IN		0x8000	/* invalidation notify bit */
+#define _SEGMENT_ENTRY_GMAP_UC		0x4000	/* dirty (migration) */
 /**
  * struct gmap_struct - guest address space
  * @list: list head for the mm->context gmap list
 int gmap_mprotect_notify(struct gmap *, unsigned long start,
 			 unsigned long len, int prot);
+void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long dirty_bitmap[4],
+			     unsigned long gaddr, unsigned long vmaddr);
 #endif /* _ASM_S390_GMAP_H */

arch/s390/include/asm/hugetlb.h+4−1

 	return 0;
 }
-#define arch_clear_hugepage_flags(page)		do { } while (0)
+static inline void arch_clear_hugepage_flags(struct page *page)
+{
+	clear_bit(PG_arch_1, &page->flags);
+}
 static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
 				  pte_t *ptep, unsigned long sz)

arch/s390/include/asm/kvm_host.h+4−7

 	__u8	reserved1c0[8];		/* 0x01c0 */
 #define ECD_HOSTREGMGMT	0x20000000
 #define ECD_MEF		0x08000000
+#define ECD_ETOKENF	0x02000000
 	__u32	ecd;			/* 0x01c8 */
 	__u8	reserved1cc[18];	/* 0x01cc */
 	__u64	pp;			/* 0x01de */
 	seqcount_t cputm_seqcount;
 	__u64 cputm_start;
 	bool gs_enabled;
+	bool skey_enabled;
 };
 struct kvm_vm_stat {
 	struct page *pages[KVM_MAX_VCPUS];
 };
-struct kvm_s390_migration_state {
-	unsigned long bitmap_size;	/* in bits (number of guest pages) */
-	atomic64_t dirty_pages;		/* number of dirty pages */
-	unsigned long *pgste_bitmap;
-};
 struct kvm_arch{
 	void *sca;
 	int use_esca;
 	struct kvm_s390_vsie vsie;
 	u8 epdx;
 	u64 epoch;
-	struct kvm_s390_migration_state *migration_state;
+	int migration_mode;
+	atomic64_t cmma_dirty_pages;
 	/* subset of available cpu features enabled by user space */
 	DECLARE_BITMAP(cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
 	struct kvm_s390_gisa *gisa;

arch/s390/include/asm/mmu.h+2−0

@@ -24,6 +24,8 @@ typedef struct { · 23 unmodified lines
24	unsigned int uses_skeys:1;	24	unsigned int uses_skeys:1;
25	/* The mmu context uses CMM. */	25	/* The mmu context uses CMM. */
26	unsigned int uses_cmm:1;	26	unsigned int uses_cmm:1;
		27	/* The gmaps associated with this context are allowed to use huge pages. */
		28	unsigned int allow_gmap_hpage_1m:1;
27	} mm_context_t;	29	} mm_context_t;
28		30
29	#define INIT_MM_CONTEXT(name) \	31	#define INIT_MM_CONTEXT(name) \

arch/s390/include/asm/mmu_context.h+1−0

@@ -32,6 +32,7 @@ static inline int init_new_context(struct task_struct *tsk, · 31 unmodified lines
32	mm->context.has_pgste = 0;	32	mm->context.has_pgste = 0;
33	mm->context.uses_skeys = 0;	33	mm->context.uses_skeys = 0;
34	mm->context.uses_cmm = 0;	34	mm->context.uses_cmm = 0;
		35	mm->context.allow_gmap_hpage_1m = 0;
35	#endif	36	#endif
36	switch (mm->context.asce_limit) {	37	switch (mm->context.asce_limit) {
37	case _REGION2_SIZE:	38	case _REGION2_SIZE:

arch/s390/include/asm/pgtable.h+10−3

 #define _REGION_ENTRY_BITS_LARGE 0xffffffff8000fe2fUL
 /* Bits in the segment table entry */
-#define _SEGMENT_ENTRY_BITS	0xfffffffffffffe33UL
+#define _SEGMENT_ENTRY_BITS			0xfffffffffffffe33UL
-#define _SEGMENT_ENTRY_BITS_LARGE 0xfffffffffff0ff33UL
+#define _SEGMENT_ENTRY_BITS_LARGE		0xfffffffffff0ff33UL
+#define _SEGMENT_ENTRY_HARDWARE_BITS		0xfffffffffffffe30UL
+#define _SEGMENT_ENTRY_HARDWARE_BITS_LARGE	0xfffffffffff00730UL
 #define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address	    */
 #define _SEGMENT_ENTRY_ORIGIN	~0x7ffUL/* page table origin		    */
 #define _SEGMENT_ENTRY_PROTECT	0x200	/* segment protection bit	    */
 		    pte_t *sptep, pte_t *tptep, pte_t pte);
 void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep);
-bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long address);
+bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long address,
+			    pte_t *ptep);
 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
 			  unsigned char key, bool nq);
 int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
 int get_pgste(struct mm_struct *mm, unsigned long hva, unsigned long *pgstep);
 int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
 			unsigned long *oldpte, unsigned long *oldpgste);
+void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr);
+void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr);
+void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr);
+void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr);
 /*
  * Certain architectures need to do special things when PTEs

arch/s390/include/uapi/asm/kvm.h+4−1

 /*
  * KVM s390 specific structures and definitions
  *
- * Copyright IBM Corp. 2008
+ * Copyright IBM Corp. 2008, 2018
  *
  *    Author(s): Carsten Otte <cotte@de.ibm.com>
  *               Christian Borntraeger <borntraeger@de.ibm.com>
 #define KVM_SYNC_FPRS   (1UL << 8)
 #define KVM_SYNC_GSCB   (1UL << 9)
 #define KVM_SYNC_BPBC   (1UL << 10)
+#define KVM_SYNC_ETOKEN (1UL << 11)
 /* length and alignment of the sdnx as a power of two */
 #define SDNXC 8
 #define SDNXL (1UL << SDNXC)
 		struct {
 			__u64 reserved1[2];
 			__u64 gscb[4];
+			__u64 etoken;
+			__u64 etoken_extension;
 		};
 	};
 };

arch/s390/kvm/kvm-s390.c+250−133

@@ -172,6 +172,10 @@ static int nested; · 171 unmodified lines
172	module_param(nested, int, S_IRUGO);	172	module_param(nested, int, S_IRUGO);
173	MODULE_PARM_DESC(nested, "Nested virtualization support");	173	MODULE_PARM_DESC(nested, "Nested virtualization support");
174		174
		175	/* allow 1m huge page guest backing, if !nested */
		176	static int hpage;
		177	module_param(hpage, int, 0444);
		178	MODULE_PARM_DESC(hpage, "1m huge page backing support");
175		179
176	/*	180	/*
177	* For now we handle at most 16 double words as this is what the s390 base	181	* For now we handle at most 16 double words as this is what the s390 base
@@ -475,6 +479,11 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 297 unmodified lines
475	case KVM_CAP_S390_AIS_MIGRATION:	479	case KVM_CAP_S390_AIS_MIGRATION:
476	r = 1;	480	r = 1;
477	break;	481	break;
		482	case KVM_CAP_S390_HPAGE_1M:
		483	r = 0;
		484	if (hpage)
		485	r = 1;
		486	break;
478	case KVM_CAP_S390_MEM_OP:	487	case KVM_CAP_S390_MEM_OP:
479	r = MEM_OP_MAX_SIZE;	488	r = MEM_OP_MAX_SIZE;
480	break;	489	break;
@@ -511,19 +520,30 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 30 unmodified lines
511	}	520	}
512		521
513	static void kvm_s390_sync_dirty_log(struct kvm *kvm,	522	static void kvm_s390_sync_dirty_log(struct kvm *kvm,
514	struct kvm_memory_slot *memslot)	523	struct kvm_memory_slot *memslot)
515	{	524	{
		525	int i;
516	gfn_t cur_gfn, last_gfn;	526	gfn_t cur_gfn, last_gfn;
517	unsigned long address;	527	unsigned long gaddr, vmaddr;
518	struct gmap *gmap = kvm->arch.gmap;	528	struct gmap *gmap = kvm->arch.gmap;
		529	DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
519		530
520	/* Loop over all guest pages */	531	/* Loop over all guest segments */
		532	cur_gfn = memslot->base_gfn;
521	last_gfn = memslot->base_gfn + memslot->npages;	533	last_gfn = memslot->base_gfn + memslot->npages;
522	for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {	534	for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
523	address = gfn_to_hva_memslot(memslot, cur_gfn);	535	gaddr = gfn_to_gpa(cur_gfn);
		536	vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
		537	if (kvm_is_error_hva(vmaddr))
		538	continue;
		539
		540	bitmap_zero(bitmap, _PAGE_ENTRIES);
		541	gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
		542	for (i = 0; i < _PAGE_ENTRIES; i++) {
		543	if (test_bit(i, bitmap))
		544	mark_page_dirty(kvm, cur_gfn + i);
		545	}
524		546
525	if (test_and_clear_guest_dirty(gmap->mm, address))
526	mark_page_dirty(kvm, cur_gfn);
527	if (fatal_signal_pending(current))	547	if (fatal_signal_pending(current))
528	return;	548	return;
529	cond_resched();	549	cond_resched();
@@ -667,6 +687,27 @@ static int kvm_vm_ioctl_enable_cap(struct kvm kvm, struct kvm_enable_cap cap) · 137 unmodified lines
667	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",	687	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
668	r ? "(not available)" : "(success)");	688	r ? "(not available)" : "(success)");
669	break;	689	break;
		690	case KVM_CAP_S390_HPAGE_1M:
		691	mutex_lock(&kvm->lock);
		692	if (kvm->created_vcpus)
		693	r = -EBUSY;
		694	else if (!hpage \|\| kvm->arch.use_cmma)
		695	r = -EINVAL;
		696	else {
		697	r = 0;
		698	kvm->mm->context.allow_gmap_hpage_1m = 1;
		699	/*
		700	* We might have to create fake 4k page
		701	* tables. To avoid that the hardware works on
		702	* stale PGSTEs, we emulate these instructions.
		703	*/
		704	kvm->arch.use_skf = 0;
		705	kvm->arch.use_pfmfi = 0;
		706	}
		707	mutex_unlock(&kvm->lock);
		708	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
		709	r ? "(not available)" : "(success)");
		710	break;
670	case KVM_CAP_S390_USER_STSI:	711	case KVM_CAP_S390_USER_STSI:
671	VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");	712	VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
672	kvm->arch.user_stsi = 1;	713	kvm->arch.user_stsi = 1;
@@ -714,10 +755,13 @@ static int kvm_s390_set_mem_control(struct kvm kvm, struct kvm_device_attr att · 41 unmodified lines
714	if (!sclp.has_cmma)	755	if (!sclp.has_cmma)
715	break;	756	break;
716		757
717	ret = -EBUSY;
718	VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");	758	VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
719	mutex_lock(&kvm->lock);	759	mutex_lock(&kvm->lock);
720	if (!kvm->created_vcpus) {	760	if (kvm->created_vcpus)
		761	ret = -EBUSY;
		762	else if (kvm->mm->context.allow_gmap_hpage_1m)
		763	ret = -EINVAL;
		764	else {
721	kvm->arch.use_cmma = 1;	765	kvm->arch.use_cmma = 1;
722	/* Not compatible with cmma. */	766	/* Not compatible with cmma. */
723	kvm->arch.use_pfmfi = 0;	767	kvm->arch.use_pfmfi = 0;
@@ -862,54 +906,37 @@ static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req) · 138 unmodified lines
862	*/	906	*/
863	static int kvm_s390_vm_start_migration(struct kvm *kvm)	907	static int kvm_s390_vm_start_migration(struct kvm *kvm)
864	{	908	{
865	struct kvm_s390_migration_state *mgs;
866	struct kvm_memory_slot *ms;	909	struct kvm_memory_slot *ms;
867	/* should be the only one */
868	struct kvm_memslots *slots;	910	struct kvm_memslots *slots;
869	unsigned long ram_pages;	911	unsigned long ram_pages = 0;
870	int slotnr;	912	int slotnr;
871		913
872	/* migration mode already enabled */	914	/* migration mode already enabled */
873	if (kvm->arch.migration_state)	915	if (kvm->arch.migration_mode)
874	return 0;	916	return 0;
875
876	slots = kvm_memslots(kvm);	917	slots = kvm_memslots(kvm);
877	if (!slots \|\| !slots->used_slots)	918	if (!slots \|\| !slots->used_slots)
878	return -EINVAL;	919	return -EINVAL;
879		920
880	mgs = kzalloc(sizeof(*mgs), GFP_KERNEL);	921	if (!kvm->arch.use_cmma) {
881	if (!mgs)	922	kvm->arch.migration_mode = 1;
882	return -ENOMEM;	923	return 0;
883	kvm->arch.migration_state = mgs;	924	}
884		925	/* mark all the pages in active slots as dirty */
885	if (kvm->arch.use_cmma) {	926	for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
		927	ms = slots->memslots + slotnr;
886	/*	928	/*
887	* Get the first slot. They are reverse sorted by base_gfn, so	929	* The second half of the bitmap is only used on x86,
888	* the first slot is also the one at the end of the address	930	* and would be wasted otherwise, so we put it to good
889	* space. We have verified above that at least one slot is	931	* use here to keep track of the state of the storage
890	* present.	932	* attributes.
891	*/	933	*/
892	ms = slots->memslots;	934	memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
893	/* round up so we only use full longs */	935	ram_pages += ms->npages;
894	ram_pages = roundup(ms->base_gfn + ms->npages, BITS_PER_LONG);
895	/* allocate enough bytes to store all the bits */
896	mgs->pgste_bitmap = vmalloc(ram_pages / 8);
897	if (!mgs->pgste_bitmap) {
898	kfree(mgs);
899	kvm->arch.migration_state = NULL;
900	return -ENOMEM;
901	}
902
903	mgs->bitmap_size = ram_pages;
904	atomic64_set(&mgs->dirty_pages, ram_pages);
905	/* mark all the pages in active slots as dirty */
906	for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
907	ms = slots->memslots + slotnr;
908	bitmap_set(mgs->pgste_bitmap, ms->base_gfn, ms->npages);
909	}
910
911	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
912	}	936	}
		937	atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
		938	kvm->arch.migration_mode = 1;
		939	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
913	return 0;	940	return 0;
914	}	941	}
915		942
@@ -919,21 +946,12 @@ static int kvm_s390_vm_start_migration(struct kvm *kvm) · 3 unmodified lines
919	*/	946	*/
920	static int kvm_s390_vm_stop_migration(struct kvm *kvm)	947	static int kvm_s390_vm_stop_migration(struct kvm *kvm)
921	{	948	{
922	struct kvm_s390_migration_state *mgs;
923
924	/* migration mode already disabled */	949	/* migration mode already disabled */
925	if (!kvm->arch.migration_state)	950	if (!kvm->arch.migration_mode)
926	return 0;	951	return 0;
927	mgs = kvm->arch.migration_state;	952	kvm->arch.migration_mode = 0;
928	kvm->arch.migration_state = NULL;	953	if (kvm->arch.use_cmma)
929
930	if (kvm->arch.use_cmma) {
931	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);	954	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
932	/* We have to wait for the essa emulation to finish */
933	synchronize_srcu(&kvm->srcu);
934	vfree(mgs->pgste_bitmap);
935	}
936	kfree(mgs);
937	return 0;	955	return 0;
938	}	956	}
939		957
@@ -961,7 +979,7 @@ static int kvm_s390_vm_set_migration(struct kvm *kvm, · 21 unmodified lines
961	static int kvm_s390_vm_get_migration(struct kvm *kvm,	979	static int kvm_s390_vm_get_migration(struct kvm *kvm,
962	struct kvm_device_attr *attr)	980	struct kvm_device_attr *attr)
963	{	981	{
964	u64 mig = (kvm->arch.migration_state != NULL);	982	u64 mig = kvm->arch.migration_mode;
965		983
966	if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)	984	if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
967	return -ENXIO;	985	return -ENXIO;
@@ -1540,6 +1558,7 @@ static long kvm_s390_set_skeys(struct kvm kvm, struct kvm_s390_skeys args) · 572 unmodified lines
1540	uint8_t *keys;	1558	uint8_t *keys;
1541	uint64_t hva;	1559	uint64_t hva;
1542	int srcu_idx, i, r = 0;	1560	int srcu_idx, i, r = 0;
		1561	bool unlocked;
1543		1562
1544	if (args->flags != 0)	1563	if (args->flags != 0)
1545	return -EINVAL;	1564	return -EINVAL;
@@ -1564,9 +1583,11 @@ static long kvm_s390_set_skeys(struct kvm kvm, struct kvm_s390_skeys args) · 18 unmodified lines
1564	if (r)	1583	if (r)
1565	goto out;	1584	goto out;
1566		1585
		1586	i = 0;
1567	down_read(&current->mm->mmap_sem);	1587	down_read(&current->mm->mmap_sem);
1568	srcu_idx = srcu_read_lock(&kvm->srcu);	1588	srcu_idx = srcu_read_lock(&kvm->srcu);
1569	for (i = 0; i < args->count; i++) {	1589	while (i < args->count) {
		1590	unlocked = false;
1570	hva = gfn_to_hva(kvm, args->start_gfn + i);	1591	hva = gfn_to_hva(kvm, args->start_gfn + i);
1571	if (kvm_is_error_hva(hva)) {	1592	if (kvm_is_error_hva(hva)) {
1572	r = -EFAULT;	1593	r = -EFAULT;
@@ -1580,8 +1601,14 @@ static long kvm_s390_set_skeys(struct kvm kvm, struct kvm_s390_skeys args) · 7 unmodified lines
1580	}	1601	}
1581		1602
1582	r = set_guest_storage_key(current->mm, hva, keys[i], 0);	1603	r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1583	if (r)	1604	if (r) {
1584	break;	1605	r = fixup_user_fault(current, current->mm, hva,
		1606	FAULT_FLAG_WRITE, &unlocked);
		1607	if (r)
		1608	break;
		1609	}
		1610	if (!r)
		1611	i++;
1585	}	1612	}
1586	srcu_read_unlock(&kvm->srcu, srcu_idx);	1613	srcu_read_unlock(&kvm->srcu, srcu_idx);
1587	up_read(&current->mm->mmap_sem);	1614	up_read(&current->mm->mmap_sem);
@@ -1599,6 +1626,134 @@ out: · 11 unmodified lines
1599	/* for consistency */	1626	/* for consistency */
1600	#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)	1627	#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1601		1628
		1629	/*
		1630	* Similar to gfn_to_memslot, but returns the index of a memslot also when the
		1631	* address falls in a hole. In that case the index of one of the memslots
		1632	* bordering the hole is returned.
		1633	*/
		1634	static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
		1635	{
		1636	int start = 0, end = slots->used_slots;
		1637	int slot = atomic_read(&slots->lru_slot);
		1638	struct kvm_memory_slot *memslots = slots->memslots;
		1639
		1640	if (gfn >= memslots[slot].base_gfn &&
		1641	gfn < memslots[slot].base_gfn + memslots[slot].npages)
		1642	return slot;
		1643
		1644	while (start < end) {
		1645	slot = start + (end - start) / 2;
		1646
		1647	if (gfn >= memslots[slot].base_gfn)
		1648	end = slot;
		1649	else
		1650	start = slot + 1;
		1651	}
		1652
		1653	if (gfn >= memslots[start].base_gfn &&
		1654	gfn < memslots[start].base_gfn + memslots[start].npages) {
		1655	atomic_set(&slots->lru_slot, start);
		1656	}
		1657
		1658	return start;
		1659	}
		1660
		1661	static int kvm_s390_peek_cmma(struct kvm kvm, struct kvm_s390_cmma_log args,
		1662	u8 *res, unsigned long bufsize)
		1663	{
		1664	unsigned long pgstev, hva, cur_gfn = args->start_gfn;
		1665
		1666	args->count = 0;
		1667	while (args->count < bufsize) {
		1668	hva = gfn_to_hva(kvm, cur_gfn);
		1669	/*
		1670	* We return an error if the first value was invalid, but we
		1671	* return successfully if at least one value was copied.
		1672	*/
		1673	if (kvm_is_error_hva(hva))
		1674	return args->count ? 0 : -EFAULT;
		1675	if (get_pgste(kvm->mm, hva, &pgstev) < 0)
		1676	pgstev = 0;
		1677	res[args->count++] = (pgstev >> 24) & 0x43;
		1678	cur_gfn++;
		1679	}
		1680
		1681	return 0;
		1682	}
		1683
		1684	static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
		1685	unsigned long cur_gfn)
		1686	{
		1687	int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
		1688	struct kvm_memory_slot *ms = slots->memslots + slotidx;
		1689	unsigned long ofs = cur_gfn - ms->base_gfn;
		1690
		1691	if (ms->base_gfn + ms->npages <= cur_gfn) {
		1692	slotidx--;
		1693	/* If we are above the highest slot, wrap around */
		1694	if (slotidx < 0)
		1695	slotidx = slots->used_slots - 1;
		1696
		1697	ms = slots->memslots + slotidx;
		1698	ofs = 0;
		1699	}
		1700	ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
		1701	while ((slotidx > 0) && (ofs >= ms->npages)) {
		1702	slotidx--;
		1703	ms = slots->memslots + slotidx;
		1704	ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
		1705	}
		1706	return ms->base_gfn + ofs;
		1707	}
		1708
		1709	static int kvm_s390_get_cmma(struct kvm kvm, struct kvm_s390_cmma_log args,
		1710	u8 *res, unsigned long bufsize)
		1711	{
		1712	unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
		1713	struct kvm_memslots *slots = kvm_memslots(kvm);
		1714	struct kvm_memory_slot *ms;
		1715
		1716	cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
		1717	ms = gfn_to_memslot(kvm, cur_gfn);
		1718	args->count = 0;
		1719	args->start_gfn = cur_gfn;
		1720	if (!ms)
		1721	return 0;
		1722	next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
		1723	mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
		1724
		1725	while (args->count < bufsize) {
		1726	hva = gfn_to_hva(kvm, cur_gfn);
		1727	if (kvm_is_error_hva(hva))
		1728	return 0;
		1729	/* Decrement only if we actually flipped the bit to 0 */
		1730	if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
		1731	atomic64_dec(&kvm->arch.cmma_dirty_pages);
		1732	if (get_pgste(kvm->mm, hva, &pgstev) < 0)
		1733	pgstev = 0;
		1734	/* Save the value */
		1735	res[args->count++] = (pgstev >> 24) & 0x43;
		1736	/* If the next bit is too far away, stop. */
		1737	if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
		1738	return 0;
		1739	/* If we reached the previous "next", find the next one */
		1740	if (cur_gfn == next_gfn)
		1741	next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
		1742	/* Reached the end of memory or of the buffer, stop */
		1743	if ((next_gfn >= mem_end) \|\|
		1744	(next_gfn - args->start_gfn >= bufsize))
		1745	return 0;
		1746	cur_gfn++;
		1747	/* Reached the end of the current memslot, take the next one. */
		1748	if (cur_gfn - ms->base_gfn >= ms->npages) {
		1749	ms = gfn_to_memslot(kvm, cur_gfn);
		1750	if (!ms)
		1751	return 0;
		1752	}
		1753	}
		1754	return 0;
		1755	}
		1756
1602	/*	1757	/*
1603	* This function searches for the next page with dirty CMMA attributes, and	1758	* This function searches for the next page with dirty CMMA attributes, and
1604	* saves the attributes in the buffer up to either the end of the buffer or	1759	* saves the attributes in the buffer up to either the end of the buffer or
@@ -1610,22 +1765,18 @@ out: · 5 unmodified lines
1610	static int kvm_s390_get_cmma_bits(struct kvm *kvm,	1765	static int kvm_s390_get_cmma_bits(struct kvm *kvm,
1611	struct kvm_s390_cmma_log *args)	1766	struct kvm_s390_cmma_log *args)
1612	{	1767	{
1613	struct kvm_s390_migration_state *s = kvm->arch.migration_state;	1768	unsigned long bufsize;
1614	unsigned long bufsize, hva, pgstev, i, next, cur;	1769	int srcu_idx, peek, ret;
1615	int srcu_idx, peek, r = 0, rr;	1770	u8 *values;
1616	u8 *res;
1617
1618	cur = args->start_gfn;
1619	i = next = pgstev = 0;
1620		1771
1621	if (unlikely(!kvm->arch.use_cmma))	1772	if (!kvm->arch.use_cmma)
1622	return -ENXIO;	1773	return -ENXIO;
1623	/* Invalid/unsupported flags were specified */	1774	/* Invalid/unsupported flags were specified */
1624	if (args->flags & ~KVM_S390_CMMA_PEEK)	1775	if (args->flags & ~KVM_S390_CMMA_PEEK)
1625	return -EINVAL;	1776	return -EINVAL;
1626	/* Migration mode query, and we are not doing a migration */	1777	/* Migration mode query, and we are not doing a migration */
1627	peek = !!(args->flags & KVM_S390_CMMA_PEEK);	1778	peek = !!(args->flags & KVM_S390_CMMA_PEEK);
1628	if (!peek && !s)	1779	if (!peek && !kvm->arch.migration_mode)
1629	return -EINVAL;	1780	return -EINVAL;
1630	/* CMMA is disabled or was not used, or the buffer has length zero */	1781	/* CMMA is disabled or was not used, or the buffer has length zero */
1631	bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);	1782	bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
@@ -1633,74 +1784,35 @@ static int kvm_s390_get_cmma_bits(struct kvm *kvm, · 1 unmodified line
1633	memset(args, 0, sizeof(*args));	1784	memset(args, 0, sizeof(*args));
1634	return 0;	1785	return 0;
1635	}	1786	}
1636		1787	/* We are not peeking, and there are no dirty pages */
1637	if (!peek) {	1788	if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
1638	/* We are not peeking, and there are no dirty pages */	1789	memset(args, 0, sizeof(*args));
1639	if (!atomic64_read(&s->dirty_pages)) {	1790	return 0;
1640	memset(args, 0, sizeof(*args));
1641	return 0;
1642	}
1643	cur = find_next_bit(s->pgste_bitmap, s->bitmap_size,
1644	args->start_gfn);
1645	if (cur >= s->bitmap_size) /* nothing found, loop back */
1646	cur = find_next_bit(s->pgste_bitmap, s->bitmap_size, 0);
1647	if (cur >= s->bitmap_size) { /* again! (very unlikely) */
1648	memset(args, 0, sizeof(*args));
1649	return 0;
1650	}
1651	next = find_next_bit(s->pgste_bitmap, s->bitmap_size, cur + 1);
1652	}	1791	}
1653		1792
1654	res = vmalloc(bufsize);	1793	values = vmalloc(bufsize);
1655	if (!res)	1794	if (!values)
1656	return -ENOMEM;	1795	return -ENOMEM;
1657		1796
1658	args->start_gfn = cur;
1659
1660	down_read(&kvm->mm->mmap_sem);	1797	down_read(&kvm->mm->mmap_sem);
1661	srcu_idx = srcu_read_lock(&kvm->srcu);	1798	srcu_idx = srcu_read_lock(&kvm->srcu);
1662	while (i < bufsize) {	1799	if (peek)
1663	hva = gfn_to_hva(kvm, cur);	1800	ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
1664	if (kvm_is_error_hva(hva)) {	1801	else
1665	r = -EFAULT;	1802	ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
1666	break;
1667	}
1668	/* decrement only if we actually flipped the bit to 0 */
1669	if (!peek && test_and_clear_bit(cur, s->pgste_bitmap))
1670	atomic64_dec(&s->dirty_pages);
1671	r = get_pgste(kvm->mm, hva, &pgstev);
1672	if (r < 0)
1673	pgstev = 0;
1674	/* save the value */
1675	res[i++] = (pgstev >> 24) & 0x43;
1676	/*
1677	* if the next bit is too far away, stop.
1678	* if we reached the previous "next", find the next one
1679	*/
1680	if (!peek) {
1681	if (next > cur + KVM_S390_MAX_BIT_DISTANCE)
1682	break;
1683	if (cur == next)
1684	next = find_next_bit(s->pgste_bitmap,
1685	s->bitmap_size, cur + 1);
1686	/* reached the end of the bitmap or of the buffer, stop */
1687	if ((next >= s->bitmap_size) \|\|
1688	(next >= args->start_gfn + bufsize))
1689	break;
1690	}
1691	cur++;
1692	}
1693	srcu_read_unlock(&kvm->srcu, srcu_idx);	1803	srcu_read_unlock(&kvm->srcu, srcu_idx);
1694	up_read(&kvm->mm->mmap_sem);	1804	up_read(&kvm->mm->mmap_sem);
1695	args->count = i;
1696	args->remaining = s ? atomic64_read(&s->dirty_pages) : 0;
1697		1805
1698	rr = copy_to_user((void __user *)args->values, res, args->count);	1806	if (kvm->arch.migration_mode)
1699	if (rr)	1807	args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
1700	r = -EFAULT;	1808	else
		1809	args->remaining = 0;
1701		1810
1702	vfree(res);	1811	if (copy_to_user((void __user *)args->values, values, args->count))
1703	return r;	1812	ret = -EFAULT;
		1813
		1814	vfree(values);
		1815	return ret;
1704	}	1816	}
1705		1817
1706	/*	1818	/*
@@ -2139,10 +2251,6 @@ void kvm_arch_destroy_vm(struct kvm *kvm) · 432 unmodified lines
2139	kvm_s390_destroy_adapters(kvm);	2251	kvm_s390_destroy_adapters(kvm);
2140	kvm_s390_clear_float_irqs(kvm);	2252	kvm_s390_clear_float_irqs(kvm);
2141	kvm_s390_vsie_destroy(kvm);	2253	kvm_s390_vsie_destroy(kvm);
2142	if (kvm->arch.migration_state) {
2143	vfree(kvm->arch.migration_state->pgste_bitmap);
2144	kfree(kvm->arch.migration_state);
2145	}
2146	KVM_EVENT(3, "vm 0x%pK destroyed", kvm);	2254	KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2147	}	2255	}
2148		2256
@@ -2300,6 +2408,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) · 151 unmodified lines
2300	vcpu->run->kvm_valid_regs \|= KVM_SYNC_BPBC;	2408	vcpu->run->kvm_valid_regs \|= KVM_SYNC_BPBC;
2301	if (test_kvm_facility(vcpu->kvm, 133))	2409	if (test_kvm_facility(vcpu->kvm, 133))
2302	vcpu->run->kvm_valid_regs \|= KVM_SYNC_GSCB;	2410	vcpu->run->kvm_valid_regs \|= KVM_SYNC_GSCB;
		2411	if (test_kvm_facility(vcpu->kvm, 156))
		2412	vcpu->run->kvm_valid_regs \|= KVM_SYNC_ETOKEN;
2303	/* fprs can be synchronized via vrs, even if the guest has no vx. With	2413	/* fprs can be synchronized via vrs, even if the guest has no vx. With
2304	* MACHINE_HAS_VX, (load\|store)_fpu_regs() will work with vrs format.	2414	* MACHINE_HAS_VX, (load\|store)_fpu_regs() will work with vrs format.
2305	*/	2415	*/
@@ -2549,7 +2659,8 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) · 243 unmodified lines
2549	}	2659	}
2550	if (test_kvm_facility(vcpu->kvm, 139))	2660	if (test_kvm_facility(vcpu->kvm, 139))
2551	vcpu->arch.sie_block->ecd \|= ECD_MEF;	2661	vcpu->arch.sie_block->ecd \|= ECD_MEF;
2552		2662	if (test_kvm_facility(vcpu->kvm, 156))
		2663	vcpu->arch.sie_block->ecd \|= ECD_ETOKENF;
2553	if (vcpu->arch.sie_block->gd) {	2664	if (vcpu->arch.sie_block->gd) {
2554	vcpu->arch.sie_block->eca \|= ECA_AIV;	2665	vcpu->arch.sie_block->eca \|= ECA_AIV;
2555	VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",	2666	VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
@@ -3467,6 +3578,7 @@ static void sync_regs(struct kvm_vcpu vcpu, struct kvm_run kvm_run) · 911 unmodified lines
3467	}	3578	}
3468	preempt_enable();	3579	preempt_enable();
3469	}	3580	}
		3581	/* SIE will load etoken directly from SDNX and therefore kvm_run */
3470		3582
3471	kvm_run->kvm_dirty_regs = 0;	3583	kvm_run->kvm_dirty_regs = 0;
3472	}	3584	}
@@ -3506,7 +3618,7 @@ static void store_regs(struct kvm_vcpu vcpu, struct kvm_run kvm_run) · 33 unmodified lines
3506	__ctl_clear_bit(2, 4);	3618	__ctl_clear_bit(2, 4);
3507	vcpu->arch.host_gscb = NULL;	3619	vcpu->arch.host_gscb = NULL;
3508	}	3620	}
3509		3621	/* SIE will save etoken directly into SDNX and therefore kvm_run */
3510	}	3622	}
3511		3623
3512	int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu vcpu, struct kvm_run kvm_run)	3624	int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu vcpu, struct kvm_run kvm_run)
@@ -4082,6 +4194,11 @@ static int __init kvm_s390_init(void) · 569 unmodified lines
4082	return -ENODEV;	4194	return -ENODEV;
4083	}	4195	}
4084		4196
		4197	if (nested && hpage) {
		4198	pr_info("nested (vSIE) and hpage (huge page backing) can currently not be activated concurrently");
		4199	return -EINVAL;
		4200	}
		4201
4085	for (i = 0; i < 16; i++)	4202	for (i = 0; i < 16; i++)
4086	kvm_s390_fac_base[i] \|=	4203	kvm_s390_fac_base[i] \|=
4087	S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);	4204	S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);

arch/s390/kvm/priv.c+94−49

@@ -205,13 +205,10 @@ static int handle_store_cpu_address(struct kvm_vcpu *vcpu) · 204 unmodified lines
205	int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu)	205	int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu)
206	{	206	{
207	int rc;	207	int rc;
208	struct kvm_s390_sie_block *sie_block = vcpu->arch.sie_block;
209		208
210	trace_kvm_s390_skey_related_inst(vcpu);	209	trace_kvm_s390_skey_related_inst(vcpu);
211	/* Already enabled? */	210	/* Already enabled? */
212	if (vcpu->kvm->arch.use_skf &&	211	if (vcpu->arch.skey_enabled)
213	!(sie_block->ictl & (ICTL_ISKE \| ICTL_SSKE \| ICTL_RRBE)) &&
214	!kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
215	return 0;	212	return 0;
216		213
217	rc = s390_enable_skey();	214	rc = s390_enable_skey();
@@ -222,9 +219,10 @@ int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu) · 4 unmodified lines
222	if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))	219	if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
223	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS);	220	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS);
224	if (!vcpu->kvm->arch.use_skf)	221	if (!vcpu->kvm->arch.use_skf)
225	sie_block->ictl \|= ICTL_ISKE \| ICTL_SSKE \| ICTL_RRBE;	222	vcpu->arch.sie_block->ictl \|= ICTL_ISKE \| ICTL_SSKE \| ICTL_RRBE;
226	else	223	else
227	sie_block->ictl &= ~(ICTL_ISKE \| ICTL_SSKE \| ICTL_RRBE);	224	vcpu->arch.sie_block->ictl &= ~(ICTL_ISKE \| ICTL_SSKE \| ICTL_RRBE);
		225	vcpu->arch.skey_enabled = true;
228	return 0;	226	return 0;
229	}	227	}
230		228
@@ -246,9 +244,10 @@ static int try_handle_skey(struct kvm_vcpu *vcpu) · 15 unmodified lines
246		244
247	static int handle_iske(struct kvm_vcpu *vcpu)	245	static int handle_iske(struct kvm_vcpu *vcpu)
248	{	246	{
249	unsigned long addr;	247	unsigned long gaddr, vmaddr;
250	unsigned char key;	248	unsigned char key;
251	int reg1, reg2;	249	int reg1, reg2;
		250	bool unlocked;
252	int rc;	251	int rc;
253		252
254	vcpu->stat.instruction_iske++;	253	vcpu->stat.instruction_iske++;
@@ -262,18 +261,28 @@ static int handle_iske(struct kvm_vcpu *vcpu) · 7 unmodified lines
262		261
263	kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);	262	kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);
264		263
265	addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;	264	gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
266	addr = kvm_s390_logical_to_effective(vcpu, addr);	265	gaddr = kvm_s390_logical_to_effective(vcpu, gaddr);
267	addr = kvm_s390_real_to_abs(vcpu, addr);	266	gaddr = kvm_s390_real_to_abs(vcpu, gaddr);
268	addr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(addr));	267	vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr));
269	if (kvm_is_error_hva(addr))	268	if (kvm_is_error_hva(vmaddr))
270	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);	269	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
271		270	retry:
		271	unlocked = false;
272	down_read(&current->mm->mmap_sem);	272	down_read(&current->mm->mmap_sem);
273	rc = get_guest_storage_key(current->mm, addr, &key);	273	rc = get_guest_storage_key(current->mm, vmaddr, &key);
274	up_read(&current->mm->mmap_sem);	274
		275	if (rc) {
		276	rc = fixup_user_fault(current, current->mm, vmaddr,
		277	FAULT_FLAG_WRITE, &unlocked);
		278	if (!rc) {
		279	up_read(&current->mm->mmap_sem);
		280	goto retry;
		281	}
		282	}
275	if (rc)	283	if (rc)
276	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);	284	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
		285	up_read(&current->mm->mmap_sem);
277	vcpu->run->s.regs.gprs[reg1] &= ~0xff;	286	vcpu->run->s.regs.gprs[reg1] &= ~0xff;
278	vcpu->run->s.regs.gprs[reg1] \|= key;	287	vcpu->run->s.regs.gprs[reg1] \|= key;
279	return 0;	288	return 0;
@@ -281,8 +290,9 @@ static int handle_iske(struct kvm_vcpu *vcpu) · 1 unmodified line
281		290
282	static int handle_rrbe(struct kvm_vcpu *vcpu)	291	static int handle_rrbe(struct kvm_vcpu *vcpu)
283	{	292	{
284	unsigned long addr;	293	unsigned long vmaddr, gaddr;
285	int reg1, reg2;	294	int reg1, reg2;
		295	bool unlocked;
286	int rc;	296	int rc;
287		297
288	vcpu->stat.instruction_rrbe++;	298	vcpu->stat.instruction_rrbe++;
@@ -296,19 +306,27 @@ static int handle_rrbe(struct kvm_vcpu *vcpu) · 7 unmodified lines
296		306
297	kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);	307	kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);
298		308
299	addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;	309	gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
300	addr = kvm_s390_logical_to_effective(vcpu, addr);	310	gaddr = kvm_s390_logical_to_effective(vcpu, gaddr);
301	addr = kvm_s390_real_to_abs(vcpu, addr);	311	gaddr = kvm_s390_real_to_abs(vcpu, gaddr);
302	addr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(addr));	312	vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr));
303	if (kvm_is_error_hva(addr))	313	if (kvm_is_error_hva(vmaddr))
304	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);	314	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
305		315	retry:
		316	unlocked = false;
306	down_read(&current->mm->mmap_sem);	317	down_read(&current->mm->mmap_sem);
307	rc = reset_guest_reference_bit(current->mm, addr);	318	rc = reset_guest_reference_bit(current->mm, vmaddr);
308	up_read(&current->mm->mmap_sem);	319	if (rc < 0) {
		320	rc = fixup_user_fault(current, current->mm, vmaddr,
		321	FAULT_FLAG_WRITE, &unlocked);
		322	if (!rc) {
		323	up_read(&current->mm->mmap_sem);
		324	goto retry;
		325	}
		326	}
309	if (rc < 0)	327	if (rc < 0)
310	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);	328	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
311		329	up_read(&current->mm->mmap_sem);
312	kvm_s390_set_psw_cc(vcpu, rc);	330	kvm_s390_set_psw_cc(vcpu, rc);
313	return 0;	331	return 0;
314	}	332	}
@@ -323,6 +341,7 @@ static int handle_sske(struct kvm_vcpu *vcpu) · 8 unmodified lines
323	unsigned long start, end;	341	unsigned long start, end;
324	unsigned char key, oldkey;	342	unsigned char key, oldkey;
325	int reg1, reg2;	343	int reg1, reg2;
		344	bool unlocked;
326	int rc;	345	int rc;
327		346
328	vcpu->stat.instruction_sske++;	347	vcpu->stat.instruction_sske++;
@@ -355,19 +374,28 @@ static int handle_sske(struct kvm_vcpu *vcpu) · 26 unmodified lines
355	}	374	}
356		375
357	while (start != end) {	376	while (start != end) {
358	unsigned long addr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start));	377	unsigned long vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start));
		378	unlocked = false;
359		379
360	if (kvm_is_error_hva(addr))	380	if (kvm_is_error_hva(vmaddr))
361	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);	381	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
362		382
363	down_read(&current->mm->mmap_sem);	383	down_read(&current->mm->mmap_sem);
364	rc = cond_set_guest_storage_key(current->mm, addr, key, &oldkey,	384	rc = cond_set_guest_storage_key(current->mm, vmaddr, key, &oldkey,
365	m3 & SSKE_NQ, m3 & SSKE_MR,	385	m3 & SSKE_NQ, m3 & SSKE_MR,
366	m3 & SSKE_MC);	386	m3 & SSKE_MC);
367	up_read(&current->mm->mmap_sem);	387
368	if (rc < 0)	388	if (rc < 0) {
		389	rc = fixup_user_fault(current, current->mm, vmaddr,
		390	FAULT_FLAG_WRITE, &unlocked);
		391	rc = !rc ? -EAGAIN : rc;
		392	}
		393	if (rc == -EFAULT)
369	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);	394	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
370	start += PAGE_SIZE;	395
		396	up_read(&current->mm->mmap_sem);
		397	if (rc >= 0)
		398	start += PAGE_SIZE;
371	}	399	}
372		400
373	if (m3 & (SSKE_MC \| SSKE_MR)) {	401	if (m3 & (SSKE_MC \| SSKE_MR)) {
@@ -948,15 +976,16 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) · 574 unmodified lines
948	}	976	}
949		977
950	while (start != end) {	978	while (start != end) {
951	unsigned long useraddr;	979	unsigned long vmaddr;
		980	bool unlocked = false;
952		981
953	/* Translate guest address to host address */	982	/* Translate guest address to host address */
954	useraddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start));	983	vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start));
955	if (kvm_is_error_hva(useraddr))	984	if (kvm_is_error_hva(vmaddr))
956	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);	985	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
957		986
958	if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) {	987	if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) {
959	if (clear_user((void __user *)useraddr, PAGE_SIZE))	988	if (kvm_clear_guest(vcpu->kvm, start, PAGE_SIZE))
960	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);	989	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
961	}	990	}
962		991
@@ -966,14 +995,20 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) · 3 unmodified lines
966	if (rc)	995	if (rc)
967	return rc;	996	return rc;
968	down_read(&current->mm->mmap_sem);	997	down_read(&current->mm->mmap_sem);
969	rc = cond_set_guest_storage_key(current->mm, useraddr,	998	rc = cond_set_guest_storage_key(current->mm, vmaddr,
970	key, NULL, nq, mr, mc);	999	key, NULL, nq, mr, mc);
971	up_read(&current->mm->mmap_sem);	1000	if (rc < 0) {
972	if (rc < 0)	1001	rc = fixup_user_fault(current, current->mm, vmaddr,
		1002	FAULT_FLAG_WRITE, &unlocked);
		1003	rc = !rc ? -EAGAIN : rc;
		1004	}
		1005	if (rc == -EFAULT)
973	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);	1006	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
974	}
975		1007
976	start += PAGE_SIZE;	1008	up_read(&current->mm->mmap_sem);
		1009	if (rc >= 0)
		1010	start += PAGE_SIZE;
		1011	}
977	}	1012	}
978	if (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) {	1013	if (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) {
979	if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) {	1014	if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) {
@@ -987,9 +1022,11 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) · 7 unmodified lines
987	return 0;	1022	return 0;
988	}	1023	}
989		1024
990	static inline int do_essa(struct kvm_vcpu *vcpu, const int orc)	1025	/*
		1026	* Must be called with relevant read locks held (kvm->mm->mmap_sem, kvm->srcu)
		1027	*/
		1028	static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc)
991	{	1029	{
992	struct kvm_s390_migration_state *ms = vcpu->kvm->arch.migration_state;
993	int r1, r2, nappended, entries;	1030	int r1, r2, nappended, entries;
994	unsigned long gfn, hva, res, pgstev, ptev;	1031	unsigned long gfn, hva, res, pgstev, ptev;
995	unsigned long *cbrlo;	1032	unsigned long *cbrlo;
@@ -1039,10 +1076,12 @@ static inline int do_essa(struct kvm_vcpu *vcpu, const int orc) · 43 unmodified lines
1039	cbrlo[entries] = gfn << PAGE_SHIFT;	1076	cbrlo[entries] = gfn << PAGE_SHIFT;
1040	}	1077	}
1041		1078
1042	if (orc && gfn < ms->bitmap_size) {	1079	if (orc) {
1043	/* increment only if we are really flipping the bit to 1 */	1080	struct kvm_memory_slot *ms = gfn_to_memslot(vcpu->kvm, gfn);
1044	if (!test_and_set_bit(gfn, ms->pgste_bitmap))	1081
1045	atomic64_inc(&ms->dirty_pages);	1082	/* Increment only if we are really flipping the bit */
		1083	if (ms && !test_and_set_bit(gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
		1084	atomic64_inc(&vcpu->kvm->arch.cmma_dirty_pages);
1046	}	1085	}
1047		1086
1048	return nappended;	1087	return nappended;
@@ -1071,7 +1110,7 @@ static int handle_essa(struct kvm_vcpu *vcpu) · 22 unmodified lines
1071	: ESSA_SET_STABLE_IF_RESIDENT))	1110	: ESSA_SET_STABLE_IF_RESIDENT))
1072	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);	1111	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1073		1112
1074	if (likely(!vcpu->kvm->arch.migration_state)) {	1113	if (!vcpu->kvm->arch.migration_mode) {
1075	/*	1114	/*
1076	* CMMA is enabled in the KVM settings, but is disabled in	1115	* CMMA is enabled in the KVM settings, but is disabled in
1077	* the SIE block and in the mm_context, and we are not doing	1116	* the SIE block and in the mm_context, and we are not doing
@@ -1099,10 +1138,16 @@ static int handle_essa(struct kvm_vcpu *vcpu) · 21 unmodified lines
1099	/* Retry the ESSA instruction */	1138	/* Retry the ESSA instruction */
1100	kvm_s390_retry_instr(vcpu);	1139	kvm_s390_retry_instr(vcpu);
1101	} else {	1140	} else {
1102	/* Account for the possible extra cbrl entry */	1141	int srcu_idx;
1103	i = do_essa(vcpu, orc);	1142
		1143	down_read(&vcpu->kvm->mm->mmap_sem);
		1144	srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
		1145	i = __do_essa(vcpu, orc);
		1146	srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
		1147	up_read(&vcpu->kvm->mm->mmap_sem);
1104	if (i < 0)	1148	if (i < 0)
1105	return i;	1149	return i;
		1150	/* Account for the possible extra cbrl entry */
1106	entries += i;	1151	entries += i;
1107	}	1152	}
1108	vcpu->arch.sie_block->cbrlo &= PAGE_MASK; /* reset nceo */	1153	vcpu->arch.sie_block->cbrlo &= PAGE_MASK; /* reset nceo */

arch/s390/kvm/vsie.c+9−2

 /*
  * kvm nested virtualization support for s390x
  *
- * Copyright IBM Corp. 2016
+ * Copyright IBM Corp. 2016, 2018
  *
  *    Author(s): David Hildenbrand <dahi@linux.vnet.ibm.com>
  */
 	if (test_kvm_facility(vcpu->kvm, 139))
 		scb_s->ecd |= scb_o->ecd & ECD_MEF;
+	/* etoken */
+	if (test_kvm_facility(vcpu->kvm, 156))
+		scb_s->ecd |= scb_o->ecd & ECD_ETOKENF;
 	prepare_ibc(vcpu, vsie_page);
 	rc = shadow_crycb(vcpu, vsie_page);
 out:
 		vsie_page->riccbd_gpa = gpa;
 		scb_s->riccbd = hpa;
 	}
-	if ((scb_s->ecb & ECB_GS) && !(scb_s->ecd & ECD_HOSTREGMGMT)) {
+	if (((scb_s->ecb & ECB_GS) && !(scb_s->ecd & ECD_HOSTREGMGMT)) ||
+	    (scb_s->ecd & ECD_ETOKENF)) {
 		unsigned long sdnxc;
 		gpa = READ_ONCE(scb_o->sdnxo) & ~0xfUL;
  *          - < 0 if an error occurred
  */
 static int do_vsie_run(struct kvm_vcpu *vcpu, struct vsie_page *vsie_page)
+	__releases(vcpu->kvm->srcu)
+	__acquires(vcpu->kvm->srcu)
 {
 	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
 	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;

arch/s390/mm/gmap.c+431−23

@@ -2,8 +2,10 @@ · 1 unmodified line
2	/*	2	/*
3	* KVM guest address space mapping code	3	* KVM guest address space mapping code
4	*	4	*
5	* Copyright IBM Corp. 2007, 2016	5	* Copyright IBM Corp. 2007, 2016, 2018
6	* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>	6	* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
		7	* David Hildenbrand <david@redhat.com>
		8	* Janosch Frank <frankja@linux.vnet.ibm.com>
7	*/	9	*/
8		10
9	#include <linux/kernel.h>	11	#include <linux/kernel.h>
@@ -521,6 +523,9 @@ void gmap_unlink(struct mm_struct mm, unsigned long table, · 511 unmodified lines
521	rcu_read_unlock();	523	rcu_read_unlock();
522	}	524	}
523		525
		526	static void gmap_pmdp_xchg(struct gmap gmap, pmd_t old, pmd_t new,
		527	unsigned long gaddr);
		528
524	/**	529	/**
525	* gmap_link - set up shadow page tables to connect a host to a guest address	530	* gmap_link - set up shadow page tables to connect a host to a guest address
526	* @gmap: pointer to guest mapping meta data structure	531	* @gmap: pointer to guest mapping meta data structure
@@ -541,6 +546,7 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) · 14 unmodified lines
541	p4d_t *p4d;	546	p4d_t *p4d;
542	pud_t *pud;	547	pud_t *pud;
543	pmd_t *pmd;	548	pmd_t *pmd;
		549	u64 unprot;
544	int rc;	550	int rc;
545		551
546	BUG_ON(gmap_is_shadow(gmap));	552	BUG_ON(gmap_is_shadow(gmap));
@@ -584,8 +590,8 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) · 37 unmodified lines
584	return -EFAULT;	590	return -EFAULT;
585	pmd = pmd_offset(pud, vmaddr);	591	pmd = pmd_offset(pud, vmaddr);
586	VM_BUG_ON(pmd_none(*pmd));	592	VM_BUG_ON(pmd_none(*pmd));
587	/* large pmds cannot yet be handled */	593	/* Are we allowed to use huge pages? */
588	if (pmd_large(*pmd))	594	if (pmd_large(*pmd) && !gmap->mm->context.allow_gmap_hpage_1m)
589	return -EFAULT;	595	return -EFAULT;
590	/* Link gmap segment table entry location to page table. */	596	/* Link gmap segment table entry location to page table. */
591	rc = radix_tree_preload(GFP_KERNEL);	597	rc = radix_tree_preload(GFP_KERNEL);
@@ -596,10 +602,22 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) · 4 unmodified lines
596	if (*table == _SEGMENT_ENTRY_EMPTY) {	602	if (*table == _SEGMENT_ENTRY_EMPTY) {
597	rc = radix_tree_insert(&gmap->host_to_guest,	603	rc = radix_tree_insert(&gmap->host_to_guest,
598	vmaddr >> PMD_SHIFT, table);	604	vmaddr >> PMD_SHIFT, table);
599	if (!rc)	605	if (!rc) {
600	table = pmd_val(pmd);	606	if (pmd_large(*pmd)) {
601	} else	607	table = (pmd_val(pmd) &
602	rc = 0;	608	_SEGMENT_ENTRY_HARDWARE_BITS_LARGE)
		609	\| _SEGMENT_ENTRY_GMAP_UC;
		610	} else
		611	table = pmd_val(pmd) &
		612	_SEGMENT_ENTRY_HARDWARE_BITS;
		613	}
		614	} else if (*table & _SEGMENT_ENTRY_PROTECT &&
		615	!(pmd_val(*pmd) & _SEGMENT_ENTRY_PROTECT)) {
		616	unprot = (u64)*table;
		617	unprot &= ~_SEGMENT_ENTRY_PROTECT;
		618	unprot \|= _SEGMENT_ENTRY_GMAP_UC;
		619	gmap_pmdp_xchg(gmap, (pmd_t *)table, __pmd(unprot), gaddr);
		620	}
603	spin_unlock(&gmap->guest_table_lock);	621	spin_unlock(&gmap->guest_table_lock);
604	spin_unlock(ptl);	622	spin_unlock(ptl);
605	radix_tree_preload_end();	623	radix_tree_preload_end();
@@ -690,6 +708,12 @@ void gmap_discard(struct gmap *gmap, unsigned long from, unsigned long to) · 84 unmodified lines
690	vmaddr \|= gaddr & ~PMD_MASK;	708	vmaddr \|= gaddr & ~PMD_MASK;
691	/* Find vma in the parent mm */	709	/* Find vma in the parent mm */
692	vma = find_vma(gmap->mm, vmaddr);	710	vma = find_vma(gmap->mm, vmaddr);
		711	/*
		712	* We do not discard pages that are backed by
		713	* hugetlbfs, so we don't have to refault them.
		714	*/
		715	if (vma && is_vm_hugetlb_page(vma))
		716	continue;
693	size = min(to - gaddr, PMD_SIZE - (gaddr & ~PMD_MASK));	717	size = min(to - gaddr, PMD_SIZE - (gaddr & ~PMD_MASK));
694	zap_page_range(vma, vmaddr, size);	718	zap_page_range(vma, vmaddr, size);
695	}	719	}
@@ -864,7 +888,128 @@ static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr, · 168 unmodified lines
864	*/	888	*/
865	static void gmap_pte_op_end(spinlock_t *ptl)	889	static void gmap_pte_op_end(spinlock_t *ptl)
866	{	890	{
867	spin_unlock(ptl);	891	if (ptl)
		892	spin_unlock(ptl);
		893	}
		894
		895	/**
		896	* gmap_pmd_op_walk - walk the gmap tables, get the guest table lock
		897	* and return the pmd pointer
		898	* @gmap: pointer to guest mapping meta data structure
		899	* @gaddr: virtual address in the guest address space
		900	*
		901	* Returns a pointer to the pmd for a guest address, or NULL
		902	*/
		903	static inline pmd_t gmap_pmd_op_walk(struct gmap gmap, unsigned long gaddr)
		904	{
		905	pmd_t *pmdp;
		906
		907	BUG_ON(gmap_is_shadow(gmap));
		908	spin_lock(&gmap->guest_table_lock);
		909	pmdp = (pmd_t *) gmap_table_walk(gmap, gaddr, 1);
		910
		911	if (!pmdp \|\| pmd_none(*pmdp)) {
		912	spin_unlock(&gmap->guest_table_lock);
		913	return NULL;
		914	}
		915
		916	/* 4k page table entries are locked via the pte (pte_alloc_map_lock). */
		917	if (!pmd_large(*pmdp))
		918	spin_unlock(&gmap->guest_table_lock);
		919	return pmdp;
		920	}
		921
		922	/**
		923	* gmap_pmd_op_end - release the guest_table_lock if needed
		924	* @gmap: pointer to the guest mapping meta data structure
		925	* @pmdp: pointer to the pmd
		926	*/
		927	static inline void gmap_pmd_op_end(struct gmap gmap, pmd_t pmdp)
		928	{
		929	if (pmd_large(*pmdp))
		930	spin_unlock(&gmap->guest_table_lock);
		931	}
		932
		933	/*
		934	* gmap_protect_pmd - remove access rights to memory and set pmd notification bits
		935	* @pmdp: pointer to the pmd to be protected
		936	* @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
		937	* @bits: notification bits to set
		938	*
		939	* Returns:
		940	* 0 if successfully protected
		941	* -EAGAIN if a fixup is needed
		942	* -EINVAL if unsupported notifier bits have been specified
		943	*
		944	* Expected to be called with sg->mm->mmap_sem in read and
		945	* guest_table_lock held.
		946	*/
		947	static int gmap_protect_pmd(struct gmap *gmap, unsigned long gaddr,
		948	pmd_t *pmdp, int prot, unsigned long bits)
		949	{
		950	int pmd_i = pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID;
		951	int pmd_p = pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT;
		952	pmd_t new = *pmdp;
		953
		954	/* Fixup needed */
		955	if ((pmd_i && (prot != PROT_NONE)) \|\| (pmd_p && (prot == PROT_WRITE)))
		956	return -EAGAIN;
		957
		958	if (prot == PROT_NONE && !pmd_i) {
		959	pmd_val(new) \|= _SEGMENT_ENTRY_INVALID;
		960	gmap_pmdp_xchg(gmap, pmdp, new, gaddr);
		961	}
		962
		963	if (prot == PROT_READ && !pmd_p) {
		964	pmd_val(new) &= ~_SEGMENT_ENTRY_INVALID;
		965	pmd_val(new) \|= _SEGMENT_ENTRY_PROTECT;
		966	gmap_pmdp_xchg(gmap, pmdp, new, gaddr);
		967	}
		968
		969	if (bits & GMAP_NOTIFY_MPROT)
		970	pmd_val(*pmdp) \|= _SEGMENT_ENTRY_GMAP_IN;
		971
		972	/* Shadow GMAP protection needs split PMDs */
		973	if (bits & GMAP_NOTIFY_SHADOW)
		974	return -EINVAL;
		975
		976	return 0;
		977	}
		978
		979	/*
		980	* gmap_protect_pte - remove access rights to memory and set pgste bits
		981	* @gmap: pointer to guest mapping meta data structure
		982	* @gaddr: virtual address in the guest address space
		983	* @pmdp: pointer to the pmd associated with the pte
		984	* @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
		985	* @bits: notification bits to set
		986	*
		987	* Returns 0 if successfully protected, -ENOMEM if out of memory and
		988	* -EAGAIN if a fixup is needed.
		989	*
		990	* Expected to be called with sg->mm->mmap_sem in read
		991	*/
		992	static int gmap_protect_pte(struct gmap *gmap, unsigned long gaddr,
		993	pmd_t *pmdp, int prot, unsigned long bits)
		994	{
		995	int rc;
		996	pte_t *ptep;
		997	spinlock_t *ptl = NULL;
		998	unsigned long pbits = 0;
		999
		1000	if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID)
		1001	return -EAGAIN;
		1002
		1003	ptep = pte_alloc_map_lock(gmap->mm, pmdp, gaddr, &ptl);
		1004	if (!ptep)
		1005	return -ENOMEM;
		1006
		1007	pbits \|= (bits & GMAP_NOTIFY_MPROT) ? PGSTE_IN_BIT : 0;
		1008	pbits \|= (bits & GMAP_NOTIFY_SHADOW) ? PGSTE_VSIE_BIT : 0;
		1009	/* Protect and unlock. */
		1010	rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, pbits);
		1011	gmap_pte_op_end(ptl);
		1012	return rc;
868	}	1013	}
869		1014
870	/*	1015	/*
@@ -883,30 +1028,45 @@ static void gmap_pte_op_end(spinlock_t *ptl) · 12 unmodified lines
883	static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr,	1028	static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr,
884	unsigned long len, int prot, unsigned long bits)	1029	unsigned long len, int prot, unsigned long bits)
885	{	1030	{
886	unsigned long vmaddr;	1031	unsigned long vmaddr, dist;
887	spinlock_t *ptl;	1032	pmd_t *pmdp;
888	pte_t *ptep;
889	int rc;	1033	int rc;
890		1034
891	BUG_ON(gmap_is_shadow(gmap));	1035	BUG_ON(gmap_is_shadow(gmap));
892	while (len) {	1036	while (len) {
893	rc = -EAGAIN;	1037	rc = -EAGAIN;
894	ptep = gmap_pte_op_walk(gmap, gaddr, &ptl);	1038	pmdp = gmap_pmd_op_walk(gmap, gaddr);
895	if (ptep) {	1039	if (pmdp) {
896	rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, bits);	1040	if (!pmd_large(*pmdp)) {
897	gmap_pte_op_end(ptl);	1041	rc = gmap_protect_pte(gmap, gaddr, pmdp, prot,
		1042	bits);
		1043	if (!rc) {
		1044	len -= PAGE_SIZE;
		1045	gaddr += PAGE_SIZE;
		1046	}
		1047	} else {
		1048	rc = gmap_protect_pmd(gmap, gaddr, pmdp, prot,
		1049	bits);
		1050	if (!rc) {
		1051	dist = HPAGE_SIZE - (gaddr & ~HPAGE_MASK);
		1052	len = len < dist ? 0 : len - dist;
		1053	gaddr = (gaddr & HPAGE_MASK) + HPAGE_SIZE;
		1054	}
		1055	}
		1056	gmap_pmd_op_end(gmap, pmdp);
898	}	1057	}
899	if (rc) {	1058	if (rc) {
		1059	if (rc == -EINVAL)
		1060	return rc;
		1061
		1062	/* -EAGAIN, fixup of userspace mm and gmap */
900	vmaddr = __gmap_translate(gmap, gaddr);	1063	vmaddr = __gmap_translate(gmap, gaddr);
901	if (IS_ERR_VALUE(vmaddr))	1064	if (IS_ERR_VALUE(vmaddr))
902	return vmaddr;	1065	return vmaddr;
903	rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot);	1066	rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot);
904	if (rc)	1067	if (rc)
905	return rc;	1068	return rc;
906	continue;
907	}	1069	}
908	gaddr += PAGE_SIZE;
909	len -= PAGE_SIZE;
910	}	1070	}
911	return 0;	1071	return 0;
912	}	1072	}
@@ -935,7 +1095,7 @@ int gmap_mprotect_notify(struct gmap *gmap, unsigned long gaddr, · 22 unmodified lines
935	if (!MACHINE_HAS_ESOP && prot == PROT_READ)	1095	if (!MACHINE_HAS_ESOP && prot == PROT_READ)
936	return -EINVAL;	1096	return -EINVAL;
937	down_read(&gmap->mm->mmap_sem);	1097	down_read(&gmap->mm->mmap_sem);
938	rc = gmap_protect_range(gmap, gaddr, len, prot, PGSTE_IN_BIT);	1098	rc = gmap_protect_range(gmap, gaddr, len, prot, GMAP_NOTIFY_MPROT);
939	up_read(&gmap->mm->mmap_sem);	1099	up_read(&gmap->mm->mmap_sem);
940	return rc;	1100	return rc;
941	}	1101	}
@@ -1474,6 +1634,7 @@ struct gmap gmap_shadow(struct gmap parent, unsigned long asce, · 532 unmodified lines
1474	unsigned long limit;	1634	unsigned long limit;
1475	int rc;	1635	int rc;
1476		1636
		1637	BUG_ON(parent->mm->context.allow_gmap_hpage_1m);
1477	BUG_ON(gmap_is_shadow(parent));	1638	BUG_ON(gmap_is_shadow(parent));
1478	spin_lock(&parent->shadow_lock);	1639	spin_lock(&parent->shadow_lock);
1479	sg = gmap_find_shadow(parent, asce, edat_level);	1640	sg = gmap_find_shadow(parent, asce, edat_level);
@@ -1526,7 +1687,7 @@ struct gmap gmap_shadow(struct gmap parent, unsigned long asce, · 46 unmodified lines
1526	down_read(&parent->mm->mmap_sem);	1687	down_read(&parent->mm->mmap_sem);
1527	rc = gmap_protect_range(parent, asce & _ASCE_ORIGIN,	1688	rc = gmap_protect_range(parent, asce & _ASCE_ORIGIN,
1528	((asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE,	1689	((asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE,
1529	PROT_READ, PGSTE_VSIE_BIT);	1690	PROT_READ, GMAP_NOTIFY_SHADOW);
1530	up_read(&parent->mm->mmap_sem);	1691	up_read(&parent->mm->mmap_sem);
1531	spin_lock(&parent->shadow_lock);	1692	spin_lock(&parent->shadow_lock);
1532	new->initialized = true;	1693	new->initialized = true;
@@ -2092,6 +2253,225 @@ void ptep_notify(struct mm_struct *mm, unsigned long vmaddr, · 559 unmodified lines
2092	}	2253	}
2093	EXPORT_SYMBOL_GPL(ptep_notify);	2254	EXPORT_SYMBOL_GPL(ptep_notify);
2094		2255
		2256	static void pmdp_notify_gmap(struct gmap gmap, pmd_t pmdp,
		2257	unsigned long gaddr)
		2258	{
		2259	pmd_val(*pmdp) &= ~_SEGMENT_ENTRY_GMAP_IN;
		2260	gmap_call_notifier(gmap, gaddr, gaddr + HPAGE_SIZE - 1);
		2261	}
		2262
		2263	/**
		2264	* gmap_pmdp_xchg - exchange a gmap pmd with another
		2265	* @gmap: pointer to the guest address space structure
		2266	* @pmdp: pointer to the pmd entry
		2267	* @new: replacement entry
		2268	* @gaddr: the affected guest address
		2269	*
		2270	* This function is assumed to be called with the guest_table_lock
		2271	* held.
		2272	*/
		2273	static void gmap_pmdp_xchg(struct gmap gmap, pmd_t pmdp, pmd_t new,
		2274	unsigned long gaddr)
		2275	{
		2276	gaddr &= HPAGE_MASK;
		2277	pmdp_notify_gmap(gmap, pmdp, gaddr);
		2278	pmd_val(new) &= ~_SEGMENT_ENTRY_GMAP_IN;
		2279	if (MACHINE_HAS_TLB_GUEST)
		2280	__pmdp_idte(gaddr, (pmd_t *)pmdp, IDTE_GUEST_ASCE, gmap->asce,
		2281	IDTE_GLOBAL);
		2282	else if (MACHINE_HAS_IDTE)
		2283	__pmdp_idte(gaddr, (pmd_t *)pmdp, 0, 0, IDTE_GLOBAL);
		2284	else
		2285	__pmdp_csp(pmdp);
		2286	*pmdp = new;
		2287	}
		2288
		2289	static void gmap_pmdp_clear(struct mm_struct *mm, unsigned long vmaddr,
		2290	int purge)
		2291	{
		2292	pmd_t *pmdp;
		2293	struct gmap *gmap;
		2294	unsigned long gaddr;
		2295
		2296	rcu_read_lock();
		2297	list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
		2298	spin_lock(&gmap->guest_table_lock);
		2299	pmdp = (pmd_t *)radix_tree_delete(&gmap->host_to_guest,
		2300	vmaddr >> PMD_SHIFT);
		2301	if (pmdp) {
		2302	gaddr = __gmap_segment_gaddr((unsigned long *)pmdp);
		2303	pmdp_notify_gmap(gmap, pmdp, gaddr);
		2304	WARN_ON(pmd_val(*pmdp) & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE \|
		2305	_SEGMENT_ENTRY_GMAP_UC));
		2306	if (purge)
		2307	__pmdp_csp(pmdp);
		2308	pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
		2309	}
		2310	spin_unlock(&gmap->guest_table_lock);
		2311	}
		2312	rcu_read_unlock();
		2313	}
		2314
		2315	/**
		2316	* gmap_pmdp_invalidate - invalidate all affected guest pmd entries without
		2317	* flushing
		2318	* @mm: pointer to the process mm_struct
		2319	* @vmaddr: virtual address in the process address space
		2320	*/
		2321	void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr)
		2322	{
		2323	gmap_pmdp_clear(mm, vmaddr, 0);
		2324	}
		2325	EXPORT_SYMBOL_GPL(gmap_pmdp_invalidate);
		2326
		2327	/**
		2328	* gmap_pmdp_csp - csp all affected guest pmd entries
		2329	* @mm: pointer to the process mm_struct
		2330	* @vmaddr: virtual address in the process address space
		2331	*/
		2332	void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr)
		2333	{
		2334	gmap_pmdp_clear(mm, vmaddr, 1);
		2335	}
		2336	EXPORT_SYMBOL_GPL(gmap_pmdp_csp);
		2337
		2338	/**
		2339	* gmap_pmdp_idte_local - invalidate and clear a guest pmd entry
		2340	* @mm: pointer to the process mm_struct
		2341	* @vmaddr: virtual address in the process address space
		2342	*/
		2343	void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr)
		2344	{
		2345	unsigned long *entry, gaddr;
		2346	struct gmap *gmap;
		2347	pmd_t *pmdp;
		2348
		2349	rcu_read_lock();
		2350	list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
		2351	spin_lock(&gmap->guest_table_lock);
		2352	entry = radix_tree_delete(&gmap->host_to_guest,
		2353	vmaddr >> PMD_SHIFT);
		2354	if (entry) {
		2355	pmdp = (pmd_t *)entry;
		2356	gaddr = __gmap_segment_gaddr(entry);
		2357	pmdp_notify_gmap(gmap, pmdp, gaddr);
		2358	WARN_ON(*entry & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE \|
		2359	_SEGMENT_ENTRY_GMAP_UC));
		2360	if (MACHINE_HAS_TLB_GUEST)
		2361	__pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE,
		2362	gmap->asce, IDTE_LOCAL);
		2363	else if (MACHINE_HAS_IDTE)
		2364	__pmdp_idte(gaddr, pmdp, 0, 0, IDTE_LOCAL);
		2365	*entry = _SEGMENT_ENTRY_EMPTY;
		2366	}
		2367	spin_unlock(&gmap->guest_table_lock);
		2368	}
		2369	rcu_read_unlock();
		2370	}
		2371	EXPORT_SYMBOL_GPL(gmap_pmdp_idte_local);
		2372
		2373	/**
		2374	* gmap_pmdp_idte_global - invalidate and clear a guest pmd entry
		2375	* @mm: pointer to the process mm_struct
		2376	* @vmaddr: virtual address in the process address space
		2377	*/
		2378	void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr)
		2379	{
		2380	unsigned long *entry, gaddr;
		2381	struct gmap *gmap;
		2382	pmd_t *pmdp;
		2383
		2384	rcu_read_lock();
		2385	list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
		2386	spin_lock(&gmap->guest_table_lock);
		2387	entry = radix_tree_delete(&gmap->host_to_guest,
		2388	vmaddr >> PMD_SHIFT);
		2389	if (entry) {
		2390	pmdp = (pmd_t *)entry;
		2391	gaddr = __gmap_segment_gaddr(entry);
		2392	pmdp_notify_gmap(gmap, pmdp, gaddr);
		2393	WARN_ON(*entry & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE \|
		2394	_SEGMENT_ENTRY_GMAP_UC));
		2395	if (MACHINE_HAS_TLB_GUEST)
		2396	__pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE,
		2397	gmap->asce, IDTE_GLOBAL);
		2398	else if (MACHINE_HAS_IDTE)
		2399	__pmdp_idte(gaddr, pmdp, 0, 0, IDTE_GLOBAL);
		2400	else
		2401	__pmdp_csp(pmdp);
		2402	*entry = _SEGMENT_ENTRY_EMPTY;
		2403	}
		2404	spin_unlock(&gmap->guest_table_lock);
		2405	}
		2406	rcu_read_unlock();
		2407	}
		2408	EXPORT_SYMBOL_GPL(gmap_pmdp_idte_global);
		2409
		2410	/**
		2411	* gmap_test_and_clear_dirty_pmd - test and reset segment dirty status
		2412	* @gmap: pointer to guest address space
		2413	* @pmdp: pointer to the pmd to be tested
		2414	* @gaddr: virtual address in the guest address space
		2415	*
		2416	* This function is assumed to be called with the guest_table_lock
		2417	* held.
		2418	*/
		2419	bool gmap_test_and_clear_dirty_pmd(struct gmap gmap, pmd_t pmdp,
		2420	unsigned long gaddr)
		2421	{
		2422	if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID)
		2423	return false;
		2424
		2425	/* Already protected memory, which did not change is clean */
		2426	if (pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT &&
		2427	!(pmd_val(*pmdp) & _SEGMENT_ENTRY_GMAP_UC))
		2428	return false;
		2429
		2430	/* Clear UC indication and reset protection */
		2431	pmd_val(*pmdp) &= ~_SEGMENT_ENTRY_GMAP_UC;
		2432	gmap_protect_pmd(gmap, gaddr, pmdp, PROT_READ, 0);
		2433	return true;
		2434	}
		2435
		2436	/**
		2437	* gmap_sync_dirty_log_pmd - set bitmap based on dirty status of segment
		2438	* @gmap: pointer to guest address space
		2439	* @bitmap: dirty bitmap for this pmd
		2440	* @gaddr: virtual address in the guest address space
		2441	* @vmaddr: virtual address in the host address space
		2442	*
		2443	* This function is assumed to be called with the guest_table_lock
		2444	* held.
		2445	*/
		2446	void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long bitmap[4],
		2447	unsigned long gaddr, unsigned long vmaddr)
		2448	{
		2449	int i;
		2450	pmd_t *pmdp;
		2451	pte_t *ptep;
		2452	spinlock_t *ptl;
		2453
		2454	pmdp = gmap_pmd_op_walk(gmap, gaddr);
		2455	if (!pmdp)
		2456	return;
		2457
		2458	if (pmd_large(*pmdp)) {
		2459	if (gmap_test_and_clear_dirty_pmd(gmap, pmdp, gaddr))
		2460	bitmap_fill(bitmap, _PAGE_ENTRIES);
		2461	} else {
		2462	for (i = 0; i < _PAGE_ENTRIES; i++, vmaddr += PAGE_SIZE) {
		2463	ptep = pte_alloc_map_lock(gmap->mm, pmdp, vmaddr, &ptl);
		2464	if (!ptep)
		2465	continue;
		2466	if (ptep_test_and_clear_uc(gmap->mm, vmaddr, ptep))
		2467	set_bit(i, bitmap);
		2468	spin_unlock(ptl);
		2469	}
		2470	}
		2471	gmap_pmd_op_end(gmap, pmdp);
		2472	}
		2473	EXPORT_SYMBOL_GPL(gmap_sync_dirty_log_pmd);
		2474
2095	static inline void thp_split_mm(struct mm_struct *mm)	2475	static inline void thp_split_mm(struct mm_struct *mm)
2096	{	2476	{
2097	#ifdef CONFIG_TRANSPARENT_HUGEPAGE	2477	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -2168,17 +2548,45 @@ EXPORT_SYMBOL_GPL(s390_enable_sie); · 70 unmodified lines
2168	* Enable storage key handling from now on and initialize the storage	2548	* Enable storage key handling from now on and initialize the storage
2169	* keys with the default key.	2549	* keys with the default key.
2170	*/	2550	*/
2171	static int __s390_enable_skey(pte_t *pte, unsigned long addr,	2551	static int __s390_enable_skey_pte(pte_t *pte, unsigned long addr,
2172	unsigned long next, struct mm_walk *walk)	2552	unsigned long next, struct mm_walk *walk)
2173	{	2553	{
2174	/* Clear storage key */	2554	/* Clear storage key */
2175	ptep_zap_key(walk->mm, addr, pte);	2555	ptep_zap_key(walk->mm, addr, pte);
2176	return 0;	2556	return 0;
2177	}	2557	}
2178		2558
		2559	static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr,
		2560	unsigned long hmask, unsigned long next,
		2561	struct mm_walk *walk)
		2562	{
		2563	pmd_t pmd = (pmd_t )pte;
		2564	unsigned long start, end;
		2565	struct page page = pmd_page(pmd);
		2566
		2567	/*
		2568	* The write check makes sure we do not set a key on shared
		2569	* memory. This is needed as the walker does not differentiate
		2570	* between actual guest memory and the process executable or
		2571	* shared libraries.
		2572	*/
		2573	if (pmd_val(*pmd) & _SEGMENT_ENTRY_INVALID \|\|
		2574	!(pmd_val(*pmd) & _SEGMENT_ENTRY_WRITE))
		2575	return 0;
		2576
		2577	start = pmd_val(*pmd) & HPAGE_MASK;
		2578	end = start + HPAGE_SIZE - 1;
		2579	__storage_key_init_range(start, end);
		2580	set_bit(PG_arch_1, &page->flags);
		2581	return 0;
		2582	}
		2583
2179	int s390_enable_skey(void)	2584	int s390_enable_skey(void)
2180	{	2585	{
2181	struct mm_walk walk = { .pte_entry = __s390_enable_skey };	2586	struct mm_walk walk = {
		2587	.hugetlb_entry = __s390_enable_skey_hugetlb,
		2588	.pte_entry = __s390_enable_skey_pte,
		2589	};
2182	struct mm_struct *mm = current->mm;	2590	struct mm_struct *mm = current->mm;
2183	struct vm_area_struct *vma;	2591	struct vm_area_struct *vma;
2184	int rc = 0;	2592	int rc = 0;

arch/s390/mm/hugetlbpage.c+24−0

@@ -123,6 +123,29 @@ static inline pte_t __rste_to_pte(unsigned long rste) · 122 unmodified lines
123	return pte;	123	return pte;
124	}	124	}
125		125
		126	static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste)
		127	{
		128	struct page *page;
		129	unsigned long size, paddr;
		130
		131	if (!mm_uses_skeys(mm) \|\|
		132	rste & _SEGMENT_ENTRY_INVALID)
		133	return;
		134
		135	if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) {
		136	page = pud_page(__pud(rste));
		137	size = PUD_SIZE;
		138	paddr = rste & PUD_MASK;
		139	} else {
		140	page = pmd_page(__pmd(rste));
		141	size = PMD_SIZE;
		142	paddr = rste & PMD_MASK;
		143	}
		144
		145	if (!test_and_set_bit(PG_arch_1, &page->flags))
		146	__storage_key_init_range(paddr, paddr + size - 1);
		147	}
		148
126	void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,	149	void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
127	pte_t *ptep, pte_t pte)	150	pte_t *ptep, pte_t pte)
128	{	151	{
@@ -137,6 +160,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, · 8 unmodified lines
137	rste \|= _REGION_ENTRY_TYPE_R3 \| _REGION3_ENTRY_LARGE;	160	rste \|= _REGION_ENTRY_TYPE_R3 \| _REGION3_ENTRY_LARGE;
138	else	161	else
139	rste \|= _SEGMENT_ENTRY_LARGE;	162	rste \|= _SEGMENT_ENTRY_LARGE;
		163	clear_huge_pte_skeys(mm, rste);
140	pte_val(*ptep) = rste;	164	pte_val(*ptep) = rste;
141	}	165	}
142		166

arch/s390/mm/pageattr.c+2−4

 static inline unsigned long sske_frame(unsigned long addr, unsigned char skey)
 {
-	asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],9,0"
+	asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],1,0"
 		     : [addr] "+a" (addr) : [skey] "d" (skey));
 	return addr;
 }
 {
 	unsigned long boundary, size;
-	if (!PAGE_DEFAULT_KEY)
-		return;
 	while (start < end) {
 		if (MACHINE_HAS_EDAT1) {
 			/* set storage keys for a 1MB frame */
 				continue;
 			}
 		}
-		page_set_storage_key(start, PAGE_DEFAULT_KEY, 0);
+		page_set_storage_key(start, PAGE_DEFAULT_KEY, 1);
 		start += PAGE_SIZE;
 	}
 }

arch/s390/mm/pgtable.c+115−44

 			    mm->context.asce, IDTE_LOCAL);
 	else
 		__pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
+	if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
+		gmap_pmdp_idte_local(mm, addr);
 }
 static inline void pmdp_idte_global(struct mm_struct *mm,
 				    unsigned long addr, pmd_t *pmdp)
 {
-	if (MACHINE_HAS_TLB_GUEST)
+	if (MACHINE_HAS_TLB_GUEST) {
 		__pmdp_idte(addr, pmdp, IDTE_NODAT | IDTE_GUEST_ASCE,
 			    mm->context.asce, IDTE_GLOBAL);
-	else if (MACHINE_HAS_IDTE)
+		if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
+			gmap_pmdp_idte_global(mm, addr);
+	} else if (MACHINE_HAS_IDTE) {
 		__pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
-	else
+		if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
+			gmap_pmdp_idte_global(mm, addr);
+	} else {
 		__pmdp_csp(pmdp);
+		if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
+			gmap_pmdp_csp(mm, addr);
+	}
 }
 static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
 			  cpumask_of(smp_processor_id()))) {
 		pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID;
 		mm->context.flush_mm = 1;
+		if (mm_has_pgste(mm))
+			gmap_pmdp_invalidate(mm, addr);
 	} else {
 		pmdp_idte_global(mm, addr, pmdp);
 	}
 	return old;
 }
+static pmd_t *pmd_alloc_map(struct mm_struct *mm, unsigned long addr)
+{
+	pgd_t *pgd;
+	p4d_t *p4d;
+	pud_t *pud;
+	pmd_t *pmd;
+	pgd = pgd_offset(mm, addr);
+	p4d = p4d_alloc(mm, pgd, addr);
+	if (!p4d)
+		return NULL;
+	pud = pud_alloc(mm, p4d, addr);
+	if (!pud)
+		return NULL;
+	pmd = pmd_alloc(mm, pud, addr);
+	return pmd;
+}
 pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
 		       pmd_t *pmdp, pmd_t new)
 {
 /*
  * Test and reset if a guest page is dirty
  */
-bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr)
+bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long addr,
+		       pte_t *ptep)
 {
-	spinlock_t *ptl;
-	pgd_t *pgd;
-	p4d_t *p4d;
-	pud_t *pud;
-	pmd_t *pmd;
 	pgste_t pgste;
-	pte_t *ptep;
 	pte_t pte;
 	bool dirty;
 	int nodat;
-	pgd = pgd_offset(mm, addr);
-	p4d = p4d_alloc(mm, pgd, addr);
-	if (!p4d)
-		return false;
-	pud = pud_alloc(mm, p4d, addr);
-	if (!pud)
-		return false;
-	pmd = pmd_alloc(mm, pud, addr);
-	if (!pmd)
-		return false;
-	/* We can't run guests backed by huge pages, but userspace can
-	 * still set them up and then try to migrate them without any
-	 * migration support.
-	 */
-	if (pmd_large(*pmd))
-		return true;
-	ptep = pte_alloc_map_lock(mm, pmd, addr, &ptl);
-	if (unlikely(!ptep))
-		return false;
 	pgste = pgste_get_lock(ptep);
 	dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
 	pgste_val(pgste) &= ~PGSTE_UC_BIT;
 		*ptep = pte;
 	}
 	pgste_set_unlock(ptep, pgste);
-	spin_unlock(ptl);
 	return dirty;
 }
-EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty);
+EXPORT_SYMBOL_GPL(ptep_test_and_clear_uc);
 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
 			  unsigned char key, bool nq)
 {
-	unsigned long keyul;
+	unsigned long keyul, paddr;
 	spinlock_t *ptl;
 	pgste_t old, new;
+	pmd_t *pmdp;
 	pte_t *ptep;
-	ptep = get_locked_pte(mm, addr, &ptl);
+	pmdp = pmd_alloc_map(mm, addr);
+	if (unlikely(!pmdp))
+		return -EFAULT;
+	ptl = pmd_lock(mm, pmdp);
+	if (!pmd_present(*pmdp)) {
+		spin_unlock(ptl);
+		return -EFAULT;
+	}
+	if (pmd_large(*pmdp)) {
+		paddr = pmd_val(*pmdp) & HPAGE_MASK;
+		paddr |= addr & ~HPAGE_MASK;
+		/*
+		 * Huge pmds need quiescing operations, they are
+		 * always mapped.
+		 */
+		page_set_storage_key(paddr, key, 1);
+		spin_unlock(ptl);
+		return 0;
+	}
+	spin_unlock(ptl);
+	ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);
 	if (unlikely(!ptep))
 		return -EFAULT;
 	pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48;
 	pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56;
 	if (!(pte_val(*ptep) & _PAGE_INVALID)) {
-		unsigned long address, bits, skey;
+		unsigned long bits, skey;
-		address = pte_val(*ptep) & PAGE_MASK;
+		paddr = pte_val(*ptep) & PAGE_MASK;
-		skey = (unsigned long) page_get_storage_key(address);
+		skey = (unsigned long) page_get_storage_key(paddr);
 		bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED);
 		skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT);
 		/* Set storage key ACC and FP */
-		page_set_storage_key(address, skey, !nq);
+		page_set_storage_key(paddr, skey, !nq);
 		/* Merge host changed & referenced into pgste  */
 		pgste_val(new) |= bits << 52;
 	}
 int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr)
 {
 	spinlock_t *ptl;
+	unsigned long paddr;
 	pgste_t old, new;
+	pmd_t *pmdp;
 	pte_t *ptep;
 	int cc = 0;
-	ptep = get_locked_pte(mm, addr, &ptl);
+	pmdp = pmd_alloc_map(mm, addr);
+	if (unlikely(!pmdp))
+		return -EFAULT;
+	ptl = pmd_lock(mm, pmdp);
+	if (!pmd_present(*pmdp)) {
+		spin_unlock(ptl);
+		return -EFAULT;
+	}
+	if (pmd_large(*pmdp)) {
+		paddr = pmd_val(*pmdp) & HPAGE_MASK;
+		paddr |= addr & ~HPAGE_MASK;
+		cc = page_reset_referenced(paddr);
+		spin_unlock(ptl);
+		return cc;
+	}
+	spin_unlock(ptl);
+	ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);
 	if (unlikely(!ptep))
 		return -EFAULT;
 	pgste_val(new) &= ~PGSTE_GR_BIT;
 	if (!(pte_val(*ptep) & _PAGE_INVALID)) {
-		cc = page_reset_referenced(pte_val(*ptep) & PAGE_MASK);
+		paddr = pte_val(*ptep) & PAGE_MASK;
+		cc = page_reset_referenced(paddr);
 		/* Merge real referenced bit into host-set */
 		pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT;
 	}
 int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
 			  unsigned char *key)
 {
+	unsigned long paddr;
 	spinlock_t *ptl;
 	pgste_t pgste;
+	pmd_t *pmdp;
 	pte_t *ptep;
-	ptep = get_locked_pte(mm, addr, &ptl);
+	pmdp = pmd_alloc_map(mm, addr);
+	if (unlikely(!pmdp))
+		return -EFAULT;
+	ptl = pmd_lock(mm, pmdp);
+	if (!pmd_present(*pmdp)) {
+		/* Not yet mapped memory has a zero key */
+		spin_unlock(ptl);
+		*key = 0;
+		return 0;
+	}
+	if (pmd_large(*pmdp)) {
+		paddr = pmd_val(*pmdp) & HPAGE_MASK;
+		paddr |= addr & ~HPAGE_MASK;
+		*key = page_get_storage_key(paddr);
+		spin_unlock(ptl);
+		return 0;
+	}
+	spin_unlock(ptl);
+	ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);
 	if (unlikely(!ptep))
 		return -EFAULT;
 	pgste = pgste_get_lock(ptep);
 	*key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56;
+	paddr = pte_val(*ptep) & PAGE_MASK;
 	if (!(pte_val(*ptep) & _PAGE_INVALID))
-		*key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK);
+		*key = page_get_storage_key(paddr);
 	/* Reflect guest's logical view, not physical */
 	*key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48;
 	pgste_set_unlock(ptep, pgste);

arch/s390/tools/gen_facilities.c+2−1

  * numbering scheme from the Princples of Operations: most significant bit
  * has bit number 0.
  *
- *    Copyright IBM Corp. 2015
+ *    Copyright IBM Corp. 2015, 2018
  *
  */
 		.name = "FACILITIES_KVM_CPUMODEL",
 		.bits = (int[]){
+			156, /* etoken facility */
 			-1  /* END */
 		}
 	},

include/linux/kvm_host.h+7−0

@@ -309,6 +309,13 @@ static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memsl · 308 unmodified lines
309	return ALIGN(memslot->npages, BITS_PER_LONG) / 8;	309	return ALIGN(memslot->npages, BITS_PER_LONG) / 8;
310	}	310	}
311		311
		312	static inline unsigned long kvm_second_dirty_bitmap(struct kvm_memory_slot memslot)
		313	{
		314	unsigned long len = kvm_dirty_bitmap_bytes(memslot);
		315
		316	return memslot->dirty_bitmap + len / sizeof(*memslot->dirty_bitmap);
		317	}
		318
312	struct kvm_s390_adapter_int {	319	struct kvm_s390_adapter_int {
313	u64 ind_addr;	320	u64 ind_addr;
314	u64 summary_addr;	321	u64 summary_addr;

include/uapi/linux/kvm.h+1−0

@@ -949,6 +949,7 @@ struct kvm_ppc_resize_hpt { · 948 unmodified lines
949	#define KVM_CAP_GET_MSR_FEATURES 153	949	#define KVM_CAP_GET_MSR_FEATURES 153
950	#define KVM_CAP_HYPERV_EVENTFD 154	950	#define KVM_CAP_HYPERV_EVENTFD 154
951	#define KVM_CAP_HYPERV_TLBFLUSH 155	951	#define KVM_CAP_HYPERV_TLBFLUSH 155
		952	#define KVM_CAP_S390_HPAGE_1M 156
952		953
953	#ifdef KVM_CAP_IRQ_ROUTING	954	#ifdef KVM_CAP_IRQ_ROUTING
954		955

virt/kvm/kvm_main.c+1−1

@@ -1169,7 +1169,7 @@ int kvm_get_dirty_log_protect(struct kvm *kvm, · 1168 unmodified lines
1169		1169
1170	n = kvm_dirty_bitmap_bytes(memslot);	1170	n = kvm_dirty_bitmap_bytes(memslot);
1171		1171
1172	dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);	1172	dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
1173	memset(dirty_bitmap_buffer, 0, n);	1173	memset(dirty_bitmap_buffer, 0, n);
1174		1174
1175	spin_lock(&kvm->mmu_lock);	1175	spin_lock(&kvm->mmu_lock);