Merge tag 'kvm-s390-next-4.19-1' of git://git.kernel.org/pub/scm/linux/kernel/git/kvms390/linux into HEAD

KVM: s390: Features for 4.19 - initial version for host large page support. Must be enabled with module parameter hpage=1 and will conflict with the nested=1 parameter. - enable etoken facility for guests - Fixes

Paolo Bonzini committed 8 years ago

Commit 85eae572 parents 3a1174c 237584619 files · +987 −269Browse files

Unified · Split

19 files

Documentation/virtual/kvm/api.txt +16 −0
arch/s390/include/asm/gmap.h +10 −0
arch/s390/include/asm/hugetlb.h +4 −1
arch/s390/include/asm/kvm_host.h +4 −7
arch/s390/include/asm/mmu.h +2 −0
arch/s390/include/asm/mmu_context.h +1 −0
arch/s390/include/asm/pgtable.h +10 −3
arch/s390/include/uapi/asm/kvm.h +4 −1
arch/s390/kvm/kvm-s390.c +250 −133
arch/s390/kvm/priv.c +94 −49
arch/s390/kvm/vsie.c +9 −2
arch/s390/mm/gmap.c +431 −23
arch/s390/mm/hugetlbpage.c +24 −0
arch/s390/mm/pageattr.c +2 −4
arch/s390/mm/pgtable.c +115 −44
arch/s390/tools/gen_facilities.c +2 −1
include/linux/kvm_host.h +7 −0
include/uapi/linux/kvm.h +1 −0
virt/kvm/kvm_main.c +1 −1

Documentation/virtual/kvm/api.txt+16−0

@@ -4391,6 +4391,22 @@ all such vmexits. · 4390 unmodified lines
4391	4391
4392	4392	Do not enable KVM_FEATURE_PV_UNHALT if you disable HLT exits.
4393	4393
	4394	+7.14 KVM_CAP_S390_HPAGE_1M
	4395	+
	4396	+Architectures: s390
	4397	+Parameters: none
	4398	+Returns: 0 on success, -EINVAL if hpage module parameter was not set
	4399	+ or cmma is enabled
	4400	+
	4401	+With this capability the KVM support for memory backing with 1m pages
	4402	+through hugetlbfs can be enabled for a VM. After the capability is
	4403	+enabled, cmma can't be enabled anymore and pfmfi and the storage key
	4404	+interpretation are disabled. If cmma has already been enabled or the
	4405	+hpage module parameter is not set to 1, -EINVAL is returned.
	4406	+
	4407	+While it is generally possible to create a huge page backed VM without
	4408	+this capability, the VM will not be able to run.
	4409	+
4394	4410	8. Other capabilities.
4395	4411	----------------------
4396	4412

arch/s390/include/asm/gmap.h+10−0

@@ -9,6 +9,14 @@ · 8 unmodified lines
9	9	#ifndef _ASM_S390_GMAP_H
10	10	#define _ASM_S390_GMAP_H
11	11
	12	+/* Generic bits for GMAP notification on DAT table entry changes. */
	13	+#define GMAP_NOTIFY_SHADOW 0x2
	14	+#define GMAP_NOTIFY_MPROT 0x1
	15	+
	16	+/* Status bits only for huge segment entries */
	17	+#define _SEGMENT_ENTRY_GMAP_IN 0x8000 /* invalidation notify bit */
	18	+#define _SEGMENT_ENTRY_GMAP_UC 0x4000 /* dirty (migration) */
	19	+
12	20	/**
13	21	* struct gmap_struct - guest address space
14	22	* @list: list head for the mm->context gmap list
@@ -132,4 +140,6 @@ void gmap_pte_notify(struct mm_struct , unsigned long addr, pte_t , · 117 unmodified lines
132	140	int gmap_mprotect_notify(struct gmap *, unsigned long start,
133	141	unsigned long len, int prot);
134	142
	143	+void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long dirty_bitmap[4],
	144	+ unsigned long gaddr, unsigned long vmaddr);
135	145	#endif /* _ASM_S390_GMAP_H */

arch/s390/include/asm/hugetlb.h+4−1

@@ -37,7 +37,10 @@ static inline int prepare_hugepage_range(struct file *file, · 36 unmodified lines
37	37	return 0;
38	38	}
39	39
40		−#define arch_clear_hugepage_flags(page) do { } while (0)
	40	+static inline void arch_clear_hugepage_flags(struct page *page)
	41	+{
	42	+ clear_bit(PG_arch_1, &page->flags);
	43	+}
41	44
42	45	static inline void huge_pte_clear(struct mm_struct *mm, unsigned long addr,
43	46	pte_t *ptep, unsigned long sz)

arch/s390/include/asm/kvm_host.h+4−7

@@ -269,6 +269,7 @@ struct kvm_s390_sie_block { · 268 unmodified lines
269	269	__u8 reserved1c0[8]; /* 0x01c0 */
270	270	#define ECD_HOSTREGMGMT 0x20000000
271	271	#define ECD_MEF 0x08000000
	272	+#define ECD_ETOKENF 0x02000000
272	273	__u32 ecd; /* 0x01c8 */
273	274	__u8 reserved1cc[18]; /* 0x01cc */
274	275	__u64 pp; /* 0x01de */
@@ -655,6 +656,7 @@ struct kvm_vcpu_arch { · 380 unmodified lines
655	656	seqcount_t cputm_seqcount;
656	657	__u64 cputm_start;
657	658	bool gs_enabled;
	659	+ bool skey_enabled;
658	660	};
659	661
660	662	struct kvm_vm_stat {
@@ -793,12 +795,6 @@ struct kvm_s390_vsie { · 132 unmodified lines
793	795	struct page *pages[KVM_MAX_VCPUS];
794	796	};
795	797
796		−struct kvm_s390_migration_state {
797		− unsigned long bitmap_size; /* in bits (number of guest pages) */
798		− atomic64_t dirty_pages; /* number of dirty pages */
799		− unsigned long *pgste_bitmap;
800		−};
801		−
802	798	struct kvm_arch{
803	799	void *sca;
804	800	int use_esca;
@@ -828,7 +824,8 @@ struct kvm_arch{ · 23 unmodified lines
828	824	struct kvm_s390_vsie vsie;
829	825	u8 epdx;
830	826	u64 epoch;
831		− struct kvm_s390_migration_state *migration_state;
	827	+ int migration_mode;
	828	+ atomic64_t cmma_dirty_pages;
832	829	/* subset of available cpu features enabled by user space */
833	830	DECLARE_BITMAP(cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
834	831	struct kvm_s390_gisa *gisa;

arch/s390/include/asm/mmu.h+2−0

@@ -24,6 +24,8 @@ typedef struct { · 23 unmodified lines
24	24	unsigned int uses_skeys:1;
25	25	/* The mmu context uses CMM. */
26	26	unsigned int uses_cmm:1;
	27	+ /* The gmaps associated with this context are allowed to use huge pages. */
	28	+ unsigned int allow_gmap_hpage_1m:1;
27	29	} mm_context_t;
28	30
29	31	#define INIT_MM_CONTEXT(name) \

arch/s390/include/asm/mmu_context.h+1−0

@@ -32,6 +32,7 @@ static inline int init_new_context(struct task_struct *tsk, · 31 unmodified lines
32	32	mm->context.has_pgste = 0;
33	33	mm->context.uses_skeys = 0;
34	34	mm->context.uses_cmm = 0;
	35	+ mm->context.allow_gmap_hpage_1m = 0;
35	36	#endif
36	37	switch (mm->context.asce_limit) {
37	38	case _REGION2_SIZE:

arch/s390/include/asm/pgtable.h+10−3

@@ -268,8 +268,10 @@ static inline int is_module_addr(void *addr) · 267 unmodified lines
268	268	#define _REGION_ENTRY_BITS_LARGE 0xffffffff8000fe2fUL
269	269
270	270	/* Bits in the segment table entry */
271		−#define _SEGMENT_ENTRY_BITS 0xfffffffffffffe33UL
272		−#define _SEGMENT_ENTRY_BITS_LARGE 0xfffffffffff0ff33UL
	271	+#define _SEGMENT_ENTRY_BITS 0xfffffffffffffe33UL
	272	+#define _SEGMENT_ENTRY_BITS_LARGE 0xfffffffffff0ff33UL
	273	+#define _SEGMENT_ENTRY_HARDWARE_BITS 0xfffffffffffffe30UL
	274	+#define _SEGMENT_ENTRY_HARDWARE_BITS_LARGE 0xfffffffffff00730UL
273	275	#define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address */
274	276	#define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL/* page table origin */
275	277	#define _SEGMENT_ENTRY_PROTECT 0x200 /* segment protection bit */
@@ -1101,7 +1103,8 @@ int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, · 825 unmodified lines
1101	1103	pte_t sptep, pte_t tptep, pte_t pte);
1102	1104	void ptep_unshadow_pte(struct mm_struct mm, unsigned long saddr, pte_t ptep);
1103	1105
1104		−bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long address);
	1106	+bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long address,
	1107	+ pte_t *ptep);
1105	1108	int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
1106	1109	unsigned char key, bool nq);
1107	1110	int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
@@ -1116,6 +1119,10 @@ int set_pgste_bits(struct mm_struct *mm, unsigned long addr, · 8 unmodified lines
1116	1119	int get_pgste(struct mm_struct mm, unsigned long hva, unsigned long pgstep);
1117	1120	int pgste_perform_essa(struct mm_struct *mm, unsigned long hva, int orc,
1118	1121	unsigned long oldpte, unsigned long oldpgste);
	1122	+void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr);
	1123	+void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr);
	1124	+void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr);
	1125	+void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr);
1119	1126
1120	1127	/*
1121	1128	* Certain architectures need to do special things when PTEs

arch/s390/include/uapi/asm/kvm.h+4−1

@@ -4,7 +4,7 @@ · 3 unmodified lines
4	4	/*
5	5	* KVM s390 specific structures and definitions
6	6	*
7		− * Copyright IBM Corp. 2008
	7	+ * Copyright IBM Corp. 2008, 2018
8	8	*
9	9	* Author(s): Carsten Otte <cotte@de.ibm.com>
10	10	* Christian Borntraeger <borntraeger@de.ibm.com>
@@ -225,6 +225,7 @@ struct kvm_guest_debug_arch { · 214 unmodified lines
225	225	#define KVM_SYNC_FPRS (1UL << 8)
226	226	#define KVM_SYNC_GSCB (1UL << 9)
227	227	#define KVM_SYNC_BPBC (1UL << 10)
	228	+#define KVM_SYNC_ETOKEN (1UL << 11)
228	229	/* length and alignment of the sdnx as a power of two */
229	230	#define SDNXC 8
230	231	#define SDNXL (1UL << SDNXC)
@@ -258,6 +259,8 @@ struct kvm_sync_regs { · 27 unmodified lines
258	259	struct {
259	260	__u64 reserved1[2];
260	261	__u64 gscb[4];
	262	+ __u64 etoken;
	263	+ __u64 etoken_extension;
261	264	};
262	265	};
263	266	};

arch/s390/kvm/kvm-s390.c+250−133

@@ -172,6 +172,10 @@ static int nested; · 171 unmodified lines
172	172	module_param(nested, int, S_IRUGO);
173	173	MODULE_PARM_DESC(nested, "Nested virtualization support");
174	174
	175	+/* allow 1m huge page guest backing, if !nested */
	176	+static int hpage;
	177	+module_param(hpage, int, 0444);
	178	+MODULE_PARM_DESC(hpage, "1m huge page backing support");
175	179
176	180	/*
177	181	* For now we handle at most 16 double words as this is what the s390 base
@@ -475,6 +479,11 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 297 unmodified lines
475	479	case KVM_CAP_S390_AIS_MIGRATION:
476	480	r = 1;
477	481	break;
	482	+ case KVM_CAP_S390_HPAGE_1M:
	483	+ r = 0;
	484	+ if (hpage)
	485	+ r = 1;
	486	+ break;
478	487	case KVM_CAP_S390_MEM_OP:
479	488	r = MEM_OP_MAX_SIZE;
480	489	break;
@@ -511,19 +520,30 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) · 30 unmodified lines
511	520	}
512	521
513	522	static void kvm_s390_sync_dirty_log(struct kvm *kvm,
514		− struct kvm_memory_slot *memslot)
	523	+ struct kvm_memory_slot *memslot)
515	524	{
	525	+ int i;
516	526	gfn_t cur_gfn, last_gfn;
517		− unsigned long address;
	527	+ unsigned long gaddr, vmaddr;
518	528	struct gmap *gmap = kvm->arch.gmap;
	529	+ DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
519	530
520		− /* Loop over all guest pages */
	531	+ /* Loop over all guest segments */
	532	+ cur_gfn = memslot->base_gfn;
521	533	last_gfn = memslot->base_gfn + memslot->npages;
522		− for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) {
523		− address = gfn_to_hva_memslot(memslot, cur_gfn);
	534	+ for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
	535	+ gaddr = gfn_to_gpa(cur_gfn);
	536	+ vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
	537	+ if (kvm_is_error_hva(vmaddr))
	538	+ continue;
	539	+
	540	+ bitmap_zero(bitmap, _PAGE_ENTRIES);
	541	+ gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
	542	+ for (i = 0; i < _PAGE_ENTRIES; i++) {
	543	+ if (test_bit(i, bitmap))
	544	+ mark_page_dirty(kvm, cur_gfn + i);
	545	+ }
524	546
525		− if (test_and_clear_guest_dirty(gmap->mm, address))
526		− mark_page_dirty(kvm, cur_gfn);
527	547	if (fatal_signal_pending(current))
528	548	return;
529	549	cond_resched();
@@ -667,6 +687,27 @@ static int kvm_vm_ioctl_enable_cap(struct kvm kvm, struct kvm_enable_cap cap) · 137 unmodified lines
667	687	VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
668	688	r ? "(not available)" : "(success)");
669	689	break;
	690	+ case KVM_CAP_S390_HPAGE_1M:
	691	+ mutex_lock(&kvm->lock);
	692	+ if (kvm->created_vcpus)
	693	+ r = -EBUSY;
	694	+ else if (!hpage \|\| kvm->arch.use_cmma)
	695	+ r = -EINVAL;
	696	+ else {
	697	+ r = 0;
	698	+ kvm->mm->context.allow_gmap_hpage_1m = 1;
	699	+ /*
	700	+ * We might have to create fake 4k page
	701	+ * tables. To avoid that the hardware works on
	702	+ * stale PGSTEs, we emulate these instructions.
	703	+ */
	704	+ kvm->arch.use_skf = 0;
	705	+ kvm->arch.use_pfmfi = 0;
	706	+ }
	707	+ mutex_unlock(&kvm->lock);
	708	+ VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
	709	+ r ? "(not available)" : "(success)");
	710	+ break;
670	711	case KVM_CAP_S390_USER_STSI:
671	712	VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
672	713	kvm->arch.user_stsi = 1;
@@ -714,10 +755,13 @@ static int kvm_s390_set_mem_control(struct kvm kvm, struct kvm_device_attr att · 41 unmodified lines
714	755	if (!sclp.has_cmma)
715	756	break;
716	757
717		− ret = -EBUSY;
718	758	VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
719	759	mutex_lock(&kvm->lock);
720		− if (!kvm->created_vcpus) {
	760	+ if (kvm->created_vcpus)
	761	+ ret = -EBUSY;
	762	+ else if (kvm->mm->context.allow_gmap_hpage_1m)
	763	+ ret = -EINVAL;
	764	+ else {
721	765	kvm->arch.use_cmma = 1;
722	766	/* Not compatible with cmma. */
723	767	kvm->arch.use_pfmfi = 0;
@@ -862,54 +906,37 @@ static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req) · 138 unmodified lines
862	906	*/
863	907	static int kvm_s390_vm_start_migration(struct kvm *kvm)
864	908	{
865		− struct kvm_s390_migration_state *mgs;
866	909	struct kvm_memory_slot *ms;
867		− /* should be the only one */
868	910	struct kvm_memslots *slots;
869		− unsigned long ram_pages;
	911	+ unsigned long ram_pages = 0;
870	912	int slotnr;
871	913
872	914	/* migration mode already enabled */
873		− if (kvm->arch.migration_state)
	915	+ if (kvm->arch.migration_mode)
874	916	return 0;
875		−
876	917	slots = kvm_memslots(kvm);
877	918	if (!slots \|\| !slots->used_slots)
878	919	return -EINVAL;
879	920
880		− mgs = kzalloc(sizeof(*mgs), GFP_KERNEL);
881		− if (!mgs)
882		− return -ENOMEM;
883		− kvm->arch.migration_state = mgs;
884		−
885		− if (kvm->arch.use_cmma) {
	921	+ if (!kvm->arch.use_cmma) {
	922	+ kvm->arch.migration_mode = 1;
	923	+ return 0;
	924	+ }
	925	+ /* mark all the pages in active slots as dirty */
	926	+ for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
	927	+ ms = slots->memslots + slotnr;
886	928	/*
887		− * Get the first slot. They are reverse sorted by base_gfn, so
888		− * the first slot is also the one at the end of the address
889		− * space. We have verified above that at least one slot is
890		− * present.
	929	+ * The second half of the bitmap is only used on x86,
	930	+ * and would be wasted otherwise, so we put it to good
	931	+ * use here to keep track of the state of the storage
	932	+ * attributes.
891	933	*/
892		− ms = slots->memslots;
893		− /* round up so we only use full longs */
894		− ram_pages = roundup(ms->base_gfn + ms->npages, BITS_PER_LONG);
895		− /* allocate enough bytes to store all the bits */
896		− mgs->pgste_bitmap = vmalloc(ram_pages / 8);
897		− if (!mgs->pgste_bitmap) {
898		− kfree(mgs);
899		− kvm->arch.migration_state = NULL;
900		− return -ENOMEM;
901		− }
902		−
903		− mgs->bitmap_size = ram_pages;
904		− atomic64_set(&mgs->dirty_pages, ram_pages);
905		− /* mark all the pages in active slots as dirty */
906		− for (slotnr = 0; slotnr < slots->used_slots; slotnr++) {
907		− ms = slots->memslots + slotnr;
908		− bitmap_set(mgs->pgste_bitmap, ms->base_gfn, ms->npages);
909		− }
910		−
911		− kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
	934	+ memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
	935	+ ram_pages += ms->npages;
912	936	}
	937	+ atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
	938	+ kvm->arch.migration_mode = 1;
	939	+ kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
913	940	return 0;
914	941	}
915	942
@@ -919,21 +946,12 @@ static int kvm_s390_vm_start_migration(struct kvm *kvm) · 3 unmodified lines
919	946	*/
920	947	static int kvm_s390_vm_stop_migration(struct kvm *kvm)
921	948	{
922		− struct kvm_s390_migration_state *mgs;
923		−
924	949	/* migration mode already disabled */
925		− if (!kvm->arch.migration_state)
	950	+ if (!kvm->arch.migration_mode)
926	951	return 0;
927		− mgs = kvm->arch.migration_state;
928		− kvm->arch.migration_state = NULL;
929		−
930		− if (kvm->arch.use_cmma) {
	952	+ kvm->arch.migration_mode = 0;
	953	+ if (kvm->arch.use_cmma)
931	954	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
932		− /* We have to wait for the essa emulation to finish */
933		− synchronize_srcu(&kvm->srcu);
934		− vfree(mgs->pgste_bitmap);
935		− }
936		− kfree(mgs);
937	955	return 0;
938	956	}
939	957
@@ -961,7 +979,7 @@ static int kvm_s390_vm_set_migration(struct kvm *kvm, · 21 unmodified lines
961	979	static int kvm_s390_vm_get_migration(struct kvm *kvm,
962	980	struct kvm_device_attr *attr)
963	981	{
964		− u64 mig = (kvm->arch.migration_state != NULL);
	982	+ u64 mig = kvm->arch.migration_mode;
965	983
966	984	if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
967	985	return -ENXIO;
@@ -1540,6 +1558,7 @@ static long kvm_s390_set_skeys(struct kvm kvm, struct kvm_s390_skeys args) · 572 unmodified lines
1540	1558	uint8_t *keys;
1541	1559	uint64_t hva;
1542	1560	int srcu_idx, i, r = 0;
	1561	+ bool unlocked;
1543	1562
1544	1563	if (args->flags != 0)
1545	1564	return -EINVAL;
@@ -1564,9 +1583,11 @@ static long kvm_s390_set_skeys(struct kvm kvm, struct kvm_s390_skeys args) · 18 unmodified lines
1564	1583	if (r)
1565	1584	goto out;
1566	1585
	1586	+ i = 0;
1567	1587	down_read(&current->mm->mmap_sem);
1568	1588	srcu_idx = srcu_read_lock(&kvm->srcu);
1569		− for (i = 0; i < args->count; i++) {
	1589	+ while (i < args->count) {
	1590	+ unlocked = false;
1570	1591	hva = gfn_to_hva(kvm, args->start_gfn + i);
1571	1592	if (kvm_is_error_hva(hva)) {
1572	1593	r = -EFAULT;
@@ -1580,8 +1601,14 @@ static long kvm_s390_set_skeys(struct kvm kvm, struct kvm_s390_skeys args) · 7 unmodified lines
1580	1601	}
1581	1602
1582	1603	r = set_guest_storage_key(current->mm, hva, keys[i], 0);
1583		− if (r)
1584		− break;
	1604	+ if (r) {
	1605	+ r = fixup_user_fault(current, current->mm, hva,
	1606	+ FAULT_FLAG_WRITE, &unlocked);
	1607	+ if (r)
	1608	+ break;
	1609	+ }
	1610	+ if (!r)
	1611	+ i++;
1585	1612	}
1586	1613	srcu_read_unlock(&kvm->srcu, srcu_idx);
1587	1614	up_read(&current->mm->mmap_sem);
@@ -1599,6 +1626,134 @@ out: · 11 unmodified lines
1599	1626	/* for consistency */
1600	1627	#define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
1601	1628
	1629	+/*
	1630	+ * Similar to gfn_to_memslot, but returns the index of a memslot also when the
	1631	+ * address falls in a hole. In that case the index of one of the memslots
	1632	+ * bordering the hole is returned.
	1633	+ */
	1634	+static int gfn_to_memslot_approx(struct kvm_memslots *slots, gfn_t gfn)
	1635	+{
	1636	+ int start = 0, end = slots->used_slots;
	1637	+ int slot = atomic_read(&slots->lru_slot);
	1638	+ struct kvm_memory_slot *memslots = slots->memslots;
	1639	+
	1640	+ if (gfn >= memslots[slot].base_gfn &&
	1641	+ gfn < memslots[slot].base_gfn + memslots[slot].npages)
	1642	+ return slot;
	1643	+
	1644	+ while (start < end) {
	1645	+ slot = start + (end - start) / 2;
	1646	+
	1647	+ if (gfn >= memslots[slot].base_gfn)
	1648	+ end = slot;
	1649	+ else
	1650	+ start = slot + 1;
	1651	+ }
	1652	+
	1653	+ if (gfn >= memslots[start].base_gfn &&
	1654	+ gfn < memslots[start].base_gfn + memslots[start].npages) {
	1655	+ atomic_set(&slots->lru_slot, start);
	1656	+ }
	1657	+
	1658	+ return start;
	1659	+}
	1660	+
	1661	+static int kvm_s390_peek_cmma(struct kvm kvm, struct kvm_s390_cmma_log args,
	1662	+ u8 *res, unsigned long bufsize)
	1663	+{
	1664	+ unsigned long pgstev, hva, cur_gfn = args->start_gfn;
	1665	+
	1666	+ args->count = 0;
	1667	+ while (args->count < bufsize) {
	1668	+ hva = gfn_to_hva(kvm, cur_gfn);
	1669	+ /*
	1670	+ * We return an error if the first value was invalid, but we
	1671	+ * return successfully if at least one value was copied.
	1672	+ */
	1673	+ if (kvm_is_error_hva(hva))
	1674	+ return args->count ? 0 : -EFAULT;
	1675	+ if (get_pgste(kvm->mm, hva, &pgstev) < 0)
	1676	+ pgstev = 0;
	1677	+ res[args->count++] = (pgstev >> 24) & 0x43;
	1678	+ cur_gfn++;
	1679	+ }
	1680	+
	1681	+ return 0;
	1682	+}
	1683	+
	1684	+static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
	1685	+ unsigned long cur_gfn)
	1686	+{
	1687	+ int slotidx = gfn_to_memslot_approx(slots, cur_gfn);
	1688	+ struct kvm_memory_slot *ms = slots->memslots + slotidx;
	1689	+ unsigned long ofs = cur_gfn - ms->base_gfn;
	1690	+
	1691	+ if (ms->base_gfn + ms->npages <= cur_gfn) {
	1692	+ slotidx--;
	1693	+ /* If we are above the highest slot, wrap around */
	1694	+ if (slotidx < 0)
	1695	+ slotidx = slots->used_slots - 1;
	1696	+
	1697	+ ms = slots->memslots + slotidx;
	1698	+ ofs = 0;
	1699	+ }
	1700	+ ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
	1701	+ while ((slotidx > 0) && (ofs >= ms->npages)) {
	1702	+ slotidx--;
	1703	+ ms = slots->memslots + slotidx;
	1704	+ ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, 0);
	1705	+ }
	1706	+ return ms->base_gfn + ofs;
	1707	+}
	1708	+
	1709	+static int kvm_s390_get_cmma(struct kvm kvm, struct kvm_s390_cmma_log args,
	1710	+ u8 *res, unsigned long bufsize)
	1711	+{
	1712	+ unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
	1713	+ struct kvm_memslots *slots = kvm_memslots(kvm);
	1714	+ struct kvm_memory_slot *ms;
	1715	+
	1716	+ cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
	1717	+ ms = gfn_to_memslot(kvm, cur_gfn);
	1718	+ args->count = 0;
	1719	+ args->start_gfn = cur_gfn;
	1720	+ if (!ms)
	1721	+ return 0;
	1722	+ next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
	1723	+ mem_end = slots->memslots[0].base_gfn + slots->memslots[0].npages;
	1724	+
	1725	+ while (args->count < bufsize) {
	1726	+ hva = gfn_to_hva(kvm, cur_gfn);
	1727	+ if (kvm_is_error_hva(hva))
	1728	+ return 0;
	1729	+ /* Decrement only if we actually flipped the bit to 0 */
	1730	+ if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
	1731	+ atomic64_dec(&kvm->arch.cmma_dirty_pages);
	1732	+ if (get_pgste(kvm->mm, hva, &pgstev) < 0)
	1733	+ pgstev = 0;
	1734	+ /* Save the value */
	1735	+ res[args->count++] = (pgstev >> 24) & 0x43;
	1736	+ /* If the next bit is too far away, stop. */
	1737	+ if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
	1738	+ return 0;
	1739	+ /* If we reached the previous "next", find the next one */
	1740	+ if (cur_gfn == next_gfn)
	1741	+ next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
	1742	+ /* Reached the end of memory or of the buffer, stop */
	1743	+ if ((next_gfn >= mem_end) \|\|
	1744	+ (next_gfn - args->start_gfn >= bufsize))
	1745	+ return 0;
	1746	+ cur_gfn++;
	1747	+ /* Reached the end of the current memslot, take the next one. */
	1748	+ if (cur_gfn - ms->base_gfn >= ms->npages) {
	1749	+ ms = gfn_to_memslot(kvm, cur_gfn);
	1750	+ if (!ms)
	1751	+ return 0;
	1752	+ }
	1753	+ }
	1754	+ return 0;
	1755	+}
	1756	+
1602	1757	/*
1603	1758	* This function searches for the next page with dirty CMMA attributes, and
1604	1759	* saves the attributes in the buffer up to either the end of the buffer or
@@ -1610,22 +1765,18 @@ out: · 5 unmodified lines
1610	1765	static int kvm_s390_get_cmma_bits(struct kvm *kvm,
1611	1766	struct kvm_s390_cmma_log *args)
1612	1767	{
1613		− struct kvm_s390_migration_state *s = kvm->arch.migration_state;
1614		− unsigned long bufsize, hva, pgstev, i, next, cur;
1615		− int srcu_idx, peek, r = 0, rr;
1616		− u8 *res;
1617		−
1618		− cur = args->start_gfn;
1619		− i = next = pgstev = 0;
	1768	+ unsigned long bufsize;
	1769	+ int srcu_idx, peek, ret;
	1770	+ u8 *values;
1620	1771
1621		− if (unlikely(!kvm->arch.use_cmma))
	1772	+ if (!kvm->arch.use_cmma)
1622	1773	return -ENXIO;
1623	1774	/* Invalid/unsupported flags were specified */
1624	1775	if (args->flags & ~KVM_S390_CMMA_PEEK)
1625	1776	return -EINVAL;
1626	1777	/* Migration mode query, and we are not doing a migration */
1627	1778	peek = !!(args->flags & KVM_S390_CMMA_PEEK);
1628		− if (!peek && !s)
	1779	+ if (!peek && !kvm->arch.migration_mode)
1629	1780	return -EINVAL;
1630	1781	/* CMMA is disabled or was not used, or the buffer has length zero */
1631	1782	bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
@@ -1633,74 +1784,35 @@ static int kvm_s390_get_cmma_bits(struct kvm *kvm, · 1 unmodified line
1633	1784	memset(args, 0, sizeof(*args));
1634	1785	return 0;
1635	1786	}
1636		−
1637		− if (!peek) {
1638		− /* We are not peeking, and there are no dirty pages */
1639		− if (!atomic64_read(&s->dirty_pages)) {
1640		− memset(args, 0, sizeof(*args));
1641		− return 0;
1642		− }
1643		− cur = find_next_bit(s->pgste_bitmap, s->bitmap_size,
1644		− args->start_gfn);
1645		− if (cur >= s->bitmap_size) /* nothing found, loop back */
1646		− cur = find_next_bit(s->pgste_bitmap, s->bitmap_size, 0);
1647		− if (cur >= s->bitmap_size) { /* again! (very unlikely) */
1648		− memset(args, 0, sizeof(*args));
1649		− return 0;
1650		− }
1651		− next = find_next_bit(s->pgste_bitmap, s->bitmap_size, cur + 1);
	1787	+ /* We are not peeking, and there are no dirty pages */
	1788	+ if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
	1789	+ memset(args, 0, sizeof(*args));
	1790	+ return 0;
1652	1791	}
1653	1792
1654		− res = vmalloc(bufsize);
1655		− if (!res)
	1793	+ values = vmalloc(bufsize);
	1794	+ if (!values)
1656	1795	return -ENOMEM;
1657	1796
1658		− args->start_gfn = cur;
1659		−
1660	1797	down_read(&kvm->mm->mmap_sem);
1661	1798	srcu_idx = srcu_read_lock(&kvm->srcu);
1662		− while (i < bufsize) {
1663		− hva = gfn_to_hva(kvm, cur);
1664		− if (kvm_is_error_hva(hva)) {
1665		− r = -EFAULT;
1666		− break;
1667		− }
1668		− /* decrement only if we actually flipped the bit to 0 */
1669		− if (!peek && test_and_clear_bit(cur, s->pgste_bitmap))
1670		− atomic64_dec(&s->dirty_pages);
1671		− r = get_pgste(kvm->mm, hva, &pgstev);
1672		− if (r < 0)
1673		− pgstev = 0;
1674		− /* save the value */
1675		− res[i++] = (pgstev >> 24) & 0x43;
1676		− /*
1677		− * if the next bit is too far away, stop.
1678		− * if we reached the previous "next", find the next one
1679		− */
1680		− if (!peek) {
1681		− if (next > cur + KVM_S390_MAX_BIT_DISTANCE)
1682		− break;
1683		− if (cur == next)
1684		− next = find_next_bit(s->pgste_bitmap,
1685		− s->bitmap_size, cur + 1);
1686		− /* reached the end of the bitmap or of the buffer, stop */
1687		− if ((next >= s->bitmap_size) \|\|
1688		− (next >= args->start_gfn + bufsize))
1689		− break;
1690		− }
1691		− cur++;
1692		− }
	1799	+ if (peek)
	1800	+ ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
	1801	+ else
	1802	+ ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
1693	1803	srcu_read_unlock(&kvm->srcu, srcu_idx);
1694	1804	up_read(&kvm->mm->mmap_sem);
1695		− args->count = i;
1696		− args->remaining = s ? atomic64_read(&s->dirty_pages) : 0;
1697	1805
1698		− rr = copy_to_user((void __user *)args->values, res, args->count);
1699		− if (rr)
1700		− r = -EFAULT;
	1806	+ if (kvm->arch.migration_mode)
	1807	+ args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
	1808	+ else
	1809	+ args->remaining = 0;
1701	1810
1702		− vfree(res);
1703		− return r;
	1811	+ if (copy_to_user((void __user *)args->values, values, args->count))
	1812	+ ret = -EFAULT;
	1813	+
	1814	+ vfree(values);
	1815	+ return ret;
1704	1816	}
1705	1817
1706	1818	/*
@@ -2139,10 +2251,6 @@ void kvm_arch_destroy_vm(struct kvm *kvm) · 432 unmodified lines
2139	2251	kvm_s390_destroy_adapters(kvm);
2140	2252	kvm_s390_clear_float_irqs(kvm);
2141	2253	kvm_s390_vsie_destroy(kvm);
2142		− if (kvm->arch.migration_state) {
2143		− vfree(kvm->arch.migration_state->pgste_bitmap);
2144		− kfree(kvm->arch.migration_state);
2145		− }
2146	2254	KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
2147	2255	}
2148	2256
@@ -2300,6 +2408,8 @@ int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) · 151 unmodified lines
2300	2408	vcpu->run->kvm_valid_regs \|= KVM_SYNC_BPBC;
2301	2409	if (test_kvm_facility(vcpu->kvm, 133))
2302	2410	vcpu->run->kvm_valid_regs \|= KVM_SYNC_GSCB;
	2411	+ if (test_kvm_facility(vcpu->kvm, 156))
	2412	+ vcpu->run->kvm_valid_regs \|= KVM_SYNC_ETOKEN;
2303	2413	/* fprs can be synchronized via vrs, even if the guest has no vx. With
2304	2414	* MACHINE_HAS_VX, (load\|store)_fpu_regs() will work with vrs format.
2305	2415	*/
@@ -2549,7 +2659,8 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) · 243 unmodified lines
2549	2659	}
2550	2660	if (test_kvm_facility(vcpu->kvm, 139))
2551	2661	vcpu->arch.sie_block->ecd \|= ECD_MEF;
2552		−
	2662	+ if (test_kvm_facility(vcpu->kvm, 156))
	2663	+ vcpu->arch.sie_block->ecd \|= ECD_ETOKENF;
2553	2664	if (vcpu->arch.sie_block->gd) {
2554	2665	vcpu->arch.sie_block->eca \|= ECA_AIV;
2555	2666	VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
@@ -3467,6 +3578,7 @@ static void sync_regs(struct kvm_vcpu vcpu, struct kvm_run kvm_run) · 911 unmodified lines
3467	3578	}
3468	3579	preempt_enable();
3469	3580	}
	3581	+ /* SIE will load etoken directly from SDNX and therefore kvm_run */
3470	3582
3471	3583	kvm_run->kvm_dirty_regs = 0;
3472	3584	}
@@ -3506,7 +3618,7 @@ static void store_regs(struct kvm_vcpu vcpu, struct kvm_run kvm_run) · 33 unmodified lines
3506	3618	__ctl_clear_bit(2, 4);
3507	3619	vcpu->arch.host_gscb = NULL;
3508	3620	}
3509		−
	3621	+ /* SIE will save etoken directly into SDNX and therefore kvm_run */
3510	3622	}
3511	3623
3512	3624	int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu vcpu, struct kvm_run kvm_run)
@@ -4082,6 +4194,11 @@ static int __init kvm_s390_init(void) · 569 unmodified lines
4082	4194	return -ENODEV;
4083	4195	}
4084	4196
	4197	+ if (nested && hpage) {
	4198	+ pr_info("nested (vSIE) and hpage (huge page backing) can currently not be activated concurrently");
	4199	+ return -EINVAL;
	4200	+ }
	4201	+
4085	4202	for (i = 0; i < 16; i++)
4086	4203	kvm_s390_fac_base[i] \|=
4087	4204	S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i);

arch/s390/kvm/priv.c+94−49

@@ -205,13 +205,10 @@ static int handle_store_cpu_address(struct kvm_vcpu *vcpu) · 204 unmodified lines
205	205	int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu)
206	206	{
207	207	int rc;
208		− struct kvm_s390_sie_block *sie_block = vcpu->arch.sie_block;
209	208
210	209	trace_kvm_s390_skey_related_inst(vcpu);
211	210	/* Already enabled? */
212		− if (vcpu->kvm->arch.use_skf &&
213		− !(sie_block->ictl & (ICTL_ISKE \| ICTL_SSKE \| ICTL_RRBE)) &&
214		− !kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
	211	+ if (vcpu->arch.skey_enabled)
215	212	return 0;
216	213
217	214	rc = s390_enable_skey();
@@ -222,9 +219,10 @@ int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu) · 4 unmodified lines
222	219	if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS))
223	220	kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS);
224	221	if (!vcpu->kvm->arch.use_skf)
225		− sie_block->ictl \|= ICTL_ISKE \| ICTL_SSKE \| ICTL_RRBE;
	222	+ vcpu->arch.sie_block->ictl \|= ICTL_ISKE \| ICTL_SSKE \| ICTL_RRBE;
226	223	else
227		− sie_block->ictl &= ~(ICTL_ISKE \| ICTL_SSKE \| ICTL_RRBE);
	224	+ vcpu->arch.sie_block->ictl &= ~(ICTL_ISKE \| ICTL_SSKE \| ICTL_RRBE);
	225	+ vcpu->arch.skey_enabled = true;
228	226	return 0;
229	227	}
230	228
@@ -246,9 +244,10 @@ static int try_handle_skey(struct kvm_vcpu *vcpu) · 15 unmodified lines
246	244
247	245	static int handle_iske(struct kvm_vcpu *vcpu)
248	246	{
249		− unsigned long addr;
	247	+ unsigned long gaddr, vmaddr;
250	248	unsigned char key;
251	249	int reg1, reg2;
	250	+ bool unlocked;
252	251	int rc;
253	252
254	253	vcpu->stat.instruction_iske++;
@@ -262,18 +261,28 @@ static int handle_iske(struct kvm_vcpu *vcpu) · 7 unmodified lines
262	261
263	262	kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);
264	263
265		− addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
266		− addr = kvm_s390_logical_to_effective(vcpu, addr);
267		− addr = kvm_s390_real_to_abs(vcpu, addr);
268		− addr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(addr));
269		− if (kvm_is_error_hva(addr))
	264	+ gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
	265	+ gaddr = kvm_s390_logical_to_effective(vcpu, gaddr);
	266	+ gaddr = kvm_s390_real_to_abs(vcpu, gaddr);
	267	+ vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr));
	268	+ if (kvm_is_error_hva(vmaddr))
270	269	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
271		−
	270	+retry:
	271	+ unlocked = false;
272	272	down_read(&current->mm->mmap_sem);
273		− rc = get_guest_storage_key(current->mm, addr, &key);
274		− up_read(&current->mm->mmap_sem);
	273	+ rc = get_guest_storage_key(current->mm, vmaddr, &key);
	274	+
	275	+ if (rc) {
	276	+ rc = fixup_user_fault(current, current->mm, vmaddr,
	277	+ FAULT_FLAG_WRITE, &unlocked);
	278	+ if (!rc) {
	279	+ up_read(&current->mm->mmap_sem);
	280	+ goto retry;
	281	+ }
	282	+ }
275	283	if (rc)
276	284	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
	285	+ up_read(&current->mm->mmap_sem);
277	286	vcpu->run->s.regs.gprs[reg1] &= ~0xff;
278	287	vcpu->run->s.regs.gprs[reg1] \|= key;
279	288	return 0;
@@ -281,8 +290,9 @@ static int handle_iske(struct kvm_vcpu *vcpu) · 1 unmodified line
281	290
282	291	static int handle_rrbe(struct kvm_vcpu *vcpu)
283	292	{
284		− unsigned long addr;
	293	+ unsigned long vmaddr, gaddr;
285	294	int reg1, reg2;
	295	+ bool unlocked;
286	296	int rc;
287	297
288	298	vcpu->stat.instruction_rrbe++;
@@ -296,19 +306,27 @@ static int handle_rrbe(struct kvm_vcpu *vcpu) · 7 unmodified lines
296	306
297	307	kvm_s390_get_regs_rre(vcpu, &reg1, &reg2);
298	308
299		− addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
300		− addr = kvm_s390_logical_to_effective(vcpu, addr);
301		− addr = kvm_s390_real_to_abs(vcpu, addr);
302		− addr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(addr));
303		− if (kvm_is_error_hva(addr))
	309	+ gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK;
	310	+ gaddr = kvm_s390_logical_to_effective(vcpu, gaddr);
	311	+ gaddr = kvm_s390_real_to_abs(vcpu, gaddr);
	312	+ vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr));
	313	+ if (kvm_is_error_hva(vmaddr))
304	314	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
305		−
	315	+retry:
	316	+ unlocked = false;
306	317	down_read(&current->mm->mmap_sem);
307		− rc = reset_guest_reference_bit(current->mm, addr);
308		− up_read(&current->mm->mmap_sem);
	318	+ rc = reset_guest_reference_bit(current->mm, vmaddr);
	319	+ if (rc < 0) {
	320	+ rc = fixup_user_fault(current, current->mm, vmaddr,
	321	+ FAULT_FLAG_WRITE, &unlocked);
	322	+ if (!rc) {
	323	+ up_read(&current->mm->mmap_sem);
	324	+ goto retry;
	325	+ }
	326	+ }
309	327	if (rc < 0)
310	328	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
311		−
	329	+ up_read(&current->mm->mmap_sem);
312	330	kvm_s390_set_psw_cc(vcpu, rc);
313	331	return 0;
314	332	}
@@ -323,6 +341,7 @@ static int handle_sske(struct kvm_vcpu *vcpu) · 8 unmodified lines
323	341	unsigned long start, end;
324	342	unsigned char key, oldkey;
325	343	int reg1, reg2;
	344	+ bool unlocked;
326	345	int rc;
327	346
328	347	vcpu->stat.instruction_sske++;
@@ -355,19 +374,28 @@ static int handle_sske(struct kvm_vcpu *vcpu) · 26 unmodified lines
355	374	}
356	375
357	376	while (start != end) {
358		− unsigned long addr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start));
	377	+ unsigned long vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start));
	378	+ unlocked = false;
359	379
360		− if (kvm_is_error_hva(addr))
	380	+ if (kvm_is_error_hva(vmaddr))
361	381	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
362	382
363	383	down_read(&current->mm->mmap_sem);
364		− rc = cond_set_guest_storage_key(current->mm, addr, key, &oldkey,
	384	+ rc = cond_set_guest_storage_key(current->mm, vmaddr, key, &oldkey,
365	385	m3 & SSKE_NQ, m3 & SSKE_MR,
366	386	m3 & SSKE_MC);
367		− up_read(&current->mm->mmap_sem);
368		− if (rc < 0)
	387	+
	388	+ if (rc < 0) {
	389	+ rc = fixup_user_fault(current, current->mm, vmaddr,
	390	+ FAULT_FLAG_WRITE, &unlocked);
	391	+ rc = !rc ? -EAGAIN : rc;
	392	+ }
	393	+ if (rc == -EFAULT)
369	394	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
370		− start += PAGE_SIZE;
	395	+
	396	+ up_read(&current->mm->mmap_sem);
	397	+ if (rc >= 0)
	398	+ start += PAGE_SIZE;
371	399	}
372	400
373	401	if (m3 & (SSKE_MC \| SSKE_MR)) {
@@ -948,15 +976,16 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) · 574 unmodified lines
948	976	}
949	977
950	978	while (start != end) {
951		− unsigned long useraddr;
	979	+ unsigned long vmaddr;
	980	+ bool unlocked = false;
952	981
953	982	/* Translate guest address to host address */
954		− useraddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start));
955		− if (kvm_is_error_hva(useraddr))
	983	+ vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start));
	984	+ if (kvm_is_error_hva(vmaddr))
956	985	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
957	986
958	987	if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) {
959		− if (clear_user((void __user *)useraddr, PAGE_SIZE))
	988	+ if (kvm_clear_guest(vcpu->kvm, start, PAGE_SIZE))
960	989	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
961	990	}
962	991
@@ -966,14 +995,20 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) · 3 unmodified lines
966	995	if (rc)
967	996	return rc;
968	997	down_read(&current->mm->mmap_sem);
969		− rc = cond_set_guest_storage_key(current->mm, useraddr,
	998	+ rc = cond_set_guest_storage_key(current->mm, vmaddr,
970	999	key, NULL, nq, mr, mc);
971		− up_read(&current->mm->mmap_sem);
972		− if (rc < 0)
	1000	+ if (rc < 0) {
	1001	+ rc = fixup_user_fault(current, current->mm, vmaddr,
	1002	+ FAULT_FLAG_WRITE, &unlocked);
	1003	+ rc = !rc ? -EAGAIN : rc;
	1004	+ }
	1005	+ if (rc == -EFAULT)
973	1006	return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING);
974		− }
975	1007
976		− start += PAGE_SIZE;
	1008	+ up_read(&current->mm->mmap_sem);
	1009	+ if (rc >= 0)
	1010	+ start += PAGE_SIZE;
	1011	+ }
977	1012	}
978	1013	if (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) {
979	1014	if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) {
@@ -987,9 +1022,11 @@ static int handle_pfmf(struct kvm_vcpu *vcpu) · 7 unmodified lines
987	1022	return 0;
988	1023	}
989	1024
990		−static inline int do_essa(struct kvm_vcpu *vcpu, const int orc)
	1025	+/*
	1026	+ * Must be called with relevant read locks held (kvm->mm->mmap_sem, kvm->srcu)
	1027	+ */
	1028	+static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc)
991	1029	{
992		− struct kvm_s390_migration_state *ms = vcpu->kvm->arch.migration_state;
993	1030	int r1, r2, nappended, entries;
994	1031	unsigned long gfn, hva, res, pgstev, ptev;
995	1032	unsigned long *cbrlo;
@@ -1039,10 +1076,12 @@ static inline int do_essa(struct kvm_vcpu *vcpu, const int orc) · 43 unmodified lines
1039	1076	cbrlo[entries] = gfn << PAGE_SHIFT;
1040	1077	}
1041	1078
1042		− if (orc && gfn < ms->bitmap_size) {
1043		− /* increment only if we are really flipping the bit to 1 */
1044		− if (!test_and_set_bit(gfn, ms->pgste_bitmap))
1045		− atomic64_inc(&ms->dirty_pages);
	1079	+ if (orc) {
	1080	+ struct kvm_memory_slot *ms = gfn_to_memslot(vcpu->kvm, gfn);
	1081	+
	1082	+ /* Increment only if we are really flipping the bit */
	1083	+ if (ms && !test_and_set_bit(gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
	1084	+ atomic64_inc(&vcpu->kvm->arch.cmma_dirty_pages);
1046	1085	}
1047	1086
1048	1087	return nappended;
@@ -1071,7 +1110,7 @@ static int handle_essa(struct kvm_vcpu *vcpu) · 22 unmodified lines
1071	1110	: ESSA_SET_STABLE_IF_RESIDENT))
1072	1111	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
1073	1112
1074		− if (likely(!vcpu->kvm->arch.migration_state)) {
	1113	+ if (!vcpu->kvm->arch.migration_mode) {
1075	1114	/*
1076	1115	* CMMA is enabled in the KVM settings, but is disabled in
1077	1116	* the SIE block and in the mm_context, and we are not doing
@@ -1099,10 +1138,16 @@ static int handle_essa(struct kvm_vcpu *vcpu) · 21 unmodified lines
1099	1138	/* Retry the ESSA instruction */
1100	1139	kvm_s390_retry_instr(vcpu);
1101	1140	} else {
1102		− /* Account for the possible extra cbrl entry */
1103		− i = do_essa(vcpu, orc);
	1141	+ int srcu_idx;
	1142	+
	1143	+ down_read(&vcpu->kvm->mm->mmap_sem);
	1144	+ srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
	1145	+ i = __do_essa(vcpu, orc);
	1146	+ srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
	1147	+ up_read(&vcpu->kvm->mm->mmap_sem);
1104	1148	if (i < 0)
1105	1149	return i;
	1150	+ /* Account for the possible extra cbrl entry */
1106	1151	entries += i;
1107	1152	}
1108	1153	vcpu->arch.sie_block->cbrlo &= PAGE_MASK; /* reset nceo */

arch/s390/kvm/vsie.c+9−2

@@ -2,7 +2,7 @@ · 1 unmodified line
2	2	/*
3	3	* kvm nested virtualization support for s390x
4	4	*
5		− * Copyright IBM Corp. 2016
	5	+ * Copyright IBM Corp. 2016, 2018
6	6	*
7	7	* Author(s): David Hildenbrand <dahi@linux.vnet.ibm.com>
8	8	*/
@@ -378,6 +378,10 @@ static int shadow_scb(struct kvm_vcpu vcpu, struct vsie_page vsie_page) · 369 unmodified lines
378	378	if (test_kvm_facility(vcpu->kvm, 139))
379	379	scb_s->ecd \|= scb_o->ecd & ECD_MEF;
380	380
	381	+ /* etoken */
	382	+ if (test_kvm_facility(vcpu->kvm, 156))
	383	+ scb_s->ecd \|= scb_o->ecd & ECD_ETOKENF;
	384	+
381	385	prepare_ibc(vcpu, vsie_page);
382	386	rc = shadow_crycb(vcpu, vsie_page);
383	387	out:
@@ -627,7 +631,8 @@ static int pin_blocks(struct kvm_vcpu vcpu, struct vsie_page vsie_page) · 243 unmodified lines
627	631	vsie_page->riccbd_gpa = gpa;
628	632	scb_s->riccbd = hpa;
629	633	}
630		− if ((scb_s->ecb & ECB_GS) && !(scb_s->ecd & ECD_HOSTREGMGMT)) {
	634	+ if (((scb_s->ecb & ECB_GS) && !(scb_s->ecd & ECD_HOSTREGMGMT)) \|\|
	635	+ (scb_s->ecd & ECD_ETOKENF)) {
631	636	unsigned long sdnxc;
632	637
633	638	gpa = READ_ONCE(scb_o->sdnxo) & ~0xfUL;
@@ -818,6 +823,8 @@ static int handle_stfle(struct kvm_vcpu vcpu, struct vsie_page vsie_page) · 184 unmodified lines
818	823	* - < 0 if an error occurred
819	824	*/
820	825	static int do_vsie_run(struct kvm_vcpu vcpu, struct vsie_page vsie_page)
	826	+ __releases(vcpu->kvm->srcu)
	827	+ __acquires(vcpu->kvm->srcu)
821	828	{
822	829	struct kvm_s390_sie_block *scb_s = &vsie_page->scb_s;
823	830	struct kvm_s390_sie_block *scb_o = vsie_page->scb_o;

arch/s390/mm/gmap.c+431−23

@@ -2,8 +2,10 @@ · 1 unmodified line
2	2	/*
3	3	* KVM guest address space mapping code
4	4	*
5		− * Copyright IBM Corp. 2007, 2016
	5	+ * Copyright IBM Corp. 2007, 2016, 2018
6	6	* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
	7	+ * David Hildenbrand <david@redhat.com>
	8	+ * Janosch Frank <frankja@linux.vnet.ibm.com>
7	9	*/
8	10
9	11	#include <linux/kernel.h>
@@ -521,6 +523,9 @@ void gmap_unlink(struct mm_struct mm, unsigned long table, · 511 unmodified lines
521	523	rcu_read_unlock();
522	524	}
523	525
	526	+static void gmap_pmdp_xchg(struct gmap gmap, pmd_t old, pmd_t new,
	527	+ unsigned long gaddr);
	528	+
524	529	/**
525	530	* gmap_link - set up shadow page tables to connect a host to a guest address
526	531	* @gmap: pointer to guest mapping meta data structure
@@ -541,6 +546,7 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) · 14 unmodified lines
541	546	p4d_t *p4d;
542	547	pud_t *pud;
543	548	pmd_t *pmd;
	549	+ u64 unprot;
544	550	int rc;
545	551
546	552	BUG_ON(gmap_is_shadow(gmap));
@@ -584,8 +590,8 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) · 37 unmodified lines
584	590	return -EFAULT;
585	591	pmd = pmd_offset(pud, vmaddr);
586	592	VM_BUG_ON(pmd_none(*pmd));
587		− /* large pmds cannot yet be handled */
588		− if (pmd_large(*pmd))
	593	+ /* Are we allowed to use huge pages? */
	594	+ if (pmd_large(*pmd) && !gmap->mm->context.allow_gmap_hpage_1m)
589	595	return -EFAULT;
590	596	/* Link gmap segment table entry location to page table. */
591	597	rc = radix_tree_preload(GFP_KERNEL);
@@ -596,10 +602,22 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr) · 4 unmodified lines
596	602	if (*table == _SEGMENT_ENTRY_EMPTY) {
597	603	rc = radix_tree_insert(&gmap->host_to_guest,
598	604	vmaddr >> PMD_SHIFT, table);
599		− if (!rc)
600		− table = pmd_val(pmd);
601		− } else
602		− rc = 0;
	605	+ if (!rc) {
	606	+ if (pmd_large(*pmd)) {
	607	+ table = (pmd_val(pmd) &
	608	+ _SEGMENT_ENTRY_HARDWARE_BITS_LARGE)
	609	+ \| _SEGMENT_ENTRY_GMAP_UC;
	610	+ } else
	611	+ table = pmd_val(pmd) &
	612	+ _SEGMENT_ENTRY_HARDWARE_BITS;
	613	+ }
	614	+ } else if (*table & _SEGMENT_ENTRY_PROTECT &&
	615	+ !(pmd_val(*pmd) & _SEGMENT_ENTRY_PROTECT)) {
	616	+ unprot = (u64)*table;
	617	+ unprot &= ~_SEGMENT_ENTRY_PROTECT;
	618	+ unprot \|= _SEGMENT_ENTRY_GMAP_UC;
	619	+ gmap_pmdp_xchg(gmap, (pmd_t *)table, __pmd(unprot), gaddr);
	620	+ }
603	621	spin_unlock(&gmap->guest_table_lock);
604	622	spin_unlock(ptl);
605	623	radix_tree_preload_end();
@@ -690,6 +708,12 @@ void gmap_discard(struct gmap *gmap, unsigned long from, unsigned long to) · 84 unmodified lines
690	708	vmaddr \|= gaddr & ~PMD_MASK;
691	709	/* Find vma in the parent mm */
692	710	vma = find_vma(gmap->mm, vmaddr);
	711	+ /*
	712	+ * We do not discard pages that are backed by
	713	+ * hugetlbfs, so we don't have to refault them.
	714	+ */
	715	+ if (vma && is_vm_hugetlb_page(vma))
	716	+ continue;
693	717	size = min(to - gaddr, PMD_SIZE - (gaddr & ~PMD_MASK));
694	718	zap_page_range(vma, vmaddr, size);
695	719	}
@@ -864,7 +888,128 @@ static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr, · 168 unmodified lines
864	888	*/
865	889	static void gmap_pte_op_end(spinlock_t *ptl)
866	890	{
867		− spin_unlock(ptl);
	891	+ if (ptl)
	892	+ spin_unlock(ptl);
	893	+}
	894	+
	895	+/**
	896	+ * gmap_pmd_op_walk - walk the gmap tables, get the guest table lock
	897	+ * and return the pmd pointer
	898	+ * @gmap: pointer to guest mapping meta data structure
	899	+ * @gaddr: virtual address in the guest address space
	900	+ *
	901	+ * Returns a pointer to the pmd for a guest address, or NULL
	902	+ */
	903	+static inline pmd_t gmap_pmd_op_walk(struct gmap gmap, unsigned long gaddr)
	904	+{
	905	+ pmd_t *pmdp;
	906	+
	907	+ BUG_ON(gmap_is_shadow(gmap));
	908	+ spin_lock(&gmap->guest_table_lock);
	909	+ pmdp = (pmd_t *) gmap_table_walk(gmap, gaddr, 1);
	910	+
	911	+ if (!pmdp \|\| pmd_none(*pmdp)) {
	912	+ spin_unlock(&gmap->guest_table_lock);
	913	+ return NULL;
	914	+ }
	915	+
	916	+ /* 4k page table entries are locked via the pte (pte_alloc_map_lock). */
	917	+ if (!pmd_large(*pmdp))
	918	+ spin_unlock(&gmap->guest_table_lock);
	919	+ return pmdp;
	920	+}
	921	+
	922	+/**
	923	+ * gmap_pmd_op_end - release the guest_table_lock if needed
	924	+ * @gmap: pointer to the guest mapping meta data structure
	925	+ * @pmdp: pointer to the pmd
	926	+ */
	927	+static inline void gmap_pmd_op_end(struct gmap gmap, pmd_t pmdp)
	928	+{
	929	+ if (pmd_large(*pmdp))
	930	+ spin_unlock(&gmap->guest_table_lock);
	931	+}
	932	+
	933	+/*
	934	+ * gmap_protect_pmd - remove access rights to memory and set pmd notification bits
	935	+ * @pmdp: pointer to the pmd to be protected
	936	+ * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
	937	+ * @bits: notification bits to set
	938	+ *
	939	+ * Returns:
	940	+ * 0 if successfully protected
	941	+ * -EAGAIN if a fixup is needed
	942	+ * -EINVAL if unsupported notifier bits have been specified
	943	+ *
	944	+ * Expected to be called with sg->mm->mmap_sem in read and
	945	+ * guest_table_lock held.
	946	+ */
	947	+static int gmap_protect_pmd(struct gmap *gmap, unsigned long gaddr,
	948	+ pmd_t *pmdp, int prot, unsigned long bits)
	949	+{
	950	+ int pmd_i = pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID;
	951	+ int pmd_p = pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT;
	952	+ pmd_t new = *pmdp;
	953	+
	954	+ /* Fixup needed */
	955	+ if ((pmd_i && (prot != PROT_NONE)) \|\| (pmd_p && (prot == PROT_WRITE)))
	956	+ return -EAGAIN;
	957	+
	958	+ if (prot == PROT_NONE && !pmd_i) {
	959	+ pmd_val(new) \|= _SEGMENT_ENTRY_INVALID;
	960	+ gmap_pmdp_xchg(gmap, pmdp, new, gaddr);
	961	+ }
	962	+
	963	+ if (prot == PROT_READ && !pmd_p) {
	964	+ pmd_val(new) &= ~_SEGMENT_ENTRY_INVALID;
	965	+ pmd_val(new) \|= _SEGMENT_ENTRY_PROTECT;
	966	+ gmap_pmdp_xchg(gmap, pmdp, new, gaddr);
	967	+ }
	968	+
	969	+ if (bits & GMAP_NOTIFY_MPROT)
	970	+ pmd_val(*pmdp) \|= _SEGMENT_ENTRY_GMAP_IN;
	971	+
	972	+ /* Shadow GMAP protection needs split PMDs */
	973	+ if (bits & GMAP_NOTIFY_SHADOW)
	974	+ return -EINVAL;
	975	+
	976	+ return 0;
	977	+}
	978	+
	979	+/*
	980	+ * gmap_protect_pte - remove access rights to memory and set pgste bits
	981	+ * @gmap: pointer to guest mapping meta data structure
	982	+ * @gaddr: virtual address in the guest address space
	983	+ * @pmdp: pointer to the pmd associated with the pte
	984	+ * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
	985	+ * @bits: notification bits to set
	986	+ *
	987	+ * Returns 0 if successfully protected, -ENOMEM if out of memory and
	988	+ * -EAGAIN if a fixup is needed.
	989	+ *
	990	+ * Expected to be called with sg->mm->mmap_sem in read
	991	+ */
	992	+static int gmap_protect_pte(struct gmap *gmap, unsigned long gaddr,
	993	+ pmd_t *pmdp, int prot, unsigned long bits)
	994	+{
	995	+ int rc;
	996	+ pte_t *ptep;
	997	+ spinlock_t *ptl = NULL;
	998	+ unsigned long pbits = 0;
	999	+
	1000	+ if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID)
	1001	+ return -EAGAIN;
	1002	+
	1003	+ ptep = pte_alloc_map_lock(gmap->mm, pmdp, gaddr, &ptl);
	1004	+ if (!ptep)
	1005	+ return -ENOMEM;
	1006	+
	1007	+ pbits \|= (bits & GMAP_NOTIFY_MPROT) ? PGSTE_IN_BIT : 0;
	1008	+ pbits \|= (bits & GMAP_NOTIFY_SHADOW) ? PGSTE_VSIE_BIT : 0;
	1009	+ /* Protect and unlock. */
	1010	+ rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, pbits);
	1011	+ gmap_pte_op_end(ptl);
	1012	+ return rc;
868	1013	}
869	1014
870	1015	/*
@@ -883,30 +1028,45 @@ static void gmap_pte_op_end(spinlock_t *ptl) · 12 unmodified lines
883	1028	static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr,
884	1029	unsigned long len, int prot, unsigned long bits)
885	1030	{
886		− unsigned long vmaddr;
887		− spinlock_t *ptl;
888		− pte_t *ptep;
	1031	+ unsigned long vmaddr, dist;
	1032	+ pmd_t *pmdp;
889	1033	int rc;
890	1034
891	1035	BUG_ON(gmap_is_shadow(gmap));
892	1036	while (len) {
893	1037	rc = -EAGAIN;
894		− ptep = gmap_pte_op_walk(gmap, gaddr, &ptl);
895		− if (ptep) {
896		− rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, bits);
897		− gmap_pte_op_end(ptl);
	1038	+ pmdp = gmap_pmd_op_walk(gmap, gaddr);
	1039	+ if (pmdp) {
	1040	+ if (!pmd_large(*pmdp)) {
	1041	+ rc = gmap_protect_pte(gmap, gaddr, pmdp, prot,
	1042	+ bits);
	1043	+ if (!rc) {
	1044	+ len -= PAGE_SIZE;
	1045	+ gaddr += PAGE_SIZE;
	1046	+ }
	1047	+ } else {
	1048	+ rc = gmap_protect_pmd(gmap, gaddr, pmdp, prot,
	1049	+ bits);
	1050	+ if (!rc) {
	1051	+ dist = HPAGE_SIZE - (gaddr & ~HPAGE_MASK);
	1052	+ len = len < dist ? 0 : len - dist;
	1053	+ gaddr = (gaddr & HPAGE_MASK) + HPAGE_SIZE;
	1054	+ }
	1055	+ }
	1056	+ gmap_pmd_op_end(gmap, pmdp);
898	1057	}
899	1058	if (rc) {
	1059	+ if (rc == -EINVAL)
	1060	+ return rc;
	1061	+
	1062	+ /* -EAGAIN, fixup of userspace mm and gmap */
900	1063	vmaddr = __gmap_translate(gmap, gaddr);
901	1064	if (IS_ERR_VALUE(vmaddr))
902	1065	return vmaddr;
903	1066	rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot);
904	1067	if (rc)
905	1068	return rc;
906		− continue;
907	1069	}
908		− gaddr += PAGE_SIZE;
909		− len -= PAGE_SIZE;
910	1070	}
911	1071	return 0;
912	1072	}
@@ -935,7 +1095,7 @@ int gmap_mprotect_notify(struct gmap *gmap, unsigned long gaddr, · 22 unmodified lines
935	1095	if (!MACHINE_HAS_ESOP && prot == PROT_READ)
936	1096	return -EINVAL;
937	1097	down_read(&gmap->mm->mmap_sem);
938		− rc = gmap_protect_range(gmap, gaddr, len, prot, PGSTE_IN_BIT);
	1098	+ rc = gmap_protect_range(gmap, gaddr, len, prot, GMAP_NOTIFY_MPROT);
939	1099	up_read(&gmap->mm->mmap_sem);
940	1100	return rc;
941	1101	}
@@ -1474,6 +1634,7 @@ struct gmap gmap_shadow(struct gmap parent, unsigned long asce, · 532 unmodified lines
1474	1634	unsigned long limit;
1475	1635	int rc;
1476	1636
	1637	+ BUG_ON(parent->mm->context.allow_gmap_hpage_1m);
1477	1638	BUG_ON(gmap_is_shadow(parent));
1478	1639	spin_lock(&parent->shadow_lock);
1479	1640	sg = gmap_find_shadow(parent, asce, edat_level);
@@ -1526,7 +1687,7 @@ struct gmap gmap_shadow(struct gmap parent, unsigned long asce, · 46 unmodified lines
1526	1687	down_read(&parent->mm->mmap_sem);
1527	1688	rc = gmap_protect_range(parent, asce & _ASCE_ORIGIN,
1528	1689	((asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE,
1529		− PROT_READ, PGSTE_VSIE_BIT);
	1690	+ PROT_READ, GMAP_NOTIFY_SHADOW);
1530	1691	up_read(&parent->mm->mmap_sem);
1531	1692	spin_lock(&parent->shadow_lock);
1532	1693	new->initialized = true;
@@ -2092,6 +2253,225 @@ void ptep_notify(struct mm_struct *mm, unsigned long vmaddr, · 559 unmodified lines
2092	2253	}
2093	2254	EXPORT_SYMBOL_GPL(ptep_notify);
2094	2255
	2256	+static void pmdp_notify_gmap(struct gmap gmap, pmd_t pmdp,
	2257	+ unsigned long gaddr)
	2258	+{
	2259	+ pmd_val(*pmdp) &= ~_SEGMENT_ENTRY_GMAP_IN;
	2260	+ gmap_call_notifier(gmap, gaddr, gaddr + HPAGE_SIZE - 1);
	2261	+}
	2262	+
	2263	+/**
	2264	+ * gmap_pmdp_xchg - exchange a gmap pmd with another
	2265	+ * @gmap: pointer to the guest address space structure
	2266	+ * @pmdp: pointer to the pmd entry
	2267	+ * @new: replacement entry
	2268	+ * @gaddr: the affected guest address
	2269	+ *
	2270	+ * This function is assumed to be called with the guest_table_lock
	2271	+ * held.
	2272	+ */
	2273	+static void gmap_pmdp_xchg(struct gmap gmap, pmd_t pmdp, pmd_t new,
	2274	+ unsigned long gaddr)
	2275	+{
	2276	+ gaddr &= HPAGE_MASK;
	2277	+ pmdp_notify_gmap(gmap, pmdp, gaddr);
	2278	+ pmd_val(new) &= ~_SEGMENT_ENTRY_GMAP_IN;
	2279	+ if (MACHINE_HAS_TLB_GUEST)
	2280	+ __pmdp_idte(gaddr, (pmd_t *)pmdp, IDTE_GUEST_ASCE, gmap->asce,
	2281	+ IDTE_GLOBAL);
	2282	+ else if (MACHINE_HAS_IDTE)
	2283	+ __pmdp_idte(gaddr, (pmd_t *)pmdp, 0, 0, IDTE_GLOBAL);
	2284	+ else
	2285	+ __pmdp_csp(pmdp);
	2286	+ *pmdp = new;
	2287	+}
	2288	+
	2289	+static void gmap_pmdp_clear(struct mm_struct *mm, unsigned long vmaddr,
	2290	+ int purge)
	2291	+{
	2292	+ pmd_t *pmdp;
	2293	+ struct gmap *gmap;
	2294	+ unsigned long gaddr;
	2295	+
	2296	+ rcu_read_lock();
	2297	+ list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
	2298	+ spin_lock(&gmap->guest_table_lock);
	2299	+ pmdp = (pmd_t *)radix_tree_delete(&gmap->host_to_guest,
	2300	+ vmaddr >> PMD_SHIFT);
	2301	+ if (pmdp) {
	2302	+ gaddr = __gmap_segment_gaddr((unsigned long *)pmdp);
	2303	+ pmdp_notify_gmap(gmap, pmdp, gaddr);
	2304	+ WARN_ON(pmd_val(*pmdp) & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE \|
	2305	+ _SEGMENT_ENTRY_GMAP_UC));
	2306	+ if (purge)
	2307	+ __pmdp_csp(pmdp);
	2308	+ pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
	2309	+ }
	2310	+ spin_unlock(&gmap->guest_table_lock);
	2311	+ }
	2312	+ rcu_read_unlock();
	2313	+}
	2314	+
	2315	+/**
	2316	+ * gmap_pmdp_invalidate - invalidate all affected guest pmd entries without
	2317	+ * flushing
	2318	+ * @mm: pointer to the process mm_struct
	2319	+ * @vmaddr: virtual address in the process address space
	2320	+ */
	2321	+void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr)
	2322	+{
	2323	+ gmap_pmdp_clear(mm, vmaddr, 0);
	2324	+}
	2325	+EXPORT_SYMBOL_GPL(gmap_pmdp_invalidate);
	2326	+
	2327	+/**
	2328	+ * gmap_pmdp_csp - csp all affected guest pmd entries
	2329	+ * @mm: pointer to the process mm_struct
	2330	+ * @vmaddr: virtual address in the process address space
	2331	+ */
	2332	+void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr)
	2333	+{
	2334	+ gmap_pmdp_clear(mm, vmaddr, 1);
	2335	+}
	2336	+EXPORT_SYMBOL_GPL(gmap_pmdp_csp);
	2337	+
	2338	+/**
	2339	+ * gmap_pmdp_idte_local - invalidate and clear a guest pmd entry
	2340	+ * @mm: pointer to the process mm_struct
	2341	+ * @vmaddr: virtual address in the process address space
	2342	+ */
	2343	+void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr)
	2344	+{
	2345	+ unsigned long *entry, gaddr;
	2346	+ struct gmap *gmap;
	2347	+ pmd_t *pmdp;
	2348	+
	2349	+ rcu_read_lock();
	2350	+ list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
	2351	+ spin_lock(&gmap->guest_table_lock);
	2352	+ entry = radix_tree_delete(&gmap->host_to_guest,
	2353	+ vmaddr >> PMD_SHIFT);
	2354	+ if (entry) {
	2355	+ pmdp = (pmd_t *)entry;
	2356	+ gaddr = __gmap_segment_gaddr(entry);
	2357	+ pmdp_notify_gmap(gmap, pmdp, gaddr);
	2358	+ WARN_ON(*entry & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE \|
	2359	+ _SEGMENT_ENTRY_GMAP_UC));
	2360	+ if (MACHINE_HAS_TLB_GUEST)
	2361	+ __pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE,
	2362	+ gmap->asce, IDTE_LOCAL);
	2363	+ else if (MACHINE_HAS_IDTE)
	2364	+ __pmdp_idte(gaddr, pmdp, 0, 0, IDTE_LOCAL);
	2365	+ *entry = _SEGMENT_ENTRY_EMPTY;
	2366	+ }
	2367	+ spin_unlock(&gmap->guest_table_lock);
	2368	+ }
	2369	+ rcu_read_unlock();
	2370	+}
	2371	+EXPORT_SYMBOL_GPL(gmap_pmdp_idte_local);
	2372	+
	2373	+/**
	2374	+ * gmap_pmdp_idte_global - invalidate and clear a guest pmd entry
	2375	+ * @mm: pointer to the process mm_struct
	2376	+ * @vmaddr: virtual address in the process address space
	2377	+ */
	2378	+void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr)
	2379	+{
	2380	+ unsigned long *entry, gaddr;
	2381	+ struct gmap *gmap;
	2382	+ pmd_t *pmdp;
	2383	+
	2384	+ rcu_read_lock();
	2385	+ list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
	2386	+ spin_lock(&gmap->guest_table_lock);
	2387	+ entry = radix_tree_delete(&gmap->host_to_guest,
	2388	+ vmaddr >> PMD_SHIFT);
	2389	+ if (entry) {
	2390	+ pmdp = (pmd_t *)entry;
	2391	+ gaddr = __gmap_segment_gaddr(entry);
	2392	+ pmdp_notify_gmap(gmap, pmdp, gaddr);
	2393	+ WARN_ON(*entry & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE \|
	2394	+ _SEGMENT_ENTRY_GMAP_UC));
	2395	+ if (MACHINE_HAS_TLB_GUEST)
	2396	+ __pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE,
	2397	+ gmap->asce, IDTE_GLOBAL);
	2398	+ else if (MACHINE_HAS_IDTE)
	2399	+ __pmdp_idte(gaddr, pmdp, 0, 0, IDTE_GLOBAL);
	2400	+ else
	2401	+ __pmdp_csp(pmdp);
	2402	+ *entry = _SEGMENT_ENTRY_EMPTY;
	2403	+ }
	2404	+ spin_unlock(&gmap->guest_table_lock);
	2405	+ }
	2406	+ rcu_read_unlock();
	2407	+}
	2408	+EXPORT_SYMBOL_GPL(gmap_pmdp_idte_global);
	2409	+
	2410	+/**
	2411	+ * gmap_test_and_clear_dirty_pmd - test and reset segment dirty status
	2412	+ * @gmap: pointer to guest address space
	2413	+ * @pmdp: pointer to the pmd to be tested
	2414	+ * @gaddr: virtual address in the guest address space
	2415	+ *
	2416	+ * This function is assumed to be called with the guest_table_lock
	2417	+ * held.
	2418	+ */
	2419	+bool gmap_test_and_clear_dirty_pmd(struct gmap gmap, pmd_t pmdp,
	2420	+ unsigned long gaddr)
	2421	+{
	2422	+ if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID)
	2423	+ return false;
	2424	+
	2425	+ /* Already protected memory, which did not change is clean */
	2426	+ if (pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT &&
	2427	+ !(pmd_val(*pmdp) & _SEGMENT_ENTRY_GMAP_UC))
	2428	+ return false;
	2429	+
	2430	+ /* Clear UC indication and reset protection */
	2431	+ pmd_val(*pmdp) &= ~_SEGMENT_ENTRY_GMAP_UC;
	2432	+ gmap_protect_pmd(gmap, gaddr, pmdp, PROT_READ, 0);
	2433	+ return true;
	2434	+}
	2435	+
	2436	+/**
	2437	+ * gmap_sync_dirty_log_pmd - set bitmap based on dirty status of segment
	2438	+ * @gmap: pointer to guest address space
	2439	+ * @bitmap: dirty bitmap for this pmd
	2440	+ * @gaddr: virtual address in the guest address space
	2441	+ * @vmaddr: virtual address in the host address space
	2442	+ *
	2443	+ * This function is assumed to be called with the guest_table_lock
	2444	+ * held.
	2445	+ */
	2446	+void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long bitmap[4],
	2447	+ unsigned long gaddr, unsigned long vmaddr)
	2448	+{
	2449	+ int i;
	2450	+ pmd_t *pmdp;
	2451	+ pte_t *ptep;
	2452	+ spinlock_t *ptl;
	2453	+
	2454	+ pmdp = gmap_pmd_op_walk(gmap, gaddr);
	2455	+ if (!pmdp)
	2456	+ return;
	2457	+
	2458	+ if (pmd_large(*pmdp)) {
	2459	+ if (gmap_test_and_clear_dirty_pmd(gmap, pmdp, gaddr))
	2460	+ bitmap_fill(bitmap, _PAGE_ENTRIES);
	2461	+ } else {
	2462	+ for (i = 0; i < _PAGE_ENTRIES; i++, vmaddr += PAGE_SIZE) {
	2463	+ ptep = pte_alloc_map_lock(gmap->mm, pmdp, vmaddr, &ptl);
	2464	+ if (!ptep)
	2465	+ continue;
	2466	+ if (ptep_test_and_clear_uc(gmap->mm, vmaddr, ptep))
	2467	+ set_bit(i, bitmap);
	2468	+ spin_unlock(ptl);
	2469	+ }
	2470	+ }
	2471	+ gmap_pmd_op_end(gmap, pmdp);
	2472	+}
	2473	+EXPORT_SYMBOL_GPL(gmap_sync_dirty_log_pmd);
	2474	+
2095	2475	static inline void thp_split_mm(struct mm_struct *mm)
2096	2476	{
2097	2477	#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -2168,17 +2548,45 @@ EXPORT_SYMBOL_GPL(s390_enable_sie); · 70 unmodified lines
2168	2548	* Enable storage key handling from now on and initialize the storage
2169	2549	* keys with the default key.
2170	2550	*/
2171		−static int __s390_enable_skey(pte_t *pte, unsigned long addr,
2172		− unsigned long next, struct mm_walk *walk)
	2551	+static int __s390_enable_skey_pte(pte_t *pte, unsigned long addr,
	2552	+ unsigned long next, struct mm_walk *walk)
2173	2553	{
2174	2554	/* Clear storage key */
2175	2555	ptep_zap_key(walk->mm, addr, pte);
2176	2556	return 0;
2177	2557	}
2178	2558
	2559	+static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr,
	2560	+ unsigned long hmask, unsigned long next,
	2561	+ struct mm_walk *walk)
	2562	+{
	2563	+ pmd_t pmd = (pmd_t )pte;
	2564	+ unsigned long start, end;
	2565	+ struct page page = pmd_page(pmd);
	2566	+
	2567	+ /*
	2568	+ * The write check makes sure we do not set a key on shared
	2569	+ * memory. This is needed as the walker does not differentiate
	2570	+ * between actual guest memory and the process executable or
	2571	+ * shared libraries.
	2572	+ */
	2573	+ if (pmd_val(*pmd) & _SEGMENT_ENTRY_INVALID \|\|
	2574	+ !(pmd_val(*pmd) & _SEGMENT_ENTRY_WRITE))
	2575	+ return 0;
	2576	+
	2577	+ start = pmd_val(*pmd) & HPAGE_MASK;
	2578	+ end = start + HPAGE_SIZE - 1;
	2579	+ __storage_key_init_range(start, end);
	2580	+ set_bit(PG_arch_1, &page->flags);
	2581	+ return 0;
	2582	+}
	2583	+
2179	2584	int s390_enable_skey(void)
2180	2585	{
2181		− struct mm_walk walk = { .pte_entry = __s390_enable_skey };
	2586	+ struct mm_walk walk = {
	2587	+ .hugetlb_entry = __s390_enable_skey_hugetlb,
	2588	+ .pte_entry = __s390_enable_skey_pte,
	2589	+ };
2182	2590	struct mm_struct *mm = current->mm;
2183	2591	struct vm_area_struct *vma;
2184	2592	int rc = 0;

arch/s390/mm/hugetlbpage.c+24−0

@@ -123,6 +123,29 @@ static inline pte_t __rste_to_pte(unsigned long rste) · 122 unmodified lines
123	123	return pte;
124	124	}
125	125
	126	+static void clear_huge_pte_skeys(struct mm_struct *mm, unsigned long rste)
	127	+{
	128	+ struct page *page;
	129	+ unsigned long size, paddr;
	130	+
	131	+ if (!mm_uses_skeys(mm) \|\|
	132	+ rste & _SEGMENT_ENTRY_INVALID)
	133	+ return;
	134	+
	135	+ if ((rste & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) {
	136	+ page = pud_page(__pud(rste));
	137	+ size = PUD_SIZE;
	138	+ paddr = rste & PUD_MASK;
	139	+ } else {
	140	+ page = pmd_page(__pmd(rste));
	141	+ size = PMD_SIZE;
	142	+ paddr = rste & PMD_MASK;
	143	+ }
	144	+
	145	+ if (!test_and_set_bit(PG_arch_1, &page->flags))
	146	+ __storage_key_init_range(paddr, paddr + size - 1);
	147	+}
	148	+
126	149	void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
127	150	pte_t *ptep, pte_t pte)
128	151	{
@@ -137,6 +160,7 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, · 8 unmodified lines
137	160	rste \|= _REGION_ENTRY_TYPE_R3 \| _REGION3_ENTRY_LARGE;
138	161	else
139	162	rste \|= _SEGMENT_ENTRY_LARGE;
	163	+ clear_huge_pte_skeys(mm, rste);
140	164	pte_val(*ptep) = rste;
141	165	}
142	166

arch/s390/mm/pageattr.c+2−4

@@ -14,7 +14,7 @@ · 13 unmodified lines
14	14
15	15	static inline unsigned long sske_frame(unsigned long addr, unsigned char skey)
16	16	{
17		− asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],9,0"
	17	+ asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],1,0"
18	18	: [addr] "+a" (addr) : [skey] "d" (skey));
19	19	return addr;
20	20	}
@@ -23,8 +23,6 @@ void __storage_key_init_range(unsigned long start, unsigned long end) · 2 unmodified lines
23	23	{
24	24	unsigned long boundary, size;
25	25
26		− if (!PAGE_DEFAULT_KEY)
27		− return;
28	26	while (start < end) {
29	27	if (MACHINE_HAS_EDAT1) {
30	28	/* set storage keys for a 1MB frame */
@@ -37,7 +35,7 @@ void __storage_key_init_range(unsigned long start, unsigned long end) · 6 unmodified lines
37	35	continue;
38	36	}
39	37	}
40		− page_set_storage_key(start, PAGE_DEFAULT_KEY, 0);
	38	+ page_set_storage_key(start, PAGE_DEFAULT_KEY, 1);
41	39	start += PAGE_SIZE;
42	40	}
43	41	}

arch/s390/mm/pgtable.c+115−44

@@ -347,18 +347,27 @@ static inline void pmdp_idte_local(struct mm_struct *mm, · 346 unmodified lines
347	347	mm->context.asce, IDTE_LOCAL);
348	348	else
349	349	__pmdp_idte(addr, pmdp, 0, 0, IDTE_LOCAL);
	350	+ if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
	351	+ gmap_pmdp_idte_local(mm, addr);
350	352	}
351	353
352	354	static inline void pmdp_idte_global(struct mm_struct *mm,
353	355	unsigned long addr, pmd_t *pmdp)
354	356	{
355		− if (MACHINE_HAS_TLB_GUEST)
	357	+ if (MACHINE_HAS_TLB_GUEST) {
356	358	__pmdp_idte(addr, pmdp, IDTE_NODAT \| IDTE_GUEST_ASCE,
357	359	mm->context.asce, IDTE_GLOBAL);
358		− else if (MACHINE_HAS_IDTE)
	360	+ if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
	361	+ gmap_pmdp_idte_global(mm, addr);
	362	+ } else if (MACHINE_HAS_IDTE) {
359	363	__pmdp_idte(addr, pmdp, 0, 0, IDTE_GLOBAL);
360		− else
	364	+ if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
	365	+ gmap_pmdp_idte_global(mm, addr);
	366	+ } else {
361	367	__pmdp_csp(pmdp);
	368	+ if (mm_has_pgste(mm) && mm->context.allow_gmap_hpage_1m)
	369	+ gmap_pmdp_csp(mm, addr);
	370	+ }
362	371	}
363	372
364	373	static inline pmd_t pmdp_flush_direct(struct mm_struct *mm,
@@ -392,6 +401,8 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, · 27 unmodified lines
392	401	cpumask_of(smp_processor_id()))) {
393	402	pmd_val(*pmdp) \|= _SEGMENT_ENTRY_INVALID;
394	403	mm->context.flush_mm = 1;
	404	+ if (mm_has_pgste(mm))
	405	+ gmap_pmdp_invalidate(mm, addr);
395	406	} else {
396	407	pmdp_idte_global(mm, addr, pmdp);
397	408	}
@@ -399,6 +410,24 @@ static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, · 1 unmodified line
399	410	return old;
400	411	}
401	412
	413	+static pmd_t pmd_alloc_map(struct mm_struct mm, unsigned long addr)
	414	+{
	415	+ pgd_t *pgd;
	416	+ p4d_t *p4d;
	417	+ pud_t *pud;
	418	+ pmd_t *pmd;
	419	+
	420	+ pgd = pgd_offset(mm, addr);
	421	+ p4d = p4d_alloc(mm, pgd, addr);
	422	+ if (!p4d)
	423	+ return NULL;
	424	+ pud = pud_alloc(mm, p4d, addr);
	425	+ if (!pud)
	426	+ return NULL;
	427	+ pmd = pmd_alloc(mm, pud, addr);
	428	+ return pmd;
	429	+}
	430	+
402	431	pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr,
403	432	pmd_t *pmdp, pmd_t new)
404	433	{
@@ -693,40 +722,14 @@ void ptep_zap_key(struct mm_struct mm, unsigned long addr, pte_t ptep) · 288 unmodified lines
693	722	/*
694	723	* Test and reset if a guest page is dirty
695	724	*/
696		−bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr)
	725	+bool ptep_test_and_clear_uc(struct mm_struct *mm, unsigned long addr,
	726	+ pte_t *ptep)
697	727	{
698		− spinlock_t *ptl;
699		− pgd_t *pgd;
700		− p4d_t *p4d;
701		− pud_t *pud;
702		− pmd_t *pmd;
703	728	pgste_t pgste;
704		− pte_t *ptep;
705	729	pte_t pte;
706	730	bool dirty;
707	731	int nodat;
708	732
709		− pgd = pgd_offset(mm, addr);
710		− p4d = p4d_alloc(mm, pgd, addr);
711		− if (!p4d)
712		− return false;
713		− pud = pud_alloc(mm, p4d, addr);
714		− if (!pud)
715		− return false;
716		− pmd = pmd_alloc(mm, pud, addr);
717		− if (!pmd)
718		− return false;
719		− /* We can't run guests backed by huge pages, but userspace can
720		− * still set them up and then try to migrate them without any
721		− * migration support.
722		− */
723		− if (pmd_large(*pmd))
724		− return true;
725		−
726		− ptep = pte_alloc_map_lock(mm, pmd, addr, &ptl);
727		− if (unlikely(!ptep))
728		− return false;
729		−
730	733	pgste = pgste_get_lock(ptep);
731	734	dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT);
732	735	pgste_val(pgste) &= ~PGSTE_UC_BIT;
@@ -742,21 +745,43 @@ bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr) · 9 unmodified lines
742	745	*ptep = pte;
743	746	}
744	747	pgste_set_unlock(ptep, pgste);
745		−
746		− spin_unlock(ptl);
747	748	return dirty;
748	749	}
749		−EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty);
	750	+EXPORT_SYMBOL_GPL(ptep_test_and_clear_uc);
750	751
751	752	int set_guest_storage_key(struct mm_struct *mm, unsigned long addr,
752	753	unsigned char key, bool nq)
753	754	{
754		− unsigned long keyul;
	755	+ unsigned long keyul, paddr;
755	756	spinlock_t *ptl;
756	757	pgste_t old, new;
	758	+ pmd_t *pmdp;
757	759	pte_t *ptep;
758	760
759		− ptep = get_locked_pte(mm, addr, &ptl);
	761	+ pmdp = pmd_alloc_map(mm, addr);
	762	+ if (unlikely(!pmdp))
	763	+ return -EFAULT;
	764	+
	765	+ ptl = pmd_lock(mm, pmdp);
	766	+ if (!pmd_present(*pmdp)) {
	767	+ spin_unlock(ptl);
	768	+ return -EFAULT;
	769	+ }
	770	+
	771	+ if (pmd_large(*pmdp)) {
	772	+ paddr = pmd_val(*pmdp) & HPAGE_MASK;
	773	+ paddr \|= addr & ~HPAGE_MASK;
	774	+ /*
	775	+ * Huge pmds need quiescing operations, they are
	776	+ * always mapped.
	777	+ */
	778	+ page_set_storage_key(paddr, key, 1);
	779	+ spin_unlock(ptl);
	780	+ return 0;
	781	+ }
	782	+ spin_unlock(ptl);
	783	+
	784	+ ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);
760	785	if (unlikely(!ptep))
761	786	return -EFAULT;
762	787
@@ -767,14 +792,14 @@ int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, · 4 unmodified lines
767	792	pgste_val(new) \|= (keyul & (_PAGE_CHANGED \| _PAGE_REFERENCED)) << 48;
768	793	pgste_val(new) \|= (keyul & (_PAGE_ACC_BITS \| _PAGE_FP_BIT)) << 56;
769	794	if (!(pte_val(*ptep) & _PAGE_INVALID)) {
770		− unsigned long address, bits, skey;
	795	+ unsigned long bits, skey;
771	796
772		− address = pte_val(*ptep) & PAGE_MASK;
773		− skey = (unsigned long) page_get_storage_key(address);
	797	+ paddr = pte_val(*ptep) & PAGE_MASK;
	798	+ skey = (unsigned long) page_get_storage_key(paddr);
774	799	bits = skey & (_PAGE_CHANGED \| _PAGE_REFERENCED);
775	800	skey = key & (_PAGE_ACC_BITS \| _PAGE_FP_BIT);
776	801	/* Set storage key ACC and FP */
777		− page_set_storage_key(address, skey, !nq);
	802	+ page_set_storage_key(paddr, skey, !nq);
778	803	/* Merge host changed & referenced into pgste */
779	804	pgste_val(new) \|= bits << 52;
780	805	}
@@ -830,11 +855,32 @@ EXPORT_SYMBOL(cond_set_guest_storage_key); · 49 unmodified lines
830	855	int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr)
831	856	{
832	857	spinlock_t *ptl;
	858	+ unsigned long paddr;
833	859	pgste_t old, new;
	860	+ pmd_t *pmdp;
834	861	pte_t *ptep;
835	862	int cc = 0;
836	863
837		− ptep = get_locked_pte(mm, addr, &ptl);
	864	+ pmdp = pmd_alloc_map(mm, addr);
	865	+ if (unlikely(!pmdp))
	866	+ return -EFAULT;
	867	+
	868	+ ptl = pmd_lock(mm, pmdp);
	869	+ if (!pmd_present(*pmdp)) {
	870	+ spin_unlock(ptl);
	871	+ return -EFAULT;
	872	+ }
	873	+
	874	+ if (pmd_large(*pmdp)) {
	875	+ paddr = pmd_val(*pmdp) & HPAGE_MASK;
	876	+ paddr \|= addr & ~HPAGE_MASK;
	877	+ cc = page_reset_referenced(paddr);
	878	+ spin_unlock(ptl);
	879	+ return cc;
	880	+ }
	881	+ spin_unlock(ptl);
	882	+
	883	+ ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);
838	884	if (unlikely(!ptep))
839	885	return -EFAULT;
840	886
@@ -843,7 +889,8 @@ int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) · 2 unmodified lines
843	889	pgste_val(new) &= ~PGSTE_GR_BIT;
844	890
845	891	if (!(pte_val(*ptep) & _PAGE_INVALID)) {
846		− cc = page_reset_referenced(pte_val(*ptep) & PAGE_MASK);
	892	+ paddr = pte_val(*ptep) & PAGE_MASK;
	893	+ cc = page_reset_referenced(paddr);
847	894	/* Merge real referenced bit into host-set */
848	895	pgste_val(new) \|= ((unsigned long) cc << 53) & PGSTE_HR_BIT;
849	896	}
@@ -862,18 +909,42 @@ EXPORT_SYMBOL(reset_guest_reference_bit); · 12 unmodified lines
862	909	int get_guest_storage_key(struct mm_struct *mm, unsigned long addr,
863	910	unsigned char *key)
864	911	{
	912	+ unsigned long paddr;
865	913	spinlock_t *ptl;
866	914	pgste_t pgste;
	915	+ pmd_t *pmdp;
867	916	pte_t *ptep;
868	917
869		− ptep = get_locked_pte(mm, addr, &ptl);
	918	+ pmdp = pmd_alloc_map(mm, addr);
	919	+ if (unlikely(!pmdp))
	920	+ return -EFAULT;
	921	+
	922	+ ptl = pmd_lock(mm, pmdp);
	923	+ if (!pmd_present(*pmdp)) {
	924	+ /* Not yet mapped memory has a zero key */
	925	+ spin_unlock(ptl);
	926	+ *key = 0;
	927	+ return 0;
	928	+ }
	929	+
	930	+ if (pmd_large(*pmdp)) {
	931	+ paddr = pmd_val(*pmdp) & HPAGE_MASK;
	932	+ paddr \|= addr & ~HPAGE_MASK;
	933	+ *key = page_get_storage_key(paddr);
	934	+ spin_unlock(ptl);
	935	+ return 0;
	936	+ }
	937	+ spin_unlock(ptl);
	938	+
	939	+ ptep = pte_alloc_map_lock(mm, pmdp, addr, &ptl);
870	940	if (unlikely(!ptep))
871	941	return -EFAULT;
872	942
873	943	pgste = pgste_get_lock(ptep);
874	944	*key = (pgste_val(pgste) & (PGSTE_ACC_BITS \| PGSTE_FP_BIT)) >> 56;
	945	+ paddr = pte_val(*ptep) & PAGE_MASK;
875	946	if (!(pte_val(*ptep) & _PAGE_INVALID))
876		− key = page_get_storage_key(pte_val(ptep) & PAGE_MASK);
	947	+ *key = page_get_storage_key(paddr);
877	948	/* Reflect guest's logical view, not physical */
878	949	*key \|= (pgste_val(pgste) & (PGSTE_GR_BIT \| PGSTE_GC_BIT)) >> 48;
879	950	pgste_set_unlock(ptep, pgste);

arch/s390/tools/gen_facilities.c+2−1

@@ -4,7 +4,7 @@ · 3 unmodified lines
4	4	* numbering scheme from the Princples of Operations: most significant bit
5	5	* has bit number 0.
6	6	*
7		− * Copyright IBM Corp. 2015
	7	+ * Copyright IBM Corp. 2015, 2018
8	8	*
9	9	*/
10	10
@@ -106,6 +106,7 @@ static struct facility_def facility_defs[] = { · 95 unmodified lines
106	106
107	107	.name = "FACILITIES_KVM_CPUMODEL",
108	108	.bits = (int[]){
	109	+ 156, /* etoken facility */
109	110	-1 /* END */
110	111	}
111	112	},

include/linux/kvm_host.h+7−0

@@ -309,6 +309,13 @@ static inline unsigned long kvm_dirty_bitmap_bytes(struct kvm_memory_slot *memsl · 308 unmodified lines
309	309	return ALIGN(memslot->npages, BITS_PER_LONG) / 8;
310	310	}
311	311
	312	+static inline unsigned long kvm_second_dirty_bitmap(struct kvm_memory_slot memslot)
	313	+{
	314	+ unsigned long len = kvm_dirty_bitmap_bytes(memslot);
	315	+
	316	+ return memslot->dirty_bitmap + len / sizeof(*memslot->dirty_bitmap);
	317	+}
	318	+
312	319	struct kvm_s390_adapter_int {
313	320	u64 ind_addr;
314	321	u64 summary_addr;

include/uapi/linux/kvm.h+1−0

@@ -949,6 +949,7 @@ struct kvm_ppc_resize_hpt { · 948 unmodified lines
949	949	#define KVM_CAP_GET_MSR_FEATURES 153
950	950	#define KVM_CAP_HYPERV_EVENTFD 154
951	951	#define KVM_CAP_HYPERV_TLBFLUSH 155
	952	+#define KVM_CAP_S390_HPAGE_1M 156
952	953
953	954	#ifdef KVM_CAP_IRQ_ROUTING
954	955

virt/kvm/kvm_main.c+1−1

@@ -1169,7 +1169,7 @@ int kvm_get_dirty_log_protect(struct kvm *kvm, · 1168 unmodified lines
1169	1169
1170	1170	n = kvm_dirty_bitmap_bytes(memslot);
1171	1171
1172		− dirty_bitmap_buffer = dirty_bitmap + n / sizeof(long);
	1172	+ dirty_bitmap_buffer = kvm_second_dirty_bitmap(memslot);
1173	1173	memset(dirty_bitmap_buffer, 0, n);
1174	1174
1175	1175	spin_lock(&kvm->mmu_lock);