Merge branch kvm-arm64/vgic-invlpir into kvmarm-master/next

* kvm-arm64/vgic-invlpir: : . : Implement MMIO-based LPI invalidation for vGICv3. : . KVM: arm64: vgic-v3: Advertise GICR_CTLR.{IR, CES} as a new GICD_IIDR revision KVM: arm64: vgic-v3: Implement MMIO-based LPI invalidation KVM: arm64: vgic-v3: Expose GICR_CTLR.RWP when disabling LPIs irqchip/gic-v3: Exposes bit values for GICR_CTLR.{IR, CES}

Marc Zyngier committed 4 years ago

Commit ec2cff62 parents 3b8e21e 49a1a2c7 files · +195 −39Browse files

Unified · Split

7 files

arch/arm64/kvm/vgic/vgic-init.c +6 −1
arch/arm64/kvm/vgic/vgic-its.c +42 −22
arch/arm64/kvm/vgic/vgic-mmio-v2.c +15 −3
arch/arm64/kvm/vgic/vgic-mmio-v3.c +114 −11
arch/arm64/kvm/vgic/vgic.h +10 −0
include/kvm/arm_vgic.h +6 −2
include/linux/irqchip/arm-gic-v3.h +2 −0

arch/arm64/kvm/vgic/vgic-init.c+6−1

@@ -319,7 +319,12 @@ int vgic_init(struct kvm *kvm) · 318 unmodified lines
319	319
320	320	vgic_debug_init(kvm);
321	321
322		− dist->implementation_rev = 2;
	322	+ /*
	323	+ * If userspace didn't set the GIC implementation revision,
	324	+ * default to the latest and greatest. You know want it.
	325	+ */
	326	+ if (!dist->implementation_rev)
	327	+ dist->implementation_rev = KVM_VGIC_IMP_REV_LATEST;
323	328	dist->initialized = true;
324	329
325	330	out:

arch/arm64/kvm/vgic/vgic-its.c+42−22

@@ -683,7 +683,7 @@ int vgic_its_resolve_lpi(struct kvm kvm, struct vgic_its its, · 682 unmodified lines
683	683	if (!vcpu)
684	684	return E_ITS_INT_UNMAPPED_INTERRUPT;
685	685
686		− if (!vcpu->arch.vgic_cpu.lpis_enabled)
	686	+ if (!vgic_lpis_enabled(vcpu))
687	687	return -EBUSY;
688	688
689	689	vgic_its_cache_translation(kvm, its, devid, eventid, ite->irq);
@@ -1272,6 +1272,11 @@ static int vgic_its_cmd_handle_clear(struct kvm kvm, struct vgic_its its, · 582 unmodified lines
1272	1272	return 0;
1273	1273	}
1274	1274
	1275	+int vgic_its_inv_lpi(struct kvm kvm, struct vgic_irq irq)
	1276	+{
	1277	+ return update_lpi_config(kvm, irq, NULL, true);
	1278	+}
	1279	+
1275	1280	/*
1276	1281	* The INV command syncs the configuration bits from the memory table.
1277	1282	* Must be called with the its_lock mutex held.
@@ -1288,7 +1293,41 @@ static int vgic_its_cmd_handle_inv(struct kvm kvm, struct vgic_its its, · 10 unmodified lines
1288	1293	if (!ite)
1289	1294	return E_ITS_INV_UNMAPPED_INTERRUPT;
1290	1295
1291		− return update_lpi_config(kvm, ite->irq, NULL, true);
	1296	+ return vgic_its_inv_lpi(kvm, ite->irq);
	1297	+}
	1298	+
	1299	+/**
	1300	+ * vgic_its_invall - invalidate all LPIs targetting a given vcpu
	1301	+ * @vcpu: the vcpu for which the RD is targetted by an invalidation
	1302	+ *
	1303	+ * Contrary to the INVALL command, this targets a RD instead of a
	1304	+ * collection, and we don't need to hold the its_lock, since no ITS is
	1305	+ * involved here.
	1306	+ */
	1307	+int vgic_its_invall(struct kvm_vcpu *vcpu)
	1308	+{
	1309	+ struct kvm *kvm = vcpu->kvm;
	1310	+ int irq_count, i = 0;
	1311	+ u32 *intids;
	1312	+
	1313	+ irq_count = vgic_copy_lpi_list(kvm, vcpu, &intids);
	1314	+ if (irq_count < 0)
	1315	+ return irq_count;
	1316	+
	1317	+ for (i = 0; i < irq_count; i++) {
	1318	+ struct vgic_irq *irq = vgic_get_irq(kvm, NULL, intids[i]);
	1319	+ if (!irq)
	1320	+ continue;
	1321	+ update_lpi_config(kvm, irq, vcpu, false);
	1322	+ vgic_put_irq(kvm, irq);
	1323	+ }
	1324	+
	1325	+ kfree(intids);
	1326	+
	1327	+ if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)
	1328	+ its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);
	1329	+
	1330	+ return 0;
1292	1331	}
1293	1332
1294	1333	/*
@@ -1305,32 +1344,13 @@ static int vgic_its_cmd_handle_invall(struct kvm kvm, struct vgic_its its, · 10 unmodified lines
1305	1344	u32 coll_id = its_cmd_get_collection(its_cmd);
1306	1345	struct its_collection *collection;
1307	1346	struct kvm_vcpu *vcpu;
1308		− struct vgic_irq *irq;
1309		− u32 *intids;
1310		− int irq_count, i;
1311	1347
1312	1348	collection = find_collection(its, coll_id);
1313	1349	if (!its_is_collection_mapped(collection))
1314	1350	return E_ITS_INVALL_UNMAPPED_COLLECTION;
1315	1351
1316	1352	vcpu = kvm_get_vcpu(kvm, collection->target_addr);
1317		−
1318		− irq_count = vgic_copy_lpi_list(kvm, vcpu, &intids);
1319		− if (irq_count < 0)
1320		− return irq_count;
1321		−
1322		− for (i = 0; i < irq_count; i++) {
1323		− irq = vgic_get_irq(kvm, NULL, intids[i]);
1324		− if (!irq)
1325		− continue;
1326		− update_lpi_config(kvm, irq, vcpu, false);
1327		− vgic_put_irq(kvm, irq);
1328		− }
1329		−
1330		− kfree(intids);
1331		−
1332		− if (vcpu->arch.vgic_cpu.vgic_v3.its_vpe.its_vm)
1333		− its_invall_vpe(&vcpu->arch.vgic_cpu.vgic_v3.its_vpe);
	1353	+ vgic_its_invall(vcpu);
1334	1354
1335	1355	return 0;
1336	1356	}

arch/arm64/kvm/vgic/vgic-mmio-v2.c+15−3

@@ -73,9 +73,13 @@ static int vgic_mmio_uaccess_write_v2_misc(struct kvm_vcpu *vcpu, · 72 unmodified lines
73	73	gpa_t addr, unsigned int len,
74	74	unsigned long val)
75	75	{
	76	+ struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
	77	+ u32 reg;
	78	+
76	79	switch (addr & 0x0c) {
77	80	case GIC_DIST_IIDR:
78		− if (val != vgic_mmio_read_v2_misc(vcpu, addr, len))
	81	+ reg = vgic_mmio_read_v2_misc(vcpu, addr, len);
	82	+ if ((reg ^ val) & ~GICD_IIDR_REVISION_MASK)
79	83	return -EINVAL;
80	84
81	85	/*
@@ -87,8 +91,16 @@ static int vgic_mmio_uaccess_write_v2_misc(struct kvm_vcpu *vcpu, · 5 unmodified lines
87	91	* migration from old kernels to new kernels with legacy
88	92	* userspace.
89	93	*/
90		− vcpu->kvm->arch.vgic.v2_groups_user_writable = true;
91		− return 0;
	94	+ reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg);
	95	+ switch (reg) {
	96	+ case KVM_VGIC_IMP_REV_2:
	97	+ case KVM_VGIC_IMP_REV_3:
	98	+ vcpu->kvm->arch.vgic.v2_groups_user_writable = true;
	99	+ dist->implementation_rev = reg;
	100	+ return 0;
	101	+ default:
	102	+ return -EINVAL;
	103	+ }
92	104	}
93	105
94	106	vgic_mmio_write_v2_misc(vcpu, addr, len, val);

arch/arm64/kvm/vgic/vgic-mmio-v3.c+114−11

@@ -155,13 +155,27 @@ static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu, · 154 unmodified lines
155	155	unsigned long val)
156	156	{
157	157	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
	158	+ u32 reg;
158	159
159	160	switch (addr & 0x0c) {
160	161	case GICD_TYPER2:
161		− case GICD_IIDR:
162	162	if (val != vgic_mmio_read_v3_misc(vcpu, addr, len))
163	163	return -EINVAL;
164	164	return 0;
	165	+ case GICD_IIDR:
	166	+ reg = vgic_mmio_read_v3_misc(vcpu, addr, len);
	167	+ if ((reg ^ val) & ~GICD_IIDR_REVISION_MASK)
	168	+ return -EINVAL;
	169	+
	170	+ reg = FIELD_GET(GICD_IIDR_REVISION_MASK, reg);
	171	+ switch (reg) {
	172	+ case KVM_VGIC_IMP_REV_2:
	173	+ case KVM_VGIC_IMP_REV_3:
	174	+ dist->implementation_rev = reg;
	175	+ return 0;
	176	+ default:
	177	+ return -EINVAL;
	178	+ }
165	179	case GICD_CTLR:
166	180	/* Not a GICv4.1? No HW SGIs */
167	181	if (!kvm_vgic_global_state.has_gicv4_1)
@@ -221,34 +235,58 @@ static void vgic_mmio_write_irouter(struct kvm_vcpu *vcpu, · 53 unmodified lines
221	235	vgic_put_irq(vcpu->kvm, irq);
222	236	}
223	237
	238	+bool vgic_lpis_enabled(struct kvm_vcpu *vcpu)
	239	+{
	240	+ struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
	241	+
	242	+ return atomic_read(&vgic_cpu->ctlr) == GICR_CTLR_ENABLE_LPIS;
	243	+}
	244	+
224	245	static unsigned long vgic_mmio_read_v3r_ctlr(struct kvm_vcpu *vcpu,
225	246	gpa_t addr, unsigned int len)
226	247	{
227	248	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
	249	+ unsigned long val;
228	250
229		− return vgic_cpu->lpis_enabled ? GICR_CTLR_ENABLE_LPIS : 0;
230		−}
	251	+ val = atomic_read(&vgic_cpu->ctlr);
	252	+ if (vgic_get_implementation_rev(vcpu) >= KVM_VGIC_IMP_REV_3)
	253	+ val \|= GICR_CTLR_IR \| GICR_CTLR_CES;
231	254
	255	+ return val;
	256	+}
232	257
233	258	static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu,
234	259	gpa_t addr, unsigned int len,
235	260	unsigned long val)
236	261	{
237	262	struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
238		− bool was_enabled = vgic_cpu->lpis_enabled;
	263	+ u32 ctlr;
239	264
240	265	if (!vgic_has_its(vcpu->kvm))
241	266	return;
242	267
243		− vgic_cpu->lpis_enabled = val & GICR_CTLR_ENABLE_LPIS;
	268	+ if (!(val & GICR_CTLR_ENABLE_LPIS)) {
	269	+ /*
	270	+ * Don't disable if RWP is set, as there already an
	271	+ * ongoing disable. Funky guest...
	272	+ */
	273	+ ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr,
	274	+ GICR_CTLR_ENABLE_LPIS,
	275	+ GICR_CTLR_RWP);
	276	+ if (ctlr != GICR_CTLR_ENABLE_LPIS)
	277	+ return;
244	278
245		− if (was_enabled && !vgic_cpu->lpis_enabled) {
246	279	vgic_flush_pending_lpis(vcpu);
247	280	vgic_its_invalidate_cache(vcpu->kvm);
248		− }
	281	+ atomic_set_release(&vgic_cpu->ctlr, 0);
	282	+ } else {
	283	+ ctlr = atomic_cmpxchg_acquire(&vgic_cpu->ctlr, 0,
	284	+ GICR_CTLR_ENABLE_LPIS);
	285	+ if (ctlr != 0)
	286	+ return;
249	287
250		− if (!was_enabled && vgic_cpu->lpis_enabled)
251	288	vgic_enable_lpis(vcpu);
	289	+ }
252	290	}
253	291
254	292	static bool vgic_mmio_vcpu_rdist_is_last(struct kvm_vcpu *vcpu)
@@ -478,11 +516,10 @@ static void vgic_mmio_write_propbase(struct kvm_vcpu *vcpu, · 223 unmodified lines
478	516	unsigned long val)
479	517	{
480	518	struct vgic_dist *dist = &vcpu->kvm->arch.vgic;
481		− struct vgic_cpu *vgic_cpu = &vcpu->arch.vgic_cpu;
482	519	u64 old_propbaser, propbaser;
483	520
484	521	/* Storing a value with LPIs already enabled is undefined */
485		− if (vgic_cpu->lpis_enabled)
	522	+ if (vgic_lpis_enabled(vcpu))
486	523	return;
487	524
488	525	do {
@@ -513,7 +550,7 @@ static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu, · 24 unmodified lines
513	550	u64 old_pendbaser, pendbaser;
514	551
515	552	/* Storing a value with LPIs already enabled is undefined */
516		− if (vgic_cpu->lpis_enabled)
	553	+ if (vgic_lpis_enabled(vcpu))
517	554	return;
518	555
519	556	do {
@@ -525,6 +562,63 @@ static void vgic_mmio_write_pendbase(struct kvm_vcpu *vcpu, · 5 unmodified lines
525	562	pendbaser) != old_pendbaser);
526	563	}
527	564
	565	+static unsigned long vgic_mmio_read_sync(struct kvm_vcpu *vcpu,
	566	+ gpa_t addr, unsigned int len)
	567	+{
	568	+ return !!atomic_read(&vcpu->arch.vgic_cpu.syncr_busy);
	569	+}
	570	+
	571	+static void vgic_set_rdist_busy(struct kvm_vcpu *vcpu, bool busy)
	572	+{
	573	+ if (busy) {
	574	+ atomic_inc(&vcpu->arch.vgic_cpu.syncr_busy);
	575	+ smp_mb__after_atomic();
	576	+ } else {
	577	+ smp_mb__before_atomic();
	578	+ atomic_dec(&vcpu->arch.vgic_cpu.syncr_busy);
	579	+ }
	580	+}
	581	+
	582	+static void vgic_mmio_write_invlpi(struct kvm_vcpu *vcpu,
	583	+ gpa_t addr, unsigned int len,
	584	+ unsigned long val)
	585	+{
	586	+ struct vgic_irq *irq;
	587	+
	588	+ /*
	589	+ * If the guest wrote only to the upper 32bit part of the
	590	+ * register, drop the write on the floor, as it is only for
	591	+ * vPEs (which we don't support for obvious reasons).
	592	+ *
	593	+ * Also discard the access if LPIs are not enabled.
	594	+ */
	595	+ if ((addr & 4) \|\| !vgic_lpis_enabled(vcpu))
	596	+ return;
	597	+
	598	+ vgic_set_rdist_busy(vcpu, true);
	599	+
	600	+ irq = vgic_get_irq(vcpu->kvm, NULL, lower_32_bits(val));
	601	+ if (irq) {
	602	+ vgic_its_inv_lpi(vcpu->kvm, irq);
	603	+ vgic_put_irq(vcpu->kvm, irq);
	604	+ }
	605	+
	606	+ vgic_set_rdist_busy(vcpu, false);
	607	+}
	608	+
	609	+static void vgic_mmio_write_invall(struct kvm_vcpu *vcpu,
	610	+ gpa_t addr, unsigned int len,
	611	+ unsigned long val)
	612	+{
	613	+ /* See vgic_mmio_write_invlpi() for the early return rationale */
	614	+ if ((addr & 4) \|\| !vgic_lpis_enabled(vcpu))
	615	+ return;
	616	+
	617	+ vgic_set_rdist_busy(vcpu, true);
	618	+ vgic_its_invall(vcpu);
	619	+ vgic_set_rdist_busy(vcpu, false);
	620	+}
	621	+
528	622	/*
529	623	* The GICv3 per-IRQ registers are split to control PPIs and SGIs in the
530	624	* redistributors, while SPIs are covered by registers in the distributor
@@ -630,6 +724,15 @@ static const struct vgic_register_region vgic_v3_rd_registers[] = { · 99 unmodified lines
630	724	REGISTER_DESC_WITH_LENGTH(GICR_PENDBASER,
631	725	vgic_mmio_read_pendbase, vgic_mmio_write_pendbase, 8,
632	726	VGIC_ACCESS_64bit \| VGIC_ACCESS_32bit),
	727	+ REGISTER_DESC_WITH_LENGTH(GICR_INVLPIR,
	728	+ vgic_mmio_read_raz, vgic_mmio_write_invlpi, 8,
	729	+ VGIC_ACCESS_64bit \| VGIC_ACCESS_32bit),
	730	+ REGISTER_DESC_WITH_LENGTH(GICR_INVALLR,
	731	+ vgic_mmio_read_raz, vgic_mmio_write_invall, 8,
	732	+ VGIC_ACCESS_64bit \| VGIC_ACCESS_32bit),
	733	+ REGISTER_DESC_WITH_LENGTH(GICR_SYNCR,
	734	+ vgic_mmio_read_sync, vgic_mmio_write_wi, 4,
	735	+ VGIC_ACCESS_32bit),
633	736	REGISTER_DESC_WITH_LENGTH(GICR_IDREGS,
634	737	vgic_mmio_read_v3_idregs, vgic_mmio_write_wi, 48,
635	738	VGIC_ACCESS_32bit),

arch/arm64/kvm/vgic/vgic.h+10−0

@@ -98,6 +98,11 @@ · 97 unmodified lines
98	98	#define DEBUG_SPINLOCK_BUG_ON(p)
99	99	#endif
100	100
	101	+static inline u32 vgic_get_implementation_rev(struct kvm_vcpu *vcpu)
	102	+{
	103	+ return vcpu->kvm->arch.vgic.implementation_rev;
	104	+}
	105	+
101	106	/* Requires the irq_lock to be held by the caller. */
102	107	static inline bool irq_is_pending(struct vgic_irq *irq)
103	108	{
@@ -308,6 +313,7 @@ static inline bool vgic_dist_overlap(struct kvm *kvm, gpa_t base, size_t size) · 204 unmodified lines
308	313	(base < d->vgic_dist_base + KVM_VGIC_V3_DIST_SIZE);
309	314	}
310	315
	316	+bool vgic_lpis_enabled(struct kvm_vcpu *vcpu);
311	317	int vgic_copy_lpi_list(struct kvm kvm, struct kvm_vcpu vcpu, u32 **intid_ptr);
312	318	int vgic_its_resolve_lpi(struct kvm kvm, struct vgic_its its,
313	319	u32 devid, u32 eventid, struct vgic_irq **irq);
@@ -317,6 +323,10 @@ void vgic_lpi_translation_cache_init(struct kvm *kvm); · 3 unmodified lines
317	323	void vgic_lpi_translation_cache_destroy(struct kvm *kvm);
318	324	void vgic_its_invalidate_cache(struct kvm *kvm);
319	325
	326	+/* GICv4.1 MMIO interface */
	327	+int vgic_its_inv_lpi(struct kvm kvm, struct vgic_irq irq);
	328	+int vgic_its_invall(struct kvm_vcpu *vcpu);
	329	+
320	330	bool vgic_supports_direct_msis(struct kvm *kvm);
321	331	int vgic_v4_init(struct kvm *kvm);
322	332	void vgic_v4_teardown(struct kvm *kvm);

include/kvm/arm_vgic.h+6−2

@@ -231,6 +231,9 @@ struct vgic_dist { · 230 unmodified lines
231	231
232	232	/* Implementation revision as reported in the GICD_IIDR */
233	233	u32 implementation_rev;
	234	+#define KVM_VGIC_IMP_REV_2 2 /* GICv2 restorable groups */
	235	+#define KVM_VGIC_IMP_REV_3 3 /* GICv3 GICR_CTLR.{IW,CES,RWP} */
	236	+#define KVM_VGIC_IMP_REV_LATEST KVM_VGIC_IMP_REV_3
234	237
235	238	/* Userspace can write to GICv2 IGROUPR */
236	239	bool v2_groups_user_writable;
@@ -344,11 +347,12 @@ struct vgic_cpu { · 107 unmodified lines
344	347	struct vgic_io_device rd_iodev;
345	348	struct vgic_redist_region *rdreg;
346	349	u32 rdreg_index;
	350	+ atomic_t syncr_busy;
347	351
348	352	/* Contains the attributes and gpa of the LPI pending tables. */
349	353	u64 pendbaser;
350		−
351		− bool lpis_enabled;
	354	+ /* GICR_CTLR.{ENABLE_LPIS,RWP} */
	355	+ atomic_t ctlr;
352	356
353	357	/* Cache guest priority bits */
354	358	u32 num_pri_bits;

include/linux/irqchip/arm-gic-v3.h+2−0

@@ -127,6 +127,8 @@ · 126 unmodified lines
127	127	#define GICR_PIDR2 GICD_PIDR2
128	128
129	129	#define GICR_CTLR_ENABLE_LPIS (1UL << 0)
	130	+#define GICR_CTLR_CES (1UL << 1)
	131	+#define GICR_CTLR_IR (1UL << 2)
130	132	#define GICR_CTLR_RWP (1UL << 3)
131	133
132	134	#define GICR_TYPER_CPU_NUMBER(r) (((r) >> 8) & 0xffff)