Merge branches 'v5.15/vfio/spdx-license-cleanups', 'v5.15/vfio/dma-valid-waited-v3', 'v5.15/vfio/vfio-pci-core-v5' and 'v5.15/vfio/vfio-ap' into v5.15/vfio/next

Alex Williamson committed 5 years ago

Commit ea870734 parents 29848a0 ffc95d1 7fa005c eb0feef28 files · +2704 −2483Browse files

Unified · Split

28 files

Documentation/PCI/pci.rst +1 −0
MAINTAINERS +1 −0
arch/s390/include/asm/kvm_host.h +3 −5
arch/s390/kvm/kvm-s390.c +28 −4
arch/s390/kvm/priv.c +9 −6
drivers/pci/pci-driver.c +21 −7
drivers/s390/crypto/vfio_ap_ops.c +132 −150
drivers/s390/crypto/vfio_ap_private.h +3 −3
drivers/vfio/Kconfig +15 −14
drivers/vfio/fsl-mc/Kconfig +2 −1
drivers/vfio/mdev/Kconfig +0 −1
drivers/vfio/pci/Kconfig +22 −18
drivers/vfio/pci/Makefile +5 −3
drivers/vfio/pci/vfio_pci.c +107 −2155
drivers/vfio/pci/vfio_pci_config.c +35 −35
drivers/vfio/pci/vfio_pci_core.c +2158 −0
drivers/vfio/pci/vfio_pci_igd.c +10 −9
drivers/vfio/pci/vfio_pci_intrs.c +21 −21
drivers/vfio/pci/vfio_pci_rdwr.c +9 −9
drivers/vfio/pci/vfio_pci_zdev.c +2 −2
drivers/vfio/platform/Kconfig +4 −2
drivers/vfio/platform/reset/Kconfig +1 −3
drivers/vfio/vfio_iommu_type1.c +4 −4
include/linux/mod_devicetable.h +6 −0
include/linux/pci.h +29 −0
include/linux/vfio_pci_core.h +60 −29
scripts/mod/devicetable-offsets.c +1 −0
scripts/mod/file2alias.c +15 −2

@@ -103,6 +103,7 @@ need pass only as many optional fields as necessary: · 102 unmodified lines
103	- subvendor and subdevice fields default to PCI_ANY_ID (FFFFFFFF)	103	- subvendor and subdevice fields default to PCI_ANY_ID (FFFFFFFF)
104	- class and classmask fields default to 0	104	- class and classmask fields default to 0
105	- driver_data defaults to 0UL.	105	- driver_data defaults to 0UL.
		106	- override_only field defaults to 0.
106		107
107	Note that driver_data must match the value used by any of the pci_device_id	108	Note that driver_data must match the value used by any of the pci_device_id
108	entries defined in the driver. This makes the driver_data field mandatory	109	entries defined in the driver. This makes the driver_data field mandatory

MAINTAINERS+1−0

@@ -19466,6 +19466,7 @@ T: git git://github.com/awilliam/linux-vfio.git · 19465 unmodified lines
19466	F: Documentation/driver-api/vfio.rst	19466	F: Documentation/driver-api/vfio.rst
19467	F: drivers/vfio/	19467	F: drivers/vfio/
19468	F: include/linux/vfio.h	19468	F: include/linux/vfio.h
		19469	F: include/linux/vfio_pci_core.h
19469	F: include/uapi/linux/vfio.h	19470	F: include/uapi/linux/vfio.h
19470		19471
19471	VFIO FSL-MC DRIVER	19472	VFIO FSL-MC DRIVER

arch/s390/include/asm/kvm_host.h+3−5

 	unsigned short ibc;
 };
-struct kvm_s390_module_hook {
+typedef int (*crypto_hook)(struct kvm_vcpu *vcpu);
-	int (*hook)(struct kvm_vcpu *vcpu);
-	struct module *owner;
-};
 struct kvm_s390_crypto {
 	struct kvm_s390_crypto_cb *crycb;
-	struct kvm_s390_module_hook *pqap_hook;
+	struct rw_semaphore pqap_hook_rwsem;
+	crypto_hook *pqap_hook;
 	__u32 crycbd;
 	__u8 aes_kw;
 	__u8 dea_kw;

arch/s390/kvm/kvm-s390.c+28−4

 		kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
 }
+/*
+ * kvm_arch_crypto_set_masks
+ *
+ * @kvm: pointer to the target guest's KVM struct containing the crypto masks
+ *	 to be set.
+ * @apm: the mask identifying the accessible AP adapters
+ * @aqm: the mask identifying the accessible AP domains
+ * @adm: the mask identifying the accessible AP control domains
+ *
+ * Set the masks that identify the adapters, domains and control domains to
+ * which the KVM guest is granted access.
+ *
+ * Note: The kvm->lock mutex must be locked by the caller before invoking this
+ *	 function.
+ */
 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
 			       unsigned long *aqm, unsigned long *adm)
 {
 	struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
-	mutex_lock(&kvm->lock);
 	kvm_s390_vcpu_block_all(kvm);
 	switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
 	/* recreate the shadow crycb for each vcpu */
 	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
 	kvm_s390_vcpu_unblock_all(kvm);
-	mutex_unlock(&kvm->lock);
 }
 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
+/*
+ * kvm_arch_crypto_clear_masks
+ *
+ * @kvm: pointer to the target guest's KVM struct containing the crypto masks
+ *	 to be cleared.
+ *
+ * Clear the masks that identify the adapters, domains and control domains to
+ * which the KVM guest is granted access.
+ *
+ * Note: The kvm->lock mutex must be locked by the caller before invoking this
+ *	 function.
+ */
 void kvm_arch_crypto_clear_masks(struct kvm *kvm)
 {
-	mutex_lock(&kvm->lock);
 	kvm_s390_vcpu_block_all(kvm);
 	memset(&kvm->arch.crypto.crycb->apcb0, 0,
 	/* recreate the shadow crycb for each vcpu */
 	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
 	kvm_s390_vcpu_unblock_all(kvm);
-	mutex_unlock(&kvm->lock);
 }
 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
 {
 	kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
 	kvm_s390_set_crycb_format(kvm);
+	init_rwsem(&kvm->arch.crypto.pqap_hook_rwsem);
 	if (!test_kvm_facility(kvm, 76))
 		return;

arch/s390/kvm/priv.c+9−6

 static int handle_pqap(struct kvm_vcpu *vcpu)
 {
 	struct ap_queue_status status = {};
+	crypto_hook pqap_hook;
 	unsigned long reg0;
 	int ret;
 	uint8_t fc;
 		return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
 	/*
-	 * Verify that the hook callback is registered, lock the owner
+	 * If the hook callback is registered, there will be a pointer to the
-	 * and call the hook.
+	 * hook function pointer in the kvm_s390_crypto structure. Lock the
+	 * owner, retrieve the hook function pointer and call the hook.
 	 */
+	down_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem);
 	if (vcpu->kvm->arch.crypto.pqap_hook) {
-		if (!try_module_get(vcpu->kvm->arch.crypto.pqap_hook->owner))
+		pqap_hook = *vcpu->kvm->arch.crypto.pqap_hook;
-			return -EOPNOTSUPP;
+		ret = pqap_hook(vcpu);
-		ret = vcpu->kvm->arch.crypto.pqap_hook->hook(vcpu);
-		module_put(vcpu->kvm->arch.crypto.pqap_hook->owner);
 		if (!ret && vcpu->run->s.regs.gprs[1] & 0x00ff0000)
 			kvm_s390_set_psw_cc(vcpu, 3);
+		up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem);
 		return ret;
 	}
+	up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem);
 	/*
 	 * A vfio_driver must register a hook.
 	 * No hook means no driver to enable the SIE CRYCB and no queues.

drivers/pci/pci-driver.c+21−7

 						    struct pci_dev *dev)
 {
 	struct pci_dynid *dynid;
-	const struct pci_device_id *found_id = NULL;
+	const struct pci_device_id *found_id = NULL, *ids;
 	/* When driver_override is set, only bind to the matching driver */
 	if (dev->driver_override && strcmp(dev->driver_override, drv->name))
 	}
 	spin_unlock(&drv->dynids.lock);
-	if (!found_id)
+	if (found_id)
-		found_id = pci_match_id(drv->id_table, dev);
+		return found_id;
-	/* driver_override will always match, send a dummy id */
+	for (ids = drv->id_table; (found_id = pci_match_id(ids, dev));
-	if (!found_id && dev->driver_override)
+	     ids = found_id + 1) {
-		found_id = &pci_device_id_any;
+		/*
+		 * The match table is split based on driver_override.
+		 * In case override_only was set, enforce driver_override
+		 * matching.
+		 */
+		if (found_id->override_only) {
+			if (dev->driver_override)
+				return found_id;
+		} else {
+			return found_id;
+		}
+	}
-	return found_id;
+	/* driver_override will always match, send a dummy id */
+	if (dev->driver_override)
+		return &pci_device_id_any;
+	return NULL;
 }
 /**

drivers/s390/crypto/vfio_ap_ops.c+132−150

@@ -24,8 +24,9 @@ · 23 unmodified lines
24	#define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"	24	#define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
25	#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"	25	#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
26		26
27	static int vfio_ap_mdev_reset_queues(struct mdev_device *mdev);	27	static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev);
28	static struct vfio_ap_queue *vfio_ap_find_queue(int apqn);	28	static struct vfio_ap_queue *vfio_ap_find_queue(int apqn);
		29	static const struct vfio_device_ops vfio_ap_matrix_dev_ops;
29		30
30	static int match_apqn(struct device dev, const void data)	31	static int match_apqn(struct device dev, const void data)
31	{	32	{
@@ -294,15 +295,6 @@ static int handle_pqap(struct kvm_vcpu *vcpu) · 262 unmodified lines
294	matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,	295	matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,
295	struct ap_matrix_mdev, pqap_hook);	296	struct ap_matrix_mdev, pqap_hook);
296		297
297	/*
298	* If the KVM pointer is in the process of being set, wait until the
299	* process has completed.
300	*/
301	wait_event_cmd(matrix_mdev->wait_for_kvm,
302	!matrix_mdev->kvm_busy,
303	mutex_unlock(&matrix_dev->lock),
304	mutex_lock(&matrix_dev->lock));
305
306	/* If the there is no guest using the mdev, there is nothing to do */	298	/* If the there is no guest using the mdev, there is nothing to do */
307	if (!matrix_mdev->kvm)	299	if (!matrix_mdev->kvm)
308	goto out_unlock;	300	goto out_unlock;
@@ -335,45 +327,57 @@ static void vfio_ap_matrix_init(struct ap_config_info *info, · 26 unmodified lines
335	matrix->adm_max = info->apxa ? info->Nd : 15;	327	matrix->adm_max = info->apxa ? info->Nd : 15;
336	}	328	}
337		329
338	static int vfio_ap_mdev_create(struct mdev_device *mdev)	330	static int vfio_ap_mdev_probe(struct mdev_device *mdev)
339	{	331	{
340	struct ap_matrix_mdev *matrix_mdev;	332	struct ap_matrix_mdev *matrix_mdev;
		333	int ret;
341		334
342	if ((atomic_dec_if_positive(&matrix_dev->available_instances) < 0))	335	if ((atomic_dec_if_positive(&matrix_dev->available_instances) < 0))
343	return -EPERM;	336	return -EPERM;
344		337
345	matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);	338	matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
346	if (!matrix_mdev) {	339	if (!matrix_mdev) {
347	atomic_inc(&matrix_dev->available_instances);	340	ret = -ENOMEM;
348	return -ENOMEM;	341	goto err_dec_available;
349	}	342	}
		343	vfio_init_group_dev(&matrix_mdev->vdev, &mdev->dev,
		344	&vfio_ap_matrix_dev_ops);
350		345
351	matrix_mdev->mdev = mdev;	346	matrix_mdev->mdev = mdev;
352	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);	347	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);
353	init_waitqueue_head(&matrix_mdev->wait_for_kvm);	348	matrix_mdev->pqap_hook = handle_pqap;
354	mdev_set_drvdata(mdev, matrix_mdev);
355	matrix_mdev->pqap_hook.hook = handle_pqap;
356	matrix_mdev->pqap_hook.owner = THIS_MODULE;
357	mutex_lock(&matrix_dev->lock);	349	mutex_lock(&matrix_dev->lock);
358	list_add(&matrix_mdev->node, &matrix_dev->mdev_list);	350	list_add(&matrix_mdev->node, &matrix_dev->mdev_list);
359	mutex_unlock(&matrix_dev->lock);	351	mutex_unlock(&matrix_dev->lock);
360		352
		353	ret = vfio_register_group_dev(&matrix_mdev->vdev);
		354	if (ret)
		355	goto err_list;
		356	dev_set_drvdata(&mdev->dev, matrix_mdev);
361	return 0;	357	return 0;
		358
		359	err_list:
		360	mutex_lock(&matrix_dev->lock);
		361	list_del(&matrix_mdev->node);
		362	mutex_unlock(&matrix_dev->lock);
		363	kfree(matrix_mdev);
		364	err_dec_available:
		365	atomic_inc(&matrix_dev->available_instances);
		366	return ret;
362	}	367	}
363		368
364	static int vfio_ap_mdev_remove(struct mdev_device *mdev)	369	static void vfio_ap_mdev_remove(struct mdev_device *mdev)
365	{	370	{
366	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);	371	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev);
		372
		373	vfio_unregister_group_dev(&matrix_mdev->vdev);
367		374
368	mutex_lock(&matrix_dev->lock);	375	mutex_lock(&matrix_dev->lock);
369	vfio_ap_mdev_reset_queues(mdev);	376	vfio_ap_mdev_reset_queues(matrix_mdev);
370	list_del(&matrix_mdev->node);	377	list_del(&matrix_mdev->node);
371	kfree(matrix_mdev);	378	kfree(matrix_mdev);
372	mdev_set_drvdata(mdev, NULL);
373	atomic_inc(&matrix_dev->available_instances);	379	atomic_inc(&matrix_dev->available_instances);
374	mutex_unlock(&matrix_dev->lock);	380	mutex_unlock(&matrix_dev->lock);
375
376	return 0;
377	}	381	}
378		382
379	static ssize_t name_show(struct mdev_type *mtype,	383	static ssize_t name_show(struct mdev_type *mtype,
@@ -615,16 +619,12 @@ static ssize_t assign_adapter_store(struct device *dev, · 235 unmodified lines
615	{	619	{
616	int ret;	620	int ret;
617	unsigned long apid;	621	unsigned long apid;
618	struct mdev_device *mdev = mdev_from_dev(dev);	622	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
619	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
620		623
621	mutex_lock(&matrix_dev->lock);	624	mutex_lock(&matrix_dev->lock);
622		625
623	/*	626	/* If the KVM guest is running, disallow assignment of adapter */
624	* If the KVM pointer is in flux or the guest is running, disallow	627	if (matrix_mdev->kvm) {
625	* un-assignment of adapter
626	*/
627	if (matrix_mdev->kvm_busy \|\| matrix_mdev->kvm) {
628	ret = -EBUSY;	628	ret = -EBUSY;
629	goto done;	629	goto done;
630	}	630	}
@@ -688,16 +688,12 @@ static ssize_t unassign_adapter_store(struct device *dev, · 57 unmodified lines
688	{	688	{
689	int ret;	689	int ret;
690	unsigned long apid;	690	unsigned long apid;
691	struct mdev_device *mdev = mdev_from_dev(dev);	691	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
692	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
693		692
694	mutex_lock(&matrix_dev->lock);	693	mutex_lock(&matrix_dev->lock);
695		694
696	/*	695	/* If the KVM guest is running, disallow unassignment of adapter */
697	* If the KVM pointer is in flux or the guest is running, disallow	696	if (matrix_mdev->kvm) {
698	* un-assignment of adapter
699	*/
700	if (matrix_mdev->kvm_busy \|\| matrix_mdev->kvm) {
701	ret = -EBUSY;	697	ret = -EBUSY;
702	goto done;	698	goto done;
703	}	699	}
@@ -777,17 +773,13 @@ static ssize_t assign_domain_store(struct device *dev, · 73 unmodified lines
777	{	773	{
778	int ret;	774	int ret;
779	unsigned long apqi;	775	unsigned long apqi;
780	struct mdev_device *mdev = mdev_from_dev(dev);	776	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
781	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
782	unsigned long max_apqi = matrix_mdev->matrix.aqm_max;	777	unsigned long max_apqi = matrix_mdev->matrix.aqm_max;
783		778
784	mutex_lock(&matrix_dev->lock);	779	mutex_lock(&matrix_dev->lock);
785		780
786	/*	781	/* If the KVM guest is running, disallow assignment of domain */
787	* If the KVM pointer is in flux or the guest is running, disallow	782	if (matrix_mdev->kvm) {
788	* assignment of domain
789	*/
790	if (matrix_mdev->kvm_busy \|\| matrix_mdev->kvm) {
791	ret = -EBUSY;	783	ret = -EBUSY;
792	goto done;	784	goto done;
793	}	785	}
@@ -846,16 +838,12 @@ static ssize_t unassign_domain_store(struct device *dev, · 52 unmodified lines
846	{	838	{
847	int ret;	839	int ret;
848	unsigned long apqi;	840	unsigned long apqi;
849	struct mdev_device *mdev = mdev_from_dev(dev);	841	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
850	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
851		842
852	mutex_lock(&matrix_dev->lock);	843	mutex_lock(&matrix_dev->lock);
853		844
854	/*	845	/* If the KVM guest is running, disallow unassignment of domain */
855	* If the KVM pointer is in flux or the guest is running, disallow	846	if (matrix_mdev->kvm) {
856	* un-assignment of domain
857	*/
858	if (matrix_mdev->kvm_busy \|\| matrix_mdev->kvm) {
859	ret = -EBUSY;	847	ret = -EBUSY;
860	goto done;	848	goto done;
861	}	849	}
@@ -900,16 +888,12 @@ static ssize_t assign_control_domain_store(struct device *dev, · 38 unmodified lines
900	{	888	{
901	int ret;	889	int ret;
902	unsigned long id;	890	unsigned long id;
903	struct mdev_device *mdev = mdev_from_dev(dev);	891	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
904	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
905		892
906	mutex_lock(&matrix_dev->lock);	893	mutex_lock(&matrix_dev->lock);
907		894
908	/*	895	/* If the KVM guest is running, disallow assignment of control domain */
909	* If the KVM pointer is in flux or the guest is running, disallow	896	if (matrix_mdev->kvm) {
910	* assignment of control domain.
911	*/
912	if (matrix_mdev->kvm_busy \|\| matrix_mdev->kvm) {
913	ret = -EBUSY;	897	ret = -EBUSY;
914	goto done;	898	goto done;
915	}	899	}
@@ -958,17 +942,13 @@ static ssize_t unassign_control_domain_store(struct device *dev, · 42 unmodified lines
958	{	942	{
959	int ret;	943	int ret;
960	unsigned long domid;	944	unsigned long domid;
961	struct mdev_device *mdev = mdev_from_dev(dev);	945	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
962	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
963	unsigned long max_domid = matrix_mdev->matrix.adm_max;	946	unsigned long max_domid = matrix_mdev->matrix.adm_max;
964		947
965	mutex_lock(&matrix_dev->lock);	948	mutex_lock(&matrix_dev->lock);
966		949
967	/*	950	/* If a KVM guest is running, disallow unassignment of control domain */
968	* If the KVM pointer is in flux or the guest is running, disallow	951	if (matrix_mdev->kvm) {
969	* un-assignment of control domain.
970	*/
971	if (matrix_mdev->kvm_busy \|\| matrix_mdev->kvm) {
972	ret = -EBUSY;	952	ret = -EBUSY;
973	goto done;	953	goto done;
974	}	954	}
@@ -997,8 +977,7 @@ static ssize_t control_domains_show(struct device *dev, · 22 unmodified lines
997	int nchars = 0;	977	int nchars = 0;
998	int n;	978	int n;
999	char *bufpos = buf;	979	char *bufpos = buf;
1000	struct mdev_device *mdev = mdev_from_dev(dev);	980	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1001	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
1002	unsigned long max_domid = matrix_mdev->matrix.adm_max;	981	unsigned long max_domid = matrix_mdev->matrix.adm_max;
1003		982
1004	mutex_lock(&matrix_dev->lock);	983	mutex_lock(&matrix_dev->lock);
@@ -1016,8 +995,7 @@ static DEVICE_ATTR_RO(control_domains); · 11 unmodified lines
1016	static ssize_t matrix_show(struct device dev, struct device_attribute attr,	995	static ssize_t matrix_show(struct device dev, struct device_attribute attr,
1017	char *buf)	996	char *buf)
1018	{	997	{
1019	struct mdev_device *mdev = mdev_from_dev(dev);	998	struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1020	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
1021	char *bufpos = buf;	999	char *bufpos = buf;
1022	unsigned long apid;	1000	unsigned long apid;
1023	unsigned long apqi;	1001	unsigned long apqi;
@@ -1109,23 +1087,30 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, · 85 unmodified lines
1109	struct ap_matrix_mdev *m;	1087	struct ap_matrix_mdev *m;
1110		1088
1111	if (kvm->arch.crypto.crycbd) {	1089	if (kvm->arch.crypto.crycbd) {
		1090	down_write(&kvm->arch.crypto.pqap_hook_rwsem);
		1091	kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
		1092	up_write(&kvm->arch.crypto.pqap_hook_rwsem);
		1093
		1094	mutex_lock(&kvm->lock);
		1095	mutex_lock(&matrix_dev->lock);
		1096
1112	list_for_each_entry(m, &matrix_dev->mdev_list, node) {	1097	list_for_each_entry(m, &matrix_dev->mdev_list, node) {
1113	if (m != matrix_mdev && m->kvm == kvm)	1098	if (m != matrix_mdev && m->kvm == kvm) {
		1099	mutex_unlock(&kvm->lock);
		1100	mutex_unlock(&matrix_dev->lock);
1114	return -EPERM;	1101	return -EPERM;
		1102	}
1115	}	1103	}
1116		1104
1117	kvm_get_kvm(kvm);	1105	kvm_get_kvm(kvm);
1118	matrix_mdev->kvm_busy = true;	1106	matrix_mdev->kvm = kvm;
1119	mutex_unlock(&matrix_dev->lock);
1120	kvm_arch_crypto_set_masks(kvm,	1107	kvm_arch_crypto_set_masks(kvm,
1121	matrix_mdev->matrix.apm,	1108	matrix_mdev->matrix.apm,
1122	matrix_mdev->matrix.aqm,	1109	matrix_mdev->matrix.aqm,
1123	matrix_mdev->matrix.adm);	1110	matrix_mdev->matrix.adm);
1124	mutex_lock(&matrix_dev->lock);	1111
1125	kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;	1112	mutex_unlock(&kvm->lock);
1126	matrix_mdev->kvm = kvm;	1113	mutex_unlock(&matrix_dev->lock);
1127	matrix_mdev->kvm_busy = false;
1128	wake_up_all(&matrix_mdev->wait_for_kvm);
1129	}	1114	}
1130		1115
1131	return 0;	1116	return 0;
@@ -1175,28 +1160,24 @@ static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb, · 43 unmodified lines
1175	* done under the @matrix_mdev->lock.	1160	* done under the @matrix_mdev->lock.
1176	*	1161	*
1177	*/	1162	*/
1178	static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev)	1163	static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev,
		1164	struct kvm *kvm)
1179	{	1165	{
1180	/*	1166	if (kvm && kvm->arch.crypto.crycbd) {
1181	* If the KVM pointer is in the process of being set, wait until the	1167	down_write(&kvm->arch.crypto.pqap_hook_rwsem);
1182	* process has completed.	1168	kvm->arch.crypto.pqap_hook = NULL;
1183	*/	1169	up_write(&kvm->arch.crypto.pqap_hook_rwsem);
1184	wait_event_cmd(matrix_mdev->wait_for_kvm,
1185	!matrix_mdev->kvm_busy,
1186	mutex_unlock(&matrix_dev->lock),
1187	mutex_lock(&matrix_dev->lock));
1188		1170
1189	if (matrix_mdev->kvm) {	1171	mutex_lock(&kvm->lock);
1190	matrix_mdev->kvm_busy = true;
1191	mutex_unlock(&matrix_dev->lock);
1192	kvm_arch_crypto_clear_masks(matrix_mdev->kvm);
1193	mutex_lock(&matrix_dev->lock);	1172	mutex_lock(&matrix_dev->lock);
1194	vfio_ap_mdev_reset_queues(matrix_mdev->mdev);	1173
1195	matrix_mdev->kvm->arch.crypto.pqap_hook = NULL;	1174	kvm_arch_crypto_clear_masks(kvm);
1196	kvm_put_kvm(matrix_mdev->kvm);	1175	vfio_ap_mdev_reset_queues(matrix_mdev);
		1176	kvm_put_kvm(kvm);
1197	matrix_mdev->kvm = NULL;	1177	matrix_mdev->kvm = NULL;
1198	matrix_mdev->kvm_busy = false;	1178
1199	wake_up_all(&matrix_mdev->wait_for_kvm);	1179	mutex_unlock(&kvm->lock);
		1180	mutex_unlock(&matrix_dev->lock);
1200	}	1181	}
1201	}	1182	}
1202		1183
@@ -1209,16 +1190,13 @@ static int vfio_ap_mdev_group_notifier(struct notifier_block *nb, · 6 unmodified lines
1209	if (action != VFIO_GROUP_NOTIFY_SET_KVM)	1190	if (action != VFIO_GROUP_NOTIFY_SET_KVM)
1210	return NOTIFY_OK;	1191	return NOTIFY_OK;
1211		1192
1212	mutex_lock(&matrix_dev->lock);
1213	matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier);	1193	matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier);
1214		1194
1215	if (!data)	1195	if (!data)
1216	vfio_ap_mdev_unset_kvm(matrix_mdev);	1196	vfio_ap_mdev_unset_kvm(matrix_mdev, matrix_mdev->kvm);
1217	else if (vfio_ap_mdev_set_kvm(matrix_mdev, data))	1197	else if (vfio_ap_mdev_set_kvm(matrix_mdev, data))
1218	notify_rc = NOTIFY_DONE;	1198	notify_rc = NOTIFY_DONE;
1219		1199
1220	mutex_unlock(&matrix_dev->lock);
1221
1222	return notify_rc;	1200	return notify_rc;
1223	}	1201	}
1224		1202
@@ -1288,13 +1266,12 @@ free_resources: · 63 unmodified lines
1288	return ret;	1266	return ret;
1289	}	1267	}
1290		1268
1291	static int vfio_ap_mdev_reset_queues(struct mdev_device *mdev)	1269	static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev)
1292	{	1270	{
1293	int ret;	1271	int ret;
1294	int rc = 0;	1272	int rc = 0;
1295	unsigned long apid, apqi;	1273	unsigned long apid, apqi;
1296	struct vfio_ap_queue *q;	1274	struct vfio_ap_queue *q;
1297	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
1298		1275
1299	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm,	1276	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm,
1300	matrix_mdev->matrix.apm_max + 1) {	1277	matrix_mdev->matrix.apm_max + 1) {
@@ -1315,52 +1292,45 @@ static int vfio_ap_mdev_reset_queues(struct mdev_device *mdev) · 14 unmodified lines
1315	return rc;	1292	return rc;
1316	}	1293	}
1317		1294
1318	static int vfio_ap_mdev_open_device(struct mdev_device *mdev)	1295	static int vfio_ap_mdev_open_device(struct vfio_device *vdev)
1319	{	1296	{
1320	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);	1297	struct ap_matrix_mdev *matrix_mdev =
		1298	container_of(vdev, struct ap_matrix_mdev, vdev);
1321	unsigned long events;	1299	unsigned long events;
1322	int ret;	1300	int ret;
1323		1301
1324
1325	if (!try_module_get(THIS_MODULE))
1326	return -ENODEV;
1327
1328	matrix_mdev->group_notifier.notifier_call = vfio_ap_mdev_group_notifier;	1302	matrix_mdev->group_notifier.notifier_call = vfio_ap_mdev_group_notifier;
1329	events = VFIO_GROUP_NOTIFY_SET_KVM;	1303	events = VFIO_GROUP_NOTIFY_SET_KVM;
1330		1304
1331	ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,	1305	ret = vfio_register_notifier(vdev->dev, VFIO_GROUP_NOTIFY,
1332	&events, &matrix_mdev->group_notifier);	1306	&events, &matrix_mdev->group_notifier);
1333	if (ret) {	1307	if (ret)
1334	module_put(THIS_MODULE);
1335	return ret;	1308	return ret;
1336	}
1337		1309
1338	matrix_mdev->iommu_notifier.notifier_call = vfio_ap_mdev_iommu_notifier;	1310	matrix_mdev->iommu_notifier.notifier_call = vfio_ap_mdev_iommu_notifier;
1339	events = VFIO_IOMMU_NOTIFY_DMA_UNMAP;	1311	events = VFIO_IOMMU_NOTIFY_DMA_UNMAP;
1340	ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,	1312	ret = vfio_register_notifier(vdev->dev, VFIO_IOMMU_NOTIFY,
1341	&events, &matrix_mdev->iommu_notifier);	1313	&events, &matrix_mdev->iommu_notifier);
1342	if (!ret)	1314	if (ret)
1343	return ret;	1315	goto out_unregister_group;
		1316	return 0;
1344		1317
1345	vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,	1318	out_unregister_group:
		1319	vfio_unregister_notifier(vdev->dev, VFIO_GROUP_NOTIFY,
1346	&matrix_mdev->group_notifier);	1320	&matrix_mdev->group_notifier);
1347	module_put(THIS_MODULE);
1348	return ret;	1321	return ret;
1349	}	1322	}
1350		1323
1351	static void vfio_ap_mdev_close_device(struct mdev_device *mdev)	1324	static void vfio_ap_mdev_close_device(struct vfio_device *vdev)
1352	{	1325	{
1353	struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);	1326	struct ap_matrix_mdev *matrix_mdev =
1354		1327	container_of(vdev, struct ap_matrix_mdev, vdev);
1355	mutex_lock(&matrix_dev->lock);
1356	vfio_ap_mdev_unset_kvm(matrix_mdev);
1357	mutex_unlock(&matrix_dev->lock);
1358		1328
1359	vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,	1329	vfio_unregister_notifier(vdev->dev, VFIO_IOMMU_NOTIFY,
1360	&matrix_mdev->iommu_notifier);	1330	&matrix_mdev->iommu_notifier);
1361	vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,	1331	vfio_unregister_notifier(vdev->dev, VFIO_GROUP_NOTIFY,
1362	&matrix_mdev->group_notifier);	1332	&matrix_mdev->group_notifier);
1363	module_put(THIS_MODULE);	1333	vfio_ap_mdev_unset_kvm(matrix_mdev, matrix_mdev->kvm);
1364	}	1334	}
1365		1335
1366	static int vfio_ap_mdev_get_device_info(unsigned long arg)	1336	static int vfio_ap_mdev_get_device_info(unsigned long arg)
@@ -1383,11 +1353,12 @@ static int vfio_ap_mdev_get_device_info(unsigned long arg) · 16 unmodified lines
1383	return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;	1353	return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
1384	}	1354	}
1385		1355
1386	static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev,	1356	static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev,
1387	unsigned int cmd, unsigned long arg)	1357	unsigned int cmd, unsigned long arg)
1388	{	1358	{
		1359	struct ap_matrix_mdev *matrix_mdev =
		1360	container_of(vdev, struct ap_matrix_mdev, vdev);
1389	int ret;	1361	int ret;
1390	struct ap_matrix_mdev *matrix_mdev;
1391		1362
1392	mutex_lock(&matrix_dev->lock);	1363	mutex_lock(&matrix_dev->lock);
1393	switch (cmd) {	1364	switch (cmd) {
@@ -1395,22 +1366,7 @@ static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev, · 1 unmodified line
1395	ret = vfio_ap_mdev_get_device_info(arg);	1366	ret = vfio_ap_mdev_get_device_info(arg);
1396	break;	1367	break;
1397	case VFIO_DEVICE_RESET:	1368	case VFIO_DEVICE_RESET:
1398	matrix_mdev = mdev_get_drvdata(mdev);	1369	ret = vfio_ap_mdev_reset_queues(matrix_mdev);
1399	if (WARN(!matrix_mdev, "Driver data missing from mdev!!")) {
1400	ret = -EINVAL;
1401	break;
1402	}
1403
1404	/*
1405	* If the KVM pointer is in the process of being set, wait until
1406	* the process has completed.
1407	*/
1408	wait_event_cmd(matrix_mdev->wait_for_kvm,
1409	!matrix_mdev->kvm_busy,
1410	mutex_unlock(&matrix_dev->lock),
1411	mutex_lock(&matrix_dev->lock));
1412
1413	ret = vfio_ap_mdev_reset_queues(mdev);
1414	break;	1370	break;
1415	default:	1371	default:
1416	ret = -EOPNOTSUPP;	1372	ret = -EOPNOTSUPP;
@@ -1421,25 +1377,51 @@ static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev, · 4 unmodified lines
1421	return ret;	1377	return ret;
1422	}	1378	}
1423		1379
		1380	static const struct vfio_device_ops vfio_ap_matrix_dev_ops = {
		1381	.open_device = vfio_ap_mdev_open_device,
		1382	.close_device = vfio_ap_mdev_close_device,
		1383	.ioctl = vfio_ap_mdev_ioctl,
		1384	};
		1385
		1386	static struct mdev_driver vfio_ap_matrix_driver = {
		1387	.driver = {
		1388	.name = "vfio_ap_mdev",
		1389	.owner = THIS_MODULE,
		1390	.mod_name = KBUILD_MODNAME,
		1391	.dev_groups = vfio_ap_mdev_attr_groups,
		1392	},
		1393	.probe = vfio_ap_mdev_probe,
		1394	.remove = vfio_ap_mdev_remove,
		1395	};
		1396
1424	static const struct mdev_parent_ops vfio_ap_matrix_ops = {	1397	static const struct mdev_parent_ops vfio_ap_matrix_ops = {
1425	.owner = THIS_MODULE,	1398	.owner = THIS_MODULE,
		1399	.device_driver = &vfio_ap_matrix_driver,
1426	.supported_type_groups = vfio_ap_mdev_type_groups,	1400	.supported_type_groups = vfio_ap_mdev_type_groups,
1427	.mdev_attr_groups = vfio_ap_mdev_attr_groups,
1428	.create = vfio_ap_mdev_create,
1429	.remove = vfio_ap_mdev_remove,
1430	.open_device = vfio_ap_mdev_open_device,
1431	.close_device = vfio_ap_mdev_close_device,
1432	.ioctl = vfio_ap_mdev_ioctl,
1433	};	1401	};
1434		1402
1435	int vfio_ap_mdev_register(void)	1403	int vfio_ap_mdev_register(void)
1436	{	1404	{
		1405	int ret;
		1406
1437	atomic_set(&matrix_dev->available_instances, MAX_ZDEV_ENTRIES_EXT);	1407	atomic_set(&matrix_dev->available_instances, MAX_ZDEV_ENTRIES_EXT);
1438		1408
1439	return mdev_register_device(&matrix_dev->device, &vfio_ap_matrix_ops);	1409	ret = mdev_register_driver(&vfio_ap_matrix_driver);
		1410	if (ret)
		1411	return ret;
		1412
		1413	ret = mdev_register_device(&matrix_dev->device, &vfio_ap_matrix_ops);
		1414	if (ret)
		1415	goto err_driver;
		1416	return 0;
		1417
		1418	err_driver:
		1419	mdev_unregister_driver(&vfio_ap_matrix_driver);
		1420	return ret;
1440	}	1421	}
1441		1422
1442	void vfio_ap_mdev_unregister(void)	1423	void vfio_ap_mdev_unregister(void)
1443	{	1424	{
1444	mdev_unregister_device(&matrix_dev->device);	1425	mdev_unregister_device(&matrix_dev->device);
		1426	mdev_unregister_driver(&vfio_ap_matrix_driver);
1445	}	1427	}

drivers/s390/crypto/vfio_ap_private.h+3−3

 #include <linux/delay.h>
 #include <linux/mutex.h>
 #include <linux/kvm_host.h>
+#include <linux/vfio.h>
 #include "ap_bus.h"
  * @kvm:	the struct holding guest's state
  */
 struct ap_matrix_mdev {
+	struct vfio_device vdev;
 	struct list_head node;
 	struct ap_matrix matrix;
 	struct notifier_block group_notifier;
 	struct notifier_block iommu_notifier;
-	bool kvm_busy;
-	wait_queue_head_t wait_for_kvm;
 	struct kvm *kvm;
-	struct kvm_s390_module_hook pqap_hook;
+	crypto_hook pqap_hook;
 	struct mdev_device *mdev;
 };

drivers/vfio/Kconfig+15−14

 # SPDX-License-Identifier: GPL-2.0-only
+menuconfig VFIO
+	tristate "VFIO Non-Privileged userspace driver framework"
+	select IOMMU_API
+	select VFIO_IOMMU_TYPE1 if MMU && (X86 || S390 || ARM || ARM64)
+	help
+	  VFIO provides a framework for secure userspace device drivers.
+	  See Documentation/driver-api/vfio.rst for more details.
+	  If you don't know what to do here, say N.
+if VFIO
 config VFIO_IOMMU_TYPE1
 	tristate
-	depends on VFIO
 	default n
 config VFIO_IOMMU_SPAPR_TCE
 	tristate
-	depends on VFIO && SPAPR_TCE_IOMMU
+	depends on SPAPR_TCE_IOMMU
 	default VFIO
 config VFIO_SPAPR_EEH
 config VFIO_VIRQFD
 	tristate
-	depends on VFIO && EVENTFD
+	select EVENTFD
 	default n
-menuconfig VFIO
-	tristate "VFIO Non-Privileged userspace driver framework"
-	select IOMMU_API
-	select VFIO_IOMMU_TYPE1 if MMU && (X86 || S390 || ARM || ARM64)
-	help
-	  VFIO provides a framework for secure userspace device drivers.
-	  See Documentation/driver-api/vfio.rst for more details.
-	  If you don't know what to do here, say N.
 config VFIO_NOIOMMU
 	bool "VFIO No-IOMMU support"
-	depends on VFIO
 	help
 	  VFIO is built on the ability to isolate devices using the IOMMU.
 	  Only with an IOMMU can userspace access to DMA capable devices be
 source "drivers/vfio/platform/Kconfig"
 source "drivers/vfio/mdev/Kconfig"
 source "drivers/vfio/fsl-mc/Kconfig"
+endif
 source "virt/lib/Kconfig"

drivers/vfio/fsl-mc/Kconfig+2−1

@@ -1,6 +1,7 @@
1	config VFIO_FSL_MC	1	config VFIO_FSL_MC
2	tristate "VFIO support for QorIQ DPAA2 fsl-mc bus devices"	2	tristate "VFIO support for QorIQ DPAA2 fsl-mc bus devices"
3	depends on VFIO && FSL_MC_BUS && EVENTFD	3	depends on FSL_MC_BUS
		4	select EVENTFD
4	help	5	help
5	Driver to enable support for the VFIO QorIQ DPAA2 fsl-mc	6	Driver to enable support for the VFIO QorIQ DPAA2 fsl-mc
6	(Management Complex) devices. This is required to passthrough	7	(Management Complex) devices. This is required to passthrough

drivers/vfio/mdev/Kconfig+0−1

@@ -2,7 +2,6 @@ · 1 unmodified line
2		2
3	config VFIO_MDEV	3	config VFIO_MDEV
4	tristate "Mediated device driver framework"	4	tristate "Mediated device driver framework"
5	depends on VFIO
6	default n	5	default n
7	help	6	help
8	Provides a framework to virtualize devices.	7	Provides a framework to virtualize devices.

drivers/vfio/pci/Kconfig+22−18

 # SPDX-License-Identifier: GPL-2.0-only
-config VFIO_PCI
+if PCI && MMU
-	tristate "VFIO support for PCI devices"
+config VFIO_PCI_CORE
-	depends on VFIO && PCI && EVENTFD
+	tristate
-	depends on MMU
 	select VFIO_VIRQFD
 	select IRQ_BYPASS_MANAGER
+config VFIO_PCI_MMAP
+	def_bool y if !S390
+config VFIO_PCI_INTX
+	def_bool y if !S390
+config VFIO_PCI
+	tristate "Generic VFIO support for any PCI device"
+	select VFIO_PCI_CORE
 	help
-	  Support for the PCI VFIO bus driver.  This is required to make
+	  Support for the generic PCI VFIO bus driver which can connect any
-	  use of PCI drivers using the VFIO framework.
+	  PCI device to the VFIO framework.
 	  If you don't know what to do here, say N.
+if VFIO_PCI
 config VFIO_PCI_VGA
-	bool "VFIO PCI support for VGA devices"
+	bool "Generic VFIO PCI support for VGA devices"
-	depends on VFIO_PCI && X86 && VGA_ARB
+	depends on X86 && VGA_ARB
 	help
 	  Support for VGA extension to VFIO PCI.  This exposes an additional
 	  region on VGA devices for accessing legacy VGA addresses used by
 	  If you don't know what to do here, say N.
-config VFIO_PCI_MMAP
-	depends on VFIO_PCI
-	def_bool y if !S390
-config VFIO_PCI_INTX
-	depends on VFIO_PCI
-	def_bool y if !S390
 config VFIO_PCI_IGD
-	bool "VFIO PCI extensions for Intel graphics (GVT-d)"
+	bool "Generic VFIO PCI extensions for Intel graphics (GVT-d)"
-	depends on VFIO_PCI && X86
+	depends on X86
 	default y
 	help
 	  Support for Intel IGD specific extensions to enable direct
 	  and LPC bridge config space.
 	  To enable Intel IGD assignment through vfio-pci, say Y.
+endif
+endif

drivers/vfio/pci/Makefile+5−3

@@ -1,7 +1,9 @@
1	# SPDX-License-Identifier: GPL-2.0-only	1	# SPDX-License-Identifier: GPL-2.0-only
2		2
3	vfio-pci-y := vfio_pci.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o	3	vfio-pci-core-y := vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o
4	vfio-pci-$(CONFIG_VFIO_PCI_IGD) += vfio_pci_igd.o	4	vfio-pci-core-$(CONFIG_S390) += vfio_pci_zdev.o
5	vfio-pci-$(CONFIG_S390) += vfio_pci_zdev.o	5	obj-$(CONFIG_VFIO_PCI_CORE) += vfio-pci-core.o
6		6
		7	vfio-pci-y := vfio_pci.o
		8	vfio-pci-$(CONFIG_VFIO_PCI_IGD) += vfio_pci_igd.o
7	obj-$(CONFIG_VFIO_PCI) += vfio-pci.o	9	obj-$(CONFIG_VFIO_PCI) += vfio-pci.o

drivers/vfio/pci/vfio_pci.c+107−2155

@@ -1,5 +1,7 @@
1	// SPDX-License-Identifier: GPL-2.0-only	1	// SPDX-License-Identifier: GPL-2.0-only
2	/*	2	/*
		3	* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved
		4	*
3	* Copyright (C) 2012 Red Hat, Inc. All rights reserved.	5	* Copyright (C) 2012 Red Hat, Inc. All rights reserved.
4	* Author: Alex Williamson <alex.williamson@redhat.com>	6	* Author: Alex Williamson <alex.williamson@redhat.com>
5	*	7	*
@@ -18,19 +20,13 @@ · 12 unmodified lines
18	#include <linux/module.h>	20	#include <linux/module.h>
19	#include <linux/mutex.h>	21	#include <linux/mutex.h>
20	#include <linux/notifier.h>	22	#include <linux/notifier.h>
21	#include <linux/pci.h>
22	#include <linux/pm_runtime.h>	23	#include <linux/pm_runtime.h>
23	#include <linux/slab.h>	24	#include <linux/slab.h>
24	#include <linux/types.h>	25	#include <linux/types.h>
25	#include <linux/uaccess.h>	26	#include <linux/uaccess.h>
26	#include <linux/vfio.h>
27	#include <linux/vgaarb.h>
28	#include <linux/nospec.h>
29	#include <linux/sched/mm.h>
30		27
31	#include "vfio_pci_private.h"	28	#include <linux/vfio_pci_core.h>
32		29
33	#define DRIVER_VERSION "0.2"
34	#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"	30	#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
35	#define DRIVER_DESC "VFIO PCI - User Level meta-driver"	31	#define DRIVER_DESC "VFIO PCI - User Level meta-driver"
36		32
@@ -64,15 +60,6 @@ static bool disable_denylist; · 27 unmodified lines
64	module_param(disable_denylist, bool, 0444);	60	module_param(disable_denylist, bool, 0444);
65	MODULE_PARM_DESC(disable_denylist, "Disable use of device denylist. Disabling the denylist allows binding to devices with known errata that may lead to exploitable stability or security issues when accessed by untrusted users.");	61	MODULE_PARM_DESC(disable_denylist, "Disable use of device denylist. Disabling the denylist allows binding to devices with known errata that may lead to exploitable stability or security issues when accessed by untrusted users.");
66		62
67	static inline bool vfio_vga_disabled(void)
68	{
69	#ifdef CONFIG_VFIO_PCI_VGA
70	return disable_vga;
71	#else
72	return true;
73	#endif
74	}
75
76	static bool vfio_pci_dev_in_denylist(struct pci_dev *pdev)	63	static bool vfio_pci_dev_in_denylist(struct pci_dev *pdev)
77	{	64	{
78	switch (pdev->vendor) {	65	switch (pdev->vendor) {
@@ -111,2210 +98,175 @@ static bool vfio_pci_is_denylisted(struct pci_dev *pdev) · 32 unmodified lines
111	return true;	98	return true;
112	}	99	}
113		100
114	/*	101	static int vfio_pci_open_device(struct vfio_device *core_vdev)
115	* Our VGA arbiter participation is limited since we don't know anything
116	* about the device itself. However, if the device is the only VGA device
117	* downstream of a bridge and VFIO VGA support is disabled, then we can
118	* safely return legacy VGA IO and memory as not decoded since the user
119	* has no way to get to it and routing can be disabled externally at the
120	* bridge.
121	*/
122	static unsigned int vfio_pci_set_vga_decode(void *opaque, bool single_vga)
123	{
124	struct vfio_pci_device *vdev = opaque;
125	struct pci_dev tmp = NULL, pdev = vdev->pdev;
126	unsigned char max_busnr;
127	unsigned int decodes;
128
129	if (single_vga \|\| !vfio_vga_disabled() \|\| pci_is_root_bus(pdev->bus))
130	return VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM \|
131	VGA_RSRC_LEGACY_IO \| VGA_RSRC_LEGACY_MEM;
132
133	max_busnr = pci_bus_max_busnr(pdev->bus);
134	decodes = VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM;
135
136	while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) {
137	if (tmp == pdev \|\|
138	pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) \|\|
139	pci_is_root_bus(tmp->bus))
140	continue;
141
142	if (tmp->bus->number >= pdev->bus->number &&
143	tmp->bus->number <= max_busnr) {
144	pci_dev_put(tmp);
145	decodes \|= VGA_RSRC_LEGACY_IO \| VGA_RSRC_LEGACY_MEM;
146	break;
147	}
148	}
149
150	return decodes;
151	}
152
153	static inline bool vfio_pci_is_vga(struct pci_dev *pdev)
154	{
155	return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;
156	}
157
158	static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev)
159	{
160	struct resource *res;
161	int i;
162	struct vfio_pci_dummy_resource *dummy_res;
163
164	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
165	int bar = i + PCI_STD_RESOURCES;
166
167	res = &vdev->pdev->resource[bar];
168
169	if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP))
170	goto no_mmap;
171
172	if (!(res->flags & IORESOURCE_MEM))
173	goto no_mmap;
174
175	/*
176	* The PCI core shouldn't set up a resource with a
177	* type but zero size. But there may be bugs that
178	* cause us to do that.
179	*/
180	if (!resource_size(res))
181	goto no_mmap;
182
183	if (resource_size(res) >= PAGE_SIZE) {
184	vdev->bar_mmap_supported[bar] = true;
185	continue;
186	}
187
188	if (!(res->start & ~PAGE_MASK)) {
189	/*
190	* Add a dummy resource to reserve the remainder
191	* of the exclusive page in case that hot-add
192	* device's bar is assigned into it.
193	*/
194	dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL);
195	if (dummy_res == NULL)
196	goto no_mmap;
197
198	dummy_res->resource.name = "vfio sub-page reserved";
199	dummy_res->resource.start = res->end + 1;
200	dummy_res->resource.end = res->start + PAGE_SIZE - 1;
201	dummy_res->resource.flags = res->flags;
202	if (request_resource(res->parent,
203	&dummy_res->resource)) {
204	kfree(dummy_res);
205	goto no_mmap;
206	}
207	dummy_res->index = bar;
208	list_add(&dummy_res->res_next,
209	&vdev->dummy_resources_list);
210	vdev->bar_mmap_supported[bar] = true;
211	continue;
212	}
213	/*
214	* Here we don't handle the case when the BAR is not page
215	* aligned because we can't expect the BAR will be
216	* assigned into the same location in a page in guest
217	* when we passthrough the BAR. And it's hard to access
218	* this BAR in userspace because we have no way to get
219	* the BAR's location in a page.
220	*/
221	no_mmap:
222	vdev->bar_mmap_supported[bar] = false;
223	}
224	}
225
226	struct vfio_pci_group_info;
227	static bool vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set);
228	static void vfio_pci_disable(struct vfio_pci_device *vdev);
229	static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
230	struct vfio_pci_group_info *groups);
231
232	/*
233	* INTx masking requires the ability to disable INTx signaling via PCI_COMMAND
234	* _and_ the ability detect when the device is asserting INTx via PCI_STATUS.
235	* If a device implements the former but not the latter we would typically
236	* expect broken_intx_masking be set and require an exclusive interrupt.
237	* However since we do have control of the device's ability to assert INTx,
238	* we can instead pretend that the device does not implement INTx, virtualizing
239	* the pin register to report zero and maintaining DisINTx set on the host.
240	*/
241	static bool vfio_pci_nointx(struct pci_dev *pdev)
242	{
243	switch (pdev->vendor) {
244	case PCI_VENDOR_ID_INTEL:
245	switch (pdev->device) {
246	/* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */
247	case 0x1572:
248	case 0x1574:
249	case 0x1580 ... 0x1581:
250	case 0x1583 ... 0x158b:
251	case 0x37d0 ... 0x37d2:
252	/* X550 */
253	case 0x1563:
254	return true;
255	default:
256	return false;
257	}
258	}
259
260	return false;
261	}
262
263	static void vfio_pci_probe_power_state(struct vfio_pci_device *vdev)
264	{
265	struct pci_dev *pdev = vdev->pdev;
266	u16 pmcsr;
267
268	if (!pdev->pm_cap)
269	return;
270
271	pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
272
273	vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
274	}
275
276	/*
277	* pci_set_power_state() wrapper handling devices which perform a soft reset on
278	* D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev,
279	* restore when returned to D0. Saved separately from pci_saved_state for use
280	* by PM capability emulation and separately from pci_dev internal saved state
281	* to avoid it being overwritten and consumed around other resets.
282	*/
283	int vfio_pci_set_power_state(struct vfio_pci_device *vdev, pci_power_t state)
284	{
285	struct pci_dev *pdev = vdev->pdev;
286	bool needs_restore = false, needs_save = false;
287	int ret;
288
289	if (vdev->needs_pm_restore) {
290	if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) {
291	pci_save_state(pdev);
292	needs_save = true;
293	}
294
295	if (pdev->current_state >= PCI_D3hot && state <= PCI_D0)
296	needs_restore = true;
297	}
298
299	ret = pci_set_power_state(pdev, state);
300
301	if (!ret) {
302	/* D3 might be unsupported via quirk, skip unless in D3 */
303	if (needs_save && pdev->current_state >= PCI_D3hot) {
304	vdev->pm_save = pci_store_saved_state(pdev);
305	} else if (needs_restore) {
306	pci_load_and_free_saved_state(pdev, &vdev->pm_save);
307	pci_restore_state(pdev);
308	}
309	}
310
311	return ret;
312	}
313
314	static int vfio_pci_enable(struct vfio_pci_device *vdev)
315	{	102	{
		103	struct vfio_pci_core_device *vdev =
		104	container_of(core_vdev, struct vfio_pci_core_device, vdev);
316	struct pci_dev *pdev = vdev->pdev;	105	struct pci_dev *pdev = vdev->pdev;
317	int ret;	106	int ret;
318	u16 cmd;
319	u8 msix_pos;
320
321	vfio_pci_set_power_state(vdev, PCI_D0);
322		107
323	/* Don't allow our initial saved state to include busmaster */	108	ret = vfio_pci_core_enable(vdev);
324	pci_clear_master(pdev);
325
326	ret = pci_enable_device(pdev);
327	if (ret)	109	if (ret)
328	return ret;	110	return ret;
329		111
330	/* If reset fails because of the device lock, fail this path entirely */
331	ret = pci_try_reset_function(pdev);
332	if (ret == -EAGAIN) {
333	pci_disable_device(pdev);
334	return ret;
335	}
336
337	vdev->reset_works = !ret;
338	pci_save_state(pdev);
339	vdev->pci_saved_state = pci_store_saved_state(pdev);
340	if (!vdev->pci_saved_state)
341	pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__);
342
343	if (likely(!nointxmask)) {
344	if (vfio_pci_nointx(pdev)) {
345	pci_info(pdev, "Masking broken INTx support\n");
346	vdev->nointx = true;
347	pci_intx(pdev, 0);
348	} else
349	vdev->pci_2_3 = pci_intx_mask_supported(pdev);
350	}
351
352	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
353	if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
354	cmd &= ~PCI_COMMAND_INTX_DISABLE;
355	pci_write_config_word(pdev, PCI_COMMAND, cmd);
356	}
357
358	ret = vfio_config_init(vdev);
359	if (ret) {
360	kfree(vdev->pci_saved_state);
361	vdev->pci_saved_state = NULL;
362	pci_disable_device(pdev);
363	return ret;
364	}
365
366	msix_pos = pdev->msix_cap;
367	if (msix_pos) {
368	u16 flags;
369	u32 table;
370
371	pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);
372	pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);
373
374	vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
375	vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
376	vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
377	} else
378	vdev->msix_bar = 0xFF;
379
380	if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))
381	vdev->has_vga = true;
382
383	if (vfio_pci_is_vga(pdev) &&	112	if (vfio_pci_is_vga(pdev) &&
384	pdev->vendor == PCI_VENDOR_ID_INTEL &&	113	pdev->vendor == PCI_VENDOR_ID_INTEL &&
385	IS_ENABLED(CONFIG_VFIO_PCI_IGD)) {	114	IS_ENABLED(CONFIG_VFIO_PCI_IGD)) {
386	ret = vfio_pci_igd_init(vdev);	115	ret = vfio_pci_igd_init(vdev);
387	if (ret && ret != -ENODEV) {	116	if (ret && ret != -ENODEV) {
388	pci_warn(pdev, "Failed to setup Intel IGD regions\n");	117	pci_warn(pdev, "Failed to setup Intel IGD regions\n");
389	goto disable_exit;	118	vfio_pci_core_disable(vdev);
		119	return ret;
390	}	120	}
391	}	121	}
392		122
393	vfio_pci_probe_mmaps(vdev);	123	vfio_pci_core_finish_enable(vdev);
394		124
395	return 0;	125	return 0;
396
397	disable_exit:
398	vfio_pci_disable(vdev);
399	return ret;
400	}
401
402	static void vfio_pci_disable(struct vfio_pci_device *vdev)
403	{
404	struct pci_dev *pdev = vdev->pdev;
405	struct vfio_pci_dummy_resource dummy_res, tmp;
406	struct vfio_pci_ioeventfd ioeventfd, ioeventfd_tmp;
407	int i, bar;
408
409	/* For needs_reset */
410	lockdep_assert_held(&vdev->vdev.dev_set->lock);
411
412	/* Stop the device from further DMA */
413	pci_clear_master(pdev);
414
415	vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE \|
416	VFIO_IRQ_SET_ACTION_TRIGGER,
417	vdev->irq_type, 0, 0, NULL);
418
419	/* Device closed, don't need mutex here */
420	list_for_each_entry_safe(ioeventfd, ioeventfd_tmp,
421	&vdev->ioeventfds_list, next) {
422	vfio_virqfd_disable(&ioeventfd->virqfd);
423	list_del(&ioeventfd->next);
424	kfree(ioeventfd);
425	}
426	vdev->ioeventfds_nr = 0;
427
428	vdev->virq_disabled = false;
429
430	for (i = 0; i < vdev->num_regions; i++)
431	vdev->region[i].ops->release(vdev, &vdev->region[i]);
432
433	vdev->num_regions = 0;
434	kfree(vdev->region);
435	vdev->region = NULL; /* don't krealloc a freed pointer */
436
437	vfio_config_free(vdev);
438
439	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
440	bar = i + PCI_STD_RESOURCES;
441	if (!vdev->barmap[bar])
442	continue;
443	pci_iounmap(pdev, vdev->barmap[bar]);
444	pci_release_selected_regions(pdev, 1 << bar);
445	vdev->barmap[bar] = NULL;
446	}
447
448	list_for_each_entry_safe(dummy_res, tmp,
449	&vdev->dummy_resources_list, res_next) {
450	list_del(&dummy_res->res_next);
451	release_resource(&dummy_res->resource);
452	kfree(dummy_res);
453	}
454
455	vdev->needs_reset = true;
456
457	/*
458	* If we have saved state, restore it. If we can reset the device,
459	* even better. Resetting with current state seems better than
460	* nothing, but saving and restoring current state without reset
461	* is just busy work.
462	*/
463	if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
464	pci_info(pdev, "%s: Couldn't reload saved state\n", __func__);
465
466	if (!vdev->reset_works)
467	goto out;
468
469	pci_save_state(pdev);
470	}
471
472	/*
473	* Disable INTx and MSI, presumably to avoid spurious interrupts
474	* during reset. Stolen from pci_reset_function()
475	*/
476	pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
477
478	/*
479	* Try to get the locks ourselves to prevent a deadlock. The
480	* success of this is dependent on being able to lock the device,
481	* which is not always possible.
482	* We can not use the "try" reset interface here, which will
483	* overwrite the previously restored configuration information.
484	*/
485	if (vdev->reset_works && pci_dev_trylock(pdev)) {
486	if (!__pci_reset_function_locked(pdev))
487	vdev->needs_reset = false;
488	pci_dev_unlock(pdev);
489	}
490
491	pci_restore_state(pdev);
492	out:
493	pci_disable_device(pdev);
494
495	if (!vfio_pci_dev_set_try_reset(vdev->vdev.dev_set) && !disable_idle_d3)
496	vfio_pci_set_power_state(vdev, PCI_D3hot);
497	}
498
499	static struct pci_driver vfio_pci_driver;
500
501	static struct vfio_pci_device get_pf_vdev(struct vfio_pci_device vdev)
502	{
503	struct pci_dev *physfn = pci_physfn(vdev->pdev);
504	struct vfio_device *pf_dev;
505
506	if (!vdev->pdev->is_virtfn)
507	return NULL;
508
509	pf_dev = vfio_device_get_from_dev(&physfn->dev);
510	if (!pf_dev)
511	return NULL;
512
513	if (pci_dev_driver(physfn) != &vfio_pci_driver) {
514	vfio_device_put(pf_dev);
515	return NULL;
516	}
517
518	return container_of(pf_dev, struct vfio_pci_device, vdev);
519	}	126	}
520		127
521	static void vfio_pci_vf_token_user_add(struct vfio_pci_device *vdev, int val)	128	static const struct vfio_device_ops vfio_pci_ops = {
522	{	129	.name = "vfio-pci",
523	struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev);	130	.open_device = vfio_pci_open_device,
524		131	.close_device = vfio_pci_core_close_device,
525	if (!pf_vdev)	132	.ioctl = vfio_pci_core_ioctl,
526	return;	133	.read = vfio_pci_core_read,
527		134	.write = vfio_pci_core_write,
528	mutex_lock(&pf_vdev->vf_token->lock);	135	.mmap = vfio_pci_core_mmap,
529	pf_vdev->vf_token->users += val;	136	.request = vfio_pci_core_request,
530	WARN_ON(pf_vdev->vf_token->users < 0);	137	.match = vfio_pci_core_match,
531	mutex_unlock(&pf_vdev->vf_token->lock);	138	};
532
533	vfio_device_put(&pf_vdev->vdev);
534	}
535		139
536	static void vfio_pci_close_device(struct vfio_device *core_vdev)	140	static int vfio_pci_probe(struct pci_dev pdev, const struct pci_device_id id)
537	{	141	{
538	struct vfio_pci_device *vdev =	142	struct vfio_pci_core_device *vdev;
539	container_of(core_vdev, struct vfio_pci_device, vdev);	143	int ret;
540
541	vfio_pci_vf_token_user_add(vdev, -1);
542	vfio_spapr_pci_eeh_release(vdev->pdev);
543	vfio_pci_disable(vdev);
544		144
545	mutex_lock(&vdev->igate);	145	if (vfio_pci_is_denylisted(pdev))
546	if (vdev->err_trigger) {	146	return -EINVAL;
547	eventfd_ctx_put(vdev->err_trigger);
548	vdev->err_trigger = NULL;
549	}
550	if (vdev->req_trigger) {
551	eventfd_ctx_put(vdev->req_trigger);
552	vdev->req_trigger = NULL;
553	}
554	mutex_unlock(&vdev->igate);
555	}
556		147
557	static int vfio_pci_open_device(struct vfio_device *core_vdev)	148	vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
558	{	149	if (!vdev)
559	struct vfio_pci_device *vdev =	150	return -ENOMEM;
560	container_of(core_vdev, struct vfio_pci_device, vdev);	151	vfio_pci_core_init_device(vdev, pdev, &vfio_pci_ops);
561	int ret = 0;
562		152
563	ret = vfio_pci_enable(vdev);	153	ret = vfio_pci_core_register_device(vdev);
564	if (ret)	154	if (ret)
565	return ret;	155	goto out_free;
566		156	dev_set_drvdata(&pdev->dev, vdev);
567	vfio_spapr_pci_eeh_open(vdev->pdev);
568	vfio_pci_vf_token_user_add(vdev, 1);
569	return 0;	157	return 0;
570	}
571
572	static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)
573	{
574	if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {
575	u8 pin;
576
577	if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) \|\|
578	vdev->nointx \|\| vdev->pdev->is_virtfn)
579	return 0;
580
581	pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);
582		158
583	return pin ? 1 : 0;	159	out_free:
584	} else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {	160	vfio_pci_core_uninit_device(vdev);
585	u8 pos;	161	kfree(vdev);
586	u16 flags;	162	return ret;
587
588	pos = vdev->pdev->msi_cap;
589	if (pos) {
590	pci_read_config_word(vdev->pdev,
591	pos + PCI_MSI_FLAGS, &flags);
592	return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);
593	}
594	} else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {
595	u8 pos;
596	u16 flags;
597
598	pos = vdev->pdev->msix_cap;
599	if (pos) {
600	pci_read_config_word(vdev->pdev,
601	pos + PCI_MSIX_FLAGS, &flags);
602
603	return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;
604	}
605	} else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) {
606	if (pci_is_pcie(vdev->pdev))
607	return 1;
608	} else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) {
609	return 1;
610	}
611
612	return 0;
613	}	163	}
614		164
615	static int vfio_pci_count_devs(struct pci_dev pdev, void data)	165	static void vfio_pci_remove(struct pci_dev *pdev)
616	{	166	{
617	((int )data)++;	167	struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
618	return 0;
619	}
620		168
621	struct vfio_pci_fill_info {	169	vfio_pci_core_unregister_device(vdev);
622	int max;	170	vfio_pci_core_uninit_device(vdev);
623	int cur;	171	kfree(vdev);
624	struct vfio_pci_dependent_device *devices;	172	}
625	};
626		173
627	static int vfio_pci_fill_devs(struct pci_dev pdev, void data)	174	static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
628	{	175	{
629	struct vfio_pci_fill_info *fill = data;	176	if (!enable_sriov)
630	struct iommu_group *iommu_group;	177	return -ENOENT;
631
632	if (fill->cur == fill->max)
633	return -EAGAIN; /* Something changed, try again */
634
635	iommu_group = iommu_group_get(&pdev->dev);
636	if (!iommu_group)
637	return -EPERM; /* Cannot reset non-isolated devices */
638		178
639	fill->devices[fill->cur].group_id = iommu_group_id(iommu_group);	179	return vfio_pci_core_sriov_configure(pdev, nr_virtfn);
640	fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus);
641	fill->devices[fill->cur].bus = pdev->bus->number;
642	fill->devices[fill->cur].devfn = pdev->devfn;
643	fill->cur++;
644	iommu_group_put(iommu_group);
645	return 0;
646	}	180	}
647		181
648	struct vfio_pci_group_info {	182	static const struct pci_device_id vfio_pci_table[] = {
649	int count;	183	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_ANY_ID, PCI_ANY_ID) }, /* match all by default */
650	struct vfio_group **groups;	184	{}
651	};	185	};
652		186
653	static bool vfio_pci_dev_below_slot(struct pci_dev pdev, struct pci_slot slot)	187	MODULE_DEVICE_TABLE(pci, vfio_pci_table);
654	{
655	for (; pdev; pdev = pdev->bus->self)
656	if (pdev->bus == slot->bus)
657	return (pdev->slot == slot);
658	return false;
659	}
660		188
661	struct vfio_pci_walk_info {	189	static struct pci_driver vfio_pci_driver = {
662	int (fn)(struct pci_dev , void *data);	190	.name = "vfio-pci",
663	void *data;	191	.id_table = vfio_pci_table,
664	struct pci_dev *pdev;	192	.probe = vfio_pci_probe,
665	bool slot;	193	.remove = vfio_pci_remove,
666	int ret;	194	.sriov_configure = vfio_pci_sriov_configure,
		195	.err_handler = &vfio_pci_core_err_handlers,
667	};	196	};
668		197
669	static int vfio_pci_walk_wrapper(struct pci_dev pdev, void data)	198	static void __init vfio_pci_fill_ids(void)
670	{	199	{
671	struct vfio_pci_walk_info *walk = data;	200	char p, id;
		201	int rc;
672		202
673	if (!walk->slot \|\| vfio_pci_dev_below_slot(pdev, walk->pdev->slot))	203	/* no ids passed actually */
674	walk->ret = walk->fn(pdev, walk->data);	204	if (ids[0] == '\0')
		205	return;
675		206
676	return walk->ret;	207	/* add ids specified in the module parameter */
677	}	208	p = ids;
		209	while ((id = strsep(&p, ","))) {
		210	unsigned int vendor, device, subvendor = PCI_ANY_ID,
		211	subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
		212	int fields;
678		213
679	static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,	214	if (!strlen(id))
680	int (fn)(struct pci_dev ,	215	continue;
681	void data), void data,	216
682	bool slot)	217	fields = sscanf(id, "%x:%x:%x:%x:%x:%x",
683	{	218	&vendor, &device, &subvendor, &subdevice,
684	struct vfio_pci_walk_info walk = {	219	&class, &class_mask);
685	.fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,
686	};
687		220
688	pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);	221	if (fields < 2) {
		222	pr_warn("invalid id string \"%s\"\n", id);
		223	continue;
		224	}
689		225
690	return walk.ret;	226	rc = pci_add_dynid(&vfio_pci_driver, vendor, device,
		227	subvendor, subdevice, class, class_mask, 0);
		228	if (rc)
		229	pr_warn("failed to add dynamic id [%04x:%04x[%04x:%04x]] class %#08x/%08x (%d)\n",
		230	vendor, device, subvendor, subdevice,
		231	class, class_mask, rc);
		232	else
		233	pr_info("add [%04x:%04x[%04x:%04x]] class %#08x/%08x\n",
		234	vendor, device, subvendor, subdevice,
		235	class, class_mask);
		236	}
691	}	237	}
692		238
693	static int msix_mmappable_cap(struct vfio_pci_device *vdev,	239	static int __init vfio_pci_init(void)
694	struct vfio_info_cap *caps)
695	{	240	{
696	struct vfio_info_cap_header header = {	241	int ret;
697	.id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE,	242	bool is_disable_vga = true;
698	.version = 1
699	};
700		243
701	return vfio_info_add_capability(caps, &header, sizeof(header));	244	#ifdef CONFIG_VFIO_PCI_VGA
702	}	245	is_disable_vga = disable_vga;
		246	#endif
703		247
704	int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,	248	vfio_pci_core_set_params(nointxmask, is_disable_vga, disable_idle_d3);
705	unsigned int type, unsigned int subtype,
706	const struct vfio_pci_regops *ops,
707	size_t size, u32 flags, void *data)
708	{
709	struct vfio_pci_region *region;
710		249
711	region = krealloc(vdev->region,	250	/* Register and scan for devices */
712	(vdev->num_regions + 1) * sizeof(*region),	251	ret = pci_register_driver(&vfio_pci_driver);
713	GFP_KERNEL);	252	if (ret)
714	if (!region)	253	return ret;
715	return -ENOMEM;
716		254
717	vdev->region = region;	255	vfio_pci_fill_ids();
718	vdev->region[vdev->num_regions].type = type;
719	vdev->region[vdev->num_regions].subtype = subtype;
720	vdev->region[vdev->num_regions].ops = ops;
721	vdev->region[vdev->num_regions].size = size;
722	vdev->region[vdev->num_regions].flags = flags;
723	vdev->region[vdev->num_regions].data = data;
724		256
725	vdev->num_regions++;	257	if (disable_denylist)
		258	pr_warn("device denylist disabled.\n");
726		259
727	return 0;	260	return 0;
728	}	261	}
		262	module_init(vfio_pci_init);
729		263
730	static long vfio_pci_ioctl(struct vfio_device *core_vdev,	264	static void __exit vfio_pci_cleanup(void)
731	unsigned int cmd, unsigned long arg)
732	{
733	struct vfio_pci_device *vdev =
734	container_of(core_vdev, struct vfio_pci_device, vdev);
735	unsigned long minsz;
736
737	if (cmd == VFIO_DEVICE_GET_INFO) {
738	struct vfio_device_info info;
739	struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
740	unsigned long capsz;
741	int ret;
742
743	minsz = offsetofend(struct vfio_device_info, num_irqs);
744
745	/* For backward compatibility, cannot require this */
746	capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset);
747
748	if (copy_from_user(&info, (void __user *)arg, minsz))
749	return -EFAULT;
750
751	if (info.argsz < minsz)
752	return -EINVAL;
753
754	if (info.argsz >= capsz) {
755	minsz = capsz;
756	info.cap_offset = 0;
757	}
758
759	info.flags = VFIO_DEVICE_FLAGS_PCI;
760
761	if (vdev->reset_works)
762	info.flags \|= VFIO_DEVICE_FLAGS_RESET;
763
764	info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
765	info.num_irqs = VFIO_PCI_NUM_IRQS;
766
767	ret = vfio_pci_info_zdev_add_caps(vdev, &caps);
768	if (ret && ret != -ENODEV) {
769	pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n");
770	return ret;
771	}
772
773	if (caps.size) {
774	info.flags \|= VFIO_DEVICE_FLAGS_CAPS;
775	if (info.argsz < sizeof(info) + caps.size) {
776	info.argsz = sizeof(info) + caps.size;
777	} else {
778	vfio_info_cap_shift(&caps, sizeof(info));
779	if (copy_to_user((void __user *)arg +
780	sizeof(info), caps.buf,
781	caps.size)) {
782	kfree(caps.buf);
783	return -EFAULT;
784	}
785	info.cap_offset = sizeof(info);
786	}
787
788	kfree(caps.buf);
789	}
790
791	return copy_to_user((void __user *)arg, &info, minsz) ?
792	-EFAULT : 0;
793
794	} else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
795	struct pci_dev *pdev = vdev->pdev;
796	struct vfio_region_info info;
797	struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
798	int i, ret;
799
800	minsz = offsetofend(struct vfio_region_info, offset);
801
802	if (copy_from_user(&info, (void __user *)arg, minsz))
803	return -EFAULT;
804
805	if (info.argsz < minsz)
806	return -EINVAL;
807
808	switch (info.index) {
809	case VFIO_PCI_CONFIG_REGION_INDEX:
810	info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
811	info.size = pdev->cfg_size;
812	info.flags = VFIO_REGION_INFO_FLAG_READ \|
813	VFIO_REGION_INFO_FLAG_WRITE;
814	break;
815	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
816	info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
817	info.size = pci_resource_len(pdev, info.index);
818	if (!info.size) {
819	info.flags = 0;
820	break;
821	}
822
823	info.flags = VFIO_REGION_INFO_FLAG_READ \|
824	VFIO_REGION_INFO_FLAG_WRITE;
825	if (vdev->bar_mmap_supported[info.index]) {
826	info.flags \|= VFIO_REGION_INFO_FLAG_MMAP;
827	if (info.index == vdev->msix_bar) {
828	ret = msix_mmappable_cap(vdev, &caps);
829	if (ret)
830	return ret;
831	}
832	}
833
834	break;
835	case VFIO_PCI_ROM_REGION_INDEX:
836	{
837	void __iomem *io;
838	size_t size;
839	u16 cmd;
840
841	info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
842	info.flags = 0;
843
844	/* Report the BAR size, not the ROM size */
845	info.size = pci_resource_len(pdev, info.index);
846	if (!info.size) {
847	/* Shadow ROMs appear as PCI option ROMs */
848	if (pdev->resource[PCI_ROM_RESOURCE].flags &
849	IORESOURCE_ROM_SHADOW)
850	info.size = 0x20000;
851	else
852	break;
853	}
854
855	/*
856	* Is it really there? Enable memory decode for
857	* implicit access in pci_map_rom().
858	*/
859	cmd = vfio_pci_memory_lock_and_enable(vdev);
860	io = pci_map_rom(pdev, &size);
861	if (io) {
862	info.flags = VFIO_REGION_INFO_FLAG_READ;
863	pci_unmap_rom(pdev, io);
864	} else {
865	info.size = 0;
866	}
867	vfio_pci_memory_unlock_and_restore(vdev, cmd);
868
869	break;
870	}
871	case VFIO_PCI_VGA_REGION_INDEX:
872	if (!vdev->has_vga)
873	return -EINVAL;
874
875	info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
876	info.size = 0xc0000;
877	info.flags = VFIO_REGION_INFO_FLAG_READ \|
878	VFIO_REGION_INFO_FLAG_WRITE;
879
880	break;
881	default:
882	{
883	struct vfio_region_info_cap_type cap_type = {
884	.header.id = VFIO_REGION_INFO_CAP_TYPE,
885	.header.version = 1 };
886
887	if (info.index >=
888	VFIO_PCI_NUM_REGIONS + vdev->num_regions)
889	return -EINVAL;
890	info.index = array_index_nospec(info.index,
891	VFIO_PCI_NUM_REGIONS +
892	vdev->num_regions);
893
894	i = info.index - VFIO_PCI_NUM_REGIONS;
895
896	info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
897	info.size = vdev->region[i].size;
898	info.flags = vdev->region[i].flags;
899
900	cap_type.type = vdev->region[i].type;
901	cap_type.subtype = vdev->region[i].subtype;
902
903	ret = vfio_info_add_capability(&caps, &cap_type.header,
904	sizeof(cap_type));
905	if (ret)
906	return ret;
907
908	if (vdev->region[i].ops->add_capability) {
909	ret = vdev->region[i].ops->add_capability(vdev,
910	&vdev->region[i], &caps);
911	if (ret)
912	return ret;
913	}
914	}
915	}
916
917	if (caps.size) {
918	info.flags \|= VFIO_REGION_INFO_FLAG_CAPS;
919	if (info.argsz < sizeof(info) + caps.size) {
920	info.argsz = sizeof(info) + caps.size;
921	info.cap_offset = 0;
922	} else {
923	vfio_info_cap_shift(&caps, sizeof(info));
924	if (copy_to_user((void __user *)arg +
925	sizeof(info), caps.buf,
926	caps.size)) {
927	kfree(caps.buf);
928	return -EFAULT;
929	}
930	info.cap_offset = sizeof(info);
931	}
932
933	kfree(caps.buf);
934	}
935
936	return copy_to_user((void __user *)arg, &info, minsz) ?
937	-EFAULT : 0;
938
939	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
940	struct vfio_irq_info info;
941
942	minsz = offsetofend(struct vfio_irq_info, count);
943
944	if (copy_from_user(&info, (void __user *)arg, minsz))
945	return -EFAULT;
946
947	if (info.argsz < minsz \|\| info.index >= VFIO_PCI_NUM_IRQS)
948	return -EINVAL;
949
950	switch (info.index) {
951	case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
952	case VFIO_PCI_REQ_IRQ_INDEX:
953	break;
954	case VFIO_PCI_ERR_IRQ_INDEX:
955	if (pci_is_pcie(vdev->pdev))
956	break;
957	fallthrough;
958	default:
959	return -EINVAL;
960	}
961
962	info.flags = VFIO_IRQ_INFO_EVENTFD;
963
964	info.count = vfio_pci_get_irq_count(vdev, info.index);
965
966	if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
967	info.flags \|= (VFIO_IRQ_INFO_MASKABLE \|
968	VFIO_IRQ_INFO_AUTOMASKED);
969	else
970	info.flags \|= VFIO_IRQ_INFO_NORESIZE;
971
972	return copy_to_user((void __user *)arg, &info, minsz) ?
973	-EFAULT : 0;
974
975	} else if (cmd == VFIO_DEVICE_SET_IRQS) {
976	struct vfio_irq_set hdr;
977	u8 *data = NULL;
978	int max, ret = 0;
979	size_t data_size = 0;
980
981	minsz = offsetofend(struct vfio_irq_set, count);
982
983	if (copy_from_user(&hdr, (void __user *)arg, minsz))
984	return -EFAULT;
985
986	max = vfio_pci_get_irq_count(vdev, hdr.index);
987
988	ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
989	VFIO_PCI_NUM_IRQS, &data_size);
990	if (ret)
991	return ret;
992
993	if (data_size) {
994	data = memdup_user((void __user *)(arg + minsz),
995	data_size);
996	if (IS_ERR(data))
997	return PTR_ERR(data);
998	}
999
1000	mutex_lock(&vdev->igate);
1001
1002	ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,
1003	hdr.start, hdr.count, data);
1004
1005	mutex_unlock(&vdev->igate);
1006	kfree(data);
1007
1008	return ret;
1009
1010	} else if (cmd == VFIO_DEVICE_RESET) {
1011	int ret;
1012
1013	if (!vdev->reset_works)
1014	return -EINVAL;
1015
1016	vfio_pci_zap_and_down_write_memory_lock(vdev);
1017	ret = pci_try_reset_function(vdev->pdev);
1018	up_write(&vdev->memory_lock);
1019
1020	return ret;
1021
1022	} else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) {
1023	struct vfio_pci_hot_reset_info hdr;
1024	struct vfio_pci_fill_info fill = { 0 };
1025	struct vfio_pci_dependent_device *devices = NULL;
1026	bool slot = false;
1027	int ret = 0;
1028
1029	minsz = offsetofend(struct vfio_pci_hot_reset_info, count);
1030
1031	if (copy_from_user(&hdr, (void __user *)arg, minsz))
1032	return -EFAULT;
1033
1034	if (hdr.argsz < minsz)
1035	return -EINVAL;
1036
1037	hdr.flags = 0;
1038
1039	/* Can we do a slot or bus reset or neither? */
1040	if (!pci_probe_reset_slot(vdev->pdev->slot))
1041	slot = true;
1042	else if (pci_probe_reset_bus(vdev->pdev->bus))
1043	return -ENODEV;
1044
1045	/* How many devices are affected? */
1046	ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
1047	vfio_pci_count_devs,
1048	&fill.max, slot);
1049	if (ret)
1050	return ret;
1051
1052	WARN_ON(!fill.max); /* Should always be at least one */
1053
1054	/*
1055	* If there's enough space, fill it now, otherwise return
1056	* -ENOSPC and the number of devices affected.
1057	*/
1058	if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) {
1059	ret = -ENOSPC;
1060	hdr.count = fill.max;
1061	goto reset_info_exit;
1062	}
1063
1064	devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL);
1065	if (!devices)
1066	return -ENOMEM;
1067
1068	fill.devices = devices;
1069
1070	ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
1071	vfio_pci_fill_devs,
1072	&fill, slot);
1073
1074	/*
1075	* If a device was removed between counting and filling,
1076	* we may come up short of fill.max. If a device was
1077	* added, we'll have a return of -EAGAIN above.
1078	*/
1079	if (!ret)
1080	hdr.count = fill.cur;
1081
1082	reset_info_exit:
1083	if (copy_to_user((void __user *)arg, &hdr, minsz))
1084	ret = -EFAULT;
1085
1086	if (!ret) {
1087	if (copy_to_user((void __user *)(arg + minsz), devices,
1088	hdr.count * sizeof(*devices)))
1089	ret = -EFAULT;
1090	}
1091
1092	kfree(devices);
1093	return ret;
1094
1095	} else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) {
1096	struct vfio_pci_hot_reset hdr;
1097	int32_t *group_fds;
1098	struct vfio_group **groups;
1099	struct vfio_pci_group_info info;
1100	bool slot = false;
1101	int group_idx, count = 0, ret = 0;
1102
1103	minsz = offsetofend(struct vfio_pci_hot_reset, count);
1104
1105	if (copy_from_user(&hdr, (void __user *)arg, minsz))
1106	return -EFAULT;
1107
1108	if (hdr.argsz < minsz \|\| hdr.flags)
1109	return -EINVAL;
1110
1111	/* Can we do a slot or bus reset or neither? */
1112	if (!pci_probe_reset_slot(vdev->pdev->slot))
1113	slot = true;
1114	else if (pci_probe_reset_bus(vdev->pdev->bus))
1115	return -ENODEV;
1116
1117	/*
1118	* We can't let userspace give us an arbitrarily large
1119	* buffer to copy, so verify how many we think there
1120	* could be. Note groups can have multiple devices so
1121	* one group per device is the max.
1122	*/
1123	ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
1124	vfio_pci_count_devs,
1125	&count, slot);
1126	if (ret)
1127	return ret;
1128
1129	/* Somewhere between 1 and count is OK */
1130	if (!hdr.count \|\| hdr.count > count)
1131	return -EINVAL;
1132
1133	group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL);
1134	groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL);
1135	if (!group_fds \|\| !groups) {
1136	kfree(group_fds);
1137	kfree(groups);
1138	return -ENOMEM;
1139	}
1140
1141	if (copy_from_user(group_fds, (void __user *)(arg + minsz),
1142	hdr.count * sizeof(*group_fds))) {
1143	kfree(group_fds);
1144	kfree(groups);
1145	return -EFAULT;
1146	}
1147
1148	/*
1149	* For each group_fd, get the group through the vfio external
1150	* user interface and store the group and iommu ID. This
1151	* ensures the group is held across the reset.
1152	*/
1153	for (group_idx = 0; group_idx < hdr.count; group_idx++) {
1154	struct vfio_group *group;
1155	struct fd f = fdget(group_fds[group_idx]);
1156	if (!f.file) {
1157	ret = -EBADF;
1158	break;
1159	}
1160
1161	group = vfio_group_get_external_user(f.file);
1162	fdput(f);
1163	if (IS_ERR(group)) {
1164	ret = PTR_ERR(group);
1165	break;
1166	}
1167
1168	groups[group_idx] = group;
1169	}
1170
1171	kfree(group_fds);
1172
1173	/* release reference to groups on error */
1174	if (ret)
1175	goto hot_reset_release;
1176
1177	info.count = hdr.count;
1178	info.groups = groups;
1179
1180	ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info);
1181
1182	hot_reset_release:
1183	for (group_idx--; group_idx >= 0; group_idx--)
1184	vfio_group_put_external_user(groups[group_idx]);
1185
1186	kfree(groups);
1187	return ret;
1188	} else if (cmd == VFIO_DEVICE_IOEVENTFD) {
1189	struct vfio_device_ioeventfd ioeventfd;
1190	int count;
1191
1192	minsz = offsetofend(struct vfio_device_ioeventfd, fd);
1193
1194	if (copy_from_user(&ioeventfd, (void __user *)arg, minsz))
1195	return -EFAULT;
1196
1197	if (ioeventfd.argsz < minsz)
1198	return -EINVAL;
1199
1200	if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK)
1201	return -EINVAL;
1202
1203	count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK;
1204
1205	if (hweight8(count) != 1 \|\| ioeventfd.fd < -1)
1206	return -EINVAL;
1207
1208	return vfio_pci_ioeventfd(vdev, ioeventfd.offset,
1209	ioeventfd.data, count, ioeventfd.fd);
1210	} else if (cmd == VFIO_DEVICE_FEATURE) {
1211	struct vfio_device_feature feature;
1212	uuid_t uuid;
1213
1214	minsz = offsetofend(struct vfio_device_feature, flags);
1215
1216	if (copy_from_user(&feature, (void __user *)arg, minsz))
1217	return -EFAULT;
1218
1219	if (feature.argsz < minsz)
1220	return -EINVAL;
1221
1222	/* Check unknown flags */
1223	if (feature.flags & ~(VFIO_DEVICE_FEATURE_MASK \|
1224	VFIO_DEVICE_FEATURE_SET \|
1225	VFIO_DEVICE_FEATURE_GET \|
1226	VFIO_DEVICE_FEATURE_PROBE))
1227	return -EINVAL;
1228
1229	/* GET & SET are mutually exclusive except with PROBE */
1230	if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) &&
1231	(feature.flags & VFIO_DEVICE_FEATURE_SET) &&
1232	(feature.flags & VFIO_DEVICE_FEATURE_GET))
1233	return -EINVAL;
1234
1235	switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) {
1236	case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN:
1237	if (!vdev->vf_token)
1238	return -ENOTTY;
1239
1240	/*
1241	* We do not support GET of the VF Token UUID as this
1242	* could expose the token of the previous device user.
1243	*/
1244	if (feature.flags & VFIO_DEVICE_FEATURE_GET)
1245	return -EINVAL;
1246
1247	if (feature.flags & VFIO_DEVICE_FEATURE_PROBE)
1248	return 0;
1249
1250	/* Don't SET unless told to do so */
1251	if (!(feature.flags & VFIO_DEVICE_FEATURE_SET))
1252	return -EINVAL;
1253
1254	if (feature.argsz < minsz + sizeof(uuid))
1255	return -EINVAL;
1256
1257	if (copy_from_user(&uuid, (void __user *)(arg + minsz),
1258	sizeof(uuid)))
1259	return -EFAULT;
1260
1261	mutex_lock(&vdev->vf_token->lock);
1262	uuid_copy(&vdev->vf_token->uuid, &uuid);
1263	mutex_unlock(&vdev->vf_token->lock);
1264
1265	return 0;
1266	default:
1267	return -ENOTTY;
1268	}
1269	}
1270
1271	return -ENOTTY;
1272	}
1273
1274	static ssize_t vfio_pci_rw(struct vfio_pci_device vdev, char __user buf,
1275	size_t count, loff_t *ppos, bool iswrite)
1276	{
1277	unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
1278
1279	if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1280	return -EINVAL;
1281
1282	switch (index) {
1283	case VFIO_PCI_CONFIG_REGION_INDEX:
1284	return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
1285
1286	case VFIO_PCI_ROM_REGION_INDEX:
1287	if (iswrite)
1288	return -EINVAL;
1289	return vfio_pci_bar_rw(vdev, buf, count, ppos, false);
1290
1291	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1292	return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
1293
1294	case VFIO_PCI_VGA_REGION_INDEX:
1295	return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
1296	default:
1297	index -= VFIO_PCI_NUM_REGIONS;
1298	return vdev->region[index].ops->rw(vdev, buf,
1299	count, ppos, iswrite);
1300	}
1301
1302	return -EINVAL;
1303	}
1304
1305	static ssize_t vfio_pci_read(struct vfio_device core_vdev, char __user buf,
1306	size_t count, loff_t *ppos)
1307	{
1308	struct vfio_pci_device *vdev =
1309	container_of(core_vdev, struct vfio_pci_device, vdev);
1310
1311	if (!count)
1312	return 0;
1313
1314	return vfio_pci_rw(vdev, buf, count, ppos, false);
1315	}
1316
1317	static ssize_t vfio_pci_write(struct vfio_device core_vdev, const char __user buf,
1318	size_t count, loff_t *ppos)
1319	{
1320	struct vfio_pci_device *vdev =
1321	container_of(core_vdev, struct vfio_pci_device, vdev);
1322
1323	if (!count)
1324	return 0;
1325
1326	return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true);
1327	}
1328
1329	/* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */
1330	static int vfio_pci_zap_and_vma_lock(struct vfio_pci_device *vdev, bool try)
1331	{
1332	struct vfio_pci_mmap_vma mmap_vma, tmp;
1333
1334	/*
1335	* Lock ordering:
1336	* vma_lock is nested under mmap_lock for vm_ops callback paths.
1337	* The memory_lock semaphore is used by both code paths calling
1338	* into this function to zap vmas and the vm_ops.fault callback
1339	* to protect the memory enable state of the device.
1340	*
1341	* When zapping vmas we need to maintain the mmap_lock => vma_lock
1342	* ordering, which requires using vma_lock to walk vma_list to
1343	* acquire an mm, then dropping vma_lock to get the mmap_lock and
1344	* reacquiring vma_lock. This logic is derived from similar
1345	* requirements in uverbs_user_mmap_disassociate().
1346	*
1347	* mmap_lock must always be the top-level lock when it is taken.
1348	* Therefore we can only hold the memory_lock write lock when
1349	* vma_list is empty, as we'd need to take mmap_lock to clear
1350	* entries. vma_list can only be guaranteed empty when holding
1351	* vma_lock, thus memory_lock is nested under vma_lock.
1352	*
1353	* This enables the vm_ops.fault callback to acquire vma_lock,
1354	* followed by memory_lock read lock, while already holding
1355	* mmap_lock without risk of deadlock.
1356	*/
1357	while (1) {
1358	struct mm_struct *mm = NULL;
1359
1360	if (try) {
1361	if (!mutex_trylock(&vdev->vma_lock))
1362	return 0;
1363	} else {
1364	mutex_lock(&vdev->vma_lock);
1365	}
1366	while (!list_empty(&vdev->vma_list)) {
1367	mmap_vma = list_first_entry(&vdev->vma_list,
1368	struct vfio_pci_mmap_vma,
1369	vma_next);
1370	mm = mmap_vma->vma->vm_mm;
1371	if (mmget_not_zero(mm))
1372	break;
1373
1374	list_del(&mmap_vma->vma_next);
1375	kfree(mmap_vma);
1376	mm = NULL;
1377	}
1378	if (!mm)
1379	return 1;
1380	mutex_unlock(&vdev->vma_lock);
1381
1382	if (try) {
1383	if (!mmap_read_trylock(mm)) {
1384	mmput(mm);
1385	return 0;
1386	}
1387	} else {
1388	mmap_read_lock(mm);
1389	}
1390	if (try) {
1391	if (!mutex_trylock(&vdev->vma_lock)) {
1392	mmap_read_unlock(mm);
1393	mmput(mm);
1394	return 0;
1395	}
1396	} else {
1397	mutex_lock(&vdev->vma_lock);
1398	}
1399	list_for_each_entry_safe(mmap_vma, tmp,
1400	&vdev->vma_list, vma_next) {
1401	struct vm_area_struct *vma = mmap_vma->vma;
1402
1403	if (vma->vm_mm != mm)
1404	continue;
1405
1406	list_del(&mmap_vma->vma_next);
1407	kfree(mmap_vma);
1408
1409	zap_vma_ptes(vma, vma->vm_start,
1410	vma->vm_end - vma->vm_start);
1411	}
1412	mutex_unlock(&vdev->vma_lock);
1413	mmap_read_unlock(mm);
1414	mmput(mm);
1415	}
1416	}
1417
1418	void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device *vdev)
1419	{
1420	vfio_pci_zap_and_vma_lock(vdev, false);
1421	down_write(&vdev->memory_lock);
1422	mutex_unlock(&vdev->vma_lock);
1423	}
1424
1425	u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_device *vdev)
1426	{
1427	u16 cmd;
1428
1429	down_write(&vdev->memory_lock);
1430	pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd);
1431	if (!(cmd & PCI_COMMAND_MEMORY))
1432	pci_write_config_word(vdev->pdev, PCI_COMMAND,
1433	cmd \| PCI_COMMAND_MEMORY);
1434
1435	return cmd;
1436	}
1437
1438	void vfio_pci_memory_unlock_and_restore(struct vfio_pci_device *vdev, u16 cmd)
1439	{
1440	pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd);
1441	up_write(&vdev->memory_lock);
1442	}
1443
1444	/* Caller holds vma_lock */
1445	static int __vfio_pci_add_vma(struct vfio_pci_device *vdev,
1446	struct vm_area_struct *vma)
1447	{
1448	struct vfio_pci_mmap_vma *mmap_vma;
1449
1450	mmap_vma = kmalloc(sizeof(*mmap_vma), GFP_KERNEL);
1451	if (!mmap_vma)
1452	return -ENOMEM;
1453
1454	mmap_vma->vma = vma;
1455	list_add(&mmap_vma->vma_next, &vdev->vma_list);
1456
1457	return 0;
1458	}
1459
1460	/*
1461	* Zap mmaps on open so that we can fault them in on access and therefore
1462	* our vma_list only tracks mappings accessed since last zap.
1463	*/
1464	static void vfio_pci_mmap_open(struct vm_area_struct *vma)
1465	{
1466	zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
1467	}
1468
1469	static void vfio_pci_mmap_close(struct vm_area_struct *vma)
1470	{
1471	struct vfio_pci_device *vdev = vma->vm_private_data;
1472	struct vfio_pci_mmap_vma *mmap_vma;
1473
1474	mutex_lock(&vdev->vma_lock);
1475	list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
1476	if (mmap_vma->vma == vma) {
1477	list_del(&mmap_vma->vma_next);
1478	kfree(mmap_vma);
1479	break;
1480	}
1481	}
1482	mutex_unlock(&vdev->vma_lock);
1483	}
1484
1485	static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf)
1486	{
1487	struct vm_area_struct *vma = vmf->vma;
1488	struct vfio_pci_device *vdev = vma->vm_private_data;
1489	struct vfio_pci_mmap_vma *mmap_vma;
1490	vm_fault_t ret = VM_FAULT_NOPAGE;
1491
1492	mutex_lock(&vdev->vma_lock);
1493	down_read(&vdev->memory_lock);
1494
1495	if (!__vfio_pci_memory_enabled(vdev)) {
1496	ret = VM_FAULT_SIGBUS;
1497	goto up_out;
1498	}
1499
1500	/*
1501	* We populate the whole vma on fault, so we need to test whether
1502	* the vma has already been mapped, such as for concurrent faults
1503	* to the same vma. io_remap_pfn_range() will trigger a BUG_ON if
1504	* we ask it to fill the same range again.
1505	*/
1506	list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
1507	if (mmap_vma->vma == vma)
1508	goto up_out;
1509	}
1510
1511	if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
1512	vma->vm_end - vma->vm_start,
1513	vma->vm_page_prot)) {
1514	ret = VM_FAULT_SIGBUS;
1515	zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
1516	goto up_out;
1517	}
1518
1519	if (__vfio_pci_add_vma(vdev, vma)) {
1520	ret = VM_FAULT_OOM;
1521	zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
1522	}
1523
1524	up_out:
1525	up_read(&vdev->memory_lock);
1526	mutex_unlock(&vdev->vma_lock);
1527	return ret;
1528	}
1529
1530	static const struct vm_operations_struct vfio_pci_mmap_ops = {
1531	.open = vfio_pci_mmap_open,
1532	.close = vfio_pci_mmap_close,
1533	.fault = vfio_pci_mmap_fault,
1534	};
1535
1536	static int vfio_pci_mmap(struct vfio_device core_vdev, struct vm_area_struct vma)
1537	{
1538	struct vfio_pci_device *vdev =
1539	container_of(core_vdev, struct vfio_pci_device, vdev);
1540	struct pci_dev *pdev = vdev->pdev;
1541	unsigned int index;
1542	u64 phys_len, req_len, pgoff, req_start;
1543	int ret;
1544
1545	index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1546
1547	if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1548	return -EINVAL;
1549	if (vma->vm_end < vma->vm_start)
1550	return -EINVAL;
1551	if ((vma->vm_flags & VM_SHARED) == 0)
1552	return -EINVAL;
1553	if (index >= VFIO_PCI_NUM_REGIONS) {
1554	int regnum = index - VFIO_PCI_NUM_REGIONS;
1555	struct vfio_pci_region *region = vdev->region + regnum;
1556
1557	if (region->ops && region->ops->mmap &&
1558	(region->flags & VFIO_REGION_INFO_FLAG_MMAP))
1559	return region->ops->mmap(vdev, region, vma);
1560	return -EINVAL;
1561	}
1562	if (index >= VFIO_PCI_ROM_REGION_INDEX)
1563	return -EINVAL;
1564	if (!vdev->bar_mmap_supported[index])
1565	return -EINVAL;
1566
1567	phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));
1568	req_len = vma->vm_end - vma->vm_start;
1569	pgoff = vma->vm_pgoff &
1570	((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1571	req_start = pgoff << PAGE_SHIFT;
1572
1573	if (req_start + req_len > phys_len)
1574	return -EINVAL;
1575
1576	/*
1577	* Even though we don't make use of the barmap for the mmap,
1578	* we need to request the region and the barmap tracks that.
1579	*/
1580	if (!vdev->barmap[index]) {
1581	ret = pci_request_selected_regions(pdev,
1582	1 << index, "vfio-pci");
1583	if (ret)
1584	return ret;
1585
1586	vdev->barmap[index] = pci_iomap(pdev, index, 0);
1587	if (!vdev->barmap[index]) {
1588	pci_release_selected_regions(pdev, 1 << index);
1589	return -ENOMEM;
1590	}
1591	}
1592
1593	vma->vm_private_data = vdev;
1594	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1595	vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;
1596
1597	/*
1598	* See remap_pfn_range(), called from vfio_pci_fault() but we can't
1599	* change vm_flags within the fault handler. Set them now.
1600	*/
1601	vma->vm_flags \|= VM_IO \| VM_PFNMAP \| VM_DONTEXPAND \| VM_DONTDUMP;
1602	vma->vm_ops = &vfio_pci_mmap_ops;
1603
1604	return 0;
1605	}
1606
1607	static void vfio_pci_request(struct vfio_device *core_vdev, unsigned int count)
1608	{
1609	struct vfio_pci_device *vdev =
1610	container_of(core_vdev, struct vfio_pci_device, vdev);
1611	struct pci_dev *pdev = vdev->pdev;
1612
1613	mutex_lock(&vdev->igate);
1614
1615	if (vdev->req_trigger) {
1616	if (!(count % 10))
1617	pci_notice_ratelimited(pdev,
1618	"Relaying device request to user (#%u)\n",
1619	count);
1620	eventfd_signal(vdev->req_trigger, 1);
1621	} else if (count == 0) {
1622	pci_warn(pdev,
1623	"No device request channel registered, blocked until released by user\n");
1624	}
1625
1626	mutex_unlock(&vdev->igate);
1627	}
1628
1629	static int vfio_pci_validate_vf_token(struct vfio_pci_device *vdev,
1630	bool vf_token, uuid_t *uuid)
1631	{
1632	/*
1633	* There's always some degree of trust or collaboration between SR-IOV
1634	* PF and VFs, even if just that the PF hosts the SR-IOV capability and
1635	* can disrupt VFs with a reset, but often the PF has more explicit
1636	* access to deny service to the VF or access data passed through the
1637	* VF. We therefore require an opt-in via a shared VF token (UUID) to
1638	* represent this trust. This both prevents that a VF driver might
1639	* assume the PF driver is a trusted, in-kernel driver, and also that
1640	* a PF driver might be replaced with a rogue driver, unknown to in-use
1641	* VF drivers.
1642	*
1643	* Therefore when presented with a VF, if the PF is a vfio device and
1644	* it is bound to the vfio-pci driver, the user needs to provide a VF
1645	* token to access the device, in the form of appending a vf_token to
1646	* the device name, for example:
1647	*
1648	* "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
1649	*
1650	* When presented with a PF which has VFs in use, the user must also
1651	* provide the current VF token to prove collaboration with existing
1652	* VF users. If VFs are not in use, the VF token provided for the PF
1653	* device will act to set the VF token.
1654	*
1655	* If the VF token is provided but unused, an error is generated.
1656	*/
1657	if (!vdev->pdev->is_virtfn && !vdev->vf_token && !vf_token)
1658	return 0; /* No VF token provided or required */
1659
1660	if (vdev->pdev->is_virtfn) {
1661	struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev);
1662	bool match;
1663
1664	if (!pf_vdev) {
1665	if (!vf_token)
1666	return 0; /* PF is not vfio-pci, no VF token */
1667
1668	pci_info_ratelimited(vdev->pdev,
1669	"VF token incorrectly provided, PF not bound to vfio-pci\n");
1670	return -EINVAL;
1671	}
1672
1673	if (!vf_token) {
1674	vfio_device_put(&pf_vdev->vdev);
1675	pci_info_ratelimited(vdev->pdev,
1676	"VF token required to access device\n");
1677	return -EACCES;
1678	}
1679
1680	mutex_lock(&pf_vdev->vf_token->lock);
1681	match = uuid_equal(uuid, &pf_vdev->vf_token->uuid);
1682	mutex_unlock(&pf_vdev->vf_token->lock);
1683
1684	vfio_device_put(&pf_vdev->vdev);
1685
1686	if (!match) {
1687	pci_info_ratelimited(vdev->pdev,
1688	"Incorrect VF token provided for device\n");
1689	return -EACCES;
1690	}
1691	} else if (vdev->vf_token) {
1692	mutex_lock(&vdev->vf_token->lock);
1693	if (vdev->vf_token->users) {
1694	if (!vf_token) {
1695	mutex_unlock(&vdev->vf_token->lock);
1696	pci_info_ratelimited(vdev->pdev,
1697	"VF token required to access device\n");
1698	return -EACCES;
1699	}
1700
1701	if (!uuid_equal(uuid, &vdev->vf_token->uuid)) {
1702	mutex_unlock(&vdev->vf_token->lock);
1703	pci_info_ratelimited(vdev->pdev,
1704	"Incorrect VF token provided for device\n");
1705	return -EACCES;
1706	}
1707	} else if (vf_token) {
1708	uuid_copy(&vdev->vf_token->uuid, uuid);
1709	}
1710
1711	mutex_unlock(&vdev->vf_token->lock);
1712	} else if (vf_token) {
1713	pci_info_ratelimited(vdev->pdev,
1714	"VF token incorrectly provided, not a PF or VF\n");
1715	return -EINVAL;
1716	}
1717
1718	return 0;
1719	}
1720
1721	#define VF_TOKEN_ARG "vf_token="
1722
1723	static int vfio_pci_match(struct vfio_device core_vdev, char buf)
1724	{
1725	struct vfio_pci_device *vdev =
1726	container_of(core_vdev, struct vfio_pci_device, vdev);
1727	bool vf_token = false;
1728	uuid_t uuid;
1729	int ret;
1730
1731	if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev))))
1732	return 0; /* No match */
1733
1734	if (strlen(buf) > strlen(pci_name(vdev->pdev))) {
1735	buf += strlen(pci_name(vdev->pdev));
1736
1737	if (*buf != ' ')
1738	return 0; /* No match: non-whitespace after name */
1739
1740	while (*buf) {
1741	if (*buf == ' ') {
1742	buf++;
1743	continue;
1744	}
1745
1746	if (!vf_token && !strncmp(buf, VF_TOKEN_ARG,
1747	strlen(VF_TOKEN_ARG))) {
1748	buf += strlen(VF_TOKEN_ARG);
1749
1750	if (strlen(buf) < UUID_STRING_LEN)
1751	return -EINVAL;
1752
1753	ret = uuid_parse(buf, &uuid);
1754	if (ret)
1755	return ret;
1756
1757	vf_token = true;
1758	buf += UUID_STRING_LEN;
1759	} else {
1760	/* Unknown/duplicate option */
1761	return -EINVAL;
1762	}
1763	}
1764	}
1765
1766	ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid);
1767	if (ret)
1768	return ret;
1769
1770	return 1; /* Match */
1771	}
1772
1773	static const struct vfio_device_ops vfio_pci_ops = {
1774	.name = "vfio-pci",
1775	.open_device = vfio_pci_open_device,
1776	.close_device = vfio_pci_close_device,
1777	.ioctl = vfio_pci_ioctl,
1778	.read = vfio_pci_read,
1779	.write = vfio_pci_write,
1780	.mmap = vfio_pci_mmap,
1781	.request = vfio_pci_request,
1782	.match = vfio_pci_match,
1783	};
1784
1785	static int vfio_pci_bus_notifier(struct notifier_block *nb,
1786	unsigned long action, void *data)
1787	{
1788	struct vfio_pci_device *vdev = container_of(nb,
1789	struct vfio_pci_device, nb);
1790	struct device *dev = data;
1791	struct pci_dev *pdev = to_pci_dev(dev);
1792	struct pci_dev *physfn = pci_physfn(pdev);
1793
1794	if (action == BUS_NOTIFY_ADD_DEVICE &&
1795	pdev->is_virtfn && physfn == vdev->pdev) {
1796	pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n",
1797	pci_name(pdev));
1798	pdev->driver_override = kasprintf(GFP_KERNEL, "%s",
1799	vfio_pci_ops.name);
1800	} else if (action == BUS_NOTIFY_BOUND_DRIVER &&
1801	pdev->is_virtfn && physfn == vdev->pdev) {
1802	struct pci_driver *drv = pci_dev_driver(pdev);
1803
1804	if (drv && drv != &vfio_pci_driver)
1805	pci_warn(vdev->pdev,
1806	"VF %s bound to driver %s while PF bound to vfio-pci\n",
1807	pci_name(pdev), drv->name);
1808	}
1809
1810	return 0;
1811	}
1812
1813	static int vfio_pci_vf_init(struct vfio_pci_device *vdev)
1814	{
1815	struct pci_dev *pdev = vdev->pdev;
1816	int ret;
1817
1818	if (!pdev->is_physfn)
1819	return 0;
1820
1821	vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL);
1822	if (!vdev->vf_token)
1823	return -ENOMEM;
1824
1825	mutex_init(&vdev->vf_token->lock);
1826	uuid_gen(&vdev->vf_token->uuid);
1827
1828	vdev->nb.notifier_call = vfio_pci_bus_notifier;
1829	ret = bus_register_notifier(&pci_bus_type, &vdev->nb);
1830	if (ret) {
1831	kfree(vdev->vf_token);
1832	return ret;
1833	}
1834	return 0;
1835	}
1836
1837	static void vfio_pci_vf_uninit(struct vfio_pci_device *vdev)
1838	{
1839	if (!vdev->vf_token)
1840	return;
1841
1842	bus_unregister_notifier(&pci_bus_type, &vdev->nb);
1843	WARN_ON(vdev->vf_token->users);
1844	mutex_destroy(&vdev->vf_token->lock);
1845	kfree(vdev->vf_token);
1846	}
1847
1848	static int vfio_pci_vga_init(struct vfio_pci_device *vdev)
1849	{
1850	struct pci_dev *pdev = vdev->pdev;
1851	int ret;
1852
1853	if (!vfio_pci_is_vga(pdev))
1854	return 0;
1855
1856	ret = vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode);
1857	if (ret)
1858	return ret;
1859	vga_set_legacy_decoding(pdev, vfio_pci_set_vga_decode(vdev, false));
1860	return 0;
1861	}
1862
1863	static void vfio_pci_vga_uninit(struct vfio_pci_device *vdev)
1864	{
1865	struct pci_dev *pdev = vdev->pdev;
1866
1867	if (!vfio_pci_is_vga(pdev))
1868	return;
1869	vga_client_register(pdev, NULL, NULL, NULL);
1870	vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM \|
1871	VGA_RSRC_LEGACY_IO \|
1872	VGA_RSRC_LEGACY_MEM);
1873	}
1874
1875	static int vfio_pci_probe(struct pci_dev pdev, const struct pci_device_id id)
1876	{
1877	struct vfio_pci_device *vdev;
1878	struct iommu_group *group;
1879	int ret;
1880
1881	if (vfio_pci_is_denylisted(pdev))
1882	return -EINVAL;
1883
1884	if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)
1885	return -EINVAL;
1886
1887	/*
1888	* Prevent binding to PFs with VFs enabled, the VFs might be in use
1889	* by the host or other users. We cannot capture the VFs if they
1890	* already exist, nor can we track VF users. Disabling SR-IOV here
1891	* would initiate removing the VFs, which would unbind the driver,
1892	* which is prone to blocking if that VF is also in use by vfio-pci.
1893	* Just reject these PFs and let the user sort it out.
1894	*/
1895	if (pci_num_vf(pdev)) {
1896	pci_warn(pdev, "Cannot bind to PF with SR-IOV enabled\n");
1897	return -EBUSY;
1898	}
1899
1900	group = vfio_iommu_group_get(&pdev->dev);
1901	if (!group)
1902	return -EINVAL;
1903
1904	vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
1905	if (!vdev) {
1906	ret = -ENOMEM;
1907	goto out_group_put;
1908	}
1909
1910	vfio_init_group_dev(&vdev->vdev, &pdev->dev, &vfio_pci_ops);
1911	vdev->pdev = pdev;
1912	vdev->irq_type = VFIO_PCI_NUM_IRQS;
1913	mutex_init(&vdev->igate);
1914	spin_lock_init(&vdev->irqlock);
1915	mutex_init(&vdev->ioeventfds_lock);
1916	INIT_LIST_HEAD(&vdev->dummy_resources_list);
1917	INIT_LIST_HEAD(&vdev->ioeventfds_list);
1918	mutex_init(&vdev->vma_lock);
1919	INIT_LIST_HEAD(&vdev->vma_list);
1920	init_rwsem(&vdev->memory_lock);
1921
1922	if (pci_is_root_bus(pdev->bus)) {
1923	ret = vfio_assign_device_set(&vdev->vdev, vdev);
1924	} else if (!pci_probe_reset_slot(pdev->slot)) {
1925	ret = vfio_assign_device_set(&vdev->vdev, pdev->slot);
1926	} else {
1927	/*
1928	* If there is no slot reset support for this device, the whole
1929	* bus needs to be grouped together to support bus-wide resets.
1930	*/
1931	ret = vfio_assign_device_set(&vdev->vdev, pdev->bus);
1932	}
1933
1934	if (ret)
1935	goto out_uninit;
1936	ret = vfio_pci_vf_init(vdev);
1937	if (ret)
1938	goto out_uninit;
1939	ret = vfio_pci_vga_init(vdev);
1940	if (ret)
1941	goto out_vf;
1942
1943	vfio_pci_probe_power_state(vdev);
1944
1945	if (!disable_idle_d3) {
1946	/*
1947	* pci-core sets the device power state to an unknown value at
1948	* bootup and after being removed from a driver. The only
1949	* transition it allows from this unknown state is to D0, which
1950	* typically happens when a driver calls pci_enable_device().
1951	* We're not ready to enable the device yet, but we do want to
1952	* be able to get to D3. Therefore first do a D0 transition
1953	* before going to D3.
1954	*/
1955	vfio_pci_set_power_state(vdev, PCI_D0);
1956	vfio_pci_set_power_state(vdev, PCI_D3hot);
1957	}
1958
1959	ret = vfio_register_group_dev(&vdev->vdev);
1960	if (ret)
1961	goto out_power;
1962	dev_set_drvdata(&pdev->dev, vdev);
1963	return 0;
1964
1965	out_power:
1966	if (!disable_idle_d3)
1967	vfio_pci_set_power_state(vdev, PCI_D0);
1968	out_vf:
1969	vfio_pci_vf_uninit(vdev);
1970	out_uninit:
1971	vfio_uninit_group_dev(&vdev->vdev);
1972	kfree(vdev->pm_save);
1973	kfree(vdev);
1974	out_group_put:
1975	vfio_iommu_group_put(group, &pdev->dev);
1976	return ret;
1977	}
1978
1979	static void vfio_pci_remove(struct pci_dev *pdev)
1980	{
1981	struct vfio_pci_device *vdev = dev_get_drvdata(&pdev->dev);
1982
1983	pci_disable_sriov(pdev);
1984
1985	vfio_unregister_group_dev(&vdev->vdev);
1986
1987	vfio_pci_vf_uninit(vdev);
1988	vfio_uninit_group_dev(&vdev->vdev);
1989	vfio_pci_vga_uninit(vdev);
1990
1991	vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev);
1992
1993	if (!disable_idle_d3)
1994	vfio_pci_set_power_state(vdev, PCI_D0);
1995
1996	mutex_destroy(&vdev->ioeventfds_lock);
1997	kfree(vdev->region);
1998	kfree(vdev->pm_save);
1999	kfree(vdev);
2000	}
2001
2002	static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,
2003	pci_channel_state_t state)
2004	{
2005	struct vfio_pci_device *vdev;
2006	struct vfio_device *device;
2007
2008	device = vfio_device_get_from_dev(&pdev->dev);
2009	if (device == NULL)
2010	return PCI_ERS_RESULT_DISCONNECT;
2011
2012	vdev = container_of(device, struct vfio_pci_device, vdev);
2013
2014	mutex_lock(&vdev->igate);
2015
2016	if (vdev->err_trigger)
2017	eventfd_signal(vdev->err_trigger, 1);
2018
2019	mutex_unlock(&vdev->igate);
2020
2021	vfio_device_put(device);
2022
2023	return PCI_ERS_RESULT_CAN_RECOVER;
2024	}
2025
2026	static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
2027	{
2028	struct vfio_device *device;
2029	int ret = 0;
2030
2031	might_sleep();
2032
2033	if (!enable_sriov)
2034	return -ENOENT;
2035
2036	device = vfio_device_get_from_dev(&pdev->dev);
2037	if (!device)
2038	return -ENODEV;
2039
2040	if (nr_virtfn == 0)
2041	pci_disable_sriov(pdev);
2042	else
2043	ret = pci_enable_sriov(pdev, nr_virtfn);
2044
2045	vfio_device_put(device);
2046
2047	return ret < 0 ? ret : nr_virtfn;
2048	}
2049
2050	static const struct pci_error_handlers vfio_err_handlers = {
2051	.error_detected = vfio_pci_aer_err_detected,
2052	};
2053
2054	static struct pci_driver vfio_pci_driver = {
2055	.name = "vfio-pci",
2056	.id_table = NULL, /* only dynamic ids */
2057	.probe = vfio_pci_probe,
2058	.remove = vfio_pci_remove,
2059	.sriov_configure = vfio_pci_sriov_configure,
2060	.err_handler = &vfio_err_handlers,
2061	};
2062
2063	static bool vfio_dev_in_groups(struct vfio_pci_device *vdev,
2064	struct vfio_pci_group_info *groups)
2065	{
2066	unsigned int i;
2067
2068	for (i = 0; i < groups->count; i++)
2069	if (groups->groups[i] == vdev->vdev.group)
2070	return true;
2071	return false;
2072	}
2073
2074	static int vfio_pci_is_device_in_set(struct pci_dev pdev, void data)
2075	{
2076	struct vfio_device_set *dev_set = data;
2077	struct vfio_device *cur;
2078
2079	list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
2080	if (cur->dev == &pdev->dev)
2081	return 0;
2082	return -EBUSY;
2083	}
2084
2085	/*
2086	* vfio-core considers a group to be viable and will create a vfio_device even
2087	* if some devices are bound to drivers like pci-stub or pcieport. Here we
2088	* require all PCI devices to be inside our dev_set since that ensures they stay
2089	* put and that every driver controlling the device can co-ordinate with the
2090	* device reset.
2091	*
2092	* Returns the pci_dev to pass to pci_reset_bus() if every PCI device to be
2093	* reset is inside the dev_set, and pci_reset_bus() can succeed. NULL otherwise.
2094	*/
2095	static struct pci_dev *
2096	vfio_pci_dev_set_resettable(struct vfio_device_set *dev_set)
2097	{
2098	struct pci_dev *pdev;
2099
2100	lockdep_assert_held(&dev_set->lock);
2101
2102	/*
2103	* By definition all PCI devices in the dev_set share the same PCI
2104	* reset, so any pci_dev will have the same outcomes for
2105	* pci_probe_reset_*() and pci_reset_bus().
2106	*/
2107	pdev = list_first_entry(&dev_set->device_list, struct vfio_pci_device,
2108	vdev.dev_set_list)->pdev;
2109
2110	/* pci_reset_bus() is supported */
2111	if (pci_probe_reset_slot(pdev->slot) && pci_probe_reset_bus(pdev->bus))
2112	return NULL;
2113
2114	if (vfio_pci_for_each_slot_or_bus(pdev, vfio_pci_is_device_in_set,
2115	dev_set,
2116	!pci_probe_reset_slot(pdev->slot)))
2117	return NULL;
2118	return pdev;
2119	}
2120
2121	/*
2122	* We need to get memory_lock for each device, but devices can share mmap_lock,
2123	* therefore we need to zap and hold the vma_lock for each device, and only then
2124	* get each memory_lock.
2125	*/
2126	static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
2127	struct vfio_pci_group_info *groups)
2128	{
2129	struct vfio_pci_device *cur_mem;
2130	struct vfio_pci_device *cur_vma;
2131	struct vfio_pci_device *cur;
2132	struct pci_dev *pdev;
2133	bool is_mem = true;
2134	int ret;
2135
2136	mutex_lock(&dev_set->lock);
2137	cur_mem = list_first_entry(&dev_set->device_list,
2138	struct vfio_pci_device, vdev.dev_set_list);
2139
2140	pdev = vfio_pci_dev_set_resettable(dev_set);
2141	if (!pdev) {
2142	ret = -EINVAL;
2143	goto err_unlock;
2144	}
2145
2146	list_for_each_entry(cur_vma, &dev_set->device_list, vdev.dev_set_list) {
2147	/*
2148	* Test whether all the affected devices are contained by the
2149	* set of groups provided by the user.
2150	*/
2151	if (!vfio_dev_in_groups(cur_vma, groups)) {
2152	ret = -EINVAL;
2153	goto err_undo;
2154	}
2155
2156	/*
2157	* Locking multiple devices is prone to deadlock, runaway and
2158	* unwind if we hit contention.
2159	*/
2160	if (!vfio_pci_zap_and_vma_lock(cur_vma, true)) {
2161	ret = -EBUSY;
2162	goto err_undo;
2163	}
2164	}
2165	cur_vma = NULL;
2166
2167	list_for_each_entry(cur_mem, &dev_set->device_list, vdev.dev_set_list) {
2168	if (!down_write_trylock(&cur_mem->memory_lock)) {
2169	ret = -EBUSY;
2170	goto err_undo;
2171	}
2172	mutex_unlock(&cur_mem->vma_lock);
2173	}
2174	cur_mem = NULL;
2175
2176	ret = pci_reset_bus(pdev);
2177
2178	err_undo:
2179	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2180	if (cur == cur_mem)
2181	is_mem = false;
2182	if (cur == cur_vma)
2183	break;
2184	if (is_mem)
2185	up_write(&cur->memory_lock);
2186	else
2187	mutex_unlock(&cur->vma_lock);
2188	}
2189	err_unlock:
2190	mutex_unlock(&dev_set->lock);
2191	return ret;
2192	}
2193
2194	static bool vfio_pci_dev_set_needs_reset(struct vfio_device_set *dev_set)
2195	{
2196	struct vfio_pci_device *cur;
2197	bool needs_reset = false;
2198
2199	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2200	/* No VFIO device in the set can have an open device FD */
2201	if (cur->vdev.open_count)
2202	return false;
2203	needs_reset \|= cur->needs_reset;
2204	}
2205	return needs_reset;
2206	}
2207
2208	/*
2209	* If a bus or slot reset is available for the provided dev_set and:
2210	* - All of the devices affected by that bus or slot reset are unused
2211	* - At least one of the affected devices is marked dirty via
2212	* needs_reset (such as by lack of FLR support)
2213	* Then attempt to perform that bus or slot reset.
2214	* Returns true if the dev_set was reset.
2215	*/
2216	static bool vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set)
2217	{
2218	struct vfio_pci_device *cur;
2219	struct pci_dev *pdev;
2220	int ret;
2221
2222	if (!vfio_pci_dev_set_needs_reset(dev_set))
2223	return false;
2224
2225	pdev = vfio_pci_dev_set_resettable(dev_set);
2226	if (!pdev)
2227	return false;
2228
2229	ret = pci_reset_bus(pdev);
2230	if (ret)
2231	return false;
2232
2233	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2234	cur->needs_reset = false;
2235	if (!disable_idle_d3)
2236	vfio_pci_set_power_state(cur, PCI_D3hot);
2237	}
2238	return true;
2239	}
2240
2241	static void __exit vfio_pci_cleanup(void)
2242	{	265	{
2243	pci_unregister_driver(&vfio_pci_driver);	266	pci_unregister_driver(&vfio_pci_driver);
2244	vfio_pci_uninit_perm_bits();
2245	}	267	}
2246
2247	static void __init vfio_pci_fill_ids(void)
2248	{
2249	char p, id;
2250	int rc;
2251
2252	/* no ids passed actually */
2253	if (ids[0] == '\0')
2254	return;
2255
2256	/* add ids specified in the module parameter */
2257	p = ids;
2258	while ((id = strsep(&p, ","))) {
2259	unsigned int vendor, device, subvendor = PCI_ANY_ID,
2260	subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
2261	int fields;
2262
2263	if (!strlen(id))
2264	continue;
2265
2266	fields = sscanf(id, "%x:%x:%x:%x:%x:%x",
2267	&vendor, &device, &subvendor, &subdevice,
2268	&class, &class_mask);
2269
2270	if (fields < 2) {
2271	pr_warn("invalid id string \"%s\"\n", id);
2272	continue;
2273	}
2274
2275	rc = pci_add_dynid(&vfio_pci_driver, vendor, device,
2276	subvendor, subdevice, class, class_mask, 0);
2277	if (rc)
2278	pr_warn("failed to add dynamic id [%04x:%04x[%04x:%04x]] class %#08x/%08x (%d)\n",
2279	vendor, device, subvendor, subdevice,
2280	class, class_mask, rc);
2281	else
2282	pr_info("add [%04x:%04x[%04x:%04x]] class %#08x/%08x\n",
2283	vendor, device, subvendor, subdevice,
2284	class, class_mask);
2285	}
2286	}
2287
2288	static int __init vfio_pci_init(void)
2289	{
2290	int ret;
2291
2292	/* Allocate shared config space permission data used by all devices */
2293	ret = vfio_pci_init_perm_bits();
2294	if (ret)
2295	return ret;
2296
2297	/* Register and scan for devices */
2298	ret = pci_register_driver(&vfio_pci_driver);
2299	if (ret)
2300	goto out_driver;
2301
2302	vfio_pci_fill_ids();
2303
2304	if (disable_denylist)
2305	pr_warn("device denylist disabled.\n");
2306
2307	return 0;
2308
2309	out_driver:
2310	vfio_pci_uninit_perm_bits();
2311	return ret;
2312	}
2313
2314	module_init(vfio_pci_init);
2315	module_exit(vfio_pci_cleanup);	268	module_exit(vfio_pci_cleanup);
2316		269
2317	MODULE_VERSION(DRIVER_VERSION);
2318	MODULE_LICENSE("GPL v2");	270	MODULE_LICENSE("GPL v2");
2319	MODULE_AUTHOR(DRIVER_AUTHOR);	271	MODULE_AUTHOR(DRIVER_AUTHOR);
2320	MODULE_DESCRIPTION(DRIVER_DESC);	272	MODULE_DESCRIPTION(DRIVER_DESC);

drivers/vfio/pci/vfio_pci_config.c+35−35

@@ -26,7 +26,7 @@ · 25 unmodified lines
26	#include <linux/vfio.h>	26	#include <linux/vfio.h>
27	#include <linux/slab.h>	27	#include <linux/slab.h>
28		28
29	#include "vfio_pci_private.h"	29	#include <linux/vfio_pci_core.h>
30		30
31	/* Fake capability ID for standard config space */	31	/* Fake capability ID for standard config space */
32	#define PCI_CAP_ID_BASIC 0	32	#define PCI_CAP_ID_BASIC 0
@@ -108,9 +108,9 @@ static const u16 pci_ext_cap_length[PCI_EXT_CAP_ID_MAX + 1] = { · 75 unmodified lines
108	struct perm_bits {	108	struct perm_bits {
109	u8 virt; / read/write virtual data, not hw */	109	u8 virt; / read/write virtual data, not hw */
110	u8 write; / writeable bits */	110	u8 write; / writeable bits */
111	int (readfn)(struct vfio_pci_device vdev, int pos, int count,	111	int (readfn)(struct vfio_pci_core_device vdev, int pos, int count,
112	struct perm_bits perm, int offset, __le32 val);	112	struct perm_bits perm, int offset, __le32 val);
113	int (writefn)(struct vfio_pci_device vdev, int pos, int count,	113	int (writefn)(struct vfio_pci_core_device vdev, int pos, int count,
114	struct perm_bits *perm, int offset, __le32 val);	114	struct perm_bits *perm, int offset, __le32 val);
115	};	115	};
116		116
@@ -171,7 +171,7 @@ static int vfio_user_config_write(struct pci_dev *pdev, int offset, · 54 unmodified lines
171	return ret;	171	return ret;
172	}	172	}
173		173
174	static int vfio_default_config_read(struct vfio_pci_device *vdev, int pos,	174	static int vfio_default_config_read(struct vfio_pci_core_device *vdev, int pos,
175	int count, struct perm_bits *perm,	175	int count, struct perm_bits *perm,
176	int offset, __le32 *val)	176	int offset, __le32 *val)
177	{	177	{
@@ -197,7 +197,7 @@ static int vfio_default_config_read(struct vfio_pci_device *vdev, int pos, · 19 unmodified lines
197	return count;	197	return count;
198	}	198	}
199		199
200	static int vfio_default_config_write(struct vfio_pci_device *vdev, int pos,	200	static int vfio_default_config_write(struct vfio_pci_core_device *vdev, int pos,
201	int count, struct perm_bits *perm,	201	int count, struct perm_bits *perm,
202	int offset, __le32 val)	202	int offset, __le32 val)
203	{	203	{
@@ -244,7 +244,7 @@ static int vfio_default_config_write(struct vfio_pci_device *vdev, int pos, · 40 unmodified lines
244	}	244	}
245		245
246	/* Allow direct read from hardware, except for capability next pointer */	246	/* Allow direct read from hardware, except for capability next pointer */
247	static int vfio_direct_config_read(struct vfio_pci_device *vdev, int pos,	247	static int vfio_direct_config_read(struct vfio_pci_core_device *vdev, int pos,
248	int count, struct perm_bits *perm,	248	int count, struct perm_bits *perm,
249	int offset, __le32 *val)	249	int offset, __le32 *val)
250	{	250	{
@@ -269,7 +269,7 @@ static int vfio_direct_config_read(struct vfio_pci_device *vdev, int pos, · 18 unmodified lines
269	}	269	}
270		270
271	/* Raw access skips any kind of virtualization */	271	/* Raw access skips any kind of virtualization */
272	static int vfio_raw_config_write(struct vfio_pci_device *vdev, int pos,	272	static int vfio_raw_config_write(struct vfio_pci_core_device *vdev, int pos,
273	int count, struct perm_bits *perm,	273	int count, struct perm_bits *perm,
274	int offset, __le32 val)	274	int offset, __le32 val)
275	{	275	{
@@ -282,7 +282,7 @@ static int vfio_raw_config_write(struct vfio_pci_device *vdev, int pos, · 6 unmodified lines
282	return count;	282	return count;
283	}	283	}
284		284
285	static int vfio_raw_config_read(struct vfio_pci_device *vdev, int pos,	285	static int vfio_raw_config_read(struct vfio_pci_core_device *vdev, int pos,
286	int count, struct perm_bits *perm,	286	int count, struct perm_bits *perm,
287	int offset, __le32 *val)	287	int offset, __le32 *val)
288	{	288	{
@@ -296,7 +296,7 @@ static int vfio_raw_config_read(struct vfio_pci_device *vdev, int pos, · 7 unmodified lines
296	}	296	}
297		297
298	/* Virt access uses only virtualization */	298	/* Virt access uses only virtualization */
299	static int vfio_virt_config_write(struct vfio_pci_device *vdev, int pos,	299	static int vfio_virt_config_write(struct vfio_pci_core_device *vdev, int pos,
300	int count, struct perm_bits *perm,	300	int count, struct perm_bits *perm,
301	int offset, __le32 val)	301	int offset, __le32 val)
302	{	302	{
@@ -304,7 +304,7 @@ static int vfio_virt_config_write(struct vfio_pci_device *vdev, int pos, · 1 unmodified line
304	return count;	304	return count;
305	}	305	}
306		306
307	static int vfio_virt_config_read(struct vfio_pci_device *vdev, int pos,	307	static int vfio_virt_config_read(struct vfio_pci_core_device *vdev, int pos,
308	int count, struct perm_bits *perm,	308	int count, struct perm_bits *perm,
309	int offset, __le32 *val)	309	int offset, __le32 *val)
310	{	310	{
@@ -396,7 +396,7 @@ static inline void p_setd(struct perm_bits *p, int off, u32 virt, u32 write) · 85 unmodified lines
396	}	396	}
397		397
398	/* Caller should hold memory_lock semaphore */	398	/* Caller should hold memory_lock semaphore */
399	bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev)	399	bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev)
400	{	400	{
401	struct pci_dev *pdev = vdev->pdev;	401	struct pci_dev *pdev = vdev->pdev;
402	u16 cmd = le16_to_cpu((__le16 )&vdev->vconfig[PCI_COMMAND]);	402	u16 cmd = le16_to_cpu((__le16 )&vdev->vconfig[PCI_COMMAND]);
@@ -413,7 +413,7 @@ bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev) · 10 unmodified lines
413	* Restore the real BARs after we detect a FLR or backdoor reset.	413	* Restore the real BARs after we detect a FLR or backdoor reset.
414	* (backdoor = some device specific technique that we didn't catch)	414	* (backdoor = some device specific technique that we didn't catch)
415	*/	415	*/
416	static void vfio_bar_restore(struct vfio_pci_device *vdev)	416	static void vfio_bar_restore(struct vfio_pci_core_device *vdev)
417	{	417	{
418	struct pci_dev *pdev = vdev->pdev;	418	struct pci_dev *pdev = vdev->pdev;
419	u32 *rbar = vdev->rbar;	419	u32 *rbar = vdev->rbar;
@@ -460,7 +460,7 @@ static __le32 vfio_generate_bar_flags(struct pci_dev *pdev, int bar) · 40 unmodified lines
460	* Pretend we're hardware and tweak the values of the virtual PCI BARs	460	* Pretend we're hardware and tweak the values of the virtual PCI BARs
461	* to reflect the hardware capabilities. This implements BAR sizing.	461	* to reflect the hardware capabilities. This implements BAR sizing.
462	*/	462	*/
463	static void vfio_bar_fixup(struct vfio_pci_device *vdev)	463	static void vfio_bar_fixup(struct vfio_pci_core_device *vdev)
464	{	464	{
465	struct pci_dev *pdev = vdev->pdev;	465	struct pci_dev *pdev = vdev->pdev;
466	int i;	466	int i;
@@ -514,7 +514,7 @@ static void vfio_bar_fixup(struct vfio_pci_device *vdev) · 47 unmodified lines
514	vdev->bardirty = false;	514	vdev->bardirty = false;
515	}	515	}
516		516
517	static int vfio_basic_config_read(struct vfio_pci_device *vdev, int pos,	517	static int vfio_basic_config_read(struct vfio_pci_core_device *vdev, int pos,
518	int count, struct perm_bits *perm,	518	int count, struct perm_bits *perm,
519	int offset, __le32 *val)	519	int offset, __le32 *val)
520	{	520	{
@@ -536,7 +536,7 @@ static int vfio_basic_config_read(struct vfio_pci_device *vdev, int pos, · 15 unmodified lines
536	}	536	}
537		537
538	/* Test whether BARs match the value we think they should contain */	538	/* Test whether BARs match the value we think they should contain */
539	static bool vfio_need_bar_restore(struct vfio_pci_device *vdev)	539	static bool vfio_need_bar_restore(struct vfio_pci_core_device *vdev)
540	{	540	{
541	int i = 0, pos = PCI_BASE_ADDRESS_0, ret;	541	int i = 0, pos = PCI_BASE_ADDRESS_0, ret;
542	u32 bar;	542	u32 bar;
@@ -552,7 +552,7 @@ static bool vfio_need_bar_restore(struct vfio_pci_device *vdev) · 9 unmodified lines
552	return false;	552	return false;
553	}	553	}
554		554
555	static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos,	555	static int vfio_basic_config_write(struct vfio_pci_core_device *vdev, int pos,
556	int count, struct perm_bits *perm,	556	int count, struct perm_bits *perm,
557	int offset, __le32 val)	557	int offset, __le32 val)
558	{	558	{
@@ -692,7 +692,7 @@ static int __init init_pci_cap_basic_perm(struct perm_bits *perm) · 133 unmodified lines
692	return 0;	692	return 0;
693	}	693	}
694		694
695	static int vfio_pm_config_write(struct vfio_pci_device *vdev, int pos,	695	static int vfio_pm_config_write(struct vfio_pci_core_device *vdev, int pos,
696	int count, struct perm_bits *perm,	696	int count, struct perm_bits *perm,
697	int offset, __le32 val)	697	int offset, __le32 val)
698	{	698	{
@@ -747,7 +747,7 @@ static int __init init_pci_cap_pm_perm(struct perm_bits *perm) · 48 unmodified lines
747	return 0;	747	return 0;
748	}	748	}
749		749
750	static int vfio_vpd_config_write(struct vfio_pci_device *vdev, int pos,	750	static int vfio_vpd_config_write(struct vfio_pci_core_device *vdev, int pos,
751	int count, struct perm_bits *perm,	751	int count, struct perm_bits *perm,
752	int offset, __le32 val)	752	int offset, __le32 val)
753	{	753	{
@@ -829,7 +829,7 @@ static int __init init_pci_cap_pcix_perm(struct perm_bits *perm) · 75 unmodified lines
829	return 0;	829	return 0;
830	}	830	}
831		831
832	static int vfio_exp_config_write(struct vfio_pci_device *vdev, int pos,	832	static int vfio_exp_config_write(struct vfio_pci_core_device *vdev, int pos,
833	int count, struct perm_bits *perm,	833	int count, struct perm_bits *perm,
834	int offset, __le32 val)	834	int offset, __le32 val)
835	{	835	{
@@ -913,7 +913,7 @@ static int __init init_pci_cap_exp_perm(struct perm_bits *perm) · 77 unmodified lines
913	return 0;	913	return 0;
914	}	914	}
915		915
916	static int vfio_af_config_write(struct vfio_pci_device *vdev, int pos,	916	static int vfio_af_config_write(struct vfio_pci_core_device *vdev, int pos,
917	int count, struct perm_bits *perm,	917	int count, struct perm_bits *perm,
918	int offset, __le32 val)	918	int offset, __le32 val)
919	{	919	{
@@ -1072,7 +1072,7 @@ int __init vfio_pci_init_perm_bits(void) · 152 unmodified lines
1072	return ret;	1072	return ret;
1073	}	1073	}
1074		1074
1075	static int vfio_find_cap_start(struct vfio_pci_device *vdev, int pos)	1075	static int vfio_find_cap_start(struct vfio_pci_core_device *vdev, int pos)
1076	{	1076	{
1077	u8 cap;	1077	u8 cap;
1078	int base = (pos >= PCI_CFG_SPACE_SIZE) ? PCI_CFG_SPACE_SIZE :	1078	int base = (pos >= PCI_CFG_SPACE_SIZE) ? PCI_CFG_SPACE_SIZE :
@@ -1089,7 +1089,7 @@ static int vfio_find_cap_start(struct vfio_pci_device *vdev, int pos) · 10 unmodified lines
1089	return pos;	1089	return pos;
1090	}	1090	}
1091		1091
1092	static int vfio_msi_config_read(struct vfio_pci_device *vdev, int pos,	1092	static int vfio_msi_config_read(struct vfio_pci_core_device *vdev, int pos,
1093	int count, struct perm_bits *perm,	1093	int count, struct perm_bits *perm,
1094	int offset, __le32 *val)	1094	int offset, __le32 *val)
1095	{	1095	{
@@ -1109,7 +1109,7 @@ static int vfio_msi_config_read(struct vfio_pci_device *vdev, int pos, · 13 unmodified lines
1109	return vfio_default_config_read(vdev, pos, count, perm, offset, val);	1109	return vfio_default_config_read(vdev, pos, count, perm, offset, val);
1110	}	1110	}
1111		1111
1112	static int vfio_msi_config_write(struct vfio_pci_device *vdev, int pos,	1112	static int vfio_msi_config_write(struct vfio_pci_core_device *vdev, int pos,
1113	int count, struct perm_bits *perm,	1113	int count, struct perm_bits *perm,
1114	int offset, __le32 val)	1114	int offset, __le32 val)
1115	{	1115	{
@@ -1189,7 +1189,7 @@ static int init_pci_cap_msi_perm(struct perm_bits *perm, int len, u16 flags) · 73 unmodified lines
1189	}	1189	}
1190		1190
1191	/* Determine MSI CAP field length; initialize msi_perms on 1st call per vdev */	1191	/* Determine MSI CAP field length; initialize msi_perms on 1st call per vdev */
1192	static int vfio_msi_cap_len(struct vfio_pci_device *vdev, u8 pos)	1192	static int vfio_msi_cap_len(struct vfio_pci_core_device *vdev, u8 pos)
1193	{	1193	{
1194	struct pci_dev *pdev = vdev->pdev;	1194	struct pci_dev *pdev = vdev->pdev;
1195	int len, ret;	1195	int len, ret;
@@ -1222,7 +1222,7 @@ static int vfio_msi_cap_len(struct vfio_pci_device *vdev, u8 pos) · 26 unmodified lines
1222	}	1222	}
1223		1223
1224	/* Determine extended capability length for VC (2 & 9) and MFVC */	1224	/* Determine extended capability length for VC (2 & 9) and MFVC */
1225	static int vfio_vc_cap_len(struct vfio_pci_device *vdev, u16 pos)	1225	static int vfio_vc_cap_len(struct vfio_pci_core_device *vdev, u16 pos)
1226	{	1226	{
1227	struct pci_dev *pdev = vdev->pdev;	1227	struct pci_dev *pdev = vdev->pdev;
1228	u32 tmp;	1228	u32 tmp;
@@ -1263,7 +1263,7 @@ static int vfio_vc_cap_len(struct vfio_pci_device *vdev, u16 pos) · 34 unmodified lines
1263	return len;	1263	return len;
1264	}	1264	}
1265		1265
1266	static int vfio_cap_len(struct vfio_pci_device *vdev, u8 cap, u8 pos)	1266	static int vfio_cap_len(struct vfio_pci_core_device *vdev, u8 cap, u8 pos)
1267	{	1267	{
1268	struct pci_dev *pdev = vdev->pdev;	1268	struct pci_dev *pdev = vdev->pdev;
1269	u32 dword;	1269	u32 dword;
@@ -1338,7 +1338,7 @@ static int vfio_cap_len(struct vfio_pci_device *vdev, u8 cap, u8 pos) · 68 unmodified lines
1338	return 0;	1338	return 0;
1339	}	1339	}
1340		1340
1341	static int vfio_ext_cap_len(struct vfio_pci_device *vdev, u16 ecap, u16 epos)	1341	static int vfio_ext_cap_len(struct vfio_pci_core_device *vdev, u16 ecap, u16 epos)
1342	{	1342	{
1343	struct pci_dev *pdev = vdev->pdev;	1343	struct pci_dev *pdev = vdev->pdev;
1344	u8 byte;	1344	u8 byte;
@@ -1412,7 +1412,7 @@ static int vfio_ext_cap_len(struct vfio_pci_device *vdev, u16 ecap, u16 epos) · 67 unmodified lines
1412	return 0;	1412	return 0;
1413	}	1413	}
1414		1414
1415	static int vfio_fill_vconfig_bytes(struct vfio_pci_device *vdev,	1415	static int vfio_fill_vconfig_bytes(struct vfio_pci_core_device *vdev,
1416	int offset, int size)	1416	int offset, int size)
1417	{	1417	{
1418	struct pci_dev *pdev = vdev->pdev;	1418	struct pci_dev *pdev = vdev->pdev;
@@ -1459,7 +1459,7 @@ static int vfio_fill_vconfig_bytes(struct vfio_pci_device *vdev, · 40 unmodified lines
1459	return ret;	1459	return ret;
1460	}	1460	}
1461		1461
1462	static int vfio_cap_init(struct vfio_pci_device *vdev)	1462	static int vfio_cap_init(struct vfio_pci_core_device *vdev)
1463	{	1463	{
1464	struct pci_dev *pdev = vdev->pdev;	1464	struct pci_dev *pdev = vdev->pdev;
1465	u8 *map = vdev->pci_config_map;	1465	u8 *map = vdev->pci_config_map;
@@ -1549,7 +1549,7 @@ static int vfio_cap_init(struct vfio_pci_device *vdev) · 83 unmodified lines
1549	return 0;	1549	return 0;
1550	}	1550	}
1551		1551
1552	static int vfio_ecap_init(struct vfio_pci_device *vdev)	1552	static int vfio_ecap_init(struct vfio_pci_core_device *vdev)
1553	{	1553	{
1554	struct pci_dev *pdev = vdev->pdev;	1554	struct pci_dev *pdev = vdev->pdev;
1555	u8 *map = vdev->pci_config_map;	1555	u8 *map = vdev->pci_config_map;
@@ -1669,7 +1669,7 @@ static const struct pci_device_id known_bogus_vf_intx_pin[] = { · 113 unmodified lines
1669	* for each area requiring emulated bits, but the array of pointers	1669	* for each area requiring emulated bits, but the array of pointers
1670	* would be comparable in size (at least for standard config space).	1670	* would be comparable in size (at least for standard config space).
1671	*/	1671	*/
1672	int vfio_config_init(struct vfio_pci_device *vdev)	1672	int vfio_config_init(struct vfio_pci_core_device *vdev)
1673	{	1673	{
1674	struct pci_dev *pdev = vdev->pdev;	1674	struct pci_dev *pdev = vdev->pdev;
1675	u8 map, vconfig;	1675	u8 map, vconfig;
@@ -1773,7 +1773,7 @@ out: · 97 unmodified lines
1773	return pcibios_err_to_errno(ret);	1773	return pcibios_err_to_errno(ret);
1774	}	1774	}
1775		1775
1776	void vfio_config_free(struct vfio_pci_device *vdev)	1776	void vfio_config_free(struct vfio_pci_core_device *vdev)
1777	{	1777	{
1778	kfree(vdev->vconfig);	1778	kfree(vdev->vconfig);
1779	vdev->vconfig = NULL;	1779	vdev->vconfig = NULL;
@@ -1790,7 +1790,7 @@ void vfio_config_free(struct vfio_pci_device *vdev) · 10 unmodified lines
1790	* Find the remaining number of bytes in a dword that match the given	1790	* Find the remaining number of bytes in a dword that match the given
1791	* position. Stop at either the end of the capability or the dword boundary.	1791	* position. Stop at either the end of the capability or the dword boundary.
1792	*/	1792	*/
1793	static size_t vfio_pci_cap_remaining_dword(struct vfio_pci_device *vdev,	1793	static size_t vfio_pci_cap_remaining_dword(struct vfio_pci_core_device *vdev,
1794	loff_t pos)	1794	loff_t pos)
1795	{	1795	{
1796	u8 cap = vdev->pci_config_map[pos];	1796	u8 cap = vdev->pci_config_map[pos];
@@ -1802,7 +1802,7 @@ static size_t vfio_pci_cap_remaining_dword(struct vfio_pci_device *vdev, · 5 unmodified lines
1802	return i;	1802	return i;
1803	}	1803	}
1804		1804
1805	static ssize_t vfio_config_do_rw(struct vfio_pci_device vdev, char __user buf,	1805	static ssize_t vfio_config_do_rw(struct vfio_pci_core_device vdev, char __user buf,
1806	size_t count, loff_t *ppos, bool iswrite)	1806	size_t count, loff_t *ppos, bool iswrite)
1807	{	1807	{
1808	struct pci_dev *pdev = vdev->pdev;	1808	struct pci_dev *pdev = vdev->pdev;
@@ -1885,7 +1885,7 @@ static ssize_t vfio_config_do_rw(struct vfio_pci_device vdev, char __user buf, · 76 unmodified lines
1885	return ret;	1885	return ret;
1886	}	1886	}
1887		1887
1888	ssize_t vfio_pci_config_rw(struct vfio_pci_device vdev, char __user buf,	1888	ssize_t vfio_pci_config_rw(struct vfio_pci_core_device vdev, char __user buf,
1889	size_t count, loff_t *ppos, bool iswrite)	1889	size_t count, loff_t *ppos, bool iswrite)
1890	{	1890	{
1891	size_t done = 0;	1891	size_t done = 0;

drivers/vfio/pci/vfio_pci_core.c+2158−0

@@ -0,0 +1,2158 @@
	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/*
	3	* Copyright (C) 2012 Red Hat, Inc. All rights reserved.
	4	* Author: Alex Williamson <alex.williamson@redhat.com>
	5	*
	6	* Derived from original vfio:
	7	* Copyright 2010 Cisco Systems, Inc. All rights reserved.
	8	* Author: Tom Lyon, pugs@cisco.com
	9	*/
	10
	11	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	12
	13	#include <linux/device.h>
	14	#include <linux/eventfd.h>
	15	#include <linux/file.h>
	16	#include <linux/interrupt.h>
	17	#include <linux/iommu.h>
	18	#include <linux/module.h>
	19	#include <linux/mutex.h>
	20	#include <linux/notifier.h>
	21	#include <linux/pci.h>
	22	#include <linux/pm_runtime.h>
	23	#include <linux/slab.h>
	24	#include <linux/types.h>
	25	#include <linux/uaccess.h>
	26	#include <linux/vgaarb.h>
	27	#include <linux/nospec.h>
	28	#include <linux/sched/mm.h>
	29
	30	#include <linux/vfio_pci_core.h>
	31
	32	#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
	33	#define DRIVER_DESC "core driver for VFIO based PCI devices"
	34
	35	static bool nointxmask;
	36	static bool disable_vga;
	37	static bool disable_idle_d3;
	38
	39	static inline bool vfio_vga_disabled(void)
	40	{
	41	#ifdef CONFIG_VFIO_PCI_VGA
	42	return disable_vga;
	43	#else
	44	return true;
	45	#endif
	46	}
	47
	48	/*
	49	* Our VGA arbiter participation is limited since we don't know anything
	50	* about the device itself. However, if the device is the only VGA device
	51	* downstream of a bridge and VFIO VGA support is disabled, then we can
	52	* safely return legacy VGA IO and memory as not decoded since the user
	53	* has no way to get to it and routing can be disabled externally at the
	54	* bridge.
	55	*/
	56	static unsigned int vfio_pci_set_vga_decode(void *opaque, bool single_vga)
	57	{
	58	struct vfio_pci_core_device *vdev = opaque;
	59	struct pci_dev tmp = NULL, pdev = vdev->pdev;
	60	unsigned char max_busnr;
	61	unsigned int decodes;
	62
	63	if (single_vga \|\| !vfio_vga_disabled() \|\| pci_is_root_bus(pdev->bus))
	64	return VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM \|
	65	VGA_RSRC_LEGACY_IO \| VGA_RSRC_LEGACY_MEM;
	66
	67	max_busnr = pci_bus_max_busnr(pdev->bus);
	68	decodes = VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM;
	69
	70	while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) {
	71	if (tmp == pdev \|\|
	72	pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) \|\|
	73	pci_is_root_bus(tmp->bus))
	74	continue;
	75
	76	if (tmp->bus->number >= pdev->bus->number &&
	77	tmp->bus->number <= max_busnr) {
	78	pci_dev_put(tmp);
	79	decodes \|= VGA_RSRC_LEGACY_IO \| VGA_RSRC_LEGACY_MEM;
	80	break;
	81	}
	82	}
	83
	84	return decodes;
	85	}
	86
	87	static void vfio_pci_probe_mmaps(struct vfio_pci_core_device *vdev)
	88	{
	89	struct resource *res;
	90	int i;
	91	struct vfio_pci_dummy_resource *dummy_res;
	92
	93	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
	94	int bar = i + PCI_STD_RESOURCES;
	95
	96	res = &vdev->pdev->resource[bar];
	97
	98	if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP))
	99	goto no_mmap;
	100
	101	if (!(res->flags & IORESOURCE_MEM))
	102	goto no_mmap;
	103
	104	/*
	105	* The PCI core shouldn't set up a resource with a
	106	* type but zero size. But there may be bugs that
	107	* cause us to do that.
	108	*/
	109	if (!resource_size(res))
	110	goto no_mmap;
	111
	112	if (resource_size(res) >= PAGE_SIZE) {
	113	vdev->bar_mmap_supported[bar] = true;
	114	continue;
	115	}
	116
	117	if (!(res->start & ~PAGE_MASK)) {
	118	/*
	119	* Add a dummy resource to reserve the remainder
	120	* of the exclusive page in case that hot-add
	121	* device's bar is assigned into it.
	122	*/
	123	dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL);
	124	if (dummy_res == NULL)
	125	goto no_mmap;
	126
	127	dummy_res->resource.name = "vfio sub-page reserved";
	128	dummy_res->resource.start = res->end + 1;
	129	dummy_res->resource.end = res->start + PAGE_SIZE - 1;
	130	dummy_res->resource.flags = res->flags;
	131	if (request_resource(res->parent,
	132	&dummy_res->resource)) {
	133	kfree(dummy_res);
	134	goto no_mmap;
	135	}
	136	dummy_res->index = bar;
	137	list_add(&dummy_res->res_next,
	138	&vdev->dummy_resources_list);
	139	vdev->bar_mmap_supported[bar] = true;
	140	continue;
	141	}
	142	/*
	143	* Here we don't handle the case when the BAR is not page
	144	* aligned because we can't expect the BAR will be
	145	* assigned into the same location in a page in guest
	146	* when we passthrough the BAR. And it's hard to access
	147	* this BAR in userspace because we have no way to get
	148	* the BAR's location in a page.
	149	*/
	150	no_mmap:
	151	vdev->bar_mmap_supported[bar] = false;
	152	}
	153	}
	154
	155	struct vfio_pci_group_info;
	156	static bool vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set);
	157	static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
	158	struct vfio_pci_group_info *groups);
	159
	160	/*
	161	* INTx masking requires the ability to disable INTx signaling via PCI_COMMAND
	162	* _and_ the ability detect when the device is asserting INTx via PCI_STATUS.
	163	* If a device implements the former but not the latter we would typically
	164	* expect broken_intx_masking be set and require an exclusive interrupt.
	165	* However since we do have control of the device's ability to assert INTx,
	166	* we can instead pretend that the device does not implement INTx, virtualizing
	167	* the pin register to report zero and maintaining DisINTx set on the host.
	168	*/
	169	static bool vfio_pci_nointx(struct pci_dev *pdev)
	170	{
	171	switch (pdev->vendor) {
	172	case PCI_VENDOR_ID_INTEL:
	173	switch (pdev->device) {
	174	/* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */
	175	case 0x1572:
	176	case 0x1574:
	177	case 0x1580 ... 0x1581:
	178	case 0x1583 ... 0x158b:
	179	case 0x37d0 ... 0x37d2:
	180	/* X550 */
	181	case 0x1563:
	182	return true;
	183	default:
	184	return false;
	185	}
	186	}
	187
	188	return false;
	189	}
	190
	191	static void vfio_pci_probe_power_state(struct vfio_pci_core_device *vdev)
	192	{
	193	struct pci_dev *pdev = vdev->pdev;
	194	u16 pmcsr;
	195
	196	if (!pdev->pm_cap)
	197	return;
	198
	199	pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
	200
	201	vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
	202	}
	203
	204	/*
	205	* pci_set_power_state() wrapper handling devices which perform a soft reset on
	206	* D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev,
	207	* restore when returned to D0. Saved separately from pci_saved_state for use
	208	* by PM capability emulation and separately from pci_dev internal saved state
	209	* to avoid it being overwritten and consumed around other resets.
	210	*/
	211	int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t state)
	212	{
	213	struct pci_dev *pdev = vdev->pdev;
	214	bool needs_restore = false, needs_save = false;
	215	int ret;
	216
	217	if (vdev->needs_pm_restore) {
	218	if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) {
	219	pci_save_state(pdev);
	220	needs_save = true;
	221	}
	222
	223	if (pdev->current_state >= PCI_D3hot && state <= PCI_D0)
	224	needs_restore = true;
	225	}
	226
	227	ret = pci_set_power_state(pdev, state);
	228
	229	if (!ret) {
	230	/* D3 might be unsupported via quirk, skip unless in D3 */
	231	if (needs_save && pdev->current_state >= PCI_D3hot) {
	232	vdev->pm_save = pci_store_saved_state(pdev);
	233	} else if (needs_restore) {
	234	pci_load_and_free_saved_state(pdev, &vdev->pm_save);
	235	pci_restore_state(pdev);
	236	}
	237	}
	238
	239	return ret;
	240	}
	241
	242	int vfio_pci_core_enable(struct vfio_pci_core_device *vdev)
	243	{
	244	struct pci_dev *pdev = vdev->pdev;
	245	int ret;
	246	u16 cmd;
	247	u8 msix_pos;
	248
	249	vfio_pci_set_power_state(vdev, PCI_D0);
	250
	251	/* Don't allow our initial saved state to include busmaster */
	252	pci_clear_master(pdev);
	253
	254	ret = pci_enable_device(pdev);
	255	if (ret)
	256	return ret;
	257
	258	/* If reset fails because of the device lock, fail this path entirely */
	259	ret = pci_try_reset_function(pdev);
	260	if (ret == -EAGAIN) {
	261	pci_disable_device(pdev);
	262	return ret;
	263	}
	264
	265	vdev->reset_works = !ret;
	266	pci_save_state(pdev);
	267	vdev->pci_saved_state = pci_store_saved_state(pdev);
	268	if (!vdev->pci_saved_state)
	269	pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__);
	270
	271	if (likely(!nointxmask)) {
	272	if (vfio_pci_nointx(pdev)) {
	273	pci_info(pdev, "Masking broken INTx support\n");
	274	vdev->nointx = true;
	275	pci_intx(pdev, 0);
	276	} else
	277	vdev->pci_2_3 = pci_intx_mask_supported(pdev);
	278	}
	279
	280	pci_read_config_word(pdev, PCI_COMMAND, &cmd);
	281	if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
	282	cmd &= ~PCI_COMMAND_INTX_DISABLE;
	283	pci_write_config_word(pdev, PCI_COMMAND, cmd);
	284	}
	285
	286	ret = vfio_config_init(vdev);
	287	if (ret) {
	288	kfree(vdev->pci_saved_state);
	289	vdev->pci_saved_state = NULL;
	290	pci_disable_device(pdev);
	291	return ret;
	292	}
	293
	294	msix_pos = pdev->msix_cap;
	295	if (msix_pos) {
	296	u16 flags;
	297	u32 table;
	298
	299	pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);
	300	pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);
	301
	302	vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
	303	vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
	304	vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
	305	} else
	306	vdev->msix_bar = 0xFF;
	307
	308	if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))
	309	vdev->has_vga = true;
	310
	311
	312	return 0;
	313	}
	314	EXPORT_SYMBOL_GPL(vfio_pci_core_enable);
	315
	316	void vfio_pci_core_disable(struct vfio_pci_core_device *vdev)
	317	{
	318	struct pci_dev *pdev = vdev->pdev;
	319	struct vfio_pci_dummy_resource dummy_res, tmp;
	320	struct vfio_pci_ioeventfd ioeventfd, ioeventfd_tmp;
	321	int i, bar;
	322
	323	/* For needs_reset */
	324	lockdep_assert_held(&vdev->vdev.dev_set->lock);
	325
	326	/* Stop the device from further DMA */
	327	pci_clear_master(pdev);
	328
	329	vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE \|
	330	VFIO_IRQ_SET_ACTION_TRIGGER,
	331	vdev->irq_type, 0, 0, NULL);
	332
	333	/* Device closed, don't need mutex here */
	334	list_for_each_entry_safe(ioeventfd, ioeventfd_tmp,
	335	&vdev->ioeventfds_list, next) {
	336	vfio_virqfd_disable(&ioeventfd->virqfd);
	337	list_del(&ioeventfd->next);
	338	kfree(ioeventfd);
	339	}
	340	vdev->ioeventfds_nr = 0;
	341
	342	vdev->virq_disabled = false;
	343
	344	for (i = 0; i < vdev->num_regions; i++)
	345	vdev->region[i].ops->release(vdev, &vdev->region[i]);
	346
	347	vdev->num_regions = 0;
	348	kfree(vdev->region);
	349	vdev->region = NULL; /* don't krealloc a freed pointer */
	350
	351	vfio_config_free(vdev);
	352
	353	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
	354	bar = i + PCI_STD_RESOURCES;
	355	if (!vdev->barmap[bar])
	356	continue;
	357	pci_iounmap(pdev, vdev->barmap[bar]);
	358	pci_release_selected_regions(pdev, 1 << bar);
	359	vdev->barmap[bar] = NULL;
	360	}
	361
	362	list_for_each_entry_safe(dummy_res, tmp,
	363	&vdev->dummy_resources_list, res_next) {
	364	list_del(&dummy_res->res_next);
	365	release_resource(&dummy_res->resource);
	366	kfree(dummy_res);
	367	}
	368
	369	vdev->needs_reset = true;
	370
	371	/*
	372	* If we have saved state, restore it. If we can reset the device,
	373	* even better. Resetting with current state seems better than
	374	* nothing, but saving and restoring current state without reset
	375	* is just busy work.
	376	*/
	377	if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
	378	pci_info(pdev, "%s: Couldn't reload saved state\n", __func__);
	379
	380	if (!vdev->reset_works)
	381	goto out;
	382
	383	pci_save_state(pdev);
	384	}
	385
	386	/*
	387	* Disable INTx and MSI, presumably to avoid spurious interrupts
	388	* during reset. Stolen from pci_reset_function()
	389	*/
	390	pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
	391
	392	/*
	393	* Try to get the locks ourselves to prevent a deadlock. The
	394	* success of this is dependent on being able to lock the device,
	395	* which is not always possible.
	396	* We can not use the "try" reset interface here, which will
	397	* overwrite the previously restored configuration information.
	398	*/
	399	if (vdev->reset_works && pci_dev_trylock(pdev)) {
	400	if (!__pci_reset_function_locked(pdev))
	401	vdev->needs_reset = false;
	402	pci_dev_unlock(pdev);
	403	}
	404
	405	pci_restore_state(pdev);
	406	out:
	407	pci_disable_device(pdev);
	408
	409	if (!vfio_pci_dev_set_try_reset(vdev->vdev.dev_set) && !disable_idle_d3)
	410	vfio_pci_set_power_state(vdev, PCI_D3hot);
	411	}
	412	EXPORT_SYMBOL_GPL(vfio_pci_core_disable);
	413
	414	static struct vfio_pci_core_device get_pf_vdev(struct vfio_pci_core_device vdev)
	415	{
	416	struct pci_dev *physfn = pci_physfn(vdev->pdev);
	417	struct vfio_device *pf_dev;
	418
	419	if (!vdev->pdev->is_virtfn)
	420	return NULL;
	421
	422	pf_dev = vfio_device_get_from_dev(&physfn->dev);
	423	if (!pf_dev)
	424	return NULL;
	425
	426	if (pci_dev_driver(physfn) != pci_dev_driver(vdev->pdev)) {
	427	vfio_device_put(pf_dev);
	428	return NULL;
	429	}
	430
	431	return container_of(pf_dev, struct vfio_pci_core_device, vdev);
	432	}
	433
	434	static void vfio_pci_vf_token_user_add(struct vfio_pci_core_device *vdev, int val)
	435	{
	436	struct vfio_pci_core_device *pf_vdev = get_pf_vdev(vdev);
	437
	438	if (!pf_vdev)
	439	return;
	440
	441	mutex_lock(&pf_vdev->vf_token->lock);
	442	pf_vdev->vf_token->users += val;
	443	WARN_ON(pf_vdev->vf_token->users < 0);
	444	mutex_unlock(&pf_vdev->vf_token->lock);
	445
	446	vfio_device_put(&pf_vdev->vdev);
	447	}
	448
	449	void vfio_pci_core_close_device(struct vfio_device *core_vdev)
	450	{
	451	struct vfio_pci_core_device *vdev =
	452	container_of(core_vdev, struct vfio_pci_core_device, vdev);
	453
	454	vfio_pci_vf_token_user_add(vdev, -1);
	455	vfio_spapr_pci_eeh_release(vdev->pdev);
	456	vfio_pci_core_disable(vdev);
	457
	458	mutex_lock(&vdev->igate);
	459	if (vdev->err_trigger) {
	460	eventfd_ctx_put(vdev->err_trigger);
	461	vdev->err_trigger = NULL;
	462	}
	463	if (vdev->req_trigger) {
	464	eventfd_ctx_put(vdev->req_trigger);
	465	vdev->req_trigger = NULL;
	466	}
	467	mutex_unlock(&vdev->igate);
	468	}
	469	EXPORT_SYMBOL_GPL(vfio_pci_core_close_device);
	470
	471	void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev)
	472	{
	473	vfio_pci_probe_mmaps(vdev);
	474	vfio_spapr_pci_eeh_open(vdev->pdev);
	475	vfio_pci_vf_token_user_add(vdev, 1);
	476	}
	477	EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable);
	478
	479	static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_type)
	480	{
	481	if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {
	482	u8 pin;
	483
	484	if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) \|\|
	485	vdev->nointx \|\| vdev->pdev->is_virtfn)
	486	return 0;
	487
	488	pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);
	489
	490	return pin ? 1 : 0;
	491	} else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {
	492	u8 pos;
	493	u16 flags;
	494
	495	pos = vdev->pdev->msi_cap;
	496	if (pos) {
	497	pci_read_config_word(vdev->pdev,
	498	pos + PCI_MSI_FLAGS, &flags);
	499	return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);
	500	}
	501	} else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {
	502	u8 pos;
	503	u16 flags;
	504
	505	pos = vdev->pdev->msix_cap;
	506	if (pos) {
	507	pci_read_config_word(vdev->pdev,
	508	pos + PCI_MSIX_FLAGS, &flags);
	509
	510	return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;
	511	}
	512	} else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) {
	513	if (pci_is_pcie(vdev->pdev))
	514	return 1;
	515	} else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) {
	516	return 1;
	517	}
	518
	519	return 0;
	520	}
	521
	522	static int vfio_pci_count_devs(struct pci_dev pdev, void data)
	523	{
	524	((int )data)++;
	525	return 0;
	526	}
	527
	528	struct vfio_pci_fill_info {
	529	int max;
	530	int cur;
	531	struct vfio_pci_dependent_device *devices;
	532	};
	533
	534	static int vfio_pci_fill_devs(struct pci_dev pdev, void data)
	535	{
	536	struct vfio_pci_fill_info *fill = data;
	537	struct iommu_group *iommu_group;
	538
	539	if (fill->cur == fill->max)
	540	return -EAGAIN; /* Something changed, try again */
	541
	542	iommu_group = iommu_group_get(&pdev->dev);
	543	if (!iommu_group)
	544	return -EPERM; /* Cannot reset non-isolated devices */
	545
	546	fill->devices[fill->cur].group_id = iommu_group_id(iommu_group);
	547	fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus);
	548	fill->devices[fill->cur].bus = pdev->bus->number;
	549	fill->devices[fill->cur].devfn = pdev->devfn;
	550	fill->cur++;
	551	iommu_group_put(iommu_group);
	552	return 0;
	553	}
	554
	555	struct vfio_pci_group_info {
	556	int count;
	557	struct vfio_group **groups;
	558	};
	559
	560	static bool vfio_pci_dev_below_slot(struct pci_dev pdev, struct pci_slot slot)
	561	{
	562	for (; pdev; pdev = pdev->bus->self)
	563	if (pdev->bus == slot->bus)
	564	return (pdev->slot == slot);
	565	return false;
	566	}
	567
	568	struct vfio_pci_walk_info {
	569	int (fn)(struct pci_dev , void *data);
	570	void *data;
	571	struct pci_dev *pdev;
	572	bool slot;
	573	int ret;
	574	};
	575
	576	static int vfio_pci_walk_wrapper(struct pci_dev pdev, void data)
	577	{
	578	struct vfio_pci_walk_info *walk = data;
	579
	580	if (!walk->slot \|\| vfio_pci_dev_below_slot(pdev, walk->pdev->slot))
	581	walk->ret = walk->fn(pdev, walk->data);
	582
	583	return walk->ret;
	584	}
	585
	586	static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,
	587	int (fn)(struct pci_dev ,
	588	void data), void data,
	589	bool slot)
	590	{
	591	struct vfio_pci_walk_info walk = {
	592	.fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,
	593	};
	594
	595	pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);
	596
	597	return walk.ret;
	598	}
	599
	600	static int msix_mmappable_cap(struct vfio_pci_core_device *vdev,
	601	struct vfio_info_cap *caps)
	602	{
	603	struct vfio_info_cap_header header = {
	604	.id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE,
	605	.version = 1
	606	};
	607
	608	return vfio_info_add_capability(caps, &header, sizeof(header));
	609	}
	610
	611	int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev,
	612	unsigned int type, unsigned int subtype,
	613	const struct vfio_pci_regops *ops,
	614	size_t size, u32 flags, void *data)
	615	{
	616	struct vfio_pci_region *region;
	617
	618	region = krealloc(vdev->region,
	619	(vdev->num_regions + 1) * sizeof(*region),
	620	GFP_KERNEL);
	621	if (!region)
	622	return -ENOMEM;
	623
	624	vdev->region = region;
	625	vdev->region[vdev->num_regions].type = type;
	626	vdev->region[vdev->num_regions].subtype = subtype;
	627	vdev->region[vdev->num_regions].ops = ops;
	628	vdev->region[vdev->num_regions].size = size;
	629	vdev->region[vdev->num_regions].flags = flags;
	630	vdev->region[vdev->num_regions].data = data;
	631
	632	vdev->num_regions++;
	633
	634	return 0;
	635	}
	636	EXPORT_SYMBOL_GPL(vfio_pci_register_dev_region);
	637
	638	long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
	639	unsigned long arg)
	640	{
	641	struct vfio_pci_core_device *vdev =
	642	container_of(core_vdev, struct vfio_pci_core_device, vdev);
	643	unsigned long minsz;
	644
	645	if (cmd == VFIO_DEVICE_GET_INFO) {
	646	struct vfio_device_info info;
	647	struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
	648	unsigned long capsz;
	649	int ret;
	650
	651	minsz = offsetofend(struct vfio_device_info, num_irqs);
	652
	653	/* For backward compatibility, cannot require this */
	654	capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset);
	655
	656	if (copy_from_user(&info, (void __user *)arg, minsz))
	657	return -EFAULT;
	658
	659	if (info.argsz < minsz)
	660	return -EINVAL;
	661
	662	if (info.argsz >= capsz) {
	663	minsz = capsz;
	664	info.cap_offset = 0;
	665	}
	666
	667	info.flags = VFIO_DEVICE_FLAGS_PCI;
	668
	669	if (vdev->reset_works)
	670	info.flags \|= VFIO_DEVICE_FLAGS_RESET;
	671
	672	info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
	673	info.num_irqs = VFIO_PCI_NUM_IRQS;
	674
	675	ret = vfio_pci_info_zdev_add_caps(vdev, &caps);
	676	if (ret && ret != -ENODEV) {
	677	pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n");
	678	return ret;
	679	}
	680
	681	if (caps.size) {
	682	info.flags \|= VFIO_DEVICE_FLAGS_CAPS;
	683	if (info.argsz < sizeof(info) + caps.size) {
	684	info.argsz = sizeof(info) + caps.size;
	685	} else {
	686	vfio_info_cap_shift(&caps, sizeof(info));
	687	if (copy_to_user((void __user *)arg +
	688	sizeof(info), caps.buf,
	689	caps.size)) {
	690	kfree(caps.buf);
	691	return -EFAULT;
	692	}
	693	info.cap_offset = sizeof(info);
	694	}
	695
	696	kfree(caps.buf);
	697	}
	698
	699	return copy_to_user((void __user *)arg, &info, minsz) ?
	700	-EFAULT : 0;
	701
	702	} else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
	703	struct pci_dev *pdev = vdev->pdev;
	704	struct vfio_region_info info;
	705	struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
	706	int i, ret;
	707
	708	minsz = offsetofend(struct vfio_region_info, offset);
	709
	710	if (copy_from_user(&info, (void __user *)arg, minsz))
	711	return -EFAULT;
	712
	713	if (info.argsz < minsz)
	714	return -EINVAL;
	715
	716	switch (info.index) {
	717	case VFIO_PCI_CONFIG_REGION_INDEX:
	718	info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
	719	info.size = pdev->cfg_size;
	720	info.flags = VFIO_REGION_INFO_FLAG_READ \|
	721	VFIO_REGION_INFO_FLAG_WRITE;
	722	break;
	723	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
	724	info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
	725	info.size = pci_resource_len(pdev, info.index);
	726	if (!info.size) {
	727	info.flags = 0;
	728	break;
	729	}
	730
	731	info.flags = VFIO_REGION_INFO_FLAG_READ \|
	732	VFIO_REGION_INFO_FLAG_WRITE;
	733	if (vdev->bar_mmap_supported[info.index]) {
	734	info.flags \|= VFIO_REGION_INFO_FLAG_MMAP;
	735	if (info.index == vdev->msix_bar) {
	736	ret = msix_mmappable_cap(vdev, &caps);
	737	if (ret)
	738	return ret;
	739	}
	740	}
	741
	742	break;
	743	case VFIO_PCI_ROM_REGION_INDEX:
	744	{
	745	void __iomem *io;
	746	size_t size;
	747	u16 cmd;
	748
	749	info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
	750	info.flags = 0;
	751
	752	/* Report the BAR size, not the ROM size */
	753	info.size = pci_resource_len(pdev, info.index);
	754	if (!info.size) {
	755	/* Shadow ROMs appear as PCI option ROMs */
	756	if (pdev->resource[PCI_ROM_RESOURCE].flags &
	757	IORESOURCE_ROM_SHADOW)
	758	info.size = 0x20000;
	759	else
	760	break;
	761	}
	762
	763	/*
	764	* Is it really there? Enable memory decode for
	765	* implicit access in pci_map_rom().
	766	*/
	767	cmd = vfio_pci_memory_lock_and_enable(vdev);
	768	io = pci_map_rom(pdev, &size);
	769	if (io) {
	770	info.flags = VFIO_REGION_INFO_FLAG_READ;
	771	pci_unmap_rom(pdev, io);
	772	} else {
	773	info.size = 0;
	774	}
	775	vfio_pci_memory_unlock_and_restore(vdev, cmd);
	776
	777	break;
	778	}
	779	case VFIO_PCI_VGA_REGION_INDEX:
	780	if (!vdev->has_vga)
	781	return -EINVAL;
	782
	783	info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
	784	info.size = 0xc0000;
	785	info.flags = VFIO_REGION_INFO_FLAG_READ \|
	786	VFIO_REGION_INFO_FLAG_WRITE;
	787
	788	break;
	789	default:
	790	{
	791	struct vfio_region_info_cap_type cap_type = {
	792	.header.id = VFIO_REGION_INFO_CAP_TYPE,
	793	.header.version = 1 };
	794
	795	if (info.index >=
	796	VFIO_PCI_NUM_REGIONS + vdev->num_regions)
	797	return -EINVAL;
	798	info.index = array_index_nospec(info.index,
	799	VFIO_PCI_NUM_REGIONS +
	800	vdev->num_regions);
	801
	802	i = info.index - VFIO_PCI_NUM_REGIONS;
	803
	804	info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
	805	info.size = vdev->region[i].size;
	806	info.flags = vdev->region[i].flags;
	807
	808	cap_type.type = vdev->region[i].type;
	809	cap_type.subtype = vdev->region[i].subtype;
	810
	811	ret = vfio_info_add_capability(&caps, &cap_type.header,
	812	sizeof(cap_type));
	813	if (ret)
	814	return ret;
	815
	816	if (vdev->region[i].ops->add_capability) {
	817	ret = vdev->region[i].ops->add_capability(vdev,
	818	&vdev->region[i], &caps);
	819	if (ret)
	820	return ret;
	821	}
	822	}
	823	}
	824
	825	if (caps.size) {
	826	info.flags \|= VFIO_REGION_INFO_FLAG_CAPS;
	827	if (info.argsz < sizeof(info) + caps.size) {
	828	info.argsz = sizeof(info) + caps.size;
	829	info.cap_offset = 0;
	830	} else {
	831	vfio_info_cap_shift(&caps, sizeof(info));
	832	if (copy_to_user((void __user *)arg +
	833	sizeof(info), caps.buf,
	834	caps.size)) {
	835	kfree(caps.buf);
	836	return -EFAULT;
	837	}
	838	info.cap_offset = sizeof(info);
	839	}
	840
	841	kfree(caps.buf);
	842	}
	843
	844	return copy_to_user((void __user *)arg, &info, minsz) ?
	845	-EFAULT : 0;
	846
	847	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
	848	struct vfio_irq_info info;
	849
	850	minsz = offsetofend(struct vfio_irq_info, count);
	851
	852	if (copy_from_user(&info, (void __user *)arg, minsz))
	853	return -EFAULT;
	854
	855	if (info.argsz < minsz \|\| info.index >= VFIO_PCI_NUM_IRQS)
	856	return -EINVAL;
	857
	858	switch (info.index) {
	859	case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
	860	case VFIO_PCI_REQ_IRQ_INDEX:
	861	break;
	862	case VFIO_PCI_ERR_IRQ_INDEX:
	863	if (pci_is_pcie(vdev->pdev))
	864	break;
	865	fallthrough;
	866	default:
	867	return -EINVAL;
	868	}
	869
	870	info.flags = VFIO_IRQ_INFO_EVENTFD;
	871
	872	info.count = vfio_pci_get_irq_count(vdev, info.index);
	873
	874	if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
	875	info.flags \|= (VFIO_IRQ_INFO_MASKABLE \|
	876	VFIO_IRQ_INFO_AUTOMASKED);
	877	else
	878	info.flags \|= VFIO_IRQ_INFO_NORESIZE;
	879
	880	return copy_to_user((void __user *)arg, &info, minsz) ?
	881	-EFAULT : 0;
	882
	883	} else if (cmd == VFIO_DEVICE_SET_IRQS) {
	884	struct vfio_irq_set hdr;
	885	u8 *data = NULL;
	886	int max, ret = 0;
	887	size_t data_size = 0;
	888
	889	minsz = offsetofend(struct vfio_irq_set, count);
	890
	891	if (copy_from_user(&hdr, (void __user *)arg, minsz))
	892	return -EFAULT;
	893
	894	max = vfio_pci_get_irq_count(vdev, hdr.index);
	895
	896	ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
	897	VFIO_PCI_NUM_IRQS, &data_size);
	898	if (ret)
	899	return ret;
	900
	901	if (data_size) {
	902	data = memdup_user((void __user *)(arg + minsz),
	903	data_size);
	904	if (IS_ERR(data))
	905	return PTR_ERR(data);
	906	}
	907
	908	mutex_lock(&vdev->igate);
	909
	910	ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,
	911	hdr.start, hdr.count, data);
	912
	913	mutex_unlock(&vdev->igate);
	914	kfree(data);
	915
	916	return ret;
	917
	918	} else if (cmd == VFIO_DEVICE_RESET) {
	919	int ret;
	920
	921	if (!vdev->reset_works)
	922	return -EINVAL;
	923
	924	vfio_pci_zap_and_down_write_memory_lock(vdev);
	925	ret = pci_try_reset_function(vdev->pdev);
	926	up_write(&vdev->memory_lock);
	927
	928	return ret;
	929
	930	} else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) {
	931	struct vfio_pci_hot_reset_info hdr;
	932	struct vfio_pci_fill_info fill = { 0 };
	933	struct vfio_pci_dependent_device *devices = NULL;
	934	bool slot = false;
	935	int ret = 0;
	936
	937	minsz = offsetofend(struct vfio_pci_hot_reset_info, count);
	938
	939	if (copy_from_user(&hdr, (void __user *)arg, minsz))
	940	return -EFAULT;
	941
	942	if (hdr.argsz < minsz)
	943	return -EINVAL;
	944
	945	hdr.flags = 0;
	946
	947	/* Can we do a slot or bus reset or neither? */
	948	if (!pci_probe_reset_slot(vdev->pdev->slot))
	949	slot = true;
	950	else if (pci_probe_reset_bus(vdev->pdev->bus))
	951	return -ENODEV;
	952
	953	/* How many devices are affected? */
	954	ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
	955	vfio_pci_count_devs,
	956	&fill.max, slot);
	957	if (ret)
	958	return ret;
	959
	960	WARN_ON(!fill.max); /* Should always be at least one */
	961
	962	/*
	963	* If there's enough space, fill it now, otherwise return
	964	* -ENOSPC and the number of devices affected.
	965	*/
	966	if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) {
	967	ret = -ENOSPC;
	968	hdr.count = fill.max;
	969	goto reset_info_exit;
	970	}
	971
	972	devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL);
	973	if (!devices)
	974	return -ENOMEM;
	975
	976	fill.devices = devices;
	977
	978	ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
	979	vfio_pci_fill_devs,
	980	&fill, slot);
	981
	982	/*
	983	* If a device was removed between counting and filling,
	984	* we may come up short of fill.max. If a device was
	985	* added, we'll have a return of -EAGAIN above.
	986	*/
	987	if (!ret)
	988	hdr.count = fill.cur;
	989
	990	reset_info_exit:
	991	if (copy_to_user((void __user *)arg, &hdr, minsz))
	992	ret = -EFAULT;
	993
	994	if (!ret) {
	995	if (copy_to_user((void __user *)(arg + minsz), devices,
	996	hdr.count * sizeof(*devices)))
	997	ret = -EFAULT;
	998	}
	999
	1000	kfree(devices);
	1001	return ret;
	1002
	1003	} else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) {
	1004	struct vfio_pci_hot_reset hdr;
	1005	int32_t *group_fds;
	1006	struct vfio_group **groups;
	1007	struct vfio_pci_group_info info;
	1008	bool slot = false;
	1009	int group_idx, count = 0, ret = 0;
	1010
	1011	minsz = offsetofend(struct vfio_pci_hot_reset, count);
	1012
	1013	if (copy_from_user(&hdr, (void __user *)arg, minsz))
	1014	return -EFAULT;
	1015
	1016	if (hdr.argsz < minsz \|\| hdr.flags)
	1017	return -EINVAL;
	1018
	1019	/* Can we do a slot or bus reset or neither? */
	1020	if (!pci_probe_reset_slot(vdev->pdev->slot))
	1021	slot = true;
	1022	else if (pci_probe_reset_bus(vdev->pdev->bus))
	1023	return -ENODEV;
	1024
	1025	/*
	1026	* We can't let userspace give us an arbitrarily large
	1027	* buffer to copy, so verify how many we think there
	1028	* could be. Note groups can have multiple devices so
	1029	* one group per device is the max.
	1030	*/
	1031	ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
	1032	vfio_pci_count_devs,
	1033	&count, slot);
	1034	if (ret)
	1035	return ret;
	1036
	1037	/* Somewhere between 1 and count is OK */
	1038	if (!hdr.count \|\| hdr.count > count)
	1039	return -EINVAL;
	1040
	1041	group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL);
	1042	groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL);
	1043	if (!group_fds \|\| !groups) {
	1044	kfree(group_fds);
	1045	kfree(groups);
	1046	return -ENOMEM;
	1047	}
	1048
	1049	if (copy_from_user(group_fds, (void __user *)(arg + minsz),
	1050	hdr.count * sizeof(*group_fds))) {
	1051	kfree(group_fds);
	1052	kfree(groups);
	1053	return -EFAULT;
	1054	}
	1055
	1056	/*
	1057	* For each group_fd, get the group through the vfio external
	1058	* user interface and store the group and iommu ID. This
	1059	* ensures the group is held across the reset.
	1060	*/
	1061	for (group_idx = 0; group_idx < hdr.count; group_idx++) {
	1062	struct vfio_group *group;
	1063	struct fd f = fdget(group_fds[group_idx]);
	1064	if (!f.file) {
	1065	ret = -EBADF;
	1066	break;
	1067	}
	1068
	1069	group = vfio_group_get_external_user(f.file);
	1070	fdput(f);
	1071	if (IS_ERR(group)) {
	1072	ret = PTR_ERR(group);
	1073	break;
	1074	}
	1075
	1076	groups[group_idx] = group;
	1077	}
	1078
	1079	kfree(group_fds);
	1080
	1081	/* release reference to groups on error */
	1082	if (ret)
	1083	goto hot_reset_release;
	1084
	1085	info.count = hdr.count;
	1086	info.groups = groups;
	1087
	1088	ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info);
	1089
	1090	hot_reset_release:
	1091	for (group_idx--; group_idx >= 0; group_idx--)
	1092	vfio_group_put_external_user(groups[group_idx]);
	1093
	1094	kfree(groups);
	1095	return ret;
	1096	} else if (cmd == VFIO_DEVICE_IOEVENTFD) {
	1097	struct vfio_device_ioeventfd ioeventfd;
	1098	int count;
	1099
	1100	minsz = offsetofend(struct vfio_device_ioeventfd, fd);
	1101
	1102	if (copy_from_user(&ioeventfd, (void __user *)arg, minsz))
	1103	return -EFAULT;
	1104
	1105	if (ioeventfd.argsz < minsz)
	1106	return -EINVAL;
	1107
	1108	if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK)
	1109	return -EINVAL;
	1110
	1111	count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK;
	1112
	1113	if (hweight8(count) != 1 \|\| ioeventfd.fd < -1)
	1114	return -EINVAL;
	1115
	1116	return vfio_pci_ioeventfd(vdev, ioeventfd.offset,
	1117	ioeventfd.data, count, ioeventfd.fd);
	1118	} else if (cmd == VFIO_DEVICE_FEATURE) {
	1119	struct vfio_device_feature feature;
	1120	uuid_t uuid;
	1121
	1122	minsz = offsetofend(struct vfio_device_feature, flags);
	1123
	1124	if (copy_from_user(&feature, (void __user *)arg, minsz))
	1125	return -EFAULT;
	1126
	1127	if (feature.argsz < minsz)
	1128	return -EINVAL;
	1129
	1130	/* Check unknown flags */
	1131	if (feature.flags & ~(VFIO_DEVICE_FEATURE_MASK \|
	1132	VFIO_DEVICE_FEATURE_SET \|
	1133	VFIO_DEVICE_FEATURE_GET \|
	1134	VFIO_DEVICE_FEATURE_PROBE))
	1135	return -EINVAL;
	1136
	1137	/* GET & SET are mutually exclusive except with PROBE */
	1138	if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) &&
	1139	(feature.flags & VFIO_DEVICE_FEATURE_SET) &&
	1140	(feature.flags & VFIO_DEVICE_FEATURE_GET))
	1141	return -EINVAL;
	1142
	1143	switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) {
	1144	case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN:
	1145	if (!vdev->vf_token)
	1146	return -ENOTTY;
	1147
	1148	/*
	1149	* We do not support GET of the VF Token UUID as this
	1150	* could expose the token of the previous device user.
	1151	*/
	1152	if (feature.flags & VFIO_DEVICE_FEATURE_GET)
	1153	return -EINVAL;
	1154
	1155	if (feature.flags & VFIO_DEVICE_FEATURE_PROBE)
	1156	return 0;
	1157
	1158	/* Don't SET unless told to do so */
	1159	if (!(feature.flags & VFIO_DEVICE_FEATURE_SET))
	1160	return -EINVAL;
	1161
	1162	if (feature.argsz < minsz + sizeof(uuid))
	1163	return -EINVAL;
	1164
	1165	if (copy_from_user(&uuid, (void __user *)(arg + minsz),
	1166	sizeof(uuid)))
	1167	return -EFAULT;
	1168
	1169	mutex_lock(&vdev->vf_token->lock);
	1170	uuid_copy(&vdev->vf_token->uuid, &uuid);
	1171	mutex_unlock(&vdev->vf_token->lock);
	1172
	1173	return 0;
	1174	default:
	1175	return -ENOTTY;
	1176	}
	1177	}
	1178
	1179	return -ENOTTY;
	1180	}
	1181	EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl);
	1182
	1183	static ssize_t vfio_pci_rw(struct vfio_pci_core_device vdev, char __user buf,
	1184	size_t count, loff_t *ppos, bool iswrite)
	1185	{
	1186	unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
	1187
	1188	if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
	1189	return -EINVAL;
	1190
	1191	switch (index) {
	1192	case VFIO_PCI_CONFIG_REGION_INDEX:
	1193	return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
	1194
	1195	case VFIO_PCI_ROM_REGION_INDEX:
	1196	if (iswrite)
	1197	return -EINVAL;
	1198	return vfio_pci_bar_rw(vdev, buf, count, ppos, false);
	1199
	1200	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
	1201	return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
	1202
	1203	case VFIO_PCI_VGA_REGION_INDEX:
	1204	return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
	1205	default:
	1206	index -= VFIO_PCI_NUM_REGIONS;
	1207	return vdev->region[index].ops->rw(vdev, buf,
	1208	count, ppos, iswrite);
	1209	}
	1210
	1211	return -EINVAL;
	1212	}
	1213
	1214	ssize_t vfio_pci_core_read(struct vfio_device core_vdev, char __user buf,
	1215	size_t count, loff_t *ppos)
	1216	{
	1217	struct vfio_pci_core_device *vdev =
	1218	container_of(core_vdev, struct vfio_pci_core_device, vdev);
	1219
	1220	if (!count)
	1221	return 0;
	1222
	1223	return vfio_pci_rw(vdev, buf, count, ppos, false);
	1224	}
	1225	EXPORT_SYMBOL_GPL(vfio_pci_core_read);
	1226
	1227	ssize_t vfio_pci_core_write(struct vfio_device core_vdev, const char __user buf,
	1228	size_t count, loff_t *ppos)
	1229	{
	1230	struct vfio_pci_core_device *vdev =
	1231	container_of(core_vdev, struct vfio_pci_core_device, vdev);
	1232
	1233	if (!count)
	1234	return 0;
	1235
	1236	return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true);
	1237	}
	1238	EXPORT_SYMBOL_GPL(vfio_pci_core_write);
	1239
	1240	/* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */
	1241	static int vfio_pci_zap_and_vma_lock(struct vfio_pci_core_device *vdev, bool try)
	1242	{
	1243	struct vfio_pci_mmap_vma mmap_vma, tmp;
	1244
	1245	/*
	1246	* Lock ordering:
	1247	* vma_lock is nested under mmap_lock for vm_ops callback paths.
	1248	* The memory_lock semaphore is used by both code paths calling
	1249	* into this function to zap vmas and the vm_ops.fault callback
	1250	* to protect the memory enable state of the device.
	1251	*
	1252	* When zapping vmas we need to maintain the mmap_lock => vma_lock
	1253	* ordering, which requires using vma_lock to walk vma_list to
	1254	* acquire an mm, then dropping vma_lock to get the mmap_lock and
	1255	* reacquiring vma_lock. This logic is derived from similar
	1256	* requirements in uverbs_user_mmap_disassociate().
	1257	*
	1258	* mmap_lock must always be the top-level lock when it is taken.
	1259	* Therefore we can only hold the memory_lock write lock when
	1260	* vma_list is empty, as we'd need to take mmap_lock to clear
	1261	* entries. vma_list can only be guaranteed empty when holding
	1262	* vma_lock, thus memory_lock is nested under vma_lock.
	1263	*
	1264	* This enables the vm_ops.fault callback to acquire vma_lock,
	1265	* followed by memory_lock read lock, while already holding
	1266	* mmap_lock without risk of deadlock.
	1267	*/
	1268	while (1) {
	1269	struct mm_struct *mm = NULL;
	1270
	1271	if (try) {
	1272	if (!mutex_trylock(&vdev->vma_lock))
	1273	return 0;
	1274	} else {
	1275	mutex_lock(&vdev->vma_lock);
	1276	}
	1277	while (!list_empty(&vdev->vma_list)) {
	1278	mmap_vma = list_first_entry(&vdev->vma_list,
	1279	struct vfio_pci_mmap_vma,
	1280	vma_next);
	1281	mm = mmap_vma->vma->vm_mm;
	1282	if (mmget_not_zero(mm))
	1283	break;
	1284
	1285	list_del(&mmap_vma->vma_next);
	1286	kfree(mmap_vma);
	1287	mm = NULL;
	1288	}
	1289	if (!mm)
	1290	return 1;
	1291	mutex_unlock(&vdev->vma_lock);
	1292
	1293	if (try) {
	1294	if (!mmap_read_trylock(mm)) {
	1295	mmput(mm);
	1296	return 0;
	1297	}
	1298	} else {
	1299	mmap_read_lock(mm);
	1300	}
	1301	if (try) {
	1302	if (!mutex_trylock(&vdev->vma_lock)) {
	1303	mmap_read_unlock(mm);
	1304	mmput(mm);
	1305	return 0;
	1306	}
	1307	} else {
	1308	mutex_lock(&vdev->vma_lock);
	1309	}
	1310	list_for_each_entry_safe(mmap_vma, tmp,
	1311	&vdev->vma_list, vma_next) {
	1312	struct vm_area_struct *vma = mmap_vma->vma;
	1313
	1314	if (vma->vm_mm != mm)
	1315	continue;
	1316
	1317	list_del(&mmap_vma->vma_next);
	1318	kfree(mmap_vma);
	1319
	1320	zap_vma_ptes(vma, vma->vm_start,
	1321	vma->vm_end - vma->vm_start);
	1322	}
	1323	mutex_unlock(&vdev->vma_lock);
	1324	mmap_read_unlock(mm);
	1325	mmput(mm);
	1326	}
	1327	}
	1328
	1329	void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev)
	1330	{
	1331	vfio_pci_zap_and_vma_lock(vdev, false);
	1332	down_write(&vdev->memory_lock);
	1333	mutex_unlock(&vdev->vma_lock);
	1334	}
	1335
	1336	u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev)
	1337	{
	1338	u16 cmd;
	1339
	1340	down_write(&vdev->memory_lock);
	1341	pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd);
	1342	if (!(cmd & PCI_COMMAND_MEMORY))
	1343	pci_write_config_word(vdev->pdev, PCI_COMMAND,
	1344	cmd \| PCI_COMMAND_MEMORY);
	1345
	1346	return cmd;
	1347	}
	1348
	1349	void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, u16 cmd)
	1350	{
	1351	pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd);
	1352	up_write(&vdev->memory_lock);
	1353	}
	1354
	1355	/* Caller holds vma_lock */
	1356	static int __vfio_pci_add_vma(struct vfio_pci_core_device *vdev,
	1357	struct vm_area_struct *vma)
	1358	{
	1359	struct vfio_pci_mmap_vma *mmap_vma;
	1360
	1361	mmap_vma = kmalloc(sizeof(*mmap_vma), GFP_KERNEL);
	1362	if (!mmap_vma)
	1363	return -ENOMEM;
	1364
	1365	mmap_vma->vma = vma;
	1366	list_add(&mmap_vma->vma_next, &vdev->vma_list);
	1367
	1368	return 0;
	1369	}
	1370
	1371	/*
	1372	* Zap mmaps on open so that we can fault them in on access and therefore
	1373	* our vma_list only tracks mappings accessed since last zap.
	1374	*/
	1375	static void vfio_pci_mmap_open(struct vm_area_struct *vma)
	1376	{
	1377	zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
	1378	}
	1379
	1380	static void vfio_pci_mmap_close(struct vm_area_struct *vma)
	1381	{
	1382	struct vfio_pci_core_device *vdev = vma->vm_private_data;
	1383	struct vfio_pci_mmap_vma *mmap_vma;
	1384
	1385	mutex_lock(&vdev->vma_lock);
	1386	list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
	1387	if (mmap_vma->vma == vma) {
	1388	list_del(&mmap_vma->vma_next);
	1389	kfree(mmap_vma);
	1390	break;
	1391	}
	1392	}
	1393	mutex_unlock(&vdev->vma_lock);
	1394	}
	1395
	1396	static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf)
	1397	{
	1398	struct vm_area_struct *vma = vmf->vma;
	1399	struct vfio_pci_core_device *vdev = vma->vm_private_data;
	1400	struct vfio_pci_mmap_vma *mmap_vma;
	1401	vm_fault_t ret = VM_FAULT_NOPAGE;
	1402
	1403	mutex_lock(&vdev->vma_lock);
	1404	down_read(&vdev->memory_lock);
	1405
	1406	if (!__vfio_pci_memory_enabled(vdev)) {
	1407	ret = VM_FAULT_SIGBUS;
	1408	goto up_out;
	1409	}
	1410
	1411	/*
	1412	* We populate the whole vma on fault, so we need to test whether
	1413	* the vma has already been mapped, such as for concurrent faults
	1414	* to the same vma. io_remap_pfn_range() will trigger a BUG_ON if
	1415	* we ask it to fill the same range again.
	1416	*/
	1417	list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
	1418	if (mmap_vma->vma == vma)
	1419	goto up_out;
	1420	}
	1421
	1422	if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
	1423	vma->vm_end - vma->vm_start,
	1424	vma->vm_page_prot)) {
	1425	ret = VM_FAULT_SIGBUS;
	1426	zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
	1427	goto up_out;
	1428	}
	1429
	1430	if (__vfio_pci_add_vma(vdev, vma)) {
	1431	ret = VM_FAULT_OOM;
	1432	zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
	1433	}
	1434
	1435	up_out:
	1436	up_read(&vdev->memory_lock);
	1437	mutex_unlock(&vdev->vma_lock);
	1438	return ret;
	1439	}
	1440
	1441	static const struct vm_operations_struct vfio_pci_mmap_ops = {
	1442	.open = vfio_pci_mmap_open,
	1443	.close = vfio_pci_mmap_close,
	1444	.fault = vfio_pci_mmap_fault,
	1445	};
	1446
	1447	int vfio_pci_core_mmap(struct vfio_device core_vdev, struct vm_area_struct vma)
	1448	{
	1449	struct vfio_pci_core_device *vdev =
	1450	container_of(core_vdev, struct vfio_pci_core_device, vdev);
	1451	struct pci_dev *pdev = vdev->pdev;
	1452	unsigned int index;
	1453	u64 phys_len, req_len, pgoff, req_start;
	1454	int ret;
	1455
	1456	index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
	1457
	1458	if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
	1459	return -EINVAL;
	1460	if (vma->vm_end < vma->vm_start)
	1461	return -EINVAL;
	1462	if ((vma->vm_flags & VM_SHARED) == 0)
	1463	return -EINVAL;
	1464	if (index >= VFIO_PCI_NUM_REGIONS) {
	1465	int regnum = index - VFIO_PCI_NUM_REGIONS;
	1466	struct vfio_pci_region *region = vdev->region + regnum;
	1467
	1468	if (region->ops && region->ops->mmap &&
	1469	(region->flags & VFIO_REGION_INFO_FLAG_MMAP))
	1470	return region->ops->mmap(vdev, region, vma);
	1471	return -EINVAL;
	1472	}
	1473	if (index >= VFIO_PCI_ROM_REGION_INDEX)
	1474	return -EINVAL;
	1475	if (!vdev->bar_mmap_supported[index])
	1476	return -EINVAL;
	1477
	1478	phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));
	1479	req_len = vma->vm_end - vma->vm_start;
	1480	pgoff = vma->vm_pgoff &
	1481	((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
	1482	req_start = pgoff << PAGE_SHIFT;
	1483
	1484	if (req_start + req_len > phys_len)
	1485	return -EINVAL;
	1486
	1487	/*
	1488	* Even though we don't make use of the barmap for the mmap,
	1489	* we need to request the region and the barmap tracks that.
	1490	*/
	1491	if (!vdev->barmap[index]) {
	1492	ret = pci_request_selected_regions(pdev,
	1493	1 << index, "vfio-pci");
	1494	if (ret)
	1495	return ret;
	1496
	1497	vdev->barmap[index] = pci_iomap(pdev, index, 0);
	1498	if (!vdev->barmap[index]) {
	1499	pci_release_selected_regions(pdev, 1 << index);
	1500	return -ENOMEM;
	1501	}
	1502	}
	1503
	1504	vma->vm_private_data = vdev;
	1505	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
	1506	vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;
	1507
	1508	/*
	1509	* See remap_pfn_range(), called from vfio_pci_fault() but we can't
	1510	* change vm_flags within the fault handler. Set them now.
	1511	*/
	1512	vma->vm_flags \|= VM_IO \| VM_PFNMAP \| VM_DONTEXPAND \| VM_DONTDUMP;
	1513	vma->vm_ops = &vfio_pci_mmap_ops;
	1514
	1515	return 0;
	1516	}
	1517	EXPORT_SYMBOL_GPL(vfio_pci_core_mmap);
	1518
	1519	void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count)
	1520	{
	1521	struct vfio_pci_core_device *vdev =
	1522	container_of(core_vdev, struct vfio_pci_core_device, vdev);
	1523	struct pci_dev *pdev = vdev->pdev;
	1524
	1525	mutex_lock(&vdev->igate);
	1526
	1527	if (vdev->req_trigger) {
	1528	if (!(count % 10))
	1529	pci_notice_ratelimited(pdev,
	1530	"Relaying device request to user (#%u)\n",
	1531	count);
	1532	eventfd_signal(vdev->req_trigger, 1);
	1533	} else if (count == 0) {
	1534	pci_warn(pdev,
	1535	"No device request channel registered, blocked until released by user\n");
	1536	}
	1537
	1538	mutex_unlock(&vdev->igate);
	1539	}
	1540	EXPORT_SYMBOL_GPL(vfio_pci_core_request);
	1541
	1542	static int vfio_pci_validate_vf_token(struct vfio_pci_core_device *vdev,
	1543	bool vf_token, uuid_t *uuid)
	1544	{
	1545	/*
	1546	* There's always some degree of trust or collaboration between SR-IOV
	1547	* PF and VFs, even if just that the PF hosts the SR-IOV capability and
	1548	* can disrupt VFs with a reset, but often the PF has more explicit
	1549	* access to deny service to the VF or access data passed through the
	1550	* VF. We therefore require an opt-in via a shared VF token (UUID) to
	1551	* represent this trust. This both prevents that a VF driver might
	1552	* assume the PF driver is a trusted, in-kernel driver, and also that
	1553	* a PF driver might be replaced with a rogue driver, unknown to in-use
	1554	* VF drivers.
	1555	*
	1556	* Therefore when presented with a VF, if the PF is a vfio device and
	1557	* it is bound to the vfio-pci driver, the user needs to provide a VF
	1558	* token to access the device, in the form of appending a vf_token to
	1559	* the device name, for example:
	1560	*
	1561	* "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
	1562	*
	1563	* When presented with a PF which has VFs in use, the user must also
	1564	* provide the current VF token to prove collaboration with existing
	1565	* VF users. If VFs are not in use, the VF token provided for the PF
	1566	* device will act to set the VF token.
	1567	*
	1568	* If the VF token is provided but unused, an error is generated.
	1569	*/
	1570	if (!vdev->pdev->is_virtfn && !vdev->vf_token && !vf_token)
	1571	return 0; /* No VF token provided or required */
	1572
	1573	if (vdev->pdev->is_virtfn) {
	1574	struct vfio_pci_core_device *pf_vdev = get_pf_vdev(vdev);
	1575	bool match;
	1576
	1577	if (!pf_vdev) {
	1578	if (!vf_token)
	1579	return 0; /* PF is not vfio-pci, no VF token */
	1580
	1581	pci_info_ratelimited(vdev->pdev,
	1582	"VF token incorrectly provided, PF not bound to vfio-pci\n");
	1583	return -EINVAL;
	1584	}
	1585
	1586	if (!vf_token) {
	1587	vfio_device_put(&pf_vdev->vdev);
	1588	pci_info_ratelimited(vdev->pdev,
	1589	"VF token required to access device\n");
	1590	return -EACCES;
	1591	}
	1592
	1593	mutex_lock(&pf_vdev->vf_token->lock);
	1594	match = uuid_equal(uuid, &pf_vdev->vf_token->uuid);
	1595	mutex_unlock(&pf_vdev->vf_token->lock);
	1596
	1597	vfio_device_put(&pf_vdev->vdev);
	1598
	1599	if (!match) {
	1600	pci_info_ratelimited(vdev->pdev,
	1601	"Incorrect VF token provided for device\n");
	1602	return -EACCES;
	1603	}
	1604	} else if (vdev->vf_token) {
	1605	mutex_lock(&vdev->vf_token->lock);
	1606	if (vdev->vf_token->users) {
	1607	if (!vf_token) {
	1608	mutex_unlock(&vdev->vf_token->lock);
	1609	pci_info_ratelimited(vdev->pdev,
	1610	"VF token required to access device\n");
	1611	return -EACCES;
	1612	}
	1613
	1614	if (!uuid_equal(uuid, &vdev->vf_token->uuid)) {
	1615	mutex_unlock(&vdev->vf_token->lock);
	1616	pci_info_ratelimited(vdev->pdev,
	1617	"Incorrect VF token provided for device\n");
	1618	return -EACCES;
	1619	}
	1620	} else if (vf_token) {
	1621	uuid_copy(&vdev->vf_token->uuid, uuid);
	1622	}
	1623
	1624	mutex_unlock(&vdev->vf_token->lock);
	1625	} else if (vf_token) {
	1626	pci_info_ratelimited(vdev->pdev,
	1627	"VF token incorrectly provided, not a PF or VF\n");
	1628	return -EINVAL;
	1629	}
	1630
	1631	return 0;
	1632	}
	1633
	1634	#define VF_TOKEN_ARG "vf_token="
	1635
	1636	int vfio_pci_core_match(struct vfio_device core_vdev, char buf)
	1637	{
	1638	struct vfio_pci_core_device *vdev =
	1639	container_of(core_vdev, struct vfio_pci_core_device, vdev);
	1640	bool vf_token = false;
	1641	uuid_t uuid;
	1642	int ret;
	1643
	1644	if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev))))
	1645	return 0; /* No match */
	1646
	1647	if (strlen(buf) > strlen(pci_name(vdev->pdev))) {
	1648	buf += strlen(pci_name(vdev->pdev));
	1649
	1650	if (*buf != ' ')
	1651	return 0; /* No match: non-whitespace after name */
	1652
	1653	while (*buf) {
	1654	if (*buf == ' ') {
	1655	buf++;
	1656	continue;
	1657	}
	1658
	1659	if (!vf_token && !strncmp(buf, VF_TOKEN_ARG,
	1660	strlen(VF_TOKEN_ARG))) {
	1661	buf += strlen(VF_TOKEN_ARG);
	1662
	1663	if (strlen(buf) < UUID_STRING_LEN)
	1664	return -EINVAL;
	1665
	1666	ret = uuid_parse(buf, &uuid);
	1667	if (ret)
	1668	return ret;
	1669
	1670	vf_token = true;
	1671	buf += UUID_STRING_LEN;
	1672	} else {
	1673	/* Unknown/duplicate option */
	1674	return -EINVAL;
	1675	}
	1676	}
	1677	}
	1678
	1679	ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid);
	1680	if (ret)
	1681	return ret;
	1682
	1683	return 1; /* Match */
	1684	}
	1685	EXPORT_SYMBOL_GPL(vfio_pci_core_match);
	1686
	1687	static int vfio_pci_bus_notifier(struct notifier_block *nb,
	1688	unsigned long action, void *data)
	1689	{
	1690	struct vfio_pci_core_device *vdev = container_of(nb,
	1691	struct vfio_pci_core_device, nb);
	1692	struct device *dev = data;
	1693	struct pci_dev *pdev = to_pci_dev(dev);
	1694	struct pci_dev *physfn = pci_physfn(pdev);
	1695
	1696	if (action == BUS_NOTIFY_ADD_DEVICE &&
	1697	pdev->is_virtfn && physfn == vdev->pdev) {
	1698	pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n",
	1699	pci_name(pdev));
	1700	pdev->driver_override = kasprintf(GFP_KERNEL, "%s",
	1701	vdev->vdev.ops->name);
	1702	} else if (action == BUS_NOTIFY_BOUND_DRIVER &&
	1703	pdev->is_virtfn && physfn == vdev->pdev) {
	1704	struct pci_driver *drv = pci_dev_driver(pdev);
	1705
	1706	if (drv && drv != pci_dev_driver(vdev->pdev))
	1707	pci_warn(vdev->pdev,
	1708	"VF %s bound to driver %s while PF bound to driver %s\n",
	1709	pci_name(pdev), drv->name,
	1710	pci_dev_driver(vdev->pdev)->name);
	1711	}
	1712
	1713	return 0;
	1714	}
	1715
	1716	static int vfio_pci_vf_init(struct vfio_pci_core_device *vdev)
	1717	{
	1718	struct pci_dev *pdev = vdev->pdev;
	1719	int ret;
	1720
	1721	if (!pdev->is_physfn)
	1722	return 0;
	1723
	1724	vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL);
	1725	if (!vdev->vf_token)
	1726	return -ENOMEM;
	1727
	1728	mutex_init(&vdev->vf_token->lock);
	1729	uuid_gen(&vdev->vf_token->uuid);
	1730
	1731	vdev->nb.notifier_call = vfio_pci_bus_notifier;
	1732	ret = bus_register_notifier(&pci_bus_type, &vdev->nb);
	1733	if (ret) {
	1734	kfree(vdev->vf_token);
	1735	return ret;
	1736	}
	1737	return 0;
	1738	}
	1739
	1740	static void vfio_pci_vf_uninit(struct vfio_pci_core_device *vdev)
	1741	{
	1742	if (!vdev->vf_token)
	1743	return;
	1744
	1745	bus_unregister_notifier(&pci_bus_type, &vdev->nb);
	1746	WARN_ON(vdev->vf_token->users);
	1747	mutex_destroy(&vdev->vf_token->lock);
	1748	kfree(vdev->vf_token);
	1749	}
	1750
	1751	static int vfio_pci_vga_init(struct vfio_pci_core_device *vdev)
	1752	{
	1753	struct pci_dev *pdev = vdev->pdev;
	1754	int ret;
	1755
	1756	if (!vfio_pci_is_vga(pdev))
	1757	return 0;
	1758
	1759	ret = vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode);
	1760	if (ret)
	1761	return ret;
	1762	vga_set_legacy_decoding(pdev, vfio_pci_set_vga_decode(vdev, false));
	1763	return 0;
	1764	}
	1765
	1766	static void vfio_pci_vga_uninit(struct vfio_pci_core_device *vdev)
	1767	{
	1768	struct pci_dev *pdev = vdev->pdev;
	1769
	1770	if (!vfio_pci_is_vga(pdev))
	1771	return;
	1772	vga_client_register(pdev, NULL, NULL, NULL);
	1773	vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM \|
	1774	VGA_RSRC_LEGACY_IO \|
	1775	VGA_RSRC_LEGACY_MEM);
	1776	}
	1777
	1778	void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev,
	1779	struct pci_dev *pdev,
	1780	const struct vfio_device_ops *vfio_pci_ops)
	1781	{
	1782	vfio_init_group_dev(&vdev->vdev, &pdev->dev, vfio_pci_ops);
	1783	vdev->pdev = pdev;
	1784	vdev->irq_type = VFIO_PCI_NUM_IRQS;
	1785	mutex_init(&vdev->igate);
	1786	spin_lock_init(&vdev->irqlock);
	1787	mutex_init(&vdev->ioeventfds_lock);
	1788	INIT_LIST_HEAD(&vdev->dummy_resources_list);
	1789	INIT_LIST_HEAD(&vdev->ioeventfds_list);
	1790	mutex_init(&vdev->vma_lock);
	1791	INIT_LIST_HEAD(&vdev->vma_list);
	1792	init_rwsem(&vdev->memory_lock);
	1793	}
	1794	EXPORT_SYMBOL_GPL(vfio_pci_core_init_device);
	1795
	1796	void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev)
	1797	{
	1798	mutex_destroy(&vdev->igate);
	1799	mutex_destroy(&vdev->ioeventfds_lock);
	1800	mutex_destroy(&vdev->vma_lock);
	1801	vfio_uninit_group_dev(&vdev->vdev);
	1802	kfree(vdev->region);
	1803	kfree(vdev->pm_save);
	1804	}
	1805	EXPORT_SYMBOL_GPL(vfio_pci_core_uninit_device);
	1806
	1807	int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev)
	1808	{
	1809	struct pci_dev *pdev = vdev->pdev;
	1810	struct iommu_group *group;
	1811	int ret;
	1812
	1813	if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)
	1814	return -EINVAL;
	1815
	1816	/*
	1817	* Prevent binding to PFs with VFs enabled, the VFs might be in use
	1818	* by the host or other users. We cannot capture the VFs if they
	1819	* already exist, nor can we track VF users. Disabling SR-IOV here
	1820	* would initiate removing the VFs, which would unbind the driver,
	1821	* which is prone to blocking if that VF is also in use by vfio-pci.
	1822	* Just reject these PFs and let the user sort it out.
	1823	*/
	1824	if (pci_num_vf(pdev)) {
	1825	pci_warn(pdev, "Cannot bind to PF with SR-IOV enabled\n");
	1826	return -EBUSY;
	1827	}
	1828
	1829	group = vfio_iommu_group_get(&pdev->dev);
	1830	if (!group)
	1831	return -EINVAL;
	1832
	1833	if (pci_is_root_bus(pdev->bus)) {
	1834	ret = vfio_assign_device_set(&vdev->vdev, vdev);
	1835	} else if (!pci_probe_reset_slot(pdev->slot)) {
	1836	ret = vfio_assign_device_set(&vdev->vdev, pdev->slot);
	1837	} else {
	1838	/*
	1839	* If there is no slot reset support for this device, the whole
	1840	* bus needs to be grouped together to support bus-wide resets.
	1841	*/
	1842	ret = vfio_assign_device_set(&vdev->vdev, pdev->bus);
	1843	}
	1844
	1845	if (ret)
	1846	goto out_group_put;
	1847	ret = vfio_pci_vf_init(vdev);
	1848	if (ret)
	1849	goto out_group_put;
	1850	ret = vfio_pci_vga_init(vdev);
	1851	if (ret)
	1852	goto out_vf;
	1853
	1854	vfio_pci_probe_power_state(vdev);
	1855
	1856	if (!disable_idle_d3) {
	1857	/*
	1858	* pci-core sets the device power state to an unknown value at
	1859	* bootup and after being removed from a driver. The only
	1860	* transition it allows from this unknown state is to D0, which
	1861	* typically happens when a driver calls pci_enable_device().
	1862	* We're not ready to enable the device yet, but we do want to
	1863	* be able to get to D3. Therefore first do a D0 transition
	1864	* before going to D3.
	1865	*/
	1866	vfio_pci_set_power_state(vdev, PCI_D0);
	1867	vfio_pci_set_power_state(vdev, PCI_D3hot);
	1868	}
	1869
	1870	ret = vfio_register_group_dev(&vdev->vdev);
	1871	if (ret)
	1872	goto out_power;
	1873	return 0;
	1874
	1875	out_power:
	1876	if (!disable_idle_d3)
	1877	vfio_pci_set_power_state(vdev, PCI_D0);
	1878	out_vf:
	1879	vfio_pci_vf_uninit(vdev);
	1880	out_group_put:
	1881	vfio_iommu_group_put(group, &pdev->dev);
	1882	return ret;
	1883	}
	1884	EXPORT_SYMBOL_GPL(vfio_pci_core_register_device);
	1885
	1886	void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev)
	1887	{
	1888	struct pci_dev *pdev = vdev->pdev;
	1889
	1890	pci_disable_sriov(pdev);
	1891
	1892	vfio_unregister_group_dev(&vdev->vdev);
	1893
	1894	vfio_pci_vf_uninit(vdev);
	1895	vfio_pci_vga_uninit(vdev);
	1896
	1897	vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev);
	1898
	1899	if (!disable_idle_d3)
	1900	vfio_pci_set_power_state(vdev, PCI_D0);
	1901	}
	1902	EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_device);
	1903
	1904	static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,
	1905	pci_channel_state_t state)
	1906	{
	1907	struct vfio_pci_core_device *vdev;
	1908	struct vfio_device *device;
	1909
	1910	device = vfio_device_get_from_dev(&pdev->dev);
	1911	if (device == NULL)
	1912	return PCI_ERS_RESULT_DISCONNECT;
	1913
	1914	vdev = container_of(device, struct vfio_pci_core_device, vdev);
	1915
	1916	mutex_lock(&vdev->igate);
	1917
	1918	if (vdev->err_trigger)
	1919	eventfd_signal(vdev->err_trigger, 1);
	1920
	1921	mutex_unlock(&vdev->igate);
	1922
	1923	vfio_device_put(device);
	1924
	1925	return PCI_ERS_RESULT_CAN_RECOVER;
	1926	}
	1927
	1928	int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
	1929	{
	1930	struct vfio_device *device;
	1931	int ret = 0;
	1932
	1933	device = vfio_device_get_from_dev(&pdev->dev);
	1934	if (!device)
	1935	return -ENODEV;
	1936
	1937	if (nr_virtfn == 0)
	1938	pci_disable_sriov(pdev);
	1939	else
	1940	ret = pci_enable_sriov(pdev, nr_virtfn);
	1941
	1942	vfio_device_put(device);
	1943
	1944	return ret < 0 ? ret : nr_virtfn;
	1945	}
	1946	EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure);
	1947
	1948	const struct pci_error_handlers vfio_pci_core_err_handlers = {
	1949	.error_detected = vfio_pci_aer_err_detected,
	1950	};
	1951	EXPORT_SYMBOL_GPL(vfio_pci_core_err_handlers);
	1952
	1953	static bool vfio_dev_in_groups(struct vfio_pci_core_device *vdev,
	1954	struct vfio_pci_group_info *groups)
	1955	{
	1956	unsigned int i;
	1957
	1958	for (i = 0; i < groups->count; i++)
	1959	if (groups->groups[i] == vdev->vdev.group)
	1960	return true;
	1961	return false;
	1962	}
	1963
	1964	static int vfio_pci_is_device_in_set(struct pci_dev pdev, void data)
	1965	{
	1966	struct vfio_device_set *dev_set = data;
	1967	struct vfio_device *cur;
	1968
	1969	list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
	1970	if (cur->dev == &pdev->dev)
	1971	return 0;
	1972	return -EBUSY;
	1973	}
	1974
	1975	/*
	1976	* vfio-core considers a group to be viable and will create a vfio_device even
	1977	* if some devices are bound to drivers like pci-stub or pcieport. Here we
	1978	* require all PCI devices to be inside our dev_set since that ensures they stay
	1979	* put and that every driver controlling the device can co-ordinate with the
	1980	* device reset.
	1981	*
	1982	* Returns the pci_dev to pass to pci_reset_bus() if every PCI device to be
	1983	* reset is inside the dev_set, and pci_reset_bus() can succeed. NULL otherwise.
	1984	*/
	1985	static struct pci_dev *
	1986	vfio_pci_dev_set_resettable(struct vfio_device_set *dev_set)
	1987	{
	1988	struct pci_dev *pdev;
	1989
	1990	lockdep_assert_held(&dev_set->lock);
	1991
	1992	/*
	1993	* By definition all PCI devices in the dev_set share the same PCI
	1994	* reset, so any pci_dev will have the same outcomes for
	1995	* pci_probe_reset_*() and pci_reset_bus().
	1996	*/
	1997	pdev = list_first_entry(&dev_set->device_list,
	1998	struct vfio_pci_core_device,
	1999	vdev.dev_set_list)->pdev;
	2000
	2001	/* pci_reset_bus() is supported */
	2002	if (pci_probe_reset_slot(pdev->slot) && pci_probe_reset_bus(pdev->bus))
	2003	return NULL;
	2004
	2005	if (vfio_pci_for_each_slot_or_bus(pdev, vfio_pci_is_device_in_set,
	2006	dev_set,
	2007	!pci_probe_reset_slot(pdev->slot)))
	2008	return NULL;
	2009	return pdev;
	2010	}
	2011
	2012	/*
	2013	* We need to get memory_lock for each device, but devices can share mmap_lock,
	2014	* therefore we need to zap and hold the vma_lock for each device, and only then
	2015	* get each memory_lock.
	2016	*/
	2017	static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
	2018	struct vfio_pci_group_info *groups)
	2019	{
	2020	struct vfio_pci_core_device *cur_mem;
	2021	struct vfio_pci_core_device *cur_vma;
	2022	struct vfio_pci_core_device *cur;
	2023	struct pci_dev *pdev;
	2024	bool is_mem = true;
	2025	int ret;
	2026
	2027	mutex_lock(&dev_set->lock);
	2028	cur_mem = list_first_entry(&dev_set->device_list,
	2029	struct vfio_pci_core_device,
	2030	vdev.dev_set_list);
	2031
	2032	pdev = vfio_pci_dev_set_resettable(dev_set);
	2033	if (!pdev) {
	2034	ret = -EINVAL;
	2035	goto err_unlock;
	2036	}
	2037
	2038	list_for_each_entry(cur_vma, &dev_set->device_list, vdev.dev_set_list) {
	2039	/*
	2040	* Test whether all the affected devices are contained by the
	2041	* set of groups provided by the user.
	2042	*/
	2043	if (!vfio_dev_in_groups(cur_vma, groups)) {
	2044	ret = -EINVAL;
	2045	goto err_undo;
	2046	}
	2047
	2048	/*
	2049	* Locking multiple devices is prone to deadlock, runaway and
	2050	* unwind if we hit contention.
	2051	*/
	2052	if (!vfio_pci_zap_and_vma_lock(cur_vma, true)) {
	2053	ret = -EBUSY;
	2054	goto err_undo;
	2055	}
	2056	}
	2057	cur_vma = NULL;
	2058
	2059	list_for_each_entry(cur_mem, &dev_set->device_list, vdev.dev_set_list) {
	2060	if (!down_write_trylock(&cur_mem->memory_lock)) {
	2061	ret = -EBUSY;
	2062	goto err_undo;
	2063	}
	2064	mutex_unlock(&cur_mem->vma_lock);
	2065	}
	2066	cur_mem = NULL;
	2067
	2068	ret = pci_reset_bus(pdev);
	2069
	2070	err_undo:
	2071	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
	2072	if (cur == cur_mem)
	2073	is_mem = false;
	2074	if (cur == cur_vma)
	2075	break;
	2076	if (is_mem)
	2077	up_write(&cur->memory_lock);
	2078	else
	2079	mutex_unlock(&cur->vma_lock);
	2080	}
	2081	err_unlock:
	2082	mutex_unlock(&dev_set->lock);
	2083	return ret;
	2084	}
	2085
	2086	static bool vfio_pci_dev_set_needs_reset(struct vfio_device_set *dev_set)
	2087	{
	2088	struct vfio_pci_core_device *cur;
	2089	bool needs_reset = false;
	2090
	2091	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
	2092	/* No VFIO device in the set can have an open device FD */
	2093	if (cur->vdev.open_count)
	2094	return false;
	2095	needs_reset \|= cur->needs_reset;
	2096	}
	2097	return needs_reset;
	2098	}
	2099
	2100	/*
	2101	* If a bus or slot reset is available for the provided dev_set and:
	2102	* - All of the devices affected by that bus or slot reset are unused
	2103	* - At least one of the affected devices is marked dirty via
	2104	* needs_reset (such as by lack of FLR support)
	2105	* Then attempt to perform that bus or slot reset.
	2106	* Returns true if the dev_set was reset.
	2107	*/
	2108	static bool vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set)
	2109	{
	2110	struct vfio_pci_core_device *cur;
	2111	struct pci_dev *pdev;
	2112	int ret;
	2113
	2114	if (!vfio_pci_dev_set_needs_reset(dev_set))
	2115	return false;
	2116
	2117	pdev = vfio_pci_dev_set_resettable(dev_set);
	2118	if (!pdev)
	2119	return false;
	2120
	2121	ret = pci_reset_bus(pdev);
	2122	if (ret)
	2123	return false;
	2124
	2125	list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
	2126	cur->needs_reset = false;
	2127	if (!disable_idle_d3)
	2128	vfio_pci_set_power_state(cur, PCI_D3hot);
	2129	}
	2130	return true;
	2131	}
	2132
	2133	void vfio_pci_core_set_params(bool is_nointxmask, bool is_disable_vga,
	2134	bool is_disable_idle_d3)
	2135	{
	2136	nointxmask = is_nointxmask;
	2137	disable_vga = is_disable_vga;
	2138	disable_idle_d3 = is_disable_idle_d3;
	2139	}
	2140	EXPORT_SYMBOL_GPL(vfio_pci_core_set_params);
	2141
	2142	static void vfio_pci_core_cleanup(void)
	2143	{
	2144	vfio_pci_uninit_perm_bits();
	2145	}
	2146
	2147	static int __init vfio_pci_core_init(void)
	2148	{
	2149	/* Allocate shared config space permission data used by all devices */
	2150	return vfio_pci_init_perm_bits();
	2151	}
	2152
	2153	module_init(vfio_pci_core_init);
	2154	module_exit(vfio_pci_core_cleanup);
	2155
	2156	MODULE_LICENSE("GPL v2");
	2157	MODULE_AUTHOR(DRIVER_AUTHOR);
	2158	MODULE_DESCRIPTION(DRIVER_DESC);

drivers/vfio/pci/vfio_pci_igd.c+10−9

 #include <linux/uaccess.h>
 #include <linux/vfio.h>
-#include "vfio_pci_private.h"
+#include <linux/vfio_pci_core.h>
 #define OPREGION_SIGNATURE	"IntelGraphicsMem"
 #define OPREGION_SIZE		(8 * 1024)
 #define OPREGION_RVDS		0x3c2
 #define OPREGION_VERSION	0x16
-static ssize_t vfio_pci_igd_rw(struct vfio_pci_device *vdev, char __user *buf,
+static ssize_t vfio_pci_igd_rw(struct vfio_pci_core_device *vdev,
-			       size_t count, loff_t *ppos, bool iswrite)
+			       char __user *buf, size_t count, loff_t *ppos,
+			       bool iswrite)
 {
 	unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - VFIO_PCI_NUM_REGIONS;
 	void *base = vdev->region[i].data;
 	return count;
 }
-static void vfio_pci_igd_release(struct vfio_pci_device *vdev,
+static void vfio_pci_igd_release(struct vfio_pci_core_device *vdev,
 				 struct vfio_pci_region *region)
 {
 	memunmap(region->data);
 	.release	= vfio_pci_igd_release,
 };
-static int vfio_pci_igd_opregion_init(struct vfio_pci_device *vdev)
+static int vfio_pci_igd_opregion_init(struct vfio_pci_core_device *vdev)
 {
 	__le32 *dwordp = (__le32 *)(vdev->vconfig + OPREGION_PCI_ADDR);
 	u32 addr, size;
 	return ret;
 }
-static ssize_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev,
+static ssize_t vfio_pci_igd_cfg_rw(struct vfio_pci_core_device *vdev,
 				   char __user *buf, size_t count, loff_t *ppos,
 				   bool iswrite)
 {
 	return count;
 }
-static void vfio_pci_igd_cfg_release(struct vfio_pci_device *vdev,
+static void vfio_pci_igd_cfg_release(struct vfio_pci_core_device *vdev,
 				     struct vfio_pci_region *region)
 {
 	struct pci_dev *pdev = region->data;
 	.release	= vfio_pci_igd_cfg_release,
 };
-static int vfio_pci_igd_cfg_init(struct vfio_pci_device *vdev)
+static int vfio_pci_igd_cfg_init(struct vfio_pci_core_device *vdev)
 {
 	struct pci_dev *host_bridge, *lpc_bridge;
 	int ret;
 	return 0;
 }
-int vfio_pci_igd_init(struct vfio_pci_device *vdev)
+int vfio_pci_igd_init(struct vfio_pci_core_device *vdev)
 {
 	int ret;

drivers/vfio/pci/vfio_pci_intrs.c+21−21

@@ -20,20 +20,20 @@ · 19 unmodified lines
20	#include <linux/wait.h>	20	#include <linux/wait.h>
21	#include <linux/slab.h>	21	#include <linux/slab.h>
22		22
23	#include "vfio_pci_private.h"	23	#include <linux/vfio_pci_core.h>
24		24
25	/*	25	/*
26	* INTx	26	* INTx
27	*/	27	*/
28	static void vfio_send_intx_eventfd(void opaque, void unused)	28	static void vfio_send_intx_eventfd(void opaque, void unused)
29	{	29	{
30	struct vfio_pci_device *vdev = opaque;	30	struct vfio_pci_core_device *vdev = opaque;
31		31
32	if (likely(is_intx(vdev) && !vdev->virq_disabled))	32	if (likely(is_intx(vdev) && !vdev->virq_disabled))
33	eventfd_signal(vdev->ctx[0].trigger, 1);	33	eventfd_signal(vdev->ctx[0].trigger, 1);
34	}	34	}
35		35
36	void vfio_pci_intx_mask(struct vfio_pci_device *vdev)	36	void vfio_pci_intx_mask(struct vfio_pci_core_device *vdev)
37	{	37	{
38	struct pci_dev *pdev = vdev->pdev;	38	struct pci_dev *pdev = vdev->pdev;
39	unsigned long flags;	39	unsigned long flags;
@@ -73,7 +73,7 @@ void vfio_pci_intx_mask(struct vfio_pci_device *vdev) · 33 unmodified lines
73	*/	73	*/
74	static int vfio_pci_intx_unmask_handler(void opaque, void unused)	74	static int vfio_pci_intx_unmask_handler(void opaque, void unused)
75	{	75	{
76	struct vfio_pci_device *vdev = opaque;	76	struct vfio_pci_core_device *vdev = opaque;
77	struct pci_dev *pdev = vdev->pdev;	77	struct pci_dev *pdev = vdev->pdev;
78	unsigned long flags;	78	unsigned long flags;
79	int ret = 0;	79	int ret = 0;
@@ -107,7 +107,7 @@ static int vfio_pci_intx_unmask_handler(void opaque, void unused) · 27 unmodified lines
107	return ret;	107	return ret;
108	}	108	}
109		109
110	void vfio_pci_intx_unmask(struct vfio_pci_device *vdev)	110	void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev)
111	{	111	{
112	if (vfio_pci_intx_unmask_handler(vdev, NULL) > 0)	112	if (vfio_pci_intx_unmask_handler(vdev, NULL) > 0)
113	vfio_send_intx_eventfd(vdev, NULL);	113	vfio_send_intx_eventfd(vdev, NULL);
@@ -115,7 +115,7 @@ void vfio_pci_intx_unmask(struct vfio_pci_device *vdev) · 1 unmodified line
115		115
116	static irqreturn_t vfio_intx_handler(int irq, void *dev_id)	116	static irqreturn_t vfio_intx_handler(int irq, void *dev_id)
117	{	117	{
118	struct vfio_pci_device *vdev = dev_id;	118	struct vfio_pci_core_device *vdev = dev_id;
119	unsigned long flags;	119	unsigned long flags;
120	int ret = IRQ_NONE;	120	int ret = IRQ_NONE;
121		121
@@ -139,7 +139,7 @@ static irqreturn_t vfio_intx_handler(int irq, void *dev_id) · 17 unmodified lines
139	return ret;	139	return ret;
140	}	140	}
141		141
142	static int vfio_intx_enable(struct vfio_pci_device *vdev)	142	static int vfio_intx_enable(struct vfio_pci_core_device *vdev)
143	{	143	{
144	if (!is_irq_none(vdev))	144	if (!is_irq_none(vdev))
145	return -EINVAL;	145	return -EINVAL;
@@ -168,7 +168,7 @@ static int vfio_intx_enable(struct vfio_pci_device *vdev) · 22 unmodified lines
168	return 0;	168	return 0;
169	}	169	}
170		170
171	static int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd)	171	static int vfio_intx_set_signal(struct vfio_pci_core_device *vdev, int fd)
172	{	172	{
173	struct pci_dev *pdev = vdev->pdev;	173	struct pci_dev *pdev = vdev->pdev;
174	unsigned long irqflags = IRQF_SHARED;	174	unsigned long irqflags = IRQF_SHARED;
@@ -223,7 +223,7 @@ static int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd) · 48 unmodified lines
223	return 0;	223	return 0;
224	}	224	}
225		225
226	static void vfio_intx_disable(struct vfio_pci_device *vdev)	226	static void vfio_intx_disable(struct vfio_pci_core_device *vdev)
227	{	227	{
228	vfio_virqfd_disable(&vdev->ctx[0].unmask);	228	vfio_virqfd_disable(&vdev->ctx[0].unmask);
229	vfio_virqfd_disable(&vdev->ctx[0].mask);	229	vfio_virqfd_disable(&vdev->ctx[0].mask);
@@ -244,7 +244,7 @@ static irqreturn_t vfio_msihandler(int irq, void *arg) · 14 unmodified lines
244	return IRQ_HANDLED;	244	return IRQ_HANDLED;
245	}	245	}
246		246
247	static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix)	247	static int vfio_msi_enable(struct vfio_pci_core_device *vdev, int nvec, bool msix)
248	{	248	{
249	struct pci_dev *pdev = vdev->pdev;	249	struct pci_dev *pdev = vdev->pdev;
250	unsigned int flag = msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI;	250	unsigned int flag = msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI;
@@ -285,7 +285,7 @@ static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix) · 34 unmodified lines
285	return 0;	285	return 0;
286	}	286	}
287		287
288	static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,	288	static int vfio_msi_set_vector_signal(struct vfio_pci_core_device *vdev,
289	int vector, int fd, bool msix)	289	int vector, int fd, bool msix)
290	{	290	{
291	struct pci_dev *pdev = vdev->pdev;	291	struct pci_dev *pdev = vdev->pdev;
@@ -364,7 +364,7 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, · 72 unmodified lines
364	return 0;	364	return 0;
365	}	365	}
366		366
367	static int vfio_msi_set_block(struct vfio_pci_device *vdev, unsigned start,	367	static int vfio_msi_set_block(struct vfio_pci_core_device *vdev, unsigned start,
368	unsigned count, int32_t *fds, bool msix)	368	unsigned count, int32_t *fds, bool msix)
369	{	369	{
370	int i, j, ret = 0;	370	int i, j, ret = 0;
@@ -385,7 +385,7 @@ static int vfio_msi_set_block(struct vfio_pci_device *vdev, unsigned start, · 14 unmodified lines
385	return ret;	385	return ret;
386	}	386	}
387		387
388	static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix)	388	static void vfio_msi_disable(struct vfio_pci_core_device *vdev, bool msix)
389	{	389	{
390	struct pci_dev *pdev = vdev->pdev;	390	struct pci_dev *pdev = vdev->pdev;
391	int i;	391	int i;
@@ -417,7 +417,7 @@ static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix) · 25 unmodified lines
417	/*	417	/*
418	* IOCTL support	418	* IOCTL support
419	*/	419	*/
420	static int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev,	420	static int vfio_pci_set_intx_unmask(struct vfio_pci_core_device *vdev,
421	unsigned index, unsigned start,	421	unsigned index, unsigned start,
422	unsigned count, uint32_t flags, void *data)	422	unsigned count, uint32_t flags, void *data)
423	{	423	{
@@ -444,7 +444,7 @@ static int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev, · 20 unmodified lines
444	return 0;	444	return 0;
445	}	445	}
446		446
447	static int vfio_pci_set_intx_mask(struct vfio_pci_device *vdev,	447	static int vfio_pci_set_intx_mask(struct vfio_pci_core_device *vdev,
448	unsigned index, unsigned start,	448	unsigned index, unsigned start,
449	unsigned count, uint32_t flags, void *data)	449	unsigned count, uint32_t flags, void *data)
450	{	450	{
@@ -464,7 +464,7 @@ static int vfio_pci_set_intx_mask(struct vfio_pci_device *vdev, · 13 unmodified lines
464	return 0;	464	return 0;
465	}	465	}
466		466
467	static int vfio_pci_set_intx_trigger(struct vfio_pci_device *vdev,	467	static int vfio_pci_set_intx_trigger(struct vfio_pci_core_device *vdev,
468	unsigned index, unsigned start,	468	unsigned index, unsigned start,
469	unsigned count, uint32_t flags, void *data)	469	unsigned count, uint32_t flags, void *data)
470	{	470	{
@@ -507,7 +507,7 @@ static int vfio_pci_set_intx_trigger(struct vfio_pci_device *vdev, · 36 unmodified lines
507	return 0;	507	return 0;
508	}	508	}
509		509
510	static int vfio_pci_set_msi_trigger(struct vfio_pci_device *vdev,	510	static int vfio_pci_set_msi_trigger(struct vfio_pci_core_device *vdev,
511	unsigned index, unsigned start,	511	unsigned index, unsigned start,
512	unsigned count, uint32_t flags, void *data)	512	unsigned count, uint32_t flags, void *data)
513	{	513	{
@@ -613,7 +613,7 @@ static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx, · 99 unmodified lines
613	return -EINVAL;	613	return -EINVAL;
614	}	614	}
615		615
616	static int vfio_pci_set_err_trigger(struct vfio_pci_device *vdev,	616	static int vfio_pci_set_err_trigger(struct vfio_pci_core_device *vdev,
617	unsigned index, unsigned start,	617	unsigned index, unsigned start,
618	unsigned count, uint32_t flags, void *data)	618	unsigned count, uint32_t flags, void *data)
619	{	619	{
@@ -624,7 +624,7 @@ static int vfio_pci_set_err_trigger(struct vfio_pci_device *vdev, · 4 unmodified lines
624	count, flags, data);	624	count, flags, data);
625	}	625	}
626		626
627	static int vfio_pci_set_req_trigger(struct vfio_pci_device *vdev,	627	static int vfio_pci_set_req_trigger(struct vfio_pci_core_device *vdev,
628	unsigned index, unsigned start,	628	unsigned index, unsigned start,
629	unsigned count, uint32_t flags, void *data)	629	unsigned count, uint32_t flags, void *data)
630	{	630	{
@@ -635,11 +635,11 @@ static int vfio_pci_set_req_trigger(struct vfio_pci_device *vdev, · 4 unmodified lines
635	count, flags, data);	635	count, flags, data);
636	}	636	}
637		637
638	int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev, uint32_t flags,	638	int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags,
639	unsigned index, unsigned start, unsigned count,	639	unsigned index, unsigned start, unsigned count,
640	void *data)	640	void *data)
641	{	641	{
642	int (func)(struct vfio_pci_device vdev, unsigned index,	642	int (func)(struct vfio_pci_core_device vdev, unsigned index,
643	unsigned start, unsigned count, uint32_t flags,	643	unsigned start, unsigned count, uint32_t flags,
644	void *data) = NULL;	644	void *data) = NULL;
645		645

drivers/vfio/pci/vfio_pci_rdwr.c+9−9

@@ -17,7 +17,7 @@ · 16 unmodified lines
17	#include <linux/vfio.h>	17	#include <linux/vfio.h>
18	#include <linux/vgaarb.h>	18	#include <linux/vgaarb.h>
19		19
20	#include "vfio_pci_private.h"	20	#include <linux/vfio_pci_core.h>
21		21
22	#ifdef __LITTLE_ENDIAN	22	#ifdef __LITTLE_ENDIAN
23	#define vfio_ioread64 ioread64	23	#define vfio_ioread64 ioread64
@@ -38,7 +38,7 @@ · 14 unmodified lines
38	#define vfio_iowrite8 iowrite8	38	#define vfio_iowrite8 iowrite8
39		39
40	#define VFIO_IOWRITE(size) \	40	#define VFIO_IOWRITE(size) \
41	static int vfio_pci_iowrite##size(struct vfio_pci_device *vdev, \	41	static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \
42	bool test_mem, u##size val, void __iomem *io) \	42	bool test_mem, u##size val, void __iomem *io) \
43	{ \	43	{ \
44	if (test_mem) { \	44	if (test_mem) { \
@@ -65,7 +65,7 @@ VFIO_IOWRITE(64) · 20 unmodified lines
65	#endif	65	#endif
66		66
67	#define VFIO_IOREAD(size) \	67	#define VFIO_IOREAD(size) \
68	static int vfio_pci_ioread##size(struct vfio_pci_device *vdev, \	68	static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \
69	bool test_mem, u##size val, void __iomem io) \	69	bool test_mem, u##size val, void __iomem io) \
70	{ \	70	{ \
71	if (test_mem) { \	71	if (test_mem) { \
@@ -94,7 +94,7 @@ VFIO_IOREAD(32) · 22 unmodified lines
94	* reads with -1. This is intended for handling MSI-X vector tables and	94	* reads with -1. This is intended for handling MSI-X vector tables and
95	* leftover space for ROM BARs.	95	* leftover space for ROM BARs.
96	*/	96	*/
97	static ssize_t do_io_rw(struct vfio_pci_device *vdev, bool test_mem,	97	static ssize_t do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
98	void __iomem io, char __user buf,	98	void __iomem io, char __user buf,
99	loff_t off, size_t count, size_t x_start,	99	loff_t off, size_t count, size_t x_start,
100	size_t x_end, bool iswrite)	100	size_t x_end, bool iswrite)
@@ -200,7 +200,7 @@ static ssize_t do_io_rw(struct vfio_pci_device *vdev, bool test_mem, · 99 unmodified lines
200	return done;	200	return done;
201	}	201	}
202		202
203	static int vfio_pci_setup_barmap(struct vfio_pci_device *vdev, int bar)	203	static int vfio_pci_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
204	{	204	{
205	struct pci_dev *pdev = vdev->pdev;	205	struct pci_dev *pdev = vdev->pdev;
206	int ret;	206	int ret;
@@ -224,7 +224,7 @@ static int vfio_pci_setup_barmap(struct vfio_pci_device *vdev, int bar) · 17 unmodified lines
224	return 0;	224	return 0;
225	}	225	}
226		226
227	ssize_t vfio_pci_bar_rw(struct vfio_pci_device vdev, char __user buf,	227	ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device vdev, char __user buf,
228	size_t count, loff_t *ppos, bool iswrite)	228	size_t count, loff_t *ppos, bool iswrite)
229	{	229	{
230	struct pci_dev *pdev = vdev->pdev;	230	struct pci_dev *pdev = vdev->pdev;
@@ -288,7 +288,7 @@ out: · 57 unmodified lines
288	return done;	288	return done;
289	}	289	}
290		290
291	ssize_t vfio_pci_vga_rw(struct vfio_pci_device vdev, char __user buf,	291	ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device vdev, char __user buf,
292	size_t count, loff_t *ppos, bool iswrite)	292	size_t count, loff_t *ppos, bool iswrite)
293	{	293	{
294	int ret;	294	int ret;
@@ -384,7 +384,7 @@ static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd, · 89 unmodified lines
384	static int vfio_pci_ioeventfd_handler(void opaque, void unused)	384	static int vfio_pci_ioeventfd_handler(void opaque, void unused)
385	{	385	{
386	struct vfio_pci_ioeventfd *ioeventfd = opaque;	386	struct vfio_pci_ioeventfd *ioeventfd = opaque;
387	struct vfio_pci_device *vdev = ioeventfd->vdev;	387	struct vfio_pci_core_device *vdev = ioeventfd->vdev;
388		388
389	if (ioeventfd->test_mem) {	389	if (ioeventfd->test_mem) {
390	if (!down_read_trylock(&vdev->memory_lock))	390	if (!down_read_trylock(&vdev->memory_lock))
@@ -410,7 +410,7 @@ static void vfio_pci_ioeventfd_thread(void opaque, void unused) · 19 unmodified lines
410	vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem);	410	vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem);
411	}	411	}
412		412
413	long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,	413	long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
414	uint64_t data, int count, int fd)	414	uint64_t data, int count, int fd)
415	{	415	{
416	struct pci_dev *pdev = vdev->pdev;	416	struct pci_dev *pdev = vdev->pdev;

drivers/vfio/pci/vfio_pci_zdev.c+2−2

@@ -14,7 +14,7 @@ · 13 unmodified lines
14	#include <asm/pci_clp.h>	14	#include <asm/pci_clp.h>
15	#include <asm/pci_io.h>	15	#include <asm/pci_io.h>
16		16
17	#include "vfio_pci_private.h"	17	#include <linux/vfio_pci_core.h>
18		18
19	/*	19	/*
20	* Add the Base PCI Function information to the device info region.	20	* Add the Base PCI Function information to the device info region.
@@ -109,7 +109,7 @@ static int zpci_pfip_cap(struct zpci_dev zdev, struct vfio_info_cap caps) · 88 unmodified lines
109	/*	109	/*
110	* Add all supported capabilities to the VFIO_DEVICE_GET_INFO capability chain.	110	* Add all supported capabilities to the VFIO_DEVICE_GET_INFO capability chain.
111	*/	111	*/
112	int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev,	112	int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
113	struct vfio_info_cap *caps)	113	struct vfio_info_cap *caps)
114	{	114	{
115	struct zpci_dev *zdev = to_zpci(vdev->pdev);	115	struct zpci_dev *zdev = to_zpci(vdev->pdev);

drivers/vfio/platform/Kconfig+4−2

 # SPDX-License-Identifier: GPL-2.0-only
 config VFIO_PLATFORM
 	tristate "VFIO support for platform devices"
-	depends on VFIO && EVENTFD && (ARM || ARM64 || COMPILE_TEST)
+	depends on ARM || ARM64 || COMPILE_TEST
 	select VFIO_VIRQFD
 	help
 	  Support for platform devices with VFIO. This is required to make
 	  If you don't know what to do here, say N.
+if VFIO_PLATFORM
 config VFIO_AMBA
 	tristate "VFIO support for AMBA devices"
-	depends on VFIO_PLATFORM && (ARM_AMBA || COMPILE_TEST)
+	depends on ARM_AMBA || COMPILE_TEST
 	help
 	  Support for ARM AMBA devices with VFIO. This is required to make
 	  use of ARM AMBA devices present on the system using the VFIO
 	  If you don't know what to do here, say N.
 source "drivers/vfio/platform/reset/Kconfig"
+endif

drivers/vfio/platform/reset/Kconfig+1−3

 # SPDX-License-Identifier: GPL-2.0-only
 config VFIO_PLATFORM_CALXEDAXGMAC_RESET
 	tristate "VFIO support for calxeda xgmac reset"
-	depends on VFIO_PLATFORM
 	help
 	  Enables the VFIO platform driver to handle reset for Calxeda xgmac
 config VFIO_PLATFORM_AMDXGBE_RESET
 	tristate "VFIO support for AMD XGBE reset"
-	depends on VFIO_PLATFORM
 	help
 	  Enables the VFIO platform driver to handle reset for AMD XGBE
 config VFIO_PLATFORM_BCMFLEXRM_RESET
 	tristate "VFIO support for Broadcom FlexRM reset"
-	depends on VFIO_PLATFORM && (ARCH_BCM_IPROC || COMPILE_TEST)
+	depends on ARCH_BCM_IPROC || COMPILE_TEST
 	default ARCH_BCM_IPROC
 	help
 	  Enables the VFIO platform driver to handle reset for Broadcom FlexRM

drivers/vfio/vfio_iommu_type1.c+4−4

@@ -612,17 +612,17 @@ static int vfio_wait(struct vfio_iommu *iommu) · 611 unmodified lines
612	static int vfio_find_dma_valid(struct vfio_iommu *iommu, dma_addr_t start,	612	static int vfio_find_dma_valid(struct vfio_iommu *iommu, dma_addr_t start,
613	size_t size, struct vfio_dma **dma_p)	613	size_t size, struct vfio_dma **dma_p)
614	{	614	{
615	int ret;	615	int ret = 0;
616		616
617	do {	617	do {
618	*dma_p = vfio_find_dma(iommu, start, size);	618	*dma_p = vfio_find_dma(iommu, start, size);
619	if (!*dma_p)	619	if (!*dma_p)
620	ret = -EINVAL;	620	return -EINVAL;
621	else if (!(*dma_p)->vaddr_invalid)	621	else if (!(*dma_p)->vaddr_invalid)
622	ret = 0;	622	return ret;
623	else	623	else
624	ret = vfio_wait(iommu);	624	ret = vfio_wait(iommu);
625	} while (ret > 0);	625	} while (ret == WAITED);
626		626
627	return ret;	627	return ret;
628	}	628	}

include/linux/mod_devicetable.h+6−0

 #define PCI_ANY_ID (~0)
+enum {
+	PCI_ID_F_VFIO_DRIVER_OVERRIDE = 1,
+};
 /**
  * struct pci_device_id - PCI device ID structure
  * @vendor:		Vendor ID to match (or PCI_ANY_ID)
  *			Best practice is to use driver_data as an index
  *			into a static list of equivalent device types,
  *			instead of using it as a pointer.
+ * @override_only:	Match only when dev->driver_override is this driver.
  */
 struct pci_device_id {
 	__u32 vendor, device;		/* Vendor and device ID or PCI_ANY_ID*/
 	__u32 subvendor, subdevice;	/* Subsystem ID's or PCI_ANY_ID */
 	__u32 class, class_mask;	/* (class,subclass,prog-if) triplet */
 	kernel_ulong_t driver_data;	/* Data private to the driver */
+	__u32 override_only;
 };

include/linux/pci.h+29−0

@@ -901,6 +901,35 @@ struct pci_driver { · 900 unmodified lines
901	.vendor = (vend), .device = (dev), \	901	.vendor = (vend), .device = (dev), \
902	.subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID	902	.subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID
903		903
		904	/**
		905	* PCI_DEVICE_DRIVER_OVERRIDE - macro used to describe a PCI device with
		906	* override_only flags.
		907	* @vend: the 16 bit PCI Vendor ID
		908	* @dev: the 16 bit PCI Device ID
		909	* @driver_override: the 32 bit PCI Device override_only
		910	*
		911	* This macro is used to create a struct pci_device_id that matches only a
		912	* driver_override device. The subvendor and subdevice fields will be set to
		913	* PCI_ANY_ID.
		914	*/
		915	#define PCI_DEVICE_DRIVER_OVERRIDE(vend, dev, driver_override) \
		916	.vendor = (vend), .device = (dev), .subvendor = PCI_ANY_ID, \
		917	.subdevice = PCI_ANY_ID, .override_only = (driver_override)
		918
		919	/**
		920	* PCI_DRIVER_OVERRIDE_DEVICE_VFIO - macro used to describe a VFIO
		921	* "driver_override" PCI device.
		922	* @vend: the 16 bit PCI Vendor ID
		923	* @dev: the 16 bit PCI Device ID
		924	*
		925	* This macro is used to create a struct pci_device_id that matches a
		926	* specific device. The subvendor and subdevice fields will be set to
		927	* PCI_ANY_ID and the driver_override will be set to
		928	* PCI_ID_F_VFIO_DRIVER_OVERRIDE.
		929	*/
		930	#define PCI_DRIVER_OVERRIDE_DEVICE_VFIO(vend, dev) \
		931	PCI_DEVICE_DRIVER_OVERRIDE(vend, dev, PCI_ID_F_VFIO_DRIVER_OVERRIDE)
		932
904	/**	933	/**
905	* PCI_DEVICE_SUB - macro used to describe a specific PCI device with subsystem	934	* PCI_DEVICE_SUB - macro used to describe a specific PCI device with subsystem
906	* @vend: the 16 bit PCI Vendor ID	935	* @vend: the 16 bit PCI Vendor ID

include/linux/vfio_pci_core.hrenamed from drivers/vfio/pci/vfio_pci_private.h+60−29

 #include <linux/mutex.h>
 #include <linux/pci.h>
+#include <linux/vfio.h>
 #include <linux/irqbypass.h>
 #include <linux/types.h>
 #include <linux/uuid.h>
 #include <linux/notifier.h>
-#ifndef VFIO_PCI_PRIVATE_H
+#ifndef VFIO_PCI_CORE_H
-#define VFIO_PCI_PRIVATE_H
+#define VFIO_PCI_CORE_H
 #define VFIO_PCI_OFFSET_SHIFT   40
 struct vfio_pci_ioeventfd {
 	struct list_head	next;
-	struct vfio_pci_device	*vdev;
+	struct vfio_pci_core_device	*vdev;
 	struct virqfd		*virqfd;
 	void __iomem		*addr;
 	uint64_t		data;
 	struct irq_bypass_producer	producer;
 };
-struct vfio_pci_device;
+struct vfio_pci_core_device;
 struct vfio_pci_region;
 struct vfio_pci_regops {
-	ssize_t	(*rw)(struct vfio_pci_device *vdev, char __user *buf,
+	ssize_t (*rw)(struct vfio_pci_core_device *vdev, char __user *buf,
 		      size_t count, loff_t *ppos, bool iswrite);
-	void	(*release)(struct vfio_pci_device *vdev,
+	void	(*release)(struct vfio_pci_core_device *vdev,
 			   struct vfio_pci_region *region);
-	int	(*mmap)(struct vfio_pci_device *vdev,
+	int	(*mmap)(struct vfio_pci_core_device *vdev,
 			struct vfio_pci_region *region,
 			struct vm_area_struct *vma);
-	int	(*add_capability)(struct vfio_pci_device *vdev,
+	int	(*add_capability)(struct vfio_pci_core_device *vdev,
 				  struct vfio_pci_region *region,
 				  struct vfio_info_cap *caps);
 };
 	struct list_head	vma_next;
 };
-struct vfio_pci_device {
+struct vfio_pci_core_device {
 	struct vfio_device	vdev;
 	struct pci_dev		*pdev;
 	void __iomem		*barmap[PCI_STD_NUM_BARS];
 #define is_irq_none(vdev) (!(is_intx(vdev) || is_msi(vdev) || is_msix(vdev)))
 #define irq_is(vdev, type) (vdev->irq_type == type)
-extern void vfio_pci_intx_mask(struct vfio_pci_device *vdev);
+extern void vfio_pci_intx_mask(struct vfio_pci_core_device *vdev);
-extern void vfio_pci_intx_unmask(struct vfio_pci_device *vdev);
+extern void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev);
-extern int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev,
+extern int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev,
 				   uint32_t flags, unsigned index,
 				   unsigned start, unsigned count, void *data);
-extern ssize_t vfio_pci_config_rw(struct vfio_pci_device *vdev,
+extern ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev,
 				  char __user *buf, size_t count,
 				  loff_t *ppos, bool iswrite);
-extern ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf,
+extern ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf,
 			       size_t count, loff_t *ppos, bool iswrite);
-extern ssize_t vfio_pci_vga_rw(struct vfio_pci_device *vdev, char __user *buf,
+extern ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf,
 			       size_t count, loff_t *ppos, bool iswrite);
-extern long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
+extern long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
 			       uint64_t data, int count, int fd);
 extern int vfio_pci_init_perm_bits(void);
 extern void vfio_pci_uninit_perm_bits(void);
-extern int vfio_config_init(struct vfio_pci_device *vdev);
+extern int vfio_config_init(struct vfio_pci_core_device *vdev);
-extern void vfio_config_free(struct vfio_pci_device *vdev);
+extern void vfio_config_free(struct vfio_pci_core_device *vdev);
-extern int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
+extern int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev,
 					unsigned int type, unsigned int subtype,
 					const struct vfio_pci_regops *ops,
 					size_t size, u32 flags, void *data);
-extern int vfio_pci_set_power_state(struct vfio_pci_device *vdev,
+extern int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev,
 				    pci_power_t state);
-extern bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev);
+extern bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev);
-extern void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device
+extern void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device
 						    *vdev);
-extern u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_device *vdev);
+extern u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev);
-extern void vfio_pci_memory_unlock_and_restore(struct vfio_pci_device *vdev,
+extern void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev,
 					       u16 cmd);
 #ifdef CONFIG_VFIO_PCI_IGD
-extern int vfio_pci_igd_init(struct vfio_pci_device *vdev);
+extern int vfio_pci_igd_init(struct vfio_pci_core_device *vdev);
 #else
-static inline int vfio_pci_igd_init(struct vfio_pci_device *vdev)
+static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev)
 {
 	return -ENODEV;
 }
 #endif
 #ifdef CONFIG_S390
-extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev,
+extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
 				       struct vfio_info_cap *caps);
 #else
-static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev,
+static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
 					      struct vfio_info_cap *caps)
 {
 	return -ENODEV;
 }
 #endif
-#endif /* VFIO_PCI_PRIVATE_H */
+/* Will be exported for vfio pci drivers usage */
+void vfio_pci_core_set_params(bool nointxmask, bool is_disable_vga,
+			      bool is_disable_idle_d3);
+void vfio_pci_core_close_device(struct vfio_device *core_vdev);
+void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev,
+			       struct pci_dev *pdev,
+			       const struct vfio_device_ops *vfio_pci_ops);
+int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev);
+void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev);
+void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev);
+int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn);
+extern const struct pci_error_handlers vfio_pci_core_err_handlers;
+long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
+		unsigned long arg);
+ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf,
+		size_t count, loff_t *ppos);
+ssize_t vfio_pci_core_write(struct vfio_device *core_vdev, const char __user *buf,
+		size_t count, loff_t *ppos);
+int vfio_pci_core_mmap(struct vfio_device *core_vdev, struct vm_area_struct *vma);
+void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count);
+int vfio_pci_core_match(struct vfio_device *core_vdev, char *buf);
+int vfio_pci_core_enable(struct vfio_pci_core_device *vdev);
+void vfio_pci_core_disable(struct vfio_pci_core_device *vdev);
+void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev);
+static inline bool vfio_pci_is_vga(struct pci_dev *pdev)
+{
+	return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;
+}
+#endif /* VFIO_PCI_CORE_H */

scripts/mod/devicetable-offsets.c+1−0

@@ -42,6 +42,7 @@ int main(void) · 41 unmodified lines
42	DEVID_FIELD(pci_device_id, subdevice);	42	DEVID_FIELD(pci_device_id, subdevice);
43	DEVID_FIELD(pci_device_id, class);	43	DEVID_FIELD(pci_device_id, class);
44	DEVID_FIELD(pci_device_id, class_mask);	44	DEVID_FIELD(pci_device_id, class_mask);
		45	DEVID_FIELD(pci_device_id, override_only);
45		46
46	DEVID(ccw_device_id);	47	DEVID(ccw_device_id);
47	DEVID_FIELD(ccw_device_id, match_flags);	48	DEVID_FIELD(ccw_device_id, match_flags);

scripts/mod/file2alias.c+15−2

 	return 1;
 }
-/* Looks like: pci:vNdNsvNsdNbcNscNiN. */
+/* Looks like: pci:vNdNsvNsdNbcNscNiN or <prefix>_pci:vNdNsvNsdNbcNscNiN. */
 static int do_pci_entry(const char *filename,
 			void *symval, char *alias)
 {
 	DEF_FIELD(symval, pci_device_id, subdevice);
 	DEF_FIELD(symval, pci_device_id, class);
 	DEF_FIELD(symval, pci_device_id, class_mask);
+	DEF_FIELD(symval, pci_device_id, override_only);
+	switch (override_only) {
+	case 0:
+		strcpy(alias, "pci:");
+		break;
+	case PCI_ID_F_VFIO_DRIVER_OVERRIDE:
+		strcpy(alias, "vfio_pci:");
+		break;
+	default:
+		warn("Unknown PCI driver_override alias %08X\n",
+		     override_only);
+		return 0;
+	}
-	strcpy(alias, "pci:");
 	ADD(alias, "v", vendor != PCI_ANY_ID, vendor);
 	ADD(alias, "d", device != PCI_ANY_ID, device);
 	ADD(alias, "sv", subvendor != PCI_ANY_ID, subvendor);

Merge branches 'v5.15/vfio/spdx-license-cleanups', 'v5.15/vfio/dma-valid-waited-v3', 'v5.15/vfio/vfio-pci-core-v5' and 'v5.15/vfio/vfio-ap' into v5.15/vfio/next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code