Merge branches 'v5.15/vfio/spdx-license-cleanups', 'v5.15/vfio/dma-valid-waited-v3', 'v5.15/vfio/vfio-pci-core-v5' and 'v5.15/vfio/vfio-ap' into v5.15/vfio/next

Alex Williamson committed 5 years ago

Commit ea870734 parents 29848a0 ffc95d1 7fa005c eb0feef28 files · +2704 −2483Browse files

Unified · Split

28 files

Documentation/PCI/pci.rst +1 −0
MAINTAINERS +1 −0
arch/s390/include/asm/kvm_host.h +3 −5
arch/s390/kvm/kvm-s390.c +28 −4
arch/s390/kvm/priv.c +9 −6
drivers/pci/pci-driver.c +21 −7
drivers/s390/crypto/vfio_ap_ops.c +132 −150
drivers/s390/crypto/vfio_ap_private.h +3 −3
drivers/vfio/Kconfig +15 −14
drivers/vfio/fsl-mc/Kconfig +2 −1
drivers/vfio/mdev/Kconfig +0 −1
drivers/vfio/pci/Kconfig +22 −18
drivers/vfio/pci/Makefile +5 −3
drivers/vfio/pci/vfio_pci.c +107 −2155
drivers/vfio/pci/vfio_pci_config.c +35 −35
drivers/vfio/pci/vfio_pci_core.c +2158 −0
drivers/vfio/pci/vfio_pci_igd.c +10 −9
drivers/vfio/pci/vfio_pci_intrs.c +21 −21
drivers/vfio/pci/vfio_pci_rdwr.c +9 −9
drivers/vfio/pci/vfio_pci_zdev.c +2 −2
drivers/vfio/platform/Kconfig +4 −2
drivers/vfio/platform/reset/Kconfig +1 −3
drivers/vfio/vfio_iommu_type1.c +4 −4
include/linux/mod_devicetable.h +6 −0
include/linux/pci.h +29 −0
include/linux/vfio_pci_core.h +60 −29
scripts/mod/devicetable-offsets.c +1 −0
scripts/mod/file2alias.c +15 −2

@@ -103,6 +103,7 @@ need pass only as many optional fields as necessary: · 102 unmodified lines
103	103	- subvendor and subdevice fields default to PCI_ANY_ID (FFFFFFFF)
104	104	- class and classmask fields default to 0
105	105	- driver_data defaults to 0UL.
	106	+ - override_only field defaults to 0.
106	107
107	108	Note that driver_data must match the value used by any of the pci_device_id
108	109	entries defined in the driver. This makes the driver_data field mandatory

MAINTAINERS+1−0

@@ -19466,6 +19466,7 @@ T: git git://github.com/awilliam/linux-vfio.git · 19465 unmodified lines
19466	19466	F: Documentation/driver-api/vfio.rst
19467	19467	F: drivers/vfio/
19468	19468	F: include/linux/vfio.h
	19469	+F: include/linux/vfio_pci_core.h
19469	19470	F: include/uapi/linux/vfio.h
19470	19471
19471	19472	VFIO FSL-MC DRIVER

arch/s390/include/asm/kvm_host.h+3−5

@@ -798,14 +798,12 @@ struct kvm_s390_cpu_model { · 797 unmodified lines
798	798	unsigned short ibc;
799	799	};
800	800
801		−struct kvm_s390_module_hook {
802		− int (hook)(struct kvm_vcpu vcpu);
803		− struct module *owner;
804		−};
	801	+typedef int (crypto_hook)(struct kvm_vcpu vcpu);
805	802
806	803	struct kvm_s390_crypto {
807	804	struct kvm_s390_crypto_cb *crycb;
808		− struct kvm_s390_module_hook *pqap_hook;
	805	+ struct rw_semaphore pqap_hook_rwsem;
	806	+ crypto_hook *pqap_hook;
809	807	__u32 crycbd;
810	808	__u8 aes_kw;
811	809	__u8 dea_kw;

arch/s390/kvm/kvm-s390.c+28−4

@@ -2559,12 +2559,26 @@ static void kvm_s390_set_crycb_format(struct kvm *kvm) · 2558 unmodified lines
2559	2559	kvm->arch.crypto.crycbd \|= CRYCB_FORMAT1;
2560	2560	}
2561	2561
	2562	+/*
	2563	+ * kvm_arch_crypto_set_masks
	2564	+ *
	2565	+ * @kvm: pointer to the target guest's KVM struct containing the crypto masks
	2566	+ * to be set.
	2567	+ * @apm: the mask identifying the accessible AP adapters
	2568	+ * @aqm: the mask identifying the accessible AP domains
	2569	+ * @adm: the mask identifying the accessible AP control domains
	2570	+ *
	2571	+ * Set the masks that identify the adapters, domains and control domains to
	2572	+ * which the KVM guest is granted access.
	2573	+ *
	2574	+ * Note: The kvm->lock mutex must be locked by the caller before invoking this
	2575	+ * function.
	2576	+ */
2562	2577	void kvm_arch_crypto_set_masks(struct kvm kvm, unsigned long apm,
2563	2578	unsigned long aqm, unsigned long adm)
2564	2579	{
2565	2580	struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
2566	2581
2567		− mutex_lock(&kvm->lock);
2568	2582	kvm_s390_vcpu_block_all(kvm);
2569	2583
2570	2584	switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
@@ -2595,13 +2609,23 @@ void kvm_arch_crypto_set_masks(struct kvm kvm, unsigned long apm, · 24 unmodified lines
2595	2609	/* recreate the shadow crycb for each vcpu */
2596	2610	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2597	2611	kvm_s390_vcpu_unblock_all(kvm);
2598		− mutex_unlock(&kvm->lock);
2599	2612	}
2600	2613	EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
2601	2614
	2615	+/*
	2616	+ * kvm_arch_crypto_clear_masks
	2617	+ *
	2618	+ * @kvm: pointer to the target guest's KVM struct containing the crypto masks
	2619	+ * to be cleared.
	2620	+ *
	2621	+ * Clear the masks that identify the adapters, domains and control domains to
	2622	+ * which the KVM guest is granted access.
	2623	+ *
	2624	+ * Note: The kvm->lock mutex must be locked by the caller before invoking this
	2625	+ * function.
	2626	+ */
2602	2627	void kvm_arch_crypto_clear_masks(struct kvm *kvm)
2603	2628	{
2604		− mutex_lock(&kvm->lock);
2605	2629	kvm_s390_vcpu_block_all(kvm);
2606	2630
2607	2631	memset(&kvm->arch.crypto.crycb->apcb0, 0,
@@ -2613,7 +2637,6 @@ void kvm_arch_crypto_clear_masks(struct kvm *kvm) · 5 unmodified lines
2613	2637	/* recreate the shadow crycb for each vcpu */
2614	2638	kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
2615	2639	kvm_s390_vcpu_unblock_all(kvm);
2616		− mutex_unlock(&kvm->lock);
2617	2640	}
2618	2641	EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
2619	2642
@@ -2630,6 +2653,7 @@ static void kvm_s390_crypto_init(struct kvm *kvm) · 10 unmodified lines
2630	2653	{
2631	2654	kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
2632	2655	kvm_s390_set_crycb_format(kvm);
	2656	+ init_rwsem(&kvm->arch.crypto.pqap_hook_rwsem);
2633	2657
2634	2658	if (!test_kvm_facility(kvm, 76))
2635	2659	return;

arch/s390/kvm/priv.c+9−6

@@ -610,6 +610,7 @@ static int handle_io_inst(struct kvm_vcpu *vcpu) · 609 unmodified lines
610	610	static int handle_pqap(struct kvm_vcpu *vcpu)
611	611	{
612	612	struct ap_queue_status status = {};
	613	+ crypto_hook pqap_hook;
613	614	unsigned long reg0;
614	615	int ret;
615	616	uint8_t fc;
@@ -654,18 +655,20 @@ static int handle_pqap(struct kvm_vcpu *vcpu) · 38 unmodified lines
654	655	return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION);
655	656
656	657	/*
657		− * Verify that the hook callback is registered, lock the owner
658		− * and call the hook.
	658	+ * If the hook callback is registered, there will be a pointer to the
	659	+ * hook function pointer in the kvm_s390_crypto structure. Lock the
	660	+ * owner, retrieve the hook function pointer and call the hook.
659	661	*/
	662	+ down_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem);
660	663	if (vcpu->kvm->arch.crypto.pqap_hook) {
661		− if (!try_module_get(vcpu->kvm->arch.crypto.pqap_hook->owner))
662		− return -EOPNOTSUPP;
663		− ret = vcpu->kvm->arch.crypto.pqap_hook->hook(vcpu);
664		− module_put(vcpu->kvm->arch.crypto.pqap_hook->owner);
	664	+ pqap_hook = *vcpu->kvm->arch.crypto.pqap_hook;
	665	+ ret = pqap_hook(vcpu);
665	666	if (!ret && vcpu->run->s.regs.gprs[1] & 0x00ff0000)
666	667	kvm_s390_set_psw_cc(vcpu, 3);
	668	+ up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem);
667	669	return ret;
668	670	}
	671	+ up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem);
669	672	/*
670	673	* A vfio_driver must register a hook.
671	674	* No hook means no driver to enable the SIE CRYCB and no queues.

drivers/pci/pci-driver.c+21−7

@@ -136,7 +136,7 @@ static const struct pci_device_id pci_match_device(struct pci_driver drv, · 135 unmodified lines
136	136	struct pci_dev *dev)
137	137	{
138	138	struct pci_dynid *dynid;
139		− const struct pci_device_id *found_id = NULL;
	139	+ const struct pci_device_id found_id = NULL, ids;
140	140
141	141	/* When driver_override is set, only bind to the matching driver */
142	142	if (dev->driver_override && strcmp(dev->driver_override, drv->name))
@@ -152,14 +152,28 @@ static const struct pci_device_id pci_match_device(struct pci_driver drv, · 9 unmodified lines
152	152	}
153	153	spin_unlock(&drv->dynids.lock);
154	154
155		− if (!found_id)
156		− found_id = pci_match_id(drv->id_table, dev);
	155	+ if (found_id)
	156	+ return found_id;
157	157
158		− /* driver_override will always match, send a dummy id */
159		− if (!found_id && dev->driver_override)
160		− found_id = &pci_device_id_any;
	158	+ for (ids = drv->id_table; (found_id = pci_match_id(ids, dev));
	159	+ ids = found_id + 1) {
	160	+ /*
	161	+ * The match table is split based on driver_override.
	162	+ * In case override_only was set, enforce driver_override
	163	+ * matching.
	164	+ */
	165	+ if (found_id->override_only) {
	166	+ if (dev->driver_override)
	167	+ return found_id;
	168	+ } else {
	169	+ return found_id;
	170	+ }
	171	+ }
161	172
162		− return found_id;
	173	+ /* driver_override will always match, send a dummy id */
	174	+ if (dev->driver_override)
	175	+ return &pci_device_id_any;
	176	+ return NULL;
163	177	}
164	178
165	179	/**

drivers/s390/crypto/vfio_ap_ops.c+132−150

@@ -24,8 +24,9 @@ · 23 unmodified lines
24	24	#define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
25	25	#define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
26	26
27		−static int vfio_ap_mdev_reset_queues(struct mdev_device *mdev);
	27	+static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev);
28	28	static struct vfio_ap_queue *vfio_ap_find_queue(int apqn);
	29	+static const struct vfio_device_ops vfio_ap_matrix_dev_ops;
29	30
30	31	static int match_apqn(struct device dev, const void data)
31	32	{
@@ -294,15 +295,6 @@ static int handle_pqap(struct kvm_vcpu *vcpu) · 262 unmodified lines
294	295	matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,
295	296	struct ap_matrix_mdev, pqap_hook);
296	297
297		− /*
298		− * If the KVM pointer is in the process of being set, wait until the
299		− * process has completed.
300		− */
301		− wait_event_cmd(matrix_mdev->wait_for_kvm,
302		− !matrix_mdev->kvm_busy,
303		− mutex_unlock(&matrix_dev->lock),
304		− mutex_lock(&matrix_dev->lock));
305		−
306	298	/* If the there is no guest using the mdev, there is nothing to do */
307	299	if (!matrix_mdev->kvm)
308	300	goto out_unlock;
@@ -335,45 +327,57 @@ static void vfio_ap_matrix_init(struct ap_config_info *info, · 26 unmodified lines
335	327	matrix->adm_max = info->apxa ? info->Nd : 15;
336	328	}
337	329
338		−static int vfio_ap_mdev_create(struct mdev_device *mdev)
	330	+static int vfio_ap_mdev_probe(struct mdev_device *mdev)
339	331	{
340	332	struct ap_matrix_mdev *matrix_mdev;
	333	+ int ret;
341	334
342	335	if ((atomic_dec_if_positive(&matrix_dev->available_instances) < 0))
343	336	return -EPERM;
344	337
345	338	matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
346	339	if (!matrix_mdev) {
347		− atomic_inc(&matrix_dev->available_instances);
348		− return -ENOMEM;
	340	+ ret = -ENOMEM;
	341	+ goto err_dec_available;
349	342	}
	343	+ vfio_init_group_dev(&matrix_mdev->vdev, &mdev->dev,
	344	+ &vfio_ap_matrix_dev_ops);
350	345
351	346	matrix_mdev->mdev = mdev;
352	347	vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);
353		− init_waitqueue_head(&matrix_mdev->wait_for_kvm);
354		− mdev_set_drvdata(mdev, matrix_mdev);
355		− matrix_mdev->pqap_hook.hook = handle_pqap;
356		− matrix_mdev->pqap_hook.owner = THIS_MODULE;
	348	+ matrix_mdev->pqap_hook = handle_pqap;
357	349	mutex_lock(&matrix_dev->lock);
358	350	list_add(&matrix_mdev->node, &matrix_dev->mdev_list);
359	351	mutex_unlock(&matrix_dev->lock);
360	352
	353	+ ret = vfio_register_group_dev(&matrix_mdev->vdev);
	354	+ if (ret)
	355	+ goto err_list;
	356	+ dev_set_drvdata(&mdev->dev, matrix_mdev);
361	357	return 0;
	358	+
	359	+err_list:
	360	+ mutex_lock(&matrix_dev->lock);
	361	+ list_del(&matrix_mdev->node);
	362	+ mutex_unlock(&matrix_dev->lock);
	363	+ kfree(matrix_mdev);
	364	+err_dec_available:
	365	+ atomic_inc(&matrix_dev->available_instances);
	366	+ return ret;
362	367	}
363	368
364		−static int vfio_ap_mdev_remove(struct mdev_device *mdev)
	369	+static void vfio_ap_mdev_remove(struct mdev_device *mdev)
365	370	{
366		− struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
	371	+ struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(&mdev->dev);
	372	+
	373	+ vfio_unregister_group_dev(&matrix_mdev->vdev);
367	374
368	375	mutex_lock(&matrix_dev->lock);
369		− vfio_ap_mdev_reset_queues(mdev);
	376	+ vfio_ap_mdev_reset_queues(matrix_mdev);
370	377	list_del(&matrix_mdev->node);
371	378	kfree(matrix_mdev);
372		− mdev_set_drvdata(mdev, NULL);
373	379	atomic_inc(&matrix_dev->available_instances);
374	380	mutex_unlock(&matrix_dev->lock);
375		−
376		− return 0;
377	381	}
378	382
379	383	static ssize_t name_show(struct mdev_type *mtype,
@@ -615,16 +619,12 @@ static ssize_t assign_adapter_store(struct device *dev, · 235 unmodified lines
615	619	{
616	620	int ret;
617	621	unsigned long apid;
618		− struct mdev_device *mdev = mdev_from_dev(dev);
619		− struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
	622	+ struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
620	623
621	624	mutex_lock(&matrix_dev->lock);
622	625
623		− /*
624		− * If the KVM pointer is in flux or the guest is running, disallow
625		− * un-assignment of adapter
626		− */
627		− if (matrix_mdev->kvm_busy \|\| matrix_mdev->kvm) {
	626	+ /* If the KVM guest is running, disallow assignment of adapter */
	627	+ if (matrix_mdev->kvm) {
628	628	ret = -EBUSY;
629	629	goto done;
630	630	}
@@ -688,16 +688,12 @@ static ssize_t unassign_adapter_store(struct device *dev, · 57 unmodified lines
688	688	{
689	689	int ret;
690	690	unsigned long apid;
691		− struct mdev_device *mdev = mdev_from_dev(dev);
692		− struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
	691	+ struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
693	692
694	693	mutex_lock(&matrix_dev->lock);
695	694
696		− /*
697		− * If the KVM pointer is in flux or the guest is running, disallow
698		− * un-assignment of adapter
699		− */
700		− if (matrix_mdev->kvm_busy \|\| matrix_mdev->kvm) {
	695	+ /* If the KVM guest is running, disallow unassignment of adapter */
	696	+ if (matrix_mdev->kvm) {
701	697	ret = -EBUSY;
702	698	goto done;
703	699	}
@@ -777,17 +773,13 @@ static ssize_t assign_domain_store(struct device *dev, · 73 unmodified lines
777	773	{
778	774	int ret;
779	775	unsigned long apqi;
780		− struct mdev_device *mdev = mdev_from_dev(dev);
781		− struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
	776	+ struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
782	777	unsigned long max_apqi = matrix_mdev->matrix.aqm_max;
783	778
784	779	mutex_lock(&matrix_dev->lock);
785	780
786		− /*
787		− * If the KVM pointer is in flux or the guest is running, disallow
788		− * assignment of domain
789		− */
790		− if (matrix_mdev->kvm_busy \|\| matrix_mdev->kvm) {
	781	+ /* If the KVM guest is running, disallow assignment of domain */
	782	+ if (matrix_mdev->kvm) {
791	783	ret = -EBUSY;
792	784	goto done;
793	785	}
@@ -846,16 +838,12 @@ static ssize_t unassign_domain_store(struct device *dev, · 52 unmodified lines
846	838	{
847	839	int ret;
848	840	unsigned long apqi;
849		− struct mdev_device *mdev = mdev_from_dev(dev);
850		− struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
	841	+ struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
851	842
852	843	mutex_lock(&matrix_dev->lock);
853	844
854		− /*
855		− * If the KVM pointer is in flux or the guest is running, disallow
856		− * un-assignment of domain
857		− */
858		− if (matrix_mdev->kvm_busy \|\| matrix_mdev->kvm) {
	845	+ /* If the KVM guest is running, disallow unassignment of domain */
	846	+ if (matrix_mdev->kvm) {
859	847	ret = -EBUSY;
860	848	goto done;
861	849	}
@@ -900,16 +888,12 @@ static ssize_t assign_control_domain_store(struct device *dev, · 38 unmodified lines
900	888	{
901	889	int ret;
902	890	unsigned long id;
903		− struct mdev_device *mdev = mdev_from_dev(dev);
904		− struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
	891	+ struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
905	892
906	893	mutex_lock(&matrix_dev->lock);
907	894
908		− /*
909		− * If the KVM pointer is in flux or the guest is running, disallow
910		− * assignment of control domain.
911		− */
912		− if (matrix_mdev->kvm_busy \|\| matrix_mdev->kvm) {
	895	+ /* If the KVM guest is running, disallow assignment of control domain */
	896	+ if (matrix_mdev->kvm) {
913	897	ret = -EBUSY;
914	898	goto done;
915	899	}
@@ -958,17 +942,13 @@ static ssize_t unassign_control_domain_store(struct device *dev, · 42 unmodified lines
958	942	{
959	943	int ret;
960	944	unsigned long domid;
961		− struct mdev_device *mdev = mdev_from_dev(dev);
962		− struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
	945	+ struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
963	946	unsigned long max_domid = matrix_mdev->matrix.adm_max;
964	947
965	948	mutex_lock(&matrix_dev->lock);
966	949
967		− /*
968		− * If the KVM pointer is in flux or the guest is running, disallow
969		− * un-assignment of control domain.
970		− */
971		− if (matrix_mdev->kvm_busy \|\| matrix_mdev->kvm) {
	950	+ /* If a KVM guest is running, disallow unassignment of control domain */
	951	+ if (matrix_mdev->kvm) {
972	952	ret = -EBUSY;
973	953	goto done;
974	954	}
@@ -997,8 +977,7 @@ static ssize_t control_domains_show(struct device *dev, · 22 unmodified lines
997	977	int nchars = 0;
998	978	int n;
999	979	char *bufpos = buf;
1000		− struct mdev_device *mdev = mdev_from_dev(dev);
1001		− struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
	980	+ struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1002	981	unsigned long max_domid = matrix_mdev->matrix.adm_max;
1003	982
1004	983	mutex_lock(&matrix_dev->lock);
@@ -1016,8 +995,7 @@ static DEVICE_ATTR_RO(control_domains); · 11 unmodified lines
1016	995	static ssize_t matrix_show(struct device dev, struct device_attribute attr,
1017	996	char *buf)
1018	997	{
1019		− struct mdev_device *mdev = mdev_from_dev(dev);
1020		− struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
	998	+ struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev);
1021	999	char *bufpos = buf;
1022	1000	unsigned long apid;
1023	1001	unsigned long apqi;
@@ -1109,23 +1087,30 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev, · 85 unmodified lines
1109	1087	struct ap_matrix_mdev *m;
1110	1088
1111	1089	if (kvm->arch.crypto.crycbd) {
	1090	+ down_write(&kvm->arch.crypto.pqap_hook_rwsem);
	1091	+ kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
	1092	+ up_write(&kvm->arch.crypto.pqap_hook_rwsem);
	1093	+
	1094	+ mutex_lock(&kvm->lock);
	1095	+ mutex_lock(&matrix_dev->lock);
	1096	+
1112	1097	list_for_each_entry(m, &matrix_dev->mdev_list, node) {
1113		− if (m != matrix_mdev && m->kvm == kvm)
	1098	+ if (m != matrix_mdev && m->kvm == kvm) {
	1099	+ mutex_unlock(&kvm->lock);
	1100	+ mutex_unlock(&matrix_dev->lock);
1114	1101	return -EPERM;
	1102	+ }
1115	1103	}
1116	1104
1117	1105	kvm_get_kvm(kvm);
1118		− matrix_mdev->kvm_busy = true;
1119		− mutex_unlock(&matrix_dev->lock);
	1106	+ matrix_mdev->kvm = kvm;
1120	1107	kvm_arch_crypto_set_masks(kvm,
1121	1108	matrix_mdev->matrix.apm,
1122	1109	matrix_mdev->matrix.aqm,
1123	1110	matrix_mdev->matrix.adm);
1124		− mutex_lock(&matrix_dev->lock);
1125		− kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
1126		− matrix_mdev->kvm = kvm;
1127		− matrix_mdev->kvm_busy = false;
1128		− wake_up_all(&matrix_mdev->wait_for_kvm);
	1111	+
	1112	+ mutex_unlock(&kvm->lock);
	1113	+ mutex_unlock(&matrix_dev->lock);
1129	1114	}
1130	1115
1131	1116	return 0;
@@ -1175,28 +1160,24 @@ static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb, · 43 unmodified lines
1175	1160	* done under the @matrix_mdev->lock.
1176	1161	*
1177	1162	*/
1178		−static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev)
	1163	+static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev,
	1164	+ struct kvm *kvm)
1179	1165	{
1180		− /*
1181		− * If the KVM pointer is in the process of being set, wait until the
1182		− * process has completed.
1183		− */
1184		− wait_event_cmd(matrix_mdev->wait_for_kvm,
1185		− !matrix_mdev->kvm_busy,
1186		− mutex_unlock(&matrix_dev->lock),
1187		− mutex_lock(&matrix_dev->lock));
	1166	+ if (kvm && kvm->arch.crypto.crycbd) {
	1167	+ down_write(&kvm->arch.crypto.pqap_hook_rwsem);
	1168	+ kvm->arch.crypto.pqap_hook = NULL;
	1169	+ up_write(&kvm->arch.crypto.pqap_hook_rwsem);
1188	1170
1189		− if (matrix_mdev->kvm) {
1190		− matrix_mdev->kvm_busy = true;
1191		− mutex_unlock(&matrix_dev->lock);
1192		− kvm_arch_crypto_clear_masks(matrix_mdev->kvm);
	1171	+ mutex_lock(&kvm->lock);
1193	1172	mutex_lock(&matrix_dev->lock);
1194		− vfio_ap_mdev_reset_queues(matrix_mdev->mdev);
1195		− matrix_mdev->kvm->arch.crypto.pqap_hook = NULL;
1196		− kvm_put_kvm(matrix_mdev->kvm);
	1173	+
	1174	+ kvm_arch_crypto_clear_masks(kvm);
	1175	+ vfio_ap_mdev_reset_queues(matrix_mdev);
	1176	+ kvm_put_kvm(kvm);
1197	1177	matrix_mdev->kvm = NULL;
1198		− matrix_mdev->kvm_busy = false;
1199		− wake_up_all(&matrix_mdev->wait_for_kvm);
	1178	+
	1179	+ mutex_unlock(&kvm->lock);
	1180	+ mutex_unlock(&matrix_dev->lock);
1200	1181	}
1201	1182	}
1202	1183
@@ -1209,16 +1190,13 @@ static int vfio_ap_mdev_group_notifier(struct notifier_block *nb, · 6 unmodified lines
1209	1190	if (action != VFIO_GROUP_NOTIFY_SET_KVM)
1210	1191	return NOTIFY_OK;
1211	1192
1212		− mutex_lock(&matrix_dev->lock);
1213	1193	matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier);
1214	1194
1215	1195	if (!data)
1216		− vfio_ap_mdev_unset_kvm(matrix_mdev);
	1196	+ vfio_ap_mdev_unset_kvm(matrix_mdev, matrix_mdev->kvm);
1217	1197	else if (vfio_ap_mdev_set_kvm(matrix_mdev, data))
1218	1198	notify_rc = NOTIFY_DONE;
1219	1199
1220		− mutex_unlock(&matrix_dev->lock);
1221		−
1222	1200	return notify_rc;
1223	1201	}
1224	1202
@@ -1288,13 +1266,12 @@ free_resources: · 63 unmodified lines
1288	1266	return ret;
1289	1267	}
1290	1268
1291		−static int vfio_ap_mdev_reset_queues(struct mdev_device *mdev)
	1269	+static int vfio_ap_mdev_reset_queues(struct ap_matrix_mdev *matrix_mdev)
1292	1270	{
1293	1271	int ret;
1294	1272	int rc = 0;
1295	1273	unsigned long apid, apqi;
1296	1274	struct vfio_ap_queue *q;
1297		− struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
1298	1275
1299	1276	for_each_set_bit_inv(apid, matrix_mdev->matrix.apm,
1300	1277	matrix_mdev->matrix.apm_max + 1) {
@@ -1315,52 +1292,45 @@ static int vfio_ap_mdev_reset_queues(struct mdev_device *mdev) · 14 unmodified lines
1315	1292	return rc;
1316	1293	}
1317	1294
1318		−static int vfio_ap_mdev_open_device(struct mdev_device *mdev)
	1295	+static int vfio_ap_mdev_open_device(struct vfio_device *vdev)
1319	1296	{
1320		− struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
	1297	+ struct ap_matrix_mdev *matrix_mdev =
	1298	+ container_of(vdev, struct ap_matrix_mdev, vdev);
1321	1299	unsigned long events;
1322	1300	int ret;
1323	1301
1324		−
1325		− if (!try_module_get(THIS_MODULE))
1326		− return -ENODEV;
1327		−
1328	1302	matrix_mdev->group_notifier.notifier_call = vfio_ap_mdev_group_notifier;
1329	1303	events = VFIO_GROUP_NOTIFY_SET_KVM;
1330	1304
1331		− ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
	1305	+ ret = vfio_register_notifier(vdev->dev, VFIO_GROUP_NOTIFY,
1332	1306	&events, &matrix_mdev->group_notifier);
1333		− if (ret) {
1334		− module_put(THIS_MODULE);
	1307	+ if (ret)
1335	1308	return ret;
1336		− }
1337	1309
1338	1310	matrix_mdev->iommu_notifier.notifier_call = vfio_ap_mdev_iommu_notifier;
1339	1311	events = VFIO_IOMMU_NOTIFY_DMA_UNMAP;
1340		− ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
	1312	+ ret = vfio_register_notifier(vdev->dev, VFIO_IOMMU_NOTIFY,
1341	1313	&events, &matrix_mdev->iommu_notifier);
1342		− if (!ret)
1343		− return ret;
	1314	+ if (ret)
	1315	+ goto out_unregister_group;
	1316	+ return 0;
1344	1317
1345		− vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
	1318	+out_unregister_group:
	1319	+ vfio_unregister_notifier(vdev->dev, VFIO_GROUP_NOTIFY,
1346	1320	&matrix_mdev->group_notifier);
1347		− module_put(THIS_MODULE);
1348	1321	return ret;
1349	1322	}
1350	1323
1351		−static void vfio_ap_mdev_close_device(struct mdev_device *mdev)
	1324	+static void vfio_ap_mdev_close_device(struct vfio_device *vdev)
1352	1325	{
1353		− struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
1354		−
1355		− mutex_lock(&matrix_dev->lock);
1356		− vfio_ap_mdev_unset_kvm(matrix_mdev);
1357		− mutex_unlock(&matrix_dev->lock);
	1326	+ struct ap_matrix_mdev *matrix_mdev =
	1327	+ container_of(vdev, struct ap_matrix_mdev, vdev);
1358	1328
1359		− vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
	1329	+ vfio_unregister_notifier(vdev->dev, VFIO_IOMMU_NOTIFY,
1360	1330	&matrix_mdev->iommu_notifier);
1361		− vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
	1331	+ vfio_unregister_notifier(vdev->dev, VFIO_GROUP_NOTIFY,
1362	1332	&matrix_mdev->group_notifier);
1363		− module_put(THIS_MODULE);
	1333	+ vfio_ap_mdev_unset_kvm(matrix_mdev, matrix_mdev->kvm);
1364	1334	}
1365	1335
1366	1336	static int vfio_ap_mdev_get_device_info(unsigned long arg)
@@ -1383,11 +1353,12 @@ static int vfio_ap_mdev_get_device_info(unsigned long arg) · 16 unmodified lines
1383	1353	return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
1384	1354	}
1385	1355
1386		−static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev,
	1356	+static ssize_t vfio_ap_mdev_ioctl(struct vfio_device *vdev,
1387	1357	unsigned int cmd, unsigned long arg)
1388	1358	{
	1359	+ struct ap_matrix_mdev *matrix_mdev =
	1360	+ container_of(vdev, struct ap_matrix_mdev, vdev);
1389	1361	int ret;
1390		− struct ap_matrix_mdev *matrix_mdev;
1391	1362
1392	1363	mutex_lock(&matrix_dev->lock);
1393	1364	switch (cmd) {
@@ -1395,22 +1366,7 @@ static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev, · 1 unmodified line
1395	1366	ret = vfio_ap_mdev_get_device_info(arg);
1396	1367	break;
1397	1368	case VFIO_DEVICE_RESET:
1398		− matrix_mdev = mdev_get_drvdata(mdev);
1399		− if (WARN(!matrix_mdev, "Driver data missing from mdev!!")) {
1400		− ret = -EINVAL;
1401		− break;
1402		− }
1403		−
1404		− /*
1405		− * If the KVM pointer is in the process of being set, wait until
1406		− * the process has completed.
1407		− */
1408		− wait_event_cmd(matrix_mdev->wait_for_kvm,
1409		− !matrix_mdev->kvm_busy,
1410		− mutex_unlock(&matrix_dev->lock),
1411		− mutex_lock(&matrix_dev->lock));
1412		−
1413		− ret = vfio_ap_mdev_reset_queues(mdev);
	1369	+ ret = vfio_ap_mdev_reset_queues(matrix_mdev);
1414	1370	break;
1415	1371	default:
1416	1372	ret = -EOPNOTSUPP;
@@ -1421,25 +1377,51 @@ static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev, · 4 unmodified lines
1421	1377	return ret;
1422	1378	}
1423	1379
	1380	+static const struct vfio_device_ops vfio_ap_matrix_dev_ops = {
	1381	+ .open_device = vfio_ap_mdev_open_device,
	1382	+ .close_device = vfio_ap_mdev_close_device,
	1383	+ .ioctl = vfio_ap_mdev_ioctl,
	1384	+};
	1385	+
	1386	+static struct mdev_driver vfio_ap_matrix_driver = {
	1387	+ .driver = {
	1388	+ .name = "vfio_ap_mdev",
	1389	+ .owner = THIS_MODULE,
	1390	+ .mod_name = KBUILD_MODNAME,
	1391	+ .dev_groups = vfio_ap_mdev_attr_groups,
	1392	+ },
	1393	+ .probe = vfio_ap_mdev_probe,
	1394	+ .remove = vfio_ap_mdev_remove,
	1395	+};
	1396	+
1424	1397	static const struct mdev_parent_ops vfio_ap_matrix_ops = {
1425	1398	.owner = THIS_MODULE,
	1399	+ .device_driver = &vfio_ap_matrix_driver,
1426	1400	.supported_type_groups = vfio_ap_mdev_type_groups,
1427		− .mdev_attr_groups = vfio_ap_mdev_attr_groups,
1428		− .create = vfio_ap_mdev_create,
1429		− .remove = vfio_ap_mdev_remove,
1430		− .open_device = vfio_ap_mdev_open_device,
1431		− .close_device = vfio_ap_mdev_close_device,
1432		− .ioctl = vfio_ap_mdev_ioctl,
1433	1401	};
1434	1402
1435	1403	int vfio_ap_mdev_register(void)
1436	1404	{
	1405	+ int ret;
	1406	+
1437	1407	atomic_set(&matrix_dev->available_instances, MAX_ZDEV_ENTRIES_EXT);
1438	1408
1439		− return mdev_register_device(&matrix_dev->device, &vfio_ap_matrix_ops);
	1409	+ ret = mdev_register_driver(&vfio_ap_matrix_driver);
	1410	+ if (ret)
	1411	+ return ret;
	1412	+
	1413	+ ret = mdev_register_device(&matrix_dev->device, &vfio_ap_matrix_ops);
	1414	+ if (ret)
	1415	+ goto err_driver;
	1416	+ return 0;
	1417	+
	1418	+err_driver:
	1419	+ mdev_unregister_driver(&vfio_ap_matrix_driver);
	1420	+ return ret;
1440	1421	}
1441	1422
1442	1423	void vfio_ap_mdev_unregister(void)
1443	1424	{
1444	1425	mdev_unregister_device(&matrix_dev->device);
	1426	+ mdev_unregister_driver(&vfio_ap_matrix_driver);
1445	1427	}

drivers/s390/crypto/vfio_ap_private.h+3−3

@@ -18,6 +18,7 @@ · 17 unmodified lines
18	18	#include <linux/delay.h>
19	19	#include <linux/mutex.h>
20	20	#include <linux/kvm_host.h>
	21	+#include <linux/vfio.h>
21	22
22	23	#include "ap_bus.h"
23	24
@@ -79,14 +80,13 @@ struct ap_matrix { · 55 unmodified lines
79	80	* @kvm: the struct holding guest's state
80	81	*/
81	82	struct ap_matrix_mdev {
	83	+ struct vfio_device vdev;
82	84	struct list_head node;
83	85	struct ap_matrix matrix;
84	86	struct notifier_block group_notifier;
85	87	struct notifier_block iommu_notifier;
86		− bool kvm_busy;
87		− wait_queue_head_t wait_for_kvm;
88	88	struct kvm *kvm;
89		− struct kvm_s390_module_hook pqap_hook;
	89	+ crypto_hook pqap_hook;
90	90	struct mdev_device *mdev;
91	91	};
92	92

drivers/vfio/Kconfig+15−14

@@ -1,12 +1,22 @@
1	1	# SPDX-License-Identifier: GPL-2.0-only
	2	+menuconfig VFIO
	3	+ tristate "VFIO Non-Privileged userspace driver framework"
	4	+ select IOMMU_API
	5	+ select VFIO_IOMMU_TYPE1 if MMU && (X86 \|\| S390 \|\| ARM \|\| ARM64)
	6	+ help
	7	+ VFIO provides a framework for secure userspace device drivers.
	8	+ See Documentation/driver-api/vfio.rst for more details.
	9	+
	10	+ If you don't know what to do here, say N.
	11	+
	12	+if VFIO
2	13	config VFIO_IOMMU_TYPE1
3	14	tristate
4		− depends on VFIO
5	15	default n
6	16
7	17	config VFIO_IOMMU_SPAPR_TCE
8	18	tristate
9		− depends on VFIO && SPAPR_TCE_IOMMU
	19	+ depends on SPAPR_TCE_IOMMU
10	20	default VFIO
11	21
12	22	config VFIO_SPAPR_EEH
@@ -16,22 +26,11 @@ config VFIO_SPAPR_EEH · 3 unmodified lines
16	26
17	27	config VFIO_VIRQFD
18	28	tristate
19		− depends on VFIO && EVENTFD
	29	+ select EVENTFD
20	30	default n
21	31
22		−menuconfig VFIO
23		− tristate "VFIO Non-Privileged userspace driver framework"
24		− select IOMMU_API
25		− select VFIO_IOMMU_TYPE1 if MMU && (X86 \|\| S390 \|\| ARM \|\| ARM64)
26		− help
27		− VFIO provides a framework for secure userspace device drivers.
28		− See Documentation/driver-api/vfio.rst for more details.
29		−
30		− If you don't know what to do here, say N.
31		−
32	32	config VFIO_NOIOMMU
33	33	bool "VFIO No-IOMMU support"
34		− depends on VFIO
35	34	help
36	35	VFIO is built on the ability to isolate devices using the IOMMU.
37	36	Only with an IOMMU can userspace access to DMA capable devices be
@@ -48,4 +47,6 @@ source "drivers/vfio/pci/Kconfig" · 10 unmodified lines
48	47	source "drivers/vfio/platform/Kconfig"
49	48	source "drivers/vfio/mdev/Kconfig"
50	49	source "drivers/vfio/fsl-mc/Kconfig"
	50	+endif
	51	+
51	52	source "virt/lib/Kconfig"

drivers/vfio/fsl-mc/Kconfig+2−1

@@ -1,6 +1,7 @@
1	1	config VFIO_FSL_MC
2	2	tristate "VFIO support for QorIQ DPAA2 fsl-mc bus devices"
3		− depends on VFIO && FSL_MC_BUS && EVENTFD
	3	+ depends on FSL_MC_BUS
	4	+ select EVENTFD
4	5	help
5	6	Driver to enable support for the VFIO QorIQ DPAA2 fsl-mc
6	7	(Management Complex) devices. This is required to passthrough

drivers/vfio/mdev/Kconfig+0−1

@@ -2,7 +2,6 @@ · 1 unmodified line
2	2
3	3	config VFIO_MDEV
4	4	tristate "Mediated device driver framework"
5		− depends on VFIO
6	5	default n
7	6	help
8	7	Provides a framework to virtualize devices.

drivers/vfio/pci/Kconfig+22−18

@@ -1,19 +1,29 @@
1	1	# SPDX-License-Identifier: GPL-2.0-only
2		−config VFIO_PCI
3		− tristate "VFIO support for PCI devices"
4		− depends on VFIO && PCI && EVENTFD
5		− depends on MMU
	2	+if PCI && MMU
	3	+config VFIO_PCI_CORE
	4	+ tristate
6	5	select VFIO_VIRQFD
7	6	select IRQ_BYPASS_MANAGER
	7	+
	8	+config VFIO_PCI_MMAP
	9	+ def_bool y if !S390
	10	+
	11	+config VFIO_PCI_INTX
	12	+ def_bool y if !S390
	13	+
	14	+config VFIO_PCI
	15	+ tristate "Generic VFIO support for any PCI device"
	16	+ select VFIO_PCI_CORE
8	17	help
9		− Support for the PCI VFIO bus driver. This is required to make
10		− use of PCI drivers using the VFIO framework.
	18	+ Support for the generic PCI VFIO bus driver which can connect any
	19	+ PCI device to the VFIO framework.
11	20
12	21	If you don't know what to do here, say N.
13	22
	23	+if VFIO_PCI
14	24	config VFIO_PCI_VGA
15		− bool "VFIO PCI support for VGA devices"
16		− depends on VFIO_PCI && X86 && VGA_ARB
	25	+ bool "Generic VFIO PCI support for VGA devices"
	26	+ depends on X86 && VGA_ARB
17	27	help
18	28	Support for VGA extension to VFIO PCI. This exposes an additional
19	29	region on VGA devices for accessing legacy VGA addresses used by
@@ -21,17 +31,9 @@ config VFIO_PCI_VGA · 1 unmodified line
21	31
22	32	If you don't know what to do here, say N.
23	33
24		−config VFIO_PCI_MMAP
25		− depends on VFIO_PCI
26		− def_bool y if !S390
27		−
28		−config VFIO_PCI_INTX
29		− depends on VFIO_PCI
30		− def_bool y if !S390
31		−
32	34	config VFIO_PCI_IGD
33		− bool "VFIO PCI extensions for Intel graphics (GVT-d)"
34		− depends on VFIO_PCI && X86
	35	+ bool "Generic VFIO PCI extensions for Intel graphics (GVT-d)"
	36	+ depends on X86
35	37	default y
36	38	help
37	39	Support for Intel IGD specific extensions to enable direct
@@ -40,3 +42,5 @@ config VFIO_PCI_IGD · 2 unmodified lines
40	42	and LPC bridge config space.
41	43
42	44	To enable Intel IGD assignment through vfio-pci, say Y.
	45	+endif
	46	+endif

drivers/vfio/pci/Makefile+5−3

@@ -1,7 +1,9 @@
1	1	# SPDX-License-Identifier: GPL-2.0-only
2	2
3		−vfio-pci-y := vfio_pci.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o
4		−vfio-pci-$(CONFIG_VFIO_PCI_IGD) += vfio_pci_igd.o
5		−vfio-pci-$(CONFIG_S390) += vfio_pci_zdev.o
	3	+vfio-pci-core-y := vfio_pci_core.o vfio_pci_intrs.o vfio_pci_rdwr.o vfio_pci_config.o
	4	+vfio-pci-core-$(CONFIG_S390) += vfio_pci_zdev.o
	5	+obj-$(CONFIG_VFIO_PCI_CORE) += vfio-pci-core.o
6	6
	7	+vfio-pci-y := vfio_pci.o
	8	+vfio-pci-$(CONFIG_VFIO_PCI_IGD) += vfio_pci_igd.o
7	9	obj-$(CONFIG_VFIO_PCI) += vfio-pci.o

drivers/vfio/pci/vfio_pci.c+107−2155

@@ -1,5 +1,7 @@
1	1	// SPDX-License-Identifier: GPL-2.0-only
2	2	/*
	3	+ * Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved
	4	+ *
3	5	* Copyright (C) 2012 Red Hat, Inc. All rights reserved.
4	6	* Author: Alex Williamson <alex.williamson@redhat.com>
5	7	*
@@ -18,19 +20,13 @@ · 12 unmodified lines
18	20	#include <linux/module.h>
19	21	#include <linux/mutex.h>
20	22	#include <linux/notifier.h>
21		−#include <linux/pci.h>
22	23	#include <linux/pm_runtime.h>
23	24	#include <linux/slab.h>
24	25	#include <linux/types.h>
25	26	#include <linux/uaccess.h>
26		−#include <linux/vfio.h>
27		−#include <linux/vgaarb.h>
28		−#include <linux/nospec.h>
29		−#include <linux/sched/mm.h>
30	27
31		−#include "vfio_pci_private.h"
	28	+#include <linux/vfio_pci_core.h>
32	29
33		−#define DRIVER_VERSION "0.2"
34	30	#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
35	31	#define DRIVER_DESC "VFIO PCI - User Level meta-driver"
36	32
@@ -64,15 +60,6 @@ static bool disable_denylist; · 27 unmodified lines
64	60	module_param(disable_denylist, bool, 0444);
65	61	MODULE_PARM_DESC(disable_denylist, "Disable use of device denylist. Disabling the denylist allows binding to devices with known errata that may lead to exploitable stability or security issues when accessed by untrusted users.");
66	62
67		−static inline bool vfio_vga_disabled(void)
68		−{
69		−#ifdef CONFIG_VFIO_PCI_VGA
70		− return disable_vga;
71		−#else
72		− return true;
73		−#endif
74		−}
75		−
76	63	static bool vfio_pci_dev_in_denylist(struct pci_dev *pdev)
77	64	{
78	65	switch (pdev->vendor) {
@@ -111,2210 +98,175 @@ static bool vfio_pci_is_denylisted(struct pci_dev *pdev) · 32 unmodified lines
111	98	return true;
112	99	}
113	100
114		−/*
115		− * Our VGA arbiter participation is limited since we don't know anything
116		− * about the device itself. However, if the device is the only VGA device
117		− * downstream of a bridge and VFIO VGA support is disabled, then we can
118		− * safely return legacy VGA IO and memory as not decoded since the user
119		− * has no way to get to it and routing can be disabled externally at the
120		− * bridge.
121		− */
122		−static unsigned int vfio_pci_set_vga_decode(void *opaque, bool single_vga)
123		−{
124		− struct vfio_pci_device *vdev = opaque;
125		− struct pci_dev tmp = NULL, pdev = vdev->pdev;
126		− unsigned char max_busnr;
127		− unsigned int decodes;
128		−
129		− if (single_vga \|\| !vfio_vga_disabled() \|\| pci_is_root_bus(pdev->bus))
130		− return VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM \|
131		− VGA_RSRC_LEGACY_IO \| VGA_RSRC_LEGACY_MEM;
132		−
133		− max_busnr = pci_bus_max_busnr(pdev->bus);
134		− decodes = VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM;
135		−
136		− while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) {
137		− if (tmp == pdev \|\|
138		− pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) \|\|
139		− pci_is_root_bus(tmp->bus))
140		− continue;
141		−
142		− if (tmp->bus->number >= pdev->bus->number &&
143		− tmp->bus->number <= max_busnr) {
144		− pci_dev_put(tmp);
145		− decodes \|= VGA_RSRC_LEGACY_IO \| VGA_RSRC_LEGACY_MEM;
146		− break;
147		− }
148		− }
149		−
150		− return decodes;
151		−}
152		−
153		−static inline bool vfio_pci_is_vga(struct pci_dev *pdev)
154		−{
155		− return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;
156		−}
157		−
158		−static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev)
159		−{
160		− struct resource *res;
161		− int i;
162		− struct vfio_pci_dummy_resource *dummy_res;
163		−
164		− for (i = 0; i < PCI_STD_NUM_BARS; i++) {
165		− int bar = i + PCI_STD_RESOURCES;
166		−
167		− res = &vdev->pdev->resource[bar];
168		−
169		− if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP))
170		− goto no_mmap;
171		−
172		− if (!(res->flags & IORESOURCE_MEM))
173		− goto no_mmap;
174		−
175		− /*
176		− * The PCI core shouldn't set up a resource with a
177		− * type but zero size. But there may be bugs that
178		− * cause us to do that.
179		− */
180		− if (!resource_size(res))
181		− goto no_mmap;
182		−
183		− if (resource_size(res) >= PAGE_SIZE) {
184		− vdev->bar_mmap_supported[bar] = true;
185		− continue;
186		− }
187		−
188		− if (!(res->start & ~PAGE_MASK)) {
189		− /*
190		− * Add a dummy resource to reserve the remainder
191		− * of the exclusive page in case that hot-add
192		− * device's bar is assigned into it.
193		− */
194		− dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL);
195		− if (dummy_res == NULL)
196		− goto no_mmap;
197		−
198		− dummy_res->resource.name = "vfio sub-page reserved";
199		− dummy_res->resource.start = res->end + 1;
200		− dummy_res->resource.end = res->start + PAGE_SIZE - 1;
201		− dummy_res->resource.flags = res->flags;
202		− if (request_resource(res->parent,
203		− &dummy_res->resource)) {
204		− kfree(dummy_res);
205		− goto no_mmap;
206		− }
207		− dummy_res->index = bar;
208		− list_add(&dummy_res->res_next,
209		− &vdev->dummy_resources_list);
210		− vdev->bar_mmap_supported[bar] = true;
211		− continue;
212		− }
213		− /*
214		− * Here we don't handle the case when the BAR is not page
215		− * aligned because we can't expect the BAR will be
216		− * assigned into the same location in a page in guest
217		− * when we passthrough the BAR. And it's hard to access
218		− * this BAR in userspace because we have no way to get
219		− * the BAR's location in a page.
220		− */
221		−no_mmap:
222		− vdev->bar_mmap_supported[bar] = false;
223		− }
224		−}
225		−
226		−struct vfio_pci_group_info;
227		−static bool vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set);
228		−static void vfio_pci_disable(struct vfio_pci_device *vdev);
229		−static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
230		− struct vfio_pci_group_info *groups);
231		−
232		−/*
233		− * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND
234		− * _and_ the ability detect when the device is asserting INTx via PCI_STATUS.
235		− * If a device implements the former but not the latter we would typically
236		− * expect broken_intx_masking be set and require an exclusive interrupt.
237		− * However since we do have control of the device's ability to assert INTx,
238		− * we can instead pretend that the device does not implement INTx, virtualizing
239		− * the pin register to report zero and maintaining DisINTx set on the host.
240		− */
241		−static bool vfio_pci_nointx(struct pci_dev *pdev)
242		−{
243		− switch (pdev->vendor) {
244		− case PCI_VENDOR_ID_INTEL:
245		− switch (pdev->device) {
246		− /* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */
247		− case 0x1572:
248		− case 0x1574:
249		− case 0x1580 ... 0x1581:
250		− case 0x1583 ... 0x158b:
251		− case 0x37d0 ... 0x37d2:
252		− /* X550 */
253		− case 0x1563:
254		− return true;
255		− default:
256		− return false;
257		− }
258		− }
259		−
260		− return false;
261		−}
262		−
263		−static void vfio_pci_probe_power_state(struct vfio_pci_device *vdev)
264		−{
265		− struct pci_dev *pdev = vdev->pdev;
266		− u16 pmcsr;
267		−
268		− if (!pdev->pm_cap)
269		− return;
270		−
271		− pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
272		−
273		− vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
274		−}
275		−
276		−/*
277		− * pci_set_power_state() wrapper handling devices which perform a soft reset on
278		− * D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev,
279		− * restore when returned to D0. Saved separately from pci_saved_state for use
280		− * by PM capability emulation and separately from pci_dev internal saved state
281		− * to avoid it being overwritten and consumed around other resets.
282		− */
283		−int vfio_pci_set_power_state(struct vfio_pci_device *vdev, pci_power_t state)
284		−{
285		− struct pci_dev *pdev = vdev->pdev;
286		− bool needs_restore = false, needs_save = false;
287		− int ret;
288		−
289		− if (vdev->needs_pm_restore) {
290		− if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) {
291		− pci_save_state(pdev);
292		− needs_save = true;
293		− }
294		−
295		− if (pdev->current_state >= PCI_D3hot && state <= PCI_D0)
296		− needs_restore = true;
297		− }
298		−
299		− ret = pci_set_power_state(pdev, state);
300		−
301		− if (!ret) {
302		− /* D3 might be unsupported via quirk, skip unless in D3 */
303		− if (needs_save && pdev->current_state >= PCI_D3hot) {
304		− vdev->pm_save = pci_store_saved_state(pdev);
305		− } else if (needs_restore) {
306		− pci_load_and_free_saved_state(pdev, &vdev->pm_save);
307		− pci_restore_state(pdev);
308		− }
309		− }
310		−
311		− return ret;
312		−}
313		−
314		−static int vfio_pci_enable(struct vfio_pci_device *vdev)
	101	+static int vfio_pci_open_device(struct vfio_device *core_vdev)
315	102	{
	103	+ struct vfio_pci_core_device *vdev =
	104	+ container_of(core_vdev, struct vfio_pci_core_device, vdev);
316	105	struct pci_dev *pdev = vdev->pdev;
317	106	int ret;
318		− u16 cmd;
319		− u8 msix_pos;
320		−
321		− vfio_pci_set_power_state(vdev, PCI_D0);
322	107
323		− /* Don't allow our initial saved state to include busmaster */
324		− pci_clear_master(pdev);
325		−
326		− ret = pci_enable_device(pdev);
	108	+ ret = vfio_pci_core_enable(vdev);
327	109	if (ret)
328	110	return ret;
329	111
330		− /* If reset fails because of the device lock, fail this path entirely */
331		− ret = pci_try_reset_function(pdev);
332		− if (ret == -EAGAIN) {
333		− pci_disable_device(pdev);
334		− return ret;
335		− }
336		−
337		− vdev->reset_works = !ret;
338		− pci_save_state(pdev);
339		− vdev->pci_saved_state = pci_store_saved_state(pdev);
340		− if (!vdev->pci_saved_state)
341		− pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__);
342		−
343		− if (likely(!nointxmask)) {
344		− if (vfio_pci_nointx(pdev)) {
345		− pci_info(pdev, "Masking broken INTx support\n");
346		− vdev->nointx = true;
347		− pci_intx(pdev, 0);
348		− } else
349		− vdev->pci_2_3 = pci_intx_mask_supported(pdev);
350		− }
351		−
352		− pci_read_config_word(pdev, PCI_COMMAND, &cmd);
353		− if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
354		− cmd &= ~PCI_COMMAND_INTX_DISABLE;
355		− pci_write_config_word(pdev, PCI_COMMAND, cmd);
356		− }
357		−
358		− ret = vfio_config_init(vdev);
359		− if (ret) {
360		− kfree(vdev->pci_saved_state);
361		− vdev->pci_saved_state = NULL;
362		− pci_disable_device(pdev);
363		− return ret;
364		− }
365		−
366		− msix_pos = pdev->msix_cap;
367		− if (msix_pos) {
368		− u16 flags;
369		− u32 table;
370		−
371		− pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);
372		− pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);
373		−
374		− vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
375		− vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
376		− vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
377		− } else
378		− vdev->msix_bar = 0xFF;
379		−
380		− if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))
381		− vdev->has_vga = true;
382		−
383	112	if (vfio_pci_is_vga(pdev) &&
384	113	pdev->vendor == PCI_VENDOR_ID_INTEL &&
385	114	IS_ENABLED(CONFIG_VFIO_PCI_IGD)) {
386	115	ret = vfio_pci_igd_init(vdev);
387	116	if (ret && ret != -ENODEV) {
388	117	pci_warn(pdev, "Failed to setup Intel IGD regions\n");
389		− goto disable_exit;
	118	+ vfio_pci_core_disable(vdev);
	119	+ return ret;
390	120	}
391	121	}
392	122
393		− vfio_pci_probe_mmaps(vdev);
	123	+ vfio_pci_core_finish_enable(vdev);
394	124
395	125	return 0;
396		−
397		−disable_exit:
398		− vfio_pci_disable(vdev);
399		− return ret;
400		−}
401		−
402		−static void vfio_pci_disable(struct vfio_pci_device *vdev)
403		−{
404		− struct pci_dev *pdev = vdev->pdev;
405		− struct vfio_pci_dummy_resource dummy_res, tmp;
406		− struct vfio_pci_ioeventfd ioeventfd, ioeventfd_tmp;
407		− int i, bar;
408		−
409		− /* For needs_reset */
410		− lockdep_assert_held(&vdev->vdev.dev_set->lock);
411		−
412		− /* Stop the device from further DMA */
413		− pci_clear_master(pdev);
414		−
415		− vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE \|
416		− VFIO_IRQ_SET_ACTION_TRIGGER,
417		− vdev->irq_type, 0, 0, NULL);
418		−
419		− /* Device closed, don't need mutex here */
420		− list_for_each_entry_safe(ioeventfd, ioeventfd_tmp,
421		− &vdev->ioeventfds_list, next) {
422		− vfio_virqfd_disable(&ioeventfd->virqfd);
423		− list_del(&ioeventfd->next);
424		− kfree(ioeventfd);
425		− }
426		− vdev->ioeventfds_nr = 0;
427		−
428		− vdev->virq_disabled = false;
429		−
430		− for (i = 0; i < vdev->num_regions; i++)
431		− vdev->region[i].ops->release(vdev, &vdev->region[i]);
432		−
433		− vdev->num_regions = 0;
434		− kfree(vdev->region);
435		− vdev->region = NULL; /* don't krealloc a freed pointer */
436		−
437		− vfio_config_free(vdev);
438		−
439		− for (i = 0; i < PCI_STD_NUM_BARS; i++) {
440		− bar = i + PCI_STD_RESOURCES;
441		− if (!vdev->barmap[bar])
442		− continue;
443		− pci_iounmap(pdev, vdev->barmap[bar]);
444		− pci_release_selected_regions(pdev, 1 << bar);
445		− vdev->barmap[bar] = NULL;
446		− }
447		−
448		− list_for_each_entry_safe(dummy_res, tmp,
449		− &vdev->dummy_resources_list, res_next) {
450		− list_del(&dummy_res->res_next);
451		− release_resource(&dummy_res->resource);
452		− kfree(dummy_res);
453		− }
454		−
455		− vdev->needs_reset = true;
456		−
457		− /*
458		− * If we have saved state, restore it. If we can reset the device,
459		− * even better. Resetting with current state seems better than
460		− * nothing, but saving and restoring current state without reset
461		− * is just busy work.
462		− */
463		− if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
464		− pci_info(pdev, "%s: Couldn't reload saved state\n", __func__);
465		−
466		− if (!vdev->reset_works)
467		− goto out;
468		−
469		− pci_save_state(pdev);
470		− }
471		−
472		− /*
473		− * Disable INTx and MSI, presumably to avoid spurious interrupts
474		− * during reset. Stolen from pci_reset_function()
475		− */
476		− pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
477		−
478		− /*
479		− * Try to get the locks ourselves to prevent a deadlock. The
480		− * success of this is dependent on being able to lock the device,
481		− * which is not always possible.
482		− * We can not use the "try" reset interface here, which will
483		− * overwrite the previously restored configuration information.
484		− */
485		− if (vdev->reset_works && pci_dev_trylock(pdev)) {
486		− if (!__pci_reset_function_locked(pdev))
487		− vdev->needs_reset = false;
488		− pci_dev_unlock(pdev);
489		− }
490		−
491		− pci_restore_state(pdev);
492		−out:
493		− pci_disable_device(pdev);
494		−
495		− if (!vfio_pci_dev_set_try_reset(vdev->vdev.dev_set) && !disable_idle_d3)
496		− vfio_pci_set_power_state(vdev, PCI_D3hot);
497		−}
498		−
499		−static struct pci_driver vfio_pci_driver;
500		−
501		−static struct vfio_pci_device get_pf_vdev(struct vfio_pci_device vdev)
502		−{
503		− struct pci_dev *physfn = pci_physfn(vdev->pdev);
504		− struct vfio_device *pf_dev;
505		−
506		− if (!vdev->pdev->is_virtfn)
507		− return NULL;
508		−
509		− pf_dev = vfio_device_get_from_dev(&physfn->dev);
510		− if (!pf_dev)
511		− return NULL;
512		−
513		− if (pci_dev_driver(physfn) != &vfio_pci_driver) {
514		− vfio_device_put(pf_dev);
515		− return NULL;
516		− }
517		−
518		− return container_of(pf_dev, struct vfio_pci_device, vdev);
519	126	}
520	127
521		−static void vfio_pci_vf_token_user_add(struct vfio_pci_device *vdev, int val)
522		−{
523		− struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev);
524		−
525		− if (!pf_vdev)
526		− return;
527		−
528		− mutex_lock(&pf_vdev->vf_token->lock);
529		− pf_vdev->vf_token->users += val;
530		− WARN_ON(pf_vdev->vf_token->users < 0);
531		− mutex_unlock(&pf_vdev->vf_token->lock);
532		−
533		− vfio_device_put(&pf_vdev->vdev);
534		−}
	128	+static const struct vfio_device_ops vfio_pci_ops = {
	129	+ .name = "vfio-pci",
	130	+ .open_device = vfio_pci_open_device,
	131	+ .close_device = vfio_pci_core_close_device,
	132	+ .ioctl = vfio_pci_core_ioctl,
	133	+ .read = vfio_pci_core_read,
	134	+ .write = vfio_pci_core_write,
	135	+ .mmap = vfio_pci_core_mmap,
	136	+ .request = vfio_pci_core_request,
	137	+ .match = vfio_pci_core_match,
	138	+};
535	139
536		−static void vfio_pci_close_device(struct vfio_device *core_vdev)
	140	+static int vfio_pci_probe(struct pci_dev pdev, const struct pci_device_id id)
537	141	{
538		− struct vfio_pci_device *vdev =
539		− container_of(core_vdev, struct vfio_pci_device, vdev);
540		−
541		− vfio_pci_vf_token_user_add(vdev, -1);
542		− vfio_spapr_pci_eeh_release(vdev->pdev);
543		− vfio_pci_disable(vdev);
	142	+ struct vfio_pci_core_device *vdev;
	143	+ int ret;
544	144
545		− mutex_lock(&vdev->igate);
546		− if (vdev->err_trigger) {
547		− eventfd_ctx_put(vdev->err_trigger);
548		− vdev->err_trigger = NULL;
549		− }
550		− if (vdev->req_trigger) {
551		− eventfd_ctx_put(vdev->req_trigger);
552		− vdev->req_trigger = NULL;
553		− }
554		− mutex_unlock(&vdev->igate);
555		−}
	145	+ if (vfio_pci_is_denylisted(pdev))
	146	+ return -EINVAL;
556	147
557		−static int vfio_pci_open_device(struct vfio_device *core_vdev)
558		−{
559		− struct vfio_pci_device *vdev =
560		− container_of(core_vdev, struct vfio_pci_device, vdev);
561		− int ret = 0;
	148	+ vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
	149	+ if (!vdev)
	150	+ return -ENOMEM;
	151	+ vfio_pci_core_init_device(vdev, pdev, &vfio_pci_ops);
562	152
563		− ret = vfio_pci_enable(vdev);
	153	+ ret = vfio_pci_core_register_device(vdev);
564	154	if (ret)
565		− return ret;
566		−
567		− vfio_spapr_pci_eeh_open(vdev->pdev);
568		− vfio_pci_vf_token_user_add(vdev, 1);
	155	+ goto out_free;
	156	+ dev_set_drvdata(&pdev->dev, vdev);
569	157	return 0;
570		−}
571		−
572		−static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)
573		−{
574		− if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {
575		− u8 pin;
576		−
577		− if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) \|\|
578		− vdev->nointx \|\| vdev->pdev->is_virtfn)
579		− return 0;
580		−
581		− pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);
582	158
583		− return pin ? 1 : 0;
584		− } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {
585		− u8 pos;
586		− u16 flags;
587		−
588		− pos = vdev->pdev->msi_cap;
589		− if (pos) {
590		− pci_read_config_word(vdev->pdev,
591		− pos + PCI_MSI_FLAGS, &flags);
592		− return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);
593		− }
594		− } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {
595		− u8 pos;
596		− u16 flags;
597		−
598		− pos = vdev->pdev->msix_cap;
599		− if (pos) {
600		− pci_read_config_word(vdev->pdev,
601		− pos + PCI_MSIX_FLAGS, &flags);
602		−
603		− return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;
604		− }
605		− } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) {
606		− if (pci_is_pcie(vdev->pdev))
607		− return 1;
608		− } else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) {
609		− return 1;
610		− }
611		−
612		− return 0;
	159	+out_free:
	160	+ vfio_pci_core_uninit_device(vdev);
	161	+ kfree(vdev);
	162	+ return ret;
613	163	}
614	164
615		−static int vfio_pci_count_devs(struct pci_dev pdev, void data)
	165	+static void vfio_pci_remove(struct pci_dev *pdev)
616	166	{
617		− ((int )data)++;
618		− return 0;
619		−}
	167	+ struct vfio_pci_core_device *vdev = dev_get_drvdata(&pdev->dev);
620	168
621		−struct vfio_pci_fill_info {
622		− int max;
623		− int cur;
624		− struct vfio_pci_dependent_device *devices;
625		−};
	169	+ vfio_pci_core_unregister_device(vdev);
	170	+ vfio_pci_core_uninit_device(vdev);
	171	+ kfree(vdev);
	172	+}
626	173
627		−static int vfio_pci_fill_devs(struct pci_dev pdev, void data)
	174	+static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
628	175	{
629		− struct vfio_pci_fill_info *fill = data;
630		− struct iommu_group *iommu_group;
631		−
632		− if (fill->cur == fill->max)
633		− return -EAGAIN; /* Something changed, try again */
634		−
635		− iommu_group = iommu_group_get(&pdev->dev);
636		− if (!iommu_group)
637		− return -EPERM; /* Cannot reset non-isolated devices */
	176	+ if (!enable_sriov)
	177	+ return -ENOENT;
638	178
639		− fill->devices[fill->cur].group_id = iommu_group_id(iommu_group);
640		− fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus);
641		− fill->devices[fill->cur].bus = pdev->bus->number;
642		− fill->devices[fill->cur].devfn = pdev->devfn;
643		− fill->cur++;
644		− iommu_group_put(iommu_group);
645		− return 0;
	179	+ return vfio_pci_core_sriov_configure(pdev, nr_virtfn);
646	180	}
647	181
648		−struct vfio_pci_group_info {
649		− int count;
650		− struct vfio_group **groups;
	182	+static const struct pci_device_id vfio_pci_table[] = {
	183	+ { PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_ANY_ID, PCI_ANY_ID) }, /* match all by default */
	184	+ {}
651	185	};
652	186
653		−static bool vfio_pci_dev_below_slot(struct pci_dev pdev, struct pci_slot slot)
654		−{
655		− for (; pdev; pdev = pdev->bus->self)
656		− if (pdev->bus == slot->bus)
657		− return (pdev->slot == slot);
658		− return false;
659		−}
	187	+MODULE_DEVICE_TABLE(pci, vfio_pci_table);
660	188
661		−struct vfio_pci_walk_info {
662		− int (fn)(struct pci_dev , void *data);
663		− void *data;
664		− struct pci_dev *pdev;
665		− bool slot;
666		− int ret;
	189	+static struct pci_driver vfio_pci_driver = {
	190	+ .name = "vfio-pci",
	191	+ .id_table = vfio_pci_table,
	192	+ .probe = vfio_pci_probe,
	193	+ .remove = vfio_pci_remove,
	194	+ .sriov_configure = vfio_pci_sriov_configure,
	195	+ .err_handler = &vfio_pci_core_err_handlers,
667	196	};
668	197
669		−static int vfio_pci_walk_wrapper(struct pci_dev pdev, void data)
	198	+static void __init vfio_pci_fill_ids(void)
670	199	{
671		− struct vfio_pci_walk_info *walk = data;
	200	+ char p, id;
	201	+ int rc;
672	202
673		− if (!walk->slot \|\| vfio_pci_dev_below_slot(pdev, walk->pdev->slot))
674		− walk->ret = walk->fn(pdev, walk->data);
	203	+ /* no ids passed actually */
	204	+ if (ids[0] == '\0')
	205	+ return;
675	206
676		− return walk->ret;
677		−}
	207	+ /* add ids specified in the module parameter */
	208	+ p = ids;
	209	+ while ((id = strsep(&p, ","))) {
	210	+ unsigned int vendor, device, subvendor = PCI_ANY_ID,
	211	+ subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
	212	+ int fields;
678	213
679		−static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,
680		− int (fn)(struct pci_dev ,
681		− void data), void data,
682		− bool slot)
683		−{
684		− struct vfio_pci_walk_info walk = {
685		− .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,
686		− };
	214	+ if (!strlen(id))
	215	+ continue;
	216	+
	217	+ fields = sscanf(id, "%x:%x:%x:%x:%x:%x",
	218	+ &vendor, &device, &subvendor, &subdevice,
	219	+ &class, &class_mask);
687	220
688		− pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);
	221	+ if (fields < 2) {
	222	+ pr_warn("invalid id string \"%s\"\n", id);
	223	+ continue;
	224	+ }
689	225
690		− return walk.ret;
	226	+ rc = pci_add_dynid(&vfio_pci_driver, vendor, device,
	227	+ subvendor, subdevice, class, class_mask, 0);
	228	+ if (rc)
	229	+ pr_warn("failed to add dynamic id [%04x:%04x[%04x:%04x]] class %#08x/%08x (%d)\n",
	230	+ vendor, device, subvendor, subdevice,
	231	+ class, class_mask, rc);
	232	+ else
	233	+ pr_info("add [%04x:%04x[%04x:%04x]] class %#08x/%08x\n",
	234	+ vendor, device, subvendor, subdevice,
	235	+ class, class_mask);
	236	+ }
691	237	}
692	238
693		−static int msix_mmappable_cap(struct vfio_pci_device *vdev,
694		− struct vfio_info_cap *caps)
	239	+static int __init vfio_pci_init(void)
695	240	{
696		− struct vfio_info_cap_header header = {
697		− .id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE,
698		− .version = 1
699		− };
	241	+ int ret;
	242	+ bool is_disable_vga = true;
700	243
701		− return vfio_info_add_capability(caps, &header, sizeof(header));
702		−}
	244	+#ifdef CONFIG_VFIO_PCI_VGA
	245	+ is_disable_vga = disable_vga;
	246	+#endif
703	247
704		−int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
705		− unsigned int type, unsigned int subtype,
706		− const struct vfio_pci_regops *ops,
707		− size_t size, u32 flags, void *data)
708		−{
709		− struct vfio_pci_region *region;
	248	+ vfio_pci_core_set_params(nointxmask, is_disable_vga, disable_idle_d3);
710	249
711		− region = krealloc(vdev->region,
712		− (vdev->num_regions + 1) * sizeof(*region),
713		− GFP_KERNEL);
714		− if (!region)
715		− return -ENOMEM;
	250	+ /* Register and scan for devices */
	251	+ ret = pci_register_driver(&vfio_pci_driver);
	252	+ if (ret)
	253	+ return ret;
716	254
717		− vdev->region = region;
718		− vdev->region[vdev->num_regions].type = type;
719		− vdev->region[vdev->num_regions].subtype = subtype;
720		− vdev->region[vdev->num_regions].ops = ops;
721		− vdev->region[vdev->num_regions].size = size;
722		− vdev->region[vdev->num_regions].flags = flags;
723		− vdev->region[vdev->num_regions].data = data;
	255	+ vfio_pci_fill_ids();
724	256
725		− vdev->num_regions++;
	257	+ if (disable_denylist)
	258	+ pr_warn("device denylist disabled.\n");
726	259
727	260	return 0;
728	261	}
	262	+module_init(vfio_pci_init);
729	263
730		−static long vfio_pci_ioctl(struct vfio_device *core_vdev,
731		− unsigned int cmd, unsigned long arg)
732		−{
733		− struct vfio_pci_device *vdev =
734		− container_of(core_vdev, struct vfio_pci_device, vdev);
735		− unsigned long minsz;
736		−
737		− if (cmd == VFIO_DEVICE_GET_INFO) {
738		− struct vfio_device_info info;
739		− struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
740		− unsigned long capsz;
741		− int ret;
742		−
743		− minsz = offsetofend(struct vfio_device_info, num_irqs);
744		−
745		− /* For backward compatibility, cannot require this */
746		− capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset);
747		−
748		− if (copy_from_user(&info, (void __user *)arg, minsz))
749		− return -EFAULT;
750		−
751		− if (info.argsz < minsz)
752		− return -EINVAL;
753		−
754		− if (info.argsz >= capsz) {
755		− minsz = capsz;
756		− info.cap_offset = 0;
757		− }
758		−
759		− info.flags = VFIO_DEVICE_FLAGS_PCI;
760		−
761		− if (vdev->reset_works)
762		− info.flags \|= VFIO_DEVICE_FLAGS_RESET;
763		−
764		− info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
765		− info.num_irqs = VFIO_PCI_NUM_IRQS;
766		−
767		− ret = vfio_pci_info_zdev_add_caps(vdev, &caps);
768		− if (ret && ret != -ENODEV) {
769		− pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n");
770		− return ret;
771		− }
772		−
773		− if (caps.size) {
774		− info.flags \|= VFIO_DEVICE_FLAGS_CAPS;
775		− if (info.argsz < sizeof(info) + caps.size) {
776		− info.argsz = sizeof(info) + caps.size;
777		− } else {
778		− vfio_info_cap_shift(&caps, sizeof(info));
779		− if (copy_to_user((void __user *)arg +
780		− sizeof(info), caps.buf,
781		− caps.size)) {
782		− kfree(caps.buf);
783		− return -EFAULT;
784		− }
785		− info.cap_offset = sizeof(info);
786		− }
787		−
788		− kfree(caps.buf);
789		− }
790		−
791		− return copy_to_user((void __user *)arg, &info, minsz) ?
792		− -EFAULT : 0;
793		−
794		− } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
795		− struct pci_dev *pdev = vdev->pdev;
796		− struct vfio_region_info info;
797		− struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
798		− int i, ret;
799		−
800		− minsz = offsetofend(struct vfio_region_info, offset);
801		−
802		− if (copy_from_user(&info, (void __user *)arg, minsz))
803		− return -EFAULT;
804		−
805		− if (info.argsz < minsz)
806		− return -EINVAL;
807		−
808		− switch (info.index) {
809		− case VFIO_PCI_CONFIG_REGION_INDEX:
810		− info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
811		− info.size = pdev->cfg_size;
812		− info.flags = VFIO_REGION_INFO_FLAG_READ \|
813		− VFIO_REGION_INFO_FLAG_WRITE;
814		− break;
815		− case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
816		− info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
817		− info.size = pci_resource_len(pdev, info.index);
818		− if (!info.size) {
819		− info.flags = 0;
820		− break;
821		− }
822		−
823		− info.flags = VFIO_REGION_INFO_FLAG_READ \|
824		− VFIO_REGION_INFO_FLAG_WRITE;
825		− if (vdev->bar_mmap_supported[info.index]) {
826		− info.flags \|= VFIO_REGION_INFO_FLAG_MMAP;
827		− if (info.index == vdev->msix_bar) {
828		− ret = msix_mmappable_cap(vdev, &caps);
829		− if (ret)
830		− return ret;
831		− }
832		− }
833		−
834		− break;
835		− case VFIO_PCI_ROM_REGION_INDEX:
836		− {
837		− void __iomem *io;
838		− size_t size;
839		− u16 cmd;
840		−
841		− info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
842		− info.flags = 0;
843		−
844		− /* Report the BAR size, not the ROM size */
845		− info.size = pci_resource_len(pdev, info.index);
846		− if (!info.size) {
847		− /* Shadow ROMs appear as PCI option ROMs */
848		− if (pdev->resource[PCI_ROM_RESOURCE].flags &
849		− IORESOURCE_ROM_SHADOW)
850		− info.size = 0x20000;
851		− else
852		− break;
853		− }
854		−
855		− /*
856		− * Is it really there? Enable memory decode for
857		− * implicit access in pci_map_rom().
858		− */
859		− cmd = vfio_pci_memory_lock_and_enable(vdev);
860		− io = pci_map_rom(pdev, &size);
861		− if (io) {
862		− info.flags = VFIO_REGION_INFO_FLAG_READ;
863		− pci_unmap_rom(pdev, io);
864		− } else {
865		− info.size = 0;
866		− }
867		− vfio_pci_memory_unlock_and_restore(vdev, cmd);
868		−
869		− break;
870		− }
871		− case VFIO_PCI_VGA_REGION_INDEX:
872		− if (!vdev->has_vga)
873		− return -EINVAL;
874		−
875		− info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
876		− info.size = 0xc0000;
877		− info.flags = VFIO_REGION_INFO_FLAG_READ \|
878		− VFIO_REGION_INFO_FLAG_WRITE;
879		−
880		− break;
881		− default:
882		− {
883		− struct vfio_region_info_cap_type cap_type = {
884		− .header.id = VFIO_REGION_INFO_CAP_TYPE,
885		− .header.version = 1 };
886		−
887		− if (info.index >=
888		− VFIO_PCI_NUM_REGIONS + vdev->num_regions)
889		− return -EINVAL;
890		− info.index = array_index_nospec(info.index,
891		− VFIO_PCI_NUM_REGIONS +
892		− vdev->num_regions);
893		−
894		− i = info.index - VFIO_PCI_NUM_REGIONS;
895		−
896		− info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
897		− info.size = vdev->region[i].size;
898		− info.flags = vdev->region[i].flags;
899		−
900		− cap_type.type = vdev->region[i].type;
901		− cap_type.subtype = vdev->region[i].subtype;
902		−
903		− ret = vfio_info_add_capability(&caps, &cap_type.header,
904		− sizeof(cap_type));
905		− if (ret)
906		− return ret;
907		−
908		− if (vdev->region[i].ops->add_capability) {
909		− ret = vdev->region[i].ops->add_capability(vdev,
910		− &vdev->region[i], &caps);
911		− if (ret)
912		− return ret;
913		− }
914		− }
915		− }
916		−
917		− if (caps.size) {
918		− info.flags \|= VFIO_REGION_INFO_FLAG_CAPS;
919		− if (info.argsz < sizeof(info) + caps.size) {
920		− info.argsz = sizeof(info) + caps.size;
921		− info.cap_offset = 0;
922		− } else {
923		− vfio_info_cap_shift(&caps, sizeof(info));
924		− if (copy_to_user((void __user *)arg +
925		− sizeof(info), caps.buf,
926		− caps.size)) {
927		− kfree(caps.buf);
928		− return -EFAULT;
929		− }
930		− info.cap_offset = sizeof(info);
931		− }
932		−
933		− kfree(caps.buf);
934		− }
935		−
936		− return copy_to_user((void __user *)arg, &info, minsz) ?
937		− -EFAULT : 0;
938		−
939		− } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
940		− struct vfio_irq_info info;
941		−
942		− minsz = offsetofend(struct vfio_irq_info, count);
943		−
944		− if (copy_from_user(&info, (void __user *)arg, minsz))
945		− return -EFAULT;
946		−
947		− if (info.argsz < minsz \|\| info.index >= VFIO_PCI_NUM_IRQS)
948		− return -EINVAL;
949		−
950		− switch (info.index) {
951		− case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
952		− case VFIO_PCI_REQ_IRQ_INDEX:
953		− break;
954		− case VFIO_PCI_ERR_IRQ_INDEX:
955		− if (pci_is_pcie(vdev->pdev))
956		− break;
957		− fallthrough;
958		− default:
959		− return -EINVAL;
960		− }
961		−
962		− info.flags = VFIO_IRQ_INFO_EVENTFD;
963		−
964		− info.count = vfio_pci_get_irq_count(vdev, info.index);
965		−
966		− if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
967		− info.flags \|= (VFIO_IRQ_INFO_MASKABLE \|
968		− VFIO_IRQ_INFO_AUTOMASKED);
969		− else
970		− info.flags \|= VFIO_IRQ_INFO_NORESIZE;
971		−
972		− return copy_to_user((void __user *)arg, &info, minsz) ?
973		− -EFAULT : 0;
974		−
975		− } else if (cmd == VFIO_DEVICE_SET_IRQS) {
976		− struct vfio_irq_set hdr;
977		− u8 *data = NULL;
978		− int max, ret = 0;
979		− size_t data_size = 0;
980		−
981		− minsz = offsetofend(struct vfio_irq_set, count);
982		−
983		− if (copy_from_user(&hdr, (void __user *)arg, minsz))
984		− return -EFAULT;
985		−
986		− max = vfio_pci_get_irq_count(vdev, hdr.index);
987		−
988		− ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
989		− VFIO_PCI_NUM_IRQS, &data_size);
990		− if (ret)
991		− return ret;
992		−
993		− if (data_size) {
994		− data = memdup_user((void __user *)(arg + minsz),
995		− data_size);
996		− if (IS_ERR(data))
997		− return PTR_ERR(data);
998		− }
999		−
1000		− mutex_lock(&vdev->igate);
1001		−
1002		− ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,
1003		− hdr.start, hdr.count, data);
1004		−
1005		− mutex_unlock(&vdev->igate);
1006		− kfree(data);
1007		−
1008		− return ret;
1009		−
1010		− } else if (cmd == VFIO_DEVICE_RESET) {
1011		− int ret;
1012		−
1013		− if (!vdev->reset_works)
1014		− return -EINVAL;
1015		−
1016		− vfio_pci_zap_and_down_write_memory_lock(vdev);
1017		− ret = pci_try_reset_function(vdev->pdev);
1018		− up_write(&vdev->memory_lock);
1019		−
1020		− return ret;
1021		−
1022		− } else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) {
1023		− struct vfio_pci_hot_reset_info hdr;
1024		− struct vfio_pci_fill_info fill = { 0 };
1025		− struct vfio_pci_dependent_device *devices = NULL;
1026		− bool slot = false;
1027		− int ret = 0;
1028		−
1029		− minsz = offsetofend(struct vfio_pci_hot_reset_info, count);
1030		−
1031		− if (copy_from_user(&hdr, (void __user *)arg, minsz))
1032		− return -EFAULT;
1033		−
1034		− if (hdr.argsz < minsz)
1035		− return -EINVAL;
1036		−
1037		− hdr.flags = 0;
1038		−
1039		− /* Can we do a slot or bus reset or neither? */
1040		− if (!pci_probe_reset_slot(vdev->pdev->slot))
1041		− slot = true;
1042		− else if (pci_probe_reset_bus(vdev->pdev->bus))
1043		− return -ENODEV;
1044		−
1045		− /* How many devices are affected? */
1046		− ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
1047		− vfio_pci_count_devs,
1048		− &fill.max, slot);
1049		− if (ret)
1050		− return ret;
1051		−
1052		− WARN_ON(!fill.max); /* Should always be at least one */
1053		−
1054		− /*
1055		− * If there's enough space, fill it now, otherwise return
1056		− * -ENOSPC and the number of devices affected.
1057		− */
1058		− if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) {
1059		− ret = -ENOSPC;
1060		− hdr.count = fill.max;
1061		− goto reset_info_exit;
1062		− }
1063		−
1064		− devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL);
1065		− if (!devices)
1066		− return -ENOMEM;
1067		−
1068		− fill.devices = devices;
1069		−
1070		− ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
1071		− vfio_pci_fill_devs,
1072		− &fill, slot);
1073		−
1074		− /*
1075		− * If a device was removed between counting and filling,
1076		− * we may come up short of fill.max. If a device was
1077		− * added, we'll have a return of -EAGAIN above.
1078		− */
1079		− if (!ret)
1080		− hdr.count = fill.cur;
1081		−
1082		−reset_info_exit:
1083		− if (copy_to_user((void __user *)arg, &hdr, minsz))
1084		− ret = -EFAULT;
1085		−
1086		− if (!ret) {
1087		− if (copy_to_user((void __user *)(arg + minsz), devices,
1088		− hdr.count * sizeof(*devices)))
1089		− ret = -EFAULT;
1090		− }
1091		−
1092		− kfree(devices);
1093		− return ret;
1094		−
1095		− } else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) {
1096		− struct vfio_pci_hot_reset hdr;
1097		− int32_t *group_fds;
1098		− struct vfio_group **groups;
1099		− struct vfio_pci_group_info info;
1100		− bool slot = false;
1101		− int group_idx, count = 0, ret = 0;
1102		−
1103		− minsz = offsetofend(struct vfio_pci_hot_reset, count);
1104		−
1105		− if (copy_from_user(&hdr, (void __user *)arg, minsz))
1106		− return -EFAULT;
1107		−
1108		− if (hdr.argsz < minsz \|\| hdr.flags)
1109		− return -EINVAL;
1110		−
1111		− /* Can we do a slot or bus reset or neither? */
1112		− if (!pci_probe_reset_slot(vdev->pdev->slot))
1113		− slot = true;
1114		− else if (pci_probe_reset_bus(vdev->pdev->bus))
1115		− return -ENODEV;
1116		−
1117		− /*
1118		− * We can't let userspace give us an arbitrarily large
1119		− * buffer to copy, so verify how many we think there
1120		− * could be. Note groups can have multiple devices so
1121		− * one group per device is the max.
1122		− */
1123		− ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
1124		− vfio_pci_count_devs,
1125		− &count, slot);
1126		− if (ret)
1127		− return ret;
1128		−
1129		− /* Somewhere between 1 and count is OK */
1130		− if (!hdr.count \|\| hdr.count > count)
1131		− return -EINVAL;
1132		−
1133		− group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL);
1134		− groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL);
1135		− if (!group_fds \|\| !groups) {
1136		− kfree(group_fds);
1137		− kfree(groups);
1138		− return -ENOMEM;
1139		− }
1140		−
1141		− if (copy_from_user(group_fds, (void __user *)(arg + minsz),
1142		− hdr.count * sizeof(*group_fds))) {
1143		− kfree(group_fds);
1144		− kfree(groups);
1145		− return -EFAULT;
1146		− }
1147		−
1148		− /*
1149		− * For each group_fd, get the group through the vfio external
1150		− * user interface and store the group and iommu ID. This
1151		− * ensures the group is held across the reset.
1152		− */
1153		− for (group_idx = 0; group_idx < hdr.count; group_idx++) {
1154		− struct vfio_group *group;
1155		− struct fd f = fdget(group_fds[group_idx]);
1156		− if (!f.file) {
1157		− ret = -EBADF;
1158		− break;
1159		− }
1160		−
1161		− group = vfio_group_get_external_user(f.file);
1162		− fdput(f);
1163		− if (IS_ERR(group)) {
1164		− ret = PTR_ERR(group);
1165		− break;
1166		− }
1167		−
1168		− groups[group_idx] = group;
1169		− }
1170		−
1171		− kfree(group_fds);
1172		−
1173		− /* release reference to groups on error */
1174		− if (ret)
1175		− goto hot_reset_release;
1176		−
1177		− info.count = hdr.count;
1178		− info.groups = groups;
1179		−
1180		− ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info);
1181		−
1182		−hot_reset_release:
1183		− for (group_idx--; group_idx >= 0; group_idx--)
1184		− vfio_group_put_external_user(groups[group_idx]);
1185		−
1186		− kfree(groups);
1187		− return ret;
1188		− } else if (cmd == VFIO_DEVICE_IOEVENTFD) {
1189		− struct vfio_device_ioeventfd ioeventfd;
1190		− int count;
1191		−
1192		− minsz = offsetofend(struct vfio_device_ioeventfd, fd);
1193		−
1194		− if (copy_from_user(&ioeventfd, (void __user *)arg, minsz))
1195		− return -EFAULT;
1196		−
1197		− if (ioeventfd.argsz < minsz)
1198		− return -EINVAL;
1199		−
1200		− if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK)
1201		− return -EINVAL;
1202		−
1203		− count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK;
1204		−
1205		− if (hweight8(count) != 1 \|\| ioeventfd.fd < -1)
1206		− return -EINVAL;
1207		−
1208		− return vfio_pci_ioeventfd(vdev, ioeventfd.offset,
1209		− ioeventfd.data, count, ioeventfd.fd);
1210		− } else if (cmd == VFIO_DEVICE_FEATURE) {
1211		− struct vfio_device_feature feature;
1212		− uuid_t uuid;
1213		−
1214		− minsz = offsetofend(struct vfio_device_feature, flags);
1215		−
1216		− if (copy_from_user(&feature, (void __user *)arg, minsz))
1217		− return -EFAULT;
1218		−
1219		− if (feature.argsz < minsz)
1220		− return -EINVAL;
1221		−
1222		− /* Check unknown flags */
1223		− if (feature.flags & ~(VFIO_DEVICE_FEATURE_MASK \|
1224		− VFIO_DEVICE_FEATURE_SET \|
1225		− VFIO_DEVICE_FEATURE_GET \|
1226		− VFIO_DEVICE_FEATURE_PROBE))
1227		− return -EINVAL;
1228		−
1229		− /* GET & SET are mutually exclusive except with PROBE */
1230		− if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) &&
1231		− (feature.flags & VFIO_DEVICE_FEATURE_SET) &&
1232		− (feature.flags & VFIO_DEVICE_FEATURE_GET))
1233		− return -EINVAL;
1234		−
1235		− switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) {
1236		− case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN:
1237		− if (!vdev->vf_token)
1238		− return -ENOTTY;
1239		−
1240		− /*
1241		− * We do not support GET of the VF Token UUID as this
1242		− * could expose the token of the previous device user.
1243		− */
1244		− if (feature.flags & VFIO_DEVICE_FEATURE_GET)
1245		− return -EINVAL;
1246		−
1247		− if (feature.flags & VFIO_DEVICE_FEATURE_PROBE)
1248		− return 0;
1249		−
1250		− /* Don't SET unless told to do so */
1251		− if (!(feature.flags & VFIO_DEVICE_FEATURE_SET))
1252		− return -EINVAL;
1253		−
1254		− if (feature.argsz < minsz + sizeof(uuid))
1255		− return -EINVAL;
1256		−
1257		− if (copy_from_user(&uuid, (void __user *)(arg + minsz),
1258		− sizeof(uuid)))
1259		− return -EFAULT;
1260		−
1261		− mutex_lock(&vdev->vf_token->lock);
1262		− uuid_copy(&vdev->vf_token->uuid, &uuid);
1263		− mutex_unlock(&vdev->vf_token->lock);
1264		−
1265		− return 0;
1266		− default:
1267		− return -ENOTTY;
1268		− }
1269		− }
1270		−
1271		− return -ENOTTY;
1272		−}
1273		−
1274		−static ssize_t vfio_pci_rw(struct vfio_pci_device vdev, char __user buf,
1275		− size_t count, loff_t *ppos, bool iswrite)
1276		−{
1277		− unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
1278		−
1279		− if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1280		− return -EINVAL;
1281		−
1282		− switch (index) {
1283		− case VFIO_PCI_CONFIG_REGION_INDEX:
1284		− return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
1285		−
1286		− case VFIO_PCI_ROM_REGION_INDEX:
1287		− if (iswrite)
1288		− return -EINVAL;
1289		− return vfio_pci_bar_rw(vdev, buf, count, ppos, false);
1290		−
1291		− case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1292		− return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
1293		−
1294		− case VFIO_PCI_VGA_REGION_INDEX:
1295		− return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
1296		− default:
1297		− index -= VFIO_PCI_NUM_REGIONS;
1298		− return vdev->region[index].ops->rw(vdev, buf,
1299		− count, ppos, iswrite);
1300		− }
1301		−
1302		− return -EINVAL;
1303		−}
1304		−
1305		−static ssize_t vfio_pci_read(struct vfio_device core_vdev, char __user buf,
1306		− size_t count, loff_t *ppos)
1307		−{
1308		− struct vfio_pci_device *vdev =
1309		− container_of(core_vdev, struct vfio_pci_device, vdev);
1310		−
1311		− if (!count)
1312		− return 0;
1313		−
1314		− return vfio_pci_rw(vdev, buf, count, ppos, false);
1315		−}
1316		−
1317		−static ssize_t vfio_pci_write(struct vfio_device core_vdev, const char __user buf,
1318		− size_t count, loff_t *ppos)
1319		−{
1320		− struct vfio_pci_device *vdev =
1321		− container_of(core_vdev, struct vfio_pci_device, vdev);
1322		−
1323		− if (!count)
1324		− return 0;
1325		−
1326		− return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true);
1327		−}
1328		−
1329		−/* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */
1330		−static int vfio_pci_zap_and_vma_lock(struct vfio_pci_device *vdev, bool try)
1331		−{
1332		− struct vfio_pci_mmap_vma mmap_vma, tmp;
1333		−
1334		− /*
1335		− * Lock ordering:
1336		− * vma_lock is nested under mmap_lock for vm_ops callback paths.
1337		− * The memory_lock semaphore is used by both code paths calling
1338		− * into this function to zap vmas and the vm_ops.fault callback
1339		− * to protect the memory enable state of the device.
1340		− *
1341		− * When zapping vmas we need to maintain the mmap_lock => vma_lock
1342		− * ordering, which requires using vma_lock to walk vma_list to
1343		− * acquire an mm, then dropping vma_lock to get the mmap_lock and
1344		− * reacquiring vma_lock. This logic is derived from similar
1345		− * requirements in uverbs_user_mmap_disassociate().
1346		− *
1347		− * mmap_lock must always be the top-level lock when it is taken.
1348		− * Therefore we can only hold the memory_lock write lock when
1349		− * vma_list is empty, as we'd need to take mmap_lock to clear
1350		− * entries. vma_list can only be guaranteed empty when holding
1351		− * vma_lock, thus memory_lock is nested under vma_lock.
1352		− *
1353		− * This enables the vm_ops.fault callback to acquire vma_lock,
1354		− * followed by memory_lock read lock, while already holding
1355		− * mmap_lock without risk of deadlock.
1356		− */
1357		− while (1) {
1358		− struct mm_struct *mm = NULL;
1359		−
1360		− if (try) {
1361		− if (!mutex_trylock(&vdev->vma_lock))
1362		− return 0;
1363		− } else {
1364		− mutex_lock(&vdev->vma_lock);
1365		− }
1366		− while (!list_empty(&vdev->vma_list)) {
1367		− mmap_vma = list_first_entry(&vdev->vma_list,
1368		− struct vfio_pci_mmap_vma,
1369		− vma_next);
1370		− mm = mmap_vma->vma->vm_mm;
1371		− if (mmget_not_zero(mm))
1372		− break;
1373		−
1374		− list_del(&mmap_vma->vma_next);
1375		− kfree(mmap_vma);
1376		− mm = NULL;
1377		− }
1378		− if (!mm)
1379		− return 1;
1380		− mutex_unlock(&vdev->vma_lock);
1381		−
1382		− if (try) {
1383		− if (!mmap_read_trylock(mm)) {
1384		− mmput(mm);
1385		− return 0;
1386		− }
1387		− } else {
1388		− mmap_read_lock(mm);
1389		− }
1390		− if (try) {
1391		− if (!mutex_trylock(&vdev->vma_lock)) {
1392		− mmap_read_unlock(mm);
1393		− mmput(mm);
1394		− return 0;
1395		− }
1396		− } else {
1397		− mutex_lock(&vdev->vma_lock);
1398		− }
1399		− list_for_each_entry_safe(mmap_vma, tmp,
1400		− &vdev->vma_list, vma_next) {
1401		− struct vm_area_struct *vma = mmap_vma->vma;
1402		−
1403		− if (vma->vm_mm != mm)
1404		− continue;
1405		−
1406		− list_del(&mmap_vma->vma_next);
1407		− kfree(mmap_vma);
1408		−
1409		− zap_vma_ptes(vma, vma->vm_start,
1410		− vma->vm_end - vma->vm_start);
1411		− }
1412		− mutex_unlock(&vdev->vma_lock);
1413		− mmap_read_unlock(mm);
1414		− mmput(mm);
1415		− }
1416		−}
1417		−
1418		−void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device *vdev)
1419		−{
1420		− vfio_pci_zap_and_vma_lock(vdev, false);
1421		− down_write(&vdev->memory_lock);
1422		− mutex_unlock(&vdev->vma_lock);
1423		−}
1424		−
1425		−u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_device *vdev)
1426		−{
1427		− u16 cmd;
1428		−
1429		− down_write(&vdev->memory_lock);
1430		− pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd);
1431		− if (!(cmd & PCI_COMMAND_MEMORY))
1432		− pci_write_config_word(vdev->pdev, PCI_COMMAND,
1433		− cmd \| PCI_COMMAND_MEMORY);
1434		−
1435		− return cmd;
1436		−}
1437		−
1438		−void vfio_pci_memory_unlock_and_restore(struct vfio_pci_device *vdev, u16 cmd)
1439		−{
1440		− pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd);
1441		− up_write(&vdev->memory_lock);
1442		−}
1443		−
1444		−/* Caller holds vma_lock */
1445		−static int __vfio_pci_add_vma(struct vfio_pci_device *vdev,
1446		− struct vm_area_struct *vma)
1447		−{
1448		− struct vfio_pci_mmap_vma *mmap_vma;
1449		−
1450		− mmap_vma = kmalloc(sizeof(*mmap_vma), GFP_KERNEL);
1451		− if (!mmap_vma)
1452		− return -ENOMEM;
1453		−
1454		− mmap_vma->vma = vma;
1455		− list_add(&mmap_vma->vma_next, &vdev->vma_list);
1456		−
1457		− return 0;
1458		−}
1459		−
1460		−/*
1461		− * Zap mmaps on open so that we can fault them in on access and therefore
1462		− * our vma_list only tracks mappings accessed since last zap.
1463		− */
1464		−static void vfio_pci_mmap_open(struct vm_area_struct *vma)
1465		−{
1466		− zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
1467		−}
1468		−
1469		−static void vfio_pci_mmap_close(struct vm_area_struct *vma)
1470		−{
1471		− struct vfio_pci_device *vdev = vma->vm_private_data;
1472		− struct vfio_pci_mmap_vma *mmap_vma;
1473		−
1474		− mutex_lock(&vdev->vma_lock);
1475		− list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
1476		− if (mmap_vma->vma == vma) {
1477		− list_del(&mmap_vma->vma_next);
1478		− kfree(mmap_vma);
1479		− break;
1480		− }
1481		− }
1482		− mutex_unlock(&vdev->vma_lock);
1483		−}
1484		−
1485		−static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf)
1486		−{
1487		− struct vm_area_struct *vma = vmf->vma;
1488		− struct vfio_pci_device *vdev = vma->vm_private_data;
1489		− struct vfio_pci_mmap_vma *mmap_vma;
1490		− vm_fault_t ret = VM_FAULT_NOPAGE;
1491		−
1492		− mutex_lock(&vdev->vma_lock);
1493		− down_read(&vdev->memory_lock);
1494		−
1495		− if (!__vfio_pci_memory_enabled(vdev)) {
1496		− ret = VM_FAULT_SIGBUS;
1497		− goto up_out;
1498		− }
1499		−
1500		− /*
1501		− * We populate the whole vma on fault, so we need to test whether
1502		− * the vma has already been mapped, such as for concurrent faults
1503		− * to the same vma. io_remap_pfn_range() will trigger a BUG_ON if
1504		− * we ask it to fill the same range again.
1505		− */
1506		− list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
1507		− if (mmap_vma->vma == vma)
1508		− goto up_out;
1509		− }
1510		−
1511		− if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
1512		− vma->vm_end - vma->vm_start,
1513		− vma->vm_page_prot)) {
1514		− ret = VM_FAULT_SIGBUS;
1515		− zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
1516		− goto up_out;
1517		− }
1518		−
1519		− if (__vfio_pci_add_vma(vdev, vma)) {
1520		− ret = VM_FAULT_OOM;
1521		− zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
1522		− }
1523		−
1524		−up_out:
1525		− up_read(&vdev->memory_lock);
1526		− mutex_unlock(&vdev->vma_lock);
1527		− return ret;
1528		−}
1529		−
1530		−static const struct vm_operations_struct vfio_pci_mmap_ops = {
1531		− .open = vfio_pci_mmap_open,
1532		− .close = vfio_pci_mmap_close,
1533		− .fault = vfio_pci_mmap_fault,
1534		−};
1535		−
1536		−static int vfio_pci_mmap(struct vfio_device core_vdev, struct vm_area_struct vma)
1537		−{
1538		− struct vfio_pci_device *vdev =
1539		− container_of(core_vdev, struct vfio_pci_device, vdev);
1540		− struct pci_dev *pdev = vdev->pdev;
1541		− unsigned int index;
1542		− u64 phys_len, req_len, pgoff, req_start;
1543		− int ret;
1544		−
1545		− index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1546		−
1547		− if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1548		− return -EINVAL;
1549		− if (vma->vm_end < vma->vm_start)
1550		− return -EINVAL;
1551		− if ((vma->vm_flags & VM_SHARED) == 0)
1552		− return -EINVAL;
1553		− if (index >= VFIO_PCI_NUM_REGIONS) {
1554		− int regnum = index - VFIO_PCI_NUM_REGIONS;
1555		− struct vfio_pci_region *region = vdev->region + regnum;
1556		−
1557		− if (region->ops && region->ops->mmap &&
1558		− (region->flags & VFIO_REGION_INFO_FLAG_MMAP))
1559		− return region->ops->mmap(vdev, region, vma);
1560		− return -EINVAL;
1561		− }
1562		− if (index >= VFIO_PCI_ROM_REGION_INDEX)
1563		− return -EINVAL;
1564		− if (!vdev->bar_mmap_supported[index])
1565		− return -EINVAL;
1566		−
1567		− phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));
1568		− req_len = vma->vm_end - vma->vm_start;
1569		− pgoff = vma->vm_pgoff &
1570		− ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1571		− req_start = pgoff << PAGE_SHIFT;
1572		−
1573		− if (req_start + req_len > phys_len)
1574		− return -EINVAL;
1575		−
1576		− /*
1577		− * Even though we don't make use of the barmap for the mmap,
1578		− * we need to request the region and the barmap tracks that.
1579		− */
1580		− if (!vdev->barmap[index]) {
1581		− ret = pci_request_selected_regions(pdev,
1582		− 1 << index, "vfio-pci");
1583		− if (ret)
1584		− return ret;
1585		−
1586		− vdev->barmap[index] = pci_iomap(pdev, index, 0);
1587		− if (!vdev->barmap[index]) {
1588		− pci_release_selected_regions(pdev, 1 << index);
1589		− return -ENOMEM;
1590		− }
1591		− }
1592		−
1593		− vma->vm_private_data = vdev;
1594		− vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1595		− vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;
1596		−
1597		− /*
1598		− * See remap_pfn_range(), called from vfio_pci_fault() but we can't
1599		− * change vm_flags within the fault handler. Set them now.
1600		− */
1601		− vma->vm_flags \|= VM_IO \| VM_PFNMAP \| VM_DONTEXPAND \| VM_DONTDUMP;
1602		− vma->vm_ops = &vfio_pci_mmap_ops;
1603		−
1604		− return 0;
1605		−}
1606		−
1607		−static void vfio_pci_request(struct vfio_device *core_vdev, unsigned int count)
1608		−{
1609		− struct vfio_pci_device *vdev =
1610		− container_of(core_vdev, struct vfio_pci_device, vdev);
1611		− struct pci_dev *pdev = vdev->pdev;
1612		−
1613		− mutex_lock(&vdev->igate);
1614		−
1615		− if (vdev->req_trigger) {
1616		− if (!(count % 10))
1617		− pci_notice_ratelimited(pdev,
1618		− "Relaying device request to user (#%u)\n",
1619		− count);
1620		− eventfd_signal(vdev->req_trigger, 1);
1621		− } else if (count == 0) {
1622		− pci_warn(pdev,
1623		− "No device request channel registered, blocked until released by user\n");
1624		− }
1625		−
1626		− mutex_unlock(&vdev->igate);
1627		−}
1628		−
1629		−static int vfio_pci_validate_vf_token(struct vfio_pci_device *vdev,
1630		− bool vf_token, uuid_t *uuid)
1631		−{
1632		− /*
1633		− * There's always some degree of trust or collaboration between SR-IOV
1634		− * PF and VFs, even if just that the PF hosts the SR-IOV capability and
1635		− * can disrupt VFs with a reset, but often the PF has more explicit
1636		− * access to deny service to the VF or access data passed through the
1637		− * VF. We therefore require an opt-in via a shared VF token (UUID) to
1638		− * represent this trust. This both prevents that a VF driver might
1639		− * assume the PF driver is a trusted, in-kernel driver, and also that
1640		− * a PF driver might be replaced with a rogue driver, unknown to in-use
1641		− * VF drivers.
1642		− *
1643		− * Therefore when presented with a VF, if the PF is a vfio device and
1644		− * it is bound to the vfio-pci driver, the user needs to provide a VF
1645		− * token to access the device, in the form of appending a vf_token to
1646		− * the device name, for example:
1647		− *
1648		− * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
1649		− *
1650		− * When presented with a PF which has VFs in use, the user must also
1651		− * provide the current VF token to prove collaboration with existing
1652		− * VF users. If VFs are not in use, the VF token provided for the PF
1653		− * device will act to set the VF token.
1654		− *
1655		− * If the VF token is provided but unused, an error is generated.
1656		− */
1657		− if (!vdev->pdev->is_virtfn && !vdev->vf_token && !vf_token)
1658		− return 0; /* No VF token provided or required */
1659		−
1660		− if (vdev->pdev->is_virtfn) {
1661		− struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev);
1662		− bool match;
1663		−
1664		− if (!pf_vdev) {
1665		− if (!vf_token)
1666		− return 0; /* PF is not vfio-pci, no VF token */
1667		−
1668		− pci_info_ratelimited(vdev->pdev,
1669		− "VF token incorrectly provided, PF not bound to vfio-pci\n");
1670		− return -EINVAL;
1671		− }
1672		−
1673		− if (!vf_token) {
1674		− vfio_device_put(&pf_vdev->vdev);
1675		− pci_info_ratelimited(vdev->pdev,
1676		− "VF token required to access device\n");
1677		− return -EACCES;
1678		− }
1679		−
1680		− mutex_lock(&pf_vdev->vf_token->lock);
1681		− match = uuid_equal(uuid, &pf_vdev->vf_token->uuid);
1682		− mutex_unlock(&pf_vdev->vf_token->lock);
1683		−
1684		− vfio_device_put(&pf_vdev->vdev);
1685		−
1686		− if (!match) {
1687		− pci_info_ratelimited(vdev->pdev,
1688		− "Incorrect VF token provided for device\n");
1689		− return -EACCES;
1690		− }
1691		− } else if (vdev->vf_token) {
1692		− mutex_lock(&vdev->vf_token->lock);
1693		− if (vdev->vf_token->users) {
1694		− if (!vf_token) {
1695		− mutex_unlock(&vdev->vf_token->lock);
1696		− pci_info_ratelimited(vdev->pdev,
1697		− "VF token required to access device\n");
1698		− return -EACCES;
1699		− }
1700		−
1701		− if (!uuid_equal(uuid, &vdev->vf_token->uuid)) {
1702		− mutex_unlock(&vdev->vf_token->lock);
1703		− pci_info_ratelimited(vdev->pdev,
1704		− "Incorrect VF token provided for device\n");
1705		− return -EACCES;
1706		− }
1707		− } else if (vf_token) {
1708		− uuid_copy(&vdev->vf_token->uuid, uuid);
1709		− }
1710		−
1711		− mutex_unlock(&vdev->vf_token->lock);
1712		− } else if (vf_token) {
1713		− pci_info_ratelimited(vdev->pdev,
1714		− "VF token incorrectly provided, not a PF or VF\n");
1715		− return -EINVAL;
1716		− }
1717		−
1718		− return 0;
1719		−}
1720		−
1721		−#define VF_TOKEN_ARG "vf_token="
1722		−
1723		−static int vfio_pci_match(struct vfio_device core_vdev, char buf)
1724		−{
1725		− struct vfio_pci_device *vdev =
1726		− container_of(core_vdev, struct vfio_pci_device, vdev);
1727		− bool vf_token = false;
1728		− uuid_t uuid;
1729		− int ret;
1730		−
1731		− if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev))))
1732		− return 0; /* No match */
1733		−
1734		− if (strlen(buf) > strlen(pci_name(vdev->pdev))) {
1735		− buf += strlen(pci_name(vdev->pdev));
1736		−
1737		− if (*buf != ' ')
1738		− return 0; /* No match: non-whitespace after name */
1739		−
1740		− while (*buf) {
1741		− if (*buf == ' ') {
1742		− buf++;
1743		− continue;
1744		− }
1745		−
1746		− if (!vf_token && !strncmp(buf, VF_TOKEN_ARG,
1747		− strlen(VF_TOKEN_ARG))) {
1748		− buf += strlen(VF_TOKEN_ARG);
1749		−
1750		− if (strlen(buf) < UUID_STRING_LEN)
1751		− return -EINVAL;
1752		−
1753		− ret = uuid_parse(buf, &uuid);
1754		− if (ret)
1755		− return ret;
1756		−
1757		− vf_token = true;
1758		− buf += UUID_STRING_LEN;
1759		− } else {
1760		− /* Unknown/duplicate option */
1761		− return -EINVAL;
1762		− }
1763		− }
1764		− }
1765		−
1766		− ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid);
1767		− if (ret)
1768		− return ret;
1769		−
1770		− return 1; /* Match */
1771		−}
1772		−
1773		−static const struct vfio_device_ops vfio_pci_ops = {
1774		− .name = "vfio-pci",
1775		− .open_device = vfio_pci_open_device,
1776		− .close_device = vfio_pci_close_device,
1777		− .ioctl = vfio_pci_ioctl,
1778		− .read = vfio_pci_read,
1779		− .write = vfio_pci_write,
1780		− .mmap = vfio_pci_mmap,
1781		− .request = vfio_pci_request,
1782		− .match = vfio_pci_match,
1783		−};
1784		−
1785		−static int vfio_pci_bus_notifier(struct notifier_block *nb,
1786		− unsigned long action, void *data)
1787		−{
1788		− struct vfio_pci_device *vdev = container_of(nb,
1789		− struct vfio_pci_device, nb);
1790		− struct device *dev = data;
1791		− struct pci_dev *pdev = to_pci_dev(dev);
1792		− struct pci_dev *physfn = pci_physfn(pdev);
1793		−
1794		− if (action == BUS_NOTIFY_ADD_DEVICE &&
1795		− pdev->is_virtfn && physfn == vdev->pdev) {
1796		− pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n",
1797		− pci_name(pdev));
1798		− pdev->driver_override = kasprintf(GFP_KERNEL, "%s",
1799		− vfio_pci_ops.name);
1800		− } else if (action == BUS_NOTIFY_BOUND_DRIVER &&
1801		− pdev->is_virtfn && physfn == vdev->pdev) {
1802		− struct pci_driver *drv = pci_dev_driver(pdev);
1803		−
1804		− if (drv && drv != &vfio_pci_driver)
1805		− pci_warn(vdev->pdev,
1806		− "VF %s bound to driver %s while PF bound to vfio-pci\n",
1807		− pci_name(pdev), drv->name);
1808		− }
1809		−
1810		− return 0;
1811		−}
1812		−
1813		−static int vfio_pci_vf_init(struct vfio_pci_device *vdev)
1814		−{
1815		− struct pci_dev *pdev = vdev->pdev;
1816		− int ret;
1817		−
1818		− if (!pdev->is_physfn)
1819		− return 0;
1820		−
1821		− vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL);
1822		− if (!vdev->vf_token)
1823		− return -ENOMEM;
1824		−
1825		− mutex_init(&vdev->vf_token->lock);
1826		− uuid_gen(&vdev->vf_token->uuid);
1827		−
1828		− vdev->nb.notifier_call = vfio_pci_bus_notifier;
1829		− ret = bus_register_notifier(&pci_bus_type, &vdev->nb);
1830		− if (ret) {
1831		− kfree(vdev->vf_token);
1832		− return ret;
1833		− }
1834		− return 0;
1835		−}
1836		−
1837		−static void vfio_pci_vf_uninit(struct vfio_pci_device *vdev)
1838		−{
1839		− if (!vdev->vf_token)
1840		− return;
1841		−
1842		− bus_unregister_notifier(&pci_bus_type, &vdev->nb);
1843		− WARN_ON(vdev->vf_token->users);
1844		− mutex_destroy(&vdev->vf_token->lock);
1845		− kfree(vdev->vf_token);
1846		−}
1847		−
1848		−static int vfio_pci_vga_init(struct vfio_pci_device *vdev)
1849		−{
1850		− struct pci_dev *pdev = vdev->pdev;
1851		− int ret;
1852		−
1853		− if (!vfio_pci_is_vga(pdev))
1854		− return 0;
1855		−
1856		− ret = vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode);
1857		− if (ret)
1858		− return ret;
1859		− vga_set_legacy_decoding(pdev, vfio_pci_set_vga_decode(vdev, false));
1860		− return 0;
1861		−}
1862		−
1863		−static void vfio_pci_vga_uninit(struct vfio_pci_device *vdev)
1864		−{
1865		− struct pci_dev *pdev = vdev->pdev;
1866		−
1867		− if (!vfio_pci_is_vga(pdev))
1868		− return;
1869		− vga_client_register(pdev, NULL, NULL, NULL);
1870		− vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM \|
1871		− VGA_RSRC_LEGACY_IO \|
1872		− VGA_RSRC_LEGACY_MEM);
1873		−}
1874		−
1875		−static int vfio_pci_probe(struct pci_dev pdev, const struct pci_device_id id)
1876		−{
1877		− struct vfio_pci_device *vdev;
1878		− struct iommu_group *group;
1879		− int ret;
1880		−
1881		− if (vfio_pci_is_denylisted(pdev))
1882		− return -EINVAL;
1883		−
1884		− if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)
1885		− return -EINVAL;
1886		−
1887		− /*
1888		− * Prevent binding to PFs with VFs enabled, the VFs might be in use
1889		− * by the host or other users. We cannot capture the VFs if they
1890		− * already exist, nor can we track VF users. Disabling SR-IOV here
1891		− * would initiate removing the VFs, which would unbind the driver,
1892		− * which is prone to blocking if that VF is also in use by vfio-pci.
1893		− * Just reject these PFs and let the user sort it out.
1894		− */
1895		− if (pci_num_vf(pdev)) {
1896		− pci_warn(pdev, "Cannot bind to PF with SR-IOV enabled\n");
1897		− return -EBUSY;
1898		− }
1899		−
1900		− group = vfio_iommu_group_get(&pdev->dev);
1901		− if (!group)
1902		− return -EINVAL;
1903		−
1904		− vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
1905		− if (!vdev) {
1906		− ret = -ENOMEM;
1907		− goto out_group_put;
1908		− }
1909		−
1910		− vfio_init_group_dev(&vdev->vdev, &pdev->dev, &vfio_pci_ops);
1911		− vdev->pdev = pdev;
1912		− vdev->irq_type = VFIO_PCI_NUM_IRQS;
1913		− mutex_init(&vdev->igate);
1914		− spin_lock_init(&vdev->irqlock);
1915		− mutex_init(&vdev->ioeventfds_lock);
1916		− INIT_LIST_HEAD(&vdev->dummy_resources_list);
1917		− INIT_LIST_HEAD(&vdev->ioeventfds_list);
1918		− mutex_init(&vdev->vma_lock);
1919		− INIT_LIST_HEAD(&vdev->vma_list);
1920		− init_rwsem(&vdev->memory_lock);
1921		−
1922		− if (pci_is_root_bus(pdev->bus)) {
1923		− ret = vfio_assign_device_set(&vdev->vdev, vdev);
1924		− } else if (!pci_probe_reset_slot(pdev->slot)) {
1925		− ret = vfio_assign_device_set(&vdev->vdev, pdev->slot);
1926		− } else {
1927		− /*
1928		− * If there is no slot reset support for this device, the whole
1929		− * bus needs to be grouped together to support bus-wide resets.
1930		− */
1931		− ret = vfio_assign_device_set(&vdev->vdev, pdev->bus);
1932		− }
1933		−
1934		− if (ret)
1935		− goto out_uninit;
1936		− ret = vfio_pci_vf_init(vdev);
1937		− if (ret)
1938		− goto out_uninit;
1939		− ret = vfio_pci_vga_init(vdev);
1940		− if (ret)
1941		− goto out_vf;
1942		−
1943		− vfio_pci_probe_power_state(vdev);
1944		−
1945		− if (!disable_idle_d3) {
1946		− /*
1947		− * pci-core sets the device power state to an unknown value at
1948		− * bootup and after being removed from a driver. The only
1949		− * transition it allows from this unknown state is to D0, which
1950		− * typically happens when a driver calls pci_enable_device().
1951		− * We're not ready to enable the device yet, but we do want to
1952		− * be able to get to D3. Therefore first do a D0 transition
1953		− * before going to D3.
1954		− */
1955		− vfio_pci_set_power_state(vdev, PCI_D0);
1956		− vfio_pci_set_power_state(vdev, PCI_D3hot);
1957		− }
1958		−
1959		− ret = vfio_register_group_dev(&vdev->vdev);
1960		− if (ret)
1961		− goto out_power;
1962		− dev_set_drvdata(&pdev->dev, vdev);
1963		− return 0;
1964		−
1965		−out_power:
1966		− if (!disable_idle_d3)
1967		− vfio_pci_set_power_state(vdev, PCI_D0);
1968		−out_vf:
1969		− vfio_pci_vf_uninit(vdev);
1970		−out_uninit:
1971		− vfio_uninit_group_dev(&vdev->vdev);
1972		− kfree(vdev->pm_save);
1973		− kfree(vdev);
1974		−out_group_put:
1975		− vfio_iommu_group_put(group, &pdev->dev);
1976		− return ret;
1977		−}
1978		−
1979		−static void vfio_pci_remove(struct pci_dev *pdev)
1980		−{
1981		− struct vfio_pci_device *vdev = dev_get_drvdata(&pdev->dev);
1982		−
1983		− pci_disable_sriov(pdev);
1984		−
1985		− vfio_unregister_group_dev(&vdev->vdev);
1986		−
1987		− vfio_pci_vf_uninit(vdev);
1988		− vfio_uninit_group_dev(&vdev->vdev);
1989		− vfio_pci_vga_uninit(vdev);
1990		−
1991		− vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev);
1992		−
1993		− if (!disable_idle_d3)
1994		− vfio_pci_set_power_state(vdev, PCI_D0);
1995		−
1996		− mutex_destroy(&vdev->ioeventfds_lock);
1997		− kfree(vdev->region);
1998		− kfree(vdev->pm_save);
1999		− kfree(vdev);
2000		−}
2001		−
2002		−static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,
2003		− pci_channel_state_t state)
2004		−{
2005		− struct vfio_pci_device *vdev;
2006		− struct vfio_device *device;
2007		−
2008		− device = vfio_device_get_from_dev(&pdev->dev);
2009		− if (device == NULL)
2010		− return PCI_ERS_RESULT_DISCONNECT;
2011		−
2012		− vdev = container_of(device, struct vfio_pci_device, vdev);
2013		−
2014		− mutex_lock(&vdev->igate);
2015		−
2016		− if (vdev->err_trigger)
2017		− eventfd_signal(vdev->err_trigger, 1);
2018		−
2019		− mutex_unlock(&vdev->igate);
2020		−
2021		− vfio_device_put(device);
2022		−
2023		− return PCI_ERS_RESULT_CAN_RECOVER;
2024		−}
2025		−
2026		−static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
2027		−{
2028		− struct vfio_device *device;
2029		− int ret = 0;
2030		−
2031		− might_sleep();
2032		−
2033		− if (!enable_sriov)
2034		− return -ENOENT;
2035		−
2036		− device = vfio_device_get_from_dev(&pdev->dev);
2037		− if (!device)
2038		− return -ENODEV;
2039		−
2040		− if (nr_virtfn == 0)
2041		− pci_disable_sriov(pdev);
2042		− else
2043		− ret = pci_enable_sriov(pdev, nr_virtfn);
2044		−
2045		− vfio_device_put(device);
2046		−
2047		− return ret < 0 ? ret : nr_virtfn;
2048		−}
2049		−
2050		−static const struct pci_error_handlers vfio_err_handlers = {
2051		− .error_detected = vfio_pci_aer_err_detected,
2052		−};
2053		−
2054		−static struct pci_driver vfio_pci_driver = {
2055		− .name = "vfio-pci",
2056		− .id_table = NULL, /* only dynamic ids */
2057		− .probe = vfio_pci_probe,
2058		− .remove = vfio_pci_remove,
2059		− .sriov_configure = vfio_pci_sriov_configure,
2060		− .err_handler = &vfio_err_handlers,
2061		−};
2062		−
2063		−static bool vfio_dev_in_groups(struct vfio_pci_device *vdev,
2064		− struct vfio_pci_group_info *groups)
2065		−{
2066		− unsigned int i;
2067		−
2068		− for (i = 0; i < groups->count; i++)
2069		− if (groups->groups[i] == vdev->vdev.group)
2070		− return true;
2071		− return false;
2072		−}
2073		−
2074		−static int vfio_pci_is_device_in_set(struct pci_dev pdev, void data)
2075		−{
2076		− struct vfio_device_set *dev_set = data;
2077		− struct vfio_device *cur;
2078		−
2079		− list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
2080		− if (cur->dev == &pdev->dev)
2081		− return 0;
2082		− return -EBUSY;
2083		−}
2084		−
2085		−/*
2086		− * vfio-core considers a group to be viable and will create a vfio_device even
2087		− * if some devices are bound to drivers like pci-stub or pcieport. Here we
2088		− * require all PCI devices to be inside our dev_set since that ensures they stay
2089		− * put and that every driver controlling the device can co-ordinate with the
2090		− * device reset.
2091		− *
2092		− * Returns the pci_dev to pass to pci_reset_bus() if every PCI device to be
2093		− * reset is inside the dev_set, and pci_reset_bus() can succeed. NULL otherwise.
2094		− */
2095		−static struct pci_dev *
2096		−vfio_pci_dev_set_resettable(struct vfio_device_set *dev_set)
2097		−{
2098		− struct pci_dev *pdev;
2099		−
2100		− lockdep_assert_held(&dev_set->lock);
2101		−
2102		− /*
2103		− * By definition all PCI devices in the dev_set share the same PCI
2104		− * reset, so any pci_dev will have the same outcomes for
2105		− * pci_probe_reset_*() and pci_reset_bus().
2106		− */
2107		− pdev = list_first_entry(&dev_set->device_list, struct vfio_pci_device,
2108		− vdev.dev_set_list)->pdev;
2109		−
2110		− /* pci_reset_bus() is supported */
2111		− if (pci_probe_reset_slot(pdev->slot) && pci_probe_reset_bus(pdev->bus))
2112		− return NULL;
2113		−
2114		− if (vfio_pci_for_each_slot_or_bus(pdev, vfio_pci_is_device_in_set,
2115		− dev_set,
2116		− !pci_probe_reset_slot(pdev->slot)))
2117		− return NULL;
2118		− return pdev;
2119		−}
2120		−
2121		−/*
2122		− * We need to get memory_lock for each device, but devices can share mmap_lock,
2123		− * therefore we need to zap and hold the vma_lock for each device, and only then
2124		− * get each memory_lock.
2125		− */
2126		−static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
2127		− struct vfio_pci_group_info *groups)
2128		−{
2129		− struct vfio_pci_device *cur_mem;
2130		− struct vfio_pci_device *cur_vma;
2131		− struct vfio_pci_device *cur;
2132		− struct pci_dev *pdev;
2133		− bool is_mem = true;
2134		− int ret;
2135		−
2136		− mutex_lock(&dev_set->lock);
2137		− cur_mem = list_first_entry(&dev_set->device_list,
2138		− struct vfio_pci_device, vdev.dev_set_list);
2139		−
2140		− pdev = vfio_pci_dev_set_resettable(dev_set);
2141		− if (!pdev) {
2142		− ret = -EINVAL;
2143		− goto err_unlock;
2144		− }
2145		−
2146		− list_for_each_entry(cur_vma, &dev_set->device_list, vdev.dev_set_list) {
2147		− /*
2148		− * Test whether all the affected devices are contained by the
2149		− * set of groups provided by the user.
2150		− */
2151		− if (!vfio_dev_in_groups(cur_vma, groups)) {
2152		− ret = -EINVAL;
2153		− goto err_undo;
2154		− }
2155		−
2156		− /*
2157		− * Locking multiple devices is prone to deadlock, runaway and
2158		− * unwind if we hit contention.
2159		− */
2160		− if (!vfio_pci_zap_and_vma_lock(cur_vma, true)) {
2161		− ret = -EBUSY;
2162		− goto err_undo;
2163		− }
2164		− }
2165		− cur_vma = NULL;
2166		−
2167		− list_for_each_entry(cur_mem, &dev_set->device_list, vdev.dev_set_list) {
2168		− if (!down_write_trylock(&cur_mem->memory_lock)) {
2169		− ret = -EBUSY;
2170		− goto err_undo;
2171		− }
2172		− mutex_unlock(&cur_mem->vma_lock);
2173		− }
2174		− cur_mem = NULL;
2175		−
2176		− ret = pci_reset_bus(pdev);
2177		−
2178		−err_undo:
2179		− list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2180		− if (cur == cur_mem)
2181		− is_mem = false;
2182		− if (cur == cur_vma)
2183		− break;
2184		− if (is_mem)
2185		− up_write(&cur->memory_lock);
2186		− else
2187		− mutex_unlock(&cur->vma_lock);
2188		− }
2189		−err_unlock:
2190		− mutex_unlock(&dev_set->lock);
2191		− return ret;
2192		−}
2193		−
2194		−static bool vfio_pci_dev_set_needs_reset(struct vfio_device_set *dev_set)
2195		−{
2196		− struct vfio_pci_device *cur;
2197		− bool needs_reset = false;
2198		−
2199		− list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2200		− /* No VFIO device in the set can have an open device FD */
2201		− if (cur->vdev.open_count)
2202		− return false;
2203		− needs_reset \|= cur->needs_reset;
2204		− }
2205		− return needs_reset;
2206		−}
2207		−
2208		−/*
2209		− * If a bus or slot reset is available for the provided dev_set and:
2210		− * - All of the devices affected by that bus or slot reset are unused
2211		− * - At least one of the affected devices is marked dirty via
2212		− * needs_reset (such as by lack of FLR support)
2213		− * Then attempt to perform that bus or slot reset.
2214		− * Returns true if the dev_set was reset.
2215		− */
2216		−static bool vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set)
2217		−{
2218		− struct vfio_pci_device *cur;
2219		− struct pci_dev *pdev;
2220		− int ret;
2221		−
2222		− if (!vfio_pci_dev_set_needs_reset(dev_set))
2223		− return false;
2224		−
2225		− pdev = vfio_pci_dev_set_resettable(dev_set);
2226		− if (!pdev)
2227		− return false;
2228		−
2229		− ret = pci_reset_bus(pdev);
2230		− if (ret)
2231		− return false;
2232		−
2233		− list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
2234		− cur->needs_reset = false;
2235		− if (!disable_idle_d3)
2236		− vfio_pci_set_power_state(cur, PCI_D3hot);
2237		− }
2238		− return true;
2239		−}
2240		−
2241		−static void __exit vfio_pci_cleanup(void)
	264	+static void __exit vfio_pci_cleanup(void)
2242	265	{
2243	266	pci_unregister_driver(&vfio_pci_driver);
2244		− vfio_pci_uninit_perm_bits();
2245	267	}
2246		−
2247		−static void __init vfio_pci_fill_ids(void)
2248		−{
2249		− char p, id;
2250		− int rc;
2251		−
2252		− /* no ids passed actually */
2253		− if (ids[0] == '\0')
2254		− return;
2255		−
2256		− /* add ids specified in the module parameter */
2257		− p = ids;
2258		− while ((id = strsep(&p, ","))) {
2259		− unsigned int vendor, device, subvendor = PCI_ANY_ID,
2260		− subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
2261		− int fields;
2262		−
2263		− if (!strlen(id))
2264		− continue;
2265		−
2266		− fields = sscanf(id, "%x:%x:%x:%x:%x:%x",
2267		− &vendor, &device, &subvendor, &subdevice,
2268		− &class, &class_mask);
2269		−
2270		− if (fields < 2) {
2271		− pr_warn("invalid id string \"%s\"\n", id);
2272		− continue;
2273		− }
2274		−
2275		− rc = pci_add_dynid(&vfio_pci_driver, vendor, device,
2276		− subvendor, subdevice, class, class_mask, 0);
2277		− if (rc)
2278		− pr_warn("failed to add dynamic id [%04x:%04x[%04x:%04x]] class %#08x/%08x (%d)\n",
2279		− vendor, device, subvendor, subdevice,
2280		− class, class_mask, rc);
2281		− else
2282		− pr_info("add [%04x:%04x[%04x:%04x]] class %#08x/%08x\n",
2283		− vendor, device, subvendor, subdevice,
2284		− class, class_mask);
2285		− }
2286		−}
2287		−
2288		−static int __init vfio_pci_init(void)
2289		−{
2290		− int ret;
2291		−
2292		− /* Allocate shared config space permission data used by all devices */
2293		− ret = vfio_pci_init_perm_bits();
2294		− if (ret)
2295		− return ret;
2296		−
2297		− /* Register and scan for devices */
2298		− ret = pci_register_driver(&vfio_pci_driver);
2299		− if (ret)
2300		− goto out_driver;
2301		−
2302		− vfio_pci_fill_ids();
2303		−
2304		− if (disable_denylist)
2305		− pr_warn("device denylist disabled.\n");
2306		−
2307		− return 0;
2308		−
2309		−out_driver:
2310		− vfio_pci_uninit_perm_bits();
2311		− return ret;
2312		−}
2313		−
2314		−module_init(vfio_pci_init);
2315	268	module_exit(vfio_pci_cleanup);
2316	269
2317		−MODULE_VERSION(DRIVER_VERSION);
2318	270	MODULE_LICENSE("GPL v2");
2319	271	MODULE_AUTHOR(DRIVER_AUTHOR);
2320	272	MODULE_DESCRIPTION(DRIVER_DESC);

drivers/vfio/pci/vfio_pci_config.c+35−35

@@ -26,7 +26,7 @@ · 25 unmodified lines
26	26	#include <linux/vfio.h>
27	27	#include <linux/slab.h>
28	28
29		−#include "vfio_pci_private.h"
	29	+#include <linux/vfio_pci_core.h>
30	30
31	31	/* Fake capability ID for standard config space */
32	32	#define PCI_CAP_ID_BASIC 0
@@ -108,9 +108,9 @@ static const u16 pci_ext_cap_length[PCI_EXT_CAP_ID_MAX + 1] = { · 75 unmodified lines
108	108	struct perm_bits {
109	109	u8 virt; / read/write virtual data, not hw */
110	110	u8 write; / writeable bits */
111		− int (readfn)(struct vfio_pci_device vdev, int pos, int count,
	111	+ int (readfn)(struct vfio_pci_core_device vdev, int pos, int count,
112	112	struct perm_bits perm, int offset, __le32 val);
113		− int (writefn)(struct vfio_pci_device vdev, int pos, int count,
	113	+ int (writefn)(struct vfio_pci_core_device vdev, int pos, int count,
114	114	struct perm_bits *perm, int offset, __le32 val);
115	115	};
116	116
@@ -171,7 +171,7 @@ static int vfio_user_config_write(struct pci_dev *pdev, int offset, · 54 unmodified lines
171	171	return ret;
172	172	}
173	173
174		−static int vfio_default_config_read(struct vfio_pci_device *vdev, int pos,
	174	+static int vfio_default_config_read(struct vfio_pci_core_device *vdev, int pos,
175	175	int count, struct perm_bits *perm,
176	176	int offset, __le32 *val)
177	177	{
@@ -197,7 +197,7 @@ static int vfio_default_config_read(struct vfio_pci_device *vdev, int pos, · 19 unmodified lines
197	197	return count;
198	198	}
199	199
200		−static int vfio_default_config_write(struct vfio_pci_device *vdev, int pos,
	200	+static int vfio_default_config_write(struct vfio_pci_core_device *vdev, int pos,
201	201	int count, struct perm_bits *perm,
202	202	int offset, __le32 val)
203	203	{
@@ -244,7 +244,7 @@ static int vfio_default_config_write(struct vfio_pci_device *vdev, int pos, · 40 unmodified lines
244	244	}
245	245
246	246	/* Allow direct read from hardware, except for capability next pointer */
247		−static int vfio_direct_config_read(struct vfio_pci_device *vdev, int pos,
	247	+static int vfio_direct_config_read(struct vfio_pci_core_device *vdev, int pos,
248	248	int count, struct perm_bits *perm,
249	249	int offset, __le32 *val)
250	250	{
@@ -269,7 +269,7 @@ static int vfio_direct_config_read(struct vfio_pci_device *vdev, int pos, · 18 unmodified lines
269	269	}
270	270
271	271	/* Raw access skips any kind of virtualization */
272		−static int vfio_raw_config_write(struct vfio_pci_device *vdev, int pos,
	272	+static int vfio_raw_config_write(struct vfio_pci_core_device *vdev, int pos,
273	273	int count, struct perm_bits *perm,
274	274	int offset, __le32 val)
275	275	{
@@ -282,7 +282,7 @@ static int vfio_raw_config_write(struct vfio_pci_device *vdev, int pos, · 6 unmodified lines
282	282	return count;
283	283	}
284	284
285		−static int vfio_raw_config_read(struct vfio_pci_device *vdev, int pos,
	285	+static int vfio_raw_config_read(struct vfio_pci_core_device *vdev, int pos,
286	286	int count, struct perm_bits *perm,
287	287	int offset, __le32 *val)
288	288	{
@@ -296,7 +296,7 @@ static int vfio_raw_config_read(struct vfio_pci_device *vdev, int pos, · 7 unmodified lines
296	296	}
297	297
298	298	/* Virt access uses only virtualization */
299		−static int vfio_virt_config_write(struct vfio_pci_device *vdev, int pos,
	299	+static int vfio_virt_config_write(struct vfio_pci_core_device *vdev, int pos,
300	300	int count, struct perm_bits *perm,
301	301	int offset, __le32 val)
302	302	{
@@ -304,7 +304,7 @@ static int vfio_virt_config_write(struct vfio_pci_device *vdev, int pos, · 1 unmodified line
304	304	return count;
305	305	}
306	306
307		−static int vfio_virt_config_read(struct vfio_pci_device *vdev, int pos,
	307	+static int vfio_virt_config_read(struct vfio_pci_core_device *vdev, int pos,
308	308	int count, struct perm_bits *perm,
309	309	int offset, __le32 *val)
310	310	{
@@ -396,7 +396,7 @@ static inline void p_setd(struct perm_bits *p, int off, u32 virt, u32 write) · 85 unmodified lines
396	396	}
397	397
398	398	/* Caller should hold memory_lock semaphore */
399		−bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev)
	399	+bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev)
400	400	{
401	401	struct pci_dev *pdev = vdev->pdev;
402	402	u16 cmd = le16_to_cpu((__le16 )&vdev->vconfig[PCI_COMMAND]);
@@ -413,7 +413,7 @@ bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev) · 10 unmodified lines
413	413	* Restore the real BARs after we detect a FLR or backdoor reset.
414	414	* (backdoor = some device specific technique that we didn't catch)
415	415	*/
416		−static void vfio_bar_restore(struct vfio_pci_device *vdev)
	416	+static void vfio_bar_restore(struct vfio_pci_core_device *vdev)
417	417	{
418	418	struct pci_dev *pdev = vdev->pdev;
419	419	u32 *rbar = vdev->rbar;
@@ -460,7 +460,7 @@ static __le32 vfio_generate_bar_flags(struct pci_dev *pdev, int bar) · 40 unmodified lines
460	460	* Pretend we're hardware and tweak the values of the virtual PCI BARs
461	461	* to reflect the hardware capabilities. This implements BAR sizing.
462	462	*/
463		−static void vfio_bar_fixup(struct vfio_pci_device *vdev)
	463	+static void vfio_bar_fixup(struct vfio_pci_core_device *vdev)
464	464	{
465	465	struct pci_dev *pdev = vdev->pdev;
466	466	int i;
@@ -514,7 +514,7 @@ static void vfio_bar_fixup(struct vfio_pci_device *vdev) · 47 unmodified lines
514	514	vdev->bardirty = false;
515	515	}
516	516
517		−static int vfio_basic_config_read(struct vfio_pci_device *vdev, int pos,
	517	+static int vfio_basic_config_read(struct vfio_pci_core_device *vdev, int pos,
518	518	int count, struct perm_bits *perm,
519	519	int offset, __le32 *val)
520	520	{
@@ -536,7 +536,7 @@ static int vfio_basic_config_read(struct vfio_pci_device *vdev, int pos, · 15 unmodified lines
536	536	}
537	537
538	538	/* Test whether BARs match the value we think they should contain */
539		−static bool vfio_need_bar_restore(struct vfio_pci_device *vdev)
	539	+static bool vfio_need_bar_restore(struct vfio_pci_core_device *vdev)
540	540	{
541	541	int i = 0, pos = PCI_BASE_ADDRESS_0, ret;
542	542	u32 bar;
@@ -552,7 +552,7 @@ static bool vfio_need_bar_restore(struct vfio_pci_device *vdev) · 9 unmodified lines
552	552	return false;
553	553	}
554	554
555		−static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos,
	555	+static int vfio_basic_config_write(struct vfio_pci_core_device *vdev, int pos,
556	556	int count, struct perm_bits *perm,
557	557	int offset, __le32 val)
558	558	{
@@ -692,7 +692,7 @@ static int __init init_pci_cap_basic_perm(struct perm_bits *perm) · 133 unmodified lines
692	692	return 0;
693	693	}
694	694
695		−static int vfio_pm_config_write(struct vfio_pci_device *vdev, int pos,
	695	+static int vfio_pm_config_write(struct vfio_pci_core_device *vdev, int pos,
696	696	int count, struct perm_bits *perm,
697	697	int offset, __le32 val)
698	698	{
@@ -747,7 +747,7 @@ static int __init init_pci_cap_pm_perm(struct perm_bits *perm) · 48 unmodified lines
747	747	return 0;
748	748	}
749	749
750		−static int vfio_vpd_config_write(struct vfio_pci_device *vdev, int pos,
	750	+static int vfio_vpd_config_write(struct vfio_pci_core_device *vdev, int pos,
751	751	int count, struct perm_bits *perm,
752	752	int offset, __le32 val)
753	753	{
@@ -829,7 +829,7 @@ static int __init init_pci_cap_pcix_perm(struct perm_bits *perm) · 75 unmodified lines
829	829	return 0;
830	830	}
831	831
832		−static int vfio_exp_config_write(struct vfio_pci_device *vdev, int pos,
	832	+static int vfio_exp_config_write(struct vfio_pci_core_device *vdev, int pos,
833	833	int count, struct perm_bits *perm,
834	834	int offset, __le32 val)
835	835	{
@@ -913,7 +913,7 @@ static int __init init_pci_cap_exp_perm(struct perm_bits *perm) · 77 unmodified lines
913	913	return 0;
914	914	}
915	915
916		−static int vfio_af_config_write(struct vfio_pci_device *vdev, int pos,
	916	+static int vfio_af_config_write(struct vfio_pci_core_device *vdev, int pos,
917	917	int count, struct perm_bits *perm,
918	918	int offset, __le32 val)
919	919	{
@@ -1072,7 +1072,7 @@ int __init vfio_pci_init_perm_bits(void) · 152 unmodified lines
1072	1072	return ret;
1073	1073	}
1074	1074
1075		−static int vfio_find_cap_start(struct vfio_pci_device *vdev, int pos)
	1075	+static int vfio_find_cap_start(struct vfio_pci_core_device *vdev, int pos)
1076	1076	{
1077	1077	u8 cap;
1078	1078	int base = (pos >= PCI_CFG_SPACE_SIZE) ? PCI_CFG_SPACE_SIZE :
@@ -1089,7 +1089,7 @@ static int vfio_find_cap_start(struct vfio_pci_device *vdev, int pos) · 10 unmodified lines
1089	1089	return pos;
1090	1090	}
1091	1091
1092		−static int vfio_msi_config_read(struct vfio_pci_device *vdev, int pos,
	1092	+static int vfio_msi_config_read(struct vfio_pci_core_device *vdev, int pos,
1093	1093	int count, struct perm_bits *perm,
1094	1094	int offset, __le32 *val)
1095	1095	{
@@ -1109,7 +1109,7 @@ static int vfio_msi_config_read(struct vfio_pci_device *vdev, int pos, · 13 unmodified lines
1109	1109	return vfio_default_config_read(vdev, pos, count, perm, offset, val);
1110	1110	}
1111	1111
1112		−static int vfio_msi_config_write(struct vfio_pci_device *vdev, int pos,
	1112	+static int vfio_msi_config_write(struct vfio_pci_core_device *vdev, int pos,
1113	1113	int count, struct perm_bits *perm,
1114	1114	int offset, __le32 val)
1115	1115	{
@@ -1189,7 +1189,7 @@ static int init_pci_cap_msi_perm(struct perm_bits *perm, int len, u16 flags) · 73 unmodified lines
1189	1189	}
1190	1190
1191	1191	/* Determine MSI CAP field length; initialize msi_perms on 1st call per vdev */
1192		−static int vfio_msi_cap_len(struct vfio_pci_device *vdev, u8 pos)
	1192	+static int vfio_msi_cap_len(struct vfio_pci_core_device *vdev, u8 pos)
1193	1193	{
1194	1194	struct pci_dev *pdev = vdev->pdev;
1195	1195	int len, ret;
@@ -1222,7 +1222,7 @@ static int vfio_msi_cap_len(struct vfio_pci_device *vdev, u8 pos) · 26 unmodified lines
1222	1222	}
1223	1223
1224	1224	/* Determine extended capability length for VC (2 & 9) and MFVC */
1225		−static int vfio_vc_cap_len(struct vfio_pci_device *vdev, u16 pos)
	1225	+static int vfio_vc_cap_len(struct vfio_pci_core_device *vdev, u16 pos)
1226	1226	{
1227	1227	struct pci_dev *pdev = vdev->pdev;
1228	1228	u32 tmp;
@@ -1263,7 +1263,7 @@ static int vfio_vc_cap_len(struct vfio_pci_device *vdev, u16 pos) · 34 unmodified lines
1263	1263	return len;
1264	1264	}
1265	1265
1266		−static int vfio_cap_len(struct vfio_pci_device *vdev, u8 cap, u8 pos)
	1266	+static int vfio_cap_len(struct vfio_pci_core_device *vdev, u8 cap, u8 pos)
1267	1267	{
1268	1268	struct pci_dev *pdev = vdev->pdev;
1269	1269	u32 dword;
@@ -1338,7 +1338,7 @@ static int vfio_cap_len(struct vfio_pci_device *vdev, u8 cap, u8 pos) · 68 unmodified lines
1338	1338	return 0;
1339	1339	}
1340	1340
1341		−static int vfio_ext_cap_len(struct vfio_pci_device *vdev, u16 ecap, u16 epos)
	1341	+static int vfio_ext_cap_len(struct vfio_pci_core_device *vdev, u16 ecap, u16 epos)
1342	1342	{
1343	1343	struct pci_dev *pdev = vdev->pdev;
1344	1344	u8 byte;
@@ -1412,7 +1412,7 @@ static int vfio_ext_cap_len(struct vfio_pci_device *vdev, u16 ecap, u16 epos) · 67 unmodified lines
1412	1412	return 0;
1413	1413	}
1414	1414
1415		−static int vfio_fill_vconfig_bytes(struct vfio_pci_device *vdev,
	1415	+static int vfio_fill_vconfig_bytes(struct vfio_pci_core_device *vdev,
1416	1416	int offset, int size)
1417	1417	{
1418	1418	struct pci_dev *pdev = vdev->pdev;
@@ -1459,7 +1459,7 @@ static int vfio_fill_vconfig_bytes(struct vfio_pci_device *vdev, · 40 unmodified lines
1459	1459	return ret;
1460	1460	}
1461	1461
1462		−static int vfio_cap_init(struct vfio_pci_device *vdev)
	1462	+static int vfio_cap_init(struct vfio_pci_core_device *vdev)
1463	1463	{
1464	1464	struct pci_dev *pdev = vdev->pdev;
1465	1465	u8 *map = vdev->pci_config_map;
@@ -1549,7 +1549,7 @@ static int vfio_cap_init(struct vfio_pci_device *vdev) · 83 unmodified lines
1549	1549	return 0;
1550	1550	}
1551	1551
1552		−static int vfio_ecap_init(struct vfio_pci_device *vdev)
	1552	+static int vfio_ecap_init(struct vfio_pci_core_device *vdev)
1553	1553	{
1554	1554	struct pci_dev *pdev = vdev->pdev;
1555	1555	u8 *map = vdev->pci_config_map;
@@ -1669,7 +1669,7 @@ static const struct pci_device_id known_bogus_vf_intx_pin[] = { · 113 unmodified lines
1669	1669	* for each area requiring emulated bits, but the array of pointers
1670	1670	* would be comparable in size (at least for standard config space).
1671	1671	*/
1672		−int vfio_config_init(struct vfio_pci_device *vdev)
	1672	+int vfio_config_init(struct vfio_pci_core_device *vdev)
1673	1673	{
1674	1674	struct pci_dev *pdev = vdev->pdev;
1675	1675	u8 map, vconfig;
@@ -1773,7 +1773,7 @@ out: · 97 unmodified lines
1773	1773	return pcibios_err_to_errno(ret);
1774	1774	}
1775	1775
1776		−void vfio_config_free(struct vfio_pci_device *vdev)
	1776	+void vfio_config_free(struct vfio_pci_core_device *vdev)
1777	1777	{
1778	1778	kfree(vdev->vconfig);
1779	1779	vdev->vconfig = NULL;
@@ -1790,7 +1790,7 @@ void vfio_config_free(struct vfio_pci_device *vdev) · 10 unmodified lines
1790	1790	* Find the remaining number of bytes in a dword that match the given
1791	1791	* position. Stop at either the end of the capability or the dword boundary.
1792	1792	*/
1793		−static size_t vfio_pci_cap_remaining_dword(struct vfio_pci_device *vdev,
	1793	+static size_t vfio_pci_cap_remaining_dword(struct vfio_pci_core_device *vdev,
1794	1794	loff_t pos)
1795	1795	{
1796	1796	u8 cap = vdev->pci_config_map[pos];
@@ -1802,7 +1802,7 @@ static size_t vfio_pci_cap_remaining_dword(struct vfio_pci_device *vdev, · 5 unmodified lines
1802	1802	return i;
1803	1803	}
1804	1804
1805		−static ssize_t vfio_config_do_rw(struct vfio_pci_device vdev, char __user buf,
	1805	+static ssize_t vfio_config_do_rw(struct vfio_pci_core_device vdev, char __user buf,
1806	1806	size_t count, loff_t *ppos, bool iswrite)
1807	1807	{
1808	1808	struct pci_dev *pdev = vdev->pdev;
@@ -1885,7 +1885,7 @@ static ssize_t vfio_config_do_rw(struct vfio_pci_device vdev, char __user buf, · 76 unmodified lines
1885	1885	return ret;
1886	1886	}
1887	1887
1888		−ssize_t vfio_pci_config_rw(struct vfio_pci_device vdev, char __user buf,
	1888	+ssize_t vfio_pci_config_rw(struct vfio_pci_core_device vdev, char __user buf,
1889	1889	size_t count, loff_t *ppos, bool iswrite)
1890	1890	{
1891	1891	size_t done = 0;

drivers/vfio/pci/vfio_pci_core.c+2158−0

@@ -0,0 +1,2158 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
	2	+/*
	3	+ * Copyright (C) 2012 Red Hat, Inc. All rights reserved.
	4	+ * Author: Alex Williamson <alex.williamson@redhat.com>
	5	+ *
	6	+ * Derived from original vfio:
	7	+ * Copyright 2010 Cisco Systems, Inc. All rights reserved.
	8	+ * Author: Tom Lyon, pugs@cisco.com
	9	+ */
	10	+
	11	+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	12	+
	13	+#include <linux/device.h>
	14	+#include <linux/eventfd.h>
	15	+#include <linux/file.h>
	16	+#include <linux/interrupt.h>
	17	+#include <linux/iommu.h>
	18	+#include <linux/module.h>
	19	+#include <linux/mutex.h>
	20	+#include <linux/notifier.h>
	21	+#include <linux/pci.h>
	22	+#include <linux/pm_runtime.h>
	23	+#include <linux/slab.h>
	24	+#include <linux/types.h>
	25	+#include <linux/uaccess.h>
	26	+#include <linux/vgaarb.h>
	27	+#include <linux/nospec.h>
	28	+#include <linux/sched/mm.h>
	29	+
	30	+#include <linux/vfio_pci_core.h>
	31	+
	32	+#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
	33	+#define DRIVER_DESC "core driver for VFIO based PCI devices"
	34	+
	35	+static bool nointxmask;
	36	+static bool disable_vga;
	37	+static bool disable_idle_d3;
	38	+
	39	+static inline bool vfio_vga_disabled(void)
	40	+{
	41	+#ifdef CONFIG_VFIO_PCI_VGA
	42	+ return disable_vga;
	43	+#else
	44	+ return true;
	45	+#endif
	46	+}
	47	+
	48	+/*
	49	+ * Our VGA arbiter participation is limited since we don't know anything
	50	+ * about the device itself. However, if the device is the only VGA device
	51	+ * downstream of a bridge and VFIO VGA support is disabled, then we can
	52	+ * safely return legacy VGA IO and memory as not decoded since the user
	53	+ * has no way to get to it and routing can be disabled externally at the
	54	+ * bridge.
	55	+ */
	56	+static unsigned int vfio_pci_set_vga_decode(void *opaque, bool single_vga)
	57	+{
	58	+ struct vfio_pci_core_device *vdev = opaque;
	59	+ struct pci_dev tmp = NULL, pdev = vdev->pdev;
	60	+ unsigned char max_busnr;
	61	+ unsigned int decodes;
	62	+
	63	+ if (single_vga \|\| !vfio_vga_disabled() \|\| pci_is_root_bus(pdev->bus))
	64	+ return VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM \|
	65	+ VGA_RSRC_LEGACY_IO \| VGA_RSRC_LEGACY_MEM;
	66	+
	67	+ max_busnr = pci_bus_max_busnr(pdev->bus);
	68	+ decodes = VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM;
	69	+
	70	+ while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) {
	71	+ if (tmp == pdev \|\|
	72	+ pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) \|\|
	73	+ pci_is_root_bus(tmp->bus))
	74	+ continue;
	75	+
	76	+ if (tmp->bus->number >= pdev->bus->number &&
	77	+ tmp->bus->number <= max_busnr) {
	78	+ pci_dev_put(tmp);
	79	+ decodes \|= VGA_RSRC_LEGACY_IO \| VGA_RSRC_LEGACY_MEM;
	80	+ break;
	81	+ }
	82	+ }
	83	+
	84	+ return decodes;
	85	+}
	86	+
	87	+static void vfio_pci_probe_mmaps(struct vfio_pci_core_device *vdev)
	88	+{
	89	+ struct resource *res;
	90	+ int i;
	91	+ struct vfio_pci_dummy_resource *dummy_res;
	92	+
	93	+ for (i = 0; i < PCI_STD_NUM_BARS; i++) {
	94	+ int bar = i + PCI_STD_RESOURCES;
	95	+
	96	+ res = &vdev->pdev->resource[bar];
	97	+
	98	+ if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP))
	99	+ goto no_mmap;
	100	+
	101	+ if (!(res->flags & IORESOURCE_MEM))
	102	+ goto no_mmap;
	103	+
	104	+ /*
	105	+ * The PCI core shouldn't set up a resource with a
	106	+ * type but zero size. But there may be bugs that
	107	+ * cause us to do that.
	108	+ */
	109	+ if (!resource_size(res))
	110	+ goto no_mmap;
	111	+
	112	+ if (resource_size(res) >= PAGE_SIZE) {
	113	+ vdev->bar_mmap_supported[bar] = true;
	114	+ continue;
	115	+ }
	116	+
	117	+ if (!(res->start & ~PAGE_MASK)) {
	118	+ /*
	119	+ * Add a dummy resource to reserve the remainder
	120	+ * of the exclusive page in case that hot-add
	121	+ * device's bar is assigned into it.
	122	+ */
	123	+ dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL);
	124	+ if (dummy_res == NULL)
	125	+ goto no_mmap;
	126	+
	127	+ dummy_res->resource.name = "vfio sub-page reserved";
	128	+ dummy_res->resource.start = res->end + 1;
	129	+ dummy_res->resource.end = res->start + PAGE_SIZE - 1;
	130	+ dummy_res->resource.flags = res->flags;
	131	+ if (request_resource(res->parent,
	132	+ &dummy_res->resource)) {
	133	+ kfree(dummy_res);
	134	+ goto no_mmap;
	135	+ }
	136	+ dummy_res->index = bar;
	137	+ list_add(&dummy_res->res_next,
	138	+ &vdev->dummy_resources_list);
	139	+ vdev->bar_mmap_supported[bar] = true;
	140	+ continue;
	141	+ }
	142	+ /*
	143	+ * Here we don't handle the case when the BAR is not page
	144	+ * aligned because we can't expect the BAR will be
	145	+ * assigned into the same location in a page in guest
	146	+ * when we passthrough the BAR. And it's hard to access
	147	+ * this BAR in userspace because we have no way to get
	148	+ * the BAR's location in a page.
	149	+ */
	150	+no_mmap:
	151	+ vdev->bar_mmap_supported[bar] = false;
	152	+ }
	153	+}
	154	+
	155	+struct vfio_pci_group_info;
	156	+static bool vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set);
	157	+static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
	158	+ struct vfio_pci_group_info *groups);
	159	+
	160	+/*
	161	+ * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND
	162	+ * _and_ the ability detect when the device is asserting INTx via PCI_STATUS.
	163	+ * If a device implements the former but not the latter we would typically
	164	+ * expect broken_intx_masking be set and require an exclusive interrupt.
	165	+ * However since we do have control of the device's ability to assert INTx,
	166	+ * we can instead pretend that the device does not implement INTx, virtualizing
	167	+ * the pin register to report zero and maintaining DisINTx set on the host.
	168	+ */
	169	+static bool vfio_pci_nointx(struct pci_dev *pdev)
	170	+{
	171	+ switch (pdev->vendor) {
	172	+ case PCI_VENDOR_ID_INTEL:
	173	+ switch (pdev->device) {
	174	+ /* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */
	175	+ case 0x1572:
	176	+ case 0x1574:
	177	+ case 0x1580 ... 0x1581:
	178	+ case 0x1583 ... 0x158b:
	179	+ case 0x37d0 ... 0x37d2:
	180	+ /* X550 */
	181	+ case 0x1563:
	182	+ return true;
	183	+ default:
	184	+ return false;
	185	+ }
	186	+ }
	187	+
	188	+ return false;
	189	+}
	190	+
	191	+static void vfio_pci_probe_power_state(struct vfio_pci_core_device *vdev)
	192	+{
	193	+ struct pci_dev *pdev = vdev->pdev;
	194	+ u16 pmcsr;
	195	+
	196	+ if (!pdev->pm_cap)
	197	+ return;
	198	+
	199	+ pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr);
	200	+
	201	+ vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
	202	+}
	203	+
	204	+/*
	205	+ * pci_set_power_state() wrapper handling devices which perform a soft reset on
	206	+ * D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev,
	207	+ * restore when returned to D0. Saved separately from pci_saved_state for use
	208	+ * by PM capability emulation and separately from pci_dev internal saved state
	209	+ * to avoid it being overwritten and consumed around other resets.
	210	+ */
	211	+int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev, pci_power_t state)
	212	+{
	213	+ struct pci_dev *pdev = vdev->pdev;
	214	+ bool needs_restore = false, needs_save = false;
	215	+ int ret;
	216	+
	217	+ if (vdev->needs_pm_restore) {
	218	+ if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) {
	219	+ pci_save_state(pdev);
	220	+ needs_save = true;
	221	+ }
	222	+
	223	+ if (pdev->current_state >= PCI_D3hot && state <= PCI_D0)
	224	+ needs_restore = true;
	225	+ }
	226	+
	227	+ ret = pci_set_power_state(pdev, state);
	228	+
	229	+ if (!ret) {
	230	+ /* D3 might be unsupported via quirk, skip unless in D3 */
	231	+ if (needs_save && pdev->current_state >= PCI_D3hot) {
	232	+ vdev->pm_save = pci_store_saved_state(pdev);
	233	+ } else if (needs_restore) {
	234	+ pci_load_and_free_saved_state(pdev, &vdev->pm_save);
	235	+ pci_restore_state(pdev);
	236	+ }
	237	+ }
	238	+
	239	+ return ret;
	240	+}
	241	+
	242	+int vfio_pci_core_enable(struct vfio_pci_core_device *vdev)
	243	+{
	244	+ struct pci_dev *pdev = vdev->pdev;
	245	+ int ret;
	246	+ u16 cmd;
	247	+ u8 msix_pos;
	248	+
	249	+ vfio_pci_set_power_state(vdev, PCI_D0);
	250	+
	251	+ /* Don't allow our initial saved state to include busmaster */
	252	+ pci_clear_master(pdev);
	253	+
	254	+ ret = pci_enable_device(pdev);
	255	+ if (ret)
	256	+ return ret;
	257	+
	258	+ /* If reset fails because of the device lock, fail this path entirely */
	259	+ ret = pci_try_reset_function(pdev);
	260	+ if (ret == -EAGAIN) {
	261	+ pci_disable_device(pdev);
	262	+ return ret;
	263	+ }
	264	+
	265	+ vdev->reset_works = !ret;
	266	+ pci_save_state(pdev);
	267	+ vdev->pci_saved_state = pci_store_saved_state(pdev);
	268	+ if (!vdev->pci_saved_state)
	269	+ pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__);
	270	+
	271	+ if (likely(!nointxmask)) {
	272	+ if (vfio_pci_nointx(pdev)) {
	273	+ pci_info(pdev, "Masking broken INTx support\n");
	274	+ vdev->nointx = true;
	275	+ pci_intx(pdev, 0);
	276	+ } else
	277	+ vdev->pci_2_3 = pci_intx_mask_supported(pdev);
	278	+ }
	279	+
	280	+ pci_read_config_word(pdev, PCI_COMMAND, &cmd);
	281	+ if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
	282	+ cmd &= ~PCI_COMMAND_INTX_DISABLE;
	283	+ pci_write_config_word(pdev, PCI_COMMAND, cmd);
	284	+ }
	285	+
	286	+ ret = vfio_config_init(vdev);
	287	+ if (ret) {
	288	+ kfree(vdev->pci_saved_state);
	289	+ vdev->pci_saved_state = NULL;
	290	+ pci_disable_device(pdev);
	291	+ return ret;
	292	+ }
	293	+
	294	+ msix_pos = pdev->msix_cap;
	295	+ if (msix_pos) {
	296	+ u16 flags;
	297	+ u32 table;
	298	+
	299	+ pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);
	300	+ pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);
	301	+
	302	+ vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
	303	+ vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
	304	+ vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
	305	+ } else
	306	+ vdev->msix_bar = 0xFF;
	307	+
	308	+ if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))
	309	+ vdev->has_vga = true;
	310	+
	311	+
	312	+ return 0;
	313	+}
	314	+EXPORT_SYMBOL_GPL(vfio_pci_core_enable);
	315	+
	316	+void vfio_pci_core_disable(struct vfio_pci_core_device *vdev)
	317	+{
	318	+ struct pci_dev *pdev = vdev->pdev;
	319	+ struct vfio_pci_dummy_resource dummy_res, tmp;
	320	+ struct vfio_pci_ioeventfd ioeventfd, ioeventfd_tmp;
	321	+ int i, bar;
	322	+
	323	+ /* For needs_reset */
	324	+ lockdep_assert_held(&vdev->vdev.dev_set->lock);
	325	+
	326	+ /* Stop the device from further DMA */
	327	+ pci_clear_master(pdev);
	328	+
	329	+ vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE \|
	330	+ VFIO_IRQ_SET_ACTION_TRIGGER,
	331	+ vdev->irq_type, 0, 0, NULL);
	332	+
	333	+ /* Device closed, don't need mutex here */
	334	+ list_for_each_entry_safe(ioeventfd, ioeventfd_tmp,
	335	+ &vdev->ioeventfds_list, next) {
	336	+ vfio_virqfd_disable(&ioeventfd->virqfd);
	337	+ list_del(&ioeventfd->next);
	338	+ kfree(ioeventfd);
	339	+ }
	340	+ vdev->ioeventfds_nr = 0;
	341	+
	342	+ vdev->virq_disabled = false;
	343	+
	344	+ for (i = 0; i < vdev->num_regions; i++)
	345	+ vdev->region[i].ops->release(vdev, &vdev->region[i]);
	346	+
	347	+ vdev->num_regions = 0;
	348	+ kfree(vdev->region);
	349	+ vdev->region = NULL; /* don't krealloc a freed pointer */
	350	+
	351	+ vfio_config_free(vdev);
	352	+
	353	+ for (i = 0; i < PCI_STD_NUM_BARS; i++) {
	354	+ bar = i + PCI_STD_RESOURCES;
	355	+ if (!vdev->barmap[bar])
	356	+ continue;
	357	+ pci_iounmap(pdev, vdev->barmap[bar]);
	358	+ pci_release_selected_regions(pdev, 1 << bar);
	359	+ vdev->barmap[bar] = NULL;
	360	+ }
	361	+
	362	+ list_for_each_entry_safe(dummy_res, tmp,
	363	+ &vdev->dummy_resources_list, res_next) {
	364	+ list_del(&dummy_res->res_next);
	365	+ release_resource(&dummy_res->resource);
	366	+ kfree(dummy_res);
	367	+ }
	368	+
	369	+ vdev->needs_reset = true;
	370	+
	371	+ /*
	372	+ * If we have saved state, restore it. If we can reset the device,
	373	+ * even better. Resetting with current state seems better than
	374	+ * nothing, but saving and restoring current state without reset
	375	+ * is just busy work.
	376	+ */
	377	+ if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
	378	+ pci_info(pdev, "%s: Couldn't reload saved state\n", __func__);
	379	+
	380	+ if (!vdev->reset_works)
	381	+ goto out;
	382	+
	383	+ pci_save_state(pdev);
	384	+ }
	385	+
	386	+ /*
	387	+ * Disable INTx and MSI, presumably to avoid spurious interrupts
	388	+ * during reset. Stolen from pci_reset_function()
	389	+ */
	390	+ pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
	391	+
	392	+ /*
	393	+ * Try to get the locks ourselves to prevent a deadlock. The
	394	+ * success of this is dependent on being able to lock the device,
	395	+ * which is not always possible.
	396	+ * We can not use the "try" reset interface here, which will
	397	+ * overwrite the previously restored configuration information.
	398	+ */
	399	+ if (vdev->reset_works && pci_dev_trylock(pdev)) {
	400	+ if (!__pci_reset_function_locked(pdev))
	401	+ vdev->needs_reset = false;
	402	+ pci_dev_unlock(pdev);
	403	+ }
	404	+
	405	+ pci_restore_state(pdev);
	406	+out:
	407	+ pci_disable_device(pdev);
	408	+
	409	+ if (!vfio_pci_dev_set_try_reset(vdev->vdev.dev_set) && !disable_idle_d3)
	410	+ vfio_pci_set_power_state(vdev, PCI_D3hot);
	411	+}
	412	+EXPORT_SYMBOL_GPL(vfio_pci_core_disable);
	413	+
	414	+static struct vfio_pci_core_device get_pf_vdev(struct vfio_pci_core_device vdev)
	415	+{
	416	+ struct pci_dev *physfn = pci_physfn(vdev->pdev);
	417	+ struct vfio_device *pf_dev;
	418	+
	419	+ if (!vdev->pdev->is_virtfn)
	420	+ return NULL;
	421	+
	422	+ pf_dev = vfio_device_get_from_dev(&physfn->dev);
	423	+ if (!pf_dev)
	424	+ return NULL;
	425	+
	426	+ if (pci_dev_driver(physfn) != pci_dev_driver(vdev->pdev)) {
	427	+ vfio_device_put(pf_dev);
	428	+ return NULL;
	429	+ }
	430	+
	431	+ return container_of(pf_dev, struct vfio_pci_core_device, vdev);
	432	+}
	433	+
	434	+static void vfio_pci_vf_token_user_add(struct vfio_pci_core_device *vdev, int val)
	435	+{
	436	+ struct vfio_pci_core_device *pf_vdev = get_pf_vdev(vdev);
	437	+
	438	+ if (!pf_vdev)
	439	+ return;
	440	+
	441	+ mutex_lock(&pf_vdev->vf_token->lock);
	442	+ pf_vdev->vf_token->users += val;
	443	+ WARN_ON(pf_vdev->vf_token->users < 0);
	444	+ mutex_unlock(&pf_vdev->vf_token->lock);
	445	+
	446	+ vfio_device_put(&pf_vdev->vdev);
	447	+}
	448	+
	449	+void vfio_pci_core_close_device(struct vfio_device *core_vdev)
	450	+{
	451	+ struct vfio_pci_core_device *vdev =
	452	+ container_of(core_vdev, struct vfio_pci_core_device, vdev);
	453	+
	454	+ vfio_pci_vf_token_user_add(vdev, -1);
	455	+ vfio_spapr_pci_eeh_release(vdev->pdev);
	456	+ vfio_pci_core_disable(vdev);
	457	+
	458	+ mutex_lock(&vdev->igate);
	459	+ if (vdev->err_trigger) {
	460	+ eventfd_ctx_put(vdev->err_trigger);
	461	+ vdev->err_trigger = NULL;
	462	+ }
	463	+ if (vdev->req_trigger) {
	464	+ eventfd_ctx_put(vdev->req_trigger);
	465	+ vdev->req_trigger = NULL;
	466	+ }
	467	+ mutex_unlock(&vdev->igate);
	468	+}
	469	+EXPORT_SYMBOL_GPL(vfio_pci_core_close_device);
	470	+
	471	+void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev)
	472	+{
	473	+ vfio_pci_probe_mmaps(vdev);
	474	+ vfio_spapr_pci_eeh_open(vdev->pdev);
	475	+ vfio_pci_vf_token_user_add(vdev, 1);
	476	+}
	477	+EXPORT_SYMBOL_GPL(vfio_pci_core_finish_enable);
	478	+
	479	+static int vfio_pci_get_irq_count(struct vfio_pci_core_device *vdev, int irq_type)
	480	+{
	481	+ if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {
	482	+ u8 pin;
	483	+
	484	+ if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) \|\|
	485	+ vdev->nointx \|\| vdev->pdev->is_virtfn)
	486	+ return 0;
	487	+
	488	+ pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);
	489	+
	490	+ return pin ? 1 : 0;
	491	+ } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {
	492	+ u8 pos;
	493	+ u16 flags;
	494	+
	495	+ pos = vdev->pdev->msi_cap;
	496	+ if (pos) {
	497	+ pci_read_config_word(vdev->pdev,
	498	+ pos + PCI_MSI_FLAGS, &flags);
	499	+ return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);
	500	+ }
	501	+ } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {
	502	+ u8 pos;
	503	+ u16 flags;
	504	+
	505	+ pos = vdev->pdev->msix_cap;
	506	+ if (pos) {
	507	+ pci_read_config_word(vdev->pdev,
	508	+ pos + PCI_MSIX_FLAGS, &flags);
	509	+
	510	+ return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;
	511	+ }
	512	+ } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) {
	513	+ if (pci_is_pcie(vdev->pdev))
	514	+ return 1;
	515	+ } else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) {
	516	+ return 1;
	517	+ }
	518	+
	519	+ return 0;
	520	+}
	521	+
	522	+static int vfio_pci_count_devs(struct pci_dev pdev, void data)
	523	+{
	524	+ ((int )data)++;
	525	+ return 0;
	526	+}
	527	+
	528	+struct vfio_pci_fill_info {
	529	+ int max;
	530	+ int cur;
	531	+ struct vfio_pci_dependent_device *devices;
	532	+};
	533	+
	534	+static int vfio_pci_fill_devs(struct pci_dev pdev, void data)
	535	+{
	536	+ struct vfio_pci_fill_info *fill = data;
	537	+ struct iommu_group *iommu_group;
	538	+
	539	+ if (fill->cur == fill->max)
	540	+ return -EAGAIN; /* Something changed, try again */
	541	+
	542	+ iommu_group = iommu_group_get(&pdev->dev);
	543	+ if (!iommu_group)
	544	+ return -EPERM; /* Cannot reset non-isolated devices */
	545	+
	546	+ fill->devices[fill->cur].group_id = iommu_group_id(iommu_group);
	547	+ fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus);
	548	+ fill->devices[fill->cur].bus = pdev->bus->number;
	549	+ fill->devices[fill->cur].devfn = pdev->devfn;
	550	+ fill->cur++;
	551	+ iommu_group_put(iommu_group);
	552	+ return 0;
	553	+}
	554	+
	555	+struct vfio_pci_group_info {
	556	+ int count;
	557	+ struct vfio_group **groups;
	558	+};
	559	+
	560	+static bool vfio_pci_dev_below_slot(struct pci_dev pdev, struct pci_slot slot)
	561	+{
	562	+ for (; pdev; pdev = pdev->bus->self)
	563	+ if (pdev->bus == slot->bus)
	564	+ return (pdev->slot == slot);
	565	+ return false;
	566	+}
	567	+
	568	+struct vfio_pci_walk_info {
	569	+ int (fn)(struct pci_dev , void *data);
	570	+ void *data;
	571	+ struct pci_dev *pdev;
	572	+ bool slot;
	573	+ int ret;
	574	+};
	575	+
	576	+static int vfio_pci_walk_wrapper(struct pci_dev pdev, void data)
	577	+{
	578	+ struct vfio_pci_walk_info *walk = data;
	579	+
	580	+ if (!walk->slot \|\| vfio_pci_dev_below_slot(pdev, walk->pdev->slot))
	581	+ walk->ret = walk->fn(pdev, walk->data);
	582	+
	583	+ return walk->ret;
	584	+}
	585	+
	586	+static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,
	587	+ int (fn)(struct pci_dev ,
	588	+ void data), void data,
	589	+ bool slot)
	590	+{
	591	+ struct vfio_pci_walk_info walk = {
	592	+ .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,
	593	+ };
	594	+
	595	+ pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);
	596	+
	597	+ return walk.ret;
	598	+}
	599	+
	600	+static int msix_mmappable_cap(struct vfio_pci_core_device *vdev,
	601	+ struct vfio_info_cap *caps)
	602	+{
	603	+ struct vfio_info_cap_header header = {
	604	+ .id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE,
	605	+ .version = 1
	606	+ };
	607	+
	608	+ return vfio_info_add_capability(caps, &header, sizeof(header));
	609	+}
	610	+
	611	+int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev,
	612	+ unsigned int type, unsigned int subtype,
	613	+ const struct vfio_pci_regops *ops,
	614	+ size_t size, u32 flags, void *data)
	615	+{
	616	+ struct vfio_pci_region *region;
	617	+
	618	+ region = krealloc(vdev->region,
	619	+ (vdev->num_regions + 1) * sizeof(*region),
	620	+ GFP_KERNEL);
	621	+ if (!region)
	622	+ return -ENOMEM;
	623	+
	624	+ vdev->region = region;
	625	+ vdev->region[vdev->num_regions].type = type;
	626	+ vdev->region[vdev->num_regions].subtype = subtype;
	627	+ vdev->region[vdev->num_regions].ops = ops;
	628	+ vdev->region[vdev->num_regions].size = size;
	629	+ vdev->region[vdev->num_regions].flags = flags;
	630	+ vdev->region[vdev->num_regions].data = data;
	631	+
	632	+ vdev->num_regions++;
	633	+
	634	+ return 0;
	635	+}
	636	+EXPORT_SYMBOL_GPL(vfio_pci_register_dev_region);
	637	+
	638	+long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
	639	+ unsigned long arg)
	640	+{
	641	+ struct vfio_pci_core_device *vdev =
	642	+ container_of(core_vdev, struct vfio_pci_core_device, vdev);
	643	+ unsigned long minsz;
	644	+
	645	+ if (cmd == VFIO_DEVICE_GET_INFO) {
	646	+ struct vfio_device_info info;
	647	+ struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
	648	+ unsigned long capsz;
	649	+ int ret;
	650	+
	651	+ minsz = offsetofend(struct vfio_device_info, num_irqs);
	652	+
	653	+ /* For backward compatibility, cannot require this */
	654	+ capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset);
	655	+
	656	+ if (copy_from_user(&info, (void __user *)arg, minsz))
	657	+ return -EFAULT;
	658	+
	659	+ if (info.argsz < minsz)
	660	+ return -EINVAL;
	661	+
	662	+ if (info.argsz >= capsz) {
	663	+ minsz = capsz;
	664	+ info.cap_offset = 0;
	665	+ }
	666	+
	667	+ info.flags = VFIO_DEVICE_FLAGS_PCI;
	668	+
	669	+ if (vdev->reset_works)
	670	+ info.flags \|= VFIO_DEVICE_FLAGS_RESET;
	671	+
	672	+ info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
	673	+ info.num_irqs = VFIO_PCI_NUM_IRQS;
	674	+
	675	+ ret = vfio_pci_info_zdev_add_caps(vdev, &caps);
	676	+ if (ret && ret != -ENODEV) {
	677	+ pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n");
	678	+ return ret;
	679	+ }
	680	+
	681	+ if (caps.size) {
	682	+ info.flags \|= VFIO_DEVICE_FLAGS_CAPS;
	683	+ if (info.argsz < sizeof(info) + caps.size) {
	684	+ info.argsz = sizeof(info) + caps.size;
	685	+ } else {
	686	+ vfio_info_cap_shift(&caps, sizeof(info));
	687	+ if (copy_to_user((void __user *)arg +
	688	+ sizeof(info), caps.buf,
	689	+ caps.size)) {
	690	+ kfree(caps.buf);
	691	+ return -EFAULT;
	692	+ }
	693	+ info.cap_offset = sizeof(info);
	694	+ }
	695	+
	696	+ kfree(caps.buf);
	697	+ }
	698	+
	699	+ return copy_to_user((void __user *)arg, &info, minsz) ?
	700	+ -EFAULT : 0;
	701	+
	702	+ } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
	703	+ struct pci_dev *pdev = vdev->pdev;
	704	+ struct vfio_region_info info;
	705	+ struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
	706	+ int i, ret;
	707	+
	708	+ minsz = offsetofend(struct vfio_region_info, offset);
	709	+
	710	+ if (copy_from_user(&info, (void __user *)arg, minsz))
	711	+ return -EFAULT;
	712	+
	713	+ if (info.argsz < minsz)
	714	+ return -EINVAL;
	715	+
	716	+ switch (info.index) {
	717	+ case VFIO_PCI_CONFIG_REGION_INDEX:
	718	+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
	719	+ info.size = pdev->cfg_size;
	720	+ info.flags = VFIO_REGION_INFO_FLAG_READ \|
	721	+ VFIO_REGION_INFO_FLAG_WRITE;
	722	+ break;
	723	+ case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
	724	+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
	725	+ info.size = pci_resource_len(pdev, info.index);
	726	+ if (!info.size) {
	727	+ info.flags = 0;
	728	+ break;
	729	+ }
	730	+
	731	+ info.flags = VFIO_REGION_INFO_FLAG_READ \|
	732	+ VFIO_REGION_INFO_FLAG_WRITE;
	733	+ if (vdev->bar_mmap_supported[info.index]) {
	734	+ info.flags \|= VFIO_REGION_INFO_FLAG_MMAP;
	735	+ if (info.index == vdev->msix_bar) {
	736	+ ret = msix_mmappable_cap(vdev, &caps);
	737	+ if (ret)
	738	+ return ret;
	739	+ }
	740	+ }
	741	+
	742	+ break;
	743	+ case VFIO_PCI_ROM_REGION_INDEX:
	744	+ {
	745	+ void __iomem *io;
	746	+ size_t size;
	747	+ u16 cmd;
	748	+
	749	+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
	750	+ info.flags = 0;
	751	+
	752	+ /* Report the BAR size, not the ROM size */
	753	+ info.size = pci_resource_len(pdev, info.index);
	754	+ if (!info.size) {
	755	+ /* Shadow ROMs appear as PCI option ROMs */
	756	+ if (pdev->resource[PCI_ROM_RESOURCE].flags &
	757	+ IORESOURCE_ROM_SHADOW)
	758	+ info.size = 0x20000;
	759	+ else
	760	+ break;
	761	+ }
	762	+
	763	+ /*
	764	+ * Is it really there? Enable memory decode for
	765	+ * implicit access in pci_map_rom().
	766	+ */
	767	+ cmd = vfio_pci_memory_lock_and_enable(vdev);
	768	+ io = pci_map_rom(pdev, &size);
	769	+ if (io) {
	770	+ info.flags = VFIO_REGION_INFO_FLAG_READ;
	771	+ pci_unmap_rom(pdev, io);
	772	+ } else {
	773	+ info.size = 0;
	774	+ }
	775	+ vfio_pci_memory_unlock_and_restore(vdev, cmd);
	776	+
	777	+ break;
	778	+ }
	779	+ case VFIO_PCI_VGA_REGION_INDEX:
	780	+ if (!vdev->has_vga)
	781	+ return -EINVAL;
	782	+
	783	+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
	784	+ info.size = 0xc0000;
	785	+ info.flags = VFIO_REGION_INFO_FLAG_READ \|
	786	+ VFIO_REGION_INFO_FLAG_WRITE;
	787	+
	788	+ break;
	789	+ default:
	790	+ {
	791	+ struct vfio_region_info_cap_type cap_type = {
	792	+ .header.id = VFIO_REGION_INFO_CAP_TYPE,
	793	+ .header.version = 1 };
	794	+
	795	+ if (info.index >=
	796	+ VFIO_PCI_NUM_REGIONS + vdev->num_regions)
	797	+ return -EINVAL;
	798	+ info.index = array_index_nospec(info.index,
	799	+ VFIO_PCI_NUM_REGIONS +
	800	+ vdev->num_regions);
	801	+
	802	+ i = info.index - VFIO_PCI_NUM_REGIONS;
	803	+
	804	+ info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
	805	+ info.size = vdev->region[i].size;
	806	+ info.flags = vdev->region[i].flags;
	807	+
	808	+ cap_type.type = vdev->region[i].type;
	809	+ cap_type.subtype = vdev->region[i].subtype;
	810	+
	811	+ ret = vfio_info_add_capability(&caps, &cap_type.header,
	812	+ sizeof(cap_type));
	813	+ if (ret)
	814	+ return ret;
	815	+
	816	+ if (vdev->region[i].ops->add_capability) {
	817	+ ret = vdev->region[i].ops->add_capability(vdev,
	818	+ &vdev->region[i], &caps);
	819	+ if (ret)
	820	+ return ret;
	821	+ }
	822	+ }
	823	+ }
	824	+
	825	+ if (caps.size) {
	826	+ info.flags \|= VFIO_REGION_INFO_FLAG_CAPS;
	827	+ if (info.argsz < sizeof(info) + caps.size) {
	828	+ info.argsz = sizeof(info) + caps.size;
	829	+ info.cap_offset = 0;
	830	+ } else {
	831	+ vfio_info_cap_shift(&caps, sizeof(info));
	832	+ if (copy_to_user((void __user *)arg +
	833	+ sizeof(info), caps.buf,
	834	+ caps.size)) {
	835	+ kfree(caps.buf);
	836	+ return -EFAULT;
	837	+ }
	838	+ info.cap_offset = sizeof(info);
	839	+ }
	840	+
	841	+ kfree(caps.buf);
	842	+ }
	843	+
	844	+ return copy_to_user((void __user *)arg, &info, minsz) ?
	845	+ -EFAULT : 0;
	846	+
	847	+ } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
	848	+ struct vfio_irq_info info;
	849	+
	850	+ minsz = offsetofend(struct vfio_irq_info, count);
	851	+
	852	+ if (copy_from_user(&info, (void __user *)arg, minsz))
	853	+ return -EFAULT;
	854	+
	855	+ if (info.argsz < minsz \|\| info.index >= VFIO_PCI_NUM_IRQS)
	856	+ return -EINVAL;
	857	+
	858	+ switch (info.index) {
	859	+ case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
	860	+ case VFIO_PCI_REQ_IRQ_INDEX:
	861	+ break;
	862	+ case VFIO_PCI_ERR_IRQ_INDEX:
	863	+ if (pci_is_pcie(vdev->pdev))
	864	+ break;
	865	+ fallthrough;
	866	+ default:
	867	+ return -EINVAL;
	868	+ }
	869	+
	870	+ info.flags = VFIO_IRQ_INFO_EVENTFD;
	871	+
	872	+ info.count = vfio_pci_get_irq_count(vdev, info.index);
	873	+
	874	+ if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
	875	+ info.flags \|= (VFIO_IRQ_INFO_MASKABLE \|
	876	+ VFIO_IRQ_INFO_AUTOMASKED);
	877	+ else
	878	+ info.flags \|= VFIO_IRQ_INFO_NORESIZE;
	879	+
	880	+ return copy_to_user((void __user *)arg, &info, minsz) ?
	881	+ -EFAULT : 0;
	882	+
	883	+ } else if (cmd == VFIO_DEVICE_SET_IRQS) {
	884	+ struct vfio_irq_set hdr;
	885	+ u8 *data = NULL;
	886	+ int max, ret = 0;
	887	+ size_t data_size = 0;
	888	+
	889	+ minsz = offsetofend(struct vfio_irq_set, count);
	890	+
	891	+ if (copy_from_user(&hdr, (void __user *)arg, minsz))
	892	+ return -EFAULT;
	893	+
	894	+ max = vfio_pci_get_irq_count(vdev, hdr.index);
	895	+
	896	+ ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
	897	+ VFIO_PCI_NUM_IRQS, &data_size);
	898	+ if (ret)
	899	+ return ret;
	900	+
	901	+ if (data_size) {
	902	+ data = memdup_user((void __user *)(arg + minsz),
	903	+ data_size);
	904	+ if (IS_ERR(data))
	905	+ return PTR_ERR(data);
	906	+ }
	907	+
	908	+ mutex_lock(&vdev->igate);
	909	+
	910	+ ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,
	911	+ hdr.start, hdr.count, data);
	912	+
	913	+ mutex_unlock(&vdev->igate);
	914	+ kfree(data);
	915	+
	916	+ return ret;
	917	+
	918	+ } else if (cmd == VFIO_DEVICE_RESET) {
	919	+ int ret;
	920	+
	921	+ if (!vdev->reset_works)
	922	+ return -EINVAL;
	923	+
	924	+ vfio_pci_zap_and_down_write_memory_lock(vdev);
	925	+ ret = pci_try_reset_function(vdev->pdev);
	926	+ up_write(&vdev->memory_lock);
	927	+
	928	+ return ret;
	929	+
	930	+ } else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) {
	931	+ struct vfio_pci_hot_reset_info hdr;
	932	+ struct vfio_pci_fill_info fill = { 0 };
	933	+ struct vfio_pci_dependent_device *devices = NULL;
	934	+ bool slot = false;
	935	+ int ret = 0;
	936	+
	937	+ minsz = offsetofend(struct vfio_pci_hot_reset_info, count);
	938	+
	939	+ if (copy_from_user(&hdr, (void __user *)arg, minsz))
	940	+ return -EFAULT;
	941	+
	942	+ if (hdr.argsz < minsz)
	943	+ return -EINVAL;
	944	+
	945	+ hdr.flags = 0;
	946	+
	947	+ /* Can we do a slot or bus reset or neither? */
	948	+ if (!pci_probe_reset_slot(vdev->pdev->slot))
	949	+ slot = true;
	950	+ else if (pci_probe_reset_bus(vdev->pdev->bus))
	951	+ return -ENODEV;
	952	+
	953	+ /* How many devices are affected? */
	954	+ ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
	955	+ vfio_pci_count_devs,
	956	+ &fill.max, slot);
	957	+ if (ret)
	958	+ return ret;
	959	+
	960	+ WARN_ON(!fill.max); /* Should always be at least one */
	961	+
	962	+ /*
	963	+ * If there's enough space, fill it now, otherwise return
	964	+ * -ENOSPC and the number of devices affected.
	965	+ */
	966	+ if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) {
	967	+ ret = -ENOSPC;
	968	+ hdr.count = fill.max;
	969	+ goto reset_info_exit;
	970	+ }
	971	+
	972	+ devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL);
	973	+ if (!devices)
	974	+ return -ENOMEM;
	975	+
	976	+ fill.devices = devices;
	977	+
	978	+ ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
	979	+ vfio_pci_fill_devs,
	980	+ &fill, slot);
	981	+
	982	+ /*
	983	+ * If a device was removed between counting and filling,
	984	+ * we may come up short of fill.max. If a device was
	985	+ * added, we'll have a return of -EAGAIN above.
	986	+ */
	987	+ if (!ret)
	988	+ hdr.count = fill.cur;
	989	+
	990	+reset_info_exit:
	991	+ if (copy_to_user((void __user *)arg, &hdr, minsz))
	992	+ ret = -EFAULT;
	993	+
	994	+ if (!ret) {
	995	+ if (copy_to_user((void __user *)(arg + minsz), devices,
	996	+ hdr.count * sizeof(*devices)))
	997	+ ret = -EFAULT;
	998	+ }
	999	+
	1000	+ kfree(devices);
	1001	+ return ret;
	1002	+
	1003	+ } else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) {
	1004	+ struct vfio_pci_hot_reset hdr;
	1005	+ int32_t *group_fds;
	1006	+ struct vfio_group **groups;
	1007	+ struct vfio_pci_group_info info;
	1008	+ bool slot = false;
	1009	+ int group_idx, count = 0, ret = 0;
	1010	+
	1011	+ minsz = offsetofend(struct vfio_pci_hot_reset, count);
	1012	+
	1013	+ if (copy_from_user(&hdr, (void __user *)arg, minsz))
	1014	+ return -EFAULT;
	1015	+
	1016	+ if (hdr.argsz < minsz \|\| hdr.flags)
	1017	+ return -EINVAL;
	1018	+
	1019	+ /* Can we do a slot or bus reset or neither? */
	1020	+ if (!pci_probe_reset_slot(vdev->pdev->slot))
	1021	+ slot = true;
	1022	+ else if (pci_probe_reset_bus(vdev->pdev->bus))
	1023	+ return -ENODEV;
	1024	+
	1025	+ /*
	1026	+ * We can't let userspace give us an arbitrarily large
	1027	+ * buffer to copy, so verify how many we think there
	1028	+ * could be. Note groups can have multiple devices so
	1029	+ * one group per device is the max.
	1030	+ */
	1031	+ ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
	1032	+ vfio_pci_count_devs,
	1033	+ &count, slot);
	1034	+ if (ret)
	1035	+ return ret;
	1036	+
	1037	+ /* Somewhere between 1 and count is OK */
	1038	+ if (!hdr.count \|\| hdr.count > count)
	1039	+ return -EINVAL;
	1040	+
	1041	+ group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL);
	1042	+ groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL);
	1043	+ if (!group_fds \|\| !groups) {
	1044	+ kfree(group_fds);
	1045	+ kfree(groups);
	1046	+ return -ENOMEM;
	1047	+ }
	1048	+
	1049	+ if (copy_from_user(group_fds, (void __user *)(arg + minsz),
	1050	+ hdr.count * sizeof(*group_fds))) {
	1051	+ kfree(group_fds);
	1052	+ kfree(groups);
	1053	+ return -EFAULT;
	1054	+ }
	1055	+
	1056	+ /*
	1057	+ * For each group_fd, get the group through the vfio external
	1058	+ * user interface and store the group and iommu ID. This
	1059	+ * ensures the group is held across the reset.
	1060	+ */
	1061	+ for (group_idx = 0; group_idx < hdr.count; group_idx++) {
	1062	+ struct vfio_group *group;
	1063	+ struct fd f = fdget(group_fds[group_idx]);
	1064	+ if (!f.file) {
	1065	+ ret = -EBADF;
	1066	+ break;
	1067	+ }
	1068	+
	1069	+ group = vfio_group_get_external_user(f.file);
	1070	+ fdput(f);
	1071	+ if (IS_ERR(group)) {
	1072	+ ret = PTR_ERR(group);
	1073	+ break;
	1074	+ }
	1075	+
	1076	+ groups[group_idx] = group;
	1077	+ }
	1078	+
	1079	+ kfree(group_fds);
	1080	+
	1081	+ /* release reference to groups on error */
	1082	+ if (ret)
	1083	+ goto hot_reset_release;
	1084	+
	1085	+ info.count = hdr.count;
	1086	+ info.groups = groups;
	1087	+
	1088	+ ret = vfio_pci_dev_set_hot_reset(vdev->vdev.dev_set, &info);
	1089	+
	1090	+hot_reset_release:
	1091	+ for (group_idx--; group_idx >= 0; group_idx--)
	1092	+ vfio_group_put_external_user(groups[group_idx]);
	1093	+
	1094	+ kfree(groups);
	1095	+ return ret;
	1096	+ } else if (cmd == VFIO_DEVICE_IOEVENTFD) {
	1097	+ struct vfio_device_ioeventfd ioeventfd;
	1098	+ int count;
	1099	+
	1100	+ minsz = offsetofend(struct vfio_device_ioeventfd, fd);
	1101	+
	1102	+ if (copy_from_user(&ioeventfd, (void __user *)arg, minsz))
	1103	+ return -EFAULT;
	1104	+
	1105	+ if (ioeventfd.argsz < minsz)
	1106	+ return -EINVAL;
	1107	+
	1108	+ if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK)
	1109	+ return -EINVAL;
	1110	+
	1111	+ count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK;
	1112	+
	1113	+ if (hweight8(count) != 1 \|\| ioeventfd.fd < -1)
	1114	+ return -EINVAL;
	1115	+
	1116	+ return vfio_pci_ioeventfd(vdev, ioeventfd.offset,
	1117	+ ioeventfd.data, count, ioeventfd.fd);
	1118	+ } else if (cmd == VFIO_DEVICE_FEATURE) {
	1119	+ struct vfio_device_feature feature;
	1120	+ uuid_t uuid;
	1121	+
	1122	+ minsz = offsetofend(struct vfio_device_feature, flags);
	1123	+
	1124	+ if (copy_from_user(&feature, (void __user *)arg, minsz))
	1125	+ return -EFAULT;
	1126	+
	1127	+ if (feature.argsz < minsz)
	1128	+ return -EINVAL;
	1129	+
	1130	+ /* Check unknown flags */
	1131	+ if (feature.flags & ~(VFIO_DEVICE_FEATURE_MASK \|
	1132	+ VFIO_DEVICE_FEATURE_SET \|
	1133	+ VFIO_DEVICE_FEATURE_GET \|
	1134	+ VFIO_DEVICE_FEATURE_PROBE))
	1135	+ return -EINVAL;
	1136	+
	1137	+ /* GET & SET are mutually exclusive except with PROBE */
	1138	+ if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) &&
	1139	+ (feature.flags & VFIO_DEVICE_FEATURE_SET) &&
	1140	+ (feature.flags & VFIO_DEVICE_FEATURE_GET))
	1141	+ return -EINVAL;
	1142	+
	1143	+ switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) {
	1144	+ case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN:
	1145	+ if (!vdev->vf_token)
	1146	+ return -ENOTTY;
	1147	+
	1148	+ /*
	1149	+ * We do not support GET of the VF Token UUID as this
	1150	+ * could expose the token of the previous device user.
	1151	+ */
	1152	+ if (feature.flags & VFIO_DEVICE_FEATURE_GET)
	1153	+ return -EINVAL;
	1154	+
	1155	+ if (feature.flags & VFIO_DEVICE_FEATURE_PROBE)
	1156	+ return 0;
	1157	+
	1158	+ /* Don't SET unless told to do so */
	1159	+ if (!(feature.flags & VFIO_DEVICE_FEATURE_SET))
	1160	+ return -EINVAL;
	1161	+
	1162	+ if (feature.argsz < minsz + sizeof(uuid))
	1163	+ return -EINVAL;
	1164	+
	1165	+ if (copy_from_user(&uuid, (void __user *)(arg + minsz),
	1166	+ sizeof(uuid)))
	1167	+ return -EFAULT;
	1168	+
	1169	+ mutex_lock(&vdev->vf_token->lock);
	1170	+ uuid_copy(&vdev->vf_token->uuid, &uuid);
	1171	+ mutex_unlock(&vdev->vf_token->lock);
	1172	+
	1173	+ return 0;
	1174	+ default:
	1175	+ return -ENOTTY;
	1176	+ }
	1177	+ }
	1178	+
	1179	+ return -ENOTTY;
	1180	+}
	1181	+EXPORT_SYMBOL_GPL(vfio_pci_core_ioctl);
	1182	+
	1183	+static ssize_t vfio_pci_rw(struct vfio_pci_core_device vdev, char __user buf,
	1184	+ size_t count, loff_t *ppos, bool iswrite)
	1185	+{
	1186	+ unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
	1187	+
	1188	+ if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
	1189	+ return -EINVAL;
	1190	+
	1191	+ switch (index) {
	1192	+ case VFIO_PCI_CONFIG_REGION_INDEX:
	1193	+ return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
	1194	+
	1195	+ case VFIO_PCI_ROM_REGION_INDEX:
	1196	+ if (iswrite)
	1197	+ return -EINVAL;
	1198	+ return vfio_pci_bar_rw(vdev, buf, count, ppos, false);
	1199	+
	1200	+ case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
	1201	+ return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
	1202	+
	1203	+ case VFIO_PCI_VGA_REGION_INDEX:
	1204	+ return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
	1205	+ default:
	1206	+ index -= VFIO_PCI_NUM_REGIONS;
	1207	+ return vdev->region[index].ops->rw(vdev, buf,
	1208	+ count, ppos, iswrite);
	1209	+ }
	1210	+
	1211	+ return -EINVAL;
	1212	+}
	1213	+
	1214	+ssize_t vfio_pci_core_read(struct vfio_device core_vdev, char __user buf,
	1215	+ size_t count, loff_t *ppos)
	1216	+{
	1217	+ struct vfio_pci_core_device *vdev =
	1218	+ container_of(core_vdev, struct vfio_pci_core_device, vdev);
	1219	+
	1220	+ if (!count)
	1221	+ return 0;
	1222	+
	1223	+ return vfio_pci_rw(vdev, buf, count, ppos, false);
	1224	+}
	1225	+EXPORT_SYMBOL_GPL(vfio_pci_core_read);
	1226	+
	1227	+ssize_t vfio_pci_core_write(struct vfio_device core_vdev, const char __user buf,
	1228	+ size_t count, loff_t *ppos)
	1229	+{
	1230	+ struct vfio_pci_core_device *vdev =
	1231	+ container_of(core_vdev, struct vfio_pci_core_device, vdev);
	1232	+
	1233	+ if (!count)
	1234	+ return 0;
	1235	+
	1236	+ return vfio_pci_rw(vdev, (char __user *)buf, count, ppos, true);
	1237	+}
	1238	+EXPORT_SYMBOL_GPL(vfio_pci_core_write);
	1239	+
	1240	+/* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */
	1241	+static int vfio_pci_zap_and_vma_lock(struct vfio_pci_core_device *vdev, bool try)
	1242	+{
	1243	+ struct vfio_pci_mmap_vma mmap_vma, tmp;
	1244	+
	1245	+ /*
	1246	+ * Lock ordering:
	1247	+ * vma_lock is nested under mmap_lock for vm_ops callback paths.
	1248	+ * The memory_lock semaphore is used by both code paths calling
	1249	+ * into this function to zap vmas and the vm_ops.fault callback
	1250	+ * to protect the memory enable state of the device.
	1251	+ *
	1252	+ * When zapping vmas we need to maintain the mmap_lock => vma_lock
	1253	+ * ordering, which requires using vma_lock to walk vma_list to
	1254	+ * acquire an mm, then dropping vma_lock to get the mmap_lock and
	1255	+ * reacquiring vma_lock. This logic is derived from similar
	1256	+ * requirements in uverbs_user_mmap_disassociate().
	1257	+ *
	1258	+ * mmap_lock must always be the top-level lock when it is taken.
	1259	+ * Therefore we can only hold the memory_lock write lock when
	1260	+ * vma_list is empty, as we'd need to take mmap_lock to clear
	1261	+ * entries. vma_list can only be guaranteed empty when holding
	1262	+ * vma_lock, thus memory_lock is nested under vma_lock.
	1263	+ *
	1264	+ * This enables the vm_ops.fault callback to acquire vma_lock,
	1265	+ * followed by memory_lock read lock, while already holding
	1266	+ * mmap_lock without risk of deadlock.
	1267	+ */
	1268	+ while (1) {
	1269	+ struct mm_struct *mm = NULL;
	1270	+
	1271	+ if (try) {
	1272	+ if (!mutex_trylock(&vdev->vma_lock))
	1273	+ return 0;
	1274	+ } else {
	1275	+ mutex_lock(&vdev->vma_lock);
	1276	+ }
	1277	+ while (!list_empty(&vdev->vma_list)) {
	1278	+ mmap_vma = list_first_entry(&vdev->vma_list,
	1279	+ struct vfio_pci_mmap_vma,
	1280	+ vma_next);
	1281	+ mm = mmap_vma->vma->vm_mm;
	1282	+ if (mmget_not_zero(mm))
	1283	+ break;
	1284	+
	1285	+ list_del(&mmap_vma->vma_next);
	1286	+ kfree(mmap_vma);
	1287	+ mm = NULL;
	1288	+ }
	1289	+ if (!mm)
	1290	+ return 1;
	1291	+ mutex_unlock(&vdev->vma_lock);
	1292	+
	1293	+ if (try) {
	1294	+ if (!mmap_read_trylock(mm)) {
	1295	+ mmput(mm);
	1296	+ return 0;
	1297	+ }
	1298	+ } else {
	1299	+ mmap_read_lock(mm);
	1300	+ }
	1301	+ if (try) {
	1302	+ if (!mutex_trylock(&vdev->vma_lock)) {
	1303	+ mmap_read_unlock(mm);
	1304	+ mmput(mm);
	1305	+ return 0;
	1306	+ }
	1307	+ } else {
	1308	+ mutex_lock(&vdev->vma_lock);
	1309	+ }
	1310	+ list_for_each_entry_safe(mmap_vma, tmp,
	1311	+ &vdev->vma_list, vma_next) {
	1312	+ struct vm_area_struct *vma = mmap_vma->vma;
	1313	+
	1314	+ if (vma->vm_mm != mm)
	1315	+ continue;
	1316	+
	1317	+ list_del(&mmap_vma->vma_next);
	1318	+ kfree(mmap_vma);
	1319	+
	1320	+ zap_vma_ptes(vma, vma->vm_start,
	1321	+ vma->vm_end - vma->vm_start);
	1322	+ }
	1323	+ mutex_unlock(&vdev->vma_lock);
	1324	+ mmap_read_unlock(mm);
	1325	+ mmput(mm);
	1326	+ }
	1327	+}
	1328	+
	1329	+void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device *vdev)
	1330	+{
	1331	+ vfio_pci_zap_and_vma_lock(vdev, false);
	1332	+ down_write(&vdev->memory_lock);
	1333	+ mutex_unlock(&vdev->vma_lock);
	1334	+}
	1335	+
	1336	+u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev)
	1337	+{
	1338	+ u16 cmd;
	1339	+
	1340	+ down_write(&vdev->memory_lock);
	1341	+ pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd);
	1342	+ if (!(cmd & PCI_COMMAND_MEMORY))
	1343	+ pci_write_config_word(vdev->pdev, PCI_COMMAND,
	1344	+ cmd \| PCI_COMMAND_MEMORY);
	1345	+
	1346	+ return cmd;
	1347	+}
	1348	+
	1349	+void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev, u16 cmd)
	1350	+{
	1351	+ pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd);
	1352	+ up_write(&vdev->memory_lock);
	1353	+}
	1354	+
	1355	+/* Caller holds vma_lock */
	1356	+static int __vfio_pci_add_vma(struct vfio_pci_core_device *vdev,
	1357	+ struct vm_area_struct *vma)
	1358	+{
	1359	+ struct vfio_pci_mmap_vma *mmap_vma;
	1360	+
	1361	+ mmap_vma = kmalloc(sizeof(*mmap_vma), GFP_KERNEL);
	1362	+ if (!mmap_vma)
	1363	+ return -ENOMEM;
	1364	+
	1365	+ mmap_vma->vma = vma;
	1366	+ list_add(&mmap_vma->vma_next, &vdev->vma_list);
	1367	+
	1368	+ return 0;
	1369	+}
	1370	+
	1371	+/*
	1372	+ * Zap mmaps on open so that we can fault them in on access and therefore
	1373	+ * our vma_list only tracks mappings accessed since last zap.
	1374	+ */
	1375	+static void vfio_pci_mmap_open(struct vm_area_struct *vma)
	1376	+{
	1377	+ zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
	1378	+}
	1379	+
	1380	+static void vfio_pci_mmap_close(struct vm_area_struct *vma)
	1381	+{
	1382	+ struct vfio_pci_core_device *vdev = vma->vm_private_data;
	1383	+ struct vfio_pci_mmap_vma *mmap_vma;
	1384	+
	1385	+ mutex_lock(&vdev->vma_lock);
	1386	+ list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
	1387	+ if (mmap_vma->vma == vma) {
	1388	+ list_del(&mmap_vma->vma_next);
	1389	+ kfree(mmap_vma);
	1390	+ break;
	1391	+ }
	1392	+ }
	1393	+ mutex_unlock(&vdev->vma_lock);
	1394	+}
	1395	+
	1396	+static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf)
	1397	+{
	1398	+ struct vm_area_struct *vma = vmf->vma;
	1399	+ struct vfio_pci_core_device *vdev = vma->vm_private_data;
	1400	+ struct vfio_pci_mmap_vma *mmap_vma;
	1401	+ vm_fault_t ret = VM_FAULT_NOPAGE;
	1402	+
	1403	+ mutex_lock(&vdev->vma_lock);
	1404	+ down_read(&vdev->memory_lock);
	1405	+
	1406	+ if (!__vfio_pci_memory_enabled(vdev)) {
	1407	+ ret = VM_FAULT_SIGBUS;
	1408	+ goto up_out;
	1409	+ }
	1410	+
	1411	+ /*
	1412	+ * We populate the whole vma on fault, so we need to test whether
	1413	+ * the vma has already been mapped, such as for concurrent faults
	1414	+ * to the same vma. io_remap_pfn_range() will trigger a BUG_ON if
	1415	+ * we ask it to fill the same range again.
	1416	+ */
	1417	+ list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
	1418	+ if (mmap_vma->vma == vma)
	1419	+ goto up_out;
	1420	+ }
	1421	+
	1422	+ if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
	1423	+ vma->vm_end - vma->vm_start,
	1424	+ vma->vm_page_prot)) {
	1425	+ ret = VM_FAULT_SIGBUS;
	1426	+ zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
	1427	+ goto up_out;
	1428	+ }
	1429	+
	1430	+ if (__vfio_pci_add_vma(vdev, vma)) {
	1431	+ ret = VM_FAULT_OOM;
	1432	+ zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
	1433	+ }
	1434	+
	1435	+up_out:
	1436	+ up_read(&vdev->memory_lock);
	1437	+ mutex_unlock(&vdev->vma_lock);
	1438	+ return ret;
	1439	+}
	1440	+
	1441	+static const struct vm_operations_struct vfio_pci_mmap_ops = {
	1442	+ .open = vfio_pci_mmap_open,
	1443	+ .close = vfio_pci_mmap_close,
	1444	+ .fault = vfio_pci_mmap_fault,
	1445	+};
	1446	+
	1447	+int vfio_pci_core_mmap(struct vfio_device core_vdev, struct vm_area_struct vma)
	1448	+{
	1449	+ struct vfio_pci_core_device *vdev =
	1450	+ container_of(core_vdev, struct vfio_pci_core_device, vdev);
	1451	+ struct pci_dev *pdev = vdev->pdev;
	1452	+ unsigned int index;
	1453	+ u64 phys_len, req_len, pgoff, req_start;
	1454	+ int ret;
	1455	+
	1456	+ index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
	1457	+
	1458	+ if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
	1459	+ return -EINVAL;
	1460	+ if (vma->vm_end < vma->vm_start)
	1461	+ return -EINVAL;
	1462	+ if ((vma->vm_flags & VM_SHARED) == 0)
	1463	+ return -EINVAL;
	1464	+ if (index >= VFIO_PCI_NUM_REGIONS) {
	1465	+ int regnum = index - VFIO_PCI_NUM_REGIONS;
	1466	+ struct vfio_pci_region *region = vdev->region + regnum;
	1467	+
	1468	+ if (region->ops && region->ops->mmap &&
	1469	+ (region->flags & VFIO_REGION_INFO_FLAG_MMAP))
	1470	+ return region->ops->mmap(vdev, region, vma);
	1471	+ return -EINVAL;
	1472	+ }
	1473	+ if (index >= VFIO_PCI_ROM_REGION_INDEX)
	1474	+ return -EINVAL;
	1475	+ if (!vdev->bar_mmap_supported[index])
	1476	+ return -EINVAL;
	1477	+
	1478	+ phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));
	1479	+ req_len = vma->vm_end - vma->vm_start;
	1480	+ pgoff = vma->vm_pgoff &
	1481	+ ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
	1482	+ req_start = pgoff << PAGE_SHIFT;
	1483	+
	1484	+ if (req_start + req_len > phys_len)
	1485	+ return -EINVAL;
	1486	+
	1487	+ /*
	1488	+ * Even though we don't make use of the barmap for the mmap,
	1489	+ * we need to request the region and the barmap tracks that.
	1490	+ */
	1491	+ if (!vdev->barmap[index]) {
	1492	+ ret = pci_request_selected_regions(pdev,
	1493	+ 1 << index, "vfio-pci");
	1494	+ if (ret)
	1495	+ return ret;
	1496	+
	1497	+ vdev->barmap[index] = pci_iomap(pdev, index, 0);
	1498	+ if (!vdev->barmap[index]) {
	1499	+ pci_release_selected_regions(pdev, 1 << index);
	1500	+ return -ENOMEM;
	1501	+ }
	1502	+ }
	1503	+
	1504	+ vma->vm_private_data = vdev;
	1505	+ vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
	1506	+ vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;
	1507	+
	1508	+ /*
	1509	+ * See remap_pfn_range(), called from vfio_pci_fault() but we can't
	1510	+ * change vm_flags within the fault handler. Set them now.
	1511	+ */
	1512	+ vma->vm_flags \|= VM_IO \| VM_PFNMAP \| VM_DONTEXPAND \| VM_DONTDUMP;
	1513	+ vma->vm_ops = &vfio_pci_mmap_ops;
	1514	+
	1515	+ return 0;
	1516	+}
	1517	+EXPORT_SYMBOL_GPL(vfio_pci_core_mmap);
	1518	+
	1519	+void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count)
	1520	+{
	1521	+ struct vfio_pci_core_device *vdev =
	1522	+ container_of(core_vdev, struct vfio_pci_core_device, vdev);
	1523	+ struct pci_dev *pdev = vdev->pdev;
	1524	+
	1525	+ mutex_lock(&vdev->igate);
	1526	+
	1527	+ if (vdev->req_trigger) {
	1528	+ if (!(count % 10))
	1529	+ pci_notice_ratelimited(pdev,
	1530	+ "Relaying device request to user (#%u)\n",
	1531	+ count);
	1532	+ eventfd_signal(vdev->req_trigger, 1);
	1533	+ } else if (count == 0) {
	1534	+ pci_warn(pdev,
	1535	+ "No device request channel registered, blocked until released by user\n");
	1536	+ }
	1537	+
	1538	+ mutex_unlock(&vdev->igate);
	1539	+}
	1540	+EXPORT_SYMBOL_GPL(vfio_pci_core_request);
	1541	+
	1542	+static int vfio_pci_validate_vf_token(struct vfio_pci_core_device *vdev,
	1543	+ bool vf_token, uuid_t *uuid)
	1544	+{
	1545	+ /*
	1546	+ * There's always some degree of trust or collaboration between SR-IOV
	1547	+ * PF and VFs, even if just that the PF hosts the SR-IOV capability and
	1548	+ * can disrupt VFs with a reset, but often the PF has more explicit
	1549	+ * access to deny service to the VF or access data passed through the
	1550	+ * VF. We therefore require an opt-in via a shared VF token (UUID) to
	1551	+ * represent this trust. This both prevents that a VF driver might
	1552	+ * assume the PF driver is a trusted, in-kernel driver, and also that
	1553	+ * a PF driver might be replaced with a rogue driver, unknown to in-use
	1554	+ * VF drivers.
	1555	+ *
	1556	+ * Therefore when presented with a VF, if the PF is a vfio device and
	1557	+ * it is bound to the vfio-pci driver, the user needs to provide a VF
	1558	+ * token to access the device, in the form of appending a vf_token to
	1559	+ * the device name, for example:
	1560	+ *
	1561	+ * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3"
	1562	+ *
	1563	+ * When presented with a PF which has VFs in use, the user must also
	1564	+ * provide the current VF token to prove collaboration with existing
	1565	+ * VF users. If VFs are not in use, the VF token provided for the PF
	1566	+ * device will act to set the VF token.
	1567	+ *
	1568	+ * If the VF token is provided but unused, an error is generated.
	1569	+ */
	1570	+ if (!vdev->pdev->is_virtfn && !vdev->vf_token && !vf_token)
	1571	+ return 0; /* No VF token provided or required */
	1572	+
	1573	+ if (vdev->pdev->is_virtfn) {
	1574	+ struct vfio_pci_core_device *pf_vdev = get_pf_vdev(vdev);
	1575	+ bool match;
	1576	+
	1577	+ if (!pf_vdev) {
	1578	+ if (!vf_token)
	1579	+ return 0; /* PF is not vfio-pci, no VF token */
	1580	+
	1581	+ pci_info_ratelimited(vdev->pdev,
	1582	+ "VF token incorrectly provided, PF not bound to vfio-pci\n");
	1583	+ return -EINVAL;
	1584	+ }
	1585	+
	1586	+ if (!vf_token) {
	1587	+ vfio_device_put(&pf_vdev->vdev);
	1588	+ pci_info_ratelimited(vdev->pdev,
	1589	+ "VF token required to access device\n");
	1590	+ return -EACCES;
	1591	+ }
	1592	+
	1593	+ mutex_lock(&pf_vdev->vf_token->lock);
	1594	+ match = uuid_equal(uuid, &pf_vdev->vf_token->uuid);
	1595	+ mutex_unlock(&pf_vdev->vf_token->lock);
	1596	+
	1597	+ vfio_device_put(&pf_vdev->vdev);
	1598	+
	1599	+ if (!match) {
	1600	+ pci_info_ratelimited(vdev->pdev,
	1601	+ "Incorrect VF token provided for device\n");
	1602	+ return -EACCES;
	1603	+ }
	1604	+ } else if (vdev->vf_token) {
	1605	+ mutex_lock(&vdev->vf_token->lock);
	1606	+ if (vdev->vf_token->users) {
	1607	+ if (!vf_token) {
	1608	+ mutex_unlock(&vdev->vf_token->lock);
	1609	+ pci_info_ratelimited(vdev->pdev,
	1610	+ "VF token required to access device\n");
	1611	+ return -EACCES;
	1612	+ }
	1613	+
	1614	+ if (!uuid_equal(uuid, &vdev->vf_token->uuid)) {
	1615	+ mutex_unlock(&vdev->vf_token->lock);
	1616	+ pci_info_ratelimited(vdev->pdev,
	1617	+ "Incorrect VF token provided for device\n");
	1618	+ return -EACCES;
	1619	+ }
	1620	+ } else if (vf_token) {
	1621	+ uuid_copy(&vdev->vf_token->uuid, uuid);
	1622	+ }
	1623	+
	1624	+ mutex_unlock(&vdev->vf_token->lock);
	1625	+ } else if (vf_token) {
	1626	+ pci_info_ratelimited(vdev->pdev,
	1627	+ "VF token incorrectly provided, not a PF or VF\n");
	1628	+ return -EINVAL;
	1629	+ }
	1630	+
	1631	+ return 0;
	1632	+}
	1633	+
	1634	+#define VF_TOKEN_ARG "vf_token="
	1635	+
	1636	+int vfio_pci_core_match(struct vfio_device core_vdev, char buf)
	1637	+{
	1638	+ struct vfio_pci_core_device *vdev =
	1639	+ container_of(core_vdev, struct vfio_pci_core_device, vdev);
	1640	+ bool vf_token = false;
	1641	+ uuid_t uuid;
	1642	+ int ret;
	1643	+
	1644	+ if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev))))
	1645	+ return 0; /* No match */
	1646	+
	1647	+ if (strlen(buf) > strlen(pci_name(vdev->pdev))) {
	1648	+ buf += strlen(pci_name(vdev->pdev));
	1649	+
	1650	+ if (*buf != ' ')
	1651	+ return 0; /* No match: non-whitespace after name */
	1652	+
	1653	+ while (*buf) {
	1654	+ if (*buf == ' ') {
	1655	+ buf++;
	1656	+ continue;
	1657	+ }
	1658	+
	1659	+ if (!vf_token && !strncmp(buf, VF_TOKEN_ARG,
	1660	+ strlen(VF_TOKEN_ARG))) {
	1661	+ buf += strlen(VF_TOKEN_ARG);
	1662	+
	1663	+ if (strlen(buf) < UUID_STRING_LEN)
	1664	+ return -EINVAL;
	1665	+
	1666	+ ret = uuid_parse(buf, &uuid);
	1667	+ if (ret)
	1668	+ return ret;
	1669	+
	1670	+ vf_token = true;
	1671	+ buf += UUID_STRING_LEN;
	1672	+ } else {
	1673	+ /* Unknown/duplicate option */
	1674	+ return -EINVAL;
	1675	+ }
	1676	+ }
	1677	+ }
	1678	+
	1679	+ ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid);
	1680	+ if (ret)
	1681	+ return ret;
	1682	+
	1683	+ return 1; /* Match */
	1684	+}
	1685	+EXPORT_SYMBOL_GPL(vfio_pci_core_match);
	1686	+
	1687	+static int vfio_pci_bus_notifier(struct notifier_block *nb,
	1688	+ unsigned long action, void *data)
	1689	+{
	1690	+ struct vfio_pci_core_device *vdev = container_of(nb,
	1691	+ struct vfio_pci_core_device, nb);
	1692	+ struct device *dev = data;
	1693	+ struct pci_dev *pdev = to_pci_dev(dev);
	1694	+ struct pci_dev *physfn = pci_physfn(pdev);
	1695	+
	1696	+ if (action == BUS_NOTIFY_ADD_DEVICE &&
	1697	+ pdev->is_virtfn && physfn == vdev->pdev) {
	1698	+ pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n",
	1699	+ pci_name(pdev));
	1700	+ pdev->driver_override = kasprintf(GFP_KERNEL, "%s",
	1701	+ vdev->vdev.ops->name);
	1702	+ } else if (action == BUS_NOTIFY_BOUND_DRIVER &&
	1703	+ pdev->is_virtfn && physfn == vdev->pdev) {
	1704	+ struct pci_driver *drv = pci_dev_driver(pdev);
	1705	+
	1706	+ if (drv && drv != pci_dev_driver(vdev->pdev))
	1707	+ pci_warn(vdev->pdev,
	1708	+ "VF %s bound to driver %s while PF bound to driver %s\n",
	1709	+ pci_name(pdev), drv->name,
	1710	+ pci_dev_driver(vdev->pdev)->name);
	1711	+ }
	1712	+
	1713	+ return 0;
	1714	+}
	1715	+
	1716	+static int vfio_pci_vf_init(struct vfio_pci_core_device *vdev)
	1717	+{
	1718	+ struct pci_dev *pdev = vdev->pdev;
	1719	+ int ret;
	1720	+
	1721	+ if (!pdev->is_physfn)
	1722	+ return 0;
	1723	+
	1724	+ vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL);
	1725	+ if (!vdev->vf_token)
	1726	+ return -ENOMEM;
	1727	+
	1728	+ mutex_init(&vdev->vf_token->lock);
	1729	+ uuid_gen(&vdev->vf_token->uuid);
	1730	+
	1731	+ vdev->nb.notifier_call = vfio_pci_bus_notifier;
	1732	+ ret = bus_register_notifier(&pci_bus_type, &vdev->nb);
	1733	+ if (ret) {
	1734	+ kfree(vdev->vf_token);
	1735	+ return ret;
	1736	+ }
	1737	+ return 0;
	1738	+}
	1739	+
	1740	+static void vfio_pci_vf_uninit(struct vfio_pci_core_device *vdev)
	1741	+{
	1742	+ if (!vdev->vf_token)
	1743	+ return;
	1744	+
	1745	+ bus_unregister_notifier(&pci_bus_type, &vdev->nb);
	1746	+ WARN_ON(vdev->vf_token->users);
	1747	+ mutex_destroy(&vdev->vf_token->lock);
	1748	+ kfree(vdev->vf_token);
	1749	+}
	1750	+
	1751	+static int vfio_pci_vga_init(struct vfio_pci_core_device *vdev)
	1752	+{
	1753	+ struct pci_dev *pdev = vdev->pdev;
	1754	+ int ret;
	1755	+
	1756	+ if (!vfio_pci_is_vga(pdev))
	1757	+ return 0;
	1758	+
	1759	+ ret = vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode);
	1760	+ if (ret)
	1761	+ return ret;
	1762	+ vga_set_legacy_decoding(pdev, vfio_pci_set_vga_decode(vdev, false));
	1763	+ return 0;
	1764	+}
	1765	+
	1766	+static void vfio_pci_vga_uninit(struct vfio_pci_core_device *vdev)
	1767	+{
	1768	+ struct pci_dev *pdev = vdev->pdev;
	1769	+
	1770	+ if (!vfio_pci_is_vga(pdev))
	1771	+ return;
	1772	+ vga_client_register(pdev, NULL, NULL, NULL);
	1773	+ vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO \| VGA_RSRC_NORMAL_MEM \|
	1774	+ VGA_RSRC_LEGACY_IO \|
	1775	+ VGA_RSRC_LEGACY_MEM);
	1776	+}
	1777	+
	1778	+void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev,
	1779	+ struct pci_dev *pdev,
	1780	+ const struct vfio_device_ops *vfio_pci_ops)
	1781	+{
	1782	+ vfio_init_group_dev(&vdev->vdev, &pdev->dev, vfio_pci_ops);
	1783	+ vdev->pdev = pdev;
	1784	+ vdev->irq_type = VFIO_PCI_NUM_IRQS;
	1785	+ mutex_init(&vdev->igate);
	1786	+ spin_lock_init(&vdev->irqlock);
	1787	+ mutex_init(&vdev->ioeventfds_lock);
	1788	+ INIT_LIST_HEAD(&vdev->dummy_resources_list);
	1789	+ INIT_LIST_HEAD(&vdev->ioeventfds_list);
	1790	+ mutex_init(&vdev->vma_lock);
	1791	+ INIT_LIST_HEAD(&vdev->vma_list);
	1792	+ init_rwsem(&vdev->memory_lock);
	1793	+}
	1794	+EXPORT_SYMBOL_GPL(vfio_pci_core_init_device);
	1795	+
	1796	+void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev)
	1797	+{
	1798	+ mutex_destroy(&vdev->igate);
	1799	+ mutex_destroy(&vdev->ioeventfds_lock);
	1800	+ mutex_destroy(&vdev->vma_lock);
	1801	+ vfio_uninit_group_dev(&vdev->vdev);
	1802	+ kfree(vdev->region);
	1803	+ kfree(vdev->pm_save);
	1804	+}
	1805	+EXPORT_SYMBOL_GPL(vfio_pci_core_uninit_device);
	1806	+
	1807	+int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev)
	1808	+{
	1809	+ struct pci_dev *pdev = vdev->pdev;
	1810	+ struct iommu_group *group;
	1811	+ int ret;
	1812	+
	1813	+ if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)
	1814	+ return -EINVAL;
	1815	+
	1816	+ /*
	1817	+ * Prevent binding to PFs with VFs enabled, the VFs might be in use
	1818	+ * by the host or other users. We cannot capture the VFs if they
	1819	+ * already exist, nor can we track VF users. Disabling SR-IOV here
	1820	+ * would initiate removing the VFs, which would unbind the driver,
	1821	+ * which is prone to blocking if that VF is also in use by vfio-pci.
	1822	+ * Just reject these PFs and let the user sort it out.
	1823	+ */
	1824	+ if (pci_num_vf(pdev)) {
	1825	+ pci_warn(pdev, "Cannot bind to PF with SR-IOV enabled\n");
	1826	+ return -EBUSY;
	1827	+ }
	1828	+
	1829	+ group = vfio_iommu_group_get(&pdev->dev);
	1830	+ if (!group)
	1831	+ return -EINVAL;
	1832	+
	1833	+ if (pci_is_root_bus(pdev->bus)) {
	1834	+ ret = vfio_assign_device_set(&vdev->vdev, vdev);
	1835	+ } else if (!pci_probe_reset_slot(pdev->slot)) {
	1836	+ ret = vfio_assign_device_set(&vdev->vdev, pdev->slot);
	1837	+ } else {
	1838	+ /*
	1839	+ * If there is no slot reset support for this device, the whole
	1840	+ * bus needs to be grouped together to support bus-wide resets.
	1841	+ */
	1842	+ ret = vfio_assign_device_set(&vdev->vdev, pdev->bus);
	1843	+ }
	1844	+
	1845	+ if (ret)
	1846	+ goto out_group_put;
	1847	+ ret = vfio_pci_vf_init(vdev);
	1848	+ if (ret)
	1849	+ goto out_group_put;
	1850	+ ret = vfio_pci_vga_init(vdev);
	1851	+ if (ret)
	1852	+ goto out_vf;
	1853	+
	1854	+ vfio_pci_probe_power_state(vdev);
	1855	+
	1856	+ if (!disable_idle_d3) {
	1857	+ /*
	1858	+ * pci-core sets the device power state to an unknown value at
	1859	+ * bootup and after being removed from a driver. The only
	1860	+ * transition it allows from this unknown state is to D0, which
	1861	+ * typically happens when a driver calls pci_enable_device().
	1862	+ * We're not ready to enable the device yet, but we do want to
	1863	+ * be able to get to D3. Therefore first do a D0 transition
	1864	+ * before going to D3.
	1865	+ */
	1866	+ vfio_pci_set_power_state(vdev, PCI_D0);
	1867	+ vfio_pci_set_power_state(vdev, PCI_D3hot);
	1868	+ }
	1869	+
	1870	+ ret = vfio_register_group_dev(&vdev->vdev);
	1871	+ if (ret)
	1872	+ goto out_power;
	1873	+ return 0;
	1874	+
	1875	+out_power:
	1876	+ if (!disable_idle_d3)
	1877	+ vfio_pci_set_power_state(vdev, PCI_D0);
	1878	+out_vf:
	1879	+ vfio_pci_vf_uninit(vdev);
	1880	+out_group_put:
	1881	+ vfio_iommu_group_put(group, &pdev->dev);
	1882	+ return ret;
	1883	+}
	1884	+EXPORT_SYMBOL_GPL(vfio_pci_core_register_device);
	1885	+
	1886	+void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev)
	1887	+{
	1888	+ struct pci_dev *pdev = vdev->pdev;
	1889	+
	1890	+ pci_disable_sriov(pdev);
	1891	+
	1892	+ vfio_unregister_group_dev(&vdev->vdev);
	1893	+
	1894	+ vfio_pci_vf_uninit(vdev);
	1895	+ vfio_pci_vga_uninit(vdev);
	1896	+
	1897	+ vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev);
	1898	+
	1899	+ if (!disable_idle_d3)
	1900	+ vfio_pci_set_power_state(vdev, PCI_D0);
	1901	+}
	1902	+EXPORT_SYMBOL_GPL(vfio_pci_core_unregister_device);
	1903	+
	1904	+static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,
	1905	+ pci_channel_state_t state)
	1906	+{
	1907	+ struct vfio_pci_core_device *vdev;
	1908	+ struct vfio_device *device;
	1909	+
	1910	+ device = vfio_device_get_from_dev(&pdev->dev);
	1911	+ if (device == NULL)
	1912	+ return PCI_ERS_RESULT_DISCONNECT;
	1913	+
	1914	+ vdev = container_of(device, struct vfio_pci_core_device, vdev);
	1915	+
	1916	+ mutex_lock(&vdev->igate);
	1917	+
	1918	+ if (vdev->err_trigger)
	1919	+ eventfd_signal(vdev->err_trigger, 1);
	1920	+
	1921	+ mutex_unlock(&vdev->igate);
	1922	+
	1923	+ vfio_device_put(device);
	1924	+
	1925	+ return PCI_ERS_RESULT_CAN_RECOVER;
	1926	+}
	1927	+
	1928	+int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn)
	1929	+{
	1930	+ struct vfio_device *device;
	1931	+ int ret = 0;
	1932	+
	1933	+ device = vfio_device_get_from_dev(&pdev->dev);
	1934	+ if (!device)
	1935	+ return -ENODEV;
	1936	+
	1937	+ if (nr_virtfn == 0)
	1938	+ pci_disable_sriov(pdev);
	1939	+ else
	1940	+ ret = pci_enable_sriov(pdev, nr_virtfn);
	1941	+
	1942	+ vfio_device_put(device);
	1943	+
	1944	+ return ret < 0 ? ret : nr_virtfn;
	1945	+}
	1946	+EXPORT_SYMBOL_GPL(vfio_pci_core_sriov_configure);
	1947	+
	1948	+const struct pci_error_handlers vfio_pci_core_err_handlers = {
	1949	+ .error_detected = vfio_pci_aer_err_detected,
	1950	+};
	1951	+EXPORT_SYMBOL_GPL(vfio_pci_core_err_handlers);
	1952	+
	1953	+static bool vfio_dev_in_groups(struct vfio_pci_core_device *vdev,
	1954	+ struct vfio_pci_group_info *groups)
	1955	+{
	1956	+ unsigned int i;
	1957	+
	1958	+ for (i = 0; i < groups->count; i++)
	1959	+ if (groups->groups[i] == vdev->vdev.group)
	1960	+ return true;
	1961	+ return false;
	1962	+}
	1963	+
	1964	+static int vfio_pci_is_device_in_set(struct pci_dev pdev, void data)
	1965	+{
	1966	+ struct vfio_device_set *dev_set = data;
	1967	+ struct vfio_device *cur;
	1968	+
	1969	+ list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
	1970	+ if (cur->dev == &pdev->dev)
	1971	+ return 0;
	1972	+ return -EBUSY;
	1973	+}
	1974	+
	1975	+/*
	1976	+ * vfio-core considers a group to be viable and will create a vfio_device even
	1977	+ * if some devices are bound to drivers like pci-stub or pcieport. Here we
	1978	+ * require all PCI devices to be inside our dev_set since that ensures they stay
	1979	+ * put and that every driver controlling the device can co-ordinate with the
	1980	+ * device reset.
	1981	+ *
	1982	+ * Returns the pci_dev to pass to pci_reset_bus() if every PCI device to be
	1983	+ * reset is inside the dev_set, and pci_reset_bus() can succeed. NULL otherwise.
	1984	+ */
	1985	+static struct pci_dev *
	1986	+vfio_pci_dev_set_resettable(struct vfio_device_set *dev_set)
	1987	+{
	1988	+ struct pci_dev *pdev;
	1989	+
	1990	+ lockdep_assert_held(&dev_set->lock);
	1991	+
	1992	+ /*
	1993	+ * By definition all PCI devices in the dev_set share the same PCI
	1994	+ * reset, so any pci_dev will have the same outcomes for
	1995	+ * pci_probe_reset_*() and pci_reset_bus().
	1996	+ */
	1997	+ pdev = list_first_entry(&dev_set->device_list,
	1998	+ struct vfio_pci_core_device,
	1999	+ vdev.dev_set_list)->pdev;
	2000	+
	2001	+ /* pci_reset_bus() is supported */
	2002	+ if (pci_probe_reset_slot(pdev->slot) && pci_probe_reset_bus(pdev->bus))
	2003	+ return NULL;
	2004	+
	2005	+ if (vfio_pci_for_each_slot_or_bus(pdev, vfio_pci_is_device_in_set,
	2006	+ dev_set,
	2007	+ !pci_probe_reset_slot(pdev->slot)))
	2008	+ return NULL;
	2009	+ return pdev;
	2010	+}
	2011	+
	2012	+/*
	2013	+ * We need to get memory_lock for each device, but devices can share mmap_lock,
	2014	+ * therefore we need to zap and hold the vma_lock for each device, and only then
	2015	+ * get each memory_lock.
	2016	+ */
	2017	+static int vfio_pci_dev_set_hot_reset(struct vfio_device_set *dev_set,
	2018	+ struct vfio_pci_group_info *groups)
	2019	+{
	2020	+ struct vfio_pci_core_device *cur_mem;
	2021	+ struct vfio_pci_core_device *cur_vma;
	2022	+ struct vfio_pci_core_device *cur;
	2023	+ struct pci_dev *pdev;
	2024	+ bool is_mem = true;
	2025	+ int ret;
	2026	+
	2027	+ mutex_lock(&dev_set->lock);
	2028	+ cur_mem = list_first_entry(&dev_set->device_list,
	2029	+ struct vfio_pci_core_device,
	2030	+ vdev.dev_set_list);
	2031	+
	2032	+ pdev = vfio_pci_dev_set_resettable(dev_set);
	2033	+ if (!pdev) {
	2034	+ ret = -EINVAL;
	2035	+ goto err_unlock;
	2036	+ }
	2037	+
	2038	+ list_for_each_entry(cur_vma, &dev_set->device_list, vdev.dev_set_list) {
	2039	+ /*
	2040	+ * Test whether all the affected devices are contained by the
	2041	+ * set of groups provided by the user.
	2042	+ */
	2043	+ if (!vfio_dev_in_groups(cur_vma, groups)) {
	2044	+ ret = -EINVAL;
	2045	+ goto err_undo;
	2046	+ }
	2047	+
	2048	+ /*
	2049	+ * Locking multiple devices is prone to deadlock, runaway and
	2050	+ * unwind if we hit contention.
	2051	+ */
	2052	+ if (!vfio_pci_zap_and_vma_lock(cur_vma, true)) {
	2053	+ ret = -EBUSY;
	2054	+ goto err_undo;
	2055	+ }
	2056	+ }
	2057	+ cur_vma = NULL;
	2058	+
	2059	+ list_for_each_entry(cur_mem, &dev_set->device_list, vdev.dev_set_list) {
	2060	+ if (!down_write_trylock(&cur_mem->memory_lock)) {
	2061	+ ret = -EBUSY;
	2062	+ goto err_undo;
	2063	+ }
	2064	+ mutex_unlock(&cur_mem->vma_lock);
	2065	+ }
	2066	+ cur_mem = NULL;
	2067	+
	2068	+ ret = pci_reset_bus(pdev);
	2069	+
	2070	+err_undo:
	2071	+ list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
	2072	+ if (cur == cur_mem)
	2073	+ is_mem = false;
	2074	+ if (cur == cur_vma)
	2075	+ break;
	2076	+ if (is_mem)
	2077	+ up_write(&cur->memory_lock);
	2078	+ else
	2079	+ mutex_unlock(&cur->vma_lock);
	2080	+ }
	2081	+err_unlock:
	2082	+ mutex_unlock(&dev_set->lock);
	2083	+ return ret;
	2084	+}
	2085	+
	2086	+static bool vfio_pci_dev_set_needs_reset(struct vfio_device_set *dev_set)
	2087	+{
	2088	+ struct vfio_pci_core_device *cur;
	2089	+ bool needs_reset = false;
	2090	+
	2091	+ list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
	2092	+ /* No VFIO device in the set can have an open device FD */
	2093	+ if (cur->vdev.open_count)
	2094	+ return false;
	2095	+ needs_reset \|= cur->needs_reset;
	2096	+ }
	2097	+ return needs_reset;
	2098	+}
	2099	+
	2100	+/*
	2101	+ * If a bus or slot reset is available for the provided dev_set and:
	2102	+ * - All of the devices affected by that bus or slot reset are unused
	2103	+ * - At least one of the affected devices is marked dirty via
	2104	+ * needs_reset (such as by lack of FLR support)
	2105	+ * Then attempt to perform that bus or slot reset.
	2106	+ * Returns true if the dev_set was reset.
	2107	+ */
	2108	+static bool vfio_pci_dev_set_try_reset(struct vfio_device_set *dev_set)
	2109	+{
	2110	+ struct vfio_pci_core_device *cur;
	2111	+ struct pci_dev *pdev;
	2112	+ int ret;
	2113	+
	2114	+ if (!vfio_pci_dev_set_needs_reset(dev_set))
	2115	+ return false;
	2116	+
	2117	+ pdev = vfio_pci_dev_set_resettable(dev_set);
	2118	+ if (!pdev)
	2119	+ return false;
	2120	+
	2121	+ ret = pci_reset_bus(pdev);
	2122	+ if (ret)
	2123	+ return false;
	2124	+
	2125	+ list_for_each_entry(cur, &dev_set->device_list, vdev.dev_set_list) {
	2126	+ cur->needs_reset = false;
	2127	+ if (!disable_idle_d3)
	2128	+ vfio_pci_set_power_state(cur, PCI_D3hot);
	2129	+ }
	2130	+ return true;
	2131	+}
	2132	+
	2133	+void vfio_pci_core_set_params(bool is_nointxmask, bool is_disable_vga,
	2134	+ bool is_disable_idle_d3)
	2135	+{
	2136	+ nointxmask = is_nointxmask;
	2137	+ disable_vga = is_disable_vga;
	2138	+ disable_idle_d3 = is_disable_idle_d3;
	2139	+}
	2140	+EXPORT_SYMBOL_GPL(vfio_pci_core_set_params);
	2141	+
	2142	+static void vfio_pci_core_cleanup(void)
	2143	+{
	2144	+ vfio_pci_uninit_perm_bits();
	2145	+}
	2146	+
	2147	+static int __init vfio_pci_core_init(void)
	2148	+{
	2149	+ /* Allocate shared config space permission data used by all devices */
	2150	+ return vfio_pci_init_perm_bits();
	2151	+}
	2152	+
	2153	+module_init(vfio_pci_core_init);
	2154	+module_exit(vfio_pci_core_cleanup);
	2155	+
	2156	+MODULE_LICENSE("GPL v2");
	2157	+MODULE_AUTHOR(DRIVER_AUTHOR);
	2158	+MODULE_DESCRIPTION(DRIVER_DESC);

drivers/vfio/pci/vfio_pci_igd.c+10−9

@@ -15,7 +15,7 @@ · 14 unmodified lines
15	15	#include <linux/uaccess.h>
16	16	#include <linux/vfio.h>
17	17
18		−#include "vfio_pci_private.h"
	18	+#include <linux/vfio_pci_core.h>
19	19
20	20	#define OPREGION_SIGNATURE "IntelGraphicsMem"
21	21	#define OPREGION_SIZE (8 * 1024)
@@ -25,8 +25,9 @@ · 3 unmodified lines
25	25	#define OPREGION_RVDS 0x3c2
26	26	#define OPREGION_VERSION 0x16
27	27
28		−static ssize_t vfio_pci_igd_rw(struct vfio_pci_device vdev, char __user buf,
29		− size_t count, loff_t *ppos, bool iswrite)
	28	+static ssize_t vfio_pci_igd_rw(struct vfio_pci_core_device *vdev,
	29	+ char __user buf, size_t count, loff_t ppos,
	30	+ bool iswrite)
30	31	{
31	32	unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) - VFIO_PCI_NUM_REGIONS;
32	33	void *base = vdev->region[i].data;
@@ -45,7 +46,7 @@ static ssize_t vfio_pci_igd_rw(struct vfio_pci_device vdev, char __user buf, · 12 unmodified lines
45	46	return count;
46	47	}
47	48
48		−static void vfio_pci_igd_release(struct vfio_pci_device *vdev,
	49	+static void vfio_pci_igd_release(struct vfio_pci_core_device *vdev,
49	50	struct vfio_pci_region *region)
50	51	{
51	52	memunmap(region->data);
@@ -56,7 +57,7 @@ static const struct vfio_pci_regops vfio_pci_igd_regops = { · 4 unmodified lines
56	57	.release = vfio_pci_igd_release,
57	58	};
58	59
59		−static int vfio_pci_igd_opregion_init(struct vfio_pci_device *vdev)
	60	+static int vfio_pci_igd_opregion_init(struct vfio_pci_core_device *vdev)
60	61	{
61	62	__le32 dwordp = (__le32 )(vdev->vconfig + OPREGION_PCI_ADDR);
62	63	u32 addr, size;
@@ -160,7 +161,7 @@ static int vfio_pci_igd_opregion_init(struct vfio_pci_device *vdev) · 97 unmodified lines
160	161	return ret;
161	162	}
162	163
163		−static ssize_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev,
	164	+static ssize_t vfio_pci_igd_cfg_rw(struct vfio_pci_core_device *vdev,
164	165	char __user buf, size_t count, loff_t ppos,
165	166	bool iswrite)
166	167	{
@@ -253,7 +254,7 @@ static ssize_t vfio_pci_igd_cfg_rw(struct vfio_pci_device *vdev, · 86 unmodified lines
253	254	return count;
254	255	}
255	256
256		−static void vfio_pci_igd_cfg_release(struct vfio_pci_device *vdev,
	257	+static void vfio_pci_igd_cfg_release(struct vfio_pci_core_device *vdev,
257	258	struct vfio_pci_region *region)
258	259	{
259	260	struct pci_dev *pdev = region->data;
@@ -266,7 +267,7 @@ static const struct vfio_pci_regops vfio_pci_igd_cfg_regops = { · 6 unmodified lines
266	267	.release = vfio_pci_igd_cfg_release,
267	268	};
268	269
269		−static int vfio_pci_igd_cfg_init(struct vfio_pci_device *vdev)
	270	+static int vfio_pci_igd_cfg_init(struct vfio_pci_core_device *vdev)
270	271	{
271	272	struct pci_dev host_bridge, lpc_bridge;
272	273	int ret;
@@ -314,7 +315,7 @@ static int vfio_pci_igd_cfg_init(struct vfio_pci_device *vdev) · 41 unmodified lines
314	315	return 0;
315	316	}
316	317
317		−int vfio_pci_igd_init(struct vfio_pci_device *vdev)
	318	+int vfio_pci_igd_init(struct vfio_pci_core_device *vdev)
318	319	{
319	320	int ret;
320	321

drivers/vfio/pci/vfio_pci_intrs.c+21−21

@@ -20,20 +20,20 @@ · 19 unmodified lines
20	20	#include <linux/wait.h>
21	21	#include <linux/slab.h>
22	22
23		−#include "vfio_pci_private.h"
	23	+#include <linux/vfio_pci_core.h>
24	24
25	25	/*
26	26	* INTx
27	27	*/
28	28	static void vfio_send_intx_eventfd(void opaque, void unused)
29	29	{
30		− struct vfio_pci_device *vdev = opaque;
	30	+ struct vfio_pci_core_device *vdev = opaque;
31	31
32	32	if (likely(is_intx(vdev) && !vdev->virq_disabled))
33	33	eventfd_signal(vdev->ctx[0].trigger, 1);
34	34	}
35	35
36		−void vfio_pci_intx_mask(struct vfio_pci_device *vdev)
	36	+void vfio_pci_intx_mask(struct vfio_pci_core_device *vdev)
37	37	{
38	38	struct pci_dev *pdev = vdev->pdev;
39	39	unsigned long flags;
@@ -73,7 +73,7 @@ void vfio_pci_intx_mask(struct vfio_pci_device *vdev) · 33 unmodified lines
73	73	*/
74	74	static int vfio_pci_intx_unmask_handler(void opaque, void unused)
75	75	{
76		− struct vfio_pci_device *vdev = opaque;
	76	+ struct vfio_pci_core_device *vdev = opaque;
77	77	struct pci_dev *pdev = vdev->pdev;
78	78	unsigned long flags;
79	79	int ret = 0;
@@ -107,7 +107,7 @@ static int vfio_pci_intx_unmask_handler(void opaque, void unused) · 27 unmodified lines
107	107	return ret;
108	108	}
109	109
110		−void vfio_pci_intx_unmask(struct vfio_pci_device *vdev)
	110	+void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev)
111	111	{
112	112	if (vfio_pci_intx_unmask_handler(vdev, NULL) > 0)
113	113	vfio_send_intx_eventfd(vdev, NULL);
@@ -115,7 +115,7 @@ void vfio_pci_intx_unmask(struct vfio_pci_device *vdev) · 1 unmodified line
115	115
116	116	static irqreturn_t vfio_intx_handler(int irq, void *dev_id)
117	117	{
118		− struct vfio_pci_device *vdev = dev_id;
	118	+ struct vfio_pci_core_device *vdev = dev_id;
119	119	unsigned long flags;
120	120	int ret = IRQ_NONE;
121	121
@@ -139,7 +139,7 @@ static irqreturn_t vfio_intx_handler(int irq, void *dev_id) · 17 unmodified lines
139	139	return ret;
140	140	}
141	141
142		−static int vfio_intx_enable(struct vfio_pci_device *vdev)
	142	+static int vfio_intx_enable(struct vfio_pci_core_device *vdev)
143	143	{
144	144	if (!is_irq_none(vdev))
145	145	return -EINVAL;
@@ -168,7 +168,7 @@ static int vfio_intx_enable(struct vfio_pci_device *vdev) · 22 unmodified lines
168	168	return 0;
169	169	}
170	170
171		−static int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd)
	171	+static int vfio_intx_set_signal(struct vfio_pci_core_device *vdev, int fd)
172	172	{
173	173	struct pci_dev *pdev = vdev->pdev;
174	174	unsigned long irqflags = IRQF_SHARED;
@@ -223,7 +223,7 @@ static int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd) · 48 unmodified lines
223	223	return 0;
224	224	}
225	225
226		−static void vfio_intx_disable(struct vfio_pci_device *vdev)
	226	+static void vfio_intx_disable(struct vfio_pci_core_device *vdev)
227	227	{
228	228	vfio_virqfd_disable(&vdev->ctx[0].unmask);
229	229	vfio_virqfd_disable(&vdev->ctx[0].mask);
@@ -244,7 +244,7 @@ static irqreturn_t vfio_msihandler(int irq, void *arg) · 14 unmodified lines
244	244	return IRQ_HANDLED;
245	245	}
246	246
247		−static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix)
	247	+static int vfio_msi_enable(struct vfio_pci_core_device *vdev, int nvec, bool msix)
248	248	{
249	249	struct pci_dev *pdev = vdev->pdev;
250	250	unsigned int flag = msix ? PCI_IRQ_MSIX : PCI_IRQ_MSI;
@@ -285,7 +285,7 @@ static int vfio_msi_enable(struct vfio_pci_device *vdev, int nvec, bool msix) · 34 unmodified lines
285	285	return 0;
286	286	}
287	287
288		−static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev,
	288	+static int vfio_msi_set_vector_signal(struct vfio_pci_core_device *vdev,
289	289	int vector, int fd, bool msix)
290	290	{
291	291	struct pci_dev *pdev = vdev->pdev;
@@ -364,7 +364,7 @@ static int vfio_msi_set_vector_signal(struct vfio_pci_device *vdev, · 72 unmodified lines
364	364	return 0;
365	365	}
366	366
367		−static int vfio_msi_set_block(struct vfio_pci_device *vdev, unsigned start,
	367	+static int vfio_msi_set_block(struct vfio_pci_core_device *vdev, unsigned start,
368	368	unsigned count, int32_t *fds, bool msix)
369	369	{
370	370	int i, j, ret = 0;
@@ -385,7 +385,7 @@ static int vfio_msi_set_block(struct vfio_pci_device *vdev, unsigned start, · 14 unmodified lines
385	385	return ret;
386	386	}
387	387
388		−static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix)
	388	+static void vfio_msi_disable(struct vfio_pci_core_device *vdev, bool msix)
389	389	{
390	390	struct pci_dev *pdev = vdev->pdev;
391	391	int i;
@@ -417,7 +417,7 @@ static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix) · 25 unmodified lines
417	417	/*
418	418	* IOCTL support
419	419	*/
420		−static int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev,
	420	+static int vfio_pci_set_intx_unmask(struct vfio_pci_core_device *vdev,
421	421	unsigned index, unsigned start,
422	422	unsigned count, uint32_t flags, void *data)
423	423	{
@@ -444,7 +444,7 @@ static int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev, · 20 unmodified lines
444	444	return 0;
445	445	}
446	446
447		−static int vfio_pci_set_intx_mask(struct vfio_pci_device *vdev,
	447	+static int vfio_pci_set_intx_mask(struct vfio_pci_core_device *vdev,
448	448	unsigned index, unsigned start,
449	449	unsigned count, uint32_t flags, void *data)
450	450	{
@@ -464,7 +464,7 @@ static int vfio_pci_set_intx_mask(struct vfio_pci_device *vdev, · 13 unmodified lines
464	464	return 0;
465	465	}
466	466
467		−static int vfio_pci_set_intx_trigger(struct vfio_pci_device *vdev,
	467	+static int vfio_pci_set_intx_trigger(struct vfio_pci_core_device *vdev,
468	468	unsigned index, unsigned start,
469	469	unsigned count, uint32_t flags, void *data)
470	470	{
@@ -507,7 +507,7 @@ static int vfio_pci_set_intx_trigger(struct vfio_pci_device *vdev, · 36 unmodified lines
507	507	return 0;
508	508	}
509	509
510		−static int vfio_pci_set_msi_trigger(struct vfio_pci_device *vdev,
	510	+static int vfio_pci_set_msi_trigger(struct vfio_pci_core_device *vdev,
511	511	unsigned index, unsigned start,
512	512	unsigned count, uint32_t flags, void *data)
513	513	{
@@ -613,7 +613,7 @@ static int vfio_pci_set_ctx_trigger_single(struct eventfd_ctx **ctx, · 99 unmodified lines
613	613	return -EINVAL;
614	614	}
615	615
616		−static int vfio_pci_set_err_trigger(struct vfio_pci_device *vdev,
	616	+static int vfio_pci_set_err_trigger(struct vfio_pci_core_device *vdev,
617	617	unsigned index, unsigned start,
618	618	unsigned count, uint32_t flags, void *data)
619	619	{
@@ -624,7 +624,7 @@ static int vfio_pci_set_err_trigger(struct vfio_pci_device *vdev, · 4 unmodified lines
624	624	count, flags, data);
625	625	}
626	626
627		−static int vfio_pci_set_req_trigger(struct vfio_pci_device *vdev,
	627	+static int vfio_pci_set_req_trigger(struct vfio_pci_core_device *vdev,
628	628	unsigned index, unsigned start,
629	629	unsigned count, uint32_t flags, void *data)
630	630	{
@@ -635,11 +635,11 @@ static int vfio_pci_set_req_trigger(struct vfio_pci_device *vdev, · 4 unmodified lines
635	635	count, flags, data);
636	636	}
637	637
638		−int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev, uint32_t flags,
	638	+int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev, uint32_t flags,
639	639	unsigned index, unsigned start, unsigned count,
640	640	void *data)
641	641	{
642		− int (func)(struct vfio_pci_device vdev, unsigned index,
	642	+ int (func)(struct vfio_pci_core_device vdev, unsigned index,
643	643	unsigned start, unsigned count, uint32_t flags,
644	644	void *data) = NULL;
645	645

drivers/vfio/pci/vfio_pci_rdwr.c+9−9

@@ -17,7 +17,7 @@ · 16 unmodified lines
17	17	#include <linux/vfio.h>
18	18	#include <linux/vgaarb.h>
19	19
20		−#include "vfio_pci_private.h"
	20	+#include <linux/vfio_pci_core.h>
21	21
22	22	#ifdef __LITTLE_ENDIAN
23	23	#define vfio_ioread64 ioread64
@@ -38,7 +38,7 @@ · 14 unmodified lines
38	38	#define vfio_iowrite8 iowrite8
39	39
40	40	#define VFIO_IOWRITE(size) \
41		−static int vfio_pci_iowrite##size(struct vfio_pci_device *vdev, \
	41	+static int vfio_pci_iowrite##size(struct vfio_pci_core_device *vdev, \
42	42	bool test_mem, u##size val, void __iomem *io) \
43	43	{ \
44	44	if (test_mem) { \
@@ -65,7 +65,7 @@ VFIO_IOWRITE(64) · 20 unmodified lines
65	65	#endif
66	66
67	67	#define VFIO_IOREAD(size) \
68		−static int vfio_pci_ioread##size(struct vfio_pci_device *vdev, \
	68	+static int vfio_pci_ioread##size(struct vfio_pci_core_device *vdev, \
69	69	bool test_mem, u##size val, void __iomem io) \
70	70	{ \
71	71	if (test_mem) { \
@@ -94,7 +94,7 @@ VFIO_IOREAD(32) · 22 unmodified lines
94	94	* reads with -1. This is intended for handling MSI-X vector tables and
95	95	* leftover space for ROM BARs.
96	96	*/
97		−static ssize_t do_io_rw(struct vfio_pci_device *vdev, bool test_mem,
	97	+static ssize_t do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem,
98	98	void __iomem io, char __user buf,
99	99	loff_t off, size_t count, size_t x_start,
100	100	size_t x_end, bool iswrite)
@@ -200,7 +200,7 @@ static ssize_t do_io_rw(struct vfio_pci_device *vdev, bool test_mem, · 99 unmodified lines
200	200	return done;
201	201	}
202	202
203		−static int vfio_pci_setup_barmap(struct vfio_pci_device *vdev, int bar)
	203	+static int vfio_pci_setup_barmap(struct vfio_pci_core_device *vdev, int bar)
204	204	{
205	205	struct pci_dev *pdev = vdev->pdev;
206	206	int ret;
@@ -224,7 +224,7 @@ static int vfio_pci_setup_barmap(struct vfio_pci_device *vdev, int bar) · 17 unmodified lines
224	224	return 0;
225	225	}
226	226
227		−ssize_t vfio_pci_bar_rw(struct vfio_pci_device vdev, char __user buf,
	227	+ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device vdev, char __user buf,
228	228	size_t count, loff_t *ppos, bool iswrite)
229	229	{
230	230	struct pci_dev *pdev = vdev->pdev;
@@ -288,7 +288,7 @@ out: · 57 unmodified lines
288	288	return done;
289	289	}
290	290
291		−ssize_t vfio_pci_vga_rw(struct vfio_pci_device vdev, char __user buf,
	291	+ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device vdev, char __user buf,
292	292	size_t count, loff_t *ppos, bool iswrite)
293	293	{
294	294	int ret;
@@ -384,7 +384,7 @@ static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd, · 89 unmodified lines
384	384	static int vfio_pci_ioeventfd_handler(void opaque, void unused)
385	385	{
386	386	struct vfio_pci_ioeventfd *ioeventfd = opaque;
387		− struct vfio_pci_device *vdev = ioeventfd->vdev;
	387	+ struct vfio_pci_core_device *vdev = ioeventfd->vdev;
388	388
389	389	if (ioeventfd->test_mem) {
390	390	if (!down_read_trylock(&vdev->memory_lock))
@@ -410,7 +410,7 @@ static void vfio_pci_ioeventfd_thread(void opaque, void unused) · 19 unmodified lines
410	410	vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem);
411	411	}
412	412
413		−long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
	413	+long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
414	414	uint64_t data, int count, int fd)
415	415	{
416	416	struct pci_dev *pdev = vdev->pdev;

drivers/vfio/pci/vfio_pci_zdev.c+2−2

@@ -14,7 +14,7 @@ · 13 unmodified lines
14	14	#include <asm/pci_clp.h>
15	15	#include <asm/pci_io.h>
16	16
17		−#include "vfio_pci_private.h"
	17	+#include <linux/vfio_pci_core.h>
18	18
19	19	/*
20	20	* Add the Base PCI Function information to the device info region.
@@ -109,7 +109,7 @@ static int zpci_pfip_cap(struct zpci_dev zdev, struct vfio_info_cap caps) · 88 unmodified lines
109	109	/*
110	110	* Add all supported capabilities to the VFIO_DEVICE_GET_INFO capability chain.
111	111	*/
112		−int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev,
	112	+int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
113	113	struct vfio_info_cap *caps)
114	114	{
115	115	struct zpci_dev *zdev = to_zpci(vdev->pdev);

drivers/vfio/platform/Kconfig+4−2

@@ -1,7 +1,7 @@
1	1	# SPDX-License-Identifier: GPL-2.0-only
2	2	config VFIO_PLATFORM
3	3	tristate "VFIO support for platform devices"
4		− depends on VFIO && EVENTFD && (ARM \|\| ARM64 \|\| COMPILE_TEST)
	4	+ depends on ARM \|\| ARM64 \|\| COMPILE_TEST
5	5	select VFIO_VIRQFD
6	6	help
7	7	Support for platform devices with VFIO. This is required to make
@@ -10,9 +10,10 @@ config VFIO_PLATFORM · 2 unmodified lines
10	10
11	11	If you don't know what to do here, say N.
12	12
	13	+if VFIO_PLATFORM
13	14	config VFIO_AMBA
14	15	tristate "VFIO support for AMBA devices"
15		− depends on VFIO_PLATFORM && (ARM_AMBA \|\| COMPILE_TEST)
	16	+ depends on ARM_AMBA \|\| COMPILE_TEST
16	17	help
17	18	Support for ARM AMBA devices with VFIO. This is required to make
18	19	use of ARM AMBA devices present on the system using the VFIO
@@ -21,3 +22,4 @@ config VFIO_AMBA · 2 unmodified lines
21	22	If you don't know what to do here, say N.
22	23
23	24	source "drivers/vfio/platform/reset/Kconfig"
	25	+endif

drivers/vfio/platform/reset/Kconfig+1−3

@@ -1,7 +1,6 @@
1	1	# SPDX-License-Identifier: GPL-2.0-only
2	2	config VFIO_PLATFORM_CALXEDAXGMAC_RESET
3	3	tristate "VFIO support for calxeda xgmac reset"
4		− depends on VFIO_PLATFORM
5	4	help
6	5	Enables the VFIO platform driver to handle reset for Calxeda xgmac
7	6
@@ -9,7 +8,6 @@ config VFIO_PLATFORM_CALXEDAXGMAC_RESET · 1 unmodified line
9	8
10	9	config VFIO_PLATFORM_AMDXGBE_RESET
11	10	tristate "VFIO support for AMD XGBE reset"
12		− depends on VFIO_PLATFORM
13	11	help
14	12	Enables the VFIO platform driver to handle reset for AMD XGBE
15	13
@@ -17,7 +15,7 @@ config VFIO_PLATFORM_AMDXGBE_RESET · 1 unmodified line
17	15
18	16	config VFIO_PLATFORM_BCMFLEXRM_RESET
19	17	tristate "VFIO support for Broadcom FlexRM reset"
20		− depends on VFIO_PLATFORM && (ARCH_BCM_IPROC \|\| COMPILE_TEST)
	18	+ depends on ARCH_BCM_IPROC \|\| COMPILE_TEST
21	19	default ARCH_BCM_IPROC
22	20	help
23	21	Enables the VFIO platform driver to handle reset for Broadcom FlexRM

drivers/vfio/vfio_iommu_type1.c+4−4

@@ -612,17 +612,17 @@ static int vfio_wait(struct vfio_iommu *iommu) · 611 unmodified lines
612	612	static int vfio_find_dma_valid(struct vfio_iommu *iommu, dma_addr_t start,
613	613	size_t size, struct vfio_dma **dma_p)
614	614	{
615		− int ret;
	615	+ int ret = 0;
616	616
617	617	do {
618	618	*dma_p = vfio_find_dma(iommu, start, size);
619	619	if (!*dma_p)
620		− ret = -EINVAL;
	620	+ return -EINVAL;
621	621	else if (!(*dma_p)->vaddr_invalid)
622		− ret = 0;
	622	+ return ret;
623	623	else
624	624	ret = vfio_wait(iommu);
625		− } while (ret > 0);
	625	+ } while (ret == WAITED);
626	626
627	627	return ret;
628	628	}

include/linux/mod_devicetable.h+6−0

@@ -16,6 +16,10 @@ typedef unsigned long kernel_ulong_t; · 15 unmodified lines
16	16
17	17	#define PCI_ANY_ID (~0)
18	18
	19	+enum {
	20	+ PCI_ID_F_VFIO_DRIVER_OVERRIDE = 1,
	21	+};
	22	+
19	23	/**
20	24	* struct pci_device_id - PCI device ID structure
21	25	* @vendor: Vendor ID to match (or PCI_ANY_ID)
@@ -34,12 +38,14 @@ typedef unsigned long kernel_ulong_t; · 12 unmodified lines
34	38	* Best practice is to use driver_data as an index
35	39	* into a static list of equivalent device types,
36	40	* instead of using it as a pointer.
	41	+ * @override_only: Match only when dev->driver_override is this driver.
37	42	*/
38	43	struct pci_device_id {
39	44	__u32 vendor, device; /* Vendor and device ID or PCI_ANY_ID*/
40	45	__u32 subvendor, subdevice; /* Subsystem ID's or PCI_ANY_ID */
41	46	__u32 class, class_mask; /* (class,subclass,prog-if) triplet */
42	47	kernel_ulong_t driver_data; /* Data private to the driver */
	48	+ __u32 override_only;
43	49	};
44	50
45	51

include/linux/pci.h+29−0

@@ -901,6 +901,35 @@ struct pci_driver { · 900 unmodified lines
901	901	.vendor = (vend), .device = (dev), \
902	902	.subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID
903	903
	904	+/**
	905	+ * PCI_DEVICE_DRIVER_OVERRIDE - macro used to describe a PCI device with
	906	+ * override_only flags.
	907	+ * @vend: the 16 bit PCI Vendor ID
	908	+ * @dev: the 16 bit PCI Device ID
	909	+ * @driver_override: the 32 bit PCI Device override_only
	910	+ *
	911	+ * This macro is used to create a struct pci_device_id that matches only a
	912	+ * driver_override device. The subvendor and subdevice fields will be set to
	913	+ * PCI_ANY_ID.
	914	+ */
	915	+#define PCI_DEVICE_DRIVER_OVERRIDE(vend, dev, driver_override) \
	916	+ .vendor = (vend), .device = (dev), .subvendor = PCI_ANY_ID, \
	917	+ .subdevice = PCI_ANY_ID, .override_only = (driver_override)
	918	+
	919	+/**
	920	+ * PCI_DRIVER_OVERRIDE_DEVICE_VFIO - macro used to describe a VFIO
	921	+ * "driver_override" PCI device.
	922	+ * @vend: the 16 bit PCI Vendor ID
	923	+ * @dev: the 16 bit PCI Device ID
	924	+ *
	925	+ * This macro is used to create a struct pci_device_id that matches a
	926	+ * specific device. The subvendor and subdevice fields will be set to
	927	+ * PCI_ANY_ID and the driver_override will be set to
	928	+ * PCI_ID_F_VFIO_DRIVER_OVERRIDE.
	929	+ */
	930	+#define PCI_DRIVER_OVERRIDE_DEVICE_VFIO(vend, dev) \
	931	+ PCI_DEVICE_DRIVER_OVERRIDE(vend, dev, PCI_ID_F_VFIO_DRIVER_OVERRIDE)
	932	+
904	933	/**
905	934	* PCI_DEVICE_SUB - macro used to describe a specific PCI device with subsystem
906	935	* @vend: the 16 bit PCI Vendor ID

include/linux/vfio_pci_core.hrenamed from drivers/vfio/pci/vfio_pci_private.h+60−29

@@ -10,13 +10,14 @@ · 9 unmodified lines
10	10
11	11	#include <linux/mutex.h>
12	12	#include <linux/pci.h>
	13	+#include <linux/vfio.h>
13	14	#include <linux/irqbypass.h>
14	15	#include <linux/types.h>
15	16	#include <linux/uuid.h>
16	17	#include <linux/notifier.h>
17	18
18		−#ifndef VFIO_PCI_PRIVATE_H
19		−#define VFIO_PCI_PRIVATE_H
	19	+#ifndef VFIO_PCI_CORE_H
	20	+#define VFIO_PCI_CORE_H
20	21
21	22	#define VFIO_PCI_OFFSET_SHIFT 40
22	23
@@ -33,7 +34,7 @@ · 10 unmodified lines
33	34
34	35	struct vfio_pci_ioeventfd {
35	36	struct list_head next;
36		− struct vfio_pci_device *vdev;
	37	+ struct vfio_pci_core_device *vdev;
37	38	struct virqfd *virqfd;
38	39	void __iomem *addr;
39	40	uint64_t data;
@@ -52,18 +53,18 @@ struct vfio_pci_irq_ctx { · 12 unmodified lines
52	53	struct irq_bypass_producer producer;
53	54	};
54	55
55		−struct vfio_pci_device;
	56	+struct vfio_pci_core_device;
56	57	struct vfio_pci_region;
57	58
58	59	struct vfio_pci_regops {
59		− ssize_t (rw)(struct vfio_pci_device vdev, char __user *buf,
	60	+ ssize_t (rw)(struct vfio_pci_core_device vdev, char __user *buf,
60	61	size_t count, loff_t *ppos, bool iswrite);
61		− void (release)(struct vfio_pci_device vdev,
	62	+ void (release)(struct vfio_pci_core_device vdev,
62	63	struct vfio_pci_region *region);
63		− int (mmap)(struct vfio_pci_device vdev,
	64	+ int (mmap)(struct vfio_pci_core_device vdev,
64	65	struct vfio_pci_region *region,
65	66	struct vm_area_struct *vma);
66		− int (add_capability)(struct vfio_pci_device vdev,
	67	+ int (add_capability)(struct vfio_pci_core_device vdev,
67	68	struct vfio_pci_region *region,
68	69	struct vfio_info_cap *caps);
69	70	};
@@ -94,7 +95,7 @@ struct vfio_pci_mmap_vma { · 24 unmodified lines
94	95	struct list_head vma_next;
95	96	};
96	97
97		−struct vfio_pci_device {
	98	+struct vfio_pci_core_device {
98	99	struct vfio_device vdev;
99	100	struct pci_dev *pdev;
100	101	void __iomem *barmap[PCI_STD_NUM_BARS];
@@ -144,65 +145,95 @@ struct vfio_pci_device { · 43 unmodified lines
144	145	#define is_irq_none(vdev) (!(is_intx(vdev) \|\| is_msi(vdev) \|\| is_msix(vdev)))
145	146	#define irq_is(vdev, type) (vdev->irq_type == type)
146	147
147		−extern void vfio_pci_intx_mask(struct vfio_pci_device *vdev);
148		−extern void vfio_pci_intx_unmask(struct vfio_pci_device *vdev);
	148	+extern void vfio_pci_intx_mask(struct vfio_pci_core_device *vdev);
	149	+extern void vfio_pci_intx_unmask(struct vfio_pci_core_device *vdev);
149	150
150		−extern int vfio_pci_set_irqs_ioctl(struct vfio_pci_device *vdev,
	151	+extern int vfio_pci_set_irqs_ioctl(struct vfio_pci_core_device *vdev,
151	152	uint32_t flags, unsigned index,
152	153	unsigned start, unsigned count, void *data);
153	154
154		−extern ssize_t vfio_pci_config_rw(struct vfio_pci_device *vdev,
	155	+extern ssize_t vfio_pci_config_rw(struct vfio_pci_core_device *vdev,
155	156	char __user *buf, size_t count,
156	157	loff_t *ppos, bool iswrite);
157	158
158		−extern ssize_t vfio_pci_bar_rw(struct vfio_pci_device vdev, char __user buf,
	159	+extern ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device vdev, char __user buf,
159	160	size_t count, loff_t *ppos, bool iswrite);
160	161
161		−extern ssize_t vfio_pci_vga_rw(struct vfio_pci_device vdev, char __user buf,
	162	+extern ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device vdev, char __user buf,
162	163	size_t count, loff_t *ppos, bool iswrite);
163	164
164		−extern long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset,
	165	+extern long vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset,
165	166	uint64_t data, int count, int fd);
166	167
167	168	extern int vfio_pci_init_perm_bits(void);
168	169	extern void vfio_pci_uninit_perm_bits(void);
169	170
170		−extern int vfio_config_init(struct vfio_pci_device *vdev);
171		−extern void vfio_config_free(struct vfio_pci_device *vdev);
	171	+extern int vfio_config_init(struct vfio_pci_core_device *vdev);
	172	+extern void vfio_config_free(struct vfio_pci_core_device *vdev);
172	173
173		−extern int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
	174	+extern int vfio_pci_register_dev_region(struct vfio_pci_core_device *vdev,
174	175	unsigned int type, unsigned int subtype,
175	176	const struct vfio_pci_regops *ops,
176	177	size_t size, u32 flags, void *data);
177	178
178		−extern int vfio_pci_set_power_state(struct vfio_pci_device *vdev,
	179	+extern int vfio_pci_set_power_state(struct vfio_pci_core_device *vdev,
179	180	pci_power_t state);
180	181
181		−extern bool __vfio_pci_memory_enabled(struct vfio_pci_device *vdev);
182		−extern void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device
	182	+extern bool __vfio_pci_memory_enabled(struct vfio_pci_core_device *vdev);
	183	+extern void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_core_device
183	184	*vdev);
184		−extern u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_device *vdev);
185		−extern void vfio_pci_memory_unlock_and_restore(struct vfio_pci_device *vdev,
	185	+extern u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_core_device *vdev);
	186	+extern void vfio_pci_memory_unlock_and_restore(struct vfio_pci_core_device *vdev,
186	187	u16 cmd);
187	188
188	189	#ifdef CONFIG_VFIO_PCI_IGD
189		−extern int vfio_pci_igd_init(struct vfio_pci_device *vdev);
	190	+extern int vfio_pci_igd_init(struct vfio_pci_core_device *vdev);
190	191	#else
191		−static inline int vfio_pci_igd_init(struct vfio_pci_device *vdev)
	192	+static inline int vfio_pci_igd_init(struct vfio_pci_core_device *vdev)
192	193	{
193	194	return -ENODEV;
194	195	}
195	196	#endif
196	197
197	198	#ifdef CONFIG_S390
198		−extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev,
	199	+extern int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
199	200	struct vfio_info_cap *caps);
200	201	#else
201		−static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_device *vdev,
	202	+static inline int vfio_pci_info_zdev_add_caps(struct vfio_pci_core_device *vdev,
202	203	struct vfio_info_cap *caps)
203	204	{
204	205	return -ENODEV;
205	206	}
206	207	#endif
207	208
208		−#endif /* VFIO_PCI_PRIVATE_H */
	209	+/* Will be exported for vfio pci drivers usage */
	210	+void vfio_pci_core_set_params(bool nointxmask, bool is_disable_vga,
	211	+ bool is_disable_idle_d3);
	212	+void vfio_pci_core_close_device(struct vfio_device *core_vdev);
	213	+void vfio_pci_core_init_device(struct vfio_pci_core_device *vdev,
	214	+ struct pci_dev *pdev,
	215	+ const struct vfio_device_ops *vfio_pci_ops);
	216	+int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev);
	217	+void vfio_pci_core_uninit_device(struct vfio_pci_core_device *vdev);
	218	+void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev);
	219	+int vfio_pci_core_sriov_configure(struct pci_dev *pdev, int nr_virtfn);
	220	+extern const struct pci_error_handlers vfio_pci_core_err_handlers;
	221	+long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd,
	222	+ unsigned long arg);
	223	+ssize_t vfio_pci_core_read(struct vfio_device core_vdev, char __user buf,
	224	+ size_t count, loff_t *ppos);
	225	+ssize_t vfio_pci_core_write(struct vfio_device core_vdev, const char __user buf,
	226	+ size_t count, loff_t *ppos);
	227	+int vfio_pci_core_mmap(struct vfio_device core_vdev, struct vm_area_struct vma);
	228	+void vfio_pci_core_request(struct vfio_device *core_vdev, unsigned int count);
	229	+int vfio_pci_core_match(struct vfio_device core_vdev, char buf);
	230	+int vfio_pci_core_enable(struct vfio_pci_core_device *vdev);
	231	+void vfio_pci_core_disable(struct vfio_pci_core_device *vdev);
	232	+void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev);
	233	+
	234	+static inline bool vfio_pci_is_vga(struct pci_dev *pdev)
	235	+{
	236	+ return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;
	237	+}
	238	+
	239	+#endif /* VFIO_PCI_CORE_H */

scripts/mod/devicetable-offsets.c+1−0

@@ -42,6 +42,7 @@ int main(void) · 41 unmodified lines
42	42	DEVID_FIELD(pci_device_id, subdevice);
43	43	DEVID_FIELD(pci_device_id, class);
44	44	DEVID_FIELD(pci_device_id, class_mask);
	45	+ DEVID_FIELD(pci_device_id, override_only);
45	46
46	47	DEVID(ccw_device_id);
47	48	DEVID_FIELD(ccw_device_id, match_flags);

scripts/mod/file2alias.c+15−2

@@ -426,7 +426,7 @@ static int do_ieee1394_entry(const char *filename, · 425 unmodified lines
426	426	return 1;
427	427	}
428	428
429		−/* Looks like: pci:vNdNsvNsdNbcNscNiN. */
	429	+/* Looks like: pci:vNdNsvNsdNbcNscNiN or <prefix>_pci:vNdNsvNsdNbcNscNiN. */
430	430	static int do_pci_entry(const char *filename,
431	431	void symval, char alias)
432	432	{
@@ -440,8 +440,21 @@ static int do_pci_entry(const char *filename, · 7 unmodified lines
440	440	DEF_FIELD(symval, pci_device_id, subdevice);
441	441	DEF_FIELD(symval, pci_device_id, class);
442	442	DEF_FIELD(symval, pci_device_id, class_mask);
	443	+ DEF_FIELD(symval, pci_device_id, override_only);
	444	+
	445	+ switch (override_only) {
	446	+ case 0:
	447	+ strcpy(alias, "pci:");
	448	+ break;
	449	+ case PCI_ID_F_VFIO_DRIVER_OVERRIDE:
	450	+ strcpy(alias, "vfio_pci:");
	451	+ break;
	452	+ default:
	453	+ warn("Unknown PCI driver_override alias %08X\n",
	454	+ override_only);
	455	+ return 0;
	456	+ }
443	457
444		− strcpy(alias, "pci:");
445	458	ADD(alias, "v", vendor != PCI_ANY_ID, vendor);
446	459	ADD(alias, "d", device != PCI_ANY_ID, device);
447	460	ADD(alias, "sv", subvendor != PCI_ANY_ID, subvendor);

Merge branches 'v5.15/vfio/spdx-license-cleanups', 'v5.15/vfio/dma-valid-waited-v3', 'v5.15/vfio/vfio-pci-core-v5' and 'v5.15/vfio/vfio-ap' into v5.15/vfio/next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code