History

iommu/vt-d: Replace intel SVM APIs with generic SVA APIs

This patch is an initial step to replace Intel SVM code with the following IOMMU SVA ops: intel_svm_bind_mm() => iommu_sva_bind_device() intel_svm_unbind_mm() => iommu_sva_unbind_device() intel_svm_is_pasid_valid() => iommu_sva_get_pasid() The features below will continue to work but are not included in this patch in that they are handled mostly within the IOMMU subsystem. - IO page fault - mmu notifier Consolidation of the above will come after merging generic IOMMU sva code[1]. There should not be any changes needed for SVA users such as accelerator device drivers during this time. [1] http://jpbrucker.net/sva/

Jacob Pan committed 6 years ago

Commit 064a57d1 parent 76fdd6c4 files · +78 −141Browse files

Unified · Split

4 files
  1. drivers/iommu/intel-iommu.c +3 −0
  2. drivers/iommu/intel-svm.c +69 −55
  3. include/linux/intel-iommu.h +6 −0
  4. include/linux/intel-svm.h +0 −86
drivers/iommu/intel-iommu.c+3−0
@@ -6071,6 +6071,9 @@ const struct iommu_ops intel_iommu_ops = { · 6070 unmodified lines
6071 .cache_invalidate = intel_iommu_sva_invalidate,6071 .cache_invalidate = intel_iommu_sva_invalidate,
6072 .sva_bind_gpasid = intel_svm_bind_gpasid,6072 .sva_bind_gpasid = intel_svm_bind_gpasid,
6073 .sva_unbind_gpasid = intel_svm_unbind_gpasid,6073 .sva_unbind_gpasid = intel_svm_unbind_gpasid,
6074 .sva_bind = intel_svm_bind,
6075 .sva_unbind = intel_svm_unbind,
6076 .sva_get_pasid = intel_svm_get_pasid,
6074#endif6077#endif
6075};6078};
6076 6079
drivers/iommu/intel-svm.c+69−55
@@ -426,13 +426,15 @@ out: · 425 unmodified lines
426 return ret;426 return ret;
427}427}
428 428
429int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ops *ops)429/* Caller must hold pasid_mutex, mm reference */
430static int
431intel_svm_bind_mm(struct device *dev, int flags, struct svm_dev_ops *ops,
432 struct mm_struct *mm, struct intel_svm_dev **sd)
430{433{
431 struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);434 struct intel_iommu *iommu = intel_svm_device_to_iommu(dev);
432 struct device_domain_info *info;435 struct device_domain_info *info;
433 struct intel_svm_dev *sdev;436 struct intel_svm_dev *sdev;
434 struct intel_svm *svm = NULL;437 struct intel_svm *svm = NULL;
435 struct mm_struct *mm = NULL;
436 int pasid_max;438 int pasid_max;
437 int ret;439 int ret;
438 440
@@ -449,16 +451,15 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ · 10 unmodified lines
449 } else451 } else
450 pasid_max = 1 << 20;452 pasid_max = 1 << 20;
451 453
454 /* Bind supervisor PASID shuld have mm = NULL */
452 if (flags & SVM_FLAG_SUPERVISOR_MODE) {455 if (flags & SVM_FLAG_SUPERVISOR_MODE) {
453 if (!ecap_srs(iommu->ecap))456 if (!ecap_srs(iommu->ecap) || mm) {
457 pr_err("Supervisor PASID with user provided mm.\n");
454 return -EINVAL;458 return -EINVAL;
455 } else if (pasid) {459 }
456 mm = get_task_mm(current);
457 BUG_ON(!mm);
458 }460 }
459 461
460 mutex_lock(&pasid_mutex);462 if (!(flags & SVM_FLAG_PRIVATE_PASID)) {
461 if (pasid && !(flags & SVM_FLAG_PRIVATE_PASID)) {
462 struct intel_svm *t;463 struct intel_svm *t;
463 464
464 list_for_each_entry(t, &global_svm_list, list) {465 list_for_each_entry(t, &global_svm_list, list) {
@@ -496,9 +497,7 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ · 31 unmodified lines
496 sdev->dev = dev;497 sdev->dev = dev;
497 498
498 ret = intel_iommu_enable_pasid(iommu, dev);499 ret = intel_iommu_enable_pasid(iommu, dev);
499 if (ret || !pasid) {500 if (ret) {
500 /* If they don't actually want to assign a PASID, this is
501 * just an enabling check/preparation. */
502 kfree(sdev);501 kfree(sdev);
503 goto out;502 goto out;
504 }503 }
@@ -597,18 +596,17 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ · 92 unmodified lines
597 }596 }
598 }597 }
599 list_add_rcu(&sdev->list, &svm->devs);598 list_add_rcu(&sdev->list, &svm->devs);
600 599success:
601 success:600 sdev->pasid = svm->pasid;
602 *pasid = svm->pasid;601 sdev->sva.dev = dev;
602 if (sd)
603 *sd = sdev;
603 ret = 0;604 ret = 0;
604 out:605 out:
605 mutex_unlock(&pasid_mutex);
606 if (mm)
607 mmput(mm);
608 return ret;606 return ret;
609}607}
610EXPORT_SYMBOL_GPL(intel_svm_bind_mm);
611 608
609/* Caller must hold pasid_mutex */
612int intel_svm_unbind_mm(struct device *dev, int pasid)610int intel_svm_unbind_mm(struct device *dev, int pasid)
613{611{
614 struct intel_svm_dev *sdev;612 struct intel_svm_dev *sdev;
@@ -616,7 +614,6 @@ int intel_svm_unbind_mm(struct device *dev, int pasid) · 1 unmodified line
616 struct intel_svm *svm;614 struct intel_svm *svm;
617 int ret = -EINVAL;615 int ret = -EINVAL;
618 616
619 mutex_lock(&pasid_mutex);
620 iommu = intel_svm_device_to_iommu(dev);617 iommu = intel_svm_device_to_iommu(dev);
621 if (!iommu)618 if (!iommu)
622 goto out;619 goto out;
@@ -662,45 +659,9 @@ int intel_svm_unbind_mm(struct device *dev, int pasid) · 39 unmodified lines
662 break;659 break;
663 }660 }
664 out:661 out:
665 mutex_unlock(&pasid_mutex);
666 662
667 return ret;663 return ret;
668}664}
669EXPORT_SYMBOL_GPL(intel_svm_unbind_mm);
670
671int intel_svm_is_pasid_valid(struct device *dev, int pasid)
672{
673 struct intel_iommu *iommu;
674 struct intel_svm *svm;
675 int ret = -EINVAL;
676
677 mutex_lock(&pasid_mutex);
678 iommu = intel_svm_device_to_iommu(dev);
679 if (!iommu)
680 goto out;
681
682 svm = ioasid_find(NULL, pasid, NULL);
683 if (!svm)
684 goto out;
685
686 if (IS_ERR(svm)) {
687 ret = PTR_ERR(svm);
688 goto out;
689 }
690 /* init_mm is used in this case */
691 if (!svm->mm)
692 ret = 1;
693 else if (atomic_read(&svm->mm->mm_users) > 0)
694 ret = 1;
695 else
696 ret = 0;
697
698 out:
699 mutex_unlock(&pasid_mutex);
700
701 return ret;
702}
703EXPORT_SYMBOL_GPL(intel_svm_is_pasid_valid);
704 665
705/* Page request queue descriptor */666/* Page request queue descriptor */
706struct page_req_dsc {667struct page_req_dsc {
@@ -894,3 +855,56 @@ static irqreturn_t prq_event_thread(int irq, void *d) · 187 unmodified lines
894 855
895 return IRQ_RETVAL(handled);856 return IRQ_RETVAL(handled);
896}857}
858
859#define to_intel_svm_dev(handle) container_of(handle, struct intel_svm_dev, sva)
860struct iommu_sva *
861intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata)
862{
863 struct iommu_sva *sva = ERR_PTR(-EINVAL);
864 struct intel_svm_dev *sdev = NULL;
865 int flags = 0;
866 int ret;
867
868 /*
869 * TODO: Consolidate with generic iommu-sva bind after it is merged.
870 * It will require shared SVM data structures, i.e. combine io_mm
871 * and intel_svm etc.
872 */
873 if (drvdata)
874 flags = *(int *)drvdata;
875 mutex_lock(&pasid_mutex);
876 ret = intel_svm_bind_mm(dev, flags, NULL, mm, &sdev);
877 if (ret)
878 sva = ERR_PTR(ret);
879 else if (sdev)
880 sva = &sdev->sva;
881 else
882 WARN(!sdev, "SVM bind succeeded with no sdev!\n");
883
884 mutex_unlock(&pasid_mutex);
885
886 return sva;
887}
888
889void intel_svm_unbind(struct iommu_sva *sva)
890{
891 struct intel_svm_dev *sdev;
892
893 mutex_lock(&pasid_mutex);
894 sdev = to_intel_svm_dev(sva);
895 intel_svm_unbind_mm(sdev->dev, sdev->pasid);
896 mutex_unlock(&pasid_mutex);
897}
898
899int intel_svm_get_pasid(struct iommu_sva *sva)
900{
901 struct intel_svm_dev *sdev;
902 int pasid;
903
904 mutex_lock(&pasid_mutex);
905 sdev = to_intel_svm_dev(sva);
906 pasid = sdev->pasid;
907 mutex_unlock(&pasid_mutex);
908
909 return pasid;
910}
include/linux/intel-iommu.h+6−0
@@ -723,6 +723,10 @@ extern int intel_svm_finish_prq(struct intel_iommu *iommu); · 722 unmodified lines
723int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev,723int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev,
724 struct iommu_gpasid_bind_data *data);724 struct iommu_gpasid_bind_data *data);
725int intel_svm_unbind_gpasid(struct device *dev, int pasid);725int intel_svm_unbind_gpasid(struct device *dev, int pasid);
726struct iommu_sva *intel_svm_bind(struct device *dev, struct mm_struct *mm,
727 void *drvdata);
728void intel_svm_unbind(struct iommu_sva *handle);
729int intel_svm_get_pasid(struct iommu_sva *handle);
726struct svm_dev_ops;730struct svm_dev_ops;
727 731
728struct intel_svm_dev {732struct intel_svm_dev {
@@ -730,6 +734,8 @@ struct intel_svm_dev { · 1 unmodified line
730 struct rcu_head rcu;734 struct rcu_head rcu;
731 struct device *dev;735 struct device *dev;
732 struct svm_dev_ops *ops;736 struct svm_dev_ops *ops;
737 struct iommu_sva sva;
738 int pasid;
733 int users;739 int users;
734 u16 did;740 u16 did;
735 u16 dev_iotlb:1;741 u16 dev_iotlb:1;
include/linux/intel-svm.h+0−86
@@ -21,7 +21,6 @@ struct svm_dev_ops { · 20 unmodified lines
21#define SVM_REQ_EXEC (1<<1)21#define SVM_REQ_EXEC (1<<1)
22#define SVM_REQ_PRIV (1<<0)22#define SVM_REQ_PRIV (1<<0)
23 23
24
25/*24/*
26 * The SVM_FLAG_PRIVATE_PASID flag requests a PASID which is *not* the "main"25 * The SVM_FLAG_PRIVATE_PASID flag requests a PASID which is *not* the "main"
27 * PASID for the current process. Even if a PASID already exists, a new one26 * PASID for the current process. Even if a PASID already exists, a new one
@@ -57,89 +56,4 @@ struct svm_dev_ops { · 29 unmodified lines
57 */56 */
58#define SVM_FLAG_GUEST_PASID (1<<3)57#define SVM_FLAG_GUEST_PASID (1<<3)
59 58
60#ifdef CONFIG_INTEL_IOMMU_SVM
61
62/**
63 * intel_svm_bind_mm() - Bind the current process to a PASID
64 * @dev: Device to be granted access
65 * @pasid: Address for allocated PASID
66 * @flags: Flags. Later for requesting supervisor mode, etc.
67 * @ops: Callbacks to device driver
68 *
69 * This function attempts to enable PASID support for the given device.
70 * If the @pasid argument is non-%NULL, a PASID is allocated for access
71 * to the MM of the current process.
72 *
73 * By using a %NULL value for the @pasid argument, this function can
74 * be used to simply validate that PASID support is available for the
75 * given device — i.e. that it is behind an IOMMU which has the
76 * requisite support, and is enabled.
77 *
78 * Page faults are handled transparently by the IOMMU code, and there
79 * should be no need for the device driver to be involved. If a page
80 * fault cannot be handled (i.e. is an invalid address rather than
81 * just needs paging in), then the page request will be completed by
82 * the core IOMMU code with appropriate status, and the device itself
83 * can then report the resulting fault to its driver via whatever
84 * mechanism is appropriate.
85 *
86 * Multiple calls from the same process may result in the same PASID
87 * being re-used. A reference count is kept.
88 */
89extern int intel_svm_bind_mm(struct device *dev, int *pasid, int flags,
90 struct svm_dev_ops *ops);
91
92/**
93 * intel_svm_unbind_mm() - Unbind a specified PASID
94 * @dev: Device for which PASID was allocated
95 * @pasid: PASID value to be unbound
96 *
97 * This function allows a PASID to be retired when the device no
98 * longer requires access to the address space of a given process.
99 *
100 * If the use count for the PASID in question reaches zero, the
101 * PASID is revoked and may no longer be used by hardware.
102 *
103 * Device drivers are required to ensure that no access (including
104 * page requests) is currently outstanding for the PASID in question,
105 * before calling this function.
106 */
107extern int intel_svm_unbind_mm(struct device *dev, int pasid);
108
109/**
110 * intel_svm_is_pasid_valid() - check if pasid is valid
111 * @dev: Device for which PASID was allocated
112 * @pasid: PASID value to be checked
113 *
114 * This function checks if the specified pasid is still valid. A
115 * valid pasid means the backing mm is still having a valid user.
116 * For kernel callers init_mm is always valid. for other mm, if mm->mm_users
117 * is non-zero, it is valid.
118 *
119 * returns -EINVAL if invalid pasid, 0 if pasid ref count is invalid
120 * 1 if pasid is valid.
121 */
122extern int intel_svm_is_pasid_valid(struct device *dev, int pasid);
123
124#else /* CONFIG_INTEL_IOMMU_SVM */
125
126static inline int intel_svm_bind_mm(struct device *dev, int *pasid,
127 int flags, struct svm_dev_ops *ops)
128{
129 return -ENOSYS;
130}
131
132static inline int intel_svm_unbind_mm(struct device *dev, int pasid)
133{
134 BUG();
135}
136
137static inline int intel_svm_is_pasid_valid(struct device *dev, int pasid)
138{
139 return -EINVAL;
140}
141#endif /* CONFIG_INTEL_IOMMU_SVM */
142
143#define intel_svm_available(dev) (!intel_svm_bind_mm((dev), NULL, 0, NULL))
144
145#endif /* __INTEL_SVM_H__ */59#endif /* __INTEL_SVM_H__ */