History

iommu: Add recoverable fault reporting

Some IOMMU hardware features, for example PCI PRI and Arm SMMU Stall, enable recoverable I/O page faults. Allow IOMMU drivers to report PRI Page Requests and Stall events through the new fault reporting API. The consumer of the fault can be either an I/O page fault handler in the host, or a guest OS. Once handled, the fault must be completed by sending a page response back to the IOMMU. Add an iommu_page_response() function to complete a page fault. There are two ways to extend the userspace API: * Add a field to iommu_page_response and a flag to iommu_page_response::flags describing the validity of this field. * Introduce a new iommu_page_response_X structure with a different version number. The kernel must then support both versions.

Jean-Philippe Brucker committed 7 years ago

Commit bf3255b1 parent 0c830e63 files · +146 −2Browse files

Unified · Split

3 files
  1. drivers/iommu/iommu.c +92 −2
  2. include/linux/iommu.h +19 −0
  3. include/uapi/linux/iommu.h +35 −0
drivers/iommu/iommu.c+92−2
@@ -891,7 +891,14 @@ EXPORT_SYMBOL_GPL(iommu_group_unregister_notifier); · 890 unmodified lines
891 * @data: private data passed as argument to the handler891 * @data: private data passed as argument to the handler
892 *892 *
893 * When an IOMMU fault event is received, this handler gets called with the893 * When an IOMMU fault event is received, this handler gets called with the
894 * fault event and data as argument. The handler should return 0 on success.894 * fault event and data as argument. The handler should return 0 on success. If
895 * the fault is recoverable (IOMMU_FAULT_PAGE_REQ), the consumer should also
896 * complete the fault by calling iommu_page_response() with one of the following
897 * response code:
898 * - IOMMU_PAGE_RESP_SUCCESS: retry the translation
899 * - IOMMU_PAGE_RESP_INVALID: terminate the fault
900 * - IOMMU_PAGE_RESP_FAILURE: terminate the fault and stop reporting
901 * page faults if possible.
895 *902 *
896 * Return 0 if the fault handler was installed successfully, or an error.903 * Return 0 if the fault handler was installed successfully, or an error.
897 */904 */
@@ -921,6 +928,8 @@ int iommu_register_device_fault_handler(struct device *dev, · 23 unmodified lines
921 }928 }
922 param->fault_param->handler = handler;929 param->fault_param->handler = handler;
923 param->fault_param->data = data;930 param->fault_param->data = data;
931 mutex_init(&param->fault_param->lock);
932 INIT_LIST_HEAD(&param->fault_param->faults);
924 933
925done_unlock:934done_unlock:
926 mutex_unlock(&param->lock);935 mutex_unlock(&param->lock);
@@ -951,6 +960,12 @@ int iommu_unregister_device_fault_handler(struct device *dev) · 24 unmodified lines
951 if (!param->fault_param)960 if (!param->fault_param)
952 goto unlock;961 goto unlock;
953 962
963 /* we cannot unregister handler if there are pending faults */
964 if (!list_empty(&param->fault_param->faults)) {
965 ret = -EBUSY;
966 goto unlock;
967 }
968
954 kfree(param->fault_param);969 kfree(param->fault_param);
955 param->fault_param = NULL;970 param->fault_param = NULL;
956 put_device(dev);971 put_device(dev);
@@ -967,13 +982,15 @@ EXPORT_SYMBOL_GPL(iommu_unregister_device_fault_handler); · 10 unmodified lines
967 * @evt: fault event data982 * @evt: fault event data
968 *983 *
969 * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ984 * Called by IOMMU drivers when a fault is detected, typically in a threaded IRQ
970 * handler.985 * handler. When this function fails and the fault is recoverable, it is the
986 * caller's responsibility to complete the fault.
971 *987 *
972 * Return 0 on success, or an error.988 * Return 0 on success, or an error.
973 */989 */
974int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt)990int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt)
975{991{
976 struct iommu_param *param = dev->iommu_param;992 struct iommu_param *param = dev->iommu_param;
993 struct iommu_fault_event *evt_pending = NULL;
977 struct iommu_fault_param *fparam;994 struct iommu_fault_param *fparam;
978 int ret = 0;995 int ret = 0;
979 996
@@ -987,13 +1004,86 @@ int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) · 7 unmodified lines
987 ret = -EINVAL;1004 ret = -EINVAL;
988 goto done_unlock;1005 goto done_unlock;
989 }1006 }
1007
1008 if (evt->fault.type == IOMMU_FAULT_PAGE_REQ &&
1009 (evt->fault.prm.flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE)) {
1010 evt_pending = kmemdup(evt, sizeof(struct iommu_fault_event),
1011 GFP_KERNEL);
1012 if (!evt_pending) {
1013 ret = -ENOMEM;
1014 goto done_unlock;
1015 }
1016 mutex_lock(&fparam->lock);
1017 list_add_tail(&evt_pending->list, &fparam->faults);
1018 mutex_unlock(&fparam->lock);
1019 }
1020
990 ret = fparam->handler(&evt->fault, fparam->data);1021 ret = fparam->handler(&evt->fault, fparam->data);
1022 if (ret && evt_pending) {
1023 mutex_lock(&fparam->lock);
1024 list_del(&evt_pending->list);
1025 mutex_unlock(&fparam->lock);
1026 kfree(evt_pending);
1027 }
991done_unlock:1028done_unlock:
992 mutex_unlock(&param->lock);1029 mutex_unlock(&param->lock);
993 return ret;1030 return ret;
994}1031}
995EXPORT_SYMBOL_GPL(iommu_report_device_fault);1032EXPORT_SYMBOL_GPL(iommu_report_device_fault);
996 1033
1034int iommu_page_response(struct device *dev,
1035 struct iommu_page_response *msg)
1036{
1037 bool pasid_valid;
1038 int ret = -EINVAL;
1039 struct iommu_fault_event *evt;
1040 struct iommu_fault_page_request *prm;
1041 struct iommu_param *param = dev->iommu_param;
1042 struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1043
1044 if (!domain || !domain->ops->page_response)
1045 return -ENODEV;
1046
1047 if (!param || !param->fault_param)
1048 return -EINVAL;
1049
1050 if (msg->version != IOMMU_PAGE_RESP_VERSION_1 ||
1051 msg->flags & ~IOMMU_PAGE_RESP_PASID_VALID)
1052 return -EINVAL;
1053
1054 /* Only send response if there is a fault report pending */
1055 mutex_lock(&param->fault_param->lock);
1056 if (list_empty(&param->fault_param->faults)) {
1057 dev_warn_ratelimited(dev, "no pending PRQ, drop response\n");
1058 goto done_unlock;
1059 }
1060 /*
1061 * Check if we have a matching page request pending to respond,
1062 * otherwise return -EINVAL
1063 */
1064 list_for_each_entry(evt, &param->fault_param->faults, list) {
1065 prm = &evt->fault.prm;
1066 pasid_valid = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
1067
1068 if ((pasid_valid && prm->pasid != msg->pasid) ||
1069 prm->grpid != msg->grpid)
1070 continue;
1071
1072 /* Sanitize the reply */
1073 msg->flags = pasid_valid ? IOMMU_PAGE_RESP_PASID_VALID : 0;
1074
1075 ret = domain->ops->page_response(dev, evt, msg);
1076 list_del(&evt->list);
1077 kfree(evt);
1078 break;
1079 }
1080
1081done_unlock:
1082 mutex_unlock(&param->fault_param->lock);
1083 return ret;
1084}
1085EXPORT_SYMBOL_GPL(iommu_page_response);
1086
997/**1087/**
998 * iommu_group_id - Return ID for a group1088 * iommu_group_id - Return ID for a group
999 * @group: the group to ID1089 * @group: the group to ID
include/linux/iommu.h+19−0
@@ -227,6 +227,7 @@ struct iommu_sva_ops { · 226 unmodified lines
227 * @sva_bind: Bind process address space to device227 * @sva_bind: Bind process address space to device
228 * @sva_unbind: Unbind process address space from device228 * @sva_unbind: Unbind process address space from device
229 * @sva_get_pasid: Get PASID associated to a SVA handle229 * @sva_get_pasid: Get PASID associated to a SVA handle
230 * @page_response: handle page request response
230 * @pgsize_bitmap: bitmap of all possible supported page sizes231 * @pgsize_bitmap: bitmap of all possible supported page sizes
231 */232 */
232struct iommu_ops {233struct iommu_ops {
@@ -287,6 +288,10 @@ struct iommu_ops { · 54 unmodified lines
287 void (*sva_unbind)(struct iommu_sva *handle);288 void (*sva_unbind)(struct iommu_sva *handle);
288 int (*sva_get_pasid)(struct iommu_sva *handle);289 int (*sva_get_pasid)(struct iommu_sva *handle);
289 290
291 int (*page_response)(struct device *dev,
292 struct iommu_fault_event *evt,
293 struct iommu_page_response *msg);
294
290 unsigned long pgsize_bitmap;295 unsigned long pgsize_bitmap;
291};296};
292 297
@@ -311,19 +316,25 @@ struct iommu_device { · 18 unmodified lines
311 * unrecoverable faults such as DMA or IRQ remapping faults.316 * unrecoverable faults such as DMA or IRQ remapping faults.
312 *317 *
313 * @fault: fault descriptor318 * @fault: fault descriptor
319 * @list: pending fault event list, used for tracking responses
314 */320 */
315struct iommu_fault_event {321struct iommu_fault_event {
316 struct iommu_fault fault;322 struct iommu_fault fault;
323 struct list_head list;
317};324};
318 325
319/**326/**
320 * struct iommu_fault_param - per-device IOMMU fault data327 * struct iommu_fault_param - per-device IOMMU fault data
321 * @handler: Callback function to handle IOMMU faults at device level328 * @handler: Callback function to handle IOMMU faults at device level
322 * @data: handler private data329 * @data: handler private data
330 * @faults: holds the pending faults which needs response
331 * @lock: protect pending faults list
323 */332 */
324struct iommu_fault_param {333struct iommu_fault_param {
325 iommu_dev_fault_handler_t handler;334 iommu_dev_fault_handler_t handler;
326 void *data;335 void *data;
336 struct list_head faults;
337 struct mutex lock;
327};338};
328 339
329/**340/**
@@ -437,6 +448,8 @@ extern int iommu_unregister_device_fault_handler(struct device *dev); · 107 unmodified lines
437 448
438extern int iommu_report_device_fault(struct device *dev,449extern int iommu_report_device_fault(struct device *dev,
439 struct iommu_fault_event *evt);450 struct iommu_fault_event *evt);
451extern int iommu_page_response(struct device *dev,
452 struct iommu_page_response *msg);
440 453
441extern int iommu_group_id(struct iommu_group *group);454extern int iommu_group_id(struct iommu_group *group);
442extern struct iommu_group *iommu_group_get_for_dev(struct device *dev);455extern struct iommu_group *iommu_group_get_for_dev(struct device *dev);
@@ -765,6 +778,12 @@ int iommu_report_device_fault(struct device *dev, struct iommu_fault_event *evt) · 322 unmodified lines
765 return -ENODEV;778 return -ENODEV;
766}779}
767 780
781static inline int iommu_page_response(struct device *dev,
782 struct iommu_page_response *msg)
783{
784 return -ENODEV;
785}
786
768static inline int iommu_group_id(struct iommu_group *group)787static inline int iommu_group_id(struct iommu_group *group)
769{788{
770 return -ENODEV;789 return -ENODEV;
include/uapi/linux/iommu.h+35−0
@@ -115,4 +115,39 @@ struct iommu_fault { · 114 unmodified lines
115 struct iommu_fault_page_request prm;115 struct iommu_fault_page_request prm;
116 };116 };
117};117};
118
119/**
120 * enum iommu_page_response_code - Return status of fault handlers
121 * @IOMMU_PAGE_RESP_SUCCESS: Fault has been handled and the page tables
122 * populated, retry the access. This is "Success" in PCI PRI.
123 * @IOMMU_PAGE_RESP_FAILURE: General error. Drop all subsequent faults from
124 * this device if possible. This is "Response Failure" in PCI PRI.
125 * @IOMMU_PAGE_RESP_INVALID: Could not handle this fault, don't retry the
126 * access. This is "Invalid Request" in PCI PRI.
127 */
128enum iommu_page_response_code {
129 IOMMU_PAGE_RESP_SUCCESS = 0,
130 IOMMU_PAGE_RESP_INVALID,
131 IOMMU_PAGE_RESP_FAILURE,
132};
133
134/**
135 * struct iommu_page_response - Generic page response information
136 * @version: API version of this structure
137 * @flags: encodes whether the corresponding fields are valid
138 * (IOMMU_FAULT_PAGE_RESPONSE_* values)
139 * @pasid: Process Address Space ID
140 * @grpid: Page Request Group Index
141 * @code: response code from &enum iommu_page_response_code
142 */
143struct iommu_page_response {
144#define IOMMU_PAGE_RESP_VERSION_1 1
145 __u32 version;
146#define IOMMU_PAGE_RESP_PASID_VALID (1 << 0)
147 __u32 flags;
148 __u32 pasid;
149 __u32 grpid;
150 __u32 code;
151};
152
118#endif /* _UAPI_IOMMU_H */153#endif /* _UAPI_IOMMU_H */