| @@ -0,0 +1,344 @@ |
| 1 | +// SPDX-License-Identifier: GPL-2.0-only |
| 2 | +/* Copyright (C) 2021 Intel Corporation |
| 3 | + * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES |
| 4 | + * |
| 5 | + * iommufd provides control over the IOMMU HW objects created by IOMMU kernel |
| 6 | + * drivers. IOMMU HW objects revolve around IO page tables that map incoming DMA |
| 7 | + * addresses (IOVA) to CPU addresses. |
| 8 | + */ |
| 9 | +#define pr_fmt(fmt) "iommufd: " fmt |
| 10 | + |
| 11 | +#include <linux/file.h> |
| 12 | +#include <linux/fs.h> |
| 13 | +#include <linux/module.h> |
| 14 | +#include <linux/slab.h> |
| 15 | +#include <linux/miscdevice.h> |
| 16 | +#include <linux/mutex.h> |
| 17 | +#include <linux/bug.h> |
| 18 | +#include <uapi/linux/iommufd.h> |
| 19 | +#include <linux/iommufd.h> |
| 20 | + |
| 21 | +#include "iommufd_private.h" |
| 22 | + |
| 23 | +struct iommufd_object_ops { |
| 24 | + void (*destroy)(struct iommufd_object *obj); |
| 25 | +}; |
| 26 | +static const struct iommufd_object_ops iommufd_object_ops[]; |
| 27 | + |
| 28 | +struct iommufd_object *_iommufd_object_alloc(struct iommufd_ctx *ictx, |
| 29 | + size_t size, |
| 30 | + enum iommufd_object_type type) |
| 31 | +{ |
| 32 | + struct iommufd_object *obj; |
| 33 | + int rc; |
| 34 | + |
| 35 | + obj = kzalloc(size, GFP_KERNEL_ACCOUNT); |
| 36 | + if (!obj) |
| 37 | + return ERR_PTR(-ENOMEM); |
| 38 | + obj->type = type; |
| 39 | + init_rwsem(&obj->destroy_rwsem); |
| 40 | + refcount_set(&obj->users, 1); |
| 41 | + |
| 42 | + /* |
| 43 | + * Reserve an ID in the xarray but do not publish the pointer yet since |
| 44 | + * the caller hasn't initialized it yet. Once the pointer is published |
| 45 | + * in the xarray and visible to other threads we can't reliably destroy |
| 46 | + * it anymore, so the caller must complete all errorable operations |
| 47 | + * before calling iommufd_object_finalize(). |
| 48 | + */ |
| 49 | + rc = xa_alloc(&ictx->objects, &obj->id, XA_ZERO_ENTRY, |
| 50 | + xa_limit_32b, GFP_KERNEL_ACCOUNT); |
| 51 | + if (rc) |
| 52 | + goto out_free; |
| 53 | + return obj; |
| 54 | +out_free: |
| 55 | + kfree(obj); |
| 56 | + return ERR_PTR(rc); |
| 57 | +} |
| 58 | + |
| 59 | +/* |
| 60 | + * Allow concurrent access to the object. |
| 61 | + * |
| 62 | + * Once another thread can see the object pointer it can prevent object |
| 63 | + * destruction. Expect for special kernel-only objects there is no in-kernel way |
| 64 | + * to reliably destroy a single object. Thus all APIs that are creating objects |
| 65 | + * must use iommufd_object_abort() to handle their errors and only call |
| 66 | + * iommufd_object_finalize() once object creation cannot fail. |
| 67 | + */ |
| 68 | +void iommufd_object_finalize(struct iommufd_ctx *ictx, |
| 69 | + struct iommufd_object *obj) |
| 70 | +{ |
| 71 | + void *old; |
| 72 | + |
| 73 | + old = xa_store(&ictx->objects, obj->id, obj, GFP_KERNEL); |
| 74 | + /* obj->id was returned from xa_alloc() so the xa_store() cannot fail */ |
| 75 | + WARN_ON(old); |
| 76 | +} |
| 77 | + |
| 78 | +/* Undo _iommufd_object_alloc() if iommufd_object_finalize() was not called */ |
| 79 | +void iommufd_object_abort(struct iommufd_ctx *ictx, struct iommufd_object *obj) |
| 80 | +{ |
| 81 | + void *old; |
| 82 | + |
| 83 | + old = xa_erase(&ictx->objects, obj->id); |
| 84 | + WARN_ON(old); |
| 85 | + kfree(obj); |
| 86 | +} |
| 87 | + |
| 88 | +/* |
| 89 | + * Abort an object that has been fully initialized and needs destroy, but has |
| 90 | + * not been finalized. |
| 91 | + */ |
| 92 | +void iommufd_object_abort_and_destroy(struct iommufd_ctx *ictx, |
| 93 | + struct iommufd_object *obj) |
| 94 | +{ |
| 95 | + iommufd_object_ops[obj->type].destroy(obj); |
| 96 | + iommufd_object_abort(ictx, obj); |
| 97 | +} |
| 98 | + |
| 99 | +struct iommufd_object *iommufd_get_object(struct iommufd_ctx *ictx, u32 id, |
| 100 | + enum iommufd_object_type type) |
| 101 | +{ |
| 102 | + struct iommufd_object *obj; |
| 103 | + |
| 104 | + xa_lock(&ictx->objects); |
| 105 | + obj = xa_load(&ictx->objects, id); |
| 106 | + if (!obj || (type != IOMMUFD_OBJ_ANY && obj->type != type) || |
| 107 | + !iommufd_lock_obj(obj)) |
| 108 | + obj = ERR_PTR(-ENOENT); |
| 109 | + xa_unlock(&ictx->objects); |
| 110 | + return obj; |
| 111 | +} |
| 112 | + |
| 113 | +/* |
| 114 | + * The caller holds a users refcount and wants to destroy the object. Returns |
| 115 | + * true if the object was destroyed. In all cases the caller no longer has a |
| 116 | + * reference on obj. |
| 117 | + */ |
| 118 | +bool iommufd_object_destroy_user(struct iommufd_ctx *ictx, |
| 119 | + struct iommufd_object *obj) |
| 120 | +{ |
| 121 | + /* |
| 122 | + * The purpose of the destroy_rwsem is to ensure deterministic |
| 123 | + * destruction of objects used by external drivers and destroyed by this |
| 124 | + * function. Any temporary increment of the refcount must hold the read |
| 125 | + * side of this, such as during ioctl execution. |
| 126 | + */ |
| 127 | + down_write(&obj->destroy_rwsem); |
| 128 | + xa_lock(&ictx->objects); |
| 129 | + refcount_dec(&obj->users); |
| 130 | + if (!refcount_dec_if_one(&obj->users)) { |
| 131 | + xa_unlock(&ictx->objects); |
| 132 | + up_write(&obj->destroy_rwsem); |
| 133 | + return false; |
| 134 | + } |
| 135 | + __xa_erase(&ictx->objects, obj->id); |
| 136 | + xa_unlock(&ictx->objects); |
| 137 | + up_write(&obj->destroy_rwsem); |
| 138 | + |
| 139 | + iommufd_object_ops[obj->type].destroy(obj); |
| 140 | + kfree(obj); |
| 141 | + return true; |
| 142 | +} |
| 143 | + |
| 144 | +static int iommufd_destroy(struct iommufd_ucmd *ucmd) |
| 145 | +{ |
| 146 | + struct iommu_destroy *cmd = ucmd->cmd; |
| 147 | + struct iommufd_object *obj; |
| 148 | + |
| 149 | + obj = iommufd_get_object(ucmd->ictx, cmd->id, IOMMUFD_OBJ_ANY); |
| 150 | + if (IS_ERR(obj)) |
| 151 | + return PTR_ERR(obj); |
| 152 | + iommufd_ref_to_users(obj); |
| 153 | + /* See iommufd_ref_to_users() */ |
| 154 | + if (!iommufd_object_destroy_user(ucmd->ictx, obj)) |
| 155 | + return -EBUSY; |
| 156 | + return 0; |
| 157 | +} |
| 158 | + |
| 159 | +static int iommufd_fops_open(struct inode *inode, struct file *filp) |
| 160 | +{ |
| 161 | + struct iommufd_ctx *ictx; |
| 162 | + |
| 163 | + ictx = kzalloc(sizeof(*ictx), GFP_KERNEL_ACCOUNT); |
| 164 | + if (!ictx) |
| 165 | + return -ENOMEM; |
| 166 | + |
| 167 | + xa_init_flags(&ictx->objects, XA_FLAGS_ALLOC1 | XA_FLAGS_ACCOUNT); |
| 168 | + ictx->file = filp; |
| 169 | + filp->private_data = ictx; |
| 170 | + return 0; |
| 171 | +} |
| 172 | + |
| 173 | +static int iommufd_fops_release(struct inode *inode, struct file *filp) |
| 174 | +{ |
| 175 | + struct iommufd_ctx *ictx = filp->private_data; |
| 176 | + struct iommufd_object *obj; |
| 177 | + |
| 178 | + /* |
| 179 | + * The objects in the xarray form a graph of "users" counts, and we have |
| 180 | + * to destroy them in a depth first manner. Leaf objects will reduce the |
| 181 | + * users count of interior objects when they are destroyed. |
| 182 | + * |
| 183 | + * Repeatedly destroying all the "1 users" leaf objects will progress |
| 184 | + * until the entire list is destroyed. If this can't progress then there |
| 185 | + * is some bug related to object refcounting. |
| 186 | + */ |
| 187 | + while (!xa_empty(&ictx->objects)) { |
| 188 | + unsigned int destroyed = 0; |
| 189 | + unsigned long index; |
| 190 | + |
| 191 | + xa_for_each(&ictx->objects, index, obj) { |
| 192 | + if (!refcount_dec_if_one(&obj->users)) |
| 193 | + continue; |
| 194 | + destroyed++; |
| 195 | + xa_erase(&ictx->objects, index); |
| 196 | + iommufd_object_ops[obj->type].destroy(obj); |
| 197 | + kfree(obj); |
| 198 | + } |
| 199 | + /* Bug related to users refcount */ |
| 200 | + if (WARN_ON(!destroyed)) |
| 201 | + break; |
| 202 | + } |
| 203 | + kfree(ictx); |
| 204 | + return 0; |
| 205 | +} |
| 206 | + |
| 207 | +union ucmd_buffer { |
| 208 | + struct iommu_destroy destroy; |
| 209 | +}; |
| 210 | + |
| 211 | +struct iommufd_ioctl_op { |
| 212 | + unsigned int size; |
| 213 | + unsigned int min_size; |
| 214 | + unsigned int ioctl_num; |
| 215 | + int (*execute)(struct iommufd_ucmd *ucmd); |
| 216 | +}; |
| 217 | + |
| 218 | +#define IOCTL_OP(_ioctl, _fn, _struct, _last) \ |
| 219 | + [_IOC_NR(_ioctl) - IOMMUFD_CMD_BASE] = { \ |
| 220 | + .size = sizeof(_struct) + \ |
| 221 | + BUILD_BUG_ON_ZERO(sizeof(union ucmd_buffer) < \ |
| 222 | + sizeof(_struct)), \ |
| 223 | + .min_size = offsetofend(_struct, _last), \ |
| 224 | + .ioctl_num = _ioctl, \ |
| 225 | + .execute = _fn, \ |
| 226 | + } |
| 227 | +static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = { |
| 228 | + IOCTL_OP(IOMMU_DESTROY, iommufd_destroy, struct iommu_destroy, id), |
| 229 | +}; |
| 230 | + |
| 231 | +static long iommufd_fops_ioctl(struct file *filp, unsigned int cmd, |
| 232 | + unsigned long arg) |
| 233 | +{ |
| 234 | + const struct iommufd_ioctl_op *op; |
| 235 | + struct iommufd_ucmd ucmd = {}; |
| 236 | + union ucmd_buffer buf; |
| 237 | + unsigned int nr; |
| 238 | + int ret; |
| 239 | + |
| 240 | + ucmd.ictx = filp->private_data; |
| 241 | + ucmd.ubuffer = (void __user *)arg; |
| 242 | + ret = get_user(ucmd.user_size, (u32 __user *)ucmd.ubuffer); |
| 243 | + if (ret) |
| 244 | + return ret; |
| 245 | + |
| 246 | + nr = _IOC_NR(cmd); |
| 247 | + if (nr < IOMMUFD_CMD_BASE || |
| 248 | + (nr - IOMMUFD_CMD_BASE) >= ARRAY_SIZE(iommufd_ioctl_ops)) |
| 249 | + return -ENOIOCTLCMD; |
| 250 | + op = &iommufd_ioctl_ops[nr - IOMMUFD_CMD_BASE]; |
| 251 | + if (op->ioctl_num != cmd) |
| 252 | + return -ENOIOCTLCMD; |
| 253 | + if (ucmd.user_size < op->min_size) |
| 254 | + return -EINVAL; |
| 255 | + |
| 256 | + ucmd.cmd = &buf; |
| 257 | + ret = copy_struct_from_user(ucmd.cmd, op->size, ucmd.ubuffer, |
| 258 | + ucmd.user_size); |
| 259 | + if (ret) |
| 260 | + return ret; |
| 261 | + ret = op->execute(&ucmd); |
| 262 | + return ret; |
| 263 | +} |
| 264 | + |
| 265 | +static const struct file_operations iommufd_fops = { |
| 266 | + .owner = THIS_MODULE, |
| 267 | + .open = iommufd_fops_open, |
| 268 | + .release = iommufd_fops_release, |
| 269 | + .unlocked_ioctl = iommufd_fops_ioctl, |
| 270 | +}; |
| 271 | + |
| 272 | +/** |
| 273 | + * iommufd_ctx_get - Get a context reference |
| 274 | + * @ictx: Context to get |
| 275 | + * |
| 276 | + * The caller must already hold a valid reference to ictx. |
| 277 | + */ |
| 278 | +void iommufd_ctx_get(struct iommufd_ctx *ictx) |
| 279 | +{ |
| 280 | + get_file(ictx->file); |
| 281 | +} |
| 282 | +EXPORT_SYMBOL_NS_GPL(iommufd_ctx_get, IOMMUFD); |
| 283 | + |
| 284 | +/** |
| 285 | + * iommufd_ctx_from_file - Acquires a reference to the iommufd context |
| 286 | + * @file: File to obtain the reference from |
| 287 | + * |
| 288 | + * Returns a pointer to the iommufd_ctx, otherwise ERR_PTR. The struct file |
| 289 | + * remains owned by the caller and the caller must still do fput. On success |
| 290 | + * the caller is responsible to call iommufd_ctx_put(). |
| 291 | + */ |
| 292 | +struct iommufd_ctx *iommufd_ctx_from_file(struct file *file) |
| 293 | +{ |
| 294 | + struct iommufd_ctx *ictx; |
| 295 | + |
| 296 | + if (file->f_op != &iommufd_fops) |
| 297 | + return ERR_PTR(-EBADFD); |
| 298 | + ictx = file->private_data; |
| 299 | + iommufd_ctx_get(ictx); |
| 300 | + return ictx; |
| 301 | +} |
| 302 | +EXPORT_SYMBOL_NS_GPL(iommufd_ctx_from_file, IOMMUFD); |
| 303 | + |
| 304 | +/** |
| 305 | + * iommufd_ctx_put - Put back a reference |
| 306 | + * @ictx: Context to put back |
| 307 | + */ |
| 308 | +void iommufd_ctx_put(struct iommufd_ctx *ictx) |
| 309 | +{ |
| 310 | + fput(ictx->file); |
| 311 | +} |
| 312 | +EXPORT_SYMBOL_NS_GPL(iommufd_ctx_put, IOMMUFD); |
| 313 | + |
| 314 | +static const struct iommufd_object_ops iommufd_object_ops[] = { |
| 315 | +}; |
| 316 | + |
| 317 | +static struct miscdevice iommu_misc_dev = { |
| 318 | + .minor = MISC_DYNAMIC_MINOR, |
| 319 | + .name = "iommu", |
| 320 | + .fops = &iommufd_fops, |
| 321 | + .nodename = "iommu", |
| 322 | + .mode = 0660, |
| 323 | +}; |
| 324 | + |
| 325 | +static int __init iommufd_init(void) |
| 326 | +{ |
| 327 | + int ret; |
| 328 | + |
| 329 | + ret = misc_register(&iommu_misc_dev); |
| 330 | + if (ret) |
| 331 | + return ret; |
| 332 | + return 0; |
| 333 | +} |
| 334 | + |
| 335 | +static void __exit iommufd_exit(void) |
| 336 | +{ |
| 337 | + misc_deregister(&iommu_misc_dev); |
| 338 | +} |
| 339 | + |
| 340 | +module_init(iommufd_init); |
| 341 | +module_exit(iommufd_exit); |
| 342 | + |
| 343 | +MODULE_DESCRIPTION("I/O Address Space Management for passthrough devices"); |
| 344 | +MODULE_LICENSE("GPL"); |