History

drm/msm: Add VM_BIND ioctl

Add a VM_BIND ioctl for binding/unbinding buffers into a VM. This is only supported if userspace has opted in to MSM_PARAM_EN_VM_BIND.

Rob Clark committed 1 year ago

Commit 2e6a8a11 parent ecfd9fa7 files · +1205 −33Browse files

Unified · Split

7 files
  1. drivers/gpu/drm/msm/msm_drv.c +1 −0
  2. drivers/gpu/drm/msm/msm_drv.h +3 −1
  3. drivers/gpu/drm/msm/msm_gem.c +29 −11
  4. drivers/gpu/drm/msm/msm_gem.h +4 −0
  5. drivers/gpu/drm/msm/msm_gem_submit.c +20 −2
  6. drivers/gpu/drm/msm/msm_gem_vma.c +1075 −18
  7. include/uapi/drm/msm_drm.h +73 −1
drivers/gpu/drm/msm/msm_drv.c+1−0
@@ -802,6 +802,7 @@ static const struct drm_ioctl_desc msm_ioctls[] = { · 801 unmodified lines
802802 DRM_IOCTL_DEF_DRV(MSM_SUBMITQUEUE_NEW, msm_ioctl_submitqueue_new, DRM_RENDER_ALLOW),
803803 DRM_IOCTL_DEF_DRV(MSM_SUBMITQUEUE_CLOSE, msm_ioctl_submitqueue_close, DRM_RENDER_ALLOW),
804804 DRM_IOCTL_DEF_DRV(MSM_SUBMITQUEUE_QUERY, msm_ioctl_submitqueue_query, DRM_RENDER_ALLOW),
805+ DRM_IOCTL_DEF_DRV(MSM_VM_BIND, msm_ioctl_vm_bind, DRM_RENDER_ALLOW),
805806 };
806807
807808 static void msm_show_fdinfo(struct drm_printer *p, struct drm_file *file)
drivers/gpu/drm/msm/msm_drv.h+3−1
@@ -255,7 +255,9 @@ struct drm_gpuvm *msm_kms_init_vm(struct drm_device *dev); · 254 unmodified lines
255255 bool msm_use_mmu(struct drm_device *dev);
256256
257257 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
258− struct drm_file *file);
258+ struct drm_file *file);
259+int msm_ioctl_vm_bind(struct drm_device *dev, void *data,
260+ struct drm_file *file);
259261
260262 #ifdef CONFIG_DEBUG_FS
261263 unsigned long msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan);
drivers/gpu/drm/msm/msm_gem.c+29−11
@@ -251,8 +251,7 @@ static void put_pages(struct drm_gem_object *obj) · 250 unmodified lines
251251 }
252252 }
253253
254−static struct page **msm_gem_get_pages_locked(struct drm_gem_object *obj,
255− unsigned madv)
254+struct page **msm_gem_get_pages_locked(struct drm_gem_object *obj, unsigned madv)
256255 {
257256 struct msm_gem_object *msm_obj = to_msm_bo(obj);
258257
@@ -1052,18 +1051,37 @@ static void msm_gem_free_object(struct drm_gem_object *obj) · 793 unmodified lines
10521051 /*
10531052 * We need to lock any VMs the object is still attached to, but not
10541053 * the object itself (see explaination in msm_gem_assert_locked()),
1055− * so just open-code this special case:
1054+ * so just open-code this special case.
1055+ *
1056+ * Note that we skip the dance if we aren't attached to any VM. This
1057+ * is load bearing. The driver needs to support two usage models:
1058+ *
1059+ * 1. Legacy kernel managed VM: Userspace expects the VMA's to be
1060+ * implicitly torn down when the object is freed, the VMA's do
1061+ * not hold a hard reference to the BO.
1062+ *
1063+ * 2. VM_BIND, userspace managed VM: The VMA holds a reference to the
1064+ * BO. This can be dropped when the VM is closed and it's associated
1065+ * VMAs are torn down. (See msm_gem_vm_close()).
1066+ *
1067+ * In the latter case the last reference to a BO can be dropped while
1068+ * we already have the VM locked. It would have already been removed
1069+ * from the gpuva list, but lockdep doesn't know that. Or understand
1070+ * the differences between the two usage models.
10561071 */
1057− drm_exec_init(&exec, 0, 0);
1058− drm_exec_until_all_locked (&exec) {
1059− struct drm_gpuvm_bo *vm_bo;
1060− drm_gem_for_each_gpuvm_bo (vm_bo, obj) {
1061− drm_exec_lock_obj(&exec, drm_gpuvm_resv_obj(vm_bo->vm));
1062− drm_exec_retry_on_contention(&exec);
1072+ if (!list_empty(&obj->gpuva.list)) {
1073+ drm_exec_init(&exec, 0, 0);
1074+ drm_exec_until_all_locked (&exec) {
1075+ struct drm_gpuvm_bo *vm_bo;
1076+ drm_gem_for_each_gpuvm_bo (vm_bo, obj) {
1077+ drm_exec_lock_obj(&exec,
1078+ drm_gpuvm_resv_obj(vm_bo->vm));
1079+ drm_exec_retry_on_contention(&exec);
1080+ }
10631081 }
1082+ put_iova_spaces(obj, NULL, true);
1083+ drm_exec_fini(&exec); /* drop locks */
10641084 }
1065− put_iova_spaces(obj, NULL, true);
1066− drm_exec_fini(&exec); /* drop locks */
10671085
10681086 if (drm_gem_is_imported(obj)) {
10691087 GEM_WARN_ON(msm_obj->vaddr);
drivers/gpu/drm/msm/msm_gem.h+4−0
@@ -73,6 +73,9 @@ struct msm_gem_vm { · 72 unmodified lines
7373 /** @mmu: The mmu object which manages the pgtables */
7474 struct msm_mmu *mmu;
7575
76+ /** @mmu_lock: Protects access to the mmu */
77+ struct mutex mmu_lock;
78+
7679 /**
7780 * @pid: For address spaces associated with a specific process, this
7881 * will be non-NULL:
@@ -205,6 +208,7 @@ int msm_gem_get_and_pin_iova(struct drm_gem_object *obj, struct drm_gpuvm *vm, · 126 unmodified lines
205208 uint64_t *iova);
206209 void msm_gem_unpin_iova(struct drm_gem_object *obj, struct drm_gpuvm *vm);
207210 void msm_gem_pin_obj_locked(struct drm_gem_object *obj);
211+struct page **msm_gem_get_pages_locked(struct drm_gem_object *obj, unsigned madv);
208212 struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj);
209213 void msm_gem_unpin_pages_locked(struct drm_gem_object *obj);
210214 int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
drivers/gpu/drm/msm/msm_gem_submit.c+20−2
@@ -194,6 +194,7 @@ out: · 193 unmodified lines
194194 static int submit_lookup_cmds(struct msm_gem_submit *submit,
195195 struct drm_msm_gem_submit *args, struct drm_file *file)
196196 {
197+ struct msm_context *ctx = file->driver_priv;
197198 unsigned i;
198199 size_t sz;
199200 int ret = 0;
@@ -225,6 +226,20 @@ static int submit_lookup_cmds(struct msm_gem_submit *submit, · 25 unmodified lines
225226 goto out;
226227 }
227228
229+ if (msm_context_is_vmbind(ctx)) {
230+ if (submit_cmd.nr_relocs) {
231+ ret = SUBMIT_ERROR(EINVAL, submit, "nr_relocs must be zero");
232+ goto out;
233+ }
234+
235+ if (submit_cmd.submit_idx || submit_cmd.submit_offset) {
236+ ret = SUBMIT_ERROR(EINVAL, submit, "submit_idx/offset must be zero");
237+ goto out;
238+ }
239+
240+ submit->cmd[i].iova = submit_cmd.iova;
241+ }
242+
228243 submit->cmd[i].type = submit_cmd.type;
229244 submit->cmd[i].size = submit_cmd.size / 4;
230245 submit->cmd[i].offset = submit_cmd.submit_offset / 4;
@@ -537,6 +552,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, · 306 unmodified lines
537552 struct msm_syncobj_post_dep *post_deps = NULL;
538553 struct drm_syncobj **syncobjs_to_reset = NULL;
539554 struct sync_file *sync_file = NULL;
555+ unsigned cmds_to_parse;
540556 int out_fence_fd = -1;
541557 unsigned i;
542558 int ret;
@@ -661,7 +677,9 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, · 118 unmodified lines
661677 if (ret)
662678 goto out;
663679
664− for (i = 0; i < args->nr_cmds; i++) {
680+ cmds_to_parse = msm_context_is_vmbind(ctx) ? 0 : args->nr_cmds;
681+
682+ for (i = 0; i < cmds_to_parse; i++) {
665683 struct drm_gem_object *obj;
666684 uint64_t iova;
667685
@@ -692,7 +710,7 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data, · 24 unmodified lines
692710 goto out;
693711 }
694712
695− submit->nr_cmds = i;
713+ submit->nr_cmds = args->nr_cmds;
696714
697715 idr_preload(GFP_KERNEL);
698716
drivers/gpu/drm/msm/msm_gem_vma.c+1075−18
@@ -4,9 +4,16 @@ · 3 unmodified lines
44 * Author: Rob Clark <robdclark@gmail.com>
55 */
66
7+#include "drm/drm_file.h"
8+#include "drm/msm_drm.h"
9+#include "linux/file.h"
10+#include "linux/sync_file.h"
11+
712 #include "msm_drv.h"
813 #include "msm_gem.h"
14+#include "msm_gpu.h"
915 #include "msm_mmu.h"
16+#include "msm_syncobj.h"
1017
1118 #define vm_dbg(fmt, ...) pr_debug("%s:%d: "fmt"\n", __func__, __LINE__, ##__VA_ARGS__)
1219
@@ -36,6 +43,97 @@ struct msm_vm_unmap_op { · 23 unmodified lines
3643 uint64_t range;
3744 };
3845
46+/**
47+ * struct msm_vma_op - A MAP or UNMAP operation
48+ */
49+struct msm_vm_op {
50+ /** @op: The operation type */
51+ enum {
52+ MSM_VM_OP_MAP = 1,
53+ MSM_VM_OP_UNMAP,
54+ } op;
55+ union {
56+ /** @map: Parameters used if op == MSM_VMA_OP_MAP */
57+ struct msm_vm_map_op map;
58+ /** @unmap: Parameters used if op == MSM_VMA_OP_UNMAP */
59+ struct msm_vm_unmap_op unmap;
60+ };
61+ /** @node: list head in msm_vm_bind_job::vm_ops */
62+ struct list_head node;
63+
64+ /**
65+ * @obj: backing object for pages to be mapped/unmapped
66+ *
67+ * Async unmap ops, in particular, must hold a reference to the
68+ * original GEM object backing the mapping that will be unmapped.
69+ * But the same can be required in the map path, for example if
70+ * there is not a corresponding unmap op, such as process exit.
71+ *
72+ * This ensures that the pages backing the mapping are not freed
73+ * before the mapping is torn down.
74+ */
75+ struct drm_gem_object *obj;
76+};
77+
78+/**
79+ * struct msm_vm_bind_job - Tracking for a VM_BIND ioctl
80+ *
81+ * A table of userspace requested VM updates (MSM_VM_BIND_OP_UNMAP/MAP/MAP_NULL)
82+ * gets applied to the vm, generating a list of VM ops (MSM_VM_OP_MAP/UNMAP)
83+ * which are applied to the pgtables asynchronously. For example a userspace
84+ * requested MSM_VM_BIND_OP_MAP could end up generating both an MSM_VM_OP_UNMAP
85+ * to unmap an existing mapping, and a MSM_VM_OP_MAP to apply the new mapping.
86+ */
87+struct msm_vm_bind_job {
88+ /** @base: base class for drm_sched jobs */
89+ struct drm_sched_job base;
90+ /** @vm: The VM being operated on */
91+ struct drm_gpuvm *vm;
92+ /** @fence: The fence that is signaled when job completes */
93+ struct dma_fence *fence;
94+ /** @queue: The queue that the job runs on */
95+ struct msm_gpu_submitqueue *queue;
96+ /** @prealloc: Tracking for pre-allocated MMU pgtable pages */
97+ struct msm_mmu_prealloc prealloc;
98+ /** @vm_ops: a list of struct msm_vm_op */
99+ struct list_head vm_ops;
100+ /** @bos_pinned: are the GEM objects being bound pinned? */
101+ bool bos_pinned;
102+ /** @nr_ops: the number of userspace requested ops */
103+ unsigned int nr_ops;
104+ /**
105+ * @ops: the userspace requested ops
106+ *
107+ * The userspace requested ops are copied/parsed and validated
108+ * before we start applying the updates to try to do as much up-
109+ * front error checking as possible, to avoid the VM being in an
110+ * undefined state due to partially executed VM_BIND.
111+ *
112+ * This table also serves to hold a reference to the backing GEM
113+ * objects.
114+ */
115+ struct msm_vm_bind_op {
116+ uint32_t op;
117+ uint32_t flags;
118+ union {
119+ struct drm_gem_object *obj;
120+ uint32_t handle;
121+ };
122+ uint64_t obj_offset;
123+ uint64_t iova;
124+ uint64_t range;
125+ } ops[];
126+};
127+
128+#define job_foreach_bo(obj, _job) \
129+ for (unsigned i = 0; i < (_job)->nr_ops; i++) \
130+ if ((obj = (_job)->ops[i].obj))
131+
132+static inline struct msm_vm_bind_job *to_msm_vm_bind_job(struct drm_sched_job *job)
133+{
134+ return container_of(job, struct msm_vm_bind_job, base);
135+}
136+
39137 static void
40138 msm_gem_vm_free(struct drm_gpuvm *gpuvm)
41139 {
@@ -52,6 +150,9 @@ msm_gem_vm_free(struct drm_gpuvm *gpuvm) · 10 unmodified lines
52150 static void
53151 vm_unmap_op(struct msm_gem_vm *vm, const struct msm_vm_unmap_op *op)
54152 {
153+ if (!vm->managed)
154+ lockdep_assert_held(&vm->mmu_lock);
155+
55156 vm_dbg("%p: %016llx %016llx", vm, op->iova, op->iova + op->range);
56157
57158 vm->mmu->funcs->unmap(vm->mmu, op->iova, op->range);
@@ -60,6 +161,9 @@ vm_unmap_op(struct msm_gem_vm *vm, const struct msm_vm_unmap_op *op) · 2 unmodified lines
60161 static int
61162 vm_map_op(struct msm_gem_vm *vm, const struct msm_vm_map_op *op)
62163 {
164+ if (!vm->managed)
165+ lockdep_assert_held(&vm->mmu_lock);
166+
63167 vm_dbg("%p: %016llx %016llx", vm, op->iova, op->iova + op->range);
64168
65169 return vm->mmu->funcs->map(vm->mmu, op->iova, op->sgt, op->offset,
@@ -69,17 +173,29 @@ vm_map_op(struct msm_gem_vm *vm, const struct msm_vm_map_op *op) · 3 unmodified lines
69173 /* Actually unmap memory for the vma */
70174 void msm_gem_vma_unmap(struct drm_gpuva *vma)
71175 {
176+ struct msm_gem_vm *vm = to_msm_vm(vma->vm);
72177 struct msm_gem_vma *msm_vma = to_msm_vma(vma);
73178
74179 /* Don't do anything if the memory isn't mapped */
75180 if (!msm_vma->mapped)
76181 return;
77182
78− vm_unmap_op(to_msm_vm(vma->vm), &(struct msm_vm_unmap_op){
183+ /*
184+ * The mmu_lock is only needed when preallocation is used. But
185+ * in that case we don't need to worry about recursion into
186+ * shrinker
187+ */
188+ if (!vm->managed)
189+ mutex_lock(&vm->mmu_lock);
190+
191+ vm_unmap_op(vm, &(struct msm_vm_unmap_op){
79192 .iova = vma->va.addr,
80193 .range = vma->va.range,
81194 });
82195
196+ if (!vm->managed)
197+ mutex_unlock(&vm->mmu_lock);
198+
83199 msm_vma->mapped = false;
84200 }
85201
@@ -87,6 +203,7 @@ void msm_gem_vma_unmap(struct drm_gpuva *vma) · 1 unmodified line
87203 int
88204 msm_gem_vma_map(struct drm_gpuva *vma, int prot, struct sg_table *sgt)
89205 {
206+ struct msm_gem_vm *vm = to_msm_vm(vma->vm);
90207 struct msm_gem_vma *msm_vma = to_msm_vma(vma);
91208 int ret;
92209
@@ -98,6 +215,14 @@ msm_gem_vma_map(struct drm_gpuva *vma, int prot, struct sg_table *sgt) · 5 unmodified lines
98215
99216 msm_vma->mapped = true;
100217
218+ /*
219+ * The mmu_lock is only needed when preallocation is used. But
220+ * in that case we don't need to worry about recursion into
221+ * shrinker
222+ */
223+ if (!vm->managed)
224+ mutex_lock(&vm->mmu_lock);
225+
101226 /*
102227 * NOTE: iommu/io-pgtable can allocate pages, so we cannot hold
103228 * a lock across map/unmap which is also used in the job_run()
@@ -107,16 +232,19 @@ msm_gem_vma_map(struct drm_gpuva *vma, int prot, struct sg_table *sgt) · 3 unmodified lines
107232 * Revisit this if we can come up with a scheme to pre-alloc pages
108233 * for the pgtable in map/unmap ops.
109234 */
110− ret = vm_map_op(to_msm_vm(vma->vm), &(struct msm_vm_map_op){
235+ ret = vm_map_op(vm, &(struct msm_vm_map_op){
111236 .iova = vma->va.addr,
112237 .range = vma->va.range,
113238 .offset = vma->gem.offset,
114239 .sgt = sgt,
115240 .prot = prot,
116241 });
117− if (ret) {
242+
243+ if (!vm->managed)
244+ mutex_unlock(&vm->mmu_lock);
245+
246+ if (ret)
118247 msm_vma->mapped = false;
119− }
120248
121249 return ret;
122250 }
@@ -131,6 +259,9 @@ void msm_gem_vma_close(struct drm_gpuva *vma) · 8 unmodified lines
131259
132260 drm_gpuvm_resv_assert_held(&vm->base);
133261
262+ if (vma->gem.obj)
263+ msm_gem_assert_locked(vma->gem.obj);
264+
134265 if (vma->va.addr && vm->managed)
135266 drm_mm_remove_node(&msm_vma->node);
136267
@@ -158,6 +289,7 @@ msm_gem_vma_new(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj, · 21 unmodified lines
158289
159290 if (vm->managed) {
160291 BUG_ON(offset != 0);
292+ BUG_ON(!obj); /* NULL mappings not valid for kernel managed VM */
161293 ret = drm_mm_insert_node_in_range(&vm->mm, &vma->node,
162294 obj->size, PAGE_SIZE, 0,
163295 range_start, range_end, 0);
@@ -169,7 +301,8 @@ msm_gem_vma_new(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj, · 5 unmodified lines
169301 range_end = range_start + obj->size;
170302 }
171303
172− GEM_WARN_ON((range_end - range_start) > obj->size);
304+ if (obj)
305+ GEM_WARN_ON((range_end - range_start) > obj->size);
173306
174307 drm_gpuva_init(&vma->base, range_start, range_end - range_start, obj, offset);
175308 vma->mapped = false;
@@ -178,6 +311,9 @@ msm_gem_vma_new(struct drm_gpuvm *gpuvm, struct drm_gem_object *obj, · 2 unmodified lines
178311 if (ret)
179312 goto err_free_range;
180313
314+ if (!obj)
315+ return &vma->base;
316+
181317 vm_bo = drm_gpuvm_bo_obtain(&vm->base, obj);
182318 if (IS_ERR(vm_bo)) {
183319 ret = PTR_ERR(vm_bo);
@@ -200,11 +336,297 @@ err_free_vma: · 16 unmodified lines
200336 return ERR_PTR(ret);
201337 }
202338
339+static int
340+msm_gem_vm_bo_validate(struct drm_gpuvm_bo *vm_bo, struct drm_exec *exec)
341+{
342+ struct drm_gem_object *obj = vm_bo->obj;
343+ struct drm_gpuva *vma;
344+ int ret;
345+
346+ vm_dbg("validate: %p", obj);
347+
348+ msm_gem_assert_locked(obj);
349+
350+ drm_gpuvm_bo_for_each_va (vma, vm_bo) {
351+ ret = msm_gem_pin_vma_locked(obj, vma);
352+ if (ret)
353+ return ret;
354+ }
355+
356+ return 0;
357+}
358+
359+struct op_arg {
360+ unsigned flags;
361+ struct msm_vm_bind_job *job;
362+};
363+
364+static void
365+vm_op_enqueue(struct op_arg *arg, struct msm_vm_op _op)
366+{
367+ struct msm_vm_op *op = kmalloc(sizeof(*op), GFP_KERNEL);
368+ *op = _op;
369+ list_add_tail(&op->node, &arg->job->vm_ops);
370+
371+ if (op->obj)
372+ drm_gem_object_get(op->obj);
373+}
374+
375+static struct drm_gpuva *
376+vma_from_op(struct op_arg *arg, struct drm_gpuva_op_map *op)
377+{
378+ return msm_gem_vma_new(arg->job->vm, op->gem.obj, op->gem.offset,
379+ op->va.addr, op->va.addr + op->va.range);
380+}
381+
382+static int
383+msm_gem_vm_sm_step_map(struct drm_gpuva_op *op, void *arg)
384+{
385+ struct drm_gem_object *obj = op->map.gem.obj;
386+ struct drm_gpuva *vma;
387+ struct sg_table *sgt;
388+ unsigned prot;
389+
390+ vma = vma_from_op(arg, &op->map);
391+ if (WARN_ON(IS_ERR(vma)))
392+ return PTR_ERR(vma);
393+
394+ vm_dbg("%p:%p:%p: %016llx %016llx", vma->vm, vma, vma->gem.obj,
395+ vma->va.addr, vma->va.range);
396+
397+ vma->flags = ((struct op_arg *)arg)->flags;
398+
399+ if (obj) {
400+ sgt = to_msm_bo(obj)->sgt;
401+ prot = msm_gem_prot(obj);
402+ } else {
403+ sgt = NULL;
404+ prot = IOMMU_READ | IOMMU_WRITE;
405+ }
406+
407+ vm_op_enqueue(arg, (struct msm_vm_op){
408+ .op = MSM_VM_OP_MAP,
409+ .map = {
410+ .sgt = sgt,
411+ .iova = vma->va.addr,
412+ .range = vma->va.range,
413+ .offset = vma->gem.offset,
414+ .prot = prot,
415+ },
416+ .obj = vma->gem.obj,
417+ });
418+
419+ to_msm_vma(vma)->mapped = true;
420+
421+ return 0;
422+}
423+
424+static int
425+msm_gem_vm_sm_step_remap(struct drm_gpuva_op *op, void *arg)
426+{
427+ struct msm_vm_bind_job *job = ((struct op_arg *)arg)->job;
428+ struct drm_gpuvm *vm = job->vm;
429+ struct drm_gpuva *orig_vma = op->remap.unmap->va;
430+ struct drm_gpuva *prev_vma = NULL, *next_vma = NULL;
431+ struct drm_gpuvm_bo *vm_bo = orig_vma->vm_bo;
432+ bool mapped = to_msm_vma(orig_vma)->mapped;
433+ unsigned flags;
434+
435+ vm_dbg("orig_vma: %p:%p:%p: %016llx %016llx", vm, orig_vma,
436+ orig_vma->gem.obj, orig_vma->va.addr, orig_vma->va.range);
437+
438+ if (mapped) {
439+ uint64_t unmap_start, unmap_range;
440+
441+ drm_gpuva_op_remap_to_unmap_range(&op->remap, &unmap_start, &unmap_range);
442+
443+ vm_op_enqueue(arg, (struct msm_vm_op){
444+ .op = MSM_VM_OP_UNMAP,
445+ .unmap = {
446+ .iova = unmap_start,
447+ .range = unmap_range,
448+ },
449+ .obj = orig_vma->gem.obj,
450+ });
451+
452+ /*
453+ * Part of this GEM obj is still mapped, but we're going to kill the
454+ * existing VMA and replace it with one or two new ones (ie. two if
455+ * the unmapped range is in the middle of the existing (unmap) VMA).
456+ * So just set the state to unmapped:
457+ */
458+ to_msm_vma(orig_vma)->mapped = false;
459+ }
460+
461+ /*
462+ * Hold a ref to the vm_bo between the msm_gem_vma_close() and the
463+ * creation of the new prev/next vma's, in case the vm_bo is tracked
464+ * in the VM's evict list:
465+ */
466+ if (vm_bo)
467+ drm_gpuvm_bo_get(vm_bo);
468+
469+ /*
470+ * The prev_vma and/or next_vma are replacing the unmapped vma, and
471+ * therefore should preserve it's flags:
472+ */
473+ flags = orig_vma->flags;
474+
475+ msm_gem_vma_close(orig_vma);
476+
477+ if (op->remap.prev) {
478+ prev_vma = vma_from_op(arg, op->remap.prev);
479+ if (WARN_ON(IS_ERR(prev_vma)))
480+ return PTR_ERR(prev_vma);
481+
482+ vm_dbg("prev_vma: %p:%p: %016llx %016llx", vm, prev_vma, prev_vma->va.addr, prev_vma->va.range);
483+ to_msm_vma(prev_vma)->mapped = mapped;
484+ prev_vma->flags = flags;
485+ }
486+
487+ if (op->remap.next) {
488+ next_vma = vma_from_op(arg, op->remap.next);
489+ if (WARN_ON(IS_ERR(next_vma)))
490+ return PTR_ERR(next_vma);
491+
492+ vm_dbg("next_vma: %p:%p: %016llx %016llx", vm, next_vma, next_vma->va.addr, next_vma->va.range);
493+ to_msm_vma(next_vma)->mapped = mapped;
494+ next_vma->flags = flags;
495+ }
496+
497+ if (!mapped)
498+ drm_gpuvm_bo_evict(vm_bo, true);
499+
500+ /* Drop the previous ref: */
501+ drm_gpuvm_bo_put(vm_bo);
502+
503+ return 0;
504+}
505+
506+static int
507+msm_gem_vm_sm_step_unmap(struct drm_gpuva_op *op, void *arg)
508+{
509+ struct drm_gpuva *vma = op->unmap.va;
510+ struct msm_gem_vma *msm_vma = to_msm_vma(vma);
511+
512+ vm_dbg("%p:%p:%p: %016llx %016llx", vma->vm, vma, vma->gem.obj,
513+ vma->va.addr, vma->va.range);
514+
515+ if (!msm_vma->mapped)
516+ goto out_close;
517+
518+ vm_op_enqueue(arg, (struct msm_vm_op){
519+ .op = MSM_VM_OP_UNMAP,
520+ .unmap = {
521+ .iova = vma->va.addr,
522+ .range = vma->va.range,
523+ },
524+ .obj = vma->gem.obj,
525+ });
526+
527+ msm_vma->mapped = false;
528+
529+out_close:
530+ msm_gem_vma_close(vma);
531+
532+ return 0;
533+}
534+
203535 static const struct drm_gpuvm_ops msm_gpuvm_ops = {
204536 .vm_free = msm_gem_vm_free,
537+ .vm_bo_validate = msm_gem_vm_bo_validate,
538+ .sm_step_map = msm_gem_vm_sm_step_map,
539+ .sm_step_remap = msm_gem_vm_sm_step_remap,
540+ .sm_step_unmap = msm_gem_vm_sm_step_unmap,
205541 };
206542
543+static struct dma_fence *
544+msm_vma_job_run(struct drm_sched_job *_job)
545+{
546+ struct msm_vm_bind_job *job = to_msm_vm_bind_job(_job);
547+ struct msm_gem_vm *vm = to_msm_vm(job->vm);
548+ struct drm_gem_object *obj;
549+ int ret = vm->unusable ? -EINVAL : 0;
550+
551+ vm_dbg("");
552+
553+ mutex_lock(&vm->mmu_lock);
554+ vm->mmu->prealloc = &job->prealloc;
555+
556+ while (!list_empty(&job->vm_ops)) {
557+ struct msm_vm_op *op =
558+ list_first_entry(&job->vm_ops, struct msm_vm_op, node);
559+
560+ switch (op->op) {
561+ case MSM_VM_OP_MAP:
562+ /*
563+ * On error, stop trying to map new things.. but we
564+ * still want to process the unmaps (or in particular,
565+ * the drm_gem_object_put()s)
566+ */
567+ if (!ret)
568+ ret = vm_map_op(vm, &op->map);
569+ break;
570+ case MSM_VM_OP_UNMAP:
571+ vm_unmap_op(vm, &op->unmap);
572+ break;
573+ }
574+ drm_gem_object_put(op->obj);
575+ list_del(&op->node);
576+ kfree(op);
577+ }
578+
579+ vm->mmu->prealloc = NULL;
580+ mutex_unlock(&vm->mmu_lock);
581+
582+ /*
583+ * We failed to perform at least _some_ of the pgtable updates, so
584+ * now the VM is in an undefined state. Game over!
585+ */
586+ if (ret)
587+ vm->unusable = true;
588+
589+ job_foreach_bo (obj, job) {
590+ msm_gem_lock(obj);
591+ msm_gem_unpin_locked(obj);
592+ msm_gem_unlock(obj);
593+ }
594+
595+ /* VM_BIND ops are synchronous, so no fence to wait on: */
596+ return NULL;
597+}
598+
599+static void
600+msm_vma_job_free(struct drm_sched_job *_job)
601+{
602+ struct msm_vm_bind_job *job = to_msm_vm_bind_job(_job);
603+ struct msm_gem_vm *vm = to_msm_vm(job->vm);
604+ struct drm_gem_object *obj;
605+
606+ vm->mmu->funcs->prealloc_cleanup(vm->mmu, &job->prealloc);
607+
608+ drm_sched_job_cleanup(_job);
609+
610+ job_foreach_bo (obj, job)
611+ drm_gem_object_put(obj);
612+
613+ msm_submitqueue_put(job->queue);
614+ dma_fence_put(job->fence);
615+
616+ /* In error paths, we could have unexecuted ops: */
617+ while (!list_empty(&job->vm_ops)) {
618+ struct msm_vm_op *op =
619+ list_first_entry(&job->vm_ops, struct msm_vm_op, node);
620+ list_del(&op->node);
621+ kfree(op);
622+ }
623+
624+ kfree(job);
625+}
626+
207627 static const struct drm_sched_backend_ops msm_vm_bind_ops = {
628+ .run_job = msm_vma_job_run,
629+ .free_job = msm_vma_job_free
208630 };
209631
210632 /**
@@ -268,6 +690,7 @@ msm_gem_vm_create(struct drm_device *drm, struct msm_mmu *mmu, const char *name, · 57 unmodified lines
268690 drm_gem_object_put(dummy_gem);
269691
270692 vm->mmu = mmu;
693+ mutex_init(&vm->mmu_lock);
271694 vm->managed = managed;
272695
273696 drm_mm_init(&vm->mm, va_start, va_size);
@@ -280,7 +703,6 @@ err_free_dummy: · 6 unmodified lines
280703 err_free_vm:
281704 kfree(vm);
282705 return ERR_PTR(ret);
283−
284706 }
285707
286708 /**
@@ -296,6 +718,7 @@ msm_gem_vm_close(struct drm_gpuvm *gpuvm) · 9 unmodified lines
296718 {
297719 struct msm_gem_vm *vm = to_msm_vm(gpuvm);
298720 struct drm_gpuva *vma, *tmp;
721+ struct drm_exec exec;
299722
300723 /*
301724 * For kernel managed VMs, the VMAs are torn down when the handle is
@@ -312,22 +735,656 @@ msm_gem_vm_close(struct drm_gpuvm *gpuvm) · 10 unmodified lines
312735 drm_sched_fini(&vm->sched);
313736
314737 /* Tear down any remaining mappings: */
315− dma_resv_lock(drm_gpuvm_resv(gpuvm), NULL);
316− drm_gpuvm_for_each_va_safe (vma, tmp, gpuvm) {
317− struct drm_gem_object *obj = vma->gem.obj;
738+ drm_exec_init(&exec, 0, 2);
739+ drm_exec_until_all_locked (&exec) {
740+ drm_exec_lock_obj(&exec, drm_gpuvm_resv_obj(gpuvm));
741+ drm_exec_retry_on_contention(&exec);
318742
319− if (obj && obj->resv != drm_gpuvm_resv(gpuvm)) {
320− drm_gem_object_get(obj);
321− msm_gem_lock(obj);
743+ drm_gpuvm_for_each_va_safe (vma, tmp, gpuvm) {
744+ struct drm_gem_object *obj = vma->gem.obj;
745+
746+ /*
747+ * MSM_BO_NO_SHARE objects share the same resv as the
748+ * VM, in which case the obj is already locked:
749+ */
750+ if (obj && (obj->resv == drm_gpuvm_resv(gpuvm)))
751+ obj = NULL;
752+
753+ if (obj) {
754+ drm_exec_lock_obj(&exec, obj);
755+ drm_exec_retry_on_contention(&exec);
756+ }
757+
758+ msm_gem_vma_unmap(vma);
759+ msm_gem_vma_close(vma);
760+
761+ if (obj) {
762+ drm_exec_unlock_obj(&exec, obj);
763+ }
322764 }
765+ }
766+ drm_exec_fini(&exec);
767+}
768+
769+
770+static struct msm_vm_bind_job *
771+vm_bind_job_create(struct drm_device *dev, struct drm_file *file,
772+ struct msm_gpu_submitqueue *queue, uint32_t nr_ops)
773+{
774+ struct msm_vm_bind_job *job;
775+ uint64_t sz;
776+ int ret;
777+
778+ sz = struct_size(job, ops, nr_ops);
779+
780+ if (sz > SIZE_MAX)
781+ return ERR_PTR(-ENOMEM);
782+
783+ job = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN);
784+ if (!job)
785+ return ERR_PTR(-ENOMEM);
786+
787+ ret = drm_sched_job_init(&job->base, queue->entity, 1, queue,
788+ file->client_id);
789+ if (ret) {
790+ kfree(job);
791+ return ERR_PTR(ret);
792+ }
323793
324− msm_gem_vma_unmap(vma);
325− msm_gem_vma_close(vma);
794+ job->vm = msm_context_vm(dev, queue->ctx);
795+ job->queue = queue;
796+ INIT_LIST_HEAD(&job->vm_ops);
326797
327− if (obj && obj->resv != drm_gpuvm_resv(gpuvm)) {
328− msm_gem_unlock(obj);
329− drm_gem_object_put(obj);
798+ return job;
799+}
800+
801+static bool invalid_alignment(uint64_t addr)
802+{
803+ /*
804+ * Technically this is about GPU alignment, not CPU alignment. But
805+ * I've not seen any qcom SoC where the SMMU does not support the
806+ * CPU's smallest page size.
807+ */
808+ return !PAGE_ALIGNED(addr);
809+}
810+
811+static int
812+lookup_op(struct msm_vm_bind_job *job, const struct drm_msm_vm_bind_op *op)
813+{
814+ struct drm_device *dev = job->vm->drm;
815+ int i = job->nr_ops++;
816+ int ret = 0;
817+
818+ job->ops[i].op = op->op;
819+ job->ops[i].handle = op->handle;
820+ job->ops[i].obj_offset = op->obj_offset;
821+ job->ops[i].iova = op->iova;
822+ job->ops[i].range = op->range;
823+ job->ops[i].flags = op->flags;
824+
825+ if (op->flags & ~MSM_VM_BIND_OP_FLAGS)
826+ ret = UERR(EINVAL, dev, "invalid flags: %x\n", op->flags);
827+
828+ if (invalid_alignment(op->iova))
829+ ret = UERR(EINVAL, dev, "invalid address: %016llx\n", op->iova);
830+
831+ if (invalid_alignment(op->obj_offset))
832+ ret = UERR(EINVAL, dev, "invalid bo_offset: %016llx\n", op->obj_offset);
833+
834+ if (invalid_alignment(op->range))
835+ ret = UERR(EINVAL, dev, "invalid range: %016llx\n", op->range);
836+
837+ if (!drm_gpuvm_range_valid(job->vm, op->iova, op->range))
838+ ret = UERR(EINVAL, dev, "invalid range: %016llx, %016llx\n", op->iova, op->range);
839+
840+ /*
841+ * MAP must specify a valid handle. But the handle MBZ for
842+ * UNMAP or MAP_NULL.
843+ */
844+ if (op->op == MSM_VM_BIND_OP_MAP) {
845+ if (!op->handle)
846+ ret = UERR(EINVAL, dev, "invalid handle\n");
847+ } else if (op->handle) {
848+ ret = UERR(EINVAL, dev, "handle must be zero\n");
849+ }
850+
851+ switch (op->op) {
852+ case MSM_VM_BIND_OP_MAP:
853+ case MSM_VM_BIND_OP_MAP_NULL:
854+ case MSM_VM_BIND_OP_UNMAP:
855+ break;
856+ default:
857+ ret = UERR(EINVAL, dev, "invalid op: %u\n", op->op);
858+ break;
859+ }
860+
861+ return ret;
862+}
863+
864+/*
865+ * ioctl parsing, parameter validation, and GEM handle lookup
866+ */
867+static int
868+vm_bind_job_lookup_ops(struct msm_vm_bind_job *job, struct drm_msm_vm_bind *args,
869+ struct drm_file *file, int *nr_bos)
870+{
871+ struct drm_device *dev = job->vm->drm;
872+ int ret = 0;
873+ int cnt = 0;
874+
875+ if (args->nr_ops == 1) {
876+ /* Single op case, the op is inlined: */
877+ ret = lookup_op(job, &args->op);
878+ } else {
879+ for (unsigned i = 0; i < args->nr_ops; i++) {
880+ struct drm_msm_vm_bind_op op;
881+ void __user *userptr =
882+ u64_to_user_ptr(args->ops + (i * sizeof(op)));
883+
884+ /* make sure we don't have garbage flags, in case we hit
885+ * error path before flags is initialized:
886+ */
887+ job->ops[i].flags = 0;
888+
889+ if (copy_from_user(&op, userptr, sizeof(op))) {
890+ ret = -EFAULT;
891+ break;
892+ }
893+
894+ ret = lookup_op(job, &op);
895+ if (ret)
896+ break;
897+ }
898+ }
899+
900+ if (ret) {
901+ job->nr_ops = 0;
902+ goto out;
903+ }
904+
905+ spin_lock(&file->table_lock);
906+
907+ for (unsigned i = 0; i < args->nr_ops; i++) {
908+ struct drm_gem_object *obj;
909+
910+ if (!job->ops[i].handle) {
911+ job->ops[i].obj = NULL;
912+ continue;
913+ }
914+
915+ /*
916+ * normally use drm_gem_object_lookup(), but for bulk lookup
917+ * all under single table_lock just hit object_idr directly:
918+ */
919+ obj = idr_find(&file->object_idr, job->ops[i].handle);
920+ if (!obj) {
921+ ret = UERR(EINVAL, dev, "invalid handle %u at index %u\n", job->ops[i].handle, i);
922+ goto out_unlock;
923+ }
924+
925+ drm_gem_object_get(obj);
926+
927+ job->ops[i].obj = obj;
928+ cnt++;
929+ }
930+
931+ *nr_bos = cnt;
932+
933+out_unlock:
934+ spin_unlock(&file->table_lock);
935+
936+out:
937+ return ret;
938+}
939+
940+static void
941+prealloc_count(struct msm_vm_bind_job *job,
942+ struct msm_vm_bind_op *first,
943+ struct msm_vm_bind_op *last)
944+{
945+ struct msm_mmu *mmu = to_msm_vm(job->vm)->mmu;
946+
947+ if (!first)
948+ return;
949+
950+ uint64_t start_iova = first->iova;
951+ uint64_t end_iova = last->iova + last->range;
952+
953+ mmu->funcs->prealloc_count(mmu, &job->prealloc, start_iova, end_iova - start_iova);
954+}
955+
956+static bool
957+ops_are_same_pte(struct msm_vm_bind_op *first, struct msm_vm_bind_op *next)
958+{
959+ /*
960+ * Last level pte covers 2MB.. so we should merge two ops, from
961+ * the PoV of figuring out how much pgtable pages to pre-allocate
962+ * if they land in the same 2MB range:
963+ */
964+ uint64_t pte_mask = ~(SZ_2M - 1);
965+ return ((first->iova + first->range) & pte_mask) == (next->iova & pte_mask);
966+}
967+
968+/*
969+ * Determine the amount of memory to prealloc for pgtables. For sparse images,
970+ * in particular, userspace plays some tricks with the order of page mappings
971+ * to get the desired swizzle pattern, resulting in a large # of tiny MAP ops.
972+ * So detect when multiple MAP operations are physically contiguous, and count
973+ * them as a single mapping. Otherwise the prealloc_count() will not realize
974+ * they can share pagetable pages and vastly overcount.
975+ */
976+static void
977+vm_bind_prealloc_count(struct msm_vm_bind_job *job)
978+{
979+ struct msm_vm_bind_op *first = NULL, *last = NULL;
980+
981+ for (int i = 0; i < job->nr_ops; i++) {
982+ struct msm_vm_bind_op *op = &job->ops[i];
983+
984+ /* We only care about MAP/MAP_NULL: */
985+ if (op->op == MSM_VM_BIND_OP_UNMAP)
986+ continue;
987+
988+ /*
989+ * If op is contiguous with last in the current range, then
990+ * it becomes the new last in the range and we continue
991+ * looping:
992+ */
993+ if (last && ops_are_same_pte(last, op)) {
994+ last = op;
995+ continue;
996+ }
997+
998+ /*
999+ * If op is not contiguous with the current range, flush
1000+ * the current range and start anew:
1001+ */
1002+ prealloc_count(job, first, last);
1003+ first = last = op;
1004+ }
1005+
1006+ /* Flush the remaining range: */
1007+ prealloc_count(job, first, last);
1008+}
1009+
1010+/*
1011+ * Lock VM and GEM objects
1012+ */
1013+static int
1014+vm_bind_job_lock_objects(struct msm_vm_bind_job *job, struct drm_exec *exec)
1015+{
1016+ int ret;
1017+
1018+ /* Lock VM and objects: */
1019+ drm_exec_until_all_locked (exec) {
1020+ ret = drm_exec_lock_obj(exec, drm_gpuvm_resv_obj(job->vm));
1021+ drm_exec_retry_on_contention(exec);
1022+ if (ret)
1023+ return ret;
1024+
1025+ for (unsigned i = 0; i < job->nr_ops; i++) {
1026+ const struct msm_vm_bind_op *op = &job->ops[i];
1027+
1028+ switch (op->op) {
1029+ case MSM_VM_BIND_OP_UNMAP:
1030+ ret = drm_gpuvm_sm_unmap_exec_lock(job->vm, exec,
1031+ op->iova,
1032+ op->obj_offset);
1033+ break;
1034+ case MSM_VM_BIND_OP_MAP:
1035+ case MSM_VM_BIND_OP_MAP_NULL:
1036+ ret = drm_gpuvm_sm_map_exec_lock(job->vm, exec, 1,
1037+ op->iova, op->range,
1038+ op->obj, op->obj_offset);
1039+ break;
1040+ default:
1041+ /*
1042+ * lookup_op() should have already thrown an error for
1043+ * invalid ops
1044+ */
1045+ WARN_ON("unreachable");
1046+ }
1047+
1048+ drm_exec_retry_on_contention(exec);
1049+ if (ret)
1050+ return ret;
1051+ }
1052+ }
1053+
1054+ return 0;
1055+}
1056+
1057+/*
1058+ * Pin GEM objects, ensuring that we have backing pages. Pinning will move
1059+ * the object to the pinned LRU so that the shrinker knows to first consider
1060+ * other objects for evicting.
1061+ */
1062+static int
1063+vm_bind_job_pin_objects(struct msm_vm_bind_job *job)
1064+{
1065+ struct drm_gem_object *obj;
1066+
1067+ /*
1068+ * First loop, before holding the LRU lock, avoids holding the
1069+ * LRU lock while calling msm_gem_pin_vma_locked (which could
1070+ * trigger get_pages())
1071+ */
1072+ job_foreach_bo (obj, job) {
1073+ struct page **pages;
1074+
1075+ pages = msm_gem_get_pages_locked(obj, MSM_MADV_WILLNEED);
1076+ if (IS_ERR(pages))
1077+ return PTR_ERR(pages);
1078+ }
1079+
1080+ struct msm_drm_private *priv = job->vm->drm->dev_private;
1081+
1082+ /*
1083+ * A second loop while holding the LRU lock (a) avoids acquiring/dropping
1084+ * the LRU lock for each individual bo, while (b) avoiding holding the
1085+ * LRU lock while calling msm_gem_pin_vma_locked() (which could trigger
1086+ * get_pages() which could trigger reclaim.. and if we held the LRU lock
1087+ * could trigger deadlock with the shrinker).
1088+ */
1089+ mutex_lock(&priv->lru.lock);
1090+ job_foreach_bo (obj, job)
1091+ msm_gem_pin_obj_locked(obj);
1092+ mutex_unlock(&priv->lru.lock);
1093+
1094+ job->bos_pinned = true;
1095+
1096+ return 0;
1097+}
1098+
1099+/*
1100+ * Unpin GEM objects. Normally this is done after the bind job is run.
1101+ */
1102+static void
1103+vm_bind_job_unpin_objects(struct msm_vm_bind_job *job)
1104+{
1105+ struct drm_gem_object *obj;
1106+
1107+ if (!job->bos_pinned)
1108+ return;
1109+
1110+ job_foreach_bo (obj, job)
1111+ msm_gem_unpin_locked(obj);
1112+
1113+ job->bos_pinned = false;
1114+}
1115+
1116+/*
1117+ * Pre-allocate pgtable memory, and translate the VM bind requests into a
1118+ * sequence of pgtable updates to be applied asynchronously.
1119+ */
1120+static int
1121+vm_bind_job_prepare(struct msm_vm_bind_job *job)
1122+{
1123+ struct msm_gem_vm *vm = to_msm_vm(job->vm);
1124+ struct msm_mmu *mmu = vm->mmu;
1125+ int ret;
1126+
1127+ ret = mmu->funcs->prealloc_allocate(mmu, &job->prealloc);
1128+ if (ret)
1129+ return ret;
1130+
1131+ for (unsigned i = 0; i < job->nr_ops; i++) {
1132+ const struct msm_vm_bind_op *op = &job->ops[i];
1133+ struct op_arg arg = {
1134+ .job = job,
1135+ };
1136+
1137+ switch (op->op) {
1138+ case MSM_VM_BIND_OP_UNMAP:
1139+ ret = drm_gpuvm_sm_unmap(job->vm, &arg, op->iova,
1140+ op->range);
1141+ break;
1142+ case MSM_VM_BIND_OP_MAP:
1143+ if (op->flags & MSM_VM_BIND_OP_DUMP)
1144+ arg.flags |= MSM_VMA_DUMP;
1145+ fallthrough;
1146+ case MSM_VM_BIND_OP_MAP_NULL:
1147+ ret = drm_gpuvm_sm_map(job->vm, &arg, op->iova,
1148+ op->range, op->obj, op->obj_offset);
1149+ break;
1150+ default:
1151+ /*
1152+ * lookup_op() should have already thrown an error for
1153+ * invalid ops
1154+ */
1155+ BUG_ON("unreachable");
1156+ }
1157+
1158+ if (ret) {
1159+ /*
1160+ * If we've already started modifying the vm, we can't
1161+ * adequetly describe to userspace the intermediate
1162+ * state the vm is in. So throw up our hands!
1163+ */
1164+ if (i > 0)
1165+ vm->unusable = true;
1166+ return ret;
1167+ }
1168+ }
1169+
1170+ return 0;
1171+}
1172+
1173+/*
1174+ * Attach fences to the GEM objects being bound. This will signify to
1175+ * the shrinker that they are busy even after dropping the locks (ie.
1176+ * drm_exec_fini())
1177+ */
1178+static void
1179+vm_bind_job_attach_fences(struct msm_vm_bind_job *job)
1180+{
1181+ for (unsigned i = 0; i < job->nr_ops; i++) {
1182+ struct drm_gem_object *obj = job->ops[i].obj;
1183+
1184+ if (!obj)
1185+ continue;
1186+
1187+ dma_resv_add_fence(obj->resv, job->fence,
1188+ DMA_RESV_USAGE_KERNEL);
1189+ }
1190+}
1191+
1192+int
1193+msm_ioctl_vm_bind(struct drm_device *dev, void *data, struct drm_file *file)
1194+{
1195+ struct msm_drm_private *priv = dev->dev_private;
1196+ struct drm_msm_vm_bind *args = data;
1197+ struct msm_context *ctx = file->driver_priv;
1198+ struct msm_vm_bind_job *job = NULL;
1199+ struct msm_gpu *gpu = priv->gpu;
1200+ struct msm_gpu_submitqueue *queue;
1201+ struct msm_syncobj_post_dep *post_deps = NULL;
1202+ struct drm_syncobj **syncobjs_to_reset = NULL;
1203+ struct sync_file *sync_file = NULL;
1204+ struct dma_fence *fence;
1205+ int out_fence_fd = -1;
1206+ int ret, nr_bos = 0;
1207+ unsigned i;
1208+
1209+ if (!gpu)
1210+ return -ENXIO;
1211+
1212+ /*
1213+ * Maybe we could allow just UNMAP ops? OTOH userspace should just
1214+ * immediately close the device file and all will be torn down.
1215+ */
1216+ if (to_msm_vm(ctx->vm)->unusable)
1217+ return UERR(EPIPE, dev, "context is unusable");
1218+
1219+ /*
1220+ * Technically, you cannot create a VM_BIND submitqueue in the first
1221+ * place, if you haven't opted in to VM_BIND context. But it is
1222+ * cleaner / less confusing, to check this case directly.
1223+ */
1224+ if (!msm_context_is_vmbind(ctx))
1225+ return UERR(EINVAL, dev, "context does not support vmbind");
1226+
1227+ if (args->flags & ~MSM_VM_BIND_FLAGS)
1228+ return UERR(EINVAL, dev, "invalid flags");
1229+
1230+ queue = msm_submitqueue_get(ctx, args->queue_id);
1231+ if (!queue)
1232+ return -ENOENT;
1233+
1234+ if (!(queue->flags & MSM_SUBMITQUEUE_VM_BIND)) {
1235+ ret = UERR(EINVAL, dev, "Invalid queue type");
1236+ goto out_post_unlock;
1237+ }
1238+
1239+ if (args->flags & MSM_VM_BIND_FENCE_FD_OUT) {
1240+ out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
1241+ if (out_fence_fd < 0) {
1242+ ret = out_fence_fd;
1243+ goto out_post_unlock;
3301244 }
3311245 }
332− dma_resv_unlock(drm_gpuvm_resv(gpuvm));
1246+
1247+ job = vm_bind_job_create(dev, file, queue, args->nr_ops);
1248+ if (IS_ERR(job)) {
1249+ ret = PTR_ERR(job);
1250+ goto out_post_unlock;
1251+ }
1252+
1253+ ret = mutex_lock_interruptible(&queue->lock);
1254+ if (ret)
1255+ goto out_post_unlock;
1256+
1257+ if (args->flags & MSM_VM_BIND_FENCE_FD_IN) {
1258+ struct dma_fence *in_fence;
1259+
1260+ in_fence = sync_file_get_fence(args->fence_fd);
1261+
1262+ if (!in_fence) {
1263+ ret = UERR(EINVAL, dev, "invalid in-fence");
1264+ goto out_unlock;
1265+ }
1266+
1267+ ret = drm_sched_job_add_dependency(&job->base, in_fence);
1268+ if (ret)
1269+ goto out_unlock;
1270+ }
1271+
1272+ if (args->in_syncobjs > 0) {
1273+ syncobjs_to_reset = msm_syncobj_parse_deps(dev, &job->base,
1274+ file, args->in_syncobjs,
1275+ args->nr_in_syncobjs,
1276+ args->syncobj_stride);
1277+ if (IS_ERR(syncobjs_to_reset)) {
1278+ ret = PTR_ERR(syncobjs_to_reset);
1279+ goto out_unlock;
1280+ }
1281+ }
1282+
1283+ if (args->out_syncobjs > 0) {
1284+ post_deps = msm_syncobj_parse_post_deps(dev, file,
1285+ args->out_syncobjs,
1286+ args->nr_out_syncobjs,
1287+ args->syncobj_stride);
1288+ if (IS_ERR(post_deps)) {
1289+ ret = PTR_ERR(post_deps);
1290+ goto out_unlock;
1291+ }
1292+ }
1293+
1294+ ret = vm_bind_job_lookup_ops(job, args, file, &nr_bos);
1295+ if (ret)
1296+ goto out_unlock;
1297+
1298+ vm_bind_prealloc_count(job);
1299+
1300+ struct drm_exec exec;
1301+ unsigned flags = DRM_EXEC_IGNORE_DUPLICATES | DRM_EXEC_INTERRUPTIBLE_WAIT;
1302+ drm_exec_init(&exec, flags, nr_bos + 1);
1303+
1304+ ret = vm_bind_job_lock_objects(job, &exec);
1305+ if (ret)
1306+ goto out;
1307+
1308+ ret = vm_bind_job_pin_objects(job);
1309+ if (ret)
1310+ goto out;
1311+
1312+ ret = vm_bind_job_prepare(job);
1313+ if (ret)
1314+ goto out;
1315+
1316+ drm_sched_job_arm(&job->base);
1317+
1318+ job->fence = dma_fence_get(&job->base.s_fence->finished);
1319+
1320+ if (args->flags & MSM_VM_BIND_FENCE_FD_OUT) {
1321+ sync_file = sync_file_create(job->fence);
1322+ if (!sync_file) {
1323+ ret = -ENOMEM;
1324+ } else {
1325+ fd_install(out_fence_fd, sync_file->file);
1326+ args->fence_fd = out_fence_fd;
1327+ }
1328+ }
1329+
1330+ if (ret)
1331+ goto out;
1332+
1333+ vm_bind_job_attach_fences(job);
1334+
1335+ /*
1336+ * The job can be free'd (and fence unref'd) at any point after
1337+ * drm_sched_entity_push_job(), so we need to hold our own ref
1338+ */
1339+ fence = dma_fence_get(job->fence);
1340+
1341+ drm_sched_entity_push_job(&job->base);
1342+
1343+ msm_syncobj_reset(syncobjs_to_reset, args->nr_in_syncobjs);
1344+ msm_syncobj_process_post_deps(post_deps, args->nr_out_syncobjs, fence);
1345+
1346+ dma_fence_put(fence);
1347+
1348+out:
1349+ if (ret)
1350+ vm_bind_job_unpin_objects(job);
1351+
1352+ drm_exec_fini(&exec);
1353+out_unlock:
1354+ mutex_unlock(&queue->lock);
1355+out_post_unlock:
1356+ if (ret && (out_fence_fd >= 0)) {
1357+ put_unused_fd(out_fence_fd);
1358+ if (sync_file)
1359+ fput(sync_file->file);
1360+ }
1361+
1362+ if (!IS_ERR_OR_NULL(job)) {
1363+ if (ret)
1364+ msm_vma_job_free(&job->base);
1365+ } else {
1366+ /*
1367+ * If the submit hasn't yet taken ownership of the queue
1368+ * then we need to drop the reference ourself:
1369+ */
1370+ msm_submitqueue_put(queue);
1371+ }
1372+
1373+ if (!IS_ERR_OR_NULL(post_deps)) {
1374+ for (i = 0; i < args->nr_out_syncobjs; ++i) {
1375+ kfree(post_deps[i].chain);
1376+ drm_syncobj_put(post_deps[i].syncobj);
1377+ }
1378+ kfree(post_deps);
1379+ }
1380+
1381+ if (!IS_ERR_OR_NULL(syncobjs_to_reset)) {
1382+ for (i = 0; i < args->nr_in_syncobjs; ++i) {
1383+ if (syncobjs_to_reset[i])
1384+ drm_syncobj_put(syncobjs_to_reset[i]);
1385+ }
1386+ kfree(syncobjs_to_reset);
1387+ }
1388+
1389+ return ret;
3331390 }
include/uapi/drm/msm_drm.h+73−1
@@ -272,7 +272,10 @@ struct drm_msm_gem_submit_cmd { · 271 unmodified lines
272272 __u32 size; /* in, cmdstream size */
273273 __u32 pad;
274274 __u32 nr_relocs; /* in, number of submit_reloc's */
275− __u64 relocs; /* in, ptr to array of submit_reloc's */
275+ union {
276+ __u64 relocs; /* in, ptr to array of submit_reloc's */
277+ __u64 iova; /* cmdstream address (for VM_BIND contexts) */
278+ };
276279 };
277280
278281 /* Each buffer referenced elsewhere in the cmdstream submit (ie. the
@@ -339,7 +342,74 @@ struct drm_msm_gem_submit { · 60 unmodified lines
339342 __u32 nr_out_syncobjs; /* in, number of entries in out_syncobj. */
340343 __u32 syncobj_stride; /* in, stride of syncobj arrays. */
341344 __u32 pad; /*in, reserved for future use, always 0. */
345+};
346+
347+#define MSM_VM_BIND_OP_UNMAP 0
348+#define MSM_VM_BIND_OP_MAP 1
349+#define MSM_VM_BIND_OP_MAP_NULL 2
350+
351+#define MSM_VM_BIND_OP_DUMP 1
352+#define MSM_VM_BIND_OP_FLAGS ( \
353+ MSM_VM_BIND_OP_DUMP | \
354+ 0)
342355
356+/**
357+ * struct drm_msm_vm_bind_op - bind/unbind op to run
358+ */
359+struct drm_msm_vm_bind_op {
360+ /** @op: one of MSM_VM_BIND_OP_x */
361+ __u32 op;
362+ /** @handle: GEM object handle, MBZ for UNMAP or MAP_NULL */
363+ __u32 handle;
364+ /** @obj_offset: Offset into GEM object, MBZ for UNMAP or MAP_NULL */
365+ __u64 obj_offset;
366+ /** @iova: Address to operate on */
367+ __u64 iova;
368+ /** @range: Number of bites to to map/unmap */
369+ __u64 range;
370+ /** @flags: Bitmask of MSM_VM_BIND_OP_FLAG_x */
371+ __u32 flags;
372+ /** @pad: MBZ */
373+ __u32 pad;
374+};
375+
376+#define MSM_VM_BIND_FENCE_FD_IN 0x00000001
377+#define MSM_VM_BIND_FENCE_FD_OUT 0x00000002
378+#define MSM_VM_BIND_FLAGS ( \
379+ MSM_VM_BIND_FENCE_FD_IN | \
380+ MSM_VM_BIND_FENCE_FD_OUT | \
381+ 0)
382+
383+/**
384+ * struct drm_msm_vm_bind - Input of &DRM_IOCTL_MSM_VM_BIND
385+ */
386+struct drm_msm_vm_bind {
387+ /** @flags: in, bitmask of MSM_VM_BIND_x */
388+ __u32 flags;
389+ /** @nr_ops: the number of bind ops in this ioctl */
390+ __u32 nr_ops;
391+ /** @fence_fd: in/out fence fd (see MSM_VM_BIND_FENCE_FD_IN/OUT) */
392+ __s32 fence_fd;
393+ /** @queue_id: in, submitqueue id */
394+ __u32 queue_id;
395+ /** @in_syncobjs: in, ptr to array of drm_msm_gem_syncobj */
396+ __u64 in_syncobjs;
397+ /** @out_syncobjs: in, ptr to array of drm_msm_gem_syncobj */
398+ __u64 out_syncobjs;
399+ /** @nr_in_syncobjs: in, number of entries in in_syncobj */
400+ __u32 nr_in_syncobjs;
401+ /** @nr_out_syncobjs: in, number of entries in out_syncobj */
402+ __u32 nr_out_syncobjs;
403+ /** @syncobj_stride: in, stride of syncobj arrays */
404+ __u32 syncobj_stride;
405+ /** @op_stride: sizeof each struct drm_msm_vm_bind_op in @ops */
406+ __u32 op_stride;
407+ union {
408+ /** @op: used if num_ops == 1 */
409+ struct drm_msm_vm_bind_op op;
410+ /** @ops: userptr to array of drm_msm_vm_bind_op if num_ops > 1 */
411+ __u64 ops;
412+ };
343413 };
344414
345415 #define MSM_WAIT_FENCE_BOOST 0x00000001
@@ -435,6 +505,7 @@ struct drm_msm_submitqueue_query { · 89 unmodified lines
435505 #define DRM_MSM_SUBMITQUEUE_NEW 0x0A
436506 #define DRM_MSM_SUBMITQUEUE_CLOSE 0x0B
437507 #define DRM_MSM_SUBMITQUEUE_QUERY 0x0C
508+#define DRM_MSM_VM_BIND 0x0D
438509
439510 #define DRM_IOCTL_MSM_GET_PARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_MSM_GET_PARAM, struct drm_msm_param)
440511 #define DRM_IOCTL_MSM_SET_PARAM DRM_IOW (DRM_COMMAND_BASE + DRM_MSM_SET_PARAM, struct drm_msm_param)
@@ -448,6 +519,7 @@ struct drm_msm_submitqueue_query { · 7 unmodified lines
448519 #define DRM_IOCTL_MSM_SUBMITQUEUE_NEW DRM_IOWR(DRM_COMMAND_BASE + DRM_MSM_SUBMITQUEUE_NEW, struct drm_msm_submitqueue)
449520 #define DRM_IOCTL_MSM_SUBMITQUEUE_CLOSE DRM_IOW (DRM_COMMAND_BASE + DRM_MSM_SUBMITQUEUE_CLOSE, __u32)
450521 #define DRM_IOCTL_MSM_SUBMITQUEUE_QUERY DRM_IOW (DRM_COMMAND_BASE + DRM_MSM_SUBMITQUEUE_QUERY, struct drm_msm_submitqueue_query)
522+#define DRM_IOCTL_MSM_VM_BIND DRM_IOWR(DRM_COMMAND_BASE + DRM_MSM_VM_BIND, struct drm_msm_vm_bind)
451523
452524 #if defined(__cplusplus)
453525 }