drm/msm: Add VM_BIND ioctl

Add a VM_BIND ioctl for binding/unbinding buffers into a VM. This is only supported if userspace has opted in to MSM_PARAM_EN_VM_BIND.

Rob Clark committed 1 year ago

Commit 2e6a8a11 parent ecfd9fa7 files · +1205 −33Browse files

Unified · Split

7 files

drivers/gpu/drm/msm/msm_drv.c +1 −0
drivers/gpu/drm/msm/msm_drv.h +3 −1
drivers/gpu/drm/msm/msm_gem.c +29 −11
drivers/gpu/drm/msm/msm_gem.h +4 −0
drivers/gpu/drm/msm/msm_gem_submit.c +20 −2
drivers/gpu/drm/msm/msm_gem_vma.c +1075 −18
include/uapi/drm/msm_drm.h +73 −1

drivers/gpu/drm/msm/msm_drv.c+1−0

@@ -802,6 +802,7 @@ static const struct drm_ioctl_desc msm_ioctls[] = { · 801 unmodified lines
802	DRM_IOCTL_DEF_DRV(MSM_SUBMITQUEUE_NEW, msm_ioctl_submitqueue_new, DRM_RENDER_ALLOW),	802	DRM_IOCTL_DEF_DRV(MSM_SUBMITQUEUE_NEW, msm_ioctl_submitqueue_new, DRM_RENDER_ALLOW),
803	DRM_IOCTL_DEF_DRV(MSM_SUBMITQUEUE_CLOSE, msm_ioctl_submitqueue_close, DRM_RENDER_ALLOW),	803	DRM_IOCTL_DEF_DRV(MSM_SUBMITQUEUE_CLOSE, msm_ioctl_submitqueue_close, DRM_RENDER_ALLOW),
804	DRM_IOCTL_DEF_DRV(MSM_SUBMITQUEUE_QUERY, msm_ioctl_submitqueue_query, DRM_RENDER_ALLOW),	804	DRM_IOCTL_DEF_DRV(MSM_SUBMITQUEUE_QUERY, msm_ioctl_submitqueue_query, DRM_RENDER_ALLOW),
		805	DRM_IOCTL_DEF_DRV(MSM_VM_BIND, msm_ioctl_vm_bind, DRM_RENDER_ALLOW),
805	};	806	};
806		807
807	static void msm_show_fdinfo(struct drm_printer p, struct drm_file file)	808	static void msm_show_fdinfo(struct drm_printer p, struct drm_file file)

drivers/gpu/drm/msm/msm_drv.h+3−1

@@ -255,7 +255,9 @@ struct drm_gpuvm msm_kms_init_vm(struct drm_device dev); · 254 unmodified lines
255	bool msm_use_mmu(struct drm_device *dev);	255	bool msm_use_mmu(struct drm_device *dev);
256		256
257	int msm_ioctl_gem_submit(struct drm_device dev, void data,	257	int msm_ioctl_gem_submit(struct drm_device dev, void data,
258	struct drm_file *file);	258	struct drm_file *file);
		259	int msm_ioctl_vm_bind(struct drm_device dev, void data,
		260	struct drm_file *file);
259		261
260	#ifdef CONFIG_DEBUG_FS	262	#ifdef CONFIG_DEBUG_FS
261	unsigned long msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan);	263	unsigned long msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan);

drivers/gpu/drm/msm/msm_gem.c+29−11

 	}
 }
-static struct page **msm_gem_get_pages_locked(struct drm_gem_object *obj,
+struct page **msm_gem_get_pages_locked(struct drm_gem_object *obj, unsigned madv)
-					      unsigned madv)
 {
 	struct msm_gem_object *msm_obj = to_msm_bo(obj);
 	/*
 	 * We need to lock any VMs the object is still attached to, but not
 	 * the object itself (see explaination in msm_gem_assert_locked()),
-	 * so just open-code this special case:
+	 * so just open-code this special case.
+	 *
+	 * Note that we skip the dance if we aren't attached to any VM.  This
+	 * is load bearing.  The driver needs to support two usage models:
+	 *
+	 * 1. Legacy kernel managed VM: Userspace expects the VMA's to be
+	 *    implicitly torn down when the object is freed, the VMA's do
+	 *    not hold a hard reference to the BO.
+	 *
+	 * 2. VM_BIND, userspace managed VM: The VMA holds a reference to the
+	 *    BO.  This can be dropped when the VM is closed and it's associated
+	 *    VMAs are torn down.  (See msm_gem_vm_close()).
+	 *
+	 * In the latter case the last reference to a BO can be dropped while
+	 * we already have the VM locked.  It would have already been removed
+	 * from the gpuva list, but lockdep doesn't know that.  Or understand
+	 * the differences between the two usage models.
 	 */
-	drm_exec_init(&exec, 0, 0);
+	if (!list_empty(&obj->gpuva.list)) {
-	drm_exec_until_all_locked (&exec) {
+		drm_exec_init(&exec, 0, 0);
-		struct drm_gpuvm_bo *vm_bo;
+		drm_exec_until_all_locked (&exec) {
-		drm_gem_for_each_gpuvm_bo (vm_bo, obj) {
+			struct drm_gpuvm_bo *vm_bo;
-			drm_exec_lock_obj(&exec, drm_gpuvm_resv_obj(vm_bo->vm));
+			drm_gem_for_each_gpuvm_bo (vm_bo, obj) {
-			drm_exec_retry_on_contention(&exec);
+				drm_exec_lock_obj(&exec,
+						  drm_gpuvm_resv_obj(vm_bo->vm));
+				drm_exec_retry_on_contention(&exec);
+			}
 		}
+		put_iova_spaces(obj, NULL, true);
+		drm_exec_fini(&exec);     /* drop locks */
 	}
-	put_iova_spaces(obj, NULL, true);
-	drm_exec_fini(&exec);     /* drop locks */
 	if (drm_gem_is_imported(obj)) {
 		GEM_WARN_ON(msm_obj->vaddr);

drivers/gpu/drm/msm/msm_gem.h+4−0

 	/** @mmu: The mmu object which manages the pgtables */
 	struct msm_mmu *mmu;
+	/** @mmu_lock: Protects access to the mmu */
+	struct mutex mmu_lock;
 	/**
 	 * @pid: For address spaces associated with a specific process, this
 	 * will be non-NULL:
 			     uint64_t *iova);
 void msm_gem_unpin_iova(struct drm_gem_object *obj, struct drm_gpuvm *vm);
 void msm_gem_pin_obj_locked(struct drm_gem_object *obj);
+struct page **msm_gem_get_pages_locked(struct drm_gem_object *obj, unsigned madv);
 struct page **msm_gem_pin_pages_locked(struct drm_gem_object *obj);
 void msm_gem_unpin_pages_locked(struct drm_gem_object *obj);
 int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev,

drivers/gpu/drm/msm/msm_gem_submit.c+20−2

 static int submit_lookup_cmds(struct msm_gem_submit *submit,
 		struct drm_msm_gem_submit *args, struct drm_file *file)
 {
+	struct msm_context *ctx = file->driver_priv;
 	unsigned i;
 	size_t sz;
 	int ret = 0;
 			goto out;
 		}
+		if (msm_context_is_vmbind(ctx)) {
+			if (submit_cmd.nr_relocs) {
+				ret = SUBMIT_ERROR(EINVAL, submit, "nr_relocs must be zero");
+				goto out;
+			}
+			if (submit_cmd.submit_idx || submit_cmd.submit_offset) {
+				ret = SUBMIT_ERROR(EINVAL, submit, "submit_idx/offset must be zero");
+				goto out;
+			}
+			submit->cmd[i].iova = submit_cmd.iova;
+		}
 		submit->cmd[i].type = submit_cmd.type;
 		submit->cmd[i].size = submit_cmd.size / 4;
 		submit->cmd[i].offset = submit_cmd.submit_offset / 4;
 	struct msm_syncobj_post_dep *post_deps = NULL;
 	struct drm_syncobj **syncobjs_to_reset = NULL;
 	struct sync_file *sync_file = NULL;
+	unsigned cmds_to_parse;
 	int out_fence_fd = -1;
 	unsigned i;
 	int ret;
 	if (ret)
 		goto out;
-	for (i = 0; i < args->nr_cmds; i++) {
+	cmds_to_parse = msm_context_is_vmbind(ctx) ? 0 : args->nr_cmds;
+	for (i = 0; i < cmds_to_parse; i++) {
 		struct drm_gem_object *obj;
 		uint64_t iova;
 			goto out;
 	}
-	submit->nr_cmds = i;
+	submit->nr_cmds = args->nr_cmds;
 	idr_preload(GFP_KERNEL);

drivers/gpu/drm/msm/msm_gem_vma.c+1075−18

@@ -4,9 +4,16 @@ · 3 unmodified lines
4	* Author: Rob Clark <robdclark@gmail.com>	4	* Author: Rob Clark <robdclark@gmail.com>
5	*/	5	*/
6		6
		7	#include "drm/drm_file.h"
		8	#include "drm/msm_drm.h"
		9	#include "linux/file.h"
		10	#include "linux/sync_file.h"
		11
7	#include "msm_drv.h"	12	#include "msm_drv.h"
8	#include "msm_gem.h"	13	#include "msm_gem.h"
		14	#include "msm_gpu.h"
9	#include "msm_mmu.h"	15	#include "msm_mmu.h"
		16	#include "msm_syncobj.h"
10		17
11	#define vm_dbg(fmt, ...) pr_debug("%s:%d: "fmt"\n", __func__, __LINE__, ##__VA_ARGS__)	18	#define vm_dbg(fmt, ...) pr_debug("%s:%d: "fmt"\n", __func__, __LINE__, ##__VA_ARGS__)
12		19
@@ -36,6 +43,97 @@ struct msm_vm_unmap_op { · 23 unmodified lines
36	uint64_t range;	43	uint64_t range;
37	};	44	};
38		45
		46	/**
		47	* struct msm_vma_op - A MAP or UNMAP operation
		48	*/
		49	struct msm_vm_op {
		50	/** @op: The operation type */
		51	enum {
		52	MSM_VM_OP_MAP = 1,
		53	MSM_VM_OP_UNMAP,
		54	} op;
		55	union {
		56	/** @map: Parameters used if op == MSM_VMA_OP_MAP */
		57	struct msm_vm_map_op map;
		58	/** @unmap: Parameters used if op == MSM_VMA_OP_UNMAP */
		59	struct msm_vm_unmap_op unmap;
		60	};
		61	/** @node: list head in msm_vm_bind_job::vm_ops */
		62	struct list_head node;
		63
		64	/**
		65	* @obj: backing object for pages to be mapped/unmapped
		66	*
		67	* Async unmap ops, in particular, must hold a reference to the
		68	* original GEM object backing the mapping that will be unmapped.
		69	* But the same can be required in the map path, for example if
		70	* there is not a corresponding unmap op, such as process exit.
		71	*
		72	* This ensures that the pages backing the mapping are not freed
		73	* before the mapping is torn down.
		74	*/
		75	struct drm_gem_object *obj;
		76	};
		77
		78	/**
		79	* struct msm_vm_bind_job - Tracking for a VM_BIND ioctl
		80	*
		81	* A table of userspace requested VM updates (MSM_VM_BIND_OP_UNMAP/MAP/MAP_NULL)
		82	* gets applied to the vm, generating a list of VM ops (MSM_VM_OP_MAP/UNMAP)
		83	* which are applied to the pgtables asynchronously. For example a userspace
		84	* requested MSM_VM_BIND_OP_MAP could end up generating both an MSM_VM_OP_UNMAP
		85	* to unmap an existing mapping, and a MSM_VM_OP_MAP to apply the new mapping.
		86	*/
		87	struct msm_vm_bind_job {
		88	/** @base: base class for drm_sched jobs */
		89	struct drm_sched_job base;
		90	/** @vm: The VM being operated on */
		91	struct drm_gpuvm *vm;
		92	/** @fence: The fence that is signaled when job completes */
		93	struct dma_fence *fence;
		94	/** @queue: The queue that the job runs on */
		95	struct msm_gpu_submitqueue *queue;
		96	/** @prealloc: Tracking for pre-allocated MMU pgtable pages */
		97	struct msm_mmu_prealloc prealloc;
		98	/** @vm_ops: a list of struct msm_vm_op */
		99	struct list_head vm_ops;
		100	/** @bos_pinned: are the GEM objects being bound pinned? */
		101	bool bos_pinned;
		102	/** @nr_ops: the number of userspace requested ops */
		103	unsigned int nr_ops;
		104	/**
		105	* @ops: the userspace requested ops
		106	*
		107	* The userspace requested ops are copied/parsed and validated
		108	* before we start applying the updates to try to do as much up-
		109	* front error checking as possible, to avoid the VM being in an
		110	* undefined state due to partially executed VM_BIND.
		111	*
		112	* This table also serves to hold a reference to the backing GEM
		113	* objects.
		114	*/
		115	struct msm_vm_bind_op {
		116	uint32_t op;
		117	uint32_t flags;
		118	union {
		119	struct drm_gem_object *obj;
		120	uint32_t handle;
		121	};
		122	uint64_t obj_offset;
		123	uint64_t iova;
		124	uint64_t range;
		125	} ops[];
		126	};
		127
		128	#define job_foreach_bo(obj, _job) \
		129	for (unsigned i = 0; i < (_job)->nr_ops; i++) \
		130	if ((obj = (_job)->ops[i].obj))
		131
		132	static inline struct msm_vm_bind_job to_msm_vm_bind_job(struct drm_sched_job job)
		133	{
		134	return container_of(job, struct msm_vm_bind_job, base);
		135	}
		136
39	static void	137	static void
40	msm_gem_vm_free(struct drm_gpuvm *gpuvm)	138	msm_gem_vm_free(struct drm_gpuvm *gpuvm)
41	{	139	{
@@ -52,6 +150,9 @@ msm_gem_vm_free(struct drm_gpuvm *gpuvm) · 10 unmodified lines
52	static void	150	static void
53	vm_unmap_op(struct msm_gem_vm vm, const struct msm_vm_unmap_op op)	151	vm_unmap_op(struct msm_gem_vm vm, const struct msm_vm_unmap_op op)
54	{	152	{
		153	if (!vm->managed)
		154	lockdep_assert_held(&vm->mmu_lock);
		155
55	vm_dbg("%p: %016llx %016llx", vm, op->iova, op->iova + op->range);	156	vm_dbg("%p: %016llx %016llx", vm, op->iova, op->iova + op->range);
56		157
57	vm->mmu->funcs->unmap(vm->mmu, op->iova, op->range);	158	vm->mmu->funcs->unmap(vm->mmu, op->iova, op->range);
@@ -60,6 +161,9 @@ vm_unmap_op(struct msm_gem_vm vm, const struct msm_vm_unmap_op op) · 2 unmodified lines
60	static int	161	static int
61	vm_map_op(struct msm_gem_vm vm, const struct msm_vm_map_op op)	162	vm_map_op(struct msm_gem_vm vm, const struct msm_vm_map_op op)
62	{	163	{
		164	if (!vm->managed)
		165	lockdep_assert_held(&vm->mmu_lock);
		166
63	vm_dbg("%p: %016llx %016llx", vm, op->iova, op->iova + op->range);	167	vm_dbg("%p: %016llx %016llx", vm, op->iova, op->iova + op->range);
64		168
65	return vm->mmu->funcs->map(vm->mmu, op->iova, op->sgt, op->offset,	169	return vm->mmu->funcs->map(vm->mmu, op->iova, op->sgt, op->offset,
@@ -69,17 +173,29 @@ vm_map_op(struct msm_gem_vm vm, const struct msm_vm_map_op op) · 3 unmodified lines
69	/* Actually unmap memory for the vma */	173	/* Actually unmap memory for the vma */
70	void msm_gem_vma_unmap(struct drm_gpuva *vma)	174	void msm_gem_vma_unmap(struct drm_gpuva *vma)
71	{	175	{
		176	struct msm_gem_vm *vm = to_msm_vm(vma->vm);
72	struct msm_gem_vma *msm_vma = to_msm_vma(vma);	177	struct msm_gem_vma *msm_vma = to_msm_vma(vma);
73		178
74	/* Don't do anything if the memory isn't mapped */	179	/* Don't do anything if the memory isn't mapped */
75	if (!msm_vma->mapped)	180	if (!msm_vma->mapped)
76	return;	181	return;
77		182
78	vm_unmap_op(to_msm_vm(vma->vm), &(struct msm_vm_unmap_op){	183	/*
		184	* The mmu_lock is only needed when preallocation is used. But
		185	* in that case we don't need to worry about recursion into
		186	* shrinker
		187	*/
		188	if (!vm->managed)
		189	mutex_lock(&vm->mmu_lock);
		190
		191	vm_unmap_op(vm, &(struct msm_vm_unmap_op){
79	.iova = vma->va.addr,	192	.iova = vma->va.addr,
80	.range = vma->va.range,	193	.range = vma->va.range,
81	});	194	});
82		195
		196	if (!vm->managed)
		197	mutex_unlock(&vm->mmu_lock);
		198
83	msm_vma->mapped = false;	199	msm_vma->mapped = false;
84	}	200	}
85		201
@@ -87,6 +203,7 @@ void msm_gem_vma_unmap(struct drm_gpuva *vma) · 1 unmodified line
87	int	203	int
88	msm_gem_vma_map(struct drm_gpuva vma, int prot, struct sg_table sgt)	204	msm_gem_vma_map(struct drm_gpuva vma, int prot, struct sg_table sgt)
89	{	205	{
		206	struct msm_gem_vm *vm = to_msm_vm(vma->vm);
90	struct msm_gem_vma *msm_vma = to_msm_vma(vma);	207	struct msm_gem_vma *msm_vma = to_msm_vma(vma);
91	int ret;	208	int ret;
92		209
@@ -98,6 +215,14 @@ msm_gem_vma_map(struct drm_gpuva vma, int prot, struct sg_table sgt) · 5 unmodified lines
98		215
99	msm_vma->mapped = true;	216	msm_vma->mapped = true;
100		217
		218	/*
		219	* The mmu_lock is only needed when preallocation is used. But
		220	* in that case we don't need to worry about recursion into
		221	* shrinker
		222	*/
		223	if (!vm->managed)
		224	mutex_lock(&vm->mmu_lock);
		225
101	/*	226	/*
102	* NOTE: iommu/io-pgtable can allocate pages, so we cannot hold	227	* NOTE: iommu/io-pgtable can allocate pages, so we cannot hold
103	* a lock across map/unmap which is also used in the job_run()	228	* a lock across map/unmap which is also used in the job_run()
@@ -107,16 +232,19 @@ msm_gem_vma_map(struct drm_gpuva vma, int prot, struct sg_table sgt) · 3 unmodified lines
107	* Revisit this if we can come up with a scheme to pre-alloc pages	232	* Revisit this if we can come up with a scheme to pre-alloc pages
108	* for the pgtable in map/unmap ops.	233	* for the pgtable in map/unmap ops.
109	*/	234	*/
110	ret = vm_map_op(to_msm_vm(vma->vm), &(struct msm_vm_map_op){	235	ret = vm_map_op(vm, &(struct msm_vm_map_op){
111	.iova = vma->va.addr,	236	.iova = vma->va.addr,
112	.range = vma->va.range,	237	.range = vma->va.range,
113	.offset = vma->gem.offset,	238	.offset = vma->gem.offset,
114	.sgt = sgt,	239	.sgt = sgt,
115	.prot = prot,	240	.prot = prot,
116	});	241	});
117	if (ret) {	242
		243	if (!vm->managed)
		244	mutex_unlock(&vm->mmu_lock);
		245
		246	if (ret)
118	msm_vma->mapped = false;	247	msm_vma->mapped = false;
119	}
120		248
121	return ret;	249	return ret;
122	}	250	}
@@ -131,6 +259,9 @@ void msm_gem_vma_close(struct drm_gpuva *vma) · 8 unmodified lines
131		259
132	drm_gpuvm_resv_assert_held(&vm->base);	260	drm_gpuvm_resv_assert_held(&vm->base);
133		261
		262	if (vma->gem.obj)
		263	msm_gem_assert_locked(vma->gem.obj);
		264
134	if (vma->va.addr && vm->managed)	265	if (vma->va.addr && vm->managed)
135	drm_mm_remove_node(&msm_vma->node);	266	drm_mm_remove_node(&msm_vma->node);
136		267
@@ -158,6 +289,7 @@ msm_gem_vma_new(struct drm_gpuvm gpuvm, struct drm_gem_object obj, · 21 unmodified lines
158		289
159	if (vm->managed) {	290	if (vm->managed) {
160	BUG_ON(offset != 0);	291	BUG_ON(offset != 0);
		292	BUG_ON(!obj); /* NULL mappings not valid for kernel managed VM */
161	ret = drm_mm_insert_node_in_range(&vm->mm, &vma->node,	293	ret = drm_mm_insert_node_in_range(&vm->mm, &vma->node,
162	obj->size, PAGE_SIZE, 0,	294	obj->size, PAGE_SIZE, 0,
163	range_start, range_end, 0);	295	range_start, range_end, 0);
@@ -169,7 +301,8 @@ msm_gem_vma_new(struct drm_gpuvm gpuvm, struct drm_gem_object obj, · 5 unmodified lines
169	range_end = range_start + obj->size;	301	range_end = range_start + obj->size;
170	}	302	}
171		303
172	GEM_WARN_ON((range_end - range_start) > obj->size);	304	if (obj)
		305	GEM_WARN_ON((range_end - range_start) > obj->size);
173		306
174	drm_gpuva_init(&vma->base, range_start, range_end - range_start, obj, offset);	307	drm_gpuva_init(&vma->base, range_start, range_end - range_start, obj, offset);
175	vma->mapped = false;	308	vma->mapped = false;
@@ -178,6 +311,9 @@ msm_gem_vma_new(struct drm_gpuvm gpuvm, struct drm_gem_object obj, · 2 unmodified lines
178	if (ret)	311	if (ret)
179	goto err_free_range;	312	goto err_free_range;
180		313
		314	if (!obj)
		315	return &vma->base;
		316
181	vm_bo = drm_gpuvm_bo_obtain(&vm->base, obj);	317	vm_bo = drm_gpuvm_bo_obtain(&vm->base, obj);
182	if (IS_ERR(vm_bo)) {	318	if (IS_ERR(vm_bo)) {
183	ret = PTR_ERR(vm_bo);	319	ret = PTR_ERR(vm_bo);
@@ -200,11 +336,297 @@ err_free_vma: · 16 unmodified lines
200	return ERR_PTR(ret);	336	return ERR_PTR(ret);
201	}	337	}
202		338
		339	static int
		340	msm_gem_vm_bo_validate(struct drm_gpuvm_bo vm_bo, struct drm_exec exec)
		341	{
		342	struct drm_gem_object *obj = vm_bo->obj;
		343	struct drm_gpuva *vma;
		344	int ret;
		345
		346	vm_dbg("validate: %p", obj);
		347
		348	msm_gem_assert_locked(obj);
		349
		350	drm_gpuvm_bo_for_each_va (vma, vm_bo) {
		351	ret = msm_gem_pin_vma_locked(obj, vma);
		352	if (ret)
		353	return ret;
		354	}
		355
		356	return 0;
		357	}
		358
		359	struct op_arg {
		360	unsigned flags;
		361	struct msm_vm_bind_job *job;
		362	};
		363
		364	static void
		365	vm_op_enqueue(struct op_arg *arg, struct msm_vm_op _op)
		366	{
		367	struct msm_vm_op op = kmalloc(sizeof(op), GFP_KERNEL);
		368	*op = _op;
		369	list_add_tail(&op->node, &arg->job->vm_ops);
		370
		371	if (op->obj)
		372	drm_gem_object_get(op->obj);
		373	}
		374
		375	static struct drm_gpuva *
		376	vma_from_op(struct op_arg arg, struct drm_gpuva_op_map op)
		377	{
		378	return msm_gem_vma_new(arg->job->vm, op->gem.obj, op->gem.offset,
		379	op->va.addr, op->va.addr + op->va.range);
		380	}
		381
		382	static int
		383	msm_gem_vm_sm_step_map(struct drm_gpuva_op op, void arg)
		384	{
		385	struct drm_gem_object *obj = op->map.gem.obj;
		386	struct drm_gpuva *vma;
		387	struct sg_table *sgt;
		388	unsigned prot;
		389
		390	vma = vma_from_op(arg, &op->map);
		391	if (WARN_ON(IS_ERR(vma)))
		392	return PTR_ERR(vma);
		393
		394	vm_dbg("%p:%p:%p: %016llx %016llx", vma->vm, vma, vma->gem.obj,
		395	vma->va.addr, vma->va.range);
		396
		397	vma->flags = ((struct op_arg *)arg)->flags;
		398
		399	if (obj) {
		400	sgt = to_msm_bo(obj)->sgt;
		401	prot = msm_gem_prot(obj);
		402	} else {
		403	sgt = NULL;
		404	prot = IOMMU_READ \| IOMMU_WRITE;
		405	}
		406
		407	vm_op_enqueue(arg, (struct msm_vm_op){
		408	.op = MSM_VM_OP_MAP,
		409	.map = {
		410	.sgt = sgt,
		411	.iova = vma->va.addr,
		412	.range = vma->va.range,
		413	.offset = vma->gem.offset,
		414	.prot = prot,
		415	},
		416	.obj = vma->gem.obj,
		417	});
		418
		419	to_msm_vma(vma)->mapped = true;
		420
		421	return 0;
		422	}
		423
		424	static int
		425	msm_gem_vm_sm_step_remap(struct drm_gpuva_op op, void arg)
		426	{
		427	struct msm_vm_bind_job job = ((struct op_arg )arg)->job;
		428	struct drm_gpuvm *vm = job->vm;
		429	struct drm_gpuva *orig_vma = op->remap.unmap->va;
		430	struct drm_gpuva prev_vma = NULL, next_vma = NULL;
		431	struct drm_gpuvm_bo *vm_bo = orig_vma->vm_bo;
		432	bool mapped = to_msm_vma(orig_vma)->mapped;
		433	unsigned flags;
		434
		435	vm_dbg("orig_vma: %p:%p:%p: %016llx %016llx", vm, orig_vma,
		436	orig_vma->gem.obj, orig_vma->va.addr, orig_vma->va.range);
		437
		438	if (mapped) {
		439	uint64_t unmap_start, unmap_range;
		440
		441	drm_gpuva_op_remap_to_unmap_range(&op->remap, &unmap_start, &unmap_range);
		442
		443	vm_op_enqueue(arg, (struct msm_vm_op){
		444	.op = MSM_VM_OP_UNMAP,
		445	.unmap = {
		446	.iova = unmap_start,
		447	.range = unmap_range,
		448	},
		449	.obj = orig_vma->gem.obj,
		450	});
		451
		452	/*
		453	* Part of this GEM obj is still mapped, but we're going to kill the
		454	* existing VMA and replace it with one or two new ones (ie. two if
		455	* the unmapped range is in the middle of the existing (unmap) VMA).
		456	* So just set the state to unmapped:
		457	*/
		458	to_msm_vma(orig_vma)->mapped = false;
		459	}
		460
		461	/*
		462	* Hold a ref to the vm_bo between the msm_gem_vma_close() and the
		463	* creation of the new prev/next vma's, in case the vm_bo is tracked
		464	* in the VM's evict list:
		465	*/
		466	if (vm_bo)
		467	drm_gpuvm_bo_get(vm_bo);
		468
		469	/*
		470	* The prev_vma and/or next_vma are replacing the unmapped vma, and
		471	* therefore should preserve it's flags:
		472	*/
		473	flags = orig_vma->flags;
		474
		475	msm_gem_vma_close(orig_vma);
		476
		477	if (op->remap.prev) {
		478	prev_vma = vma_from_op(arg, op->remap.prev);
		479	if (WARN_ON(IS_ERR(prev_vma)))
		480	return PTR_ERR(prev_vma);
		481
		482	vm_dbg("prev_vma: %p:%p: %016llx %016llx", vm, prev_vma, prev_vma->va.addr, prev_vma->va.range);
		483	to_msm_vma(prev_vma)->mapped = mapped;
		484	prev_vma->flags = flags;
		485	}
		486
		487	if (op->remap.next) {
		488	next_vma = vma_from_op(arg, op->remap.next);
		489	if (WARN_ON(IS_ERR(next_vma)))
		490	return PTR_ERR(next_vma);
		491
		492	vm_dbg("next_vma: %p:%p: %016llx %016llx", vm, next_vma, next_vma->va.addr, next_vma->va.range);
		493	to_msm_vma(next_vma)->mapped = mapped;
		494	next_vma->flags = flags;
		495	}
		496
		497	if (!mapped)
		498	drm_gpuvm_bo_evict(vm_bo, true);
		499
		500	/* Drop the previous ref: */
		501	drm_gpuvm_bo_put(vm_bo);
		502
		503	return 0;
		504	}
		505
		506	static int
		507	msm_gem_vm_sm_step_unmap(struct drm_gpuva_op op, void arg)
		508	{
		509	struct drm_gpuva *vma = op->unmap.va;
		510	struct msm_gem_vma *msm_vma = to_msm_vma(vma);
		511
		512	vm_dbg("%p:%p:%p: %016llx %016llx", vma->vm, vma, vma->gem.obj,
		513	vma->va.addr, vma->va.range);
		514
		515	if (!msm_vma->mapped)
		516	goto out_close;
		517
		518	vm_op_enqueue(arg, (struct msm_vm_op){
		519	.op = MSM_VM_OP_UNMAP,
		520	.unmap = {
		521	.iova = vma->va.addr,
		522	.range = vma->va.range,
		523	},
		524	.obj = vma->gem.obj,
		525	});
		526
		527	msm_vma->mapped = false;
		528
		529	out_close:
		530	msm_gem_vma_close(vma);
		531
		532	return 0;
		533	}
		534
203	static const struct drm_gpuvm_ops msm_gpuvm_ops = {	535	static const struct drm_gpuvm_ops msm_gpuvm_ops = {
204	.vm_free = msm_gem_vm_free,	536	.vm_free = msm_gem_vm_free,
		537	.vm_bo_validate = msm_gem_vm_bo_validate,
		538	.sm_step_map = msm_gem_vm_sm_step_map,
		539	.sm_step_remap = msm_gem_vm_sm_step_remap,
		540	.sm_step_unmap = msm_gem_vm_sm_step_unmap,
205	};	541	};
206		542
		543	static struct dma_fence *
		544	msm_vma_job_run(struct drm_sched_job *_job)
		545	{
		546	struct msm_vm_bind_job *job = to_msm_vm_bind_job(_job);
		547	struct msm_gem_vm *vm = to_msm_vm(job->vm);
		548	struct drm_gem_object *obj;
		549	int ret = vm->unusable ? -EINVAL : 0;
		550
		551	vm_dbg("");
		552
		553	mutex_lock(&vm->mmu_lock);
		554	vm->mmu->prealloc = &job->prealloc;
		555
		556	while (!list_empty(&job->vm_ops)) {
		557	struct msm_vm_op *op =
		558	list_first_entry(&job->vm_ops, struct msm_vm_op, node);
		559
		560	switch (op->op) {
		561	case MSM_VM_OP_MAP:
		562	/*
		563	* On error, stop trying to map new things.. but we
		564	* still want to process the unmaps (or in particular,
		565	* the drm_gem_object_put()s)
		566	*/
		567	if (!ret)
		568	ret = vm_map_op(vm, &op->map);
		569	break;
		570	case MSM_VM_OP_UNMAP:
		571	vm_unmap_op(vm, &op->unmap);
		572	break;
		573	}
		574	drm_gem_object_put(op->obj);
		575	list_del(&op->node);
		576	kfree(op);
		577	}
		578
		579	vm->mmu->prealloc = NULL;
		580	mutex_unlock(&vm->mmu_lock);
		581
		582	/*
		583	* We failed to perform at least _some_ of the pgtable updates, so
		584	* now the VM is in an undefined state. Game over!
		585	*/
		586	if (ret)
		587	vm->unusable = true;
		588
		589	job_foreach_bo (obj, job) {
		590	msm_gem_lock(obj);
		591	msm_gem_unpin_locked(obj);
		592	msm_gem_unlock(obj);
		593	}
		594
		595	/* VM_BIND ops are synchronous, so no fence to wait on: */
		596	return NULL;
		597	}
		598
		599	static void
		600	msm_vma_job_free(struct drm_sched_job *_job)
		601	{
		602	struct msm_vm_bind_job *job = to_msm_vm_bind_job(_job);
		603	struct msm_gem_vm *vm = to_msm_vm(job->vm);
		604	struct drm_gem_object *obj;
		605
		606	vm->mmu->funcs->prealloc_cleanup(vm->mmu, &job->prealloc);
		607
		608	drm_sched_job_cleanup(_job);
		609
		610	job_foreach_bo (obj, job)
		611	drm_gem_object_put(obj);
		612
		613	msm_submitqueue_put(job->queue);
		614	dma_fence_put(job->fence);
		615
		616	/* In error paths, we could have unexecuted ops: */
		617	while (!list_empty(&job->vm_ops)) {
		618	struct msm_vm_op *op =
		619	list_first_entry(&job->vm_ops, struct msm_vm_op, node);
		620	list_del(&op->node);
		621	kfree(op);
		622	}
		623
		624	kfree(job);
		625	}
		626
207	static const struct drm_sched_backend_ops msm_vm_bind_ops = {	627	static const struct drm_sched_backend_ops msm_vm_bind_ops = {
		628	.run_job = msm_vma_job_run,
		629	.free_job = msm_vma_job_free
208	};	630	};
209		631
210	/**	632	/**
@@ -268,6 +690,7 @@ msm_gem_vm_create(struct drm_device drm, struct msm_mmu mmu, const char *name, · 57 unmodified lines
268	drm_gem_object_put(dummy_gem);	690	drm_gem_object_put(dummy_gem);
269		691
270	vm->mmu = mmu;	692	vm->mmu = mmu;
		693	mutex_init(&vm->mmu_lock);
271	vm->managed = managed;	694	vm->managed = managed;
272		695
273	drm_mm_init(&vm->mm, va_start, va_size);	696	drm_mm_init(&vm->mm, va_start, va_size);
@@ -280,7 +703,6 @@ err_free_dummy: · 6 unmodified lines
280	err_free_vm:	703	err_free_vm:
281	kfree(vm);	704	kfree(vm);
282	return ERR_PTR(ret);	705	return ERR_PTR(ret);
283
284	}	706	}
285		707
286	/**	708	/**
@@ -296,6 +718,7 @@ msm_gem_vm_close(struct drm_gpuvm *gpuvm) · 9 unmodified lines
296	{	718	{
297	struct msm_gem_vm *vm = to_msm_vm(gpuvm);	719	struct msm_gem_vm *vm = to_msm_vm(gpuvm);
298	struct drm_gpuva vma, tmp;	720	struct drm_gpuva vma, tmp;
		721	struct drm_exec exec;
299		722
300	/*	723	/*
301	* For kernel managed VMs, the VMAs are torn down when the handle is	724	* For kernel managed VMs, the VMAs are torn down when the handle is
@@ -312,22 +735,656 @@ msm_gem_vm_close(struct drm_gpuvm *gpuvm) · 10 unmodified lines
312	drm_sched_fini(&vm->sched);	735	drm_sched_fini(&vm->sched);
313		736
314	/* Tear down any remaining mappings: */	737	/* Tear down any remaining mappings: */
315	dma_resv_lock(drm_gpuvm_resv(gpuvm), NULL);	738	drm_exec_init(&exec, 0, 2);
316	drm_gpuvm_for_each_va_safe (vma, tmp, gpuvm) {	739	drm_exec_until_all_locked (&exec) {
317	struct drm_gem_object *obj = vma->gem.obj;	740	drm_exec_lock_obj(&exec, drm_gpuvm_resv_obj(gpuvm));
		741	drm_exec_retry_on_contention(&exec);
318		742
319	if (obj && obj->resv != drm_gpuvm_resv(gpuvm)) {	743	drm_gpuvm_for_each_va_safe (vma, tmp, gpuvm) {
320	drm_gem_object_get(obj);	744	struct drm_gem_object *obj = vma->gem.obj;
321	msm_gem_lock(obj);	745
		746	/*
		747	* MSM_BO_NO_SHARE objects share the same resv as the
		748	* VM, in which case the obj is already locked:
		749	*/
		750	if (obj && (obj->resv == drm_gpuvm_resv(gpuvm)))
		751	obj = NULL;
		752
		753	if (obj) {
		754	drm_exec_lock_obj(&exec, obj);
		755	drm_exec_retry_on_contention(&exec);
		756	}
		757
		758	msm_gem_vma_unmap(vma);
		759	msm_gem_vma_close(vma);
		760
		761	if (obj) {
		762	drm_exec_unlock_obj(&exec, obj);
		763	}
322	}	764	}
		765	}
		766	drm_exec_fini(&exec);
		767	}
		768
		769
		770	static struct msm_vm_bind_job *
		771	vm_bind_job_create(struct drm_device dev, struct drm_file file,
		772	struct msm_gpu_submitqueue *queue, uint32_t nr_ops)
		773	{
		774	struct msm_vm_bind_job *job;
		775	uint64_t sz;
		776	int ret;
		777
		778	sz = struct_size(job, ops, nr_ops);
		779
		780	if (sz > SIZE_MAX)
		781	return ERR_PTR(-ENOMEM);
		782
		783	job = kzalloc(sz, GFP_KERNEL \| __GFP_NOWARN);
		784	if (!job)
		785	return ERR_PTR(-ENOMEM);
		786
		787	ret = drm_sched_job_init(&job->base, queue->entity, 1, queue,
		788	file->client_id);
		789	if (ret) {
		790	kfree(job);
		791	return ERR_PTR(ret);
		792	}
323		793
324	msm_gem_vma_unmap(vma);	794	job->vm = msm_context_vm(dev, queue->ctx);
325	msm_gem_vma_close(vma);	795	job->queue = queue;
		796	INIT_LIST_HEAD(&job->vm_ops);
326		797
327	if (obj && obj->resv != drm_gpuvm_resv(gpuvm)) {	798	return job;
328	msm_gem_unlock(obj);	799	}
329	drm_gem_object_put(obj);	800
		801	static bool invalid_alignment(uint64_t addr)
		802	{
		803	/*
		804	* Technically this is about GPU alignment, not CPU alignment. But
		805	* I've not seen any qcom SoC where the SMMU does not support the
		806	* CPU's smallest page size.
		807	*/
		808	return !PAGE_ALIGNED(addr);
		809	}
		810
		811	static int
		812	lookup_op(struct msm_vm_bind_job job, const struct drm_msm_vm_bind_op op)
		813	{
		814	struct drm_device *dev = job->vm->drm;
		815	int i = job->nr_ops++;
		816	int ret = 0;
		817
		818	job->ops[i].op = op->op;
		819	job->ops[i].handle = op->handle;
		820	job->ops[i].obj_offset = op->obj_offset;
		821	job->ops[i].iova = op->iova;
		822	job->ops[i].range = op->range;
		823	job->ops[i].flags = op->flags;
		824
		825	if (op->flags & ~MSM_VM_BIND_OP_FLAGS)
		826	ret = UERR(EINVAL, dev, "invalid flags: %x\n", op->flags);
		827
		828	if (invalid_alignment(op->iova))
		829	ret = UERR(EINVAL, dev, "invalid address: %016llx\n", op->iova);
		830
		831	if (invalid_alignment(op->obj_offset))
		832	ret = UERR(EINVAL, dev, "invalid bo_offset: %016llx\n", op->obj_offset);
		833
		834	if (invalid_alignment(op->range))
		835	ret = UERR(EINVAL, dev, "invalid range: %016llx\n", op->range);
		836
		837	if (!drm_gpuvm_range_valid(job->vm, op->iova, op->range))
		838	ret = UERR(EINVAL, dev, "invalid range: %016llx, %016llx\n", op->iova, op->range);
		839
		840	/*
		841	* MAP must specify a valid handle. But the handle MBZ for
		842	* UNMAP or MAP_NULL.
		843	*/
		844	if (op->op == MSM_VM_BIND_OP_MAP) {
		845	if (!op->handle)
		846	ret = UERR(EINVAL, dev, "invalid handle\n");
		847	} else if (op->handle) {
		848	ret = UERR(EINVAL, dev, "handle must be zero\n");
		849	}
		850
		851	switch (op->op) {
		852	case MSM_VM_BIND_OP_MAP:
		853	case MSM_VM_BIND_OP_MAP_NULL:
		854	case MSM_VM_BIND_OP_UNMAP:
		855	break;
		856	default:
		857	ret = UERR(EINVAL, dev, "invalid op: %u\n", op->op);
		858	break;
		859	}
		860
		861	return ret;
		862	}
		863
		864	/*
		865	* ioctl parsing, parameter validation, and GEM handle lookup
		866	*/
		867	static int
		868	vm_bind_job_lookup_ops(struct msm_vm_bind_job job, struct drm_msm_vm_bind args,
		869	struct drm_file file, int nr_bos)
		870	{
		871	struct drm_device *dev = job->vm->drm;
		872	int ret = 0;
		873	int cnt = 0;
		874
		875	if (args->nr_ops == 1) {
		876	/* Single op case, the op is inlined: */
		877	ret = lookup_op(job, &args->op);
		878	} else {
		879	for (unsigned i = 0; i < args->nr_ops; i++) {
		880	struct drm_msm_vm_bind_op op;
		881	void __user *userptr =
		882	u64_to_user_ptr(args->ops + (i * sizeof(op)));
		883
		884	/* make sure we don't have garbage flags, in case we hit
		885	* error path before flags is initialized:
		886	*/
		887	job->ops[i].flags = 0;
		888
		889	if (copy_from_user(&op, userptr, sizeof(op))) {
		890	ret = -EFAULT;
		891	break;
		892	}
		893
		894	ret = lookup_op(job, &op);
		895	if (ret)
		896	break;
		897	}
		898	}
		899
		900	if (ret) {
		901	job->nr_ops = 0;
		902	goto out;
		903	}
		904
		905	spin_lock(&file->table_lock);
		906
		907	for (unsigned i = 0; i < args->nr_ops; i++) {
		908	struct drm_gem_object *obj;
		909
		910	if (!job->ops[i].handle) {
		911	job->ops[i].obj = NULL;
		912	continue;
		913	}
		914
		915	/*
		916	* normally use drm_gem_object_lookup(), but for bulk lookup
		917	* all under single table_lock just hit object_idr directly:
		918	*/
		919	obj = idr_find(&file->object_idr, job->ops[i].handle);
		920	if (!obj) {
		921	ret = UERR(EINVAL, dev, "invalid handle %u at index %u\n", job->ops[i].handle, i);
		922	goto out_unlock;
		923	}
		924
		925	drm_gem_object_get(obj);
		926
		927	job->ops[i].obj = obj;
		928	cnt++;
		929	}
		930
		931	*nr_bos = cnt;
		932
		933	out_unlock:
		934	spin_unlock(&file->table_lock);
		935
		936	out:
		937	return ret;
		938	}
		939
		940	static void
		941	prealloc_count(struct msm_vm_bind_job *job,
		942	struct msm_vm_bind_op *first,
		943	struct msm_vm_bind_op *last)
		944	{
		945	struct msm_mmu *mmu = to_msm_vm(job->vm)->mmu;
		946
		947	if (!first)
		948	return;
		949
		950	uint64_t start_iova = first->iova;
		951	uint64_t end_iova = last->iova + last->range;
		952
		953	mmu->funcs->prealloc_count(mmu, &job->prealloc, start_iova, end_iova - start_iova);
		954	}
		955
		956	static bool
		957	ops_are_same_pte(struct msm_vm_bind_op first, struct msm_vm_bind_op next)
		958	{
		959	/*
		960	* Last level pte covers 2MB.. so we should merge two ops, from
		961	* the PoV of figuring out how much pgtable pages to pre-allocate
		962	* if they land in the same 2MB range:
		963	*/
		964	uint64_t pte_mask = ~(SZ_2M - 1);
		965	return ((first->iova + first->range) & pte_mask) == (next->iova & pte_mask);
		966	}
		967
		968	/*
		969	* Determine the amount of memory to prealloc for pgtables. For sparse images,
		970	* in particular, userspace plays some tricks with the order of page mappings
		971	* to get the desired swizzle pattern, resulting in a large # of tiny MAP ops.
		972	* So detect when multiple MAP operations are physically contiguous, and count
		973	* them as a single mapping. Otherwise the prealloc_count() will not realize
		974	* they can share pagetable pages and vastly overcount.
		975	*/
		976	static void
		977	vm_bind_prealloc_count(struct msm_vm_bind_job *job)
		978	{
		979	struct msm_vm_bind_op first = NULL, last = NULL;
		980
		981	for (int i = 0; i < job->nr_ops; i++) {
		982	struct msm_vm_bind_op *op = &job->ops[i];
		983
		984	/* We only care about MAP/MAP_NULL: */
		985	if (op->op == MSM_VM_BIND_OP_UNMAP)
		986	continue;
		987
		988	/*
		989	* If op is contiguous with last in the current range, then
		990	* it becomes the new last in the range and we continue
		991	* looping:
		992	*/
		993	if (last && ops_are_same_pte(last, op)) {
		994	last = op;
		995	continue;
		996	}
		997
		998	/*
		999	* If op is not contiguous with the current range, flush
		1000	* the current range and start anew:
		1001	*/
		1002	prealloc_count(job, first, last);
		1003	first = last = op;
		1004	}
		1005
		1006	/* Flush the remaining range: */
		1007	prealloc_count(job, first, last);
		1008	}
		1009
		1010	/*
		1011	* Lock VM and GEM objects
		1012	*/
		1013	static int
		1014	vm_bind_job_lock_objects(struct msm_vm_bind_job job, struct drm_exec exec)
		1015	{
		1016	int ret;
		1017
		1018	/* Lock VM and objects: */
		1019	drm_exec_until_all_locked (exec) {
		1020	ret = drm_exec_lock_obj(exec, drm_gpuvm_resv_obj(job->vm));
		1021	drm_exec_retry_on_contention(exec);
		1022	if (ret)
		1023	return ret;
		1024
		1025	for (unsigned i = 0; i < job->nr_ops; i++) {
		1026	const struct msm_vm_bind_op *op = &job->ops[i];
		1027
		1028	switch (op->op) {
		1029	case MSM_VM_BIND_OP_UNMAP:
		1030	ret = drm_gpuvm_sm_unmap_exec_lock(job->vm, exec,
		1031	op->iova,
		1032	op->obj_offset);
		1033	break;
		1034	case MSM_VM_BIND_OP_MAP:
		1035	case MSM_VM_BIND_OP_MAP_NULL:
		1036	ret = drm_gpuvm_sm_map_exec_lock(job->vm, exec, 1,
		1037	op->iova, op->range,
		1038	op->obj, op->obj_offset);
		1039	break;
		1040	default:
		1041	/*
		1042	* lookup_op() should have already thrown an error for
		1043	* invalid ops
		1044	*/
		1045	WARN_ON("unreachable");
		1046	}
		1047
		1048	drm_exec_retry_on_contention(exec);
		1049	if (ret)
		1050	return ret;
		1051	}
		1052	}
		1053
		1054	return 0;
		1055	}
		1056
		1057	/*
		1058	* Pin GEM objects, ensuring that we have backing pages. Pinning will move
		1059	* the object to the pinned LRU so that the shrinker knows to first consider
		1060	* other objects for evicting.
		1061	*/
		1062	static int
		1063	vm_bind_job_pin_objects(struct msm_vm_bind_job *job)
		1064	{
		1065	struct drm_gem_object *obj;
		1066
		1067	/*
		1068	* First loop, before holding the LRU lock, avoids holding the
		1069	* LRU lock while calling msm_gem_pin_vma_locked (which could
		1070	* trigger get_pages())
		1071	*/
		1072	job_foreach_bo (obj, job) {
		1073	struct page **pages;
		1074
		1075	pages = msm_gem_get_pages_locked(obj, MSM_MADV_WILLNEED);
		1076	if (IS_ERR(pages))
		1077	return PTR_ERR(pages);
		1078	}
		1079
		1080	struct msm_drm_private *priv = job->vm->drm->dev_private;
		1081
		1082	/*
		1083	* A second loop while holding the LRU lock (a) avoids acquiring/dropping
		1084	* the LRU lock for each individual bo, while (b) avoiding holding the
		1085	* LRU lock while calling msm_gem_pin_vma_locked() (which could trigger
		1086	* get_pages() which could trigger reclaim.. and if we held the LRU lock
		1087	* could trigger deadlock with the shrinker).
		1088	*/
		1089	mutex_lock(&priv->lru.lock);
		1090	job_foreach_bo (obj, job)
		1091	msm_gem_pin_obj_locked(obj);
		1092	mutex_unlock(&priv->lru.lock);
		1093
		1094	job->bos_pinned = true;
		1095
		1096	return 0;
		1097	}
		1098
		1099	/*
		1100	* Unpin GEM objects. Normally this is done after the bind job is run.
		1101	*/
		1102	static void
		1103	vm_bind_job_unpin_objects(struct msm_vm_bind_job *job)
		1104	{
		1105	struct drm_gem_object *obj;
		1106
		1107	if (!job->bos_pinned)
		1108	return;
		1109
		1110	job_foreach_bo (obj, job)
		1111	msm_gem_unpin_locked(obj);
		1112
		1113	job->bos_pinned = false;
		1114	}
		1115
		1116	/*
		1117	* Pre-allocate pgtable memory, and translate the VM bind requests into a
		1118	* sequence of pgtable updates to be applied asynchronously.
		1119	*/
		1120	static int
		1121	vm_bind_job_prepare(struct msm_vm_bind_job *job)
		1122	{
		1123	struct msm_gem_vm *vm = to_msm_vm(job->vm);
		1124	struct msm_mmu *mmu = vm->mmu;
		1125	int ret;
		1126
		1127	ret = mmu->funcs->prealloc_allocate(mmu, &job->prealloc);
		1128	if (ret)
		1129	return ret;
		1130
		1131	for (unsigned i = 0; i < job->nr_ops; i++) {
		1132	const struct msm_vm_bind_op *op = &job->ops[i];
		1133	struct op_arg arg = {
		1134	.job = job,
		1135	};
		1136
		1137	switch (op->op) {
		1138	case MSM_VM_BIND_OP_UNMAP:
		1139	ret = drm_gpuvm_sm_unmap(job->vm, &arg, op->iova,
		1140	op->range);
		1141	break;
		1142	case MSM_VM_BIND_OP_MAP:
		1143	if (op->flags & MSM_VM_BIND_OP_DUMP)
		1144	arg.flags \|= MSM_VMA_DUMP;
		1145	fallthrough;
		1146	case MSM_VM_BIND_OP_MAP_NULL:
		1147	ret = drm_gpuvm_sm_map(job->vm, &arg, op->iova,
		1148	op->range, op->obj, op->obj_offset);
		1149	break;
		1150	default:
		1151	/*
		1152	* lookup_op() should have already thrown an error for
		1153	* invalid ops
		1154	*/
		1155	BUG_ON("unreachable");
		1156	}
		1157
		1158	if (ret) {
		1159	/*
		1160	* If we've already started modifying the vm, we can't
		1161	* adequetly describe to userspace the intermediate
		1162	* state the vm is in. So throw up our hands!
		1163	*/
		1164	if (i > 0)
		1165	vm->unusable = true;
		1166	return ret;
		1167	}
		1168	}
		1169
		1170	return 0;
		1171	}
		1172
		1173	/*
		1174	* Attach fences to the GEM objects being bound. This will signify to
		1175	* the shrinker that they are busy even after dropping the locks (ie.
		1176	* drm_exec_fini())
		1177	*/
		1178	static void
		1179	vm_bind_job_attach_fences(struct msm_vm_bind_job *job)
		1180	{
		1181	for (unsigned i = 0; i < job->nr_ops; i++) {
		1182	struct drm_gem_object *obj = job->ops[i].obj;
		1183
		1184	if (!obj)
		1185	continue;
		1186
		1187	dma_resv_add_fence(obj->resv, job->fence,
		1188	DMA_RESV_USAGE_KERNEL);
		1189	}
		1190	}
		1191
		1192	int
		1193	msm_ioctl_vm_bind(struct drm_device dev, void data, struct drm_file *file)
		1194	{
		1195	struct msm_drm_private *priv = dev->dev_private;
		1196	struct drm_msm_vm_bind *args = data;
		1197	struct msm_context *ctx = file->driver_priv;
		1198	struct msm_vm_bind_job *job = NULL;
		1199	struct msm_gpu *gpu = priv->gpu;
		1200	struct msm_gpu_submitqueue *queue;
		1201	struct msm_syncobj_post_dep *post_deps = NULL;
		1202	struct drm_syncobj **syncobjs_to_reset = NULL;
		1203	struct sync_file *sync_file = NULL;
		1204	struct dma_fence *fence;
		1205	int out_fence_fd = -1;
		1206	int ret, nr_bos = 0;
		1207	unsigned i;
		1208
		1209	if (!gpu)
		1210	return -ENXIO;
		1211
		1212	/*
		1213	* Maybe we could allow just UNMAP ops? OTOH userspace should just
		1214	* immediately close the device file and all will be torn down.
		1215	*/
		1216	if (to_msm_vm(ctx->vm)->unusable)
		1217	return UERR(EPIPE, dev, "context is unusable");
		1218
		1219	/*
		1220	* Technically, you cannot create a VM_BIND submitqueue in the first
		1221	* place, if you haven't opted in to VM_BIND context. But it is
		1222	* cleaner / less confusing, to check this case directly.
		1223	*/
		1224	if (!msm_context_is_vmbind(ctx))
		1225	return UERR(EINVAL, dev, "context does not support vmbind");
		1226
		1227	if (args->flags & ~MSM_VM_BIND_FLAGS)
		1228	return UERR(EINVAL, dev, "invalid flags");
		1229
		1230	queue = msm_submitqueue_get(ctx, args->queue_id);
		1231	if (!queue)
		1232	return -ENOENT;
		1233
		1234	if (!(queue->flags & MSM_SUBMITQUEUE_VM_BIND)) {
		1235	ret = UERR(EINVAL, dev, "Invalid queue type");
		1236	goto out_post_unlock;
		1237	}
		1238
		1239	if (args->flags & MSM_VM_BIND_FENCE_FD_OUT) {
		1240	out_fence_fd = get_unused_fd_flags(O_CLOEXEC);
		1241	if (out_fence_fd < 0) {
		1242	ret = out_fence_fd;
		1243	goto out_post_unlock;
330	}	1244	}
331	}	1245	}
332	dma_resv_unlock(drm_gpuvm_resv(gpuvm));	1246
		1247	job = vm_bind_job_create(dev, file, queue, args->nr_ops);
		1248	if (IS_ERR(job)) {
		1249	ret = PTR_ERR(job);
		1250	goto out_post_unlock;
		1251	}
		1252
		1253	ret = mutex_lock_interruptible(&queue->lock);
		1254	if (ret)
		1255	goto out_post_unlock;
		1256
		1257	if (args->flags & MSM_VM_BIND_FENCE_FD_IN) {
		1258	struct dma_fence *in_fence;
		1259
		1260	in_fence = sync_file_get_fence(args->fence_fd);
		1261
		1262	if (!in_fence) {
		1263	ret = UERR(EINVAL, dev, "invalid in-fence");
		1264	goto out_unlock;
		1265	}
		1266
		1267	ret = drm_sched_job_add_dependency(&job->base, in_fence);
		1268	if (ret)
		1269	goto out_unlock;
		1270	}
		1271
		1272	if (args->in_syncobjs > 0) {
		1273	syncobjs_to_reset = msm_syncobj_parse_deps(dev, &job->base,
		1274	file, args->in_syncobjs,
		1275	args->nr_in_syncobjs,
		1276	args->syncobj_stride);
		1277	if (IS_ERR(syncobjs_to_reset)) {
		1278	ret = PTR_ERR(syncobjs_to_reset);
		1279	goto out_unlock;
		1280	}
		1281	}
		1282
		1283	if (args->out_syncobjs > 0) {
		1284	post_deps = msm_syncobj_parse_post_deps(dev, file,
		1285	args->out_syncobjs,
		1286	args->nr_out_syncobjs,
		1287	args->syncobj_stride);
		1288	if (IS_ERR(post_deps)) {
		1289	ret = PTR_ERR(post_deps);
		1290	goto out_unlock;
		1291	}
		1292	}
		1293
		1294	ret = vm_bind_job_lookup_ops(job, args, file, &nr_bos);
		1295	if (ret)
		1296	goto out_unlock;
		1297
		1298	vm_bind_prealloc_count(job);
		1299
		1300	struct drm_exec exec;
		1301	unsigned flags = DRM_EXEC_IGNORE_DUPLICATES \| DRM_EXEC_INTERRUPTIBLE_WAIT;
		1302	drm_exec_init(&exec, flags, nr_bos + 1);
		1303
		1304	ret = vm_bind_job_lock_objects(job, &exec);
		1305	if (ret)
		1306	goto out;
		1307
		1308	ret = vm_bind_job_pin_objects(job);
		1309	if (ret)
		1310	goto out;
		1311
		1312	ret = vm_bind_job_prepare(job);
		1313	if (ret)
		1314	goto out;
		1315
		1316	drm_sched_job_arm(&job->base);
		1317
		1318	job->fence = dma_fence_get(&job->base.s_fence->finished);
		1319
		1320	if (args->flags & MSM_VM_BIND_FENCE_FD_OUT) {
		1321	sync_file = sync_file_create(job->fence);
		1322	if (!sync_file) {
		1323	ret = -ENOMEM;
		1324	} else {
		1325	fd_install(out_fence_fd, sync_file->file);
		1326	args->fence_fd = out_fence_fd;
		1327	}
		1328	}
		1329
		1330	if (ret)
		1331	goto out;
		1332
		1333	vm_bind_job_attach_fences(job);
		1334
		1335	/*
		1336	* The job can be free'd (and fence unref'd) at any point after
		1337	* drm_sched_entity_push_job(), so we need to hold our own ref
		1338	*/
		1339	fence = dma_fence_get(job->fence);
		1340
		1341	drm_sched_entity_push_job(&job->base);
		1342
		1343	msm_syncobj_reset(syncobjs_to_reset, args->nr_in_syncobjs);
		1344	msm_syncobj_process_post_deps(post_deps, args->nr_out_syncobjs, fence);
		1345
		1346	dma_fence_put(fence);
		1347
		1348	out:
		1349	if (ret)
		1350	vm_bind_job_unpin_objects(job);
		1351
		1352	drm_exec_fini(&exec);
		1353	out_unlock:
		1354	mutex_unlock(&queue->lock);
		1355	out_post_unlock:
		1356	if (ret && (out_fence_fd >= 0)) {
		1357	put_unused_fd(out_fence_fd);
		1358	if (sync_file)
		1359	fput(sync_file->file);
		1360	}
		1361
		1362	if (!IS_ERR_OR_NULL(job)) {
		1363	if (ret)
		1364	msm_vma_job_free(&job->base);
		1365	} else {
		1366	/*
		1367	* If the submit hasn't yet taken ownership of the queue
		1368	* then we need to drop the reference ourself:
		1369	*/
		1370	msm_submitqueue_put(queue);
		1371	}
		1372
		1373	if (!IS_ERR_OR_NULL(post_deps)) {
		1374	for (i = 0; i < args->nr_out_syncobjs; ++i) {
		1375	kfree(post_deps[i].chain);
		1376	drm_syncobj_put(post_deps[i].syncobj);
		1377	}
		1378	kfree(post_deps);
		1379	}
		1380
		1381	if (!IS_ERR_OR_NULL(syncobjs_to_reset)) {
		1382	for (i = 0; i < args->nr_in_syncobjs; ++i) {
		1383	if (syncobjs_to_reset[i])
		1384	drm_syncobj_put(syncobjs_to_reset[i]);
		1385	}
		1386	kfree(syncobjs_to_reset);
		1387	}
		1388
		1389	return ret;
333	}	1390	}

include/uapi/drm/msm_drm.h+73−1

@@ -272,7 +272,10 @@ struct drm_msm_gem_submit_cmd { · 271 unmodified lines
272	__u32 size; /* in, cmdstream size */	272	__u32 size; /* in, cmdstream size */
273	__u32 pad;	273	__u32 pad;
274	__u32 nr_relocs; /* in, number of submit_reloc's */	274	__u32 nr_relocs; /* in, number of submit_reloc's */
275	__u64 relocs; /* in, ptr to array of submit_reloc's */	275	union {
		276	__u64 relocs; /* in, ptr to array of submit_reloc's */
		277	__u64 iova; /* cmdstream address (for VM_BIND contexts) */
		278	};
276	};	279	};
277		280
278	/* Each buffer referenced elsewhere in the cmdstream submit (ie. the	281	/* Each buffer referenced elsewhere in the cmdstream submit (ie. the
@@ -339,7 +342,74 @@ struct drm_msm_gem_submit { · 60 unmodified lines
339	__u32 nr_out_syncobjs; /* in, number of entries in out_syncobj. */	342	__u32 nr_out_syncobjs; /* in, number of entries in out_syncobj. */
340	__u32 syncobj_stride; /* in, stride of syncobj arrays. */	343	__u32 syncobj_stride; /* in, stride of syncobj arrays. */
341	__u32 pad; /in, reserved for future use, always 0. /	344	__u32 pad; /in, reserved for future use, always 0. /
		345	};
		346
		347	#define MSM_VM_BIND_OP_UNMAP 0
		348	#define MSM_VM_BIND_OP_MAP 1
		349	#define MSM_VM_BIND_OP_MAP_NULL 2
		350
		351	#define MSM_VM_BIND_OP_DUMP 1
		352	#define MSM_VM_BIND_OP_FLAGS ( \
		353	MSM_VM_BIND_OP_DUMP \| \
		354	0)
342		355
		356	/**
		357	* struct drm_msm_vm_bind_op - bind/unbind op to run
		358	*/
		359	struct drm_msm_vm_bind_op {
		360	/** @op: one of MSM_VM_BIND_OP_x */
		361	__u32 op;
		362	/** @handle: GEM object handle, MBZ for UNMAP or MAP_NULL */
		363	__u32 handle;
		364	/** @obj_offset: Offset into GEM object, MBZ for UNMAP or MAP_NULL */
		365	__u64 obj_offset;
		366	/** @iova: Address to operate on */
		367	__u64 iova;
		368	/** @range: Number of bites to to map/unmap */
		369	__u64 range;
		370	/** @flags: Bitmask of MSM_VM_BIND_OP_FLAG_x */
		371	__u32 flags;
		372	/** @pad: MBZ */
		373	__u32 pad;
		374	};
		375
		376	#define MSM_VM_BIND_FENCE_FD_IN 0x00000001
		377	#define MSM_VM_BIND_FENCE_FD_OUT 0x00000002
		378	#define MSM_VM_BIND_FLAGS ( \
		379	MSM_VM_BIND_FENCE_FD_IN \| \
		380	MSM_VM_BIND_FENCE_FD_OUT \| \
		381	0)
		382
		383	/**
		384	* struct drm_msm_vm_bind - Input of &DRM_IOCTL_MSM_VM_BIND
		385	*/
		386	struct drm_msm_vm_bind {
		387	/** @flags: in, bitmask of MSM_VM_BIND_x */
		388	__u32 flags;
		389	/** @nr_ops: the number of bind ops in this ioctl */
		390	__u32 nr_ops;
		391	/** @fence_fd: in/out fence fd (see MSM_VM_BIND_FENCE_FD_IN/OUT) */
		392	__s32 fence_fd;
		393	/** @queue_id: in, submitqueue id */
		394	__u32 queue_id;
		395	/** @in_syncobjs: in, ptr to array of drm_msm_gem_syncobj */
		396	__u64 in_syncobjs;
		397	/** @out_syncobjs: in, ptr to array of drm_msm_gem_syncobj */
		398	__u64 out_syncobjs;
		399	/** @nr_in_syncobjs: in, number of entries in in_syncobj */
		400	__u32 nr_in_syncobjs;
		401	/** @nr_out_syncobjs: in, number of entries in out_syncobj */
		402	__u32 nr_out_syncobjs;
		403	/** @syncobj_stride: in, stride of syncobj arrays */
		404	__u32 syncobj_stride;
		405	/** @op_stride: sizeof each struct drm_msm_vm_bind_op in @ops */
		406	__u32 op_stride;
		407	union {
		408	/** @op: used if num_ops == 1 */
		409	struct drm_msm_vm_bind_op op;
		410	/** @ops: userptr to array of drm_msm_vm_bind_op if num_ops > 1 */
		411	__u64 ops;
		412	};
343	};	413	};
344		414
345	#define MSM_WAIT_FENCE_BOOST 0x00000001	415	#define MSM_WAIT_FENCE_BOOST 0x00000001
@@ -435,6 +505,7 @@ struct drm_msm_submitqueue_query { · 89 unmodified lines
435	#define DRM_MSM_SUBMITQUEUE_NEW 0x0A	505	#define DRM_MSM_SUBMITQUEUE_NEW 0x0A
436	#define DRM_MSM_SUBMITQUEUE_CLOSE 0x0B	506	#define DRM_MSM_SUBMITQUEUE_CLOSE 0x0B
437	#define DRM_MSM_SUBMITQUEUE_QUERY 0x0C	507	#define DRM_MSM_SUBMITQUEUE_QUERY 0x0C
		508	#define DRM_MSM_VM_BIND 0x0D
438		509
439	#define DRM_IOCTL_MSM_GET_PARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_MSM_GET_PARAM, struct drm_msm_param)	510	#define DRM_IOCTL_MSM_GET_PARAM DRM_IOWR(DRM_COMMAND_BASE + DRM_MSM_GET_PARAM, struct drm_msm_param)
440	#define DRM_IOCTL_MSM_SET_PARAM DRM_IOW (DRM_COMMAND_BASE + DRM_MSM_SET_PARAM, struct drm_msm_param)	511	#define DRM_IOCTL_MSM_SET_PARAM DRM_IOW (DRM_COMMAND_BASE + DRM_MSM_SET_PARAM, struct drm_msm_param)
@@ -448,6 +519,7 @@ struct drm_msm_submitqueue_query { · 7 unmodified lines
448	#define DRM_IOCTL_MSM_SUBMITQUEUE_NEW DRM_IOWR(DRM_COMMAND_BASE + DRM_MSM_SUBMITQUEUE_NEW, struct drm_msm_submitqueue)	519	#define DRM_IOCTL_MSM_SUBMITQUEUE_NEW DRM_IOWR(DRM_COMMAND_BASE + DRM_MSM_SUBMITQUEUE_NEW, struct drm_msm_submitqueue)
449	#define DRM_IOCTL_MSM_SUBMITQUEUE_CLOSE DRM_IOW (DRM_COMMAND_BASE + DRM_MSM_SUBMITQUEUE_CLOSE, __u32)	520	#define DRM_IOCTL_MSM_SUBMITQUEUE_CLOSE DRM_IOW (DRM_COMMAND_BASE + DRM_MSM_SUBMITQUEUE_CLOSE, __u32)
450	#define DRM_IOCTL_MSM_SUBMITQUEUE_QUERY DRM_IOW (DRM_COMMAND_BASE + DRM_MSM_SUBMITQUEUE_QUERY, struct drm_msm_submitqueue_query)	521	#define DRM_IOCTL_MSM_SUBMITQUEUE_QUERY DRM_IOW (DRM_COMMAND_BASE + DRM_MSM_SUBMITQUEUE_QUERY, struct drm_msm_submitqueue_query)
		522	#define DRM_IOCTL_MSM_VM_BIND DRM_IOWR(DRM_COMMAND_BASE + DRM_MSM_VM_BIND, struct drm_msm_vm_bind)
451		523
452	#if defined(__cplusplus)	524	#if defined(__cplusplus)
453	}	525	}