lael-labs/linux

Documentation/gpu/index.rst+1−0

@@ -16,6 +16,7 @@ Linux GPU Driver Developer's Guide · 15 unmodified lines
16	vga-switcheroo	16	vga-switcheroo
17	vgaarbiter	17	vgaarbiter
18	todo	18	todo
		19	rfc/index
19		20
20	.. only:: subproject and html	21	.. only:: subproject and html
21		22

Documentation/gpu/rfc/index.rst+17−0

@@ -0,0 +1,17 @@
	1	===============
	2	GPU RFC Section
	3	===============
	4
	5	For complex work, especially new uapi, it is often good to nail the high level
	6	design issues before getting lost in the code details. This section is meant to
	7	host such documentation:
	8
	9	* Each RFC should be a section in this file, explaining the goal and main design
	10	considerations. Especially for uapi make sure you Cc: all relevant project
	11	mailing lists and involved people outside of dri-devel.
	12
	13	* For uapi structures add a file to this directory with and then pull the
	14	kerneldoc in like with real uapi headers.
	15
	16	* Once the code has landed move all the documentation to the right places in
	17	the main core, helper or driver sections.

drivers/gpu/drm/i915/Kconfig.profile+14−0

@@ -1,3 +1,17 @@
		1	config DRM_I915_REQUEST_TIMEOUT
		2	int "Default timeout for requests (ms)"
		3	default 20000 # milliseconds
		4	help
		5	Configures the default timeout after which any user submissions will
		6	be forcefully terminated.
		7
		8	Beware setting this value lower, or close to heartbeat interval
		9	rounded to whole seconds times three, in order to avoid allowing
		10	misbehaving applications causing total rendering failure in unrelated
		11	clients.
		12
		13	May be 0 to disable the timeout.
		14
1	config DRM_I915_FENCE_TIMEOUT	15	config DRM_I915_FENCE_TIMEOUT
2	int "Timeout for unsignaled foreign fences (ms, jiffy granularity)"	16	int "Timeout for unsignaled foreign fences (ms, jiffy granularity)"
3	default 10000 # milliseconds	17	default 10000 # milliseconds

drivers/gpu/drm/i915/Makefile+0−1

@@ -139,7 +139,6 @@ gem-y += \ · 138 unmodified lines
139	gem/i915_gem_dmabuf.o \	139	gem/i915_gem_dmabuf.o \
140	gem/i915_gem_domain.o \	140	gem/i915_gem_domain.o \
141	gem/i915_gem_execbuffer.o \	141	gem/i915_gem_execbuffer.o \
142	gem/i915_gem_fence.o \
143	gem/i915_gem_internal.o \	142	gem/i915_gem_internal.o \
144	gem/i915_gem_object.o \	143	gem/i915_gem_object.o \
145	gem/i915_gem_object_blt.o \	144	gem/i915_gem_object_blt.o \

drivers/gpu/drm/i915/TODO.txt+41−0

@@ -0,0 +1,41 @@
	1	gem/gt TODO items
	2	-----------------
	3
	4	- For discrete memory manager, merge enough dg1 to be able to refactor it to
	5	TTM. Then land pci ids (just in case that turns up an uapi problem). TTM has
	6	improved a lot the past 2 years, there's no reason anymore not to use it.
	7
	8	- Come up with a plan what to do with drm/scheduler and how to get there.
	9
	10	- Roll out dma_fence critical section annotations.
	11
	12	- There's a lot of complexity added past few years to make relocations faster.
	13	That doesn't make sense given hw and gpu apis moved away from this model years
	14	ago:
	15	1. Land a modern pre-bound uapi like VM_BIND
	16	2. Any complexity added in this area past few years which can't be justified
	17	with VM_BIND using userspace should be removed. Looking at amdgpu dma_resv on
	18	the bo and vm, plus some lru locks is all that needed. No complex rcu,
	19	refcounts, caching, ... on everything.
	20	This is the matching task on the vm side compared to ttm/dma_resv on the
	21	backing storage side.
	22
	23	- i915_sw_fence seems to be the main structure for the i915-gem dma_fence model.
	24	How-to-dma_fence is core and drivers really shouldn't build their own world
	25	here, treating everything else as a fixed platform. i915_sw_fence concepts
	26	should be moved to dma_fence, drm/scheduler or atomic commit helpers. Or
	27	removed if dri-devel consensus is that it's not a good idea. Once that's done
	28	maybe even remove it if there's nothing left.
	29
	30	Smaller things:
	31	- i915_utils.h needs to be moved to the right places.
	32
	33	- dma_fence_work should be in drivers/dma-buf
	34
	35	- i915_mm.c should be moved to the right places. Some of the helpers also look a
	36	bit fishy:
	37
	38	https://lore.kernel.org/linux-mm/20210301083320.943079-1-hch@lst.de/
	39
	40	- tasklet helpers in i915_gem.h also look a bit misplaced and should
	41	probably be moved to tasklet headers.

drivers/gpu/drm/i915/display/intel_display.c+44−27

 struct i915_vma *
 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
+			   bool phys_cursor,
 			   const struct i915_ggtt_view *view,
 			   bool uses_fence,
 			   unsigned long *out_flags)
 	struct drm_i915_private *dev_priv = to_i915(dev);
 	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	intel_wakeref_t wakeref;
+	struct i915_gem_ww_ctx ww;
 	struct i915_vma *vma;
 	unsigned int pinctl;
 	u32 alignment;
+	int ret;
 	if (drm_WARN_ON(dev, !i915_gem_object_is_framebuffer(obj)))
 		return ERR_PTR(-EINVAL);
-	alignment = intel_surf_alignment(fb, 0);
+	if (phys_cursor)
+		alignment = intel_cursor_alignment(dev_priv);
+	else
+		alignment = intel_surf_alignment(fb, 0);
 	if (drm_WARN_ON(dev, alignment && !is_power_of_2(alignment)))
 		return ERR_PTR(-EINVAL);
 	if (HAS_GMCH(dev_priv))
 		pinctl |= PIN_MAPPABLE;
-	vma = i915_gem_object_pin_to_display_plane(obj,
+	i915_gem_ww_ctx_init(&ww, true);
-						   alignment, view, pinctl);
+retry:
-	if (IS_ERR(vma))
+	ret = i915_gem_object_lock(obj, &ww);
+	if (!ret && phys_cursor)
+		ret = i915_gem_object_attach_phys(obj, alignment);
+	if (!ret)
+		ret = i915_gem_object_pin_pages(obj);
+	if (ret)
 		goto err;
-	if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
+	if (!ret) {
-		int ret;
+		vma = i915_gem_object_pin_to_display_plane(obj, &ww, alignment,
+							   view, pinctl);
+		if (IS_ERR(vma)) {
+			ret = PTR_ERR(vma);
+			goto err_unpin;
+		}
+	}
+	if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
 		/*
 		 * Install a fence for tiled scan-out. Pre-i965 always needs a
 		 * fence, whereas 965+ only requires a fence if using
 		ret = i915_vma_pin_fence(vma);
 		if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
 			i915_vma_unpin(vma);
-			vma = ERR_PTR(ret);
+			goto err_unpin;
-			goto err;
 		}
+		ret = 0;
-		if (ret == 0 && vma->fence)
+		if (vma->fence)
 			*out_flags |= PLANE_HAS_FENCE;
 	}
 	i915_vma_get(vma);
+err_unpin:
+	i915_gem_object_unpin_pages(obj);
 err:
+	if (ret == -EDEADLK) {
+		ret = i915_gem_ww_ctx_backoff(&ww);
+		if (!ret)
+			goto retry;
+	}
+	i915_gem_ww_ctx_fini(&ww);
+	if (ret)
+		vma = ERR_PTR(ret);
 	atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
 	intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
 	return vma;
 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
 	struct drm_framebuffer *fb = plane_state->hw.fb;
 	struct i915_vma *vma;
+	bool phys_cursor =
+		plane->id == PLANE_CURSOR &&
+		INTEL_INFO(dev_priv)->display.cursor_needs_physical;
-	if (plane->id == PLANE_CURSOR &&
+	vma = intel_pin_and_fence_fb_obj(fb, phys_cursor,
-	    INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
-		struct drm_i915_gem_object *obj = intel_fb_obj(fb);
-		const int align = intel_cursor_alignment(dev_priv);
-		int err;
-		err = i915_gem_object_attach_phys(obj, align);
-		if (err)
-			return err;
-	}
-	vma = intel_pin_and_fence_fb_obj(fb,
 					 &plane_state->view,
 					 intel_plane_uses_fence(plane_state),
 					 &plane_state->flags);
 	if (!obj)
 		return 0;
-	ret = i915_gem_object_pin_pages(obj);
-	if (ret)
-		return ret;
 	ret = intel_plane_pin_fb(new_plane_state);
-	i915_gem_object_unpin_pages(obj);
 	if (ret)
 		return ret;
 	struct drm_i915_gem_object *obj = intel_fb_obj(fb);
 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
-	if (obj->userptr.mm) {
+	if (i915_gem_object_is_userptr(obj)) {
 		drm_dbg(&i915->drm,
 			"attempting to use a userptr for a framebuffer, denied\n");
 		return -EINVAL;

drivers/gpu/drm/i915/display/intel_display.h+1−1

@@ -573,7 +573,7 @@ void intel_release_load_detect_pipe(struct drm_connector *connector, · 572 unmodified lines
573	struct intel_load_detect_pipe *old,	573	struct intel_load_detect_pipe *old,
574	struct drm_modeset_acquire_ctx *ctx);	574	struct drm_modeset_acquire_ctx *ctx);
575	struct i915_vma *	575	struct i915_vma *
576	intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,	576	intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, bool phys_cursor,
577	const struct i915_ggtt_view *view,	577	const struct i915_ggtt_view *view,
578	bool uses_fence,	578	bool uses_fence,
579	unsigned long *out_flags);	579	unsigned long *out_flags);

drivers/gpu/drm/i915/display/intel_dsb.c+1−1

@@ -293,7 +293,7 @@ void intel_dsb_prepare(struct intel_crtc_state *crtc_state) · 292 unmodified lines
293	goto out;	293	goto out;
294	}	294	}
295		295
296	buf = i915_gem_object_pin_map(vma->obj, I915_MAP_WC);	296	buf = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC);
297	if (IS_ERR(buf)) {	297	if (IS_ERR(buf)) {
298	drm_err(&i915->drm, "Command buffer creation failed\n");	298	drm_err(&i915->drm, "Command buffer creation failed\n");
299	i915_vma_unpin_and_release(&vma, I915_VMA_RELEASE_MAP);	299	i915_vma_unpin_and_release(&vma, I915_VMA_RELEASE_MAP);

drivers/gpu/drm/i915/display/intel_fbdev.c+1−1

@@ -211,7 +211,7 @@ static int intelfb_create(struct drm_fb_helper *helper, · 210 unmodified lines
211	* This also validates that any existing fb inherited from the	211	* This also validates that any existing fb inherited from the
212	* BIOS is suitable for own access.	212	* BIOS is suitable for own access.
213	*/	213	*/
214	vma = intel_pin_and_fence_fb_obj(&ifbdev->fb->base,	214	vma = intel_pin_and_fence_fb_obj(&ifbdev->fb->base, false,
215	&view, false, &flags);	215	&view, false, &flags);
216	if (IS_ERR(vma)) {	216	if (IS_ERR(vma)) {
217	ret = PTR_ERR(vma);	217	ret = PTR_ERR(vma);

drivers/gpu/drm/i915/display/intel_overlay.c+29−5

@@ -755,6 +755,32 @@ static u32 overlay_cmd_reg(struct drm_intel_overlay_put_image *params) · 754 unmodified lines
755	return cmd;	755	return cmd;
756	}	756	}
757		757
		758	static struct i915_vma intel_overlay_pin_fb(struct drm_i915_gem_object new_bo)
		759	{
		760	struct i915_gem_ww_ctx ww;
		761	struct i915_vma *vma;
		762	int ret;
		763
		764	i915_gem_ww_ctx_init(&ww, true);
		765	retry:
		766	ret = i915_gem_object_lock(new_bo, &ww);
		767	if (!ret) {
		768	vma = i915_gem_object_pin_to_display_plane(new_bo, &ww, 0,
		769	NULL, PIN_MAPPABLE);
		770	ret = PTR_ERR_OR_ZERO(vma);
		771	}
		772	if (ret == -EDEADLK) {
		773	ret = i915_gem_ww_ctx_backoff(&ww);
		774	if (!ret)
		775	goto retry;
		776	}
		777	i915_gem_ww_ctx_fini(&ww);
		778	if (ret)
		779	return ERR_PTR(ret);
		780
		781	return vma;
		782	}
		783
758	static int intel_overlay_do_put_image(struct intel_overlay *overlay,	784	static int intel_overlay_do_put_image(struct intel_overlay *overlay,
759	struct drm_i915_gem_object *new_bo,	785	struct drm_i915_gem_object *new_bo,
760	struct drm_intel_overlay_put_image *params)	786	struct drm_intel_overlay_put_image *params)
@@ -776,12 +802,10 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, · 15 unmodified lines
776		802
777	atomic_inc(&dev_priv->gpu_error.pending_fb_pin);	803	atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
778		804
779	vma = i915_gem_object_pin_to_display_plane(new_bo,	805	vma = intel_overlay_pin_fb(new_bo);
780	0, NULL, PIN_MAPPABLE);	806	if (IS_ERR(vma))
781	if (IS_ERR(vma)) {
782	ret = PTR_ERR(vma);
783	goto out_pin_section;	807	goto out_pin_section;
784	}	808
785	i915_gem_object_flush_frontbuffer(new_bo, ORIGIN_DIRTYFB);	809	i915_gem_object_flush_frontbuffer(new_bo, ORIGIN_DIRTYFB);
786		810
787	if (!overlay->active) {	811	if (!overlay->active) {

drivers/gpu/drm/i915/gem/i915_gem_clflush.c+7−8

 static int clflush_work(struct dma_fence_work *base)
 {
 	struct clflush *clflush = container_of(base, typeof(*clflush), base);
-	struct drm_i915_gem_object *obj = clflush->obj;
-	int err;
-	err = i915_gem_object_pin_pages(obj);
+	__do_clflush(clflush->obj);
-	if (err)
-		return err;
-	__do_clflush(obj);
-	i915_gem_object_unpin_pages(obj);
 	return 0;
 }
 {
 	struct clflush *clflush = container_of(base, typeof(*clflush), base);
+	i915_gem_object_unpin_pages(clflush->obj);
 	i915_gem_object_put(clflush->obj);
 }
 	if (!clflush)
 		return NULL;
+	if (__i915_gem_object_get_pages(obj) < 0) {
+		kfree(clflush);
+		return NULL;
+	}
 	dma_fence_work_init(&clflush->base, &clflush_ops);
 	clflush->obj = i915_gem_object_get(obj); /* obj <-> clflush cycle */

drivers/gpu/drm/i915/gem/i915_gem_context.c+40−33

 	if (ctx->sched.priority >= I915_PRIORITY_NORMAL &&
 	    intel_engine_has_timeslices(ce->engine))
 		__set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags);
+	intel_context_set_watchdog_us(ce, ctx->watchdog.timeout_us);
 }
 static void __free_engines(struct i915_gem_engines *e, unsigned int count)
 	return intel_engine_pulse(engine) == 0;
 }
-static bool
-__active_engine(struct i915_request *rq, struct intel_engine_cs **active)
-{
-	struct intel_engine_cs *engine, *locked;
-	bool ret = false;
-	/*
-	 * Serialise with __i915_request_submit() so that it sees
-	 * is-banned?, or we know the request is already inflight.
-	 *
-	 * Note that rq->engine is unstable, and so we double
-	 * check that we have acquired the lock on the final engine.
-	 */
-	locked = READ_ONCE(rq->engine);
-	spin_lock_irq(&locked->active.lock);
-	while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) {
-		spin_unlock(&locked->active.lock);
-		locked = engine;
-		spin_lock(&locked->active.lock);
-	}
-	if (i915_request_is_active(rq)) {
-		if (!__i915_request_is_complete(rq))
-			*active = locked;
-		ret = true;
-	}
-	spin_unlock_irq(&locked->active.lock);
-	return ret;
-}
 static struct intel_engine_cs *active_engine(struct intel_context *ce)
 {
 	struct intel_engine_cs *engine = NULL;
 		/* Check with the backend if the request is inflight */
 		found = true;
 		if (likely(rcu_access_pointer(rq->timeline) == ce->timeline))
-			found = __active_engine(rq, &engine);
+			found = i915_request_active_engine(rq, &engine);
 		i915_request_put(rq);
 		if (found)
 	context_apply_all(ctx, __apply_timeline, timeline);
 }
+static int __apply_watchdog(struct intel_context *ce, void *timeout_us)
+{
+	return intel_context_set_watchdog_us(ce, (uintptr_t)timeout_us);
+}
+static int
+__set_watchdog(struct i915_gem_context *ctx, unsigned long timeout_us)
+{
+	int ret;
+	ret = context_apply_all(ctx, __apply_watchdog,
+				(void *)(uintptr_t)timeout_us);
+	if (!ret)
+		ctx->watchdog.timeout_us = timeout_us;
+	return ret;
+}
+static void __set_default_fence_expiry(struct i915_gem_context *ctx)
+{
+	struct drm_i915_private *i915 = ctx->i915;
+	int ret;
+	if (!IS_ACTIVE(CONFIG_DRM_I915_REQUEST_TIMEOUT) ||
+	    !i915->params.request_timeout_ms)
+		return;
+	/* Default expiry for user fences. */
+	ret = __set_watchdog(ctx, i915->params.request_timeout_ms * 1000);
+	if (ret)
+		drm_notice(&i915->drm,
+			   "Failed to configure default fence expiry! (%d)",
+			   ret);
+}
 static struct i915_gem_context *
 i915_gem_create_context(struct drm_i915_private *i915, unsigned int flags)
 {
 		intel_timeline_put(timeline);
 	}
+	__set_default_fence_expiry(ctx);
 	trace_i915_context_create(ctx);
 	return ctx;

drivers/gpu/drm/i915/gem/i915_gem_context_types.h+4−0

 	 */
 	atomic_t active_count;
+	struct {
+		u64 timeout_us;
+	} watchdog;
 	/**
 	 * @hang_timestamp: The last time(s) this context caused a GPU hang
 	 */

drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c+35−29

@@ -25,7 +25,7 @@ static struct sg_table i915_gem_map_dma_buf(struct dma_buf_attachment attachme · 24 unmodified lines
25	struct scatterlist src, dst;	25	struct scatterlist src, dst;
26	int ret, i;	26	int ret, i;
27		27
28	ret = i915_gem_object_pin_pages(obj);	28	ret = i915_gem_object_pin_pages_unlocked(obj);
29	if (ret)	29	if (ret)
30	goto err;	30	goto err;
31		31
@@ -82,7 +82,7 @@ static int i915_gem_dmabuf_vmap(struct dma_buf dma_buf, struct dma_buf_map map · 50 unmodified lines
82	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);	82	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
83	void *vaddr;	83	void *vaddr;
84		84
85	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);	85	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
86	if (IS_ERR(vaddr))	86	if (IS_ERR(vaddr))
87	return PTR_ERR(vaddr);	87	return PTR_ERR(vaddr);
88		88
@@ -123,42 +123,48 @@ static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_dire · 34 unmodified lines
123	{	123	{
124	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);	124	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
125	bool write = (direction == DMA_BIDIRECTIONAL \|\| direction == DMA_TO_DEVICE);	125	bool write = (direction == DMA_BIDIRECTIONAL \|\| direction == DMA_TO_DEVICE);
		126	struct i915_gem_ww_ctx ww;
126	int err;	127	int err;
127		128
128	err = i915_gem_object_pin_pages(obj);	129	i915_gem_ww_ctx_init(&ww, true);
129	if (err)	130	retry:
130	return err;	131	err = i915_gem_object_lock(obj, &ww);
131		132	if (!err)
132	err = i915_gem_object_lock_interruptible(obj, NULL);	133	err = i915_gem_object_pin_pages(obj);
133	if (err)	134	if (!err) {
134	goto out;	135	err = i915_gem_object_set_to_cpu_domain(obj, write);
135		136	i915_gem_object_unpin_pages(obj);
136	err = i915_gem_object_set_to_cpu_domain(obj, write);	137	}
137	i915_gem_object_unlock(obj);	138	if (err == -EDEADLK) {
138		139	err = i915_gem_ww_ctx_backoff(&ww);
139	out:	140	if (!err)
140	i915_gem_object_unpin_pages(obj);	141	goto retry;
		142	}
		143	i915_gem_ww_ctx_fini(&ww);
141	return err;	144	return err;
142	}	145	}
143		146
144	static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction)	147	static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction)
145	{	148	{
146	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);	149	struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
		150	struct i915_gem_ww_ctx ww;
147	int err;	151	int err;
148		152
149	err = i915_gem_object_pin_pages(obj);	153	i915_gem_ww_ctx_init(&ww, true);
150	if (err)	154	retry:
151	return err;	155	err = i915_gem_object_lock(obj, &ww);
152		156	if (!err)
153	err = i915_gem_object_lock_interruptible(obj, NULL);	157	err = i915_gem_object_pin_pages(obj);
154	if (err)	158	if (!err) {
155	goto out;	159	err = i915_gem_object_set_to_gtt_domain(obj, false);
156		160	i915_gem_object_unpin_pages(obj);
157	err = i915_gem_object_set_to_gtt_domain(obj, false);	161	}
158	i915_gem_object_unlock(obj);	162	if (err == -EDEADLK) {
159		163	err = i915_gem_ww_ctx_backoff(&ww);
160	out:	164	if (!err)
161	i915_gem_object_unpin_pages(obj);	165	goto retry;
		166	}
		167	i915_gem_ww_ctx_fini(&ww);
162	return err;	168	return err;
163	}	169	}
164		170
@@ -258,7 +264,7 @@ struct drm_gem_object i915_gem_prime_import(struct drm_device dev, · 93 unmodified lines
258	}	264	}
259		265
260	drm_gem_private_object_init(dev, &obj->base, dma_buf->size);	266	drm_gem_private_object_init(dev, &obj->base, dma_buf->size);
261	i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops, &lock_class);	267	i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops, &lock_class, 0);
262	obj->base.import_attach = attach;	268	obj->base.import_attach = attach;
263	obj->base.resv = dma_buf->resv;	269	obj->base.resv = dma_buf->resv;
264		270

drivers/gpu/drm/i915/gem/i915_gem_domain.c+39−33

 	 * not allowed to be changed by userspace.
 	 */
 	if (i915_gem_object_is_proxy(obj)) {
-		ret = -ENXIO;
+		/*
+		 * Silently allow cached for userptr; the vulkan driver
+		 * sets all objects to cached
+		 */
+		if (!i915_gem_object_is_userptr(obj) ||
+		    args->caching != I915_CACHING_CACHED)
+			ret = -ENXIO;
 		goto out;
 	}
  */
 struct i915_vma *
 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
+				     struct i915_gem_ww_ctx *ww,
 				     u32 alignment,
 				     const struct i915_ggtt_view *view,
 				     unsigned int flags)
 {
 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
-	struct i915_gem_ww_ctx ww;
 	struct i915_vma *vma;
 	int ret;
 	if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj))
 		return ERR_PTR(-EINVAL);
-	i915_gem_ww_ctx_init(&ww, true);
-retry:
-	ret = i915_gem_object_lock(obj, &ww);
-	if (ret)
-		goto err;
 	/*
 	 * The display engine is not coherent with the LLC cache on gen6.  As
 	 * a result, we make sure that the pinning that is about to occur is
 					      HAS_WT(i915) ?
 					      I915_CACHE_WT : I915_CACHE_NONE);
 	if (ret)
-		goto err;
+		return ERR_PTR(ret);
 	/*
 	 * As the user may map the buffer once pinned in the display plane
 	vma = ERR_PTR(-ENOSPC);
 	if ((flags & PIN_MAPPABLE) == 0 &&
 	    (!view || view->type == I915_GGTT_VIEW_NORMAL))
-		vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0, alignment,
+		vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0, alignment,
 						  flags | PIN_MAPPABLE |
 						  PIN_NONBLOCK);
 	if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK))
-		vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0,
+		vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0,
 						  alignment, flags);
-	if (IS_ERR(vma)) {
+	if (IS_ERR(vma))
-		ret = PTR_ERR(vma);
+		return vma;
-		goto err;
-	}
 	vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
 	i915_vma_mark_scanout(vma);
 	i915_gem_object_flush_if_display_locked(obj);
-err:
-	if (ret == -EDEADLK) {
-		ret = i915_gem_ww_ctx_backoff(&ww);
-		if (!ret)
-			goto retry;
-	}
-	i915_gem_ww_ctx_fini(&ww);
-	if (ret)
-		return ERR_PTR(ret);
 	return vma;
 }
 	if (err)
 		goto out;
+	if (i915_gem_object_is_userptr(obj)) {
+		/*
+		 * Try to grab userptr pages, iris uses set_domain to check
+		 * userptr validity
+		 */
+		err = i915_gem_object_userptr_validate(obj);
+		if (!err)
+			err = i915_gem_object_wait(obj,
+						   I915_WAIT_INTERRUPTIBLE |
+						   I915_WAIT_PRIORITY |
+						   (write_domain ? I915_WAIT_ALL : 0),
+						   MAX_SCHEDULE_TIMEOUT);
+		goto out;
+	}
 	/*
 	 * Proxy objects do not control access to the backing storage, ergo
 	 * they cannot be used as a means to manipulate the cache domain
 		goto out;
 	}
+	err = i915_gem_object_lock_interruptible(obj, NULL);
+	if (err)
+		goto out;
 	/*
 	 * Flush and acquire obj->pages so that we are coherent through
 	 * direct access in memory with previous cached writes through
 	 */
 	err = i915_gem_object_pin_pages(obj);
 	if (err)
-		goto out;
+		goto out_unlock;
 	/*
 	 * Already in the desired write domain? Nothing for us to do!
 	if (READ_ONCE(obj->write_domain) == read_domains)
 		goto out_unpin;
-	err = i915_gem_object_lock_interruptible(obj, NULL);
-	if (err)
-		goto out_unpin;
 	if (read_domains & I915_GEM_DOMAIN_WC)
 		err = i915_gem_object_set_to_wc_domain(obj, write_domain);
 	else if (read_domains & I915_GEM_DOMAIN_GTT)
 	else
 		err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
+out_unpin:
+	i915_gem_object_unpin_pages(obj);
+out_unlock:
 	i915_gem_object_unlock(obj);
-	if (write_domain)
+	if (!err && write_domain)
 		i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
-out_unpin:
-	i915_gem_object_unpin_pages(obj);
 out:
 	i915_gem_object_put(obj);
 	return err;

drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c+204−131

 #include "i915_sw_fence_work.h"
 #include "i915_trace.h"
 #include "i915_user_extensions.h"
+#include "i915_memcpy.h"
 struct eb_vma {
 	struct i915_vma *vma;
 #define DBG_FORCE_RELOC 0 /* choose one of the above! */
 };
-#define __EXEC_OBJECT_HAS_PIN		BIT(31)
+/* __EXEC_OBJECT_NO_RESERVE is BIT(31), defined in i915_vma.h */
-#define __EXEC_OBJECT_HAS_FENCE		BIT(30)
+#define __EXEC_OBJECT_HAS_PIN		BIT(30)
-#define __EXEC_OBJECT_NEEDS_MAP		BIT(29)
+#define __EXEC_OBJECT_HAS_FENCE		BIT(29)
-#define __EXEC_OBJECT_NEEDS_BIAS	BIT(28)
+#define __EXEC_OBJECT_USERPTR_INIT	BIT(28)
-#define __EXEC_OBJECT_INTERNAL_FLAGS	(~0u << 28) /* all of the above */
+#define __EXEC_OBJECT_NEEDS_MAP		BIT(27)
+#define __EXEC_OBJECT_NEEDS_BIAS	BIT(26)
+#define __EXEC_OBJECT_INTERNAL_FLAGS	(~0u << 26) /* all of the above + */
 #define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE)
 #define __EXEC_HAS_RELOC	BIT(31)
 #define __EXEC_ENGINE_PINNED	BIT(30)
-#define __EXEC_INTERNAL_FLAGS	(~0u << 30)
+#define __EXEC_USERPTR_USED	BIT(29)
+#define __EXEC_INTERNAL_FLAGS	(~0u << 29)
 #define UPDATE			PIN_OFFSET_FIXED
 #define BATCH_OFFSET_BIAS (256*1024)
 	return pin_flags;
 }
-static inline bool
+static inline int
 eb_pin_vma(struct i915_execbuffer *eb,
 	   const struct drm_i915_gem_exec_object2 *entry,
 	   struct eb_vma *ev)
 {
 	struct i915_vma *vma = ev->vma;
 	u64 pin_flags;
+	int err;
 	if (vma->node.size)
 		pin_flags = vma->node.start;
 		pin_flags |= PIN_GLOBAL;
 	/* Attempt to reuse the current location if available */
-	/* TODO: Add -EDEADLK handling here */
+	err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags);
-	if (unlikely(i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags))) {
+	if (err == -EDEADLK)
+		return err;
+	if (unlikely(err)) {
 		if (entry->flags & EXEC_OBJECT_PINNED)
-			return false;
+			return err;
 		/* Failing that pick any _free_ space if suitable */
-		if (unlikely(i915_vma_pin_ww(vma, &eb->ww,
+		err = i915_vma_pin_ww(vma, &eb->ww,
 					     entry->pad_to_size,
 					     entry->alignment,
 					     eb_pin_flags(entry, ev->flags) |
-					     PIN_USER | PIN_NOEVICT)))
+					     PIN_USER | PIN_NOEVICT);
-			return false;
+		if (unlikely(err))
+			return err;
 	}
 	if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) {
-		if (unlikely(i915_vma_pin_fence(vma))) {
+		err = i915_vma_pin_fence(vma);
+		if (unlikely(err)) {
 			i915_vma_unpin(vma);
-			return false;
+			return err;
 		}
 		if (vma->fence)
 	}
 	ev->flags |= __EXEC_OBJECT_HAS_PIN;
-	return !eb_vma_misplaced(entry, vma, ev->flags);
+	if (eb_vma_misplaced(entry, vma, ev->flags))
+		return -EBADSLT;
+	return 0;
 }
 static inline void
 		struct drm_i915_gem_exec_object2 *entry,
 		struct i915_vma *vma)
 {
+	/* Relocations are disallowed for all platforms after TGL-LP.  This
+	 * also covers all platforms with local memory.
+	 */
+	if (entry->relocation_count &&
+	    INTEL_GEN(eb->i915) >= 12 && !IS_TIGERLAKE(eb->i915))
+		return -EINVAL;
 	if (unlikely(entry->flags & eb->invalid_flags))
 		return -EINVAL;
 		}
 		eb_add_vma(eb, i, batch, vma);
+		if (i915_gem_object_is_userptr(vma->obj)) {
+			err = i915_gem_object_userptr_submit_init(vma->obj);
+			if (err) {
+				if (i + 1 < eb->buffer_count) {
+					/*
+					 * Execbuffer code expects last vma entry to be NULL,
+					 * since we already initialized this entry,
+					 * set the next value to NULL or we mess up
+					 * cleanup handling.
+					 */
+					eb->vma[i + 1].vma = NULL;
+				}
+				return err;
+			}
+			eb->vma[i].flags |= __EXEC_OBJECT_USERPTR_INIT;
+			eb->args->flags |= __EXEC_USERPTR_USED;
+		}
 	}
 	if (unlikely(eb->batch->flags & EXEC_OBJECT_WRITE)) {
 		if (err)
 			return err;
-		if (eb_pin_vma(eb, entry, ev)) {
+		err = eb_pin_vma(eb, entry, ev);
+		if (err == -EDEADLK)
+			return err;
+		if (!err) {
 			if (entry->offset != vma->node.start) {
 				entry->offset = vma->node.start | UPDATE;
 				eb->args->flags |= __EXEC_HAS_RELOC;
 			}
 		}
+		if (!(ev->flags & EXEC_OBJECT_WRITE)) {
+			err = dma_resv_reserve_shared(vma->resv, 1);
+			if (err)
+				return err;
+		}
 		GEM_BUG_ON(drm_mm_node_allocated(&vma->node) &&
 			   eb_vma_misplaced(&eb->exec[i], vma, ev->flags));
 	}
 	}
 }
-static void eb_release_vmas(struct i915_execbuffer *eb, bool final)
+static void eb_release_vmas(struct i915_execbuffer *eb, bool final, bool release_userptr)
 {
 	const unsigned int count = eb->buffer_count;
 	unsigned int i;
 		eb_unreserve_vma(ev);
+		if (release_userptr && ev->flags & __EXEC_OBJECT_USERPTR_INIT) {
+			ev->flags &= ~__EXEC_OBJECT_USERPTR_INIT;
+			i915_gem_object_userptr_submit_fini(vma->obj);
+		}
 		if (final)
 			i915_vma_put(vma);
 	}
 		err = PTR_ERR(cmd);
 		goto err_pool;
 	}
+	intel_gt_buffer_pool_mark_used(pool);
 	memset32(cmd, 0, pool->obj->base.size / sizeof(u32));
 	return 0;
 }
+static int eb_reinit_userptr(struct i915_execbuffer *eb)
+{
+	const unsigned int count = eb->buffer_count;
+	unsigned int i;
+	int ret;
+	if (likely(!(eb->args->flags & __EXEC_USERPTR_USED)))
+		return 0;
+	for (i = 0; i < count; i++) {
+		struct eb_vma *ev = &eb->vma[i];
+		if (!i915_gem_object_is_userptr(ev->vma->obj))
+			continue;
+		ret = i915_gem_object_userptr_submit_init(ev->vma->obj);
+		if (ret)
+			return ret;
+		ev->flags |= __EXEC_OBJECT_USERPTR_INIT;
+	}
+	return 0;
+}
 static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb,
 					   struct i915_request *rq)
 {
 	}
 	/* We may process another execbuffer during the unlock... */
-	eb_release_vmas(eb, false);
+	eb_release_vmas(eb, false, true);
 	i915_gem_ww_ctx_fini(&eb->ww);
 	if (rq) {
 	}
 	if (!err)
-		flush_workqueue(eb->i915->mm.userptr_wq);
+		err = eb_reinit_userptr(eb);
 err_relock:
 	i915_gem_ww_ctx_init(&eb->ww, true);
 err:
 	if (err == -EDEADLK) {
-		eb_release_vmas(eb, false);
+		eb_release_vmas(eb, false, false);
 		err = i915_gem_ww_ctx_backoff(&eb->ww);
 		if (!err)
 			goto repeat_validate;
 err:
 	if (err == -EDEADLK) {
-		eb_release_vmas(eb, false);
+		eb_release_vmas(eb, false, false);
 		err = i915_gem_ww_ctx_backoff(&eb->ww);
 		if (!err)
 			goto retry;
 		}
 		if (err == 0)
-			err = i915_vma_move_to_active(vma, eb->request, flags);
+			err = i915_vma_move_to_active(vma, eb->request,
+						      flags | __EXEC_OBJECT_NO_RESERVE);
+	}
+#ifdef CONFIG_MMU_NOTIFIER
+	if (!err && (eb->args->flags & __EXEC_USERPTR_USED)) {
+		spin_lock(&eb->i915->mm.notifier_lock);
+		/*
+		 * count is always at least 1, otherwise __EXEC_USERPTR_USED
+		 * could not have been set
+		 */
+		for (i = 0; i < count; i++) {
+			struct eb_vma *ev = &eb->vma[i];
+			struct drm_i915_gem_object *obj = ev->vma->obj;
+			if (!i915_gem_object_is_userptr(obj))
+				continue;
+			err = i915_gem_object_userptr_submit_done(obj);
+			if (err)
+				break;
+		}
+		spin_unlock(&eb->i915->mm.notifier_lock);
 	}
+#endif
 	if (unlikely(err))
 		goto err_skip;
 	struct i915_vma *trampoline;
 	unsigned long batch_offset;
 	unsigned long batch_length;
+	unsigned long *jump_whitelist;
+	const void *batch_map;
+	void *shadow_map;
 };
 static int __eb_parse(struct dma_fence_work *work)
 {
 	struct eb_parse_work *pw = container_of(work, typeof(*pw), base);
+	int ret;
+	bool cookie;
-	return intel_engine_cmd_parser(pw->engine,
+	cookie = dma_fence_begin_signalling();
-				       pw->batch,
+	ret = intel_engine_cmd_parser(pw->engine,
-				       pw->batch_offset,
+				      pw->batch,
-				       pw->batch_length,
+				      pw->batch_offset,
-				       pw->shadow,
+				      pw->batch_length,
-				       pw->trampoline);
+				      pw->shadow,
+				      pw->jump_whitelist,
+				      pw->shadow_map,
+				      pw->batch_map);
+	dma_fence_end_signalling(cookie);
+	return ret;
 }
 static void __eb_parse_release(struct dma_fence_work *work)
 {
 	struct eb_parse_work *pw = container_of(work, typeof(*pw), base);
+	if (!IS_ERR_OR_NULL(pw->jump_whitelist))
+		kfree(pw->jump_whitelist);
+	if (pw->batch_map)
+		i915_gem_object_unpin_map(pw->batch->obj);
+	else
+		i915_gem_object_unpin_pages(pw->batch->obj);
+	i915_gem_object_unpin_map(pw->shadow->obj);
 	if (pw->trampoline)
 		i915_active_release(&pw->trampoline->active);
 	i915_active_release(&pw->shadow->active);
 			     struct i915_vma *trampoline)
 {
 	struct eb_parse_work *pw;
+	struct drm_i915_gem_object *batch = eb->batch->vma->obj;
+	bool needs_clflush;
 	int err;
 	GEM_BUG_ON(overflows_type(eb->batch_start_offset, pw->batch_offset));
 			goto err_shadow;
 	}
+	pw->shadow_map = i915_gem_object_pin_map(shadow->obj, I915_MAP_WB);
+	if (IS_ERR(pw->shadow_map)) {
+		err = PTR_ERR(pw->shadow_map);
+		goto err_trampoline;
+	}
+	needs_clflush =
+		!(batch->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ);
+	pw->batch_map = ERR_PTR(-ENODEV);
+	if (needs_clflush && i915_has_memcpy_from_wc())
+		pw->batch_map = i915_gem_object_pin_map(batch, I915_MAP_WC);
+	if (IS_ERR(pw->batch_map)) {
+		err = i915_gem_object_pin_pages(batch);
+		if (err)
+			goto err_unmap_shadow;
+		pw->batch_map = NULL;
+	}
+	pw->jump_whitelist =
+		intel_engine_cmd_parser_alloc_jump_whitelist(eb->batch_len,
+							     trampoline);
+	if (IS_ERR(pw->jump_whitelist)) {
+		err = PTR_ERR(pw->jump_whitelist);
+		goto err_unmap_batch;
+	}
 	dma_fence_work_init(&pw->base, &eb_parse_ops);
 	pw->engine = eb->engine;
 	if (err)
 		goto err_commit;
+	err = dma_resv_reserve_shared(shadow->resv, 1);
+	if (err)
+		goto err_commit;
 	/* Wait for all writes (and relocs) into the batch to complete */
 	err = i915_sw_fence_await_reservation(&pw->base.chain,
 					      pw->batch->resv, NULL, false,
 	dma_fence_work_commit_imm(&pw->base);
 	return err;
+err_unmap_batch:
+	if (pw->batch_map)
+		i915_gem_object_unpin_map(batch);
+	else
+		i915_gem_object_unpin_pages(batch);
+err_unmap_shadow:
+	i915_gem_object_unpin_map(shadow->obj);
+err_trampoline:
+	if (trampoline)
+		i915_active_release(&trampoline->active);
 err_shadow:
 	i915_active_release(&shadow->active);
 err_batch:
 		err = PTR_ERR(shadow);
 		goto err;
 	}
+	intel_gt_buffer_pool_mark_used(pool);
 	i915_gem_object_set_readonly(shadow->obj);
 	shadow->private = pool;
 	err = eb_lookup_vmas(&eb);
 	if (err) {
-		eb_release_vmas(&eb, true);
+		eb_release_vmas(&eb, true, true);
 		goto err_engine;
 	}
 	trace_i915_request_queue(eb.request, eb.batch_flags);
 	err = eb_submit(&eb, batch);
 err_request:
 	i915_request_get(eb.request);
 	err = eb_request_add(&eb, err);
 	i915_request_put(eb.request);
 err_vma:
-	eb_release_vmas(&eb, true);
+	eb_release_vmas(&eb, true, true);
 	if (eb.trampoline)
 		i915_vma_unpin(eb.trampoline);
 	WARN_ON(err == -EDEADLK);
 	return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1);
 }
-/*
- * Legacy execbuffer just creates an exec2 list from the original exec object
- * list array and passes it to the real function.
- */
-int
-i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data,
-			  struct drm_file *file)
-{
-	struct drm_i915_private *i915 = to_i915(dev);
-	struct drm_i915_gem_execbuffer *args = data;
-	struct drm_i915_gem_execbuffer2 exec2;
-	struct drm_i915_gem_exec_object *exec_list = NULL;
-	struct drm_i915_gem_exec_object2 *exec2_list = NULL;
-	const size_t count = args->buffer_count;
-	unsigned int i;
-	int err;
-	if (!check_buffer_count(count)) {
-		drm_dbg(&i915->drm, "execbuf2 with %zd buffers\n", count);
-		return -EINVAL;
-	}
-	exec2.buffers_ptr = args->buffers_ptr;
-	exec2.buffer_count = args->buffer_count;
-	exec2.batch_start_offset = args->batch_start_offset;
-	exec2.batch_len = args->batch_len;
-	exec2.DR1 = args->DR1;
-	exec2.DR4 = args->DR4;
-	exec2.num_cliprects = args->num_cliprects;
-	exec2.cliprects_ptr = args->cliprects_ptr;
-	exec2.flags = I915_EXEC_RENDER;
-	i915_execbuffer2_set_context_id(exec2, 0);
-	err = i915_gem_check_execbuffer(&exec2);
-	if (err)
-		return err;
-	/* Copy in the exec list from userland */
-	exec_list = kvmalloc_array(count, sizeof(*exec_list),
-				   __GFP_NOWARN | GFP_KERNEL);
-	/* Allocate extra slots for use by the command parser */
-	exec2_list = kvmalloc_array(count + 2, eb_element_size(),
-				    __GFP_NOWARN | GFP_KERNEL);
-	if (exec_list == NULL || exec2_list == NULL) {
-		drm_dbg(&i915->drm,
-			"Failed to allocate exec list for %d buffers\n",
-			args->buffer_count);
-		kvfree(exec_list);
-		kvfree(exec2_list);
-		return -ENOMEM;
-	}
-	err = copy_from_user(exec_list,
-			     u64_to_user_ptr(args->buffers_ptr),
-			     sizeof(*exec_list) * count);
-	if (err) {
-		drm_dbg(&i915->drm, "copy %d exec entries failed %d\n",
-			args->buffer_count, err);
-		kvfree(exec_list);
-		kvfree(exec2_list);
-		return -EFAULT;
-	}
-	for (i = 0; i < args->buffer_count; i++) {
-		exec2_list[i].handle = exec_list[i].handle;
-		exec2_list[i].relocation_count = exec_list[i].relocation_count;
-		exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
-		exec2_list[i].alignment = exec_list[i].alignment;
-		exec2_list[i].offset = exec_list[i].offset;
-		if (INTEL_GEN(to_i915(dev)) < 4)
-			exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
-		else
-			exec2_list[i].flags = 0;
-	}
-	err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list);
-	if (exec2.flags & __EXEC_HAS_RELOC) {
-		struct drm_i915_gem_exec_object __user *user_exec_list =
-			u64_to_user_ptr(args->buffers_ptr);
-		/* Copy the new buffer offsets back to the user's exec list. */
-		for (i = 0; i < args->buffer_count; i++) {
-			if (!(exec2_list[i].offset & UPDATE))
-				continue;
-			exec2_list[i].offset =
-				gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
-			exec2_list[i].offset &= PIN_OFFSET_MASK;
-			if (__copy_to_user(&user_exec_list[i].offset,
-					   &exec2_list[i].offset,
-					   sizeof(user_exec_list[i].offset)))
-				break;
-		}
-	}
-	kvfree(exec_list);
-	kvfree(exec2_list);
-	return err;
-}
 int
 i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data,
 			   struct drm_file *file)

drivers/gpu/drm/i915/gem/i915_gem_fence.c+0−95

@@ -1,95 +0,0 @@
1	/*
2	* SPDX-License-Identifier: MIT
3	*
4	* Copyright © 2019 Intel Corporation
5	*/
6
7	#include "i915_drv.h"
8	#include "i915_gem_object.h"
9
10	struct stub_fence {
11	struct dma_fence dma;
12	struct i915_sw_fence chain;
13	};
14
15	static int __i915_sw_fence_call
16	stub_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
17	{
18	struct stub_fence stub = container_of(fence, typeof(stub), chain);
19
20	switch (state) {
21	case FENCE_COMPLETE:
22	dma_fence_signal(&stub->dma);
23	break;
24
25	case FENCE_FREE:
26	dma_fence_put(&stub->dma);
27	break;
28	}
29
30	return NOTIFY_DONE;
31	}
32
33	static const char stub_driver_name(struct dma_fence fence)
34	{
35	return DRIVER_NAME;
36	}
37
38	static const char stub_timeline_name(struct dma_fence fence)
39	{
40	return "object";
41	}
42
43	static void stub_release(struct dma_fence *fence)
44	{
45	struct stub_fence stub = container_of(fence, typeof(stub), dma);
46
47	i915_sw_fence_fini(&stub->chain);
48
49	BUILD_BUG_ON(offsetof(typeof(*stub), dma));
50	dma_fence_free(&stub->dma);
51	}
52
53	static const struct dma_fence_ops stub_fence_ops = {
54	.get_driver_name = stub_driver_name,
55	.get_timeline_name = stub_timeline_name,
56	.release = stub_release,
57	};
58
59	struct dma_fence *
60	i915_gem_object_lock_fence(struct drm_i915_gem_object *obj)
61	{
62	struct stub_fence *stub;
63
64	assert_object_held(obj);
65
66	stub = kmalloc(sizeof(*stub), GFP_KERNEL);
67	if (!stub)
68	return NULL;
69
70	i915_sw_fence_init(&stub->chain, stub_notify);
71	dma_fence_init(&stub->dma, &stub_fence_ops, &stub->chain.wait.lock,
72	0, 0);
73
74	if (i915_sw_fence_await_reservation(&stub->chain,
75	obj->base.resv, NULL, true,
76	i915_fence_timeout(to_i915(obj->base.dev)),
77	I915_FENCE_GFP) < 0)
78	goto err;
79
80	dma_resv_add_excl_fence(obj->base.resv, &stub->dma);
81
82	return &stub->dma;
83
84	err:
85	stub_release(&stub->dma);
86	return NULL;
87	}
88
89	void i915_gem_object_unlock_fence(struct drm_i915_gem_object *obj,
90	struct dma_fence *fence)
91	{
92	struct stub_fence stub = container_of(fence, typeof(stub), dma);
93
94	i915_sw_fence_commit(&stub->chain);
95	}

drivers/gpu/drm/i915/gem/i915_gem_internal.c+3−3

 static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {
 	.name = "i915_gem_object_internal",
-	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
+	.flags = I915_GEM_OBJECT_IS_SHRINKABLE,
-		 I915_GEM_OBJECT_IS_SHRINKABLE,
 	.get_pages = i915_gem_object_get_pages_internal,
 	.put_pages = i915_gem_object_put_pages_internal,
 };
 		return ERR_PTR(-ENOMEM);
 	drm_gem_private_object_init(&i915->drm, &obj->base, size);
-	i915_gem_object_init(obj, &i915_gem_object_internal_ops, &lock_class);
+	i915_gem_object_init(obj, &i915_gem_object_internal_ops, &lock_class,
+			     I915_BO_ALLOC_STRUCT_PAGE);
 	/*
 	 * Mark the object as volatile, such that the pages are marked as

drivers/gpu/drm/i915/gem/i915_gem_ioctls.h+0−2

 			struct drm_file *file);
 int i915_gem_create_ioctl(struct drm_device *dev, void *data,
 			  struct drm_file *file);
-int i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data,
-			      struct drm_file *file);
 int i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data,
 			       struct drm_file *file);
 int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,

drivers/gpu/drm/i915/gem/i915_gem_lmem.c+2−2

@@ -40,13 +40,13 @@ int __i915_gem_lmem_object_init(struct intel_memory_region *mem, · 39 unmodified lines
40	struct drm_i915_private *i915 = mem->i915;	40	struct drm_i915_private *i915 = mem->i915;
41		41
42	drm_gem_private_object_init(&i915->drm, &obj->base, size);	42	drm_gem_private_object_init(&i915->drm, &obj->base, size);
43	i915_gem_object_init(obj, &i915_gem_lmem_obj_ops, &lock_class);	43	i915_gem_object_init(obj, &i915_gem_lmem_obj_ops, &lock_class, flags);
44		44
45	obj->read_domains = I915_GEM_DOMAIN_WC \| I915_GEM_DOMAIN_GTT;	45	obj->read_domains = I915_GEM_DOMAIN_WC \| I915_GEM_DOMAIN_GTT;
46		46
47	i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);	47	i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);
48		48
49	i915_gem_object_init_memory_region(obj, mem, flags);	49	i915_gem_object_init_memory_region(obj, mem);
50		50
51	return 0;	51	return 0;
52	}	52	}

drivers/gpu/drm/i915/gem/i915_gem_mman.c+29−6

 		     area->vm_flags & VM_WRITE))
 		return VM_FAULT_SIGBUS;
+	if (i915_gem_object_lock_interruptible(obj, NULL))
+		return VM_FAULT_NOPAGE;
 	err = i915_gem_object_pin_pages(obj);
 	if (err)
 		goto out;
 	iomap = -1;
-	if (!i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_STRUCT_PAGE)) {
+	if (!i915_gem_object_has_struct_page(obj)) {
 		iomap = obj->mm.region->iomap.base;
 		iomap -= obj->mm.region->region.start;
 	}
 	i915_gem_object_unpin_pages(obj);
 out:
+	i915_gem_object_unlock(obj);
 	return i915_error_to_vmf_fault(err);
 }
 {
 	struct i915_mmap_offset *mmo = area->vm_private_data;
 	struct drm_i915_gem_object *obj = mmo->obj;
+	struct i915_gem_ww_ctx ww;
 	void *vaddr;
+	int err = 0;
 	if (i915_gem_object_is_readonly(obj) && write)
 		return -EACCES;
 	if (addr >= obj->base.size)
 		return -EINVAL;
+	i915_gem_ww_ctx_init(&ww, true);
+retry:
+	err = i915_gem_object_lock(obj, &ww);
+	if (err)
+		goto out;
 	/* As this is primarily for debugging, let's focus on simplicity */
 	vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC);
-	if (IS_ERR(vaddr))
+	if (IS_ERR(vaddr)) {
-		return PTR_ERR(vaddr);
+		err = PTR_ERR(vaddr);
+		goto out;
+	}
 	if (write) {
 		memcpy(vaddr + addr, buf, len);
 	}
 	i915_gem_object_unpin_map(obj);
+out:
+	if (err == -EDEADLK) {
+		err = i915_gem_ww_ctx_backoff(&ww);
+		if (!err)
+			goto retry;
+	}
+	i915_gem_ww_ctx_fini(&ww);
+	if (err)
+		return err;
 	return len;
 }
 	}
 	if (mmap_type != I915_MMAP_TYPE_GTT &&
-	    !i915_gem_object_type_has(obj,
+	    !i915_gem_object_has_struct_page(obj) &&
-				      I915_GEM_OBJECT_HAS_STRUCT_PAGE |
+	    !i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM)) {
-				      I915_GEM_OBJECT_HAS_IOMEM)) {
 		err = -ENODEV;
 		goto out;
 	}

drivers/gpu/drm/i915/gem/i915_gem_object.c+3−7

 void i915_gem_object_init(struct drm_i915_gem_object *obj,
 			  const struct drm_i915_gem_object_ops *ops,
-			  struct lock_class_key *key)
+			  struct lock_class_key *key, unsigned flags)
 {
-	__mutex_init(&obj->mm.lock, ops->name ?: "obj->mm.lock", key);
 	spin_lock_init(&obj->vma.lock);
 	INIT_LIST_HEAD(&obj->vma.list);
 	init_rcu_head(&obj->rcu);
 	obj->ops = ops;
+	GEM_BUG_ON(flags & ~I915_BO_ALLOC_FLAGS);
+	obj->flags = flags;
 	obj->mm.madv = I915_MADV_WILLNEED;
 	INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN);
 	mutex_init(&obj->mm.get_page.lock);
 	INIT_RADIX_TREE(&obj->mm.get_dma_page.radix, GFP_KERNEL | __GFP_NOWARN);
 	mutex_init(&obj->mm.get_dma_page.lock);
-	if (IS_ENABLED(CONFIG_LOCKDEP) && i915_gem_object_is_shrinkable(obj))
-		i915_gem_shrinker_taints_mutex(to_i915(obj->base.dev),
-					       &obj->mm.lock);
 }
 /**

drivers/gpu/drm/i915/gem/i915_gem_object.h+60−36

 void i915_gem_object_init(struct drm_i915_gem_object *obj,
 			  const struct drm_i915_gem_object_ops *ops,
-			  struct lock_class_key *key);
+			  struct lock_class_key *key,
+			  unsigned alloc_flags);
 struct drm_i915_gem_object *
 i915_gem_object_create_shmem(struct drm_i915_private *i915,
 			     resource_size_t size);
 				       const void *data, resource_size_t size);
 extern const struct drm_i915_gem_object_ops i915_gem_shmem_ops;
 void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
 				     struct sg_table *pages,
 				     bool needs_clflush);
+int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj,
+				const struct drm_i915_gem_pwrite *args);
+int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj,
+			       const struct drm_i915_gem_pread *args);
 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align);
+void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj,
+				     struct sg_table *pages);
+void i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
+				    struct sg_table *pages);
 void i915_gem_flush_free_objects(struct drm_i915_private *i915);
 #define assert_object_held(obj) dma_resv_assert_held((obj)->base.resv)
+/*
+ * If more than one potential simultaneous locker, assert held.
+ */
+static inline void assert_object_held_shared(struct drm_i915_gem_object *obj)
+{
+	/*
+	 * Note mm list lookup is protected by
+	 * kref_get_unless_zero().
+	 */
+	if (IS_ENABLED(CONFIG_LOCKDEP) &&
+	    kref_read(&obj->base.refcount) > 0)
+		assert_object_held(obj);
+}
 static inline int __i915_gem_object_lock(struct drm_i915_gem_object *obj,
 					 struct i915_gem_ww_ctx *ww,
 					 bool intr)
 	dma_resv_unlock(obj->base.resv);
 }
-struct dma_fence *
-i915_gem_object_lock_fence(struct drm_i915_gem_object *obj);
-void i915_gem_object_unlock_fence(struct drm_i915_gem_object *obj,
-				  struct dma_fence *fence);
 static inline void
 i915_gem_object_set_readonly(struct drm_i915_gem_object *obj)
 {
 static inline bool
 i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
 {
-	return i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_STRUCT_PAGE);
+	return obj->flags & I915_BO_ALLOC_STRUCT_PAGE;
 }
 static inline bool
 	return i915_gem_object_type_has(obj, I915_GEM_OBJECT_NO_MMAP);
 }
-static inline bool
-i915_gem_object_needs_async_cancel(const struct drm_i915_gem_object *obj)
-{
-	return i915_gem_object_type_has(obj, I915_GEM_OBJECT_ASYNC_CANCEL);
-}
 static inline bool
 i915_gem_object_is_framebuffer(const struct drm_i915_gem_object *obj)
 {
 __i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
 			 struct i915_gem_object_page_iter *iter,
 			 unsigned int n,
-			 unsigned int *offset);
+			 unsigned int *offset, bool allow_alloc);
 static inline struct scatterlist *
 i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
 		       unsigned int n,
-		       unsigned int *offset)
+		       unsigned int *offset, bool allow_alloc)
 {
-	return __i915_gem_object_get_sg(obj, &obj->mm.get_page, n, offset);
+	return __i915_gem_object_get_sg(obj, &obj->mm.get_page, n, offset, allow_alloc);
 }
 static inline struct scatterlist *
 i915_gem_object_get_sg_dma(struct drm_i915_gem_object *obj,
 			   unsigned int n,
-			   unsigned int *offset)
+			   unsigned int *offset, bool allow_alloc)
 {
-	return __i915_gem_object_get_sg(obj, &obj->mm.get_dma_page, n, offset);
+	return __i915_gem_object_get_sg(obj, &obj->mm.get_dma_page, n, offset, allow_alloc);
 }
 struct page *
 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
-enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock/struct_mutex */
-	I915_MM_NORMAL = 0,
-	/*
-	 * Only used by struct_mutex, when called "recursively" from
-	 * direct-reclaim-esque. Safe because there is only every one
-	 * struct_mutex in the entire system.
-	 */
-	I915_MM_SHRINKER = 1,
-	/*
-	 * Used for obj->mm.lock when allocating pages. Safe because the object
-	 * isn't yet on any LRU, and therefore the shrinker can't deadlock on
-	 * it. As soon as the object has pages, obj->mm.lock nests within
-	 * fs_reclaim.
-	 */
-	I915_MM_GET_PAGES = 1,
-};
 static inline int __must_check
 i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
 {
-	might_lock_nested(&obj->mm.lock, I915_MM_GET_PAGES);
+	assert_object_held(obj);
 	if (atomic_inc_not_zero(&obj->mm.pages_pin_count))
 		return 0;
 	return __i915_gem_object_get_pages(obj);
 }
+int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj);
 static inline bool
 i915_gem_object_has_pages(struct drm_i915_gem_object *obj)
 {
 void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
 					   enum i915_map_type type);
+void *__must_check i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj,
+						    enum i915_map_type type);
 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
 				 unsigned long offset,
 				 unsigned long size);
 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write);
 struct i915_vma * __must_check
 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
+				     struct i915_gem_ww_ctx *ww,
 				     u32 alignment,
 				     const struct i915_ggtt_view *view,
 				     unsigned int flags);
 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj);
+#ifdef CONFIG_MMU_NOTIFIER
+static inline bool
+i915_gem_object_is_userptr(struct drm_i915_gem_object *obj)
+{
+	return obj->userptr.notifier.mm;
+}
+int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj);
+int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj);
+void i915_gem_object_userptr_submit_fini(struct drm_i915_gem_object *obj);
+int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj);
+#else
+static inline bool i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) { return false; }
+static inline int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; }
+static inline int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; }
+static inline void i915_gem_object_userptr_submit_fini(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); }
+static inline int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; }
+#endif
 #endif

drivers/gpu/drm/i915/gem/i915_gem_object_blt.c+6−0

 	if (unlikely(err))
 		goto out_put;
+	/* we pinned the pool, mark it as such */
+	intel_gt_buffer_pool_mark_used(pool);
 	cmd = i915_gem_object_pin_map(pool->obj, pool->type);
 	if (IS_ERR(cmd)) {
 		err = PTR_ERR(cmd);
 	if (unlikely(err))
 		goto out_put;
+	/* we pinned the pool, mark it as such */
+	intel_gt_buffer_pool_mark_used(pool);
 	cmd = i915_gem_object_pin_map(pool->obj, pool->type);
 	if (IS_ERR(cmd)) {
 		err = PTR_ERR(cmd);

drivers/gpu/drm/i915/gem/i915_gem_object_types.h+14−9

 #ifndef __I915_GEM_OBJECT_TYPES_H__
 #define __I915_GEM_OBJECT_TYPES_H__
+#include <linux/mmu_notifier.h>
 #include <drm/drm_gem.h>
 #include <uapi/drm/i915_drm.h>
 struct drm_i915_gem_object_ops {
 	unsigned int flags;
-#define I915_GEM_OBJECT_HAS_STRUCT_PAGE	BIT(0)
 #define I915_GEM_OBJECT_HAS_IOMEM	BIT(1)
 #define I915_GEM_OBJECT_IS_SHRINKABLE	BIT(2)
 #define I915_GEM_OBJECT_IS_PROXY	BIT(3)
 #define I915_GEM_OBJECT_NO_MMAP		BIT(4)
-#define I915_GEM_OBJECT_ASYNC_CANCEL	BIT(5)
 	/* Interface between the GEM object and its backing storage.
 	 * get_pages() is called once prior to the use of the associated set
 	unsigned long flags;
 #define I915_BO_ALLOC_CONTIGUOUS BIT(0)
 #define I915_BO_ALLOC_VOLATILE   BIT(1)
-#define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | I915_BO_ALLOC_VOLATILE)
+#define I915_BO_ALLOC_STRUCT_PAGE BIT(2)
-#define I915_BO_READONLY         BIT(2)
+#define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | \
-#define I915_TILING_QUIRK_BIT    3 /* unknown swizzling; do not release! */
+			     I915_BO_ALLOC_VOLATILE | \
+			     I915_BO_ALLOC_STRUCT_PAGE)
+#define I915_BO_READONLY         BIT(3)
+#define I915_TILING_QUIRK_BIT    4 /* unknown swizzling; do not release! */
 	/*
 	 * Is the object to be mapped as read-only to the GPU
 		 * Protects the pages and their use. Do not use directly, but
 		 * instead go through the pin/unpin interfaces.
 		 */
-		struct mutex lock;
 		atomic_t pages_pin_count;
 		atomic_t shrink_pin;
 	unsigned long *bit_17;
 	union {
+#ifdef CONFIG_MMU_NOTIFIER
 		struct i915_gem_userptr {
 			uintptr_t ptr;
+			unsigned long notifier_seq;
-			struct i915_mm_struct *mm;
+			struct mmu_interval_notifier notifier;
-			struct i915_mmu_object *mmu_object;
+			struct page **pvec;
-			struct work_struct *work;
+			int page_ref;
 		} userptr;
+#endif
 		struct drm_mm_node *stolen;

drivers/gpu/drm/i915/gem/i915_gem_pages.c+69−40

@@ -19,7 +19,7 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, · 18 unmodified lines
19	bool shrinkable;	19	bool shrinkable;
20	int i;	20	int i;
21		21
22	lockdep_assert_held(&obj->mm.lock);	22	assert_object_held_shared(obj);
23		23
24	if (i915_gem_object_is_volatile(obj))	24	if (i915_gem_object_is_volatile(obj))
25	obj->mm.madv = I915_MADV_DONTNEED;	25	obj->mm.madv = I915_MADV_DONTNEED;
@@ -70,6 +70,7 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, · 44 unmodified lines
70	struct list_head *list;	70	struct list_head *list;
71	unsigned long flags;	71	unsigned long flags;
72		72
		73	assert_object_held(obj);
73	spin_lock_irqsave(&i915->mm.obj_lock, flags);	74	spin_lock_irqsave(&i915->mm.obj_lock, flags);
74		75
75	i915->mm.shrink_count++;	76	i915->mm.shrink_count++;
@@ -91,6 +92,8 @@ int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj) · 15 unmodified lines
91	struct drm_i915_private *i915 = to_i915(obj->base.dev);	92	struct drm_i915_private *i915 = to_i915(obj->base.dev);
92	int err;	93	int err;
93		94
		95	assert_object_held_shared(obj);
		96
94	if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {	97	if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
95	drm_dbg(&i915->drm,	98	drm_dbg(&i915->drm,
96	"Attempting to obtain a purgeable object\n");	99	"Attempting to obtain a purgeable object\n");
@@ -114,23 +117,41 @@ int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) · 17 unmodified lines
114	{	117	{
115	int err;	118	int err;
116		119
117	err = mutex_lock_interruptible_nested(&obj->mm.lock, I915_MM_GET_PAGES);	120	assert_object_held(obj);
118	if (err)	121
119	return err;	122	assert_object_held_shared(obj);
120		123
121	if (unlikely(!i915_gem_object_has_pages(obj))) {	124	if (unlikely(!i915_gem_object_has_pages(obj))) {
122	GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));	125	GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
123		126
124	err = ____i915_gem_object_get_pages(obj);	127	err = ____i915_gem_object_get_pages(obj);
125	if (err)	128	if (err)
126	goto unlock;	129	return err;
127		130
128	smp_mb__before_atomic();	131	smp_mb__before_atomic();
129	}	132	}
130	atomic_inc(&obj->mm.pages_pin_count);	133	atomic_inc(&obj->mm.pages_pin_count);
131		134
132	unlock:	135	return 0;
133	mutex_unlock(&obj->mm.lock);	136	}
		137
		138	int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj)
		139	{
		140	struct i915_gem_ww_ctx ww;
		141	int err;
		142
		143	i915_gem_ww_ctx_init(&ww, true);
		144	retry:
		145	err = i915_gem_object_lock(obj, &ww);
		146	if (!err)
		147	err = i915_gem_object_pin_pages(obj);
		148
		149	if (err == -EDEADLK) {
		150	err = i915_gem_ww_ctx_backoff(&ww);
		151	if (!err)
		152	goto retry;
		153	}
		154	i915_gem_ww_ctx_fini(&ww);
134	return err;	155	return err;
135	}	156	}
136		157
@@ -145,7 +166,7 @@ void i915_gem_object_truncate(struct drm_i915_gem_object *obj) · 8 unmodified lines
145	/* Try to discard unwanted pages */	166	/* Try to discard unwanted pages */
146	void i915_gem_object_writeback(struct drm_i915_gem_object *obj)	167	void i915_gem_object_writeback(struct drm_i915_gem_object *obj)
147	{	168	{
148	lockdep_assert_held(&obj->mm.lock);	169	assert_object_held_shared(obj);
149	GEM_BUG_ON(i915_gem_object_has_pages(obj));	170	GEM_BUG_ON(i915_gem_object_has_pages(obj));
150		171
151	if (obj->ops->writeback)	172	if (obj->ops->writeback)
@@ -176,6 +197,8 @@ __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) · 24 unmodified lines
176	{	197	{
177	struct sg_table *pages;	198	struct sg_table *pages;
178		199
		200	assert_object_held_shared(obj);
		201
179	pages = fetch_and_zero(&obj->mm.pages);	202	pages = fetch_and_zero(&obj->mm.pages);
180	if (IS_ERR_OR_NULL(pages))	203	if (IS_ERR_OR_NULL(pages))
181	return pages;	204	return pages;
@@ -199,17 +222,12 @@ __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) · 17 unmodified lines
199	int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)	222	int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
200	{	223	{
201	struct sg_table *pages;	224	struct sg_table *pages;
202	int err;
203		225
204	if (i915_gem_object_has_pinned_pages(obj))	226	if (i915_gem_object_has_pinned_pages(obj))
205	return -EBUSY;	227	return -EBUSY;
206		228
207	/* May be called by shrinker from within get_pages() (on another bo) */	229	/* May be called by shrinker from within get_pages() (on another bo) */
208	mutex_lock(&obj->mm.lock);	230	assert_object_held_shared(obj);
209	if (unlikely(atomic_read(&obj->mm.pages_pin_count))) {
210	err = -EBUSY;
211	goto unlock;
212	}
213		231
214	i915_gem_object_release_mmap_offset(obj);	232	i915_gem_object_release_mmap_offset(obj);
215		233
@@ -226,17 +244,10 @@ int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj) · 10 unmodified lines
226	* get_pages backends we should be better able to handle the	244	* get_pages backends we should be better able to handle the
227	* cancellation of the async task in a more uniform manner.	245	* cancellation of the async task in a more uniform manner.
228	*/	246	*/
229	if (!pages && !i915_gem_object_needs_async_cancel(obj))	247	if (!IS_ERR_OR_NULL(pages))
230	pages = ERR_PTR(-EINVAL);
231
232	if (!IS_ERR(pages))
233	obj->ops->put_pages(obj, pages);	248	obj->ops->put_pages(obj, pages);
234		249
235	err = 0;	250	return 0;
236	unlock:
237	mutex_unlock(&obj->mm.lock);
238
239	return err;
240	}	251	}
241		252
242	/* The 'mapping' part of i915_gem_object_pin_map() below */	253	/* The 'mapping' part of i915_gem_object_pin_map() below */
@@ -333,18 +344,15 @@ void i915_gem_object_pin_map(struct drm_i915_gem_object obj, · 90 unmodified lines
333	enum i915_map_type type)	344	enum i915_map_type type)
334	{	345	{
335	enum i915_map_type has_type;	346	enum i915_map_type has_type;
336	unsigned int flags;
337	bool pinned;	347	bool pinned;
338	void *ptr;	348	void *ptr;
339	int err;	349	int err;
340		350
341	flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE \| I915_GEM_OBJECT_HAS_IOMEM;	351	if (!i915_gem_object_has_struct_page(obj) &&
342	if (!i915_gem_object_type_has(obj, flags))	352	!i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM))
343	return ERR_PTR(-ENXIO);	353	return ERR_PTR(-ENXIO);
344		354
345	err = mutex_lock_interruptible_nested(&obj->mm.lock, I915_MM_GET_PAGES);	355	assert_object_held(obj);
346	if (err)
347	return ERR_PTR(err);
348		356
349	pinned = !(type & I915_MAP_OVERRIDE);	357	pinned = !(type & I915_MAP_OVERRIDE);
350	type &= ~I915_MAP_OVERRIDE;	358	type &= ~I915_MAP_OVERRIDE;
@@ -354,10 +362,8 @@ void i915_gem_object_pin_map(struct drm_i915_gem_object obj, · 3 unmodified lines
354	GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));	362	GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
355		363
356	err = ____i915_gem_object_get_pages(obj);	364	err = ____i915_gem_object_get_pages(obj);
357	if (err) {	365	if (err)
358	ptr = ERR_PTR(err);	366	return ERR_PTR(err);
359	goto out_unlock;
360	}
361		367
362	smp_mb__before_atomic();	368	smp_mb__before_atomic();
363	}	369	}
@@ -392,13 +398,23 @@ void i915_gem_object_pin_map(struct drm_i915_gem_object obj, · 28 unmodified lines
392	obj->mm.mapping = page_pack_bits(ptr, type);	398	obj->mm.mapping = page_pack_bits(ptr, type);
393	}	399	}
394		400
395	out_unlock:
396	mutex_unlock(&obj->mm.lock);
397	return ptr;	401	return ptr;
398		402
399	err_unpin:	403	err_unpin:
400	atomic_dec(&obj->mm.pages_pin_count);	404	atomic_dec(&obj->mm.pages_pin_count);
401	goto out_unlock;	405	return ptr;
		406	}
		407
		408	void i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object obj,
		409	enum i915_map_type type)
		410	{
		411	void *ret;
		412
		413	i915_gem_object_lock(obj, NULL);
		414	ret = i915_gem_object_pin_map(obj, type);
		415	i915_gem_object_unlock(obj);
		416
		417	return ret;
402	}	418	}
403		419
404	void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,	420	void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
@@ -448,7 +464,8 @@ struct scatterlist * · 43 unmodified lines
448	__i915_gem_object_get_sg(struct drm_i915_gem_object *obj,	464	__i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
449	struct i915_gem_object_page_iter *iter,	465	struct i915_gem_object_page_iter *iter,
450	unsigned int n,	466	unsigned int n,
451	unsigned int *offset)	467	unsigned int *offset,
		468	bool allow_alloc)
452	{	469	{
453	const bool dma = iter == &obj->mm.get_dma_page;	470	const bool dma = iter == &obj->mm.get_dma_page;
454	struct scatterlist *sg;	471	struct scatterlist *sg;
@@ -470,6 +487,9 @@ __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, · 15 unmodified lines
470	if (n < READ_ONCE(iter->sg_idx))	487	if (n < READ_ONCE(iter->sg_idx))
471	goto lookup;	488	goto lookup;
472		489
		490	if (!allow_alloc)
		491	goto manual_lookup;
		492
473	mutex_lock(&iter->lock);	493	mutex_lock(&iter->lock);
474		494
475	/* We prefer to reuse the last sg so that repeated lookup of this	495	/* We prefer to reuse the last sg so that repeated lookup of this
@@ -519,7 +539,16 @@ scan: · 43 unmodified lines
519	if (unlikely(n < idx)) /* insertion completed by another thread */	539	if (unlikely(n < idx)) /* insertion completed by another thread */
520	goto lookup;	540	goto lookup;
521		541
522	/* In case we failed to insert the entry into the radixtree, we need	542	goto manual_walk;
		543
		544	manual_lookup:
		545	idx = 0;
		546	sg = obj->mm.pages->sgl;
		547	count = __sg_page_count(sg);
		548
		549	manual_walk:
		550	/*
		551	* In case we failed to insert the entry into the radixtree, we need
523	* to look beyond the current sg.	552	* to look beyond the current sg.
524	*/	553	*/
525	while (idx + count <= n) {	554	while (idx + count <= n) {
@@ -566,7 +595,7 @@ i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) · 40 unmodified lines
566		595
567	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));	596	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
568		597
569	sg = i915_gem_object_get_sg(obj, n, &offset);	598	sg = i915_gem_object_get_sg(obj, n, &offset, true);
570	return nth_page(sg_page(sg), offset);	599	return nth_page(sg_page(sg), offset);
571	}	600	}
572		601
@@ -592,7 +621,7 @@ i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, · 19 unmodified lines
592	struct scatterlist *sg;	621	struct scatterlist *sg;
593	unsigned int offset;	622	unsigned int offset;
594		623
595	sg = i915_gem_object_get_sg_dma(obj, n, &offset);	624	sg = i915_gem_object_get_sg_dma(obj, n, &offset, true);
596		625
597	if (len)	626	if (len)
598	*len = sg_dma_len(sg) - (offset << PAGE_SHIFT);	627	*len = sg_dma_len(sg) - (offset << PAGE_SHIFT);

drivers/gpu/drm/i915/gem/i915_gem_phys.c+47−63

 	intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt);
+	/* We're no longer struct page backed */
+	obj->flags &= ~I915_BO_ALLOC_STRUCT_PAGE;
 	__i915_gem_object_set_pages(obj, st, sg->length);
 	return 0;
 	return -ENOMEM;
 }
-static void
+void
 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
 			       struct sg_table *pages)
 {
 			  vaddr, dma);
 }
-static int
+int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj,
-phys_pwrite(struct drm_i915_gem_object *obj,
+				const struct drm_i915_gem_pwrite *args)
-	    const struct drm_i915_gem_pwrite *args)
 {
 	void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset;
 	char __user *user_data = u64_to_user_ptr(args->data_ptr);
 	return 0;
 }
-static int
+int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj,
-phys_pread(struct drm_i915_gem_object *obj,
+			       const struct drm_i915_gem_pread *args)
-	   const struct drm_i915_gem_pread *args)
 {
 	void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset;
 	char __user *user_data = u64_to_user_ptr(args->data_ptr);
 	return 0;
 }
-static void phys_release(struct drm_i915_gem_object *obj)
+static int i915_gem_object_shmem_to_phys(struct drm_i915_gem_object *obj)
-{
-	fput(obj->base.filp);
-}
-static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
-	.name = "i915_gem_object_phys",
-	.get_pages = i915_gem_object_get_pages_phys,
-	.put_pages = i915_gem_object_put_pages_phys,
-	.pread  = phys_pread,
-	.pwrite = phys_pwrite,
-	.release = phys_release,
-};
-int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
 {
 	struct sg_table *pages;
 	int err;
-	if (align > obj->base.size)
-		return -EINVAL;
-	if (obj->ops == &i915_gem_phys_ops)
-		return 0;
-	if (!i915_gem_object_is_shmem(obj))
-		return -EINVAL;
-	err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
-	if (err)
-		return err;
-	mutex_lock_nested(&obj->mm.lock, I915_MM_GET_PAGES);
-	if (obj->mm.madv != I915_MADV_WILLNEED) {
-		err = -EFAULT;
-		goto err_unlock;
-	}
-	if (i915_gem_object_has_tiling_quirk(obj)) {
-		err = -EFAULT;
-		goto err_unlock;
-	}
-	if (obj->mm.mapping) {
-		err = -EBUSY;
-		goto err_unlock;
-	}
 	pages = __i915_gem_object_unset_pages(obj);
-	obj->ops = &i915_gem_phys_ops;
+	err = i915_gem_object_get_pages_phys(obj);
-	err = ____i915_gem_object_get_pages(obj);
 	if (err)
 		goto err_xfer;
 	__i915_gem_object_pin_pages(obj);
 	if (!IS_ERR_OR_NULL(pages))
-		i915_gem_shmem_ops.put_pages(obj, pages);
+		i915_gem_object_put_pages_shmem(obj, pages);
 	i915_gem_object_release_memory_region(obj);
-	mutex_unlock(&obj->mm.lock);
 	return 0;
 err_xfer:
-	obj->ops = &i915_gem_shmem_ops;
 	if (!IS_ERR_OR_NULL(pages)) {
 		unsigned int sg_page_sizes = i915_sg_page_sizes(pages->sgl);
 		__i915_gem_object_set_pages(obj, pages, sg_page_sizes);
 	}
-err_unlock:
-	mutex_unlock(&obj->mm.lock);
 	return err;
 }
+int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
+{
+	int err;
+	assert_object_held(obj);
+	if (align > obj->base.size)
+		return -EINVAL;
+	if (!i915_gem_object_is_shmem(obj))
+		return -EINVAL;
+	if (!i915_gem_object_has_struct_page(obj))
+		return 0;
+	err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
+	if (err)
+		return err;
+	if (obj->mm.madv != I915_MADV_WILLNEED)
+		return -EFAULT;
+	if (i915_gem_object_has_tiling_quirk(obj))
+		return -EFAULT;
+	if (obj->mm.mapping || i915_gem_object_has_pinned_pages(obj))
+		return -EBUSY;
+	if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
+		drm_dbg(obj->base.dev,
+			"Attempting to obtain a purgeable object\n");
+		return -EFAULT;
+	}
+	return i915_gem_object_shmem_to_phys(obj);
+}
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
 #include "selftests/i915_gem_phys.c"
 #endif

drivers/gpu/drm/i915/gem/i915_gem_pm.c+1−1

@@ -116,7 +116,7 @@ int i915_gem_freeze_late(struct drm_i915_private *i915) · 115 unmodified lines
116	*/	116	*/
117		117
118	with_intel_runtime_pm(&i915->runtime_pm, wakeref)	118	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
119	i915_gem_shrink(i915, -1UL, NULL, ~0);	119	i915_gem_shrink(NULL, i915, -1UL, NULL, ~0);
120	i915_gem_drain_freed_objects(i915);	120	i915_gem_drain_freed_objects(i915);
121		121
122	wbinvd_on_all_cpus();	122	wbinvd_on_all_cpus();

drivers/gpu/drm/i915/gem/i915_gem_region.c+1−3

 }
 void i915_gem_object_init_memory_region(struct drm_i915_gem_object *obj,
-					struct intel_memory_region *mem,
+					struct intel_memory_region *mem)
-					unsigned long flags)
 {
 	INIT_LIST_HEAD(&obj->mm.blocks);
 	obj->mm.region = intel_memory_region_get(mem);
-	obj->flags |= flags;
 	if (obj->base.size <= mem->min_page_size)
 		obj->flags |= I915_BO_ALLOC_CONTIGUOUS;

drivers/gpu/drm/i915/gem/i915_gem_region.h+1−2

@@ -17,8 +17,7 @@ void i915_gem_object_put_pages_buddy(struct drm_i915_gem_object *obj, · 16 unmodified lines
17	struct sg_table *pages);	17	struct sg_table *pages);
18		18
19	void i915_gem_object_init_memory_region(struct drm_i915_gem_object *obj,	19	void i915_gem_object_init_memory_region(struct drm_i915_gem_object *obj,
20	struct intel_memory_region *mem,	20	struct intel_memory_region *mem);
21	unsigned long flags);
22	void i915_gem_object_release_memory_region(struct drm_i915_gem_object *obj);	21	void i915_gem_object_release_memory_region(struct drm_i915_gem_object *obj);
23		22
24	struct drm_i915_gem_object *	23	struct drm_i915_gem_object *

drivers/gpu/drm/i915/gem/i915_gem_shmem.c+31−8

 				goto err_sg;
 			}
-			i915_gem_shrink(i915, 2 * page_count, NULL, *s++);
+			i915_gem_shrink(NULL, i915, 2 * page_count, NULL, *s++);
 			/*
 			 * We've tried hard to allocate the memory by reaping
 	__start_cpu_write(obj);
 }
-static void
+void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages)
-shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages)
 {
 	struct sgt_iter sgt_iter;
 	struct pagevec pvec;
 	kfree(pages);
 }
+static void
+shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages)
+{
+	if (likely(i915_gem_object_has_struct_page(obj)))
+		i915_gem_object_put_pages_shmem(obj, pages);
+	else
+		i915_gem_object_put_pages_phys(obj, pages);
+}
 static int
 shmem_pwrite(struct drm_i915_gem_object *obj,
 	     const struct drm_i915_gem_pwrite *arg)
 	/* Caller already validated user args */
 	GEM_BUG_ON(!access_ok(user_data, arg->size));
+	if (!i915_gem_object_has_struct_page(obj))
+		return i915_gem_object_pwrite_phys(obj, arg);
 	/*
 	 * Before we instantiate/pin the backing store for our use, we
 	 * can prepopulate the shmemfs filp efficiently using a write into
 	return 0;
 }
+static int
+shmem_pread(struct drm_i915_gem_object *obj,
+	    const struct drm_i915_gem_pread *arg)
+{
+	if (!i915_gem_object_has_struct_page(obj))
+		return i915_gem_object_pread_phys(obj, arg);
+	return -ENODEV;
+}
 static void shmem_release(struct drm_i915_gem_object *obj)
 {
-	i915_gem_object_release_memory_region(obj);
+	if (obj->flags & I915_BO_ALLOC_STRUCT_PAGE)
+		i915_gem_object_release_memory_region(obj);
 	fput(obj->base.filp);
 }
 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = {
 	.name = "i915_gem_object_shmem",
-	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
+	.flags = I915_GEM_OBJECT_IS_SHRINKABLE,
-		 I915_GEM_OBJECT_IS_SHRINKABLE,
 	.get_pages = shmem_get_pages,
 	.put_pages = shmem_put_pages,
 	.writeback = shmem_writeback,
 	.pwrite = shmem_pwrite,
+	.pread = shmem_pread,
 	.release = shmem_release,
 };
 	mapping_set_gfp_mask(mapping, mask);
 	GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));
-	i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class);
+	i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class,
+			     I915_BO_ALLOC_STRUCT_PAGE);
 	obj->write_domain = I915_GEM_DOMAIN_CPU;
 	obj->read_domains = I915_GEM_DOMAIN_CPU;
 	i915_gem_object_set_cache_coherency(obj, cache_level);
-	i915_gem_object_init_memory_region(obj, mem, 0);
+	i915_gem_object_init_memory_region(obj, mem);
 	return 0;
 }

drivers/gpu/drm/i915/gem/i915_gem_shrinker.c+28−11

 		flags = I915_GEM_OBJECT_UNBIND_TEST;
 	if (i915_gem_object_unbind(obj, flags) == 0)
-		__i915_gem_object_put_pages(obj);
+		return true;
-	return !i915_gem_object_has_pages(obj);
+	return false;
 }
 static void try_to_writeback(struct drm_i915_gem_object *obj,
  * The number of pages of backing storage actually released.
  */
 unsigned long
-i915_gem_shrink(struct drm_i915_private *i915,
+i915_gem_shrink(struct i915_gem_ww_ctx *ww,
+		struct drm_i915_private *i915,
 		unsigned long target,
 		unsigned long *nr_scanned,
 		unsigned int shrink)
 	intel_wakeref_t wakeref = 0;
 	unsigned long count = 0;
 	unsigned long scanned = 0;
+	int err;
 	trace_i915_gem_shrink(i915, target, shrink);
 			spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
+			err = 0;
 			if (unsafe_drop_pages(obj, shrink)) {
 				/* May arrive from get_pages on another bo */
-				mutex_lock(&obj->mm.lock);
+				if (!ww) {
-				if (!i915_gem_object_has_pages(obj)) {
+					if (!i915_gem_object_trylock(obj))
+						goto skip;
+				} else {
+					err = i915_gem_object_lock(obj, ww);
+					if (err)
+						goto skip;
+				}
+				if (!__i915_gem_object_put_pages(obj)) {
 					try_to_writeback(obj, shrink);
 					count += obj->base.size >> PAGE_SHIFT;
 				}
-				mutex_unlock(&obj->mm.lock);
+				if (!ww)
+					i915_gem_object_unlock(obj);
 			}
 			dma_resv_prune(obj->base.resv);
 			scanned += obj->base.size >> PAGE_SHIFT;
+skip:
 			i915_gem_object_put(obj);
 			spin_lock_irqsave(&i915->mm.obj_lock, flags);
+			if (err)
+				break;
 		}
 		list_splice_tail(&still_in_list, phase->list);
 		spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
+		if (err)
+			return err;
 	}
 	if (shrink & I915_SHRINK_BOUND)
 	unsigned long freed = 0;
 	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
-		freed = i915_gem_shrink(i915, -1UL, NULL,
+		freed = i915_gem_shrink(NULL, i915, -1UL, NULL,
 					I915_SHRINK_BOUND |
 					I915_SHRINK_UNBOUND);
 	}
 	sc->nr_scanned = 0;
-	freed = i915_gem_shrink(i915,
+	freed = i915_gem_shrink(NULL, i915,
 				sc->nr_to_scan,
 				&sc->nr_scanned,
 				I915_SHRINK_BOUND |
 		intel_wakeref_t wakeref;
 		with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
-			freed += i915_gem_shrink(i915,
+			freed += i915_gem_shrink(NULL, i915,
 						 sc->nr_to_scan - sc->nr_scanned,
 						 &sc->nr_scanned,
 						 I915_SHRINK_ACTIVE |
 	freed_pages = 0;
 	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
-		freed_pages += i915_gem_shrink(i915, -1UL, NULL,
+		freed_pages += i915_gem_shrink(NULL, i915, -1UL, NULL,
 					       I915_SHRINK_BOUND |
 					       I915_SHRINK_UNBOUND |
 					       I915_SHRINK_WRITEBACK);
 	intel_wakeref_t wakeref;
 	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
-		freed_pages += i915_gem_shrink(i915, -1UL, NULL,
+		freed_pages += i915_gem_shrink(NULL, i915, -1UL, NULL,
 					       I915_SHRINK_BOUND |
 					       I915_SHRINK_UNBOUND |
 					       I915_SHRINK_VMAPS);

drivers/gpu/drm/i915/gem/i915_gem_shrinker.h+3−1

 #include <linux/bits.h>
 struct drm_i915_private;
+struct i915_gem_ww_ctx;
 struct mutex;
 /* i915_gem_shrinker.c */
-unsigned long i915_gem_shrink(struct drm_i915_private *i915,
+unsigned long i915_gem_shrink(struct i915_gem_ww_ctx *ww,
+			      struct drm_i915_private *i915,
 			      unsigned long target,
 			      unsigned long *nr_scanned,
 			      unsigned flags);

drivers/gpu/drm/i915/gem/i915_gem_stolen.c+8−6

 	int err;
 	drm_gem_private_object_init(&mem->i915->drm, &obj->base, stolen->size);
-	i915_gem_object_init(obj, &i915_gem_object_stolen_ops, &lock_class);
+	i915_gem_object_init(obj, &i915_gem_object_stolen_ops, &lock_class, 0);
 	obj->stolen = stolen;
 	obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
 	cache_level = HAS_LLC(mem->i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
 	i915_gem_object_set_cache_coherency(obj, cache_level);
-	err = i915_gem_object_pin_pages(obj);
+	if (WARN_ON(!i915_gem_object_trylock(obj)))
-	if (err)
+		return -EBUSY;
-		return err;
-	i915_gem_object_init_memory_region(obj, mem, 0);
+	err = i915_gem_object_pin_pages(obj);
+	if (!err)
+		i915_gem_object_init_memory_region(obj, mem);
+	i915_gem_object_unlock(obj);
-	return 0;
+	return err;
 }
 static int _i915_gem_object_stolen_init(struct intel_memory_region *mem,

drivers/gpu/drm/i915/gem/i915_gem_tiling.c+0−2

 	 * pages to prevent them being swapped out and causing corruption
 	 * due to the change in swizzling.
 	 */
-	mutex_lock(&obj->mm.lock);
 	if (i915_gem_object_has_pages(obj) &&
 	    obj->mm.madv == I915_MADV_WILLNEED &&
 	    i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
 			i915_gem_object_set_tiling_quirk(obj);
 		}
 	}
-	mutex_unlock(&obj->mm.lock);
 	spin_lock(&obj->vma.lock);
 	for_each_ggtt_vma(vma, obj) {

drivers/gpu/drm/i915/gem/i915_gem_userptr.c+320−582

@@ -2,10 +2,39 @@ · 1 unmodified line
2	* SPDX-License-Identifier: MIT	2	* SPDX-License-Identifier: MIT
3	*	3	*
4	* Copyright © 2012-2014 Intel Corporation	4	* Copyright © 2012-2014 Intel Corporation
		5	*
		6	* Based on amdgpu_mn, which bears the following notice:
		7	*
		8	* Copyright 2014 Advanced Micro Devices, Inc.
		9	* All Rights Reserved.
		10	*
		11	* Permission is hereby granted, free of charge, to any person obtaining a
		12	* copy of this software and associated documentation files (the
		13	* "Software"), to deal in the Software without restriction, including
		14	* without limitation the rights to use, copy, modify, merge, publish,
		15	* distribute, sub license, and/or sell copies of the Software, and to
		16	* permit persons to whom the Software is furnished to do so, subject to
		17	* the following conditions:
		18	*
		19	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
		20	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
		21	* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
		22	* THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
		23	* DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
		24	* OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
		25	* USE OR OTHER DEALINGS IN THE SOFTWARE.
		26	*
		27	* The above copyright notice and this permission notice (including the
		28	* next paragraph) shall be included in all copies or substantial portions
		29	* of the Software.
		30	*
		31	*/
		32	/*
		33	* Authors:
		34	* Christian König <christian.koenig@amd.com>
5	*/	35	*/
6		36
7	#include <linux/mmu_context.h>	37	#include <linux/mmu_context.h>
8	#include <linux/mmu_notifier.h>
9	#include <linux/mempolicy.h>	38	#include <linux/mempolicy.h>
10	#include <linux/swap.h>	39	#include <linux/swap.h>
11	#include <linux/sched/mm.h>	40	#include <linux/sched/mm.h>
@@ -15,408 +44,121 @@ · 3 unmodified lines
15	#include "i915_gem_object.h"	44	#include "i915_gem_object.h"
16	#include "i915_scatterlist.h"	45	#include "i915_scatterlist.h"
17		46
18	struct i915_mm_struct {	47	#ifdef CONFIG_MMU_NOTIFIER
19	struct mm_struct *mm;
20	struct drm_i915_private *i915;
21	struct i915_mmu_notifier *mn;
22	struct hlist_node node;
23	struct kref kref;
24	struct rcu_work work;
25	};
26
27	#if defined(CONFIG_MMU_NOTIFIER)
28	#include <linux/interval_tree.h>
29		48
30	struct i915_mmu_notifier {	49	/**
31	spinlock_t lock;	50	* i915_gem_userptr_invalidate - callback to notify about mm change
32	struct hlist_node node;	51	*
33	struct mmu_notifier mn;	52	* @mni: the range (mm) is about to update
34	struct rb_root_cached objects;	53	* @range: details on the invalidation
35	struct i915_mm_struct *mm;	54	* @cur_seq: Value to pass to mmu_interval_set_seq()
36	};	55	*
37		56	* Block for operations on BOs to finish and mark pages as accessed and
38	struct i915_mmu_object {	57	* potentially dirty.
39	struct i915_mmu_notifier *mn;	58	*/
40	struct drm_i915_gem_object *obj;	59	static bool i915_gem_userptr_invalidate(struct mmu_interval_notifier *mni,
41	struct interval_tree_node it;	60	const struct mmu_notifier_range *range,
42	};	61	unsigned long cur_seq)
43
44	static void add_object(struct i915_mmu_object *mo)
45	{	62	{
46	GEM_BUG_ON(!RB_EMPTY_NODE(&mo->it.rb));	63	struct drm_i915_gem_object *obj = container_of(mni, struct drm_i915_gem_object, userptr.notifier);
47	interval_tree_insert(&mo->it, &mo->mn->objects);	64	struct drm_i915_private *i915 = to_i915(obj->base.dev);
48	}	65	long r;
49		66
50	static void del_object(struct i915_mmu_object *mo)	67	if (!mmu_notifier_range_blockable(range))
51	{	68	return false;
52	if (RB_EMPTY_NODE(&mo->it.rb))
53	return;
54		69
55	interval_tree_remove(&mo->it, &mo->mn->objects);	70	spin_lock(&i915->mm.notifier_lock);
56	RB_CLEAR_NODE(&mo->it.rb);
57	}
58		71
59	static void	72	mmu_interval_set_seq(mni, cur_seq);
60	__i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value)	73
61	{	74	spin_unlock(&i915->mm.notifier_lock);
62	struct i915_mmu_object *mo = obj->userptr.mmu_object;
63		75
64	/*	76	/*
65	* During mm_invalidate_range we need to cancel any userptr that	77	* We don't wait when the process is exiting. This is valid
66	* overlaps the range being invalidated. Doing so requires the	78	* because the object will be cleaned up anyway.
67	* struct_mutex, and that risks recursion. In order to cause	79	*
68	* recursion, the user must alias the userptr address space with	80	* This is also temporarily required as a hack, because we
69	* a GTT mmapping (possible with a MAP_FIXED) - then when we have	81	* cannot currently force non-consistent batch buffers to preempt
70	* to invalidate that mmaping, mm_invalidate_range is called with	82	* and reschedule by waiting on it, hanging processes on exit.
71	* the userptr address and the struct_mutex held. To prevent that
72	* we set a flag under the i915_mmu_notifier spinlock to indicate
73	* whether this object is valid.
74	*/	83	*/
75	if (!mo)	84	if (current->flags & PF_EXITING)
76	return;	85	return true;
77		86
78	spin_lock(&mo->mn->lock);	87	/* we will unbind on next submission, still have userptr pins */
79	if (value)	88	r = dma_resv_wait_timeout_rcu(obj->base.resv, true, false,
80	add_object(mo);	89	MAX_SCHEDULE_TIMEOUT);
81	else	90	if (r <= 0)
82	del_object(mo);	91	drm_err(&i915->drm, "(%ld) failed to wait for idle\n", r);
83	spin_unlock(&mo->mn->lock);
84	}
85
86	static int
87	userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
88	const struct mmu_notifier_range *range)
89	{
90	struct i915_mmu_notifier *mn =
91	container_of(_mn, struct i915_mmu_notifier, mn);
92	struct interval_tree_node *it;
93	unsigned long end;
94	int ret = 0;
95
96	if (RB_EMPTY_ROOT(&mn->objects.rb_root))
97	return 0;
98
99	/* interval ranges are inclusive, but invalidate range is exclusive */
100	end = range->end - 1;
101
102	spin_lock(&mn->lock);
103	it = interval_tree_iter_first(&mn->objects, range->start, end);
104	while (it) {
105	struct drm_i915_gem_object *obj;
106
107	if (!mmu_notifier_range_blockable(range)) {
108	ret = -EAGAIN;
109	break;
110	}
111
112	/*
113	* The mmu_object is released late when destroying the
114	* GEM object so it is entirely possible to gain a
115	* reference on an object in the process of being freed
116	* since our serialisation is via the spinlock and not
117	* the struct_mutex - and consequently use it after it
118	* is freed and then double free it. To prevent that
119	* use-after-free we only acquire a reference on the
120	* object if it is not in the process of being destroyed.
121	*/
122	obj = container_of(it, struct i915_mmu_object, it)->obj;
123	if (!kref_get_unless_zero(&obj->base.refcount)) {
124	it = interval_tree_iter_next(it, range->start, end);
125	continue;
126	}
127	spin_unlock(&mn->lock);
128
129	ret = i915_gem_object_unbind(obj,
130	I915_GEM_OBJECT_UNBIND_ACTIVE \|
131	I915_GEM_OBJECT_UNBIND_BARRIER);
132	if (ret == 0)
133	ret = __i915_gem_object_put_pages(obj);
134	i915_gem_object_put(obj);
135	if (ret)
136	return ret;
137
138	spin_lock(&mn->lock);
139
140	/*
141	* As we do not (yet) protect the mmu from concurrent insertion
142	* over this range, there is no guarantee that this search will
143	* terminate given a pathologic workload.
144	*/
145	it = interval_tree_iter_first(&mn->objects, range->start, end);
146	}
147	spin_unlock(&mn->lock);
148
149	return ret;
150		92
		93	return true;
151	}	94	}
152		95
153	static const struct mmu_notifier_ops i915_gem_userptr_notifier = {	96	static const struct mmu_interval_notifier_ops i915_gem_userptr_notifier_ops = {
154	.invalidate_range_start = userptr_mn_invalidate_range_start,	97	.invalidate = i915_gem_userptr_invalidate,
155	};	98	};
156		99
157	static struct i915_mmu_notifier *
158	i915_mmu_notifier_create(struct i915_mm_struct *mm)
159	{
160	struct i915_mmu_notifier *mn;
161
162	mn = kmalloc(sizeof(*mn), GFP_KERNEL);
163	if (mn == NULL)
164	return ERR_PTR(-ENOMEM);
165
166	spin_lock_init(&mn->lock);
167	mn->mn.ops = &i915_gem_userptr_notifier;
168	mn->objects = RB_ROOT_CACHED;
169	mn->mm = mm;
170
171	return mn;
172	}
173
174	static void
175	i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)
176	{
177	struct i915_mmu_object *mo;
178
179	mo = fetch_and_zero(&obj->userptr.mmu_object);
180	if (!mo)
181	return;
182
183	spin_lock(&mo->mn->lock);
184	del_object(mo);
185	spin_unlock(&mo->mn->lock);
186	kfree(mo);
187	}
188
189	static struct i915_mmu_notifier *
190	i915_mmu_notifier_find(struct i915_mm_struct *mm)
191	{
192	struct i915_mmu_notifier mn, old;
193	int err;
194
195	mn = READ_ONCE(mm->mn);
196	if (likely(mn))
197	return mn;
198
199	mn = i915_mmu_notifier_create(mm);
200	if (IS_ERR(mn))
201	return mn;
202
203	err = mmu_notifier_register(&mn->mn, mm->mm);
204	if (err) {
205	kfree(mn);
206	return ERR_PTR(err);
207	}
208
209	old = cmpxchg(&mm->mn, NULL, mn);
210	if (old) {
211	mmu_notifier_unregister(&mn->mn, mm->mm);
212	kfree(mn);
213	mn = old;
214	}
215
216	return mn;
217	}
218
219	static int
220	i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,
221	unsigned flags)
222	{
223	struct i915_mmu_notifier *mn;
224	struct i915_mmu_object *mo;
225
226	if (flags & I915_USERPTR_UNSYNCHRONIZED)
227	return capable(CAP_SYS_ADMIN) ? 0 : -EPERM;
228
229	if (GEM_WARN_ON(!obj->userptr.mm))
230	return -EINVAL;
231
232	mn = i915_mmu_notifier_find(obj->userptr.mm);
233	if (IS_ERR(mn))
234	return PTR_ERR(mn);
235
236	mo = kzalloc(sizeof(*mo), GFP_KERNEL);
237	if (!mo)
238	return -ENOMEM;
239
240	mo->mn = mn;
241	mo->obj = obj;
242	mo->it.start = obj->userptr.ptr;
243	mo->it.last = obj->userptr.ptr + obj->base.size - 1;
244	RB_CLEAR_NODE(&mo->it.rb);
245
246	obj->userptr.mmu_object = mo;
247	return 0;
248	}
249
250	static void
251	i915_mmu_notifier_free(struct i915_mmu_notifier *mn,
252	struct mm_struct *mm)
253	{
254	if (mn == NULL)
255	return;
256
257	mmu_notifier_unregister(&mn->mn, mm);
258	kfree(mn);
259	}
260
261	#else
262
263	static void
264	__i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value)
265	{
266	}
267
268	static void
269	i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)
270	{
271	}
272
273	static int	100	static int
274	i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,	101	i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj)
275	unsigned flags)
276	{	102	{
277	if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0)	103	return mmu_interval_notifier_insert(&obj->userptr.notifier, current->mm,
278	return -ENODEV;	104	obj->userptr.ptr, obj->base.size,
279		105	&i915_gem_userptr_notifier_ops);
280	if (!capable(CAP_SYS_ADMIN))
281	return -EPERM;
282
283	return 0;
284	}	106	}
285		107
286	static void	108	static void i915_gem_object_userptr_drop_ref(struct drm_i915_gem_object *obj)
287	i915_mmu_notifier_free(struct i915_mmu_notifier *mn,
288	struct mm_struct *mm)
289	{
290	}
291
292	#endif
293
294	static struct i915_mm_struct *
295	__i915_mm_struct_find(struct drm_i915_private i915, struct mm_struct real)
296	{
297	struct i915_mm_struct it, mm = NULL;
298
299	rcu_read_lock();
300	hash_for_each_possible_rcu(i915->mm_structs,
301	it, node,
302	(unsigned long)real)
303	if (it->mm == real && kref_get_unless_zero(&it->kref)) {
304	mm = it;
305	break;
306	}
307	rcu_read_unlock();
308
309	return mm;
310	}
311
312	static int
313	i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj)
314	{	109	{
315	struct drm_i915_private *i915 = to_i915(obj->base.dev);	110	struct drm_i915_private *i915 = to_i915(obj->base.dev);
316	struct i915_mm_struct mm, new;	111	struct page **pvec = NULL;
317	int ret = 0;
318
319	/* During release of the GEM object we hold the struct_mutex. This
320	* precludes us from calling mmput() at that time as that may be
321	* the last reference and so call exit_mmap(). exit_mmap() will
322	* attempt to reap the vma, and if we were holding a GTT mmap
323	* would then call drm_gem_vm_close() and attempt to reacquire
324	* the struct mutex. So in order to avoid that recursion, we have
325	* to defer releasing the mm reference until after we drop the
326	* struct_mutex, i.e. we need to schedule a worker to do the clean
327	* up.
328	*/
329	mm = __i915_mm_struct_find(i915, current->mm);
330	if (mm)
331	goto out;
332
333	new = kmalloc(sizeof(*mm), GFP_KERNEL);
334	if (!new)
335	return -ENOMEM;
336		112
337	kref_init(&new->kref);	113	spin_lock(&i915->mm.notifier_lock);
338	new->i915 = to_i915(obj->base.dev);	114	if (!--obj->userptr.page_ref) {
339	new->mm = current->mm;	115	pvec = obj->userptr.pvec;
340	new->mn = NULL;	116	obj->userptr.pvec = NULL;
341
342	spin_lock(&i915->mm_lock);
343	mm = __i915_mm_struct_find(i915, current->mm);
344	if (!mm) {
345	hash_add_rcu(i915->mm_structs,
346	&new->node,
347	(unsigned long)new->mm);
348	mmgrab(current->mm);
349	mm = new;
350	}	117	}
351	spin_unlock(&i915->mm_lock);	118	GEM_BUG_ON(obj->userptr.page_ref < 0);
352	if (mm != new)	119	spin_unlock(&i915->mm.notifier_lock);
353	kfree(new);
354		120
355	out:	121	if (pvec) {
356	obj->userptr.mm = mm;	122	const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
357	return ret;
358	}
359
360	static void
361	__i915_mm_struct_free__worker(struct work_struct *work)
362	{
363	struct i915_mm_struct mm = container_of(work, typeof(mm), work.work);
364
365	i915_mmu_notifier_free(mm->mn, mm->mm);
366	mmdrop(mm->mm);
367	kfree(mm);
368	}
369
370	static void
371	__i915_mm_struct_free(struct kref *kref)
372	{
373	struct i915_mm_struct mm = container_of(kref, typeof(mm), kref);
374		123
375	spin_lock(&mm->i915->mm_lock);	124	unpin_user_pages(pvec, num_pages);
376	hash_del_rcu(&mm->node);	125	kvfree(pvec);
377	spin_unlock(&mm->i915->mm_lock);	126	}
378
379	INIT_RCU_WORK(&mm->work, __i915_mm_struct_free__worker);
380	queue_rcu_work(system_wq, &mm->work);
381	}
382
383	static void
384	i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj)
385	{
386	if (obj->userptr.mm == NULL)
387	return;
388
389	kref_put(&obj->userptr.mm->kref, __i915_mm_struct_free);
390	obj->userptr.mm = NULL;
391	}	127	}
392		128
393	struct get_pages_work {	129	static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
394	struct work_struct work;
395	struct drm_i915_gem_object *obj;
396	struct task_struct *task;
397	};
398
399	static struct sg_table *
400	__i915_gem_userptr_alloc_pages(struct drm_i915_gem_object *obj,
401	struct page **pvec, unsigned long num_pages)
402	{	130	{
		131	struct drm_i915_private *i915 = to_i915(obj->base.dev);
		132	const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
403	unsigned int max_segment = i915_sg_segment_size();	133	unsigned int max_segment = i915_sg_segment_size();
404	struct sg_table *st;	134	struct sg_table *st;
405	unsigned int sg_page_sizes;	135	unsigned int sg_page_sizes;
406	struct scatterlist *sg;	136	struct scatterlist *sg;
		137	struct page **pvec;
407	int ret;	138	int ret;
408		139
409	st = kmalloc(sizeof(*st), GFP_KERNEL);	140	st = kmalloc(sizeof(*st), GFP_KERNEL);
410	if (!st)	141	if (!st)
411	return ERR_PTR(-ENOMEM);	142	return -ENOMEM;
		143
		144	spin_lock(&i915->mm.notifier_lock);
		145	if (GEM_WARN_ON(!obj->userptr.page_ref)) {
		146	spin_unlock(&i915->mm.notifier_lock);
		147	ret = -EFAULT;
		148	goto err_free;
		149	}
		150
		151	obj->userptr.page_ref++;
		152	pvec = obj->userptr.pvec;
		153	spin_unlock(&i915->mm.notifier_lock);
412		154
413	alloc_table:	155	alloc_table:
414	sg = __sg_alloc_table_from_pages(st, pvec, num_pages, 0,	156	sg = __sg_alloc_table_from_pages(st, pvec, num_pages, 0,
415	num_pages << PAGE_SHIFT, max_segment,	157	num_pages << PAGE_SHIFT, max_segment,
416	NULL, 0, GFP_KERNEL);	158	NULL, 0, GFP_KERNEL);
417	if (IS_ERR(sg)) {	159	if (IS_ERR(sg)) {
418	kfree(st);	160	ret = PTR_ERR(sg);
419	return ERR_CAST(sg);	161	goto err;
420	}	162	}
421		163
422	ret = i915_gem_gtt_prepare_pages(obj, st);	164	ret = i915_gem_gtt_prepare_pages(obj, st);
@@ -428,203 +170,20 @@ alloc_table: · 5 unmodified lines
428	goto alloc_table;	170	goto alloc_table;
429	}	171	}
430		172
431	kfree(st);	173	goto err;
432	return ERR_PTR(ret);
433	}	174	}
434		175
435	sg_page_sizes = i915_sg_page_sizes(st->sgl);	176	sg_page_sizes = i915_sg_page_sizes(st->sgl);
436		177
437	__i915_gem_object_set_pages(obj, st, sg_page_sizes);	178	__i915_gem_object_set_pages(obj, st, sg_page_sizes);
438		179
439	return st;	180	return 0;
440	}
441
442	static void
443	__i915_gem_userptr_get_pages_worker(struct work_struct *_work)
444	{
445	struct get_pages_work work = container_of(_work, typeof(work), work);
446	struct drm_i915_gem_object *obj = work->obj;
447	const unsigned long npages = obj->base.size >> PAGE_SHIFT;
448	unsigned long pinned;
449	struct page **pvec;
450	int ret;
451
452	ret = -ENOMEM;
453	pinned = 0;
454
455	pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
456	if (pvec != NULL) {
457	struct mm_struct *mm = obj->userptr.mm->mm;
458	unsigned int flags = 0;
459	int locked = 0;
460
461	if (!i915_gem_object_is_readonly(obj))
462	flags \|= FOLL_WRITE;
463
464	ret = -EFAULT;
465	if (mmget_not_zero(mm)) {
466	while (pinned < npages) {
467	if (!locked) {
468	mmap_read_lock(mm);
469	locked = 1;
470	}
471	ret = pin_user_pages_remote
472	(mm,
473	obj->userptr.ptr + pinned * PAGE_SIZE,
474	npages - pinned,
475	flags,
476	pvec + pinned, NULL, &locked);
477	if (ret < 0)
478	break;
479
480	pinned += ret;
481	}
482	if (locked)
483	mmap_read_unlock(mm);
484	mmput(mm);
485	}
486	}
487
488	mutex_lock_nested(&obj->mm.lock, I915_MM_GET_PAGES);
489	if (obj->userptr.work == &work->work) {
490	struct sg_table *pages = ERR_PTR(ret);
491
492	if (pinned == npages) {
493	pages = __i915_gem_userptr_alloc_pages(obj, pvec,
494	npages);
495	if (!IS_ERR(pages)) {
496	pinned = 0;
497	pages = NULL;
498	}
499	}
500
501	obj->userptr.work = ERR_CAST(pages);
502	if (IS_ERR(pages))
503	__i915_gem_userptr_set_active(obj, false);
504	}
505	mutex_unlock(&obj->mm.lock);
506
507	unpin_user_pages(pvec, pinned);
508	kvfree(pvec);
509
510	i915_gem_object_put(obj);
511	put_task_struct(work->task);
512	kfree(work);
513	}
514
515	static struct sg_table *
516	__i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj)
517	{
518	struct get_pages_work *work;
519
520	/* Spawn a worker so that we can acquire the
521	* user pages without holding our mutex. Access
522	* to the user pages requires mmap_lock, and we have
523	* a strict lock ordering of mmap_lock, struct_mutex -
524	* we already hold struct_mutex here and so cannot
525	* call gup without encountering a lock inversion.
526	*
527	* Userspace will keep on repeating the operation
528	* (thanks to EAGAIN) until either we hit the fast
529	* path or the worker completes. If the worker is
530	* cancelled or superseded, the task is still run
531	* but the results ignored. (This leads to
532	* complications that we may have a stray object
533	* refcount that we need to be wary of when
534	* checking for existing objects during creation.)
535	* If the worker encounters an error, it reports
536	* that error back to this function through
537	* obj->userptr.work = ERR_PTR.
538	*/
539	work = kmalloc(sizeof(*work), GFP_KERNEL);
540	if (work == NULL)
541	return ERR_PTR(-ENOMEM);
542
543	obj->userptr.work = &work->work;
544
545	work->obj = i915_gem_object_get(obj);
546
547	work->task = current;
548	get_task_struct(work->task);
549
550	INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker);
551	queue_work(to_i915(obj->base.dev)->mm.userptr_wq, &work->work);
552
553	return ERR_PTR(-EAGAIN);
554	}
555
556	static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
557	{
558	const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
559	struct mm_struct *mm = obj->userptr.mm->mm;
560	struct page **pvec;
561	struct sg_table *pages;
562	bool active;
563	int pinned;
564	unsigned int gup_flags = 0;
565
566	/* If userspace should engineer that these pages are replaced in
567	* the vma between us binding this page into the GTT and completion
568	* of rendering... Their loss. If they change the mapping of their
569	* pages they need to create a new bo to point to the new vma.
570	*
571	* However, that still leaves open the possibility of the vma
572	* being copied upon fork. Which falls under the same userspace
573	* synchronisation issue as a regular bo, except that this time
574	* the process may not be expecting that a particular piece of
575	* memory is tied to the GPU.
576	*
577	* Fortunately, we can hook into the mmu_notifier in order to
578	* discard the page references prior to anything nasty happening
579	* to the vma (discard or cloning) which should prevent the more
580	* egregious cases from causing harm.
581	*/
582
583	if (obj->userptr.work) {
584	/* active flag should still be held for the pending work */
585	if (IS_ERR(obj->userptr.work))
586	return PTR_ERR(obj->userptr.work);
587	else
588	return -EAGAIN;
589	}
590
591	pvec = NULL;
592	pinned = 0;
593
594	if (mm == current->mm) {
595	pvec = kvmalloc_array(num_pages, sizeof(struct page *),
596	GFP_KERNEL \|
597	__GFP_NORETRY \|
598	__GFP_NOWARN);
599	if (pvec) {
600	/* defer to worker if malloc fails */
601	if (!i915_gem_object_is_readonly(obj))
602	gup_flags \|= FOLL_WRITE;
603	pinned = pin_user_pages_fast_only(obj->userptr.ptr,
604	num_pages, gup_flags,
605	pvec);
606	}
607	}
608
609	active = false;
610	if (pinned < 0) {
611	pages = ERR_PTR(pinned);
612	pinned = 0;
613	} else if (pinned < num_pages) {
614	pages = __i915_gem_userptr_get_pages_schedule(obj);
615	active = pages == ERR_PTR(-EAGAIN);
616	} else {
617	pages = __i915_gem_userptr_alloc_pages(obj, pvec, num_pages);
618	active = !IS_ERR(pages);
619	}
620	if (active)
621	__i915_gem_userptr_set_active(obj, true);
622
623	if (IS_ERR(pages))
624	unpin_user_pages(pvec, pinned);
625	kvfree(pvec);
626		181
627	return PTR_ERR_OR_ZERO(pages);	182	err:
		183	i915_gem_object_userptr_drop_ref(obj);
		184	err_free:
		185	kfree(st);
		186	return ret;
628	}	187	}
629		188
630	static void	189	static void
@@ -634,9 +193,6 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj, · 3 unmodified lines
634	struct sgt_iter sgt_iter;	193	struct sgt_iter sgt_iter;
635	struct page *page;	194	struct page *page;
636		195
637	/* Cancel any inflight work and force them to restart their gup */
638	obj->userptr.work = NULL;
639	__i915_gem_userptr_set_active(obj, false);
640	if (!pages)	196	if (!pages)
641	return;	197	return;
642		198
@@ -676,42 +232,224 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj, · 33 unmodified lines
676	}	232	}
677		233
678	mark_page_accessed(page);	234	mark_page_accessed(page);
679	unpin_user_page(page);
680	}	235	}
681	obj->mm.dirty = false;	236	obj->mm.dirty = false;
682		237
683	sg_free_table(pages);	238	sg_free_table(pages);
684	kfree(pages);	239	kfree(pages);
		240
		241	i915_gem_object_userptr_drop_ref(obj);
		242	}
		243
		244	static int i915_gem_object_userptr_unbind(struct drm_i915_gem_object *obj, bool get_pages)
		245	{
		246	struct sg_table *pages;
		247	int err;
		248
		249	err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
		250	if (err)
		251	return err;
		252
		253	if (GEM_WARN_ON(i915_gem_object_has_pinned_pages(obj)))
		254	return -EBUSY;
		255
		256	assert_object_held(obj);
		257
		258	pages = __i915_gem_object_unset_pages(obj);
		259	if (!IS_ERR_OR_NULL(pages))
		260	i915_gem_userptr_put_pages(obj, pages);
		261
		262	if (get_pages)
		263	err = ____i915_gem_object_get_pages(obj);
		264
		265	return err;
		266	}
		267
		268	int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj)
		269	{
		270	struct drm_i915_private *i915 = to_i915(obj->base.dev);
		271	const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
		272	struct page **pvec;
		273	unsigned int gup_flags = 0;
		274	unsigned long notifier_seq;
		275	int pinned, ret;
		276
		277	if (obj->userptr.notifier.mm != current->mm)
		278	return -EFAULT;
		279
		280	ret = i915_gem_object_lock_interruptible(obj, NULL);
		281	if (ret)
		282	return ret;
		283
		284	/* optimistically try to preserve current pages while unlocked */
		285	if (i915_gem_object_has_pages(obj) &&
		286	!mmu_interval_check_retry(&obj->userptr.notifier,
		287	obj->userptr.notifier_seq)) {
		288	spin_lock(&i915->mm.notifier_lock);
		289	if (obj->userptr.pvec &&
		290	!mmu_interval_read_retry(&obj->userptr.notifier,
		291	obj->userptr.notifier_seq)) {
		292	obj->userptr.page_ref++;
		293
		294	/* We can keep using the current binding, this is the fastpath */
		295	ret = 1;
		296	}
		297	spin_unlock(&i915->mm.notifier_lock);
		298	}
		299
		300	if (!ret) {
		301	/* Make sure userptr is unbound for next attempt, so we don't use stale pages. */
		302	ret = i915_gem_object_userptr_unbind(obj, false);
		303	}
		304	i915_gem_object_unlock(obj);
		305	if (ret < 0)
		306	return ret;
		307
		308	if (ret > 0)
		309	return 0;
		310
		311	notifier_seq = mmu_interval_read_begin(&obj->userptr.notifier);
		312
		313	pvec = kvmalloc_array(num_pages, sizeof(struct page *), GFP_KERNEL);
		314	if (!pvec)
		315	return -ENOMEM;
		316
		317	if (!i915_gem_object_is_readonly(obj))
		318	gup_flags \|= FOLL_WRITE;
		319
		320	pinned = ret = 0;
		321	while (pinned < num_pages) {
		322	ret = pin_user_pages_fast(obj->userptr.ptr + pinned * PAGE_SIZE,
		323	num_pages - pinned, gup_flags,
		324	&pvec[pinned]);
		325	if (ret < 0)
		326	goto out;
		327
		328	pinned += ret;
		329	}
		330	ret = 0;
		331
		332	spin_lock(&i915->mm.notifier_lock);
		333
		334	if (mmu_interval_read_retry(&obj->userptr.notifier,
		335	!obj->userptr.page_ref ? notifier_seq :
		336	obj->userptr.notifier_seq)) {
		337	ret = -EAGAIN;
		338	goto out_unlock;
		339	}
		340
		341	if (!obj->userptr.page_ref++) {
		342	obj->userptr.pvec = pvec;
		343	obj->userptr.notifier_seq = notifier_seq;
		344
		345	pvec = NULL;
		346	}
		347
		348	out_unlock:
		349	spin_unlock(&i915->mm.notifier_lock);
		350
		351	out:
		352	if (pvec) {
		353	unpin_user_pages(pvec, pinned);
		354	kvfree(pvec);
		355	}
		356
		357	return ret;
		358	}
		359
		360	int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj)
		361	{
		362	if (mmu_interval_read_retry(&obj->userptr.notifier,
		363	obj->userptr.notifier_seq)) {
		364	/* We collided with the mmu notifier, need to retry */
		365
		366	return -EAGAIN;
		367	}
		368
		369	return 0;
		370	}
		371
		372	void i915_gem_object_userptr_submit_fini(struct drm_i915_gem_object *obj)
		373	{
		374	i915_gem_object_userptr_drop_ref(obj);
		375	}
		376
		377	int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj)
		378	{
		379	int err;
		380
		381	err = i915_gem_object_userptr_submit_init(obj);
		382	if (err)
		383	return err;
		384
		385	err = i915_gem_object_lock_interruptible(obj, NULL);
		386	if (!err) {
		387	/*
		388	* Since we only check validity, not use the pages,
		389	* it doesn't matter if we collide with the mmu notifier,
		390	* and -EAGAIN handling is not required.
		391	*/
		392	err = i915_gem_object_pin_pages(obj);
		393	if (!err)
		394	i915_gem_object_unpin_pages(obj);
		395
		396	i915_gem_object_unlock(obj);
		397	}
		398
		399	i915_gem_object_userptr_submit_fini(obj);
		400	return err;
685	}	401	}
686		402
687	static void	403	static void
688	i915_gem_userptr_release(struct drm_i915_gem_object *obj)	404	i915_gem_userptr_release(struct drm_i915_gem_object *obj)
689	{	405	{
690	i915_gem_userptr_release__mmu_notifier(obj);	406	GEM_WARN_ON(obj->userptr.page_ref);
691	i915_gem_userptr_release__mm_struct(obj);	407
		408	mmu_interval_notifier_remove(&obj->userptr.notifier);
		409	obj->userptr.notifier.mm = NULL;
692	}	410	}
693		411
694	static int	412	static int
695	i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)	413	i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)
696	{	414	{
697	if (obj->userptr.mmu_object)	415	drm_dbg(obj->base.dev, "Exporting userptr no longer allowed\n");
698	return 0;
699		416
700	return i915_gem_userptr_init__mmu_notifier(obj, 0);	417	return -EINVAL;
		418	}
		419
		420	static int
		421	i915_gem_userptr_pwrite(struct drm_i915_gem_object *obj,
		422	const struct drm_i915_gem_pwrite *args)
		423	{
		424	drm_dbg(obj->base.dev, "pwrite to userptr no longer allowed\n");
		425
		426	return -EINVAL;
		427	}
		428
		429	static int
		430	i915_gem_userptr_pread(struct drm_i915_gem_object *obj,
		431	const struct drm_i915_gem_pread *args)
		432	{
		433	drm_dbg(obj->base.dev, "pread from userptr no longer allowed\n");
		434
		435	return -EINVAL;
701	}	436	}
702		437
703	static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {	438	static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {
704	.name = "i915_gem_object_userptr",	439	.name = "i915_gem_object_userptr",
705	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE \|	440	.flags = I915_GEM_OBJECT_IS_SHRINKABLE \|
706	I915_GEM_OBJECT_IS_SHRINKABLE \|
707	I915_GEM_OBJECT_NO_MMAP \|	441	I915_GEM_OBJECT_NO_MMAP \|
708	I915_GEM_OBJECT_ASYNC_CANCEL,	442	I915_GEM_OBJECT_IS_PROXY,
709	.get_pages = i915_gem_userptr_get_pages,	443	.get_pages = i915_gem_userptr_get_pages,
710	.put_pages = i915_gem_userptr_put_pages,	444	.put_pages = i915_gem_userptr_put_pages,
711	.dmabuf_export = i915_gem_userptr_dmabuf_export,	445	.dmabuf_export = i915_gem_userptr_dmabuf_export,
		446	.pwrite = i915_gem_userptr_pwrite,
		447	.pread = i915_gem_userptr_pread,
712	.release = i915_gem_userptr_release,	448	.release = i915_gem_userptr_release,
713	};	449	};
714		450
		451	#endif
		452
715	/*	453	/*
716	* Creates a new mm object that wraps some normal memory from the process	454	* Creates a new mm object that wraps some normal memory from the process
717	* context - user memory.	455	* context - user memory.
@@ -752,12 +490,12 @@ i915_gem_userptr_ioctl(struct drm_device *dev, · 34 unmodified lines
752	void *data,	490	void *data,
753	struct drm_file *file)	491	struct drm_file *file)
754	{	492	{
755	static struct lock_class_key lock_class;	493	static struct lock_class_key __maybe_unused lock_class;
756	struct drm_i915_private *dev_priv = to_i915(dev);	494	struct drm_i915_private *dev_priv = to_i915(dev);
757	struct drm_i915_gem_userptr *args = data;	495	struct drm_i915_gem_userptr *args = data;
758	struct drm_i915_gem_object *obj;	496	struct drm_i915_gem_object __maybe_unused *obj;
759	int ret;	497	int __maybe_unused ret;
760	u32 handle;	498	u32 __maybe_unused handle;
761		499
762	if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) {	500	if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) {
763	/* We cannot support coherent userptr objects on hw without	501	/* We cannot support coherent userptr objects on hw without
@@ -796,6 +534,9 @@ i915_gem_userptr_ioctl(struct drm_device *dev, · 32 unmodified lines
796	if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size))	534	if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size))
797	return -EFAULT;	535	return -EFAULT;
798		536
		537	if (args->flags & I915_USERPTR_UNSYNCHRONIZED)
		538	return -ENODEV;
		539
799	if (args->flags & I915_USERPTR_READ_ONLY) {	540	if (args->flags & I915_USERPTR_READ_ONLY) {
800	/*	541	/*
801	* On almost all of the older hw, we cannot tell the GPU that	542	* On almost all of the older hw, we cannot tell the GPU that
@@ -805,17 +546,20 @@ i915_gem_userptr_ioctl(struct drm_device *dev, · 3 unmodified lines
805	return -ENODEV;	546	return -ENODEV;
806	}	547	}
807		548
		549	#ifdef CONFIG_MMU_NOTIFIER
808	obj = i915_gem_object_alloc();	550	obj = i915_gem_object_alloc();
809	if (obj == NULL)	551	if (obj == NULL)
810	return -ENOMEM;	552	return -ENOMEM;
811		553
812	drm_gem_private_object_init(dev, &obj->base, args->user_size);	554	drm_gem_private_object_init(dev, &obj->base, args->user_size);
813	i915_gem_object_init(obj, &i915_gem_userptr_ops, &lock_class);	555	i915_gem_object_init(obj, &i915_gem_userptr_ops, &lock_class,
		556	I915_BO_ALLOC_STRUCT_PAGE);
814	obj->read_domains = I915_GEM_DOMAIN_CPU;	557	obj->read_domains = I915_GEM_DOMAIN_CPU;
815	obj->write_domain = I915_GEM_DOMAIN_CPU;	558	obj->write_domain = I915_GEM_DOMAIN_CPU;
816	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);	559	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
817		560
818	obj->userptr.ptr = args->user_ptr;	561	obj->userptr.ptr = args->user_ptr;
		562	obj->userptr.notifier_seq = ULONG_MAX;
819	if (args->flags & I915_USERPTR_READ_ONLY)	563	if (args->flags & I915_USERPTR_READ_ONLY)
820	i915_gem_object_set_readonly(obj);	564	i915_gem_object_set_readonly(obj);
821		565
@@ -823,9 +567,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev, · 1 unmodified line
823	* at binding. This means that we need to hook into the mmu_notifier	567	* at binding. This means that we need to hook into the mmu_notifier
824	* in order to detect if the mmu is destroyed.	568	* in order to detect if the mmu is destroyed.
825	*/	569	*/
826	ret = i915_gem_userptr_init__mm_struct(obj);	570	ret = i915_gem_userptr_init__mmu_notifier(obj);
827	if (ret == 0)
828	ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags);
829	if (ret == 0)	571	if (ret == 0)
830	ret = drm_gem_handle_create(file, &obj->base, &handle);	572	ret = drm_gem_handle_create(file, &obj->base, &handle);
831		573
@@ -836,24 +578,20 @@ i915_gem_userptr_ioctl(struct drm_device *dev, · 4 unmodified lines
836		578
837	args->handle = handle;	579	args->handle = handle;
838	return 0;	580	return 0;
		581	#else
		582	return -ENODEV;
		583	#endif
839	}	584	}
840		585
841	int i915_gem_init_userptr(struct drm_i915_private *dev_priv)	586	int i915_gem_init_userptr(struct drm_i915_private *dev_priv)
842	{	587	{
843	spin_lock_init(&dev_priv->mm_lock);	588	#ifdef CONFIG_MMU_NOTIFIER
844	hash_init(dev_priv->mm_structs);	589	spin_lock_init(&dev_priv->mm.notifier_lock);
845		590	#endif
846	dev_priv->mm.userptr_wq =
847	alloc_workqueue("i915-userptr-acquire",
848	WQ_HIGHPRI \| WQ_UNBOUND,
849	0);
850	if (!dev_priv->mm.userptr_wq)
851	return -ENOMEM;
852		591
853	return 0;	592	return 0;
854	}	593	}
855		594
856	void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv)	595	void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv)
857	{	596	{
858	destroy_workqueue(dev_priv->mm.userptr_wq);
859	}	597	}

drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c+2−2

 static const struct drm_i915_gem_object_ops huge_ops = {
 	.name = "huge-gem",
-	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
 	.get_pages = huge_get_pages,
 	.put_pages = huge_put_pages,
 };
 		return ERR_PTR(-ENOMEM);
 	drm_gem_private_object_init(&i915->drm, &obj->base, dma_size);
-	i915_gem_object_init(obj, &huge_ops, &lock_class);
+	i915_gem_object_init(obj, &huge_ops, &lock_class,
+			     I915_BO_ALLOC_STRUCT_PAGE);
 	obj->read_domains = I915_GEM_DOMAIN_CPU;
 	obj->write_domain = I915_GEM_DOMAIN_CPU;

drivers/gpu/drm/i915/gem/selftests/huge_pages.c+26−12

 static const struct drm_i915_gem_object_ops huge_page_ops = {
 	.name = "huge-gem",
-	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
+	.flags = I915_GEM_OBJECT_IS_SHRINKABLE,
-		 I915_GEM_OBJECT_IS_SHRINKABLE,
 	.get_pages = get_huge_pages,
 	.put_pages = put_huge_pages,
 };
 		return ERR_PTR(-ENOMEM);
 	drm_gem_private_object_init(&i915->drm, &obj->base, size);
-	i915_gem_object_init(obj, &huge_page_ops, &lock_class);
+	i915_gem_object_init(obj, &huge_page_ops, &lock_class,
+			     I915_BO_ALLOC_STRUCT_PAGE);
 	i915_gem_object_set_volatile(obj);
 	drm_gem_private_object_init(&i915->drm, &obj->base, size);
 	if (single)
-		i915_gem_object_init(obj, &fake_ops_single, &lock_class);
+		i915_gem_object_init(obj, &fake_ops_single, &lock_class, 0);
 	else
-		i915_gem_object_init(obj, &fake_ops, &lock_class);
+		i915_gem_object_init(obj, &fake_ops, &lock_class, 0);
 	i915_gem_object_set_volatile(obj);
 			goto out_put;
 		}
-		err = i915_gem_object_pin_pages(obj);
+		err = i915_gem_object_pin_pages_unlocked(obj);
 		if (err)
 			goto out_put;
 				break;
 		}
+		i915_gem_object_lock(obj, NULL);
 		i915_gem_object_unpin_pages(obj);
 		__i915_gem_object_put_pages(obj);
+		i915_gem_object_unlock(obj);
 		i915_gem_object_put(obj);
 	}
 	return 0;
 out_unpin:
+	i915_gem_object_lock(obj, NULL);
 	i915_gem_object_unpin_pages(obj);
+	i915_gem_object_unlock(obj);
 out_put:
 	i915_gem_object_put(obj);
 	list_for_each_entry_safe(obj, on, objects, st_link) {
 		list_del(&obj->st_link);
+		i915_gem_object_lock(obj, NULL);
 		i915_gem_object_unpin_pages(obj);
 		__i915_gem_object_put_pages(obj);
+		i915_gem_object_unlock(obj);
 		i915_gem_object_put(obj);
 	}
 }
 			break;
 		}
-		err = i915_gem_object_pin_pages(obj);
+		err = i915_gem_object_pin_pages_unlocked(obj);
 		if (err) {
 			i915_gem_object_put(obj);
 			break;
 			if (IS_ERR(obj))
 				return PTR_ERR(obj);
-			err = i915_gem_object_pin_pages(obj);
+			err = i915_gem_object_pin_pages_unlocked(obj);
 			if (err)
 				goto out_object_put;
 			}
 			i915_vma_unpin(vma);
+			i915_gem_object_lock(obj, NULL);
 			i915_gem_object_unpin_pages(obj);
 			__i915_gem_object_put_pages(obj);
+			i915_gem_object_unlock(obj);
 			i915_gem_object_put(obj);
 		}
 	}
 out_vma_unpin:
 	i915_vma_unpin(vma);
 out_object_unpin:
+	i915_gem_object_lock(obj, NULL);
 	i915_gem_object_unpin_pages(obj);
+	i915_gem_object_unlock(obj);
 out_object_put:
 	i915_gem_object_put(obj);
 	if (err)
 		return err;
-	ptr = i915_gem_object_pin_map(obj, I915_MAP_WC);
+	ptr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 	if (IS_ERR(ptr))
 		return PTR_ERR(ptr);
 			return err;
 		}
-		err = i915_gem_object_pin_pages(obj);
+		err = i915_gem_object_pin_pages_unlocked(obj);
 		if (err) {
 			if (err == -ENXIO || err == -E2BIG) {
 				i915_gem_object_put(obj);
 			       __func__, size, i);
 		}
 out_unpin:
+		i915_gem_object_lock(obj, NULL);
 		i915_gem_object_unpin_pages(obj);
 		__i915_gem_object_put_pages(obj);
+		i915_gem_object_unlock(obj);
 out_put:
 		i915_gem_object_put(obj);
 				return err;
 			}
-			err = i915_gem_object_pin_pages(obj);
+			err = i915_gem_object_pin_pages_unlocked(obj);
 			if (err) {
 				i915_gem_object_put(obj);
 				goto out;
 			err = igt_write_huge(ctx, obj);
+			i915_gem_object_lock(obj, NULL);
 			i915_gem_object_unpin_pages(obj);
 			__i915_gem_object_put_pages(obj);
+			i915_gem_object_unlock(obj);
 			i915_gem_object_put(obj);
 			if (err) {
 		goto out_restore;
 	}
-	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
+	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
 	if (IS_ERR(vaddr)) {
 		err = PTR_ERR(vaddr);
 		goto out_put;

drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c+4−4

@@ -45,7 +45,7 @@ static int __igt_client_fill(struct intel_engine_cs *engine) · 44 unmodified lines
45	goto err_flush;	45	goto err_flush;
46	}	46	}
47		47
48	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);	48	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
49	if (IS_ERR(vaddr)) {	49	if (IS_ERR(vaddr)) {
50	err = PTR_ERR(vaddr);	50	err = PTR_ERR(vaddr);
51	goto err_put;	51	goto err_put;
@@ -157,7 +157,7 @@ static int prepare_blit(const struct tiled_blits *t, · 105 unmodified lines
157	u32 src_pitch, dst_pitch;	157	u32 src_pitch, dst_pitch;
158	u32 cmd, *cs;	158	u32 cmd, *cs;
159		159
160	cs = i915_gem_object_pin_map(batch, I915_MAP_WC);	160	cs = i915_gem_object_pin_map_unlocked(batch, I915_MAP_WC);
161	if (IS_ERR(cs))	161	if (IS_ERR(cs))
162	return PTR_ERR(cs);	162	return PTR_ERR(cs);
163		163
@@ -377,7 +377,7 @@ static int verify_buffer(const struct tiled_blits *t, · 213 unmodified lines
377	y = i915_prandom_u32_max_state(t->height, prng);	377	y = i915_prandom_u32_max_state(t->height, prng);
378	p = y * t->width + x;	378	p = y * t->width + x;
379		379
380	vaddr = i915_gem_object_pin_map(buf->vma->obj, I915_MAP_WC);	380	vaddr = i915_gem_object_pin_map_unlocked(buf->vma->obj, I915_MAP_WC);
381	if (IS_ERR(vaddr))	381	if (IS_ERR(vaddr))
382	return PTR_ERR(vaddr);	382	return PTR_ERR(vaddr);
383		383
@@ -564,7 +564,7 @@ static int tiled_blits_prepare(struct tiled_blits *t, · 180 unmodified lines
564	int err;	564	int err;
565	int i;	565	int i;
566		566
567	map = i915_gem_object_pin_map(t->scratch.vma->obj, I915_MAP_WC);	567	map = i915_gem_object_pin_map_unlocked(t->scratch.vma->obj, I915_MAP_WC);
568	if (IS_ERR(map))	568	if (IS_ERR(map))
569	return PTR_ERR(map);	569	return PTR_ERR(map);
570		570

drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c+6−8

 	if (err)
 		return err;
-	map = i915_gem_object_pin_map(ctx->obj, I915_MAP_WC);
+	map = i915_gem_object_pin_map_unlocked(ctx->obj, I915_MAP_WC);
 	if (IS_ERR(map))
 		return PTR_ERR(map);
 	if (err)
 		return err;
-	map = i915_gem_object_pin_map(ctx->obj, I915_MAP_WC);
+	map = i915_gem_object_pin_map_unlocked(ctx->obj, I915_MAP_WC);
 	if (IS_ERR(map))
 		return PTR_ERR(map);
 	u32 *cs;
 	int err;
+	vma = i915_gem_object_ggtt_pin(ctx->obj, NULL, 0, 0, 0);
+	if (IS_ERR(vma))
+		return PTR_ERR(vma);
 	i915_gem_object_lock(ctx->obj, NULL);
 	err = i915_gem_object_set_to_gtt_domain(ctx->obj, true);
 	if (err)
 		goto out_unlock;
-	vma = i915_gem_object_ggtt_pin(ctx->obj, NULL, 0, 0, 0);
-	if (IS_ERR(vma)) {
-		err = PTR_ERR(vma);
-		goto out_unlock;
-	}
 	rq = intel_engine_create_kernel_request(ctx->engine);
 	if (IS_ERR(rq)) {
 		err = PTR_ERR(rq);

drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c+5−5

@@ -1094,7 +1094,7 @@ __read_slice_count(struct intel_context *ce, · 1093 unmodified lines
1094	if (ret < 0)	1094	if (ret < 0)
1095	return ret;	1095	return ret;
1096		1096
1097	buf = i915_gem_object_pin_map(obj, I915_MAP_WB);	1097	buf = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1098	if (IS_ERR(buf)) {	1098	if (IS_ERR(buf)) {
1099	ret = PTR_ERR(buf);	1099	ret = PTR_ERR(buf);
1100	return ret;	1100	return ret;
@@ -1511,7 +1511,7 @@ static int write_to_scratch(struct i915_gem_context *ctx, · 410 unmodified lines
1511	if (IS_ERR(obj))	1511	if (IS_ERR(obj))
1512	return PTR_ERR(obj);	1512	return PTR_ERR(obj);
1513		1513
1514	cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);	1514	cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1515	if (IS_ERR(cmd)) {	1515	if (IS_ERR(cmd)) {
1516	err = PTR_ERR(cmd);	1516	err = PTR_ERR(cmd);
1517	goto out;	1517	goto out;
@@ -1622,7 +1622,7 @@ static int read_from_scratch(struct i915_gem_context *ctx, · 104 unmodified lines
1622	if (err)	1622	if (err)
1623	goto out_vm;	1623	goto out_vm;
1624		1624
1625	cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);	1625	cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1626	if (IS_ERR(cmd)) {	1626	if (IS_ERR(cmd)) {
1627	err = PTR_ERR(cmd);	1627	err = PTR_ERR(cmd);
1628	goto out;	1628	goto out;
@@ -1658,7 +1658,7 @@ static int read_from_scratch(struct i915_gem_context *ctx, · 29 unmodified lines
1658	if (err)	1658	if (err)
1659	goto out_vm;	1659	goto out_vm;
1660		1660
1661	cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);	1661	cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1662	if (IS_ERR(cmd)) {	1662	if (IS_ERR(cmd)) {
1663	err = PTR_ERR(cmd);	1663	err = PTR_ERR(cmd);
1664	goto out;	1664	goto out;
@@ -1715,7 +1715,7 @@ static int read_from_scratch(struct i915_gem_context *ctx, · 50 unmodified lines
1715	if (err)	1715	if (err)
1716	goto out_vm;	1716	goto out_vm;
1717		1717
1718	cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);	1718	cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
1719	if (IS_ERR(cmd)) {	1719	if (IS_ERR(cmd)) {
1720	err = PTR_ERR(cmd);	1720	err = PTR_ERR(cmd);
1721	goto out_vm;	1721	goto out_vm;

drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c+1−1

@@ -194,7 +194,7 @@ static int igt_dmabuf_import_ownership(void *arg) · 193 unmodified lines
194		194
195	dma_buf_put(dmabuf);	195	dma_buf_put(dmabuf);
196		196
197	err = i915_gem_object_pin_pages(obj);	197	err = i915_gem_object_pin_pages_unlocked(obj);
198	if (err) {	198	if (err) {
199	pr_err("i915_gem_object_pin_pages failed with err=%d\n", err);	199	pr_err("i915_gem_object_pin_pages failed with err=%d\n", err);
200	goto out_obj;	200	goto out_obj;

drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c+1−1

@@ -116,7 +116,7 @@ static int igt_gpu_reloc(void *arg) · 115 unmodified lines
116	if (IS_ERR(scratch))	116	if (IS_ERR(scratch))
117	return PTR_ERR(scratch);	117	return PTR_ERR(scratch);
118		118
119	map = i915_gem_object_pin_map(scratch, I915_MAP_WC);	119	map = i915_gem_object_pin_map_unlocked(scratch, I915_MAP_WC);
120	if (IS_ERR(map)) {	120	if (IS_ERR(map)) {
121	err = PTR_ERR(map);	121	err = PTR_ERR(map);
122	goto err_scratch;	122	goto err_scratch;

drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c+10−11

 	if (IS_ERR(obj))
 		return PTR_ERR(obj);
-	err = i915_gem_object_pin_pages(obj);
+	err = i915_gem_object_pin_pages_unlocked(obj);
 	if (err) {
 		pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
 		       nreal, obj->base.size / PAGE_SIZE, err);
 	if (IS_ERR(obj))
 		return PTR_ERR(obj);
-	err = i915_gem_object_pin_pages(obj);
+	err = i915_gem_object_pin_pages_unlocked(obj);
 	if (err) {
 		pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
 		       nreal, obj->base.size / PAGE_SIZE, err);
 {
 	void *vaddr;
-	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
+	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 	if (IS_ERR(vaddr))
 		return PTR_ERR(vaddr);
 	void *vaddr;
 	int err = 0;
-	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
+	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 	if (IS_ERR(vaddr))
 		return PTR_ERR(vaddr);
 		return false;
 	if (type != I915_MMAP_TYPE_GTT &&
-	    !i915_gem_object_type_has(obj,
+	    !i915_gem_object_has_struct_page(obj) &&
-				      I915_GEM_OBJECT_HAS_STRUCT_PAGE |
+	    !i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM))
-				      I915_GEM_OBJECT_HAS_IOMEM))
 		return false;
 	return true;
 static bool can_access(const struct drm_i915_gem_object *obj)
 {
-	unsigned int flags =
+	return i915_gem_object_has_struct_page(obj) ||
-		I915_GEM_OBJECT_HAS_STRUCT_PAGE | I915_GEM_OBJECT_HAS_IOMEM;
+	       i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM);
-	return i915_gem_object_type_has(obj, flags);
 }
 static int __igt_mmap_access(struct drm_i915_private *i915,
 	}
 	if (type != I915_MMAP_TYPE_GTT) {
+		i915_gem_object_lock(obj, NULL);
 		__i915_gem_object_put_pages(obj);
+		i915_gem_object_unlock(obj);
 		if (i915_gem_object_has_pages(obj)) {
 			pr_err("Failed to put-pages object!\n");
 			err = -EINVAL;

drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c+1−1

@@ -47,7 +47,7 @@ static int igt_gem_huge(void *arg) · 46 unmodified lines
47	if (IS_ERR(obj))	47	if (IS_ERR(obj))
48	return PTR_ERR(obj);	48	return PTR_ERR(obj);
49		49
50	err = i915_gem_object_pin_pages(obj);	50	err = i915_gem_object_pin_pages_unlocked(obj);
51	if (err) {	51	if (err) {
52	pr_err("Failed to allocate %u pages (%lu total), err=%d\n",	52	pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
53	nreal, obj->base.size / PAGE_SIZE, err);	53	nreal, obj->base.size / PAGE_SIZE, err);

drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c+3−3

@@ -262,7 +262,7 @@ static int igt_fill_blt_thread(void *arg) · 261 unmodified lines
262	goto err_flush;	262	goto err_flush;
263	}	263	}
264		264
265	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);	265	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
266	if (IS_ERR(vaddr)) {	266	if (IS_ERR(vaddr)) {
267	err = PTR_ERR(vaddr);	267	err = PTR_ERR(vaddr);
268	goto err_put;	268	goto err_put;
@@ -380,7 +380,7 @@ static int igt_copy_blt_thread(void *arg) · 111 unmodified lines
380	goto err_flush;	380	goto err_flush;
381	}	381	}
382		382
383	vaddr = i915_gem_object_pin_map(src, I915_MAP_WB);	383	vaddr = i915_gem_object_pin_map_unlocked(src, I915_MAP_WB);
384	if (IS_ERR(vaddr)) {	384	if (IS_ERR(vaddr)) {
385	err = PTR_ERR(vaddr);	385	err = PTR_ERR(vaddr);
386	goto err_put_src;	386	goto err_put_src;
@@ -400,7 +400,7 @@ static int igt_copy_blt_thread(void *arg) · 13 unmodified lines
400	goto err_put_src;	400	goto err_put_src;
401	}	401	}
402		402
403	vaddr = i915_gem_object_pin_map(dst, I915_MAP_WB);	403	vaddr = i915_gem_object_pin_map_unlocked(dst, I915_MAP_WB);
404	if (IS_ERR(vaddr)) {	404	if (IS_ERR(vaddr)) {
405	err = PTR_ERR(vaddr);	405	err = PTR_ERR(vaddr);
406	goto err_put_dst;	406	goto err_put_dst;

drivers/gpu/drm/i915/gem/selftests/i915_gem_phys.c+9−1

 		goto out;
 	}
+	if (!i915_gem_object_has_struct_page(obj)) {
+		err = -EINVAL;
+		pr_err("shmem has no struct page\n");
+		goto out_obj;
+	}
+	i915_gem_object_lock(obj, NULL);
 	err = i915_gem_object_attach_phys(obj, PAGE_SIZE);
+	i915_gem_object_unlock(obj);
 	if (err) {
 		pr_err("i915_gem_object_attach_phys failed, err=%d\n", err);
 		goto out_obj;
 	}
-	if (obj->ops != &i915_gem_phys_ops) {
+	if (i915_gem_object_has_struct_page(obj)) {
 		pr_err("i915_gem_object_attach_phys did not create a phys object\n");
 		err = -EINVAL;
 		goto out_obj;

drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c+1−1

@@ -55,7 +55,7 @@ igt_emit_store_dw(struct i915_vma *vma, · 54 unmodified lines
55	if (IS_ERR(obj))	55	if (IS_ERR(obj))
56	return ERR_CAST(obj);	56	return ERR_CAST(obj);
57		57
58	cmd = i915_gem_object_pin_map(obj, I915_MAP_WC);	58	cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
59	if (IS_ERR(cmd)) {	59	if (IS_ERR(cmd)) {
60	err = PTR_ERR(cmd);	60	err = PTR_ERR(cmd);
61	goto err;	61	goto err;

drivers/gpu/drm/i915/gt/gen2_engine_cs.c+1−1

@@ -143,7 +143,7 @@ static u32 __gen2_emit_breadcrumb(struct i915_request rq, u32 *cs, · 142 unmodified lines
143	int flush, int post)	143	int flush, int post)
144	{	144	{
145	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);	145	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
146	GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);	146	GEM_BUG_ON(offset_in_page(rq->hwsp_seqno) != I915_GEM_HWS_SEQNO_ADDR);
147		147
148	*cs++ = MI_FLUSH;	148	*cs++ = MI_FLUSH;
149		149

drivers/gpu/drm/i915/gt/gen6_engine_cs.c+4−4

@@ -161,7 +161,7 @@ u32 gen6_emit_breadcrumb_rcs(struct i915_request rq, u32 *cs) · 160 unmodified lines
161	PIPE_CONTROL_DC_FLUSH_ENABLE \|	161	PIPE_CONTROL_DC_FLUSH_ENABLE \|
162	PIPE_CONTROL_QW_WRITE \|	162	PIPE_CONTROL_QW_WRITE \|
163	PIPE_CONTROL_CS_STALL);	163	PIPE_CONTROL_CS_STALL);
164	*cs++ = i915_request_active_timeline(rq)->hwsp_offset \|	164	*cs++ = i915_request_active_seqno(rq) \|
165	PIPE_CONTROL_GLOBAL_GTT;	165	PIPE_CONTROL_GLOBAL_GTT;
166	*cs++ = rq->fence.seqno;	166	*cs++ = rq->fence.seqno;
167		167
@@ -359,7 +359,7 @@ u32 gen7_emit_breadcrumb_rcs(struct i915_request rq, u32 *cs) · 191 unmodified lines
359	PIPE_CONTROL_QW_WRITE \|	359	PIPE_CONTROL_QW_WRITE \|
360	PIPE_CONTROL_GLOBAL_GTT_IVB \|	360	PIPE_CONTROL_GLOBAL_GTT_IVB \|
361	PIPE_CONTROL_CS_STALL);	361	PIPE_CONTROL_CS_STALL);
362	*cs++ = i915_request_active_timeline(rq)->hwsp_offset;	362	*cs++ = i915_request_active_seqno(rq);
363	*cs++ = rq->fence.seqno;	363	*cs++ = rq->fence.seqno;
364		364
365	*cs++ = MI_USER_INTERRUPT;	365	*cs++ = MI_USER_INTERRUPT;
@@ -374,7 +374,7 @@ u32 gen7_emit_breadcrumb_rcs(struct i915_request rq, u32 *cs) · 8 unmodified lines
374	u32 gen6_emit_breadcrumb_xcs(struct i915_request rq, u32 *cs)	374	u32 gen6_emit_breadcrumb_xcs(struct i915_request rq, u32 *cs)
375	{	375	{
376	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);	376	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
377	GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);	377	GEM_BUG_ON(offset_in_page(rq->hwsp_seqno) != I915_GEM_HWS_SEQNO_ADDR);
378		378
379	*cs++ = MI_FLUSH_DW \| MI_FLUSH_DW_OP_STOREDW \| MI_FLUSH_DW_STORE_INDEX;	379	*cs++ = MI_FLUSH_DW \| MI_FLUSH_DW_OP_STOREDW \| MI_FLUSH_DW_STORE_INDEX;
380	*cs++ = I915_GEM_HWS_SEQNO_ADDR \| MI_FLUSH_DW_USE_GTT;	380	*cs++ = I915_GEM_HWS_SEQNO_ADDR \| MI_FLUSH_DW_USE_GTT;
@@ -394,7 +394,7 @@ u32 gen7_emit_breadcrumb_xcs(struct i915_request rq, u32 *cs) · 13 unmodified lines
394	int i;	394	int i;
395		395
396	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);	396	GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
397	GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);	397	GEM_BUG_ON(offset_in_page(rq->hwsp_seqno) != I915_GEM_HWS_SEQNO_ADDR);
398		398
399	*cs++ = MI_FLUSH_DW \| MI_INVALIDATE_TLB \|	399	*cs++ = MI_FLUSH_DW \| MI_INVALIDATE_TLB \|
400	MI_FLUSH_DW_OP_STOREDW \| MI_FLUSH_DW_STORE_INDEX;	400	MI_FLUSH_DW_OP_STOREDW \| MI_FLUSH_DW_STORE_INDEX;

drivers/gpu/drm/i915/gt/gen8_engine_cs.c+6−7

 static u32 hwsp_offset(const struct i915_request *rq)
 {
-	const struct intel_timeline_cacheline *cl;
+	const struct intel_timeline *tl;
-	/* Before the request is executed, the timeline/cachline is fixed */
+	/* Before the request is executed, the timeline is fixed */
+	tl = rcu_dereference_protected(rq->timeline,
+				       !i915_request_signaled(rq));
-	cl = rcu_dereference_protected(rq->hwsp_cacheline, 1);
+	/* See the comment in i915_request_active_seqno(). */
-	if (cl)
+	return page_mask_bits(tl->hwsp_offset) + offset_in_page(rq->hwsp_seqno);
-		return cl->ggtt_offset;
-	return rcu_dereference_protected(rq->timeline, 1)->hwsp_offset;
 }
 int gen8_emit_init_breadcrumb(struct i915_request *rq)

drivers/gpu/drm/i915/gt/intel_context_param.h+10−1

 #ifndef INTEL_CONTEXT_PARAM_H
 #define INTEL_CONTEXT_PARAM_H
-struct intel_context;
+#include <linux/types.h>
+#include "intel_context.h"
 int intel_context_set_ring_size(struct intel_context *ce, long sz);
 long intel_context_get_ring_size(struct intel_context *ce);
+static inline int
+intel_context_set_watchdog_us(struct intel_context *ce, u64 timeout_us)
+{
+	ce->watchdog.timeout_us = timeout_us;
+	return 0;
+}
 #endif /* INTEL_CONTEXT_PARAM_H */

drivers/gpu/drm/i915/gt/intel_context_types.h+4−0

 #define CONTEXT_FORCE_SINGLE_SUBMISSION	7
 #define CONTEXT_NOPREEMPT		8
+	struct {
+		u64 timeout_us;
+	} watchdog;
 	u32 *lrc_reg_state;
 	union {
 		struct {

drivers/gpu/drm/i915/gt/intel_engine_cs.c+25−13

 }
 static int pin_ggtt_status_page(struct intel_engine_cs *engine,
+				struct i915_gem_ww_ctx *ww,
 				struct i915_vma *vma)
 {
 	unsigned int flags;
 	else
 		flags = PIN_HIGH;
-	return i915_ggtt_pin(vma, NULL, 0, flags);
+	return i915_ggtt_pin(vma, ww, 0, flags);
 }
 static int init_status_page(struct intel_engine_cs *engine)
 {
 	struct drm_i915_gem_object *obj;
+	struct i915_gem_ww_ctx ww;
 	struct i915_vma *vma;
 	void *vaddr;
 	int ret;
 	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
-		goto err;
+		goto err_put;
 	}
+	i915_gem_ww_ctx_init(&ww, true);
+retry:
+	ret = i915_gem_object_lock(obj, &ww);
+	if (!ret && !HWS_NEEDS_PHYSICAL(engine->i915))
+		ret = pin_ggtt_status_page(engine, &ww, vma);
+	if (ret)
+		goto err;
 	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
 	if (IS_ERR(vaddr)) {
 		ret = PTR_ERR(vaddr);
-		goto err;
+		goto err_unpin;
 	}
 	engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE);
 	engine->status_page.vma = vma;
-	if (!HWS_NEEDS_PHYSICAL(engine->i915)) {
-		ret = pin_ggtt_status_page(engine, vma);
-		if (ret)
-			goto err_unpin;
-	}
-	return 0;
 err_unpin:
-	i915_gem_object_unpin_map(obj);
+	if (ret)
+		i915_vma_unpin(vma);
 err:
-	i915_gem_object_put(obj);
+	if (ret == -EDEADLK) {
+		ret = i915_gem_ww_ctx_backoff(&ww);
+		if (!ret)
+			goto retry;
+	}
+	i915_gem_ww_ctx_fini(&ww);
+err_put:
+	if (ret)
+		i915_gem_object_put(obj);
 	return ret;
 }
 	frame->rq.engine = engine;
 	frame->rq.context = ce;
 	rcu_assign_pointer(frame->rq.timeline, ce->timeline);
+	frame->rq.hwsp_seqno = ce->timeline->hwsp_seqno;
 	frame->ring.vaddr = frame->cs;
 	frame->ring.size = sizeof(frame->cs);

drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c+1−0

@@ -279,6 +279,7 @@ int intel_engine_pulse(struct intel_engine_cs *engine) · 278 unmodified lines
279	mutex_unlock(&ce->timeline->mutex);	279	mutex_unlock(&ce->timeline->mutex);
280	}	280	}
281		281
		282	intel_engine_flush_submission(engine);
282	intel_engine_pm_put(engine);	283	intel_engine_pm_put(engine);
283	return err;	284	return err;
284	}	285	}

drivers/gpu/drm/i915/gt/intel_engine_pm.c+4−0

 		int type = i915_coherent_map_type(ce->engine->i915);
 		void *map;
+		if (!i915_gem_object_trylock(obj))
+			return;
 		map = i915_gem_object_pin_map(obj, type);
 		if (!IS_ERR(map)) {
 			memset(map, CONTEXT_REDZONE, obj->base.size);
 			i915_gem_object_flush_map(obj);
 			i915_gem_object_unpin_map(obj);
 		}
+		i915_gem_object_unlock(obj);
 	}
 }

drivers/gpu/drm/i915/gt/intel_execlists_submission.c+39−10

 	ce->lrc.lrca = lrc_update_regs(ce, engine, head);
 }
+static bool bad_request(const struct i915_request *rq)
+{
+	return rq->fence.error && i915_request_started(rq);
+}
 static struct intel_engine_cs *
 __execlists_schedule_in(struct i915_request *rq)
 {
 		     !intel_engine_has_heartbeat(engine)))
 		intel_context_set_banned(ce);
-	if (unlikely(intel_context_is_banned(ce)))
+	if (unlikely(intel_context_is_banned(ce) || bad_request(rq)))
 		reset_active(rq, engine);
 	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
 {
 	struct intel_engine_cs *engine =
 		container_of(execlists, typeof(*engine), execlists);
-	struct i915_request * const *port, *rq;
+	struct i915_request * const *port, *rq, *prev = NULL;
 	struct intel_context *ce = NULL;
-	bool sentinel = false;
 	u32 ccid = -1;
 	trace_ports(execlists, msg, execlists->pending);
 		 * Sentinels are supposed to be the last request so they flush
 		 * the current execution off the HW. Check that they are the only
 		 * request in the pending submission.
+		 *
+		 * NB: Due to the async nature of preempt-to-busy and request
+		 * cancellation we need to handle the case where request
+		 * becomes a sentinel in parallel to CSB processing.
 		 */
-		if (sentinel) {
+		if (prev && i915_request_has_sentinel(prev) &&
+		    !READ_ONCE(prev->fence.error)) {
 			GEM_TRACE_ERR("%s: context:%llx after sentinel in pending[%zd]\n",
 				      engine->name,
 				      ce->timeline->fence_context,
 				      port - execlists->pending);
 			return false;
 		}
-		sentinel = i915_request_has_sentinel(rq);
+		prev = rq;
 		/*
 		 * We want virtual requests to only be in the first slot so
 	if (__i915_request_is_complete(next))
 		return true;
-	if (unlikely((i915_request_flags(prev) ^ i915_request_flags(next)) &
+	if (unlikely((i915_request_flags(prev) | i915_request_flags(next)) &
 		     (BIT(I915_FENCE_FLAG_NOPREEMPT) |
 		      BIT(I915_FENCE_FLAG_SENTINEL))))
 		return false;
 		return 0;
 	/* Force a fast reset for terminated contexts (ignoring sysfs!) */
-	if (unlikely(intel_context_is_banned(rq->context)))
+	if (unlikely(intel_context_is_banned(rq->context) || bad_request(rq)))
 		return 1;
 	return READ_ONCE(engine->props.preempt_timeout_ms);
 	spin_unlock_irqrestore(&engine->active.lock, flags);
 }
+static int
+__execlists_context_pre_pin(struct intel_context *ce,
+			    struct intel_engine_cs *engine,
+			    struct i915_gem_ww_ctx *ww, void **vaddr)
+{
+	int err;
+	err = lrc_pre_pin(ce, engine, ww, vaddr);
+	if (err)
+		return err;
+	if (!__test_and_set_bit(CONTEXT_INIT_BIT, &ce->flags)) {
+		lrc_init_state(ce, engine, *vaddr);
+		 __i915_gem_object_flush_map(ce->state->obj, 0, engine->context_size);
+	}
+	return 0;
+}
 static int execlists_context_pre_pin(struct intel_context *ce,
 				     struct i915_gem_ww_ctx *ww,
 				     void **vaddr)
 {
-	return lrc_pre_pin(ce, ce->engine, ww, vaddr);
+	return __execlists_context_pre_pin(ce, ce->engine, ww, vaddr);
 }
 static int execlists_context_pin(struct intel_context *ce, void *vaddr)
 {
 	struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
-	/* Note: we must use a real engine class for setting up reg state */
+	 /* Note: we must use a real engine class for setting up reg state */
-	return lrc_pre_pin(ce, ve->siblings[0], ww, vaddr);
+	return __execlists_context_pre_pin(ce, ve->siblings[0], ww, vaddr);
 }
 static int virtual_context_pin(struct intel_context *ce, void *vaddr)

drivers/gpu/drm/i915/gt/intel_execlists_submission.h+2−0

 #ifndef __INTEL_EXECLISTS_SUBMISSION_H__
 #define __INTEL_EXECLISTS_SUBMISSION_H__
+#include <linux/llist.h>
 #include <linux/types.h>
 struct drm_printer;
 struct i915_request;
 struct intel_context;
 struct intel_engine_cs;
+struct intel_gt;
 enum {
 	INTEL_CONTEXT_SCHEDULE_IN = 0,

drivers/gpu/drm/i915/gt/intel_ggtt.c+8−2

 	if (err)
 		goto err_ppgtt;
+	i915_gem_object_lock(ppgtt->vm.scratch[0], NULL);
 	err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash);
+	i915_gem_object_unlock(ppgtt->vm.scratch[0]);
 	if (err)
 		goto err_stash;
 	mutex_unlock(&ggtt->vm.mutex);
 	i915_address_space_fini(&ggtt->vm);
+	dma_resv_fini(&ggtt->vm.resv);
 	arch_phys_wc_del(ggtt->mtrr);
 	ggtt->vm.gt = gt;
 	ggtt->vm.i915 = i915;
 	ggtt->vm.dma = i915->drm.dev;
+	dma_resv_init(&ggtt->vm.resv);
 	if (INTEL_GEN(i915) <= 5)
 		ret = i915_gmch_probe(ggtt);
 		ret = gen6_gmch_probe(ggtt);
 	else
 		ret = gen8_gmch_probe(ggtt);
-	if (ret)
+	if (ret) {
+		dma_resv_fini(&ggtt->vm.resv);
 		return ret;
+	}
 	if ((ggtt->vm.total - 1) >> 32) {
 		drm_err(&i915->drm,
 	if (ret)
 		goto err_sg_alloc;
-	iter = i915_gem_object_get_sg_dma(obj, view->partial.offset, &offset);
+	iter = i915_gem_object_get_sg_dma(obj, view->partial.offset, &offset, true);
 	GEM_BUG_ON(!iter);
 	sg = st->sgl;

drivers/gpu/drm/i915/gt/intel_gt.c+3−0

@@ -29,6 +29,9 @@ void intel_gt_init_early(struct intel_gt gt, struct drm_i915_private i915) · 28 unmodified lines
29	INIT_LIST_HEAD(&gt->closed_vma);	29	INIT_LIST_HEAD(&gt->closed_vma);
30	spin_lock_init(&gt->closed_lock);	30	spin_lock_init(&gt->closed_lock);
31		31
		32	init_llist_head(&gt->watchdog.list);
		33	INIT_WORK(&gt->watchdog.work, intel_gt_watchdog_work);
		34
32	intel_gt_init_buffer_pool(gt);	35	intel_gt_init_buffer_pool(gt);
33	intel_gt_init_reset(gt);	36	intel_gt_init_reset(gt);
34	intel_gt_init_requests(gt);	37	intel_gt_init_requests(gt);

drivers/gpu/drm/i915/gt/intel_gt.h+2−0

@@ -77,4 +77,6 @@ static inline bool intel_gt_is_wedged(const struct intel_gt *gt) · 76 unmodified lines
77	void intel_gt_info_print(const struct intel_gt_info *info,	77	void intel_gt_info_print(const struct intel_gt_info *info,
78	struct drm_printer *p);	78	struct drm_printer *p);
79		79
		80	void intel_gt_watchdog_work(struct work_struct *work);
		81
80	#endif /* __INTEL_GT_H__ */	82	#endif /* __INTEL_GT_H__ */

drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c+21−26

 				      round_jiffies_up_relative(HZ));
 }
-static int pool_active(struct i915_active *ref)
-{
-	struct intel_gt_buffer_pool_node *node =
-		container_of(ref, typeof(*node), active);
-	struct dma_resv *resv = node->obj->base.resv;
-	int err;
-	if (dma_resv_trylock(resv)) {
-		dma_resv_add_excl_fence(resv, NULL);
-		dma_resv_unlock(resv);
-	}
-	err = i915_gem_object_pin_pages(node->obj);
-	if (err)
-		return err;
-	/* Hide this pinned object from the shrinker until retired */
-	i915_gem_object_make_unshrinkable(node->obj);
-	return 0;
-}
 __i915_active_call
 static void pool_retire(struct i915_active *ref)
 {
 	struct list_head *list = bucket_for_size(pool, node->obj->base.size);
 	unsigned long flags;
-	i915_gem_object_unpin_pages(node->obj);
+	if (node->pinned) {
+		i915_gem_object_unpin_pages(node->obj);
-	/* Return this object to the shrinker pool */
+		/* Return this object to the shrinker pool */
-	i915_gem_object_make_purgeable(node->obj);
+		i915_gem_object_make_purgeable(node->obj);
+		node->pinned = false;
+	}
 	GEM_BUG_ON(node->age);
 	spin_lock_irqsave(&pool->lock, flags);
 			      round_jiffies_up_relative(HZ));
 }
+void intel_gt_buffer_pool_mark_used(struct intel_gt_buffer_pool_node *node)
+{
+	assert_object_held(node->obj);
+	if (node->pinned)
+		return;
+	__i915_gem_object_pin_pages(node->obj);
+	/* Hide this pinned object from the shrinker until retired */
+	i915_gem_object_make_unshrinkable(node->obj);
+	node->pinned = true;
+}
 static struct intel_gt_buffer_pool_node *
 node_create(struct intel_gt_buffer_pool *pool, size_t sz,
 	    enum i915_map_type type)
 	node->age = 0;
 	node->pool = pool;
-	i915_active_init(&node->active, pool_active, pool_retire);
+	node->pinned = false;
+	i915_active_init(&node->active, NULL, pool_retire);
 	obj = i915_gem_object_create_internal(gt->i915, sz);
 	if (IS_ERR(obj)) {

drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.h+5−0

 intel_gt_get_buffer_pool(struct intel_gt *gt, size_t size,
 			 enum i915_map_type type);
+void intel_gt_buffer_pool_mark_used(struct intel_gt_buffer_pool_node *node);
 static inline int
 intel_gt_buffer_pool_mark_active(struct intel_gt_buffer_pool_node *node,
 				 struct i915_request *rq)
 {
+	/* did we call mark_used? */
+	GEM_WARN_ON(!node->pinned);
 	return i915_active_add_request(&node->active, rq);
 }

drivers/gpu/drm/i915/gt/intel_gt_buffer_pool_types.h+1−0

@@ -31,6 +31,7 @@ struct intel_gt_buffer_pool_node { · 30 unmodified lines
31	};	31	};
32	unsigned long age;	32	unsigned long age;
33	enum i915_map_type type;	33	enum i915_map_type type;
		34	u32 pinned;
34	};	35	};
35		36
36	#endif /* INTEL_GT_BUFFER_POOL_TYPES_H */	37	#endif /* INTEL_GT_BUFFER_POOL_TYPES_H */

drivers/gpu/drm/i915/gt/intel_gt_requests.c+28−0

@@ -9,6 +9,7 @@ · 8 unmodified lines
9	#include "i915_drv.h" /* for_each_engine() */	9	#include "i915_drv.h" /* for_each_engine() */
10	#include "i915_request.h"	10	#include "i915_request.h"
11	#include "intel_engine_heartbeat.h"	11	#include "intel_engine_heartbeat.h"
		12	#include "intel_execlists_submission.h"
12	#include "intel_gt.h"	13	#include "intel_gt.h"
13	#include "intel_gt_pm.h"	14	#include "intel_gt_pm.h"
14	#include "intel_gt_requests.h"	15	#include "intel_gt_requests.h"
@@ -243,4 +244,31 @@ void intel_gt_fini_requests(struct intel_gt *gt) · 228 unmodified lines
243	{	244	{
244	/* Wait until the work is marked as finished before unloading! */	245	/* Wait until the work is marked as finished before unloading! */
245	cancel_delayed_work_sync(&gt->requests.retire_work);	246	cancel_delayed_work_sync(&gt->requests.retire_work);
		247
		248	flush_work(&gt->watchdog.work);
		249	}
		250
		251	void intel_gt_watchdog_work(struct work_struct *work)
		252	{
		253	struct intel_gt *gt =
		254	container_of(work, typeof(*gt), watchdog.work);
		255	struct i915_request rq, rn;
		256	struct llist_node *first;
		257
		258	first = llist_del_all(&gt->watchdog.list);
		259	if (!first)
		260	return;
		261
		262	llist_for_each_entry_safe(rq, rn, first, watchdog.link) {
		263	if (!i915_request_completed(rq)) {
		264	struct dma_fence *f = &rq->fence;
		265
		266	pr_notice("Fence expiration time out i915-%s:%s:%llx!\n",
		267	f->ops->get_driver_name(f),
		268	f->ops->get_timeline_name(f),
		269	f->seqno);
		270	i915_request_cancel(rq, -EINTR);
		271	}
		272	i915_request_put(rq);
		273	}
246	}	274	}

drivers/gpu/drm/i915/gt/intel_gt_types.h+7−4

 #include <linux/ktime.h>
 #include <linux/list.h>
+#include <linux/llist.h>
 #include <linux/mutex.h>
 #include <linux/notifier.h>
 #include <linux/spinlock.h>
 #include <linux/types.h>
+#include <linux/workqueue.h>
 #include "uc/intel_uc.h"
 	struct intel_gt_timelines {
 		spinlock_t lock; /* protects active_list */
 		struct list_head active_list;
-		/* Pack multiple timelines' seqnos into the same page */
-		spinlock_t hwsp_lock;
-		struct list_head hwsp_free_list;
 	} timelines;
 	struct intel_gt_requests {
 		struct delayed_work retire_work;
 	} requests;
+	struct {
+		struct llist_head list;
+		struct work_struct work;
+	} watchdog;
 	struct intel_wakeref wakeref;
 	atomic_t user_wakeref;

drivers/gpu/drm/i915/gt/intel_gtt.c+50−2

@@ -13,16 +13,36 @@ · 12 unmodified lines
13		13
14	struct drm_i915_gem_object alloc_pt_dma(struct i915_address_space vm, int sz)	14	struct drm_i915_gem_object alloc_pt_dma(struct i915_address_space vm, int sz)
15	{	15	{
		16	struct drm_i915_gem_object *obj;
		17
16	if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))	18	if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
17	i915_gem_shrink_all(vm->i915);	19	i915_gem_shrink_all(vm->i915);
18		20
19	return i915_gem_object_create_internal(vm->i915, sz);	21	obj = i915_gem_object_create_internal(vm->i915, sz);
		22	/* ensure all dma objects have the same reservation class */
		23	if (!IS_ERR(obj))
		24	obj->base.resv = &vm->resv;
		25	return obj;
20	}	26	}
21		27
22	int pin_pt_dma(struct i915_address_space vm, struct drm_i915_gem_object obj)	28	int pin_pt_dma(struct i915_address_space vm, struct drm_i915_gem_object obj)
23	{	29	{
24	int err;	30	int err;
25		31
		32	i915_gem_object_lock(obj, NULL);
		33	err = i915_gem_object_pin_pages(obj);
		34	i915_gem_object_unlock(obj);
		35	if (err)
		36	return err;
		37
		38	i915_gem_object_make_unshrinkable(obj);
		39	return 0;
		40	}
		41
		42	int pin_pt_dma_locked(struct i915_address_space vm, struct drm_i915_gem_object obj)
		43	{
		44	int err;
		45
26	err = i915_gem_object_pin_pages(obj);	46	err = i915_gem_object_pin_pages(obj);
27	if (err)	47	if (err)
28	return err;	48	return err;
@@ -56,6 +76,20 @@ void __i915_vm_close(struct i915_address_space *vm) · 27 unmodified lines
56	mutex_unlock(&vm->mutex);	76	mutex_unlock(&vm->mutex);
57	}	77	}
58		78
		79	/* lock the vm into the current ww, if we lock one, we lock all */
		80	int i915_vm_lock_objects(struct i915_address_space *vm,
		81	struct i915_gem_ww_ctx *ww)
		82	{
		83	if (vm->scratch[0]->base.resv == &vm->resv) {
		84	return i915_gem_object_lock(vm->scratch[0], ww);
		85	} else {
		86	struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
		87
		88	/* We borrowed the scratch page from ggtt, take the top level object */
		89	return i915_gem_object_lock(ppgtt->pd->pt.base, ww);
		90	}
		91	}
		92
59	void i915_address_space_fini(struct i915_address_space *vm)	93	void i915_address_space_fini(struct i915_address_space *vm)
60	{	94	{
61	drm_mm_takedown(&vm->mm);	95	drm_mm_takedown(&vm->mm);
@@ -69,6 +103,7 @@ static void __i915_vm_release(struct work_struct *work) · 7 unmodified lines
69		103
70	vm->cleanup(vm);	104	vm->cleanup(vm);
71	i915_address_space_fini(vm);	105	i915_address_space_fini(vm);
		106	dma_resv_fini(&vm->resv);
72		107
73	kfree(vm);	108	kfree(vm);
74	}	109	}
@@ -98,6 +133,7 @@ void i915_address_space_init(struct i915_address_space *vm, int subclass) · 23 unmodified lines
98	mutex_init(&vm->mutex);	133	mutex_init(&vm->mutex);
99	lockdep_set_subclass(&vm->mutex, subclass);	134	lockdep_set_subclass(&vm->mutex, subclass);
100	i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex);	135	i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex);
		136	dma_resv_init(&vm->resv);
101		137
102	GEM_BUG_ON(!vm->total);	138	GEM_BUG_ON(!vm->total);
103	drm_mm_init(&vm->mm, 0, vm->total);	139	drm_mm_init(&vm->mm, 0, vm->total);
@@ -427,7 +463,6 @@ __vm_create_scratch_for_read(struct i915_address_space *vm, unsigned long size) · 323 unmodified lines
427	{	463	{
428	struct drm_i915_gem_object *obj;	464	struct drm_i915_gem_object *obj;
429	struct i915_vma *vma;	465	struct i915_vma *vma;
430	int err;
431		466
432	obj = i915_gem_object_create_internal(vm->i915, PAGE_ALIGN(size));	467	obj = i915_gem_object_create_internal(vm->i915, PAGE_ALIGN(size));
433	if (IS_ERR(obj))	468	if (IS_ERR(obj))
@@ -441,6 +476,19 @@ __vm_create_scratch_for_read(struct i915_address_space *vm, unsigned long size) · 7 unmodified lines
441	return vma;	476	return vma;
442	}	477	}
443		478
		479	return vma;
		480	}
		481
		482	struct i915_vma *
		483	__vm_create_scratch_for_read_pinned(struct i915_address_space *vm, unsigned long size)
		484	{
		485	struct i915_vma *vma;
		486	int err;
		487
		488	vma = __vm_create_scratch_for_read(vm, size);
		489	if (IS_ERR(vma))
		490	return vma;
		491
444	err = i915_vma_pin(vma, 0, 0,	492	err = i915_vma_pin(vma, 0, 0,
445	i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);	493	i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
446	if (err) {	494	if (err) {

drivers/gpu/drm/i915/gt/intel_gtt.h+8−0

 	atomic_t open;
 	struct mutex mutex; /* protects vma and our lists */
+	struct dma_resv resv; /* reservation lock for all pd objects, and buffer pool */
 #define VM_CLASS_GGTT 0
 #define VM_CLASS_PPGTT 1
 #define i915_is_ggtt(vm) ((vm)->is_ggtt)
+int __must_check
+i915_vm_lock_objects(struct i915_address_space *vm, struct i915_gem_ww_ctx *ww);
 static inline bool
 i915_vm_is_4lvl(const struct i915_address_space *vm)
 {
 struct i915_page_directory *__alloc_pd(int npde);
 int pin_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj);
+int pin_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj);
 void free_px(struct i915_address_space *vm,
 	     struct i915_page_table *pt, int lvl);
 struct i915_vma *
 __vm_create_scratch_for_read(struct i915_address_space *vm, unsigned long size);
+struct i915_vma *
+__vm_create_scratch_for_read_pinned(struct i915_address_space *vm, unsigned long size);
 static inline struct sgt_dma {
 	struct scatterlist *sg;
 	dma_addr_t dma, max;

drivers/gpu/drm/i915/gt/intel_lrc.c+38−11

 #define CTX_WA_BB_SIZE (PAGE_SIZE)
-static int lrc_setup_wa_ctx(struct intel_engine_cs *engine)
+static int lrc_create_wa_ctx(struct intel_engine_cs *engine)
 {
 	struct drm_i915_gem_object *obj;
 	struct i915_vma *vma;
 		goto err;
 	}
-	err = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH);
-	if (err)
-		goto err;
 	engine->wa_ctx.vma = vma;
 	return 0;
 void lrc_fini_wa_ctx(struct intel_engine_cs *engine)
 {
 	i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0);
-	/* Called on error unwind, clear all flags to prevent further use */
-	memset(&engine->wa_ctx, 0, sizeof(engine->wa_ctx));
 }
 typedef u32 *(*wa_bb_func_t)(struct intel_engine_cs *engine, u32 *batch);
 		&wa_ctx->indirect_ctx, &wa_ctx->per_ctx
 	};
 	wa_bb_func_t wa_bb_fn[ARRAY_SIZE(wa_bb)];
+	struct i915_gem_ww_ctx ww;
 	void *batch, *batch_ptr;
 	unsigned int i;
 	int err;
 		return;
 	}
-	err = lrc_setup_wa_ctx(engine);
+	err = lrc_create_wa_ctx(engine);
 	if (err) {
 		/*
 		 * We continue even if we fail to initialize WA batch
 		return;
 	}
+	if (!engine->wa_ctx.vma)
+		return;
+	i915_gem_ww_ctx_init(&ww, true);
+retry:
+	err = i915_gem_object_lock(wa_ctx->vma->obj, &ww);
+	if (!err)
+		err = i915_ggtt_pin(wa_ctx->vma, &ww, 0, PIN_HIGH);
+	if (err)
+		goto err;
 	batch = i915_gem_object_pin_map(wa_ctx->vma->obj, I915_MAP_WB);
+	if (IS_ERR(batch)) {
+		err = PTR_ERR(batch);
+		goto err_unpin;
+	}
 	/*
 	 * Emit the two workaround batch buffers, recording the offset from the
 	__i915_gem_object_release_map(wa_ctx->vma->obj);
 	/* Verify that we can handle failure to setup the wa_ctx */
-	if (err || i915_inject_probe_error(engine->i915, -ENODEV))
+	if (!err)
-		lrc_fini_wa_ctx(engine);
+		err = i915_inject_probe_error(engine->i915, -ENODEV);
+err_unpin:
+	if (err)
+		i915_vma_unpin(wa_ctx->vma);
+err:
+	if (err == -EDEADLK) {
+		err = i915_gem_ww_ctx_backoff(&ww);
+		if (!err)
+			goto retry;
+	}
+	i915_gem_ww_ctx_fini(&ww);
+	if (err) {
+		i915_vma_put(engine->wa_ctx.vma);
+		/* Clear all flags to prevent further use */
+		memset(wa_ctx, 0, sizeof(*wa_ctx));
+	}
 }
 static void st_update_runtime_underflow(struct intel_context *ce, s32 dt)

drivers/gpu/drm/i915/gt/intel_ppgtt.c+2−1

 	for (n = 0; n < ARRAY_SIZE(stash->pt); n++) {
 		for (pt = stash->pt[n]; pt; pt = pt->stash) {
-			err = pin_pt_dma(vm, pt->base);
+			err = pin_pt_dma_locked(vm, pt->base);
 			if (err)
 				return err;
 		}
 	ppgtt->vm.dma = i915->drm.dev;
 	ppgtt->vm.total = BIT_ULL(INTEL_INFO(i915)->ppgtt_size);
+	dma_resv_init(&ppgtt->vm.resv);
 	i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT);
 	ppgtt->vm.vma_ops.bind_vma    = ppgtt_bind_vma;

drivers/gpu/drm/i915/gt/intel_renderstate.c+1−1

@@ -197,7 +197,7 @@ retry: · 196 unmodified lines
197	if (err)	197	if (err)
198	goto err_context;	198	goto err_context;
199		199
200	err = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL \| PIN_HIGH);	200	err = i915_vma_pin_ww(so->vma, &so->ww, 0, 0, PIN_GLOBAL \| PIN_HIGH);
201	if (err)	201	if (err)
202	goto err_context;	202	goto err_context;
203		203

drivers/gpu/drm/i915/gt/intel_reset.c+7−2

 {
 	int err, i;
-	gt_revoke(gt);
 	err = __intel_gt_reset(gt, ALL_ENGINES);
 	for (i = 0; err && i < RESET_MAX_RETRIES; i++) {
 		msleep(10 * (i + 1));
 	might_sleep();
 	GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, &gt->reset.flags));
+	/*
+	 * FIXME: Revoking cpu mmap ptes cannot be done from a dma_fence
+	 * critical section like gpu reset.
+	 */
+	gt_revoke(gt);
 	mutex_lock(&gt->reset.mutex);
 	/* Clear any previous failed attempts at recovery. Time to try again. */

drivers/gpu/drm/i915/gt/intel_ring_submission.c+118−66

@@ -466,6 +466,26 @@ static void ring_context_destroy(struct kref *ref) · 465 unmodified lines
466	intel_context_free(ce);	466	intel_context_free(ce);
467	}	467	}
468		468
		469	static int ring_context_init_default_state(struct intel_context *ce,
		470	struct i915_gem_ww_ctx *ww)
		471	{
		472	struct drm_i915_gem_object *obj = ce->state->obj;
		473	void *vaddr;
		474
		475	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
		476	if (IS_ERR(vaddr))
		477	return PTR_ERR(vaddr);
		478
		479	shmem_read(ce->engine->default_state, 0,
		480	vaddr, ce->engine->context_size);
		481
		482	i915_gem_object_flush_map(obj);
		483	__i915_gem_object_release_map(obj);
		484
		485	__set_bit(CONTEXT_VALID_BIT, &ce->flags);
		486	return 0;
		487	}
		488
469	static int ring_context_pre_pin(struct intel_context *ce,	489	static int ring_context_pre_pin(struct intel_context *ce,
470	struct i915_gem_ww_ctx *ww,	490	struct i915_gem_ww_ctx *ww,
471	void **unused)	491	void **unused)
@@ -473,6 +493,13 @@ static int ring_context_pre_pin(struct intel_context *ce, · 1 unmodified line
473	struct i915_address_space *vm;	493	struct i915_address_space *vm;
474	int err = 0;	494	int err = 0;
475		495
		496	if (ce->engine->default_state &&
		497	!test_bit(CONTEXT_VALID_BIT, &ce->flags)) {
		498	err = ring_context_init_default_state(ce, ww);
		499	if (err)
		500	return err;
		501	}
		502
476	vm = vm_alias(ce->vm);	503	vm = vm_alias(ce->vm);
477	if (vm)	504	if (vm)
478	err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)), ww);	505	err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)), ww);
@@ -528,22 +555,6 @@ alloc_context_vma(struct intel_engine_cs *engine) · 49 unmodified lines
528	if (IS_IVYBRIDGE(i915))	555	if (IS_IVYBRIDGE(i915))
529	i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC);	556	i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC);
530		557
531	if (engine->default_state) {
532	void *vaddr;
533
534	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
535	if (IS_ERR(vaddr)) {
536	err = PTR_ERR(vaddr);
537	goto err_obj;
538	}
539
540	shmem_read(engine->default_state, 0,
541	vaddr, engine->context_size);
542
543	i915_gem_object_flush_map(obj);
544	__i915_gem_object_release_map(obj);
545	}
546
547	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);	558	vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
548	if (IS_ERR(vma)) {	559	if (IS_ERR(vma)) {
549	err = PTR_ERR(vma);	560	err = PTR_ERR(vma);
@@ -575,8 +586,6 @@ static int ring_context_alloc(struct intel_context *ce) · 25 unmodified lines
575	return PTR_ERR(vma);	586	return PTR_ERR(vma);
576		587
577	ce->state = vma;	588	ce->state = vma;
578	if (engine->default_state)
579	__set_bit(CONTEXT_VALID_BIT, &ce->flags);
580	}	589	}
581		590
582	return 0;	591	return 0;
@@ -1176,37 +1185,15 @@ static int gen7_ctx_switch_bb_setup(struct intel_engine_cs * const engine, · 593 unmodified lines
1176	return gen7_setup_clear_gpr_bb(engine, vma);	1185	return gen7_setup_clear_gpr_bb(engine, vma);
1177	}	1186	}
1178		1187
1179	static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine)	1188	static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine,
		1189	struct i915_gem_ww_ctx *ww,
		1190	struct i915_vma *vma)
1180	{	1191	{
1181	struct drm_i915_gem_object *obj;
1182	struct i915_vma *vma;
1183	int size;
1184	int err;	1192	int err;
1185		1193
1186	size = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */);	1194	err = i915_vma_pin_ww(vma, ww, 0, 0, PIN_USER \| PIN_HIGH);
1187	if (size <= 0)
1188	return size;
1189
1190	size = ALIGN(size, PAGE_SIZE);
1191	obj = i915_gem_object_create_internal(engine->i915, size);
1192	if (IS_ERR(obj))
1193	return PTR_ERR(obj);
1194
1195	vma = i915_vma_instance(obj, engine->gt->vm, NULL);
1196	if (IS_ERR(vma)) {
1197	err = PTR_ERR(vma);
1198	goto err_obj;
1199	}
1200
1201	vma->private = intel_context_create(engine); /* dummy residuals */
1202	if (IS_ERR(vma->private)) {
1203	err = PTR_ERR(vma->private);
1204	goto err_obj;
1205	}
1206
1207	err = i915_vma_pin(vma, 0, 0, PIN_USER \| PIN_HIGH);
1208	if (err)	1195	if (err)
1209	goto err_private;	1196	return err;
1210		1197
1211	err = i915_vma_sync(vma);	1198	err = i915_vma_sync(vma);
1212	if (err)	1199	if (err)
@@ -1221,17 +1208,53 @@ static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine) · 8 unmodified lines
1221		1208
1222	err_unpin:	1209	err_unpin:
1223	i915_vma_unpin(vma);	1210	i915_vma_unpin(vma);
1224	err_private:
1225	intel_context_put(vma->private);
1226	err_obj:
1227	i915_gem_object_put(obj);
1228	return err;	1211	return err;
1229	}	1212	}
1230		1213
		1214	static struct i915_vma gen7_ctx_vma(struct intel_engine_cs engine)
		1215	{
		1216	struct drm_i915_gem_object *obj;
		1217	struct i915_vma *vma;
		1218	int size, err;
		1219
		1220	if (!IS_GEN(engine->i915, 7) \|\| engine->class != RENDER_CLASS)
		1221	return 0;
		1222
		1223	err = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */);
		1224	if (err < 0)
		1225	return ERR_PTR(err);
		1226	if (!err)
		1227	return NULL;
		1228
		1229	size = ALIGN(err, PAGE_SIZE);
		1230
		1231	obj = i915_gem_object_create_internal(engine->i915, size);
		1232	if (IS_ERR(obj))
		1233	return ERR_CAST(obj);
		1234
		1235	vma = i915_vma_instance(obj, engine->gt->vm, NULL);
		1236	if (IS_ERR(vma)) {
		1237	i915_gem_object_put(obj);
		1238	return ERR_CAST(vma);
		1239	}
		1240
		1241	vma->private = intel_context_create(engine); /* dummy residuals */
		1242	if (IS_ERR(vma->private)) {
		1243	err = PTR_ERR(vma->private);
		1244	vma->private = NULL;
		1245	i915_gem_object_put(obj);
		1246	return ERR_PTR(err);
		1247	}
		1248
		1249	return vma;
		1250	}
		1251
1231	int intel_ring_submission_setup(struct intel_engine_cs *engine)	1252	int intel_ring_submission_setup(struct intel_engine_cs *engine)
1232	{	1253	{
		1254	struct i915_gem_ww_ctx ww;
1233	struct intel_timeline *timeline;	1255	struct intel_timeline *timeline;
1234	struct intel_ring *ring;	1256	struct intel_ring *ring;
		1257	struct i915_vma *gen7_wa_vma;
1235	int err;	1258	int err;
1236		1259
1237	setup_common(engine);	1260	setup_common(engine);
@@ -1262,43 +1285,72 @@ int intel_ring_submission_setup(struct intel_engine_cs *engine) · 24 unmodified lines
1262	}	1285	}
1263	GEM_BUG_ON(timeline->has_initial_breadcrumb);	1286	GEM_BUG_ON(timeline->has_initial_breadcrumb);
1264		1287
1265	err = intel_timeline_pin(timeline, NULL);
1266	if (err)
1267	goto err_timeline;
1268
1269	ring = intel_engine_create_ring(engine, SZ_16K);	1288	ring = intel_engine_create_ring(engine, SZ_16K);
1270	if (IS_ERR(ring)) {	1289	if (IS_ERR(ring)) {
1271	err = PTR_ERR(ring);	1290	err = PTR_ERR(ring);
1272	goto err_timeline_unpin;	1291	goto err_timeline;
1273	}	1292	}
1274		1293
1275	err = intel_ring_pin(ring, NULL);
1276	if (err)
1277	goto err_ring;
1278
1279	GEM_BUG_ON(engine->legacy.ring);	1294	GEM_BUG_ON(engine->legacy.ring);
1280	engine->legacy.ring = ring;	1295	engine->legacy.ring = ring;
1281	engine->legacy.timeline = timeline;	1296	engine->legacy.timeline = timeline;
1282		1297
1283	GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma);	1298	gen7_wa_vma = gen7_ctx_vma(engine);
		1299	if (IS_ERR(gen7_wa_vma)) {
		1300	err = PTR_ERR(gen7_wa_vma);
		1301	goto err_ring;
		1302	}
1284		1303
1285	if (IS_GEN(engine->i915, 7) && engine->class == RENDER_CLASS) {	1304	i915_gem_ww_ctx_init(&ww, false);
1286	err = gen7_ctx_switch_bb_init(engine);	1305
		1306	retry:
		1307	err = i915_gem_object_lock(timeline->hwsp_ggtt->obj, &ww);
		1308	if (!err && gen7_wa_vma)
		1309	err = i915_gem_object_lock(gen7_wa_vma->obj, &ww);
		1310	if (!err && engine->legacy.ring->vma->obj)
		1311	err = i915_gem_object_lock(engine->legacy.ring->vma->obj, &ww);
		1312	if (!err)
		1313	err = intel_timeline_pin(timeline, &ww);
		1314	if (!err) {
		1315	err = intel_ring_pin(ring, &ww);
1287	if (err)	1316	if (err)
1288	goto err_ring_unpin;	1317	intel_timeline_unpin(timeline);
		1318	}
		1319	if (err)
		1320	goto out;
		1321
		1322	GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma);
		1323
		1324	if (gen7_wa_vma) {
		1325	err = gen7_ctx_switch_bb_init(engine, &ww, gen7_wa_vma);
		1326	if (err) {
		1327	intel_ring_unpin(ring);
		1328	intel_timeline_unpin(timeline);
		1329	}
1289	}	1330	}
1290		1331
		1332	out:
		1333	if (err == -EDEADLK) {
		1334	err = i915_gem_ww_ctx_backoff(&ww);
		1335	if (!err)
		1336	goto retry;
		1337	}
		1338	i915_gem_ww_ctx_fini(&ww);
		1339	if (err)
		1340	goto err_gen7_put;
		1341
1291	/* Finally, take ownership and responsibility for cleanup! */	1342	/* Finally, take ownership and responsibility for cleanup! */
1292	engine->release = ring_release;	1343	engine->release = ring_release;
1293		1344
1294	return 0;	1345	return 0;
1295		1346
1296	err_ring_unpin:	1347	err_gen7_put:
1297	intel_ring_unpin(ring);	1348	if (gen7_wa_vma) {
		1349	intel_context_put(gen7_wa_vma->private);
		1350	i915_gem_object_put(gen7_wa_vma->obj);
		1351	}
1298	err_ring:	1352	err_ring:
1299	intel_ring_put(ring);	1353	intel_ring_put(ring);
1300	err_timeline_unpin:
1301	intel_timeline_unpin(timeline);
1302	err_timeline:	1354	err_timeline:
1303	intel_timeline_put(timeline);	1355	intel_timeline_put(timeline);
1304	err:	1356	err:

drivers/gpu/drm/i915/gt/intel_timeline.c+94−332

@@ -12,21 +12,9 @@ · 11 unmodified lines
12	#include "intel_ring.h"	12	#include "intel_ring.h"
13	#include "intel_timeline.h"	13	#include "intel_timeline.h"
14		14
15	#define ptr_set_bit(ptr, bit) ((typeof(ptr))((unsigned long)(ptr) \| BIT(bit)))	15	#define TIMELINE_SEQNO_BYTES 8
16	#define ptr_test_bit(ptr, bit) ((unsigned long)(ptr) & BIT(bit))
17		16
18	#define CACHELINE_BITS 6	17	static struct i915_vma hwsp_alloc(struct intel_gt gt)
19	#define CACHELINE_FREE CACHELINE_BITS
20
21	struct intel_timeline_hwsp {
22	struct intel_gt *gt;
23	struct intel_gt_timelines *gt_timelines;
24	struct list_head free_link;
25	struct i915_vma *vma;
26	u64 free_bitmap;
27	};
28
29	static struct i915_vma __hwsp_alloc(struct intel_gt gt)
30	{	18	{
31	struct drm_i915_private *i915 = gt->i915;	19	struct drm_i915_private *i915 = gt->i915;
32	struct drm_i915_gem_object *obj;	20	struct drm_i915_gem_object *obj;
@@ -45,174 +33,42 @@ static struct i915_vma __hwsp_alloc(struct intel_gt gt) · 12 unmodified lines
45	return vma;	33	return vma;
46	}	34	}
47		35
48	static struct i915_vma *
49	hwsp_alloc(struct intel_timeline timeline, unsigned int cacheline)
50	{
51	struct intel_gt_timelines *gt = &timeline->gt->timelines;
52	struct intel_timeline_hwsp *hwsp;
53
54	BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);
55
56	spin_lock_irq(&gt->hwsp_lock);
57
58	/* hwsp_free_list only contains HWSP that have available cachelines */
59	hwsp = list_first_entry_or_null(&gt->hwsp_free_list,
60	typeof(*hwsp), free_link);
61	if (!hwsp) {
62	struct i915_vma *vma;
63
64	spin_unlock_irq(&gt->hwsp_lock);
65
66	hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
67	if (!hwsp)
68	return ERR_PTR(-ENOMEM);
69
70	vma = __hwsp_alloc(timeline->gt);
71	if (IS_ERR(vma)) {
72	kfree(hwsp);
73	return vma;
74	}
75
76	GT_TRACE(timeline->gt, "new HWSP allocated\n");
77
78	vma->private = hwsp;
79	hwsp->gt = timeline->gt;
80	hwsp->vma = vma;
81	hwsp->free_bitmap = ~0ull;
82	hwsp->gt_timelines = gt;
83
84	spin_lock_irq(&gt->hwsp_lock);
85	list_add(&hwsp->free_link, &gt->hwsp_free_list);
86	}
87
88	GEM_BUG_ON(!hwsp->free_bitmap);
89	*cacheline = __ffs64(hwsp->free_bitmap);
90	hwsp->free_bitmap &= ~BIT_ULL(*cacheline);
91	if (!hwsp->free_bitmap)
92	list_del(&hwsp->free_link);
93
94	spin_unlock_irq(&gt->hwsp_lock);
95
96	GEM_BUG_ON(hwsp->vma->private != hwsp);
97	return hwsp->vma;
98	}
99
100	static void __idle_hwsp_free(struct intel_timeline_hwsp *hwsp, int cacheline)
101	{
102	struct intel_gt_timelines *gt = hwsp->gt_timelines;
103	unsigned long flags;
104
105	spin_lock_irqsave(&gt->hwsp_lock, flags);
106
107	/* As a cacheline becomes available, publish the HWSP on the freelist */
108	if (!hwsp->free_bitmap)
109	list_add_tail(&hwsp->free_link, &gt->hwsp_free_list);
110
111	GEM_BUG_ON(cacheline >= BITS_PER_TYPE(hwsp->free_bitmap));
112	hwsp->free_bitmap \|= BIT_ULL(cacheline);
113
114	/* And if no one is left using it, give the page back to the system */
115	if (hwsp->free_bitmap == ~0ull) {
116	i915_vma_put(hwsp->vma);
117	list_del(&hwsp->free_link);
118	kfree(hwsp);
119	}
120
121	spin_unlock_irqrestore(&gt->hwsp_lock, flags);
122	}
123
124	static void __rcu_cacheline_free(struct rcu_head *rcu)
125	{
126	struct intel_timeline_cacheline *cl =
127	container_of(rcu, typeof(*cl), rcu);
128
129	/* Must wait until after all rq->hwsp are complete before removing /
130	i915_gem_object_unpin_map(cl->hwsp->vma->obj);
131	__idle_hwsp_free(cl->hwsp, ptr_unmask_bits(cl->vaddr, CACHELINE_BITS));
132
133	i915_active_fini(&cl->active);
134	kfree(cl);
135	}
136
137	static void __idle_cacheline_free(struct intel_timeline_cacheline *cl)
138	{
139	GEM_BUG_ON(!i915_active_is_idle(&cl->active));
140	call_rcu(&cl->rcu, __rcu_cacheline_free);
141	}
142
143	__i915_active_call	36	__i915_active_call
144	static void __cacheline_retire(struct i915_active *active)	37	static void __timeline_retire(struct i915_active *active)
145	{	38	{
146	struct intel_timeline_cacheline *cl =	39	struct intel_timeline *tl =
147	container_of(active, typeof(*cl), active);	40	container_of(active, typeof(*tl), active);
148		41
149	i915_vma_unpin(cl->hwsp->vma);	42	i915_vma_unpin(tl->hwsp_ggtt);
150	if (ptr_test_bit(cl->vaddr, CACHELINE_FREE))	43	intel_timeline_put(tl);
151	__idle_cacheline_free(cl);
152	}	44	}
153		45
154	static int __cacheline_active(struct i915_active *active)	46	static int __timeline_active(struct i915_active *active)
155	{	47	{
156	struct intel_timeline_cacheline *cl =	48	struct intel_timeline *tl =
157	container_of(active, typeof(*cl), active);	49	container_of(active, typeof(*tl), active);
158		50
159	__i915_vma_pin(cl->hwsp->vma);	51	__i915_vma_pin(tl->hwsp_ggtt);
		52	intel_timeline_get(tl);
160	return 0;	53	return 0;
161	}	54	}
162		55
163	static struct intel_timeline_cacheline *	56	I915_SELFTEST_EXPORT int
164	cacheline_alloc(struct intel_timeline_hwsp *hwsp, unsigned int cacheline)	57	intel_timeline_pin_map(struct intel_timeline *timeline)
165	{	58	{
166	struct intel_timeline_cacheline *cl;	59	struct drm_i915_gem_object *obj = timeline->hwsp_ggtt->obj;
		60	u32 ofs = offset_in_page(timeline->hwsp_offset);
167	void *vaddr;	61	void *vaddr;
168		62
169	GEM_BUG_ON(cacheline >= BIT(CACHELINE_BITS));	63	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
170		64	if (IS_ERR(vaddr))
171	cl = kmalloc(sizeof(*cl), GFP_KERNEL);	65	return PTR_ERR(vaddr);
172	if (!cl)
173	return ERR_PTR(-ENOMEM);
174
175	vaddr = i915_gem_object_pin_map(hwsp->vma->obj, I915_MAP_WB);
176	if (IS_ERR(vaddr)) {
177	kfree(cl);
178	return ERR_CAST(vaddr);
179	}
180
181	cl->hwsp = hwsp;
182	cl->vaddr = page_pack_bits(vaddr, cacheline);
183
184	i915_active_init(&cl->active, __cacheline_active, __cacheline_retire);
185
186	return cl;
187	}
188
189	static void cacheline_acquire(struct intel_timeline_cacheline *cl,
190	u32 ggtt_offset)
191	{
192	if (!cl)
193	return;
194
195	cl->ggtt_offset = ggtt_offset;
196	i915_active_acquire(&cl->active);
197	}
198
199	static void cacheline_release(struct intel_timeline_cacheline *cl)
200	{
201	if (cl)
202	i915_active_release(&cl->active);
203	}
204
205	static void cacheline_free(struct intel_timeline_cacheline *cl)
206	{
207	if (!i915_active_acquire_if_busy(&cl->active)) {
208	__idle_cacheline_free(cl);
209	return;
210	}
211		66
212	GEM_BUG_ON(ptr_test_bit(cl->vaddr, CACHELINE_FREE));	67	timeline->hwsp_map = vaddr;
213	cl->vaddr = ptr_set_bit(cl->vaddr, CACHELINE_FREE);	68	timeline->hwsp_seqno = memset(vaddr + ofs, 0, TIMELINE_SEQNO_BYTES);
		69	clflush(vaddr + ofs);
214		70
215	i915_active_release(&cl->active);	71	return 0;
216	}	72	}
217		73
218	static int intel_timeline_init(struct intel_timeline *timeline,	74	static int intel_timeline_init(struct intel_timeline *timeline,
@@ -220,45 +76,25 @@ static int intel_timeline_init(struct intel_timeline *timeline, · 1 unmodified line
220	struct i915_vma *hwsp,	76	struct i915_vma *hwsp,
221	unsigned int offset)	77	unsigned int offset)
222	{	78	{
223	void *vaddr;
224
225	kref_init(&timeline->kref);	79	kref_init(&timeline->kref);
226	atomic_set(&timeline->pin_count, 0);	80	atomic_set(&timeline->pin_count, 0);
227		81
228	timeline->gt = gt;	82	timeline->gt = gt;
229		83
230	timeline->has_initial_breadcrumb = !hwsp;	84	if (hwsp) {
231	timeline->hwsp_cacheline = NULL;	85	timeline->hwsp_offset = offset;
232		86	timeline->hwsp_ggtt = i915_vma_get(hwsp);
233	if (!hwsp) {	87	} else {
234	struct intel_timeline_cacheline *cl;	88	timeline->has_initial_breadcrumb = true;
235	unsigned int cacheline;	89	hwsp = hwsp_alloc(gt);
236
237	hwsp = hwsp_alloc(timeline, &cacheline);
238	if (IS_ERR(hwsp))	90	if (IS_ERR(hwsp))
239	return PTR_ERR(hwsp);	91	return PTR_ERR(hwsp);
240		92	timeline->hwsp_ggtt = hwsp;
241	cl = cacheline_alloc(hwsp->private, cacheline);
242	if (IS_ERR(cl)) {
243	__idle_hwsp_free(hwsp->private, cacheline);
244	return PTR_ERR(cl);
245	}
246
247	timeline->hwsp_cacheline = cl;
248	timeline->hwsp_offset = cacheline * CACHELINE_BYTES;
249
250	vaddr = page_mask_bits(cl->vaddr);
251	} else {
252	timeline->hwsp_offset = offset;
253	vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
254	if (IS_ERR(vaddr))
255	return PTR_ERR(vaddr);
256	}	93	}
257		94
258	timeline->hwsp_seqno =	95	timeline->hwsp_map = NULL;
259	memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);	96	timeline->hwsp_seqno = (void *)(long)timeline->hwsp_offset;
260		97
261	timeline->hwsp_ggtt = i915_vma_get(hwsp);
262	GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size);	98	GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size);
263		99
264	timeline->fence_context = dma_fence_context_alloc(1);	100	timeline->fence_context = dma_fence_context_alloc(1);
@@ -269,6 +105,7 @@ static int intel_timeline_init(struct intel_timeline *timeline, · 4 unmodified lines
269	INIT_LIST_HEAD(&timeline->requests);	105	INIT_LIST_HEAD(&timeline->requests);
270		106
271	i915_syncmap_init(&timeline->sync);	107	i915_syncmap_init(&timeline->sync);
		108	i915_active_init(&timeline->active, __timeline_active, __timeline_retire);
272		109
273	return 0;	110	return 0;
274	}	111	}
@@ -279,23 +116,19 @@ void intel_gt_init_timelines(struct intel_gt *gt) · 4 unmodified lines
279		116
280	spin_lock_init(&timelines->lock);	117	spin_lock_init(&timelines->lock);
281	INIT_LIST_HEAD(&timelines->active_list);	118	INIT_LIST_HEAD(&timelines->active_list);
282
283	spin_lock_init(&timelines->hwsp_lock);
284	INIT_LIST_HEAD(&timelines->hwsp_free_list);
285	}	119	}
286		120
287	static void intel_timeline_fini(struct intel_timeline *timeline)	121	static void intel_timeline_fini(struct rcu_head *rcu)
288	{	122	{
289	GEM_BUG_ON(atomic_read(&timeline->pin_count));	123	struct intel_timeline *timeline =
290	GEM_BUG_ON(!list_empty(&timeline->requests));	124	container_of(rcu, struct intel_timeline, rcu);
291	GEM_BUG_ON(timeline->retire);
292		125
293	if (timeline->hwsp_cacheline)	126	if (timeline->hwsp_map)
294	cacheline_free(timeline->hwsp_cacheline);
295	else
296	i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);	127	i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
297		128
298	i915_vma_put(timeline->hwsp_ggtt);	129	i915_vma_put(timeline->hwsp_ggtt);
		130	i915_active_fini(&timeline->active);
		131	kfree(timeline);
299	}	132	}
300		133
301	struct intel_timeline *	134	struct intel_timeline *
@@ -351,6 +184,12 @@ int intel_timeline_pin(struct intel_timeline tl, struct i915_gem_ww_ctx ww) · 49 unmodified lines
351	if (atomic_add_unless(&tl->pin_count, 1, 0))	184	if (atomic_add_unless(&tl->pin_count, 1, 0))
352	return 0;	185	return 0;
353		186
		187	if (!tl->hwsp_map) {
		188	err = intel_timeline_pin_map(tl);
		189	if (err)
		190	return err;
		191	}
		192
354	err = i915_ggtt_pin(tl->hwsp_ggtt, ww, 0, PIN_HIGH);	193	err = i915_ggtt_pin(tl->hwsp_ggtt, ww, 0, PIN_HIGH);
355	if (err)	194	if (err)
356	return err;	195	return err;
@@ -361,9 +200,9 @@ int intel_timeline_pin(struct intel_timeline tl, struct i915_gem_ww_ctx ww) · 4 unmodified lines
361	GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",	200	GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",
362	tl->fence_context, tl->hwsp_offset);	201	tl->fence_context, tl->hwsp_offset);
363		202
364	cacheline_acquire(tl->hwsp_cacheline, tl->hwsp_offset);	203	i915_active_acquire(&tl->active);
365	if (atomic_fetch_inc(&tl->pin_count)) {	204	if (atomic_fetch_inc(&tl->pin_count)) {
366	cacheline_release(tl->hwsp_cacheline);	205	i915_active_release(&tl->active);
367	__i915_vma_unpin(tl->hwsp_ggtt);	206	__i915_vma_unpin(tl->hwsp_ggtt);
368	}	207	}
369		208
@@ -372,9 +211,13 @@ int intel_timeline_pin(struct intel_timeline tl, struct i915_gem_ww_ctx ww) · 2 unmodified lines
372		211
373	void intel_timeline_reset_seqno(const struct intel_timeline *tl)	212	void intel_timeline_reset_seqno(const struct intel_timeline *tl)
374	{	213	{
		214	u32 hwsp_seqno = (u32 )tl->hwsp_seqno;
375	/* Must be pinned to be writable, and no requests in flight. */	215	/* Must be pinned to be writable, and no requests in flight. */
376	GEM_BUG_ON(!atomic_read(&tl->pin_count));	216	GEM_BUG_ON(!atomic_read(&tl->pin_count));
377	WRITE_ONCE((u32 )tl->hwsp_seqno, tl->seqno);	217
		218	memset(hwsp_seqno + 1, 0, TIMELINE_SEQNO_BYTES - sizeof(*hwsp_seqno));
		219	WRITE_ONCE(*hwsp_seqno, tl->seqno);
		220	clflush(hwsp_seqno);
378	}	221	}
379		222
380	void intel_timeline_enter(struct intel_timeline *tl)	223	void intel_timeline_enter(struct intel_timeline *tl)
@@ -450,106 +293,23 @@ static u32 timeline_advance(struct intel_timeline *tl) · 69 unmodified lines
450	return tl->seqno += 1 + tl->has_initial_breadcrumb;	293	return tl->seqno += 1 + tl->has_initial_breadcrumb;
451	}	294	}
452		295
453	static void timeline_rollback(struct intel_timeline *tl)
454	{
455	tl->seqno -= 1 + tl->has_initial_breadcrumb;
456	}
457
458	static noinline int	296	static noinline int
459	__intel_timeline_get_seqno(struct intel_timeline *tl,	297	__intel_timeline_get_seqno(struct intel_timeline *tl,
460	struct i915_request *rq,
461	u32 *seqno)	298	u32 *seqno)
462	{	299	{
463	struct intel_timeline_cacheline *cl;	300	u32 next_ofs = offset_in_page(tl->hwsp_offset + TIMELINE_SEQNO_BYTES);
464	unsigned int cacheline;
465	struct i915_vma *vma;
466	void *vaddr;
467	int err;
468
469	might_lock(&tl->gt->ggtt->vm.mutex);
470	GT_TRACE(tl->gt, "timeline:%llx wrapped\n", tl->fence_context);
471
472	/*
473	* If there is an outstanding GPU reference to this cacheline,
474	* such as it being sampled by a HW semaphore on another timeline,
475	* we cannot wraparound our seqno value (the HW semaphore does
476	* a strict greater-than-or-equals compare, not i915_seqno_passed).
477	* So if the cacheline is still busy, we must detach ourselves
478	* from it and leave it inflight alongside its users.
479	*
480	* However, if nobody is watching and we can guarantee that nobody
481	* will, we could simply reuse the same cacheline.
482	*
483	* if (i915_active_request_is_signaled(&tl->last_request) &&
484	* i915_active_is_signaled(&tl->hwsp_cacheline->active))
485	* return 0;
486	*
487	* That seems unlikely for a busy timeline that needed to wrap in
488	* the first place, so just replace the cacheline.
489	*/
490
491	vma = hwsp_alloc(tl, &cacheline);
492	if (IS_ERR(vma)) {
493	err = PTR_ERR(vma);
494	goto err_rollback;
495	}
496
497	err = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH);
498	if (err) {
499	__idle_hwsp_free(vma->private, cacheline);
500	goto err_rollback;
501	}
502
503	cl = cacheline_alloc(vma->private, cacheline);
504	if (IS_ERR(cl)) {
505	err = PTR_ERR(cl);
506	__idle_hwsp_free(vma->private, cacheline);
507	goto err_unpin;
508	}
509	GEM_BUG_ON(cl->hwsp->vma != vma);
510
511	/*
512	* Attach the old cacheline to the current request, so that we only
513	* free it after the current request is retired, which ensures that
514	* all writes into the cacheline from previous requests are complete.
515	*/
516	err = i915_active_ref(&tl->hwsp_cacheline->active,
517	tl->fence_context,
518	&rq->fence);
519	if (err)
520	goto err_cacheline;
521
522	cacheline_release(tl->hwsp_cacheline); /* ownership now xfered to rq */
523	cacheline_free(tl->hwsp_cacheline);
524
525	i915_vma_unpin(tl->hwsp_ggtt); /* binding kept alive by old cacheline */
526	i915_vma_put(tl->hwsp_ggtt);
527
528	tl->hwsp_ggtt = i915_vma_get(vma);
529		301
530	vaddr = page_mask_bits(cl->vaddr);	302	/* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
531	tl->hwsp_offset = cacheline * CACHELINE_BYTES;	303	if (TIMELINE_SEQNO_BYTES <= BIT(5) && (next_ofs & BIT(5)))
532	tl->hwsp_seqno =	304	next_ofs = offset_in_page(next_ofs + BIT(5));
533	memset(vaddr + tl->hwsp_offset, 0, CACHELINE_BYTES);
534		305
535	tl->hwsp_offset += i915_ggtt_offset(vma);	306	tl->hwsp_offset = i915_ggtt_offset(tl->hwsp_ggtt) + next_ofs;
536	GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",	307	tl->hwsp_seqno = tl->hwsp_map + next_ofs;
537	tl->fence_context, tl->hwsp_offset);	308	intel_timeline_reset_seqno(tl);
538
539	cacheline_acquire(cl, tl->hwsp_offset);
540	tl->hwsp_cacheline = cl;
541		309
542	*seqno = timeline_advance(tl);	310	*seqno = timeline_advance(tl);
543	GEM_BUG_ON(i915_seqno_passed(tl->hwsp_seqno, seqno));	311	GEM_BUG_ON(i915_seqno_passed(tl->hwsp_seqno, seqno));
544	return 0;	312	return 0;
545
546	err_cacheline:
547	cacheline_free(cl);
548	err_unpin:
549	i915_vma_unpin(vma);
550	err_rollback:
551	timeline_rollback(tl);
552	return err;
553	}	313	}
554		314
555	int intel_timeline_get_seqno(struct intel_timeline *tl,	315	int intel_timeline_get_seqno(struct intel_timeline *tl,
@@ -559,51 +319,52 @@ int intel_timeline_get_seqno(struct intel_timeline *tl, · 3 unmodified lines
559	*seqno = timeline_advance(tl);	319	*seqno = timeline_advance(tl);
560		320
561	/* Replace the HWSP on wraparound for HW semaphores */	321	/* Replace the HWSP on wraparound for HW semaphores */
562	if (unlikely(!*seqno && tl->hwsp_cacheline))	322	if (unlikely(!*seqno && tl->has_initial_breadcrumb))
563	return __intel_timeline_get_seqno(tl, rq, seqno);	323	return __intel_timeline_get_seqno(tl, seqno);
564		324
565	return 0;	325	return 0;
566	}	326	}
567		327
568	static int cacheline_ref(struct intel_timeline_cacheline *cl,
569	struct i915_request *rq)
570	{
571	return i915_active_add_request(&cl->active, rq);
572	}
573
574	int intel_timeline_read_hwsp(struct i915_request *from,	328	int intel_timeline_read_hwsp(struct i915_request *from,
575	struct i915_request *to,	329	struct i915_request *to,
576	u32 *hwsp)	330	u32 *hwsp)
577	{	331	{
578	struct intel_timeline_cacheline *cl;	332	struct intel_timeline *tl;
579	int err;	333	int err;
580		334
581	GEM_BUG_ON(!rcu_access_pointer(from->hwsp_cacheline));
582
583	rcu_read_lock();	335	rcu_read_lock();
584	cl = rcu_dereference(from->hwsp_cacheline);	336	tl = rcu_dereference(from->timeline);
585	if (i915_request_signaled(from)) /* confirm cacheline is valid */	337	if (i915_request_signaled(from) \|\|
586	goto unlock;	338	!i915_active_acquire_if_busy(&tl->active))
587	if (unlikely(!i915_active_acquire_if_busy(&cl->active)))	339	tl = NULL;
588	goto unlock; /* seqno wrapped and completed! */	340
589	if (unlikely(__i915_request_is_complete(from)))	341	if (tl) {
590	goto release;	342	/* hwsp_offset may wraparound, so use from->hwsp_seqno */
		343	*hwsp = i915_ggtt_offset(tl->hwsp_ggtt) +
		344	offset_in_page(from->hwsp_seqno);
		345	}
		346
		347	/* ensure we wait on the right request, if not, we completed */
		348	if (tl && __i915_request_is_complete(from)) {
		349	i915_active_release(&tl->active);
		350	tl = NULL;
		351	}
591	rcu_read_unlock();	352	rcu_read_unlock();
592		353
593	err = cacheline_ref(cl, to);	354	if (!tl)
594	if (err)	355	return 1;
		356
		357	/* Can't do semaphore waits on kernel context */
		358	if (!tl->has_initial_breadcrumb) {
		359	err = -EINVAL;
595	goto out;	360	goto out;
		361	}
		362
		363	err = i915_active_add_request(&tl->active, to);
596		364
597	*hwsp = cl->ggtt_offset;
598	out:	365	out:
599	i915_active_release(&cl->active);	366	i915_active_release(&tl->active);
600	return err;	367	return err;
601
602	release:
603	i915_active_release(&cl->active);
604	unlock:
605	rcu_read_unlock();
606	return 1;
607	}	368	}
608		369
609	void intel_timeline_unpin(struct intel_timeline *tl)	370	void intel_timeline_unpin(struct intel_timeline *tl)
@@ -612,8 +373,7 @@ void intel_timeline_unpin(struct intel_timeline *tl) · 2 unmodified lines
612	if (!atomic_dec_and_test(&tl->pin_count))	373	if (!atomic_dec_and_test(&tl->pin_count))
613	return;	374	return;
614		375
615	cacheline_release(tl->hwsp_cacheline);	376	i915_active_release(&tl->active);
616
617	__i915_vma_unpin(tl->hwsp_ggtt);	377	__i915_vma_unpin(tl->hwsp_ggtt);
618	}	378	}
619		379
@@ -622,8 +382,11 @@ void __intel_timeline_free(struct kref *kref) · 2 unmodified lines
622	struct intel_timeline *timeline =	382	struct intel_timeline *timeline =
623	container_of(kref, typeof(*timeline), kref);	383	container_of(kref, typeof(*timeline), kref);
624		384
625	intel_timeline_fini(timeline);	385	GEM_BUG_ON(atomic_read(&timeline->pin_count));
626	kfree_rcu(timeline, rcu);	386	GEM_BUG_ON(!list_empty(&timeline->requests));
		387	GEM_BUG_ON(timeline->retire);
		388
		389	call_rcu(&timeline->rcu, intel_timeline_fini);
627	}	390	}
628		391
629	void intel_gt_fini_timelines(struct intel_gt *gt)	392	void intel_gt_fini_timelines(struct intel_gt *gt)
@@ -631,7 +394,6 @@ void intel_gt_fini_timelines(struct intel_gt *gt) · 1 unmodified line
631	struct intel_gt_timelines *timelines = &gt->timelines;	394	struct intel_gt_timelines *timelines = &gt->timelines;
632		395
633	GEM_BUG_ON(!list_empty(&timelines->active_list));	396	GEM_BUG_ON(!list_empty(&timelines->active_list));
634	GEM_BUG_ON(!list_empty(&timelines->hwsp_free_list));
635	}	397	}
636		398
637	void intel_gt_show_timelines(struct intel_gt *gt,	399	void intel_gt_show_timelines(struct intel_gt *gt,

drivers/gpu/drm/i915/gt/intel_timeline.h+2−0

@@ -117,4 +117,6 @@ intel_timeline_is_last(const struct intel_timeline *tl, · 116 unmodified lines
117	return list_is_last_rcu(&rq->link, &tl->requests);	117	return list_is_last_rcu(&rq->link, &tl->requests);
118	}	118	}
119		119
		120	I915_SELFTEST_DECLARE(int intel_timeline_pin_map(struct intel_timeline *tl));
		121
120	#endif	122	#endif

drivers/gpu/drm/i915/gt/intel_timeline_types.h+3−14

 struct i915_vma;
 struct i915_syncmap;
 struct intel_gt;
-struct intel_timeline_hwsp;
 struct intel_timeline {
 	u64 fence_context;
 	atomic_t pin_count;
 	atomic_t active_count;
+	void *hwsp_map;
 	const u32 *hwsp_seqno;
 	struct i915_vma *hwsp_ggtt;
 	u32 hwsp_offset;
-	struct intel_timeline_cacheline *hwsp_cacheline;
 	bool has_initial_breadcrumb;
 	/**
 	 */
 	struct i915_active_fence last_request;
+	struct i915_active active;
 	/** A chain of completed timelines ready for early retirement. */
 	struct intel_timeline *retire;
 	struct rcu_head rcu;
 };
-struct intel_timeline_cacheline {
-	struct i915_active active;
-	struct intel_timeline_hwsp *hwsp;
-	void *vaddr;
-	u32 ggtt_offset;
-	struct rcu_head rcu;
-};
 #endif /* __I915_TIMELINE_TYPES_H__ */

drivers/gpu/drm/i915/gt/intel_workarounds.c+8−2

 	if (err)
 		goto err_pm;
+	err = i915_vma_pin_ww(vma, &ww, 0, 0,
+			   i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
+	if (err)
+		goto err_unpin;
 	rq = i915_request_create(ce);
 	if (IS_ERR(rq)) {
 		err = PTR_ERR(rq);
-		goto err_unpin;
+		goto err_vma;
 	}
 	err = i915_request_await_object(rq, vma->obj, true);
 err_rq:
 	i915_request_put(rq);
+err_vma:
+	i915_vma_unpin(vma);
 err_unpin:
 	intel_context_unpin(ce);
 err_pm:
 	}
 	i915_gem_ww_ctx_fini(&ww);
 	intel_engine_pm_put(ce->engine);
-	i915_vma_unpin(vma);
 	i915_vma_put(vma);
 	return err;
 }

drivers/gpu/drm/i915/gt/mock_engine.c+20−2

 #include "mock_engine.h"
 #include "selftests/mock_request.h"
-static void mock_timeline_pin(struct intel_timeline *tl)
+static int mock_timeline_pin(struct intel_timeline *tl)
 {
+	int err;
+	if (WARN_ON(!i915_gem_object_trylock(tl->hwsp_ggtt->obj)))
+		return -EBUSY;
+	err = intel_timeline_pin_map(tl);
+	i915_gem_object_unlock(tl->hwsp_ggtt->obj);
+	if (err)
+		return err;
 	atomic_inc(&tl->pin_count);
+	return 0;
 }
 static void mock_timeline_unpin(struct intel_timeline *tl)
 static int mock_context_alloc(struct intel_context *ce)
 {
+	int err;
 	ce->ring = mock_ring(ce->engine);
 	if (!ce->ring)
 		return -ENOMEM;
 		return PTR_ERR(ce->timeline);
 	}
-	mock_timeline_pin(ce->timeline);
+	err = mock_timeline_pin(ce->timeline);
+	if (err) {
+		intel_timeline_put(ce->timeline);
+		ce->timeline = NULL;
+		return err;
+	}
 	return 0;
 }

drivers/gpu/drm/i915/gt/selftest_context.c+2−2

@@ -88,8 +88,8 @@ static int __live_context_size(struct intel_engine_cs *engine) · 87 unmodified lines
88	if (err)	88	if (err)
89	goto err;	89	goto err;
90		90
91	vaddr = i915_gem_object_pin_map(ce->state->obj,	91	vaddr = i915_gem_object_pin_map_unlocked(ce->state->obj,
92	i915_coherent_map_type(engine->i915));	92	i915_coherent_map_type(engine->i915));
93	if (IS_ERR(vaddr)) {	93	if (IS_ERR(vaddr)) {
94	err = PTR_ERR(vaddr);	94	err = PTR_ERR(vaddr);
95	intel_context_unpin(ce);	95	intel_context_unpin(ce);

drivers/gpu/drm/i915/gt/selftest_engine_cs.c+6−3

 static int write_timestamp(struct i915_request *rq, int slot)
 {
+	struct intel_timeline *tl =
+		rcu_dereference_protected(rq->timeline,
+					  !i915_request_signaled(rq));
 	u32 cmd;
 	u32 *cs;
 		cmd++;
 	*cs++ = cmd;
 	*cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(rq->engine->mmio_base));
-	*cs++ = i915_request_timeline(rq)->hwsp_offset + slot * sizeof(u32);
+	*cs++ = tl->hwsp_offset + slot * sizeof(u32);
 	*cs++ = 0;
 	intel_ring_advance(rq, cs);
 	if (IS_ERR(obj))
 		return ERR_CAST(obj);
-	cs = i915_gem_object_pin_map(obj, I915_MAP_WB);
+	cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
 	if (IS_ERR(cs)) {
 		err = PTR_ERR(cs);
 		goto err_put;
 	if (IS_ERR(obj))
 		return ERR_CAST(obj);
-	cs = i915_gem_object_pin_map(obj, I915_MAP_WB);
+	cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
 	if (IS_ERR(cs)) {
 		err = PTR_ERR(cs);
 		goto err_put;

drivers/gpu/drm/i915/gt/selftest_execlists.c+12−11

 		goto err_obj;
 	}
-	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
+	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 	if (IS_ERR(vaddr)) {
 		err = PTR_ERR(vaddr);
 		goto err_obj;
 		goto err_obj;
 	}
-	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
+	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 	if (IS_ERR(vaddr)) {
 		err = PTR_ERR(vaddr);
 		goto err_obj;
 		goto err_ctx_lo;
 	}
-	map = i915_gem_object_pin_map(obj, I915_MAP_WC);
+	map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 	if (IS_ERR(map)) {
 		err = PTR_ERR(map);
 		goto err_obj;
 	if (err)
 		goto err_obj;
-	cs = i915_gem_object_pin_map(obj, I915_MAP_WC);
+	cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 	if (IS_ERR(cs)) {
 		err = PTR_ERR(cs);
 		goto err_obj;
 		 * it will terminate the next lowest spinner until there
 		 * are no more spinners and the gang is complete.
 		 */
-		cs = i915_gem_object_pin_map(rq->batch->obj, I915_MAP_WC);
+		cs = i915_gem_object_pin_map_unlocked(rq->batch->obj, I915_MAP_WC);
 		if (!IS_ERR(cs)) {
 			*cs = 0;
 			i915_gem_object_unpin_map(rq->batch->obj);
 		return ERR_PTR(err);
 	}
-	cs = i915_gem_object_pin_map(obj, I915_MAP_WC);
+	cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 	if (IS_ERR(cs)) {
 		i915_vma_put(vma);
 		return ERR_CAST(cs);
 	if (IS_ERR(global))
 		return PTR_ERR(global);
-	result = i915_gem_object_pin_map(global->obj, I915_MAP_WC);
+	result = i915_gem_object_pin_map_unlocked(global->obj, I915_MAP_WC);
 	if (IS_ERR(result)) {
 		i915_vma_unpin_and_release(&global, 0);
 		return PTR_ERR(result);
 		goto err_free;
 	}
-	cs = i915_gem_object_pin_map(smoke.batch, I915_MAP_WB);
+	cs = i915_gem_object_pin_map_unlocked(smoke.batch, I915_MAP_WB);
 	if (IS_ERR(cs)) {
 		err = PTR_ERR(cs);
 		goto err_batch;
 	int err = 0;
 	u32 *cs;
-	scratch = __vm_create_scratch_for_read(&siblings[0]->gt->ggtt->vm,
+	scratch =
-					       PAGE_SIZE);
+		__vm_create_scratch_for_read_pinned(&siblings[0]->gt->ggtt->vm,
+						    PAGE_SIZE);
 	if (IS_ERR(scratch))
 		return PTR_ERR(scratch);
 		goto out_end;
 	}
-	cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
+	cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
 	if (IS_ERR(cs)) {
 		err = PTR_ERR(cs);
 		goto out_end;

drivers/gpu/drm/i915/gt/selftest_hangcheck.c+4−4

@@ -80,15 +80,15 @@ static int hang_init(struct hang h, struct intel_gt gt) · 79 unmodified lines
80	}	80	}
81		81
82	i915_gem_object_set_cache_coherency(h->hws, I915_CACHE_LLC);	82	i915_gem_object_set_cache_coherency(h->hws, I915_CACHE_LLC);
83	vaddr = i915_gem_object_pin_map(h->hws, I915_MAP_WB);	83	vaddr = i915_gem_object_pin_map_unlocked(h->hws, I915_MAP_WB);
84	if (IS_ERR(vaddr)) {	84	if (IS_ERR(vaddr)) {
85	err = PTR_ERR(vaddr);	85	err = PTR_ERR(vaddr);
86	goto err_obj;	86	goto err_obj;
87	}	87	}
88	h->seqno = memset(vaddr, 0xff, PAGE_SIZE);	88	h->seqno = memset(vaddr, 0xff, PAGE_SIZE);
89		89
90	vaddr = i915_gem_object_pin_map(h->obj,	90	vaddr = i915_gem_object_pin_map_unlocked(h->obj,
91	i915_coherent_map_type(gt->i915));	91	i915_coherent_map_type(gt->i915));
92	if (IS_ERR(vaddr)) {	92	if (IS_ERR(vaddr)) {
93	err = PTR_ERR(vaddr);	93	err = PTR_ERR(vaddr);
94	goto err_unpin_hws;	94	goto err_unpin_hws;
@@ -149,7 +149,7 @@ hang_create_request(struct hang h, struct intel_engine_cs engine) · 54 unmodified lines
149	return ERR_CAST(obj);	149	return ERR_CAST(obj);
150	}	150	}
151		151
152	vaddr = i915_gem_object_pin_map(obj, i915_coherent_map_type(gt->i915));	152	vaddr = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(gt->i915));
153	if (IS_ERR(vaddr)) {	153	if (IS_ERR(vaddr)) {
154	i915_gem_object_put(obj);	154	i915_gem_object_put(obj);
155	i915_vm_put(vm);	155	i915_vm_put(vm);

drivers/gpu/drm/i915/gt/selftest_lrc.c+9−9

@@ -27,7 +27,7 @@ · 26 unmodified lines
27		27
28	static struct i915_vma create_scratch(struct intel_gt gt)	28	static struct i915_vma create_scratch(struct intel_gt gt)
29	{	29	{
30	return __vm_create_scratch_for_read(&gt->ggtt->vm, PAGE_SIZE);	30	return __vm_create_scratch_for_read_pinned(&gt->ggtt->vm, PAGE_SIZE);
31	}	31	}
32		32
33	static bool is_active(struct i915_request *rq)	33	static bool is_active(struct i915_request *rq)
@@ -627,7 +627,7 @@ static int __live_lrc_gpr(struct intel_engine_cs *engine, · 593 unmodified lines
627	goto err_rq;	627	goto err_rq;
628	}	628	}
629		629
630	cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);	630	cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
631	if (IS_ERR(cs)) {	631	if (IS_ERR(cs)) {
632	err = PTR_ERR(cs);	632	err = PTR_ERR(cs);
633	goto err_rq;	633	goto err_rq;
@@ -921,7 +921,7 @@ store_context(struct intel_context ce, struct i915_vma scratch) · 287 unmodified lines
921	if (IS_ERR(batch))	921	if (IS_ERR(batch))
922	return batch;	922	return batch;
923		923
924	cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);	924	cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
925	if (IS_ERR(cs)) {	925	if (IS_ERR(cs)) {
926	i915_vma_put(batch);	926	i915_vma_put(batch);
927	return ERR_CAST(cs);	927	return ERR_CAST(cs);
@@ -1085,7 +1085,7 @@ static struct i915_vma load_context(struct intel_context ce, u32 poison) · 157 unmodified lines
1085	if (IS_ERR(batch))	1085	if (IS_ERR(batch))
1086	return batch;	1086	return batch;
1087		1087
1088	cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);	1088	cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1089	if (IS_ERR(cs)) {	1089	if (IS_ERR(cs)) {
1090	i915_vma_put(batch);	1090	i915_vma_put(batch);
1091	return ERR_CAST(cs);	1091	return ERR_CAST(cs);
@@ -1199,29 +1199,29 @@ static int compare_isolation(struct intel_engine_cs *engine, · 107 unmodified lines
1199	u32 *defaults;	1199	u32 *defaults;
1200	int err = 0;	1200	int err = 0;
1201		1201
1202	A[0] = i915_gem_object_pin_map(ref[0]->obj, I915_MAP_WC);	1202	A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);
1203	if (IS_ERR(A[0]))	1203	if (IS_ERR(A[0]))
1204	return PTR_ERR(A[0]);	1204	return PTR_ERR(A[0]);
1205		1205
1206	A[1] = i915_gem_object_pin_map(ref[1]->obj, I915_MAP_WC);	1206	A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);
1207	if (IS_ERR(A[1])) {	1207	if (IS_ERR(A[1])) {
1208	err = PTR_ERR(A[1]);	1208	err = PTR_ERR(A[1]);
1209	goto err_A0;	1209	goto err_A0;
1210	}	1210	}
1211		1211
1212	B[0] = i915_gem_object_pin_map(result[0]->obj, I915_MAP_WC);	1212	B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);
1213	if (IS_ERR(B[0])) {	1213	if (IS_ERR(B[0])) {
1214	err = PTR_ERR(B[0]);	1214	err = PTR_ERR(B[0]);
1215	goto err_A1;	1215	goto err_A1;
1216	}	1216	}
1217		1217
1218	B[1] = i915_gem_object_pin_map(result[1]->obj, I915_MAP_WC);	1218	B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);
1219	if (IS_ERR(B[1])) {	1219	if (IS_ERR(B[1])) {
1220	err = PTR_ERR(B[1]);	1220	err = PTR_ERR(B[1]);
1221	goto err_B0;	1221	goto err_B0;
1222	}	1222	}
1223		1223
1224	lrc = i915_gem_object_pin_map(ce->state->obj,	1224	lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,
1225	i915_coherent_map_type(engine->i915));	1225	i915_coherent_map_type(engine->i915));
1226	if (IS_ERR(lrc)) {	1226	if (IS_ERR(lrc)) {
1227	err = PTR_ERR(lrc);	1227	err = PTR_ERR(lrc);

drivers/gpu/drm/i915/gt/selftest_mocs.c+3−2

 	if (flags & (HAS_GLOBAL_MOCS | HAS_ENGINE_MOCS))
 		arg->mocs = table;
-	arg->scratch = __vm_create_scratch_for_read(&gt->ggtt->vm, PAGE_SIZE);
+	arg->scratch =
+		__vm_create_scratch_for_read_pinned(&gt->ggtt->vm, PAGE_SIZE);
 	if (IS_ERR(arg->scratch))
 		return PTR_ERR(arg->scratch);
-	arg->vaddr = i915_gem_object_pin_map(arg->scratch->obj, I915_MAP_WB);
+	arg->vaddr = i915_gem_object_pin_map_unlocked(arg->scratch->obj, I915_MAP_WB);
 	if (IS_ERR(arg->vaddr)) {
 		err = PTR_ERR(arg->vaddr);
 		goto err_scratch;

drivers/gpu/drm/i915/gt/selftest_ring_submission.c+2−2

@@ -35,7 +35,7 @@ static struct i915_vma create_wally(struct intel_engine_cs engine) · 34 unmodified lines
35	return ERR_PTR(err);	35	return ERR_PTR(err);
36	}	36	}
37		37
38	cs = i915_gem_object_pin_map(obj, I915_MAP_WC);	38	cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
39	if (IS_ERR(cs)) {	39	if (IS_ERR(cs)) {
40	i915_gem_object_put(obj);	40	i915_gem_object_put(obj);
41	return ERR_CAST(cs);	41	return ERR_CAST(cs);
@@ -212,7 +212,7 @@ static int __live_ctx_switch_wa(struct intel_engine_cs *engine) · 170 unmodified lines
212	if (IS_ERR(bb))	212	if (IS_ERR(bb))
213	return PTR_ERR(bb);	213	return PTR_ERR(bb);
214		214
215	result = i915_gem_object_pin_map(bb->obj, I915_MAP_WC);	215	result = i915_gem_object_pin_map_unlocked(bb->obj, I915_MAP_WC);
216	if (IS_ERR(result)) {	216	if (IS_ERR(result)) {
217	intel_context_put(bb->private);	217	intel_context_put(bb->private);
218	i915_vma_unpin_and_release(&bb, 0);	218	i915_vma_unpin_and_release(&bb, 0);

drivers/gpu/drm/i915/gt/selftest_timeline.c+109−68

@@ -35,10 +35,31 @@ static unsigned long hwsp_cacheline(struct intel_timeline *tl) · 34 unmodified lines
35	{	35	{
36	unsigned long address = (unsigned long)page_address(hwsp_page(tl));	36	unsigned long address = (unsigned long)page_address(hwsp_page(tl));
37		37
38	return (address + tl->hwsp_offset) / CACHELINE_BYTES;	38	return (address + offset_in_page(tl->hwsp_offset)) / TIMELINE_SEQNO_BYTES;
39	}	39	}
40		40
41	#define CACHELINES_PER_PAGE (PAGE_SIZE / CACHELINE_BYTES)	41	static int selftest_tl_pin(struct intel_timeline *tl)
		42	{
		43	struct i915_gem_ww_ctx ww;
		44	int err;
		45
		46	i915_gem_ww_ctx_init(&ww, false);
		47	retry:
		48	err = i915_gem_object_lock(tl->hwsp_ggtt->obj, &ww);
		49	if (!err)
		50	err = intel_timeline_pin(tl, &ww);
		51
		52	if (err == -EDEADLK) {
		53	err = i915_gem_ww_ctx_backoff(&ww);
		54	if (!err)
		55	goto retry;
		56	}
		57	i915_gem_ww_ctx_fini(&ww);
		58	return err;
		59	}
		60
		61	/* Only half of seqno's are usable, see __intel_timeline_get_seqno() */
		62	#define CACHELINES_PER_PAGE (PAGE_SIZE / TIMELINE_SEQNO_BYTES / 2)
42		63
43	struct mock_hwsp_freelist {	64	struct mock_hwsp_freelist {
44	struct intel_gt *gt;	65	struct intel_gt *gt;
@@ -59,6 +80,7 @@ static void __mock_hwsp_record(struct mock_hwsp_freelist *state, · 14 unmodified lines
59	tl = xchg(&state->history[idx], tl);	80	tl = xchg(&state->history[idx], tl);
60	if (tl) {	81	if (tl) {
61	radix_tree_delete(&state->cachelines, hwsp_cacheline(tl));	82	radix_tree_delete(&state->cachelines, hwsp_cacheline(tl));
		83	intel_timeline_unpin(tl);
62	intel_timeline_put(tl);	84	intel_timeline_put(tl);
63	}	85	}
64	}	86	}
@@ -78,6 +100,12 @@ static int __mock_hwsp_timeline(struct mock_hwsp_freelist *state, · 13 unmodified lines
78	if (IS_ERR(tl))	100	if (IS_ERR(tl))
79	return PTR_ERR(tl);	101	return PTR_ERR(tl);
80		102
		103	err = selftest_tl_pin(tl);
		104	if (err) {
		105	intel_timeline_put(tl);
		106	return err;
		107	}
		108
81	cacheline = hwsp_cacheline(tl);	109	cacheline = hwsp_cacheline(tl);
82	err = radix_tree_insert(&state->cachelines, cacheline, tl);	110	err = radix_tree_insert(&state->cachelines, cacheline, tl);
83	if (err) {	111	if (err) {
@@ -85,6 +113,7 @@ static int __mock_hwsp_timeline(struct mock_hwsp_freelist *state, · 1 unmodified line
85	pr_err("HWSP cacheline %lu already used; duplicate allocation!\n",	113	pr_err("HWSP cacheline %lu already used; duplicate allocation!\n",
86	cacheline);	114	cacheline);
87	}	115	}
		116	intel_timeline_unpin(tl);
88	intel_timeline_put(tl);	117	intel_timeline_put(tl);
89	return err;	118	return err;
90	}	119	}
@@ -452,17 +481,24 @@ static int emit_ggtt_store_dw(struct i915_request *rq, u32 addr, u32 value) · 361 unmodified lines
452	}	481	}
453		482
454	static struct i915_request *	483	static struct i915_request *
455	tl_write(struct intel_timeline tl, struct intel_engine_cs engine, u32 value)	484	checked_tl_write(struct intel_timeline tl, struct intel_engine_cs engine, u32 value)
456	{	485	{
457	struct i915_request *rq;	486	struct i915_request *rq;
458	int err;	487	int err;
459		488
460	err = intel_timeline_pin(tl, NULL);	489	err = selftest_tl_pin(tl);
461	if (err) {	490	if (err) {
462	rq = ERR_PTR(err);	491	rq = ERR_PTR(err);
463	goto out;	492	goto out;
464	}	493	}
465		494
		495	if (READ_ONCE(*tl->hwsp_seqno) != tl->seqno) {
		496	pr_err("Timeline created with incorrect breadcrumb, found %x, expected %x\n",
		497	*tl->hwsp_seqno, tl->seqno);
		498	intel_timeline_unpin(tl);
		499	return ERR_PTR(-EINVAL);
		500	}
		501
466	rq = intel_engine_create_kernel_request(engine);	502	rq = intel_engine_create_kernel_request(engine);
467	if (IS_ERR(rq))	503	if (IS_ERR(rq))
468	goto out_unpin;	504	goto out_unpin;
@@ -484,25 +520,6 @@ out: · 15 unmodified lines
484	return rq;	520	return rq;
485	}	521	}
486		522
487	static struct intel_timeline *
488	checked_intel_timeline_create(struct intel_gt *gt)
489	{
490	struct intel_timeline *tl;
491
492	tl = intel_timeline_create(gt);
493	if (IS_ERR(tl))
494	return tl;
495
496	if (READ_ONCE(*tl->hwsp_seqno) != tl->seqno) {
497	pr_err("Timeline created with incorrect breadcrumb, found %x, expected %x\n",
498	*tl->hwsp_seqno, tl->seqno);
499	intel_timeline_put(tl);
500	return ERR_PTR(-EINVAL);
501	}
502
503	return tl;
504	}
505
506	static int live_hwsp_engine(void *arg)	523	static int live_hwsp_engine(void *arg)
507	{	524	{
508	#define NUM_TIMELINES 4096	525	#define NUM_TIMELINES 4096
@@ -535,13 +552,13 @@ static int live_hwsp_engine(void *arg) · 26 unmodified lines
535	struct intel_timeline *tl;	552	struct intel_timeline *tl;
536	struct i915_request *rq;	553	struct i915_request *rq;
537		554
538	tl = checked_intel_timeline_create(gt);	555	tl = intel_timeline_create(gt);
539	if (IS_ERR(tl)) {	556	if (IS_ERR(tl)) {
540	err = PTR_ERR(tl);	557	err = PTR_ERR(tl);
541	break;	558	break;
542	}	559	}
543		560
544	rq = tl_write(tl, engine, count);	561	rq = checked_tl_write(tl, engine, count);
545	if (IS_ERR(rq)) {	562	if (IS_ERR(rq)) {
546	intel_timeline_put(tl);	563	intel_timeline_put(tl);
547	err = PTR_ERR(rq);	564	err = PTR_ERR(rq);
@@ -608,14 +625,14 @@ static int live_hwsp_alternate(void *arg) · 60 unmodified lines
608	if (!intel_engine_can_store_dword(engine))	625	if (!intel_engine_can_store_dword(engine))
609	continue;	626	continue;
610		627
611	tl = checked_intel_timeline_create(gt);	628	tl = intel_timeline_create(gt);
612	if (IS_ERR(tl)) {	629	if (IS_ERR(tl)) {
613	err = PTR_ERR(tl);	630	err = PTR_ERR(tl);
614	goto out;	631	goto out;
615	}	632	}
616		633
617	intel_engine_pm_get(engine);	634	intel_engine_pm_get(engine);
618	rq = tl_write(tl, engine, count);	635	rq = checked_tl_write(tl, engine, count);
619	intel_engine_pm_put(engine);	636	intel_engine_pm_put(engine);
620	if (IS_ERR(rq)) {	637	if (IS_ERR(rq)) {
621	intel_timeline_put(tl);	638	intel_timeline_put(tl);
@@ -666,10 +683,10 @@ static int live_hwsp_wrap(void *arg) · 44 unmodified lines
666	if (IS_ERR(tl))	683	if (IS_ERR(tl))
667	return PTR_ERR(tl);	684	return PTR_ERR(tl);
668		685
669	if (!tl->has_initial_breadcrumb \|\| !tl->hwsp_cacheline)	686	if (!tl->has_initial_breadcrumb)
670	goto out_free;	687	goto out_free;
671		688
672	err = intel_timeline_pin(tl, NULL);	689	err = selftest_tl_pin(tl);
673	if (err)	690	if (err)
674	goto out_free;	691	goto out_free;
675		692
@@ -816,13 +833,13 @@ static int setup_watcher(struct hwsp_watcher w, struct intel_gt gt) · 140 unmodified lines
816	if (IS_ERR(obj))	833	if (IS_ERR(obj))
817	return PTR_ERR(obj);	834	return PTR_ERR(obj);
818		835
819	w->map = i915_gem_object_pin_map(obj, I915_MAP_WB);	836	w->map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
820	if (IS_ERR(w->map)) {	837	if (IS_ERR(w->map)) {
821	i915_gem_object_put(obj);	838	i915_gem_object_put(obj);
822	return PTR_ERR(w->map);	839	return PTR_ERR(w->map);
823	}	840	}
824		841
825	vma = i915_gem_object_ggtt_pin_ww(obj, NULL, NULL, 0, 0, 0);	842	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
826	if (IS_ERR(vma)) {	843	if (IS_ERR(vma)) {
827	i915_gem_object_put(obj);	844	i915_gem_object_put(obj);
828	return PTR_ERR(vma);	845	return PTR_ERR(vma);
@@ -833,12 +850,26 @@ static int setup_watcher(struct hwsp_watcher w, struct intel_gt gt) · 4 unmodified lines
833	return 0;	850	return 0;
834	}	851	}
835		852
		853	static void switch_tl_lock(struct i915_request from, struct i915_request to)
		854	{
		855	/* some light mutex juggling required; think co-routines */
		856
		857	if (from) {
		858	lockdep_unpin_lock(&from->context->timeline->mutex, from->cookie);
		859	mutex_unlock(&from->context->timeline->mutex);
		860	}
		861
		862	if (to) {
		863	mutex_lock(&to->context->timeline->mutex);
		864	to->cookie = lockdep_pin_lock(&to->context->timeline->mutex);
		865	}
		866	}
		867
836	static int create_watcher(struct hwsp_watcher *w,	868	static int create_watcher(struct hwsp_watcher *w,
837	struct intel_engine_cs *engine,	869	struct intel_engine_cs *engine,
838	int ringsz)	870	int ringsz)
839	{	871	{
840	struct intel_context *ce;	872	struct intel_context *ce;
841	struct intel_timeline *tl;
842		873
843	ce = intel_context_create(engine);	874	ce = intel_context_create(engine);
844	if (IS_ERR(ce))	875	if (IS_ERR(ce))
@@ -851,11 +882,8 @@ static int create_watcher(struct hwsp_watcher *w, · 6 unmodified lines
851	return PTR_ERR(w->rq);	882	return PTR_ERR(w->rq);
852		883
853	w->addr = i915_ggtt_offset(w->vma);	884	w->addr = i915_ggtt_offset(w->vma);
854	tl = w->rq->context->timeline;
855		885
856	/* some light mutex juggling required; think co-routines */	886	switch_tl_lock(w->rq, NULL);
857	lockdep_unpin_lock(&tl->mutex, w->rq->cookie);
858	mutex_unlock(&tl->mutex);
859		887
860	return 0;	888	return 0;
861	}	889	}
@@ -864,15 +892,13 @@ static int check_watcher(struct hwsp_watcher w, const char name, · 2 unmodified lines
864	bool (*op)(u32 hwsp, u32 seqno))	892	bool (*op)(u32 hwsp, u32 seqno))
865	{	893	{
866	struct i915_request *rq = fetch_and_zero(&w->rq);	894	struct i915_request *rq = fetch_and_zero(&w->rq);
867	struct intel_timeline *tl = rq->context->timeline;
868	u32 offset, end;	895	u32 offset, end;
869	int err;	896	int err;
870		897
871	GEM_BUG_ON(w->addr - i915_ggtt_offset(w->vma) > w->vma->size);	898	GEM_BUG_ON(w->addr - i915_ggtt_offset(w->vma) > w->vma->size);
872		899
873	i915_request_get(rq);	900	i915_request_get(rq);
874	mutex_lock(&tl->mutex);	901	switch_tl_lock(NULL, rq);
875	rq->cookie = lockdep_pin_lock(&tl->mutex);
876	i915_request_add(rq);	902	i915_request_add(rq);
877		903
878	if (i915_request_wait(rq, 0, HZ) < 0) {	904	if (i915_request_wait(rq, 0, HZ) < 0) {
@@ -901,10 +927,7 @@ out: · 22 unmodified lines
901	static void cleanup_watcher(struct hwsp_watcher *w)	927	static void cleanup_watcher(struct hwsp_watcher *w)
902	{	928	{
903	if (w->rq) {	929	if (w->rq) {
904	struct intel_timeline *tl = w->rq->context->timeline;	930	switch_tl_lock(NULL, w->rq);
905
906	mutex_lock(&tl->mutex);
907	w->rq->cookie = lockdep_pin_lock(&tl->mutex);
908		931
909	i915_request_add(w->rq);	932	i915_request_add(w->rq);
910	}	933	}
@@ -942,7 +965,7 @@ static struct i915_request wrap_timeline(struct i915_request rq) · 31 unmodified lines
942	}	965	}
943		966
944	i915_request_put(rq);	967	i915_request_put(rq);
945	rq = intel_context_create_request(ce);	968	rq = i915_request_create(ce);
946	if (IS_ERR(rq))	969	if (IS_ERR(rq))
947	return rq;	970	return rq;
948		971
@@ -977,7 +1000,7 @@ static int live_hwsp_read(void *arg) · 28 unmodified lines
977	if (IS_ERR(tl))	1000	if (IS_ERR(tl))
978	return PTR_ERR(tl);	1001	return PTR_ERR(tl);
979		1002
980	if (!tl->hwsp_cacheline)	1003	if (!tl->has_initial_breadcrumb)
981	goto out_free;	1004	goto out_free;
982		1005
983	for (i = 0; i < ARRAY_SIZE(watcher); i++) {	1006	for (i = 0; i < ARRAY_SIZE(watcher); i++) {
@@ -999,7 +1022,7 @@ static int live_hwsp_read(void *arg) · 15 unmodified lines
999	do {	1022	do {
1000	struct i915_sw_fence *submit;	1023	struct i915_sw_fence *submit;
1001	struct i915_request *rq;	1024	struct i915_request *rq;
1002	u32 hwsp;	1025	u32 hwsp, dummy;
1003		1026
1004	submit = heap_fence_create(GFP_KERNEL);	1027	submit = heap_fence_create(GFP_KERNEL);
1005	if (!submit) {	1028	if (!submit) {
@@ -1017,14 +1040,26 @@ static int live_hwsp_read(void *arg) · 11 unmodified lines
1017	goto out;	1040	goto out;
1018	}	1041	}
1019		1042
1020	/* Skip to the end, saving 30 minutes of nops */
1021	tl->seqno = -10u + 2 * (count & 3);
1022	WRITE_ONCE((u32 )tl->hwsp_seqno, tl->seqno);
1023	ce->timeline = intel_timeline_get(tl);	1043	ce->timeline = intel_timeline_get(tl);
1024		1044
1025	rq = intel_context_create_request(ce);	1045	/* Ensure timeline is mapped, done during first pin */
		1046	err = intel_context_pin(ce);
		1047	if (err) {
		1048	intel_context_put(ce);
		1049	goto out;
		1050	}
		1051
		1052	/*
		1053	* Start at a new wrap, and set seqno right before another wrap,
		1054	* saving 30 minutes of nops
		1055	*/
		1056	tl->seqno = -12u + 2 * (count & 3);
		1057	__intel_timeline_get_seqno(tl, &dummy);
		1058
		1059	rq = i915_request_create(ce);
1026	if (IS_ERR(rq)) {	1060	if (IS_ERR(rq)) {
1027	err = PTR_ERR(rq);	1061	err = PTR_ERR(rq);
		1062	intel_context_unpin(ce);
1028	intel_context_put(ce);	1063	intel_context_put(ce);
1029	goto out;	1064	goto out;
1030	}	1065	}
@@ -1034,32 +1069,35 @@ static int live_hwsp_read(void *arg) · 3 unmodified lines
1034	GFP_KERNEL);	1069	GFP_KERNEL);
1035	if (err < 0) {	1070	if (err < 0) {
1036	i915_request_add(rq);	1071	i915_request_add(rq);
		1072	intel_context_unpin(ce);
1037	intel_context_put(ce);	1073	intel_context_put(ce);
1038	goto out;	1074	goto out;
1039	}	1075	}
1040		1076
1041	mutex_lock(&watcher[0].rq->context->timeline->mutex);	1077	switch_tl_lock(rq, watcher[0].rq);
1042	err = intel_timeline_read_hwsp(rq, watcher[0].rq, &hwsp);	1078	err = intel_timeline_read_hwsp(rq, watcher[0].rq, &hwsp);
1043	if (err == 0)	1079	if (err == 0)
1044	err = emit_read_hwsp(watcher[0].rq, /* before */	1080	err = emit_read_hwsp(watcher[0].rq, /* before */
1045	rq->fence.seqno, hwsp,	1081	rq->fence.seqno, hwsp,
1046	&watcher[0].addr);	1082	&watcher[0].addr);
1047	mutex_unlock(&watcher[0].rq->context->timeline->mutex);	1083	switch_tl_lock(watcher[0].rq, rq);
1048	if (err) {	1084	if (err) {
1049	i915_request_add(rq);	1085	i915_request_add(rq);
		1086	intel_context_unpin(ce);
1050	intel_context_put(ce);	1087	intel_context_put(ce);
1051	goto out;	1088	goto out;
1052	}	1089	}
1053		1090
1054	mutex_lock(&watcher[1].rq->context->timeline->mutex);	1091	switch_tl_lock(rq, watcher[1].rq);
1055	err = intel_timeline_read_hwsp(rq, watcher[1].rq, &hwsp);	1092	err = intel_timeline_read_hwsp(rq, watcher[1].rq, &hwsp);
1056	if (err == 0)	1093	if (err == 0)
1057	err = emit_read_hwsp(watcher[1].rq, /* after */	1094	err = emit_read_hwsp(watcher[1].rq, /* after */
1058	rq->fence.seqno, hwsp,	1095	rq->fence.seqno, hwsp,
1059	&watcher[1].addr);	1096	&watcher[1].addr);
1060	mutex_unlock(&watcher[1].rq->context->timeline->mutex);	1097	switch_tl_lock(watcher[1].rq, rq);
1061	if (err) {	1098	if (err) {
1062	i915_request_add(rq);	1099	i915_request_add(rq);
		1100	intel_context_unpin(ce);
1063	intel_context_put(ce);	1101	intel_context_put(ce);
1064	goto out;	1102	goto out;
1065	}	1103	}
@@ -1068,6 +1106,7 @@ static int live_hwsp_read(void *arg) · 2 unmodified lines
1068	i915_request_add(rq);	1106	i915_request_add(rq);
1069		1107
1070	rq = wrap_timeline(rq);	1108	rq = wrap_timeline(rq);
		1109	intel_context_unpin(ce);
1071	intel_context_put(ce);	1110	intel_context_put(ce);
1072	if (IS_ERR(rq)) {	1111	if (IS_ERR(rq)) {
1073	err = PTR_ERR(rq);	1112	err = PTR_ERR(rq);
@@ -1107,8 +1146,8 @@ static int live_hwsp_read(void *arg) · 33 unmodified lines
1107	3 * watcher[1].rq->ring->size)	1146	3 * watcher[1].rq->ring->size)
1108	break;	1147	break;
1109		1148
1110	} while (!__igt_timeout(end_time, NULL));	1149	} while (!__igt_timeout(end_time, NULL) &&
1111	WRITE_ONCE((u32 )tl->hwsp_seqno, 0xdeadbeef);	1150	count < (PAGE_SIZE / TIMELINE_SEQNO_BYTES - 1) / 2);
1112		1151
1113	pr_info("%s: simulated %lu wraps\n", engine->name, count);	1152	pr_info("%s: simulated %lu wraps\n", engine->name, count);
1114	err = check_watcher(&watcher[1], "after", cmp_gte);	1153	err = check_watcher(&watcher[1], "after", cmp_gte);
@@ -1153,9 +1192,7 @@ static int live_hwsp_rollover_kernel(void *arg) · 38 unmodified lines
1153	}	1192	}
1154		1193
1155	GEM_BUG_ON(i915_active_fence_isset(&tl->last_request));	1194	GEM_BUG_ON(i915_active_fence_isset(&tl->last_request));
1156	tl->seqno = 0;	1195	tl->seqno = -2u;
1157	timeline_rollback(tl);
1158	timeline_rollback(tl);
1159	WRITE_ONCE((u32 )tl->hwsp_seqno, tl->seqno);	1196	WRITE_ONCE((u32 )tl->hwsp_seqno, tl->seqno);
1160		1197
1161	for (i = 0; i < ARRAY_SIZE(rq); i++) {	1198	for (i = 0; i < ARRAY_SIZE(rq); i++) {
@@ -1235,11 +1272,14 @@ static int live_hwsp_rollover_user(void *arg) · 73 unmodified lines
1235	goto out;	1272	goto out;
1236		1273
1237	tl = ce->timeline;	1274	tl = ce->timeline;
1238	if (!tl->has_initial_breadcrumb \|\| !tl->hwsp_cacheline)	1275	if (!tl->has_initial_breadcrumb)
		1276	goto out;
		1277
		1278	err = intel_context_pin(ce);
		1279	if (err)
1239	goto out;	1280	goto out;
1240		1281
1241	timeline_rollback(tl);	1282	tl->seqno = -4u;
1242	timeline_rollback(tl);
1243	WRITE_ONCE((u32 )tl->hwsp_seqno, tl->seqno);	1283	WRITE_ONCE((u32 )tl->hwsp_seqno, tl->seqno);
1244		1284
1245	for (i = 0; i < ARRAY_SIZE(rq); i++) {	1285	for (i = 0; i < ARRAY_SIZE(rq); i++) {
@@ -1248,7 +1288,7 @@ static int live_hwsp_rollover_user(void *arg) · 2 unmodified lines
1248	this = intel_context_create_request(ce);	1288	this = intel_context_create_request(ce);
1249	if (IS_ERR(this)) {	1289	if (IS_ERR(this)) {
1250	err = PTR_ERR(this);	1290	err = PTR_ERR(this);
1251	goto out;	1291	goto out_unpin;
1252	}	1292	}
1253		1293
1254	pr_debug("%s: create fence.seqnp:%d\n",	1294	pr_debug("%s: create fence.seqnp:%d\n",
@@ -1267,17 +1307,18 @@ static int live_hwsp_rollover_user(void *arg) · 12 unmodified lines
1267	if (i915_request_wait(rq[2], 0, HZ / 5) < 0) {	1307	if (i915_request_wait(rq[2], 0, HZ / 5) < 0) {
1268	pr_err("Wait for timeline wrap timed out!\n");	1308	pr_err("Wait for timeline wrap timed out!\n");
1269	err = -EIO;	1309	err = -EIO;
1270	goto out;	1310	goto out_unpin;
1271	}	1311	}
1272		1312
1273	for (i = 0; i < ARRAY_SIZE(rq); i++) {	1313	for (i = 0; i < ARRAY_SIZE(rq); i++) {
1274	if (!i915_request_completed(rq[i])) {	1314	if (!i915_request_completed(rq[i])) {
1275	pr_err("Pre-wrap request not completed!\n");	1315	pr_err("Pre-wrap request not completed!\n");
1276	err = -EINVAL;	1316	err = -EINVAL;
1277	goto out;	1317	goto out_unpin;
1278	}	1318	}
1279	}	1319	}
1280		1320	out_unpin:
		1321	intel_context_unpin(ce);
1281	out:	1322	out:
1282	for (i = 0; i < ARRAY_SIZE(rq); i++)	1323	for (i = 0; i < ARRAY_SIZE(rq); i++)
1283	i915_request_put(rq[i]);	1324	i915_request_put(rq[i]);
@@ -1319,13 +1360,13 @@ static int live_hwsp_recycle(void *arg) · 35 unmodified lines
1319	struct intel_timeline *tl;	1360	struct intel_timeline *tl;
1320	struct i915_request *rq;	1361	struct i915_request *rq;
1321		1362
1322	tl = checked_intel_timeline_create(gt);	1363	tl = intel_timeline_create(gt);
1323	if (IS_ERR(tl)) {	1364	if (IS_ERR(tl)) {
1324	err = PTR_ERR(tl);	1365	err = PTR_ERR(tl);
1325	break;	1366	break;
1326	}	1367	}
1327		1368
1328	rq = tl_write(tl, engine, count);	1369	rq = checked_tl_write(tl, engine, count);
1329	if (IS_ERR(rq)) {	1370	if (IS_ERR(rq)) {
1330	intel_timeline_put(tl);	1371	intel_timeline_put(tl);
1331	err = PTR_ERR(rq);	1372	err = PTR_ERR(rq);

drivers/gpu/drm/i915/gt/selftest_workarounds.c+52−30

@@ -112,7 +112,7 @@ read_nonprivs(struct intel_context *ce) · 111 unmodified lines
112		112
113	i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);	113	i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
114		114
115	cs = i915_gem_object_pin_map(result, I915_MAP_WB);	115	cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB);
116	if (IS_ERR(cs)) {	116	if (IS_ERR(cs)) {
117	err = PTR_ERR(cs);	117	err = PTR_ERR(cs);
118	goto err_obj;	118	goto err_obj;
@@ -218,7 +218,7 @@ static int check_whitelist(struct intel_context *ce) · 99 unmodified lines
218	i915_gem_object_lock(results, NULL);	218	i915_gem_object_lock(results, NULL);
219	intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */	219	intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */
220	err = i915_gem_object_set_to_cpu_domain(results, false);	220	err = i915_gem_object_set_to_cpu_domain(results, false);
221	i915_gem_object_unlock(results);	221
222	if (intel_gt_is_wedged(engine->gt))	222	if (intel_gt_is_wedged(engine->gt))
223	err = -EIO;	223	err = -EIO;
224	if (err)	224	if (err)
@@ -246,6 +246,7 @@ static int check_whitelist(struct intel_context *ce) · 21 unmodified lines
246		246
247	i915_gem_object_unpin_map(results);	247	i915_gem_object_unpin_map(results);
248	out_put:	248	out_put:
		249	i915_gem_object_unlock(results);
249	i915_gem_object_put(results);	250	i915_gem_object_put(results);
250	return err;	251	return err;
251	}	252	}
@@ -490,7 +491,7 @@ static int check_dirty_whitelist(struct intel_context *ce) · 238 unmodified lines
490	u32 cs, results;	491	u32 cs, results;
491		492
492	sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);	493	sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);
493	scratch = __vm_create_scratch_for_read(ce->vm, sz);	494	scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz);
494	if (IS_ERR(scratch))	495	if (IS_ERR(scratch))
495	return PTR_ERR(scratch);	496	return PTR_ERR(scratch);
496		497
@@ -502,6 +503,7 @@ static int check_dirty_whitelist(struct intel_context *ce) · 5 unmodified lines
502		503
503	for (i = 0; i < engine->whitelist.count; i++) {	504	for (i = 0; i < engine->whitelist.count; i++) {
504	u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);	505	u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
		506	struct i915_gem_ww_ctx ww;
505	u64 addr = scratch->node.start;	507	u64 addr = scratch->node.start;
506	struct i915_request *rq;	508	struct i915_request *rq;
507	u32 srm, lrm, rsvd;	509	u32 srm, lrm, rsvd;
@@ -517,6 +519,29 @@ static int check_dirty_whitelist(struct intel_context *ce) · 9 unmodified lines
517		519
518	ro_reg = ro_register(reg);	520	ro_reg = ro_register(reg);
519		521
		522	i915_gem_ww_ctx_init(&ww, false);
		523	retry:
		524	cs = NULL;
		525	err = i915_gem_object_lock(scratch->obj, &ww);
		526	if (!err)
		527	err = i915_gem_object_lock(batch->obj, &ww);
		528	if (!err)
		529	err = intel_context_pin_ww(ce, &ww);
		530	if (err)
		531	goto out;
		532
		533	cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
		534	if (IS_ERR(cs)) {
		535	err = PTR_ERR(cs);
		536	goto out_ctx;
		537	}
		538
		539	results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
		540	if (IS_ERR(results)) {
		541	err = PTR_ERR(results);
		542	goto out_unmap_batch;
		543	}
		544
520	/* Clear non priv flags */	545	/* Clear non priv flags */
521	reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;	546	reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
522		547
@@ -528,12 +553,6 @@ static int check_dirty_whitelist(struct intel_context *ce) · 5 unmodified lines
528	pr_debug("%s: Writing garbage to %x\n",	553	pr_debug("%s: Writing garbage to %x\n",
529	engine->name, reg);	554	engine->name, reg);
530		555
531	cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
532	if (IS_ERR(cs)) {
533	err = PTR_ERR(cs);
534	goto out_batch;
535	}
536
537	/* SRM original */	556	/* SRM original */
538	*cs++ = srm;	557	*cs++ = srm;
539	*cs++ = reg;	558	*cs++ = reg;
@@ -580,11 +599,12 @@ static int check_dirty_whitelist(struct intel_context *ce) · 40 unmodified lines
580	i915_gem_object_flush_map(batch->obj);	599	i915_gem_object_flush_map(batch->obj);
581	i915_gem_object_unpin_map(batch->obj);	600	i915_gem_object_unpin_map(batch->obj);
582	intel_gt_chipset_flush(engine->gt);	601	intel_gt_chipset_flush(engine->gt);
		602	cs = NULL;
583		603
584	rq = intel_context_create_request(ce);	604	rq = i915_request_create(ce);
585	if (IS_ERR(rq)) {	605	if (IS_ERR(rq)) {
586	err = PTR_ERR(rq);	606	err = PTR_ERR(rq);
587	goto out_batch;	607	goto out_unmap_scratch;
588	}	608	}
589		609
590	if (engine->emit_init_breadcrumb) { /* Be nice if we hang */	610	if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
@@ -593,20 +613,16 @@ static int check_dirty_whitelist(struct intel_context *ce) · 2 unmodified lines
593	goto err_request;	613	goto err_request;
594	}	614	}
595		615
596	i915_vma_lock(batch);
597	err = i915_request_await_object(rq, batch->obj, false);	616	err = i915_request_await_object(rq, batch->obj, false);
598	if (err == 0)	617	if (err == 0)
599	err = i915_vma_move_to_active(batch, rq, 0);	618	err = i915_vma_move_to_active(batch, rq, 0);
600	i915_vma_unlock(batch);
601	if (err)	619	if (err)
602	goto err_request;	620	goto err_request;
603		621
604	i915_vma_lock(scratch);
605	err = i915_request_await_object(rq, scratch->obj, true);	622	err = i915_request_await_object(rq, scratch->obj, true);
606	if (err == 0)	623	if (err == 0)
607	err = i915_vma_move_to_active(scratch, rq,	624	err = i915_vma_move_to_active(scratch, rq,
608	EXEC_OBJECT_WRITE);	625	EXEC_OBJECT_WRITE);
609	i915_vma_unlock(scratch);
610	if (err)	626	if (err)
611	goto err_request;	627	goto err_request;
612		628
@@ -622,13 +638,7 @@ err_request: · 9 unmodified lines
622	pr_err("%s: Futzing %x timedout; cancelling test\n",	638	pr_err("%s: Futzing %x timedout; cancelling test\n",
623	engine->name, reg);	639	engine->name, reg);
624	intel_gt_set_wedged(engine->gt);	640	intel_gt_set_wedged(engine->gt);
625	goto out_batch;	641	goto out_unmap_scratch;
626	}
627
628	results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
629	if (IS_ERR(results)) {
630	err = PTR_ERR(results);
631	goto out_batch;
632	}	642	}
633		643
634	GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);	644	GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
@@ -639,7 +649,7 @@ err_request: · 4 unmodified lines
639	pr_err("%s: Unable to write to whitelisted register %x\n",	649	pr_err("%s: Unable to write to whitelisted register %x\n",
640	engine->name, reg);	650	engine->name, reg);
641	err = -EINVAL;	651	err = -EINVAL;
642	goto out_unpin;	652	goto out_unmap_scratch;
643	}	653	}
644	} else {	654	} else {
645	rsvd = 0;	655	rsvd = 0;
@@ -705,15 +715,27 @@ err_request: · 59 unmodified lines
705		715
706	err = -EINVAL;	716	err = -EINVAL;
707	}	717	}
708	out_unpin:	718	out_unmap_scratch:
709	i915_gem_object_unpin_map(scratch->obj);	719	i915_gem_object_unpin_map(scratch->obj);
		720	out_unmap_batch:
		721	if (cs)
		722	i915_gem_object_unpin_map(batch->obj);
		723	out_ctx:
		724	intel_context_unpin(ce);
		725	out:
		726	if (err == -EDEADLK) {
		727	err = i915_gem_ww_ctx_backoff(&ww);
		728	if (!err)
		729	goto retry;
		730	}
		731	i915_gem_ww_ctx_fini(&ww);
710	if (err)	732	if (err)
711	break;	733	break;
712	}	734	}
713		735
714	if (igt_flush_test(engine->i915))	736	if (igt_flush_test(engine->i915))
715	err = -EIO;	737	err = -EIO;
716	out_batch:	738
717	i915_vma_unpin_and_release(&batch, 0);	739	i915_vma_unpin_and_release(&batch, 0);
718	out_scratch:	740	out_scratch:
719	i915_vma_unpin_and_release(&scratch, 0);	741	i915_vma_unpin_and_release(&scratch, 0);
@@ -847,7 +869,7 @@ static int scrub_whitelisted_registers(struct intel_context *ce) · 127 unmodified lines
847	if (IS_ERR(batch))	869	if (IS_ERR(batch))
848	return PTR_ERR(batch);	870	return PTR_ERR(batch);
849		871
850	cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);	872	cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
851	if (IS_ERR(cs)) {	873	if (IS_ERR(cs)) {
852	err = PTR_ERR(cs);	874	err = PTR_ERR(cs);
853	goto err_batch;	875	goto err_batch;
@@ -982,11 +1004,11 @@ check_whitelisted_registers(struct intel_engine_cs *engine, · 128 unmodified lines
982	u32 a, b;	1004	u32 a, b;
983	int i, err;	1005	int i, err;
984		1006
985	a = i915_gem_object_pin_map(A->obj, I915_MAP_WB);	1007	a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB);
986	if (IS_ERR(a))	1008	if (IS_ERR(a))
987	return PTR_ERR(a);	1009	return PTR_ERR(a);
988		1010
989	b = i915_gem_object_pin_map(B->obj, I915_MAP_WB);	1011	b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB);
990	if (IS_ERR(b)) {	1012	if (IS_ERR(b)) {
991	err = PTR_ERR(b);	1013	err = PTR_ERR(b);
992	goto err_a;	1014	goto err_a;
@@ -1030,14 +1052,14 @@ static int live_isolated_whitelist(void *arg) · 37 unmodified lines
1030		1052
1031	for (i = 0; i < ARRAY_SIZE(client); i++) {	1053	for (i = 0; i < ARRAY_SIZE(client); i++) {
1032	client[i].scratch[0] =	1054	client[i].scratch[0] =
1033	__vm_create_scratch_for_read(gt->vm, 4096);	1055	__vm_create_scratch_for_read_pinned(gt->vm, 4096);
1034	if (IS_ERR(client[i].scratch[0])) {	1056	if (IS_ERR(client[i].scratch[0])) {
1035	err = PTR_ERR(client[i].scratch[0]);	1057	err = PTR_ERR(client[i].scratch[0]);
1036	goto err;	1058	goto err;
1037	}	1059	}
1038		1060
1039	client[i].scratch[1] =	1061	client[i].scratch[1] =
1040	__vm_create_scratch_for_read(gt->vm, 4096);	1062	__vm_create_scratch_for_read_pinned(gt->vm, 4096);
1041	if (IS_ERR(client[i].scratch[1])) {	1063	if (IS_ERR(client[i].scratch[1])) {
1042	err = PTR_ERR(client[i].scratch[1]);	1064	err = PTR_ERR(client[i].scratch[1]);
1043	i915_vma_unpin_and_release(&client[i].scratch[0], 0);	1065	i915_vma_unpin_and_release(&client[i].scratch[0], 0);

drivers/gpu/drm/i915/gt/shmem_utils.c+1−1

@@ -39,7 +39,7 @@ struct file shmem_create_from_object(struct drm_i915_gem_object obj) · 38 unmodified lines
39	return file;	39	return file;
40	}	40	}
41		41
42	ptr = i915_gem_object_pin_map(obj, I915_MAP_WB);	42	ptr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
43	if (IS_ERR(ptr))	43	if (IS_ERR(ptr))
44	return ERR_CAST(ptr);	44	return ERR_CAST(ptr);
45		45

drivers/gpu/drm/i915/gt/uc/intel_guc.c+1−1

@@ -682,7 +682,7 @@ int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size, · 681 unmodified lines
682	if (IS_ERR(vma))	682	if (IS_ERR(vma))
683	return PTR_ERR(vma);	683	return PTR_ERR(vma);
684		684
685	vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);	685	vaddr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WB);
686	if (IS_ERR(vaddr)) {	686	if (IS_ERR(vaddr)) {
687	i915_vma_unpin_and_release(&vma, 0);	687	i915_vma_unpin_and_release(&vma, 0);
688	return PTR_ERR(vaddr);	688	return PTR_ERR(vaddr);

drivers/gpu/drm/i915/gt/uc/intel_guc_log.c+2−2

@@ -335,7 +335,7 @@ static int guc_log_map(struct intel_guc_log *log) · 334 unmodified lines
335	* buffer pages, so that we can directly get the data	335	* buffer pages, so that we can directly get the data
336	* (up-to-date) from memory.	336	* (up-to-date) from memory.
337	*/	337	*/
338	vaddr = i915_gem_object_pin_map(log->vma->obj, I915_MAP_WC);	338	vaddr = i915_gem_object_pin_map_unlocked(log->vma->obj, I915_MAP_WC);
339	if (IS_ERR(vaddr))	339	if (IS_ERR(vaddr))
340	return PTR_ERR(vaddr);	340	return PTR_ERR(vaddr);
341		341
@@ -744,7 +744,7 @@ int intel_guc_log_dump(struct intel_guc_log log, struct drm_printer p, · 402 unmodified lines
744	if (!obj)	744	if (!obj)
745	return 0;	745	return 0;
746		746
747	map = i915_gem_object_pin_map(obj, I915_MAP_WC);	747	map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
748	if (IS_ERR(map)) {	748	if (IS_ERR(map)) {
749	DRM_DEBUG("Failed to pin object\n");	749	DRM_DEBUG("Failed to pin object\n");
750	drm_puts(p, "(log data unaccessible)\n");	750	drm_puts(p, "(log data unaccessible)\n");

drivers/gpu/drm/i915/gt/uc/intel_huc.c+1−1

@@ -82,7 +82,7 @@ static int intel_huc_rsa_data_create(struct intel_huc *huc) · 81 unmodified lines
82	if (IS_ERR(vma))	82	if (IS_ERR(vma))
83	return PTR_ERR(vma);	83	return PTR_ERR(vma);
84		84
85	vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);	85	vaddr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WB);
86	if (IS_ERR(vaddr)) {	86	if (IS_ERR(vaddr)) {
87	i915_vma_unpin_and_release(&vma, 0);	87	i915_vma_unpin_and_release(&vma, 0);
88	return PTR_ERR(vaddr);	88	return PTR_ERR(vaddr);

drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c+1−1

@@ -539,7 +539,7 @@ int intel_uc_fw_init(struct intel_uc_fw *uc_fw) · 538 unmodified lines
539	if (!intel_uc_fw_is_available(uc_fw))	539	if (!intel_uc_fw_is_available(uc_fw))
540	return -ENOEXEC;	540	return -ENOEXEC;
541		541
542	err = i915_gem_object_pin_pages(uc_fw->obj);	542	err = i915_gem_object_pin_pages_unlocked(uc_fw->obj);
543	if (err) {	543	if (err) {
544	DRM_DEBUG_DRIVER("%s fw pin-pages err=%d\n",	544	DRM_DEBUG_DRIVER("%s fw pin-pages err=%d\n",
545	intel_uc_fw_type_repr(uc_fw->type), err);	545	intel_uc_fw_type_repr(uc_fw->type), err);

drivers/gpu/drm/i915/gvt/dmabuf.c+1−1

@@ -218,7 +218,7 @@ static struct drm_i915_gem_object vgpu_create_gem(struct drm_device dev, · 217 unmodified lines
218		218
219	drm_gem_private_object_init(dev, &obj->base,	219	drm_gem_private_object_init(dev, &obj->base,
220	roundup(info->size, PAGE_SIZE));	220	roundup(info->size, PAGE_SIZE));
221	i915_gem_object_init(obj, &intel_vgpu_gem_ops, &lock_class);	221	i915_gem_object_init(obj, &intel_vgpu_gem_ops, &lock_class, 0);
222	i915_gem_object_set_readonly(obj);	222	i915_gem_object_set_readonly(obj);
223		223
224	obj->read_domains = I915_GEM_DOMAIN_GTT;	224	obj->read_domains = I915_GEM_DOMAIN_GTT;

drivers/gpu/drm/i915/i915_active.c+10−10

 static struct i915_active_fence *
 active_instance(struct i915_active *ref, u64 idx)
 {
-	struct active_node *node, *prealloc;
+	struct active_node *node;
 	struct rb_node **p, *parent;
 	node = __active_lookup(ref, idx);
 	if (likely(node))
 		return &node->base;
-	/* Preallocate a replacement, just in case */
-	prealloc = kmem_cache_alloc(global.slab_cache, GFP_KERNEL);
-	if (!prealloc)
-		return NULL;
 	spin_lock_irq(&ref->tree_lock);
 	GEM_BUG_ON(i915_active_is_idle(ref));
 		parent = *p;
 		node = rb_entry(parent, struct active_node, node);
-		if (node->timeline == idx) {
+		if (node->timeline == idx)
-			kmem_cache_free(global.slab_cache, prealloc);
 			goto out;
-		}
 		if (node->timeline < idx)
 			p = &parent->rb_right;
 			p = &parent->rb_left;
 	}
-	node = prealloc;
+	/*
+	 * XXX: We should preallocate this before i915_active_ref() is ever
+	 *  called, but we cannot call into fs_reclaim() anyway, so use GFP_ATOMIC.
+	 */
+	node = kmem_cache_alloc(global.slab_cache, GFP_ATOMIC);
+	if (!node)
+		goto out;
 	__i915_active_fence_init(&node->base, NULL, node_retire);
 	node->ref = ref;
 	node->timeline = idx;

drivers/gpu/drm/i915/i915_cmd_parser.c+43−61

@@ -1144,38 +1144,20 @@ find_reg(const struct intel_engine_cs *engine, u32 addr) · 1143 unmodified lines
1144	/* Returns a vmap'd pointer to dst_obj, which the caller must unmap */	1144	/* Returns a vmap'd pointer to dst_obj, which the caller must unmap */
1145	static u32 copy_batch(struct drm_i915_gem_object dst_obj,	1145	static u32 copy_batch(struct drm_i915_gem_object dst_obj,
1146	struct drm_i915_gem_object *src_obj,	1146	struct drm_i915_gem_object *src_obj,
1147	unsigned long offset, unsigned long length)	1147	unsigned long offset, unsigned long length,
		1148	void dst, const void src)
1148	{	1149	{
1149	bool needs_clflush;	1150	bool needs_clflush =
1150	void dst, src;
1151	int ret;
1152
1153	dst = i915_gem_object_pin_map(dst_obj, I915_MAP_WB);
1154	if (IS_ERR(dst))
1155	return dst;
1156
1157	ret = i915_gem_object_pin_pages(src_obj);
1158	if (ret) {
1159	i915_gem_object_unpin_map(dst_obj);
1160	return ERR_PTR(ret);
1161	}
1162
1163	needs_clflush =
1164	!(src_obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ);	1151	!(src_obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ);
1165		1152
1166	src = ERR_PTR(-ENODEV);	1153	if (src) {
1167	if (needs_clflush && i915_has_memcpy_from_wc()) {	1154	GEM_BUG_ON(!needs_clflush);
1168	src = i915_gem_object_pin_map(src_obj, I915_MAP_WC);	1155	i915_unaligned_memcpy_from_wc(dst, src + offset, length);
1169	if (!IS_ERR(src)) {	1156	} else {
1170	i915_unaligned_memcpy_from_wc(dst,	1157	struct scatterlist *sg;
1171	src + offset,
1172	length);
1173	i915_gem_object_unpin_map(src_obj);
1174	}
1175	}
1176	if (IS_ERR(src)) {
1177	unsigned long x, n, remain;
1178	void *ptr;	1158	void *ptr;
		1159	unsigned int x, sg_ofs;
		1160	unsigned long remain;
1179		1161
1180	/*	1162	/*
1181	* We can avoid clflushing partial cachelines before the write	1163	* We can avoid clflushing partial cachelines before the write
@@ -1192,23 +1174,31 @@ static u32 copy_batch(struct drm_i915_gem_object dst_obj, · 10 unmodified lines
1192		1174
1193	ptr = dst;	1175	ptr = dst;
1194	x = offset_in_page(offset);	1176	x = offset_in_page(offset);
1195	for (n = offset >> PAGE_SHIFT; remain; n++) {	1177	sg = i915_gem_object_get_sg(src_obj, offset >> PAGE_SHIFT, &sg_ofs, false);
1196	int len = min(remain, PAGE_SIZE - x);	1178
1197		1179	while (remain) {
1198	src = kmap_atomic(i915_gem_object_get_page(src_obj, n));	1180	unsigned long sg_max = sg->length >> PAGE_SHIFT;
1199	if (needs_clflush)	1181
1200	drm_clflush_virt_range(src + x, len);	1182	for (; remain && sg_ofs < sg_max; sg_ofs++) {
1201	memcpy(ptr, src + x, len);	1183	unsigned long len = min(remain, PAGE_SIZE - x);
1202	kunmap_atomic(src);	1184	void *map;
1203		1185
1204	ptr += len;	1186	map = kmap_atomic(nth_page(sg_page(sg), sg_ofs));
1205	remain -= len;	1187	if (needs_clflush)
1206	x = 0;	1188	drm_clflush_virt_range(map + x, len);
		1189	memcpy(ptr, map + x, len);
		1190	kunmap_atomic(map);
		1191
		1192	ptr += len;
		1193	remain -= len;
		1194	x = 0;
		1195	}
		1196
		1197	sg_ofs = 0;
		1198	sg = sg_next(sg);
1207	}	1199	}
1208	}	1200	}
1209		1201
1210	i915_gem_object_unpin_pages(src_obj);
1211
1212	memset32(dst + length, 0, (dst_obj->base.size - length) / sizeof(u32));	1202	memset32(dst + length, 0, (dst_obj->base.size - length) / sizeof(u32));
1213		1203
1214	/* dst_obj is returned with vmap pinned */	1204	/* dst_obj is returned with vmap pinned */
@@ -1370,9 +1360,6 @@ static int check_bbstart(u32 *cmd, u32 offset, u32 length, · 155 unmodified lines
1370	if (target_cmd_index == offset)	1360	if (target_cmd_index == offset)
1371	return 0;	1361	return 0;
1372		1362
1373	if (IS_ERR(jump_whitelist))
1374	return PTR_ERR(jump_whitelist);
1375
1376	if (!test_bit(target_cmd_index, jump_whitelist)) {	1363	if (!test_bit(target_cmd_index, jump_whitelist)) {
1377	DRM_DEBUG("CMD: BB_START to 0x%llx not a previously executed cmd\n",	1364	DRM_DEBUG("CMD: BB_START to 0x%llx not a previously executed cmd\n",
1378	jump_target);	1365	jump_target);
@@ -1382,10 +1369,14 @@ static int check_bbstart(u32 *cmd, u32 offset, u32 length, · 3 unmodified lines
1382	return 0;	1369	return 0;
1383	}	1370	}
1384		1371
1385	static unsigned long *alloc_whitelist(u32 batch_length)	1372	unsigned long *intel_engine_cmd_parser_alloc_jump_whitelist(u32 batch_length,
		1373	bool trampoline)
1386	{	1374	{
1387	unsigned long *jmp;	1375	unsigned long *jmp;
1388		1376
		1377	if (trampoline)
		1378	return NULL;
		1379
1389	/*	1380	/*
1390	* We expect batch_length to be less than 256KiB for known users,	1381	* We expect batch_length to be less than 256KiB for known users,
1391	* i.e. we need at most an 8KiB bitmap allocation which should be	1382	* i.e. we need at most an 8KiB bitmap allocation which should be
@@ -1423,14 +1414,16 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, · 31 unmodified lines
1423	unsigned long batch_offset,	1414	unsigned long batch_offset,
1424	unsigned long batch_length,	1415	unsigned long batch_length,
1425	struct i915_vma *shadow,	1416	struct i915_vma *shadow,
1426	bool trampoline)	1417	unsigned long *jump_whitelist,
		1418	void *shadow_map,
		1419	const void *batch_map)
1427	{	1420	{
1428	u32 cmd, batch_end, offset = 0;	1421	u32 cmd, batch_end, offset = 0;
1429	struct drm_i915_cmd_descriptor default_desc = noop_desc;	1422	struct drm_i915_cmd_descriptor default_desc = noop_desc;
1430	const struct drm_i915_cmd_descriptor *desc = &default_desc;	1423	const struct drm_i915_cmd_descriptor *desc = &default_desc;
1431	unsigned long *jump_whitelist;
1432	u64 batch_addr, shadow_addr;	1424	u64 batch_addr, shadow_addr;
1433	int ret = 0;	1425	int ret = 0;
		1426	bool trampoline = !jump_whitelist;
1434		1427
1435	GEM_BUG_ON(!IS_ALIGNED(batch_offset, sizeof(*cmd)));	1428	GEM_BUG_ON(!IS_ALIGNED(batch_offset, sizeof(*cmd)));
1436	GEM_BUG_ON(!IS_ALIGNED(batch_length, sizeof(*cmd)));	1429	GEM_BUG_ON(!IS_ALIGNED(batch_length, sizeof(*cmd)));
@@ -1438,16 +1431,8 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, · 1 unmodified line
1438	batch->size));	1431	batch->size));
1439	GEM_BUG_ON(!batch_length);	1432	GEM_BUG_ON(!batch_length);
1440		1433
1441	cmd = copy_batch(shadow->obj, batch->obj, batch_offset, batch_length);	1434	cmd = copy_batch(shadow->obj, batch->obj, batch_offset, batch_length,
1442	if (IS_ERR(cmd)) {	1435	shadow_map, batch_map);
1443	DRM_DEBUG("CMD: Failed to copy batch\n");
1444	return PTR_ERR(cmd);
1445	}
1446
1447	jump_whitelist = NULL;
1448	if (!trampoline)
1449	/* Defer failure until attempted use */
1450	jump_whitelist = alloc_whitelist(batch_length);
1451		1436
1452	shadow_addr = gen8_canonical_addr(shadow->node.start);	1437	shadow_addr = gen8_canonical_addr(shadow->node.start);
1453	batch_addr = gen8_canonical_addr(batch->node.start + batch_offset);	1438	batch_addr = gen8_canonical_addr(batch->node.start + batch_offset);
@@ -1548,9 +1533,6 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, · 94 unmodified lines
1548		1533
1549	i915_gem_object_flush_map(shadow->obj);	1534	i915_gem_object_flush_map(shadow->obj);
1550		1535
1551	if (!IS_ERR_OR_NULL(jump_whitelist))
1552	kfree(jump_whitelist);
1553	i915_gem_object_unpin_map(shadow->obj);
1554	return ret;	1536	return ret;
1555	}	1537	}
1556		1538

drivers/gpu/drm/i915/i915_debugfs.c+2−2

@@ -904,10 +904,10 @@ i915_drop_caches_set(void *data, u64 val) · 903 unmodified lines
904		904
905	fs_reclaim_acquire(GFP_KERNEL);	905	fs_reclaim_acquire(GFP_KERNEL);
906	if (val & DROP_BOUND)	906	if (val & DROP_BOUND)
907	i915_gem_shrink(i915, LONG_MAX, NULL, I915_SHRINK_BOUND);	907	i915_gem_shrink(NULL, i915, LONG_MAX, NULL, I915_SHRINK_BOUND);
908		908
909	if (val & DROP_UNBOUND)	909	if (val & DROP_UNBOUND)
910	i915_gem_shrink(i915, LONG_MAX, NULL, I915_SHRINK_UNBOUND);	910	i915_gem_shrink(NULL, i915, LONG_MAX, NULL, I915_SHRINK_UNBOUND);
911		911
912	if (val & DROP_SHRINK_ALL)	912	if (val & DROP_SHRINK_ALL)
913	i915_gem_shrink_all(i915);	913	i915_gem_shrink_all(i915);

drivers/gpu/drm/i915/i915_drv.c+1−1

@@ -1691,7 +1691,7 @@ static const struct drm_ioctl_desc i915_ioctls[] = { · 1690 unmodified lines
1691	DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),	1691	DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
1692	DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH\|DRM_MASTER\|DRM_ROOT_ONLY),	1692	DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH\|DRM_MASTER\|DRM_ROOT_ONLY),
1693	DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH\|DRM_MASTER\|DRM_ROOT_ONLY),	1693	DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH\|DRM_MASTER\|DRM_ROOT_ONLY),
1694	DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer_ioctl, DRM_AUTH),	1694	DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, drm_invalid_op, DRM_AUTH),
1695	DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2_WR, i915_gem_execbuffer2_ioctl, DRM_RENDER_ALLOW),	1695	DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2_WR, i915_gem_execbuffer2_ioctl, DRM_RENDER_ALLOW),
1696	DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH\|DRM_ROOT_ONLY),	1696	DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH\|DRM_ROOT_ONLY),
1697	DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH\|DRM_ROOT_ONLY),	1697	DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH\|DRM_ROOT_ONLY),

drivers/gpu/drm/i915/i915_drv.h+11−7

 	struct notifier_block vmap_notifier;
 	struct shrinker shrinker;
+#ifdef CONFIG_MMU_NOTIFIER
 	/**
-	 * Workqueue to fault in userptr pages, flushed by the execbuf
+	 * notifier_lock for mmu notifiers, memory may not be allocated
-	 * when required but otherwise left to userspace to try again
+	 * while holding this lock.
-	 * on EAGAIN.
 	 */
-	struct workqueue_struct *userptr_wq;
+	spinlock_t notifier_lock;
+#endif
 	/* shrinker accounting, also useful for userland debugging */
 	u64 shrink_memory;
 	struct i915_ggtt ggtt; /* VM representing the global address space */
 	struct i915_gem_mm mm;
-	DECLARE_HASHTABLE(mm_structs, 7);
-	spinlock_t mm_lock;
 	/* Kernel Modesetting */
 int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv);
 int intel_engine_init_cmd_parser(struct intel_engine_cs *engine);
 void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine);
+unsigned long *intel_engine_cmd_parser_alloc_jump_whitelist(u32 batch_length,
+							    bool trampoline);
 int intel_engine_cmd_parser(struct intel_engine_cs *engine,
 			    struct i915_vma *batch,
 			    unsigned long batch_offset,
 			    unsigned long batch_length,
 			    struct i915_vma *shadow,
-			    bool trampoline);
+			    unsigned long *jump_whitelist,
+			    void *shadow_map,
+			    const void *batch_map);
 #define I915_CMD_PARSER_TRAMPOLINE_SIZE 8
 /* intel_device_info.c */

drivers/gpu/drm/i915/i915_gem.c+141−117

 {
 	unsigned int needs_clflush;
 	unsigned int idx, offset;
-	struct dma_fence *fence;
 	char __user *user_data;
 	u64 remain;
 	int ret;
 	if (ret)
 		return ret;
+	ret = i915_gem_object_pin_pages(obj);
+	if (ret)
+		goto err_unlock;
 	ret = i915_gem_object_prepare_read(obj, &needs_clflush);
-	if (ret) {
+	if (ret)
-		i915_gem_object_unlock(obj);
+		goto err_unpin;
-		return ret;
-	}
-	fence = i915_gem_object_lock_fence(obj);
 	i915_gem_object_finish_access(obj);
 	i915_gem_object_unlock(obj);
-	if (!fence)
-		return -ENOMEM;
 	remain = args->size;
 	user_data = u64_to_user_ptr(args->data_ptr);
 	offset = offset_in_page(args->offset);
 		offset = 0;
 	}
-	i915_gem_object_unlock_fence(obj, fence);
+	i915_gem_object_unpin_pages(obj);
+	return ret;
+err_unpin:
+	i915_gem_object_unpin_pages(obj);
+err_unlock:
+	i915_gem_object_unlock(obj);
 	return ret;
 }
 	return unwritten;
 }
-static int
+static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj,
-i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
+					     struct drm_mm_node *node,
-		   const struct drm_i915_gem_pread *args)
+					     bool write)
 {
 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 	struct i915_ggtt *ggtt = &i915->ggtt;
-	intel_wakeref_t wakeref;
-	struct drm_mm_node node;
-	struct dma_fence *fence;
-	void __user *user_data;
 	struct i915_vma *vma;
-	u64 remain, offset;
+	struct i915_gem_ww_ctx ww;
 	int ret;
-	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
+	i915_gem_ww_ctx_init(&ww, true);
+retry:
 	vma = ERR_PTR(-ENODEV);
+	ret = i915_gem_object_lock(obj, &ww);
+	if (ret)
+		goto err_ww;
+	ret = i915_gem_object_set_to_gtt_domain(obj, write);
+	if (ret)
+		goto err_ww;
 	if (!i915_gem_object_is_tiled(obj))
-		vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
+		vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
-					       PIN_MAPPABLE |
+						  PIN_MAPPABLE |
-					       PIN_NONBLOCK /* NOWARN */ |
+						  PIN_NONBLOCK /* NOWARN */ |
-					       PIN_NOEVICT);
+						  PIN_NOEVICT);
-	if (!IS_ERR(vma)) {
+	if (vma == ERR_PTR(-EDEADLK)) {
-		node.start = i915_ggtt_offset(vma);
+		ret = -EDEADLK;
-		node.flags = 0;
+		goto err_ww;
+	} else if (!IS_ERR(vma)) {
+		node->start = i915_ggtt_offset(vma);
+		node->flags = 0;
 	} else {
-		ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
+		ret = insert_mappable_node(ggtt, node, PAGE_SIZE);
 		if (ret)
-			goto out_rpm;
+			goto err_ww;
-		GEM_BUG_ON(!drm_mm_node_allocated(&node));
+		GEM_BUG_ON(!drm_mm_node_allocated(node));
+		vma = NULL;
 	}
-	ret = i915_gem_object_lock_interruptible(obj, NULL);
+	ret = i915_gem_object_pin_pages(obj);
-	if (ret)
-		goto out_unpin;
-	ret = i915_gem_object_set_to_gtt_domain(obj, false);
 	if (ret) {
-		i915_gem_object_unlock(obj);
+		if (drm_mm_node_allocated(node)) {
-		goto out_unpin;
+			ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
+			remove_mappable_node(ggtt, node);
+		} else {
+			i915_vma_unpin(vma);
+		}
 	}
-	fence = i915_gem_object_lock_fence(obj);
+err_ww:
-	i915_gem_object_unlock(obj);
+	if (ret == -EDEADLK) {
-	if (!fence) {
+		ret = i915_gem_ww_ctx_backoff(&ww);
-		ret = -ENOMEM;
+		if (!ret)
-		goto out_unpin;
+			goto retry;
+	}
+	i915_gem_ww_ctx_fini(&ww);
+	return ret ? ERR_PTR(ret) : vma;
+}
+static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj,
+				 struct drm_mm_node *node,
+				 struct i915_vma *vma)
+{
+	struct drm_i915_private *i915 = to_i915(obj->base.dev);
+	struct i915_ggtt *ggtt = &i915->ggtt;
+	i915_gem_object_unpin_pages(obj);
+	if (drm_mm_node_allocated(node)) {
+		ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
+		remove_mappable_node(ggtt, node);
+	} else {
+		i915_vma_unpin(vma);
+	}
+}
+static int
+i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
+		   const struct drm_i915_gem_pread *args)
+{
+	struct drm_i915_private *i915 = to_i915(obj->base.dev);
+	struct i915_ggtt *ggtt = &i915->ggtt;
+	intel_wakeref_t wakeref;
+	struct drm_mm_node node;
+	void __user *user_data;
+	struct i915_vma *vma;
+	u64 remain, offset;
+	int ret = 0;
+	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
+	vma = i915_gem_gtt_prepare(obj, &node, false);
+	if (IS_ERR(vma)) {
+		ret = PTR_ERR(vma);
+		goto out_rpm;
 	}
 	user_data = u64_to_user_ptr(args->data_ptr);
 		offset += page_length;
 	}
-	i915_gem_object_unlock_fence(obj, fence);
+	i915_gem_gtt_cleanup(obj, &node, vma);
-out_unpin:
-	if (drm_mm_node_allocated(&node)) {
-		ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
-		remove_mappable_node(ggtt, &node);
-	} else {
-		i915_vma_unpin(vma);
-	}
 out_rpm:
 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 	return ret;
 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 		     struct drm_file *file)
 {
+	struct drm_i915_private *i915 = to_i915(dev);
 	struct drm_i915_gem_pread *args = data;
 	struct drm_i915_gem_object *obj;
 	int ret;
+	/* PREAD is disallowed for all platforms after TGL-LP.  This also
+	 * covers all platforms with local memory.
+	 */
+	if (INTEL_GEN(i915) >= 12 && !IS_TIGERLAKE(i915))
+		return -EOPNOTSUPP;
 	if (args->size == 0)
 		return 0;
 	}
 	trace_i915_gem_object_pread(obj, args->offset, args->size);
+	ret = -ENODEV;
+	if (obj->ops->pread)
+		ret = obj->ops->pread(obj, args);
+	if (ret != -ENODEV)
+		goto out;
 	ret = -ENODEV;
 	if (obj->ops->pread)
 	if (ret)
 		goto out;
-	ret = i915_gem_object_pin_pages(obj);
-	if (ret)
-		goto out;
 	ret = i915_gem_shmem_pread(obj, args);
 	if (ret == -EFAULT || ret == -ENODEV)
 		ret = i915_gem_gtt_pread(obj, args);
-	i915_gem_object_unpin_pages(obj);
 out:
 	i915_gem_object_put(obj);
 	return ret;
 	struct intel_runtime_pm *rpm = &i915->runtime_pm;
 	intel_wakeref_t wakeref;
 	struct drm_mm_node node;
-	struct dma_fence *fence;
 	struct i915_vma *vma;
 	u64 remain, offset;
 	void __user *user_data;
-	int ret;
+	int ret = 0;
 	if (i915_gem_object_has_struct_page(obj)) {
 		/*
 		wakeref = intel_runtime_pm_get(rpm);
 	}
-	vma = ERR_PTR(-ENODEV);
+	vma = i915_gem_gtt_prepare(obj, &node, true);
-	if (!i915_gem_object_is_tiled(obj))
+	if (IS_ERR(vma)) {
-		vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
+		ret = PTR_ERR(vma);
-					       PIN_MAPPABLE |
+		goto out_rpm;
-					       PIN_NONBLOCK /* NOWARN */ |
-					       PIN_NOEVICT);
-	if (!IS_ERR(vma)) {
-		node.start = i915_ggtt_offset(vma);
-		node.flags = 0;
-	} else {
-		ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
-		if (ret)
-			goto out_rpm;
-		GEM_BUG_ON(!drm_mm_node_allocated(&node));
-	}
-	ret = i915_gem_object_lock_interruptible(obj, NULL);
-	if (ret)
-		goto out_unpin;
-	ret = i915_gem_object_set_to_gtt_domain(obj, true);
-	if (ret) {
-		i915_gem_object_unlock(obj);
-		goto out_unpin;
-	}
-	fence = i915_gem_object_lock_fence(obj);
-	i915_gem_object_unlock(obj);
-	if (!fence) {
-		ret = -ENOMEM;
-		goto out_unpin;
 	}
 	i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
 	intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
-	i915_gem_object_unlock_fence(obj, fence);
+	i915_gem_gtt_cleanup(obj, &node, vma);
-out_unpin:
-	if (drm_mm_node_allocated(&node)) {
-		ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
-		remove_mappable_node(ggtt, &node);
-	} else {
-		i915_vma_unpin(vma);
-	}
 out_rpm:
 	intel_runtime_pm_put(rpm, wakeref);
 	return ret;
 	unsigned int partial_cacheline_write;
 	unsigned int needs_clflush;
 	unsigned int offset, idx;
-	struct dma_fence *fence;
 	void __user *user_data;
 	u64 remain;
 	int ret;
 	if (ret)
 		return ret;
+	ret = i915_gem_object_pin_pages(obj);
+	if (ret)
+		goto err_unlock;
 	ret = i915_gem_object_prepare_write(obj, &needs_clflush);
-	if (ret) {
+	if (ret)
-		i915_gem_object_unlock(obj);
+		goto err_unpin;
-		return ret;
-	}
-	fence = i915_gem_object_lock_fence(obj);
 	i915_gem_object_finish_access(obj);
 	i915_gem_object_unlock(obj);
-	if (!fence)
-		return -ENOMEM;
 	/* If we don't overwrite a cacheline completely we need to be
 	 * careful to have up-to-date data by first clflushing. Don't
 	 * overcomplicate things and flush the entire patch.
 	}
 	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
-	i915_gem_object_unlock_fence(obj, fence);
+	i915_gem_object_unpin_pages(obj);
+	return ret;
+err_unpin:
+	i915_gem_object_unpin_pages(obj);
+err_unlock:
+	i915_gem_object_unlock(obj);
 	return ret;
 }
 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 		      struct drm_file *file)
 {
+	struct drm_i915_private *i915 = to_i915(dev);
 	struct drm_i915_gem_pwrite *args = data;
 	struct drm_i915_gem_object *obj;
 	int ret;
+	/* PWRITE is disallowed for all platforms after TGL-LP.  This also
+	 * covers all platforms with local memory.
+	 */
+	if (INTEL_GEN(i915) >= 12 && !IS_TIGERLAKE(i915))
+		return -EOPNOTSUPP;
 	if (args->size == 0)
 		return 0;
 	if (ret)
 		goto err;
-	ret = i915_gem_object_pin_pages(obj);
-	if (ret)
-		goto err;
 	ret = -EFAULT;
 	/* We can only do the GTT pwrite on untiled buffers, as otherwise
 	 * it would end up going through the fenced access, and we'll get
 			ret = i915_gem_shmem_pwrite(obj, args);
 	}
-	i915_gem_object_unpin_pages(obj);
 err:
 	i915_gem_object_put(obj);
 	return ret;
 			return ERR_PTR(ret);
 	}
-	ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);
+	if (ww)
+		ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);
+	else
+		ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
 	if (ret)
 		return ERR_PTR(ret);
 	if (!obj)
 		return -ENOENT;
-	err = mutex_lock_interruptible(&obj->mm.lock);
+	err = i915_gem_object_lock_interruptible(obj, NULL);
 	if (err)
 		goto out;
 		i915_gem_object_truncate(obj);
 	args->retained = obj->mm.madv != __I915_MADV_PURGED;
-	mutex_unlock(&obj->mm.lock);
+	i915_gem_object_unlock(obj);
 out:
 	i915_gem_object_put(obj);
 	return err;
 err_unlock:
 	i915_gem_drain_workqueue(dev_priv);
-	if (ret != -EIO) {
+	if (ret != -EIO)
 		intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
-		i915_gem_cleanup_userptr(dev_priv);
-	}
 	if (ret == -EIO) {
 		/*
 	intel_wa_list_free(&dev_priv->gt_wa_list);
 	intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
-	i915_gem_cleanup_userptr(dev_priv);
 	i915_gem_drain_freed_objects(dev_priv);

drivers/gpu/drm/i915/i915_gem_gtt.c+1−1

@@ -44,7 +44,7 @@ int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, · 43 unmodified lines
44	* the DMA remapper, i915_gem_shrink will return 0.	44	* the DMA remapper, i915_gem_shrink will return 0.
45	*/	45	*/
46	GEM_BUG_ON(obj->mm.pages == pages);	46	GEM_BUG_ON(obj->mm.pages == pages);
47	} while (i915_gem_shrink(to_i915(obj->base.dev),	47	} while (i915_gem_shrink(NULL, to_i915(obj->base.dev),
48	obj->base.size >> PAGE_SHIFT, NULL,	48	obj->base.size >> PAGE_SHIFT, NULL,
49	I915_SHRINK_BOUND \|	49	I915_SHRINK_BOUND \|
50	I915_SHRINK_UNBOUND));	50	I915_SHRINK_UNBOUND));

drivers/gpu/drm/i915/i915_memcpy.c+1−1

@@ -135,7 +135,7 @@ bool i915_memcpy_from_wc(void dst, const void src, unsigned long len) · 134 unmodified lines
135	* accepts that its arguments may not be aligned, but are valid for the	135	* accepts that its arguments may not be aligned, but are valid for the
136	* potential 16-byte read past the end.	136	* potential 16-byte read past the end.
137	*/	137	*/
138	void i915_unaligned_memcpy_from_wc(void dst, void src, unsigned long len)	138	void i915_unaligned_memcpy_from_wc(void dst, const void src, unsigned long len)
139	{	139	{
140	unsigned long addr;	140	unsigned long addr;
141		141

drivers/gpu/drm/i915/i915_memcpy.h+1−1

@@ -13,7 +13,7 @@ struct drm_i915_private; · 12 unmodified lines
13	void i915_memcpy_init_early(struct drm_i915_private *i915);	13	void i915_memcpy_init_early(struct drm_i915_private *i915);
14		14
15	bool i915_memcpy_from_wc(void dst, const void src, unsigned long len);	15	bool i915_memcpy_from_wc(void dst, const void src, unsigned long len);
16	void i915_unaligned_memcpy_from_wc(void dst, void src, unsigned long len);	16	void i915_unaligned_memcpy_from_wc(void dst, const void src, unsigned long len);
17		17
18	/* The movntdqa instructions used for memcpy-from-wc require 16-byte alignment,	18	/* The movntdqa instructions used for memcpy-from-wc require 16-byte alignment,
19	* as well as SSE4.1 support. i915_memcpy_from_wc() will report if it cannot	19	* as well as SSE4.1 support. i915_memcpy_from_wc() will report if it cannot

drivers/gpu/drm/i915/i915_params.c+5−0

 	"Fake LMEM start offset (default: 0)");
 #endif
+#if CONFIG_DRM_I915_REQUEST_TIMEOUT
+i915_param_named_unsafe(request_timeout_ms, uint, 0600,
+			"Default request/fence/batch buffer expiration timeout.");
+#endif
 static __always_inline void _print_param(struct drm_printer *p,
 					 const char *name,
 					 const char *type,

drivers/gpu/drm/i915/i915_params.h+1−0

@@ -72,6 +72,7 @@ struct drm_printer; · 71 unmodified lines
72	param(int, enable_dpcd_backlight, -1, 0600) \	72	param(int, enable_dpcd_backlight, -1, 0600) \
73	param(char *, force_probe, CONFIG_DRM_I915_FORCE_PROBE, 0400) \	73	param(char *, force_probe, CONFIG_DRM_I915_FORCE_PROBE, 0400) \
74	param(unsigned long, fake_lmem_start, 0, 0400) \	74	param(unsigned long, fake_lmem_start, 0, 0400) \
		75	param(unsigned int, request_timeout_ms, CONFIG_DRM_I915_REQUEST_TIMEOUT, 0600) \
75	/* leave bools at the end to not create holes */ \	76	/* leave bools at the end to not create holes */ \
76	param(bool, enable_hangcheck, true, 0600) \	77	param(bool, enable_hangcheck, true, 0600) \
77	param(bool, load_detect_test, false, 0600) \	78	param(bool, load_detect_test, false, 0600) \

drivers/gpu/drm/i915/i915_perf.c+43−13

 	stream->oa_buffer.vma = vma;
 	stream->oa_buffer.vaddr =
-		i915_gem_object_pin_map(bo, I915_MAP_WB);
+		i915_gem_object_pin_map_unlocked(bo, I915_MAP_WB);
 	if (IS_ERR(stream->oa_buffer.vaddr)) {
 		ret = PTR_ERR(stream->oa_buffer.vaddr);
 		goto err_unpin;
 	const u32 base = stream->engine->mmio_base;
 #define CS_GPR(x) GEN8_RING_CS_GPR(base, x)
 	u32 *batch, *ts0, *cs, *jump;
+	struct i915_gem_ww_ctx ww;
 	int ret, i;
 	enum {
 		START_TS,
 		return PTR_ERR(bo);
 	}
+	i915_gem_ww_ctx_init(&ww, true);
+retry:
+	ret = i915_gem_object_lock(bo, &ww);
+	if (ret)
+		goto out_ww;
 	/*
 	 * We pin in GGTT because we jump into this buffer now because
 	 * multiple OA config BOs will have a jump to this address and it
 	 * needs to be fixed during the lifetime of the i915/perf stream.
 	 */
-	vma = i915_gem_object_ggtt_pin(bo, NULL, 0, 0, PIN_HIGH);
+	vma = i915_gem_object_ggtt_pin_ww(bo, &ww, NULL, 0, 0, PIN_HIGH);
 	if (IS_ERR(vma)) {
 		ret = PTR_ERR(vma);
-		goto err_unref;
+		goto out_ww;
 	}
 	batch = cs = i915_gem_object_pin_map(bo, I915_MAP_WB);
 	__i915_gem_object_release_map(bo);
 	stream->noa_wait = vma;
-	return 0;
+	goto out_ww;
 err_unpin:
 	i915_vma_unpin_and_release(&vma, 0);
-err_unref:
+out_ww:
-	i915_gem_object_put(bo);
+	if (ret == -EDEADLK) {
+		ret = i915_gem_ww_ctx_backoff(&ww);
+		if (!ret)
+			goto retry;
+	}
+	i915_gem_ww_ctx_fini(&ww);
+	if (ret)
+		i915_gem_object_put(bo);
 	return ret;
 }
 {
 	struct drm_i915_gem_object *obj;
 	struct i915_oa_config_bo *oa_bo;
+	struct i915_gem_ww_ctx ww;
 	size_t config_length = 0;
 	u32 *cs;
 	int err;
 		goto err_free;
 	}
+	i915_gem_ww_ctx_init(&ww, true);
+retry:
+	err = i915_gem_object_lock(obj, &ww);
+	if (err)
+		goto out_ww;
 	cs = i915_gem_object_pin_map(obj, I915_MAP_WB);
 	if (IS_ERR(cs)) {
 		err = PTR_ERR(cs);
-		goto err_oa_bo;
+		goto out_ww;
 	}
 	cs = write_cs_mi_lri(cs,
 				       NULL);
 	if (IS_ERR(oa_bo->vma)) {
 		err = PTR_ERR(oa_bo->vma);
-		goto err_oa_bo;
+		goto out_ww;
 	}
 	oa_bo->oa_config = i915_oa_config_get(oa_config);
 	llist_add(&oa_bo->node, &stream->oa_config_bos);
-	return oa_bo;
+out_ww:
+	if (err == -EDEADLK) {
+		err = i915_gem_ww_ctx_backoff(&ww);
+		if (!err)
+			goto retry;
+	}
+	i915_gem_ww_ctx_fini(&ww);
-err_oa_bo:
+	if (err)
-	i915_gem_object_put(obj);
+		i915_gem_object_put(obj);
 err_free:
-	kfree(oa_bo);
+	if (err) {
-	return ERR_PTR(err);
+		kfree(oa_bo);
+		return ERR_PTR(err);
+	}
+	return oa_bo;
 }
 static struct i915_vma *

drivers/gpu/drm/i915/i915_request.c+126−7

@@ -33,7 +33,10 @@ · 32 unmodified lines
33	#include "gem/i915_gem_context.h"	33	#include "gem/i915_gem_context.h"
34	#include "gt/intel_breadcrumbs.h"	34	#include "gt/intel_breadcrumbs.h"
35	#include "gt/intel_context.h"	35	#include "gt/intel_context.h"
		36	#include "gt/intel_engine.h"
		37	#include "gt/intel_engine_heartbeat.h"
36	#include "gt/intel_gpu_commands.h"	38	#include "gt/intel_gpu_commands.h"
		39	#include "gt/intel_reset.h"
37	#include "gt/intel_ring.h"	40	#include "gt/intel_ring.h"
38	#include "gt/intel_rps.h"	41	#include "gt/intel_rps.h"
39		42
@@ -244,6 +247,50 @@ static void __i915_request_fill(struct i915_request *rq, u8 val) · 204 unmodified lines
244	memset(vaddr + head, val, rq->postfix - head);	247	memset(vaddr + head, val, rq->postfix - head);
245	}	248	}
246		249
		250	/**
		251	* i915_request_active_engine
		252	* @rq: request to inspect
		253	* @active: pointer in which to return the active engine
		254	*
		255	* Fills the currently active engine to the @active pointer if the request
		256	* is active and still not completed.
		257	*
		258	* Returns true if request was active or false otherwise.
		259	*/
		260	bool
		261	i915_request_active_engine(struct i915_request *rq,
		262	struct intel_engine_cs **active)
		263	{
		264	struct intel_engine_cs engine, locked;
		265	bool ret = false;
		266
		267	/*
		268	* Serialise with __i915_request_submit() so that it sees
		269	* is-banned?, or we know the request is already inflight.
		270	*
		271	* Note that rq->engine is unstable, and so we double
		272	* check that we have acquired the lock on the final engine.
		273	*/
		274	locked = READ_ONCE(rq->engine);
		275	spin_lock_irq(&locked->active.lock);
		276	while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) {
		277	spin_unlock(&locked->active.lock);
		278	locked = engine;
		279	spin_lock(&locked->active.lock);
		280	}
		281
		282	if (i915_request_is_active(rq)) {
		283	if (!__i915_request_is_complete(rq))
		284	*active = locked;
		285	ret = true;
		286	}
		287
		288	spin_unlock_irq(&locked->active.lock);
		289
		290	return ret;
		291	}
		292
		293
247	static void remove_from_engine(struct i915_request *rq)	294	static void remove_from_engine(struct i915_request *rq)
248	{	295	{
249	struct intel_engine_cs engine, locked;	296	struct intel_engine_cs engine, locked;
@@ -274,6 +321,53 @@ static void remove_from_engine(struct i915_request *rq) · 24 unmodified lines
274	__notify_execute_cb_imm(rq);	321	__notify_execute_cb_imm(rq);
275	}	322	}
276		323
		324	static void __rq_init_watchdog(struct i915_request *rq)
		325	{
		326	rq->watchdog.timer.function = NULL;
		327	}
		328
		329	static enum hrtimer_restart __rq_watchdog_expired(struct hrtimer *hrtimer)
		330	{
		331	struct i915_request *rq =
		332	container_of(hrtimer, struct i915_request, watchdog.timer);
		333	struct intel_gt *gt = rq->engine->gt;
		334
		335	if (!i915_request_completed(rq)) {
		336	if (llist_add(&rq->watchdog.link, &gt->watchdog.list))
		337	schedule_work(&gt->watchdog.work);
		338	} else {
		339	i915_request_put(rq);
		340	}
		341
		342	return HRTIMER_NORESTART;
		343	}
		344
		345	static void __rq_arm_watchdog(struct i915_request *rq)
		346	{
		347	struct i915_request_watchdog *wdg = &rq->watchdog;
		348	struct intel_context *ce = rq->context;
		349
		350	if (!ce->watchdog.timeout_us)
		351	return;
		352
		353	hrtimer_init(&wdg->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
		354	wdg->timer.function = __rq_watchdog_expired;
		355	hrtimer_start_range_ns(&wdg->timer,
		356	ns_to_ktime(ce->watchdog.timeout_us *
		357	NSEC_PER_USEC),
		358	NSEC_PER_MSEC,
		359	HRTIMER_MODE_REL);
		360	i915_request_get(rq);
		361	}
		362
		363	static void __rq_cancel_watchdog(struct i915_request *rq)
		364	{
		365	struct i915_request_watchdog *wdg = &rq->watchdog;
		366
		367	if (wdg->timer.function && hrtimer_try_to_cancel(&wdg->timer) > 0)
		368	i915_request_put(rq);
		369	}
		370
277	bool i915_request_retire(struct i915_request *rq)	371	bool i915_request_retire(struct i915_request *rq)
278	{	372	{
279	if (!__i915_request_is_complete(rq))	373	if (!__i915_request_is_complete(rq))
@@ -285,6 +379,8 @@ bool i915_request_retire(struct i915_request *rq) · 5 unmodified lines
285	trace_i915_request_retire(rq);	379	trace_i915_request_retire(rq);
286	i915_request_mark_complete(rq);	380	i915_request_mark_complete(rq);
287		381
		382	__rq_cancel_watchdog(rq);
		383
288	/*	384	/*
289	* We know the GPU must have read the request to have	385	* We know the GPU must have read the request to have
290	* sent us the seqno + interrupt, so use the position	386	* sent us the seqno + interrupt, so use the position
@@ -498,20 +594,22 @@ void __i915_request_skip(struct i915_request *rq) · 207 unmodified lines
498	rq->infix = rq->postfix;	594	rq->infix = rq->postfix;
499	}	595	}
500		596
501	void i915_request_set_error_once(struct i915_request *rq, int error)	597	bool i915_request_set_error_once(struct i915_request *rq, int error)
502	{	598	{
503	int old;	599	int old;
504		600
505	GEM_BUG_ON(!IS_ERR_VALUE((long)error));	601	GEM_BUG_ON(!IS_ERR_VALUE((long)error));
506		602
507	if (i915_request_signaled(rq))	603	if (i915_request_signaled(rq))
508	return;	604	return false;
509		605
510	old = READ_ONCE(rq->fence.error);	606	old = READ_ONCE(rq->fence.error);
511	do {	607	do {
512	if (fatal_error(old))	608	if (fatal_error(old))
513	return;	609	return false;
514	} while (!try_cmpxchg(&rq->fence.error, &old, error));	610	} while (!try_cmpxchg(&rq->fence.error, &old, error));
		611
		612	return true;
515	}	613	}
516		614
517	void i915_request_mark_eio(struct i915_request *rq)	615	void i915_request_mark_eio(struct i915_request *rq)
@@ -678,6 +776,28 @@ void i915_request_unsubmit(struct i915_request *request) · 160 unmodified lines
678	spin_unlock_irqrestore(&engine->active.lock, flags);	776	spin_unlock_irqrestore(&engine->active.lock, flags);
679	}	777	}
680		778
		779	static void __cancel_request(struct i915_request *rq)
		780	{
		781	struct intel_engine_cs *engine = NULL;
		782
		783	i915_request_active_engine(rq, &engine);
		784
		785	if (engine && intel_engine_pulse(engine))
		786	intel_gt_handle_error(engine->gt, engine->mask, 0,
		787	"request cancellation by %s",
		788	current->comm);
		789	}
		790
		791	void i915_request_cancel(struct i915_request *rq, int error)
		792	{
		793	if (!i915_request_set_error_once(rq, error))
		794	return;
		795
		796	set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags);
		797
		798	__cancel_request(rq);
		799	}
		800
681	static int __i915_sw_fence_call	801	static int __i915_sw_fence_call
682	submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)	802	submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
683	{	803	{
@@ -690,6 +810,8 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) · 6 unmodified lines
690		810
691	if (unlikely(fence->error))	811	if (unlikely(fence->error))
692	i915_request_set_error_once(request, fence->error);	812	i915_request_set_error_once(request, fence->error);
		813	else
		814	__rq_arm_watchdog(request);
693		815
694	/*	816	/*
695	* We need to serialize use of the submit_request() callback	817	* We need to serialize use of the submit_request() callback
@@ -863,7 +985,6 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) · 167 unmodified lines
863	rq->fence.seqno = seqno;	985	rq->fence.seqno = seqno;
864		986
865	RCU_INIT_POINTER(rq->timeline, tl);	987	RCU_INIT_POINTER(rq->timeline, tl);
866	RCU_INIT_POINTER(rq->hwsp_cacheline, tl->hwsp_cacheline);
867	rq->hwsp_seqno = tl->hwsp_seqno;	988	rq->hwsp_seqno = tl->hwsp_seqno;
868	GEM_BUG_ON(__i915_request_is_complete(rq));	989	GEM_BUG_ON(__i915_request_is_complete(rq));
869		990
@@ -877,6 +998,7 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) · 7 unmodified lines
877		998
878	/* No zalloc, everything must be cleared after use */	999	/* No zalloc, everything must be cleared after use */
879	rq->batch = NULL;	1000	rq->batch = NULL;
		1001	__rq_init_watchdog(rq);
880	GEM_BUG_ON(rq->capture_list);	1002	GEM_BUG_ON(rq->capture_list);
881	GEM_BUG_ON(!llist_empty(&rq->execute_cb));	1003	GEM_BUG_ON(!llist_empty(&rq->execute_cb));
882		1004
@@ -1108,9 +1230,6 @@ emit_semaphore_wait(struct i915_request *to, · 225 unmodified lines
1108	if (i915_request_has_initial_breadcrumb(to))	1230	if (i915_request_has_initial_breadcrumb(to))
1109	goto await_fence;	1231	goto await_fence;
1110		1232
1111	if (!rcu_access_pointer(from->hwsp_cacheline))
1112	goto await_fence;
1113
1114	/*	1233	/*
1115	* If this or its dependents are waiting on an external fence	1234	* If this or its dependents are waiting on an external fence
1116	* that may fail catastrophically, then we want to avoid using	1235	* that may fail catastrophically, then we want to avoid using

drivers/gpu/drm/i915/i915_request.h+36−11

 #define I915_REQUEST_H
 #include <linux/dma-fence.h>
+#include <linux/hrtimer.h>
 #include <linux/irq_work.h>
+#include <linux/llist.h>
 #include <linux/lockdep.h>
 #include "gem/i915_gem_context_types.h"
 	 */
 	const u32 *hwsp_seqno;
-	/*
-	 * If we need to access the timeline's seqno for this request in
-	 * another request, we need to keep a read reference to this associated
-	 * cacheline, so that we do not free and recycle it before the foreign
-	 * observers have completed. Hence, we keep a pointer to the cacheline
-	 * inside the timeline's HWSP vma, but it is only valid while this
-	 * request has not completed and guarded by the timeline mutex.
-	 */
-	struct intel_timeline_cacheline __rcu *hwsp_cacheline;
 	/** Position in the ring of the start of the request */
 	u32 head;
 	/** timeline->request entry for this request */
 	struct list_head link;
+	/** Watchdog support fields. */
+	struct i915_request_watchdog {
+		struct llist_node link;
+		struct hrtimer timer;
+	} watchdog;
 	I915_SELFTEST_DECLARE(struct {
 		struct list_head link;
 		unsigned long delay;
 i915_request_create(struct intel_context *ce);
 void __i915_request_skip(struct i915_request *rq);
-void i915_request_set_error_once(struct i915_request *rq, int error);
+bool i915_request_set_error_once(struct i915_request *rq, int error);
 void i915_request_mark_eio(struct i915_request *rq);
 struct i915_request *__i915_request_commit(struct i915_request *request);
 void __i915_request_unsubmit(struct i915_request *request);
 void i915_request_unsubmit(struct i915_request *request);
+void i915_request_cancel(struct i915_request *rq, int error);
 long i915_request_wait(struct i915_request *rq,
 		       unsigned int flags,
 		       long timeout)
 					 lockdep_is_held(&rq->engine->active.lock));
 }
+static inline u32
+i915_request_active_seqno(const struct i915_request *rq)
+{
+	u32 hwsp_phys_base =
+		page_mask_bits(i915_request_active_timeline(rq)->hwsp_offset);
+	u32 hwsp_relative_offset = offset_in_page(rq->hwsp_seqno);
+	/*
+	 * Because of wraparound, we cannot simply take tl->hwsp_offset,
+	 * but instead use the fact that the relative for vaddr is the
+	 * offset as for hwsp_offset. Take the top bits from tl->hwsp_offset
+	 * and combine them with the relative offset in rq->hwsp_seqno.
+	 *
+	 * As rw->hwsp_seqno is rewritten when signaled, this only works
+	 * when the request isn't signaled yet, but at that point you
+	 * no longer need the offset.
+	 */
+	return hwsp_phys_base + hwsp_relative_offset;
+}
+bool
+i915_request_active_engine(struct i915_request *rq,
+			   struct intel_engine_cs **active);
 #endif /* I915_REQUEST_H */

drivers/gpu/drm/i915/i915_selftest.h+2−0

 #define I915_SELFTEST_DECLARE(x) x
 #define I915_SELFTEST_ONLY(x) unlikely(x)
+#define I915_SELFTEST_EXPORT
 #else /* !IS_ENABLED(CONFIG_DRM_I915_SELFTEST) */
 #define I915_SELFTEST_DECLARE(x)
 #define I915_SELFTEST_ONLY(x) 0
+#define I915_SELFTEST_EXPORT static
 #endif

drivers/gpu/drm/i915/i915_vma.c+22−8

 }
 static struct i915_vma *
-vma_lookup(struct drm_i915_gem_object *obj,
+i915_vma_lookup(struct drm_i915_gem_object *obj,
 	   struct i915_address_space *vm,
 	   const struct i915_ggtt_view *view)
 {
 	GEM_BUG_ON(!atomic_read(&vm->open));
 	spin_lock(&obj->vma.lock);
-	vma = vma_lookup(obj, vm, view);
+	vma = i915_vma_lookup(obj, vm, view);
 	spin_unlock(&obj->vma.lock);
 	/* vma_create() will resolve the race if another creates the vma */
 	int err;
 #ifdef CONFIG_PROVE_LOCKING
-	if (debug_locks && lockdep_is_held(&vma->vm->i915->drm.struct_mutex))
+	if (debug_locks && !WARN_ON(!ww) && vma->resv)
-		WARN_ON(!ww);
+		assert_vma_held(vma);
 #endif
 	BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
 		wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm);
 	if (flags & vma->vm->bind_async_flags) {
+		/* lock VM */
+		err = i915_vm_lock_objects(vma->vm, ww);
+		if (err)
+			goto err_rpm;
 		work = i915_vma_work();
 		if (!work) {
 			err = -ENOMEM;
 	GEM_BUG_ON(!i915_vma_is_ggtt(vma));
+#ifdef CONFIG_LOCKDEP
+	WARN_ON(!ww && vma->resv && dma_resv_held(vma->resv));
+#endif
 	do {
-		err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL);
+		if (ww)
+			err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL);
+		else
+			err = i915_vma_pin(vma, 0, align, flags | PIN_GLOBAL);
 		if (err != -ENOSPC) {
 			if (!err) {
 				err = i915_vma_wait_for_bind(vma);
 		obj->write_domain = I915_GEM_DOMAIN_RENDER;
 		obj->read_domains = 0;
 	} else {
-		err = dma_resv_reserve_shared(vma->resv, 1);
+		if (!(flags & __EXEC_OBJECT_NO_RESERVE)) {
-		if (unlikely(err))
+			err = dma_resv_reserve_shared(vma->resv, 1);
-			return err;
+			if (unlikely(err))
+				return err;
+		}
 		dma_resv_add_shared_fence(vma->resv, &rq->fence);
 		obj->write_domain = 0;

drivers/gpu/drm/i915/i915_vma.h+19−1

@@ -52,6 +52,9 @@ static inline bool i915_vma_is_active(const struct i915_vma *vma) · 51 unmodified lines
52	return !i915_active_is_idle(&vma->active);	52	return !i915_active_is_idle(&vma->active);
53	}	53	}
54		54
		55	/* do not reserve memory to prevent deadlocks */
		56	#define __EXEC_OBJECT_NO_RESERVE BIT(31)
		57
55	int __must_check __i915_vma_move_to_active(struct i915_vma *vma,	58	int __must_check __i915_vma_move_to_active(struct i915_vma *vma,
56	struct i915_request *rq);	59	struct i915_request *rq);
57	int __must_check i915_vma_move_to_active(struct i915_vma *vma,	60	int __must_check i915_vma_move_to_active(struct i915_vma *vma,
@@ -243,7 +246,22 @@ i915_vma_pin_ww(struct i915_vma vma, struct i915_gem_ww_ctx ww, · 185 unmodified lines
243	static inline int __must_check	246	static inline int __must_check
244	i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)	247	i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
245	{	248	{
246	return i915_vma_pin_ww(vma, NULL, size, alignment, flags);	249	struct i915_gem_ww_ctx ww;
		250	int err;
		251
		252	i915_gem_ww_ctx_init(&ww, true);
		253	retry:
		254	err = i915_gem_object_lock(vma->obj, &ww);
		255	if (!err)
		256	err = i915_vma_pin_ww(vma, &ww, size, alignment, flags);
		257	if (err == -EDEADLK) {
		258	err = i915_gem_ww_ctx_backoff(&ww);
		259	if (!err)
		260	goto retry;
		261	}
		262	i915_gem_ww_ctx_fini(&ww);
		263
		264	return err;
247	}	265	}
248		266
249	int i915_ggtt_pin(struct i915_vma vma, struct i915_gem_ww_ctx ww,	267	int i915_ggtt_pin(struct i915_vma vma, struct i915_gem_ww_ctx ww,

drivers/gpu/drm/i915/selftests/i915_gem_gtt.c+68−26

 		goto err;
 	drm_gem_private_object_init(&i915->drm, &obj->base, size);
-	i915_gem_object_init(obj, &fake_ops, &lock_class);
+	i915_gem_object_init(obj, &fake_ops, &lock_class, 0);
 	i915_gem_object_set_volatile(obj);
 	obj->cache_level = I915_CACHE_NONE;
 	/* Preallocate the "backing storage" */
-	if (i915_gem_object_pin_pages(obj))
+	if (i915_gem_object_pin_pages_unlocked(obj))
 		goto err_obj;
 	i915_gem_object_unpin_pages(obj);
 {
 	struct drm_i915_private *dev_priv = arg;
 	struct i915_ppgtt *ppgtt;
+	struct i915_gem_ww_ctx ww;
 	u64 size, last, limit;
 	int err = 0;
 	limit = totalram_pages() << PAGE_SHIFT;
 	limit = min(ppgtt->vm.total, limit);
+	i915_gem_ww_ctx_init(&ww, false);
+retry:
+	err = i915_vm_lock_objects(&ppgtt->vm, &ww);
+	if (err)
+		goto err_ppgtt_cleanup;
 	/* Check we can allocate the entire range */
 	for (size = 4096; size <= limit; size <<= 2) {
 		struct i915_vm_pt_stash stash = {};
 	}
 err_ppgtt_cleanup:
+	if (err == -EDEADLK) {
+		err = i915_gem_ww_ctx_backoff(&ww);
+		if (!err)
+			goto retry;
+	}
+	i915_gem_ww_ctx_fini(&ww);
 	i915_vm_put(&ppgtt->vm);
 	return err;
 }
 		GEM_BUG_ON(obj->base.size != BIT_ULL(size));
-		if (i915_gem_object_pin_pages(obj)) {
+		if (i915_gem_object_pin_pages_unlocked(obj)) {
 			i915_gem_object_put(obj);
 			kfree(order);
 			break;
 			if (vm->allocate_va_range) {
 				struct i915_vm_pt_stash stash = {};
+				struct i915_gem_ww_ctx ww;
+				int err;
+				i915_gem_ww_ctx_init(&ww, false);
+retry:
+				err = i915_vm_lock_objects(vm, &ww);
+				if (err)
+					goto alloc_vm_end;
+				err = -ENOMEM;
 				if (i915_vm_alloc_pt_stash(vm, &stash,
 							   BIT_ULL(size)))
-					break;
+					goto alloc_vm_end;
-				if (i915_vm_pin_pt_stash(vm, &stash)) {
-					i915_vm_free_pt_stash(vm, &stash);
-					break;
-				}
-				vm->allocate_va_range(vm, &stash,
+				err = i915_vm_pin_pt_stash(vm, &stash);
-						      addr, BIT_ULL(size));
+				if (!err)
+					vm->allocate_va_range(vm, &stash,
+							      addr, BIT_ULL(size));
 				i915_vm_free_pt_stash(vm, &stash);
+alloc_vm_end:
+				if (err == -EDEADLK) {
+					err = i915_gem_ww_ctx_backoff(&ww);
+					if (!err)
+						goto retry;
+				}
+				i915_gem_ww_ctx_fini(&ww);
+				if (err)
+					break;
 			}
 			mock_vma->pages = obj->mm.pages;
 	if (IS_ERR(obj))
 		return PTR_ERR(obj);
-	err = i915_gem_object_pin_pages(obj);
+	err = i915_gem_object_pin_pages_unlocked(obj);
 	if (err)
 		goto out_free;
 			goto out;
 		}
-		err = i915_gem_object_pin_pages(obj);
+		err = i915_gem_object_pin_pages_unlocked(obj);
 		if (err) {
 			i915_gem_object_put(obj);
 			goto out;
 			goto out;
 		}
-		err = i915_gem_object_pin_pages(obj);
+		err = i915_gem_object_pin_pages_unlocked(obj);
 		if (err) {
 			i915_gem_object_put(obj);
 			goto out;
 			goto out;
 		}
-		err = i915_gem_object_pin_pages(obj);
+		err = i915_gem_object_pin_pages_unlocked(obj);
 		if (err) {
 			i915_gem_object_put(obj);
 			goto out;
 			goto out;
 		}
-		err = i915_gem_object_pin_pages(obj);
+		err = i915_gem_object_pin_pages_unlocked(obj);
 		if (err) {
 			i915_gem_object_put(obj);
 			goto out;
 		goto out_vm;
 	}
-	batch = i915_gem_object_pin_map(bbe, I915_MAP_WC);
+	batch = i915_gem_object_pin_map_unlocked(bbe, I915_MAP_WC);
 	if (IS_ERR(batch)) {
 		err = PTR_ERR(batch);
 		goto out_put_bbe;
 	}
 	/* Track the execution of each request by writing into different slot */
-	batch = i915_gem_object_pin_map(act, I915_MAP_WC);
+	batch = i915_gem_object_pin_map_unlocked(act, I915_MAP_WC);
 	if (IS_ERR(batch)) {
 		err = PTR_ERR(batch);
 		goto out_put_act;
 		goto out_put_out;
 	GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE);
-	result = i915_gem_object_pin_map(out, I915_MAP_WB);
+	result = i915_gem_object_pin_map_unlocked(out, I915_MAP_WB);
 	if (IS_ERR(result)) {
 		err = PTR_ERR(result);
 		goto out_put_out;
 		while (!__igt_timeout(end_time, NULL)) {
 			struct i915_vm_pt_stash stash = {};
 			struct i915_request *rq;
+			struct i915_gem_ww_ctx ww;
 			u64 offset;
 			offset = igt_random_offset(&prng,
 			if (err)
 				goto end;
+			i915_gem_ww_ctx_init(&ww, false);
+retry:
+			err = i915_vm_lock_objects(vm, &ww);
+			if (err)
+				goto end_ww;
 			err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size);
 			if (err)
-				goto end;
+				goto end_ww;
 			err = i915_vm_pin_pt_stash(vm, &stash);
-			if (err) {
+			if (!err)
-				i915_vm_free_pt_stash(vm, &stash);
+				vm->allocate_va_range(vm, &stash, offset, chunk_size);
-				goto end;
-			}
-			vm->allocate_va_range(vm, &stash, offset, chunk_size);
 			i915_vm_free_pt_stash(vm, &stash);
+end_ww:
+			if (err == -EDEADLK) {
+				err = i915_gem_ww_ctx_backoff(&ww);
+				if (!err)
+					goto retry;
+			}
+			i915_gem_ww_ctx_fini(&ww);
+			if (err)
+				goto end;
 			/* Prime the TLB with the dummy pages */
 			for (i = 0; i < count; i++) {

drivers/gpu/drm/i915/selftests/i915_request.c+206−5

@@ -609,6 +609,206 @@ static int live_nop_request(void *arg) · 608 unmodified lines
609	return err;	609	return err;
610	}	610	}
611		611
		612	static int __cancel_inactive(struct intel_engine_cs *engine)
		613	{
		614	struct intel_context *ce;
		615	struct igt_spinner spin;
		616	struct i915_request *rq;
		617	int err = 0;
		618
		619	if (igt_spinner_init(&spin, engine->gt))
		620	return -ENOMEM;
		621
		622	ce = intel_context_create(engine);
		623	if (IS_ERR(ce)) {
		624	err = PTR_ERR(ce);
		625	goto out_spin;
		626	}
		627
		628	rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
		629	if (IS_ERR(rq)) {
		630	err = PTR_ERR(rq);
		631	goto out_ce;
		632	}
		633
		634	pr_debug("%s: Cancelling inactive request\n", engine->name);
		635	i915_request_cancel(rq, -EINTR);
		636	i915_request_get(rq);
		637	i915_request_add(rq);
		638
		639	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
		640	struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
		641
		642	pr_err("%s: Failed to cancel inactive request\n", engine->name);
		643	intel_engine_dump(engine, &p, "%s\n", engine->name);
		644	err = -ETIME;
		645	goto out_rq;
		646	}
		647
		648	if (rq->fence.error != -EINTR) {
		649	pr_err("%s: fence not cancelled (%u)\n",
		650	engine->name, rq->fence.error);
		651	err = -EINVAL;
		652	}
		653
		654	out_rq:
		655	i915_request_put(rq);
		656	out_ce:
		657	intel_context_put(ce);
		658	out_spin:
		659	igt_spinner_fini(&spin);
		660	if (err)
		661	pr_err("%s: %s error %d\n", __func__, engine->name, err);
		662	return err;
		663	}
		664
		665	static int __cancel_active(struct intel_engine_cs *engine)
		666	{
		667	struct intel_context *ce;
		668	struct igt_spinner spin;
		669	struct i915_request *rq;
		670	int err = 0;
		671
		672	if (igt_spinner_init(&spin, engine->gt))
		673	return -ENOMEM;
		674
		675	ce = intel_context_create(engine);
		676	if (IS_ERR(ce)) {
		677	err = PTR_ERR(ce);
		678	goto out_spin;
		679	}
		680
		681	rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
		682	if (IS_ERR(rq)) {
		683	err = PTR_ERR(rq);
		684	goto out_ce;
		685	}
		686
		687	pr_debug("%s: Cancelling active request\n", engine->name);
		688	i915_request_get(rq);
		689	i915_request_add(rq);
		690	if (!igt_wait_for_spinner(&spin, rq)) {
		691	struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
		692
		693	pr_err("Failed to start spinner on %s\n", engine->name);
		694	intel_engine_dump(engine, &p, "%s\n", engine->name);
		695	err = -ETIME;
		696	goto out_rq;
		697	}
		698	i915_request_cancel(rq, -EINTR);
		699
		700	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
		701	struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
		702
		703	pr_err("%s: Failed to cancel active request\n", engine->name);
		704	intel_engine_dump(engine, &p, "%s\n", engine->name);
		705	err = -ETIME;
		706	goto out_rq;
		707	}
		708
		709	if (rq->fence.error != -EINTR) {
		710	pr_err("%s: fence not cancelled (%u)\n",
		711	engine->name, rq->fence.error);
		712	err = -EINVAL;
		713	}
		714
		715	out_rq:
		716	i915_request_put(rq);
		717	out_ce:
		718	intel_context_put(ce);
		719	out_spin:
		720	igt_spinner_fini(&spin);
		721	if (err)
		722	pr_err("%s: %s error %d\n", __func__, engine->name, err);
		723	return err;
		724	}
		725
		726	static int __cancel_completed(struct intel_engine_cs *engine)
		727	{
		728	struct intel_context *ce;
		729	struct igt_spinner spin;
		730	struct i915_request *rq;
		731	int err = 0;
		732
		733	if (igt_spinner_init(&spin, engine->gt))
		734	return -ENOMEM;
		735
		736	ce = intel_context_create(engine);
		737	if (IS_ERR(ce)) {
		738	err = PTR_ERR(ce);
		739	goto out_spin;
		740	}
		741
		742	rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
		743	if (IS_ERR(rq)) {
		744	err = PTR_ERR(rq);
		745	goto out_ce;
		746	}
		747	igt_spinner_end(&spin);
		748	i915_request_get(rq);
		749	i915_request_add(rq);
		750
		751	if (i915_request_wait(rq, 0, HZ / 5) < 0) {
		752	err = -ETIME;
		753	goto out_rq;
		754	}
		755
		756	pr_debug("%s: Cancelling completed request\n", engine->name);
		757	i915_request_cancel(rq, -EINTR);
		758	if (rq->fence.error) {
		759	pr_err("%s: fence not cancelled (%u)\n",
		760	engine->name, rq->fence.error);
		761	err = -EINVAL;
		762	}
		763
		764	out_rq:
		765	i915_request_put(rq);
		766	out_ce:
		767	intel_context_put(ce);
		768	out_spin:
		769	igt_spinner_fini(&spin);
		770	if (err)
		771	pr_err("%s: %s error %d\n", __func__, engine->name, err);
		772	return err;
		773	}
		774
		775	static int live_cancel_request(void *arg)
		776	{
		777	struct drm_i915_private *i915 = arg;
		778	struct intel_engine_cs *engine;
		779
		780	/*
		781	* Check cancellation of requests. We expect to be able to immediately
		782	* cancel active requests, even if they are currently on the GPU.
		783	*/
		784
		785	for_each_uabi_engine(engine, i915) {
		786	struct igt_live_test t;
		787	int err, err2;
		788
		789	if (!intel_engine_has_preemption(engine))
		790	continue;
		791
		792	err = igt_live_test_begin(&t, i915, __func__, engine->name);
		793	if (err)
		794	return err;
		795
		796	err = __cancel_inactive(engine);
		797	if (err == 0)
		798	err = __cancel_active(engine);
		799	if (err == 0)
		800	err = __cancel_completed(engine);
		801
		802	err2 = igt_live_test_end(&t);
		803	if (err)
		804	return err;
		805	if (err2)
		806	return err2;
		807	}
		808
		809	return 0;
		810	}
		811
612	static struct i915_vma empty_batch(struct drm_i915_private i915)	812	static struct i915_vma empty_batch(struct drm_i915_private i915)
613	{	813	{
614	struct drm_i915_gem_object *obj;	814	struct drm_i915_gem_object *obj;
@@ -620,7 +820,7 @@ static struct i915_vma empty_batch(struct drm_i915_private i915) · 5 unmodified lines
620	if (IS_ERR(obj))	820	if (IS_ERR(obj))
621	return ERR_CAST(obj);	821	return ERR_CAST(obj);
622		822
623	cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);	823	cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
624	if (IS_ERR(cmd)) {	824	if (IS_ERR(cmd)) {
625	err = PTR_ERR(cmd);	825	err = PTR_ERR(cmd);
626	goto err;	826	goto err;
@@ -782,7 +982,7 @@ static struct i915_vma recursive_batch(struct drm_i915_private i915) · 155 unmodified lines
782	if (err)	982	if (err)
783	goto err;	983	goto err;
784		984
785	cmd = i915_gem_object_pin_map(obj, I915_MAP_WC);	985	cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
786	if (IS_ERR(cmd)) {	986	if (IS_ERR(cmd)) {
787	err = PTR_ERR(cmd);	987	err = PTR_ERR(cmd);
788	goto err;	988	goto err;
@@ -817,7 +1017,7 @@ static int recursive_batch_resolve(struct i915_vma *batch) · 28 unmodified lines
817	{	1017	{
818	u32 *cmd;	1018	u32 *cmd;
819		1019
820	cmd = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);	1020	cmd = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
821	if (IS_ERR(cmd))	1021	if (IS_ERR(cmd))
822	return PTR_ERR(cmd);	1022	return PTR_ERR(cmd);
823		1023
@@ -1070,8 +1270,8 @@ out_request: · 246 unmodified lines
1070	if (!request[idx])	1270	if (!request[idx])
1071	break;	1271	break;
1072		1272
1073	cmd = i915_gem_object_pin_map(request[idx]->batch->obj,	1273	cmd = i915_gem_object_pin_map_unlocked(request[idx]->batch->obj,
1074	I915_MAP_WC);	1274	I915_MAP_WC);
1075	if (!IS_ERR(cmd)) {	1275	if (!IS_ERR(cmd)) {
1076	*cmd = MI_BATCH_BUFFER_END;	1276	*cmd = MI_BATCH_BUFFER_END;
1077		1277
@@ -1486,6 +1686,7 @@ int i915_request_live_selftests(struct drm_i915_private *i915) · 408 unmodified lines
1486	SUBTEST(live_sequential_engines),	1686	SUBTEST(live_sequential_engines),
1487	SUBTEST(live_parallel_engines),	1687	SUBTEST(live_parallel_engines),
1488	SUBTEST(live_empty_request),	1688	SUBTEST(live_empty_request),
		1689	SUBTEST(live_cancel_request),
1489	SUBTEST(live_breadcrumbs_smoketest),	1690	SUBTEST(live_breadcrumbs_smoketest),
1490	};	1691	};
1491		1692

drivers/gpu/drm/i915/selftests/igt_spinner.c+90−46

@@ -12,8 +12,6 @@ · 11 unmodified lines
12		12
13	int igt_spinner_init(struct igt_spinner spin, struct intel_gt gt)	13	int igt_spinner_init(struct igt_spinner spin, struct intel_gt gt)
14	{	14	{
15	unsigned int mode;
16	void *vaddr;
17	int err;	15	int err;
18		16
19	memset(spin, 0, sizeof(*spin));	17	memset(spin, 0, sizeof(*spin));
@@ -24,6 +22,7 @@ int igt_spinner_init(struct igt_spinner spin, struct intel_gt gt) · 4 unmodified lines
24	err = PTR_ERR(spin->hws);	22	err = PTR_ERR(spin->hws);
25	goto err;	23	goto err;
26	}	24	}
		25	i915_gem_object_set_cache_coherency(spin->hws, I915_CACHE_LLC);
27		26
28	spin->obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);	27	spin->obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
29	if (IS_ERR(spin->obj)) {	28	if (IS_ERR(spin->obj)) {
@@ -31,34 +30,83 @@ int igt_spinner_init(struct igt_spinner spin, struct intel_gt gt) · 1 unmodified line
31	goto err_hws;	30	goto err_hws;
32	}	31	}
33		32
34	i915_gem_object_set_cache_coherency(spin->hws, I915_CACHE_LLC);
35	vaddr = i915_gem_object_pin_map(spin->hws, I915_MAP_WB);
36	if (IS_ERR(vaddr)) {
37	err = PTR_ERR(vaddr);
38	goto err_obj;
39	}
40	spin->seqno = memset(vaddr, 0xff, PAGE_SIZE);
41
42	mode = i915_coherent_map_type(gt->i915);
43	vaddr = i915_gem_object_pin_map(spin->obj, mode);
44	if (IS_ERR(vaddr)) {
45	err = PTR_ERR(vaddr);
46	goto err_unpin_hws;
47	}
48	spin->batch = vaddr;
49
50	return 0;	33	return 0;
51		34
52	err_unpin_hws:
53	i915_gem_object_unpin_map(spin->hws);
54	err_obj:
55	i915_gem_object_put(spin->obj);
56	err_hws:	35	err_hws:
57	i915_gem_object_put(spin->hws);	36	i915_gem_object_put(spin->hws);
58	err:	37	err:
59	return err;	38	return err;
60	}	39	}
61		40
		41	static void igt_spinner_pin_obj(struct intel_context ce,
		42	struct i915_gem_ww_ctx *ww,
		43	struct drm_i915_gem_object *obj,
		44	unsigned int mode, struct i915_vma **vma)
		45	{
		46	void *vaddr;
		47	int ret;
		48
		49	*vma = i915_vma_instance(obj, ce->vm, NULL);
		50	if (IS_ERR(*vma))
		51	return ERR_CAST(*vma);
		52
		53	ret = i915_gem_object_lock(obj, ww);
		54	if (ret)
		55	return ERR_PTR(ret);
		56
		57	vaddr = i915_gem_object_pin_map(obj, mode);
		58
		59	if (!ww)
		60	i915_gem_object_unlock(obj);
		61
		62	if (IS_ERR(vaddr))
		63	return vaddr;
		64
		65	if (ww)
		66	ret = i915_vma_pin_ww(*vma, ww, 0, 0, PIN_USER);
		67	else
		68	ret = i915_vma_pin(*vma, 0, 0, PIN_USER);
		69
		70	if (ret) {
		71	i915_gem_object_unpin_map(obj);
		72	return ERR_PTR(ret);
		73	}
		74
		75	return vaddr;
		76	}
		77
		78	int igt_spinner_pin(struct igt_spinner *spin,
		79	struct intel_context *ce,
		80	struct i915_gem_ww_ctx *ww)
		81	{
		82	void *vaddr;
		83
		84	if (spin->ce && WARN_ON(spin->ce != ce))
		85	return -ENODEV;
		86	spin->ce = ce;
		87
		88	if (!spin->seqno) {
		89	vaddr = igt_spinner_pin_obj(ce, ww, spin->hws, I915_MAP_WB, &spin->hws_vma);
		90	if (IS_ERR(vaddr))
		91	return PTR_ERR(vaddr);
		92
		93	spin->seqno = memset(vaddr, 0xff, PAGE_SIZE);
		94	}
		95
		96	if (!spin->batch) {
		97	unsigned int mode =
		98	i915_coherent_map_type(spin->gt->i915);
		99
		100	vaddr = igt_spinner_pin_obj(ce, ww, spin->obj, mode, &spin->batch_vma);
		101	if (IS_ERR(vaddr))
		102	return PTR_ERR(vaddr);
		103
		104	spin->batch = vaddr;
		105	}
		106
		107	return 0;
		108	}
		109
62	static unsigned int seqno_offset(u64 fence)	110	static unsigned int seqno_offset(u64 fence)
63	{	111	{
64	return offset_in_page(sizeof(u32) * fence);	112	return offset_in_page(sizeof(u32) * fence);
@@ -103,27 +151,18 @@ igt_spinner_create_request(struct igt_spinner *spin, · 38 unmodified lines
103	if (!intel_engine_can_store_dword(ce->engine))	151	if (!intel_engine_can_store_dword(ce->engine))
104	return ERR_PTR(-ENODEV);	152	return ERR_PTR(-ENODEV);
105		153
106	vma = i915_vma_instance(spin->obj, ce->vm, NULL);	154	if (!spin->batch) {
107	if (IS_ERR(vma))	155	err = igt_spinner_pin(spin, ce, NULL);
108	return ERR_CAST(vma);	156	if (err)
109		157	return ERR_PTR(err);
110	hws = i915_vma_instance(spin->hws, ce->vm, NULL);	158	}
111	if (IS_ERR(hws))
112	return ERR_CAST(hws);
113		159
114	err = i915_vma_pin(vma, 0, 0, PIN_USER);	160	hws = spin->hws_vma;
115	if (err)	161	vma = spin->batch_vma;
116	return ERR_PTR(err);
117
118	err = i915_vma_pin(hws, 0, 0, PIN_USER);
119	if (err)
120	goto unpin_vma;
121		162
122	rq = intel_context_create_request(ce);	163	rq = intel_context_create_request(ce);
123	if (IS_ERR(rq)) {	164	if (IS_ERR(rq))
124	err = PTR_ERR(rq);	165	return ERR_CAST(rq);
125	goto unpin_hws;
126	}
127		166
128	err = move_to_active(vma, rq, 0);	167	err = move_to_active(vma, rq, 0);
129	if (err)	168	if (err)
@@ -186,10 +225,6 @@ cancel_rq: · 56 unmodified lines
186	i915_request_set_error_once(rq, err);	225	i915_request_set_error_once(rq, err);
187	i915_request_add(rq);	226	i915_request_add(rq);
188	}	227	}
189	unpin_hws:
190	i915_vma_unpin(hws);
191	unpin_vma:
192	i915_vma_unpin(vma);
193	return err ? ERR_PTR(err) : rq;	228	return err ? ERR_PTR(err) : rq;
194	}	229	}
195		230
@@ -203,6 +238,9 @@ hws_seqno(const struct igt_spinner spin, const struct i915_request rq) · 7 unmodified lines
203		238
204	void igt_spinner_end(struct igt_spinner *spin)	239	void igt_spinner_end(struct igt_spinner *spin)
205	{	240	{
		241	if (!spin->batch)
		242	return;
		243
206	*spin->batch = MI_BATCH_BUFFER_END;	244	*spin->batch = MI_BATCH_BUFFER_END;
207	intel_gt_chipset_flush(spin->gt);	245	intel_gt_chipset_flush(spin->gt);
208	}	246	}
@@ -211,10 +249,16 @@ void igt_spinner_fini(struct igt_spinner *spin) · 2 unmodified lines
211	{	249	{
212	igt_spinner_end(spin);	250	igt_spinner_end(spin);
213		251
214	i915_gem_object_unpin_map(spin->obj);	252	if (spin->batch) {
		253	i915_vma_unpin(spin->batch_vma);
		254	i915_gem_object_unpin_map(spin->obj);
		255	}
215	i915_gem_object_put(spin->obj);	256	i915_gem_object_put(spin->obj);
216		257
217	i915_gem_object_unpin_map(spin->hws);	258	if (spin->seqno) {
		259	i915_vma_unpin(spin->hws_vma);
		260	i915_gem_object_unpin_map(spin->hws);
		261	}
218	i915_gem_object_put(spin->hws);	262	i915_gem_object_put(spin->hws);
219	}	263	}
220		264

drivers/gpu/drm/i915/selftests/igt_spinner.h+5−0

 	struct intel_gt *gt;
 	struct drm_i915_gem_object *hws;
 	struct drm_i915_gem_object *obj;
+	struct intel_context *ce;
+	struct i915_vma *hws_vma, *batch_vma;
 	u32 *batch;
 	void *seqno;
 };
 int igt_spinner_init(struct igt_spinner *spin, struct intel_gt *gt);
+int igt_spinner_pin(struct igt_spinner *spin,
+		    struct intel_context *ce,
+		    struct i915_gem_ww_ctx *ww);
 void igt_spinner_fini(struct igt_spinner *spin);
 struct i915_request *

drivers/gpu/drm/i915/selftests/intel_memory_region.c+11−7

 	struct drm_i915_gem_object *obj, *on;
 	list_for_each_entry_safe(obj, on, objects, st_link) {
+		i915_gem_object_lock(obj, NULL);
 		if (i915_gem_object_has_pinned_pages(obj))
 			i915_gem_object_unpin_pages(obj);
 		/* No polluting the memory region between tests */
 		__i915_gem_object_put_pages(obj);
+		i915_gem_object_unlock(obj);
 		list_del(&obj->st_link);
 		i915_gem_object_put(obj);
 	}
 			break;
 		}
-		err = i915_gem_object_pin_pages(obj);
+		err = i915_gem_object_pin_pages_unlocked(obj);
 		if (err) {
 			i915_gem_object_put(obj);
 			break;
 	if (IS_ERR(obj))
 		return obj;
-	err = i915_gem_object_pin_pages(obj);
+	err = i915_gem_object_pin_pages_unlocked(obj);
 	if (err)
 		goto put;
 static void igt_object_release(struct drm_i915_gem_object *obj)
 {
+	i915_gem_object_lock(obj, NULL);
 	i915_gem_object_unpin_pages(obj);
 	__i915_gem_object_put_pages(obj);
+	i915_gem_object_unlock(obj);
 	list_del(&obj->st_link);
 	i915_gem_object_put(obj);
 }
 	if (err)
 		return err;
-	ptr = i915_gem_object_pin_map(obj, I915_MAP_WC);
+	ptr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 	if (IS_ERR(ptr))
 		return PTR_ERR(ptr);
 	if (IS_ERR(obj))
 		return PTR_ERR(obj);
-	err = i915_gem_object_pin_pages(obj);
+	err = i915_gem_object_pin_pages_unlocked(obj);
 	if (err)
 		goto out_put;
 		goto out_file;
 	}
-	err = i915_gem_object_pin_pages(obj);
+	err = i915_gem_object_pin_pages_unlocked(obj);
 	if (err)
 		goto out_put;
 	if (IS_ERR(obj))
 		return PTR_ERR(obj);
-	vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
+	vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
 	if (IS_ERR(vaddr)) {
 		err = PTR_ERR(vaddr);
 		goto out_put;
 		return obj;
 	}
-	addr = i915_gem_object_pin_map(obj, type);
+	addr = i915_gem_object_pin_map_unlocked(obj, type);
 	if (IS_ERR(addr)) {
 		i915_gem_object_put(obj);
 		if (PTR_ERR(addr) == -ENXIO)

drivers/gpu/drm/i915/selftests/mock_region.c+2−2

@@ -27,13 +27,13 @@ static int mock_object_init(struct intel_memory_region *mem, · 26 unmodified lines
27	return -E2BIG;	27	return -E2BIG;
28		28
29	drm_gem_private_object_init(&i915->drm, &obj->base, size);	29	drm_gem_private_object_init(&i915->drm, &obj->base, size);
30	i915_gem_object_init(obj, &mock_region_obj_ops, &lock_class);	30	i915_gem_object_init(obj, &mock_region_obj_ops, &lock_class, flags);
31		31
32	obj->read_domains = I915_GEM_DOMAIN_CPU \| I915_GEM_DOMAIN_GTT;	32	obj->read_domains = I915_GEM_DOMAIN_CPU \| I915_GEM_DOMAIN_GTT;
33		33
34	i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);	34	i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);
35		35
36	i915_gem_object_init_memory_region(obj, mem, flags);	36	i915_gem_object_init_memory_region(obj, mem);
37		37
38	return 0;	38	return 0;
39	}	39	}

include/uapi/drm/i915_drm.h+1−0

@@ -943,6 +943,7 @@ struct drm_i915_gem_exec_object { · 942 unmodified lines
943	__u64 offset;	943	__u64 offset;
944	};	944	};
945		945
		946	/* DRM_IOCTL_I915_GEM_EXECBUFFER was removed in Linux 5.13 */
946	struct drm_i915_gem_execbuffer {	947	struct drm_i915_gem_execbuffer {
947	/**	948	/**
948	* List of buffers to be validated with their relocations to be	949	* List of buffers to be validated with their relocations to be