History

Merge tag 'topic/i915-gem-next-2021-03-26' of ssh://git.freedesktop.org/git/drm/drm into drm-next

special i915-gem-next pull as requested - Conversion to dma_resv_locking, obj->mm.lock is gone (Maarten, with help from Thomas Hellström) - watchdog (Tvrtko, one patch to cancel individual request from Chris) - legacy ioctl cleanup (Jason+Ashutosh) - i915-gem TODO and RFC process doc (me) - i915_ prefix for vma_lookup (Liam Howlett) just because I spotted it and put it in here too Signed-off-by: Dave Airlie <airlied@redhat.com>

Dave Airlie committed 5 years ago

Commit 2f835b52 parents fe8a057 e1a5e6a117 files · +2749 −2165Browse files

Unified · Split

117 files
  1. Documentation/gpu/index.rst +1 −0
  2. Documentation/gpu/rfc/index.rst +17 −0
  3. drivers/gpu/drm/i915/Kconfig.profile +14 −0
  4. drivers/gpu/drm/i915/Makefile +0 −1
  5. drivers/gpu/drm/i915/TODO.txt +41 −0
  6. drivers/gpu/drm/i915/display/intel_display.c +44 −27
  7. drivers/gpu/drm/i915/display/intel_display.h +1 −1
  8. drivers/gpu/drm/i915/display/intel_dsb.c +1 −1
  9. drivers/gpu/drm/i915/display/intel_fbdev.c +1 −1
  10. drivers/gpu/drm/i915/display/intel_overlay.c +29 −5
  11. drivers/gpu/drm/i915/gem/i915_gem_clflush.c +7 −8
  12. drivers/gpu/drm/i915/gem/i915_gem_context.c +40 −33
  13. drivers/gpu/drm/i915/gem/i915_gem_context_types.h +4 −0
  14. drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c +35 −29
  15. drivers/gpu/drm/i915/gem/i915_gem_domain.c +39 −33
  16. drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +204 −131
  17. drivers/gpu/drm/i915/gem/i915_gem_fence.c +0 −95
  18. drivers/gpu/drm/i915/gem/i915_gem_internal.c +3 −3
  19. drivers/gpu/drm/i915/gem/i915_gem_ioctls.h +0 −2
  20. drivers/gpu/drm/i915/gem/i915_gem_lmem.c +2 −2
  21. drivers/gpu/drm/i915/gem/i915_gem_mman.c +29 −6
  22. drivers/gpu/drm/i915/gem/i915_gem_object.c +3 −7
  23. drivers/gpu/drm/i915/gem/i915_gem_object.h +60 −36
  24. drivers/gpu/drm/i915/gem/i915_gem_object_blt.c +6 −0
  25. drivers/gpu/drm/i915/gem/i915_gem_object_types.h +14 −9
  26. drivers/gpu/drm/i915/gem/i915_gem_pages.c +69 −40
  27. drivers/gpu/drm/i915/gem/i915_gem_phys.c +47 −63
  28. drivers/gpu/drm/i915/gem/i915_gem_pm.c +1 −1
  29. drivers/gpu/drm/i915/gem/i915_gem_region.c +1 −3
  30. drivers/gpu/drm/i915/gem/i915_gem_region.h +1 −2
  31. drivers/gpu/drm/i915/gem/i915_gem_shmem.c +31 −8
  32. drivers/gpu/drm/i915/gem/i915_gem_shrinker.c +28 −11
  33. drivers/gpu/drm/i915/gem/i915_gem_shrinker.h +3 −1
  34. drivers/gpu/drm/i915/gem/i915_gem_stolen.c +8 −6
  35. drivers/gpu/drm/i915/gem/i915_gem_tiling.c +0 −2
  36. drivers/gpu/drm/i915/gem/i915_gem_userptr.c +320 −582
  37. drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c +2 −2
  38. drivers/gpu/drm/i915/gem/selftests/huge_pages.c +26 −12
  39. drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c +4 −4
  40. drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c +6 −8
  41. drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c +5 −5
  42. drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c +1 −1
  43. drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c +1 −1
  44. drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c +10 −11
  45. drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c +1 −1
  46. drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c +3 −3
  47. drivers/gpu/drm/i915/gem/selftests/i915_gem_phys.c +9 −1
  48. drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c +1 −1
  49. drivers/gpu/drm/i915/gt/gen2_engine_cs.c +1 −1
  50. drivers/gpu/drm/i915/gt/gen6_engine_cs.c +4 −4
  51. drivers/gpu/drm/i915/gt/gen8_engine_cs.c +6 −7
  52. drivers/gpu/drm/i915/gt/intel_context_param.h +10 −1
  53. drivers/gpu/drm/i915/gt/intel_context_types.h +4 −0
  54. drivers/gpu/drm/i915/gt/intel_engine_cs.c +25 −13
  55. drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c +1 −0
  56. drivers/gpu/drm/i915/gt/intel_engine_pm.c +4 −0
  57. drivers/gpu/drm/i915/gt/intel_execlists_submission.c +39 −10
  58. drivers/gpu/drm/i915/gt/intel_execlists_submission.h +2 −0
  59. drivers/gpu/drm/i915/gt/intel_ggtt.c +8 −2
  60. drivers/gpu/drm/i915/gt/intel_gt.c +3 −0
  61. drivers/gpu/drm/i915/gt/intel_gt.h +2 −0
  62. drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c +21 −26
  63. drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.h +5 −0
  64. drivers/gpu/drm/i915/gt/intel_gt_buffer_pool_types.h +1 −0
  65. drivers/gpu/drm/i915/gt/intel_gt_requests.c +28 −0
  66. drivers/gpu/drm/i915/gt/intel_gt_types.h +7 −4
  67. drivers/gpu/drm/i915/gt/intel_gtt.c +50 −2
  68. drivers/gpu/drm/i915/gt/intel_gtt.h +8 −0
  69. drivers/gpu/drm/i915/gt/intel_lrc.c +38 −11
  70. drivers/gpu/drm/i915/gt/intel_ppgtt.c +2 −1
  71. drivers/gpu/drm/i915/gt/intel_renderstate.c +1 −1
  72. drivers/gpu/drm/i915/gt/intel_reset.c +7 −2
  73. drivers/gpu/drm/i915/gt/intel_ring_submission.c +118 −66
  74. drivers/gpu/drm/i915/gt/intel_timeline.c +94 −332
  75. drivers/gpu/drm/i915/gt/intel_timeline.h +2 −0
  76. drivers/gpu/drm/i915/gt/intel_timeline_types.h +3 −14
  77. drivers/gpu/drm/i915/gt/intel_workarounds.c +8 −2
  78. drivers/gpu/drm/i915/gt/mock_engine.c +20 −2
  79. drivers/gpu/drm/i915/gt/selftest_context.c +2 −2
  80. drivers/gpu/drm/i915/gt/selftest_engine_cs.c +6 −3
  81. drivers/gpu/drm/i915/gt/selftest_execlists.c +12 −11
  82. drivers/gpu/drm/i915/gt/selftest_hangcheck.c +4 −4
  83. drivers/gpu/drm/i915/gt/selftest_lrc.c +9 −9
  84. drivers/gpu/drm/i915/gt/selftest_mocs.c +3 −2
  85. drivers/gpu/drm/i915/gt/selftest_ring_submission.c +2 −2
  86. drivers/gpu/drm/i915/gt/selftest_timeline.c +109 −68
  87. drivers/gpu/drm/i915/gt/selftest_workarounds.c +52 −30
  88. drivers/gpu/drm/i915/gt/shmem_utils.c +1 −1
  89. drivers/gpu/drm/i915/gt/uc/intel_guc.c +1 −1
  90. drivers/gpu/drm/i915/gt/uc/intel_guc_log.c +2 −2
  91. drivers/gpu/drm/i915/gt/uc/intel_huc.c +1 −1
  92. drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c +1 −1
  93. drivers/gpu/drm/i915/gvt/dmabuf.c +1 −1
  94. drivers/gpu/drm/i915/i915_active.c +10 −10
  95. drivers/gpu/drm/i915/i915_cmd_parser.c +43 −61
  96. drivers/gpu/drm/i915/i915_debugfs.c +2 −2
  97. drivers/gpu/drm/i915/i915_drv.c +1 −1
  98. drivers/gpu/drm/i915/i915_drv.h +11 −7
  99. drivers/gpu/drm/i915/i915_gem.c +141 −117
  100. drivers/gpu/drm/i915/i915_gem_gtt.c +1 −1
  101. drivers/gpu/drm/i915/i915_memcpy.c +1 −1
  102. drivers/gpu/drm/i915/i915_memcpy.h +1 −1
  103. drivers/gpu/drm/i915/i915_params.c +5 −0
  104. drivers/gpu/drm/i915/i915_params.h +1 −0
  105. drivers/gpu/drm/i915/i915_perf.c +43 −13
  106. drivers/gpu/drm/i915/i915_request.c +126 −7
  107. drivers/gpu/drm/i915/i915_request.h +36 −11
  108. drivers/gpu/drm/i915/i915_selftest.h +2 −0
  109. drivers/gpu/drm/i915/i915_vma.c +22 −8
  110. drivers/gpu/drm/i915/i915_vma.h +19 −1
  111. drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +68 −26
  112. drivers/gpu/drm/i915/selftests/i915_request.c +206 −5
  113. drivers/gpu/drm/i915/selftests/igt_spinner.c +90 −46
  114. drivers/gpu/drm/i915/selftests/igt_spinner.h +5 −0
  115. drivers/gpu/drm/i915/selftests/intel_memory_region.c +11 −7
  116. drivers/gpu/drm/i915/selftests/mock_region.c +2 −2
  117. include/uapi/drm/i915_drm.h +1 −0
Documentation/gpu/index.rst+1−0
@@ -16,6 +16,7 @@ Linux GPU Driver Developer's Guide · 15 unmodified lines
1616 vga-switcheroo
1717 vgaarbiter
1818 todo
19+ rfc/index
1920
2021 .. only:: subproject and html
2122
Documentation/gpu/rfc/index.rst+17−0
@@ -0,0 +1,17 @@
1+===============
2+GPU RFC Section
3+===============
4+
5+For complex work, especially new uapi, it is often good to nail the high level
6+design issues before getting lost in the code details. This section is meant to
7+host such documentation:
8+
9+* Each RFC should be a section in this file, explaining the goal and main design
10+ considerations. Especially for uapi make sure you Cc: all relevant project
11+ mailing lists and involved people outside of dri-devel.
12+
13+* For uapi structures add a file to this directory with and then pull the
14+ kerneldoc in like with real uapi headers.
15+
16+* Once the code has landed move all the documentation to the right places in
17+ the main core, helper or driver sections.
drivers/gpu/drm/i915/Kconfig.profile+14−0
@@ -1,3 +1,17 @@
1+config DRM_I915_REQUEST_TIMEOUT
2+ int "Default timeout for requests (ms)"
3+ default 20000 # milliseconds
4+ help
5+ Configures the default timeout after which any user submissions will
6+ be forcefully terminated.
7+
8+ Beware setting this value lower, or close to heartbeat interval
9+ rounded to whole seconds times three, in order to avoid allowing
10+ misbehaving applications causing total rendering failure in unrelated
11+ clients.
12+
13+ May be 0 to disable the timeout.
14+
115 config DRM_I915_FENCE_TIMEOUT
216 int "Timeout for unsignaled foreign fences (ms, jiffy granularity)"
317 default 10000 # milliseconds
drivers/gpu/drm/i915/Makefile+0−1
@@ -139,7 +139,6 @@ gem-y += \ · 138 unmodified lines
139139 gem/i915_gem_dmabuf.o \
140140 gem/i915_gem_domain.o \
141141 gem/i915_gem_execbuffer.o \
142− gem/i915_gem_fence.o \
143142 gem/i915_gem_internal.o \
144143 gem/i915_gem_object.o \
145144 gem/i915_gem_object_blt.o \
drivers/gpu/drm/i915/TODO.txt+41−0
@@ -0,0 +1,41 @@
1+gem/gt TODO items
2+-----------------
3+
4+- For discrete memory manager, merge enough dg1 to be able to refactor it to
5+ TTM. Then land pci ids (just in case that turns up an uapi problem). TTM has
6+ improved a lot the past 2 years, there's no reason anymore not to use it.
7+
8+- Come up with a plan what to do with drm/scheduler and how to get there.
9+
10+- Roll out dma_fence critical section annotations.
11+
12+- There's a lot of complexity added past few years to make relocations faster.
13+ That doesn't make sense given hw and gpu apis moved away from this model years
14+ ago:
15+ 1. Land a modern pre-bound uapi like VM_BIND
16+ 2. Any complexity added in this area past few years which can't be justified
17+ with VM_BIND using userspace should be removed. Looking at amdgpu dma_resv on
18+ the bo and vm, plus some lru locks is all that needed. No complex rcu,
19+ refcounts, caching, ... on everything.
20+ This is the matching task on the vm side compared to ttm/dma_resv on the
21+ backing storage side.
22+
23+- i915_sw_fence seems to be the main structure for the i915-gem dma_fence model.
24+ How-to-dma_fence is core and drivers really shouldn't build their own world
25+ here, treating everything else as a fixed platform. i915_sw_fence concepts
26+ should be moved to dma_fence, drm/scheduler or atomic commit helpers. Or
27+ removed if dri-devel consensus is that it's not a good idea. Once that's done
28+ maybe even remove it if there's nothing left.
29+
30+Smaller things:
31+- i915_utils.h needs to be moved to the right places.
32+
33+- dma_fence_work should be in drivers/dma-buf
34+
35+- i915_mm.c should be moved to the right places. Some of the helpers also look a
36+ bit fishy:
37+
38+ https://lore.kernel.org/linux-mm/20210301083320.943079-1-hch@lst.de/
39+
40+- tasklet helpers in i915_gem.h also look a bit misplaced and should
41+ probably be moved to tasklet headers.
drivers/gpu/drm/i915/display/intel_display.c+44−27
@@ -1091,6 +1091,7 @@ static bool intel_plane_uses_fence(const struct intel_plane_state *plane_state) · 1090 unmodified lines
10911091
10921092 struct i915_vma *
10931093 intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
1094+ bool phys_cursor,
10941095 const struct i915_ggtt_view *view,
10951096 bool uses_fence,
10961097 unsigned long *out_flags)
@@ -1099,14 +1100,19 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, · 2 unmodified lines
10991100 struct drm_i915_private *dev_priv = to_i915(dev);
11001101 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
11011102 intel_wakeref_t wakeref;
1103+ struct i915_gem_ww_ctx ww;
11021104 struct i915_vma *vma;
11031105 unsigned int pinctl;
11041106 u32 alignment;
1107+ int ret;
11051108
11061109 if (drm_WARN_ON(dev, !i915_gem_object_is_framebuffer(obj)))
11071110 return ERR_PTR(-EINVAL);
11081111
1109− alignment = intel_surf_alignment(fb, 0);
1112+ if (phys_cursor)
1113+ alignment = intel_cursor_alignment(dev_priv);
1114+ else
1115+ alignment = intel_surf_alignment(fb, 0);
11101116 if (drm_WARN_ON(dev, alignment && !is_power_of_2(alignment)))
11111117 return ERR_PTR(-EINVAL);
11121118
@@ -1141,14 +1147,26 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, · 28 unmodified lines
11411147 if (HAS_GMCH(dev_priv))
11421148 pinctl |= PIN_MAPPABLE;
11431149
1144− vma = i915_gem_object_pin_to_display_plane(obj,
1145− alignment, view, pinctl);
1146− if (IS_ERR(vma))
1150+ i915_gem_ww_ctx_init(&ww, true);
1151+retry:
1152+ ret = i915_gem_object_lock(obj, &ww);
1153+ if (!ret && phys_cursor)
1154+ ret = i915_gem_object_attach_phys(obj, alignment);
1155+ if (!ret)
1156+ ret = i915_gem_object_pin_pages(obj);
1157+ if (ret)
11471158 goto err;
11481159
1149− if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
1150− int ret;
1160+ if (!ret) {
1161+ vma = i915_gem_object_pin_to_display_plane(obj, &ww, alignment,
1162+ view, pinctl);
1163+ if (IS_ERR(vma)) {
1164+ ret = PTR_ERR(vma);
1165+ goto err_unpin;
1166+ }
1167+ }
11511168
1169+ if (uses_fence && i915_vma_is_map_and_fenceable(vma)) {
11521170 /*
11531171 * Install a fence for tiled scan-out. Pre-i965 always needs a
11541172 * fence, whereas 965+ only requires a fence if using
@@ -1169,16 +1187,28 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, · 14 unmodified lines
11691187 ret = i915_vma_pin_fence(vma);
11701188 if (ret != 0 && INTEL_GEN(dev_priv) < 4) {
11711189 i915_vma_unpin(vma);
1172− vma = ERR_PTR(ret);
1173− goto err;
1190+ goto err_unpin;
11741191 }
1192+ ret = 0;
11751193
1176− if (ret == 0 && vma->fence)
1194+ if (vma->fence)
11771195 *out_flags |= PLANE_HAS_FENCE;
11781196 }
11791197
11801198 i915_vma_get(vma);
1199+
1200+err_unpin:
1201+ i915_gem_object_unpin_pages(obj);
11811202 err:
1203+ if (ret == -EDEADLK) {
1204+ ret = i915_gem_ww_ctx_backoff(&ww);
1205+ if (!ret)
1206+ goto retry;
1207+ }
1208+ i915_gem_ww_ctx_fini(&ww);
1209+ if (ret)
1210+ vma = ERR_PTR(ret);
1211+
11821212 atomic_dec(&dev_priv->gpu_error.pending_fb_pin);
11831213 intel_runtime_pm_put(&dev_priv->runtime_pm, wakeref);
11841214 return vma;
@@ -11333,19 +11363,11 @@ int intel_plane_pin_fb(struct intel_plane_state *plane_state) · 10148 unmodified lines
1133311363 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1133411364 struct drm_framebuffer *fb = plane_state->hw.fb;
1133511365 struct i915_vma *vma;
11366+ bool phys_cursor =
11367+ plane->id == PLANE_CURSOR &&
11368+ INTEL_INFO(dev_priv)->display.cursor_needs_physical;
1133611369
11337− if (plane->id == PLANE_CURSOR &&
11338− INTEL_INFO(dev_priv)->display.cursor_needs_physical) {
11339− struct drm_i915_gem_object *obj = intel_fb_obj(fb);
11340− const int align = intel_cursor_alignment(dev_priv);
11341− int err;
11342−
11343− err = i915_gem_object_attach_phys(obj, align);
11344− if (err)
11345− return err;
11346− }
11347−
11348− vma = intel_pin_and_fence_fb_obj(fb,
11370+ vma = intel_pin_and_fence_fb_obj(fb, phys_cursor,
1134911371 &plane_state->view,
1135011372 intel_plane_uses_fence(plane_state),
1135111373 &plane_state->flags);
@@ -11437,13 +11459,8 @@ intel_prepare_plane_fb(struct drm_plane *_plane, · 85 unmodified lines
1143711459 if (!obj)
1143811460 return 0;
1143911461
11440− ret = i915_gem_object_pin_pages(obj);
11441− if (ret)
11442− return ret;
1144311462
1144411463 ret = intel_plane_pin_fb(new_plane_state);
11445−
11446− i915_gem_object_unpin_pages(obj);
1144711464 if (ret)
1144811465 return ret;
1144911466
@@ -11905,7 +11922,7 @@ static int intel_user_framebuffer_create_handle(struct drm_framebuffer *fb, · 455 unmodified lines
1190511922 struct drm_i915_gem_object *obj = intel_fb_obj(fb);
1190611923 struct drm_i915_private *i915 = to_i915(obj->base.dev);
1190711924
11908− if (obj->userptr.mm) {
11925+ if (i915_gem_object_is_userptr(obj)) {
1190911926 drm_dbg(&i915->drm,
1191011927 "attempting to use a userptr for a framebuffer, denied\n");
1191111928 return -EINVAL;
drivers/gpu/drm/i915/display/intel_display.h+1−1
@@ -573,7 +573,7 @@ void intel_release_load_detect_pipe(struct drm_connector *connector, · 572 unmodified lines
573573 struct intel_load_detect_pipe *old,
574574 struct drm_modeset_acquire_ctx *ctx);
575575 struct i915_vma *
576−intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
576+intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb, bool phys_cursor,
577577 const struct i915_ggtt_view *view,
578578 bool uses_fence,
579579 unsigned long *out_flags);
drivers/gpu/drm/i915/display/intel_dsb.c+1−1
@@ -293,7 +293,7 @@ void intel_dsb_prepare(struct intel_crtc_state *crtc_state) · 292 unmodified lines
293293 goto out;
294294 }
295295
296− buf = i915_gem_object_pin_map(vma->obj, I915_MAP_WC);
296+ buf = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC);
297297 if (IS_ERR(buf)) {
298298 drm_err(&i915->drm, "Command buffer creation failed\n");
299299 i915_vma_unpin_and_release(&vma, I915_VMA_RELEASE_MAP);
drivers/gpu/drm/i915/display/intel_fbdev.c+1−1
@@ -211,7 +211,7 @@ static int intelfb_create(struct drm_fb_helper *helper, · 210 unmodified lines
211211 * This also validates that any existing fb inherited from the
212212 * BIOS is suitable for own access.
213213 */
214− vma = intel_pin_and_fence_fb_obj(&ifbdev->fb->base,
214+ vma = intel_pin_and_fence_fb_obj(&ifbdev->fb->base, false,
215215 &view, false, &flags);
216216 if (IS_ERR(vma)) {
217217 ret = PTR_ERR(vma);
drivers/gpu/drm/i915/display/intel_overlay.c+29−5
@@ -755,6 +755,32 @@ static u32 overlay_cmd_reg(struct drm_intel_overlay_put_image *params) · 754 unmodified lines
755755 return cmd;
756756 }
757757
758+static struct i915_vma *intel_overlay_pin_fb(struct drm_i915_gem_object *new_bo)
759+{
760+ struct i915_gem_ww_ctx ww;
761+ struct i915_vma *vma;
762+ int ret;
763+
764+ i915_gem_ww_ctx_init(&ww, true);
765+retry:
766+ ret = i915_gem_object_lock(new_bo, &ww);
767+ if (!ret) {
768+ vma = i915_gem_object_pin_to_display_plane(new_bo, &ww, 0,
769+ NULL, PIN_MAPPABLE);
770+ ret = PTR_ERR_OR_ZERO(vma);
771+ }
772+ if (ret == -EDEADLK) {
773+ ret = i915_gem_ww_ctx_backoff(&ww);
774+ if (!ret)
775+ goto retry;
776+ }
777+ i915_gem_ww_ctx_fini(&ww);
778+ if (ret)
779+ return ERR_PTR(ret);
780+
781+ return vma;
782+}
783+
758784 static int intel_overlay_do_put_image(struct intel_overlay *overlay,
759785 struct drm_i915_gem_object *new_bo,
760786 struct drm_intel_overlay_put_image *params)
@@ -776,12 +802,10 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay, · 15 unmodified lines
776802
777803 atomic_inc(&dev_priv->gpu_error.pending_fb_pin);
778804
779− vma = i915_gem_object_pin_to_display_plane(new_bo,
780− 0, NULL, PIN_MAPPABLE);
781− if (IS_ERR(vma)) {
782− ret = PTR_ERR(vma);
805+ vma = intel_overlay_pin_fb(new_bo);
806+ if (IS_ERR(vma))
783807 goto out_pin_section;
784− }
808+
785809 i915_gem_object_flush_frontbuffer(new_bo, ORIGIN_DIRTYFB);
786810
787811 if (!overlay->active) {
drivers/gpu/drm/i915/gem/i915_gem_clflush.c+7−8
@@ -27,15 +27,8 @@ static void __do_clflush(struct drm_i915_gem_object *obj) · 26 unmodified lines
2727 static int clflush_work(struct dma_fence_work *base)
2828 {
2929 struct clflush *clflush = container_of(base, typeof(*clflush), base);
30− struct drm_i915_gem_object *obj = clflush->obj;
31− int err;
3230
33− err = i915_gem_object_pin_pages(obj);
34− if (err)
35− return err;
36−
37− __do_clflush(obj);
38− i915_gem_object_unpin_pages(obj);
31+ __do_clflush(clflush->obj);
3932
4033 return 0;
4134 }
@@ -44,6 +37,7 @@ static void clflush_release(struct dma_fence_work *base) · 2 unmodified lines
4437 {
4538 struct clflush *clflush = container_of(base, typeof(*clflush), base);
4639
40+ i915_gem_object_unpin_pages(clflush->obj);
4741 i915_gem_object_put(clflush->obj);
4842 }
4943
@@ -63,6 +57,11 @@ static struct clflush *clflush_work_create(struct drm_i915_gem_object *obj) · 13 unmodified lines
6357 if (!clflush)
6458 return NULL;
6559
60+ if (__i915_gem_object_get_pages(obj) < 0) {
61+ kfree(clflush);
62+ return NULL;
63+ }
64+
6665 dma_fence_work_init(&clflush->base, &clflush_ops);
6766 clflush->obj = i915_gem_object_get(obj); /* obj <-> clflush cycle */
6867
drivers/gpu/drm/i915/gem/i915_gem_context.c+40−33
@@ -232,6 +232,8 @@ static void intel_context_set_gem(struct intel_context *ce, · 231 unmodified lines
232232 if (ctx->sched.priority >= I915_PRIORITY_NORMAL &&
233233 intel_engine_has_timeslices(ce->engine))
234234 __set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags);
235+
236+ intel_context_set_watchdog_us(ce, ctx->watchdog.timeout_us);
235237 }
236238
237239 static void __free_engines(struct i915_gem_engines *e, unsigned int count)
@@ -386,38 +388,6 @@ static bool __cancel_engine(struct intel_engine_cs *engine) · 148 unmodified lines
386388 return intel_engine_pulse(engine) == 0;
387389 }
388390
389−static bool
390−__active_engine(struct i915_request *rq, struct intel_engine_cs **active)
391−{
392− struct intel_engine_cs *engine, *locked;
393− bool ret = false;
394−
395− /*
396− * Serialise with __i915_request_submit() so that it sees
397− * is-banned?, or we know the request is already inflight.
398− *
399− * Note that rq->engine is unstable, and so we double
400− * check that we have acquired the lock on the final engine.
401− */
402− locked = READ_ONCE(rq->engine);
403− spin_lock_irq(&locked->active.lock);
404− while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) {
405− spin_unlock(&locked->active.lock);
406− locked = engine;
407− spin_lock(&locked->active.lock);
408− }
409−
410− if (i915_request_is_active(rq)) {
411− if (!__i915_request_is_complete(rq))
412− *active = locked;
413− ret = true;
414− }
415−
416− spin_unlock_irq(&locked->active.lock);
417−
418− return ret;
419−}
420−
421391 static struct intel_engine_cs *active_engine(struct intel_context *ce)
422392 {
423393 struct intel_engine_cs *engine = NULL;
@@ -445,7 +415,7 @@ static struct intel_engine_cs *active_engine(struct intel_context *ce) · 21 unmodified lines
445415 /* Check with the backend if the request is inflight */
446416 found = true;
447417 if (likely(rcu_access_pointer(rq->timeline) == ce->timeline))
448− found = __active_engine(rq, &engine);
418+ found = i915_request_active_engine(rq, &engine);
449419
450420 i915_request_put(rq);
451421 if (found)
@@ -822,6 +792,41 @@ static void __assign_timeline(struct i915_gem_context *ctx, · 370 unmodified lines
822792 context_apply_all(ctx, __apply_timeline, timeline);
823793 }
824794
795+static int __apply_watchdog(struct intel_context *ce, void *timeout_us)
796+{
797+ return intel_context_set_watchdog_us(ce, (uintptr_t)timeout_us);
798+}
799+
800+static int
801+__set_watchdog(struct i915_gem_context *ctx, unsigned long timeout_us)
802+{
803+ int ret;
804+
805+ ret = context_apply_all(ctx, __apply_watchdog,
806+ (void *)(uintptr_t)timeout_us);
807+ if (!ret)
808+ ctx->watchdog.timeout_us = timeout_us;
809+
810+ return ret;
811+}
812+
813+static void __set_default_fence_expiry(struct i915_gem_context *ctx)
814+{
815+ struct drm_i915_private *i915 = ctx->i915;
816+ int ret;
817+
818+ if (!IS_ACTIVE(CONFIG_DRM_I915_REQUEST_TIMEOUT) ||
819+ !i915->params.request_timeout_ms)
820+ return;
821+
822+ /* Default expiry for user fences. */
823+ ret = __set_watchdog(ctx, i915->params.request_timeout_ms * 1000);
824+ if (ret)
825+ drm_notice(&i915->drm,
826+ "Failed to configure default fence expiry! (%d)",
827+ ret);
828+}
829+
825830 static struct i915_gem_context *
826831 i915_gem_create_context(struct drm_i915_private *i915, unsigned int flags)
827832 {
@@ -866,6 +871,8 @@ i915_gem_create_context(struct drm_i915_private *i915, unsigned int flags) · 38 unmodified lines
866871 intel_timeline_put(timeline);
867872 }
868873
874+ __set_default_fence_expiry(ctx);
875+
869876 trace_i915_context_create(ctx);
870877
871878 return ctx;
drivers/gpu/drm/i915/gem/i915_gem_context_types.h+4−0
@@ -154,6 +154,10 @@ struct i915_gem_context { · 153 unmodified lines
154154 */
155155 atomic_t active_count;
156156
157+ struct {
158+ u64 timeout_us;
159+ } watchdog;
160+
157161 /**
158162 * @hang_timestamp: The last time(s) this context caused a GPU hang
159163 */
drivers/gpu/drm/i915/gem/i915_gem_dmabuf.c+35−29
@@ -25,7 +25,7 @@ static struct sg_table *i915_gem_map_dma_buf(struct dma_buf_attachment *attachme · 24 unmodified lines
2525 struct scatterlist *src, *dst;
2626 int ret, i;
2727
28− ret = i915_gem_object_pin_pages(obj);
28+ ret = i915_gem_object_pin_pages_unlocked(obj);
2929 if (ret)
3030 goto err;
3131
@@ -82,7 +82,7 @@ static int i915_gem_dmabuf_vmap(struct dma_buf *dma_buf, struct dma_buf_map *map · 50 unmodified lines
8282 struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
8383 void *vaddr;
8484
85− vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
85+ vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
8686 if (IS_ERR(vaddr))
8787 return PTR_ERR(vaddr);
8888
@@ -123,42 +123,48 @@ static int i915_gem_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_dire · 34 unmodified lines
123123 {
124124 struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
125125 bool write = (direction == DMA_BIDIRECTIONAL || direction == DMA_TO_DEVICE);
126+ struct i915_gem_ww_ctx ww;
126127 int err;
127128
128− err = i915_gem_object_pin_pages(obj);
129− if (err)
130− return err;
131−
132− err = i915_gem_object_lock_interruptible(obj, NULL);
133− if (err)
134− goto out;
135−
136− err = i915_gem_object_set_to_cpu_domain(obj, write);
137− i915_gem_object_unlock(obj);
138−
139−out:
140− i915_gem_object_unpin_pages(obj);
129+ i915_gem_ww_ctx_init(&ww, true);
130+retry:
131+ err = i915_gem_object_lock(obj, &ww);
132+ if (!err)
133+ err = i915_gem_object_pin_pages(obj);
134+ if (!err) {
135+ err = i915_gem_object_set_to_cpu_domain(obj, write);
136+ i915_gem_object_unpin_pages(obj);
137+ }
138+ if (err == -EDEADLK) {
139+ err = i915_gem_ww_ctx_backoff(&ww);
140+ if (!err)
141+ goto retry;
142+ }
143+ i915_gem_ww_ctx_fini(&ww);
141144 return err;
142145 }
143146
144147 static int i915_gem_end_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction direction)
145148 {
146149 struct drm_i915_gem_object *obj = dma_buf_to_obj(dma_buf);
150+ struct i915_gem_ww_ctx ww;
147151 int err;
148152
149− err = i915_gem_object_pin_pages(obj);
150− if (err)
151− return err;
152−
153− err = i915_gem_object_lock_interruptible(obj, NULL);
154− if (err)
155− goto out;
156−
157− err = i915_gem_object_set_to_gtt_domain(obj, false);
158− i915_gem_object_unlock(obj);
159−
160−out:
161− i915_gem_object_unpin_pages(obj);
153+ i915_gem_ww_ctx_init(&ww, true);
154+retry:
155+ err = i915_gem_object_lock(obj, &ww);
156+ if (!err)
157+ err = i915_gem_object_pin_pages(obj);
158+ if (!err) {
159+ err = i915_gem_object_set_to_gtt_domain(obj, false);
160+ i915_gem_object_unpin_pages(obj);
161+ }
162+ if (err == -EDEADLK) {
163+ err = i915_gem_ww_ctx_backoff(&ww);
164+ if (!err)
165+ goto retry;
166+ }
167+ i915_gem_ww_ctx_fini(&ww);
162168 return err;
163169 }
164170
@@ -258,7 +264,7 @@ struct drm_gem_object *i915_gem_prime_import(struct drm_device *dev, · 93 unmodified lines
258264 }
259265
260266 drm_gem_private_object_init(dev, &obj->base, dma_buf->size);
261− i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops, &lock_class);
267+ i915_gem_object_init(obj, &i915_gem_object_dmabuf_ops, &lock_class, 0);
262268 obj->base.import_attach = attach;
263269 obj->base.resv = dma_buf->resv;
264270
drivers/gpu/drm/i915/gem/i915_gem_domain.c+39−33
@@ -335,7 +335,14 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data, · 334 unmodified lines
335335 * not allowed to be changed by userspace.
336336 */
337337 if (i915_gem_object_is_proxy(obj)) {
338− ret = -ENXIO;
338+ /*
339+ * Silently allow cached for userptr; the vulkan driver
340+ * sets all objects to cached
341+ */
342+ if (!i915_gem_object_is_userptr(obj) ||
343+ args->caching != I915_CACHING_CACHED)
344+ ret = -ENXIO;
345+
339346 goto out;
340347 }
341348
@@ -359,12 +366,12 @@ out: · 17 unmodified lines
359366 */
360367 struct i915_vma *
361368 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
369+ struct i915_gem_ww_ctx *ww,
362370 u32 alignment,
363371 const struct i915_ggtt_view *view,
364372 unsigned int flags)
365373 {
366374 struct drm_i915_private *i915 = to_i915(obj->base.dev);
367− struct i915_gem_ww_ctx ww;
368375 struct i915_vma *vma;
369376 int ret;
370377
@@ -372,11 +379,6 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, · 1 unmodified line
372379 if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj))
373380 return ERR_PTR(-EINVAL);
374381
375− i915_gem_ww_ctx_init(&ww, true);
376−retry:
377− ret = i915_gem_object_lock(obj, &ww);
378− if (ret)
379− goto err;
380382 /*
381383 * The display engine is not coherent with the LLC cache on gen6. As
382384 * a result, we make sure that the pinning that is about to occur is
@@ -391,7 +393,7 @@ retry: · 8 unmodified lines
391393 HAS_WT(i915) ?
392394 I915_CACHE_WT : I915_CACHE_NONE);
393395 if (ret)
394− goto err;
396+ return ERR_PTR(ret);
395397
396398 /*
397399 * As the user may map the buffer once pinned in the display plane
@@ -404,33 +406,20 @@ retry: · 6 unmodified lines
404406 vma = ERR_PTR(-ENOSPC);
405407 if ((flags & PIN_MAPPABLE) == 0 &&
406408 (!view || view->type == I915_GGTT_VIEW_NORMAL))
407− vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0, alignment,
409+ vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0, alignment,
408410 flags | PIN_MAPPABLE |
409411 PIN_NONBLOCK);
410412 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK))
411− vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0,
413+ vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0,
412414 alignment, flags);
413− if (IS_ERR(vma)) {
414− ret = PTR_ERR(vma);
415− goto err;
416− }
415+ if (IS_ERR(vma))
416+ return vma;
417417
418418 vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
419419 i915_vma_mark_scanout(vma);
420420
421421 i915_gem_object_flush_if_display_locked(obj);
422422
423−err:
424− if (ret == -EDEADLK) {
425− ret = i915_gem_ww_ctx_backoff(&ww);
426− if (!ret)
427− goto retry;
428− }
429− i915_gem_ww_ctx_fini(&ww);
430−
431− if (ret)
432− return ERR_PTR(ret);
433−
434423 return vma;
435424 }
436425
@@ -526,6 +515,21 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, · 89 unmodified lines
526515 if (err)
527516 goto out;
528517
518+ if (i915_gem_object_is_userptr(obj)) {
519+ /*
520+ * Try to grab userptr pages, iris uses set_domain to check
521+ * userptr validity
522+ */
523+ err = i915_gem_object_userptr_validate(obj);
524+ if (!err)
525+ err = i915_gem_object_wait(obj,
526+ I915_WAIT_INTERRUPTIBLE |
527+ I915_WAIT_PRIORITY |
528+ (write_domain ? I915_WAIT_ALL : 0),
529+ MAX_SCHEDULE_TIMEOUT);
530+ goto out;
531+ }
532+
529533 /*
530534 * Proxy objects do not control access to the backing storage, ergo
531535 * they cannot be used as a means to manipulate the cache domain
@@ -537,6 +541,10 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, · 5 unmodified lines
537541 goto out;
538542 }
539543
544+ err = i915_gem_object_lock_interruptible(obj, NULL);
545+ if (err)
546+ goto out;
547+
540548 /*
541549 * Flush and acquire obj->pages so that we are coherent through
542550 * direct access in memory with previous cached writes through
@@ -548,7 +556,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, · 5 unmodified lines
548556 */
549557 err = i915_gem_object_pin_pages(obj);
550558 if (err)
551− goto out;
559+ goto out_unlock;
552560
553561 /*
554562 * Already in the desired write domain? Nothing for us to do!
@@ -563,10 +571,6 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, · 8 unmodified lines
563571 if (READ_ONCE(obj->write_domain) == read_domains)
564572 goto out_unpin;
565573
566− err = i915_gem_object_lock_interruptible(obj, NULL);
567− if (err)
568− goto out_unpin;
569−
570574 if (read_domains & I915_GEM_DOMAIN_WC)
571575 err = i915_gem_object_set_to_wc_domain(obj, write_domain);
572576 else if (read_domains & I915_GEM_DOMAIN_GTT)
@@ -574,13 +578,15 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, · 1 unmodified line
574578 else
575579 err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
576580
581+out_unpin:
582+ i915_gem_object_unpin_pages(obj);
583+
584+out_unlock:
577585 i915_gem_object_unlock(obj);
578586
579− if (write_domain)
587+ if (!err && write_domain)
580588 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
581589
582−out_unpin:
583− i915_gem_object_unpin_pages(obj);
584590 out:
585591 i915_gem_object_put(obj);
586592 return err;
drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c+204−131
@@ -28,6 +28,7 @@ · 27 unmodified lines
2828 #include "i915_sw_fence_work.h"
2929 #include "i915_trace.h"
3030 #include "i915_user_extensions.h"
31+#include "i915_memcpy.h"
3132
3233 struct eb_vma {
3334 struct i915_vma *vma;
@@ -49,16 +50,19 @@ enum { · 15 unmodified lines
4950 #define DBG_FORCE_RELOC 0 /* choose one of the above! */
5051 };
5152
52−#define __EXEC_OBJECT_HAS_PIN BIT(31)
53−#define __EXEC_OBJECT_HAS_FENCE BIT(30)
54−#define __EXEC_OBJECT_NEEDS_MAP BIT(29)
55−#define __EXEC_OBJECT_NEEDS_BIAS BIT(28)
56−#define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 28) /* all of the above */
53+/* __EXEC_OBJECT_NO_RESERVE is BIT(31), defined in i915_vma.h */
54+#define __EXEC_OBJECT_HAS_PIN BIT(30)
55+#define __EXEC_OBJECT_HAS_FENCE BIT(29)
56+#define __EXEC_OBJECT_USERPTR_INIT BIT(28)
57+#define __EXEC_OBJECT_NEEDS_MAP BIT(27)
58+#define __EXEC_OBJECT_NEEDS_BIAS BIT(26)
59+#define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 26) /* all of the above + */
5760 #define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE)
5861
5962 #define __EXEC_HAS_RELOC BIT(31)
6063 #define __EXEC_ENGINE_PINNED BIT(30)
61−#define __EXEC_INTERNAL_FLAGS (~0u << 30)
64+#define __EXEC_USERPTR_USED BIT(29)
65+#define __EXEC_INTERNAL_FLAGS (~0u << 29)
6266 #define UPDATE PIN_OFFSET_FIXED
6367
6468 #define BATCH_OFFSET_BIAS (256*1024)
@@ -419,13 +423,14 @@ static u64 eb_pin_flags(const struct drm_i915_gem_exec_object2 *entry, · 354 unmodified lines
419423 return pin_flags;
420424 }
421425
422−static inline bool
426+static inline int
423427 eb_pin_vma(struct i915_execbuffer *eb,
424428 const struct drm_i915_gem_exec_object2 *entry,
425429 struct eb_vma *ev)
426430 {
427431 struct i915_vma *vma = ev->vma;
428432 u64 pin_flags;
433+ int err;
429434
430435 if (vma->node.size)
431436 pin_flags = vma->node.start;
@@ -437,24 +442,29 @@ eb_pin_vma(struct i915_execbuffer *eb, · 5 unmodified lines
437442 pin_flags |= PIN_GLOBAL;
438443
439444 /* Attempt to reuse the current location if available */
440− /* TODO: Add -EDEADLK handling here */
441− if (unlikely(i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags))) {
445+ err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags);
446+ if (err == -EDEADLK)
447+ return err;
448+
449+ if (unlikely(err)) {
442450 if (entry->flags & EXEC_OBJECT_PINNED)
443− return false;
451+ return err;
444452
445453 /* Failing that pick any _free_ space if suitable */
446− if (unlikely(i915_vma_pin_ww(vma, &eb->ww,
454+ err = i915_vma_pin_ww(vma, &eb->ww,
447455 entry->pad_to_size,
448456 entry->alignment,
449457 eb_pin_flags(entry, ev->flags) |
450− PIN_USER | PIN_NOEVICT)))
451− return false;
458+ PIN_USER | PIN_NOEVICT);
459+ if (unlikely(err))
460+ return err;
452461 }
453462
454463 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) {
455− if (unlikely(i915_vma_pin_fence(vma))) {
464+ err = i915_vma_pin_fence(vma);
465+ if (unlikely(err)) {
456466 i915_vma_unpin(vma);
457− return false;
467+ return err;
458468 }
459469
460470 if (vma->fence)
@@ -462,7 +472,10 @@ eb_pin_vma(struct i915_execbuffer *eb, · 1 unmodified line
462472 }
463473
464474 ev->flags |= __EXEC_OBJECT_HAS_PIN;
465− return !eb_vma_misplaced(entry, vma, ev->flags);
475+ if (eb_vma_misplaced(entry, vma, ev->flags))
476+ return -EBADSLT;
477+
478+ return 0;
466479 }
467480
468481 static inline void
@@ -483,6 +496,13 @@ eb_validate_vma(struct i915_execbuffer *eb, · 14 unmodified lines
483496 struct drm_i915_gem_exec_object2 *entry,
484497 struct i915_vma *vma)
485498 {
499+ /* Relocations are disallowed for all platforms after TGL-LP. This
500+ * also covers all platforms with local memory.
501+ */
502+ if (entry->relocation_count &&
503+ INTEL_GEN(eb->i915) >= 12 && !IS_TIGERLAKE(eb->i915))
504+ return -EINVAL;
505+
486506 if (unlikely(entry->flags & eb->invalid_flags))
487507 return -EINVAL;
488508
@@ -853,6 +873,26 @@ static int eb_lookup_vmas(struct i915_execbuffer *eb) · 364 unmodified lines
853873 }
854874
855875 eb_add_vma(eb, i, batch, vma);
876+
877+ if (i915_gem_object_is_userptr(vma->obj)) {
878+ err = i915_gem_object_userptr_submit_init(vma->obj);
879+ if (err) {
880+ if (i + 1 < eb->buffer_count) {
881+ /*
882+ * Execbuffer code expects last vma entry to be NULL,
883+ * since we already initialized this entry,
884+ * set the next value to NULL or we mess up
885+ * cleanup handling.
886+ */
887+ eb->vma[i + 1].vma = NULL;
888+ }
889+
890+ return err;
891+ }
892+
893+ eb->vma[i].flags |= __EXEC_OBJECT_USERPTR_INIT;
894+ eb->args->flags |= __EXEC_USERPTR_USED;
895+ }
856896 }
857897
858898 if (unlikely(eb->batch->flags & EXEC_OBJECT_WRITE)) {
@@ -898,7 +938,11 @@ static int eb_validate_vmas(struct i915_execbuffer *eb) · 39 unmodified lines
898938 if (err)
899939 return err;
900940
901− if (eb_pin_vma(eb, entry, ev)) {
941+ err = eb_pin_vma(eb, entry, ev);
942+ if (err == -EDEADLK)
943+ return err;
944+
945+ if (!err) {
902946 if (entry->offset != vma->node.start) {
903947 entry->offset = vma->node.start | UPDATE;
904948 eb->args->flags |= __EXEC_HAS_RELOC;
@@ -914,6 +958,12 @@ static int eb_validate_vmas(struct i915_execbuffer *eb) · 9 unmodified lines
914958 }
915959 }
916960
961+ if (!(ev->flags & EXEC_OBJECT_WRITE)) {
962+ err = dma_resv_reserve_shared(vma->resv, 1);
963+ if (err)
964+ return err;
965+ }
966+
917967 GEM_BUG_ON(drm_mm_node_allocated(&vma->node) &&
918968 eb_vma_misplaced(&eb->exec[i], vma, ev->flags));
919969 }
@@ -944,7 +994,7 @@ eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle) · 24 unmodified lines
944994 }
945995 }
946996
947−static void eb_release_vmas(struct i915_execbuffer *eb, bool final)
997+static void eb_release_vmas(struct i915_execbuffer *eb, bool final, bool release_userptr)
948998 {
949999 const unsigned int count = eb->buffer_count;
9501000 unsigned int i;
@@ -958,6 +1008,11 @@ static void eb_release_vmas(struct i915_execbuffer *eb, bool final) · 7 unmodified lines
9581008
9591009 eb_unreserve_vma(ev);
9601010
1011+ if (release_userptr && ev->flags & __EXEC_OBJECT_USERPTR_INIT) {
1012+ ev->flags &= ~__EXEC_OBJECT_USERPTR_INIT;
1013+ i915_gem_object_userptr_submit_fini(vma->obj);
1014+ }
1015+
9611016 if (final)
9621017 i915_vma_put(vma);
9631018 }
@@ -1294,6 +1349,7 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb, · 330 unmodified lines
12941349 err = PTR_ERR(cmd);
12951350 goto err_pool;
12961351 }
1352+ intel_gt_buffer_pool_mark_used(pool);
12971353
12981354 memset32(cmd, 0, pool->obj->base.size / sizeof(u32));
12991355
@@ -1895,6 +1951,31 @@ static int eb_prefault_relocations(const struct i915_execbuffer *eb) · 595 unmodified lines
18951951 return 0;
18961952 }
18971953
1954+static int eb_reinit_userptr(struct i915_execbuffer *eb)
1955+{
1956+ const unsigned int count = eb->buffer_count;
1957+ unsigned int i;
1958+ int ret;
1959+
1960+ if (likely(!(eb->args->flags & __EXEC_USERPTR_USED)))
1961+ return 0;
1962+
1963+ for (i = 0; i < count; i++) {
1964+ struct eb_vma *ev = &eb->vma[i];
1965+
1966+ if (!i915_gem_object_is_userptr(ev->vma->obj))
1967+ continue;
1968+
1969+ ret = i915_gem_object_userptr_submit_init(ev->vma->obj);
1970+ if (ret)
1971+ return ret;
1972+
1973+ ev->flags |= __EXEC_OBJECT_USERPTR_INIT;
1974+ }
1975+
1976+ return 0;
1977+}
1978+
18981979 static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb,
18991980 struct i915_request *rq)
19001981 {
@@ -1909,7 +1990,7 @@ repeat: · 8 unmodified lines
19091990 }
19101991
19111992 /* We may process another execbuffer during the unlock... */
1912− eb_release_vmas(eb, false);
1993+ eb_release_vmas(eb, false, true);
19131994 i915_gem_ww_ctx_fini(&eb->ww);
19141995
19151996 if (rq) {
@@ -1951,7 +2032,7 @@ repeat: · 35 unmodified lines
19512032 }
19522033
19532034 if (!err)
1954− flush_workqueue(eb->i915->mm.userptr_wq);
2035+ err = eb_reinit_userptr(eb);
19552036
19562037 err_relock:
19572038 i915_gem_ww_ctx_init(&eb->ww, true);
@@ -2013,7 +2094,7 @@ repeat_validate: · 55 unmodified lines
20132094
20142095 err:
20152096 if (err == -EDEADLK) {
2016− eb_release_vmas(eb, false);
2097+ eb_release_vmas(eb, false, false);
20172098 err = i915_gem_ww_ctx_backoff(&eb->ww);
20182099 if (!err)
20192100 goto repeat_validate;
@@ -2110,7 +2191,7 @@ retry: · 90 unmodified lines
21102191
21112192 err:
21122193 if (err == -EDEADLK) {
2113− eb_release_vmas(eb, false);
2194+ eb_release_vmas(eb, false, false);
21142195 err = i915_gem_ww_ctx_backoff(&eb->ww);
21152196 if (!err)
21162197 goto retry;
@@ -2181,8 +2262,33 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb) · 64 unmodified lines
21812262 }
21822263
21832264 if (err == 0)
2184− err = i915_vma_move_to_active(vma, eb->request, flags);
2265+ err = i915_vma_move_to_active(vma, eb->request,
2266+ flags | __EXEC_OBJECT_NO_RESERVE);
2267+ }
2268+
2269+#ifdef CONFIG_MMU_NOTIFIER
2270+ if (!err && (eb->args->flags & __EXEC_USERPTR_USED)) {
2271+ spin_lock(&eb->i915->mm.notifier_lock);
2272+
2273+ /*
2274+ * count is always at least 1, otherwise __EXEC_USERPTR_USED
2275+ * could not have been set
2276+ */
2277+ for (i = 0; i < count; i++) {
2278+ struct eb_vma *ev = &eb->vma[i];
2279+ struct drm_i915_gem_object *obj = ev->vma->obj;
2280+
2281+ if (!i915_gem_object_is_userptr(obj))
2282+ continue;
2283+
2284+ err = i915_gem_object_userptr_submit_done(obj);
2285+ if (err)
2286+ break;
2287+ }
2288+
2289+ spin_unlock(&eb->i915->mm.notifier_lock);
21852290 }
2291+#endif
21862292
21872293 if (unlikely(err))
21882294 goto err_skip;
@@ -2274,24 +2380,45 @@ struct eb_parse_work { · 85 unmodified lines
22742380 struct i915_vma *trampoline;
22752381 unsigned long batch_offset;
22762382 unsigned long batch_length;
2383+ unsigned long *jump_whitelist;
2384+ const void *batch_map;
2385+ void *shadow_map;
22772386 };
22782387
22792388 static int __eb_parse(struct dma_fence_work *work)
22802389 {
22812390 struct eb_parse_work *pw = container_of(work, typeof(*pw), base);
2391+ int ret;
2392+ bool cookie;
22822393
2283− return intel_engine_cmd_parser(pw->engine,
2284− pw->batch,
2285− pw->batch_offset,
2286− pw->batch_length,
2287− pw->shadow,
2288− pw->trampoline);
2394+ cookie = dma_fence_begin_signalling();
2395+ ret = intel_engine_cmd_parser(pw->engine,
2396+ pw->batch,
2397+ pw->batch_offset,
2398+ pw->batch_length,
2399+ pw->shadow,
2400+ pw->jump_whitelist,
2401+ pw->shadow_map,
2402+ pw->batch_map);
2403+ dma_fence_end_signalling(cookie);
2404+
2405+ return ret;
22892406 }
22902407
22912408 static void __eb_parse_release(struct dma_fence_work *work)
22922409 {
22932410 struct eb_parse_work *pw = container_of(work, typeof(*pw), base);
22942411
2412+ if (!IS_ERR_OR_NULL(pw->jump_whitelist))
2413+ kfree(pw->jump_whitelist);
2414+
2415+ if (pw->batch_map)
2416+ i915_gem_object_unpin_map(pw->batch->obj);
2417+ else
2418+ i915_gem_object_unpin_pages(pw->batch->obj);
2419+
2420+ i915_gem_object_unpin_map(pw->shadow->obj);
2421+
22952422 if (pw->trampoline)
22962423 i915_active_release(&pw->trampoline->active);
22972424 i915_active_release(&pw->shadow->active);
@@ -2341,6 +2468,8 @@ static int eb_parse_pipeline(struct i915_execbuffer *eb, · 43 unmodified lines
23412468 struct i915_vma *trampoline)
23422469 {
23432470 struct eb_parse_work *pw;
2471+ struct drm_i915_gem_object *batch = eb->batch->vma->obj;
2472+ bool needs_clflush;
23442473 int err;
23452474
23462475 GEM_BUG_ON(overflows_type(eb->batch_start_offset, pw->batch_offset));
@@ -2364,6 +2493,34 @@ static int eb_parse_pipeline(struct i915_execbuffer *eb, · 17 unmodified lines
23642493 goto err_shadow;
23652494 }
23662495
2496+ pw->shadow_map = i915_gem_object_pin_map(shadow->obj, I915_MAP_WB);
2497+ if (IS_ERR(pw->shadow_map)) {
2498+ err = PTR_ERR(pw->shadow_map);
2499+ goto err_trampoline;
2500+ }
2501+
2502+ needs_clflush =
2503+ !(batch->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ);
2504+
2505+ pw->batch_map = ERR_PTR(-ENODEV);
2506+ if (needs_clflush && i915_has_memcpy_from_wc())
2507+ pw->batch_map = i915_gem_object_pin_map(batch, I915_MAP_WC);
2508+
2509+ if (IS_ERR(pw->batch_map)) {
2510+ err = i915_gem_object_pin_pages(batch);
2511+ if (err)
2512+ goto err_unmap_shadow;
2513+ pw->batch_map = NULL;
2514+ }
2515+
2516+ pw->jump_whitelist =
2517+ intel_engine_cmd_parser_alloc_jump_whitelist(eb->batch_len,
2518+ trampoline);
2519+ if (IS_ERR(pw->jump_whitelist)) {
2520+ err = PTR_ERR(pw->jump_whitelist);
2521+ goto err_unmap_batch;
2522+ }
2523+
23672524 dma_fence_work_init(&pw->base, &eb_parse_ops);
23682525
23692526 pw->engine = eb->engine;
@@ -2382,6 +2539,10 @@ static int eb_parse_pipeline(struct i915_execbuffer *eb, · 12 unmodified lines
23822539 if (err)
23832540 goto err_commit;
23842541
2542+ err = dma_resv_reserve_shared(shadow->resv, 1);
2543+ if (err)
2544+ goto err_commit;
2545+
23852546 /* Wait for all writes (and relocs) into the batch to complete */
23862547 err = i915_sw_fence_await_reservation(&pw->base.chain,
23872548 pw->batch->resv, NULL, false,
@@ -2403,6 +2564,16 @@ err_commit: · 15 unmodified lines
24032564 dma_fence_work_commit_imm(&pw->base);
24042565 return err;
24052566
2567+err_unmap_batch:
2568+ if (pw->batch_map)
2569+ i915_gem_object_unpin_map(batch);
2570+ else
2571+ i915_gem_object_unpin_pages(batch);
2572+err_unmap_shadow:
2573+ i915_gem_object_unpin_map(shadow->obj);
2574+err_trampoline:
2575+ if (trampoline)
2576+ i915_active_release(&trampoline->active);
24062577 err_shadow:
24072578 i915_active_release(&shadow->active);
24082579 err_batch:
@@ -2474,6 +2645,7 @@ static int eb_parse(struct i915_execbuffer *eb) · 65 unmodified lines
24742645 err = PTR_ERR(shadow);
24752646 goto err;
24762647 }
2648+ intel_gt_buffer_pool_mark_used(pool);
24772649 i915_gem_object_set_readonly(shadow->obj);
24782650 shadow->private = pool;
24792651
@@ -3263,7 +3435,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, · 783 unmodified lines
32633435
32643436 err = eb_lookup_vmas(&eb);
32653437 if (err) {
3266− eb_release_vmas(&eb, true);
3438+ eb_release_vmas(&eb, true, true);
32673439 goto err_engine;
32683440 }
32693441
@@ -3335,6 +3507,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, · 65 unmodified lines
33353507
33363508 trace_i915_request_queue(eb.request, eb.batch_flags);
33373509 err = eb_submit(&eb, batch);
3510+
33383511 err_request:
33393512 i915_request_get(eb.request);
33403513 err = eb_request_add(&eb, err);
@@ -3355,7 +3528,7 @@ err_request: · 14 unmodified lines
33553528 i915_request_put(eb.request);
33563529
33573530 err_vma:
3358− eb_release_vmas(&eb, true);
3531+ eb_release_vmas(&eb, true, true);
33593532 if (eb.trampoline)
33603533 i915_vma_unpin(eb.trampoline);
33613534 WARN_ON(err == -EDEADLK);
@@ -3401,106 +3574,6 @@ static bool check_buffer_count(size_t count) · 39 unmodified lines
34013574 return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1);
34023575 }
34033576
3404−/*
3405− * Legacy execbuffer just creates an exec2 list from the original exec object
3406− * list array and passes it to the real function.
3407− */
3408−int
3409−i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data,
3410− struct drm_file *file)
3411−{
3412− struct drm_i915_private *i915 = to_i915(dev);
3413− struct drm_i915_gem_execbuffer *args = data;
3414− struct drm_i915_gem_execbuffer2 exec2;
3415− struct drm_i915_gem_exec_object *exec_list = NULL;
3416− struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3417− const size_t count = args->buffer_count;
3418− unsigned int i;
3419− int err;
3420−
3421− if (!check_buffer_count(count)) {
3422− drm_dbg(&i915->drm, "execbuf2 with %zd buffers\n", count);
3423− return -EINVAL;
3424− }
3425−
3426− exec2.buffers_ptr = args->buffers_ptr;
3427− exec2.buffer_count = args->buffer_count;
3428− exec2.batch_start_offset = args->batch_start_offset;
3429− exec2.batch_len = args->batch_len;
3430− exec2.DR1 = args->DR1;
3431− exec2.DR4 = args->DR4;
3432− exec2.num_cliprects = args->num_cliprects;
3433− exec2.cliprects_ptr = args->cliprects_ptr;
3434− exec2.flags = I915_EXEC_RENDER;
3435− i915_execbuffer2_set_context_id(exec2, 0);
3436−
3437− err = i915_gem_check_execbuffer(&exec2);
3438− if (err)
3439− return err;
3440−
3441− /* Copy in the exec list from userland */
3442− exec_list = kvmalloc_array(count, sizeof(*exec_list),
3443− __GFP_NOWARN | GFP_KERNEL);
3444−
3445− /* Allocate extra slots for use by the command parser */
3446− exec2_list = kvmalloc_array(count + 2, eb_element_size(),
3447− __GFP_NOWARN | GFP_KERNEL);
3448− if (exec_list == NULL || exec2_list == NULL) {
3449− drm_dbg(&i915->drm,
3450− "Failed to allocate exec list for %d buffers\n",
3451− args->buffer_count);
3452− kvfree(exec_list);
3453− kvfree(exec2_list);
3454− return -ENOMEM;
3455− }
3456− err = copy_from_user(exec_list,
3457− u64_to_user_ptr(args->buffers_ptr),
3458− sizeof(*exec_list) * count);
3459− if (err) {
3460− drm_dbg(&i915->drm, "copy %d exec entries failed %d\n",
3461− args->buffer_count, err);
3462− kvfree(exec_list);
3463− kvfree(exec2_list);
3464− return -EFAULT;
3465− }
3466−
3467− for (i = 0; i < args->buffer_count; i++) {
3468− exec2_list[i].handle = exec_list[i].handle;
3469− exec2_list[i].relocation_count = exec_list[i].relocation_count;
3470− exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3471− exec2_list[i].alignment = exec_list[i].alignment;
3472− exec2_list[i].offset = exec_list[i].offset;
3473− if (INTEL_GEN(to_i915(dev)) < 4)
3474− exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3475− else
3476− exec2_list[i].flags = 0;
3477− }
3478−
3479− err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list);
3480− if (exec2.flags & __EXEC_HAS_RELOC) {
3481− struct drm_i915_gem_exec_object __user *user_exec_list =
3482− u64_to_user_ptr(args->buffers_ptr);
3483−
3484− /* Copy the new buffer offsets back to the user's exec list. */
3485− for (i = 0; i < args->buffer_count; i++) {
3486− if (!(exec2_list[i].offset & UPDATE))
3487− continue;
3488−
3489− exec2_list[i].offset =
3490− gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK);
3491− exec2_list[i].offset &= PIN_OFFSET_MASK;
3492− if (__copy_to_user(&user_exec_list[i].offset,
3493− &exec2_list[i].offset,
3494− sizeof(user_exec_list[i].offset)))
3495− break;
3496− }
3497− }
3498−
3499− kvfree(exec_list);
3500− kvfree(exec2_list);
3501− return err;
3502−}
3503−
35043577 int
35053578 i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data,
35063579 struct drm_file *file)
drivers/gpu/drm/i915/gem/i915_gem_fence.c+0−95
@@ -1,95 +0,0 @@
1−/*
2− * SPDX-License-Identifier: MIT
3− *
4− * Copyright © 2019 Intel Corporation
5− */
6−
7−#include "i915_drv.h"
8−#include "i915_gem_object.h"
9−
10−struct stub_fence {
11− struct dma_fence dma;
12− struct i915_sw_fence chain;
13−};
14−
15−static int __i915_sw_fence_call
16−stub_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
17−{
18− struct stub_fence *stub = container_of(fence, typeof(*stub), chain);
19−
20− switch (state) {
21− case FENCE_COMPLETE:
22− dma_fence_signal(&stub->dma);
23− break;
24−
25− case FENCE_FREE:
26− dma_fence_put(&stub->dma);
27− break;
28− }
29−
30− return NOTIFY_DONE;
31−}
32−
33−static const char *stub_driver_name(struct dma_fence *fence)
34−{
35− return DRIVER_NAME;
36−}
37−
38−static const char *stub_timeline_name(struct dma_fence *fence)
39−{
40− return "object";
41−}
42−
43−static void stub_release(struct dma_fence *fence)
44−{
45− struct stub_fence *stub = container_of(fence, typeof(*stub), dma);
46−
47− i915_sw_fence_fini(&stub->chain);
48−
49− BUILD_BUG_ON(offsetof(typeof(*stub), dma));
50− dma_fence_free(&stub->dma);
51−}
52−
53−static const struct dma_fence_ops stub_fence_ops = {
54− .get_driver_name = stub_driver_name,
55− .get_timeline_name = stub_timeline_name,
56− .release = stub_release,
57−};
58−
59−struct dma_fence *
60−i915_gem_object_lock_fence(struct drm_i915_gem_object *obj)
61−{
62− struct stub_fence *stub;
63−
64− assert_object_held(obj);
65−
66− stub = kmalloc(sizeof(*stub), GFP_KERNEL);
67− if (!stub)
68− return NULL;
69−
70− i915_sw_fence_init(&stub->chain, stub_notify);
71− dma_fence_init(&stub->dma, &stub_fence_ops, &stub->chain.wait.lock,
72− 0, 0);
73−
74− if (i915_sw_fence_await_reservation(&stub->chain,
75− obj->base.resv, NULL, true,
76− i915_fence_timeout(to_i915(obj->base.dev)),
77− I915_FENCE_GFP) < 0)
78− goto err;
79−
80− dma_resv_add_excl_fence(obj->base.resv, &stub->dma);
81−
82− return &stub->dma;
83−
84−err:
85− stub_release(&stub->dma);
86− return NULL;
87−}
88−
89−void i915_gem_object_unlock_fence(struct drm_i915_gem_object *obj,
90− struct dma_fence *fence)
91−{
92− struct stub_fence *stub = container_of(fence, typeof(*stub), dma);
93−
94− i915_sw_fence_commit(&stub->chain);
95−}
drivers/gpu/drm/i915/gem/i915_gem_internal.c+3−3
@@ -138,8 +138,7 @@ static void i915_gem_object_put_pages_internal(struct drm_i915_gem_object *obj, · 137 unmodified lines
138138
139139 static const struct drm_i915_gem_object_ops i915_gem_object_internal_ops = {
140140 .name = "i915_gem_object_internal",
141− .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
142− I915_GEM_OBJECT_IS_SHRINKABLE,
141+ .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
143142 .get_pages = i915_gem_object_get_pages_internal,
144143 .put_pages = i915_gem_object_put_pages_internal,
145144 };
@@ -178,7 +177,8 @@ i915_gem_object_create_internal(struct drm_i915_private *i915, · 32 unmodified lines
178177 return ERR_PTR(-ENOMEM);
179178
180179 drm_gem_private_object_init(&i915->drm, &obj->base, size);
181− i915_gem_object_init(obj, &i915_gem_object_internal_ops, &lock_class);
180+ i915_gem_object_init(obj, &i915_gem_object_internal_ops, &lock_class,
181+ I915_BO_ALLOC_STRUCT_PAGE);
182182
183183 /*
184184 * Mark the object as volatile, such that the pages are marked as
drivers/gpu/drm/i915/gem/i915_gem_ioctls.h+0−2
@@ -14,8 +14,6 @@ int i915_gem_busy_ioctl(struct drm_device *dev, void *data, · 13 unmodified lines
1414 struct drm_file *file);
1515 int i915_gem_create_ioctl(struct drm_device *dev, void *data,
1616 struct drm_file *file);
17−int i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data,
18− struct drm_file *file);
1917 int i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data,
2018 struct drm_file *file);
2119 int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
drivers/gpu/drm/i915/gem/i915_gem_lmem.c+2−2
@@ -40,13 +40,13 @@ int __i915_gem_lmem_object_init(struct intel_memory_region *mem, · 39 unmodified lines
4040 struct drm_i915_private *i915 = mem->i915;
4141
4242 drm_gem_private_object_init(&i915->drm, &obj->base, size);
43− i915_gem_object_init(obj, &i915_gem_lmem_obj_ops, &lock_class);
43+ i915_gem_object_init(obj, &i915_gem_lmem_obj_ops, &lock_class, flags);
4444
4545 obj->read_domains = I915_GEM_DOMAIN_WC | I915_GEM_DOMAIN_GTT;
4646
4747 i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);
4848
49− i915_gem_object_init_memory_region(obj, mem, flags);
49+ i915_gem_object_init_memory_region(obj, mem);
5050
5151 return 0;
5252 }
drivers/gpu/drm/i915/gem/i915_gem_mman.c+29−6
@@ -246,12 +246,15 @@ static vm_fault_t vm_fault_cpu(struct vm_fault *vmf) · 245 unmodified lines
246246 area->vm_flags & VM_WRITE))
247247 return VM_FAULT_SIGBUS;
248248
249+ if (i915_gem_object_lock_interruptible(obj, NULL))
250+ return VM_FAULT_NOPAGE;
251+
249252 err = i915_gem_object_pin_pages(obj);
250253 if (err)
251254 goto out;
252255
253256 iomap = -1;
254− if (!i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_STRUCT_PAGE)) {
257+ if (!i915_gem_object_has_struct_page(obj)) {
255258 iomap = obj->mm.region->iomap.base;
256259 iomap -= obj->mm.region->region.start;
257260 }
@@ -269,6 +272,7 @@ static vm_fault_t vm_fault_cpu(struct vm_fault *vmf) · 11 unmodified lines
269272 i915_gem_object_unpin_pages(obj);
270273
271274 out:
275+ i915_gem_object_unlock(obj);
272276 return i915_error_to_vmf_fault(err);
273277 }
274278
@@ -417,7 +421,9 @@ vm_access(struct vm_area_struct *area, unsigned long addr, · 142 unmodified lines
417421 {
418422 struct i915_mmap_offset *mmo = area->vm_private_data;
419423 struct drm_i915_gem_object *obj = mmo->obj;
424+ struct i915_gem_ww_ctx ww;
420425 void *vaddr;
426+ int err = 0;
421427
422428 if (i915_gem_object_is_readonly(obj) && write)
423429 return -EACCES;
@@ -426,10 +432,18 @@ vm_access(struct vm_area_struct *area, unsigned long addr, · 2 unmodified lines
426432 if (addr >= obj->base.size)
427433 return -EINVAL;
428434
435+ i915_gem_ww_ctx_init(&ww, true);
436+retry:
437+ err = i915_gem_object_lock(obj, &ww);
438+ if (err)
439+ goto out;
440+
429441 /* As this is primarily for debugging, let's focus on simplicity */
430442 vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC);
431− if (IS_ERR(vaddr))
432− return PTR_ERR(vaddr);
443+ if (IS_ERR(vaddr)) {
444+ err = PTR_ERR(vaddr);
445+ goto out;
446+ }
433447
434448 if (write) {
435449 memcpy(vaddr + addr, buf, len);
@@ -439,6 +453,16 @@ vm_access(struct vm_area_struct *area, unsigned long addr, · 3 unmodified lines
439453 }
440454
441455 i915_gem_object_unpin_map(obj);
456+out:
457+ if (err == -EDEADLK) {
458+ err = i915_gem_ww_ctx_backoff(&ww);
459+ if (!err)
460+ goto retry;
461+ }
462+ i915_gem_ww_ctx_fini(&ww);
463+
464+ if (err)
465+ return err;
442466
443467 return len;
444468 }
@@ -653,9 +677,8 @@ __assign_mmap_offset(struct drm_file *file, · 208 unmodified lines
653677 }
654678
655679 if (mmap_type != I915_MMAP_TYPE_GTT &&
656− !i915_gem_object_type_has(obj,
657− I915_GEM_OBJECT_HAS_STRUCT_PAGE |
658− I915_GEM_OBJECT_HAS_IOMEM)) {
680+ !i915_gem_object_has_struct_page(obj) &&
681+ !i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM)) {
659682 err = -ENODEV;
660683 goto out;
661684 }
drivers/gpu/drm/i915/gem/i915_gem_object.c+3−7
@@ -60,10 +60,8 @@ void i915_gem_object_free(struct drm_i915_gem_object *obj) · 59 unmodified lines
6060
6161 void i915_gem_object_init(struct drm_i915_gem_object *obj,
6262 const struct drm_i915_gem_object_ops *ops,
63− struct lock_class_key *key)
63+ struct lock_class_key *key, unsigned flags)
6464 {
65− __mutex_init(&obj->mm.lock, ops->name ?: "obj->mm.lock", key);
66−
6765 spin_lock_init(&obj->vma.lock);
6866 INIT_LIST_HEAD(&obj->vma.list);
6967
@@ -78,16 +76,14 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj, · 8 unmodified lines
7876 init_rcu_head(&obj->rcu);
7977
8078 obj->ops = ops;
79+ GEM_BUG_ON(flags & ~I915_BO_ALLOC_FLAGS);
80+ obj->flags = flags;
8181
8282 obj->mm.madv = I915_MADV_WILLNEED;
8383 INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN);
8484 mutex_init(&obj->mm.get_page.lock);
8585 INIT_RADIX_TREE(&obj->mm.get_dma_page.radix, GFP_KERNEL | __GFP_NOWARN);
8686 mutex_init(&obj->mm.get_dma_page.lock);
87−
88− if (IS_ENABLED(CONFIG_LOCKDEP) && i915_gem_object_is_shrinkable(obj))
89− i915_gem_shrinker_taints_mutex(to_i915(obj->base.dev),
90− &obj->mm.lock);
9187 }
9288
9389 /**
drivers/gpu/drm/i915/gem/i915_gem_object.h+60−36
@@ -23,7 +23,8 @@ void i915_gem_object_free(struct drm_i915_gem_object *obj); · 22 unmodified lines
2323
2424 void i915_gem_object_init(struct drm_i915_gem_object *obj,
2525 const struct drm_i915_gem_object_ops *ops,
26− struct lock_class_key *key);
26+ struct lock_class_key *key,
27+ unsigned alloc_flags);
2728 struct drm_i915_gem_object *
2829 i915_gem_object_create_shmem(struct drm_i915_private *i915,
2930 resource_size_t size);
@@ -32,11 +33,21 @@ i915_gem_object_create_shmem_from_data(struct drm_i915_private *i915, · 2 unmodified lines
3233 const void *data, resource_size_t size);
3334
3435 extern const struct drm_i915_gem_object_ops i915_gem_shmem_ops;
36+
3537 void __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
3638 struct sg_table *pages,
3739 bool needs_clflush);
3840
41+int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj,
42+ const struct drm_i915_gem_pwrite *args);
43+int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj,
44+ const struct drm_i915_gem_pread *args);
45+
3946 int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align);
47+void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj,
48+ struct sg_table *pages);
49+void i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
50+ struct sg_table *pages);
4051
4152 void i915_gem_flush_free_objects(struct drm_i915_private *i915);
4253
@@ -107,6 +118,20 @@ i915_gem_object_put(struct drm_i915_gem_object *obj) · 64 unmodified lines
107118
108119 #define assert_object_held(obj) dma_resv_assert_held((obj)->base.resv)
109120
121+/*
122+ * If more than one potential simultaneous locker, assert held.
123+ */
124+static inline void assert_object_held_shared(struct drm_i915_gem_object *obj)
125+{
126+ /*
127+ * Note mm list lookup is protected by
128+ * kref_get_unless_zero().
129+ */
130+ if (IS_ENABLED(CONFIG_LOCKDEP) &&
131+ kref_read(&obj->base.refcount) > 0)
132+ assert_object_held(obj);
133+}
134+
110135 static inline int __i915_gem_object_lock(struct drm_i915_gem_object *obj,
111136 struct i915_gem_ww_ctx *ww,
112137 bool intr)
@@ -152,11 +177,6 @@ static inline void i915_gem_object_unlock(struct drm_i915_gem_object *obj) · 39 unmodified lines
152177 dma_resv_unlock(obj->base.resv);
153178 }
154179
155−struct dma_fence *
156−i915_gem_object_lock_fence(struct drm_i915_gem_object *obj);
157−void i915_gem_object_unlock_fence(struct drm_i915_gem_object *obj,
158− struct dma_fence *fence);
159−
160180 static inline void
161181 i915_gem_object_set_readonly(struct drm_i915_gem_object *obj)
162182 {
@@ -215,7 +235,7 @@ i915_gem_object_type_has(const struct drm_i915_gem_object *obj, · 52 unmodified lines
215235 static inline bool
216236 i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
217237 {
218− return i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_STRUCT_PAGE);
238+ return obj->flags & I915_BO_ALLOC_STRUCT_PAGE;
219239 }
220240
221241 static inline bool
@@ -242,12 +262,6 @@ i915_gem_object_never_mmap(const struct drm_i915_gem_object *obj) · 20 unmodified lines
242262 return i915_gem_object_type_has(obj, I915_GEM_OBJECT_NO_MMAP);
243263 }
244264
245−static inline bool
246−i915_gem_object_needs_async_cancel(const struct drm_i915_gem_object *obj)
247−{
248− return i915_gem_object_type_has(obj, I915_GEM_OBJECT_ASYNC_CANCEL);
249−}
250−
251265 static inline bool
252266 i915_gem_object_is_framebuffer(const struct drm_i915_gem_object *obj)
253267 {
@@ -299,22 +313,22 @@ struct scatterlist * · 45 unmodified lines
299313 __i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
300314 struct i915_gem_object_page_iter *iter,
301315 unsigned int n,
302− unsigned int *offset);
316+ unsigned int *offset, bool allow_alloc);
303317
304318 static inline struct scatterlist *
305319 i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
306320 unsigned int n,
307− unsigned int *offset)
321+ unsigned int *offset, bool allow_alloc)
308322 {
309− return __i915_gem_object_get_sg(obj, &obj->mm.get_page, n, offset);
323+ return __i915_gem_object_get_sg(obj, &obj->mm.get_page, n, offset, allow_alloc);
310324 }
311325
312326 static inline struct scatterlist *
313327 i915_gem_object_get_sg_dma(struct drm_i915_gem_object *obj,
314328 unsigned int n,
315− unsigned int *offset)
329+ unsigned int *offset, bool allow_alloc)
316330 {
317− return __i915_gem_object_get_sg(obj, &obj->mm.get_dma_page, n, offset);
331+ return __i915_gem_object_get_sg(obj, &obj->mm.get_dma_page, n, offset, allow_alloc);
318332 }
319333
320334 struct page *
@@ -341,27 +355,10 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, · 20 unmodified lines
341355 int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
342356 int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
343357
344−enum i915_mm_subclass { /* lockdep subclass for obj->mm.lock/struct_mutex */
345− I915_MM_NORMAL = 0,
346− /*
347− * Only used by struct_mutex, when called "recursively" from
348− * direct-reclaim-esque. Safe because there is only every one
349− * struct_mutex in the entire system.
350− */
351− I915_MM_SHRINKER = 1,
352− /*
353− * Used for obj->mm.lock when allocating pages. Safe because the object
354− * isn't yet on any LRU, and therefore the shrinker can't deadlock on
355− * it. As soon as the object has pages, obj->mm.lock nests within
356− * fs_reclaim.
357− */
358− I915_MM_GET_PAGES = 1,
359−};
360−
361358 static inline int __must_check
362359 i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
363360 {
364− might_lock_nested(&obj->mm.lock, I915_MM_GET_PAGES);
361+ assert_object_held(obj);
365362
366363 if (atomic_inc_not_zero(&obj->mm.pages_pin_count))
367364 return 0;
@@ -369,6 +366,8 @@ i915_gem_object_pin_pages(struct drm_i915_gem_object *obj) · 1 unmodified line
369366 return __i915_gem_object_get_pages(obj);
370367 }
371368
369+int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj);
370+
372371 static inline bool
373372 i915_gem_object_has_pages(struct drm_i915_gem_object *obj)
374373 {
@@ -427,6 +426,9 @@ void i915_gem_object_writeback(struct drm_i915_gem_object *obj); · 52 unmodified lines
427426 void *__must_check i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
428427 enum i915_map_type type);
429428
429+void *__must_check i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj,
430+ enum i915_map_type type);
431+
430432 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
431433 unsigned long offset,
432434 unsigned long size);
@@ -495,6 +497,7 @@ int __must_check · 62 unmodified lines
495497 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write);
496498 struct i915_vma * __must_check
497499 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
500+ struct i915_gem_ww_ctx *ww,
498501 u32 alignment,
499502 const struct i915_ggtt_view *view,
500503 unsigned int flags);
@@ -558,4 +561,25 @@ int i915_gem_object_read_from_page(struct drm_i915_gem_object *obj, u64 offset, · 57 unmodified lines
558561
559562 bool i915_gem_object_is_shmem(const struct drm_i915_gem_object *obj);
560563
564+#ifdef CONFIG_MMU_NOTIFIER
565+static inline bool
566+i915_gem_object_is_userptr(struct drm_i915_gem_object *obj)
567+{
568+ return obj->userptr.notifier.mm;
569+}
570+
571+int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj);
572+int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj);
573+void i915_gem_object_userptr_submit_fini(struct drm_i915_gem_object *obj);
574+int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj);
575+#else
576+static inline bool i915_gem_object_is_userptr(struct drm_i915_gem_object *obj) { return false; }
577+
578+static inline int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; }
579+static inline int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; }
580+static inline void i915_gem_object_userptr_submit_fini(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); }
581+static inline int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj) { GEM_BUG_ON(1); return -ENODEV; }
582+
583+#endif
584+
561585 #endif
drivers/gpu/drm/i915/gem/i915_gem_object_blt.c+6−0
@@ -55,6 +55,9 @@ struct i915_vma *intel_emit_vma_fill_blt(struct intel_context *ce, · 54 unmodified lines
5555 if (unlikely(err))
5656 goto out_put;
5757
58+ /* we pinned the pool, mark it as such */
59+ intel_gt_buffer_pool_mark_used(pool);
60+
5861 cmd = i915_gem_object_pin_map(pool->obj, pool->type);
5962 if (IS_ERR(cmd)) {
6063 err = PTR_ERR(cmd);
@@ -277,6 +280,9 @@ struct i915_vma *intel_emit_vma_copy_blt(struct intel_context *ce, · 216 unmodified lines
277280 if (unlikely(err))
278281 goto out_put;
279282
283+ /* we pinned the pool, mark it as such */
284+ intel_gt_buffer_pool_mark_used(pool);
285+
280286 cmd = i915_gem_object_pin_map(pool->obj, pool->type);
281287 if (IS_ERR(cmd)) {
282288 err = PTR_ERR(cmd);
drivers/gpu/drm/i915/gem/i915_gem_object_types.h+14−9
@@ -7,6 +7,8 @@ · 6 unmodified lines
77 #ifndef __I915_GEM_OBJECT_TYPES_H__
88 #define __I915_GEM_OBJECT_TYPES_H__
99
10+#include <linux/mmu_notifier.h>
11+
1012 #include <drm/drm_gem.h>
1113 #include <uapi/drm/i915_drm.h>
1214
@@ -30,12 +32,10 @@ struct i915_lut_handle { · 17 unmodified lines
3032
3133 struct drm_i915_gem_object_ops {
3234 unsigned int flags;
33−#define I915_GEM_OBJECT_HAS_STRUCT_PAGE BIT(0)
3435 #define I915_GEM_OBJECT_HAS_IOMEM BIT(1)
3536 #define I915_GEM_OBJECT_IS_SHRINKABLE BIT(2)
3637 #define I915_GEM_OBJECT_IS_PROXY BIT(3)
3738 #define I915_GEM_OBJECT_NO_MMAP BIT(4)
38−#define I915_GEM_OBJECT_ASYNC_CANCEL BIT(5)
3939
4040 /* Interface between the GEM object and its backing storage.
4141 * get_pages() is called once prior to the use of the associated set
@@ -171,9 +171,12 @@ struct drm_i915_gem_object { · 129 unmodified lines
171171 unsigned long flags;
172172 #define I915_BO_ALLOC_CONTIGUOUS BIT(0)
173173 #define I915_BO_ALLOC_VOLATILE BIT(1)
174−#define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | I915_BO_ALLOC_VOLATILE)
175−#define I915_BO_READONLY BIT(2)
176−#define I915_TILING_QUIRK_BIT 3 /* unknown swizzling; do not release! */
174+#define I915_BO_ALLOC_STRUCT_PAGE BIT(2)
175+#define I915_BO_ALLOC_FLAGS (I915_BO_ALLOC_CONTIGUOUS | \
176+ I915_BO_ALLOC_VOLATILE | \
177+ I915_BO_ALLOC_STRUCT_PAGE)
178+#define I915_BO_READONLY BIT(3)
179+#define I915_TILING_QUIRK_BIT 4 /* unknown swizzling; do not release! */
177180
178181 /*
179182 * Is the object to be mapped as read-only to the GPU
@@ -213,7 +216,6 @@ struct drm_i915_gem_object { · 33 unmodified lines
213216 * Protects the pages and their use. Do not use directly, but
214217 * instead go through the pin/unpin interfaces.
215218 */
216− struct mutex lock;
217219 atomic_t pages_pin_count;
218220 atomic_t shrink_pin;
219221
@@ -288,13 +290,16 @@ struct drm_i915_gem_object { · 68 unmodified lines
288290 unsigned long *bit_17;
289291
290292 union {
293+#ifdef CONFIG_MMU_NOTIFIER
291294 struct i915_gem_userptr {
292295 uintptr_t ptr;
296+ unsigned long notifier_seq;
293297
294− struct i915_mm_struct *mm;
295− struct i915_mmu_object *mmu_object;
296− struct work_struct *work;
298+ struct mmu_interval_notifier notifier;
299+ struct page **pvec;
300+ int page_ref;
297301 } userptr;
302+#endif
298303
299304 struct drm_mm_node *stolen;
300305
drivers/gpu/drm/i915/gem/i915_gem_pages.c+69−40
@@ -19,7 +19,7 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, · 18 unmodified lines
1919 bool shrinkable;
2020 int i;
2121
22− lockdep_assert_held(&obj->mm.lock);
22+ assert_object_held_shared(obj);
2323
2424 if (i915_gem_object_is_volatile(obj))
2525 obj->mm.madv = I915_MADV_DONTNEED;
@@ -70,6 +70,7 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, · 44 unmodified lines
7070 struct list_head *list;
7171 unsigned long flags;
7272
73+ assert_object_held(obj);
7374 spin_lock_irqsave(&i915->mm.obj_lock, flags);
7475
7576 i915->mm.shrink_count++;
@@ -91,6 +92,8 @@ int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj) · 15 unmodified lines
9192 struct drm_i915_private *i915 = to_i915(obj->base.dev);
9293 int err;
9394
95+ assert_object_held_shared(obj);
96+
9497 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
9598 drm_dbg(&i915->drm,
9699 "Attempting to obtain a purgeable object\n");
@@ -114,23 +117,41 @@ int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj) · 17 unmodified lines
114117 {
115118 int err;
116119
117− err = mutex_lock_interruptible_nested(&obj->mm.lock, I915_MM_GET_PAGES);
118− if (err)
119− return err;
120+ assert_object_held(obj);
121+
122+ assert_object_held_shared(obj);
120123
121124 if (unlikely(!i915_gem_object_has_pages(obj))) {
122125 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
123126
124127 err = ____i915_gem_object_get_pages(obj);
125128 if (err)
126− goto unlock;
129+ return err;
127130
128131 smp_mb__before_atomic();
129132 }
130133 atomic_inc(&obj->mm.pages_pin_count);
131134
132−unlock:
133− mutex_unlock(&obj->mm.lock);
135+ return 0;
136+}
137+
138+int i915_gem_object_pin_pages_unlocked(struct drm_i915_gem_object *obj)
139+{
140+ struct i915_gem_ww_ctx ww;
141+ int err;
142+
143+ i915_gem_ww_ctx_init(&ww, true);
144+retry:
145+ err = i915_gem_object_lock(obj, &ww);
146+ if (!err)
147+ err = i915_gem_object_pin_pages(obj);
148+
149+ if (err == -EDEADLK) {
150+ err = i915_gem_ww_ctx_backoff(&ww);
151+ if (!err)
152+ goto retry;
153+ }
154+ i915_gem_ww_ctx_fini(&ww);
134155 return err;
135156 }
136157
@@ -145,7 +166,7 @@ void i915_gem_object_truncate(struct drm_i915_gem_object *obj) · 8 unmodified lines
145166 /* Try to discard unwanted pages */
146167 void i915_gem_object_writeback(struct drm_i915_gem_object *obj)
147168 {
148− lockdep_assert_held(&obj->mm.lock);
169+ assert_object_held_shared(obj);
149170 GEM_BUG_ON(i915_gem_object_has_pages(obj));
150171
151172 if (obj->ops->writeback)
@@ -176,6 +197,8 @@ __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) · 24 unmodified lines
176197 {
177198 struct sg_table *pages;
178199
200+ assert_object_held_shared(obj);
201+
179202 pages = fetch_and_zero(&obj->mm.pages);
180203 if (IS_ERR_OR_NULL(pages))
181204 return pages;
@@ -199,17 +222,12 @@ __i915_gem_object_unset_pages(struct drm_i915_gem_object *obj) · 17 unmodified lines
199222 int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
200223 {
201224 struct sg_table *pages;
202− int err;
203225
204226 if (i915_gem_object_has_pinned_pages(obj))
205227 return -EBUSY;
206228
207229 /* May be called by shrinker from within get_pages() (on another bo) */
208− mutex_lock(&obj->mm.lock);
209− if (unlikely(atomic_read(&obj->mm.pages_pin_count))) {
210− err = -EBUSY;
211− goto unlock;
212− }
230+ assert_object_held_shared(obj);
213231
214232 i915_gem_object_release_mmap_offset(obj);
215233
@@ -226,17 +244,10 @@ int __i915_gem_object_put_pages(struct drm_i915_gem_object *obj) · 10 unmodified lines
226244 * get_pages backends we should be better able to handle the
227245 * cancellation of the async task in a more uniform manner.
228246 */
229− if (!pages && !i915_gem_object_needs_async_cancel(obj))
230− pages = ERR_PTR(-EINVAL);
231−
232− if (!IS_ERR(pages))
247+ if (!IS_ERR_OR_NULL(pages))
233248 obj->ops->put_pages(obj, pages);
234249
235− err = 0;
236−unlock:
237− mutex_unlock(&obj->mm.lock);
238−
239− return err;
250+ return 0;
240251 }
241252
242253 /* The 'mapping' part of i915_gem_object_pin_map() below */
@@ -333,18 +344,15 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, · 90 unmodified lines
333344 enum i915_map_type type)
334345 {
335346 enum i915_map_type has_type;
336− unsigned int flags;
337347 bool pinned;
338348 void *ptr;
339349 int err;
340350
341− flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE | I915_GEM_OBJECT_HAS_IOMEM;
342− if (!i915_gem_object_type_has(obj, flags))
351+ if (!i915_gem_object_has_struct_page(obj) &&
352+ !i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM))
343353 return ERR_PTR(-ENXIO);
344354
345− err = mutex_lock_interruptible_nested(&obj->mm.lock, I915_MM_GET_PAGES);
346− if (err)
347− return ERR_PTR(err);
355+ assert_object_held(obj);
348356
349357 pinned = !(type & I915_MAP_OVERRIDE);
350358 type &= ~I915_MAP_OVERRIDE;
@@ -354,10 +362,8 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, · 3 unmodified lines
354362 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
355363
356364 err = ____i915_gem_object_get_pages(obj);
357− if (err) {
358− ptr = ERR_PTR(err);
359− goto out_unlock;
360− }
365+ if (err)
366+ return ERR_PTR(err);
361367
362368 smp_mb__before_atomic();
363369 }
@@ -392,13 +398,23 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj, · 28 unmodified lines
392398 obj->mm.mapping = page_pack_bits(ptr, type);
393399 }
394400
395−out_unlock:
396− mutex_unlock(&obj->mm.lock);
397401 return ptr;
398402
399403 err_unpin:
400404 atomic_dec(&obj->mm.pages_pin_count);
401− goto out_unlock;
405+ return ptr;
406+}
407+
408+void *i915_gem_object_pin_map_unlocked(struct drm_i915_gem_object *obj,
409+ enum i915_map_type type)
410+{
411+ void *ret;
412+
413+ i915_gem_object_lock(obj, NULL);
414+ ret = i915_gem_object_pin_map(obj, type);
415+ i915_gem_object_unlock(obj);
416+
417+ return ret;
402418 }
403419
404420 void __i915_gem_object_flush_map(struct drm_i915_gem_object *obj,
@@ -448,7 +464,8 @@ struct scatterlist * · 43 unmodified lines
448464 __i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
449465 struct i915_gem_object_page_iter *iter,
450466 unsigned int n,
451− unsigned int *offset)
467+ unsigned int *offset,
468+ bool allow_alloc)
452469 {
453470 const bool dma = iter == &obj->mm.get_dma_page;
454471 struct scatterlist *sg;
@@ -470,6 +487,9 @@ __i915_gem_object_get_sg(struct drm_i915_gem_object *obj, · 15 unmodified lines
470487 if (n < READ_ONCE(iter->sg_idx))
471488 goto lookup;
472489
490+ if (!allow_alloc)
491+ goto manual_lookup;
492+
473493 mutex_lock(&iter->lock);
474494
475495 /* We prefer to reuse the last sg so that repeated lookup of this
@@ -519,7 +539,16 @@ scan: · 43 unmodified lines
519539 if (unlikely(n < idx)) /* insertion completed by another thread */
520540 goto lookup;
521541
522− /* In case we failed to insert the entry into the radixtree, we need
542+ goto manual_walk;
543+
544+manual_lookup:
545+ idx = 0;
546+ sg = obj->mm.pages->sgl;
547+ count = __sg_page_count(sg);
548+
549+manual_walk:
550+ /*
551+ * In case we failed to insert the entry into the radixtree, we need
523552 * to look beyond the current sg.
524553 */
525554 while (idx + count <= n) {
@@ -566,7 +595,7 @@ i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n) · 40 unmodified lines
566595
567596 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
568597
569− sg = i915_gem_object_get_sg(obj, n, &offset);
598+ sg = i915_gem_object_get_sg(obj, n, &offset, true);
570599 return nth_page(sg_page(sg), offset);
571600 }
572601
@@ -592,7 +621,7 @@ i915_gem_object_get_dma_address_len(struct drm_i915_gem_object *obj, · 19 unmodified lines
592621 struct scatterlist *sg;
593622 unsigned int offset;
594623
595− sg = i915_gem_object_get_sg_dma(obj, n, &offset);
624+ sg = i915_gem_object_get_sg_dma(obj, n, &offset, true);
596625
597626 if (len)
598627 *len = sg_dma_len(sg) - (offset << PAGE_SHIFT);
drivers/gpu/drm/i915/gem/i915_gem_phys.c+47−63
@@ -76,6 +76,8 @@ static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj) · 75 unmodified lines
7676
7777 intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt);
7878
79+ /* We're no longer struct page backed */
80+ obj->flags &= ~I915_BO_ALLOC_STRUCT_PAGE;
7981 __i915_gem_object_set_pages(obj, st, sg->length);
8082
8183 return 0;
@@ -89,7 +91,7 @@ err_pci: · 7 unmodified lines
8991 return -ENOMEM;
9092 }
9193
92−static void
94+void
9395 i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
9496 struct sg_table *pages)
9597 {
@@ -134,9 +136,8 @@ i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj, · 38 unmodified lines
134136 vaddr, dma);
135137 }
136138
137−static int
138−phys_pwrite(struct drm_i915_gem_object *obj,
139− const struct drm_i915_gem_pwrite *args)
139+int i915_gem_object_pwrite_phys(struct drm_i915_gem_object *obj,
140+ const struct drm_i915_gem_pwrite *args)
140141 {
141142 void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset;
142143 char __user *user_data = u64_to_user_ptr(args->data_ptr);
@@ -165,9 +166,8 @@ phys_pwrite(struct drm_i915_gem_object *obj, · 22 unmodified lines
165166 return 0;
166167 }
167168
168−static int
169−phys_pread(struct drm_i915_gem_object *obj,
170− const struct drm_i915_gem_pread *args)
169+int i915_gem_object_pread_phys(struct drm_i915_gem_object *obj,
170+ const struct drm_i915_gem_pread *args)
171171 {
172172 void *vaddr = sg_page(obj->mm.pages->sgl) + args->offset;
173173 char __user *user_data = u64_to_user_ptr(args->data_ptr);
@@ -186,62 +186,14 @@ phys_pread(struct drm_i915_gem_object *obj, · 12 unmodified lines
186186 return 0;
187187 }
188188
189−static void phys_release(struct drm_i915_gem_object *obj)
190−{
191− fput(obj->base.filp);
192−}
193−
194−static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
195− .name = "i915_gem_object_phys",
196− .get_pages = i915_gem_object_get_pages_phys,
197− .put_pages = i915_gem_object_put_pages_phys,
198−
199− .pread = phys_pread,
200− .pwrite = phys_pwrite,
201−
202− .release = phys_release,
203−};
204−
205−int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
189+static int i915_gem_object_shmem_to_phys(struct drm_i915_gem_object *obj)
206190 {
207191 struct sg_table *pages;
208192 int err;
209193
210− if (align > obj->base.size)
211− return -EINVAL;
212−
213− if (obj->ops == &i915_gem_phys_ops)
214− return 0;
215−
216− if (!i915_gem_object_is_shmem(obj))
217− return -EINVAL;
218−
219− err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
220− if (err)
221− return err;
222−
223− mutex_lock_nested(&obj->mm.lock, I915_MM_GET_PAGES);
224−
225− if (obj->mm.madv != I915_MADV_WILLNEED) {
226− err = -EFAULT;
227− goto err_unlock;
228− }
229−
230− if (i915_gem_object_has_tiling_quirk(obj)) {
231− err = -EFAULT;
232− goto err_unlock;
233− }
234−
235− if (obj->mm.mapping) {
236− err = -EBUSY;
237− goto err_unlock;
238− }
239−
240194 pages = __i915_gem_object_unset_pages(obj);
241195
242− obj->ops = &i915_gem_phys_ops;
243−
244− err = ____i915_gem_object_get_pages(obj);
196+ err = i915_gem_object_get_pages_phys(obj);
245197 if (err)
246198 goto err_xfer;
247199
@@ -249,25 +201,57 @@ int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align) · 1 unmodified line
249201 __i915_gem_object_pin_pages(obj);
250202
251203 if (!IS_ERR_OR_NULL(pages))
252− i915_gem_shmem_ops.put_pages(obj, pages);
204+ i915_gem_object_put_pages_shmem(obj, pages);
253205
254206 i915_gem_object_release_memory_region(obj);
255−
256− mutex_unlock(&obj->mm.lock);
257207 return 0;
258208
259209 err_xfer:
260− obj->ops = &i915_gem_shmem_ops;
261210 if (!IS_ERR_OR_NULL(pages)) {
262211 unsigned int sg_page_sizes = i915_sg_page_sizes(pages->sgl);
263212
264213 __i915_gem_object_set_pages(obj, pages, sg_page_sizes);
265214 }
266−err_unlock:
267− mutex_unlock(&obj->mm.lock);
268215 return err;
269216 }
270217
218+int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
219+{
220+ int err;
221+
222+ assert_object_held(obj);
223+
224+ if (align > obj->base.size)
225+ return -EINVAL;
226+
227+ if (!i915_gem_object_is_shmem(obj))
228+ return -EINVAL;
229+
230+ if (!i915_gem_object_has_struct_page(obj))
231+ return 0;
232+
233+ err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
234+ if (err)
235+ return err;
236+
237+ if (obj->mm.madv != I915_MADV_WILLNEED)
238+ return -EFAULT;
239+
240+ if (i915_gem_object_has_tiling_quirk(obj))
241+ return -EFAULT;
242+
243+ if (obj->mm.mapping || i915_gem_object_has_pinned_pages(obj))
244+ return -EBUSY;
245+
246+ if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
247+ drm_dbg(obj->base.dev,
248+ "Attempting to obtain a purgeable object\n");
249+ return -EFAULT;
250+ }
251+
252+ return i915_gem_object_shmem_to_phys(obj);
253+}
254+
271255 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
272256 #include "selftests/i915_gem_phys.c"
273257 #endif
drivers/gpu/drm/i915/gem/i915_gem_pm.c+1−1
@@ -116,7 +116,7 @@ int i915_gem_freeze_late(struct drm_i915_private *i915) · 115 unmodified lines
116116 */
117117
118118 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
119− i915_gem_shrink(i915, -1UL, NULL, ~0);
119+ i915_gem_shrink(NULL, i915, -1UL, NULL, ~0);
120120 i915_gem_drain_freed_objects(i915);
121121
122122 wbinvd_on_all_cpus();
drivers/gpu/drm/i915/gem/i915_gem_region.c+1−3
@@ -106,13 +106,11 @@ err_free_sg: · 105 unmodified lines
106106 }
107107
108108 void i915_gem_object_init_memory_region(struct drm_i915_gem_object *obj,
109− struct intel_memory_region *mem,
110− unsigned long flags)
109+ struct intel_memory_region *mem)
111110 {
112111 INIT_LIST_HEAD(&obj->mm.blocks);
113112 obj->mm.region = intel_memory_region_get(mem);
114113
115− obj->flags |= flags;
116114 if (obj->base.size <= mem->min_page_size)
117115 obj->flags |= I915_BO_ALLOC_CONTIGUOUS;
118116
drivers/gpu/drm/i915/gem/i915_gem_region.h+1−2
@@ -17,8 +17,7 @@ void i915_gem_object_put_pages_buddy(struct drm_i915_gem_object *obj, · 16 unmodified lines
1717 struct sg_table *pages);
1818
1919 void i915_gem_object_init_memory_region(struct drm_i915_gem_object *obj,
20− struct intel_memory_region *mem,
21− unsigned long flags);
20+ struct intel_memory_region *mem);
2221 void i915_gem_object_release_memory_region(struct drm_i915_gem_object *obj);
2322
2423 struct drm_i915_gem_object *
drivers/gpu/drm/i915/gem/i915_gem_shmem.c+31−8
@@ -99,7 +99,7 @@ rebuild_st: · 98 unmodified lines
9999 goto err_sg;
100100 }
101101
102− i915_gem_shrink(i915, 2 * page_count, NULL, *s++);
102+ i915_gem_shrink(NULL, i915, 2 * page_count, NULL, *s++);
103103
104104 /*
105105 * We've tried hard to allocate the memory by reaping
@@ -296,8 +296,7 @@ __i915_gem_object_release_shmem(struct drm_i915_gem_object *obj, · 190 unmodified lines
296296 __start_cpu_write(obj);
297297 }
298298
299−static void
300−shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages)
299+void i915_gem_object_put_pages_shmem(struct drm_i915_gem_object *obj, struct sg_table *pages)
301300 {
302301 struct sgt_iter sgt_iter;
303302 struct pagevec pvec;
@@ -331,6 +330,15 @@ shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) · 27 unmodified lines
331330 kfree(pages);
332331 }
333332
333+static void
334+shmem_put_pages(struct drm_i915_gem_object *obj, struct sg_table *pages)
335+{
336+ if (likely(i915_gem_object_has_struct_page(obj)))
337+ i915_gem_object_put_pages_shmem(obj, pages);
338+ else
339+ i915_gem_object_put_pages_phys(obj, pages);
340+}
341+
334342 static int
335343 shmem_pwrite(struct drm_i915_gem_object *obj,
336344 const struct drm_i915_gem_pwrite *arg)
@@ -343,6 +351,9 @@ shmem_pwrite(struct drm_i915_gem_object *obj, · 6 unmodified lines
343351 /* Caller already validated user args */
344352 GEM_BUG_ON(!access_ok(user_data, arg->size));
345353
354+ if (!i915_gem_object_has_struct_page(obj))
355+ return i915_gem_object_pwrite_phys(obj, arg);
356+
346357 /*
347358 * Before we instantiate/pin the backing store for our use, we
348359 * can prepopulate the shmemfs filp efficiently using a write into
@@ -421,17 +432,27 @@ shmem_pwrite(struct drm_i915_gem_object *obj, · 72 unmodified lines
421432 return 0;
422433 }
423434
435+static int
436+shmem_pread(struct drm_i915_gem_object *obj,
437+ const struct drm_i915_gem_pread *arg)
438+{
439+ if (!i915_gem_object_has_struct_page(obj))
440+ return i915_gem_object_pread_phys(obj, arg);
441+
442+ return -ENODEV;
443+}
444+
424445 static void shmem_release(struct drm_i915_gem_object *obj)
425446 {
426− i915_gem_object_release_memory_region(obj);
447+ if (obj->flags & I915_BO_ALLOC_STRUCT_PAGE)
448+ i915_gem_object_release_memory_region(obj);
427449
428450 fput(obj->base.filp);
429451 }
430452
431453 const struct drm_i915_gem_object_ops i915_gem_shmem_ops = {
432454 .name = "i915_gem_object_shmem",
433− .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
434− I915_GEM_OBJECT_IS_SHRINKABLE,
455+ .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
435456
436457 .get_pages = shmem_get_pages,
437458 .put_pages = shmem_put_pages,
@@ -439,6 +460,7 @@ const struct drm_i915_gem_object_ops i915_gem_shmem_ops = { · 1 unmodified line
439460 .writeback = shmem_writeback,
440461
441462 .pwrite = shmem_pwrite,
463+ .pread = shmem_pread,
442464
443465 .release = shmem_release,
444466 };
@@ -491,7 +513,8 @@ static int shmem_object_init(struct intel_memory_region *mem, · 46 unmodified lines
491513 mapping_set_gfp_mask(mapping, mask);
492514 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));
493515
494− i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class);
516+ i915_gem_object_init(obj, &i915_gem_shmem_ops, &lock_class,
517+ I915_BO_ALLOC_STRUCT_PAGE);
495518
496519 obj->write_domain = I915_GEM_DOMAIN_CPU;
497520 obj->read_domains = I915_GEM_DOMAIN_CPU;
@@ -515,7 +538,7 @@ static int shmem_object_init(struct intel_memory_region *mem, · 17 unmodified lines
515538
516539 i915_gem_object_set_cache_coherency(obj, cache_level);
517540
518− i915_gem_object_init_memory_region(obj, mem, 0);
541+ i915_gem_object_init_memory_region(obj, mem);
519542
520543 return 0;
521544 }
drivers/gpu/drm/i915/gem/i915_gem_shrinker.c+28−11
@@ -49,9 +49,9 @@ static bool unsafe_drop_pages(struct drm_i915_gem_object *obj, · 48 unmodified lines
4949 flags = I915_GEM_OBJECT_UNBIND_TEST;
5050
5151 if (i915_gem_object_unbind(obj, flags) == 0)
52− __i915_gem_object_put_pages(obj);
52+ return true;
5353
54− return !i915_gem_object_has_pages(obj);
54+ return false;
5555 }
5656
5757 static void try_to_writeback(struct drm_i915_gem_object *obj,
@@ -94,7 +94,8 @@ static void try_to_writeback(struct drm_i915_gem_object *obj, · 36 unmodified lines
9494 * The number of pages of backing storage actually released.
9595 */
9696 unsigned long
97−i915_gem_shrink(struct drm_i915_private *i915,
97+i915_gem_shrink(struct i915_gem_ww_ctx *ww,
98+ struct drm_i915_private *i915,
9899 unsigned long target,
99100 unsigned long *nr_scanned,
100101 unsigned int shrink)
@@ -113,6 +114,7 @@ i915_gem_shrink(struct drm_i915_private *i915, · 12 unmodified lines
113114 intel_wakeref_t wakeref = 0;
114115 unsigned long count = 0;
115116 unsigned long scanned = 0;
117+ int err;
116118
117119 trace_i915_gem_shrink(i915, target, shrink);
118120
@@ -200,25 +202,40 @@ i915_gem_shrink(struct drm_i915_private *i915, · 81 unmodified lines
200202
201203 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
202204
205+ err = 0;
203206 if (unsafe_drop_pages(obj, shrink)) {
204207 /* May arrive from get_pages on another bo */
205− mutex_lock(&obj->mm.lock);
206− if (!i915_gem_object_has_pages(obj)) {
208+ if (!ww) {
209+ if (!i915_gem_object_trylock(obj))
210+ goto skip;
211+ } else {
212+ err = i915_gem_object_lock(obj, ww);
213+ if (err)
214+ goto skip;
215+ }
216+
217+ if (!__i915_gem_object_put_pages(obj)) {
207218 try_to_writeback(obj, shrink);
208219 count += obj->base.size >> PAGE_SHIFT;
209220 }
210− mutex_unlock(&obj->mm.lock);
221+ if (!ww)
222+ i915_gem_object_unlock(obj);
211223 }
212224
213225 dma_resv_prune(obj->base.resv);
214226
215227 scanned += obj->base.size >> PAGE_SHIFT;
228+skip:
216229 i915_gem_object_put(obj);
217230
218231 spin_lock_irqsave(&i915->mm.obj_lock, flags);
232+ if (err)
233+ break;
219234 }
220235 list_splice_tail(&still_in_list, phase->list);
221236 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
237+ if (err)
238+ return err;
222239 }
223240
224241 if (shrink & I915_SHRINK_BOUND)
@@ -249,7 +266,7 @@ unsigned long i915_gem_shrink_all(struct drm_i915_private *i915) · 24 unmodified lines
249266 unsigned long freed = 0;
250267
251268 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
252− freed = i915_gem_shrink(i915, -1UL, NULL,
269+ freed = i915_gem_shrink(NULL, i915, -1UL, NULL,
253270 I915_SHRINK_BOUND |
254271 I915_SHRINK_UNBOUND);
255272 }
@@ -295,7 +312,7 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) · 39 unmodified lines
295312
296313 sc->nr_scanned = 0;
297314
298− freed = i915_gem_shrink(i915,
315+ freed = i915_gem_shrink(NULL, i915,
299316 sc->nr_to_scan,
300317 &sc->nr_scanned,
301318 I915_SHRINK_BOUND |
@@ -304,7 +321,7 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) · 2 unmodified lines
304321 intel_wakeref_t wakeref;
305322
306323 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
307− freed += i915_gem_shrink(i915,
324+ freed += i915_gem_shrink(NULL, i915,
308325 sc->nr_to_scan - sc->nr_scanned,
309326 &sc->nr_scanned,
310327 I915_SHRINK_ACTIVE |
@@ -329,7 +346,7 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr) · 18 unmodified lines
329346
330347 freed_pages = 0;
331348 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
332− freed_pages += i915_gem_shrink(i915, -1UL, NULL,
349+ freed_pages += i915_gem_shrink(NULL, i915, -1UL, NULL,
333350 I915_SHRINK_BOUND |
334351 I915_SHRINK_UNBOUND |
335352 I915_SHRINK_WRITEBACK);
@@ -367,7 +384,7 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr · 31 unmodified lines
367384 intel_wakeref_t wakeref;
368385
369386 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
370− freed_pages += i915_gem_shrink(i915, -1UL, NULL,
387+ freed_pages += i915_gem_shrink(NULL, i915, -1UL, NULL,
371388 I915_SHRINK_BOUND |
372389 I915_SHRINK_UNBOUND |
373390 I915_SHRINK_VMAPS);
drivers/gpu/drm/i915/gem/i915_gem_shrinker.h+3−1
@@ -9,10 +9,12 @@ · 8 unmodified lines
99 #include <linux/bits.h>
1010
1111 struct drm_i915_private;
12+struct i915_gem_ww_ctx;
1213 struct mutex;
1314
1415 /* i915_gem_shrinker.c */
15−unsigned long i915_gem_shrink(struct drm_i915_private *i915,
16+unsigned long i915_gem_shrink(struct i915_gem_ww_ctx *ww,
17+ struct drm_i915_private *i915,
1618 unsigned long target,
1719 unsigned long *nr_scanned,
1820 unsigned flags);
drivers/gpu/drm/i915/gem/i915_gem_stolen.c+8−6
@@ -630,20 +630,22 @@ static int __i915_gem_object_create_stolen(struct intel_memory_region *mem, · 629 unmodified lines
630630 int err;
631631
632632 drm_gem_private_object_init(&mem->i915->drm, &obj->base, stolen->size);
633− i915_gem_object_init(obj, &i915_gem_object_stolen_ops, &lock_class);
633+ i915_gem_object_init(obj, &i915_gem_object_stolen_ops, &lock_class, 0);
634634
635635 obj->stolen = stolen;
636636 obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
637637 cache_level = HAS_LLC(mem->i915) ? I915_CACHE_LLC : I915_CACHE_NONE;
638638 i915_gem_object_set_cache_coherency(obj, cache_level);
639639
640− err = i915_gem_object_pin_pages(obj);
641− if (err)
642− return err;
640+ if (WARN_ON(!i915_gem_object_trylock(obj)))
641+ return -EBUSY;
643642
644− i915_gem_object_init_memory_region(obj, mem, 0);
643+ err = i915_gem_object_pin_pages(obj);
644+ if (!err)
645+ i915_gem_object_init_memory_region(obj, mem);
646+ i915_gem_object_unlock(obj);
645647
646− return 0;
648+ return err;
647649 }
648650
649651 static int _i915_gem_object_stolen_init(struct intel_memory_region *mem,
drivers/gpu/drm/i915/gem/i915_gem_tiling.c+0−2
@@ -265,7 +265,6 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, · 264 unmodified lines
265265 * pages to prevent them being swapped out and causing corruption
266266 * due to the change in swizzling.
267267 */
268− mutex_lock(&obj->mm.lock);
269268 if (i915_gem_object_has_pages(obj) &&
270269 obj->mm.madv == I915_MADV_WILLNEED &&
271270 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
@@ -280,7 +279,6 @@ i915_gem_object_set_tiling(struct drm_i915_gem_object *obj, · 8 unmodified lines
280279 i915_gem_object_set_tiling_quirk(obj);
281280 }
282281 }
283− mutex_unlock(&obj->mm.lock);
284282
285283 spin_lock(&obj->vma.lock);
286284 for_each_ggtt_vma(vma, obj) {
drivers/gpu/drm/i915/gem/i915_gem_userptr.c+320−582
@@ -2,10 +2,39 @@ · 1 unmodified line
22 * SPDX-License-Identifier: MIT
33 *
44 * Copyright © 2012-2014 Intel Corporation
5+ *
6+ * Based on amdgpu_mn, which bears the following notice:
7+ *
8+ * Copyright 2014 Advanced Micro Devices, Inc.
9+ * All Rights Reserved.
10+ *
11+ * Permission is hereby granted, free of charge, to any person obtaining a
12+ * copy of this software and associated documentation files (the
13+ * "Software"), to deal in the Software without restriction, including
14+ * without limitation the rights to use, copy, modify, merge, publish,
15+ * distribute, sub license, and/or sell copies of the Software, and to
16+ * permit persons to whom the Software is furnished to do so, subject to
17+ * the following conditions:
18+ *
19+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
26+ *
27+ * The above copyright notice and this permission notice (including the
28+ * next paragraph) shall be included in all copies or substantial portions
29+ * of the Software.
30+ *
31+ */
32+/*
33+ * Authors:
34+ * Christian König <christian.koenig@amd.com>
535 */
636
737 #include <linux/mmu_context.h>
8−#include <linux/mmu_notifier.h>
938 #include <linux/mempolicy.h>
1039 #include <linux/swap.h>
1140 #include <linux/sched/mm.h>
@@ -15,408 +44,121 @@ · 3 unmodified lines
1544 #include "i915_gem_object.h"
1645 #include "i915_scatterlist.h"
1746
18−struct i915_mm_struct {
19− struct mm_struct *mm;
20− struct drm_i915_private *i915;
21− struct i915_mmu_notifier *mn;
22− struct hlist_node node;
23− struct kref kref;
24− struct rcu_work work;
25−};
26−
27−#if defined(CONFIG_MMU_NOTIFIER)
28−#include <linux/interval_tree.h>
47+#ifdef CONFIG_MMU_NOTIFIER
2948
30−struct i915_mmu_notifier {
31− spinlock_t lock;
32− struct hlist_node node;
33− struct mmu_notifier mn;
34− struct rb_root_cached objects;
35− struct i915_mm_struct *mm;
36−};
37−
38−struct i915_mmu_object {
39− struct i915_mmu_notifier *mn;
40− struct drm_i915_gem_object *obj;
41− struct interval_tree_node it;
42−};
43−
44−static void add_object(struct i915_mmu_object *mo)
49+/**
50+ * i915_gem_userptr_invalidate - callback to notify about mm change
51+ *
52+ * @mni: the range (mm) is about to update
53+ * @range: details on the invalidation
54+ * @cur_seq: Value to pass to mmu_interval_set_seq()
55+ *
56+ * Block for operations on BOs to finish and mark pages as accessed and
57+ * potentially dirty.
58+ */
59+static bool i915_gem_userptr_invalidate(struct mmu_interval_notifier *mni,
60+ const struct mmu_notifier_range *range,
61+ unsigned long cur_seq)
4562 {
46− GEM_BUG_ON(!RB_EMPTY_NODE(&mo->it.rb));
47− interval_tree_insert(&mo->it, &mo->mn->objects);
48−}
63+ struct drm_i915_gem_object *obj = container_of(mni, struct drm_i915_gem_object, userptr.notifier);
64+ struct drm_i915_private *i915 = to_i915(obj->base.dev);
65+ long r;
4966
50−static void del_object(struct i915_mmu_object *mo)
51−{
52− if (RB_EMPTY_NODE(&mo->it.rb))
53− return;
67+ if (!mmu_notifier_range_blockable(range))
68+ return false;
5469
55− interval_tree_remove(&mo->it, &mo->mn->objects);
56− RB_CLEAR_NODE(&mo->it.rb);
57−}
70+ spin_lock(&i915->mm.notifier_lock);
5871
59−static void
60−__i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value)
61−{
62− struct i915_mmu_object *mo = obj->userptr.mmu_object;
72+ mmu_interval_set_seq(mni, cur_seq);
73+
74+ spin_unlock(&i915->mm.notifier_lock);
6375
6476 /*
65− * During mm_invalidate_range we need to cancel any userptr that
66− * overlaps the range being invalidated. Doing so requires the
67− * struct_mutex, and that risks recursion. In order to cause
68− * recursion, the user must alias the userptr address space with
69− * a GTT mmapping (possible with a MAP_FIXED) - then when we have
70− * to invalidate that mmaping, mm_invalidate_range is called with
71− * the userptr address *and* the struct_mutex held. To prevent that
72− * we set a flag under the i915_mmu_notifier spinlock to indicate
73− * whether this object is valid.
77+ * We don't wait when the process is exiting. This is valid
78+ * because the object will be cleaned up anyway.
79+ *
80+ * This is also temporarily required as a hack, because we
81+ * cannot currently force non-consistent batch buffers to preempt
82+ * and reschedule by waiting on it, hanging processes on exit.
7483 */
75− if (!mo)
76− return;
84+ if (current->flags & PF_EXITING)
85+ return true;
7786
78− spin_lock(&mo->mn->lock);
79− if (value)
80− add_object(mo);
81− else
82− del_object(mo);
83− spin_unlock(&mo->mn->lock);
84−}
85−
86−static int
87−userptr_mn_invalidate_range_start(struct mmu_notifier *_mn,
88− const struct mmu_notifier_range *range)
89−{
90− struct i915_mmu_notifier *mn =
91− container_of(_mn, struct i915_mmu_notifier, mn);
92− struct interval_tree_node *it;
93− unsigned long end;
94− int ret = 0;
95−
96− if (RB_EMPTY_ROOT(&mn->objects.rb_root))
97− return 0;
98−
99− /* interval ranges are inclusive, but invalidate range is exclusive */
100− end = range->end - 1;
101−
102− spin_lock(&mn->lock);
103− it = interval_tree_iter_first(&mn->objects, range->start, end);
104− while (it) {
105− struct drm_i915_gem_object *obj;
106−
107− if (!mmu_notifier_range_blockable(range)) {
108− ret = -EAGAIN;
109− break;
110− }
111−
112− /*
113− * The mmu_object is released late when destroying the
114− * GEM object so it is entirely possible to gain a
115− * reference on an object in the process of being freed
116− * since our serialisation is via the spinlock and not
117− * the struct_mutex - and consequently use it after it
118− * is freed and then double free it. To prevent that
119− * use-after-free we only acquire a reference on the
120− * object if it is not in the process of being destroyed.
121− */
122− obj = container_of(it, struct i915_mmu_object, it)->obj;
123− if (!kref_get_unless_zero(&obj->base.refcount)) {
124− it = interval_tree_iter_next(it, range->start, end);
125− continue;
126− }
127− spin_unlock(&mn->lock);
128−
129− ret = i915_gem_object_unbind(obj,
130− I915_GEM_OBJECT_UNBIND_ACTIVE |
131− I915_GEM_OBJECT_UNBIND_BARRIER);
132− if (ret == 0)
133− ret = __i915_gem_object_put_pages(obj);
134− i915_gem_object_put(obj);
135− if (ret)
136− return ret;
137−
138− spin_lock(&mn->lock);
139−
140− /*
141− * As we do not (yet) protect the mmu from concurrent insertion
142− * over this range, there is no guarantee that this search will
143− * terminate given a pathologic workload.
144− */
145− it = interval_tree_iter_first(&mn->objects, range->start, end);
146− }
147− spin_unlock(&mn->lock);
148−
149− return ret;
87+ /* we will unbind on next submission, still have userptr pins */
88+ r = dma_resv_wait_timeout_rcu(obj->base.resv, true, false,
89+ MAX_SCHEDULE_TIMEOUT);
90+ if (r <= 0)
91+ drm_err(&i915->drm, "(%ld) failed to wait for idle\n", r);
15092
93+ return true;
15194 }
15295
153−static const struct mmu_notifier_ops i915_gem_userptr_notifier = {
154− .invalidate_range_start = userptr_mn_invalidate_range_start,
96+static const struct mmu_interval_notifier_ops i915_gem_userptr_notifier_ops = {
97+ .invalidate = i915_gem_userptr_invalidate,
15598 };
15699
157−static struct i915_mmu_notifier *
158−i915_mmu_notifier_create(struct i915_mm_struct *mm)
159−{
160− struct i915_mmu_notifier *mn;
161−
162− mn = kmalloc(sizeof(*mn), GFP_KERNEL);
163− if (mn == NULL)
164− return ERR_PTR(-ENOMEM);
165−
166− spin_lock_init(&mn->lock);
167− mn->mn.ops = &i915_gem_userptr_notifier;
168− mn->objects = RB_ROOT_CACHED;
169− mn->mm = mm;
170−
171− return mn;
172−}
173−
174−static void
175−i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)
176−{
177− struct i915_mmu_object *mo;
178−
179− mo = fetch_and_zero(&obj->userptr.mmu_object);
180− if (!mo)
181− return;
182−
183− spin_lock(&mo->mn->lock);
184− del_object(mo);
185− spin_unlock(&mo->mn->lock);
186− kfree(mo);
187−}
188−
189−static struct i915_mmu_notifier *
190−i915_mmu_notifier_find(struct i915_mm_struct *mm)
191−{
192− struct i915_mmu_notifier *mn, *old;
193− int err;
194−
195− mn = READ_ONCE(mm->mn);
196− if (likely(mn))
197− return mn;
198−
199− mn = i915_mmu_notifier_create(mm);
200− if (IS_ERR(mn))
201− return mn;
202−
203− err = mmu_notifier_register(&mn->mn, mm->mm);
204− if (err) {
205− kfree(mn);
206− return ERR_PTR(err);
207− }
208−
209− old = cmpxchg(&mm->mn, NULL, mn);
210− if (old) {
211− mmu_notifier_unregister(&mn->mn, mm->mm);
212− kfree(mn);
213− mn = old;
214− }
215−
216− return mn;
217−}
218−
219−static int
220−i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,
221− unsigned flags)
222−{
223− struct i915_mmu_notifier *mn;
224− struct i915_mmu_object *mo;
225−
226− if (flags & I915_USERPTR_UNSYNCHRONIZED)
227− return capable(CAP_SYS_ADMIN) ? 0 : -EPERM;
228−
229− if (GEM_WARN_ON(!obj->userptr.mm))
230− return -EINVAL;
231−
232− mn = i915_mmu_notifier_find(obj->userptr.mm);
233− if (IS_ERR(mn))
234− return PTR_ERR(mn);
235−
236− mo = kzalloc(sizeof(*mo), GFP_KERNEL);
237− if (!mo)
238− return -ENOMEM;
239−
240− mo->mn = mn;
241− mo->obj = obj;
242− mo->it.start = obj->userptr.ptr;
243− mo->it.last = obj->userptr.ptr + obj->base.size - 1;
244− RB_CLEAR_NODE(&mo->it.rb);
245−
246− obj->userptr.mmu_object = mo;
247− return 0;
248−}
249−
250−static void
251−i915_mmu_notifier_free(struct i915_mmu_notifier *mn,
252− struct mm_struct *mm)
253−{
254− if (mn == NULL)
255− return;
256−
257− mmu_notifier_unregister(&mn->mn, mm);
258− kfree(mn);
259−}
260−
261−#else
262−
263−static void
264−__i915_gem_userptr_set_active(struct drm_i915_gem_object *obj, bool value)
265−{
266−}
267−
268−static void
269−i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)
270−{
271−}
272−
273100 static int
274−i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj,
275− unsigned flags)
101+i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj)
276102 {
277− if ((flags & I915_USERPTR_UNSYNCHRONIZED) == 0)
278− return -ENODEV;
279−
280− if (!capable(CAP_SYS_ADMIN))
281− return -EPERM;
282−
283− return 0;
103+ return mmu_interval_notifier_insert(&obj->userptr.notifier, current->mm,
104+ obj->userptr.ptr, obj->base.size,
105+ &i915_gem_userptr_notifier_ops);
284106 }
285107
286−static void
287−i915_mmu_notifier_free(struct i915_mmu_notifier *mn,
288− struct mm_struct *mm)
289−{
290−}
291−
292−#endif
293−
294−static struct i915_mm_struct *
295−__i915_mm_struct_find(struct drm_i915_private *i915, struct mm_struct *real)
296−{
297− struct i915_mm_struct *it, *mm = NULL;
298−
299− rcu_read_lock();
300− hash_for_each_possible_rcu(i915->mm_structs,
301− it, node,
302− (unsigned long)real)
303− if (it->mm == real && kref_get_unless_zero(&it->kref)) {
304− mm = it;
305− break;
306− }
307− rcu_read_unlock();
308−
309− return mm;
310−}
311−
312−static int
313−i915_gem_userptr_init__mm_struct(struct drm_i915_gem_object *obj)
108+static void i915_gem_object_userptr_drop_ref(struct drm_i915_gem_object *obj)
314109 {
315110 struct drm_i915_private *i915 = to_i915(obj->base.dev);
316− struct i915_mm_struct *mm, *new;
317− int ret = 0;
318−
319− /* During release of the GEM object we hold the struct_mutex. This
320− * precludes us from calling mmput() at that time as that may be
321− * the last reference and so call exit_mmap(). exit_mmap() will
322− * attempt to reap the vma, and if we were holding a GTT mmap
323− * would then call drm_gem_vm_close() and attempt to reacquire
324− * the struct mutex. So in order to avoid that recursion, we have
325− * to defer releasing the mm reference until after we drop the
326− * struct_mutex, i.e. we need to schedule a worker to do the clean
327− * up.
328− */
329− mm = __i915_mm_struct_find(i915, current->mm);
330− if (mm)
331− goto out;
332−
333− new = kmalloc(sizeof(*mm), GFP_KERNEL);
334− if (!new)
335− return -ENOMEM;
111+ struct page **pvec = NULL;
336112
337− kref_init(&new->kref);
338− new->i915 = to_i915(obj->base.dev);
339− new->mm = current->mm;
340− new->mn = NULL;
341−
342− spin_lock(&i915->mm_lock);
343− mm = __i915_mm_struct_find(i915, current->mm);
344− if (!mm) {
345− hash_add_rcu(i915->mm_structs,
346− &new->node,
347− (unsigned long)new->mm);
348− mmgrab(current->mm);
349− mm = new;
113+ spin_lock(&i915->mm.notifier_lock);
114+ if (!--obj->userptr.page_ref) {
115+ pvec = obj->userptr.pvec;
116+ obj->userptr.pvec = NULL;
350117 }
351− spin_unlock(&i915->mm_lock);
352− if (mm != new)
353− kfree(new);
118+ GEM_BUG_ON(obj->userptr.page_ref < 0);
119+ spin_unlock(&i915->mm.notifier_lock);
354120
355−out:
356− obj->userptr.mm = mm;
357− return ret;
358−}
359−
360−static void
361−__i915_mm_struct_free__worker(struct work_struct *work)
362−{
363− struct i915_mm_struct *mm = container_of(work, typeof(*mm), work.work);
364−
365− i915_mmu_notifier_free(mm->mn, mm->mm);
366− mmdrop(mm->mm);
367− kfree(mm);
368−}
369−
370−static void
371−__i915_mm_struct_free(struct kref *kref)
372−{
373− struct i915_mm_struct *mm = container_of(kref, typeof(*mm), kref);
121+ if (pvec) {
122+ const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
374123
375− spin_lock(&mm->i915->mm_lock);
376− hash_del_rcu(&mm->node);
377− spin_unlock(&mm->i915->mm_lock);
378−
379− INIT_RCU_WORK(&mm->work, __i915_mm_struct_free__worker);
380− queue_rcu_work(system_wq, &mm->work);
381−}
382−
383−static void
384−i915_gem_userptr_release__mm_struct(struct drm_i915_gem_object *obj)
385−{
386− if (obj->userptr.mm == NULL)
387− return;
388−
389− kref_put(&obj->userptr.mm->kref, __i915_mm_struct_free);
390− obj->userptr.mm = NULL;
124+ unpin_user_pages(pvec, num_pages);
125+ kvfree(pvec);
126+ }
391127 }
392128
393−struct get_pages_work {
394− struct work_struct work;
395− struct drm_i915_gem_object *obj;
396− struct task_struct *task;
397−};
398−
399−static struct sg_table *
400−__i915_gem_userptr_alloc_pages(struct drm_i915_gem_object *obj,
401− struct page **pvec, unsigned long num_pages)
129+static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
402130 {
131+ struct drm_i915_private *i915 = to_i915(obj->base.dev);
132+ const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
403133 unsigned int max_segment = i915_sg_segment_size();
404134 struct sg_table *st;
405135 unsigned int sg_page_sizes;
406136 struct scatterlist *sg;
137+ struct page **pvec;
407138 int ret;
408139
409140 st = kmalloc(sizeof(*st), GFP_KERNEL);
410141 if (!st)
411− return ERR_PTR(-ENOMEM);
142+ return -ENOMEM;
143+
144+ spin_lock(&i915->mm.notifier_lock);
145+ if (GEM_WARN_ON(!obj->userptr.page_ref)) {
146+ spin_unlock(&i915->mm.notifier_lock);
147+ ret = -EFAULT;
148+ goto err_free;
149+ }
150+
151+ obj->userptr.page_ref++;
152+ pvec = obj->userptr.pvec;
153+ spin_unlock(&i915->mm.notifier_lock);
412154
413155 alloc_table:
414156 sg = __sg_alloc_table_from_pages(st, pvec, num_pages, 0,
415157 num_pages << PAGE_SHIFT, max_segment,
416158 NULL, 0, GFP_KERNEL);
417159 if (IS_ERR(sg)) {
418− kfree(st);
419− return ERR_CAST(sg);
160+ ret = PTR_ERR(sg);
161+ goto err;
420162 }
421163
422164 ret = i915_gem_gtt_prepare_pages(obj, st);
@@ -428,203 +170,20 @@ alloc_table: · 5 unmodified lines
428170 goto alloc_table;
429171 }
430172
431− kfree(st);
432− return ERR_PTR(ret);
173+ goto err;
433174 }
434175
435176 sg_page_sizes = i915_sg_page_sizes(st->sgl);
436177
437178 __i915_gem_object_set_pages(obj, st, sg_page_sizes);
438179
439− return st;
440−}
441−
442−static void
443−__i915_gem_userptr_get_pages_worker(struct work_struct *_work)
444−{
445− struct get_pages_work *work = container_of(_work, typeof(*work), work);
446− struct drm_i915_gem_object *obj = work->obj;
447− const unsigned long npages = obj->base.size >> PAGE_SHIFT;
448− unsigned long pinned;
449− struct page **pvec;
450− int ret;
451−
452− ret = -ENOMEM;
453− pinned = 0;
454−
455− pvec = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
456− if (pvec != NULL) {
457− struct mm_struct *mm = obj->userptr.mm->mm;
458− unsigned int flags = 0;
459− int locked = 0;
460−
461− if (!i915_gem_object_is_readonly(obj))
462− flags |= FOLL_WRITE;
463−
464− ret = -EFAULT;
465− if (mmget_not_zero(mm)) {
466− while (pinned < npages) {
467− if (!locked) {
468− mmap_read_lock(mm);
469− locked = 1;
470− }
471− ret = pin_user_pages_remote
472− (mm,
473− obj->userptr.ptr + pinned * PAGE_SIZE,
474− npages - pinned,
475− flags,
476− pvec + pinned, NULL, &locked);
477− if (ret < 0)
478− break;
479−
480− pinned += ret;
481− }
482− if (locked)
483− mmap_read_unlock(mm);
484− mmput(mm);
485− }
486− }
487−
488− mutex_lock_nested(&obj->mm.lock, I915_MM_GET_PAGES);
489− if (obj->userptr.work == &work->work) {
490− struct sg_table *pages = ERR_PTR(ret);
491−
492− if (pinned == npages) {
493− pages = __i915_gem_userptr_alloc_pages(obj, pvec,
494− npages);
495− if (!IS_ERR(pages)) {
496− pinned = 0;
497− pages = NULL;
498− }
499− }
500−
501− obj->userptr.work = ERR_CAST(pages);
502− if (IS_ERR(pages))
503− __i915_gem_userptr_set_active(obj, false);
504− }
505− mutex_unlock(&obj->mm.lock);
506−
507− unpin_user_pages(pvec, pinned);
508− kvfree(pvec);
509−
510− i915_gem_object_put(obj);
511− put_task_struct(work->task);
512− kfree(work);
513−}
514−
515−static struct sg_table *
516−__i915_gem_userptr_get_pages_schedule(struct drm_i915_gem_object *obj)
517−{
518− struct get_pages_work *work;
519−
520− /* Spawn a worker so that we can acquire the
521− * user pages without holding our mutex. Access
522− * to the user pages requires mmap_lock, and we have
523− * a strict lock ordering of mmap_lock, struct_mutex -
524− * we already hold struct_mutex here and so cannot
525− * call gup without encountering a lock inversion.
526− *
527− * Userspace will keep on repeating the operation
528− * (thanks to EAGAIN) until either we hit the fast
529− * path or the worker completes. If the worker is
530− * cancelled or superseded, the task is still run
531− * but the results ignored. (This leads to
532− * complications that we may have a stray object
533− * refcount that we need to be wary of when
534− * checking for existing objects during creation.)
535− * If the worker encounters an error, it reports
536− * that error back to this function through
537− * obj->userptr.work = ERR_PTR.
538− */
539− work = kmalloc(sizeof(*work), GFP_KERNEL);
540− if (work == NULL)
541− return ERR_PTR(-ENOMEM);
542−
543− obj->userptr.work = &work->work;
544−
545− work->obj = i915_gem_object_get(obj);
546−
547− work->task = current;
548− get_task_struct(work->task);
549−
550− INIT_WORK(&work->work, __i915_gem_userptr_get_pages_worker);
551− queue_work(to_i915(obj->base.dev)->mm.userptr_wq, &work->work);
552−
553− return ERR_PTR(-EAGAIN);
554−}
555−
556−static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
557−{
558− const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
559− struct mm_struct *mm = obj->userptr.mm->mm;
560− struct page **pvec;
561− struct sg_table *pages;
562− bool active;
563− int pinned;
564− unsigned int gup_flags = 0;
565−
566− /* If userspace should engineer that these pages are replaced in
567− * the vma between us binding this page into the GTT and completion
568− * of rendering... Their loss. If they change the mapping of their
569− * pages they need to create a new bo to point to the new vma.
570− *
571− * However, that still leaves open the possibility of the vma
572− * being copied upon fork. Which falls under the same userspace
573− * synchronisation issue as a regular bo, except that this time
574− * the process may not be expecting that a particular piece of
575− * memory is tied to the GPU.
576− *
577− * Fortunately, we can hook into the mmu_notifier in order to
578− * discard the page references prior to anything nasty happening
579− * to the vma (discard or cloning) which should prevent the more
580− * egregious cases from causing harm.
581− */
582−
583− if (obj->userptr.work) {
584− /* active flag should still be held for the pending work */
585− if (IS_ERR(obj->userptr.work))
586− return PTR_ERR(obj->userptr.work);
587− else
588− return -EAGAIN;
589− }
590−
591− pvec = NULL;
592− pinned = 0;
593−
594− if (mm == current->mm) {
595− pvec = kvmalloc_array(num_pages, sizeof(struct page *),
596− GFP_KERNEL |
597− __GFP_NORETRY |
598− __GFP_NOWARN);
599− if (pvec) {
600− /* defer to worker if malloc fails */
601− if (!i915_gem_object_is_readonly(obj))
602− gup_flags |= FOLL_WRITE;
603− pinned = pin_user_pages_fast_only(obj->userptr.ptr,
604− num_pages, gup_flags,
605− pvec);
606− }
607− }
608−
609− active = false;
610− if (pinned < 0) {
611− pages = ERR_PTR(pinned);
612− pinned = 0;
613− } else if (pinned < num_pages) {
614− pages = __i915_gem_userptr_get_pages_schedule(obj);
615− active = pages == ERR_PTR(-EAGAIN);
616− } else {
617− pages = __i915_gem_userptr_alloc_pages(obj, pvec, num_pages);
618− active = !IS_ERR(pages);
619− }
620− if (active)
621− __i915_gem_userptr_set_active(obj, true);
622−
623− if (IS_ERR(pages))
624− unpin_user_pages(pvec, pinned);
625− kvfree(pvec);
180+ return 0;
626181
627− return PTR_ERR_OR_ZERO(pages);
182+err:
183+ i915_gem_object_userptr_drop_ref(obj);
184+err_free:
185+ kfree(st);
186+ return ret;
628187 }
629188
630189 static void
@@ -634,9 +193,6 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj, · 3 unmodified lines
634193 struct sgt_iter sgt_iter;
635194 struct page *page;
636195
637− /* Cancel any inflight work and force them to restart their gup */
638− obj->userptr.work = NULL;
639− __i915_gem_userptr_set_active(obj, false);
640196 if (!pages)
641197 return;
642198
@@ -676,42 +232,224 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj, · 33 unmodified lines
676232 }
677233
678234 mark_page_accessed(page);
679− unpin_user_page(page);
680235 }
681236 obj->mm.dirty = false;
682237
683238 sg_free_table(pages);
684239 kfree(pages);
240+
241+ i915_gem_object_userptr_drop_ref(obj);
242+}
243+
244+static int i915_gem_object_userptr_unbind(struct drm_i915_gem_object *obj, bool get_pages)
245+{
246+ struct sg_table *pages;
247+ int err;
248+
249+ err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
250+ if (err)
251+ return err;
252+
253+ if (GEM_WARN_ON(i915_gem_object_has_pinned_pages(obj)))
254+ return -EBUSY;
255+
256+ assert_object_held(obj);
257+
258+ pages = __i915_gem_object_unset_pages(obj);
259+ if (!IS_ERR_OR_NULL(pages))
260+ i915_gem_userptr_put_pages(obj, pages);
261+
262+ if (get_pages)
263+ err = ____i915_gem_object_get_pages(obj);
264+
265+ return err;
266+}
267+
268+int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj)
269+{
270+ struct drm_i915_private *i915 = to_i915(obj->base.dev);
271+ const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
272+ struct page **pvec;
273+ unsigned int gup_flags = 0;
274+ unsigned long notifier_seq;
275+ int pinned, ret;
276+
277+ if (obj->userptr.notifier.mm != current->mm)
278+ return -EFAULT;
279+
280+ ret = i915_gem_object_lock_interruptible(obj, NULL);
281+ if (ret)
282+ return ret;
283+
284+ /* optimistically try to preserve current pages while unlocked */
285+ if (i915_gem_object_has_pages(obj) &&
286+ !mmu_interval_check_retry(&obj->userptr.notifier,
287+ obj->userptr.notifier_seq)) {
288+ spin_lock(&i915->mm.notifier_lock);
289+ if (obj->userptr.pvec &&
290+ !mmu_interval_read_retry(&obj->userptr.notifier,
291+ obj->userptr.notifier_seq)) {
292+ obj->userptr.page_ref++;
293+
294+ /* We can keep using the current binding, this is the fastpath */
295+ ret = 1;
296+ }
297+ spin_unlock(&i915->mm.notifier_lock);
298+ }
299+
300+ if (!ret) {
301+ /* Make sure userptr is unbound for next attempt, so we don't use stale pages. */
302+ ret = i915_gem_object_userptr_unbind(obj, false);
303+ }
304+ i915_gem_object_unlock(obj);
305+ if (ret < 0)
306+ return ret;
307+
308+ if (ret > 0)
309+ return 0;
310+
311+ notifier_seq = mmu_interval_read_begin(&obj->userptr.notifier);
312+
313+ pvec = kvmalloc_array(num_pages, sizeof(struct page *), GFP_KERNEL);
314+ if (!pvec)
315+ return -ENOMEM;
316+
317+ if (!i915_gem_object_is_readonly(obj))
318+ gup_flags |= FOLL_WRITE;
319+
320+ pinned = ret = 0;
321+ while (pinned < num_pages) {
322+ ret = pin_user_pages_fast(obj->userptr.ptr + pinned * PAGE_SIZE,
323+ num_pages - pinned, gup_flags,
324+ &pvec[pinned]);
325+ if (ret < 0)
326+ goto out;
327+
328+ pinned += ret;
329+ }
330+ ret = 0;
331+
332+ spin_lock(&i915->mm.notifier_lock);
333+
334+ if (mmu_interval_read_retry(&obj->userptr.notifier,
335+ !obj->userptr.page_ref ? notifier_seq :
336+ obj->userptr.notifier_seq)) {
337+ ret = -EAGAIN;
338+ goto out_unlock;
339+ }
340+
341+ if (!obj->userptr.page_ref++) {
342+ obj->userptr.pvec = pvec;
343+ obj->userptr.notifier_seq = notifier_seq;
344+
345+ pvec = NULL;
346+ }
347+
348+out_unlock:
349+ spin_unlock(&i915->mm.notifier_lock);
350+
351+out:
352+ if (pvec) {
353+ unpin_user_pages(pvec, pinned);
354+ kvfree(pvec);
355+ }
356+
357+ return ret;
358+}
359+
360+int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj)
361+{
362+ if (mmu_interval_read_retry(&obj->userptr.notifier,
363+ obj->userptr.notifier_seq)) {
364+ /* We collided with the mmu notifier, need to retry */
365+
366+ return -EAGAIN;
367+ }
368+
369+ return 0;
370+}
371+
372+void i915_gem_object_userptr_submit_fini(struct drm_i915_gem_object *obj)
373+{
374+ i915_gem_object_userptr_drop_ref(obj);
375+}
376+
377+int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj)
378+{
379+ int err;
380+
381+ err = i915_gem_object_userptr_submit_init(obj);
382+ if (err)
383+ return err;
384+
385+ err = i915_gem_object_lock_interruptible(obj, NULL);
386+ if (!err) {
387+ /*
388+ * Since we only check validity, not use the pages,
389+ * it doesn't matter if we collide with the mmu notifier,
390+ * and -EAGAIN handling is not required.
391+ */
392+ err = i915_gem_object_pin_pages(obj);
393+ if (!err)
394+ i915_gem_object_unpin_pages(obj);
395+
396+ i915_gem_object_unlock(obj);
397+ }
398+
399+ i915_gem_object_userptr_submit_fini(obj);
400+ return err;
685401 }
686402
687403 static void
688404 i915_gem_userptr_release(struct drm_i915_gem_object *obj)
689405 {
690− i915_gem_userptr_release__mmu_notifier(obj);
691− i915_gem_userptr_release__mm_struct(obj);
406+ GEM_WARN_ON(obj->userptr.page_ref);
407+
408+ mmu_interval_notifier_remove(&obj->userptr.notifier);
409+ obj->userptr.notifier.mm = NULL;
692410 }
693411
694412 static int
695413 i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)
696414 {
697− if (obj->userptr.mmu_object)
698− return 0;
415+ drm_dbg(obj->base.dev, "Exporting userptr no longer allowed\n");
699416
700− return i915_gem_userptr_init__mmu_notifier(obj, 0);
417+ return -EINVAL;
418+}
419+
420+static int
421+i915_gem_userptr_pwrite(struct drm_i915_gem_object *obj,
422+ const struct drm_i915_gem_pwrite *args)
423+{
424+ drm_dbg(obj->base.dev, "pwrite to userptr no longer allowed\n");
425+
426+ return -EINVAL;
427+}
428+
429+static int
430+i915_gem_userptr_pread(struct drm_i915_gem_object *obj,
431+ const struct drm_i915_gem_pread *args)
432+{
433+ drm_dbg(obj->base.dev, "pread from userptr no longer allowed\n");
434+
435+ return -EINVAL;
701436 }
702437
703438 static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {
704439 .name = "i915_gem_object_userptr",
705− .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
706− I915_GEM_OBJECT_IS_SHRINKABLE |
440+ .flags = I915_GEM_OBJECT_IS_SHRINKABLE |
707441 I915_GEM_OBJECT_NO_MMAP |
708− I915_GEM_OBJECT_ASYNC_CANCEL,
442+ I915_GEM_OBJECT_IS_PROXY,
709443 .get_pages = i915_gem_userptr_get_pages,
710444 .put_pages = i915_gem_userptr_put_pages,
711445 .dmabuf_export = i915_gem_userptr_dmabuf_export,
446+ .pwrite = i915_gem_userptr_pwrite,
447+ .pread = i915_gem_userptr_pread,
712448 .release = i915_gem_userptr_release,
713449 };
714450
451+#endif
452+
715453 /*
716454 * Creates a new mm object that wraps some normal memory from the process
717455 * context - user memory.
@@ -752,12 +490,12 @@ i915_gem_userptr_ioctl(struct drm_device *dev, · 34 unmodified lines
752490 void *data,
753491 struct drm_file *file)
754492 {
755− static struct lock_class_key lock_class;
493+ static struct lock_class_key __maybe_unused lock_class;
756494 struct drm_i915_private *dev_priv = to_i915(dev);
757495 struct drm_i915_gem_userptr *args = data;
758− struct drm_i915_gem_object *obj;
759− int ret;
760− u32 handle;
496+ struct drm_i915_gem_object __maybe_unused *obj;
497+ int __maybe_unused ret;
498+ u32 __maybe_unused handle;
761499
762500 if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) {
763501 /* We cannot support coherent userptr objects on hw without
@@ -796,6 +534,9 @@ i915_gem_userptr_ioctl(struct drm_device *dev, · 32 unmodified lines
796534 if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size))
797535 return -EFAULT;
798536
537+ if (args->flags & I915_USERPTR_UNSYNCHRONIZED)
538+ return -ENODEV;
539+
799540 if (args->flags & I915_USERPTR_READ_ONLY) {
800541 /*
801542 * On almost all of the older hw, we cannot tell the GPU that
@@ -805,17 +546,20 @@ i915_gem_userptr_ioctl(struct drm_device *dev, · 3 unmodified lines
805546 return -ENODEV;
806547 }
807548
549+#ifdef CONFIG_MMU_NOTIFIER
808550 obj = i915_gem_object_alloc();
809551 if (obj == NULL)
810552 return -ENOMEM;
811553
812554 drm_gem_private_object_init(dev, &obj->base, args->user_size);
813− i915_gem_object_init(obj, &i915_gem_userptr_ops, &lock_class);
555+ i915_gem_object_init(obj, &i915_gem_userptr_ops, &lock_class,
556+ I915_BO_ALLOC_STRUCT_PAGE);
814557 obj->read_domains = I915_GEM_DOMAIN_CPU;
815558 obj->write_domain = I915_GEM_DOMAIN_CPU;
816559 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
817560
818561 obj->userptr.ptr = args->user_ptr;
562+ obj->userptr.notifier_seq = ULONG_MAX;
819563 if (args->flags & I915_USERPTR_READ_ONLY)
820564 i915_gem_object_set_readonly(obj);
821565
@@ -823,9 +567,7 @@ i915_gem_userptr_ioctl(struct drm_device *dev, · 1 unmodified line
823567 * at binding. This means that we need to hook into the mmu_notifier
824568 * in order to detect if the mmu is destroyed.
825569 */
826− ret = i915_gem_userptr_init__mm_struct(obj);
827− if (ret == 0)
828− ret = i915_gem_userptr_init__mmu_notifier(obj, args->flags);
570+ ret = i915_gem_userptr_init__mmu_notifier(obj);
829571 if (ret == 0)
830572 ret = drm_gem_handle_create(file, &obj->base, &handle);
831573
@@ -836,24 +578,20 @@ i915_gem_userptr_ioctl(struct drm_device *dev, · 4 unmodified lines
836578
837579 args->handle = handle;
838580 return 0;
581+#else
582+ return -ENODEV;
583+#endif
839584 }
840585
841586 int i915_gem_init_userptr(struct drm_i915_private *dev_priv)
842587 {
843− spin_lock_init(&dev_priv->mm_lock);
844− hash_init(dev_priv->mm_structs);
845−
846− dev_priv->mm.userptr_wq =
847− alloc_workqueue("i915-userptr-acquire",
848− WQ_HIGHPRI | WQ_UNBOUND,
849− 0);
850− if (!dev_priv->mm.userptr_wq)
851− return -ENOMEM;
588+#ifdef CONFIG_MMU_NOTIFIER
589+ spin_lock_init(&dev_priv->mm.notifier_lock);
590+#endif
852591
853592 return 0;
854593 }
855594
856595 void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv)
857596 {
858− destroy_workqueue(dev_priv->mm.userptr_wq);
859597 }
drivers/gpu/drm/i915/gem/selftests/huge_gem_object.c+2−2
@@ -89,7 +89,6 @@ static void huge_put_pages(struct drm_i915_gem_object *obj, · 88 unmodified lines
8989
9090 static const struct drm_i915_gem_object_ops huge_ops = {
9191 .name = "huge-gem",
92− .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
9392 .get_pages = huge_get_pages,
9493 .put_pages = huge_put_pages,
9594 };
@@ -115,7 +114,8 @@ huge_gem_object(struct drm_i915_private *i915, · 19 unmodified lines
115114 return ERR_PTR(-ENOMEM);
116115
117116 drm_gem_private_object_init(&i915->drm, &obj->base, dma_size);
118− i915_gem_object_init(obj, &huge_ops, &lock_class);
117+ i915_gem_object_init(obj, &huge_ops, &lock_class,
118+ I915_BO_ALLOC_STRUCT_PAGE);
119119
120120 obj->read_domains = I915_GEM_DOMAIN_CPU;
121121 obj->write_domain = I915_GEM_DOMAIN_CPU;
drivers/gpu/drm/i915/gem/selftests/huge_pages.c+26−12
@@ -140,8 +140,7 @@ static void put_huge_pages(struct drm_i915_gem_object *obj, · 139 unmodified lines
140140
141141 static const struct drm_i915_gem_object_ops huge_page_ops = {
142142 .name = "huge-gem",
143− .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
144− I915_GEM_OBJECT_IS_SHRINKABLE,
143+ .flags = I915_GEM_OBJECT_IS_SHRINKABLE,
145144 .get_pages = get_huge_pages,
146145 .put_pages = put_huge_pages,
147146 };
@@ -168,7 +167,8 @@ huge_pages_object(struct drm_i915_private *i915, · 20 unmodified lines
168167 return ERR_PTR(-ENOMEM);
169168
170169 drm_gem_private_object_init(&i915->drm, &obj->base, size);
171− i915_gem_object_init(obj, &huge_page_ops, &lock_class);
170+ i915_gem_object_init(obj, &huge_page_ops, &lock_class,
171+ I915_BO_ALLOC_STRUCT_PAGE);
172172
173173 i915_gem_object_set_volatile(obj);
174174
@@ -319,9 +319,9 @@ fake_huge_pages_object(struct drm_i915_private *i915, u64 size, bool single) · 144 unmodified lines
319319 drm_gem_private_object_init(&i915->drm, &obj->base, size);
320320
321321 if (single)
322− i915_gem_object_init(obj, &fake_ops_single, &lock_class);
322+ i915_gem_object_init(obj, &fake_ops_single, &lock_class, 0);
323323 else
324− i915_gem_object_init(obj, &fake_ops, &lock_class);
324+ i915_gem_object_init(obj, &fake_ops, &lock_class, 0);
325325
326326 i915_gem_object_set_volatile(obj);
327327
@@ -589,7 +589,7 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg) · 261 unmodified lines
589589 goto out_put;
590590 }
591591
592− err = i915_gem_object_pin_pages(obj);
592+ err = i915_gem_object_pin_pages_unlocked(obj);
593593 if (err)
594594 goto out_put;
595595
@@ -653,15 +653,19 @@ static int igt_mock_ppgtt_misaligned_dma(void *arg) · 57 unmodified lines
653653 break;
654654 }
655655
656+ i915_gem_object_lock(obj, NULL);
656657 i915_gem_object_unpin_pages(obj);
657658 __i915_gem_object_put_pages(obj);
659+ i915_gem_object_unlock(obj);
658660 i915_gem_object_put(obj);
659661 }
660662
661663 return 0;
662664
663665 out_unpin:
666+ i915_gem_object_lock(obj, NULL);
664667 i915_gem_object_unpin_pages(obj);
668+ i915_gem_object_unlock(obj);
665669 out_put:
666670 i915_gem_object_put(obj);
667671
@@ -675,8 +679,10 @@ static void close_object_list(struct list_head *objects, · 7 unmodified lines
675679
676680 list_for_each_entry_safe(obj, on, objects, st_link) {
677681 list_del(&obj->st_link);
682+ i915_gem_object_lock(obj, NULL);
678683 i915_gem_object_unpin_pages(obj);
679684 __i915_gem_object_put_pages(obj);
685+ i915_gem_object_unlock(obj);
680686 i915_gem_object_put(obj);
681687 }
682688 }
@@ -713,7 +719,7 @@ static int igt_mock_ppgtt_huge_fill(void *arg) · 30 unmodified lines
713719 break;
714720 }
715721
716− err = i915_gem_object_pin_pages(obj);
722+ err = i915_gem_object_pin_pages_unlocked(obj);
717723 if (err) {
718724 i915_gem_object_put(obj);
719725 break;
@@ -889,7 +895,7 @@ static int igt_mock_ppgtt_64K(void *arg) · 169 unmodified lines
889895 if (IS_ERR(obj))
890896 return PTR_ERR(obj);
891897
892− err = i915_gem_object_pin_pages(obj);
898+ err = i915_gem_object_pin_pages_unlocked(obj);
893899 if (err)
894900 goto out_object_put;
895901
@@ -943,8 +949,10 @@ static int igt_mock_ppgtt_64K(void *arg) · 47 unmodified lines
943949 }
944950
945951 i915_vma_unpin(vma);
952+ i915_gem_object_lock(obj, NULL);
946953 i915_gem_object_unpin_pages(obj);
947954 __i915_gem_object_put_pages(obj);
955+ i915_gem_object_unlock(obj);
948956 i915_gem_object_put(obj);
949957 }
950958 }
@@ -954,7 +962,9 @@ static int igt_mock_ppgtt_64K(void *arg) · 3 unmodified lines
954962 out_vma_unpin:
955963 i915_vma_unpin(vma);
956964 out_object_unpin:
965+ i915_gem_object_lock(obj, NULL);
957966 i915_gem_object_unpin_pages(obj);
967+ i915_gem_object_unlock(obj);
958968 out_object_put:
959969 i915_gem_object_put(obj);
960970
@@ -1024,7 +1034,7 @@ static int __cpu_check_vmap(struct drm_i915_gem_object *obj, u32 dword, u32 val) · 63 unmodified lines
10241034 if (err)
10251035 return err;
10261036
1027− ptr = i915_gem_object_pin_map(obj, I915_MAP_WC);
1037+ ptr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
10281038 if (IS_ERR(ptr))
10291039 return PTR_ERR(ptr);
10301040
@@ -1304,7 +1314,7 @@ try_again: · 273 unmodified lines
13041314 return err;
13051315 }
13061316
1307− err = i915_gem_object_pin_pages(obj);
1317+ err = i915_gem_object_pin_pages_unlocked(obj);
13081318 if (err) {
13091319 if (err == -ENXIO || err == -E2BIG) {
13101320 i915_gem_object_put(obj);
@@ -1327,8 +1337,10 @@ try_again: · 16 unmodified lines
13271337 __func__, size, i);
13281338 }
13291339 out_unpin:
1340+ i915_gem_object_lock(obj, NULL);
13301341 i915_gem_object_unpin_pages(obj);
13311342 __i915_gem_object_put_pages(obj);
1343+ i915_gem_object_unlock(obj);
13321344 out_put:
13331345 i915_gem_object_put(obj);
13341346
@@ -1402,7 +1414,7 @@ static int igt_ppgtt_sanity_check(void *arg) · 67 unmodified lines
14021414 return err;
14031415 }
14041416
1405− err = i915_gem_object_pin_pages(obj);
1417+ err = i915_gem_object_pin_pages_unlocked(obj);
14061418 if (err) {
14071419 i915_gem_object_put(obj);
14081420 goto out;
@@ -1416,8 +1428,10 @@ static int igt_ppgtt_sanity_check(void *arg) · 7 unmodified lines
14161428
14171429 err = igt_write_huge(ctx, obj);
14181430
1431+ i915_gem_object_lock(obj, NULL);
14191432 i915_gem_object_unpin_pages(obj);
14201433 __i915_gem_object_put_pages(obj);
1434+ i915_gem_object_unlock(obj);
14211435 i915_gem_object_put(obj);
14221436
14231437 if (err) {
@@ -1462,7 +1476,7 @@ static int igt_tmpfs_fallback(void *arg) · 38 unmodified lines
14621476 goto out_restore;
14631477 }
14641478
1465− vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
1479+ vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
14661480 if (IS_ERR(vaddr)) {
14671481 err = PTR_ERR(vaddr);
14681482 goto out_put;
drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c+4−4
@@ -45,7 +45,7 @@ static int __igt_client_fill(struct intel_engine_cs *engine) · 44 unmodified lines
4545 goto err_flush;
4646 }
4747
48− vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
48+ vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
4949 if (IS_ERR(vaddr)) {
5050 err = PTR_ERR(vaddr);
5151 goto err_put;
@@ -157,7 +157,7 @@ static int prepare_blit(const struct tiled_blits *t, · 105 unmodified lines
157157 u32 src_pitch, dst_pitch;
158158 u32 cmd, *cs;
159159
160− cs = i915_gem_object_pin_map(batch, I915_MAP_WC);
160+ cs = i915_gem_object_pin_map_unlocked(batch, I915_MAP_WC);
161161 if (IS_ERR(cs))
162162 return PTR_ERR(cs);
163163
@@ -377,7 +377,7 @@ static int verify_buffer(const struct tiled_blits *t, · 213 unmodified lines
377377 y = i915_prandom_u32_max_state(t->height, prng);
378378 p = y * t->width + x;
379379
380− vaddr = i915_gem_object_pin_map(buf->vma->obj, I915_MAP_WC);
380+ vaddr = i915_gem_object_pin_map_unlocked(buf->vma->obj, I915_MAP_WC);
381381 if (IS_ERR(vaddr))
382382 return PTR_ERR(vaddr);
383383
@@ -564,7 +564,7 @@ static int tiled_blits_prepare(struct tiled_blits *t, · 180 unmodified lines
564564 int err;
565565 int i;
566566
567− map = i915_gem_object_pin_map(t->scratch.vma->obj, I915_MAP_WC);
567+ map = i915_gem_object_pin_map_unlocked(t->scratch.vma->obj, I915_MAP_WC);
568568 if (IS_ERR(map))
569569 return PTR_ERR(map);
570570
drivers/gpu/drm/i915/gem/selftests/i915_gem_coherency.c+6−8
@@ -160,7 +160,7 @@ static int wc_set(struct context *ctx, unsigned long offset, u32 v) · 159 unmodified lines
160160 if (err)
161161 return err;
162162
163− map = i915_gem_object_pin_map(ctx->obj, I915_MAP_WC);
163+ map = i915_gem_object_pin_map_unlocked(ctx->obj, I915_MAP_WC);
164164 if (IS_ERR(map))
165165 return PTR_ERR(map);
166166
@@ -183,7 +183,7 @@ static int wc_get(struct context *ctx, unsigned long offset, u32 *v) · 16 unmodified lines
183183 if (err)
184184 return err;
185185
186− map = i915_gem_object_pin_map(ctx->obj, I915_MAP_WC);
186+ map = i915_gem_object_pin_map_unlocked(ctx->obj, I915_MAP_WC);
187187 if (IS_ERR(map))
188188 return PTR_ERR(map);
189189
@@ -200,17 +200,15 @@ static int gpu_set(struct context *ctx, unsigned long offset, u32 v) · 10 unmodified lines
200200 u32 *cs;
201201 int err;
202202
203+ vma = i915_gem_object_ggtt_pin(ctx->obj, NULL, 0, 0, 0);
204+ if (IS_ERR(vma))
205+ return PTR_ERR(vma);
206+
203207 i915_gem_object_lock(ctx->obj, NULL);
204208 err = i915_gem_object_set_to_gtt_domain(ctx->obj, true);
205209 if (err)
206210 goto out_unlock;
207211
208− vma = i915_gem_object_ggtt_pin(ctx->obj, NULL, 0, 0, 0);
209− if (IS_ERR(vma)) {
210− err = PTR_ERR(vma);
211− goto out_unlock;
212− }
213−
214212 rq = intel_engine_create_kernel_request(ctx->engine);
215213 if (IS_ERR(rq)) {
216214 err = PTR_ERR(rq);
drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c+5−5
@@ -1094,7 +1094,7 @@ __read_slice_count(struct intel_context *ce, · 1093 unmodified lines
10941094 if (ret < 0)
10951095 return ret;
10961096
1097− buf = i915_gem_object_pin_map(obj, I915_MAP_WB);
1097+ buf = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
10981098 if (IS_ERR(buf)) {
10991099 ret = PTR_ERR(buf);
11001100 return ret;
@@ -1511,7 +1511,7 @@ static int write_to_scratch(struct i915_gem_context *ctx, · 410 unmodified lines
15111511 if (IS_ERR(obj))
15121512 return PTR_ERR(obj);
15131513
1514− cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
1514+ cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
15151515 if (IS_ERR(cmd)) {
15161516 err = PTR_ERR(cmd);
15171517 goto out;
@@ -1622,7 +1622,7 @@ static int read_from_scratch(struct i915_gem_context *ctx, · 104 unmodified lines
16221622 if (err)
16231623 goto out_vm;
16241624
1625− cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
1625+ cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
16261626 if (IS_ERR(cmd)) {
16271627 err = PTR_ERR(cmd);
16281628 goto out;
@@ -1658,7 +1658,7 @@ static int read_from_scratch(struct i915_gem_context *ctx, · 29 unmodified lines
16581658 if (err)
16591659 goto out_vm;
16601660
1661− cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
1661+ cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
16621662 if (IS_ERR(cmd)) {
16631663 err = PTR_ERR(cmd);
16641664 goto out;
@@ -1715,7 +1715,7 @@ static int read_from_scratch(struct i915_gem_context *ctx, · 50 unmodified lines
17151715 if (err)
17161716 goto out_vm;
17171717
1718− cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
1718+ cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
17191719 if (IS_ERR(cmd)) {
17201720 err = PTR_ERR(cmd);
17211721 goto out_vm;
drivers/gpu/drm/i915/gem/selftests/i915_gem_dmabuf.c+1−1
@@ -194,7 +194,7 @@ static int igt_dmabuf_import_ownership(void *arg) · 193 unmodified lines
194194
195195 dma_buf_put(dmabuf);
196196
197− err = i915_gem_object_pin_pages(obj);
197+ err = i915_gem_object_pin_pages_unlocked(obj);
198198 if (err) {
199199 pr_err("i915_gem_object_pin_pages failed with err=%d\n", err);
200200 goto out_obj;
drivers/gpu/drm/i915/gem/selftests/i915_gem_execbuffer.c+1−1
@@ -116,7 +116,7 @@ static int igt_gpu_reloc(void *arg) · 115 unmodified lines
116116 if (IS_ERR(scratch))
117117 return PTR_ERR(scratch);
118118
119− map = i915_gem_object_pin_map(scratch, I915_MAP_WC);
119+ map = i915_gem_object_pin_map_unlocked(scratch, I915_MAP_WC);
120120 if (IS_ERR(map)) {
121121 err = PTR_ERR(map);
122122 goto err_scratch;
drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c+10−11
@@ -322,7 +322,7 @@ static int igt_partial_tiling(void *arg) · 321 unmodified lines
322322 if (IS_ERR(obj))
323323 return PTR_ERR(obj);
324324
325− err = i915_gem_object_pin_pages(obj);
325+ err = i915_gem_object_pin_pages_unlocked(obj);
326326 if (err) {
327327 pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
328328 nreal, obj->base.size / PAGE_SIZE, err);
@@ -459,7 +459,7 @@ static int igt_smoke_tiling(void *arg) · 130 unmodified lines
459459 if (IS_ERR(obj))
460460 return PTR_ERR(obj);
461461
462− err = i915_gem_object_pin_pages(obj);
462+ err = i915_gem_object_pin_pages_unlocked(obj);
463463 if (err) {
464464 pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
465465 nreal, obj->base.size / PAGE_SIZE, err);
@@ -798,7 +798,7 @@ static int wc_set(struct drm_i915_gem_object *obj) · 332 unmodified lines
798798 {
799799 void *vaddr;
800800
801− vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
801+ vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
802802 if (IS_ERR(vaddr))
803803 return PTR_ERR(vaddr);
804804
@@ -814,7 +814,7 @@ static int wc_check(struct drm_i915_gem_object *obj) · 9 unmodified lines
814814 void *vaddr;
815815 int err = 0;
816816
817− vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
817+ vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
818818 if (IS_ERR(vaddr))
819819 return PTR_ERR(vaddr);
820820
@@ -835,9 +835,8 @@ static bool can_mmap(struct drm_i915_gem_object *obj, enum i915_mmap_type type) · 14 unmodified lines
835835 return false;
836836
837837 if (type != I915_MMAP_TYPE_GTT &&
838− !i915_gem_object_type_has(obj,
839− I915_GEM_OBJECT_HAS_STRUCT_PAGE |
840− I915_GEM_OBJECT_HAS_IOMEM))
838+ !i915_gem_object_has_struct_page(obj) &&
839+ !i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM))
841840 return false;
842841
843842 return true;
@@ -977,10 +976,8 @@ static const char *repr_mmap_type(enum i915_mmap_type type) · 133 unmodified lines
977976
978977 static bool can_access(const struct drm_i915_gem_object *obj)
979978 {
980− unsigned int flags =
981− I915_GEM_OBJECT_HAS_STRUCT_PAGE | I915_GEM_OBJECT_HAS_IOMEM;
982−
983− return i915_gem_object_type_has(obj, flags);
979+ return i915_gem_object_has_struct_page(obj) ||
980+ i915_gem_object_type_has(obj, I915_GEM_OBJECT_HAS_IOMEM);
984981 }
985982
986983 static int __igt_mmap_access(struct drm_i915_private *i915,
@@ -1319,7 +1316,9 @@ static int __igt_mmap_revoke(struct drm_i915_private *i915, · 332 unmodified lines
13191316 }
13201317
13211318 if (type != I915_MMAP_TYPE_GTT) {
1319+ i915_gem_object_lock(obj, NULL);
13221320 __i915_gem_object_put_pages(obj);
1321+ i915_gem_object_unlock(obj);
13231322 if (i915_gem_object_has_pages(obj)) {
13241323 pr_err("Failed to put-pages object!\n");
13251324 err = -EINVAL;
drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c+1−1
@@ -47,7 +47,7 @@ static int igt_gem_huge(void *arg) · 46 unmodified lines
4747 if (IS_ERR(obj))
4848 return PTR_ERR(obj);
4949
50− err = i915_gem_object_pin_pages(obj);
50+ err = i915_gem_object_pin_pages_unlocked(obj);
5151 if (err) {
5252 pr_err("Failed to allocate %u pages (%lu total), err=%d\n",
5353 nreal, obj->base.size / PAGE_SIZE, err);
drivers/gpu/drm/i915/gem/selftests/i915_gem_object_blt.c+3−3
@@ -262,7 +262,7 @@ static int igt_fill_blt_thread(void *arg) · 261 unmodified lines
262262 goto err_flush;
263263 }
264264
265− vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
265+ vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
266266 if (IS_ERR(vaddr)) {
267267 err = PTR_ERR(vaddr);
268268 goto err_put;
@@ -380,7 +380,7 @@ static int igt_copy_blt_thread(void *arg) · 111 unmodified lines
380380 goto err_flush;
381381 }
382382
383− vaddr = i915_gem_object_pin_map(src, I915_MAP_WB);
383+ vaddr = i915_gem_object_pin_map_unlocked(src, I915_MAP_WB);
384384 if (IS_ERR(vaddr)) {
385385 err = PTR_ERR(vaddr);
386386 goto err_put_src;
@@ -400,7 +400,7 @@ static int igt_copy_blt_thread(void *arg) · 13 unmodified lines
400400 goto err_put_src;
401401 }
402402
403− vaddr = i915_gem_object_pin_map(dst, I915_MAP_WB);
403+ vaddr = i915_gem_object_pin_map_unlocked(dst, I915_MAP_WB);
404404 if (IS_ERR(vaddr)) {
405405 err = PTR_ERR(vaddr);
406406 goto err_put_dst;
drivers/gpu/drm/i915/gem/selftests/i915_gem_phys.c+9−1
@@ -25,13 +25,21 @@ static int mock_phys_object(void *arg) · 24 unmodified lines
2525 goto out;
2626 }
2727
28+ if (!i915_gem_object_has_struct_page(obj)) {
29+ err = -EINVAL;
30+ pr_err("shmem has no struct page\n");
31+ goto out_obj;
32+ }
33+
34+ i915_gem_object_lock(obj, NULL);
2835 err = i915_gem_object_attach_phys(obj, PAGE_SIZE);
36+ i915_gem_object_unlock(obj);
2937 if (err) {
3038 pr_err("i915_gem_object_attach_phys failed, err=%d\n", err);
3139 goto out_obj;
3240 }
3341
34− if (obj->ops != &i915_gem_phys_ops) {
42+ if (i915_gem_object_has_struct_page(obj)) {
3543 pr_err("i915_gem_object_attach_phys did not create a phys object\n");
3644 err = -EINVAL;
3745 goto out_obj;
drivers/gpu/drm/i915/gem/selftests/igt_gem_utils.c+1−1
@@ -55,7 +55,7 @@ igt_emit_store_dw(struct i915_vma *vma, · 54 unmodified lines
5555 if (IS_ERR(obj))
5656 return ERR_CAST(obj);
5757
58− cmd = i915_gem_object_pin_map(obj, I915_MAP_WC);
58+ cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
5959 if (IS_ERR(cmd)) {
6060 err = PTR_ERR(cmd);
6161 goto err;
drivers/gpu/drm/i915/gt/gen2_engine_cs.c+1−1
@@ -143,7 +143,7 @@ static u32 *__gen2_emit_breadcrumb(struct i915_request *rq, u32 *cs, · 142 unmodified lines
143143 int flush, int post)
144144 {
145145 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
146− GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
146+ GEM_BUG_ON(offset_in_page(rq->hwsp_seqno) != I915_GEM_HWS_SEQNO_ADDR);
147147
148148 *cs++ = MI_FLUSH;
149149
drivers/gpu/drm/i915/gt/gen6_engine_cs.c+4−4
@@ -161,7 +161,7 @@ u32 *gen6_emit_breadcrumb_rcs(struct i915_request *rq, u32 *cs) · 160 unmodified lines
161161 PIPE_CONTROL_DC_FLUSH_ENABLE |
162162 PIPE_CONTROL_QW_WRITE |
163163 PIPE_CONTROL_CS_STALL);
164− *cs++ = i915_request_active_timeline(rq)->hwsp_offset |
164+ *cs++ = i915_request_active_seqno(rq) |
165165 PIPE_CONTROL_GLOBAL_GTT;
166166 *cs++ = rq->fence.seqno;
167167
@@ -359,7 +359,7 @@ u32 *gen7_emit_breadcrumb_rcs(struct i915_request *rq, u32 *cs) · 191 unmodified lines
359359 PIPE_CONTROL_QW_WRITE |
360360 PIPE_CONTROL_GLOBAL_GTT_IVB |
361361 PIPE_CONTROL_CS_STALL);
362− *cs++ = i915_request_active_timeline(rq)->hwsp_offset;
362+ *cs++ = i915_request_active_seqno(rq);
363363 *cs++ = rq->fence.seqno;
364364
365365 *cs++ = MI_USER_INTERRUPT;
@@ -374,7 +374,7 @@ u32 *gen7_emit_breadcrumb_rcs(struct i915_request *rq, u32 *cs) · 8 unmodified lines
374374 u32 *gen6_emit_breadcrumb_xcs(struct i915_request *rq, u32 *cs)
375375 {
376376 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
377− GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
377+ GEM_BUG_ON(offset_in_page(rq->hwsp_seqno) != I915_GEM_HWS_SEQNO_ADDR);
378378
379379 *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;
380380 *cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT;
@@ -394,7 +394,7 @@ u32 *gen7_emit_breadcrumb_xcs(struct i915_request *rq, u32 *cs) · 13 unmodified lines
394394 int i;
395395
396396 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma);
397− GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR);
397+ GEM_BUG_ON(offset_in_page(rq->hwsp_seqno) != I915_GEM_HWS_SEQNO_ADDR);
398398
399399 *cs++ = MI_FLUSH_DW | MI_INVALIDATE_TLB |
400400 MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX;
drivers/gpu/drm/i915/gt/gen8_engine_cs.c+6−7
@@ -338,15 +338,14 @@ static u32 preempt_address(struct intel_engine_cs *engine) · 337 unmodified lines
338338
339339 static u32 hwsp_offset(const struct i915_request *rq)
340340 {
341− const struct intel_timeline_cacheline *cl;
341+ const struct intel_timeline *tl;
342342
343− /* Before the request is executed, the timeline/cachline is fixed */
343+ /* Before the request is executed, the timeline is fixed */
344+ tl = rcu_dereference_protected(rq->timeline,
345+ !i915_request_signaled(rq));
344346
345− cl = rcu_dereference_protected(rq->hwsp_cacheline, 1);
346− if (cl)
347− return cl->ggtt_offset;
348−
349− return rcu_dereference_protected(rq->timeline, 1)->hwsp_offset;
347+ /* See the comment in i915_request_active_seqno(). */
348+ return page_mask_bits(tl->hwsp_offset) + offset_in_page(rq->hwsp_seqno);
350349 }
351350
352351 int gen8_emit_init_breadcrumb(struct i915_request *rq)
drivers/gpu/drm/i915/gt/intel_context_param.h+10−1
@@ -6,9 +6,18 @@ · 5 unmodified lines
66 #ifndef INTEL_CONTEXT_PARAM_H
77 #define INTEL_CONTEXT_PARAM_H
88
9−struct intel_context;
9+#include <linux/types.h>
10+
11+#include "intel_context.h"
1012
1113 int intel_context_set_ring_size(struct intel_context *ce, long sz);
1214 long intel_context_get_ring_size(struct intel_context *ce);
1315
16+static inline int
17+intel_context_set_watchdog_us(struct intel_context *ce, u64 timeout_us)
18+{
19+ ce->watchdog.timeout_us = timeout_us;
20+ return 0;
21+}
22+
1423 #endif /* INTEL_CONTEXT_PARAM_H */
drivers/gpu/drm/i915/gt/intel_context_types.h+4−0
@@ -97,6 +97,10 @@ struct intel_context { · 96 unmodified lines
9797 #define CONTEXT_FORCE_SINGLE_SUBMISSION 7
9898 #define CONTEXT_NOPREEMPT 8
9999
100+ struct {
101+ u64 timeout_us;
102+ } watchdog;
103+
100104 u32 *lrc_reg_state;
101105 union {
102106 struct {
drivers/gpu/drm/i915/gt/intel_engine_cs.c+25−13
@@ -619,6 +619,7 @@ static void cleanup_status_page(struct intel_engine_cs *engine) · 618 unmodified lines
619619 }
620620
621621 static int pin_ggtt_status_page(struct intel_engine_cs *engine,
622+ struct i915_gem_ww_ctx *ww,
622623 struct i915_vma *vma)
623624 {
624625 unsigned int flags;
@@ -639,12 +640,13 @@ static int pin_ggtt_status_page(struct intel_engine_cs *engine, · 14 unmodified lines
639640 else
640641 flags = PIN_HIGH;
641642
642− return i915_ggtt_pin(vma, NULL, 0, flags);
643+ return i915_ggtt_pin(vma, ww, 0, flags);
643644 }
644645
645646 static int init_status_page(struct intel_engine_cs *engine)
646647 {
647648 struct drm_i915_gem_object *obj;
649+ struct i915_gem_ww_ctx ww;
648650 struct i915_vma *vma;
649651 void *vaddr;
650652 int ret;
@@ -670,30 +672,39 @@ static int init_status_page(struct intel_engine_cs *engine) · 19 unmodified lines
670672 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
671673 if (IS_ERR(vma)) {
672674 ret = PTR_ERR(vma);
673− goto err;
675+ goto err_put;
674676 }
675677
678+ i915_gem_ww_ctx_init(&ww, true);
679+retry:
680+ ret = i915_gem_object_lock(obj, &ww);
681+ if (!ret && !HWS_NEEDS_PHYSICAL(engine->i915))
682+ ret = pin_ggtt_status_page(engine, &ww, vma);
683+ if (ret)
684+ goto err;
685+
676686 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
677687 if (IS_ERR(vaddr)) {
678688 ret = PTR_ERR(vaddr);
679− goto err;
689+ goto err_unpin;
680690 }
681691
682692 engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE);
683693 engine->status_page.vma = vma;
684694
685− if (!HWS_NEEDS_PHYSICAL(engine->i915)) {
686− ret = pin_ggtt_status_page(engine, vma);
687− if (ret)
688− goto err_unpin;
689− }
690−
691− return 0;
692−
693695 err_unpin:
694− i915_gem_object_unpin_map(obj);
696+ if (ret)
697+ i915_vma_unpin(vma);
695698 err:
696− i915_gem_object_put(obj);
699+ if (ret == -EDEADLK) {
700+ ret = i915_gem_ww_ctx_backoff(&ww);
701+ if (!ret)
702+ goto retry;
703+ }
704+ i915_gem_ww_ctx_fini(&ww);
705+err_put:
706+ if (ret)
707+ i915_gem_object_put(obj);
697708 return ret;
698709 }
699710
@@ -763,6 +774,7 @@ static int measure_breadcrumb_dw(struct intel_context *ce) · 63 unmodified lines
763774 frame->rq.engine = engine;
764775 frame->rq.context = ce;
765776 rcu_assign_pointer(frame->rq.timeline, ce->timeline);
777+ frame->rq.hwsp_seqno = ce->timeline->hwsp_seqno;
766778
767779 frame->ring.vaddr = frame->cs;
768780 frame->ring.size = sizeof(frame->cs);
drivers/gpu/drm/i915/gt/intel_engine_heartbeat.c+1−0
@@ -279,6 +279,7 @@ int intel_engine_pulse(struct intel_engine_cs *engine) · 278 unmodified lines
279279 mutex_unlock(&ce->timeline->mutex);
280280 }
281281
282+ intel_engine_flush_submission(engine);
282283 intel_engine_pm_put(engine);
283284 return err;
284285 }
drivers/gpu/drm/i915/gt/intel_engine_pm.c+4−0
@@ -27,12 +27,16 @@ static void dbg_poison_ce(struct intel_context *ce) · 26 unmodified lines
2727 int type = i915_coherent_map_type(ce->engine->i915);
2828 void *map;
2929
30+ if (!i915_gem_object_trylock(obj))
31+ return;
32+
3033 map = i915_gem_object_pin_map(obj, type);
3134 if (!IS_ERR(map)) {
3235 memset(map, CONTEXT_REDZONE, obj->base.size);
3336 i915_gem_object_flush_map(obj);
3437 i915_gem_object_unpin_map(obj);
3538 }
39+ i915_gem_object_unlock(obj);
3640 }
3741 }
3842
drivers/gpu/drm/i915/gt/intel_execlists_submission.c+39−10
@@ -470,6 +470,11 @@ static void reset_active(struct i915_request *rq, · 469 unmodified lines
470470 ce->lrc.lrca = lrc_update_regs(ce, engine, head);
471471 }
472472
473+static bool bad_request(const struct i915_request *rq)
474+{
475+ return rq->fence.error && i915_request_started(rq);
476+}
477+
473478 static struct intel_engine_cs *
474479 __execlists_schedule_in(struct i915_request *rq)
475480 {
@@ -482,7 +487,7 @@ __execlists_schedule_in(struct i915_request *rq) · 6 unmodified lines
482487 !intel_engine_has_heartbeat(engine)))
483488 intel_context_set_banned(ce);
484489
485− if (unlikely(intel_context_is_banned(ce)))
490+ if (unlikely(intel_context_is_banned(ce) || bad_request(rq)))
486491 reset_active(rq, engine);
487492
488493 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
@@ -752,9 +757,8 @@ assert_pending_valid(const struct intel_engine_execlists *execlists, · 263 unmodified lines
752757 {
753758 struct intel_engine_cs *engine =
754759 container_of(execlists, typeof(*engine), execlists);
755− struct i915_request * const *port, *rq;
760+ struct i915_request * const *port, *rq, *prev = NULL;
756761 struct intel_context *ce = NULL;
757− bool sentinel = false;
758762 u32 ccid = -1;
759763
760764 trace_ports(execlists, msg, execlists->pending);
@@ -804,15 +808,20 @@ assert_pending_valid(const struct intel_engine_execlists *execlists, · 43 unmodified lines
804808 * Sentinels are supposed to be the last request so they flush
805809 * the current execution off the HW. Check that they are the only
806810 * request in the pending submission.
811+ *
812+ * NB: Due to the async nature of preempt-to-busy and request
813+ * cancellation we need to handle the case where request
814+ * becomes a sentinel in parallel to CSB processing.
807815 */
808− if (sentinel) {
816+ if (prev && i915_request_has_sentinel(prev) &&
817+ !READ_ONCE(prev->fence.error)) {
809818 GEM_TRACE_ERR("%s: context:%llx after sentinel in pending[%zd]\n",
810819 engine->name,
811820 ce->timeline->fence_context,
812821 port - execlists->pending);
813822 return false;
814823 }
815− sentinel = i915_request_has_sentinel(rq);
824+ prev = rq;
816825
817826 /*
818827 * We want virtual requests to only be in the first slot so
@@ -948,7 +957,7 @@ static bool can_merge_rq(const struct i915_request *prev, · 129 unmodified lines
948957 if (__i915_request_is_complete(next))
949958 return true;
950959
951− if (unlikely((i915_request_flags(prev) ^ i915_request_flags(next)) &
960+ if (unlikely((i915_request_flags(prev) | i915_request_flags(next)) &
952961 (BIT(I915_FENCE_FLAG_NOPREEMPT) |
953962 BIT(I915_FENCE_FLAG_SENTINEL))))
954963 return false;
@@ -1208,7 +1217,7 @@ static unsigned long active_preempt_timeout(struct intel_engine_cs *engine, · 253 unmodified lines
12081217 return 0;
12091218
12101219 /* Force a fast reset for terminated contexts (ignoring sysfs!) */
1211− if (unlikely(intel_context_is_banned(rq->context)))
1220+ if (unlikely(intel_context_is_banned(rq->context) || bad_request(rq)))
12121221 return 1;
12131222
12141223 return READ_ONCE(engine->props.preempt_timeout_ms);
@@ -2457,11 +2466,31 @@ static void execlists_submit_request(struct i915_request *request) · 1242 unmodified lines
24572466 spin_unlock_irqrestore(&engine->active.lock, flags);
24582467 }
24592468
2469+static int
2470+__execlists_context_pre_pin(struct intel_context *ce,
2471+ struct intel_engine_cs *engine,
2472+ struct i915_gem_ww_ctx *ww, void **vaddr)
2473+{
2474+ int err;
2475+
2476+ err = lrc_pre_pin(ce, engine, ww, vaddr);
2477+ if (err)
2478+ return err;
2479+
2480+ if (!__test_and_set_bit(CONTEXT_INIT_BIT, &ce->flags)) {
2481+ lrc_init_state(ce, engine, *vaddr);
2482+
2483+ __i915_gem_object_flush_map(ce->state->obj, 0, engine->context_size);
2484+ }
2485+
2486+ return 0;
2487+}
2488+
24602489 static int execlists_context_pre_pin(struct intel_context *ce,
24612490 struct i915_gem_ww_ctx *ww,
24622491 void **vaddr)
24632492 {
2464− return lrc_pre_pin(ce, ce->engine, ww, vaddr);
2493+ return __execlists_context_pre_pin(ce, ce->engine, ww, vaddr);
24652494 }
24662495
24672496 static int execlists_context_pin(struct intel_context *ce, void *vaddr)
@@ -3365,8 +3394,8 @@ static int virtual_context_pre_pin(struct intel_context *ce, · 897 unmodified lines
33653394 {
33663395 struct virtual_engine *ve = container_of(ce, typeof(*ve), context);
33673396
3368− /* Note: we must use a real engine class for setting up reg state */
3369− return lrc_pre_pin(ce, ve->siblings[0], ww, vaddr);
3397+ /* Note: we must use a real engine class for setting up reg state */
3398+ return __execlists_context_pre_pin(ce, ve->siblings[0], ww, vaddr);
33703399 }
33713400
33723401 static int virtual_context_pin(struct intel_context *ce, void *vaddr)
drivers/gpu/drm/i915/gt/intel_execlists_submission.h+2−0
@@ -6,6 +6,7 @@ · 5 unmodified lines
66 #ifndef __INTEL_EXECLISTS_SUBMISSION_H__
77 #define __INTEL_EXECLISTS_SUBMISSION_H__
88
9+#include <linux/llist.h>
910 #include <linux/types.h>
1011
1112 struct drm_printer;
@@ -13,6 +14,7 @@ struct drm_printer; · 1 unmodified line
1314 struct i915_request;
1415 struct intel_context;
1516 struct intel_engine_cs;
17+struct intel_gt;
1618
1719 enum {
1820 INTEL_CONTEXT_SCHEDULE_IN = 0,
drivers/gpu/drm/i915/gt/intel_ggtt.c+8−2
@@ -647,7 +647,9 @@ static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) · 646 unmodified lines
647647 if (err)
648648 goto err_ppgtt;
649649
650+ i915_gem_object_lock(ppgtt->vm.scratch[0], NULL);
650651 err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash);
652+ i915_gem_object_unlock(ppgtt->vm.scratch[0]);
651653 if (err)
652654 goto err_stash;
653655
@@ -734,6 +736,7 @@ static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) · 80 unmodified lines
734736
735737 mutex_unlock(&ggtt->vm.mutex);
736738 i915_address_space_fini(&ggtt->vm);
739+ dma_resv_fini(&ggtt->vm.resv);
737740
738741 arch_phys_wc_del(ggtt->mtrr);
739742
@@ -1115,6 +1118,7 @@ static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt) · 375 unmodified lines
11151118 ggtt->vm.gt = gt;
11161119 ggtt->vm.i915 = i915;
11171120 ggtt->vm.dma = i915->drm.dev;
1121+ dma_resv_init(&ggtt->vm.resv);
11181122
11191123 if (INTEL_GEN(i915) <= 5)
11201124 ret = i915_gmch_probe(ggtt);
@@ -1122,8 +1126,10 @@ static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt) · 1 unmodified line
11221126 ret = gen6_gmch_probe(ggtt);
11231127 else
11241128 ret = gen8_gmch_probe(ggtt);
1125− if (ret)
1129+ if (ret) {
1130+ dma_resv_fini(&ggtt->vm.resv);
11261131 return ret;
1132+ }
11271133
11281134 if ((ggtt->vm.total - 1) >> 32) {
11291135 drm_err(&i915->drm,
@@ -1420,7 +1426,7 @@ intel_partial_pages(const struct i915_ggtt_view *view, · 290 unmodified lines
14201426 if (ret)
14211427 goto err_sg_alloc;
14221428
1423− iter = i915_gem_object_get_sg_dma(obj, view->partial.offset, &offset);
1429+ iter = i915_gem_object_get_sg_dma(obj, view->partial.offset, &offset, true);
14241430 GEM_BUG_ON(!iter);
14251431
14261432 sg = st->sgl;
drivers/gpu/drm/i915/gt/intel_gt.c+3−0
@@ -29,6 +29,9 @@ void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915) · 28 unmodified lines
2929 INIT_LIST_HEAD(&gt->closed_vma);
3030 spin_lock_init(&gt->closed_lock);
3131
32+ init_llist_head(&gt->watchdog.list);
33+ INIT_WORK(&gt->watchdog.work, intel_gt_watchdog_work);
34+
3235 intel_gt_init_buffer_pool(gt);
3336 intel_gt_init_reset(gt);
3437 intel_gt_init_requests(gt);
drivers/gpu/drm/i915/gt/intel_gt.h+2−0
@@ -77,4 +77,6 @@ static inline bool intel_gt_is_wedged(const struct intel_gt *gt) · 76 unmodified lines
7777 void intel_gt_info_print(const struct intel_gt_info *info,
7878 struct drm_printer *p);
7979
80+void intel_gt_watchdog_work(struct work_struct *work);
81+
8082 #endif /* __INTEL_GT_H__ */
drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.c+21−26
@@ -98,28 +98,6 @@ static void pool_free_work(struct work_struct *wrk) · 97 unmodified lines
9898 round_jiffies_up_relative(HZ));
9999 }
100100
101−static int pool_active(struct i915_active *ref)
102−{
103− struct intel_gt_buffer_pool_node *node =
104− container_of(ref, typeof(*node), active);
105− struct dma_resv *resv = node->obj->base.resv;
106− int err;
107−
108− if (dma_resv_trylock(resv)) {
109− dma_resv_add_excl_fence(resv, NULL);
110− dma_resv_unlock(resv);
111− }
112−
113− err = i915_gem_object_pin_pages(node->obj);
114− if (err)
115− return err;
116−
117− /* Hide this pinned object from the shrinker until retired */
118− i915_gem_object_make_unshrinkable(node->obj);
119−
120− return 0;
121−}
122−
123101 __i915_active_call
124102 static void pool_retire(struct i915_active *ref)
125103 {
@@ -129,10 +107,13 @@ static void pool_retire(struct i915_active *ref) · 3 unmodified lines
129107 struct list_head *list = bucket_for_size(pool, node->obj->base.size);
130108 unsigned long flags;
131109
132− i915_gem_object_unpin_pages(node->obj);
110+ if (node->pinned) {
111+ i915_gem_object_unpin_pages(node->obj);
133112
134− /* Return this object to the shrinker pool */
135− i915_gem_object_make_purgeable(node->obj);
113+ /* Return this object to the shrinker pool */
114+ i915_gem_object_make_purgeable(node->obj);
115+ node->pinned = false;
116+ }
136117
137118 GEM_BUG_ON(node->age);
138119 spin_lock_irqsave(&pool->lock, flags);
@@ -144,6 +125,19 @@ static void pool_retire(struct i915_active *ref) · 5 unmodified lines
144125 round_jiffies_up_relative(HZ));
145126 }
146127
128+void intel_gt_buffer_pool_mark_used(struct intel_gt_buffer_pool_node *node)
129+{
130+ assert_object_held(node->obj);
131+
132+ if (node->pinned)
133+ return;
134+
135+ __i915_gem_object_pin_pages(node->obj);
136+ /* Hide this pinned object from the shrinker until retired */
137+ i915_gem_object_make_unshrinkable(node->obj);
138+ node->pinned = true;
139+}
140+
147141 static struct intel_gt_buffer_pool_node *
148142 node_create(struct intel_gt_buffer_pool *pool, size_t sz,
149143 enum i915_map_type type)
@@ -159,7 +153,8 @@ node_create(struct intel_gt_buffer_pool *pool, size_t sz, · 9 unmodified lines
159153
160154 node->age = 0;
161155 node->pool = pool;
162− i915_active_init(&node->active, pool_active, pool_retire);
156+ node->pinned = false;
157+ i915_active_init(&node->active, NULL, pool_retire);
163158
164159 obj = i915_gem_object_create_internal(gt->i915, sz);
165160 if (IS_ERR(obj)) {
drivers/gpu/drm/i915/gt/intel_gt_buffer_pool.h+5−0
@@ -18,10 +18,15 @@ struct intel_gt_buffer_pool_node * · 17 unmodified lines
1818 intel_gt_get_buffer_pool(struct intel_gt *gt, size_t size,
1919 enum i915_map_type type);
2020
21+void intel_gt_buffer_pool_mark_used(struct intel_gt_buffer_pool_node *node);
22+
2123 static inline int
2224 intel_gt_buffer_pool_mark_active(struct intel_gt_buffer_pool_node *node,
2325 struct i915_request *rq)
2426 {
27+ /* did we call mark_used? */
28+ GEM_WARN_ON(!node->pinned);
29+
2530 return i915_active_add_request(&node->active, rq);
2631 }
2732
drivers/gpu/drm/i915/gt/intel_gt_buffer_pool_types.h+1−0
@@ -31,6 +31,7 @@ struct intel_gt_buffer_pool_node { · 30 unmodified lines
3131 };
3232 unsigned long age;
3333 enum i915_map_type type;
34+ u32 pinned;
3435 };
3536
3637 #endif /* INTEL_GT_BUFFER_POOL_TYPES_H */
drivers/gpu/drm/i915/gt/intel_gt_requests.c+28−0
@@ -9,6 +9,7 @@ · 8 unmodified lines
99 #include "i915_drv.h" /* for_each_engine() */
1010 #include "i915_request.h"
1111 #include "intel_engine_heartbeat.h"
12+#include "intel_execlists_submission.h"
1213 #include "intel_gt.h"
1314 #include "intel_gt_pm.h"
1415 #include "intel_gt_requests.h"
@@ -243,4 +244,31 @@ void intel_gt_fini_requests(struct intel_gt *gt) · 228 unmodified lines
243244 {
244245 /* Wait until the work is marked as finished before unloading! */
245246 cancel_delayed_work_sync(&gt->requests.retire_work);
247+
248+ flush_work(&gt->watchdog.work);
249+}
250+
251+void intel_gt_watchdog_work(struct work_struct *work)
252+{
253+ struct intel_gt *gt =
254+ container_of(work, typeof(*gt), watchdog.work);
255+ struct i915_request *rq, *rn;
256+ struct llist_node *first;
257+
258+ first = llist_del_all(&gt->watchdog.list);
259+ if (!first)
260+ return;
261+
262+ llist_for_each_entry_safe(rq, rn, first, watchdog.link) {
263+ if (!i915_request_completed(rq)) {
264+ struct dma_fence *f = &rq->fence;
265+
266+ pr_notice("Fence expiration time out i915-%s:%s:%llx!\n",
267+ f->ops->get_driver_name(f),
268+ f->ops->get_timeline_name(f),
269+ f->seqno);
270+ i915_request_cancel(rq, -EINTR);
271+ }
272+ i915_request_put(rq);
273+ }
246274 }
drivers/gpu/drm/i915/gt/intel_gt_types.h+7−4
@@ -8,10 +8,12 @@ · 7 unmodified lines
88
99 #include <linux/ktime.h>
1010 #include <linux/list.h>
11+#include <linux/llist.h>
1112 #include <linux/mutex.h>
1213 #include <linux/notifier.h>
1314 #include <linux/spinlock.h>
1415 #include <linux/types.h>
16+#include <linux/workqueue.h>
1517
1618 #include "uc/intel_uc.h"
1719
@@ -39,10 +41,6 @@ struct intel_gt { · 21 unmodified lines
3941 struct intel_gt_timelines {
4042 spinlock_t lock; /* protects active_list */
4143 struct list_head active_list;
42−
43− /* Pack multiple timelines' seqnos into the same page */
44− spinlock_t hwsp_lock;
45− struct list_head hwsp_free_list;
4644 } timelines;
4745
4846 struct intel_gt_requests {
@@ -56,6 +54,11 @@ struct intel_gt { · 7 unmodified lines
5654 struct delayed_work retire_work;
5755 } requests;
5856
57+ struct {
58+ struct llist_head list;
59+ struct work_struct work;
60+ } watchdog;
61+
5962 struct intel_wakeref wakeref;
6063 atomic_t user_wakeref;
6164
drivers/gpu/drm/i915/gt/intel_gtt.c+50−2
@@ -13,16 +13,36 @@ · 12 unmodified lines
1313
1414 struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz)
1515 {
16+ struct drm_i915_gem_object *obj;
17+
1618 if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
1719 i915_gem_shrink_all(vm->i915);
1820
19− return i915_gem_object_create_internal(vm->i915, sz);
21+ obj = i915_gem_object_create_internal(vm->i915, sz);
22+ /* ensure all dma objects have the same reservation class */
23+ if (!IS_ERR(obj))
24+ obj->base.resv = &vm->resv;
25+ return obj;
2026 }
2127
2228 int pin_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj)
2329 {
2430 int err;
2531
32+ i915_gem_object_lock(obj, NULL);
33+ err = i915_gem_object_pin_pages(obj);
34+ i915_gem_object_unlock(obj);
35+ if (err)
36+ return err;
37+
38+ i915_gem_object_make_unshrinkable(obj);
39+ return 0;
40+}
41+
42+int pin_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj)
43+{
44+ int err;
45+
2646 err = i915_gem_object_pin_pages(obj);
2747 if (err)
2848 return err;
@@ -56,6 +76,20 @@ void __i915_vm_close(struct i915_address_space *vm) · 27 unmodified lines
5676 mutex_unlock(&vm->mutex);
5777 }
5878
79+/* lock the vm into the current ww, if we lock one, we lock all */
80+int i915_vm_lock_objects(struct i915_address_space *vm,
81+ struct i915_gem_ww_ctx *ww)
82+{
83+ if (vm->scratch[0]->base.resv == &vm->resv) {
84+ return i915_gem_object_lock(vm->scratch[0], ww);
85+ } else {
86+ struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
87+
88+ /* We borrowed the scratch page from ggtt, take the top level object */
89+ return i915_gem_object_lock(ppgtt->pd->pt.base, ww);
90+ }
91+}
92+
5993 void i915_address_space_fini(struct i915_address_space *vm)
6094 {
6195 drm_mm_takedown(&vm->mm);
@@ -69,6 +103,7 @@ static void __i915_vm_release(struct work_struct *work) · 7 unmodified lines
69103
70104 vm->cleanup(vm);
71105 i915_address_space_fini(vm);
106+ dma_resv_fini(&vm->resv);
72107
73108 kfree(vm);
74109 }
@@ -98,6 +133,7 @@ void i915_address_space_init(struct i915_address_space *vm, int subclass) · 23 unmodified lines
98133 mutex_init(&vm->mutex);
99134 lockdep_set_subclass(&vm->mutex, subclass);
100135 i915_gem_shrinker_taints_mutex(vm->i915, &vm->mutex);
136+ dma_resv_init(&vm->resv);
101137
102138 GEM_BUG_ON(!vm->total);
103139 drm_mm_init(&vm->mm, 0, vm->total);
@@ -427,7 +463,6 @@ __vm_create_scratch_for_read(struct i915_address_space *vm, unsigned long size) · 323 unmodified lines
427463 {
428464 struct drm_i915_gem_object *obj;
429465 struct i915_vma *vma;
430− int err;
431466
432467 obj = i915_gem_object_create_internal(vm->i915, PAGE_ALIGN(size));
433468 if (IS_ERR(obj))
@@ -441,6 +476,19 @@ __vm_create_scratch_for_read(struct i915_address_space *vm, unsigned long size) · 7 unmodified lines
441476 return vma;
442477 }
443478
479+ return vma;
480+}
481+
482+struct i915_vma *
483+__vm_create_scratch_for_read_pinned(struct i915_address_space *vm, unsigned long size)
484+{
485+ struct i915_vma *vma;
486+ int err;
487+
488+ vma = __vm_create_scratch_for_read(vm, size);
489+ if (IS_ERR(vma))
490+ return vma;
491+
444492 err = i915_vma_pin(vma, 0, 0,
445493 i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
446494 if (err) {
drivers/gpu/drm/i915/gt/intel_gtt.h+8−0
@@ -238,6 +238,7 @@ struct i915_address_space { · 237 unmodified lines
238238 atomic_t open;
239239
240240 struct mutex mutex; /* protects vma and our lists */
241+ struct dma_resv resv; /* reservation lock for all pd objects, and buffer pool */
241242 #define VM_CLASS_GGTT 0
242243 #define VM_CLASS_PPGTT 1
243244
@@ -346,6 +347,9 @@ struct i915_ppgtt { · 102 unmodified lines
346347
347348 #define i915_is_ggtt(vm) ((vm)->is_ggtt)
348349
350+int __must_check
351+i915_vm_lock_objects(struct i915_address_space *vm, struct i915_gem_ww_ctx *ww);
352+
349353 static inline bool
350354 i915_vm_is_4lvl(const struct i915_address_space *vm)
351355 {
@@ -522,6 +526,7 @@ struct i915_page_directory *alloc_pd(struct i915_address_space *vm); · 170 unmodified lines
522526 struct i915_page_directory *__alloc_pd(int npde);
523527
524528 int pin_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj);
529+int pin_pt_dma_locked(struct i915_address_space *vm, struct drm_i915_gem_object *obj);
525530
526531 void free_px(struct i915_address_space *vm,
527532 struct i915_page_table *pt, int lvl);
@@ -576,6 +581,9 @@ void i915_vm_free_pt_stash(struct i915_address_space *vm, · 48 unmodified lines
576581 struct i915_vma *
577582 __vm_create_scratch_for_read(struct i915_address_space *vm, unsigned long size);
578583
584+struct i915_vma *
585+__vm_create_scratch_for_read_pinned(struct i915_address_space *vm, unsigned long size);
586+
579587 static inline struct sgt_dma {
580588 struct scatterlist *sg;
581589 dma_addr_t dma, max;
drivers/gpu/drm/i915/gt/intel_lrc.c+38−11
@@ -1417,7 +1417,7 @@ gen10_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) · 1416 unmodified lines
14171417
14181418 #define CTX_WA_BB_SIZE (PAGE_SIZE)
14191419
1420−static int lrc_setup_wa_ctx(struct intel_engine_cs *engine)
1420+static int lrc_create_wa_ctx(struct intel_engine_cs *engine)
14211421 {
14221422 struct drm_i915_gem_object *obj;
14231423 struct i915_vma *vma;
@@ -1433,10 +1433,6 @@ static int lrc_setup_wa_ctx(struct intel_engine_cs *engine) · 9 unmodified lines
14331433 goto err;
14341434 }
14351435
1436− err = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH);
1437− if (err)
1438− goto err;
1439−
14401436 engine->wa_ctx.vma = vma;
14411437 return 0;
14421438
@@ -1448,9 +1444,6 @@ err: · 5 unmodified lines
14481444 void lrc_fini_wa_ctx(struct intel_engine_cs *engine)
14491445 {
14501446 i915_vma_unpin_and_release(&engine->wa_ctx.vma, 0);
1451−
1452− /* Called on error unwind, clear all flags to prevent further use */
1453− memset(&engine->wa_ctx, 0, sizeof(engine->wa_ctx));
14541447 }
14551448
14561449 typedef u32 *(*wa_bb_func_t)(struct intel_engine_cs *engine, u32 *batch);
@@ -1462,6 +1455,7 @@ void lrc_init_wa_ctx(struct intel_engine_cs *engine) · 5 unmodified lines
14621455 &wa_ctx->indirect_ctx, &wa_ctx->per_ctx
14631456 };
14641457 wa_bb_func_t wa_bb_fn[ARRAY_SIZE(wa_bb)];
1458+ struct i915_gem_ww_ctx ww;
14651459 void *batch, *batch_ptr;
14661460 unsigned int i;
14671461 int err;
@@ -1490,7 +1484,7 @@ void lrc_init_wa_ctx(struct intel_engine_cs *engine) · 22 unmodified lines
14901484 return;
14911485 }
14921486
1493− err = lrc_setup_wa_ctx(engine);
1487+ err = lrc_create_wa_ctx(engine);
14941488 if (err) {
14951489 /*
14961490 * We continue even if we fail to initialize WA batch
@@ -1503,7 +1497,22 @@ void lrc_init_wa_ctx(struct intel_engine_cs *engine) · 6 unmodified lines
15031497 return;
15041498 }
15051499
1500+ if (!engine->wa_ctx.vma)
1501+ return;
1502+
1503+ i915_gem_ww_ctx_init(&ww, true);
1504+retry:
1505+ err = i915_gem_object_lock(wa_ctx->vma->obj, &ww);
1506+ if (!err)
1507+ err = i915_ggtt_pin(wa_ctx->vma, &ww, 0, PIN_HIGH);
1508+ if (err)
1509+ goto err;
1510+
15061511 batch = i915_gem_object_pin_map(wa_ctx->vma->obj, I915_MAP_WB);
1512+ if (IS_ERR(batch)) {
1513+ err = PTR_ERR(batch);
1514+ goto err_unpin;
1515+ }
15071516
15081517 /*
15091518 * Emit the two workaround batch buffers, recording the offset from the
@@ -1528,8 +1537,26 @@ void lrc_init_wa_ctx(struct intel_engine_cs *engine) · 18 unmodified lines
15281537 __i915_gem_object_release_map(wa_ctx->vma->obj);
15291538
15301539 /* Verify that we can handle failure to setup the wa_ctx */
1531− if (err || i915_inject_probe_error(engine->i915, -ENODEV))
1532− lrc_fini_wa_ctx(engine);
1540+ if (!err)
1541+ err = i915_inject_probe_error(engine->i915, -ENODEV);
1542+
1543+err_unpin:
1544+ if (err)
1545+ i915_vma_unpin(wa_ctx->vma);
1546+err:
1547+ if (err == -EDEADLK) {
1548+ err = i915_gem_ww_ctx_backoff(&ww);
1549+ if (!err)
1550+ goto retry;
1551+ }
1552+ i915_gem_ww_ctx_fini(&ww);
1553+
1554+ if (err) {
1555+ i915_vma_put(engine->wa_ctx.vma);
1556+
1557+ /* Clear all flags to prevent further use */
1558+ memset(wa_ctx, 0, sizeof(*wa_ctx));
1559+ }
15331560 }
15341561
15351562 static void st_update_runtime_underflow(struct intel_context *ce, s32 dt)
drivers/gpu/drm/i915/gt/intel_ppgtt.c+2−1
@@ -262,7 +262,7 @@ int i915_vm_pin_pt_stash(struct i915_address_space *vm, · 261 unmodified lines
262262
263263 for (n = 0; n < ARRAY_SIZE(stash->pt); n++) {
264264 for (pt = stash->pt[n]; pt; pt = pt->stash) {
265− err = pin_pt_dma(vm, pt->base);
265+ err = pin_pt_dma_locked(vm, pt->base);
266266 if (err)
267267 return err;
268268 }
@@ -304,6 +304,7 @@ void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt) · 35 unmodified lines
304304 ppgtt->vm.dma = i915->drm.dev;
305305 ppgtt->vm.total = BIT_ULL(INTEL_INFO(i915)->ppgtt_size);
306306
307+ dma_resv_init(&ppgtt->vm.resv);
307308 i915_address_space_init(&ppgtt->vm, VM_CLASS_PPGTT);
308309
309310 ppgtt->vm.vma_ops.bind_vma = ppgtt_bind_vma;
drivers/gpu/drm/i915/gt/intel_renderstate.c+1−1
@@ -197,7 +197,7 @@ retry: · 196 unmodified lines
197197 if (err)
198198 goto err_context;
199199
200− err = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
200+ err = i915_vma_pin_ww(so->vma, &so->ww, 0, 0, PIN_GLOBAL | PIN_HIGH);
201201 if (err)
202202 goto err_context;
203203
drivers/gpu/drm/i915/gt/intel_reset.c+7−2
@@ -974,8 +974,6 @@ static int do_reset(struct intel_gt *gt, intel_engine_mask_t stalled_mask) · 973 unmodified lines
974974 {
975975 int err, i;
976976
977− gt_revoke(gt);
978−
979977 err = __intel_gt_reset(gt, ALL_ENGINES);
980978 for (i = 0; err && i < RESET_MAX_RETRIES; i++) {
981979 msleep(10 * (i + 1));
@@ -1030,6 +1028,13 @@ void intel_gt_reset(struct intel_gt *gt, · 48 unmodified lines
10301028
10311029 might_sleep();
10321030 GEM_BUG_ON(!test_bit(I915_RESET_BACKOFF, &gt->reset.flags));
1031+
1032+ /*
1033+ * FIXME: Revoking cpu mmap ptes cannot be done from a dma_fence
1034+ * critical section like gpu reset.
1035+ */
1036+ gt_revoke(gt);
1037+
10331038 mutex_lock(&gt->reset.mutex);
10341039
10351040 /* Clear any previous failed attempts at recovery. Time to try again. */
drivers/gpu/drm/i915/gt/intel_ring_submission.c+118−66
@@ -466,6 +466,26 @@ static void ring_context_destroy(struct kref *ref) · 465 unmodified lines
466466 intel_context_free(ce);
467467 }
468468
469+static int ring_context_init_default_state(struct intel_context *ce,
470+ struct i915_gem_ww_ctx *ww)
471+{
472+ struct drm_i915_gem_object *obj = ce->state->obj;
473+ void *vaddr;
474+
475+ vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
476+ if (IS_ERR(vaddr))
477+ return PTR_ERR(vaddr);
478+
479+ shmem_read(ce->engine->default_state, 0,
480+ vaddr, ce->engine->context_size);
481+
482+ i915_gem_object_flush_map(obj);
483+ __i915_gem_object_release_map(obj);
484+
485+ __set_bit(CONTEXT_VALID_BIT, &ce->flags);
486+ return 0;
487+}
488+
469489 static int ring_context_pre_pin(struct intel_context *ce,
470490 struct i915_gem_ww_ctx *ww,
471491 void **unused)
@@ -473,6 +493,13 @@ static int ring_context_pre_pin(struct intel_context *ce, · 1 unmodified line
473493 struct i915_address_space *vm;
474494 int err = 0;
475495
496+ if (ce->engine->default_state &&
497+ !test_bit(CONTEXT_VALID_BIT, &ce->flags)) {
498+ err = ring_context_init_default_state(ce, ww);
499+ if (err)
500+ return err;
501+ }
502+
476503 vm = vm_alias(ce->vm);
477504 if (vm)
478505 err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm)), ww);
@@ -528,22 +555,6 @@ alloc_context_vma(struct intel_engine_cs *engine) · 49 unmodified lines
528555 if (IS_IVYBRIDGE(i915))
529556 i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC);
530557
531− if (engine->default_state) {
532− void *vaddr;
533−
534− vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
535− if (IS_ERR(vaddr)) {
536− err = PTR_ERR(vaddr);
537− goto err_obj;
538− }
539−
540− shmem_read(engine->default_state, 0,
541− vaddr, engine->context_size);
542−
543− i915_gem_object_flush_map(obj);
544− __i915_gem_object_release_map(obj);
545− }
546−
547558 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
548559 if (IS_ERR(vma)) {
549560 err = PTR_ERR(vma);
@@ -575,8 +586,6 @@ static int ring_context_alloc(struct intel_context *ce) · 25 unmodified lines
575586 return PTR_ERR(vma);
576587
577588 ce->state = vma;
578− if (engine->default_state)
579− __set_bit(CONTEXT_VALID_BIT, &ce->flags);
580589 }
581590
582591 return 0;
@@ -1176,37 +1185,15 @@ static int gen7_ctx_switch_bb_setup(struct intel_engine_cs * const engine, · 593 unmodified lines
11761185 return gen7_setup_clear_gpr_bb(engine, vma);
11771186 }
11781187
1179−static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine)
1188+static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine,
1189+ struct i915_gem_ww_ctx *ww,
1190+ struct i915_vma *vma)
11801191 {
1181− struct drm_i915_gem_object *obj;
1182− struct i915_vma *vma;
1183− int size;
11841192 int err;
11851193
1186− size = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */);
1187− if (size <= 0)
1188− return size;
1189−
1190− size = ALIGN(size, PAGE_SIZE);
1191− obj = i915_gem_object_create_internal(engine->i915, size);
1192− if (IS_ERR(obj))
1193− return PTR_ERR(obj);
1194−
1195− vma = i915_vma_instance(obj, engine->gt->vm, NULL);
1196− if (IS_ERR(vma)) {
1197− err = PTR_ERR(vma);
1198− goto err_obj;
1199− }
1200−
1201− vma->private = intel_context_create(engine); /* dummy residuals */
1202− if (IS_ERR(vma->private)) {
1203− err = PTR_ERR(vma->private);
1204− goto err_obj;
1205− }
1206−
1207− err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_HIGH);
1194+ err = i915_vma_pin_ww(vma, ww, 0, 0, PIN_USER | PIN_HIGH);
12081195 if (err)
1209− goto err_private;
1196+ return err;
12101197
12111198 err = i915_vma_sync(vma);
12121199 if (err)
@@ -1221,17 +1208,53 @@ static int gen7_ctx_switch_bb_init(struct intel_engine_cs *engine) · 8 unmodified lines
12211208
12221209 err_unpin:
12231210 i915_vma_unpin(vma);
1224−err_private:
1225− intel_context_put(vma->private);
1226−err_obj:
1227− i915_gem_object_put(obj);
12281211 return err;
12291212 }
12301213
1214+static struct i915_vma *gen7_ctx_vma(struct intel_engine_cs *engine)
1215+{
1216+ struct drm_i915_gem_object *obj;
1217+ struct i915_vma *vma;
1218+ int size, err;
1219+
1220+ if (!IS_GEN(engine->i915, 7) || engine->class != RENDER_CLASS)
1221+ return 0;
1222+
1223+ err = gen7_ctx_switch_bb_setup(engine, NULL /* probe size */);
1224+ if (err < 0)
1225+ return ERR_PTR(err);
1226+ if (!err)
1227+ return NULL;
1228+
1229+ size = ALIGN(err, PAGE_SIZE);
1230+
1231+ obj = i915_gem_object_create_internal(engine->i915, size);
1232+ if (IS_ERR(obj))
1233+ return ERR_CAST(obj);
1234+
1235+ vma = i915_vma_instance(obj, engine->gt->vm, NULL);
1236+ if (IS_ERR(vma)) {
1237+ i915_gem_object_put(obj);
1238+ return ERR_CAST(vma);
1239+ }
1240+
1241+ vma->private = intel_context_create(engine); /* dummy residuals */
1242+ if (IS_ERR(vma->private)) {
1243+ err = PTR_ERR(vma->private);
1244+ vma->private = NULL;
1245+ i915_gem_object_put(obj);
1246+ return ERR_PTR(err);
1247+ }
1248+
1249+ return vma;
1250+}
1251+
12311252 int intel_ring_submission_setup(struct intel_engine_cs *engine)
12321253 {
1254+ struct i915_gem_ww_ctx ww;
12331255 struct intel_timeline *timeline;
12341256 struct intel_ring *ring;
1257+ struct i915_vma *gen7_wa_vma;
12351258 int err;
12361259
12371260 setup_common(engine);
@@ -1262,43 +1285,72 @@ int intel_ring_submission_setup(struct intel_engine_cs *engine) · 24 unmodified lines
12621285 }
12631286 GEM_BUG_ON(timeline->has_initial_breadcrumb);
12641287
1265− err = intel_timeline_pin(timeline, NULL);
1266− if (err)
1267− goto err_timeline;
1268−
12691288 ring = intel_engine_create_ring(engine, SZ_16K);
12701289 if (IS_ERR(ring)) {
12711290 err = PTR_ERR(ring);
1272− goto err_timeline_unpin;
1291+ goto err_timeline;
12731292 }
12741293
1275− err = intel_ring_pin(ring, NULL);
1276− if (err)
1277− goto err_ring;
1278−
12791294 GEM_BUG_ON(engine->legacy.ring);
12801295 engine->legacy.ring = ring;
12811296 engine->legacy.timeline = timeline;
12821297
1283− GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma);
1298+ gen7_wa_vma = gen7_ctx_vma(engine);
1299+ if (IS_ERR(gen7_wa_vma)) {
1300+ err = PTR_ERR(gen7_wa_vma);
1301+ goto err_ring;
1302+ }
12841303
1285− if (IS_GEN(engine->i915, 7) && engine->class == RENDER_CLASS) {
1286− err = gen7_ctx_switch_bb_init(engine);
1304+ i915_gem_ww_ctx_init(&ww, false);
1305+
1306+retry:
1307+ err = i915_gem_object_lock(timeline->hwsp_ggtt->obj, &ww);
1308+ if (!err && gen7_wa_vma)
1309+ err = i915_gem_object_lock(gen7_wa_vma->obj, &ww);
1310+ if (!err && engine->legacy.ring->vma->obj)
1311+ err = i915_gem_object_lock(engine->legacy.ring->vma->obj, &ww);
1312+ if (!err)
1313+ err = intel_timeline_pin(timeline, &ww);
1314+ if (!err) {
1315+ err = intel_ring_pin(ring, &ww);
12871316 if (err)
1288− goto err_ring_unpin;
1317+ intel_timeline_unpin(timeline);
1318+ }
1319+ if (err)
1320+ goto out;
1321+
1322+ GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma);
1323+
1324+ if (gen7_wa_vma) {
1325+ err = gen7_ctx_switch_bb_init(engine, &ww, gen7_wa_vma);
1326+ if (err) {
1327+ intel_ring_unpin(ring);
1328+ intel_timeline_unpin(timeline);
1329+ }
12891330 }
12901331
1332+out:
1333+ if (err == -EDEADLK) {
1334+ err = i915_gem_ww_ctx_backoff(&ww);
1335+ if (!err)
1336+ goto retry;
1337+ }
1338+ i915_gem_ww_ctx_fini(&ww);
1339+ if (err)
1340+ goto err_gen7_put;
1341+
12911342 /* Finally, take ownership and responsibility for cleanup! */
12921343 engine->release = ring_release;
12931344
12941345 return 0;
12951346
1296−err_ring_unpin:
1297− intel_ring_unpin(ring);
1347+err_gen7_put:
1348+ if (gen7_wa_vma) {
1349+ intel_context_put(gen7_wa_vma->private);
1350+ i915_gem_object_put(gen7_wa_vma->obj);
1351+ }
12981352 err_ring:
12991353 intel_ring_put(ring);
1300−err_timeline_unpin:
1301− intel_timeline_unpin(timeline);
13021354 err_timeline:
13031355 intel_timeline_put(timeline);
13041356 err:
drivers/gpu/drm/i915/gt/intel_timeline.c+94−332
@@ -12,21 +12,9 @@ · 11 unmodified lines
1212 #include "intel_ring.h"
1313 #include "intel_timeline.h"
1414
15−#define ptr_set_bit(ptr, bit) ((typeof(ptr))((unsigned long)(ptr) | BIT(bit)))
16−#define ptr_test_bit(ptr, bit) ((unsigned long)(ptr) & BIT(bit))
15+#define TIMELINE_SEQNO_BYTES 8
1716
18−#define CACHELINE_BITS 6
19−#define CACHELINE_FREE CACHELINE_BITS
20−
21−struct intel_timeline_hwsp {
22− struct intel_gt *gt;
23− struct intel_gt_timelines *gt_timelines;
24− struct list_head free_link;
25− struct i915_vma *vma;
26− u64 free_bitmap;
27−};
28−
29−static struct i915_vma *__hwsp_alloc(struct intel_gt *gt)
17+static struct i915_vma *hwsp_alloc(struct intel_gt *gt)
3018 {
3119 struct drm_i915_private *i915 = gt->i915;
3220 struct drm_i915_gem_object *obj;
@@ -45,174 +33,42 @@ static struct i915_vma *__hwsp_alloc(struct intel_gt *gt) · 12 unmodified lines
4533 return vma;
4634 }
4735
48−static struct i915_vma *
49−hwsp_alloc(struct intel_timeline *timeline, unsigned int *cacheline)
50−{
51− struct intel_gt_timelines *gt = &timeline->gt->timelines;
52− struct intel_timeline_hwsp *hwsp;
53−
54− BUILD_BUG_ON(BITS_PER_TYPE(u64) * CACHELINE_BYTES > PAGE_SIZE);
55−
56− spin_lock_irq(&gt->hwsp_lock);
57−
58− /* hwsp_free_list only contains HWSP that have available cachelines */
59− hwsp = list_first_entry_or_null(&gt->hwsp_free_list,
60− typeof(*hwsp), free_link);
61− if (!hwsp) {
62− struct i915_vma *vma;
63−
64− spin_unlock_irq(&gt->hwsp_lock);
65−
66− hwsp = kmalloc(sizeof(*hwsp), GFP_KERNEL);
67− if (!hwsp)
68− return ERR_PTR(-ENOMEM);
69−
70− vma = __hwsp_alloc(timeline->gt);
71− if (IS_ERR(vma)) {
72− kfree(hwsp);
73− return vma;
74− }
75−
76− GT_TRACE(timeline->gt, "new HWSP allocated\n");
77−
78− vma->private = hwsp;
79− hwsp->gt = timeline->gt;
80− hwsp->vma = vma;
81− hwsp->free_bitmap = ~0ull;
82− hwsp->gt_timelines = gt;
83−
84− spin_lock_irq(&gt->hwsp_lock);
85− list_add(&hwsp->free_link, &gt->hwsp_free_list);
86− }
87−
88− GEM_BUG_ON(!hwsp->free_bitmap);
89− *cacheline = __ffs64(hwsp->free_bitmap);
90− hwsp->free_bitmap &= ~BIT_ULL(*cacheline);
91− if (!hwsp->free_bitmap)
92− list_del(&hwsp->free_link);
93−
94− spin_unlock_irq(&gt->hwsp_lock);
95−
96− GEM_BUG_ON(hwsp->vma->private != hwsp);
97− return hwsp->vma;
98−}
99−
100−static void __idle_hwsp_free(struct intel_timeline_hwsp *hwsp, int cacheline)
101−{
102− struct intel_gt_timelines *gt = hwsp->gt_timelines;
103− unsigned long flags;
104−
105− spin_lock_irqsave(&gt->hwsp_lock, flags);
106−
107− /* As a cacheline becomes available, publish the HWSP on the freelist */
108− if (!hwsp->free_bitmap)
109− list_add_tail(&hwsp->free_link, &gt->hwsp_free_list);
110−
111− GEM_BUG_ON(cacheline >= BITS_PER_TYPE(hwsp->free_bitmap));
112− hwsp->free_bitmap |= BIT_ULL(cacheline);
113−
114− /* And if no one is left using it, give the page back to the system */
115− if (hwsp->free_bitmap == ~0ull) {
116− i915_vma_put(hwsp->vma);
117− list_del(&hwsp->free_link);
118− kfree(hwsp);
119− }
120−
121− spin_unlock_irqrestore(&gt->hwsp_lock, flags);
122−}
123−
124−static void __rcu_cacheline_free(struct rcu_head *rcu)
125−{
126− struct intel_timeline_cacheline *cl =
127− container_of(rcu, typeof(*cl), rcu);
128−
129− /* Must wait until after all *rq->hwsp are complete before removing */
130− i915_gem_object_unpin_map(cl->hwsp->vma->obj);
131− __idle_hwsp_free(cl->hwsp, ptr_unmask_bits(cl->vaddr, CACHELINE_BITS));
132−
133− i915_active_fini(&cl->active);
134− kfree(cl);
135−}
136−
137−static void __idle_cacheline_free(struct intel_timeline_cacheline *cl)
138−{
139− GEM_BUG_ON(!i915_active_is_idle(&cl->active));
140− call_rcu(&cl->rcu, __rcu_cacheline_free);
141−}
142−
14336 __i915_active_call
144−static void __cacheline_retire(struct i915_active *active)
37+static void __timeline_retire(struct i915_active *active)
14538 {
146− struct intel_timeline_cacheline *cl =
147− container_of(active, typeof(*cl), active);
39+ struct intel_timeline *tl =
40+ container_of(active, typeof(*tl), active);
14841
149− i915_vma_unpin(cl->hwsp->vma);
150− if (ptr_test_bit(cl->vaddr, CACHELINE_FREE))
151− __idle_cacheline_free(cl);
42+ i915_vma_unpin(tl->hwsp_ggtt);
43+ intel_timeline_put(tl);
15244 }
15345
154−static int __cacheline_active(struct i915_active *active)
46+static int __timeline_active(struct i915_active *active)
15547 {
156− struct intel_timeline_cacheline *cl =
157− container_of(active, typeof(*cl), active);
48+ struct intel_timeline *tl =
49+ container_of(active, typeof(*tl), active);
15850
159− __i915_vma_pin(cl->hwsp->vma);
51+ __i915_vma_pin(tl->hwsp_ggtt);
52+ intel_timeline_get(tl);
16053 return 0;
16154 }
16255
163−static struct intel_timeline_cacheline *
164−cacheline_alloc(struct intel_timeline_hwsp *hwsp, unsigned int cacheline)
56+I915_SELFTEST_EXPORT int
57+intel_timeline_pin_map(struct intel_timeline *timeline)
16558 {
166− struct intel_timeline_cacheline *cl;
59+ struct drm_i915_gem_object *obj = timeline->hwsp_ggtt->obj;
60+ u32 ofs = offset_in_page(timeline->hwsp_offset);
16761 void *vaddr;
16862
169− GEM_BUG_ON(cacheline >= BIT(CACHELINE_BITS));
170−
171− cl = kmalloc(sizeof(*cl), GFP_KERNEL);
172− if (!cl)
173− return ERR_PTR(-ENOMEM);
174−
175− vaddr = i915_gem_object_pin_map(hwsp->vma->obj, I915_MAP_WB);
176− if (IS_ERR(vaddr)) {
177− kfree(cl);
178− return ERR_CAST(vaddr);
179− }
180−
181− cl->hwsp = hwsp;
182− cl->vaddr = page_pack_bits(vaddr, cacheline);
183−
184− i915_active_init(&cl->active, __cacheline_active, __cacheline_retire);
185−
186− return cl;
187−}
188−
189−static void cacheline_acquire(struct intel_timeline_cacheline *cl,
190− u32 ggtt_offset)
191−{
192− if (!cl)
193− return;
194−
195− cl->ggtt_offset = ggtt_offset;
196− i915_active_acquire(&cl->active);
197−}
198−
199−static void cacheline_release(struct intel_timeline_cacheline *cl)
200−{
201− if (cl)
202− i915_active_release(&cl->active);
203−}
204−
205−static void cacheline_free(struct intel_timeline_cacheline *cl)
206−{
207− if (!i915_active_acquire_if_busy(&cl->active)) {
208− __idle_cacheline_free(cl);
209− return;
210− }
63+ vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
64+ if (IS_ERR(vaddr))
65+ return PTR_ERR(vaddr);
21166
212− GEM_BUG_ON(ptr_test_bit(cl->vaddr, CACHELINE_FREE));
213− cl->vaddr = ptr_set_bit(cl->vaddr, CACHELINE_FREE);
67+ timeline->hwsp_map = vaddr;
68+ timeline->hwsp_seqno = memset(vaddr + ofs, 0, TIMELINE_SEQNO_BYTES);
69+ clflush(vaddr + ofs);
21470
215− i915_active_release(&cl->active);
71+ return 0;
21672 }
21773
21874 static int intel_timeline_init(struct intel_timeline *timeline,
@@ -220,45 +76,25 @@ static int intel_timeline_init(struct intel_timeline *timeline, · 1 unmodified line
22076 struct i915_vma *hwsp,
22177 unsigned int offset)
22278 {
223− void *vaddr;
224−
22579 kref_init(&timeline->kref);
22680 atomic_set(&timeline->pin_count, 0);
22781
22882 timeline->gt = gt;
22983
230− timeline->has_initial_breadcrumb = !hwsp;
231− timeline->hwsp_cacheline = NULL;
232−
233− if (!hwsp) {
234− struct intel_timeline_cacheline *cl;
235− unsigned int cacheline;
236−
237− hwsp = hwsp_alloc(timeline, &cacheline);
84+ if (hwsp) {
85+ timeline->hwsp_offset = offset;
86+ timeline->hwsp_ggtt = i915_vma_get(hwsp);
87+ } else {
88+ timeline->has_initial_breadcrumb = true;
89+ hwsp = hwsp_alloc(gt);
23890 if (IS_ERR(hwsp))
23991 return PTR_ERR(hwsp);
240−
241− cl = cacheline_alloc(hwsp->private, cacheline);
242− if (IS_ERR(cl)) {
243− __idle_hwsp_free(hwsp->private, cacheline);
244− return PTR_ERR(cl);
245− }
246−
247− timeline->hwsp_cacheline = cl;
248− timeline->hwsp_offset = cacheline * CACHELINE_BYTES;
249−
250− vaddr = page_mask_bits(cl->vaddr);
251− } else {
252− timeline->hwsp_offset = offset;
253− vaddr = i915_gem_object_pin_map(hwsp->obj, I915_MAP_WB);
254− if (IS_ERR(vaddr))
255− return PTR_ERR(vaddr);
92+ timeline->hwsp_ggtt = hwsp;
25693 }
25794
258− timeline->hwsp_seqno =
259− memset(vaddr + timeline->hwsp_offset, 0, CACHELINE_BYTES);
95+ timeline->hwsp_map = NULL;
96+ timeline->hwsp_seqno = (void *)(long)timeline->hwsp_offset;
26097
261− timeline->hwsp_ggtt = i915_vma_get(hwsp);
26298 GEM_BUG_ON(timeline->hwsp_offset >= hwsp->size);
26399
264100 timeline->fence_context = dma_fence_context_alloc(1);
@@ -269,6 +105,7 @@ static int intel_timeline_init(struct intel_timeline *timeline, · 4 unmodified lines
269105 INIT_LIST_HEAD(&timeline->requests);
270106
271107 i915_syncmap_init(&timeline->sync);
108+ i915_active_init(&timeline->active, __timeline_active, __timeline_retire);
272109
273110 return 0;
274111 }
@@ -279,23 +116,19 @@ void intel_gt_init_timelines(struct intel_gt *gt) · 4 unmodified lines
279116
280117 spin_lock_init(&timelines->lock);
281118 INIT_LIST_HEAD(&timelines->active_list);
282−
283− spin_lock_init(&timelines->hwsp_lock);
284− INIT_LIST_HEAD(&timelines->hwsp_free_list);
285119 }
286120
287−static void intel_timeline_fini(struct intel_timeline *timeline)
121+static void intel_timeline_fini(struct rcu_head *rcu)
288122 {
289− GEM_BUG_ON(atomic_read(&timeline->pin_count));
290− GEM_BUG_ON(!list_empty(&timeline->requests));
291− GEM_BUG_ON(timeline->retire);
123+ struct intel_timeline *timeline =
124+ container_of(rcu, struct intel_timeline, rcu);
292125
293− if (timeline->hwsp_cacheline)
294− cacheline_free(timeline->hwsp_cacheline);
295− else
126+ if (timeline->hwsp_map)
296127 i915_gem_object_unpin_map(timeline->hwsp_ggtt->obj);
297128
298129 i915_vma_put(timeline->hwsp_ggtt);
130+ i915_active_fini(&timeline->active);
131+ kfree(timeline);
299132 }
300133
301134 struct intel_timeline *
@@ -351,6 +184,12 @@ int intel_timeline_pin(struct intel_timeline *tl, struct i915_gem_ww_ctx *ww) · 49 unmodified lines
351184 if (atomic_add_unless(&tl->pin_count, 1, 0))
352185 return 0;
353186
187+ if (!tl->hwsp_map) {
188+ err = intel_timeline_pin_map(tl);
189+ if (err)
190+ return err;
191+ }
192+
354193 err = i915_ggtt_pin(tl->hwsp_ggtt, ww, 0, PIN_HIGH);
355194 if (err)
356195 return err;
@@ -361,9 +200,9 @@ int intel_timeline_pin(struct intel_timeline *tl, struct i915_gem_ww_ctx *ww) · 4 unmodified lines
361200 GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",
362201 tl->fence_context, tl->hwsp_offset);
363202
364− cacheline_acquire(tl->hwsp_cacheline, tl->hwsp_offset);
203+ i915_active_acquire(&tl->active);
365204 if (atomic_fetch_inc(&tl->pin_count)) {
366− cacheline_release(tl->hwsp_cacheline);
205+ i915_active_release(&tl->active);
367206 __i915_vma_unpin(tl->hwsp_ggtt);
368207 }
369208
@@ -372,9 +211,13 @@ int intel_timeline_pin(struct intel_timeline *tl, struct i915_gem_ww_ctx *ww) · 2 unmodified lines
372211
373212 void intel_timeline_reset_seqno(const struct intel_timeline *tl)
374213 {
214+ u32 *hwsp_seqno = (u32 *)tl->hwsp_seqno;
375215 /* Must be pinned to be writable, and no requests in flight. */
376216 GEM_BUG_ON(!atomic_read(&tl->pin_count));
377− WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);
217+
218+ memset(hwsp_seqno + 1, 0, TIMELINE_SEQNO_BYTES - sizeof(*hwsp_seqno));
219+ WRITE_ONCE(*hwsp_seqno, tl->seqno);
220+ clflush(hwsp_seqno);
378221 }
379222
380223 void intel_timeline_enter(struct intel_timeline *tl)
@@ -450,106 +293,23 @@ static u32 timeline_advance(struct intel_timeline *tl) · 69 unmodified lines
450293 return tl->seqno += 1 + tl->has_initial_breadcrumb;
451294 }
452295
453−static void timeline_rollback(struct intel_timeline *tl)
454−{
455− tl->seqno -= 1 + tl->has_initial_breadcrumb;
456−}
457−
458296 static noinline int
459297 __intel_timeline_get_seqno(struct intel_timeline *tl,
460− struct i915_request *rq,
461298 u32 *seqno)
462299 {
463− struct intel_timeline_cacheline *cl;
464− unsigned int cacheline;
465− struct i915_vma *vma;
466− void *vaddr;
467− int err;
468−
469− might_lock(&tl->gt->ggtt->vm.mutex);
470− GT_TRACE(tl->gt, "timeline:%llx wrapped\n", tl->fence_context);
471−
472− /*
473− * If there is an outstanding GPU reference to this cacheline,
474− * such as it being sampled by a HW semaphore on another timeline,
475− * we cannot wraparound our seqno value (the HW semaphore does
476− * a strict greater-than-or-equals compare, not i915_seqno_passed).
477− * So if the cacheline is still busy, we must detach ourselves
478− * from it and leave it inflight alongside its users.
479− *
480− * However, if nobody is watching and we can guarantee that nobody
481− * will, we could simply reuse the same cacheline.
482− *
483− * if (i915_active_request_is_signaled(&tl->last_request) &&
484− * i915_active_is_signaled(&tl->hwsp_cacheline->active))
485− * return 0;
486− *
487− * That seems unlikely for a busy timeline that needed to wrap in
488− * the first place, so just replace the cacheline.
489− */
490−
491− vma = hwsp_alloc(tl, &cacheline);
492− if (IS_ERR(vma)) {
493− err = PTR_ERR(vma);
494− goto err_rollback;
495− }
496−
497− err = i915_ggtt_pin(vma, NULL, 0, PIN_HIGH);
498− if (err) {
499− __idle_hwsp_free(vma->private, cacheline);
500− goto err_rollback;
501− }
502−
503− cl = cacheline_alloc(vma->private, cacheline);
504− if (IS_ERR(cl)) {
505− err = PTR_ERR(cl);
506− __idle_hwsp_free(vma->private, cacheline);
507− goto err_unpin;
508− }
509− GEM_BUG_ON(cl->hwsp->vma != vma);
510−
511− /*
512− * Attach the old cacheline to the current request, so that we only
513− * free it after the current request is retired, which ensures that
514− * all writes into the cacheline from previous requests are complete.
515− */
516− err = i915_active_ref(&tl->hwsp_cacheline->active,
517− tl->fence_context,
518− &rq->fence);
519− if (err)
520− goto err_cacheline;
521−
522− cacheline_release(tl->hwsp_cacheline); /* ownership now xfered to rq */
523− cacheline_free(tl->hwsp_cacheline);
524−
525− i915_vma_unpin(tl->hwsp_ggtt); /* binding kept alive by old cacheline */
526− i915_vma_put(tl->hwsp_ggtt);
527−
528− tl->hwsp_ggtt = i915_vma_get(vma);
300+ u32 next_ofs = offset_in_page(tl->hwsp_offset + TIMELINE_SEQNO_BYTES);
529301
530− vaddr = page_mask_bits(cl->vaddr);
531− tl->hwsp_offset = cacheline * CACHELINE_BYTES;
532− tl->hwsp_seqno =
533− memset(vaddr + tl->hwsp_offset, 0, CACHELINE_BYTES);
302+ /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
303+ if (TIMELINE_SEQNO_BYTES <= BIT(5) && (next_ofs & BIT(5)))
304+ next_ofs = offset_in_page(next_ofs + BIT(5));
534305
535− tl->hwsp_offset += i915_ggtt_offset(vma);
536− GT_TRACE(tl->gt, "timeline:%llx using HWSP offset:%x\n",
537− tl->fence_context, tl->hwsp_offset);
538−
539− cacheline_acquire(cl, tl->hwsp_offset);
540− tl->hwsp_cacheline = cl;
306+ tl->hwsp_offset = i915_ggtt_offset(tl->hwsp_ggtt) + next_ofs;
307+ tl->hwsp_seqno = tl->hwsp_map + next_ofs;
308+ intel_timeline_reset_seqno(tl);
541309
542310 *seqno = timeline_advance(tl);
543311 GEM_BUG_ON(i915_seqno_passed(*tl->hwsp_seqno, *seqno));
544312 return 0;
545−
546−err_cacheline:
547− cacheline_free(cl);
548−err_unpin:
549− i915_vma_unpin(vma);
550−err_rollback:
551− timeline_rollback(tl);
552− return err;
553313 }
554314
555315 int intel_timeline_get_seqno(struct intel_timeline *tl,
@@ -559,51 +319,52 @@ int intel_timeline_get_seqno(struct intel_timeline *tl, · 3 unmodified lines
559319 *seqno = timeline_advance(tl);
560320
561321 /* Replace the HWSP on wraparound for HW semaphores */
562− if (unlikely(!*seqno && tl->hwsp_cacheline))
563− return __intel_timeline_get_seqno(tl, rq, seqno);
322+ if (unlikely(!*seqno && tl->has_initial_breadcrumb))
323+ return __intel_timeline_get_seqno(tl, seqno);
564324
565325 return 0;
566326 }
567327
568−static int cacheline_ref(struct intel_timeline_cacheline *cl,
569− struct i915_request *rq)
570−{
571− return i915_active_add_request(&cl->active, rq);
572−}
573−
574328 int intel_timeline_read_hwsp(struct i915_request *from,
575329 struct i915_request *to,
576330 u32 *hwsp)
577331 {
578− struct intel_timeline_cacheline *cl;
332+ struct intel_timeline *tl;
579333 int err;
580334
581− GEM_BUG_ON(!rcu_access_pointer(from->hwsp_cacheline));
582−
583335 rcu_read_lock();
584− cl = rcu_dereference(from->hwsp_cacheline);
585− if (i915_request_signaled(from)) /* confirm cacheline is valid */
586− goto unlock;
587− if (unlikely(!i915_active_acquire_if_busy(&cl->active)))
588− goto unlock; /* seqno wrapped and completed! */
589− if (unlikely(__i915_request_is_complete(from)))
590− goto release;
336+ tl = rcu_dereference(from->timeline);
337+ if (i915_request_signaled(from) ||
338+ !i915_active_acquire_if_busy(&tl->active))
339+ tl = NULL;
340+
341+ if (tl) {
342+ /* hwsp_offset may wraparound, so use from->hwsp_seqno */
343+ *hwsp = i915_ggtt_offset(tl->hwsp_ggtt) +
344+ offset_in_page(from->hwsp_seqno);
345+ }
346+
347+ /* ensure we wait on the right request, if not, we completed */
348+ if (tl && __i915_request_is_complete(from)) {
349+ i915_active_release(&tl->active);
350+ tl = NULL;
351+ }
591352 rcu_read_unlock();
592353
593− err = cacheline_ref(cl, to);
594− if (err)
354+ if (!tl)
355+ return 1;
356+
357+ /* Can't do semaphore waits on kernel context */
358+ if (!tl->has_initial_breadcrumb) {
359+ err = -EINVAL;
595360 goto out;
361+ }
362+
363+ err = i915_active_add_request(&tl->active, to);
596364
597− *hwsp = cl->ggtt_offset;
598365 out:
599− i915_active_release(&cl->active);
366+ i915_active_release(&tl->active);
600367 return err;
601−
602−release:
603− i915_active_release(&cl->active);
604−unlock:
605− rcu_read_unlock();
606− return 1;
607368 }
608369
609370 void intel_timeline_unpin(struct intel_timeline *tl)
@@ -612,8 +373,7 @@ void intel_timeline_unpin(struct intel_timeline *tl) · 2 unmodified lines
612373 if (!atomic_dec_and_test(&tl->pin_count))
613374 return;
614375
615− cacheline_release(tl->hwsp_cacheline);
616−
376+ i915_active_release(&tl->active);
617377 __i915_vma_unpin(tl->hwsp_ggtt);
618378 }
619379
@@ -622,8 +382,11 @@ void __intel_timeline_free(struct kref *kref) · 2 unmodified lines
622382 struct intel_timeline *timeline =
623383 container_of(kref, typeof(*timeline), kref);
624384
625− intel_timeline_fini(timeline);
626− kfree_rcu(timeline, rcu);
385+ GEM_BUG_ON(atomic_read(&timeline->pin_count));
386+ GEM_BUG_ON(!list_empty(&timeline->requests));
387+ GEM_BUG_ON(timeline->retire);
388+
389+ call_rcu(&timeline->rcu, intel_timeline_fini);
627390 }
628391
629392 void intel_gt_fini_timelines(struct intel_gt *gt)
@@ -631,7 +394,6 @@ void intel_gt_fini_timelines(struct intel_gt *gt) · 1 unmodified line
631394 struct intel_gt_timelines *timelines = &gt->timelines;
632395
633396 GEM_BUG_ON(!list_empty(&timelines->active_list));
634− GEM_BUG_ON(!list_empty(&timelines->hwsp_free_list));
635397 }
636398
637399 void intel_gt_show_timelines(struct intel_gt *gt,
drivers/gpu/drm/i915/gt/intel_timeline.h+2−0
@@ -117,4 +117,6 @@ intel_timeline_is_last(const struct intel_timeline *tl, · 116 unmodified lines
117117 return list_is_last_rcu(&rq->link, &tl->requests);
118118 }
119119
120+I915_SELFTEST_DECLARE(int intel_timeline_pin_map(struct intel_timeline *tl));
121+
120122 #endif
drivers/gpu/drm/i915/gt/intel_timeline_types.h+3−14
@@ -18,7 +18,6 @@ · 17 unmodified lines
1818 struct i915_vma;
1919 struct i915_syncmap;
2020 struct intel_gt;
21−struct intel_timeline_hwsp;
2221
2322 struct intel_timeline {
2423 u64 fence_context;
@@ -45,12 +44,11 @@ struct intel_timeline { · 20 unmodified lines
4544 atomic_t pin_count;
4645 atomic_t active_count;
4746
47+ void *hwsp_map;
4848 const u32 *hwsp_seqno;
4949 struct i915_vma *hwsp_ggtt;
5050 u32 hwsp_offset;
5151
52− struct intel_timeline_cacheline *hwsp_cacheline;
53−
5452 bool has_initial_breadcrumb;
5553
5654 /**
@@ -67,6 +65,8 @@ struct intel_timeline { · 10 unmodified lines
6765 */
6866 struct i915_active_fence last_request;
6967
68+ struct i915_active active;
69+
7070 /** A chain of completed timelines ready for early retirement. */
7171 struct intel_timeline *retire;
7272
@@ -90,15 +90,4 @@ struct intel_timeline { · 17 unmodified lines
9090 struct rcu_head rcu;
9191 };
9292
93−struct intel_timeline_cacheline {
94− struct i915_active active;
95−
96− struct intel_timeline_hwsp *hwsp;
97− void *vaddr;
98−
99− u32 ggtt_offset;
100−
101− struct rcu_head rcu;
102−};
103−
10493 #endif /* __I915_TIMELINE_TYPES_H__ */
drivers/gpu/drm/i915/gt/intel_workarounds.c+8−2
@@ -2213,10 +2213,15 @@ retry: · 2212 unmodified lines
22132213 if (err)
22142214 goto err_pm;
22152215
2216+ err = i915_vma_pin_ww(vma, &ww, 0, 0,
2217+ i915_vma_is_ggtt(vma) ? PIN_GLOBAL : PIN_USER);
2218+ if (err)
2219+ goto err_unpin;
2220+
22162221 rq = i915_request_create(ce);
22172222 if (IS_ERR(rq)) {
22182223 err = PTR_ERR(rq);
2219− goto err_unpin;
2224+ goto err_vma;
22202225 }
22212226
22222227 err = i915_request_await_object(rq, vma->obj, true);
@@ -2257,6 +2262,8 @@ retry: · 34 unmodified lines
22572262
22582263 err_rq:
22592264 i915_request_put(rq);
2265+err_vma:
2266+ i915_vma_unpin(vma);
22602267 err_unpin:
22612268 intel_context_unpin(ce);
22622269 err_pm:
@@ -2267,7 +2274,6 @@ err_pm: · 4 unmodified lines
22672274 }
22682275 i915_gem_ww_ctx_fini(&ww);
22692276 intel_engine_pm_put(ce->engine);
2270− i915_vma_unpin(vma);
22712277 i915_vma_put(vma);
22722278 return err;
22732279 }
drivers/gpu/drm/i915/gt/mock_engine.c+20−2
@@ -32,9 +32,20 @@ · 31 unmodified lines
3232 #include "mock_engine.h"
3333 #include "selftests/mock_request.h"
3434
35−static void mock_timeline_pin(struct intel_timeline *tl)
35+static int mock_timeline_pin(struct intel_timeline *tl)
3636 {
37+ int err;
38+
39+ if (WARN_ON(!i915_gem_object_trylock(tl->hwsp_ggtt->obj)))
40+ return -EBUSY;
41+
42+ err = intel_timeline_pin_map(tl);
43+ i915_gem_object_unlock(tl->hwsp_ggtt->obj);
44+ if (err)
45+ return err;
46+
3747 atomic_inc(&tl->pin_count);
48+ return 0;
3849 }
3950
4051 static void mock_timeline_unpin(struct intel_timeline *tl)
@@ -152,6 +163,8 @@ static void mock_context_destroy(struct kref *ref) · 111 unmodified lines
152163
153164 static int mock_context_alloc(struct intel_context *ce)
154165 {
166+ int err;
167+
155168 ce->ring = mock_ring(ce->engine);
156169 if (!ce->ring)
157170 return -ENOMEM;
@@ -162,7 +175,12 @@ static int mock_context_alloc(struct intel_context *ce) · 4 unmodified lines
162175 return PTR_ERR(ce->timeline);
163176 }
164177
165− mock_timeline_pin(ce->timeline);
178+ err = mock_timeline_pin(ce->timeline);
179+ if (err) {
180+ intel_timeline_put(ce->timeline);
181+ ce->timeline = NULL;
182+ return err;
183+ }
166184
167185 return 0;
168186 }
drivers/gpu/drm/i915/gt/selftest_context.c+2−2
@@ -88,8 +88,8 @@ static int __live_context_size(struct intel_engine_cs *engine) · 87 unmodified lines
8888 if (err)
8989 goto err;
9090
91− vaddr = i915_gem_object_pin_map(ce->state->obj,
92− i915_coherent_map_type(engine->i915));
91+ vaddr = i915_gem_object_pin_map_unlocked(ce->state->obj,
92+ i915_coherent_map_type(engine->i915));
9393 if (IS_ERR(vaddr)) {
9494 err = PTR_ERR(vaddr);
9595 intel_context_unpin(ce);
drivers/gpu/drm/i915/gt/selftest_engine_cs.c+6−3
@@ -42,6 +42,9 @@ static int perf_end(struct intel_gt *gt) · 41 unmodified lines
4242
4343 static int write_timestamp(struct i915_request *rq, int slot)
4444 {
45+ struct intel_timeline *tl =
46+ rcu_dereference_protected(rq->timeline,
47+ !i915_request_signaled(rq));
4548 u32 cmd;
4649 u32 *cs;
4750
@@ -54,7 +57,7 @@ static int write_timestamp(struct i915_request *rq, int slot) · 6 unmodified lines
5457 cmd++;
5558 *cs++ = cmd;
5659 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP(rq->engine->mmio_base));
57− *cs++ = i915_request_timeline(rq)->hwsp_offset + slot * sizeof(u32);
60+ *cs++ = tl->hwsp_offset + slot * sizeof(u32);
5861 *cs++ = 0;
5962
6063 intel_ring_advance(rq, cs);
@@ -73,7 +76,7 @@ static struct i915_vma *create_empty_batch(struct intel_context *ce) · 12 unmodified lines
7376 if (IS_ERR(obj))
7477 return ERR_CAST(obj);
7578
76− cs = i915_gem_object_pin_map(obj, I915_MAP_WB);
79+ cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
7780 if (IS_ERR(cs)) {
7881 err = PTR_ERR(cs);
7982 goto err_put;
@@ -209,7 +212,7 @@ static struct i915_vma *create_nop_batch(struct intel_context *ce) · 129 unmodified lines
209212 if (IS_ERR(obj))
210213 return ERR_CAST(obj);
211214
212− cs = i915_gem_object_pin_map(obj, I915_MAP_WB);
215+ cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
213216 if (IS_ERR(cs)) {
214217 err = PTR_ERR(cs);
215218 goto err_put;
drivers/gpu/drm/i915/gt/selftest_execlists.c+12−11
@@ -989,7 +989,7 @@ static int live_timeslice_preempt(void *arg) · 988 unmodified lines
989989 goto err_obj;
990990 }
991991
992− vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
992+ vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
993993 if (IS_ERR(vaddr)) {
994994 err = PTR_ERR(vaddr);
995995 goto err_obj;
@@ -1297,7 +1297,7 @@ static int live_timeslice_queue(void *arg) · 301 unmodified lines
12971297 goto err_obj;
12981298 }
12991299
1300− vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
1300+ vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
13011301 if (IS_ERR(vaddr)) {
13021302 err = PTR_ERR(vaddr);
13031303 goto err_obj;
@@ -1544,7 +1544,7 @@ static int live_busywait_preempt(void *arg) · 240 unmodified lines
15441544 goto err_ctx_lo;
15451545 }
15461546
1547− map = i915_gem_object_pin_map(obj, I915_MAP_WC);
1547+ map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
15481548 if (IS_ERR(map)) {
15491549 err = PTR_ERR(map);
15501550 goto err_obj;
@@ -2714,7 +2714,7 @@ static int create_gang(struct intel_engine_cs *engine, · 1163 unmodified lines
27142714 if (err)
27152715 goto err_obj;
27162716
2717− cs = i915_gem_object_pin_map(obj, I915_MAP_WC);
2717+ cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
27182718 if (IS_ERR(cs)) {
27192719 err = PTR_ERR(cs);
27202720 goto err_obj;
@@ -2997,7 +2997,7 @@ static int live_preempt_gang(void *arg) · 276 unmodified lines
29972997 * it will terminate the next lowest spinner until there
29982998 * are no more spinners and the gang is complete.
29992999 */
3000− cs = i915_gem_object_pin_map(rq->batch->obj, I915_MAP_WC);
3000+ cs = i915_gem_object_pin_map_unlocked(rq->batch->obj, I915_MAP_WC);
30013001 if (!IS_ERR(cs)) {
30023002 *cs = 0;
30033003 i915_gem_object_unpin_map(rq->batch->obj);
@@ -3062,7 +3062,7 @@ create_gpr_user(struct intel_engine_cs *engine, · 58 unmodified lines
30623062 return ERR_PTR(err);
30633063 }
30643064
3065− cs = i915_gem_object_pin_map(obj, I915_MAP_WC);
3065+ cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
30663066 if (IS_ERR(cs)) {
30673067 i915_vma_put(vma);
30683068 return ERR_CAST(cs);
@@ -3269,7 +3269,7 @@ static int live_preempt_user(void *arg) · 200 unmodified lines
32693269 if (IS_ERR(global))
32703270 return PTR_ERR(global);
32713271
3272− result = i915_gem_object_pin_map(global->obj, I915_MAP_WC);
3272+ result = i915_gem_object_pin_map_unlocked(global->obj, I915_MAP_WC);
32733273 if (IS_ERR(result)) {
32743274 i915_vma_unpin_and_release(&global, 0);
32753275 return PTR_ERR(result);
@@ -3658,7 +3658,7 @@ static int live_preempt_smoke(void *arg) · 382 unmodified lines
36583658 goto err_free;
36593659 }
36603660
3661− cs = i915_gem_object_pin_map(smoke.batch, I915_MAP_WB);
3661+ cs = i915_gem_object_pin_map_unlocked(smoke.batch, I915_MAP_WB);
36623662 if (IS_ERR(cs)) {
36633663 err = PTR_ERR(cs);
36643664 goto err_batch;
@@ -4197,8 +4197,9 @@ static int preserved_virtual_engine(struct intel_gt *gt, · 532 unmodified lines
41974197 int err = 0;
41984198 u32 *cs;
41994199
4200− scratch = __vm_create_scratch_for_read(&siblings[0]->gt->ggtt->vm,
4201− PAGE_SIZE);
4200+ scratch =
4201+ __vm_create_scratch_for_read_pinned(&siblings[0]->gt->ggtt->vm,
4202+ PAGE_SIZE);
42024203 if (IS_ERR(scratch))
42034204 return PTR_ERR(scratch);
42044205
@@ -4262,7 +4263,7 @@ static int preserved_virtual_engine(struct intel_gt *gt, · 57 unmodified lines
42624263 goto out_end;
42634264 }
42644265
4265− cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
4266+ cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
42664267 if (IS_ERR(cs)) {
42674268 err = PTR_ERR(cs);
42684269 goto out_end;
drivers/gpu/drm/i915/gt/selftest_hangcheck.c+4−4
@@ -80,15 +80,15 @@ static int hang_init(struct hang *h, struct intel_gt *gt) · 79 unmodified lines
8080 }
8181
8282 i915_gem_object_set_cache_coherency(h->hws, I915_CACHE_LLC);
83− vaddr = i915_gem_object_pin_map(h->hws, I915_MAP_WB);
83+ vaddr = i915_gem_object_pin_map_unlocked(h->hws, I915_MAP_WB);
8484 if (IS_ERR(vaddr)) {
8585 err = PTR_ERR(vaddr);
8686 goto err_obj;
8787 }
8888 h->seqno = memset(vaddr, 0xff, PAGE_SIZE);
8989
90− vaddr = i915_gem_object_pin_map(h->obj,
91− i915_coherent_map_type(gt->i915));
90+ vaddr = i915_gem_object_pin_map_unlocked(h->obj,
91+ i915_coherent_map_type(gt->i915));
9292 if (IS_ERR(vaddr)) {
9393 err = PTR_ERR(vaddr);
9494 goto err_unpin_hws;
@@ -149,7 +149,7 @@ hang_create_request(struct hang *h, struct intel_engine_cs *engine) · 54 unmodified lines
149149 return ERR_CAST(obj);
150150 }
151151
152− vaddr = i915_gem_object_pin_map(obj, i915_coherent_map_type(gt->i915));
152+ vaddr = i915_gem_object_pin_map_unlocked(obj, i915_coherent_map_type(gt->i915));
153153 if (IS_ERR(vaddr)) {
154154 i915_gem_object_put(obj);
155155 i915_vm_put(vm);
drivers/gpu/drm/i915/gt/selftest_lrc.c+9−9
@@ -27,7 +27,7 @@ · 26 unmodified lines
2727
2828 static struct i915_vma *create_scratch(struct intel_gt *gt)
2929 {
30− return __vm_create_scratch_for_read(&gt->ggtt->vm, PAGE_SIZE);
30+ return __vm_create_scratch_for_read_pinned(&gt->ggtt->vm, PAGE_SIZE);
3131 }
3232
3333 static bool is_active(struct i915_request *rq)
@@ -627,7 +627,7 @@ static int __live_lrc_gpr(struct intel_engine_cs *engine, · 593 unmodified lines
627627 goto err_rq;
628628 }
629629
630− cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
630+ cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
631631 if (IS_ERR(cs)) {
632632 err = PTR_ERR(cs);
633633 goto err_rq;
@@ -921,7 +921,7 @@ store_context(struct intel_context *ce, struct i915_vma *scratch) · 287 unmodified lines
921921 if (IS_ERR(batch))
922922 return batch;
923923
924− cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
924+ cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
925925 if (IS_ERR(cs)) {
926926 i915_vma_put(batch);
927927 return ERR_CAST(cs);
@@ -1085,7 +1085,7 @@ static struct i915_vma *load_context(struct intel_context *ce, u32 poison) · 157 unmodified lines
10851085 if (IS_ERR(batch))
10861086 return batch;
10871087
1088− cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
1088+ cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
10891089 if (IS_ERR(cs)) {
10901090 i915_vma_put(batch);
10911091 return ERR_CAST(cs);
@@ -1199,29 +1199,29 @@ static int compare_isolation(struct intel_engine_cs *engine, · 107 unmodified lines
11991199 u32 *defaults;
12001200 int err = 0;
12011201
1202− A[0] = i915_gem_object_pin_map(ref[0]->obj, I915_MAP_WC);
1202+ A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);
12031203 if (IS_ERR(A[0]))
12041204 return PTR_ERR(A[0]);
12051205
1206− A[1] = i915_gem_object_pin_map(ref[1]->obj, I915_MAP_WC);
1206+ A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);
12071207 if (IS_ERR(A[1])) {
12081208 err = PTR_ERR(A[1]);
12091209 goto err_A0;
12101210 }
12111211
1212− B[0] = i915_gem_object_pin_map(result[0]->obj, I915_MAP_WC);
1212+ B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);
12131213 if (IS_ERR(B[0])) {
12141214 err = PTR_ERR(B[0]);
12151215 goto err_A1;
12161216 }
12171217
1218− B[1] = i915_gem_object_pin_map(result[1]->obj, I915_MAP_WC);
1218+ B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);
12191219 if (IS_ERR(B[1])) {
12201220 err = PTR_ERR(B[1]);
12211221 goto err_B0;
12221222 }
12231223
1224− lrc = i915_gem_object_pin_map(ce->state->obj,
1224+ lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,
12251225 i915_coherent_map_type(engine->i915));
12261226 if (IS_ERR(lrc)) {
12271227 err = PTR_ERR(lrc);
drivers/gpu/drm/i915/gt/selftest_mocs.c+3−2
@@ -75,11 +75,12 @@ static int live_mocs_init(struct live_mocs *arg, struct intel_gt *gt) · 74 unmodified lines
7575 if (flags & (HAS_GLOBAL_MOCS | HAS_ENGINE_MOCS))
7676 arg->mocs = table;
7777
78− arg->scratch = __vm_create_scratch_for_read(&gt->ggtt->vm, PAGE_SIZE);
78+ arg->scratch =
79+ __vm_create_scratch_for_read_pinned(&gt->ggtt->vm, PAGE_SIZE);
7980 if (IS_ERR(arg->scratch))
8081 return PTR_ERR(arg->scratch);
8182
82− arg->vaddr = i915_gem_object_pin_map(arg->scratch->obj, I915_MAP_WB);
83+ arg->vaddr = i915_gem_object_pin_map_unlocked(arg->scratch->obj, I915_MAP_WB);
8384 if (IS_ERR(arg->vaddr)) {
8485 err = PTR_ERR(arg->vaddr);
8586 goto err_scratch;
drivers/gpu/drm/i915/gt/selftest_ring_submission.c+2−2
@@ -35,7 +35,7 @@ static struct i915_vma *create_wally(struct intel_engine_cs *engine) · 34 unmodified lines
3535 return ERR_PTR(err);
3636 }
3737
38− cs = i915_gem_object_pin_map(obj, I915_MAP_WC);
38+ cs = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
3939 if (IS_ERR(cs)) {
4040 i915_gem_object_put(obj);
4141 return ERR_CAST(cs);
@@ -212,7 +212,7 @@ static int __live_ctx_switch_wa(struct intel_engine_cs *engine) · 170 unmodified lines
212212 if (IS_ERR(bb))
213213 return PTR_ERR(bb);
214214
215− result = i915_gem_object_pin_map(bb->obj, I915_MAP_WC);
215+ result = i915_gem_object_pin_map_unlocked(bb->obj, I915_MAP_WC);
216216 if (IS_ERR(result)) {
217217 intel_context_put(bb->private);
218218 i915_vma_unpin_and_release(&bb, 0);
drivers/gpu/drm/i915/gt/selftest_timeline.c+109−68
@@ -35,10 +35,31 @@ static unsigned long hwsp_cacheline(struct intel_timeline *tl) · 34 unmodified lines
3535 {
3636 unsigned long address = (unsigned long)page_address(hwsp_page(tl));
3737
38− return (address + tl->hwsp_offset) / CACHELINE_BYTES;
38+ return (address + offset_in_page(tl->hwsp_offset)) / TIMELINE_SEQNO_BYTES;
3939 }
4040
41−#define CACHELINES_PER_PAGE (PAGE_SIZE / CACHELINE_BYTES)
41+static int selftest_tl_pin(struct intel_timeline *tl)
42+{
43+ struct i915_gem_ww_ctx ww;
44+ int err;
45+
46+ i915_gem_ww_ctx_init(&ww, false);
47+retry:
48+ err = i915_gem_object_lock(tl->hwsp_ggtt->obj, &ww);
49+ if (!err)
50+ err = intel_timeline_pin(tl, &ww);
51+
52+ if (err == -EDEADLK) {
53+ err = i915_gem_ww_ctx_backoff(&ww);
54+ if (!err)
55+ goto retry;
56+ }
57+ i915_gem_ww_ctx_fini(&ww);
58+ return err;
59+}
60+
61+/* Only half of seqno's are usable, see __intel_timeline_get_seqno() */
62+#define CACHELINES_PER_PAGE (PAGE_SIZE / TIMELINE_SEQNO_BYTES / 2)
4263
4364 struct mock_hwsp_freelist {
4465 struct intel_gt *gt;
@@ -59,6 +80,7 @@ static void __mock_hwsp_record(struct mock_hwsp_freelist *state, · 14 unmodified lines
5980 tl = xchg(&state->history[idx], tl);
6081 if (tl) {
6182 radix_tree_delete(&state->cachelines, hwsp_cacheline(tl));
83+ intel_timeline_unpin(tl);
6284 intel_timeline_put(tl);
6385 }
6486 }
@@ -78,6 +100,12 @@ static int __mock_hwsp_timeline(struct mock_hwsp_freelist *state, · 13 unmodified lines
78100 if (IS_ERR(tl))
79101 return PTR_ERR(tl);
80102
103+ err = selftest_tl_pin(tl);
104+ if (err) {
105+ intel_timeline_put(tl);
106+ return err;
107+ }
108+
81109 cacheline = hwsp_cacheline(tl);
82110 err = radix_tree_insert(&state->cachelines, cacheline, tl);
83111 if (err) {
@@ -85,6 +113,7 @@ static int __mock_hwsp_timeline(struct mock_hwsp_freelist *state, · 1 unmodified line
85113 pr_err("HWSP cacheline %lu already used; duplicate allocation!\n",
86114 cacheline);
87115 }
116+ intel_timeline_unpin(tl);
88117 intel_timeline_put(tl);
89118 return err;
90119 }
@@ -452,17 +481,24 @@ static int emit_ggtt_store_dw(struct i915_request *rq, u32 addr, u32 value) · 361 unmodified lines
452481 }
453482
454483 static struct i915_request *
455−tl_write(struct intel_timeline *tl, struct intel_engine_cs *engine, u32 value)
484+checked_tl_write(struct intel_timeline *tl, struct intel_engine_cs *engine, u32 value)
456485 {
457486 struct i915_request *rq;
458487 int err;
459488
460− err = intel_timeline_pin(tl, NULL);
489+ err = selftest_tl_pin(tl);
461490 if (err) {
462491 rq = ERR_PTR(err);
463492 goto out;
464493 }
465494
495+ if (READ_ONCE(*tl->hwsp_seqno) != tl->seqno) {
496+ pr_err("Timeline created with incorrect breadcrumb, found %x, expected %x\n",
497+ *tl->hwsp_seqno, tl->seqno);
498+ intel_timeline_unpin(tl);
499+ return ERR_PTR(-EINVAL);
500+ }
501+
466502 rq = intel_engine_create_kernel_request(engine);
467503 if (IS_ERR(rq))
468504 goto out_unpin;
@@ -484,25 +520,6 @@ out: · 15 unmodified lines
484520 return rq;
485521 }
486522
487−static struct intel_timeline *
488−checked_intel_timeline_create(struct intel_gt *gt)
489−{
490− struct intel_timeline *tl;
491−
492− tl = intel_timeline_create(gt);
493− if (IS_ERR(tl))
494− return tl;
495−
496− if (READ_ONCE(*tl->hwsp_seqno) != tl->seqno) {
497− pr_err("Timeline created with incorrect breadcrumb, found %x, expected %x\n",
498− *tl->hwsp_seqno, tl->seqno);
499− intel_timeline_put(tl);
500− return ERR_PTR(-EINVAL);
501− }
502−
503− return tl;
504−}
505−
506523 static int live_hwsp_engine(void *arg)
507524 {
508525 #define NUM_TIMELINES 4096
@@ -535,13 +552,13 @@ static int live_hwsp_engine(void *arg) · 26 unmodified lines
535552 struct intel_timeline *tl;
536553 struct i915_request *rq;
537554
538− tl = checked_intel_timeline_create(gt);
555+ tl = intel_timeline_create(gt);
539556 if (IS_ERR(tl)) {
540557 err = PTR_ERR(tl);
541558 break;
542559 }
543560
544− rq = tl_write(tl, engine, count);
561+ rq = checked_tl_write(tl, engine, count);
545562 if (IS_ERR(rq)) {
546563 intel_timeline_put(tl);
547564 err = PTR_ERR(rq);
@@ -608,14 +625,14 @@ static int live_hwsp_alternate(void *arg) · 60 unmodified lines
608625 if (!intel_engine_can_store_dword(engine))
609626 continue;
610627
611− tl = checked_intel_timeline_create(gt);
628+ tl = intel_timeline_create(gt);
612629 if (IS_ERR(tl)) {
613630 err = PTR_ERR(tl);
614631 goto out;
615632 }
616633
617634 intel_engine_pm_get(engine);
618− rq = tl_write(tl, engine, count);
635+ rq = checked_tl_write(tl, engine, count);
619636 intel_engine_pm_put(engine);
620637 if (IS_ERR(rq)) {
621638 intel_timeline_put(tl);
@@ -666,10 +683,10 @@ static int live_hwsp_wrap(void *arg) · 44 unmodified lines
666683 if (IS_ERR(tl))
667684 return PTR_ERR(tl);
668685
669− if (!tl->has_initial_breadcrumb || !tl->hwsp_cacheline)
686+ if (!tl->has_initial_breadcrumb)
670687 goto out_free;
671688
672− err = intel_timeline_pin(tl, NULL);
689+ err = selftest_tl_pin(tl);
673690 if (err)
674691 goto out_free;
675692
@@ -816,13 +833,13 @@ static int setup_watcher(struct hwsp_watcher *w, struct intel_gt *gt) · 140 unmodified lines
816833 if (IS_ERR(obj))
817834 return PTR_ERR(obj);
818835
819− w->map = i915_gem_object_pin_map(obj, I915_MAP_WB);
836+ w->map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
820837 if (IS_ERR(w->map)) {
821838 i915_gem_object_put(obj);
822839 return PTR_ERR(w->map);
823840 }
824841
825− vma = i915_gem_object_ggtt_pin_ww(obj, NULL, NULL, 0, 0, 0);
842+ vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
826843 if (IS_ERR(vma)) {
827844 i915_gem_object_put(obj);
828845 return PTR_ERR(vma);
@@ -833,12 +850,26 @@ static int setup_watcher(struct hwsp_watcher *w, struct intel_gt *gt) · 4 unmodified lines
833850 return 0;
834851 }
835852
853+static void switch_tl_lock(struct i915_request *from, struct i915_request *to)
854+{
855+ /* some light mutex juggling required; think co-routines */
856+
857+ if (from) {
858+ lockdep_unpin_lock(&from->context->timeline->mutex, from->cookie);
859+ mutex_unlock(&from->context->timeline->mutex);
860+ }
861+
862+ if (to) {
863+ mutex_lock(&to->context->timeline->mutex);
864+ to->cookie = lockdep_pin_lock(&to->context->timeline->mutex);
865+ }
866+}
867+
836868 static int create_watcher(struct hwsp_watcher *w,
837869 struct intel_engine_cs *engine,
838870 int ringsz)
839871 {
840872 struct intel_context *ce;
841− struct intel_timeline *tl;
842873
843874 ce = intel_context_create(engine);
844875 if (IS_ERR(ce))
@@ -851,11 +882,8 @@ static int create_watcher(struct hwsp_watcher *w, · 6 unmodified lines
851882 return PTR_ERR(w->rq);
852883
853884 w->addr = i915_ggtt_offset(w->vma);
854− tl = w->rq->context->timeline;
855885
856− /* some light mutex juggling required; think co-routines */
857− lockdep_unpin_lock(&tl->mutex, w->rq->cookie);
858− mutex_unlock(&tl->mutex);
886+ switch_tl_lock(w->rq, NULL);
859887
860888 return 0;
861889 }
@@ -864,15 +892,13 @@ static int check_watcher(struct hwsp_watcher *w, const char *name, · 2 unmodified lines
864892 bool (*op)(u32 hwsp, u32 seqno))
865893 {
866894 struct i915_request *rq = fetch_and_zero(&w->rq);
867− struct intel_timeline *tl = rq->context->timeline;
868895 u32 offset, end;
869896 int err;
870897
871898 GEM_BUG_ON(w->addr - i915_ggtt_offset(w->vma) > w->vma->size);
872899
873900 i915_request_get(rq);
874− mutex_lock(&tl->mutex);
875− rq->cookie = lockdep_pin_lock(&tl->mutex);
901+ switch_tl_lock(NULL, rq);
876902 i915_request_add(rq);
877903
878904 if (i915_request_wait(rq, 0, HZ) < 0) {
@@ -901,10 +927,7 @@ out: · 22 unmodified lines
901927 static void cleanup_watcher(struct hwsp_watcher *w)
902928 {
903929 if (w->rq) {
904− struct intel_timeline *tl = w->rq->context->timeline;
905−
906− mutex_lock(&tl->mutex);
907− w->rq->cookie = lockdep_pin_lock(&tl->mutex);
930+ switch_tl_lock(NULL, w->rq);
908931
909932 i915_request_add(w->rq);
910933 }
@@ -942,7 +965,7 @@ static struct i915_request *wrap_timeline(struct i915_request *rq) · 31 unmodified lines
942965 }
943966
944967 i915_request_put(rq);
945− rq = intel_context_create_request(ce);
968+ rq = i915_request_create(ce);
946969 if (IS_ERR(rq))
947970 return rq;
948971
@@ -977,7 +1000,7 @@ static int live_hwsp_read(void *arg) · 28 unmodified lines
9771000 if (IS_ERR(tl))
9781001 return PTR_ERR(tl);
9791002
980− if (!tl->hwsp_cacheline)
1003+ if (!tl->has_initial_breadcrumb)
9811004 goto out_free;
9821005
9831006 for (i = 0; i < ARRAY_SIZE(watcher); i++) {
@@ -999,7 +1022,7 @@ static int live_hwsp_read(void *arg) · 15 unmodified lines
9991022 do {
10001023 struct i915_sw_fence *submit;
10011024 struct i915_request *rq;
1002− u32 hwsp;
1025+ u32 hwsp, dummy;
10031026
10041027 submit = heap_fence_create(GFP_KERNEL);
10051028 if (!submit) {
@@ -1017,14 +1040,26 @@ static int live_hwsp_read(void *arg) · 11 unmodified lines
10171040 goto out;
10181041 }
10191042
1020− /* Skip to the end, saving 30 minutes of nops */
1021− tl->seqno = -10u + 2 * (count & 3);
1022− WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);
10231043 ce->timeline = intel_timeline_get(tl);
10241044
1025− rq = intel_context_create_request(ce);
1045+ /* Ensure timeline is mapped, done during first pin */
1046+ err = intel_context_pin(ce);
1047+ if (err) {
1048+ intel_context_put(ce);
1049+ goto out;
1050+ }
1051+
1052+ /*
1053+ * Start at a new wrap, and set seqno right before another wrap,
1054+ * saving 30 minutes of nops
1055+ */
1056+ tl->seqno = -12u + 2 * (count & 3);
1057+ __intel_timeline_get_seqno(tl, &dummy);
1058+
1059+ rq = i915_request_create(ce);
10261060 if (IS_ERR(rq)) {
10271061 err = PTR_ERR(rq);
1062+ intel_context_unpin(ce);
10281063 intel_context_put(ce);
10291064 goto out;
10301065 }
@@ -1034,32 +1069,35 @@ static int live_hwsp_read(void *arg) · 3 unmodified lines
10341069 GFP_KERNEL);
10351070 if (err < 0) {
10361071 i915_request_add(rq);
1072+ intel_context_unpin(ce);
10371073 intel_context_put(ce);
10381074 goto out;
10391075 }
10401076
1041− mutex_lock(&watcher[0].rq->context->timeline->mutex);
1077+ switch_tl_lock(rq, watcher[0].rq);
10421078 err = intel_timeline_read_hwsp(rq, watcher[0].rq, &hwsp);
10431079 if (err == 0)
10441080 err = emit_read_hwsp(watcher[0].rq, /* before */
10451081 rq->fence.seqno, hwsp,
10461082 &watcher[0].addr);
1047− mutex_unlock(&watcher[0].rq->context->timeline->mutex);
1083+ switch_tl_lock(watcher[0].rq, rq);
10481084 if (err) {
10491085 i915_request_add(rq);
1086+ intel_context_unpin(ce);
10501087 intel_context_put(ce);
10511088 goto out;
10521089 }
10531090
1054− mutex_lock(&watcher[1].rq->context->timeline->mutex);
1091+ switch_tl_lock(rq, watcher[1].rq);
10551092 err = intel_timeline_read_hwsp(rq, watcher[1].rq, &hwsp);
10561093 if (err == 0)
10571094 err = emit_read_hwsp(watcher[1].rq, /* after */
10581095 rq->fence.seqno, hwsp,
10591096 &watcher[1].addr);
1060− mutex_unlock(&watcher[1].rq->context->timeline->mutex);
1097+ switch_tl_lock(watcher[1].rq, rq);
10611098 if (err) {
10621099 i915_request_add(rq);
1100+ intel_context_unpin(ce);
10631101 intel_context_put(ce);
10641102 goto out;
10651103 }
@@ -1068,6 +1106,7 @@ static int live_hwsp_read(void *arg) · 2 unmodified lines
10681106 i915_request_add(rq);
10691107
10701108 rq = wrap_timeline(rq);
1109+ intel_context_unpin(ce);
10711110 intel_context_put(ce);
10721111 if (IS_ERR(rq)) {
10731112 err = PTR_ERR(rq);
@@ -1107,8 +1146,8 @@ static int live_hwsp_read(void *arg) · 33 unmodified lines
11071146 3 * watcher[1].rq->ring->size)
11081147 break;
11091148
1110− } while (!__igt_timeout(end_time, NULL));
1111− WRITE_ONCE(*(u32 *)tl->hwsp_seqno, 0xdeadbeef);
1149+ } while (!__igt_timeout(end_time, NULL) &&
1150+ count < (PAGE_SIZE / TIMELINE_SEQNO_BYTES - 1) / 2);
11121151
11131152 pr_info("%s: simulated %lu wraps\n", engine->name, count);
11141153 err = check_watcher(&watcher[1], "after", cmp_gte);
@@ -1153,9 +1192,7 @@ static int live_hwsp_rollover_kernel(void *arg) · 38 unmodified lines
11531192 }
11541193
11551194 GEM_BUG_ON(i915_active_fence_isset(&tl->last_request));
1156− tl->seqno = 0;
1157− timeline_rollback(tl);
1158− timeline_rollback(tl);
1195+ tl->seqno = -2u;
11591196 WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);
11601197
11611198 for (i = 0; i < ARRAY_SIZE(rq); i++) {
@@ -1235,11 +1272,14 @@ static int live_hwsp_rollover_user(void *arg) · 73 unmodified lines
12351272 goto out;
12361273
12371274 tl = ce->timeline;
1238− if (!tl->has_initial_breadcrumb || !tl->hwsp_cacheline)
1275+ if (!tl->has_initial_breadcrumb)
1276+ goto out;
1277+
1278+ err = intel_context_pin(ce);
1279+ if (err)
12391280 goto out;
12401281
1241− timeline_rollback(tl);
1242− timeline_rollback(tl);
1282+ tl->seqno = -4u;
12431283 WRITE_ONCE(*(u32 *)tl->hwsp_seqno, tl->seqno);
12441284
12451285 for (i = 0; i < ARRAY_SIZE(rq); i++) {
@@ -1248,7 +1288,7 @@ static int live_hwsp_rollover_user(void *arg) · 2 unmodified lines
12481288 this = intel_context_create_request(ce);
12491289 if (IS_ERR(this)) {
12501290 err = PTR_ERR(this);
1251− goto out;
1291+ goto out_unpin;
12521292 }
12531293
12541294 pr_debug("%s: create fence.seqnp:%d\n",
@@ -1267,17 +1307,18 @@ static int live_hwsp_rollover_user(void *arg) · 12 unmodified lines
12671307 if (i915_request_wait(rq[2], 0, HZ / 5) < 0) {
12681308 pr_err("Wait for timeline wrap timed out!\n");
12691309 err = -EIO;
1270− goto out;
1310+ goto out_unpin;
12711311 }
12721312
12731313 for (i = 0; i < ARRAY_SIZE(rq); i++) {
12741314 if (!i915_request_completed(rq[i])) {
12751315 pr_err("Pre-wrap request not completed!\n");
12761316 err = -EINVAL;
1277− goto out;
1317+ goto out_unpin;
12781318 }
12791319 }
1280−
1320+out_unpin:
1321+ intel_context_unpin(ce);
12811322 out:
12821323 for (i = 0; i < ARRAY_SIZE(rq); i++)
12831324 i915_request_put(rq[i]);
@@ -1319,13 +1360,13 @@ static int live_hwsp_recycle(void *arg) · 35 unmodified lines
13191360 struct intel_timeline *tl;
13201361 struct i915_request *rq;
13211362
1322− tl = checked_intel_timeline_create(gt);
1363+ tl = intel_timeline_create(gt);
13231364 if (IS_ERR(tl)) {
13241365 err = PTR_ERR(tl);
13251366 break;
13261367 }
13271368
1328− rq = tl_write(tl, engine, count);
1369+ rq = checked_tl_write(tl, engine, count);
13291370 if (IS_ERR(rq)) {
13301371 intel_timeline_put(tl);
13311372 err = PTR_ERR(rq);
drivers/gpu/drm/i915/gt/selftest_workarounds.c+52−30
@@ -112,7 +112,7 @@ read_nonprivs(struct intel_context *ce) · 111 unmodified lines
112112
113113 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
114114
115− cs = i915_gem_object_pin_map(result, I915_MAP_WB);
115+ cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB);
116116 if (IS_ERR(cs)) {
117117 err = PTR_ERR(cs);
118118 goto err_obj;
@@ -218,7 +218,7 @@ static int check_whitelist(struct intel_context *ce) · 99 unmodified lines
218218 i915_gem_object_lock(results, NULL);
219219 intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */
220220 err = i915_gem_object_set_to_cpu_domain(results, false);
221− i915_gem_object_unlock(results);
221+
222222 if (intel_gt_is_wedged(engine->gt))
223223 err = -EIO;
224224 if (err)
@@ -246,6 +246,7 @@ static int check_whitelist(struct intel_context *ce) · 21 unmodified lines
246246
247247 i915_gem_object_unpin_map(results);
248248 out_put:
249+ i915_gem_object_unlock(results);
249250 i915_gem_object_put(results);
250251 return err;
251252 }
@@ -490,7 +491,7 @@ static int check_dirty_whitelist(struct intel_context *ce) · 238 unmodified lines
490491 u32 *cs, *results;
491492
492493 sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);
493− scratch = __vm_create_scratch_for_read(ce->vm, sz);
494+ scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz);
494495 if (IS_ERR(scratch))
495496 return PTR_ERR(scratch);
496497
@@ -502,6 +503,7 @@ static int check_dirty_whitelist(struct intel_context *ce) · 5 unmodified lines
502503
503504 for (i = 0; i < engine->whitelist.count; i++) {
504505 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
506+ struct i915_gem_ww_ctx ww;
505507 u64 addr = scratch->node.start;
506508 struct i915_request *rq;
507509 u32 srm, lrm, rsvd;
@@ -517,6 +519,29 @@ static int check_dirty_whitelist(struct intel_context *ce) · 9 unmodified lines
517519
518520 ro_reg = ro_register(reg);
519521
522+ i915_gem_ww_ctx_init(&ww, false);
523+retry:
524+ cs = NULL;
525+ err = i915_gem_object_lock(scratch->obj, &ww);
526+ if (!err)
527+ err = i915_gem_object_lock(batch->obj, &ww);
528+ if (!err)
529+ err = intel_context_pin_ww(ce, &ww);
530+ if (err)
531+ goto out;
532+
533+ cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
534+ if (IS_ERR(cs)) {
535+ err = PTR_ERR(cs);
536+ goto out_ctx;
537+ }
538+
539+ results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
540+ if (IS_ERR(results)) {
541+ err = PTR_ERR(results);
542+ goto out_unmap_batch;
543+ }
544+
520545 /* Clear non priv flags */
521546 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
522547
@@ -528,12 +553,6 @@ static int check_dirty_whitelist(struct intel_context *ce) · 5 unmodified lines
528553 pr_debug("%s: Writing garbage to %x\n",
529554 engine->name, reg);
530555
531− cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
532− if (IS_ERR(cs)) {
533− err = PTR_ERR(cs);
534− goto out_batch;
535− }
536−
537556 /* SRM original */
538557 *cs++ = srm;
539558 *cs++ = reg;
@@ -580,11 +599,12 @@ static int check_dirty_whitelist(struct intel_context *ce) · 40 unmodified lines
580599 i915_gem_object_flush_map(batch->obj);
581600 i915_gem_object_unpin_map(batch->obj);
582601 intel_gt_chipset_flush(engine->gt);
602+ cs = NULL;
583603
584− rq = intel_context_create_request(ce);
604+ rq = i915_request_create(ce);
585605 if (IS_ERR(rq)) {
586606 err = PTR_ERR(rq);
587− goto out_batch;
607+ goto out_unmap_scratch;
588608 }
589609
590610 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
@@ -593,20 +613,16 @@ static int check_dirty_whitelist(struct intel_context *ce) · 2 unmodified lines
593613 goto err_request;
594614 }
595615
596− i915_vma_lock(batch);
597616 err = i915_request_await_object(rq, batch->obj, false);
598617 if (err == 0)
599618 err = i915_vma_move_to_active(batch, rq, 0);
600− i915_vma_unlock(batch);
601619 if (err)
602620 goto err_request;
603621
604− i915_vma_lock(scratch);
605622 err = i915_request_await_object(rq, scratch->obj, true);
606623 if (err == 0)
607624 err = i915_vma_move_to_active(scratch, rq,
608625 EXEC_OBJECT_WRITE);
609− i915_vma_unlock(scratch);
610626 if (err)
611627 goto err_request;
612628
@@ -622,13 +638,7 @@ err_request: · 9 unmodified lines
622638 pr_err("%s: Futzing %x timedout; cancelling test\n",
623639 engine->name, reg);
624640 intel_gt_set_wedged(engine->gt);
625− goto out_batch;
626− }
627−
628− results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
629− if (IS_ERR(results)) {
630− err = PTR_ERR(results);
631− goto out_batch;
641+ goto out_unmap_scratch;
632642 }
633643
634644 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
@@ -639,7 +649,7 @@ err_request: · 4 unmodified lines
639649 pr_err("%s: Unable to write to whitelisted register %x\n",
640650 engine->name, reg);
641651 err = -EINVAL;
642− goto out_unpin;
652+ goto out_unmap_scratch;
643653 }
644654 } else {
645655 rsvd = 0;
@@ -705,15 +715,27 @@ err_request: · 59 unmodified lines
705715
706716 err = -EINVAL;
707717 }
708−out_unpin:
718+out_unmap_scratch:
709719 i915_gem_object_unpin_map(scratch->obj);
720+out_unmap_batch:
721+ if (cs)
722+ i915_gem_object_unpin_map(batch->obj);
723+out_ctx:
724+ intel_context_unpin(ce);
725+out:
726+ if (err == -EDEADLK) {
727+ err = i915_gem_ww_ctx_backoff(&ww);
728+ if (!err)
729+ goto retry;
730+ }
731+ i915_gem_ww_ctx_fini(&ww);
710732 if (err)
711733 break;
712734 }
713735
714736 if (igt_flush_test(engine->i915))
715737 err = -EIO;
716−out_batch:
738+
717739 i915_vma_unpin_and_release(&batch, 0);
718740 out_scratch:
719741 i915_vma_unpin_and_release(&scratch, 0);
@@ -847,7 +869,7 @@ static int scrub_whitelisted_registers(struct intel_context *ce) · 127 unmodified lines
847869 if (IS_ERR(batch))
848870 return PTR_ERR(batch);
849871
850− cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
872+ cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
851873 if (IS_ERR(cs)) {
852874 err = PTR_ERR(cs);
853875 goto err_batch;
@@ -982,11 +1004,11 @@ check_whitelisted_registers(struct intel_engine_cs *engine, · 128 unmodified lines
9821004 u32 *a, *b;
9831005 int i, err;
9841006
985− a = i915_gem_object_pin_map(A->obj, I915_MAP_WB);
1007+ a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB);
9861008 if (IS_ERR(a))
9871009 return PTR_ERR(a);
9881010
989− b = i915_gem_object_pin_map(B->obj, I915_MAP_WB);
1011+ b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB);
9901012 if (IS_ERR(b)) {
9911013 err = PTR_ERR(b);
9921014 goto err_a;
@@ -1030,14 +1052,14 @@ static int live_isolated_whitelist(void *arg) · 37 unmodified lines
10301052
10311053 for (i = 0; i < ARRAY_SIZE(client); i++) {
10321054 client[i].scratch[0] =
1033− __vm_create_scratch_for_read(gt->vm, 4096);
1055+ __vm_create_scratch_for_read_pinned(gt->vm, 4096);
10341056 if (IS_ERR(client[i].scratch[0])) {
10351057 err = PTR_ERR(client[i].scratch[0]);
10361058 goto err;
10371059 }
10381060
10391061 client[i].scratch[1] =
1040− __vm_create_scratch_for_read(gt->vm, 4096);
1062+ __vm_create_scratch_for_read_pinned(gt->vm, 4096);
10411063 if (IS_ERR(client[i].scratch[1])) {
10421064 err = PTR_ERR(client[i].scratch[1]);
10431065 i915_vma_unpin_and_release(&client[i].scratch[0], 0);
drivers/gpu/drm/i915/gt/shmem_utils.c+1−1
@@ -39,7 +39,7 @@ struct file *shmem_create_from_object(struct drm_i915_gem_object *obj) · 38 unmodified lines
3939 return file;
4040 }
4141
42− ptr = i915_gem_object_pin_map(obj, I915_MAP_WB);
42+ ptr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
4343 if (IS_ERR(ptr))
4444 return ERR_CAST(ptr);
4545
drivers/gpu/drm/i915/gt/uc/intel_guc.c+1−1
@@ -682,7 +682,7 @@ int intel_guc_allocate_and_map_vma(struct intel_guc *guc, u32 size, · 681 unmodified lines
682682 if (IS_ERR(vma))
683683 return PTR_ERR(vma);
684684
685− vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
685+ vaddr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WB);
686686 if (IS_ERR(vaddr)) {
687687 i915_vma_unpin_and_release(&vma, 0);
688688 return PTR_ERR(vaddr);
drivers/gpu/drm/i915/gt/uc/intel_guc_log.c+2−2
@@ -335,7 +335,7 @@ static int guc_log_map(struct intel_guc_log *log) · 334 unmodified lines
335335 * buffer pages, so that we can directly get the data
336336 * (up-to-date) from memory.
337337 */
338− vaddr = i915_gem_object_pin_map(log->vma->obj, I915_MAP_WC);
338+ vaddr = i915_gem_object_pin_map_unlocked(log->vma->obj, I915_MAP_WC);
339339 if (IS_ERR(vaddr))
340340 return PTR_ERR(vaddr);
341341
@@ -744,7 +744,7 @@ int intel_guc_log_dump(struct intel_guc_log *log, struct drm_printer *p, · 402 unmodified lines
744744 if (!obj)
745745 return 0;
746746
747− map = i915_gem_object_pin_map(obj, I915_MAP_WC);
747+ map = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
748748 if (IS_ERR(map)) {
749749 DRM_DEBUG("Failed to pin object\n");
750750 drm_puts(p, "(log data unaccessible)\n");
drivers/gpu/drm/i915/gt/uc/intel_huc.c+1−1
@@ -82,7 +82,7 @@ static int intel_huc_rsa_data_create(struct intel_huc *huc) · 81 unmodified lines
8282 if (IS_ERR(vma))
8383 return PTR_ERR(vma);
8484
85− vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
85+ vaddr = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WB);
8686 if (IS_ERR(vaddr)) {
8787 i915_vma_unpin_and_release(&vma, 0);
8888 return PTR_ERR(vaddr);
drivers/gpu/drm/i915/gt/uc/intel_uc_fw.c+1−1
@@ -539,7 +539,7 @@ int intel_uc_fw_init(struct intel_uc_fw *uc_fw) · 538 unmodified lines
539539 if (!intel_uc_fw_is_available(uc_fw))
540540 return -ENOEXEC;
541541
542− err = i915_gem_object_pin_pages(uc_fw->obj);
542+ err = i915_gem_object_pin_pages_unlocked(uc_fw->obj);
543543 if (err) {
544544 DRM_DEBUG_DRIVER("%s fw pin-pages err=%d\n",
545545 intel_uc_fw_type_repr(uc_fw->type), err);
drivers/gpu/drm/i915/gvt/dmabuf.c+1−1
@@ -218,7 +218,7 @@ static struct drm_i915_gem_object *vgpu_create_gem(struct drm_device *dev, · 217 unmodified lines
218218
219219 drm_gem_private_object_init(dev, &obj->base,
220220 roundup(info->size, PAGE_SIZE));
221− i915_gem_object_init(obj, &intel_vgpu_gem_ops, &lock_class);
221+ i915_gem_object_init(obj, &intel_vgpu_gem_ops, &lock_class, 0);
222222 i915_gem_object_set_readonly(obj);
223223
224224 obj->read_domains = I915_GEM_DOMAIN_GTT;
drivers/gpu/drm/i915/i915_active.c+10−10
@@ -293,18 +293,13 @@ static struct active_node *__active_lookup(struct i915_active *ref, u64 idx) · 292 unmodified lines
293293 static struct i915_active_fence *
294294 active_instance(struct i915_active *ref, u64 idx)
295295 {
296− struct active_node *node, *prealloc;
296+ struct active_node *node;
297297 struct rb_node **p, *parent;
298298
299299 node = __active_lookup(ref, idx);
300300 if (likely(node))
301301 return &node->base;
302302
303− /* Preallocate a replacement, just in case */
304− prealloc = kmem_cache_alloc(global.slab_cache, GFP_KERNEL);
305− if (!prealloc)
306− return NULL;
307−
308303 spin_lock_irq(&ref->tree_lock);
309304 GEM_BUG_ON(i915_active_is_idle(ref));
310305
@@ -314,10 +309,8 @@ active_instance(struct i915_active *ref, u64 idx) · 3 unmodified lines
314309 parent = *p;
315310
316311 node = rb_entry(parent, struct active_node, node);
317− if (node->timeline == idx) {
318− kmem_cache_free(global.slab_cache, prealloc);
312+ if (node->timeline == idx)
319313 goto out;
320− }
321314
322315 if (node->timeline < idx)
323316 p = &parent->rb_right;
@@ -325,7 +318,14 @@ active_instance(struct i915_active *ref, u64 idx) · 1 unmodified line
325318 p = &parent->rb_left;
326319 }
327320
328− node = prealloc;
321+ /*
322+ * XXX: We should preallocate this before i915_active_ref() is ever
323+ * called, but we cannot call into fs_reclaim() anyway, so use GFP_ATOMIC.
324+ */
325+ node = kmem_cache_alloc(global.slab_cache, GFP_ATOMIC);
326+ if (!node)
327+ goto out;
328+
329329 __i915_active_fence_init(&node->base, NULL, node_retire);
330330 node->ref = ref;
331331 node->timeline = idx;
drivers/gpu/drm/i915/i915_cmd_parser.c+43−61
@@ -1144,38 +1144,20 @@ find_reg(const struct intel_engine_cs *engine, u32 addr) · 1143 unmodified lines
11441144 /* Returns a vmap'd pointer to dst_obj, which the caller must unmap */
11451145 static u32 *copy_batch(struct drm_i915_gem_object *dst_obj,
11461146 struct drm_i915_gem_object *src_obj,
1147− unsigned long offset, unsigned long length)
1147+ unsigned long offset, unsigned long length,
1148+ void *dst, const void *src)
11481149 {
1149− bool needs_clflush;
1150− void *dst, *src;
1151− int ret;
1152−
1153− dst = i915_gem_object_pin_map(dst_obj, I915_MAP_WB);
1154− if (IS_ERR(dst))
1155− return dst;
1156−
1157− ret = i915_gem_object_pin_pages(src_obj);
1158− if (ret) {
1159− i915_gem_object_unpin_map(dst_obj);
1160− return ERR_PTR(ret);
1161− }
1162−
1163− needs_clflush =
1150+ bool needs_clflush =
11641151 !(src_obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ);
11651152
1166− src = ERR_PTR(-ENODEV);
1167− if (needs_clflush && i915_has_memcpy_from_wc()) {
1168− src = i915_gem_object_pin_map(src_obj, I915_MAP_WC);
1169− if (!IS_ERR(src)) {
1170− i915_unaligned_memcpy_from_wc(dst,
1171− src + offset,
1172− length);
1173− i915_gem_object_unpin_map(src_obj);
1174− }
1175− }
1176− if (IS_ERR(src)) {
1177− unsigned long x, n, remain;
1153+ if (src) {
1154+ GEM_BUG_ON(!needs_clflush);
1155+ i915_unaligned_memcpy_from_wc(dst, src + offset, length);
1156+ } else {
1157+ struct scatterlist *sg;
11781158 void *ptr;
1159+ unsigned int x, sg_ofs;
1160+ unsigned long remain;
11791161
11801162 /*
11811163 * We can avoid clflushing partial cachelines before the write
@@ -1192,23 +1174,31 @@ static u32 *copy_batch(struct drm_i915_gem_object *dst_obj, · 10 unmodified lines
11921174
11931175 ptr = dst;
11941176 x = offset_in_page(offset);
1195− for (n = offset >> PAGE_SHIFT; remain; n++) {
1196− int len = min(remain, PAGE_SIZE - x);
1197−
1198− src = kmap_atomic(i915_gem_object_get_page(src_obj, n));
1199− if (needs_clflush)
1200− drm_clflush_virt_range(src + x, len);
1201− memcpy(ptr, src + x, len);
1202− kunmap_atomic(src);
1203−
1204− ptr += len;
1205− remain -= len;
1206− x = 0;
1177+ sg = i915_gem_object_get_sg(src_obj, offset >> PAGE_SHIFT, &sg_ofs, false);
1178+
1179+ while (remain) {
1180+ unsigned long sg_max = sg->length >> PAGE_SHIFT;
1181+
1182+ for (; remain && sg_ofs < sg_max; sg_ofs++) {
1183+ unsigned long len = min(remain, PAGE_SIZE - x);
1184+ void *map;
1185+
1186+ map = kmap_atomic(nth_page(sg_page(sg), sg_ofs));
1187+ if (needs_clflush)
1188+ drm_clflush_virt_range(map + x, len);
1189+ memcpy(ptr, map + x, len);
1190+ kunmap_atomic(map);
1191+
1192+ ptr += len;
1193+ remain -= len;
1194+ x = 0;
1195+ }
1196+
1197+ sg_ofs = 0;
1198+ sg = sg_next(sg);
12071199 }
12081200 }
12091201
1210− i915_gem_object_unpin_pages(src_obj);
1211−
12121202 memset32(dst + length, 0, (dst_obj->base.size - length) / sizeof(u32));
12131203
12141204 /* dst_obj is returned with vmap pinned */
@@ -1370,9 +1360,6 @@ static int check_bbstart(u32 *cmd, u32 offset, u32 length, · 155 unmodified lines
13701360 if (target_cmd_index == offset)
13711361 return 0;
13721362
1373− if (IS_ERR(jump_whitelist))
1374− return PTR_ERR(jump_whitelist);
1375−
13761363 if (!test_bit(target_cmd_index, jump_whitelist)) {
13771364 DRM_DEBUG("CMD: BB_START to 0x%llx not a previously executed cmd\n",
13781365 jump_target);
@@ -1382,10 +1369,14 @@ static int check_bbstart(u32 *cmd, u32 offset, u32 length, · 3 unmodified lines
13821369 return 0;
13831370 }
13841371
1385−static unsigned long *alloc_whitelist(u32 batch_length)
1372+unsigned long *intel_engine_cmd_parser_alloc_jump_whitelist(u32 batch_length,
1373+ bool trampoline)
13861374 {
13871375 unsigned long *jmp;
13881376
1377+ if (trampoline)
1378+ return NULL;
1379+
13891380 /*
13901381 * We expect batch_length to be less than 256KiB for known users,
13911382 * i.e. we need at most an 8KiB bitmap allocation which should be
@@ -1423,14 +1414,16 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, · 31 unmodified lines
14231414 unsigned long batch_offset,
14241415 unsigned long batch_length,
14251416 struct i915_vma *shadow,
1426− bool trampoline)
1417+ unsigned long *jump_whitelist,
1418+ void *shadow_map,
1419+ const void *batch_map)
14271420 {
14281421 u32 *cmd, *batch_end, offset = 0;
14291422 struct drm_i915_cmd_descriptor default_desc = noop_desc;
14301423 const struct drm_i915_cmd_descriptor *desc = &default_desc;
1431− unsigned long *jump_whitelist;
14321424 u64 batch_addr, shadow_addr;
14331425 int ret = 0;
1426+ bool trampoline = !jump_whitelist;
14341427
14351428 GEM_BUG_ON(!IS_ALIGNED(batch_offset, sizeof(*cmd)));
14361429 GEM_BUG_ON(!IS_ALIGNED(batch_length, sizeof(*cmd)));
@@ -1438,16 +1431,8 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, · 1 unmodified line
14381431 batch->size));
14391432 GEM_BUG_ON(!batch_length);
14401433
1441− cmd = copy_batch(shadow->obj, batch->obj, batch_offset, batch_length);
1442− if (IS_ERR(cmd)) {
1443− DRM_DEBUG("CMD: Failed to copy batch\n");
1444− return PTR_ERR(cmd);
1445− }
1446−
1447− jump_whitelist = NULL;
1448− if (!trampoline)
1449− /* Defer failure until attempted use */
1450− jump_whitelist = alloc_whitelist(batch_length);
1434+ cmd = copy_batch(shadow->obj, batch->obj, batch_offset, batch_length,
1435+ shadow_map, batch_map);
14511436
14521437 shadow_addr = gen8_canonical_addr(shadow->node.start);
14531438 batch_addr = gen8_canonical_addr(batch->node.start + batch_offset);
@@ -1548,9 +1533,6 @@ int intel_engine_cmd_parser(struct intel_engine_cs *engine, · 94 unmodified lines
15481533
15491534 i915_gem_object_flush_map(shadow->obj);
15501535
1551− if (!IS_ERR_OR_NULL(jump_whitelist))
1552− kfree(jump_whitelist);
1553− i915_gem_object_unpin_map(shadow->obj);
15541536 return ret;
15551537 }
15561538
drivers/gpu/drm/i915/i915_debugfs.c+2−2
@@ -904,10 +904,10 @@ i915_drop_caches_set(void *data, u64 val) · 903 unmodified lines
904904
905905 fs_reclaim_acquire(GFP_KERNEL);
906906 if (val & DROP_BOUND)
907− i915_gem_shrink(i915, LONG_MAX, NULL, I915_SHRINK_BOUND);
907+ i915_gem_shrink(NULL, i915, LONG_MAX, NULL, I915_SHRINK_BOUND);
908908
909909 if (val & DROP_UNBOUND)
910− i915_gem_shrink(i915, LONG_MAX, NULL, I915_SHRINK_UNBOUND);
910+ i915_gem_shrink(NULL, i915, LONG_MAX, NULL, I915_SHRINK_UNBOUND);
911911
912912 if (val & DROP_SHRINK_ALL)
913913 i915_gem_shrink_all(i915);
drivers/gpu/drm/i915/i915_drv.c+1−1
@@ -1691,7 +1691,7 @@ static const struct drm_ioctl_desc i915_ioctls[] = { · 1690 unmodified lines
16911691 DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
16921692 DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
16931693 DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1694− DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer_ioctl, DRM_AUTH),
1694+ DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, drm_invalid_op, DRM_AUTH),
16951695 DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2_WR, i915_gem_execbuffer2_ioctl, DRM_RENDER_ALLOW),
16961696 DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
16971697 DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
drivers/gpu/drm/i915/i915_drv.h+11−7
@@ -554,12 +554,13 @@ struct i915_gem_mm { · 553 unmodified lines
554554 struct notifier_block vmap_notifier;
555555 struct shrinker shrinker;
556556
557+#ifdef CONFIG_MMU_NOTIFIER
557558 /**
558− * Workqueue to fault in userptr pages, flushed by the execbuf
559− * when required but otherwise left to userspace to try again
560− * on EAGAIN.
559+ * notifier_lock for mmu notifiers, memory may not be allocated
560+ * while holding this lock.
561561 */
562− struct workqueue_struct *userptr_wq;
562+ spinlock_t notifier_lock;
563+#endif
563564
564565 /* shrinker accounting, also useful for userland debugging */
565566 u64 shrink_memory;
@@ -938,8 +939,6 @@ struct drm_i915_private { · 372 unmodified lines
938939 struct i915_ggtt ggtt; /* VM representing the global address space */
939940
940941 struct i915_gem_mm mm;
941− DECLARE_HASHTABLE(mm_structs, 7);
942− spinlock_t mm_lock;
943942
944943 /* Kernel Modesetting */
945944
@@ -1946,12 +1945,17 @@ const char *i915_cache_level_str(struct drm_i915_private *i915, int type); · 1000 unmodified lines
19461945 int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv);
19471946 int intel_engine_init_cmd_parser(struct intel_engine_cs *engine);
19481947 void intel_engine_cleanup_cmd_parser(struct intel_engine_cs *engine);
1948+unsigned long *intel_engine_cmd_parser_alloc_jump_whitelist(u32 batch_length,
1949+ bool trampoline);
1950+
19491951 int intel_engine_cmd_parser(struct intel_engine_cs *engine,
19501952 struct i915_vma *batch,
19511953 unsigned long batch_offset,
19521954 unsigned long batch_length,
19531955 struct i915_vma *shadow,
1954− bool trampoline);
1956+ unsigned long *jump_whitelist,
1957+ void *shadow_map,
1958+ const void *batch_map);
19551959 #define I915_CMD_PARSER_TRAMPOLINE_SIZE 8
19561960
19571961 /* intel_device_info.c */
drivers/gpu/drm/i915/i915_gem.c+141−117
@@ -204,7 +204,6 @@ i915_gem_shmem_pread(struct drm_i915_gem_object *obj, · 203 unmodified lines
204204 {
205205 unsigned int needs_clflush;
206206 unsigned int idx, offset;
207− struct dma_fence *fence;
208207 char __user *user_data;
209208 u64 remain;
210209 int ret;
@@ -213,19 +212,17 @@ i915_gem_shmem_pread(struct drm_i915_gem_object *obj, · 2 unmodified lines
213212 if (ret)
214213 return ret;
215214
215+ ret = i915_gem_object_pin_pages(obj);
216+ if (ret)
217+ goto err_unlock;
218+
216219 ret = i915_gem_object_prepare_read(obj, &needs_clflush);
217− if (ret) {
218− i915_gem_object_unlock(obj);
219− return ret;
220− }
220+ if (ret)
221+ goto err_unpin;
221222
222− fence = i915_gem_object_lock_fence(obj);
223223 i915_gem_object_finish_access(obj);
224224 i915_gem_object_unlock(obj);
225225
226− if (!fence)
227− return -ENOMEM;
228−
229226 remain = args->size;
230227 user_data = u64_to_user_ptr(args->data_ptr);
231228 offset = offset_in_page(args->offset);
@@ -243,7 +240,13 @@ i915_gem_shmem_pread(struct drm_i915_gem_object *obj, · 11 unmodified lines
243240 offset = 0;
244241 }
245242
246− i915_gem_object_unlock_fence(obj, fence);
243+ i915_gem_object_unpin_pages(obj);
244+ return ret;
245+
246+err_unpin:
247+ i915_gem_object_unpin_pages(obj);
248+err_unlock:
249+ i915_gem_object_unlock(obj);
247250 return ret;
248251 }
249252
@@ -271,52 +274,102 @@ gtt_user_read(struct io_mapping *mapping, · 21 unmodified lines
271274 return unwritten;
272275 }
273276
274−static int
275−i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
276− const struct drm_i915_gem_pread *args)
277+static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj,
278+ struct drm_mm_node *node,
279+ bool write)
277280 {
278281 struct drm_i915_private *i915 = to_i915(obj->base.dev);
279282 struct i915_ggtt *ggtt = &i915->ggtt;
280− intel_wakeref_t wakeref;
281− struct drm_mm_node node;
282− struct dma_fence *fence;
283− void __user *user_data;
284283 struct i915_vma *vma;
285− u64 remain, offset;
284+ struct i915_gem_ww_ctx ww;
286285 int ret;
287286
288− wakeref = intel_runtime_pm_get(&i915->runtime_pm);
287+ i915_gem_ww_ctx_init(&ww, true);
288+retry:
289289 vma = ERR_PTR(-ENODEV);
290+ ret = i915_gem_object_lock(obj, &ww);
291+ if (ret)
292+ goto err_ww;
293+
294+ ret = i915_gem_object_set_to_gtt_domain(obj, write);
295+ if (ret)
296+ goto err_ww;
297+
290298 if (!i915_gem_object_is_tiled(obj))
291− vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
292− PIN_MAPPABLE |
293− PIN_NONBLOCK /* NOWARN */ |
294− PIN_NOEVICT);
295− if (!IS_ERR(vma)) {
296− node.start = i915_ggtt_offset(vma);
297− node.flags = 0;
299+ vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
300+ PIN_MAPPABLE |
301+ PIN_NONBLOCK /* NOWARN */ |
302+ PIN_NOEVICT);
303+ if (vma == ERR_PTR(-EDEADLK)) {
304+ ret = -EDEADLK;
305+ goto err_ww;
306+ } else if (!IS_ERR(vma)) {
307+ node->start = i915_ggtt_offset(vma);
308+ node->flags = 0;
298309 } else {
299− ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
310+ ret = insert_mappable_node(ggtt, node, PAGE_SIZE);
300311 if (ret)
301− goto out_rpm;
302− GEM_BUG_ON(!drm_mm_node_allocated(&node));
312+ goto err_ww;
313+ GEM_BUG_ON(!drm_mm_node_allocated(node));
314+ vma = NULL;
303315 }
304316
305− ret = i915_gem_object_lock_interruptible(obj, NULL);
306− if (ret)
307− goto out_unpin;
308−
309− ret = i915_gem_object_set_to_gtt_domain(obj, false);
317+ ret = i915_gem_object_pin_pages(obj);
310318 if (ret) {
311− i915_gem_object_unlock(obj);
312− goto out_unpin;
319+ if (drm_mm_node_allocated(node)) {
320+ ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
321+ remove_mappable_node(ggtt, node);
322+ } else {
323+ i915_vma_unpin(vma);
324+ }
313325 }
314326
315− fence = i915_gem_object_lock_fence(obj);
316− i915_gem_object_unlock(obj);
317− if (!fence) {
318− ret = -ENOMEM;
319− goto out_unpin;
327+err_ww:
328+ if (ret == -EDEADLK) {
329+ ret = i915_gem_ww_ctx_backoff(&ww);
330+ if (!ret)
331+ goto retry;
332+ }
333+ i915_gem_ww_ctx_fini(&ww);
334+
335+ return ret ? ERR_PTR(ret) : vma;
336+}
337+
338+static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj,
339+ struct drm_mm_node *node,
340+ struct i915_vma *vma)
341+{
342+ struct drm_i915_private *i915 = to_i915(obj->base.dev);
343+ struct i915_ggtt *ggtt = &i915->ggtt;
344+
345+ i915_gem_object_unpin_pages(obj);
346+ if (drm_mm_node_allocated(node)) {
347+ ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
348+ remove_mappable_node(ggtt, node);
349+ } else {
350+ i915_vma_unpin(vma);
351+ }
352+}
353+
354+static int
355+i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
356+ const struct drm_i915_gem_pread *args)
357+{
358+ struct drm_i915_private *i915 = to_i915(obj->base.dev);
359+ struct i915_ggtt *ggtt = &i915->ggtt;
360+ intel_wakeref_t wakeref;
361+ struct drm_mm_node node;
362+ void __user *user_data;
363+ struct i915_vma *vma;
364+ u64 remain, offset;
365+ int ret = 0;
366+
367+ wakeref = intel_runtime_pm_get(&i915->runtime_pm);
368+
369+ vma = i915_gem_gtt_prepare(obj, &node, false);
370+ if (IS_ERR(vma)) {
371+ ret = PTR_ERR(vma);
372+ goto out_rpm;
320373 }
321374
322375 user_data = u64_to_user_ptr(args->data_ptr);
@@ -353,14 +406,7 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj, · 30 unmodified lines
353406 offset += page_length;
354407 }
355408
356− i915_gem_object_unlock_fence(obj, fence);
357−out_unpin:
358− if (drm_mm_node_allocated(&node)) {
359− ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
360− remove_mappable_node(ggtt, &node);
361− } else {
362− i915_vma_unpin(vma);
363− }
409+ i915_gem_gtt_cleanup(obj, &node, vma);
364410 out_rpm:
365411 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
366412 return ret;
@@ -378,10 +424,17 @@ int · 11 unmodified lines
378424 i915_gem_pread_ioctl(struct drm_device *dev, void *data,
379425 struct drm_file *file)
380426 {
427+ struct drm_i915_private *i915 = to_i915(dev);
381428 struct drm_i915_gem_pread *args = data;
382429 struct drm_i915_gem_object *obj;
383430 int ret;
384431
432+ /* PREAD is disallowed for all platforms after TGL-LP. This also
433+ * covers all platforms with local memory.
434+ */
435+ if (INTEL_GEN(i915) >= 12 && !IS_TIGERLAKE(i915))
436+ return -EOPNOTSUPP;
437+
385438 if (args->size == 0)
386439 return 0;
387440
@@ -400,6 +453,11 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, · 12 unmodified lines
400453 }
401454
402455 trace_i915_gem_object_pread(obj, args->offset, args->size);
456+ ret = -ENODEV;
457+ if (obj->ops->pread)
458+ ret = obj->ops->pread(obj, args);
459+ if (ret != -ENODEV)
460+ goto out;
403461
404462 ret = -ENODEV;
405463 if (obj->ops->pread)
@@ -413,15 +471,10 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data, · 7 unmodified lines
413471 if (ret)
414472 goto out;
415473
416− ret = i915_gem_object_pin_pages(obj);
417− if (ret)
418− goto out;
419−
420474 ret = i915_gem_shmem_pread(obj, args);
421475 if (ret == -EFAULT || ret == -ENODEV)
422476 ret = i915_gem_gtt_pread(obj, args);
423477
424− i915_gem_object_unpin_pages(obj);
425478 out:
426479 i915_gem_object_put(obj);
427480 return ret;
@@ -469,11 +522,10 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, · 41 unmodified lines
469522 struct intel_runtime_pm *rpm = &i915->runtime_pm;
470523 intel_wakeref_t wakeref;
471524 struct drm_mm_node node;
472− struct dma_fence *fence;
473525 struct i915_vma *vma;
474526 u64 remain, offset;
475527 void __user *user_data;
476− int ret;
528+ int ret = 0;
477529
478530 if (i915_gem_object_has_struct_page(obj)) {
479531 /*
@@ -491,37 +543,10 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, · 11 unmodified lines
491543 wakeref = intel_runtime_pm_get(rpm);
492544 }
493545
494− vma = ERR_PTR(-ENODEV);
495− if (!i915_gem_object_is_tiled(obj))
496− vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
497− PIN_MAPPABLE |
498− PIN_NONBLOCK /* NOWARN */ |
499− PIN_NOEVICT);
500− if (!IS_ERR(vma)) {
501− node.start = i915_ggtt_offset(vma);
502− node.flags = 0;
503− } else {
504− ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
505− if (ret)
506− goto out_rpm;
507− GEM_BUG_ON(!drm_mm_node_allocated(&node));
508− }
509−
510− ret = i915_gem_object_lock_interruptible(obj, NULL);
511− if (ret)
512− goto out_unpin;
513−
514− ret = i915_gem_object_set_to_gtt_domain(obj, true);
515− if (ret) {
516− i915_gem_object_unlock(obj);
517− goto out_unpin;
518− }
519−
520− fence = i915_gem_object_lock_fence(obj);
521− i915_gem_object_unlock(obj);
522− if (!fence) {
523− ret = -ENOMEM;
524− goto out_unpin;
546+ vma = i915_gem_gtt_prepare(obj, &node, true);
547+ if (IS_ERR(vma)) {
548+ ret = PTR_ERR(vma);
549+ goto out_rpm;
525550 }
526551
527552 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
@@ -570,14 +595,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, · 42 unmodified lines
570595 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
571596 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
572597
573− i915_gem_object_unlock_fence(obj, fence);
574−out_unpin:
575− if (drm_mm_node_allocated(&node)) {
576− ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
577− remove_mappable_node(ggtt, &node);
578− } else {
579− i915_vma_unpin(vma);
580− }
598+ i915_gem_gtt_cleanup(obj, &node, vma);
581599 out_rpm:
582600 intel_runtime_pm_put(rpm, wakeref);
583601 return ret;
@@ -617,7 +635,6 @@ i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, · 33 unmodified lines
617635 unsigned int partial_cacheline_write;
618636 unsigned int needs_clflush;
619637 unsigned int offset, idx;
620− struct dma_fence *fence;
621638 void __user *user_data;
622639 u64 remain;
623640 int ret;
@@ -626,19 +643,17 @@ i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, · 2 unmodified lines
626643 if (ret)
627644 return ret;
628645
646+ ret = i915_gem_object_pin_pages(obj);
647+ if (ret)
648+ goto err_unlock;
649+
629650 ret = i915_gem_object_prepare_write(obj, &needs_clflush);
630− if (ret) {
631− i915_gem_object_unlock(obj);
632− return ret;
633− }
651+ if (ret)
652+ goto err_unpin;
634653
635− fence = i915_gem_object_lock_fence(obj);
636654 i915_gem_object_finish_access(obj);
637655 i915_gem_object_unlock(obj);
638656
639− if (!fence)
640− return -ENOMEM;
641−
642657 /* If we don't overwrite a cacheline completely we need to be
643658 * careful to have up-to-date data by first clflushing. Don't
644659 * overcomplicate things and flush the entire patch.
@@ -666,8 +681,14 @@ i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj, · 21 unmodified lines
666681 }
667682
668683 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
669− i915_gem_object_unlock_fence(obj, fence);
670684
685+ i915_gem_object_unpin_pages(obj);
686+ return ret;
687+
688+err_unpin:
689+ i915_gem_object_unpin_pages(obj);
690+err_unlock:
691+ i915_gem_object_unlock(obj);
671692 return ret;
672693 }
673694
@@ -683,10 +704,17 @@ int · 9 unmodified lines
683704 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
684705 struct drm_file *file)
685706 {
707+ struct drm_i915_private *i915 = to_i915(dev);
686708 struct drm_i915_gem_pwrite *args = data;
687709 struct drm_i915_gem_object *obj;
688710 int ret;
689711
712+ /* PWRITE is disallowed for all platforms after TGL-LP. This also
713+ * covers all platforms with local memory.
714+ */
715+ if (INTEL_GEN(i915) >= 12 && !IS_TIGERLAKE(i915))
716+ return -EOPNOTSUPP;
717+
690718 if (args->size == 0)
691719 return 0;
692720
@@ -724,10 +752,6 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, · 31 unmodified lines
724752 if (ret)
725753 goto err;
726754
727− ret = i915_gem_object_pin_pages(obj);
728− if (ret)
729− goto err;
730−
731755 ret = -EFAULT;
732756 /* We can only do the GTT pwrite on untiled buffers, as otherwise
733757 * it would end up going through the fenced access, and we'll get
@@ -748,7 +772,6 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, · 14 unmodified lines
748772 ret = i915_gem_shmem_pwrite(obj, args);
749773 }
750774
751− i915_gem_object_unpin_pages(obj);
752775 err:
753776 i915_gem_object_put(obj);
754777 return ret;
@@ -909,7 +932,11 @@ new_vma: · 154 unmodified lines
909932 return ERR_PTR(ret);
910933 }
911934
912− ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);
935+ if (ww)
936+ ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);
937+ else
938+ ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
939+
913940 if (ret)
914941 return ERR_PTR(ret);
915942
@@ -949,7 +976,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, · 33 unmodified lines
949976 if (!obj)
950977 return -ENOENT;
951978
952− err = mutex_lock_interruptible(&obj->mm.lock);
979+ err = i915_gem_object_lock_interruptible(obj, NULL);
953980 if (err)
954981 goto out;
955982
@@ -995,8 +1022,8 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, · 39 unmodified lines
9951022 i915_gem_object_truncate(obj);
9961023
9971024 args->retained = obj->mm.madv != __I915_MADV_PURGED;
998− mutex_unlock(&obj->mm.lock);
9991025
1026+ i915_gem_object_unlock(obj);
10001027 out:
10011028 i915_gem_object_put(obj);
10021029 return err;
@@ -1050,10 +1077,8 @@ int i915_gem_init(struct drm_i915_private *dev_priv) · 47 unmodified lines
10501077 err_unlock:
10511078 i915_gem_drain_workqueue(dev_priv);
10521079
1053− if (ret != -EIO) {
1080+ if (ret != -EIO)
10541081 intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
1055− i915_gem_cleanup_userptr(dev_priv);
1056− }
10571082
10581083 if (ret == -EIO) {
10591084 /*
@@ -1110,7 +1135,6 @@ void i915_gem_driver_release(struct drm_i915_private *dev_priv) · 50 unmodified lines
11101135 intel_wa_list_free(&dev_priv->gt_wa_list);
11111136
11121137 intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
1113− i915_gem_cleanup_userptr(dev_priv);
11141138
11151139 i915_gem_drain_freed_objects(dev_priv);
11161140
drivers/gpu/drm/i915/i915_gem_gtt.c+1−1
@@ -44,7 +44,7 @@ int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj, · 43 unmodified lines
4444 * the DMA remapper, i915_gem_shrink will return 0.
4545 */
4646 GEM_BUG_ON(obj->mm.pages == pages);
47− } while (i915_gem_shrink(to_i915(obj->base.dev),
47+ } while (i915_gem_shrink(NULL, to_i915(obj->base.dev),
4848 obj->base.size >> PAGE_SHIFT, NULL,
4949 I915_SHRINK_BOUND |
5050 I915_SHRINK_UNBOUND));
drivers/gpu/drm/i915/i915_memcpy.c+1−1
@@ -135,7 +135,7 @@ bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len) · 134 unmodified lines
135135 * accepts that its arguments may not be aligned, but are valid for the
136136 * potential 16-byte read past the end.
137137 */
138−void i915_unaligned_memcpy_from_wc(void *dst, void *src, unsigned long len)
138+void i915_unaligned_memcpy_from_wc(void *dst, const void *src, unsigned long len)
139139 {
140140 unsigned long addr;
141141
drivers/gpu/drm/i915/i915_memcpy.h+1−1
@@ -13,7 +13,7 @@ struct drm_i915_private; · 12 unmodified lines
1313 void i915_memcpy_init_early(struct drm_i915_private *i915);
1414
1515 bool i915_memcpy_from_wc(void *dst, const void *src, unsigned long len);
16−void i915_unaligned_memcpy_from_wc(void *dst, void *src, unsigned long len);
16+void i915_unaligned_memcpy_from_wc(void *dst, const void *src, unsigned long len);
1717
1818 /* The movntdqa instructions used for memcpy-from-wc require 16-byte alignment,
1919 * as well as SSE4.1 support. i915_memcpy_from_wc() will report if it cannot
drivers/gpu/drm/i915/i915_params.c+5−0
@@ -197,6 +197,11 @@ i915_param_named_unsafe(fake_lmem_start, ulong, 0400, · 196 unmodified lines
197197 "Fake LMEM start offset (default: 0)");
198198 #endif
199199
200+#if CONFIG_DRM_I915_REQUEST_TIMEOUT
201+i915_param_named_unsafe(request_timeout_ms, uint, 0600,
202+ "Default request/fence/batch buffer expiration timeout.");
203+#endif
204+
200205 static __always_inline void _print_param(struct drm_printer *p,
201206 const char *name,
202207 const char *type,
drivers/gpu/drm/i915/i915_params.h+1−0
@@ -72,6 +72,7 @@ struct drm_printer; · 71 unmodified lines
7272 param(int, enable_dpcd_backlight, -1, 0600) \
7373 param(char *, force_probe, CONFIG_DRM_I915_FORCE_PROBE, 0400) \
7474 param(unsigned long, fake_lmem_start, 0, 0400) \
75+ param(unsigned int, request_timeout_ms, CONFIG_DRM_I915_REQUEST_TIMEOUT, 0600) \
7576 /* leave bools at the end to not create holes */ \
7677 param(bool, enable_hangcheck, true, 0600) \
7778 param(bool, load_detect_test, false, 0600) \
drivers/gpu/drm/i915/i915_perf.c+43−13
@@ -1573,7 +1573,7 @@ static int alloc_oa_buffer(struct i915_perf_stream *stream) · 1572 unmodified lines
15731573 stream->oa_buffer.vma = vma;
15741574
15751575 stream->oa_buffer.vaddr =
1576− i915_gem_object_pin_map(bo, I915_MAP_WB);
1576+ i915_gem_object_pin_map_unlocked(bo, I915_MAP_WB);
15771577 if (IS_ERR(stream->oa_buffer.vaddr)) {
15781578 ret = PTR_ERR(stream->oa_buffer.vaddr);
15791579 goto err_unpin;
@@ -1627,6 +1627,7 @@ static int alloc_noa_wait(struct i915_perf_stream *stream) · 47 unmodified lines
16271627 const u32 base = stream->engine->mmio_base;
16281628 #define CS_GPR(x) GEN8_RING_CS_GPR(base, x)
16291629 u32 *batch, *ts0, *cs, *jump;
1630+ struct i915_gem_ww_ctx ww;
16301631 int ret, i;
16311632 enum {
16321633 START_TS,
@@ -1644,15 +1645,21 @@ static int alloc_noa_wait(struct i915_perf_stream *stream) · 11 unmodified lines
16441645 return PTR_ERR(bo);
16451646 }
16461647
1648+ i915_gem_ww_ctx_init(&ww, true);
1649+retry:
1650+ ret = i915_gem_object_lock(bo, &ww);
1651+ if (ret)
1652+ goto out_ww;
1653+
16471654 /*
16481655 * We pin in GGTT because we jump into this buffer now because
16491656 * multiple OA config BOs will have a jump to this address and it
16501657 * needs to be fixed during the lifetime of the i915/perf stream.
16511658 */
1652− vma = i915_gem_object_ggtt_pin(bo, NULL, 0, 0, PIN_HIGH);
1659+ vma = i915_gem_object_ggtt_pin_ww(bo, &ww, NULL, 0, 0, PIN_HIGH);
16531660 if (IS_ERR(vma)) {
16541661 ret = PTR_ERR(vma);
1655− goto err_unref;
1662+ goto out_ww;
16561663 }
16571664
16581665 batch = cs = i915_gem_object_pin_map(bo, I915_MAP_WB);
@@ -1786,12 +1793,19 @@ static int alloc_noa_wait(struct i915_perf_stream *stream) · 127 unmodified lines
17861793 __i915_gem_object_release_map(bo);
17871794
17881795 stream->noa_wait = vma;
1789− return 0;
1796+ goto out_ww;
17901797
17911798 err_unpin:
17921799 i915_vma_unpin_and_release(&vma, 0);
1793−err_unref:
1794− i915_gem_object_put(bo);
1800+out_ww:
1801+ if (ret == -EDEADLK) {
1802+ ret = i915_gem_ww_ctx_backoff(&ww);
1803+ if (!ret)
1804+ goto retry;
1805+ }
1806+ i915_gem_ww_ctx_fini(&ww);
1807+ if (ret)
1808+ i915_gem_object_put(bo);
17951809 return ret;
17961810 }
17971811
@@ -1834,6 +1848,7 @@ alloc_oa_config_buffer(struct i915_perf_stream *stream, · 36 unmodified lines
18341848 {
18351849 struct drm_i915_gem_object *obj;
18361850 struct i915_oa_config_bo *oa_bo;
1851+ struct i915_gem_ww_ctx ww;
18371852 size_t config_length = 0;
18381853 u32 *cs;
18391854 int err;
@@ -1854,10 +1869,16 @@ alloc_oa_config_buffer(struct i915_perf_stream *stream, · 14 unmodified lines
18541869 goto err_free;
18551870 }
18561871
1872+ i915_gem_ww_ctx_init(&ww, true);
1873+retry:
1874+ err = i915_gem_object_lock(obj, &ww);
1875+ if (err)
1876+ goto out_ww;
1877+
18571878 cs = i915_gem_object_pin_map(obj, I915_MAP_WB);
18581879 if (IS_ERR(cs)) {
18591880 err = PTR_ERR(cs);
1860− goto err_oa_bo;
1881+ goto out_ww;
18611882 }
18621883
18631884 cs = write_cs_mi_lri(cs,
@@ -1885,19 +1906,28 @@ alloc_oa_config_buffer(struct i915_perf_stream *stream, · 21 unmodified lines
18851906 NULL);
18861907 if (IS_ERR(oa_bo->vma)) {
18871908 err = PTR_ERR(oa_bo->vma);
1888− goto err_oa_bo;
1909+ goto out_ww;
18891910 }
18901911
18911912 oa_bo->oa_config = i915_oa_config_get(oa_config);
18921913 llist_add(&oa_bo->node, &stream->oa_config_bos);
18931914
1894− return oa_bo;
1915+out_ww:
1916+ if (err == -EDEADLK) {
1917+ err = i915_gem_ww_ctx_backoff(&ww);
1918+ if (!err)
1919+ goto retry;
1920+ }
1921+ i915_gem_ww_ctx_fini(&ww);
18951922
1896−err_oa_bo:
1897− i915_gem_object_put(obj);
1923+ if (err)
1924+ i915_gem_object_put(obj);
18981925 err_free:
1899− kfree(oa_bo);
1900− return ERR_PTR(err);
1926+ if (err) {
1927+ kfree(oa_bo);
1928+ return ERR_PTR(err);
1929+ }
1930+ return oa_bo;
19011931 }
19021932
19031933 static struct i915_vma *
drivers/gpu/drm/i915/i915_request.c+126−7
@@ -33,7 +33,10 @@ · 32 unmodified lines
3333 #include "gem/i915_gem_context.h"
3434 #include "gt/intel_breadcrumbs.h"
3535 #include "gt/intel_context.h"
36+#include "gt/intel_engine.h"
37+#include "gt/intel_engine_heartbeat.h"
3638 #include "gt/intel_gpu_commands.h"
39+#include "gt/intel_reset.h"
3740 #include "gt/intel_ring.h"
3841 #include "gt/intel_rps.h"
3942
@@ -244,6 +247,50 @@ static void __i915_request_fill(struct i915_request *rq, u8 val) · 204 unmodified lines
244247 memset(vaddr + head, val, rq->postfix - head);
245248 }
246249
250+/**
251+ * i915_request_active_engine
252+ * @rq: request to inspect
253+ * @active: pointer in which to return the active engine
254+ *
255+ * Fills the currently active engine to the @active pointer if the request
256+ * is active and still not completed.
257+ *
258+ * Returns true if request was active or false otherwise.
259+ */
260+bool
261+i915_request_active_engine(struct i915_request *rq,
262+ struct intel_engine_cs **active)
263+{
264+ struct intel_engine_cs *engine, *locked;
265+ bool ret = false;
266+
267+ /*
268+ * Serialise with __i915_request_submit() so that it sees
269+ * is-banned?, or we know the request is already inflight.
270+ *
271+ * Note that rq->engine is unstable, and so we double
272+ * check that we have acquired the lock on the final engine.
273+ */
274+ locked = READ_ONCE(rq->engine);
275+ spin_lock_irq(&locked->active.lock);
276+ while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) {
277+ spin_unlock(&locked->active.lock);
278+ locked = engine;
279+ spin_lock(&locked->active.lock);
280+ }
281+
282+ if (i915_request_is_active(rq)) {
283+ if (!__i915_request_is_complete(rq))
284+ *active = locked;
285+ ret = true;
286+ }
287+
288+ spin_unlock_irq(&locked->active.lock);
289+
290+ return ret;
291+}
292+
293+
247294 static void remove_from_engine(struct i915_request *rq)
248295 {
249296 struct intel_engine_cs *engine, *locked;
@@ -274,6 +321,53 @@ static void remove_from_engine(struct i915_request *rq) · 24 unmodified lines
274321 __notify_execute_cb_imm(rq);
275322 }
276323
324+static void __rq_init_watchdog(struct i915_request *rq)
325+{
326+ rq->watchdog.timer.function = NULL;
327+}
328+
329+static enum hrtimer_restart __rq_watchdog_expired(struct hrtimer *hrtimer)
330+{
331+ struct i915_request *rq =
332+ container_of(hrtimer, struct i915_request, watchdog.timer);
333+ struct intel_gt *gt = rq->engine->gt;
334+
335+ if (!i915_request_completed(rq)) {
336+ if (llist_add(&rq->watchdog.link, &gt->watchdog.list))
337+ schedule_work(&gt->watchdog.work);
338+ } else {
339+ i915_request_put(rq);
340+ }
341+
342+ return HRTIMER_NORESTART;
343+}
344+
345+static void __rq_arm_watchdog(struct i915_request *rq)
346+{
347+ struct i915_request_watchdog *wdg = &rq->watchdog;
348+ struct intel_context *ce = rq->context;
349+
350+ if (!ce->watchdog.timeout_us)
351+ return;
352+
353+ hrtimer_init(&wdg->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
354+ wdg->timer.function = __rq_watchdog_expired;
355+ hrtimer_start_range_ns(&wdg->timer,
356+ ns_to_ktime(ce->watchdog.timeout_us *
357+ NSEC_PER_USEC),
358+ NSEC_PER_MSEC,
359+ HRTIMER_MODE_REL);
360+ i915_request_get(rq);
361+}
362+
363+static void __rq_cancel_watchdog(struct i915_request *rq)
364+{
365+ struct i915_request_watchdog *wdg = &rq->watchdog;
366+
367+ if (wdg->timer.function && hrtimer_try_to_cancel(&wdg->timer) > 0)
368+ i915_request_put(rq);
369+}
370+
277371 bool i915_request_retire(struct i915_request *rq)
278372 {
279373 if (!__i915_request_is_complete(rq))
@@ -285,6 +379,8 @@ bool i915_request_retire(struct i915_request *rq) · 5 unmodified lines
285379 trace_i915_request_retire(rq);
286380 i915_request_mark_complete(rq);
287381
382+ __rq_cancel_watchdog(rq);
383+
288384 /*
289385 * We know the GPU must have read the request to have
290386 * sent us the seqno + interrupt, so use the position
@@ -498,20 +594,22 @@ void __i915_request_skip(struct i915_request *rq) · 207 unmodified lines
498594 rq->infix = rq->postfix;
499595 }
500596
501−void i915_request_set_error_once(struct i915_request *rq, int error)
597+bool i915_request_set_error_once(struct i915_request *rq, int error)
502598 {
503599 int old;
504600
505601 GEM_BUG_ON(!IS_ERR_VALUE((long)error));
506602
507603 if (i915_request_signaled(rq))
508− return;
604+ return false;
509605
510606 old = READ_ONCE(rq->fence.error);
511607 do {
512608 if (fatal_error(old))
513− return;
609+ return false;
514610 } while (!try_cmpxchg(&rq->fence.error, &old, error));
611+
612+ return true;
515613 }
516614
517615 void i915_request_mark_eio(struct i915_request *rq)
@@ -678,6 +776,28 @@ void i915_request_unsubmit(struct i915_request *request) · 160 unmodified lines
678776 spin_unlock_irqrestore(&engine->active.lock, flags);
679777 }
680778
779+static void __cancel_request(struct i915_request *rq)
780+{
781+ struct intel_engine_cs *engine = NULL;
782+
783+ i915_request_active_engine(rq, &engine);
784+
785+ if (engine && intel_engine_pulse(engine))
786+ intel_gt_handle_error(engine->gt, engine->mask, 0,
787+ "request cancellation by %s",
788+ current->comm);
789+}
790+
791+void i915_request_cancel(struct i915_request *rq, int error)
792+{
793+ if (!i915_request_set_error_once(rq, error))
794+ return;
795+
796+ set_bit(I915_FENCE_FLAG_SENTINEL, &rq->fence.flags);
797+
798+ __cancel_request(rq);
799+}
800+
681801 static int __i915_sw_fence_call
682802 submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state)
683803 {
@@ -690,6 +810,8 @@ submit_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) · 6 unmodified lines
690810
691811 if (unlikely(fence->error))
692812 i915_request_set_error_once(request, fence->error);
813+ else
814+ __rq_arm_watchdog(request);
693815
694816 /*
695817 * We need to serialize use of the submit_request() callback
@@ -863,7 +985,6 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) · 167 unmodified lines
863985 rq->fence.seqno = seqno;
864986
865987 RCU_INIT_POINTER(rq->timeline, tl);
866− RCU_INIT_POINTER(rq->hwsp_cacheline, tl->hwsp_cacheline);
867988 rq->hwsp_seqno = tl->hwsp_seqno;
868989 GEM_BUG_ON(__i915_request_is_complete(rq));
869990
@@ -877,6 +998,7 @@ __i915_request_create(struct intel_context *ce, gfp_t gfp) · 7 unmodified lines
877998
878999 /* No zalloc, everything must be cleared after use */
8791000 rq->batch = NULL;
1001+ __rq_init_watchdog(rq);
8801002 GEM_BUG_ON(rq->capture_list);
8811003 GEM_BUG_ON(!llist_empty(&rq->execute_cb));
8821004
@@ -1108,9 +1230,6 @@ emit_semaphore_wait(struct i915_request *to, · 225 unmodified lines
11081230 if (i915_request_has_initial_breadcrumb(to))
11091231 goto await_fence;
11101232
1111− if (!rcu_access_pointer(from->hwsp_cacheline))
1112− goto await_fence;
1113−
11141233 /*
11151234 * If this or its dependents are waiting on an external fence
11161235 * that may fail catastrophically, then we want to avoid using
drivers/gpu/drm/i915/i915_request.h+36−11
@@ -26,7 +26,9 @@ · 25 unmodified lines
2626 #define I915_REQUEST_H
2727
2828 #include <linux/dma-fence.h>
29+#include <linux/hrtimer.h>
2930 #include <linux/irq_work.h>
31+#include <linux/llist.h>
3032 #include <linux/lockdep.h>
3133
3234 #include "gem/i915_gem_context_types.h"
@@ -237,16 +239,6 @@ struct i915_request { · 204 unmodified lines
237239 */
238240 const u32 *hwsp_seqno;
239241
240− /*
241− * If we need to access the timeline's seqno for this request in
242− * another request, we need to keep a read reference to this associated
243− * cacheline, so that we do not free and recycle it before the foreign
244− * observers have completed. Hence, we keep a pointer to the cacheline
245− * inside the timeline's HWSP vma, but it is only valid while this
246− * request has not completed and guarded by the timeline mutex.
247− */
248− struct intel_timeline_cacheline __rcu *hwsp_cacheline;
249−
250242 /** Position in the ring of the start of the request */
251243 u32 head;
252244
@@ -287,6 +279,12 @@ struct i915_request { · 34 unmodified lines
287279 /** timeline->request entry for this request */
288280 struct list_head link;
289281
282+ /** Watchdog support fields. */
283+ struct i915_request_watchdog {
284+ struct llist_node link;
285+ struct hrtimer timer;
286+ } watchdog;
287+
290288 I915_SELFTEST_DECLARE(struct {
291289 struct list_head link;
292290 unsigned long delay;
@@ -310,7 +308,7 @@ struct i915_request * __must_check · 17 unmodified lines
310308 i915_request_create(struct intel_context *ce);
311309
312310 void __i915_request_skip(struct i915_request *rq);
313−void i915_request_set_error_once(struct i915_request *rq, int error);
311+bool i915_request_set_error_once(struct i915_request *rq, int error);
314312 void i915_request_mark_eio(struct i915_request *rq);
315313
316314 struct i915_request *__i915_request_commit(struct i915_request *request);
@@ -366,6 +364,8 @@ void i915_request_submit(struct i915_request *request); · 49 unmodified lines
366364 void __i915_request_unsubmit(struct i915_request *request);
367365 void i915_request_unsubmit(struct i915_request *request);
368366
367+void i915_request_cancel(struct i915_request *rq, int error);
368+
369369 long i915_request_wait(struct i915_request *rq,
370370 unsigned int flags,
371371 long timeout)
@@ -616,4 +616,29 @@ i915_request_active_timeline(const struct i915_request *rq) · 244 unmodified lines
616616 lockdep_is_held(&rq->engine->active.lock));
617617 }
618618
619+static inline u32
620+i915_request_active_seqno(const struct i915_request *rq)
621+{
622+ u32 hwsp_phys_base =
623+ page_mask_bits(i915_request_active_timeline(rq)->hwsp_offset);
624+ u32 hwsp_relative_offset = offset_in_page(rq->hwsp_seqno);
625+
626+ /*
627+ * Because of wraparound, we cannot simply take tl->hwsp_offset,
628+ * but instead use the fact that the relative for vaddr is the
629+ * offset as for hwsp_offset. Take the top bits from tl->hwsp_offset
630+ * and combine them with the relative offset in rq->hwsp_seqno.
631+ *
632+ * As rw->hwsp_seqno is rewritten when signaled, this only works
633+ * when the request isn't signaled yet, but at that point you
634+ * no longer need the offset.
635+ */
636+
637+ return hwsp_phys_base + hwsp_relative_offset;
638+}
639+
640+bool
641+i915_request_active_engine(struct i915_request *rq,
642+ struct intel_engine_cs **active);
643+
619644 #endif /* I915_REQUEST_H */
drivers/gpu/drm/i915/i915_selftest.h+2−0
@@ -107,6 +107,7 @@ int __i915_subtests(const char *caller, · 106 unmodified lines
107107
108108 #define I915_SELFTEST_DECLARE(x) x
109109 #define I915_SELFTEST_ONLY(x) unlikely(x)
110+#define I915_SELFTEST_EXPORT
110111
111112 #else /* !IS_ENABLED(CONFIG_DRM_I915_SELFTEST) */
112113
@@ -116,6 +117,7 @@ static inline int i915_perf_selftests(struct pci_dev *pdev) { return 0; } · 3 unmodified lines
116117
117118 #define I915_SELFTEST_DECLARE(x)
118119 #define I915_SELFTEST_ONLY(x) 0
120+#define I915_SELFTEST_EXPORT static
119121
120122 #endif
121123
drivers/gpu/drm/i915/i915_vma.c+22−8
@@ -230,7 +230,7 @@ err_vma: · 229 unmodified lines
230230 }
231231
232232 static struct i915_vma *
233−vma_lookup(struct drm_i915_gem_object *obj,
233+i915_vma_lookup(struct drm_i915_gem_object *obj,
234234 struct i915_address_space *vm,
235235 const struct i915_ggtt_view *view)
236236 {
@@ -278,7 +278,7 @@ i915_vma_instance(struct drm_i915_gem_object *obj, · 41 unmodified lines
278278 GEM_BUG_ON(!atomic_read(&vm->open));
279279
280280 spin_lock(&obj->vma.lock);
281− vma = vma_lookup(obj, vm, view);
281+ vma = i915_vma_lookup(obj, vm, view);
282282 spin_unlock(&obj->vma.lock);
283283
284284 /* vma_create() will resolve the race if another creates the vma */
@@ -863,8 +863,8 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, · 578 unmodified lines
863863 int err;
864864
865865 #ifdef CONFIG_PROVE_LOCKING
866− if (debug_locks && lockdep_is_held(&vma->vm->i915->drm.struct_mutex))
867− WARN_ON(!ww);
866+ if (debug_locks && !WARN_ON(!ww) && vma->resv)
867+ assert_vma_held(vma);
868868 #endif
869869
870870 BUILD_BUG_ON(PIN_GLOBAL != I915_VMA_GLOBAL_BIND);
@@ -884,6 +884,11 @@ int i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, · 13 unmodified lines
884884 wakeref = intel_runtime_pm_get(&vma->vm->i915->runtime_pm);
885885
886886 if (flags & vma->vm->bind_async_flags) {
887+ /* lock VM */
888+ err = i915_vm_lock_objects(vma->vm, ww);
889+ if (err)
890+ goto err_rpm;
891+
887892 work = i915_vma_work();
888893 if (!work) {
889894 err = -ENOMEM;
@@ -1020,8 +1025,15 @@ int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, · 130 unmodified lines
10201025
10211026 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
10221027
1028+#ifdef CONFIG_LOCKDEP
1029+ WARN_ON(!ww && vma->resv && dma_resv_held(vma->resv));
1030+#endif
1031+
10231032 do {
1024− err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL);
1033+ if (ww)
1034+ err = i915_vma_pin_ww(vma, ww, 0, align, flags | PIN_GLOBAL);
1035+ else
1036+ err = i915_vma_pin(vma, 0, align, flags | PIN_GLOBAL);
10251037 if (err != -ENOSPC) {
10261038 if (!err) {
10271039 err = i915_vma_wait_for_bind(vma);
@@ -1238,9 +1250,11 @@ int i915_vma_move_to_active(struct i915_vma *vma, · 210 unmodified lines
12381250 obj->write_domain = I915_GEM_DOMAIN_RENDER;
12391251 obj->read_domains = 0;
12401252 } else {
1241− err = dma_resv_reserve_shared(vma->resv, 1);
1242− if (unlikely(err))
1243− return err;
1253+ if (!(flags & __EXEC_OBJECT_NO_RESERVE)) {
1254+ err = dma_resv_reserve_shared(vma->resv, 1);
1255+ if (unlikely(err))
1256+ return err;
1257+ }
12441258
12451259 dma_resv_add_shared_fence(vma->resv, &rq->fence);
12461260 obj->write_domain = 0;
drivers/gpu/drm/i915/i915_vma.h+19−1
@@ -52,6 +52,9 @@ static inline bool i915_vma_is_active(const struct i915_vma *vma) · 51 unmodified lines
5252 return !i915_active_is_idle(&vma->active);
5353 }
5454
55+/* do not reserve memory to prevent deadlocks */
56+#define __EXEC_OBJECT_NO_RESERVE BIT(31)
57+
5558 int __must_check __i915_vma_move_to_active(struct i915_vma *vma,
5659 struct i915_request *rq);
5760 int __must_check i915_vma_move_to_active(struct i915_vma *vma,
@@ -243,7 +246,22 @@ i915_vma_pin_ww(struct i915_vma *vma, struct i915_gem_ww_ctx *ww, · 185 unmodified lines
243246 static inline int __must_check
244247 i915_vma_pin(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
245248 {
246− return i915_vma_pin_ww(vma, NULL, size, alignment, flags);
249+ struct i915_gem_ww_ctx ww;
250+ int err;
251+
252+ i915_gem_ww_ctx_init(&ww, true);
253+retry:
254+ err = i915_gem_object_lock(vma->obj, &ww);
255+ if (!err)
256+ err = i915_vma_pin_ww(vma, &ww, size, alignment, flags);
257+ if (err == -EDEADLK) {
258+ err = i915_gem_ww_ctx_backoff(&ww);
259+ if (!err)
260+ goto retry;
261+ }
262+ i915_gem_ww_ctx_fini(&ww);
263+
264+ return err;
247265 }
248266
249267 int i915_ggtt_pin(struct i915_vma *vma, struct i915_gem_ww_ctx *ww,
drivers/gpu/drm/i915/selftests/i915_gem_gtt.c+68−26
@@ -121,7 +121,7 @@ fake_dma_object(struct drm_i915_private *i915, u64 size) · 120 unmodified lines
121121 goto err;
122122
123123 drm_gem_private_object_init(&i915->drm, &obj->base, size);
124− i915_gem_object_init(obj, &fake_ops, &lock_class);
124+ i915_gem_object_init(obj, &fake_ops, &lock_class, 0);
125125
126126 i915_gem_object_set_volatile(obj);
127127
@@ -130,7 +130,7 @@ fake_dma_object(struct drm_i915_private *i915, u64 size) · 2 unmodified lines
130130 obj->cache_level = I915_CACHE_NONE;
131131
132132 /* Preallocate the "backing storage" */
133− if (i915_gem_object_pin_pages(obj))
133+ if (i915_gem_object_pin_pages_unlocked(obj))
134134 goto err_obj;
135135
136136 i915_gem_object_unpin_pages(obj);
@@ -146,6 +146,7 @@ static int igt_ppgtt_alloc(void *arg) · 9 unmodified lines
146146 {
147147 struct drm_i915_private *dev_priv = arg;
148148 struct i915_ppgtt *ppgtt;
149+ struct i915_gem_ww_ctx ww;
149150 u64 size, last, limit;
150151 int err = 0;
151152
@@ -171,6 +172,12 @@ static int igt_ppgtt_alloc(void *arg) · 19 unmodified lines
171172 limit = totalram_pages() << PAGE_SHIFT;
172173 limit = min(ppgtt->vm.total, limit);
173174
175+ i915_gem_ww_ctx_init(&ww, false);
176+retry:
177+ err = i915_vm_lock_objects(&ppgtt->vm, &ww);
178+ if (err)
179+ goto err_ppgtt_cleanup;
180+
174181 /* Check we can allocate the entire range */
175182 for (size = 4096; size <= limit; size <<= 2) {
176183 struct i915_vm_pt_stash stash = {};
@@ -215,6 +222,13 @@ static int igt_ppgtt_alloc(void *arg) · 38 unmodified lines
215222 }
216223
217224 err_ppgtt_cleanup:
225+ if (err == -EDEADLK) {
226+ err = i915_gem_ww_ctx_backoff(&ww);
227+ if (!err)
228+ goto retry;
229+ }
230+ i915_gem_ww_ctx_fini(&ww);
231+
218232 i915_vm_put(&ppgtt->vm);
219233 return err;
220234 }
@@ -276,7 +290,7 @@ static int lowlevel_hole(struct i915_address_space *vm, · 55 unmodified lines
276290
277291 GEM_BUG_ON(obj->base.size != BIT_ULL(size));
278292
279− if (i915_gem_object_pin_pages(obj)) {
293+ if (i915_gem_object_pin_pages_unlocked(obj)) {
280294 i915_gem_object_put(obj);
281295 kfree(order);
282296 break;
@@ -297,20 +311,36 @@ static int lowlevel_hole(struct i915_address_space *vm, · 14 unmodified lines
297311
298312 if (vm->allocate_va_range) {
299313 struct i915_vm_pt_stash stash = {};
314+ struct i915_gem_ww_ctx ww;
315+ int err;
316+
317+ i915_gem_ww_ctx_init(&ww, false);
318+retry:
319+ err = i915_vm_lock_objects(vm, &ww);
320+ if (err)
321+ goto alloc_vm_end;
300322
323+ err = -ENOMEM;
301324 if (i915_vm_alloc_pt_stash(vm, &stash,
302325 BIT_ULL(size)))
303− break;
304−
305− if (i915_vm_pin_pt_stash(vm, &stash)) {
306− i915_vm_free_pt_stash(vm, &stash);
307− break;
308− }
326+ goto alloc_vm_end;
309327
310− vm->allocate_va_range(vm, &stash,
311− addr, BIT_ULL(size));
328+ err = i915_vm_pin_pt_stash(vm, &stash);
329+ if (!err)
330+ vm->allocate_va_range(vm, &stash,
331+ addr, BIT_ULL(size));
312332
313333 i915_vm_free_pt_stash(vm, &stash);
334+alloc_vm_end:
335+ if (err == -EDEADLK) {
336+ err = i915_gem_ww_ctx_backoff(&ww);
337+ if (!err)
338+ goto retry;
339+ }
340+ i915_gem_ww_ctx_fini(&ww);
341+
342+ if (err)
343+ break;
314344 }
315345
316346 mock_vma->pages = obj->mm.pages;
@@ -1166,7 +1196,7 @@ static int igt_ggtt_page(void *arg) · 849 unmodified lines
11661196 if (IS_ERR(obj))
11671197 return PTR_ERR(obj);
11681198
1169− err = i915_gem_object_pin_pages(obj);
1199+ err = i915_gem_object_pin_pages_unlocked(obj);
11701200 if (err)
11711201 goto out_free;
11721202
@@ -1333,7 +1363,7 @@ static int igt_gtt_reserve(void *arg) · 160 unmodified lines
13331363 goto out;
13341364 }
13351365
1336− err = i915_gem_object_pin_pages(obj);
1366+ err = i915_gem_object_pin_pages_unlocked(obj);
13371367 if (err) {
13381368 i915_gem_object_put(obj);
13391369 goto out;
@@ -1385,7 +1415,7 @@ static int igt_gtt_reserve(void *arg) · 45 unmodified lines
13851415 goto out;
13861416 }
13871417
1388− err = i915_gem_object_pin_pages(obj);
1418+ err = i915_gem_object_pin_pages_unlocked(obj);
13891419 if (err) {
13901420 i915_gem_object_put(obj);
13911421 goto out;
@@ -1549,7 +1579,7 @@ static int igt_gtt_insert(void *arg) · 157 unmodified lines
15491579 goto out;
15501580 }
15511581
1552− err = i915_gem_object_pin_pages(obj);
1582+ err = i915_gem_object_pin_pages_unlocked(obj);
15531583 if (err) {
15541584 i915_gem_object_put(obj);
15551585 goto out;
@@ -1658,7 +1688,7 @@ static int igt_gtt_insert(void *arg) · 102 unmodified lines
16581688 goto out;
16591689 }
16601690
1661− err = i915_gem_object_pin_pages(obj);
1691+ err = i915_gem_object_pin_pages_unlocked(obj);
16621692 if (err) {
16631693 i915_gem_object_put(obj);
16641694 goto out;
@@ -1829,7 +1859,7 @@ static int igt_cs_tlb(void *arg) · 164 unmodified lines
18291859 goto out_vm;
18301860 }
18311861
1832− batch = i915_gem_object_pin_map(bbe, I915_MAP_WC);
1862+ batch = i915_gem_object_pin_map_unlocked(bbe, I915_MAP_WC);
18331863 if (IS_ERR(batch)) {
18341864 err = PTR_ERR(batch);
18351865 goto out_put_bbe;
@@ -1845,7 +1875,7 @@ static int igt_cs_tlb(void *arg) · 9 unmodified lines
18451875 }
18461876
18471877 /* Track the execution of each request by writing into different slot */
1848− batch = i915_gem_object_pin_map(act, I915_MAP_WC);
1878+ batch = i915_gem_object_pin_map_unlocked(act, I915_MAP_WC);
18491879 if (IS_ERR(batch)) {
18501880 err = PTR_ERR(batch);
18511881 goto out_put_act;
@@ -1892,7 +1922,7 @@ static int igt_cs_tlb(void *arg) · 40 unmodified lines
18921922 goto out_put_out;
18931923 GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE);
18941924
1895− result = i915_gem_object_pin_map(out, I915_MAP_WB);
1925+ result = i915_gem_object_pin_map_unlocked(out, I915_MAP_WB);
18961926 if (IS_ERR(result)) {
18971927 err = PTR_ERR(result);
18981928 goto out_put_out;
@@ -1908,6 +1938,7 @@ static int igt_cs_tlb(void *arg) · 9 unmodified lines
19081938 while (!__igt_timeout(end_time, NULL)) {
19091939 struct i915_vm_pt_stash stash = {};
19101940 struct i915_request *rq;
1941+ struct i915_gem_ww_ctx ww;
19111942 u64 offset;
19121943
19131944 offset = igt_random_offset(&prng,
@@ -1926,19 +1957,30 @@ static int igt_cs_tlb(void *arg) · 12 unmodified lines
19261957 if (err)
19271958 goto end;
19281959
1960+ i915_gem_ww_ctx_init(&ww, false);
1961+retry:
1962+ err = i915_vm_lock_objects(vm, &ww);
1963+ if (err)
1964+ goto end_ww;
1965+
19291966 err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size);
19301967 if (err)
1931− goto end;
1968+ goto end_ww;
19321969
19331970 err = i915_vm_pin_pt_stash(vm, &stash);
1934− if (err) {
1935− i915_vm_free_pt_stash(vm, &stash);
1936− goto end;
1937− }
1938−
1939− vm->allocate_va_range(vm, &stash, offset, chunk_size);
1971+ if (!err)
1972+ vm->allocate_va_range(vm, &stash, offset, chunk_size);
19401973
19411974 i915_vm_free_pt_stash(vm, &stash);
1975+end_ww:
1976+ if (err == -EDEADLK) {
1977+ err = i915_gem_ww_ctx_backoff(&ww);
1978+ if (!err)
1979+ goto retry;
1980+ }
1981+ i915_gem_ww_ctx_fini(&ww);
1982+ if (err)
1983+ goto end;
19421984
19431985 /* Prime the TLB with the dummy pages */
19441986 for (i = 0; i < count; i++) {
drivers/gpu/drm/i915/selftests/i915_request.c+206−5
@@ -609,6 +609,206 @@ static int live_nop_request(void *arg) · 608 unmodified lines
609609 return err;
610610 }
611611
612+static int __cancel_inactive(struct intel_engine_cs *engine)
613+{
614+ struct intel_context *ce;
615+ struct igt_spinner spin;
616+ struct i915_request *rq;
617+ int err = 0;
618+
619+ if (igt_spinner_init(&spin, engine->gt))
620+ return -ENOMEM;
621+
622+ ce = intel_context_create(engine);
623+ if (IS_ERR(ce)) {
624+ err = PTR_ERR(ce);
625+ goto out_spin;
626+ }
627+
628+ rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
629+ if (IS_ERR(rq)) {
630+ err = PTR_ERR(rq);
631+ goto out_ce;
632+ }
633+
634+ pr_debug("%s: Cancelling inactive request\n", engine->name);
635+ i915_request_cancel(rq, -EINTR);
636+ i915_request_get(rq);
637+ i915_request_add(rq);
638+
639+ if (i915_request_wait(rq, 0, HZ / 5) < 0) {
640+ struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
641+
642+ pr_err("%s: Failed to cancel inactive request\n", engine->name);
643+ intel_engine_dump(engine, &p, "%s\n", engine->name);
644+ err = -ETIME;
645+ goto out_rq;
646+ }
647+
648+ if (rq->fence.error != -EINTR) {
649+ pr_err("%s: fence not cancelled (%u)\n",
650+ engine->name, rq->fence.error);
651+ err = -EINVAL;
652+ }
653+
654+out_rq:
655+ i915_request_put(rq);
656+out_ce:
657+ intel_context_put(ce);
658+out_spin:
659+ igt_spinner_fini(&spin);
660+ if (err)
661+ pr_err("%s: %s error %d\n", __func__, engine->name, err);
662+ return err;
663+}
664+
665+static int __cancel_active(struct intel_engine_cs *engine)
666+{
667+ struct intel_context *ce;
668+ struct igt_spinner spin;
669+ struct i915_request *rq;
670+ int err = 0;
671+
672+ if (igt_spinner_init(&spin, engine->gt))
673+ return -ENOMEM;
674+
675+ ce = intel_context_create(engine);
676+ if (IS_ERR(ce)) {
677+ err = PTR_ERR(ce);
678+ goto out_spin;
679+ }
680+
681+ rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
682+ if (IS_ERR(rq)) {
683+ err = PTR_ERR(rq);
684+ goto out_ce;
685+ }
686+
687+ pr_debug("%s: Cancelling active request\n", engine->name);
688+ i915_request_get(rq);
689+ i915_request_add(rq);
690+ if (!igt_wait_for_spinner(&spin, rq)) {
691+ struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
692+
693+ pr_err("Failed to start spinner on %s\n", engine->name);
694+ intel_engine_dump(engine, &p, "%s\n", engine->name);
695+ err = -ETIME;
696+ goto out_rq;
697+ }
698+ i915_request_cancel(rq, -EINTR);
699+
700+ if (i915_request_wait(rq, 0, HZ / 5) < 0) {
701+ struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
702+
703+ pr_err("%s: Failed to cancel active request\n", engine->name);
704+ intel_engine_dump(engine, &p, "%s\n", engine->name);
705+ err = -ETIME;
706+ goto out_rq;
707+ }
708+
709+ if (rq->fence.error != -EINTR) {
710+ pr_err("%s: fence not cancelled (%u)\n",
711+ engine->name, rq->fence.error);
712+ err = -EINVAL;
713+ }
714+
715+out_rq:
716+ i915_request_put(rq);
717+out_ce:
718+ intel_context_put(ce);
719+out_spin:
720+ igt_spinner_fini(&spin);
721+ if (err)
722+ pr_err("%s: %s error %d\n", __func__, engine->name, err);
723+ return err;
724+}
725+
726+static int __cancel_completed(struct intel_engine_cs *engine)
727+{
728+ struct intel_context *ce;
729+ struct igt_spinner spin;
730+ struct i915_request *rq;
731+ int err = 0;
732+
733+ if (igt_spinner_init(&spin, engine->gt))
734+ return -ENOMEM;
735+
736+ ce = intel_context_create(engine);
737+ if (IS_ERR(ce)) {
738+ err = PTR_ERR(ce);
739+ goto out_spin;
740+ }
741+
742+ rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
743+ if (IS_ERR(rq)) {
744+ err = PTR_ERR(rq);
745+ goto out_ce;
746+ }
747+ igt_spinner_end(&spin);
748+ i915_request_get(rq);
749+ i915_request_add(rq);
750+
751+ if (i915_request_wait(rq, 0, HZ / 5) < 0) {
752+ err = -ETIME;
753+ goto out_rq;
754+ }
755+
756+ pr_debug("%s: Cancelling completed request\n", engine->name);
757+ i915_request_cancel(rq, -EINTR);
758+ if (rq->fence.error) {
759+ pr_err("%s: fence not cancelled (%u)\n",
760+ engine->name, rq->fence.error);
761+ err = -EINVAL;
762+ }
763+
764+out_rq:
765+ i915_request_put(rq);
766+out_ce:
767+ intel_context_put(ce);
768+out_spin:
769+ igt_spinner_fini(&spin);
770+ if (err)
771+ pr_err("%s: %s error %d\n", __func__, engine->name, err);
772+ return err;
773+}
774+
775+static int live_cancel_request(void *arg)
776+{
777+ struct drm_i915_private *i915 = arg;
778+ struct intel_engine_cs *engine;
779+
780+ /*
781+ * Check cancellation of requests. We expect to be able to immediately
782+ * cancel active requests, even if they are currently on the GPU.
783+ */
784+
785+ for_each_uabi_engine(engine, i915) {
786+ struct igt_live_test t;
787+ int err, err2;
788+
789+ if (!intel_engine_has_preemption(engine))
790+ continue;
791+
792+ err = igt_live_test_begin(&t, i915, __func__, engine->name);
793+ if (err)
794+ return err;
795+
796+ err = __cancel_inactive(engine);
797+ if (err == 0)
798+ err = __cancel_active(engine);
799+ if (err == 0)
800+ err = __cancel_completed(engine);
801+
802+ err2 = igt_live_test_end(&t);
803+ if (err)
804+ return err;
805+ if (err2)
806+ return err2;
807+ }
808+
809+ return 0;
810+}
811+
612812 static struct i915_vma *empty_batch(struct drm_i915_private *i915)
613813 {
614814 struct drm_i915_gem_object *obj;
@@ -620,7 +820,7 @@ static struct i915_vma *empty_batch(struct drm_i915_private *i915) · 5 unmodified lines
620820 if (IS_ERR(obj))
621821 return ERR_CAST(obj);
622822
623− cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
823+ cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
624824 if (IS_ERR(cmd)) {
625825 err = PTR_ERR(cmd);
626826 goto err;
@@ -782,7 +982,7 @@ static struct i915_vma *recursive_batch(struct drm_i915_private *i915) · 155 unmodified lines
782982 if (err)
783983 goto err;
784984
785− cmd = i915_gem_object_pin_map(obj, I915_MAP_WC);
985+ cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
786986 if (IS_ERR(cmd)) {
787987 err = PTR_ERR(cmd);
788988 goto err;
@@ -817,7 +1017,7 @@ static int recursive_batch_resolve(struct i915_vma *batch) · 28 unmodified lines
8171017 {
8181018 u32 *cmd;
8191019
820− cmd = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
1020+ cmd = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
8211021 if (IS_ERR(cmd))
8221022 return PTR_ERR(cmd);
8231023
@@ -1070,8 +1270,8 @@ out_request: · 246 unmodified lines
10701270 if (!request[idx])
10711271 break;
10721272
1073− cmd = i915_gem_object_pin_map(request[idx]->batch->obj,
1074− I915_MAP_WC);
1273+ cmd = i915_gem_object_pin_map_unlocked(request[idx]->batch->obj,
1274+ I915_MAP_WC);
10751275 if (!IS_ERR(cmd)) {
10761276 *cmd = MI_BATCH_BUFFER_END;
10771277
@@ -1486,6 +1686,7 @@ int i915_request_live_selftests(struct drm_i915_private *i915) · 408 unmodified lines
14861686 SUBTEST(live_sequential_engines),
14871687 SUBTEST(live_parallel_engines),
14881688 SUBTEST(live_empty_request),
1689+ SUBTEST(live_cancel_request),
14891690 SUBTEST(live_breadcrumbs_smoketest),
14901691 };
14911692
drivers/gpu/drm/i915/selftests/igt_spinner.c+90−46
@@ -12,8 +12,6 @@ · 11 unmodified lines
1212
1313 int igt_spinner_init(struct igt_spinner *spin, struct intel_gt *gt)
1414 {
15− unsigned int mode;
16− void *vaddr;
1715 int err;
1816
1917 memset(spin, 0, sizeof(*spin));
@@ -24,6 +22,7 @@ int igt_spinner_init(struct igt_spinner *spin, struct intel_gt *gt) · 4 unmodified lines
2422 err = PTR_ERR(spin->hws);
2523 goto err;
2624 }
25+ i915_gem_object_set_cache_coherency(spin->hws, I915_CACHE_LLC);
2726
2827 spin->obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
2928 if (IS_ERR(spin->obj)) {
@@ -31,34 +30,83 @@ int igt_spinner_init(struct igt_spinner *spin, struct intel_gt *gt) · 1 unmodified line
3130 goto err_hws;
3231 }
3332
34− i915_gem_object_set_cache_coherency(spin->hws, I915_CACHE_LLC);
35− vaddr = i915_gem_object_pin_map(spin->hws, I915_MAP_WB);
36− if (IS_ERR(vaddr)) {
37− err = PTR_ERR(vaddr);
38− goto err_obj;
39− }
40− spin->seqno = memset(vaddr, 0xff, PAGE_SIZE);
41−
42− mode = i915_coherent_map_type(gt->i915);
43− vaddr = i915_gem_object_pin_map(spin->obj, mode);
44− if (IS_ERR(vaddr)) {
45− err = PTR_ERR(vaddr);
46− goto err_unpin_hws;
47− }
48− spin->batch = vaddr;
49−
5033 return 0;
5134
52−err_unpin_hws:
53− i915_gem_object_unpin_map(spin->hws);
54−err_obj:
55− i915_gem_object_put(spin->obj);
5635 err_hws:
5736 i915_gem_object_put(spin->hws);
5837 err:
5938 return err;
6039 }
6140
41+static void *igt_spinner_pin_obj(struct intel_context *ce,
42+ struct i915_gem_ww_ctx *ww,
43+ struct drm_i915_gem_object *obj,
44+ unsigned int mode, struct i915_vma **vma)
45+{
46+ void *vaddr;
47+ int ret;
48+
49+ *vma = i915_vma_instance(obj, ce->vm, NULL);
50+ if (IS_ERR(*vma))
51+ return ERR_CAST(*vma);
52+
53+ ret = i915_gem_object_lock(obj, ww);
54+ if (ret)
55+ return ERR_PTR(ret);
56+
57+ vaddr = i915_gem_object_pin_map(obj, mode);
58+
59+ if (!ww)
60+ i915_gem_object_unlock(obj);
61+
62+ if (IS_ERR(vaddr))
63+ return vaddr;
64+
65+ if (ww)
66+ ret = i915_vma_pin_ww(*vma, ww, 0, 0, PIN_USER);
67+ else
68+ ret = i915_vma_pin(*vma, 0, 0, PIN_USER);
69+
70+ if (ret) {
71+ i915_gem_object_unpin_map(obj);
72+ return ERR_PTR(ret);
73+ }
74+
75+ return vaddr;
76+}
77+
78+int igt_spinner_pin(struct igt_spinner *spin,
79+ struct intel_context *ce,
80+ struct i915_gem_ww_ctx *ww)
81+{
82+ void *vaddr;
83+
84+ if (spin->ce && WARN_ON(spin->ce != ce))
85+ return -ENODEV;
86+ spin->ce = ce;
87+
88+ if (!spin->seqno) {
89+ vaddr = igt_spinner_pin_obj(ce, ww, spin->hws, I915_MAP_WB, &spin->hws_vma);
90+ if (IS_ERR(vaddr))
91+ return PTR_ERR(vaddr);
92+
93+ spin->seqno = memset(vaddr, 0xff, PAGE_SIZE);
94+ }
95+
96+ if (!spin->batch) {
97+ unsigned int mode =
98+ i915_coherent_map_type(spin->gt->i915);
99+
100+ vaddr = igt_spinner_pin_obj(ce, ww, spin->obj, mode, &spin->batch_vma);
101+ if (IS_ERR(vaddr))
102+ return PTR_ERR(vaddr);
103+
104+ spin->batch = vaddr;
105+ }
106+
107+ return 0;
108+}
109+
62110 static unsigned int seqno_offset(u64 fence)
63111 {
64112 return offset_in_page(sizeof(u32) * fence);
@@ -103,27 +151,18 @@ igt_spinner_create_request(struct igt_spinner *spin, · 38 unmodified lines
103151 if (!intel_engine_can_store_dword(ce->engine))
104152 return ERR_PTR(-ENODEV);
105153
106− vma = i915_vma_instance(spin->obj, ce->vm, NULL);
107− if (IS_ERR(vma))
108− return ERR_CAST(vma);
109−
110− hws = i915_vma_instance(spin->hws, ce->vm, NULL);
111− if (IS_ERR(hws))
112− return ERR_CAST(hws);
154+ if (!spin->batch) {
155+ err = igt_spinner_pin(spin, ce, NULL);
156+ if (err)
157+ return ERR_PTR(err);
158+ }
113159
114− err = i915_vma_pin(vma, 0, 0, PIN_USER);
115− if (err)
116− return ERR_PTR(err);
117−
118− err = i915_vma_pin(hws, 0, 0, PIN_USER);
119− if (err)
120− goto unpin_vma;
160+ hws = spin->hws_vma;
161+ vma = spin->batch_vma;
121162
122163 rq = intel_context_create_request(ce);
123− if (IS_ERR(rq)) {
124− err = PTR_ERR(rq);
125− goto unpin_hws;
126− }
164+ if (IS_ERR(rq))
165+ return ERR_CAST(rq);
127166
128167 err = move_to_active(vma, rq, 0);
129168 if (err)
@@ -186,10 +225,6 @@ cancel_rq: · 56 unmodified lines
186225 i915_request_set_error_once(rq, err);
187226 i915_request_add(rq);
188227 }
189−unpin_hws:
190− i915_vma_unpin(hws);
191−unpin_vma:
192− i915_vma_unpin(vma);
193228 return err ? ERR_PTR(err) : rq;
194229 }
195230
@@ -203,6 +238,9 @@ hws_seqno(const struct igt_spinner *spin, const struct i915_request *rq) · 7 unmodified lines
203238
204239 void igt_spinner_end(struct igt_spinner *spin)
205240 {
241+ if (!spin->batch)
242+ return;
243+
206244 *spin->batch = MI_BATCH_BUFFER_END;
207245 intel_gt_chipset_flush(spin->gt);
208246 }
@@ -211,10 +249,16 @@ void igt_spinner_fini(struct igt_spinner *spin) · 2 unmodified lines
211249 {
212250 igt_spinner_end(spin);
213251
214− i915_gem_object_unpin_map(spin->obj);
252+ if (spin->batch) {
253+ i915_vma_unpin(spin->batch_vma);
254+ i915_gem_object_unpin_map(spin->obj);
255+ }
215256 i915_gem_object_put(spin->obj);
216257
217− i915_gem_object_unpin_map(spin->hws);
258+ if (spin->seqno) {
259+ i915_vma_unpin(spin->hws_vma);
260+ i915_gem_object_unpin_map(spin->hws);
261+ }
218262 i915_gem_object_put(spin->hws);
219263 }
220264
drivers/gpu/drm/i915/selftests/igt_spinner.h+5−0
@@ -20,11 +20,16 @@ struct igt_spinner { · 19 unmodified lines
2020 struct intel_gt *gt;
2121 struct drm_i915_gem_object *hws;
2222 struct drm_i915_gem_object *obj;
23+ struct intel_context *ce;
24+ struct i915_vma *hws_vma, *batch_vma;
2325 u32 *batch;
2426 void *seqno;
2527 };
2628
2729 int igt_spinner_init(struct igt_spinner *spin, struct intel_gt *gt);
30+int igt_spinner_pin(struct igt_spinner *spin,
31+ struct intel_context *ce,
32+ struct i915_gem_ww_ctx *ww);
2833 void igt_spinner_fini(struct igt_spinner *spin);
2934
3035 struct i915_request *
drivers/gpu/drm/i915/selftests/intel_memory_region.c+11−7
@@ -31,10 +31,12 @@ static void close_objects(struct intel_memory_region *mem, · 30 unmodified lines
3131 struct drm_i915_gem_object *obj, *on;
3232
3333 list_for_each_entry_safe(obj, on, objects, st_link) {
34+ i915_gem_object_lock(obj, NULL);
3435 if (i915_gem_object_has_pinned_pages(obj))
3536 i915_gem_object_unpin_pages(obj);
3637 /* No polluting the memory region between tests */
3738 __i915_gem_object_put_pages(obj);
39+ i915_gem_object_unlock(obj);
3840 list_del(&obj->st_link);
3941 i915_gem_object_put(obj);
4042 }
@@ -69,7 +71,7 @@ static int igt_mock_fill(void *arg) · 28 unmodified lines
6971 break;
7072 }
7173
72− err = i915_gem_object_pin_pages(obj);
74+ err = i915_gem_object_pin_pages_unlocked(obj);
7375 if (err) {
7476 i915_gem_object_put(obj);
7577 break;
@@ -109,7 +111,7 @@ igt_object_create(struct intel_memory_region *mem, · 33 unmodified lines
109111 if (IS_ERR(obj))
110112 return obj;
111113
112− err = i915_gem_object_pin_pages(obj);
114+ err = i915_gem_object_pin_pages_unlocked(obj);
113115 if (err)
114116 goto put;
115117
@@ -123,8 +125,10 @@ put: · 7 unmodified lines
123125
124126 static void igt_object_release(struct drm_i915_gem_object *obj)
125127 {
128+ i915_gem_object_lock(obj, NULL);
126129 i915_gem_object_unpin_pages(obj);
127130 __i915_gem_object_put_pages(obj);
131+ i915_gem_object_unlock(obj);
128132 list_del(&obj->st_link);
129133 i915_gem_object_put(obj);
130134 }
@@ -433,7 +437,7 @@ static int igt_cpu_check(struct drm_i915_gem_object *obj, u32 dword, u32 val) · 302 unmodified lines
433437 if (err)
434438 return err;
435439
436− ptr = i915_gem_object_pin_map(obj, I915_MAP_WC);
440+ ptr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
437441 if (IS_ERR(ptr))
438442 return PTR_ERR(ptr);
439443
@@ -538,7 +542,7 @@ static int igt_lmem_create(void *arg) · 98 unmodified lines
538542 if (IS_ERR(obj))
539543 return PTR_ERR(obj);
540544
541− err = i915_gem_object_pin_pages(obj);
545+ err = i915_gem_object_pin_pages_unlocked(obj);
542546 if (err)
543547 goto out_put;
544548
@@ -577,7 +581,7 @@ static int igt_lmem_write_gpu(void *arg) · 32 unmodified lines
577581 goto out_file;
578582 }
579583
580− err = i915_gem_object_pin_pages(obj);
584+ err = i915_gem_object_pin_pages_unlocked(obj);
581585 if (err)
582586 goto out_put;
583587
@@ -649,7 +653,7 @@ static int igt_lmem_write_cpu(void *arg) · 65 unmodified lines
649653 if (IS_ERR(obj))
650654 return PTR_ERR(obj);
651655
652− vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
656+ vaddr = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
653657 if (IS_ERR(vaddr)) {
654658 err = PTR_ERR(vaddr);
655659 goto out_put;
@@ -753,7 +757,7 @@ create_region_for_mapping(struct intel_memory_region *mr, u64 size, u32 type, · 97 unmodified lines
753757 return obj;
754758 }
755759
756− addr = i915_gem_object_pin_map(obj, type);
760+ addr = i915_gem_object_pin_map_unlocked(obj, type);
757761 if (IS_ERR(addr)) {
758762 i915_gem_object_put(obj);
759763 if (PTR_ERR(addr) == -ENXIO)
drivers/gpu/drm/i915/selftests/mock_region.c+2−2
@@ -27,13 +27,13 @@ static int mock_object_init(struct intel_memory_region *mem, · 26 unmodified lines
2727 return -E2BIG;
2828
2929 drm_gem_private_object_init(&i915->drm, &obj->base, size);
30− i915_gem_object_init(obj, &mock_region_obj_ops, &lock_class);
30+ i915_gem_object_init(obj, &mock_region_obj_ops, &lock_class, flags);
3131
3232 obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;
3333
3434 i915_gem_object_set_cache_coherency(obj, I915_CACHE_NONE);
3535
36− i915_gem_object_init_memory_region(obj, mem, flags);
36+ i915_gem_object_init_memory_region(obj, mem);
3737
3838 return 0;
3939 }
include/uapi/drm/i915_drm.h+1−0
@@ -943,6 +943,7 @@ struct drm_i915_gem_exec_object { · 942 unmodified lines
943943 __u64 offset;
944944 };
945945
946+/* DRM_IOCTL_I915_GEM_EXECBUFFER was removed in Linux 5.13 */
946947 struct drm_i915_gem_execbuffer {
947948 /**
948949 * List of buffers to be validated with their relocations to be