History

drm/i915/execlists: Preemption!

When we write to ELSP, it triggers a context preemption at the earliest arbitration point (3DPRIMITIVE, some PIPECONTROLs, a few other operations and the explicit MI_ARB_CHECK). If this is to the same context, it triggers a LITE_RESTORE where the RING_TAIL is merely updated (used currently to chain requests from the same context together, avoiding bubbles). However, if it is to a different context, a full context-switch is performed and it will start to execute the new context saving the image of the old for later execution. Previously we avoided preemption by only submitting a new context when the old was idle. But now we wish embrace it, and if the new request has a higher priority than the currently executing request, we write to the ELSP regardless, thus triggering preemption, but we tell the GPU to switch to our special preemption context (not the target). In the context-switch interrupt handler, we know that the previous contexts have finished execution and so can unwind all the incomplete requests and compute the new highest priority request to execute. It would be feasible to avoid the switch-to-idle intermediate by programming the ELSP with the target context. The difficulty is in tracking which request that should be whilst maintaining the dependency change, the error comes in with coalesced requests. As we only track the most recent request and its priority, we may run into the issue of being tricked in preempting a high priority request that was followed by a low priority request from the same context (e.g. for PI); worse still that earlier request may be our own dependency and the order then broken by preemption. By injecting the switch-to-idle and then recomputing the priority queue, we avoid the issue with tracking in-flight coalesced requests. Having tried the preempt-to-busy approach, and failed to find a way around the coalesced priority issue, Michal's original proposal to inject an idle context (based on handling GuC preemption) succeeds. The current heuristic for deciding when to preempt are only if the new request is of higher priority, and has the privileged priority of greater than 0. Note that the scheduler remains unfair! v2: Disable for gen8 (bdw/bsw) as we need additional w/a for GPGPU. Since, the feature is now conditional and not always available when we have a scheduler, make it known via the HAS_SCHEDULER GETPARAM (now a capability mask). v3: Stylistic tweaks. v4: Appease Joonas with a snippet of kerneldoc, only to fuel to fire of the preempt vs preempting debate.

Chris Wilson committed 9 years ago

Commit beecec91 parent bf64e0b5 files · +143 −36Browse files

Unified · Split

5 files
  1. drivers/gpu/drm/i915/i915_drv.c +8 −1
  2. drivers/gpu/drm/i915/i915_irq.c +2 −4
  3. drivers/gpu/drm/i915/i915_pci.c +2 −0
  4. drivers/gpu/drm/i915/intel_lrc.c +126 −31
  5. drivers/gpu/drm/i915/intel_ringbuffer.h +5 −0
drivers/gpu/drm/i915/i915_drv.c+8−1
@@ -368,9 +368,16 @@ static int i915_getparam(struct drm_device *dev, void *data, · 367 unmodified lines
368368 break;
369369 case I915_PARAM_HAS_SCHEDULER:
370370 value = 0;
371− if (dev_priv->engine[RCS] && dev_priv->engine[RCS]->schedule)
371+ if (dev_priv->engine[RCS] && dev_priv->engine[RCS]->schedule) {
372372 value |= I915_SCHEDULER_CAP_ENABLED;
373+
374+ if (INTEL_INFO(dev_priv)->has_logical_ring_preemption &&
375+ i915_modparams.enable_execlists &&
376+ !i915_modparams.enable_guc_submission)
377+ value |= I915_SCHEDULER_CAP_PREEMPTION;
378+ }
373379 break;
380+
374381 case I915_PARAM_MMAP_VERSION:
375382 /* Remember to bump this if the version changes! */
376383 case I915_PARAM_HAS_GEM:
drivers/gpu/drm/i915/i915_irq.c+2−4
@@ -1382,10 +1382,8 @@ gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift) · 1381 unmodified lines
13821382 bool tasklet = false;
13831383
13841384 if (iir & (GT_CONTEXT_SWITCH_INTERRUPT << test_shift)) {
1385− if (port_count(&execlists->port[0])) {
1386− __set_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
1387− tasklet = true;
1388− }
1385+ __set_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
1386+ tasklet = true;
13891387 }
13901388
13911389 if (iir & (GT_RENDER_USER_INTERRUPT << test_shift)) {
drivers/gpu/drm/i915/i915_pci.c+2−0
@@ -424,6 +424,7 @@ static const struct intel_device_info intel_cherryview_info __initconst = { · 423 unmodified lines
424424
425425 #define GEN9_FEATURES \
426426 GEN8_FEATURES, \
427+ .has_logical_ring_preemption = 1, \
427428 .has_csr = 1, \
428429 .has_guc = 1, \
429430 .has_ipc = 1, \
@@ -477,6 +478,7 @@ static const struct intel_device_info intel_skylake_gt4_info __initconst = { · 47 unmodified lines
477478 .has_rc6 = 1, \
478479 .has_dp_mst = 1, \
479480 .has_logical_ring_contexts = 1, \
481+ .has_logical_ring_preemption = 1, \
480482 .has_guc = 1, \
481483 .has_aliasing_ppgtt = 1, \
482484 .has_full_ppgtt = 1, \
drivers/gpu/drm/i915/intel_lrc.c+126−31
@@ -208,9 +208,9 @@ · 207 unmodified lines
208208
209209 /* Typical size of the average request (2 pipecontrols and a MI_BB) */
210210 #define EXECLISTS_REQUEST_SIZE 64 /* bytes */
211−
212211 #define WA_TAIL_DWORDS 2
213212 #define WA_TAIL_BYTES (sizeof(u32) * WA_TAIL_DWORDS)
213+#define PREEMPT_ID 0x1
214214
215215 static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
216216 struct intel_engine_cs *engine);
@@ -429,6 +429,12 @@ static u64 execlists_update_context(struct drm_i915_gem_request *rq) · 212 unmodified lines
429429 return ce->lrc_desc;
430430 }
431431
432+static inline void elsp_write(u64 desc, u32 __iomem *elsp)
433+{
434+ writel(upper_32_bits(desc), elsp);
435+ writel(lower_32_bits(desc), elsp);
436+}
437+
432438 static void execlists_submit_ports(struct intel_engine_cs *engine)
433439 {
434440 struct execlist_port *port = engine->execlists.port;
@@ -454,8 +460,7 @@ static void execlists_submit_ports(struct intel_engine_cs *engine) · 19 unmodified lines
454460 desc = 0;
455461 }
456462
457− writel(upper_32_bits(desc), elsp);
458− writel(lower_32_bits(desc), elsp);
463+ elsp_write(desc, elsp);
459464 }
460465 }
461466
@@ -488,26 +493,43 @@ static void port_assign(struct execlist_port *port, · 26 unmodified lines
488493 port_set(port, port_pack(i915_gem_request_get(rq), port_count(port)));
489494 }
490495
496+static void inject_preempt_context(struct intel_engine_cs *engine)
497+{
498+ struct intel_context *ce =
499+ &engine->i915->preempt_context->engine[engine->id];
500+ u32 __iomem *elsp =
501+ engine->i915->regs + i915_mmio_reg_offset(RING_ELSP(engine));
502+ unsigned int n;
503+
504+ GEM_BUG_ON(engine->i915->preempt_context->hw_id != PREEMPT_ID);
505+ GEM_BUG_ON(!IS_ALIGNED(ce->ring->size, WA_TAIL_BYTES));
506+
507+ memset(ce->ring->vaddr + ce->ring->tail, 0, WA_TAIL_BYTES);
508+ ce->ring->tail += WA_TAIL_BYTES;
509+ ce->ring->tail &= (ce->ring->size - 1);
510+ ce->lrc_reg_state[CTX_RING_TAIL+1] = ce->ring->tail;
511+
512+ for (n = execlists_num_ports(&engine->execlists); --n; )
513+ elsp_write(0, elsp);
514+
515+ elsp_write(ce->lrc_desc, elsp);
516+}
517+
518+static bool can_preempt(struct intel_engine_cs *engine)
519+{
520+ return INTEL_INFO(engine->i915)->has_logical_ring_preemption;
521+}
522+
491523 static void execlists_dequeue(struct intel_engine_cs *engine)
492524 {
493− struct drm_i915_gem_request *last;
494525 struct intel_engine_execlists * const execlists = &engine->execlists;
495526 struct execlist_port *port = execlists->port;
496527 const struct execlist_port * const last_port =
497528 &execlists->port[execlists->port_mask];
529+ struct drm_i915_gem_request *last = port_request(port);
498530 struct rb_node *rb;
499531 bool submit = false;
500532
501− last = port_request(port);
502− if (last)
503− /* WaIdleLiteRestore:bdw,skl
504− * Apply the wa NOOPs to prevent ring:HEAD == req:TAIL
505− * as we resubmit the request. See gen8_emit_breadcrumb()
506− * for where we prepare the padding after the end of the
507− * request.
508− */
509− last->tail = last->wa_tail;
510−
511533 /* Hardware submission is through 2 ports. Conceptually each port
512534 * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
513535 * static for a context, and unique to each, so we only execute
@@ -532,7 +554,65 @@ static void execlists_dequeue(struct intel_engine_cs *engine) · 18 unmodified lines
532554 spin_lock_irq(&engine->timeline->lock);
533555 rb = execlists->first;
534556 GEM_BUG_ON(rb_first(&execlists->queue) != rb);
535− while (rb) {
557+ if (!rb)
558+ goto unlock;
559+
560+ if (last) {
561+ /*
562+ * Don't resubmit or switch until all outstanding
563+ * preemptions (lite-restore) are seen. Then we
564+ * know the next preemption status we see corresponds
565+ * to this ELSP update.
566+ */
567+ if (port_count(&port[0]) > 1)
568+ goto unlock;
569+
570+ if (can_preempt(engine) &&
571+ rb_entry(rb, struct i915_priolist, node)->priority >
572+ max(last->priotree.priority, 0)) {
573+ /*
574+ * Switch to our empty preempt context so
575+ * the state of the GPU is known (idle).
576+ */
577+ inject_preempt_context(engine);
578+ execlists->preempt = true;
579+ goto unlock;
580+ } else {
581+ /*
582+ * In theory, we could coalesce more requests onto
583+ * the second port (the first port is active, with
584+ * no preemptions pending). However, that means we
585+ * then have to deal with the possible lite-restore
586+ * of the second port (as we submit the ELSP, there
587+ * may be a context-switch) but also we may complete
588+ * the resubmission before the context-switch. Ergo,
589+ * coalescing onto the second port will cause a
590+ * preemption event, but we cannot predict whether
591+ * that will affect port[0] or port[1].
592+ *
593+ * If the second port is already active, we can wait
594+ * until the next context-switch before contemplating
595+ * new requests. The GPU will be busy and we should be
596+ * able to resubmit the new ELSP before it idles,
597+ * avoiding pipeline bubbles (momentary pauses where
598+ * the driver is unable to keep up the supply of new
599+ * work).
600+ */
601+ if (port_count(&port[1]))
602+ goto unlock;
603+
604+ /* WaIdleLiteRestore:bdw,skl
605+ * Apply the wa NOOPs to prevent
606+ * ring:HEAD == req:TAIL as we resubmit the
607+ * request. See gen8_emit_breadcrumb() for
608+ * where we prepare the padding after the
609+ * end of the request.
610+ */
611+ last->tail = last->wa_tail;
612+ }
613+ }
614+
615+ do {
536616 struct i915_priolist *p = rb_entry(rb, typeof(*p), node);
537617 struct drm_i915_gem_request *rq, *rn;
538618
@@ -595,11 +675,12 @@ static void execlists_dequeue(struct intel_engine_cs *engine) · 56 unmodified lines
595675 INIT_LIST_HEAD(&p->requests);
596676 if (p->priority != I915_PRIORITY_NORMAL)
597677 kmem_cache_free(engine->i915->priorities, p);
598− }
678+ } while (rb);
599679 done:
600680 execlists->first = rb;
601681 if (submit)
602682 port_assign(port, last);
683+unlock:
603684 spin_unlock_irq(&engine->timeline->lock);
604685
605686 if (submit)
@@ -680,13 +761,6 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine) · 74 unmodified lines
680761 spin_unlock_irqrestore(&engine->timeline->lock, flags);
681762 }
682763
683−static bool execlists_elsp_ready(const struct intel_engine_cs *engine)
684−{
685− const struct execlist_port *port = engine->execlists.port;
686−
687− return port_count(&port[0]) + port_count(&port[1]) < 2;
688−}
689−
690764 /*
691765 * Check the unread Context Status Buffers and manage the submission of new
692766 * contexts to the ELSP accordingly.
@@ -695,7 +769,7 @@ static void intel_lrc_irq_handler(unsigned long data) · 2 unmodified lines
695769 {
696770 struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
697771 struct intel_engine_execlists * const execlists = &engine->execlists;
698− struct execlist_port *port = execlists->port;
772+ struct execlist_port * const port = execlists->port;
699773 struct drm_i915_private *dev_priv = engine->i915;
700774
701775 /* We can skip acquiring intel_runtime_pm_get() here as it was taken
@@ -780,6 +854,23 @@ static void intel_lrc_irq_handler(unsigned long data) · 78 unmodified lines
780854 if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
781855 continue;
782856
857+ if (status & GEN8_CTX_STATUS_ACTIVE_IDLE &&
858+ buf[2*head + 1] == PREEMPT_ID) {
859+ execlist_cancel_port_requests(execlists);
860+
861+ spin_lock_irq(&engine->timeline->lock);
862+ unwind_incomplete_requests(engine);
863+ spin_unlock_irq(&engine->timeline->lock);
864+
865+ GEM_BUG_ON(!execlists->preempt);
866+ execlists->preempt = false;
867+ continue;
868+ }
869+
870+ if (status & GEN8_CTX_STATUS_PREEMPTED &&
871+ execlists->preempt)
872+ continue;
873+
783874 /* Check the context/desc id for this event matches */
784875 GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id);
785876
@@ -811,7 +902,7 @@ static void intel_lrc_irq_handler(unsigned long data) · 25 unmodified lines
811902 }
812903 }
813904
814− if (execlists_elsp_ready(engine))
905+ if (!execlists->preempt)
815906 execlists_dequeue(engine);
816907
817908 intel_uncore_forcewake_put(dev_priv, execlists->fw_domains);
@@ -824,7 +915,7 @@ static void insert_request(struct intel_engine_cs *engine, · 6 unmodified lines
824915 struct i915_priolist *p = lookup_priolist(engine, pt, prio);
825916
826917 list_add_tail(&pt->link, &ptr_mask_bits(p, 1)->requests);
827− if (ptr_unmask_bits(p, 1) && execlists_elsp_ready(engine))
918+ if (ptr_unmask_bits(p, 1))
828919 tasklet_hi_schedule(&engine->execlists.irq_tasklet);
829920 }
830921
@@ -954,8 +1045,6 @@ static void execlists_schedule(struct drm_i915_gem_request *request, int prio) · 123 unmodified lines
9541045 }
9551046
9561047 spin_unlock_irq(&engine->timeline->lock);
957−
958− /* XXX Do we need to preempt to make room for us and our deps? */
9591048 }
9601049
9611050 static struct intel_ring *
@@ -1151,6 +1240,8 @@ static u32 *gen8_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) · 189 unmodified lines
11511240 i915_ggtt_offset(engine->scratch) +
11521241 2 * CACHELINE_BYTES);
11531242
1243+ *batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1244+
11541245 /* Pad to end of cacheline */
11551246 while ((unsigned long)batch % CACHELINE_BYTES)
11561247 *batch++ = MI_NOOP;
@@ -1166,6 +1257,8 @@ static u32 *gen8_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) · 9 unmodified lines
11661257
11671258 static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch)
11681259 {
1260+ *batch++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1261+
11691262 /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt,glk */
11701263 batch = gen8_emit_flush_coherentl3_wa(engine, batch);
11711264
@@ -1211,6 +1304,8 @@ static u32 *gen9_init_indirectctx_bb(struct intel_engine_cs *engine, u32 *batch) · 39 unmodified lines
12111304 *batch++ = 0;
12121305 }
12131306
1307+ *batch++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1308+
12141309 /* Pad to end of cacheline */
12151310 while ((unsigned long)batch % CACHELINE_BYTES)
12161311 *batch++ = MI_NOOP;
@@ -1364,6 +1459,7 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine) · 147 unmodified lines
13641459 GT_CONTEXT_SWITCH_INTERRUPT << engine->irq_shift);
13651460 clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
13661461 execlists->csb_head = -1;
1462+ execlists->preempt = false;
13671463
13681464 /* After a GPU reset, we may have requests to replay */
13691465 if (!i915_modparams.enable_guc_submission && execlists->first)
@@ -1659,7 +1755,8 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request, · 289 unmodified lines
16591755 */
16601756 static void gen8_emit_wa_tail(struct drm_i915_gem_request *request, u32 *cs)
16611757 {
1662− *cs++ = MI_NOOP;
1758+ /* Ensure there's always at least one preemption point per-request. */
1759+ *cs++ = MI_ARB_CHECK;
16631760 *cs++ = MI_NOOP;
16641761 request->wa_tail = intel_ring_offset(request, cs);
16651762 }
@@ -1680,7 +1777,6 @@ static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request, u32 *cs) · 14 unmodified lines
16801777
16811778 gen8_emit_wa_tail(request, cs);
16821779 }
1683−
16841780 static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS;
16851781
16861782 static void gen8_emit_breadcrumb_render(struct drm_i915_gem_request *request,
@@ -1708,7 +1804,6 @@ static void gen8_emit_breadcrumb_render(struct drm_i915_gem_request *request, · 21 unmodified lines
17081804
17091805 gen8_emit_wa_tail(request, cs);
17101806 }
1711−
17121807 static const int gen8_emit_breadcrumb_render_sz = 8 + WA_TAIL_DWORDS;
17131808
17141809 static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
drivers/gpu/drm/i915/intel_ringbuffer.h+5−0
@@ -238,6 +238,11 @@ struct intel_engine_execlists { · 237 unmodified lines
238238 #define EXECLIST_MAX_PORTS 2
239239 } port[EXECLIST_MAX_PORTS];
240240
241+ /**
242+ * @preempt: are we currently handling a preempting context switch?
243+ */
244+ bool preempt;
245+
241246 /**
242247 * @port_mask: number of execlist ports - 1
243248 */

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t