History

drm/xe: fix job timeout recovery for unstarted jobs and kernel queues

A job that GuC never scheduled (never started) indicates a GuC scheduling failure; previously such jobs were silently errored out instead of triggering a GT reset to recover. Trigger a GT reset and resubmit them, but only when the queue was not already killed or banned: an unstarted job on an already banned queue is the ban working as intended and must neither clear the ban nor kick off a reset, otherwise a banned userspace queue could be resurrected and spam GT resets. Kernel queues are always recovered this way and wedge the device once recovery attempts are exhausted, since kernel work must not silently fail. A started job that times out on a userspace VM bind queue stays banned rather than being reset and retried. The queue is banned early in the timeout handler to signal the G2H scheduling-done handler so it wakes the disable-scheduling waiter; without it the waiter sleeps the full 5s timeout. When a reset is warranted the ban is cleared before rearming so that guc_exec_queue_start() can resubmit jobs after the GT reset - a still-banned queue would block resubmission and cause an infinite TDR loop. The already-banned case is gated out before this point via skip_timeout_check, so it is unaffected. v2: (Himal) Do it for any queue type, not just kernel/migration v3: - (Sashiko and Sanjay): don't clear the ban / GT reset for already killed/banned queues on unstarted-job timeout - Update commit message - (Matt) Add Fixes tag

Rodrigo Vivi committed 5 months ago

Commit 347ccc01 parent ba367861 file · +35 −14Browse files

Unified · Split

drivers/gpu/drm/xe/xe_guc_submit.c+35−14
@@ -157,6 +157,11 @@ static void set_exec_queue_banned(struct xe_exec_queue *q) · 156 unmodified lines
157157 atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state);
158158 }
159159
160+static void clear_exec_queue_banned(struct xe_exec_queue *q)
161+{
162+ atomic_andnot(EXEC_QUEUE_STATE_BANNED, &q->guc->state);
163+}
164+
160165 static bool exec_queue_suspended(struct xe_exec_queue *q)
161166 {
162167 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED;
@@ -1361,7 +1366,8 @@ static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job) · 1198 unmodified lines
13611366 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
13621367 q->guc->id);
13631368
1364− return xe_sched_invalidate_job(job, 2);
1369+ /* GuC never scheduled this job - let the caller trigger a GT reset. */
1370+ return true;
13651371 }
13661372
13671373 ctx_timestamp = lower_32_bits(xe_lrc_timestamp(q->lrc[0]));
@@ -1458,6 +1464,21 @@ static void disable_scheduling(struct xe_exec_queue *q, bool immediate) · 90 unmodified lines
14581464 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
14591465 }
14601466
1467+/*
1468+ * Recover via GT reset for a kernel queue, or for a GuC scheduling failure (job
1469+ * never started) on a queue that was not already killed or banned. An already
1470+ * banned queue must stay banned, so its unstarted jobs do not clear the ban or
1471+ * trigger a reset.
1472+ */
1473+static bool timeout_needs_gt_reset(struct xe_exec_queue *q, struct xe_sched_job *job,
1474+ bool skip_timeout_check)
1475+{
1476+ if (q->flags & EXEC_QUEUE_FLAG_KERNEL)
1477+ return true;
1478+
1479+ return !skip_timeout_check && !xe_sched_job_started(job);
1480+}
1481+
14611482 static enum drm_gpu_sched_stat
14621483 guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
14631484 {
@@ -1606,19 +1627,19 @@ trigger_reset: · 142 unmodified lines
16061627 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
16071628 q->guc->id, q->flags);
16081629
1609− /*
1610− * Kernel jobs should never fail, nor should VM jobs if they do
1611− * somethings has gone wrong and the GT needs a reset
1612− */
1613− xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL,
1614− "Kernel-submitted job timed out\n");
1615− xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q),
1616− "VM job timed out on non-killed execqueue\n");
1617− if (!wedged && (q->flags & EXEC_QUEUE_FLAG_KERNEL ||
1618− (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)))) {
1619− if (!xe_sched_invalidate_job(job, 2)) {
1620− xe_gt_reset_async(q->gt);
1621− goto rearm;
1630+ if (!wedged) {
1631+ if (timeout_needs_gt_reset(q, job, skip_timeout_check)) {
1632+ if (!xe_sched_invalidate_job(job, 2)) {
1633+ clear_exec_queue_banned(q);
1634+ xe_gt_reset_async(q->gt);
1635+ goto rearm;
1636+ }
1637+ if (q->flags & EXEC_QUEUE_FLAG_KERNEL) {
1638+ xe_gt_WARN(q->gt, true, "Kernel-submitted job timed out\n");
1639+ xe_device_declare_wedged(gt_to_xe(q->gt));
1640+ }
1641+ } else if (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)) {
1642+ xe_gt_WARN(q->gt, true, "VM job timed out on non-killed execqueue\n");
16221643 }
16231644 }
16241645