History

drm/xe: fix job timeout recovery for unstarted jobs and kernel queues

A job that GuC never scheduled (never started) indicates a GuC scheduling failure; previously such jobs were silently errored out instead of triggering a GT reset to recover. Trigger a GT reset and resubmit them, but only when the queue was not already killed or banned: an unstarted job on an already banned queue is the ban working as intended and must neither clear the ban nor kick off a reset, otherwise a banned userspace queue could be resurrected and spam GT resets. Kernel queues are always recovered this way and wedge the device once recovery attempts are exhausted, since kernel work must not silently fail. A started job that times out on a userspace VM bind queue stays banned rather than being reset and retried. The queue is banned early in the timeout handler to signal the G2H scheduling-done handler so it wakes the disable-scheduling waiter; without it the waiter sleeps the full 5s timeout. When a reset is warranted the ban is cleared before rearming so that guc_exec_queue_start() can resubmit jobs after the GT reset - a still-banned queue would block resubmission and cause an infinite TDR loop. The already-banned case is gated out before this point via skip_timeout_check, so it is unaffected. v2: (Himal) Do it for any queue type, not just kernel/migration v3: - (Sashiko and Sanjay): don't clear the ban / GT reset for already killed/banned queues on unstarted-job timeout - Update commit message - (Matt) Add Fixes tag

Rodrigo Vivi committed 5 months ago

Commit 347ccc01 parent ba367861 file · +35 −14Browse files

Unified · Split

drivers/gpu/drm/xe/xe_guc_submit.c+35−14
@@ -157,6 +157,11 @@ static void set_exec_queue_banned(struct xe_exec_queue *q) · 156 unmodified lines
157 atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state);157 atomic_or(EXEC_QUEUE_STATE_BANNED, &q->guc->state);
158}158}
159 159
160static void clear_exec_queue_banned(struct xe_exec_queue *q)
161{
162 atomic_andnot(EXEC_QUEUE_STATE_BANNED, &q->guc->state);
163}
164
160static bool exec_queue_suspended(struct xe_exec_queue *q)165static bool exec_queue_suspended(struct xe_exec_queue *q)
161{166{
162 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED;167 return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_SUSPENDED;
@@ -1361,7 +1366,8 @@ static bool check_timeout(struct xe_exec_queue *q, struct xe_sched_job *job) · 1198 unmodified lines
1361 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),1366 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
1362 q->guc->id);1367 q->guc->id);
1363 1368
1364 return xe_sched_invalidate_job(job, 2);1369 /* GuC never scheduled this job - let the caller trigger a GT reset. */
1370 return true;
1365 }1371 }
1366 1372
1367 ctx_timestamp = lower_32_bits(xe_lrc_timestamp(q->lrc[0]));1373 ctx_timestamp = lower_32_bits(xe_lrc_timestamp(q->lrc[0]));
@@ -1458,6 +1464,21 @@ static void disable_scheduling(struct xe_exec_queue *q, bool immediate) · 90 unmodified lines
1458 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);1464 G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1);
1459}1465}
1460 1466
1467/*
1468 * Recover via GT reset for a kernel queue, or for a GuC scheduling failure (job
1469 * never started) on a queue that was not already killed or banned. An already
1470 * banned queue must stay banned, so its unstarted jobs do not clear the ban or
1471 * trigger a reset.
1472 */
1473static bool timeout_needs_gt_reset(struct xe_exec_queue *q, struct xe_sched_job *job,
1474 bool skip_timeout_check)
1475{
1476 if (q->flags & EXEC_QUEUE_FLAG_KERNEL)
1477 return true;
1478
1479 return !skip_timeout_check && !xe_sched_job_started(job);
1480}
1481
1461static enum drm_gpu_sched_stat1482static enum drm_gpu_sched_stat
1462guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)1483guc_exec_queue_timedout_job(struct drm_sched_job *drm_job)
1463{1484{
@@ -1606,19 +1627,19 @@ trigger_reset: · 142 unmodified lines
1606 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),1627 xe_sched_job_seqno(job), xe_sched_job_lrc_seqno(job),
1607 q->guc->id, q->flags);1628 q->guc->id, q->flags);
1608 1629
1609 /*1630 if (!wedged) {
1610 * Kernel jobs should never fail, nor should VM jobs if they do1631 if (timeout_needs_gt_reset(q, job, skip_timeout_check)) {
1611 * somethings has gone wrong and the GT needs a reset1632 if (!xe_sched_invalidate_job(job, 2)) {
1612 */1633 clear_exec_queue_banned(q);
1613 xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_KERNEL,1634 xe_gt_reset_async(q->gt);
1614 "Kernel-submitted job timed out\n");1635 goto rearm;
1615 xe_gt_WARN(q->gt, q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q),1636 }
1616 "VM job timed out on non-killed execqueue\n");1637 if (q->flags & EXEC_QUEUE_FLAG_KERNEL) {
1617 if (!wedged && (q->flags & EXEC_QUEUE_FLAG_KERNEL ||1638 xe_gt_WARN(q->gt, true, "Kernel-submitted job timed out\n");
1618 (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)))) {1639 xe_device_declare_wedged(gt_to_xe(q->gt));
1619 if (!xe_sched_invalidate_job(job, 2)) {1640 }
1620 xe_gt_reset_async(q->gt);1641 } else if (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)) {
1621 goto rearm;1642 xe_gt_WARN(q->gt, true, "VM job timed out on non-killed execqueue\n");
1622 }1643 }
1623 }1644 }
1624 1645