History

drm/amdgpu: revert "implement tdr advanced mode"

This reverts commit e6c6338f393b74ac0b303d567bb918b44ae7ad75. This feature basically re-submits one job after another to figure out which one was the one causing a hang. This is obviously incompatible with gang-submit which requires that multiple jobs run at the same time. It's also absolutely not helpful to crash the hardware multiple times if a clean recovery is desired. For testing and debugging environments we should rather disable recovery alltogether to be able to inspect the state with a hw debugger. Additional to that the sw implementation is clearly buggy and causes reference count issues for the hardware fence.

Christian König committed 4 years ago

Commit 06a2d7c1 parent d27252b4 files · +10 −156Browse files

Unified · Split

4 files
  1. drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 −103
  2. drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +1 −1
  3. drivers/gpu/drm/scheduler/sched_main.c +9 −49
  4. include/drm/gpu_scheduler.h +0 −3
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c+0−103
@@ -5088,94 +5088,6 @@ static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev) · 5087 unmodified lines
50885088 return 0;
50895089 }
50905090
5091−static void amdgpu_device_recheck_guilty_jobs(
5092− struct amdgpu_device *adev, struct list_head *device_list_handle,
5093− struct amdgpu_reset_context *reset_context)
5094−{
5095− int i, r = 0;
5096−
5097− for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5098− struct amdgpu_ring *ring = adev->rings[i];
5099− int ret = 0;
5100− struct drm_sched_job *s_job;
5101−
5102− if (!ring || !ring->sched.thread)
5103− continue;
5104−
5105− s_job = list_first_entry_or_null(&ring->sched.pending_list,
5106− struct drm_sched_job, list);
5107− if (s_job == NULL)
5108− continue;
5109−
5110− /* clear job's guilty and depend the folowing step to decide the real one */
5111− drm_sched_reset_karma(s_job);
5112− drm_sched_resubmit_jobs_ext(&ring->sched, 1);
5113−
5114− if (!s_job->s_fence->parent) {
5115− DRM_WARN("Failed to get a HW fence for job!");
5116− continue;
5117− }
5118−
5119− ret = dma_fence_wait_timeout(s_job->s_fence->parent, false, ring->sched.timeout);
5120− if (ret == 0) { /* timeout */
5121− DRM_ERROR("Found the real bad job! ring:%s, job_id:%llx\n",
5122− ring->sched.name, s_job->id);
5123−
5124−
5125− amdgpu_fence_driver_isr_toggle(adev, true);
5126−
5127− /* Clear this failed job from fence array */
5128− amdgpu_fence_driver_clear_job_fences(ring);
5129−
5130− amdgpu_fence_driver_isr_toggle(adev, false);
5131−
5132− /* Since the job won't signal and we go for
5133− * another resubmit drop this parent pointer
5134− */
5135− dma_fence_put(s_job->s_fence->parent);
5136− s_job->s_fence->parent = NULL;
5137−
5138− /* set guilty */
5139− drm_sched_increase_karma(s_job);
5140− amdgpu_reset_prepare_hwcontext(adev, reset_context);
5141−retry:
5142− /* do hw reset */
5143− if (amdgpu_sriov_vf(adev)) {
5144− amdgpu_virt_fini_data_exchange(adev);
5145− r = amdgpu_device_reset_sriov(adev, false);
5146− if (r)
5147− adev->asic_reset_res = r;
5148− } else {
5149− clear_bit(AMDGPU_SKIP_HW_RESET,
5150− &reset_context->flags);
5151− r = amdgpu_do_asic_reset(device_list_handle,
5152− reset_context);
5153− if (r && r == -EAGAIN)
5154− goto retry;
5155− }
5156−
5157− /*
5158− * add reset counter so that the following
5159− * resubmitted job could flush vmid
5160− */
5161− atomic_inc(&adev->gpu_reset_counter);
5162− continue;
5163− }
5164−
5165− /* got the hw fence, signal finished fence */
5166− atomic_dec(ring->sched.score);
5167− dma_fence_get(&s_job->s_fence->finished);
5168− dma_fence_signal(&s_job->s_fence->finished);
5169− dma_fence_put(&s_job->s_fence->finished);
5170−
5171− /* remove node from list and free the job */
5172− spin_lock(&ring->sched.job_list_lock);
5173− list_del_init(&s_job->list);
5174− spin_unlock(&ring->sched.job_list_lock);
5175− ring->sched.ops->free_job(s_job);
5176− }
5177−}
5178−
51795091 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
51805092 {
51815093 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
@@ -5196,7 +5108,6 @@ static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev) · 14 unmodified lines
51965108
51975109 }
51985110
5199−
52005111 /**
52015112 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
52025113 *
@@ -5219,7 +5130,6 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · 16 unmodified lines
52195130 int i, r = 0;
52205131 bool need_emergency_restart = false;
52215132 bool audio_suspended = false;
5222− int tmp_vram_lost_counter;
52235133 bool gpu_reset_for_dev_remove = false;
52245134
52255135 gpu_reset_for_dev_remove =
@@ -5365,7 +5275,6 @@ retry: /* Rest of adevs pre asic reset from XGMI hive. */ · 139 unmodified lines
53655275 amdgpu_device_stop_pending_resets(tmp_adev);
53665276 }
53675277
5368− tmp_vram_lost_counter = atomic_read(&((adev)->vram_lost_counter));
53695278 /* Actual ASIC resets if needed.*/
53705279 /* Host driver will handle XGMI hive reset for SRIOV */
53715280 if (amdgpu_sriov_vf(adev)) {
@@ -5390,18 +5299,6 @@ skip_hw_reset: · 18 unmodified lines
53905299 /* Post ASIC reset for all devs .*/
53915300 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
53925301
5393− /*
5394− * Sometimes a later bad compute job can block a good gfx job as gfx
5395− * and compute ring share internal GC HW mutually. We add an additional
5396− * guilty jobs recheck step to find the real guilty job, it synchronously
5397− * submits and pends for the first job being signaled. If it gets timeout,
5398− * we identify it as a real guilty job.
5399− */
5400− if (amdgpu_gpu_recovery == 2 &&
5401− !(tmp_vram_lost_counter < atomic_read(&adev->vram_lost_counter)))
5402− amdgpu_device_recheck_guilty_jobs(
5403− tmp_adev, device_list_handle, reset_context);
5404−
54055302 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
54065303 struct amdgpu_ring *ring = tmp_adev->rings[i];
54075304
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c+1−1
@@ -533,7 +533,7 @@ module_param_named(compute_multipipe, amdgpu_compute_multipipe, int, 0444); · 532 unmodified lines
533533 * DOC: gpu_recovery (int)
534534 * Set to enable GPU recovery mechanism (1 = enable, 0 = disable). The default is -1 (auto, disabled except SRIOV).
535535 */
536−MODULE_PARM_DESC(gpu_recovery, "Enable GPU recovery mechanism, (2 = advanced tdr mode, 1 = enable, 0 = disable, -1 = auto)");
536+MODULE_PARM_DESC(gpu_recovery, "Enable GPU recovery mechanism, (1 = enable, 0 = disable, -1 = auto)");
537537 module_param_named(gpu_recovery, amdgpu_gpu_recovery, int, 0444);
538538
539539 /**
drivers/gpu/drm/scheduler/sched_main.c+9−49
@@ -443,27 +443,6 @@ static void drm_sched_job_timedout(struct work_struct *work) · 442 unmodified lines
443443 }
444444 }
445445
446− /**
447− * drm_sched_increase_karma - Update sched_entity guilty flag
448− *
449− * @bad: The job guilty of time out
450− *
451− * Increment on every hang caused by the 'bad' job. If this exceeds the hang
452− * limit of the scheduler then the respective sched entity is marked guilty and
453− * jobs from it will not be scheduled further
454− */
455−void drm_sched_increase_karma(struct drm_sched_job *bad)
456−{
457− drm_sched_increase_karma_ext(bad, 1);
458−}
459−EXPORT_SYMBOL(drm_sched_increase_karma);
460−
461−void drm_sched_reset_karma(struct drm_sched_job *bad)
462−{
463− drm_sched_increase_karma_ext(bad, 0);
464−}
465−EXPORT_SYMBOL(drm_sched_reset_karma);
466−
467446 /**
468447 * drm_sched_stop - stop the scheduler
469448 *
@@ -604,32 +583,15 @@ EXPORT_SYMBOL(drm_sched_start); · 134 unmodified lines
604583 *
605584 */
606585 void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
607−{
608− drm_sched_resubmit_jobs_ext(sched, INT_MAX);
609−}
610−EXPORT_SYMBOL(drm_sched_resubmit_jobs);
611−
612−/**
613− * drm_sched_resubmit_jobs_ext - helper to relunch certain number of jobs from mirror ring list
614− *
615− * @sched: scheduler instance
616− * @max: job numbers to relaunch
617− *
618− */
619−void drm_sched_resubmit_jobs_ext(struct drm_gpu_scheduler *sched, int max)
620586 {
621587 struct drm_sched_job *s_job, *tmp;
622588 uint64_t guilty_context;
623589 bool found_guilty = false;
624590 struct dma_fence *fence;
625− int i = 0;
626591
627592 list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
628593 struct drm_sched_fence *s_fence = s_job->s_fence;
629594
630− if (i >= max)
631− break;
632−
633595 if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {
634596 found_guilty = true;
635597 guilty_context = s_job->s_fence->scheduled.context;
@@ -639,7 +601,6 @@ void drm_sched_resubmit_jobs_ext(struct drm_gpu_scheduler *sched, int max) · 3 unmodified lines
639601 dma_fence_set_error(&s_fence->finished, -ECANCELED);
640602
641603 fence = sched->ops->run_job(s_job);
642− i++;
643604
644605 if (IS_ERR_OR_NULL(fence)) {
645606 if (IS_ERR(fence))
@@ -655,7 +616,7 @@ void drm_sched_resubmit_jobs_ext(struct drm_gpu_scheduler *sched, int max) · 9 unmodified lines
655616 }
656617 }
657618 }
658−EXPORT_SYMBOL(drm_sched_resubmit_jobs_ext);
619+EXPORT_SYMBOL(drm_sched_resubmit_jobs);
659620
660621 /**
661622 * drm_sched_job_init - init a scheduler job
@@ -1172,13 +1133,15 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) · 510 unmodified lines
11721133 EXPORT_SYMBOL(drm_sched_fini);
11731134
11741135 /**
1175− * drm_sched_increase_karma_ext - Update sched_entity guilty flag
1136+ * drm_sched_increase_karma - Update sched_entity guilty flag
11761137 *
11771138 * @bad: The job guilty of time out
1178− * @type: type for increase/reset karma
11791139 *
1140+ * Increment on every hang caused by the 'bad' job. If this exceeds the hang
1141+ * limit of the scheduler then the respective sched entity is marked guilty and
1142+ * jobs from it will not be scheduled further
11801143 */
1181−void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type)
1144+void drm_sched_increase_karma(struct drm_sched_job *bad)
11821145 {
11831146 int i;
11841147 struct drm_sched_entity *tmp;
@@ -1190,10 +1153,7 @@ void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type) · 5 unmodified lines
11901153 * corrupt but keep in mind that kernel jobs always considered good.
11911154 */
11921155 if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
1193− if (type == 0)
1194− atomic_set(&bad->karma, 0);
1195− else if (type == 1)
1196− atomic_inc(&bad->karma);
1156+ atomic_inc(&bad->karma);
11971157
11981158 for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;
11991159 i++) {
@@ -1204,7 +1164,7 @@ void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type) · 4 unmodified lines
12041164 if (bad->s_fence->scheduled.context ==
12051165 entity->fence_context) {
12061166 if (entity->guilty)
1207− atomic_set(entity->guilty, type);
1167+ atomic_set(entity->guilty, 1);
12081168 break;
12091169 }
12101170 }
@@ -1214,4 +1174,4 @@ void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type) · 3 unmodified lines
12141174 }
12151175 }
12161176 }
1217−EXPORT_SYMBOL(drm_sched_increase_karma_ext);
1177+EXPORT_SYMBOL(drm_sched_increase_karma);
include/drm/gpu_scheduler.h+0−3
@@ -528,10 +528,7 @@ void drm_sched_wakeup(struct drm_gpu_scheduler *sched); · 527 unmodified lines
528528 void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad);
529529 void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery);
530530 void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
531−void drm_sched_resubmit_jobs_ext(struct drm_gpu_scheduler *sched, int max);
532531 void drm_sched_increase_karma(struct drm_sched_job *bad);
533−void drm_sched_reset_karma(struct drm_sched_job *bad);
534−void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type);
535532 bool drm_sched_dependency_optimized(struct dma_fence* fence,
536533 struct drm_sched_entity *entity);
537534 void drm_sched_fault(struct drm_gpu_scheduler *sched);