History

drm/amdgpu: revert "implement tdr advanced mode"

This reverts commit e6c6338f393b74ac0b303d567bb918b44ae7ad75. This feature basically re-submits one job after another to figure out which one was the one causing a hang. This is obviously incompatible with gang-submit which requires that multiple jobs run at the same time. It's also absolutely not helpful to crash the hardware multiple times if a clean recovery is desired. For testing and debugging environments we should rather disable recovery alltogether to be able to inspect the state with a hw debugger. Additional to that the sw implementation is clearly buggy and causes reference count issues for the hardware fence.

Christian König committed 4 years ago

Commit 06a2d7c1 parent d27252b4 files · +10 −156Browse files

Unified · Split

4 files
  1. drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +0 −103
  2. drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +1 −1
  3. drivers/gpu/drm/scheduler/sched_main.c +9 −49
  4. include/drm/gpu_scheduler.h +0 −3
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c+0−103
@@ -5088,94 +5088,6 @@ static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev) · 5087 unmodified lines
5088 return 0;5088 return 0;
5089}5089}
5090 5090
5091static void amdgpu_device_recheck_guilty_jobs(
5092 struct amdgpu_device *adev, struct list_head *device_list_handle,
5093 struct amdgpu_reset_context *reset_context)
5094{
5095 int i, r = 0;
5096
5097 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5098 struct amdgpu_ring *ring = adev->rings[i];
5099 int ret = 0;
5100 struct drm_sched_job *s_job;
5101
5102 if (!ring || !ring->sched.thread)
5103 continue;
5104
5105 s_job = list_first_entry_or_null(&ring->sched.pending_list,
5106 struct drm_sched_job, list);
5107 if (s_job == NULL)
5108 continue;
5109
5110 /* clear job's guilty and depend the folowing step to decide the real one */
5111 drm_sched_reset_karma(s_job);
5112 drm_sched_resubmit_jobs_ext(&ring->sched, 1);
5113
5114 if (!s_job->s_fence->parent) {
5115 DRM_WARN("Failed to get a HW fence for job!");
5116 continue;
5117 }
5118
5119 ret = dma_fence_wait_timeout(s_job->s_fence->parent, false, ring->sched.timeout);
5120 if (ret == 0) { /* timeout */
5121 DRM_ERROR("Found the real bad job! ring:%s, job_id:%llx\n",
5122 ring->sched.name, s_job->id);
5123
5124
5125 amdgpu_fence_driver_isr_toggle(adev, true);
5126
5127 /* Clear this failed job from fence array */
5128 amdgpu_fence_driver_clear_job_fences(ring);
5129
5130 amdgpu_fence_driver_isr_toggle(adev, false);
5131
5132 /* Since the job won't signal and we go for
5133 * another resubmit drop this parent pointer
5134 */
5135 dma_fence_put(s_job->s_fence->parent);
5136 s_job->s_fence->parent = NULL;
5137
5138 /* set guilty */
5139 drm_sched_increase_karma(s_job);
5140 amdgpu_reset_prepare_hwcontext(adev, reset_context);
5141retry:
5142 /* do hw reset */
5143 if (amdgpu_sriov_vf(adev)) {
5144 amdgpu_virt_fini_data_exchange(adev);
5145 r = amdgpu_device_reset_sriov(adev, false);
5146 if (r)
5147 adev->asic_reset_res = r;
5148 } else {
5149 clear_bit(AMDGPU_SKIP_HW_RESET,
5150 &reset_context->flags);
5151 r = amdgpu_do_asic_reset(device_list_handle,
5152 reset_context);
5153 if (r && r == -EAGAIN)
5154 goto retry;
5155 }
5156
5157 /*
5158 * add reset counter so that the following
5159 * resubmitted job could flush vmid
5160 */
5161 atomic_inc(&adev->gpu_reset_counter);
5162 continue;
5163 }
5164
5165 /* got the hw fence, signal finished fence */
5166 atomic_dec(ring->sched.score);
5167 dma_fence_get(&s_job->s_fence->finished);
5168 dma_fence_signal(&s_job->s_fence->finished);
5169 dma_fence_put(&s_job->s_fence->finished);
5170
5171 /* remove node from list and free the job */
5172 spin_lock(&ring->sched.job_list_lock);
5173 list_del_init(&s_job->list);
5174 spin_unlock(&ring->sched.job_list_lock);
5175 ring->sched.ops->free_job(s_job);
5176 }
5177}
5178
5179static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)5091static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5180{5092{
5181 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);5093 struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
@@ -5196,7 +5108,6 @@ static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev) · 14 unmodified lines
5196 5108
5197}5109}
5198 5110
5199
5200/**5111/**
5201 * amdgpu_device_gpu_recover - reset the asic and recover scheduler5112 * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5202 *5113 *
@@ -5219,7 +5130,6 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · 16 unmodified lines
5219 int i, r = 0;5130 int i, r = 0;
5220 bool need_emergency_restart = false;5131 bool need_emergency_restart = false;
5221 bool audio_suspended = false;5132 bool audio_suspended = false;
5222 int tmp_vram_lost_counter;
5223 bool gpu_reset_for_dev_remove = false;5133 bool gpu_reset_for_dev_remove = false;
5224 5134
5225 gpu_reset_for_dev_remove =5135 gpu_reset_for_dev_remove =
@@ -5365,7 +5275,6 @@ retry: /* Rest of adevs pre asic reset from XGMI hive. */ · 139 unmodified lines
5365 amdgpu_device_stop_pending_resets(tmp_adev);5275 amdgpu_device_stop_pending_resets(tmp_adev);
5366 }5276 }
5367 5277
5368 tmp_vram_lost_counter = atomic_read(&((adev)->vram_lost_counter));
5369 /* Actual ASIC resets if needed.*/5278 /* Actual ASIC resets if needed.*/
5370 /* Host driver will handle XGMI hive reset for SRIOV */5279 /* Host driver will handle XGMI hive reset for SRIOV */
5371 if (amdgpu_sriov_vf(adev)) {5280 if (amdgpu_sriov_vf(adev)) {
@@ -5390,18 +5299,6 @@ skip_hw_reset: · 18 unmodified lines
5390 /* Post ASIC reset for all devs .*/5299 /* Post ASIC reset for all devs .*/
5391 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {5300 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5392 5301
5393 /*
5394 * Sometimes a later bad compute job can block a good gfx job as gfx
5395 * and compute ring share internal GC HW mutually. We add an additional
5396 * guilty jobs recheck step to find the real guilty job, it synchronously
5397 * submits and pends for the first job being signaled. If it gets timeout,
5398 * we identify it as a real guilty job.
5399 */
5400 if (amdgpu_gpu_recovery == 2 &&
5401 !(tmp_vram_lost_counter < atomic_read(&adev->vram_lost_counter)))
5402 amdgpu_device_recheck_guilty_jobs(
5403 tmp_adev, device_list_handle, reset_context);
5404
5405 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {5302 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5406 struct amdgpu_ring *ring = tmp_adev->rings[i];5303 struct amdgpu_ring *ring = tmp_adev->rings[i];
5407 5304
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c+1−1
@@ -533,7 +533,7 @@ module_param_named(compute_multipipe, amdgpu_compute_multipipe, int, 0444); · 532 unmodified lines
533 * DOC: gpu_recovery (int)533 * DOC: gpu_recovery (int)
534 * Set to enable GPU recovery mechanism (1 = enable, 0 = disable). The default is -1 (auto, disabled except SRIOV).534 * Set to enable GPU recovery mechanism (1 = enable, 0 = disable). The default is -1 (auto, disabled except SRIOV).
535 */535 */
536MODULE_PARM_DESC(gpu_recovery, "Enable GPU recovery mechanism, (2 = advanced tdr mode, 1 = enable, 0 = disable, -1 = auto)");536MODULE_PARM_DESC(gpu_recovery, "Enable GPU recovery mechanism, (1 = enable, 0 = disable, -1 = auto)");
537module_param_named(gpu_recovery, amdgpu_gpu_recovery, int, 0444);537module_param_named(gpu_recovery, amdgpu_gpu_recovery, int, 0444);
538 538
539/**539/**
drivers/gpu/drm/scheduler/sched_main.c+9−49
@@ -443,27 +443,6 @@ static void drm_sched_job_timedout(struct work_struct *work) · 442 unmodified lines
443 }443 }
444}444}
445 445
446 /**
447 * drm_sched_increase_karma - Update sched_entity guilty flag
448 *
449 * @bad: The job guilty of time out
450 *
451 * Increment on every hang caused by the 'bad' job. If this exceeds the hang
452 * limit of the scheduler then the respective sched entity is marked guilty and
453 * jobs from it will not be scheduled further
454 */
455void drm_sched_increase_karma(struct drm_sched_job *bad)
456{
457 drm_sched_increase_karma_ext(bad, 1);
458}
459EXPORT_SYMBOL(drm_sched_increase_karma);
460
461void drm_sched_reset_karma(struct drm_sched_job *bad)
462{
463 drm_sched_increase_karma_ext(bad, 0);
464}
465EXPORT_SYMBOL(drm_sched_reset_karma);
466
467/**446/**
468 * drm_sched_stop - stop the scheduler447 * drm_sched_stop - stop the scheduler
469 *448 *
@@ -604,32 +583,15 @@ EXPORT_SYMBOL(drm_sched_start); · 134 unmodified lines
604 *583 *
605 */584 */
606void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)585void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched)
607{
608 drm_sched_resubmit_jobs_ext(sched, INT_MAX);
609}
610EXPORT_SYMBOL(drm_sched_resubmit_jobs);
611
612/**
613 * drm_sched_resubmit_jobs_ext - helper to relunch certain number of jobs from mirror ring list
614 *
615 * @sched: scheduler instance
616 * @max: job numbers to relaunch
617 *
618 */
619void drm_sched_resubmit_jobs_ext(struct drm_gpu_scheduler *sched, int max)
620{586{
621 struct drm_sched_job *s_job, *tmp;587 struct drm_sched_job *s_job, *tmp;
622 uint64_t guilty_context;588 uint64_t guilty_context;
623 bool found_guilty = false;589 bool found_guilty = false;
624 struct dma_fence *fence;590 struct dma_fence *fence;
625 int i = 0;
626 591
627 list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {592 list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
628 struct drm_sched_fence *s_fence = s_job->s_fence;593 struct drm_sched_fence *s_fence = s_job->s_fence;
629 594
630 if (i >= max)
631 break;
632
633 if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {595 if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) {
634 found_guilty = true;596 found_guilty = true;
635 guilty_context = s_job->s_fence->scheduled.context;597 guilty_context = s_job->s_fence->scheduled.context;
@@ -639,7 +601,6 @@ void drm_sched_resubmit_jobs_ext(struct drm_gpu_scheduler *sched, int max) · 3 unmodified lines
639 dma_fence_set_error(&s_fence->finished, -ECANCELED);601 dma_fence_set_error(&s_fence->finished, -ECANCELED);
640 602
641 fence = sched->ops->run_job(s_job);603 fence = sched->ops->run_job(s_job);
642 i++;
643 604
644 if (IS_ERR_OR_NULL(fence)) {605 if (IS_ERR_OR_NULL(fence)) {
645 if (IS_ERR(fence))606 if (IS_ERR(fence))
@@ -655,7 +616,7 @@ void drm_sched_resubmit_jobs_ext(struct drm_gpu_scheduler *sched, int max) · 9 unmodified lines
655 }616 }
656 }617 }
657}618}
658EXPORT_SYMBOL(drm_sched_resubmit_jobs_ext);619EXPORT_SYMBOL(drm_sched_resubmit_jobs);
659 620
660/**621/**
661 * drm_sched_job_init - init a scheduler job622 * drm_sched_job_init - init a scheduler job
@@ -1172,13 +1133,15 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) · 510 unmodified lines
1172EXPORT_SYMBOL(drm_sched_fini);1133EXPORT_SYMBOL(drm_sched_fini);
1173 1134
1174/**1135/**
1175 * drm_sched_increase_karma_ext - Update sched_entity guilty flag1136 * drm_sched_increase_karma - Update sched_entity guilty flag
1176 *1137 *
1177 * @bad: The job guilty of time out1138 * @bad: The job guilty of time out
1178 * @type: type for increase/reset karma
1179 *1139 *
1140 * Increment on every hang caused by the 'bad' job. If this exceeds the hang
1141 * limit of the scheduler then the respective sched entity is marked guilty and
1142 * jobs from it will not be scheduled further
1180 */1143 */
1181void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type)1144void drm_sched_increase_karma(struct drm_sched_job *bad)
1182{1145{
1183 int i;1146 int i;
1184 struct drm_sched_entity *tmp;1147 struct drm_sched_entity *tmp;
@@ -1190,10 +1153,7 @@ void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type) · 5 unmodified lines
1190 * corrupt but keep in mind that kernel jobs always considered good.1153 * corrupt but keep in mind that kernel jobs always considered good.
1191 */1154 */
1192 if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {1155 if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) {
1193 if (type == 0)1156 atomic_inc(&bad->karma);
1194 atomic_set(&bad->karma, 0);
1195 else if (type == 1)
1196 atomic_inc(&bad->karma);
1197 1157
1198 for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;1158 for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL;
1199 i++) {1159 i++) {
@@ -1204,7 +1164,7 @@ void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type) · 4 unmodified lines
1204 if (bad->s_fence->scheduled.context ==1164 if (bad->s_fence->scheduled.context ==
1205 entity->fence_context) {1165 entity->fence_context) {
1206 if (entity->guilty)1166 if (entity->guilty)
1207 atomic_set(entity->guilty, type);1167 atomic_set(entity->guilty, 1);
1208 break;1168 break;
1209 }1169 }
1210 }1170 }
@@ -1214,4 +1174,4 @@ void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type) · 3 unmodified lines
1214 }1174 }
1215 }1175 }
1216}1176}
1217EXPORT_SYMBOL(drm_sched_increase_karma_ext);1177EXPORT_SYMBOL(drm_sched_increase_karma);
include/drm/gpu_scheduler.h+0−3
@@ -528,10 +528,7 @@ void drm_sched_wakeup(struct drm_gpu_scheduler *sched); · 527 unmodified lines
528void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad);528void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad);
529void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery);529void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery);
530void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);530void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched);
531void drm_sched_resubmit_jobs_ext(struct drm_gpu_scheduler *sched, int max);
532void drm_sched_increase_karma(struct drm_sched_job *bad);531void drm_sched_increase_karma(struct drm_sched_job *bad);
533void drm_sched_reset_karma(struct drm_sched_job *bad);
534void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type);
535bool drm_sched_dependency_optimized(struct dma_fence* fence,532bool drm_sched_dependency_optimized(struct dma_fence* fence,
536 struct drm_sched_entity *entity);533 struct drm_sched_entity *entity);
537void drm_sched_fault(struct drm_gpu_scheduler *sched);534void drm_sched_fault(struct drm_gpu_scheduler *sched);