History

Merge tag 'drm-amdkfd-next-2018-07-28' of git://people.freedesktop.org/~gabbayo/linux into drm-next

This is amdkfd pull for 4.19. The major changes are: - Add Raven support. Raven refers to Ryzen APUs with integrated GFXv9 GPU. - Integrate GPU reset support In addition, there are a couple of small fixes and improvements, such as: - Better handling and reporting to user of VM faults - Fix race upon context restore - Allow the user to use specific Compute Units - Basic power management

Dave Airlie committed 8 years ago

Commit ecd79632 parents 6d52aac b5aa3f445 files · +1460 −447Browse files

Unified · Split

45 files
  1. drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +35 −0
  2. drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +10 −0
  3. drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c +27 −0
  4. drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c +9 −0
  5. drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +8 −0
  6. drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +14 −0
  7. drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +6 −1
  8. drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +2 −0
  9. drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +8 −5
  10. drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c +32 −1
  11. drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c +32 −1
  12. drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c +49 −5
  13. drivers/gpu/drm/amd/amdkfd/cik_int.h +4 −3
  14. drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h +233 −225
  15. drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm +15 −3
  16. drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm +14 −2
  17. drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +63 −6
  18. drivers/gpu/drm/amd/amdkfd/kfd_crat.c +46 −11
  19. drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c +0 −1
  20. drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h +37 −0
  21. drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c +48 −0
  22. drivers/gpu/drm/amd/amdkfd/kfd_device.c +115 −7
  23. drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +177 −106
  24. drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h +26 −6
  25. drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c +1 −1
  26. drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c +5 −4
  27. drivers/gpu/drm/amd/amdkfd/kfd_events.c +98 −20
  28. drivers/gpu/drm/amd/amdkfd/kfd_events.h +1 −0
  29. drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c +19 −3
  30. drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c +4 −2
  31. drivers/gpu/drm/amd/amdkfd/kfd_iommu.c +1 −1
  32. drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c +10 −8
  33. drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h +1 −1
  34. drivers/gpu/drm/amd/amdkfd/kfd_module.c +12 −4
  35. drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c +40 −1
  36. drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h +4 −0
  37. drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c +28 −1
  38. drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +29 −2
  39. drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c +28 −1
  40. drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c +26 −0
  41. drivers/gpu/drm/amd/amdkfd/kfd_priv.h +35 −5
  42. drivers/gpu/drm/amd/amdkfd/kfd_process.c +2 −0
  43. drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +36 −7
  44. drivers/gpu/drm/amd/include/kgd_kfd_interface.h +40 −0
  45. include/uapi/linux/kfd_ioctl.h +30 −3
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c+35−0
@@ -243,6 +243,33 @@ int amdgpu_amdkfd_resume(struct amdgpu_device *adev) · 242 unmodified lines
243243 return r;
244244 }
245245
246+int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev)
247+{
248+ int r = 0;
249+
250+ if (adev->kfd)
251+ r = kgd2kfd->pre_reset(adev->kfd);
252+
253+ return r;
254+}
255+
256+int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev)
257+{
258+ int r = 0;
259+
260+ if (adev->kfd)
261+ r = kgd2kfd->post_reset(adev->kfd);
262+
263+ return r;
264+}
265+
266+void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd)
267+{
268+ struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
269+
270+ amdgpu_device_gpu_recover(adev, NULL, false);
271+}
272+
246273 int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
247274 void **mem_obj, uint64_t *gpu_addr,
248275 void **cpu_ptr)
@@ -461,6 +488,14 @@ err: · 212 unmodified lines
461488 return ret;
462489 }
463490
491+void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle)
492+{
493+ struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
494+
495+ amdgpu_dpm_switch_power_profile(adev,
496+ PP_SMC_POWER_PROFILE_COMPUTE, !idle);
497+}
498+
464499 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid)
465500 {
466501 if (adev->kfd) {
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h+10−0
@@ -119,6 +119,7 @@ int amdgpu_amdkfd_evict_userptr(struct kgd_mem *mem, struct mm_struct *mm); · 118 unmodified lines
119119 int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
120120 uint32_t vmid, uint64_t gpu_addr,
121121 uint32_t *ib_cmd, uint32_t ib_len);
122+void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);
122123
123124 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
124125 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
@@ -126,6 +127,12 @@ struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void); · 1 unmodified line
126127
127128 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
128129
130+int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev);
131+
132+int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev);
133+
134+void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd);
135+
129136 /* Shared API */
130137 int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
131138 void **mem_obj, uint64_t *gpu_addr,
@@ -183,6 +190,9 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd, · 51 unmodified lines
183190 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
184191 struct dma_fence **ef);
185192
193+int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
194+ struct kfd_vm_fault_info *info);
195+
186196 void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
187197 void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo);
188198
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c+27−0
@@ -145,6 +145,7 @@ static void set_vm_context_page_table_base(struct kgd_dev *kgd, uint32_t vmid, · 144 unmodified lines
145145 uint32_t page_table_base);
146146 static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid);
147147 static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid);
148+static uint32_t read_vmid_from_vmfault_reg(struct kgd_dev *kgd);
148149
149150 /* Because of REG_GET_FIELD() being used, we put this function in the
150151 * asic specific file.
@@ -216,6 +217,10 @@ static const struct kfd2kgd_calls kfd2kgd = { · 65 unmodified lines
216217 .invalidate_tlbs = invalidate_tlbs,
217218 .invalidate_tlbs_vmid = invalidate_tlbs_vmid,
218219 .submit_ib = amdgpu_amdkfd_submit_ib,
220+ .get_vm_fault_info = amdgpu_amdkfd_gpuvm_get_vm_fault_info,
221+ .read_vmid_from_vmfault_reg = read_vmid_from_vmfault_reg,
222+ .gpu_recover = amdgpu_amdkfd_gpu_reset,
223+ .set_compute_idle = amdgpu_amdkfd_set_compute_idle
219224 };
220225
221226 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void)
@@ -571,6 +576,9 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, · 349 unmodified lines
571576 unsigned long flags, end_jiffies;
572577 int retry;
573578
579+ if (adev->in_gpu_reset)
580+ return -EIO;
581+
574582 acquire_queue(kgd, pipe_id, queue_id);
575583 WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
576584
@@ -882,6 +890,9 @@ static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid) · 305 unmodified lines
882890 int vmid;
883891 unsigned int tmp;
884892
893+ if (adev->in_gpu_reset)
894+ return -EIO;
895+
885896 for (vmid = 0; vmid < 16; vmid++) {
886897 if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))
887898 continue;
@@ -911,3 +922,19 @@ static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid) · 23 unmodified lines
911922 RREG32(mmVM_INVALIDATE_RESPONSE);
912923 return 0;
913924 }
925+
926+ /**
927+ * read_vmid_from_vmfault_reg - read vmid from register
928+ *
929+ * adev: amdgpu_device pointer
930+ * @vmid: vmid pointer
931+ * read vmid from register (CIK).
932+ */
933+static uint32_t read_vmid_from_vmfault_reg(struct kgd_dev *kgd)
934+{
935+ struct amdgpu_device *adev = get_amdgpu_device(kgd);
936+
937+ uint32_t status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
938+
939+ return REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
940+}
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c+9−0
@@ -176,6 +176,9 @@ static const struct kfd2kgd_calls kfd2kgd = { · 175 unmodified lines
176176 .invalidate_tlbs = invalidate_tlbs,
177177 .invalidate_tlbs_vmid = invalidate_tlbs_vmid,
178178 .submit_ib = amdgpu_amdkfd_submit_ib,
179+ .get_vm_fault_info = amdgpu_amdkfd_gpuvm_get_vm_fault_info,
180+ .gpu_recover = amdgpu_amdkfd_gpu_reset,
181+ .set_compute_idle = amdgpu_amdkfd_set_compute_idle
179182 };
180183
181184 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void)
@@ -568,6 +571,9 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, · 386 unmodified lines
568571 int retry;
569572 struct vi_mqd *m = get_mqd(mqd);
570573
574+ if (adev->in_gpu_reset)
575+ return -EIO;
576+
571577 acquire_queue(kgd, pipe_id, queue_id);
572578
573579 if (m->cp_hqd_vmid == 0)
@@ -844,6 +850,9 @@ static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid) · 270 unmodified lines
844850 int vmid;
845851 unsigned int tmp;
846852
853+ if (adev->in_gpu_reset)
854+ return -EIO;
855+
847856 for (vmid = 0; vmid < 16; vmid++) {
848857 if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))
849858 continue;
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c+8−0
@@ -213,6 +213,8 @@ static const struct kfd2kgd_calls kfd2kgd = { · 212 unmodified lines
213213 .invalidate_tlbs = invalidate_tlbs,
214214 .invalidate_tlbs_vmid = invalidate_tlbs_vmid,
215215 .submit_ib = amdgpu_amdkfd_submit_ib,
216+ .gpu_recover = amdgpu_amdkfd_gpu_reset,
217+ .set_compute_idle = amdgpu_amdkfd_set_compute_idle
216218 };
217219
218220 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void)
@@ -679,6 +681,9 @@ static int kgd_hqd_destroy(struct kgd_dev *kgd, void *mqd, · 460 unmodified lines
679681 uint32_t temp;
680682 struct v9_mqd *m = get_mqd(mqd);
681683
684+ if (adev->in_gpu_reset)
685+ return -EIO;
686+
682687 acquire_queue(kgd, pipe_id, queue_id);
683688
684689 if (m->cp_hqd_vmid == 0)
@@ -866,6 +871,9 @@ static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid) · 181 unmodified lines
866871 int vmid;
867872 struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
868873
874+ if (adev->in_gpu_reset)
875+ return -EIO;
876+
869877 if (ring->ready)
870878 return invalidate_tlbs_with_kiq(adev, pasid);
871879
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c+14−0
@@ -1621,6 +1621,20 @@ bo_reserve_failed: · 1620 unmodified lines
16211621 return ret;
16221622 }
16231623
1624+int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
1625+ struct kfd_vm_fault_info *mem)
1626+{
1627+ struct amdgpu_device *adev;
1628+
1629+ adev = (struct amdgpu_device *)kgd;
1630+ if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) {
1631+ *mem = *adev->gmc.vm_fault_info;
1632+ mb();
1633+ atomic_set(&adev->gmc.vm_fault_info_updated, 0);
1634+ }
1635+ return 0;
1636+}
1637+
16241638 /* Evict a userptr BO by stopping the queues if necessary
16251639 *
16261640 * Runs in MMU notifier, may be in RECLAIM_FS context. This means it
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c+6−1
@@ -3307,6 +3307,9 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · 3306 unmodified lines
33073307 atomic_inc(&adev->gpu_reset_counter);
33083308 adev->in_gpu_reset = 1;
33093309
3310+ /* Block kfd */
3311+ amdgpu_amdkfd_pre_reset(adev);
3312+
33103313 /* block TTM */
33113314 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
33123315
@@ -3322,7 +3325,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · 9 unmodified lines
33223325 if (job && job->base.sched == &ring->sched)
33233326 continue;
33243327
3325− drm_sched_hw_job_reset(&ring->sched, &job->base);
3328+ drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
33263329
33273330 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
33283331 amdgpu_fence_driver_force_completion(ring);
@@ -3363,6 +3366,8 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, · 34 unmodified lines
33633366 dev_info(adev->dev, "GPU reset(%d) succeeded!\n",atomic_read(&adev->gpu_reset_counter));
33643367 }
33653368
3369+ /*unlock kfd */
3370+ amdgpu_amdkfd_post_reset(adev);
33663371 amdgpu_vf_error_trans_all(adev);
33673372 adev->in_gpu_reset = 0;
33683373 mutex_unlock(&adev->lock_reset);
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h+2−0
@@ -105,6 +105,8 @@ struct amdgpu_gmc { · 104 unmodified lines
105105 /* protects concurrent invalidation */
106106 spinlock_t invalidate_lock;
107107 bool translate_further;
108+ struct kfd_vm_fault_info *vm_fault_info;
109+ atomic_t vm_fault_info_updated;
108110
109111 const struct amdgpu_gmc_funcs *gmc_funcs;
110112 };
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c+8−5
@@ -495,11 +495,12 @@ static int amdgpu_vm_alloc_levels(struct amdgpu_device *adev, · 494 unmodified lines
495495 eaddr = eaddr & ((1 << shift) - 1);
496496
497497 flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
498+ if (vm->root.base.bo->shadow)
499+ flags |= AMDGPU_GEM_CREATE_SHADOW;
498500 if (vm->use_cpu_for_update)
499501 flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
500502 else
501− flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
502− AMDGPU_GEM_CREATE_SHADOW);
503+ flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
503504
504505 /* walk over the address space and allocate the page tables */
505506 for (pt_idx = from; pt_idx <= to; ++pt_idx) {
@@ -2587,7 +2588,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, · 2081 unmodified lines
25872588 flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
25882589 if (vm->use_cpu_for_update)
25892590 flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
2590− else
2591+ else if (vm_context != AMDGPU_VM_CONTEXT_COMPUTE)
25912592 flags |= AMDGPU_GEM_CREATE_SHADOW;
25922593
25932594 size = amdgpu_vm_bo_size(adev, adev->vm_manager.root_level);
@@ -2662,8 +2663,7 @@ error_free_sched_entity: · 68 unmodified lines
26622663 * - pasid (old PASID is released, because compute manages its own PASIDs)
26632664 *
26642665 * Reinitializes the page directory to reflect the changed ATS
2665− * setting. May leave behind an unused shadow BO for the page
2666− * directory when switching from SDMA updates to CPU updates.
2666+ * setting.
26672667 *
26682668 * Returns:
26692669 * 0 for success, -errno for errors.
@@ -2713,6 +2713,9 @@ int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm) · 43 unmodified lines
27132713 vm->pasid = 0;
27142714 }
27152715
2716+ /* Free the shadow bo for compute VM */
2717+ amdgpu_bo_unref(&vm->root.base.bo->shadow);
2718+
27162719 error:
27172720 amdgpu_bo_unreserve(vm->root.base.bo);
27182721 return r;
drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c+32−1
@@ -28,6 +28,7 @@ · 27 unmodified lines
2828 #include "cik.h"
2929 #include "gmc_v7_0.h"
3030 #include "amdgpu_ucode.h"
31+#include "amdgpu_amdkfd.h"
3132
3233 #include "bif/bif_4_1_d.h"
3334 #include "bif/bif_4_1_sh_mask.h"
@@ -1078,6 +1079,12 @@ static int gmc_v7_0_sw_init(void *handle) · 1044 unmodified lines
10781079 adev->vm_manager.vram_base_offset = 0;
10791080 }
10801081
1082+ adev->gmc.vm_fault_info = kmalloc(sizeof(struct kfd_vm_fault_info),
1083+ GFP_KERNEL);
1084+ if (!adev->gmc.vm_fault_info)
1085+ return -ENOMEM;
1086+ atomic_set(&adev->gmc.vm_fault_info_updated, 0);
1087+
10811088 return 0;
10821089 }
10831090
@@ -1087,6 +1094,7 @@ static int gmc_v7_0_sw_fini(void *handle) · 3 unmodified lines
10871094
10881095 amdgpu_gem_force_release(adev);
10891096 amdgpu_vm_manager_fini(adev);
1097+ kfree(adev->gmc.vm_fault_info);
10901098 gmc_v7_0_gart_fini(adev);
10911099 amdgpu_bo_fini(adev);
10921100 release_firmware(adev->gmc.fw);
@@ -1276,7 +1284,7 @@ static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev, · 183 unmodified lines
12761284 struct amdgpu_irq_src *source,
12771285 struct amdgpu_iv_entry *entry)
12781286 {
1279− u32 addr, status, mc_client;
1287+ u32 addr, status, mc_client, vmid;
12801288
12811289 addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
12821290 status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
@@ -1301,6 +1309,29 @@ static int gmc_v7_0_process_interrupt(struct amdgpu_device *adev, · 18 unmodified lines
13011309 entry->pasid);
13021310 }
13031311
1312+ vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
1313+ VMID);
1314+ if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid)
1315+ && !atomic_read(&adev->gmc.vm_fault_info_updated)) {
1316+ struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info;
1317+ u32 protections = REG_GET_FIELD(status,
1318+ VM_CONTEXT1_PROTECTION_FAULT_STATUS,
1319+ PROTECTIONS);
1320+
1321+ info->vmid = vmid;
1322+ info->mc_id = REG_GET_FIELD(status,
1323+ VM_CONTEXT1_PROTECTION_FAULT_STATUS,
1324+ MEMORY_CLIENT_ID);
1325+ info->status = status;
1326+ info->page_addr = addr;
1327+ info->prot_valid = protections & 0x7 ? true : false;
1328+ info->prot_read = protections & 0x8 ? true : false;
1329+ info->prot_write = protections & 0x10 ? true : false;
1330+ info->prot_exec = protections & 0x20 ? true : false;
1331+ mb();
1332+ atomic_set(&adev->gmc.vm_fault_info_updated, 1);
1333+ }
1334+
13041335 return 0;
13051336 }
13061337
drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c+32−1
@@ -26,6 +26,7 @@ · 25 unmodified lines
2626 #include "amdgpu.h"
2727 #include "gmc_v8_0.h"
2828 #include "amdgpu_ucode.h"
29+#include "amdgpu_amdkfd.h"
2930
3031 #include "gmc/gmc_8_1_d.h"
3132 #include "gmc/gmc_8_1_sh_mask.h"
@@ -1182,6 +1183,12 @@ static int gmc_v8_0_sw_init(void *handle) · 1150 unmodified lines
11821183 adev->vm_manager.vram_base_offset = 0;
11831184 }
11841185
1186+ adev->gmc.vm_fault_info = kmalloc(sizeof(struct kfd_vm_fault_info),
1187+ GFP_KERNEL);
1188+ if (!adev->gmc.vm_fault_info)
1189+ return -ENOMEM;
1190+ atomic_set(&adev->gmc.vm_fault_info_updated, 0);
1191+
11851192 return 0;
11861193 }
11871194
@@ -1191,6 +1198,7 @@ static int gmc_v8_0_sw_fini(void *handle) · 3 unmodified lines
11911198
11921199 amdgpu_gem_force_release(adev);
11931200 amdgpu_vm_manager_fini(adev);
1201+ kfree(adev->gmc.vm_fault_info);
11941202 gmc_v8_0_gart_fini(adev);
11951203 amdgpu_bo_fini(adev);
11961204 release_firmware(adev->gmc.fw);
@@ -1426,7 +1434,7 @@ static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev, · 229 unmodified lines
14261434 struct amdgpu_irq_src *source,
14271435 struct amdgpu_iv_entry *entry)
14281436 {
1429− u32 addr, status, mc_client;
1437+ u32 addr, status, mc_client, vmid;
14301438
14311439 if (amdgpu_sriov_vf(adev)) {
14321440 dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
@@ -1463,6 +1471,29 @@ static int gmc_v8_0_process_interrupt(struct amdgpu_device *adev, · 30 unmodified lines
14631471 entry->pasid);
14641472 }
14651473
1474+ vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
1475+ VMID);
1476+ if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid)
1477+ && !atomic_read(&adev->gmc.vm_fault_info_updated)) {
1478+ struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info;
1479+ u32 protections = REG_GET_FIELD(status,
1480+ VM_CONTEXT1_PROTECTION_FAULT_STATUS,
1481+ PROTECTIONS);
1482+
1483+ info->vmid = vmid;
1484+ info->mc_id = REG_GET_FIELD(status,
1485+ VM_CONTEXT1_PROTECTION_FAULT_STATUS,
1486+ MEMORY_CLIENT_ID);
1487+ info->status = status;
1488+ info->page_addr = addr;
1489+ info->prot_valid = protections & 0x7 ? true : false;
1490+ info->prot_read = protections & 0x8 ? true : false;
1491+ info->prot_write = protections & 0x10 ? true : false;
1492+ info->prot_exec = protections & 0x20 ? true : false;
1493+ mb();
1494+ atomic_set(&adev->gmc.vm_fault_info_updated, 1);
1495+ }
1496+
14661497 return 0;
14671498 }
14681499
drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c+49−5
@@ -25,12 +25,39 @@ · 24 unmodified lines
2525 #include "cik_int.h"
2626
2727 static bool cik_event_interrupt_isr(struct kfd_dev *dev,
28− const uint32_t *ih_ring_entry)
28+ const uint32_t *ih_ring_entry,
29+ uint32_t *patched_ihre,
30+ bool *patched_flag)
2931 {
3032 const struct cik_ih_ring_entry *ihre =
3133 (const struct cik_ih_ring_entry *)ih_ring_entry;
34+ const struct kfd2kgd_calls *f2g = dev->kfd2kgd;
3235 unsigned int vmid, pasid;
3336
37+ /* This workaround is due to HW/FW limitation on Hawaii that
38+ * VMID and PASID are not written into ih_ring_entry
39+ */
40+ if ((ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
41+ ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) &&
42+ dev->device_info->asic_family == CHIP_HAWAII) {
43+ struct cik_ih_ring_entry *tmp_ihre =
44+ (struct cik_ih_ring_entry *)patched_ihre;
45+
46+ *patched_flag = true;
47+ *tmp_ihre = *ihre;
48+
49+ vmid = f2g->read_vmid_from_vmfault_reg(dev->kgd);
50+ pasid = f2g->get_atc_vmid_pasid_mapping_pasid(dev->kgd, vmid);
51+
52+ tmp_ihre->ring_id &= 0x000000ff;
53+ tmp_ihre->ring_id |= vmid << 8;
54+ tmp_ihre->ring_id |= pasid << 16;
55+
56+ return (pasid != 0) &&
57+ vmid >= dev->vm_info.first_vmid_kfd &&
58+ vmid <= dev->vm_info.last_vmid_kfd;
59+ }
60+
3461 /* Only handle interrupts from KFD VMIDs */
3562 vmid = (ihre->ring_id & 0x0000ff00) >> 8;
3663 if (vmid < dev->vm_info.first_vmid_kfd ||
@@ -48,18 +75,19 @@ static bool cik_event_interrupt_isr(struct kfd_dev *dev, · 11 unmodified lines
4875 return ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE ||
4976 ihre->source_id == CIK_INTSRC_SDMA_TRAP ||
5077 ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG ||
51− ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE;
78+ ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE ||
79+ ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
80+ ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT;
5281 }
5382
5483 static void cik_event_interrupt_wq(struct kfd_dev *dev,
5584 const uint32_t *ih_ring_entry)
5685 {
57− unsigned int pasid;
5886 const struct cik_ih_ring_entry *ihre =
5987 (const struct cik_ih_ring_entry *)ih_ring_entry;
6088 uint32_t context_id = ihre->data & 0xfffffff;
61−
62− pasid = (ihre->ring_id & 0xffff0000) >> 16;
89+ unsigned int vmid = (ihre->ring_id & 0x0000ff00) >> 8;
90+ unsigned int pasid = (ihre->ring_id & 0xffff0000) >> 16;
6391
6492 if (pasid == 0)
6593 return;
@@ -72,6 +100,22 @@ static void cik_event_interrupt_wq(struct kfd_dev *dev, · 6 unmodified lines
72100 kfd_signal_event_interrupt(pasid, context_id & 0xff, 8);
73101 else if (ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE)
74102 kfd_signal_hw_exception_event(pasid);
103+ else if (ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
104+ ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) {
105+ struct kfd_vm_fault_info info;
106+
107+ kfd_process_vm_fault(dev->dqm, pasid);
108+
109+ memset(&info, 0, sizeof(info));
110+ dev->kfd2kgd->get_vm_fault_info(dev->kgd, &info);
111+ if (!info.page_addr && !info.status)
112+ return;
113+
114+ if (info.vmid == vmid)
115+ kfd_signal_vm_fault_event(dev, pasid, &info);
116+ else
117+ kfd_signal_vm_fault_event(dev, pasid, NULL);
118+ }
75119 }
76120
77121 const struct kfd_event_interrupt_class event_interrupt_class_cik = {
drivers/gpu/drm/amd/amdkfd/cik_int.h+4−3
@@ -20,8 +20,8 @@ · 19 unmodified lines
2020 * OTHER DEALINGS IN THE SOFTWARE.
2121 */
2222
23−#ifndef HSA_RADEON_CIK_INT_H_INCLUDED
24−#define HSA_RADEON_CIK_INT_H_INCLUDED
23+#ifndef CIK_INT_H_INCLUDED
24+#define CIK_INT_H_INCLUDED
2525
2626 #include <linux/types.h>
2727
@@ -34,9 +34,10 @@ struct cik_ih_ring_entry { · 6 unmodified lines
3434
3535 #define CIK_INTSRC_CP_END_OF_PIPE 0xB5
3636 #define CIK_INTSRC_CP_BAD_OPCODE 0xB7
37−#define CIK_INTSRC_DEQUEUE_COMPLETE 0xC6
3837 #define CIK_INTSRC_SDMA_TRAP 0xE0
3938 #define CIK_INTSRC_SQ_INTERRUPT_MSG 0xEF
39+#define CIK_INTSRC_GFX_PAGE_INV_FAULT 0x92
40+#define CIK_INTSRC_GFX_MEM_PROT_FAULT 0x93
4041
4142 #endif
4243
drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h+233−225
@@ -21,18 +21,21 @@ · 20 unmodified lines
2121 */
2222
2323 static const uint32_t cwsr_trap_gfx8_hex[] = {
24− 0xbf820001, 0xbf820125,
24+ 0xbf820001, 0xbf82012b,
2525 0xb8f4f802, 0x89748674,
2626 0xb8f5f803, 0x8675ff75,
27− 0x00000400, 0xbf850011,
27+ 0x00000400, 0xbf850017,
2828 0xc00a1e37, 0x00000000,
2929 0xbf8c007f, 0x87777978,
30− 0xbf840002, 0xb974f802,
31− 0xbe801d78, 0xb8f5f803,
32− 0x8675ff75, 0x000001ff,
33− 0xbf850002, 0x80708470,
34− 0x82718071, 0x8671ff71,
35− 0x0000ffff, 0xb974f802,
30+ 0xbf840005, 0x8f728374,
31+ 0xb972e0c2, 0xbf800002,
32+ 0xb9740002, 0xbe801d78,
33+ 0xb8f5f803, 0x8675ff75,
34+ 0x000001ff, 0xbf850002,
35+ 0x80708470, 0x82718071,
36+ 0x8671ff71, 0x0000ffff,
37+ 0x8f728374, 0xb972e0c2,
38+ 0xbf800002, 0xb9740002,
3639 0xbe801f70, 0xb8f5f803,
3740 0x8675ff75, 0x00000100,
3841 0xbf840006, 0xbefa0080,
@@ -168,7 +171,7 @@ static const uint32_t cwsr_trap_gfx8_hex[] = { · 129 unmodified lines
168171 0x807c847c, 0x806eff6e,
169172 0x00000400, 0xbf0a757c,
170173 0xbf85ffef, 0xbf9c0000,
171− 0xbf8200ca, 0xbef8007e,
174+ 0xbf8200cd, 0xbef8007e,
172175 0x8679ff7f, 0x0000ffff,
173176 0x8779ff79, 0x00040000,
174177 0xbefa0080, 0xbefb00ff,
@@ -268,16 +271,18 @@ static const uint32_t cwsr_trap_gfx8_hex[] = { · 93 unmodified lines
268271 0x8f739773, 0xb976f807,
269272 0x8671ff71, 0x0000ffff,
270273 0x86fe7e7e, 0x86ea6a6a,
271− 0xb974f802, 0xbf8a0000,
272− 0x95807370, 0xbf810000,
274+ 0x8f768374, 0xb976e0c2,
275+ 0xbf800002, 0xb9740002,
276+ 0xbf8a0000, 0x95807370,
277+ 0xbf810000, 0x00000000,
273278 };
274279
275280
276281 static const uint32_t cwsr_trap_gfx9_hex[] = {
277− 0xbf820001, 0xbf82015a,
282+ 0xbf820001, 0xbf82015d,
278283 0xb8f8f802, 0x89788678,
279284 0xb8f1f803, 0x866eff71,
280− 0x00000400, 0xbf850034,
285+ 0x00000400, 0xbf850037,
281286 0x866eff71, 0x00000800,
282287 0xbf850003, 0x866eff71,
283288 0x00000100, 0xbf840008,
@@ -303,258 +308,261 @@ static const uint32_t cwsr_trap_gfx9_hex[] = { · 19 unmodified lines
303308 0x8f6e8b77, 0x866eff6e,
304309 0x001f8000, 0xb96ef807,
305310 0x86fe7e7e, 0x86ea6a6a,
306− 0xb978f802, 0xbe801f6c,
307− 0x866dff6d, 0x0000ffff,
308− 0xbef00080, 0xb9700283,
309− 0xb8f02407, 0x8e709c70,
310− 0x876d706d, 0xb8f003c7,
311− 0x8e709b70, 0x876d706d,
312− 0xb8f0f807, 0x8670ff70,
313− 0x00007fff, 0xb970f807,
314− 0xbeee007e, 0xbeef007f,
315− 0xbefe0180, 0xbf900004,
316− 0x87708478, 0xb970f802,
317− 0xbf8e0002, 0xbf88fffe,
318− 0xb8f02a05, 0x80708170,
319− 0x8e708a70, 0xb8f11605,
320− 0x80718171, 0x8e718671,
321− 0x80707170, 0x80707e70,
322− 0x8271807f, 0x8671ff71,
323− 0x0000ffff, 0xc0471cb8,
324− 0x00000040, 0xbf8cc07f,
325− 0xc04b1d38, 0x00000048,
326− 0xbf8cc07f, 0xc0431e78,
327− 0x00000058, 0xbf8cc07f,
328− 0xc0471eb8, 0x0000005c,
329− 0xbf8cc07f, 0xbef4007e,
330− 0x8675ff7f, 0x0000ffff,
331− 0x8775ff75, 0x00040000,
332− 0xbef60080, 0xbef700ff,
333− 0x00807fac, 0x8670ff7f,
334− 0x08000000, 0x8f708370,
335− 0x87777077, 0x8670ff7f,
336− 0x70000000, 0x8f708170,
337− 0x87777077, 0xbefb007c,
338− 0xbefa0080, 0xb8fa2a05,
339− 0x807a817a, 0x8e7a8a7a,
340− 0xb8f01605, 0x80708170,
341− 0x8e708670, 0x807a707a,
342− 0xbef60084, 0xbef600ff,
343− 0x01000000, 0xbefe007c,
344− 0xbefc007a, 0xc0611efa,
345− 0x0000007c, 0xbf8cc07f,
346− 0x807a847a, 0xbefc007e,
311+ 0x8f6e8378, 0xb96ee0c2,
312+ 0xbf800002, 0xb9780002,
313+ 0xbe801f6c, 0x866dff6d,
314+ 0x0000ffff, 0xbef00080,
315+ 0xb9700283, 0xb8f02407,
316+ 0x8e709c70, 0x876d706d,
317+ 0xb8f003c7, 0x8e709b70,
318+ 0x876d706d, 0xb8f0f807,
319+ 0x8670ff70, 0x00007fff,
320+ 0xb970f807, 0xbeee007e,
321+ 0xbeef007f, 0xbefe0180,
322+ 0xbf900004, 0x87708478,
323+ 0xb970f802, 0xbf8e0002,
324+ 0xbf88fffe, 0xb8f02a05,
325+ 0x80708170, 0x8e708a70,
326+ 0xb8f11605, 0x80718171,
327+ 0x8e718671, 0x80707170,
328+ 0x80707e70, 0x8271807f,
329+ 0x8671ff71, 0x0000ffff,
330+ 0xc0471cb8, 0x00000040,
331+ 0xbf8cc07f, 0xc04b1d38,
332+ 0x00000048, 0xbf8cc07f,
333+ 0xc0431e78, 0x00000058,
334+ 0xbf8cc07f, 0xc0471eb8,
335+ 0x0000005c, 0xbf8cc07f,
336+ 0xbef4007e, 0x8675ff7f,
337+ 0x0000ffff, 0x8775ff75,
338+ 0x00040000, 0xbef60080,
339+ 0xbef700ff, 0x00807fac,
340+ 0x8670ff7f, 0x08000000,
341+ 0x8f708370, 0x87777077,
342+ 0x8670ff7f, 0x70000000,
343+ 0x8f708170, 0x87777077,
344+ 0xbefb007c, 0xbefa0080,
345+ 0xb8fa2a05, 0x807a817a,
346+ 0x8e7a8a7a, 0xb8f01605,
347+ 0x80708170, 0x8e708670,
348+ 0x807a707a, 0xbef60084,
349+ 0xbef600ff, 0x01000000,
347350 0xbefe007c, 0xbefc007a,
348− 0xc0611b3a, 0x0000007c,
351+ 0xc0611efa, 0x0000007c,
349352 0xbf8cc07f, 0x807a847a,
350353 0xbefc007e, 0xbefe007c,
351− 0xbefc007a, 0xc0611b7a,
354+ 0xbefc007a, 0xc0611b3a,
352355 0x0000007c, 0xbf8cc07f,
353356 0x807a847a, 0xbefc007e,
354357 0xbefe007c, 0xbefc007a,
355− 0xc0611bba, 0x0000007c,
358+ 0xc0611b7a, 0x0000007c,
356359 0xbf8cc07f, 0x807a847a,
357360 0xbefc007e, 0xbefe007c,
358− 0xbefc007a, 0xc0611bfa,
361+ 0xbefc007a, 0xc0611bba,
359362 0x0000007c, 0xbf8cc07f,
360363 0x807a847a, 0xbefc007e,
361364 0xbefe007c, 0xbefc007a,
362− 0xc0611e3a, 0x0000007c,
363− 0xbf8cc07f, 0x807a847a,
364− 0xbefc007e, 0xb8f1f803,
365− 0xbefe007c, 0xbefc007a,
366− 0xc0611c7a, 0x0000007c,
365+ 0xc0611bfa, 0x0000007c,
367366 0xbf8cc07f, 0x807a847a,
368367 0xbefc007e, 0xbefe007c,
369− 0xbefc007a, 0xc0611a3a,
368+ 0xbefc007a, 0xc0611e3a,
369+ 0x0000007c, 0xbf8cc07f,
370+ 0x807a847a, 0xbefc007e,
371+ 0xb8f1f803, 0xbefe007c,
372+ 0xbefc007a, 0xc0611c7a,
370373 0x0000007c, 0xbf8cc07f,
371374 0x807a847a, 0xbefc007e,
372375 0xbefe007c, 0xbefc007a,
373− 0xc0611a7a, 0x0000007c,
374− 0xbf8cc07f, 0x807a847a,
375− 0xbefc007e, 0xb8fbf801,
376− 0xbefe007c, 0xbefc007a,
377− 0xc0611efa, 0x0000007c,
376+ 0xc0611a3a, 0x0000007c,
378377 0xbf8cc07f, 0x807a847a,
379− 0xbefc007e, 0x8670ff7f,
380− 0x04000000, 0xbeef0080,
381− 0x876f6f70, 0xb8fa2a05,
378+ 0xbefc007e, 0xbefe007c,
379+ 0xbefc007a, 0xc0611a7a,
380+ 0x0000007c, 0xbf8cc07f,
381+ 0x807a847a, 0xbefc007e,
382+ 0xb8fbf801, 0xbefe007c,
383+ 0xbefc007a, 0xc0611efa,
384+ 0x0000007c, 0xbf8cc07f,
385+ 0x807a847a, 0xbefc007e,
386+ 0x8670ff7f, 0x04000000,
387+ 0xbeef0080, 0x876f6f70,
388+ 0xb8fa2a05, 0x807a817a,
389+ 0x8e7a8a7a, 0xb8f11605,
390+ 0x80718171, 0x8e718471,
391+ 0x8e768271, 0xbef600ff,
392+ 0x01000000, 0xbef20174,
393+ 0x80747a74, 0x82758075,
394+ 0xbefc0080, 0xbf800000,
395+ 0xbe802b00, 0xbe822b02,
396+ 0xbe842b04, 0xbe862b06,
397+ 0xbe882b08, 0xbe8a2b0a,
398+ 0xbe8c2b0c, 0xbe8e2b0e,
399+ 0xc06b003a, 0x00000000,
400+ 0xbf8cc07f, 0xc06b013a,
401+ 0x00000010, 0xbf8cc07f,
402+ 0xc06b023a, 0x00000020,
403+ 0xbf8cc07f, 0xc06b033a,
404+ 0x00000030, 0xbf8cc07f,
405+ 0x8074c074, 0x82758075,
406+ 0x807c907c, 0xbf0a717c,
407+ 0xbf85ffe7, 0xbef40172,
408+ 0xbefa0080, 0xbefe00c1,
409+ 0xbeff00c1, 0xbee80080,
410+ 0xbee90080, 0xbef600ff,
411+ 0x01000000, 0xe0724000,
412+ 0x7a1d0000, 0xe0724100,
413+ 0x7a1d0100, 0xe0724200,
414+ 0x7a1d0200, 0xe0724300,
415+ 0x7a1d0300, 0xbefe00c1,
416+ 0xbeff00c1, 0xb8f14306,
417+ 0x8671c171, 0xbf84002c,
418+ 0xbf8a0000, 0x8670ff6f,
419+ 0x04000000, 0xbf840028,
420+ 0x8e718671, 0x8e718271,
421+ 0xbef60071, 0xb8fa2a05,
382422 0x807a817a, 0x8e7a8a7a,
383− 0xb8f11605, 0x80718171,
384− 0x8e718471, 0x8e768271,
423+ 0xb8f01605, 0x80708170,
424+ 0x8e708670, 0x807a707a,
425+ 0x807aff7a, 0x00000080,
385426 0xbef600ff, 0x01000000,
386− 0xbef20174, 0x80747a74,
387− 0x82758075, 0xbefc0080,
388− 0xbf800000, 0xbe802b00,
389− 0xbe822b02, 0xbe842b04,
390− 0xbe862b06, 0xbe882b08,
391− 0xbe8a2b0a, 0xbe8c2b0c,
392− 0xbe8e2b0e, 0xc06b003a,
393− 0x00000000, 0xbf8cc07f,
394− 0xc06b013a, 0x00000010,
395− 0xbf8cc07f, 0xc06b023a,
396− 0x00000020, 0xbf8cc07f,
397− 0xc06b033a, 0x00000030,
398− 0xbf8cc07f, 0x8074c074,
399− 0x82758075, 0x807c907c,
400− 0xbf0a717c, 0xbf85ffe7,
401− 0xbef40172, 0xbefa0080,
427+ 0xbefc0080, 0xd28c0002,
428+ 0x000100c1, 0xd28d0003,
429+ 0x000204c1, 0xd1060002,
430+ 0x00011103, 0x7e0602ff,
431+ 0x00000200, 0xbefc00ff,
432+ 0x00010000, 0xbe800077,
433+ 0x8677ff77, 0xff7fffff,
434+ 0x8777ff77, 0x00058000,
435+ 0xd8ec0000, 0x00000002,
436+ 0xbf8cc07f, 0xe0765000,
437+ 0x7a1d0002, 0x68040702,
438+ 0xd0c9006a, 0x0000e302,
439+ 0xbf87fff7, 0xbef70000,
440+ 0xbefa00ff, 0x00000400,
402441 0xbefe00c1, 0xbeff00c1,
403− 0xbee80080, 0xbee90080,
442+ 0xb8f12a05, 0x80718171,
443+ 0x8e718271, 0x8e768871,
404444 0xbef600ff, 0x01000000,
445+ 0xbefc0084, 0xbf0a717c,
446+ 0xbf840015, 0xbf11017c,
447+ 0x8071ff71, 0x00001000,
448+ 0x7e000300, 0x7e020301,
449+ 0x7e040302, 0x7e060303,
405450 0xe0724000, 0x7a1d0000,
406451 0xe0724100, 0x7a1d0100,
407452 0xe0724200, 0x7a1d0200,
408453 0xe0724300, 0x7a1d0300,
454+ 0x807c847c, 0x807aff7a,
455+ 0x00000400, 0xbf0a717c,
456+ 0xbf85ffef, 0xbf9c0000,
457+ 0xbf8200dc, 0xbef4007e,
458+ 0x8675ff7f, 0x0000ffff,
459+ 0x8775ff75, 0x00040000,
460+ 0xbef60080, 0xbef700ff,
461+ 0x00807fac, 0x866eff7f,
462+ 0x08000000, 0x8f6e836e,
463+ 0x87776e77, 0x866eff7f,
464+ 0x70000000, 0x8f6e816e,
465+ 0x87776e77, 0x866eff7f,
466+ 0x04000000, 0xbf84001e,
409467 0xbefe00c1, 0xbeff00c1,
410− 0xb8f14306, 0x8671c171,
411− 0xbf84002c, 0xbf8a0000,
412− 0x8670ff6f, 0x04000000,
413− 0xbf840028, 0x8e718671,
414− 0x8e718271, 0xbef60071,
415− 0xb8fa2a05, 0x807a817a,
416− 0x8e7a8a7a, 0xb8f01605,
417− 0x80708170, 0x8e708670,
418− 0x807a707a, 0x807aff7a,
468+ 0xb8ef4306, 0x866fc16f,
469+ 0xbf840019, 0x8e6f866f,
470+ 0x8e6f826f, 0xbef6006f,
471+ 0xb8f82a05, 0x80788178,
472+ 0x8e788a78, 0xb8ee1605,
473+ 0x806e816e, 0x8e6e866e,
474+ 0x80786e78, 0x8078ff78,
419475 0x00000080, 0xbef600ff,
420476 0x01000000, 0xbefc0080,
421− 0xd28c0002, 0x000100c1,
422− 0xd28d0003, 0x000204c1,
423− 0xd1060002, 0x00011103,
424− 0x7e0602ff, 0x00000200,
425− 0xbefc00ff, 0x00010000,
426− 0xbe800077, 0x8677ff77,
427− 0xff7fffff, 0x8777ff77,
428− 0x00058000, 0xd8ec0000,
429− 0x00000002, 0xbf8cc07f,
430− 0xe0765000, 0x7a1d0002,
431− 0x68040702, 0xd0c9006a,
432− 0x0000e302, 0xbf87fff7,
433− 0xbef70000, 0xbefa00ff,
434− 0x00000400, 0xbefe00c1,
435− 0xbeff00c1, 0xb8f12a05,
436− 0x80718171, 0x8e718271,
437− 0x8e768871, 0xbef600ff,
438− 0x01000000, 0xbefc0084,
439− 0xbf0a717c, 0xbf840015,
440− 0xbf11017c, 0x8071ff71,
441− 0x00001000, 0x7e000300,
477+ 0xe0510000, 0x781d0000,
478+ 0xe0510100, 0x781d0000,
479+ 0x807cff7c, 0x00000200,
480+ 0x8078ff78, 0x00000200,
481+ 0xbf0a6f7c, 0xbf85fff6,
482+ 0xbef80080, 0xbefe00c1,
483+ 0xbeff00c1, 0xb8ef2a05,
484+ 0x806f816f, 0x8e6f826f,
485+ 0x8e76886f, 0xbef600ff,
486+ 0x01000000, 0xbeee0078,
487+ 0x8078ff78, 0x00000400,
488+ 0xbefc0084, 0xbf11087c,
489+ 0x806fff6f, 0x00008000,
490+ 0xe0524000, 0x781d0000,
491+ 0xe0524100, 0x781d0100,
492+ 0xe0524200, 0x781d0200,
493+ 0xe0524300, 0x781d0300,
494+ 0xbf8c0f70, 0x7e000300,
442495 0x7e020301, 0x7e040302,
443− 0x7e060303, 0xe0724000,
444− 0x7a1d0000, 0xe0724100,
445− 0x7a1d0100, 0xe0724200,
446− 0x7a1d0200, 0xe0724300,
447− 0x7a1d0300, 0x807c847c,
448− 0x807aff7a, 0x00000400,
449− 0xbf0a717c, 0xbf85ffef,
450− 0xbf9c0000, 0xbf8200d9,
451− 0xbef4007e, 0x8675ff7f,
452− 0x0000ffff, 0x8775ff75,
453− 0x00040000, 0xbef60080,
454− 0xbef700ff, 0x00807fac,
455− 0x866eff7f, 0x08000000,
456− 0x8f6e836e, 0x87776e77,
457− 0x866eff7f, 0x70000000,
458− 0x8f6e816e, 0x87776e77,
459− 0x866eff7f, 0x04000000,
460− 0xbf84001e, 0xbefe00c1,
461− 0xbeff00c1, 0xb8ef4306,
462− 0x866fc16f, 0xbf840019,
463− 0x8e6f866f, 0x8e6f826f,
464− 0xbef6006f, 0xb8f82a05,
496+ 0x7e060303, 0x807c847c,
497+ 0x8078ff78, 0x00000400,
498+ 0xbf0a6f7c, 0xbf85ffee,
499+ 0xbf9c0000, 0xe0524000,
500+ 0x6e1d0000, 0xe0524100,
501+ 0x6e1d0100, 0xe0524200,
502+ 0x6e1d0200, 0xe0524300,
503+ 0x6e1d0300, 0xb8f82a05,
465504 0x80788178, 0x8e788a78,
466505 0xb8ee1605, 0x806e816e,
467506 0x8e6e866e, 0x80786e78,
468− 0x8078ff78, 0x00000080,
469− 0xbef600ff, 0x01000000,
470− 0xbefc0080, 0xe0510000,
471− 0x781d0000, 0xe0510100,
472− 0x781d0000, 0x807cff7c,
473− 0x00000200, 0x8078ff78,
474− 0x00000200, 0xbf0a6f7c,
475− 0xbf85fff6, 0xbef80080,
476− 0xbefe00c1, 0xbeff00c1,
477− 0xb8ef2a05, 0x806f816f,
478− 0x8e6f826f, 0x8e76886f,
479− 0xbef600ff, 0x01000000,
480− 0xbeee0078, 0x8078ff78,
481− 0x00000400, 0xbefc0084,
482− 0xbf11087c, 0x806fff6f,
483− 0x00008000, 0xe0524000,
484− 0x781d0000, 0xe0524100,
485− 0x781d0100, 0xe0524200,
486− 0x781d0200, 0xe0524300,
487− 0x781d0300, 0xbf8c0f70,
488− 0x7e000300, 0x7e020301,
489− 0x7e040302, 0x7e060303,
490− 0x807c847c, 0x8078ff78,
491− 0x00000400, 0xbf0a6f7c,
492− 0xbf85ffee, 0xbf9c0000,
493− 0xe0524000, 0x6e1d0000,
494− 0xe0524100, 0x6e1d0100,
495− 0xe0524200, 0x6e1d0200,
496− 0xe0524300, 0x6e1d0300,
507+ 0x80f8c078, 0xb8ef1605,
508+ 0x806f816f, 0x8e6f846f,
509+ 0x8e76826f, 0xbef600ff,
510+ 0x01000000, 0xbefc006f,
511+ 0xc031003a, 0x00000078,
512+ 0x80f8c078, 0xbf8cc07f,
513+ 0x80fc907c, 0xbf800000,
514+ 0xbe802d00, 0xbe822d02,
515+ 0xbe842d04, 0xbe862d06,
516+ 0xbe882d08, 0xbe8a2d0a,
517+ 0xbe8c2d0c, 0xbe8e2d0e,
518+ 0xbf06807c, 0xbf84fff0,
497519 0xb8f82a05, 0x80788178,
498520 0x8e788a78, 0xb8ee1605,
499521 0x806e816e, 0x8e6e866e,
500− 0x80786e78, 0x80f8c078,
501− 0xb8ef1605, 0x806f816f,
502− 0x8e6f846f, 0x8e76826f,
522+ 0x80786e78, 0xbef60084,
503523 0xbef600ff, 0x01000000,
504− 0xbefc006f, 0xc031003a,
505− 0x00000078, 0x80f8c078,
506− 0xbf8cc07f, 0x80fc907c,
507− 0xbf800000, 0xbe802d00,
508− 0xbe822d02, 0xbe842d04,
509− 0xbe862d06, 0xbe882d08,
510− 0xbe8a2d0a, 0xbe8c2d0c,
511− 0xbe8e2d0e, 0xbf06807c,
512− 0xbf84fff0, 0xb8f82a05,
513− 0x80788178, 0x8e788a78,
514− 0xb8ee1605, 0x806e816e,
515− 0x8e6e866e, 0x80786e78,
516− 0xbef60084, 0xbef600ff,
517− 0x01000000, 0xc0211bfa,
524+ 0xc0211bfa, 0x00000078,
525+ 0x80788478, 0xc0211b3a,
518526 0x00000078, 0x80788478,
519− 0xc0211b3a, 0x00000078,
520− 0x80788478, 0xc0211b7a,
527+ 0xc0211b7a, 0x00000078,
528+ 0x80788478, 0xc0211eba,
521529 0x00000078, 0x80788478,
522− 0xc0211eba, 0x00000078,
523− 0x80788478, 0xc0211efa,
530+ 0xc0211efa, 0x00000078,
531+ 0x80788478, 0xc0211c3a,
524532 0x00000078, 0x80788478,
525− 0xc0211c3a, 0x00000078,
526− 0x80788478, 0xc0211c7a,
533+ 0xc0211c7a, 0x00000078,
534+ 0x80788478, 0xc0211a3a,
527535 0x00000078, 0x80788478,
528− 0xc0211a3a, 0x00000078,
529− 0x80788478, 0xc0211a7a,
536+ 0xc0211a7a, 0x00000078,
537+ 0x80788478, 0xc0211cfa,
530538 0x00000078, 0x80788478,
531− 0xc0211cfa, 0x00000078,
532− 0x80788478, 0xbf8cc07f,
533− 0xbefc006f, 0xbefe007a,
534− 0xbeff007b, 0x866f71ff,
535− 0x000003ff, 0xb96f4803,
536− 0x866f71ff, 0xfffff800,
537− 0x8f6f8b6f, 0xb96fa2c3,
538− 0xb973f801, 0xb8ee2a05,
539− 0x806e816e, 0x8e6e8a6e,
540− 0xb8ef1605, 0x806f816f,
541− 0x8e6f866f, 0x806e6f6e,
542− 0x806e746e, 0x826f8075,
543− 0x866fff6f, 0x0000ffff,
544− 0xc0071cb7, 0x00000040,
545− 0xc00b1d37, 0x00000048,
546− 0xc0031e77, 0x00000058,
547− 0xc0071eb7, 0x0000005c,
548− 0xbf8cc07f, 0x866fff6d,
549− 0xf0000000, 0x8f6f9c6f,
550− 0x8e6f906f, 0xbeee0080,
551− 0x876e6f6e, 0x866fff6d,
552− 0x08000000, 0x8f6f9b6f,
553− 0x8e6f8f6f, 0x876e6f6e,
554− 0x866fff70, 0x00800000,
555− 0x8f6f976f, 0xb96ef807,
556− 0x866dff6d, 0x0000ffff,
557− 0x86fe7e7e, 0x86ea6a6a,
558− 0xb970f802, 0xbf8a0000,
539+ 0xbf8cc07f, 0xbefc006f,
540+ 0xbefe007a, 0xbeff007b,
541+ 0x866f71ff, 0x000003ff,
542+ 0xb96f4803, 0x866f71ff,
543+ 0xfffff800, 0x8f6f8b6f,
544+ 0xb96fa2c3, 0xb973f801,
545+ 0xb8ee2a05, 0x806e816e,
546+ 0x8e6e8a6e, 0xb8ef1605,
547+ 0x806f816f, 0x8e6f866f,
548+ 0x806e6f6e, 0x806e746e,
549+ 0x826f8075, 0x866fff6f,
550+ 0x0000ffff, 0xc0071cb7,
551+ 0x00000040, 0xc00b1d37,
552+ 0x00000048, 0xc0031e77,
553+ 0x00000058, 0xc0071eb7,
554+ 0x0000005c, 0xbf8cc07f,
555+ 0x866fff6d, 0xf0000000,
556+ 0x8f6f9c6f, 0x8e6f906f,
557+ 0xbeee0080, 0x876e6f6e,
558+ 0x866fff6d, 0x08000000,
559+ 0x8f6f9b6f, 0x8e6f8f6f,
560+ 0x876e6f6e, 0x866fff70,
561+ 0x00800000, 0x8f6f976f,
562+ 0xb96ef807, 0x866dff6d,
563+ 0x0000ffff, 0x86fe7e7e,
564+ 0x86ea6a6a, 0x8f6e8370,
565+ 0xb96ee0c2, 0xbf800002,
566+ 0xb9700002, 0xbf8a0000,
559567 0x95806f6c, 0xbf810000,
560568 };
drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm+15−3
@@ -103,6 +103,10 @@ var SQ_WAVE_STATUS_INST_ATC_SHIFT = 23 · 102 unmodified lines
103103 var SQ_WAVE_STATUS_INST_ATC_MASK = 0x00800000
104104 var SQ_WAVE_STATUS_SPI_PRIO_SHIFT = 1
105105 var SQ_WAVE_STATUS_SPI_PRIO_MASK = 0x00000006
106+var SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT = 0
107+var SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE = 1
108+var SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT = 3
109+var SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE = 29
106110
107111 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT = 12
108112 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE = 9
@@ -251,7 +255,7 @@ if (!EMU_RUN_HACK) · 142 unmodified lines
251255 s_waitcnt lgkmcnt(0)
252256 s_or_b32 ttmp7, ttmp8, ttmp9
253257 s_cbranch_scc0 L_NO_NEXT_TRAP //next level trap handler not been set
254− s_setreg_b32 hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)
258+ set_status_without_spi_prio(s_save_status, ttmp2) //restore HW status(SCC)
255259 s_setpc_b64 [ttmp8,ttmp9] //jump to next level trap handler
256260
257261 L_NO_NEXT_TRAP:
@@ -262,7 +266,7 @@ L_NO_NEXT_TRAP: · 4 unmodified lines
262266 s_addc_u32 ttmp1, ttmp1, 0
263267 L_EXCP_CASE:
264268 s_and_b32 ttmp1, ttmp1, 0xFFFF
265− s_setreg_b32 hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)
269+ set_status_without_spi_prio(s_save_status, ttmp2) //restore HW status(SCC)
266270 s_rfe_b64 [ttmp0, ttmp1]
267271 end
268272 // ********* End handling of non-CWSR traps *******************
@@ -1053,7 +1057,7 @@ end · 784 unmodified lines
10531057 s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff //pc[47:32] //Do it here in order not to affect STATUS
10541058 s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
10551059 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32
1056− s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status // SCC is included, which is changed by previous salu
1060+ set_status_without_spi_prio(s_restore_status, s_restore_tmp) // SCC is included, which is changed by previous salu
10571061
10581062 s_barrier //barrier to ensure the readiness of LDS before access attempts from any other wave in the same TG //FIXME not performance-optimal at this time
10591063
@@ -1134,3 +1138,11 @@ end · 74 unmodified lines
11341138 function get_hwreg_size_bytes
11351139 return 128 //HWREG size 128 bytes
11361140 end
1141+
1142+function set_status_without_spi_prio(status, tmp)
1143+ // Do not restore STATUS.SPI_PRIO since scheduler may have raised it.
1144+ s_lshr_b32 tmp, status, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT
1145+ s_setreg_b32 hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE), tmp
1146+ s_nop 0x2 // avoid S_SETREG => S_SETREG hazard
1147+ s_setreg_b32 hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE), status
1148+end
drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm+14−2
@@ -103,6 +103,10 @@ var SQ_WAVE_STATUS_INST_ATC_MASK = 0x00800000 · 102 unmodified lines
103103 var SQ_WAVE_STATUS_SPI_PRIO_SHIFT = 1
104104 var SQ_WAVE_STATUS_SPI_PRIO_MASK = 0x00000006
105105 var SQ_WAVE_STATUS_HALT_MASK = 0x2000
106+var SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT = 0
107+var SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE = 1
108+var SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT = 3
109+var SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE = 29
106110
107111 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT = 12
108112 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE = 9
@@ -317,7 +321,7 @@ L_EXCP_CASE: · 208 unmodified lines
317321 // Restore SQ_WAVE_STATUS.
318322 s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
319323 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32
320− s_setreg_b32 hwreg(HW_REG_STATUS), s_save_status
324+ set_status_without_spi_prio(s_save_status, ttmp2)
321325
322326 s_rfe_b64 [ttmp0, ttmp1]
323327 end
@@ -1120,7 +1124,7 @@ end · 796 unmodified lines
11201124 s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff //pc[47:32] //Do it here in order not to affect STATUS
11211125 s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
11221126 s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32
1123− s_setreg_b32 hwreg(HW_REG_STATUS), s_restore_status // SCC is included, which is changed by previous salu
1127+ set_status_without_spi_prio(s_restore_status, s_restore_tmp) // SCC is included, which is changed by previous salu
11241128
11251129 s_barrier //barrier to ensure the readiness of LDS before access attempts from any other wave in the same TG //FIXME not performance-optimal at this time
11261130
@@ -1212,3 +1216,11 @@ function ack_sqc_store_workaround · 85 unmodified lines
12121216 s_waitcnt lgkmcnt(0)
12131217 end
12141218 end
1219+
1220+function set_status_without_spi_prio(status, tmp)
1221+ // Do not restore STATUS.SPI_PRIO since scheduler may have raised it.
1222+ s_lshr_b32 tmp, status, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT
1223+ s_setreg_b32 hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE), tmp
1224+ s_nop 0x2 // avoid S_SETREG => S_SETREG hazard
1225+ s_setreg_b32 hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE), status
1226+end
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c+63−6
@@ -122,6 +122,9 @@ static int kfd_open(struct inode *inode, struct file *filep) · 121 unmodified lines
122122 if (IS_ERR(process))
123123 return PTR_ERR(process);
124124
125+ if (kfd_is_locked())
126+ return -EAGAIN;
127+
125128 dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
126129 process->pasid, process->is_32bit_user_mode);
127130
@@ -389,6 +392,61 @@ static int kfd_ioctl_update_queue(struct file *filp, struct kfd_process *p, · 261 unmodified lines
389392 return retval;
390393 }
391394
395+static int kfd_ioctl_set_cu_mask(struct file *filp, struct kfd_process *p,
396+ void *data)
397+{
398+ int retval;
399+ const int max_num_cus = 1024;
400+ struct kfd_ioctl_set_cu_mask_args *args = data;
401+ struct queue_properties properties;
402+ uint32_t __user *cu_mask_ptr = (uint32_t __user *)args->cu_mask_ptr;
403+ size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
404+
405+ if ((args->num_cu_mask % 32) != 0) {
406+ pr_debug("num_cu_mask 0x%x must be a multiple of 32",
407+ args->num_cu_mask);
408+ return -EINVAL;
409+ }
410+
411+ properties.cu_mask_count = args->num_cu_mask;
412+ if (properties.cu_mask_count == 0) {
413+ pr_debug("CU mask cannot be 0");
414+ return -EINVAL;
415+ }
416+
417+ /* To prevent an unreasonably large CU mask size, set an arbitrary
418+ * limit of max_num_cus bits. We can then just drop any CU mask bits
419+ * past max_num_cus bits and just use the first max_num_cus bits.
420+ */
421+ if (properties.cu_mask_count > max_num_cus) {
422+ pr_debug("CU mask cannot be greater than 1024 bits");
423+ properties.cu_mask_count = max_num_cus;
424+ cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
425+ }
426+
427+ properties.cu_mask = kzalloc(cu_mask_size, GFP_KERNEL);
428+ if (!properties.cu_mask)
429+ return -ENOMEM;
430+
431+ retval = copy_from_user(properties.cu_mask, cu_mask_ptr, cu_mask_size);
432+ if (retval) {
433+ pr_debug("Could not copy CU mask from userspace");
434+ kfree(properties.cu_mask);
435+ return -EFAULT;
436+ }
437+
438+ mutex_lock(&p->mutex);
439+
440+ retval = pqm_set_cu_mask(&p->pqm, args->queue_id, &properties);
441+
442+ mutex_unlock(&p->mutex);
443+
444+ if (retval)
445+ kfree(properties.cu_mask);
446+
447+ return retval;
448+}
449+
392450 static int kfd_ioctl_set_memory_policy(struct file *filep,
393451 struct kfd_process *p, void *data)
394452 {
@@ -754,7 +812,6 @@ static int kfd_ioctl_get_clock_counters(struct file *filep, · 359 unmodified lines
754812 {
755813 struct kfd_ioctl_get_clock_counters_args *args = data;
756814 struct kfd_dev *dev;
757− struct timespec64 time;
758815
759816 dev = kfd_device_by_id(args->gpu_id);
760817 if (dev)
@@ -766,11 +823,8 @@ static int kfd_ioctl_get_clock_counters(struct file *filep, · 5 unmodified lines
766823 args->gpu_clock_counter = 0;
767824
768825 /* No access to rdtsc. Using raw monotonic time */
769− getrawmonotonic64(&time);
770− args->cpu_clock_counter = (uint64_t)timespec64_to_ns(&time);
771−
772− get_monotonic_boottime64(&time);
773− args->system_clock_counter = (uint64_t)timespec64_to_ns(&time);
826+ args->cpu_clock_counter = ktime_get_raw_ns();
827+ args->system_clock_counter = ktime_get_boot_ns();
774828
775829 /* Since the counter is in nano-seconds we use 1GHz frequency */
776830 args->system_clock_freq = 1000000000;
@@ -1558,6 +1612,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { · 781 unmodified lines
15581612 AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
15591613 kfd_ioctl_unmap_memory_from_gpu, 0),
15601614
1615+ AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
1616+ kfd_ioctl_set_cu_mask, 0),
1617+
15611618 };
15621619
15631620 #define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)
drivers/gpu/drm/amd/amdkfd/kfd_crat.c+46−11
@@ -189,6 +189,21 @@ static int kfd_parse_subtype_cu(struct crat_subtype_computeunit *cu, · 188 unmodified lines
189189 return 0;
190190 }
191191
192+static struct kfd_mem_properties *
193+find_subtype_mem(uint32_t heap_type, uint32_t flags, uint32_t width,
194+ struct kfd_topology_device *dev)
195+{
196+ struct kfd_mem_properties *props;
197+
198+ list_for_each_entry(props, &dev->mem_props, list) {
199+ if (props->heap_type == heap_type
200+ && props->flags == flags
201+ && props->width == width)
202+ return props;
203+ }
204+
205+ return NULL;
206+}
192207 /* kfd_parse_subtype_mem - parse memory subtypes and attach it to correct
193208 * topology device present in the device_list
194209 */
@@ -197,36 +212,56 @@ static int kfd_parse_subtype_mem(struct crat_subtype_memory *mem, · 2 unmodified lines
197212 {
198213 struct kfd_mem_properties *props;
199214 struct kfd_topology_device *dev;
215+ uint32_t heap_type;
216+ uint64_t size_in_bytes;
217+ uint32_t flags = 0;
218+ uint32_t width;
200219
201220 pr_debug("Found memory entry in CRAT table with proximity_domain=%d\n",
202221 mem->proximity_domain);
203222 list_for_each_entry(dev, device_list, list) {
204223 if (mem->proximity_domain == dev->proximity_domain) {
205− props = kfd_alloc_struct(props);
206− if (!props)
207− return -ENOMEM;
208−
209224 /* We're on GPU node */
210225 if (dev->node_props.cpu_cores_count == 0) {
211226 /* APU */
212227 if (mem->visibility_type == 0)
213− props->heap_type =
228+ heap_type =
214229 HSA_MEM_HEAP_TYPE_FB_PRIVATE;
215230 /* dGPU */
216231 else
217− props->heap_type = mem->visibility_type;
232+ heap_type = mem->visibility_type;
218233 } else
219− props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
234+ heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
220235
221236 if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
222− props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
237+ flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
223238 if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
224− props->flags |= HSA_MEM_FLAGS_NON_VOLATILE;
239+ flags |= HSA_MEM_FLAGS_NON_VOLATILE;
225240
226− props->size_in_bytes =
241+ size_in_bytes =
227242 ((uint64_t)mem->length_high << 32) +
228243 mem->length_low;
229− props->width = mem->width;
244+ width = mem->width;
245+
246+ /* Multiple banks of the same type are aggregated into
247+ * one. User mode doesn't care about multiple physical
248+ * memory segments. It's managed as a single virtual
249+ * heap for user mode.
250+ */
251+ props = find_subtype_mem(heap_type, flags, width, dev);
252+ if (props) {
253+ props->size_in_bytes += size_in_bytes;
254+ break;
255+ }
256+
257+ props = kfd_alloc_struct(props);
258+ if (!props)
259+ return -ENOMEM;
260+
261+ props->heap_type = heap_type;
262+ props->flags = flags;
263+ props->size_in_bytes = size_in_bytes;
264+ props->width = width;
230265
231266 dev->node_props.mem_banks_count++;
232267 list_add_tail(&props->list, &dev->mem_props);
drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c+0−1
@@ -38,7 +38,6 @@ · 37 unmodified lines
3838 #include "kfd_dbgmgr.h"
3939 #include "kfd_dbgdev.h"
4040 #include "kfd_device_queue_manager.h"
41−#include "../../radeon/cik_reg.h"
4241
4342 static void dbgdev_address_watch_disable_nodiq(struct kfd_dev *dev)
4443 {
drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h+37−0
@@ -60,6 +60,9 @@ enum { · 59 unmodified lines
6060 SH_REG_SIZE = SH_REG_END - SH_REG_BASE
6161 };
6262
63+/* SQ_CMD definitions */
64+#define SQ_CMD 0x8DEC
65+
6366 enum SQ_IND_CMD_CMD {
6467 SQ_IND_CMD_CMD_NULL = 0x00000000,
6568 SQ_IND_CMD_CMD_HALT = 0x00000001,
@@ -190,4 +193,38 @@ union ULARGE_INTEGER { · 124 unmodified lines
190193 void kfd_dbgdev_init(struct kfd_dbgdev *pdbgdev, struct kfd_dev *pdev,
191194 enum DBGDEV_TYPE type);
192195
196+union TCP_WATCH_CNTL_BITS {
197+ struct {
198+ uint32_t mask:24;
199+ uint32_t vmid:4;
200+ uint32_t atc:1;
201+ uint32_t mode:2;
202+ uint32_t valid:1;
203+ } bitfields, bits;
204+ uint32_t u32All;
205+ signed int i32All;
206+ float f32All;
207+};
208+
209+enum {
210+ ADDRESS_WATCH_REG_CNTL_ATC_BIT = 0x10000000UL,
211+ ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK = 0x00FFFFFF,
212+ ADDRESS_WATCH_REG_ADDLOW_MASK_EXTENSION = 0x03000000,
213+ /* extend the mask to 26 bits in order to match the low address field */
214+ ADDRESS_WATCH_REG_ADDLOW_SHIFT = 6,
215+ ADDRESS_WATCH_REG_ADDHIGH_MASK = 0xFFFF
216+};
217+
218+enum {
219+ MAX_TRAPID = 8, /* 3 bits in the bitfield. */
220+ MAX_WATCH_ADDRESSES = 4
221+};
222+
223+enum {
224+ ADDRESS_WATCH_REG_ADDR_HI = 0,
225+ ADDRESS_WATCH_REG_ADDR_LO,
226+ ADDRESS_WATCH_REG_CNTL,
227+ ADDRESS_WATCH_REG_MAX
228+};
229+
193230 #endif /* KFD_DBGDEV_H_ */
drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c+48−0
@@ -21,6 +21,8 @@ · 20 unmodified lines
2121 */
2222
2323 #include <linux/debugfs.h>
24+#include <linux/uaccess.h>
25+
2426 #include "kfd_priv.h"
2527
2628 static struct dentry *debugfs_root;
@@ -32,6 +34,38 @@ static int kfd_debugfs_open(struct inode *inode, struct file *file) · 5 unmodified lines
3234 return single_open(file, show, NULL);
3335 }
3436
37+static ssize_t kfd_debugfs_hang_hws_write(struct file *file,
38+ const char __user *user_buf, size_t size, loff_t *ppos)
39+{
40+ struct kfd_dev *dev;
41+ char tmp[16];
42+ uint32_t gpu_id;
43+ int ret = -EINVAL;
44+
45+ memset(tmp, 0, 16);
46+ if (size >= 16) {
47+ pr_err("Invalid input for gpu id.\n");
48+ goto out;
49+ }
50+ if (copy_from_user(tmp, user_buf, size)) {
51+ ret = -EFAULT;
52+ goto out;
53+ }
54+ if (kstrtoint(tmp, 10, &gpu_id)) {
55+ pr_err("Invalid input for gpu id.\n");
56+ goto out;
57+ }
58+ dev = kfd_device_by_id(gpu_id);
59+ if (dev) {
60+ kfd_debugfs_hang_hws(dev);
61+ ret = size;
62+ } else
63+ pr_err("Cannot find device %d.\n", gpu_id);
64+
65+out:
66+ return ret;
67+}
68+
3569 static const struct file_operations kfd_debugfs_fops = {
3670 .owner = THIS_MODULE,
3771 .open = kfd_debugfs_open,
@@ -40,6 +74,15 @@ static const struct file_operations kfd_debugfs_fops = { · 2 unmodified lines
4074 .release = single_release,
4175 };
4276
77+static const struct file_operations kfd_debugfs_hang_hws_fops = {
78+ .owner = THIS_MODULE,
79+ .open = kfd_debugfs_open,
80+ .read = seq_read,
81+ .write = kfd_debugfs_hang_hws_write,
82+ .llseek = seq_lseek,
83+ .release = single_release,
84+};
85+
4386 void kfd_debugfs_init(void)
4487 {
4588 struct dentry *ent;
@@ -65,6 +108,11 @@ void kfd_debugfs_init(void) · 19 unmodified lines
65108 ent = debugfs_create_file("rls", S_IFREG | 0444, debugfs_root,
66109 kfd_debugfs_rls_by_device,
67110 &kfd_debugfs_fops);
111+
112+ ent = debugfs_create_file("hang_hws", S_IFREG | 0644, debugfs_root,
113+ NULL,
114+ &kfd_debugfs_hang_hws_fops);
115+
68116 if (!ent)
69117 pr_warn("Failed to create rls in kfd debugfs\n");
70118 }
drivers/gpu/drm/amd/amdkfd/kfd_device.c+115−7
@@ -30,7 +30,13 @@ · 29 unmodified lines
3030 #include "kfd_iommu.h"
3131
3232 #define MQD_SIZE_ALIGNED 768
33−static atomic_t kfd_device_suspended = ATOMIC_INIT(0);
33+
34+/*
35+ * kfd_locked is used to lock the kfd driver during suspend or reset
36+ * once locked, kfd driver will stop any further GPU execution.
37+ * create process (open) will return -EAGAIN.
38+ */
39+static atomic_t kfd_locked = ATOMIC_INIT(0);
3440
3541 #ifdef KFD_SUPPORT_IOMMU_V2
3642 static const struct kfd_device_info kaveri_device_info = {
@@ -46,6 +52,7 @@ static const struct kfd_device_info kaveri_device_info = { · 9 unmodified lines
4652 .supports_cwsr = false,
4753 .needs_iommu_device = true,
4854 .needs_pci_atomics = false,
55+ .num_sdma_engines = 2,
4956 };
5057
5158 static const struct kfd_device_info carrizo_device_info = {
@@ -61,6 +68,22 @@ static const struct kfd_device_info carrizo_device_info = { · 9 unmodified lines
6168 .supports_cwsr = true,
6269 .needs_iommu_device = true,
6370 .needs_pci_atomics = false,
71+ .num_sdma_engines = 2,
72+};
73+
74+static const struct kfd_device_info raven_device_info = {
75+ .asic_family = CHIP_RAVEN,
76+ .max_pasid_bits = 16,
77+ .max_no_of_hqd = 24,
78+ .doorbell_size = 8,
79+ .ih_ring_entry_size = 8 * sizeof(uint32_t),
80+ .event_interrupt_class = &event_interrupt_class_v9,
81+ .num_of_watch_points = 4,
82+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
83+ .supports_cwsr = true,
84+ .needs_iommu_device = true,
85+ .needs_pci_atomics = true,
86+ .num_sdma_engines = 1,
6487 };
6588 #endif
6689
@@ -77,6 +100,7 @@ static const struct kfd_device_info hawaii_device_info = { · 10 unmodified lines
77100 .supports_cwsr = false,
78101 .needs_iommu_device = false,
79102 .needs_pci_atomics = false,
103+ .num_sdma_engines = 2,
80104 };
81105
82106 static const struct kfd_device_info tonga_device_info = {
@@ -91,6 +115,7 @@ static const struct kfd_device_info tonga_device_info = { · 8 unmodified lines
91115 .supports_cwsr = false,
92116 .needs_iommu_device = false,
93117 .needs_pci_atomics = true,
118+ .num_sdma_engines = 2,
94119 };
95120
96121 static const struct kfd_device_info tonga_vf_device_info = {
@@ -105,6 +130,7 @@ static const struct kfd_device_info tonga_vf_device_info = { · 8 unmodified lines
105130 .supports_cwsr = false,
106131 .needs_iommu_device = false,
107132 .needs_pci_atomics = false,
133+ .num_sdma_engines = 2,
108134 };
109135
110136 static const struct kfd_device_info fiji_device_info = {
@@ -119,6 +145,7 @@ static const struct kfd_device_info fiji_device_info = { · 8 unmodified lines
119145 .supports_cwsr = true,
120146 .needs_iommu_device = false,
121147 .needs_pci_atomics = true,
148+ .num_sdma_engines = 2,
122149 };
123150
124151 static const struct kfd_device_info fiji_vf_device_info = {
@@ -133,6 +160,7 @@ static const struct kfd_device_info fiji_vf_device_info = { · 8 unmodified lines
133160 .supports_cwsr = true,
134161 .needs_iommu_device = false,
135162 .needs_pci_atomics = false,
163+ .num_sdma_engines = 2,
136164 };
137165
138166
@@ -148,6 +176,7 @@ static const struct kfd_device_info polaris10_device_info = { · 9 unmodified lines
148176 .supports_cwsr = true,
149177 .needs_iommu_device = false,
150178 .needs_pci_atomics = true,
179+ .num_sdma_engines = 2,
151180 };
152181
153182 static const struct kfd_device_info polaris10_vf_device_info = {
@@ -162,6 +191,7 @@ static const struct kfd_device_info polaris10_vf_device_info = { · 8 unmodified lines
162191 .supports_cwsr = true,
163192 .needs_iommu_device = false,
164193 .needs_pci_atomics = false,
194+ .num_sdma_engines = 2,
165195 };
166196
167197 static const struct kfd_device_info polaris11_device_info = {
@@ -176,6 +206,7 @@ static const struct kfd_device_info polaris11_device_info = { · 8 unmodified lines
176206 .supports_cwsr = true,
177207 .needs_iommu_device = false,
178208 .needs_pci_atomics = true,
209+ .num_sdma_engines = 2,
179210 };
180211
181212 static const struct kfd_device_info vega10_device_info = {
@@ -190,6 +221,7 @@ static const struct kfd_device_info vega10_device_info = { · 8 unmodified lines
190221 .supports_cwsr = true,
191222 .needs_iommu_device = false,
192223 .needs_pci_atomics = false,
224+ .num_sdma_engines = 2,
193225 };
194226
195227 static const struct kfd_device_info vega10_vf_device_info = {
@@ -204,6 +236,7 @@ static const struct kfd_device_info vega10_vf_device_info = { · 8 unmodified lines
204236 .supports_cwsr = true,
205237 .needs_iommu_device = false,
206238 .needs_pci_atomics = false,
239+ .num_sdma_engines = 2,
207240 };
208241
209242
@@ -241,6 +274,7 @@ static const struct kfd_deviceid supported_devices[] = { · 31 unmodified lines
241274 { 0x9875, &carrizo_device_info }, /* Carrizo */
242275 { 0x9876, &carrizo_device_info }, /* Carrizo */
243276 { 0x9877, &carrizo_device_info }, /* Carrizo */
277+ { 0x15DD, &raven_device_info }, /* Raven */
244278 #endif
245279 { 0x67A0, &hawaii_device_info }, /* Hawaii */
246280 { 0x67A1, &hawaii_device_info }, /* Hawaii */
@@ -514,13 +548,54 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd) · 267 unmodified lines
514548 kfree(kfd);
515549 }
516550
551+int kgd2kfd_pre_reset(struct kfd_dev *kfd)
552+{
553+ if (!kfd->init_complete)
554+ return 0;
555+ kgd2kfd_suspend(kfd);
556+
557+ /* hold dqm->lock to prevent further execution*/
558+ dqm_lock(kfd->dqm);
559+
560+ kfd_signal_reset_event(kfd);
561+ return 0;
562+}
563+
564+/*
565+ * Fix me. KFD won't be able to resume existing process for now.
566+ * We will keep all existing process in a evicted state and
567+ * wait the process to be terminated.
568+ */
569+
570+int kgd2kfd_post_reset(struct kfd_dev *kfd)
571+{
572+ int ret, count;
573+
574+ if (!kfd->init_complete)
575+ return 0;
576+
577+ dqm_unlock(kfd->dqm);
578+
579+ ret = kfd_resume(kfd);
580+ if (ret)
581+ return ret;
582+ count = atomic_dec_return(&kfd_locked);
583+ WARN_ONCE(count != 0, "KFD reset ref. error");
584+ return 0;
585+}
586+
587+bool kfd_is_locked(void)
588+{
589+ return (atomic_read(&kfd_locked) > 0);
590+}
591+
517592 void kgd2kfd_suspend(struct kfd_dev *kfd)
518593 {
519594 if (!kfd->init_complete)
520595 return;
521596
522597 /* For first KFD device suspend all the KFD processes */
523− if (atomic_inc_return(&kfd_device_suspended) == 1)
598+ if (atomic_inc_return(&kfd_locked) == 1)
524599 kfd_suspend_all_processes();
525600
526601 kfd->dqm->ops.stop(kfd->dqm);
@@ -539,7 +614,7 @@ int kgd2kfd_resume(struct kfd_dev *kfd) · 12 unmodified lines
539614 if (ret)
540615 return ret;
541616
542− count = atomic_dec_return(&kfd_device_suspended);
617+ count = atomic_dec_return(&kfd_locked);
543618 WARN_ONCE(count < 0, "KFD suspend / resume ref. error");
544619 if (count == 0)
545620 ret = kfd_resume_all_processes();
@@ -577,14 +652,24 @@ dqm_start_error: · 31 unmodified lines
577652 /* This is called directly from KGD at ISR. */
578653 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
579654 {
655+ uint32_t patched_ihre[KFD_MAX_RING_ENTRY_SIZE];
656+ bool is_patched = false;
657+
580658 if (!kfd->init_complete)
581659 return;
582660
661+ if (kfd->device_info->ih_ring_entry_size > sizeof(patched_ihre)) {
662+ dev_err_once(kfd_device, "Ring entry too small\n");
663+ return;
664+ }
665+
583666 spin_lock(&kfd->interrupt_lock);
584667
585668 if (kfd->interrupts_active
586− && interrupt_is_wanted(kfd, ih_ring_entry)
587− && enqueue_ih_ring_entry(kfd, ih_ring_entry))
669+ && interrupt_is_wanted(kfd, ih_ring_entry,
670+ patched_ihre, &is_patched)
671+ && enqueue_ih_ring_entry(kfd,
672+ is_patched ? patched_ihre : ih_ring_entry))
588673 queue_work(kfd->ih_wq, &kfd->interrupt_work);
589674
590675 spin_unlock(&kfd->interrupt_lock);
@@ -739,8 +824,8 @@ int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size, · 148 unmodified lines
739824 if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size)
740825 return -ENOMEM;
741826
742− *mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
743− if ((*mem_obj) == NULL)
827+ *mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
828+ if (!(*mem_obj))
744829 return -ENOMEM;
745830
746831 pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size);
@@ -857,3 +942,26 @@ int kfd_gtt_sa_free(struct kfd_dev *kfd, struct kfd_mem_obj *mem_obj) · 110 unmodified lines
857942 kfree(mem_obj);
858943 return 0;
859944 }
945+
946+#if defined(CONFIG_DEBUG_FS)
947+
948+/* This function will send a package to HIQ to hang the HWS
949+ * which will trigger a GPU reset and bring the HWS back to normal state
950+ */
951+int kfd_debugfs_hang_hws(struct kfd_dev *dev)
952+{
953+ int r = 0;
954+
955+ if (dev->dqm->sched_policy != KFD_SCHED_POLICY_HWS) {
956+ pr_err("HWS is not enabled");
957+ return -EINVAL;
958+ }
959+
960+ r = pm_debugfs_hang_hws(&dev->dqm->packets);
961+ if (!r)
962+ r = dqm_debugfs_execute_queues(dev->dqm);
963+
964+ return r;
965+}
966+
967+#endif
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c+177−106
@@ -61,6 +61,8 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, · 60 unmodified lines
6161 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
6262 unsigned int sdma_queue_id);
6363
64+static void kfd_process_hw_exception(struct work_struct *work);
65+
6466 static inline
6567 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
6668 {
@@ -99,6 +101,17 @@ unsigned int get_pipes_per_mec(struct device_queue_manager *dqm) · 32 unmodified lines
99101 return dqm->dev->shared_resources.num_pipe_per_mec;
100102 }
101103
104+static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm)
105+{
106+ return dqm->dev->device_info->num_sdma_engines;
107+}
108+
109+unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
110+{
111+ return dqm->dev->device_info->num_sdma_engines
112+ * KFD_SDMA_QUEUES_PER_ENGINE;
113+}
114+
102115 void program_sh_mem_settings(struct device_queue_manager *dqm,
103116 struct qcm_process_device *qpd)
104117 {
@@ -240,7 +253,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm, · 135 unmodified lines
240253
241254 print_queue(q);
242255
243− mutex_lock(&dqm->lock);
256+ dqm_lock(dqm);
244257
245258 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
246259 pr_warn("Can't create new usermode queue because %d queues were already created\n",
@@ -297,7 +310,7 @@ static int create_queue_nocpsch(struct device_queue_manager *dqm, · 50 unmodified lines
297310 dqm->total_queue_count);
298311
299312 out_unlock:
300− mutex_unlock(&dqm->lock);
313+ dqm_unlock(dqm);
301314 return retval;
302315 }
303316
@@ -346,10 +359,10 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, · 42 unmodified lines
346359 struct qcm_process_device *qpd)
347360 {
348361 int retval;
349− struct mqd_manager *mqd;
362+ struct mqd_manager *mqd_mgr;
350363
351− mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
352− if (!mqd)
364+ mqd_mgr = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
365+ if (!mqd_mgr)
353366 return -ENOMEM;
354367
355368 retval = allocate_hqd(dqm, q);
@@ -360,7 +373,7 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, · 4 unmodified lines
360373 if (retval)
361374 goto out_deallocate_hqd;
362375
363− retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
376+ retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
364377 &q->gart_mqd_addr, &q->properties);
365378 if (retval)
366379 goto out_deallocate_doorbell;
@@ -374,15 +387,15 @@ static int create_compute_queue_nocpsch(struct device_queue_manager *dqm, · 7 unmodified lines
374387 if (!q->properties.is_active)
375388 return 0;
376389
377− retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue, &q->properties,
378− q->process->mm);
390+ retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, q->queue,
391+ &q->properties, q->process->mm);
379392 if (retval)
380393 goto out_uninit_mqd;
381394
382395 return 0;
383396
384397 out_uninit_mqd:
385− mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
398+ mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
386399 out_deallocate_doorbell:
387400 deallocate_doorbell(qpd, q);
388401 out_deallocate_hqd:
@@ -399,11 +412,11 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, · 10 unmodified lines
399412 struct queue *q)
400413 {
401414 int retval;
402− struct mqd_manager *mqd;
415+ struct mqd_manager *mqd_mgr;
403416
404− mqd = dqm->ops.get_mqd_manager(dqm,
417+ mqd_mgr = dqm->ops.get_mqd_manager(dqm,
405418 get_mqd_type_from_queue_type(q->properties.type));
406− if (!mqd)
419+ if (!mqd_mgr)
407420 return -ENOMEM;
408421
409422 if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
@@ -420,14 +433,14 @@ static int destroy_queue_nocpsch_locked(struct device_queue_manager *dqm, · 10 unmodified lines
420433
421434 deallocate_doorbell(qpd, q);
422435
423− retval = mqd->destroy_mqd(mqd, q->mqd,
436+ retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
424437 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
425438 KFD_UNMAP_LATENCY_MS,
426439 q->pipe, q->queue);
427440 if (retval == -ETIME)
428441 qpd->reset_wavefronts = true;
429442
430− mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
443+ mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
431444
432445 list_del(&q->list);
433446 if (list_empty(&qpd->queues_list)) {
@@ -457,9 +470,9 @@ static int destroy_queue_nocpsch(struct device_queue_manager *dqm, · 23 unmodified lines
457470 {
458471 int retval;
459472
460− mutex_lock(&dqm->lock);
473+ dqm_lock(dqm);
461474 retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
462− mutex_unlock(&dqm->lock);
475+ dqm_unlock(dqm);
463476
464477 return retval;
465478 }
@@ -467,19 +480,19 @@ static int destroy_queue_nocpsch(struct device_queue_manager *dqm, · 1 unmodified line
467480 static int update_queue(struct device_queue_manager *dqm, struct queue *q)
468481 {
469482 int retval;
470− struct mqd_manager *mqd;
483+ struct mqd_manager *mqd_mgr;
471484 struct kfd_process_device *pdd;
472485 bool prev_active = false;
473486
474− mutex_lock(&dqm->lock);
487+ dqm_lock(dqm);
475488 pdd = kfd_get_process_device_data(q->device, q->process);
476489 if (!pdd) {
477490 retval = -ENODEV;
478491 goto out_unlock;
479492 }
480− mqd = dqm->ops.get_mqd_manager(dqm,
493+ mqd_mgr = dqm->ops.get_mqd_manager(dqm,
481494 get_mqd_type_from_queue_type(q->properties.type));
482− if (!mqd) {
495+ if (!mqd_mgr) {
483496 retval = -ENOMEM;
484497 goto out_unlock;
485498 }
@@ -506,7 +519,7 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q) · 20 unmodified lines
506519 } else if (prev_active &&
507520 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
508521 q->properties.type == KFD_QUEUE_TYPE_SDMA)) {
509− retval = mqd->destroy_mqd(mqd, q->mqd,
522+ retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
510523 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
511524 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
512525 if (retval) {
@@ -515,7 +528,7 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q) · 2 unmodified lines
515528 }
516529 }
517530
518− retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
531+ retval = mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties);
519532
520533 /*
521534 * check active state vs. the previous state and modify
@@ -533,44 +546,44 @@ static int update_queue(struct device_queue_manager *dqm, struct queue *q) · 11 unmodified lines
533546 else if (q->properties.is_active &&
534547 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
535548 q->properties.type == KFD_QUEUE_TYPE_SDMA))
536− retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue,
549+ retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, q->queue,
537550 &q->properties, q->process->mm);
538551
539552 out_unlock:
540− mutex_unlock(&dqm->lock);
553+ dqm_unlock(dqm);
541554 return retval;
542555 }
543556
544557 static struct mqd_manager *get_mqd_manager(
545558 struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
546559 {
547− struct mqd_manager *mqd;
560+ struct mqd_manager *mqd_mgr;
548561
549562 if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
550563 return NULL;
551564
552565 pr_debug("mqd type %d\n", type);
553566
554− mqd = dqm->mqds[type];
555− if (!mqd) {
556− mqd = mqd_manager_init(type, dqm->dev);
557− if (!mqd)
567+ mqd_mgr = dqm->mqd_mgrs[type];
568+ if (!mqd_mgr) {
569+ mqd_mgr = mqd_manager_init(type, dqm->dev);
570+ if (!mqd_mgr)
558571 pr_err("mqd manager is NULL");
559− dqm->mqds[type] = mqd;
572+ dqm->mqd_mgrs[type] = mqd_mgr;
560573 }
561574
562− return mqd;
575+ return mqd_mgr;
563576 }
564577
565578 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
566579 struct qcm_process_device *qpd)
567580 {
568581 struct queue *q;
569− struct mqd_manager *mqd;
582+ struct mqd_manager *mqd_mgr;
570583 struct kfd_process_device *pdd;
571584 int retval = 0;
572585
573− mutex_lock(&dqm->lock);
586+ dqm_lock(dqm);
574587 if (qpd->evicted++ > 0) /* already evicted, do nothing */
575588 goto out;
576589
@@ -582,16 +595,16 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, · 5 unmodified lines
582595 list_for_each_entry(q, &qpd->queues_list, list) {
583596 if (!q->properties.is_active)
584597 continue;
585− mqd = dqm->ops.get_mqd_manager(dqm,
598+ mqd_mgr = dqm->ops.get_mqd_manager(dqm,
586599 get_mqd_type_from_queue_type(q->properties.type));
587− if (!mqd) { /* should not be here */
600+ if (!mqd_mgr) { /* should not be here */
588601 pr_err("Cannot evict queue, mqd mgr is NULL\n");
589602 retval = -ENOMEM;
590603 goto out;
591604 }
592605 q->properties.is_evicted = true;
593606 q->properties.is_active = false;
594− retval = mqd->destroy_mqd(mqd, q->mqd,
607+ retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
595608 KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
596609 KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
597610 if (retval)
@@ -600,7 +613,7 @@ static int evict_process_queues_nocpsch(struct device_queue_manager *dqm, · 2 unmodified lines
600613 }
601614
602615 out:
603− mutex_unlock(&dqm->lock);
616+ dqm_unlock(dqm);
604617 return retval;
605618 }
606619
@@ -611,7 +624,7 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm, · 4 unmodified lines
611624 struct kfd_process_device *pdd;
612625 int retval = 0;
613626
614− mutex_lock(&dqm->lock);
627+ dqm_lock(dqm);
615628 if (qpd->evicted++ > 0) /* already evicted, do nothing */
616629 goto out;
617630
@@ -633,7 +646,7 @@ static int evict_process_queues_cpsch(struct device_queue_manager *dqm, · 15 unmodified lines
633646 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
634647
635648 out:
636− mutex_unlock(&dqm->lock);
649+ dqm_unlock(dqm);
637650 return retval;
638651 }
639652
@@ -641,7 +654,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, · 1 unmodified line
641654 struct qcm_process_device *qpd)
642655 {
643656 struct queue *q;
644− struct mqd_manager *mqd;
657+ struct mqd_manager *mqd_mgr;
645658 struct kfd_process_device *pdd;
646659 uint32_t pd_base;
647660 int retval = 0;
@@ -650,7 +663,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, · 2 unmodified lines
650663 /* Retrieve PD base */
651664 pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
652665
653− mutex_lock(&dqm->lock);
666+ dqm_lock(dqm);
654667 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
655668 goto out;
656669 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
@@ -677,16 +690,16 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, · 20 unmodified lines
677690 list_for_each_entry(q, &qpd->queues_list, list) {
678691 if (!q->properties.is_evicted)
679692 continue;
680− mqd = dqm->ops.get_mqd_manager(dqm,
693+ mqd_mgr = dqm->ops.get_mqd_manager(dqm,
681694 get_mqd_type_from_queue_type(q->properties.type));
682− if (!mqd) { /* should not be here */
695+ if (!mqd_mgr) { /* should not be here */
683696 pr_err("Cannot restore queue, mqd mgr is NULL\n");
684697 retval = -ENOMEM;
685698 goto out;
686699 }
687700 q->properties.is_evicted = false;
688701 q->properties.is_active = true;
689− retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
702+ retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
690703 q->queue, &q->properties,
691704 q->process->mm);
692705 if (retval)
@@ -695,7 +708,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, · 2 unmodified lines
695708 }
696709 qpd->evicted = 0;
697710 out:
698− mutex_unlock(&dqm->lock);
711+ dqm_unlock(dqm);
699712 return retval;
700713 }
701714
@@ -711,7 +724,7 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm, · 9 unmodified lines
711724 /* Retrieve PD base */
712725 pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
713726
714− mutex_lock(&dqm->lock);
727+ dqm_lock(dqm);
715728 if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
716729 goto out;
717730 if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
@@ -739,7 +752,7 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm, · 21 unmodified lines
739752 if (!retval)
740753 qpd->evicted = 0;
741754 out:
742− mutex_unlock(&dqm->lock);
755+ dqm_unlock(dqm);
743756 return retval;
744757 }
745758
@@ -761,7 +774,7 @@ static int register_process(struct device_queue_manager *dqm, · 15 unmodified lines
761774 /* Retrieve PD base */
762775 pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
763776
764− mutex_lock(&dqm->lock);
777+ dqm_lock(dqm);
765778 list_add(&n->list, &dqm->queues);
766779
767780 /* Update PD Base in QPD */
@@ -769,9 +782,10 @@ static int register_process(struct device_queue_manager *dqm, · 1 unmodified line
769782
770783 retval = dqm->asic_ops.update_qpd(dqm, qpd);
771784
772− dqm->processes_count++;
785+ if (dqm->processes_count++ == 0)
786+ dqm->dev->kfd2kgd->set_compute_idle(dqm->dev->kgd, false);
773787
774− mutex_unlock(&dqm->lock);
788+ dqm_unlock(dqm);
775789
776790 return retval;
777791 }
@@ -786,20 +800,22 @@ static int unregister_process(struct device_queue_manager *dqm, · 8 unmodified lines
786800 list_empty(&qpd->queues_list) ? "empty" : "not empty");
787801
788802 retval = 0;
789− mutex_lock(&dqm->lock);
803+ dqm_lock(dqm);
790804
791805 list_for_each_entry_safe(cur, next, &dqm->queues, list) {
792806 if (qpd == cur->qpd) {
793807 list_del(&cur->list);
794808 kfree(cur);
795− dqm->processes_count--;
809+ if (--dqm->processes_count == 0)
810+ dqm->dev->kfd2kgd->set_compute_idle(
811+ dqm->dev->kgd, true);
796812 goto out;
797813 }
798814 }
799815 /* qpd not found in dqm list */
800816 retval = 1;
801817 out:
802− mutex_unlock(&dqm->lock);
818+ dqm_unlock(dqm);
803819 return retval;
804820 }
805821
@@ -838,7 +854,7 @@ static int initialize_nocpsch(struct device_queue_manager *dqm) · 32 unmodified lines
838854 if (!dqm->allocated_queues)
839855 return -ENOMEM;
840856
841− mutex_init(&dqm->lock);
857+ mutex_init(&dqm->lock_hidden);
842858 INIT_LIST_HEAD(&dqm->queues);
843859 dqm->queue_count = dqm->next_pipe_to_allocate = 0;
844860 dqm->sdma_queue_count = 0;
@@ -853,7 +869,7 @@ static int initialize_nocpsch(struct device_queue_manager *dqm) · 8 unmodified lines
853869 }
854870
855871 dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1;
856− dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
872+ dqm->sdma_bitmap = (1 << get_num_sdma_queues(dqm)) - 1;
857873
858874 return 0;
859875 }
@@ -866,8 +882,8 @@ static void uninitialize(struct device_queue_manager *dqm) · 6 unmodified lines
866882
867883 kfree(dqm->allocated_queues);
868884 for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
869− kfree(dqm->mqds[i]);
870− mutex_destroy(&dqm->lock);
885+ kfree(dqm->mqd_mgrs[i]);
886+ mutex_destroy(&dqm->lock_hidden);
871887 kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
872888 }
873889
@@ -901,7 +917,7 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm, · 27 unmodified lines
901917 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
902918 unsigned int sdma_queue_id)
903919 {
904− if (sdma_queue_id >= CIK_SDMA_QUEUES)
920+ if (sdma_queue_id >= get_num_sdma_queues(dqm))
905921 return;
906922 dqm->sdma_bitmap |= (1 << sdma_queue_id);
907923 }
@@ -910,19 +926,19 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, · 2 unmodified lines
910926 struct queue *q,
911927 struct qcm_process_device *qpd)
912928 {
913− struct mqd_manager *mqd;
929+ struct mqd_manager *mqd_mgr;
914930 int retval;
915931
916− mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
917− if (!mqd)
932+ mqd_mgr = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
933+ if (!mqd_mgr)
918934 return -ENOMEM;
919935
920936 retval = allocate_sdma_queue(dqm, &q->sdma_id);
921937 if (retval)
922938 return retval;
923939
924− q->properties.sdma_queue_id = q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
925− q->properties.sdma_engine_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
940+ q->properties.sdma_queue_id = q->sdma_id / get_num_sdma_engines(dqm);
941+ q->properties.sdma_engine_id = q->sdma_id % get_num_sdma_engines(dqm);
926942
927943 retval = allocate_doorbell(qpd, q);
928944 if (retval)
@@ -933,19 +949,20 @@ static int create_sdma_queue_nocpsch(struct device_queue_manager *dqm, · 4 unmodified lines
933949 pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
934950
935951 dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
936− retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
952+ retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
937953 &q->gart_mqd_addr, &q->properties);
938954 if (retval)
939955 goto out_deallocate_doorbell;
940956
941− retval = mqd->load_mqd(mqd, q->mqd, 0, 0, &q->properties, NULL);
957+ retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, 0, 0, &q->properties,
958+ NULL);
942959 if (retval)
943960 goto out_uninit_mqd;
944961
945962 return 0;
946963
947964 out_uninit_mqd:
948− mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
965+ mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
949966 out_deallocate_doorbell:
950967 deallocate_doorbell(qpd, q);
951968 out_deallocate_sdma_queue:
@@ -1003,12 +1020,14 @@ static int initialize_cpsch(struct device_queue_manager *dqm) · 51 unmodified lines
10031020 {
10041021 pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
10051022
1006− mutex_init(&dqm->lock);
1023+ mutex_init(&dqm->lock_hidden);
10071024 INIT_LIST_HEAD(&dqm->queues);
10081025 dqm->queue_count = dqm->processes_count = 0;
10091026 dqm->sdma_queue_count = 0;
10101027 dqm->active_runlist = false;
1011− dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
1028+ dqm->sdma_bitmap = (1 << get_num_sdma_queues(dqm)) - 1;
1029+
1030+ INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception);
10121031
10131032 return 0;
10141033 }
@@ -1041,9 +1060,11 @@ static int start_cpsch(struct device_queue_manager *dqm) · 26 unmodified lines
10411060
10421061 init_interrupts(dqm);
10431062
1044− mutex_lock(&dqm->lock);
1063+ dqm_lock(dqm);
1064+ /* clear hang status when driver try to start the hw scheduler */
1065+ dqm->is_hws_hang = false;
10451066 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
1046− mutex_unlock(&dqm->lock);
1067+ dqm_unlock(dqm);
10471068
10481069 return 0;
10491070 fail_allocate_vidmem:
@@ -1055,9 +1076,9 @@ fail_packet_manager_init: · 5 unmodified lines
10551076
10561077 static int stop_cpsch(struct device_queue_manager *dqm)
10571078 {
1058− mutex_lock(&dqm->lock);
1079+ dqm_lock(dqm);
10591080 unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
1060− mutex_unlock(&dqm->lock);
1081+ dqm_unlock(dqm);
10611082
10621083 kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
10631084 pm_uninit(&dqm->packets);
@@ -1069,11 +1090,11 @@ static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, · 5 unmodified lines
10691090 struct kernel_queue *kq,
10701091 struct qcm_process_device *qpd)
10711092 {
1072− mutex_lock(&dqm->lock);
1093+ dqm_lock(dqm);
10731094 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
10741095 pr_warn("Can't create new kernel queue because %d queues were already created\n",
10751096 dqm->total_queue_count);
1076− mutex_unlock(&dqm->lock);
1097+ dqm_unlock(dqm);
10771098 return -EPERM;
10781099 }
10791100
@@ -1089,7 +1110,7 @@ static int create_kernel_queue_cpsch(struct device_queue_manager *dqm, · 9 unmodified lines
10891110 dqm->queue_count++;
10901111 qpd->is_debug = true;
10911112 execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
1092− mutex_unlock(&dqm->lock);
1113+ dqm_unlock(dqm);
10931114
10941115 return 0;
10951116 }
@@ -1098,7 +1119,7 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, · 2 unmodified lines
10981119 struct kernel_queue *kq,
10991120 struct qcm_process_device *qpd)
11001121 {
1101− mutex_lock(&dqm->lock);
1122+ dqm_lock(dqm);
11021123 list_del(&kq->list);
11031124 dqm->queue_count--;
11041125 qpd->is_debug = false;
@@ -1110,18 +1131,18 @@ static void destroy_kernel_queue_cpsch(struct device_queue_manager *dqm, · 5 unmodified lines
11101131 dqm->total_queue_count--;
11111132 pr_debug("Total of %d queues are accountable so far\n",
11121133 dqm->total_queue_count);
1113− mutex_unlock(&dqm->lock);
1134+ dqm_unlock(dqm);
11141135 }
11151136
11161137 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
11171138 struct qcm_process_device *qpd)
11181139 {
11191140 int retval;
1120− struct mqd_manager *mqd;
1141+ struct mqd_manager *mqd_mgr;
11211142
11221143 retval = 0;
11231144
1124− mutex_lock(&dqm->lock);
1145+ dqm_lock(dqm);
11251146
11261147 if (dqm->total_queue_count >= max_num_of_queues_per_device) {
11271148 pr_warn("Can't create new usermode queue because %d queues were already created\n",
@@ -1135,19 +1156,19 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, · 7 unmodified lines
11351156 if (retval)
11361157 goto out_unlock;
11371158 q->properties.sdma_queue_id =
1138− q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
1159+ q->sdma_id / get_num_sdma_engines(dqm);
11391160 q->properties.sdma_engine_id =
1140− q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
1161+ q->sdma_id % get_num_sdma_engines(dqm);
11411162 }
11421163
11431164 retval = allocate_doorbell(qpd, q);
11441165 if (retval)
11451166 goto out_deallocate_sdma_queue;
11461167
1147− mqd = dqm->ops.get_mqd_manager(dqm,
1168+ mqd_mgr = dqm->ops.get_mqd_manager(dqm,
11481169 get_mqd_type_from_queue_type(q->properties.type));
11491170
1150− if (!mqd) {
1171+ if (!mqd_mgr) {
11511172 retval = -ENOMEM;
11521173 goto out_deallocate_doorbell;
11531174 }
@@ -1164,7 +1185,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, · 10 unmodified lines
11641185
11651186 q->properties.tba_addr = qpd->tba_addr;
11661187 q->properties.tma_addr = qpd->tma_addr;
1167− retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
1188+ retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
11681189 &q->gart_mqd_addr, &q->properties);
11691190 if (retval)
11701191 goto out_deallocate_doorbell;
@@ -1188,7 +1209,7 @@ static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q, · 17 unmodified lines
11881209 pr_debug("Total of %d queues are accountable so far\n",
11891210 dqm->total_queue_count);
11901211
1191− mutex_unlock(&dqm->lock);
1212+ dqm_unlock(dqm);
11921213 return retval;
11931214
11941215 out_deallocate_doorbell:
@@ -1197,7 +1218,8 @@ out_deallocate_sdma_queue: · 2 unmodified lines
11971218 if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
11981219 deallocate_sdma_queue(dqm, q->sdma_id);
11991220 out_unlock:
1200− mutex_unlock(&dqm->lock);
1221+ dqm_unlock(dqm);
1222+
12011223 return retval;
12021224 }
12031225
@@ -1210,6 +1232,13 @@ int amdkfd_fence_wait_timeout(unsigned int *fence_addr, · 6 unmodified lines
12101232 while (*fence_addr != fence_value) {
12111233 if (time_after(jiffies, end_jiffies)) {
12121234 pr_err("qcm fence wait loop timeout expired\n");
1235+ /* In HWS case, this is used to halt the driver thread
1236+ * in order not to mess up CP states before doing
1237+ * scandumps for FW debugging.
1238+ */
1239+ while (halt_if_hws_hang)
1240+ schedule();
1241+
12131242 return -ETIME;
12141243 }
12151244 schedule();
@@ -1254,6 +1283,8 @@ static int unmap_queues_cpsch(struct device_queue_manager *dqm, · 38 unmodified lines
12541283 {
12551284 int retval = 0;
12561285
1286+ if (dqm->is_hws_hang)
1287+ return -EIO;
12571288 if (!dqm->active_runlist)
12581289 return retval;
12591290
@@ -1292,9 +1323,13 @@ static int execute_queues_cpsch(struct device_queue_manager *dqm, · 32 unmodified lines
12921323 {
12931324 int retval;
12941325
1326+ if (dqm->is_hws_hang)
1327+ return -EIO;
12951328 retval = unmap_queues_cpsch(dqm, filter, filter_param);
12961329 if (retval) {
12971330 pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
1331+ dqm->is_hws_hang = true;
1332+ schedule_work(&dqm->hw_exception_work);
12981333 return retval;
12991334 }
13001335
@@ -1306,7 +1341,7 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, · 5 unmodified lines
13061341 struct queue *q)
13071342 {
13081343 int retval;
1309− struct mqd_manager *mqd;
1344+ struct mqd_manager *mqd_mgr;
13101345 bool preempt_all_queues;
13111346
13121347 preempt_all_queues = false;
@@ -1314,7 +1349,7 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, · 1 unmodified line
13141349 retval = 0;
13151350
13161351 /* remove queue from list to prevent rescheduling after preemption */
1317− mutex_lock(&dqm->lock);
1352+ dqm_lock(dqm);
13181353
13191354 if (qpd->is_debug) {
13201355 /*
@@ -1326,9 +1361,9 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, · 5 unmodified lines
13261361
13271362 }
13281363
1329− mqd = dqm->ops.get_mqd_manager(dqm,
1364+ mqd_mgr = dqm->ops.get_mqd_manager(dqm,
13301365 get_mqd_type_from_queue_type(q->properties.type));
1331− if (!mqd) {
1366+ if (!mqd_mgr) {
13321367 retval = -ENOMEM;
13331368 goto failed;
13341369 }
@@ -1350,7 +1385,7 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, · 15 unmodified lines
13501385 qpd->reset_wavefronts = true;
13511386 }
13521387
1353− mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
1388+ mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
13541389
13551390 /*
13561391 * Unconditionally decrement this counter, regardless of the queue's
@@ -1360,14 +1395,14 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, · 3 unmodified lines
13601395 pr_debug("Total of %d queues are accountable so far\n",
13611396 dqm->total_queue_count);
13621397
1363− mutex_unlock(&dqm->lock);
1398+ dqm_unlock(dqm);
13641399
13651400 return retval;
13661401
13671402 failed:
13681403 failed_try_destroy_debugged_queue:
13691404
1370− mutex_unlock(&dqm->lock);
1405+ dqm_unlock(dqm);
13711406 return retval;
13721407 }
13731408
@@ -1391,7 +1426,7 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm, · 17 unmodified lines
13911426 if (!dqm->asic_ops.set_cache_memory_policy)
13921427 return retval;
13931428
1394− mutex_lock(&dqm->lock);
1429+ dqm_lock(dqm);
13951430
13961431 if (alternate_aperture_size == 0) {
13971432 /* base > limit disables APE1 */
@@ -1437,7 +1472,7 @@ static bool set_cache_memory_policy(struct device_queue_manager *dqm, · 39 unmodified lines
14371472 qpd->sh_mem_ape1_limit);
14381473
14391474 out:
1440− mutex_unlock(&dqm->lock);
1475+ dqm_unlock(dqm);
14411476 return retval;
14421477 }
14431478
@@ -1468,7 +1503,7 @@ static int process_termination_nocpsch(struct device_queue_manager *dqm, · 24 unmodified lines
14681503 struct device_process_node *cur, *next_dpn;
14691504 int retval = 0;
14701505
1471− mutex_lock(&dqm->lock);
1506+ dqm_lock(dqm);
14721507
14731508 /* Clear all user mode queues */
14741509 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
@@ -1489,7 +1524,7 @@ static int process_termination_nocpsch(struct device_queue_manager *dqm, · 14 unmodified lines
14891524 }
14901525 }
14911526
1492− mutex_unlock(&dqm->lock);
1527+ dqm_unlock(dqm);
14931528 return retval;
14941529 }
14951530
@@ -1500,14 +1535,14 @@ static int process_termination_cpsch(struct device_queue_manager *dqm, · 4 unmodified lines
15001535 int retval;
15011536 struct queue *q, *next;
15021537 struct kernel_queue *kq, *kq_next;
1503− struct mqd_manager *mqd;
1538+ struct mqd_manager *mqd_mgr;
15041539 struct device_process_node *cur, *next_dpn;
15051540 enum kfd_unmap_queues_filter filter =
15061541 KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
15071542
15081543 retval = 0;
15091544
1510− mutex_lock(&dqm->lock);
1545+ dqm_lock(dqm);
15111546
15121547 /* Clean all kernel queues */
15131548 list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
@@ -1542,7 +1577,7 @@ static int process_termination_cpsch(struct device_queue_manager *dqm, · 28 unmodified lines
15421577 }
15431578
15441579 retval = execute_queues_cpsch(dqm, filter, 0);
1545− if (retval || qpd->reset_wavefronts) {
1580+ if ((!dqm->is_hws_hang) && (retval || qpd->reset_wavefronts)) {
15461581 pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
15471582 dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
15481583 qpd->reset_wavefronts = false;
@@ -1550,19 +1585,19 @@ static int process_termination_cpsch(struct device_queue_manager *dqm, · 1 unmodified line
15501585
15511586 /* lastly, free mqd resources */
15521587 list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
1553− mqd = dqm->ops.get_mqd_manager(dqm,
1588+ mqd_mgr = dqm->ops.get_mqd_manager(dqm,
15541589 get_mqd_type_from_queue_type(q->properties.type));
1555− if (!mqd) {
1590+ if (!mqd_mgr) {
15561591 retval = -ENOMEM;
15571592 goto out;
15581593 }
15591594 list_del(&q->list);
15601595 qpd->queue_count--;
1561− mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
1596+ mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
15621597 }
15631598
15641599 out:
1565− mutex_unlock(&dqm->lock);
1600+ dqm_unlock(dqm);
15661601 return retval;
15671602 }
15681603
@@ -1683,6 +1718,30 @@ void device_queue_manager_uninit(struct device_queue_manager *dqm) · 114 unmodified lines
16831718 kfree(dqm);
16841719 }
16851720
1721+int kfd_process_vm_fault(struct device_queue_manager *dqm,
1722+ unsigned int pasid)
1723+{
1724+ struct kfd_process_device *pdd;
1725+ struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
1726+ int ret = 0;
1727+
1728+ if (!p)
1729+ return -EINVAL;
1730+ pdd = kfd_get_process_device_data(dqm->dev, p);
1731+ if (pdd)
1732+ ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd);
1733+ kfd_unref_process(p);
1734+
1735+ return ret;
1736+}
1737+
1738+static void kfd_process_hw_exception(struct work_struct *work)
1739+{
1740+ struct device_queue_manager *dqm = container_of(work,
1741+ struct device_queue_manager, hw_exception_work);
1742+ dqm->dev->kfd2kgd->gpu_recover(dqm->dev->kgd);
1743+}
1744+
16861745 #if defined(CONFIG_DEBUG_FS)
16871746
16881747 static void seq_reg_dump(struct seq_file *m,
@@ -1746,8 +1805,8 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data) · 57 unmodified lines
17461805 }
17471806 }
17481807
1749− for (pipe = 0; pipe < CIK_SDMA_ENGINE_NUM; pipe++) {
1750− for (queue = 0; queue < CIK_SDMA_QUEUES_PER_ENGINE; queue++) {
1808+ for (pipe = 0; pipe < get_num_sdma_engines(dqm); pipe++) {
1809+ for (queue = 0; queue < KFD_SDMA_QUEUES_PER_ENGINE; queue++) {
17511810 r = dqm->dev->kfd2kgd->hqd_sdma_dump(
17521811 dqm->dev->kgd, pipe, queue, &dump, &n_regs);
17531812 if (r)
@@ -1764,4 +1823,16 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data) · 10 unmodified lines
17641823 return r;
17651824 }
17661825
1826+int dqm_debugfs_execute_queues(struct device_queue_manager *dqm)
1827+{
1828+ int r = 0;
1829+
1830+ dqm_lock(dqm);
1831+ dqm->active_runlist = true;
1832+ r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
1833+ dqm_unlock(dqm);
1834+
1835+ return r;
1836+}
1837+
17671838 #endif
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h+26−6
@@ -26,15 +26,14 @@ · 25 unmodified lines
2626
2727 #include <linux/rwsem.h>
2828 #include <linux/list.h>
29+#include <linux/mutex.h>
30+#include <linux/sched/mm.h>
2931 #include "kfd_priv.h"
3032 #include "kfd_mqd_manager.h"
3133
3234 #define KFD_UNMAP_LATENCY_MS (4000)
3335 #define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (2 * KFD_UNMAP_LATENCY_MS + 1000)
34−
35−#define CIK_SDMA_QUEUES (4)
36−#define CIK_SDMA_QUEUES_PER_ENGINE (2)
37−#define CIK_SDMA_ENGINE_NUM (2)
36+#define KFD_SDMA_QUEUES_PER_ENGINE (2)
3837
3938 struct device_process_node {
4039 struct qcm_process_device *qpd;
@@ -170,11 +169,12 @@ struct device_queue_manager { · 129 unmodified lines
170169 struct device_queue_manager_ops ops;
171170 struct device_queue_manager_asic_ops asic_ops;
172171
173− struct mqd_manager *mqds[KFD_MQD_TYPE_MAX];
172+ struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX];
174173 struct packet_manager packets;
175174 struct kfd_dev *dev;
176− struct mutex lock;
175+ struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */
177176 struct list_head queues;
177+ unsigned int saved_flags;
178178 unsigned int processes_count;
179179 unsigned int queue_count;
180180 unsigned int sdma_queue_count;
@@ -190,6 +190,10 @@ struct device_queue_manager { · 9 unmodified lines
190190 struct kfd_mem_obj *fence_mem;
191191 bool active_runlist;
192192 int sched_policy;
193+
194+ /* hw exception */
195+ bool is_hws_hang;
196+ struct work_struct hw_exception_work;
193197 };
194198
195199 void device_queue_manager_init_cik(
@@ -207,6 +211,7 @@ void program_sh_mem_settings(struct device_queue_manager *dqm, · 11 unmodified lines
207211 unsigned int get_queues_num(struct device_queue_manager *dqm);
208212 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
209213 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
214+unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
210215
211216 static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
212217 {
@@ -219,4 +224,19 @@ get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd) · 6 unmodified lines
219224 return (pdd->lds_base >> 60) & 0x0E;
220225 }
221226
227+/* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
228+ * happens while holding this lock anywhere to prevent deadlocks when
229+ * an MMU notifier runs in reclaim-FS context.
230+ */
231+static inline void dqm_lock(struct device_queue_manager *dqm)
232+{
233+ mutex_lock(&dqm->lock_hidden);
234+ dqm->saved_flags = memalloc_nofs_save();
235+}
236+static inline void dqm_unlock(struct device_queue_manager *dqm)
237+{
238+ memalloc_nofs_restore(dqm->saved_flags);
239+ mutex_unlock(&dqm->lock_hidden);
240+}
241+
222242 #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c+1−1
@@ -60,7 +60,7 @@ static int update_qpd_v9(struct device_queue_manager *dqm, · 59 unmodified lines
6060 qpd->sh_mem_config =
6161 SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
6262 SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
63− if (vega10_noretry &&
63+ if (noretry &&
6464 !dqm->dev->device_info->needs_iommu_device)
6565 qpd->sh_mem_config |=
6666 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c+5−4
@@ -188,9 +188,9 @@ void __iomem *kfd_get_kernel_doorbell(struct kfd_dev *kfd, · 187 unmodified lines
188188 *doorbell_off = kfd->doorbell_id_offset + inx;
189189
190190 pr_debug("Get kernel queue doorbell\n"
191− " doorbell offset == 0x%08X\n"
192− " kernel address == %p\n",
193− *doorbell_off, (kfd->doorbell_kernel_ptr + inx));
191+ " doorbell offset == 0x%08X\n"
192+ " doorbell index == 0x%x\n",
193+ *doorbell_off, inx);
194194
195195 return kfd->doorbell_kernel_ptr + inx;
196196 }
@@ -199,7 +199,8 @@ void kfd_release_kernel_doorbell(struct kfd_dev *kfd, u32 __iomem *db_addr) · 2 unmodified lines
199199 {
200200 unsigned int inx;
201201
202− inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr);
202+ inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr)
203+ * sizeof(u32) / kfd->device_info->doorbell_size;
203204
204205 mutex_lock(&kfd->doorbell_mutex);
205206 __clear_bit(inx, kfd->doorbell_available_index);
drivers/gpu/drm/amd/amdkfd/kfd_events.c+98−20
@@ -850,6 +850,13 @@ static void lookup_events_by_type_and_signal(struct kfd_process *p, · 849 unmodified lines
850850 ev->memory_exception_data = *ev_data;
851851 }
852852
853+ if (type == KFD_EVENT_TYPE_MEMORY) {
854+ dev_warn(kfd_device,
855+ "Sending SIGSEGV to HSA Process with PID %d ",
856+ p->lead_thread->pid);
857+ send_sig(SIGSEGV, p->lead_thread, 0);
858+ }
859+
853860 /* Send SIGTERM no event of type "type" has been found*/
854861 if (send_signal) {
855862 if (send_sigterm) {
@@ -904,34 +911,41 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid, · 48 unmodified lines
904911 memory_exception_data.failure.NotPresent = 1;
905912 memory_exception_data.failure.NoExecute = 0;
906913 memory_exception_data.failure.ReadOnly = 0;
907− if (vma) {
908− if (vma->vm_start > address) {
909− memory_exception_data.failure.NotPresent = 1;
910− memory_exception_data.failure.NoExecute = 0;
914+ if (vma && address >= vma->vm_start) {
915+ memory_exception_data.failure.NotPresent = 0;
916+
917+ if (is_write_requested && !(vma->vm_flags & VM_WRITE))
918+ memory_exception_data.failure.ReadOnly = 1;
919+ else
911920 memory_exception_data.failure.ReadOnly = 0;
912− } else {
913− memory_exception_data.failure.NotPresent = 0;
914− if (is_write_requested && !(vma->vm_flags & VM_WRITE))
915− memory_exception_data.failure.ReadOnly = 1;
916− else
917− memory_exception_data.failure.ReadOnly = 0;
918− if (is_execute_requested && !(vma->vm_flags & VM_EXEC))
919− memory_exception_data.failure.NoExecute = 1;
920− else
921− memory_exception_data.failure.NoExecute = 0;
922− }
921+
922+ if (is_execute_requested && !(vma->vm_flags & VM_EXEC))
923+ memory_exception_data.failure.NoExecute = 1;
924+ else
925+ memory_exception_data.failure.NoExecute = 0;
923926 }
924927
925928 up_read(&mm->mmap_sem);
926929 mmput(mm);
927930
928− mutex_lock(&p->event_mutex);
931+ pr_debug("notpresent %d, noexecute %d, readonly %d\n",
932+ memory_exception_data.failure.NotPresent,
933+ memory_exception_data.failure.NoExecute,
934+ memory_exception_data.failure.ReadOnly);
929935
930− /* Lookup events by type and signal them */
931− lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_MEMORY,
932− &memory_exception_data);
936+ /* Workaround on Raven to not kill the process when memory is freed
937+ * before IOMMU is able to finish processing all the excessive PPRs
938+ */
939+ if (dev->device_info->asic_family != CHIP_RAVEN) {
940+ mutex_lock(&p->event_mutex);
941+
942+ /* Lookup events by type and signal them */
943+ lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_MEMORY,
944+ &memory_exception_data);
945+
946+ mutex_unlock(&p->event_mutex);
947+ }
933948
934− mutex_unlock(&p->event_mutex);
935949 kfd_unref_process(p);
936950 }
937951 #endif /* KFD_SUPPORT_IOMMU_V2 */
@@ -956,3 +970,67 @@ void kfd_signal_hw_exception_event(unsigned int pasid) · 18 unmodified lines
956970 mutex_unlock(&p->event_mutex);
957971 kfd_unref_process(p);
958972 }
973+
974+void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
975+ struct kfd_vm_fault_info *info)
976+{
977+ struct kfd_event *ev;
978+ uint32_t id;
979+ struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
980+ struct kfd_hsa_memory_exception_data memory_exception_data;
981+
982+ if (!p)
983+ return; /* Presumably process exited. */
984+ memset(&memory_exception_data, 0, sizeof(memory_exception_data));
985+ memory_exception_data.gpu_id = dev->id;
986+ memory_exception_data.failure.imprecise = 1;
987+ /* Set failure reason */
988+ if (info) {
989+ memory_exception_data.va = (info->page_addr) << PAGE_SHIFT;
990+ memory_exception_data.failure.NotPresent =
991+ info->prot_valid ? 1 : 0;
992+ memory_exception_data.failure.NoExecute =
993+ info->prot_exec ? 1 : 0;
994+ memory_exception_data.failure.ReadOnly =
995+ info->prot_write ? 1 : 0;
996+ memory_exception_data.failure.imprecise = 0;
997+ }
998+ mutex_lock(&p->event_mutex);
999+
1000+ id = KFD_FIRST_NONSIGNAL_EVENT_ID;
1001+ idr_for_each_entry_continue(&p->event_idr, ev, id)
1002+ if (ev->type == KFD_EVENT_TYPE_MEMORY) {
1003+ ev->memory_exception_data = memory_exception_data;
1004+ set_event(ev);
1005+ }
1006+
1007+ mutex_unlock(&p->event_mutex);
1008+ kfd_unref_process(p);
1009+}
1010+
1011+void kfd_signal_reset_event(struct kfd_dev *dev)
1012+{
1013+ struct kfd_hsa_hw_exception_data hw_exception_data;
1014+ struct kfd_process *p;
1015+ struct kfd_event *ev;
1016+ unsigned int temp;
1017+ uint32_t id, idx;
1018+
1019+ /* Whole gpu reset caused by GPU hang and memory is lost */
1020+ memset(&hw_exception_data, 0, sizeof(hw_exception_data));
1021+ hw_exception_data.gpu_id = dev->id;
1022+ hw_exception_data.memory_lost = 1;
1023+
1024+ idx = srcu_read_lock(&kfd_processes_srcu);
1025+ hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1026+ mutex_lock(&p->event_mutex);
1027+ id = KFD_FIRST_NONSIGNAL_EVENT_ID;
1028+ idr_for_each_entry_continue(&p->event_idr, ev, id)
1029+ if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) {
1030+ ev->hw_exception_data = hw_exception_data;
1031+ set_event(ev);
1032+ }
1033+ mutex_unlock(&p->event_mutex);
1034+ }
1035+ srcu_read_unlock(&kfd_processes_srcu, idx);
1036+}
drivers/gpu/drm/amd/amdkfd/kfd_events.h+1−0
@@ -66,6 +66,7 @@ struct kfd_event { · 65 unmodified lines
6666 /* type specific data */
6767 union {
6868 struct kfd_hsa_memory_exception_data memory_exception_data;
69+ struct kfd_hsa_hw_exception_data hw_exception_data;
6970 };
7071 };
7172
drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c+19−3
@@ -26,7 +26,9 @@ · 25 unmodified lines
2626
2727
2828 static bool event_interrupt_isr_v9(struct kfd_dev *dev,
29− const uint32_t *ih_ring_entry)
29+ const uint32_t *ih_ring_entry,
30+ uint32_t *patched_ihre,
31+ bool *patched_flag)
3032 {
3133 uint16_t source_id, client_id, pasid, vmid;
3234 const uint32_t *data = ih_ring_entry;
@@ -57,7 +59,9 @@ static bool event_interrupt_isr_v9(struct kfd_dev *dev, · 24 unmodified lines
5759 return source_id == SOC15_INTSRC_CP_END_OF_PIPE ||
5860 source_id == SOC15_INTSRC_SDMA_TRAP ||
5961 source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG ||
60− source_id == SOC15_INTSRC_CP_BAD_OPCODE;
62+ source_id == SOC15_INTSRC_CP_BAD_OPCODE ||
63+ client_id == SOC15_IH_CLIENTID_VMC ||
64+ client_id == SOC15_IH_CLIENTID_UTCL2;
6165 }
6266
6367 static void event_interrupt_wq_v9(struct kfd_dev *dev,
@@ -82,7 +86,19 @@ static void event_interrupt_wq_v9(struct kfd_dev *dev, · 18 unmodified lines
8286 kfd_signal_hw_exception_event(pasid);
8387 else if (client_id == SOC15_IH_CLIENTID_VMC ||
8488 client_id == SOC15_IH_CLIENTID_UTCL2) {
85− /* TODO */
89+ struct kfd_vm_fault_info info = {0};
90+ uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry);
91+
92+ info.vmid = vmid;
93+ info.mc_id = client_id;
94+ info.page_addr = ih_ring_entry[4] |
95+ (uint64_t)(ih_ring_entry[5] & 0xf) << 32;
96+ info.prot_valid = ring_id & 0x08;
97+ info.prot_read = ring_id & 0x10;
98+ info.prot_write = ring_id & 0x20;
99+
100+ kfd_process_vm_fault(dev->dqm, pasid);
101+ kfd_signal_vm_fault_event(dev, pasid, &info);
86102 }
87103 }
88104
drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c+4−2
@@ -151,13 +151,15 @@ static void interrupt_wq(struct work_struct *work) · 150 unmodified lines
151151 ih_ring_entry);
152152 }
153153
154−bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry)
154+bool interrupt_is_wanted(struct kfd_dev *dev,
155+ const uint32_t *ih_ring_entry,
156+ uint32_t *patched_ihre, bool *flag)
155157 {
156158 /* integer and bitwise OR so there is no boolean short-circuiting */
157159 unsigned int wanted = 0;
158160
159161 wanted |= dev->device_info->event_interrupt_class->interrupt_isr(dev,
160− ih_ring_entry);
162+ ih_ring_entry, patched_ihre, flag);
161163
162164 return wanted != 0;
163165 }
drivers/gpu/drm/amd/amdkfd/kfd_iommu.c+1−1
@@ -190,7 +190,7 @@ static int iommu_invalid_ppr_cb(struct pci_dev *pdev, int pasid, · 189 unmodified lines
190190 {
191191 struct kfd_dev *dev;
192192
193− dev_warn(kfd_device,
193+ dev_warn_ratelimited(kfd_device,
194194 "Invalid PPR device %x:%x.%x pasid %d address 0x%lX flags 0x%X",
195195 PCI_BUS_NUM(pdev->devfn),
196196 PCI_SLOT(pdev->devfn),
drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c+10−8
@@ -59,7 +59,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev, · 58 unmodified lines
5959 switch (type) {
6060 case KFD_QUEUE_TYPE_DIQ:
6161 case KFD_QUEUE_TYPE_HIQ:
62− kq->mqd = dev->dqm->ops.get_mqd_manager(dev->dqm,
62+ kq->mqd_mgr = dev->dqm->ops.get_mqd_manager(dev->dqm,
6363 KFD_MQD_TYPE_HIQ);
6464 break;
6565 default:
@@ -67,7 +67,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev, · 1 unmodified line
6767 return false;
6868 }
6969
70− if (!kq->mqd)
70+ if (!kq->mqd_mgr)
7171 return false;
7272
7373 prop.doorbell_ptr = kfd_get_kernel_doorbell(dev, &prop.doorbell_off);
@@ -123,6 +123,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev, · 49 unmodified lines
123123 prop.write_ptr = (uint32_t *) kq->wptr_gpu_addr;
124124 prop.eop_ring_buffer_address = kq->eop_gpu_addr;
125125 prop.eop_ring_buffer_size = PAGE_SIZE;
126+ prop.cu_mask = NULL;
126127
127128 if (init_queue(&kq->queue, &prop) != 0)
128129 goto err_init_queue;
@@ -130,7 +131,7 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev, · 1 unmodified line
130131 kq->queue->device = dev;
131132 kq->queue->process = kfd_get_process(current);
132133
133− retval = kq->mqd->init_mqd(kq->mqd, &kq->queue->mqd,
134+ retval = kq->mqd_mgr->init_mqd(kq->mqd_mgr, &kq->queue->mqd,
134135 &kq->queue->mqd_mem_obj,
135136 &kq->queue->gart_mqd_addr,
136137 &kq->queue->properties);
@@ -142,9 +143,9 @@ static bool initialize(struct kernel_queue *kq, struct kfd_dev *dev, · 5 unmodified lines
142143 pr_debug("Assigning hiq to hqd\n");
143144 kq->queue->pipe = KFD_CIK_HIQ_PIPE;
144145 kq->queue->queue = KFD_CIK_HIQ_QUEUE;
145− kq->mqd->load_mqd(kq->mqd, kq->queue->mqd, kq->queue->pipe,
146− kq->queue->queue, &kq->queue->properties,
147− NULL);
146+ kq->mqd_mgr->load_mqd(kq->mqd_mgr, kq->queue->mqd,
147+ kq->queue->pipe, kq->queue->queue,
148+ &kq->queue->properties, NULL);
148149 } else {
149150 /* allocate fence for DIQ */
150151
@@ -182,7 +183,7 @@ err_get_kernel_doorbell: · 31 unmodified lines
182183 static void uninitialize(struct kernel_queue *kq)
183184 {
184185 if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ)
185− kq->mqd->destroy_mqd(kq->mqd,
186+ kq->mqd_mgr->destroy_mqd(kq->mqd_mgr,
186187 kq->queue->mqd,
187188 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
188189 KFD_UNMAP_LATENCY_MS,
@@ -191,7 +192,8 @@ static void uninitialize(struct kernel_queue *kq) · 2 unmodified lines
191192 else if (kq->queue->properties.type == KFD_QUEUE_TYPE_DIQ)
192193 kfd_gtt_sa_free(kq->dev, kq->fence_mem_obj);
193194
194− kq->mqd->uninit_mqd(kq->mqd, kq->queue->mqd, kq->queue->mqd_mem_obj);
195+ kq->mqd_mgr->uninit_mqd(kq->mqd_mgr, kq->queue->mqd,
196+ kq->queue->mqd_mem_obj);
195197
196198 kfd_gtt_sa_free(kq->dev, kq->rptr_mem);
197199 kfd_gtt_sa_free(kq->dev, kq->wptr_mem);
drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h+1−1
@@ -70,7 +70,7 @@ struct kernel_queue { · 69 unmodified lines
7070
7171 /* data */
7272 struct kfd_dev *dev;
73− struct mqd_manager *mqd;
73+ struct mqd_manager *mqd_mgr;
7474 struct queue *queue;
7575 uint64_t pending_wptr64;
7676 uint32_t pending_wptr;
drivers/gpu/drm/amd/amdkfd/kfd_module.c+12−4
@@ -47,6 +47,8 @@ static const struct kgd2kfd_calls kgd2kfd = { · 46 unmodified lines
4747 .resume_mm = kgd2kfd_resume_mm,
4848 .schedule_evict_and_restore_process =
4949 kgd2kfd_schedule_evict_and_restore_process,
50+ .pre_reset = kgd2kfd_pre_reset,
51+ .post_reset = kgd2kfd_post_reset,
5052 };
5153
5254 int sched_policy = KFD_SCHED_POLICY_HWS;
@@ -61,7 +63,7 @@ MODULE_PARM_DESC(hws_max_conc_proc, · 8 unmodified lines
6163
6264 int cwsr_enable = 1;
6365 module_param(cwsr_enable, int, 0444);
64−MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = Off, 1 = On (Default))");
66+MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = off, 1 = on (default))");
6567
6668 int max_num_of_queues_per_device = KFD_MAX_NUM_OF_QUEUES_PER_DEVICE_DEFAULT;
6769 module_param(max_num_of_queues_per_device, int, 0444);
@@ -83,13 +85,19 @@ module_param(ignore_crat, int, 0444); · 15 unmodified lines
8385 MODULE_PARM_DESC(ignore_crat,
8486 "Ignore CRAT table during KFD initialization (0 = use CRAT (default), 1 = ignore CRAT)");
8587
86−int vega10_noretry;
87−module_param_named(noretry, vega10_noretry, int, 0644);
88+int noretry;
89+module_param(noretry, int, 0644);
8890 MODULE_PARM_DESC(noretry,
89− "Set sh_mem_config.retry_disable on Vega10 (0 = retry enabled (default), 1 = retry disabled)");
91+ "Set sh_mem_config.retry_disable on GFXv9+ dGPUs (0 = retry enabled (default), 1 = retry disabled)");
92+
93+int halt_if_hws_hang;
94+module_param(halt_if_hws_hang, int, 0644);
95+MODULE_PARM_DESC(halt_if_hws_hang, "Halt if HWS hang is detected (0 = off (default), 1 = on)");
96+
9097
9198 static int amdkfd_init_completed;
9299
100+
93101 int kgd2kfd_init(unsigned int interface_version,
94102 const struct kgd2kfd_calls **g2f)
95103 {
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c+40−1
@@ -21,7 +21,7 @@ · 20 unmodified lines
2121 *
2222 */
2323
24−#include "kfd_priv.h"
24+#include "kfd_mqd_manager.h"
2525
2626 struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
2727 struct kfd_dev *dev)
@@ -48,3 +48,42 @@ struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, · 20 unmodified lines
4848
4949 return NULL;
5050 }
51+
52+void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm,
53+ const uint32_t *cu_mask, uint32_t cu_mask_count,
54+ uint32_t *se_mask)
55+{
56+ struct kfd_cu_info cu_info;
57+ uint32_t cu_per_sh[4] = {0};
58+ int i, se, cu = 0;
59+
60+ mm->dev->kfd2kgd->get_cu_info(mm->dev->kgd, &cu_info);
61+
62+ if (cu_mask_count > cu_info.cu_active_number)
63+ cu_mask_count = cu_info.cu_active_number;
64+
65+ for (se = 0; se < cu_info.num_shader_engines; se++)
66+ for (i = 0; i < 4; i++)
67+ cu_per_sh[se] += hweight32(cu_info.cu_bitmap[se][i]);
68+
69+ /* Symmetrically map cu_mask to all SEs:
70+ * cu_mask[0] bit0 -> se_mask[0] bit0;
71+ * cu_mask[0] bit1 -> se_mask[1] bit0;
72+ * ... (if # SE is 4)
73+ * cu_mask[0] bit4 -> se_mask[0] bit1;
74+ * ...
75+ */
76+ se = 0;
77+ for (i = 0; i < cu_mask_count; i++) {
78+ if (cu_mask[i / 32] & (1 << (i % 32)))
79+ se_mask[se] |= 1 << cu;
80+
81+ do {
82+ se++;
83+ if (se == cu_info.num_shader_engines) {
84+ se = 0;
85+ cu++;
86+ }
87+ } while (cu >= cu_per_sh[se] && cu < 32);
88+ }
89+}
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h+4−0
@@ -93,4 +93,8 @@ struct mqd_manager { · 92 unmodified lines
9393 struct kfd_dev *dev;
9494 };
9595
96+void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm,
97+ const uint32_t *cu_mask, uint32_t cu_mask_count,
98+ uint32_t *se_mask);
99+
96100 #endif /* KFD_MQD_MANAGER_H_ */
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c+28−1
@@ -41,6 +41,31 @@ static inline struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) · 40 unmodified lines
4141 return (struct cik_sdma_rlc_registers *)mqd;
4242 }
4343
44+static void update_cu_mask(struct mqd_manager *mm, void *mqd,
45+ struct queue_properties *q)
46+{
47+ struct cik_mqd *m;
48+ uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
49+
50+ if (q->cu_mask_count == 0)
51+ return;
52+
53+ mqd_symmetrically_map_cu_mask(mm,
54+ q->cu_mask, q->cu_mask_count, se_mask);
55+
56+ m = get_mqd(mqd);
57+ m->compute_static_thread_mgmt_se0 = se_mask[0];
58+ m->compute_static_thread_mgmt_se1 = se_mask[1];
59+ m->compute_static_thread_mgmt_se2 = se_mask[2];
60+ m->compute_static_thread_mgmt_se3 = se_mask[3];
61+
62+ pr_debug("Update cu mask to %#x %#x %#x %#x\n",
63+ m->compute_static_thread_mgmt_se0,
64+ m->compute_static_thread_mgmt_se1,
65+ m->compute_static_thread_mgmt_se2,
66+ m->compute_static_thread_mgmt_se3);
67+}
68+
4469 static int init_mqd(struct mqd_manager *mm, void **mqd,
4570 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
4671 struct queue_properties *q)
@@ -196,6 +221,8 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd, · 149 unmodified lines
196221 if (q->format == KFD_QUEUE_FORMAT_AQL)
197222 m->cp_hqd_pq_control |= NO_UPDATE_RPTR;
198223
224+ update_cu_mask(mm, mqd, q);
225+
199226 q->is_active = (q->queue_size > 0 &&
200227 q->queue_address != 0 &&
201228 q->queue_percent > 0 &&
@@ -408,7 +435,7 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type, · 206 unmodified lines
408435 if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
409436 return NULL;
410437
411− mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
438+ mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
412439 if (!mqd)
413440 return NULL;
414441
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c+29−2
@@ -41,6 +41,31 @@ static inline struct v9_sdma_mqd *get_sdma_mqd(void *mqd) · 40 unmodified lines
4141 return (struct v9_sdma_mqd *)mqd;
4242 }
4343
44+static void update_cu_mask(struct mqd_manager *mm, void *mqd,
45+ struct queue_properties *q)
46+{
47+ struct v9_mqd *m;
48+ uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
49+
50+ if (q->cu_mask_count == 0)
51+ return;
52+
53+ mqd_symmetrically_map_cu_mask(mm,
54+ q->cu_mask, q->cu_mask_count, se_mask);
55+
56+ m = get_mqd(mqd);
57+ m->compute_static_thread_mgmt_se0 = se_mask[0];
58+ m->compute_static_thread_mgmt_se1 = se_mask[1];
59+ m->compute_static_thread_mgmt_se2 = se_mask[2];
60+ m->compute_static_thread_mgmt_se3 = se_mask[3];
61+
62+ pr_debug("update cu mask to %#x %#x %#x %#x\n",
63+ m->compute_static_thread_mgmt_se0,
64+ m->compute_static_thread_mgmt_se1,
65+ m->compute_static_thread_mgmt_se2,
66+ m->compute_static_thread_mgmt_se3);
67+}
68+
4469 static int init_mqd(struct mqd_manager *mm, void **mqd,
4570 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
4671 struct queue_properties *q)
@@ -55,7 +80,7 @@ static int init_mqd(struct mqd_manager *mm, void **mqd, · 8 unmodified lines
5580 * instead of sub-allocation function.
5681 */
5782 if (kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
58− *mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);
83+ *mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
5984 if (!*mqd_mem_obj)
6085 return -ENOMEM;
6186 retval = kfd->kfd2kgd->init_gtt_mem_allocation(kfd->kgd,
@@ -198,6 +223,8 @@ static int update_mqd(struct mqd_manager *mm, void *mqd, · 136 unmodified lines
198223 if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address)
199224 m->cp_hqd_ctx_save_control = 0;
200225
226+ update_cu_mask(mm, mqd, q);
227+
201228 q->is_active = (q->queue_size > 0 &&
202229 q->queue_address != 0 &&
203230 q->queue_percent > 0 &&
@@ -393,7 +420,7 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type, · 189 unmodified lines
393420 if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
394421 return NULL;
395422
396− mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
423+ mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
397424 if (!mqd)
398425 return NULL;
399426
drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c+28−1
@@ -43,6 +43,31 @@ static inline struct vi_sdma_mqd *get_sdma_mqd(void *mqd) · 42 unmodified lines
4343 return (struct vi_sdma_mqd *)mqd;
4444 }
4545
46+static void update_cu_mask(struct mqd_manager *mm, void *mqd,
47+ struct queue_properties *q)
48+{
49+ struct vi_mqd *m;
50+ uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
51+
52+ if (q->cu_mask_count == 0)
53+ return;
54+
55+ mqd_symmetrically_map_cu_mask(mm,
56+ q->cu_mask, q->cu_mask_count, se_mask);
57+
58+ m = get_mqd(mqd);
59+ m->compute_static_thread_mgmt_se0 = se_mask[0];
60+ m->compute_static_thread_mgmt_se1 = se_mask[1];
61+ m->compute_static_thread_mgmt_se2 = se_mask[2];
62+ m->compute_static_thread_mgmt_se3 = se_mask[3];
63+
64+ pr_debug("Update cu mask to %#x %#x %#x %#x\n",
65+ m->compute_static_thread_mgmt_se0,
66+ m->compute_static_thread_mgmt_se1,
67+ m->compute_static_thread_mgmt_se2,
68+ m->compute_static_thread_mgmt_se3);
69+}
70+
4671 static int init_mqd(struct mqd_manager *mm, void **mqd,
4772 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr,
4873 struct queue_properties *q)
@@ -196,6 +221,8 @@ static int __update_mqd(struct mqd_manager *mm, void *mqd, · 147 unmodified lines
196221 atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT |
197222 mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;
198223
224+ update_cu_mask(mm, mqd, q);
225+
199226 q->is_active = (q->queue_size > 0 &&
200227 q->queue_address != 0 &&
201228 q->queue_percent > 0 &&
@@ -394,7 +421,7 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type, · 192 unmodified lines
394421 if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
395422 return NULL;
396423
397− mqd = kzalloc(sizeof(*mqd), GFP_NOIO);
424+ mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
398425 if (!mqd)
399426 return NULL;
400427
drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c+26−0
@@ -418,4 +418,30 @@ out: · 417 unmodified lines
418418 return 0;
419419 }
420420
421+int pm_debugfs_hang_hws(struct packet_manager *pm)
422+{
423+ uint32_t *buffer, size;
424+ int r = 0;
425+
426+ size = pm->pmf->query_status_size;
427+ mutex_lock(&pm->lock);
428+ pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,
429+ size / sizeof(uint32_t), (unsigned int **)&buffer);
430+ if (!buffer) {
431+ pr_err("Failed to allocate buffer on kernel queue\n");
432+ r = -ENOMEM;
433+ goto out;
434+ }
435+ memset(buffer, 0x55, size);
436+ pm->priv_queue->ops.submit_packet(pm->priv_queue);
437+
438+ pr_info("Submitting %x %x %x %x %x %x %x to HIQ to hang the HWS.",
439+ buffer[0], buffer[1], buffer[2], buffer[3],
440+ buffer[4], buffer[5], buffer[6]);
441+out:
442+ mutex_unlock(&pm->lock);
443+ return r;
444+}
445+
446+
421447 #endif
drivers/gpu/drm/amd/amdkfd/kfd_priv.h+35−5
@@ -73,7 +73,7 @@ · 72 unmodified lines
7373
7474 /*
7575 * When working with cp scheduler we should assign the HIQ manually or via
76− * the radeon driver to a fixed hqd slot, here are the fixed HIQ hqd slot
76+ * the amdgpu driver to a fixed hqd slot, here are the fixed HIQ hqd slot
7777 * definitions for Kaveri. In Kaveri only the first ME queues participates
7878 * in the cp scheduling taking that in mind we set the HIQ slot in the
7979 * second ME.
@@ -142,7 +142,12 @@ extern int ignore_crat; · 62 unmodified lines
142142 /*
143143 * Set sh_mem_config.retry_disable on Vega10
144144 */
145−extern int vega10_noretry;
145+extern int noretry;
146+
147+/*
148+ * Halt if HWS hang is detected
149+ */
150+extern int halt_if_hws_hang;
146151
147152 /**
148153 * enum kfd_sched_policy
@@ -180,9 +185,10 @@ enum cache_policy { · 31 unmodified lines
180185
181186 struct kfd_event_interrupt_class {
182187 bool (*interrupt_isr)(struct kfd_dev *dev,
183− const uint32_t *ih_ring_entry);
188+ const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
189+ bool *patched_flag);
184190 void (*interrupt_wq)(struct kfd_dev *dev,
185− const uint32_t *ih_ring_entry);
191+ const uint32_t *ih_ring_entry);
186192 };
187193
188194 struct kfd_device_info {
@@ -197,6 +203,7 @@ struct kfd_device_info { · 8 unmodified lines
197203 bool supports_cwsr;
198204 bool needs_iommu_device;
199205 bool needs_pci_atomics;
206+ unsigned int num_sdma_engines;
200207 };
201208
202209 struct kfd_mem_obj {
@@ -415,6 +422,9 @@ struct queue_properties { · 212 unmodified lines
415422 uint32_t ctl_stack_size;
416423 uint64_t tba_addr;
417424 uint64_t tma_addr;
425+ /* Relevant for CU */
426+ uint32_t cu_mask_count; /* Must be a multiple of 32 */
427+ uint32_t *cu_mask;
418428 };
419429
420430 /**
@@ -806,12 +816,18 @@ int kfd_interrupt_init(struct kfd_dev *dev); · 385 unmodified lines
806816 void kfd_interrupt_exit(struct kfd_dev *dev);
807817 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);
808818 bool enqueue_ih_ring_entry(struct kfd_dev *kfd, const void *ih_ring_entry);
809−bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry);
819+bool interrupt_is_wanted(struct kfd_dev *dev,
820+ const uint32_t *ih_ring_entry,
821+ uint32_t *patched_ihre, bool *flag);
810822
811823 /* Power Management */
812824 void kgd2kfd_suspend(struct kfd_dev *kfd);
813825 int kgd2kfd_resume(struct kfd_dev *kfd);
814826
827+/* GPU reset */
828+int kgd2kfd_pre_reset(struct kfd_dev *kfd);
829+int kgd2kfd_post_reset(struct kfd_dev *kfd);
830+
815831 /* amdkfd Apertures */
816832 int kfd_init_apertures(struct kfd_process *process);
817833
@@ -838,6 +854,7 @@ void device_queue_manager_uninit(struct device_queue_manager *dqm); · 20 unmodified lines
838854 struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
839855 enum kfd_queue_type type);
840856 void kernel_queue_uninit(struct kernel_queue *kq);
857+int kfd_process_vm_fault(struct device_queue_manager *dqm, unsigned int pasid);
841858
842859 /* Process Queue Manager */
843860 struct process_queue_node {
@@ -858,6 +875,8 @@ int pqm_create_queue(struct process_queue_manager *pqm, · 14 unmodified lines
858875 int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
859876 int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
860877 struct queue_properties *p);
878+int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
879+ struct queue_properties *p);
861880 struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
862881 unsigned int qid);
863882
@@ -964,10 +983,17 @@ int kfd_event_create(struct file *devkfd, struct kfd_process *p, · 100 unmodified lines
964983 uint64_t *event_page_offset, uint32_t *event_slot_index);
965984 int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
966985
986+void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
987+ struct kfd_vm_fault_info *info);
988+
989+void kfd_signal_reset_event(struct kfd_dev *dev);
990+
967991 void kfd_flush_tlb(struct kfd_process_device *pdd);
968992
969993 int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);
970994
995+bool kfd_is_locked(void);
996+
971997 /* Debugfs */
972998 #if defined(CONFIG_DEBUG_FS)
973999
@@ -980,6 +1006,10 @@ int dqm_debugfs_hqds(struct seq_file *m, void *data); · 6 unmodified lines
9801006 int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
9811007 int pm_debugfs_runlist(struct seq_file *m, void *data);
9821008
1009+int kfd_debugfs_hang_hws(struct kfd_dev *dev);
1010+int pm_debugfs_hang_hws(struct packet_manager *pm);
1011+int dqm_debugfs_execute_queues(struct device_queue_manager *dqm);
1012+
9831013 #else
9841014
9851015 static inline void kfd_debugfs_init(void) {}
drivers/gpu/drm/amd/amdkfd/kfd_process.c+2−0
@@ -244,6 +244,8 @@ struct kfd_process *kfd_get_process(const struct task_struct *thread) · 243 unmodified lines
244244 return ERR_PTR(-EINVAL);
245245
246246 process = find_process(thread);
247+ if (!process)
248+ return ERR_PTR(-EINVAL);
247249
248250 return process;
249251 }
drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c+36−7
@@ -186,8 +186,7 @@ int pqm_create_queue(struct process_queue_manager *pqm, · 185 unmodified lines
186186
187187 switch (type) {
188188 case KFD_QUEUE_TYPE_SDMA:
189− if (dev->dqm->queue_count >=
190− CIK_SDMA_QUEUES_PER_ENGINE * CIK_SDMA_ENGINE_NUM) {
189+ if (dev->dqm->queue_count >= get_num_sdma_queues(dev->dqm)) {
191190 pr_err("Over-subscription is not allowed for SDMA.\n");
192191 retval = -EPERM;
193192 goto err_create_queue;
@@ -209,7 +208,7 @@ int pqm_create_queue(struct process_queue_manager *pqm, · 15 unmodified lines
209208 KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
210209 ((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
211210 (dev->dqm->queue_count >= get_queues_num(dev->dqm)))) {
212− pr_err("Over-subscription is not allowed in radeon_kfd.sched_policy == 1\n");
211+ pr_debug("Over-subscription is not allowed when amdkfd.sched_policy == 1\n");
213212 retval = -EPERM;
214213 goto err_create_queue;
215214 }
@@ -326,6 +325,8 @@ int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid) · 110 unmodified lines
326325 if (retval != -ETIME)
327326 goto err_destroy_queue;
328327 }
328+ kfree(pqn->q->properties.cu_mask);
329+ pqn->q->properties.cu_mask = NULL;
329330 uninit_queue(pqn->q);
330331 }
331332
@@ -366,6 +367,34 @@ int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid, · 34 unmodified lines
366367 return 0;
367368 }
368369
370+int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
371+ struct queue_properties *p)
372+{
373+ int retval;
374+ struct process_queue_node *pqn;
375+
376+ pqn = get_queue_by_qid(pqm, qid);
377+ if (!pqn) {
378+ pr_debug("No queue %d exists for update operation\n", qid);
379+ return -EFAULT;
380+ }
381+
382+ /* Free the old CU mask memory if it is already allocated, then
383+ * allocate memory for the new CU mask.
384+ */
385+ kfree(pqn->q->properties.cu_mask);
386+
387+ pqn->q->properties.cu_mask_count = p->cu_mask_count;
388+ pqn->q->properties.cu_mask = p->cu_mask;
389+
390+ retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
391+ pqn->q);
392+ if (retval != 0)
393+ return retval;
394+
395+ return 0;
396+}
397+
369398 struct kernel_queue *pqm_get_kernel_queue(
370399 struct process_queue_manager *pqm,
371400 unsigned int qid)
@@ -387,7 +416,7 @@ int pqm_debugfs_mqds(struct seq_file *m, void *data) · 15 unmodified lines
387416 struct process_queue_node *pqn;
388417 struct queue *q;
389418 enum KFD_MQD_TYPE mqd_type;
390− struct mqd_manager *mqd_manager;
419+ struct mqd_manager *mqd_mgr;
391420 int r = 0;
392421
393422 list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
@@ -410,11 +439,11 @@ int pqm_debugfs_mqds(struct seq_file *m, void *data) · 16 unmodified lines
410439 q->properties.type, q->device->id);
411440 continue;
412441 }
413− mqd_manager = q->device->dqm->ops.get_mqd_manager(
442+ mqd_mgr = q->device->dqm->ops.get_mqd_manager(
414443 q->device->dqm, mqd_type);
415444 } else if (pqn->kq) {
416445 q = pqn->kq->queue;
417− mqd_manager = pqn->kq->mqd;
446+ mqd_mgr = pqn->kq->mqd_mgr;
418447 switch (q->properties.type) {
419448 case KFD_QUEUE_TYPE_DIQ:
420449 seq_printf(m, " DIQ on device %x\n",
@@ -434,7 +463,7 @@ int pqm_debugfs_mqds(struct seq_file *m, void *data) · 13 unmodified lines
434463 continue;
435464 }
436465
437− r = mqd_manager->debugfs_show_mqd(m, q->mqd);
466+ r = mqd_mgr->debugfs_show_mqd(m, q->mqd);
438467 if (r != 0)
439468 break;
440469 }
drivers/gpu/drm/amd/include/kgd_kfd_interface.h+40−0
@@ -47,6 +47,17 @@ enum kfd_preempt_type { · 46 unmodified lines
4747 KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
4848 };
4949
50+struct kfd_vm_fault_info {
51+ uint64_t page_addr;
52+ uint32_t vmid;
53+ uint32_t mc_id;
54+ uint32_t status;
55+ bool prot_valid;
56+ bool prot_read;
57+ bool prot_write;
58+ bool prot_exec;
59+};
60+
5061 struct kfd_cu_info {
5162 uint32_t num_shader_engines;
5263 uint32_t num_shader_arrays_per_engine;
@@ -259,6 +270,21 @@ struct tile_config { · 206 unmodified lines
259270 * IB to the corresponding ring (ring type). The IB is executed with the
260271 * specified VMID in a user mode context.
261272 *
273+ * @get_vm_fault_info: Return information about a recent VM fault on
274+ * GFXv7 and v8. If multiple VM faults occurred since the last call of
275+ * this function, it will return information about the first of those
276+ * faults. On GFXv9 VM fault information is fully contained in the IH
277+ * packet and this function is not needed.
278+ *
279+ * @read_vmid_from_vmfault_reg: On Hawaii the VMID is not set in the
280+ * IH ring entry. This function allows the KFD ISR to get the VMID
281+ * from the fault status register as early as possible.
282+ *
283+ * @gpu_recover: let kgd reset gpu after kfd detect CPC hang
284+ *
285+ * @set_compute_idle: Indicates that compute is idle on a device. This
286+ * can be used to change power profiles depending on compute activity.
287+ *
262288 * This structure contains function pointers to services that the kgd driver
263289 * provides to amdkfd driver.
264290 *
@@ -374,6 +400,14 @@ struct kfd2kgd_calls { · 109 unmodified lines
374400 int (*submit_ib)(struct kgd_dev *kgd, enum kgd_engine_type engine,
375401 uint32_t vmid, uint64_t gpu_addr,
376402 uint32_t *ib_cmd, uint32_t ib_len);
403+
404+ int (*get_vm_fault_info)(struct kgd_dev *kgd,
405+ struct kfd_vm_fault_info *info);
406+ uint32_t (*read_vmid_from_vmfault_reg)(struct kgd_dev *kgd);
407+
408+ void (*gpu_recover)(struct kgd_dev *kgd);
409+
410+ void (*set_compute_idle)(struct kgd_dev *kgd, bool idle);
377411 };
378412
379413 /**
@@ -399,6 +433,10 @@ struct kfd2kgd_calls { · 19 unmodified lines
399433 * @schedule_evict_and_restore_process: Schedules work queue that will prepare
400434 * for safe eviction of KFD BOs that belong to the specified process.
401435 *
436+ * @pre_reset: Notifies amdkfd that amdgpu about to reset the gpu
437+ *
438+ * @post_reset: Notify amdkfd that amgpu successfully reseted the gpu
439+ *
402440 * This structure contains function callback pointers so the kgd driver
403441 * will notify to the amdkfd about certain status changes.
404442 *
@@ -417,6 +455,8 @@ struct kgd2kfd_calls { · 12 unmodified lines
417455 int (*resume_mm)(struct mm_struct *mm);
418456 int (*schedule_evict_and_restore_process)(struct mm_struct *mm,
419457 struct dma_fence *fence);
458+ int (*pre_reset)(struct kfd_dev *kfd);
459+ int (*post_reset)(struct kfd_dev *kfd);
420460 };
421461
422462 int kgd2kfd_init(unsigned interface_version,
include/uapi/linux/kfd_ioctl.h+30−3
@@ -76,6 +76,12 @@ struct kfd_ioctl_update_queue_args { · 75 unmodified lines
7676 __u32 queue_priority; /* to KFD */
7777 };
7878
79+struct kfd_ioctl_set_cu_mask_args {
80+ __u32 queue_id; /* to KFD */
81+ __u32 num_cu_mask; /* to KFD */
82+ __u64 cu_mask_ptr; /* to KFD */
83+};
84+
7985 /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */
8086 #define KFD_IOC_CACHE_POLICY_COHERENT 0
8187 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1
@@ -189,6 +195,15 @@ struct kfd_ioctl_dbg_wave_control_args { · 107 unmodified lines
189195
190196 #define KFD_SIGNAL_EVENT_LIMIT 4096
191197
198+/* For kfd_event_data.hw_exception_data.reset_type. */
199+#define KFD_HW_EXCEPTION_WHOLE_GPU_RESET 0
200+#define KFD_HW_EXCEPTION_PER_ENGINE_RESET 1
201+
202+/* For kfd_event_data.hw_exception_data.reset_cause. */
203+#define KFD_HW_EXCEPTION_GPU_HANG 0
204+#define KFD_HW_EXCEPTION_ECC 1
205+
206+
192207 struct kfd_ioctl_create_event_args {
193208 __u64 event_page_offset; /* from KFD */
194209 __u32 event_trigger_data; /* from KFD - signal events only */
@@ -219,7 +234,7 @@ struct kfd_memory_exception_failure { · 24 unmodified lines
219234 __u32 NotPresent; /* Page not present or supervisor privilege */
220235 __u32 ReadOnly; /* Write access to a read-only page */
221236 __u32 NoExecute; /* Execute access to a page marked NX */
222− __u32 pad;
237+ __u32 imprecise; /* Can't determine the exact fault address */
223238 };
224239
225240 /* memory exception data*/
@@ -230,10 +245,19 @@ struct kfd_hsa_memory_exception_data { · 4 unmodified lines
230245 __u32 pad;
231246 };
232247
233−/* Event data*/
248+/* hw exception data */
249+struct kfd_hsa_hw_exception_data {
250+ uint32_t reset_type;
251+ uint32_t reset_cause;
252+ uint32_t memory_lost;
253+ uint32_t gpu_id;
254+};
255+
256+/* Event data */
234257 struct kfd_event_data {
235258 union {
236259 struct kfd_hsa_memory_exception_data memory_exception_data;
260+ struct kfd_hsa_hw_exception_data hw_exception_data;
237261 }; /* From KFD */
238262 __u64 kfd_event_data_ext; /* pointer to an extension structure
239263 for future exception types */
@@ -448,7 +472,10 @@ struct kfd_ioctl_unmap_memory_from_gpu_args { · 208 unmodified lines
448472 #define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU \
449473 AMDKFD_IOWR(0x19, struct kfd_ioctl_unmap_memory_from_gpu_args)
450474
475+#define AMDKFD_IOC_SET_CU_MASK \
476+ AMDKFD_IOW(0x1A, struct kfd_ioctl_set_cu_mask_args)
477+
451478 #define AMDKFD_COMMAND_START 0x01
452−#define AMDKFD_COMMAND_END 0x1A
479+#define AMDKFD_COMMAND_END 0x1B
453480
454481 #endif

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t