lael-labs/linux

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c+35−0

@@ -243,6 +243,33 @@ int amdgpu_amdkfd_resume(struct amdgpu_device *adev) · 242 unmodified lines
243	return r;	243	return r;
244	}	244	}
245		245
		246	int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev)
		247	{
		248	int r = 0;
		249
		250	if (adev->kfd)
		251	r = kgd2kfd->pre_reset(adev->kfd);
		252
		253	return r;
		254	}
		255
		256	int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev)
		257	{
		258	int r = 0;
		259
		260	if (adev->kfd)
		261	r = kgd2kfd->post_reset(adev->kfd);
		262
		263	return r;
		264	}
		265
		266	void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd)
		267	{
		268	struct amdgpu_device adev = (struct amdgpu_device )kgd;
		269
		270	amdgpu_device_gpu_recover(adev, NULL, false);
		271	}
		272
246	int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,	273	int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
247	void *mem_obj, uint64_t gpu_addr,	274	void *mem_obj, uint64_t gpu_addr,
248	void **cpu_ptr)	275	void **cpu_ptr)
@@ -461,6 +488,14 @@ err: · 212 unmodified lines
461	return ret;	488	return ret;
462	}	489	}
463		490
		491	void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle)
		492	{
		493	struct amdgpu_device adev = (struct amdgpu_device )kgd;
		494
		495	amdgpu_dpm_switch_power_profile(adev,
		496	PP_SMC_POWER_PROFILE_COMPUTE, !idle);
		497	}
		498
464	bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid)	499	bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid)
465	{	500	{
466	if (adev->kfd) {	501	if (adev->kfd) {

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h+10−0

 int amdgpu_amdkfd_submit_ib(struct kgd_dev *kgd, enum kgd_engine_type engine,
 				uint32_t vmid, uint64_t gpu_addr,
 				uint32_t *ib_cmd, uint32_t ib_len);
+void amdgpu_amdkfd_set_compute_idle(struct kgd_dev *kgd, bool idle);
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void);
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void);
 bool amdgpu_amdkfd_is_kfd_vmid(struct amdgpu_device *adev, u32 vmid);
+int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev);
+int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev);
+void amdgpu_amdkfd_gpu_reset(struct kgd_dev *kgd);
 /* Shared API */
 int alloc_gtt_mem(struct kgd_dev *kgd, size_t size,
 			void **mem_obj, uint64_t *gpu_addr,
 int amdgpu_amdkfd_gpuvm_restore_process_bos(void *process_info,
 					    struct dma_fence **ef);
+int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
+					      struct kfd_vm_fault_info *info);
 void amdgpu_amdkfd_gpuvm_init_mem_limits(void);
 void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo);

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c+27−0

 		uint32_t page_table_base);
 static int invalidate_tlbs(struct kgd_dev *kgd, uint16_t pasid);
 static int invalidate_tlbs_vmid(struct kgd_dev *kgd, uint16_t vmid);
+static uint32_t read_vmid_from_vmfault_reg(struct kgd_dev *kgd);
 /* Because of REG_GET_FIELD() being used, we put this function in the
  * asic specific file.
 	.invalidate_tlbs = invalidate_tlbs,
 	.invalidate_tlbs_vmid = invalidate_tlbs_vmid,
 	.submit_ib = amdgpu_amdkfd_submit_ib,
+	.get_vm_fault_info = amdgpu_amdkfd_gpuvm_get_vm_fault_info,
+	.read_vmid_from_vmfault_reg = read_vmid_from_vmfault_reg,
+	.gpu_recover = amdgpu_amdkfd_gpu_reset,
+	.set_compute_idle = amdgpu_amdkfd_set_compute_idle
 };
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_7_get_functions(void)
 	unsigned long flags, end_jiffies;
 	int retry;
+	if (adev->in_gpu_reset)
+		return -EIO;
 	acquire_queue(kgd, pipe_id, queue_id);
 	WREG32(mmCP_HQD_PQ_DOORBELL_CONTROL, 0);
 	int vmid;
 	unsigned int tmp;
+	if (adev->in_gpu_reset)
+		return -EIO;
 	for (vmid = 0; vmid < 16; vmid++) {
 		if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))
 			continue;
 	RREG32(mmVM_INVALIDATE_RESPONSE);
 	return 0;
 }
+ /**
+  * read_vmid_from_vmfault_reg - read vmid from register
+  *
+  * adev: amdgpu_device pointer
+  * @vmid: vmid pointer
+  * read vmid from register (CIK).
+  */
+static uint32_t read_vmid_from_vmfault_reg(struct kgd_dev *kgd)
+{
+	struct amdgpu_device *adev = get_amdgpu_device(kgd);
+	uint32_t status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
+	return REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS, VMID);
+}

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c+9−0

 	.invalidate_tlbs = invalidate_tlbs,
 	.invalidate_tlbs_vmid = invalidate_tlbs_vmid,
 	.submit_ib = amdgpu_amdkfd_submit_ib,
+	.get_vm_fault_info = amdgpu_amdkfd_gpuvm_get_vm_fault_info,
+	.gpu_recover = amdgpu_amdkfd_gpu_reset,
+	.set_compute_idle = amdgpu_amdkfd_set_compute_idle
 };
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_8_0_get_functions(void)
 	int retry;
 	struct vi_mqd *m = get_mqd(mqd);
+	if (adev->in_gpu_reset)
+		return -EIO;
 	acquire_queue(kgd, pipe_id, queue_id);
 	if (m->cp_hqd_vmid == 0)
 	int vmid;
 	unsigned int tmp;
+	if (adev->in_gpu_reset)
+		return -EIO;
 	for (vmid = 0; vmid < 16; vmid++) {
 		if (!amdgpu_amdkfd_is_kfd_vmid(adev, vmid))
 			continue;

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c+8−0

 	.invalidate_tlbs = invalidate_tlbs,
 	.invalidate_tlbs_vmid = invalidate_tlbs_vmid,
 	.submit_ib = amdgpu_amdkfd_submit_ib,
+	.gpu_recover = amdgpu_amdkfd_gpu_reset,
+	.set_compute_idle = amdgpu_amdkfd_set_compute_idle
 };
 struct kfd2kgd_calls *amdgpu_amdkfd_gfx_9_0_get_functions(void)
 	uint32_t temp;
 	struct v9_mqd *m = get_mqd(mqd);
+	if (adev->in_gpu_reset)
+		return -EIO;
 	acquire_queue(kgd, pipe_id, queue_id);
 	if (m->cp_hqd_vmid == 0)
 	int vmid;
 	struct amdgpu_ring *ring = &adev->gfx.kiq.ring;
+	if (adev->in_gpu_reset)
+		return -EIO;
 	if (ring->ready)
 		return invalidate_tlbs_with_kiq(adev, pasid);

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c+14−0

@@ -1621,6 +1621,20 @@ bo_reserve_failed: · 1620 unmodified lines
1621	return ret;	1621	return ret;
1622	}	1622	}
1623		1623
		1624	int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd,
		1625	struct kfd_vm_fault_info *mem)
		1626	{
		1627	struct amdgpu_device *adev;
		1628
		1629	adev = (struct amdgpu_device *)kgd;
		1630	if (atomic_read(&adev->gmc.vm_fault_info_updated) == 1) {
		1631	mem = adev->gmc.vm_fault_info;
		1632	mb();
		1633	atomic_set(&adev->gmc.vm_fault_info_updated, 0);
		1634	}
		1635	return 0;
		1636	}
		1637
1624	/* Evict a userptr BO by stopping the queues if necessary	1638	/* Evict a userptr BO by stopping the queues if necessary
1625	*	1639	*
1626	* Runs in MMU notifier, may be in RECLAIM_FS context. This means it	1640	* Runs in MMU notifier, may be in RECLAIM_FS context. This means it

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c+6−1

 	atomic_inc(&adev->gpu_reset_counter);
 	adev->in_gpu_reset = 1;
+	/* Block kfd */
+	amdgpu_amdkfd_pre_reset(adev);
 	/* block TTM */
 	resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
 		if (job && job->base.sched == &ring->sched)
 			continue;
-		drm_sched_hw_job_reset(&ring->sched, &job->base);
+		drm_sched_hw_job_reset(&ring->sched, job ? &job->base : NULL);
 		/* after all hw jobs are reset, hw fence is meaningless, so force_completion */
 		amdgpu_fence_driver_force_completion(ring);
 		dev_info(adev->dev, "GPU reset(%d) succeeded!\n",atomic_read(&adev->gpu_reset_counter));
 	}
+	/*unlock kfd */
+	amdgpu_amdkfd_post_reset(adev);
 	amdgpu_vf_error_trans_all(adev);
 	adev->in_gpu_reset = 0;
 	mutex_unlock(&adev->lock_reset);

drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h+2−0

@@ -105,6 +105,8 @@ struct amdgpu_gmc { · 104 unmodified lines
105	/* protects concurrent invalidation */	105	/* protects concurrent invalidation */
106	spinlock_t invalidate_lock;	106	spinlock_t invalidate_lock;
107	bool translate_further;	107	bool translate_further;
		108	struct kfd_vm_fault_info *vm_fault_info;
		109	atomic_t vm_fault_info_updated;
108		110
109	const struct amdgpu_gmc_funcs *gmc_funcs;	111	const struct amdgpu_gmc_funcs *gmc_funcs;
110	};	112	};

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c+8−5

 	eaddr = eaddr & ((1 << shift) - 1);
 	flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
+	if (vm->root.base.bo->shadow)
+		flags |= AMDGPU_GEM_CREATE_SHADOW;
 	if (vm->use_cpu_for_update)
 		flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
 	else
-		flags |= (AMDGPU_GEM_CREATE_NO_CPU_ACCESS |
+		flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
-				AMDGPU_GEM_CREATE_SHADOW);
 	/* walk over the address space and allocate the page tables */
 	for (pt_idx = from; pt_idx <= to; ++pt_idx) {
 	flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS;
 	if (vm->use_cpu_for_update)
 		flags |= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
-	else
+	else if (vm_context != AMDGPU_VM_CONTEXT_COMPUTE)
 		flags |= AMDGPU_GEM_CREATE_SHADOW;
 	size = amdgpu_vm_bo_size(adev, adev->vm_manager.root_level);
  * - pasid (old PASID is released, because compute manages its own PASIDs)
  *
  * Reinitializes the page directory to reflect the changed ATS
- * setting. May leave behind an unused shadow BO for the page
+ * setting.
- * directory when switching from SDMA updates to CPU updates.
  *
  * Returns:
  * 0 for success, -errno for errors.
 		vm->pasid = 0;
 	}
+	/* Free the shadow bo for compute VM */
+	amdgpu_bo_unref(&vm->root.base.bo->shadow);
 error:
 	amdgpu_bo_unreserve(vm->root.base.bo);
 	return r;

drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c+32−1

 #include "cik.h"
 #include "gmc_v7_0.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_amdkfd.h"
 #include "bif/bif_4_1_d.h"
 #include "bif/bif_4_1_sh_mask.h"
 		adev->vm_manager.vram_base_offset = 0;
 	}
+	adev->gmc.vm_fault_info = kmalloc(sizeof(struct kfd_vm_fault_info),
+					GFP_KERNEL);
+	if (!adev->gmc.vm_fault_info)
+		return -ENOMEM;
+	atomic_set(&adev->gmc.vm_fault_info_updated, 0);
 	return 0;
 }
 	amdgpu_gem_force_release(adev);
 	amdgpu_vm_manager_fini(adev);
+	kfree(adev->gmc.vm_fault_info);
 	gmc_v7_0_gart_fini(adev);
 	amdgpu_bo_fini(adev);
 	release_firmware(adev->gmc.fw);
 				      struct amdgpu_irq_src *source,
 				      struct amdgpu_iv_entry *entry)
 {
-	u32 addr, status, mc_client;
+	u32 addr, status, mc_client, vmid;
 	addr = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_ADDR);
 	status = RREG32(mmVM_CONTEXT1_PROTECTION_FAULT_STATUS);
 					 entry->pasid);
 	}
+	vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+			     VMID);
+	if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid)
+		&& !atomic_read(&adev->gmc.vm_fault_info_updated)) {
+		struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info;
+		u32 protections = REG_GET_FIELD(status,
+					VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+					PROTECTIONS);
+		info->vmid = vmid;
+		info->mc_id = REG_GET_FIELD(status,
+					    VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+					    MEMORY_CLIENT_ID);
+		info->status = status;
+		info->page_addr = addr;
+		info->prot_valid = protections & 0x7 ? true : false;
+		info->prot_read = protections & 0x8 ? true : false;
+		info->prot_write = protections & 0x10 ? true : false;
+		info->prot_exec = protections & 0x20 ? true : false;
+		mb();
+		atomic_set(&adev->gmc.vm_fault_info_updated, 1);
+	}
 	return 0;
 }

drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c+32−1

 #include "amdgpu.h"
 #include "gmc_v8_0.h"
 #include "amdgpu_ucode.h"
+#include "amdgpu_amdkfd.h"
 #include "gmc/gmc_8_1_d.h"
 #include "gmc/gmc_8_1_sh_mask.h"
 		adev->vm_manager.vram_base_offset = 0;
 	}
+	adev->gmc.vm_fault_info = kmalloc(sizeof(struct kfd_vm_fault_info),
+					GFP_KERNEL);
+	if (!adev->gmc.vm_fault_info)
+		return -ENOMEM;
+	atomic_set(&adev->gmc.vm_fault_info_updated, 0);
 	return 0;
 }
 	amdgpu_gem_force_release(adev);
 	amdgpu_vm_manager_fini(adev);
+	kfree(adev->gmc.vm_fault_info);
 	gmc_v8_0_gart_fini(adev);
 	amdgpu_bo_fini(adev);
 	release_firmware(adev->gmc.fw);
 				      struct amdgpu_irq_src *source,
 				      struct amdgpu_iv_entry *entry)
 {
-	u32 addr, status, mc_client;
+	u32 addr, status, mc_client, vmid;
 	if (amdgpu_sriov_vf(adev)) {
 		dev_err(adev->dev, "GPU fault detected: %d 0x%08x\n",
 					 entry->pasid);
 	}
+	vmid = REG_GET_FIELD(status, VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+			     VMID);
+	if (amdgpu_amdkfd_is_kfd_vmid(adev, vmid)
+		&& !atomic_read(&adev->gmc.vm_fault_info_updated)) {
+		struct kfd_vm_fault_info *info = adev->gmc.vm_fault_info;
+		u32 protections = REG_GET_FIELD(status,
+					VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+					PROTECTIONS);
+		info->vmid = vmid;
+		info->mc_id = REG_GET_FIELD(status,
+					    VM_CONTEXT1_PROTECTION_FAULT_STATUS,
+					    MEMORY_CLIENT_ID);
+		info->status = status;
+		info->page_addr = addr;
+		info->prot_valid = protections & 0x7 ? true : false;
+		info->prot_read = protections & 0x8 ? true : false;
+		info->prot_write = protections & 0x10 ? true : false;
+		info->prot_exec = protections & 0x20 ? true : false;
+		mb();
+		atomic_set(&adev->gmc.vm_fault_info_updated, 1);
+	}
 	return 0;
 }

drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c+49−5

@@ -25,12 +25,39 @@ · 24 unmodified lines
25	#include "cik_int.h"	25	#include "cik_int.h"
26		26
27	static bool cik_event_interrupt_isr(struct kfd_dev *dev,	27	static bool cik_event_interrupt_isr(struct kfd_dev *dev,
28	const uint32_t *ih_ring_entry)	28	const uint32_t *ih_ring_entry,
		29	uint32_t *patched_ihre,
		30	bool *patched_flag)
29	{	31	{
30	const struct cik_ih_ring_entry *ihre =	32	const struct cik_ih_ring_entry *ihre =
31	(const struct cik_ih_ring_entry *)ih_ring_entry;	33	(const struct cik_ih_ring_entry *)ih_ring_entry;
		34	const struct kfd2kgd_calls *f2g = dev->kfd2kgd;
32	unsigned int vmid, pasid;	35	unsigned int vmid, pasid;
33		36
		37	/* This workaround is due to HW/FW limitation on Hawaii that
		38	* VMID and PASID are not written into ih_ring_entry
		39	*/
		40	if ((ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT \|\|
		41	ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) &&
		42	dev->device_info->asic_family == CHIP_HAWAII) {
		43	struct cik_ih_ring_entry *tmp_ihre =
		44	(struct cik_ih_ring_entry *)patched_ihre;
		45
		46	*patched_flag = true;
		47	tmp_ihre = ihre;
		48
		49	vmid = f2g->read_vmid_from_vmfault_reg(dev->kgd);
		50	pasid = f2g->get_atc_vmid_pasid_mapping_pasid(dev->kgd, vmid);
		51
		52	tmp_ihre->ring_id &= 0x000000ff;
		53	tmp_ihre->ring_id \|= vmid << 8;
		54	tmp_ihre->ring_id \|= pasid << 16;
		55
		56	return (pasid != 0) &&
		57	vmid >= dev->vm_info.first_vmid_kfd &&
		58	vmid <= dev->vm_info.last_vmid_kfd;
		59	}
		60
34	/* Only handle interrupts from KFD VMIDs */	61	/* Only handle interrupts from KFD VMIDs */
35	vmid = (ihre->ring_id & 0x0000ff00) >> 8;	62	vmid = (ihre->ring_id & 0x0000ff00) >> 8;
36	if (vmid < dev->vm_info.first_vmid_kfd \|\|	63	if (vmid < dev->vm_info.first_vmid_kfd \|\|
@@ -48,18 +75,19 @@ static bool cik_event_interrupt_isr(struct kfd_dev *dev, · 11 unmodified lines
48	return ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE \|\|	75	return ihre->source_id == CIK_INTSRC_CP_END_OF_PIPE \|\|
49	ihre->source_id == CIK_INTSRC_SDMA_TRAP \|\|	76	ihre->source_id == CIK_INTSRC_SDMA_TRAP \|\|
50	ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG \|\|	77	ihre->source_id == CIK_INTSRC_SQ_INTERRUPT_MSG \|\|
51	ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE;	78	ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE \|\|
		79	ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT \|\|
		80	ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT;
52	}	81	}
53		82
54	static void cik_event_interrupt_wq(struct kfd_dev *dev,	83	static void cik_event_interrupt_wq(struct kfd_dev *dev,
55	const uint32_t *ih_ring_entry)	84	const uint32_t *ih_ring_entry)
56	{	85	{
57	unsigned int pasid;
58	const struct cik_ih_ring_entry *ihre =	86	const struct cik_ih_ring_entry *ihre =
59	(const struct cik_ih_ring_entry *)ih_ring_entry;	87	(const struct cik_ih_ring_entry *)ih_ring_entry;
60	uint32_t context_id = ihre->data & 0xfffffff;	88	uint32_t context_id = ihre->data & 0xfffffff;
61		89	unsigned int vmid = (ihre->ring_id & 0x0000ff00) >> 8;
62	pasid = (ihre->ring_id & 0xffff0000) >> 16;	90	unsigned int pasid = (ihre->ring_id & 0xffff0000) >> 16;
63		91
64	if (pasid == 0)	92	if (pasid == 0)
65	return;	93	return;
@@ -72,6 +100,22 @@ static void cik_event_interrupt_wq(struct kfd_dev *dev, · 6 unmodified lines
72	kfd_signal_event_interrupt(pasid, context_id & 0xff, 8);	100	kfd_signal_event_interrupt(pasid, context_id & 0xff, 8);
73	else if (ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE)	101	else if (ihre->source_id == CIK_INTSRC_CP_BAD_OPCODE)
74	kfd_signal_hw_exception_event(pasid);	102	kfd_signal_hw_exception_event(pasid);
		103	else if (ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT \|\|
		104	ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) {
		105	struct kfd_vm_fault_info info;
		106
		107	kfd_process_vm_fault(dev->dqm, pasid);
		108
		109	memset(&info, 0, sizeof(info));
		110	dev->kfd2kgd->get_vm_fault_info(dev->kgd, &info);
		111	if (!info.page_addr && !info.status)
		112	return;
		113
		114	if (info.vmid == vmid)
		115	kfd_signal_vm_fault_event(dev, pasid, &info);
		116	else
		117	kfd_signal_vm_fault_event(dev, pasid, NULL);
		118	}
75	}	119	}
76		120
77	const struct kfd_event_interrupt_class event_interrupt_class_cik = {	121	const struct kfd_event_interrupt_class event_interrupt_class_cik = {

drivers/gpu/drm/amd/amdkfd/cik_int.h+4−3

  * OTHER DEALINGS IN THE SOFTWARE.
  */
-#ifndef HSA_RADEON_CIK_INT_H_INCLUDED
+#ifndef CIK_INT_H_INCLUDED
-#define HSA_RADEON_CIK_INT_H_INCLUDED
+#define CIK_INT_H_INCLUDED
 #include <linux/types.h>
 #define CIK_INTSRC_CP_END_OF_PIPE	0xB5
 #define CIK_INTSRC_CP_BAD_OPCODE	0xB7
-#define CIK_INTSRC_DEQUEUE_COMPLETE	0xC6
 #define CIK_INTSRC_SDMA_TRAP		0xE0
 #define CIK_INTSRC_SQ_INTERRUPT_MSG	0xEF
+#define CIK_INTSRC_GFX_PAGE_INV_FAULT	0x92
+#define CIK_INTSRC_GFX_MEM_PROT_FAULT	0x93
 #endif

drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler.h+233−225

@@ -21,18 +21,21 @@ · 20 unmodified lines
21	*/	21	*/
22		22
23	static const uint32_t cwsr_trap_gfx8_hex[] = {	23	static const uint32_t cwsr_trap_gfx8_hex[] = {
24	0xbf820001, 0xbf820125,	24	0xbf820001, 0xbf82012b,
25	0xb8f4f802, 0x89748674,	25	0xb8f4f802, 0x89748674,
26	0xb8f5f803, 0x8675ff75,	26	0xb8f5f803, 0x8675ff75,
27	0x00000400, 0xbf850011,	27	0x00000400, 0xbf850017,
28	0xc00a1e37, 0x00000000,	28	0xc00a1e37, 0x00000000,
29	0xbf8c007f, 0x87777978,	29	0xbf8c007f, 0x87777978,
30	0xbf840002, 0xb974f802,	30	0xbf840005, 0x8f728374,
31	0xbe801d78, 0xb8f5f803,	31	0xb972e0c2, 0xbf800002,
32	0x8675ff75, 0x000001ff,	32	0xb9740002, 0xbe801d78,
33	0xbf850002, 0x80708470,	33	0xb8f5f803, 0x8675ff75,
34	0x82718071, 0x8671ff71,	34	0x000001ff, 0xbf850002,
35	0x0000ffff, 0xb974f802,	35	0x80708470, 0x82718071,
		36	0x8671ff71, 0x0000ffff,
		37	0x8f728374, 0xb972e0c2,
		38	0xbf800002, 0xb9740002,
36	0xbe801f70, 0xb8f5f803,	39	0xbe801f70, 0xb8f5f803,
37	0x8675ff75, 0x00000100,	40	0x8675ff75, 0x00000100,
38	0xbf840006, 0xbefa0080,	41	0xbf840006, 0xbefa0080,
@@ -168,7 +171,7 @@ static const uint32_t cwsr_trap_gfx8_hex[] = { · 129 unmodified lines
168	0x807c847c, 0x806eff6e,	171	0x807c847c, 0x806eff6e,
169	0x00000400, 0xbf0a757c,	172	0x00000400, 0xbf0a757c,
170	0xbf85ffef, 0xbf9c0000,	173	0xbf85ffef, 0xbf9c0000,
171	0xbf8200ca, 0xbef8007e,	174	0xbf8200cd, 0xbef8007e,
172	0x8679ff7f, 0x0000ffff,	175	0x8679ff7f, 0x0000ffff,
173	0x8779ff79, 0x00040000,	176	0x8779ff79, 0x00040000,
174	0xbefa0080, 0xbefb00ff,	177	0xbefa0080, 0xbefb00ff,
@@ -268,16 +271,18 @@ static const uint32_t cwsr_trap_gfx8_hex[] = { · 93 unmodified lines
268	0x8f739773, 0xb976f807,	271	0x8f739773, 0xb976f807,
269	0x8671ff71, 0x0000ffff,	272	0x8671ff71, 0x0000ffff,
270	0x86fe7e7e, 0x86ea6a6a,	273	0x86fe7e7e, 0x86ea6a6a,
271	0xb974f802, 0xbf8a0000,	274	0x8f768374, 0xb976e0c2,
272	0x95807370, 0xbf810000,	275	0xbf800002, 0xb9740002,
		276	0xbf8a0000, 0x95807370,
		277	0xbf810000, 0x00000000,
273	};	278	};
274		279
275		280
276	static const uint32_t cwsr_trap_gfx9_hex[] = {	281	static const uint32_t cwsr_trap_gfx9_hex[] = {
277	0xbf820001, 0xbf82015a,	282	0xbf820001, 0xbf82015d,
278	0xb8f8f802, 0x89788678,	283	0xb8f8f802, 0x89788678,
279	0xb8f1f803, 0x866eff71,	284	0xb8f1f803, 0x866eff71,
280	0x00000400, 0xbf850034,	285	0x00000400, 0xbf850037,
281	0x866eff71, 0x00000800,	286	0x866eff71, 0x00000800,
282	0xbf850003, 0x866eff71,	287	0xbf850003, 0x866eff71,
283	0x00000100, 0xbf840008,	288	0x00000100, 0xbf840008,
@@ -303,258 +308,261 @@ static const uint32_t cwsr_trap_gfx9_hex[] = { · 19 unmodified lines
303	0x8f6e8b77, 0x866eff6e,	308	0x8f6e8b77, 0x866eff6e,
304	0x001f8000, 0xb96ef807,	309	0x001f8000, 0xb96ef807,
305	0x86fe7e7e, 0x86ea6a6a,	310	0x86fe7e7e, 0x86ea6a6a,
306	0xb978f802, 0xbe801f6c,	311	0x8f6e8378, 0xb96ee0c2,
307	0x866dff6d, 0x0000ffff,	312	0xbf800002, 0xb9780002,
308	0xbef00080, 0xb9700283,	313	0xbe801f6c, 0x866dff6d,
309	0xb8f02407, 0x8e709c70,	314	0x0000ffff, 0xbef00080,
310	0x876d706d, 0xb8f003c7,	315	0xb9700283, 0xb8f02407,
311	0x8e709b70, 0x876d706d,	316	0x8e709c70, 0x876d706d,
312	0xb8f0f807, 0x8670ff70,	317	0xb8f003c7, 0x8e709b70,
313	0x00007fff, 0xb970f807,	318	0x876d706d, 0xb8f0f807,
314	0xbeee007e, 0xbeef007f,	319	0x8670ff70, 0x00007fff,
315	0xbefe0180, 0xbf900004,	320	0xb970f807, 0xbeee007e,
316	0x87708478, 0xb970f802,	321	0xbeef007f, 0xbefe0180,
317	0xbf8e0002, 0xbf88fffe,	322	0xbf900004, 0x87708478,
318	0xb8f02a05, 0x80708170,	323	0xb970f802, 0xbf8e0002,
319	0x8e708a70, 0xb8f11605,	324	0xbf88fffe, 0xb8f02a05,
320	0x80718171, 0x8e718671,	325	0x80708170, 0x8e708a70,
321	0x80707170, 0x80707e70,	326	0xb8f11605, 0x80718171,
322	0x8271807f, 0x8671ff71,	327	0x8e718671, 0x80707170,
323	0x0000ffff, 0xc0471cb8,	328	0x80707e70, 0x8271807f,
324	0x00000040, 0xbf8cc07f,	329	0x8671ff71, 0x0000ffff,
325	0xc04b1d38, 0x00000048,	330	0xc0471cb8, 0x00000040,
326	0xbf8cc07f, 0xc0431e78,	331	0xbf8cc07f, 0xc04b1d38,
327	0x00000058, 0xbf8cc07f,	332	0x00000048, 0xbf8cc07f,
328	0xc0471eb8, 0x0000005c,	333	0xc0431e78, 0x00000058,
329	0xbf8cc07f, 0xbef4007e,	334	0xbf8cc07f, 0xc0471eb8,
330	0x8675ff7f, 0x0000ffff,	335	0x0000005c, 0xbf8cc07f,
331	0x8775ff75, 0x00040000,	336	0xbef4007e, 0x8675ff7f,
332	0xbef60080, 0xbef700ff,	337	0x0000ffff, 0x8775ff75,
333	0x00807fac, 0x8670ff7f,	338	0x00040000, 0xbef60080,
334	0x08000000, 0x8f708370,	339	0xbef700ff, 0x00807fac,
335	0x87777077, 0x8670ff7f,	340	0x8670ff7f, 0x08000000,
336	0x70000000, 0x8f708170,	341	0x8f708370, 0x87777077,
337	0x87777077, 0xbefb007c,	342	0x8670ff7f, 0x70000000,
338	0xbefa0080, 0xb8fa2a05,	343	0x8f708170, 0x87777077,
339	0x807a817a, 0x8e7a8a7a,	344	0xbefb007c, 0xbefa0080,
340	0xb8f01605, 0x80708170,	345	0xb8fa2a05, 0x807a817a,
341	0x8e708670, 0x807a707a,	346	0x8e7a8a7a, 0xb8f01605,
342	0xbef60084, 0xbef600ff,	347	0x80708170, 0x8e708670,
343	0x01000000, 0xbefe007c,	348	0x807a707a, 0xbef60084,
344	0xbefc007a, 0xc0611efa,	349	0xbef600ff, 0x01000000,
345	0x0000007c, 0xbf8cc07f,
346	0x807a847a, 0xbefc007e,
347	0xbefe007c, 0xbefc007a,	350	0xbefe007c, 0xbefc007a,
348	0xc0611b3a, 0x0000007c,	351	0xc0611efa, 0x0000007c,
349	0xbf8cc07f, 0x807a847a,	352	0xbf8cc07f, 0x807a847a,
350	0xbefc007e, 0xbefe007c,	353	0xbefc007e, 0xbefe007c,
351	0xbefc007a, 0xc0611b7a,	354	0xbefc007a, 0xc0611b3a,
352	0x0000007c, 0xbf8cc07f,	355	0x0000007c, 0xbf8cc07f,
353	0x807a847a, 0xbefc007e,	356	0x807a847a, 0xbefc007e,
354	0xbefe007c, 0xbefc007a,	357	0xbefe007c, 0xbefc007a,
355	0xc0611bba, 0x0000007c,	358	0xc0611b7a, 0x0000007c,
356	0xbf8cc07f, 0x807a847a,	359	0xbf8cc07f, 0x807a847a,
357	0xbefc007e, 0xbefe007c,	360	0xbefc007e, 0xbefe007c,
358	0xbefc007a, 0xc0611bfa,	361	0xbefc007a, 0xc0611bba,
359	0x0000007c, 0xbf8cc07f,	362	0x0000007c, 0xbf8cc07f,
360	0x807a847a, 0xbefc007e,	363	0x807a847a, 0xbefc007e,
361	0xbefe007c, 0xbefc007a,	364	0xbefe007c, 0xbefc007a,
362	0xc0611e3a, 0x0000007c,	365	0xc0611bfa, 0x0000007c,
363	0xbf8cc07f, 0x807a847a,
364	0xbefc007e, 0xb8f1f803,
365	0xbefe007c, 0xbefc007a,
366	0xc0611c7a, 0x0000007c,
367	0xbf8cc07f, 0x807a847a,	366	0xbf8cc07f, 0x807a847a,
368	0xbefc007e, 0xbefe007c,	367	0xbefc007e, 0xbefe007c,
369	0xbefc007a, 0xc0611a3a,	368	0xbefc007a, 0xc0611e3a,
		369	0x0000007c, 0xbf8cc07f,
		370	0x807a847a, 0xbefc007e,
		371	0xb8f1f803, 0xbefe007c,
		372	0xbefc007a, 0xc0611c7a,
370	0x0000007c, 0xbf8cc07f,	373	0x0000007c, 0xbf8cc07f,
371	0x807a847a, 0xbefc007e,	374	0x807a847a, 0xbefc007e,
372	0xbefe007c, 0xbefc007a,	375	0xbefe007c, 0xbefc007a,
373	0xc0611a7a, 0x0000007c,	376	0xc0611a3a, 0x0000007c,
374	0xbf8cc07f, 0x807a847a,
375	0xbefc007e, 0xb8fbf801,
376	0xbefe007c, 0xbefc007a,
377	0xc0611efa, 0x0000007c,
378	0xbf8cc07f, 0x807a847a,	377	0xbf8cc07f, 0x807a847a,
379	0xbefc007e, 0x8670ff7f,	378	0xbefc007e, 0xbefe007c,
380	0x04000000, 0xbeef0080,	379	0xbefc007a, 0xc0611a7a,
381	0x876f6f70, 0xb8fa2a05,	380	0x0000007c, 0xbf8cc07f,
		381	0x807a847a, 0xbefc007e,
		382	0xb8fbf801, 0xbefe007c,
		383	0xbefc007a, 0xc0611efa,
		384	0x0000007c, 0xbf8cc07f,
		385	0x807a847a, 0xbefc007e,
		386	0x8670ff7f, 0x04000000,
		387	0xbeef0080, 0x876f6f70,
		388	0xb8fa2a05, 0x807a817a,
		389	0x8e7a8a7a, 0xb8f11605,
		390	0x80718171, 0x8e718471,
		391	0x8e768271, 0xbef600ff,
		392	0x01000000, 0xbef20174,
		393	0x80747a74, 0x82758075,
		394	0xbefc0080, 0xbf800000,
		395	0xbe802b00, 0xbe822b02,
		396	0xbe842b04, 0xbe862b06,
		397	0xbe882b08, 0xbe8a2b0a,
		398	0xbe8c2b0c, 0xbe8e2b0e,
		399	0xc06b003a, 0x00000000,
		400	0xbf8cc07f, 0xc06b013a,
		401	0x00000010, 0xbf8cc07f,
		402	0xc06b023a, 0x00000020,
		403	0xbf8cc07f, 0xc06b033a,
		404	0x00000030, 0xbf8cc07f,
		405	0x8074c074, 0x82758075,
		406	0x807c907c, 0xbf0a717c,
		407	0xbf85ffe7, 0xbef40172,
		408	0xbefa0080, 0xbefe00c1,
		409	0xbeff00c1, 0xbee80080,
		410	0xbee90080, 0xbef600ff,
		411	0x01000000, 0xe0724000,
		412	0x7a1d0000, 0xe0724100,
		413	0x7a1d0100, 0xe0724200,
		414	0x7a1d0200, 0xe0724300,
		415	0x7a1d0300, 0xbefe00c1,
		416	0xbeff00c1, 0xb8f14306,
		417	0x8671c171, 0xbf84002c,
		418	0xbf8a0000, 0x8670ff6f,
		419	0x04000000, 0xbf840028,
		420	0x8e718671, 0x8e718271,
		421	0xbef60071, 0xb8fa2a05,
382	0x807a817a, 0x8e7a8a7a,	422	0x807a817a, 0x8e7a8a7a,
383	0xb8f11605, 0x80718171,	423	0xb8f01605, 0x80708170,
384	0x8e718471, 0x8e768271,	424	0x8e708670, 0x807a707a,
		425	0x807aff7a, 0x00000080,
385	0xbef600ff, 0x01000000,	426	0xbef600ff, 0x01000000,
386	0xbef20174, 0x80747a74,	427	0xbefc0080, 0xd28c0002,
387	0x82758075, 0xbefc0080,	428	0x000100c1, 0xd28d0003,
388	0xbf800000, 0xbe802b00,	429	0x000204c1, 0xd1060002,
389	0xbe822b02, 0xbe842b04,	430	0x00011103, 0x7e0602ff,
390	0xbe862b06, 0xbe882b08,	431	0x00000200, 0xbefc00ff,
391	0xbe8a2b0a, 0xbe8c2b0c,	432	0x00010000, 0xbe800077,
392	0xbe8e2b0e, 0xc06b003a,	433	0x8677ff77, 0xff7fffff,
393	0x00000000, 0xbf8cc07f,	434	0x8777ff77, 0x00058000,
394	0xc06b013a, 0x00000010,	435	0xd8ec0000, 0x00000002,
395	0xbf8cc07f, 0xc06b023a,	436	0xbf8cc07f, 0xe0765000,
396	0x00000020, 0xbf8cc07f,	437	0x7a1d0002, 0x68040702,
397	0xc06b033a, 0x00000030,	438	0xd0c9006a, 0x0000e302,
398	0xbf8cc07f, 0x8074c074,	439	0xbf87fff7, 0xbef70000,
399	0x82758075, 0x807c907c,	440	0xbefa00ff, 0x00000400,
400	0xbf0a717c, 0xbf85ffe7,
401	0xbef40172, 0xbefa0080,
402	0xbefe00c1, 0xbeff00c1,	441	0xbefe00c1, 0xbeff00c1,
403	0xbee80080, 0xbee90080,	442	0xb8f12a05, 0x80718171,
		443	0x8e718271, 0x8e768871,
404	0xbef600ff, 0x01000000,	444	0xbef600ff, 0x01000000,
		445	0xbefc0084, 0xbf0a717c,
		446	0xbf840015, 0xbf11017c,
		447	0x8071ff71, 0x00001000,
		448	0x7e000300, 0x7e020301,
		449	0x7e040302, 0x7e060303,
405	0xe0724000, 0x7a1d0000,	450	0xe0724000, 0x7a1d0000,
406	0xe0724100, 0x7a1d0100,	451	0xe0724100, 0x7a1d0100,
407	0xe0724200, 0x7a1d0200,	452	0xe0724200, 0x7a1d0200,
408	0xe0724300, 0x7a1d0300,	453	0xe0724300, 0x7a1d0300,
		454	0x807c847c, 0x807aff7a,
		455	0x00000400, 0xbf0a717c,
		456	0xbf85ffef, 0xbf9c0000,
		457	0xbf8200dc, 0xbef4007e,
		458	0x8675ff7f, 0x0000ffff,
		459	0x8775ff75, 0x00040000,
		460	0xbef60080, 0xbef700ff,
		461	0x00807fac, 0x866eff7f,
		462	0x08000000, 0x8f6e836e,
		463	0x87776e77, 0x866eff7f,
		464	0x70000000, 0x8f6e816e,
		465	0x87776e77, 0x866eff7f,
		466	0x04000000, 0xbf84001e,
409	0xbefe00c1, 0xbeff00c1,	467	0xbefe00c1, 0xbeff00c1,
410	0xb8f14306, 0x8671c171,	468	0xb8ef4306, 0x866fc16f,
411	0xbf84002c, 0xbf8a0000,	469	0xbf840019, 0x8e6f866f,
412	0x8670ff6f, 0x04000000,	470	0x8e6f826f, 0xbef6006f,
413	0xbf840028, 0x8e718671,	471	0xb8f82a05, 0x80788178,
414	0x8e718271, 0xbef60071,	472	0x8e788a78, 0xb8ee1605,
415	0xb8fa2a05, 0x807a817a,	473	0x806e816e, 0x8e6e866e,
416	0x8e7a8a7a, 0xb8f01605,	474	0x80786e78, 0x8078ff78,
417	0x80708170, 0x8e708670,
418	0x807a707a, 0x807aff7a,
419	0x00000080, 0xbef600ff,	475	0x00000080, 0xbef600ff,
420	0x01000000, 0xbefc0080,	476	0x01000000, 0xbefc0080,
421	0xd28c0002, 0x000100c1,	477	0xe0510000, 0x781d0000,
422	0xd28d0003, 0x000204c1,	478	0xe0510100, 0x781d0000,
423	0xd1060002, 0x00011103,	479	0x807cff7c, 0x00000200,
424	0x7e0602ff, 0x00000200,	480	0x8078ff78, 0x00000200,
425	0xbefc00ff, 0x00010000,	481	0xbf0a6f7c, 0xbf85fff6,
426	0xbe800077, 0x8677ff77,	482	0xbef80080, 0xbefe00c1,
427	0xff7fffff, 0x8777ff77,	483	0xbeff00c1, 0xb8ef2a05,
428	0x00058000, 0xd8ec0000,	484	0x806f816f, 0x8e6f826f,
429	0x00000002, 0xbf8cc07f,	485	0x8e76886f, 0xbef600ff,
430	0xe0765000, 0x7a1d0002,	486	0x01000000, 0xbeee0078,
431	0x68040702, 0xd0c9006a,	487	0x8078ff78, 0x00000400,
432	0x0000e302, 0xbf87fff7,	488	0xbefc0084, 0xbf11087c,
433	0xbef70000, 0xbefa00ff,	489	0x806fff6f, 0x00008000,
434	0x00000400, 0xbefe00c1,	490	0xe0524000, 0x781d0000,
435	0xbeff00c1, 0xb8f12a05,	491	0xe0524100, 0x781d0100,
436	0x80718171, 0x8e718271,	492	0xe0524200, 0x781d0200,
437	0x8e768871, 0xbef600ff,	493	0xe0524300, 0x781d0300,
438	0x01000000, 0xbefc0084,	494	0xbf8c0f70, 0x7e000300,
439	0xbf0a717c, 0xbf840015,
440	0xbf11017c, 0x8071ff71,
441	0x00001000, 0x7e000300,
442	0x7e020301, 0x7e040302,	495	0x7e020301, 0x7e040302,
443	0x7e060303, 0xe0724000,	496	0x7e060303, 0x807c847c,
444	0x7a1d0000, 0xe0724100,	497	0x8078ff78, 0x00000400,
445	0x7a1d0100, 0xe0724200,	498	0xbf0a6f7c, 0xbf85ffee,
446	0x7a1d0200, 0xe0724300,	499	0xbf9c0000, 0xe0524000,
447	0x7a1d0300, 0x807c847c,	500	0x6e1d0000, 0xe0524100,
448	0x807aff7a, 0x00000400,	501	0x6e1d0100, 0xe0524200,
449	0xbf0a717c, 0xbf85ffef,	502	0x6e1d0200, 0xe0524300,
450	0xbf9c0000, 0xbf8200d9,	503	0x6e1d0300, 0xb8f82a05,
451	0xbef4007e, 0x8675ff7f,
452	0x0000ffff, 0x8775ff75,
453	0x00040000, 0xbef60080,
454	0xbef700ff, 0x00807fac,
455	0x866eff7f, 0x08000000,
456	0x8f6e836e, 0x87776e77,
457	0x866eff7f, 0x70000000,
458	0x8f6e816e, 0x87776e77,
459	0x866eff7f, 0x04000000,
460	0xbf84001e, 0xbefe00c1,
461	0xbeff00c1, 0xb8ef4306,
462	0x866fc16f, 0xbf840019,
463	0x8e6f866f, 0x8e6f826f,
464	0xbef6006f, 0xb8f82a05,
465	0x80788178, 0x8e788a78,	504	0x80788178, 0x8e788a78,
466	0xb8ee1605, 0x806e816e,	505	0xb8ee1605, 0x806e816e,
467	0x8e6e866e, 0x80786e78,	506	0x8e6e866e, 0x80786e78,
468	0x8078ff78, 0x00000080,	507	0x80f8c078, 0xb8ef1605,
469	0xbef600ff, 0x01000000,	508	0x806f816f, 0x8e6f846f,
470	0xbefc0080, 0xe0510000,	509	0x8e76826f, 0xbef600ff,
471	0x781d0000, 0xe0510100,	510	0x01000000, 0xbefc006f,
472	0x781d0000, 0x807cff7c,	511	0xc031003a, 0x00000078,
473	0x00000200, 0x8078ff78,	512	0x80f8c078, 0xbf8cc07f,
474	0x00000200, 0xbf0a6f7c,	513	0x80fc907c, 0xbf800000,
475	0xbf85fff6, 0xbef80080,	514	0xbe802d00, 0xbe822d02,
476	0xbefe00c1, 0xbeff00c1,	515	0xbe842d04, 0xbe862d06,
477	0xb8ef2a05, 0x806f816f,	516	0xbe882d08, 0xbe8a2d0a,
478	0x8e6f826f, 0x8e76886f,	517	0xbe8c2d0c, 0xbe8e2d0e,
479	0xbef600ff, 0x01000000,	518	0xbf06807c, 0xbf84fff0,
480	0xbeee0078, 0x8078ff78,
481	0x00000400, 0xbefc0084,
482	0xbf11087c, 0x806fff6f,
483	0x00008000, 0xe0524000,
484	0x781d0000, 0xe0524100,
485	0x781d0100, 0xe0524200,
486	0x781d0200, 0xe0524300,
487	0x781d0300, 0xbf8c0f70,
488	0x7e000300, 0x7e020301,
489	0x7e040302, 0x7e060303,
490	0x807c847c, 0x8078ff78,
491	0x00000400, 0xbf0a6f7c,
492	0xbf85ffee, 0xbf9c0000,
493	0xe0524000, 0x6e1d0000,
494	0xe0524100, 0x6e1d0100,
495	0xe0524200, 0x6e1d0200,
496	0xe0524300, 0x6e1d0300,
497	0xb8f82a05, 0x80788178,	519	0xb8f82a05, 0x80788178,
498	0x8e788a78, 0xb8ee1605,	520	0x8e788a78, 0xb8ee1605,
499	0x806e816e, 0x8e6e866e,	521	0x806e816e, 0x8e6e866e,
500	0x80786e78, 0x80f8c078,	522	0x80786e78, 0xbef60084,
501	0xb8ef1605, 0x806f816f,
502	0x8e6f846f, 0x8e76826f,
503	0xbef600ff, 0x01000000,	523	0xbef600ff, 0x01000000,
504	0xbefc006f, 0xc031003a,	524	0xc0211bfa, 0x00000078,
505	0x00000078, 0x80f8c078,	525	0x80788478, 0xc0211b3a,
506	0xbf8cc07f, 0x80fc907c,
507	0xbf800000, 0xbe802d00,
508	0xbe822d02, 0xbe842d04,
509	0xbe862d06, 0xbe882d08,
510	0xbe8a2d0a, 0xbe8c2d0c,
511	0xbe8e2d0e, 0xbf06807c,
512	0xbf84fff0, 0xb8f82a05,
513	0x80788178, 0x8e788a78,
514	0xb8ee1605, 0x806e816e,
515	0x8e6e866e, 0x80786e78,
516	0xbef60084, 0xbef600ff,
517	0x01000000, 0xc0211bfa,
518	0x00000078, 0x80788478,	526	0x00000078, 0x80788478,
519	0xc0211b3a, 0x00000078,	527	0xc0211b7a, 0x00000078,
520	0x80788478, 0xc0211b7a,	528	0x80788478, 0xc0211eba,
521	0x00000078, 0x80788478,	529	0x00000078, 0x80788478,
522	0xc0211eba, 0x00000078,	530	0xc0211efa, 0x00000078,
523	0x80788478, 0xc0211efa,	531	0x80788478, 0xc0211c3a,
524	0x00000078, 0x80788478,	532	0x00000078, 0x80788478,
525	0xc0211c3a, 0x00000078,	533	0xc0211c7a, 0x00000078,
526	0x80788478, 0xc0211c7a,	534	0x80788478, 0xc0211a3a,
527	0x00000078, 0x80788478,	535	0x00000078, 0x80788478,
528	0xc0211a3a, 0x00000078,	536	0xc0211a7a, 0x00000078,
529	0x80788478, 0xc0211a7a,	537	0x80788478, 0xc0211cfa,
530	0x00000078, 0x80788478,	538	0x00000078, 0x80788478,
531	0xc0211cfa, 0x00000078,	539	0xbf8cc07f, 0xbefc006f,
532	0x80788478, 0xbf8cc07f,	540	0xbefe007a, 0xbeff007b,
533	0xbefc006f, 0xbefe007a,	541	0x866f71ff, 0x000003ff,
534	0xbeff007b, 0x866f71ff,	542	0xb96f4803, 0x866f71ff,
535	0x000003ff, 0xb96f4803,	543	0xfffff800, 0x8f6f8b6f,
536	0x866f71ff, 0xfffff800,	544	0xb96fa2c3, 0xb973f801,
537	0x8f6f8b6f, 0xb96fa2c3,	545	0xb8ee2a05, 0x806e816e,
538	0xb973f801, 0xb8ee2a05,	546	0x8e6e8a6e, 0xb8ef1605,
539	0x806e816e, 0x8e6e8a6e,	547	0x806f816f, 0x8e6f866f,
540	0xb8ef1605, 0x806f816f,	548	0x806e6f6e, 0x806e746e,
541	0x8e6f866f, 0x806e6f6e,	549	0x826f8075, 0x866fff6f,
542	0x806e746e, 0x826f8075,	550	0x0000ffff, 0xc0071cb7,
543	0x866fff6f, 0x0000ffff,	551	0x00000040, 0xc00b1d37,
544	0xc0071cb7, 0x00000040,	552	0x00000048, 0xc0031e77,
545	0xc00b1d37, 0x00000048,	553	0x00000058, 0xc0071eb7,
546	0xc0031e77, 0x00000058,	554	0x0000005c, 0xbf8cc07f,
547	0xc0071eb7, 0x0000005c,	555	0x866fff6d, 0xf0000000,
548	0xbf8cc07f, 0x866fff6d,	556	0x8f6f9c6f, 0x8e6f906f,
549	0xf0000000, 0x8f6f9c6f,	557	0xbeee0080, 0x876e6f6e,
550	0x8e6f906f, 0xbeee0080,	558	0x866fff6d, 0x08000000,
551	0x876e6f6e, 0x866fff6d,	559	0x8f6f9b6f, 0x8e6f8f6f,
552	0x08000000, 0x8f6f9b6f,	560	0x876e6f6e, 0x866fff70,
553	0x8e6f8f6f, 0x876e6f6e,	561	0x00800000, 0x8f6f976f,
554	0x866fff70, 0x00800000,	562	0xb96ef807, 0x866dff6d,
555	0x8f6f976f, 0xb96ef807,	563	0x0000ffff, 0x86fe7e7e,
556	0x866dff6d, 0x0000ffff,	564	0x86ea6a6a, 0x8f6e8370,
557	0x86fe7e7e, 0x86ea6a6a,	565	0xb96ee0c2, 0xbf800002,
558	0xb970f802, 0xbf8a0000,	566	0xb9700002, 0xbf8a0000,
559	0x95806f6c, 0xbf810000,	567	0x95806f6c, 0xbf810000,
560	};	568	};

drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx8.asm+15−3

 var SQ_WAVE_STATUS_INST_ATC_MASK   = 0x00800000
 var SQ_WAVE_STATUS_SPI_PRIO_SHIFT  = 1
 var SQ_WAVE_STATUS_SPI_PRIO_MASK   = 0x00000006
+var SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT   = 0
+var SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE    = 1
+var SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT  = 3
+var SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE   = 29
 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT    = 12
 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE     = 9
     s_waitcnt lgkmcnt(0)
     s_or_b32        ttmp7, ttmp8, ttmp9
     s_cbranch_scc0  L_NO_NEXT_TRAP //next level trap handler not been set
-    s_setreg_b32    hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)
+    set_status_without_spi_prio(s_save_status, ttmp2) //restore HW status(SCC)
     s_setpc_b64     [ttmp8,ttmp9] //jump to next level trap handler
 L_NO_NEXT_TRAP:
     s_addc_u32  ttmp1, ttmp1, 0
 L_EXCP_CASE:
     s_and_b32   ttmp1, ttmp1, 0xFFFF
-    s_setreg_b32    hwreg(HW_REG_STATUS), s_save_status //restore HW status(SCC)
+    set_status_without_spi_prio(s_save_status, ttmp2) //restore HW status(SCC)
     s_rfe_b64       [ttmp0, ttmp1]
 end
     // *********        End handling of non-CWSR traps   *******************
     s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff      //pc[47:32]        //Do it here in order not to affect STATUS
     s_and_b64    exec, exec, exec  // Restore STATUS.EXECZ, not writable by s_setreg_b32
     s_and_b64    vcc, vcc, vcc  // Restore STATUS.VCCZ, not writable by s_setreg_b32
-    s_setreg_b32    hwreg(HW_REG_STATUS),   s_restore_status     // SCC is included, which is changed by previous salu
+    set_status_without_spi_prio(s_restore_status, s_restore_tmp) // SCC is included, which is changed by previous salu
     s_barrier                                                   //barrier to ensure the readiness of LDS before access attempts from any other wave in the same TG //FIXME not performance-optimal at this time
 function get_hwreg_size_bytes
     return 128 //HWREG size 128 bytes
 end
+function set_status_without_spi_prio(status, tmp)
+    // Do not restore STATUS.SPI_PRIO since scheduler may have raised it.
+    s_lshr_b32      tmp, status, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT
+    s_setreg_b32    hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE), tmp
+    s_nop           0x2 // avoid S_SETREG => S_SETREG hazard
+    s_setreg_b32    hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE), status
+end

drivers/gpu/drm/amd/amdkfd/cwsr_trap_handler_gfx9.asm+14−2

 var SQ_WAVE_STATUS_SPI_PRIO_SHIFT  = 1
 var SQ_WAVE_STATUS_SPI_PRIO_MASK   = 0x00000006
 var SQ_WAVE_STATUS_HALT_MASK       = 0x2000
+var SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT   = 0
+var SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE    = 1
+var SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT  = 3
+var SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE   = 29
 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SHIFT	= 12
 var SQ_WAVE_LDS_ALLOC_LDS_SIZE_SIZE	= 9
     // Restore SQ_WAVE_STATUS.
     s_and_b64       exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32
     s_and_b64       vcc, vcc, vcc    // Restore STATUS.VCCZ, not writable by s_setreg_b32
-    s_setreg_b32    hwreg(HW_REG_STATUS), s_save_status
+    set_status_without_spi_prio(s_save_status, ttmp2)
     s_rfe_b64       [ttmp0, ttmp1]
 end
     s_and_b32 s_restore_pc_hi, s_restore_pc_hi, 0x0000ffff	//pc[47:32]	   //Do it here in order not to affect STATUS
     s_and_b64	 exec, exec, exec  // Restore STATUS.EXECZ, not writable by s_setreg_b32
     s_and_b64	 vcc, vcc, vcc	// Restore STATUS.VCCZ, not writable by s_setreg_b32
-    s_setreg_b32    hwreg(HW_REG_STATUS),   s_restore_status	 // SCC is included, which is changed by previous salu
+    set_status_without_spi_prio(s_restore_status, s_restore_tmp) // SCC is included, which is changed by previous salu
     s_barrier							//barrier to ensure the readiness of LDS before access attempts from any other wave in the same TG //FIXME not performance-optimal at this time
         s_waitcnt lgkmcnt(0)
     end
 end
+function set_status_without_spi_prio(status, tmp)
+    // Do not restore STATUS.SPI_PRIO since scheduler may have raised it.
+    s_lshr_b32      tmp, status, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT
+    s_setreg_b32    hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_POST_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_POST_SPI_PRIO_SIZE), tmp
+    s_nop           0x2 // avoid S_SETREG => S_SETREG hazard
+    s_setreg_b32    hwreg(HW_REG_STATUS, SQ_WAVE_STATUS_PRE_SPI_PRIO_SHIFT, SQ_WAVE_STATUS_PRE_SPI_PRIO_SIZE), status
+end

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c+63−6

@@ -122,6 +122,9 @@ static int kfd_open(struct inode inode, struct file filep) · 121 unmodified lines
122	if (IS_ERR(process))	122	if (IS_ERR(process))
123	return PTR_ERR(process);	123	return PTR_ERR(process);
124		124
		125	if (kfd_is_locked())
		126	return -EAGAIN;
		127
125	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",	128	dev_dbg(kfd_device, "process %d opened, compat mode (32 bit) - %d\n",
126	process->pasid, process->is_32bit_user_mode);	129	process->pasid, process->is_32bit_user_mode);
127		130
@@ -389,6 +392,61 @@ static int kfd_ioctl_update_queue(struct file filp, struct kfd_process p, · 261 unmodified lines
389	return retval;	392	return retval;
390	}	393	}
391		394
		395	static int kfd_ioctl_set_cu_mask(struct file filp, struct kfd_process p,
		396	void *data)
		397	{
		398	int retval;
		399	const int max_num_cus = 1024;
		400	struct kfd_ioctl_set_cu_mask_args *args = data;
		401	struct queue_properties properties;
		402	uint32_t __user cu_mask_ptr = (uint32_t __user )args->cu_mask_ptr;
		403	size_t cu_mask_size = sizeof(uint32_t) * (args->num_cu_mask / 32);
		404
		405	if ((args->num_cu_mask % 32) != 0) {
		406	pr_debug("num_cu_mask 0x%x must be a multiple of 32",
		407	args->num_cu_mask);
		408	return -EINVAL;
		409	}
		410
		411	properties.cu_mask_count = args->num_cu_mask;
		412	if (properties.cu_mask_count == 0) {
		413	pr_debug("CU mask cannot be 0");
		414	return -EINVAL;
		415	}
		416
		417	/* To prevent an unreasonably large CU mask size, set an arbitrary
		418	* limit of max_num_cus bits. We can then just drop any CU mask bits
		419	* past max_num_cus bits and just use the first max_num_cus bits.
		420	*/
		421	if (properties.cu_mask_count > max_num_cus) {
		422	pr_debug("CU mask cannot be greater than 1024 bits");
		423	properties.cu_mask_count = max_num_cus;
		424	cu_mask_size = sizeof(uint32_t) * (max_num_cus/32);
		425	}
		426
		427	properties.cu_mask = kzalloc(cu_mask_size, GFP_KERNEL);
		428	if (!properties.cu_mask)
		429	return -ENOMEM;
		430
		431	retval = copy_from_user(properties.cu_mask, cu_mask_ptr, cu_mask_size);
		432	if (retval) {
		433	pr_debug("Could not copy CU mask from userspace");
		434	kfree(properties.cu_mask);
		435	return -EFAULT;
		436	}
		437
		438	mutex_lock(&p->mutex);
		439
		440	retval = pqm_set_cu_mask(&p->pqm, args->queue_id, &properties);
		441
		442	mutex_unlock(&p->mutex);
		443
		444	if (retval)
		445	kfree(properties.cu_mask);
		446
		447	return retval;
		448	}
		449
392	static int kfd_ioctl_set_memory_policy(struct file *filep,	450	static int kfd_ioctl_set_memory_policy(struct file *filep,
393	struct kfd_process p, void data)	451	struct kfd_process p, void data)
394	{	452	{
@@ -754,7 +812,6 @@ static int kfd_ioctl_get_clock_counters(struct file *filep, · 359 unmodified lines
754	{	812	{
755	struct kfd_ioctl_get_clock_counters_args *args = data;	813	struct kfd_ioctl_get_clock_counters_args *args = data;
756	struct kfd_dev *dev;	814	struct kfd_dev *dev;
757	struct timespec64 time;
758		815
759	dev = kfd_device_by_id(args->gpu_id);	816	dev = kfd_device_by_id(args->gpu_id);
760	if (dev)	817	if (dev)
@@ -766,11 +823,8 @@ static int kfd_ioctl_get_clock_counters(struct file *filep, · 5 unmodified lines
766	args->gpu_clock_counter = 0;	823	args->gpu_clock_counter = 0;
767		824
768	/* No access to rdtsc. Using raw monotonic time */	825	/* No access to rdtsc. Using raw monotonic time */
769	getrawmonotonic64(&time);	826	args->cpu_clock_counter = ktime_get_raw_ns();
770	args->cpu_clock_counter = (uint64_t)timespec64_to_ns(&time);	827	args->system_clock_counter = ktime_get_boot_ns();
771
772	get_monotonic_boottime64(&time);
773	args->system_clock_counter = (uint64_t)timespec64_to_ns(&time);
774		828
775	/* Since the counter is in nano-seconds we use 1GHz frequency */	829	/* Since the counter is in nano-seconds we use 1GHz frequency */
776	args->system_clock_freq = 1000000000;	830	args->system_clock_freq = 1000000000;
@@ -1558,6 +1612,9 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { · 781 unmodified lines
1558	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,	1612	AMDKFD_IOCTL_DEF(AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU,
1559	kfd_ioctl_unmap_memory_from_gpu, 0),	1613	kfd_ioctl_unmap_memory_from_gpu, 0),
1560		1614
		1615	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_CU_MASK,
		1616	kfd_ioctl_set_cu_mask, 0),
		1617
1561	};	1618	};
1562		1619
1563	#define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)	1620	#define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)

drivers/gpu/drm/amd/amdkfd/kfd_crat.c+46−11

 	return 0;
 }
+static struct kfd_mem_properties *
+find_subtype_mem(uint32_t heap_type, uint32_t flags, uint32_t width,
+		struct kfd_topology_device *dev)
+{
+	struct kfd_mem_properties *props;
+	list_for_each_entry(props, &dev->mem_props, list) {
+		if (props->heap_type == heap_type
+				&& props->flags == flags
+				&& props->width == width)
+			return props;
+	}
+	return NULL;
+}
 /* kfd_parse_subtype_mem - parse memory subtypes and attach it to correct
  * topology device present in the device_list
  */
 {
 	struct kfd_mem_properties *props;
 	struct kfd_topology_device *dev;
+	uint32_t heap_type;
+	uint64_t size_in_bytes;
+	uint32_t flags = 0;
+	uint32_t width;
 	pr_debug("Found memory entry in CRAT table with proximity_domain=%d\n",
 			mem->proximity_domain);
 	list_for_each_entry(dev, device_list, list) {
 		if (mem->proximity_domain == dev->proximity_domain) {
-			props = kfd_alloc_struct(props);
-			if (!props)
-				return -ENOMEM;
 			/* We're on GPU node */
 			if (dev->node_props.cpu_cores_count == 0) {
 				/* APU */
 				if (mem->visibility_type == 0)
-					props->heap_type =
+					heap_type =
 						HSA_MEM_HEAP_TYPE_FB_PRIVATE;
 				/* dGPU */
 				else
-					props->heap_type = mem->visibility_type;
+					heap_type = mem->visibility_type;
 			} else
-				props->heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
+				heap_type = HSA_MEM_HEAP_TYPE_SYSTEM;
 			if (mem->flags & CRAT_MEM_FLAGS_HOT_PLUGGABLE)
-				props->flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
+				flags |= HSA_MEM_FLAGS_HOT_PLUGGABLE;
 			if (mem->flags & CRAT_MEM_FLAGS_NON_VOLATILE)
-				props->flags |= HSA_MEM_FLAGS_NON_VOLATILE;
+				flags |= HSA_MEM_FLAGS_NON_VOLATILE;
-			props->size_in_bytes =
+			size_in_bytes =
 				((uint64_t)mem->length_high << 32) +
 							mem->length_low;
-			props->width = mem->width;
+			width = mem->width;
+			/* Multiple banks of the same type are aggregated into
+			 * one. User mode doesn't care about multiple physical
+			 * memory segments. It's managed as a single virtual
+			 * heap for user mode.
+			 */
+			props = find_subtype_mem(heap_type, flags, width, dev);
+			if (props) {
+				props->size_in_bytes += size_in_bytes;
+				break;
+			}
+			props = kfd_alloc_struct(props);
+			if (!props)
+				return -ENOMEM;
+			props->heap_type = heap_type;
+			props->flags = flags;
+			props->size_in_bytes = size_in_bytes;
+			props->width = width;
 			dev->node_props.mem_banks_count++;
 			list_add_tail(&props->list, &dev->mem_props);

drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.c+0−1

@@ -38,7 +38,6 @@ · 37 unmodified lines
38	#include "kfd_dbgmgr.h"	38	#include "kfd_dbgmgr.h"
39	#include "kfd_dbgdev.h"	39	#include "kfd_dbgdev.h"
40	#include "kfd_device_queue_manager.h"	40	#include "kfd_device_queue_manager.h"
41	#include "../../radeon/cik_reg.h"
42		41
43	static void dbgdev_address_watch_disable_nodiq(struct kfd_dev *dev)	42	static void dbgdev_address_watch_disable_nodiq(struct kfd_dev *dev)
44	{	43	{

drivers/gpu/drm/amd/amdkfd/kfd_dbgdev.h+37−0

@@ -60,6 +60,9 @@ enum { · 59 unmodified lines
60	SH_REG_SIZE = SH_REG_END - SH_REG_BASE	60	SH_REG_SIZE = SH_REG_END - SH_REG_BASE
61	};	61	};
62		62
		63	/* SQ_CMD definitions */
		64	#define SQ_CMD 0x8DEC
		65
63	enum SQ_IND_CMD_CMD {	66	enum SQ_IND_CMD_CMD {
64	SQ_IND_CMD_CMD_NULL = 0x00000000,	67	SQ_IND_CMD_CMD_NULL = 0x00000000,
65	SQ_IND_CMD_CMD_HALT = 0x00000001,	68	SQ_IND_CMD_CMD_HALT = 0x00000001,
@@ -190,4 +193,38 @@ union ULARGE_INTEGER { · 124 unmodified lines
190	void kfd_dbgdev_init(struct kfd_dbgdev pdbgdev, struct kfd_dev pdev,	193	void kfd_dbgdev_init(struct kfd_dbgdev pdbgdev, struct kfd_dev pdev,
191	enum DBGDEV_TYPE type);	194	enum DBGDEV_TYPE type);
192		195
		196	union TCP_WATCH_CNTL_BITS {
		197	struct {
		198	uint32_t mask:24;
		199	uint32_t vmid:4;
		200	uint32_t atc:1;
		201	uint32_t mode:2;
		202	uint32_t valid:1;
		203	} bitfields, bits;
		204	uint32_t u32All;
		205	signed int i32All;
		206	float f32All;
		207	};
		208
		209	enum {
		210	ADDRESS_WATCH_REG_CNTL_ATC_BIT = 0x10000000UL,
		211	ADDRESS_WATCH_REG_CNTL_DEFAULT_MASK = 0x00FFFFFF,
		212	ADDRESS_WATCH_REG_ADDLOW_MASK_EXTENSION = 0x03000000,
		213	/* extend the mask to 26 bits in order to match the low address field */
		214	ADDRESS_WATCH_REG_ADDLOW_SHIFT = 6,
		215	ADDRESS_WATCH_REG_ADDHIGH_MASK = 0xFFFF
		216	};
		217
		218	enum {
		219	MAX_TRAPID = 8, /* 3 bits in the bitfield. */
		220	MAX_WATCH_ADDRESSES = 4
		221	};
		222
		223	enum {
		224	ADDRESS_WATCH_REG_ADDR_HI = 0,
		225	ADDRESS_WATCH_REG_ADDR_LO,
		226	ADDRESS_WATCH_REG_CNTL,
		227	ADDRESS_WATCH_REG_MAX
		228	};
		229
193	#endif /* KFD_DBGDEV_H_ */	230	#endif /* KFD_DBGDEV_H_ */

drivers/gpu/drm/amd/amdkfd/kfd_debugfs.c+48−0

@@ -21,6 +21,8 @@ · 20 unmodified lines
21	*/	21	*/
22		22
23	#include <linux/debugfs.h>	23	#include <linux/debugfs.h>
		24	#include <linux/uaccess.h>
		25
24	#include "kfd_priv.h"	26	#include "kfd_priv.h"
25		27
26	static struct dentry *debugfs_root;	28	static struct dentry *debugfs_root;
@@ -32,6 +34,38 @@ static int kfd_debugfs_open(struct inode inode, struct file file) · 5 unmodified lines
32	return single_open(file, show, NULL);	34	return single_open(file, show, NULL);
33	}	35	}
34		36
		37	static ssize_t kfd_debugfs_hang_hws_write(struct file *file,
		38	const char __user user_buf, size_t size, loff_t ppos)
		39	{
		40	struct kfd_dev *dev;
		41	char tmp[16];
		42	uint32_t gpu_id;
		43	int ret = -EINVAL;
		44
		45	memset(tmp, 0, 16);
		46	if (size >= 16) {
		47	pr_err("Invalid input for gpu id.\n");
		48	goto out;
		49	}
		50	if (copy_from_user(tmp, user_buf, size)) {
		51	ret = -EFAULT;
		52	goto out;
		53	}
		54	if (kstrtoint(tmp, 10, &gpu_id)) {
		55	pr_err("Invalid input for gpu id.\n");
		56	goto out;
		57	}
		58	dev = kfd_device_by_id(gpu_id);
		59	if (dev) {
		60	kfd_debugfs_hang_hws(dev);
		61	ret = size;
		62	} else
		63	pr_err("Cannot find device %d.\n", gpu_id);
		64
		65	out:
		66	return ret;
		67	}
		68
35	static const struct file_operations kfd_debugfs_fops = {	69	static const struct file_operations kfd_debugfs_fops = {
36	.owner = THIS_MODULE,	70	.owner = THIS_MODULE,
37	.open = kfd_debugfs_open,	71	.open = kfd_debugfs_open,
@@ -40,6 +74,15 @@ static const struct file_operations kfd_debugfs_fops = { · 2 unmodified lines
40	.release = single_release,	74	.release = single_release,
41	};	75	};
42		76
		77	static const struct file_operations kfd_debugfs_hang_hws_fops = {
		78	.owner = THIS_MODULE,
		79	.open = kfd_debugfs_open,
		80	.read = seq_read,
		81	.write = kfd_debugfs_hang_hws_write,
		82	.llseek = seq_lseek,
		83	.release = single_release,
		84	};
		85
43	void kfd_debugfs_init(void)	86	void kfd_debugfs_init(void)
44	{	87	{
45	struct dentry *ent;	88	struct dentry *ent;
@@ -65,6 +108,11 @@ void kfd_debugfs_init(void) · 19 unmodified lines
65	ent = debugfs_create_file("rls", S_IFREG \| 0444, debugfs_root,	108	ent = debugfs_create_file("rls", S_IFREG \| 0444, debugfs_root,
66	kfd_debugfs_rls_by_device,	109	kfd_debugfs_rls_by_device,
67	&kfd_debugfs_fops);	110	&kfd_debugfs_fops);
		111
		112	ent = debugfs_create_file("hang_hws", S_IFREG \| 0644, debugfs_root,
		113	NULL,
		114	&kfd_debugfs_hang_hws_fops);
		115
68	if (!ent)	116	if (!ent)
69	pr_warn("Failed to create rls in kfd debugfs\n");	117	pr_warn("Failed to create rls in kfd debugfs\n");
70	}	118	}

drivers/gpu/drm/amd/amdkfd/kfd_device.c+115−7

@@ -30,7 +30,13 @@ · 29 unmodified lines
30	#include "kfd_iommu.h"	30	#include "kfd_iommu.h"
31		31
32	#define MQD_SIZE_ALIGNED 768	32	#define MQD_SIZE_ALIGNED 768
33	static atomic_t kfd_device_suspended = ATOMIC_INIT(0);	33
		34	/*
		35	* kfd_locked is used to lock the kfd driver during suspend or reset
		36	* once locked, kfd driver will stop any further GPU execution.
		37	* create process (open) will return -EAGAIN.
		38	*/
		39	static atomic_t kfd_locked = ATOMIC_INIT(0);
34		40
35	#ifdef KFD_SUPPORT_IOMMU_V2	41	#ifdef KFD_SUPPORT_IOMMU_V2
36	static const struct kfd_device_info kaveri_device_info = {	42	static const struct kfd_device_info kaveri_device_info = {
@@ -46,6 +52,7 @@ static const struct kfd_device_info kaveri_device_info = { · 9 unmodified lines
46	.supports_cwsr = false,	52	.supports_cwsr = false,
47	.needs_iommu_device = true,	53	.needs_iommu_device = true,
48	.needs_pci_atomics = false,	54	.needs_pci_atomics = false,
		55	.num_sdma_engines = 2,
49	};	56	};
50		57
51	static const struct kfd_device_info carrizo_device_info = {	58	static const struct kfd_device_info carrizo_device_info = {
@@ -61,6 +68,22 @@ static const struct kfd_device_info carrizo_device_info = { · 9 unmodified lines
61	.supports_cwsr = true,	68	.supports_cwsr = true,
62	.needs_iommu_device = true,	69	.needs_iommu_device = true,
63	.needs_pci_atomics = false,	70	.needs_pci_atomics = false,
		71	.num_sdma_engines = 2,
		72	};
		73
		74	static const struct kfd_device_info raven_device_info = {
		75	.asic_family = CHIP_RAVEN,
		76	.max_pasid_bits = 16,
		77	.max_no_of_hqd = 24,
		78	.doorbell_size = 8,
		79	.ih_ring_entry_size = 8 * sizeof(uint32_t),
		80	.event_interrupt_class = &event_interrupt_class_v9,
		81	.num_of_watch_points = 4,
		82	.mqd_size_aligned = MQD_SIZE_ALIGNED,
		83	.supports_cwsr = true,
		84	.needs_iommu_device = true,
		85	.needs_pci_atomics = true,
		86	.num_sdma_engines = 1,
64	};	87	};
65	#endif	88	#endif
66		89
@@ -77,6 +100,7 @@ static const struct kfd_device_info hawaii_device_info = { · 10 unmodified lines
77	.supports_cwsr = false,	100	.supports_cwsr = false,
78	.needs_iommu_device = false,	101	.needs_iommu_device = false,
79	.needs_pci_atomics = false,	102	.needs_pci_atomics = false,
		103	.num_sdma_engines = 2,
80	};	104	};
81		105
82	static const struct kfd_device_info tonga_device_info = {	106	static const struct kfd_device_info tonga_device_info = {
@@ -91,6 +115,7 @@ static const struct kfd_device_info tonga_device_info = { · 8 unmodified lines
91	.supports_cwsr = false,	115	.supports_cwsr = false,
92	.needs_iommu_device = false,	116	.needs_iommu_device = false,
93	.needs_pci_atomics = true,	117	.needs_pci_atomics = true,
		118	.num_sdma_engines = 2,
94	};	119	};
95		120
96	static const struct kfd_device_info tonga_vf_device_info = {	121	static const struct kfd_device_info tonga_vf_device_info = {
@@ -105,6 +130,7 @@ static const struct kfd_device_info tonga_vf_device_info = { · 8 unmodified lines
105	.supports_cwsr = false,	130	.supports_cwsr = false,
106	.needs_iommu_device = false,	131	.needs_iommu_device = false,
107	.needs_pci_atomics = false,	132	.needs_pci_atomics = false,
		133	.num_sdma_engines = 2,
108	};	134	};
109		135
110	static const struct kfd_device_info fiji_device_info = {	136	static const struct kfd_device_info fiji_device_info = {
@@ -119,6 +145,7 @@ static const struct kfd_device_info fiji_device_info = { · 8 unmodified lines
119	.supports_cwsr = true,	145	.supports_cwsr = true,
120	.needs_iommu_device = false,	146	.needs_iommu_device = false,
121	.needs_pci_atomics = true,	147	.needs_pci_atomics = true,
		148	.num_sdma_engines = 2,
122	};	149	};
123		150
124	static const struct kfd_device_info fiji_vf_device_info = {	151	static const struct kfd_device_info fiji_vf_device_info = {
@@ -133,6 +160,7 @@ static const struct kfd_device_info fiji_vf_device_info = { · 8 unmodified lines
133	.supports_cwsr = true,	160	.supports_cwsr = true,
134	.needs_iommu_device = false,	161	.needs_iommu_device = false,
135	.needs_pci_atomics = false,	162	.needs_pci_atomics = false,
		163	.num_sdma_engines = 2,
136	};	164	};
137		165
138		166
@@ -148,6 +176,7 @@ static const struct kfd_device_info polaris10_device_info = { · 9 unmodified lines
148	.supports_cwsr = true,	176	.supports_cwsr = true,
149	.needs_iommu_device = false,	177	.needs_iommu_device = false,
150	.needs_pci_atomics = true,	178	.needs_pci_atomics = true,
		179	.num_sdma_engines = 2,
151	};	180	};
152		181
153	static const struct kfd_device_info polaris10_vf_device_info = {	182	static const struct kfd_device_info polaris10_vf_device_info = {
@@ -162,6 +191,7 @@ static const struct kfd_device_info polaris10_vf_device_info = { · 8 unmodified lines
162	.supports_cwsr = true,	191	.supports_cwsr = true,
163	.needs_iommu_device = false,	192	.needs_iommu_device = false,
164	.needs_pci_atomics = false,	193	.needs_pci_atomics = false,
		194	.num_sdma_engines = 2,
165	};	195	};
166		196
167	static const struct kfd_device_info polaris11_device_info = {	197	static const struct kfd_device_info polaris11_device_info = {
@@ -176,6 +206,7 @@ static const struct kfd_device_info polaris11_device_info = { · 8 unmodified lines
176	.supports_cwsr = true,	206	.supports_cwsr = true,
177	.needs_iommu_device = false,	207	.needs_iommu_device = false,
178	.needs_pci_atomics = true,	208	.needs_pci_atomics = true,
		209	.num_sdma_engines = 2,
179	};	210	};
180		211
181	static const struct kfd_device_info vega10_device_info = {	212	static const struct kfd_device_info vega10_device_info = {
@@ -190,6 +221,7 @@ static const struct kfd_device_info vega10_device_info = { · 8 unmodified lines
190	.supports_cwsr = true,	221	.supports_cwsr = true,
191	.needs_iommu_device = false,	222	.needs_iommu_device = false,
192	.needs_pci_atomics = false,	223	.needs_pci_atomics = false,
		224	.num_sdma_engines = 2,
193	};	225	};
194		226
195	static const struct kfd_device_info vega10_vf_device_info = {	227	static const struct kfd_device_info vega10_vf_device_info = {
@@ -204,6 +236,7 @@ static const struct kfd_device_info vega10_vf_device_info = { · 8 unmodified lines
204	.supports_cwsr = true,	236	.supports_cwsr = true,
205	.needs_iommu_device = false,	237	.needs_iommu_device = false,
206	.needs_pci_atomics = false,	238	.needs_pci_atomics = false,
		239	.num_sdma_engines = 2,
207	};	240	};
208		241
209		242
@@ -241,6 +274,7 @@ static const struct kfd_deviceid supported_devices[] = { · 31 unmodified lines
241	{ 0x9875, &carrizo_device_info }, /* Carrizo */	274	{ 0x9875, &carrizo_device_info }, /* Carrizo */
242	{ 0x9876, &carrizo_device_info }, /* Carrizo */	275	{ 0x9876, &carrizo_device_info }, /* Carrizo */
243	{ 0x9877, &carrizo_device_info }, /* Carrizo */	276	{ 0x9877, &carrizo_device_info }, /* Carrizo */
		277	{ 0x15DD, &raven_device_info }, /* Raven */
244	#endif	278	#endif
245	{ 0x67A0, &hawaii_device_info }, /* Hawaii */	279	{ 0x67A0, &hawaii_device_info }, /* Hawaii */
246	{ 0x67A1, &hawaii_device_info }, /* Hawaii */	280	{ 0x67A1, &hawaii_device_info }, /* Hawaii */
@@ -514,13 +548,54 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd) · 267 unmodified lines
514	kfree(kfd);	548	kfree(kfd);
515	}	549	}
516		550
		551	int kgd2kfd_pre_reset(struct kfd_dev *kfd)
		552	{
		553	if (!kfd->init_complete)
		554	return 0;
		555	kgd2kfd_suspend(kfd);
		556
		557	/* hold dqm->lock to prevent further execution*/
		558	dqm_lock(kfd->dqm);
		559
		560	kfd_signal_reset_event(kfd);
		561	return 0;
		562	}
		563
		564	/*
		565	* Fix me. KFD won't be able to resume existing process for now.
		566	* We will keep all existing process in a evicted state and
		567	* wait the process to be terminated.
		568	*/
		569
		570	int kgd2kfd_post_reset(struct kfd_dev *kfd)
		571	{
		572	int ret, count;
		573
		574	if (!kfd->init_complete)
		575	return 0;
		576
		577	dqm_unlock(kfd->dqm);
		578
		579	ret = kfd_resume(kfd);
		580	if (ret)
		581	return ret;
		582	count = atomic_dec_return(&kfd_locked);
		583	WARN_ONCE(count != 0, "KFD reset ref. error");
		584	return 0;
		585	}
		586
		587	bool kfd_is_locked(void)
		588	{
		589	return (atomic_read(&kfd_locked) > 0);
		590	}
		591
517	void kgd2kfd_suspend(struct kfd_dev *kfd)	592	void kgd2kfd_suspend(struct kfd_dev *kfd)
518	{	593	{
519	if (!kfd->init_complete)	594	if (!kfd->init_complete)
520	return;	595	return;
521		596
522	/* For first KFD device suspend all the KFD processes */	597	/* For first KFD device suspend all the KFD processes */
523	if (atomic_inc_return(&kfd_device_suspended) == 1)	598	if (atomic_inc_return(&kfd_locked) == 1)
524	kfd_suspend_all_processes();	599	kfd_suspend_all_processes();
525		600
526	kfd->dqm->ops.stop(kfd->dqm);	601	kfd->dqm->ops.stop(kfd->dqm);
@@ -539,7 +614,7 @@ int kgd2kfd_resume(struct kfd_dev *kfd) · 12 unmodified lines
539	if (ret)	614	if (ret)
540	return ret;	615	return ret;
541		616
542	count = atomic_dec_return(&kfd_device_suspended);	617	count = atomic_dec_return(&kfd_locked);
543	WARN_ONCE(count < 0, "KFD suspend / resume ref. error");	618	WARN_ONCE(count < 0, "KFD suspend / resume ref. error");
544	if (count == 0)	619	if (count == 0)
545	ret = kfd_resume_all_processes();	620	ret = kfd_resume_all_processes();
@@ -577,14 +652,24 @@ dqm_start_error: · 31 unmodified lines
577	/* This is called directly from KGD at ISR. */	652	/* This is called directly from KGD at ISR. */
578	void kgd2kfd_interrupt(struct kfd_dev kfd, const void ih_ring_entry)	653	void kgd2kfd_interrupt(struct kfd_dev kfd, const void ih_ring_entry)
579	{	654	{
		655	uint32_t patched_ihre[KFD_MAX_RING_ENTRY_SIZE];
		656	bool is_patched = false;
		657
580	if (!kfd->init_complete)	658	if (!kfd->init_complete)
581	return;	659	return;
582		660
		661	if (kfd->device_info->ih_ring_entry_size > sizeof(patched_ihre)) {
		662	dev_err_once(kfd_device, "Ring entry too small\n");
		663	return;
		664	}
		665
583	spin_lock(&kfd->interrupt_lock);	666	spin_lock(&kfd->interrupt_lock);
584		667
585	if (kfd->interrupts_active	668	if (kfd->interrupts_active
586	&& interrupt_is_wanted(kfd, ih_ring_entry)	669	&& interrupt_is_wanted(kfd, ih_ring_entry,
587	&& enqueue_ih_ring_entry(kfd, ih_ring_entry))	670	patched_ihre, &is_patched)
		671	&& enqueue_ih_ring_entry(kfd,
		672	is_patched ? patched_ihre : ih_ring_entry))
588	queue_work(kfd->ih_wq, &kfd->interrupt_work);	673	queue_work(kfd->ih_wq, &kfd->interrupt_work);
589		674
590	spin_unlock(&kfd->interrupt_lock);	675	spin_unlock(&kfd->interrupt_lock);
@@ -739,8 +824,8 @@ int kfd_gtt_sa_allocate(struct kfd_dev *kfd, unsigned int size, · 148 unmodified lines
739	if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size)	824	if (size > kfd->gtt_sa_num_of_chunks * kfd->gtt_sa_chunk_size)
740	return -ENOMEM;	825	return -ENOMEM;
741		826
742	*mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);	827	*mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
743	if ((*mem_obj) == NULL)	828	if (!(*mem_obj))
744	return -ENOMEM;	829	return -ENOMEM;
745		830
746	pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size);	831	pr_debug("Allocated mem_obj = %p for size = %d\n", *mem_obj, size);
@@ -857,3 +942,26 @@ int kfd_gtt_sa_free(struct kfd_dev kfd, struct kfd_mem_obj mem_obj) · 110 unmodified lines
857	kfree(mem_obj);	942	kfree(mem_obj);
858	return 0;	943	return 0;
859	}	944	}
		945
		946	#if defined(CONFIG_DEBUG_FS)
		947
		948	/* This function will send a package to HIQ to hang the HWS
		949	* which will trigger a GPU reset and bring the HWS back to normal state
		950	*/
		951	int kfd_debugfs_hang_hws(struct kfd_dev *dev)
		952	{
		953	int r = 0;
		954
		955	if (dev->dqm->sched_policy != KFD_SCHED_POLICY_HWS) {
		956	pr_err("HWS is not enabled");
		957	return -EINVAL;
		958	}
		959
		960	r = pm_debugfs_hang_hws(&dev->dqm->packets);
		961	if (!r)
		962	r = dqm_debugfs_execute_queues(dev->dqm);
		963
		964	return r;
		965	}
		966
		967	#endif

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c+177−106

 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
 				unsigned int sdma_queue_id);
+static void kfd_process_hw_exception(struct work_struct *work);
 static inline
 enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type)
 {
 	return dqm->dev->shared_resources.num_pipe_per_mec;
 }
+static unsigned int get_num_sdma_engines(struct device_queue_manager *dqm)
+{
+	return dqm->dev->device_info->num_sdma_engines;
+}
+unsigned int get_num_sdma_queues(struct device_queue_manager *dqm)
+{
+	return dqm->dev->device_info->num_sdma_engines
+			* KFD_SDMA_QUEUES_PER_ENGINE;
+}
 void program_sh_mem_settings(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd)
 {
 	print_queue(q);
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 		pr_warn("Can't create new usermode queue because %d queues were already created\n",
 			dqm->total_queue_count);
 out_unlock:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 					struct qcm_process_device *qpd)
 {
 	int retval;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
-	mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
+	mqd_mgr = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_COMPUTE);
-	if (!mqd)
+	if (!mqd_mgr)
 		return -ENOMEM;
 	retval = allocate_hqd(dqm, q);
 	if (retval)
 		goto out_deallocate_hqd;
-	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
+	retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
 	if (retval)
 		goto out_deallocate_doorbell;
 	if (!q->properties.is_active)
 		return 0;
-	retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue, &q->properties,
+	retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, q->queue,
-			       q->process->mm);
+			&q->properties, q->process->mm);
 	if (retval)
 		goto out_uninit_mqd;
 	return 0;
 out_uninit_mqd:
-	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+	mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 out_deallocate_doorbell:
 	deallocate_doorbell(qpd, q);
 out_deallocate_hqd:
 				struct queue *q)
 {
 	int retval;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
-	mqd = dqm->ops.get_mqd_manager(dqm,
+	mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 		get_mqd_type_from_queue_type(q->properties.type));
-	if (!mqd)
+	if (!mqd_mgr)
 		return -ENOMEM;
 	if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE) {
 	deallocate_doorbell(qpd, q);
-	retval = mqd->destroy_mqd(mqd, q->mqd,
+	retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
 				KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
 				KFD_UNMAP_LATENCY_MS,
 				q->pipe, q->queue);
 	if (retval == -ETIME)
 		qpd->reset_wavefronts = true;
-	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+	mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 	list_del(&q->list);
 	if (list_empty(&qpd->queues_list)) {
 {
 	int retval;
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	retval = destroy_queue_nocpsch_locked(dqm, qpd, q);
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 static int update_queue(struct device_queue_manager *dqm, struct queue *q)
 {
 	int retval;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	struct kfd_process_device *pdd;
 	bool prev_active = false;
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	pdd = kfd_get_process_device_data(q->device, q->process);
 	if (!pdd) {
 		retval = -ENODEV;
 		goto out_unlock;
 	}
-	mqd = dqm->ops.get_mqd_manager(dqm,
+	mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
-	if (!mqd) {
+	if (!mqd_mgr) {
 		retval = -ENOMEM;
 		goto out_unlock;
 	}
 	} else if (prev_active &&
 		   (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
 		    q->properties.type == KFD_QUEUE_TYPE_SDMA)) {
-		retval = mqd->destroy_mqd(mqd, q->mqd,
+		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
 				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
 				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
 		if (retval) {
 		}
 	}
-	retval = mqd->update_mqd(mqd, q->mqd, &q->properties);
+	retval = mqd_mgr->update_mqd(mqd_mgr, q->mqd, &q->properties);
 	/*
 	 * check active state vs. the previous state and modify
 	else if (q->properties.is_active &&
 		 (q->properties.type == KFD_QUEUE_TYPE_COMPUTE ||
 		  q->properties.type == KFD_QUEUE_TYPE_SDMA))
-		retval = mqd->load_mqd(mqd, q->mqd, q->pipe, q->queue,
+		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe, q->queue,
 				       &q->properties, q->process->mm);
 out_unlock:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 static struct mqd_manager *get_mqd_manager(
 		struct device_queue_manager *dqm, enum KFD_MQD_TYPE type)
 {
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
 		return NULL;
 	pr_debug("mqd type %d\n", type);
-	mqd = dqm->mqds[type];
+	mqd_mgr = dqm->mqd_mgrs[type];
-	if (!mqd) {
+	if (!mqd_mgr) {
-		mqd = mqd_manager_init(type, dqm->dev);
+		mqd_mgr = mqd_manager_init(type, dqm->dev);
-		if (!mqd)
+		if (!mqd_mgr)
 			pr_err("mqd manager is NULL");
-		dqm->mqds[type] = mqd;
+		dqm->mqd_mgrs[type] = mqd_mgr;
 	}
-	return mqd;
+	return mqd_mgr;
 }
 static int evict_process_queues_nocpsch(struct device_queue_manager *dqm,
 					struct qcm_process_device *qpd)
 {
 	struct queue *q;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	struct kfd_process_device *pdd;
 	int retval = 0;
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (qpd->evicted++ > 0) /* already evicted, do nothing */
 		goto out;
 	list_for_each_entry(q, &qpd->queues_list, list) {
 		if (!q->properties.is_active)
 			continue;
-		mqd = dqm->ops.get_mqd_manager(dqm,
+		mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
-		if (!mqd) { /* should not be here */
+		if (!mqd_mgr) { /* should not be here */
 			pr_err("Cannot evict queue, mqd mgr is NULL\n");
 			retval = -ENOMEM;
 			goto out;
 		}
 		q->properties.is_evicted = true;
 		q->properties.is_active = false;
-		retval = mqd->destroy_mqd(mqd, q->mqd,
+		retval = mqd_mgr->destroy_mqd(mqd_mgr, q->mqd,
 				KFD_PREEMPT_TYPE_WAVEFRONT_DRAIN,
 				KFD_UNMAP_LATENCY_MS, q->pipe, q->queue);
 		if (retval)
 	}
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 	struct kfd_process_device *pdd;
 	int retval = 0;
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (qpd->evicted++ > 0) /* already evicted, do nothing */
 		goto out;
 				KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 					  struct qcm_process_device *qpd)
 {
 	struct queue *q;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	struct kfd_process_device *pdd;
 	uint32_t pd_base;
 	int retval = 0;
 	/* Retrieve PD base */
 	pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
 		goto out;
 	if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
 	list_for_each_entry(q, &qpd->queues_list, list) {
 		if (!q->properties.is_evicted)
 			continue;
-		mqd = dqm->ops.get_mqd_manager(dqm,
+		mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
-		if (!mqd) { /* should not be here */
+		if (!mqd_mgr) { /* should not be here */
 			pr_err("Cannot restore queue, mqd mgr is NULL\n");
 			retval = -ENOMEM;
 			goto out;
 		}
 		q->properties.is_evicted = false;
 		q->properties.is_active = true;
-		retval = mqd->load_mqd(mqd, q->mqd, q->pipe,
+		retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, q->pipe,
 				       q->queue, &q->properties,
 				       q->process->mm);
 		if (retval)
 	}
 	qpd->evicted = 0;
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 	/* Retrieve PD base */
 	pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
 		goto out;
 	if (qpd->evicted > 1) { /* ref count still > 0, decrement & quit */
 	if (!retval)
 		qpd->evicted = 0;
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 	/* Retrieve PD base */
 	pd_base = dqm->dev->kfd2kgd->get_process_page_dir(pdd->vm);
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	list_add(&n->list, &dqm->queues);
 	/* Update PD Base in QPD */
 	retval = dqm->asic_ops.update_qpd(dqm, qpd);
-	dqm->processes_count++;
+	if (dqm->processes_count++ == 0)
+		dqm->dev->kfd2kgd->set_compute_idle(dqm->dev->kgd, false);
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 			list_empty(&qpd->queues_list) ? "empty" : "not empty");
 	retval = 0;
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	list_for_each_entry_safe(cur, next, &dqm->queues, list) {
 		if (qpd == cur->qpd) {
 			list_del(&cur->list);
 			kfree(cur);
-			dqm->processes_count--;
+			if (--dqm->processes_count == 0)
+				dqm->dev->kfd2kgd->set_compute_idle(
+					dqm->dev->kgd, true);
 			goto out;
 		}
 	}
 	/* qpd not found in dqm list */
 	retval = 1;
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 	if (!dqm->allocated_queues)
 		return -ENOMEM;
-	mutex_init(&dqm->lock);
+	mutex_init(&dqm->lock_hidden);
 	INIT_LIST_HEAD(&dqm->queues);
 	dqm->queue_count = dqm->next_pipe_to_allocate = 0;
 	dqm->sdma_queue_count = 0;
 	}
 	dqm->vmid_bitmap = (1 << dqm->dev->vm_info.vmid_num_kfd) - 1;
-	dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
+	dqm->sdma_bitmap = (1 << get_num_sdma_queues(dqm)) - 1;
 	return 0;
 }
 	kfree(dqm->allocated_queues);
 	for (i = 0 ; i < KFD_MQD_TYPE_MAX ; i++)
-		kfree(dqm->mqds[i]);
+		kfree(dqm->mqd_mgrs[i]);
-	mutex_destroy(&dqm->lock);
+	mutex_destroy(&dqm->lock_hidden);
 	kfd_gtt_sa_free(dqm->dev, dqm->pipeline_mem);
 }
 static void deallocate_sdma_queue(struct device_queue_manager *dqm,
 				unsigned int sdma_queue_id)
 {
-	if (sdma_queue_id >= CIK_SDMA_QUEUES)
+	if (sdma_queue_id >= get_num_sdma_queues(dqm))
 		return;
 	dqm->sdma_bitmap |= (1 << sdma_queue_id);
 }
 					struct queue *q,
 					struct qcm_process_device *qpd)
 {
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	int retval;
-	mqd = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
+	mqd_mgr = dqm->ops.get_mqd_manager(dqm, KFD_MQD_TYPE_SDMA);
-	if (!mqd)
+	if (!mqd_mgr)
 		return -ENOMEM;
 	retval = allocate_sdma_queue(dqm, &q->sdma_id);
 	if (retval)
 		return retval;
-	q->properties.sdma_queue_id = q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
+	q->properties.sdma_queue_id = q->sdma_id / get_num_sdma_engines(dqm);
-	q->properties.sdma_engine_id = q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
+	q->properties.sdma_engine_id = q->sdma_id % get_num_sdma_engines(dqm);
 	retval = allocate_doorbell(qpd, q);
 	if (retval)
 	pr_debug("SDMA engine id: %d\n", q->properties.sdma_engine_id);
 	dqm->asic_ops.init_sdma_vm(dqm, q, qpd);
-	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
+	retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
 	if (retval)
 		goto out_deallocate_doorbell;
-	retval = mqd->load_mqd(mqd, q->mqd, 0, 0, &q->properties, NULL);
+	retval = mqd_mgr->load_mqd(mqd_mgr, q->mqd, 0, 0, &q->properties,
+				NULL);
 	if (retval)
 		goto out_uninit_mqd;
 	return 0;
 out_uninit_mqd:
-	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+	mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 out_deallocate_doorbell:
 	deallocate_doorbell(qpd, q);
 out_deallocate_sdma_queue:
 {
 	pr_debug("num of pipes: %d\n", get_pipes_per_mec(dqm));
-	mutex_init(&dqm->lock);
+	mutex_init(&dqm->lock_hidden);
 	INIT_LIST_HEAD(&dqm->queues);
 	dqm->queue_count = dqm->processes_count = 0;
 	dqm->sdma_queue_count = 0;
 	dqm->active_runlist = false;
-	dqm->sdma_bitmap = (1 << CIK_SDMA_QUEUES) - 1;
+	dqm->sdma_bitmap = (1 << get_num_sdma_queues(dqm)) - 1;
+	INIT_WORK(&dqm->hw_exception_work, kfd_process_hw_exception);
 	return 0;
 }
 	init_interrupts(dqm);
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
+	/* clear hang status when driver try to start the hw scheduler */
+	dqm->is_hws_hang = false;
 	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return 0;
 fail_allocate_vidmem:
 static int stop_cpsch(struct device_queue_manager *dqm)
 {
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	unmap_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	kfd_gtt_sa_free(dqm->dev, dqm->fence_mem);
 	pm_uninit(&dqm->packets);
 					struct kernel_queue *kq,
 					struct qcm_process_device *qpd)
 {
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 		pr_warn("Can't create new kernel queue because %d queues were already created\n",
 				dqm->total_queue_count);
-		mutex_unlock(&dqm->lock);
+		dqm_unlock(dqm);
 		return -EPERM;
 	}
 	dqm->queue_count++;
 	qpd->is_debug = true;
 	execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES, 0);
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return 0;
 }
 					struct kernel_queue *kq,
 					struct qcm_process_device *qpd)
 {
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	list_del(&kq->list);
 	dqm->queue_count--;
 	qpd->is_debug = false;
 	dqm->total_queue_count--;
 	pr_debug("Total of %d queues are accountable so far\n",
 			dqm->total_queue_count);
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 }
 static int create_queue_cpsch(struct device_queue_manager *dqm, struct queue *q,
 			struct qcm_process_device *qpd)
 {
 	int retval;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	retval = 0;
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (dqm->total_queue_count >= max_num_of_queues_per_device) {
 		pr_warn("Can't create new usermode queue because %d queues were already created\n",
 		if (retval)
 			goto out_unlock;
 		q->properties.sdma_queue_id =
-			q->sdma_id / CIK_SDMA_QUEUES_PER_ENGINE;
+			q->sdma_id / get_num_sdma_engines(dqm);
 		q->properties.sdma_engine_id =
-			q->sdma_id % CIK_SDMA_QUEUES_PER_ENGINE;
+			q->sdma_id % get_num_sdma_engines(dqm);
 	}
 	retval = allocate_doorbell(qpd, q);
 	if (retval)
 		goto out_deallocate_sdma_queue;
-	mqd = dqm->ops.get_mqd_manager(dqm,
+	mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
-	if (!mqd) {
+	if (!mqd_mgr) {
 		retval = -ENOMEM;
 		goto out_deallocate_doorbell;
 	}
 	q->properties.tba_addr = qpd->tba_addr;
 	q->properties.tma_addr = qpd->tma_addr;
-	retval = mqd->init_mqd(mqd, &q->mqd, &q->mqd_mem_obj,
+	retval = mqd_mgr->init_mqd(mqd_mgr, &q->mqd, &q->mqd_mem_obj,
 				&q->gart_mqd_addr, &q->properties);
 	if (retval)
 		goto out_deallocate_doorbell;
 	pr_debug("Total of %d queues are accountable so far\n",
 			dqm->total_queue_count);
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 out_deallocate_doorbell:
 	if (q->properties.type == KFD_QUEUE_TYPE_SDMA)
 		deallocate_sdma_queue(dqm, q->sdma_id);
 out_unlock:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 	while (*fence_addr != fence_value) {
 		if (time_after(jiffies, end_jiffies)) {
 			pr_err("qcm fence wait loop timeout expired\n");
+			/* In HWS case, this is used to halt the driver thread
+			 * in order not to mess up CP states before doing
+			 * scandumps for FW debugging.
+			 */
+			while (halt_if_hws_hang)
+				schedule();
 			return -ETIME;
 		}
 		schedule();
 {
 	int retval = 0;
+	if (dqm->is_hws_hang)
+		return -EIO;
 	if (!dqm->active_runlist)
 		return retval;
 {
 	int retval;
+	if (dqm->is_hws_hang)
+		return -EIO;
 	retval = unmap_queues_cpsch(dqm, filter, filter_param);
 	if (retval) {
 		pr_err("The cp might be in an unrecoverable state due to an unsuccessful queues preemption\n");
+		dqm->is_hws_hang = true;
+		schedule_work(&dqm->hw_exception_work);
 		return retval;
 	}
 				struct queue *q)
 {
 	int retval;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	bool preempt_all_queues;
 	preempt_all_queues = false;
 	retval = 0;
 	/* remove queue from list to prevent rescheduling after preemption */
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (qpd->is_debug) {
 		/*
 	}
-	mqd = dqm->ops.get_mqd_manager(dqm,
+	mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
-	if (!mqd) {
+	if (!mqd_mgr) {
 		retval = -ENOMEM;
 		goto failed;
 	}
 			qpd->reset_wavefronts = true;
 	}
-	mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+	mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 	/*
 	 * Unconditionally decrement this counter, regardless of the queue's
 	pr_debug("Total of %d queues are accountable so far\n",
 			dqm->total_queue_count);
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 failed:
 failed_try_destroy_debugged_queue:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 	if (!dqm->asic_ops.set_cache_memory_policy)
 		return retval;
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	if (alternate_aperture_size == 0) {
 		/* base > limit disables APE1 */
 		qpd->sh_mem_ape1_limit);
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 	struct device_process_node *cur, *next_dpn;
 	int retval = 0;
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	/* Clear all user mode queues */
 	list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
 		}
 	}
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 	int retval;
 	struct queue *q, *next;
 	struct kernel_queue *kq, *kq_next;
-	struct mqd_manager *mqd;
+	struct mqd_manager *mqd_mgr;
 	struct device_process_node *cur, *next_dpn;
 	enum kfd_unmap_queues_filter filter =
 		KFD_UNMAP_QUEUES_FILTER_DYNAMIC_QUEUES;
 	retval = 0;
-	mutex_lock(&dqm->lock);
+	dqm_lock(dqm);
 	/* Clean all kernel queues */
 	list_for_each_entry_safe(kq, kq_next, &qpd->priv_queue_list, list) {
 	}
 	retval = execute_queues_cpsch(dqm, filter, 0);
-	if (retval || qpd->reset_wavefronts) {
+	if ((!dqm->is_hws_hang) && (retval || qpd->reset_wavefronts)) {
 		pr_warn("Resetting wave fronts (cpsch) on dev %p\n", dqm->dev);
 		dbgdev_wave_reset_wavefronts(dqm->dev, qpd->pqm->process);
 		qpd->reset_wavefronts = false;
 	/* lastly, free mqd resources */
 	list_for_each_entry_safe(q, next, &qpd->queues_list, list) {
-		mqd = dqm->ops.get_mqd_manager(dqm,
+		mqd_mgr = dqm->ops.get_mqd_manager(dqm,
 			get_mqd_type_from_queue_type(q->properties.type));
-		if (!mqd) {
+		if (!mqd_mgr) {
 			retval = -ENOMEM;
 			goto out;
 		}
 		list_del(&q->list);
 		qpd->queue_count--;
-		mqd->uninit_mqd(mqd, q->mqd, q->mqd_mem_obj);
+		mqd_mgr->uninit_mqd(mqd_mgr, q->mqd, q->mqd_mem_obj);
 	}
 out:
-	mutex_unlock(&dqm->lock);
+	dqm_unlock(dqm);
 	return retval;
 }
 	kfree(dqm);
 }
+int kfd_process_vm_fault(struct device_queue_manager *dqm,
+			 unsigned int pasid)
+{
+	struct kfd_process_device *pdd;
+	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
+	int ret = 0;
+	if (!p)
+		return -EINVAL;
+	pdd = kfd_get_process_device_data(dqm->dev, p);
+	if (pdd)
+		ret = dqm->ops.evict_process_queues(dqm, &pdd->qpd);
+	kfd_unref_process(p);
+	return ret;
+}
+static void kfd_process_hw_exception(struct work_struct *work)
+{
+	struct device_queue_manager *dqm = container_of(work,
+			struct device_queue_manager, hw_exception_work);
+	dqm->dev->kfd2kgd->gpu_recover(dqm->dev->kgd);
+}
 #if defined(CONFIG_DEBUG_FS)
 static void seq_reg_dump(struct seq_file *m,
 		}
 	}
-	for (pipe = 0; pipe < CIK_SDMA_ENGINE_NUM; pipe++) {
+	for (pipe = 0; pipe < get_num_sdma_engines(dqm); pipe++) {
-		for (queue = 0; queue < CIK_SDMA_QUEUES_PER_ENGINE; queue++) {
+		for (queue = 0; queue < KFD_SDMA_QUEUES_PER_ENGINE; queue++) {
 			r = dqm->dev->kfd2kgd->hqd_sdma_dump(
 				dqm->dev->kgd, pipe, queue, &dump, &n_regs);
 			if (r)
 	return r;
 }
+int dqm_debugfs_execute_queues(struct device_queue_manager *dqm)
+{
+	int r = 0;
+	dqm_lock(dqm);
+	dqm->active_runlist = true;
+	r = execute_queues_cpsch(dqm, KFD_UNMAP_QUEUES_FILTER_ALL_QUEUES, 0);
+	dqm_unlock(dqm);
+	return r;
+}
 #endif

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h+26−6

@@ -26,15 +26,14 @@ · 25 unmodified lines
26		26
27	#include <linux/rwsem.h>	27	#include <linux/rwsem.h>
28	#include <linux/list.h>	28	#include <linux/list.h>
		29	#include <linux/mutex.h>
		30	#include <linux/sched/mm.h>
29	#include "kfd_priv.h"	31	#include "kfd_priv.h"
30	#include "kfd_mqd_manager.h"	32	#include "kfd_mqd_manager.h"
31		33
32	#define KFD_UNMAP_LATENCY_MS (4000)	34	#define KFD_UNMAP_LATENCY_MS (4000)
33	#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (2 * KFD_UNMAP_LATENCY_MS + 1000)	35	#define QUEUE_PREEMPT_DEFAULT_TIMEOUT_MS (2 * KFD_UNMAP_LATENCY_MS + 1000)
34		36	#define KFD_SDMA_QUEUES_PER_ENGINE (2)
35	#define CIK_SDMA_QUEUES (4)
36	#define CIK_SDMA_QUEUES_PER_ENGINE (2)
37	#define CIK_SDMA_ENGINE_NUM (2)
38		37
39	struct device_process_node {	38	struct device_process_node {
40	struct qcm_process_device *qpd;	39	struct qcm_process_device *qpd;
@@ -170,11 +169,12 @@ struct device_queue_manager { · 129 unmodified lines
170	struct device_queue_manager_ops ops;	169	struct device_queue_manager_ops ops;
171	struct device_queue_manager_asic_ops asic_ops;	170	struct device_queue_manager_asic_ops asic_ops;
172		171
173	struct mqd_manager *mqds[KFD_MQD_TYPE_MAX];	172	struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX];
174	struct packet_manager packets;	173	struct packet_manager packets;
175	struct kfd_dev *dev;	174	struct kfd_dev *dev;
176	struct mutex lock;	175	struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */
177	struct list_head queues;	176	struct list_head queues;
		177	unsigned int saved_flags;
178	unsigned int processes_count;	178	unsigned int processes_count;
179	unsigned int queue_count;	179	unsigned int queue_count;
180	unsigned int sdma_queue_count;	180	unsigned int sdma_queue_count;
@@ -190,6 +190,10 @@ struct device_queue_manager { · 9 unmodified lines
190	struct kfd_mem_obj *fence_mem;	190	struct kfd_mem_obj *fence_mem;
191	bool active_runlist;	191	bool active_runlist;
192	int sched_policy;	192	int sched_policy;
		193
		194	/* hw exception */
		195	bool is_hws_hang;
		196	struct work_struct hw_exception_work;
193	};	197	};
194		198
195	void device_queue_manager_init_cik(	199	void device_queue_manager_init_cik(
@@ -207,6 +211,7 @@ void program_sh_mem_settings(struct device_queue_manager *dqm, · 11 unmodified lines
207	unsigned int get_queues_num(struct device_queue_manager *dqm);	211	unsigned int get_queues_num(struct device_queue_manager *dqm);
208	unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);	212	unsigned int get_queues_per_pipe(struct device_queue_manager *dqm);
209	unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);	213	unsigned int get_pipes_per_mec(struct device_queue_manager *dqm);
		214	unsigned int get_num_sdma_queues(struct device_queue_manager *dqm);
210		215
211	static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)	216	static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
212	{	217	{
@@ -219,4 +224,19 @@ get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd) · 6 unmodified lines
219	return (pdd->lds_base >> 60) & 0x0E;	224	return (pdd->lds_base >> 60) & 0x0E;
220	}	225	}
221		226
		227	/* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS
		228	* happens while holding this lock anywhere to prevent deadlocks when
		229	* an MMU notifier runs in reclaim-FS context.
		230	*/
		231	static inline void dqm_lock(struct device_queue_manager *dqm)
		232	{
		233	mutex_lock(&dqm->lock_hidden);
		234	dqm->saved_flags = memalloc_nofs_save();
		235	}
		236	static inline void dqm_unlock(struct device_queue_manager *dqm)
		237	{
		238	memalloc_nofs_restore(dqm->saved_flags);
		239	mutex_unlock(&dqm->lock_hidden);
		240	}
		241
222	#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */	242	#endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c+1−1

@@ -60,7 +60,7 @@ static int update_qpd_v9(struct device_queue_manager *dqm, · 59 unmodified lines
60	qpd->sh_mem_config =	60	qpd->sh_mem_config =
61	SH_MEM_ALIGNMENT_MODE_UNALIGNED <<	61	SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
62	SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;	62	SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
63	if (vega10_noretry &&	63	if (noretry &&
64	!dqm->dev->device_info->needs_iommu_device)	64	!dqm->dev->device_info->needs_iommu_device)
65	qpd->sh_mem_config \|=	65	qpd->sh_mem_config \|=
66	1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;	66	1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;

drivers/gpu/drm/amd/amdkfd/kfd_doorbell.c+5−4

 	*doorbell_off = kfd->doorbell_id_offset + inx;
 	pr_debug("Get kernel queue doorbell\n"
-			 "     doorbell offset   == 0x%08X\n"
+			"     doorbell offset   == 0x%08X\n"
-			 "     kernel address    == %p\n",
+			"     doorbell index    == 0x%x\n",
-		*doorbell_off, (kfd->doorbell_kernel_ptr + inx));
+		*doorbell_off, inx);
 	return kfd->doorbell_kernel_ptr + inx;
 }
 {
 	unsigned int inx;
-	inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr);
+	inx = (unsigned int)(db_addr - kfd->doorbell_kernel_ptr)
+		* sizeof(u32) / kfd->device_info->doorbell_size;
 	mutex_lock(&kfd->doorbell_mutex);
 	__clear_bit(inx, kfd->doorbell_available_index);

drivers/gpu/drm/amd/amdkfd/kfd_events.c+98−20

@@ -850,6 +850,13 @@ static void lookup_events_by_type_and_signal(struct kfd_process *p, · 849 unmodified lines
850	ev->memory_exception_data = *ev_data;	850	ev->memory_exception_data = *ev_data;
851	}	851	}
852		852
		853	if (type == KFD_EVENT_TYPE_MEMORY) {
		854	dev_warn(kfd_device,
		855	"Sending SIGSEGV to HSA Process with PID %d ",
		856	p->lead_thread->pid);
		857	send_sig(SIGSEGV, p->lead_thread, 0);
		858	}
		859
853	/* Send SIGTERM no event of type "type" has been found*/	860	/* Send SIGTERM no event of type "type" has been found*/
854	if (send_signal) {	861	if (send_signal) {
855	if (send_sigterm) {	862	if (send_sigterm) {
@@ -904,34 +911,41 @@ void kfd_signal_iommu_event(struct kfd_dev *dev, unsigned int pasid, · 48 unmodified lines
904	memory_exception_data.failure.NotPresent = 1;	911	memory_exception_data.failure.NotPresent = 1;
905	memory_exception_data.failure.NoExecute = 0;	912	memory_exception_data.failure.NoExecute = 0;
906	memory_exception_data.failure.ReadOnly = 0;	913	memory_exception_data.failure.ReadOnly = 0;
907	if (vma) {	914	if (vma && address >= vma->vm_start) {
908	if (vma->vm_start > address) {	915	memory_exception_data.failure.NotPresent = 0;
909	memory_exception_data.failure.NotPresent = 1;	916
910	memory_exception_data.failure.NoExecute = 0;	917	if (is_write_requested && !(vma->vm_flags & VM_WRITE))
		918	memory_exception_data.failure.ReadOnly = 1;
		919	else
911	memory_exception_data.failure.ReadOnly = 0;	920	memory_exception_data.failure.ReadOnly = 0;
912	} else {	921
913	memory_exception_data.failure.NotPresent = 0;	922	if (is_execute_requested && !(vma->vm_flags & VM_EXEC))
914	if (is_write_requested && !(vma->vm_flags & VM_WRITE))	923	memory_exception_data.failure.NoExecute = 1;
915	memory_exception_data.failure.ReadOnly = 1;	924	else
916	else	925	memory_exception_data.failure.NoExecute = 0;
917	memory_exception_data.failure.ReadOnly = 0;
918	if (is_execute_requested && !(vma->vm_flags & VM_EXEC))
919	memory_exception_data.failure.NoExecute = 1;
920	else
921	memory_exception_data.failure.NoExecute = 0;
922	}
923	}	926	}
924		927
925	up_read(&mm->mmap_sem);	928	up_read(&mm->mmap_sem);
926	mmput(mm);	929	mmput(mm);
927		930
928	mutex_lock(&p->event_mutex);	931	pr_debug("notpresent %d, noexecute %d, readonly %d\n",
		932	memory_exception_data.failure.NotPresent,
		933	memory_exception_data.failure.NoExecute,
		934	memory_exception_data.failure.ReadOnly);
929		935
930	/* Lookup events by type and signal them */	936	/* Workaround on Raven to not kill the process when memory is freed
931	lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_MEMORY,	937	* before IOMMU is able to finish processing all the excessive PPRs
932	&memory_exception_data);	938	*/
		939	if (dev->device_info->asic_family != CHIP_RAVEN) {
		940	mutex_lock(&p->event_mutex);
		941
		942	/* Lookup events by type and signal them */
		943	lookup_events_by_type_and_signal(p, KFD_EVENT_TYPE_MEMORY,
		944	&memory_exception_data);
		945
		946	mutex_unlock(&p->event_mutex);
		947	}
933		948
934	mutex_unlock(&p->event_mutex);
935	kfd_unref_process(p);	949	kfd_unref_process(p);
936	}	950	}
937	#endif /* KFD_SUPPORT_IOMMU_V2 */	951	#endif /* KFD_SUPPORT_IOMMU_V2 */
@@ -956,3 +970,67 @@ void kfd_signal_hw_exception_event(unsigned int pasid) · 18 unmodified lines
956	mutex_unlock(&p->event_mutex);	970	mutex_unlock(&p->event_mutex);
957	kfd_unref_process(p);	971	kfd_unref_process(p);
958	}	972	}
		973
		974	void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
		975	struct kfd_vm_fault_info *info)
		976	{
		977	struct kfd_event *ev;
		978	uint32_t id;
		979	struct kfd_process *p = kfd_lookup_process_by_pasid(pasid);
		980	struct kfd_hsa_memory_exception_data memory_exception_data;
		981
		982	if (!p)
		983	return; /* Presumably process exited. */
		984	memset(&memory_exception_data, 0, sizeof(memory_exception_data));
		985	memory_exception_data.gpu_id = dev->id;
		986	memory_exception_data.failure.imprecise = 1;
		987	/* Set failure reason */
		988	if (info) {
		989	memory_exception_data.va = (info->page_addr) << PAGE_SHIFT;
		990	memory_exception_data.failure.NotPresent =
		991	info->prot_valid ? 1 : 0;
		992	memory_exception_data.failure.NoExecute =
		993	info->prot_exec ? 1 : 0;
		994	memory_exception_data.failure.ReadOnly =
		995	info->prot_write ? 1 : 0;
		996	memory_exception_data.failure.imprecise = 0;
		997	}
		998	mutex_lock(&p->event_mutex);
		999
		1000	id = KFD_FIRST_NONSIGNAL_EVENT_ID;
		1001	idr_for_each_entry_continue(&p->event_idr, ev, id)
		1002	if (ev->type == KFD_EVENT_TYPE_MEMORY) {
		1003	ev->memory_exception_data = memory_exception_data;
		1004	set_event(ev);
		1005	}
		1006
		1007	mutex_unlock(&p->event_mutex);
		1008	kfd_unref_process(p);
		1009	}
		1010
		1011	void kfd_signal_reset_event(struct kfd_dev *dev)
		1012	{
		1013	struct kfd_hsa_hw_exception_data hw_exception_data;
		1014	struct kfd_process *p;
		1015	struct kfd_event *ev;
		1016	unsigned int temp;
		1017	uint32_t id, idx;
		1018
		1019	/* Whole gpu reset caused by GPU hang and memory is lost */
		1020	memset(&hw_exception_data, 0, sizeof(hw_exception_data));
		1021	hw_exception_data.gpu_id = dev->id;
		1022	hw_exception_data.memory_lost = 1;
		1023
		1024	idx = srcu_read_lock(&kfd_processes_srcu);
		1025	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
		1026	mutex_lock(&p->event_mutex);
		1027	id = KFD_FIRST_NONSIGNAL_EVENT_ID;
		1028	idr_for_each_entry_continue(&p->event_idr, ev, id)
		1029	if (ev->type == KFD_EVENT_TYPE_HW_EXCEPTION) {
		1030	ev->hw_exception_data = hw_exception_data;
		1031	set_event(ev);
		1032	}
		1033	mutex_unlock(&p->event_mutex);
		1034	}
		1035	srcu_read_unlock(&kfd_processes_srcu, idx);
		1036	}

drivers/gpu/drm/amd/amdkfd/kfd_events.h+1−0

@@ -66,6 +66,7 @@ struct kfd_event { · 65 unmodified lines
66	/* type specific data */	66	/* type specific data */
67	union {	67	union {
68	struct kfd_hsa_memory_exception_data memory_exception_data;	68	struct kfd_hsa_memory_exception_data memory_exception_data;
		69	struct kfd_hsa_hw_exception_data hw_exception_data;
69	};	70	};
70	};	71	};
71		72

drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c+19−3

 static bool event_interrupt_isr_v9(struct kfd_dev *dev,
-					const uint32_t *ih_ring_entry)
+					const uint32_t *ih_ring_entry,
+					uint32_t *patched_ihre,
+					bool *patched_flag)
 {
 	uint16_t source_id, client_id, pasid, vmid;
 	const uint32_t *data = ih_ring_entry;
 	return source_id == SOC15_INTSRC_CP_END_OF_PIPE ||
 		source_id == SOC15_INTSRC_SDMA_TRAP ||
 		source_id == SOC15_INTSRC_SQ_INTERRUPT_MSG ||
-		source_id == SOC15_INTSRC_CP_BAD_OPCODE;
+		source_id == SOC15_INTSRC_CP_BAD_OPCODE ||
+		client_id == SOC15_IH_CLIENTID_VMC ||
+		client_id == SOC15_IH_CLIENTID_UTCL2;
 }
 static void event_interrupt_wq_v9(struct kfd_dev *dev,
 		kfd_signal_hw_exception_event(pasid);
 	else if (client_id == SOC15_IH_CLIENTID_VMC ||
 		 client_id == SOC15_IH_CLIENTID_UTCL2) {
-		/* TODO */
+		struct kfd_vm_fault_info info = {0};
+		uint16_t ring_id = SOC15_RING_ID_FROM_IH_ENTRY(ih_ring_entry);
+		info.vmid = vmid;
+		info.mc_id = client_id;
+		info.page_addr = ih_ring_entry[4] |
+			(uint64_t)(ih_ring_entry[5] & 0xf) << 32;
+		info.prot_valid = ring_id & 0x08;
+		info.prot_read  = ring_id & 0x10;
+		info.prot_write = ring_id & 0x20;
+		kfd_process_vm_fault(dev->dqm, pasid);
+		kfd_signal_vm_fault_event(dev, pasid, &info);
 	}
 }

drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c+4−2

 								ih_ring_entry);
 }
-bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry)
+bool interrupt_is_wanted(struct kfd_dev *dev,
+			const uint32_t *ih_ring_entry,
+			uint32_t *patched_ihre, bool *flag)
 {
 	/* integer and bitwise OR so there is no boolean short-circuiting */
 	unsigned int wanted = 0;
 	wanted |= dev->device_info->event_interrupt_class->interrupt_isr(dev,
-								ih_ring_entry);
+					 ih_ring_entry, patched_ihre, flag);
 	return wanted != 0;
 }

drivers/gpu/drm/amd/amdkfd/kfd_iommu.c+1−1

@@ -190,7 +190,7 @@ static int iommu_invalid_ppr_cb(struct pci_dev *pdev, int pasid, · 189 unmodified lines
190	{	190	{
191	struct kfd_dev *dev;	191	struct kfd_dev *dev;
192		192
193	dev_warn(kfd_device,	193	dev_warn_ratelimited(kfd_device,
194	"Invalid PPR device %x:%x.%x pasid %d address 0x%lX flags 0x%X",	194	"Invalid PPR device %x:%x.%x pasid %d address 0x%lX flags 0x%X",
195	PCI_BUS_NUM(pdev->devfn),	195	PCI_BUS_NUM(pdev->devfn),
196	PCI_SLOT(pdev->devfn),	196	PCI_SLOT(pdev->devfn),

drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.c+10−8

 	switch (type) {
 	case KFD_QUEUE_TYPE_DIQ:
 	case KFD_QUEUE_TYPE_HIQ:
-		kq->mqd = dev->dqm->ops.get_mqd_manager(dev->dqm,
+		kq->mqd_mgr = dev->dqm->ops.get_mqd_manager(dev->dqm,
 						KFD_MQD_TYPE_HIQ);
 		break;
 	default:
 		return false;
 	}
-	if (!kq->mqd)
+	if (!kq->mqd_mgr)
 		return false;
 	prop.doorbell_ptr = kfd_get_kernel_doorbell(dev, &prop.doorbell_off);
 	prop.write_ptr = (uint32_t *) kq->wptr_gpu_addr;
 	prop.eop_ring_buffer_address = kq->eop_gpu_addr;
 	prop.eop_ring_buffer_size = PAGE_SIZE;
+	prop.cu_mask = NULL;
 	if (init_queue(&kq->queue, &prop) != 0)
 		goto err_init_queue;
 	kq->queue->device = dev;
 	kq->queue->process = kfd_get_process(current);
-	retval = kq->mqd->init_mqd(kq->mqd, &kq->queue->mqd,
+	retval = kq->mqd_mgr->init_mqd(kq->mqd_mgr, &kq->queue->mqd,
 					&kq->queue->mqd_mem_obj,
 					&kq->queue->gart_mqd_addr,
 					&kq->queue->properties);
 		pr_debug("Assigning hiq to hqd\n");
 		kq->queue->pipe = KFD_CIK_HIQ_PIPE;
 		kq->queue->queue = KFD_CIK_HIQ_QUEUE;
-		kq->mqd->load_mqd(kq->mqd, kq->queue->mqd, kq->queue->pipe,
+		kq->mqd_mgr->load_mqd(kq->mqd_mgr, kq->queue->mqd,
-				  kq->queue->queue, &kq->queue->properties,
+				kq->queue->pipe, kq->queue->queue,
-				  NULL);
+				&kq->queue->properties, NULL);
 	} else {
 		/* allocate fence for DIQ */
 static void uninitialize(struct kernel_queue *kq)
 {
 	if (kq->queue->properties.type == KFD_QUEUE_TYPE_HIQ)
-		kq->mqd->destroy_mqd(kq->mqd,
+		kq->mqd_mgr->destroy_mqd(kq->mqd_mgr,
 					kq->queue->mqd,
 					KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
 					KFD_UNMAP_LATENCY_MS,
 	else if (kq->queue->properties.type == KFD_QUEUE_TYPE_DIQ)
 		kfd_gtt_sa_free(kq->dev, kq->fence_mem_obj);
-	kq->mqd->uninit_mqd(kq->mqd, kq->queue->mqd, kq->queue->mqd_mem_obj);
+	kq->mqd_mgr->uninit_mqd(kq->mqd_mgr, kq->queue->mqd,
+				kq->queue->mqd_mem_obj);
 	kfd_gtt_sa_free(kq->dev, kq->rptr_mem);
 	kfd_gtt_sa_free(kq->dev, kq->wptr_mem);

drivers/gpu/drm/amd/amdkfd/kfd_kernel_queue.h+1−1

@@ -70,7 +70,7 @@ struct kernel_queue { · 69 unmodified lines
70		70
71	/* data */	71	/* data */
72	struct kfd_dev *dev;	72	struct kfd_dev *dev;
73	struct mqd_manager *mqd;	73	struct mqd_manager *mqd_mgr;
74	struct queue *queue;	74	struct queue *queue;
75	uint64_t pending_wptr64;	75	uint64_t pending_wptr64;
76	uint32_t pending_wptr;	76	uint32_t pending_wptr;

drivers/gpu/drm/amd/amdkfd/kfd_module.c+12−4

 	.resume_mm	= kgd2kfd_resume_mm,
 	.schedule_evict_and_restore_process =
 			  kgd2kfd_schedule_evict_and_restore_process,
+	.pre_reset	= kgd2kfd_pre_reset,
+	.post_reset	= kgd2kfd_post_reset,
 };
 int sched_policy = KFD_SCHED_POLICY_HWS;
 int cwsr_enable = 1;
 module_param(cwsr_enable, int, 0444);
-MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = Off, 1 = On (Default))");
+MODULE_PARM_DESC(cwsr_enable, "CWSR enable (0 = off, 1 = on (default))");
 int max_num_of_queues_per_device = KFD_MAX_NUM_OF_QUEUES_PER_DEVICE_DEFAULT;
 module_param(max_num_of_queues_per_device, int, 0444);
 MODULE_PARM_DESC(ignore_crat,
 	"Ignore CRAT table during KFD initialization (0 = use CRAT (default), 1 = ignore CRAT)");
-int vega10_noretry;
+int noretry;
-module_param_named(noretry, vega10_noretry, int, 0644);
+module_param(noretry, int, 0644);
 MODULE_PARM_DESC(noretry,
-	"Set sh_mem_config.retry_disable on Vega10 (0 = retry enabled (default), 1 = retry disabled)");
+	"Set sh_mem_config.retry_disable on GFXv9+ dGPUs (0 = retry enabled (default), 1 = retry disabled)");
+int halt_if_hws_hang;
+module_param(halt_if_hws_hang, int, 0644);
+MODULE_PARM_DESC(halt_if_hws_hang, "Halt if HWS hang is detected (0 = off (default), 1 = on)");
 static int amdkfd_init_completed;
 int kgd2kfd_init(unsigned int interface_version,
 		const struct kgd2kfd_calls **g2f)
 {

drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c+40−1

@@ -21,7 +21,7 @@ · 20 unmodified lines
21	*	21	*
22	*/	22	*/
23		23
24	#include "kfd_priv.h"	24	#include "kfd_mqd_manager.h"
25		25
26	struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,	26	struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type,
27	struct kfd_dev *dev)	27	struct kfd_dev *dev)
@@ -48,3 +48,42 @@ struct mqd_manager *mqd_manager_init(enum KFD_MQD_TYPE type, · 20 unmodified lines
48		48
49	return NULL;	49	return NULL;
50	}	50	}
		51
		52	void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm,
		53	const uint32_t *cu_mask, uint32_t cu_mask_count,
		54	uint32_t *se_mask)
		55	{
		56	struct kfd_cu_info cu_info;
		57	uint32_t cu_per_sh[4] = {0};
		58	int i, se, cu = 0;
		59
		60	mm->dev->kfd2kgd->get_cu_info(mm->dev->kgd, &cu_info);
		61
		62	if (cu_mask_count > cu_info.cu_active_number)
		63	cu_mask_count = cu_info.cu_active_number;
		64
		65	for (se = 0; se < cu_info.num_shader_engines; se++)
		66	for (i = 0; i < 4; i++)
		67	cu_per_sh[se] += hweight32(cu_info.cu_bitmap[se][i]);
		68
		69	/* Symmetrically map cu_mask to all SEs:
		70	* cu_mask[0] bit0 -> se_mask[0] bit0;
		71	* cu_mask[0] bit1 -> se_mask[1] bit0;
		72	* ... (if # SE is 4)
		73	* cu_mask[0] bit4 -> se_mask[0] bit1;
		74	* ...
		75	*/
		76	se = 0;
		77	for (i = 0; i < cu_mask_count; i++) {
		78	if (cu_mask[i / 32] & (1 << (i % 32)))
		79	se_mask[se] \|= 1 << cu;
		80
		81	do {
		82	se++;
		83	if (se == cu_info.num_shader_engines) {
		84	se = 0;
		85	cu++;
		86	}
		87	} while (cu >= cu_per_sh[se] && cu < 32);
		88	}
		89	}

drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.h+4−0

@@ -93,4 +93,8 @@ struct mqd_manager { · 92 unmodified lines
93	struct kfd_dev *dev;	93	struct kfd_dev *dev;
94	};	94	};
95		95
		96	void mqd_symmetrically_map_cu_mask(struct mqd_manager *mm,
		97	const uint32_t *cu_mask, uint32_t cu_mask_count,
		98	uint32_t *se_mask);
		99
96	#endif /* KFD_MQD_MANAGER_H_ */	100	#endif /* KFD_MQD_MANAGER_H_ */

drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_cik.c+28−1

@@ -41,6 +41,31 @@ static inline struct cik_sdma_rlc_registers get_sdma_mqd(void mqd) · 40 unmodified lines
41	return (struct cik_sdma_rlc_registers *)mqd;	41	return (struct cik_sdma_rlc_registers *)mqd;
42	}	42	}
43		43
		44	static void update_cu_mask(struct mqd_manager mm, void mqd,
		45	struct queue_properties *q)
		46	{
		47	struct cik_mqd *m;
		48	uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
		49
		50	if (q->cu_mask_count == 0)
		51	return;
		52
		53	mqd_symmetrically_map_cu_mask(mm,
		54	q->cu_mask, q->cu_mask_count, se_mask);
		55
		56	m = get_mqd(mqd);
		57	m->compute_static_thread_mgmt_se0 = se_mask[0];
		58	m->compute_static_thread_mgmt_se1 = se_mask[1];
		59	m->compute_static_thread_mgmt_se2 = se_mask[2];
		60	m->compute_static_thread_mgmt_se3 = se_mask[3];
		61
		62	pr_debug("Update cu mask to %#x %#x %#x %#x\n",
		63	m->compute_static_thread_mgmt_se0,
		64	m->compute_static_thread_mgmt_se1,
		65	m->compute_static_thread_mgmt_se2,
		66	m->compute_static_thread_mgmt_se3);
		67	}
		68
44	static int init_mqd(struct mqd_manager mm, void *mqd,	69	static int init_mqd(struct mqd_manager mm, void *mqd,
45	struct kfd_mem_obj *mqd_mem_obj, uint64_t gart_addr,	70	struct kfd_mem_obj *mqd_mem_obj, uint64_t gart_addr,
46	struct queue_properties *q)	71	struct queue_properties *q)
@@ -196,6 +221,8 @@ static int __update_mqd(struct mqd_manager mm, void mqd, · 149 unmodified lines
196	if (q->format == KFD_QUEUE_FORMAT_AQL)	221	if (q->format == KFD_QUEUE_FORMAT_AQL)
197	m->cp_hqd_pq_control \|= NO_UPDATE_RPTR;	222	m->cp_hqd_pq_control \|= NO_UPDATE_RPTR;
198		223
		224	update_cu_mask(mm, mqd, q);
		225
199	q->is_active = (q->queue_size > 0 &&	226	q->is_active = (q->queue_size > 0 &&
200	q->queue_address != 0 &&	227	q->queue_address != 0 &&
201	q->queue_percent > 0 &&	228	q->queue_percent > 0 &&
@@ -408,7 +435,7 @@ struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type, · 206 unmodified lines
408	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))	435	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
409	return NULL;	436	return NULL;
410		437
411	mqd = kzalloc(sizeof(*mqd), GFP_NOIO);	438	mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
412	if (!mqd)	439	if (!mqd)
413	return NULL;	440	return NULL;
414		441

drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c+29−2

@@ -41,6 +41,31 @@ static inline struct v9_sdma_mqd get_sdma_mqd(void mqd) · 40 unmodified lines
41	return (struct v9_sdma_mqd *)mqd;	41	return (struct v9_sdma_mqd *)mqd;
42	}	42	}
43		43
		44	static void update_cu_mask(struct mqd_manager mm, void mqd,
		45	struct queue_properties *q)
		46	{
		47	struct v9_mqd *m;
		48	uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
		49
		50	if (q->cu_mask_count == 0)
		51	return;
		52
		53	mqd_symmetrically_map_cu_mask(mm,
		54	q->cu_mask, q->cu_mask_count, se_mask);
		55
		56	m = get_mqd(mqd);
		57	m->compute_static_thread_mgmt_se0 = se_mask[0];
		58	m->compute_static_thread_mgmt_se1 = se_mask[1];
		59	m->compute_static_thread_mgmt_se2 = se_mask[2];
		60	m->compute_static_thread_mgmt_se3 = se_mask[3];
		61
		62	pr_debug("update cu mask to %#x %#x %#x %#x\n",
		63	m->compute_static_thread_mgmt_se0,
		64	m->compute_static_thread_mgmt_se1,
		65	m->compute_static_thread_mgmt_se2,
		66	m->compute_static_thread_mgmt_se3);
		67	}
		68
44	static int init_mqd(struct mqd_manager mm, void *mqd,	69	static int init_mqd(struct mqd_manager mm, void *mqd,
45	struct kfd_mem_obj *mqd_mem_obj, uint64_t gart_addr,	70	struct kfd_mem_obj *mqd_mem_obj, uint64_t gart_addr,
46	struct queue_properties *q)	71	struct queue_properties *q)
@@ -55,7 +80,7 @@ static int init_mqd(struct mqd_manager mm, void *mqd, · 8 unmodified lines
55	* instead of sub-allocation function.	80	* instead of sub-allocation function.
56	*/	81	*/
57	if (kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {	82	if (kfd->cwsr_enabled && (q->type == KFD_QUEUE_TYPE_COMPUTE)) {
58	*mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_NOIO);	83	*mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL);
59	if (!*mqd_mem_obj)	84	if (!*mqd_mem_obj)
60	return -ENOMEM;	85	return -ENOMEM;
61	retval = kfd->kfd2kgd->init_gtt_mem_allocation(kfd->kgd,	86	retval = kfd->kfd2kgd->init_gtt_mem_allocation(kfd->kgd,
@@ -198,6 +223,8 @@ static int update_mqd(struct mqd_manager mm, void mqd, · 136 unmodified lines
198	if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address)	223	if (mm->dev->cwsr_enabled && q->ctx_save_restore_area_address)
199	m->cp_hqd_ctx_save_control = 0;	224	m->cp_hqd_ctx_save_control = 0;
200		225
		226	update_cu_mask(mm, mqd, q);
		227
201	q->is_active = (q->queue_size > 0 &&	228	q->is_active = (q->queue_size > 0 &&
202	q->queue_address != 0 &&	229	q->queue_address != 0 &&
203	q->queue_percent > 0 &&	230	q->queue_percent > 0 &&
@@ -393,7 +420,7 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type, · 189 unmodified lines
393	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))	420	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
394	return NULL;	421	return NULL;
395		422
396	mqd = kzalloc(sizeof(*mqd), GFP_NOIO);	423	mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
397	if (!mqd)	424	if (!mqd)
398	return NULL;	425	return NULL;
399		426

drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_vi.c+28−1

@@ -43,6 +43,31 @@ static inline struct vi_sdma_mqd get_sdma_mqd(void mqd) · 42 unmodified lines
43	return (struct vi_sdma_mqd *)mqd;	43	return (struct vi_sdma_mqd *)mqd;
44	}	44	}
45		45
		46	static void update_cu_mask(struct mqd_manager mm, void mqd,
		47	struct queue_properties *q)
		48	{
		49	struct vi_mqd *m;
		50	uint32_t se_mask[4] = {0}; /* 4 is the max # of SEs */
		51
		52	if (q->cu_mask_count == 0)
		53	return;
		54
		55	mqd_symmetrically_map_cu_mask(mm,
		56	q->cu_mask, q->cu_mask_count, se_mask);
		57
		58	m = get_mqd(mqd);
		59	m->compute_static_thread_mgmt_se0 = se_mask[0];
		60	m->compute_static_thread_mgmt_se1 = se_mask[1];
		61	m->compute_static_thread_mgmt_se2 = se_mask[2];
		62	m->compute_static_thread_mgmt_se3 = se_mask[3];
		63
		64	pr_debug("Update cu mask to %#x %#x %#x %#x\n",
		65	m->compute_static_thread_mgmt_se0,
		66	m->compute_static_thread_mgmt_se1,
		67	m->compute_static_thread_mgmt_se2,
		68	m->compute_static_thread_mgmt_se3);
		69	}
		70
46	static int init_mqd(struct mqd_manager mm, void *mqd,	71	static int init_mqd(struct mqd_manager mm, void *mqd,
47	struct kfd_mem_obj *mqd_mem_obj, uint64_t gart_addr,	72	struct kfd_mem_obj *mqd_mem_obj, uint64_t gart_addr,
48	struct queue_properties *q)	73	struct queue_properties *q)
@@ -196,6 +221,8 @@ static int __update_mqd(struct mqd_manager mm, void mqd, · 147 unmodified lines
196	atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT \|	221	atc_bit << CP_HQD_CTX_SAVE_CONTROL__ATC__SHIFT \|
197	mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;	222	mtype << CP_HQD_CTX_SAVE_CONTROL__MTYPE__SHIFT;
198		223
		224	update_cu_mask(mm, mqd, q);
		225
199	q->is_active = (q->queue_size > 0 &&	226	q->is_active = (q->queue_size > 0 &&
200	q->queue_address != 0 &&	227	q->queue_address != 0 &&
201	q->queue_percent > 0 &&	228	q->queue_percent > 0 &&
@@ -394,7 +421,7 @@ struct mqd_manager *mqd_manager_init_vi(enum KFD_MQD_TYPE type, · 192 unmodified lines
394	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))	421	if (WARN_ON(type >= KFD_MQD_TYPE_MAX))
395	return NULL;	422	return NULL;
396		423
397	mqd = kzalloc(sizeof(*mqd), GFP_NOIO);	424	mqd = kzalloc(sizeof(*mqd), GFP_KERNEL);
398	if (!mqd)	425	if (!mqd)
399	return NULL;	426	return NULL;
400		427

drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c+26−0

@@ -418,4 +418,30 @@ out: · 417 unmodified lines
418	return 0;	418	return 0;
419	}	419	}
420		420
		421	int pm_debugfs_hang_hws(struct packet_manager *pm)
		422	{
		423	uint32_t *buffer, size;
		424	int r = 0;
		425
		426	size = pm->pmf->query_status_size;
		427	mutex_lock(&pm->lock);
		428	pm->priv_queue->ops.acquire_packet_buffer(pm->priv_queue,
		429	size / sizeof(uint32_t), (unsigned int **)&buffer);
		430	if (!buffer) {
		431	pr_err("Failed to allocate buffer on kernel queue\n");
		432	r = -ENOMEM;
		433	goto out;
		434	}
		435	memset(buffer, 0x55, size);
		436	pm->priv_queue->ops.submit_packet(pm->priv_queue);
		437
		438	pr_info("Submitting %x %x %x %x %x %x %x to HIQ to hang the HWS.",
		439	buffer[0], buffer[1], buffer[2], buffer[3],
		440	buffer[4], buffer[5], buffer[6]);
		441	out:
		442	mutex_unlock(&pm->lock);
		443	return r;
		444	}
		445
		446
421	#endif	447	#endif

drivers/gpu/drm/amd/amdkfd/kfd_priv.h+35−5

 /*
  * When working with cp scheduler we should assign the HIQ manually or via
- * the radeon driver to a fixed hqd slot, here are the fixed HIQ hqd slot
+ * the amdgpu driver to a fixed hqd slot, here are the fixed HIQ hqd slot
  * definitions for Kaveri. In Kaveri only the first ME queues participates
  * in the cp scheduling taking that in mind we set the HIQ slot in the
  * second ME.
 /*
  * Set sh_mem_config.retry_disable on Vega10
  */
-extern int vega10_noretry;
+extern int noretry;
+/*
+ * Halt if HWS hang is detected
+ */
+extern int halt_if_hws_hang;
 /**
  * enum kfd_sched_policy
 struct kfd_event_interrupt_class {
 	bool (*interrupt_isr)(struct kfd_dev *dev,
-				const uint32_t *ih_ring_entry);
+			const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
+			bool *patched_flag);
 	void (*interrupt_wq)(struct kfd_dev *dev,
-				const uint32_t *ih_ring_entry);
+			const uint32_t *ih_ring_entry);
 };
 struct kfd_device_info {
 	bool supports_cwsr;
 	bool needs_iommu_device;
 	bool needs_pci_atomics;
+	unsigned int num_sdma_engines;
 };
 struct kfd_mem_obj {
 	uint32_t ctl_stack_size;
 	uint64_t tba_addr;
 	uint64_t tma_addr;
+	/* Relevant for CU */
+	uint32_t cu_mask_count; /* Must be a multiple of 32 */
+	uint32_t *cu_mask;
 };
 /**
 void kfd_interrupt_exit(struct kfd_dev *dev);
 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);
 bool enqueue_ih_ring_entry(struct kfd_dev *kfd,	const void *ih_ring_entry);
-bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry);
+bool interrupt_is_wanted(struct kfd_dev *dev,
+				const uint32_t *ih_ring_entry,
+				uint32_t *patched_ihre, bool *flag);
 /* Power Management */
 void kgd2kfd_suspend(struct kfd_dev *kfd);
 int kgd2kfd_resume(struct kfd_dev *kfd);
+/* GPU reset */
+int kgd2kfd_pre_reset(struct kfd_dev *kfd);
+int kgd2kfd_post_reset(struct kfd_dev *kfd);
 /* amdkfd Apertures */
 int kfd_init_apertures(struct kfd_process *process);
 struct kernel_queue *kernel_queue_init(struct kfd_dev *dev,
 					enum kfd_queue_type type);
 void kernel_queue_uninit(struct kernel_queue *kq);
+int kfd_process_vm_fault(struct device_queue_manager *dqm, unsigned int pasid);
 /* Process Queue Manager */
 struct process_queue_node {
 int pqm_destroy_queue(struct process_queue_manager *pqm, unsigned int qid);
 int pqm_update_queue(struct process_queue_manager *pqm, unsigned int qid,
 			struct queue_properties *p);
+int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
+			struct queue_properties *p);
 struct kernel_queue *pqm_get_kernel_queue(struct process_queue_manager *pqm,
 						unsigned int qid);
 		     uint64_t *event_page_offset, uint32_t *event_slot_index);
 int kfd_event_destroy(struct kfd_process *p, uint32_t event_id);
+void kfd_signal_vm_fault_event(struct kfd_dev *dev, unsigned int pasid,
+				struct kfd_vm_fault_info *info);
+void kfd_signal_reset_event(struct kfd_dev *dev);
 void kfd_flush_tlb(struct kfd_process_device *pdd);
 int dbgdev_wave_reset_wavefronts(struct kfd_dev *dev, struct kfd_process *p);
+bool kfd_is_locked(void);
 /* Debugfs */
 #if defined(CONFIG_DEBUG_FS)
 int kfd_debugfs_rls_by_device(struct seq_file *m, void *data);
 int pm_debugfs_runlist(struct seq_file *m, void *data);
+int kfd_debugfs_hang_hws(struct kfd_dev *dev);
+int pm_debugfs_hang_hws(struct packet_manager *pm);
+int dqm_debugfs_execute_queues(struct device_queue_manager *dqm);
 #else
 static inline void kfd_debugfs_init(void) {}

drivers/gpu/drm/amd/amdkfd/kfd_process.c+2−0

@@ -244,6 +244,8 @@ struct kfd_process kfd_get_process(const struct task_struct thread) · 243 unmodified lines
244	return ERR_PTR(-EINVAL);	244	return ERR_PTR(-EINVAL);
245		245
246	process = find_process(thread);	246	process = find_process(thread);
		247	if (!process)
		248	return ERR_PTR(-EINVAL);
247		249
248	return process;	250	return process;
249	}	251	}

drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c+36−7

 	switch (type) {
 	case KFD_QUEUE_TYPE_SDMA:
-		if (dev->dqm->queue_count >=
+		if (dev->dqm->queue_count >= get_num_sdma_queues(dev->dqm)) {
-			CIK_SDMA_QUEUES_PER_ENGINE * CIK_SDMA_ENGINE_NUM) {
 			pr_err("Over-subscription is not allowed for SDMA.\n");
 			retval = -EPERM;
 			goto err_create_queue;
 		     KFD_SCHED_POLICY_HWS_NO_OVERSUBSCRIPTION) &&
 		((dev->dqm->processes_count >= dev->vm_info.vmid_num_kfd) ||
 		(dev->dqm->queue_count >= get_queues_num(dev->dqm)))) {
-			pr_err("Over-subscription is not allowed in radeon_kfd.sched_policy == 1\n");
+			pr_debug("Over-subscription is not allowed when amdkfd.sched_policy == 1\n");
 			retval = -EPERM;
 			goto err_create_queue;
 		}
 			if (retval != -ETIME)
 				goto err_destroy_queue;
 		}
+		kfree(pqn->q->properties.cu_mask);
+		pqn->q->properties.cu_mask = NULL;
 		uninit_queue(pqn->q);
 	}
 	return 0;
 }
+int pqm_set_cu_mask(struct process_queue_manager *pqm, unsigned int qid,
+			struct queue_properties *p)
+{
+	int retval;
+	struct process_queue_node *pqn;
+	pqn = get_queue_by_qid(pqm, qid);
+	if (!pqn) {
+		pr_debug("No queue %d exists for update operation\n", qid);
+		return -EFAULT;
+	}
+	/* Free the old CU mask memory if it is already allocated, then
+	 * allocate memory for the new CU mask.
+	 */
+	kfree(pqn->q->properties.cu_mask);
+	pqn->q->properties.cu_mask_count = p->cu_mask_count;
+	pqn->q->properties.cu_mask = p->cu_mask;
+	retval = pqn->q->device->dqm->ops.update_queue(pqn->q->device->dqm,
+							pqn->q);
+	if (retval != 0)
+		return retval;
+	return 0;
+}
 struct kernel_queue *pqm_get_kernel_queue(
 					struct process_queue_manager *pqm,
 					unsigned int qid)
 	struct process_queue_node *pqn;
 	struct queue *q;
 	enum KFD_MQD_TYPE mqd_type;
-	struct mqd_manager *mqd_manager;
+	struct mqd_manager *mqd_mgr;
 	int r = 0;
 	list_for_each_entry(pqn, &pqm->queues, process_queue_list) {
 					   q->properties.type, q->device->id);
 				continue;
 			}
-			mqd_manager = q->device->dqm->ops.get_mqd_manager(
+			mqd_mgr = q->device->dqm->ops.get_mqd_manager(
 				q->device->dqm, mqd_type);
 		} else if (pqn->kq) {
 			q = pqn->kq->queue;
-			mqd_manager = pqn->kq->mqd;
+			mqd_mgr = pqn->kq->mqd_mgr;
 			switch (q->properties.type) {
 			case KFD_QUEUE_TYPE_DIQ:
 				seq_printf(m, "  DIQ on device %x\n",
 			continue;
 		}
-		r = mqd_manager->debugfs_show_mqd(m, q->mqd);
+		r = mqd_mgr->debugfs_show_mqd(m, q->mqd);
 		if (r != 0)
 			break;
 	}

drivers/gpu/drm/amd/include/kgd_kfd_interface.h+40−0

@@ -47,6 +47,17 @@ enum kfd_preempt_type { · 46 unmodified lines
47	KFD_PREEMPT_TYPE_WAVEFRONT_RESET,	47	KFD_PREEMPT_TYPE_WAVEFRONT_RESET,
48	};	48	};
49		49
		50	struct kfd_vm_fault_info {
		51	uint64_t page_addr;
		52	uint32_t vmid;
		53	uint32_t mc_id;
		54	uint32_t status;
		55	bool prot_valid;
		56	bool prot_read;
		57	bool prot_write;
		58	bool prot_exec;
		59	};
		60
50	struct kfd_cu_info {	61	struct kfd_cu_info {
51	uint32_t num_shader_engines;	62	uint32_t num_shader_engines;
52	uint32_t num_shader_arrays_per_engine;	63	uint32_t num_shader_arrays_per_engine;
@@ -259,6 +270,21 @@ struct tile_config { · 206 unmodified lines
259	* IB to the corresponding ring (ring type). The IB is executed with the	270	* IB to the corresponding ring (ring type). The IB is executed with the
260	* specified VMID in a user mode context.	271	* specified VMID in a user mode context.
261	*	272	*
		273	* @get_vm_fault_info: Return information about a recent VM fault on
		274	* GFXv7 and v8. If multiple VM faults occurred since the last call of
		275	* this function, it will return information about the first of those
		276	* faults. On GFXv9 VM fault information is fully contained in the IH
		277	* packet and this function is not needed.
		278	*
		279	* @read_vmid_from_vmfault_reg: On Hawaii the VMID is not set in the
		280	* IH ring entry. This function allows the KFD ISR to get the VMID
		281	* from the fault status register as early as possible.
		282	*
		283	* @gpu_recover: let kgd reset gpu after kfd detect CPC hang
		284	*
		285	* @set_compute_idle: Indicates that compute is idle on a device. This
		286	* can be used to change power profiles depending on compute activity.
		287	*
262	* This structure contains function pointers to services that the kgd driver	288	* This structure contains function pointers to services that the kgd driver
263	* provides to amdkfd driver.	289	* provides to amdkfd driver.
264	*	290	*
@@ -374,6 +400,14 @@ struct kfd2kgd_calls { · 109 unmodified lines
374	int (submit_ib)(struct kgd_dev kgd, enum kgd_engine_type engine,	400	int (submit_ib)(struct kgd_dev kgd, enum kgd_engine_type engine,
375	uint32_t vmid, uint64_t gpu_addr,	401	uint32_t vmid, uint64_t gpu_addr,
376	uint32_t *ib_cmd, uint32_t ib_len);	402	uint32_t *ib_cmd, uint32_t ib_len);
		403
		404	int (get_vm_fault_info)(struct kgd_dev kgd,
		405	struct kfd_vm_fault_info *info);
		406	uint32_t (read_vmid_from_vmfault_reg)(struct kgd_dev kgd);
		407
		408	void (gpu_recover)(struct kgd_dev kgd);
		409
		410	void (set_compute_idle)(struct kgd_dev kgd, bool idle);
377	};	411	};
378		412
379	/**	413	/**
@@ -399,6 +433,10 @@ struct kfd2kgd_calls { · 19 unmodified lines
399	* @schedule_evict_and_restore_process: Schedules work queue that will prepare	433	* @schedule_evict_and_restore_process: Schedules work queue that will prepare
400	* for safe eviction of KFD BOs that belong to the specified process.	434	* for safe eviction of KFD BOs that belong to the specified process.
401	*	435	*
		436	* @pre_reset: Notifies amdkfd that amdgpu about to reset the gpu
		437	*
		438	* @post_reset: Notify amdkfd that amgpu successfully reseted the gpu
		439	*
402	* This structure contains function callback pointers so the kgd driver	440	* This structure contains function callback pointers so the kgd driver
403	* will notify to the amdkfd about certain status changes.	441	* will notify to the amdkfd about certain status changes.
404	*	442	*
@@ -417,6 +455,8 @@ struct kgd2kfd_calls { · 12 unmodified lines
417	int (resume_mm)(struct mm_struct mm);	455	int (resume_mm)(struct mm_struct mm);
418	int (schedule_evict_and_restore_process)(struct mm_struct mm,	456	int (schedule_evict_and_restore_process)(struct mm_struct mm,
419	struct dma_fence *fence);	457	struct dma_fence *fence);
		458	int (pre_reset)(struct kfd_dev kfd);
		459	int (post_reset)(struct kfd_dev kfd);
420	};	460	};
421		461
422	int kgd2kfd_init(unsigned interface_version,	462	int kgd2kfd_init(unsigned interface_version,

include/uapi/linux/kfd_ioctl.h+30−3

 	__u32 queue_priority;	/* to KFD */
 };
+struct kfd_ioctl_set_cu_mask_args {
+	__u32 queue_id;		/* to KFD */
+	__u32 num_cu_mask;		/* to KFD */
+	__u64 cu_mask_ptr;		/* to KFD */
+};
 /* For kfd_ioctl_set_memory_policy_args.default_policy and alternate_policy */
 #define KFD_IOC_CACHE_POLICY_COHERENT 0
 #define KFD_IOC_CACHE_POLICY_NONCOHERENT 1
 #define KFD_SIGNAL_EVENT_LIMIT			4096
+/* For kfd_event_data.hw_exception_data.reset_type. */
+#define KFD_HW_EXCEPTION_WHOLE_GPU_RESET	0
+#define KFD_HW_EXCEPTION_PER_ENGINE_RESET	1
+/* For kfd_event_data.hw_exception_data.reset_cause. */
+#define KFD_HW_EXCEPTION_GPU_HANG	0
+#define KFD_HW_EXCEPTION_ECC		1
 struct kfd_ioctl_create_event_args {
 	__u64 event_page_offset;	/* from KFD */
 	__u32 event_trigger_data;	/* from KFD - signal events only */
 	__u32 NotPresent;	/* Page not present or supervisor privilege */
 	__u32 ReadOnly;	/* Write access to a read-only page */
 	__u32 NoExecute;	/* Execute access to a page marked NX */
-	__u32 pad;
+	__u32 imprecise;	/* Can't determine the	exact fault address */
 };
 /* memory exception data*/
 	__u32 pad;
 };
-/* Event data*/
+/* hw exception data */
+struct kfd_hsa_hw_exception_data {
+	uint32_t reset_type;
+	uint32_t reset_cause;
+	uint32_t memory_lost;
+	uint32_t gpu_id;
+};
+/* Event data */
 struct kfd_event_data {
 	union {
 		struct kfd_hsa_memory_exception_data memory_exception_data;
+		struct kfd_hsa_hw_exception_data hw_exception_data;
 	};				/* From KFD */
 	__u64 kfd_event_data_ext;	/* pointer to an extension structure
 					   for future exception types */
 #define AMDKFD_IOC_UNMAP_MEMORY_FROM_GPU	\
 		AMDKFD_IOWR(0x19, struct kfd_ioctl_unmap_memory_from_gpu_args)
+#define AMDKFD_IOC_SET_CU_MASK		\
+		AMDKFD_IOW(0x1A, struct kfd_ioctl_set_cu_mask_args)
 #define AMDKFD_COMMAND_START		0x01
-#define AMDKFD_COMMAND_END		0x1A
+#define AMDKFD_COMMAND_END		0x1B
 #endif

Merge tag 'drm-amdkfd-next-2018-07-28' of git://people.freedesktop.org/~gabbayo/linux into drm-next

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code