lael-labs/linux

MAINTAINERS+2−2

@@ -878,7 +878,7 @@ M: Harry Wentland <harry.wentland@amd.com> · 877 unmodified lines
878	M: Leo Li <sunpeng.li@amd.com>	878	M: Leo Li <sunpeng.li@amd.com>
879	L: amd-gfx@lists.freedesktop.org	879	L: amd-gfx@lists.freedesktop.org
880	S: Supported	880	S: Supported
881	T: git git://people.freedesktop.org/~agd5f/linux	881	T: git https://gitlab.freedesktop.org/agd5f/linux.git
882	F: drivers/gpu/drm/amd/display/	882	F: drivers/gpu/drm/amd/display/
883		883
884	AMD FAM15H PROCESSOR POWER MONITORING DRIVER	884	AMD FAM15H PROCESSOR POWER MONITORING DRIVER
@@ -954,7 +954,7 @@ AMD POWERPLAY · 69 unmodified lines
954	M: Evan Quan <evan.quan@amd.com>	954	M: Evan Quan <evan.quan@amd.com>
955	L: amd-gfx@lists.freedesktop.org	955	L: amd-gfx@lists.freedesktop.org
956	S: Supported	956	S: Supported
957	T: git git://people.freedesktop.org/~agd5f/linux	957	T: git https://gitlab.freedesktop.org/agd5f/linux.git
958	F: drivers/gpu/drm/amd/pm/powerplay/	958	F: drivers/gpu/drm/amd/pm/powerplay/
959		959
960	AMD SEATTLE DEVICE TREE SUPPORT	960	AMD SEATTLE DEVICE TREE SUPPORT

drivers/gpu/drm/amd/amdgpu/Makefile+1−1

@@ -56,7 +56,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \ · 55 unmodified lines
56	amdgpu_gmc.o amdgpu_mmhub.o amdgpu_xgmi.o amdgpu_csa.o amdgpu_ras.o amdgpu_vm_cpu.o \	56	amdgpu_gmc.o amdgpu_mmhub.o amdgpu_xgmi.o amdgpu_csa.o amdgpu_ras.o amdgpu_vm_cpu.o \
57	amdgpu_vm_sdma.o amdgpu_discovery.o amdgpu_ras_eeprom.o amdgpu_nbio.o \	57	amdgpu_vm_sdma.o amdgpu_discovery.o amdgpu_ras_eeprom.o amdgpu_nbio.o \
58	amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o \	58	amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o \
59	amdgpu_fw_attestation.o amdgpu_securedisplay.o	59	amdgpu_fw_attestation.o amdgpu_securedisplay.o amdgpu_hdp.o
60		60
61	amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o	61	amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o
62		62

drivers/gpu/drm/amd/amdgpu/amdgpu.h+2−1

@@ -1075,7 +1075,8 @@ struct amdgpu_device { · 1074 unmodified lines
1075		1075
1076	atomic_t throttling_logging_enabled;	1076	atomic_t throttling_logging_enabled;
1077	struct ratelimit_state throttling_logging_rs;	1077	struct ratelimit_state throttling_logging_rs;
1078	uint32_t ras_features;	1078	uint32_t ras_hw_enabled;
		1079	uint32_t ras_enabled;
1079		1080
1080	bool in_pci_err_recovery;	1081	bool in_pci_err_recovery;
1081	struct pci_saved_state *pci_state;	1082	struct pci_saved_state *pci_state;

drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c+6−6

 /**
  * amdgpu_atif_call - call an ATIF method
  *
- * @handle: acpi handle
+ * @atif: acpi handle
  * @function: the ATIF function to execute
  * @params: ATIF function params
  *
 /**
  * amdgpu_atif_verify_interface - verify ATIF
  *
- * @handle: acpi handle
  * @atif: amdgpu atif struct
  *
  * Execute the ATIF_FUNCTION_VERIFY_INTERFACE ATIF function
 /**
  * amdgpu_atif_get_notification_params - determine notify configuration
  *
- * @handle: acpi handle
+ * @atif: acpi handle
- * @n: atif notification configuration struct
  *
  * Execute the ATIF_FUNCTION_GET_SYSTEM_PARAMETERS ATIF function
  * to determine if a notifier is used and if so which one
 /**
  * amdgpu_atif_query_backlight_caps - get min and max backlight input signal
  *
- * @handle: acpi handle
+ * @atif: acpi handle
  *
  * Execute the QUERY_BRIGHTNESS_TRANSFER_CHARACTERISTICS ATIF function
  * to determine the acceptable range of backlight values
 /**
  * amdgpu_atif_get_sbios_requests - get requested sbios event
  *
- * @handle: acpi handle
+ * @atif: acpi handle
  * @req: atif sbios request struct
  *
  * Execute the ATIF_FUNCTION_GET_SYSTEM_BIOS_REQUESTS ATIF function
 /**
  * amdgpu_acpi_is_s0ix_supported
  *
+ * @adev: amdgpu_device_pointer
+ *
  * returns true if supported, false if not.
  */
 bool amdgpu_acpi_is_s0ix_supported(struct amdgpu_device *adev)

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h+17−9

 	struct mm_struct *mm;
 	spinlock_t lock;
 	char timeline_name[TASK_COMM_LEN];
+	struct svm_range_bo *svm_bo;
 };
 struct amdgpu_kfd_dev {
 					int queue_bit);
 struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
-								struct mm_struct *mm);
+				struct mm_struct *mm,
+				struct svm_range_bo *svm_bo);
 #if IS_ENABLED(CONFIG_HSA_AMD)
 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm);
 struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f);
 	})
 /* GPUVM API */
+#define drm_priv_to_vm(drm_priv)					\
+	(&((struct amdgpu_fpriv *)					\
+		((struct drm_file *)(drm_priv))->driver_priv)->vm)
 int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
 					struct file *filp, u32 pasid,
-					void **vm, void **process_info,
+					void **process_info,
 					struct dma_fence **ef);
-void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *vm);
+void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *drm_priv);
-uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
+uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv);
 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 		struct kgd_dev *kgd, uint64_t va, uint64_t size,
-		void *vm, struct kgd_mem **mem,
+		void *drm_priv, struct kgd_mem **mem,
 		uint64_t *offset, uint32_t flags);
 int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
-		struct kgd_dev *kgd, struct kgd_mem *mem, uint64_t *size);
+		struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv,
+		uint64_t *size);
 int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
-		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
+		struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv);
 int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
-		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm);
+		struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv);
 int amdgpu_amdkfd_gpuvm_sync_memory(
 		struct kgd_dev *kgd, struct kgd_mem *mem, bool intr);
 int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
 					      struct kfd_vm_fault_info *info);
 int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
 				      struct dma_buf *dmabuf,
-				      uint64_t va, void *vm,
+				      uint64_t va, void *drm_priv,
 				      struct kgd_mem **mem, uint64_t *size,
 				      uint64_t *mmap_offset);
 int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd,
 void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
 				struct amdgpu_vm *vm);
 void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo);
+void amdgpu_amdkfd_reserve_system_mem(uint64_t size);
 #else
 static inline
 void amdgpu_amdkfd_gpuvm_init_mem_limits(void)

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c+1−0

@@ -25,6 +25,7 @@ · 24 unmodified lines
25	#include <linux/firmware.h>	25	#include <linux/firmware.h>
26	#include "amdgpu.h"	26	#include "amdgpu.h"
27	#include "amdgpu_amdkfd.h"	27	#include "amdgpu_amdkfd.h"
		28	#include "amdgpu_amdkfd_arcturus.h"
28	#include "sdma0/sdma0_4_2_2_offset.h"	29	#include "sdma0/sdma0_4_2_2_offset.h"
29	#include "sdma0/sdma0_4_2_2_sh_mask.h"	30	#include "sdma0/sdma0_4_2_2_sh_mask.h"
30	#include "sdma1/sdma1_4_2_2_offset.h"	31	#include "sdma1/sdma1_4_2_2_offset.h"

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c+14−6

@@ -28,6 +28,7 @@ · 27 unmodified lines
28	#include <linux/slab.h>	28	#include <linux/slab.h>
29	#include <linux/sched/mm.h>	29	#include <linux/sched/mm.h>
30	#include "amdgpu_amdkfd.h"	30	#include "amdgpu_amdkfd.h"
		31	#include "kfd_svm.h"
31		32
32	static const struct dma_fence_ops amdkfd_fence_ops;	33	static const struct dma_fence_ops amdkfd_fence_ops;
33	static atomic_t fence_seq = ATOMIC_INIT(0);	34	static atomic_t fence_seq = ATOMIC_INIT(0);
@@ -60,7 +61,8 @@ static atomic_t fence_seq = ATOMIC_INIT(0); · 26 unmodified lines
60	*/	61	*/
61		62
62	struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,	63	struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
63	struct mm_struct *mm)	64	struct mm_struct *mm,
		65	struct svm_range_bo *svm_bo)
64	{	66	{
65	struct amdgpu_amdkfd_fence *fence;	67	struct amdgpu_amdkfd_fence *fence;
66		68
@@ -73,7 +75,7 @@ struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, · 6 unmodified lines
73	fence->mm = mm;	75	fence->mm = mm;
74	get_task_comm(fence->timeline_name, current);	76	get_task_comm(fence->timeline_name, current);
75	spin_lock_init(&fence->lock);	77	spin_lock_init(&fence->lock);
76		78	fence->svm_bo = svm_bo;
77	dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,	79	dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,
78	context, atomic_inc_return(&fence_seq));	80	context, atomic_inc_return(&fence_seq));
79		81
@@ -111,6 +113,8 @@ static const char amdkfd_fence_get_timeline_name(struct dma_fence f) · 31 unmodified lines
111	* a KFD BO and schedules a job to move the BO.	113	* a KFD BO and schedules a job to move the BO.
112	* If fence is already signaled return true.	114	* If fence is already signaled return true.
113	* If fence is not signaled schedule a evict KFD process work item.	115	* If fence is not signaled schedule a evict KFD process work item.
		116	*
		117	* @f: dma_fence
114	*/	118	*/
115	static bool amdkfd_fence_enable_signaling(struct dma_fence *f)	119	static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
116	{	120	{
@@ -122,16 +126,20 @@ static bool amdkfd_fence_enable_signaling(struct dma_fence *f) · 5 unmodified lines
122	if (dma_fence_is_signaled(f))	126	if (dma_fence_is_signaled(f))
123	return true;	127	return true;
124		128
125	if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f))	129	if (!fence->svm_bo) {
126	return true;	130	if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f))
127		131	return true;
		132	} else {
		133	if (!svm_range_schedule_evict_svm_bo(fence))
		134	return true;
		135	}
128	return false;	136	return false;
129	}	137	}
130		138
131	/**	139	/**
132	* amdkfd_fence_release - callback that fence can be freed	140	* amdkfd_fence_release - callback that fence can be freed
133	*	141	*
134	* @fence: fence	142	* @f: dma_fence
135	*	143	*
136	* This function is called when the reference count becomes zero.	144	* This function is called when the reference count becomes zero.
137	* Drops the mm_struct reference and RCU schedules freeing up the fence.	145	* Drops the mm_struct reference and RCU schedules freeing up the fence.

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c+4−4

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c+62−36

 #include <uapi/linux/kfd_ioctl.h>
 #include "amdgpu_xgmi.h"
-/* BO flag to indicate a KFD userptr BO */
-#define AMDGPU_AMDKFD_USERPTR_BO (1ULL << 63)
 /* Userptr restore delay, just long enough to allow consecutive VM
  * changes to accumulate
  */
 		(kfd_mem_limit.max_ttm_mem_limit >> 20));
 }
+void amdgpu_amdkfd_reserve_system_mem(uint64_t size)
+{
+	kfd_mem_limit.system_mem_used += size;
+}
 /* Estimate page table size needed to represent a given memory size
  *
  * With 4KB pages, we need one 8 byte PTE for each 4KB of memory
 	u32 domain = bo->preferred_domains;
 	bool sg = (bo->preferred_domains == AMDGPU_GEM_DOMAIN_CPU);
-	if (bo->flags & AMDGPU_AMDKFD_USERPTR_BO) {
+	if (bo->flags & AMDGPU_AMDKFD_CREATE_USERPTR_BO) {
 		domain = AMDGPU_GEM_DOMAIN_CPU;
 		sg = false;
 	}
 		info->eviction_fence =
 			amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
-						   current->mm);
+						   current->mm,
+						   NULL);
 		if (!info->eviction_fence) {
 			pr_err("Failed to create eviction fence\n");
 			ret = -ENOMEM;
 int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
 					   struct file *filp, u32 pasid,
-					   void **vm, void **process_info,
+					   void **process_info,
 					   struct dma_fence **ef)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
-	struct drm_file *drm_priv = filp->private_data;
+	struct amdgpu_fpriv *drv_priv;
-	struct amdgpu_fpriv *drv_priv = drm_priv->driver_priv;
+	struct amdgpu_vm *avm;
-	struct amdgpu_vm *avm = &drv_priv->vm;
 	int ret;
+	ret = amdgpu_file_to_fpriv(filp, &drv_priv);
+	if (ret)
+		return ret;
+	avm = &drv_priv->vm;
 	/* Already a compute VM? */
 	if (avm->process_info)
 		return -EINVAL;
 	if (ret)
 		return ret;
-	*vm = (void *)avm;
+	amdgpu_vm_set_task_info(avm);
 	return 0;
 }
 	}
 }
-void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *vm)
+void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev *kgd, void *drm_priv)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
-	struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
+	struct amdgpu_vm *avm;
-	if (WARN_ON(!kgd || !vm))
+	if (WARN_ON(!kgd || !drm_priv))
 		return;
-	pr_debug("Releasing process vm %p\n", vm);
+	avm = drm_priv_to_vm(drm_priv);
+	pr_debug("Releasing process vm %p\n", avm);
 	/* The original pasid of amdgpu vm has already been
 	 * released during making a amdgpu vm to a compute vm
 	amdgpu_vm_release_compute(adev, avm);
 }
-uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm)
+uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv)
 {
-	struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
+	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
 	struct amdgpu_bo *pd = avm->root.base.bo;
 	struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
 int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 		struct kgd_dev *kgd, uint64_t va, uint64_t size,
-		void *vm, struct kgd_mem **mem,
+		void *drm_priv, struct kgd_mem **mem,
 		uint64_t *offset, uint32_t flags)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
-	struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
+	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
 	enum ttm_bo_type bo_type = ttm_bo_type_device;
 	struct sg_table *sg = NULL;
 	uint64_t user_addr = 0;
 			 domain_string(alloc_domain), ret);
 		goto err_bo_create;
 	}
+	ret = drm_vma_node_allow(&gobj->vma_node, drm_priv);
+	if (ret) {
+		pr_debug("Failed to allow vma node access. ret %d\n", ret);
+		goto err_node_allow;
+	}
 	bo = gem_to_amdgpu_bo(gobj);
 	if (bo_type == ttm_bo_type_sg) {
 		bo->tbo.sg = sg;
 	bo->kfd_bo = *mem;
 	(*mem)->bo = bo;
 	if (user_addr)
-		bo->flags |= AMDGPU_AMDKFD_USERPTR_BO;
+		bo->flags |= AMDGPU_AMDKFD_CREATE_USERPTR_BO;
 	(*mem)->va = va;
 	(*mem)->domain = domain;
 allocate_init_user_pages_failed:
 	remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
+	drm_vma_node_revoke(&gobj->vma_node, drm_priv);
+err_node_allow:
 	amdgpu_bo_unref(&bo);
 	/* Don't unreserve system mem limit twice */
 	goto err_reserve_limit;
 }
 int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
-		struct kgd_dev *kgd, struct kgd_mem *mem, uint64_t *size)
+		struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv,
+		uint64_t *size)
 {
 	struct amdkfd_process_info *process_info = mem->process_info;
 	unsigned long bo_size = mem->bo->tbo.base.size;
 	}
 	/* Free the BO*/
+	drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv);
 	drm_gem_object_put(&mem->bo->tbo.base);
 	mutex_destroy(&mem->lock);
 	kfree(mem);
 }
 int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
-		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm)
+		struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
-	struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
+	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
 	int ret;
 	struct amdgpu_bo *bo;
 	uint32_t domain;
 	pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",
 			mem->va,
 			mem->va + bo_size * (1 + mem->aql_queue),
-			vm, domain_string(domain));
+			avm, domain_string(domain));
-	ret = reserve_bo_and_vm(mem, vm, &ctx);
+	ret = reserve_bo_and_vm(mem, avm, &ctx);
 	if (unlikely(ret))
 		goto out;
 	}
 	list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
-		if (entry->bo_va->base.vm == vm && !entry->is_mapped) {
+		if (entry->bo_va->base.vm == avm && !entry->is_mapped) {
 			pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
 					entry->va, entry->va + bo_size,
 					entry);
 				goto map_bo_to_gpuvm_failed;
 			}
-			ret = vm_update_pds(vm, ctx.sync);
+			ret = vm_update_pds(avm, ctx.sync);
 			if (ret) {
 				pr_err("Failed to update page directories\n");
 				goto map_bo_to_gpuvm_failed;
 }
 int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
-		struct kgd_dev *kgd, struct kgd_mem *mem, void *vm)
+		struct kgd_dev *kgd, struct kgd_mem *mem, void *drm_priv)
 {
 	struct amdgpu_device *adev = get_amdgpu_device(kgd);
-	struct amdkfd_process_info *process_info =
+	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
-		((struct amdgpu_vm *)vm)->process_info;
+	struct amdkfd_process_info *process_info = avm->process_info;
 	unsigned long bo_size = mem->bo->tbo.base.size;
 	struct kfd_bo_va_list *entry;
 	struct bo_vm_reservation_context ctx;
 	mutex_lock(&mem->lock);
-	ret = reserve_bo_and_cond_vms(mem, vm, BO_VM_MAPPED, &ctx);
+	ret = reserve_bo_and_cond_vms(mem, avm, BO_VM_MAPPED, &ctx);
 	if (unlikely(ret))
 		goto out;
 	/* If no VMs were reserved, it means the BO wasn't actually mapped */
 		goto unreserve_out;
 	}
-	ret = vm_validate_pt_pd_bos((struct amdgpu_vm *)vm);
+	ret = vm_validate_pt_pd_bos(avm);
 	if (unlikely(ret))
 		goto unreserve_out;
 	pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",
 		mem->va,
 		mem->va + bo_size * (1 + mem->aql_queue),
-		vm);
+		avm);
 	list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
-		if (entry->bo_va->base.vm == vm && entry->is_mapped) {
+		if (entry->bo_va->base.vm == avm && entry->is_mapped) {
 			pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
 					entry->va,
 					entry->va + bo_size,
 int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
 				      struct dma_buf *dma_buf,
-				      uint64_t va, void *vm,
+				      uint64_t va, void *drm_priv,
 				      struct kgd_mem **mem, uint64_t *size,
 				      uint64_t *mmap_offset)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)kgd;
+	struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
 	struct drm_gem_object *obj;
 	struct amdgpu_bo *bo;
-	struct amdgpu_vm *avm = (struct amdgpu_vm *)vm;
+	int ret;
 	if (dma_buf->ops != &amdgpu_dmabuf_ops)
 		/* Can't handle non-graphics buffers */
 	if (!*mem)
 		return -ENOMEM;
+	ret = drm_vma_node_allow(&obj->vma_node, drm_priv);
+	if (ret) {
+		kfree(mem);
+		return ret;
+	}
 	if (size)
 		*size = amdgpu_bo_size(bo);
 	 */
 	new_fence = amdgpu_amdkfd_fence_create(
 				process_info->eviction_fence->base.context,
-				process_info->eviction_fence->mm);
+				process_info->eviction_fence->mm,
+				NULL);
 	if (!new_fence) {
 		pr_err("Failed to create eviction fence\n");
 		ret = -ENOMEM;

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c+3−3

@@ -672,7 +672,7 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) · 671 unmodified lines
672	}	672	}
673		673
674	/**	674	/**
675	* cs_parser_fini() - clean parser states	675	* amdgpu_cs_parser_fini() - clean parser states
676	* @parser: parser structure holding parsing context.	676	* @parser: parser structure holding parsing context.
677	* @error: error number	677	* @error: error number
678	* @backoff: indicator to backoff the reservation	678	* @backoff: indicator to backoff the reservation
@@ -1488,7 +1488,7 @@ int amdgpu_cs_fence_to_handle_ioctl(struct drm_device dev, void data, · 809 unmodified lines
1488	}	1488	}
1489		1489
1490	/**	1490	/**
1491	* amdgpu_cs_wait_all_fence - wait on all fences to signal	1491	* amdgpu_cs_wait_all_fences - wait on all fences to signal
1492	*	1492	*
1493	* @adev: amdgpu device	1493	* @adev: amdgpu device
1494	* @filp: file private	1494	* @filp: file private
@@ -1639,7 +1639,7 @@ err_free_fences: · 144 unmodified lines
1639	}	1639	}
1640		1640
1641	/**	1641	/**
1642	* amdgpu_cs_find_bo_va - find bo_va for VM address	1642	* amdgpu_cs_find_mapping - find bo_va for VM address
1643	*	1643	*
1644	* @parser: command submission parser context	1644	* @parser: command submission parser context
1645	* @addr: VM address	1645	* @addr: VM address

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c+10−11

 		AMD_IP_BLOCK_TYPE_IH,
 	};
-	for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
+	for (i = 0; i < adev->num_ip_blocks; i++) {
 		int j;
 		struct amdgpu_ip_block *block;
 	int ret = 0;
 	/*
-	 * By default timeout for non compute jobs is 10000.
+	 * By default timeout for non compute jobs is 10000
-	 * And there is no timeout enforced on compute jobs.
+	 * and 60000 for compute jobs.
 	 * In SR-IOV or passthrough mode, timeout for compute
 	 * jobs are 60000 by default.
 	 */
 	if (amdgpu_sriov_vf(adev))
 		adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
 					msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
-	else if (amdgpu_passthrough(adev))
-		adev->compute_timeout =  msecs_to_jiffies(60000);
 	else
-		adev->compute_timeout = MAX_SCHEDULE_TIMEOUT;
+		adev->compute_timeout =  msecs_to_jiffies(60000);
 	if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
 		while ((timeout_setting = strsep(&input, ",")) &&
 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
 {
 	struct amdgpu_device *adev = drm_to_adev(dev);
-	int r;
 	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
 		return 0;
 	amdgpu_ras_suspend(adev);
-	r = amdgpu_device_ip_suspend_phase1(adev);
+	amdgpu_device_ip_suspend_phase1(adev);
 	if (!adev->in_s0ix)
 		amdgpu_amdkfd_suspend(adev, adev->in_runpm);
 	amdgpu_fence_driver_suspend(adev);
-	r = amdgpu_device_ip_suspend_phase2(adev);
+	amdgpu_device_ip_suspend_phase2(adev);
 	/* evict remaining vram memory
 	 * This second call to evict vram is to evict the gart page table
 	 * using the CPU.
 	if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
 		return -ENOTSUPP;
-	if (ras && ras->supported && adev->nbio.funcs->enable_doorbell_interrupt)
+	if (ras && adev->ras_enabled &&
+	    adev->nbio.funcs->enable_doorbell_interrupt)
 		adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
 	return amdgpu_dpm_baco_enter(adev);
 	if (ret)
 		return ret;
-	if (ras && ras->supported && adev->nbio.funcs->enable_doorbell_interrupt)
+	if (ras && adev->ras_enabled &&
+	    adev->nbio.funcs->enable_doorbell_interrupt)
 		adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
 	return 0;

drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c+9−9

  *   for SDMA and Video.
  *
  * By default(with no lockup_timeout settings), the timeout for all non-compute(GFX, SDMA and Video)
- * jobs is 10000. And there is no timeout enforced on compute jobs.
+ * jobs is 10000. The timeout for compute is 60000.
  */
-MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms (default: for bare metal 10000 for non-compute jobs and infinity timeout for compute jobs; "
+MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms (default: for bare metal 10000 for non-compute jobs and 60000 for compute jobs; "
 		"for passthrough or sriov, 10000 for all jobs."
 		" 0: keep default value. negative: infinity timeout), "
 		"format: for bare metal [Non-Compute] or [GFX,Compute,SDMA,Video]; "
 /**
  * DOC: noretry (int)
- * Disable retry faults in the GPU memory controller.
+ * Disable XNACK retry in the SQ by default on GFXv9 hardware. On ASICs that
+ * do not support per-process XNACK this also disables retry page faults.
  * (0 = retry enabled, 1 = retry disabled, -1 auto (default))
  */
 MODULE_PARM_DESC(noretry,
 	{0x1002, 0x7408, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ALDEBARAN|AMD_EXP_HW_SUPPORT},
 	{0x1002, 0x740C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ALDEBARAN|AMD_EXP_HW_SUPPORT},
 	{0x1002, 0x740F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ALDEBARAN|AMD_EXP_HW_SUPPORT},
+	{0x1002, 0x7410, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ALDEBARAN|AMD_EXP_HW_SUPPORT},
 	{0, 0, 0}
 };
 	if (amdgpu_device_has_dc_support(adev)) {
 		struct drm_crtc *crtc;
-		drm_modeset_lock_all(drm_dev);
 		drm_for_each_crtc(crtc, drm_dev) {
-			if (crtc->state->active) {
+			drm_modeset_lock(&crtc->mutex, NULL);
+			if (crtc->state->active)
 				ret = -EBUSY;
+			drm_modeset_unlock(&crtc->mutex);
+			if (ret < 0)
 				break;
-			}
 		}
-		drm_modeset_unlock_all(drm_dev);
 	} else {
 		struct drm_connector *list_connector;
 		struct drm_connector_list_iter iter;

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c+1−0

@@ -434,6 +434,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, · 433 unmodified lines
434	*	434	*
435	* @ring: ring to init the fence driver on	435	* @ring: ring to init the fence driver on
436	* @num_hw_submission: number of entries on the hardware queue	436	* @num_hw_submission: number of entries on the hardware queue
		437	* @sched_score: optional score atomic shared with other schedulers
437	*	438	*
438	* Init the fence driver for the requested ring (all asics).	439	* Init the fence driver for the requested ring (all asics).
439	* Helper function for amdgpu_fence_driver_init().	440	* Helper function for amdgpu_fence_driver_init().

drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c+2−2

@@ -60,7 +60,7 @@ · 59 unmodified lines
60	*/	60	*/
61		61
62	/**	62	/**
63	* amdgpu_dummy_page_init - init dummy page used by the driver	63	* amdgpu_gart_dummy_page_init - init dummy page used by the driver
64	*	64	*
65	* @adev: amdgpu_device pointer	65	* @adev: amdgpu_device pointer
66	*	66	*
@@ -86,7 +86,7 @@ static int amdgpu_gart_dummy_page_init(struct amdgpu_device *adev) · 19 unmodified lines
86	}	86	}
87		87
88	/**	88	/**
89	* amdgpu_dummy_page_fini - free dummy page used by the driver	89	* amdgpu_gart_dummy_page_fini - free dummy page used by the driver
90	*	90	*
91	* @adev: amdgpu_device pointer	91	* @adev: amdgpu_device pointer
92	*	92	*

drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c+56−5

@@ -332,6 +332,17 @@ void amdgpu_gmc_agp_location(struct amdgpu_device adev, struct amdgpu_gmc mc) · 331 unmodified lines
332	mc->agp_size >> 20, mc->agp_start, mc->agp_end);	332	mc->agp_size >> 20, mc->agp_start, mc->agp_end);
333	}	333	}
334		334
		335	/**
		336	* amdgpu_gmc_fault_key - get hask key from vm fault address and pasid
		337	*
		338	* @addr: 48 bit physical address, page aligned (36 significant bits)
		339	* @pasid: 16 bit process address space identifier
		340	*/
		341	static inline uint64_t amdgpu_gmc_fault_key(uint64_t addr, uint16_t pasid)
		342	{
		343	return addr << 4 \| pasid;
		344	}
		345
335	/**	346	/**
336	* amdgpu_gmc_filter_faults - filter VM faults	347	* amdgpu_gmc_filter_faults - filter VM faults
337	*	348	*
@@ -348,8 +359,7 @@ bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr, · 10 unmodified lines
348	uint16_t pasid, uint64_t timestamp)	359	uint16_t pasid, uint64_t timestamp)
349	{	360	{
350	struct amdgpu_gmc *gmc = &adev->gmc;	361	struct amdgpu_gmc *gmc = &adev->gmc;
351		362	uint64_t stamp, key = amdgpu_gmc_fault_key(addr, pasid);
352	uint64_t stamp, key = addr << 4 \| pasid;
353	struct amdgpu_gmc_fault *fault;	363	struct amdgpu_gmc_fault *fault;
354	uint32_t hash;	364	uint32_t hash;
355		365
@@ -365,7 +375,7 @@ bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr, · 9 unmodified lines
365	while (fault->timestamp >= stamp) {	375	while (fault->timestamp >= stamp) {
366	uint64_t tmp;	376	uint64_t tmp;
367		377
368	if (fault->key == key)	378	if (atomic64_read(&fault->key) == key)
369	return true;	379	return true;
370		380
371	tmp = fault->timestamp;	381	tmp = fault->timestamp;
@@ -378,7 +388,7 @@ bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr, · 6 unmodified lines
378		388
379	/* Add the fault to the ring */	389	/* Add the fault to the ring */
380	fault = &gmc->fault_ring[gmc->last_fault];	390	fault = &gmc->fault_ring[gmc->last_fault];
381	fault->key = key;	391	atomic64_set(&fault->key, key);
382	fault->timestamp = timestamp;	392	fault->timestamp = timestamp;
383		393
384	/* And update the hash */	394	/* And update the hash */
@@ -387,6 +397,36 @@ bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr, · 2 unmodified lines
387	return false;	397	return false;
388	}	398	}
389		399
		400	/**
		401	* amdgpu_gmc_filter_faults_remove - remove address from VM faults filter
		402	*
		403	* @adev: amdgpu device structure
		404	* @addr: address of the VM fault
		405	* @pasid: PASID of the process causing the fault
		406	*
		407	* Remove the address from fault filter, then future vm fault on this address
		408	* will pass to retry fault handler to recover.
		409	*/
		410	void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr,
		411	uint16_t pasid)
		412	{
		413	struct amdgpu_gmc *gmc = &adev->gmc;
		414	uint64_t key = amdgpu_gmc_fault_key(addr, pasid);
		415	struct amdgpu_gmc_fault *fault;
		416	uint32_t hash;
		417	uint64_t tmp;
		418
		419	hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER);
		420	fault = &gmc->fault_ring[gmc->fault_hash[hash].idx];
		421	do {
		422	if (atomic64_cmpxchg(&fault->key, key, 0) == key)
		423	break;
		424
		425	tmp = fault->timestamp;
		426	fault = &gmc->fault_ring[fault->next];
		427	} while (fault->timestamp < tmp);
		428	}
		429
390	int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev)	430	int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev)
391	{	431	{
392	int r;	432	int r;
@@ -415,6 +455,13 @@ int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev) · 22 unmodified lines
415	return r;	455	return r;
416	}	456	}
417		457
		458	if (adev->hdp.ras_funcs &&
		459	adev->hdp.ras_funcs->ras_late_init) {
		460	r = adev->hdp.ras_funcs->ras_late_init(adev);
		461	if (r)
		462	return r;
		463	}
		464
418	return 0;	465	return 0;
419	}	466	}
420		467
@@ -426,11 +473,15 @@ void amdgpu_gmc_ras_fini(struct amdgpu_device *adev) · 5 unmodified lines
426		473
427	if (adev->mmhub.ras_funcs &&	474	if (adev->mmhub.ras_funcs &&
428	adev->mmhub.ras_funcs->ras_fini)	475	adev->mmhub.ras_funcs->ras_fini)
429	amdgpu_mmhub_ras_fini(adev);	476	adev->mmhub.ras_funcs->ras_fini(adev);
430		477
431	if (adev->gmc.xgmi.ras_funcs &&	478	if (adev->gmc.xgmi.ras_funcs &&
432	adev->gmc.xgmi.ras_funcs->ras_fini)	479	adev->gmc.xgmi.ras_funcs->ras_fini)
433	adev->gmc.xgmi.ras_funcs->ras_fini(adev);	480	adev->gmc.xgmi.ras_funcs->ras_fini(adev);
		481
		482	if (adev->hdp.ras_funcs &&
		483	adev->hdp.ras_funcs->ras_fini)
		484	adev->hdp.ras_funcs->ras_fini(adev);
434	}	485	}
435		486
436	/*	487	/*

drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h+4−2

  * GMC page fault information
  */
 struct amdgpu_gmc_fault {
-	uint64_t	timestamp;
+	uint64_t	timestamp:48;
 	uint64_t	next:AMDGPU_GMC_FAULT_RING_ORDER;
-	uint64_t	key:52;
+	atomic64_t	key;
 };
 /*
 			     struct amdgpu_gmc *mc);
 bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr,
 			      uint16_t pasid, uint64_t timestamp);
+void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr,
+				     uint16_t pasid);
 int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev);
 void amdgpu_gmc_ras_fini(struct amdgpu_device *adev);
 int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev);

drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c+90−79

 #include "amdgpu.h"
-static inline struct amdgpu_gtt_mgr *to_gtt_mgr(struct ttm_resource_manager *man)
+static inline struct amdgpu_gtt_mgr *
+to_gtt_mgr(struct ttm_resource_manager *man)
 {
 	return container_of(man, struct amdgpu_gtt_mgr, manager);
 }
  * the GTT block, in bytes
  */
 static ssize_t amdgpu_mem_info_gtt_total_show(struct device *dev,
-		struct device_attribute *attr, char *buf)
+					      struct device_attribute *attr,
+					      char *buf)
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = drm_to_adev(ddev);
-	struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
+	struct ttm_resource_manager *man;
+	man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
 	return sysfs_emit(buf, "%llu\n", man->size * PAGE_SIZE);
 }
  * size of the GTT block, in bytes
  */
 static ssize_t amdgpu_mem_info_gtt_used_show(struct device *dev,
-		struct device_attribute *attr, char *buf)
+					     struct device_attribute *attr,
+					     char *buf)
 {
 	struct drm_device *ddev = dev_get_drvdata(dev);
 	struct amdgpu_device *adev = drm_to_adev(ddev);
-	struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
+	struct ttm_resource_manager *man;
+	man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
 	return sysfs_emit(buf, "%llu\n", amdgpu_gtt_mgr_usage(man));
 }
 static DEVICE_ATTR(mem_info_gtt_used, S_IRUGO,
 	           amdgpu_mem_info_gtt_used_show, NULL);
-static const struct ttm_resource_manager_func amdgpu_gtt_mgr_func;
-/**
- * amdgpu_gtt_mgr_init - init GTT manager and DRM MM
- *
- * @adev: amdgpu_device pointer
- * @gtt_size: maximum size of GTT
- *
- * Allocate and initialize the GTT manager.
- */
-int amdgpu_gtt_mgr_init(struct amdgpu_device *adev, uint64_t gtt_size)
-{
-	struct amdgpu_gtt_mgr *mgr = &adev->mman.gtt_mgr;
-	struct ttm_resource_manager *man = &mgr->manager;
-	uint64_t start, size;
-	int ret;
-	man->use_tt = true;
-	man->func = &amdgpu_gtt_mgr_func;
-	ttm_resource_manager_init(man, gtt_size >> PAGE_SHIFT);
-	start = AMDGPU_GTT_MAX_TRANSFER_SIZE * AMDGPU_GTT_NUM_TRANSFER_WINDOWS;
-	size = (adev->gmc.gart_size >> PAGE_SHIFT) - start;
-	drm_mm_init(&mgr->mm, start, size);
-	spin_lock_init(&mgr->lock);
-	atomic64_set(&mgr->available, gtt_size >> PAGE_SHIFT);
-	ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_total);
-	if (ret) {
-		DRM_ERROR("Failed to create device file mem_info_gtt_total\n");
-		return ret;
-	}
-	ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_used);
-	if (ret) {
-		DRM_ERROR("Failed to create device file mem_info_gtt_used\n");
-		return ret;
-	}
-	ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_TT, &mgr->manager);
-	ttm_resource_manager_set_used(man, true);
-	return 0;
-}
-/**
- * amdgpu_gtt_mgr_fini - free and destroy GTT manager
- *
- * @adev: amdgpu_device pointer
- *
- * Destroy and free the GTT manager, returns -EBUSY if ranges are still
- * allocated inside it.
- */
-void amdgpu_gtt_mgr_fini(struct amdgpu_device *adev)
-{
-	struct amdgpu_gtt_mgr *mgr = &adev->mman.gtt_mgr;
-	struct ttm_resource_manager *man = &mgr->manager;
-	int ret;
-	ttm_resource_manager_set_used(man, false);
-	ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
-	if (ret)
-		return;
-	spin_lock(&mgr->lock);
-	drm_mm_takedown(&mgr->mm);
-	spin_unlock(&mgr->lock);
-	device_remove_file(adev->dev, &dev_attr_mem_info_gtt_total);
-	device_remove_file(adev->dev, &dev_attr_mem_info_gtt_used);
-	ttm_resource_manager_cleanup(man);
-	ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_TT, NULL);
-}
 /**
  * amdgpu_gtt_mgr_has_gart_addr - Check if mem has address space
  *
 	return (result > 0 ? result : 0) * PAGE_SIZE;
 }
+/**
+ * amdgpu_gtt_mgr_recover - re-init gart
+ *
+ * @man: TTM memory type manager
+ *
+ * Re-init the gart for each known BO in the GTT.
+ */
 int amdgpu_gtt_mgr_recover(struct ttm_resource_manager *man)
 {
 	struct amdgpu_gtt_mgr *mgr = to_gtt_mgr(man);
 	.free = amdgpu_gtt_mgr_del,
 	.debug = amdgpu_gtt_mgr_debug
 };
+/**
+ * amdgpu_gtt_mgr_init - init GTT manager and DRM MM
+ *
+ * @adev: amdgpu_device pointer
+ * @gtt_size: maximum size of GTT
+ *
+ * Allocate and initialize the GTT manager.
+ */
+int amdgpu_gtt_mgr_init(struct amdgpu_device *adev, uint64_t gtt_size)
+{
+	struct amdgpu_gtt_mgr *mgr = &adev->mman.gtt_mgr;
+	struct ttm_resource_manager *man = &mgr->manager;
+	uint64_t start, size;
+	int ret;
+	man->use_tt = true;
+	man->func = &amdgpu_gtt_mgr_func;
+	ttm_resource_manager_init(man, gtt_size >> PAGE_SHIFT);
+	start = AMDGPU_GTT_MAX_TRANSFER_SIZE * AMDGPU_GTT_NUM_TRANSFER_WINDOWS;
+	size = (adev->gmc.gart_size >> PAGE_SHIFT) - start;
+	drm_mm_init(&mgr->mm, start, size);
+	spin_lock_init(&mgr->lock);
+	atomic64_set(&mgr->available, gtt_size >> PAGE_SHIFT);
+	ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_total);
+	if (ret) {
+		DRM_ERROR("Failed to create device file mem_info_gtt_total\n");
+		return ret;
+	}
+	ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_used);
+	if (ret) {
+		DRM_ERROR("Failed to create device file mem_info_gtt_used\n");
+		return ret;
+	}
+	ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_TT, &mgr->manager);
+	ttm_resource_manager_set_used(man, true);
+	return 0;
+}
+/**
+ * amdgpu_gtt_mgr_fini - free and destroy GTT manager
+ *
+ * @adev: amdgpu_device pointer
+ *
+ * Destroy and free the GTT manager, returns -EBUSY if ranges are still
+ * allocated inside it.
+ */
+void amdgpu_gtt_mgr_fini(struct amdgpu_device *adev)
+{
+	struct amdgpu_gtt_mgr *mgr = &adev->mman.gtt_mgr;
+	struct ttm_resource_manager *man = &mgr->manager;
+	int ret;
+	ttm_resource_manager_set_used(man, false);
+	ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
+	if (ret)
+		return;
+	spin_lock(&mgr->lock);
+	drm_mm_takedown(&mgr->mm);
+	spin_unlock(&mgr->lock);
+	device_remove_file(adev->dev, &dev_attr_mem_info_gtt_total);
+	device_remove_file(adev->dev, &dev_attr_mem_info_gtt_used);
+	ttm_resource_manager_cleanup(man);
+	ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_TT, NULL);
+}

drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.c+69−0

@@ -0,0 +1,69 @@
	1	/*
	2	* Copyright 2021 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*
	22	*/
	23
	24	#include "amdgpu.h"
	25	#include "amdgpu_ras.h"
	26
	27	int amdgpu_hdp_ras_late_init(struct amdgpu_device *adev)
	28	{
	29	int r;
	30	struct ras_ih_if ih_info = {
	31	.cb = NULL,
	32	};
	33	struct ras_fs_if fs_info = {
	34	.sysfs_name = "hdp_err_count",
	35	};
	36
	37	if (!adev->hdp.ras_if) {
	38	adev->hdp.ras_if = kmalloc(sizeof(struct ras_common_if), GFP_KERNEL);
	39	if (!adev->hdp.ras_if)
	40	return -ENOMEM;
	41	adev->hdp.ras_if->block = AMDGPU_RAS_BLOCK__HDP;
	42	adev->hdp.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
	43	adev->hdp.ras_if->sub_block_index = 0;
	44	strcpy(adev->hdp.ras_if->name, "hdp");
	45	}
	46	ih_info.head = fs_info.head = *adev->hdp.ras_if;
	47	r = amdgpu_ras_late_init(adev, adev->hdp.ras_if,
	48	&fs_info, &ih_info);
	49	if (r \|\| !amdgpu_ras_is_supported(adev, adev->hdp.ras_if->block)) {
	50	kfree(adev->hdp.ras_if);
	51	adev->hdp.ras_if = NULL;
	52	}
	53
	54	return r;
	55	}
	56
	57	void amdgpu_hdp_ras_fini(struct amdgpu_device *adev)
	58	{
	59	if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP) &&
	60	adev->hdp.ras_if) {
	61	struct ras_common_if *ras_if = adev->hdp.ras_if;
	62	struct ras_ih_if ih_info = {
	63	.cb = NULL,
	64	};
	65
	66	amdgpu_ras_late_fini(adev, ras_if, &ih_info);
	67	kfree(ras_if);
	68	}
	69	}

drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h+12−1

 #ifndef __AMDGPU_HDP_H__
 #define __AMDGPU_HDP_H__
+struct amdgpu_hdp_ras_funcs {
+	int (*ras_late_init)(struct amdgpu_device *adev);
+	void (*ras_fini)(struct amdgpu_device *adev);
+	void (*query_ras_error_count)(struct amdgpu_device *adev,
+				      void *ras_error_status);
+	void (*reset_ras_error_count)(struct amdgpu_device *adev);
+};
 struct amdgpu_hdp_funcs {
 	void (*flush_hdp)(struct amdgpu_device *adev, struct amdgpu_ring *ring);
 	void (*invalidate_hdp)(struct amdgpu_device *adev,
 			       struct amdgpu_ring *ring);
-	void (*reset_ras_error_count)(struct amdgpu_device *adev);
 	void (*update_clock_gating)(struct amdgpu_device *adev, bool enable);
 	void (*get_clock_gating_state)(struct amdgpu_device *adev, u32 *flags);
 	void (*init_registers)(struct amdgpu_device *adev);
 };
 struct amdgpu_hdp {
+	struct ras_common_if			*ras_if;
 	const struct amdgpu_hdp_funcs		*funcs;
+	const struct amdgpu_hdp_ras_funcs	*ras_funcs;
 };
+int amdgpu_hdp_ras_late_init(struct amdgpu_device *adev);
+void amdgpu_hdp_ras_fini(struct amdgpu_device *adev);
 #endif /* __AMDGPU_HDP_H__ */

drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c+1−1

@@ -328,7 +328,7 @@ int amdgpu_ib_pool_init(struct amdgpu_device *adev) · 327 unmodified lines
328		328
329	for (i = 0; i < AMDGPU_IB_POOL_MAX; i++) {	329	for (i = 0; i < AMDGPU_IB_POOL_MAX; i++) {
330	if (i == AMDGPU_IB_POOL_DIRECT)	330	if (i == AMDGPU_IB_POOL_DIRECT)
331	size = PAGE_SIZE * 2;	331	size = PAGE_SIZE * 6;
332	else	332	else
333	size = AMDGPU_IB_POOL_SIZE;	333	size = AMDGPU_IB_POOL_SIZE;
334		334

drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c+3−1

@@ -175,7 +175,9 @@ static bool amdgpu_ih_has_checkpoint_processed(struct amdgpu_device *adev, · 174 unmodified lines
175	cur_rptr += ih->ptr_mask + 1;	175	cur_rptr += ih->ptr_mask + 1;
176	*prev_rptr = cur_rptr;	176	*prev_rptr = cur_rptr;
177		177
178	return cur_rptr >= checkpoint_wptr;	178	/* check ring is empty to workaround missing wptr overflow flag */
		179	return cur_rptr >= checkpoint_wptr \|\|
		180	(cur_rptr & ih->ptr_mask) == amdgpu_ih_get_wptr(adev, ih);
179	}	181	}
180		182
181	/**	183	/**

drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c+3−2

@@ -986,7 +986,7 @@ int amdgpu_info_ioctl(struct drm_device dev, void data, struct drm_file *filp) · 985 unmodified lines
986		986
987	if (!ras)	987	if (!ras)
988	return -EINVAL;	988	return -EINVAL;
989	ras_mask = (uint64_t)ras->supported << 32 \| ras->features;	989	ras_mask = (uint64_t)adev->ras_enabled << 32 \| ras->features;
990		990
991	return copy_to_user(out, &ras_mask,	991	return copy_to_user(out, &ras_mask,
992	min_t(u64, size, sizeof(ras_mask))) ?	992	min_t(u64, size, sizeof(ras_mask))) ?
@@ -1114,7 +1114,8 @@ int amdgpu_driver_open_kms(struct drm_device dev, struct drm_file file_priv) · 121 unmodified lines
1114	dev_warn(adev->dev, "No more PASIDs available!");	1114	dev_warn(adev->dev, "No more PASIDs available!");
1115	pasid = 0;	1115	pasid = 0;
1116	}	1116	}
1117	r = amdgpu_vm_init(adev, &fpriv->vm, AMDGPU_VM_CONTEXT_GFX, pasid);	1117
		1118	r = amdgpu_vm_init(adev, &fpriv->vm, pasid);
1118	if (r)	1119	if (r)
1119	goto error_pasid;	1120	goto error_pasid;
1120		1121

drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h+1−0

@@ -28,6 +28,7 @@ struct amdgpu_mmhub_ras_funcs { · 27 unmodified lines
28	void *ras_error_status);	28	void *ras_error_status);
29	void (query_ras_error_status)(struct amdgpu_device adev);	29	void (query_ras_error_status)(struct amdgpu_device adev);
30	void (reset_ras_error_count)(struct amdgpu_device adev);	30	void (reset_ras_error_count)(struct amdgpu_device adev);
		31	void (reset_ras_error_status)(struct amdgpu_device adev);
31	};	32	};
32		33
33	struct amdgpu_mmhub_funcs {	34	struct amdgpu_mmhub_funcs {

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c+86−0

@@ -155,3 +155,89 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo) · 154 unmodified lines
155	mmu_interval_notifier_remove(&bo->notifier);	155	mmu_interval_notifier_remove(&bo->notifier);
156	bo->notifier.mm = NULL;	156	bo->notifier.mm = NULL;
157	}	157	}
		158
		159	int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier,
		160	struct mm_struct mm, struct page *pages,
		161	uint64_t start, uint64_t npages,
		162	struct hmm_range **phmm_range, bool readonly,
		163	bool mmap_locked)
		164	{
		165	struct hmm_range *hmm_range;
		166	unsigned long timeout;
		167	unsigned long i;
		168	unsigned long *pfns;
		169	int r = 0;
		170
		171	hmm_range = kzalloc(sizeof(*hmm_range), GFP_KERNEL);
		172	if (unlikely(!hmm_range))
		173	return -ENOMEM;
		174
		175	pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
		176	if (unlikely(!pfns)) {
		177	r = -ENOMEM;
		178	goto out_free_range;
		179	}
		180
		181	hmm_range->notifier = notifier;
		182	hmm_range->default_flags = HMM_PFN_REQ_FAULT;
		183	if (!readonly)
		184	hmm_range->default_flags \|= HMM_PFN_REQ_WRITE;
		185	hmm_range->hmm_pfns = pfns;
		186	hmm_range->start = start;
		187	hmm_range->end = start + npages * PAGE_SIZE;
		188
		189	/* Assuming 512MB takes maxmium 1 second to fault page address */
		190	timeout = max(npages >> 17, 1ULL) * HMM_RANGE_DEFAULT_TIMEOUT;
		191	timeout = jiffies + msecs_to_jiffies(timeout);
		192
		193	retry:
		194	hmm_range->notifier_seq = mmu_interval_read_begin(notifier);
		195
		196	if (likely(!mmap_locked))
		197	mmap_read_lock(mm);
		198
		199	r = hmm_range_fault(hmm_range);
		200
		201	if (likely(!mmap_locked))
		202	mmap_read_unlock(mm);
		203	if (unlikely(r)) {
		204	/*
		205	* FIXME: This timeout should encompass the retry from
		206	* mmu_interval_read_retry() as well.
		207	*/
		208	if (r == -EBUSY && !time_after(jiffies, timeout))
		209	goto retry;
		210	goto out_free_pfns;
		211	}
		212
		213	/*
		214	* Due to default_flags, all pages are HMM_PFN_VALID or
		215	* hmm_range_fault() fails. FIXME: The pages cannot be touched outside
		216	* the notifier_lock, and mmu_interval_read_retry() must be done first.
		217	*/
		218	for (i = 0; pages && i < npages; i++)
		219	pages[i] = hmm_pfn_to_page(pfns[i]);
		220
		221	*phmm_range = hmm_range;
		222
		223	return 0;
		224
		225	out_free_pfns:
		226	kvfree(pfns);
		227	out_free_range:
		228	kfree(hmm_range);
		229
		230	return r;
		231	}
		232
		233	int amdgpu_hmm_range_get_pages_done(struct hmm_range *hmm_range)
		234	{
		235	int r;
		236
		237	r = mmu_interval_read_retry(hmm_range->notifier,
		238	hmm_range->notifier_seq);
		239	kvfree(hmm_range->hmm_pfns);
		240	kfree(hmm_range);
		241
		242	return r;
		243	}

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h+7−0

@@ -30,6 +30,13 @@ · 29 unmodified lines
30	#include <linux/workqueue.h>	30	#include <linux/workqueue.h>
31	#include <linux/interval_tree.h>	31	#include <linux/interval_tree.h>
32		32
		33	int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier,
		34	struct mm_struct mm, struct page *pages,
		35	uint64_t start, uint64_t npages,
		36	struct hmm_range **phmm_range, bool readonly,
		37	bool mmap_locked);
		38	int amdgpu_hmm_range_get_pages_done(struct hmm_range *hmm_range);
		39
33	#if defined(CONFIG_HMM_MIRROR)	40	#if defined(CONFIG_HMM_MIRROR)
34	int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);	41	int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
35	void amdgpu_mn_unregister(struct amdgpu_bo *bo);	42	void amdgpu_mn_unregister(struct amdgpu_bo *bo);

drivers/gpu/drm/amd/amdgpu/amdgpu_object.c+18−54

 #endif
 }
-static int amdgpu_bo_do_create(struct amdgpu_device *adev,
+/**
+ * amdgpu_bo_create - create an &amdgpu_bo buffer object
+ * @adev: amdgpu device object
+ * @bp: parameters to be used for the buffer object
+ * @bo_ptr: pointer to the buffer object pointer
+ *
+ * Creates an &amdgpu_bo buffer object.
+ *
+ * Returns:
+ * 0 for success or a negative error code on failure.
+ */
+int amdgpu_bo_create(struct amdgpu_device *adev,
 			       struct amdgpu_bo_param *bp,
 			       struct amdgpu_bo **bo_ptr)
 {
 	return r;
 }
-static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
+int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
-				   unsigned long size,
+			    unsigned long size,
-				   struct amdgpu_bo *bo)
+			    struct amdgpu_bo *bo)
 {
 	struct amdgpu_bo_param bp;
 	int r;
 	memset(&bp, 0, sizeof(bp));
 	bp.size = size;
 	bp.domain = AMDGPU_GEM_DOMAIN_GTT;
-	bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC |
+	bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC;
-		AMDGPU_GEM_CREATE_SHADOW;
 	bp.type = ttm_bo_type_kernel;
 	bp.resv = bo->tbo.base.resv;
 	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
-	r = amdgpu_bo_do_create(adev, &bp, &bo->shadow);
+	r = amdgpu_bo_create(adev, &bp, &bo->shadow);
 	if (!r) {
 		bo->shadow->parent = amdgpu_bo_ref(bo);
 		mutex_lock(&adev->shadow_list_lock);
 	return r;
 }
-/**
- * amdgpu_bo_create - create an &amdgpu_bo buffer object
- * @adev: amdgpu device object
- * @bp: parameters to be used for the buffer object
- * @bo_ptr: pointer to the buffer object pointer
- *
- * Creates an &amdgpu_bo buffer object; and if requested, also creates a
- * shadow object.
- * Shadow object is used to backup the original buffer object, and is always
- * in GTT.
- *
- * Returns:
- * 0 for success or a negative error code on failure.
- */
-int amdgpu_bo_create(struct amdgpu_device *adev,
-		     struct amdgpu_bo_param *bp,
-		     struct amdgpu_bo **bo_ptr)
-{
-	u64 flags = bp->flags;
-	int r;
-	bp->flags = bp->flags & ~AMDGPU_GEM_CREATE_SHADOW;
-	r = amdgpu_bo_do_create(adev, bp, bo_ptr);
-	if (r)
-		return r;
-	if ((flags & AMDGPU_GEM_CREATE_SHADOW) && !(adev->flags & AMD_IS_APU)) {
-		if (!bp->resv)
-			WARN_ON(dma_resv_lock((*bo_ptr)->tbo.base.resv,
-							NULL));
-		r = amdgpu_bo_create_shadow(adev, bp->size, *bo_ptr);
-		if (!bp->resv)
-			dma_resv_unlock((*bo_ptr)->tbo.base.resv);
-		if (r)
-			amdgpu_bo_unref(bo_ptr);
-	}
-	return r;
-}
 /**
  * amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object
  * @adev: amdgpu device object
 	struct amdgpu_bo *bo_ptr;
 	int r;
-	bp->flags = bp->flags & ~AMDGPU_GEM_CREATE_SHADOW;
 	bp->bo_ptr_size = sizeof(struct amdgpu_bo_user);
-	r = amdgpu_bo_do_create(adev, bp, &bo_ptr);
+	r = amdgpu_bo_create(adev, bp, &bo_ptr);
 	if (r)
 		return r;
 	amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);
 	amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC);
 	amdgpu_bo_print_flag(m, bo, VRAM_CLEARED);
-	amdgpu_bo_print_flag(m, bo, SHADOW);
 	amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS);
 	amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID);
 	amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC);

drivers/gpu/drm/amd/amdgpu/amdgpu_object.h+7−0

 #define AMDGPU_BO_INVALID_OFFSET	LONG_MAX
 #define AMDGPU_BO_MAX_PLACEMENTS	3
+/* BO flag to indicate a KFD userptr BO */
+#define AMDGPU_AMDKFD_CREATE_USERPTR_BO	(1ULL << 63)
+#define AMDGPU_AMDKFD_CREATE_SVM_BO	(1ULL << 62)
 #define to_amdgpu_bo_user(abo) container_of((abo), struct amdgpu_bo_user, bo)
 struct amdgpu_bo_param {
 			  struct amdgpu_bo_user **ubo_ptr);
 void amdgpu_bo_free_kernel(struct amdgpu_bo **bo, u64 *gpu_addr,
 			   void **cpu_addr);
+int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
+			    unsigned long size,
+			    struct amdgpu_bo *bo);
 int amdgpu_bo_kmap(struct amdgpu_bo *bo, void **ptr);
 void *amdgpu_bo_kptr(struct amdgpu_bo *bo);
 void amdgpu_bo_kunmap(struct amdgpu_bo *bo);

drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c+36−37

 	return ret;
 }
-static int psp_clear_vf_fw(struct psp_context *psp)
-{
-	int ret;
-	struct psp_gfx_cmd_resp *cmd;
-	if (!amdgpu_sriov_vf(psp->adev) || psp->adev->asic_type != CHIP_NAVI12)
-		return 0;
-	cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
-	if (!cmd)
-		return -ENOMEM;
-	cmd->cmd_id = GFX_CMD_ID_CLEAR_VF_FW;
-	ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
-	kfree(cmd);
-	return ret;
-}
 static bool psp_skip_tmr(struct psp_context *psp)
 {
 	switch (psp->adev->asic_type) {
 	case CHIP_NAVI12:
 	case CHIP_SIENNA_CICHLID:
+	case CHIP_ALDEBARAN:
 		return true;
 	default:
 		return false;
 	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 	memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
+	ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
+	if (psp->adev->gmc.xgmi.connected_to_cpu)
+		ras_cmd->ras_in_message.init_flags.poison_mode_en = 1;
+	else
+		ras_cmd->ras_in_message.init_flags.dgpu_mode = 1;
 	psp_prep_ta_load_cmd_buf(cmd,
 				 psp->fw_pri_mc_addr,
 				 psp->ta_ras_ucode_size,
 	ret = psp_cmd_submit_buf(psp, NULL, cmd,
 			psp->fence_buf_mc_addr);
-	ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 	if (!ret) {
 		psp->ras.session_id = cmd->resp.session_id;
 	return ret;
 }
+static int psp_ras_status_to_errno(struct amdgpu_device *adev,
+					 enum ta_ras_status ras_status)
+{
+	int ret = -EINVAL;
+	switch (ras_status) {
+	case TA_RAS_STATUS__SUCCESS:
+		ret = 0;
+		break;
+	case TA_RAS_STATUS__RESET_NEEDED:
+		ret = -EAGAIN;
+		break;
+	case TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE:
+		dev_warn(adev->dev, "RAS WARN: ras function unavailable\n");
+		break;
+	case TA_RAS_STATUS__ERROR_ASD_READ_WRITE:
+		dev_warn(adev->dev, "RAS WARN: asd read or write failed\n");
+		break;
+	default:
+		dev_err(adev->dev, "RAS ERROR: ras function failed ret 0x%X\n", ret);
+	}
+	return ret;
+}
 int psp_ras_enable_features(struct psp_context *psp,
 		union ta_ras_cmd_input *info, bool enable)
 {
 	if (ret)
 		return -EINVAL;
-	return ras_cmd->ras_status;
+	return psp_ras_status_to_errno(psp->adev, ras_cmd->ras_status);
 }
 static int psp_ras_terminate(struct psp_context *psp)
 	if (amdgpu_ras_intr_triggered())
 		return 0;
-	return ras_cmd->ras_status;
+	return psp_ras_status_to_errno(psp->adev, ras_cmd->ras_status);
 }
 // ras end
 		return ret;
 	}
-	ret = psp_clear_vf_fw(psp);
-	if (ret) {
-		DRM_ERROR("PSP clear vf fw!\n");
-		return ret;
-	}
 	ret = psp_boot_config_set(adev);
 	if (ret) {
 		DRM_WARN("PSP set boot config@\n");
 		return 0;
 	if ((amdgpu_in_reset(adev) &&
-	     ras && ras->supported &&
+	     ras && adev->ras_enabled &&
 	     (adev->asic_type == CHIP_ARCTURUS ||
 	      adev->asic_type == CHIP_VEGA20)) ||
 	     (adev->in_runpm &&
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	struct psp_context *psp = &adev->psp;
-	int ret;
 	if (psp->adev->psp.ta_fw) {
 		psp_ras_terminate(psp);
 	}
 	psp_asd_unload(psp);
-	ret = psp_clear_vf_fw(psp);
-	if (ret) {
-		DRM_ERROR("PSP clear vf fw!\n");
-		return ret;
-	}
 	psp_tmr_terminate(psp);
 	psp_ring_destroy(psp, PSP_RING_TYPE__KM);

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c+114−85

@@ -33,6 +33,7 @@ · 32 unmodified lines
33	#include "amdgpu_atomfirmware.h"	33	#include "amdgpu_atomfirmware.h"
34	#include "amdgpu_xgmi.h"	34	#include "amdgpu_xgmi.h"
35	#include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"	35	#include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
		36	#include "atom.h"
36		37
37	static const char *RAS_FS_NAME = "ras";	38	static const char *RAS_FS_NAME = "ras";
38		39
@@ -320,11 +321,14 @@ static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, · 281 unmodified lines
320	* "disable" requires only the block.	321	* "disable" requires only the block.
321	* "enable" requires the block and error type.	322	* "enable" requires the block and error type.
322	* "inject" requires the block, error type, address, and value.	323	* "inject" requires the block, error type, address, and value.
		324	*
323	* The block is one of: umc, sdma, gfx, etc.	325	* The block is one of: umc, sdma, gfx, etc.
324	* see ras_block_string[] for details	326	* see ras_block_string[] for details
		327	*
325	* The error type is one of: ue, ce, where,	328	* The error type is one of: ue, ce, where,
326	* ue is multi-uncorrectable	329	* ue is multi-uncorrectable
327	* ce is single-correctable	330	* ce is single-correctable
		331	*
328	* The sub-block is a the sub-block index, pass 0 if there is no sub-block.	332	* The sub-block is a the sub-block index, pass 0 if there is no sub-block.
329	* The address and value are hexadecimal numbers, leading 0x is optional.	333	* The address and value are hexadecimal numbers, leading 0x is optional.
330	*	334	*
@@ -531,7 +535,7 @@ static struct ras_manager amdgpu_ras_create_obj(struct amdgpu_device adev, · 200 unmodified lines
531	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);	535	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
532	struct ras_manager *obj;	536	struct ras_manager *obj;
533		537
534	if (!adev->ras_features \|\| !con)	538	if (!adev->ras_enabled \|\| !con)
535	return NULL;	539	return NULL;
536		540
537	if (head->block >= AMDGPU_RAS_BLOCK_COUNT)	541	if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
@@ -558,7 +562,7 @@ struct ras_manager amdgpu_ras_find_obj(struct amdgpu_device adev, · 20 unmodified lines
558	struct ras_manager *obj;	562	struct ras_manager *obj;
559	int i;	563	int i;
560		564
561	if (!adev->ras_features \|\| !con)	565	if (!adev->ras_enabled \|\| !con)
562	return NULL;	566	return NULL;
563		567
564	if (head) {	568	if (head) {
@@ -585,36 +589,11 @@ struct ras_manager amdgpu_ras_find_obj(struct amdgpu_device adev, · 20 unmodified lines
585	}	589	}
586	/* obj end */	590	/* obj end */
587		591
588	static void amdgpu_ras_parse_status_code(struct amdgpu_device *adev,
589	const char* invoke_type,
590	const char* block_name,
591	enum ta_ras_status ret)
592	{
593	switch (ret) {
594	case TA_RAS_STATUS__SUCCESS:
595	return;
596	case TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE:
597	dev_warn(adev->dev,
598	"RAS WARN: %s %s currently unavailable\n",
599	invoke_type,
600	block_name);
601	break;
602	default:
603	dev_err(adev->dev,
604	"RAS ERROR: %s %s error failed ret 0x%X\n",
605	invoke_type,
606	block_name,
607	ret);
608	}
609	}
610
611	/* feature ctl begin */	592	/* feature ctl begin */
612	static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,	593	static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
613	struct ras_common_if *head)	594	struct ras_common_if *head)
614	{	595	{
615	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);	596	return adev->ras_hw_enabled & BIT(head->block);
616
617	return con->hw_supported & BIT(head->block);
618	}	597	}
619		598
620	static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,	599	static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
@@ -658,11 +637,7 @@ static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, · 37 unmodified lines
658	con->features \|= BIT(head->block);	637	con->features \|= BIT(head->block);
659	} else {	638	} else {
660	if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {	639	if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
661	/* skip clean gfx ras context feature for VEGA20 Gaming.	640	con->features &= ~BIT(head->block);
662	* will clean later
663	*/
664	if (!(!adev->ras_features && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)))
665	con->features &= ~BIT(head->block);
666	put_obj(obj);	641	put_obj(obj);
667	}	642	}
668	}	643	}
@@ -708,15 +683,10 @@ int amdgpu_ras_feature_enable(struct amdgpu_device *adev, · 39 unmodified lines
708	if (!amdgpu_ras_intr_triggered()) {	683	if (!amdgpu_ras_intr_triggered()) {
709	ret = psp_ras_enable_features(&adev->psp, info, enable);	684	ret = psp_ras_enable_features(&adev->psp, info, enable);
710	if (ret) {	685	if (ret) {
711	amdgpu_ras_parse_status_code(adev,	686	dev_err(adev->dev, "ras %s %s failed %d\n",
712	enable ? "enable":"disable",	687	enable ? "enable":"disable",
713	ras_block_str(head->block),	688	ras_block_str(head->block),
714	(enum ta_ras_status)ret);	689	ret);
715	if (ret == TA_RAS_STATUS__RESET_NEEDED)
716	ret = -EAGAIN;
717	else
718	ret = -EINVAL;
719
720	goto out;	690	goto out;
721	}	691	}
722	}	692	}
@@ -770,6 +740,10 @@ int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, · 47 unmodified lines
770	con->features \|= BIT(head->block);	740	con->features \|= BIT(head->block);
771		741
772	ret = amdgpu_ras_feature_enable(adev, head, 0);	742	ret = amdgpu_ras_feature_enable(adev, head, 0);
		743
		744	/* clean gfx block ras features flag */
		745	if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
		746	con->features &= ~BIT(head->block);
773	}	747	}
774	} else	748	} else
775	ret = amdgpu_ras_feature_enable(adev, head, enable);	749	ret = amdgpu_ras_feature_enable(adev, head, enable);
@@ -890,6 +864,11 @@ int amdgpu_ras_query_error_status(struct amdgpu_device *adev, · 114 unmodified lines
890	adev->gmc.xgmi.ras_funcs->query_ras_error_count)	864	adev->gmc.xgmi.ras_funcs->query_ras_error_count)
891	adev->gmc.xgmi.ras_funcs->query_ras_error_count(adev, &err_data);	865	adev->gmc.xgmi.ras_funcs->query_ras_error_count(adev, &err_data);
892	break;	866	break;
		867	case AMDGPU_RAS_BLOCK__HDP:
		868	if (adev->hdp.ras_funcs &&
		869	adev->hdp.ras_funcs->query_ras_error_count)
		870	adev->hdp.ras_funcs->query_ras_error_count(adev, &err_data);
		871	break;
893	default:	872	default:
894	break;	873	break;
895	}	874	}
@@ -901,17 +880,42 @@ int amdgpu_ras_query_error_status(struct amdgpu_device *adev, · 5 unmodified lines
901	info->ce_count = obj->err_data.ce_count;	880	info->ce_count = obj->err_data.ce_count;
902		881
903	if (err_data.ce_count) {	882	if (err_data.ce_count) {
904	dev_info(adev->dev, "%ld correctable hardware errors "	883	if (adev->smuio.funcs &&
		884	adev->smuio.funcs->get_socket_id &&
		885	adev->smuio.funcs->get_die_id) {
		886	dev_info(adev->dev, "socket: %d, die: %d "
		887	"%ld correctable hardware errors "
		888	"detected in %s block, no user "
		889	"action is needed.\n",
		890	adev->smuio.funcs->get_socket_id(adev),
		891	adev->smuio.funcs->get_die_id(adev),
		892	obj->err_data.ce_count,
		893	ras_block_str(info->head.block));
		894	} else {
		895	dev_info(adev->dev, "%ld correctable hardware errors "
905	"detected in %s block, no user "	896	"detected in %s block, no user "
906	"action is needed.\n",	897	"action is needed.\n",
907	obj->err_data.ce_count,	898	obj->err_data.ce_count,
908	ras_block_str(info->head.block));	899	ras_block_str(info->head.block));
		900	}
909	}	901	}
910	if (err_data.ue_count) {	902	if (err_data.ue_count) {
911	dev_info(adev->dev, "%ld uncorrectable hardware errors "	903	if (adev->smuio.funcs &&
		904	adev->smuio.funcs->get_socket_id &&
		905	adev->smuio.funcs->get_die_id) {
		906	dev_info(adev->dev, "socket: %d, die: %d "
		907	"%ld uncorrectable hardware errors "
		908	"detected in %s block\n",
		909	adev->smuio.funcs->get_socket_id(adev),
		910	adev->smuio.funcs->get_die_id(adev),
		911	obj->err_data.ue_count,
		912	ras_block_str(info->head.block));
		913	} else {
		914	dev_info(adev->dev, "%ld uncorrectable hardware errors "
912	"detected in %s block\n",	915	"detected in %s block\n",
913	obj->err_data.ue_count,	916	obj->err_data.ue_count,
914	ras_block_str(info->head.block));	917	ras_block_str(info->head.block));
		918	}
915	}	919	}
916		920
917	return 0;	921	return 0;
@@ -937,11 +941,20 @@ int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, · 19 unmodified lines
937	if (adev->mmhub.ras_funcs &&	941	if (adev->mmhub.ras_funcs &&
938	adev->mmhub.ras_funcs->reset_ras_error_count)	942	adev->mmhub.ras_funcs->reset_ras_error_count)
939	adev->mmhub.ras_funcs->reset_ras_error_count(adev);	943	adev->mmhub.ras_funcs->reset_ras_error_count(adev);
		944
		945	if (adev->mmhub.ras_funcs &&
		946	adev->mmhub.ras_funcs->reset_ras_error_status)
		947	adev->mmhub.ras_funcs->reset_ras_error_status(adev);
940	break;	948	break;
941	case AMDGPU_RAS_BLOCK__SDMA:	949	case AMDGPU_RAS_BLOCK__SDMA:
942	if (adev->sdma.funcs->reset_ras_error_count)	950	if (adev->sdma.funcs->reset_ras_error_count)
943	adev->sdma.funcs->reset_ras_error_count(adev);	951	adev->sdma.funcs->reset_ras_error_count(adev);
944	break;	952	break;
		953	case AMDGPU_RAS_BLOCK__HDP:
		954	if (adev->hdp.ras_funcs &&
		955	adev->hdp.ras_funcs->reset_ras_error_count)
		956	adev->hdp.ras_funcs->reset_ras_error_count(adev);
		957	break;
945	default:	958	default:
946	break;	959	break;
947	}	960	}
@@ -1022,10 +1035,9 @@ int amdgpu_ras_error_inject(struct amdgpu_device *adev, · 74 unmodified lines
1022	ret = -EINVAL;	1035	ret = -EINVAL;
1023	}	1036	}
1024		1037
1025	amdgpu_ras_parse_status_code(adev,	1038	if (ret)
1026	"inject",	1039	dev_err(adev->dev, "ras inject %s failed %d\n",
1027	ras_block_str(info->head.block),	1040	ras_block_str(info->head.block), ret);
1028	(enum ta_ras_status)ret);
1029		1041
1030	return ret;	1042	return ret;
1031	}	1043	}
@@ -1038,7 +1050,7 @@ unsigned long amdgpu_ras_query_error_count(struct amdgpu_device *adev, · 6 unmodified lines
1038	struct ras_manager *obj;	1050	struct ras_manager *obj;
1039	struct ras_err_data data = {0, 0};	1051	struct ras_err_data data = {0, 0};
1040		1052
1041	if (!adev->ras_features \|\| !con)	1053	if (!adev->ras_enabled \|\| !con)
1042	return 0;	1054	return 0;
1043		1055
1044	list_for_each_entry(obj, &con->head, node) {	1056	list_for_each_entry(obj, &con->head, node) {
@@ -1265,8 +1277,8 @@ static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) · 220 unmodified lines
1265	static struct dentry amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device adev)	1277	static struct dentry amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device adev)
1266	{	1278	{
1267	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);	1279	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1268	struct dentry *dir;	1280	struct drm_minor *minor = adev_to_drm(adev)->primary;
1269	struct drm_minor *minor = adev_to_drm(adev)->primary;	1281	struct dentry *dir;
1270		1282
1271	dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);	1283	dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1272	debugfs_create_file("ras_ctrl", S_IWUGO \| S_IRUGO, dir, adev,	1284	debugfs_create_file("ras_ctrl", S_IWUGO \| S_IRUGO, dir, adev,
@@ -1275,6 +1287,8 @@ static struct dentry amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device · 2 unmodified lines
1275	&amdgpu_ras_debugfs_eeprom_ops);	1287	&amdgpu_ras_debugfs_eeprom_ops);
1276	debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,	1288	debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1277	&con->bad_page_cnt_threshold);	1289	&con->bad_page_cnt_threshold);
		1290	debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
		1291	debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
1278		1292
1279	/*	1293	/*
1280	* After one uncorrectable error happens, usually GPU recovery will	1294	* After one uncorrectable error happens, usually GPU recovery will
@@ -1561,7 +1575,7 @@ static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev) · 280 unmodified lines
1561	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);	1575	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1562	struct ras_manager *obj;	1576	struct ras_manager *obj;
1563		1577
1564	if (!adev->ras_features \|\| !con)	1578	if (!adev->ras_enabled \|\| !con)
1565	return;	1579	return;
1566		1580
1567	list_for_each_entry(obj, &con->head, node) {	1581	list_for_each_entry(obj, &con->head, node) {
@@ -1611,7 +1625,7 @@ static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) · 43 unmodified lines
1611	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);	1625	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1612	struct ras_manager *obj;	1626	struct ras_manager *obj;
1613		1627
1614	if (!adev->ras_features \|\| !con)	1628	if (!adev->ras_enabled \|\| !con)
1615	return;	1629	return;
1616		1630
1617	list_for_each_entry(obj, &con->head, node) {	1631	list_for_each_entry(obj, &con->head, node) {
@@ -1925,7 +1939,7 @@ int amdgpu_ras_recovery_init(struct amdgpu_device *adev) · 307 unmodified lines
1925	bool exc_err_limit = false;	1939	bool exc_err_limit = false;
1926	int ret;	1940	int ret;
1927		1941
1928	if (adev->ras_features && con)	1942	if (adev->ras_enabled && con)
1929	data = &con->eh_data;	1943	data = &con->eh_data;
1930	else	1944	else
1931	return 0;	1945	return 0;
@@ -2028,6 +2042,23 @@ static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) · 96 unmodified lines
2028	adev->asic_type == CHIP_SIENNA_CICHLID;	2042	adev->asic_type == CHIP_SIENNA_CICHLID;
2029	}	2043	}
2030		2044
		2045	/*
		2046	* this is workaround for vega20 workstation sku,
		2047	* force enable gfx ras, ignore vbios gfx ras flag
		2048	* due to GC EDC can not write
		2049	*/
		2050	static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
		2051	{
		2052	struct atom_context *ctx = adev->mode_info.atom_context;
		2053
		2054	if (!ctx)
		2055	return;
		2056
		2057	if (strnstr(ctx->vbios_version, "D16406",
		2058	sizeof(ctx->vbios_version)))
		2059	adev->ras_hw_enabled \|= (1 << AMDGPU_RAS_BLOCK__GFX);
		2060	}
		2061
2031	/*	2062	/*
2032	* check hardware's ras ability which will be saved in hw_supported.	2063	* check hardware's ras ability which will be saved in hw_supported.
2033	* if hardware does not support ras, we can skip some ras initializtion and	2064	* if hardware does not support ras, we can skip some ras initializtion and
@@ -2037,11 +2068,9 @@ static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) · 3 unmodified lines
2037	* we have to initialize ras as normal. but need check if operation is	2068	* we have to initialize ras as normal. but need check if operation is
2038	* allowed or not in each function.	2069	* allowed or not in each function.
2039	*/	2070	*/
2040	static void amdgpu_ras_check_supported(struct amdgpu_device *adev,	2071	static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
2041	uint32_t hw_supported, uint32_t supported)
2042	{	2072	{
2043	*hw_supported = 0;	2073	adev->ras_hw_enabled = adev->ras_enabled = 0;
2044	*supported = 0;
2045		2074
2046	if (amdgpu_sriov_vf(adev) \|\| !adev->is_atom_fw \|\|	2075	if (amdgpu_sriov_vf(adev) \|\| !adev->is_atom_fw \|\|
2047	!amdgpu_ras_asic_supported(adev))	2076	!amdgpu_ras_asic_supported(adev))
@@ -2050,33 +2079,34 @@ static void amdgpu_ras_check_supported(struct amdgpu_device *adev, · 2 unmodified lines
2050	if (!adev->gmc.xgmi.connected_to_cpu) {	2079	if (!adev->gmc.xgmi.connected_to_cpu) {
2051	if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {	2080	if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
2052	dev_info(adev->dev, "MEM ECC is active.\n");	2081	dev_info(adev->dev, "MEM ECC is active.\n");
2053	*hw_supported \|= (1 << AMDGPU_RAS_BLOCK__UMC \|	2082	adev->ras_hw_enabled \|= (1 << AMDGPU_RAS_BLOCK__UMC \|
2054	1 << AMDGPU_RAS_BLOCK__DF);	2083	1 << AMDGPU_RAS_BLOCK__DF);
2055	} else {	2084	} else {
2056	dev_info(adev->dev, "MEM ECC is not presented.\n");	2085	dev_info(adev->dev, "MEM ECC is not presented.\n");
2057	}	2086	}
2058		2087
2059	if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {	2088	if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
2060	dev_info(adev->dev, "SRAM ECC is active.\n");	2089	dev_info(adev->dev, "SRAM ECC is active.\n");
2061	*hw_supported \|= ~(1 << AMDGPU_RAS_BLOCK__UMC \|	2090	adev->ras_hw_enabled \|= ~(1 << AMDGPU_RAS_BLOCK__UMC \|
2062	1 << AMDGPU_RAS_BLOCK__DF);	2091	1 << AMDGPU_RAS_BLOCK__DF);
2063	} else {	2092	} else {
2064	dev_info(adev->dev, "SRAM ECC is not presented.\n");	2093	dev_info(adev->dev, "SRAM ECC is not presented.\n");
2065	}	2094	}
2066	} else {	2095	} else {
2067	/* driver only manages a few IP blocks RAS feature	2096	/* driver only manages a few IP blocks RAS feature
2068	* when GPU is connected cpu through XGMI */	2097	* when GPU is connected cpu through XGMI */
2069	*hw_supported \|= (1 << AMDGPU_RAS_BLOCK__GFX \|	2098	adev->ras_hw_enabled \|= (1 << AMDGPU_RAS_BLOCK__GFX \|
2070	1 << AMDGPU_RAS_BLOCK__SDMA \|	2099	1 << AMDGPU_RAS_BLOCK__SDMA \|
2071	1 << AMDGPU_RAS_BLOCK__MMHUB);	2100	1 << AMDGPU_RAS_BLOCK__MMHUB);
2072	}	2101	}
2073		2102
		2103	amdgpu_ras_get_quirks(adev);
		2104
2074	/* hw_supported needs to be aligned with RAS block mask. */	2105	/* hw_supported needs to be aligned with RAS block mask. */
2075	*hw_supported &= AMDGPU_RAS_BLOCK_MASK;	2106	adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
2076		2107
2077	*supported = amdgpu_ras_enable == 0 ?	2108	adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
2078	0 : *hw_supported & amdgpu_ras_mask;	2109	adev->ras_hw_enabled & amdgpu_ras_mask;
2079	adev->ras_features = *supported;
2080	}	2110	}
2081		2111
2082	int amdgpu_ras_init(struct amdgpu_device *adev)	2112	int amdgpu_ras_init(struct amdgpu_device *adev)
@@ -2097,13 +2127,13 @@ int amdgpu_ras_init(struct amdgpu_device *adev) · 14 unmodified lines
2097		2127
2098	amdgpu_ras_set_context(adev, con);	2128	amdgpu_ras_set_context(adev, con);
2099		2129
2100	amdgpu_ras_check_supported(adev, &con->hw_supported,	2130	amdgpu_ras_check_supported(adev);
2101	&con->supported);	2131
2102	if (!con->hw_supported \|\| (adev->asic_type == CHIP_VEGA10)) {	2132	if (!adev->ras_enabled \|\| adev->asic_type == CHIP_VEGA10) {
2103	/* set gfx block ras context feature for VEGA20 Gaming	2133	/* set gfx block ras context feature for VEGA20 Gaming
2104	* send ras disable cmd to ras ta during ras late init.	2134	* send ras disable cmd to ras ta during ras late init.
2105	*/	2135	*/
2106	if (!adev->ras_features && adev->asic_type == CHIP_VEGA20) {	2136	if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
2107	con->features \|= BIT(AMDGPU_RAS_BLOCK__GFX);	2137	con->features \|= BIT(AMDGPU_RAS_BLOCK__GFX);
2108		2138
2109	return 0;	2139	return 0;
@@ -2153,8 +2183,9 @@ int amdgpu_ras_init(struct amdgpu_device *adev) · 43 unmodified lines
2153	}	2183	}
2154		2184
2155	dev_info(adev->dev, "RAS INFO: ras initialized successfully, "	2185	dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
2156	"hardware ability[%x] ras_mask[%x]\n",	2186	"hardware ability[%x] ras_mask[%x]\n",
2157	con->hw_supported, con->supported);	2187	adev->ras_hw_enabled, adev->ras_enabled);
		2188
2158	return 0;	2189	return 0;
2159	release_con:	2190	release_con:
2160	amdgpu_ras_set_context(adev, NULL);	2191	amdgpu_ras_set_context(adev, NULL);
@@ -2268,7 +2299,7 @@ void amdgpu_ras_resume(struct amdgpu_device *adev) · 107 unmodified lines
2268	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);	2299	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2269	struct ras_manager obj, tmp;	2300	struct ras_manager obj, tmp;
2270		2301
2271	if (!adev->ras_features \|\| !con) {	2302	if (!adev->ras_enabled \|\| !con) {
2272	/* clean ras context for VEGA20 Gaming after send ras disable cmd */	2303	/* clean ras context for VEGA20 Gaming after send ras disable cmd */
2273	amdgpu_release_ras_context(adev);	2304	amdgpu_release_ras_context(adev);
2274		2305
@@ -2314,7 +2345,7 @@ void amdgpu_ras_suspend(struct amdgpu_device *adev) · 39 unmodified lines
2314	{	2345	{
2315	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);	2346	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2316		2347
2317	if (!adev->ras_features \|\| !con)	2348	if (!adev->ras_enabled \|\| !con)
2318	return;	2349	return;
2319		2350
2320	amdgpu_ras_disable_all_features(adev, 0);	2351	amdgpu_ras_disable_all_features(adev, 0);
@@ -2328,7 +2359,7 @@ int amdgpu_ras_pre_fini(struct amdgpu_device *adev) · 7 unmodified lines
2328	{	2359	{
2329	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);	2360	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2330		2361
2331	if (!adev->ras_features \|\| !con)	2362	if (!adev->ras_enabled \|\| !con)
2332	return 0;	2363	return 0;
2333		2364
2334	/* Need disable ras on all IPs here before ip [hw/sw]fini */	2365	/* Need disable ras on all IPs here before ip [hw/sw]fini */
@@ -2341,7 +2372,7 @@ int amdgpu_ras_fini(struct amdgpu_device *adev) · 6 unmodified lines
2341	{	2372	{
2342	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);	2373	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2343		2374
2344	if (!adev->ras_features \|\| !con)	2375	if (!adev->ras_enabled \|\| !con)
2345	return 0;	2376	return 0;
2346		2377
2347	amdgpu_ras_fs_fini(adev);	2378	amdgpu_ras_fs_fini(adev);
@@ -2360,10 +2391,8 @@ int amdgpu_ras_fini(struct amdgpu_device *adev) · 12 unmodified lines
2360		2391
2361	void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)	2392	void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
2362	{	2393	{
2363	uint32_t hw_supported, supported;	2394	amdgpu_ras_check_supported(adev);
2364		2395	if (!adev->ras_hw_enabled)
2365	amdgpu_ras_check_supported(adev, &hw_supported, &supported);
2366	if (!hw_supported)
2367	return;	2396	return;
2368		2397
2369	if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {	2398	if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
@@ -2392,7 +2421,7 @@ void amdgpu_release_ras_context(struct amdgpu_device *adev) · 22 unmodified lines
2392	if (!con)	2421	if (!con)
2393	return;	2422	return;
2394		2423
2395	if (!adev->ras_features && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {	2424	if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
2396	con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);	2425	con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
2397	amdgpu_ras_set_context(adev, NULL);	2426	amdgpu_ras_set_context(adev, NULL);
2398	kfree(con);	2427	kfree(con);

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h+1−4

 struct amdgpu_ras {
 	/* ras infrastructure */
 	/* for ras itself. */
-	uint32_t hw_supported;
-	/* for IP to check its ras ability. */
-	uint32_t supported;
 	uint32_t features;
 	struct list_head head;
 	/* sysfs */
 	if (block >= AMDGPU_RAS_BLOCK_COUNT)
 		return 0;
-	return ras && (ras->supported & (1 << block));
+	return ras && (adev->ras_enabled & (1 << block));
 }
 int amdgpu_ras_recovery_init(struct amdgpu_device *adev);

drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c+1−0

@@ -158,6 +158,7 @@ void amdgpu_ring_undo(struct amdgpu_ring *ring) · 157 unmodified lines
158	* @irq_src: interrupt source to use for this ring	158	* @irq_src: interrupt source to use for this ring
159	* @irq_type: interrupt type to use for this ring	159	* @irq_type: interrupt type to use for this ring
160	* @hw_prio: ring priority (NORMAL/HIGH)	160	* @hw_prio: ring priority (NORMAL/HIGH)
		161	* @sched_score: optional score atomic shared with other schedulers
161	*	162	*
162	* Initialize the driver information for the selected ring (all asics).	163	* Initialize the driver information for the selected ring (all asics).
163	* Returns 0 on success, error on failure.	164	* Returns 0 on success, error on failure.

drivers/gpu/drm/amd/amdgpu/amdgpu_smuio.h+1−0

@@ -29,6 +29,7 @@ struct amdgpu_smuio_funcs { · 28 unmodified lines
29	void (update_rom_clock_gating)(struct amdgpu_device adev, bool enable);	29	void (update_rom_clock_gating)(struct amdgpu_device adev, bool enable);
30	void (get_clock_gating_state)(struct amdgpu_device adev, u32 *flags);	30	void (get_clock_gating_state)(struct amdgpu_device adev, u32 *flags);
31	u32 (get_die_id)(struct amdgpu_device adev);	31	u32 (get_die_id)(struct amdgpu_device adev);
		32	u32 (get_socket_id)(struct amdgpu_device adev);
32	bool (is_host_gpu_xgmi_supported)(struct amdgpu_device adev);	33	bool (is_host_gpu_xgmi_supported)(struct amdgpu_device adev);
33	};	34	};
34		35

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c+25−74

 #include <linux/dma-mapping.h>
 #include <linux/iommu.h>
-#include <linux/hmm.h>
 #include <linux/pagemap.h>
 #include <linux/sched/task.h>
 #include <linux/sched/mm.h>
 	}
 	abo = ttm_to_amdgpu_bo(bo);
+	if (abo->flags & AMDGPU_AMDKFD_CREATE_SVM_BO) {
+		struct dma_fence *fence;
+		struct dma_resv *resv = &bo->base._resv;
+		rcu_read_lock();
+		fence = rcu_dereference(resv->fence_excl);
+		if (fence && !fence->ops->signaled)
+			dma_fence_enable_sw_signaling(fence);
+		placement->num_placement = 0;
+		placement->num_busy_placement = 0;
+		rcu_read_unlock();
+		return;
+	}
 	switch (bo->mem.mem_type) {
 	case AMDGPU_PL_GDS:
 	case AMDGPU_PL_GWS:
 {
 	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
-	/*
-	 * Don't verify access for KFD BOs. They don't have a GEM
-	 * object associated with them.
-	 */
-	if (abo->kfd_bo)
-		return 0;
 	if (amdgpu_ttm_tt_get_usermm(bo->ttm))
 		return -EPERM;
 	return drm_vma_node_verify_access(&abo->tbo.base.vma_node,
 }
 /**
- * amdgpu_copy_ttm_mem_to_mem - Helper function for copy
+ * amdgpu_ttm_copy_mem_to_mem - Helper function for copy
  * @adev: amdgpu device
  * @src: buffer/address where to read from
  * @dst: buffer/address where to write to
 	struct amdgpu_ttm_tt *gtt = (void *)ttm;
 	unsigned long start = gtt->userptr;
 	struct vm_area_struct *vma;
-	struct hmm_range *range;
-	unsigned long timeout;
 	struct mm_struct *mm;
-	unsigned long i;
+	bool readonly;
 	int r = 0;
 	mm = bo->notifier.mm;
 	if (!mmget_not_zero(mm)) /* Happens during process shutdown */
 		return -ESRCH;
-	range = kzalloc(sizeof(*range), GFP_KERNEL);
-	if (unlikely(!range)) {
-		r = -ENOMEM;
-		goto out;
-	}
-	range->notifier = &bo->notifier;
-	range->start = bo->notifier.interval_tree.start;
-	range->end = bo->notifier.interval_tree.last + 1;
-	range->default_flags = HMM_PFN_REQ_FAULT;
-	if (!amdgpu_ttm_tt_is_readonly(ttm))
-		range->default_flags |= HMM_PFN_REQ_WRITE;
-	range->hmm_pfns = kvmalloc_array(ttm->num_pages,
-					 sizeof(*range->hmm_pfns), GFP_KERNEL);
-	if (unlikely(!range->hmm_pfns)) {
-		r = -ENOMEM;
-		goto out_free_ranges;
-	}
 	mmap_read_lock(mm);
 	vma = find_vma(mm, start);
+	mmap_read_unlock(mm);
 	if (unlikely(!vma || start < vma->vm_start)) {
 		r = -EFAULT;
-		goto out_unlock;
+		goto out_putmm;
 	}
 	if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&
 		vma->vm_file)) {
 		r = -EPERM;
-		goto out_unlock;
+		goto out_putmm;
-	}
-	mmap_read_unlock(mm);
-	timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
-retry:
-	range->notifier_seq = mmu_interval_read_begin(&bo->notifier);
-	mmap_read_lock(mm);
-	r = hmm_range_fault(range);
-	mmap_read_unlock(mm);
-	if (unlikely(r)) {
-		/*
-		 * FIXME: This timeout should encompass the retry from
-		 * mmu_interval_read_retry() as well.
-		 */
-		if (r == -EBUSY && !time_after(jiffies, timeout))
-			goto retry;
-		goto out_free_pfns;
 	}
-	/*
+	readonly = amdgpu_ttm_tt_is_readonly(ttm);
-	 * Due to default_flags, all pages are HMM_PFN_VALID or
+	r = amdgpu_hmm_range_get_pages(&bo->notifier, mm, pages, start,
-	 * hmm_range_fault() fails. FIXME: The pages cannot be touched outside
+				       ttm->num_pages, &gtt->range, readonly,
-	 * the notifier_lock, and mmu_interval_read_retry() must be done first.
+				       false);
-	 */
+out_putmm:
-	for (i = 0; i < ttm->num_pages; i++)
-		pages[i] = hmm_pfn_to_page(range->hmm_pfns[i]);
-	gtt->range = range;
 	mmput(mm);
-	return 0;
-out_unlock:
-	mmap_read_unlock(mm);
-out_free_pfns:
-	kvfree(range->hmm_pfns);
-out_free_ranges:
-	kfree(range);
-out:
-	mmput(mm);
 	return r;
 }
 		 * FIXME: Must always hold notifier_lock for this, and must
 		 * not ignore the return code.
 		 */
-		r = mmu_interval_read_retry(gtt->range->notifier,
+		r = amdgpu_hmm_range_get_pages_done(gtt->range);
-					 gtt->range->notifier_seq);
-		kvfree(gtt->range->hmm_pfns);
-		kfree(gtt->range);
 		gtt->range = NULL;
 	}

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c+7−3

 	struct drm_device *ddev = adev_to_drm(adev);
 	/* enable virtual display */
-	if (adev->mode_info.num_crtc == 0)
+	if (adev->asic_type != CHIP_ALDEBARAN &&
-		adev->mode_info.num_crtc = 1;
+	    adev->asic_type != CHIP_ARCTURUS) {
-	adev->enable_virtual_display = true;
+		if (adev->mode_info.num_crtc == 0)
+			adev->mode_info.num_crtc = 1;
+		adev->enable_virtual_display = true;
+	}
 	ddev->driver_features &= ~DRIVER_ATOMIC;
 	adev->cg_flags = 0;
 	adev->pg_flags = 0;
 		case CHIP_VEGA10:
 		case CHIP_VEGA20:
 		case CHIP_ARCTURUS:
+		case CHIP_ALDEBARAN:
 			soc15_set_virt_ops(adev);
 			break;
 		case CHIP_NAVI10:

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c+72−52

@@ -38,6 +38,7 @@ · 37 unmodified lines
38	#include "amdgpu_gmc.h"	38	#include "amdgpu_gmc.h"
39	#include "amdgpu_xgmi.h"	39	#include "amdgpu_xgmi.h"
40	#include "amdgpu_dma_buf.h"	40	#include "amdgpu_dma_buf.h"
		41	#include "kfd_svm.h"
41		42
42	/**	43	/**
43	* DOC: GPUVM	44	* DOC: GPUVM
@@ -850,35 +851,60 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, · 806 unmodified lines
850	}	851	}
851		852
852	/**	853	/**
853	* amdgpu_vm_bo_param - fill in parameters for PD/PT allocation	854	* amdgpu_vm_pt_create - create bo for PD/PT
854	*	855	*
855	* @adev: amdgpu_device pointer	856	* @adev: amdgpu_device pointer
856	* @vm: requesting vm	857	* @vm: requesting vm
857	* @level: the page table level	858	* @level: the page table level
858	* @immediate: use a immediate update	859	* @immediate: use a immediate update
859	* @bp: resulting BO allocation parameters	860	* @bo: pointer to the buffer object pointer
860	*/	861	*/
861	static void amdgpu_vm_bo_param(struct amdgpu_device adev, struct amdgpu_vm vm,	862	static int amdgpu_vm_pt_create(struct amdgpu_device *adev,
		863	struct amdgpu_vm *vm,
862	int level, bool immediate,	864	int level, bool immediate,
863	struct amdgpu_bo_param *bp)	865	struct amdgpu_bo **bo)
864	{	866	{
865	memset(bp, 0, sizeof(*bp));	867	struct amdgpu_bo_param bp;
		868	int r;
866		869
867	bp->size = amdgpu_vm_bo_size(adev, level);	870	memset(&bp, 0, sizeof(bp));
868	bp->byte_align = AMDGPU_GPU_PAGE_SIZE;	871
869	bp->domain = AMDGPU_GEM_DOMAIN_VRAM;	872	bp.size = amdgpu_vm_bo_size(adev, level);
870	bp->domain = amdgpu_bo_get_preferred_pin_domain(adev, bp->domain);	873	bp.byte_align = AMDGPU_GPU_PAGE_SIZE;
871	bp->flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS \|	874	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
		875	bp.domain = amdgpu_bo_get_preferred_pin_domain(adev, bp.domain);
		876	bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS \|
872	AMDGPU_GEM_CREATE_CPU_GTT_USWC;	877	AMDGPU_GEM_CREATE_CPU_GTT_USWC;
873	bp->bo_ptr_size = sizeof(struct amdgpu_bo);	878	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
874	if (vm->use_cpu_for_update)	879	if (vm->use_cpu_for_update)
875	bp->flags \|= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;	880	bp.flags \|= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
876	else if (!vm->root.base.bo \|\| vm->root.base.bo->shadow)	881
877	bp->flags \|= AMDGPU_GEM_CREATE_SHADOW;	882	bp.type = ttm_bo_type_kernel;
878	bp->type = ttm_bo_type_kernel;	883	bp.no_wait_gpu = immediate;
879	bp->no_wait_gpu = immediate;
880	if (vm->root.base.bo)	884	if (vm->root.base.bo)
881	bp->resv = vm->root.base.bo->tbo.base.resv;	885	bp.resv = vm->root.base.bo->tbo.base.resv;
		886
		887	r = amdgpu_bo_create(adev, &bp, bo);
		888	if (r)
		889	return r;
		890
		891	if (vm->is_compute_context && (adev->flags & AMD_IS_APU))
		892	return 0;
		893
		894	if (!bp.resv)
		895	WARN_ON(dma_resv_lock((*bo)->tbo.base.resv,
		896	NULL));
		897	r = amdgpu_bo_create_shadow(adev, bp.size, *bo);
		898
		899	if (!bp.resv)
		900	dma_resv_unlock((*bo)->tbo.base.resv);
		901
		902	if (r) {
		903	amdgpu_bo_unref(bo);
		904	return r;
		905	}
		906
		907	return 0;
882	}	908	}
883		909
884	/**	910	/**
@@ -901,7 +927,6 @@ static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, · 16 unmodified lines
901	bool immediate)	927	bool immediate)
902	{	928	{
903	struct amdgpu_vm_pt *entry = cursor->entry;	929	struct amdgpu_vm_pt *entry = cursor->entry;
904	struct amdgpu_bo_param bp;
905	struct amdgpu_bo *pt;	930	struct amdgpu_bo *pt;
906	int r;	931	int r;
907		932
@@ -919,9 +944,7 @@ static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, · 11 unmodified lines
919	if (entry->base.bo)	944	if (entry->base.bo)
920	return 0;	945	return 0;
921		946
922	amdgpu_vm_bo_param(adev, vm, cursor->level, immediate, &bp);	947	r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt);
923
924	r = amdgpu_bo_create(adev, &bp, &pt);
925	if (r)	948	if (r)
926	return r;	949	return r;
927		950
@@ -1593,15 +1616,15 @@ static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params, · 665 unmodified lines
1593	* Returns:	1616	* Returns:
1594	* 0 for success, -EINVAL for failure.	1617	* 0 for success, -EINVAL for failure.
1595	*/	1618	*/
1596	static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,	1619	int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
1597	struct amdgpu_device *bo_adev,	1620	struct amdgpu_device *bo_adev,
1598	struct amdgpu_vm *vm, bool immediate,	1621	struct amdgpu_vm *vm, bool immediate,
1599	bool unlocked, struct dma_resv *resv,	1622	bool unlocked, struct dma_resv *resv,
1600	uint64_t start, uint64_t last,	1623	uint64_t start, uint64_t last,
1601	uint64_t flags, uint64_t offset,	1624	uint64_t flags, uint64_t offset,
1602	struct drm_mm_node *nodes,	1625	struct drm_mm_node *nodes,
1603	dma_addr_t *pages_addr,	1626	dma_addr_t *pages_addr,
1604	struct dma_fence **fence)	1627	struct dma_fence **fence)
1605	{	1628	{
1606	struct amdgpu_vm_update_params params;	1629	struct amdgpu_vm_update_params params;
1607	enum amdgpu_sync_mode sync_mode;	1630	enum amdgpu_sync_mode sync_mode;
@@ -2818,7 +2841,6 @@ long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout) · 1210 unmodified lines
2818	*	2841	*
2819	* @adev: amdgpu_device pointer	2842	* @adev: amdgpu_device pointer
2820	* @vm: requested vm	2843	* @vm: requested vm
2821	* @vm_context: Indicates if it GFX or Compute context
2822	* @pasid: Process address space identifier	2844	* @pasid: Process address space identifier
2823	*	2845	*
2824	* Init @vm fields.	2846	* Init @vm fields.
@@ -2826,10 +2848,8 @@ long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout) · 1 unmodified line
2826	* Returns:	2848	* Returns:
2827	* 0 for success, error for failure.	2849	* 0 for success, error for failure.
2828	*/	2850	*/
2829	int amdgpu_vm_init(struct amdgpu_device adev, struct amdgpu_vm vm,	2851	int amdgpu_vm_init(struct amdgpu_device adev, struct amdgpu_vm vm, u32 pasid)
2830	int vm_context, u32 pasid)
2831	{	2852	{
2832	struct amdgpu_bo_param bp;
2833	struct amdgpu_bo *root;	2853	struct amdgpu_bo *root;
2834	int r, i;	2854	int r, i;
2835		2855
@@ -2861,16 +2881,9 @@ int amdgpu_vm_init(struct amdgpu_device adev, struct amdgpu_vm vm, · 25 unmodified lines
2861	vm->pte_support_ats = false;	2881	vm->pte_support_ats = false;
2862	vm->is_compute_context = false;	2882	vm->is_compute_context = false;
2863		2883
2864	if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {	2884	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2865	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &	2885	AMDGPU_VM_USE_CPU_FOR_GFX);
2866	AMDGPU_VM_USE_CPU_FOR_COMPUTE);
2867		2886
2868	if (adev->asic_type == CHIP_RAVEN)
2869	vm->pte_support_ats = true;
2870	} else {
2871	vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2872	AMDGPU_VM_USE_CPU_FOR_GFX);
2873	}
2874	DRM_DEBUG_DRIVER("VM update mode is %s\n",	2887	DRM_DEBUG_DRIVER("VM update mode is %s\n",
2875	vm->use_cpu_for_update ? "CPU" : "SDMA");	2888	vm->use_cpu_for_update ? "CPU" : "SDMA");
2876	WARN_ONCE((vm->use_cpu_for_update &&	2889	WARN_ONCE((vm->use_cpu_for_update &&
@@ -2887,10 +2900,8 @@ int amdgpu_vm_init(struct amdgpu_device adev, struct amdgpu_vm vm, · 10 unmodified lines
2887	mutex_init(&vm->eviction_lock);	2900	mutex_init(&vm->eviction_lock);
2888	vm->evicting = false;	2901	vm->evicting = false;
2889		2902
2890	amdgpu_vm_bo_param(adev, vm, adev->vm_manager.root_level, false, &bp);	2903	r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,
2891	if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE)	2904	false, &root);
2892	bp.flags &= ~AMDGPU_GEM_CREATE_SHADOW;
2893	r = amdgpu_bo_create(adev, &bp, &root);
2894	if (r)	2905	if (r)
2895	goto error_free_delayed;	2906	goto error_free_delayed;
2896		2907
@@ -3349,6 +3360,7 @@ void amdgpu_vm_set_task_info(struct amdgpu_vm *vm) · 452 unmodified lines
3349	bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,	3360	bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
3350	uint64_t addr)	3361	uint64_t addr)
3351	{	3362	{
		3363	bool is_compute_context = false;
3352	struct amdgpu_bo *root;	3364	struct amdgpu_bo *root;
3353	uint64_t value, flags;	3365	uint64_t value, flags;
3354	struct amdgpu_vm *vm;	3366	struct amdgpu_vm *vm;
@@ -3356,15 +3368,25 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, · 1 unmodified line
3356		3368
3357	spin_lock(&adev->vm_manager.pasid_lock);	3369	spin_lock(&adev->vm_manager.pasid_lock);
3358	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);	3370	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
3359	if (vm)	3371	if (vm) {
3360	root = amdgpu_bo_ref(vm->root.base.bo);	3372	root = amdgpu_bo_ref(vm->root.base.bo);
3361	else	3373	is_compute_context = vm->is_compute_context;
		3374	} else {
3362	root = NULL;	3375	root = NULL;
		3376	}
3363	spin_unlock(&adev->vm_manager.pasid_lock);	3377	spin_unlock(&adev->vm_manager.pasid_lock);
3364		3378
3365	if (!root)	3379	if (!root)
3366	return false;	3380	return false;
3367		3381
		3382	addr /= AMDGPU_GPU_PAGE_SIZE;
		3383
		3384	if (is_compute_context &&
		3385	!svm_range_restore_pages(adev, pasid, addr)) {
		3386	amdgpu_bo_unref(&root);
		3387	return true;
		3388	}
		3389
3368	r = amdgpu_bo_reserve(root, true);	3390	r = amdgpu_bo_reserve(root, true);
3369	if (r)	3391	if (r)
3370	goto error_unref;	3392	goto error_unref;
@@ -3378,18 +3400,16 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, · 7 unmodified lines
3378	if (!vm)	3400	if (!vm)
3379	goto error_unlock;	3401	goto error_unlock;
3380		3402
3381	addr /= AMDGPU_GPU_PAGE_SIZE;
3382	flags = AMDGPU_PTE_VALID \| AMDGPU_PTE_SNOOPED \|	3403	flags = AMDGPU_PTE_VALID \| AMDGPU_PTE_SNOOPED \|
3383	AMDGPU_PTE_SYSTEM;	3404	AMDGPU_PTE_SYSTEM;
3384		3405
3385	if (vm->is_compute_context) {	3406	if (is_compute_context) {
3386	/* Intentionally setting invalid PTE flag	3407	/* Intentionally setting invalid PTE flag
3387	* combination to force a no-retry-fault	3408	* combination to force a no-retry-fault
3388	*/	3409	*/
3389	flags = AMDGPU_PTE_EXECUTABLE \| AMDGPU_PDE_PTE \|	3410	flags = AMDGPU_PTE_EXECUTABLE \| AMDGPU_PDE_PTE \|
3390	AMDGPU_PTE_TF;	3411	AMDGPU_PTE_TF;
3391	value = 0;	3412	value = 0;
3392
3393	} else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {	3413	} else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {
3394	/* Redirect the access to the dummy page */	3414	/* Redirect the access to the dummy page */
3395	value = adev->dummy_page_addr;	3415	value = adev->dummy_page_addr;

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h+12−5

 /* max vmids dedicated for process */
 #define AMDGPU_VM_MAX_RESERVED_VMID	1
-#define AMDGPU_VM_CONTEXT_GFX 0
-#define AMDGPU_VM_CONTEXT_COMPUTE 1
 /* See vm_update_mode */
 #define AMDGPU_VM_USE_CPU_FOR_GFX (1 << 0)
 #define AMDGPU_VM_USE_CPU_FOR_COMPUTE (1 << 1)
 	spinlock_t				pasid_lock;
 };
+struct amdgpu_bo_va_mapping;
 #define amdgpu_vm_copy_pte(adev, ib, pe, src, count) ((adev)->vm_manager.vm_pte_funcs->copy_pte((ib), (pe), (src), (count)))
 #define amdgpu_vm_write_pte(adev, ib, pe, value, count, incr) ((adev)->vm_manager.vm_pte_funcs->write_pte((ib), (pe), (value), (count), (incr)))
 #define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags)))
 void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
 long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout);
-int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, u32 pasid);
-		   int vm_context, u32 pasid);
 int amdgpu_vm_make_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm, u32 pasid);
 void amdgpu_vm_release_compute(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 			  struct dma_fence **fence);
 int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
 			   struct amdgpu_vm *vm);
+int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
+				struct amdgpu_device *bo_adev,
+				struct amdgpu_vm *vm, bool immediate,
+				bool unlocked, struct dma_resv *resv,
+				uint64_t start, uint64_t last,
+				uint64_t flags, uint64_t offset,
+				struct drm_mm_node *nodes,
+				dma_addr_t *pages_addr,
+				struct dma_fence **fence);
 int amdgpu_vm_bo_update(struct amdgpu_device *adev,
 			struct amdgpu_bo_va *bo_va,
 			bool clear);

drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c+120−112

@@ -29,12 +29,14 @@ · 28 unmodified lines
29	#include "amdgpu_atomfirmware.h"	29	#include "amdgpu_atomfirmware.h"
30	#include "atom.h"	30	#include "atom.h"
31		31
32	static inline struct amdgpu_vram_mgr to_vram_mgr(struct ttm_resource_manager man)	32	static inline struct amdgpu_vram_mgr *
		33	to_vram_mgr(struct ttm_resource_manager *man)
33	{	34	{
34	return container_of(man, struct amdgpu_vram_mgr, manager);	35	return container_of(man, struct amdgpu_vram_mgr, manager);
35	}	36	}
36		37
37	static inline struct amdgpu_device to_amdgpu_device(struct amdgpu_vram_mgr mgr)	38	static inline struct amdgpu_device *
		39	to_amdgpu_device(struct amdgpu_vram_mgr *mgr)
38	{	40	{
39	return container_of(mgr, struct amdgpu_device, mman.vram_mgr);	41	return container_of(mgr, struct amdgpu_device, mman.vram_mgr);
40	}	42	}
@@ -82,12 +84,14 @@ static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev, · 41 unmodified lines
82	* amount of currently used VRAM in bytes	84	* amount of currently used VRAM in bytes
83	*/	85	*/
84	static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev,	86	static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev,
85	struct device_attribute attr, char buf)	87	struct device_attribute *attr,
		88	char *buf)
86	{	89	{
87	struct drm_device *ddev = dev_get_drvdata(dev);	90	struct drm_device *ddev = dev_get_drvdata(dev);
88	struct amdgpu_device *adev = drm_to_adev(ddev);	91	struct amdgpu_device *adev = drm_to_adev(ddev);
89	struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);	92	struct ttm_resource_manager *man;
90		93
		94	man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
91	return sysfs_emit(buf, "%llu\n", amdgpu_vram_mgr_usage(man));	95	return sysfs_emit(buf, "%llu\n", amdgpu_vram_mgr_usage(man));
92	}	96	}
93		97
@@ -100,18 +104,28 @@ static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev, · 6 unmodified lines
100	* amount of currently used visible VRAM in bytes	104	* amount of currently used visible VRAM in bytes
101	*/	105	*/
102	static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev,	106	static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev,
103	struct device_attribute attr, char buf)	107	struct device_attribute *attr,
		108	char *buf)
104	{	109	{
105	struct drm_device *ddev = dev_get_drvdata(dev);	110	struct drm_device *ddev = dev_get_drvdata(dev);
106	struct amdgpu_device *adev = drm_to_adev(ddev);	111	struct amdgpu_device *adev = drm_to_adev(ddev);
107	struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);	112	struct ttm_resource_manager *man;
108		113
		114	man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
109	return sysfs_emit(buf, "%llu\n", amdgpu_vram_mgr_vis_usage(man));	115	return sysfs_emit(buf, "%llu\n", amdgpu_vram_mgr_vis_usage(man));
110	}	116	}
111		117
		118	/**
		119	* DOC: mem_info_vram_vendor
		120	*
		121	* The amdgpu driver provides a sysfs API for reporting the vendor of the
		122	* installed VRAM
		123	* The file mem_info_vram_vendor is used for this and returns the name of the
		124	* vendor.
		125	*/
112	static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev,	126	static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev,
113	struct device_attribute *attr,	127	struct device_attribute *attr,
114	char *buf)	128	char *buf)
115	{	129	{
116	struct drm_device *ddev = dev_get_drvdata(dev);	130	struct drm_device *ddev = dev_get_drvdata(dev);
117	struct amdgpu_device *adev = drm_to_adev(ddev);	131	struct amdgpu_device *adev = drm_to_adev(ddev);
@@ -162,78 +176,6 @@ static const struct attribute *amdgpu_vram_mgr_attributes[] = { · 44 unmodified lines
162	NULL	176	NULL
163	};	177	};
164		178
165	static const struct ttm_resource_manager_func amdgpu_vram_mgr_func;
166
167	/**
168	* amdgpu_vram_mgr_init - init VRAM manager and DRM MM
169	*
170	* @adev: amdgpu_device pointer
171	*
172	* Allocate and initialize the VRAM manager.
173	*/
174	int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
175	{
176	struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
177	struct ttm_resource_manager *man = &mgr->manager;
178	int ret;
179
180	ttm_resource_manager_init(man, adev->gmc.real_vram_size >> PAGE_SHIFT);
181
182	man->func = &amdgpu_vram_mgr_func;
183
184	drm_mm_init(&mgr->mm, 0, man->size);
185	spin_lock_init(&mgr->lock);
186	INIT_LIST_HEAD(&mgr->reservations_pending);
187	INIT_LIST_HEAD(&mgr->reserved_pages);
188
189	/* Add the two VRAM-related sysfs files */
190	ret = sysfs_create_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes);
191	if (ret)
192	DRM_ERROR("Failed to register sysfs\n");
193
194	ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);
195	ttm_resource_manager_set_used(man, true);
196	return 0;
197	}
198
199	/**
200	* amdgpu_vram_mgr_fini - free and destroy VRAM manager
201	*
202	* @adev: amdgpu_device pointer
203	*
204	* Destroy and free the VRAM manager, returns -EBUSY if ranges are still
205	* allocated inside it.
206	*/
207	void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
208	{
209	struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
210	struct ttm_resource_manager *man = &mgr->manager;
211	int ret;
212	struct amdgpu_vram_reservation rsv, temp;
213
214	ttm_resource_manager_set_used(man, false);
215
216	ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
217	if (ret)
218	return;
219
220	spin_lock(&mgr->lock);
221	list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node)
222	kfree(rsv);
223
224	list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, node) {
225	drm_mm_remove_node(&rsv->mm_node);
226	kfree(rsv);
227	}
228	drm_mm_takedown(&mgr->mm);
229	spin_unlock(&mgr->lock);
230
231	sysfs_remove_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes);
232
233	ttm_resource_manager_cleanup(man);
234	ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL);
235	}
236
237	/**	179	/**
238	* amdgpu_vram_mgr_vis_size - Calculate visible node size	180	* amdgpu_vram_mgr_vis_size - Calculate visible node size
239	*	181	*
@@ -283,6 +225,7 @@ u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) · 43 unmodified lines
283	return usage;	225	return usage;
284	}	226	}
285		227
		228	/* Commit the reservation of VRAM pages */
286	static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man)	229	static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man)
287	{	230	{
288	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);	231	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
@@ -415,13 +358,13 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, · 126 unmodified lines
415	const struct ttm_place *place,	358	const struct ttm_place *place,
416	struct ttm_resource *mem)	359	struct ttm_resource *mem)
417	{	360	{
		361	unsigned long lpfn, num_nodes, pages_per_node, pages_left, pages;
418	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);	362	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
419	struct amdgpu_device *adev = to_amdgpu_device(mgr);	363	struct amdgpu_device *adev = to_amdgpu_device(mgr);
		364	uint64_t vis_usage = 0, mem_bytes, max_bytes;
420	struct drm_mm *mm = &mgr->mm;	365	struct drm_mm *mm = &mgr->mm;
421	struct drm_mm_node *nodes;
422	enum drm_mm_insert_mode mode;	366	enum drm_mm_insert_mode mode;
423	unsigned long lpfn, num_nodes, pages_per_node, pages_left;	367	struct drm_mm_node *nodes;
424	uint64_t vis_usage = 0, mem_bytes, max_bytes;
425	unsigned i;	368	unsigned i;
426	int r;	369	int r;
427		370
@@ -448,10 +391,10 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, · 20 unmodified lines
448	pages_per_node = HPAGE_PMD_NR;	391	pages_per_node = HPAGE_PMD_NR;
449	#else	392	#else
450	/* default to 2MB */	393	/* default to 2MB */
451	pages_per_node = (2UL << (20UL - PAGE_SHIFT));	394	pages_per_node = 2UL << (20UL - PAGE_SHIFT);
452	#endif	395	#endif
453	pages_per_node = max((uint32_t)pages_per_node,	396	pages_per_node = max_t(uint32_t, pages_per_node,
454	tbo->page_alignment);	397	tbo->page_alignment);
455	num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node);	398	num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node);
456	}	399	}
457		400
@@ -469,42 +412,37 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, · 11 unmodified lines
469	mem->start = 0;	412	mem->start = 0;
470	pages_left = mem->num_pages;	413	pages_left = mem->num_pages;
471		414
472	spin_lock(&mgr->lock);	415	/* Limit maximum size to 2GB due to SG table limitations */
473	for (i = 0; pages_left >= pages_per_node; ++i) {	416	pages = min(pages_left, 2UL << (30 - PAGE_SHIFT));
474	unsigned long pages = rounddown_pow_of_two(pages_left);
475		417
476	/* Limit maximum size to 2GB due to SG table limitations */	418	i = 0;
477	pages = min(pages, (2UL << (30 - PAGE_SHIFT)));	419	spin_lock(&mgr->lock);
478		420	while (pages_left) {
479	r = drm_mm_insert_node_in_range(mm, &nodes[i], pages,
480	pages_per_node, 0,
481	place->fpfn, lpfn,
482	mode);
483	if (unlikely(r))
484	break;
485
486	vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);
487	amdgpu_vram_mgr_virt_start(mem, &nodes[i]);
488	pages_left -= pages;
489	}
490
491	for (; pages_left; ++i) {
492	unsigned long pages = min(pages_left, pages_per_node);
493	uint32_t alignment = tbo->page_alignment;	421	uint32_t alignment = tbo->page_alignment;
494		422
495	if (pages == pages_per_node)	423	if (pages >= pages_per_node)
496	alignment = pages_per_node;	424	alignment = pages_per_node;
497		425
498	r = drm_mm_insert_node_in_range(mm, &nodes[i],	426	r = drm_mm_insert_node_in_range(mm, &nodes[i], pages, alignment,
499	pages, alignment, 0,	427	0, place->fpfn, lpfn, mode);
500	place->fpfn, lpfn,	428	if (unlikely(r)) {
501	mode);	429	if (pages > pages_per_node) {
502	if (unlikely(r))	430	if (is_power_of_2(pages))
		431	pages = pages / 2;
		432	else
		433	pages = rounddown_pow_of_two(pages);
		434	continue;
		435	}
503	goto error;	436	goto error;
		437	}
504		438
505	vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);	439	vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);
506	amdgpu_vram_mgr_virt_start(mem, &nodes[i]);	440	amdgpu_vram_mgr_virt_start(mem, &nodes[i]);
507	pages_left -= pages;	441	pages_left -= pages;
		442	++i;
		443
		444	if (pages > pages_left)
		445	pages = pages_left;
508	}	446	}
509	spin_unlock(&mgr->lock);	447	spin_unlock(&mgr->lock);
510		448
@@ -728,3 +666,73 @@ static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = { · 217 unmodified lines
728	.free = amdgpu_vram_mgr_del,	666	.free = amdgpu_vram_mgr_del,
729	.debug = amdgpu_vram_mgr_debug	667	.debug = amdgpu_vram_mgr_debug
730	};	668	};
		669
		670	/**
		671	* amdgpu_vram_mgr_init - init VRAM manager and DRM MM
		672	*
		673	* @adev: amdgpu_device pointer
		674	*
		675	* Allocate and initialize the VRAM manager.
		676	*/
		677	int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
		678	{
		679	struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
		680	struct ttm_resource_manager *man = &mgr->manager;
		681	int ret;
		682
		683	ttm_resource_manager_init(man, adev->gmc.real_vram_size >> PAGE_SHIFT);
		684
		685	man->func = &amdgpu_vram_mgr_func;
		686
		687	drm_mm_init(&mgr->mm, 0, man->size);
		688	spin_lock_init(&mgr->lock);
		689	INIT_LIST_HEAD(&mgr->reservations_pending);
		690	INIT_LIST_HEAD(&mgr->reserved_pages);
		691
		692	/* Add the two VRAM-related sysfs files */
		693	ret = sysfs_create_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes);
		694	if (ret)
		695	DRM_ERROR("Failed to register sysfs\n");
		696
		697	ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);
		698	ttm_resource_manager_set_used(man, true);
		699	return 0;
		700	}
		701
		702	/**
		703	* amdgpu_vram_mgr_fini - free and destroy VRAM manager
		704	*
		705	* @adev: amdgpu_device pointer
		706	*
		707	* Destroy and free the VRAM manager, returns -EBUSY if ranges are still
		708	* allocated inside it.
		709	*/
		710	void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
		711	{
		712	struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
		713	struct ttm_resource_manager *man = &mgr->manager;
		714	int ret;
		715	struct amdgpu_vram_reservation rsv, temp;
		716
		717	ttm_resource_manager_set_used(man, false);
		718
		719	ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
		720	if (ret)
		721	return;
		722
		723	spin_lock(&mgr->lock);
		724	list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node)
		725	kfree(rsv);
		726
		727	list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, node) {
		728	drm_mm_remove_node(&rsv->mm_node);
		729	kfree(rsv);
		730	}
		731	drm_mm_takedown(&mgr->mm);
		732	spin_unlock(&mgr->lock);
		733
		734	sysfs_remove_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes);
		735
		736	ttm_resource_manager_cleanup(man);
		737	ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL);
		738	}

drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h+3−3

@@ -98,9 +98,9 @@ union amd_sriov_msg_feature_flags { · 97 unmodified lines
98		98
99	union amd_sriov_reg_access_flags {	99	union amd_sriov_reg_access_flags {
100	struct {	100	struct {
101	uint32_t vf_reg_access_ih : 1;	101	uint32_t vf_reg_psp_access_ih : 1;
102	uint32_t vf_reg_access_mmhub : 1;	102	uint32_t vf_reg_rlc_access_mmhub : 1;
103	uint32_t vf_reg_access_gc : 1;	103	uint32_t vf_reg_rlc_access_gc : 1;
104	uint32_t reserved : 29;	104	uint32_t reserved : 29;
105	} flags;	105	} flags;
106	uint32_t all;	106	uint32_t all;

drivers/gpu/drm/amd/amdgpu/dce_virtual.c+5−7

 static int dce_virtual_sw_fini(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+	int i = 0;
+	for (i = 0; i < adev->mode_info.num_crtc; i++)
+		if (adev->mode_info.crtcs[i])
+			hrtimer_cancel(&adev->mode_info.crtcs[i]->vblank_timer);
 	kfree(adev->mode_info.bios_hardcoded_edid);
 static int dce_virtual_hw_fini(void *handle)
 {
-	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
-	int i = 0;
-	for (i = 0; i<adev->mode_info.num_crtc; i++)
-		if (adev->mode_info.crtcs[i])
-			hrtimer_cancel(&adev->mode_info.crtcs[i]->vblank_timer);
 	return 0;
 }

drivers/gpu/drm/amd/amdgpu/df_v3_6.c+11−6

@@ -219,11 +219,11 @@ static void df_v3_6_query_hashes(struct amdgpu_device *adev) · 218 unmodified lines
219	adev->df.hash_status.hash_2m = false;	219	adev->df.hash_status.hash_2m = false;
220	adev->df.hash_status.hash_1g = false;	220	adev->df.hash_status.hash_1g = false;
221		221
222	if (adev->asic_type != CHIP_ARCTURUS)	222	/* encoding for hash-enabled on Arcturus and Aldebaran */
223	return;	223	if ((adev->asic_type == CHIP_ARCTURUS &&
224		224	adev->df.funcs->get_fb_channel_number(adev) == 0xe) \|\|
225	/* encoding for hash-enabled on Arcturus */	225	(adev->asic_type == CHIP_ALDEBARAN &&
226	if (adev->df.funcs->get_fb_channel_number(adev) == 0xe) {	226	adev->df.funcs->get_fb_channel_number(adev) == 0x1e)) {
227	tmp = RREG32_SOC15(DF, 0, mmDF_CS_UMC_AON0_DfGlobalCtrl);	227	tmp = RREG32_SOC15(DF, 0, mmDF_CS_UMC_AON0_DfGlobalCtrl);
228	adev->df.hash_status.hash_64k = REG_GET_FIELD(tmp,	228	adev->df.hash_status.hash_64k = REG_GET_FIELD(tmp,
229	DF_CS_UMC_AON0_DfGlobalCtrl,	229	DF_CS_UMC_AON0_DfGlobalCtrl,
@@ -278,7 +278,12 @@ static u32 df_v3_6_get_fb_channel_number(struct amdgpu_device *adev) · 48 unmodified lines
278	u32 tmp;	278	u32 tmp;
279		279
280	tmp = RREG32_SOC15(DF, 0, mmDF_CS_UMC_AON0_DramBaseAddress0);	280	tmp = RREG32_SOC15(DF, 0, mmDF_CS_UMC_AON0_DramBaseAddress0);
281	tmp &= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK;	281	if (adev->asic_type == CHIP_ALDEBARAN)
		282	tmp &=
		283	ALDEBARAN_DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK;
		284	else
		285	tmp &= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK;
		286
282	tmp >>= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;	287	tmp >>= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;
283		288
284	return tmp;	289	return tmp;

drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c+8−4

 {
 	u32 tmp;
-	if (adev->asic_type != CHIP_ARCTURUS)
+	if (adev->asic_type != CHIP_ARCTURUS &&
+	    adev->asic_type != CHIP_ALDEBARAN)
 		return;
 	tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
 	if (!ring->sched.ready)
 		return 0;
-	if (adev->asic_type == CHIP_ARCTURUS ||
+	if (adev->asic_type == CHIP_ARCTURUS) {
-	    adev->asic_type == CHIP_ALDEBARAN) {
 		vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
 		vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
 		vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
 	}
 	/* requires IBs so do in late init after IB pool is initialized */
-	r = gfx_v9_0_do_edc_gpr_workarounds(adev);
+	if (adev->asic_type == CHIP_ALDEBARAN)
+		r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);
+	else
+		r = gfx_v9_0_do_edc_gpr_workarounds(adev);
 	if (r)
 		return r;

drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c+641−3

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	*/	22	*/
23	#include "amdgpu.h"	23	#include "amdgpu.h"
24	#include "soc15.h"	24	#include "soc15.h"
		25	#include "soc15d.h"
25		26
26	#include "gc/gc_9_4_2_offset.h"	27	#include "gc/gc_9_4_2_offset.h"
27	#include "gc/gc_9_4_2_sh_mask.h"	28	#include "gc/gc_9_4_2_sh_mask.h"
@@ -31,6 +32,11 @@ · 3 unmodified lines
31	#include "amdgpu_ras.h"	32	#include "amdgpu_ras.h"
32	#include "amdgpu_gfx.h"	33	#include "amdgpu_gfx.h"
33		34
		35	#define SE_ID_MAX 8
		36	#define CU_ID_MAX 16
		37	#define SIMD_ID_MAX 4
		38	#define WAVE_ID_MAX 10
		39
34	enum gfx_v9_4_2_utc_type {	40	enum gfx_v9_4_2_utc_type {
35	VML2_MEM,	41	VML2_MEM,
36	VML2_WALKER_MEM,	42	VML2_WALKER_MEM,
@@ -79,6 +85,634 @@ static const struct soc15_reg_golden golden_settings_gc_9_4_2_alde[] = { · 42 unmodified lines
79	SOC15_REG_GOLDEN_VALUE(GC, 0, regTCI_CNTL_3, 0xff, 0x20),	85	SOC15_REG_GOLDEN_VALUE(GC, 0, regTCI_CNTL_3, 0xff, 0x20),
80	};	86	};
81		87
		88	/**
		89	* This shader is used to clear VGPRS and LDS, and also write the input
		90	* pattern into the write back buffer, which will be used by driver to
		91	* check whether all SIMDs have been covered.
		92	*/
		93	static const u32 vgpr_init_compute_shader_aldebaran[] = {
		94	0xb8840904, 0xb8851a04, 0xb8861344, 0xb8831804, 0x9208ff06, 0x00000280,
		95	0x9209a805, 0x920a8a04, 0x81080908, 0x81080a08, 0x81080308, 0x8e078208,
		96	0x81078407, 0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8a0000, 0xd3d94000,
		97	0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 0xd3d94003,
		98	0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 0xd3d94006,
		99	0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 0xd3d94009,
		100	0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 0xd3d9400c,
		101	0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 0xd3d9400f,
		102	0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 0xd3d94012,
		103	0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 0xd3d94015,
		104	0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 0xd3d94018,
		105	0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 0xd3d9401b,
		106	0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 0xd3d9401e,
		107	0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 0xd3d94021,
		108	0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 0xd3d94024,
		109	0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 0xd3d94027,
		110	0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 0xd3d9402a,
		111	0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 0xd3d9402d,
		112	0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 0xd3d94030,
		113	0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 0xd3d94033,
		114	0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 0xd3d94036,
		115	0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 0xd3d94039,
		116	0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 0xd3d9403c,
		117	0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 0xd3d9403f,
		118	0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 0xd3d94042,
		119	0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 0xd3d94045,
		120	0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 0xd3d94048,
		121	0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 0xd3d9404b,
		122	0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 0xd3d9404e,
		123	0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 0xd3d94051,
		124	0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 0xd3d94054,
		125	0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 0xd3d94057,
		126	0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 0xd3d9405a,
		127	0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 0xd3d9405d,
		128	0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 0xd3d94060,
		129	0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 0xd3d94063,
		130	0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 0xd3d94066,
		131	0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 0xd3d94069,
		132	0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 0xd3d9406c,
		133	0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 0xd3d9406f,
		134	0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 0xd3d94072,
		135	0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 0xd3d94075,
		136	0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 0xd3d94078,
		137	0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 0xd3d9407b,
		138	0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 0xd3d9407e,
		139	0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 0xd3d94081,
		140	0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 0xd3d94084,
		141	0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 0xd3d94087,
		142	0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 0xd3d9408a,
		143	0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 0xd3d9408d,
		144	0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 0xd3d94090,
		145	0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 0xd3d94093,
		146	0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 0xd3d94096,
		147	0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 0xd3d94099,
		148	0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 0xd3d9409c,
		149	0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 0xd3d9409f,
		150	0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 0xd3d940a2,
		151	0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 0xd3d940a5,
		152	0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 0xd3d940a8,
		153	0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 0xd3d940ab,
		154	0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 0xd3d940ae,
		155	0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 0xd3d940b1,
		156	0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 0xd3d940b4,
		157	0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 0xd3d940b7,
		158	0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 0xd3d940ba,
		159	0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 0xd3d940bd,
		160	0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 0xd3d940c0,
		161	0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 0xd3d940c3,
		162	0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 0xd3d940c6,
		163	0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 0xd3d940c9,
		164	0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 0xd3d940cc,
		165	0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 0xd3d940cf,
		166	0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 0xd3d940d2,
		167	0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 0xd3d940d5,
		168	0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 0xd3d940d8,
		169	0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 0xd3d940db,
		170	0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 0xd3d940de,
		171	0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 0xd3d940e1,
		172	0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 0xd3d940e4,
		173	0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 0xd3d940e7,
		174	0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 0xd3d940ea,
		175	0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 0xd3d940ed,
		176	0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 0xd3d940f0,
		177	0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 0xd3d940f3,
		178	0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 0xd3d940f6,
		179	0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 0xd3d940f9,
		180	0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 0xd3d940fc,
		181	0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 0xd3d940ff,
		182	0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 0x7e000280,
		183	0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 0x7e0c0280,
		184	0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 0xd28c0001,
		185	0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xbe8b0004, 0xb78b4000,
		186	0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 0x00020201,
		187	0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 0xbf84fff8,
		188	0xbf810000,
		189	};
		190
		191	const struct soc15_reg_entry vgpr_init_regs_aldebaran[] = {
		192	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
		193	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_X), 0x40 },
		194	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Y), 4 },
		195	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Z), 1 },
		196	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC1), 0xbf },
		197	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC2), 0x400006 }, /* 64KB LDS */
		198	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC3), 0x3F }, /* 63 - accum-offset = 256 */
		199	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
		200	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
		201	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
		202	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
		203	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
		204	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
		205	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
		206	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
		207	};
		208
		209	/**
		210	* The below shaders are used to clear SGPRS, and also write the input
		211	* pattern into the write back buffer. The first two dispatch should be
		212	* scheduled simultaneously which make sure that all SGPRS could be
		213	* allocated, so the dispatch 1 need check write back buffer before scheduled,
		214	* make sure that waves of dispatch 0 are all dispacthed to all simds
		215	* balanced. both dispatch 0 and dispatch 1 should be halted until all waves
		216	* are dispatched, and then driver write a pattern to the shared memory to make
		217	* all waves continue.
		218	*/
		219	static const u32 sgpr112_init_compute_shader_aldebaran[] = {
		220	0xb8840904, 0xb8851a04, 0xb8861344, 0xb8831804, 0x9208ff06, 0x00000280,
		221	0x9209a805, 0x920a8a04, 0x81080908, 0x81080a08, 0x81080308, 0x8e078208,
		222	0x81078407, 0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8e003f, 0xc0030200,
		223	0x00000000, 0xbf8c0000, 0xbf06ff08, 0xdeadbeaf, 0xbf84fff9, 0x81028102,
		224	0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8a0000, 0xbefc0080, 0xbeea0080,
		225	0xbeeb0080, 0xbf00f280, 0xbee60080, 0xbee70080, 0xbee80080, 0xbee90080,
		226	0xbefe0080, 0xbeff0080, 0xbe880080, 0xbe890080, 0xbe8a0080, 0xbe8b0080,
		227	0xbe8c0080, 0xbe8d0080, 0xbe8e0080, 0xbe8f0080, 0xbe900080, 0xbe910080,
		228	0xbe920080, 0xbe930080, 0xbe940080, 0xbe950080, 0xbe960080, 0xbe970080,
		229	0xbe980080, 0xbe990080, 0xbe9a0080, 0xbe9b0080, 0xbe9c0080, 0xbe9d0080,
		230	0xbe9e0080, 0xbe9f0080, 0xbea00080, 0xbea10080, 0xbea20080, 0xbea30080,
		231	0xbea40080, 0xbea50080, 0xbea60080, 0xbea70080, 0xbea80080, 0xbea90080,
		232	0xbeaa0080, 0xbeab0080, 0xbeac0080, 0xbead0080, 0xbeae0080, 0xbeaf0080,
		233	0xbeb00080, 0xbeb10080, 0xbeb20080, 0xbeb30080, 0xbeb40080, 0xbeb50080,
		234	0xbeb60080, 0xbeb70080, 0xbeb80080, 0xbeb90080, 0xbeba0080, 0xbebb0080,
		235	0xbebc0080, 0xbebd0080, 0xbebe0080, 0xbebf0080, 0xbec00080, 0xbec10080,
		236	0xbec20080, 0xbec30080, 0xbec40080, 0xbec50080, 0xbec60080, 0xbec70080,
		237	0xbec80080, 0xbec90080, 0xbeca0080, 0xbecb0080, 0xbecc0080, 0xbecd0080,
		238	0xbece0080, 0xbecf0080, 0xbed00080, 0xbed10080, 0xbed20080, 0xbed30080,
		239	0xbed40080, 0xbed50080, 0xbed60080, 0xbed70080, 0xbed80080, 0xbed90080,
		240	0xbeda0080, 0xbedb0080, 0xbedc0080, 0xbedd0080, 0xbede0080, 0xbedf0080,
		241	0xbee00080, 0xbee10080, 0xbee20080, 0xbee30080, 0xbee40080, 0xbee50080,
		242	0xbf810000
		243	};
		244
		245	const struct soc15_reg_entry sgpr112_init_regs_aldebaran[] = {
		246	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
		247	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_X), 0x40 },
		248	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Y), 8 },
		249	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Z), 1 },
		250	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC1), 0x340 },
		251	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC2), 0x6 },
		252	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC3), 0x0 },
		253	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
		254	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
		255	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
		256	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
		257	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
		258	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
		259	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
		260	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
		261	};
		262
		263	static const u32 sgpr96_init_compute_shader_aldebaran[] = {
		264	0xb8840904, 0xb8851a04, 0xb8861344, 0xb8831804, 0x9208ff06, 0x00000280,
		265	0x9209a805, 0x920a8a04, 0x81080908, 0x81080a08, 0x81080308, 0x8e078208,
		266	0x81078407, 0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8e003f, 0xc0030200,
		267	0x00000000, 0xbf8c0000, 0xbf06ff08, 0xdeadbeaf, 0xbf84fff9, 0x81028102,
		268	0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8a0000, 0xbefc0080, 0xbeea0080,
		269	0xbeeb0080, 0xbf00f280, 0xbee60080, 0xbee70080, 0xbee80080, 0xbee90080,
		270	0xbefe0080, 0xbeff0080, 0xbe880080, 0xbe890080, 0xbe8a0080, 0xbe8b0080,
		271	0xbe8c0080, 0xbe8d0080, 0xbe8e0080, 0xbe8f0080, 0xbe900080, 0xbe910080,
		272	0xbe920080, 0xbe930080, 0xbe940080, 0xbe950080, 0xbe960080, 0xbe970080,
		273	0xbe980080, 0xbe990080, 0xbe9a0080, 0xbe9b0080, 0xbe9c0080, 0xbe9d0080,
		274	0xbe9e0080, 0xbe9f0080, 0xbea00080, 0xbea10080, 0xbea20080, 0xbea30080,
		275	0xbea40080, 0xbea50080, 0xbea60080, 0xbea70080, 0xbea80080, 0xbea90080,
		276	0xbeaa0080, 0xbeab0080, 0xbeac0080, 0xbead0080, 0xbeae0080, 0xbeaf0080,
		277	0xbeb00080, 0xbeb10080, 0xbeb20080, 0xbeb30080, 0xbeb40080, 0xbeb50080,
		278	0xbeb60080, 0xbeb70080, 0xbeb80080, 0xbeb90080, 0xbeba0080, 0xbebb0080,
		279	0xbebc0080, 0xbebd0080, 0xbebe0080, 0xbebf0080, 0xbec00080, 0xbec10080,
		280	0xbec20080, 0xbec30080, 0xbec40080, 0xbec50080, 0xbec60080, 0xbec70080,
		281	0xbec80080, 0xbec90080, 0xbeca0080, 0xbecb0080, 0xbecc0080, 0xbecd0080,
		282	0xbece0080, 0xbecf0080, 0xbed00080, 0xbed10080, 0xbed20080, 0xbed30080,
		283	0xbed40080, 0xbed50080, 0xbed60080, 0xbed70080, 0xbed80080, 0xbed90080,
		284	0xbf810000,
		285	};
		286
		287	const struct soc15_reg_entry sgpr96_init_regs_aldebaran[] = {
		288	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
		289	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_X), 0x40 },
		290	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Y), 0xc },
		291	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Z), 1 },
		292	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC1), 0x2c0 },
		293	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC2), 0x6 },
		294	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC3), 0x0 },
		295	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
		296	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
		297	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
		298	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
		299	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
		300	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
		301	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
		302	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
		303	};
		304
		305	/**
		306	* This shader is used to clear the uninitiated sgprs after the above
		307	* two dispatches, because of hardware feature, dispath 0 couldn't clear
		308	* top hole sgprs. Therefore need 4 waves per SIMD to cover these sgprs
		309	*/
		310	static const u32 sgpr64_init_compute_shader_aldebaran[] = {
		311	0xb8840904, 0xb8851a04, 0xb8861344, 0xb8831804, 0x9208ff06, 0x00000280,
		312	0x9209a805, 0x920a8a04, 0x81080908, 0x81080a08, 0x81080308, 0x8e078208,
		313	0x81078407, 0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8e003f, 0xc0030200,
		314	0x00000000, 0xbf8c0000, 0xbf06ff08, 0xdeadbeaf, 0xbf84fff9, 0x81028102,
		315	0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8a0000, 0xbefc0080, 0xbeea0080,
		316	0xbeeb0080, 0xbf00f280, 0xbee60080, 0xbee70080, 0xbee80080, 0xbee90080,
		317	0xbefe0080, 0xbeff0080, 0xbe880080, 0xbe890080, 0xbe8a0080, 0xbe8b0080,
		318	0xbe8c0080, 0xbe8d0080, 0xbe8e0080, 0xbe8f0080, 0xbe900080, 0xbe910080,
		319	0xbe920080, 0xbe930080, 0xbe940080, 0xbe950080, 0xbe960080, 0xbe970080,
		320	0xbe980080, 0xbe990080, 0xbe9a0080, 0xbe9b0080, 0xbe9c0080, 0xbe9d0080,
		321	0xbe9e0080, 0xbe9f0080, 0xbea00080, 0xbea10080, 0xbea20080, 0xbea30080,
		322	0xbea40080, 0xbea50080, 0xbea60080, 0xbea70080, 0xbea80080, 0xbea90080,
		323	0xbeaa0080, 0xbeab0080, 0xbeac0080, 0xbead0080, 0xbeae0080, 0xbeaf0080,
		324	0xbeb00080, 0xbeb10080, 0xbeb20080, 0xbeb30080, 0xbeb40080, 0xbeb50080,
		325	0xbeb60080, 0xbeb70080, 0xbeb80080, 0xbeb90080, 0xbf810000,
		326	};
		327
		328	const struct soc15_reg_entry sgpr64_init_regs_aldebaran[] = {
		329	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
		330	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_X), 0x40 },
		331	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Y), 0x10 },
		332	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Z), 1 },
		333	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC1), 0x1c0 },
		334	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC2), 0x6 },
		335	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC3), 0x0 },
		336	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
		337	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
		338	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
		339	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
		340	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
		341	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
		342	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
		343	{ SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
		344	};
		345
		346	static int gfx_v9_4_2_run_shader(struct amdgpu_device *adev,
		347	struct amdgpu_ring *ring,
		348	struct amdgpu_ib *ib,
		349	const u32 *shader_ptr, u32 shader_size,
		350	const struct soc15_reg_entry *init_regs, u32 regs_size,
		351	u32 compute_dim_x, u64 wb_gpu_addr, u32 pattern,
		352	struct dma_fence **fence_ptr)
		353	{
		354	int r, i;
		355	uint32_t total_size, shader_offset;
		356	u64 gpu_addr;
		357
		358	total_size = (regs_size * 3 + 4 + 5 + 5) * 4;
		359	total_size = ALIGN(total_size, 256);
		360	shader_offset = total_size;
		361	total_size += ALIGN(shader_size, 256);
		362
		363	/* allocate an indirect buffer to put the commands in */
		364	memset(ib, 0, sizeof(*ib));
		365	r = amdgpu_ib_get(adev, NULL, total_size,
		366	AMDGPU_IB_POOL_DIRECT, ib);
		367	if (r) {
		368	dev_err(adev->dev, "failed to get ib (%d).\n", r);
		369	return r;
		370	}
		371
		372	/* load the compute shaders */
		373	for (i = 0; i < shader_size/sizeof(u32); i++)
		374	ib->ptr[i + (shader_offset / 4)] = shader_ptr[i];
		375
		376	/* init the ib length to 0 */
		377	ib->length_dw = 0;
		378
		379	/* write the register state for the compute dispatch */
		380	for (i = 0; i < regs_size; i++) {
		381	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
		382	ib->ptr[ib->length_dw++] = SOC15_REG_ENTRY_OFFSET(init_regs[i])
		383	- PACKET3_SET_SH_REG_START;
		384	ib->ptr[ib->length_dw++] = init_regs[i].reg_value;
		385	}
		386
		387	/* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
		388	gpu_addr = (ib->gpu_addr + (u64)shader_offset) >> 8;
		389	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
		390	ib->ptr[ib->length_dw++] = SOC15_REG_OFFSET(GC, 0, regCOMPUTE_PGM_LO)
		391	- PACKET3_SET_SH_REG_START;
		392	ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr);
		393	ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr);
		394
		395	/* write the wb buffer address */
		396	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 3);
		397	ib->ptr[ib->length_dw++] = SOC15_REG_OFFSET(GC, 0, regCOMPUTE_USER_DATA_0)
		398	- PACKET3_SET_SH_REG_START;
		399	ib->ptr[ib->length_dw++] = lower_32_bits(wb_gpu_addr);
		400	ib->ptr[ib->length_dw++] = upper_32_bits(wb_gpu_addr);
		401	ib->ptr[ib->length_dw++] = pattern;
		402
		403	/* write dispatch packet */
		404	ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
		405	ib->ptr[ib->length_dw++] = compute_dim_x; /* x */
		406	ib->ptr[ib->length_dw++] = 1; /* y */
		407	ib->ptr[ib->length_dw++] = 1; /* z */
		408	ib->ptr[ib->length_dw++] =
		409	REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
		410
		411	/* shedule the ib on the ring */
		412	r = amdgpu_ib_schedule(ring, 1, ib, NULL, fence_ptr);
		413	if (r) {
		414	dev_err(adev->dev, "ib submit failed (%d).\n", r);
		415	amdgpu_ib_free(adev, ib, NULL);
		416	}
		417	return r;
		418	}
		419
		420	static void gfx_v9_4_2_log_wave_assignment(struct amdgpu_device adev, uint32_t wb_ptr)
		421	{
		422	uint32_t se, cu, simd, wave;
		423	uint32_t offset = 0;
		424	char *str;
		425	int size;
		426
		427	str = kmalloc(256, GFP_KERNEL);
		428	if (!str)
		429	return;
		430
		431	dev_dbg(adev->dev, "wave assignment:\n");
		432
		433	for (se = 0; se < adev->gfx.config.max_shader_engines; se++) {
		434	for (cu = 0; cu < CU_ID_MAX; cu++) {
		435	memset(str, 0, 256);
		436	size = sprintf(str, "SE[%02d]CU[%02d]: ", se, cu);
		437	for (simd = 0; simd < SIMD_ID_MAX; simd++) {
		438	size += sprintf(str + size, "[");
		439	for (wave = 0; wave < WAVE_ID_MAX; wave++) {
		440	size += sprintf(str + size, "%x", wb_ptr[offset]);
		441	offset++;
		442	}
		443	size += sprintf(str + size, "] ");
		444	}
		445	dev_dbg(adev->dev, "%s\n", str);
		446	}
		447	}
		448
		449	kfree(str);
		450	}
		451
		452	static int gfx_v9_4_2_wait_for_waves_assigned(struct amdgpu_device *adev,
		453	uint32_t *wb_ptr, uint32_t mask,
		454	uint32_t pattern, uint32_t num_wave, bool wait)
		455	{
		456	uint32_t se, cu, simd, wave;
		457	uint32_t loop = 0;
		458	uint32_t wave_cnt;
		459	uint32_t offset;
		460
		461	do {
		462	wave_cnt = 0;
		463	offset = 0;
		464
		465	for (se = 0; se < adev->gfx.config.max_shader_engines; se++)
		466	for (cu = 0; cu < CU_ID_MAX; cu++)
		467	for (simd = 0; simd < SIMD_ID_MAX; simd++)
		468	for (wave = 0; wave < WAVE_ID_MAX; wave++) {
		469	if (((1 << wave) & mask) &&
		470	(wb_ptr[offset] == pattern))
		471	wave_cnt++;
		472
		473	offset++;
		474	}
		475
		476	if (wave_cnt == num_wave)
		477	return 0;
		478
		479	mdelay(1);
		480	} while (++loop < 2000 && wait);
		481
		482	dev_err(adev->dev, "actual wave num: %d, expected wave num: %d\n",
		483	wave_cnt, num_wave);
		484
		485	gfx_v9_4_2_log_wave_assignment(adev, wb_ptr);
		486
		487	return -EBADSLT;
		488	}
		489
		490	static int gfx_v9_4_2_do_sgprs_init(struct amdgpu_device *adev)
		491	{
		492	int r;
		493	int wb_size = adev->gfx.config.max_shader_engines *
		494	CU_ID_MAX * SIMD_ID_MAX * WAVE_ID_MAX;
		495	struct amdgpu_ib wb_ib;
		496	struct amdgpu_ib disp_ibs[3];
		497	struct dma_fence *fences[3];
		498	u32 pattern[3] = { 0x1, 0x5, 0xa };
		499
		500	/* bail if the compute ring is not ready */
		501	if (!adev->gfx.compute_ring[0].sched.ready \|\|
		502	!adev->gfx.compute_ring[1].sched.ready)
		503	return 0;
		504
		505	/* allocate the write-back buffer from IB */
		506	memset(&wb_ib, 0, sizeof(wb_ib));
		507	r = amdgpu_ib_get(adev, NULL, (1 + wb_size) * sizeof(uint32_t),
		508	AMDGPU_IB_POOL_DIRECT, &wb_ib);
		509	if (r) {
		510	dev_err(adev->dev, "failed to get ib (%d) for wb\n", r);
		511	return r;
		512	}
		513	memset(wb_ib.ptr, 0, (1 + wb_size) * sizeof(uint32_t));
		514
		515	r = gfx_v9_4_2_run_shader(adev,
		516	&adev->gfx.compute_ring[0],
		517	&disp_ibs[0],
		518	sgpr112_init_compute_shader_aldebaran,
		519	sizeof(sgpr112_init_compute_shader_aldebaran),
		520	sgpr112_init_regs_aldebaran,
		521	ARRAY_SIZE(sgpr112_init_regs_aldebaran),
		522	adev->gfx.cu_info.number,
		523	wb_ib.gpu_addr, pattern[0], &fences[0]);
		524	if (r) {
		525	dev_err(adev->dev, "failed to clear first 224 sgprs\n");
		526	goto pro_end;
		527	}
		528
		529	r = gfx_v9_4_2_wait_for_waves_assigned(adev,
		530	&wb_ib.ptr[1], 0b11,
		531	pattern[0],
		532	adev->gfx.cu_info.number * SIMD_ID_MAX * 2,
		533	true);
		534	if (r) {
		535	dev_err(adev->dev, "wave coverage failed when clear first 224 sgprs\n");
		536	wb_ib.ptr[0] = 0xdeadbeaf; /* stop waves */
		537	goto disp0_failed;
		538	}
		539
		540	r = gfx_v9_4_2_run_shader(adev,
		541	&adev->gfx.compute_ring[1],
		542	&disp_ibs[1],
		543	sgpr96_init_compute_shader_aldebaran,
		544	sizeof(sgpr96_init_compute_shader_aldebaran),
		545	sgpr96_init_regs_aldebaran,
		546	ARRAY_SIZE(sgpr96_init_regs_aldebaran),
		547	adev->gfx.cu_info.number * 2,
		548	wb_ib.gpu_addr, pattern[1], &fences[1]);
		549	if (r) {
		550	dev_err(adev->dev, "failed to clear next 576 sgprs\n");
		551	goto disp0_failed;
		552	}
		553
		554	r = gfx_v9_4_2_wait_for_waves_assigned(adev,
		555	&wb_ib.ptr[1], 0b11111100,
		556	pattern[1], adev->gfx.cu_info.number * SIMD_ID_MAX * 6,
		557	true);
		558	if (r) {
		559	dev_err(adev->dev, "wave coverage failed when clear first 576 sgprs\n");
		560	wb_ib.ptr[0] = 0xdeadbeaf; /* stop waves */
		561	goto disp1_failed;
		562	}
		563
		564	wb_ib.ptr[0] = 0xdeadbeaf; /* stop waves */
		565
		566	/* wait for the GPU to finish processing the IB */
		567	r = dma_fence_wait(fences[0], false);
		568	if (r) {
		569	dev_err(adev->dev, "timeout to clear first 224 sgprs\n");
		570	goto disp1_failed;
		571	}
		572
		573	r = dma_fence_wait(fences[1], false);
		574	if (r) {
		575	dev_err(adev->dev, "timeout to clear first 576 sgprs\n");
		576	goto disp1_failed;
		577	}
		578
		579	memset(wb_ib.ptr, 0, (1 + wb_size) * sizeof(uint32_t));
		580	r = gfx_v9_4_2_run_shader(adev,
		581	&adev->gfx.compute_ring[0],
		582	&disp_ibs[2],
		583	sgpr64_init_compute_shader_aldebaran,
		584	sizeof(sgpr64_init_compute_shader_aldebaran),
		585	sgpr64_init_regs_aldebaran,
		586	ARRAY_SIZE(sgpr64_init_regs_aldebaran),
		587	adev->gfx.cu_info.number,
		588	wb_ib.gpu_addr, pattern[2], &fences[2]);
		589	if (r) {
		590	dev_err(adev->dev, "failed to clear first 256 sgprs\n");
		591	goto disp1_failed;
		592	}
		593
		594	r = gfx_v9_4_2_wait_for_waves_assigned(adev,
		595	&wb_ib.ptr[1], 0b1111,
		596	pattern[2],
		597	adev->gfx.cu_info.number * SIMD_ID_MAX * 4,
		598	true);
		599	if (r) {
		600	dev_err(adev->dev, "wave coverage failed when clear first 256 sgprs\n");
		601	wb_ib.ptr[0] = 0xdeadbeaf; /* stop waves */
		602	goto disp2_failed;
		603	}
		604
		605	wb_ib.ptr[0] = 0xdeadbeaf; /* stop waves */
		606
		607	r = dma_fence_wait(fences[2], false);
		608	if (r) {
		609	dev_err(adev->dev, "timeout to clear first 256 sgprs\n");
		610	goto disp2_failed;
		611	}
		612
		613	disp2_failed:
		614	amdgpu_ib_free(adev, &disp_ibs[2], NULL);
		615	dma_fence_put(fences[2]);
		616	disp1_failed:
		617	amdgpu_ib_free(adev, &disp_ibs[1], NULL);
		618	dma_fence_put(fences[1]);
		619	disp0_failed:
		620	amdgpu_ib_free(adev, &disp_ibs[0], NULL);
		621	dma_fence_put(fences[0]);
		622	pro_end:
		623	amdgpu_ib_free(adev, &wb_ib, NULL);
		624
		625	if (r)
		626	dev_info(adev->dev, "Init SGPRS Failed\n");
		627	else
		628	dev_info(adev->dev, "Init SGPRS Successfully\n");
		629
		630	return r;
		631	}
		632
		633	static int gfx_v9_4_2_do_vgprs_init(struct amdgpu_device *adev)
		634	{
		635	int r;
		636	/* CU_ID: 0~15, SIMD_ID: 0~3, WAVE_ID: 0 ~ 9 */
		637	int wb_size = adev->gfx.config.max_shader_engines *
		638	CU_ID_MAX * SIMD_ID_MAX * WAVE_ID_MAX;
		639	struct amdgpu_ib wb_ib;
		640	struct amdgpu_ib disp_ib;
		641	struct dma_fence *fence;
		642	u32 pattern = 0xa;
		643
		644	/* bail if the compute ring is not ready */
		645	if (!adev->gfx.compute_ring[0].sched.ready)
		646	return 0;
		647
		648	/* allocate the write-back buffer from IB */
		649	memset(&wb_ib, 0, sizeof(wb_ib));
		650	r = amdgpu_ib_get(adev, NULL, (1 + wb_size) * sizeof(uint32_t),
		651	AMDGPU_IB_POOL_DIRECT, &wb_ib);
		652	if (r) {
		653	dev_err(adev->dev, "failed to get ib (%d) for wb.\n", r);
		654	return r;
		655	}
		656	memset(wb_ib.ptr, 0, (1 + wb_size) * sizeof(uint32_t));
		657
		658	r = gfx_v9_4_2_run_shader(adev,
		659	&adev->gfx.compute_ring[0],
		660	&disp_ib,
		661	vgpr_init_compute_shader_aldebaran,
		662	sizeof(vgpr_init_compute_shader_aldebaran),
		663	vgpr_init_regs_aldebaran,
		664	ARRAY_SIZE(vgpr_init_regs_aldebaran),
		665	adev->gfx.cu_info.number,
		666	wb_ib.gpu_addr, pattern, &fence);
		667	if (r) {
		668	dev_err(adev->dev, "failed to clear vgprs\n");
		669	goto pro_end;
		670	}
		671
		672	/* wait for the GPU to finish processing the IB */
		673	r = dma_fence_wait(fence, false);
		674	if (r) {
		675	dev_err(adev->dev, "timeout to clear vgprs\n");
		676	goto disp_failed;
		677	}
		678
		679	r = gfx_v9_4_2_wait_for_waves_assigned(adev,
		680	&wb_ib.ptr[1], 0b1,
		681	pattern,
		682	adev->gfx.cu_info.number * SIMD_ID_MAX,
		683	false);
		684	if (r) {
		685	dev_err(adev->dev, "failed to cover all simds when clearing vgprs\n");
		686	goto disp_failed;
		687	}
		688
		689	disp_failed:
		690	amdgpu_ib_free(adev, &disp_ib, NULL);
		691	dma_fence_put(fence);
		692	pro_end:
		693	amdgpu_ib_free(adev, &wb_ib, NULL);
		694
		695	if (r)
		696	dev_info(adev->dev, "Init VGPRS Failed\n");
		697	else
		698	dev_info(adev->dev, "Init VGPRS Successfully\n");
		699
		700	return r;
		701	}
		702
		703	int gfx_v9_4_2_do_edc_gpr_workarounds(struct amdgpu_device *adev)
		704	{
		705	/* only support when RAS is enabled */
		706	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
		707	return 0;
		708
		709	gfx_v9_4_2_do_sgprs_init(adev);
		710
		711	gfx_v9_4_2_do_vgprs_init(adev);
		712
		713	return 0;
		714	}
		715
82	static void gfx_v9_4_2_query_sq_timeout_status(struct amdgpu_device *adev);	716	static void gfx_v9_4_2_query_sq_timeout_status(struct amdgpu_device *adev);
83	static void gfx_v9_4_2_reset_sq_timeout_status(struct amdgpu_device *adev);	717	static void gfx_v9_4_2_reset_sq_timeout_status(struct amdgpu_device *adev);
84		718
@@ -808,8 +1442,9 @@ static struct gfx_v9_4_2_utc_block gfx_v9_4_2_utc_blocks[] = { · 723 unmodified lines
808	REG_SET_FIELD(0, ATC_L2_CACHE_4K_DSM_CNTL, WRITE_COUNTERS, 1) },	1442	REG_SET_FIELD(0, ATC_L2_CACHE_4K_DSM_CNTL, WRITE_COUNTERS, 1) },
809	};	1443	};
810		1444
811	static const struct soc15_reg_entry gfx_v9_4_2_ea_err_status_regs =	1445	static const struct soc15_reg_entry gfx_v9_4_2_ea_err_status_regs = {
812	{ SOC15_REG_ENTRY(GC, 0, regGCEA_ERR_STATUS), 0, 1, 16 };	1446	SOC15_REG_ENTRY(GC, 0, regGCEA_ERR_STATUS), 0, 1, 16
		1447	};
813		1448
814	static int gfx_v9_4_2_get_reg_error_count(struct amdgpu_device *adev,	1449	static int gfx_v9_4_2_get_reg_error_count(struct amdgpu_device *adev,
815	const struct soc15_reg_entry *reg,	1450	const struct soc15_reg_entry *reg,
@@ -1039,13 +1674,16 @@ static void gfx_v9_4_2_reset_utc_err_status(struct amdgpu_device *adev) · 223 unmodified lines
1039	static void gfx_v9_4_2_reset_ea_err_status(struct amdgpu_device *adev)	1674	static void gfx_v9_4_2_reset_ea_err_status(struct amdgpu_device *adev)
1040	{	1675	{
1041	uint32_t i, j;	1676	uint32_t i, j;
		1677	uint32_t value;
		1678
		1679	value = REG_SET_FIELD(0, GCEA_ERR_STATUS, CLEAR_ERROR_STATUS, 0x1);
1042		1680
1043	mutex_lock(&adev->grbm_idx_mutex);	1681	mutex_lock(&adev->grbm_idx_mutex);
1044	for (i = 0; i < gfx_v9_4_2_ea_err_status_regs.se_num; i++) {	1682	for (i = 0; i < gfx_v9_4_2_ea_err_status_regs.se_num; i++) {
1045	for (j = 0; j < gfx_v9_4_2_ea_err_status_regs.instance;	1683	for (j = 0; j < gfx_v9_4_2_ea_err_status_regs.instance;
1046	j++) {	1684	j++) {
1047	gfx_v9_4_2_select_se_sh(adev, i, 0, j);	1685	gfx_v9_4_2_select_se_sh(adev, i, 0, j);
1048	WREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_4_2_ea_err_status_regs), 0x10);	1686	WREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_4_2_ea_err_status_regs), value);
1049	}	1687	}
1050	}	1688	}
1051	gfx_v9_4_2_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);	1689	gfx_v9_4_2_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);

drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h+1−0

@@ -29,6 +29,7 @@ void gfx_v9_4_2_debug_trap_config_init(struct amdgpu_device *adev, · 28 unmodified lines
29	void gfx_v9_4_2_init_golden_registers(struct amdgpu_device *adev,	29	void gfx_v9_4_2_init_golden_registers(struct amdgpu_device *adev,
30	uint32_t die_id);	30	uint32_t die_id);
31	void gfx_v9_4_2_set_power_brake_sequence(struct amdgpu_device *adev);	31	void gfx_v9_4_2_set_power_brake_sequence(struct amdgpu_device *adev);
		32	int gfx_v9_4_2_do_edc_gpr_workarounds(struct amdgpu_device *adev);
32		33
33	extern const struct amdgpu_gfx_ras_funcs gfx_v9_4_2_ras_funcs;	34	extern const struct amdgpu_gfx_ras_funcs gfx_v9_4_2_ras_funcs;
34		35

drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c+6−2

 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 				    PAGE_TABLE_BLOCK_SIZE,
 				    block_size);
-		/* Send no-retry XNACK on fault to suppress VM fault storm. */
+		/* Send no-retry XNACK on fault to suppress VM fault storm.
+		 * On Aldebaran, XNACK can be enabled in the SQ per-process.
+		 * Retry faults need to be enabled for that to work.
+		 */
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
-				    !adev->gmc.noretry);
+				    !adev->gmc.noretry ||
+				    adev->asic_type == CHIP_ALDEBARAN);
 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL,
 				    i * hub->ctx_distance, tmp);
 		WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c+12−1

 #include "mmhub_v1_7.h"
 #include "umc_v6_1.h"
 #include "umc_v6_0.h"
+#include "hdp_v4_0.h"
 #include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
 	adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
 }
+static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev)
+{
+	adev->hdp.ras_funcs = &hdp_v4_0_ras_funcs;
+}
 static int gmc_v9_0_early_init(void *handle)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 	gmc_v9_0_set_mmhub_funcs(adev);
 	gmc_v9_0_set_mmhub_ras_funcs(adev);
 	gmc_v9_0_set_gfxhub_funcs(adev);
+	gmc_v9_0_set_hdp_ras_funcs(adev);
 	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
 	adev->gmc.shared_aperture_end =
 	 * writes, while disables HBM ECC for vega10.
 	 */
 	if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) {
-		if (!(adev->ras_features & (1 << AMDGPU_RAS_BLOCK__UMC))) {
+		if (!(adev->ras_enabled & (1 << AMDGPU_RAS_BLOCK__UMC))) {
 			if (adev->df.funcs->enable_ecc_force_par_wr_rmw)
 				adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
 		}
 	    adev->mmhub.ras_funcs->reset_ras_error_count)
 		adev->mmhub.ras_funcs->reset_ras_error_count(adev);
+	if (adev->hdp.ras_funcs &&
+	    adev->hdp.ras_funcs->reset_ras_error_count)
+		adev->hdp.ras_funcs->reset_ras_error_count(adev);
 	r = amdgpu_gmc_ras_late_init(adev);
 	if (r)
 		return r;

drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c+28−3

@@ -59,12 +59,31 @@ static void hdp_v4_0_invalidate_hdp(struct amdgpu_device *adev, · 58 unmodified lines
59	HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1);	59	HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1);
60	}	60	}
61		61
		62	static void hdp_v4_0_query_ras_error_count(struct amdgpu_device *adev,
		63	void *ras_error_status)
		64	{
		65	struct ras_err_data err_data = (struct ras_err_data )ras_error_status;
		66
		67	err_data->ue_count = 0;
		68	err_data->ce_count = 0;
		69
		70	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP))
		71	return;
		72
		73	/* HDP SRAM errors are uncorrectable ones (i.e. fatal errors) */
		74	err_data->ue_count += RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT);
		75	};
		76
62	static void hdp_v4_0_reset_ras_error_count(struct amdgpu_device *adev)	77	static void hdp_v4_0_reset_ras_error_count(struct amdgpu_device *adev)
63	{	78	{
64	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP))	79	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP))
65	return;	80	return;
66	/read back hdp ras counter to reset it to 0 /	81
67	RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT);	82	if (adev->asic_type >= CHIP_ALDEBARAN)
		83	WREG32_SOC15(HDP, 0, mmHDP_EDC_CNT, 0);
		84	else
		85	/read back hdp ras counter to reset it to 0 /
		86	RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT);
68	}	87	}
69		88
70	static void hdp_v4_0_update_clock_gating(struct amdgpu_device *adev,	89	static void hdp_v4_0_update_clock_gating(struct amdgpu_device *adev,
@@ -130,10 +149,16 @@ static void hdp_v4_0_init_registers(struct amdgpu_device *adev) · 59 unmodified lines
130	WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40));	149	WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40));
131	}	150	}
132		151
		152	const struct amdgpu_hdp_ras_funcs hdp_v4_0_ras_funcs = {
		153	.ras_late_init = amdgpu_hdp_ras_late_init,
		154	.ras_fini = amdgpu_hdp_ras_fini,
		155	.query_ras_error_count = hdp_v4_0_query_ras_error_count,
		156	.reset_ras_error_count = hdp_v4_0_reset_ras_error_count,
		157	};
		158
133	const struct amdgpu_hdp_funcs hdp_v4_0_funcs = {	159	const struct amdgpu_hdp_funcs hdp_v4_0_funcs = {
134	.flush_hdp = hdp_v4_0_flush_hdp,	160	.flush_hdp = hdp_v4_0_flush_hdp,
135	.invalidate_hdp = hdp_v4_0_invalidate_hdp,	161	.invalidate_hdp = hdp_v4_0_invalidate_hdp,
136	.reset_ras_error_count = hdp_v4_0_reset_ras_error_count,
137	.update_clock_gating = hdp_v4_0_update_clock_gating,	162	.update_clock_gating = hdp_v4_0_update_clock_gating,
138	.get_clock_gating_state = hdp_v4_0_get_clockgating_state,	163	.get_clock_gating_state = hdp_v4_0_get_clockgating_state,
139	.init_registers = hdp_v4_0_init_registers,	164	.init_registers = hdp_v4_0_init_registers,

drivers/gpu/drm/amd/amdgpu/hdp_v4_0.h+1−0

@@ -27,5 +27,6 @@ · 26 unmodified lines
27	#include "soc15_common.h"	27	#include "soc15_common.h"
28		28
29	extern const struct amdgpu_hdp_funcs hdp_v4_0_funcs;	29	extern const struct amdgpu_hdp_funcs hdp_v4_0_funcs;
		30	extern const struct amdgpu_hdp_ras_funcs hdp_v4_0_ras_funcs;
30		31
31	#endif	32	#endif

drivers/gpu/drm/amd/amdgpu/mmhub_v1_7.c+23−2

 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 				    PAGE_TABLE_BLOCK_SIZE,
 				    block_size);
-		/* Send no-retry XNACK on fault to suppress VM fault storm. */
+		/* On Aldebaran, XNACK can be enabled in the SQ per-process.
+		 * Retry faults need to be enabled for that to work.
+		 */
 		tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
 				    RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
-				    !adev->gmc.noretry);
+				    1);
 		WREG32_SOC15_OFFSET(MMHUB, 0, regVM_CONTEXT1_CNTL,
 				    i * hub->ctx_distance, tmp);
 		WREG32_SOC15_OFFSET(MMHUB, 0, regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
 	}
 }
+static void mmhub_v1_7_reset_ras_error_status(struct amdgpu_device *adev)
+{
+	int i;
+	uint32_t reg_value;
+	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB))
+		return;
+	for (i = 0; i < ARRAY_SIZE(mmhub_v1_7_ea_err_status_regs); i++) {
+		reg_value = RREG32(SOC15_REG_ENTRY_OFFSET(
+			mmhub_v1_7_ea_err_status_regs[i]));
+		reg_value = REG_SET_FIELD(reg_value, MMEA0_ERR_STATUS,
+					  CLEAR_ERROR_STATUS, 0x01);
+		WREG32(SOC15_REG_ENTRY_OFFSET(mmhub_v1_7_ea_err_status_regs[i]),
+		       reg_value);
+	}
+}
 const struct amdgpu_mmhub_ras_funcs mmhub_v1_7_ras_funcs = {
 	.ras_late_init = amdgpu_mmhub_ras_late_init,
 	.ras_fini = amdgpu_mmhub_ras_fini,
 	.query_ras_error_count = mmhub_v1_7_query_ras_error_count,
 	.reset_ras_error_count = mmhub_v1_7_reset_ras_error_count,
 	.query_ras_error_status = mmhub_v1_7_query_ras_error_status,
+	.reset_ras_error_status = mmhub_v1_7_reset_ras_error_status,
 };
 const struct amdgpu_mmhub_funcs mmhub_v1_7_funcs = {

drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c+2−2

@@ -569,9 +569,9 @@ static int mmhub_v2_3_set_clockgating(struct amdgpu_device *adev, · 568 unmodified lines
569	return 0;	569	return 0;
570		570
571	mmhub_v2_3_update_medium_grain_clock_gating(adev,	571	mmhub_v2_3_update_medium_grain_clock_gating(adev,
572	state == AMD_CG_STATE_GATE ? true : false);	572	state == AMD_CG_STATE_GATE);
573	mmhub_v2_3_update_medium_grain_light_sleep(adev,	573	mmhub_v2_3_update_medium_grain_light_sleep(adev,
574	state == AMD_CG_STATE_GATE ? true : false);	574	state == AMD_CG_STATE_GATE);
575		575
576	return 0;	576	return 0;
577	}	577	}

drivers/gpu/drm/amd/amdgpu/nv.c+6−1

 static void nv_program_aspm(struct amdgpu_device *adev)
 {
-	if (amdgpu_aspm != 1)
+	if (!amdgpu_aspm)
 		return;
 	if (!(adev->flags & AMD_IS_APU) &&
 	case CHIP_SIENNA_CICHLID:
 		adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
 			AMD_CG_SUPPORT_GFX_CGCG |
+			AMD_CG_SUPPORT_GFX_CGLS |
 			AMD_CG_SUPPORT_GFX_3D_CGCG |
 			AMD_CG_SUPPORT_MC_MGCG |
 			AMD_CG_SUPPORT_VCN_MGCG |
 	case CHIP_NAVY_FLOUNDER:
 		adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
 			AMD_CG_SUPPORT_GFX_CGCG |
+			AMD_CG_SUPPORT_GFX_CGLS |
 			AMD_CG_SUPPORT_GFX_3D_CGCG |
 			AMD_CG_SUPPORT_VCN_MGCG |
 			AMD_CG_SUPPORT_JPEG_MGCG |
 			AMD_CG_SUPPORT_MC_LS |
 			AMD_CG_SUPPORT_GFX_FGCG |
 			AMD_CG_SUPPORT_VCN_MGCG |
+			AMD_CG_SUPPORT_SDMA_MGCG |
+			AMD_CG_SUPPORT_SDMA_LS |
 			AMD_CG_SUPPORT_JPEG_MGCG;
 		adev->pg_flags = AMD_PG_SUPPORT_GFX_PG |
 			AMD_PG_SUPPORT_VCN |
 	case CHIP_DIMGREY_CAVEFISH:
 		adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG |
 			AMD_CG_SUPPORT_GFX_CGCG |
+			AMD_CG_SUPPORT_GFX_CGLS |
 			AMD_CG_SUPPORT_GFX_3D_CGCG |
 			AMD_CG_SUPPORT_VCN_MGCG |
 			AMD_CG_SUPPORT_JPEG_MGCG |

drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h+0−1

@@ -97,7 +97,6 @@ enum psp_gfx_cmd_id · 96 unmodified lines
97	GFX_CMD_ID_SETUP_VMR = 0x00000009, /* setup VMR region */	97	GFX_CMD_ID_SETUP_VMR = 0x00000009, /* setup VMR region */
98	GFX_CMD_ID_DESTROY_VMR = 0x0000000A, /* destroy VMR region */	98	GFX_CMD_ID_DESTROY_VMR = 0x0000000A, /* destroy VMR region */
99	GFX_CMD_ID_PROG_REG = 0x0000000B, /* program regs */	99	GFX_CMD_ID_PROG_REG = 0x0000000B, /* program regs */
100	GFX_CMD_ID_CLEAR_VF_FW = 0x0000000D, /* Clear VF FW, to be used on VF shutdown. */
101	GFX_CMD_ID_GET_FW_ATTESTATION = 0x0000000F, /* Query GPUVA of the Fw Attestation DB */	100	GFX_CMD_ID_GET_FW_ATTESTATION = 0x0000000F, /* Query GPUVA of the Fw Attestation DB */
102	/* IDs upto 0x1F are reserved for older programs (Raven, Vega 10/12/20) */	101	/* IDs upto 0x1F are reserved for older programs (Raven, Vega 10/12/20) */
103	GFX_CMD_ID_LOAD_TOC = 0x00000020, /* Load TOC and obtain TMR size */	102	GFX_CMD_ID_LOAD_TOC = 0x00000020, /* Load TOC and obtain TMR size */

drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c+14−21

 		if (adev->asic_type == CHIP_ARCTURUS &&
 		    adev->sdma.instance[i].fw_version >= 14)
 			WREG32_SDMA(i, mmSDMA0_PUB_DUMMY_REG2, enable);
+		/* Extend page fault timeout to avoid interrupt storm */
+		WREG32_SDMA(i, mmSDMA0_UTCL1_TIMEOUT, 0x00800080);
 	}
 }
 	memset(&task_info, 0, sizeof(struct amdgpu_task_info));
 	amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
-	dev_info(adev->dev,
+	dev_dbg_ratelimited(adev->dev,
 		   "[sdma%d] address:0x%016llx src_id:%u ring:%u vmid:%u "
 		   "pasid:%u, for process %s pid %d thread %s pid %d\n",
 		   instance, addr, entry->src_id, entry->ring_id, entry->vmid,
 					      struct amdgpu_irq_src *source,
 					      struct amdgpu_iv_entry *entry)
 {
-	dev_err(adev->dev, "MC or SEM address in VM hole\n");
+	dev_dbg_ratelimited(adev->dev, "MC or SEM address in VM hole\n");
 	sdma_v4_0_print_iv_entry(adev, entry);
 	return 0;
 }
 					      struct amdgpu_irq_src *source,
 					      struct amdgpu_iv_entry *entry)
 {
-	dev_err(adev->dev, "SDMA received a doorbell from BIF with byte_enable !=0xff\n");
+	dev_dbg_ratelimited(adev->dev, "SDMA received a doorbell from BIF with byte_enable !=0xff\n");
 	sdma_v4_0_print_iv_entry(adev, entry);
 	return 0;
 }
 					      struct amdgpu_irq_src *source,
 					      struct amdgpu_iv_entry *entry)
 {
-	dev_err(adev->dev,
+	dev_dbg_ratelimited(adev->dev,
 		"Polling register/memory timeout executing POLL_REG/MEM with finite timer\n");
 	sdma_v4_0_print_iv_entry(adev, entry);
 	return 0;
 					      struct amdgpu_irq_src *source,
 					      struct amdgpu_iv_entry *entry)
 {
-	dev_err(adev->dev,
+	dev_dbg_ratelimited(adev->dev,
 		"SDMA gets an Register Write SRBM_WRITE command in non-privilege command buffer\n");
 	sdma_v4_0_print_iv_entry(adev, entry);
 	return 0;
 static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev)
 {
+	adev->sdma.trap_irq.num_types = adev->sdma.num_instances;
+	adev->sdma.ecc_irq.num_types = adev->sdma.num_instances;
+	/*For Arcturus and Aldebaran, add another 4 irq handler*/
 	switch (adev->sdma.num_instances) {
-	case 1:
-		adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE1;
-		adev->sdma.ecc_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE1;
-		break;
 	case 5:
-		adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE5;
-		adev->sdma.ecc_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE5;
-		break;
 	case 8:
-		adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
+		adev->sdma.vm_hole_irq.num_types = adev->sdma.num_instances;
-		adev->sdma.ecc_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
+		adev->sdma.doorbell_invalid_irq.num_types = adev->sdma.num_instances;
-		adev->sdma.vm_hole_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE5;
+		adev->sdma.pool_timeout_irq.num_types = adev->sdma.num_instances;
-		adev->sdma.doorbell_invalid_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
+		adev->sdma.srbm_write_irq.num_types = adev->sdma.num_instances;
-		adev->sdma.pool_timeout_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
-		adev->sdma.srbm_write_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
 		break;
-	case 2:
 	default:
-		adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE2;
-		adev->sdma.ecc_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE2;
 		break;
 	}
 	adev->sdma.trap_irq.funcs = &sdma_v4_0_trap_irq_funcs;

drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c+28−12

 	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
 	uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid);
-	/* Invalidate L2, because if we don't do it, we might get stale cache
-	 * lines from previous IBs.
-	 */
-	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_GCR_REQ));
-	amdgpu_ring_write(ring, 0);
-	amdgpu_ring_write(ring, (SDMA_GCR_GL2_INV |
-				 SDMA_GCR_GL2_WB |
-				 SDMA_GCR_GLM_INV |
-				 SDMA_GCR_GLM_WB) << 16);
-	amdgpu_ring_write(ring, 0xffffff80);
-	amdgpu_ring_write(ring, 0xffff);
 	/* An IB packet must end on a 8 DW boundary--the next dword
 	 * must be on a 8-dword boundary. Our IB packet below is 6
 	 * dwords long, thus add x number of NOPs, such that, in
 	amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr));
 }
+/**
+ * sdma_v5_0_ring_emit_mem_sync - flush the IB by graphics cache rinse
+ *
+ * @ring: amdgpu ring pointer
+ * @job: job to retrieve vmid from
+ * @ib: IB object to schedule
+ *
+ * flush the IB by graphics cache rinse.
+ */
+static void sdma_v5_0_ring_emit_mem_sync(struct amdgpu_ring *ring)
+{
+    uint32_t gcr_cntl =
+		    SDMA_GCR_GL2_INV | SDMA_GCR_GL2_WB | SDMA_GCR_GLM_INV |
+			SDMA_GCR_GL1_INV | SDMA_GCR_GLV_INV | SDMA_GCR_GLK_INV |
+			SDMA_GCR_GLI_INV(1);
+	/* flush entire cache L0/L1/L2, this can be optimized by performance requirement */
+	amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_GCR_REQ));
+	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0));
+	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD2_GCR_CONTROL_15_0(gcr_cntl) |
+			SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_47_32(0));
+	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_31_7(0) |
+			SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_16(gcr_cntl >> 16));
+	amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_32(0) |
+			SDMA_PKT_GCR_REQ_PAYLOAD4_VMID(0));
+}
 /**
  * sdma_v5_0_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
  *
 		10 + 10 + 10, /* sdma_v5_0_ring_emit_fence x3 for user fence, vm fence */
 	.emit_ib_size = 5 + 7 + 6, /* sdma_v5_0_ring_emit_ib */
 	.emit_ib = sdma_v5_0_ring_emit_ib,
+	.emit_mem_sync = sdma_v5_0_ring_emit_mem_sync,
 	.emit_fence = sdma_v5_0_ring_emit_fence,
 	.emit_pipeline_sync = sdma_v5_0_ring_emit_pipeline_sync,
 	.emit_vm_flush = sdma_v5_0_ring_emit_vm_flush,

drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c+8−0

 	int i;
 	for (i = 0; i < adev->sdma.num_instances; i++) {
+		if (adev->sdma.instance[i].fw_version < 70 && adev->asic_type == CHIP_VANGOGH)
+			adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_MGCG;
 		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) {
 			/* Enable sdma clock gating */
 			def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL));
 	int i;
 	for (i = 0; i < adev->sdma.num_instances; i++) {
+		if (adev->sdma.instance[i].fw_version < 70 && adev->asic_type == CHIP_VANGOGH)
+			adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_LS;
 		if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) {
 			/* Enable sdma mem light sleep */
 			def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL));

drivers/gpu/drm/amd/amdgpu/smuio_v13_0.c+18−0

@@ -88,6 +88,23 @@ static u32 smuio_v13_0_get_die_id(struct amdgpu_device *adev) · 87 unmodified lines
88	return die_id;	88	return die_id;
89	}	89	}
90		90
		91	/**
		92	* smuio_v13_0_get_socket_id - query socket id from FCH
		93	*
		94	* @adev: amdgpu device pointer
		95	*
		96	* Returns socket id
		97	*/
		98	static u32 smuio_v13_0_get_socket_id(struct amdgpu_device *adev)
		99	{
		100	u32 data, socket_id;
		101
		102	data = RREG32_SOC15(SMUIO, 0, regSMUIO_MCM_CONFIG);
		103	socket_id = REG_GET_FIELD(data, SMUIO_MCM_CONFIG, SOCKET_ID);
		104
		105	return socket_id;
		106	}
		107
91	/**	108	/**
92	* smuio_v13_0_supports_host_gpu_xgmi - detect xgmi interface between cpu and gpu/s.	109	* smuio_v13_0_supports_host_gpu_xgmi - detect xgmi interface between cpu and gpu/s.
93	*	110	*
@@ -115,6 +132,7 @@ const struct amdgpu_smuio_funcs smuio_v13_0_funcs = { · 21 unmodified lines
115	.get_rom_index_offset = smuio_v13_0_get_rom_index_offset,	132	.get_rom_index_offset = smuio_v13_0_get_rom_index_offset,
116	.get_rom_data_offset = smuio_v13_0_get_rom_data_offset,	133	.get_rom_data_offset = smuio_v13_0_get_rom_data_offset,
117	.get_die_id = smuio_v13_0_get_die_id,	134	.get_die_id = smuio_v13_0_get_die_id,
		135	.get_socket_id = smuio_v13_0_get_socket_id,
118	.is_host_gpu_xgmi_supported = smuio_v13_0_is_host_gpu_xgmi_supported,	136	.is_host_gpu_xgmi_supported = smuio_v13_0_is_host_gpu_xgmi_supported,
119	.update_rom_clock_gating = smuio_v13_0_update_rom_clock_gating,	137	.update_rom_clock_gating = smuio_v13_0_update_rom_clock_gating,
120	.get_clock_gating_state = smuio_v13_0_get_clock_gating_state,	138	.get_clock_gating_state = smuio_v13_0_get_clock_gating_state,

drivers/gpu/drm/amd/amdgpu/soc15.c+5−7

 	int ret = 0;
 	/* avoid NBIF got stuck when do RAS recovery in BACO reset */
-	if (ras && ras->supported)
+	if (ras && adev->ras_enabled)
 		adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
 	ret = amdgpu_dpm_baco_reset(adev);
 		return ret;
 	/* re-enable doorbell interrupt after BACO exit */
-	if (ras && ras->supported)
+	if (ras && adev->ras_enabled)
 		adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
 	return 0;
 		 * 1. PMFW version > 0x284300: all cases use baco
 		 * 2. PMFW version <= 0x284300: only sGPU w/o RAS use baco
 		 */
-		if ((ras && ras->supported) && adev->pm.fw_version <= 0x283400)
+		if (ras && adev->ras_enabled &&
+		    adev->pm.fw_version <= 0x283400)
 			baco_reset = false;
 		break;
 	case CHIP_ALDEBARAN:
 static void soc15_program_aspm(struct amdgpu_device *adev)
 {
-	if (amdgpu_aspm != 1)
+	if (!amdgpu_aspm)
 		return;
 	if (!(adev->flags & AMD_IS_APU) &&
 	if (amdgpu_sriov_vf(adev))
 		xgpu_ai_mailbox_get_irq(adev);
-	if (adev->hdp.funcs->reset_ras_error_count)
-		adev->hdp.funcs->reset_ras_error_count(adev);
 	if (adev->nbio.ras_funcs &&
 	    adev->nbio.ras_funcs->ras_late_init)
 		r = adev->nbio.ras_funcs->ras_late_init(adev);

drivers/gpu/drm/amd/amdgpu/ta_ras_if.h+7−0

 	uint64_t		value;			// method if error injection. i.e persistent, coherent etc.
 };
+struct ta_ras_init_flags
+{
+    uint8_t     poison_mode_en;
+    uint8_t     dgpu_mode;
+};
 struct ta_ras_output_flags
 {
 	uint8_t    ras_init_success_flag;
 /* Common input structure for RAS callbacks */
 /**********************************************************/
 union ta_ras_cmd_input {
+	struct ta_ras_init_flags		init_flags;
 	struct ta_ras_enable_features_input	enable_features;
 	struct ta_ras_disable_features_input	disable_features;
 	struct ta_ras_trigger_error_input	trigger_error;

drivers/gpu/drm/amd/amdgpu/vega10_ih.c+1−0

@@ -104,6 +104,7 @@ static int vega10_ih_toggle_ring_interrupts(struct amdgpu_device *adev, · 103 unmodified lines
104		104
105	tmp = RREG32(ih_regs->ih_rb_cntl);	105	tmp = RREG32(ih_regs->ih_rb_cntl);
106	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0));	106	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0));
		107	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_GPU_TS_ENABLE, 1);
107	/* enable_intr field is only valid in ring0 */	108	/* enable_intr field is only valid in ring0 */
108	if (ih == &adev->irq.ih)	109	if (ih == &adev->irq.ih)
109	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0));	110	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0));

drivers/gpu/drm/amd/amdgpu/vi.c+1−1

@@ -1136,7 +1136,7 @@ static void vi_program_aspm(struct amdgpu_device *adev) · 1135 unmodified lines
1136	bool bL1SS = false;	1136	bool bL1SS = false;
1137	bool bClkReqSupport = true;	1137	bool bClkReqSupport = true;
1138		1138
1139	if (amdgpu_aspm != 1)	1139	if (!amdgpu_aspm)
1140	return;	1140	return;
1141		1141
1142	if (adev->flags & AMD_IS_APU \|\|	1142	if (adev->flags & AMD_IS_APU \|\|

drivers/gpu/drm/amd/amdkfd/Kconfig+13−0

@@ -12,3 +12,16 @@ config HSA_AMD · 11 unmodified lines
12	select DRM_AMDGPU_USERPTR	12	select DRM_AMDGPU_USERPTR
13	help	13	help
14	Enable this if you want to use HSA features on AMD GPU devices.	14	Enable this if you want to use HSA features on AMD GPU devices.
		15
		16	config HSA_AMD_SVM
		17	bool "Enable HMM-based shared virtual memory manager"
		18	depends on HSA_AMD && DEVICE_PRIVATE
		19	default y
		20	select HMM_MIRROR
		21	select MMU_NOTIFIER
		22	help
		23	Enable this to use unified memory and managed memory in HIP. This
		24	memory manager supports two modes of operation. One based on
		25	preemptions and one based on page faults. To enable page fault
		26	based memory management on most GFXv9 GPUs, set the module
		27	parameter amdgpu.noretry=0.

drivers/gpu/drm/amd/amdkfd/Makefile+5−0

@@ -63,3 +63,8 @@ endif · 62 unmodified lines
63	ifneq ($(CONFIG_DEBUG_FS),)	63	ifneq ($(CONFIG_DEBUG_FS),)
64	AMDKFD_FILES += $(AMDKFD_PATH)/kfd_debugfs.o	64	AMDKFD_FILES += $(AMDKFD_PATH)/kfd_debugfs.o
65	endif	65	endif
		66
		67	ifneq ($(CONFIG_HSA_AMD_SVM),)
		68	AMDKFD_FILES += $(AMDKFD_PATH)/kfd_svm.o \
		69	$(AMDKFD_PATH)/kfd_migrate.o
		70	endif

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c+73−7

 #include "kfd_priv.h"
 #include "kfd_device_queue_manager.h"
 #include "kfd_dbgmgr.h"
+#include "kfd_svm.h"
 #include "amdgpu_amdkfd.h"
 #include "kfd_smi_events.h"
 	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
 		dev->kgd, args->va_addr, args->size,
-		pdd->vm, (struct kgd_mem **) &mem, &offset,
+		pdd->drm_priv, (struct kgd_mem **) &mem, &offset,
 		flags);
 	if (err)
 	return 0;
 err_free:
-	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem, NULL);
+	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem,
+					       pdd->drm_priv, NULL);
 err_unlock:
 	mutex_unlock(&p->mutex);
 	return err;
 	}
 	ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd,
-						(struct kgd_mem *)mem, &size);
+				(struct kgd_mem *)mem, pdd->drm_priv, &size);
 	/* If freeing the buffer failed, leave the handle in place for
 	 * clean-up during process tear-down.
 			goto get_mem_obj_from_handle_failed;
 		}
 		err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
-			peer->kgd, (struct kgd_mem *)mem, peer_pdd->vm);
+			peer->kgd, (struct kgd_mem *)mem, peer_pdd->drm_priv);
 		if (err) {
 			pr_err("Failed to map to gpu %d/%d\n",
 			       i, args->n_devices);
 			goto get_mem_obj_from_handle_failed;
 		}
 		err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
-			peer->kgd, (struct kgd_mem *)mem, peer_pdd->vm);
+			peer->kgd, (struct kgd_mem *)mem, peer_pdd->drm_priv);
 		if (err) {
 			pr_err("Failed to unmap from gpu %d/%d\n",
 			       i, args->n_devices);
 	}
 	r = amdgpu_amdkfd_gpuvm_import_dmabuf(dev->kgd, dmabuf,
-					      args->va_addr, pdd->vm,
+					      args->va_addr, pdd->drm_priv,
 					      (struct kgd_mem **)&mem, &size,
 					      NULL);
 	if (r)
 	return 0;
 err_free:
-	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem, NULL);
+	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem,
+					       pdd->drm_priv, NULL);
 err_unlock:
 	mutex_unlock(&p->mutex);
 	dma_buf_put(dmabuf);
 	return kfd_smi_event_open(dev, &args->anon_fd);
 }
+static int kfd_ioctl_set_xnack_mode(struct file *filep,
+				    struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_set_xnack_mode_args *args = data;
+	int r = 0;
+	mutex_lock(&p->mutex);
+	if (args->xnack_enabled >= 0) {
+		if (!list_empty(&p->pqm.queues)) {
+			pr_debug("Process has user queues running\n");
+			mutex_unlock(&p->mutex);
+			return -EBUSY;
+		}
+		if (args->xnack_enabled && !kfd_process_xnack_mode(p, true))
+			r = -EPERM;
+		else
+			p->xnack_enabled = args->xnack_enabled;
+	} else {
+		args->xnack_enabled = p->xnack_enabled;
+	}
+	mutex_unlock(&p->mutex);
+	return r;
+}
+#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
+static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
+{
+	struct kfd_ioctl_svm_args *args = data;
+	int r = 0;
+	if (p->svm_disabled)
+		return -EPERM;
+	pr_debug("start 0x%llx size 0x%llx op 0x%x nattr 0x%x\n",
+		 args->start_addr, args->size, args->op, args->nattr);
+	if ((args->start_addr & ~PAGE_MASK) || (args->size & ~PAGE_MASK))
+		return -EINVAL;
+	if (!args->start_addr || !args->size)
+		return -EINVAL;
+	mutex_lock(&p->mutex);
+	r = svm_ioctl(p, args->op, args->start_addr, args->size, args->nattr,
+		      args->attrs);
+	mutex_unlock(&p->mutex);
+	return r;
+}
+#else
+static int kfd_ioctl_svm(struct file *filep, struct kfd_process *p, void *data)
+{
+	return -EPERM;
+}
+#endif
 #define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
 	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
 			    .cmd_drv = 0, .name = #ioctl}
 	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
 			kfd_ioctl_smi_events, 0),
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0),
+	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
+			kfd_ioctl_set_xnack_mode, 0),
 };
 #define AMDKFD_CORE_IOCTL_COUNT	ARRAY_SIZE(amdkfd_ioctls)

drivers/gpu/drm/amd/amdkfd/kfd_crat.c+811−70

@@ -55,7 +55,7 @@ struct kfd_gpu_cache_info { · 54 unmodified lines
55	uint32_t cache_level;	55	uint32_t cache_level;
56	uint32_t flags;	56	uint32_t flags;
57	/* Indicates how many Compute Units share this cache	57	/* Indicates how many Compute Units share this cache
58	* Value = 1 indicates the cache is not shared	58	* within a SA. Value = 1 indicates the cache is not shared
59	*/	59	*/
60	uint32_t num_cu_shared;	60	uint32_t num_cu_shared;
61	};	61	};
@@ -69,32 +69,456 @@ static struct kfd_gpu_cache_info kaveri_cache_info[] = { · 7 unmodified lines
69	CRAT_CACHE_FLAGS_DATA_CACHE \|	69	CRAT_CACHE_FLAGS_DATA_CACHE \|
70	CRAT_CACHE_FLAGS_SIMD_CACHE),	70	CRAT_CACHE_FLAGS_SIMD_CACHE),
71	.num_cu_shared = 1,	71	.num_cu_shared = 1,
		72	},
		73	{
		74	/* Scalar L1 Instruction Cache (in SQC module) per bank */
		75	.cache_size = 16,
		76	.cache_level = 1,
		77	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		78	CRAT_CACHE_FLAGS_INST_CACHE \|
		79	CRAT_CACHE_FLAGS_SIMD_CACHE),
		80	.num_cu_shared = 2,
		81	},
		82	{
		83	/* Scalar L1 Data Cache (in SQC module) per bank */
		84	.cache_size = 8,
		85	.cache_level = 1,
		86	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		87	CRAT_CACHE_FLAGS_DATA_CACHE \|
		88	CRAT_CACHE_FLAGS_SIMD_CACHE),
		89	.num_cu_shared = 2,
		90	},
72		91
		92	/* TODO: Add L2 Cache information */
		93	};
		94
		95
		96	static struct kfd_gpu_cache_info carrizo_cache_info[] = {
		97	{
		98	/* TCP L1 Cache per CU */
		99	.cache_size = 16,
		100	.cache_level = 1,
		101	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		102	CRAT_CACHE_FLAGS_DATA_CACHE \|
		103	CRAT_CACHE_FLAGS_SIMD_CACHE),
		104	.num_cu_shared = 1,
		105	},
		106	{
		107	/* Scalar L1 Instruction Cache (in SQC module) per bank */
		108	.cache_size = 8,
		109	.cache_level = 1,
		110	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		111	CRAT_CACHE_FLAGS_INST_CACHE \|
		112	CRAT_CACHE_FLAGS_SIMD_CACHE),
		113	.num_cu_shared = 4,
		114	},
		115	{
		116	/* Scalar L1 Data Cache (in SQC module) per bank. */
		117	.cache_size = 4,
		118	.cache_level = 1,
		119	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		120	CRAT_CACHE_FLAGS_DATA_CACHE \|
		121	CRAT_CACHE_FLAGS_SIMD_CACHE),
		122	.num_cu_shared = 4,
		123	},
		124
		125	/* TODO: Add L2 Cache information */
		126	};
		127
		128	#define hawaii_cache_info kaveri_cache_info
		129	#define tonga_cache_info carrizo_cache_info
		130	#define fiji_cache_info carrizo_cache_info
		131	#define polaris10_cache_info carrizo_cache_info
		132	#define polaris11_cache_info carrizo_cache_info
		133	#define polaris12_cache_info carrizo_cache_info
		134	#define vegam_cache_info carrizo_cache_info
		135
		136	/* NOTE: L1 cache information has been updated and L2/L3
		137	* cache information has been added for Vega10 and
		138	* newer ASICs. The unit for cache_size is KiB.
		139	* In future, check & update cache details
		140	* for every new ASIC is required.
		141	*/
		142
		143	static struct kfd_gpu_cache_info vega10_cache_info[] = {
		144	{
		145	/* TCP L1 Cache per CU */
		146	.cache_size = 16,
		147	.cache_level = 1,
		148	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		149	CRAT_CACHE_FLAGS_DATA_CACHE \|
		150	CRAT_CACHE_FLAGS_SIMD_CACHE),
		151	.num_cu_shared = 1,
		152	},
		153	{
		154	/* Scalar L1 Instruction Cache per SQC */
		155	.cache_size = 32,
		156	.cache_level = 1,
		157	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		158	CRAT_CACHE_FLAGS_INST_CACHE \|
		159	CRAT_CACHE_FLAGS_SIMD_CACHE),
		160	.num_cu_shared = 3,
		161	},
		162	{
		163	/* Scalar L1 Data Cache per SQC */
		164	.cache_size = 16,
		165	.cache_level = 1,
		166	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		167	CRAT_CACHE_FLAGS_DATA_CACHE \|
		168	CRAT_CACHE_FLAGS_SIMD_CACHE),
		169	.num_cu_shared = 3,
		170	},
		171	{
		172	/* L2 Data Cache per GPU (Total Tex Cache) */
		173	.cache_size = 4096,
		174	.cache_level = 2,
		175	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		176	CRAT_CACHE_FLAGS_DATA_CACHE \|
		177	CRAT_CACHE_FLAGS_SIMD_CACHE),
		178	.num_cu_shared = 16,
		179	},
		180	};
		181
		182	static struct kfd_gpu_cache_info raven_cache_info[] = {
		183	{
		184	/* TCP L1 Cache per CU */
		185	.cache_size = 16,
		186	.cache_level = 1,
		187	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		188	CRAT_CACHE_FLAGS_DATA_CACHE \|
		189	CRAT_CACHE_FLAGS_SIMD_CACHE),
		190	.num_cu_shared = 1,
		191	},
		192	{
		193	/* Scalar L1 Instruction Cache per SQC */
		194	.cache_size = 32,
		195	.cache_level = 1,
		196	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		197	CRAT_CACHE_FLAGS_INST_CACHE \|
		198	CRAT_CACHE_FLAGS_SIMD_CACHE),
		199	.num_cu_shared = 3,
		200	},
		201	{
		202	/* Scalar L1 Data Cache per SQC */
		203	.cache_size = 16,
		204	.cache_level = 1,
		205	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		206	CRAT_CACHE_FLAGS_DATA_CACHE \|
		207	CRAT_CACHE_FLAGS_SIMD_CACHE),
		208	.num_cu_shared = 3,
		209	},
		210	{
		211	/* L2 Data Cache per GPU (Total Tex Cache) */
		212	.cache_size = 1024,
		213	.cache_level = 2,
		214	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		215	CRAT_CACHE_FLAGS_DATA_CACHE \|
		216	CRAT_CACHE_FLAGS_SIMD_CACHE),
		217	.num_cu_shared = 11,
		218	},
		219	};
		220
		221	static struct kfd_gpu_cache_info renoir_cache_info[] = {
		222	{
		223	/* TCP L1 Cache per CU */
		224	.cache_size = 16,
		225	.cache_level = 1,
		226	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		227	CRAT_CACHE_FLAGS_DATA_CACHE \|
		228	CRAT_CACHE_FLAGS_SIMD_CACHE),
		229	.num_cu_shared = 1,
		230	},
		231	{
		232	/* Scalar L1 Instruction Cache per SQC */
		233	.cache_size = 32,
		234	.cache_level = 1,
		235	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		236	CRAT_CACHE_FLAGS_INST_CACHE \|
		237	CRAT_CACHE_FLAGS_SIMD_CACHE),
		238	.num_cu_shared = 3,
		239	},
		240	{
		241	/* Scalar L1 Data Cache per SQC */
		242	.cache_size = 16,
		243	.cache_level = 1,
		244	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		245	CRAT_CACHE_FLAGS_DATA_CACHE \|
		246	CRAT_CACHE_FLAGS_SIMD_CACHE),
		247	.num_cu_shared = 3,
		248	},
		249	{
		250	/* L2 Data Cache per GPU (Total Tex Cache) */
		251	.cache_size = 1024,
		252	.cache_level = 2,
		253	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		254	CRAT_CACHE_FLAGS_DATA_CACHE \|
		255	CRAT_CACHE_FLAGS_SIMD_CACHE),
		256	.num_cu_shared = 8,
		257	},
		258	};
		259
		260	static struct kfd_gpu_cache_info vega12_cache_info[] = {
		261	{
		262	/* TCP L1 Cache per CU */
		263	.cache_size = 16,
		264	.cache_level = 1,
		265	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		266	CRAT_CACHE_FLAGS_DATA_CACHE \|
		267	CRAT_CACHE_FLAGS_SIMD_CACHE),
		268	.num_cu_shared = 1,
		269	},
		270	{
		271	/* Scalar L1 Instruction Cache per SQC */
		272	.cache_size = 32,
		273	.cache_level = 1,
		274	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		275	CRAT_CACHE_FLAGS_INST_CACHE \|
		276	CRAT_CACHE_FLAGS_SIMD_CACHE),
		277	.num_cu_shared = 3,
		278	},
		279	{
		280	/* Scalar L1 Data Cache per SQC */
		281	.cache_size = 16,
		282	.cache_level = 1,
		283	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		284	CRAT_CACHE_FLAGS_DATA_CACHE \|
		285	CRAT_CACHE_FLAGS_SIMD_CACHE),
		286	.num_cu_shared = 3,
		287	},
		288	{
		289	/* L2 Data Cache per GPU (Total Tex Cache) */
		290	.cache_size = 2048,
		291	.cache_level = 2,
		292	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		293	CRAT_CACHE_FLAGS_DATA_CACHE \|
		294	CRAT_CACHE_FLAGS_SIMD_CACHE),
		295	.num_cu_shared = 5,
		296	},
		297	};
		298
		299	static struct kfd_gpu_cache_info vega20_cache_info[] = {
		300	{
		301	/* TCP L1 Cache per CU */
		302	.cache_size = 16,
		303	.cache_level = 1,
		304	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		305	CRAT_CACHE_FLAGS_DATA_CACHE \|
		306	CRAT_CACHE_FLAGS_SIMD_CACHE),
		307	.num_cu_shared = 1,
		308	},
		309	{
		310	/* Scalar L1 Instruction Cache per SQC */
		311	.cache_size = 32,
		312	.cache_level = 1,
		313	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		314	CRAT_CACHE_FLAGS_INST_CACHE \|
		315	CRAT_CACHE_FLAGS_SIMD_CACHE),
		316	.num_cu_shared = 3,
		317	},
		318	{
		319	/* Scalar L1 Data Cache per SQC */
		320	.cache_size = 16,
		321	.cache_level = 1,
		322	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		323	CRAT_CACHE_FLAGS_DATA_CACHE \|
		324	CRAT_CACHE_FLAGS_SIMD_CACHE),
		325	.num_cu_shared = 3,
		326	},
		327	{
		328	/* L2 Data Cache per GPU (Total Tex Cache) */
		329	.cache_size = 8192,
		330	.cache_level = 2,
		331	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		332	CRAT_CACHE_FLAGS_DATA_CACHE \|
		333	CRAT_CACHE_FLAGS_SIMD_CACHE),
		334	.num_cu_shared = 16,
		335	},
		336	};
		337
		338	static struct kfd_gpu_cache_info aldebaran_cache_info[] = {
		339	{
		340	/* TCP L1 Cache per CU */
		341	.cache_size = 16,
		342	.cache_level = 1,
		343	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		344	CRAT_CACHE_FLAGS_DATA_CACHE \|
		345	CRAT_CACHE_FLAGS_SIMD_CACHE),
		346	.num_cu_shared = 1,
		347	},
		348	{
		349	/* Scalar L1 Instruction Cache per SQC */
		350	.cache_size = 32,
		351	.cache_level = 1,
		352	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		353	CRAT_CACHE_FLAGS_INST_CACHE \|
		354	CRAT_CACHE_FLAGS_SIMD_CACHE),
		355	.num_cu_shared = 2,
		356	},
		357	{
		358	/* Scalar L1 Data Cache per SQC */
		359	.cache_size = 16,
		360	.cache_level = 1,
		361	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		362	CRAT_CACHE_FLAGS_DATA_CACHE \|
		363	CRAT_CACHE_FLAGS_SIMD_CACHE),
		364	.num_cu_shared = 2,
		365	},
		366	{
		367	/* L2 Data Cache per GPU (Total Tex Cache) */
		368	.cache_size = 8192,
		369	.cache_level = 2,
		370	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		371	CRAT_CACHE_FLAGS_DATA_CACHE \|
		372	CRAT_CACHE_FLAGS_SIMD_CACHE),
		373	.num_cu_shared = 14,
		374	},
		375	};
		376
		377	static struct kfd_gpu_cache_info navi10_cache_info[] = {
		378	{
		379	/* TCP L1 Cache per CU */
		380	.cache_size = 16,
		381	.cache_level = 1,
		382	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		383	CRAT_CACHE_FLAGS_DATA_CACHE \|
		384	CRAT_CACHE_FLAGS_SIMD_CACHE),
		385	.num_cu_shared = 1,
		386	},
		387	{
		388	/* Scalar L1 Instruction Cache per SQC */
		389	.cache_size = 32,
		390	.cache_level = 1,
		391	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		392	CRAT_CACHE_FLAGS_INST_CACHE \|
		393	CRAT_CACHE_FLAGS_SIMD_CACHE),
		394	.num_cu_shared = 2,
		395	},
		396	{
		397	/* Scalar L1 Data Cache per SQC */
		398	.cache_size = 16,
		399	.cache_level = 1,
		400	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		401	CRAT_CACHE_FLAGS_DATA_CACHE \|
		402	CRAT_CACHE_FLAGS_SIMD_CACHE),
		403	.num_cu_shared = 2,
		404	},
		405	{
		406	/* GL1 Data Cache per SA */
		407	.cache_size = 128,
		408	.cache_level = 1,
		409	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		410	CRAT_CACHE_FLAGS_DATA_CACHE \|
		411	CRAT_CACHE_FLAGS_SIMD_CACHE),
		412	.num_cu_shared = 10,
		413	},
		414	{
		415	/* L2 Data Cache per GPU (Total Tex Cache) */
		416	.cache_size = 4096,
		417	.cache_level = 2,
		418	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		419	CRAT_CACHE_FLAGS_DATA_CACHE \|
		420	CRAT_CACHE_FLAGS_SIMD_CACHE),
		421	.num_cu_shared = 10,
		422	},
		423	};
		424
		425	static struct kfd_gpu_cache_info vangogh_cache_info[] = {
		426	{
		427	/* TCP L1 Cache per CU */
		428	.cache_size = 16,
		429	.cache_level = 1,
		430	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		431	CRAT_CACHE_FLAGS_DATA_CACHE \|
		432	CRAT_CACHE_FLAGS_SIMD_CACHE),
		433	.num_cu_shared = 1,
		434	},
		435	{
		436	/* Scalar L1 Instruction Cache per SQC */
		437	.cache_size = 32,
		438	.cache_level = 1,
		439	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		440	CRAT_CACHE_FLAGS_INST_CACHE \|
		441	CRAT_CACHE_FLAGS_SIMD_CACHE),
		442	.num_cu_shared = 2,
		443	},
		444	{
		445	/* Scalar L1 Data Cache per SQC */
		446	.cache_size = 16,
		447	.cache_level = 1,
		448	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		449	CRAT_CACHE_FLAGS_DATA_CACHE \|
		450	CRAT_CACHE_FLAGS_SIMD_CACHE),
		451	.num_cu_shared = 2,
		452	},
		453	{
		454	/* GL1 Data Cache per SA */
		455	.cache_size = 128,
		456	.cache_level = 1,
		457	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		458	CRAT_CACHE_FLAGS_DATA_CACHE \|
		459	CRAT_CACHE_FLAGS_SIMD_CACHE),
		460	.num_cu_shared = 8,
73	},	461	},
74	{	462	{
75	/* Scalar L1 Instruction Cache (in SQC module) per bank */	463	/* L2 Data Cache per GPU (Total Tex Cache) */
		464	.cache_size = 1024,
		465	.cache_level = 2,
		466	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		467	CRAT_CACHE_FLAGS_DATA_CACHE \|
		468	CRAT_CACHE_FLAGS_SIMD_CACHE),
		469	.num_cu_shared = 8,
		470	},
		471	};
		472
		473	static struct kfd_gpu_cache_info navi14_cache_info[] = {
		474	{
		475	/* TCP L1 Cache per CU */
76	.cache_size = 16,	476	.cache_size = 16,
77	.cache_level = 1,	477	.cache_level = 1,
		478	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		479	CRAT_CACHE_FLAGS_DATA_CACHE \|
		480	CRAT_CACHE_FLAGS_SIMD_CACHE),
		481	.num_cu_shared = 1,
		482	},
		483	{
		484	/* Scalar L1 Instruction Cache per SQC */
		485	.cache_size = 32,
		486	.cache_level = 1,
78	.flags = (CRAT_CACHE_FLAGS_ENABLED \|	487	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
79	CRAT_CACHE_FLAGS_INST_CACHE \|	488	CRAT_CACHE_FLAGS_INST_CACHE \|
80	CRAT_CACHE_FLAGS_SIMD_CACHE),	489	CRAT_CACHE_FLAGS_SIMD_CACHE),
81	.num_cu_shared = 2,	490	.num_cu_shared = 2,
82	},	491	},
83	{	492	{
84	/* Scalar L1 Data Cache (in SQC module) per bank */	493	/* Scalar L1 Data Cache per SQC */
85	.cache_size = 8,	494	.cache_size = 16,
86	.cache_level = 1,	495	.cache_level = 1,
87	.flags = (CRAT_CACHE_FLAGS_ENABLED \|	496	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
88	CRAT_CACHE_FLAGS_DATA_CACHE \|	497	CRAT_CACHE_FLAGS_DATA_CACHE \|
89	CRAT_CACHE_FLAGS_SIMD_CACHE),	498	CRAT_CACHE_FLAGS_SIMD_CACHE),
90	.num_cu_shared = 2,	499	.num_cu_shared = 2,
91	},	500	},
92		501	{
93	/* TODO: Add L2 Cache information */	502	/* GL1 Data Cache per SA */
		503	.cache_size = 128,
		504	.cache_level = 1,
		505	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		506	CRAT_CACHE_FLAGS_DATA_CACHE \|
		507	CRAT_CACHE_FLAGS_SIMD_CACHE),
		508	.num_cu_shared = 12,
		509	},
		510	{
		511	/* L2 Data Cache per GPU (Total Tex Cache) */
		512	.cache_size = 2048,
		513	.cache_level = 2,
		514	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		515	CRAT_CACHE_FLAGS_DATA_CACHE \|
		516	CRAT_CACHE_FLAGS_SIMD_CACHE),
		517	.num_cu_shared = 12,
		518	},
94	};	519	};
95		520
96		521	static struct kfd_gpu_cache_info sienna_cichlid_cache_info[] = {
97	static struct kfd_gpu_cache_info carrizo_cache_info[] = {
98	{	522	{
99	/* TCP L1 Cache per CU */	523	/* TCP L1 Cache per CU */
100	.cache_size = 16,	524	.cache_size = 16,
@@ -105,44 +529,165 @@ static struct kfd_gpu_cache_info carrizo_cache_info[] = { · 4 unmodified lines
105	.num_cu_shared = 1,	529	.num_cu_shared = 1,
106	},	530	},
107	{	531	{
108	/* Scalar L1 Instruction Cache (in SQC module) per bank */	532	/* Scalar L1 Instruction Cache per SQC */
109	.cache_size = 8,	533	.cache_size = 32,
110	.cache_level = 1,	534	.cache_level = 1,
111	.flags = (CRAT_CACHE_FLAGS_ENABLED \|	535	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
112	CRAT_CACHE_FLAGS_INST_CACHE \|	536	CRAT_CACHE_FLAGS_INST_CACHE \|
113	CRAT_CACHE_FLAGS_SIMD_CACHE),	537	CRAT_CACHE_FLAGS_SIMD_CACHE),
114	.num_cu_shared = 4,	538	.num_cu_shared = 2,
115	},	539	},
116	{	540	{
117	/* Scalar L1 Data Cache (in SQC module) per bank. */	541	/* Scalar L1 Data Cache per SQC */
118	.cache_size = 4,	542	.cache_size = 16,
119	.cache_level = 1,	543	.cache_level = 1,
120	.flags = (CRAT_CACHE_FLAGS_ENABLED \|	544	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
121	CRAT_CACHE_FLAGS_DATA_CACHE \|	545	CRAT_CACHE_FLAGS_DATA_CACHE \|
122	CRAT_CACHE_FLAGS_SIMD_CACHE),	546	CRAT_CACHE_FLAGS_SIMD_CACHE),
123	.num_cu_shared = 4,	547	.num_cu_shared = 2,
		548	},
		549	{
		550	/* GL1 Data Cache per SA */
		551	.cache_size = 128,
		552	.cache_level = 1,
		553	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		554	CRAT_CACHE_FLAGS_DATA_CACHE \|
		555	CRAT_CACHE_FLAGS_SIMD_CACHE),
		556	.num_cu_shared = 10,
		557	},
		558	{
		559	/* L2 Data Cache per GPU (Total Tex Cache) */
		560	.cache_size = 4096,
		561	.cache_level = 2,
		562	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		563	CRAT_CACHE_FLAGS_DATA_CACHE \|
		564	CRAT_CACHE_FLAGS_SIMD_CACHE),
		565	.num_cu_shared = 10,
124	},	566	},
		567	{
		568	/* L3 Data Cache per GPU */
		569	.cache_size = 128*1024,
		570	.cache_level = 3,
		571	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		572	CRAT_CACHE_FLAGS_DATA_CACHE \|
		573	CRAT_CACHE_FLAGS_SIMD_CACHE),
		574	.num_cu_shared = 10,
		575	},
		576	};
125		577
126	/* TODO: Add L2 Cache information */	578	static struct kfd_gpu_cache_info navy_flounder_cache_info[] = {
		579	{
		580	/* TCP L1 Cache per CU */
		581	.cache_size = 16,
		582	.cache_level = 1,
		583	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		584	CRAT_CACHE_FLAGS_DATA_CACHE \|
		585	CRAT_CACHE_FLAGS_SIMD_CACHE),
		586	.num_cu_shared = 1,
		587	},
		588	{
		589	/* Scalar L1 Instruction Cache per SQC */
		590	.cache_size = 32,
		591	.cache_level = 1,
		592	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		593	CRAT_CACHE_FLAGS_INST_CACHE \|
		594	CRAT_CACHE_FLAGS_SIMD_CACHE),
		595	.num_cu_shared = 2,
		596	},
		597	{
		598	/* Scalar L1 Data Cache per SQC */
		599	.cache_size = 16,
		600	.cache_level = 1,
		601	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		602	CRAT_CACHE_FLAGS_DATA_CACHE \|
		603	CRAT_CACHE_FLAGS_SIMD_CACHE),
		604	.num_cu_shared = 2,
		605	},
		606	{
		607	/* GL1 Data Cache per SA */
		608	.cache_size = 128,
		609	.cache_level = 1,
		610	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		611	CRAT_CACHE_FLAGS_DATA_CACHE \|
		612	CRAT_CACHE_FLAGS_SIMD_CACHE),
		613	.num_cu_shared = 10,
		614	},
		615	{
		616	/* L2 Data Cache per GPU (Total Tex Cache) */
		617	.cache_size = 3072,
		618	.cache_level = 2,
		619	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		620	CRAT_CACHE_FLAGS_DATA_CACHE \|
		621	CRAT_CACHE_FLAGS_SIMD_CACHE),
		622	.num_cu_shared = 10,
		623	},
		624	{
		625	/* L3 Data Cache per GPU */
		626	.cache_size = 96*1024,
		627	.cache_level = 3,
		628	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		629	CRAT_CACHE_FLAGS_DATA_CACHE \|
		630	CRAT_CACHE_FLAGS_SIMD_CACHE),
		631	.num_cu_shared = 10,
		632	},
127	};	633	};
128		634
129	/* NOTE: In future if more information is added to struct kfd_gpu_cache_info	635	static struct kfd_gpu_cache_info dimgrey_cavefish_cache_info[] = {
130	* the following ASICs may need a separate table.	636	{
131	*/	637	/* TCP L1 Cache per CU */
132	#define hawaii_cache_info kaveri_cache_info	638	.cache_size = 16,
133	#define tonga_cache_info carrizo_cache_info	639	.cache_level = 1,
134	#define fiji_cache_info carrizo_cache_info	640	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
135	#define polaris10_cache_info carrizo_cache_info	641	CRAT_CACHE_FLAGS_DATA_CACHE \|
136	#define polaris11_cache_info carrizo_cache_info	642	CRAT_CACHE_FLAGS_SIMD_CACHE),
137	#define polaris12_cache_info carrizo_cache_info	643	.num_cu_shared = 1,
138	#define vegam_cache_info carrizo_cache_info	644	},
139	/* TODO - check & update Vega10 cache details */	645	{
140	#define vega10_cache_info carrizo_cache_info	646	/* Scalar L1 Instruction Cache per SQC */
141	#define raven_cache_info carrizo_cache_info	647	.cache_size = 32,
142	#define renoir_cache_info carrizo_cache_info	648	.cache_level = 1,
143	/* TODO - check & update Navi10 cache details */	649	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
144	#define navi10_cache_info carrizo_cache_info	650	CRAT_CACHE_FLAGS_INST_CACHE \|
145	#define vangogh_cache_info carrizo_cache_info	651	CRAT_CACHE_FLAGS_SIMD_CACHE),
		652	.num_cu_shared = 2,
		653	},
		654	{
		655	/* Scalar L1 Data Cache per SQC */
		656	.cache_size = 16,
		657	.cache_level = 1,
		658	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		659	CRAT_CACHE_FLAGS_DATA_CACHE \|
		660	CRAT_CACHE_FLAGS_SIMD_CACHE),
		661	.num_cu_shared = 2,
		662	},
		663	{
		664	/* GL1 Data Cache per SA */
		665	.cache_size = 128,
		666	.cache_level = 1,
		667	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		668	CRAT_CACHE_FLAGS_DATA_CACHE \|
		669	CRAT_CACHE_FLAGS_SIMD_CACHE),
		670	.num_cu_shared = 8,
		671	},
		672	{
		673	/* L2 Data Cache per GPU (Total Tex Cache) */
		674	.cache_size = 2048,
		675	.cache_level = 2,
		676	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		677	CRAT_CACHE_FLAGS_DATA_CACHE \|
		678	CRAT_CACHE_FLAGS_SIMD_CACHE),
		679	.num_cu_shared = 8,
		680	},
		681	{
		682	/* L3 Data Cache per GPU */
		683	.cache_size = 32*1024,
		684	.cache_level = 3,
		685	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
		686	CRAT_CACHE_FLAGS_DATA_CACHE \|
		687	CRAT_CACHE_FLAGS_SIMD_CACHE),
		688	.num_cu_shared = 8,
		689	},
		690	};
146		691
147	static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,	692	static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
148	struct crat_subtype_computeunit *cu)	693	struct crat_subtype_computeunit *cu)
@@ -544,7 +1089,7 @@ err: · 395 unmodified lines
544	}	1089	}
545		1090
546	/* Helper function. See kfd_fill_gpu_cache_info for parameter description */	1091	/* Helper function. See kfd_fill_gpu_cache_info for parameter description */
547	static int fill_in_pcache(struct crat_subtype_cache *pcache,	1092	static int fill_in_l1_pcache(struct crat_subtype_cache *pcache,
548	struct kfd_gpu_cache_info *pcache_info,	1093	struct kfd_gpu_cache_info *pcache_info,
549	struct kfd_cu_info *cu_info,	1094	struct kfd_cu_info *cu_info,
550	int mem_available,	1095	int mem_available,
@@ -597,6 +1142,70 @@ static int fill_in_pcache(struct crat_subtype_cache *pcache, · 46 unmodified lines
597	return 1;	1142	return 1;
598	}	1143	}
599		1144
		1145	/* Helper function. See kfd_fill_gpu_cache_info for parameter description */
		1146	static int fill_in_l2_l3_pcache(struct crat_subtype_cache *pcache,
		1147	struct kfd_gpu_cache_info *pcache_info,
		1148	struct kfd_cu_info *cu_info,
		1149	int mem_available,
		1150	int cache_type, unsigned int cu_processor_id)
		1151	{
		1152	unsigned int cu_sibling_map_mask;
		1153	int first_active_cu;
		1154	int i, j, k;
		1155
		1156	/* First check if enough memory is available */
		1157	if (sizeof(struct crat_subtype_cache) > mem_available)
		1158	return -ENOMEM;
		1159
		1160	cu_sibling_map_mask = cu_info->cu_bitmap[0][0];
		1161	cu_sibling_map_mask &=
		1162	((1 << pcache_info[cache_type].num_cu_shared) - 1);
		1163	first_active_cu = ffs(cu_sibling_map_mask);
		1164
		1165	/* CU could be inactive. In case of shared cache find the first active
		1166	* CU. and incase of non-shared cache check if the CU is inactive. If
		1167	* inactive active skip it
		1168	*/
		1169	if (first_active_cu) {
		1170	memset(pcache, 0, sizeof(struct crat_subtype_cache));
		1171	pcache->type = CRAT_SUBTYPE_CACHE_AFFINITY;
		1172	pcache->length = sizeof(struct crat_subtype_cache);
		1173	pcache->flags = pcache_info[cache_type].flags;
		1174	pcache->processor_id_low = cu_processor_id
		1175	+ (first_active_cu - 1);
		1176	pcache->cache_level = pcache_info[cache_type].cache_level;
		1177	pcache->cache_size = pcache_info[cache_type].cache_size;
		1178
		1179	/* Sibling map is w.r.t processor_id_low, so shift out
		1180	* inactive CU
		1181	*/
		1182	cu_sibling_map_mask =
		1183	cu_sibling_map_mask >> (first_active_cu - 1);
		1184	k = 0;
		1185	for (i = 0; i < cu_info->num_shader_engines; i++) {
		1186	for (j = 0; j < cu_info->num_shader_arrays_per_engine;
		1187	j++) {
		1188	pcache->sibling_map[k] =
		1189	(uint8_t)(cu_sibling_map_mask & 0xFF);
		1190	pcache->sibling_map[k+1] =
		1191	(uint8_t)((cu_sibling_map_mask >> 8) & 0xFF);
		1192	pcache->sibling_map[k+2] =
		1193	(uint8_t)((cu_sibling_map_mask >> 16) & 0xFF);
		1194	pcache->sibling_map[k+3] =
		1195	(uint8_t)((cu_sibling_map_mask >> 24) & 0xFF);
		1196	k += 4;
		1197	cu_sibling_map_mask =
		1198	cu_info->cu_bitmap[i % 4][j + i / 4];
		1199	cu_sibling_map_mask &= (
		1200	(1 << pcache_info[cache_type].num_cu_shared)
		1201	- 1);
		1202	}
		1203	}
		1204	return 0;
		1205	}
		1206	return 1;
		1207	}
		1208
600	/* kfd_fill_gpu_cache_info - Fill GPU cache info using kfd_gpu_cache_info	1209	/* kfd_fill_gpu_cache_info - Fill GPU cache info using kfd_gpu_cache_info
601	* tables	1210	* tables
602	*	1211	*
@@ -624,6 +1233,7 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, · 21 unmodified lines
624	int mem_available = available_size;	1233	int mem_available = available_size;
625	unsigned int cu_processor_id;	1234	unsigned int cu_processor_id;
626	int ret;	1235	int ret;
		1236	unsigned int num_cu_shared;
627		1237
628	switch (kdev->device_info->asic_family) {	1238	switch (kdev->device_info->asic_family) {
629	case CHIP_KAVERI:	1239	case CHIP_KAVERI:
@@ -663,12 +1273,21 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, · 33 unmodified lines
663	num_of_cache_types = ARRAY_SIZE(vegam_cache_info);	1273	num_of_cache_types = ARRAY_SIZE(vegam_cache_info);
664	break;	1274	break;
665	case CHIP_VEGA10:	1275	case CHIP_VEGA10:
		1276	pcache_info = vega10_cache_info;
		1277	num_of_cache_types = ARRAY_SIZE(vega10_cache_info);
		1278	break;
666	case CHIP_VEGA12:	1279	case CHIP_VEGA12:
		1280	pcache_info = vega12_cache_info;
		1281	num_of_cache_types = ARRAY_SIZE(vega12_cache_info);
		1282	break;
667	case CHIP_VEGA20:	1283	case CHIP_VEGA20:
668	case CHIP_ARCTURUS:	1284	case CHIP_ARCTURUS:
		1285	pcache_info = vega20_cache_info;
		1286	num_of_cache_types = ARRAY_SIZE(vega20_cache_info);
		1287	break;
669	case CHIP_ALDEBARAN:	1288	case CHIP_ALDEBARAN:
670	pcache_info = vega10_cache_info;	1289	pcache_info = aldebaran_cache_info;
671	num_of_cache_types = ARRAY_SIZE(vega10_cache_info);	1290	num_of_cache_types = ARRAY_SIZE(aldebaran_cache_info);
672	break;	1291	break;
673	case CHIP_RAVEN:	1292	case CHIP_RAVEN:
674	pcache_info = raven_cache_info;	1293	pcache_info = raven_cache_info;
@@ -680,12 +1299,24 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, · 5 unmodified lines
680	break;	1299	break;
681	case CHIP_NAVI10:	1300	case CHIP_NAVI10:
682	case CHIP_NAVI12:	1301	case CHIP_NAVI12:
		1302	pcache_info = navi10_cache_info;
		1303	num_of_cache_types = ARRAY_SIZE(navi10_cache_info);
		1304	break;
683	case CHIP_NAVI14:	1305	case CHIP_NAVI14:
		1306	pcache_info = navi14_cache_info;
		1307	num_of_cache_types = ARRAY_SIZE(navi14_cache_info);
		1308	break;
684	case CHIP_SIENNA_CICHLID:	1309	case CHIP_SIENNA_CICHLID:
		1310	pcache_info = sienna_cichlid_cache_info;
		1311	num_of_cache_types = ARRAY_SIZE(sienna_cichlid_cache_info);
		1312	break;
685	case CHIP_NAVY_FLOUNDER:	1313	case CHIP_NAVY_FLOUNDER:
		1314	pcache_info = navy_flounder_cache_info;
		1315	num_of_cache_types = ARRAY_SIZE(navy_flounder_cache_info);
		1316	break;
686	case CHIP_DIMGREY_CAVEFISH:	1317	case CHIP_DIMGREY_CAVEFISH:
687	pcache_info = navi10_cache_info;	1318	pcache_info = dimgrey_cavefish_cache_info;
688	num_of_cache_types = ARRAY_SIZE(navi10_cache_info);	1319	num_of_cache_types = ARRAY_SIZE(dimgrey_cavefish_cache_info);
689	break;	1320	break;
690	case CHIP_VANGOGH:	1321	case CHIP_VANGOGH:
691	pcache_info = vangogh_cache_info;	1322	pcache_info = vangogh_cache_info;
@@ -709,40 +1340,58 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, · 17 unmodified lines
709	*/	1340	*/
710		1341
711	for (ct = 0; ct < num_of_cache_types; ct++) {	1342	for (ct = 0; ct < num_of_cache_types; ct++) {
712	cu_processor_id = gpu_processor_id;	1343	cu_processor_id = gpu_processor_id;
713	for (i = 0; i < cu_info->num_shader_engines; i++) {	1344	if (pcache_info[ct].cache_level == 1) {
714	for (j = 0; j < cu_info->num_shader_arrays_per_engine;	1345	for (i = 0; i < cu_info->num_shader_engines; i++) {
715	j++) {	1346	for (j = 0; j < cu_info->num_shader_arrays_per_engine; j++) {
716	for (k = 0; k < cu_info->num_cu_per_sh;	1347	for (k = 0; k < cu_info->num_cu_per_sh;
717	k += pcache_info[ct].num_cu_shared) {	1348	k += pcache_info[ct].num_cu_shared) {
718		1349	ret = fill_in_l1_pcache(pcache,
719	ret = fill_in_pcache(pcache,	1350	pcache_info,
720	pcache_info,	1351	cu_info,
721	cu_info,	1352	mem_available,
722	mem_available,	1353	cu_info->cu_bitmap[i % 4][j + i / 4],
723	cu_info->cu_bitmap[i % 4][j + i / 4],	1354	ct,
724	ct,	1355	cu_processor_id,
725	cu_processor_id,	1356	k);
726	k);	1357
727		1358	if (ret < 0)
728	if (ret < 0)	1359	break;
729	break;	1360
730		1361	if (!ret) {
731	if (!ret) {	1362	pcache++;
732	pcache++;	1363	(*num_of_entries)++;
733	(*num_of_entries)++;	1364	mem_available -= sizeof(*pcache);
734	mem_available -=	1365	(size_filled) += sizeof(pcache);
735	sizeof(*pcache);	1366	}
736	(*size_filled) +=	1367
737	sizeof(*pcache);	1368	/* Move to next CU block */
738	}	1369	num_cu_shared = ((k + pcache_info[ct].num_cu_shared) <=
739		1370	cu_info->num_cu_per_sh) ?
740	/* Move to next CU block */	1371	pcache_info[ct].num_cu_shared :
741	cu_processor_id +=	1372	(cu_info->num_cu_per_sh - k);
742	pcache_info[ct].num_cu_shared;	1373	cu_processor_id += num_cu_shared;
743	}
744	}
745	}	1374	}
		1375	}
		1376	}
		1377	} else {
		1378	ret = fill_in_l2_l3_pcache(pcache,
		1379	pcache_info,
		1380	cu_info,
		1381	mem_available,
		1382	ct,
		1383	cu_processor_id);
		1384
		1385	if (ret < 0)
		1386	break;
		1387
		1388	if (!ret) {
		1389	pcache++;
		1390	(*num_of_entries)++;
		1391	mem_available -= sizeof(*pcache);
		1392	(size_filled) += sizeof(pcache);
		1393	}
		1394	}
746	}	1395	}
747		1396
748	pr_debug("Added [%d] GPU cache entries\n", *num_of_entries);	1397	pr_debug("Added [%d] GPU cache entries\n", *num_of_entries);
@@ -1100,6 +1749,92 @@ static int kfd_fill_gpu_memory_affinity(int *avail_size, · 351 unmodified lines
1100	return 0;	1749	return 0;
1101	}	1750	}
1102		1751
		1752	#ifdef CONFIG_ACPI_NUMA
		1753	static void kfd_find_numa_node_in_srat(struct kfd_dev *kdev)
		1754	{
		1755	struct acpi_table_header *table_header = NULL;
		1756	struct acpi_subtable_header *sub_header = NULL;
		1757	unsigned long table_end, subtable_len;
		1758	u32 pci_id = pci_domain_nr(kdev->pdev->bus) << 16 \|
		1759	pci_dev_id(kdev->pdev);
		1760	u32 bdf;
		1761	acpi_status status;
		1762	struct acpi_srat_cpu_affinity *cpu;
		1763	struct acpi_srat_generic_affinity *gpu;
		1764	int pxm = 0, max_pxm = 0;
		1765	int numa_node = NUMA_NO_NODE;
		1766	bool found = false;
		1767
		1768	/* Fetch the SRAT table from ACPI */
		1769	status = acpi_get_table(ACPI_SIG_SRAT, 0, &table_header);
		1770	if (status == AE_NOT_FOUND) {
		1771	pr_warn("SRAT table not found\n");
		1772	return;
		1773	} else if (ACPI_FAILURE(status)) {
		1774	const char *err = acpi_format_exception(status);
		1775	pr_err("SRAT table error: %s\n", err);
		1776	return;
		1777	}
		1778
		1779	table_end = (unsigned long)table_header + table_header->length;
		1780
		1781	/* Parse all entries looking for a match. */
		1782	sub_header = (struct acpi_subtable_header *)
		1783	((unsigned long)table_header +
		1784	sizeof(struct acpi_table_srat));
		1785	subtable_len = sub_header->length;
		1786
		1787	while (((unsigned long)sub_header) + subtable_len < table_end) {
		1788	/*
		1789	* If length is 0, break from this loop to avoid
		1790	* infinite loop.
		1791	*/
		1792	if (subtable_len == 0) {
		1793	pr_err("SRAT invalid zero length\n");
		1794	break;
		1795	}
		1796
		1797	switch (sub_header->type) {
		1798	case ACPI_SRAT_TYPE_CPU_AFFINITY:
		1799	cpu = (struct acpi_srat_cpu_affinity *)sub_header;
		1800	pxm = ((u32 )cpu->proximity_domain_hi) << 8 \|
		1801	cpu->proximity_domain_lo;
		1802	if (pxm > max_pxm)
		1803	max_pxm = pxm;
		1804	break;
		1805	case ACPI_SRAT_TYPE_GENERIC_AFFINITY:
		1806	gpu = (struct acpi_srat_generic_affinity *)sub_header;
		1807	bdf = ((u16 )(&gpu->device_handle[0])) << 16 \|
		1808	((u16 )(&gpu->device_handle[2]));
		1809	if (bdf == pci_id) {
		1810	found = true;
		1811	numa_node = pxm_to_node(gpu->proximity_domain);
		1812	}
		1813	break;
		1814	default:
		1815	break;
		1816	}
		1817
		1818	if (found)
		1819	break;
		1820
		1821	sub_header = (struct acpi_subtable_header *)
		1822	((unsigned long)sub_header + subtable_len);
		1823	subtable_len = sub_header->length;
		1824	}
		1825
		1826	acpi_put_table(table_header);
		1827
		1828	/* Workaround bad cpu-gpu binding case */
		1829	if (found && (numa_node < 0 \|\|
		1830	numa_node > pxm_to_node(max_pxm)))
		1831	numa_node = 0;
		1832
		1833	if (numa_node != NUMA_NO_NODE)
		1834	set_dev_node(&kdev->pdev->dev, numa_node);
		1835	}
		1836	#endif
		1837
1103	/* kfd_fill_gpu_direct_io_link - Fill in direct io link from GPU	1838	/* kfd_fill_gpu_direct_io_link - Fill in direct io link from GPU
1104	* to its NUMA node	1839	* to its NUMA node
1105	* @avail_size: Available size in the memory	1840	* @avail_size: Available size in the memory
@@ -1140,11 +1875,17 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size, · 34 unmodified lines
1140	*/	1875	*/
1141	sub_type_hdr->flags \|= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;	1876	sub_type_hdr->flags \|= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
1142	sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;	1877	sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
		1878	sub_type_hdr->num_hops_xgmi = 1;
1143	} else {	1879	} else {
1144	sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;	1880	sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
1145	}	1881	}
1146		1882
1147	sub_type_hdr->proximity_domain_from = proximity_domain;	1883	sub_type_hdr->proximity_domain_from = proximity_domain;
		1884
		1885	#ifdef CONFIG_ACPI_NUMA
		1886	if (kdev->pdev->dev.numa_node == NUMA_NO_NODE)
		1887	kfd_find_numa_node_in_srat(kdev);
		1888	#endif
1148	#ifdef CONFIG_NUMA	1889	#ifdef CONFIG_NUMA
1149	if (kdev->pdev->dev.numa_node == NUMA_NO_NODE)	1890	if (kdev->pdev->dev.numa_node == NUMA_NO_NODE)
1150	sub_type_hdr->proximity_domain_to = 0;	1891	sub_type_hdr->proximity_domain_to = 0;

drivers/gpu/drm/amd/amdkfd/kfd_device.c+15−4

 #include "kfd_priv.h"
 #include "kfd_device_queue_manager.h"
 #include "kfd_pm4_headers_vi.h"
+#include "kfd_pm4_headers_aldebaran.h"
 #include "cwsr_trap_handler.h"
 #include "kfd_iommu.h"
 #include "amdgpu_amdkfd.h"
 #include "kfd_smi_events.h"
+#include "kfd_migrate.h"
 #define MQD_SIZE_ALIGNED 768
 	[CHIP_VEGA20] = {&vega20_device_info, NULL},
 	[CHIP_RENOIR] = {&renoir_device_info, NULL},
 	[CHIP_ARCTURUS] = {&arcturus_device_info, &arcturus_device_info},
-	[CHIP_ALDEBARAN] = {&aldebaran_device_info, NULL},
+	[CHIP_ALDEBARAN] = {&aldebaran_device_info, &aldebaran_device_info},
 	[CHIP_NAVI10] = {&navi10_device_info, NULL},
 	[CHIP_NAVI12] = {&navi12_device_info, &navi12_device_info},
 	[CHIP_NAVI14] = {&navi14_device_info, NULL},
 			&& kfd->device_info->asic_family <= CHIP_RAVEN
 			&& kfd->mec2_fw_version >= 0x1b3)
 		|| (kfd->device_info->asic_family == CHIP_ARCTURUS
-			&& kfd->mec2_fw_version >= 0x30))
+			&& kfd->mec2_fw_version >= 0x30)
+		|| (kfd->device_info->asic_family == CHIP_ALDEBARAN
+			&& kfd->mec2_fw_version >= 0x28))
 		ret = amdgpu_amdkfd_alloc_gws(kfd->kgd,
 				amdgpu_amdkfd_get_num_gws(kfd->kgd), &kfd->gws);
 			 struct drm_device *ddev,
 			 const struct kgd2kfd_shared_resources *gpu_resources)
 {
-	unsigned int size;
+	unsigned int size, map_process_packet_size;
 	kfd->ddev = ddev;
 	kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
 	 * calculate max size of runlist packet.
 	 * There can be only 2 packets at once
 	 */
-	size += (KFD_MAX_NUM_OF_PROCESSES * sizeof(struct pm4_mes_map_process) +
+	map_process_packet_size =
+			kfd->device_info->asic_family == CHIP_ALDEBARAN ?
+				sizeof(struct pm4_mes_map_process_aldebaran) :
+					sizeof(struct pm4_mes_map_process);
+	size += (KFD_MAX_NUM_OF_PROCESSES * map_process_packet_size +
 		max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues)
 		+ sizeof(struct pm4_mes_runlist)) * 2;
 	kfd_cwsr_init(kfd);
+	svm_migrate_init((struct amdgpu_device *)kfd->kgd);
 	if (kfd_resume(kfd))
 		goto kfd_resume_error;
 {
 	if (kfd->init_complete) {
 		kgd2kfd_suspend(kfd, false);
+		svm_migrate_fini((struct amdgpu_device *)kfd->kgd);
 		device_queue_manager_uninit(kfd->dqm);
 		kfd_interrupt_exit(kfd);
 		kfd_topology_remove_device(kfd);

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c+3−3

@@ -738,7 +738,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, · 737 unmodified lines
738		738
739	pdd = qpd_to_pdd(qpd);	739	pdd = qpd_to_pdd(qpd);
740	/* Retrieve PD base */	740	/* Retrieve PD base */
741	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);	741	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
742		742
743	dqm_lock(dqm);	743	dqm_lock(dqm);
744	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */	744	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
@@ -821,7 +821,7 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm, · 76 unmodified lines
821		821
822	pdd = qpd_to_pdd(qpd);	822	pdd = qpd_to_pdd(qpd);
823	/* Retrieve PD base */	823	/* Retrieve PD base */
824	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);	824	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
825		825
826	dqm_lock(dqm);	826	dqm_lock(dqm);
827	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */	827	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
@@ -873,7 +873,7 @@ static int register_process(struct device_queue_manager *dqm, · 45 unmodified lines
873		873
874	pdd = qpd_to_pdd(qpd);	874	pdd = qpd_to_pdd(qpd);
875	/* Retrieve PD base */	875	/* Retrieve PD base */
876	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);	876	pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
877		877
878	dqm_lock(dqm);	878	dqm_lock(dqm);
879	list_add(&n->list, &dqm->queues);	879	list_add(&n->list, &dqm->queues);

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c+11−2

@@ -61,10 +61,19 @@ static int update_qpd_v9(struct device_queue_manager *dqm, · 60 unmodified lines
61	qpd->sh_mem_config =	61	qpd->sh_mem_config =
62	SH_MEM_ALIGNMENT_MODE_UNALIGNED <<	62	SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
63	SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;	63	SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
64	if (dqm->dev->noretry &&	64
65	!dqm->dev->use_iommu_v2)	65	if (dqm->dev->device_info->asic_family == CHIP_ALDEBARAN) {
		66	/* Aldebaran can safely support different XNACK modes
		67	* per process
		68	*/
		69	if (!pdd->process->xnack_enabled)
		70	qpd->sh_mem_config \|=
		71	1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
		72	} else if (dqm->dev->noretry &&
		73	!dqm->dev->use_iommu_v2) {
66	qpd->sh_mem_config \|=	74	qpd->sh_mem_config \|=
67	1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;	75	1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
		76	}
68		77
69	qpd->sh_mem_ape1_limit = 0;	78	qpd->sh_mem_ape1_limit = 0;
70	qpd->sh_mem_ape1_base = 0;	79	qpd->sh_mem_ape1_base = 0;

drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c+4−0

 			case CHIP_POLARIS12:
 			case CHIP_VEGAM:
 				kfd_init_apertures_vi(pdd, id);
+				/* VI GPUs cannot support SVM with only
+				 * 40 bits of virtual address space.
+				 */
+				process->svm_disabled = true;
 				break;
 			case CHIP_VEGA10:
 			case CHIP_VEGA12:

drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c+0−1

@@ -25,7 +25,6 @@ · 24 unmodified lines
25	#include "soc15_int.h"	25	#include "soc15_int.h"
26	#include "kfd_device_queue_manager.h"	26	#include "kfd_device_queue_manager.h"
27	#include "kfd_smi_events.h"	27	#include "kfd_smi_events.h"
28	#include "amdgpu.h"
29		28
30	enum SQ_INTERRUPT_WORD_ENCODING {	29	enum SQ_INTERRUPT_WORD_ENCODING {
31	SQ_INTERRUPT_WORD_ENCODING_AUTO = 0x0,	30	SQ_INTERRUPT_WORD_ENCODING_AUTO = 0x0,

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c+937−0

@@ -0,0 +1,937 @@
	1	// SPDX-License-Identifier: GPL-2.0 OR MIT
	2	/*
	3	* Copyright 2020-2021 Advanced Micro Devices, Inc.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining a
	6	* copy of this software and associated documentation files (the "Software"),
	7	* to deal in the Software without restriction, including without limitation
	8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	9	* and/or sell copies of the Software, and to permit persons to whom the
	10	* Software is furnished to do so, subject to the following conditions:
	11	*
	12	* The above copyright notice and this permission notice shall be included in
	13	* all copies or substantial portions of the Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	21	* OTHER DEALINGS IN THE SOFTWARE.
	22	*/
	23
	24	#include <linux/types.h>
	25	#include <linux/hmm.h>
	26	#include <linux/dma-direction.h>
	27	#include <linux/dma-mapping.h>
	28	#include "amdgpu_sync.h"
	29	#include "amdgpu_object.h"
	30	#include "amdgpu_vm.h"
	31	#include "amdgpu_mn.h"
	32	#include "kfd_priv.h"
	33	#include "kfd_svm.h"
	34	#include "kfd_migrate.h"
	35
	36	static uint64_t
	37	svm_migrate_direct_mapping_addr(struct amdgpu_device *adev, uint64_t addr)
	38	{
	39	return addr + amdgpu_ttm_domain_start(adev, TTM_PL_VRAM);
	40	}
	41
	42	static int
	43	svm_migrate_gart_map(struct amdgpu_ring *ring, uint64_t npages,
	44	dma_addr_t addr, uint64_t gart_addr, uint64_t flags)
	45	{
	46	struct amdgpu_device *adev = ring->adev;
	47	struct amdgpu_job *job;
	48	unsigned int num_dw, num_bytes;
	49	struct dma_fence *fence;
	50	uint64_t src_addr, dst_addr;
	51	uint64_t pte_flags;
	52	void *cpu_addr;
	53	int r;
	54
	55	/* use gart window 0 */
	56	*gart_addr = adev->gmc.gart_start;
	57
	58	num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
	59	num_bytes = npages * 8;
	60
	61	r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes,
	62	AMDGPU_IB_POOL_DELAYED, &job);
	63	if (r)
	64	return r;
	65
	66	src_addr = num_dw * 4;
	67	src_addr += job->ibs[0].gpu_addr;
	68
	69	dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
	70	amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
	71	dst_addr, num_bytes, false);
	72
	73	amdgpu_ring_pad_ib(ring, &job->ibs[0]);
	74	WARN_ON(job->ibs[0].length_dw > num_dw);
	75
	76	pte_flags = AMDGPU_PTE_VALID \| AMDGPU_PTE_READABLE;
	77	pte_flags \|= AMDGPU_PTE_SYSTEM \| AMDGPU_PTE_SNOOPED;
	78	if (!(flags & KFD_IOCTL_SVM_FLAG_GPU_RO))
	79	pte_flags \|= AMDGPU_PTE_WRITEABLE;
	80	pte_flags \|= adev->gart.gart_pte_flags;
	81
	82	cpu_addr = &job->ibs[0].ptr[num_dw];
	83
	84	r = amdgpu_gart_map(adev, 0, npages, addr, pte_flags, cpu_addr);
	85	if (r)
	86	goto error_free;
	87
	88	r = amdgpu_job_submit(job, &adev->mman.entity,
	89	AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
	90	if (r)
	91	goto error_free;
	92
	93	dma_fence_put(fence);
	94
	95	return r;
	96
	97	error_free:
	98	amdgpu_job_free(job);
	99	return r;
	100	}
	101
	102	/**
	103	* svm_migrate_copy_memory_gart - sdma copy data between ram and vram
	104	*
	105	* @adev: amdgpu device the sdma ring running
	106	* @src: source page address array
	107	* @dst: destination page address array
	108	* @npages: number of pages to copy
	109	* @direction: enum MIGRATION_COPY_DIR
	110	* @mfence: output, sdma fence to signal after sdma is done
	111	*
	112	* ram address uses GART table continuous entries mapping to ram pages,
	113	* vram address uses direct mapping of vram pages, which must have npages
	114	* number of continuous pages.
	115	* GART update and sdma uses same buf copy function ring, sdma is splited to
	116	* multiple GTT_MAX_PAGES transfer, all sdma operations are serialized, wait for
	117	* the last sdma finish fence which is returned to check copy memory is done.
	118	*
	119	* Context: Process context, takes and releases gtt_window_lock
	120	*
	121	* Return:
	122	* 0 - OK, otherwise error code
	123	*/
	124
	125	static int
	126	svm_migrate_copy_memory_gart(struct amdgpu_device adev, dma_addr_t sys,
	127	uint64_t *vram, uint64_t npages,
	128	enum MIGRATION_COPY_DIR direction,
	129	struct dma_fence **mfence)
	130	{
	131	const uint64_t GTT_MAX_PAGES = AMDGPU_GTT_MAX_TRANSFER_SIZE;
	132	struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
	133	uint64_t gart_s, gart_d;
	134	struct dma_fence *next;
	135	uint64_t size;
	136	int r;
	137
	138	mutex_lock(&adev->mman.gtt_window_lock);
	139
	140	while (npages) {
	141	size = min(GTT_MAX_PAGES, npages);
	142
	143	if (direction == FROM_VRAM_TO_RAM) {
	144	gart_s = svm_migrate_direct_mapping_addr(adev, *vram);
	145	r = svm_migrate_gart_map(ring, size, sys, &gart_d, 0);
	146
	147	} else if (direction == FROM_RAM_TO_VRAM) {
	148	r = svm_migrate_gart_map(ring, size, sys, &gart_s,
	149	KFD_IOCTL_SVM_FLAG_GPU_RO);
	150	gart_d = svm_migrate_direct_mapping_addr(adev, *vram);
	151	}
	152	if (r) {
	153	pr_debug("failed %d to create gart mapping\n", r);
	154	goto out_unlock;
	155	}
	156
	157	r = amdgpu_copy_buffer(ring, gart_s, gart_d, size * PAGE_SIZE,
	158	NULL, &next, false, true, false);
	159	if (r) {
	160	pr_debug("failed %d to copy memory\n", r);
	161	goto out_unlock;
	162	}
	163
	164	dma_fence_put(*mfence);
	165	*mfence = next;
	166	npages -= size;
	167	if (npages) {
	168	sys += size;
	169	vram += size;
	170	}
	171	}
	172
	173	out_unlock:
	174	mutex_unlock(&adev->mman.gtt_window_lock);
	175
	176	return r;
	177	}
	178
	179	/**
	180	* svm_migrate_copy_done - wait for memory copy sdma is done
	181	*
	182	* @adev: amdgpu device the sdma memory copy is executing on
	183	* @mfence: migrate fence
	184	*
	185	* Wait for dma fence is signaled, if the copy ssplit into multiple sdma
	186	* operations, this is the last sdma operation fence.
	187	*
	188	* Context: called after svm_migrate_copy_memory
	189	*
	190	* Return:
	191	* 0 - success
	192	* otherwise - error code from dma fence signal
	193	*/
	194	static int
	195	svm_migrate_copy_done(struct amdgpu_device adev, struct dma_fence mfence)
	196	{
	197	int r = 0;
	198
	199	if (mfence) {
	200	r = dma_fence_wait(mfence, false);
	201	dma_fence_put(mfence);
	202	pr_debug("sdma copy memory fence done\n");
	203	}
	204
	205	return r;
	206	}
	207
	208	static uint64_t
	209	svm_migrate_node_physical_addr(struct amdgpu_device *adev,
	210	struct drm_mm_node *mm_node, uint64_t offset)
	211	{
	212	struct drm_mm_node node = mm_node;
	213	uint64_t pos = *offset;
	214
	215	if (node->start == AMDGPU_BO_INVALID_OFFSET) {
	216	pr_debug("drm node is not validated\n");
	217	return 0;
	218	}
	219
	220	pr_debug("vram node start 0x%llx npages 0x%llx\n", node->start,
	221	node->size);
	222
	223	if (pos >= node->size) {
	224	do {
	225	pos -= node->size;
	226	node++;
	227	} while (pos >= node->size);
	228
	229	*mm_node = node;
	230	*offset = pos;
	231	}
	232
	233	return (node->start + pos) << PAGE_SHIFT;
	234	}
	235
	236	unsigned long
	237	svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr)
	238	{
	239	return (addr + adev->kfd.dev->pgmap.range.start) >> PAGE_SHIFT;
	240	}
	241
	242	static void
	243	svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)
	244	{
	245	struct page *page;
	246
	247	page = pfn_to_page(pfn);
	248	page->zone_device_data = prange;
	249	get_page(page);
	250	lock_page(page);
	251	}
	252
	253	static void
	254	svm_migrate_put_vram_page(struct amdgpu_device *adev, unsigned long addr)
	255	{
	256	struct page *page;
	257
	258	page = pfn_to_page(svm_migrate_addr_to_pfn(adev, addr));
	259	unlock_page(page);
	260	put_page(page);
	261	}
	262
	263	static unsigned long
	264	svm_migrate_addr(struct amdgpu_device adev, struct page page)
	265	{
	266	unsigned long addr;
	267
	268	addr = page_to_pfn(page) << PAGE_SHIFT;
	269	return (addr - adev->kfd.dev->pgmap.range.start);
	270	}
	271
	272	static struct page *
	273	svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr)
	274	{
	275	struct page *page;
	276
	277	page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
	278	if (page)
	279	lock_page(page);
	280
	281	return page;
	282	}
	283
	284	static void svm_migrate_put_sys_page(unsigned long addr)
	285	{
	286	struct page *page;
	287
	288	page = pfn_to_page(addr >> PAGE_SHIFT);
	289	unlock_page(page);
	290	put_page(page);
	291	}
	292
	293	static int
	294	svm_migrate_copy_to_vram(struct amdgpu_device adev, struct svm_range prange,
	295	struct migrate_vma migrate, struct dma_fence *mfence,
	296	dma_addr_t *scratch)
	297	{
	298	uint64_t npages = migrate->cpages;
	299	struct device *dev = adev->dev;
	300	struct drm_mm_node *node;
	301	dma_addr_t *src;
	302	uint64_t *dst;
	303	uint64_t vram_addr;
	304	uint64_t offset;
	305	uint64_t i, j;
	306	int r;
	307
	308	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
	309	prange->last);
	310
	311	src = scratch;
	312	dst = (uint64_t *)(scratch + npages);
	313
	314	r = svm_range_vram_node_new(adev, prange, true);
	315	if (r) {
	316	pr_debug("failed %d get 0x%llx pages from vram\n", r, npages);
	317	goto out;
	318	}
	319
	320	node = prange->ttm_res->mm_node;
	321	offset = prange->offset;
	322	vram_addr = svm_migrate_node_physical_addr(adev, &node, &offset);
	323	if (!vram_addr) {
	324	WARN_ONCE(1, "vram node address is 0\n");
	325	r = -ENOMEM;
	326	goto out;
	327	}
	328
	329	for (i = j = 0; i < npages; i++) {
	330	struct page *spage;
	331
	332	dst[i] = vram_addr + (j << PAGE_SHIFT);
	333	migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
	334	svm_migrate_get_vram_page(prange, migrate->dst[i]);
	335
	336	migrate->dst[i] = migrate_pfn(migrate->dst[i]);
	337	migrate->dst[i] \|= MIGRATE_PFN_LOCKED;
	338
	339	if (migrate->src[i] & MIGRATE_PFN_VALID) {
	340	spage = migrate_pfn_to_page(migrate->src[i]);
	341	src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
	342	DMA_TO_DEVICE);
	343	r = dma_mapping_error(dev, src[i]);
	344	if (r) {
	345	pr_debug("failed %d dma_map_page\n", r);
	346	goto out_free_vram_pages;
	347	}
	348	} else {
	349	if (j) {
	350	r = svm_migrate_copy_memory_gart(
	351	adev, src + i - j,
	352	dst + i - j, j,
	353	FROM_RAM_TO_VRAM,
	354	mfence);
	355	if (r)
	356	goto out_free_vram_pages;
	357	offset += j;
	358	vram_addr = (node->start + offset) << PAGE_SHIFT;
	359	j = 0;
	360	} else {
	361	offset++;
	362	vram_addr += PAGE_SIZE;
	363	}
	364	if (offset >= node->size) {
	365	node++;
	366	pr_debug("next node size 0x%llx\n", node->size);
	367	vram_addr = node->start << PAGE_SHIFT;
	368	offset = 0;
	369	}
	370	continue;
	371	}
	372
	373	pr_debug("dma mapping src to 0x%llx, page_to_pfn 0x%lx\n",
	374	src[i] >> PAGE_SHIFT, page_to_pfn(spage));
	375
	376	if (j + offset >= node->size - 1 && i < npages - 1) {
	377	r = svm_migrate_copy_memory_gart(adev, src + i - j,
	378	dst + i - j, j + 1,
	379	FROM_RAM_TO_VRAM,
	380	mfence);
	381	if (r)
	382	goto out_free_vram_pages;
	383
	384	node++;
	385	pr_debug("next node size 0x%llx\n", node->size);
	386	vram_addr = node->start << PAGE_SHIFT;
	387	offset = 0;
	388	j = 0;
	389	} else {
	390	j++;
	391	}
	392	}
	393
	394	r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
	395	FROM_RAM_TO_VRAM, mfence);
	396
	397	out_free_vram_pages:
	398	if (r) {
	399	pr_debug("failed %d to copy memory to vram\n", r);
	400	while (i--) {
	401	svm_migrate_put_vram_page(adev, dst[i]);
	402	migrate->dst[i] = 0;
	403	}
	404	}
	405
	406	out:
	407	return r;
	408	}
	409
	410	static int
	411	svm_migrate_vma_to_vram(struct amdgpu_device adev, struct svm_range prange,
	412	struct vm_area_struct *vma, uint64_t start,
	413	uint64_t end)
	414	{
	415	uint64_t npages = (end - start) >> PAGE_SHIFT;
	416	struct dma_fence *mfence = NULL;
	417	struct migrate_vma migrate;
	418	dma_addr_t *scratch;
	419	size_t size;
	420	void *buf;
	421	int r = -ENOMEM;
	422	int retry = 0;
	423
	424	memset(&migrate, 0, sizeof(migrate));
	425	migrate.vma = vma;
	426	migrate.start = start;
	427	migrate.end = end;
	428	migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;
	429	migrate.pgmap_owner = adev;
	430
	431	size = 2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t);
	432	size *= npages;
	433	buf = kvmalloc(size, GFP_KERNEL \| __GFP_ZERO);
	434	if (!buf)
	435	goto out;
	436
	437	migrate.src = buf;
	438	migrate.dst = migrate.src + npages;
	439	scratch = (dma_addr_t *)(migrate.dst + npages);
	440
	441	retry:
	442	r = migrate_vma_setup(&migrate);
	443	if (r) {
	444	pr_debug("failed %d prepare migrate svms 0x%p [0x%lx 0x%lx]\n",
	445	r, prange->svms, prange->start, prange->last);
	446	goto out_free;
	447	}
	448	if (migrate.cpages != npages) {
	449	pr_debug("collect 0x%lx/0x%llx pages, retry\n", migrate.cpages,
	450	npages);
	451	migrate_vma_finalize(&migrate);
	452	if (retry++ >= 3) {
	453	r = -ENOMEM;
	454	pr_debug("failed %d migrate svms 0x%p [0x%lx 0x%lx]\n",
	455	r, prange->svms, prange->start, prange->last);
	456	goto out_free;
	457	}
	458
	459	goto retry;
	460	}
	461
	462	if (migrate.cpages) {
	463	r = svm_migrate_copy_to_vram(adev, prange, &migrate, &mfence,
	464	scratch);
	465	migrate_vma_pages(&migrate);
	466	svm_migrate_copy_done(adev, mfence);
	467	migrate_vma_finalize(&migrate);
	468	}
	469
	470	svm_range_dma_unmap(adev->dev, scratch, 0, npages);
	471	svm_range_free_dma_mappings(prange);
	472
	473	out_free:
	474	kvfree(buf);
	475	out:
	476	return r;
	477	}
	478
	479	/**
	480	* svm_migrate_ram_to_vram - migrate svm range from system to device
	481	* @prange: range structure
	482	* @best_loc: the device to migrate to
	483	* @mm: the process mm structure
	484	*
	485	* Context: Process context, caller hold mmap read lock, svms lock, prange lock
	486	*
	487	* Return:
	488	* 0 - OK, otherwise error code
	489	*/
	490	static int
	491	svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
	492	struct mm_struct *mm)
	493	{
	494	unsigned long addr, start, end;
	495	struct vm_area_struct *vma;
	496	struct amdgpu_device *adev;
	497	int r = 0;
	498
	499	if (prange->actual_loc == best_loc) {
	500	pr_debug("svms 0x%p [0x%lx 0x%lx] already on best_loc 0x%x\n",
	501	prange->svms, prange->start, prange->last, best_loc);
	502	return 0;
	503	}
	504
	505	adev = svm_range_get_adev_by_id(prange, best_loc);
	506	if (!adev) {
	507	pr_debug("failed to get device by id 0x%x\n", best_loc);
	508	return -ENODEV;
	509	}
	510
	511	pr_debug("svms 0x%p [0x%lx 0x%lx] to gpu 0x%x\n", prange->svms,
	512	prange->start, prange->last, best_loc);
	513
	514	/* FIXME: workaround for page locking bug with invalid pages */
	515	svm_range_prefault(prange, mm);
	516
	517	start = prange->start << PAGE_SHIFT;
	518	end = (prange->last + 1) << PAGE_SHIFT;
	519
	520	for (addr = start; addr < end;) {
	521	unsigned long next;
	522
	523	vma = find_vma(mm, addr);
	524	if (!vma \|\| addr < vma->vm_start)
	525	break;
	526
	527	next = min(vma->vm_end, end);
	528	r = svm_migrate_vma_to_vram(adev, prange, vma, addr, next);
	529	if (r) {
	530	pr_debug("failed to migrate\n");
	531	break;
	532	}
	533	addr = next;
	534	}
	535
	536	if (!r)
	537	prange->actual_loc = best_loc;
	538
	539	return r;
	540	}
	541
	542	static void svm_migrate_page_free(struct page *page)
	543	{
	544	/* Keep this function to avoid warning */
	545	}
	546
	547	static int
	548	svm_migrate_copy_to_ram(struct amdgpu_device adev, struct svm_range prange,
	549	struct migrate_vma migrate, struct dma_fence *mfence,
	550	dma_addr_t *scratch)
	551	{
	552	uint64_t npages = migrate->cpages;
	553	struct device *dev = adev->dev;
	554	uint64_t *src;
	555	dma_addr_t *dst;
	556	struct page *dpage;
	557	uint64_t i = 0, j;
	558	uint64_t addr;
	559	int r = 0;
	560
	561	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
	562	prange->last);
	563
	564	addr = prange->start << PAGE_SHIFT;
	565
	566	src = (uint64_t *)(scratch + npages);
	567	dst = scratch;
	568
	569	for (i = 0, j = 0; i < npages; i++, j++, addr += PAGE_SIZE) {
	570	struct page *spage;
	571
	572	spage = migrate_pfn_to_page(migrate->src[i]);
	573	if (!spage) {
	574	pr_debug("failed get spage svms 0x%p [0x%lx 0x%lx]\n",
	575	prange->svms, prange->start, prange->last);
	576	r = -ENOMEM;
	577	goto out_oom;
	578	}
	579	src[i] = svm_migrate_addr(adev, spage);
	580	if (i > 0 && src[i] != src[i - 1] + PAGE_SIZE) {
	581	r = svm_migrate_copy_memory_gart(adev, dst + i - j,
	582	src + i - j, j,
	583	FROM_VRAM_TO_RAM,
	584	mfence);
	585	if (r)
	586	goto out_oom;
	587	j = 0;
	588	}
	589
	590	dpage = svm_migrate_get_sys_page(migrate->vma, addr);
	591	if (!dpage) {
	592	pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
	593	prange->svms, prange->start, prange->last);
	594	r = -ENOMEM;
	595	goto out_oom;
	596	}
	597
	598	dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_FROM_DEVICE);
	599	r = dma_mapping_error(dev, dst[i]);
	600	if (r) {
	601	pr_debug("failed %d dma_map_page\n", r);
	602	goto out_oom;
	603	}
	604
	605	pr_debug("dma mapping dst to 0x%llx, page_to_pfn 0x%lx\n",
	606	dst[i] >> PAGE_SHIFT, page_to_pfn(dpage));
	607
	608	migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));
	609	migrate->dst[i] \|= MIGRATE_PFN_LOCKED;
	610	}
	611
	612	r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j,
	613	FROM_VRAM_TO_RAM, mfence);
	614
	615	out_oom:
	616	if (r) {
	617	pr_debug("failed %d copy to ram\n", r);
	618	while (i--) {
	619	svm_migrate_put_sys_page(dst[i]);
	620	migrate->dst[i] = 0;
	621	}
	622	}
	623
	624	return r;
	625	}
	626
	627	static int
	628	svm_migrate_vma_to_ram(struct amdgpu_device adev, struct svm_range prange,
	629	struct vm_area_struct *vma, uint64_t start, uint64_t end)
	630	{
	631	uint64_t npages = (end - start) >> PAGE_SHIFT;
	632	struct dma_fence *mfence = NULL;
	633	struct migrate_vma migrate;
	634	dma_addr_t *scratch;
	635	size_t size;
	636	void *buf;
	637	int r = -ENOMEM;
	638
	639	memset(&migrate, 0, sizeof(migrate));
	640	migrate.vma = vma;
	641	migrate.start = start;
	642	migrate.end = end;
	643	migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
	644	migrate.pgmap_owner = adev;
	645
	646	size = 2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t);
	647	size *= npages;
	648	buf = kvmalloc(size, GFP_KERNEL \| __GFP_ZERO);
	649	if (!buf)
	650	goto out;
	651
	652	migrate.src = buf;
	653	migrate.dst = migrate.src + npages;
	654	scratch = (dma_addr_t *)(migrate.dst + npages);
	655
	656	r = migrate_vma_setup(&migrate);
	657	if (r) {
	658	pr_debug("failed %d prepare migrate svms 0x%p [0x%lx 0x%lx]\n",
	659	r, prange->svms, prange->start, prange->last);
	660	goto out_free;
	661	}
	662
	663	pr_debug("cpages %ld\n", migrate.cpages);
	664
	665	if (migrate.cpages) {
	666	r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,
	667	scratch);
	668	migrate_vma_pages(&migrate);
	669	svm_migrate_copy_done(adev, mfence);
	670	migrate_vma_finalize(&migrate);
	671	} else {
	672	pr_debug("failed collect migrate device pages [0x%lx 0x%lx]\n",
	673	prange->start, prange->last);
	674	}
	675
	676	svm_range_dma_unmap(adev->dev, scratch, 0, npages);
	677
	678	out_free:
	679	kvfree(buf);
	680	out:
	681	return r;
	682	}
	683
	684	/**
	685	* svm_migrate_vram_to_ram - migrate svm range from device to system
	686	* @prange: range structure
	687	* @mm: process mm, use current->mm if NULL
	688	*
	689	* Context: Process context, caller hold mmap read lock, svms lock, prange lock
	690	*
	691	* Return:
	692	* 0 - OK, otherwise error code
	693	*/
	694	int svm_migrate_vram_to_ram(struct svm_range prange, struct mm_struct mm)
	695	{
	696	struct amdgpu_device *adev;
	697	struct vm_area_struct *vma;
	698	unsigned long addr;
	699	unsigned long start;
	700	unsigned long end;
	701	int r = 0;
	702
	703	if (!prange->actual_loc) {
	704	pr_debug("[0x%lx 0x%lx] already migrated to ram\n",
	705	prange->start, prange->last);
	706	return 0;
	707	}
	708
	709	adev = svm_range_get_adev_by_id(prange, prange->actual_loc);
	710	if (!adev) {
	711	pr_debug("failed to get device by id 0x%x\n",
	712	prange->actual_loc);
	713	return -ENODEV;
	714	}
	715
	716	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] from gpu 0x%x to ram\n",
	717	prange->svms, prange, prange->start, prange->last,
	718	prange->actual_loc);
	719
	720	start = prange->start << PAGE_SHIFT;
	721	end = (prange->last + 1) << PAGE_SHIFT;
	722
	723	for (addr = start; addr < end;) {
	724	unsigned long next;
	725
	726	vma = find_vma(mm, addr);
	727	if (!vma \|\| addr < vma->vm_start)
	728	break;
	729
	730	next = min(vma->vm_end, end);
	731	r = svm_migrate_vma_to_ram(adev, prange, vma, addr, next);
	732	if (r) {
	733	pr_debug("failed %d to migrate\n", r);
	734	break;
	735	}
	736	addr = next;
	737	}
	738
	739	if (!r) {
	740	svm_range_vram_node_free(prange);
	741	prange->actual_loc = 0;
	742	}
	743	return r;
	744	}
	745
	746	/**
	747	* svm_migrate_vram_to_vram - migrate svm range from device to device
	748	* @prange: range structure
	749	* @best_loc: the device to migrate to
	750	* @mm: process mm, use current->mm if NULL
	751	*
	752	* Context: Process context, caller hold mmap read lock, svms lock, prange lock
	753	*
	754	* Return:
	755	* 0 - OK, otherwise error code
	756	*/
	757	static int
	758	svm_migrate_vram_to_vram(struct svm_range *prange, uint32_t best_loc,
	759	struct mm_struct *mm)
	760	{
	761	int r;
	762
	763	/*
	764	* TODO: for both devices with PCIe large bar or on same xgmi hive, skip
	765	* system memory as migration bridge
	766	*/
	767
	768	pr_debug("from gpu 0x%x to gpu 0x%x\n", prange->actual_loc, best_loc);
	769
	770	r = svm_migrate_vram_to_ram(prange, mm);
	771	if (r)
	772	return r;
	773
	774	return svm_migrate_ram_to_vram(prange, best_loc, mm);
	775	}
	776
	777	int
	778	svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,
	779	struct mm_struct *mm)
	780	{
	781	if (!prange->actual_loc)
	782	return svm_migrate_ram_to_vram(prange, best_loc, mm);
	783	else
	784	return svm_migrate_vram_to_vram(prange, best_loc, mm);
	785
	786	}
	787
	788	/**
	789	* svm_migrate_to_ram - CPU page fault handler
	790	* @vmf: CPU vm fault vma, address
	791	*
	792	* Context: vm fault handler, caller holds the mmap read lock
	793	*
	794	* Return:
	795	* 0 - OK
	796	* VM_FAULT_SIGBUS - notice application to have SIGBUS page fault
	797	*/
	798	static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
	799	{
	800	unsigned long addr = vmf->address;
	801	struct vm_area_struct *vma;
	802	enum svm_work_list_ops op;
	803	struct svm_range *parent;
	804	struct svm_range *prange;
	805	struct kfd_process *p;
	806	struct mm_struct *mm;
	807	int r = 0;
	808
	809	vma = vmf->vma;
	810	mm = vma->vm_mm;
	811
	812	p = kfd_lookup_process_by_mm(vma->vm_mm);
	813	if (!p) {
	814	pr_debug("failed find process at fault address 0x%lx\n", addr);
	815	return VM_FAULT_SIGBUS;
	816	}
	817	addr >>= PAGE_SHIFT;
	818	pr_debug("CPU page fault svms 0x%p address 0x%lx\n", &p->svms, addr);
	819
	820	mutex_lock(&p->svms.lock);
	821
	822	prange = svm_range_from_addr(&p->svms, addr, &parent);
	823	if (!prange) {
	824	pr_debug("cannot find svm range at 0x%lx\n", addr);
	825	r = -EFAULT;
	826	goto out;
	827	}
	828
	829	mutex_lock(&parent->migrate_mutex);
	830	if (prange != parent)
	831	mutex_lock_nested(&prange->migrate_mutex, 1);
	832
	833	if (!prange->actual_loc)
	834	goto out_unlock_prange;
	835
	836	svm_range_lock(parent);
	837	if (prange != parent)
	838	mutex_lock_nested(&prange->lock, 1);
	839	r = svm_range_split_by_granularity(p, mm, addr, parent, prange);
	840	if (prange != parent)
	841	mutex_unlock(&prange->lock);
	842	svm_range_unlock(parent);
	843	if (r) {
	844	pr_debug("failed %d to split range by granularity\n", r);
	845	goto out_unlock_prange;
	846	}
	847
	848	r = svm_migrate_vram_to_ram(prange, mm);
	849	if (r)
	850	pr_debug("failed %d migrate 0x%p [0x%lx 0x%lx] to ram\n", r,
	851	prange, prange->start, prange->last);
	852
	853	/* xnack on, update mapping on GPUs with ACCESS_IN_PLACE */
	854	if (p->xnack_enabled && parent == prange)
	855	op = SVM_OP_UPDATE_RANGE_NOTIFIER_AND_MAP;
	856	else
	857	op = SVM_OP_UPDATE_RANGE_NOTIFIER;
	858	svm_range_add_list_work(&p->svms, parent, mm, op);
	859	schedule_deferred_list_work(&p->svms);
	860
	861	out_unlock_prange:
	862	if (prange != parent)
	863	mutex_unlock(&prange->migrate_mutex);
	864	mutex_unlock(&parent->migrate_mutex);
	865	out:
	866	mutex_unlock(&p->svms.lock);
	867	kfd_unref_process(p);
	868
	869	pr_debug("CPU fault svms 0x%p address 0x%lx done\n", &p->svms, addr);
	870
	871	return r ? VM_FAULT_SIGBUS : 0;
	872	}
	873
	874	static const struct dev_pagemap_ops svm_migrate_pgmap_ops = {
	875	.page_free = svm_migrate_page_free,
	876	.migrate_to_ram = svm_migrate_to_ram,
	877	};
	878
	879	/* Each VRAM page uses sizeof(struct page) on system memory */
	880	#define SVM_HMM_PAGE_STRUCT_SIZE(size) ((size)/PAGE_SIZE * sizeof(struct page))
	881
	882	int svm_migrate_init(struct amdgpu_device *adev)
	883	{
	884	struct kfd_dev *kfddev = adev->kfd.dev;
	885	struct dev_pagemap *pgmap;
	886	struct resource *res;
	887	unsigned long size;
	888	void *r;
	889
	890	/* Page migration works on Vega10 or newer */
	891	if (kfddev->device_info->asic_family < CHIP_VEGA10)
	892	return -EINVAL;
	893
	894	pgmap = &kfddev->pgmap;
	895	memset(pgmap, 0, sizeof(*pgmap));
	896
	897	/* TODO: register all vram to HMM for now.
	898	* should remove reserved size
	899	*/
	900	size = ALIGN(adev->gmc.real_vram_size, 2ULL << 20);
	901	res = devm_request_free_mem_region(adev->dev, &iomem_resource, size);
	902	if (IS_ERR(res))
	903	return -ENOMEM;
	904
	905	pgmap->type = MEMORY_DEVICE_PRIVATE;
	906	pgmap->nr_range = 1;
	907	pgmap->range.start = res->start;
	908	pgmap->range.end = res->end;
	909	pgmap->ops = &svm_migrate_pgmap_ops;
	910	pgmap->owner = adev;
	911	pgmap->flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
	912	r = devm_memremap_pages(adev->dev, pgmap);
	913	if (IS_ERR(r)) {
	914	pr_err("failed to register HMM device memory\n");
	915	devm_release_mem_region(adev->dev, res->start,
	916	res->end - res->start + 1);
	917	return PTR_ERR(r);
	918	}
	919
	920	pr_debug("reserve %ldMB system memory for VRAM pages struct\n",
	921	SVM_HMM_PAGE_STRUCT_SIZE(size) >> 20);
	922
	923	amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
	924
	925	pr_info("HMM registered %ldMB device memory\n", size >> 20);
	926
	927	return 0;
	928	}
	929
	930	void svm_migrate_fini(struct amdgpu_device *adev)
	931	{
	932	struct dev_pagemap *pgmap = &adev->kfd.dev->pgmap;
	933
	934	devm_memunmap_pages(adev->dev, pgmap);
	935	devm_release_mem_region(adev->dev, pgmap->range.start,
	936	pgmap->range.end - pgmap->range.start + 1);
	937	}

drivers/gpu/drm/amd/amdkfd/kfd_migrate.h+65−0

@@ -0,0 +1,65 @@
	1	/* SPDX-License-Identifier: GPL-2.0 OR MIT */
	2	/*
	3	* Copyright 2020-2021 Advanced Micro Devices, Inc.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining a
	6	* copy of this software and associated documentation files (the "Software"),
	7	* to deal in the Software without restriction, including without limitation
	8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	9	* and/or sell copies of the Software, and to permit persons to whom the
	10	* Software is furnished to do so, subject to the following conditions:
	11	*
	12	* The above copyright notice and this permission notice shall be included in
	13	* all copies or substantial portions of the Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	21	* OTHER DEALINGS IN THE SOFTWARE.
	22	*
	23	*/
	24
	25	#ifndef KFD_MIGRATE_H_
	26	#define KFD_MIGRATE_H_
	27
	28	#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
	29
	30	#include <linux/rwsem.h>
	31	#include <linux/list.h>
	32	#include <linux/mutex.h>
	33	#include <linux/sched/mm.h>
	34	#include <linux/hmm.h>
	35	#include "kfd_priv.h"
	36	#include "kfd_svm.h"
	37
	38	enum MIGRATION_COPY_DIR {
	39	FROM_RAM_TO_VRAM = 0,
	40	FROM_VRAM_TO_RAM
	41	};
	42
	43	int svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,
	44	struct mm_struct *mm);
	45	int svm_migrate_vram_to_ram(struct svm_range prange, struct mm_struct mm);
	46	unsigned long
	47	svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr);
	48
	49	int svm_migrate_init(struct amdgpu_device *adev);
	50	void svm_migrate_fini(struct amdgpu_device *adev);
	51
	52	#else
	53
	54	static inline int svm_migrate_init(struct amdgpu_device *adev)
	55	{
	56	return 0;
	57	}
	58	static inline void svm_migrate_fini(struct amdgpu_device *adev)
	59	{
	60	/* empty */
	61	}
	62
	63	#endif /* IS_ENABLED(CONFIG_HSA_AMD_SVM) */
	64
	65	#endif /* KFD_MIGRATE_H_ */

drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c+7−5

 {
 	unsigned int alloc_size_bytes;
 	unsigned int *rl_buffer, rl_wptr, i;
-	int retval, proccesses_mapped;
+	int retval, processes_mapped;
 	struct device_process_node *cur;
 	struct qcm_process_device *qpd;
 	struct queue *q;
 	struct kernel_queue *kq;
 	bool is_over_subscription;
-	rl_wptr = retval = proccesses_mapped = 0;
+	rl_wptr = retval = processes_mapped = 0;
 	retval = pm_allocate_runlist_ib(pm, &rl_buffer, rl_gpu_addr,
 				&alloc_size_bytes, &is_over_subscription);
 	list_for_each_entry(cur, queues, list) {
 		qpd = cur->qpd;
 		/* build map process packet */
-		if (proccesses_mapped >= pm->dqm->processes_count) {
+		if (processes_mapped >= pm->dqm->processes_count) {
 			pr_debug("Not enough space left in runlist IB\n");
 			pm_release_ib(pm);
 			return -ENOMEM;
 		if (retval)
 			return retval;
-		proccesses_mapped++;
+		processes_mapped++;
 		inc_wptr(&rl_wptr, pm->pmf->map_process_size,
 				alloc_size_bytes);
 	case CHIP_RAVEN:
 	case CHIP_RENOIR:
 	case CHIP_ARCTURUS:
-	case CHIP_ALDEBARAN:
 	case CHIP_NAVI10:
 	case CHIP_NAVI12:
 	case CHIP_NAVI14:
 	case CHIP_DIMGREY_CAVEFISH:
 		pm->pmf = &kfd_v9_pm_funcs;
 		break;
+	case CHIP_ALDEBARAN:
+		pm->pmf = &kfd_aldebaran_pm_funcs;
+		break;
 	default:
 		WARN(1, "Unexpected ASIC family %u",
 		     dqm->dev->device_info->asic_family);

drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c+57−1

@@ -24,6 +24,7 @@ · 23 unmodified lines
24	#include "kfd_kernel_queue.h"	24	#include "kfd_kernel_queue.h"
25	#include "kfd_device_queue_manager.h"	25	#include "kfd_device_queue_manager.h"
26	#include "kfd_pm4_headers_ai.h"	26	#include "kfd_pm4_headers_ai.h"
		27	#include "kfd_pm4_headers_aldebaran.h"
27	#include "kfd_pm4_opcodes.h"	28	#include "kfd_pm4_opcodes.h"
28	#include "gc/gc_10_1_0_sh_mask.h"	29	#include "gc/gc_10_1_0_sh_mask.h"
29		30
@@ -35,7 +36,6 @@ static int pm_map_process_v9(struct packet_manager *pm, · 5 unmodified lines
35		36
36	packet = (struct pm4_mes_map_process *)buffer;	37	packet = (struct pm4_mes_map_process *)buffer;
37	memset(buffer, 0, sizeof(struct pm4_mes_map_process));	38	memset(buffer, 0, sizeof(struct pm4_mes_map_process));
38
39	packet->header.u32All = pm_build_pm4_header(IT_MAP_PROCESS,	39	packet->header.u32All = pm_build_pm4_header(IT_MAP_PROCESS,
40	sizeof(struct pm4_mes_map_process));	40	sizeof(struct pm4_mes_map_process));
41	packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;	41	packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
@@ -73,6 +73,45 @@ static int pm_map_process_v9(struct packet_manager *pm, · 31 unmodified lines
73	return 0;	73	return 0;
74	}	74	}
75		75
		76	static int pm_map_process_aldebaran(struct packet_manager *pm,
		77	uint32_t buffer, struct qcm_process_device qpd)
		78	{
		79	struct pm4_mes_map_process_aldebaran *packet;
		80	uint64_t vm_page_table_base_addr = qpd->page_table_base;
		81
		82	packet = (struct pm4_mes_map_process_aldebaran *)buffer;
		83	memset(buffer, 0, sizeof(struct pm4_mes_map_process_aldebaran));
		84	packet->header.u32All = pm_build_pm4_header(IT_MAP_PROCESS,
		85	sizeof(struct pm4_mes_map_process_aldebaran));
		86	packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
		87	packet->bitfields2.process_quantum = 10;
		88	packet->bitfields2.pasid = qpd->pqm->process->pasid;
		89	packet->bitfields14.gds_size = qpd->gds_size & 0x3F;
		90	packet->bitfields14.gds_size_hi = (qpd->gds_size >> 6) & 0xF;
		91	packet->bitfields14.num_gws = (qpd->mapped_gws_queue) ? qpd->num_gws : 0;
		92	packet->bitfields14.num_oac = qpd->num_oac;
		93	packet->bitfields14.sdma_enable = 1;
		94	packet->bitfields14.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count;
		95
		96	packet->sh_mem_config = qpd->sh_mem_config;
		97	packet->sh_mem_bases = qpd->sh_mem_bases;
		98	if (qpd->tba_addr) {
		99	packet->sq_shader_tba_lo = lower_32_bits(qpd->tba_addr >> 8);
		100	packet->sq_shader_tma_lo = lower_32_bits(qpd->tma_addr >> 8);
		101	packet->sq_shader_tma_hi = upper_32_bits(qpd->tma_addr >> 8);
		102	}
		103
		104	packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
		105	packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
		106
		107	packet->vm_context_page_table_base_addr_lo32 =
		108	lower_32_bits(vm_page_table_base_addr);
		109	packet->vm_context_page_table_base_addr_hi32 =
		110	upper_32_bits(vm_page_table_base_addr);
		111
		112	return 0;
		113	}
		114
76	static int pm_runlist_v9(struct packet_manager pm, uint32_t buffer,	115	static int pm_runlist_v9(struct packet_manager pm, uint32_t buffer,
77	uint64_t ib, size_t ib_size_in_dwords, bool chain)	116	uint64_t ib, size_t ib_size_in_dwords, bool chain)
78	{	117	{
@@ -324,3 +363,20 @@ const struct packet_manager_funcs kfd_v9_pm_funcs = { · 245 unmodified lines
324	.query_status_size = sizeof(struct pm4_mes_query_status),	363	.query_status_size = sizeof(struct pm4_mes_query_status),
325	.release_mem_size = 0,	364	.release_mem_size = 0,
326	};	365	};
		366
		367	const struct packet_manager_funcs kfd_aldebaran_pm_funcs = {
		368	.map_process = pm_map_process_aldebaran,
		369	.runlist = pm_runlist_v9,
		370	.set_resources = pm_set_resources_v9,
		371	.map_queues = pm_map_queues_v9,
		372	.unmap_queues = pm_unmap_queues_v9,
		373	.query_status = pm_query_status_v9,
		374	.release_mem = NULL,
		375	.map_process_size = sizeof(struct pm4_mes_map_process_aldebaran),
		376	.runlist_size = sizeof(struct pm4_mes_runlist),
		377	.set_resources_size = sizeof(struct pm4_mes_set_resources),
		378	.map_queues_size = sizeof(struct pm4_mes_map_queues),
		379	.unmap_queues_size = sizeof(struct pm4_mes_unmap_queues),
		380	.query_status_size = sizeof(struct pm4_mes_query_status),
		381	.release_mem_size = 0,
		382	};

drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_aldebaran.h+93−0

@@ -0,0 +1,93 @@
	1	/*
	2	* Copyright 2020 Advanced Micro Devices, Inc.
	3	*
	4	* Permission is hereby granted, free of charge, to any person obtaining a
	5	* copy of this software and associated documentation files (the "Software"),
	6	* to deal in the Software without restriction, including without limitation
	7	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	* and/or sell copies of the Software, and to permit persons to whom the
	9	* Software is furnished to do so, subject to the following conditions:
	10	*
	11	* The above copyright notice and this permission notice shall be included in
	12	* all copies or substantial portions of the Software.
	13	*
	14	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	* OTHER DEALINGS IN THE SOFTWARE.
	21	*
	22	*/
	23
	24	/--------------------MES_MAP_PROCESS (PER DEBUG VMID)--------------------/
	25
	26	#ifndef PM4_MES_MAP_PROCESS_PER_DEBUG_VMID_DEFINED
	27	#define PM4_MES_MAP_PROCESS_PER_DEBUG_VMID_DEFINED
	28
	29	struct pm4_mes_map_process_aldebaran {
	30	union {
	31	union PM4_MES_TYPE_3_HEADER header; /* header */
	32	uint32_t ordinal1;
	33	};
	34
	35	union {
	36	struct {
	37	uint32_t pasid:16; /* 0 - 15 */
	38	uint32_t single_memops:1; /* 16 */
	39	uint32_t reserved1:1; /* 17 */
	40	uint32_t debug_vmid:4; /* 18 - 21 */
	41	uint32_t new_debug:1; /* 22 */
	42	uint32_t tmz:1; /* 23 */
	43	uint32_t diq_enable:1; /* 24 */
	44	uint32_t process_quantum:7; /* 25 - 31 */
	45	} bitfields2;
	46	uint32_t ordinal2;
	47	};
	48
	49	uint32_t vm_context_page_table_base_addr_lo32;
	50
	51	uint32_t vm_context_page_table_base_addr_hi32;
	52
	53	uint32_t sh_mem_bases;
	54
	55	uint32_t sh_mem_config;
	56
	57	uint32_t sq_shader_tba_lo;
	58
	59	uint32_t sq_shader_tba_hi;
	60
	61	uint32_t sq_shader_tma_lo;
	62
	63	uint32_t sq_shader_tma_hi;
	64
	65	uint32_t reserved6;
	66
	67	uint32_t gds_addr_lo;
	68
	69	uint32_t gds_addr_hi;
	70
	71	union {
	72	struct {
	73	uint32_t num_gws:7;
	74	uint32_t sdma_enable:1;
	75	uint32_t num_oac:4;
	76	uint32_t gds_size_hi:4;
	77	uint32_t gds_size:6;
	78	uint32_t num_queues:10;
	79	} bitfields14;
	80	uint32_t ordinal14;
	81	};
	82
	83	uint32_t spi_gdbg_per_vmid_cntl;
	84
	85	uint32_t tcp_watch_cntl[4];
	86
	87	uint32_t completion_signal_lo;
	88
	89	uint32_t completion_signal_hi;
	90
	91	};
	92
	93	#endif

drivers/gpu/drm/amd/amdkfd/kfd_priv.h+38−1

 	unsigned int max_doorbell_slices;
 	int noretry;
+	/* HMM page migration MEMORY_DEVICE_PRIVATE mapping */
+	struct dev_pagemap pgmap;
 };
 enum kfd_mempool {
 	/* VM context for GPUVM allocations */
 	struct file *drm_file;
-	void *vm;
+	void *drm_priv;
 	/* GPUVM allocations storage */
 	struct idr alloc_idr;
 #define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
+struct svm_range_list {
+	struct mutex			lock;
+	struct rb_root_cached		objects;
+	struct list_head		list;
+	struct work_struct		deferred_list_work;
+	struct list_head		deferred_range_list;
+	spinlock_t			deferred_list_lock;
+	atomic_t			evicted_ranges;
+	struct delayed_work		restore_work;
+};
 /* Process data */
 struct kfd_process {
 	/*
 	struct kobject *kobj;
 	struct kobject *kobj_queues;
 	struct attribute attr_pasid;
+	/* shared virtual memory registered by this process */
+	struct svm_range_list svms;
+	bool svm_disabled;
+	bool xnack_enabled;
 };
 #define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
 struct kfd_process *kfd_get_process(const struct task_struct *);
 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid);
 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm);
+int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id);
+int kfd_process_gpuid_from_kgd(struct kfd_process *p,
+			       struct amdgpu_device *adev, uint32_t *gpuid,
+			       uint32_t *gpuidx);
+static inline int kfd_process_gpuid_from_gpuidx(struct kfd_process *p,
+				uint32_t gpuidx, uint32_t *gpuid) {
+	return gpuidx < p->n_pdds ? p->pdds[gpuidx]->dev->id : -EINVAL;
+}
+static inline struct kfd_process_device *kfd_process_device_from_gpuidx(
+				struct kfd_process *p, uint32_t gpuidx) {
+	return gpuidx < p->n_pdds ? p->pdds[gpuidx] : NULL;
+}
 void kfd_unref_process(struct kfd_process *p);
 int kfd_process_evict_queues(struct kfd_process *p);
 int kfd_process_restore_queues(struct kfd_process *p);
 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
 							struct kfd_process *p);
+bool kfd_process_xnack_mode(struct kfd_process *p, bool supported);
 int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
 			  struct vm_area_struct *vma);
 extern const struct packet_manager_funcs kfd_vi_pm_funcs;
 extern const struct packet_manager_funcs kfd_v9_pm_funcs;
+extern const struct packet_manager_funcs kfd_aldebaran_pm_funcs;
 int pm_init(struct packet_manager *pm, struct device_queue_manager *dqm);
 void pm_uninit(struct packet_manager *pm, bool hanging);

drivers/gpu/drm/amd/amdkfd/kfd_process.c+108−16

@@ -35,6 +35,7 @@ · 34 unmodified lines
35	#include <linux/pm_runtime.h>	35	#include <linux/pm_runtime.h>
36	#include "amdgpu_amdkfd.h"	36	#include "amdgpu_amdkfd.h"
37	#include "amdgpu.h"	37	#include "amdgpu.h"
		38	#include "kfd_svm.h"
38		39
39	struct mm_struct;	40	struct mm_struct;
40		41
@@ -42,6 +43,7 @@ struct mm_struct; · 1 unmodified line
42	#include "kfd_device_queue_manager.h"	43	#include "kfd_device_queue_manager.h"
43	#include "kfd_dbgmgr.h"	44	#include "kfd_dbgmgr.h"
44	#include "kfd_iommu.h"	45	#include "kfd_iommu.h"
		46	#include "kfd_svm.h"
45		47
46	/*	48	/*
47	* List of struct kfd_process (field kfd_process).	49	* List of struct kfd_process (field kfd_process).
@@ -250,7 +252,7 @@ cleanup: · 202 unmodified lines
250	}	252	}
251		253
252	/**	254	/**
253	* @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device	255	* @kfd_get_cu_occupancy - Collect number of waves in-flight on this device
254	* by current process. Translates acquired wave count into number of compute units	256	* by current process. Translates acquired wave count into number of compute units
255	* that are occupied.	257	* that are occupied.
256	*	258	*
@@ -647,8 +649,9 @@ static void kfd_process_free_gpuvm(struct kgd_mem *mem, · 390 unmodified lines
647	{	649	{
648	struct kfd_dev *dev = pdd->dev;	650	struct kfd_dev *dev = pdd->dev;
649		651
650	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->vm);	652	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->drm_priv);
651	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, NULL);	653	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, pdd->drm_priv,
		654	NULL);
652	}	655	}
653		656
654	/* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process	657	/* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
@@ -667,11 +670,11 @@ static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd, · 12 unmodified lines
667	int err;	670	int err;
668		671
669	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size,	672	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size,
670	pdd->vm, &mem, NULL, flags);	673	pdd->drm_priv, &mem, NULL, flags);
671	if (err)	674	if (err)
672	goto err_alloc_mem;	675	goto err_alloc_mem;
673		676
674	err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, pdd->vm);	677	err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, pdd->drm_priv);
675	if (err)	678	if (err)
676	goto err_map_mem;	679	goto err_map_mem;
677		680
@@ -712,7 +715,8 @@ sync_memory_failed: · 34 unmodified lines
712	return err;	715	return err;
713		716
714	err_map_mem:	717	err_map_mem:
715	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, NULL);	718	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, pdd->drm_priv,
		719	NULL);
716	err_alloc_mem:	720	err_alloc_mem:
717	*kptr = NULL;	721	*kptr = NULL;
718	return err;	722	return err;
@@ -901,13 +905,14 @@ static void kfd_process_device_free_bos(struct kfd_process_device *pdd) · 182 unmodified lines
901	for (i = 0; i < p->n_pdds; i++) {	905	for (i = 0; i < p->n_pdds; i++) {
902	struct kfd_process_device *peer_pdd = p->pdds[i];	906	struct kfd_process_device *peer_pdd = p->pdds[i];
903		907
904	if (!peer_pdd->vm)	908	if (!peer_pdd->drm_priv)
905	continue;	909	continue;
906	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(	910	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
907	peer_pdd->dev->kgd, mem, peer_pdd->vm);	911	peer_pdd->dev->kgd, mem, peer_pdd->drm_priv);
908	}	912	}
909		913
910	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem, NULL);	914	amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem,
		915	pdd->drm_priv, NULL);
911	kfd_process_device_remove_obj_handle(pdd, id);	916	kfd_process_device_remove_obj_handle(pdd, id);
912	}	917	}
913	}	918	}
@@ -932,7 +937,7 @@ static void kfd_process_destroy_pdds(struct kfd_process *p) · 18 unmodified lines
932		937
933	if (pdd->drm_file) {	938	if (pdd->drm_file) {
934	amdgpu_amdkfd_gpuvm_release_process_vm(	939	amdgpu_amdkfd_gpuvm_release_process_vm(
935	pdd->dev->kgd, pdd->vm);	940	pdd->dev->kgd, pdd->drm_priv);
936	fput(pdd->drm_file);	941	fput(pdd->drm_file);
937	}	942	}
938		943
@@ -1000,6 +1005,7 @@ static void kfd_process_wq_release(struct work_struct *work) · 61 unmodified lines
1000	kfd_iommu_unbind_process(p);	1005	kfd_iommu_unbind_process(p);
1001		1006
1002	kfd_process_free_outstanding_kfd_bos(p);	1007	kfd_process_free_outstanding_kfd_bos(p);
		1008	svm_range_list_fini(p);
1003		1009
1004	kfd_process_destroy_pdds(p);	1010	kfd_process_destroy_pdds(p);
1005	dma_fence_put(p->ef);	1011	dma_fence_put(p->ef);
@@ -1058,6 +1064,7 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn, · 52 unmodified lines
1058		1064
1059	cancel_delayed_work_sync(&p->eviction_work);	1065	cancel_delayed_work_sync(&p->eviction_work);
1060	cancel_delayed_work_sync(&p->restore_work);	1066	cancel_delayed_work_sync(&p->restore_work);
		1067	cancel_delayed_work_sync(&p->svms.restore_work);
1061		1068
1062	mutex_lock(&p->mutex);	1069	mutex_lock(&p->mutex);
1063		1070
@@ -1186,6 +1193,56 @@ void kfd_process_set_trap_handler(struct qcm_process_device *qpd, · 122 unmodified lines
1186	}	1193	}
1187	}	1194	}
1188		1195
		1196	bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
		1197	{
		1198	int i;
		1199
		1200	/* On most GFXv9 GPUs, the retry mode in the SQ must match the
		1201	* boot time retry setting. Mixing processes with different
		1202	* XNACK/retry settings can hang the GPU.
		1203	*
		1204	* Different GPUs can have different noretry settings depending
		1205	* on HW bugs or limitations. We need to find at least one
		1206	* XNACK mode for this process that's compatible with all GPUs.
		1207	* Fortunately GPUs with retry enabled (noretry=0) can run code
		1208	* built for XNACK-off. On GFXv9 it may perform slower.
		1209	*
		1210	* Therefore applications built for XNACK-off can always be
		1211	* supported and will be our fallback if any GPU does not
		1212	* support retry.
		1213	*/
		1214	for (i = 0; i < p->n_pdds; i++) {
		1215	struct kfd_dev *dev = p->pdds[i]->dev;
		1216
		1217	/* Only consider GFXv9 and higher GPUs. Older GPUs don't
		1218	* support the SVM APIs and don't need to be considered
		1219	* for the XNACK mode selection.
		1220	*/
		1221	if (dev->device_info->asic_family < CHIP_VEGA10)
		1222	continue;
		1223	/* Aldebaran can always support XNACK because it can support
		1224	* per-process XNACK mode selection. But let the dev->noretry
		1225	* setting still influence the default XNACK mode.
		1226	*/
		1227	if (supported &&
		1228	dev->device_info->asic_family == CHIP_ALDEBARAN)
		1229	continue;
		1230
		1231	/* GFXv10 and later GPUs do not support shader preemption
		1232	* during page faults. This can lead to poor QoS for queue
		1233	* management and memory-manager-related preemptions or
		1234	* even deadlocks.
		1235	*/
		1236	if (dev->device_info->asic_family >= CHIP_NAVI10)
		1237	return false;
		1238
		1239	if (dev->noretry)
		1240	return false;
		1241	}
		1242
		1243	return true;
		1244	}
		1245
1189	/*	1246	/*
1190	* On return the kfd_process is fully operational and will be freed when the	1247	* On return the kfd_process is fully operational and will be freed when the
1191	* mm is released	1248	* mm is released
@@ -1205,6 +1262,7 @@ static struct kfd_process create_process(const struct task_struct thread) · 13 unmodified lines
1205	process->mm = thread->mm;	1262	process->mm = thread->mm;
1206	process->lead_thread = thread->group_leader;	1263	process->lead_thread = thread->group_leader;
1207	process->n_pdds = 0;	1264	process->n_pdds = 0;
		1265	process->svm_disabled = false;
1208	INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);	1266	INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1209	INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);	1267	INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1210	process->last_restore_timestamp = get_jiffies_64();	1268	process->last_restore_timestamp = get_jiffies_64();
@@ -1224,6 +1282,13 @@ static struct kfd_process create_process(const struct task_struct thread) · 13 unmodified lines
1224	if (err != 0)	1282	if (err != 0)
1225	goto err_init_apertures;	1283	goto err_init_apertures;
1226		1284
		1285	/* Check XNACK support after PDDs are created in kfd_init_apertures */
		1286	process->xnack_enabled = kfd_process_xnack_mode(process, false);
		1287
		1288	err = svm_range_list_init(process);
		1289	if (err)
		1290	goto err_init_svm_range_list;
		1291
1227	/* alloc_notifier needs to find the process in the hash table */	1292	/* alloc_notifier needs to find the process in the hash table */
1228	hash_add_rcu(kfd_processes_table, &process->kfd_processes,	1293	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1229	(uintptr_t)process->mm);	1294	(uintptr_t)process->mm);
@@ -1246,6 +1311,8 @@ static struct kfd_process create_process(const struct task_struct thread) · 16 unmodified lines
1246		1311
1247	err_register_notifier:	1312	err_register_notifier:
1248	hash_del_rcu(&process->kfd_processes);	1313	hash_del_rcu(&process->kfd_processes);
		1314	svm_range_list_fini(process);
		1315	err_init_svm_range_list:
1249	kfd_process_free_outstanding_kfd_bos(process);	1316	kfd_process_free_outstanding_kfd_bos(process);
1250	kfd_process_destroy_pdds(process);	1317	kfd_process_destroy_pdds(process);
1251	err_init_apertures:	1318	err_init_apertures:
@@ -1375,7 +1442,7 @@ int kfd_process_device_init_vm(struct kfd_process_device *pdd, · 123 unmodified lines
1375	if (!drm_file)	1442	if (!drm_file)
1376	return -EINVAL;	1443	return -EINVAL;
1377		1444
1378	if (pdd->vm)	1445	if (pdd->drm_priv)
1379	return -EBUSY;	1446	return -EBUSY;
1380		1447
1381	p = pdd->process;	1448	p = pdd->process;
@@ -1383,13 +1450,12 @@ int kfd_process_device_init_vm(struct kfd_process_device *pdd, · 1 unmodified line
1383		1450
1384	ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(	1451	ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(
1385	dev->kgd, drm_file, p->pasid,	1452	dev->kgd, drm_file, p->pasid,
1386	&pdd->vm, &p->kgd_process_info, &p->ef);	1453	&p->kgd_process_info, &p->ef);
1387	if (ret) {	1454	if (ret) {
1388	pr_err("Failed to create process VM object\n");	1455	pr_err("Failed to create process VM object\n");
1389	return ret;	1456	return ret;
1390	}	1457	}
1391		1458	pdd->drm_priv = drm_file->private_data;
1392	amdgpu_vm_set_task_info(pdd->vm);
1393		1459
1394	ret = kfd_process_device_reserve_ib_mem(pdd);	1460	ret = kfd_process_device_reserve_ib_mem(pdd);
1395	if (ret)	1461	if (ret)
@@ -1405,7 +1471,7 @@ int kfd_process_device_init_vm(struct kfd_process_device *pdd, · 9 unmodified lines
1405	err_init_cwsr:	1471	err_init_cwsr:
1406	err_reserve_ib_mem:	1472	err_reserve_ib_mem:
1407	kfd_process_device_free_bos(pdd);	1473	kfd_process_device_free_bos(pdd);
1408	pdd->vm = NULL;	1474	pdd->drm_priv = NULL;
1409		1475
1410	return ret;	1476	return ret;
1411	}	1477	}
@@ -1429,7 +1495,7 @@ struct kfd_process_device kfd_bind_process_to_device(struct kfd_dev dev, · 17 unmodified lines
1429	return ERR_PTR(-ENOMEM);	1495	return ERR_PTR(-ENOMEM);
1430	}	1496	}
1431		1497
1432	if (!pdd->vm)	1498	if (!pdd->drm_priv)
1433	return ERR_PTR(-ENODEV);	1499	return ERR_PTR(-ENODEV);
1434		1500
1435	/*	1501	/*
@@ -1600,6 +1666,32 @@ int kfd_process_restore_queues(struct kfd_process *p) · 164 unmodified lines
1600	return ret;	1666	return ret;
1601	}	1667	}
1602		1668
		1669	int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
		1670	{
		1671	int i;
		1672
		1673	for (i = 0; i < p->n_pdds; i++)
		1674	if (p->pdds[i] && gpu_id == p->pdds[i]->dev->id)
		1675	return i;
		1676	return -EINVAL;
		1677	}
		1678
		1679	int
		1680	kfd_process_gpuid_from_kgd(struct kfd_process p, struct amdgpu_device adev,
		1681	uint32_t gpuid, uint32_t gpuidx)
		1682	{
		1683	struct kgd_dev kgd = (struct kgd_dev )adev;
		1684	int i;
		1685
		1686	for (i = 0; i < p->n_pdds; i++)
		1687	if (p->pdds[i] && p->pdds[i]->dev->kgd == kgd) {
		1688	*gpuid = p->pdds[i]->dev->id;
		1689	*gpuidx = i;
		1690	return 0;
		1691	}
		1692	return -EINVAL;
		1693	}
		1694
1603	static void evict_process_worker(struct work_struct *work)	1695	static void evict_process_worker(struct work_struct *work)
1604	{	1696	{
1605	int ret;	1697	int ret;

drivers/gpu/drm/amd/amdkfd/kfd_svm.c+3085−0

@@ -0,0 +1,3085 @@
	1	// SPDX-License-Identifier: GPL-2.0 OR MIT
	2	/*
	3	* Copyright 2020-2021 Advanced Micro Devices, Inc.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining a
	6	* copy of this software and associated documentation files (the "Software"),
	7	* to deal in the Software without restriction, including without limitation
	8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	9	* and/or sell copies of the Software, and to permit persons to whom the
	10	* Software is furnished to do so, subject to the following conditions:
	11	*
	12	* The above copyright notice and this permission notice shall be included in
	13	* all copies or substantial portions of the Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	21	* OTHER DEALINGS IN THE SOFTWARE.
	22	*/
	23
	24	#include <linux/types.h>
	25	#include <linux/sched/task.h>
	26	#include "amdgpu_sync.h"
	27	#include "amdgpu_object.h"
	28	#include "amdgpu_vm.h"
	29	#include "amdgpu_mn.h"
	30	#include "amdgpu.h"
	31	#include "amdgpu_xgmi.h"
	32	#include "kfd_priv.h"
	33	#include "kfd_svm.h"
	34	#include "kfd_migrate.h"
	35
	36	#define AMDGPU_SVM_RANGE_RESTORE_DELAY_MS 1
	37
	38	/* Long enough to ensure no retry fault comes after svm range is restored and
	39	* page table is updated.
	40	*/
	41	#define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING 2000
	42
	43	static void svm_range_evict_svm_bo_worker(struct work_struct *work);
	44	static bool
	45	svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,
	46	const struct mmu_notifier_range *range,
	47	unsigned long cur_seq);
	48
	49	static const struct mmu_interval_notifier_ops svm_range_mn_ops = {
	50	.invalidate = svm_range_cpu_invalidate_pagetables,
	51	};
	52
	53	/**
	54	* svm_range_unlink - unlink svm_range from lists and interval tree
	55	* @prange: svm range structure to be removed
	56	*
	57	* Remove the svm_range from the svms and svm_bo lists and the svms
	58	* interval tree.
	59	*
	60	* Context: The caller must hold svms->lock
	61	*/
	62	static void svm_range_unlink(struct svm_range *prange)
	63	{
	64	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,
	65	prange, prange->start, prange->last);
	66
	67	if (prange->svm_bo) {
	68	spin_lock(&prange->svm_bo->list_lock);
	69	list_del(&prange->svm_bo_list);
	70	spin_unlock(&prange->svm_bo->list_lock);
	71	}
	72
	73	list_del(&prange->list);
	74	if (prange->it_node.start != 0 && prange->it_node.last != 0)
	75	interval_tree_remove(&prange->it_node, &prange->svms->objects);
	76	}
	77
	78	static void
	79	svm_range_add_notifier_locked(struct mm_struct mm, struct svm_range prange)
	80	{
	81	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,
	82	prange, prange->start, prange->last);
	83
	84	mmu_interval_notifier_insert_locked(&prange->notifier, mm,
	85	prange->start << PAGE_SHIFT,
	86	prange->npages << PAGE_SHIFT,
	87	&svm_range_mn_ops);
	88	}
	89
	90	/**
	91	* svm_range_add_to_svms - add svm range to svms
	92	* @prange: svm range structure to be added
	93	*
	94	* Add the svm range to svms interval tree and link list
	95	*
	96	* Context: The caller must hold svms->lock
	97	*/
	98	static void svm_range_add_to_svms(struct svm_range *prange)
	99	{
	100	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,
	101	prange, prange->start, prange->last);
	102
	103	list_add_tail(&prange->list, &prange->svms->list);
	104	prange->it_node.start = prange->start;
	105	prange->it_node.last = prange->last;
	106	interval_tree_insert(&prange->it_node, &prange->svms->objects);
	107	}
	108
	109	static void svm_range_remove_notifier(struct svm_range *prange)
	110	{
	111	pr_debug("remove notifier svms 0x%p prange 0x%p [0x%lx 0x%lx]\n",
	112	prange->svms, prange,
	113	prange->notifier.interval_tree.start >> PAGE_SHIFT,
	114	prange->notifier.interval_tree.last >> PAGE_SHIFT);
	115
	116	if (prange->notifier.interval_tree.start != 0 &&
	117	prange->notifier.interval_tree.last != 0)
	118	mmu_interval_notifier_remove(&prange->notifier);
	119	}
	120
	121	static int
	122	svm_range_dma_map_dev(struct device dev, dma_addr_t *dma_addr,
	123	unsigned long *hmm_pfns, uint64_t npages)
	124	{
	125	enum dma_data_direction dir = DMA_BIDIRECTIONAL;
	126	dma_addr_t addr = dma_addr;
	127	struct page *page;
	128	int i, r;
	129
	130	if (!addr) {
	131	addr = kvmalloc_array(npages, sizeof(*addr),
	132	GFP_KERNEL \| __GFP_ZERO);
	133	if (!addr)
	134	return -ENOMEM;
	135	*dma_addr = addr;
	136	}
	137
	138	for (i = 0; i < npages; i++) {
	139	if (WARN_ONCE(addr[i] && !dma_mapping_error(dev, addr[i]),
	140	"leaking dma mapping\n"))
	141	dma_unmap_page(dev, addr[i], PAGE_SIZE, dir);
	142
	143	page = hmm_pfn_to_page(hmm_pfns[i]);
	144	addr[i] = dma_map_page(dev, page, 0, PAGE_SIZE, dir);
	145	r = dma_mapping_error(dev, addr[i]);
	146	if (r) {
	147	pr_debug("failed %d dma_map_page\n", r);
	148	return r;
	149	}
	150	pr_debug("dma mapping 0x%llx for page addr 0x%lx\n",
	151	addr[i] >> PAGE_SHIFT, page_to_pfn(page));
	152	}
	153	return 0;
	154	}
	155
	156	static int
	157	svm_range_dma_map(struct svm_range prange, unsigned long bitmap,
	158	unsigned long *hmm_pfns)
	159	{
	160	struct kfd_process *p;
	161	uint32_t gpuidx;
	162	int r;
	163
	164	p = container_of(prange->svms, struct kfd_process, svms);
	165
	166	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
	167	struct kfd_process_device *pdd;
	168	struct amdgpu_device *adev;
	169
	170	pr_debug("mapping to gpu idx 0x%x\n", gpuidx);
	171	pdd = kfd_process_device_from_gpuidx(p, gpuidx);
	172	if (!pdd) {
	173	pr_debug("failed to find device idx %d\n", gpuidx);
	174	return -EINVAL;
	175	}
	176	adev = (struct amdgpu_device *)pdd->dev->kgd;
	177
	178	r = svm_range_dma_map_dev(adev->dev, &prange->dma_addr[gpuidx],
	179	hmm_pfns, prange->npages);
	180	if (r)
	181	break;
	182	}
	183
	184	return r;
	185	}
	186
	187	void svm_range_dma_unmap(struct device dev, dma_addr_t dma_addr,
	188	unsigned long offset, unsigned long npages)
	189	{
	190	enum dma_data_direction dir = DMA_BIDIRECTIONAL;
	191	int i;
	192
	193	if (!dma_addr)
	194	return;
	195
	196	for (i = offset; i < offset + npages; i++) {
	197	if (!dma_addr[i] \|\| dma_mapping_error(dev, dma_addr[i]))
	198	continue;
	199	pr_debug("dma unmapping 0x%llx\n", dma_addr[i] >> PAGE_SHIFT);
	200	dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
	201	dma_addr[i] = 0;
	202	}
	203	}
	204
	205	void svm_range_free_dma_mappings(struct svm_range *prange)
	206	{
	207	struct kfd_process_device *pdd;
	208	dma_addr_t *dma_addr;
	209	struct device *dev;
	210	struct kfd_process *p;
	211	uint32_t gpuidx;
	212
	213	p = container_of(prange->svms, struct kfd_process, svms);
	214
	215	for (gpuidx = 0; gpuidx < MAX_GPU_INSTANCE; gpuidx++) {
	216	dma_addr = prange->dma_addr[gpuidx];
	217	if (!dma_addr)
	218	continue;
	219
	220	pdd = kfd_process_device_from_gpuidx(p, gpuidx);
	221	if (!pdd) {
	222	pr_debug("failed to find device idx %d\n", gpuidx);
	223	continue;
	224	}
	225	dev = &pdd->dev->pdev->dev;
	226	svm_range_dma_unmap(dev, dma_addr, 0, prange->npages);
	227	kvfree(dma_addr);
	228	prange->dma_addr[gpuidx] = NULL;
	229	}
	230	}
	231
	232	static void svm_range_free(struct svm_range *prange)
	233	{
	234	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, prange,
	235	prange->start, prange->last);
	236
	237	svm_range_vram_node_free(prange);
	238	svm_range_free_dma_mappings(prange);
	239	mutex_destroy(&prange->lock);
	240	mutex_destroy(&prange->migrate_mutex);
	241	kfree(prange);
	242	}
	243
	244	static void
	245	svm_range_set_default_attributes(int32_t location, int32_t prefetch_loc,
	246	uint8_t granularity, uint32_t flags)
	247	{
	248	*location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
	249	*prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
	250	*granularity = 9;
	251	*flags =
	252	KFD_IOCTL_SVM_FLAG_HOST_ACCESS \| KFD_IOCTL_SVM_FLAG_COHERENT;
	253	}
	254
	255	static struct
	256	svm_range svm_range_new(struct svm_range_list svms, uint64_t start,
	257	uint64_t last)
	258	{
	259	uint64_t size = last - start + 1;
	260	struct svm_range *prange;
	261	struct kfd_process *p;
	262
	263	prange = kzalloc(sizeof(*prange), GFP_KERNEL);
	264	if (!prange)
	265	return NULL;
	266	prange->npages = size;
	267	prange->svms = svms;
	268	prange->start = start;
	269	prange->last = last;
	270	INIT_LIST_HEAD(&prange->list);
	271	INIT_LIST_HEAD(&prange->update_list);
	272	INIT_LIST_HEAD(&prange->remove_list);
	273	INIT_LIST_HEAD(&prange->insert_list);
	274	INIT_LIST_HEAD(&prange->svm_bo_list);
	275	INIT_LIST_HEAD(&prange->deferred_list);
	276	INIT_LIST_HEAD(&prange->child_list);
	277	atomic_set(&prange->invalid, 0);
	278	prange->validate_timestamp = 0;
	279	mutex_init(&prange->migrate_mutex);
	280	mutex_init(&prange->lock);
	281
	282	p = container_of(svms, struct kfd_process, svms);
	283	if (p->xnack_enabled)
	284	bitmap_fill(prange->bitmap_access, MAX_GPU_INSTANCE);
	285
	286	svm_range_set_default_attributes(&prange->preferred_loc,
	287	&prange->prefetch_loc,
	288	&prange->granularity, &prange->flags);
	289
	290	pr_debug("svms 0x%p [0x%llx 0x%llx]\n", svms, start, last);
	291
	292	return prange;
	293	}
	294
	295	static bool svm_bo_ref_unless_zero(struct svm_range_bo *svm_bo)
	296	{
	297	if (!svm_bo \|\| !kref_get_unless_zero(&svm_bo->kref))
	298	return false;
	299
	300	return true;
	301	}
	302
	303	static struct svm_range_bo svm_range_bo_ref(struct svm_range_bo svm_bo)
	304	{
	305	if (svm_bo)
	306	kref_get(&svm_bo->kref);
	307
	308	return svm_bo;
	309	}
	310
	311	static void svm_range_bo_release(struct kref *kref)
	312	{
	313	struct svm_range_bo *svm_bo;
	314
	315	svm_bo = container_of(kref, struct svm_range_bo, kref);
	316	spin_lock(&svm_bo->list_lock);
	317	while (!list_empty(&svm_bo->range_list)) {
	318	struct svm_range *prange =
	319	list_first_entry(&svm_bo->range_list,
	320	struct svm_range, svm_bo_list);
	321	/* list_del_init tells a concurrent svm_range_vram_node_new when
	322	* it's safe to reuse the svm_bo pointer and svm_bo_list head.
	323	*/
	324	list_del_init(&prange->svm_bo_list);
	325	spin_unlock(&svm_bo->list_lock);
	326
	327	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,
	328	prange->start, prange->last);
	329	mutex_lock(&prange->lock);
	330	prange->svm_bo = NULL;
	331	mutex_unlock(&prange->lock);
	332
	333	spin_lock(&svm_bo->list_lock);
	334	}
	335	spin_unlock(&svm_bo->list_lock);
	336	if (!dma_fence_is_signaled(&svm_bo->eviction_fence->base)) {
	337	/* We're not in the eviction worker.
	338	* Signal the fence and synchronize with any
	339	* pending eviction work.
	340	*/
	341	dma_fence_signal(&svm_bo->eviction_fence->base);
	342	cancel_work_sync(&svm_bo->eviction_work);
	343	}
	344	dma_fence_put(&svm_bo->eviction_fence->base);
	345	amdgpu_bo_unref(&svm_bo->bo);
	346	kfree(svm_bo);
	347	}
	348
	349	static void svm_range_bo_unref(struct svm_range_bo *svm_bo)
	350	{
	351	if (!svm_bo)
	352	return;
	353
	354	kref_put(&svm_bo->kref, svm_range_bo_release);
	355	}
	356
	357	static bool
	358	svm_range_validate_svm_bo(struct amdgpu_device adev, struct svm_range prange)
	359	{
	360	struct amdgpu_device *bo_adev;
	361
	362	mutex_lock(&prange->lock);
	363	if (!prange->svm_bo) {
	364	mutex_unlock(&prange->lock);
	365	return false;
	366	}
	367	if (prange->ttm_res) {
	368	/* We still have a reference, all is well */
	369	mutex_unlock(&prange->lock);
	370	return true;
	371	}
	372	if (svm_bo_ref_unless_zero(prange->svm_bo)) {
	373	/*
	374	* Migrate from GPU to GPU, remove range from source bo_adev
	375	* svm_bo range list, and return false to allocate svm_bo from
	376	* destination adev.
	377	*/
	378	bo_adev = amdgpu_ttm_adev(prange->svm_bo->bo->tbo.bdev);
	379	if (bo_adev != adev) {
	380	mutex_unlock(&prange->lock);
	381
	382	spin_lock(&prange->svm_bo->list_lock);
	383	list_del_init(&prange->svm_bo_list);
	384	spin_unlock(&prange->svm_bo->list_lock);
	385
	386	svm_range_bo_unref(prange->svm_bo);
	387	return false;
	388	}
	389	if (READ_ONCE(prange->svm_bo->evicting)) {
	390	struct dma_fence *f;
	391	struct svm_range_bo *svm_bo;
	392	/* The BO is getting evicted,
	393	* we need to get a new one
	394	*/
	395	mutex_unlock(&prange->lock);
	396	svm_bo = prange->svm_bo;
	397	f = dma_fence_get(&svm_bo->eviction_fence->base);
	398	svm_range_bo_unref(prange->svm_bo);
	399	/* wait for the fence to avoid long spin-loop
	400	* at list_empty_careful
	401	*/
	402	dma_fence_wait(f, false);
	403	dma_fence_put(f);
	404	} else {
	405	/* The BO was still around and we got
	406	* a new reference to it
	407	*/
	408	mutex_unlock(&prange->lock);
	409	pr_debug("reuse old bo svms 0x%p [0x%lx 0x%lx]\n",
	410	prange->svms, prange->start, prange->last);
	411
	412	prange->ttm_res = &prange->svm_bo->bo->tbo.mem;
	413	return true;
	414	}
	415
	416	} else {
	417	mutex_unlock(&prange->lock);
	418	}
	419
	420	/* We need a new svm_bo. Spin-loop to wait for concurrent
	421	* svm_range_bo_release to finish removing this range from
	422	* its range list. After this, it is safe to reuse the
	423	* svm_bo pointer and svm_bo_list head.
	424	*/
	425	while (!list_empty_careful(&prange->svm_bo_list))
	426	;
	427
	428	return false;
	429	}
	430
	431	static struct svm_range_bo *svm_range_bo_new(void)
	432	{
	433	struct svm_range_bo *svm_bo;
	434
	435	svm_bo = kzalloc(sizeof(*svm_bo), GFP_KERNEL);
	436	if (!svm_bo)
	437	return NULL;
	438
	439	kref_init(&svm_bo->kref);
	440	INIT_LIST_HEAD(&svm_bo->range_list);
	441	spin_lock_init(&svm_bo->list_lock);
	442
	443	return svm_bo;
	444	}
	445
	446	int
	447	svm_range_vram_node_new(struct amdgpu_device adev, struct svm_range prange,
	448	bool clear)
	449	{
	450	struct amdgpu_bo_param bp;
	451	struct svm_range_bo *svm_bo;
	452	struct amdgpu_bo_user *ubo;
	453	struct amdgpu_bo *bo;
	454	struct kfd_process *p;
	455	struct mm_struct *mm;
	456	int r;
	457
	458	p = container_of(prange->svms, struct kfd_process, svms);
	459	pr_debug("pasid: %x svms 0x%p [0x%lx 0x%lx]\n", p->pasid, prange->svms,
	460	prange->start, prange->last);
	461
	462	if (svm_range_validate_svm_bo(adev, prange))
	463	return 0;
	464
	465	svm_bo = svm_range_bo_new();
	466	if (!svm_bo) {
	467	pr_debug("failed to alloc svm bo\n");
	468	return -ENOMEM;
	469	}
	470	mm = get_task_mm(p->lead_thread);
	471	if (!mm) {
	472	pr_debug("failed to get mm\n");
	473	kfree(svm_bo);
	474	return -ESRCH;
	475	}
	476	svm_bo->svms = prange->svms;
	477	svm_bo->eviction_fence =
	478	amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
	479	mm,
	480	svm_bo);
	481	mmput(mm);
	482	INIT_WORK(&svm_bo->eviction_work, svm_range_evict_svm_bo_worker);
	483	svm_bo->evicting = 0;
	484	memset(&bp, 0, sizeof(bp));
	485	bp.size = prange->npages * PAGE_SIZE;
	486	bp.byte_align = PAGE_SIZE;
	487	bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
	488	bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
	489	bp.flags \|= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0;
	490	bp.flags \|= AMDGPU_AMDKFD_CREATE_SVM_BO;
	491	bp.type = ttm_bo_type_device;
	492	bp.resv = NULL;
	493
	494	r = amdgpu_bo_create_user(adev, &bp, &ubo);
	495	if (r) {
	496	pr_debug("failed %d to create bo\n", r);
	497	goto create_bo_failed;
	498	}
	499	bo = &ubo->bo;
	500	r = amdgpu_bo_reserve(bo, true);
	501	if (r) {
	502	pr_debug("failed %d to reserve bo\n", r);
	503	goto reserve_bo_failed;
	504	}
	505
	506	r = dma_resv_reserve_shared(bo->tbo.base.resv, 1);
	507	if (r) {
	508	pr_debug("failed %d to reserve bo\n", r);
	509	amdgpu_bo_unreserve(bo);
	510	goto reserve_bo_failed;
	511	}
	512	amdgpu_bo_fence(bo, &svm_bo->eviction_fence->base, true);
	513
	514	amdgpu_bo_unreserve(bo);
	515
	516	svm_bo->bo = bo;
	517	prange->svm_bo = svm_bo;
	518	prange->ttm_res = &bo->tbo.mem;
	519	prange->offset = 0;
	520
	521	spin_lock(&svm_bo->list_lock);
	522	list_add(&prange->svm_bo_list, &svm_bo->range_list);
	523	spin_unlock(&svm_bo->list_lock);
	524
	525	return 0;
	526
	527	reserve_bo_failed:
	528	amdgpu_bo_unref(&bo);
	529	create_bo_failed:
	530	dma_fence_put(&svm_bo->eviction_fence->base);
	531	kfree(svm_bo);
	532	prange->ttm_res = NULL;
	533
	534	return r;
	535	}
	536
	537	void svm_range_vram_node_free(struct svm_range *prange)
	538	{
	539	svm_range_bo_unref(prange->svm_bo);
	540	prange->ttm_res = NULL;
	541	}
	542
	543	struct amdgpu_device *
	544	svm_range_get_adev_by_id(struct svm_range *prange, uint32_t gpu_id)
	545	{
	546	struct kfd_process_device *pdd;
	547	struct kfd_process *p;
	548	int32_t gpu_idx;
	549
	550	p = container_of(prange->svms, struct kfd_process, svms);
	551
	552	gpu_idx = kfd_process_gpuidx_from_gpuid(p, gpu_id);
	553	if (gpu_idx < 0) {
	554	pr_debug("failed to get device by id 0x%x\n", gpu_id);
	555	return NULL;
	556	}
	557	pdd = kfd_process_device_from_gpuidx(p, gpu_idx);
	558	if (!pdd) {
	559	pr_debug("failed to get device by idx 0x%x\n", gpu_idx);
	560	return NULL;
	561	}
	562
	563	return (struct amdgpu_device *)pdd->dev->kgd;
	564	}
	565
	566	static int svm_range_bo_validate(void param, struct amdgpu_bo bo)
	567	{
	568	struct ttm_operation_ctx ctx = { false, false };
	569
	570	amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM);
	571
	572	return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
	573	}
	574
	575	static int
	576	svm_range_check_attr(struct kfd_process *p,
	577	uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs)
	578	{
	579	uint32_t i;
	580	int gpuidx;
	581
	582	for (i = 0; i < nattr; i++) {
	583	switch (attrs[i].type) {
	584	case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
	585	if (attrs[i].value != KFD_IOCTL_SVM_LOCATION_SYSMEM &&
	586	attrs[i].value != KFD_IOCTL_SVM_LOCATION_UNDEFINED &&
	587	kfd_process_gpuidx_from_gpuid(p,
	588	attrs[i].value) < 0) {
	589	pr_debug("no GPU 0x%x found\n", attrs[i].value);
	590	return -EINVAL;
	591	}
	592	break;
	593	case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
	594	if (attrs[i].value != KFD_IOCTL_SVM_LOCATION_SYSMEM &&
	595	kfd_process_gpuidx_from_gpuid(p,
	596	attrs[i].value) < 0) {
	597	pr_debug("no GPU 0x%x found\n", attrs[i].value);
	598	return -EINVAL;
	599	}
	600	break;
	601	case KFD_IOCTL_SVM_ATTR_ACCESS:
	602	case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:
	603	case KFD_IOCTL_SVM_ATTR_NO_ACCESS:
	604	gpuidx = kfd_process_gpuidx_from_gpuid(p,
	605	attrs[i].value);
	606	if (gpuidx < 0) {
	607	pr_debug("no GPU 0x%x found\n", attrs[i].value);
	608	return -EINVAL;
	609	}
	610	break;
	611	case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
	612	break;
	613	case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:
	614	break;
	615	case KFD_IOCTL_SVM_ATTR_GRANULARITY:
	616	break;
	617	default:
	618	pr_debug("unknown attr type 0x%x\n", attrs[i].type);
	619	return -EINVAL;
	620	}
	621	}
	622
	623	return 0;
	624	}
	625
	626	static void
	627	svm_range_apply_attrs(struct kfd_process p, struct svm_range prange,
	628	uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs)
	629	{
	630	uint32_t i;
	631	int gpuidx;
	632
	633	for (i = 0; i < nattr; i++) {
	634	switch (attrs[i].type) {
	635	case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
	636	prange->preferred_loc = attrs[i].value;
	637	break;
	638	case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
	639	prange->prefetch_loc = attrs[i].value;
	640	break;
	641	case KFD_IOCTL_SVM_ATTR_ACCESS:
	642	case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:
	643	case KFD_IOCTL_SVM_ATTR_NO_ACCESS:
	644	gpuidx = kfd_process_gpuidx_from_gpuid(p,
	645	attrs[i].value);
	646	if (attrs[i].type == KFD_IOCTL_SVM_ATTR_NO_ACCESS) {
	647	bitmap_clear(prange->bitmap_access, gpuidx, 1);
	648	bitmap_clear(prange->bitmap_aip, gpuidx, 1);
	649	} else if (attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS) {
	650	bitmap_set(prange->bitmap_access, gpuidx, 1);
	651	bitmap_clear(prange->bitmap_aip, gpuidx, 1);
	652	} else {
	653	bitmap_clear(prange->bitmap_access, gpuidx, 1);
	654	bitmap_set(prange->bitmap_aip, gpuidx, 1);
	655	}
	656	break;
	657	case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
	658	prange->flags \|= attrs[i].value;
	659	break;
	660	case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:
	661	prange->flags &= ~attrs[i].value;
	662	break;
	663	case KFD_IOCTL_SVM_ATTR_GRANULARITY:
	664	prange->granularity = attrs[i].value;
	665	break;
	666	default:
	667	WARN_ONCE(1, "svm_range_check_attrs wasn't called?");
	668	}
	669	}
	670	}
	671
	672	/**
	673	* svm_range_debug_dump - print all range information from svms
	674	* @svms: svm range list header
	675	*
	676	* debug output svm range start, end, prefetch location from svms
	677	* interval tree and link list
	678	*
	679	* Context: The caller must hold svms->lock
	680	*/
	681	static void svm_range_debug_dump(struct svm_range_list *svms)
	682	{
	683	struct interval_tree_node *node;
	684	struct svm_range *prange;
	685
	686	pr_debug("dump svms 0x%p list\n", svms);
	687	pr_debug("range\tstart\tpage\tend\t\tlocation\n");
	688
	689	list_for_each_entry(prange, &svms->list, list) {
	690	pr_debug("0x%p 0x%lx\t0x%llx\t0x%llx\t0x%x\n",
	691	prange, prange->start, prange->npages,
	692	prange->start + prange->npages - 1,
	693	prange->actual_loc);
	694	}
	695
	696	pr_debug("dump svms 0x%p interval tree\n", svms);
	697	pr_debug("range\tstart\tpage\tend\t\tlocation\n");
	698	node = interval_tree_iter_first(&svms->objects, 0, ~0ULL);
	699	while (node) {
	700	prange = container_of(node, struct svm_range, it_node);
	701	pr_debug("0x%p 0x%lx\t0x%llx\t0x%llx\t0x%x\n",
	702	prange, prange->start, prange->npages,
	703	prange->start + prange->npages - 1,
	704	prange->actual_loc);
	705	node = interval_tree_iter_next(node, 0, ~0ULL);
	706	}
	707	}
	708
	709	static bool
	710	svm_range_is_same_attrs(struct svm_range old, struct svm_range new)
	711	{
	712	return (old->prefetch_loc == new->prefetch_loc &&
	713	old->flags == new->flags &&
	714	old->granularity == new->granularity);
	715	}
	716
	717	static int
	718	svm_range_split_array(void ppnew, void ppold, size_t size,
	719	uint64_t old_start, uint64_t old_n,
	720	uint64_t new_start, uint64_t new_n)
	721	{
	722	unsigned char new, old, *pold;
	723	uint64_t d;
	724
	725	if (!ppold)
	726	return 0;
	727	pold = (unsigned char *)ppold;
	728	if (!pold)
	729	return 0;
	730
	731	new = kvmalloc_array(new_n, size, GFP_KERNEL);
	732	if (!new)
	733	return -ENOMEM;
	734
	735	d = (new_start - old_start) * size;
	736	memcpy(new, pold + d, new_n * size);
	737
	738	old = kvmalloc_array(old_n, size, GFP_KERNEL);
	739	if (!old) {
	740	kvfree(new);
	741	return -ENOMEM;
	742	}
	743
	744	d = (new_start == old_start) ? new_n * size : 0;
	745	memcpy(old, pold + d, old_n * size);
	746
	747	kvfree(pold);
	748	(void *)ppold = old;
	749	(void *)ppnew = new;
	750
	751	return 0;
	752	}
	753
	754	static int
	755	svm_range_split_pages(struct svm_range new, struct svm_range old,
	756	uint64_t start, uint64_t last)
	757	{
	758	uint64_t npages = last - start + 1;
	759	int i, r;
	760
	761	for (i = 0; i < MAX_GPU_INSTANCE; i++) {
	762	r = svm_range_split_array(&new->dma_addr[i], &old->dma_addr[i],
	763	sizeof(*old->dma_addr[i]), old->start,
	764	npages, new->start, new->npages);
	765	if (r)
	766	return r;
	767	}
	768
	769	return 0;
	770	}
	771
	772	static int
	773	svm_range_split_nodes(struct svm_range new, struct svm_range old,
	774	uint64_t start, uint64_t last)
	775	{
	776	uint64_t npages = last - start + 1;
	777
	778	pr_debug("svms 0x%p new prange 0x%p start 0x%lx [0x%llx 0x%llx]\n",
	779	new->svms, new, new->start, start, last);
	780
	781	if (new->start == old->start) {
	782	new->offset = old->offset;
	783	old->offset += new->npages;
	784	} else {
	785	new->offset = old->offset + npages;
	786	}
	787
	788	new->svm_bo = svm_range_bo_ref(old->svm_bo);
	789	new->ttm_res = old->ttm_res;
	790
	791	spin_lock(&new->svm_bo->list_lock);
	792	list_add(&new->svm_bo_list, &new->svm_bo->range_list);
	793	spin_unlock(&new->svm_bo->list_lock);
	794
	795	return 0;
	796	}
	797
	798	/**
	799	* svm_range_split_adjust - split range and adjust
	800	*
	801	* @new: new range
	802	* @old: the old range
	803	* @start: the old range adjust to start address in pages
	804	* @last: the old range adjust to last address in pages
	805	*
	806	* Copy system memory dma_addr or vram ttm_res in old range to new
	807	* range from new_start up to size new->npages, the remaining old range is from
	808	* start to last
	809	*
	810	* Return:
	811	* 0 - OK, -ENOMEM - out of memory
	812	*/
	813	static int
	814	svm_range_split_adjust(struct svm_range new, struct svm_range old,
	815	uint64_t start, uint64_t last)
	816	{
	817	int r;
	818
	819	pr_debug("svms 0x%p new 0x%lx old [0x%lx 0x%lx] => [0x%llx 0x%llx]\n",
	820	new->svms, new->start, old->start, old->last, start, last);
	821
	822	if (new->start < old->start \|\|
	823	new->last > old->last) {
	824	WARN_ONCE(1, "invalid new range start or last\n");
	825	return -EINVAL;
	826	}
	827
	828	r = svm_range_split_pages(new, old, start, last);
	829	if (r)
	830	return r;
	831
	832	if (old->actual_loc && old->ttm_res) {
	833	r = svm_range_split_nodes(new, old, start, last);
	834	if (r)
	835	return r;
	836	}
	837
	838	old->npages = last - start + 1;
	839	old->start = start;
	840	old->last = last;
	841	new->flags = old->flags;
	842	new->preferred_loc = old->preferred_loc;
	843	new->prefetch_loc = old->prefetch_loc;
	844	new->actual_loc = old->actual_loc;
	845	new->granularity = old->granularity;
	846	bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE);
	847	bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE);
	848
	849	return 0;
	850	}
	851
	852	/**
	853	* svm_range_split - split a range in 2 ranges
	854	*
	855	* @prange: the svm range to split
	856	* @start: the remaining range start address in pages
	857	* @last: the remaining range last address in pages
	858	* @new: the result new range generated
	859	*
	860	* Two cases only:
	861	* case 1: if start == prange->start
	862	* prange ==> prange[start, last]
	863	* new range [last + 1, prange->last]
	864	*
	865	* case 2: if last == prange->last
	866	* prange ==> prange[start, last]
	867	* new range [prange->start, start - 1]
	868	*
	869	* Return:
	870	* 0 - OK, -ENOMEM - out of memory, -EINVAL - invalid start, last
	871	*/
	872	static int
	873	svm_range_split(struct svm_range *prange, uint64_t start, uint64_t last,
	874	struct svm_range **new)
	875	{
	876	uint64_t old_start = prange->start;
	877	uint64_t old_last = prange->last;
	878	struct svm_range_list *svms;
	879	int r = 0;
	880
	881	pr_debug("svms 0x%p [0x%llx 0x%llx] to [0x%llx 0x%llx]\n", prange->svms,
	882	old_start, old_last, start, last);
	883
	884	if (old_start != start && old_last != last)
	885	return -EINVAL;
	886	if (start < old_start \|\| last > old_last)
	887	return -EINVAL;
	888
	889	svms = prange->svms;
	890	if (old_start == start)
	891	*new = svm_range_new(svms, last + 1, old_last);
	892	else
	893	*new = svm_range_new(svms, old_start, start - 1);
	894	if (!*new)
	895	return -ENOMEM;
	896
	897	r = svm_range_split_adjust(*new, prange, start, last);
	898	if (r) {
	899	pr_debug("failed %d split [0x%llx 0x%llx] to [0x%llx 0x%llx]\n",
	900	r, old_start, old_last, start, last);
	901	svm_range_free(*new);
	902	*new = NULL;
	903	}
	904
	905	return r;
	906	}
	907
	908	static int
	909	svm_range_split_tail(struct svm_range prange, struct svm_range new,
	910	uint64_t new_last, struct list_head *insert_list)
	911	{
	912	struct svm_range *tail;
	913	int r = svm_range_split(prange, prange->start, new_last, &tail);
	914
	915	if (!r)
	916	list_add(&tail->insert_list, insert_list);
	917	return r;
	918	}
	919
	920	static int
	921	svm_range_split_head(struct svm_range prange, struct svm_range new,
	922	uint64_t new_start, struct list_head *insert_list)
	923	{
	924	struct svm_range *head;
	925	int r = svm_range_split(prange, new_start, prange->last, &head);
	926
	927	if (!r)
	928	list_add(&head->insert_list, insert_list);
	929	return r;
	930	}
	931
	932	static void
	933	svm_range_add_child(struct svm_range prange, struct mm_struct mm,
	934	struct svm_range *pchild, enum svm_work_list_ops op)
	935	{
	936	pr_debug("add child 0x%p [0x%lx 0x%lx] to prange 0x%p child list %d\n",
	937	pchild, pchild->start, pchild->last, prange, op);
	938
	939	pchild->work_item.mm = mm;
	940	pchild->work_item.op = op;
	941	list_add_tail(&pchild->child_list, &prange->child_list);
	942	}
	943
	944	/**
	945	* svm_range_split_by_granularity - collect ranges within granularity boundary
	946	*
	947	* @p: the process with svms list
	948	* @mm: mm structure
	949	* @addr: the vm fault address in pages, to split the prange
	950	* @parent: parent range if prange is from child list
	951	* @prange: prange to split
	952	*
	953	* Trims @prange to be a single aligned block of prange->granularity if
	954	* possible. The head and tail are added to the child_list in @parent.
	955	*
	956	* Context: caller must hold mmap_read_lock and prange->lock
	957	*
	958	* Return:
	959	* 0 - OK, otherwise error code
	960	*/
	961	int
	962	svm_range_split_by_granularity(struct kfd_process p, struct mm_struct mm,
	963	unsigned long addr, struct svm_range *parent,
	964	struct svm_range *prange)
	965	{
	966	struct svm_range head, tail;
	967	unsigned long start, last, size;
	968	int r;
	969
	970	/* Align splited range start and size to granularity size, then a single
	971	* PTE will be used for whole range, this reduces the number of PTE
	972	* updated and the L1 TLB space used for translation.
	973	*/
	974	size = 1UL << prange->granularity;
	975	start = ALIGN_DOWN(addr, size);
	976	last = ALIGN(addr + 1, size) - 1;
	977
	978	pr_debug("svms 0x%p split [0x%lx 0x%lx] to [0x%lx 0x%lx] size 0x%lx\n",
	979	prange->svms, prange->start, prange->last, start, last, size);
	980
	981	if (start > prange->start) {
	982	r = svm_range_split(prange, start, prange->last, &head);
	983	if (r)
	984	return r;
	985	svm_range_add_child(parent, mm, head, SVM_OP_ADD_RANGE);
	986	}
	987
	988	if (last < prange->last) {
	989	r = svm_range_split(prange, prange->start, last, &tail);
	990	if (r)
	991	return r;
	992	svm_range_add_child(parent, mm, tail, SVM_OP_ADD_RANGE);
	993	}
	994
	995	/* xnack on, update mapping on GPUs with ACCESS_IN_PLACE */
	996	if (p->xnack_enabled && prange->work_item.op == SVM_OP_ADD_RANGE) {
	997	prange->work_item.op = SVM_OP_ADD_RANGE_AND_MAP;
	998	pr_debug("change prange 0x%p [0x%lx 0x%lx] op %d\n",
	999	prange, prange->start, prange->last,
	1000	SVM_OP_ADD_RANGE_AND_MAP);
	1001	}
	1002	return 0;
	1003	}
	1004
	1005	static uint64_t
	1006	svm_range_get_pte_flags(struct amdgpu_device adev, struct svm_range prange)
	1007	{
	1008	struct amdgpu_device *bo_adev;
	1009	uint32_t flags = prange->flags;
	1010	uint32_t mapping_flags = 0;
	1011	uint64_t pte_flags;
	1012	bool snoop = !prange->ttm_res;
	1013	bool coherent = flags & KFD_IOCTL_SVM_FLAG_COHERENT;
	1014
	1015	if (prange->svm_bo && prange->ttm_res)
	1016	bo_adev = amdgpu_ttm_adev(prange->svm_bo->bo->tbo.bdev);
	1017
	1018	switch (adev->asic_type) {
	1019	case CHIP_ARCTURUS:
	1020	if (prange->svm_bo && prange->ttm_res) {
	1021	if (bo_adev == adev) {
	1022	mapping_flags \|= coherent ?
	1023	AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW;
	1024	} else {
	1025	mapping_flags \|= AMDGPU_VM_MTYPE_UC;
	1026	if (amdgpu_xgmi_same_hive(adev, bo_adev))
	1027	snoop = true;
	1028	}
	1029	} else {
	1030	mapping_flags \|= coherent ?
	1031	AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
	1032	}
	1033	break;
	1034	case CHIP_ALDEBARAN:
	1035	if (prange->svm_bo && prange->ttm_res) {
	1036	if (bo_adev == adev) {
	1037	mapping_flags \|= coherent ?
	1038	AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW;
	1039	if (adev->gmc.xgmi.connected_to_cpu)
	1040	snoop = true;
	1041	} else {
	1042	mapping_flags \|= AMDGPU_VM_MTYPE_UC;
	1043	if (amdgpu_xgmi_same_hive(adev, bo_adev))
	1044	snoop = true;
	1045	}
	1046	} else {
	1047	mapping_flags \|= coherent ?
	1048	AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
	1049	}
	1050	break;
	1051	default:
	1052	mapping_flags \|= coherent ?
	1053	AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
	1054	}
	1055
	1056	mapping_flags \|= AMDGPU_VM_PAGE_READABLE \| AMDGPU_VM_PAGE_WRITEABLE;
	1057
	1058	if (flags & KFD_IOCTL_SVM_FLAG_GPU_RO)
	1059	mapping_flags &= ~AMDGPU_VM_PAGE_WRITEABLE;
	1060	if (flags & KFD_IOCTL_SVM_FLAG_GPU_EXEC)
	1061	mapping_flags \|= AMDGPU_VM_PAGE_EXECUTABLE;
	1062
	1063	pte_flags = AMDGPU_PTE_VALID;
	1064	pte_flags \|= prange->ttm_res ? 0 : AMDGPU_PTE_SYSTEM;
	1065	pte_flags \|= snoop ? AMDGPU_PTE_SNOOPED : 0;
	1066
	1067	pte_flags \|= amdgpu_gem_va_map_flags(adev, mapping_flags);
	1068
	1069	pr_debug("svms 0x%p [0x%lx 0x%lx] vram %d PTE 0x%llx mapping 0x%x\n",
	1070	prange->svms, prange->start, prange->last,
	1071	prange->ttm_res ? 1:0, pte_flags, mapping_flags);
	1072
	1073	return pte_flags;
	1074	}
	1075
	1076	static int
	1077	svm_range_unmap_from_gpu(struct amdgpu_device adev, struct amdgpu_vm vm,
	1078	uint64_t start, uint64_t last,
	1079	struct dma_fence **fence)
	1080	{
	1081	uint64_t init_pte_value = 0;
	1082
	1083	pr_debug("[0x%llx 0x%llx]\n", start, last);
	1084
	1085	return amdgpu_vm_bo_update_mapping(adev, adev, vm, false, true, NULL,
	1086	start, last, init_pte_value, 0,
	1087	NULL, NULL, fence);
	1088	}
	1089
	1090	static int
	1091	svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
	1092	unsigned long last)
	1093	{
	1094	DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);
	1095	struct kfd_process_device *pdd;
	1096	struct dma_fence *fence = NULL;
	1097	struct amdgpu_device *adev;
	1098	struct kfd_process *p;
	1099	uint32_t gpuidx;
	1100	int r = 0;
	1101
	1102	bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip,
	1103	MAX_GPU_INSTANCE);
	1104	p = container_of(prange->svms, struct kfd_process, svms);
	1105
	1106	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
	1107	pr_debug("unmap from gpu idx 0x%x\n", gpuidx);
	1108	pdd = kfd_process_device_from_gpuidx(p, gpuidx);
	1109	if (!pdd) {
	1110	pr_debug("failed to find device idx %d\n", gpuidx);
	1111	return -EINVAL;
	1112	}
	1113	adev = (struct amdgpu_device *)pdd->dev->kgd;
	1114
	1115	r = svm_range_unmap_from_gpu(adev, drm_priv_to_vm(pdd->drm_priv),
	1116	start, last, &fence);
	1117	if (r)
	1118	break;
	1119
	1120	if (fence) {
	1121	r = dma_fence_wait(fence, false);
	1122	dma_fence_put(fence);
	1123	fence = NULL;
	1124	if (r)
	1125	break;
	1126	}
	1127	amdgpu_amdkfd_flush_gpu_tlb_pasid((struct kgd_dev *)adev,
	1128	p->pasid);
	1129	}
	1130
	1131	return r;
	1132	}
	1133
	1134	static int
	1135	svm_range_map_to_gpu(struct amdgpu_device adev, struct amdgpu_vm vm,
	1136	struct svm_range prange, dma_addr_t dma_addr,
	1137	struct amdgpu_device bo_adev, struct dma_fence *fence)
	1138	{
	1139	struct amdgpu_bo_va bo_va;
	1140	uint64_t pte_flags;
	1141	int r = 0;
	1142
	1143	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
	1144	prange->last);
	1145
	1146	if (prange->svm_bo && prange->ttm_res) {
	1147	bo_va.is_xgmi = amdgpu_xgmi_same_hive(adev, bo_adev);
	1148	prange->mapping.bo_va = &bo_va;
	1149	}
	1150
	1151	prange->mapping.start = prange->start;
	1152	prange->mapping.last = prange->last;
	1153	prange->mapping.offset = prange->offset;
	1154	pte_flags = svm_range_get_pte_flags(adev, prange);
	1155
	1156	r = amdgpu_vm_bo_update_mapping(adev, bo_adev, vm, false, false, NULL,
	1157	prange->mapping.start,
	1158	prange->mapping.last, pte_flags,
	1159	prange->mapping.offset,
	1160	prange->ttm_res ?
	1161	prange->ttm_res->mm_node : NULL,
	1162	dma_addr, &vm->last_update);
	1163	if (r) {
	1164	pr_debug("failed %d to map to gpu 0x%lx\n", r, prange->start);
	1165	goto out;
	1166	}
	1167
	1168	r = amdgpu_vm_update_pdes(adev, vm, false);
	1169	if (r) {
	1170	pr_debug("failed %d to update directories 0x%lx\n", r,
	1171	prange->start);
	1172	goto out;
	1173	}
	1174
	1175	if (fence)
	1176	*fence = dma_fence_get(vm->last_update);
	1177
	1178	out:
	1179	prange->mapping.bo_va = NULL;
	1180	return r;
	1181	}
	1182
	1183	static int svm_range_map_to_gpus(struct svm_range *prange,
	1184	unsigned long *bitmap, bool wait)
	1185	{
	1186	struct kfd_process_device *pdd;
	1187	struct amdgpu_device *bo_adev;
	1188	struct amdgpu_device *adev;
	1189	struct kfd_process *p;
	1190	struct dma_fence *fence = NULL;
	1191	uint32_t gpuidx;
	1192	int r = 0;
	1193
	1194	if (prange->svm_bo && prange->ttm_res)
	1195	bo_adev = amdgpu_ttm_adev(prange->svm_bo->bo->tbo.bdev);
	1196	else
	1197	bo_adev = NULL;
	1198
	1199	p = container_of(prange->svms, struct kfd_process, svms);
	1200	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
	1201	pr_debug("mapping to gpu idx 0x%x\n", gpuidx);
	1202	pdd = kfd_process_device_from_gpuidx(p, gpuidx);
	1203	if (!pdd) {
	1204	pr_debug("failed to find device idx %d\n", gpuidx);
	1205	return -EINVAL;
	1206	}
	1207	adev = (struct amdgpu_device *)pdd->dev->kgd;
	1208
	1209	pdd = kfd_bind_process_to_device(pdd->dev, p);
	1210	if (IS_ERR(pdd))
	1211	return -EINVAL;
	1212
	1213	if (bo_adev && adev != bo_adev &&
	1214	!amdgpu_xgmi_same_hive(adev, bo_adev)) {
	1215	pr_debug("cannot map to device idx %d\n", gpuidx);
	1216	continue;
	1217	}
	1218
	1219	r = svm_range_map_to_gpu(adev, drm_priv_to_vm(pdd->drm_priv),
	1220	prange, prange->dma_addr[gpuidx],
	1221	bo_adev, wait ? &fence : NULL);
	1222	if (r)
	1223	break;
	1224
	1225	if (fence) {
	1226	r = dma_fence_wait(fence, false);
	1227	dma_fence_put(fence);
	1228	fence = NULL;
	1229	if (r) {
	1230	pr_debug("failed %d to dma fence wait\n", r);
	1231	break;
	1232	}
	1233	}
	1234
	1235	amdgpu_amdkfd_flush_gpu_tlb_pasid((struct kgd_dev *)adev,
	1236	p->pasid);
	1237	}
	1238
	1239	return r;
	1240	}
	1241
	1242	struct svm_validate_context {
	1243	struct kfd_process *process;
	1244	struct svm_range *prange;
	1245	bool intr;
	1246	unsigned long bitmap[MAX_GPU_INSTANCE];
	1247	struct ttm_validate_buffer tv[MAX_GPU_INSTANCE+1];
	1248	struct list_head validate_list;
	1249	struct ww_acquire_ctx ticket;
	1250	};
	1251
	1252	static int svm_range_reserve_bos(struct svm_validate_context *ctx)
	1253	{
	1254	struct kfd_process_device *pdd;
	1255	struct amdgpu_device *adev;
	1256	struct amdgpu_vm *vm;
	1257	uint32_t gpuidx;
	1258	int r;
	1259
	1260	INIT_LIST_HEAD(&ctx->validate_list);
	1261	for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) {
	1262	pdd = kfd_process_device_from_gpuidx(ctx->process, gpuidx);
	1263	if (!pdd) {
	1264	pr_debug("failed to find device idx %d\n", gpuidx);
	1265	return -EINVAL;
	1266	}
	1267	adev = (struct amdgpu_device *)pdd->dev->kgd;
	1268	vm = drm_priv_to_vm(pdd->drm_priv);
	1269
	1270	ctx->tv[gpuidx].bo = &vm->root.base.bo->tbo;
	1271	ctx->tv[gpuidx].num_shared = 4;
	1272	list_add(&ctx->tv[gpuidx].head, &ctx->validate_list);
	1273	}
	1274	if (ctx->prange->svm_bo && ctx->prange->ttm_res) {
	1275	ctx->tv[MAX_GPU_INSTANCE].bo = &ctx->prange->svm_bo->bo->tbo;
	1276	ctx->tv[MAX_GPU_INSTANCE].num_shared = 1;
	1277	list_add(&ctx->tv[MAX_GPU_INSTANCE].head, &ctx->validate_list);
	1278	}
	1279
	1280	r = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->validate_list,
	1281	ctx->intr, NULL);
	1282	if (r) {
	1283	pr_debug("failed %d to reserve bo\n", r);
	1284	return r;
	1285	}
	1286
	1287	for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) {
	1288	pdd = kfd_process_device_from_gpuidx(ctx->process, gpuidx);
	1289	if (!pdd) {
	1290	pr_debug("failed to find device idx %d\n", gpuidx);
	1291	r = -EINVAL;
	1292	goto unreserve_out;
	1293	}
	1294	adev = (struct amdgpu_device *)pdd->dev->kgd;
	1295
	1296	r = amdgpu_vm_validate_pt_bos(adev, drm_priv_to_vm(pdd->drm_priv),
	1297	svm_range_bo_validate, NULL);
	1298	if (r) {
	1299	pr_debug("failed %d validate pt bos\n", r);
	1300	goto unreserve_out;
	1301	}
	1302	}
	1303
	1304	return 0;
	1305
	1306	unreserve_out:
	1307	ttm_eu_backoff_reservation(&ctx->ticket, &ctx->validate_list);
	1308	return r;
	1309	}
	1310
	1311	static void svm_range_unreserve_bos(struct svm_validate_context *ctx)
	1312	{
	1313	ttm_eu_backoff_reservation(&ctx->ticket, &ctx->validate_list);
	1314	}
	1315
	1316	/*
	1317	* Validation+GPU mapping with concurrent invalidation (MMU notifiers)
	1318	*
	1319	* To prevent concurrent destruction or change of range attributes, the
	1320	* svm_read_lock must be held. The caller must not hold the svm_write_lock
	1321	* because that would block concurrent evictions and lead to deadlocks. To
	1322	* serialize concurrent migrations or validations of the same range, the
	1323	* prange->migrate_mutex must be held.
	1324	*
	1325	* For VRAM ranges, the SVM BO must be allocated and valid (protected by its
	1326	* eviction fence.
	1327	*
	1328	* The following sequence ensures race-free validation and GPU mapping:
	1329	*
	1330	* 1. Reserve page table (and SVM BO if range is in VRAM)
	1331	* 2. hmm_range_fault to get page addresses (if system memory)
	1332	* 3. DMA-map pages (if system memory)
	1333	* 4-a. Take notifier lock
	1334	* 4-b. Check that pages still valid (mmu_interval_read_retry)
	1335	* 4-c. Check that the range was not split or otherwise invalidated
	1336	* 4-d. Update GPU page table
	1337	* 4.e. Release notifier lock
	1338	* 5. Release page table (and SVM BO) reservation
	1339	*/
	1340	static int svm_range_validate_and_map(struct mm_struct *mm,
	1341	struct svm_range *prange,
	1342	int32_t gpuidx, bool intr, bool wait)
	1343	{
	1344	struct svm_validate_context ctx;
	1345	struct hmm_range *hmm_range;
	1346	int r = 0;
	1347
	1348	ctx.process = container_of(prange->svms, struct kfd_process, svms);
	1349	ctx.prange = prange;
	1350	ctx.intr = intr;
	1351
	1352	if (gpuidx < MAX_GPU_INSTANCE) {
	1353	bitmap_zero(ctx.bitmap, MAX_GPU_INSTANCE);
	1354	bitmap_set(ctx.bitmap, gpuidx, 1);
	1355	} else if (ctx.process->xnack_enabled) {
	1356	bitmap_copy(ctx.bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE);
	1357
	1358	/* If prefetch range to GPU, or GPU retry fault migrate range to
	1359	* GPU, which has ACCESS attribute to the range, create mapping
	1360	* on that GPU.
	1361	*/
	1362	if (prange->actual_loc) {
	1363	gpuidx = kfd_process_gpuidx_from_gpuid(ctx.process,
	1364	prange->actual_loc);
	1365	if (gpuidx < 0) {
	1366	WARN_ONCE(1, "failed get device by id 0x%x\n",
	1367	prange->actual_loc);
	1368	return -EINVAL;
	1369	}
	1370	if (test_bit(gpuidx, prange->bitmap_access))
	1371	bitmap_set(ctx.bitmap, gpuidx, 1);
	1372	}
	1373	} else {
	1374	bitmap_or(ctx.bitmap, prange->bitmap_access,
	1375	prange->bitmap_aip, MAX_GPU_INSTANCE);
	1376	}
	1377
	1378	if (bitmap_empty(ctx.bitmap, MAX_GPU_INSTANCE))
	1379	return 0;
	1380
	1381	if (prange->actual_loc && !prange->ttm_res) {
	1382	/* This should never happen. actual_loc gets set by
	1383	* svm_migrate_ram_to_vram after allocating a BO.
	1384	*/
	1385	WARN(1, "VRAM BO missing during validation\n");
	1386	return -EINVAL;
	1387	}
	1388
	1389	svm_range_reserve_bos(&ctx);
	1390
	1391	if (!prange->actual_loc) {
	1392	r = amdgpu_hmm_range_get_pages(&prange->notifier, mm, NULL,
	1393	prange->start << PAGE_SHIFT,
	1394	prange->npages, &hmm_range,
	1395	false, true);
	1396	if (r) {
	1397	pr_debug("failed %d to get svm range pages\n", r);
	1398	goto unreserve_out;
	1399	}
	1400
	1401	r = svm_range_dma_map(prange, ctx.bitmap,
	1402	hmm_range->hmm_pfns);
	1403	if (r) {
	1404	pr_debug("failed %d to dma map range\n", r);
	1405	goto unreserve_out;
	1406	}
	1407
	1408	prange->validated_once = true;
	1409	}
	1410
	1411	svm_range_lock(prange);
	1412	if (!prange->actual_loc) {
	1413	if (amdgpu_hmm_range_get_pages_done(hmm_range)) {
	1414	pr_debug("hmm update the range, need validate again\n");
	1415	r = -EAGAIN;
	1416	goto unlock_out;
	1417	}
	1418	}
	1419	if (!list_empty(&prange->child_list)) {
	1420	pr_debug("range split by unmap in parallel, validate again\n");
	1421	r = -EAGAIN;
	1422	goto unlock_out;
	1423	}
	1424
	1425	r = svm_range_map_to_gpus(prange, ctx.bitmap, wait);
	1426
	1427	unlock_out:
	1428	svm_range_unlock(prange);
	1429	unreserve_out:
	1430	svm_range_unreserve_bos(&ctx);
	1431
	1432	if (!r)
	1433	prange->validate_timestamp = ktime_to_us(ktime_get());
	1434
	1435	return r;
	1436	}
	1437
	1438	/**
	1439	* svm_range_list_lock_and_flush_work - flush pending deferred work
	1440	*
	1441	* @svms: the svm range list
	1442	* @mm: the mm structure
	1443	*
	1444	* Context: Returns with mmap write lock held, pending deferred work flushed
	1445	*
	1446	*/
	1447	static void
	1448	svm_range_list_lock_and_flush_work(struct svm_range_list *svms,
	1449	struct mm_struct *mm)
	1450	{
	1451	retry_flush_work:
	1452	flush_work(&svms->deferred_list_work);
	1453	mmap_write_lock(mm);
	1454
	1455	if (list_empty(&svms->deferred_range_list))
	1456	return;
	1457	mmap_write_unlock(mm);
	1458	pr_debug("retry flush\n");
	1459	goto retry_flush_work;
	1460	}
	1461
	1462	static void svm_range_restore_work(struct work_struct *work)
	1463	{
	1464	struct delayed_work *dwork = to_delayed_work(work);
	1465	struct amdkfd_process_info *process_info;
	1466	struct svm_range_list *svms;
	1467	struct svm_range *prange;
	1468	struct kfd_process *p;
	1469	struct mm_struct *mm;
	1470	int evicted_ranges;
	1471	int invalid;
	1472	int r;
	1473
	1474	svms = container_of(dwork, struct svm_range_list, restore_work);
	1475	evicted_ranges = atomic_read(&svms->evicted_ranges);
	1476	if (!evicted_ranges)
	1477	return;
	1478
	1479	pr_debug("restore svm ranges\n");
	1480
	1481	/* kfd_process_notifier_release destroys this worker thread. So during
	1482	* the lifetime of this thread, kfd_process and mm will be valid.
	1483	*/
	1484	p = container_of(svms, struct kfd_process, svms);
	1485	process_info = p->kgd_process_info;
	1486	mm = p->mm;
	1487	if (!mm)
	1488	return;
	1489
	1490	mutex_lock(&process_info->lock);
	1491	svm_range_list_lock_and_flush_work(svms, mm);
	1492	mutex_lock(&svms->lock);
	1493
	1494	evicted_ranges = atomic_read(&svms->evicted_ranges);
	1495
	1496	list_for_each_entry(prange, &svms->list, list) {
	1497	invalid = atomic_read(&prange->invalid);
	1498	if (!invalid)
	1499	continue;
	1500
	1501	pr_debug("restoring svms 0x%p prange 0x%p [0x%lx %lx] inv %d\n",
	1502	prange->svms, prange, prange->start, prange->last,
	1503	invalid);
	1504
	1505	/*
	1506	* If range is migrating, wait for migration is done.
	1507	*/
	1508	mutex_lock(&prange->migrate_mutex);
	1509
	1510	r = svm_range_validate_and_map(mm, prange, MAX_GPU_INSTANCE,
	1511	false, true);
	1512	if (r)
	1513	pr_debug("failed %d to map 0x%lx to gpus\n", r,
	1514	prange->start);
	1515
	1516	mutex_unlock(&prange->migrate_mutex);
	1517	if (r)
	1518	goto out_reschedule;
	1519
	1520	if (atomic_cmpxchg(&prange->invalid, invalid, 0) != invalid)
	1521	goto out_reschedule;
	1522	}
	1523
	1524	if (atomic_cmpxchg(&svms->evicted_ranges, evicted_ranges, 0) !=
	1525	evicted_ranges)
	1526	goto out_reschedule;
	1527
	1528	evicted_ranges = 0;
	1529
	1530	r = kgd2kfd_resume_mm(mm);
	1531	if (r) {
	1532	/* No recovery from this failure. Probably the CP is
	1533	* hanging. No point trying again.
	1534	*/
	1535	pr_debug("failed %d to resume KFD\n", r);
	1536	}
	1537
	1538	pr_debug("restore svm ranges successfully\n");
	1539
	1540	out_reschedule:
	1541	mutex_unlock(&svms->lock);
	1542	mmap_write_unlock(mm);
	1543	mutex_unlock(&process_info->lock);
	1544
	1545	/* If validation failed, reschedule another attempt */
	1546	if (evicted_ranges) {
	1547	pr_debug("reschedule to restore svm range\n");
	1548	schedule_delayed_work(&svms->restore_work,
	1549	msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS));
	1550	}
	1551	}
	1552
	1553	/**
	1554	* svm_range_evict - evict svm range
	1555	*
	1556	* Stop all queues of the process to ensure GPU doesn't access the memory, then
	1557	* return to let CPU evict the buffer and proceed CPU pagetable update.
	1558	*
	1559	* Don't need use lock to sync cpu pagetable invalidation with GPU execution.
	1560	* If invalidation happens while restore work is running, restore work will
	1561	* restart to ensure to get the latest CPU pages mapping to GPU, then start
	1562	* the queues.
	1563	*/
	1564	static int
	1565	svm_range_evict(struct svm_range prange, struct mm_struct mm,
	1566	unsigned long start, unsigned long last)
	1567	{
	1568	struct svm_range_list *svms = prange->svms;
	1569	struct kfd_process *p;
	1570	int r = 0;
	1571
	1572	p = container_of(svms, struct kfd_process, svms);
	1573
	1574	pr_debug("invalidate svms 0x%p prange [0x%lx 0x%lx] [0x%lx 0x%lx]\n",
	1575	svms, prange->start, prange->last, start, last);
	1576
	1577	if (!p->xnack_enabled) {
	1578	int evicted_ranges;
	1579
	1580	atomic_inc(&prange->invalid);
	1581	evicted_ranges = atomic_inc_return(&svms->evicted_ranges);
	1582	if (evicted_ranges != 1)
	1583	return r;
	1584
	1585	pr_debug("evicting svms 0x%p range [0x%lx 0x%lx]\n",
	1586	prange->svms, prange->start, prange->last);
	1587
	1588	/* First eviction, stop the queues */
	1589	r = kgd2kfd_quiesce_mm(mm);
	1590	if (r)
	1591	pr_debug("failed to quiesce KFD\n");
	1592
	1593	pr_debug("schedule to restore svm %p ranges\n", svms);
	1594	schedule_delayed_work(&svms->restore_work,
	1595	msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS));
	1596	} else {
	1597	struct svm_range *pchild;
	1598	unsigned long s, l;
	1599
	1600	pr_debug("invalidate unmap svms 0x%p [0x%lx 0x%lx] from GPUs\n",
	1601	prange->svms, start, last);
	1602	list_for_each_entry(pchild, &prange->child_list, child_list) {
	1603	mutex_lock_nested(&pchild->lock, 1);
	1604	s = max(start, pchild->start);
	1605	l = min(last, pchild->last);
	1606	if (l >= s)
	1607	svm_range_unmap_from_gpus(pchild, s, l);
	1608	mutex_unlock(&pchild->lock);
	1609	}
	1610	s = max(start, prange->start);
	1611	l = min(last, prange->last);
	1612	if (l >= s)
	1613	svm_range_unmap_from_gpus(prange, s, l);
	1614	}
	1615
	1616	return r;
	1617	}
	1618
	1619	static struct svm_range svm_range_clone(struct svm_range old)
	1620	{
	1621	struct svm_range *new;
	1622
	1623	new = svm_range_new(old->svms, old->start, old->last);
	1624	if (!new)
	1625	return NULL;
	1626
	1627	if (old->svm_bo) {
	1628	new->ttm_res = old->ttm_res;
	1629	new->offset = old->offset;
	1630	new->svm_bo = svm_range_bo_ref(old->svm_bo);
	1631	spin_lock(&new->svm_bo->list_lock);
	1632	list_add(&new->svm_bo_list, &new->svm_bo->range_list);
	1633	spin_unlock(&new->svm_bo->list_lock);
	1634	}
	1635	new->flags = old->flags;
	1636	new->preferred_loc = old->preferred_loc;
	1637	new->prefetch_loc = old->prefetch_loc;
	1638	new->actual_loc = old->actual_loc;
	1639	new->granularity = old->granularity;
	1640	bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE);
	1641	bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE);
	1642
	1643	return new;
	1644	}
	1645
	1646	/**
	1647	* svm_range_handle_overlap - split overlap ranges
	1648	* @svms: svm range list header
	1649	* @new: range added with this attributes
	1650	* @start: range added start address, in pages
	1651	* @last: range last address, in pages
	1652	* @update_list: output, the ranges attributes are updated. For set_attr, this
	1653	* will do validation and map to GPUs. For unmap, this will be
	1654	* removed and unmap from GPUs
	1655	* @insert_list: output, the ranges will be inserted into svms, attributes are
	1656	* not changes. For set_attr, this will add into svms.
	1657	* @remove_list:output, the ranges will be removed from svms
	1658	* @left: the remaining range after overlap, For set_attr, this will be added
	1659	* as new range.
	1660	*
	1661	* Total have 5 overlap cases.
	1662	*
	1663	* This function handles overlap of an address interval with existing
	1664	* struct svm_ranges for applying new attributes. This may require
	1665	* splitting existing struct svm_ranges. All changes should be applied to
	1666	* the range_list and interval tree transactionally. If any split operation
	1667	* fails, the entire update fails. Therefore the existing overlapping
	1668	* svm_ranges are cloned and the original svm_ranges left unchanged. If the
	1669	* transaction succeeds, the modified clones are added and the originals
	1670	* freed. Otherwise the clones are removed and the old svm_ranges remain.
	1671	*
	1672	* Context: The caller must hold svms->lock
	1673	*/
	1674	static int
	1675	svm_range_handle_overlap(struct svm_range_list svms, struct svm_range new,
	1676	unsigned long start, unsigned long last,
	1677	struct list_head *update_list,
	1678	struct list_head *insert_list,
	1679	struct list_head *remove_list,
	1680	unsigned long *left)
	1681	{
	1682	struct interval_tree_node *node;
	1683	struct svm_range *prange;
	1684	struct svm_range *tmp;
	1685	int r = 0;
	1686
	1687	INIT_LIST_HEAD(update_list);
	1688	INIT_LIST_HEAD(insert_list);
	1689	INIT_LIST_HEAD(remove_list);
	1690
	1691	node = interval_tree_iter_first(&svms->objects, start, last);
	1692	while (node) {
	1693	struct interval_tree_node *next;
	1694	struct svm_range *old;
	1695	unsigned long next_start;
	1696
	1697	pr_debug("found overlap node [0x%lx 0x%lx]\n", node->start,
	1698	node->last);
	1699
	1700	old = container_of(node, struct svm_range, it_node);
	1701	next = interval_tree_iter_next(node, start, last);
	1702	next_start = min(node->last, last) + 1;
	1703
	1704	if (node->start < start \|\| node->last > last) {
	1705	/* node intersects the updated range, clone+split it */
	1706	prange = svm_range_clone(old);
	1707	if (!prange) {
	1708	r = -ENOMEM;
	1709	goto out;
	1710	}
	1711
	1712	list_add(&old->remove_list, remove_list);
	1713	list_add(&prange->insert_list, insert_list);
	1714
	1715	if (node->start < start) {
	1716	pr_debug("change old range start\n");
	1717	r = svm_range_split_head(prange, new, start,
	1718	insert_list);
	1719	if (r)
	1720	goto out;
	1721	}
	1722	if (node->last > last) {
	1723	pr_debug("change old range last\n");
	1724	r = svm_range_split_tail(prange, new, last,
	1725	insert_list);
	1726	if (r)
	1727	goto out;
	1728	}
	1729	} else {
	1730	/* The node is contained within start..last,
	1731	* just update it
	1732	*/
	1733	prange = old;
	1734	}
	1735
	1736	if (!svm_range_is_same_attrs(prange, new))
	1737	list_add(&prange->update_list, update_list);
	1738
	1739	/* insert a new node if needed */
	1740	if (node->start > start) {
	1741	prange = svm_range_new(prange->svms, start,
	1742	node->start - 1);
	1743	if (!prange) {
	1744	r = -ENOMEM;
	1745	goto out;
	1746	}
	1747
	1748	list_add(&prange->insert_list, insert_list);
	1749	list_add(&prange->update_list, update_list);
	1750	}
	1751
	1752	node = next;
	1753	start = next_start;
	1754	}
	1755
	1756	if (left && start <= last)
	1757	*left = last - start + 1;
	1758
	1759	out:
	1760	if (r)
	1761	list_for_each_entry_safe(prange, tmp, insert_list, insert_list)
	1762	svm_range_free(prange);
	1763
	1764	return r;
	1765	}
	1766
	1767	static void
	1768	svm_range_update_notifier_and_interval_tree(struct mm_struct *mm,
	1769	struct svm_range *prange)
	1770	{
	1771	unsigned long start;
	1772	unsigned long last;
	1773
	1774	start = prange->notifier.interval_tree.start >> PAGE_SHIFT;
	1775	last = prange->notifier.interval_tree.last >> PAGE_SHIFT;
	1776
	1777	if (prange->start == start && prange->last == last)
	1778	return;
	1779
	1780	pr_debug("up notifier 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n",
	1781	prange->svms, prange, start, last, prange->start,
	1782	prange->last);
	1783
	1784	if (start != 0 && last != 0) {
	1785	interval_tree_remove(&prange->it_node, &prange->svms->objects);
	1786	svm_range_remove_notifier(prange);
	1787	}
	1788	prange->it_node.start = prange->start;
	1789	prange->it_node.last = prange->last;
	1790
	1791	interval_tree_insert(&prange->it_node, &prange->svms->objects);
	1792	svm_range_add_notifier_locked(mm, prange);
	1793	}
	1794
	1795	static void
	1796	svm_range_handle_list_op(struct svm_range_list svms, struct svm_range prange)
	1797	{
	1798	struct mm_struct *mm = prange->work_item.mm;
	1799
	1800	switch (prange->work_item.op) {
	1801	case SVM_OP_NULL:
	1802	pr_debug("NULL OP 0x%p prange 0x%p [0x%lx 0x%lx]\n",
	1803	svms, prange, prange->start, prange->last);
	1804	break;
	1805	case SVM_OP_UNMAP_RANGE:
	1806	pr_debug("remove 0x%p prange 0x%p [0x%lx 0x%lx]\n",
	1807	svms, prange, prange->start, prange->last);
	1808	svm_range_unlink(prange);
	1809	svm_range_remove_notifier(prange);
	1810	svm_range_free(prange);
	1811	break;
	1812	case SVM_OP_UPDATE_RANGE_NOTIFIER:
	1813	pr_debug("update notifier 0x%p prange 0x%p [0x%lx 0x%lx]\n",
	1814	svms, prange, prange->start, prange->last);
	1815	svm_range_update_notifier_and_interval_tree(mm, prange);
	1816	break;
	1817	case SVM_OP_UPDATE_RANGE_NOTIFIER_AND_MAP:
	1818	pr_debug("update and map 0x%p prange 0x%p [0x%lx 0x%lx]\n",
	1819	svms, prange, prange->start, prange->last);
	1820	svm_range_update_notifier_and_interval_tree(mm, prange);
	1821	/* TODO: implement deferred validation and mapping */
	1822	break;
	1823	case SVM_OP_ADD_RANGE:
	1824	pr_debug("add 0x%p prange 0x%p [0x%lx 0x%lx]\n", svms, prange,
	1825	prange->start, prange->last);
	1826	svm_range_add_to_svms(prange);
	1827	svm_range_add_notifier_locked(mm, prange);
	1828	break;
	1829	case SVM_OP_ADD_RANGE_AND_MAP:
	1830	pr_debug("add and map 0x%p prange 0x%p [0x%lx 0x%lx]\n", svms,
	1831	prange, prange->start, prange->last);
	1832	svm_range_add_to_svms(prange);
	1833	svm_range_add_notifier_locked(mm, prange);
	1834	/* TODO: implement deferred validation and mapping */
	1835	break;
	1836	default:
	1837	WARN_ONCE(1, "Unknown prange 0x%p work op %d\n", prange,
	1838	prange->work_item.op);
	1839	}
	1840	}
	1841
	1842	static void svm_range_drain_retry_fault(struct svm_range_list *svms)
	1843	{
	1844	struct kfd_process_device *pdd;
	1845	struct amdgpu_device *adev;
	1846	struct kfd_process *p;
	1847	uint32_t i;
	1848
	1849	p = container_of(svms, struct kfd_process, svms);
	1850
	1851	for (i = 0; i < p->n_pdds; i++) {
	1852	pdd = p->pdds[i];
	1853	if (!pdd)
	1854	continue;
	1855
	1856	pr_debug("drain retry fault gpu %d svms %p\n", i, svms);
	1857	adev = (struct amdgpu_device *)pdd->dev->kgd;
	1858
	1859	amdgpu_ih_wait_on_checkpoint_process(adev, &adev->irq.ih1);
	1860	pr_debug("drain retry fault gpu %d svms 0x%p done\n", i, svms);
	1861	}
	1862	}
	1863
	1864	static void svm_range_deferred_list_work(struct work_struct *work)
	1865	{
	1866	struct svm_range_list *svms;
	1867	struct svm_range *prange;
	1868	struct mm_struct *mm;
	1869
	1870	svms = container_of(work, struct svm_range_list, deferred_list_work);
	1871	pr_debug("enter svms 0x%p\n", svms);
	1872
	1873	spin_lock(&svms->deferred_list_lock);
	1874	while (!list_empty(&svms->deferred_range_list)) {
	1875	prange = list_first_entry(&svms->deferred_range_list,
	1876	struct svm_range, deferred_list);
	1877	spin_unlock(&svms->deferred_list_lock);
	1878	pr_debug("prange 0x%p [0x%lx 0x%lx] op %d\n", prange,
	1879	prange->start, prange->last, prange->work_item.op);
	1880
	1881	/* Make sure no stale retry fault coming after range is freed */
	1882	if (prange->work_item.op == SVM_OP_UNMAP_RANGE)
	1883	svm_range_drain_retry_fault(prange->svms);
	1884
	1885	mm = prange->work_item.mm;
	1886	mmap_write_lock(mm);
	1887	mutex_lock(&svms->lock);
	1888
	1889	/* Remove from deferred_list must be inside mmap write lock,
	1890	* otherwise, svm_range_list_lock_and_flush_work may hold mmap
	1891	* write lock, and continue because deferred_list is empty, then
	1892	* deferred_list handle is blocked by mmap write lock.
	1893	*/
	1894	spin_lock(&svms->deferred_list_lock);
	1895	list_del_init(&prange->deferred_list);
	1896	spin_unlock(&svms->deferred_list_lock);
	1897
	1898	mutex_lock(&prange->migrate_mutex);
	1899	while (!list_empty(&prange->child_list)) {
	1900	struct svm_range *pchild;
	1901
	1902	pchild = list_first_entry(&prange->child_list,
	1903	struct svm_range, child_list);
	1904	pr_debug("child prange 0x%p op %d\n", pchild,
	1905	pchild->work_item.op);
	1906	list_del_init(&pchild->child_list);
	1907	svm_range_handle_list_op(svms, pchild);
	1908	}
	1909	mutex_unlock(&prange->migrate_mutex);
	1910
	1911	svm_range_handle_list_op(svms, prange);
	1912	mutex_unlock(&svms->lock);
	1913	mmap_write_unlock(mm);
	1914
	1915	spin_lock(&svms->deferred_list_lock);
	1916	}
	1917	spin_unlock(&svms->deferred_list_lock);
	1918
	1919	pr_debug("exit svms 0x%p\n", svms);
	1920	}
	1921
	1922	void
	1923	svm_range_add_list_work(struct svm_range_list svms, struct svm_range prange,
	1924	struct mm_struct *mm, enum svm_work_list_ops op)
	1925	{
	1926	spin_lock(&svms->deferred_list_lock);
	1927	/* if prange is on the deferred list */
	1928	if (!list_empty(&prange->deferred_list)) {
	1929	pr_debug("update exist prange 0x%p work op %d\n", prange, op);
	1930	WARN_ONCE(prange->work_item.mm != mm, "unmatch mm\n");
	1931	if (op != SVM_OP_NULL &&
	1932	prange->work_item.op != SVM_OP_UNMAP_RANGE)
	1933	prange->work_item.op = op;
	1934	} else {
	1935	prange->work_item.op = op;
	1936	prange->work_item.mm = mm;
	1937	list_add_tail(&prange->deferred_list,
	1938	&prange->svms->deferred_range_list);
	1939	pr_debug("add prange 0x%p [0x%lx 0x%lx] to work list op %d\n",
	1940	prange, prange->start, prange->last, op);
	1941	}
	1942	spin_unlock(&svms->deferred_list_lock);
	1943	}
	1944
	1945	void schedule_deferred_list_work(struct svm_range_list *svms)
	1946	{
	1947	spin_lock(&svms->deferred_list_lock);
	1948	if (!list_empty(&svms->deferred_range_list))
	1949	schedule_work(&svms->deferred_list_work);
	1950	spin_unlock(&svms->deferred_list_lock);
	1951	}
	1952
	1953	static void
	1954	svm_range_unmap_split(struct mm_struct mm, struct svm_range parent,
	1955	struct svm_range *prange, unsigned long start,
	1956	unsigned long last)
	1957	{
	1958	struct svm_range *head;
	1959	struct svm_range *tail;
	1960
	1961	if (prange->work_item.op == SVM_OP_UNMAP_RANGE) {
	1962	pr_debug("prange 0x%p [0x%lx 0x%lx] is already freed\n", prange,
	1963	prange->start, prange->last);
	1964	return;
	1965	}
	1966	if (start > prange->last \|\| last < prange->start)
	1967	return;
	1968
	1969	head = tail = prange;
	1970	if (start > prange->start)
	1971	svm_range_split(prange, prange->start, start - 1, &tail);
	1972	if (last < tail->last)
	1973	svm_range_split(tail, last + 1, tail->last, &head);
	1974
	1975	if (head != prange && tail != prange) {
	1976	svm_range_add_child(parent, mm, head, SVM_OP_UNMAP_RANGE);
	1977	svm_range_add_child(parent, mm, tail, SVM_OP_ADD_RANGE);
	1978	} else if (tail != prange) {
	1979	svm_range_add_child(parent, mm, tail, SVM_OP_UNMAP_RANGE);
	1980	} else if (head != prange) {
	1981	svm_range_add_child(parent, mm, head, SVM_OP_UNMAP_RANGE);
	1982	} else if (parent != prange) {
	1983	prange->work_item.op = SVM_OP_UNMAP_RANGE;
	1984	}
	1985	}
	1986
	1987	static void
	1988	svm_range_unmap_from_cpu(struct mm_struct mm, struct svm_range prange,
	1989	unsigned long start, unsigned long last)
	1990	{
	1991	struct svm_range_list *svms;
	1992	struct svm_range *pchild;
	1993	struct kfd_process *p;
	1994	unsigned long s, l;
	1995	bool unmap_parent;
	1996
	1997	p = kfd_lookup_process_by_mm(mm);
	1998	if (!p)
	1999	return;
	2000	svms = &p->svms;
	2001
	2002	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n", svms,
	2003	prange, prange->start, prange->last, start, last);
	2004
	2005	unmap_parent = start <= prange->start && last >= prange->last;
	2006
	2007	list_for_each_entry(pchild, &prange->child_list, child_list) {
	2008	mutex_lock_nested(&pchild->lock, 1);
	2009	s = max(start, pchild->start);
	2010	l = min(last, pchild->last);
	2011	if (l >= s)
	2012	svm_range_unmap_from_gpus(pchild, s, l);
	2013	svm_range_unmap_split(mm, prange, pchild, start, last);
	2014	mutex_unlock(&pchild->lock);
	2015	}
	2016	s = max(start, prange->start);
	2017	l = min(last, prange->last);
	2018	if (l >= s)
	2019	svm_range_unmap_from_gpus(prange, s, l);
	2020	svm_range_unmap_split(mm, prange, prange, start, last);
	2021
	2022	if (unmap_parent)
	2023	svm_range_add_list_work(svms, prange, mm, SVM_OP_UNMAP_RANGE);
	2024	else
	2025	svm_range_add_list_work(svms, prange, mm,
	2026	SVM_OP_UPDATE_RANGE_NOTIFIER);
	2027	schedule_deferred_list_work(svms);
	2028
	2029	kfd_unref_process(p);
	2030	}
	2031
	2032	/**
	2033	* svm_range_cpu_invalidate_pagetables - interval notifier callback
	2034	*
	2035	* If event is MMU_NOTIFY_UNMAP, this is from CPU unmap range, otherwise, it
	2036	* is from migration, or CPU page invalidation callback.
	2037	*
	2038	* For unmap event, unmap range from GPUs, remove prange from svms in a delayed
	2039	* work thread, and split prange if only part of prange is unmapped.
	2040	*
	2041	* For invalidation event, if GPU retry fault is not enabled, evict the queues,
	2042	* then schedule svm_range_restore_work to update GPU mapping and resume queues.
	2043	* If GPU retry fault is enabled, unmap the svm range from GPU, retry fault will
	2044	* update GPU mapping to recover.
	2045	*
	2046	* Context: mmap lock, notifier_invalidate_start lock are held
	2047	* for invalidate event, prange lock is held if this is from migration
	2048	*/
	2049	static bool
	2050	svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,
	2051	const struct mmu_notifier_range *range,
	2052	unsigned long cur_seq)
	2053	{
	2054	struct svm_range *prange;
	2055	unsigned long start;
	2056	unsigned long last;
	2057
	2058	if (range->event == MMU_NOTIFY_RELEASE)
	2059	return true;
	2060
	2061	start = mni->interval_tree.start;
	2062	last = mni->interval_tree.last;
	2063	start = (start > range->start ? start : range->start) >> PAGE_SHIFT;
	2064	last = (last < (range->end - 1) ? last : range->end - 1) >> PAGE_SHIFT;
	2065	pr_debug("[0x%lx 0x%lx] range[0x%lx 0x%lx] notifier[0x%lx 0x%lx] %d\n",
	2066	start, last, range->start >> PAGE_SHIFT,
	2067	(range->end - 1) >> PAGE_SHIFT,
	2068	mni->interval_tree.start >> PAGE_SHIFT,
	2069	mni->interval_tree.last >> PAGE_SHIFT, range->event);
	2070
	2071	prange = container_of(mni, struct svm_range, notifier);
	2072
	2073	svm_range_lock(prange);
	2074	mmu_interval_set_seq(mni, cur_seq);
	2075
	2076	switch (range->event) {
	2077	case MMU_NOTIFY_UNMAP:
	2078	svm_range_unmap_from_cpu(mni->mm, prange, start, last);
	2079	break;
	2080	default:
	2081	svm_range_evict(prange, mni->mm, start, last);
	2082	break;
	2083	}
	2084
	2085	svm_range_unlock(prange);
	2086
	2087	return true;
	2088	}
	2089
	2090	/**
	2091	* svm_range_from_addr - find svm range from fault address
	2092	* @svms: svm range list header
	2093	* @addr: address to search range interval tree, in pages
	2094	* @parent: parent range if range is on child list
	2095	*
	2096	* Context: The caller must hold svms->lock
	2097	*
	2098	* Return: the svm_range found or NULL
	2099	*/
	2100	struct svm_range *
	2101	svm_range_from_addr(struct svm_range_list *svms, unsigned long addr,
	2102	struct svm_range **parent)
	2103	{
	2104	struct interval_tree_node *node;
	2105	struct svm_range *prange;
	2106	struct svm_range *pchild;
	2107
	2108	node = interval_tree_iter_first(&svms->objects, addr, addr);
	2109	if (!node)
	2110	return NULL;
	2111
	2112	prange = container_of(node, struct svm_range, it_node);
	2113	pr_debug("address 0x%lx prange [0x%lx 0x%lx] node [0x%lx 0x%lx]\n",
	2114	addr, prange->start, prange->last, node->start, node->last);
	2115
	2116	if (addr >= prange->start && addr <= prange->last) {
	2117	if (parent)
	2118	*parent = prange;
	2119	return prange;
	2120	}
	2121	list_for_each_entry(pchild, &prange->child_list, child_list)
	2122	if (addr >= pchild->start && addr <= pchild->last) {
	2123	pr_debug("found address 0x%lx pchild [0x%lx 0x%lx]\n",
	2124	addr, pchild->start, pchild->last);
	2125	if (parent)
	2126	*parent = prange;
	2127	return pchild;
	2128	}
	2129
	2130	return NULL;
	2131	}
	2132
	2133	/* svm_range_best_restore_location - decide the best fault restore location
	2134	* @prange: svm range structure
	2135	* @adev: the GPU on which vm fault happened
	2136	*
	2137	* This is only called when xnack is on, to decide the best location to restore
	2138	* the range mapping after GPU vm fault. Caller uses the best location to do
	2139	* migration if actual loc is not best location, then update GPU page table
	2140	* mapping to the best location.
	2141	*
	2142	* If vm fault gpu is range preferred loc, the best_loc is preferred loc.
	2143	* If vm fault gpu idx is on range ACCESSIBLE bitmap, best_loc is vm fault gpu
	2144	* If vm fault gpu idx is on range ACCESSIBLE_IN_PLACE bitmap, then
	2145	* if range actual loc is cpu, best_loc is cpu
	2146	* if vm fault gpu is on xgmi same hive of range actual loc gpu, best_loc is
	2147	* range actual loc.
	2148	* Otherwise, GPU no access, best_loc is -1.
	2149	*
	2150	* Return:
	2151	* -1 means vm fault GPU no access
	2152	* 0 for CPU or GPU id
	2153	*/
	2154	static int32_t
	2155	svm_range_best_restore_location(struct svm_range *prange,
	2156	struct amdgpu_device *adev,
	2157	int32_t *gpuidx)
	2158	{
	2159	struct amdgpu_device *bo_adev;
	2160	struct kfd_process *p;
	2161	uint32_t gpuid;
	2162	int r;
	2163
	2164	p = container_of(prange->svms, struct kfd_process, svms);
	2165
	2166	r = kfd_process_gpuid_from_kgd(p, adev, &gpuid, gpuidx);
	2167	if (r < 0) {
	2168	pr_debug("failed to get gpuid from kgd\n");
	2169	return -1;
	2170	}
	2171
	2172	if (prange->preferred_loc == gpuid)
	2173	return prange->preferred_loc;
	2174
	2175	if (test_bit(*gpuidx, prange->bitmap_access))
	2176	return gpuid;
	2177
	2178	if (test_bit(*gpuidx, prange->bitmap_aip)) {
	2179	if (!prange->actual_loc)
	2180	return 0;
	2181
	2182	bo_adev = svm_range_get_adev_by_id(prange, prange->actual_loc);
	2183	if (amdgpu_xgmi_same_hive(adev, bo_adev))
	2184	return prange->actual_loc;
	2185	else
	2186	return 0;
	2187	}
	2188
	2189	return -1;
	2190	}
	2191	static int
	2192	svm_range_get_range_boundaries(struct kfd_process *p, int64_t addr,
	2193	unsigned long start, unsigned long last)
	2194	{
	2195	struct vm_area_struct *vma;
	2196	struct interval_tree_node *node;
	2197	unsigned long start_limit, end_limit;
	2198
	2199	vma = find_vma(p->mm, addr << PAGE_SHIFT);
	2200	if (!vma \|\| (addr << PAGE_SHIFT) < vma->vm_start) {
	2201	pr_debug("VMA does not exist in address [0x%llx]\n", addr);
	2202	return -EFAULT;
	2203	}
	2204	start_limit = max(vma->vm_start >> PAGE_SHIFT,
	2205	(unsigned long)ALIGN_DOWN(addr, 2UL << 8));
	2206	end_limit = min(vma->vm_end >> PAGE_SHIFT,
	2207	(unsigned long)ALIGN(addr + 1, 2UL << 8));
	2208	/* First range that starts after the fault address */
	2209	node = interval_tree_iter_first(&p->svms.objects, addr + 1, ULONG_MAX);
	2210	if (node) {
	2211	end_limit = min(end_limit, node->start);
	2212	/* Last range that ends before the fault address */
	2213	node = container_of(rb_prev(&node->rb),
	2214	struct interval_tree_node, rb);
	2215	} else {
	2216	/* Last range must end before addr because
	2217	* there was no range after addr
	2218	*/
	2219	node = container_of(rb_last(&p->svms.objects.rb_root),
	2220	struct interval_tree_node, rb);
	2221	}
	2222	if (node) {
	2223	if (node->last >= addr) {
	2224	WARN(1, "Overlap with prev node and page fault addr\n");
	2225	return -EFAULT;
	2226	}
	2227	start_limit = max(start_limit, node->last + 1);
	2228	}
	2229
	2230	*start = start_limit;
	2231	*last = end_limit - 1;
	2232
	2233	pr_debug("vma start: 0x%lx start: 0x%lx vma end: 0x%lx last: 0x%lx\n",
	2234	vma->vm_start >> PAGE_SHIFT, *start,
	2235	vma->vm_end >> PAGE_SHIFT, *last);
	2236
	2237	return 0;
	2238
	2239	}
	2240	static struct
	2241	svm_range svm_range_create_unregistered_range(struct amdgpu_device adev,
	2242	struct kfd_process *p,
	2243	struct mm_struct *mm,
	2244	int64_t addr)
	2245	{
	2246	struct svm_range *prange = NULL;
	2247	unsigned long start, last;
	2248	uint32_t gpuid, gpuidx;
	2249
	2250	if (svm_range_get_range_boundaries(p, addr, &start, &last))
	2251	return NULL;
	2252
	2253	prange = svm_range_new(&p->svms, start, last);
	2254	if (!prange) {
	2255	pr_debug("Failed to create prange in address [0x%llx]\n", addr);
	2256	return NULL;
	2257	}
	2258	if (kfd_process_gpuid_from_kgd(p, adev, &gpuid, &gpuidx)) {
	2259	pr_debug("failed to get gpuid from kgd\n");
	2260	svm_range_free(prange);
	2261	return NULL;
	2262	}
	2263
	2264	svm_range_add_to_svms(prange);
	2265	svm_range_add_notifier_locked(mm, prange);
	2266
	2267	return prange;
	2268	}
	2269
	2270	/* svm_range_skip_recover - decide if prange can be recovered
	2271	* @prange: svm range structure
	2272	*
	2273	* GPU vm retry fault handle skip recover the range for cases:
	2274	* 1. prange is on deferred list to be removed after unmap, it is stale fault,
	2275	* deferred list work will drain the stale fault before free the prange.
	2276	* 2. prange is on deferred list to add interval notifier after split, or
	2277	* 3. prange is child range, it is split from parent prange, recover later
	2278	* after interval notifier is added.
	2279	*
	2280	* Return: true to skip recover, false to recover
	2281	*/
	2282	static bool svm_range_skip_recover(struct svm_range *prange)
	2283	{
	2284	struct svm_range_list *svms = prange->svms;
	2285
	2286	spin_lock(&svms->deferred_list_lock);
	2287	if (list_empty(&prange->deferred_list) &&
	2288	list_empty(&prange->child_list)) {
	2289	spin_unlock(&svms->deferred_list_lock);
	2290	return false;
	2291	}
	2292	spin_unlock(&svms->deferred_list_lock);
	2293
	2294	if (prange->work_item.op == SVM_OP_UNMAP_RANGE) {
	2295	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] unmapped\n",
	2296	svms, prange, prange->start, prange->last);
	2297	return true;
	2298	}
	2299	if (prange->work_item.op == SVM_OP_ADD_RANGE_AND_MAP \|\|
	2300	prange->work_item.op == SVM_OP_ADD_RANGE) {
	2301	pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] not added yet\n",
	2302	svms, prange, prange->start, prange->last);
	2303	return true;
	2304	}
	2305	return false;
	2306	}
	2307
	2308	int
	2309	svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
	2310	uint64_t addr)
	2311	{
	2312	struct mm_struct *mm = NULL;
	2313	struct svm_range_list *svms;
	2314	struct svm_range *prange;
	2315	struct kfd_process *p;
	2316	uint64_t timestamp;
	2317	int32_t best_loc, gpuidx;
	2318	bool write_locked = false;
	2319	int r = 0;
	2320
	2321	p = kfd_lookup_process_by_pasid(pasid);
	2322	if (!p) {
	2323	pr_debug("kfd process not founded pasid 0x%x\n", pasid);
	2324	return -ESRCH;
	2325	}
	2326	if (!p->xnack_enabled) {
	2327	pr_debug("XNACK not enabled for pasid 0x%x\n", pasid);
	2328	return -EFAULT;
	2329	}
	2330	svms = &p->svms;
	2331
	2332	pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr);
	2333
	2334	mm = get_task_mm(p->lead_thread);
	2335	if (!mm) {
	2336	pr_debug("svms 0x%p failed to get mm\n", svms);
	2337	r = -ESRCH;
	2338	goto out;
	2339	}
	2340
	2341	mmap_read_lock(mm);
	2342	retry_write_locked:
	2343	mutex_lock(&svms->lock);
	2344	prange = svm_range_from_addr(svms, addr, NULL);
	2345	if (!prange) {
	2346	pr_debug("failed to find prange svms 0x%p address [0x%llx]\n",
	2347	svms, addr);
	2348	if (!write_locked) {
	2349	/* Need the write lock to create new range with MMU notifier.
	2350	* Also flush pending deferred work to make sure the interval
	2351	* tree is up to date before we add a new range
	2352	*/
	2353	mutex_unlock(&svms->lock);
	2354	mmap_read_unlock(mm);
	2355	mmap_write_lock(mm);
	2356	write_locked = true;
	2357	goto retry_write_locked;
	2358	}
	2359	prange = svm_range_create_unregistered_range(adev, p, mm, addr);
	2360	if (!prange) {
	2361	pr_debug("failed to create unregistered range svms 0x%p address [0x%llx]\n",
	2362	svms, addr);
	2363	mmap_write_downgrade(mm);
	2364	r = -EFAULT;
	2365	goto out_unlock_svms;
	2366	}
	2367	}
	2368	if (write_locked)
	2369	mmap_write_downgrade(mm);
	2370
	2371	mutex_lock(&prange->migrate_mutex);
	2372
	2373	if (svm_range_skip_recover(prange)) {
	2374	amdgpu_gmc_filter_faults_remove(adev, addr, pasid);
	2375	goto out_unlock_range;
	2376	}
	2377
	2378	timestamp = ktime_to_us(ktime_get()) - prange->validate_timestamp;
	2379	/* skip duplicate vm fault on different pages of same range */
	2380	if (timestamp < AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING) {
	2381	pr_debug("svms 0x%p [0x%lx %lx] already restored\n",
	2382	svms, prange->start, prange->last);
	2383	goto out_unlock_range;
	2384	}
	2385
	2386	best_loc = svm_range_best_restore_location(prange, adev, &gpuidx);
	2387	if (best_loc == -1) {
	2388	pr_debug("svms %p failed get best restore loc [0x%lx 0x%lx]\n",
	2389	svms, prange->start, prange->last);
	2390	r = -EACCES;
	2391	goto out_unlock_range;
	2392	}
	2393
	2394	pr_debug("svms %p [0x%lx 0x%lx] best restore 0x%x, actual loc 0x%x\n",
	2395	svms, prange->start, prange->last, best_loc,
	2396	prange->actual_loc);
	2397
	2398	if (prange->actual_loc != best_loc) {
	2399	if (best_loc) {
	2400	r = svm_migrate_to_vram(prange, best_loc, mm);
	2401	if (r) {
	2402	pr_debug("svm_migrate_to_vram failed (%d) at %llx, falling back to system memory\n",
	2403	r, addr);
	2404	/* Fallback to system memory if migration to
	2405	* VRAM failed
	2406	*/
	2407	if (prange->actual_loc)
	2408	r = svm_migrate_vram_to_ram(prange, mm);
	2409	else
	2410	r = 0;
	2411	}
	2412	} else {
	2413	r = svm_migrate_vram_to_ram(prange, mm);
	2414	}
	2415	if (r) {
	2416	pr_debug("failed %d to migrate svms %p [0x%lx 0x%lx]\n",
	2417	r, svms, prange->start, prange->last);
	2418	goto out_unlock_range;
	2419	}
	2420	}
	2421
	2422	r = svm_range_validate_and_map(mm, prange, gpuidx, false, false);
	2423	if (r)
	2424	pr_debug("failed %d to map svms 0x%p [0x%lx 0x%lx] to gpus\n",
	2425	r, svms, prange->start, prange->last);
	2426
	2427	out_unlock_range:
	2428	mutex_unlock(&prange->migrate_mutex);
	2429	out_unlock_svms:
	2430	mutex_unlock(&svms->lock);
	2431	mmap_read_unlock(mm);
	2432	mmput(mm);
	2433	out:
	2434	kfd_unref_process(p);
	2435
	2436	if (r == -EAGAIN) {
	2437	pr_debug("recover vm fault later\n");
	2438	amdgpu_gmc_filter_faults_remove(adev, addr, pasid);
	2439	r = 0;
	2440	}
	2441	return r;
	2442	}
	2443
	2444	void svm_range_list_fini(struct kfd_process *p)
	2445	{
	2446	struct svm_range *prange;
	2447	struct svm_range *next;
	2448
	2449	pr_debug("pasid 0x%x svms 0x%p\n", p->pasid, &p->svms);
	2450
	2451	/* Ensure list work is finished before process is destroyed */
	2452	flush_work(&p->svms.deferred_list_work);
	2453
	2454	list_for_each_entry_safe(prange, next, &p->svms.list, list) {
	2455	svm_range_unlink(prange);
	2456	svm_range_remove_notifier(prange);
	2457	svm_range_free(prange);
	2458	}
	2459
	2460	mutex_destroy(&p->svms.lock);
	2461
	2462	pr_debug("pasid 0x%x svms 0x%p done\n", p->pasid, &p->svms);
	2463	}
	2464
	2465	int svm_range_list_init(struct kfd_process *p)
	2466	{
	2467	struct svm_range_list *svms = &p->svms;
	2468
	2469	svms->objects = RB_ROOT_CACHED;
	2470	mutex_init(&svms->lock);
	2471	INIT_LIST_HEAD(&svms->list);
	2472	atomic_set(&svms->evicted_ranges, 0);
	2473	INIT_DELAYED_WORK(&svms->restore_work, svm_range_restore_work);
	2474	INIT_WORK(&svms->deferred_list_work, svm_range_deferred_list_work);
	2475	INIT_LIST_HEAD(&svms->deferred_range_list);
	2476	spin_lock_init(&svms->deferred_list_lock);
	2477
	2478	return 0;
	2479	}
	2480
	2481	/**
	2482	* svm_range_is_valid - check if virtual address range is valid
	2483	* @mm: current process mm_struct
	2484	* @start: range start address, in pages
	2485	* @size: range size, in pages
	2486	*
	2487	* Valid virtual address range means it belongs to one or more VMAs
	2488	*
	2489	* Context: Process context
	2490	*
	2491	* Return:
	2492	* true - valid svm range
	2493	* false - invalid svm range
	2494	*/
	2495	static bool
	2496	svm_range_is_valid(struct mm_struct *mm, uint64_t start, uint64_t size)
	2497	{
	2498	const unsigned long device_vma = VM_IO \| VM_PFNMAP \| VM_MIXEDMAP;
	2499	struct vm_area_struct *vma;
	2500	unsigned long end;
	2501
	2502	start <<= PAGE_SHIFT;
	2503	end = start + (size << PAGE_SHIFT);
	2504
	2505	do {
	2506	vma = find_vma(mm, start);
	2507	if (!vma \|\| start < vma->vm_start \|\|
	2508	(vma->vm_flags & device_vma))
	2509	return false;
	2510	start = min(end, vma->vm_end);
	2511	} while (start < end);
	2512
	2513	return true;
	2514	}
	2515
	2516	/**
	2517	* svm_range_add - add svm range and handle overlap
	2518	* @p: the range add to this process svms
	2519	* @start: page size aligned
	2520	* @size: page size aligned
	2521	* @nattr: number of attributes
	2522	* @attrs: array of attributes
	2523	* @update_list: output, the ranges need validate and update GPU mapping
	2524	* @insert_list: output, the ranges need insert to svms
	2525	* @remove_list: output, the ranges are replaced and need remove from svms
	2526	*
	2527	* Check if the virtual address range has overlap with the registered ranges,
	2528	* split the overlapped range, copy and adjust pages address and vram nodes in
	2529	* old and new ranges.
	2530	*
	2531	* Context: Process context, caller must hold svms->lock
	2532	*
	2533	* Return:
	2534	* 0 - OK, otherwise error code
	2535	*/
	2536	static int
	2537	svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size,
	2538	uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs,
	2539	struct list_head update_list, struct list_head insert_list,
	2540	struct list_head *remove_list)
	2541	{
	2542	uint64_t last = start + size - 1UL;
	2543	struct svm_range_list *svms;
	2544	struct svm_range new = {0};
	2545	struct svm_range *prange;
	2546	unsigned long left = 0;
	2547	int r = 0;
	2548
	2549	pr_debug("svms 0x%p [0x%llx 0x%llx]\n", &p->svms, start, last);
	2550
	2551	svm_range_apply_attrs(p, &new, nattr, attrs);
	2552
	2553	svms = &p->svms;
	2554
	2555	r = svm_range_handle_overlap(svms, &new, start, last, update_list,
	2556	insert_list, remove_list, &left);
	2557	if (r)
	2558	return r;
	2559
	2560	if (left) {
	2561	prange = svm_range_new(svms, last - left + 1, last);
	2562	list_add(&prange->insert_list, insert_list);
	2563	list_add(&prange->update_list, update_list);
	2564	}
	2565
	2566	return 0;
	2567	}
	2568
	2569	/* svm_range_best_prefetch_location - decide the best prefetch location
	2570	* @prange: svm range structure
	2571	*
	2572	* For xnack off:
	2573	* If range map to single GPU, the best acutal location is prefetch loc, which
	2574	* can be CPU or GPU.
	2575	*
	2576	* If range map to multiple GPUs, only if mGPU connection on xgmi same hive,
	2577	* the best actual location could be prefetch_loc GPU. If mGPU connection on
	2578	* PCIe, the best actual location is always CPU, because GPU cannot access vram
	2579	* of other GPUs, assuming PCIe small bar (large bar support is not upstream).
	2580	*
	2581	* For xnack on:
	2582	* The best actual location is prefetch location. If mGPU connection on xgmi
	2583	* same hive, range map to multiple GPUs. Otherwise, the range only map to
	2584	* actual location GPU. Other GPU access vm fault will trigger migration.
	2585	*
	2586	* Context: Process context
	2587	*
	2588	* Return:
	2589	* 0 for CPU or GPU id
	2590	*/
	2591	static uint32_t
	2592	svm_range_best_prefetch_location(struct svm_range *prange)
	2593	{
	2594	DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);
	2595	uint32_t best_loc = prange->prefetch_loc;
	2596	struct kfd_process_device *pdd;
	2597	struct amdgpu_device *bo_adev;
	2598	struct amdgpu_device *adev;
	2599	struct kfd_process *p;
	2600	uint32_t gpuidx;
	2601
	2602	p = container_of(prange->svms, struct kfd_process, svms);
	2603
	2604	/* xnack on */
	2605	if (p->xnack_enabled)
	2606	goto out;
	2607
	2608	/* xnack off */
	2609	if (!best_loc \|\| best_loc == KFD_IOCTL_SVM_LOCATION_UNDEFINED)
	2610	goto out;
	2611
	2612	bo_adev = svm_range_get_adev_by_id(prange, best_loc);
	2613	if (!bo_adev) {
	2614	WARN_ONCE(1, "failed to get device by id 0x%x\n", best_loc);
	2615	best_loc = 0;
	2616	goto out;
	2617	}
	2618	bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip,
	2619	MAX_GPU_INSTANCE);
	2620
	2621	for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
	2622	pdd = kfd_process_device_from_gpuidx(p, gpuidx);
	2623	if (!pdd) {
	2624	pr_debug("failed to get device by idx 0x%x\n", gpuidx);
	2625	continue;
	2626	}
	2627	adev = (struct amdgpu_device *)pdd->dev->kgd;
	2628
	2629	if (adev == bo_adev)
	2630	continue;
	2631
	2632	if (!amdgpu_xgmi_same_hive(adev, bo_adev)) {
	2633	best_loc = 0;
	2634	break;
	2635	}
	2636	}
	2637
	2638	out:
	2639	pr_debug("xnack %d svms 0x%p [0x%lx 0x%lx] best loc 0x%x\n",
	2640	p->xnack_enabled, &p->svms, prange->start, prange->last,
	2641	best_loc);
	2642
	2643	return best_loc;
	2644	}
	2645
	2646	/* FIXME: This is a workaround for page locking bug when some pages are
	2647	* invalid during migration to VRAM
	2648	*/
	2649	void svm_range_prefault(struct svm_range prange, struct mm_struct mm)
	2650	{
	2651	struct hmm_range *hmm_range;
	2652	int r;
	2653
	2654	if (prange->validated_once)
	2655	return;
	2656
	2657	r = amdgpu_hmm_range_get_pages(&prange->notifier, mm, NULL,
	2658	prange->start << PAGE_SHIFT,
	2659	prange->npages, &hmm_range,
	2660	false, true);
	2661	if (!r) {
	2662	amdgpu_hmm_range_get_pages_done(hmm_range);
	2663	prange->validated_once = true;
	2664	}
	2665	}
	2666
	2667	/* svm_range_trigger_migration - start page migration if prefetch loc changed
	2668	* @mm: current process mm_struct
	2669	* @prange: svm range structure
	2670	* @migrated: output, true if migration is triggered
	2671	*
	2672	* If range perfetch_loc is GPU, actual loc is cpu 0, then migrate the range
	2673	* from ram to vram.
	2674	* If range prefetch_loc is cpu 0, actual loc is GPU, then migrate the range
	2675	* from vram to ram.
	2676	*
	2677	* If GPU vm fault retry is not enabled, migration interact with MMU notifier
	2678	* and restore work:
	2679	* 1. migrate_vma_setup invalidate pages, MMU notifier callback svm_range_evict
	2680	* stops all queues, schedule restore work
	2681	* 2. svm_range_restore_work wait for migration is done by
	2682	* a. svm_range_validate_vram takes prange->migrate_mutex
	2683	* b. svm_range_validate_ram HMM get pages wait for CPU fault handle returns
	2684	* 3. restore work update mappings of GPU, resume all queues.
	2685	*
	2686	* Context: Process context
	2687	*
	2688	* Return:
	2689	* 0 - OK, otherwise - error code of migration
	2690	*/
	2691	static int
	2692	svm_range_trigger_migration(struct mm_struct mm, struct svm_range prange,
	2693	bool *migrated)
	2694	{
	2695	uint32_t best_loc;
	2696	int r = 0;
	2697
	2698	*migrated = false;
	2699	best_loc = svm_range_best_prefetch_location(prange);
	2700
	2701	if (best_loc == KFD_IOCTL_SVM_LOCATION_UNDEFINED \|\|
	2702	best_loc == prange->actual_loc)
	2703	return 0;
	2704
	2705	/*
	2706	* Prefetch to GPU without host access flag, set actual_loc to gpu, then
	2707	* validate on gpu and map to gpus will be handled afterwards.
	2708	*/
	2709	if (best_loc && !prange->actual_loc &&
	2710	!(prange->flags & KFD_IOCTL_SVM_FLAG_HOST_ACCESS)) {
	2711	prange->actual_loc = best_loc;
	2712	return 0;
	2713	}
	2714
	2715	if (!best_loc) {
	2716	r = svm_migrate_vram_to_ram(prange, mm);
	2717	*migrated = !r;
	2718	return r;
	2719	}
	2720
	2721	r = svm_migrate_to_vram(prange, best_loc, mm);
	2722	*migrated = !r;
	2723
	2724	return r;
	2725	}
	2726
	2727	int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence)
	2728	{
	2729	if (!fence)
	2730	return -EINVAL;
	2731
	2732	if (dma_fence_is_signaled(&fence->base))
	2733	return 0;
	2734
	2735	if (fence->svm_bo) {
	2736	WRITE_ONCE(fence->svm_bo->evicting, 1);
	2737	schedule_work(&fence->svm_bo->eviction_work);
	2738	}
	2739
	2740	return 0;
	2741	}
	2742
	2743	static void svm_range_evict_svm_bo_worker(struct work_struct *work)
	2744	{
	2745	struct svm_range_bo *svm_bo;
	2746	struct kfd_process *p;
	2747	struct mm_struct *mm;
	2748
	2749	svm_bo = container_of(work, struct svm_range_bo, eviction_work);
	2750	if (!svm_bo_ref_unless_zero(svm_bo))
	2751	return; /* svm_bo was freed while eviction was pending */
	2752
	2753	/* svm_range_bo_release destroys this worker thread. So during
	2754	* the lifetime of this thread, kfd_process and mm will be valid.
	2755	*/
	2756	p = container_of(svm_bo->svms, struct kfd_process, svms);
	2757	mm = p->mm;
	2758	if (!mm)
	2759	return;
	2760
	2761	mmap_read_lock(mm);
	2762	spin_lock(&svm_bo->list_lock);
	2763	while (!list_empty(&svm_bo->range_list)) {
	2764	struct svm_range *prange =
	2765	list_first_entry(&svm_bo->range_list,
	2766	struct svm_range, svm_bo_list);
	2767	list_del_init(&prange->svm_bo_list);
	2768	spin_unlock(&svm_bo->list_lock);
	2769
	2770	pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,
	2771	prange->start, prange->last);
	2772
	2773	mutex_lock(&prange->migrate_mutex);
	2774	svm_migrate_vram_to_ram(prange, svm_bo->eviction_fence->mm);
	2775
	2776	mutex_lock(&prange->lock);
	2777	prange->svm_bo = NULL;
	2778	mutex_unlock(&prange->lock);
	2779
	2780	mutex_unlock(&prange->migrate_mutex);
	2781
	2782	spin_lock(&svm_bo->list_lock);
	2783	}
	2784	spin_unlock(&svm_bo->list_lock);
	2785	mmap_read_unlock(mm);
	2786
	2787	dma_fence_signal(&svm_bo->eviction_fence->base);
	2788	/* This is the last reference to svm_bo, after svm_range_vram_node_free
	2789	* has been called in svm_migrate_vram_to_ram
	2790	*/
	2791	WARN_ONCE(kref_read(&svm_bo->kref) != 1, "This was not the last reference\n");
	2792	svm_range_bo_unref(svm_bo);
	2793	}
	2794
	2795	static int
	2796	svm_range_set_attr(struct kfd_process *p, uint64_t start, uint64_t size,
	2797	uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs)
	2798	{
	2799	struct amdkfd_process_info *process_info = p->kgd_process_info;
	2800	struct mm_struct *mm = current->mm;
	2801	struct list_head update_list;
	2802	struct list_head insert_list;
	2803	struct list_head remove_list;
	2804	struct svm_range_list *svms;
	2805	struct svm_range *prange;
	2806	struct svm_range *next;
	2807	int r = 0;
	2808
	2809	pr_debug("pasid 0x%x svms 0x%p [0x%llx 0x%llx] pages 0x%llx\n",
	2810	p->pasid, &p->svms, start, start + size - 1, size);
	2811
	2812	r = svm_range_check_attr(p, nattr, attrs);
	2813	if (r)
	2814	return r;
	2815
	2816	svms = &p->svms;
	2817
	2818	mutex_lock(&process_info->lock);
	2819
	2820	svm_range_list_lock_and_flush_work(svms, mm);
	2821
	2822	if (!svm_range_is_valid(mm, start, size)) {
	2823	pr_debug("invalid range\n");
	2824	r = -EFAULT;
	2825	mmap_write_unlock(mm);
	2826	goto out;
	2827	}
	2828
	2829	mutex_lock(&svms->lock);
	2830
	2831	/* Add new range and split existing ranges as needed */
	2832	r = svm_range_add(p, start, size, nattr, attrs, &update_list,
	2833	&insert_list, &remove_list);
	2834	if (r) {
	2835	mutex_unlock(&svms->lock);
	2836	mmap_write_unlock(mm);
	2837	goto out;
	2838	}
	2839	/* Apply changes as a transaction */
	2840	list_for_each_entry_safe(prange, next, &insert_list, insert_list) {
	2841	svm_range_add_to_svms(prange);
	2842	svm_range_add_notifier_locked(mm, prange);
	2843	}
	2844	list_for_each_entry(prange, &update_list, update_list) {
	2845	svm_range_apply_attrs(p, prange, nattr, attrs);
	2846	/* TODO: unmap ranges from GPU that lost access */
	2847	}
	2848	list_for_each_entry_safe(prange, next, &remove_list,
	2849	remove_list) {
	2850	pr_debug("unlink old 0x%p prange 0x%p [0x%lx 0x%lx]\n",
	2851	prange->svms, prange, prange->start,
	2852	prange->last);
	2853	svm_range_unlink(prange);
	2854	svm_range_remove_notifier(prange);
	2855	svm_range_free(prange);
	2856	}
	2857
	2858	mmap_write_downgrade(mm);
	2859	/* Trigger migrations and revalidate and map to GPUs as needed. If
	2860	* this fails we may be left with partially completed actions. There
	2861	* is no clean way of rolling back to the previous state in such a
	2862	* case because the rollback wouldn't be guaranteed to work either.
	2863	*/
	2864	list_for_each_entry(prange, &update_list, update_list) {
	2865	bool migrated;
	2866
	2867	mutex_lock(&prange->migrate_mutex);
	2868
	2869	r = svm_range_trigger_migration(mm, prange, &migrated);
	2870	if (r)
	2871	goto out_unlock_range;
	2872
	2873	if (migrated && !p->xnack_enabled) {
	2874	pr_debug("restore_work will update mappings of GPUs\n");
	2875	mutex_unlock(&prange->migrate_mutex);
	2876	continue;
	2877	}
	2878
	2879	r = svm_range_validate_and_map(mm, prange, MAX_GPU_INSTANCE,
	2880	true, true);
	2881	if (r)
	2882	pr_debug("failed %d to map svm range\n", r);
	2883
	2884	out_unlock_range:
	2885	mutex_unlock(&prange->migrate_mutex);
	2886	if (r)
	2887	break;
	2888	}
	2889
	2890	svm_range_debug_dump(svms);
	2891
	2892	mutex_unlock(&svms->lock);
	2893	mmap_read_unlock(mm);
	2894	out:
	2895	mutex_unlock(&process_info->lock);
	2896
	2897	pr_debug("pasid 0x%x svms 0x%p [0x%llx 0x%llx] done, r=%d\n", p->pasid,
	2898	&p->svms, start, start + size - 1, r);
	2899
	2900	return r;
	2901	}
	2902
	2903	static int
	2904	svm_range_get_attr(struct kfd_process *p, uint64_t start, uint64_t size,
	2905	uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs)
	2906	{
	2907	DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE);
	2908	DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE);
	2909	bool get_preferred_loc = false;
	2910	bool get_prefetch_loc = false;
	2911	bool get_granularity = false;
	2912	bool get_accessible = false;
	2913	bool get_flags = false;
	2914	uint64_t last = start + size - 1UL;
	2915	struct mm_struct *mm = current->mm;
	2916	uint8_t granularity = 0xff;
	2917	struct interval_tree_node *node;
	2918	struct svm_range_list *svms;
	2919	struct svm_range *prange;
	2920	uint32_t prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
	2921	uint32_t location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
	2922	uint32_t flags = 0xffffffff;
	2923	int gpuidx;
	2924	uint32_t i;
	2925
	2926	pr_debug("svms 0x%p [0x%llx 0x%llx] nattr 0x%x\n", &p->svms, start,
	2927	start + size - 1, nattr);
	2928
	2929	mmap_read_lock(mm);
	2930	if (!svm_range_is_valid(mm, start, size)) {
	2931	pr_debug("invalid range\n");
	2932	mmap_read_unlock(mm);
	2933	return -EINVAL;
	2934	}
	2935	mmap_read_unlock(mm);
	2936
	2937	for (i = 0; i < nattr; i++) {
	2938	switch (attrs[i].type) {
	2939	case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
	2940	get_preferred_loc = true;
	2941	break;
	2942	case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
	2943	get_prefetch_loc = true;
	2944	break;
	2945	case KFD_IOCTL_SVM_ATTR_ACCESS:
	2946	get_accessible = true;
	2947	break;
	2948	case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
	2949	get_flags = true;
	2950	break;
	2951	case KFD_IOCTL_SVM_ATTR_GRANULARITY:
	2952	get_granularity = true;
	2953	break;
	2954	case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:
	2955	case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:
	2956	case KFD_IOCTL_SVM_ATTR_NO_ACCESS:
	2957	fallthrough;
	2958	default:
	2959	pr_debug("get invalid attr type 0x%x\n", attrs[i].type);
	2960	return -EINVAL;
	2961	}
	2962	}
	2963
	2964	svms = &p->svms;
	2965
	2966	mutex_lock(&svms->lock);
	2967
	2968	node = interval_tree_iter_first(&svms->objects, start, last);
	2969	if (!node) {
	2970	pr_debug("range attrs not found return default values\n");
	2971	svm_range_set_default_attributes(&location, &prefetch_loc,
	2972	&granularity, &flags);
	2973	if (p->xnack_enabled)
	2974	bitmap_fill(bitmap_access, MAX_GPU_INSTANCE);
	2975	else
	2976	bitmap_zero(bitmap_access, MAX_GPU_INSTANCE);
	2977	bitmap_zero(bitmap_aip, MAX_GPU_INSTANCE);
	2978	goto fill_values;
	2979	}
	2980	bitmap_fill(bitmap_access, MAX_GPU_INSTANCE);
	2981	bitmap_fill(bitmap_aip, MAX_GPU_INSTANCE);
	2982
	2983	while (node) {
	2984	struct interval_tree_node *next;
	2985
	2986	prange = container_of(node, struct svm_range, it_node);
	2987	next = interval_tree_iter_next(node, start, last);
	2988
	2989	if (get_preferred_loc) {
	2990	if (prange->preferred_loc ==
	2991	KFD_IOCTL_SVM_LOCATION_UNDEFINED \|\|
	2992	(location != KFD_IOCTL_SVM_LOCATION_UNDEFINED &&
	2993	location != prange->preferred_loc)) {
	2994	location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
	2995	get_preferred_loc = false;
	2996	} else {
	2997	location = prange->preferred_loc;
	2998	}
	2999	}
	3000	if (get_prefetch_loc) {
	3001	if (prange->prefetch_loc ==
	3002	KFD_IOCTL_SVM_LOCATION_UNDEFINED \|\|
	3003	(prefetch_loc != KFD_IOCTL_SVM_LOCATION_UNDEFINED &&
	3004	prefetch_loc != prange->prefetch_loc)) {
	3005	prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
	3006	get_prefetch_loc = false;
	3007	} else {
	3008	prefetch_loc = prange->prefetch_loc;
	3009	}
	3010	}
	3011	if (get_accessible) {
	3012	bitmap_and(bitmap_access, bitmap_access,
	3013	prange->bitmap_access, MAX_GPU_INSTANCE);
	3014	bitmap_and(bitmap_aip, bitmap_aip,
	3015	prange->bitmap_aip, MAX_GPU_INSTANCE);
	3016	}
	3017	if (get_flags)
	3018	flags &= prange->flags;
	3019
	3020	if (get_granularity && prange->granularity < granularity)
	3021	granularity = prange->granularity;
	3022
	3023	node = next;
	3024	}
	3025	fill_values:
	3026	mutex_unlock(&svms->lock);
	3027
	3028	for (i = 0; i < nattr; i++) {
	3029	switch (attrs[i].type) {
	3030	case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
	3031	attrs[i].value = location;
	3032	break;
	3033	case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
	3034	attrs[i].value = prefetch_loc;
	3035	break;
	3036	case KFD_IOCTL_SVM_ATTR_ACCESS:
	3037	gpuidx = kfd_process_gpuidx_from_gpuid(p,
	3038	attrs[i].value);
	3039	if (gpuidx < 0) {
	3040	pr_debug("invalid gpuid %x\n", attrs[i].value);
	3041	return -EINVAL;
	3042	}
	3043	if (test_bit(gpuidx, bitmap_access))
	3044	attrs[i].type = KFD_IOCTL_SVM_ATTR_ACCESS;
	3045	else if (test_bit(gpuidx, bitmap_aip))
	3046	attrs[i].type =
	3047	KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE;
	3048	else
	3049	attrs[i].type = KFD_IOCTL_SVM_ATTR_NO_ACCESS;
	3050	break;
	3051	case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
	3052	attrs[i].value = flags;
	3053	break;
	3054	case KFD_IOCTL_SVM_ATTR_GRANULARITY:
	3055	attrs[i].value = (uint32_t)granularity;
	3056	break;
	3057	}
	3058	}
	3059
	3060	return 0;
	3061	}
	3062
	3063	int
	3064	svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,
	3065	uint64_t size, uint32_t nattrs, struct kfd_ioctl_svm_attribute *attrs)
	3066	{
	3067	int r;
	3068
	3069	start >>= PAGE_SHIFT;
	3070	size >>= PAGE_SHIFT;
	3071
	3072	switch (op) {
	3073	case KFD_IOCTL_SVM_OP_SET_ATTR:
	3074	r = svm_range_set_attr(p, start, size, nattrs, attrs);
	3075	break;
	3076	case KFD_IOCTL_SVM_OP_GET_ATTR:
	3077	r = svm_range_get_attr(p, start, size, nattrs, attrs);
	3078	break;
	3079	default:
	3080	r = EINVAL;
	3081	break;
	3082	}
	3083
	3084	return r;
	3085	}

drivers/gpu/drm/amd/amdkfd/kfd_svm.h+206−0

@@ -0,0 +1,206 @@
	1	/* SPDX-License-Identifier: GPL-2.0 OR MIT */
	2	/*
	3	* Copyright 2020-2021 Advanced Micro Devices, Inc.
	4	*
	5	* Permission is hereby granted, free of charge, to any person obtaining a
	6	* copy of this software and associated documentation files (the "Software"),
	7	* to deal in the Software without restriction, including without limitation
	8	* the rights to use, copy, modify, merge, publish, distribute, sublicense,
	9	* and/or sell copies of the Software, and to permit persons to whom the
	10	* Software is furnished to do so, subject to the following conditions:
	11	*
	12	* The above copyright notice and this permission notice shall be included in
	13	* all copies or substantial portions of the Software.
	14	*
	15	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	19	* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	20	* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	21	* OTHER DEALINGS IN THE SOFTWARE.
	22	*
	23	*/
	24
	25	#ifndef KFD_SVM_H_
	26	#define KFD_SVM_H_
	27
	28	#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
	29
	30	#include <linux/rwsem.h>
	31	#include <linux/list.h>
	32	#include <linux/mutex.h>
	33	#include <linux/sched/mm.h>
	34	#include <linux/hmm.h>
	35	#include "amdgpu.h"
	36	#include "kfd_priv.h"
	37
	38	struct svm_range_bo {
	39	struct amdgpu_bo *bo;
	40	struct kref kref;
	41	struct list_head range_list; /* all svm ranges shared this bo */
	42	spinlock_t list_lock;
	43	struct amdgpu_amdkfd_fence *eviction_fence;
	44	struct work_struct eviction_work;
	45	struct svm_range_list *svms;
	46	uint32_t evicting;
	47	};
	48
	49	enum svm_work_list_ops {
	50	SVM_OP_NULL,
	51	SVM_OP_UNMAP_RANGE,
	52	SVM_OP_UPDATE_RANGE_NOTIFIER,
	53	SVM_OP_UPDATE_RANGE_NOTIFIER_AND_MAP,
	54	SVM_OP_ADD_RANGE,
	55	SVM_OP_ADD_RANGE_AND_MAP
	56	};
	57
	58	struct svm_work_list_item {
	59	enum svm_work_list_ops op;
	60	struct mm_struct *mm;
	61	};
	62
	63	/**
	64	* struct svm_range - shared virtual memory range
	65	*
	66	* @svms: list of svm ranges, structure defined in kfd_process
	67	* @migrate_mutex: to serialize range migration, validation and mapping update
	68	* @start: range start address in pages
	69	* @last: range last address in pages
	70	* @it_node: node [start, last] stored in interval tree, start, last are page
	71	* aligned, page size is (last - start + 1)
	72	* @list: link list node, used to scan all ranges of svms
	73	* @update_list:link list node used to add to update_list
	74	* @remove_list:link list node used to add to remove list
	75	* @insert_list:link list node used to add to insert list
	76	* @mapping: bo_va mapping structure to create and update GPU page table
	77	* @npages: number of pages
	78	* @dma_addr: dma mapping address on each GPU for system memory physical page
	79	* @ttm_res: vram ttm resource map
	80	* @offset: range start offset within mm_nodes
	81	* @svm_bo: struct to manage splited amdgpu_bo
	82	* @svm_bo_list:link list node, to scan all ranges which share same svm_bo
	83	* @lock: protect prange start, last, child_list, svm_bo_list
	84	* @saved_flags:save/restore current PF_MEMALLOC flags
	85	* @flags: flags defined as KFD_IOCTL_SVM_FLAG_*
	86	* @perferred_loc: perferred location, 0 for CPU, or GPU id
	87	* @perfetch_loc: last prefetch location, 0 for CPU, or GPU id
	88	* @actual_loc: the actual location, 0 for CPU, or GPU id
	89	* @granularity:migration granularity, log2 num pages
	90	* @invalid: not 0 means cpu page table is invalidated
	91	* @validate_timestamp: system timestamp when range is validated
	92	* @notifier: register mmu interval notifier
	93	* @work_item: deferred work item information
	94	* @deferred_list: list header used to add range to deferred list
	95	* @child_list: list header for split ranges which are not added to svms yet
	96	* @bitmap_access: index bitmap of GPUs which can access the range
	97	* @bitmap_aip: index bitmap of GPUs which can access the range in place
	98	*
	99	* Data structure for virtual memory range shared by CPU and GPUs, it can be
	100	* allocated from system memory ram or device vram, and migrate from ram to vram
	101	* or from vram to ram.
	102	*/
	103	struct svm_range {
	104	struct svm_range_list *svms;
	105	struct mutex migrate_mutex;
	106	unsigned long start;
	107	unsigned long last;
	108	struct interval_tree_node it_node;
	109	struct list_head list;
	110	struct list_head update_list;
	111	struct list_head remove_list;
	112	struct list_head insert_list;
	113	struct amdgpu_bo_va_mapping mapping;
	114	uint64_t npages;
	115	dma_addr_t *dma_addr[MAX_GPU_INSTANCE];
	116	struct ttm_resource *ttm_res;
	117	uint64_t offset;
	118	struct svm_range_bo *svm_bo;
	119	struct list_head svm_bo_list;
	120	struct mutex lock;
	121	unsigned int saved_flags;
	122	uint32_t flags;
	123	uint32_t preferred_loc;
	124	uint32_t prefetch_loc;
	125	uint32_t actual_loc;
	126	uint8_t granularity;
	127	atomic_t invalid;
	128	uint64_t validate_timestamp;
	129	struct mmu_interval_notifier notifier;
	130	struct svm_work_list_item work_item;
	131	struct list_head deferred_list;
	132	struct list_head child_list;
	133	DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE);
	134	DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE);
	135	bool validated_once;
	136	};
	137
	138	static inline void svm_range_lock(struct svm_range *prange)
	139	{
	140	mutex_lock(&prange->lock);
	141	prange->saved_flags = memalloc_noreclaim_save();
	142
	143	}
	144	static inline void svm_range_unlock(struct svm_range *prange)
	145	{
	146	memalloc_noreclaim_restore(prange->saved_flags);
	147	mutex_unlock(&prange->lock);
	148	}
	149
	150	int svm_range_list_init(struct kfd_process *p);
	151	void svm_range_list_fini(struct kfd_process *p);
	152	int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,
	153	uint64_t size, uint32_t nattrs,
	154	struct kfd_ioctl_svm_attribute *attrs);
	155	struct svm_range svm_range_from_addr(struct svm_range_list svms,
	156	unsigned long addr,
	157	struct svm_range **parent);
	158	struct amdgpu_device svm_range_get_adev_by_id(struct svm_range prange,
	159	uint32_t id);
	160	int svm_range_vram_node_new(struct amdgpu_device *adev,
	161	struct svm_range *prange, bool clear);
	162	void svm_range_vram_node_free(struct svm_range *prange);
	163	int svm_range_split_by_granularity(struct kfd_process p, struct mm_struct mm,
	164	unsigned long addr, struct svm_range *parent,
	165	struct svm_range *prange);
	166	int svm_range_restore_pages(struct amdgpu_device *adev,
	167	unsigned int pasid, uint64_t addr);
	168	int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence);
	169	void svm_range_add_list_work(struct svm_range_list *svms,
	170	struct svm_range prange, struct mm_struct mm,
	171	enum svm_work_list_ops op);
	172	void schedule_deferred_list_work(struct svm_range_list *svms);
	173	void svm_range_dma_unmap(struct device dev, dma_addr_t dma_addr,
	174	unsigned long offset, unsigned long npages);
	175	void svm_range_free_dma_mappings(struct svm_range *prange);
	176	void svm_range_prefault(struct svm_range prange, struct mm_struct mm);
	177
	178	#else
	179
	180	struct kfd_process;
	181
	182	static inline int svm_range_list_init(struct kfd_process *p)
	183	{
	184	return 0;
	185	}
	186	static inline void svm_range_list_fini(struct kfd_process *p)
	187	{
	188	/* empty */
	189	}
	190
	191	static inline int svm_range_restore_pages(struct amdgpu_device *adev,
	192	unsigned int pasid, uint64_t addr)
	193	{
	194	return -EFAULT;
	195	}
	196
	197	static inline int svm_range_schedule_evict_svm_bo(
	198	struct amdgpu_amdkfd_fence *fence)
	199	{
	200	WARN_ONCE(1, "SVM eviction fence triggered, but SVM is disabled");
	201	return -EINVAL;
	202	}
	203
	204	#endif /* IS_ENABLED(CONFIG_HSA_AMD_SVM) */
	205
	206	#endif /* KFD_SVM_H_ */

drivers/gpu/drm/amd/amdkfd/kfd_topology.c+54−28

@@ -1192,40 +1192,60 @@ static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev) · 1191 unmodified lines
1192	mem->mem_clk_max = local_mem_info.mem_clk_max;	1192	mem->mem_clk_max = local_mem_info.mem_clk_max;
1193	}	1193	}
1194		1194
1195	static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)	1195	static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev,
		1196	struct kfd_topology_device *target_gpu_dev,
		1197	struct kfd_iolink_properties *link)
1196	{	1198	{
1197	struct kfd_iolink_properties link, cpu_link;	1199	/* xgmi always supports atomics between links. */
1198	struct kfd_topology_device *cpu_dev;	1200	if (link->iolink_type == CRAT_IOLINK_TYPE_XGMI)
1199	uint32_t cap;
1200	uint32_t cpu_flag = CRAT_IOLINK_FLAGS_ENABLED;
1201	uint32_t flag = CRAT_IOLINK_FLAGS_ENABLED;
1202
1203	if (!dev \|\| !dev->gpu)
1204	return;	1201	return;
1205		1202
1206	pcie_capability_read_dword(dev->gpu->pdev,	1203	/* check pcie support to set cpu(dev) flags for target_gpu_dev link. */
1207	PCI_EXP_DEVCAP2, &cap);	1204	if (target_gpu_dev) {
		1205	uint32_t cap;
1208		1206
1209	if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 \|	1207	pcie_capability_read_dword(target_gpu_dev->gpu->pdev,
1210	PCI_EXP_DEVCAP2_ATOMIC_COMP64)))	1208	PCI_EXP_DEVCAP2, &cap);
1211	cpu_flag \|= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT \|	1209
1212	CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;	1210	if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 \|
		1211	PCI_EXP_DEVCAP2_ATOMIC_COMP64)))
		1212	link->flags \|= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT \|
		1213	CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
		1214	/* set gpu (dev) flags. */
		1215	} else {
		1216	if (!dev->gpu->pci_atomic_requested \|\|
		1217	dev->gpu->device_info->asic_family ==
		1218	CHIP_HAWAII)
		1219	link->flags \|= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT \|
		1220	CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
		1221	}
		1222	}
		1223
		1224	static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
		1225	{
		1226	struct kfd_iolink_properties link, inbound_link;
		1227	struct kfd_topology_device *peer_dev;
1213		1228
1214	if (!dev->gpu->pci_atomic_requested \|\|	1229	if (!dev \|\| !dev->gpu)
1215	dev->gpu->device_info->asic_family == CHIP_HAWAII)	1230	return;
1216	flag \|= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT \|
1217	CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
1218		1231
1219	/* GPU only creates direct links so apply flags setting to all */	1232	/* GPU only creates direct links so apply flags setting to all */
1220	list_for_each_entry(link, &dev->io_link_props, list) {	1233	list_for_each_entry(link, &dev->io_link_props, list) {
1221	link->flags = flag;	1234	link->flags = CRAT_IOLINK_FLAGS_ENABLED;
1222	cpu_dev = kfd_topology_device_by_proximity_domain(	1235	kfd_set_iolink_no_atomics(dev, NULL, link);
		1236	peer_dev = kfd_topology_device_by_proximity_domain(
1223	link->node_to);	1237	link->node_to);
1224	if (cpu_dev) {	1238
1225	list_for_each_entry(cpu_link,	1239	if (!peer_dev)
1226	&cpu_dev->io_link_props, list)	1240	continue;
1227	if (cpu_link->node_to == link->node_from)	1241
1228	cpu_link->flags = cpu_flag;	1242	list_for_each_entry(inbound_link, &peer_dev->io_link_props,
		1243	list) {
		1244	if (inbound_link->node_to != link->node_from)
		1245	continue;
		1246
		1247	inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED;
		1248	kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link);
1229	}	1249	}
1230	}	1250	}
1231	}	1251	}
@@ -1410,15 +1430,21 @@ int kfd_topology_add_device(struct kfd_dev *gpu) · 178 unmodified lines
1410	adev = (struct amdgpu_device *)(dev->gpu->kgd);	1430	adev = (struct amdgpu_device *)(dev->gpu->kgd);
1411	/* kfd only concerns sram ecc on GFX and HBM ecc on UMC */	1431	/* kfd only concerns sram ecc on GFX and HBM ecc on UMC */
1412	dev->node_props.capability \|=	1432	dev->node_props.capability \|=
1413	((adev->ras_features & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0) ?	1433	((adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0) ?
1414	HSA_CAP_SRAM_EDCSUPPORTED : 0;	1434	HSA_CAP_SRAM_EDCSUPPORTED : 0;
1415	dev->node_props.capability \|= ((adev->ras_features & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ?	1435	dev->node_props.capability \|= ((adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ?
1416	HSA_CAP_MEM_EDCSUPPORTED : 0;	1436	HSA_CAP_MEM_EDCSUPPORTED : 0;
1417		1437
1418	if (adev->asic_type != CHIP_VEGA10)	1438	if (adev->asic_type != CHIP_VEGA10)
1419	dev->node_props.capability \|= (adev->ras_features != 0) ?	1439	dev->node_props.capability \|= (adev->ras_enabled != 0) ?
1420	HSA_CAP_RASEVENTNOTIFY : 0;	1440	HSA_CAP_RASEVENTNOTIFY : 0;
1421		1441
		1442	/* SVM API and HMM page migration work together, device memory type
		1443	* is initialized to not 0 when page migration register device memory.
		1444	*/
		1445	if (adev->kfd.dev->pgmap.type != 0)
		1446	dev->node_props.capability \|= HSA_CAP_SVMAPI_SUPPORTED;
		1447
1422	kfd_debug_print_topology();	1448	kfd_debug_print_topology();
1423		1449
1424	if (!res)	1450	if (!res)

drivers/gpu/drm/amd/amdkfd/kfd_topology.h+6−4

 #define HSA_CAP_ASIC_REVISION_MASK		0x03c00000
 #define HSA_CAP_ASIC_REVISION_SHIFT		22
 #define HSA_CAP_SRAM_EDCSUPPORTED		0x04000000
+#define HSA_CAP_SVMAPI_SUPPORTED		0x08000000
-#define HSA_CAP_RESERVED			0xf80f8000
+#define HSA_CAP_RESERVED			0xf00f8000
 struct kfd_node_properties {
 	uint64_t hive_id;
 #define HSA_MEM_HEAP_TYPE_GPU_LDS	4
 #define HSA_MEM_HEAP_TYPE_GPU_SCRATCH	5
-#define HSA_MEM_FLAGS_HOT_PLUGGABLE	0x00000001
+#define HSA_MEM_FLAGS_HOT_PLUGGABLE		0x00000001
-#define HSA_MEM_FLAGS_NON_VOLATILE	0x00000002
+#define HSA_MEM_FLAGS_NON_VOLATILE		0x00000002
-#define HSA_MEM_FLAGS_RESERVED		0xfffffffc
+#define HSA_MEM_FLAGS_COHERENTHOSTACCESS	0x00000004
+#define HSA_MEM_FLAGS_RESERVED			0xfffffff8
 struct kfd_mem_properties {
 	struct list_head	list;

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c+188−71

@@ -35,6 +35,7 @@ · 34 unmodified lines
35	#include "dc/inc/hw/abm.h"	35	#include "dc/inc/hw/abm.h"
36	#include "dc/dc_dmub_srv.h"	36	#include "dc/dc_dmub_srv.h"
37	#include "dc/dc_edid_parser.h"	37	#include "dc/dc_edid_parser.h"
		38	#include "dc/dc_stat.h"
38	#include "amdgpu_dm_trace.h"	39	#include "amdgpu_dm_trace.h"
39		40
40	#include "vid.h"	41	#include "vid.h"
@@ -59,6 +60,7 @@ · 18 unmodified lines
59		60
60	#include "ivsrcid/ivsrcid_vislands30.h"	61	#include "ivsrcid/ivsrcid_vislands30.h"
61		62
		63	#include "i2caux_interface.h"
62	#include <linux/module.h>	64	#include <linux/module.h>
63	#include <linux/moduleparam.h>	65	#include <linux/moduleparam.h>
64	#include <linux/types.h>	66	#include <linux/types.h>
@@ -618,6 +620,58 @@ static void dm_dcn_vertical_interrupt0_high_irq(void *interrupt_params) · 553 unmodified lines
618	amdgpu_dm_crtc_handle_crc_window_irq(&acrtc->base);	620	amdgpu_dm_crtc_handle_crc_window_irq(&acrtc->base);
619	}	621	}
620	#endif	622	#endif
		623
		624	/**
		625	* dm_dmub_outbox1_low_irq() - Handles Outbox interrupt
		626	* @interrupt_params: used for determining the Outbox instance
		627	*
		628	* Handles the Outbox Interrupt
		629	* event handler.
		630	*/
		631	#define DMUB_TRACE_MAX_READ 64
		632	static void dm_dmub_outbox1_low_irq(void *interrupt_params)
		633	{
		634	struct dmub_notification notify;
		635	struct common_irq_params *irq_params = interrupt_params;
		636	struct amdgpu_device *adev = irq_params->adev;
		637	struct amdgpu_display_manager *dm = &adev->dm;
		638	struct dmcub_trace_buf_entry entry = { 0 };
		639	uint32_t count = 0;
		640
		641	if (dc_enable_dmub_notifications(adev->dm.dc)) {
		642	if (irq_params->irq_src == DC_IRQ_SOURCE_DMCUB_OUTBOX) {
		643	do {
		644	dc_stat_get_dmub_notification(adev->dm.dc, &notify);
		645	} while (notify.pending_notification);
		646
		647	if (adev->dm.dmub_notify)
		648	memcpy(adev->dm.dmub_notify, &notify, sizeof(struct dmub_notification));
		649	if (notify.type == DMUB_NOTIFICATION_AUX_REPLY)
		650	complete(&adev->dm.dmub_aux_transfer_done);
		651	// TODO : HPD Implementation
		652
		653	} else {
		654	DRM_ERROR("DM: Failed to receive correct outbox IRQ !");
		655	}
		656	}
		657
		658
		659	do {
		660	if (dc_dmub_srv_get_dmub_outbox0_msg(dm->dc, &entry)) {
		661	trace_amdgpu_dmub_trace_high_irq(entry.trace_code, entry.tick_count,
		662	entry.param0, entry.param1);
		663
		664	DRM_DEBUG_DRIVER("trace_code:%u, tick_count:%u, param0:%u, param1:%u\n",
		665	entry.trace_code, entry.tick_count, entry.param0, entry.param1);
		666	} else
		667	break;
		668
		669	count++;
		670
		671	} while (count <= DMUB_TRACE_MAX_READ);
		672
		673	ASSERT(count <= DMUB_TRACE_MAX_READ);
		674	}
621	#endif	675	#endif
622		676
623	static int dm_set_clockgating_state(void *handle,	677	static int dm_set_clockgating_state(void *handle,
@@ -938,32 +992,6 @@ static int dm_dmub_hw_init(struct amdgpu_device *adev) · 314 unmodified lines
938	}	992	}
939		993
940	#if defined(CONFIG_DRM_AMD_DC_DCN)	994	#if defined(CONFIG_DRM_AMD_DC_DCN)
941	#define DMUB_TRACE_MAX_READ 64
942	static void dm_dmub_trace_high_irq(void *interrupt_params)
943	{
944	struct common_irq_params *irq_params = interrupt_params;
945	struct amdgpu_device *adev = irq_params->adev;
946	struct amdgpu_display_manager *dm = &adev->dm;
947	struct dmcub_trace_buf_entry entry = { 0 };
948	uint32_t count = 0;
949
950	do {
951	if (dc_dmub_srv_get_dmub_outbox0_msg(dm->dc, &entry)) {
952	trace_amdgpu_dmub_trace_high_irq(entry.trace_code, entry.tick_count,
953	entry.param0, entry.param1);
954
955	DRM_DEBUG_DRIVER("trace_code:%u, tick_count:%u, param0:%u, param1:%u\n",
956	entry.trace_code, entry.tick_count, entry.param0, entry.param1);
957	} else
958	break;
959
960	count++;
961
962	} while (count <= DMUB_TRACE_MAX_READ);
963
964	ASSERT(count <= DMUB_TRACE_MAX_READ);
965	}
966
967	static void mmhub_read_system_context(struct amdgpu_device adev, struct dc_phy_addr_space_config pa_config)	995	static void mmhub_read_system_context(struct amdgpu_device adev, struct dc_phy_addr_space_config pa_config)
968	{	996	{
969	uint64_t pt_base;	997	uint64_t pt_base;
@@ -1220,6 +1248,16 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) · 250 unmodified lines
1220	#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)	1248	#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
1221	adev->dm.crc_rd_wrk = amdgpu_dm_crtc_secure_display_create_work();	1249	adev->dm.crc_rd_wrk = amdgpu_dm_crtc_secure_display_create_work();
1222	#endif	1250	#endif
		1251	if (dc_enable_dmub_notifications(adev->dm.dc)) {
		1252	init_completion(&adev->dm.dmub_aux_transfer_done);
		1253	adev->dm.dmub_notify = kzalloc(sizeof(struct dmub_notification), GFP_KERNEL);
		1254	if (!adev->dm.dmub_notify) {
		1255	DRM_INFO("amdgpu: fail to allocate adev->dm.dmub_notify");
		1256	goto error;
		1257	}
		1258	amdgpu_dm_outbox_init(adev);
		1259	}
		1260
1223	if (amdgpu_dm_initialize_drm_device(adev)) {	1261	if (amdgpu_dm_initialize_drm_device(adev)) {
1224	DRM_ERROR(	1262	DRM_ERROR(
1225	"amdgpu: failed to initialize sw for display support.\n");	1263	"amdgpu: failed to initialize sw for display support.\n");
@@ -1293,6 +1331,11 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev) · 67 unmodified lines
1293	adev->dm.dc->ctx->dmub_srv = NULL;	1331	adev->dm.dc->ctx->dmub_srv = NULL;
1294	}	1332	}
1295		1333
		1334	if (dc_enable_dmub_notifications(adev->dm.dc)) {
		1335	kfree(adev->dm.dmub_notify);
		1336	adev->dm.dmub_notify = NULL;
		1337	}
		1338
1296	if (adev->dm.dmub_bo)	1339	if (adev->dm.dmub_bo)
1297	amdgpu_bo_free_kernel(&adev->dm.dmub_bo,	1340	amdgpu_bo_free_kernel(&adev->dm.dmub_bo,
1298	&adev->dm.dmub_bo_gpu_addr,	1341	&adev->dm.dmub_bo_gpu_addr,
@@ -2708,8 +2751,7 @@ static void handle_hpd_rx_irq(void *param) · 1409 unmodified lines
2708	* conflict, after implement i2c helper, this mutex should be	2751	* conflict, after implement i2c helper, this mutex should be
2709	* retired.	2752	* retired.
2710	*/	2753	*/
2711	if (dc_link->type != dc_connection_mst_branch)	2754	mutex_lock(&aconnector->hpd_lock);
2712	mutex_lock(&aconnector->hpd_lock);
2713		2755
2714	read_hpd_rx_irq_data(dc_link, &hpd_irq_data);	2756	read_hpd_rx_irq_data(dc_link, &hpd_irq_data);
2715		2757
@@ -2726,13 +2768,15 @@ static void handle_hpd_rx_irq(void *param) · 10 unmodified lines
2726	}	2768	}
2727	}	2769	}
2728		2770
2729	mutex_lock(&adev->dm.dc_lock);	2771	if (!amdgpu_in_reset(adev)) {
		2772	mutex_lock(&adev->dm.dc_lock);
2730	#ifdef CONFIG_DRM_AMD_DC_HDCP	2773	#ifdef CONFIG_DRM_AMD_DC_HDCP
2731	result = dc_link_handle_hpd_rx_irq(dc_link, &hpd_irq_data, NULL);	2774	result = dc_link_handle_hpd_rx_irq(dc_link, &hpd_irq_data, NULL);
2732	#else	2775	#else
2733	result = dc_link_handle_hpd_rx_irq(dc_link, NULL, NULL);	2776	result = dc_link_handle_hpd_rx_irq(dc_link, NULL, NULL);
2734	#endif	2777	#endif
2735	mutex_unlock(&adev->dm.dc_lock);	2778	mutex_unlock(&adev->dm.dc_lock);
		2779	}
2736		2780
2737	out:	2781	out:
2738	if (result && !is_mst_root_connector) {	2782	if (result && !is_mst_root_connector) {
@@ -2776,10 +2820,10 @@ out: · 37 unmodified lines
2776	}	2820	}
2777	#endif	2821	#endif
2778		2822
2779	if (dc_link->type != dc_connection_mst_branch) {	2823	if (dc_link->type != dc_connection_mst_branch)
2780	drm_dp_cec_irq(&aconnector->dm_dp_aux.aux);	2824	drm_dp_cec_irq(&aconnector->dm_dp_aux.aux);
2781	mutex_unlock(&aconnector->hpd_lock);	2825
2782	}	2826	mutex_unlock(&aconnector->hpd_lock);
2783	}	2827	}
2784		2828
2785	static void register_hpd_handlers(struct amdgpu_device *adev)	2829	static void register_hpd_handlers(struct amdgpu_device *adev)
@@ -3151,38 +3195,51 @@ static int dcn10_register_irq_handlers(struct amdgpu_device *adev) · 365 unmodified lines
3151		3195
3152	}	3196	}
3153		3197
3154	if (dc->ctx->dmub_srv) {	3198	/* HPD */
3155	i = DCN_1_0__SRCID__DMCUB_OUTBOX_HIGH_PRIORITY_READY_INT;	3199	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT,
3156	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->dmub_trace_irq);	3200	&adev->hpd_irq);
		3201	if (r) {
		3202	DRM_ERROR("Failed to add hpd irq id!\n");
		3203	return r;
		3204	}
3157		3205
3158	if (r) {	3206	register_hpd_handlers(adev);
3159	DRM_ERROR("Failed to add dmub trace irq id!\n");
3160	return r;
3161	}
3162		3207
3163	int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;	3208	return 0;
		3209	}
		3210	/* Register Outbox IRQ sources and initialize IRQ callbacks */
		3211	static int register_outbox_irq_handlers(struct amdgpu_device *adev)
		3212	{
		3213	struct dc *dc = adev->dm.dc;
		3214	struct common_irq_params *c_irq_params;
		3215	struct dc_interrupt_params int_params = {0};
		3216	int r, i;
		3217
		3218	int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
		3219	int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
		3220
		3221	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT,
		3222	&adev->dmub_outbox_irq);
		3223	if (r) {
		3224	DRM_ERROR("Failed to add outbox irq id!\n");
		3225	return r;
		3226	}
		3227
		3228	if (dc->ctx->dmub_srv) {
		3229	i = DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT;
		3230	int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
3164	int_params.irq_source =	3231	int_params.irq_source =
3165	dc_interrupt_to_irq_source(dc, i, 0);	3232	dc_interrupt_to_irq_source(dc, i, 0);
3166		3233
3167	c_irq_params = &adev->dm.dmub_trace_params[0];	3234	c_irq_params = &adev->dm.dmub_outbox_params[0];
3168		3235
3169	c_irq_params->adev = adev;	3236	c_irq_params->adev = adev;
3170	c_irq_params->irq_src = int_params.irq_source;	3237	c_irq_params->irq_src = int_params.irq_source;
3171		3238
3172	amdgpu_dm_irq_register_interrupt(adev, &int_params,	3239	amdgpu_dm_irq_register_interrupt(adev, &int_params,
3173	dm_dmub_trace_high_irq, c_irq_params);	3240	dm_dmub_outbox1_low_irq, c_irq_params);
3174	}
3175
3176	/* HPD */
3177	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT,
3178	&adev->hpd_irq);
3179	if (r) {
3180	DRM_ERROR("Failed to add hpd irq id!\n");
3181	return r;
3182	}	3241	}
3183		3242
3184	register_hpd_handlers(adev);
3185
3186	return 0;	3243	return 0;
3187	}	3244	}
3188	#endif	3245	#endif
@@ -3414,22 +3471,37 @@ static int amdgpu_dm_backlight_update_status(struct backlight_device *bd) · 225 unmodified lines
3414	{	3471	{
3415	struct amdgpu_display_manager *dm = bl_get_data(bd);	3472	struct amdgpu_display_manager *dm = bl_get_data(bd);
3416	struct amdgpu_dm_backlight_caps caps;	3473	struct amdgpu_dm_backlight_caps caps;
3417	struct dc_link *link = NULL;	3474	struct dc_link *link[AMDGPU_DM_MAX_NUM_EDP];
3418	u32 brightness;	3475	u32 brightness;
3419	bool rc;	3476	bool rc;
		3477	int i;
3420		3478
3421	amdgpu_dm_update_backlight_caps(dm);	3479	amdgpu_dm_update_backlight_caps(dm);
3422	caps = dm->backlight_caps;	3480	caps = dm->backlight_caps;
3423		3481
3424	link = (struct dc_link *)dm->backlight_link;	3482	for (i = 0; i < dm->num_of_edps; i++)
		3483	link[i] = (struct dc_link *)dm->backlight_link[i];
3425		3484
3426	brightness = convert_brightness_from_user(&caps, bd->props.brightness);	3485	brightness = convert_brightness_from_user(&caps, bd->props.brightness);
3427	// Change brightness based on AUX property	3486	// Change brightness based on AUX property
3428	if (caps.aux_support)	3487	if (caps.aux_support) {
3429	rc = dc_link_set_backlight_level_nits(link, true, brightness,	3488	for (i = 0; i < dm->num_of_edps; i++) {
3430	AUX_BL_DEFAULT_TRANSITION_TIME_MS);	3489	rc = dc_link_set_backlight_level_nits(link[i], true, brightness,
3431	else	3490	AUX_BL_DEFAULT_TRANSITION_TIME_MS);
3432	rc = dc_link_set_backlight_level(dm->backlight_link, brightness, 0);	3491	if (!rc) {
		3492	DRM_ERROR("DM: Failed to update backlight via AUX on eDP[%d]\n", i);
		3493	break;
		3494	}
		3495	}
		3496	} else {
		3497	for (i = 0; i < dm->num_of_edps; i++) {
		3498	rc = dc_link_set_backlight_level(dm->backlight_link[i], brightness, 0);
		3499	if (!rc) {
		3500	DRM_ERROR("DM: Failed to update backlight on eDP[%d]\n", i);
		3501	break;
		3502	}
		3503	}
		3504	}
3433		3505
3434	return rc ? 0 : 1;	3506	return rc ? 0 : 1;
3435	}	3507	}
@@ -3443,7 +3515,7 @@ static int amdgpu_dm_backlight_get_brightness(struct backlight_device *bd) · 7 unmodified lines
3443	caps = dm->backlight_caps;	3515	caps = dm->backlight_caps;
3444		3516
3445	if (caps.aux_support) {	3517	if (caps.aux_support) {
3446	struct dc_link link = (struct dc_link )dm->backlight_link;	3518	struct dc_link link = (struct dc_link )dm->backlight_link[0];
3447	u32 avg, peak;	3519	u32 avg, peak;
3448	bool rc;	3520	bool rc;
3449		3521
@@ -3452,7 +3524,7 @@ static int amdgpu_dm_backlight_get_brightness(struct backlight_device *bd) · 2 unmodified lines
3452	return bd->props.brightness;	3524	return bd->props.brightness;
3453	return convert_brightness_to_user(&caps, avg);	3525	return convert_brightness_to_user(&caps, avg);
3454	} else {	3526	} else {
3455	int ret = dc_link_get_backlight_level(dm->backlight_link);	3527	int ret = dc_link_get_backlight_level(dm->backlight_link[0]);
3456		3528
3457	if (ret == DC_ERROR_UNEXPECTED)	3529	if (ret == DC_ERROR_UNEXPECTED)
3458	return bd->props.brightness;	3530	return bd->props.brightness;
@@ -3549,10 +3621,13 @@ static void register_backlight_device(struct amdgpu_display_manager *dm, · 90 unmodified lines
3549	* DM initialization because not having a backlight control	3621	* DM initialization because not having a backlight control
3550	* is better then a black screen.	3622	* is better then a black screen.
3551	*/	3623	*/
3552	amdgpu_dm_register_backlight_device(dm);	3624	if (!dm->backlight_dev)
		3625	amdgpu_dm_register_backlight_device(dm);
3553		3626
3554	if (dm->backlight_dev)	3627	if (dm->backlight_dev) {
3555	dm->backlight_link = link;	3628	dm->backlight_link[dm->num_of_edps] = link;
		3629	dm->num_of_edps++;
		3630	}
3556	}	3631	}
3557	#endif	3632	#endif
3558	}	3633	}
@@ -3643,6 +3718,22 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev) · 84 unmodified lines
3643	goto fail;	3718	goto fail;
3644	}	3719	}
3645		3720
		3721	#if defined(CONFIG_DRM_AMD_DC_DCN)
		3722	/* Use Outbox interrupt */
		3723	switch (adev->asic_type) {
		3724	case CHIP_SIENNA_CICHLID:
		3725	case CHIP_NAVY_FLOUNDER:
		3726	case CHIP_RENOIR:
		3727	if (register_outbox_irq_handlers(dm->adev)) {
		3728	DRM_ERROR("DM: Failed to initialize IRQ\n");
		3729	goto fail;
		3730	}
		3731	break;
		3732	default:
		3733	DRM_DEBUG_KMS("Unsupported ASIC type for outbox: 0x%X\n", adev->asic_type);
		3734	}
		3735	#endif
		3736
3646	/* loops over all connectors on the board */	3737	/* loops over all connectors on the board */
3647	for (i = 0; i < link_cnt; i++) {	3738	for (i = 0; i < link_cnt; i++) {
3648	struct dc_link *link = NULL;	3739	struct dc_link *link = NULL;
@@ -6560,13 +6651,13 @@ static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_state *state, · 2911 unmodified lines
6560	{	6651	{
6561	struct dc_stream_state *stream = NULL;	6652	struct dc_stream_state *stream = NULL;
6562	struct drm_connector *connector;	6653	struct drm_connector *connector;
6563	struct drm_connector_state new_con_state, old_con_state;	6654	struct drm_connector_state *new_con_state;
6564	struct amdgpu_dm_connector *aconnector;	6655	struct amdgpu_dm_connector *aconnector;
6565	struct dm_connector_state *dm_conn_state;	6656	struct dm_connector_state *dm_conn_state;
6566	int i, j, clock, bpp;	6657	int i, j, clock, bpp;
6567	int vcpi, pbn_div, pbn = 0;	6658	int vcpi, pbn_div, pbn = 0;
6568		6659
6569	for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) {	6660	for_each_new_connector_in_state(state, connector, new_con_state, i) {
6570		6661
6571	aconnector = to_amdgpu_dm_connector(connector);	6662	aconnector = to_amdgpu_dm_connector(connector);
6572		6663
@@ -8164,15 +8255,14 @@ static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state, · 1591 unmodified lines
8164	static void amdgpu_dm_commit_cursors(struct drm_atomic_state *state)	8255	static void amdgpu_dm_commit_cursors(struct drm_atomic_state *state)
8165	{	8256	{
8166	struct drm_plane *plane;	8257	struct drm_plane *plane;
8167	struct drm_plane_state old_plane_state, new_plane_state;	8258	struct drm_plane_state *old_plane_state;
8168	int i;	8259	int i;
8169		8260
8170	/*	8261	/*
8171	* TODO: Make this per-stream so we don't issue redundant updates for	8262	* TODO: Make this per-stream so we don't issue redundant updates for
8172	* commits with multiple streams.	8263	* commits with multiple streams.
8173	*/	8264	*/
8174	for_each_oldnew_plane_in_state(state, plane, old_plane_state,	8265	for_each_old_plane_in_state(state, plane, old_plane_state, i)
8175	new_plane_state, i)
8176	if (plane->type == DRM_PLANE_TYPE_CURSOR)	8266	if (plane->type == DRM_PLANE_TYPE_CURSOR)
8177	handle_cursor_update(plane, old_plane_state);	8267	handle_cursor_update(plane, old_plane_state);
8178	}	8268	}
@@ -10668,3 +10758,30 @@ uint32_t dm_read_reg_func(const struct dc_context *ctx, uint32_t address, · 2489 unmodified lines
10668		10758
10669	return value;	10759	return value;
10670	}	10760	}
		10761
		10762	int amdgpu_dm_process_dmub_aux_transfer_sync(struct dc_context *ctx, unsigned int linkIndex,
		10763	struct aux_payload payload, enum aux_return_code_type operation_result)
		10764	{
		10765	struct amdgpu_device *adev = ctx->driver_context;
		10766	int ret = 0;
		10767
		10768	dc_process_dmub_aux_transfer_async(ctx->dc, linkIndex, payload);
		10769	ret = wait_for_completion_interruptible_timeout(&adev->dm.dmub_aux_transfer_done, 10*HZ);
		10770	if (ret == 0) {
		10771	*operation_result = AUX_RET_ERROR_TIMEOUT;
		10772	return -1;
		10773	}
		10774	*operation_result = (enum aux_return_code_type)adev->dm.dmub_notify->result;
		10775
		10776	if (adev->dm.dmub_notify->result == AUX_RET_SUCCESS) {
		10777	(*payload->reply) = adev->dm.dmub_notify->aux_reply.command;
		10778
		10779	// For read case, Copy data to payload
		10780	if (!payload->write && adev->dm.dmub_notify->aux_reply.length &&
		10781	(*payload->reply == AUX_TRANSACTION_REPLY_AUX_ACK))
		10782	memcpy(payload->data, adev->dm.dmub_notify->aux_reply.data,
		10783	adev->dm.dmub_notify->aux_reply.length);
		10784	}
		10785
		10786	return adev->dm.dmub_notify->aux_reply.length;
		10787	}

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h+21−1

 #define AMDGPU_DM_MAX_CRTC 6
+#define AMDGPU_DM_MAX_NUM_EDP 2
 /*
 #include "include/amdgpu_dal_power_if.h"
 #include "amdgpu_dm_irq.h"
 #include "irq_types.h"
 #include "signal_types.h"
 #include "amdgpu_dm_crc.h"
+struct aux_payload;
+enum aux_return_code_type;
 /* Forward declarations */
 struct amdgpu_device;
 struct amdgpu_bo;
 struct dmub_srv;
 struct dc_plane_state;
+struct dmub_notification;
 struct common_irq_params {
 	struct amdgpu_device *adev;
 /**
  * struct dal_allocation - Tracks mapped FB memory for SMU communication
+ * @list: list of dal allocations
+ * @bo: GPU buffer object
+ * @cpu_ptr: CPU virtual address of the GPU buffer object
+ * @gpu_addr: GPU virtual address of the GPU buffer object
  */
 struct dal_allocation {
 	struct list_head list;
  * @compressor: Frame buffer compression buffer. See &struct dm_compressor_info
  * @force_timing_sync: set via debugfs. When set, indicates that all connected
  *		       displays will be forced to synchronize.
+ * @dmcub_trace_event_en: enable dmcub trace events
  */
 struct amdgpu_display_manager {
 	 */
 	struct dmub_srv *dmub_srv;
+	struct dmub_notification *dmub_notify;
 	/**
 	 * @dmub_fb_info:
 	 *
 	struct common_irq_params
 	dmub_trace_params[1];
+	struct common_irq_params
+	dmub_outbox_params[1];
 	spinlock_t irq_handler_list_table_lock;
 	struct backlight_device *backlight_dev;
-	const struct dc_link *backlight_link;
+	const struct dc_link *backlight_link[AMDGPU_DM_MAX_NUM_EDP];
+	uint8_t num_of_edps;
 	struct amdgpu_dm_backlight_caps backlight_caps;
 	struct mod_freesync *freesync_module;
 	 * DAL fb memory allocation list, for communication with SMU.
 	 */
 	struct list_head da_list;
+	struct completion dmub_aux_transfer_done;
 };
 enum dsc_clock_force_state {
 extern const struct drm_encoder_helper_funcs amdgpu_dm_encoder_helper_funcs;
+int amdgpu_dm_process_dmub_aux_transfer_sync(struct dc_context *ctx, unsigned int linkIndex,
+					struct aux_payload *payload, enum aux_return_code_type *operation_result);
 #endif /* __AMDGPU_DM_H__ */

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c+66−40

 	return 0;
 }
 #endif
+/*
+ * Returns whether the connected display is internal and not hotpluggable.
+ * Example usage: cat /sys/kernel/debug/dri/0/DP-1/internal_display
+ */
+static int internal_display_show(struct seq_file *m, void *data)
+{
+	struct drm_connector *connector = m->private;
+	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
+	struct dc_link *link = aconnector->dc_link;
+	seq_printf(m, "Internal: %u\n", link->is_internal_display);
+	return 0;
+}
 /* function description
  *
  * generic SDP message access for testing
 	return size;
 }
+/*
+ * Backlight at this moment.  Read only.
+ * As written to display, taking ABM and backlight lut into account.
+ * Ranges from 0x0 to 0x10000 (= 100% PWM)
+ *
+ * Example usage: cat /sys/kernel/debug/dri/0/eDP-1/current_backlight
+ */
+static int current_backlight_show(struct seq_file *m, void *unused)
+{
+	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(m->private);
+	struct dc_link *link = aconnector->dc_link;
+	unsigned int backlight;
+	backlight = dc_link_get_backlight_level(link);
+	seq_printf(m, "0x%x\n", backlight);
+	return 0;
+}
+/*
+ * Backlight value that is being approached.  Read only.
+ * As written to display, taking ABM and backlight lut into account.
+ * Ranges from 0x0 to 0x10000 (= 100% PWM)
+ *
+ * Example usage: cat /sys/kernel/debug/dri/0/eDP-1/target_backlight
+ */
+static int target_backlight_show(struct seq_file *m, void *unused)
+{
+	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(m->private);
+	struct dc_link *link = aconnector->dc_link;
+	unsigned int backlight;
+	backlight = dc_link_get_target_backlight_pwm(link);
+	seq_printf(m, "0x%x\n", backlight);
+	return 0;
+}
 DEFINE_SHOW_ATTRIBUTE(dp_dsc_fec_support);
 DEFINE_SHOW_ATTRIBUTE(dmub_fw_state);
 DEFINE_SHOW_ATTRIBUTE(dmub_tracebuffer);
 #ifdef CONFIG_DRM_AMD_DC_HDCP
 DEFINE_SHOW_ATTRIBUTE(hdcp_sink_capability);
 #endif
+DEFINE_SHOW_ATTRIBUTE(internal_display);
 static const struct file_operations dp_dsc_clock_en_debugfs_fops = {
 	.owner = THIS_MODULE,
 DEFINE_DEBUGFS_ATTRIBUTE(psr_fops, psr_get, NULL, "%llu\n");
+DEFINE_SHOW_ATTRIBUTE(current_backlight);
+DEFINE_SHOW_ATTRIBUTE(target_backlight);
 static const struct {
 	char *name;
 	const struct file_operations *fops;
 } connector_debugfs_entries[] = {
 		{"force_yuv420_output", &force_yuv420_output_fops},
 		{"output_bpc", &output_bpc_fops},
-		{"trigger_hotplug", &trigger_hotplug_debugfs_fops}
+		{"trigger_hotplug", &trigger_hotplug_debugfs_fops},
+		{"internal_display", &internal_display_fops}
 };
 void connector_debugfs_init(struct amdgpu_dm_connector *connector)
 					    dp_debugfs_entries[i].fops);
 		}
 	}
-	if (connector->base.connector_type == DRM_MODE_CONNECTOR_eDP)
+	if (connector->base.connector_type == DRM_MODE_CONNECTOR_eDP) {
 		debugfs_create_file_unsafe("psr_state", 0444, dir, connector, &psr_fops);
+		debugfs_create_file("amdgpu_current_backlight_pwm", 0444, dir, connector,
+				    &current_backlight_fops);
+		debugfs_create_file("amdgpu_target_backlight_pwm", 0444, dir, connector,
+				    &target_backlight_fops);
+	}
 	for (i = 0; i < ARRAY_SIZE(connector_debugfs_entries); i++) {
 		debugfs_create_file(connector_debugfs_entries[i].name,
 	return size;
 }
-/*
- * Backlight at this moment.  Read only.
- * As written to display, taking ABM and backlight lut into account.
- * Ranges from 0x0 to 0x10000 (= 100% PWM)
- */
-static int current_backlight_show(struct seq_file *m, void *unused)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
-	struct amdgpu_display_manager *dm = &adev->dm;
-	unsigned int backlight = dc_link_get_backlight_level(dm->backlight_link);
-	seq_printf(m, "0x%x\n", backlight);
-	return 0;
-}
-/*
- * Backlight value that is being approached.  Read only.
- * As written to display, taking ABM and backlight lut into account.
- * Ranges from 0x0 to 0x10000 (= 100% PWM)
- */
-static int target_backlight_show(struct seq_file *m, void *unused)
-{
-	struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
-	struct amdgpu_display_manager *dm = &adev->dm;
-	unsigned int backlight = dc_link_get_target_backlight_pwm(dm->backlight_link);
-	seq_printf(m, "0x%x\n", backlight);
-	return 0;
-}
 static int mst_topo_show(struct seq_file *m, void *unused)
 {
 	struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
 	return 0;
 }
-DEFINE_SHOW_ATTRIBUTE(current_backlight);
-DEFINE_SHOW_ATTRIBUTE(target_backlight);
 DEFINE_SHOW_ATTRIBUTE(mst_topo);
 DEFINE_DEBUGFS_ATTRIBUTE(visual_confirm_fops, visual_confirm_get,
 			 visual_confirm_set, "%llu\n");
 	struct drm_minor *minor = adev_to_drm(adev)->primary;
 	struct dentry *root = minor->debugfs_root;
-	debugfs_create_file("amdgpu_current_backlight_pwm", 0444,
-			    root, adev, &current_backlight_fops);
-	debugfs_create_file("amdgpu_target_backlight_pwm", 0444,
-			    root, adev, &target_backlight_fops);
 	debugfs_create_file("amdgpu_mst_topology", 0444, root,
 			    adev, &mst_topo_fops);
 	debugfs_create_file("amdgpu_dm_dtn_log", 0644, root, adev,

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c+14−5

@@ -544,8 +544,10 @@ bool dm_helpers_dp_write_dsc_enable( · 543 unmodified lines
544	ret = drm_dp_dpcd_write(aconnector->dsc_aux, DP_DSC_ENABLE, &enable_dsc, 1);	544	ret = drm_dp_dpcd_write(aconnector->dsc_aux, DP_DSC_ENABLE, &enable_dsc, 1);
545	}	545	}
546		546
547	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT)	547	if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT) {
548	return dm_helpers_dp_write_dpcd(ctx, stream->link, DP_DSC_ENABLE, &enable_dsc, 1);	548	ret = dm_helpers_dp_write_dpcd(ctx, stream->link, DP_DSC_ENABLE, &enable_dsc, 1);
		549	DC_LOG_DC("Send DSC %s to sst display\n", enable_dsc ? "enable" : "disable");
		550	}
549		551
550	return (ret > 0);	552	return (ret > 0);
551	}	553	}
@@ -640,7 +642,14 @@ enum dc_edid_status dm_helpers_read_local_edid( · 88 unmodified lines
640		642
641	return edid_status;	643	return edid_status;
642	}	644	}
643		645	int dm_helper_dmub_aux_transfer_sync(
		646	struct dc_context *ctx,
		647	const struct dc_link *link,
		648	struct aux_payload *payload,
		649	enum aux_return_code_type *operation_result)
		650	{
		651	return amdgpu_dm_process_dmub_aux_transfer_sync(ctx, link->link_index, payload, operation_result);
		652	}
644	void dm_set_dcn_clocks(struct dc_context ctx, struct dc_clocks clks)	653	void dm_set_dcn_clocks(struct dc_context ctx, struct dc_clocks clks)
645	{	654	{
646	/* TODO: something */	655	/* TODO: something */
@@ -698,12 +707,12 @@ void dm_helpers_free_gpu_mem( · 51 unmodified lines
698	}	707	}
699	}	708	}
700		709
701	bool dm_helpers_dmub_outbox0_interrupt_control(struct dc_context *ctx, bool enable)	710	bool dm_helpers_dmub_outbox_interrupt_control(struct dc_context *ctx, bool enable)
702	{	711	{
703	enum dc_irq_source irq_source;	712	enum dc_irq_source irq_source;
704	bool ret;	713	bool ret;
705		714
706	irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX0;	715	irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX;
707		716
708	ret = dc_interrupt_set(ctx->dc, irq_source, enable);	717	ret = dc_interrupt_set(ctx->dc, irq_source, enable);
709		718

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c+26−1

 		__func__);
 }
+static int amdgpu_dm_set_dmub_outbox_irq_state(struct amdgpu_device *adev,
+					struct amdgpu_irq_src *source,
+					unsigned int crtc_id,
+					enum amdgpu_interrupt_state state)
+{
+	enum dc_irq_source irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX;
+	bool st = (state == AMDGPU_IRQ_STATE_ENABLE);
+	dc_interrupt_set(adev->dm.dc, irq_source, st);
+	return 0;
+}
 static int amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device *adev,
 					   struct amdgpu_irq_src *source,
 					   unsigned int crtc_id,
 	.process = amdgpu_dm_irq_handler,
 };
+static const struct amdgpu_irq_src_funcs dm_dmub_outbox_irq_funcs = {
+	.set = amdgpu_dm_set_dmub_outbox_irq_state,
+	.process = amdgpu_dm_irq_handler,
+};
 static const struct amdgpu_irq_src_funcs dm_vupdate_irq_funcs = {
 	.set = amdgpu_dm_set_vupdate_irq_state,
 	.process = amdgpu_dm_irq_handler,
 void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev)
 {
 	adev->crtc_irq.num_types = adev->mode_info.num_crtc;
 	adev->crtc_irq.funcs = &dm_crtc_irq_funcs;
 	adev->vline0_irq.num_types = adev->mode_info.num_crtc;
 	adev->vline0_irq.funcs = &dm_vline0_irq_funcs;
+	adev->dmub_outbox_irq.num_types = 1;
+	adev->dmub_outbox_irq.funcs = &dm_dmub_outbox_irq_funcs;
 	adev->vupdate_irq.num_types = adev->mode_info.num_crtc;
 	adev->vupdate_irq.funcs = &dm_vupdate_irq_funcs;
 	adev->hpd_irq.num_types = adev->mode_info.num_hpd;
 	adev->hpd_irq.funcs = &dm_hpd_irq_funcs;
 }
+void amdgpu_dm_outbox_init(struct amdgpu_device *adev)
+{
+	dc_interrupt_set(adev->dm.dc,
+		DC_IRQ_SOURCE_DMCUB_OUTBOX,
+		true);
+}
 /**
  * amdgpu_dm_hpd_init - hpd setup callback.

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h+1−0

@@ -82,6 +82,7 @@ void amdgpu_dm_irq_unregister_interrupt(struct amdgpu_device *adev, · 81 unmodified lines
82		82
83	void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev);	83	void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev);
84		84
		85	void amdgpu_dm_outbox_init(struct amdgpu_device *adev);
85	void amdgpu_dm_hpd_init(struct amdgpu_device *adev);	86	void amdgpu_dm_hpd_init(struct amdgpu_device *adev);
86	void amdgpu_dm_hpd_fini(struct amdgpu_device *adev);	87	void amdgpu_dm_hpd_fini(struct amdgpu_device *adev);
87		88

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c+3−0

@@ -278,6 +278,9 @@ dm_dp_mst_detect(struct drm_connector *connector, · 277 unmodified lines
278	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);	278	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
279	struct amdgpu_dm_connector *master = aconnector->mst_port;	279	struct amdgpu_dm_connector *master = aconnector->mst_port;
280		280
		281	if (drm_connector_is_unregistered(connector))
		282	return connector_status_disconnected;
		283
281	return drm_dp_mst_detect_port(connector, ctx, &master->mst_mgr,	284	return drm_dp_mst_detect_port(connector, ctx, &master->mst_mgr,
282	aconnector->port);	285	aconnector->port);
283	}	286	}

drivers/gpu/drm/amd/display/dc/Makefile+1−1

@@ -54,7 +54,7 @@ AMD_DC = $(addsuffix /Makefile, $(addprefix $(FULL_AMD_DISPLAY_PATH)/dc/,$(DC_LI · 53 unmodified lines
54		54
55	include $(AMD_DC)	55	include $(AMD_DC)
56		56
57	DISPLAY_CORE = dc.o dc_link.o dc_resource.o dc_hw_sequencer.o dc_sink.o \	57	DISPLAY_CORE = dc.o dc_stat.o dc_link.o dc_resource.o dc_hw_sequencer.o dc_sink.o \
58	dc_surface.o dc_link_hwss.o dc_link_dp.o dc_link_ddc.o dc_debug.o dc_stream.o \	58	dc_surface.o dc_link_hwss.o dc_link_dp.o dc_link_ddc.o dc_debug.o dc_stream.o \
59	dc_link_enc_cfg.o	59	dc_link_enc_cfg.o
60		60

drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c+2−0

 			return get_ss_info_v4_1(bp, signal, index, ss_info);
 		case 2:
 		case 3:
+		case 4:
 			return get_ss_info_v4_2(bp, signal, index, ss_info);
 		default:
+			ASSERT(0);
 			break;
 		}
 		break;

drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c+59−85

@@ -106,10 +106,10 @@ static void rn_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr, · 105 unmodified lines
106	for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {	106	for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {
107	int dpp_inst, dppclk_khz, prev_dppclk_khz;	107	int dpp_inst, dppclk_khz, prev_dppclk_khz;
108		108
109	/* Loop index will match dpp->inst if resource exists,	109	/* Loop index may not match dpp->inst if some pipes disabled,
110	* and we want to avoid dependency on dpp object	110	* so select correct inst from res_pool
111	*/	111	*/
112	dpp_inst = i;	112	dpp_inst = clk_mgr->base.ctx->dc->res_pool->dpps[i]->inst;
113	dppclk_khz = context->res_ctx.pipe_ctx[i].plane_res.bw.dppclk_khz;	113	dppclk_khz = context->res_ctx.pipe_ctx[i].plane_res.bw.dppclk_khz;
114		114
115	prev_dppclk_khz = clk_mgr->dccg->pipe_dppclk_khz[i];	115	prev_dppclk_khz = clk_mgr->dccg->pipe_dppclk_khz[i];
@@ -128,7 +128,7 @@ void rn_update_clocks(struct clk_mgr *clk_mgr_base, · 12 unmodified lines
128	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);	128	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
129	struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk;	129	struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk;
130	struct dc *dc = clk_mgr_base->ctx->dc;	130	struct dc *dc = clk_mgr_base->ctx->dc;
131	int display_count, i;	131	int display_count;
132	bool update_dppclk = false;	132	bool update_dppclk = false;
133	bool update_dispclk = false;	133	bool update_dispclk = false;
134	bool dpp_clock_lowered = false;	134	bool dpp_clock_lowered = false;
@@ -210,14 +210,6 @@ void rn_update_clocks(struct clk_mgr *clk_mgr_base, · 75 unmodified lines
210	clk_mgr_base->clks.dppclk_khz,	210	clk_mgr_base->clks.dppclk_khz,
211	safe_to_lower);	211	safe_to_lower);
212		212
213	for (i = 0; i < context->stream_count; i++) {
214	if (context->streams[i]->signal == SIGNAL_TYPE_EDP &&
215	context->streams[i]->apply_seamless_boot_optimization) {
216	dc_wait_for_vblank(dc, context->streams[i]);
217	break;
218	}
219	}
220
221	clk_mgr_base->clks.actual_dppclk_khz =	213	clk_mgr_base->clks.actual_dppclk_khz =
222	rn_vbios_smu_set_dppclk(clk_mgr, clk_mgr_base->clks.dppclk_khz);	214	rn_vbios_smu_set_dppclk(clk_mgr, clk_mgr_base->clks.dppclk_khz);
223		215
@@ -769,43 +761,6 @@ static struct wm_table ddr4_wm_table_rn = { · 545 unmodified lines
769	}	761	}
770	};	762	};
771		763
772	static struct wm_table ddr4_1R_wm_table_rn = {
773	.entries = {
774	{
775	.wm_inst = WM_A,
776	.wm_type = WM_TYPE_PSTATE_CHG,
777	.pstate_latency_us = 11.72,
778	.sr_exit_time_us = 13.90,
779	.sr_enter_plus_exit_time_us = 14.80,
780	.valid = true,
781	},
782	{
783	.wm_inst = WM_B,
784	.wm_type = WM_TYPE_PSTATE_CHG,
785	.pstate_latency_us = 11.72,
786	.sr_exit_time_us = 13.90,
787	.sr_enter_plus_exit_time_us = 14.80,
788	.valid = true,
789	},
790	{
791	.wm_inst = WM_C,
792	.wm_type = WM_TYPE_PSTATE_CHG,
793	.pstate_latency_us = 11.72,
794	.sr_exit_time_us = 13.90,
795	.sr_enter_plus_exit_time_us = 14.80,
796	.valid = true,
797	},
798	{
799	.wm_inst = WM_D,
800	.wm_type = WM_TYPE_PSTATE_CHG,
801	.pstate_latency_us = 11.72,
802	.sr_exit_time_us = 13.90,
803	.sr_enter_plus_exit_time_us = 14.80,
804	.valid = true,
805	},
806	}
807	};
808
809	static struct wm_table lpddr4_wm_table_rn = {	764	static struct wm_table lpddr4_wm_table_rn = {
810	.entries = {	765	.entries = {
811	{	766	{
@@ -842,46 +797,67 @@ static struct wm_table lpddr4_wm_table_rn = { · 30 unmodified lines
842	},	797	},
843	}	798	}
844	};	799	};
845	static unsigned int find_socclk_for_voltage(struct dpm_clocks *clock_table, unsigned int voltage)	800
		801	static unsigned int find_max_fclk_for_voltage(struct dpm_clocks *clock_table,
		802	unsigned int voltage)
846	{	803	{
847	int i;	804	int i;
		805	uint32_t max_clk = 0;
848		806
849	for (i = 0; i < PP_SMU_NUM_SOCCLK_DPM_LEVELS; i++) {	807	for (i = 0; i < PP_SMU_NUM_FCLK_DPM_LEVELS; i++) {
850	if (clock_table->SocClocks[i].Vol == voltage)	808	if (clock_table->FClocks[i].Vol <= voltage) {
851	return clock_table->SocClocks[i].Freq;	809	max_clk = clock_table->FClocks[i].Freq > max_clk ?
		810	clock_table->FClocks[i].Freq : max_clk;
		811	}
		812	}
		813
		814	return max_clk;
		815	}
		816
		817	static unsigned int find_max_memclk_for_voltage(struct dpm_clocks *clock_table,
		818	unsigned int voltage)
		819	{
		820	int i;
		821	uint32_t max_clk = 0;
		822
		823	for (i = 0; i < PP_SMU_NUM_MEMCLK_DPM_LEVELS; i++) {
		824	if (clock_table->MemClocks[i].Vol <= voltage) {
		825	max_clk = clock_table->MemClocks[i].Freq > max_clk ?
		826	clock_table->MemClocks[i].Freq : max_clk;
		827	}
852	}	828	}
853		829
854	ASSERT(0);	830	return max_clk;
855	return 0;
856	}	831	}
857	static unsigned int find_dcfclk_for_voltage(struct dpm_clocks *clock_table, unsigned int voltage)	832
		833	static unsigned int find_max_socclk_for_voltage(struct dpm_clocks *clock_table,
		834	unsigned int voltage)
858	{	835	{
859	int i;	836	int i;
		837	uint32_t max_clk = 0;
860		838
861	for (i = 0; i < PP_SMU_NUM_DCFCLK_DPM_LEVELS; i++) {	839	for (i = 0; i < PP_SMU_NUM_SOCCLK_DPM_LEVELS; i++) {
862	if (clock_table->DcfClocks[i].Vol == voltage)	840	if (clock_table->SocClocks[i].Vol <= voltage) {
863	return clock_table->DcfClocks[i].Freq;	841	max_clk = clock_table->SocClocks[i].Freq > max_clk ?
		842	clock_table->SocClocks[i].Freq : max_clk;
		843	}
864	}	844	}
865		845
866	ASSERT(0);	846	return max_clk;
867	return 0;
868	}	847	}
869		848
870	static void rn_clk_mgr_helper_populate_bw_params(struct clk_bw_params bw_params, struct dpm_clocks clock_table, struct integrated_info *bios_info)	849	static void rn_clk_mgr_helper_populate_bw_params(struct clk_bw_params bw_params, struct dpm_clocks clock_table, struct integrated_info *bios_info)
871	{	850	{
872	int i, j = 0;	851	int i, j = 0;
		852	unsigned int volt;
873		853
874	j = -1;	854	j = -1;
875		855
876	ASSERT(PP_SMU_NUM_FCLK_DPM_LEVELS <= MAX_NUM_DPM_LVL);	856	/* Find max DPM */
877		857	for (i = 0; i < PP_SMU_NUM_DCFCLK_DPM_LEVELS; ++i) {
878	/* Find lowest DPM, FCLK is filled in reverse order*/	858	if (clock_table->DcfClocks[i].Freq != 0 &&
879		859	clock_table->DcfClocks[i].Vol != 0)
880	for (i = PP_SMU_NUM_FCLK_DPM_LEVELS - 1; i >= 0; i--) {
881	if (clock_table->FClocks[i].Freq != 0 && clock_table->FClocks[i].Vol != 0) {
882	j = i;	860	j = i;
883	break;
884	}
885	}	861	}
886		862
887	if (j == -1) {	863	if (j == -1) {
@@ -892,13 +868,18 @@ static void rn_clk_mgr_helper_populate_bw_params(struct clk_bw_params *bw_params · 4 unmodified lines
892		868
893	bw_params->clk_table.num_entries = j + 1;	869	bw_params->clk_table.num_entries = j + 1;
894		870
895	for (i = 0; i < bw_params->clk_table.num_entries; i++, j--) {	871	for (i = 0; i < bw_params->clk_table.num_entries; i++) {
896	bw_params->clk_table.entries[i].fclk_mhz = clock_table->FClocks[j].Freq;	872	volt = clock_table->DcfClocks[i].Vol;
897	bw_params->clk_table.entries[i].memclk_mhz = clock_table->MemClocks[j].Freq;	873
898	bw_params->clk_table.entries[i].voltage = clock_table->FClocks[j].Vol;	874	bw_params->clk_table.entries[i].voltage = volt;
899	bw_params->clk_table.entries[i].dcfclk_mhz = find_dcfclk_for_voltage(clock_table, clock_table->FClocks[j].Vol);	875	bw_params->clk_table.entries[i].dcfclk_mhz =
900	bw_params->clk_table.entries[i].socclk_mhz = find_socclk_for_voltage(clock_table,	876	clock_table->DcfClocks[i].Freq;
901	bw_params->clk_table.entries[i].voltage);	877	bw_params->clk_table.entries[i].fclk_mhz =
		878	find_max_fclk_for_voltage(clock_table, volt);
		879	bw_params->clk_table.entries[i].memclk_mhz =
		880	find_max_memclk_for_voltage(clock_table, volt);
		881	bw_params->clk_table.entries[i].socclk_mhz =
		882	find_max_socclk_for_voltage(clock_table, volt);
902	}	883	}
903		884
904	bw_params->vram_type = bios_info->memory_type;	885	bw_params->vram_type = bios_info->memory_type;
@@ -990,12 +971,8 @@ void rn_clk_mgr_construct( · 85 unmodified lines
990	} else {	971	} else {
991	if (is_green_sardine)	972	if (is_green_sardine)
992	rn_bw_params.wm_table = ddr4_wm_table_gs;	973	rn_bw_params.wm_table = ddr4_wm_table_gs;
993	else {	974	else
994	if (ctx->dc->config.is_single_rank_dimm)	975	rn_bw_params.wm_table = ddr4_wm_table_rn;
995	rn_bw_params.wm_table = ddr4_1R_wm_table_rn;
996	else
997	rn_bw_params.wm_table = ddr4_wm_table_rn;
998	}
999	}	976	}
1000	/* Saved clocks configured at boot for debug purposes */	977	/* Saved clocks configured at boot for debug purposes */
1001	rn_dump_clk_registers(&clk_mgr->base.boot_snapshot, &clk_mgr->base, &log_info);	978	rn_dump_clk_registers(&clk_mgr->base.boot_snapshot, &clk_mgr->base, &log_info);
@@ -1013,9 +990,6 @@ void rn_clk_mgr_construct( · 11 unmodified lines
1013	if (status == PP_SMU_RESULT_OK &&	990	if (status == PP_SMU_RESULT_OK &&
1014	ctx->dc_bios && ctx->dc_bios->integrated_info) {	991	ctx->dc_bios && ctx->dc_bios->integrated_info) {
1015	rn_clk_mgr_helper_populate_bw_params (clk_mgr->base.bw_params, &clock_table, ctx->dc_bios->integrated_info);	992	rn_clk_mgr_helper_populate_bw_params (clk_mgr->base.bw_params, &clock_table, ctx->dc_bios->integrated_info);
1016	/* treat memory config as single channel if memory is asymmetrics. */
1017	if (ctx->dc->config.is_asymmetric_memory)
1018	clk_mgr->base.bw_params->num_channels = 1;
1019	}	993	}
1020	}	994	}
1021		995

drivers/gpu/drm/amd/display/dc/core/dc.c+1−15

 #include "dc_link_ddc.h"
 #include "dm_helpers.h"
 #include "mem_input.h"
-#include "hubp.h"
 #include "dc_link_dp.h"
 #include "dc_dmub_srv.h"
 	}
 }
-void dc_wait_for_vblank(struct dc *dc, struct dc_stream_state *stream)
-{
-	int i;
-	for (i = 0; i < dc->res_pool->pipe_count; i++)
-		if (dc->current_state->res_ctx.pipe_ctx[i].stream == stream) {
-			struct timing_generator *tg =
-				dc->current_state->res_ctx.pipe_ctx[i].stream_res.tg;
-			tg->funcs->wait_for_state(tg, CRTC_STATE_VBLANK);
-			break;
-		}
-}
 void get_clock_requirements_for_state(struct dc_state *state, struct AsicStateEx *info)
 {
 	info->displayClock				= (unsigned int)state->bw_ctx.bw.dcn.clk.dispclk_khz;
 	if (dc->debug.disable_idle_power_optimizations)
 		return;
-	if (dc->clk_mgr->funcs->is_smu_present)
+	if (dc->clk_mgr != NULL && dc->clk_mgr->funcs->is_smu_present)
 		if (!dc->clk_mgr->funcs->is_smu_present(dc->clk_mgr))
 			return;

drivers/gpu/drm/amd/display/dc/core/dc_link.c+71−31

 #include "dce/dmub_psr.h"
 #include "dmub/dmub_srv.h"
 #include "inc/hw/panel_cntl.h"
+#include "inc/link_enc_cfg.h"
 #define DC_LOGGER_INIT(logger)
 		link->dc->hwss.edp_wait_for_hpd_ready(link, true);
 	}
+	/* Link may not have physical HPD pin. */
+	if (link->ep_type != DISPLAY_ENDPOINT_PHY) {
+		if (link->hpd_status)
+			*type = dc_connection_single;
+		else
+			*type = dc_connection_none;
+		return true;
+	}
 	/* todo: may need to lock gpio access */
 	hpd_pin = get_hpd_gpio(link->ctx->dc_bios, link->link_id,
 			       link->ctx->gpio_service);
 static enum signal_type link_detect_sink(struct dc_link *link,
 					 enum dc_detect_reason reason)
 {
-	enum signal_type result = get_basic_signal_type(link->link_enc->id,
+	enum signal_type result;
-							link->link_id);
+	struct graphics_object_id enc_id;
+	if (link->is_dig_mapping_flexible)
+		enc_id = (struct graphics_object_id){.id = ENCODER_ID_UNKNOWN};
+	else
+		enc_id = link->link_enc->id;
+	result = get_basic_signal_type(enc_id, link->link_id);
+	/* Use basic signal type for link without physical connector. */
+	if (link->ep_type != DISPLAY_ENDPOINT_PHY)
+		return result;
 	/* Internal digital encoder will detect only dongles
 	 * that require digital signal
 		}
 		if (link->type != dc_connection_mst_branch &&
-		    is_dp_active_dongle(link)) {
+		    is_dp_branch_device(link)) {
-			/* DP active dongles */
+			/* DP SST branch */
-			link->type = dc_connection_active_dongle;
+			link->type = dc_connection_sst_branch;
 			if (!link->dpcd_caps.sink_count.bits.SINK_COUNT) {
 				/*
-				 * active dongle unplug processing for short irq
+				 * SST branch unplug processing for short irq
 				 */
 				link_disconnect_sink(link);
 				return true;
 			}
-			if (link->dpcd_caps.dongle_type !=
+			if (is_dp_active_dongle(link) &&
-			    DISPLAY_DONGLE_DP_HDMI_CONVERTER)
+				(link->dpcd_caps.dongle_type !=
+					DISPLAY_DONGLE_DP_HDMI_CONVERTER))
 				*converter_disable_audio = true;
 		}
 	} else {
 		case SIGNAL_TYPE_DISPLAY_PORT: {
 			/* wa HPD high coming too early*/
-			if (link->link_enc->features.flags.bits.DP_IS_USB_C == 1) {
+			if (link->ep_type == DISPLAY_ENDPOINT_PHY &&
+			    link->link_enc->features.flags.bits.DP_IS_USB_C == 1) {
 				/* if alt mode times out, return false */
 				if (!wait_for_entering_dp_alt_mode(link))
 					return false;
 					   sizeof(struct dpcd_caps)))
 					same_dpcd = false;
 			}
-			/* Active dongle downstream unplug*/
+			/* Active SST downstream branch device unplug*/
-			if (link->type == dc_connection_active_dongle &&
+			if (link->type == dc_connection_sst_branch &&
 			    link->dpcd_caps.sink_count.bits.SINK_COUNT == 0) {
 				if (prev_sink)
 					/* Downstream unplug */
 {
 	const struct dc *dc = link->dc;
 	bool ret;
+	bool can_apply_seamless_boot = false;
+	int i;
+	for (i = 0; i < dc->current_state->stream_count; i++) {
+		if (dc->current_state->streams[i]->apply_seamless_boot_optimization) {
+			can_apply_seamless_boot = true;
+			break;
+		}
+	}
 	/* get out of low power state */
-	clk_mgr_exit_optimized_pwr_state(dc, dc->clk_mgr);
+	if (!can_apply_seamless_boot && reason != DETECT_REASON_BOOT)
+		clk_mgr_exit_optimized_pwr_state(dc, dc->clk_mgr);
 	ret = dc_link_detect_helper(link, reason);
 	/* Go back to power optimized state */
-	clk_mgr_optimize_pwr_state(dc, dc->clk_mgr);
+	if (!can_apply_seamless_boot && reason != DETECT_REASON_BOOT)
+		clk_mgr_optimize_pwr_state(dc, dc->clk_mgr);
 	return ret;
 }
 	bool apply_seamless_boot_optimization = false;
 	uint32_t bl_oled_enable_delay = 50; // in ms
 	const uint32_t post_oui_delay = 30; // 30ms
+	/* Reduce link bandwidth between failed link training attempts. */
+	bool do_fallback = false;
 	// check for seamless boot
 	for (i = 0; i < state->stream_count; i++) {
 					       skip_video_pattern,
 					       LINK_TRAINING_ATTEMPTS,
 					       pipe_ctx,
-					       pipe_ctx->stream->signal)) {
+					       pipe_ctx->stream->signal,
+					       do_fallback)) {
 		link->cur_link_settings = link_settings;
 		status = DC_OK;
 	} else {
 	uint32_t kbps;
 #if defined(CONFIG_DRM_AMD_DC_DCN)
-	if (timing->flags.DSC) {
+	if (timing->flags.DSC)
-		return dc_dsc_stream_bandwidth_in_kbps(timing->pix_clk_100hz, timing->dsc_cfg.bits_per_pixel);
+		return dc_dsc_stream_bandwidth_in_kbps(timing,
-	}
+				timing->dsc_cfg.bits_per_pixel,
+				timing->dsc_cfg.num_slices_h,
+				timing->dsc_cfg.is_dp);
 #endif
 	switch (timing->display_color_depth) {
 	dc_link_dp_set_drive_settings(dc->links[i], lt_settings);
 }
-void dc_link_perform_link_training(struct dc *dc,
-				   struct dc_link_settings *link_setting,
-				   bool skip_video_pattern)
-{
-	int i;
-	for (i = 0; i < dc->link_count; i++)
-		dc_link_dp_perform_link_training(
-			dc->links[i],
-			link_setting,
-			skip_video_pattern);
-}
 void dc_link_set_preferred_link_settings(struct dc *dc,
 					 struct dc_link_settings *link_setting,
 					 struct dc_link *link)
 bool dc_link_is_fec_supported(const struct dc_link *link)
 {
+	struct link_encoder *link_enc = NULL;
+	/* Links supporting dynamically assigned link encoder will be assigned next
+	 * available encoder if one not already assigned.
+	 */
+	if (link->is_dig_mapping_flexible &&
+			link->dc->res_pool->funcs->link_encs_assign) {
+		link_enc = link_enc_cfg_get_link_enc_used_by_link(link->dc->current_state, link);
+		if (link_enc == NULL)
+			link_enc = link_enc_cfg_get_next_avail_link_enc(link->dc, link->dc->current_state);
+	} else
+		link_enc = link->link_enc;
+	ASSERT(link_enc);
 	return (dc_is_dp_signal(link->connector_signal) &&
-			link->link_enc->features.fec_supported &&
+			link_enc->features.fec_supported &&
 			link->dpcd_caps.fec_cap.bits.FEC_CAPABLE &&
 			!IS_FPGA_MAXIMUS_DC(link->ctx->dce_environment));
 }

drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c+8−1

 		struct aux_payload *payload,
 		enum aux_return_code_type *operation_result)
 {
-	return dce_aux_transfer_raw(ddc, payload, operation_result);
+	if (dc_enable_dmub_notifications(ddc->ctx->dc))
+		return dce_aux_transfer_dmub_raw(ddc, payload, operation_result);
+	else
+		return dce_aux_transfer_raw(ddc, payload, operation_result);
 }
 /* dc_link_aux_transfer_with_retries() - Attempt to submit an
 	bool result = false;
 	struct ddc *ddc_pin = ddc->ddc_pin;
+	/* Do not try to access nonexistent DDC pin. */
+	if (ddc->link->ep_type != DISPLAY_ENDPOINT_PHY)
+		return true;
 	if (ddc->ctx->dc->res_pool->engines[ddc_pin->pin_data->en]->funcs->configure_timeout) {
 		ddc->ctx->dc->res_pool->engines[ddc_pin->pin_data->en]->funcs->configure_timeout(ddc, timeout);
 		result = true;

drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c+157−74

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	#include "dpcd_defs.h"	14	#include "dpcd_defs.h"
15	#include "dc_dmub_srv.h"	15	#include "dc_dmub_srv.h"
16	#include "dce/dmub_hw_lock_mgr.h"	16	#include "dce/dmub_hw_lock_mgr.h"
		17	#include "inc/link_enc_cfg.h"
17		18
18	/Travis/	19	/Travis/
19	static const uint8_t DP_VGA_LVDS_CONVERTER_ID_2[] = "sivarT";	20	static const uint8_t DP_VGA_LVDS_CONVERTER_ID_2[] = "sivarT";
@@ -107,10 +108,50 @@ static void wait_for_training_aux_rd_interval( · 87 unmodified lines
107	wait_in_micro_secs);	108	wait_in_micro_secs);
108	}	109	}
109		110
		111	static enum dpcd_training_patterns
		112	dc_dp_training_pattern_to_dpcd_training_pattern(
		113	struct dc_link *link,
		114	enum dc_dp_training_pattern pattern)
		115	{
		116	enum dpcd_training_patterns dpcd_tr_pattern =
		117	DPCD_TRAINING_PATTERN_VIDEOIDLE;
		118
		119	switch (pattern) {
		120	case DP_TRAINING_PATTERN_SEQUENCE_1:
		121	dpcd_tr_pattern = DPCD_TRAINING_PATTERN_1;
		122	break;
		123	case DP_TRAINING_PATTERN_SEQUENCE_2:
		124	dpcd_tr_pattern = DPCD_TRAINING_PATTERN_2;
		125	break;
		126	case DP_TRAINING_PATTERN_SEQUENCE_3:
		127	dpcd_tr_pattern = DPCD_TRAINING_PATTERN_3;
		128	break;
		129	case DP_TRAINING_PATTERN_SEQUENCE_4:
		130	dpcd_tr_pattern = DPCD_TRAINING_PATTERN_4;
		131	break;
		132	case DP_TRAINING_PATTERN_VIDEOIDLE:
		133	dpcd_tr_pattern = DPCD_TRAINING_PATTERN_VIDEOIDLE;
		134	break;
		135	default:
		136	ASSERT(0);
		137	DC_LOG_HW_LINK_TRAINING("%s: Invalid HW Training pattern: %d\n",
		138	__func__, pattern);
		139	break;
		140	}
		141
		142	return dpcd_tr_pattern;
		143	}
		144
110	static void dpcd_set_training_pattern(	145	static void dpcd_set_training_pattern(
111	struct dc_link *link,	146	struct dc_link *link,
112	union dpcd_training_pattern dpcd_pattern)	147	enum dc_dp_training_pattern training_pattern)
113	{	148	{
		149	union dpcd_training_pattern dpcd_pattern = { {0} };
		150
		151	dpcd_pattern.v1_4.TRAINING_PATTERN_SET =
		152	dc_dp_training_pattern_to_dpcd_training_pattern(
		153	link, training_pattern);
		154
114	core_link_write_dpcd(	155	core_link_write_dpcd(
115	link,	156	link,
116	DP_TRAINING_PATTERN_SET,	157	DP_TRAINING_PATTERN_SET,
@@ -132,10 +173,22 @@ static enum dc_dp_training_pattern decide_cr_training_pattern( · 15 unmodified lines
132	static enum dc_dp_training_pattern decide_eq_training_pattern(struct dc_link *link,	173	static enum dc_dp_training_pattern decide_eq_training_pattern(struct dc_link *link,
133	const struct dc_link_settings *link_settings)	174	const struct dc_link_settings *link_settings)
134	{	175	{
		176	struct link_encoder *link_enc;
135	enum dc_dp_training_pattern highest_tp = DP_TRAINING_PATTERN_SEQUENCE_2;	177	enum dc_dp_training_pattern highest_tp = DP_TRAINING_PATTERN_SEQUENCE_2;
136	struct encoder_feature_support *features = &link->link_enc->features;	178	struct encoder_feature_support *features;
137	struct dpcd_caps *dpcd_caps = &link->dpcd_caps;	179	struct dpcd_caps *dpcd_caps = &link->dpcd_caps;
138		180
		181	/* Access link encoder capability based on whether it is statically
		182	* or dynamically assigned to a link.
		183	*/
		184	if (link->is_dig_mapping_flexible &&
		185	link->dc->res_pool->funcs->link_encs_assign)
		186	link_enc = link_enc_cfg_get_link_enc_used_by_link(link->dc->current_state, link);
		187	else
		188	link_enc = link->link_enc;
		189	ASSERT(link_enc);
		190	features = &link_enc->features;
		191
139	if (features->flags.bits.IS_TPS3_CAPABLE)	192	if (features->flags.bits.IS_TPS3_CAPABLE)
140	highest_tp = DP_TRAINING_PATTERN_SEQUENCE_3;	193	highest_tp = DP_TRAINING_PATTERN_SEQUENCE_3;
141		194
@@ -227,37 +280,6 @@ static void dpcd_set_link_settings( · 85 unmodified lines
227	}	280	}
228	}	281	}
229		282
230	static enum dpcd_training_patterns
231	dc_dp_training_pattern_to_dpcd_training_pattern(
232	struct dc_link *link,
233	enum dc_dp_training_pattern pattern)
234	{
235	enum dpcd_training_patterns dpcd_tr_pattern =
236	DPCD_TRAINING_PATTERN_VIDEOIDLE;
237
238	switch (pattern) {
239	case DP_TRAINING_PATTERN_SEQUENCE_1:
240	dpcd_tr_pattern = DPCD_TRAINING_PATTERN_1;
241	break;
242	case DP_TRAINING_PATTERN_SEQUENCE_2:
243	dpcd_tr_pattern = DPCD_TRAINING_PATTERN_2;
244	break;
245	case DP_TRAINING_PATTERN_SEQUENCE_3:
246	dpcd_tr_pattern = DPCD_TRAINING_PATTERN_3;
247	break;
248	case DP_TRAINING_PATTERN_SEQUENCE_4:
249	dpcd_tr_pattern = DPCD_TRAINING_PATTERN_4;
250	break;
251	default:
252	ASSERT(0);
253	DC_LOG_HW_LINK_TRAINING("%s: Invalid HW Training pattern: %d\n",
254	__func__, pattern);
255	break;
256	}
257
258	return dpcd_tr_pattern;
259	}
260
261	static uint8_t dc_dp_initialize_scrambling_data_symbols(	283	static uint8_t dc_dp_initialize_scrambling_data_symbols(
262	struct dc_link *link,	284	struct dc_link *link,
263	enum dc_dp_training_pattern pattern)	285	enum dc_dp_training_pattern pattern)
@@ -420,20 +442,30 @@ static bool is_cr_done(enum dc_lane_count ln_count, · 156 unmodified lines
420	}	442	}
421		443
422	static bool is_ch_eq_done(enum dc_lane_count ln_count,	444	static bool is_ch_eq_done(enum dc_lane_count ln_count,
423	union lane_status *dpcd_lane_status,	445	union lane_status *dpcd_lane_status)
424	union lane_align_status_updated *lane_status_updated)
425	{	446	{
		447	bool done = true;
426	uint32_t lane;	448	uint32_t lane;
427	if (!lane_status_updated->bits.INTERLANE_ALIGN_DONE)	449	for (lane = 0; lane < (uint32_t)(ln_count); lane++)
428	return false;	450	if (!dpcd_lane_status[lane].bits.CHANNEL_EQ_DONE_0)
429	else {	451	done = false;
430	for (lane = 0; lane < (uint32_t)(ln_count); lane++) {	452	return done;
431	if (!dpcd_lane_status[lane].bits.SYMBOL_LOCKED_0 \|\|	453	}
432	!dpcd_lane_status[lane].bits.CHANNEL_EQ_DONE_0)	454
433	return false;	455	static bool is_symbol_locked(enum dc_lane_count ln_count,
434	}	456	union lane_status *dpcd_lane_status)
435	}	457	{
436	return true;	458	bool locked = true;
		459	uint32_t lane;
		460	for (lane = 0; lane < (uint32_t)(ln_count); lane++)
		461	if (!dpcd_lane_status[lane].bits.SYMBOL_LOCKED_0)
		462	locked = false;
		463	return locked;
		464	}
		465
		466	static inline bool is_interlane_aligned(union lane_align_status_updated align_status)
		467	{
		468	return align_status.bits.INTERLANE_ALIGN_DONE == 1;
437	}	469	}
438		470
439	static void update_drive_settings(	471	static void update_drive_settings(
@@ -835,10 +867,9 @@ static bool perform_post_lt_adj_req_sequence( · 395 unmodified lines
835	if (!is_cr_done(lane_count, dpcd_lane_status))	867	if (!is_cr_done(lane_count, dpcd_lane_status))
836	return false;	868	return false;
837		869
838	if (!is_ch_eq_done(	870	if (!is_ch_eq_done(lane_count, dpcd_lane_status) \|\|
839	lane_count,	871	!is_symbol_locked(lane_count, dpcd_lane_status) \|\|
840	dpcd_lane_status,	872	!is_interlane_aligned(dpcd_lane_status_updated))
841	&dpcd_lane_status_updated))
842	return false;	873	return false;
843		874
844	for (lane = 0; lane < (uint32_t)(lane_count); lane++) {	875	for (lane = 0; lane < (uint32_t)(lane_count); lane++) {
@@ -992,9 +1023,9 @@ static enum link_training_result perform_channel_equalization_sequence( · 147 unmodified lines
992	return LINK_TRAINING_EQ_FAIL_CR;	1023	return LINK_TRAINING_EQ_FAIL_CR;
993		1024
994	/* 6. check CHEQ done*/	1025	/* 6. check CHEQ done*/
995	if (is_ch_eq_done(lane_count,	1026	if (is_ch_eq_done(lane_count, dpcd_lane_status) &&
996	dpcd_lane_status,	1027	is_symbol_locked(lane_count, dpcd_lane_status) &&
997	&dpcd_lane_status_updated))	1028	is_interlane_aligned(dpcd_lane_status_updated))
998	return LINK_TRAINING_SUCCESS;	1029	return LINK_TRAINING_SUCCESS;
999		1030
1000	/* 7. update VS/PE/PC2 in lt_settings*/	1031	/* 7. update VS/PE/PC2 in lt_settings*/
@@ -1162,7 +1193,7 @@ static inline enum link_training_result perform_link_training_int( · 161 unmodified lines
1162	return status;	1193	return status;
1163	}	1194	}
1164		1195
1165	static enum link_training_result check_link_loss_status(	1196	enum link_training_result dp_check_link_loss_status(
1166	struct dc_link *link,	1197	struct dc_link *link,
1167	const struct link_training_settings *link_training_setting)	1198	const struct link_training_settings *link_training_setting)
1168	{	1199	{
@@ -1296,7 +1327,7 @@ static void initialize_training_settings( · 127 unmodified lines
1296	lt_settings->enhanced_framing = 1;	1327	lt_settings->enhanced_framing = 1;
1297	}	1328	}
1298		1329
1299	static uint8_t convert_to_count(uint8_t lttpr_repeater_count)	1330	uint8_t dp_convert_to_count(uint8_t lttpr_repeater_count)
1300	{	1331	{
1301	switch (lttpr_repeater_count) {	1332	switch (lttpr_repeater_count) {
1302	case 0x80: // 1 lttpr repeater	1333	case 0x80: // 1 lttpr repeater
@@ -1365,7 +1396,8 @@ static void configure_lttpr_mode_non_transparent(struct dc_link *link) · 62 unmodified lines
1365	link->dpcd_caps.lttpr_caps.mode = repeater_mode;	1396	link->dpcd_caps.lttpr_caps.mode = repeater_mode;
1366	}	1397	}
1367		1398
1368	repeater_cnt = convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt);	1399	repeater_cnt = dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt);
		1400
1369	for (repeater_id = repeater_cnt; repeater_id > 0; repeater_id--) {	1401	for (repeater_id = repeater_cnt; repeater_id > 0; repeater_id--) {
1370	aux_interval_address = DP_TRAINING_AUX_RD_INTERVAL_PHY_REPEATER1 +	1402	aux_interval_address = DP_TRAINING_AUX_RD_INTERVAL_PHY_REPEATER1 +
1371	((DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE) * (repeater_id - 1));	1403	((DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE) * (repeater_id - 1));
@@ -1555,7 +1587,6 @@ enum link_training_result dc_link_dp_perform_link_training( · 183 unmodified lines
1555	{	1587	{
1556	enum link_training_result status = LINK_TRAINING_SUCCESS;	1588	enum link_training_result status = LINK_TRAINING_SUCCESS;
1557	struct link_training_settings lt_settings;	1589	struct link_training_settings lt_settings;
1558	union dpcd_training_pattern dpcd_pattern = { { 0 } };
1559		1590
1560	bool fec_enable;	1591	bool fec_enable;
1561	uint8_t repeater_cnt;	1592	uint8_t repeater_cnt;
@@ -1591,7 +1622,7 @@ enum link_training_result dc_link_dp_perform_link_training( · 29 unmodified lines
1591	/* 2. perform link training (set link training done	1622	/* 2. perform link training (set link training done
1592	* to false is done as well)	1623	* to false is done as well)
1593	*/	1624	*/
1594	repeater_cnt = convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt);	1625	repeater_cnt = dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt);
1595		1626
1596	for (repeater_id = repeater_cnt; (repeater_id > 0 && status == LINK_TRAINING_SUCCESS);	1627	for (repeater_id = repeater_cnt; (repeater_id > 0 && status == LINK_TRAINING_SUCCESS);
1597	repeater_id--) {	1628	repeater_id--) {
@@ -1621,8 +1652,7 @@ enum link_training_result dc_link_dp_perform_link_training( · 23 unmodified lines
1621	}	1652	}
1622		1653
1623	/* 3. set training not in progress*/	1654	/* 3. set training not in progress*/
1624	dpcd_pattern.v1_4.TRAINING_PATTERN_SET = DPCD_TRAINING_PATTERN_VIDEOIDLE;	1655	dpcd_set_training_pattern(link, DP_TRAINING_PATTERN_VIDEOIDLE);
1625	dpcd_set_training_pattern(link, dpcd_pattern);
1626	if ((status == LINK_TRAINING_SUCCESS) \|\| !skip_video_pattern) {	1656	if ((status == LINK_TRAINING_SUCCESS) \|\| !skip_video_pattern) {
1627	status = perform_link_training_int(link,	1657	status = perform_link_training_int(link,
1628	&lt_settings,	1658	&lt_settings,
@@ -1634,7 +1664,7 @@ enum link_training_result dc_link_dp_perform_link_training( · 5 unmodified lines
1634	*/	1664	*/
1635	if (link->connector_signal != SIGNAL_TYPE_EDP && status == LINK_TRAINING_SUCCESS) {	1665	if (link->connector_signal != SIGNAL_TYPE_EDP && status == LINK_TRAINING_SUCCESS) {
1636	msleep(5);	1666	msleep(5);
1637	status = check_link_loss_status(link, &lt_settings);	1667	status = dp_check_link_loss_status(link, &lt_settings);
1638	}	1668	}
1639		1669
1640	/* 6. print status message*/	1670	/* 6. print status message*/
@@ -1687,18 +1717,31 @@ bool perform_link_training_with_retries( · 46 unmodified lines
1687	bool skip_video_pattern,	1717	bool skip_video_pattern,
1688	int attempts,	1718	int attempts,
1689	struct pipe_ctx *pipe_ctx,	1719	struct pipe_ctx *pipe_ctx,
1690	enum signal_type signal)	1720	enum signal_type signal,
		1721	bool do_fallback)
1691	{	1722	{
1692	uint8_t j;	1723	uint8_t j;
1693	uint8_t delay_between_attempts = LINK_TRAINING_RETRY_DELAY;	1724	uint8_t delay_between_attempts = LINK_TRAINING_RETRY_DELAY;
1694	struct dc_stream_state *stream = pipe_ctx->stream;	1725	struct dc_stream_state *stream = pipe_ctx->stream;
1695	struct dc_link *link = stream->link;	1726	struct dc_link *link = stream->link;
1696	enum dp_panel_mode panel_mode;	1727	enum dp_panel_mode panel_mode;
		1728	struct link_encoder *link_enc;
		1729	enum link_training_result status = LINK_TRAINING_CR_FAIL_LANE0;
		1730	struct dc_link_settings currnet_setting = *link_setting;
		1731
		1732	/* Dynamically assigned link encoders associated with stream rather than
		1733	* link.
		1734	*/
		1735	if (link->dc->res_pool->funcs->link_encs_assign)
		1736	link_enc = stream->link_enc;
		1737	else
		1738	link_enc = link->link_enc;
		1739	ASSERT(link_enc);
1697		1740
1698	/* We need to do this before the link training to ensure the idle pattern in SST	1741	/* We need to do this before the link training to ensure the idle pattern in SST
1699	* mode will be sent right after the link training	1742	* mode will be sent right after the link training
1700	*/	1743	*/
1701	link->link_enc->funcs->connect_dig_be_to_fe(link->link_enc,	1744	link_enc->funcs->connect_dig_be_to_fe(link_enc,
1702	pipe_ctx->stream_res.stream_enc->id, true);	1745	pipe_ctx->stream_res.stream_enc->id, true);
1703		1746
1704	for (j = 0; j < attempts; ++j) {	1747	for (j = 0; j < attempts; ++j) {
@@ -1710,7 +1753,7 @@ bool perform_link_training_with_retries( · 5 unmodified lines
1710	link,	1753	link,
1711	signal,	1754	signal,
1712	pipe_ctx->clock_source->id,	1755	pipe_ctx->clock_source->id,
1713	link_setting);	1756	&currnet_setting);
1714		1757
1715	if (stream->sink_patches.dppowerup_delay > 0) {	1758	if (stream->sink_patches.dppowerup_delay > 0) {
1716	int delay_dp_power_up_in_ms = stream->sink_patches.dppowerup_delay;	1759	int delay_dp_power_up_in_ms = stream->sink_patches.dppowerup_delay;
@@ -1725,14 +1768,12 @@ bool perform_link_training_with_retries( · 8 unmodified lines
1725	panel_mode != DP_PANEL_MODE_DEFAULT);	1768	panel_mode != DP_PANEL_MODE_DEFAULT);
1726		1769
1727	if (link->aux_access_disabled) {	1770	if (link->aux_access_disabled) {
1728	dc_link_dp_perform_link_training_skip_aux(link, link_setting);	1771	dc_link_dp_perform_link_training_skip_aux(link, &currnet_setting);
1729	return true;	1772	return true;
1730	} else {	1773	} else {
1731	enum link_training_result status = LINK_TRAINING_CR_FAIL_LANE0;
1732
1733	status = dc_link_dp_perform_link_training(	1774	status = dc_link_dp_perform_link_training(
1734	link,	1775	link,
1735	link_setting,	1776	&currnet_setting,
1736	skip_video_pattern);	1777	skip_video_pattern);
1737	if (status == LINK_TRAINING_SUCCESS)	1778	if (status == LINK_TRAINING_SUCCESS)
1738	return true;	1779	return true;
@@ -1740,7 +1781,7 @@ bool perform_link_training_with_retries( · 1 unmodified line
1740		1781
1741	/* latest link training still fail, skip delay and keep PHY on	1782	/* latest link training still fail, skip delay and keep PHY on
1742	*/	1783	*/
1743	if (j == (attempts - 1))	1784	if (j == (attempts - 1) && link->ep_type == DISPLAY_ENDPOINT_PHY)
1744	break;	1785	break;
1745		1786
1746	DC_LOG_WARNING("%s: Link training attempt %u of %d failed\n",	1787	DC_LOG_WARNING("%s: Link training attempt %u of %d failed\n",
@@ -1748,6 +1789,19 @@ bool perform_link_training_with_retries( · 1 unmodified line
1748		1789
1749	dp_disable_link_phy(link, signal);	1790	dp_disable_link_phy(link, signal);
1750		1791
		1792	/* Abort link training if failure due to sink being unplugged. */
		1793	if (status == LINK_TRAINING_ABORT)
		1794	break;
		1795	else if (do_fallback) {
		1796	decide_fallback_link_setting(*link_setting, &currnet_setting, status);
		1797	/* Fail link training if reduced link bandwidth no longer meets
		1798	* stream requirements.
		1799	*/
		1800	if (dc_bandwidth_in_kbps_from_timing(&stream->timing) <
		1801	dc_link_bandwidth_kbps(link, &currnet_setting))
		1802	break;
		1803	}
		1804
1751	msleep(delay_between_attempts);	1805	msleep(delay_between_attempts);
1752		1806
1753	delay_between_attempts += LINK_TRAINING_RETRY_DELAY;	1807	delay_between_attempts += LINK_TRAINING_RETRY_DELAY;
@@ -2429,6 +2483,12 @@ bool dp_validate_mode_timing( · 675 unmodified lines
2429		2483
2430	const struct dc_link_settings *link_setting;	2484	const struct dc_link_settings *link_setting;
2431		2485
		2486	/* According to spec, VSC SDP should be used if pixel format is YCbCr420 */
		2487	if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR420 &&
		2488	!link->dpcd_caps.dprx_feature.bits.VSC_SDP_COLORIMETRY_SUPPORTED &&
		2489	dal_graphics_object_id_get_connector_id(link->link_id) != CONNECTOR_ID_VIRTUAL)
		2490	return false;
		2491
2432	/always DP fail safe mode/	2492	/always DP fail safe mode/
2433	if ((timing->pix_clk_100hz / 10) == (uint32_t) 25175 &&	2493	if ((timing->pix_clk_100hz / 10) == (uint32_t) 25175 &&
2434	timing->h_addressable == (uint32_t) 640 &&	2494	timing->h_addressable == (uint32_t) 640 &&
@@ -2611,13 +2671,11 @@ static bool allow_hpd_rx_irq(const struct dc_link *link) · 176 unmodified lines
2611	/*	2671	/*
2612	* Don't handle RX IRQ unless one of following is met:	2672	* Don't handle RX IRQ unless one of following is met:
2613	* 1) The link is established (cur_link_settings != unknown)	2673	* 1) The link is established (cur_link_settings != unknown)
2614	* 2) We kicked off MST detection	2674	* 2) We know we're dealing with a branch device, SST or MST
2615	* 3) We know we're dealing with an active dongle
2616	*/	2675	*/
2617		2676
2618	if ((link->cur_link_settings.lane_count != LANE_COUNT_UNKNOWN) \|\|	2677	if ((link->cur_link_settings.lane_count != LANE_COUNT_UNKNOWN) \|\|
2619	(link->type == dc_connection_mst_branch) \|\|	2678	is_dp_branch_device(link))
2620	is_dp_active_dongle(link))
2621	return true;	2679	return true;
2622		2680
2623	return false;	2681	return false;
@@ -2917,6 +2975,22 @@ static void dp_test_send_link_test_pattern(struct dc_link *link) · 293 unmodified lines
2917	break;	2975	break;
2918	}	2976	}
2919		2977
		2978	switch (dpcd_test_params.bits.CLR_FORMAT) {
		2979	case 0:
		2980	pipe_ctx->stream->timing.pixel_encoding = PIXEL_ENCODING_RGB;
		2981	break;
		2982	case 1:
		2983	pipe_ctx->stream->timing.pixel_encoding = PIXEL_ENCODING_YCBCR422;
		2984	break;
		2985	case 2:
		2986	pipe_ctx->stream->timing.pixel_encoding = PIXEL_ENCODING_YCBCR444;
		2987	break;
		2988	default:
		2989	pipe_ctx->stream->timing.pixel_encoding = PIXEL_ENCODING_RGB;
		2990	break;
		2991	}
		2992
		2993
2920	if (requestColorDepth != COLOR_DEPTH_UNDEFINED	2994	if (requestColorDepth != COLOR_DEPTH_UNDEFINED
2921	&& pipe_ctx->stream->timing.display_color_depth != requestColorDepth) {	2995	&& pipe_ctx->stream->timing.display_color_depth != requestColorDepth) {
2922	DC_LOG_DEBUG("%s: original bpc %d, changing to %d\n",	2996	DC_LOG_DEBUG("%s: original bpc %d, changing to %d\n",
@@ -2924,9 +2998,10 @@ static void dp_test_send_link_test_pattern(struct dc_link *link) · 1 unmodified line
2924	pipe_ctx->stream->timing.display_color_depth,	2998	pipe_ctx->stream->timing.display_color_depth,
2925	requestColorDepth);	2999	requestColorDepth);
2926	pipe_ctx->stream->timing.display_color_depth = requestColorDepth;	3000	pipe_ctx->stream->timing.display_color_depth = requestColorDepth;
2927	dp_update_dsc_config(pipe_ctx);
2928	}	3001	}
2929		3002
		3003	dp_update_dsc_config(pipe_ctx);
		3004
2930	dc_link_dp_set_test_pattern(	3005	dc_link_dp_set_test_pattern(
2931	link,	3006	link,
2932	test_pattern,	3007	test_pattern,
@@ -3182,7 +3257,7 @@ bool dc_link_handle_hpd_rx_irq(struct dc_link link, union hpd_irq_data out_hpd · 249 unmodified lines
3182	*out_link_loss = true;	3257	*out_link_loss = true;
3183	}	3258	}
3184		3259
3185	if (link->type == dc_connection_active_dongle &&	3260	if (link->type == dc_connection_sst_branch &&
3186	hpd_irq_dpcd_data.bytes.sink_cnt.bits.SINK_COUNT	3261	hpd_irq_dpcd_data.bytes.sink_cnt.bits.SINK_COUNT
3187	!= link->dpcd_sink_count)	3262	!= link->dpcd_sink_count)
3188	status = true;	3263	status = true;
@@ -3232,6 +3307,12 @@ bool is_mst_supported(struct dc_link *link) · 43 unmodified lines
3232	}	3307	}
3233		3308
3234	bool is_dp_active_dongle(const struct dc_link *link)	3309	bool is_dp_active_dongle(const struct dc_link *link)
		3310	{
		3311	return (link->dpcd_caps.dongle_type >= DISPLAY_DONGLE_DP_VGA_CONVERTER) &&
		3312	(link->dpcd_caps.dongle_type <= DISPLAY_DONGLE_DP_HDMI_CONVERTER);
		3313	}
		3314
		3315	bool is_dp_branch_device(const struct dc_link *link)
3235	{	3316	{
3236	return link->dpcd_caps.is_branch_dev;	3317	return link->dpcd_caps.is_branch_dev;
3237	}	3318	}
@@ -3593,7 +3674,9 @@ static bool retrieve_link_cap(struct dc_link *link) · 355 unmodified lines
3593	lttpr_dpcd_data[DP_PHY_REPEATER_EXTENDED_WAIT_TIMEOUT -	3674	lttpr_dpcd_data[DP_PHY_REPEATER_EXTENDED_WAIT_TIMEOUT -
3594	DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];	3675	DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
3595		3676
		3677	/* Attempt to train in LTTPR transparent mode if repeater count exceeds 8. */
3596	is_lttpr_present = (link->dpcd_caps.lttpr_caps.phy_repeater_cnt > 0 &&	3678	is_lttpr_present = (link->dpcd_caps.lttpr_caps.phy_repeater_cnt > 0 &&
		3679	link->dpcd_caps.lttpr_caps.phy_repeater_cnt < 0xff &&
3597	link->dpcd_caps.lttpr_caps.max_lane_count > 0 &&	3680	link->dpcd_caps.lttpr_caps.max_lane_count > 0 &&
3598	link->dpcd_caps.lttpr_caps.max_lane_count <= 4 &&	3681	link->dpcd_caps.lttpr_caps.max_lane_count <= 4 &&
3599	link->dpcd_caps.lttpr_caps.revision.raw >= 0x14);	3682	link->dpcd_caps.lttpr_caps.revision.raw >= 0x14);

drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c+17−5

@@ -112,8 +112,8 @@ static void update_link_enc_assignment( · 111 unmodified lines
112		112
113	/* Return first available DIG link encoder. */	113	/* Return first available DIG link encoder. */
114	static enum engine_id find_first_avail_link_enc(	114	static enum engine_id find_first_avail_link_enc(
115	struct dc_context *ctx,	115	const struct dc_context *ctx,
116	struct dc_state *state)	116	const struct dc_state *state)
117	{	117	{
118	enum engine_id eng_id = ENGINE_ID_UNKNOWN;	118	enum engine_id eng_id = ENGINE_ID_UNKNOWN;
119	int i;	119	int i;
@@ -270,7 +270,7 @@ struct dc_link *link_enc_cfg_get_link_using_link_enc( · 150 unmodified lines
270		270
271	struct link_encoder *link_enc_cfg_get_link_enc_used_by_link(	271	struct link_encoder *link_enc_cfg_get_link_enc_used_by_link(
272	struct dc_state *state,	272	struct dc_state *state,
273	struct dc_link *link)	273	const struct dc_link *link)
274	{	274	{
275	struct link_encoder *link_enc = NULL;	275	struct link_encoder *link_enc = NULL;
276	struct display_endpoint_id ep_id;	276	struct display_endpoint_id ep_id;
@@ -296,8 +296,20 @@ struct link_encoder *link_enc_cfg_get_link_enc_used_by_link( · 19 unmodified lines
296		296
297	if (stream_idx != -1)	297	if (stream_idx != -1)
298	link_enc = state->streams[stream_idx]->link_enc;	298	link_enc = state->streams[stream_idx]->link_enc;
299	else	299
300	dm_output_to_console("%s: No link encoder used by link(%d).\n", __func__, link->link_index);	300	return link_enc;
		301	}
		302
		303	struct link_encoder *link_enc_cfg_get_next_avail_link_enc(
		304	const struct dc *dc,
		305	const struct dc_state *state)
		306	{
		307	struct link_encoder *link_enc = NULL;
		308	enum engine_id eng_id = ENGINE_ID_UNKNOWN;
		309
		310	eng_id = find_first_avail_link_enc(dc->ctx, state);
		311	if (eng_id != ENGINE_ID_UNKNOWN)
		312	link_enc = dc->res_pool->link_encoders[eng_id - ENGINE_ID_DIGA];
301		313
302	return link_enc;	314	return link_enc;
303	}	315	}

drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c+2−1

@@ -384,7 +384,8 @@ void dp_retrain_link_dp_test(struct dc_link *link, · 383 unmodified lines
384	skip_video_pattern,	384	skip_video_pattern,
385	LINK_TRAINING_ATTEMPTS,	385	LINK_TRAINING_ATTEMPTS,
386	&pipes[i],	386	&pipes[i],
387	SIGNAL_TYPE_DISPLAY_PORT);	387	SIGNAL_TYPE_DISPLAY_PORT,
		388	false);
388		389
389	link->dc->hwss.enable_stream(&pipes[i]);	390	link->dc->hwss.enable_stream(&pipes[i]);
390		391

drivers/gpu/drm/amd/display/dc/core/dc_resource.c+1−6

 	if (cur_stream == NULL)
 		return true;
-	/* If sink pointer changed, it means this is a hotplug, we should do
-	 * full hw setting.
-	 */
-	if (cur_stream->sink != new_stream->sink)
-		return true;
 	/* If output color space is changed, need to reprogram info frames */
 	if (cur_stream->output_color_space != new_stream->output_color_space)
 		return true;
 		dc_stream_release(context->streams[i]);
 		context->streams[i] = NULL;
 	}
+	context->stream_count = 0;
 }
 void dc_resource_state_copy_construct(

drivers/gpu/drm/amd/display/dc/dc.h+1−5

 /* forward declaration */
 struct aux_payload;
-#define DC_VER "3.2.132"
+#define DC_VER "3.2.135.1"
 #define MAX_SURFACES 3
 #define MAX_PLANES 6
 #endif
 	uint64_t vblank_alignment_dto_params;
 	uint8_t  vblank_alignment_max_frame_time_diff;
-	bool is_asymmetric_memory;
-	bool is_single_rank_dimm;
 };
 enum visual_confirm {
 	int min_dcfclk_mhz;
 };
-struct dc_state;
 struct resource_pool;
 struct dce_hwseq;
 struct gpu_info_soc_bounding_box_v1_0;
 void dc_deinit_callbacks(struct dc *dc);
 void dc_destroy(struct dc **dc);
-void dc_wait_for_vblank(struct dc *dc, struct dc_stream_state *stream);
 /*******************************************************************************
  * Surface Interfaces
  ******************************************************************************/

drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c+1−1

@@ -180,5 +180,5 @@ bool dc_dmub_srv_get_dmub_outbox0_msg(const struct dc *dc, struct dmcub_trace_bu · 179 unmodified lines
180		180
181	void dc_dmub_trace_event_control(struct dc *dc, bool enable)	181	void dc_dmub_trace_event_control(struct dc *dc, bool enable)
182	{	182	{
183	dm_helpers_dmub_outbox0_interrupt_control(dc->ctx, enable);	183	dm_helpers_dmub_outbox_interrupt_control(dc->ctx, enable);
184	}	184	}

drivers/gpu/drm/amd/display/dc/dc_dp_types.h+1−0

@@ -95,6 +95,7 @@ enum dc_dp_training_pattern { · 94 unmodified lines
95	DP_TRAINING_PATTERN_SEQUENCE_2,	95	DP_TRAINING_PATTERN_SEQUENCE_2,
96	DP_TRAINING_PATTERN_SEQUENCE_3,	96	DP_TRAINING_PATTERN_SEQUENCE_3,
97	DP_TRAINING_PATTERN_SEQUENCE_4,	97	DP_TRAINING_PATTERN_SEQUENCE_4,
		98	DP_TRAINING_PATTERN_VIDEOIDLE,
98	};	99	};
99		100
100	struct dc_link_settings {	101	struct dc_link_settings {

drivers/gpu/drm/amd/display/dc/dc_dsc.h+3−2

 		const struct dc_crtc_timing *timing,
 		struct dc_dsc_config *dsc_cfg);
-uint32_t dc_dsc_stream_bandwidth_in_kbps(uint32_t pix_clk_100hz, uint32_t bpp_x16);
+uint32_t dc_dsc_stream_bandwidth_in_kbps(const struct dc_crtc_timing *timing,
+		uint32_t bpp_x16, uint32_t num_slices_h, bool is_dp);
 void dc_dsc_get_policy_for_timing(const struct dc_crtc_timing *timing,
 		uint32_t max_target_bpp_limit_override_x16,
 void dc_dsc_policy_set_enable_dsc_when_not_needed(bool enable);
-uint32_t dc_dsc_stream_bandwidth_in_kbps(uint32_t pix_clk_100hz, uint32_t bpp_x16);
+void dc_dsc_policy_set_disable_dsc_stream_overhead(bool disable);
 #endif

drivers/gpu/drm/amd/display/dc/dc_hw_types.h+1−0

@@ -739,6 +739,7 @@ struct dc_dsc_config { · 738 unmodified lines
739	uint32_t version_minor; /* DSC minor version. Full version is formed as 1.version_minor. */	739	uint32_t version_minor; /* DSC minor version. Full version is formed as 1.version_minor. */
740	bool ycbcr422_simple; /* Tell DSC engine to convert YCbCr 4:2:2 to 'YCbCr 4:2:2 simple'. */	740	bool ycbcr422_simple; /* Tell DSC engine to convert YCbCr 4:2:2 to 'YCbCr 4:2:2 simple'. */
741	int32_t rc_buffer_size; /* DSC RC buffer block size in bytes */	741	int32_t rc_buffer_size; /* DSC RC buffer block size in bytes */
		742	bool is_dp; /* indicate if DSC is applied based on DP's capability */
742	};	743	};
743	struct dc_crtc_timing {	744	struct dc_crtc_timing {
744	uint32_t h_total;	745	uint32_t h_total;

drivers/gpu/drm/amd/display/dc/dc_link.h+1−9

 	dc_link_fec_enabled
 };
-enum lttpr_mode {
-	LTTPR_MODE_NON_LTTPR,
-	LTTPR_MODE_TRANSPARENT,
-	LTTPR_MODE_NON_TRANSPARENT,
-};
 struct dc_link_status {
 	bool link_active;
 	struct dpcd_caps *dpcd_caps;
 	/* TODO: Rename. Flag an endpoint as having a programmable mapping to a
 	 * DIG encoder. */
 	bool is_dig_mapping_flexible;
+	bool hpd_status; /* HPD status of link without physical HPD pin. */
 	bool edp_sink_present;
 void dc_link_set_drive_settings(struct dc *dc,
 				struct link_training_settings *lt_settings,
 				const struct dc_link *link);
-void dc_link_perform_link_training(struct dc *dc,
-				   struct dc_link_settings *link_setting,
-				   bool skip_video_pattern);
 void dc_link_set_preferred_link_settings(struct dc *dc,
 					 struct dc_link_settings *link_setting,
 					 struct dc_link *link);

drivers/gpu/drm/amd/display/dc/dc_types.h+2−1

 	dc_connection_none,
 	dc_connection_single,
 	dc_connection_mst_branch,
-	dc_connection_active_dongle
+	dc_connection_sst_branch
 };
 struct dc_csc_adjustments {
 	uint32_t branch_overall_throughput_0_mps; /* In MPs */
 	uint32_t branch_overall_throughput_1_mps; /* In MPs */
 	uint32_t branch_max_line_width;
+	bool is_dp;
 };
 struct dc_golden_table {

drivers/gpu/drm/amd/display/dc/dce/dce_aux.c+19−0

@@ -595,6 +595,25 @@ int dce_aux_transfer_raw(struct ddc_service *ddc, · 594 unmodified lines
595	return res;	595	return res;
596	}	596	}
597		597
		598	int dce_aux_transfer_dmub_raw(struct ddc_service *ddc,
		599	struct aux_payload *payload,
		600	enum aux_return_code_type *operation_result)
		601	{
		602	struct ddc *ddc_pin = ddc->ddc_pin;
		603
		604	if (ddc_pin != NULL) {
		605	struct dce_aux *aux_engine = ddc->ctx->dc->res_pool->engines[ddc_pin->pin_data->en];
		606	/* XXX: Workaround to configure ddc channels for aux transactions */
		607	if (!acquire(aux_engine, ddc_pin)) {
		608	*operation_result = AUX_RET_ERROR_ENGINE_ACQUIRE;
		609	return -1;
		610	}
		611	release_engine(aux_engine);
		612	}
		613
		614	return dm_helper_dmub_aux_transfer_sync(ddc->ctx, ddc->link, payload, operation_result);
		615	}
		616
598	#define AUX_MAX_RETRIES 7	617	#define AUX_MAX_RETRIES 7
599	#define AUX_MAX_DEFER_RETRIES 7	618	#define AUX_MAX_DEFER_RETRIES 7
600	#define AUX_MAX_I2C_DEFER_RETRIES 7	619	#define AUX_MAX_I2C_DEFER_RETRIES 7

drivers/gpu/drm/amd/display/dc/dce/dce_aux.h+3−0

@@ -304,6 +304,9 @@ int dce_aux_transfer_raw(struct ddc_service *ddc, · 303 unmodified lines
304	struct aux_payload *cmd,	304	struct aux_payload *cmd,
305	enum aux_return_code_type *operation_result);	305	enum aux_return_code_type *operation_result);
306		306
		307	int dce_aux_transfer_dmub_raw(struct ddc_service *ddc,
		308	struct aux_payload *payload,
		309	enum aux_return_code_type *operation_result);
307	bool dce_aux_transfer_with_retries(struct ddc_service *ddc,	310	bool dce_aux_transfer_with_retries(struct ddc_service *ddc,
308	struct aux_payload *cmd);	311	struct aux_payload *cmd);
309		312

drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c+30−0

 {
 	union dmub_rb_cmd cmd;
 	uint32_t fractional_pwm = (dc->dc->config.disable_fractional_pwm == false) ? 1 : 0;
+	uint32_t edp_id_count = dc->dc_edp_id_count;
+	int i;
+	uint8_t panel_mask = 0;
+	for (i = 0; i < edp_id_count; i++)
+		panel_mask |= 0x01 << i;
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.abm_set_pwm_frac.header.type = DMUB_CMD__ABM;
 	cmd.abm_set_pwm_frac.header.sub_type = DMUB_CMD__ABM_SET_PWM_FRAC;
 	cmd.abm_set_pwm_frac.abm_set_pwm_frac_data.fractional_pwm = fractional_pwm;
+	cmd.abm_set_pwm_frac.abm_set_pwm_frac_data.version = DMUB_CMD_ABM_CONTROL_VERSION_1;
+	cmd.abm_set_pwm_frac.abm_set_pwm_frac_data.panel_mask = panel_mask;
 	cmd.abm_set_pwm_frac.header.payload_bytes = sizeof(struct dmub_cmd_abm_set_pwm_frac_data);
 	dc_dmub_srv_cmd_queue(dc->dmub_srv, &cmd);
 {
 	union dmub_rb_cmd cmd;
 	struct dc_context *dc = abm->ctx;
+	struct dc_link *edp_links[MAX_NUM_EDP];
+	int i;
+	int edp_num;
+	uint8_t panel_mask = 0;
+	get_edp_links(dc->dc, edp_links, &edp_num);
+	for (i = 0; i < edp_num; i++) {
+		if (edp_links[i]->link_status.link_active)
+			panel_mask |= (0x01 << i);
+	}
 	memset(&cmd, 0, sizeof(cmd));
 	cmd.abm_set_level.header.type = DMUB_CMD__ABM;
 	cmd.abm_set_level.header.sub_type = DMUB_CMD__ABM_SET_LEVEL;
 	cmd.abm_set_level.abm_set_level_data.level = level;
+	cmd.abm_set_level.abm_set_level_data.version = DMUB_CMD_ABM_CONTROL_VERSION_1;
+	cmd.abm_set_level.abm_set_level_data.panel_mask = panel_mask;
 	cmd.abm_set_level.header.payload_bytes = sizeof(struct dmub_cmd_abm_set_level_data);
 	dc_dmub_srv_cmd_queue(dc->dmub_srv, &cmd);
 {
 	union dmub_rb_cmd cmd;
 	struct dc_context *dc = abm->ctx;
+	uint32_t edp_id_count = dc->dc_edp_id_count;
+	int i;
+	uint8_t panel_mask = 0;
+	for (i = 0; i < edp_id_count; i++)
+		panel_mask |= 0x01 << i;
 	// TODO: Optimize by only reading back final 4 bytes
 	dmub_flush_buffer_mem(&dc->dmub_srv->dmub->scratch_mem_fb);
 	cmd.abm_init_config.header.sub_type = DMUB_CMD__ABM_INIT_CONFIG;
 	cmd.abm_init_config.abm_init_config_data.src.quad_part = dc->dmub_srv->dmub->scratch_mem_fb.gpu_addr;
 	cmd.abm_init_config.abm_init_config_data.bytes = bytes;
+	cmd.abm_init_config.abm_init_config_data.version = DMUB_CMD_ABM_CONTROL_VERSION_1;
+	cmd.abm_init_config.abm_init_config_data.panel_mask = panel_mask;
 	cmd.abm_init_config.header.payload_bytes = sizeof(struct dmub_cmd_abm_init_config_data);
 	dc_dmub_srv_cmd_queue(dc->dmub_srv, &cmd);

drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c+6−3

 	int vtaps_c = scl_data->taps.v_taps_c;
 	int ceil_vratio = dc_fixpt_ceil(scl_data->ratios.vert);
 	int ceil_vratio_c = dc_fixpt_ceil(scl_data->ratios.vert_c);
-	enum lb_memory_config mem_cfg = LB_MEMORY_CONFIG_0;
-	if (dpp->base.ctx->dc->debug.use_max_lb)
+	if (dpp->base.ctx->dc->debug.use_max_lb) {
-		return mem_cfg;
+		if (scl_data->format == PIXEL_FORMAT_420BPP8
+				|| scl_data->format == PIXEL_FORMAT_420BPP10)
+			return LB_MEMORY_CONFIG_3;
+		return LB_MEMORY_CONFIG_0;
+	}
 	dpp->base.caps->dscl_calc_lb_num_partitions(
 			scl_data, LB_MEMORY_CONFIG_1, &num_part_y, &num_part_c);

drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c+16−139

@@ -2956,35 +2956,6 @@ void dcn10_program_pipe( · 2955 unmodified lines
2956	{	2956	{
2957	struct dce_hwseq *hws = dc->hwseq;	2957	struct dce_hwseq *hws = dc->hwseq;
2958		2958
2959	if (pipe_ctx->plane_state->update_flags.bits.full_update)
2960	dcn10_enable_plane(dc, pipe_ctx, context);
2961
2962	dcn10_update_dchubp_dpp(dc, pipe_ctx, context);
2963
2964	hws->funcs.set_hdr_multiplier(pipe_ctx);
2965
2966	if (pipe_ctx->plane_state->update_flags.bits.full_update \|\|
2967	pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change \|\|
2968	pipe_ctx->plane_state->update_flags.bits.gamma_change)
2969	hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
2970
2971	/* dcn10_translate_regamma_to_hw_format takes 750us to finish
2972	* only do gamma programming for full update.
2973	* TODO: This can be further optimized/cleaned up
2974	* Always call this for now since it does memcmp inside before
2975	* doing heavy calculation and programming
2976	*/
2977	if (pipe_ctx->plane_state->update_flags.bits.full_update)
2978	hws->funcs.set_output_transfer_func(dc, pipe_ctx, pipe_ctx->stream);
2979	}
2980
2981	static void dcn10_program_all_pipe_in_tree(
2982	struct dc *dc,
2983	struct pipe_ctx *pipe_ctx,
2984	struct dc_state *context)
2985	{
2986	struct dce_hwseq *hws = dc->hwseq;
2987
2988	if (pipe_ctx->top_pipe == NULL) {	2959	if (pipe_ctx->top_pipe == NULL) {
2989	bool blank = !is_pipe_tree_visible(pipe_ctx);	2960	bool blank = !is_pipe_tree_visible(pipe_ctx);
2990		2961
@@ -3004,35 +2975,26 @@ static void dcn10_program_all_pipe_in_tree( · 13 unmodified lines
3004	hws->funcs.blank_pixel_data(dc, pipe_ctx, blank);	2975	hws->funcs.blank_pixel_data(dc, pipe_ctx, blank);
3005	}	2976	}
3006		2977
3007	if (pipe_ctx->plane_state != NULL)	2978	if (pipe_ctx->plane_state->update_flags.bits.full_update)
3008	hws->funcs.program_pipe(dc, pipe_ctx, context);	2979	dcn10_enable_plane(dc, pipe_ctx, context);
3009
3010	if (pipe_ctx->bottom_pipe != NULL && pipe_ctx->bottom_pipe != pipe_ctx)
3011	dcn10_program_all_pipe_in_tree(dc, pipe_ctx->bottom_pipe, context);
3012	}
3013
3014	static struct pipe_ctx *dcn10_find_top_pipe_for_stream(
3015	struct dc *dc,
3016	struct dc_state *context,
3017	const struct dc_stream_state *stream)
3018	{
3019	int i;
3020		2980
3021	for (i = 0; i < dc->res_pool->pipe_count; i++) {	2981	dcn10_update_dchubp_dpp(dc, pipe_ctx, context);
3022	struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
3023	struct pipe_ctx *old_pipe_ctx =
3024	&dc->current_state->res_ctx.pipe_ctx[i];
3025		2982
3026	if (!pipe_ctx->plane_state && !old_pipe_ctx->plane_state)	2983	hws->funcs.set_hdr_multiplier(pipe_ctx);
3027	continue;
3028		2984
3029	if (pipe_ctx->stream != stream)	2985	if (pipe_ctx->plane_state->update_flags.bits.full_update \|\|
3030	continue;	2986	pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change \|\|
		2987	pipe_ctx->plane_state->update_flags.bits.gamma_change)
		2988	hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
3031		2989
3032	if (!pipe_ctx->top_pipe && !pipe_ctx->prev_odm_pipe)	2990	/* dcn10_translate_regamma_to_hw_format takes 750us to finish
3033	return pipe_ctx;	2991	* only do gamma programming for full update.
3034	}	2992	* TODO: This can be further optimized/cleaned up
3035	return NULL;	2993	* Always call this for now since it does memcmp inside before
		2994	* doing heavy calculation and programming
		2995	*/
		2996	if (pipe_ctx->plane_state->update_flags.bits.full_update)
		2997	hws->funcs.set_output_transfer_func(dc, pipe_ctx, pipe_ctx->stream);
3036	}	2998	}
3037		2999
3038	void dcn10_wait_for_pending_cleared(struct dc *dc,	3000	void dcn10_wait_for_pending_cleared(struct dc *dc,
@@ -3067,91 +3029,6 @@ void dcn10_wait_for_pending_cleared(struct dc *dc, · 28 unmodified lines
3067	}	3029	}
3068	}	3030	}
3069		3031
3070	void dcn10_apply_ctx_for_surface(
3071	struct dc *dc,
3072	const struct dc_stream_state *stream,
3073	int num_planes,
3074	struct dc_state *context)
3075	{
3076	struct dce_hwseq *hws = dc->hwseq;
3077	int i;
3078	struct timing_generator *tg;
3079	uint32_t underflow_check_delay_us;
3080	bool interdependent_update = false;
3081	struct pipe_ctx *top_pipe_to_program =
3082	dcn10_find_top_pipe_for_stream(dc, context, stream);
3083	DC_LOGGER_INIT(dc->ctx->logger);
3084
3085	// Clear pipe_ctx flag
3086	for (i = 0; i < dc->res_pool->pipe_count; i++) {
3087	struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
3088	pipe_ctx->update_flags.raw = 0;
3089	}
3090
3091	if (!top_pipe_to_program)
3092	return;
3093
3094	tg = top_pipe_to_program->stream_res.tg;
3095
3096	interdependent_update = top_pipe_to_program->plane_state &&
3097	top_pipe_to_program->plane_state->update_flags.bits.full_update;
3098
3099	underflow_check_delay_us = dc->debug.underflow_assert_delay_us;
3100
3101	if (underflow_check_delay_us != 0xFFFFFFFF && hws->funcs.did_underflow_occur)
3102	ASSERT(hws->funcs.did_underflow_occur(dc, top_pipe_to_program));
3103
3104	if (underflow_check_delay_us != 0xFFFFFFFF)
3105	udelay(underflow_check_delay_us);
3106
3107	if (underflow_check_delay_us != 0xFFFFFFFF && hws->funcs.did_underflow_occur)
3108	ASSERT(hws->funcs.did_underflow_occur(dc, top_pipe_to_program));
3109
3110	if (num_planes == 0) {
3111	/* OTG blank before remove all front end */
3112	hws->funcs.blank_pixel_data(dc, top_pipe_to_program, true);
3113	}
3114
3115	/* Disconnect unused mpcc */
3116	for (i = 0; i < dc->res_pool->pipe_count; i++) {
3117	struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
3118	struct pipe_ctx *old_pipe_ctx =
3119	&dc->current_state->res_ctx.pipe_ctx[i];
3120
3121	if ((!pipe_ctx->plane_state \|\|
3122	pipe_ctx->stream_res.tg != old_pipe_ctx->stream_res.tg) &&
3123	old_pipe_ctx->plane_state &&
3124	old_pipe_ctx->stream_res.tg == tg) {
3125
3126	hws->funcs.plane_atomic_disconnect(dc, old_pipe_ctx);
3127	pipe_ctx->update_flags.bits.disable = 1;
3128
3129	DC_LOG_DC("Reset mpcc for pipe %d\n",
3130	old_pipe_ctx->pipe_idx);
3131	}
3132	}
3133
3134	if (num_planes > 0)
3135	dcn10_program_all_pipe_in_tree(dc, top_pipe_to_program, context);
3136
3137	/* Program secondary blending tree and writeback pipes */
3138	if ((stream->num_wb_info > 0) && (hws->funcs.program_all_writeback_pipes_in_tree))
3139	hws->funcs.program_all_writeback_pipes_in_tree(dc, stream, context);
3140	if (interdependent_update)
3141	for (i = 0; i < dc->res_pool->pipe_count; i++) {
3142	struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
3143	/* Skip inactive pipes and ones already updated */
3144	if (!pipe_ctx->stream \|\| pipe_ctx->stream == stream \|\|
3145	!pipe_ctx->plane_state \|\| !tg->funcs->is_tg_enabled(tg))
3146	continue;
3147
3148	pipe_ctx->plane_res.hubp->funcs->hubp_setup_interdependent(
3149	pipe_ctx->plane_res.hubp,
3150	&pipe_ctx->dlg_regs,
3151	&pipe_ctx->ttu_regs);
3152	}
3153	}
3154
3155	void dcn10_post_unlock_program_front_end(	3032	void dcn10_post_unlock_program_front_end(
3156	struct dc *dc,	3033	struct dc *dc,
3157	struct dc_state *context)	3034	struct dc_state *context)

drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h+0−5

 		struct dc *dc,
 		struct dc_state *context,
 		bool lock);
-void dcn10_apply_ctx_for_surface(
-		struct dc *dc,
-		const struct dc_stream_state *stream,
-		int num_planes,
-		struct dc_state *context);
 void dcn10_post_unlock_program_front_end(
 		struct dc *dc,
 		struct dc_state *context);

drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c+3−1

 #include "hw_sequencer_private.h"
 #include "dce110/dce110_hw_sequencer.h"
 #include "dcn10_hw_sequencer.h"
+#include "dcn20/dcn20_hwseq.h"
 static const struct hw_sequencer_funcs dcn10_funcs = {
 	.program_gamut_remap = dcn10_program_gamut_remap,
 	.init_hw = dcn10_init_hw,
 	.power_down_on_boot = dcn10_power_down_on_boot,
 	.apply_ctx_to_hw = dce110_apply_ctx_to_hw,
-	.apply_ctx_for_surface = dcn10_apply_ctx_for_surface,
+	.apply_ctx_for_surface = NULL,
+	.program_front_end_for_ctx = dcn20_program_front_end_for_ctx,
 	.post_unlock_program_front_end = dcn10_post_unlock_program_front_end,
 	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
 	.update_plane_addr = dcn10_update_plane_addr,

drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp_cm.c+2−2

@@ -482,7 +482,7 @@ bool dpp20_program_blnd_lut( · 481 unmodified lines
482	next_mode = LUT_RAM_A;	482	next_mode = LUT_RAM_A;
483		483
484	dpp20_power_on_blnd_lut(dpp_base, true);	484	dpp20_power_on_blnd_lut(dpp_base, true);
485	dpp20_configure_blnd_lut(dpp_base, next_mode == LUT_RAM_A ? true:false);	485	dpp20_configure_blnd_lut(dpp_base, next_mode == LUT_RAM_A);
486		486
487	if (next_mode == LUT_RAM_A)	487	if (next_mode == LUT_RAM_A)
488	dpp20_program_blnd_luta_settings(dpp_base, params);	488	dpp20_program_blnd_luta_settings(dpp_base, params);
@@ -893,7 +893,7 @@ bool dpp20_program_shaper( · 404 unmodified lines
893	else	893	else
894	next_mode = LUT_RAM_A;	894	next_mode = LUT_RAM_A;
895		895
896	dpp20_configure_shaper_lut(dpp_base, next_mode == LUT_RAM_A ? true:false);	896	dpp20_configure_shaper_lut(dpp_base, next_mode == LUT_RAM_A);
897		897
898	if (next_mode == LUT_RAM_A)	898	if (next_mode == LUT_RAM_A)
899	dpp20_program_shaper_luta_settings(dpp_base, params);	899	dpp20_program_shaper_luta_settings(dpp_base, params);

drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c+5−1

 		if (pipe->plane_state && !pipe->top_pipe) {
 			while (pipe) {
-				dcn20_program_pipe(dc, pipe, context);
+				if (hws->funcs.program_pipe)
+					hws->funcs.program_pipe(dc, pipe, context);
+				else
+					dcn20_program_pipe(dc, pipe, context);
 				pipe = pipe->bottom_pipe;
 			}
 			/* Program secondary blending tree and writeback pipes */

drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c+1−1

@@ -3667,7 +3667,7 @@ static bool dcn20_resource_construct( · 3666 unmodified lines
3667	int i;	3667	int i;
3668	struct dc_context *ctx = dc->ctx;	3668	struct dc_context *ctx = dc->ctx;
3669	struct irq_service_init_data init_data;	3669	struct irq_service_init_data init_data;
3670	struct ddc_service_init_data ddc_init_data;	3670	struct ddc_service_init_data ddc_init_data = {0};
3671	struct _vcs_dpi_soc_bounding_box_st *loaded_bb =	3671	struct _vcs_dpi_soc_bounding_box_st *loaded_bb =
3672	get_asic_rev_soc_bb(ctx->asic_id.hw_internal_rev);	3672	get_asic_rev_soc_bb(ctx->asic_id.hw_internal_rev);
3673	struct _vcs_dpi_ip_params_st *loaded_ip =	3673	struct _vcs_dpi_ip_params_st *loaded_ip =

drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c+1−1

@@ -218,7 +218,7 @@ bool dcn21_set_backlight_level(struct pipe_ctx *pipe_ctx, · 217 unmodified lines
218	cmd.abm_set_backlight.header.sub_type = DMUB_CMD__ABM_SET_BACKLIGHT;	218	cmd.abm_set_backlight.header.sub_type = DMUB_CMD__ABM_SET_BACKLIGHT;
219	cmd.abm_set_backlight.abm_set_backlight_data.frame_ramp = frame_ramp;	219	cmd.abm_set_backlight.abm_set_backlight_data.frame_ramp = frame_ramp;
220	cmd.abm_set_backlight.abm_set_backlight_data.backlight_user_level = backlight_pwm_u16_16;	220	cmd.abm_set_backlight.abm_set_backlight_data.backlight_user_level = backlight_pwm_u16_16;
221	cmd.abm_set_backlight.abm_set_backlight_data.version = DMUB_CMD_ABM_SET_BACKLIGHT_VERSION_1;	221	cmd.abm_set_backlight.abm_set_backlight_data.version = DMUB_CMD_ABM_CONTROL_VERSION_1;
222	cmd.abm_set_backlight.abm_set_backlight_data.panel_mask = (0x01 << panel_cntl->inst);	222	cmd.abm_set_backlight.abm_set_backlight_data.panel_mask = (0x01 << panel_cntl->inst);
223	cmd.abm_set_backlight.header.payload_bytes = sizeof(struct dmub_cmd_abm_set_backlight_data);	223	cmd.abm_set_backlight.header.payload_bytes = sizeof(struct dmub_cmd_abm_set_backlight_data);
224		224

drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c+21−12

@@ -1575,10 +1575,12 @@ static struct _vcs_dpi_voltage_scaling_st construct_low_pstate_lvl(struct clk_li · 1574 unmodified lines
1575	low_pstate_lvl.phyclk_d18_mhz = dcn2_1_soc.clock_limits[high_voltage_lvl].phyclk_d18_mhz;	1575	low_pstate_lvl.phyclk_d18_mhz = dcn2_1_soc.clock_limits[high_voltage_lvl].phyclk_d18_mhz;
1576	low_pstate_lvl.phyclk_mhz = dcn2_1_soc.clock_limits[high_voltage_lvl].phyclk_mhz;	1576	low_pstate_lvl.phyclk_mhz = dcn2_1_soc.clock_limits[high_voltage_lvl].phyclk_mhz;
1577		1577
1578	for (i = clk_table->num_entries; i > 1; i--)	1578	if (clk_table->num_entries < MAX_NUM_DPM_LVL) {
1579	clk_table->entries[i] = clk_table->entries[i-1];	1579	for (i = clk_table->num_entries; i > 1; i--)
1580	clk_table->entries[1] = clk_table->entries[0];	1580	clk_table->entries[i] = clk_table->entries[i-1];
1581	clk_table->num_entries++;	1581	clk_table->entries[1] = clk_table->entries[0];
		1582	clk_table->num_entries++;
		1583	}
1582		1584
1583	return low_pstate_lvl;	1585	return low_pstate_lvl;
1584	}	1586	}
@@ -1610,10 +1612,6 @@ static void update_bw_bounding_box(struct dc dc, struct clk_bw_params bw_param · 25 unmodified lines
1610	}	1612	}
1611	}	1613	}
1612		1614
1613	/* clk_table[1] is reserved for min DF PState. skip here to fill in later. */
1614	if (i == 1)
1615	k++;
1616
1617	clock_limits[k].state = k;	1615	clock_limits[k].state = k;
1618	clock_limits[k].dcfclk_mhz = clk_table->entries[i].dcfclk_mhz;	1616	clock_limits[k].dcfclk_mhz = clk_table->entries[i].dcfclk_mhz;
1619	clock_limits[k].fabricclk_mhz = clk_table->entries[i].fclk_mhz;	1617	clock_limits[k].fabricclk_mhz = clk_table->entries[i].fclk_mhz;
@@ -1630,14 +1628,25 @@ static void update_bw_bounding_box(struct dc dc, struct clk_bw_params bw_param · 10 unmodified lines
1630		1628
1631	k++;	1629	k++;
1632	}	1630	}
1633	for (i = 0; i < clk_table->num_entries + 1; i++)	1631
1634	dcn2_1_soc.clock_limits[i] = clock_limits[i];	1632	if (clk_table->num_entries >= MAX_NUM_DPM_LVL) {
		1633	for (i = 0; i < clk_table->num_entries + 1; i++)
		1634	dcn2_1_soc.clock_limits[i] = clock_limits[i];
		1635	} else {
		1636	dcn2_1_soc.clock_limits[0] = clock_limits[0];
		1637	for (i = 2; i < clk_table->num_entries + 1; i++) {
		1638	dcn2_1_soc.clock_limits[i] = clock_limits[i - 1];
		1639	dcn2_1_soc.clock_limits[i].state = i;
		1640	}
		1641	}
		1642
1635	if (clk_table->num_entries) {	1643	if (clk_table->num_entries) {
1636	dcn2_1_soc.num_states = clk_table->num_entries + 1;
1637	/* fill in min DF PState */	1644	/* fill in min DF PState */
1638	dcn2_1_soc.clock_limits[1] = construct_low_pstate_lvl(clk_table, closest_clk_lvl);	1645	dcn2_1_soc.clock_limits[1] = construct_low_pstate_lvl(clk_table, closest_clk_lvl);
		1646	dcn2_1_soc.num_states = clk_table->num_entries;
1639	/* duplicate last level */	1647	/* duplicate last level */
1640	dcn2_1_soc.clock_limits[dcn2_1_soc.num_states] = dcn2_1_soc.clock_limits[dcn2_1_soc.num_states - 1];	1648	dcn2_1_soc.clock_limits[dcn2_1_soc.num_states] =
		1649	dcn2_1_soc.clock_limits[dcn2_1_soc.num_states - 1];
1641	dcn2_1_soc.clock_limits[dcn2_1_soc.num_states].state = dcn2_1_soc.num_states;	1650	dcn2_1_soc.clock_limits[dcn2_1_soc.num_states].state = dcn2_1_soc.num_states;
1642	}	1651	}
1643		1652

drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_stream_encoder.h+4−0

 	SRI(DP_MSE_RATE_UPDATE, DP, id), \
 	SRI(DP_PIXEL_FORMAT, DP, id), \
 	SRI(DP_SEC_CNTL, DP, id), \
+	SRI(DP_SEC_CNTL1, DP, id), \
 	SRI(DP_SEC_CNTL2, DP, id), \
+	SRI(DP_SEC_CNTL5, DP, id), \
 	SRI(DP_SEC_CNTL6, DP, id), \
 	SRI(DP_STEER_FIFO, DP, id), \
 	SRI(DP_VID_M, DP, id), \
 	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP5_ENABLE, mask_sh),\
 	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP6_ENABLE, mask_sh),\
 	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP7_ENABLE, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL1, DP_SEC_GSP5_LINE_REFERENCE, mask_sh),\
 	SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP7_SEND, mask_sh),\
+	SE_SF(DP0_DP_SEC_CNTL5, DP_SEC_GSP5_LINE_NUM, mask_sh),\
 	SE_SF(DP0_DP_SEC_CNTL6, DP_SEC_GSP7_LINE_NUM, mask_sh),\
 	SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP11_PPS, mask_sh),\
 	SE_SF(DP0_DP_GSP11_CNTL, DP_SEC_GSP11_ENABLE, mask_sh),\

drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c+1−1

@@ -874,7 +874,7 @@ bool mpc3_program_shaper( · 873 unmodified lines
874	else	874	else
875	next_mode = LUT_RAM_A;	875	next_mode = LUT_RAM_A;
876		876
877	mpc3_configure_shaper_lut(mpc, next_mode == LUT_RAM_A ? true:false, rmu_idx);	877	mpc3_configure_shaper_lut(mpc, next_mode == LUT_RAM_A, rmu_idx);
878		878
879	if (next_mode == LUT_RAM_A)	879	if (next_mode == LUT_RAM_A)
880	mpc3_program_shaper_luta_settings(mpc, params, rmu_idx);	880	mpc3_program_shaper_luta_settings(mpc, params, rmu_idx);

drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c+1−1

@@ -97,7 +97,7 @@ void optc3_lock_doublebuffer_disable(struct timing_generator *optc) · 96 unmodified lines
97	MASTER_UPDATE_LOCK_DB_END_Y, 0);	97	MASTER_UPDATE_LOCK_DB_END_Y, 0);
98		98
99	REG_UPDATE(OTG_GLOBAL_CONTROL2, GLOBAL_UPDATE_LOCK_EN, 0);	99	REG_UPDATE(OTG_GLOBAL_CONTROL2, GLOBAL_UPDATE_LOCK_EN, 0);
100	REG_UPDATE(OTG_GLOBAL_CONTROL0, MASTER_UPDATE_LOCK_DB_EN, 1);	100	REG_UPDATE(OTG_GLOBAL_CONTROL0, MASTER_UPDATE_LOCK_DB_EN, 0);
101	}	101	}
102		102
103	void optc3_lock(struct timing_generator *optc)	103	void optc3_lock(struct timing_generator *optc)

drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c+1−1

@@ -2538,7 +2538,7 @@ static bool dcn30_resource_construct( · 2537 unmodified lines
2538	int i;	2538	int i;
2539	struct dc_context *ctx = dc->ctx;	2539	struct dc_context *ctx = dc->ctx;
2540	struct irq_service_init_data init_data;	2540	struct irq_service_init_data init_data;
2541	struct ddc_service_init_data ddc_init_data;	2541	struct ddc_service_init_data ddc_init_data = {0};
2542	uint32_t pipe_fuses = read_pipe_fuses(ctx);	2542	uint32_t pipe_fuses = read_pipe_fuses(ctx);
2543	uint32_t num_pipes = 0;	2543	uint32_t num_pipes = 0;
2544		2544

drivers/gpu/drm/amd/display/dc/dm_helpers.h+8−1

 #include "dc.h"
 struct dp_mst_stream_allocation_table;
+struct aux_payload;
+enum aux_return_code_type;
 /*
  * Allocate memory accessible by the GPU
 		struct dc_context *ctx,
 		struct dc_clocks *clks);
-bool dm_helpers_dmub_outbox0_interrupt_control(struct dc_context *ctx, bool enable);
+bool dm_helpers_dmub_outbox_interrupt_control(struct dc_context *ctx, bool enable);
+int dm_helper_dmub_aux_transfer_sync(
+		struct dc_context *ctx,
+		const struct dc_link *link,
+		struct aux_payload *payload,
+		enum aux_return_code_type *operation_result);
 #endif /* __DM_HELPERS__ */

drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c+13−13

drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c+13−13

drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c+29−29

@@ -148,7 +148,7 @@ static double CalculateDCCConfiguration( · 147 unmodified lines
148	bool DCCProgrammingAssumesScanDirectionUnknown,	148	bool DCCProgrammingAssumesScanDirectionUnknown,
149	unsigned int ViewportWidth,	149	unsigned int ViewportWidth,
150	unsigned int ViewportHeight,	150	unsigned int ViewportHeight,
151	double DETBufferSize,	151	unsigned int DETBufferSize,
152	unsigned int RequestHeight256Byte,	152	unsigned int RequestHeight256Byte,
153	unsigned int SwathHeight,	153	unsigned int SwathHeight,
154	enum dm_swizzle_mode TilingFormat,	154	enum dm_swizzle_mode TilingFormat,
@@ -289,7 +289,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 134 unmodified lines
289	unsigned int MaxLineBufferLines,	289	unsigned int MaxLineBufferLines,
290	unsigned int LineBufferSize,	290	unsigned int LineBufferSize,
291	unsigned int DPPOutputBufferPixels,	291	unsigned int DPPOutputBufferPixels,
292	double DETBufferSizeInKByte,	292	unsigned int DETBufferSizeInKByte,
293	unsigned int WritebackInterfaceLumaBufferSize,	293	unsigned int WritebackInterfaceLumaBufferSize,
294	unsigned int WritebackInterfaceChromaBufferSize,	294	unsigned int WritebackInterfaceChromaBufferSize,
295	double DCFCLK,	295	double DCFCLK,
@@ -354,11 +354,11 @@ static void CalculateDCFCLKDeepSleep( · 58 unmodified lines
354	double DPPCLK[],	354	double DPPCLK[],
355	double *DCFCLKDeepSleep);	355	double *DCFCLKDeepSleep);
356	static void CalculateDETBufferSize(	356	static void CalculateDETBufferSize(
357	double DETBufferSizeInKByte,	357	unsigned int DETBufferSizeInKByte,
358	unsigned int SwathHeightY,	358	unsigned int SwathHeightY,
359	unsigned int SwathHeightC,	359	unsigned int SwathHeightC,
360	double *DETBufferSizeY,	360	unsigned int *DETBufferSizeY,
361	double *DETBufferSizeC);	361	unsigned int *DETBufferSizeC);
362	static void CalculateUrgentBurstFactor(	362	static void CalculateUrgentBurstFactor(
363	unsigned int DETBufferSizeInKByte,	363	unsigned int DETBufferSizeInKByte,
364	unsigned int SwathHeightY,	364	unsigned int SwathHeightY,
@@ -1074,7 +1074,7 @@ static double CalculateDCCConfiguration( · 709 unmodified lines
1074	bool DCCProgrammingAssumesScanDirectionUnknown,	1074	bool DCCProgrammingAssumesScanDirectionUnknown,
1075	unsigned int ViewportWidth,	1075	unsigned int ViewportWidth,
1076	unsigned int ViewportHeight,	1076	unsigned int ViewportHeight,
1077	double DETBufferSize,	1077	unsigned int DETBufferSize,
1078	unsigned int RequestHeight256Byte,	1078	unsigned int RequestHeight256Byte,
1079	unsigned int SwathHeight,	1079	unsigned int SwathHeight,
1080	enum dm_swizzle_mode TilingFormat,	1080	enum dm_swizzle_mode TilingFormat,
@@ -2246,7 +2246,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 1165 unmodified lines
2246	}	2246	}
2247		2247
2248	CalculateUrgentBurstFactor(	2248	CalculateUrgentBurstFactor(
2249	mode_lib->vba.DETBufferSizeInKByte,	2249	mode_lib->vba.DETBufferSizeInKByte[0],
2250	mode_lib->vba.SwathHeightY[k],	2250	mode_lib->vba.SwathHeightY[k],
2251	mode_lib->vba.SwathHeightC[k],	2251	mode_lib->vba.SwathHeightC[k],
2252	locals->SwathWidthY[k],	2252	locals->SwathWidthY[k],
@@ -2415,7 +2415,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 162 unmodified lines
2415	mode_lib->vba.MaxLineBufferLines,	2415	mode_lib->vba.MaxLineBufferLines,
2416	mode_lib->vba.LineBufferSize,	2416	mode_lib->vba.LineBufferSize,
2417	mode_lib->vba.DPPOutputBufferPixels,	2417	mode_lib->vba.DPPOutputBufferPixels,
2418	mode_lib->vba.DETBufferSizeInKByte,	2418	mode_lib->vba.DETBufferSizeInKByte[0],
2419	mode_lib->vba.WritebackInterfaceLumaBufferSize,	2419	mode_lib->vba.WritebackInterfaceLumaBufferSize,
2420	mode_lib->vba.WritebackInterfaceChromaBufferSize,	2420	mode_lib->vba.WritebackInterfaceChromaBufferSize,
2421	mode_lib->vba.DCFCLK,	2421	mode_lib->vba.DCFCLK,
@@ -2588,7 +2588,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 166 unmodified lines
2588	false, // We should always know the direction DCCProgrammingAssumesScanDirectionUnknown,	2588	false, // We should always know the direction DCCProgrammingAssumesScanDirectionUnknown,
2589	mode_lib->vba.ViewportWidth[k],	2589	mode_lib->vba.ViewportWidth[k],
2590	mode_lib->vba.ViewportHeight[k],	2590	mode_lib->vba.ViewportHeight[k],
2591	mode_lib->vba.DETBufferSizeInKByte * 1024,	2591	mode_lib->vba.DETBufferSizeInKByte[0] * 1024,
2592	locals->BlockHeight256BytesY[k],	2592	locals->BlockHeight256BytesY[k],
2593	mode_lib->vba.SwathHeightY[k],	2593	mode_lib->vba.SwathHeightY[k],
2594	mode_lib->vba.SurfaceTiling[k],	2594	mode_lib->vba.SurfaceTiling[k],
@@ -2689,13 +2689,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 94 unmodified lines
2689	// Stutter Efficiency	2689	// Stutter Efficiency
2690	for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {	2690	for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
2691	CalculateDETBufferSize(	2691	CalculateDETBufferSize(
2692	mode_lib->vba.DETBufferSizeInKByte,	2692	mode_lib->vba.DETBufferSizeInKByte[0],
2693	mode_lib->vba.SwathHeightY[k],	2693	mode_lib->vba.SwathHeightY[k],
2694	mode_lib->vba.SwathHeightC[k],	2694	mode_lib->vba.SwathHeightC[k],
2695	&locals->DETBufferSizeY[k],	2695	&locals->DETBufferSizeY[k],
2696	&locals->DETBufferSizeC[k]);	2696	&locals->DETBufferSizeC[k]);
2697		2697
2698	locals->LinesInDETY[k] = locals->DETBufferSizeY[k]	2698	locals->LinesInDETY[k] = (double)locals->DETBufferSizeY[k]
2699	/ locals->BytePerPixelDETY[k] / locals->SwathWidthY[k];	2699	/ locals->BytePerPixelDETY[k] / locals->SwathWidthY[k];
2700	locals->LinesInDETYRoundedDownToSwath[k] = dml_floor(	2700	locals->LinesInDETYRoundedDownToSwath[k] = dml_floor(
2701	locals->LinesInDETY[k],	2701	locals->LinesInDETY[k],
@@ -2984,7 +2984,7 @@ static void DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 282 unmodified lines
2984	RoundedUpMaxSwathSizeBytesC = 0.0;	2984	RoundedUpMaxSwathSizeBytesC = 0.0;
2985		2985
2986	if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC	2986	if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC
2987	<= mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0) {	2987	<= mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0) {
2988	mode_lib->vba.SwathHeightY[k] = MaximumSwathHeightY;	2988	mode_lib->vba.SwathHeightY[k] = MaximumSwathHeightY;
2989	mode_lib->vba.SwathHeightC[k] = MaximumSwathHeightC;	2989	mode_lib->vba.SwathHeightC[k] = MaximumSwathHeightC;
2990	} else {	2990	} else {
@@ -2993,7 +2993,7 @@ static void DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 2 unmodified lines
2993	}	2993	}
2994		2994
2995	CalculateDETBufferSize(	2995	CalculateDETBufferSize(
2996	mode_lib->vba.DETBufferSizeInKByte,	2996	mode_lib->vba.DETBufferSizeInKByte[0],
2997	mode_lib->vba.SwathHeightY[k],	2997	mode_lib->vba.SwathHeightY[k],
2998	mode_lib->vba.SwathHeightC[k],	2998	mode_lib->vba.SwathHeightC[k],
2999	&mode_lib->vba.DETBufferSizeY[k],	2999	&mode_lib->vba.DETBufferSizeY[k],
@@ -3888,7 +3888,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 888 unmodified lines
3888	mode_lib->vba.MaximumSwathWidthInDETBuffer =	3888	mode_lib->vba.MaximumSwathWidthInDETBuffer =
3889	dml_min(	3889	dml_min(
3890	mode_lib->vba.MaximumSwathWidthSupport,	3890	mode_lib->vba.MaximumSwathWidthSupport,
3891	mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0	3891	mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0
3892	/ (locals->BytePerPixelInDETY[k]	3892	/ (locals->BytePerPixelInDETY[k]
3893	* locals->MinSwathHeightY[k]	3893	* locals->MinSwathHeightY[k]
3894	+ locals->BytePerPixelInDETC[k]	3894	+ locals->BytePerPixelInDETC[k]
@@ -4437,7 +4437,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 542 unmodified lines
4437	mode_lib->vba.RoundedUpMaxSwathSizeBytesC = 0.0;	4437	mode_lib->vba.RoundedUpMaxSwathSizeBytesC = 0.0;
4438	}	4438	}
4439	if (mode_lib->vba.RoundedUpMaxSwathSizeBytesY + mode_lib->vba.RoundedUpMaxSwathSizeBytesC	4439	if (mode_lib->vba.RoundedUpMaxSwathSizeBytesY + mode_lib->vba.RoundedUpMaxSwathSizeBytesC
4440	<= mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0) {	4440	<= mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0) {
4441	locals->SwathHeightYThisState[k] = locals->MaxSwathHeightY[k];	4441	locals->SwathHeightYThisState[k] = locals->MaxSwathHeightY[k];
4442	locals->SwathHeightCThisState[k] = locals->MaxSwathHeightC[k];	4442	locals->SwathHeightCThisState[k] = locals->MaxSwathHeightC[k];
4443	} else {	4443	} else {
@@ -4801,7 +4801,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 357 unmodified lines
4801	}	4801	}
4802		4802
4803	CalculateUrgentBurstFactor(	4803	CalculateUrgentBurstFactor(
4804	mode_lib->vba.DETBufferSizeInKByte,	4804	mode_lib->vba.DETBufferSizeInKByte[0],
4805	locals->SwathHeightYThisState[k],	4805	locals->SwathHeightYThisState[k],
4806	locals->SwathHeightCThisState[k],	4806	locals->SwathHeightCThisState[k],
4807	locals->SwathWidthYThisState[k],	4807	locals->SwathWidthYThisState[k],
@@ -4975,7 +4975,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 167 unmodified lines
4975	mode_lib->vba.MaxLineBufferLines,	4975	mode_lib->vba.MaxLineBufferLines,
4976	mode_lib->vba.LineBufferSize,	4976	mode_lib->vba.LineBufferSize,
4977	mode_lib->vba.DPPOutputBufferPixels,	4977	mode_lib->vba.DPPOutputBufferPixels,
4978	mode_lib->vba.DETBufferSizeInKByte,	4978	mode_lib->vba.DETBufferSizeInKByte[0],
4979	mode_lib->vba.WritebackInterfaceLumaBufferSize,	4979	mode_lib->vba.WritebackInterfaceLumaBufferSize,
4980	mode_lib->vba.WritebackInterfaceChromaBufferSize,	4980	mode_lib->vba.WritebackInterfaceChromaBufferSize,
4981	mode_lib->vba.DCFCLKPerState[i],	4981	mode_lib->vba.DCFCLKPerState[i],
@@ -5230,7 +5230,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 248 unmodified lines
5230	unsigned int MaxLineBufferLines,	5230	unsigned int MaxLineBufferLines,
5231	unsigned int LineBufferSize,	5231	unsigned int LineBufferSize,
5232	unsigned int DPPOutputBufferPixels,	5232	unsigned int DPPOutputBufferPixels,
5233	double DETBufferSizeInKByte,	5233	unsigned int DETBufferSizeInKByte,
5234	unsigned int WritebackInterfaceLumaBufferSize,	5234	unsigned int WritebackInterfaceLumaBufferSize,
5235	unsigned int WritebackInterfaceChromaBufferSize,	5235	unsigned int WritebackInterfaceChromaBufferSize,
5236	double DCFCLK,	5236	double DCFCLK,
@@ -5285,8 +5285,8 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 48 unmodified lines
5285	double EffectiveLBLatencyHidingC;	5285	double EffectiveLBLatencyHidingC;
5286	double DPPOutputBufferLinesY;	5286	double DPPOutputBufferLinesY;
5287	double DPPOutputBufferLinesC;	5287	double DPPOutputBufferLinesC;
5288	double DETBufferSizeY;	5288	unsigned int DETBufferSizeY;
5289	double DETBufferSizeC;	5289	unsigned int DETBufferSizeC;
5290	double LinesInDETY[DC__NUM_DPP__MAX];	5290	double LinesInDETY[DC__NUM_DPP__MAX];
5291	double LinesInDETC;	5291	double LinesInDETC;
5292	unsigned int LinesInDETYRoundedDownToSwath[DC__NUM_DPP__MAX];	5292	unsigned int LinesInDETYRoundedDownToSwath[DC__NUM_DPP__MAX];
@@ -5382,12 +5382,12 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 89 unmodified lines
5382	&DETBufferSizeY,	5382	&DETBufferSizeY,
5383	&DETBufferSizeC);	5383	&DETBufferSizeC);
5384		5384
5385	LinesInDETY[k] = DETBufferSizeY / BytePerPixelDETY[k] / SwathWidthY[k];	5385	LinesInDETY[k] = (double)DETBufferSizeY / BytePerPixelDETY[k] / SwathWidthY[k];
5386	LinesInDETYRoundedDownToSwath[k] = dml_floor(LinesInDETY[k], SwathHeightY[k]);	5386	LinesInDETYRoundedDownToSwath[k] = dml_floor(LinesInDETY[k], SwathHeightY[k]);
5387	FullDETBufferingTimeY[k] = LinesInDETYRoundedDownToSwath[k]	5387	FullDETBufferingTimeY[k] = LinesInDETYRoundedDownToSwath[k]
5388	* (HTotal[k] / PixelClock[k]) / VRatio[k];	5388	* (HTotal[k] / PixelClock[k]) / VRatio[k];
5389	if (BytePerPixelDETC[k] > 0) {	5389	if (BytePerPixelDETC[k] > 0) {
5390	LinesInDETC = DETBufferSizeC / BytePerPixelDETC[k] / (SwathWidthY[k] / 2.0);	5390	LinesInDETC = (double)DETBufferSizeC / BytePerPixelDETC[k] / (SwathWidthY[k] / 2.0);
5391	LinesInDETCRoundedDownToSwath = dml_floor(LinesInDETC, SwathHeightC[k]);	5391	LinesInDETCRoundedDownToSwath = dml_floor(LinesInDETC, SwathHeightC[k]);
5392	FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath	5392	FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath
5393	* (HTotal[k] / PixelClock[k]) / (VRatio[k] / 2);	5393	* (HTotal[k] / PixelClock[k]) / (VRatio[k] / 2);
@@ -5574,11 +5574,11 @@ static void CalculateDCFCLKDeepSleep( · 180 unmodified lines
5574	}	5574	}
5575		5575
5576	static void CalculateDETBufferSize(	5576	static void CalculateDETBufferSize(
5577	double DETBufferSizeInKByte,	5577	unsigned int DETBufferSizeInKByte,
5578	unsigned int SwathHeightY,	5578	unsigned int SwathHeightY,
5579	unsigned int SwathHeightC,	5579	unsigned int SwathHeightC,
5580	double *DETBufferSizeY,	5580	unsigned int *DETBufferSizeY,
5581	double *DETBufferSizeC)	5581	unsigned int *DETBufferSizeC)
5582	{	5582	{
5583	if (SwathHeightC == 0) {	5583	if (SwathHeightC == 0) {
5584	DETBufferSizeY = DETBufferSizeInKByte 1024;	5584	DETBufferSizeY = DETBufferSizeInKByte 1024;
@@ -5625,8 +5625,8 @@ static void CalculateUrgentBurstFactor( · 40 unmodified lines
5625	double DETBufferSizeInTimeLumaPre;	5625	double DETBufferSizeInTimeLumaPre;
5626	double DETBufferSizeInTimeChroma;	5626	double DETBufferSizeInTimeChroma;
5627	double DETBufferSizeInTimeChromaPre;	5627	double DETBufferSizeInTimeChromaPre;
5628	double DETBufferSizeY;	5628	unsigned int DETBufferSizeY;
5629	double DETBufferSizeC;	5629	unsigned int DETBufferSizeC;
5630		5630
5631	*NotEnoughUrgentLatencyHiding = 0;	5631	*NotEnoughUrgentLatencyHiding = 0;
5632	*NotEnoughUrgentLatencyHidingPre = 0;	5632	*NotEnoughUrgentLatencyHidingPre = 0;
@@ -5663,7 +5663,7 @@ static void CalculateUrgentBurstFactor( · 30 unmodified lines
5663	&DETBufferSizeY,	5663	&DETBufferSizeY,
5664	&DETBufferSizeC);	5664	&DETBufferSizeC);
5665		5665
5666	LinesInDETLuma = DETBufferSizeY / BytePerPixelInDETY / SwathWidthY;	5666	LinesInDETLuma = (double)DETBufferSizeY / BytePerPixelInDETY / SwathWidthY;
5667	DETBufferSizeInTimeLuma = dml_floor(LinesInDETLuma, SwathHeightY) * LineTime / VRatio;	5667	DETBufferSizeInTimeLuma = dml_floor(LinesInDETLuma, SwathHeightY) * LineTime / VRatio;
5668	if (DETBufferSizeInTimeLuma - UrgentLatency <= 0) {	5668	if (DETBufferSizeInTimeLuma - UrgentLatency <= 0) {
5669	*NotEnoughUrgentLatencyHiding = 1;	5669	*NotEnoughUrgentLatencyHiding = 1;
@@ -5687,7 +5687,7 @@ static void CalculateUrgentBurstFactor( · 17 unmodified lines
5687	}	5687	}
5688		5688
5689	if (BytePerPixelInDETC > 0) {	5689	if (BytePerPixelInDETC > 0) {
5690	LinesInDETChroma = DETBufferSizeC / BytePerPixelInDETC / (SwathWidthY / 2);	5690	LinesInDETChroma = (double)DETBufferSizeC / BytePerPixelInDETC / (SwathWidthY / 2);
5691	DETBufferSizeInTimeChroma = dml_floor(LinesInDETChroma, SwathHeightC) * LineTime	5691	DETBufferSizeInTimeChroma = dml_floor(LinesInDETChroma, SwathHeightC) * LineTime
5692	/ (VRatio / 2);	5692	/ (VRatio / 2);
5693	if (DETBufferSizeInTimeChroma - UrgentLatency <= 0) {	5693	if (DETBufferSizeInTimeChroma - UrgentLatency <= 0) {

drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c+24−24

@@ -299,7 +299,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 298 unmodified lines
299	unsigned int MaxLineBufferLines,	299	unsigned int MaxLineBufferLines,
300	unsigned int LineBufferSize,	300	unsigned int LineBufferSize,
301	unsigned int DPPOutputBufferPixels,	301	unsigned int DPPOutputBufferPixels,
302	double DETBufferSizeInKByte,	302	unsigned int DETBufferSizeInKByte,
303	unsigned int WritebackInterfaceBufferSize,	303	unsigned int WritebackInterfaceBufferSize,
304	double DCFCLK,	304	double DCFCLK,
305	double ReturnBW,	305	double ReturnBW,
@@ -318,8 +318,8 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 12 unmodified lines
318	unsigned int DPPPerPlane[],	318	unsigned int DPPPerPlane[],
319	bool DCCEnable[],	319	bool DCCEnable[],
320	double DPPCLK[],	320	double DPPCLK[],
321	double DETBufferSizeY[],	321	unsigned int DETBufferSizeY[],
322	double DETBufferSizeC[],	322	unsigned int DETBufferSizeC[],
323	unsigned int SwathHeightY[],	323	unsigned int SwathHeightY[],
324	unsigned int SwathHeightC[],	324	unsigned int SwathHeightC[],
325	unsigned int LBBitPerPixel[],	325	unsigned int LBBitPerPixel[],
@@ -570,7 +570,7 @@ static void CalculateStutterEfficiency( · 244 unmodified lines
570	double SRExitTime,	570	double SRExitTime,
571	bool SynchronizedVBlank,	571	bool SynchronizedVBlank,
572	int DPPPerPlane[],	572	int DPPPerPlane[],
573	double DETBufferSizeY[],	573	unsigned int DETBufferSizeY[],
574	int BytePerPixelY[],	574	int BytePerPixelY[],
575	double BytePerPixelDETY[],	575	double BytePerPixelDETY[],
576	double SwathWidthY[],	576	double SwathWidthY[],
@@ -603,7 +603,7 @@ static void CalculateStutterEfficiency( · 26 unmodified lines
603	static void CalculateSwathAndDETConfiguration(	603	static void CalculateSwathAndDETConfiguration(
604	bool ForceSingleDPP,	604	bool ForceSingleDPP,
605	int NumberOfActivePlanes,	605	int NumberOfActivePlanes,
606	long DETBufferSizeInKByte,	606	unsigned int DETBufferSizeInKByte,
607	double MaximumSwathWidthLuma[],	607	double MaximumSwathWidthLuma[],
608	double MaximumSwathWidthChroma[],	608	double MaximumSwathWidthChroma[],
609	enum scan_direction_class SourceScan[],	609	enum scan_direction_class SourceScan[],
@@ -635,8 +635,8 @@ static void CalculateSwathAndDETConfiguration( · 25 unmodified lines
635	double SwathWidthChroma[],	635	double SwathWidthChroma[],
636	int SwathHeightY[],	636	int SwathHeightY[],
637	int SwathHeightC[],	637	int SwathHeightC[],
638	double DETBufferSizeY[],	638	unsigned int DETBufferSizeY[],
639	double DETBufferSizeC[],	639	unsigned int DETBufferSizeC[],
640	bool ViewportSizeSupportPerPlane[],	640	bool ViewportSizeSupportPerPlane[],
641	bool *ViewportSizeSupport);	641	bool *ViewportSizeSupport);
642	static void CalculateSwathWidth(	642	static void CalculateSwathWidth(
@@ -2613,7 +2613,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 1970 unmodified lines
2613	CalculateUrgentBurstFactor(	2613	CalculateUrgentBurstFactor(
2614	v->swath_width_luma_ub[k],	2614	v->swath_width_luma_ub[k],
2615	v->swath_width_chroma_ub[k],	2615	v->swath_width_chroma_ub[k],
2616	v->DETBufferSizeInKByte,	2616	v->DETBufferSizeInKByte[0],
2617	v->SwathHeightY[k],	2617	v->SwathHeightY[k],
2618	v->SwathHeightC[k],	2618	v->SwathHeightC[k],
2619	v->HTotal[k] / v->PixelClock[k],	2619	v->HTotal[k] / v->PixelClock[k],
@@ -2635,7 +2635,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 15 unmodified lines
2635	CalculateUrgentBurstFactor(	2635	CalculateUrgentBurstFactor(
2636	v->swath_width_luma_ub[k],	2636	v->swath_width_luma_ub[k],
2637	v->swath_width_chroma_ub[k],	2637	v->swath_width_chroma_ub[k],
2638	v->DETBufferSizeInKByte,	2638	v->DETBufferSizeInKByte[0],
2639	v->SwathHeightY[k],	2639	v->SwathHeightY[k],
2640	v->SwathHeightC[k],	2640	v->SwathHeightC[k],
2641	v->HTotal[k] / v->PixelClock[k],	2641	v->HTotal[k] / v->PixelClock[k],
@@ -2808,7 +2808,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 166 unmodified lines
2808	v->MaxLineBufferLines,	2808	v->MaxLineBufferLines,
2809	v->LineBufferSize,	2809	v->LineBufferSize,
2810	v->DPPOutputBufferPixels,	2810	v->DPPOutputBufferPixels,
2811	v->DETBufferSizeInKByte,	2811	v->DETBufferSizeInKByte[0],
2812	v->WritebackInterfaceBufferSize,	2812	v->WritebackInterfaceBufferSize,
2813	v->DCFCLK,	2813	v->DCFCLK,
2814	v->ReturnBW,	2814	v->ReturnBW,
@@ -3027,7 +3027,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 212 unmodified lines
3027	v->SurfaceWidthC[k],	3027	v->SurfaceWidthC[k],
3028	v->SurfaceHeightY[k],	3028	v->SurfaceHeightY[k],
3029	v->SurfaceHeightC[k],	3029	v->SurfaceHeightC[k],
3030	v->DETBufferSizeInKByte * 1024,	3030	v->DETBufferSizeInKByte[0] * 1024,
3031	v->BlockHeight256BytesY[k],	3031	v->BlockHeight256BytesY[k],
3032	v->BlockHeight256BytesC[k],	3032	v->BlockHeight256BytesC[k],
3033	v->SurfaceTiling[k],	3033	v->SurfaceTiling[k],
@@ -3177,7 +3177,7 @@ static void DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 143 unmodified lines
3177	CalculateSwathAndDETConfiguration(	3177	CalculateSwathAndDETConfiguration(
3178	false,	3178	false,
3179	mode_lib->vba.NumberOfActivePlanes,	3179	mode_lib->vba.NumberOfActivePlanes,
3180	mode_lib->vba.DETBufferSizeInKByte,	3180	mode_lib->vba.DETBufferSizeInKByte[0],
3181	dummy1,	3181	dummy1,
3182	dummy2,	3182	dummy2,
3183	mode_lib->vba.SourceScan,	3183	mode_lib->vba.SourceScan,
@@ -3911,7 +3911,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 727 unmodified lines
3911	CalculateSwathAndDETConfiguration(	3911	CalculateSwathAndDETConfiguration(
3912	true,	3912	true,
3913	v->NumberOfActivePlanes,	3913	v->NumberOfActivePlanes,
3914	v->DETBufferSizeInKByte,	3914	v->DETBufferSizeInKByte[0],
3915	v->MaximumSwathWidthLuma,	3915	v->MaximumSwathWidthLuma,
3916	v->MaximumSwathWidthChroma,	3916	v->MaximumSwathWidthChroma,
3917	v->SourceScan,	3917	v->SourceScan,
@@ -4399,7 +4399,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 481 unmodified lines
4399	CalculateSwathAndDETConfiguration(	4399	CalculateSwathAndDETConfiguration(
4400	false,	4400	false,
4401	v->NumberOfActivePlanes,	4401	v->NumberOfActivePlanes,
4402	v->DETBufferSizeInKByte,	4402	v->DETBufferSizeInKByte[0],
4403	v->MaximumSwathWidthLuma,	4403	v->MaximumSwathWidthLuma,
4404	v->MaximumSwathWidthChroma,	4404	v->MaximumSwathWidthChroma,
4405	v->SourceScan,	4405	v->SourceScan,
@@ -4622,7 +4622,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 216 unmodified lines
4622	CalculateUrgentBurstFactor(	4622	CalculateUrgentBurstFactor(
4623	v->swath_width_luma_ub_this_state[k],	4623	v->swath_width_luma_ub_this_state[k],
4624	v->swath_width_chroma_ub_this_state[k],	4624	v->swath_width_chroma_ub_this_state[k],
4625	v->DETBufferSizeInKByte,	4625	v->DETBufferSizeInKByte[0],
4626	v->SwathHeightYThisState[k],	4626	v->SwathHeightYThisState[k],
4627	v->SwathHeightCThisState[k],	4627	v->SwathHeightCThisState[k],
4628	v->HTotal[k] / v->PixelClock[k],	4628	v->HTotal[k] / v->PixelClock[k],
@@ -5025,7 +5025,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 396 unmodified lines
5025	CalculateUrgentBurstFactor(	5025	CalculateUrgentBurstFactor(
5026	v->swath_width_luma_ub_this_state[k],	5026	v->swath_width_luma_ub_this_state[k],
5027	v->swath_width_chroma_ub_this_state[k],	5027	v->swath_width_chroma_ub_this_state[k],
5028	v->DETBufferSizeInKByte,	5028	v->DETBufferSizeInKByte[0],
5029	v->SwathHeightYThisState[k],	5029	v->SwathHeightYThisState[k],
5030	v->SwathHeightCThisState[k],	5030	v->SwathHeightCThisState[k],
5031	v->HTotal[k] / v->PixelClock[k],	5031	v->HTotal[k] / v->PixelClock[k],
@@ -5197,7 +5197,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 165 unmodified lines
5197	v->MaxLineBufferLines,	5197	v->MaxLineBufferLines,
5198	v->LineBufferSize,	5198	v->LineBufferSize,
5199	v->DPPOutputBufferPixels,	5199	v->DPPOutputBufferPixels,
5200	v->DETBufferSizeInKByte,	5200	v->DETBufferSizeInKByte[0],
5201	v->WritebackInterfaceBufferSize,	5201	v->WritebackInterfaceBufferSize,
5202	v->DCFCLKState[i][j],	5202	v->DCFCLKState[i][j],
5203	v->ReturnBWPerState[i][j],	5203	v->ReturnBWPerState[i][j],
@@ -5369,7 +5369,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 165 unmodified lines
5369	unsigned int MaxLineBufferLines,	5369	unsigned int MaxLineBufferLines,
5370	unsigned int LineBufferSize,	5370	unsigned int LineBufferSize,
5371	unsigned int DPPOutputBufferPixels,	5371	unsigned int DPPOutputBufferPixels,
5372	double DETBufferSizeInKByte,	5372	unsigned int DETBufferSizeInKByte,
5373	unsigned int WritebackInterfaceBufferSize,	5373	unsigned int WritebackInterfaceBufferSize,
5374	double DCFCLK,	5374	double DCFCLK,
5375	double ReturnBW,	5375	double ReturnBW,
@@ -5388,8 +5388,8 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 12 unmodified lines
5388	unsigned int DPPPerPlane[],	5388	unsigned int DPPPerPlane[],
5389	bool DCCEnable[],	5389	bool DCCEnable[],
5390	double DPPCLK[],	5390	double DPPCLK[],
5391	double DETBufferSizeY[],	5391	unsigned int DETBufferSizeY[],
5392	double DETBufferSizeC[],	5392	unsigned int DETBufferSizeC[],
5393	unsigned int SwathHeightY[],	5393	unsigned int SwathHeightY[],
5394	unsigned int SwathHeightC[],	5394	unsigned int SwathHeightC[],
5395	unsigned int LBBitPerPixel[],	5395	unsigned int LBBitPerPixel[],
@@ -6126,7 +6126,7 @@ static void CalculateStutterEfficiency( · 730 unmodified lines
6126	double SRExitTime,	6126	double SRExitTime,
6127	bool SynchronizedVBlank,	6127	bool SynchronizedVBlank,
6128	int DPPPerPlane[],	6128	int DPPPerPlane[],
6129	double DETBufferSizeY[],	6129	unsigned int DETBufferSizeY[],
6130	int BytePerPixelY[],	6130	int BytePerPixelY[],
6131	double BytePerPixelDETY[],	6131	double BytePerPixelDETY[],
6132	double SwathWidthY[],	6132	double SwathWidthY[],
@@ -6273,7 +6273,7 @@ static void CalculateStutterEfficiency( · 140 unmodified lines
6273	static void CalculateSwathAndDETConfiguration(	6273	static void CalculateSwathAndDETConfiguration(
6274	bool ForceSingleDPP,	6274	bool ForceSingleDPP,
6275	int NumberOfActivePlanes,	6275	int NumberOfActivePlanes,
6276	long DETBufferSizeInKByte,	6276	unsigned int DETBufferSizeInKByte,
6277	double MaximumSwathWidthLuma[],	6277	double MaximumSwathWidthLuma[],
6278	double MaximumSwathWidthChroma[],	6278	double MaximumSwathWidthChroma[],
6279	enum scan_direction_class SourceScan[],	6279	enum scan_direction_class SourceScan[],
@@ -6305,8 +6305,8 @@ static void CalculateSwathAndDETConfiguration( · 25 unmodified lines
6305	double SwathWidthChroma[],	6305	double SwathWidthChroma[],
6306	int SwathHeightY[],	6306	int SwathHeightY[],
6307	int SwathHeightC[],	6307	int SwathHeightC[],
6308	double DETBufferSizeY[],	6308	unsigned int DETBufferSizeY[],
6309	double DETBufferSizeC[],	6309	unsigned int DETBufferSizeC[],
6310	bool ViewportSizeSupportPerPlane[],	6310	bool ViewportSizeSupportPerPlane[],
6311	bool *ViewportSizeSupport)	6311	bool *ViewportSizeSupport)
6312	{	6312	{

drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c+1−1

@@ -299,7 +299,7 @@ static void fetch_ip_params(struct display_mode_lib *mode_lib) · 298 unmodified lines
299	mode_lib->vba.MaxDCHUBToPSCLThroughput = ip->max_dchub_pscl_bw_pix_per_clk;	299	mode_lib->vba.MaxDCHUBToPSCLThroughput = ip->max_dchub_pscl_bw_pix_per_clk;
300	mode_lib->vba.MaxPSCLToLBThroughput = ip->max_pscl_lb_bw_pix_per_clk;	300	mode_lib->vba.MaxPSCLToLBThroughput = ip->max_pscl_lb_bw_pix_per_clk;
301	mode_lib->vba.ROBBufferSizeInKByte = ip->rob_buffer_size_kbytes;	301	mode_lib->vba.ROBBufferSizeInKByte = ip->rob_buffer_size_kbytes;
302	mode_lib->vba.DETBufferSizeInKByte = ip->det_buffer_size_kbytes;	302	mode_lib->vba.DETBufferSizeInKByte[0] = ip->det_buffer_size_kbytes;
303		303
304	mode_lib->vba.PixelChunkSizeInKByte = ip->pixel_chunk_size_kbytes;	304	mode_lib->vba.PixelChunkSizeInKByte = ip->pixel_chunk_size_kbytes;
305	mode_lib->vba.MetaChunkSize = ip->meta_chunk_size_kbytes;	305	mode_lib->vba.MetaChunkSize = ip->meta_chunk_size_kbytes;

drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h+7−7

@@ -233,7 +233,7 @@ struct vba_vars_st { · 232 unmodified lines
233	// IP Parameters	233	// IP Parameters
234	//	234	//
235	unsigned int ROBBufferSizeInKByte;	235	unsigned int ROBBufferSizeInKByte;
236	double DETBufferSizeInKByte;	236	unsigned int DETBufferSizeInKByte[DC__NUM_DPP__MAX];
237	double DETBufferSizeInTime;	237	double DETBufferSizeInTime;
238	unsigned int DPPOutputBufferPixels;	238	unsigned int DPPOutputBufferPixels;
239	unsigned int OPPOutputBufferLines;	239	unsigned int OPPOutputBufferLines;
@@ -351,8 +351,8 @@ struct vba_vars_st { · 111 unmodified lines
351		351
352	// Intermediates/Informational	352	// Intermediates/Informational
353	bool ImmediateFlipSupport;	353	bool ImmediateFlipSupport;
354	double DETBufferSizeY[DC__NUM_DPP__MAX];	354	unsigned int DETBufferSizeY[DC__NUM_DPP__MAX];
355	double DETBufferSizeC[DC__NUM_DPP__MAX];	355	unsigned int DETBufferSizeC[DC__NUM_DPP__MAX];
356	unsigned int SwathHeightY[DC__NUM_DPP__MAX];	356	unsigned int SwathHeightY[DC__NUM_DPP__MAX];
357	unsigned int SwathHeightC[DC__NUM_DPP__MAX];	357	unsigned int SwathHeightC[DC__NUM_DPP__MAX];
358	unsigned int LBBitPerPixel[DC__NUM_DPP__MAX];	358	unsigned int LBBitPerPixel[DC__NUM_DPP__MAX];
@@ -631,8 +631,8 @@ struct vba_vars_st { · 272 unmodified lines
631	enum odm_combine_mode odm_combine_dummy[DC__NUM_DPP__MAX];	631	enum odm_combine_mode odm_combine_dummy[DC__NUM_DPP__MAX];
632	double dummy1[DC__NUM_DPP__MAX];	632	double dummy1[DC__NUM_DPP__MAX];
633	double dummy2[DC__NUM_DPP__MAX];	633	double dummy2[DC__NUM_DPP__MAX];
634	double dummy3[DC__NUM_DPP__MAX];	634	unsigned int dummy3[DC__NUM_DPP__MAX];
635	double dummy4[DC__NUM_DPP__MAX];	635	unsigned int dummy4[DC__NUM_DPP__MAX];
636	double dummy5;	636	double dummy5;
637	double dummy6;	637	double dummy6;
638	double dummy7[DC__NUM_DPP__MAX];	638	double dummy7[DC__NUM_DPP__MAX];
@@ -872,8 +872,8 @@ struct vba_vars_st { · 233 unmodified lines
872	int PercentMarginOverMinimumRequiredDCFCLK;	872	int PercentMarginOverMinimumRequiredDCFCLK;
873	bool DynamicMetadataSupported[DC__VOLTAGE_STATES][2];	873	bool DynamicMetadataSupported[DC__VOLTAGE_STATES][2];
874	enum immediate_flip_requirement ImmediateFlipRequirement;	874	enum immediate_flip_requirement ImmediateFlipRequirement;
875	double DETBufferSizeYThisState[DC__NUM_DPP__MAX];	875	unsigned int DETBufferSizeYThisState[DC__NUM_DPP__MAX];
876	double DETBufferSizeCThisState[DC__NUM_DPP__MAX];	876	unsigned int DETBufferSizeCThisState[DC__NUM_DPP__MAX];
877	bool NoUrgentLatencyHiding[DC__NUM_DPP__MAX];	877	bool NoUrgentLatencyHiding[DC__NUM_DPP__MAX];
878	bool NoUrgentLatencyHidingPre[DC__NUM_DPP__MAX];	878	bool NoUrgentLatencyHidingPre[DC__NUM_DPP__MAX];
879	int swath_width_luma_ub_this_state[DC__NUM_DPP__MAX];	879	int swath_width_luma_ub_this_state[DC__NUM_DPP__MAX];

drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c+103−25

@@ -37,6 +37,8 @@ static uint32_t dsc_policy_max_target_bpp_limit = 16; · 36 unmodified lines
37	/* default DSC policy enables DSC only when needed */	37	/* default DSC policy enables DSC only when needed */
38	static bool dsc_policy_enable_dsc_when_not_needed;	38	static bool dsc_policy_enable_dsc_when_not_needed;
39		39
		40	static bool dsc_policy_disable_dsc_stream_overhead;
		41
40	static bool dsc_buff_block_size_from_dpcd(int dpcd_buff_block_size, int *buff_block_size)	42	static bool dsc_buff_block_size_from_dpcd(int dpcd_buff_block_size, int *buff_block_size)
41	{	43	{
42		44
@@ -250,6 +252,7 @@ static bool intersect_dsc_caps( · 207 unmodified lines
250	if (pixel_encoding == PIXEL_ENCODING_YCBCR422 \|\| pixel_encoding == PIXEL_ENCODING_YCBCR420)	252	if (pixel_encoding == PIXEL_ENCODING_YCBCR422 \|\| pixel_encoding == PIXEL_ENCODING_YCBCR420)
251	dsc_common_caps->bpp_increment_div = min(dsc_common_caps->bpp_increment_div, (uint32_t)8);	253	dsc_common_caps->bpp_increment_div = min(dsc_common_caps->bpp_increment_div, (uint32_t)8);
252		254
		255	dsc_common_caps->is_dp = dsc_sink_caps->is_dp;
253	return true;	256	return true;
254	}	257	}
255		258
@@ -258,12 +261,63 @@ static inline uint32_t dsc_div_by_10_round_up(uint32_t value) · 2 unmodified lines
258	return (value + 9) / 10;	261	return (value + 9) / 10;
259	}	262	}
260		263
		264	static struct fixed31_32 compute_dsc_max_bandwidth_overhead(
		265	const struct dc_crtc_timing *timing,
		266	const int num_slices_h,
		267	const bool is_dp)
		268	{
		269	struct fixed31_32 max_dsc_overhead;
		270	struct fixed31_32 refresh_rate;
		271
		272	if (dsc_policy_disable_dsc_stream_overhead \|\| !is_dp)
		273	return dc_fixpt_from_int(0);
		274
		275	/* use target bpp that can take entire target bandwidth */
		276	refresh_rate = dc_fixpt_from_int(timing->pix_clk_100hz);
		277	refresh_rate = dc_fixpt_div_int(refresh_rate, timing->h_total);
		278	refresh_rate = dc_fixpt_div_int(refresh_rate, timing->v_total);
		279	refresh_rate = dc_fixpt_mul_int(refresh_rate, 100);
		280
		281	max_dsc_overhead = dc_fixpt_from_int(num_slices_h);
		282	max_dsc_overhead = dc_fixpt_mul_int(max_dsc_overhead, timing->v_total);
		283	max_dsc_overhead = dc_fixpt_mul_int(max_dsc_overhead, 256);
		284	max_dsc_overhead = dc_fixpt_div_int(max_dsc_overhead, 1000);
		285	max_dsc_overhead = dc_fixpt_mul(max_dsc_overhead, refresh_rate);
		286
		287	return max_dsc_overhead;
		288	}
		289
		290	static uint32_t compute_bpp_x16_from_target_bandwidth(
		291	const uint32_t bandwidth_in_kbps,
		292	const struct dc_crtc_timing *timing,
		293	const uint32_t num_slices_h,
		294	const uint32_t bpp_increment_div,
		295	const bool is_dp)
		296	{
		297	struct fixed31_32 overhead_in_kbps;
		298	struct fixed31_32 effective_bandwidth_in_kbps;
		299	struct fixed31_32 bpp_x16;
		300
		301	overhead_in_kbps = compute_dsc_max_bandwidth_overhead(
		302	timing, num_slices_h, is_dp);
		303	effective_bandwidth_in_kbps = dc_fixpt_from_int(bandwidth_in_kbps);
		304	effective_bandwidth_in_kbps = dc_fixpt_sub(effective_bandwidth_in_kbps,
		305	overhead_in_kbps);
		306	bpp_x16 = dc_fixpt_mul_int(effective_bandwidth_in_kbps, 10);
		307	bpp_x16 = dc_fixpt_div_int(bpp_x16, timing->pix_clk_100hz);
		308	bpp_x16 = dc_fixpt_from_int(dc_fixpt_floor(dc_fixpt_mul_int(bpp_x16, bpp_increment_div)));
		309	bpp_x16 = dc_fixpt_div_int(bpp_x16, bpp_increment_div);
		310	bpp_x16 = dc_fixpt_mul_int(bpp_x16, 16);
		311	return dc_fixpt_floor(bpp_x16);
		312	}
		313
261	/* Get DSC bandwidth range based on [min_bpp, max_bpp] target bitrate range, and timing's pixel clock	314	/* Get DSC bandwidth range based on [min_bpp, max_bpp] target bitrate range, and timing's pixel clock
262	* and uncompressed bandwidth.	315	* and uncompressed bandwidth.
263	*/	316	*/
264	static void get_dsc_bandwidth_range(	317	static void get_dsc_bandwidth_range(
265	const uint32_t min_bpp_x16,	318	const uint32_t min_bpp_x16,
266	const uint32_t max_bpp_x16,	319	const uint32_t max_bpp_x16,
		320	const uint32_t num_slices_h,
267	const struct dsc_enc_caps *dsc_caps,	321	const struct dsc_enc_caps *dsc_caps,
268	const struct dc_crtc_timing *timing,	322	const struct dc_crtc_timing *timing,
269	struct dc_dsc_bw_range *range)	323	struct dc_dsc_bw_range *range)
@@ -272,16 +326,21 @@ static void get_dsc_bandwidth_range( · 2 unmodified lines
272	range->stream_kbps = dc_bandwidth_in_kbps_from_timing(timing);	326	range->stream_kbps = dc_bandwidth_in_kbps_from_timing(timing);
273		327
274	/* max dsc target bpp */	328	/* max dsc target bpp */
275	range->max_kbps = dc_dsc_stream_bandwidth_in_kbps(timing->pix_clk_100hz, max_bpp_x16);	329	range->max_kbps = dc_dsc_stream_bandwidth_in_kbps(timing,
		330	max_bpp_x16, num_slices_h, dsc_caps->is_dp);
276	range->max_target_bpp_x16 = max_bpp_x16;	331	range->max_target_bpp_x16 = max_bpp_x16;
277	if (range->max_kbps > range->stream_kbps) {	332	if (range->max_kbps > range->stream_kbps) {
278	/* max dsc target bpp is capped to native bandwidth */	333	/* max dsc target bpp is capped to native bandwidth */
279	range->max_kbps = range->stream_kbps;	334	range->max_kbps = range->stream_kbps;
280	range->max_target_bpp_x16 = calc_dsc_bpp_x16(range->stream_kbps, timing->pix_clk_100hz, dsc_caps->bpp_increment_div);	335	range->max_target_bpp_x16 = compute_bpp_x16_from_target_bandwidth(
		336	range->max_kbps, timing, num_slices_h,
		337	dsc_caps->bpp_increment_div,
		338	dsc_caps->is_dp);
281	}	339	}
282		340
283	/* min dsc target bpp */	341	/* min dsc target bpp */
284	range->min_kbps = dc_dsc_stream_bandwidth_in_kbps(timing->pix_clk_100hz, min_bpp_x16);	342	range->min_kbps = dc_dsc_stream_bandwidth_in_kbps(timing,
		343	min_bpp_x16, num_slices_h, dsc_caps->is_dp);
285	range->min_target_bpp_x16 = min_bpp_x16;	344	range->min_target_bpp_x16 = min_bpp_x16;
286	if (range->min_kbps > range->max_kbps) {	345	if (range->min_kbps > range->max_kbps) {
287	/* min dsc target bpp is capped to max dsc bandwidth*/	346	/* min dsc target bpp is capped to max dsc bandwidth*/
@@ -290,7 +349,6 @@ static void get_dsc_bandwidth_range( · 2 unmodified lines
290	}	349	}
291	}	350	}
292		351
293
294	/* Decides if DSC should be used and calculates target bpp if it should, applying DSC policy.	352	/* Decides if DSC should be used and calculates target bpp if it should, applying DSC policy.
295	*	353	*
296	* Returns:	354	* Returns:
@@ -303,6 +361,7 @@ static bool decide_dsc_target_bpp_x16( · 6 unmodified lines
303	const struct dsc_enc_caps *dsc_common_caps,	361	const struct dsc_enc_caps *dsc_common_caps,
304	const int target_bandwidth_kbps,	362	const int target_bandwidth_kbps,
305	const struct dc_crtc_timing *timing,	363	const struct dc_crtc_timing *timing,
		364	const int num_slices_h,
306	int *target_bpp_x16)	365	int *target_bpp_x16)
307	{	366	{
308	bool should_use_dsc = false;	367	bool should_use_dsc = false;
@@ -311,7 +370,7 @@ static bool decide_dsc_target_bpp_x16( · 2 unmodified lines
311	memset(&range, 0, sizeof(range));	370	memset(&range, 0, sizeof(range));
312		371
313	get_dsc_bandwidth_range(policy->min_target_bpp * 16, policy->max_target_bpp * 16,	372	get_dsc_bandwidth_range(policy->min_target_bpp * 16, policy->max_target_bpp * 16,
314	dsc_common_caps, timing, &range);	373	num_slices_h, dsc_common_caps, timing, &range);
315	if (!policy->enable_dsc_when_not_needed && target_bandwidth_kbps >= range.stream_kbps) {	374	if (!policy->enable_dsc_when_not_needed && target_bandwidth_kbps >= range.stream_kbps) {
316	/* enough bandwidth without dsc */	375	/* enough bandwidth without dsc */
317	*target_bpp_x16 = 0;	376	*target_bpp_x16 = 0;
@@ -327,7 +386,10 @@ static bool decide_dsc_target_bpp_x16( · 9 unmodified lines
327	should_use_dsc = true;	386	should_use_dsc = true;
328	} else if (target_bandwidth_kbps >= range.min_kbps) {	387	} else if (target_bandwidth_kbps >= range.min_kbps) {
329	/* use target bpp that can take entire target bandwidth */	388	/* use target bpp that can take entire target bandwidth */
330	*target_bpp_x16 = calc_dsc_bpp_x16(target_bandwidth_kbps, timing->pix_clk_100hz, dsc_common_caps->bpp_increment_div);	389	*target_bpp_x16 = compute_bpp_x16_from_target_bandwidth(
		390	target_bandwidth_kbps, timing, num_slices_h,
		391	dsc_common_caps->bpp_increment_div,
		392	dsc_common_caps->is_dp);
331	should_use_dsc = true;	393	should_use_dsc = true;
332	} else {	394	} else {
333	/* not enough bandwidth to fulfill minimum requirement */	395	/* not enough bandwidth to fulfill minimum requirement */
@@ -531,18 +593,6 @@ static bool setup_dsc_config( · 197 unmodified lines
531	if (!is_dsc_possible)	593	if (!is_dsc_possible)
532	goto done;	594	goto done;
533		595
534	if (target_bandwidth_kbps > 0) {
535	is_dsc_possible = decide_dsc_target_bpp_x16(
536	&policy,
537	&dsc_common_caps,
538	target_bandwidth_kbps,
539	timing,
540	&target_bpp);
541	dsc_cfg->bits_per_pixel = target_bpp;
542	}
543	if (!is_dsc_possible)
544	goto done;
545
546	sink_per_slice_throughput_mps = 0;	596	sink_per_slice_throughput_mps = 0;
547		597
548	// Validate available DSC settings against the mode timing	598	// Validate available DSC settings against the mode timing
@@ -690,12 +740,26 @@ static bool setup_dsc_config( · 141 unmodified lines
690		740
691	dsc_cfg->num_slices_v = pic_height/slice_height;	741	dsc_cfg->num_slices_v = pic_height/slice_height;
692		742
		743	if (target_bandwidth_kbps > 0) {
		744	is_dsc_possible = decide_dsc_target_bpp_x16(
		745	&policy,
		746	&dsc_common_caps,
		747	target_bandwidth_kbps,
		748	timing,
		749	num_slices_h,
		750	&target_bpp);
		751	dsc_cfg->bits_per_pixel = target_bpp;
		752	}
		753	if (!is_dsc_possible)
		754	goto done;
		755
693	// Final decission: can we do DSC or not?	756	// Final decission: can we do DSC or not?
694	if (is_dsc_possible) {	757	if (is_dsc_possible) {
695	// Fill out the rest of DSC settings	758	// Fill out the rest of DSC settings
696	dsc_cfg->block_pred_enable = dsc_common_caps.is_block_pred_supported;	759	dsc_cfg->block_pred_enable = dsc_common_caps.is_block_pred_supported;
697	dsc_cfg->linebuf_depth = dsc_common_caps.lb_bit_depth;	760	dsc_cfg->linebuf_depth = dsc_common_caps.lb_bit_depth;
698	dsc_cfg->version_minor = (dsc_common_caps.dsc_version & 0xf0) >> 4;	761	dsc_cfg->version_minor = (dsc_common_caps.dsc_version & 0xf0) >> 4;
		762	dsc_cfg->is_dp = dsc_sink_caps->is_dp;
699	}	763	}
700		764
701	done:	765	done:
@@ -806,6 +870,7 @@ bool dc_dsc_parse_dsc_dpcd(const struct dc dc, const uint8_t dpcd_dsc_basic_da · 104 unmodified lines
806	dsc_sink_caps->branch_max_line_width = dpcd_dsc_branch_decoder_caps[DP_DSC_BRANCH_MAX_LINE_WIDTH - DP_DSC_BRANCH_OVERALL_THROUGHPUT_0] * 320;	870	dsc_sink_caps->branch_max_line_width = dpcd_dsc_branch_decoder_caps[DP_DSC_BRANCH_MAX_LINE_WIDTH - DP_DSC_BRANCH_OVERALL_THROUGHPUT_0] * 320;
807	ASSERT(dsc_sink_caps->branch_max_line_width == 0 \|\| dsc_sink_caps->branch_max_line_width >= 5120);	871	ASSERT(dsc_sink_caps->branch_max_line_width == 0 \|\| dsc_sink_caps->branch_max_line_width >= 5120);
808		872
		873	dsc_sink_caps->is_dp = true;
809	return true;	874	return true;
810	}	875	}
811		876
@@ -838,7 +903,8 @@ bool dc_dsc_compute_bandwidth_range( · 26 unmodified lines
838	dsc_min_slice_height_override, max_bpp_x16, &config);	903	dsc_min_slice_height_override, max_bpp_x16, &config);
839		904
840	if (is_dsc_possible)	905	if (is_dsc_possible)
841	get_dsc_bandwidth_range(min_bpp_x16, max_bpp_x16, &dsc_common_caps, timing, range);	906	get_dsc_bandwidth_range(min_bpp_x16, max_bpp_x16,
		907	config.num_slices_h, &dsc_common_caps, timing, range);
842		908
843	return is_dsc_possible;	909	return is_dsc_possible;
844	}	910	}
@@ -864,13 +930,20 @@ bool dc_dsc_compute_config( · 19 unmodified lines
864	return is_dsc_possible;	930	return is_dsc_possible;
865	}	931	}
866		932
867	uint32_t dc_dsc_stream_bandwidth_in_kbps(uint32_t pix_clk_100hz, uint32_t bpp_x16)	933	uint32_t dc_dsc_stream_bandwidth_in_kbps(const struct dc_crtc_timing *timing,
		934	uint32_t bpp_x16, uint32_t num_slices_h, bool is_dp)
868	{	935	{
869	struct fixed31_32 link_bw_kbps;	936	struct fixed31_32 overhead_in_kbps;
870	link_bw_kbps = dc_fixpt_from_int(pix_clk_100hz);	937	struct fixed31_32 bpp;
871	link_bw_kbps = dc_fixpt_div_int(link_bw_kbps, 160);	938	struct fixed31_32 actual_bandwidth_in_kbps;
872	link_bw_kbps = dc_fixpt_mul_int(link_bw_kbps, bpp_x16);	939
873	return dc_fixpt_ceil(link_bw_kbps);	940	overhead_in_kbps = compute_dsc_max_bandwidth_overhead(
		941	timing, num_slices_h, is_dp);
		942	bpp = dc_fixpt_from_fraction(bpp_x16, 16);
		943	actual_bandwidth_in_kbps = dc_fixpt_from_fraction(timing->pix_clk_100hz, 10);
		944	actual_bandwidth_in_kbps = dc_fixpt_mul(actual_bandwidth_in_kbps, bpp);
		945	actual_bandwidth_in_kbps = dc_fixpt_add(actual_bandwidth_in_kbps, overhead_in_kbps);
		946	return dc_fixpt_ceil(actual_bandwidth_in_kbps);
874	}	947	}
875		948
876	void dc_dsc_get_policy_for_timing(const struct dc_crtc_timing timing, uint32_t max_target_bpp_limit_override_x16, struct dc_dsc_policy policy)	949	void dc_dsc_get_policy_for_timing(const struct dc_crtc_timing timing, uint32_t max_target_bpp_limit_override_x16, struct dc_dsc_policy policy)
@@ -954,3 +1027,8 @@ void dc_dsc_policy_set_enable_dsc_when_not_needed(bool enable) · 77 unmodified lines
954	{	1027	{
955	dsc_policy_enable_dsc_when_not_needed = enable;	1028	dsc_policy_enable_dsc_when_not_needed = enable;
956	}	1029	}
		1030
		1031	void dc_dsc_policy_set_disable_dsc_stream_overhead(bool disable)
		1032	{
		1033	dsc_policy_disable_dsc_stream_overhead = disable;
		1034	}

drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c+0−43

@@ -284,26 +284,6 @@ static u32 _do_bytes_per_pixel_calc(int slice_width, u16 drm_bpp, · 283 unmodified lines
284	return bytes_per_pixel;	284	return bytes_per_pixel;
285	}	285	}
286		286
287	static u32 _do_calc_dsc_bpp_x16(u32 stream_bandwidth_kbps, u32 pix_clk_100hz,
288	u32 bpp_increment_div)
289	{
290	u32 dsc_target_bpp_x16;
291	float f_dsc_target_bpp;
292	float f_stream_bandwidth_100bps;
293	// bpp_increment_div is actually precision
294	u32 precision = bpp_increment_div;
295
296	f_stream_bandwidth_100bps = stream_bandwidth_kbps * 10.0f;
297	f_dsc_target_bpp = f_stream_bandwidth_100bps / pix_clk_100hz;
298
299	// Round down to the nearest precision stop to bring it into DSC spec
300	// range
301	dsc_target_bpp_x16 = (u32)(f_dsc_target_bpp * precision);
302	dsc_target_bpp_x16 = (dsc_target_bpp_x16 * 16) / precision;
303
304	return dsc_target_bpp_x16;
305	}
306
307	/**	287	/**
308	* calc_rc_params - reads the user's cmdline mode	288	* calc_rc_params - reads the user's cmdline mode
309	* @rc: DC internal DSC parameters	289	* @rc: DC internal DSC parameters
@@ -367,26 +347,3 @@ u32 calc_dsc_bytes_per_pixel(const struct drm_dsc_config *pps) · 57 unmodified lines
367	DC_FP_END();	347	DC_FP_END();
368	return ret;	348	return ret;
369	}	349	}
370
371	/**
372	* calc_dsc_bpp_x16 - retrieve the dsc bits per pixel
373	* @stream_bandwidth_kbps:
374	* @pix_clk_100hz:
375	* @bpp_increment_div:
376	*
377	* Calculate the total of bits per pixel for DSC configuration.
378	*
379	* @note This calculation requires float point operation, most of it executes
380	* under kernel_fpu_{begin,end}.
381	*/
382	u32 calc_dsc_bpp_x16(u32 stream_bandwidth_kbps, u32 pix_clk_100hz,
383	u32 bpp_increment_div)
384	{
385	u32 dsc_bpp;
386
387	DC_FP_START();
388	dsc_bpp = _do_calc_dsc_bpp_x16(stream_bandwidth_kbps, pix_clk_100hz,
389	bpp_increment_div);
390	DC_FP_END();
391	return dsc_bpp;
392	}

drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h+0−2

@@ -79,8 +79,6 @@ typedef struct qp_entry qp_table[]; · 78 unmodified lines
79		79
80	void calc_rc_params(struct rc_params rc, const struct drm_dsc_config pps);	80	void calc_rc_params(struct rc_params rc, const struct drm_dsc_config pps);
81	u32 calc_dsc_bytes_per_pixel(const struct drm_dsc_config *pps);	81	u32 calc_dsc_bytes_per_pixel(const struct drm_dsc_config *pps);
82	u32 calc_dsc_bpp_x16(u32 stream_bandwidth_kbps, u32 pix_clk_100hz,
83	u32 bpp_increment_div);
84		82
85	#endif	83	#endif
86		84

drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h+11−1

 	bool skip_video_pattern,
 	int attempts,
 	struct pipe_ctx *pipe_ctx,
-	enum signal_type signal);
+	enum signal_type signal,
+	bool do_fallback);
 bool is_mst_supported(struct dc_link *link);
 bool is_dp_active_dongle(const struct dc_link *link);
+bool is_dp_branch_device(const struct dc_link *link);
 bool is_edp_ilr_optimization_required(struct dc_link *link, struct dc_crtc_timing *crtc_timing);
 void dp_enable_mst_on_sink(struct dc_link *link, bool enable);
 bool dp_update_dsc_config(struct pipe_ctx *pipe_ctx);
 bool dp_set_dsc_on_rx(struct pipe_ctx *pipe_ctx, bool enable);
+/* Convert PHY repeater count read from DPCD uint8_t. */
+uint8_t dp_convert_to_count(uint8_t lttpr_repeater_count);
+/* Check DPCD training status registers to detect link loss. */
+enum link_training_result dp_check_link_loss_status(
+		struct dc_link *link,
+		const struct link_training_settings *link_training_setting);
 #endif /* __DC_LINK_DP_H__ */

drivers/gpu/drm/amd/display/dc/inc/hw/dsc.h+1−0

@@ -88,6 +88,7 @@ struct dsc_enc_caps { · 87 unmodified lines
88	int32_t max_total_throughput_mps; /* Maximum total throughput with all the slices combined */	88	int32_t max_total_throughput_mps; /* Maximum total throughput with all the slices combined */
89	int32_t max_slice_width;	89	int32_t max_slice_width;
90	uint32_t bpp_increment_div; /* bpp increment divisor, e.g. if 16, it's 1/16th of a bit */	90	uint32_t bpp_increment_div; /* bpp increment divisor, e.g. if 16, it's 1/16th of a bit */
		91	bool is_dp;
91	};	92	};
92		93
93	struct dsc_funcs {	94	struct dsc_funcs {

drivers/gpu/drm/amd/display/dc/inc/link_enc_cfg.h+6−1

 /* Return DIG link encoder used by link. NULL if unused. */
 struct link_encoder *link_enc_cfg_get_link_enc_used_by_link(
 		struct dc_state *state,
-		struct dc_link *link);
+		const struct dc_link *link);
+/* Return next available DIG link encoder. NULL if none available. */
+struct link_encoder *link_enc_cfg_get_next_avail_link_enc(
+	const struct dc *dc,
+	const struct dc_state *state);
 #endif /* DC_INC_LINK_ENC_CFG_H_ */

drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c+9−9

@@ -58,8 +58,8 @@ enum dc_irq_source to_dal_irq_source_dcn21( · 57 unmodified lines
58	return DC_IRQ_SOURCE_VBLANK5;	58	return DC_IRQ_SOURCE_VBLANK5;
59	case DCN_1_0__SRCID__DC_D6_OTG_VSTARTUP:	59	case DCN_1_0__SRCID__DC_D6_OTG_VSTARTUP:
60	return DC_IRQ_SOURCE_VBLANK6;	60	return DC_IRQ_SOURCE_VBLANK6;
61	case DCN_1_0__SRCID__DMCUB_OUTBOX_HIGH_PRIORITY_READY_INT:	61	case DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT:
62	return DC_IRQ_SOURCE_DMCUB_OUTBOX0;	62	return DC_IRQ_SOURCE_DMCUB_OUTBOX;
63	case DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL:	63	case DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL:
64	return DC_IRQ_SOURCE_DC1_VLINE0;	64	return DC_IRQ_SOURCE_DC1_VLINE0;
65	case DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL:	65	case DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL:
@@ -187,7 +187,7 @@ static const struct irq_source_info_funcs vupdate_no_lock_irq_info_funcs = { · 121 unmodified lines
187	.ack = NULL	187	.ack = NULL
188	};	188	};
189		189
190	static const struct irq_source_info_funcs dmub_trace_irq_info_funcs = {	190	static const struct irq_source_info_funcs dmub_outbox_irq_info_funcs = {
191	.set = NULL,	191	.set = NULL,
192	.ack = NULL	192	.ack = NULL
193	};	193	};
@@ -301,11 +301,11 @@ static const struct irq_source_info_funcs vline0_irq_info_funcs = { · 107 unmodified lines
301	.funcs = &vline0_irq_info_funcs\	301	.funcs = &vline0_irq_info_funcs\
302	}	302	}
303		303
304	#define dmub_trace_int_entry()\	304	#define dmub_outbox_int_entry()\
305	[DC_IRQ_SOURCE_DMCUB_OUTBOX0] = {\	305	[DC_IRQ_SOURCE_DMCUB_OUTBOX] = {\
306	IRQ_REG_ENTRY_DMUB(DMCUB_INTERRUPT_ENABLE, DMCUB_OUTBOX0_READY_INT_EN,\	306	IRQ_REG_ENTRY_DMUB(DMCUB_INTERRUPT_ENABLE, DMCUB_OUTBOX1_READY_INT_EN,\
307	DMCUB_INTERRUPT_ACK, DMCUB_OUTBOX0_READY_INT_ACK),\	307	DMCUB_INTERRUPT_ACK, DMCUB_OUTBOX1_READY_INT_ACK),\
308	.funcs = &dmub_trace_irq_info_funcs\	308	.funcs = &dmub_outbox_irq_info_funcs\
309	}	309	}
310		310
311	#define dummy_irq_entry() \	311	#define dummy_irq_entry() \
@@ -426,7 +426,7 @@ irq_source_info_dcn21[DAL_IRQ_SOURCES_NUMBER] = { · 114 unmodified lines
426	vline0_int_entry(3),	426	vline0_int_entry(3),
427	vline0_int_entry(4),	427	vline0_int_entry(4),
428	vline0_int_entry(5),	428	vline0_int_entry(5),
429	dmub_trace_int_entry(),	429	dmub_outbox_int_entry(),
430	};	430	};
431		431
432	static const struct irq_service_funcs irq_service_funcs_dcn21 = {	432	static const struct irq_service_funcs irq_service_funcs_dcn21 = {

drivers/gpu/drm/amd/display/dc/irq_types.h+1−1

@@ -150,7 +150,7 @@ enum dc_irq_source { · 149 unmodified lines
150	DC_IRQ_SOURCE_DC4_VLINE1,	150	DC_IRQ_SOURCE_DC4_VLINE1,
151	DC_IRQ_SOURCE_DC5_VLINE1,	151	DC_IRQ_SOURCE_DC5_VLINE1,
152	DC_IRQ_SOURCE_DC6_VLINE1,	152	DC_IRQ_SOURCE_DC6_VLINE1,
153	DC_IRQ_DMCUB_OUTBOX1,	153	DC_IRQ_SOURCE_DMCUB_OUTBOX,
154	DC_IRQ_SOURCE_DMCUB_OUTBOX0,	154	DC_IRQ_SOURCE_DMCUB_OUTBOX0,
155		155
156	DAL_IRQ_SOURCES_NUMBER	156	DAL_IRQ_SOURCES_NUMBER

drivers/gpu/drm/amd/display/dmub/dmub_srv.h+19−18

 	struct dmub_fb fb[DMUB_WINDOW_TOTAL];
 };
+/*
+ * struct dmub_srv_hw_params - params for dmub hardware initialization
+ * @fb: framebuffer info for each region
+ * @fb_base: base of the framebuffer aperture
+ * @fb_offset: offset of the framebuffer aperture
+ * @psp_version: psp version to pass for DMCU init
+ * @load_inst_const: true if DMUB should load inst const fw
+ */
+struct dmub_srv_hw_params {
+	struct dmub_fb *fb[DMUB_WINDOW_TOTAL];
+	uint64_t fb_base;
+	uint64_t fb_offset;
+	uint32_t psp_version;
+	bool load_inst_const;
+	bool skip_panel_power_sequence;
+};
 /**
  * struct dmub_srv_base_funcs - Driver specific base callbacks
  */
 	bool (*is_hw_init)(struct dmub_srv *dmub);
 	bool (*is_phy_init)(struct dmub_srv *dmub);
-	void (*enable_dmub_boot_options)(struct dmub_srv *dmub);
+	void (*enable_dmub_boot_options)(struct dmub_srv *dmub,
+				const struct dmub_srv_hw_params *params);
 	void (*skip_dmub_panel_power_sequence)(struct dmub_srv *dmub, bool skip);
 	bool is_virtual;
 };
-/*
- * struct dmub_srv_hw_params - params for dmub hardware initialization
- * @fb: framebuffer info for each region
- * @fb_base: base of the framebuffer aperture
- * @fb_offset: offset of the framebuffer aperture
- * @psp_version: psp version to pass for DMCU init
- * @load_inst_const: true if DMUB should load inst const fw
- */
-struct dmub_srv_hw_params {
-	struct dmub_fb *fb[DMUB_WINDOW_TOTAL];
-	uint64_t fb_base;
-	uint64_t fb_offset;
-	uint32_t psp_version;
-	bool load_inst_const;
-	bool skip_panel_power_sequence;
-};
 /**
  * struct dmub_srv - software state for dmcub
  * @asic: dmub asic identifier

drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h+194−13

@@ -47,10 +47,10 @@ · 46 unmodified lines
47		47
48	/* Firmware versioning. */	48	/* Firmware versioning. */
49	#ifdef DMUB_EXPOSE_VERSION	49	#ifdef DMUB_EXPOSE_VERSION
50	#define DMUB_FW_VERSION_GIT_HASH 0x23db9b126	50	#define DMUB_FW_VERSION_GIT_HASH 0x2cab49dfb
51	#define DMUB_FW_VERSION_MAJOR 0	51	#define DMUB_FW_VERSION_MAJOR 0
52	#define DMUB_FW_VERSION_MINOR 0	52	#define DMUB_FW_VERSION_MINOR 0
53	#define DMUB_FW_VERSION_REVISION 62	53	#define DMUB_FW_VERSION_REVISION 65
54	#define DMUB_FW_VERSION_TEST 0	54	#define DMUB_FW_VERSION_TEST 0
55	#define DMUB_FW_VERSION_VBIOS 0	55	#define DMUB_FW_VERSION_VBIOS 0
56	#define DMUB_FW_VERSION_HOTFIX 0	56	#define DMUB_FW_VERSION_HOTFIX 0
@@ -117,18 +117,20 @@ · 60 unmodified lines
117	/* Maximum number of planes on any ASIC. */	117	/* Maximum number of planes on any ASIC. */
118	#define DMUB_MAX_PLANES 6	118	#define DMUB_MAX_PLANES 6
119		119
		120	#define DMUB_MAX_SUBVP_STREAMS 2
		121
120	/* Trace buffer offset for entry */	122	/* Trace buffer offset for entry */
121	#define TRACE_BUFFER_ENTRY_OFFSET 16	123	#define TRACE_BUFFER_ENTRY_OFFSET 16
122		124
123	/**	125	/**
124	* ABM backlight control version legacy	126	* ABM control version legacy
125	*/	127	*/
126	#define DMUB_CMD_ABM_SET_BACKLIGHT_VERSION_UNKNOWN 0x0	128	#define DMUB_CMD_ABM_CONTROL_VERSION_UNKNOWN 0x0
127		129
128	/**	130	/**
129	* ABM backlight control version with multi edp support	131	* ABM control version with multi edp support
130	*/	132	*/
131	#define DMUB_CMD_ABM_SET_BACKLIGHT_VERSION_1 0x1	133	#define DMUB_CMD_ABM_CONTROL_VERSION_1 0x1
132		134
133	/**	135	/**
134	* Physical framebuffer address location, 64-bit.	136	* Physical framebuffer address location, 64-bit.
@@ -162,6 +164,13 @@ extern "C" { · 27 unmodified lines
162	#define dmub_udelay(microseconds) udelay(microseconds)	164	#define dmub_udelay(microseconds) udelay(microseconds)
163	#endif	165	#endif
164		166
		167	/**
		168	* Number of nanoseconds per DMUB tick.
		169	* DMCUB_TIMER_CURRENT increments in DMUB ticks, which are 10ns by default.
		170	* If DMCUB_TIMER_WINDOW is non-zero this will no longer be true.
		171	*/
		172	#define NS_PER_DMUB_TICK 10
		173
165	/**	174	/**
166	* union dmub_addr - DMUB physical/virtual 64-bit address.	175	* union dmub_addr - DMUB physical/virtual 64-bit address.
167	*/	176	*/
@@ -328,7 +337,8 @@ union dmub_fw_boot_options { · 160 unmodified lines
328	uint32_t skip_phy_access : 1; /*< 1 if PHY access should be skipped /	337	uint32_t skip_phy_access : 1; /*< 1 if PHY access should be skipped /
329	uint32_t disable_clk_gate: 1; /*< 1 if clock gating should be disabled /	338	uint32_t disable_clk_gate: 1; /*< 1 if clock gating should be disabled /
330	uint32_t skip_phy_init_panel_sequence: 1; /*< 1 to skip panel init seq /	339	uint32_t skip_phy_init_panel_sequence: 1; /*< 1 to skip panel init seq /
331	uint32_t reserved : 26; /*< reserved /	340	uint32_t reserved_unreleased: 1; /*< reserved for an unreleased feature /
		341	uint32_t reserved : 25; /*< reserved /
332	} bits; /*< boot bits /	342	} bits; /*< boot bits /
333	uint32_t all; /*< 32-bit access to bits /	343	uint32_t all; /*< 32-bit access to bits /
334	};	344	};
@@ -452,6 +462,61 @@ enum dmub_gpint_command { · 117 unmodified lines
452	DMUB_GPINT__PSR_RESIDENCY = 9,	462	DMUB_GPINT__PSR_RESIDENCY = 9,
453	};	463	};
454		464
		465	/**
		466	* INBOX0 generic command definition
		467	*/
		468	union dmub_inbox0_cmd_common {
		469	struct {
		470	uint32_t command_code: 8; /*< INBOX0 command code /
		471	uint32_t param: 24; /*< 24-bit parameter /
		472	} bits;
		473	uint32_t all;
		474	};
		475
		476	/**
		477	* INBOX0 hw_lock command definition
		478	*/
		479	union dmub_inbox0_cmd_lock_hw {
		480	struct {
		481	uint32_t command_code: 8;
		482
		483	/* NOTE: Must be have enough bits to match: enum hw_lock_client */
		484	uint32_t hw_lock_client: 1;
		485
		486	/* NOTE: Below fields must match with: struct dmub_hw_lock_inst_flags */
		487	uint32_t otg_inst: 3;
		488	uint32_t opp_inst: 3;
		489	uint32_t dig_inst: 3;
		490
		491	/* NOTE: Below fields must match with: union dmub_hw_lock_flags */
		492	uint32_t lock_pipe: 1;
		493	uint32_t lock_cursor: 1;
		494	uint32_t lock_dig: 1;
		495	uint32_t triple_buffer_lock: 1;
		496
		497	uint32_t lock: 1; /*< Lock /
		498	uint32_t should_release: 1; /*< Release /
		499	uint32_t reserved: 8; /*< Reserved for extending more clients, HW, etc. /
		500	} bits;
		501	uint32_t all;
		502	};
		503
		504	union dmub_inbox0_data_register {
		505	union dmub_inbox0_cmd_common inbox0_cmd_common;
		506	union dmub_inbox0_cmd_lock_hw inbox0_cmd_lock_hw;
		507	};
		508
		509	enum dmub_inbox0_command {
		510	/**
		511	* DESC: Invalid command, ignored.
		512	*/
		513	DMUB_INBOX0_CMD__INVALID_COMMAND = 0,
		514	/**
		515	* DESC: Notification to acquire/release HW lock
		516	* ARGS:
		517	*/
		518	DMUB_INBOX0_CMD__HW_LOCK = 1,
		519	};
455	//==============================================================================	520	//==============================================================================
456	//</DMUB_GPINT>=================================================================	521	//</DMUB_GPINT>=================================================================
457	//==============================================================================	522	//==============================================================================
@@ -570,7 +635,8 @@ struct dmub_cmd_header { · 112 unmodified lines
570	unsigned int type : 8; /*< command type /	635	unsigned int type : 8; /*< command type /
571	unsigned int sub_type : 8; /*< command sub type /	636	unsigned int sub_type : 8; /*< command sub type /
572	unsigned int ret_status : 1; /*< 1 if returned data, 0 otherwise /	637	unsigned int ret_status : 1; /*< 1 if returned data, 0 otherwise /
573	unsigned int reserved0 : 7; /*< reserved bits /	638	unsigned int multi_cmd_pending : 1; /*< 1 if multiple commands chained together /
		639	unsigned int reserved0 : 6; /*< reserved bits /
574	unsigned int payload_bytes : 6; /* payload excluding header - up to 60 bytes */	640	unsigned int payload_bytes : 6; /* payload excluding header - up to 60 bytes */
575	unsigned int reserved1 : 2; /*< reserved bits /	641	unsigned int reserved1 : 2; /*< reserved bits /
576	};	642	};
@@ -1343,6 +1409,9 @@ struct dmub_rb_cmd_psr_force_static { · 766 unmodified lines
1343		1409
1344	/**	1410	/**
1345	* Set of HW components that can be locked.	1411	* Set of HW components that can be locked.
		1412	*
		1413	* Note: If updating with more HW components, fields
		1414	* in dmub_inbox0_cmd_lock_hw must be updated to match.
1346	*/	1415	*/
1347	union dmub_hw_lock_flags {	1416	union dmub_hw_lock_flags {
1348	/**	1417	/**
@@ -1375,6 +1444,9 @@ union dmub_hw_lock_flags { · 26 unmodified lines
1375		1444
1376	/**	1445	/**
1377	* Instances of HW to be locked.	1446	* Instances of HW to be locked.
		1447	*
		1448	* Note: If updating with more HW components, fields
		1449	* in dmub_inbox0_cmd_lock_hw must be updated to match.
1378	*/	1450	*/
1379	struct dmub_hw_lock_inst_flags {	1451	struct dmub_hw_lock_inst_flags {
1380	/**	1452	/**
@@ -1398,16 +1470,16 @@ struct dmub_hw_lock_inst_flags { · 17 unmodified lines
1398		1470
1399	/**	1471	/**
1400	* Clients that can acquire the HW Lock Manager.	1472	* Clients that can acquire the HW Lock Manager.
		1473	*
		1474	* Note: If updating with more clients, fields in
		1475	* dmub_inbox0_cmd_lock_hw must be updated to match.
1401	*/	1476	*/
1402	enum hw_lock_client {	1477	enum hw_lock_client {
1403	/**	1478	/**
1404	* Driver is the client of HW Lock Manager.	1479	* Driver is the client of HW Lock Manager.
1405	*/	1480	*/
1406	HW_LOCK_CLIENT_DRIVER = 0,	1481	HW_LOCK_CLIENT_DRIVER = 0,
1407	/**	1482	HW_LOCK_CLIENT_SUBVP = 3,
1408	* FW is the client of HW Lock Manager.
1409	*/
1410	HW_LOCK_CLIENT_FW,
1411	/**	1483	/**
1412	* Invalid client.	1484	* Invalid client.
1413	*/	1485	*/
@@ -1637,7 +1709,7 @@ struct dmub_cmd_abm_set_backlight_data { · 223 unmodified lines
1637	uint32_t backlight_user_level;	1709	uint32_t backlight_user_level;
1638		1710
1639	/**	1711	/**
1640	* Backlight data version.	1712	* ABM control version.
1641	*/	1713	*/
1642	uint8_t version;	1714	uint8_t version;
1643		1715
@@ -1677,6 +1749,23 @@ struct dmub_cmd_abm_set_level_data { · 33 unmodified lines
1677	* Set current ABM operating/aggression level.	1749	* Set current ABM operating/aggression level.
1678	*/	1750	*/
1679	uint32_t level;	1751	uint32_t level;
		1752
		1753	/**
		1754	* ABM control version.
		1755	*/
		1756	uint8_t version;
		1757
		1758	/**
		1759	* Panel Control HW instance mask.
		1760	* Bit 0 is Panel Control HW instance 0.
		1761	* Bit 1 is Panel Control HW instance 1.
		1762	*/
		1763	uint8_t panel_mask;
		1764
		1765	/**
		1766	* Explicit padding to 4 byte boundary.
		1767	*/
		1768	uint8_t pad[2];
1680	};	1769	};
1681		1770
1682	/**	1771	/**
@@ -1702,6 +1791,23 @@ struct dmub_cmd_abm_set_ambient_level_data { · 19 unmodified lines
1702	* Ambient light sensor reading from OS.	1791	* Ambient light sensor reading from OS.
1703	*/	1792	*/
1704	uint32_t ambient_lux;	1793	uint32_t ambient_lux;
		1794
		1795	/**
		1796	* ABM control version.
		1797	*/
		1798	uint8_t version;
		1799
		1800	/**
		1801	* Panel Control HW instance mask.
		1802	* Bit 0 is Panel Control HW instance 0.
		1803	* Bit 1 is Panel Control HW instance 1.
		1804	*/
		1805	uint8_t panel_mask;
		1806
		1807	/**
		1808	* Explicit padding to 4 byte boundary.
		1809	*/
		1810	uint8_t pad[2];
1705	};	1811	};
1706		1812
1707	/**	1813	/**
@@ -1728,6 +1834,23 @@ struct dmub_cmd_abm_set_pwm_frac_data { · 20 unmodified lines
1728	* TODO: Convert to uint8_t.	1834	* TODO: Convert to uint8_t.
1729	*/	1835	*/
1730	uint32_t fractional_pwm;	1836	uint32_t fractional_pwm;
		1837
		1838	/**
		1839	* ABM control version.
		1840	*/
		1841	uint8_t version;
		1842
		1843	/**
		1844	* Panel Control HW instance mask.
		1845	* Bit 0 is Panel Control HW instance 0.
		1846	* Bit 1 is Panel Control HW instance 1.
		1847	*/
		1848	uint8_t panel_mask;
		1849
		1850	/**
		1851	* Explicit padding to 4 byte boundary.
		1852	*/
		1853	uint8_t pad[2];
1731	};	1854	};
1732		1855
1733	/**	1856	/**
@@ -1758,6 +1881,24 @@ struct dmub_cmd_abm_init_config_data { · 24 unmodified lines
1758	* Indirect buffer length.	1881	* Indirect buffer length.
1759	*/	1882	*/
1760	uint16_t bytes;	1883	uint16_t bytes;
		1884
		1885
		1886	/**
		1887	* ABM control version.
		1888	*/
		1889	uint8_t version;
		1890
		1891	/**
		1892	* Panel Control HW instance mask.
		1893	* Bit 0 is Panel Control HW instance 0.
		1894	* Bit 1 is Panel Control HW instance 1.
		1895	*/
		1896	uint8_t panel_mask;
		1897
		1898	/**
		1899	* Explicit padding to 4 byte boundary.
		1900	*/
		1901	uint8_t pad[2];
1761	};	1902	};
1762		1903
1763	/**	1904	/**
@@ -2126,6 +2267,46 @@ static inline bool dmub_rb_front(struct dmub_rb *rb, · 362 unmodified lines
2126	return true;	2267	return true;
2127	}	2268	}
2128		2269
		2270	/**
		2271	* @brief Determines the next ringbuffer offset.
		2272	*
		2273	* @param rb DMUB inbox ringbuffer
		2274	* @param num_cmds Number of commands
		2275	* @param next_rptr The next offset in the ringbuffer
		2276	*/
		2277	static inline void dmub_rb_get_rptr_with_offset(struct dmub_rb *rb,
		2278	uint32_t num_cmds,
		2279	uint32_t *next_rptr)
		2280	{
		2281	next_rptr = rb->rptr + DMUB_RB_CMD_SIZE num_cmds;
		2282
		2283	if (*next_rptr >= rb->capacity)
		2284	*next_rptr %= rb->capacity;
		2285	}
		2286
		2287	/**
		2288	* @brief Returns a pointer to a command in the inbox.
		2289	*
		2290	* @param rb DMUB inbox ringbuffer
		2291	* @param cmd The inbox command to return
		2292	* @param rptr The ringbuffer offset
		2293	* @return true if not empty
		2294	* @return false otherwise
		2295	*/
		2296	static inline bool dmub_rb_peek_offset(struct dmub_rb *rb,
		2297	union dmub_rb_cmd **cmd,
		2298	uint32_t rptr)
		2299	{
		2300	uint8_t rb_cmd = (uint8_t )(rb->base_address) + rptr;
		2301
		2302	if (dmub_rb_empty(rb))
		2303	return false;
		2304
		2305	cmd = (union dmub_rb_cmd )rb_cmd;
		2306
		2307	return true;
		2308	}
		2309
2129	/**	2310	/**
2130	* @brief Returns the next unprocessed command in the outbox.	2311	* @brief Returns the next unprocessed command in the outbox.
2131	*	2312	*

drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c+1−1

@@ -385,7 +385,7 @@ union dmub_fw_boot_status dmub_dcn20_get_fw_boot_status(struct dmub_srv *dmub) · 384 unmodified lines
385	return status;	385	return status;
386	}	386	}
387		387
388	void dmub_dcn20_enable_dmub_boot_options(struct dmub_srv *dmub)	388	void dmub_dcn20_enable_dmub_boot_options(struct dmub_srv dmub, const struct dmub_srv_hw_params params)
389	{	389	{
390	union dmub_fw_boot_options boot_options = {0};	390	union dmub_fw_boot_options boot_options = {0};
391		391

drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h+1−1

@@ -221,7 +221,7 @@ bool dmub_dcn20_is_gpint_acked(struct dmub_srv *dmub, · 220 unmodified lines
221		221
222	uint32_t dmub_dcn20_get_gpint_response(struct dmub_srv *dmub);	222	uint32_t dmub_dcn20_get_gpint_response(struct dmub_srv *dmub);
223		223
224	void dmub_dcn20_enable_dmub_boot_options(struct dmub_srv *dmub);	224	void dmub_dcn20_enable_dmub_boot_options(struct dmub_srv dmub, const struct dmub_srv_hw_params params);
225		225
226	void dmub_dcn20_skip_dmub_panel_power_sequence(struct dmub_srv *dmub, bool skip);	226	void dmub_dcn20_skip_dmub_panel_power_sequence(struct dmub_srv *dmub, bool skip);
227		227

drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c+4−0

@@ -514,6 +514,10 @@ enum dmub_status dmub_srv_hw_init(struct dmub_srv *dmub, · 513 unmodified lines
514	outbox0_rb_params.capacity = tracebuff_fb->size - dmub_align(TRACE_BUFFER_ENTRY_OFFSET, 64);	514	outbox0_rb_params.capacity = tracebuff_fb->size - dmub_align(TRACE_BUFFER_ENTRY_OFFSET, 64);
515	dmub_rb_init(&dmub->outbox0_rb, &outbox0_rb_params);	515	dmub_rb_init(&dmub->outbox0_rb, &outbox0_rb_params);
516		516
		517	/* Report to DMUB what features are supported by current driver */
		518	if (dmub->hw_funcs.enable_dmub_boot_options)
		519	dmub->hw_funcs.enable_dmub_boot_options(dmub, params);
		520
517	if (dmub->hw_funcs.reset_release)	521	if (dmub->hw_funcs.reset_release)
518	dmub->hw_funcs.reset_release(dmub);	522	dmub->hw_funcs.reset_release(dmub);
519		523

drivers/gpu/drm/amd/display/include/link_service_types.h+8−0

@@ -68,6 +68,14 @@ enum link_training_result { · 67 unmodified lines
68	LINK_TRAINING_LQA_FAIL,	68	LINK_TRAINING_LQA_FAIL,
69	/* one of the CR,EQ or symbol lock is dropped */	69	/* one of the CR,EQ or symbol lock is dropped */
70	LINK_TRAINING_LINK_LOSS,	70	LINK_TRAINING_LINK_LOSS,
		71	/* Abort link training (because sink unplugged) */
		72	LINK_TRAINING_ABORT,
		73	};
		74
		75	enum lttpr_mode {
		76	LTTPR_MODE_NON_LTTPR,
		77	LTTPR_MODE_TRANSPARENT,
		78	LTTPR_MODE_NON_TRANSPARENT,
71	};	79	};
72		80
73	struct link_training_settings {	81	struct link_training_settings {

drivers/gpu/drm/amd/display/modules/hdcp/hdcp.c+0−1

@@ -260,7 +260,6 @@ enum mod_hdcp_status mod_hdcp_setup(struct mod_hdcp *hdcp, · 259 unmodified lines
260	struct mod_hdcp_output output;	260	struct mod_hdcp_output output;
261	enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS;	261	enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS;
262		262
263	memset(hdcp, 0, sizeof(struct mod_hdcp));
264	memset(&output, 0, sizeof(output));	263	memset(&output, 0, sizeof(output));
265	hdcp->config = *config;	264	hdcp->config = *config;
266	HDCP_TOP_INTERFACE_TRACE(hdcp);	265	HDCP_TOP_INTERFACE_TRACE(hdcp);

drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_execution.c+5−0

 static inline enum mod_hdcp_status check_device_count(struct mod_hdcp *hdcp)
 {
+	/* Avoid device count == 0 to do authentication */
+	if (0 == get_device_count(hdcp)) {
+		return MOD_HDCP_STATUS_HDCP1_DEVICE_COUNT_MISMATCH_FAILURE;
+	}
 	/* Some MST display may choose to report the internal panel as an HDCP RX.
 	 * To update this condition with 1(because the immediate repeater's internal
 	 * panel is possibly not included in DEVICE_COUNT) + get_device_count(hdcp).

drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_execution.c+5−0

 static enum mod_hdcp_status check_device_count(struct mod_hdcp *hdcp)
 {
+	/* Avoid device count == 0 to do authentication */
+	if (0 == get_device_count(hdcp)) {
+		return MOD_HDCP_STATUS_HDCP1_DEVICE_COUNT_MISMATCH_FAILURE;
+	}
 	/* Some MST display may choose to report the internal panel as an HDCP RX.   */
 	/* To update this condition with 1(because the immediate repeater's internal */
 	/* panel is possibly not included in DEVICE_COUNT) + get_device_count(hdcp). */

drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_sh_mask.h+1−0

@@ -50,6 +50,7 @@ · 49 unmodified lines
50	#define DF_CS_UMC_AON0_DramBaseAddress0__AddrRngVal_MASK 0x00000001L	50	#define DF_CS_UMC_AON0_DramBaseAddress0__AddrRngVal_MASK 0x00000001L
51	#define DF_CS_UMC_AON0_DramBaseAddress0__LgcyMmioHoleEn_MASK 0x00000002L	51	#define DF_CS_UMC_AON0_DramBaseAddress0__LgcyMmioHoleEn_MASK 0x00000002L
52	#define DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK 0x0000003CL	52	#define DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK 0x0000003CL
		53	#define ALDEBARAN_DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK 0x0000007CL
53	#define DF_CS_UMC_AON0_DramBaseAddress0__IntLvAddrSel_MASK 0x00000E00L	54	#define DF_CS_UMC_AON0_DramBaseAddress0__IntLvAddrSel_MASK 0x00000E00L
54	#define DF_CS_UMC_AON0_DramBaseAddress0__DramBaseAddr_MASK 0xFFFFF000L	55	#define DF_CS_UMC_AON0_DramBaseAddress0__DramBaseAddr_MASK 0xFFFFF000L
55		56

drivers/gpu/drm/amd/include/kgd_pp_interface.h+63−0

@@ -350,6 +350,7 @@ struct amd_pm_funcs { · 349 unmodified lines
350	unsigned int *num_states);	350	unsigned int *num_states);
351	int (get_dpm_clock_table)(void handle,	351	int (get_dpm_clock_table)(void handle,
352	struct dpm_clocks *clock_table);	352	struct dpm_clocks *clock_table);
		353	int (get_smu_prv_buf_details)(void handle, void *addr, size_t size);
353	};	354	};
354		355
355	struct metrics_table_header {	356	struct metrics_table_header {
@@ -473,6 +474,68 @@ struct gpu_metrics_v1_1 { · 117 unmodified lines
473	uint16_t temperature_hbm[NUM_HBM_INSTANCES];	474	uint16_t temperature_hbm[NUM_HBM_INSTANCES];
474	};	475	};
475		476
		477	struct gpu_metrics_v1_2 {
		478	struct metrics_table_header common_header;
		479
		480	/* Temperature */
		481	uint16_t temperature_edge;
		482	uint16_t temperature_hotspot;
		483	uint16_t temperature_mem;
		484	uint16_t temperature_vrgfx;
		485	uint16_t temperature_vrsoc;
		486	uint16_t temperature_vrmem;
		487
		488	/* Utilization */
		489	uint16_t average_gfx_activity;
		490	uint16_t average_umc_activity; // memory controller
		491	uint16_t average_mm_activity; // UVD or VCN
		492
		493	/* Power/Energy */
		494	uint16_t average_socket_power;
		495	uint64_t energy_accumulator;
		496
		497	/* Driver attached timestamp (in ns) */
		498	uint64_t system_clock_counter;
		499
		500	/* Average clocks */
		501	uint16_t average_gfxclk_frequency;
		502	uint16_t average_socclk_frequency;
		503	uint16_t average_uclk_frequency;
		504	uint16_t average_vclk0_frequency;
		505	uint16_t average_dclk0_frequency;
		506	uint16_t average_vclk1_frequency;
		507	uint16_t average_dclk1_frequency;
		508
		509	/* Current clocks */
		510	uint16_t current_gfxclk;
		511	uint16_t current_socclk;
		512	uint16_t current_uclk;
		513	uint16_t current_vclk0;
		514	uint16_t current_dclk0;
		515	uint16_t current_vclk1;
		516	uint16_t current_dclk1;
		517
		518	/* Throttle status */
		519	uint32_t throttle_status;
		520
		521	/* Fans */
		522	uint16_t current_fan_speed;
		523
		524	/* Link width/speed */
		525	uint16_t pcie_link_width;
		526	uint16_t pcie_link_speed; // in 0.1 GT/s
		527
		528	uint16_t padding;
		529
		530	uint32_t gfx_activity_acc;
		531	uint32_t mem_activity_acc;
		532
		533	uint16_t temperature_hbm[NUM_HBM_INSTANCES];
		534
		535	/* PMFW attached timestamp (10ns resolution) */
		536	uint64_t firmware_timestamp;
		537	};
		538
476	/*	539	/*
477	* gpu_metrics_v2_0 is not recommended as it's not naturally aligned.	540	* gpu_metrics_v2_0 is not recommended as it's not naturally aligned.
478	* Use gpu_metrics_v2_1 or later instead.	541	* Use gpu_metrics_v2_1 or later instead.

drivers/gpu/drm/amd/pm/amdgpu_pm.c+65−1

@@ -735,6 +735,23 @@ static ssize_t amdgpu_set_pp_table(struct device *dev, · 734 unmodified lines
735	* - a list of valid ranges for sclk, mclk, and voltage curve points	735	* - a list of valid ranges for sclk, mclk, and voltage curve points
736	* labeled OD_RANGE	736	* labeled OD_RANGE
737	*	737	*
		738	* < For APUs >
		739	*
		740	* Reading the file will display:
		741	*
		742	* - minimum and maximum engine clock labeled OD_SCLK
		743	*
		744	* - a list of valid ranges for sclk labeled OD_RANGE
		745	*
		746	* < For VanGogh >
		747	*
		748	* Reading the file will display:
		749	*
		750	* - minimum and maximum engine clock labeled OD_SCLK
		751	* - minimum and maximum core clocks labeled OD_CCLK
		752	*
		753	* - a list of valid ranges for sclk and cclk labeled OD_RANGE
		754	*
738	* To manually adjust these settings:	755	* To manually adjust these settings:
739	*	756	*
740	* - First select manual using power_dpm_force_performance_level	757	* - First select manual using power_dpm_force_performance_level
@@ -743,7 +760,10 @@ static ssize_t amdgpu_set_pp_table(struct device *dev, · 2 unmodified lines
743	* string that contains "s/m index clock" to the file. The index	760	* string that contains "s/m index clock" to the file. The index
744	* should be 0 if to set minimum clock. And 1 if to set maximum	761	* should be 0 if to set minimum clock. And 1 if to set maximum
745	* clock. E.g., "s 0 500" will update minimum sclk to be 500 MHz.	762	* clock. E.g., "s 0 500" will update minimum sclk to be 500 MHz.
746	* "m 1 800" will update maximum mclk to be 800Mhz.	763	* "m 1 800" will update maximum mclk to be 800Mhz. For core
		764	* clocks on VanGogh, the string contains "p core index clock".
		765	* E.g., "p 2 0 800" would set the minimum core clock on core
		766	* 2 to 800Mhz.
747	*	767	*
748	* For sclk voltage curve, enter the new values by writing a	768	* For sclk voltage curve, enter the new values by writing a
749	* string that contains "vc point clock voltage" to the file. The	769	* string that contains "vc point clock voltage" to the file. The
@@ -3534,6 +3554,45 @@ out: · 2784 unmodified lines
3534		3554
3535	DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_pm_info);	3555	DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_pm_info);
3536		3556
		3557	/*
		3558	* amdgpu_pm_priv_buffer_read - Read memory region allocated to FW
		3559	*
		3560	* Reads debug memory region allocated to PMFW
		3561	*/
		3562	static ssize_t amdgpu_pm_prv_buffer_read(struct file f, char __user buf,
		3563	size_t size, loff_t *pos)
		3564	{
		3565	struct amdgpu_device *adev = file_inode(f)->i_private;
		3566	const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
		3567	void *pp_handle = adev->powerplay.pp_handle;
		3568	size_t smu_prv_buf_size;
		3569	void *smu_prv_buf;
		3570
		3571	if (amdgpu_in_reset(adev))
		3572	return -EPERM;
		3573	if (adev->in_suspend && !adev->in_runpm)
		3574	return -EPERM;
		3575
		3576	if (pp_funcs && pp_funcs->get_smu_prv_buf_details)
		3577	pp_funcs->get_smu_prv_buf_details(pp_handle, &smu_prv_buf,
		3578	&smu_prv_buf_size);
		3579	else
		3580	return -ENOSYS;
		3581
		3582	if (!smu_prv_buf \|\| !smu_prv_buf_size)
		3583	return -EINVAL;
		3584
		3585	return simple_read_from_buffer(buf, size, pos, smu_prv_buf,
		3586	smu_prv_buf_size);
		3587	}
		3588
		3589	static const struct file_operations amdgpu_debugfs_pm_prv_buffer_fops = {
		3590	.owner = THIS_MODULE,
		3591	.open = simple_open,
		3592	.read = amdgpu_pm_prv_buffer_read,
		3593	.llseek = default_llseek,
		3594	};
		3595
3537	#endif	3596	#endif
3538		3597
3539	void amdgpu_debugfs_pm_init(struct amdgpu_device *adev)	3598	void amdgpu_debugfs_pm_init(struct amdgpu_device *adev)
@@ -3545,5 +3604,10 @@ void amdgpu_debugfs_pm_init(struct amdgpu_device *adev) · 5 unmodified lines
3545	debugfs_create_file("amdgpu_pm_info", 0444, root, adev,	3604	debugfs_create_file("amdgpu_pm_info", 0444, root, adev,
3546	&amdgpu_debugfs_pm_info_fops);	3605	&amdgpu_debugfs_pm_info_fops);
3547		3606
		3607	if (adev->pm.smu_prv_buffer_size > 0)
		3608	debugfs_create_file_size("amdgpu_pm_prv_buffer", 0444, root,
		3609	adev,
		3610	&amdgpu_debugfs_pm_prv_buffer_fops,
		3611	adev->pm.smu_prv_buffer_size);
3548	#endif	3612	#endif
3549	}	3613	}

drivers/gpu/drm/amd/pm/inc/smu13_driver_if_aldebaran.h+5−2

 #define FEATURE_SMUIO_CG_BIT            28
 #define FEATURE_THM_CG_BIT              29
 #define FEATURE_CLK_CG_BIT              30
-#define FEATURE_SPARE_31_BIT            31
+#define FEATURE_EDC_BIT                 31
 #define FEATURE_SPARE_32_BIT            32
 #define FEATURE_SPARE_33_BIT            33
 #define FEATURE_SPARE_34_BIT            34
   int8_t   XgmiOffset;     // in Amps
   uint8_t  Padding_TelemetryXgmi;
+  uint16_t  EdcPowerLimit;
+  uint16_t  spare6;
   //reserved
-  uint32_t reserved[15];
+  uint32_t reserved[14];
 } PPTable_t;

drivers/gpu/drm/amd/pm/inc/smu_v13_0.h+1−1

@@ -26,7 +26,7 @@ · 25 unmodified lines
26	#include "amdgpu_smu.h"	26	#include "amdgpu_smu.h"
27		27
28	#define SMU13_DRIVER_IF_VERSION_INV 0xFFFFFFFF	28	#define SMU13_DRIVER_IF_VERSION_INV 0xFFFFFFFF
29	#define SMU13_DRIVER_IF_VERSION_ALDE 0x6	29	#define SMU13_DRIVER_IF_VERSION_ALDE 0x07
30		30
31	/* MP Apertures */	31	/* MP Apertures */
32	#define MP0_Public 0x03800000	32	#define MP0_Public 0x03800000

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c+21−0

@@ -1651,6 +1651,26 @@ static int pp_gfx_state_change_set(void *handle, uint32_t state) · 1650 unmodified lines
1651	return 0;	1651	return 0;
1652	}	1652	}
1653		1653
		1654	static int pp_get_prv_buffer_details(void handle, void addr, size_t size)
		1655	{
		1656	struct pp_hwmgr *hwmgr = handle;
		1657	struct amdgpu_device *adev = hwmgr->adev;
		1658
		1659	if (!addr \|\| !size)
		1660	return -EINVAL;
		1661
		1662	*addr = NULL;
		1663	*size = 0;
		1664	mutex_lock(&hwmgr->smu_lock);
		1665	if (adev->pm.smu_prv_buffer) {
		1666	amdgpu_bo_kmap(adev->pm.smu_prv_buffer, addr);
		1667	*size = adev->pm.smu_prv_buffer_size;
		1668	}
		1669	mutex_unlock(&hwmgr->smu_lock);
		1670
		1671	return 0;
		1672	}
		1673
1654	static const struct amd_pm_funcs pp_dpm_funcs = {	1674	static const struct amd_pm_funcs pp_dpm_funcs = {
1655	.load_firmware = pp_dpm_load_fw,	1675	.load_firmware = pp_dpm_load_fw,
1656	.wait_for_fw_loading_complete = pp_dpm_fw_loading_complete,	1676	.wait_for_fw_loading_complete = pp_dpm_fw_loading_complete,
@@ -1714,4 +1734,5 @@ static const struct amd_pm_funcs pp_dpm_funcs = { · 57 unmodified lines
1714	.set_xgmi_pstate = pp_set_xgmi_pstate,	1734	.set_xgmi_pstate = pp_set_xgmi_pstate,
1715	.get_gpu_metrics = pp_get_gpu_metrics,	1735	.get_gpu_metrics = pp_get_gpu_metrics,
1716	.gfx_state_change_set = pp_gfx_state_change_set,	1736	.gfx_state_change_set = pp_gfx_state_change_set,
		1737	.get_smu_prv_buf_details = pp_get_prv_buffer_details,
1717	};	1738	};

drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c+4−4

drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c+1−1

@@ -85,7 +85,7 @@ int vega20_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state) · 84 unmodified lines
85	return 0;	85	return 0;
86		86
87	if (state == BACO_STATE_IN) {	87	if (state == BACO_STATE_IN) {
88	if (!ras \|\| !ras->supported) {	88	if (!ras \|\| !adev->ras_enabled) {
89	data = RREG32_SOC15(THM, 0, mmTHM_BACO_CNTL);	89	data = RREG32_SOC15(THM, 0, mmTHM_BACO_CNTL);
90	data \|= 0x80000000;	90	data \|= 0x80000000;
91	WREG32_SOC15(THM, 0, mmTHM_BACO_CNTL, data);	91	WREG32_SOC15(THM, 0, mmTHM_BACO_CNTL, data);

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c+21−0

@@ -2933,6 +2933,26 @@ int smu_set_light_sbr(struct smu_context *smu, bool enable) · 2932 unmodified lines
2933	return ret;	2933	return ret;
2934	}	2934	}
2935		2935
		2936	static int smu_get_prv_buffer_details(void handle, void addr, size_t size)
		2937	{
		2938	struct smu_context *smu = handle;
		2939	struct smu_table_context *smu_table = &smu->smu_table;
		2940	struct smu_table *memory_pool = &smu_table->memory_pool;
		2941
		2942	if (!addr \|\| !size)
		2943	return -EINVAL;
		2944
		2945	*addr = NULL;
		2946	*size = 0;
		2947	mutex_lock(&smu->mutex);
		2948	if (memory_pool->bo) {
		2949	*addr = memory_pool->cpu_addr;
		2950	*size = memory_pool->size;
		2951	}
		2952	mutex_unlock(&smu->mutex);
		2953
		2954	return 0;
		2955	}
2936		2956
2937	static const struct amd_pm_funcs swsmu_pm_funcs = {	2957	static const struct amd_pm_funcs swsmu_pm_funcs = {
2938	/* export for sysfs */	2958	/* export for sysfs */
@@ -2984,6 +3004,7 @@ static const struct amd_pm_funcs swsmu_pm_funcs = { · 45 unmodified lines
2984	.get_max_sustainable_clocks_by_dc = smu_get_max_sustainable_clocks_by_dc,	3004	.get_max_sustainable_clocks_by_dc = smu_get_max_sustainable_clocks_by_dc,
2985	.load_firmware = smu_load_microcode,	3005	.load_firmware = smu_load_microcode,
2986	.gfx_state_change_set = smu_gfx_state_change_set,	3006	.gfx_state_change_set = smu_gfx_state_change_set,
		3007	.get_smu_prv_buf_details = smu_get_prv_buffer_details,
2987	};	3008	};
2988		3009
2989	int smu_wait_for_event(struct amdgpu_device *adev, enum smu_event_type event,	3010	int smu_wait_for_event(struct amdgpu_device *adev, enum smu_event_type event,

drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c+1−1

@@ -302,7 +302,7 @@ sienna_cichlid_get_allowed_feature_mask(struct smu_context *smu, · 301 unmodified lines
302	if (smu->dc_controlled_by_gpio)	302	if (smu->dc_controlled_by_gpio)
303	(uint64_t )feature_mask \|= FEATURE_MASK(FEATURE_ACDC_BIT);	303	(uint64_t )feature_mask \|= FEATURE_MASK(FEATURE_ACDC_BIT);
304		304
305	if (amdgpu_aspm == 1)	305	if (amdgpu_aspm)
306	(uint64_t )feature_mask \|= FEATURE_MASK(FEATURE_DS_LCLK_BIT);	306	(uint64_t )feature_mask \|= FEATURE_MASK(FEATURE_DS_LCLK_BIT);
307		307
308	return 0;	308	return 0;

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c+2−1

 								      NULL);
 			break;
 		default:
-			if (!ras || !ras->supported || adev->gmc.xgmi.pending_reset) {
+			if (!ras || !adev->ras_enabled ||
+			    adev->gmc.xgmi.pending_reset) {
 				if (adev->asic_type == CHIP_ARCTURUS) {
 					data = RREG32_SOC15(THM, 0, mmTHM_BACO_CNTL_ARCT);
 					data |= 0x80000000;

drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c+11−6

 		return -ENOMEM;
 	smu_table->metrics_time = 0;
-	smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v1_1);
+	smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v1_2);
 	smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL);
 	if (!smu_table->gpu_metrics_table) {
 		kfree(smu_table->metrics_table);
 					 void **table)
 {
 	struct smu_table_context *smu_table = &smu->smu_table;
-	struct gpu_metrics_v1_1 *gpu_metrics =
+	struct gpu_metrics_v1_2 *gpu_metrics =
-		(struct gpu_metrics_v1_1 *)smu_table->gpu_metrics_table;
+		(struct gpu_metrics_v1_2 *)smu_table->gpu_metrics_table;
 	SmuMetrics_t metrics;
 	int i, ret = 0;
 	if (ret)
 		return ret;
-	smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 1);
+	smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 2);
 	gpu_metrics->temperature_edge = metrics.TemperatureEdge;
 	gpu_metrics->temperature_hotspot = metrics.TemperatureHotspot;
 	gpu_metrics->average_mm_activity = 0;
 	gpu_metrics->average_socket_power = metrics.AverageSocketPower;
-	gpu_metrics->energy_accumulator = 0;
+	gpu_metrics->energy_accumulator =
+			(uint64_t)metrics.EnergyAcc64bitHigh << 32 |
+			metrics.EnergyAcc64bitLow;
 	gpu_metrics->average_gfxclk_frequency = metrics.AverageGfxclkFrequency;
 	gpu_metrics->average_socclk_frequency = metrics.AverageSocclkFrequency;
 	for (i = 0; i < NUM_HBM_INSTANCES; i++)
 		gpu_metrics->temperature_hbm[i] = metrics.TemperatureAllHBM[i];
+	gpu_metrics->firmware_timestamp = ((uint64_t)metrics.TimeStampHigh << 32) |
+					metrics.TimeStampLow;
 	*table = (void *)gpu_metrics;
-	return sizeof(struct gpu_metrics_v1_1);
+	return sizeof(struct gpu_metrics_v1_2);
 }
 static int aldebaran_mode2_reset(struct smu_context *smu)

drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c+5−2

 	ret = smu_cmn_wait_for_response(smu);
 	if (ret != 0x1) {
-		dev_err(adev->dev, "Msg issuing pre-check failed and "
+		dev_err(adev->dev, "Msg issuing pre-check failed(0x%x) and "
-		       "SMU may be not in the right state!\n");
+		       "SMU may be not in the right state!\n", ret);
 		if (ret != -ETIME)
 			ret = -EIO;
 		return ret;
 	case METRICS_VERSION(1, 1):
 		structure_size = sizeof(struct gpu_metrics_v1_1);
 		break;
+	case METRICS_VERSION(1, 2):
+		structure_size = sizeof(struct gpu_metrics_v1_2);
+		break;
 	case METRICS_VERSION(2, 0):
 		structure_size = sizeof(struct gpu_metrics_v2_0);
 		break;

drivers/gpu/drm/radeon/radeon_device.c+2−1

@@ -1098,7 +1098,8 @@ static bool radeon_check_pot_argument(int arg) · 1097 unmodified lines
1098	}	1098	}
1099		1099
1100	/**	1100	/**
1101	* Determine a sensible default GART size according to ASIC family.	1101	* radeon_gart_size_auto - Determine a sensible default GART size
		1102	* according to ASIC family.
1102	*	1103	*
1103	* @family: ASIC family name	1104	* @family: ASIC family name
1104	*/	1105	*/

include/uapi/drm/amdgpu_drm.h+0−2

 #define AMDGPU_GEM_CREATE_CPU_GTT_USWC		(1 << 2)
 /* Flag that the memory should be in VRAM and cleared */
 #define AMDGPU_GEM_CREATE_VRAM_CLEARED		(1 << 3)
-/* Flag that create shadow bo(GTT) while allocating vram bo */
-#define AMDGPU_GEM_CREATE_SHADOW		(1 << 4)
 /* Flag that allocating the BO should use linear VRAM */
 #define AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS	(1 << 5)
 /* Flag that BO is always valid in this VM */

include/uapi/linux/kfd_ioctl.h+169−2

@@ -30,9 +30,10 @@ · 29 unmodified lines
30	* - 1.1 - initial version	30	* - 1.1 - initial version
31	* - 1.3 - Add SMI events support	31	* - 1.3 - Add SMI events support
32	* - 1.4 - Indicate new SRAM EDC bit in device properties	32	* - 1.4 - Indicate new SRAM EDC bit in device properties
		33	* - 1.5 - Add SVM API
33	*/	34	*/
34	#define KFD_IOCTL_MAJOR_VERSION 1	35	#define KFD_IOCTL_MAJOR_VERSION 1
35	#define KFD_IOCTL_MINOR_VERSION 4	36	#define KFD_IOCTL_MINOR_VERSION 5
36		37
37	struct kfd_ioctl_get_version_args {	38	struct kfd_ioctl_get_version_args {
38	__u32 major_version; /* from KFD */	39	__u32 major_version; /* from KFD */
@@ -473,6 +474,167 @@ enum kfd_mmio_remap { · 434 unmodified lines
473	KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL = 4,	474	KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL = 4,
474	};	475	};
475		476
		477	/* Guarantee host access to memory */
		478	#define KFD_IOCTL_SVM_FLAG_HOST_ACCESS 0x00000001
		479	/* Fine grained coherency between all devices with access */
		480	#define KFD_IOCTL_SVM_FLAG_COHERENT 0x00000002
		481	/* Use any GPU in same hive as preferred device */
		482	#define KFD_IOCTL_SVM_FLAG_HIVE_LOCAL 0x00000004
		483	/* GPUs only read, allows replication */
		484	#define KFD_IOCTL_SVM_FLAG_GPU_RO 0x00000008
		485	/* Allow execution on GPU */
		486	#define KFD_IOCTL_SVM_FLAG_GPU_EXEC 0x00000010
		487	/* GPUs mostly read, may allow similar optimizations as RO, but writes fault */
		488	#define KFD_IOCTL_SVM_FLAG_GPU_READ_MOSTLY 0x00000020
		489
		490	/**
		491	* kfd_ioctl_svm_op - SVM ioctl operations
		492	*
		493	* @KFD_IOCTL_SVM_OP_SET_ATTR: Modify one or more attributes
		494	* @KFD_IOCTL_SVM_OP_GET_ATTR: Query one or more attributes
		495	*/
		496	enum kfd_ioctl_svm_op {
		497	KFD_IOCTL_SVM_OP_SET_ATTR,
		498	KFD_IOCTL_SVM_OP_GET_ATTR
		499	};
		500
		501	/** kfd_ioctl_svm_location - Enum for preferred and prefetch locations
		502	*
		503	* GPU IDs are used to specify GPUs as preferred and prefetch locations.
		504	* Below definitions are used for system memory or for leaving the preferred
		505	* location unspecified.
		506	*/
		507	enum kfd_ioctl_svm_location {
		508	KFD_IOCTL_SVM_LOCATION_SYSMEM = 0,
		509	KFD_IOCTL_SVM_LOCATION_UNDEFINED = 0xffffffff
		510	};
		511
		512	/**
		513	* kfd_ioctl_svm_attr_type - SVM attribute types
		514	*
		515	* @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: gpuid of the preferred location, 0 for
		516	* system memory
		517	* @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: gpuid of the prefetch location, 0 for
		518	* system memory. Setting this triggers an
		519	* immediate prefetch (migration).
		520	* @KFD_IOCTL_SVM_ATTR_ACCESS:
		521	* @KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:
		522	* @KFD_IOCTL_SVM_ATTR_NO_ACCESS: specify memory access for the gpuid given
		523	* by the attribute value
		524	* @KFD_IOCTL_SVM_ATTR_SET_FLAGS: bitmask of flags to set (see
		525	* KFD_IOCTL_SVM_FLAG_...)
		526	* @KFD_IOCTL_SVM_ATTR_CLR_FLAGS: bitmask of flags to clear
		527	* @KFD_IOCTL_SVM_ATTR_GRANULARITY: migration granularity
		528	* (log2 num pages)
		529	*/
		530	enum kfd_ioctl_svm_attr_type {
		531	KFD_IOCTL_SVM_ATTR_PREFERRED_LOC,
		532	KFD_IOCTL_SVM_ATTR_PREFETCH_LOC,
		533	KFD_IOCTL_SVM_ATTR_ACCESS,
		534	KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE,
		535	KFD_IOCTL_SVM_ATTR_NO_ACCESS,
		536	KFD_IOCTL_SVM_ATTR_SET_FLAGS,
		537	KFD_IOCTL_SVM_ATTR_CLR_FLAGS,
		538	KFD_IOCTL_SVM_ATTR_GRANULARITY
		539	};
		540
		541	/**
		542	* kfd_ioctl_svm_attribute - Attributes as pairs of type and value
		543	*
		544	* The meaning of the @value depends on the attribute type.
		545	*
		546	* @type: attribute type (see enum @kfd_ioctl_svm_attr_type)
		547	* @value: attribute value
		548	*/
		549	struct kfd_ioctl_svm_attribute {
		550	__u32 type;
		551	__u32 value;
		552	};
		553
		554	/**
		555	* kfd_ioctl_svm_args - Arguments for SVM ioctl
		556	*
		557	* @op specifies the operation to perform (see enum
		558	* @kfd_ioctl_svm_op). @start_addr and @size are common for all
		559	* operations.
		560	*
		561	* A variable number of attributes can be given in @attrs.
		562	* @nattr specifies the number of attributes. New attributes can be
		563	* added in the future without breaking the ABI. If unknown attributes
		564	* are given, the function returns -EINVAL.
		565	*
		566	* @KFD_IOCTL_SVM_OP_SET_ATTR sets attributes for a virtual address
		567	* range. It may overlap existing virtual address ranges. If it does,
		568	* the existing ranges will be split such that the attribute changes
		569	* only apply to the specified address range.
		570	*
		571	* @KFD_IOCTL_SVM_OP_GET_ATTR returns the intersection of attributes
		572	* over all memory in the given range and returns the result as the
		573	* attribute value. If different pages have different preferred or
		574	* prefetch locations, 0xffffffff will be returned for
		575	* @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC or
		576	* @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC resepctively. For
		577	* @KFD_IOCTL_SVM_ATTR_SET_FLAGS, flags of all pages will be
		578	* aggregated by bitwise AND. The minimum migration granularity
		579	* throughout the range will be returned for
		580	* @KFD_IOCTL_SVM_ATTR_GRANULARITY.
		581	*
		582	* Querying of accessibility attributes works by initializing the
		583	* attribute type to @KFD_IOCTL_SVM_ATTR_ACCESS and the value to the
		584	* GPUID being queried. Multiple attributes can be given to allow
		585	* querying multiple GPUIDs. The ioctl function overwrites the
		586	* attribute type to indicate the access for the specified GPU.
		587	*
		588	* @KFD_IOCTL_SVM_ATTR_CLR_FLAGS is invalid for
		589	* @KFD_IOCTL_SVM_OP_GET_ATTR.
		590	*/
		591	struct kfd_ioctl_svm_args {
		592	__u64 start_addr;
		593	__u64 size;
		594	__u32 op;
		595	__u32 nattr;
		596	/* Variable length array of attributes */
		597	struct kfd_ioctl_svm_attribute attrs[0];
		598	};
		599
		600	/**
		601	* kfd_ioctl_set_xnack_mode_args - Arguments for set_xnack_mode
		602	*
		603	* @xnack_enabled: [in/out] Whether to enable XNACK mode for this process
		604	*
		605	* @xnack_enabled indicates whether recoverable page faults should be
		606	* enabled for the current process. 0 means disabled, positive means
		607	* enabled, negative means leave unchanged. If enabled, virtual address
		608	* translations on GFXv9 and later AMD GPUs can return XNACK and retry
		609	* the access until a valid PTE is available. This is used to implement
		610	* device page faults.
		611	*
		612	* On output, @xnack_enabled returns the (new) current mode (0 or
		613	* positive). Therefore, a negative input value can be used to query
		614	* the current mode without changing it.
		615	*
		616	* The XNACK mode fundamentally changes the way SVM managed memory works
		617	* in the driver, with subtle effects on application performance and
		618	* functionality.
		619	*
		620	* Enabling XNACK mode requires shader programs to be compiled
		621	* differently. Furthermore, not all GPUs support changing the mode
		622	* per-process. Therefore changing the mode is only allowed while no
		623	* user mode queues exist in the process. This ensure that no shader
		624	* code is running that may be compiled for the wrong mode. And GPUs
		625	* that cannot change to the requested mode will prevent the XNACK
		626	* mode from occurring. All GPUs used by the process must be in the
		627	* same XNACK mode.
		628	*
		629	* GFXv8 or older GPUs do not support 48 bit virtual addresses or SVM.
		630	* Therefore those GPUs are not considered for the XNACK mode switch.
		631	*
		632	* Return: 0 on success, -errno on failure
		633	*/
		634	struct kfd_ioctl_set_xnack_mode_args {
		635	__s32 xnack_enabled;
		636	};
		637
476	#define AMDKFD_IOCTL_BASE 'K'	638	#define AMDKFD_IOCTL_BASE 'K'
477	#define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr)	639	#define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr)
478	#define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type)	640	#define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type)
@@ -573,7 +735,12 @@ enum kfd_mmio_remap { · 94 unmodified lines
573	#define AMDKFD_IOC_SMI_EVENTS \	735	#define AMDKFD_IOC_SMI_EVENTS \
574	AMDKFD_IOWR(0x1F, struct kfd_ioctl_smi_events_args)	736	AMDKFD_IOWR(0x1F, struct kfd_ioctl_smi_events_args)
575		737
		738	#define AMDKFD_IOC_SVM AMDKFD_IOWR(0x20, struct kfd_ioctl_svm_args)
		739
		740	#define AMDKFD_IOC_SET_XNACK_MODE \
		741	AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args)
		742
576	#define AMDKFD_COMMAND_START 0x01	743	#define AMDKFD_COMMAND_START 0x01
577	#define AMDKFD_COMMAND_END 0x20	744	#define AMDKFD_COMMAND_END 0x22
578		745
579	#endif	746	#endif

@@ -719,7 +719,7 @@ static void unlock_spi_csq_mutexes(struct amdgpu_device *adev) · 718 unmodified lines
719	}	719	}
720		720
721	/**	721	/**
722	* @get_wave_count: Read device registers to get number of waves in flight for	722	* get_wave_count: Read device registers to get number of waves in flight for
723	* a particular queue. The method also returns the VMID associated with the	723	* a particular queue. The method also returns the VMID associated with the
724	* queue.	724	* queue.
725	*	725	*
@@ -755,19 +755,19 @@ static void get_wave_count(struct amdgpu_device *adev, int queue_idx, · 29 unmodified lines
755	}	755	}
756		756
757	/**	757	/**
758	* @kgd_gfx_v9_get_cu_occupancy: Reads relevant registers associated with each	758	* kgd_gfx_v9_get_cu_occupancy: Reads relevant registers associated with each
759	* shader engine and aggregates the number of waves that are in flight for the	759	* shader engine and aggregates the number of waves that are in flight for the
760	* process whose pasid is provided as a parameter. The process could have ZERO	760	* process whose pasid is provided as a parameter. The process could have ZERO
761	* or more queues running and submitting waves to compute units.	761	* or more queues running and submitting waves to compute units.
762	*	762	*
763	* @kgd: Handle of device from which to get number of waves in flight	763	* @kgd: Handle of device from which to get number of waves in flight
764	* @pasid: Identifies the process for which this query call is invoked	764	* @pasid: Identifies the process for which this query call is invoked
765	* @wave_cnt: Output parameter updated with number of waves in flight that	765	* @pasid_wave_cnt: Output parameter updated with number of waves in flight that
766	* belong to process with given pasid	766	* belong to process with given pasid
767	* @max_waves_per_cu: Output parameter updated with maximum number of waves	767	* @max_waves_per_cu: Output parameter updated with maximum number of waves
768	* possible per Compute Unit	768	* possible per Compute Unit
769	*	769	*
770	* @note: It's possible that the device has too many queues (oversubscription)	770	* Note: It's possible that the device has too many queues (oversubscription)
771	* in which case a VMID could be remapped to a different PASID. This could lead	771	* in which case a VMID could be remapped to a different PASID. This could lead
772	* to an iaccurate wave count. Following is a high-level sequence:	772	* to an iaccurate wave count. Following is a high-level sequence:
773	* Time T1: vmid = getVmid(); vmid is associated with Pasid P1	773	* Time T1: vmid = getVmid(); vmid is associated with Pasid P1

@@ -2895,7 +2895,7 @@ static void dml20_DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 2894 unmodified lines
2895	RoundedUpMaxSwathSizeBytesC = 0.0;	2895	RoundedUpMaxSwathSizeBytesC = 0.0;
2896		2896
2897	if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC	2897	if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC
2898	<= mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0) {	2898	<= mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0) {
2899	mode_lib->vba.SwathHeightY[k] = MaximumSwathHeightY;	2899	mode_lib->vba.SwathHeightY[k] = MaximumSwathHeightY;
2900	mode_lib->vba.SwathHeightC[k] = MaximumSwathHeightC;	2900	mode_lib->vba.SwathHeightC[k] = MaximumSwathHeightC;
2901	} else {	2901	} else {
@@ -2904,17 +2904,17 @@ static void dml20_DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 2 unmodified lines
2904	}	2904	}
2905		2905
2906	if (mode_lib->vba.SwathHeightC[k] == 0) {	2906	if (mode_lib->vba.SwathHeightC[k] == 0) {
2907	mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte * 1024;	2907	mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0] * 1024;
2908	mode_lib->vba.DETBufferSizeC[k] = 0;	2908	mode_lib->vba.DETBufferSizeC[k] = 0;
2909	} else if (mode_lib->vba.SwathHeightY[k] <= mode_lib->vba.SwathHeightC[k]) {	2909	} else if (mode_lib->vba.SwathHeightY[k] <= mode_lib->vba.SwathHeightC[k]) {
2910	mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte	2910	mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2911	* 1024.0 / 2;	2911	* 1024.0 / 2;
2912	mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte	2912	mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2913	* 1024.0 / 2;	2913	* 1024.0 / 2;
2914	} else {	2914	} else {
2915	mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte	2915	mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2916	* 1024.0 * 2 / 3;	2916	* 1024.0 * 2 / 3;
2917	mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte	2917	mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2918	* 1024.0 / 3;	2918	* 1024.0 / 3;
2919	}	2919	}
2920	}	2920	}
@@ -3819,7 +3819,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 898 unmodified lines
3819	mode_lib->vba.MaximumSwathWidthInDETBuffer =	3819	mode_lib->vba.MaximumSwathWidthInDETBuffer =
3820	dml_min(	3820	dml_min(
3821	mode_lib->vba.MaximumSwathWidthSupport,	3821	mode_lib->vba.MaximumSwathWidthSupport,
3822	mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0	3822	mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0
3823	/ (locals->BytePerPixelInDETY[k]	3823	/ (locals->BytePerPixelInDETY[k]
3824	* locals->MinSwathHeightY[k]	3824	* locals->MinSwathHeightY[k]
3825	+ locals->BytePerPixelInDETC[k]	3825	+ locals->BytePerPixelInDETC[k]
@@ -4322,7 +4322,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 496 unmodified lines
4322	locals->RoundedUpMaxSwathSizeBytesC = 0;	4322	locals->RoundedUpMaxSwathSizeBytesC = 0;
4323	}	4323	}
4324		4324
4325	if (locals->RoundedUpMaxSwathSizeBytesY + locals->RoundedUpMaxSwathSizeBytesC <= locals->DETBufferSizeInKByte * 1024 / 2) {	4325	if (locals->RoundedUpMaxSwathSizeBytesY + locals->RoundedUpMaxSwathSizeBytesC <= locals->DETBufferSizeInKByte[0] * 1024 / 2) {
4326	locals->SwathHeightYPerState[i][j][k] = locals->MaxSwathHeightY[k];	4326	locals->SwathHeightYPerState[i][j][k] = locals->MaxSwathHeightY[k];
4327	locals->SwathHeightCPerState[i][j][k] = locals->MaxSwathHeightC[k];	4327	locals->SwathHeightCPerState[i][j][k] = locals->MaxSwathHeightC[k];
4328	} else {	4328	} else {
@@ -4331,15 +4331,15 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 2 unmodified lines
4331	}	4331	}
4332		4332
4333	if (locals->BytePerPixelInDETC[k] == 0) {	4333	if (locals->BytePerPixelInDETC[k] == 0) {
4334	locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];	4334	locals->LinesInDETLuma = locals->DETBufferSizeInKByte[0] * 1024 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
4335	locals->LinesInDETChroma = 0;	4335	locals->LinesInDETChroma = 0;
4336	} else if (locals->SwathHeightYPerState[i][j][k] <= locals->SwathHeightCPerState[i][j][k]) {	4336	} else if (locals->SwathHeightYPerState[i][j][k] <= locals->SwathHeightCPerState[i][j][k]) {
4337	locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 / 2 / locals->BytePerPixelInDETY[k] /	4337	locals->LinesInDETLuma = locals->DETBufferSizeInKByte[0] * 1024 / 2 / locals->BytePerPixelInDETY[k] /
4338	locals->SwathWidthYPerState[i][j][k];	4338	locals->SwathWidthYPerState[i][j][k];
4339	locals->LinesInDETChroma = locals->DETBufferSizeInKByte * 1024 / 2 / locals->BytePerPixelInDETC[k] / (locals->SwathWidthYPerState[i][j][k] / 2);	4339	locals->LinesInDETChroma = locals->DETBufferSizeInKByte[0] * 1024 / 2 / locals->BytePerPixelInDETC[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
4340	} else {	4340	} else {
4341	locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 * 2 / 3 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];	4341	locals->LinesInDETLuma = locals->DETBufferSizeInKByte[0] * 1024 * 2 / 3 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
4342	locals->LinesInDETChroma = locals->DETBufferSizeInKByte * 1024 / 3 / locals->BytePerPixelInDETY[k] / (locals->SwathWidthYPerState[i][j][k] / 2);	4342	locals->LinesInDETChroma = locals->DETBufferSizeInKByte[0] * 1024 / 3 / locals->BytePerPixelInDETY[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
4343	}	4343	}
4344		4344
4345	locals->EffectiveLBLatencyHidingSourceLinesLuma = dml_min(locals->MaxLineBufferLines,	4345	locals->EffectiveLBLatencyHidingSourceLinesLuma = dml_min(locals->MaxLineBufferLines,

@@ -2968,7 +2968,7 @@ static void dml20v2_DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 2967 unmodified lines
2968	RoundedUpMaxSwathSizeBytesC = 0.0;	2968	RoundedUpMaxSwathSizeBytesC = 0.0;
2969		2969
2970	if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC	2970	if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC
2971	<= mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0) {	2971	<= mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0) {
2972	mode_lib->vba.SwathHeightY[k] = MaximumSwathHeightY;	2972	mode_lib->vba.SwathHeightY[k] = MaximumSwathHeightY;
2973	mode_lib->vba.SwathHeightC[k] = MaximumSwathHeightC;	2973	mode_lib->vba.SwathHeightC[k] = MaximumSwathHeightC;
2974	} else {	2974	} else {
@@ -2977,17 +2977,17 @@ static void dml20v2_DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 2 unmodified lines
2977	}	2977	}
2978		2978
2979	if (mode_lib->vba.SwathHeightC[k] == 0) {	2979	if (mode_lib->vba.SwathHeightC[k] == 0) {
2980	mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte * 1024;	2980	mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0] * 1024;
2981	mode_lib->vba.DETBufferSizeC[k] = 0;	2981	mode_lib->vba.DETBufferSizeC[k] = 0;
2982	} else if (mode_lib->vba.SwathHeightY[k] <= mode_lib->vba.SwathHeightC[k]) {	2982	} else if (mode_lib->vba.SwathHeightY[k] <= mode_lib->vba.SwathHeightC[k]) {
2983	mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte	2983	mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2984	* 1024.0 / 2;	2984	* 1024.0 / 2;
2985	mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte	2985	mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2986	* 1024.0 / 2;	2986	* 1024.0 / 2;
2987	} else {	2987	} else {
2988	mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte	2988	mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2989	* 1024.0 * 2 / 3;	2989	* 1024.0 * 2 / 3;
2990	mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte	2990	mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2991	* 1024.0 / 3;	2991	* 1024.0 / 3;
2992	}	2992	}
2993	}	2993	}
@@ -3926,7 +3926,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode · 932 unmodified lines
3926	mode_lib->vba.MaximumSwathWidthInDETBuffer =	3926	mode_lib->vba.MaximumSwathWidthInDETBuffer =
3927	dml_min(	3927	dml_min(
3928	mode_lib->vba.MaximumSwathWidthSupport,	3928	mode_lib->vba.MaximumSwathWidthSupport,
3929	mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0	3929	mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0
3930	/ (locals->BytePerPixelInDETY[k]	3930	/ (locals->BytePerPixelInDETY[k]
3931	* locals->MinSwathHeightY[k]	3931	* locals->MinSwathHeightY[k]
3932	+ locals->BytePerPixelInDETC[k]	3932	+ locals->BytePerPixelInDETC[k]
@@ -4443,7 +4443,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode · 510 unmodified lines
4443	locals->RoundedUpMaxSwathSizeBytesC = 0;	4443	locals->RoundedUpMaxSwathSizeBytesC = 0;
4444	}	4444	}
4445		4445
4446	if (locals->RoundedUpMaxSwathSizeBytesY + locals->RoundedUpMaxSwathSizeBytesC <= locals->DETBufferSizeInKByte * 1024 / 2) {	4446	if (locals->RoundedUpMaxSwathSizeBytesY + locals->RoundedUpMaxSwathSizeBytesC <= locals->DETBufferSizeInKByte[0] * 1024 / 2) {
4447	locals->SwathHeightYPerState[i][j][k] = locals->MaxSwathHeightY[k];	4447	locals->SwathHeightYPerState[i][j][k] = locals->MaxSwathHeightY[k];
4448	locals->SwathHeightCPerState[i][j][k] = locals->MaxSwathHeightC[k];	4448	locals->SwathHeightCPerState[i][j][k] = locals->MaxSwathHeightC[k];
4449	} else {	4449	} else {
@@ -4452,15 +4452,15 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode · 2 unmodified lines
4452	}	4452	}
4453		4453
4454	if (locals->BytePerPixelInDETC[k] == 0) {	4454	if (locals->BytePerPixelInDETC[k] == 0) {
4455	locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];	4455	locals->LinesInDETLuma = locals->DETBufferSizeInKByte[0] * 1024 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
4456	locals->LinesInDETChroma = 0;	4456	locals->LinesInDETChroma = 0;
4457	} else if (locals->SwathHeightYPerState[i][j][k] <= locals->SwathHeightCPerState[i][j][k]) {	4457	} else if (locals->SwathHeightYPerState[i][j][k] <= locals->SwathHeightCPerState[i][j][k]) {
4458	locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 / 2 / locals->BytePerPixelInDETY[k] /	4458	locals->LinesInDETLuma = locals->DETBufferSizeInKByte[0] * 1024 / 2 / locals->BytePerPixelInDETY[k] /
4459	locals->SwathWidthYPerState[i][j][k];	4459	locals->SwathWidthYPerState[i][j][k];
4460	locals->LinesInDETChroma = locals->DETBufferSizeInKByte * 1024 / 2 / locals->BytePerPixelInDETC[k] / (locals->SwathWidthYPerState[i][j][k] / 2);	4460	locals->LinesInDETChroma = locals->DETBufferSizeInKByte[0] * 1024 / 2 / locals->BytePerPixelInDETC[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
4461	} else {	4461	} else {
4462	locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 * 2 / 3 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];	4462	locals->LinesInDETLuma = locals->DETBufferSizeInKByte[0] * 1024 * 2 / 3 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
4463	locals->LinesInDETChroma = locals->DETBufferSizeInKByte * 1024 / 3 / locals->BytePerPixelInDETY[k] / (locals->SwathWidthYPerState[i][j][k] / 2);	4463	locals->LinesInDETChroma = locals->DETBufferSizeInKByte[0] * 1024 / 3 / locals->BytePerPixelInDETY[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
4464	}	4464	}
4465		4465
4466	locals->EffectiveLBLatencyHidingSourceLinesLuma = dml_min(locals->MaxLineBufferLines,	4466	locals->EffectiveLBLatencyHidingSourceLinesLuma = dml_min(locals->MaxLineBufferLines,

@@ -275,7 +275,7 @@ static const ATOM_VOLTAGE_OBJECT_V3 *atomctrl_lookup_voltage_type_v3( · 274 unmodified lines
275	}	275	}
276		276
277	/**	277	/**
278	* atomctrl_get_memory_pll_dividers_si().	278	* atomctrl_get_memory_pll_dividers_si
279	*	279	*
280	* @hwmgr: input parameter: pointer to HwMgr	280	* @hwmgr: input parameter: pointer to HwMgr
281	* @clock_value: input parameter: memory clock	281	* @clock_value: input parameter: memory clock
@@ -328,7 +328,7 @@ int atomctrl_get_memory_pll_dividers_si( · 46 unmodified lines
328	}	328	}
329		329
330	/**	330	/**
331	* atomctrl_get_memory_pll_dividers_vi().	331	* atomctrl_get_memory_pll_dividers_vi
332	*	332	*
333	* @hwmgr: input parameter: pointer to HwMgr	333	* @hwmgr: input parameter: pointer to HwMgr
334	* @clock_value: input parameter: memory clock	334	* @clock_value: input parameter: memory clock
@@ -1104,7 +1104,7 @@ int atomctrl_calculate_voltage_evv_on_sclk( · 769 unmodified lines
1104	}	1104	}
1105		1105
1106	/**	1106	/**
1107	* atomctrl_get_voltage_evv_on_sclk gets voltage via call to ATOM COMMAND table.	1107	* atomctrl_get_voltage_evv_on_sclk: gets voltage via call to ATOM COMMAND table.
1108	* @hwmgr: input: pointer to hwManager	1108	* @hwmgr: input: pointer to hwManager
1109	* @voltage_type: input: type of EVV voltage VDDC or VDDGFX	1109	* @voltage_type: input: type of EVV voltage VDDC or VDDGFX
1110	* @sclk: input: in 10Khz unit. DPM state SCLK frequency	1110	* @sclk: input: in 10Khz unit. DPM state SCLK frequency
@@ -1144,7 +1144,7 @@ int atomctrl_get_voltage_evv_on_sclk( · 33 unmodified lines
1144	}	1144	}
1145		1145
1146	/**	1146	/**
1147	* atomctrl_get_voltage_evv gets voltage via call to ATOM COMMAND table.	1147	* atomctrl_get_voltage_evv: gets voltage via call to ATOM COMMAND table.
1148	* @hwmgr: input: pointer to hwManager	1148	* @hwmgr: input: pointer to hwManager
1149	* @virtual_voltage_id: input: voltage id which match per voltage DPM state: 0xff01, 0xff02.. 0xff08	1149	* @virtual_voltage_id: input: voltage id which match per voltage DPM state: 0xff01, 0xff02.. 0xff08
1150	* @voltage: output: real voltage level in unit of mv	1150	* @voltage: output: real voltage level in unit of mv