Merge tag 'amd-drm-next-5.14-2021-05-19' of https://gitlab.freedesktop.org/agd5f/linux into drm-next

amd-drm-next-5.14-2021-05-19: amdgpu: - Aldebaran updates - More LTTPR display work - Vangogh updates - SDMA 5.x GCR fixes - RAS fixes - PCIe ASPM support - Modifier fixes - Enable TMZ on Renoir - Buffer object code cleanup - Display overlay fixes - Initial support for multiple eDP panels - Initial SR-IOV support for Aldebaran - DP link training refactor - Misc code cleanups and bug fixes - SMU regression fixes for variable sized arrays - MAINTAINERS fixes for amdgpu amdkfd: - Initial SR-IOV support for Aldebaran - Topology fixes - Initial HMM SVM support - Misc code cleanups and bug fixes radeon: - Misc code cleanups and bug fixes - SMU regression fixes for variable sized arrays - Flickering fix for Oland with multiple 4K displays UAPI: - amdgpu: Drop AMDGPU_GEM_CREATE_SHADOW flag. This was always a kernel internal flag and userspace use of it has always been blocked. It's no longer needed so remove it. - amdkgd: HMM SVM support Overview: https://patchwork.freedesktop.org/series/85562/ Porposed userspace: https://github.com/RadeonOpenCompute/ROCT-Thunk-Interface/tree/fxkamd/hmm-wip Signed-off-by: Dave Airlie <airlied@redhat.com>

Dave Airlie committed 5 years ago

Commit c99c4d02 parents 2ba0478 2bb5b5f159 files · +8897 −1576Browse files

Unified · Split

159 files

MAINTAINERS +2 −2
drivers/gpu/drm/amd/amdgpu/Makefile +1 −1
drivers/gpu/drm/amd/amdgpu/amdgpu.h +2 −1
drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c +6 −6
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +17 −9
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c +1 −0
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c +14 −6
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +4 −4
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +62 −36
drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +3 −3
drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +10 −11
drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +9 −9
drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c +1 −0
drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c +2 −2
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +56 −5
drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h +4 −2
drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c +90 −79
drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.c +69 −0
drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h +12 −1
drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c +1 −1
drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c +3 −1
drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +3 −2
drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h +1 −0
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c +86 −0
drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h +7 −0
drivers/gpu/drm/amd/amdgpu/amdgpu_object.c +18 −54
drivers/gpu/drm/amd/amdgpu/amdgpu_object.h +7 −0
drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +36 −37
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c +114 −85
drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h +1 −4
drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c +1 −0
drivers/gpu/drm/amd/amdgpu/amdgpu_smuio.h +1 −0
drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c +25 −74
drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c +7 −3
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +72 −52
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h +12 −5
drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c +120 −112
drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h +3 −3
drivers/gpu/drm/amd/amdgpu/dce_virtual.c +5 −7
drivers/gpu/drm/amd/amdgpu/df_v3_6.c +11 −6
drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c +8 −4
drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c +641 −3
drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h +1 −0
drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c +6 −2
drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c +12 −1
drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c +28 −3
drivers/gpu/drm/amd/amdgpu/hdp_v4_0.h +1 −0
drivers/gpu/drm/amd/amdgpu/mmhub_v1_7.c +23 −2
drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c +2 −2
drivers/gpu/drm/amd/amdgpu/nv.c +6 −1
drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h +0 −1
drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c +14 −21
drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c +28 −12
drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c +8 −0
drivers/gpu/drm/amd/amdgpu/smuio_v13_0.c +18 −0
drivers/gpu/drm/amd/amdgpu/soc15.c +5 −7
drivers/gpu/drm/amd/amdgpu/ta_ras_if.h +7 −0
drivers/gpu/drm/amd/amdgpu/vega10_ih.c +1 −0
drivers/gpu/drm/amd/amdgpu/vi.c +1 −1
drivers/gpu/drm/amd/amdkfd/Kconfig +13 −0
drivers/gpu/drm/amd/amdkfd/Makefile +5 −0
drivers/gpu/drm/amd/amdkfd/kfd_chardev.c +73 −7
drivers/gpu/drm/amd/amdkfd/kfd_crat.c +811 −70
drivers/gpu/drm/amd/amdkfd/kfd_device.c +15 −4
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +3 −3
drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c +11 −2
drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c +4 −0
drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c +0 −1
drivers/gpu/drm/amd/amdkfd/kfd_migrate.c +937 −0
drivers/gpu/drm/amd/amdkfd/kfd_migrate.h +65 −0
drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c +7 −5
drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c +57 −1
drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_aldebaran.h +93 −0
drivers/gpu/drm/amd/amdkfd/kfd_priv.h +38 −1
drivers/gpu/drm/amd/amdkfd/kfd_process.c +108 −16
drivers/gpu/drm/amd/amdkfd/kfd_svm.c +3085 −0
drivers/gpu/drm/amd/amdkfd/kfd_svm.h +206 −0
drivers/gpu/drm/amd/amdkfd/kfd_topology.c +54 −28
drivers/gpu/drm/amd/amdkfd/kfd_topology.h +6 −4
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +188 −71
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h +21 −1
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c +66 −40
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +14 −5
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c +26 −1
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h +1 −0
drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c +3 −0
drivers/gpu/drm/amd/display/dc/Makefile +1 −1
drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c +2 −0
drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c +59 −85
drivers/gpu/drm/amd/display/dc/core/dc.c +1 −15
drivers/gpu/drm/amd/display/dc/core/dc_link.c +71 −31
drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c +8 −1
drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c +157 −74
drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c +17 −5
drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c +2 −1
drivers/gpu/drm/amd/display/dc/core/dc_resource.c +1 −6
drivers/gpu/drm/amd/display/dc/dc.h +1 −5
drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c +1 −1
drivers/gpu/drm/amd/display/dc/dc_dp_types.h +1 −0
drivers/gpu/drm/amd/display/dc/dc_dsc.h +3 −2
drivers/gpu/drm/amd/display/dc/dc_hw_types.h +1 −0
drivers/gpu/drm/amd/display/dc/dc_link.h +1 −9
drivers/gpu/drm/amd/display/dc/dc_types.h +2 −1
drivers/gpu/drm/amd/display/dc/dce/dce_aux.c +19 −0
drivers/gpu/drm/amd/display/dc/dce/dce_aux.h +3 −0
drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c +30 −0
drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c +6 −3
drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c +16 −139
drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h +0 −5
drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c +3 −1
drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp_cm.c +2 −2
drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c +5 −1
drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c +1 −1
drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c +1 −1
drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c +21 −12
drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_stream_encoder.h +4 −0
drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c +1 −1
drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c +1 −1
drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c +1 −1
drivers/gpu/drm/amd/display/dc/dm_helpers.h +8 −1
drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c +13 −13
drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c +13 −13
drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c +29 −29
drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c +24 −24
drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c +1 −1
drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h +7 −7
drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c +103 −25
drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c +0 −43
drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h +0 −2
drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h +11 −1
drivers/gpu/drm/amd/display/dc/inc/hw/dsc.h +1 −0
drivers/gpu/drm/amd/display/dc/inc/link_enc_cfg.h +6 −1
drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c +9 −9
drivers/gpu/drm/amd/display/dc/irq_types.h +1 −1
drivers/gpu/drm/amd/display/dmub/dmub_srv.h +19 −18
drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h +194 −13
drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c +1 −1
drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h +1 −1
drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c +4 −0
drivers/gpu/drm/amd/display/include/link_service_types.h +8 −0
drivers/gpu/drm/amd/display/modules/hdcp/hdcp.c +0 −1
drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_execution.c +5 −0
drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_execution.c +5 −0
drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_sh_mask.h +1 −0
drivers/gpu/drm/amd/include/kgd_pp_interface.h +63 −0
drivers/gpu/drm/amd/pm/amdgpu_pm.c +65 −1
drivers/gpu/drm/amd/pm/inc/smu13_driver_if_aldebaran.h +5 −2
drivers/gpu/drm/amd/pm/inc/smu_v13_0.h +1 −1
drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c +21 −0
drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c +4 −4
drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c +1 −1
drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +21 −0
drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +1 −1
drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c +2 −1
drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c +11 −6
drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +5 −2
drivers/gpu/drm/radeon/radeon_device.c +2 −1
include/uapi/drm/amdgpu_drm.h +0 −2
include/uapi/linux/kfd_ioctl.h +169 −2

MAINTAINERS+2−2

@@ -878,7 +878,7 @@ M: Harry Wentland <harry.wentland@amd.com> · 877 unmodified lines
878	878	M: Leo Li <sunpeng.li@amd.com>
879	879	L: amd-gfx@lists.freedesktop.org
880	880	S: Supported
881		−T: git git://people.freedesktop.org/~agd5f/linux
	881	+T: git https://gitlab.freedesktop.org/agd5f/linux.git
882	882	F: drivers/gpu/drm/amd/display/
883	883
884	884	AMD FAM15H PROCESSOR POWER MONITORING DRIVER
@@ -954,7 +954,7 @@ AMD POWERPLAY · 69 unmodified lines
954	954	M: Evan Quan <evan.quan@amd.com>
955	955	L: amd-gfx@lists.freedesktop.org
956	956	S: Supported
957		−T: git git://people.freedesktop.org/~agd5f/linux
	957	+T: git https://gitlab.freedesktop.org/agd5f/linux.git
958	958	F: drivers/gpu/drm/amd/pm/powerplay/
959	959
960	960	AMD SEATTLE DEVICE TREE SUPPORT

drivers/gpu/drm/amd/amdgpu/Makefile+1−1

@@ -56,7 +56,7 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \ · 55 unmodified lines
56	56	amdgpu_gmc.o amdgpu_mmhub.o amdgpu_xgmi.o amdgpu_csa.o amdgpu_ras.o amdgpu_vm_cpu.o \
57	57	amdgpu_vm_sdma.o amdgpu_discovery.o amdgpu_ras_eeprom.o amdgpu_nbio.o \
58	58	amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o \
59		− amdgpu_fw_attestation.o amdgpu_securedisplay.o
	59	+ amdgpu_fw_attestation.o amdgpu_securedisplay.o amdgpu_hdp.o
60	60
61	61	amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o
62	62

drivers/gpu/drm/amd/amdgpu/amdgpu.h+2−1

@@ -1075,7 +1075,8 @@ struct amdgpu_device { · 1074 unmodified lines
1075	1075
1076	1076	atomic_t throttling_logging_enabled;
1077	1077	struct ratelimit_state throttling_logging_rs;
1078		− uint32_t ras_features;
	1078	+ uint32_t ras_hw_enabled;
	1079	+ uint32_t ras_enabled;
1079	1080
1080	1081	bool in_pci_err_recovery;
1081	1082	struct pci_saved_state *pci_state;

drivers/gpu/drm/amd/amdgpu/amdgpu_acpi.c+6−6

@@ -76,7 +76,7 @@ struct amdgpu_atif { · 75 unmodified lines
76	76	/**
77	77	* amdgpu_atif_call - call an ATIF method
78	78	*
79		− * @handle: acpi handle
	79	+ * @atif: acpi handle
80	80	* @function: the ATIF function to execute
81	81	* @params: ATIF function params
82	82	*
@@ -166,7 +166,6 @@ static void amdgpu_atif_parse_functions(struct amdgpu_atif_functions *f, u32 mas · 83 unmodified lines
166	166	/**
167	167	* amdgpu_atif_verify_interface - verify ATIF
168	168	*
169		− * @handle: acpi handle
170	169	* @atif: amdgpu atif struct
171	170	*
172	171	* Execute the ATIF_FUNCTION_VERIFY_INTERFACE ATIF function
@@ -240,8 +239,7 @@ out: · 67 unmodified lines
240	239	/**
241	240	* amdgpu_atif_get_notification_params - determine notify configuration
242	241	*
243		− * @handle: acpi handle
244		− * @n: atif notification configuration struct
	242	+ * @atif: acpi handle
245	243	*
246	244	* Execute the ATIF_FUNCTION_GET_SYSTEM_PARAMETERS ATIF function
247	245	* to determine if a notifier is used and if so which one
@@ -304,7 +302,7 @@ out: · 56 unmodified lines
304	302	/**
305	303	* amdgpu_atif_query_backlight_caps - get min and max backlight input signal
306	304	*
307		− * @handle: acpi handle
	305	+ * @atif: acpi handle
308	306	*
309	307	* Execute the QUERY_BRIGHTNESS_TRANSFER_CHARACTERISTICS ATIF function
310	308	* to determine the acceptable range of backlight values
@@ -363,7 +361,7 @@ out: · 52 unmodified lines
363	361	/**
364	362	* amdgpu_atif_get_sbios_requests - get requested sbios event
365	363	*
366		− * @handle: acpi handle
	364	+ * @atif: acpi handle
367	365	* @req: atif sbios request struct
368	366	*
369	367	* Execute the ATIF_FUNCTION_GET_SYSTEM_BIOS_REQUESTS ATIF function
@@ -899,6 +897,8 @@ void amdgpu_acpi_fini(struct amdgpu_device *adev) · 529 unmodified lines
899	897	/**
900	898	* amdgpu_acpi_is_s0ix_supported
901	899	*
	900	+ * @adev: amdgpu_device_pointer
	901	+ *
902	902	* returns true if supported, false if not.
903	903	*/
904	904	bool amdgpu_acpi_is_s0ix_supported(struct amdgpu_device *adev)

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h+17−9

@@ -75,6 +75,7 @@ struct amdgpu_amdkfd_fence { · 74 unmodified lines
75	75	struct mm_struct *mm;
76	76	spinlock_t lock;
77	77	char timeline_name[TASK_COMM_LEN];
	78	+ struct svm_range_bo *svm_bo;
78	79	};
79	80
80	81	struct amdgpu_kfd_dev {
@@ -148,7 +149,8 @@ int amdgpu_queue_mask_bit_to_set_resource_bit(struct amdgpu_device *adev, · 67 unmodified lines
148	149	int queue_bit);
149	150
150	151	struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
151		− struct mm_struct *mm);
	152	+ struct mm_struct *mm,
	153	+ struct svm_range_bo *svm_bo);
152	154	#if IS_ENABLED(CONFIG_HSA_AMD)
153	155	bool amdkfd_fence_check_mm(struct dma_fence f, struct mm_struct mm);
154	156	struct amdgpu_amdkfd_fence to_amdgpu_amdkfd_fence(struct dma_fence f);
@@ -234,22 +236,27 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev dst, struct kgd_dev s · 79 unmodified lines
234	236	})
235	237
236	238	/* GPUVM API */
	239	+#define drm_priv_to_vm(drm_priv) \
	240	+ (&((struct amdgpu_fpriv *) \
	241	+ ((struct drm_file *)(drm_priv))->driver_priv)->vm)
	242	+
237	243	int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
238	244	struct file *filp, u32 pasid,
239		− void vm, void process_info,
	245	+ void **process_info,
240	246	struct dma_fence **ef);
241		−void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev kgd, void vm);
242		−uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm);
	247	+void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev kgd, void drm_priv);
	248	+uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv);
243	249	int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
244	250	struct kgd_dev *kgd, uint64_t va, uint64_t size,
245		− void vm, struct kgd_mem *mem,
	251	+ void drm_priv, struct kgd_mem *mem,
246	252	uint64_t *offset, uint32_t flags);
247	253	int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
248		− struct kgd_dev kgd, struct kgd_mem mem, uint64_t *size);
	254	+ struct kgd_dev kgd, struct kgd_mem mem, void *drm_priv,
	255	+ uint64_t *size);
249	256	int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
250		− struct kgd_dev kgd, struct kgd_mem mem, void *vm);
	257	+ struct kgd_dev kgd, struct kgd_mem mem, void *drm_priv);
251	258	int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
252		− struct kgd_dev kgd, struct kgd_mem mem, void *vm);
	259	+ struct kgd_dev kgd, struct kgd_mem mem, void *drm_priv);
253	260	int amdgpu_amdkfd_gpuvm_sync_memory(
254	261	struct kgd_dev kgd, struct kgd_mem mem, bool intr);
255	262	int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_dev *kgd,
@@ -260,7 +267,7 @@ int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd, · 4 unmodified lines
260	267	struct kfd_vm_fault_info *info);
261	268	int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
262	269	struct dma_buf *dmabuf,
263		− uint64_t va, void *vm,
	270	+ uint64_t va, void *drm_priv,
264	271	struct kgd_mem *mem, uint64_t size,
265	272	uint64_t *mmap_offset);
266	273	int amdgpu_amdkfd_get_tile_config(struct kgd_dev *kgd,
@@ -270,6 +277,7 @@ void amdgpu_amdkfd_gpuvm_init_mem_limits(void); · 3 unmodified lines
270	277	void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev,
271	278	struct amdgpu_vm *vm);
272	279	void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo);
	280	+void amdgpu_amdkfd_reserve_system_mem(uint64_t size);
273	281	#else
274	282	static inline
275	283	void amdgpu_amdkfd_gpuvm_init_mem_limits(void)

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c+1−0

@@ -25,6 +25,7 @@ · 24 unmodified lines
25	25	#include <linux/firmware.h>
26	26	#include "amdgpu.h"
27	27	#include "amdgpu_amdkfd.h"
	28	+#include "amdgpu_amdkfd_arcturus.h"
28	29	#include "sdma0/sdma0_4_2_2_offset.h"
29	30	#include "sdma0/sdma0_4_2_2_sh_mask.h"
30	31	#include "sdma1/sdma1_4_2_2_offset.h"

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_fence.c+14−6

@@ -28,6 +28,7 @@ · 27 unmodified lines
28	28	#include <linux/slab.h>
29	29	#include <linux/sched/mm.h>
30	30	#include "amdgpu_amdkfd.h"
	31	+#include "kfd_svm.h"
31	32
32	33	static const struct dma_fence_ops amdkfd_fence_ops;
33	34	static atomic_t fence_seq = ATOMIC_INIT(0);
@@ -60,7 +61,8 @@ static atomic_t fence_seq = ATOMIC_INIT(0); · 26 unmodified lines
60	61	*/
61	62
62	63	struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context,
63		− struct mm_struct *mm)
	64	+ struct mm_struct *mm,
	65	+ struct svm_range_bo *svm_bo)
64	66	{
65	67	struct amdgpu_amdkfd_fence *fence;
66	68
@@ -73,7 +75,7 @@ struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, · 6 unmodified lines
73	75	fence->mm = mm;
74	76	get_task_comm(fence->timeline_name, current);
75	77	spin_lock_init(&fence->lock);
76		−
	78	+ fence->svm_bo = svm_bo;
77	79	dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock,
78	80	context, atomic_inc_return(&fence_seq));
79	81
@@ -111,6 +113,8 @@ static const char amdkfd_fence_get_timeline_name(struct dma_fence f) · 31 unmodified lines
111	113	* a KFD BO and schedules a job to move the BO.
112	114	* If fence is already signaled return true.
113	115	* If fence is not signaled schedule a evict KFD process work item.
	116	+ *
	117	+ * @f: dma_fence
114	118	*/
115	119	static bool amdkfd_fence_enable_signaling(struct dma_fence *f)
116	120	{
@@ -122,16 +126,20 @@ static bool amdkfd_fence_enable_signaling(struct dma_fence *f) · 5 unmodified lines
122	126	if (dma_fence_is_signaled(f))
123	127	return true;
124	128
125		− if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f))
126		− return true;
127		−
	129	+ if (!fence->svm_bo) {
	130	+ if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, f))
	131	+ return true;
	132	+ } else {
	133	+ if (!svm_range_schedule_evict_svm_bo(fence))
	134	+ return true;
	135	+ }
128	136	return false;
129	137	}
130	138
131	139	/**
132	140	* amdkfd_fence_release - callback that fence can be freed
133	141	*
134		− * @fence: fence
	142	+ * @f: dma_fence
135	143	*
136	144	* This function is called when the reference count becomes zero.
137	145	* Drops the mm_struct reference and RCU schedules freeing up the fence.

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c+4−4

@@ -719,7 +719,7 @@ static void unlock_spi_csq_mutexes(struct amdgpu_device *adev) · 718 unmodified lines
719	719	}
720	720
721	721	/**
722		− * @get_wave_count: Read device registers to get number of waves in flight for
	722	+ * get_wave_count: Read device registers to get number of waves in flight for
723	723	* a particular queue. The method also returns the VMID associated with the
724	724	* queue.
725	725	*
@@ -755,19 +755,19 @@ static void get_wave_count(struct amdgpu_device *adev, int queue_idx, · 29 unmodified lines
755	755	}
756	756
757	757	/**
758		− * @kgd_gfx_v9_get_cu_occupancy: Reads relevant registers associated with each
	758	+ * kgd_gfx_v9_get_cu_occupancy: Reads relevant registers associated with each
759	759	* shader engine and aggregates the number of waves that are in flight for the
760	760	* process whose pasid is provided as a parameter. The process could have ZERO
761	761	* or more queues running and submitting waves to compute units.
762	762	*
763	763	* @kgd: Handle of device from which to get number of waves in flight
764	764	* @pasid: Identifies the process for which this query call is invoked
765		− * @wave_cnt: Output parameter updated with number of waves in flight that
	765	+ * @pasid_wave_cnt: Output parameter updated with number of waves in flight that
766	766	* belong to process with given pasid
767	767	* @max_waves_per_cu: Output parameter updated with maximum number of waves
768	768	* possible per Compute Unit
769	769	*
770		− * @note: It's possible that the device has too many queues (oversubscription)
	770	+ * Note: It's possible that the device has too many queues (oversubscription)
771	771	* in which case a VMID could be remapped to a different PASID. This could lead
772	772	* to an iaccurate wave count. Following is a high-level sequence:
773	773	* Time T1: vmid = getVmid(); vmid is associated with Pasid P1

drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c+62−36

@@ -33,9 +33,6 @@ · 32 unmodified lines
33	33	#include <uapi/linux/kfd_ioctl.h>
34	34	#include "amdgpu_xgmi.h"
35	35
36		−/* BO flag to indicate a KFD userptr BO */
37		−#define AMDGPU_AMDKFD_USERPTR_BO (1ULL << 63)
38		−
39	36	/* Userptr restore delay, just long enough to allow consecutive VM
40	37	* changes to accumulate
41	38	*/
@@ -108,6 +105,11 @@ void amdgpu_amdkfd_gpuvm_init_mem_limits(void) · 66 unmodified lines
108	105	(kfd_mem_limit.max_ttm_mem_limit >> 20));
109	106	}
110	107
	108	+void amdgpu_amdkfd_reserve_system_mem(uint64_t size)
	109	+{
	110	+ kfd_mem_limit.system_mem_used += size;
	111	+}
	112	+
111	113	/* Estimate page table size needed to represent a given memory size
112	114	*
113	115	* With 4KB pages, we need one 8 byte PTE for each 4KB of memory
@@ -217,7 +219,7 @@ void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo) · 103 unmodified lines
217	219	u32 domain = bo->preferred_domains;
218	220	bool sg = (bo->preferred_domains == AMDGPU_GEM_DOMAIN_CPU);
219	221
220		− if (bo->flags & AMDGPU_AMDKFD_USERPTR_BO) {
	222	+ if (bo->flags & AMDGPU_AMDKFD_CREATE_USERPTR_BO) {
221	223	domain = AMDGPU_GEM_DOMAIN_CPU;
222	224	sg = false;
223	225	}
@@ -967,7 +969,8 @@ static int init_kfd_vm(struct amdgpu_vm vm, void *process_info, · 743 unmodified lines
967	969
968	970	info->eviction_fence =
969	971	amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
970		− current->mm);
	972	+ current->mm,
	973	+ NULL);
971	974	if (!info->eviction_fence) {
972	975	pr_err("Failed to create eviction fence\n");
973	976	ret = -ENOMEM;
@@ -1036,15 +1039,19 @@ create_evict_fence_fail: · 62 unmodified lines
1036	1039
1037	1040	int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd,
1038	1041	struct file *filp, u32 pasid,
1039		− void vm, void process_info,
	1042	+ void **process_info,
1040	1043	struct dma_fence **ef)
1041	1044	{
1042	1045	struct amdgpu_device *adev = get_amdgpu_device(kgd);
1043		− struct drm_file *drm_priv = filp->private_data;
1044		− struct amdgpu_fpriv *drv_priv = drm_priv->driver_priv;
1045		− struct amdgpu_vm *avm = &drv_priv->vm;
	1046	+ struct amdgpu_fpriv *drv_priv;
	1047	+ struct amdgpu_vm *avm;
1046	1048	int ret;
1047	1049
	1050	+ ret = amdgpu_file_to_fpriv(filp, &drv_priv);
	1051	+ if (ret)
	1052	+ return ret;
	1053	+ avm = &drv_priv->vm;
	1054	+
1048	1055	/* Already a compute VM? */
1049	1056	if (avm->process_info)
1050	1057	return -EINVAL;
@@ -1059,7 +1066,7 @@ int amdgpu_amdkfd_gpuvm_acquire_process_vm(struct kgd_dev *kgd, · 8 unmodified lines
1059	1066	if (ret)
1060	1067	return ret;
1061	1068
1062		− vm = (void )avm;
	1069	+ amdgpu_vm_set_task_info(avm);
1063	1070
1064	1071	return 0;
1065	1072	}
@@ -1100,15 +1107,17 @@ void amdgpu_amdkfd_gpuvm_destroy_cb(struct amdgpu_device *adev, · 34 unmodified lines
1100	1107	}
1101	1108	}
1102	1109
1103		−void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev kgd, void vm)
	1110	+void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev kgd, void drm_priv)
1104	1111	{
1105	1112	struct amdgpu_device *adev = get_amdgpu_device(kgd);
1106		− struct amdgpu_vm avm = (struct amdgpu_vm )vm;
	1113	+ struct amdgpu_vm *avm;
1107	1114
1108		− if (WARN_ON(!kgd \|\| !vm))
	1115	+ if (WARN_ON(!kgd \|\| !drm_priv))
1109	1116	return;
1110	1117
1111		− pr_debug("Releasing process vm %p\n", vm);
	1118	+ avm = drm_priv_to_vm(drm_priv);
	1119	+
	1120	+ pr_debug("Releasing process vm %p\n", avm);
1112	1121
1113	1122	/* The original pasid of amdgpu vm has already been
1114	1123	* released during making a amdgpu vm to a compute vm
@@ -1119,9 +1128,9 @@ void amdgpu_amdkfd_gpuvm_release_process_vm(struct kgd_dev kgd, void vm) · 4 unmodified lines
1119	1128	amdgpu_vm_release_compute(adev, avm);
1120	1129	}
1121	1130
1122		−uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm)
	1131	+uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *drm_priv)
1123	1132	{
1124		− struct amdgpu_vm avm = (struct amdgpu_vm )vm;
	1133	+ struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1125	1134	struct amdgpu_bo *pd = avm->root.base.bo;
1126	1135	struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
1127	1136
@@ -1132,11 +1141,11 @@ uint64_t amdgpu_amdkfd_gpuvm_get_process_page_dir(void *vm) · 4 unmodified lines
1132	1141
1133	1142	int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1134	1143	struct kgd_dev *kgd, uint64_t va, uint64_t size,
1135		− void vm, struct kgd_mem *mem,
	1144	+ void drm_priv, struct kgd_mem *mem,
1136	1145	uint64_t *offset, uint32_t flags)
1137	1146	{
1138	1147	struct amdgpu_device *adev = get_amdgpu_device(kgd);
1139		− struct amdgpu_vm avm = (struct amdgpu_vm )vm;
	1148	+ struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1140	1149	enum ttm_bo_type bo_type = ttm_bo_type_device;
1141	1150	struct sg_table *sg = NULL;
1142	1151	uint64_t user_addr = 0;
@@ -1216,6 +1225,11 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( · 73 unmodified lines
1216	1225	domain_string(alloc_domain), ret);
1217	1226	goto err_bo_create;
1218	1227	}
	1228	+ ret = drm_vma_node_allow(&gobj->vma_node, drm_priv);
	1229	+ if (ret) {
	1230	+ pr_debug("Failed to allow vma node access. ret %d\n", ret);
	1231	+ goto err_node_allow;
	1232	+ }
1219	1233	bo = gem_to_amdgpu_bo(gobj);
1220	1234	if (bo_type == ttm_bo_type_sg) {
1221	1235	bo->tbo.sg = sg;
@@ -1224,7 +1238,7 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( · 2 unmodified lines
1224	1238	bo->kfd_bo = *mem;
1225	1239	(*mem)->bo = bo;
1226	1240	if (user_addr)
1227		− bo->flags \|= AMDGPU_AMDKFD_USERPTR_BO;
	1241	+ bo->flags \|= AMDGPU_AMDKFD_CREATE_USERPTR_BO;
1228	1242
1229	1243	(*mem)->va = va;
1230	1244	(*mem)->domain = domain;
@@ -1245,6 +1259,8 @@ int amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu( · 14 unmodified lines
1245	1259
1246	1260	allocate_init_user_pages_failed:
1247	1261	remove_kgd_mem_from_kfd_bo_list(*mem, avm->process_info);
	1262	+ drm_vma_node_revoke(&gobj->vma_node, drm_priv);
	1263	+err_node_allow:
1248	1264	amdgpu_bo_unref(&bo);
1249	1265	/* Don't unreserve system mem limit twice */
1250	1266	goto err_reserve_limit;
@@ -1262,7 +1278,8 @@ err: · 11 unmodified lines
1262	1278	}
1263	1279
1264	1280	int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
1265		− struct kgd_dev kgd, struct kgd_mem mem, uint64_t *size)
	1281	+ struct kgd_dev kgd, struct kgd_mem mem, void *drm_priv,
	1282	+ uint64_t *size)
1266	1283	{
1267	1284	struct amdkfd_process_info *process_info = mem->process_info;
1268	1285	unsigned long bo_size = mem->bo->tbo.base.size;
@@ -1339,6 +1356,7 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( · 70 unmodified lines
1339	1356	}
1340	1357
1341	1358	/* Free the BO*/
	1359	+ drm_vma_node_revoke(&mem->bo->tbo.base.vma_node, drm_priv);
1342	1360	drm_gem_object_put(&mem->bo->tbo.base);
1343	1361	mutex_destroy(&mem->lock);
1344	1362	kfree(mem);
@@ -1347,10 +1365,10 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu( · 2 unmodified lines
1347	1365	}
1348	1366
1349	1367	int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1350		− struct kgd_dev kgd, struct kgd_mem mem, void *vm)
	1368	+ struct kgd_dev kgd, struct kgd_mem mem, void *drm_priv)
1351	1369	{
1352	1370	struct amdgpu_device *adev = get_amdgpu_device(kgd);
1353		− struct amdgpu_vm avm = (struct amdgpu_vm )vm;
	1371	+ struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1354	1372	int ret;
1355	1373	struct amdgpu_bo *bo;
1356	1374	uint32_t domain;
@@ -1391,9 +1409,9 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( · 34 unmodified lines
1391	1409	pr_debug("Map VA 0x%llx - 0x%llx to vm %p domain %s\n",
1392	1410	mem->va,
1393	1411	mem->va + bo_size * (1 + mem->aql_queue),
1394		− vm, domain_string(domain));
	1412	+ avm, domain_string(domain));
1395	1413
1396		− ret = reserve_bo_and_vm(mem, vm, &ctx);
	1414	+ ret = reserve_bo_and_vm(mem, avm, &ctx);
1397	1415	if (unlikely(ret))
1398	1416	goto out;
1399	1417
@@ -1437,7 +1455,7 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( · 37 unmodified lines
1437	1455	}
1438	1456
1439	1457	list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
1440		− if (entry->bo_va->base.vm == vm && !entry->is_mapped) {
	1458	+ if (entry->bo_va->base.vm == avm && !entry->is_mapped) {
1441	1459	pr_debug("\t map VA 0x%llx - 0x%llx in entry %p\n",
1442	1460	entry->va, entry->va + bo_size,
1443	1461	entry);
@@ -1449,7 +1467,7 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu( · 5 unmodified lines
1449	1467	goto map_bo_to_gpuvm_failed;
1450	1468	}
1451	1469
1452		− ret = vm_update_pds(vm, ctx.sync);
	1470	+ ret = vm_update_pds(avm, ctx.sync);
1453	1471	if (ret) {
1454	1472	pr_err("Failed to update page directories\n");
1455	1473	goto map_bo_to_gpuvm_failed;
@@ -1485,11 +1503,11 @@ out: · 29 unmodified lines
1485	1503	}
1486	1504
1487	1505	int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1488		− struct kgd_dev kgd, struct kgd_mem mem, void *vm)
	1506	+ struct kgd_dev kgd, struct kgd_mem mem, void *drm_priv)
1489	1507	{
1490	1508	struct amdgpu_device *adev = get_amdgpu_device(kgd);
1491		− struct amdkfd_process_info *process_info =
1492		− ((struct amdgpu_vm *)vm)->process_info;
	1509	+ struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
	1510	+ struct amdkfd_process_info *process_info = avm->process_info;
1493	1511	unsigned long bo_size = mem->bo->tbo.base.size;
1494	1512	struct kfd_bo_va_list *entry;
1495	1513	struct bo_vm_reservation_context ctx;
@@ -1497,7 +1515,7 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( · 1 unmodified line
1497	1515
1498	1516	mutex_lock(&mem->lock);
1499	1517
1500		− ret = reserve_bo_and_cond_vms(mem, vm, BO_VM_MAPPED, &ctx);
	1518	+ ret = reserve_bo_and_cond_vms(mem, avm, BO_VM_MAPPED, &ctx);
1501	1519	if (unlikely(ret))
1502	1520	goto out;
1503	1521	/* If no VMs were reserved, it means the BO wasn't actually mapped */
@@ -1506,17 +1524,17 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( · 2 unmodified lines
1506	1524	goto unreserve_out;
1507	1525	}
1508	1526
1509		− ret = vm_validate_pt_pd_bos((struct amdgpu_vm *)vm);
	1527	+ ret = vm_validate_pt_pd_bos(avm);
1510	1528	if (unlikely(ret))
1511	1529	goto unreserve_out;
1512	1530
1513	1531	pr_debug("Unmap VA 0x%llx - 0x%llx from vm %p\n",
1514	1532	mem->va,
1515	1533	mem->va + bo_size * (1 + mem->aql_queue),
1516		− vm);
	1534	+ avm);
1517	1535
1518	1536	list_for_each_entry(entry, &mem->bo_va_list, bo_list) {
1519		− if (entry->bo_va->base.vm == vm && entry->is_mapped) {
	1537	+ if (entry->bo_va->base.vm == avm && entry->is_mapped) {
1520	1538	pr_debug("\t unmap VA 0x%llx - 0x%llx from entry %p\n",
1521	1539	entry->va,
1522	1540	entry->va + bo_size,
@@ -1642,14 +1660,15 @@ int amdgpu_amdkfd_gpuvm_get_vm_fault_info(struct kgd_dev *kgd, · 119 unmodified lines
1642	1660
1643	1661	int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd,
1644	1662	struct dma_buf *dma_buf,
1645		− uint64_t va, void *vm,
	1663	+ uint64_t va, void *drm_priv,
1646	1664	struct kgd_mem *mem, uint64_t size,
1647	1665	uint64_t *mmap_offset)
1648	1666	{
1649	1667	struct amdgpu_device adev = (struct amdgpu_device )kgd;
	1668	+ struct amdgpu_vm *avm = drm_priv_to_vm(drm_priv);
1650	1669	struct drm_gem_object *obj;
1651	1670	struct amdgpu_bo *bo;
1652		− struct amdgpu_vm avm = (struct amdgpu_vm )vm;
	1671	+ int ret;
1653	1672
1654	1673	if (dma_buf->ops != &amdgpu_dmabuf_ops)
1655	1674	/* Can't handle non-graphics buffers */
@@ -1670,6 +1689,12 @@ int amdgpu_amdkfd_gpuvm_import_dmabuf(struct kgd_dev *kgd, · 14 unmodified lines
1670	1689	if (!*mem)
1671	1690	return -ENOMEM;
1672	1691
	1692	+ ret = drm_vma_node_allow(&obj->vma_node, drm_priv);
	1693	+ if (ret) {
	1694	+ kfree(mem);
	1695	+ return ret;
	1696	+ }
	1697	+
1673	1698	if (size)
1674	1699	*size = amdgpu_bo_size(bo);
1675	1700
@@ -2135,7 +2160,8 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void info, struct dma_fence *ef) · 459 unmodified lines
2135	2160	*/
2136	2161	new_fence = amdgpu_amdkfd_fence_create(
2137	2162	process_info->eviction_fence->base.context,
2138		− process_info->eviction_fence->mm);
	2163	+ process_info->eviction_fence->mm,
	2164	+ NULL);
2139	2165	if (!new_fence) {
2140	2166	pr_err("Failed to create eviction fence\n");
2141	2167	ret = -ENOMEM;

drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c+3−3

@@ -672,7 +672,7 @@ static int amdgpu_cs_sync_rings(struct amdgpu_cs_parser *p) · 671 unmodified lines
672	672	}
673	673
674	674	/**
675		− * cs_parser_fini() - clean parser states
	675	+ * amdgpu_cs_parser_fini() - clean parser states
676	676	* @parser: parser structure holding parsing context.
677	677	* @error: error number
678	678	* @backoff: indicator to backoff the reservation
@@ -1488,7 +1488,7 @@ int amdgpu_cs_fence_to_handle_ioctl(struct drm_device dev, void data, · 809 unmodified lines
1488	1488	}
1489	1489
1490	1490	/**
1491		− * amdgpu_cs_wait_all_fence - wait on all fences to signal
	1491	+ * amdgpu_cs_wait_all_fences - wait on all fences to signal
1492	1492	*
1493	1493	* @adev: amdgpu device
1494	1494	* @filp: file private
@@ -1639,7 +1639,7 @@ err_free_fences: · 144 unmodified lines
1639	1639	}
1640	1640
1641	1641	/**
1642		− * amdgpu_cs_find_bo_va - find bo_va for VM address
	1642	+ * amdgpu_cs_find_mapping - find bo_va for VM address
1643	1643	*
1644	1644	* @parser: command submission parser context
1645	1645	* @addr: VM address

drivers/gpu/drm/amd/amdgpu/amdgpu_device.c+10−11

@@ -2856,7 +2856,7 @@ static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev) · 2855 unmodified lines
2856	2856	AMD_IP_BLOCK_TYPE_IH,
2857	2857	};
2858	2858
2859		− for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
	2859	+ for (i = 0; i < adev->num_ip_blocks; i++) {
2860	2860	int j;
2861	2861	struct amdgpu_ip_block *block;
2862	2862
@@ -3179,8 +3179,8 @@ static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) · 316 unmodified lines
3179	3179	int ret = 0;
3180	3180
3181	3181	/*
3182		− * By default timeout for non compute jobs is 10000.
3183		− * And there is no timeout enforced on compute jobs.
	3182	+ * By default timeout for non compute jobs is 10000
	3183	+ * and 60000 for compute jobs.
3184	3184	* In SR-IOV or passthrough mode, timeout for compute
3185	3185	* jobs are 60000 by default.
3186	3186	*/
@@ -3189,10 +3189,8 @@ static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev) · 2 unmodified lines
3189	3189	if (amdgpu_sriov_vf(adev))
3190	3190	adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3191	3191	msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3192		− else if (amdgpu_passthrough(adev))
3193		− adev->compute_timeout = msecs_to_jiffies(60000);
3194	3192	else
3195		− adev->compute_timeout = MAX_SCHEDULE_TIMEOUT;
	3193	+ adev->compute_timeout = msecs_to_jiffies(60000);
3196	3194
3197	3195	if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3198	3196	while ((timeout_setting = strsep(&input, ",")) &&
@@ -3741,7 +3739,6 @@ void amdgpu_device_fini(struct amdgpu_device *adev) · 542 unmodified lines
3741	3739	int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
3742	3740	{
3743	3741	struct amdgpu_device *adev = drm_to_adev(dev);
3744		− int r;
3745	3742
3746	3743	if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3747	3744	return 0;
@@ -3756,7 +3753,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) · 8 unmodified lines
3756	3753
3757	3754	amdgpu_ras_suspend(adev);
3758	3755
3759		− r = amdgpu_device_ip_suspend_phase1(adev);
	3756	+ amdgpu_device_ip_suspend_phase1(adev);
3760	3757
3761	3758	if (!adev->in_s0ix)
3762	3759	amdgpu_amdkfd_suspend(adev, adev->in_runpm);
@@ -3766,7 +3763,7 @@ int amdgpu_device_suspend(struct drm_device *dev, bool fbcon) · 3 unmodified lines
3766	3763
3767	3764	amdgpu_fence_driver_suspend(adev);
3768	3765
3769		− r = amdgpu_device_ip_suspend_phase2(adev);
	3766	+ amdgpu_device_ip_suspend_phase2(adev);
3770	3767	/* evict remaining vram memory
3771	3768	* This second call to evict vram is to evict the gart page table
3772	3769	* using the CPU.
@@ -5124,7 +5121,8 @@ int amdgpu_device_baco_enter(struct drm_device *dev) · 1351 unmodified lines
5124	5121	if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5125	5122	return -ENOTSUPP;
5126	5123
5127		− if (ras && ras->supported && adev->nbio.funcs->enable_doorbell_interrupt)
	5124	+ if (ras && adev->ras_enabled &&
	5125	+ adev->nbio.funcs->enable_doorbell_interrupt)
5128	5126	adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5129	5127
5130	5128	return amdgpu_dpm_baco_enter(adev);
@@ -5143,7 +5141,8 @@ int amdgpu_device_baco_exit(struct drm_device *dev) · 12 unmodified lines
5143	5141	if (ret)
5144	5142	return ret;
5145	5143
5146		− if (ras && ras->supported && adev->nbio.funcs->enable_doorbell_interrupt)
	5144	+ if (ras && adev->ras_enabled &&
	5145	+ adev->nbio.funcs->enable_doorbell_interrupt)
5147	5146	adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5148	5147
5149	5148	return 0;

drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c+9−9

@@ -288,9 +288,9 @@ module_param_named(msi, amdgpu_msi, int, 0444); · 287 unmodified lines
288	288	* for SDMA and Video.
289	289	*
290	290	* By default(with no lockup_timeout settings), the timeout for all non-compute(GFX, SDMA and Video)
291		− * jobs is 10000. And there is no timeout enforced on compute jobs.
	291	+ * jobs is 10000. The timeout for compute is 60000.
292	292	*/
293		−MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms (default: for bare metal 10000 for non-compute jobs and infinity timeout for compute jobs; "
	293	+MODULE_PARM_DESC(lockup_timeout, "GPU lockup timeout in ms (default: for bare metal 10000 for non-compute jobs and 60000 for compute jobs; "
294	294	"for passthrough or sriov, 10000 for all jobs."
295	295	" 0: keep default value. negative: infinity timeout), "
296	296	"format: for bare metal [Non-Compute] or [GFX,Compute,SDMA,Video]; "
@@ -641,7 +641,8 @@ module_param_named(mes, amdgpu_mes, int, 0444); · 344 unmodified lines
641	641
642	642	/**
643	643	* DOC: noretry (int)
644		− * Disable retry faults in the GPU memory controller.
	644	+ * Disable XNACK retry in the SQ by default on GFXv9 hardware. On ASICs that
	645	+ * do not support per-process XNACK this also disables retry page faults.
645	646	* (0 = retry enabled, 1 = retry disabled, -1 auto (default))
646	647	*/
647	648	MODULE_PARM_DESC(noretry,
@@ -1186,6 +1187,7 @@ static const struct pci_device_id pciidlist[] = { · 538 unmodified lines
1186	1187	{0x1002, 0x7408, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ALDEBARAN\|AMD_EXP_HW_SUPPORT},
1187	1188	{0x1002, 0x740C, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ALDEBARAN\|AMD_EXP_HW_SUPPORT},
1188	1189	{0x1002, 0x740F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ALDEBARAN\|AMD_EXP_HW_SUPPORT},
	1190	+ {0x1002, 0x7410, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_ALDEBARAN\|AMD_EXP_HW_SUPPORT},
1189	1191
1190	1192	{0, 0, 0}
1191	1193	};
@@ -1598,17 +1600,15 @@ static int amdgpu_pmops_runtime_idle(struct device *dev) · 406 unmodified lines
1598	1600	if (amdgpu_device_has_dc_support(adev)) {
1599	1601	struct drm_crtc *crtc;
1600	1602
1601		− drm_modeset_lock_all(drm_dev);
1602		−
1603	1603	drm_for_each_crtc(crtc, drm_dev) {
1604		− if (crtc->state->active) {
	1604	+ drm_modeset_lock(&crtc->mutex, NULL);
	1605	+ if (crtc->state->active)
1605	1606	ret = -EBUSY;
	1607	+ drm_modeset_unlock(&crtc->mutex);
	1608	+ if (ret < 0)
1606	1609	break;
1607		− }
1608	1610	}
1609	1611
1610		− drm_modeset_unlock_all(drm_dev);
1611		−
1612	1612	} else {
1613	1613	struct drm_connector *list_connector;
1614	1614	struct drm_connector_list_iter iter;

drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c+1−0

@@ -434,6 +434,7 @@ int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring, · 433 unmodified lines
434	434	*
435	435	* @ring: ring to init the fence driver on
436	436	* @num_hw_submission: number of entries on the hardware queue
	437	+ * @sched_score: optional score atomic shared with other schedulers
437	438	*
438	439	* Init the fence driver for the requested ring (all asics).
439	440	* Helper function for amdgpu_fence_driver_init().

drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c+2−2

@@ -60,7 +60,7 @@ · 59 unmodified lines
60	60	*/
61	61
62	62	/**
63		− * amdgpu_dummy_page_init - init dummy page used by the driver
	63	+ * amdgpu_gart_dummy_page_init - init dummy page used by the driver
64	64	*
65	65	* @adev: amdgpu_device pointer
66	66	*
@@ -86,7 +86,7 @@ static int amdgpu_gart_dummy_page_init(struct amdgpu_device *adev) · 19 unmodified lines
86	86	}
87	87
88	88	/**
89		− * amdgpu_dummy_page_fini - free dummy page used by the driver
	89	+ * amdgpu_gart_dummy_page_fini - free dummy page used by the driver
90	90	*
91	91	* @adev: amdgpu_device pointer
92	92	*

drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c+56−5

@@ -332,6 +332,17 @@ void amdgpu_gmc_agp_location(struct amdgpu_device adev, struct amdgpu_gmc mc) · 331 unmodified lines
332	332	mc->agp_size >> 20, mc->agp_start, mc->agp_end);
333	333	}
334	334
	335	+/**
	336	+ * amdgpu_gmc_fault_key - get hask key from vm fault address and pasid
	337	+ *
	338	+ * @addr: 48 bit physical address, page aligned (36 significant bits)
	339	+ * @pasid: 16 bit process address space identifier
	340	+ */
	341	+static inline uint64_t amdgpu_gmc_fault_key(uint64_t addr, uint16_t pasid)
	342	+{
	343	+ return addr << 4 \| pasid;
	344	+}
	345	+
335	346	/**
336	347	* amdgpu_gmc_filter_faults - filter VM faults
337	348	*
@@ -348,8 +359,7 @@ bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr, · 10 unmodified lines
348	359	uint16_t pasid, uint64_t timestamp)
349	360	{
350	361	struct amdgpu_gmc *gmc = &adev->gmc;
351		−
352		− uint64_t stamp, key = addr << 4 \| pasid;
	362	+ uint64_t stamp, key = amdgpu_gmc_fault_key(addr, pasid);
353	363	struct amdgpu_gmc_fault *fault;
354	364	uint32_t hash;
355	365
@@ -365,7 +375,7 @@ bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr, · 9 unmodified lines
365	375	while (fault->timestamp >= stamp) {
366	376	uint64_t tmp;
367	377
368		− if (fault->key == key)
	378	+ if (atomic64_read(&fault->key) == key)
369	379	return true;
370	380
371	381	tmp = fault->timestamp;
@@ -378,7 +388,7 @@ bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr, · 6 unmodified lines
378	388
379	389	/* Add the fault to the ring */
380	390	fault = &gmc->fault_ring[gmc->last_fault];
381		− fault->key = key;
	391	+ atomic64_set(&fault->key, key);
382	392	fault->timestamp = timestamp;
383	393
384	394	/* And update the hash */
@@ -387,6 +397,36 @@ bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr, · 2 unmodified lines
387	397	return false;
388	398	}
389	399
	400	+/**
	401	+ * amdgpu_gmc_filter_faults_remove - remove address from VM faults filter
	402	+ *
	403	+ * @adev: amdgpu device structure
	404	+ * @addr: address of the VM fault
	405	+ * @pasid: PASID of the process causing the fault
	406	+ *
	407	+ * Remove the address from fault filter, then future vm fault on this address
	408	+ * will pass to retry fault handler to recover.
	409	+ */
	410	+void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr,
	411	+ uint16_t pasid)
	412	+{
	413	+ struct amdgpu_gmc *gmc = &adev->gmc;
	414	+ uint64_t key = amdgpu_gmc_fault_key(addr, pasid);
	415	+ struct amdgpu_gmc_fault *fault;
	416	+ uint32_t hash;
	417	+ uint64_t tmp;
	418	+
	419	+ hash = hash_64(key, AMDGPU_GMC_FAULT_HASH_ORDER);
	420	+ fault = &gmc->fault_ring[gmc->fault_hash[hash].idx];
	421	+ do {
	422	+ if (atomic64_cmpxchg(&fault->key, key, 0) == key)
	423	+ break;
	424	+
	425	+ tmp = fault->timestamp;
	426	+ fault = &gmc->fault_ring[fault->next];
	427	+ } while (fault->timestamp < tmp);
	428	+}
	429	+
390	430	int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev)
391	431	{
392	432	int r;
@@ -415,6 +455,13 @@ int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev) · 22 unmodified lines
415	455	return r;
416	456	}
417	457
	458	+ if (adev->hdp.ras_funcs &&
	459	+ adev->hdp.ras_funcs->ras_late_init) {
	460	+ r = adev->hdp.ras_funcs->ras_late_init(adev);
	461	+ if (r)
	462	+ return r;
	463	+ }
	464	+
418	465	return 0;
419	466	}
420	467
@@ -426,11 +473,15 @@ void amdgpu_gmc_ras_fini(struct amdgpu_device *adev) · 5 unmodified lines
426	473
427	474	if (adev->mmhub.ras_funcs &&
428	475	adev->mmhub.ras_funcs->ras_fini)
429		− amdgpu_mmhub_ras_fini(adev);
	476	+ adev->mmhub.ras_funcs->ras_fini(adev);
430	477
431	478	if (adev->gmc.xgmi.ras_funcs &&
432	479	adev->gmc.xgmi.ras_funcs->ras_fini)
433	480	adev->gmc.xgmi.ras_funcs->ras_fini(adev);
	481	+
	482	+ if (adev->hdp.ras_funcs &&
	483	+ adev->hdp.ras_funcs->ras_fini)
	484	+ adev->hdp.ras_funcs->ras_fini(adev);
434	485	}
435	486
436	487	/*

drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.h+4−2

@@ -66,9 +66,9 @@ struct firmware; · 65 unmodified lines
66	66	* GMC page fault information
67	67	*/
68	68	struct amdgpu_gmc_fault {
69		− uint64_t timestamp;
	69	+ uint64_t timestamp:48;
70	70	uint64_t next:AMDGPU_GMC_FAULT_RING_ORDER;
71		− uint64_t key:52;
	71	+ atomic64_t key;
72	72	};
73	73
74	74	/*
@@ -318,6 +318,8 @@ void amdgpu_gmc_agp_location(struct amdgpu_device *adev, · 243 unmodified lines
318	318	struct amdgpu_gmc *mc);
319	319	bool amdgpu_gmc_filter_faults(struct amdgpu_device *adev, uint64_t addr,
320	320	uint16_t pasid, uint64_t timestamp);
	321	+void amdgpu_gmc_filter_faults_remove(struct amdgpu_device *adev, uint64_t addr,
	322	+ uint16_t pasid);
321	323	int amdgpu_gmc_ras_late_init(struct amdgpu_device *adev);
322	324	void amdgpu_gmc_ras_fini(struct amdgpu_device *adev);
323	325	int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev);

drivers/gpu/drm/amd/amdgpu/amdgpu_gtt_mgr.c+90−79

@@ -24,7 +24,8 @@ · 23 unmodified lines
24	24
25	25	#include "amdgpu.h"
26	26
27		−static inline struct amdgpu_gtt_mgr to_gtt_mgr(struct ttm_resource_manager man)
	27	+static inline struct amdgpu_gtt_mgr *
	28	+to_gtt_mgr(struct ttm_resource_manager *man)
28	29	{
29	30	return container_of(man, struct amdgpu_gtt_mgr, manager);
30	31	}
@@ -43,12 +44,14 @@ struct amdgpu_gtt_node { · 12 unmodified lines
43	44	* the GTT block, in bytes
44	45	*/
45	46	static ssize_t amdgpu_mem_info_gtt_total_show(struct device *dev,
46		− struct device_attribute attr, char buf)
	47	+ struct device_attribute *attr,
	48	+ char *buf)
47	49	{
48	50	struct drm_device *ddev = dev_get_drvdata(dev);
49	51	struct amdgpu_device *adev = drm_to_adev(ddev);
50		− struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
	52	+ struct ttm_resource_manager *man;
51	53
	54	+ man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
52	55	return sysfs_emit(buf, "%llu\n", man->size * PAGE_SIZE);
53	56	}
54	57
@@ -61,12 +64,14 @@ static ssize_t amdgpu_mem_info_gtt_total_show(struct device *dev, · 6 unmodified lines
61	64	* size of the GTT block, in bytes
62	65	*/
63	66	static ssize_t amdgpu_mem_info_gtt_used_show(struct device *dev,
64		− struct device_attribute attr, char buf)
	67	+ struct device_attribute *attr,
	68	+ char *buf)
65	69	{
66	70	struct drm_device *ddev = dev_get_drvdata(dev);
67	71	struct amdgpu_device *adev = drm_to_adev(ddev);
68		− struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
	72	+ struct ttm_resource_manager *man;
69	73
	74	+ man = ttm_manager_type(&adev->mman.bdev, TTM_PL_TT);
70	75	return sysfs_emit(buf, "%llu\n", amdgpu_gtt_mgr_usage(man));
71	76	}
72	77
@@ -75,80 +80,6 @@ static DEVICE_ATTR(mem_info_gtt_total, S_IRUGO, · 2 unmodified lines
75	80	static DEVICE_ATTR(mem_info_gtt_used, S_IRUGO,
76	81	amdgpu_mem_info_gtt_used_show, NULL);
77	82
78		−static const struct ttm_resource_manager_func amdgpu_gtt_mgr_func;
79		−/**
80		− * amdgpu_gtt_mgr_init - init GTT manager and DRM MM
81		− *
82		− * @adev: amdgpu_device pointer
83		− * @gtt_size: maximum size of GTT
84		− *
85		− * Allocate and initialize the GTT manager.
86		− */
87		−int amdgpu_gtt_mgr_init(struct amdgpu_device *adev, uint64_t gtt_size)
88		−{
89		− struct amdgpu_gtt_mgr *mgr = &adev->mman.gtt_mgr;
90		− struct ttm_resource_manager *man = &mgr->manager;
91		− uint64_t start, size;
92		− int ret;
93		−
94		− man->use_tt = true;
95		− man->func = &amdgpu_gtt_mgr_func;
96		−
97		− ttm_resource_manager_init(man, gtt_size >> PAGE_SHIFT);
98		−
99		− start = AMDGPU_GTT_MAX_TRANSFER_SIZE * AMDGPU_GTT_NUM_TRANSFER_WINDOWS;
100		− size = (adev->gmc.gart_size >> PAGE_SHIFT) - start;
101		− drm_mm_init(&mgr->mm, start, size);
102		− spin_lock_init(&mgr->lock);
103		− atomic64_set(&mgr->available, gtt_size >> PAGE_SHIFT);
104		−
105		− ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_total);
106		− if (ret) {
107		− DRM_ERROR("Failed to create device file mem_info_gtt_total\n");
108		− return ret;
109		− }
110		− ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_used);
111		− if (ret) {
112		− DRM_ERROR("Failed to create device file mem_info_gtt_used\n");
113		− return ret;
114		− }
115		−
116		− ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_TT, &mgr->manager);
117		− ttm_resource_manager_set_used(man, true);
118		− return 0;
119		−}
120		−
121		−/**
122		− * amdgpu_gtt_mgr_fini - free and destroy GTT manager
123		− *
124		− * @adev: amdgpu_device pointer
125		− *
126		− * Destroy and free the GTT manager, returns -EBUSY if ranges are still
127		− * allocated inside it.
128		− */
129		−void amdgpu_gtt_mgr_fini(struct amdgpu_device *adev)
130		−{
131		− struct amdgpu_gtt_mgr *mgr = &adev->mman.gtt_mgr;
132		− struct ttm_resource_manager *man = &mgr->manager;
133		− int ret;
134		−
135		− ttm_resource_manager_set_used(man, false);
136		−
137		− ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
138		− if (ret)
139		− return;
140		−
141		− spin_lock(&mgr->lock);
142		− drm_mm_takedown(&mgr->mm);
143		− spin_unlock(&mgr->lock);
144		−
145		− device_remove_file(adev->dev, &dev_attr_mem_info_gtt_total);
146		− device_remove_file(adev->dev, &dev_attr_mem_info_gtt_used);
147		−
148		− ttm_resource_manager_cleanup(man);
149		− ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_TT, NULL);
150		−}
151		−
152	83	/**
153	84	* amdgpu_gtt_mgr_has_gart_addr - Check if mem has address space
154	85	*
@@ -265,6 +196,13 @@ uint64_t amdgpu_gtt_mgr_usage(struct ttm_resource_manager *man) · 110 unmodified lines
265	196	return (result > 0 ? result : 0) * PAGE_SIZE;
266	197	}
267	198
	199	+/**
	200	+ * amdgpu_gtt_mgr_recover - re-init gart
	201	+ *
	202	+ * @man: TTM memory type manager
	203	+ *
	204	+ * Re-init the gart for each known BO in the GTT.
	205	+ */
268	206	int amdgpu_gtt_mgr_recover(struct ttm_resource_manager *man)
269	207	{
270	208	struct amdgpu_gtt_mgr *mgr = to_gtt_mgr(man);
@@ -311,3 +249,76 @@ static const struct ttm_resource_manager_func amdgpu_gtt_mgr_func = { · 40 unmodified lines
311	249	.free = amdgpu_gtt_mgr_del,
312	250	.debug = amdgpu_gtt_mgr_debug
313	251	};
	252	+
	253	+/**
	254	+ * amdgpu_gtt_mgr_init - init GTT manager and DRM MM
	255	+ *
	256	+ * @adev: amdgpu_device pointer
	257	+ * @gtt_size: maximum size of GTT
	258	+ *
	259	+ * Allocate and initialize the GTT manager.
	260	+ */
	261	+int amdgpu_gtt_mgr_init(struct amdgpu_device *adev, uint64_t gtt_size)
	262	+{
	263	+ struct amdgpu_gtt_mgr *mgr = &adev->mman.gtt_mgr;
	264	+ struct ttm_resource_manager *man = &mgr->manager;
	265	+ uint64_t start, size;
	266	+ int ret;
	267	+
	268	+ man->use_tt = true;
	269	+ man->func = &amdgpu_gtt_mgr_func;
	270	+
	271	+ ttm_resource_manager_init(man, gtt_size >> PAGE_SHIFT);
	272	+
	273	+ start = AMDGPU_GTT_MAX_TRANSFER_SIZE * AMDGPU_GTT_NUM_TRANSFER_WINDOWS;
	274	+ size = (adev->gmc.gart_size >> PAGE_SHIFT) - start;
	275	+ drm_mm_init(&mgr->mm, start, size);
	276	+ spin_lock_init(&mgr->lock);
	277	+ atomic64_set(&mgr->available, gtt_size >> PAGE_SHIFT);
	278	+
	279	+ ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_total);
	280	+ if (ret) {
	281	+ DRM_ERROR("Failed to create device file mem_info_gtt_total\n");
	282	+ return ret;
	283	+ }
	284	+ ret = device_create_file(adev->dev, &dev_attr_mem_info_gtt_used);
	285	+ if (ret) {
	286	+ DRM_ERROR("Failed to create device file mem_info_gtt_used\n");
	287	+ return ret;
	288	+ }
	289	+
	290	+ ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_TT, &mgr->manager);
	291	+ ttm_resource_manager_set_used(man, true);
	292	+ return 0;
	293	+}
	294	+
	295	+/**
	296	+ * amdgpu_gtt_mgr_fini - free and destroy GTT manager
	297	+ *
	298	+ * @adev: amdgpu_device pointer
	299	+ *
	300	+ * Destroy and free the GTT manager, returns -EBUSY if ranges are still
	301	+ * allocated inside it.
	302	+ */
	303	+void amdgpu_gtt_mgr_fini(struct amdgpu_device *adev)
	304	+{
	305	+ struct amdgpu_gtt_mgr *mgr = &adev->mman.gtt_mgr;
	306	+ struct ttm_resource_manager *man = &mgr->manager;
	307	+ int ret;
	308	+
	309	+ ttm_resource_manager_set_used(man, false);
	310	+
	311	+ ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
	312	+ if (ret)
	313	+ return;
	314	+
	315	+ spin_lock(&mgr->lock);
	316	+ drm_mm_takedown(&mgr->mm);
	317	+ spin_unlock(&mgr->lock);
	318	+
	319	+ device_remove_file(adev->dev, &dev_attr_mem_info_gtt_total);
	320	+ device_remove_file(adev->dev, &dev_attr_mem_info_gtt_used);
	321	+
	322	+ ttm_resource_manager_cleanup(man);
	323	+ ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_TT, NULL);
	324	+}

drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.c+69−0

@@ -0,0 +1,69 @@
	1	+/*
	2	+ * Copyright 2021 Advanced Micro Devices, Inc.
	3	+ *
	4	+ * Permission is hereby granted, free of charge, to any person obtaining a
	5	+ * copy of this software and associated documentation files (the "Software"),
	6	+ * to deal in the Software without restriction, including without limitation
	7	+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	+ * and/or sell copies of the Software, and to permit persons to whom the
	9	+ * Software is furnished to do so, subject to the following conditions:
	10	+ *
	11	+ * The above copyright notice and this permission notice shall be included in
	12	+ * all copies or substantial portions of the Software.
	13	+ *
	14	+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	+ * OTHER DEALINGS IN THE SOFTWARE.
	21	+ *
	22	+ */
	23	+
	24	+#include "amdgpu.h"
	25	+#include "amdgpu_ras.h"
	26	+
	27	+int amdgpu_hdp_ras_late_init(struct amdgpu_device *adev)
	28	+{
	29	+ int r;
	30	+ struct ras_ih_if ih_info = {
	31	+ .cb = NULL,
	32	+ };
	33	+ struct ras_fs_if fs_info = {
	34	+ .sysfs_name = "hdp_err_count",
	35	+ };
	36	+
	37	+ if (!adev->hdp.ras_if) {
	38	+ adev->hdp.ras_if = kmalloc(sizeof(struct ras_common_if), GFP_KERNEL);
	39	+ if (!adev->hdp.ras_if)
	40	+ return -ENOMEM;
	41	+ adev->hdp.ras_if->block = AMDGPU_RAS_BLOCK__HDP;
	42	+ adev->hdp.ras_if->type = AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE;
	43	+ adev->hdp.ras_if->sub_block_index = 0;
	44	+ strcpy(adev->hdp.ras_if->name, "hdp");
	45	+ }
	46	+ ih_info.head = fs_info.head = *adev->hdp.ras_if;
	47	+ r = amdgpu_ras_late_init(adev, adev->hdp.ras_if,
	48	+ &fs_info, &ih_info);
	49	+ if (r \|\| !amdgpu_ras_is_supported(adev, adev->hdp.ras_if->block)) {
	50	+ kfree(adev->hdp.ras_if);
	51	+ adev->hdp.ras_if = NULL;
	52	+ }
	53	+
	54	+ return r;
	55	+}
	56	+
	57	+void amdgpu_hdp_ras_fini(struct amdgpu_device *adev)
	58	+{
	59	+ if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP) &&
	60	+ adev->hdp.ras_if) {
	61	+ struct ras_common_if *ras_if = adev->hdp.ras_if;
	62	+ struct ras_ih_if ih_info = {
	63	+ .cb = NULL,
	64	+ };
	65	+
	66	+ amdgpu_ras_late_fini(adev, ras_if, &ih_info);
	67	+ kfree(ras_if);
	68	+ }
	69	+}

drivers/gpu/drm/amd/amdgpu/amdgpu_hdp.h+12−1

@@ -23,18 +23,29 @@ · 22 unmodified lines
23	23	#ifndef __AMDGPU_HDP_H__
24	24	#define __AMDGPU_HDP_H__
25	25
	26	+struct amdgpu_hdp_ras_funcs {
	27	+ int (ras_late_init)(struct amdgpu_device adev);
	28	+ void (ras_fini)(struct amdgpu_device adev);
	29	+ void (query_ras_error_count)(struct amdgpu_device adev,
	30	+ void *ras_error_status);
	31	+ void (reset_ras_error_count)(struct amdgpu_device adev);
	32	+};
	33	+
26	34	struct amdgpu_hdp_funcs {
27	35	void (flush_hdp)(struct amdgpu_device adev, struct amdgpu_ring *ring);
28	36	void (invalidate_hdp)(struct amdgpu_device adev,
29	37	struct amdgpu_ring *ring);
30		− void (reset_ras_error_count)(struct amdgpu_device adev);
31	38	void (update_clock_gating)(struct amdgpu_device adev, bool enable);
32	39	void (get_clock_gating_state)(struct amdgpu_device adev, u32 *flags);
33	40	void (init_registers)(struct amdgpu_device adev);
34	41	};
35	42
36	43	struct amdgpu_hdp {
	44	+ struct ras_common_if *ras_if;
37	45	const struct amdgpu_hdp_funcs *funcs;
	46	+ const struct amdgpu_hdp_ras_funcs *ras_funcs;
38	47	};
39	48
	49	+int amdgpu_hdp_ras_late_init(struct amdgpu_device *adev);
	50	+void amdgpu_hdp_ras_fini(struct amdgpu_device *adev);
40	51	#endif /* __AMDGPU_HDP_H__ */

drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c+1−1

@@ -328,7 +328,7 @@ int amdgpu_ib_pool_init(struct amdgpu_device *adev) · 327 unmodified lines
328	328
329	329	for (i = 0; i < AMDGPU_IB_POOL_MAX; i++) {
330	330	if (i == AMDGPU_IB_POOL_DIRECT)
331		− size = PAGE_SIZE * 2;
	331	+ size = PAGE_SIZE * 6;
332	332	else
333	333	size = AMDGPU_IB_POOL_SIZE;
334	334

drivers/gpu/drm/amd/amdgpu/amdgpu_ih.c+3−1

@@ -175,7 +175,9 @@ static bool amdgpu_ih_has_checkpoint_processed(struct amdgpu_device *adev, · 174 unmodified lines
175	175	cur_rptr += ih->ptr_mask + 1;
176	176	*prev_rptr = cur_rptr;
177	177
178		− return cur_rptr >= checkpoint_wptr;
	178	+ /* check ring is empty to workaround missing wptr overflow flag */
	179	+ return cur_rptr >= checkpoint_wptr \|\|
	180	+ (cur_rptr & ih->ptr_mask) == amdgpu_ih_get_wptr(adev, ih);
179	181	}
180	182
181	183	/**

drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c+3−2

@@ -986,7 +986,7 @@ int amdgpu_info_ioctl(struct drm_device dev, void data, struct drm_file *filp) · 985 unmodified lines
986	986
987	987	if (!ras)
988	988	return -EINVAL;
989		− ras_mask = (uint64_t)ras->supported << 32 \| ras->features;
	989	+ ras_mask = (uint64_t)adev->ras_enabled << 32 \| ras->features;
990	990
991	991	return copy_to_user(out, &ras_mask,
992	992	min_t(u64, size, sizeof(ras_mask))) ?
@@ -1114,7 +1114,8 @@ int amdgpu_driver_open_kms(struct drm_device dev, struct drm_file file_priv) · 121 unmodified lines
1114	1114	dev_warn(adev->dev, "No more PASIDs available!");
1115	1115	pasid = 0;
1116	1116	}
1117		− r = amdgpu_vm_init(adev, &fpriv->vm, AMDGPU_VM_CONTEXT_GFX, pasid);
	1117	+
	1118	+ r = amdgpu_vm_init(adev, &fpriv->vm, pasid);
1118	1119	if (r)
1119	1120	goto error_pasid;
1120	1121

drivers/gpu/drm/amd/amdgpu/amdgpu_mmhub.h+1−0

@@ -28,6 +28,7 @@ struct amdgpu_mmhub_ras_funcs { · 27 unmodified lines
28	28	void *ras_error_status);
29	29	void (query_ras_error_status)(struct amdgpu_device adev);
30	30	void (reset_ras_error_count)(struct amdgpu_device adev);
	31	+ void (reset_ras_error_status)(struct amdgpu_device adev);
31	32	};
32	33
33	34	struct amdgpu_mmhub_funcs {

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.c+86−0

@@ -155,3 +155,89 @@ void amdgpu_mn_unregister(struct amdgpu_bo *bo) · 154 unmodified lines
155	155	mmu_interval_notifier_remove(&bo->notifier);
156	156	bo->notifier.mm = NULL;
157	157	}
	158	+
	159	+int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier,
	160	+ struct mm_struct mm, struct page *pages,
	161	+ uint64_t start, uint64_t npages,
	162	+ struct hmm_range **phmm_range, bool readonly,
	163	+ bool mmap_locked)
	164	+{
	165	+ struct hmm_range *hmm_range;
	166	+ unsigned long timeout;
	167	+ unsigned long i;
	168	+ unsigned long *pfns;
	169	+ int r = 0;
	170	+
	171	+ hmm_range = kzalloc(sizeof(*hmm_range), GFP_KERNEL);
	172	+ if (unlikely(!hmm_range))
	173	+ return -ENOMEM;
	174	+
	175	+ pfns = kvmalloc_array(npages, sizeof(*pfns), GFP_KERNEL);
	176	+ if (unlikely(!pfns)) {
	177	+ r = -ENOMEM;
	178	+ goto out_free_range;
	179	+ }
	180	+
	181	+ hmm_range->notifier = notifier;
	182	+ hmm_range->default_flags = HMM_PFN_REQ_FAULT;
	183	+ if (!readonly)
	184	+ hmm_range->default_flags \|= HMM_PFN_REQ_WRITE;
	185	+ hmm_range->hmm_pfns = pfns;
	186	+ hmm_range->start = start;
	187	+ hmm_range->end = start + npages * PAGE_SIZE;
	188	+
	189	+ /* Assuming 512MB takes maxmium 1 second to fault page address */
	190	+ timeout = max(npages >> 17, 1ULL) * HMM_RANGE_DEFAULT_TIMEOUT;
	191	+ timeout = jiffies + msecs_to_jiffies(timeout);
	192	+
	193	+retry:
	194	+ hmm_range->notifier_seq = mmu_interval_read_begin(notifier);
	195	+
	196	+ if (likely(!mmap_locked))
	197	+ mmap_read_lock(mm);
	198	+
	199	+ r = hmm_range_fault(hmm_range);
	200	+
	201	+ if (likely(!mmap_locked))
	202	+ mmap_read_unlock(mm);
	203	+ if (unlikely(r)) {
	204	+ /*
	205	+ * FIXME: This timeout should encompass the retry from
	206	+ * mmu_interval_read_retry() as well.
	207	+ */
	208	+ if (r == -EBUSY && !time_after(jiffies, timeout))
	209	+ goto retry;
	210	+ goto out_free_pfns;
	211	+ }
	212	+
	213	+ /*
	214	+ * Due to default_flags, all pages are HMM_PFN_VALID or
	215	+ * hmm_range_fault() fails. FIXME: The pages cannot be touched outside
	216	+ * the notifier_lock, and mmu_interval_read_retry() must be done first.
	217	+ */
	218	+ for (i = 0; pages && i < npages; i++)
	219	+ pages[i] = hmm_pfn_to_page(pfns[i]);
	220	+
	221	+ *phmm_range = hmm_range;
	222	+
	223	+ return 0;
	224	+
	225	+out_free_pfns:
	226	+ kvfree(pfns);
	227	+out_free_range:
	228	+ kfree(hmm_range);
	229	+
	230	+ return r;
	231	+}
	232	+
	233	+int amdgpu_hmm_range_get_pages_done(struct hmm_range *hmm_range)
	234	+{
	235	+ int r;
	236	+
	237	+ r = mmu_interval_read_retry(hmm_range->notifier,
	238	+ hmm_range->notifier_seq);
	239	+ kvfree(hmm_range->hmm_pfns);
	240	+ kfree(hmm_range);
	241	+
	242	+ return r;
	243	+}

drivers/gpu/drm/amd/amdgpu/amdgpu_mn.h+7−0

@@ -30,6 +30,13 @@ · 29 unmodified lines
30	30	#include <linux/workqueue.h>
31	31	#include <linux/interval_tree.h>
32	32
	33	+int amdgpu_hmm_range_get_pages(struct mmu_interval_notifier *notifier,
	34	+ struct mm_struct mm, struct page *pages,
	35	+ uint64_t start, uint64_t npages,
	36	+ struct hmm_range **phmm_range, bool readonly,
	37	+ bool mmap_locked);
	38	+int amdgpu_hmm_range_get_pages_done(struct hmm_range *hmm_range);
	39	+
33	40	#if defined(CONFIG_HMM_MIRROR)
34	41	int amdgpu_mn_register(struct amdgpu_bo *bo, unsigned long addr);
35	42	void amdgpu_mn_unregister(struct amdgpu_bo *bo);

drivers/gpu/drm/amd/amdgpu/amdgpu_object.c+18−54

@@ -491,7 +491,18 @@ bool amdgpu_bo_support_uswc(u64 bo_flags) · 490 unmodified lines
491	491	#endif
492	492	}
493	493
494		−static int amdgpu_bo_do_create(struct amdgpu_device *adev,
	494	+/**
	495	+ * amdgpu_bo_create - create an &amdgpu_bo buffer object
	496	+ * @adev: amdgpu device object
	497	+ * @bp: parameters to be used for the buffer object
	498	+ * @bo_ptr: pointer to the buffer object pointer
	499	+ *
	500	+ * Creates an &amdgpu_bo buffer object.
	501	+ *
	502	+ * Returns:
	503	+ * 0 for success or a negative error code on failure.
	504	+ */
	505	+int amdgpu_bo_create(struct amdgpu_device *adev,
495	506	struct amdgpu_bo_param *bp,
496	507	struct amdgpu_bo **bo_ptr)
497	508	{
@@ -601,9 +612,9 @@ fail_unreserve: · 103 unmodified lines
601	612	return r;
602	613	}
603	614
604		−static int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
605		− unsigned long size,
606		− struct amdgpu_bo *bo)
	615	+int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
	616	+ unsigned long size,
	617	+ struct amdgpu_bo *bo)
607	618	{
608	619	struct amdgpu_bo_param bp;
609	620	int r;
@@ -614,13 +625,12 @@ static int amdgpu_bo_create_shadow(struct amdgpu_device *adev, · 4 unmodified lines
614	625	memset(&bp, 0, sizeof(bp));
615	626	bp.size = size;
616	627	bp.domain = AMDGPU_GEM_DOMAIN_GTT;
617		− bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC \|
618		− AMDGPU_GEM_CREATE_SHADOW;
	628	+ bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC;
619	629	bp.type = ttm_bo_type_kernel;
620	630	bp.resv = bo->tbo.base.resv;
621	631	bp.bo_ptr_size = sizeof(struct amdgpu_bo);
622	632
623		− r = amdgpu_bo_do_create(adev, &bp, &bo->shadow);
	633	+ r = amdgpu_bo_create(adev, &bp, &bo->shadow);
624	634	if (!r) {
625	635	bo->shadow->parent = amdgpu_bo_ref(bo);
626	636	mutex_lock(&adev->shadow_list_lock);
@@ -631,50 +641,6 @@ static int amdgpu_bo_create_shadow(struct amdgpu_device *adev, · 4 unmodified lines
631	641	return r;
632	642	}
633	643
634		−/**
635		− * amdgpu_bo_create - create an &amdgpu_bo buffer object
636		− * @adev: amdgpu device object
637		− * @bp: parameters to be used for the buffer object
638		− * @bo_ptr: pointer to the buffer object pointer
639		− *
640		− * Creates an &amdgpu_bo buffer object; and if requested, also creates a
641		− * shadow object.
642		− * Shadow object is used to backup the original buffer object, and is always
643		− * in GTT.
644		− *
645		− * Returns:
646		− * 0 for success or a negative error code on failure.
647		− */
648		−int amdgpu_bo_create(struct amdgpu_device *adev,
649		− struct amdgpu_bo_param *bp,
650		− struct amdgpu_bo **bo_ptr)
651		−{
652		− u64 flags = bp->flags;
653		− int r;
654		−
655		− bp->flags = bp->flags & ~AMDGPU_GEM_CREATE_SHADOW;
656		−
657		− r = amdgpu_bo_do_create(adev, bp, bo_ptr);
658		− if (r)
659		− return r;
660		−
661		− if ((flags & AMDGPU_GEM_CREATE_SHADOW) && !(adev->flags & AMD_IS_APU)) {
662		− if (!bp->resv)
663		− WARN_ON(dma_resv_lock((*bo_ptr)->tbo.base.resv,
664		− NULL));
665		−
666		− r = amdgpu_bo_create_shadow(adev, bp->size, *bo_ptr);
667		−
668		− if (!bp->resv)
669		− dma_resv_unlock((*bo_ptr)->tbo.base.resv);
670		−
671		− if (r)
672		− amdgpu_bo_unref(bo_ptr);
673		− }
674		−
675		− return r;
676		−}
677		−
678	644	/**
679	645	* amdgpu_bo_create_user - create an &amdgpu_bo_user buffer object
680	646	* @adev: amdgpu device object
@@ -694,9 +660,8 @@ int amdgpu_bo_create_user(struct amdgpu_device *adev, · 13 unmodified lines
694	660	struct amdgpu_bo *bo_ptr;
695	661	int r;
696	662
697		− bp->flags = bp->flags & ~AMDGPU_GEM_CREATE_SHADOW;
698	663	bp->bo_ptr_size = sizeof(struct amdgpu_bo_user);
699		− r = amdgpu_bo_do_create(adev, bp, &bo_ptr);
	664	+ r = amdgpu_bo_create(adev, bp, &bo_ptr);
700	665	if (r)
701	666	return r;
702	667
@@ -1595,7 +1560,6 @@ u64 amdgpu_bo_print_info(int id, struct amdgpu_bo bo, struct seq_file m) · 892 unmodified lines
1595	1560	amdgpu_bo_print_flag(m, bo, NO_CPU_ACCESS);
1596	1561	amdgpu_bo_print_flag(m, bo, CPU_GTT_USWC);
1597	1562	amdgpu_bo_print_flag(m, bo, VRAM_CLEARED);
1598		− amdgpu_bo_print_flag(m, bo, SHADOW);
1599	1563	amdgpu_bo_print_flag(m, bo, VRAM_CONTIGUOUS);
1600	1564	amdgpu_bo_print_flag(m, bo, VM_ALWAYS_VALID);
1601	1565	amdgpu_bo_print_flag(m, bo, EXPLICIT_SYNC);

drivers/gpu/drm/amd/amdgpu/amdgpu_object.h+7−0

@@ -37,6 +37,10 @@ · 36 unmodified lines
37	37	#define AMDGPU_BO_INVALID_OFFSET LONG_MAX
38	38	#define AMDGPU_BO_MAX_PLACEMENTS 3
39	39
	40	+/* BO flag to indicate a KFD userptr BO */
	41	+#define AMDGPU_AMDKFD_CREATE_USERPTR_BO (1ULL << 63)
	42	+#define AMDGPU_AMDKFD_CREATE_SVM_BO (1ULL << 62)
	43	+
40	44	#define to_amdgpu_bo_user(abo) container_of((abo), struct amdgpu_bo_user, bo)
41	45
42	46	struct amdgpu_bo_param {
@@ -267,6 +271,9 @@ int amdgpu_bo_create_user(struct amdgpu_device *adev, · 224 unmodified lines
267	271	struct amdgpu_bo_user **ubo_ptr);
268	272	void amdgpu_bo_free_kernel(struct amdgpu_bo *bo, u64 gpu_addr,
269	273	void **cpu_addr);
	274	+int amdgpu_bo_create_shadow(struct amdgpu_device *adev,
	275	+ unsigned long size,
	276	+ struct amdgpu_bo *bo);
270	277	int amdgpu_bo_kmap(struct amdgpu_bo bo, void *ptr);
271	278	void amdgpu_bo_kptr(struct amdgpu_bo bo);
272	279	void amdgpu_bo_kunmap(struct amdgpu_bo *bo);

drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c+36−37

@@ -417,31 +417,12 @@ static int psp_tmr_init(struct psp_context *psp) · 416 unmodified lines
417	417	return ret;
418	418	}
419	419
420		−static int psp_clear_vf_fw(struct psp_context *psp)
421		−{
422		− int ret;
423		− struct psp_gfx_cmd_resp *cmd;
424		−
425		− if (!amdgpu_sriov_vf(psp->adev) \|\| psp->adev->asic_type != CHIP_NAVI12)
426		− return 0;
427		−
428		− cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
429		− if (!cmd)
430		− return -ENOMEM;
431		−
432		− cmd->cmd_id = GFX_CMD_ID_CLEAR_VF_FW;
433		−
434		− ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
435		− kfree(cmd);
436		−
437		− return ret;
438		−}
439		−
440	420	static bool psp_skip_tmr(struct psp_context *psp)
441	421	{
442	422	switch (psp->adev->asic_type) {
443	423	case CHIP_NAVI12:
444	424	case CHIP_SIENNA_CICHLID:
	425	+ case CHIP_ALDEBARAN:
445	426	return true;
446	427	default:
447	428	return false;
@@ -1037,6 +1018,13 @@ static int psp_ras_load(struct psp_context *psp) · 589 unmodified lines
1037	1018	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
1038	1019	memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
1039	1020
	1021	+ ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
	1022	+
	1023	+ if (psp->adev->gmc.xgmi.connected_to_cpu)
	1024	+ ras_cmd->ras_in_message.init_flags.poison_mode_en = 1;
	1025	+ else
	1026	+ ras_cmd->ras_in_message.init_flags.dgpu_mode = 1;
	1027	+
1040	1028	psp_prep_ta_load_cmd_buf(cmd,
1041	1029	psp->fw_pri_mc_addr,
1042	1030	psp->ta_ras_ucode_size,
@@ -1046,8 +1034,6 @@ static int psp_ras_load(struct psp_context *psp) · 3 unmodified lines
1046	1034	ret = psp_cmd_submit_buf(psp, NULL, cmd,
1047	1035	psp->fence_buf_mc_addr);
1048	1036
1049		− ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
1050		−
1051	1037	if (!ret) {
1052	1038	psp->ras.session_id = cmd->resp.session_id;
1053	1039
@@ -1128,6 +1114,31 @@ int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id) · 74 unmodified lines
1128	1114	return ret;
1129	1115	}
1130	1116
	1117	+static int psp_ras_status_to_errno(struct amdgpu_device *adev,
	1118	+ enum ta_ras_status ras_status)
	1119	+{
	1120	+ int ret = -EINVAL;
	1121	+
	1122	+ switch (ras_status) {
	1123	+ case TA_RAS_STATUS__SUCCESS:
	1124	+ ret = 0;
	1125	+ break;
	1126	+ case TA_RAS_STATUS__RESET_NEEDED:
	1127	+ ret = -EAGAIN;
	1128	+ break;
	1129	+ case TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE:
	1130	+ dev_warn(adev->dev, "RAS WARN: ras function unavailable\n");
	1131	+ break;
	1132	+ case TA_RAS_STATUS__ERROR_ASD_READ_WRITE:
	1133	+ dev_warn(adev->dev, "RAS WARN: asd read or write failed\n");
	1134	+ break;
	1135	+ default:
	1136	+ dev_err(adev->dev, "RAS ERROR: ras function failed ret 0x%X\n", ret);
	1137	+ }
	1138	+
	1139	+ return ret;
	1140	+}
	1141	+
1131	1142	int psp_ras_enable_features(struct psp_context *psp,
1132	1143	union ta_ras_cmd_input *info, bool enable)
1133	1144	{
@@ -1151,7 +1162,7 @@ int psp_ras_enable_features(struct psp_context *psp, · 17 unmodified lines
1151	1162	if (ret)
1152	1163	return -EINVAL;
1153	1164
1154		− return ras_cmd->ras_status;
	1165	+ return psp_ras_status_to_errno(psp->adev, ras_cmd->ras_status);
1155	1166	}
1156	1167
1157	1168	static int psp_ras_terminate(struct psp_context *psp)
@@ -1234,7 +1245,7 @@ int psp_ras_trigger_error(struct psp_context *psp, · 76 unmodified lines
1234	1245	if (amdgpu_ras_intr_triggered())
1235	1246	return 0;
1236	1247
1237		− return ras_cmd->ras_status;
	1248	+ return psp_ras_status_to_errno(psp->adev, ras_cmd->ras_status);
1238	1249	}
1239	1250	// ras end
1240	1251
@@ -1920,12 +1931,6 @@ static int psp_hw_start(struct psp_context *psp) · 679 unmodified lines
1920	1931	return ret;
1921	1932	}
1922	1933
1923		− ret = psp_clear_vf_fw(psp);
1924		− if (ret) {
1925		− DRM_ERROR("PSP clear vf fw!\n");
1926		− return ret;
1927		− }
1928		−
1929	1934	ret = psp_boot_config_set(adev);
1930	1935	if (ret) {
1931	1936	DRM_WARN("PSP set boot config@\n");
@@ -2166,7 +2171,7 @@ static int psp_load_smu_fw(struct psp_context *psp) · 234 unmodified lines
2166	2171	return 0;
2167	2172
2168	2173	if ((amdgpu_in_reset(adev) &&
2169		− ras && ras->supported &&
	2174	+ ras && adev->ras_enabled &&
2170	2175	(adev->asic_type == CHIP_ARCTURUS \|\|
2171	2176	adev->asic_type == CHIP_VEGA20)) \|\|
2172	2177	(adev->in_runpm &&
@@ -2434,7 +2439,6 @@ static int psp_hw_fini(void *handle) · 261 unmodified lines
2434	2439	{
2435	2440	struct amdgpu_device adev = (struct amdgpu_device )handle;
2436	2441	struct psp_context *psp = &adev->psp;
2437		− int ret;
2438	2442
2439	2443	if (psp->adev->psp.ta_fw) {
2440	2444	psp_ras_terminate(psp);
@@ -2445,11 +2449,6 @@ static int psp_hw_fini(void *handle) · 4 unmodified lines
2445	2449	}
2446	2450
2447	2451	psp_asd_unload(psp);
2448		− ret = psp_clear_vf_fw(psp);
2449		− if (ret) {
2450		− DRM_ERROR("PSP clear vf fw!\n");
2451		− return ret;
2452		− }
2453	2452
2454	2453	psp_tmr_terminate(psp);
2455	2454	psp_ring_destroy(psp, PSP_RING_TYPE__KM);

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.c+114−85

@@ -33,6 +33,7 @@ · 32 unmodified lines
33	33	#include "amdgpu_atomfirmware.h"
34	34	#include "amdgpu_xgmi.h"
35	35	#include "ivsrcid/nbio/irqsrcs_nbif_7_4.h"
	36	+#include "atom.h"
36	37
37	38	static const char *RAS_FS_NAME = "ras";
38	39
@@ -320,11 +321,14 @@ static int amdgpu_ras_debugfs_ctrl_parse_data(struct file *f, · 281 unmodified lines
320	321	* "disable" requires only the block.
321	322	* "enable" requires the block and error type.
322	323	* "inject" requires the block, error type, address, and value.
	324	+ *
323	325	* The block is one of: umc, sdma, gfx, etc.
324	326	* see ras_block_string[] for details
	327	+ *
325	328	* The error type is one of: ue, ce, where,
326	329	* ue is multi-uncorrectable
327	330	* ce is single-correctable
	331	+ *
328	332	* The sub-block is a the sub-block index, pass 0 if there is no sub-block.
329	333	* The address and value are hexadecimal numbers, leading 0x is optional.
330	334	*
@@ -531,7 +535,7 @@ static struct ras_manager amdgpu_ras_create_obj(struct amdgpu_device adev, · 200 unmodified lines
531	535	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
532	536	struct ras_manager *obj;
533	537
534		− if (!adev->ras_features \|\| !con)
	538	+ if (!adev->ras_enabled \|\| !con)
535	539	return NULL;
536	540
537	541	if (head->block >= AMDGPU_RAS_BLOCK_COUNT)
@@ -558,7 +562,7 @@ struct ras_manager amdgpu_ras_find_obj(struct amdgpu_device adev, · 20 unmodified lines
558	562	struct ras_manager *obj;
559	563	int i;
560	564
561		− if (!adev->ras_features \|\| !con)
	565	+ if (!adev->ras_enabled \|\| !con)
562	566	return NULL;
563	567
564	568	if (head) {
@@ -585,36 +589,11 @@ struct ras_manager amdgpu_ras_find_obj(struct amdgpu_device adev, · 20 unmodified lines
585	589	}
586	590	/* obj end */
587	591
588		−static void amdgpu_ras_parse_status_code(struct amdgpu_device *adev,
589		− const char* invoke_type,
590		− const char* block_name,
591		− enum ta_ras_status ret)
592		−{
593		− switch (ret) {
594		− case TA_RAS_STATUS__SUCCESS:
595		− return;
596		− case TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE:
597		− dev_warn(adev->dev,
598		− "RAS WARN: %s %s currently unavailable\n",
599		− invoke_type,
600		− block_name);
601		− break;
602		− default:
603		− dev_err(adev->dev,
604		− "RAS ERROR: %s %s error failed ret 0x%X\n",
605		− invoke_type,
606		− block_name,
607		− ret);
608		− }
609		−}
610		−
611	592	/* feature ctl begin */
612	593	static int amdgpu_ras_is_feature_allowed(struct amdgpu_device *adev,
613		− struct ras_common_if *head)
	594	+ struct ras_common_if *head)
614	595	{
615		− struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
616		−
617		− return con->hw_supported & BIT(head->block);
	596	+ return adev->ras_hw_enabled & BIT(head->block);
618	597	}
619	598
620	599	static int amdgpu_ras_is_feature_enabled(struct amdgpu_device *adev,
@@ -658,11 +637,7 @@ static int __amdgpu_ras_feature_enable(struct amdgpu_device *adev, · 37 unmodified lines
658	637	con->features \|= BIT(head->block);
659	638	} else {
660	639	if (obj && amdgpu_ras_is_feature_enabled(adev, head)) {
661		− /* skip clean gfx ras context feature for VEGA20 Gaming.
662		− * will clean later
663		− */
664		− if (!(!adev->ras_features && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)))
665		− con->features &= ~BIT(head->block);
	640	+ con->features &= ~BIT(head->block);
666	641	put_obj(obj);
667	642	}
668	643	}
@@ -708,15 +683,10 @@ int amdgpu_ras_feature_enable(struct amdgpu_device *adev, · 39 unmodified lines
708	683	if (!amdgpu_ras_intr_triggered()) {
709	684	ret = psp_ras_enable_features(&adev->psp, info, enable);
710	685	if (ret) {
711		− amdgpu_ras_parse_status_code(adev,
712		− enable ? "enable":"disable",
713		− ras_block_str(head->block),
714		− (enum ta_ras_status)ret);
715		− if (ret == TA_RAS_STATUS__RESET_NEEDED)
716		− ret = -EAGAIN;
717		− else
718		− ret = -EINVAL;
719		−
	686	+ dev_err(adev->dev, "ras %s %s failed %d\n",
	687	+ enable ? "enable":"disable",
	688	+ ras_block_str(head->block),
	689	+ ret);
720	690	goto out;
721	691	}
722	692	}
@@ -770,6 +740,10 @@ int amdgpu_ras_feature_enable_on_boot(struct amdgpu_device *adev, · 47 unmodified lines
770	740	con->features \|= BIT(head->block);
771	741
772	742	ret = amdgpu_ras_feature_enable(adev, head, 0);
	743	+
	744	+ /* clean gfx block ras features flag */
	745	+ if (adev->ras_enabled && head->block == AMDGPU_RAS_BLOCK__GFX)
	746	+ con->features &= ~BIT(head->block);
773	747	}
774	748	} else
775	749	ret = amdgpu_ras_feature_enable(adev, head, enable);
@@ -890,6 +864,11 @@ int amdgpu_ras_query_error_status(struct amdgpu_device *adev, · 114 unmodified lines
890	864	adev->gmc.xgmi.ras_funcs->query_ras_error_count)
891	865	adev->gmc.xgmi.ras_funcs->query_ras_error_count(adev, &err_data);
892	866	break;
	867	+ case AMDGPU_RAS_BLOCK__HDP:
	868	+ if (adev->hdp.ras_funcs &&
	869	+ adev->hdp.ras_funcs->query_ras_error_count)
	870	+ adev->hdp.ras_funcs->query_ras_error_count(adev, &err_data);
	871	+ break;
893	872	default:
894	873	break;
895	874	}
@@ -901,17 +880,42 @@ int amdgpu_ras_query_error_status(struct amdgpu_device *adev, · 5 unmodified lines
901	880	info->ce_count = obj->err_data.ce_count;
902	881
903	882	if (err_data.ce_count) {
904		− dev_info(adev->dev, "%ld correctable hardware errors "
	883	+ if (adev->smuio.funcs &&
	884	+ adev->smuio.funcs->get_socket_id &&
	885	+ adev->smuio.funcs->get_die_id) {
	886	+ dev_info(adev->dev, "socket: %d, die: %d "
	887	+ "%ld correctable hardware errors "
	888	+ "detected in %s block, no user "
	889	+ "action is needed.\n",
	890	+ adev->smuio.funcs->get_socket_id(adev),
	891	+ adev->smuio.funcs->get_die_id(adev),
	892	+ obj->err_data.ce_count,
	893	+ ras_block_str(info->head.block));
	894	+ } else {
	895	+ dev_info(adev->dev, "%ld correctable hardware errors "
905	896	"detected in %s block, no user "
906	897	"action is needed.\n",
907	898	obj->err_data.ce_count,
908	899	ras_block_str(info->head.block));
	900	+ }
909	901	}
910	902	if (err_data.ue_count) {
911		− dev_info(adev->dev, "%ld uncorrectable hardware errors "
	903	+ if (adev->smuio.funcs &&
	904	+ adev->smuio.funcs->get_socket_id &&
	905	+ adev->smuio.funcs->get_die_id) {
	906	+ dev_info(adev->dev, "socket: %d, die: %d "
	907	+ "%ld uncorrectable hardware errors "
	908	+ "detected in %s block\n",
	909	+ adev->smuio.funcs->get_socket_id(adev),
	910	+ adev->smuio.funcs->get_die_id(adev),
	911	+ obj->err_data.ue_count,
	912	+ ras_block_str(info->head.block));
	913	+ } else {
	914	+ dev_info(adev->dev, "%ld uncorrectable hardware errors "
912	915	"detected in %s block\n",
913	916	obj->err_data.ue_count,
914	917	ras_block_str(info->head.block));
	918	+ }
915	919	}
916	920
917	921	return 0;
@@ -937,11 +941,20 @@ int amdgpu_ras_reset_error_status(struct amdgpu_device *adev, · 19 unmodified lines
937	941	if (adev->mmhub.ras_funcs &&
938	942	adev->mmhub.ras_funcs->reset_ras_error_count)
939	943	adev->mmhub.ras_funcs->reset_ras_error_count(adev);
	944	+
	945	+ if (adev->mmhub.ras_funcs &&
	946	+ adev->mmhub.ras_funcs->reset_ras_error_status)
	947	+ adev->mmhub.ras_funcs->reset_ras_error_status(adev);
940	948	break;
941	949	case AMDGPU_RAS_BLOCK__SDMA:
942	950	if (adev->sdma.funcs->reset_ras_error_count)
943	951	adev->sdma.funcs->reset_ras_error_count(adev);
944	952	break;
	953	+ case AMDGPU_RAS_BLOCK__HDP:
	954	+ if (adev->hdp.ras_funcs &&
	955	+ adev->hdp.ras_funcs->reset_ras_error_count)
	956	+ adev->hdp.ras_funcs->reset_ras_error_count(adev);
	957	+ break;
945	958	default:
946	959	break;
947	960	}
@@ -1022,10 +1035,9 @@ int amdgpu_ras_error_inject(struct amdgpu_device *adev, · 74 unmodified lines
1022	1035	ret = -EINVAL;
1023	1036	}
1024	1037
1025		− amdgpu_ras_parse_status_code(adev,
1026		− "inject",
1027		− ras_block_str(info->head.block),
1028		− (enum ta_ras_status)ret);
	1038	+ if (ret)
	1039	+ dev_err(adev->dev, "ras inject %s failed %d\n",
	1040	+ ras_block_str(info->head.block), ret);
1029	1041
1030	1042	return ret;
1031	1043	}
@@ -1038,7 +1050,7 @@ unsigned long amdgpu_ras_query_error_count(struct amdgpu_device *adev, · 6 unmodified lines
1038	1050	struct ras_manager *obj;
1039	1051	struct ras_err_data data = {0, 0};
1040	1052
1041		− if (!adev->ras_features \|\| !con)
	1053	+ if (!adev->ras_enabled \|\| !con)
1042	1054	return 0;
1043	1055
1044	1056	list_for_each_entry(obj, &con->head, node) {
@@ -1265,8 +1277,8 @@ static int amdgpu_ras_sysfs_remove_all(struct amdgpu_device *adev) · 220 unmodified lines
1265	1277	static struct dentry amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device adev)
1266	1278	{
1267	1279	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1268		− struct dentry *dir;
1269		− struct drm_minor *minor = adev_to_drm(adev)->primary;
	1280	+ struct drm_minor *minor = adev_to_drm(adev)->primary;
	1281	+ struct dentry *dir;
1270	1282
1271	1283	dir = debugfs_create_dir(RAS_FS_NAME, minor->debugfs_root);
1272	1284	debugfs_create_file("ras_ctrl", S_IWUGO \| S_IRUGO, dir, adev,
@@ -1275,6 +1287,8 @@ static struct dentry amdgpu_ras_debugfs_create_ctrl_node(struct amdgpu_device · 2 unmodified lines
1275	1287	&amdgpu_ras_debugfs_eeprom_ops);
1276	1288	debugfs_create_u32("bad_page_cnt_threshold", 0444, dir,
1277	1289	&con->bad_page_cnt_threshold);
	1290	+ debugfs_create_x32("ras_hw_enabled", 0444, dir, &adev->ras_hw_enabled);
	1291	+ debugfs_create_x32("ras_enabled", 0444, dir, &adev->ras_enabled);
1278	1292
1279	1293	/*
1280	1294	* After one uncorrectable error happens, usually GPU recovery will
@@ -1561,7 +1575,7 @@ static void amdgpu_ras_log_on_err_counter(struct amdgpu_device *adev) · 280 unmodified lines
1561	1575	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1562	1576	struct ras_manager *obj;
1563	1577
1564		− if (!adev->ras_features \|\| !con)
	1578	+ if (!adev->ras_enabled \|\| !con)
1565	1579	return;
1566	1580
1567	1581	list_for_each_entry(obj, &con->head, node) {
@@ -1611,7 +1625,7 @@ static void amdgpu_ras_query_err_status(struct amdgpu_device *adev) · 43 unmodified lines
1611	1625	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
1612	1626	struct ras_manager *obj;
1613	1627
1614		− if (!adev->ras_features \|\| !con)
	1628	+ if (!adev->ras_enabled \|\| !con)
1615	1629	return;
1616	1630
1617	1631	list_for_each_entry(obj, &con->head, node) {
@@ -1925,7 +1939,7 @@ int amdgpu_ras_recovery_init(struct amdgpu_device *adev) · 307 unmodified lines
1925	1939	bool exc_err_limit = false;
1926	1940	int ret;
1927	1941
1928		− if (adev->ras_features && con)
	1942	+ if (adev->ras_enabled && con)
1929	1943	data = &con->eh_data;
1930	1944	else
1931	1945	return 0;
@@ -2028,6 +2042,23 @@ static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) · 96 unmodified lines
2028	2042	adev->asic_type == CHIP_SIENNA_CICHLID;
2029	2043	}
2030	2044
	2045	+/*
	2046	+ * this is workaround for vega20 workstation sku,
	2047	+ * force enable gfx ras, ignore vbios gfx ras flag
	2048	+ * due to GC EDC can not write
	2049	+ */
	2050	+static void amdgpu_ras_get_quirks(struct amdgpu_device *adev)
	2051	+{
	2052	+ struct atom_context *ctx = adev->mode_info.atom_context;
	2053	+
	2054	+ if (!ctx)
	2055	+ return;
	2056	+
	2057	+ if (strnstr(ctx->vbios_version, "D16406",
	2058	+ sizeof(ctx->vbios_version)))
	2059	+ adev->ras_hw_enabled \|= (1 << AMDGPU_RAS_BLOCK__GFX);
	2060	+}
	2061	+
2031	2062	/*
2032	2063	* check hardware's ras ability which will be saved in hw_supported.
2033	2064	* if hardware does not support ras, we can skip some ras initializtion and
@@ -2037,11 +2068,9 @@ static bool amdgpu_ras_asic_supported(struct amdgpu_device *adev) · 3 unmodified lines
2037	2068	* we have to initialize ras as normal. but need check if operation is
2038	2069	* allowed or not in each function.
2039	2070	*/
2040		−static void amdgpu_ras_check_supported(struct amdgpu_device *adev,
2041		− uint32_t hw_supported, uint32_t supported)
	2071	+static void amdgpu_ras_check_supported(struct amdgpu_device *adev)
2042	2072	{
2043		− *hw_supported = 0;
2044		− *supported = 0;
	2073	+ adev->ras_hw_enabled = adev->ras_enabled = 0;
2045	2074
2046	2075	if (amdgpu_sriov_vf(adev) \|\| !adev->is_atom_fw \|\|
2047	2076	!amdgpu_ras_asic_supported(adev))
@@ -2050,33 +2079,34 @@ static void amdgpu_ras_check_supported(struct amdgpu_device *adev, · 2 unmodified lines
2050	2079	if (!adev->gmc.xgmi.connected_to_cpu) {
2051	2080	if (amdgpu_atomfirmware_mem_ecc_supported(adev)) {
2052	2081	dev_info(adev->dev, "MEM ECC is active.\n");
2053		− *hw_supported \|= (1 << AMDGPU_RAS_BLOCK__UMC \|
2054		− 1 << AMDGPU_RAS_BLOCK__DF);
	2082	+ adev->ras_hw_enabled \|= (1 << AMDGPU_RAS_BLOCK__UMC \|
	2083	+ 1 << AMDGPU_RAS_BLOCK__DF);
2055	2084	} else {
2056	2085	dev_info(adev->dev, "MEM ECC is not presented.\n");
2057	2086	}
2058	2087
2059	2088	if (amdgpu_atomfirmware_sram_ecc_supported(adev)) {
2060	2089	dev_info(adev->dev, "SRAM ECC is active.\n");
2061		− *hw_supported \|= ~(1 << AMDGPU_RAS_BLOCK__UMC \|
2062		− 1 << AMDGPU_RAS_BLOCK__DF);
	2090	+ adev->ras_hw_enabled \|= ~(1 << AMDGPU_RAS_BLOCK__UMC \|
	2091	+ 1 << AMDGPU_RAS_BLOCK__DF);
2063	2092	} else {
2064	2093	dev_info(adev->dev, "SRAM ECC is not presented.\n");
2065	2094	}
2066	2095	} else {
2067	2096	/* driver only manages a few IP blocks RAS feature
2068	2097	* when GPU is connected cpu through XGMI */
2069		− *hw_supported \|= (1 << AMDGPU_RAS_BLOCK__GFX \|
2070		− 1 << AMDGPU_RAS_BLOCK__SDMA \|
2071		− 1 << AMDGPU_RAS_BLOCK__MMHUB);
	2098	+ adev->ras_hw_enabled \|= (1 << AMDGPU_RAS_BLOCK__GFX \|
	2099	+ 1 << AMDGPU_RAS_BLOCK__SDMA \|
	2100	+ 1 << AMDGPU_RAS_BLOCK__MMHUB);
2072	2101	}
2073	2102
	2103	+ amdgpu_ras_get_quirks(adev);
	2104	+
2074	2105	/* hw_supported needs to be aligned with RAS block mask. */
2075		− *hw_supported &= AMDGPU_RAS_BLOCK_MASK;
	2106	+ adev->ras_hw_enabled &= AMDGPU_RAS_BLOCK_MASK;
2076	2107
2077		− *supported = amdgpu_ras_enable == 0 ?
2078		− 0 : *hw_supported & amdgpu_ras_mask;
2079		− adev->ras_features = *supported;
	2108	+ adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 :
	2109	+ adev->ras_hw_enabled & amdgpu_ras_mask;
2080	2110	}
2081	2111
2082	2112	int amdgpu_ras_init(struct amdgpu_device *adev)
@@ -2097,13 +2127,13 @@ int amdgpu_ras_init(struct amdgpu_device *adev) · 14 unmodified lines
2097	2127
2098	2128	amdgpu_ras_set_context(adev, con);
2099	2129
2100		− amdgpu_ras_check_supported(adev, &con->hw_supported,
2101		− &con->supported);
2102		− if (!con->hw_supported \|\| (adev->asic_type == CHIP_VEGA10)) {
	2130	+ amdgpu_ras_check_supported(adev);
	2131	+
	2132	+ if (!adev->ras_enabled \|\| adev->asic_type == CHIP_VEGA10) {
2103	2133	/* set gfx block ras context feature for VEGA20 Gaming
2104	2134	* send ras disable cmd to ras ta during ras late init.
2105	2135	*/
2106		− if (!adev->ras_features && adev->asic_type == CHIP_VEGA20) {
	2136	+ if (!adev->ras_enabled && adev->asic_type == CHIP_VEGA20) {
2107	2137	con->features \|= BIT(AMDGPU_RAS_BLOCK__GFX);
2108	2138
2109	2139	return 0;
@@ -2153,8 +2183,9 @@ int amdgpu_ras_init(struct amdgpu_device *adev) · 43 unmodified lines
2153	2183	}
2154	2184
2155	2185	dev_info(adev->dev, "RAS INFO: ras initialized successfully, "
2156		− "hardware ability[%x] ras_mask[%x]\n",
2157		− con->hw_supported, con->supported);
	2186	+ "hardware ability[%x] ras_mask[%x]\n",
	2187	+ adev->ras_hw_enabled, adev->ras_enabled);
	2188	+
2158	2189	return 0;
2159	2190	release_con:
2160	2191	amdgpu_ras_set_context(adev, NULL);
@@ -2268,7 +2299,7 @@ void amdgpu_ras_resume(struct amdgpu_device *adev) · 107 unmodified lines
2268	2299	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2269	2300	struct ras_manager obj, tmp;
2270	2301
2271		− if (!adev->ras_features \|\| !con) {
	2302	+ if (!adev->ras_enabled \|\| !con) {
2272	2303	/* clean ras context for VEGA20 Gaming after send ras disable cmd */
2273	2304	amdgpu_release_ras_context(adev);
2274	2305
@@ -2314,7 +2345,7 @@ void amdgpu_ras_suspend(struct amdgpu_device *adev) · 39 unmodified lines
2314	2345	{
2315	2346	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2316	2347
2317		− if (!adev->ras_features \|\| !con)
	2348	+ if (!adev->ras_enabled \|\| !con)
2318	2349	return;
2319	2350
2320	2351	amdgpu_ras_disable_all_features(adev, 0);
@@ -2328,7 +2359,7 @@ int amdgpu_ras_pre_fini(struct amdgpu_device *adev) · 7 unmodified lines
2328	2359	{
2329	2360	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2330	2361
2331		− if (!adev->ras_features \|\| !con)
	2362	+ if (!adev->ras_enabled \|\| !con)
2332	2363	return 0;
2333	2364
2334	2365	/* Need disable ras on all IPs here before ip [hw/sw]fini */
@@ -2341,7 +2372,7 @@ int amdgpu_ras_fini(struct amdgpu_device *adev) · 6 unmodified lines
2341	2372	{
2342	2373	struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
2343	2374
2344		− if (!adev->ras_features \|\| !con)
	2375	+ if (!adev->ras_enabled \|\| !con)
2345	2376	return 0;
2346	2377
2347	2378	amdgpu_ras_fs_fini(adev);
@@ -2360,10 +2391,8 @@ int amdgpu_ras_fini(struct amdgpu_device *adev) · 12 unmodified lines
2360	2391
2361	2392	void amdgpu_ras_global_ras_isr(struct amdgpu_device *adev)
2362	2393	{
2363		− uint32_t hw_supported, supported;
2364		−
2365		− amdgpu_ras_check_supported(adev, &hw_supported, &supported);
2366		− if (!hw_supported)
	2394	+ amdgpu_ras_check_supported(adev);
	2395	+ if (!adev->ras_hw_enabled)
2367	2396	return;
2368	2397
2369	2398	if (atomic_cmpxchg(&amdgpu_ras_in_intr, 0, 1) == 0) {
@@ -2392,7 +2421,7 @@ void amdgpu_release_ras_context(struct amdgpu_device *adev) · 22 unmodified lines
2392	2421	if (!con)
2393	2422	return;
2394	2423
2395		− if (!adev->ras_features && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
	2424	+ if (!adev->ras_enabled && con->features & BIT(AMDGPU_RAS_BLOCK__GFX)) {
2396	2425	con->features &= ~BIT(AMDGPU_RAS_BLOCK__GFX);
2397	2426	amdgpu_ras_set_context(adev, NULL);
2398	2427	kfree(con);

drivers/gpu/drm/amd/amdgpu/amdgpu_ras.h+1−4

@@ -313,9 +313,6 @@ struct ras_common_if { · 312 unmodified lines
313	313	struct amdgpu_ras {
314	314	/* ras infrastructure */
315	315	/* for ras itself. */
316		− uint32_t hw_supported;
317		− /* for IP to check its ras ability. */
318		− uint32_t supported;
319	316	uint32_t features;
320	317	struct list_head head;
321	318	/* sysfs */
@@ -478,7 +475,7 @@ static inline int amdgpu_ras_is_supported(struct amdgpu_device *adev, · 156 unmodified lines
478	475
479	476	if (block >= AMDGPU_RAS_BLOCK_COUNT)
480	477	return 0;
481		− return ras && (ras->supported & (1 << block));
	478	+ return ras && (adev->ras_enabled & (1 << block));
482	479	}
483	480
484	481	int amdgpu_ras_recovery_init(struct amdgpu_device *adev);

drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c+1−0

@@ -158,6 +158,7 @@ void amdgpu_ring_undo(struct amdgpu_ring *ring) · 157 unmodified lines
158	158	* @irq_src: interrupt source to use for this ring
159	159	* @irq_type: interrupt type to use for this ring
160	160	* @hw_prio: ring priority (NORMAL/HIGH)
	161	+ * @sched_score: optional score atomic shared with other schedulers
161	162	*
162	163	* Initialize the driver information for the selected ring (all asics).
163	164	* Returns 0 on success, error on failure.

drivers/gpu/drm/amd/amdgpu/amdgpu_smuio.h+1−0

@@ -29,6 +29,7 @@ struct amdgpu_smuio_funcs { · 28 unmodified lines
29	29	void (update_rom_clock_gating)(struct amdgpu_device adev, bool enable);
30	30	void (get_clock_gating_state)(struct amdgpu_device adev, u32 *flags);
31	31	u32 (get_die_id)(struct amdgpu_device adev);
	32	+ u32 (get_socket_id)(struct amdgpu_device adev);
32	33	bool (is_host_gpu_xgmi_supported)(struct amdgpu_device adev);
33	34	};
34	35

drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c+25−74

@@ -32,7 +32,6 @@ · 31 unmodified lines
32	32
33	33	#include <linux/dma-mapping.h>
34	34	#include <linux/iommu.h>
35		−#include <linux/hmm.h>
36	35	#include <linux/pagemap.h>
37	36	#include <linux/sched/task.h>
38	37	#include <linux/sched/mm.h>
@@ -112,6 +111,20 @@ static void amdgpu_evict_flags(struct ttm_buffer_object *bo, · 73 unmodified lines
112	111	}
113	112
114	113	abo = ttm_to_amdgpu_bo(bo);
	114	+ if (abo->flags & AMDGPU_AMDKFD_CREATE_SVM_BO) {
	115	+ struct dma_fence *fence;
	116	+ struct dma_resv *resv = &bo->base._resv;
	117	+
	118	+ rcu_read_lock();
	119	+ fence = rcu_dereference(resv->fence_excl);
	120	+ if (fence && !fence->ops->signaled)
	121	+ dma_fence_enable_sw_signaling(fence);
	122	+
	123	+ placement->num_placement = 0;
	124	+ placement->num_busy_placement = 0;
	125	+ rcu_read_unlock();
	126	+ return;
	127	+ }
115	128	switch (bo->mem.mem_type) {
116	129	case AMDGPU_PL_GDS:
117	130	case AMDGPU_PL_GWS:
@@ -165,13 +178,6 @@ static int amdgpu_verify_access(struct ttm_buffer_object bo, struct file filp) · 47 unmodified lines
165	178	{
166	179	struct amdgpu_bo *abo = ttm_to_amdgpu_bo(bo);
167	180
168		− /*
169		− * Don't verify access for KFD BOs. They don't have a GEM
170		− * object associated with them.
171		− */
172		− if (abo->kfd_bo)
173		− return 0;
174		−
175	181	if (amdgpu_ttm_tt_get_usermm(bo->ttm))
176	182	return -EPERM;
177	183	return drm_vma_node_verify_access(&abo->tbo.base.vma_node,
@@ -288,7 +294,7 @@ error_free: · 110 unmodified lines
288	294	}
289	295
290	296	/**
291		− * amdgpu_copy_ttm_mem_to_mem - Helper function for copy
	297	+ * amdgpu_ttm_copy_mem_to_mem - Helper function for copy
292	298	* @adev: amdgpu device
293	299	* @src: buffer/address where to read from
294	300	* @dst: buffer/address where to write to
@@ -670,10 +676,8 @@ int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo bo, struct page *pages) · 375 unmodified lines
670	676	struct amdgpu_ttm_tt gtt = (void )ttm;
671	677	unsigned long start = gtt->userptr;
672	678	struct vm_area_struct *vma;
673		− struct hmm_range *range;
674		− unsigned long timeout;
675	679	struct mm_struct *mm;
676		− unsigned long i;
	680	+ bool readonly;
677	681	int r = 0;
678	682
679	683	mm = bo->notifier.mm;
@@ -689,76 +693,26 @@ int amdgpu_ttm_tt_get_user_pages(struct amdgpu_bo bo, struct page *pages) · 9 unmodified lines
689	693	if (!mmget_not_zero(mm)) /* Happens during process shutdown */
690	694	return -ESRCH;
691	695
692		− range = kzalloc(sizeof(*range), GFP_KERNEL);
693		− if (unlikely(!range)) {
694		− r = -ENOMEM;
695		− goto out;
696		− }
697		− range->notifier = &bo->notifier;
698		− range->start = bo->notifier.interval_tree.start;
699		− range->end = bo->notifier.interval_tree.last + 1;
700		− range->default_flags = HMM_PFN_REQ_FAULT;
701		− if (!amdgpu_ttm_tt_is_readonly(ttm))
702		− range->default_flags \|= HMM_PFN_REQ_WRITE;
703		−
704		− range->hmm_pfns = kvmalloc_array(ttm->num_pages,
705		− sizeof(*range->hmm_pfns), GFP_KERNEL);
706		− if (unlikely(!range->hmm_pfns)) {
707		− r = -ENOMEM;
708		− goto out_free_ranges;
709		− }
710		−
711	696	mmap_read_lock(mm);
712	697	vma = find_vma(mm, start);
	698	+ mmap_read_unlock(mm);
713	699	if (unlikely(!vma \|\| start < vma->vm_start)) {
714	700	r = -EFAULT;
715		− goto out_unlock;
	701	+ goto out_putmm;
716	702	}
717	703	if (unlikely((gtt->userflags & AMDGPU_GEM_USERPTR_ANONONLY) &&
718	704	vma->vm_file)) {
719	705	r = -EPERM;
720		− goto out_unlock;
721		− }
722		− mmap_read_unlock(mm);
723		− timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
724		−
725		−retry:
726		− range->notifier_seq = mmu_interval_read_begin(&bo->notifier);
727		−
728		− mmap_read_lock(mm);
729		− r = hmm_range_fault(range);
730		− mmap_read_unlock(mm);
731		− if (unlikely(r)) {
732		− /*
733		− * FIXME: This timeout should encompass the retry from
734		− * mmu_interval_read_retry() as well.
735		− */
736		− if (r == -EBUSY && !time_after(jiffies, timeout))
737		− goto retry;
738		− goto out_free_pfns;
	706	+ goto out_putmm;
739	707	}
740	708
741		− /*
742		− * Due to default_flags, all pages are HMM_PFN_VALID or
743		− * hmm_range_fault() fails. FIXME: The pages cannot be touched outside
744		− * the notifier_lock, and mmu_interval_read_retry() must be done first.
745		− */
746		− for (i = 0; i < ttm->num_pages; i++)
747		− pages[i] = hmm_pfn_to_page(range->hmm_pfns[i]);
748		−
749		− gtt->range = range;
	709	+ readonly = amdgpu_ttm_tt_is_readonly(ttm);
	710	+ r = amdgpu_hmm_range_get_pages(&bo->notifier, mm, pages, start,
	711	+ ttm->num_pages, &gtt->range, readonly,
	712	+ false);
	713	+out_putmm:
750	714	mmput(mm);
751	715
752		− return 0;
753		−
754		−out_unlock:
755		− mmap_read_unlock(mm);
756		−out_free_pfns:
757		− kvfree(range->hmm_pfns);
758		−out_free_ranges:
759		− kfree(range);
760		−out:
761		− mmput(mm);
762	716	return r;
763	717	}
764	718
@@ -787,10 +741,7 @@ bool amdgpu_ttm_tt_get_user_pages_done(struct ttm_tt *ttm) · 22 unmodified lines
787	741	* FIXME: Must always hold notifier_lock for this, and must
788	742	* not ignore the return code.
789	743	*/
790		− r = mmu_interval_read_retry(gtt->range->notifier,
791		− gtt->range->notifier_seq);
792		− kvfree(gtt->range->hmm_pfns);
793		− kfree(gtt->range);
	744	+ r = amdgpu_hmm_range_get_pages_done(gtt->range);
794	745	gtt->range = NULL;
795	746	}
796	747

drivers/gpu/drm/amd/amdgpu/amdgpu_virt.c+7−3

@@ -50,9 +50,12 @@ void amdgpu_virt_init_setting(struct amdgpu_device *adev) · 49 unmodified lines
50	50	struct drm_device *ddev = adev_to_drm(adev);
51	51
52	52	/* enable virtual display */
53		− if (adev->mode_info.num_crtc == 0)
54		− adev->mode_info.num_crtc = 1;
55		− adev->enable_virtual_display = true;
	53	+ if (adev->asic_type != CHIP_ALDEBARAN &&
	54	+ adev->asic_type != CHIP_ARCTURUS) {
	55	+ if (adev->mode_info.num_crtc == 0)
	56	+ adev->mode_info.num_crtc = 1;
	57	+ adev->enable_virtual_display = true;
	58	+ }
56	59	ddev->driver_features &= ~DRIVER_ATOMIC;
57	60	adev->cg_flags = 0;
58	61	adev->pg_flags = 0;
@@ -679,6 +682,7 @@ void amdgpu_detect_virtualization(struct amdgpu_device *adev) · 620 unmodified lines
679	682	case CHIP_VEGA10:
680	683	case CHIP_VEGA20:
681	684	case CHIP_ARCTURUS:
	685	+ case CHIP_ALDEBARAN:
682	686	soc15_set_virt_ops(adev);
683	687	break;
684	688	case CHIP_NAVI10:

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c+72−52

@@ -38,6 +38,7 @@ · 37 unmodified lines
38	38	#include "amdgpu_gmc.h"
39	39	#include "amdgpu_xgmi.h"
40	40	#include "amdgpu_dma_buf.h"
	41	+#include "kfd_svm.h"
41	42
42	43	/**
43	44	* DOC: GPUVM
@@ -850,35 +851,60 @@ static int amdgpu_vm_clear_bo(struct amdgpu_device *adev, · 806 unmodified lines
850	851	}
851	852
852	853	/**
853		− * amdgpu_vm_bo_param - fill in parameters for PD/PT allocation
	854	+ * amdgpu_vm_pt_create - create bo for PD/PT
854	855	*
855	856	* @adev: amdgpu_device pointer
856	857	* @vm: requesting vm
857	858	* @level: the page table level
858	859	* @immediate: use a immediate update
859		− * @bp: resulting BO allocation parameters
	860	+ * @bo: pointer to the buffer object pointer
860	861	*/
861		−static void amdgpu_vm_bo_param(struct amdgpu_device adev, struct amdgpu_vm vm,
	862	+static int amdgpu_vm_pt_create(struct amdgpu_device *adev,
	863	+ struct amdgpu_vm *vm,
862	864	int level, bool immediate,
863		− struct amdgpu_bo_param *bp)
	865	+ struct amdgpu_bo **bo)
864	866	{
865		− memset(bp, 0, sizeof(*bp));
	867	+ struct amdgpu_bo_param bp;
	868	+ int r;
866	869
867		− bp->size = amdgpu_vm_bo_size(adev, level);
868		− bp->byte_align = AMDGPU_GPU_PAGE_SIZE;
869		− bp->domain = AMDGPU_GEM_DOMAIN_VRAM;
870		− bp->domain = amdgpu_bo_get_preferred_pin_domain(adev, bp->domain);
871		− bp->flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS \|
	870	+ memset(&bp, 0, sizeof(bp));
	871	+
	872	+ bp.size = amdgpu_vm_bo_size(adev, level);
	873	+ bp.byte_align = AMDGPU_GPU_PAGE_SIZE;
	874	+ bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
	875	+ bp.domain = amdgpu_bo_get_preferred_pin_domain(adev, bp.domain);
	876	+ bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS \|
872	877	AMDGPU_GEM_CREATE_CPU_GTT_USWC;
873		− bp->bo_ptr_size = sizeof(struct amdgpu_bo);
	878	+ bp.bo_ptr_size = sizeof(struct amdgpu_bo);
874	879	if (vm->use_cpu_for_update)
875		− bp->flags \|= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
876		− else if (!vm->root.base.bo \|\| vm->root.base.bo->shadow)
877		− bp->flags \|= AMDGPU_GEM_CREATE_SHADOW;
878		− bp->type = ttm_bo_type_kernel;
879		− bp->no_wait_gpu = immediate;
	880	+ bp.flags \|= AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED;
	881	+
	882	+ bp.type = ttm_bo_type_kernel;
	883	+ bp.no_wait_gpu = immediate;
880	884	if (vm->root.base.bo)
881		− bp->resv = vm->root.base.bo->tbo.base.resv;
	885	+ bp.resv = vm->root.base.bo->tbo.base.resv;
	886	+
	887	+ r = amdgpu_bo_create(adev, &bp, bo);
	888	+ if (r)
	889	+ return r;
	890	+
	891	+ if (vm->is_compute_context && (adev->flags & AMD_IS_APU))
	892	+ return 0;
	893	+
	894	+ if (!bp.resv)
	895	+ WARN_ON(dma_resv_lock((*bo)->tbo.base.resv,
	896	+ NULL));
	897	+ r = amdgpu_bo_create_shadow(adev, bp.size, *bo);
	898	+
	899	+ if (!bp.resv)
	900	+ dma_resv_unlock((*bo)->tbo.base.resv);
	901	+
	902	+ if (r) {
	903	+ amdgpu_bo_unref(bo);
	904	+ return r;
	905	+ }
	906	+
	907	+ return 0;
882	908	}
883	909
884	910	/**
@@ -901,7 +927,6 @@ static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, · 16 unmodified lines
901	927	bool immediate)
902	928	{
903	929	struct amdgpu_vm_pt *entry = cursor->entry;
904		− struct amdgpu_bo_param bp;
905	930	struct amdgpu_bo *pt;
906	931	int r;
907	932
@@ -919,9 +944,7 @@ static int amdgpu_vm_alloc_pts(struct amdgpu_device *adev, · 11 unmodified lines
919	944	if (entry->base.bo)
920	945	return 0;
921	946
922		− amdgpu_vm_bo_param(adev, vm, cursor->level, immediate, &bp);
923		−
924		− r = amdgpu_bo_create(adev, &bp, &pt);
	947	+ r = amdgpu_vm_pt_create(adev, vm, cursor->level, immediate, &pt);
925	948	if (r)
926	949	return r;
927	950
@@ -1593,15 +1616,15 @@ static int amdgpu_vm_update_ptes(struct amdgpu_vm_update_params *params, · 665 unmodified lines
1593	1616	* Returns:
1594	1617	* 0 for success, -EINVAL for failure.
1595	1618	*/
1596		−static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
1597		− struct amdgpu_device *bo_adev,
1598		− struct amdgpu_vm *vm, bool immediate,
1599		− bool unlocked, struct dma_resv *resv,
1600		− uint64_t start, uint64_t last,
1601		− uint64_t flags, uint64_t offset,
1602		− struct drm_mm_node *nodes,
1603		− dma_addr_t *pages_addr,
1604		− struct dma_fence **fence)
	1619	+int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
	1620	+ struct amdgpu_device *bo_adev,
	1621	+ struct amdgpu_vm *vm, bool immediate,
	1622	+ bool unlocked, struct dma_resv *resv,
	1623	+ uint64_t start, uint64_t last,
	1624	+ uint64_t flags, uint64_t offset,
	1625	+ struct drm_mm_node *nodes,
	1626	+ dma_addr_t *pages_addr,
	1627	+ struct dma_fence **fence)
1605	1628	{
1606	1629	struct amdgpu_vm_update_params params;
1607	1630	enum amdgpu_sync_mode sync_mode;
@@ -2818,7 +2841,6 @@ long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout) · 1210 unmodified lines
2818	2841	*
2819	2842	* @adev: amdgpu_device pointer
2820	2843	* @vm: requested vm
2821		− * @vm_context: Indicates if it GFX or Compute context
2822	2844	* @pasid: Process address space identifier
2823	2845	*
2824	2846	* Init @vm fields.
@@ -2826,10 +2848,8 @@ long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout) · 1 unmodified line
2826	2848	* Returns:
2827	2849	* 0 for success, error for failure.
2828	2850	*/
2829		−int amdgpu_vm_init(struct amdgpu_device adev, struct amdgpu_vm vm,
2830		− int vm_context, u32 pasid)
	2851	+int amdgpu_vm_init(struct amdgpu_device adev, struct amdgpu_vm vm, u32 pasid)
2831	2852	{
2832		− struct amdgpu_bo_param bp;
2833	2853	struct amdgpu_bo *root;
2834	2854	int r, i;
2835	2855
@@ -2861,16 +2881,9 @@ int amdgpu_vm_init(struct amdgpu_device adev, struct amdgpu_vm vm, · 25 unmodified lines
2861	2881	vm->pte_support_ats = false;
2862	2882	vm->is_compute_context = false;
2863	2883
2864		− if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE) {
2865		− vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2866		− AMDGPU_VM_USE_CPU_FOR_COMPUTE);
	2884	+ vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
	2885	+ AMDGPU_VM_USE_CPU_FOR_GFX);
2867	2886
2868		− if (adev->asic_type == CHIP_RAVEN)
2869		− vm->pte_support_ats = true;
2870		− } else {
2871		− vm->use_cpu_for_update = !!(adev->vm_manager.vm_update_mode &
2872		− AMDGPU_VM_USE_CPU_FOR_GFX);
2873		− }
2874	2887	DRM_DEBUG_DRIVER("VM update mode is %s\n",
2875	2888	vm->use_cpu_for_update ? "CPU" : "SDMA");
2876	2889	WARN_ONCE((vm->use_cpu_for_update &&
@@ -2887,10 +2900,8 @@ int amdgpu_vm_init(struct amdgpu_device adev, struct amdgpu_vm vm, · 10 unmodified lines
2887	2900	mutex_init(&vm->eviction_lock);
2888	2901	vm->evicting = false;
2889	2902
2890		− amdgpu_vm_bo_param(adev, vm, adev->vm_manager.root_level, false, &bp);
2891		− if (vm_context == AMDGPU_VM_CONTEXT_COMPUTE)
2892		− bp.flags &= ~AMDGPU_GEM_CREATE_SHADOW;
2893		− r = amdgpu_bo_create(adev, &bp, &root);
	2903	+ r = amdgpu_vm_pt_create(adev, vm, adev->vm_manager.root_level,
	2904	+ false, &root);
2894	2905	if (r)
2895	2906	goto error_free_delayed;
2896	2907
@@ -3349,6 +3360,7 @@ void amdgpu_vm_set_task_info(struct amdgpu_vm *vm) · 452 unmodified lines
3349	3360	bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid,
3350	3361	uint64_t addr)
3351	3362	{
	3363	+ bool is_compute_context = false;
3352	3364	struct amdgpu_bo *root;
3353	3365	uint64_t value, flags;
3354	3366	struct amdgpu_vm *vm;
@@ -3356,15 +3368,25 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, · 1 unmodified line
3356	3368
3357	3369	spin_lock(&adev->vm_manager.pasid_lock);
3358	3370	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
3359		− if (vm)
	3371	+ if (vm) {
3360	3372	root = amdgpu_bo_ref(vm->root.base.bo);
3361		− else
	3373	+ is_compute_context = vm->is_compute_context;
	3374	+ } else {
3362	3375	root = NULL;
	3376	+ }
3363	3377	spin_unlock(&adev->vm_manager.pasid_lock);
3364	3378
3365	3379	if (!root)
3366	3380	return false;
3367	3381
	3382	+ addr /= AMDGPU_GPU_PAGE_SIZE;
	3383	+
	3384	+ if (is_compute_context &&
	3385	+ !svm_range_restore_pages(adev, pasid, addr)) {
	3386	+ amdgpu_bo_unref(&root);
	3387	+ return true;
	3388	+ }
	3389	+
3368	3390	r = amdgpu_bo_reserve(root, true);
3369	3391	if (r)
3370	3392	goto error_unref;
@@ -3378,18 +3400,16 @@ bool amdgpu_vm_handle_fault(struct amdgpu_device *adev, u32 pasid, · 7 unmodified lines
3378	3400	if (!vm)
3379	3401	goto error_unlock;
3380	3402
3381		− addr /= AMDGPU_GPU_PAGE_SIZE;
3382	3403	flags = AMDGPU_PTE_VALID \| AMDGPU_PTE_SNOOPED \|
3383	3404	AMDGPU_PTE_SYSTEM;
3384	3405
3385		− if (vm->is_compute_context) {
	3406	+ if (is_compute_context) {
3386	3407	/* Intentionally setting invalid PTE flag
3387	3408	* combination to force a no-retry-fault
3388	3409	*/
3389	3410	flags = AMDGPU_PTE_EXECUTABLE \| AMDGPU_PDE_PTE \|
3390	3411	AMDGPU_PTE_TF;
3391	3412	value = 0;
3392		−
3393	3413	} else if (amdgpu_vm_fault_stop == AMDGPU_VM_FAULT_STOP_NEVER) {
3394	3414	/* Redirect the access to the dummy page */
3395	3415	value = adev->dummy_page_addr;

drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h+12−5

@@ -121,9 +121,6 @@ struct amdgpu_bo_list_entry; · 120 unmodified lines
121	121	/* max vmids dedicated for process */
122	122	#define AMDGPU_VM_MAX_RESERVED_VMID 1
123	123
124		−#define AMDGPU_VM_CONTEXT_GFX 0
125		−#define AMDGPU_VM_CONTEXT_COMPUTE 1
126		−
127	124	/* See vm_update_mode */
128	125	#define AMDGPU_VM_USE_CPU_FOR_GFX (1 << 0)
129	126	#define AMDGPU_VM_USE_CPU_FOR_COMPUTE (1 << 1)
@@ -367,6 +364,8 @@ struct amdgpu_vm_manager { · 237 unmodified lines
367	364	spinlock_t pasid_lock;
368	365	};
369	366
	367	+struct amdgpu_bo_va_mapping;
	368	+
370	369	#define amdgpu_vm_copy_pte(adev, ib, pe, src, count) ((adev)->vm_manager.vm_pte_funcs->copy_pte((ib), (pe), (src), (count)))
371	370	#define amdgpu_vm_write_pte(adev, ib, pe, value, count, incr) ((adev)->vm_manager.vm_pte_funcs->write_pte((ib), (pe), (value), (count), (incr)))
372	371	#define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags)))
@@ -378,8 +377,7 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev); · 5 unmodified lines
378	377	void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
379	378
380	379	long amdgpu_vm_wait_idle(struct amdgpu_vm *vm, long timeout);
381		−int amdgpu_vm_init(struct amdgpu_device adev, struct amdgpu_vm vm,
382		− int vm_context, u32 pasid);
	380	+int amdgpu_vm_init(struct amdgpu_device adev, struct amdgpu_vm vm, u32 pasid);
383	381	int amdgpu_vm_make_compute(struct amdgpu_device adev, struct amdgpu_vm vm, u32 pasid);
384	382	void amdgpu_vm_release_compute(struct amdgpu_device adev, struct amdgpu_vm vm);
385	383	void amdgpu_vm_fini(struct amdgpu_device adev, struct amdgpu_vm vm);
@@ -398,6 +396,15 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev, · 12 unmodified lines
398	396	struct dma_fence **fence);
399	397	int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
400	398	struct amdgpu_vm *vm);
	399	+int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
	400	+ struct amdgpu_device *bo_adev,
	401	+ struct amdgpu_vm *vm, bool immediate,
	402	+ bool unlocked, struct dma_resv *resv,
	403	+ uint64_t start, uint64_t last,
	404	+ uint64_t flags, uint64_t offset,
	405	+ struct drm_mm_node *nodes,
	406	+ dma_addr_t *pages_addr,
	407	+ struct dma_fence **fence);
401	408	int amdgpu_vm_bo_update(struct amdgpu_device *adev,
402	409	struct amdgpu_bo_va *bo_va,
403	410	bool clear);

drivers/gpu/drm/amd/amdgpu/amdgpu_vram_mgr.c+120−112

@@ -29,12 +29,14 @@ · 28 unmodified lines
29	29	#include "amdgpu_atomfirmware.h"
30	30	#include "atom.h"
31	31
32		−static inline struct amdgpu_vram_mgr to_vram_mgr(struct ttm_resource_manager man)
	32	+static inline struct amdgpu_vram_mgr *
	33	+to_vram_mgr(struct ttm_resource_manager *man)
33	34	{
34	35	return container_of(man, struct amdgpu_vram_mgr, manager);
35	36	}
36	37
37		−static inline struct amdgpu_device to_amdgpu_device(struct amdgpu_vram_mgr mgr)
	38	+static inline struct amdgpu_device *
	39	+to_amdgpu_device(struct amdgpu_vram_mgr *mgr)
38	40	{
39	41	return container_of(mgr, struct amdgpu_device, mman.vram_mgr);
40	42	}
@@ -82,12 +84,14 @@ static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev, · 41 unmodified lines
82	84	* amount of currently used VRAM in bytes
83	85	*/
84	86	static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev,
85		− struct device_attribute attr, char buf)
	87	+ struct device_attribute *attr,
	88	+ char *buf)
86	89	{
87	90	struct drm_device *ddev = dev_get_drvdata(dev);
88	91	struct amdgpu_device *adev = drm_to_adev(ddev);
89		− struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
	92	+ struct ttm_resource_manager *man;
90	93
	94	+ man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
91	95	return sysfs_emit(buf, "%llu\n", amdgpu_vram_mgr_usage(man));
92	96	}
93	97
@@ -100,18 +104,28 @@ static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev, · 6 unmodified lines
100	104	* amount of currently used visible VRAM in bytes
101	105	*/
102	106	static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev,
103		− struct device_attribute attr, char buf)
	107	+ struct device_attribute *attr,
	108	+ char *buf)
104	109	{
105	110	struct drm_device *ddev = dev_get_drvdata(dev);
106	111	struct amdgpu_device *adev = drm_to_adev(ddev);
107		− struct ttm_resource_manager *man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
	112	+ struct ttm_resource_manager *man;
108	113
	114	+ man = ttm_manager_type(&adev->mman.bdev, TTM_PL_VRAM);
109	115	return sysfs_emit(buf, "%llu\n", amdgpu_vram_mgr_vis_usage(man));
110	116	}
111	117
	118	+/**
	119	+ * DOC: mem_info_vram_vendor
	120	+ *
	121	+ * The amdgpu driver provides a sysfs API for reporting the vendor of the
	122	+ * installed VRAM
	123	+ * The file mem_info_vram_vendor is used for this and returns the name of the
	124	+ * vendor.
	125	+ */
112	126	static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev,
113		− struct device_attribute *attr,
114		− char *buf)
	127	+ struct device_attribute *attr,
	128	+ char *buf)
115	129	{
116	130	struct drm_device *ddev = dev_get_drvdata(dev);
117	131	struct amdgpu_device *adev = drm_to_adev(ddev);
@@ -162,78 +176,6 @@ static const struct attribute *amdgpu_vram_mgr_attributes[] = { · 44 unmodified lines
162	176	NULL
163	177	};
164	178
165		−static const struct ttm_resource_manager_func amdgpu_vram_mgr_func;
166		−
167		−/**
168		− * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
169		− *
170		− * @adev: amdgpu_device pointer
171		− *
172		− * Allocate and initialize the VRAM manager.
173		− */
174		−int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
175		−{
176		− struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
177		− struct ttm_resource_manager *man = &mgr->manager;
178		− int ret;
179		−
180		− ttm_resource_manager_init(man, adev->gmc.real_vram_size >> PAGE_SHIFT);
181		−
182		− man->func = &amdgpu_vram_mgr_func;
183		−
184		− drm_mm_init(&mgr->mm, 0, man->size);
185		− spin_lock_init(&mgr->lock);
186		− INIT_LIST_HEAD(&mgr->reservations_pending);
187		− INIT_LIST_HEAD(&mgr->reserved_pages);
188		−
189		− /* Add the two VRAM-related sysfs files */
190		− ret = sysfs_create_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes);
191		− if (ret)
192		− DRM_ERROR("Failed to register sysfs\n");
193		−
194		− ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);
195		− ttm_resource_manager_set_used(man, true);
196		− return 0;
197		−}
198		−
199		−/**
200		− * amdgpu_vram_mgr_fini - free and destroy VRAM manager
201		− *
202		− * @adev: amdgpu_device pointer
203		− *
204		− * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
205		− * allocated inside it.
206		− */
207		−void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
208		−{
209		− struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
210		− struct ttm_resource_manager *man = &mgr->manager;
211		− int ret;
212		− struct amdgpu_vram_reservation rsv, temp;
213		−
214		− ttm_resource_manager_set_used(man, false);
215		−
216		− ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
217		− if (ret)
218		− return;
219		−
220		− spin_lock(&mgr->lock);
221		− list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node)
222		− kfree(rsv);
223		−
224		− list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, node) {
225		− drm_mm_remove_node(&rsv->mm_node);
226		− kfree(rsv);
227		− }
228		− drm_mm_takedown(&mgr->mm);
229		− spin_unlock(&mgr->lock);
230		−
231		− sysfs_remove_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes);
232		−
233		− ttm_resource_manager_cleanup(man);
234		− ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL);
235		−}
236		−
237	179	/**
238	180	* amdgpu_vram_mgr_vis_size - Calculate visible node size
239	181	*
@@ -283,6 +225,7 @@ u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo) · 43 unmodified lines
283	225	return usage;
284	226	}
285	227
	228	+/* Commit the reservation of VRAM pages */
286	229	static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man)
287	230	{
288	231	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
@@ -415,13 +358,13 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, · 126 unmodified lines
415	358	const struct ttm_place *place,
416	359	struct ttm_resource *mem)
417	360	{
	361	+ unsigned long lpfn, num_nodes, pages_per_node, pages_left, pages;
418	362	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
419	363	struct amdgpu_device *adev = to_amdgpu_device(mgr);
	364	+ uint64_t vis_usage = 0, mem_bytes, max_bytes;
420	365	struct drm_mm *mm = &mgr->mm;
421		− struct drm_mm_node *nodes;
422	366	enum drm_mm_insert_mode mode;
423		− unsigned long lpfn, num_nodes, pages_per_node, pages_left;
424		− uint64_t vis_usage = 0, mem_bytes, max_bytes;
	367	+ struct drm_mm_node *nodes;
425	368	unsigned i;
426	369	int r;
427	370
@@ -448,10 +391,10 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, · 20 unmodified lines
448	391	pages_per_node = HPAGE_PMD_NR;
449	392	#else
450	393	/* default to 2MB */
451		− pages_per_node = (2UL << (20UL - PAGE_SHIFT));
	394	+ pages_per_node = 2UL << (20UL - PAGE_SHIFT);
452	395	#endif
453		− pages_per_node = max((uint32_t)pages_per_node,
454		− tbo->page_alignment);
	396	+ pages_per_node = max_t(uint32_t, pages_per_node,
	397	+ tbo->page_alignment);
455	398	num_nodes = DIV_ROUND_UP(mem->num_pages, pages_per_node);
456	399	}
457	400
@@ -469,42 +412,37 @@ static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man, · 11 unmodified lines
469	412	mem->start = 0;
470	413	pages_left = mem->num_pages;
471	414
472		− spin_lock(&mgr->lock);
473		− for (i = 0; pages_left >= pages_per_node; ++i) {
474		− unsigned long pages = rounddown_pow_of_two(pages_left);
	415	+ /* Limit maximum size to 2GB due to SG table limitations */
	416	+ pages = min(pages_left, 2UL << (30 - PAGE_SHIFT));
475	417
476		− /* Limit maximum size to 2GB due to SG table limitations */
477		− pages = min(pages, (2UL << (30 - PAGE_SHIFT)));
478		−
479		− r = drm_mm_insert_node_in_range(mm, &nodes[i], pages,
480		− pages_per_node, 0,
481		− place->fpfn, lpfn,
482		− mode);
483		− if (unlikely(r))
484		− break;
485		−
486		− vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);
487		− amdgpu_vram_mgr_virt_start(mem, &nodes[i]);
488		− pages_left -= pages;
489		− }
490		−
491		− for (; pages_left; ++i) {
492		− unsigned long pages = min(pages_left, pages_per_node);
	418	+ i = 0;
	419	+ spin_lock(&mgr->lock);
	420	+ while (pages_left) {
493	421	uint32_t alignment = tbo->page_alignment;
494	422
495		− if (pages == pages_per_node)
	423	+ if (pages >= pages_per_node)
496	424	alignment = pages_per_node;
497	425
498		− r = drm_mm_insert_node_in_range(mm, &nodes[i],
499		− pages, alignment, 0,
500		− place->fpfn, lpfn,
501		− mode);
502		− if (unlikely(r))
	426	+ r = drm_mm_insert_node_in_range(mm, &nodes[i], pages, alignment,
	427	+ 0, place->fpfn, lpfn, mode);
	428	+ if (unlikely(r)) {
	429	+ if (pages > pages_per_node) {
	430	+ if (is_power_of_2(pages))
	431	+ pages = pages / 2;
	432	+ else
	433	+ pages = rounddown_pow_of_two(pages);
	434	+ continue;
	435	+ }
503	436	goto error;
	437	+ }
504	438
505	439	vis_usage += amdgpu_vram_mgr_vis_size(adev, &nodes[i]);
506	440	amdgpu_vram_mgr_virt_start(mem, &nodes[i]);
507	441	pages_left -= pages;
	442	+ ++i;
	443	+
	444	+ if (pages > pages_left)
	445	+ pages = pages_left;
508	446	}
509	447	spin_unlock(&mgr->lock);
510	448
@@ -728,3 +666,73 @@ static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = { · 217 unmodified lines
728	666	.free = amdgpu_vram_mgr_del,
729	667	.debug = amdgpu_vram_mgr_debug
730	668	};
	669	+
	670	+/**
	671	+ * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
	672	+ *
	673	+ * @adev: amdgpu_device pointer
	674	+ *
	675	+ * Allocate and initialize the VRAM manager.
	676	+ */
	677	+int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
	678	+{
	679	+ struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
	680	+ struct ttm_resource_manager *man = &mgr->manager;
	681	+ int ret;
	682	+
	683	+ ttm_resource_manager_init(man, adev->gmc.real_vram_size >> PAGE_SHIFT);
	684	+
	685	+ man->func = &amdgpu_vram_mgr_func;
	686	+
	687	+ drm_mm_init(&mgr->mm, 0, man->size);
	688	+ spin_lock_init(&mgr->lock);
	689	+ INIT_LIST_HEAD(&mgr->reservations_pending);
	690	+ INIT_LIST_HEAD(&mgr->reserved_pages);
	691	+
	692	+ /* Add the two VRAM-related sysfs files */
	693	+ ret = sysfs_create_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes);
	694	+ if (ret)
	695	+ DRM_ERROR("Failed to register sysfs\n");
	696	+
	697	+ ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);
	698	+ ttm_resource_manager_set_used(man, true);
	699	+ return 0;
	700	+}
	701	+
	702	+/**
	703	+ * amdgpu_vram_mgr_fini - free and destroy VRAM manager
	704	+ *
	705	+ * @adev: amdgpu_device pointer
	706	+ *
	707	+ * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
	708	+ * allocated inside it.
	709	+ */
	710	+void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
	711	+{
	712	+ struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
	713	+ struct ttm_resource_manager *man = &mgr->manager;
	714	+ int ret;
	715	+ struct amdgpu_vram_reservation rsv, temp;
	716	+
	717	+ ttm_resource_manager_set_used(man, false);
	718	+
	719	+ ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
	720	+ if (ret)
	721	+ return;
	722	+
	723	+ spin_lock(&mgr->lock);
	724	+ list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, node)
	725	+ kfree(rsv);
	726	+
	727	+ list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, node) {
	728	+ drm_mm_remove_node(&rsv->mm_node);
	729	+ kfree(rsv);
	730	+ }
	731	+ drm_mm_takedown(&mgr->mm);
	732	+ spin_unlock(&mgr->lock);
	733	+
	734	+ sysfs_remove_files(&adev->dev->kobj, amdgpu_vram_mgr_attributes);
	735	+
	736	+ ttm_resource_manager_cleanup(man);
	737	+ ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL);
	738	+}

drivers/gpu/drm/amd/amdgpu/amdgv_sriovmsg.h+3−3

@@ -98,9 +98,9 @@ union amd_sriov_msg_feature_flags { · 97 unmodified lines
98	98
99	99	union amd_sriov_reg_access_flags {
100	100	struct {
101		− uint32_t vf_reg_access_ih : 1;
102		− uint32_t vf_reg_access_mmhub : 1;
103		− uint32_t vf_reg_access_gc : 1;
	101	+ uint32_t vf_reg_psp_access_ih : 1;
	102	+ uint32_t vf_reg_rlc_access_mmhub : 1;
	103	+ uint32_t vf_reg_rlc_access_gc : 1;
104	104	uint32_t reserved : 29;
105	105	} flags;
106	106	uint32_t all;

drivers/gpu/drm/amd/amdgpu/dce_virtual.c+5−7

@@ -421,6 +421,11 @@ static int dce_virtual_sw_init(void *handle) · 420 unmodified lines
421	421	static int dce_virtual_sw_fini(void *handle)
422	422	{
423	423	struct amdgpu_device adev = (struct amdgpu_device )handle;
	424	+ int i = 0;
	425	+
	426	+ for (i = 0; i < adev->mode_info.num_crtc; i++)
	427	+ if (adev->mode_info.crtcs[i])
	428	+ hrtimer_cancel(&adev->mode_info.crtcs[i]->vblank_timer);
424	429
425	430	kfree(adev->mode_info.bios_hardcoded_edid);
426	431
@@ -480,13 +485,6 @@ static int dce_virtual_hw_init(void *handle) · 53 unmodified lines
480	485
481	486	static int dce_virtual_hw_fini(void *handle)
482	487	{
483		− struct amdgpu_device adev = (struct amdgpu_device )handle;
484		− int i = 0;
485		−
486		− for (i = 0; i<adev->mode_info.num_crtc; i++)
487		− if (adev->mode_info.crtcs[i])
488		− hrtimer_cancel(&adev->mode_info.crtcs[i]->vblank_timer);
489		−
490	488	return 0;
491	489	}
492	490

drivers/gpu/drm/amd/amdgpu/df_v3_6.c+11−6

@@ -219,11 +219,11 @@ static void df_v3_6_query_hashes(struct amdgpu_device *adev) · 218 unmodified lines
219	219	adev->df.hash_status.hash_2m = false;
220	220	adev->df.hash_status.hash_1g = false;
221	221
222		− if (adev->asic_type != CHIP_ARCTURUS)
223		− return;
224		−
225		− /* encoding for hash-enabled on Arcturus */
226		− if (adev->df.funcs->get_fb_channel_number(adev) == 0xe) {
	222	+ /* encoding for hash-enabled on Arcturus and Aldebaran */
	223	+ if ((adev->asic_type == CHIP_ARCTURUS &&
	224	+ adev->df.funcs->get_fb_channel_number(adev) == 0xe) \|\|
	225	+ (adev->asic_type == CHIP_ALDEBARAN &&
	226	+ adev->df.funcs->get_fb_channel_number(adev) == 0x1e)) {
227	227	tmp = RREG32_SOC15(DF, 0, mmDF_CS_UMC_AON0_DfGlobalCtrl);
228	228	adev->df.hash_status.hash_64k = REG_GET_FIELD(tmp,
229	229	DF_CS_UMC_AON0_DfGlobalCtrl,
@@ -278,7 +278,12 @@ static u32 df_v3_6_get_fb_channel_number(struct amdgpu_device *adev) · 48 unmodified lines
278	278	u32 tmp;
279	279
280	280	tmp = RREG32_SOC15(DF, 0, mmDF_CS_UMC_AON0_DramBaseAddress0);
281		− tmp &= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK;
	281	+ if (adev->asic_type == CHIP_ALDEBARAN)
	282	+ tmp &=
	283	+ ALDEBARAN_DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK;
	284	+ else
	285	+ tmp &= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK;
	286	+
282	287	tmp >>= DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan__SHIFT;
283	288
284	289	return tmp;

drivers/gpu/drm/amd/amdgpu/gfx_v9_0.c+8−4

@@ -3937,7 +3937,8 @@ static void gfx_v9_0_init_tcp_config(struct amdgpu_device *adev) · 3936 unmodified lines
3937	3937	{
3938	3938	u32 tmp;
3939	3939
3940		− if (adev->asic_type != CHIP_ARCTURUS)
	3940	+ if (adev->asic_type != CHIP_ARCTURUS &&
	3941	+ adev->asic_type != CHIP_ALDEBARAN)
3941	3942	return;
3942	3943
3943	3944	tmp = RREG32_SOC15(GC, 0, mmTCP_ADDR_CONFIG);
@@ -4559,8 +4560,7 @@ static int gfx_v9_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) · 615 unmodified lines
4559	4560	if (!ring->sched.ready)
4560	4561	return 0;
4561	4562
4562		− if (adev->asic_type == CHIP_ARCTURUS \|\|
4563		− adev->asic_type == CHIP_ALDEBARAN) {
	4563	+ if (adev->asic_type == CHIP_ARCTURUS) {
4564	4564	vgpr_init_shader_ptr = vgpr_init_compute_shader_arcturus;
4565	4565	vgpr_init_shader_size = sizeof(vgpr_init_compute_shader_arcturus);
4566	4566	vgpr_init_regs_ptr = vgpr_init_regs_arcturus;
@@ -4745,7 +4745,11 @@ static int gfx_v9_0_ecc_late_init(void *handle) · 178 unmodified lines
4745	4745	}
4746	4746
4747	4747	/* requires IBs so do in late init after IB pool is initialized */
4748		− r = gfx_v9_0_do_edc_gpr_workarounds(adev);
	4748	+ if (adev->asic_type == CHIP_ALDEBARAN)
	4749	+ r = gfx_v9_4_2_do_edc_gpr_workarounds(adev);
	4750	+ else
	4751	+ r = gfx_v9_0_do_edc_gpr_workarounds(adev);
	4752	+
4749	4753	if (r)
4750	4754	return r;
4751	4755

drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.c+641−3

@@ -22,6 +22,7 @@ · 21 unmodified lines
22	22	*/
23	23	#include "amdgpu.h"
24	24	#include "soc15.h"
	25	+#include "soc15d.h"
25	26
26	27	#include "gc/gc_9_4_2_offset.h"
27	28	#include "gc/gc_9_4_2_sh_mask.h"
@@ -31,6 +32,11 @@ · 3 unmodified lines
31	32	#include "amdgpu_ras.h"
32	33	#include "amdgpu_gfx.h"
33	34
	35	+#define SE_ID_MAX 8
	36	+#define CU_ID_MAX 16
	37	+#define SIMD_ID_MAX 4
	38	+#define WAVE_ID_MAX 10
	39	+
34	40	enum gfx_v9_4_2_utc_type {
35	41	VML2_MEM,
36	42	VML2_WALKER_MEM,
@@ -79,6 +85,634 @@ static const struct soc15_reg_golden golden_settings_gc_9_4_2_alde[] = { · 42 unmodified lines
79	85	SOC15_REG_GOLDEN_VALUE(GC, 0, regTCI_CNTL_3, 0xff, 0x20),
80	86	};
81	87
	88	+/**
	89	+ * This shader is used to clear VGPRS and LDS, and also write the input
	90	+ * pattern into the write back buffer, which will be used by driver to
	91	+ * check whether all SIMDs have been covered.
	92	+*/
	93	+static const u32 vgpr_init_compute_shader_aldebaran[] = {
	94	+ 0xb8840904, 0xb8851a04, 0xb8861344, 0xb8831804, 0x9208ff06, 0x00000280,
	95	+ 0x9209a805, 0x920a8a04, 0x81080908, 0x81080a08, 0x81080308, 0x8e078208,
	96	+ 0x81078407, 0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8a0000, 0xd3d94000,
	97	+ 0x18000080, 0xd3d94001, 0x18000080, 0xd3d94002, 0x18000080, 0xd3d94003,
	98	+ 0x18000080, 0xd3d94004, 0x18000080, 0xd3d94005, 0x18000080, 0xd3d94006,
	99	+ 0x18000080, 0xd3d94007, 0x18000080, 0xd3d94008, 0x18000080, 0xd3d94009,
	100	+ 0x18000080, 0xd3d9400a, 0x18000080, 0xd3d9400b, 0x18000080, 0xd3d9400c,
	101	+ 0x18000080, 0xd3d9400d, 0x18000080, 0xd3d9400e, 0x18000080, 0xd3d9400f,
	102	+ 0x18000080, 0xd3d94010, 0x18000080, 0xd3d94011, 0x18000080, 0xd3d94012,
	103	+ 0x18000080, 0xd3d94013, 0x18000080, 0xd3d94014, 0x18000080, 0xd3d94015,
	104	+ 0x18000080, 0xd3d94016, 0x18000080, 0xd3d94017, 0x18000080, 0xd3d94018,
	105	+ 0x18000080, 0xd3d94019, 0x18000080, 0xd3d9401a, 0x18000080, 0xd3d9401b,
	106	+ 0x18000080, 0xd3d9401c, 0x18000080, 0xd3d9401d, 0x18000080, 0xd3d9401e,
	107	+ 0x18000080, 0xd3d9401f, 0x18000080, 0xd3d94020, 0x18000080, 0xd3d94021,
	108	+ 0x18000080, 0xd3d94022, 0x18000080, 0xd3d94023, 0x18000080, 0xd3d94024,
	109	+ 0x18000080, 0xd3d94025, 0x18000080, 0xd3d94026, 0x18000080, 0xd3d94027,
	110	+ 0x18000080, 0xd3d94028, 0x18000080, 0xd3d94029, 0x18000080, 0xd3d9402a,
	111	+ 0x18000080, 0xd3d9402b, 0x18000080, 0xd3d9402c, 0x18000080, 0xd3d9402d,
	112	+ 0x18000080, 0xd3d9402e, 0x18000080, 0xd3d9402f, 0x18000080, 0xd3d94030,
	113	+ 0x18000080, 0xd3d94031, 0x18000080, 0xd3d94032, 0x18000080, 0xd3d94033,
	114	+ 0x18000080, 0xd3d94034, 0x18000080, 0xd3d94035, 0x18000080, 0xd3d94036,
	115	+ 0x18000080, 0xd3d94037, 0x18000080, 0xd3d94038, 0x18000080, 0xd3d94039,
	116	+ 0x18000080, 0xd3d9403a, 0x18000080, 0xd3d9403b, 0x18000080, 0xd3d9403c,
	117	+ 0x18000080, 0xd3d9403d, 0x18000080, 0xd3d9403e, 0x18000080, 0xd3d9403f,
	118	+ 0x18000080, 0xd3d94040, 0x18000080, 0xd3d94041, 0x18000080, 0xd3d94042,
	119	+ 0x18000080, 0xd3d94043, 0x18000080, 0xd3d94044, 0x18000080, 0xd3d94045,
	120	+ 0x18000080, 0xd3d94046, 0x18000080, 0xd3d94047, 0x18000080, 0xd3d94048,
	121	+ 0x18000080, 0xd3d94049, 0x18000080, 0xd3d9404a, 0x18000080, 0xd3d9404b,
	122	+ 0x18000080, 0xd3d9404c, 0x18000080, 0xd3d9404d, 0x18000080, 0xd3d9404e,
	123	+ 0x18000080, 0xd3d9404f, 0x18000080, 0xd3d94050, 0x18000080, 0xd3d94051,
	124	+ 0x18000080, 0xd3d94052, 0x18000080, 0xd3d94053, 0x18000080, 0xd3d94054,
	125	+ 0x18000080, 0xd3d94055, 0x18000080, 0xd3d94056, 0x18000080, 0xd3d94057,
	126	+ 0x18000080, 0xd3d94058, 0x18000080, 0xd3d94059, 0x18000080, 0xd3d9405a,
	127	+ 0x18000080, 0xd3d9405b, 0x18000080, 0xd3d9405c, 0x18000080, 0xd3d9405d,
	128	+ 0x18000080, 0xd3d9405e, 0x18000080, 0xd3d9405f, 0x18000080, 0xd3d94060,
	129	+ 0x18000080, 0xd3d94061, 0x18000080, 0xd3d94062, 0x18000080, 0xd3d94063,
	130	+ 0x18000080, 0xd3d94064, 0x18000080, 0xd3d94065, 0x18000080, 0xd3d94066,
	131	+ 0x18000080, 0xd3d94067, 0x18000080, 0xd3d94068, 0x18000080, 0xd3d94069,
	132	+ 0x18000080, 0xd3d9406a, 0x18000080, 0xd3d9406b, 0x18000080, 0xd3d9406c,
	133	+ 0x18000080, 0xd3d9406d, 0x18000080, 0xd3d9406e, 0x18000080, 0xd3d9406f,
	134	+ 0x18000080, 0xd3d94070, 0x18000080, 0xd3d94071, 0x18000080, 0xd3d94072,
	135	+ 0x18000080, 0xd3d94073, 0x18000080, 0xd3d94074, 0x18000080, 0xd3d94075,
	136	+ 0x18000080, 0xd3d94076, 0x18000080, 0xd3d94077, 0x18000080, 0xd3d94078,
	137	+ 0x18000080, 0xd3d94079, 0x18000080, 0xd3d9407a, 0x18000080, 0xd3d9407b,
	138	+ 0x18000080, 0xd3d9407c, 0x18000080, 0xd3d9407d, 0x18000080, 0xd3d9407e,
	139	+ 0x18000080, 0xd3d9407f, 0x18000080, 0xd3d94080, 0x18000080, 0xd3d94081,
	140	+ 0x18000080, 0xd3d94082, 0x18000080, 0xd3d94083, 0x18000080, 0xd3d94084,
	141	+ 0x18000080, 0xd3d94085, 0x18000080, 0xd3d94086, 0x18000080, 0xd3d94087,
	142	+ 0x18000080, 0xd3d94088, 0x18000080, 0xd3d94089, 0x18000080, 0xd3d9408a,
	143	+ 0x18000080, 0xd3d9408b, 0x18000080, 0xd3d9408c, 0x18000080, 0xd3d9408d,
	144	+ 0x18000080, 0xd3d9408e, 0x18000080, 0xd3d9408f, 0x18000080, 0xd3d94090,
	145	+ 0x18000080, 0xd3d94091, 0x18000080, 0xd3d94092, 0x18000080, 0xd3d94093,
	146	+ 0x18000080, 0xd3d94094, 0x18000080, 0xd3d94095, 0x18000080, 0xd3d94096,
	147	+ 0x18000080, 0xd3d94097, 0x18000080, 0xd3d94098, 0x18000080, 0xd3d94099,
	148	+ 0x18000080, 0xd3d9409a, 0x18000080, 0xd3d9409b, 0x18000080, 0xd3d9409c,
	149	+ 0x18000080, 0xd3d9409d, 0x18000080, 0xd3d9409e, 0x18000080, 0xd3d9409f,
	150	+ 0x18000080, 0xd3d940a0, 0x18000080, 0xd3d940a1, 0x18000080, 0xd3d940a2,
	151	+ 0x18000080, 0xd3d940a3, 0x18000080, 0xd3d940a4, 0x18000080, 0xd3d940a5,
	152	+ 0x18000080, 0xd3d940a6, 0x18000080, 0xd3d940a7, 0x18000080, 0xd3d940a8,
	153	+ 0x18000080, 0xd3d940a9, 0x18000080, 0xd3d940aa, 0x18000080, 0xd3d940ab,
	154	+ 0x18000080, 0xd3d940ac, 0x18000080, 0xd3d940ad, 0x18000080, 0xd3d940ae,
	155	+ 0x18000080, 0xd3d940af, 0x18000080, 0xd3d940b0, 0x18000080, 0xd3d940b1,
	156	+ 0x18000080, 0xd3d940b2, 0x18000080, 0xd3d940b3, 0x18000080, 0xd3d940b4,
	157	+ 0x18000080, 0xd3d940b5, 0x18000080, 0xd3d940b6, 0x18000080, 0xd3d940b7,
	158	+ 0x18000080, 0xd3d940b8, 0x18000080, 0xd3d940b9, 0x18000080, 0xd3d940ba,
	159	+ 0x18000080, 0xd3d940bb, 0x18000080, 0xd3d940bc, 0x18000080, 0xd3d940bd,
	160	+ 0x18000080, 0xd3d940be, 0x18000080, 0xd3d940bf, 0x18000080, 0xd3d940c0,
	161	+ 0x18000080, 0xd3d940c1, 0x18000080, 0xd3d940c2, 0x18000080, 0xd3d940c3,
	162	+ 0x18000080, 0xd3d940c4, 0x18000080, 0xd3d940c5, 0x18000080, 0xd3d940c6,
	163	+ 0x18000080, 0xd3d940c7, 0x18000080, 0xd3d940c8, 0x18000080, 0xd3d940c9,
	164	+ 0x18000080, 0xd3d940ca, 0x18000080, 0xd3d940cb, 0x18000080, 0xd3d940cc,
	165	+ 0x18000080, 0xd3d940cd, 0x18000080, 0xd3d940ce, 0x18000080, 0xd3d940cf,
	166	+ 0x18000080, 0xd3d940d0, 0x18000080, 0xd3d940d1, 0x18000080, 0xd3d940d2,
	167	+ 0x18000080, 0xd3d940d3, 0x18000080, 0xd3d940d4, 0x18000080, 0xd3d940d5,
	168	+ 0x18000080, 0xd3d940d6, 0x18000080, 0xd3d940d7, 0x18000080, 0xd3d940d8,
	169	+ 0x18000080, 0xd3d940d9, 0x18000080, 0xd3d940da, 0x18000080, 0xd3d940db,
	170	+ 0x18000080, 0xd3d940dc, 0x18000080, 0xd3d940dd, 0x18000080, 0xd3d940de,
	171	+ 0x18000080, 0xd3d940df, 0x18000080, 0xd3d940e0, 0x18000080, 0xd3d940e1,
	172	+ 0x18000080, 0xd3d940e2, 0x18000080, 0xd3d940e3, 0x18000080, 0xd3d940e4,
	173	+ 0x18000080, 0xd3d940e5, 0x18000080, 0xd3d940e6, 0x18000080, 0xd3d940e7,
	174	+ 0x18000080, 0xd3d940e8, 0x18000080, 0xd3d940e9, 0x18000080, 0xd3d940ea,
	175	+ 0x18000080, 0xd3d940eb, 0x18000080, 0xd3d940ec, 0x18000080, 0xd3d940ed,
	176	+ 0x18000080, 0xd3d940ee, 0x18000080, 0xd3d940ef, 0x18000080, 0xd3d940f0,
	177	+ 0x18000080, 0xd3d940f1, 0x18000080, 0xd3d940f2, 0x18000080, 0xd3d940f3,
	178	+ 0x18000080, 0xd3d940f4, 0x18000080, 0xd3d940f5, 0x18000080, 0xd3d940f6,
	179	+ 0x18000080, 0xd3d940f7, 0x18000080, 0xd3d940f8, 0x18000080, 0xd3d940f9,
	180	+ 0x18000080, 0xd3d940fa, 0x18000080, 0xd3d940fb, 0x18000080, 0xd3d940fc,
	181	+ 0x18000080, 0xd3d940fd, 0x18000080, 0xd3d940fe, 0x18000080, 0xd3d940ff,
	182	+ 0x18000080, 0xb07c0000, 0xbe8a00ff, 0x000000f8, 0xbf11080a, 0x7e000280,
	183	+ 0x7e020280, 0x7e040280, 0x7e060280, 0x7e080280, 0x7e0a0280, 0x7e0c0280,
	184	+ 0x7e0e0280, 0x808a880a, 0xbe80320a, 0xbf84fff5, 0xbf9c0000, 0xd28c0001,
	185	+ 0x0001007f, 0xd28d0001, 0x0002027e, 0x10020288, 0xbe8b0004, 0xb78b4000,
	186	+ 0xd1196a01, 0x00001701, 0xbe8a0087, 0xbefc00c1, 0xd89c4000, 0x00020201,
	187	+ 0xd89cc080, 0x00040401, 0x320202ff, 0x00000800, 0x808a810a, 0xbf84fff8,
	188	+ 0xbf810000,
	189	+};
	190	+
	191	+const struct soc15_reg_entry vgpr_init_regs_aldebaran[] = {
	192	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
	193	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_X), 0x40 },
	194	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Y), 4 },
	195	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Z), 1 },
	196	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC1), 0xbf },
	197	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC2), 0x400006 }, /* 64KB LDS */
	198	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC3), 0x3F }, /* 63 - accum-offset = 256 */
	199	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
	200	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
	201	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
	202	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
	203	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
	204	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
	205	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
	206	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
	207	+};
	208	+
	209	+/**
	210	+ * The below shaders are used to clear SGPRS, and also write the input
	211	+ * pattern into the write back buffer. The first two dispatch should be
	212	+ * scheduled simultaneously which make sure that all SGPRS could be
	213	+ * allocated, so the dispatch 1 need check write back buffer before scheduled,
	214	+ * make sure that waves of dispatch 0 are all dispacthed to all simds
	215	+ * balanced. both dispatch 0 and dispatch 1 should be halted until all waves
	216	+ * are dispatched, and then driver write a pattern to the shared memory to make
	217	+ * all waves continue.
	218	+*/
	219	+static const u32 sgpr112_init_compute_shader_aldebaran[] = {
	220	+ 0xb8840904, 0xb8851a04, 0xb8861344, 0xb8831804, 0x9208ff06, 0x00000280,
	221	+ 0x9209a805, 0x920a8a04, 0x81080908, 0x81080a08, 0x81080308, 0x8e078208,
	222	+ 0x81078407, 0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8e003f, 0xc0030200,
	223	+ 0x00000000, 0xbf8c0000, 0xbf06ff08, 0xdeadbeaf, 0xbf84fff9, 0x81028102,
	224	+ 0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8a0000, 0xbefc0080, 0xbeea0080,
	225	+ 0xbeeb0080, 0xbf00f280, 0xbee60080, 0xbee70080, 0xbee80080, 0xbee90080,
	226	+ 0xbefe0080, 0xbeff0080, 0xbe880080, 0xbe890080, 0xbe8a0080, 0xbe8b0080,
	227	+ 0xbe8c0080, 0xbe8d0080, 0xbe8e0080, 0xbe8f0080, 0xbe900080, 0xbe910080,
	228	+ 0xbe920080, 0xbe930080, 0xbe940080, 0xbe950080, 0xbe960080, 0xbe970080,
	229	+ 0xbe980080, 0xbe990080, 0xbe9a0080, 0xbe9b0080, 0xbe9c0080, 0xbe9d0080,
	230	+ 0xbe9e0080, 0xbe9f0080, 0xbea00080, 0xbea10080, 0xbea20080, 0xbea30080,
	231	+ 0xbea40080, 0xbea50080, 0xbea60080, 0xbea70080, 0xbea80080, 0xbea90080,
	232	+ 0xbeaa0080, 0xbeab0080, 0xbeac0080, 0xbead0080, 0xbeae0080, 0xbeaf0080,
	233	+ 0xbeb00080, 0xbeb10080, 0xbeb20080, 0xbeb30080, 0xbeb40080, 0xbeb50080,
	234	+ 0xbeb60080, 0xbeb70080, 0xbeb80080, 0xbeb90080, 0xbeba0080, 0xbebb0080,
	235	+ 0xbebc0080, 0xbebd0080, 0xbebe0080, 0xbebf0080, 0xbec00080, 0xbec10080,
	236	+ 0xbec20080, 0xbec30080, 0xbec40080, 0xbec50080, 0xbec60080, 0xbec70080,
	237	+ 0xbec80080, 0xbec90080, 0xbeca0080, 0xbecb0080, 0xbecc0080, 0xbecd0080,
	238	+ 0xbece0080, 0xbecf0080, 0xbed00080, 0xbed10080, 0xbed20080, 0xbed30080,
	239	+ 0xbed40080, 0xbed50080, 0xbed60080, 0xbed70080, 0xbed80080, 0xbed90080,
	240	+ 0xbeda0080, 0xbedb0080, 0xbedc0080, 0xbedd0080, 0xbede0080, 0xbedf0080,
	241	+ 0xbee00080, 0xbee10080, 0xbee20080, 0xbee30080, 0xbee40080, 0xbee50080,
	242	+ 0xbf810000
	243	+};
	244	+
	245	+const struct soc15_reg_entry sgpr112_init_regs_aldebaran[] = {
	246	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
	247	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_X), 0x40 },
	248	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Y), 8 },
	249	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Z), 1 },
	250	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC1), 0x340 },
	251	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC2), 0x6 },
	252	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC3), 0x0 },
	253	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
	254	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
	255	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
	256	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
	257	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
	258	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
	259	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
	260	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
	261	+};
	262	+
	263	+static const u32 sgpr96_init_compute_shader_aldebaran[] = {
	264	+ 0xb8840904, 0xb8851a04, 0xb8861344, 0xb8831804, 0x9208ff06, 0x00000280,
	265	+ 0x9209a805, 0x920a8a04, 0x81080908, 0x81080a08, 0x81080308, 0x8e078208,
	266	+ 0x81078407, 0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8e003f, 0xc0030200,
	267	+ 0x00000000, 0xbf8c0000, 0xbf06ff08, 0xdeadbeaf, 0xbf84fff9, 0x81028102,
	268	+ 0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8a0000, 0xbefc0080, 0xbeea0080,
	269	+ 0xbeeb0080, 0xbf00f280, 0xbee60080, 0xbee70080, 0xbee80080, 0xbee90080,
	270	+ 0xbefe0080, 0xbeff0080, 0xbe880080, 0xbe890080, 0xbe8a0080, 0xbe8b0080,
	271	+ 0xbe8c0080, 0xbe8d0080, 0xbe8e0080, 0xbe8f0080, 0xbe900080, 0xbe910080,
	272	+ 0xbe920080, 0xbe930080, 0xbe940080, 0xbe950080, 0xbe960080, 0xbe970080,
	273	+ 0xbe980080, 0xbe990080, 0xbe9a0080, 0xbe9b0080, 0xbe9c0080, 0xbe9d0080,
	274	+ 0xbe9e0080, 0xbe9f0080, 0xbea00080, 0xbea10080, 0xbea20080, 0xbea30080,
	275	+ 0xbea40080, 0xbea50080, 0xbea60080, 0xbea70080, 0xbea80080, 0xbea90080,
	276	+ 0xbeaa0080, 0xbeab0080, 0xbeac0080, 0xbead0080, 0xbeae0080, 0xbeaf0080,
	277	+ 0xbeb00080, 0xbeb10080, 0xbeb20080, 0xbeb30080, 0xbeb40080, 0xbeb50080,
	278	+ 0xbeb60080, 0xbeb70080, 0xbeb80080, 0xbeb90080, 0xbeba0080, 0xbebb0080,
	279	+ 0xbebc0080, 0xbebd0080, 0xbebe0080, 0xbebf0080, 0xbec00080, 0xbec10080,
	280	+ 0xbec20080, 0xbec30080, 0xbec40080, 0xbec50080, 0xbec60080, 0xbec70080,
	281	+ 0xbec80080, 0xbec90080, 0xbeca0080, 0xbecb0080, 0xbecc0080, 0xbecd0080,
	282	+ 0xbece0080, 0xbecf0080, 0xbed00080, 0xbed10080, 0xbed20080, 0xbed30080,
	283	+ 0xbed40080, 0xbed50080, 0xbed60080, 0xbed70080, 0xbed80080, 0xbed90080,
	284	+ 0xbf810000,
	285	+};
	286	+
	287	+const struct soc15_reg_entry sgpr96_init_regs_aldebaran[] = {
	288	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
	289	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_X), 0x40 },
	290	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Y), 0xc },
	291	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Z), 1 },
	292	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC1), 0x2c0 },
	293	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC2), 0x6 },
	294	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC3), 0x0 },
	295	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
	296	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
	297	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
	298	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
	299	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
	300	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
	301	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
	302	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
	303	+};
	304	+
	305	+/**
	306	+ * This shader is used to clear the uninitiated sgprs after the above
	307	+ * two dispatches, because of hardware feature, dispath 0 couldn't clear
	308	+ * top hole sgprs. Therefore need 4 waves per SIMD to cover these sgprs
	309	+*/
	310	+static const u32 sgpr64_init_compute_shader_aldebaran[] = {
	311	+ 0xb8840904, 0xb8851a04, 0xb8861344, 0xb8831804, 0x9208ff06, 0x00000280,
	312	+ 0x9209a805, 0x920a8a04, 0x81080908, 0x81080a08, 0x81080308, 0x8e078208,
	313	+ 0x81078407, 0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8e003f, 0xc0030200,
	314	+ 0x00000000, 0xbf8c0000, 0xbf06ff08, 0xdeadbeaf, 0xbf84fff9, 0x81028102,
	315	+ 0xc0410080, 0x00000007, 0xbf8c0000, 0xbf8a0000, 0xbefc0080, 0xbeea0080,
	316	+ 0xbeeb0080, 0xbf00f280, 0xbee60080, 0xbee70080, 0xbee80080, 0xbee90080,
	317	+ 0xbefe0080, 0xbeff0080, 0xbe880080, 0xbe890080, 0xbe8a0080, 0xbe8b0080,
	318	+ 0xbe8c0080, 0xbe8d0080, 0xbe8e0080, 0xbe8f0080, 0xbe900080, 0xbe910080,
	319	+ 0xbe920080, 0xbe930080, 0xbe940080, 0xbe950080, 0xbe960080, 0xbe970080,
	320	+ 0xbe980080, 0xbe990080, 0xbe9a0080, 0xbe9b0080, 0xbe9c0080, 0xbe9d0080,
	321	+ 0xbe9e0080, 0xbe9f0080, 0xbea00080, 0xbea10080, 0xbea20080, 0xbea30080,
	322	+ 0xbea40080, 0xbea50080, 0xbea60080, 0xbea70080, 0xbea80080, 0xbea90080,
	323	+ 0xbeaa0080, 0xbeab0080, 0xbeac0080, 0xbead0080, 0xbeae0080, 0xbeaf0080,
	324	+ 0xbeb00080, 0xbeb10080, 0xbeb20080, 0xbeb30080, 0xbeb40080, 0xbeb50080,
	325	+ 0xbeb60080, 0xbeb70080, 0xbeb80080, 0xbeb90080, 0xbf810000,
	326	+};
	327	+
	328	+const struct soc15_reg_entry sgpr64_init_regs_aldebaran[] = {
	329	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_RESOURCE_LIMITS), 0x0000000 },
	330	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_X), 0x40 },
	331	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Y), 0x10 },
	332	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_NUM_THREAD_Z), 1 },
	333	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC1), 0x1c0 },
	334	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC2), 0x6 },
	335	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_PGM_RSRC3), 0x0 },
	336	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE0), 0xffffffff },
	337	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE1), 0xffffffff },
	338	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE2), 0xffffffff },
	339	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE3), 0xffffffff },
	340	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE4), 0xffffffff },
	341	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE5), 0xffffffff },
	342	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE6), 0xffffffff },
	343	+ { SOC15_REG_ENTRY(GC, 0, regCOMPUTE_STATIC_THREAD_MGMT_SE7), 0xffffffff },
	344	+};
	345	+
	346	+static int gfx_v9_4_2_run_shader(struct amdgpu_device *adev,
	347	+ struct amdgpu_ring *ring,
	348	+ struct amdgpu_ib *ib,
	349	+ const u32 *shader_ptr, u32 shader_size,
	350	+ const struct soc15_reg_entry *init_regs, u32 regs_size,
	351	+ u32 compute_dim_x, u64 wb_gpu_addr, u32 pattern,
	352	+ struct dma_fence **fence_ptr)
	353	+{
	354	+ int r, i;
	355	+ uint32_t total_size, shader_offset;
	356	+ u64 gpu_addr;
	357	+
	358	+ total_size = (regs_size * 3 + 4 + 5 + 5) * 4;
	359	+ total_size = ALIGN(total_size, 256);
	360	+ shader_offset = total_size;
	361	+ total_size += ALIGN(shader_size, 256);
	362	+
	363	+ /* allocate an indirect buffer to put the commands in */
	364	+ memset(ib, 0, sizeof(*ib));
	365	+ r = amdgpu_ib_get(adev, NULL, total_size,
	366	+ AMDGPU_IB_POOL_DIRECT, ib);
	367	+ if (r) {
	368	+ dev_err(adev->dev, "failed to get ib (%d).\n", r);
	369	+ return r;
	370	+ }
	371	+
	372	+ /* load the compute shaders */
	373	+ for (i = 0; i < shader_size/sizeof(u32); i++)
	374	+ ib->ptr[i + (shader_offset / 4)] = shader_ptr[i];
	375	+
	376	+ /* init the ib length to 0 */
	377	+ ib->length_dw = 0;
	378	+
	379	+ /* write the register state for the compute dispatch */
	380	+ for (i = 0; i < regs_size; i++) {
	381	+ ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 1);
	382	+ ib->ptr[ib->length_dw++] = SOC15_REG_ENTRY_OFFSET(init_regs[i])
	383	+ - PACKET3_SET_SH_REG_START;
	384	+ ib->ptr[ib->length_dw++] = init_regs[i].reg_value;
	385	+ }
	386	+
	387	+ /* write the shader start address: mmCOMPUTE_PGM_LO, mmCOMPUTE_PGM_HI */
	388	+ gpu_addr = (ib->gpu_addr + (u64)shader_offset) >> 8;
	389	+ ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 2);
	390	+ ib->ptr[ib->length_dw++] = SOC15_REG_OFFSET(GC, 0, regCOMPUTE_PGM_LO)
	391	+ - PACKET3_SET_SH_REG_START;
	392	+ ib->ptr[ib->length_dw++] = lower_32_bits(gpu_addr);
	393	+ ib->ptr[ib->length_dw++] = upper_32_bits(gpu_addr);
	394	+
	395	+ /* write the wb buffer address */
	396	+ ib->ptr[ib->length_dw++] = PACKET3(PACKET3_SET_SH_REG, 3);
	397	+ ib->ptr[ib->length_dw++] = SOC15_REG_OFFSET(GC, 0, regCOMPUTE_USER_DATA_0)
	398	+ - PACKET3_SET_SH_REG_START;
	399	+ ib->ptr[ib->length_dw++] = lower_32_bits(wb_gpu_addr);
	400	+ ib->ptr[ib->length_dw++] = upper_32_bits(wb_gpu_addr);
	401	+ ib->ptr[ib->length_dw++] = pattern;
	402	+
	403	+ /* write dispatch packet */
	404	+ ib->ptr[ib->length_dw++] = PACKET3(PACKET3_DISPATCH_DIRECT, 3);
	405	+ ib->ptr[ib->length_dw++] = compute_dim_x; /* x */
	406	+ ib->ptr[ib->length_dw++] = 1; /* y */
	407	+ ib->ptr[ib->length_dw++] = 1; /* z */
	408	+ ib->ptr[ib->length_dw++] =
	409	+ REG_SET_FIELD(0, COMPUTE_DISPATCH_INITIATOR, COMPUTE_SHADER_EN, 1);
	410	+
	411	+ /* shedule the ib on the ring */
	412	+ r = amdgpu_ib_schedule(ring, 1, ib, NULL, fence_ptr);
	413	+ if (r) {
	414	+ dev_err(adev->dev, "ib submit failed (%d).\n", r);
	415	+ amdgpu_ib_free(adev, ib, NULL);
	416	+ }
	417	+ return r;
	418	+}
	419	+
	420	+static void gfx_v9_4_2_log_wave_assignment(struct amdgpu_device adev, uint32_t wb_ptr)
	421	+{
	422	+ uint32_t se, cu, simd, wave;
	423	+ uint32_t offset = 0;
	424	+ char *str;
	425	+ int size;
	426	+
	427	+ str = kmalloc(256, GFP_KERNEL);
	428	+ if (!str)
	429	+ return;
	430	+
	431	+ dev_dbg(adev->dev, "wave assignment:\n");
	432	+
	433	+ for (se = 0; se < adev->gfx.config.max_shader_engines; se++) {
	434	+ for (cu = 0; cu < CU_ID_MAX; cu++) {
	435	+ memset(str, 0, 256);
	436	+ size = sprintf(str, "SE[%02d]CU[%02d]: ", se, cu);
	437	+ for (simd = 0; simd < SIMD_ID_MAX; simd++) {
	438	+ size += sprintf(str + size, "[");
	439	+ for (wave = 0; wave < WAVE_ID_MAX; wave++) {
	440	+ size += sprintf(str + size, "%x", wb_ptr[offset]);
	441	+ offset++;
	442	+ }
	443	+ size += sprintf(str + size, "] ");
	444	+ }
	445	+ dev_dbg(adev->dev, "%s\n", str);
	446	+ }
	447	+ }
	448	+
	449	+ kfree(str);
	450	+}
	451	+
	452	+static int gfx_v9_4_2_wait_for_waves_assigned(struct amdgpu_device *adev,
	453	+ uint32_t *wb_ptr, uint32_t mask,
	454	+ uint32_t pattern, uint32_t num_wave, bool wait)
	455	+{
	456	+ uint32_t se, cu, simd, wave;
	457	+ uint32_t loop = 0;
	458	+ uint32_t wave_cnt;
	459	+ uint32_t offset;
	460	+
	461	+ do {
	462	+ wave_cnt = 0;
	463	+ offset = 0;
	464	+
	465	+ for (se = 0; se < adev->gfx.config.max_shader_engines; se++)
	466	+ for (cu = 0; cu < CU_ID_MAX; cu++)
	467	+ for (simd = 0; simd < SIMD_ID_MAX; simd++)
	468	+ for (wave = 0; wave < WAVE_ID_MAX; wave++) {
	469	+ if (((1 << wave) & mask) &&
	470	+ (wb_ptr[offset] == pattern))
	471	+ wave_cnt++;
	472	+
	473	+ offset++;
	474	+ }
	475	+
	476	+ if (wave_cnt == num_wave)
	477	+ return 0;
	478	+
	479	+ mdelay(1);
	480	+ } while (++loop < 2000 && wait);
	481	+
	482	+ dev_err(adev->dev, "actual wave num: %d, expected wave num: %d\n",
	483	+ wave_cnt, num_wave);
	484	+
	485	+ gfx_v9_4_2_log_wave_assignment(adev, wb_ptr);
	486	+
	487	+ return -EBADSLT;
	488	+}
	489	+
	490	+static int gfx_v9_4_2_do_sgprs_init(struct amdgpu_device *adev)
	491	+{
	492	+ int r;
	493	+ int wb_size = adev->gfx.config.max_shader_engines *
	494	+ CU_ID_MAX * SIMD_ID_MAX * WAVE_ID_MAX;
	495	+ struct amdgpu_ib wb_ib;
	496	+ struct amdgpu_ib disp_ibs[3];
	497	+ struct dma_fence *fences[3];
	498	+ u32 pattern[3] = { 0x1, 0x5, 0xa };
	499	+
	500	+ /* bail if the compute ring is not ready */
	501	+ if (!adev->gfx.compute_ring[0].sched.ready \|\|
	502	+ !adev->gfx.compute_ring[1].sched.ready)
	503	+ return 0;
	504	+
	505	+ /* allocate the write-back buffer from IB */
	506	+ memset(&wb_ib, 0, sizeof(wb_ib));
	507	+ r = amdgpu_ib_get(adev, NULL, (1 + wb_size) * sizeof(uint32_t),
	508	+ AMDGPU_IB_POOL_DIRECT, &wb_ib);
	509	+ if (r) {
	510	+ dev_err(adev->dev, "failed to get ib (%d) for wb\n", r);
	511	+ return r;
	512	+ }
	513	+ memset(wb_ib.ptr, 0, (1 + wb_size) * sizeof(uint32_t));
	514	+
	515	+ r = gfx_v9_4_2_run_shader(adev,
	516	+ &adev->gfx.compute_ring[0],
	517	+ &disp_ibs[0],
	518	+ sgpr112_init_compute_shader_aldebaran,
	519	+ sizeof(sgpr112_init_compute_shader_aldebaran),
	520	+ sgpr112_init_regs_aldebaran,
	521	+ ARRAY_SIZE(sgpr112_init_regs_aldebaran),
	522	+ adev->gfx.cu_info.number,
	523	+ wb_ib.gpu_addr, pattern[0], &fences[0]);
	524	+ if (r) {
	525	+ dev_err(adev->dev, "failed to clear first 224 sgprs\n");
	526	+ goto pro_end;
	527	+ }
	528	+
	529	+ r = gfx_v9_4_2_wait_for_waves_assigned(adev,
	530	+ &wb_ib.ptr[1], 0b11,
	531	+ pattern[0],
	532	+ adev->gfx.cu_info.number * SIMD_ID_MAX * 2,
	533	+ true);
	534	+ if (r) {
	535	+ dev_err(adev->dev, "wave coverage failed when clear first 224 sgprs\n");
	536	+ wb_ib.ptr[0] = 0xdeadbeaf; /* stop waves */
	537	+ goto disp0_failed;
	538	+ }
	539	+
	540	+ r = gfx_v9_4_2_run_shader(adev,
	541	+ &adev->gfx.compute_ring[1],
	542	+ &disp_ibs[1],
	543	+ sgpr96_init_compute_shader_aldebaran,
	544	+ sizeof(sgpr96_init_compute_shader_aldebaran),
	545	+ sgpr96_init_regs_aldebaran,
	546	+ ARRAY_SIZE(sgpr96_init_regs_aldebaran),
	547	+ adev->gfx.cu_info.number * 2,
	548	+ wb_ib.gpu_addr, pattern[1], &fences[1]);
	549	+ if (r) {
	550	+ dev_err(adev->dev, "failed to clear next 576 sgprs\n");
	551	+ goto disp0_failed;
	552	+ }
	553	+
	554	+ r = gfx_v9_4_2_wait_for_waves_assigned(adev,
	555	+ &wb_ib.ptr[1], 0b11111100,
	556	+ pattern[1], adev->gfx.cu_info.number * SIMD_ID_MAX * 6,
	557	+ true);
	558	+ if (r) {
	559	+ dev_err(adev->dev, "wave coverage failed when clear first 576 sgprs\n");
	560	+ wb_ib.ptr[0] = 0xdeadbeaf; /* stop waves */
	561	+ goto disp1_failed;
	562	+ }
	563	+
	564	+ wb_ib.ptr[0] = 0xdeadbeaf; /* stop waves */
	565	+
	566	+ /* wait for the GPU to finish processing the IB */
	567	+ r = dma_fence_wait(fences[0], false);
	568	+ if (r) {
	569	+ dev_err(adev->dev, "timeout to clear first 224 sgprs\n");
	570	+ goto disp1_failed;
	571	+ }
	572	+
	573	+ r = dma_fence_wait(fences[1], false);
	574	+ if (r) {
	575	+ dev_err(adev->dev, "timeout to clear first 576 sgprs\n");
	576	+ goto disp1_failed;
	577	+ }
	578	+
	579	+ memset(wb_ib.ptr, 0, (1 + wb_size) * sizeof(uint32_t));
	580	+ r = gfx_v9_4_2_run_shader(adev,
	581	+ &adev->gfx.compute_ring[0],
	582	+ &disp_ibs[2],
	583	+ sgpr64_init_compute_shader_aldebaran,
	584	+ sizeof(sgpr64_init_compute_shader_aldebaran),
	585	+ sgpr64_init_regs_aldebaran,
	586	+ ARRAY_SIZE(sgpr64_init_regs_aldebaran),
	587	+ adev->gfx.cu_info.number,
	588	+ wb_ib.gpu_addr, pattern[2], &fences[2]);
	589	+ if (r) {
	590	+ dev_err(adev->dev, "failed to clear first 256 sgprs\n");
	591	+ goto disp1_failed;
	592	+ }
	593	+
	594	+ r = gfx_v9_4_2_wait_for_waves_assigned(adev,
	595	+ &wb_ib.ptr[1], 0b1111,
	596	+ pattern[2],
	597	+ adev->gfx.cu_info.number * SIMD_ID_MAX * 4,
	598	+ true);
	599	+ if (r) {
	600	+ dev_err(adev->dev, "wave coverage failed when clear first 256 sgprs\n");
	601	+ wb_ib.ptr[0] = 0xdeadbeaf; /* stop waves */
	602	+ goto disp2_failed;
	603	+ }
	604	+
	605	+ wb_ib.ptr[0] = 0xdeadbeaf; /* stop waves */
	606	+
	607	+ r = dma_fence_wait(fences[2], false);
	608	+ if (r) {
	609	+ dev_err(adev->dev, "timeout to clear first 256 sgprs\n");
	610	+ goto disp2_failed;
	611	+ }
	612	+
	613	+disp2_failed:
	614	+ amdgpu_ib_free(adev, &disp_ibs[2], NULL);
	615	+ dma_fence_put(fences[2]);
	616	+disp1_failed:
	617	+ amdgpu_ib_free(adev, &disp_ibs[1], NULL);
	618	+ dma_fence_put(fences[1]);
	619	+disp0_failed:
	620	+ amdgpu_ib_free(adev, &disp_ibs[0], NULL);
	621	+ dma_fence_put(fences[0]);
	622	+pro_end:
	623	+ amdgpu_ib_free(adev, &wb_ib, NULL);
	624	+
	625	+ if (r)
	626	+ dev_info(adev->dev, "Init SGPRS Failed\n");
	627	+ else
	628	+ dev_info(adev->dev, "Init SGPRS Successfully\n");
	629	+
	630	+ return r;
	631	+}
	632	+
	633	+static int gfx_v9_4_2_do_vgprs_init(struct amdgpu_device *adev)
	634	+{
	635	+ int r;
	636	+ /* CU_ID: 0~15, SIMD_ID: 0~3, WAVE_ID: 0 ~ 9 */
	637	+ int wb_size = adev->gfx.config.max_shader_engines *
	638	+ CU_ID_MAX * SIMD_ID_MAX * WAVE_ID_MAX;
	639	+ struct amdgpu_ib wb_ib;
	640	+ struct amdgpu_ib disp_ib;
	641	+ struct dma_fence *fence;
	642	+ u32 pattern = 0xa;
	643	+
	644	+ /* bail if the compute ring is not ready */
	645	+ if (!adev->gfx.compute_ring[0].sched.ready)
	646	+ return 0;
	647	+
	648	+ /* allocate the write-back buffer from IB */
	649	+ memset(&wb_ib, 0, sizeof(wb_ib));
	650	+ r = amdgpu_ib_get(adev, NULL, (1 + wb_size) * sizeof(uint32_t),
	651	+ AMDGPU_IB_POOL_DIRECT, &wb_ib);
	652	+ if (r) {
	653	+ dev_err(adev->dev, "failed to get ib (%d) for wb.\n", r);
	654	+ return r;
	655	+ }
	656	+ memset(wb_ib.ptr, 0, (1 + wb_size) * sizeof(uint32_t));
	657	+
	658	+ r = gfx_v9_4_2_run_shader(adev,
	659	+ &adev->gfx.compute_ring[0],
	660	+ &disp_ib,
	661	+ vgpr_init_compute_shader_aldebaran,
	662	+ sizeof(vgpr_init_compute_shader_aldebaran),
	663	+ vgpr_init_regs_aldebaran,
	664	+ ARRAY_SIZE(vgpr_init_regs_aldebaran),
	665	+ adev->gfx.cu_info.number,
	666	+ wb_ib.gpu_addr, pattern, &fence);
	667	+ if (r) {
	668	+ dev_err(adev->dev, "failed to clear vgprs\n");
	669	+ goto pro_end;
	670	+ }
	671	+
	672	+ /* wait for the GPU to finish processing the IB */
	673	+ r = dma_fence_wait(fence, false);
	674	+ if (r) {
	675	+ dev_err(adev->dev, "timeout to clear vgprs\n");
	676	+ goto disp_failed;
	677	+ }
	678	+
	679	+ r = gfx_v9_4_2_wait_for_waves_assigned(adev,
	680	+ &wb_ib.ptr[1], 0b1,
	681	+ pattern,
	682	+ adev->gfx.cu_info.number * SIMD_ID_MAX,
	683	+ false);
	684	+ if (r) {
	685	+ dev_err(adev->dev, "failed to cover all simds when clearing vgprs\n");
	686	+ goto disp_failed;
	687	+ }
	688	+
	689	+disp_failed:
	690	+ amdgpu_ib_free(adev, &disp_ib, NULL);
	691	+ dma_fence_put(fence);
	692	+pro_end:
	693	+ amdgpu_ib_free(adev, &wb_ib, NULL);
	694	+
	695	+ if (r)
	696	+ dev_info(adev->dev, "Init VGPRS Failed\n");
	697	+ else
	698	+ dev_info(adev->dev, "Init VGPRS Successfully\n");
	699	+
	700	+ return r;
	701	+}
	702	+
	703	+int gfx_v9_4_2_do_edc_gpr_workarounds(struct amdgpu_device *adev)
	704	+{
	705	+ /* only support when RAS is enabled */
	706	+ if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__GFX))
	707	+ return 0;
	708	+
	709	+ gfx_v9_4_2_do_sgprs_init(adev);
	710	+
	711	+ gfx_v9_4_2_do_vgprs_init(adev);
	712	+
	713	+ return 0;
	714	+}
	715	+
82	716	static void gfx_v9_4_2_query_sq_timeout_status(struct amdgpu_device *adev);
83	717	static void gfx_v9_4_2_reset_sq_timeout_status(struct amdgpu_device *adev);
84	718
@@ -808,8 +1442,9 @@ static struct gfx_v9_4_2_utc_block gfx_v9_4_2_utc_blocks[] = { · 723 unmodified lines
808	1442	REG_SET_FIELD(0, ATC_L2_CACHE_4K_DSM_CNTL, WRITE_COUNTERS, 1) },
809	1443	};
810	1444
811		−static const struct soc15_reg_entry gfx_v9_4_2_ea_err_status_regs =
812		− { SOC15_REG_ENTRY(GC, 0, regGCEA_ERR_STATUS), 0, 1, 16 };
	1445	+static const struct soc15_reg_entry gfx_v9_4_2_ea_err_status_regs = {
	1446	+ SOC15_REG_ENTRY(GC, 0, regGCEA_ERR_STATUS), 0, 1, 16
	1447	+};
813	1448
814	1449	static int gfx_v9_4_2_get_reg_error_count(struct amdgpu_device *adev,
815	1450	const struct soc15_reg_entry *reg,
@@ -1039,13 +1674,16 @@ static void gfx_v9_4_2_reset_utc_err_status(struct amdgpu_device *adev) · 223 unmodified lines
1039	1674	static void gfx_v9_4_2_reset_ea_err_status(struct amdgpu_device *adev)
1040	1675	{
1041	1676	uint32_t i, j;
	1677	+ uint32_t value;
	1678	+
	1679	+ value = REG_SET_FIELD(0, GCEA_ERR_STATUS, CLEAR_ERROR_STATUS, 0x1);
1042	1680
1043	1681	mutex_lock(&adev->grbm_idx_mutex);
1044	1682	for (i = 0; i < gfx_v9_4_2_ea_err_status_regs.se_num; i++) {
1045	1683	for (j = 0; j < gfx_v9_4_2_ea_err_status_regs.instance;
1046	1684	j++) {
1047	1685	gfx_v9_4_2_select_se_sh(adev, i, 0, j);
1048		− WREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_4_2_ea_err_status_regs), 0x10);
	1686	+ WREG32(SOC15_REG_ENTRY_OFFSET(gfx_v9_4_2_ea_err_status_regs), value);
1049	1687	}
1050	1688	}
1051	1689	gfx_v9_4_2_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);

drivers/gpu/drm/amd/amdgpu/gfx_v9_4_2.h+1−0

@@ -29,6 +29,7 @@ void gfx_v9_4_2_debug_trap_config_init(struct amdgpu_device *adev, · 28 unmodified lines
29	29	void gfx_v9_4_2_init_golden_registers(struct amdgpu_device *adev,
30	30	uint32_t die_id);
31	31	void gfx_v9_4_2_set_power_brake_sequence(struct amdgpu_device *adev);
	32	+int gfx_v9_4_2_do_edc_gpr_workarounds(struct amdgpu_device *adev);
32	33
33	34	extern const struct amdgpu_gfx_ras_funcs gfx_v9_4_2_ras_funcs;
34	35

drivers/gpu/drm/amd/amdgpu/gfxhub_v1_0.c+6−2

@@ -283,10 +283,14 @@ static void gfxhub_v1_0_setup_vmid_config(struct amdgpu_device *adev) · 282 unmodified lines
283	283	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
284	284	PAGE_TABLE_BLOCK_SIZE,
285	285	block_size);
286		− /* Send no-retry XNACK on fault to suppress VM fault storm. */
	286	+ /* Send no-retry XNACK on fault to suppress VM fault storm.
	287	+ * On Aldebaran, XNACK can be enabled in the SQ per-process.
	288	+ * Retry faults need to be enabled for that to work.
	289	+ */
287	290	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
288	291	RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
289		− !adev->gmc.noretry);
	292	+ !adev->gmc.noretry \|\|
	293	+ adev->asic_type == CHIP_ALDEBARAN);
290	294	WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_CNTL,
291	295	i * hub->ctx_distance, tmp);
292	296	WREG32_SOC15_OFFSET(GC, 0, mmVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,

drivers/gpu/drm/amd/amdgpu/gmc_v9_0.c+12−1

@@ -53,6 +53,7 @@ · 52 unmodified lines
53	53	#include "mmhub_v1_7.h"
54	54	#include "umc_v6_1.h"
55	55	#include "umc_v6_0.h"
	56	+#include "hdp_v4_0.h"
56	57
57	58	#include "ivsrcid/vmc/irqsrcs_vmc_1_0.h"
58	59
@@ -1210,6 +1211,11 @@ static void gmc_v9_0_set_gfxhub_funcs(struct amdgpu_device *adev) · 1151 unmodified lines
1210	1211	adev->gfxhub.funcs = &gfxhub_v1_0_funcs;
1211	1212	}
1212	1213
	1214	+static void gmc_v9_0_set_hdp_ras_funcs(struct amdgpu_device *adev)
	1215	+{
	1216	+ adev->hdp.ras_funcs = &hdp_v4_0_ras_funcs;
	1217	+}
	1218	+
1213	1219	static int gmc_v9_0_early_init(void *handle)
1214	1220	{
1215	1221	struct amdgpu_device adev = (struct amdgpu_device )handle;
@@ -1230,6 +1236,7 @@ static int gmc_v9_0_early_init(void *handle) · 14 unmodified lines
1230	1236	gmc_v9_0_set_mmhub_funcs(adev);
1231	1237	gmc_v9_0_set_mmhub_ras_funcs(adev);
1232	1238	gmc_v9_0_set_gfxhub_funcs(adev);
	1239	+ gmc_v9_0_set_hdp_ras_funcs(adev);
1233	1240
1234	1241	adev->gmc.shared_aperture_start = 0x2000000000000000ULL;
1235	1242	adev->gmc.shared_aperture_end =
@@ -1255,7 +1262,7 @@ static int gmc_v9_0_late_init(void *handle) · 19 unmodified lines
1255	1262	* writes, while disables HBM ECC for vega10.
1256	1263	*/
1257	1264	if (!amdgpu_sriov_vf(adev) && (adev->asic_type == CHIP_VEGA10)) {
1258		− if (!(adev->ras_features & (1 << AMDGPU_RAS_BLOCK__UMC))) {
	1265	+ if (!(adev->ras_enabled & (1 << AMDGPU_RAS_BLOCK__UMC))) {
1259	1266	if (adev->df.funcs->enable_ecc_force_par_wr_rmw)
1260	1267	adev->df.funcs->enable_ecc_force_par_wr_rmw(adev, false);
1261	1268	}
@@ -1265,6 +1272,10 @@ static int gmc_v9_0_late_init(void *handle) · 3 unmodified lines
1265	1272	adev->mmhub.ras_funcs->reset_ras_error_count)
1266	1273	adev->mmhub.ras_funcs->reset_ras_error_count(adev);
1267	1274
	1275	+ if (adev->hdp.ras_funcs &&
	1276	+ adev->hdp.ras_funcs->reset_ras_error_count)
	1277	+ adev->hdp.ras_funcs->reset_ras_error_count(adev);
	1278	+
1268	1279	r = amdgpu_gmc_ras_late_init(adev);
1269	1280	if (r)
1270	1281	return r;

drivers/gpu/drm/amd/amdgpu/hdp_v4_0.c+28−3

@@ -59,12 +59,31 @@ static void hdp_v4_0_invalidate_hdp(struct amdgpu_device *adev, · 58 unmodified lines
59	59	HDP, 0, mmHDP_READ_CACHE_INVALIDATE), 1);
60	60	}
61	61
	62	+static void hdp_v4_0_query_ras_error_count(struct amdgpu_device *adev,
	63	+ void *ras_error_status)
	64	+{
	65	+ struct ras_err_data err_data = (struct ras_err_data )ras_error_status;
	66	+
	67	+ err_data->ue_count = 0;
	68	+ err_data->ce_count = 0;
	69	+
	70	+ if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP))
	71	+ return;
	72	+
	73	+ /* HDP SRAM errors are uncorrectable ones (i.e. fatal errors) */
	74	+ err_data->ue_count += RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT);
	75	+};
	76	+
62	77	static void hdp_v4_0_reset_ras_error_count(struct amdgpu_device *adev)
63	78	{
64	79	if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__HDP))
65	80	return;
66		− /read back hdp ras counter to reset it to 0 /
67		− RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT);
	81	+
	82	+ if (adev->asic_type >= CHIP_ALDEBARAN)
	83	+ WREG32_SOC15(HDP, 0, mmHDP_EDC_CNT, 0);
	84	+ else
	85	+ /read back hdp ras counter to reset it to 0 /
	86	+ RREG32_SOC15(HDP, 0, mmHDP_EDC_CNT);
68	87	}
69	88
70	89	static void hdp_v4_0_update_clock_gating(struct amdgpu_device *adev,
@@ -130,10 +149,16 @@ static void hdp_v4_0_init_registers(struct amdgpu_device *adev) · 59 unmodified lines
130	149	WREG32_SOC15(HDP, 0, mmHDP_NONSURFACE_BASE_HI, (adev->gmc.vram_start >> 40));
131	150	}
132	151
	152	+const struct amdgpu_hdp_ras_funcs hdp_v4_0_ras_funcs = {
	153	+ .ras_late_init = amdgpu_hdp_ras_late_init,
	154	+ .ras_fini = amdgpu_hdp_ras_fini,
	155	+ .query_ras_error_count = hdp_v4_0_query_ras_error_count,
	156	+ .reset_ras_error_count = hdp_v4_0_reset_ras_error_count,
	157	+};
	158	+
133	159	const struct amdgpu_hdp_funcs hdp_v4_0_funcs = {
134	160	.flush_hdp = hdp_v4_0_flush_hdp,
135	161	.invalidate_hdp = hdp_v4_0_invalidate_hdp,
136		− .reset_ras_error_count = hdp_v4_0_reset_ras_error_count,
137	162	.update_clock_gating = hdp_v4_0_update_clock_gating,
138	163	.get_clock_gating_state = hdp_v4_0_get_clockgating_state,
139	164	.init_registers = hdp_v4_0_init_registers,

drivers/gpu/drm/amd/amdgpu/hdp_v4_0.h+1−0

@@ -27,5 +27,6 @@ · 26 unmodified lines
27	27	#include "soc15_common.h"
28	28
29	29	extern const struct amdgpu_hdp_funcs hdp_v4_0_funcs;
	30	+extern const struct amdgpu_hdp_ras_funcs hdp_v4_0_ras_funcs;
30	31
31	32	#endif

drivers/gpu/drm/amd/amdgpu/mmhub_v1_7.c+23−2

@@ -296,10 +296,12 @@ static void mmhub_v1_7_setup_vmid_config(struct amdgpu_device *adev) · 295 unmodified lines
296	296	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
297	297	PAGE_TABLE_BLOCK_SIZE,
298	298	block_size);
299		− /* Send no-retry XNACK on fault to suppress VM fault storm. */
	299	+ /* On Aldebaran, XNACK can be enabled in the SQ per-process.
	300	+ * Retry faults need to be enabled for that to work.
	301	+ */
300	302	tmp = REG_SET_FIELD(tmp, VM_CONTEXT1_CNTL,
301	303	RETRY_PERMISSION_OR_INVALID_PAGE_FAULT,
302		− !adev->gmc.noretry);
	304	+ 1);
303	305	WREG32_SOC15_OFFSET(MMHUB, 0, regVM_CONTEXT1_CNTL,
304	306	i * hub->ctx_distance, tmp);
305	307	WREG32_SOC15_OFFSET(MMHUB, 0, regVM_CONTEXT1_PAGE_TABLE_START_ADDR_LO32,
@@ -1313,12 +1315,31 @@ static void mmhub_v1_7_query_ras_error_status(struct amdgpu_device *adev) · 1007 unmodified lines
1313	1315	}
1314	1316	}
1315	1317
	1318	+static void mmhub_v1_7_reset_ras_error_status(struct amdgpu_device *adev)
	1319	+{
	1320	+ int i;
	1321	+ uint32_t reg_value;
	1322	+
	1323	+ if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB))
	1324	+ return;
	1325	+
	1326	+ for (i = 0; i < ARRAY_SIZE(mmhub_v1_7_ea_err_status_regs); i++) {
	1327	+ reg_value = RREG32(SOC15_REG_ENTRY_OFFSET(
	1328	+ mmhub_v1_7_ea_err_status_regs[i]));
	1329	+ reg_value = REG_SET_FIELD(reg_value, MMEA0_ERR_STATUS,
	1330	+ CLEAR_ERROR_STATUS, 0x01);
	1331	+ WREG32(SOC15_REG_ENTRY_OFFSET(mmhub_v1_7_ea_err_status_regs[i]),
	1332	+ reg_value);
	1333	+ }
	1334	+}
	1335	+
1316	1336	const struct amdgpu_mmhub_ras_funcs mmhub_v1_7_ras_funcs = {
1317	1337	.ras_late_init = amdgpu_mmhub_ras_late_init,
1318	1338	.ras_fini = amdgpu_mmhub_ras_fini,
1319	1339	.query_ras_error_count = mmhub_v1_7_query_ras_error_count,
1320	1340	.reset_ras_error_count = mmhub_v1_7_reset_ras_error_count,
1321	1341	.query_ras_error_status = mmhub_v1_7_query_ras_error_status,
	1342	+ .reset_ras_error_status = mmhub_v1_7_reset_ras_error_status,
1322	1343	};
1323	1344
1324	1345	const struct amdgpu_mmhub_funcs mmhub_v1_7_funcs = {

drivers/gpu/drm/amd/amdgpu/mmhub_v2_3.c+2−2

@@ -569,9 +569,9 @@ static int mmhub_v2_3_set_clockgating(struct amdgpu_device *adev, · 568 unmodified lines
569	569	return 0;
570	570
571	571	mmhub_v2_3_update_medium_grain_clock_gating(adev,
572		− state == AMD_CG_STATE_GATE ? true : false);
	572	+ state == AMD_CG_STATE_GATE);
573	573	mmhub_v2_3_update_medium_grain_light_sleep(adev,
574		− state == AMD_CG_STATE_GATE ? true : false);
	574	+ state == AMD_CG_STATE_GATE);
575	575
576	576	return 0;
577	577	}

drivers/gpu/drm/amd/amdgpu/nv.c+6−1

@@ -598,7 +598,7 @@ static void nv_pcie_gen3_enable(struct amdgpu_device *adev) · 597 unmodified lines
598	598
599	599	static void nv_program_aspm(struct amdgpu_device *adev)
600	600	{
601		− if (amdgpu_aspm != 1)
	601	+ if (!amdgpu_aspm)
602	602	return;
603	603
604	604	if (!(adev->flags & AMD_IS_APU) &&
@@ -1068,6 +1068,7 @@ static int nv_common_early_init(void *handle) · 463 unmodified lines
1068	1068	case CHIP_SIENNA_CICHLID:
1069	1069	adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG \|
1070	1070	AMD_CG_SUPPORT_GFX_CGCG \|
	1071	+ AMD_CG_SUPPORT_GFX_CGLS \|
1071	1072	AMD_CG_SUPPORT_GFX_3D_CGCG \|
1072	1073	AMD_CG_SUPPORT_MC_MGCG \|
1073	1074	AMD_CG_SUPPORT_VCN_MGCG \|
@@ -1091,6 +1092,7 @@ static int nv_common_early_init(void *handle) · 17 unmodified lines
1091	1092	case CHIP_NAVY_FLOUNDER:
1092	1093	adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG \|
1093	1094	AMD_CG_SUPPORT_GFX_CGCG \|
	1095	+ AMD_CG_SUPPORT_GFX_CGLS \|
1094	1096	AMD_CG_SUPPORT_GFX_3D_CGCG \|
1095	1097	AMD_CG_SUPPORT_VCN_MGCG \|
1096	1098	AMD_CG_SUPPORT_JPEG_MGCG \|
@@ -1121,6 +1123,8 @@ static int nv_common_early_init(void *handle) · 24 unmodified lines
1121	1123	AMD_CG_SUPPORT_MC_LS \|
1122	1124	AMD_CG_SUPPORT_GFX_FGCG \|
1123	1125	AMD_CG_SUPPORT_VCN_MGCG \|
	1126	+ AMD_CG_SUPPORT_SDMA_MGCG \|
	1127	+ AMD_CG_SUPPORT_SDMA_LS \|
1124	1128	AMD_CG_SUPPORT_JPEG_MGCG;
1125	1129	adev->pg_flags = AMD_PG_SUPPORT_GFX_PG \|
1126	1130	AMD_PG_SUPPORT_VCN \|
@@ -1132,6 +1136,7 @@ static int nv_common_early_init(void *handle) · 5 unmodified lines
1132	1136	case CHIP_DIMGREY_CAVEFISH:
1133	1137	adev->cg_flags = AMD_CG_SUPPORT_GFX_MGCG \|
1134	1138	AMD_CG_SUPPORT_GFX_CGCG \|
	1139	+ AMD_CG_SUPPORT_GFX_CGLS \|
1135	1140	AMD_CG_SUPPORT_GFX_3D_CGCG \|
1136	1141	AMD_CG_SUPPORT_VCN_MGCG \|
1137	1142	AMD_CG_SUPPORT_JPEG_MGCG \|

drivers/gpu/drm/amd/amdgpu/psp_gfx_if.h+0−1

@@ -97,7 +97,6 @@ enum psp_gfx_cmd_id · 96 unmodified lines
97	97	GFX_CMD_ID_SETUP_VMR = 0x00000009, /* setup VMR region */
98	98	GFX_CMD_ID_DESTROY_VMR = 0x0000000A, /* destroy VMR region */
99	99	GFX_CMD_ID_PROG_REG = 0x0000000B, /* program regs */
100		− GFX_CMD_ID_CLEAR_VF_FW = 0x0000000D, /* Clear VF FW, to be used on VF shutdown. */
101	100	GFX_CMD_ID_GET_FW_ATTESTATION = 0x0000000F, /* Query GPUVA of the Fw Attestation DB */
102	101	/* IDs upto 0x1F are reserved for older programs (Raven, Vega 10/12/20) */
103	102	GFX_CMD_ID_LOAD_TOC = 0x00000020, /* Load TOC and obtain TMR size */

drivers/gpu/drm/amd/amdgpu/sdma_v4_0.c+14−21

@@ -1109,6 +1109,8 @@ static void sdma_v4_0_ctx_switch_enable(struct amdgpu_device *adev, bool enable) · 1108 unmodified lines
1109	1109	if (adev->asic_type == CHIP_ARCTURUS &&
1110	1110	adev->sdma.instance[i].fw_version >= 14)
1111	1111	WREG32_SDMA(i, mmSDMA0_PUB_DUMMY_REG2, enable);
	1112	+ /* Extend page fault timeout to avoid interrupt storm */
	1113	+ WREG32_SDMA(i, mmSDMA0_UTCL1_TIMEOUT, 0x00800080);
1112	1114	}
1113	1115
1114	1116	}
@@ -2227,7 +2229,7 @@ static int sdma_v4_0_print_iv_entry(struct amdgpu_device *adev, · 1112 unmodified lines
2227	2229	memset(&task_info, 0, sizeof(struct amdgpu_task_info));
2228	2230	amdgpu_vm_get_task_info(adev, entry->pasid, &task_info);
2229	2231
2230		− dev_info(adev->dev,
	2232	+ dev_dbg_ratelimited(adev->dev,
2231	2233	"[sdma%d] address:0x%016llx src_id:%u ring:%u vmid:%u "
2232	2234	"pasid:%u, for process %s pid %d thread %s pid %d\n",
2233	2235	instance, addr, entry->src_id, entry->ring_id, entry->vmid,
@@ -2240,7 +2242,7 @@ static int sdma_v4_0_process_vm_hole_irq(struct amdgpu_device *adev, · 6 unmodified lines
2240	2242	struct amdgpu_irq_src *source,
2241	2243	struct amdgpu_iv_entry *entry)
2242	2244	{
2243		− dev_err(adev->dev, "MC or SEM address in VM hole\n");
	2245	+ dev_dbg_ratelimited(adev->dev, "MC or SEM address in VM hole\n");
2244	2246	sdma_v4_0_print_iv_entry(adev, entry);
2245	2247	return 0;
2246	2248	}
@@ -2249,7 +2251,7 @@ static int sdma_v4_0_process_doorbell_invalid_irq(struct amdgpu_device *adev, · 2 unmodified lines
2249	2251	struct amdgpu_irq_src *source,
2250	2252	struct amdgpu_iv_entry *entry)
2251	2253	{
2252		− dev_err(adev->dev, "SDMA received a doorbell from BIF with byte_enable !=0xff\n");
	2254	+ dev_dbg_ratelimited(adev->dev, "SDMA received a doorbell from BIF with byte_enable !=0xff\n");
2253	2255	sdma_v4_0_print_iv_entry(adev, entry);
2254	2256	return 0;
2255	2257	}
@@ -2258,7 +2260,7 @@ static int sdma_v4_0_process_pool_timeout_irq(struct amdgpu_device *adev, · 2 unmodified lines
2258	2260	struct amdgpu_irq_src *source,
2259	2261	struct amdgpu_iv_entry *entry)
2260	2262	{
2261		− dev_err(adev->dev,
	2263	+ dev_dbg_ratelimited(adev->dev,
2262	2264	"Polling register/memory timeout executing POLL_REG/MEM with finite timer\n");
2263	2265	sdma_v4_0_print_iv_entry(adev, entry);
2264	2266	return 0;
@@ -2268,7 +2270,7 @@ static int sdma_v4_0_process_srbm_write_irq(struct amdgpu_device *adev, · 3 unmodified lines
2268	2270	struct amdgpu_irq_src *source,
2269	2271	struct amdgpu_iv_entry *entry)
2270	2272	{
2271		− dev_err(adev->dev,
	2273	+ dev_dbg_ratelimited(adev->dev,
2272	2274	"SDMA gets an Register Write SRBM_WRITE command in non-privilege command buffer\n");
2273	2275	sdma_v4_0_print_iv_entry(adev, entry);
2274	2276	return 0;
@@ -2597,27 +2599,18 @@ static const struct amdgpu_irq_src_funcs sdma_v4_0_srbm_write_irq_funcs = { · 322 unmodified lines
2597	2599
2598	2600	static void sdma_v4_0_set_irq_funcs(struct amdgpu_device *adev)
2599	2601	{
	2602	+ adev->sdma.trap_irq.num_types = adev->sdma.num_instances;
	2603	+ adev->sdma.ecc_irq.num_types = adev->sdma.num_instances;
	2604	+ /For Arcturus and Aldebaran, add another 4 irq handler/
2600	2605	switch (adev->sdma.num_instances) {
2601		− case 1:
2602		− adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE1;
2603		− adev->sdma.ecc_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE1;
2604		− break;
2605	2606	case 5:
2606		− adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE5;
2607		− adev->sdma.ecc_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE5;
2608		− break;
2609	2607	case 8:
2610		− adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
2611		− adev->sdma.ecc_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
2612		− adev->sdma.vm_hole_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE5;
2613		− adev->sdma.doorbell_invalid_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
2614		− adev->sdma.pool_timeout_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
2615		− adev->sdma.srbm_write_irq.num_types = AMDGPU_SDMA_IRQ_LAST;
	2608	+ adev->sdma.vm_hole_irq.num_types = adev->sdma.num_instances;
	2609	+ adev->sdma.doorbell_invalid_irq.num_types = adev->sdma.num_instances;
	2610	+ adev->sdma.pool_timeout_irq.num_types = adev->sdma.num_instances;
	2611	+ adev->sdma.srbm_write_irq.num_types = adev->sdma.num_instances;
2616	2612	break;
2617		− case 2:
2618	2613	default:
2619		− adev->sdma.trap_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE2;
2620		− adev->sdma.ecc_irq.num_types = AMDGPU_SDMA_IRQ_INSTANCE2;
2621	2614	break;
2622	2615	}
2623	2616	adev->sdma.trap_irq.funcs = &sdma_v4_0_trap_irq_funcs;

drivers/gpu/drm/amd/amdgpu/sdma_v5_0.c+28−12

@@ -405,18 +405,6 @@ static void sdma_v5_0_ring_emit_ib(struct amdgpu_ring *ring, · 404 unmodified lines
405	405	unsigned vmid = AMDGPU_JOB_GET_VMID(job);
406	406	uint64_t csa_mc_addr = amdgpu_sdma_get_csa_mc_addr(ring, vmid);
407	407
408		− /* Invalidate L2, because if we don't do it, we might get stale cache
409		− * lines from previous IBs.
410		− */
411		− amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_GCR_REQ));
412		− amdgpu_ring_write(ring, 0);
413		− amdgpu_ring_write(ring, (SDMA_GCR_GL2_INV \|
414		− SDMA_GCR_GL2_WB \|
415		− SDMA_GCR_GLM_INV \|
416		− SDMA_GCR_GLM_WB) << 16);
417		− amdgpu_ring_write(ring, 0xffffff80);
418		− amdgpu_ring_write(ring, 0xffff);
419		−
420	408	/* An IB packet must end on a 8 DW boundary--the next dword
421	409	* must be on a 8-dword boundary. Our IB packet below is 6
422	410	* dwords long, thus add x number of NOPs, such that, in
@@ -437,6 +425,33 @@ static void sdma_v5_0_ring_emit_ib(struct amdgpu_ring *ring, · 14 unmodified lines
437	425	amdgpu_ring_write(ring, upper_32_bits(csa_mc_addr));
438	426	}
439	427
	428	+/**
	429	+ * sdma_v5_0_ring_emit_mem_sync - flush the IB by graphics cache rinse
	430	+ *
	431	+ * @ring: amdgpu ring pointer
	432	+ * @job: job to retrieve vmid from
	433	+ * @ib: IB object to schedule
	434	+ *
	435	+ * flush the IB by graphics cache rinse.
	436	+ */
	437	+static void sdma_v5_0_ring_emit_mem_sync(struct amdgpu_ring *ring)
	438	+{
	439	+ uint32_t gcr_cntl =
	440	+ SDMA_GCR_GL2_INV \| SDMA_GCR_GL2_WB \| SDMA_GCR_GLM_INV \|
	441	+ SDMA_GCR_GL1_INV \| SDMA_GCR_GLV_INV \| SDMA_GCR_GLK_INV \|
	442	+ SDMA_GCR_GLI_INV(1);
	443	+
	444	+ /* flush entire cache L0/L1/L2, this can be optimized by performance requirement */
	445	+ amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_GCR_REQ));
	446	+ amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD1_BASE_VA_31_7(0));
	447	+ amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD2_GCR_CONTROL_15_0(gcr_cntl) \|
	448	+ SDMA_PKT_GCR_REQ_PAYLOAD2_BASE_VA_47_32(0));
	449	+ amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD3_LIMIT_VA_31_7(0) \|
	450	+ SDMA_PKT_GCR_REQ_PAYLOAD3_GCR_CONTROL_18_16(gcr_cntl >> 16));
	451	+ amdgpu_ring_write(ring, SDMA_PKT_GCR_REQ_PAYLOAD4_LIMIT_VA_47_32(0) \|
	452	+ SDMA_PKT_GCR_REQ_PAYLOAD4_VMID(0));
	453	+}
	454	+
440	455	/**
441	456	* sdma_v5_0_ring_emit_hdp_flush - emit an hdp flush on the DMA ring
442	457	*
@@ -1643,6 +1658,7 @@ static const struct amdgpu_ring_funcs sdma_v5_0_ring_funcs = { · 1200 unmodified lines
1643	1658	10 + 10 + 10, /* sdma_v5_0_ring_emit_fence x3 for user fence, vm fence */
1644	1659	.emit_ib_size = 5 + 7 + 6, /* sdma_v5_0_ring_emit_ib */
1645	1660	.emit_ib = sdma_v5_0_ring_emit_ib,
	1661	+ .emit_mem_sync = sdma_v5_0_ring_emit_mem_sync,
1646	1662	.emit_fence = sdma_v5_0_ring_emit_fence,
1647	1663	.emit_pipeline_sync = sdma_v5_0_ring_emit_pipeline_sync,
1648	1664	.emit_vm_flush = sdma_v5_0_ring_emit_vm_flush,

drivers/gpu/drm/amd/amdgpu/sdma_v5_2.c+8−0

@@ -1556,6 +1556,10 @@ static void sdma_v5_2_update_medium_grain_clock_gating(struct amdgpu_device *ade · 1555 unmodified lines
1556	1556	int i;
1557	1557
1558	1558	for (i = 0; i < adev->sdma.num_instances; i++) {
	1559	+
	1560	+ if (adev->sdma.instance[i].fw_version < 70 && adev->asic_type == CHIP_VANGOGH)
	1561	+ adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_MGCG;
	1562	+
1559	1563	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_MGCG)) {
1560	1564	/* Enable sdma clock gating */
1561	1565	def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_CLK_CTRL));
@@ -1589,6 +1593,10 @@ static void sdma_v5_2_update_medium_grain_light_sleep(struct amdgpu_device *adev · 27 unmodified lines
1589	1593	int i;
1590	1594
1591	1595	for (i = 0; i < adev->sdma.num_instances; i++) {
	1596	+
	1597	+ if (adev->sdma.instance[i].fw_version < 70 && adev->asic_type == CHIP_VANGOGH)
	1598	+ adev->cg_flags &= ~AMD_CG_SUPPORT_SDMA_LS;
	1599	+
1592	1600	if (enable && (adev->cg_flags & AMD_CG_SUPPORT_SDMA_LS)) {
1593	1601	/* Enable sdma mem light sleep */
1594	1602	def = data = RREG32(sdma_v5_2_get_reg_offset(adev, i, mmSDMA0_POWER_CNTL));

drivers/gpu/drm/amd/amdgpu/smuio_v13_0.c+18−0

@@ -88,6 +88,23 @@ static u32 smuio_v13_0_get_die_id(struct amdgpu_device *adev) · 87 unmodified lines
88	88	return die_id;
89	89	}
90	90
	91	+/**
	92	+ * smuio_v13_0_get_socket_id - query socket id from FCH
	93	+ *
	94	+ * @adev: amdgpu device pointer
	95	+ *
	96	+ * Returns socket id
	97	+ */
	98	+static u32 smuio_v13_0_get_socket_id(struct amdgpu_device *adev)
	99	+{
	100	+ u32 data, socket_id;
	101	+
	102	+ data = RREG32_SOC15(SMUIO, 0, regSMUIO_MCM_CONFIG);
	103	+ socket_id = REG_GET_FIELD(data, SMUIO_MCM_CONFIG, SOCKET_ID);
	104	+
	105	+ return socket_id;
	106	+}
	107	+
91	108	/**
92	109	* smuio_v13_0_supports_host_gpu_xgmi - detect xgmi interface between cpu and gpu/s.
93	110	*
@@ -115,6 +132,7 @@ const struct amdgpu_smuio_funcs smuio_v13_0_funcs = { · 21 unmodified lines
115	132	.get_rom_index_offset = smuio_v13_0_get_rom_index_offset,
116	133	.get_rom_data_offset = smuio_v13_0_get_rom_data_offset,
117	134	.get_die_id = smuio_v13_0_get_die_id,
	135	+ .get_socket_id = smuio_v13_0_get_socket_id,
118	136	.is_host_gpu_xgmi_supported = smuio_v13_0_is_host_gpu_xgmi_supported,
119	137	.update_rom_clock_gating = smuio_v13_0_update_rom_clock_gating,
120	138	.get_clock_gating_state = smuio_v13_0_get_clock_gating_state,

drivers/gpu/drm/amd/amdgpu/soc15.c+5−7

@@ -655,7 +655,7 @@ static int soc15_asic_baco_reset(struct amdgpu_device *adev) · 654 unmodified lines
655	655	int ret = 0;
656	656
657	657	/* avoid NBIF got stuck when do RAS recovery in BACO reset */
658		− if (ras && ras->supported)
	658	+ if (ras && adev->ras_enabled)
659	659	adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
660	660
661	661	ret = amdgpu_dpm_baco_reset(adev);
@@ -663,7 +663,7 @@ static int soc15_asic_baco_reset(struct amdgpu_device *adev) · 1 unmodified line
663	663	return ret;
664	664
665	665	/* re-enable doorbell interrupt after BACO exit */
666		− if (ras && ras->supported)
	666	+ if (ras && adev->ras_enabled)
667	667	adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
668	668
669	669	return 0;
@@ -710,7 +710,8 @@ soc15_asic_reset_method(struct amdgpu_device *adev) · 40 unmodified lines
710	710	* 1. PMFW version > 0x284300: all cases use baco
711	711	* 2. PMFW version <= 0x284300: only sGPU w/o RAS use baco
712	712	*/
713		− if ((ras && ras->supported) && adev->pm.fw_version <= 0x283400)
	713	+ if (ras && adev->ras_enabled &&
	714	+ adev->pm.fw_version <= 0x283400)
714	715	baco_reset = false;
715	716	break;
716	717	case CHIP_ALDEBARAN:
@@ -816,7 +817,7 @@ static void soc15_pcie_gen3_enable(struct amdgpu_device *adev) · 99 unmodified lines
816	817
817	818	static void soc15_program_aspm(struct amdgpu_device *adev)
818	819	{
819		− if (amdgpu_aspm != 1)
	820	+ if (!amdgpu_aspm)
820	821	return;
821	822
822	823	if (!(adev->flags & AMD_IS_APU) &&
@@ -1522,9 +1523,6 @@ static int soc15_common_late_init(void *handle) · 699 unmodified lines
1522	1523	if (amdgpu_sriov_vf(adev))
1523	1524	xgpu_ai_mailbox_get_irq(adev);
1524	1525
1525		− if (adev->hdp.funcs->reset_ras_error_count)
1526		− adev->hdp.funcs->reset_ras_error_count(adev);
1527		−
1528	1526	if (adev->nbio.ras_funcs &&
1529	1527	adev->nbio.ras_funcs->ras_late_init)
1530	1528	r = adev->nbio.ras_funcs->ras_late_init(adev);

drivers/gpu/drm/amd/amdgpu/ta_ras_if.h+7−0

@@ -105,6 +105,12 @@ struct ta_ras_trigger_error_input { · 104 unmodified lines
105	105	uint64_t value; // method if error injection. i.e persistent, coherent etc.
106	106	};
107	107
	108	+struct ta_ras_init_flags
	109	+{
	110	+ uint8_t poison_mode_en;
	111	+ uint8_t dgpu_mode;
	112	+};
	113	+
108	114	struct ta_ras_output_flags
109	115	{
110	116	uint8_t ras_init_success_flag;
@@ -115,6 +121,7 @@ struct ta_ras_output_flags · 4 unmodified lines
115	121	/* Common input structure for RAS callbacks */
116	122	/**********************************************************/
117	123	union ta_ras_cmd_input {
	124	+ struct ta_ras_init_flags init_flags;
118	125	struct ta_ras_enable_features_input enable_features;
119	126	struct ta_ras_disable_features_input disable_features;
120	127	struct ta_ras_trigger_error_input trigger_error;

drivers/gpu/drm/amd/amdgpu/vega10_ih.c+1−0

@@ -104,6 +104,7 @@ static int vega10_ih_toggle_ring_interrupts(struct amdgpu_device *adev, · 103 unmodified lines
104	104
105	105	tmp = RREG32(ih_regs->ih_rb_cntl);
106	106	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_ENABLE, (enable ? 1 : 0));
	107	+ tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, RB_GPU_TS_ENABLE, 1);
107	108	/* enable_intr field is only valid in ring0 */
108	109	if (ih == &adev->irq.ih)
109	110	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, ENABLE_INTR, (enable ? 1 : 0));

drivers/gpu/drm/amd/amdgpu/vi.c+1−1

@@ -1136,7 +1136,7 @@ static void vi_program_aspm(struct amdgpu_device *adev) · 1135 unmodified lines
1136	1136	bool bL1SS = false;
1137	1137	bool bClkReqSupport = true;
1138	1138
1139		− if (amdgpu_aspm != 1)
	1139	+ if (!amdgpu_aspm)
1140	1140	return;
1141	1141
1142	1142	if (adev->flags & AMD_IS_APU \|\|

drivers/gpu/drm/amd/amdkfd/Kconfig+13−0

@@ -12,3 +12,16 @@ config HSA_AMD · 11 unmodified lines
12	12	select DRM_AMDGPU_USERPTR
13	13	help
14	14	Enable this if you want to use HSA features on AMD GPU devices.
	15	+
	16	+config HSA_AMD_SVM
	17	+ bool "Enable HMM-based shared virtual memory manager"
	18	+ depends on HSA_AMD && DEVICE_PRIVATE
	19	+ default y
	20	+ select HMM_MIRROR
	21	+ select MMU_NOTIFIER
	22	+ help
	23	+ Enable this to use unified memory and managed memory in HIP. This
	24	+ memory manager supports two modes of operation. One based on
	25	+ preemptions and one based on page faults. To enable page fault
	26	+ based memory management on most GFXv9 GPUs, set the module
	27	+ parameter amdgpu.noretry=0.

drivers/gpu/drm/amd/amdkfd/Makefile+5−0

@@ -63,3 +63,8 @@ endif · 62 unmodified lines
63	63	ifneq ($(CONFIG_DEBUG_FS),)
64	64	AMDKFD_FILES += $(AMDKFD_PATH)/kfd_debugfs.o
65	65	endif
	66	+
	67	+ifneq ($(CONFIG_HSA_AMD_SVM),)
	68	+AMDKFD_FILES += $(AMDKFD_PATH)/kfd_svm.o \
	69	+ $(AMDKFD_PATH)/kfd_migrate.o
	70	+endif

drivers/gpu/drm/amd/amdkfd/kfd_chardev.c+73−7

@@ -38,6 +38,7 @@ · 37 unmodified lines
38	38	#include "kfd_priv.h"
39	39	#include "kfd_device_queue_manager.h"
40	40	#include "kfd_dbgmgr.h"
	41	+#include "kfd_svm.h"
41	42	#include "amdgpu_amdkfd.h"
42	43	#include "kfd_smi_events.h"
43	44
@@ -1297,7 +1298,7 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep, · 1253 unmodified lines
1297	1298
1298	1299	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(
1299	1300	dev->kgd, args->va_addr, args->size,
1300		− pdd->vm, (struct kgd_mem **) &mem, &offset,
	1301	+ pdd->drm_priv, (struct kgd_mem **) &mem, &offset,
1301	1302	flags);
1302	1303
1303	1304	if (err)
@@ -1328,7 +1329,8 @@ static int kfd_ioctl_alloc_memory_of_gpu(struct file *filep, · 24 unmodified lines
1328	1329	return 0;
1329	1330
1330	1331	err_free:
1331		− amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem, NULL);
	1332	+ amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem,
	1333	+ pdd->drm_priv, NULL);
1332	1334	err_unlock:
1333	1335	mutex_unlock(&p->mutex);
1334	1336	return err;
@@ -1365,7 +1367,7 @@ static int kfd_ioctl_free_memory_of_gpu(struct file *filep, · 30 unmodified lines
1365	1367	}
1366	1368
1367	1369	ret = amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd,
1368		− (struct kgd_mem *)mem, &size);
	1370	+ (struct kgd_mem *)mem, pdd->drm_priv, &size);
1369	1371
1370	1372	/* If freeing the buffer failed, leave the handle in place for
1371	1373	* clean-up during process tear-down.
@@ -1448,7 +1450,7 @@ static int kfd_ioctl_map_memory_to_gpu(struct file *filep, · 76 unmodified lines
1448	1450	goto get_mem_obj_from_handle_failed;
1449	1451	}
1450	1452	err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
1451		− peer->kgd, (struct kgd_mem *)mem, peer_pdd->vm);
	1453	+ peer->kgd, (struct kgd_mem *)mem, peer_pdd->drm_priv);
1452	1454	if (err) {
1453	1455	pr_err("Failed to map to gpu %d/%d\n",
1454	1456	i, args->n_devices);
@@ -1555,7 +1557,7 @@ static int kfd_ioctl_unmap_memory_from_gpu(struct file *filep, · 100 unmodified lines
1555	1557	goto get_mem_obj_from_handle_failed;
1556	1558	}
1557	1559	err = amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
1558		− peer->kgd, (struct kgd_mem *)mem, peer_pdd->vm);
	1560	+ peer->kgd, (struct kgd_mem *)mem, peer_pdd->drm_priv);
1559	1561	if (err) {
1560	1562	pr_err("Failed to unmap from gpu %d/%d\n",
1561	1563	i, args->n_devices);
@@ -1701,7 +1703,7 @@ static int kfd_ioctl_import_dmabuf(struct file *filep, · 139 unmodified lines
1701	1703	}
1702	1704
1703	1705	r = amdgpu_amdkfd_gpuvm_import_dmabuf(dev->kgd, dmabuf,
1704		− args->va_addr, pdd->vm,
	1706	+ args->va_addr, pdd->drm_priv,
1705	1707	(struct kgd_mem **)&mem, &size,
1706	1708	NULL);
1707	1709	if (r)
@@ -1721,7 +1723,8 @@ static int kfd_ioctl_import_dmabuf(struct file *filep, · 13 unmodified lines
1721	1723	return 0;
1722	1724
1723	1725	err_free:
1724		− amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem, NULL);
	1726	+ amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, (struct kgd_mem *)mem,
	1727	+ pdd->drm_priv, NULL);
1725	1728	err_unlock:
1726	1729	mutex_unlock(&p->mutex);
1727	1730	dma_buf_put(dmabuf);
@@ -1742,6 +1745,64 @@ static int kfd_ioctl_smi_events(struct file *filep, · 14 unmodified lines
1742	1745	return kfd_smi_event_open(dev, &args->anon_fd);
1743	1746	}
1744	1747
	1748	+static int kfd_ioctl_set_xnack_mode(struct file *filep,
	1749	+ struct kfd_process p, void data)
	1750	+{
	1751	+ struct kfd_ioctl_set_xnack_mode_args *args = data;
	1752	+ int r = 0;
	1753	+
	1754	+ mutex_lock(&p->mutex);
	1755	+ if (args->xnack_enabled >= 0) {
	1756	+ if (!list_empty(&p->pqm.queues)) {
	1757	+ pr_debug("Process has user queues running\n");
	1758	+ mutex_unlock(&p->mutex);
	1759	+ return -EBUSY;
	1760	+ }
	1761	+ if (args->xnack_enabled && !kfd_process_xnack_mode(p, true))
	1762	+ r = -EPERM;
	1763	+ else
	1764	+ p->xnack_enabled = args->xnack_enabled;
	1765	+ } else {
	1766	+ args->xnack_enabled = p->xnack_enabled;
	1767	+ }
	1768	+ mutex_unlock(&p->mutex);
	1769	+
	1770	+ return r;
	1771	+}
	1772	+
	1773	+#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
	1774	+static int kfd_ioctl_svm(struct file filep, struct kfd_process p, void *data)
	1775	+{
	1776	+ struct kfd_ioctl_svm_args *args = data;
	1777	+ int r = 0;
	1778	+
	1779	+ if (p->svm_disabled)
	1780	+ return -EPERM;
	1781	+
	1782	+ pr_debug("start 0x%llx size 0x%llx op 0x%x nattr 0x%x\n",
	1783	+ args->start_addr, args->size, args->op, args->nattr);
	1784	+
	1785	+ if ((args->start_addr & ~PAGE_MASK) \|\| (args->size & ~PAGE_MASK))
	1786	+ return -EINVAL;
	1787	+ if (!args->start_addr \|\| !args->size)
	1788	+ return -EINVAL;
	1789	+
	1790	+ mutex_lock(&p->mutex);
	1791	+
	1792	+ r = svm_ioctl(p, args->op, args->start_addr, args->size, args->nattr,
	1793	+ args->attrs);
	1794	+
	1795	+ mutex_unlock(&p->mutex);
	1796	+
	1797	+ return r;
	1798	+}
	1799	+#else
	1800	+static int kfd_ioctl_svm(struct file filep, struct kfd_process p, void *data)
	1801	+{
	1802	+ return -EPERM;
	1803	+}
	1804	+#endif
	1805	+
1745	1806	#define AMDKFD_IOCTL_DEF(ioctl, _func, _flags) \
1746	1807	[_IOC_NR(ioctl)] = {.cmd = ioctl, .func = _func, .flags = _flags, \
1747	1808	.cmd_drv = 0, .name = #ioctl}
@@ -1840,6 +1901,11 @@ static const struct amdkfd_ioctl_desc amdkfd_ioctls[] = { · 92 unmodified lines
1840	1901
1841	1902	AMDKFD_IOCTL_DEF(AMDKFD_IOC_SMI_EVENTS,
1842	1903	kfd_ioctl_smi_events, 0),
	1904	+
	1905	+ AMDKFD_IOCTL_DEF(AMDKFD_IOC_SVM, kfd_ioctl_svm, 0),
	1906	+
	1907	+ AMDKFD_IOCTL_DEF(AMDKFD_IOC_SET_XNACK_MODE,
	1908	+ kfd_ioctl_set_xnack_mode, 0),
1843	1909	};
1844	1910
1845	1911	#define AMDKFD_CORE_IOCTL_COUNT ARRAY_SIZE(amdkfd_ioctls)

drivers/gpu/drm/amd/amdkfd/kfd_crat.c+811−70

@@ -55,7 +55,7 @@ struct kfd_gpu_cache_info { · 54 unmodified lines
55	55	uint32_t cache_level;
56	56	uint32_t flags;
57	57	/* Indicates how many Compute Units share this cache
58		− * Value = 1 indicates the cache is not shared
	58	+ * within a SA. Value = 1 indicates the cache is not shared
59	59	*/
60	60	uint32_t num_cu_shared;
61	61	};
@@ -69,32 +69,456 @@ static struct kfd_gpu_cache_info kaveri_cache_info[] = { · 7 unmodified lines
69	69	CRAT_CACHE_FLAGS_DATA_CACHE \|
70	70	CRAT_CACHE_FLAGS_SIMD_CACHE),
71	71	.num_cu_shared = 1,
	72	+ },
	73	+ {
	74	+ /* Scalar L1 Instruction Cache (in SQC module) per bank */
	75	+ .cache_size = 16,
	76	+ .cache_level = 1,
	77	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	78	+ CRAT_CACHE_FLAGS_INST_CACHE \|
	79	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	80	+ .num_cu_shared = 2,
	81	+ },
	82	+ {
	83	+ /* Scalar L1 Data Cache (in SQC module) per bank */
	84	+ .cache_size = 8,
	85	+ .cache_level = 1,
	86	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	87	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	88	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	89	+ .num_cu_shared = 2,
	90	+ },
72	91
	92	+ /* TODO: Add L2 Cache information */
	93	+};
	94	+
	95	+
	96	+static struct kfd_gpu_cache_info carrizo_cache_info[] = {
	97	+ {
	98	+ /* TCP L1 Cache per CU */
	99	+ .cache_size = 16,
	100	+ .cache_level = 1,
	101	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	102	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	103	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	104	+ .num_cu_shared = 1,
	105	+ },
	106	+ {
	107	+ /* Scalar L1 Instruction Cache (in SQC module) per bank */
	108	+ .cache_size = 8,
	109	+ .cache_level = 1,
	110	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	111	+ CRAT_CACHE_FLAGS_INST_CACHE \|
	112	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	113	+ .num_cu_shared = 4,
	114	+ },
	115	+ {
	116	+ /* Scalar L1 Data Cache (in SQC module) per bank. */
	117	+ .cache_size = 4,
	118	+ .cache_level = 1,
	119	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	120	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	121	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	122	+ .num_cu_shared = 4,
	123	+ },
	124	+
	125	+ /* TODO: Add L2 Cache information */
	126	+};
	127	+
	128	+#define hawaii_cache_info kaveri_cache_info
	129	+#define tonga_cache_info carrizo_cache_info
	130	+#define fiji_cache_info carrizo_cache_info
	131	+#define polaris10_cache_info carrizo_cache_info
	132	+#define polaris11_cache_info carrizo_cache_info
	133	+#define polaris12_cache_info carrizo_cache_info
	134	+#define vegam_cache_info carrizo_cache_info
	135	+
	136	+/* NOTE: L1 cache information has been updated and L2/L3
	137	+ * cache information has been added for Vega10 and
	138	+ * newer ASICs. The unit for cache_size is KiB.
	139	+ * In future, check & update cache details
	140	+ * for every new ASIC is required.
	141	+ */
	142	+
	143	+static struct kfd_gpu_cache_info vega10_cache_info[] = {
	144	+ {
	145	+ /* TCP L1 Cache per CU */
	146	+ .cache_size = 16,
	147	+ .cache_level = 1,
	148	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	149	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	150	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	151	+ .num_cu_shared = 1,
	152	+ },
	153	+ {
	154	+ /* Scalar L1 Instruction Cache per SQC */
	155	+ .cache_size = 32,
	156	+ .cache_level = 1,
	157	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	158	+ CRAT_CACHE_FLAGS_INST_CACHE \|
	159	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	160	+ .num_cu_shared = 3,
	161	+ },
	162	+ {
	163	+ /* Scalar L1 Data Cache per SQC */
	164	+ .cache_size = 16,
	165	+ .cache_level = 1,
	166	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	167	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	168	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	169	+ .num_cu_shared = 3,
	170	+ },
	171	+ {
	172	+ /* L2 Data Cache per GPU (Total Tex Cache) */
	173	+ .cache_size = 4096,
	174	+ .cache_level = 2,
	175	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	176	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	177	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	178	+ .num_cu_shared = 16,
	179	+ },
	180	+};
	181	+
	182	+static struct kfd_gpu_cache_info raven_cache_info[] = {
	183	+ {
	184	+ /* TCP L1 Cache per CU */
	185	+ .cache_size = 16,
	186	+ .cache_level = 1,
	187	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	188	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	189	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	190	+ .num_cu_shared = 1,
	191	+ },
	192	+ {
	193	+ /* Scalar L1 Instruction Cache per SQC */
	194	+ .cache_size = 32,
	195	+ .cache_level = 1,
	196	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	197	+ CRAT_CACHE_FLAGS_INST_CACHE \|
	198	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	199	+ .num_cu_shared = 3,
	200	+ },
	201	+ {
	202	+ /* Scalar L1 Data Cache per SQC */
	203	+ .cache_size = 16,
	204	+ .cache_level = 1,
	205	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	206	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	207	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	208	+ .num_cu_shared = 3,
	209	+ },
	210	+ {
	211	+ /* L2 Data Cache per GPU (Total Tex Cache) */
	212	+ .cache_size = 1024,
	213	+ .cache_level = 2,
	214	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	215	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	216	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	217	+ .num_cu_shared = 11,
	218	+ },
	219	+};
	220	+
	221	+static struct kfd_gpu_cache_info renoir_cache_info[] = {
	222	+ {
	223	+ /* TCP L1 Cache per CU */
	224	+ .cache_size = 16,
	225	+ .cache_level = 1,
	226	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	227	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	228	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	229	+ .num_cu_shared = 1,
	230	+ },
	231	+ {
	232	+ /* Scalar L1 Instruction Cache per SQC */
	233	+ .cache_size = 32,
	234	+ .cache_level = 1,
	235	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	236	+ CRAT_CACHE_FLAGS_INST_CACHE \|
	237	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	238	+ .num_cu_shared = 3,
	239	+ },
	240	+ {
	241	+ /* Scalar L1 Data Cache per SQC */
	242	+ .cache_size = 16,
	243	+ .cache_level = 1,
	244	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	245	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	246	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	247	+ .num_cu_shared = 3,
	248	+ },
	249	+ {
	250	+ /* L2 Data Cache per GPU (Total Tex Cache) */
	251	+ .cache_size = 1024,
	252	+ .cache_level = 2,
	253	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	254	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	255	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	256	+ .num_cu_shared = 8,
	257	+ },
	258	+};
	259	+
	260	+static struct kfd_gpu_cache_info vega12_cache_info[] = {
	261	+ {
	262	+ /* TCP L1 Cache per CU */
	263	+ .cache_size = 16,
	264	+ .cache_level = 1,
	265	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	266	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	267	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	268	+ .num_cu_shared = 1,
	269	+ },
	270	+ {
	271	+ /* Scalar L1 Instruction Cache per SQC */
	272	+ .cache_size = 32,
	273	+ .cache_level = 1,
	274	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	275	+ CRAT_CACHE_FLAGS_INST_CACHE \|
	276	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	277	+ .num_cu_shared = 3,
	278	+ },
	279	+ {
	280	+ /* Scalar L1 Data Cache per SQC */
	281	+ .cache_size = 16,
	282	+ .cache_level = 1,
	283	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	284	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	285	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	286	+ .num_cu_shared = 3,
	287	+ },
	288	+ {
	289	+ /* L2 Data Cache per GPU (Total Tex Cache) */
	290	+ .cache_size = 2048,
	291	+ .cache_level = 2,
	292	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	293	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	294	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	295	+ .num_cu_shared = 5,
	296	+ },
	297	+};
	298	+
	299	+static struct kfd_gpu_cache_info vega20_cache_info[] = {
	300	+ {
	301	+ /* TCP L1 Cache per CU */
	302	+ .cache_size = 16,
	303	+ .cache_level = 1,
	304	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	305	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	306	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	307	+ .num_cu_shared = 1,
	308	+ },
	309	+ {
	310	+ /* Scalar L1 Instruction Cache per SQC */
	311	+ .cache_size = 32,
	312	+ .cache_level = 1,
	313	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	314	+ CRAT_CACHE_FLAGS_INST_CACHE \|
	315	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	316	+ .num_cu_shared = 3,
	317	+ },
	318	+ {
	319	+ /* Scalar L1 Data Cache per SQC */
	320	+ .cache_size = 16,
	321	+ .cache_level = 1,
	322	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	323	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	324	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	325	+ .num_cu_shared = 3,
	326	+ },
	327	+ {
	328	+ /* L2 Data Cache per GPU (Total Tex Cache) */
	329	+ .cache_size = 8192,
	330	+ .cache_level = 2,
	331	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	332	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	333	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	334	+ .num_cu_shared = 16,
	335	+ },
	336	+};
	337	+
	338	+static struct kfd_gpu_cache_info aldebaran_cache_info[] = {
	339	+ {
	340	+ /* TCP L1 Cache per CU */
	341	+ .cache_size = 16,
	342	+ .cache_level = 1,
	343	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	344	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	345	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	346	+ .num_cu_shared = 1,
	347	+ },
	348	+ {
	349	+ /* Scalar L1 Instruction Cache per SQC */
	350	+ .cache_size = 32,
	351	+ .cache_level = 1,
	352	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	353	+ CRAT_CACHE_FLAGS_INST_CACHE \|
	354	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	355	+ .num_cu_shared = 2,
	356	+ },
	357	+ {
	358	+ /* Scalar L1 Data Cache per SQC */
	359	+ .cache_size = 16,
	360	+ .cache_level = 1,
	361	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	362	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	363	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	364	+ .num_cu_shared = 2,
	365	+ },
	366	+ {
	367	+ /* L2 Data Cache per GPU (Total Tex Cache) */
	368	+ .cache_size = 8192,
	369	+ .cache_level = 2,
	370	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	371	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	372	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	373	+ .num_cu_shared = 14,
	374	+ },
	375	+};
	376	+
	377	+static struct kfd_gpu_cache_info navi10_cache_info[] = {
	378	+ {
	379	+ /* TCP L1 Cache per CU */
	380	+ .cache_size = 16,
	381	+ .cache_level = 1,
	382	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	383	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	384	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	385	+ .num_cu_shared = 1,
	386	+ },
	387	+ {
	388	+ /* Scalar L1 Instruction Cache per SQC */
	389	+ .cache_size = 32,
	390	+ .cache_level = 1,
	391	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	392	+ CRAT_CACHE_FLAGS_INST_CACHE \|
	393	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	394	+ .num_cu_shared = 2,
	395	+ },
	396	+ {
	397	+ /* Scalar L1 Data Cache per SQC */
	398	+ .cache_size = 16,
	399	+ .cache_level = 1,
	400	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	401	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	402	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	403	+ .num_cu_shared = 2,
	404	+ },
	405	+ {
	406	+ /* GL1 Data Cache per SA */
	407	+ .cache_size = 128,
	408	+ .cache_level = 1,
	409	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	410	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	411	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	412	+ .num_cu_shared = 10,
	413	+ },
	414	+ {
	415	+ /* L2 Data Cache per GPU (Total Tex Cache) */
	416	+ .cache_size = 4096,
	417	+ .cache_level = 2,
	418	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	419	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	420	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	421	+ .num_cu_shared = 10,
	422	+ },
	423	+};
	424	+
	425	+static struct kfd_gpu_cache_info vangogh_cache_info[] = {
	426	+ {
	427	+ /* TCP L1 Cache per CU */
	428	+ .cache_size = 16,
	429	+ .cache_level = 1,
	430	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	431	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	432	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	433	+ .num_cu_shared = 1,
	434	+ },
	435	+ {
	436	+ /* Scalar L1 Instruction Cache per SQC */
	437	+ .cache_size = 32,
	438	+ .cache_level = 1,
	439	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	440	+ CRAT_CACHE_FLAGS_INST_CACHE \|
	441	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	442	+ .num_cu_shared = 2,
	443	+ },
	444	+ {
	445	+ /* Scalar L1 Data Cache per SQC */
	446	+ .cache_size = 16,
	447	+ .cache_level = 1,
	448	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	449	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	450	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	451	+ .num_cu_shared = 2,
	452	+ },
	453	+ {
	454	+ /* GL1 Data Cache per SA */
	455	+ .cache_size = 128,
	456	+ .cache_level = 1,
	457	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	458	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	459	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	460	+ .num_cu_shared = 8,
73	461	},
74	462	{
75		− /* Scalar L1 Instruction Cache (in SQC module) per bank */
	463	+ /* L2 Data Cache per GPU (Total Tex Cache) */
	464	+ .cache_size = 1024,
	465	+ .cache_level = 2,
	466	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	467	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	468	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	469	+ .num_cu_shared = 8,
	470	+ },
	471	+};
	472	+
	473	+static struct kfd_gpu_cache_info navi14_cache_info[] = {
	474	+ {
	475	+ /* TCP L1 Cache per CU */
76	476	.cache_size = 16,
77	477	.cache_level = 1,
	478	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	479	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	480	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	481	+ .num_cu_shared = 1,
	482	+ },
	483	+ {
	484	+ /* Scalar L1 Instruction Cache per SQC */
	485	+ .cache_size = 32,
	486	+ .cache_level = 1,
78	487	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
79	488	CRAT_CACHE_FLAGS_INST_CACHE \|
80	489	CRAT_CACHE_FLAGS_SIMD_CACHE),
81	490	.num_cu_shared = 2,
82	491	},
83	492	{
84		− /* Scalar L1 Data Cache (in SQC module) per bank */
85		− .cache_size = 8,
	493	+ /* Scalar L1 Data Cache per SQC */
	494	+ .cache_size = 16,
86	495	.cache_level = 1,
87	496	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
88	497	CRAT_CACHE_FLAGS_DATA_CACHE \|
89	498	CRAT_CACHE_FLAGS_SIMD_CACHE),
90	499	.num_cu_shared = 2,
91	500	},
92		−
93		− /* TODO: Add L2 Cache information */
	501	+ {
	502	+ /* GL1 Data Cache per SA */
	503	+ .cache_size = 128,
	504	+ .cache_level = 1,
	505	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	506	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	507	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	508	+ .num_cu_shared = 12,
	509	+ },
	510	+ {
	511	+ /* L2 Data Cache per GPU (Total Tex Cache) */
	512	+ .cache_size = 2048,
	513	+ .cache_level = 2,
	514	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	515	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	516	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	517	+ .num_cu_shared = 12,
	518	+ },
94	519	};
95	520
96		−
97		−static struct kfd_gpu_cache_info carrizo_cache_info[] = {
	521	+static struct kfd_gpu_cache_info sienna_cichlid_cache_info[] = {
98	522	{
99	523	/* TCP L1 Cache per CU */
100	524	.cache_size = 16,
@@ -105,44 +529,165 @@ static struct kfd_gpu_cache_info carrizo_cache_info[] = { · 4 unmodified lines
105	529	.num_cu_shared = 1,
106	530	},
107	531	{
108		− /* Scalar L1 Instruction Cache (in SQC module) per bank */
109		− .cache_size = 8,
	532	+ /* Scalar L1 Instruction Cache per SQC */
	533	+ .cache_size = 32,
110	534	.cache_level = 1,
111	535	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
112	536	CRAT_CACHE_FLAGS_INST_CACHE \|
113	537	CRAT_CACHE_FLAGS_SIMD_CACHE),
114		− .num_cu_shared = 4,
	538	+ .num_cu_shared = 2,
115	539	},
116	540	{
117		− /* Scalar L1 Data Cache (in SQC module) per bank. */
118		− .cache_size = 4,
	541	+ /* Scalar L1 Data Cache per SQC */
	542	+ .cache_size = 16,
119	543	.cache_level = 1,
120	544	.flags = (CRAT_CACHE_FLAGS_ENABLED \|
121	545	CRAT_CACHE_FLAGS_DATA_CACHE \|
122	546	CRAT_CACHE_FLAGS_SIMD_CACHE),
123		− .num_cu_shared = 4,
	547	+ .num_cu_shared = 2,
	548	+ },
	549	+ {
	550	+ /* GL1 Data Cache per SA */
	551	+ .cache_size = 128,
	552	+ .cache_level = 1,
	553	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	554	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	555	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	556	+ .num_cu_shared = 10,
	557	+ },
	558	+ {
	559	+ /* L2 Data Cache per GPU (Total Tex Cache) */
	560	+ .cache_size = 4096,
	561	+ .cache_level = 2,
	562	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	563	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	564	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	565	+ .num_cu_shared = 10,
124	566	},
	567	+ {
	568	+ /* L3 Data Cache per GPU */
	569	+ .cache_size = 128*1024,
	570	+ .cache_level = 3,
	571	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	572	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	573	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	574	+ .num_cu_shared = 10,
	575	+ },
	576	+};
125	577
126		− /* TODO: Add L2 Cache information */
	578	+static struct kfd_gpu_cache_info navy_flounder_cache_info[] = {
	579	+ {
	580	+ /* TCP L1 Cache per CU */
	581	+ .cache_size = 16,
	582	+ .cache_level = 1,
	583	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	584	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	585	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	586	+ .num_cu_shared = 1,
	587	+ },
	588	+ {
	589	+ /* Scalar L1 Instruction Cache per SQC */
	590	+ .cache_size = 32,
	591	+ .cache_level = 1,
	592	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	593	+ CRAT_CACHE_FLAGS_INST_CACHE \|
	594	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	595	+ .num_cu_shared = 2,
	596	+ },
	597	+ {
	598	+ /* Scalar L1 Data Cache per SQC */
	599	+ .cache_size = 16,
	600	+ .cache_level = 1,
	601	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	602	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	603	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	604	+ .num_cu_shared = 2,
	605	+ },
	606	+ {
	607	+ /* GL1 Data Cache per SA */
	608	+ .cache_size = 128,
	609	+ .cache_level = 1,
	610	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	611	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	612	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	613	+ .num_cu_shared = 10,
	614	+ },
	615	+ {
	616	+ /* L2 Data Cache per GPU (Total Tex Cache) */
	617	+ .cache_size = 3072,
	618	+ .cache_level = 2,
	619	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	620	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	621	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	622	+ .num_cu_shared = 10,
	623	+ },
	624	+ {
	625	+ /* L3 Data Cache per GPU */
	626	+ .cache_size = 96*1024,
	627	+ .cache_level = 3,
	628	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	629	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	630	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	631	+ .num_cu_shared = 10,
	632	+ },
127	633	};
128	634
129		−/* NOTE: In future if more information is added to struct kfd_gpu_cache_info
130		− * the following ASICs may need a separate table.
131		− */
132		−#define hawaii_cache_info kaveri_cache_info
133		−#define tonga_cache_info carrizo_cache_info
134		−#define fiji_cache_info carrizo_cache_info
135		−#define polaris10_cache_info carrizo_cache_info
136		−#define polaris11_cache_info carrizo_cache_info
137		−#define polaris12_cache_info carrizo_cache_info
138		−#define vegam_cache_info carrizo_cache_info
139		−/* TODO - check & update Vega10 cache details */
140		−#define vega10_cache_info carrizo_cache_info
141		−#define raven_cache_info carrizo_cache_info
142		−#define renoir_cache_info carrizo_cache_info
143		−/* TODO - check & update Navi10 cache details */
144		−#define navi10_cache_info carrizo_cache_info
145		−#define vangogh_cache_info carrizo_cache_info
	635	+static struct kfd_gpu_cache_info dimgrey_cavefish_cache_info[] = {
	636	+ {
	637	+ /* TCP L1 Cache per CU */
	638	+ .cache_size = 16,
	639	+ .cache_level = 1,
	640	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	641	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	642	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	643	+ .num_cu_shared = 1,
	644	+ },
	645	+ {
	646	+ /* Scalar L1 Instruction Cache per SQC */
	647	+ .cache_size = 32,
	648	+ .cache_level = 1,
	649	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	650	+ CRAT_CACHE_FLAGS_INST_CACHE \|
	651	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	652	+ .num_cu_shared = 2,
	653	+ },
	654	+ {
	655	+ /* Scalar L1 Data Cache per SQC */
	656	+ .cache_size = 16,
	657	+ .cache_level = 1,
	658	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	659	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	660	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	661	+ .num_cu_shared = 2,
	662	+ },
	663	+ {
	664	+ /* GL1 Data Cache per SA */
	665	+ .cache_size = 128,
	666	+ .cache_level = 1,
	667	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	668	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	669	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	670	+ .num_cu_shared = 8,
	671	+ },
	672	+ {
	673	+ /* L2 Data Cache per GPU (Total Tex Cache) */
	674	+ .cache_size = 2048,
	675	+ .cache_level = 2,
	676	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	677	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	678	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	679	+ .num_cu_shared = 8,
	680	+ },
	681	+ {
	682	+ /* L3 Data Cache per GPU */
	683	+ .cache_size = 32*1024,
	684	+ .cache_level = 3,
	685	+ .flags = (CRAT_CACHE_FLAGS_ENABLED \|
	686	+ CRAT_CACHE_FLAGS_DATA_CACHE \|
	687	+ CRAT_CACHE_FLAGS_SIMD_CACHE),
	688	+ .num_cu_shared = 8,
	689	+ },
	690	+};
146	691
147	692	static void kfd_populated_cu_info_cpu(struct kfd_topology_device *dev,
148	693	struct crat_subtype_computeunit *cu)
@@ -544,7 +1089,7 @@ err: · 395 unmodified lines
544	1089	}
545	1090
546	1091	/* Helper function. See kfd_fill_gpu_cache_info for parameter description */
547		−static int fill_in_pcache(struct crat_subtype_cache *pcache,
	1092	+static int fill_in_l1_pcache(struct crat_subtype_cache *pcache,
548	1093	struct kfd_gpu_cache_info *pcache_info,
549	1094	struct kfd_cu_info *cu_info,
550	1095	int mem_available,
@@ -597,6 +1142,70 @@ static int fill_in_pcache(struct crat_subtype_cache *pcache, · 46 unmodified lines
597	1142	return 1;
598	1143	}
599	1144
	1145	+/* Helper function. See kfd_fill_gpu_cache_info for parameter description */
	1146	+static int fill_in_l2_l3_pcache(struct crat_subtype_cache *pcache,
	1147	+ struct kfd_gpu_cache_info *pcache_info,
	1148	+ struct kfd_cu_info *cu_info,
	1149	+ int mem_available,
	1150	+ int cache_type, unsigned int cu_processor_id)
	1151	+{
	1152	+ unsigned int cu_sibling_map_mask;
	1153	+ int first_active_cu;
	1154	+ int i, j, k;
	1155	+
	1156	+ /* First check if enough memory is available */
	1157	+ if (sizeof(struct crat_subtype_cache) > mem_available)
	1158	+ return -ENOMEM;
	1159	+
	1160	+ cu_sibling_map_mask = cu_info->cu_bitmap[0][0];
	1161	+ cu_sibling_map_mask &=
	1162	+ ((1 << pcache_info[cache_type].num_cu_shared) - 1);
	1163	+ first_active_cu = ffs(cu_sibling_map_mask);
	1164	+
	1165	+ /* CU could be inactive. In case of shared cache find the first active
	1166	+ * CU. and incase of non-shared cache check if the CU is inactive. If
	1167	+ * inactive active skip it
	1168	+ */
	1169	+ if (first_active_cu) {
	1170	+ memset(pcache, 0, sizeof(struct crat_subtype_cache));
	1171	+ pcache->type = CRAT_SUBTYPE_CACHE_AFFINITY;
	1172	+ pcache->length = sizeof(struct crat_subtype_cache);
	1173	+ pcache->flags = pcache_info[cache_type].flags;
	1174	+ pcache->processor_id_low = cu_processor_id
	1175	+ + (first_active_cu - 1);
	1176	+ pcache->cache_level = pcache_info[cache_type].cache_level;
	1177	+ pcache->cache_size = pcache_info[cache_type].cache_size;
	1178	+
	1179	+ /* Sibling map is w.r.t processor_id_low, so shift out
	1180	+ * inactive CU
	1181	+ */
	1182	+ cu_sibling_map_mask =
	1183	+ cu_sibling_map_mask >> (first_active_cu - 1);
	1184	+ k = 0;
	1185	+ for (i = 0; i < cu_info->num_shader_engines; i++) {
	1186	+ for (j = 0; j < cu_info->num_shader_arrays_per_engine;
	1187	+ j++) {
	1188	+ pcache->sibling_map[k] =
	1189	+ (uint8_t)(cu_sibling_map_mask & 0xFF);
	1190	+ pcache->sibling_map[k+1] =
	1191	+ (uint8_t)((cu_sibling_map_mask >> 8) & 0xFF);
	1192	+ pcache->sibling_map[k+2] =
	1193	+ (uint8_t)((cu_sibling_map_mask >> 16) & 0xFF);
	1194	+ pcache->sibling_map[k+3] =
	1195	+ (uint8_t)((cu_sibling_map_mask >> 24) & 0xFF);
	1196	+ k += 4;
	1197	+ cu_sibling_map_mask =
	1198	+ cu_info->cu_bitmap[i % 4][j + i / 4];
	1199	+ cu_sibling_map_mask &= (
	1200	+ (1 << pcache_info[cache_type].num_cu_shared)
	1201	+ - 1);
	1202	+ }
	1203	+ }
	1204	+ return 0;
	1205	+ }
	1206	+ return 1;
	1207	+}
	1208	+
600	1209	/* kfd_fill_gpu_cache_info - Fill GPU cache info using kfd_gpu_cache_info
601	1210	* tables
602	1211	*
@@ -624,6 +1233,7 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, · 21 unmodified lines
624	1233	int mem_available = available_size;
625	1234	unsigned int cu_processor_id;
626	1235	int ret;
	1236	+ unsigned int num_cu_shared;
627	1237
628	1238	switch (kdev->device_info->asic_family) {
629	1239	case CHIP_KAVERI:
@@ -663,12 +1273,21 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, · 33 unmodified lines
663	1273	num_of_cache_types = ARRAY_SIZE(vegam_cache_info);
664	1274	break;
665	1275	case CHIP_VEGA10:
	1276	+ pcache_info = vega10_cache_info;
	1277	+ num_of_cache_types = ARRAY_SIZE(vega10_cache_info);
	1278	+ break;
666	1279	case CHIP_VEGA12:
	1280	+ pcache_info = vega12_cache_info;
	1281	+ num_of_cache_types = ARRAY_SIZE(vega12_cache_info);
	1282	+ break;
667	1283	case CHIP_VEGA20:
668	1284	case CHIP_ARCTURUS:
	1285	+ pcache_info = vega20_cache_info;
	1286	+ num_of_cache_types = ARRAY_SIZE(vega20_cache_info);
	1287	+ break;
669	1288	case CHIP_ALDEBARAN:
670		− pcache_info = vega10_cache_info;
671		− num_of_cache_types = ARRAY_SIZE(vega10_cache_info);
	1289	+ pcache_info = aldebaran_cache_info;
	1290	+ num_of_cache_types = ARRAY_SIZE(aldebaran_cache_info);
672	1291	break;
673	1292	case CHIP_RAVEN:
674	1293	pcache_info = raven_cache_info;
@@ -680,12 +1299,24 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, · 5 unmodified lines
680	1299	break;
681	1300	case CHIP_NAVI10:
682	1301	case CHIP_NAVI12:
	1302	+ pcache_info = navi10_cache_info;
	1303	+ num_of_cache_types = ARRAY_SIZE(navi10_cache_info);
	1304	+ break;
683	1305	case CHIP_NAVI14:
	1306	+ pcache_info = navi14_cache_info;
	1307	+ num_of_cache_types = ARRAY_SIZE(navi14_cache_info);
	1308	+ break;
684	1309	case CHIP_SIENNA_CICHLID:
	1310	+ pcache_info = sienna_cichlid_cache_info;
	1311	+ num_of_cache_types = ARRAY_SIZE(sienna_cichlid_cache_info);
	1312	+ break;
685	1313	case CHIP_NAVY_FLOUNDER:
	1314	+ pcache_info = navy_flounder_cache_info;
	1315	+ num_of_cache_types = ARRAY_SIZE(navy_flounder_cache_info);
	1316	+ break;
686	1317	case CHIP_DIMGREY_CAVEFISH:
687		− pcache_info = navi10_cache_info;
688		− num_of_cache_types = ARRAY_SIZE(navi10_cache_info);
	1318	+ pcache_info = dimgrey_cavefish_cache_info;
	1319	+ num_of_cache_types = ARRAY_SIZE(dimgrey_cavefish_cache_info);
689	1320	break;
690	1321	case CHIP_VANGOGH:
691	1322	pcache_info = vangogh_cache_info;
@@ -709,40 +1340,58 @@ static int kfd_fill_gpu_cache_info(struct kfd_dev *kdev, · 17 unmodified lines
709	1340	*/
710	1341
711	1342	for (ct = 0; ct < num_of_cache_types; ct++) {
712		− cu_processor_id = gpu_processor_id;
713		− for (i = 0; i < cu_info->num_shader_engines; i++) {
714		− for (j = 0; j < cu_info->num_shader_arrays_per_engine;
715		− j++) {
716		− for (k = 0; k < cu_info->num_cu_per_sh;
717		− k += pcache_info[ct].num_cu_shared) {
718		−
719		− ret = fill_in_pcache(pcache,
720		− pcache_info,
721		− cu_info,
722		− mem_available,
723		− cu_info->cu_bitmap[i % 4][j + i / 4],
724		− ct,
725		− cu_processor_id,
726		− k);
727		−
728		− if (ret < 0)
729		− break;
730		−
731		− if (!ret) {
732		− pcache++;
733		− (*num_of_entries)++;
734		− mem_available -=
735		− sizeof(*pcache);
736		− (*size_filled) +=
737		− sizeof(*pcache);
738		− }
739		−
740		− /* Move to next CU block */
741		− cu_processor_id +=
742		− pcache_info[ct].num_cu_shared;
743		− }
744		− }
	1343	+ cu_processor_id = gpu_processor_id;
	1344	+ if (pcache_info[ct].cache_level == 1) {
	1345	+ for (i = 0; i < cu_info->num_shader_engines; i++) {
	1346	+ for (j = 0; j < cu_info->num_shader_arrays_per_engine; j++) {
	1347	+ for (k = 0; k < cu_info->num_cu_per_sh;
	1348	+ k += pcache_info[ct].num_cu_shared) {
	1349	+ ret = fill_in_l1_pcache(pcache,
	1350	+ pcache_info,
	1351	+ cu_info,
	1352	+ mem_available,
	1353	+ cu_info->cu_bitmap[i % 4][j + i / 4],
	1354	+ ct,
	1355	+ cu_processor_id,
	1356	+ k);
	1357	+
	1358	+ if (ret < 0)
	1359	+ break;
	1360	+
	1361	+ if (!ret) {
	1362	+ pcache++;
	1363	+ (*num_of_entries)++;
	1364	+ mem_available -= sizeof(*pcache);
	1365	+ (size_filled) += sizeof(pcache);
	1366	+ }
	1367	+
	1368	+ /* Move to next CU block */
	1369	+ num_cu_shared = ((k + pcache_info[ct].num_cu_shared) <=
	1370	+ cu_info->num_cu_per_sh) ?
	1371	+ pcache_info[ct].num_cu_shared :
	1372	+ (cu_info->num_cu_per_sh - k);
	1373	+ cu_processor_id += num_cu_shared;
745	1374	}
	1375	+ }
	1376	+ }
	1377	+ } else {
	1378	+ ret = fill_in_l2_l3_pcache(pcache,
	1379	+ pcache_info,
	1380	+ cu_info,
	1381	+ mem_available,
	1382	+ ct,
	1383	+ cu_processor_id);
	1384	+
	1385	+ if (ret < 0)
	1386	+ break;
	1387	+
	1388	+ if (!ret) {
	1389	+ pcache++;
	1390	+ (*num_of_entries)++;
	1391	+ mem_available -= sizeof(*pcache);
	1392	+ (size_filled) += sizeof(pcache);
	1393	+ }
	1394	+ }
746	1395	}
747	1396
748	1397	pr_debug("Added [%d] GPU cache entries\n", *num_of_entries);
@@ -1100,6 +1749,92 @@ static int kfd_fill_gpu_memory_affinity(int *avail_size, · 351 unmodified lines
1100	1749	return 0;
1101	1750	}
1102	1751
	1752	+#ifdef CONFIG_ACPI_NUMA
	1753	+static void kfd_find_numa_node_in_srat(struct kfd_dev *kdev)
	1754	+{
	1755	+ struct acpi_table_header *table_header = NULL;
	1756	+ struct acpi_subtable_header *sub_header = NULL;
	1757	+ unsigned long table_end, subtable_len;
	1758	+ u32 pci_id = pci_domain_nr(kdev->pdev->bus) << 16 \|
	1759	+ pci_dev_id(kdev->pdev);
	1760	+ u32 bdf;
	1761	+ acpi_status status;
	1762	+ struct acpi_srat_cpu_affinity *cpu;
	1763	+ struct acpi_srat_generic_affinity *gpu;
	1764	+ int pxm = 0, max_pxm = 0;
	1765	+ int numa_node = NUMA_NO_NODE;
	1766	+ bool found = false;
	1767	+
	1768	+ /* Fetch the SRAT table from ACPI */
	1769	+ status = acpi_get_table(ACPI_SIG_SRAT, 0, &table_header);
	1770	+ if (status == AE_NOT_FOUND) {
	1771	+ pr_warn("SRAT table not found\n");
	1772	+ return;
	1773	+ } else if (ACPI_FAILURE(status)) {
	1774	+ const char *err = acpi_format_exception(status);
	1775	+ pr_err("SRAT table error: %s\n", err);
	1776	+ return;
	1777	+ }
	1778	+
	1779	+ table_end = (unsigned long)table_header + table_header->length;
	1780	+
	1781	+ /* Parse all entries looking for a match. */
	1782	+ sub_header = (struct acpi_subtable_header *)
	1783	+ ((unsigned long)table_header +
	1784	+ sizeof(struct acpi_table_srat));
	1785	+ subtable_len = sub_header->length;
	1786	+
	1787	+ while (((unsigned long)sub_header) + subtable_len < table_end) {
	1788	+ /*
	1789	+ * If length is 0, break from this loop to avoid
	1790	+ * infinite loop.
	1791	+ */
	1792	+ if (subtable_len == 0) {
	1793	+ pr_err("SRAT invalid zero length\n");
	1794	+ break;
	1795	+ }
	1796	+
	1797	+ switch (sub_header->type) {
	1798	+ case ACPI_SRAT_TYPE_CPU_AFFINITY:
	1799	+ cpu = (struct acpi_srat_cpu_affinity *)sub_header;
	1800	+ pxm = ((u32 )cpu->proximity_domain_hi) << 8 \|
	1801	+ cpu->proximity_domain_lo;
	1802	+ if (pxm > max_pxm)
	1803	+ max_pxm = pxm;
	1804	+ break;
	1805	+ case ACPI_SRAT_TYPE_GENERIC_AFFINITY:
	1806	+ gpu = (struct acpi_srat_generic_affinity *)sub_header;
	1807	+ bdf = ((u16 )(&gpu->device_handle[0])) << 16 \|
	1808	+ ((u16 )(&gpu->device_handle[2]));
	1809	+ if (bdf == pci_id) {
	1810	+ found = true;
	1811	+ numa_node = pxm_to_node(gpu->proximity_domain);
	1812	+ }
	1813	+ break;
	1814	+ default:
	1815	+ break;
	1816	+ }
	1817	+
	1818	+ if (found)
	1819	+ break;
	1820	+
	1821	+ sub_header = (struct acpi_subtable_header *)
	1822	+ ((unsigned long)sub_header + subtable_len);
	1823	+ subtable_len = sub_header->length;
	1824	+ }
	1825	+
	1826	+ acpi_put_table(table_header);
	1827	+
	1828	+ /* Workaround bad cpu-gpu binding case */
	1829	+ if (found && (numa_node < 0 \|\|
	1830	+ numa_node > pxm_to_node(max_pxm)))
	1831	+ numa_node = 0;
	1832	+
	1833	+ if (numa_node != NUMA_NO_NODE)
	1834	+ set_dev_node(&kdev->pdev->dev, numa_node);
	1835	+}
	1836	+#endif
	1837	+
1103	1838	/* kfd_fill_gpu_direct_io_link - Fill in direct io link from GPU
1104	1839	* to its NUMA node
1105	1840	* @avail_size: Available size in the memory
@@ -1140,11 +1875,17 @@ static int kfd_fill_gpu_direct_io_link_to_cpu(int *avail_size, · 34 unmodified lines
1140	1875	*/
1141	1876	sub_type_hdr->flags \|= CRAT_IOLINK_FLAGS_BI_DIRECTIONAL;
1142	1877	sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_XGMI;
	1878	+ sub_type_hdr->num_hops_xgmi = 1;
1143	1879	} else {
1144	1880	sub_type_hdr->io_interface_type = CRAT_IOLINK_TYPE_PCIEXPRESS;
1145	1881	}
1146	1882
1147	1883	sub_type_hdr->proximity_domain_from = proximity_domain;
	1884	+
	1885	+#ifdef CONFIG_ACPI_NUMA
	1886	+ if (kdev->pdev->dev.numa_node == NUMA_NO_NODE)
	1887	+ kfd_find_numa_node_in_srat(kdev);
	1888	+#endif
1148	1889	#ifdef CONFIG_NUMA
1149	1890	if (kdev->pdev->dev.numa_node == NUMA_NO_NODE)
1150	1891	sub_type_hdr->proximity_domain_to = 0;

drivers/gpu/drm/amd/amdkfd/kfd_device.c+15−4

@@ -26,10 +26,12 @@ · 25 unmodified lines
26	26	#include "kfd_priv.h"
27	27	#include "kfd_device_queue_manager.h"
28	28	#include "kfd_pm4_headers_vi.h"
	29	+#include "kfd_pm4_headers_aldebaran.h"
29	30	#include "cwsr_trap_handler.h"
30	31	#include "kfd_iommu.h"
31	32	#include "amdgpu_amdkfd.h"
32	33	#include "kfd_smi_events.h"
	34	+#include "kfd_migrate.h"
33	35
34	36	#define MQD_SIZE_ALIGNED 768
35	37
@@ -576,7 +578,7 @@ static const struct kfd_device_info *kfd_supported_devices[][2] = { · 540 unmodified lines
576	578	[CHIP_VEGA20] = {&vega20_device_info, NULL},
577	579	[CHIP_RENOIR] = {&renoir_device_info, NULL},
578	580	[CHIP_ARCTURUS] = {&arcturus_device_info, &arcturus_device_info},
579		− [CHIP_ALDEBARAN] = {&aldebaran_device_info, NULL},
	581	+ [CHIP_ALDEBARAN] = {&aldebaran_device_info, &aldebaran_device_info},
580	582	[CHIP_NAVI10] = {&navi10_device_info, NULL},
581	583	[CHIP_NAVI12] = {&navi12_device_info, &navi12_device_info},
582	584	[CHIP_NAVI14] = {&navi14_device_info, NULL},
@@ -697,7 +699,9 @@ static int kfd_gws_init(struct kfd_dev *kfd) · 114 unmodified lines
697	699	&& kfd->device_info->asic_family <= CHIP_RAVEN
698	700	&& kfd->mec2_fw_version >= 0x1b3)
699	701	\|\| (kfd->device_info->asic_family == CHIP_ARCTURUS
700		− && kfd->mec2_fw_version >= 0x30))
	702	+ && kfd->mec2_fw_version >= 0x30)
	703	+ \|\| (kfd->device_info->asic_family == CHIP_ALDEBARAN
	704	+ && kfd->mec2_fw_version >= 0x28))
701	705	ret = amdgpu_amdkfd_alloc_gws(kfd->kgd,
702	706	amdgpu_amdkfd_get_num_gws(kfd->kgd), &kfd->gws);
703	707
@@ -713,7 +717,7 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, · 9 unmodified lines
713	717	struct drm_device *ddev,
714	718	const struct kgd2kfd_shared_resources *gpu_resources)
715	719	{
716		− unsigned int size;
	720	+ unsigned int size, map_process_packet_size;
717	721
718	722	kfd->ddev = ddev;
719	723	kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
@@ -748,7 +752,11 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, · 28 unmodified lines
748	752	* calculate max size of runlist packet.
749	753	* There can be only 2 packets at once
750	754	*/
751		− size += (KFD_MAX_NUM_OF_PROCESSES * sizeof(struct pm4_mes_map_process) +
	755	+ map_process_packet_size =
	756	+ kfd->device_info->asic_family == CHIP_ALDEBARAN ?
	757	+ sizeof(struct pm4_mes_map_process_aldebaran) :
	758	+ sizeof(struct pm4_mes_map_process);
	759	+ size += (KFD_MAX_NUM_OF_PROCESSES * map_process_packet_size +
752	760	max_num_of_queues_per_device * sizeof(struct pm4_mes_map_queues)
753	761	+ sizeof(struct pm4_mes_runlist)) * 2;
754	762
@@ -814,6 +822,8 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, · 59 unmodified lines
814	822
815	823	kfd_cwsr_init(kfd);
816	824
	825	+ svm_migrate_init((struct amdgpu_device *)kfd->kgd);
	826	+
817	827	if (kfd_resume(kfd))
818	828	goto kfd_resume_error;
819	829
@@ -862,6 +872,7 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd) · 42 unmodified lines
862	872	{
863	873	if (kfd->init_complete) {
864	874	kgd2kfd_suspend(kfd, false);
	875	+ svm_migrate_fini((struct amdgpu_device *)kfd->kgd);
865	876	device_queue_manager_uninit(kfd->dqm);
866	877	kfd_interrupt_exit(kfd);
867	878	kfd_topology_remove_device(kfd);

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c+3−3

@@ -738,7 +738,7 @@ static int restore_process_queues_nocpsch(struct device_queue_manager *dqm, · 737 unmodified lines
738	738
739	739	pdd = qpd_to_pdd(qpd);
740	740	/* Retrieve PD base */
741		− pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);
	741	+ pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
742	742
743	743	dqm_lock(dqm);
744	744	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
@@ -821,7 +821,7 @@ static int restore_process_queues_cpsch(struct device_queue_manager *dqm, · 76 unmodified lines
821	821
822	822	pdd = qpd_to_pdd(qpd);
823	823	/* Retrieve PD base */
824		− pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);
	824	+ pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
825	825
826	826	dqm_lock(dqm);
827	827	if (WARN_ON_ONCE(!qpd->evicted)) /* already restored, do nothing */
@@ -873,7 +873,7 @@ static int register_process(struct device_queue_manager *dqm, · 45 unmodified lines
873	873
874	874	pdd = qpd_to_pdd(qpd);
875	875	/* Retrieve PD base */
876		− pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->vm);
	876	+ pd_base = amdgpu_amdkfd_gpuvm_get_process_page_dir(pdd->drm_priv);
877	877
878	878	dqm_lock(dqm);
879	879	list_add(&n->list, &dqm->queues);

drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager_v9.c+11−2

@@ -61,10 +61,19 @@ static int update_qpd_v9(struct device_queue_manager *dqm, · 60 unmodified lines
61	61	qpd->sh_mem_config =
62	62	SH_MEM_ALIGNMENT_MODE_UNALIGNED <<
63	63	SH_MEM_CONFIG__ALIGNMENT_MODE__SHIFT;
64		− if (dqm->dev->noretry &&
65		− !dqm->dev->use_iommu_v2)
	64	+
	65	+ if (dqm->dev->device_info->asic_family == CHIP_ALDEBARAN) {
	66	+ /* Aldebaran can safely support different XNACK modes
	67	+ * per process
	68	+ */
	69	+ if (!pdd->process->xnack_enabled)
	70	+ qpd->sh_mem_config \|=
	71	+ 1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
	72	+ } else if (dqm->dev->noretry &&
	73	+ !dqm->dev->use_iommu_v2) {
66	74	qpd->sh_mem_config \|=
67	75	1 << SH_MEM_CONFIG__RETRY_DISABLE__SHIFT;
	76	+ }
68	77
69	78	qpd->sh_mem_ape1_limit = 0;
70	79	qpd->sh_mem_ape1_base = 0;

drivers/gpu/drm/amd/amdkfd/kfd_flat_memory.c+4−0

@@ -405,6 +405,10 @@ int kfd_init_apertures(struct kfd_process *process) · 404 unmodified lines
405	405	case CHIP_POLARIS12:
406	406	case CHIP_VEGAM:
407	407	kfd_init_apertures_vi(pdd, id);
	408	+ /* VI GPUs cannot support SVM with only
	409	+ * 40 bits of virtual address space.
	410	+ */
	411	+ process->svm_disabled = true;
408	412	break;
409	413	case CHIP_VEGA10:
410	414	case CHIP_VEGA12:

drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c+0−1

@@ -25,7 +25,6 @@ · 24 unmodified lines
25	25	#include "soc15_int.h"
26	26	#include "kfd_device_queue_manager.h"
27	27	#include "kfd_smi_events.h"
28		−#include "amdgpu.h"
29	28
30	29	enum SQ_INTERRUPT_WORD_ENCODING {
31	30	SQ_INTERRUPT_WORD_ENCODING_AUTO = 0x0,

drivers/gpu/drm/amd/amdkfd/kfd_migrate.c+937−0

@@ -0,0 +1,937 @@
	1	+// SPDX-License-Identifier: GPL-2.0 OR MIT
	2	+/*
	3	+ * Copyright 2020-2021 Advanced Micro Devices, Inc.
	4	+ *
	5	+ * Permission is hereby granted, free of charge, to any person obtaining a
	6	+ * copy of this software and associated documentation files (the "Software"),
	7	+ * to deal in the Software without restriction, including without limitation
	8	+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
	9	+ * and/or sell copies of the Software, and to permit persons to whom the
	10	+ * Software is furnished to do so, subject to the following conditions:
	11	+ *
	12	+ * The above copyright notice and this permission notice shall be included in
	13	+ * all copies or substantial portions of the Software.
	14	+ *
	15	+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	19	+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	20	+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	21	+ * OTHER DEALINGS IN THE SOFTWARE.
	22	+ */
	23	+
	24	+#include <linux/types.h>
	25	+#include <linux/hmm.h>
	26	+#include <linux/dma-direction.h>
	27	+#include <linux/dma-mapping.h>
	28	+#include "amdgpu_sync.h"
	29	+#include "amdgpu_object.h"
	30	+#include "amdgpu_vm.h"
	31	+#include "amdgpu_mn.h"
	32	+#include "kfd_priv.h"
	33	+#include "kfd_svm.h"
	34	+#include "kfd_migrate.h"
	35	+
	36	+static uint64_t
	37	+svm_migrate_direct_mapping_addr(struct amdgpu_device *adev, uint64_t addr)
	38	+{
	39	+ return addr + amdgpu_ttm_domain_start(adev, TTM_PL_VRAM);
	40	+}
	41	+
	42	+static int
	43	+svm_migrate_gart_map(struct amdgpu_ring *ring, uint64_t npages,
	44	+ dma_addr_t addr, uint64_t gart_addr, uint64_t flags)
	45	+{
	46	+ struct amdgpu_device *adev = ring->adev;
	47	+ struct amdgpu_job *job;
	48	+ unsigned int num_dw, num_bytes;
	49	+ struct dma_fence *fence;
	50	+ uint64_t src_addr, dst_addr;
	51	+ uint64_t pte_flags;
	52	+ void *cpu_addr;
	53	+ int r;
	54	+
	55	+ /* use gart window 0 */
	56	+ *gart_addr = adev->gmc.gart_start;
	57	+
	58	+ num_dw = ALIGN(adev->mman.buffer_funcs->copy_num_dw, 8);
	59	+ num_bytes = npages * 8;
	60	+
	61	+ r = amdgpu_job_alloc_with_ib(adev, num_dw * 4 + num_bytes,
	62	+ AMDGPU_IB_POOL_DELAYED, &job);
	63	+ if (r)
	64	+ return r;
	65	+
	66	+ src_addr = num_dw * 4;
	67	+ src_addr += job->ibs[0].gpu_addr;
	68	+
	69	+ dst_addr = amdgpu_bo_gpu_offset(adev->gart.bo);
	70	+ amdgpu_emit_copy_buffer(adev, &job->ibs[0], src_addr,
	71	+ dst_addr, num_bytes, false);
	72	+
	73	+ amdgpu_ring_pad_ib(ring, &job->ibs[0]);
	74	+ WARN_ON(job->ibs[0].length_dw > num_dw);
	75	+
	76	+ pte_flags = AMDGPU_PTE_VALID \| AMDGPU_PTE_READABLE;
	77	+ pte_flags \|= AMDGPU_PTE_SYSTEM \| AMDGPU_PTE_SNOOPED;
	78	+ if (!(flags & KFD_IOCTL_SVM_FLAG_GPU_RO))
	79	+ pte_flags \|= AMDGPU_PTE_WRITEABLE;
	80	+ pte_flags \|= adev->gart.gart_pte_flags;
	81	+
	82	+ cpu_addr = &job->ibs[0].ptr[num_dw];
	83	+
	84	+ r = amdgpu_gart_map(adev, 0, npages, addr, pte_flags, cpu_addr);
	85	+ if (r)
	86	+ goto error_free;
	87	+
	88	+ r = amdgpu_job_submit(job, &adev->mman.entity,
	89	+ AMDGPU_FENCE_OWNER_UNDEFINED, &fence);
	90	+ if (r)
	91	+ goto error_free;
	92	+
	93	+ dma_fence_put(fence);
	94	+
	95	+ return r;
	96	+
	97	+error_free:
	98	+ amdgpu_job_free(job);
	99	+ return r;
	100	+}
	101	+
	102	+/**
	103	+ * svm_migrate_copy_memory_gart - sdma copy data between ram and vram
	104	+ *
	105	+ * @adev: amdgpu device the sdma ring running
	106	+ * @src: source page address array
	107	+ * @dst: destination page address array
	108	+ * @npages: number of pages to copy
	109	+ * @direction: enum MIGRATION_COPY_DIR
	110	+ * @mfence: output, sdma fence to signal after sdma is done
	111	+ *
	112	+ * ram address uses GART table continuous entries mapping to ram pages,
	113	+ * vram address uses direct mapping of vram pages, which must have npages
	114	+ * number of continuous pages.
	115	+ * GART update and sdma uses same buf copy function ring, sdma is splited to
	116	+ * multiple GTT_MAX_PAGES transfer, all sdma operations are serialized, wait for
	117	+ * the last sdma finish fence which is returned to check copy memory is done.
	118	+ *
	119	+ * Context: Process context, takes and releases gtt_window_lock
	120	+ *
	121	+ * Return:
	122	+ * 0 - OK, otherwise error code
	123	+ */
	124	+
	125	+static int
	126	+svm_migrate_copy_memory_gart(struct amdgpu_device adev, dma_addr_t sys,
	127	+ uint64_t *vram, uint64_t npages,
	128	+ enum MIGRATION_COPY_DIR direction,
	129	+ struct dma_fence **mfence)
	130	+{
	131	+ const uint64_t GTT_MAX_PAGES = AMDGPU_GTT_MAX_TRANSFER_SIZE;
	132	+ struct amdgpu_ring *ring = adev->mman.buffer_funcs_ring;
	133	+ uint64_t gart_s, gart_d;
	134	+ struct dma_fence *next;
	135	+ uint64_t size;
	136	+ int r;
	137	+
	138	+ mutex_lock(&adev->mman.gtt_window_lock);
	139	+
	140	+ while (npages) {
	141	+ size = min(GTT_MAX_PAGES, npages);
	142	+
	143	+ if (direction == FROM_VRAM_TO_RAM) {
	144	+ gart_s = svm_migrate_direct_mapping_addr(adev, *vram);
	145	+ r = svm_migrate_gart_map(ring, size, sys, &gart_d, 0);
	146	+
	147	+ } else if (direction == FROM_RAM_TO_VRAM) {
	148	+ r = svm_migrate_gart_map(ring, size, sys, &gart_s,
	149	+ KFD_IOCTL_SVM_FLAG_GPU_RO);
	150	+ gart_d = svm_migrate_direct_mapping_addr(adev, *vram);
	151	+ }
	152	+ if (r) {
	153	+ pr_debug("failed %d to create gart mapping\n", r);
	154	+ goto out_unlock;
	155	+ }
	156	+
	157	+ r = amdgpu_copy_buffer(ring, gart_s, gart_d, size * PAGE_SIZE,
	158	+ NULL, &next, false, true, false);
	159	+ if (r) {
	160	+ pr_debug("failed %d to copy memory\n", r);
	161	+ goto out_unlock;
	162	+ }
	163	+
	164	+ dma_fence_put(*mfence);
	165	+ *mfence = next;
	166	+ npages -= size;
	167	+ if (npages) {
	168	+ sys += size;
	169	+ vram += size;
	170	+ }
	171	+ }
	172	+
	173	+out_unlock:
	174	+ mutex_unlock(&adev->mman.gtt_window_lock);
	175	+
	176	+ return r;
	177	+}
	178	+
	179	+/**
	180	+ * svm_migrate_copy_done - wait for memory copy sdma is done
	181	+ *
	182	+ * @adev: amdgpu device the sdma memory copy is executing on
	183	+ * @mfence: migrate fence
	184	+ *
	185	+ * Wait for dma fence is signaled, if the copy ssplit into multiple sdma
	186	+ * operations, this is the last sdma operation fence.
	187	+ *
	188	+ * Context: called after svm_migrate_copy_memory
	189	+ *
	190	+ * Return:
	191	+ * 0 - success
	192	+ * otherwise - error code from dma fence signal
	193	+ */
	194	+static int
	195	+svm_migrate_copy_done(struct amdgpu_device adev, struct dma_fence mfence)
	196	+{
	197	+ int r = 0;
	198	+
	199	+ if (mfence) {
	200	+ r = dma_fence_wait(mfence, false);
	201	+ dma_fence_put(mfence);
	202	+ pr_debug("sdma copy memory fence done\n");
	203	+ }
	204	+
	205	+ return r;
	206	+}
	207	+
	208	+static uint64_t
	209	+svm_migrate_node_physical_addr(struct amdgpu_device *adev,
	210	+ struct drm_mm_node *mm_node, uint64_t offset)
	211	+{
	212	+ struct drm_mm_node node = mm_node;
	213	+ uint64_t pos = *offset;
	214	+
	215	+ if (node->start == AMDGPU_BO_INVALID_OFFSET) {
	216	+ pr_debug("drm node is not validated\n");
	217	+ return 0;
	218	+ }
	219	+
	220	+ pr_debug("vram node start 0x%llx npages 0x%llx\n", node->start,
	221	+ node->size);
	222	+
	223	+ if (pos >= node->size) {
	224	+ do {
	225	+ pos -= node->size;
	226	+ node++;
	227	+ } while (pos >= node->size);
	228	+
	229	+ *mm_node = node;
	230	+ *offset = pos;
	231	+ }
	232	+
	233	+ return (node->start + pos) << PAGE_SHIFT;
	234	+}
	235	+
	236	+unsigned long
	237	+svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr)
	238	+{
	239	+ return (addr + adev->kfd.dev->pgmap.range.start) >> PAGE_SHIFT;
	240	+}
	241	+
	242	+static void
	243	+svm_migrate_get_vram_page(struct svm_range *prange, unsigned long pfn)
	244	+{
	245	+ struct page *page;
	246	+
	247	+ page = pfn_to_page(pfn);
	248	+ page->zone_device_data = prange;
	249	+ get_page(page);
	250	+ lock_page(page);
	251	+}
	252	+
	253	+static void
	254	+svm_migrate_put_vram_page(struct amdgpu_device *adev, unsigned long addr)
	255	+{
	256	+ struct page *page;
	257	+
	258	+ page = pfn_to_page(svm_migrate_addr_to_pfn(adev, addr));
	259	+ unlock_page(page);
	260	+ put_page(page);
	261	+}
	262	+
	263	+static unsigned long
	264	+svm_migrate_addr(struct amdgpu_device adev, struct page page)
	265	+{
	266	+ unsigned long addr;
	267	+
	268	+ addr = page_to_pfn(page) << PAGE_SHIFT;
	269	+ return (addr - adev->kfd.dev->pgmap.range.start);
	270	+}
	271	+
	272	+static struct page *
	273	+svm_migrate_get_sys_page(struct vm_area_struct *vma, unsigned long addr)
	274	+{
	275	+ struct page *page;
	276	+
	277	+ page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
	278	+ if (page)
	279	+ lock_page(page);
	280	+
	281	+ return page;
	282	+}
	283	+
	284	+static void svm_migrate_put_sys_page(unsigned long addr)
	285	+{
	286	+ struct page *page;
	287	+
	288	+ page = pfn_to_page(addr >> PAGE_SHIFT);
	289	+ unlock_page(page);
	290	+ put_page(page);
	291	+}
	292	+
	293	+static int
	294	+svm_migrate_copy_to_vram(struct amdgpu_device adev, struct svm_range prange,
	295	+ struct migrate_vma migrate, struct dma_fence *mfence,
	296	+ dma_addr_t *scratch)
	297	+{
	298	+ uint64_t npages = migrate->cpages;
	299	+ struct device *dev = adev->dev;
	300	+ struct drm_mm_node *node;
	301	+ dma_addr_t *src;
	302	+ uint64_t *dst;
	303	+ uint64_t vram_addr;
	304	+ uint64_t offset;
	305	+ uint64_t i, j;
	306	+ int r;
	307	+
	308	+ pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
	309	+ prange->last);
	310	+
	311	+ src = scratch;
	312	+ dst = (uint64_t *)(scratch + npages);
	313	+
	314	+ r = svm_range_vram_node_new(adev, prange, true);
	315	+ if (r) {
	316	+ pr_debug("failed %d get 0x%llx pages from vram\n", r, npages);
	317	+ goto out;
	318	+ }
	319	+
	320	+ node = prange->ttm_res->mm_node;
	321	+ offset = prange->offset;
	322	+ vram_addr = svm_migrate_node_physical_addr(adev, &node, &offset);
	323	+ if (!vram_addr) {
	324	+ WARN_ONCE(1, "vram node address is 0\n");
	325	+ r = -ENOMEM;
	326	+ goto out;
	327	+ }
	328	+
	329	+ for (i = j = 0; i < npages; i++) {
	330	+ struct page *spage;
	331	+
	332	+ dst[i] = vram_addr + (j << PAGE_SHIFT);
	333	+ migrate->dst[i] = svm_migrate_addr_to_pfn(adev, dst[i]);
	334	+ svm_migrate_get_vram_page(prange, migrate->dst[i]);
	335	+
	336	+ migrate->dst[i] = migrate_pfn(migrate->dst[i]);
	337	+ migrate->dst[i] \|= MIGRATE_PFN_LOCKED;
	338	+
	339	+ if (migrate->src[i] & MIGRATE_PFN_VALID) {
	340	+ spage = migrate_pfn_to_page(migrate->src[i]);
	341	+ src[i] = dma_map_page(dev, spage, 0, PAGE_SIZE,
	342	+ DMA_TO_DEVICE);
	343	+ r = dma_mapping_error(dev, src[i]);
	344	+ if (r) {
	345	+ pr_debug("failed %d dma_map_page\n", r);
	346	+ goto out_free_vram_pages;
	347	+ }
	348	+ } else {
	349	+ if (j) {
	350	+ r = svm_migrate_copy_memory_gart(
	351	+ adev, src + i - j,
	352	+ dst + i - j, j,
	353	+ FROM_RAM_TO_VRAM,
	354	+ mfence);
	355	+ if (r)
	356	+ goto out_free_vram_pages;
	357	+ offset += j;
	358	+ vram_addr = (node->start + offset) << PAGE_SHIFT;
	359	+ j = 0;
	360	+ } else {
	361	+ offset++;
	362	+ vram_addr += PAGE_SIZE;
	363	+ }
	364	+ if (offset >= node->size) {
	365	+ node++;
	366	+ pr_debug("next node size 0x%llx\n", node->size);
	367	+ vram_addr = node->start << PAGE_SHIFT;
	368	+ offset = 0;
	369	+ }
	370	+ continue;
	371	+ }
	372	+
	373	+ pr_debug("dma mapping src to 0x%llx, page_to_pfn 0x%lx\n",
	374	+ src[i] >> PAGE_SHIFT, page_to_pfn(spage));
	375	+
	376	+ if (j + offset >= node->size - 1 && i < npages - 1) {
	377	+ r = svm_migrate_copy_memory_gart(adev, src + i - j,
	378	+ dst + i - j, j + 1,
	379	+ FROM_RAM_TO_VRAM,
	380	+ mfence);
	381	+ if (r)
	382	+ goto out_free_vram_pages;
	383	+
	384	+ node++;
	385	+ pr_debug("next node size 0x%llx\n", node->size);
	386	+ vram_addr = node->start << PAGE_SHIFT;
	387	+ offset = 0;
	388	+ j = 0;
	389	+ } else {
	390	+ j++;
	391	+ }
	392	+ }
	393	+
	394	+ r = svm_migrate_copy_memory_gart(adev, src + i - j, dst + i - j, j,
	395	+ FROM_RAM_TO_VRAM, mfence);
	396	+
	397	+out_free_vram_pages:
	398	+ if (r) {
	399	+ pr_debug("failed %d to copy memory to vram\n", r);
	400	+ while (i--) {
	401	+ svm_migrate_put_vram_page(adev, dst[i]);
	402	+ migrate->dst[i] = 0;
	403	+ }
	404	+ }
	405	+
	406	+out:
	407	+ return r;
	408	+}
	409	+
	410	+static int
	411	+svm_migrate_vma_to_vram(struct amdgpu_device adev, struct svm_range prange,
	412	+ struct vm_area_struct *vma, uint64_t start,
	413	+ uint64_t end)
	414	+{
	415	+ uint64_t npages = (end - start) >> PAGE_SHIFT;
	416	+ struct dma_fence *mfence = NULL;
	417	+ struct migrate_vma migrate;
	418	+ dma_addr_t *scratch;
	419	+ size_t size;
	420	+ void *buf;
	421	+ int r = -ENOMEM;
	422	+ int retry = 0;
	423	+
	424	+ memset(&migrate, 0, sizeof(migrate));
	425	+ migrate.vma = vma;
	426	+ migrate.start = start;
	427	+ migrate.end = end;
	428	+ migrate.flags = MIGRATE_VMA_SELECT_SYSTEM;
	429	+ migrate.pgmap_owner = adev;
	430	+
	431	+ size = 2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t);
	432	+ size *= npages;
	433	+ buf = kvmalloc(size, GFP_KERNEL \| __GFP_ZERO);
	434	+ if (!buf)
	435	+ goto out;
	436	+
	437	+ migrate.src = buf;
	438	+ migrate.dst = migrate.src + npages;
	439	+ scratch = (dma_addr_t *)(migrate.dst + npages);
	440	+
	441	+retry:
	442	+ r = migrate_vma_setup(&migrate);
	443	+ if (r) {
	444	+ pr_debug("failed %d prepare migrate svms 0x%p [0x%lx 0x%lx]\n",
	445	+ r, prange->svms, prange->start, prange->last);
	446	+ goto out_free;
	447	+ }
	448	+ if (migrate.cpages != npages) {
	449	+ pr_debug("collect 0x%lx/0x%llx pages, retry\n", migrate.cpages,
	450	+ npages);
	451	+ migrate_vma_finalize(&migrate);
	452	+ if (retry++ >= 3) {
	453	+ r = -ENOMEM;
	454	+ pr_debug("failed %d migrate svms 0x%p [0x%lx 0x%lx]\n",
	455	+ r, prange->svms, prange->start, prange->last);
	456	+ goto out_free;
	457	+ }
	458	+
	459	+ goto retry;
	460	+ }
	461	+
	462	+ if (migrate.cpages) {
	463	+ r = svm_migrate_copy_to_vram(adev, prange, &migrate, &mfence,
	464	+ scratch);
	465	+ migrate_vma_pages(&migrate);
	466	+ svm_migrate_copy_done(adev, mfence);
	467	+ migrate_vma_finalize(&migrate);
	468	+ }
	469	+
	470	+ svm_range_dma_unmap(adev->dev, scratch, 0, npages);
	471	+ svm_range_free_dma_mappings(prange);
	472	+
	473	+out_free:
	474	+ kvfree(buf);
	475	+out:
	476	+ return r;
	477	+}
	478	+
	479	+/**
	480	+ * svm_migrate_ram_to_vram - migrate svm range from system to device
	481	+ * @prange: range structure
	482	+ * @best_loc: the device to migrate to
	483	+ * @mm: the process mm structure
	484	+ *
	485	+ * Context: Process context, caller hold mmap read lock, svms lock, prange lock
	486	+ *
	487	+ * Return:
	488	+ * 0 - OK, otherwise error code
	489	+ */
	490	+static int
	491	+svm_migrate_ram_to_vram(struct svm_range *prange, uint32_t best_loc,
	492	+ struct mm_struct *mm)
	493	+{
	494	+ unsigned long addr, start, end;
	495	+ struct vm_area_struct *vma;
	496	+ struct amdgpu_device *adev;
	497	+ int r = 0;
	498	+
	499	+ if (prange->actual_loc == best_loc) {
	500	+ pr_debug("svms 0x%p [0x%lx 0x%lx] already on best_loc 0x%x\n",
	501	+ prange->svms, prange->start, prange->last, best_loc);
	502	+ return 0;
	503	+ }
	504	+
	505	+ adev = svm_range_get_adev_by_id(prange, best_loc);
	506	+ if (!adev) {
	507	+ pr_debug("failed to get device by id 0x%x\n", best_loc);
	508	+ return -ENODEV;
	509	+ }
	510	+
	511	+ pr_debug("svms 0x%p [0x%lx 0x%lx] to gpu 0x%x\n", prange->svms,
	512	+ prange->start, prange->last, best_loc);
	513	+
	514	+ /* FIXME: workaround for page locking bug with invalid pages */
	515	+ svm_range_prefault(prange, mm);
	516	+
	517	+ start = prange->start << PAGE_SHIFT;
	518	+ end = (prange->last + 1) << PAGE_SHIFT;
	519	+
	520	+ for (addr = start; addr < end;) {
	521	+ unsigned long next;
	522	+
	523	+ vma = find_vma(mm, addr);
	524	+ if (!vma \|\| addr < vma->vm_start)
	525	+ break;
	526	+
	527	+ next = min(vma->vm_end, end);
	528	+ r = svm_migrate_vma_to_vram(adev, prange, vma, addr, next);
	529	+ if (r) {
	530	+ pr_debug("failed to migrate\n");
	531	+ break;
	532	+ }
	533	+ addr = next;
	534	+ }
	535	+
	536	+ if (!r)
	537	+ prange->actual_loc = best_loc;
	538	+
	539	+ return r;
	540	+}
	541	+
	542	+static void svm_migrate_page_free(struct page *page)
	543	+{
	544	+ /* Keep this function to avoid warning */
	545	+}
	546	+
	547	+static int
	548	+svm_migrate_copy_to_ram(struct amdgpu_device adev, struct svm_range prange,
	549	+ struct migrate_vma migrate, struct dma_fence *mfence,
	550	+ dma_addr_t *scratch)
	551	+{
	552	+ uint64_t npages = migrate->cpages;
	553	+ struct device *dev = adev->dev;
	554	+ uint64_t *src;
	555	+ dma_addr_t *dst;
	556	+ struct page *dpage;
	557	+ uint64_t i = 0, j;
	558	+ uint64_t addr;
	559	+ int r = 0;
	560	+
	561	+ pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
	562	+ prange->last);
	563	+
	564	+ addr = prange->start << PAGE_SHIFT;
	565	+
	566	+ src = (uint64_t *)(scratch + npages);
	567	+ dst = scratch;
	568	+
	569	+ for (i = 0, j = 0; i < npages; i++, j++, addr += PAGE_SIZE) {
	570	+ struct page *spage;
	571	+
	572	+ spage = migrate_pfn_to_page(migrate->src[i]);
	573	+ if (!spage) {
	574	+ pr_debug("failed get spage svms 0x%p [0x%lx 0x%lx]\n",
	575	+ prange->svms, prange->start, prange->last);
	576	+ r = -ENOMEM;
	577	+ goto out_oom;
	578	+ }
	579	+ src[i] = svm_migrate_addr(adev, spage);
	580	+ if (i > 0 && src[i] != src[i - 1] + PAGE_SIZE) {
	581	+ r = svm_migrate_copy_memory_gart(adev, dst + i - j,
	582	+ src + i - j, j,
	583	+ FROM_VRAM_TO_RAM,
	584	+ mfence);
	585	+ if (r)
	586	+ goto out_oom;
	587	+ j = 0;
	588	+ }
	589	+
	590	+ dpage = svm_migrate_get_sys_page(migrate->vma, addr);
	591	+ if (!dpage) {
	592	+ pr_debug("failed get page svms 0x%p [0x%lx 0x%lx]\n",
	593	+ prange->svms, prange->start, prange->last);
	594	+ r = -ENOMEM;
	595	+ goto out_oom;
	596	+ }
	597	+
	598	+ dst[i] = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_FROM_DEVICE);
	599	+ r = dma_mapping_error(dev, dst[i]);
	600	+ if (r) {
	601	+ pr_debug("failed %d dma_map_page\n", r);
	602	+ goto out_oom;
	603	+ }
	604	+
	605	+ pr_debug("dma mapping dst to 0x%llx, page_to_pfn 0x%lx\n",
	606	+ dst[i] >> PAGE_SHIFT, page_to_pfn(dpage));
	607	+
	608	+ migrate->dst[i] = migrate_pfn(page_to_pfn(dpage));
	609	+ migrate->dst[i] \|= MIGRATE_PFN_LOCKED;
	610	+ }
	611	+
	612	+ r = svm_migrate_copy_memory_gart(adev, dst + i - j, src + i - j, j,
	613	+ FROM_VRAM_TO_RAM, mfence);
	614	+
	615	+out_oom:
	616	+ if (r) {
	617	+ pr_debug("failed %d copy to ram\n", r);
	618	+ while (i--) {
	619	+ svm_migrate_put_sys_page(dst[i]);
	620	+ migrate->dst[i] = 0;
	621	+ }
	622	+ }
	623	+
	624	+ return r;
	625	+}
	626	+
	627	+static int
	628	+svm_migrate_vma_to_ram(struct amdgpu_device adev, struct svm_range prange,
	629	+ struct vm_area_struct *vma, uint64_t start, uint64_t end)
	630	+{
	631	+ uint64_t npages = (end - start) >> PAGE_SHIFT;
	632	+ struct dma_fence *mfence = NULL;
	633	+ struct migrate_vma migrate;
	634	+ dma_addr_t *scratch;
	635	+ size_t size;
	636	+ void *buf;
	637	+ int r = -ENOMEM;
	638	+
	639	+ memset(&migrate, 0, sizeof(migrate));
	640	+ migrate.vma = vma;
	641	+ migrate.start = start;
	642	+ migrate.end = end;
	643	+ migrate.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
	644	+ migrate.pgmap_owner = adev;
	645	+
	646	+ size = 2 * sizeof(*migrate.src) + sizeof(uint64_t) + sizeof(dma_addr_t);
	647	+ size *= npages;
	648	+ buf = kvmalloc(size, GFP_KERNEL \| __GFP_ZERO);
	649	+ if (!buf)
	650	+ goto out;
	651	+
	652	+ migrate.src = buf;
	653	+ migrate.dst = migrate.src + npages;
	654	+ scratch = (dma_addr_t *)(migrate.dst + npages);
	655	+
	656	+ r = migrate_vma_setup(&migrate);
	657	+ if (r) {
	658	+ pr_debug("failed %d prepare migrate svms 0x%p [0x%lx 0x%lx]\n",
	659	+ r, prange->svms, prange->start, prange->last);
	660	+ goto out_free;
	661	+ }
	662	+
	663	+ pr_debug("cpages %ld\n", migrate.cpages);
	664	+
	665	+ if (migrate.cpages) {
	666	+ r = svm_migrate_copy_to_ram(adev, prange, &migrate, &mfence,
	667	+ scratch);
	668	+ migrate_vma_pages(&migrate);
	669	+ svm_migrate_copy_done(adev, mfence);
	670	+ migrate_vma_finalize(&migrate);
	671	+ } else {
	672	+ pr_debug("failed collect migrate device pages [0x%lx 0x%lx]\n",
	673	+ prange->start, prange->last);
	674	+ }
	675	+
	676	+ svm_range_dma_unmap(adev->dev, scratch, 0, npages);
	677	+
	678	+out_free:
	679	+ kvfree(buf);
	680	+out:
	681	+ return r;
	682	+}
	683	+
	684	+/**
	685	+ * svm_migrate_vram_to_ram - migrate svm range from device to system
	686	+ * @prange: range structure
	687	+ * @mm: process mm, use current->mm if NULL
	688	+ *
	689	+ * Context: Process context, caller hold mmap read lock, svms lock, prange lock
	690	+ *
	691	+ * Return:
	692	+ * 0 - OK, otherwise error code
	693	+ */
	694	+int svm_migrate_vram_to_ram(struct svm_range prange, struct mm_struct mm)
	695	+{
	696	+ struct amdgpu_device *adev;
	697	+ struct vm_area_struct *vma;
	698	+ unsigned long addr;
	699	+ unsigned long start;
	700	+ unsigned long end;
	701	+ int r = 0;
	702	+
	703	+ if (!prange->actual_loc) {
	704	+ pr_debug("[0x%lx 0x%lx] already migrated to ram\n",
	705	+ prange->start, prange->last);
	706	+ return 0;
	707	+ }
	708	+
	709	+ adev = svm_range_get_adev_by_id(prange, prange->actual_loc);
	710	+ if (!adev) {
	711	+ pr_debug("failed to get device by id 0x%x\n",
	712	+ prange->actual_loc);
	713	+ return -ENODEV;
	714	+ }
	715	+
	716	+ pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] from gpu 0x%x to ram\n",
	717	+ prange->svms, prange, prange->start, prange->last,
	718	+ prange->actual_loc);
	719	+
	720	+ start = prange->start << PAGE_SHIFT;
	721	+ end = (prange->last + 1) << PAGE_SHIFT;
	722	+
	723	+ for (addr = start; addr < end;) {
	724	+ unsigned long next;
	725	+
	726	+ vma = find_vma(mm, addr);
	727	+ if (!vma \|\| addr < vma->vm_start)
	728	+ break;
	729	+
	730	+ next = min(vma->vm_end, end);
	731	+ r = svm_migrate_vma_to_ram(adev, prange, vma, addr, next);
	732	+ if (r) {
	733	+ pr_debug("failed %d to migrate\n", r);
	734	+ break;
	735	+ }
	736	+ addr = next;
	737	+ }
	738	+
	739	+ if (!r) {
	740	+ svm_range_vram_node_free(prange);
	741	+ prange->actual_loc = 0;
	742	+ }
	743	+ return r;
	744	+}
	745	+
	746	+/**
	747	+ * svm_migrate_vram_to_vram - migrate svm range from device to device
	748	+ * @prange: range structure
	749	+ * @best_loc: the device to migrate to
	750	+ * @mm: process mm, use current->mm if NULL
	751	+ *
	752	+ * Context: Process context, caller hold mmap read lock, svms lock, prange lock
	753	+ *
	754	+ * Return:
	755	+ * 0 - OK, otherwise error code
	756	+ */
	757	+static int
	758	+svm_migrate_vram_to_vram(struct svm_range *prange, uint32_t best_loc,
	759	+ struct mm_struct *mm)
	760	+{
	761	+ int r;
	762	+
	763	+ /*
	764	+ * TODO: for both devices with PCIe large bar or on same xgmi hive, skip
	765	+ * system memory as migration bridge
	766	+ */
	767	+
	768	+ pr_debug("from gpu 0x%x to gpu 0x%x\n", prange->actual_loc, best_loc);
	769	+
	770	+ r = svm_migrate_vram_to_ram(prange, mm);
	771	+ if (r)
	772	+ return r;
	773	+
	774	+ return svm_migrate_ram_to_vram(prange, best_loc, mm);
	775	+}
	776	+
	777	+int
	778	+svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,
	779	+ struct mm_struct *mm)
	780	+{
	781	+ if (!prange->actual_loc)
	782	+ return svm_migrate_ram_to_vram(prange, best_loc, mm);
	783	+ else
	784	+ return svm_migrate_vram_to_vram(prange, best_loc, mm);
	785	+
	786	+}
	787	+
	788	+/**
	789	+ * svm_migrate_to_ram - CPU page fault handler
	790	+ * @vmf: CPU vm fault vma, address
	791	+ *
	792	+ * Context: vm fault handler, caller holds the mmap read lock
	793	+ *
	794	+ * Return:
	795	+ * 0 - OK
	796	+ * VM_FAULT_SIGBUS - notice application to have SIGBUS page fault
	797	+ */
	798	+static vm_fault_t svm_migrate_to_ram(struct vm_fault *vmf)
	799	+{
	800	+ unsigned long addr = vmf->address;
	801	+ struct vm_area_struct *vma;
	802	+ enum svm_work_list_ops op;
	803	+ struct svm_range *parent;
	804	+ struct svm_range *prange;
	805	+ struct kfd_process *p;
	806	+ struct mm_struct *mm;
	807	+ int r = 0;
	808	+
	809	+ vma = vmf->vma;
	810	+ mm = vma->vm_mm;
	811	+
	812	+ p = kfd_lookup_process_by_mm(vma->vm_mm);
	813	+ if (!p) {
	814	+ pr_debug("failed find process at fault address 0x%lx\n", addr);
	815	+ return VM_FAULT_SIGBUS;
	816	+ }
	817	+ addr >>= PAGE_SHIFT;
	818	+ pr_debug("CPU page fault svms 0x%p address 0x%lx\n", &p->svms, addr);
	819	+
	820	+ mutex_lock(&p->svms.lock);
	821	+
	822	+ prange = svm_range_from_addr(&p->svms, addr, &parent);
	823	+ if (!prange) {
	824	+ pr_debug("cannot find svm range at 0x%lx\n", addr);
	825	+ r = -EFAULT;
	826	+ goto out;
	827	+ }
	828	+
	829	+ mutex_lock(&parent->migrate_mutex);
	830	+ if (prange != parent)
	831	+ mutex_lock_nested(&prange->migrate_mutex, 1);
	832	+
	833	+ if (!prange->actual_loc)
	834	+ goto out_unlock_prange;
	835	+
	836	+ svm_range_lock(parent);
	837	+ if (prange != parent)
	838	+ mutex_lock_nested(&prange->lock, 1);
	839	+ r = svm_range_split_by_granularity(p, mm, addr, parent, prange);
	840	+ if (prange != parent)
	841	+ mutex_unlock(&prange->lock);
	842	+ svm_range_unlock(parent);
	843	+ if (r) {
	844	+ pr_debug("failed %d to split range by granularity\n", r);
	845	+ goto out_unlock_prange;
	846	+ }
	847	+
	848	+ r = svm_migrate_vram_to_ram(prange, mm);
	849	+ if (r)
	850	+ pr_debug("failed %d migrate 0x%p [0x%lx 0x%lx] to ram\n", r,
	851	+ prange, prange->start, prange->last);
	852	+
	853	+ /* xnack on, update mapping on GPUs with ACCESS_IN_PLACE */
	854	+ if (p->xnack_enabled && parent == prange)
	855	+ op = SVM_OP_UPDATE_RANGE_NOTIFIER_AND_MAP;
	856	+ else
	857	+ op = SVM_OP_UPDATE_RANGE_NOTIFIER;
	858	+ svm_range_add_list_work(&p->svms, parent, mm, op);
	859	+ schedule_deferred_list_work(&p->svms);
	860	+
	861	+out_unlock_prange:
	862	+ if (prange != parent)
	863	+ mutex_unlock(&prange->migrate_mutex);
	864	+ mutex_unlock(&parent->migrate_mutex);
	865	+out:
	866	+ mutex_unlock(&p->svms.lock);
	867	+ kfd_unref_process(p);
	868	+
	869	+ pr_debug("CPU fault svms 0x%p address 0x%lx done\n", &p->svms, addr);
	870	+
	871	+ return r ? VM_FAULT_SIGBUS : 0;
	872	+}
	873	+
	874	+static const struct dev_pagemap_ops svm_migrate_pgmap_ops = {
	875	+ .page_free = svm_migrate_page_free,
	876	+ .migrate_to_ram = svm_migrate_to_ram,
	877	+};
	878	+
	879	+/* Each VRAM page uses sizeof(struct page) on system memory */
	880	+#define SVM_HMM_PAGE_STRUCT_SIZE(size) ((size)/PAGE_SIZE * sizeof(struct page))
	881	+
	882	+int svm_migrate_init(struct amdgpu_device *adev)
	883	+{
	884	+ struct kfd_dev *kfddev = adev->kfd.dev;
	885	+ struct dev_pagemap *pgmap;
	886	+ struct resource *res;
	887	+ unsigned long size;
	888	+ void *r;
	889	+
	890	+ /* Page migration works on Vega10 or newer */
	891	+ if (kfddev->device_info->asic_family < CHIP_VEGA10)
	892	+ return -EINVAL;
	893	+
	894	+ pgmap = &kfddev->pgmap;
	895	+ memset(pgmap, 0, sizeof(*pgmap));
	896	+
	897	+ /* TODO: register all vram to HMM for now.
	898	+ * should remove reserved size
	899	+ */
	900	+ size = ALIGN(adev->gmc.real_vram_size, 2ULL << 20);
	901	+ res = devm_request_free_mem_region(adev->dev, &iomem_resource, size);
	902	+ if (IS_ERR(res))
	903	+ return -ENOMEM;
	904	+
	905	+ pgmap->type = MEMORY_DEVICE_PRIVATE;
	906	+ pgmap->nr_range = 1;
	907	+ pgmap->range.start = res->start;
	908	+ pgmap->range.end = res->end;
	909	+ pgmap->ops = &svm_migrate_pgmap_ops;
	910	+ pgmap->owner = adev;
	911	+ pgmap->flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
	912	+ r = devm_memremap_pages(adev->dev, pgmap);
	913	+ if (IS_ERR(r)) {
	914	+ pr_err("failed to register HMM device memory\n");
	915	+ devm_release_mem_region(adev->dev, res->start,
	916	+ res->end - res->start + 1);
	917	+ return PTR_ERR(r);
	918	+ }
	919	+
	920	+ pr_debug("reserve %ldMB system memory for VRAM pages struct\n",
	921	+ SVM_HMM_PAGE_STRUCT_SIZE(size) >> 20);
	922	+
	923	+ amdgpu_amdkfd_reserve_system_mem(SVM_HMM_PAGE_STRUCT_SIZE(size));
	924	+
	925	+ pr_info("HMM registered %ldMB device memory\n", size >> 20);
	926	+
	927	+ return 0;
	928	+}
	929	+
	930	+void svm_migrate_fini(struct amdgpu_device *adev)
	931	+{
	932	+ struct dev_pagemap *pgmap = &adev->kfd.dev->pgmap;
	933	+
	934	+ devm_memunmap_pages(adev->dev, pgmap);
	935	+ devm_release_mem_region(adev->dev, pgmap->range.start,
	936	+ pgmap->range.end - pgmap->range.start + 1);
	937	+}

drivers/gpu/drm/amd/amdkfd/kfd_migrate.h+65−0

@@ -0,0 +1,65 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
	2	+/*
	3	+ * Copyright 2020-2021 Advanced Micro Devices, Inc.
	4	+ *
	5	+ * Permission is hereby granted, free of charge, to any person obtaining a
	6	+ * copy of this software and associated documentation files (the "Software"),
	7	+ * to deal in the Software without restriction, including without limitation
	8	+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
	9	+ * and/or sell copies of the Software, and to permit persons to whom the
	10	+ * Software is furnished to do so, subject to the following conditions:
	11	+ *
	12	+ * The above copyright notice and this permission notice shall be included in
	13	+ * all copies or substantial portions of the Software.
	14	+ *
	15	+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	19	+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	20	+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	21	+ * OTHER DEALINGS IN THE SOFTWARE.
	22	+ *
	23	+ */
	24	+
	25	+#ifndef KFD_MIGRATE_H_
	26	+#define KFD_MIGRATE_H_
	27	+
	28	+#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
	29	+
	30	+#include <linux/rwsem.h>
	31	+#include <linux/list.h>
	32	+#include <linux/mutex.h>
	33	+#include <linux/sched/mm.h>
	34	+#include <linux/hmm.h>
	35	+#include "kfd_priv.h"
	36	+#include "kfd_svm.h"
	37	+
	38	+enum MIGRATION_COPY_DIR {
	39	+ FROM_RAM_TO_VRAM = 0,
	40	+ FROM_VRAM_TO_RAM
	41	+};
	42	+
	43	+int svm_migrate_to_vram(struct svm_range *prange, uint32_t best_loc,
	44	+ struct mm_struct *mm);
	45	+int svm_migrate_vram_to_ram(struct svm_range prange, struct mm_struct mm);
	46	+unsigned long
	47	+svm_migrate_addr_to_pfn(struct amdgpu_device *adev, unsigned long addr);
	48	+
	49	+int svm_migrate_init(struct amdgpu_device *adev);
	50	+void svm_migrate_fini(struct amdgpu_device *adev);
	51	+
	52	+#else
	53	+
	54	+static inline int svm_migrate_init(struct amdgpu_device *adev)
	55	+{
	56	+ return 0;
	57	+}
	58	+static inline void svm_migrate_fini(struct amdgpu_device *adev)
	59	+{
	60	+ /* empty */
	61	+}
	62	+
	63	+#endif /* IS_ENABLED(CONFIG_HSA_AMD_SVM) */
	64	+
	65	+#endif /* KFD_MIGRATE_H_ */

drivers/gpu/drm/amd/amdkfd/kfd_packet_manager.c+7−5

@@ -124,14 +124,14 @@ static int pm_create_runlist_ib(struct packet_manager *pm, · 123 unmodified lines
124	124	{
125	125	unsigned int alloc_size_bytes;
126	126	unsigned int *rl_buffer, rl_wptr, i;
127		− int retval, proccesses_mapped;
	127	+ int retval, processes_mapped;
128	128	struct device_process_node *cur;
129	129	struct qcm_process_device *qpd;
130	130	struct queue *q;
131	131	struct kernel_queue *kq;
132	132	bool is_over_subscription;
133	133
134		− rl_wptr = retval = proccesses_mapped = 0;
	134	+ rl_wptr = retval = processes_mapped = 0;
135	135
136	136	retval = pm_allocate_runlist_ib(pm, &rl_buffer, rl_gpu_addr,
137	137	&alloc_size_bytes, &is_over_subscription);
@@ -148,7 +148,7 @@ static int pm_create_runlist_ib(struct packet_manager *pm, · 10 unmodified lines
148	148	list_for_each_entry(cur, queues, list) {
149	149	qpd = cur->qpd;
150	150	/* build map process packet */
151		− if (proccesses_mapped >= pm->dqm->processes_count) {
	151	+ if (processes_mapped >= pm->dqm->processes_count) {
152	152	pr_debug("Not enough space left in runlist IB\n");
153	153	pm_release_ib(pm);
154	154	return -ENOMEM;
@@ -158,7 +158,7 @@ static int pm_create_runlist_ib(struct packet_manager *pm, · 3 unmodified lines
158	158	if (retval)
159	159	return retval;
160	160
161		− proccesses_mapped++;
	161	+ processes_mapped++;
162	162	inc_wptr(&rl_wptr, pm->pmf->map_process_size,
163	163	alloc_size_bytes);
164	164
@@ -242,7 +242,6 @@ int pm_init(struct packet_manager pm, struct device_queue_manager dqm) · 77 unmodified lines
242	242	case CHIP_RAVEN:
243	243	case CHIP_RENOIR:
244	244	case CHIP_ARCTURUS:
245		− case CHIP_ALDEBARAN:
246	245	case CHIP_NAVI10:
247	246	case CHIP_NAVI12:
248	247	case CHIP_NAVI14:
@@ -252,6 +251,9 @@ int pm_init(struct packet_manager pm, struct device_queue_manager dqm) · 3 unmodified lines
252	251	case CHIP_DIMGREY_CAVEFISH:
253	252	pm->pmf = &kfd_v9_pm_funcs;
254	253	break;
	254	+ case CHIP_ALDEBARAN:
	255	+ pm->pmf = &kfd_aldebaran_pm_funcs;
	256	+ break;
255	257	default:
256	258	WARN(1, "Unexpected ASIC family %u",
257	259	dqm->dev->device_info->asic_family);

drivers/gpu/drm/amd/amdkfd/kfd_packet_manager_v9.c+57−1

@@ -24,6 +24,7 @@ · 23 unmodified lines
24	24	#include "kfd_kernel_queue.h"
25	25	#include "kfd_device_queue_manager.h"
26	26	#include "kfd_pm4_headers_ai.h"
	27	+#include "kfd_pm4_headers_aldebaran.h"
27	28	#include "kfd_pm4_opcodes.h"
28	29	#include "gc/gc_10_1_0_sh_mask.h"
29	30
@@ -35,7 +36,6 @@ static int pm_map_process_v9(struct packet_manager *pm, · 5 unmodified lines
35	36
36	37	packet = (struct pm4_mes_map_process *)buffer;
37	38	memset(buffer, 0, sizeof(struct pm4_mes_map_process));
38		−
39	39	packet->header.u32All = pm_build_pm4_header(IT_MAP_PROCESS,
40	40	sizeof(struct pm4_mes_map_process));
41	41	packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
@@ -73,6 +73,45 @@ static int pm_map_process_v9(struct packet_manager *pm, · 31 unmodified lines
73	73	return 0;
74	74	}
75	75
	76	+static int pm_map_process_aldebaran(struct packet_manager *pm,
	77	+ uint32_t buffer, struct qcm_process_device qpd)
	78	+{
	79	+ struct pm4_mes_map_process_aldebaran *packet;
	80	+ uint64_t vm_page_table_base_addr = qpd->page_table_base;
	81	+
	82	+ packet = (struct pm4_mes_map_process_aldebaran *)buffer;
	83	+ memset(buffer, 0, sizeof(struct pm4_mes_map_process_aldebaran));
	84	+ packet->header.u32All = pm_build_pm4_header(IT_MAP_PROCESS,
	85	+ sizeof(struct pm4_mes_map_process_aldebaran));
	86	+ packet->bitfields2.diq_enable = (qpd->is_debug) ? 1 : 0;
	87	+ packet->bitfields2.process_quantum = 10;
	88	+ packet->bitfields2.pasid = qpd->pqm->process->pasid;
	89	+ packet->bitfields14.gds_size = qpd->gds_size & 0x3F;
	90	+ packet->bitfields14.gds_size_hi = (qpd->gds_size >> 6) & 0xF;
	91	+ packet->bitfields14.num_gws = (qpd->mapped_gws_queue) ? qpd->num_gws : 0;
	92	+ packet->bitfields14.num_oac = qpd->num_oac;
	93	+ packet->bitfields14.sdma_enable = 1;
	94	+ packet->bitfields14.num_queues = (qpd->is_debug) ? 0 : qpd->queue_count;
	95	+
	96	+ packet->sh_mem_config = qpd->sh_mem_config;
	97	+ packet->sh_mem_bases = qpd->sh_mem_bases;
	98	+ if (qpd->tba_addr) {
	99	+ packet->sq_shader_tba_lo = lower_32_bits(qpd->tba_addr >> 8);
	100	+ packet->sq_shader_tma_lo = lower_32_bits(qpd->tma_addr >> 8);
	101	+ packet->sq_shader_tma_hi = upper_32_bits(qpd->tma_addr >> 8);
	102	+ }
	103	+
	104	+ packet->gds_addr_lo = lower_32_bits(qpd->gds_context_area);
	105	+ packet->gds_addr_hi = upper_32_bits(qpd->gds_context_area);
	106	+
	107	+ packet->vm_context_page_table_base_addr_lo32 =
	108	+ lower_32_bits(vm_page_table_base_addr);
	109	+ packet->vm_context_page_table_base_addr_hi32 =
	110	+ upper_32_bits(vm_page_table_base_addr);
	111	+
	112	+ return 0;
	113	+}
	114	+
76	115	static int pm_runlist_v9(struct packet_manager pm, uint32_t buffer,
77	116	uint64_t ib, size_t ib_size_in_dwords, bool chain)
78	117	{
@@ -324,3 +363,20 @@ const struct packet_manager_funcs kfd_v9_pm_funcs = { · 245 unmodified lines
324	363	.query_status_size = sizeof(struct pm4_mes_query_status),
325	364	.release_mem_size = 0,
326	365	};
	366	+
	367	+const struct packet_manager_funcs kfd_aldebaran_pm_funcs = {
	368	+ .map_process = pm_map_process_aldebaran,
	369	+ .runlist = pm_runlist_v9,
	370	+ .set_resources = pm_set_resources_v9,
	371	+ .map_queues = pm_map_queues_v9,
	372	+ .unmap_queues = pm_unmap_queues_v9,
	373	+ .query_status = pm_query_status_v9,
	374	+ .release_mem = NULL,
	375	+ .map_process_size = sizeof(struct pm4_mes_map_process_aldebaran),
	376	+ .runlist_size = sizeof(struct pm4_mes_runlist),
	377	+ .set_resources_size = sizeof(struct pm4_mes_set_resources),
	378	+ .map_queues_size = sizeof(struct pm4_mes_map_queues),
	379	+ .unmap_queues_size = sizeof(struct pm4_mes_unmap_queues),
	380	+ .query_status_size = sizeof(struct pm4_mes_query_status),
	381	+ .release_mem_size = 0,
	382	+};

drivers/gpu/drm/amd/amdkfd/kfd_pm4_headers_aldebaran.h+93−0

@@ -0,0 +1,93 @@
	1	+/*
	2	+ * Copyright 2020 Advanced Micro Devices, Inc.
	3	+ *
	4	+ * Permission is hereby granted, free of charge, to any person obtaining a
	5	+ * copy of this software and associated documentation files (the "Software"),
	6	+ * to deal in the Software without restriction, including without limitation
	7	+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
	8	+ * and/or sell copies of the Software, and to permit persons to whom the
	9	+ * Software is furnished to do so, subject to the following conditions:
	10	+ *
	11	+ * The above copyright notice and this permission notice shall be included in
	12	+ * all copies or substantial portions of the Software.
	13	+ *
	14	+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	15	+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	16	+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	17	+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	18	+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	19	+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	20	+ * OTHER DEALINGS IN THE SOFTWARE.
	21	+ *
	22	+ */
	23	+
	24	+/--------------------MES_MAP_PROCESS (PER DEBUG VMID)--------------------/
	25	+
	26	+#ifndef PM4_MES_MAP_PROCESS_PER_DEBUG_VMID_DEFINED
	27	+#define PM4_MES_MAP_PROCESS_PER_DEBUG_VMID_DEFINED
	28	+
	29	+struct pm4_mes_map_process_aldebaran {
	30	+ union {
	31	+ union PM4_MES_TYPE_3_HEADER header; /* header */
	32	+ uint32_t ordinal1;
	33	+ };
	34	+
	35	+ union {
	36	+ struct {
	37	+ uint32_t pasid:16; /* 0 - 15 */
	38	+ uint32_t single_memops:1; /* 16 */
	39	+ uint32_t reserved1:1; /* 17 */
	40	+ uint32_t debug_vmid:4; /* 18 - 21 */
	41	+ uint32_t new_debug:1; /* 22 */
	42	+ uint32_t tmz:1; /* 23 */
	43	+ uint32_t diq_enable:1; /* 24 */
	44	+ uint32_t process_quantum:7; /* 25 - 31 */
	45	+ } bitfields2;
	46	+ uint32_t ordinal2;
	47	+ };
	48	+
	49	+ uint32_t vm_context_page_table_base_addr_lo32;
	50	+
	51	+ uint32_t vm_context_page_table_base_addr_hi32;
	52	+
	53	+ uint32_t sh_mem_bases;
	54	+
	55	+ uint32_t sh_mem_config;
	56	+
	57	+ uint32_t sq_shader_tba_lo;
	58	+
	59	+ uint32_t sq_shader_tba_hi;
	60	+
	61	+ uint32_t sq_shader_tma_lo;
	62	+
	63	+ uint32_t sq_shader_tma_hi;
	64	+
	65	+ uint32_t reserved6;
	66	+
	67	+ uint32_t gds_addr_lo;
	68	+
	69	+ uint32_t gds_addr_hi;
	70	+
	71	+ union {
	72	+ struct {
	73	+ uint32_t num_gws:7;
	74	+ uint32_t sdma_enable:1;
	75	+ uint32_t num_oac:4;
	76	+ uint32_t gds_size_hi:4;
	77	+ uint32_t gds_size:6;
	78	+ uint32_t num_queues:10;
	79	+ } bitfields14;
	80	+ uint32_t ordinal14;
	81	+ };
	82	+
	83	+ uint32_t spi_gdbg_per_vmid_cntl;
	84	+
	85	+ uint32_t tcp_watch_cntl[4];
	86	+
	87	+ uint32_t completion_signal_lo;
	88	+
	89	+ uint32_t completion_signal_hi;
	90	+
	91	+};
	92	+
	93	+#endif

drivers/gpu/drm/amd/amdkfd/kfd_priv.h+38−1

@@ -322,6 +322,9 @@ struct kfd_dev { · 321 unmodified lines
322	322	unsigned int max_doorbell_slices;
323	323
324	324	int noretry;
	325	+
	326	+ /* HMM page migration MEMORY_DEVICE_PRIVATE mapping */
	327	+ struct dev_pagemap pgmap;
325	328	};
326	329
327	330	enum kfd_mempool {
@@ -669,7 +672,7 @@ struct kfd_process_device { · 341 unmodified lines
669	672
670	673	/* VM context for GPUVM allocations */
671	674	struct file *drm_file;
672		− void *vm;
	675	+ void *drm_priv;
673	676
674	677	/* GPUVM allocations storage */
675	678	struct idr alloc_idr;
@@ -731,6 +734,17 @@ struct kfd_process_device { · 55 unmodified lines
731	734
732	735	#define qpd_to_pdd(x) container_of(x, struct kfd_process_device, qpd)
733	736
	737	+struct svm_range_list {
	738	+ struct mutex lock;
	739	+ struct rb_root_cached objects;
	740	+ struct list_head list;
	741	+ struct work_struct deferred_list_work;
	742	+ struct list_head deferred_range_list;
	743	+ spinlock_t deferred_list_lock;
	744	+ atomic_t evicted_ranges;
	745	+ struct delayed_work restore_work;
	746	+};
	747	+
734	748	/* Process data */
735	749	struct kfd_process {
736	750	/*
@@ -809,6 +823,12 @@ struct kfd_process { · 72 unmodified lines
809	823	struct kobject *kobj;
810	824	struct kobject *kobj_queues;
811	825	struct attribute attr_pasid;
	826	+
	827	+ /* shared virtual memory registered by this process */
	828	+ struct svm_range_list svms;
	829	+ bool svm_disabled;
	830	+
	831	+ bool xnack_enabled;
812	832	};
813	833
814	834	#define KFD_PROCESS_TABLE_SIZE 5 /* bits: 32 entries */
@@ -842,6 +862,20 @@ struct kfd_process kfd_create_process(struct file filep); · 27 unmodified lines
842	862	struct kfd_process kfd_get_process(const struct task_struct );
843	863	struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid);
844	864	struct kfd_process kfd_lookup_process_by_mm(const struct mm_struct mm);
	865	+
	866	+int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id);
	867	+int kfd_process_gpuid_from_kgd(struct kfd_process *p,
	868	+ struct amdgpu_device adev, uint32_t gpuid,
	869	+ uint32_t *gpuidx);
	870	+static inline int kfd_process_gpuid_from_gpuidx(struct kfd_process *p,
	871	+ uint32_t gpuidx, uint32_t *gpuid) {
	872	+ return gpuidx < p->n_pdds ? p->pdds[gpuidx]->dev->id : -EINVAL;
	873	+}
	874	+static inline struct kfd_process_device *kfd_process_device_from_gpuidx(
	875	+ struct kfd_process *p, uint32_t gpuidx) {
	876	+ return gpuidx < p->n_pdds ? p->pdds[gpuidx] : NULL;
	877	+}
	878	+
845	879	void kfd_unref_process(struct kfd_process *p);
846	880	int kfd_process_evict_queues(struct kfd_process *p);
847	881	int kfd_process_restore_queues(struct kfd_process *p);
@@ -857,6 +891,8 @@ struct kfd_process_device kfd_get_process_device_data(struct kfd_dev dev, · 9 unmodified lines
857	891	struct kfd_process_device kfd_create_process_device_data(struct kfd_dev dev,
858	892	struct kfd_process *p);
859	893
	894	+bool kfd_process_xnack_mode(struct kfd_process *p, bool supported);
	895	+
860	896	int kfd_reserved_mem_mmap(struct kfd_dev dev, struct kfd_process process,
861	897	struct vm_area_struct *vma);
862	898
@@ -1052,6 +1088,7 @@ struct packet_manager_funcs { · 189 unmodified lines
1052	1088
1053	1089	extern const struct packet_manager_funcs kfd_vi_pm_funcs;
1054	1090	extern const struct packet_manager_funcs kfd_v9_pm_funcs;
	1091	+extern const struct packet_manager_funcs kfd_aldebaran_pm_funcs;
1055	1092
1056	1093	int pm_init(struct packet_manager pm, struct device_queue_manager dqm);
1057	1094	void pm_uninit(struct packet_manager *pm, bool hanging);

drivers/gpu/drm/amd/amdkfd/kfd_process.c+108−16

@@ -35,6 +35,7 @@ · 34 unmodified lines
35	35	#include <linux/pm_runtime.h>
36	36	#include "amdgpu_amdkfd.h"
37	37	#include "amdgpu.h"
	38	+#include "kfd_svm.h"
38	39
39	40	struct mm_struct;
40	41
@@ -42,6 +43,7 @@ struct mm_struct; · 1 unmodified line
42	43	#include "kfd_device_queue_manager.h"
43	44	#include "kfd_dbgmgr.h"
44	45	#include "kfd_iommu.h"
	46	+#include "kfd_svm.h"
45	47
46	48	/*
47	49	* List of struct kfd_process (field kfd_process).
@@ -250,7 +252,7 @@ cleanup: · 202 unmodified lines
250	252	}
251	253
252	254	/**
253		− * @kfd_get_cu_occupancy() - Collect number of waves in-flight on this device
	255	+ * @kfd_get_cu_occupancy - Collect number of waves in-flight on this device
254	256	* by current process. Translates acquired wave count into number of compute units
255	257	* that are occupied.
256	258	*
@@ -647,8 +649,9 @@ static void kfd_process_free_gpuvm(struct kgd_mem *mem, · 390 unmodified lines
647	649	{
648	650	struct kfd_dev *dev = pdd->dev;
649	651
650		− amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->vm);
651		− amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, NULL);
	652	+ amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->drm_priv);
	653	+ amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, pdd->drm_priv,
	654	+ NULL);
652	655	}
653	656
654	657	/* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
@@ -667,11 +670,11 @@ static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd, · 12 unmodified lines
667	670	int err;
668	671
669	672	err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size,
670		− pdd->vm, &mem, NULL, flags);
	673	+ pdd->drm_priv, &mem, NULL, flags);
671	674	if (err)
672	675	goto err_alloc_mem;
673	676
674		− err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, pdd->vm);
	677	+ err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem, pdd->drm_priv);
675	678	if (err)
676	679	goto err_map_mem;
677	680
@@ -712,7 +715,8 @@ sync_memory_failed: · 34 unmodified lines
712	715	return err;
713	716
714	717	err_map_mem:
715		− amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, NULL);
	718	+ amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, pdd->drm_priv,
	719	+ NULL);
716	720	err_alloc_mem:
717	721	*kptr = NULL;
718	722	return err;
@@ -901,13 +905,14 @@ static void kfd_process_device_free_bos(struct kfd_process_device *pdd) · 182 unmodified lines
901	905	for (i = 0; i < p->n_pdds; i++) {
902	906	struct kfd_process_device *peer_pdd = p->pdds[i];
903	907
904		− if (!peer_pdd->vm)
	908	+ if (!peer_pdd->drm_priv)
905	909	continue;
906	910	amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
907		− peer_pdd->dev->kgd, mem, peer_pdd->vm);
	911	+ peer_pdd->dev->kgd, mem, peer_pdd->drm_priv);
908	912	}
909	913
910		− amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem, NULL);
	914	+ amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem,
	915	+ pdd->drm_priv, NULL);
911	916	kfd_process_device_remove_obj_handle(pdd, id);
912	917	}
913	918	}
@@ -932,7 +937,7 @@ static void kfd_process_destroy_pdds(struct kfd_process *p) · 18 unmodified lines
932	937
933	938	if (pdd->drm_file) {
934	939	amdgpu_amdkfd_gpuvm_release_process_vm(
935		− pdd->dev->kgd, pdd->vm);
	940	+ pdd->dev->kgd, pdd->drm_priv);
936	941	fput(pdd->drm_file);
937	942	}
938	943
@@ -1000,6 +1005,7 @@ static void kfd_process_wq_release(struct work_struct *work) · 61 unmodified lines
1000	1005	kfd_iommu_unbind_process(p);
1001	1006
1002	1007	kfd_process_free_outstanding_kfd_bos(p);
	1008	+ svm_range_list_fini(p);
1003	1009
1004	1010	kfd_process_destroy_pdds(p);
1005	1011	dma_fence_put(p->ef);
@@ -1058,6 +1064,7 @@ static void kfd_process_notifier_release(struct mmu_notifier *mn, · 52 unmodified lines
1058	1064
1059	1065	cancel_delayed_work_sync(&p->eviction_work);
1060	1066	cancel_delayed_work_sync(&p->restore_work);
	1067	+ cancel_delayed_work_sync(&p->svms.restore_work);
1061	1068
1062	1069	mutex_lock(&p->mutex);
1063	1070
@@ -1186,6 +1193,56 @@ void kfd_process_set_trap_handler(struct qcm_process_device *qpd, · 122 unmodified lines
1186	1193	}
1187	1194	}
1188	1195
	1196	+bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
	1197	+{
	1198	+ int i;
	1199	+
	1200	+ /* On most GFXv9 GPUs, the retry mode in the SQ must match the
	1201	+ * boot time retry setting. Mixing processes with different
	1202	+ * XNACK/retry settings can hang the GPU.
	1203	+ *
	1204	+ * Different GPUs can have different noretry settings depending
	1205	+ * on HW bugs or limitations. We need to find at least one
	1206	+ * XNACK mode for this process that's compatible with all GPUs.
	1207	+ * Fortunately GPUs with retry enabled (noretry=0) can run code
	1208	+ * built for XNACK-off. On GFXv9 it may perform slower.
	1209	+ *
	1210	+ * Therefore applications built for XNACK-off can always be
	1211	+ * supported and will be our fallback if any GPU does not
	1212	+ * support retry.
	1213	+ */
	1214	+ for (i = 0; i < p->n_pdds; i++) {
	1215	+ struct kfd_dev *dev = p->pdds[i]->dev;
	1216	+
	1217	+ /* Only consider GFXv9 and higher GPUs. Older GPUs don't
	1218	+ * support the SVM APIs and don't need to be considered
	1219	+ * for the XNACK mode selection.
	1220	+ */
	1221	+ if (dev->device_info->asic_family < CHIP_VEGA10)
	1222	+ continue;
	1223	+ /* Aldebaran can always support XNACK because it can support
	1224	+ * per-process XNACK mode selection. But let the dev->noretry
	1225	+ * setting still influence the default XNACK mode.
	1226	+ */
	1227	+ if (supported &&
	1228	+ dev->device_info->asic_family == CHIP_ALDEBARAN)
	1229	+ continue;
	1230	+
	1231	+ /* GFXv10 and later GPUs do not support shader preemption
	1232	+ * during page faults. This can lead to poor QoS for queue
	1233	+ * management and memory-manager-related preemptions or
	1234	+ * even deadlocks.
	1235	+ */
	1236	+ if (dev->device_info->asic_family >= CHIP_NAVI10)
	1237	+ return false;
	1238	+
	1239	+ if (dev->noretry)
	1240	+ return false;
	1241	+ }
	1242	+
	1243	+ return true;
	1244	+}
	1245	+
1189	1246	/*
1190	1247	* On return the kfd_process is fully operational and will be freed when the
1191	1248	* mm is released
@@ -1205,6 +1262,7 @@ static struct kfd_process create_process(const struct task_struct thread) · 13 unmodified lines
1205	1262	process->mm = thread->mm;
1206	1263	process->lead_thread = thread->group_leader;
1207	1264	process->n_pdds = 0;
	1265	+ process->svm_disabled = false;
1208	1266	INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1209	1267	INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1210	1268	process->last_restore_timestamp = get_jiffies_64();
@@ -1224,6 +1282,13 @@ static struct kfd_process create_process(const struct task_struct thread) · 13 unmodified lines
1224	1282	if (err != 0)
1225	1283	goto err_init_apertures;
1226	1284
	1285	+ /* Check XNACK support after PDDs are created in kfd_init_apertures */
	1286	+ process->xnack_enabled = kfd_process_xnack_mode(process, false);
	1287	+
	1288	+ err = svm_range_list_init(process);
	1289	+ if (err)
	1290	+ goto err_init_svm_range_list;
	1291	+
1227	1292	/* alloc_notifier needs to find the process in the hash table */
1228	1293	hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1229	1294	(uintptr_t)process->mm);
@@ -1246,6 +1311,8 @@ static struct kfd_process create_process(const struct task_struct thread) · 16 unmodified lines
1246	1311
1247	1312	err_register_notifier:
1248	1313	hash_del_rcu(&process->kfd_processes);
	1314	+ svm_range_list_fini(process);
	1315	+err_init_svm_range_list:
1249	1316	kfd_process_free_outstanding_kfd_bos(process);
1250	1317	kfd_process_destroy_pdds(process);
1251	1318	err_init_apertures:
@@ -1375,7 +1442,7 @@ int kfd_process_device_init_vm(struct kfd_process_device *pdd, · 123 unmodified lines
1375	1442	if (!drm_file)
1376	1443	return -EINVAL;
1377	1444
1378		− if (pdd->vm)
	1445	+ if (pdd->drm_priv)
1379	1446	return -EBUSY;
1380	1447
1381	1448	p = pdd->process;
@@ -1383,13 +1450,12 @@ int kfd_process_device_init_vm(struct kfd_process_device *pdd, · 1 unmodified line
1383	1450
1384	1451	ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(
1385	1452	dev->kgd, drm_file, p->pasid,
1386		− &pdd->vm, &p->kgd_process_info, &p->ef);
	1453	+ &p->kgd_process_info, &p->ef);
1387	1454	if (ret) {
1388	1455	pr_err("Failed to create process VM object\n");
1389	1456	return ret;
1390	1457	}
1391		−
1392		− amdgpu_vm_set_task_info(pdd->vm);
	1458	+ pdd->drm_priv = drm_file->private_data;
1393	1459
1394	1460	ret = kfd_process_device_reserve_ib_mem(pdd);
1395	1461	if (ret)
@@ -1405,7 +1471,7 @@ int kfd_process_device_init_vm(struct kfd_process_device *pdd, · 9 unmodified lines
1405	1471	err_init_cwsr:
1406	1472	err_reserve_ib_mem:
1407	1473	kfd_process_device_free_bos(pdd);
1408		− pdd->vm = NULL;
	1474	+ pdd->drm_priv = NULL;
1409	1475
1410	1476	return ret;
1411	1477	}
@@ -1429,7 +1495,7 @@ struct kfd_process_device kfd_bind_process_to_device(struct kfd_dev dev, · 17 unmodified lines
1429	1495	return ERR_PTR(-ENOMEM);
1430	1496	}
1431	1497
1432		− if (!pdd->vm)
	1498	+ if (!pdd->drm_priv)
1433	1499	return ERR_PTR(-ENODEV);
1434	1500
1435	1501	/*
@@ -1600,6 +1666,32 @@ int kfd_process_restore_queues(struct kfd_process *p) · 164 unmodified lines
1600	1666	return ret;
1601	1667	}
1602	1668
	1669	+int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
	1670	+{
	1671	+ int i;
	1672	+
	1673	+ for (i = 0; i < p->n_pdds; i++)
	1674	+ if (p->pdds[i] && gpu_id == p->pdds[i]->dev->id)
	1675	+ return i;
	1676	+ return -EINVAL;
	1677	+}
	1678	+
	1679	+int
	1680	+kfd_process_gpuid_from_kgd(struct kfd_process p, struct amdgpu_device adev,
	1681	+ uint32_t gpuid, uint32_t gpuidx)
	1682	+{
	1683	+ struct kgd_dev kgd = (struct kgd_dev )adev;
	1684	+ int i;
	1685	+
	1686	+ for (i = 0; i < p->n_pdds; i++)
	1687	+ if (p->pdds[i] && p->pdds[i]->dev->kgd == kgd) {
	1688	+ *gpuid = p->pdds[i]->dev->id;
	1689	+ *gpuidx = i;
	1690	+ return 0;
	1691	+ }
	1692	+ return -EINVAL;
	1693	+}
	1694	+
1603	1695	static void evict_process_worker(struct work_struct *work)
1604	1696	{
1605	1697	int ret;

drivers/gpu/drm/amd/amdkfd/kfd_svm.c+3085−0

@@ -0,0 +1,3085 @@
	1	+// SPDX-License-Identifier: GPL-2.0 OR MIT
	2	+/*
	3	+ * Copyright 2020-2021 Advanced Micro Devices, Inc.
	4	+ *
	5	+ * Permission is hereby granted, free of charge, to any person obtaining a
	6	+ * copy of this software and associated documentation files (the "Software"),
	7	+ * to deal in the Software without restriction, including without limitation
	8	+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
	9	+ * and/or sell copies of the Software, and to permit persons to whom the
	10	+ * Software is furnished to do so, subject to the following conditions:
	11	+ *
	12	+ * The above copyright notice and this permission notice shall be included in
	13	+ * all copies or substantial portions of the Software.
	14	+ *
	15	+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	19	+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	20	+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	21	+ * OTHER DEALINGS IN THE SOFTWARE.
	22	+ */
	23	+
	24	+#include <linux/types.h>
	25	+#include <linux/sched/task.h>
	26	+#include "amdgpu_sync.h"
	27	+#include "amdgpu_object.h"
	28	+#include "amdgpu_vm.h"
	29	+#include "amdgpu_mn.h"
	30	+#include "amdgpu.h"
	31	+#include "amdgpu_xgmi.h"
	32	+#include "kfd_priv.h"
	33	+#include "kfd_svm.h"
	34	+#include "kfd_migrate.h"
	35	+
	36	+#define AMDGPU_SVM_RANGE_RESTORE_DELAY_MS 1
	37	+
	38	+/* Long enough to ensure no retry fault comes after svm range is restored and
	39	+ * page table is updated.
	40	+ */
	41	+#define AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING 2000
	42	+
	43	+static void svm_range_evict_svm_bo_worker(struct work_struct *work);
	44	+static bool
	45	+svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,
	46	+ const struct mmu_notifier_range *range,
	47	+ unsigned long cur_seq);
	48	+
	49	+static const struct mmu_interval_notifier_ops svm_range_mn_ops = {
	50	+ .invalidate = svm_range_cpu_invalidate_pagetables,
	51	+};
	52	+
	53	+/**
	54	+ * svm_range_unlink - unlink svm_range from lists and interval tree
	55	+ * @prange: svm range structure to be removed
	56	+ *
	57	+ * Remove the svm_range from the svms and svm_bo lists and the svms
	58	+ * interval tree.
	59	+ *
	60	+ * Context: The caller must hold svms->lock
	61	+ */
	62	+static void svm_range_unlink(struct svm_range *prange)
	63	+{
	64	+ pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,
	65	+ prange, prange->start, prange->last);
	66	+
	67	+ if (prange->svm_bo) {
	68	+ spin_lock(&prange->svm_bo->list_lock);
	69	+ list_del(&prange->svm_bo_list);
	70	+ spin_unlock(&prange->svm_bo->list_lock);
	71	+ }
	72	+
	73	+ list_del(&prange->list);
	74	+ if (prange->it_node.start != 0 && prange->it_node.last != 0)
	75	+ interval_tree_remove(&prange->it_node, &prange->svms->objects);
	76	+}
	77	+
	78	+static void
	79	+svm_range_add_notifier_locked(struct mm_struct mm, struct svm_range prange)
	80	+{
	81	+ pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,
	82	+ prange, prange->start, prange->last);
	83	+
	84	+ mmu_interval_notifier_insert_locked(&prange->notifier, mm,
	85	+ prange->start << PAGE_SHIFT,
	86	+ prange->npages << PAGE_SHIFT,
	87	+ &svm_range_mn_ops);
	88	+}
	89	+
	90	+/**
	91	+ * svm_range_add_to_svms - add svm range to svms
	92	+ * @prange: svm range structure to be added
	93	+ *
	94	+ * Add the svm range to svms interval tree and link list
	95	+ *
	96	+ * Context: The caller must hold svms->lock
	97	+ */
	98	+static void svm_range_add_to_svms(struct svm_range *prange)
	99	+{
	100	+ pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms,
	101	+ prange, prange->start, prange->last);
	102	+
	103	+ list_add_tail(&prange->list, &prange->svms->list);
	104	+ prange->it_node.start = prange->start;
	105	+ prange->it_node.last = prange->last;
	106	+ interval_tree_insert(&prange->it_node, &prange->svms->objects);
	107	+}
	108	+
	109	+static void svm_range_remove_notifier(struct svm_range *prange)
	110	+{
	111	+ pr_debug("remove notifier svms 0x%p prange 0x%p [0x%lx 0x%lx]\n",
	112	+ prange->svms, prange,
	113	+ prange->notifier.interval_tree.start >> PAGE_SHIFT,
	114	+ prange->notifier.interval_tree.last >> PAGE_SHIFT);
	115	+
	116	+ if (prange->notifier.interval_tree.start != 0 &&
	117	+ prange->notifier.interval_tree.last != 0)
	118	+ mmu_interval_notifier_remove(&prange->notifier);
	119	+}
	120	+
	121	+static int
	122	+svm_range_dma_map_dev(struct device dev, dma_addr_t *dma_addr,
	123	+ unsigned long *hmm_pfns, uint64_t npages)
	124	+{
	125	+ enum dma_data_direction dir = DMA_BIDIRECTIONAL;
	126	+ dma_addr_t addr = dma_addr;
	127	+ struct page *page;
	128	+ int i, r;
	129	+
	130	+ if (!addr) {
	131	+ addr = kvmalloc_array(npages, sizeof(*addr),
	132	+ GFP_KERNEL \| __GFP_ZERO);
	133	+ if (!addr)
	134	+ return -ENOMEM;
	135	+ *dma_addr = addr;
	136	+ }
	137	+
	138	+ for (i = 0; i < npages; i++) {
	139	+ if (WARN_ONCE(addr[i] && !dma_mapping_error(dev, addr[i]),
	140	+ "leaking dma mapping\n"))
	141	+ dma_unmap_page(dev, addr[i], PAGE_SIZE, dir);
	142	+
	143	+ page = hmm_pfn_to_page(hmm_pfns[i]);
	144	+ addr[i] = dma_map_page(dev, page, 0, PAGE_SIZE, dir);
	145	+ r = dma_mapping_error(dev, addr[i]);
	146	+ if (r) {
	147	+ pr_debug("failed %d dma_map_page\n", r);
	148	+ return r;
	149	+ }
	150	+ pr_debug("dma mapping 0x%llx for page addr 0x%lx\n",
	151	+ addr[i] >> PAGE_SHIFT, page_to_pfn(page));
	152	+ }
	153	+ return 0;
	154	+}
	155	+
	156	+static int
	157	+svm_range_dma_map(struct svm_range prange, unsigned long bitmap,
	158	+ unsigned long *hmm_pfns)
	159	+{
	160	+ struct kfd_process *p;
	161	+ uint32_t gpuidx;
	162	+ int r;
	163	+
	164	+ p = container_of(prange->svms, struct kfd_process, svms);
	165	+
	166	+ for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
	167	+ struct kfd_process_device *pdd;
	168	+ struct amdgpu_device *adev;
	169	+
	170	+ pr_debug("mapping to gpu idx 0x%x\n", gpuidx);
	171	+ pdd = kfd_process_device_from_gpuidx(p, gpuidx);
	172	+ if (!pdd) {
	173	+ pr_debug("failed to find device idx %d\n", gpuidx);
	174	+ return -EINVAL;
	175	+ }
	176	+ adev = (struct amdgpu_device *)pdd->dev->kgd;
	177	+
	178	+ r = svm_range_dma_map_dev(adev->dev, &prange->dma_addr[gpuidx],
	179	+ hmm_pfns, prange->npages);
	180	+ if (r)
	181	+ break;
	182	+ }
	183	+
	184	+ return r;
	185	+}
	186	+
	187	+void svm_range_dma_unmap(struct device dev, dma_addr_t dma_addr,
	188	+ unsigned long offset, unsigned long npages)
	189	+{
	190	+ enum dma_data_direction dir = DMA_BIDIRECTIONAL;
	191	+ int i;
	192	+
	193	+ if (!dma_addr)
	194	+ return;
	195	+
	196	+ for (i = offset; i < offset + npages; i++) {
	197	+ if (!dma_addr[i] \|\| dma_mapping_error(dev, dma_addr[i]))
	198	+ continue;
	199	+ pr_debug("dma unmapping 0x%llx\n", dma_addr[i] >> PAGE_SHIFT);
	200	+ dma_unmap_page(dev, dma_addr[i], PAGE_SIZE, dir);
	201	+ dma_addr[i] = 0;
	202	+ }
	203	+}
	204	+
	205	+void svm_range_free_dma_mappings(struct svm_range *prange)
	206	+{
	207	+ struct kfd_process_device *pdd;
	208	+ dma_addr_t *dma_addr;
	209	+ struct device *dev;
	210	+ struct kfd_process *p;
	211	+ uint32_t gpuidx;
	212	+
	213	+ p = container_of(prange->svms, struct kfd_process, svms);
	214	+
	215	+ for (gpuidx = 0; gpuidx < MAX_GPU_INSTANCE; gpuidx++) {
	216	+ dma_addr = prange->dma_addr[gpuidx];
	217	+ if (!dma_addr)
	218	+ continue;
	219	+
	220	+ pdd = kfd_process_device_from_gpuidx(p, gpuidx);
	221	+ if (!pdd) {
	222	+ pr_debug("failed to find device idx %d\n", gpuidx);
	223	+ continue;
	224	+ }
	225	+ dev = &pdd->dev->pdev->dev;
	226	+ svm_range_dma_unmap(dev, dma_addr, 0, prange->npages);
	227	+ kvfree(dma_addr);
	228	+ prange->dma_addr[gpuidx] = NULL;
	229	+ }
	230	+}
	231	+
	232	+static void svm_range_free(struct svm_range *prange)
	233	+{
	234	+ pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, prange,
	235	+ prange->start, prange->last);
	236	+
	237	+ svm_range_vram_node_free(prange);
	238	+ svm_range_free_dma_mappings(prange);
	239	+ mutex_destroy(&prange->lock);
	240	+ mutex_destroy(&prange->migrate_mutex);
	241	+ kfree(prange);
	242	+}
	243	+
	244	+static void
	245	+svm_range_set_default_attributes(int32_t location, int32_t prefetch_loc,
	246	+ uint8_t granularity, uint32_t flags)
	247	+{
	248	+ *location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
	249	+ *prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
	250	+ *granularity = 9;
	251	+ *flags =
	252	+ KFD_IOCTL_SVM_FLAG_HOST_ACCESS \| KFD_IOCTL_SVM_FLAG_COHERENT;
	253	+}
	254	+
	255	+static struct
	256	+svm_range svm_range_new(struct svm_range_list svms, uint64_t start,
	257	+ uint64_t last)
	258	+{
	259	+ uint64_t size = last - start + 1;
	260	+ struct svm_range *prange;
	261	+ struct kfd_process *p;
	262	+
	263	+ prange = kzalloc(sizeof(*prange), GFP_KERNEL);
	264	+ if (!prange)
	265	+ return NULL;
	266	+ prange->npages = size;
	267	+ prange->svms = svms;
	268	+ prange->start = start;
	269	+ prange->last = last;
	270	+ INIT_LIST_HEAD(&prange->list);
	271	+ INIT_LIST_HEAD(&prange->update_list);
	272	+ INIT_LIST_HEAD(&prange->remove_list);
	273	+ INIT_LIST_HEAD(&prange->insert_list);
	274	+ INIT_LIST_HEAD(&prange->svm_bo_list);
	275	+ INIT_LIST_HEAD(&prange->deferred_list);
	276	+ INIT_LIST_HEAD(&prange->child_list);
	277	+ atomic_set(&prange->invalid, 0);
	278	+ prange->validate_timestamp = 0;
	279	+ mutex_init(&prange->migrate_mutex);
	280	+ mutex_init(&prange->lock);
	281	+
	282	+ p = container_of(svms, struct kfd_process, svms);
	283	+ if (p->xnack_enabled)
	284	+ bitmap_fill(prange->bitmap_access, MAX_GPU_INSTANCE);
	285	+
	286	+ svm_range_set_default_attributes(&prange->preferred_loc,
	287	+ &prange->prefetch_loc,
	288	+ &prange->granularity, &prange->flags);
	289	+
	290	+ pr_debug("svms 0x%p [0x%llx 0x%llx]\n", svms, start, last);
	291	+
	292	+ return prange;
	293	+}
	294	+
	295	+static bool svm_bo_ref_unless_zero(struct svm_range_bo *svm_bo)
	296	+{
	297	+ if (!svm_bo \|\| !kref_get_unless_zero(&svm_bo->kref))
	298	+ return false;
	299	+
	300	+ return true;
	301	+}
	302	+
	303	+static struct svm_range_bo svm_range_bo_ref(struct svm_range_bo svm_bo)
	304	+{
	305	+ if (svm_bo)
	306	+ kref_get(&svm_bo->kref);
	307	+
	308	+ return svm_bo;
	309	+}
	310	+
	311	+static void svm_range_bo_release(struct kref *kref)
	312	+{
	313	+ struct svm_range_bo *svm_bo;
	314	+
	315	+ svm_bo = container_of(kref, struct svm_range_bo, kref);
	316	+ spin_lock(&svm_bo->list_lock);
	317	+ while (!list_empty(&svm_bo->range_list)) {
	318	+ struct svm_range *prange =
	319	+ list_first_entry(&svm_bo->range_list,
	320	+ struct svm_range, svm_bo_list);
	321	+ /* list_del_init tells a concurrent svm_range_vram_node_new when
	322	+ * it's safe to reuse the svm_bo pointer and svm_bo_list head.
	323	+ */
	324	+ list_del_init(&prange->svm_bo_list);
	325	+ spin_unlock(&svm_bo->list_lock);
	326	+
	327	+ pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,
	328	+ prange->start, prange->last);
	329	+ mutex_lock(&prange->lock);
	330	+ prange->svm_bo = NULL;
	331	+ mutex_unlock(&prange->lock);
	332	+
	333	+ spin_lock(&svm_bo->list_lock);
	334	+ }
	335	+ spin_unlock(&svm_bo->list_lock);
	336	+ if (!dma_fence_is_signaled(&svm_bo->eviction_fence->base)) {
	337	+ /* We're not in the eviction worker.
	338	+ * Signal the fence and synchronize with any
	339	+ * pending eviction work.
	340	+ */
	341	+ dma_fence_signal(&svm_bo->eviction_fence->base);
	342	+ cancel_work_sync(&svm_bo->eviction_work);
	343	+ }
	344	+ dma_fence_put(&svm_bo->eviction_fence->base);
	345	+ amdgpu_bo_unref(&svm_bo->bo);
	346	+ kfree(svm_bo);
	347	+}
	348	+
	349	+static void svm_range_bo_unref(struct svm_range_bo *svm_bo)
	350	+{
	351	+ if (!svm_bo)
	352	+ return;
	353	+
	354	+ kref_put(&svm_bo->kref, svm_range_bo_release);
	355	+}
	356	+
	357	+static bool
	358	+svm_range_validate_svm_bo(struct amdgpu_device adev, struct svm_range prange)
	359	+{
	360	+ struct amdgpu_device *bo_adev;
	361	+
	362	+ mutex_lock(&prange->lock);
	363	+ if (!prange->svm_bo) {
	364	+ mutex_unlock(&prange->lock);
	365	+ return false;
	366	+ }
	367	+ if (prange->ttm_res) {
	368	+ /* We still have a reference, all is well */
	369	+ mutex_unlock(&prange->lock);
	370	+ return true;
	371	+ }
	372	+ if (svm_bo_ref_unless_zero(prange->svm_bo)) {
	373	+ /*
	374	+ * Migrate from GPU to GPU, remove range from source bo_adev
	375	+ * svm_bo range list, and return false to allocate svm_bo from
	376	+ * destination adev.
	377	+ */
	378	+ bo_adev = amdgpu_ttm_adev(prange->svm_bo->bo->tbo.bdev);
	379	+ if (bo_adev != adev) {
	380	+ mutex_unlock(&prange->lock);
	381	+
	382	+ spin_lock(&prange->svm_bo->list_lock);
	383	+ list_del_init(&prange->svm_bo_list);
	384	+ spin_unlock(&prange->svm_bo->list_lock);
	385	+
	386	+ svm_range_bo_unref(prange->svm_bo);
	387	+ return false;
	388	+ }
	389	+ if (READ_ONCE(prange->svm_bo->evicting)) {
	390	+ struct dma_fence *f;
	391	+ struct svm_range_bo *svm_bo;
	392	+ /* The BO is getting evicted,
	393	+ * we need to get a new one
	394	+ */
	395	+ mutex_unlock(&prange->lock);
	396	+ svm_bo = prange->svm_bo;
	397	+ f = dma_fence_get(&svm_bo->eviction_fence->base);
	398	+ svm_range_bo_unref(prange->svm_bo);
	399	+ /* wait for the fence to avoid long spin-loop
	400	+ * at list_empty_careful
	401	+ */
	402	+ dma_fence_wait(f, false);
	403	+ dma_fence_put(f);
	404	+ } else {
	405	+ /* The BO was still around and we got
	406	+ * a new reference to it
	407	+ */
	408	+ mutex_unlock(&prange->lock);
	409	+ pr_debug("reuse old bo svms 0x%p [0x%lx 0x%lx]\n",
	410	+ prange->svms, prange->start, prange->last);
	411	+
	412	+ prange->ttm_res = &prange->svm_bo->bo->tbo.mem;
	413	+ return true;
	414	+ }
	415	+
	416	+ } else {
	417	+ mutex_unlock(&prange->lock);
	418	+ }
	419	+
	420	+ /* We need a new svm_bo. Spin-loop to wait for concurrent
	421	+ * svm_range_bo_release to finish removing this range from
	422	+ * its range list. After this, it is safe to reuse the
	423	+ * svm_bo pointer and svm_bo_list head.
	424	+ */
	425	+ while (!list_empty_careful(&prange->svm_bo_list))
	426	+ ;
	427	+
	428	+ return false;
	429	+}
	430	+
	431	+static struct svm_range_bo *svm_range_bo_new(void)
	432	+{
	433	+ struct svm_range_bo *svm_bo;
	434	+
	435	+ svm_bo = kzalloc(sizeof(*svm_bo), GFP_KERNEL);
	436	+ if (!svm_bo)
	437	+ return NULL;
	438	+
	439	+ kref_init(&svm_bo->kref);
	440	+ INIT_LIST_HEAD(&svm_bo->range_list);
	441	+ spin_lock_init(&svm_bo->list_lock);
	442	+
	443	+ return svm_bo;
	444	+}
	445	+
	446	+int
	447	+svm_range_vram_node_new(struct amdgpu_device adev, struct svm_range prange,
	448	+ bool clear)
	449	+{
	450	+ struct amdgpu_bo_param bp;
	451	+ struct svm_range_bo *svm_bo;
	452	+ struct amdgpu_bo_user *ubo;
	453	+ struct amdgpu_bo *bo;
	454	+ struct kfd_process *p;
	455	+ struct mm_struct *mm;
	456	+ int r;
	457	+
	458	+ p = container_of(prange->svms, struct kfd_process, svms);
	459	+ pr_debug("pasid: %x svms 0x%p [0x%lx 0x%lx]\n", p->pasid, prange->svms,
	460	+ prange->start, prange->last);
	461	+
	462	+ if (svm_range_validate_svm_bo(adev, prange))
	463	+ return 0;
	464	+
	465	+ svm_bo = svm_range_bo_new();
	466	+ if (!svm_bo) {
	467	+ pr_debug("failed to alloc svm bo\n");
	468	+ return -ENOMEM;
	469	+ }
	470	+ mm = get_task_mm(p->lead_thread);
	471	+ if (!mm) {
	472	+ pr_debug("failed to get mm\n");
	473	+ kfree(svm_bo);
	474	+ return -ESRCH;
	475	+ }
	476	+ svm_bo->svms = prange->svms;
	477	+ svm_bo->eviction_fence =
	478	+ amdgpu_amdkfd_fence_create(dma_fence_context_alloc(1),
	479	+ mm,
	480	+ svm_bo);
	481	+ mmput(mm);
	482	+ INIT_WORK(&svm_bo->eviction_work, svm_range_evict_svm_bo_worker);
	483	+ svm_bo->evicting = 0;
	484	+ memset(&bp, 0, sizeof(bp));
	485	+ bp.size = prange->npages * PAGE_SIZE;
	486	+ bp.byte_align = PAGE_SIZE;
	487	+ bp.domain = AMDGPU_GEM_DOMAIN_VRAM;
	488	+ bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS;
	489	+ bp.flags \|= clear ? AMDGPU_GEM_CREATE_VRAM_CLEARED : 0;
	490	+ bp.flags \|= AMDGPU_AMDKFD_CREATE_SVM_BO;
	491	+ bp.type = ttm_bo_type_device;
	492	+ bp.resv = NULL;
	493	+
	494	+ r = amdgpu_bo_create_user(adev, &bp, &ubo);
	495	+ if (r) {
	496	+ pr_debug("failed %d to create bo\n", r);
	497	+ goto create_bo_failed;
	498	+ }
	499	+ bo = &ubo->bo;
	500	+ r = amdgpu_bo_reserve(bo, true);
	501	+ if (r) {
	502	+ pr_debug("failed %d to reserve bo\n", r);
	503	+ goto reserve_bo_failed;
	504	+ }
	505	+
	506	+ r = dma_resv_reserve_shared(bo->tbo.base.resv, 1);
	507	+ if (r) {
	508	+ pr_debug("failed %d to reserve bo\n", r);
	509	+ amdgpu_bo_unreserve(bo);
	510	+ goto reserve_bo_failed;
	511	+ }
	512	+ amdgpu_bo_fence(bo, &svm_bo->eviction_fence->base, true);
	513	+
	514	+ amdgpu_bo_unreserve(bo);
	515	+
	516	+ svm_bo->bo = bo;
	517	+ prange->svm_bo = svm_bo;
	518	+ prange->ttm_res = &bo->tbo.mem;
	519	+ prange->offset = 0;
	520	+
	521	+ spin_lock(&svm_bo->list_lock);
	522	+ list_add(&prange->svm_bo_list, &svm_bo->range_list);
	523	+ spin_unlock(&svm_bo->list_lock);
	524	+
	525	+ return 0;
	526	+
	527	+reserve_bo_failed:
	528	+ amdgpu_bo_unref(&bo);
	529	+create_bo_failed:
	530	+ dma_fence_put(&svm_bo->eviction_fence->base);
	531	+ kfree(svm_bo);
	532	+ prange->ttm_res = NULL;
	533	+
	534	+ return r;
	535	+}
	536	+
	537	+void svm_range_vram_node_free(struct svm_range *prange)
	538	+{
	539	+ svm_range_bo_unref(prange->svm_bo);
	540	+ prange->ttm_res = NULL;
	541	+}
	542	+
	543	+struct amdgpu_device *
	544	+svm_range_get_adev_by_id(struct svm_range *prange, uint32_t gpu_id)
	545	+{
	546	+ struct kfd_process_device *pdd;
	547	+ struct kfd_process *p;
	548	+ int32_t gpu_idx;
	549	+
	550	+ p = container_of(prange->svms, struct kfd_process, svms);
	551	+
	552	+ gpu_idx = kfd_process_gpuidx_from_gpuid(p, gpu_id);
	553	+ if (gpu_idx < 0) {
	554	+ pr_debug("failed to get device by id 0x%x\n", gpu_id);
	555	+ return NULL;
	556	+ }
	557	+ pdd = kfd_process_device_from_gpuidx(p, gpu_idx);
	558	+ if (!pdd) {
	559	+ pr_debug("failed to get device by idx 0x%x\n", gpu_idx);
	560	+ return NULL;
	561	+ }
	562	+
	563	+ return (struct amdgpu_device *)pdd->dev->kgd;
	564	+}
	565	+
	566	+static int svm_range_bo_validate(void param, struct amdgpu_bo bo)
	567	+{
	568	+ struct ttm_operation_ctx ctx = { false, false };
	569	+
	570	+ amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_VRAM);
	571	+
	572	+ return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
	573	+}
	574	+
	575	+static int
	576	+svm_range_check_attr(struct kfd_process *p,
	577	+ uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs)
	578	+{
	579	+ uint32_t i;
	580	+ int gpuidx;
	581	+
	582	+ for (i = 0; i < nattr; i++) {
	583	+ switch (attrs[i].type) {
	584	+ case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
	585	+ if (attrs[i].value != KFD_IOCTL_SVM_LOCATION_SYSMEM &&
	586	+ attrs[i].value != KFD_IOCTL_SVM_LOCATION_UNDEFINED &&
	587	+ kfd_process_gpuidx_from_gpuid(p,
	588	+ attrs[i].value) < 0) {
	589	+ pr_debug("no GPU 0x%x found\n", attrs[i].value);
	590	+ return -EINVAL;
	591	+ }
	592	+ break;
	593	+ case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
	594	+ if (attrs[i].value != KFD_IOCTL_SVM_LOCATION_SYSMEM &&
	595	+ kfd_process_gpuidx_from_gpuid(p,
	596	+ attrs[i].value) < 0) {
	597	+ pr_debug("no GPU 0x%x found\n", attrs[i].value);
	598	+ return -EINVAL;
	599	+ }
	600	+ break;
	601	+ case KFD_IOCTL_SVM_ATTR_ACCESS:
	602	+ case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:
	603	+ case KFD_IOCTL_SVM_ATTR_NO_ACCESS:
	604	+ gpuidx = kfd_process_gpuidx_from_gpuid(p,
	605	+ attrs[i].value);
	606	+ if (gpuidx < 0) {
	607	+ pr_debug("no GPU 0x%x found\n", attrs[i].value);
	608	+ return -EINVAL;
	609	+ }
	610	+ break;
	611	+ case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
	612	+ break;
	613	+ case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:
	614	+ break;
	615	+ case KFD_IOCTL_SVM_ATTR_GRANULARITY:
	616	+ break;
	617	+ default:
	618	+ pr_debug("unknown attr type 0x%x\n", attrs[i].type);
	619	+ return -EINVAL;
	620	+ }
	621	+ }
	622	+
	623	+ return 0;
	624	+}
	625	+
	626	+static void
	627	+svm_range_apply_attrs(struct kfd_process p, struct svm_range prange,
	628	+ uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs)
	629	+{
	630	+ uint32_t i;
	631	+ int gpuidx;
	632	+
	633	+ for (i = 0; i < nattr; i++) {
	634	+ switch (attrs[i].type) {
	635	+ case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
	636	+ prange->preferred_loc = attrs[i].value;
	637	+ break;
	638	+ case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
	639	+ prange->prefetch_loc = attrs[i].value;
	640	+ break;
	641	+ case KFD_IOCTL_SVM_ATTR_ACCESS:
	642	+ case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:
	643	+ case KFD_IOCTL_SVM_ATTR_NO_ACCESS:
	644	+ gpuidx = kfd_process_gpuidx_from_gpuid(p,
	645	+ attrs[i].value);
	646	+ if (attrs[i].type == KFD_IOCTL_SVM_ATTR_NO_ACCESS) {
	647	+ bitmap_clear(prange->bitmap_access, gpuidx, 1);
	648	+ bitmap_clear(prange->bitmap_aip, gpuidx, 1);
	649	+ } else if (attrs[i].type == KFD_IOCTL_SVM_ATTR_ACCESS) {
	650	+ bitmap_set(prange->bitmap_access, gpuidx, 1);
	651	+ bitmap_clear(prange->bitmap_aip, gpuidx, 1);
	652	+ } else {
	653	+ bitmap_clear(prange->bitmap_access, gpuidx, 1);
	654	+ bitmap_set(prange->bitmap_aip, gpuidx, 1);
	655	+ }
	656	+ break;
	657	+ case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
	658	+ prange->flags \|= attrs[i].value;
	659	+ break;
	660	+ case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:
	661	+ prange->flags &= ~attrs[i].value;
	662	+ break;
	663	+ case KFD_IOCTL_SVM_ATTR_GRANULARITY:
	664	+ prange->granularity = attrs[i].value;
	665	+ break;
	666	+ default:
	667	+ WARN_ONCE(1, "svm_range_check_attrs wasn't called?");
	668	+ }
	669	+ }
	670	+}
	671	+
	672	+/**
	673	+ * svm_range_debug_dump - print all range information from svms
	674	+ * @svms: svm range list header
	675	+ *
	676	+ * debug output svm range start, end, prefetch location from svms
	677	+ * interval tree and link list
	678	+ *
	679	+ * Context: The caller must hold svms->lock
	680	+ */
	681	+static void svm_range_debug_dump(struct svm_range_list *svms)
	682	+{
	683	+ struct interval_tree_node *node;
	684	+ struct svm_range *prange;
	685	+
	686	+ pr_debug("dump svms 0x%p list\n", svms);
	687	+ pr_debug("range\tstart\tpage\tend\t\tlocation\n");
	688	+
	689	+ list_for_each_entry(prange, &svms->list, list) {
	690	+ pr_debug("0x%p 0x%lx\t0x%llx\t0x%llx\t0x%x\n",
	691	+ prange, prange->start, prange->npages,
	692	+ prange->start + prange->npages - 1,
	693	+ prange->actual_loc);
	694	+ }
	695	+
	696	+ pr_debug("dump svms 0x%p interval tree\n", svms);
	697	+ pr_debug("range\tstart\tpage\tend\t\tlocation\n");
	698	+ node = interval_tree_iter_first(&svms->objects, 0, ~0ULL);
	699	+ while (node) {
	700	+ prange = container_of(node, struct svm_range, it_node);
	701	+ pr_debug("0x%p 0x%lx\t0x%llx\t0x%llx\t0x%x\n",
	702	+ prange, prange->start, prange->npages,
	703	+ prange->start + prange->npages - 1,
	704	+ prange->actual_loc);
	705	+ node = interval_tree_iter_next(node, 0, ~0ULL);
	706	+ }
	707	+}
	708	+
	709	+static bool
	710	+svm_range_is_same_attrs(struct svm_range old, struct svm_range new)
	711	+{
	712	+ return (old->prefetch_loc == new->prefetch_loc &&
	713	+ old->flags == new->flags &&
	714	+ old->granularity == new->granularity);
	715	+}
	716	+
	717	+static int
	718	+svm_range_split_array(void ppnew, void ppold, size_t size,
	719	+ uint64_t old_start, uint64_t old_n,
	720	+ uint64_t new_start, uint64_t new_n)
	721	+{
	722	+ unsigned char new, old, *pold;
	723	+ uint64_t d;
	724	+
	725	+ if (!ppold)
	726	+ return 0;
	727	+ pold = (unsigned char *)ppold;
	728	+ if (!pold)
	729	+ return 0;
	730	+
	731	+ new = kvmalloc_array(new_n, size, GFP_KERNEL);
	732	+ if (!new)
	733	+ return -ENOMEM;
	734	+
	735	+ d = (new_start - old_start) * size;
	736	+ memcpy(new, pold + d, new_n * size);
	737	+
	738	+ old = kvmalloc_array(old_n, size, GFP_KERNEL);
	739	+ if (!old) {
	740	+ kvfree(new);
	741	+ return -ENOMEM;
	742	+ }
	743	+
	744	+ d = (new_start == old_start) ? new_n * size : 0;
	745	+ memcpy(old, pold + d, old_n * size);
	746	+
	747	+ kvfree(pold);
	748	+ (void *)ppold = old;
	749	+ (void *)ppnew = new;
	750	+
	751	+ return 0;
	752	+}
	753	+
	754	+static int
	755	+svm_range_split_pages(struct svm_range new, struct svm_range old,
	756	+ uint64_t start, uint64_t last)
	757	+{
	758	+ uint64_t npages = last - start + 1;
	759	+ int i, r;
	760	+
	761	+ for (i = 0; i < MAX_GPU_INSTANCE; i++) {
	762	+ r = svm_range_split_array(&new->dma_addr[i], &old->dma_addr[i],
	763	+ sizeof(*old->dma_addr[i]), old->start,
	764	+ npages, new->start, new->npages);
	765	+ if (r)
	766	+ return r;
	767	+ }
	768	+
	769	+ return 0;
	770	+}
	771	+
	772	+static int
	773	+svm_range_split_nodes(struct svm_range new, struct svm_range old,
	774	+ uint64_t start, uint64_t last)
	775	+{
	776	+ uint64_t npages = last - start + 1;
	777	+
	778	+ pr_debug("svms 0x%p new prange 0x%p start 0x%lx [0x%llx 0x%llx]\n",
	779	+ new->svms, new, new->start, start, last);
	780	+
	781	+ if (new->start == old->start) {
	782	+ new->offset = old->offset;
	783	+ old->offset += new->npages;
	784	+ } else {
	785	+ new->offset = old->offset + npages;
	786	+ }
	787	+
	788	+ new->svm_bo = svm_range_bo_ref(old->svm_bo);
	789	+ new->ttm_res = old->ttm_res;
	790	+
	791	+ spin_lock(&new->svm_bo->list_lock);
	792	+ list_add(&new->svm_bo_list, &new->svm_bo->range_list);
	793	+ spin_unlock(&new->svm_bo->list_lock);
	794	+
	795	+ return 0;
	796	+}
	797	+
	798	+/**
	799	+ * svm_range_split_adjust - split range and adjust
	800	+ *
	801	+ * @new: new range
	802	+ * @old: the old range
	803	+ * @start: the old range adjust to start address in pages
	804	+ * @last: the old range adjust to last address in pages
	805	+ *
	806	+ * Copy system memory dma_addr or vram ttm_res in old range to new
	807	+ * range from new_start up to size new->npages, the remaining old range is from
	808	+ * start to last
	809	+ *
	810	+ * Return:
	811	+ * 0 - OK, -ENOMEM - out of memory
	812	+ */
	813	+static int
	814	+svm_range_split_adjust(struct svm_range new, struct svm_range old,
	815	+ uint64_t start, uint64_t last)
	816	+{
	817	+ int r;
	818	+
	819	+ pr_debug("svms 0x%p new 0x%lx old [0x%lx 0x%lx] => [0x%llx 0x%llx]\n",
	820	+ new->svms, new->start, old->start, old->last, start, last);
	821	+
	822	+ if (new->start < old->start \|\|
	823	+ new->last > old->last) {
	824	+ WARN_ONCE(1, "invalid new range start or last\n");
	825	+ return -EINVAL;
	826	+ }
	827	+
	828	+ r = svm_range_split_pages(new, old, start, last);
	829	+ if (r)
	830	+ return r;
	831	+
	832	+ if (old->actual_loc && old->ttm_res) {
	833	+ r = svm_range_split_nodes(new, old, start, last);
	834	+ if (r)
	835	+ return r;
	836	+ }
	837	+
	838	+ old->npages = last - start + 1;
	839	+ old->start = start;
	840	+ old->last = last;
	841	+ new->flags = old->flags;
	842	+ new->preferred_loc = old->preferred_loc;
	843	+ new->prefetch_loc = old->prefetch_loc;
	844	+ new->actual_loc = old->actual_loc;
	845	+ new->granularity = old->granularity;
	846	+ bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE);
	847	+ bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE);
	848	+
	849	+ return 0;
	850	+}
	851	+
	852	+/**
	853	+ * svm_range_split - split a range in 2 ranges
	854	+ *
	855	+ * @prange: the svm range to split
	856	+ * @start: the remaining range start address in pages
	857	+ * @last: the remaining range last address in pages
	858	+ * @new: the result new range generated
	859	+ *
	860	+ * Two cases only:
	861	+ * case 1: if start == prange->start
	862	+ * prange ==> prange[start, last]
	863	+ * new range [last + 1, prange->last]
	864	+ *
	865	+ * case 2: if last == prange->last
	866	+ * prange ==> prange[start, last]
	867	+ * new range [prange->start, start - 1]
	868	+ *
	869	+ * Return:
	870	+ * 0 - OK, -ENOMEM - out of memory, -EINVAL - invalid start, last
	871	+ */
	872	+static int
	873	+svm_range_split(struct svm_range *prange, uint64_t start, uint64_t last,
	874	+ struct svm_range **new)
	875	+{
	876	+ uint64_t old_start = prange->start;
	877	+ uint64_t old_last = prange->last;
	878	+ struct svm_range_list *svms;
	879	+ int r = 0;
	880	+
	881	+ pr_debug("svms 0x%p [0x%llx 0x%llx] to [0x%llx 0x%llx]\n", prange->svms,
	882	+ old_start, old_last, start, last);
	883	+
	884	+ if (old_start != start && old_last != last)
	885	+ return -EINVAL;
	886	+ if (start < old_start \|\| last > old_last)
	887	+ return -EINVAL;
	888	+
	889	+ svms = prange->svms;
	890	+ if (old_start == start)
	891	+ *new = svm_range_new(svms, last + 1, old_last);
	892	+ else
	893	+ *new = svm_range_new(svms, old_start, start - 1);
	894	+ if (!*new)
	895	+ return -ENOMEM;
	896	+
	897	+ r = svm_range_split_adjust(*new, prange, start, last);
	898	+ if (r) {
	899	+ pr_debug("failed %d split [0x%llx 0x%llx] to [0x%llx 0x%llx]\n",
	900	+ r, old_start, old_last, start, last);
	901	+ svm_range_free(*new);
	902	+ *new = NULL;
	903	+ }
	904	+
	905	+ return r;
	906	+}
	907	+
	908	+static int
	909	+svm_range_split_tail(struct svm_range prange, struct svm_range new,
	910	+ uint64_t new_last, struct list_head *insert_list)
	911	+{
	912	+ struct svm_range *tail;
	913	+ int r = svm_range_split(prange, prange->start, new_last, &tail);
	914	+
	915	+ if (!r)
	916	+ list_add(&tail->insert_list, insert_list);
	917	+ return r;
	918	+}
	919	+
	920	+static int
	921	+svm_range_split_head(struct svm_range prange, struct svm_range new,
	922	+ uint64_t new_start, struct list_head *insert_list)
	923	+{
	924	+ struct svm_range *head;
	925	+ int r = svm_range_split(prange, new_start, prange->last, &head);
	926	+
	927	+ if (!r)
	928	+ list_add(&head->insert_list, insert_list);
	929	+ return r;
	930	+}
	931	+
	932	+static void
	933	+svm_range_add_child(struct svm_range prange, struct mm_struct mm,
	934	+ struct svm_range *pchild, enum svm_work_list_ops op)
	935	+{
	936	+ pr_debug("add child 0x%p [0x%lx 0x%lx] to prange 0x%p child list %d\n",
	937	+ pchild, pchild->start, pchild->last, prange, op);
	938	+
	939	+ pchild->work_item.mm = mm;
	940	+ pchild->work_item.op = op;
	941	+ list_add_tail(&pchild->child_list, &prange->child_list);
	942	+}
	943	+
	944	+/**
	945	+ * svm_range_split_by_granularity - collect ranges within granularity boundary
	946	+ *
	947	+ * @p: the process with svms list
	948	+ * @mm: mm structure
	949	+ * @addr: the vm fault address in pages, to split the prange
	950	+ * @parent: parent range if prange is from child list
	951	+ * @prange: prange to split
	952	+ *
	953	+ * Trims @prange to be a single aligned block of prange->granularity if
	954	+ * possible. The head and tail are added to the child_list in @parent.
	955	+ *
	956	+ * Context: caller must hold mmap_read_lock and prange->lock
	957	+ *
	958	+ * Return:
	959	+ * 0 - OK, otherwise error code
	960	+ */
	961	+int
	962	+svm_range_split_by_granularity(struct kfd_process p, struct mm_struct mm,
	963	+ unsigned long addr, struct svm_range *parent,
	964	+ struct svm_range *prange)
	965	+{
	966	+ struct svm_range head, tail;
	967	+ unsigned long start, last, size;
	968	+ int r;
	969	+
	970	+ /* Align splited range start and size to granularity size, then a single
	971	+ * PTE will be used for whole range, this reduces the number of PTE
	972	+ * updated and the L1 TLB space used for translation.
	973	+ */
	974	+ size = 1UL << prange->granularity;
	975	+ start = ALIGN_DOWN(addr, size);
	976	+ last = ALIGN(addr + 1, size) - 1;
	977	+
	978	+ pr_debug("svms 0x%p split [0x%lx 0x%lx] to [0x%lx 0x%lx] size 0x%lx\n",
	979	+ prange->svms, prange->start, prange->last, start, last, size);
	980	+
	981	+ if (start > prange->start) {
	982	+ r = svm_range_split(prange, start, prange->last, &head);
	983	+ if (r)
	984	+ return r;
	985	+ svm_range_add_child(parent, mm, head, SVM_OP_ADD_RANGE);
	986	+ }
	987	+
	988	+ if (last < prange->last) {
	989	+ r = svm_range_split(prange, prange->start, last, &tail);
	990	+ if (r)
	991	+ return r;
	992	+ svm_range_add_child(parent, mm, tail, SVM_OP_ADD_RANGE);
	993	+ }
	994	+
	995	+ /* xnack on, update mapping on GPUs with ACCESS_IN_PLACE */
	996	+ if (p->xnack_enabled && prange->work_item.op == SVM_OP_ADD_RANGE) {
	997	+ prange->work_item.op = SVM_OP_ADD_RANGE_AND_MAP;
	998	+ pr_debug("change prange 0x%p [0x%lx 0x%lx] op %d\n",
	999	+ prange, prange->start, prange->last,
	1000	+ SVM_OP_ADD_RANGE_AND_MAP);
	1001	+ }
	1002	+ return 0;
	1003	+}
	1004	+
	1005	+static uint64_t
	1006	+svm_range_get_pte_flags(struct amdgpu_device adev, struct svm_range prange)
	1007	+{
	1008	+ struct amdgpu_device *bo_adev;
	1009	+ uint32_t flags = prange->flags;
	1010	+ uint32_t mapping_flags = 0;
	1011	+ uint64_t pte_flags;
	1012	+ bool snoop = !prange->ttm_res;
	1013	+ bool coherent = flags & KFD_IOCTL_SVM_FLAG_COHERENT;
	1014	+
	1015	+ if (prange->svm_bo && prange->ttm_res)
	1016	+ bo_adev = amdgpu_ttm_adev(prange->svm_bo->bo->tbo.bdev);
	1017	+
	1018	+ switch (adev->asic_type) {
	1019	+ case CHIP_ARCTURUS:
	1020	+ if (prange->svm_bo && prange->ttm_res) {
	1021	+ if (bo_adev == adev) {
	1022	+ mapping_flags \|= coherent ?
	1023	+ AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW;
	1024	+ } else {
	1025	+ mapping_flags \|= AMDGPU_VM_MTYPE_UC;
	1026	+ if (amdgpu_xgmi_same_hive(adev, bo_adev))
	1027	+ snoop = true;
	1028	+ }
	1029	+ } else {
	1030	+ mapping_flags \|= coherent ?
	1031	+ AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
	1032	+ }
	1033	+ break;
	1034	+ case CHIP_ALDEBARAN:
	1035	+ if (prange->svm_bo && prange->ttm_res) {
	1036	+ if (bo_adev == adev) {
	1037	+ mapping_flags \|= coherent ?
	1038	+ AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW;
	1039	+ if (adev->gmc.xgmi.connected_to_cpu)
	1040	+ snoop = true;
	1041	+ } else {
	1042	+ mapping_flags \|= AMDGPU_VM_MTYPE_UC;
	1043	+ if (amdgpu_xgmi_same_hive(adev, bo_adev))
	1044	+ snoop = true;
	1045	+ }
	1046	+ } else {
	1047	+ mapping_flags \|= coherent ?
	1048	+ AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
	1049	+ }
	1050	+ break;
	1051	+ default:
	1052	+ mapping_flags \|= coherent ?
	1053	+ AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC;
	1054	+ }
	1055	+
	1056	+ mapping_flags \|= AMDGPU_VM_PAGE_READABLE \| AMDGPU_VM_PAGE_WRITEABLE;
	1057	+
	1058	+ if (flags & KFD_IOCTL_SVM_FLAG_GPU_RO)
	1059	+ mapping_flags &= ~AMDGPU_VM_PAGE_WRITEABLE;
	1060	+ if (flags & KFD_IOCTL_SVM_FLAG_GPU_EXEC)
	1061	+ mapping_flags \|= AMDGPU_VM_PAGE_EXECUTABLE;
	1062	+
	1063	+ pte_flags = AMDGPU_PTE_VALID;
	1064	+ pte_flags \|= prange->ttm_res ? 0 : AMDGPU_PTE_SYSTEM;
	1065	+ pte_flags \|= snoop ? AMDGPU_PTE_SNOOPED : 0;
	1066	+
	1067	+ pte_flags \|= amdgpu_gem_va_map_flags(adev, mapping_flags);
	1068	+
	1069	+ pr_debug("svms 0x%p [0x%lx 0x%lx] vram %d PTE 0x%llx mapping 0x%x\n",
	1070	+ prange->svms, prange->start, prange->last,
	1071	+ prange->ttm_res ? 1:0, pte_flags, mapping_flags);
	1072	+
	1073	+ return pte_flags;
	1074	+}
	1075	+
	1076	+static int
	1077	+svm_range_unmap_from_gpu(struct amdgpu_device adev, struct amdgpu_vm vm,
	1078	+ uint64_t start, uint64_t last,
	1079	+ struct dma_fence **fence)
	1080	+{
	1081	+ uint64_t init_pte_value = 0;
	1082	+
	1083	+ pr_debug("[0x%llx 0x%llx]\n", start, last);
	1084	+
	1085	+ return amdgpu_vm_bo_update_mapping(adev, adev, vm, false, true, NULL,
	1086	+ start, last, init_pte_value, 0,
	1087	+ NULL, NULL, fence);
	1088	+}
	1089	+
	1090	+static int
	1091	+svm_range_unmap_from_gpus(struct svm_range *prange, unsigned long start,
	1092	+ unsigned long last)
	1093	+{
	1094	+ DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);
	1095	+ struct kfd_process_device *pdd;
	1096	+ struct dma_fence *fence = NULL;
	1097	+ struct amdgpu_device *adev;
	1098	+ struct kfd_process *p;
	1099	+ uint32_t gpuidx;
	1100	+ int r = 0;
	1101	+
	1102	+ bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip,
	1103	+ MAX_GPU_INSTANCE);
	1104	+ p = container_of(prange->svms, struct kfd_process, svms);
	1105	+
	1106	+ for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
	1107	+ pr_debug("unmap from gpu idx 0x%x\n", gpuidx);
	1108	+ pdd = kfd_process_device_from_gpuidx(p, gpuidx);
	1109	+ if (!pdd) {
	1110	+ pr_debug("failed to find device idx %d\n", gpuidx);
	1111	+ return -EINVAL;
	1112	+ }
	1113	+ adev = (struct amdgpu_device *)pdd->dev->kgd;
	1114	+
	1115	+ r = svm_range_unmap_from_gpu(adev, drm_priv_to_vm(pdd->drm_priv),
	1116	+ start, last, &fence);
	1117	+ if (r)
	1118	+ break;
	1119	+
	1120	+ if (fence) {
	1121	+ r = dma_fence_wait(fence, false);
	1122	+ dma_fence_put(fence);
	1123	+ fence = NULL;
	1124	+ if (r)
	1125	+ break;
	1126	+ }
	1127	+ amdgpu_amdkfd_flush_gpu_tlb_pasid((struct kgd_dev *)adev,
	1128	+ p->pasid);
	1129	+ }
	1130	+
	1131	+ return r;
	1132	+}
	1133	+
	1134	+static int
	1135	+svm_range_map_to_gpu(struct amdgpu_device adev, struct amdgpu_vm vm,
	1136	+ struct svm_range prange, dma_addr_t dma_addr,
	1137	+ struct amdgpu_device bo_adev, struct dma_fence *fence)
	1138	+{
	1139	+ struct amdgpu_bo_va bo_va;
	1140	+ uint64_t pte_flags;
	1141	+ int r = 0;
	1142	+
	1143	+ pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, prange->start,
	1144	+ prange->last);
	1145	+
	1146	+ if (prange->svm_bo && prange->ttm_res) {
	1147	+ bo_va.is_xgmi = amdgpu_xgmi_same_hive(adev, bo_adev);
	1148	+ prange->mapping.bo_va = &bo_va;
	1149	+ }
	1150	+
	1151	+ prange->mapping.start = prange->start;
	1152	+ prange->mapping.last = prange->last;
	1153	+ prange->mapping.offset = prange->offset;
	1154	+ pte_flags = svm_range_get_pte_flags(adev, prange);
	1155	+
	1156	+ r = amdgpu_vm_bo_update_mapping(adev, bo_adev, vm, false, false, NULL,
	1157	+ prange->mapping.start,
	1158	+ prange->mapping.last, pte_flags,
	1159	+ prange->mapping.offset,
	1160	+ prange->ttm_res ?
	1161	+ prange->ttm_res->mm_node : NULL,
	1162	+ dma_addr, &vm->last_update);
	1163	+ if (r) {
	1164	+ pr_debug("failed %d to map to gpu 0x%lx\n", r, prange->start);
	1165	+ goto out;
	1166	+ }
	1167	+
	1168	+ r = amdgpu_vm_update_pdes(adev, vm, false);
	1169	+ if (r) {
	1170	+ pr_debug("failed %d to update directories 0x%lx\n", r,
	1171	+ prange->start);
	1172	+ goto out;
	1173	+ }
	1174	+
	1175	+ if (fence)
	1176	+ *fence = dma_fence_get(vm->last_update);
	1177	+
	1178	+out:
	1179	+ prange->mapping.bo_va = NULL;
	1180	+ return r;
	1181	+}
	1182	+
	1183	+static int svm_range_map_to_gpus(struct svm_range *prange,
	1184	+ unsigned long *bitmap, bool wait)
	1185	+{
	1186	+ struct kfd_process_device *pdd;
	1187	+ struct amdgpu_device *bo_adev;
	1188	+ struct amdgpu_device *adev;
	1189	+ struct kfd_process *p;
	1190	+ struct dma_fence *fence = NULL;
	1191	+ uint32_t gpuidx;
	1192	+ int r = 0;
	1193	+
	1194	+ if (prange->svm_bo && prange->ttm_res)
	1195	+ bo_adev = amdgpu_ttm_adev(prange->svm_bo->bo->tbo.bdev);
	1196	+ else
	1197	+ bo_adev = NULL;
	1198	+
	1199	+ p = container_of(prange->svms, struct kfd_process, svms);
	1200	+ for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
	1201	+ pr_debug("mapping to gpu idx 0x%x\n", gpuidx);
	1202	+ pdd = kfd_process_device_from_gpuidx(p, gpuidx);
	1203	+ if (!pdd) {
	1204	+ pr_debug("failed to find device idx %d\n", gpuidx);
	1205	+ return -EINVAL;
	1206	+ }
	1207	+ adev = (struct amdgpu_device *)pdd->dev->kgd;
	1208	+
	1209	+ pdd = kfd_bind_process_to_device(pdd->dev, p);
	1210	+ if (IS_ERR(pdd))
	1211	+ return -EINVAL;
	1212	+
	1213	+ if (bo_adev && adev != bo_adev &&
	1214	+ !amdgpu_xgmi_same_hive(adev, bo_adev)) {
	1215	+ pr_debug("cannot map to device idx %d\n", gpuidx);
	1216	+ continue;
	1217	+ }
	1218	+
	1219	+ r = svm_range_map_to_gpu(adev, drm_priv_to_vm(pdd->drm_priv),
	1220	+ prange, prange->dma_addr[gpuidx],
	1221	+ bo_adev, wait ? &fence : NULL);
	1222	+ if (r)
	1223	+ break;
	1224	+
	1225	+ if (fence) {
	1226	+ r = dma_fence_wait(fence, false);
	1227	+ dma_fence_put(fence);
	1228	+ fence = NULL;
	1229	+ if (r) {
	1230	+ pr_debug("failed %d to dma fence wait\n", r);
	1231	+ break;
	1232	+ }
	1233	+ }
	1234	+
	1235	+ amdgpu_amdkfd_flush_gpu_tlb_pasid((struct kgd_dev *)adev,
	1236	+ p->pasid);
	1237	+ }
	1238	+
	1239	+ return r;
	1240	+}
	1241	+
	1242	+struct svm_validate_context {
	1243	+ struct kfd_process *process;
	1244	+ struct svm_range *prange;
	1245	+ bool intr;
	1246	+ unsigned long bitmap[MAX_GPU_INSTANCE];
	1247	+ struct ttm_validate_buffer tv[MAX_GPU_INSTANCE+1];
	1248	+ struct list_head validate_list;
	1249	+ struct ww_acquire_ctx ticket;
	1250	+};
	1251	+
	1252	+static int svm_range_reserve_bos(struct svm_validate_context *ctx)
	1253	+{
	1254	+ struct kfd_process_device *pdd;
	1255	+ struct amdgpu_device *adev;
	1256	+ struct amdgpu_vm *vm;
	1257	+ uint32_t gpuidx;
	1258	+ int r;
	1259	+
	1260	+ INIT_LIST_HEAD(&ctx->validate_list);
	1261	+ for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) {
	1262	+ pdd = kfd_process_device_from_gpuidx(ctx->process, gpuidx);
	1263	+ if (!pdd) {
	1264	+ pr_debug("failed to find device idx %d\n", gpuidx);
	1265	+ return -EINVAL;
	1266	+ }
	1267	+ adev = (struct amdgpu_device *)pdd->dev->kgd;
	1268	+ vm = drm_priv_to_vm(pdd->drm_priv);
	1269	+
	1270	+ ctx->tv[gpuidx].bo = &vm->root.base.bo->tbo;
	1271	+ ctx->tv[gpuidx].num_shared = 4;
	1272	+ list_add(&ctx->tv[gpuidx].head, &ctx->validate_list);
	1273	+ }
	1274	+ if (ctx->prange->svm_bo && ctx->prange->ttm_res) {
	1275	+ ctx->tv[MAX_GPU_INSTANCE].bo = &ctx->prange->svm_bo->bo->tbo;
	1276	+ ctx->tv[MAX_GPU_INSTANCE].num_shared = 1;
	1277	+ list_add(&ctx->tv[MAX_GPU_INSTANCE].head, &ctx->validate_list);
	1278	+ }
	1279	+
	1280	+ r = ttm_eu_reserve_buffers(&ctx->ticket, &ctx->validate_list,
	1281	+ ctx->intr, NULL);
	1282	+ if (r) {
	1283	+ pr_debug("failed %d to reserve bo\n", r);
	1284	+ return r;
	1285	+ }
	1286	+
	1287	+ for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) {
	1288	+ pdd = kfd_process_device_from_gpuidx(ctx->process, gpuidx);
	1289	+ if (!pdd) {
	1290	+ pr_debug("failed to find device idx %d\n", gpuidx);
	1291	+ r = -EINVAL;
	1292	+ goto unreserve_out;
	1293	+ }
	1294	+ adev = (struct amdgpu_device *)pdd->dev->kgd;
	1295	+
	1296	+ r = amdgpu_vm_validate_pt_bos(adev, drm_priv_to_vm(pdd->drm_priv),
	1297	+ svm_range_bo_validate, NULL);
	1298	+ if (r) {
	1299	+ pr_debug("failed %d validate pt bos\n", r);
	1300	+ goto unreserve_out;
	1301	+ }
	1302	+ }
	1303	+
	1304	+ return 0;
	1305	+
	1306	+unreserve_out:
	1307	+ ttm_eu_backoff_reservation(&ctx->ticket, &ctx->validate_list);
	1308	+ return r;
	1309	+}
	1310	+
	1311	+static void svm_range_unreserve_bos(struct svm_validate_context *ctx)
	1312	+{
	1313	+ ttm_eu_backoff_reservation(&ctx->ticket, &ctx->validate_list);
	1314	+}
	1315	+
	1316	+/*
	1317	+ * Validation+GPU mapping with concurrent invalidation (MMU notifiers)
	1318	+ *
	1319	+ * To prevent concurrent destruction or change of range attributes, the
	1320	+ * svm_read_lock must be held. The caller must not hold the svm_write_lock
	1321	+ * because that would block concurrent evictions and lead to deadlocks. To
	1322	+ * serialize concurrent migrations or validations of the same range, the
	1323	+ * prange->migrate_mutex must be held.
	1324	+ *
	1325	+ * For VRAM ranges, the SVM BO must be allocated and valid (protected by its
	1326	+ * eviction fence.
	1327	+ *
	1328	+ * The following sequence ensures race-free validation and GPU mapping:
	1329	+ *
	1330	+ * 1. Reserve page table (and SVM BO if range is in VRAM)
	1331	+ * 2. hmm_range_fault to get page addresses (if system memory)
	1332	+ * 3. DMA-map pages (if system memory)
	1333	+ * 4-a. Take notifier lock
	1334	+ * 4-b. Check that pages still valid (mmu_interval_read_retry)
	1335	+ * 4-c. Check that the range was not split or otherwise invalidated
	1336	+ * 4-d. Update GPU page table
	1337	+ * 4.e. Release notifier lock
	1338	+ * 5. Release page table (and SVM BO) reservation
	1339	+ */
	1340	+static int svm_range_validate_and_map(struct mm_struct *mm,
	1341	+ struct svm_range *prange,
	1342	+ int32_t gpuidx, bool intr, bool wait)
	1343	+{
	1344	+ struct svm_validate_context ctx;
	1345	+ struct hmm_range *hmm_range;
	1346	+ int r = 0;
	1347	+
	1348	+ ctx.process = container_of(prange->svms, struct kfd_process, svms);
	1349	+ ctx.prange = prange;
	1350	+ ctx.intr = intr;
	1351	+
	1352	+ if (gpuidx < MAX_GPU_INSTANCE) {
	1353	+ bitmap_zero(ctx.bitmap, MAX_GPU_INSTANCE);
	1354	+ bitmap_set(ctx.bitmap, gpuidx, 1);
	1355	+ } else if (ctx.process->xnack_enabled) {
	1356	+ bitmap_copy(ctx.bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE);
	1357	+
	1358	+ /* If prefetch range to GPU, or GPU retry fault migrate range to
	1359	+ * GPU, which has ACCESS attribute to the range, create mapping
	1360	+ * on that GPU.
	1361	+ */
	1362	+ if (prange->actual_loc) {
	1363	+ gpuidx = kfd_process_gpuidx_from_gpuid(ctx.process,
	1364	+ prange->actual_loc);
	1365	+ if (gpuidx < 0) {
	1366	+ WARN_ONCE(1, "failed get device by id 0x%x\n",
	1367	+ prange->actual_loc);
	1368	+ return -EINVAL;
	1369	+ }
	1370	+ if (test_bit(gpuidx, prange->bitmap_access))
	1371	+ bitmap_set(ctx.bitmap, gpuidx, 1);
	1372	+ }
	1373	+ } else {
	1374	+ bitmap_or(ctx.bitmap, prange->bitmap_access,
	1375	+ prange->bitmap_aip, MAX_GPU_INSTANCE);
	1376	+ }
	1377	+
	1378	+ if (bitmap_empty(ctx.bitmap, MAX_GPU_INSTANCE))
	1379	+ return 0;
	1380	+
	1381	+ if (prange->actual_loc && !prange->ttm_res) {
	1382	+ /* This should never happen. actual_loc gets set by
	1383	+ * svm_migrate_ram_to_vram after allocating a BO.
	1384	+ */
	1385	+ WARN(1, "VRAM BO missing during validation\n");
	1386	+ return -EINVAL;
	1387	+ }
	1388	+
	1389	+ svm_range_reserve_bos(&ctx);
	1390	+
	1391	+ if (!prange->actual_loc) {
	1392	+ r = amdgpu_hmm_range_get_pages(&prange->notifier, mm, NULL,
	1393	+ prange->start << PAGE_SHIFT,
	1394	+ prange->npages, &hmm_range,
	1395	+ false, true);
	1396	+ if (r) {
	1397	+ pr_debug("failed %d to get svm range pages\n", r);
	1398	+ goto unreserve_out;
	1399	+ }
	1400	+
	1401	+ r = svm_range_dma_map(prange, ctx.bitmap,
	1402	+ hmm_range->hmm_pfns);
	1403	+ if (r) {
	1404	+ pr_debug("failed %d to dma map range\n", r);
	1405	+ goto unreserve_out;
	1406	+ }
	1407	+
	1408	+ prange->validated_once = true;
	1409	+ }
	1410	+
	1411	+ svm_range_lock(prange);
	1412	+ if (!prange->actual_loc) {
	1413	+ if (amdgpu_hmm_range_get_pages_done(hmm_range)) {
	1414	+ pr_debug("hmm update the range, need validate again\n");
	1415	+ r = -EAGAIN;
	1416	+ goto unlock_out;
	1417	+ }
	1418	+ }
	1419	+ if (!list_empty(&prange->child_list)) {
	1420	+ pr_debug("range split by unmap in parallel, validate again\n");
	1421	+ r = -EAGAIN;
	1422	+ goto unlock_out;
	1423	+ }
	1424	+
	1425	+ r = svm_range_map_to_gpus(prange, ctx.bitmap, wait);
	1426	+
	1427	+unlock_out:
	1428	+ svm_range_unlock(prange);
	1429	+unreserve_out:
	1430	+ svm_range_unreserve_bos(&ctx);
	1431	+
	1432	+ if (!r)
	1433	+ prange->validate_timestamp = ktime_to_us(ktime_get());
	1434	+
	1435	+ return r;
	1436	+}
	1437	+
	1438	+/**
	1439	+ * svm_range_list_lock_and_flush_work - flush pending deferred work
	1440	+ *
	1441	+ * @svms: the svm range list
	1442	+ * @mm: the mm structure
	1443	+ *
	1444	+ * Context: Returns with mmap write lock held, pending deferred work flushed
	1445	+ *
	1446	+ */
	1447	+static void
	1448	+svm_range_list_lock_and_flush_work(struct svm_range_list *svms,
	1449	+ struct mm_struct *mm)
	1450	+{
	1451	+retry_flush_work:
	1452	+ flush_work(&svms->deferred_list_work);
	1453	+ mmap_write_lock(mm);
	1454	+
	1455	+ if (list_empty(&svms->deferred_range_list))
	1456	+ return;
	1457	+ mmap_write_unlock(mm);
	1458	+ pr_debug("retry flush\n");
	1459	+ goto retry_flush_work;
	1460	+}
	1461	+
	1462	+static void svm_range_restore_work(struct work_struct *work)
	1463	+{
	1464	+ struct delayed_work *dwork = to_delayed_work(work);
	1465	+ struct amdkfd_process_info *process_info;
	1466	+ struct svm_range_list *svms;
	1467	+ struct svm_range *prange;
	1468	+ struct kfd_process *p;
	1469	+ struct mm_struct *mm;
	1470	+ int evicted_ranges;
	1471	+ int invalid;
	1472	+ int r;
	1473	+
	1474	+ svms = container_of(dwork, struct svm_range_list, restore_work);
	1475	+ evicted_ranges = atomic_read(&svms->evicted_ranges);
	1476	+ if (!evicted_ranges)
	1477	+ return;
	1478	+
	1479	+ pr_debug("restore svm ranges\n");
	1480	+
	1481	+ /* kfd_process_notifier_release destroys this worker thread. So during
	1482	+ * the lifetime of this thread, kfd_process and mm will be valid.
	1483	+ */
	1484	+ p = container_of(svms, struct kfd_process, svms);
	1485	+ process_info = p->kgd_process_info;
	1486	+ mm = p->mm;
	1487	+ if (!mm)
	1488	+ return;
	1489	+
	1490	+ mutex_lock(&process_info->lock);
	1491	+ svm_range_list_lock_and_flush_work(svms, mm);
	1492	+ mutex_lock(&svms->lock);
	1493	+
	1494	+ evicted_ranges = atomic_read(&svms->evicted_ranges);
	1495	+
	1496	+ list_for_each_entry(prange, &svms->list, list) {
	1497	+ invalid = atomic_read(&prange->invalid);
	1498	+ if (!invalid)
	1499	+ continue;
	1500	+
	1501	+ pr_debug("restoring svms 0x%p prange 0x%p [0x%lx %lx] inv %d\n",
	1502	+ prange->svms, prange, prange->start, prange->last,
	1503	+ invalid);
	1504	+
	1505	+ /*
	1506	+ * If range is migrating, wait for migration is done.
	1507	+ */
	1508	+ mutex_lock(&prange->migrate_mutex);
	1509	+
	1510	+ r = svm_range_validate_and_map(mm, prange, MAX_GPU_INSTANCE,
	1511	+ false, true);
	1512	+ if (r)
	1513	+ pr_debug("failed %d to map 0x%lx to gpus\n", r,
	1514	+ prange->start);
	1515	+
	1516	+ mutex_unlock(&prange->migrate_mutex);
	1517	+ if (r)
	1518	+ goto out_reschedule;
	1519	+
	1520	+ if (atomic_cmpxchg(&prange->invalid, invalid, 0) != invalid)
	1521	+ goto out_reschedule;
	1522	+ }
	1523	+
	1524	+ if (atomic_cmpxchg(&svms->evicted_ranges, evicted_ranges, 0) !=
	1525	+ evicted_ranges)
	1526	+ goto out_reschedule;
	1527	+
	1528	+ evicted_ranges = 0;
	1529	+
	1530	+ r = kgd2kfd_resume_mm(mm);
	1531	+ if (r) {
	1532	+ /* No recovery from this failure. Probably the CP is
	1533	+ * hanging. No point trying again.
	1534	+ */
	1535	+ pr_debug("failed %d to resume KFD\n", r);
	1536	+ }
	1537	+
	1538	+ pr_debug("restore svm ranges successfully\n");
	1539	+
	1540	+out_reschedule:
	1541	+ mutex_unlock(&svms->lock);
	1542	+ mmap_write_unlock(mm);
	1543	+ mutex_unlock(&process_info->lock);
	1544	+
	1545	+ /* If validation failed, reschedule another attempt */
	1546	+ if (evicted_ranges) {
	1547	+ pr_debug("reschedule to restore svm range\n");
	1548	+ schedule_delayed_work(&svms->restore_work,
	1549	+ msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS));
	1550	+ }
	1551	+}
	1552	+
	1553	+/**
	1554	+ * svm_range_evict - evict svm range
	1555	+ *
	1556	+ * Stop all queues of the process to ensure GPU doesn't access the memory, then
	1557	+ * return to let CPU evict the buffer and proceed CPU pagetable update.
	1558	+ *
	1559	+ * Don't need use lock to sync cpu pagetable invalidation with GPU execution.
	1560	+ * If invalidation happens while restore work is running, restore work will
	1561	+ * restart to ensure to get the latest CPU pages mapping to GPU, then start
	1562	+ * the queues.
	1563	+ */
	1564	+static int
	1565	+svm_range_evict(struct svm_range prange, struct mm_struct mm,
	1566	+ unsigned long start, unsigned long last)
	1567	+{
	1568	+ struct svm_range_list *svms = prange->svms;
	1569	+ struct kfd_process *p;
	1570	+ int r = 0;
	1571	+
	1572	+ p = container_of(svms, struct kfd_process, svms);
	1573	+
	1574	+ pr_debug("invalidate svms 0x%p prange [0x%lx 0x%lx] [0x%lx 0x%lx]\n",
	1575	+ svms, prange->start, prange->last, start, last);
	1576	+
	1577	+ if (!p->xnack_enabled) {
	1578	+ int evicted_ranges;
	1579	+
	1580	+ atomic_inc(&prange->invalid);
	1581	+ evicted_ranges = atomic_inc_return(&svms->evicted_ranges);
	1582	+ if (evicted_ranges != 1)
	1583	+ return r;
	1584	+
	1585	+ pr_debug("evicting svms 0x%p range [0x%lx 0x%lx]\n",
	1586	+ prange->svms, prange->start, prange->last);
	1587	+
	1588	+ /* First eviction, stop the queues */
	1589	+ r = kgd2kfd_quiesce_mm(mm);
	1590	+ if (r)
	1591	+ pr_debug("failed to quiesce KFD\n");
	1592	+
	1593	+ pr_debug("schedule to restore svm %p ranges\n", svms);
	1594	+ schedule_delayed_work(&svms->restore_work,
	1595	+ msecs_to_jiffies(AMDGPU_SVM_RANGE_RESTORE_DELAY_MS));
	1596	+ } else {
	1597	+ struct svm_range *pchild;
	1598	+ unsigned long s, l;
	1599	+
	1600	+ pr_debug("invalidate unmap svms 0x%p [0x%lx 0x%lx] from GPUs\n",
	1601	+ prange->svms, start, last);
	1602	+ list_for_each_entry(pchild, &prange->child_list, child_list) {
	1603	+ mutex_lock_nested(&pchild->lock, 1);
	1604	+ s = max(start, pchild->start);
	1605	+ l = min(last, pchild->last);
	1606	+ if (l >= s)
	1607	+ svm_range_unmap_from_gpus(pchild, s, l);
	1608	+ mutex_unlock(&pchild->lock);
	1609	+ }
	1610	+ s = max(start, prange->start);
	1611	+ l = min(last, prange->last);
	1612	+ if (l >= s)
	1613	+ svm_range_unmap_from_gpus(prange, s, l);
	1614	+ }
	1615	+
	1616	+ return r;
	1617	+}
	1618	+
	1619	+static struct svm_range svm_range_clone(struct svm_range old)
	1620	+{
	1621	+ struct svm_range *new;
	1622	+
	1623	+ new = svm_range_new(old->svms, old->start, old->last);
	1624	+ if (!new)
	1625	+ return NULL;
	1626	+
	1627	+ if (old->svm_bo) {
	1628	+ new->ttm_res = old->ttm_res;
	1629	+ new->offset = old->offset;
	1630	+ new->svm_bo = svm_range_bo_ref(old->svm_bo);
	1631	+ spin_lock(&new->svm_bo->list_lock);
	1632	+ list_add(&new->svm_bo_list, &new->svm_bo->range_list);
	1633	+ spin_unlock(&new->svm_bo->list_lock);
	1634	+ }
	1635	+ new->flags = old->flags;
	1636	+ new->preferred_loc = old->preferred_loc;
	1637	+ new->prefetch_loc = old->prefetch_loc;
	1638	+ new->actual_loc = old->actual_loc;
	1639	+ new->granularity = old->granularity;
	1640	+ bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE);
	1641	+ bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE);
	1642	+
	1643	+ return new;
	1644	+}
	1645	+
	1646	+/**
	1647	+ * svm_range_handle_overlap - split overlap ranges
	1648	+ * @svms: svm range list header
	1649	+ * @new: range added with this attributes
	1650	+ * @start: range added start address, in pages
	1651	+ * @last: range last address, in pages
	1652	+ * @update_list: output, the ranges attributes are updated. For set_attr, this
	1653	+ * will do validation and map to GPUs. For unmap, this will be
	1654	+ * removed and unmap from GPUs
	1655	+ * @insert_list: output, the ranges will be inserted into svms, attributes are
	1656	+ * not changes. For set_attr, this will add into svms.
	1657	+ * @remove_list:output, the ranges will be removed from svms
	1658	+ * @left: the remaining range after overlap, For set_attr, this will be added
	1659	+ * as new range.
	1660	+ *
	1661	+ * Total have 5 overlap cases.
	1662	+ *
	1663	+ * This function handles overlap of an address interval with existing
	1664	+ * struct svm_ranges for applying new attributes. This may require
	1665	+ * splitting existing struct svm_ranges. All changes should be applied to
	1666	+ * the range_list and interval tree transactionally. If any split operation
	1667	+ * fails, the entire update fails. Therefore the existing overlapping
	1668	+ * svm_ranges are cloned and the original svm_ranges left unchanged. If the
	1669	+ * transaction succeeds, the modified clones are added and the originals
	1670	+ * freed. Otherwise the clones are removed and the old svm_ranges remain.
	1671	+ *
	1672	+ * Context: The caller must hold svms->lock
	1673	+ */
	1674	+static int
	1675	+svm_range_handle_overlap(struct svm_range_list svms, struct svm_range new,
	1676	+ unsigned long start, unsigned long last,
	1677	+ struct list_head *update_list,
	1678	+ struct list_head *insert_list,
	1679	+ struct list_head *remove_list,
	1680	+ unsigned long *left)
	1681	+{
	1682	+ struct interval_tree_node *node;
	1683	+ struct svm_range *prange;
	1684	+ struct svm_range *tmp;
	1685	+ int r = 0;
	1686	+
	1687	+ INIT_LIST_HEAD(update_list);
	1688	+ INIT_LIST_HEAD(insert_list);
	1689	+ INIT_LIST_HEAD(remove_list);
	1690	+
	1691	+ node = interval_tree_iter_first(&svms->objects, start, last);
	1692	+ while (node) {
	1693	+ struct interval_tree_node *next;
	1694	+ struct svm_range *old;
	1695	+ unsigned long next_start;
	1696	+
	1697	+ pr_debug("found overlap node [0x%lx 0x%lx]\n", node->start,
	1698	+ node->last);
	1699	+
	1700	+ old = container_of(node, struct svm_range, it_node);
	1701	+ next = interval_tree_iter_next(node, start, last);
	1702	+ next_start = min(node->last, last) + 1;
	1703	+
	1704	+ if (node->start < start \|\| node->last > last) {
	1705	+ /* node intersects the updated range, clone+split it */
	1706	+ prange = svm_range_clone(old);
	1707	+ if (!prange) {
	1708	+ r = -ENOMEM;
	1709	+ goto out;
	1710	+ }
	1711	+
	1712	+ list_add(&old->remove_list, remove_list);
	1713	+ list_add(&prange->insert_list, insert_list);
	1714	+
	1715	+ if (node->start < start) {
	1716	+ pr_debug("change old range start\n");
	1717	+ r = svm_range_split_head(prange, new, start,
	1718	+ insert_list);
	1719	+ if (r)
	1720	+ goto out;
	1721	+ }
	1722	+ if (node->last > last) {
	1723	+ pr_debug("change old range last\n");
	1724	+ r = svm_range_split_tail(prange, new, last,
	1725	+ insert_list);
	1726	+ if (r)
	1727	+ goto out;
	1728	+ }
	1729	+ } else {
	1730	+ /* The node is contained within start..last,
	1731	+ * just update it
	1732	+ */
	1733	+ prange = old;
	1734	+ }
	1735	+
	1736	+ if (!svm_range_is_same_attrs(prange, new))
	1737	+ list_add(&prange->update_list, update_list);
	1738	+
	1739	+ /* insert a new node if needed */
	1740	+ if (node->start > start) {
	1741	+ prange = svm_range_new(prange->svms, start,
	1742	+ node->start - 1);
	1743	+ if (!prange) {
	1744	+ r = -ENOMEM;
	1745	+ goto out;
	1746	+ }
	1747	+
	1748	+ list_add(&prange->insert_list, insert_list);
	1749	+ list_add(&prange->update_list, update_list);
	1750	+ }
	1751	+
	1752	+ node = next;
	1753	+ start = next_start;
	1754	+ }
	1755	+
	1756	+ if (left && start <= last)
	1757	+ *left = last - start + 1;
	1758	+
	1759	+out:
	1760	+ if (r)
	1761	+ list_for_each_entry_safe(prange, tmp, insert_list, insert_list)
	1762	+ svm_range_free(prange);
	1763	+
	1764	+ return r;
	1765	+}
	1766	+
	1767	+static void
	1768	+svm_range_update_notifier_and_interval_tree(struct mm_struct *mm,
	1769	+ struct svm_range *prange)
	1770	+{
	1771	+ unsigned long start;
	1772	+ unsigned long last;
	1773	+
	1774	+ start = prange->notifier.interval_tree.start >> PAGE_SHIFT;
	1775	+ last = prange->notifier.interval_tree.last >> PAGE_SHIFT;
	1776	+
	1777	+ if (prange->start == start && prange->last == last)
	1778	+ return;
	1779	+
	1780	+ pr_debug("up notifier 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n",
	1781	+ prange->svms, prange, start, last, prange->start,
	1782	+ prange->last);
	1783	+
	1784	+ if (start != 0 && last != 0) {
	1785	+ interval_tree_remove(&prange->it_node, &prange->svms->objects);
	1786	+ svm_range_remove_notifier(prange);
	1787	+ }
	1788	+ prange->it_node.start = prange->start;
	1789	+ prange->it_node.last = prange->last;
	1790	+
	1791	+ interval_tree_insert(&prange->it_node, &prange->svms->objects);
	1792	+ svm_range_add_notifier_locked(mm, prange);
	1793	+}
	1794	+
	1795	+static void
	1796	+svm_range_handle_list_op(struct svm_range_list svms, struct svm_range prange)
	1797	+{
	1798	+ struct mm_struct *mm = prange->work_item.mm;
	1799	+
	1800	+ switch (prange->work_item.op) {
	1801	+ case SVM_OP_NULL:
	1802	+ pr_debug("NULL OP 0x%p prange 0x%p [0x%lx 0x%lx]\n",
	1803	+ svms, prange, prange->start, prange->last);
	1804	+ break;
	1805	+ case SVM_OP_UNMAP_RANGE:
	1806	+ pr_debug("remove 0x%p prange 0x%p [0x%lx 0x%lx]\n",
	1807	+ svms, prange, prange->start, prange->last);
	1808	+ svm_range_unlink(prange);
	1809	+ svm_range_remove_notifier(prange);
	1810	+ svm_range_free(prange);
	1811	+ break;
	1812	+ case SVM_OP_UPDATE_RANGE_NOTIFIER:
	1813	+ pr_debug("update notifier 0x%p prange 0x%p [0x%lx 0x%lx]\n",
	1814	+ svms, prange, prange->start, prange->last);
	1815	+ svm_range_update_notifier_and_interval_tree(mm, prange);
	1816	+ break;
	1817	+ case SVM_OP_UPDATE_RANGE_NOTIFIER_AND_MAP:
	1818	+ pr_debug("update and map 0x%p prange 0x%p [0x%lx 0x%lx]\n",
	1819	+ svms, prange, prange->start, prange->last);
	1820	+ svm_range_update_notifier_and_interval_tree(mm, prange);
	1821	+ /* TODO: implement deferred validation and mapping */
	1822	+ break;
	1823	+ case SVM_OP_ADD_RANGE:
	1824	+ pr_debug("add 0x%p prange 0x%p [0x%lx 0x%lx]\n", svms, prange,
	1825	+ prange->start, prange->last);
	1826	+ svm_range_add_to_svms(prange);
	1827	+ svm_range_add_notifier_locked(mm, prange);
	1828	+ break;
	1829	+ case SVM_OP_ADD_RANGE_AND_MAP:
	1830	+ pr_debug("add and map 0x%p prange 0x%p [0x%lx 0x%lx]\n", svms,
	1831	+ prange, prange->start, prange->last);
	1832	+ svm_range_add_to_svms(prange);
	1833	+ svm_range_add_notifier_locked(mm, prange);
	1834	+ /* TODO: implement deferred validation and mapping */
	1835	+ break;
	1836	+ default:
	1837	+ WARN_ONCE(1, "Unknown prange 0x%p work op %d\n", prange,
	1838	+ prange->work_item.op);
	1839	+ }
	1840	+}
	1841	+
	1842	+static void svm_range_drain_retry_fault(struct svm_range_list *svms)
	1843	+{
	1844	+ struct kfd_process_device *pdd;
	1845	+ struct amdgpu_device *adev;
	1846	+ struct kfd_process *p;
	1847	+ uint32_t i;
	1848	+
	1849	+ p = container_of(svms, struct kfd_process, svms);
	1850	+
	1851	+ for (i = 0; i < p->n_pdds; i++) {
	1852	+ pdd = p->pdds[i];
	1853	+ if (!pdd)
	1854	+ continue;
	1855	+
	1856	+ pr_debug("drain retry fault gpu %d svms %p\n", i, svms);
	1857	+ adev = (struct amdgpu_device *)pdd->dev->kgd;
	1858	+
	1859	+ amdgpu_ih_wait_on_checkpoint_process(adev, &adev->irq.ih1);
	1860	+ pr_debug("drain retry fault gpu %d svms 0x%p done\n", i, svms);
	1861	+ }
	1862	+}
	1863	+
	1864	+static void svm_range_deferred_list_work(struct work_struct *work)
	1865	+{
	1866	+ struct svm_range_list *svms;
	1867	+ struct svm_range *prange;
	1868	+ struct mm_struct *mm;
	1869	+
	1870	+ svms = container_of(work, struct svm_range_list, deferred_list_work);
	1871	+ pr_debug("enter svms 0x%p\n", svms);
	1872	+
	1873	+ spin_lock(&svms->deferred_list_lock);
	1874	+ while (!list_empty(&svms->deferred_range_list)) {
	1875	+ prange = list_first_entry(&svms->deferred_range_list,
	1876	+ struct svm_range, deferred_list);
	1877	+ spin_unlock(&svms->deferred_list_lock);
	1878	+ pr_debug("prange 0x%p [0x%lx 0x%lx] op %d\n", prange,
	1879	+ prange->start, prange->last, prange->work_item.op);
	1880	+
	1881	+ /* Make sure no stale retry fault coming after range is freed */
	1882	+ if (prange->work_item.op == SVM_OP_UNMAP_RANGE)
	1883	+ svm_range_drain_retry_fault(prange->svms);
	1884	+
	1885	+ mm = prange->work_item.mm;
	1886	+ mmap_write_lock(mm);
	1887	+ mutex_lock(&svms->lock);
	1888	+
	1889	+ /* Remove from deferred_list must be inside mmap write lock,
	1890	+ * otherwise, svm_range_list_lock_and_flush_work may hold mmap
	1891	+ * write lock, and continue because deferred_list is empty, then
	1892	+ * deferred_list handle is blocked by mmap write lock.
	1893	+ */
	1894	+ spin_lock(&svms->deferred_list_lock);
	1895	+ list_del_init(&prange->deferred_list);
	1896	+ spin_unlock(&svms->deferred_list_lock);
	1897	+
	1898	+ mutex_lock(&prange->migrate_mutex);
	1899	+ while (!list_empty(&prange->child_list)) {
	1900	+ struct svm_range *pchild;
	1901	+
	1902	+ pchild = list_first_entry(&prange->child_list,
	1903	+ struct svm_range, child_list);
	1904	+ pr_debug("child prange 0x%p op %d\n", pchild,
	1905	+ pchild->work_item.op);
	1906	+ list_del_init(&pchild->child_list);
	1907	+ svm_range_handle_list_op(svms, pchild);
	1908	+ }
	1909	+ mutex_unlock(&prange->migrate_mutex);
	1910	+
	1911	+ svm_range_handle_list_op(svms, prange);
	1912	+ mutex_unlock(&svms->lock);
	1913	+ mmap_write_unlock(mm);
	1914	+
	1915	+ spin_lock(&svms->deferred_list_lock);
	1916	+ }
	1917	+ spin_unlock(&svms->deferred_list_lock);
	1918	+
	1919	+ pr_debug("exit svms 0x%p\n", svms);
	1920	+}
	1921	+
	1922	+void
	1923	+svm_range_add_list_work(struct svm_range_list svms, struct svm_range prange,
	1924	+ struct mm_struct *mm, enum svm_work_list_ops op)
	1925	+{
	1926	+ spin_lock(&svms->deferred_list_lock);
	1927	+ /* if prange is on the deferred list */
	1928	+ if (!list_empty(&prange->deferred_list)) {
	1929	+ pr_debug("update exist prange 0x%p work op %d\n", prange, op);
	1930	+ WARN_ONCE(prange->work_item.mm != mm, "unmatch mm\n");
	1931	+ if (op != SVM_OP_NULL &&
	1932	+ prange->work_item.op != SVM_OP_UNMAP_RANGE)
	1933	+ prange->work_item.op = op;
	1934	+ } else {
	1935	+ prange->work_item.op = op;
	1936	+ prange->work_item.mm = mm;
	1937	+ list_add_tail(&prange->deferred_list,
	1938	+ &prange->svms->deferred_range_list);
	1939	+ pr_debug("add prange 0x%p [0x%lx 0x%lx] to work list op %d\n",
	1940	+ prange, prange->start, prange->last, op);
	1941	+ }
	1942	+ spin_unlock(&svms->deferred_list_lock);
	1943	+}
	1944	+
	1945	+void schedule_deferred_list_work(struct svm_range_list *svms)
	1946	+{
	1947	+ spin_lock(&svms->deferred_list_lock);
	1948	+ if (!list_empty(&svms->deferred_range_list))
	1949	+ schedule_work(&svms->deferred_list_work);
	1950	+ spin_unlock(&svms->deferred_list_lock);
	1951	+}
	1952	+
	1953	+static void
	1954	+svm_range_unmap_split(struct mm_struct mm, struct svm_range parent,
	1955	+ struct svm_range *prange, unsigned long start,
	1956	+ unsigned long last)
	1957	+{
	1958	+ struct svm_range *head;
	1959	+ struct svm_range *tail;
	1960	+
	1961	+ if (prange->work_item.op == SVM_OP_UNMAP_RANGE) {
	1962	+ pr_debug("prange 0x%p [0x%lx 0x%lx] is already freed\n", prange,
	1963	+ prange->start, prange->last);
	1964	+ return;
	1965	+ }
	1966	+ if (start > prange->last \|\| last < prange->start)
	1967	+ return;
	1968	+
	1969	+ head = tail = prange;
	1970	+ if (start > prange->start)
	1971	+ svm_range_split(prange, prange->start, start - 1, &tail);
	1972	+ if (last < tail->last)
	1973	+ svm_range_split(tail, last + 1, tail->last, &head);
	1974	+
	1975	+ if (head != prange && tail != prange) {
	1976	+ svm_range_add_child(parent, mm, head, SVM_OP_UNMAP_RANGE);
	1977	+ svm_range_add_child(parent, mm, tail, SVM_OP_ADD_RANGE);
	1978	+ } else if (tail != prange) {
	1979	+ svm_range_add_child(parent, mm, tail, SVM_OP_UNMAP_RANGE);
	1980	+ } else if (head != prange) {
	1981	+ svm_range_add_child(parent, mm, head, SVM_OP_UNMAP_RANGE);
	1982	+ } else if (parent != prange) {
	1983	+ prange->work_item.op = SVM_OP_UNMAP_RANGE;
	1984	+ }
	1985	+}
	1986	+
	1987	+static void
	1988	+svm_range_unmap_from_cpu(struct mm_struct mm, struct svm_range prange,
	1989	+ unsigned long start, unsigned long last)
	1990	+{
	1991	+ struct svm_range_list *svms;
	1992	+ struct svm_range *pchild;
	1993	+ struct kfd_process *p;
	1994	+ unsigned long s, l;
	1995	+ bool unmap_parent;
	1996	+
	1997	+ p = kfd_lookup_process_by_mm(mm);
	1998	+ if (!p)
	1999	+ return;
	2000	+ svms = &p->svms;
	2001	+
	2002	+ pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] [0x%lx 0x%lx]\n", svms,
	2003	+ prange, prange->start, prange->last, start, last);
	2004	+
	2005	+ unmap_parent = start <= prange->start && last >= prange->last;
	2006	+
	2007	+ list_for_each_entry(pchild, &prange->child_list, child_list) {
	2008	+ mutex_lock_nested(&pchild->lock, 1);
	2009	+ s = max(start, pchild->start);
	2010	+ l = min(last, pchild->last);
	2011	+ if (l >= s)
	2012	+ svm_range_unmap_from_gpus(pchild, s, l);
	2013	+ svm_range_unmap_split(mm, prange, pchild, start, last);
	2014	+ mutex_unlock(&pchild->lock);
	2015	+ }
	2016	+ s = max(start, prange->start);
	2017	+ l = min(last, prange->last);
	2018	+ if (l >= s)
	2019	+ svm_range_unmap_from_gpus(prange, s, l);
	2020	+ svm_range_unmap_split(mm, prange, prange, start, last);
	2021	+
	2022	+ if (unmap_parent)
	2023	+ svm_range_add_list_work(svms, prange, mm, SVM_OP_UNMAP_RANGE);
	2024	+ else
	2025	+ svm_range_add_list_work(svms, prange, mm,
	2026	+ SVM_OP_UPDATE_RANGE_NOTIFIER);
	2027	+ schedule_deferred_list_work(svms);
	2028	+
	2029	+ kfd_unref_process(p);
	2030	+}
	2031	+
	2032	+/**
	2033	+ * svm_range_cpu_invalidate_pagetables - interval notifier callback
	2034	+ *
	2035	+ * If event is MMU_NOTIFY_UNMAP, this is from CPU unmap range, otherwise, it
	2036	+ * is from migration, or CPU page invalidation callback.
	2037	+ *
	2038	+ * For unmap event, unmap range from GPUs, remove prange from svms in a delayed
	2039	+ * work thread, and split prange if only part of prange is unmapped.
	2040	+ *
	2041	+ * For invalidation event, if GPU retry fault is not enabled, evict the queues,
	2042	+ * then schedule svm_range_restore_work to update GPU mapping and resume queues.
	2043	+ * If GPU retry fault is enabled, unmap the svm range from GPU, retry fault will
	2044	+ * update GPU mapping to recover.
	2045	+ *
	2046	+ * Context: mmap lock, notifier_invalidate_start lock are held
	2047	+ * for invalidate event, prange lock is held if this is from migration
	2048	+ */
	2049	+static bool
	2050	+svm_range_cpu_invalidate_pagetables(struct mmu_interval_notifier *mni,
	2051	+ const struct mmu_notifier_range *range,
	2052	+ unsigned long cur_seq)
	2053	+{
	2054	+ struct svm_range *prange;
	2055	+ unsigned long start;
	2056	+ unsigned long last;
	2057	+
	2058	+ if (range->event == MMU_NOTIFY_RELEASE)
	2059	+ return true;
	2060	+
	2061	+ start = mni->interval_tree.start;
	2062	+ last = mni->interval_tree.last;
	2063	+ start = (start > range->start ? start : range->start) >> PAGE_SHIFT;
	2064	+ last = (last < (range->end - 1) ? last : range->end - 1) >> PAGE_SHIFT;
	2065	+ pr_debug("[0x%lx 0x%lx] range[0x%lx 0x%lx] notifier[0x%lx 0x%lx] %d\n",
	2066	+ start, last, range->start >> PAGE_SHIFT,
	2067	+ (range->end - 1) >> PAGE_SHIFT,
	2068	+ mni->interval_tree.start >> PAGE_SHIFT,
	2069	+ mni->interval_tree.last >> PAGE_SHIFT, range->event);
	2070	+
	2071	+ prange = container_of(mni, struct svm_range, notifier);
	2072	+
	2073	+ svm_range_lock(prange);
	2074	+ mmu_interval_set_seq(mni, cur_seq);
	2075	+
	2076	+ switch (range->event) {
	2077	+ case MMU_NOTIFY_UNMAP:
	2078	+ svm_range_unmap_from_cpu(mni->mm, prange, start, last);
	2079	+ break;
	2080	+ default:
	2081	+ svm_range_evict(prange, mni->mm, start, last);
	2082	+ break;
	2083	+ }
	2084	+
	2085	+ svm_range_unlock(prange);
	2086	+
	2087	+ return true;
	2088	+}
	2089	+
	2090	+/**
	2091	+ * svm_range_from_addr - find svm range from fault address
	2092	+ * @svms: svm range list header
	2093	+ * @addr: address to search range interval tree, in pages
	2094	+ * @parent: parent range if range is on child list
	2095	+ *
	2096	+ * Context: The caller must hold svms->lock
	2097	+ *
	2098	+ * Return: the svm_range found or NULL
	2099	+ */
	2100	+struct svm_range *
	2101	+svm_range_from_addr(struct svm_range_list *svms, unsigned long addr,
	2102	+ struct svm_range **parent)
	2103	+{
	2104	+ struct interval_tree_node *node;
	2105	+ struct svm_range *prange;
	2106	+ struct svm_range *pchild;
	2107	+
	2108	+ node = interval_tree_iter_first(&svms->objects, addr, addr);
	2109	+ if (!node)
	2110	+ return NULL;
	2111	+
	2112	+ prange = container_of(node, struct svm_range, it_node);
	2113	+ pr_debug("address 0x%lx prange [0x%lx 0x%lx] node [0x%lx 0x%lx]\n",
	2114	+ addr, prange->start, prange->last, node->start, node->last);
	2115	+
	2116	+ if (addr >= prange->start && addr <= prange->last) {
	2117	+ if (parent)
	2118	+ *parent = prange;
	2119	+ return prange;
	2120	+ }
	2121	+ list_for_each_entry(pchild, &prange->child_list, child_list)
	2122	+ if (addr >= pchild->start && addr <= pchild->last) {
	2123	+ pr_debug("found address 0x%lx pchild [0x%lx 0x%lx]\n",
	2124	+ addr, pchild->start, pchild->last);
	2125	+ if (parent)
	2126	+ *parent = prange;
	2127	+ return pchild;
	2128	+ }
	2129	+
	2130	+ return NULL;
	2131	+}
	2132	+
	2133	+/* svm_range_best_restore_location - decide the best fault restore location
	2134	+ * @prange: svm range structure
	2135	+ * @adev: the GPU on which vm fault happened
	2136	+ *
	2137	+ * This is only called when xnack is on, to decide the best location to restore
	2138	+ * the range mapping after GPU vm fault. Caller uses the best location to do
	2139	+ * migration if actual loc is not best location, then update GPU page table
	2140	+ * mapping to the best location.
	2141	+ *
	2142	+ * If vm fault gpu is range preferred loc, the best_loc is preferred loc.
	2143	+ * If vm fault gpu idx is on range ACCESSIBLE bitmap, best_loc is vm fault gpu
	2144	+ * If vm fault gpu idx is on range ACCESSIBLE_IN_PLACE bitmap, then
	2145	+ * if range actual loc is cpu, best_loc is cpu
	2146	+ * if vm fault gpu is on xgmi same hive of range actual loc gpu, best_loc is
	2147	+ * range actual loc.
	2148	+ * Otherwise, GPU no access, best_loc is -1.
	2149	+ *
	2150	+ * Return:
	2151	+ * -1 means vm fault GPU no access
	2152	+ * 0 for CPU or GPU id
	2153	+ */
	2154	+static int32_t
	2155	+svm_range_best_restore_location(struct svm_range *prange,
	2156	+ struct amdgpu_device *adev,
	2157	+ int32_t *gpuidx)
	2158	+{
	2159	+ struct amdgpu_device *bo_adev;
	2160	+ struct kfd_process *p;
	2161	+ uint32_t gpuid;
	2162	+ int r;
	2163	+
	2164	+ p = container_of(prange->svms, struct kfd_process, svms);
	2165	+
	2166	+ r = kfd_process_gpuid_from_kgd(p, adev, &gpuid, gpuidx);
	2167	+ if (r < 0) {
	2168	+ pr_debug("failed to get gpuid from kgd\n");
	2169	+ return -1;
	2170	+ }
	2171	+
	2172	+ if (prange->preferred_loc == gpuid)
	2173	+ return prange->preferred_loc;
	2174	+
	2175	+ if (test_bit(*gpuidx, prange->bitmap_access))
	2176	+ return gpuid;
	2177	+
	2178	+ if (test_bit(*gpuidx, prange->bitmap_aip)) {
	2179	+ if (!prange->actual_loc)
	2180	+ return 0;
	2181	+
	2182	+ bo_adev = svm_range_get_adev_by_id(prange, prange->actual_loc);
	2183	+ if (amdgpu_xgmi_same_hive(adev, bo_adev))
	2184	+ return prange->actual_loc;
	2185	+ else
	2186	+ return 0;
	2187	+ }
	2188	+
	2189	+ return -1;
	2190	+}
	2191	+static int
	2192	+svm_range_get_range_boundaries(struct kfd_process *p, int64_t addr,
	2193	+ unsigned long start, unsigned long last)
	2194	+{
	2195	+ struct vm_area_struct *vma;
	2196	+ struct interval_tree_node *node;
	2197	+ unsigned long start_limit, end_limit;
	2198	+
	2199	+ vma = find_vma(p->mm, addr << PAGE_SHIFT);
	2200	+ if (!vma \|\| (addr << PAGE_SHIFT) < vma->vm_start) {
	2201	+ pr_debug("VMA does not exist in address [0x%llx]\n", addr);
	2202	+ return -EFAULT;
	2203	+ }
	2204	+ start_limit = max(vma->vm_start >> PAGE_SHIFT,
	2205	+ (unsigned long)ALIGN_DOWN(addr, 2UL << 8));
	2206	+ end_limit = min(vma->vm_end >> PAGE_SHIFT,
	2207	+ (unsigned long)ALIGN(addr + 1, 2UL << 8));
	2208	+ /* First range that starts after the fault address */
	2209	+ node = interval_tree_iter_first(&p->svms.objects, addr + 1, ULONG_MAX);
	2210	+ if (node) {
	2211	+ end_limit = min(end_limit, node->start);
	2212	+ /* Last range that ends before the fault address */
	2213	+ node = container_of(rb_prev(&node->rb),
	2214	+ struct interval_tree_node, rb);
	2215	+ } else {
	2216	+ /* Last range must end before addr because
	2217	+ * there was no range after addr
	2218	+ */
	2219	+ node = container_of(rb_last(&p->svms.objects.rb_root),
	2220	+ struct interval_tree_node, rb);
	2221	+ }
	2222	+ if (node) {
	2223	+ if (node->last >= addr) {
	2224	+ WARN(1, "Overlap with prev node and page fault addr\n");
	2225	+ return -EFAULT;
	2226	+ }
	2227	+ start_limit = max(start_limit, node->last + 1);
	2228	+ }
	2229	+
	2230	+ *start = start_limit;
	2231	+ *last = end_limit - 1;
	2232	+
	2233	+ pr_debug("vma start: 0x%lx start: 0x%lx vma end: 0x%lx last: 0x%lx\n",
	2234	+ vma->vm_start >> PAGE_SHIFT, *start,
	2235	+ vma->vm_end >> PAGE_SHIFT, *last);
	2236	+
	2237	+ return 0;
	2238	+
	2239	+}
	2240	+static struct
	2241	+svm_range svm_range_create_unregistered_range(struct amdgpu_device adev,
	2242	+ struct kfd_process *p,
	2243	+ struct mm_struct *mm,
	2244	+ int64_t addr)
	2245	+{
	2246	+ struct svm_range *prange = NULL;
	2247	+ unsigned long start, last;
	2248	+ uint32_t gpuid, gpuidx;
	2249	+
	2250	+ if (svm_range_get_range_boundaries(p, addr, &start, &last))
	2251	+ return NULL;
	2252	+
	2253	+ prange = svm_range_new(&p->svms, start, last);
	2254	+ if (!prange) {
	2255	+ pr_debug("Failed to create prange in address [0x%llx]\n", addr);
	2256	+ return NULL;
	2257	+ }
	2258	+ if (kfd_process_gpuid_from_kgd(p, adev, &gpuid, &gpuidx)) {
	2259	+ pr_debug("failed to get gpuid from kgd\n");
	2260	+ svm_range_free(prange);
	2261	+ return NULL;
	2262	+ }
	2263	+
	2264	+ svm_range_add_to_svms(prange);
	2265	+ svm_range_add_notifier_locked(mm, prange);
	2266	+
	2267	+ return prange;
	2268	+}
	2269	+
	2270	+/* svm_range_skip_recover - decide if prange can be recovered
	2271	+ * @prange: svm range structure
	2272	+ *
	2273	+ * GPU vm retry fault handle skip recover the range for cases:
	2274	+ * 1. prange is on deferred list to be removed after unmap, it is stale fault,
	2275	+ * deferred list work will drain the stale fault before free the prange.
	2276	+ * 2. prange is on deferred list to add interval notifier after split, or
	2277	+ * 3. prange is child range, it is split from parent prange, recover later
	2278	+ * after interval notifier is added.
	2279	+ *
	2280	+ * Return: true to skip recover, false to recover
	2281	+ */
	2282	+static bool svm_range_skip_recover(struct svm_range *prange)
	2283	+{
	2284	+ struct svm_range_list *svms = prange->svms;
	2285	+
	2286	+ spin_lock(&svms->deferred_list_lock);
	2287	+ if (list_empty(&prange->deferred_list) &&
	2288	+ list_empty(&prange->child_list)) {
	2289	+ spin_unlock(&svms->deferred_list_lock);
	2290	+ return false;
	2291	+ }
	2292	+ spin_unlock(&svms->deferred_list_lock);
	2293	+
	2294	+ if (prange->work_item.op == SVM_OP_UNMAP_RANGE) {
	2295	+ pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] unmapped\n",
	2296	+ svms, prange, prange->start, prange->last);
	2297	+ return true;
	2298	+ }
	2299	+ if (prange->work_item.op == SVM_OP_ADD_RANGE_AND_MAP \|\|
	2300	+ prange->work_item.op == SVM_OP_ADD_RANGE) {
	2301	+ pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx] not added yet\n",
	2302	+ svms, prange, prange->start, prange->last);
	2303	+ return true;
	2304	+ }
	2305	+ return false;
	2306	+}
	2307	+
	2308	+int
	2309	+svm_range_restore_pages(struct amdgpu_device *adev, unsigned int pasid,
	2310	+ uint64_t addr)
	2311	+{
	2312	+ struct mm_struct *mm = NULL;
	2313	+ struct svm_range_list *svms;
	2314	+ struct svm_range *prange;
	2315	+ struct kfd_process *p;
	2316	+ uint64_t timestamp;
	2317	+ int32_t best_loc, gpuidx;
	2318	+ bool write_locked = false;
	2319	+ int r = 0;
	2320	+
	2321	+ p = kfd_lookup_process_by_pasid(pasid);
	2322	+ if (!p) {
	2323	+ pr_debug("kfd process not founded pasid 0x%x\n", pasid);
	2324	+ return -ESRCH;
	2325	+ }
	2326	+ if (!p->xnack_enabled) {
	2327	+ pr_debug("XNACK not enabled for pasid 0x%x\n", pasid);
	2328	+ return -EFAULT;
	2329	+ }
	2330	+ svms = &p->svms;
	2331	+
	2332	+ pr_debug("restoring svms 0x%p fault address 0x%llx\n", svms, addr);
	2333	+
	2334	+ mm = get_task_mm(p->lead_thread);
	2335	+ if (!mm) {
	2336	+ pr_debug("svms 0x%p failed to get mm\n", svms);
	2337	+ r = -ESRCH;
	2338	+ goto out;
	2339	+ }
	2340	+
	2341	+ mmap_read_lock(mm);
	2342	+retry_write_locked:
	2343	+ mutex_lock(&svms->lock);
	2344	+ prange = svm_range_from_addr(svms, addr, NULL);
	2345	+ if (!prange) {
	2346	+ pr_debug("failed to find prange svms 0x%p address [0x%llx]\n",
	2347	+ svms, addr);
	2348	+ if (!write_locked) {
	2349	+ /* Need the write lock to create new range with MMU notifier.
	2350	+ * Also flush pending deferred work to make sure the interval
	2351	+ * tree is up to date before we add a new range
	2352	+ */
	2353	+ mutex_unlock(&svms->lock);
	2354	+ mmap_read_unlock(mm);
	2355	+ mmap_write_lock(mm);
	2356	+ write_locked = true;
	2357	+ goto retry_write_locked;
	2358	+ }
	2359	+ prange = svm_range_create_unregistered_range(adev, p, mm, addr);
	2360	+ if (!prange) {
	2361	+ pr_debug("failed to create unregistered range svms 0x%p address [0x%llx]\n",
	2362	+ svms, addr);
	2363	+ mmap_write_downgrade(mm);
	2364	+ r = -EFAULT;
	2365	+ goto out_unlock_svms;
	2366	+ }
	2367	+ }
	2368	+ if (write_locked)
	2369	+ mmap_write_downgrade(mm);
	2370	+
	2371	+ mutex_lock(&prange->migrate_mutex);
	2372	+
	2373	+ if (svm_range_skip_recover(prange)) {
	2374	+ amdgpu_gmc_filter_faults_remove(adev, addr, pasid);
	2375	+ goto out_unlock_range;
	2376	+ }
	2377	+
	2378	+ timestamp = ktime_to_us(ktime_get()) - prange->validate_timestamp;
	2379	+ /* skip duplicate vm fault on different pages of same range */
	2380	+ if (timestamp < AMDGPU_SVM_RANGE_RETRY_FAULT_PENDING) {
	2381	+ pr_debug("svms 0x%p [0x%lx %lx] already restored\n",
	2382	+ svms, prange->start, prange->last);
	2383	+ goto out_unlock_range;
	2384	+ }
	2385	+
	2386	+ best_loc = svm_range_best_restore_location(prange, adev, &gpuidx);
	2387	+ if (best_loc == -1) {
	2388	+ pr_debug("svms %p failed get best restore loc [0x%lx 0x%lx]\n",
	2389	+ svms, prange->start, prange->last);
	2390	+ r = -EACCES;
	2391	+ goto out_unlock_range;
	2392	+ }
	2393	+
	2394	+ pr_debug("svms %p [0x%lx 0x%lx] best restore 0x%x, actual loc 0x%x\n",
	2395	+ svms, prange->start, prange->last, best_loc,
	2396	+ prange->actual_loc);
	2397	+
	2398	+ if (prange->actual_loc != best_loc) {
	2399	+ if (best_loc) {
	2400	+ r = svm_migrate_to_vram(prange, best_loc, mm);
	2401	+ if (r) {
	2402	+ pr_debug("svm_migrate_to_vram failed (%d) at %llx, falling back to system memory\n",
	2403	+ r, addr);
	2404	+ /* Fallback to system memory if migration to
	2405	+ * VRAM failed
	2406	+ */
	2407	+ if (prange->actual_loc)
	2408	+ r = svm_migrate_vram_to_ram(prange, mm);
	2409	+ else
	2410	+ r = 0;
	2411	+ }
	2412	+ } else {
	2413	+ r = svm_migrate_vram_to_ram(prange, mm);
	2414	+ }
	2415	+ if (r) {
	2416	+ pr_debug("failed %d to migrate svms %p [0x%lx 0x%lx]\n",
	2417	+ r, svms, prange->start, prange->last);
	2418	+ goto out_unlock_range;
	2419	+ }
	2420	+ }
	2421	+
	2422	+ r = svm_range_validate_and_map(mm, prange, gpuidx, false, false);
	2423	+ if (r)
	2424	+ pr_debug("failed %d to map svms 0x%p [0x%lx 0x%lx] to gpus\n",
	2425	+ r, svms, prange->start, prange->last);
	2426	+
	2427	+out_unlock_range:
	2428	+ mutex_unlock(&prange->migrate_mutex);
	2429	+out_unlock_svms:
	2430	+ mutex_unlock(&svms->lock);
	2431	+ mmap_read_unlock(mm);
	2432	+ mmput(mm);
	2433	+out:
	2434	+ kfd_unref_process(p);
	2435	+
	2436	+ if (r == -EAGAIN) {
	2437	+ pr_debug("recover vm fault later\n");
	2438	+ amdgpu_gmc_filter_faults_remove(adev, addr, pasid);
	2439	+ r = 0;
	2440	+ }
	2441	+ return r;
	2442	+}
	2443	+
	2444	+void svm_range_list_fini(struct kfd_process *p)
	2445	+{
	2446	+ struct svm_range *prange;
	2447	+ struct svm_range *next;
	2448	+
	2449	+ pr_debug("pasid 0x%x svms 0x%p\n", p->pasid, &p->svms);
	2450	+
	2451	+ /* Ensure list work is finished before process is destroyed */
	2452	+ flush_work(&p->svms.deferred_list_work);
	2453	+
	2454	+ list_for_each_entry_safe(prange, next, &p->svms.list, list) {
	2455	+ svm_range_unlink(prange);
	2456	+ svm_range_remove_notifier(prange);
	2457	+ svm_range_free(prange);
	2458	+ }
	2459	+
	2460	+ mutex_destroy(&p->svms.lock);
	2461	+
	2462	+ pr_debug("pasid 0x%x svms 0x%p done\n", p->pasid, &p->svms);
	2463	+}
	2464	+
	2465	+int svm_range_list_init(struct kfd_process *p)
	2466	+{
	2467	+ struct svm_range_list *svms = &p->svms;
	2468	+
	2469	+ svms->objects = RB_ROOT_CACHED;
	2470	+ mutex_init(&svms->lock);
	2471	+ INIT_LIST_HEAD(&svms->list);
	2472	+ atomic_set(&svms->evicted_ranges, 0);
	2473	+ INIT_DELAYED_WORK(&svms->restore_work, svm_range_restore_work);
	2474	+ INIT_WORK(&svms->deferred_list_work, svm_range_deferred_list_work);
	2475	+ INIT_LIST_HEAD(&svms->deferred_range_list);
	2476	+ spin_lock_init(&svms->deferred_list_lock);
	2477	+
	2478	+ return 0;
	2479	+}
	2480	+
	2481	+/**
	2482	+ * svm_range_is_valid - check if virtual address range is valid
	2483	+ * @mm: current process mm_struct
	2484	+ * @start: range start address, in pages
	2485	+ * @size: range size, in pages
	2486	+ *
	2487	+ * Valid virtual address range means it belongs to one or more VMAs
	2488	+ *
	2489	+ * Context: Process context
	2490	+ *
	2491	+ * Return:
	2492	+ * true - valid svm range
	2493	+ * false - invalid svm range
	2494	+ */
	2495	+static bool
	2496	+svm_range_is_valid(struct mm_struct *mm, uint64_t start, uint64_t size)
	2497	+{
	2498	+ const unsigned long device_vma = VM_IO \| VM_PFNMAP \| VM_MIXEDMAP;
	2499	+ struct vm_area_struct *vma;
	2500	+ unsigned long end;
	2501	+
	2502	+ start <<= PAGE_SHIFT;
	2503	+ end = start + (size << PAGE_SHIFT);
	2504	+
	2505	+ do {
	2506	+ vma = find_vma(mm, start);
	2507	+ if (!vma \|\| start < vma->vm_start \|\|
	2508	+ (vma->vm_flags & device_vma))
	2509	+ return false;
	2510	+ start = min(end, vma->vm_end);
	2511	+ } while (start < end);
	2512	+
	2513	+ return true;
	2514	+}
	2515	+
	2516	+/**
	2517	+ * svm_range_add - add svm range and handle overlap
	2518	+ * @p: the range add to this process svms
	2519	+ * @start: page size aligned
	2520	+ * @size: page size aligned
	2521	+ * @nattr: number of attributes
	2522	+ * @attrs: array of attributes
	2523	+ * @update_list: output, the ranges need validate and update GPU mapping
	2524	+ * @insert_list: output, the ranges need insert to svms
	2525	+ * @remove_list: output, the ranges are replaced and need remove from svms
	2526	+ *
	2527	+ * Check if the virtual address range has overlap with the registered ranges,
	2528	+ * split the overlapped range, copy and adjust pages address and vram nodes in
	2529	+ * old and new ranges.
	2530	+ *
	2531	+ * Context: Process context, caller must hold svms->lock
	2532	+ *
	2533	+ * Return:
	2534	+ * 0 - OK, otherwise error code
	2535	+ */
	2536	+static int
	2537	+svm_range_add(struct kfd_process *p, uint64_t start, uint64_t size,
	2538	+ uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs,
	2539	+ struct list_head update_list, struct list_head insert_list,
	2540	+ struct list_head *remove_list)
	2541	+{
	2542	+ uint64_t last = start + size - 1UL;
	2543	+ struct svm_range_list *svms;
	2544	+ struct svm_range new = {0};
	2545	+ struct svm_range *prange;
	2546	+ unsigned long left = 0;
	2547	+ int r = 0;
	2548	+
	2549	+ pr_debug("svms 0x%p [0x%llx 0x%llx]\n", &p->svms, start, last);
	2550	+
	2551	+ svm_range_apply_attrs(p, &new, nattr, attrs);
	2552	+
	2553	+ svms = &p->svms;
	2554	+
	2555	+ r = svm_range_handle_overlap(svms, &new, start, last, update_list,
	2556	+ insert_list, remove_list, &left);
	2557	+ if (r)
	2558	+ return r;
	2559	+
	2560	+ if (left) {
	2561	+ prange = svm_range_new(svms, last - left + 1, last);
	2562	+ list_add(&prange->insert_list, insert_list);
	2563	+ list_add(&prange->update_list, update_list);
	2564	+ }
	2565	+
	2566	+ return 0;
	2567	+}
	2568	+
	2569	+/* svm_range_best_prefetch_location - decide the best prefetch location
	2570	+ * @prange: svm range structure
	2571	+ *
	2572	+ * For xnack off:
	2573	+ * If range map to single GPU, the best acutal location is prefetch loc, which
	2574	+ * can be CPU or GPU.
	2575	+ *
	2576	+ * If range map to multiple GPUs, only if mGPU connection on xgmi same hive,
	2577	+ * the best actual location could be prefetch_loc GPU. If mGPU connection on
	2578	+ * PCIe, the best actual location is always CPU, because GPU cannot access vram
	2579	+ * of other GPUs, assuming PCIe small bar (large bar support is not upstream).
	2580	+ *
	2581	+ * For xnack on:
	2582	+ * The best actual location is prefetch location. If mGPU connection on xgmi
	2583	+ * same hive, range map to multiple GPUs. Otherwise, the range only map to
	2584	+ * actual location GPU. Other GPU access vm fault will trigger migration.
	2585	+ *
	2586	+ * Context: Process context
	2587	+ *
	2588	+ * Return:
	2589	+ * 0 for CPU or GPU id
	2590	+ */
	2591	+static uint32_t
	2592	+svm_range_best_prefetch_location(struct svm_range *prange)
	2593	+{
	2594	+ DECLARE_BITMAP(bitmap, MAX_GPU_INSTANCE);
	2595	+ uint32_t best_loc = prange->prefetch_loc;
	2596	+ struct kfd_process_device *pdd;
	2597	+ struct amdgpu_device *bo_adev;
	2598	+ struct amdgpu_device *adev;
	2599	+ struct kfd_process *p;
	2600	+ uint32_t gpuidx;
	2601	+
	2602	+ p = container_of(prange->svms, struct kfd_process, svms);
	2603	+
	2604	+ /* xnack on */
	2605	+ if (p->xnack_enabled)
	2606	+ goto out;
	2607	+
	2608	+ /* xnack off */
	2609	+ if (!best_loc \|\| best_loc == KFD_IOCTL_SVM_LOCATION_UNDEFINED)
	2610	+ goto out;
	2611	+
	2612	+ bo_adev = svm_range_get_adev_by_id(prange, best_loc);
	2613	+ if (!bo_adev) {
	2614	+ WARN_ONCE(1, "failed to get device by id 0x%x\n", best_loc);
	2615	+ best_loc = 0;
	2616	+ goto out;
	2617	+ }
	2618	+ bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip,
	2619	+ MAX_GPU_INSTANCE);
	2620	+
	2621	+ for_each_set_bit(gpuidx, bitmap, MAX_GPU_INSTANCE) {
	2622	+ pdd = kfd_process_device_from_gpuidx(p, gpuidx);
	2623	+ if (!pdd) {
	2624	+ pr_debug("failed to get device by idx 0x%x\n", gpuidx);
	2625	+ continue;
	2626	+ }
	2627	+ adev = (struct amdgpu_device *)pdd->dev->kgd;
	2628	+
	2629	+ if (adev == bo_adev)
	2630	+ continue;
	2631	+
	2632	+ if (!amdgpu_xgmi_same_hive(adev, bo_adev)) {
	2633	+ best_loc = 0;
	2634	+ break;
	2635	+ }
	2636	+ }
	2637	+
	2638	+out:
	2639	+ pr_debug("xnack %d svms 0x%p [0x%lx 0x%lx] best loc 0x%x\n",
	2640	+ p->xnack_enabled, &p->svms, prange->start, prange->last,
	2641	+ best_loc);
	2642	+
	2643	+ return best_loc;
	2644	+}
	2645	+
	2646	+/* FIXME: This is a workaround for page locking bug when some pages are
	2647	+ * invalid during migration to VRAM
	2648	+ */
	2649	+void svm_range_prefault(struct svm_range prange, struct mm_struct mm)
	2650	+{
	2651	+ struct hmm_range *hmm_range;
	2652	+ int r;
	2653	+
	2654	+ if (prange->validated_once)
	2655	+ return;
	2656	+
	2657	+ r = amdgpu_hmm_range_get_pages(&prange->notifier, mm, NULL,
	2658	+ prange->start << PAGE_SHIFT,
	2659	+ prange->npages, &hmm_range,
	2660	+ false, true);
	2661	+ if (!r) {
	2662	+ amdgpu_hmm_range_get_pages_done(hmm_range);
	2663	+ prange->validated_once = true;
	2664	+ }
	2665	+}
	2666	+
	2667	+/* svm_range_trigger_migration - start page migration if prefetch loc changed
	2668	+ * @mm: current process mm_struct
	2669	+ * @prange: svm range structure
	2670	+ * @migrated: output, true if migration is triggered
	2671	+ *
	2672	+ * If range perfetch_loc is GPU, actual loc is cpu 0, then migrate the range
	2673	+ * from ram to vram.
	2674	+ * If range prefetch_loc is cpu 0, actual loc is GPU, then migrate the range
	2675	+ * from vram to ram.
	2676	+ *
	2677	+ * If GPU vm fault retry is not enabled, migration interact with MMU notifier
	2678	+ * and restore work:
	2679	+ * 1. migrate_vma_setup invalidate pages, MMU notifier callback svm_range_evict
	2680	+ * stops all queues, schedule restore work
	2681	+ * 2. svm_range_restore_work wait for migration is done by
	2682	+ * a. svm_range_validate_vram takes prange->migrate_mutex
	2683	+ * b. svm_range_validate_ram HMM get pages wait for CPU fault handle returns
	2684	+ * 3. restore work update mappings of GPU, resume all queues.
	2685	+ *
	2686	+ * Context: Process context
	2687	+ *
	2688	+ * Return:
	2689	+ * 0 - OK, otherwise - error code of migration
	2690	+ */
	2691	+static int
	2692	+svm_range_trigger_migration(struct mm_struct mm, struct svm_range prange,
	2693	+ bool *migrated)
	2694	+{
	2695	+ uint32_t best_loc;
	2696	+ int r = 0;
	2697	+
	2698	+ *migrated = false;
	2699	+ best_loc = svm_range_best_prefetch_location(prange);
	2700	+
	2701	+ if (best_loc == KFD_IOCTL_SVM_LOCATION_UNDEFINED \|\|
	2702	+ best_loc == prange->actual_loc)
	2703	+ return 0;
	2704	+
	2705	+ /*
	2706	+ * Prefetch to GPU without host access flag, set actual_loc to gpu, then
	2707	+ * validate on gpu and map to gpus will be handled afterwards.
	2708	+ */
	2709	+ if (best_loc && !prange->actual_loc &&
	2710	+ !(prange->flags & KFD_IOCTL_SVM_FLAG_HOST_ACCESS)) {
	2711	+ prange->actual_loc = best_loc;
	2712	+ return 0;
	2713	+ }
	2714	+
	2715	+ if (!best_loc) {
	2716	+ r = svm_migrate_vram_to_ram(prange, mm);
	2717	+ *migrated = !r;
	2718	+ return r;
	2719	+ }
	2720	+
	2721	+ r = svm_migrate_to_vram(prange, best_loc, mm);
	2722	+ *migrated = !r;
	2723	+
	2724	+ return r;
	2725	+}
	2726	+
	2727	+int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence)
	2728	+{
	2729	+ if (!fence)
	2730	+ return -EINVAL;
	2731	+
	2732	+ if (dma_fence_is_signaled(&fence->base))
	2733	+ return 0;
	2734	+
	2735	+ if (fence->svm_bo) {
	2736	+ WRITE_ONCE(fence->svm_bo->evicting, 1);
	2737	+ schedule_work(&fence->svm_bo->eviction_work);
	2738	+ }
	2739	+
	2740	+ return 0;
	2741	+}
	2742	+
	2743	+static void svm_range_evict_svm_bo_worker(struct work_struct *work)
	2744	+{
	2745	+ struct svm_range_bo *svm_bo;
	2746	+ struct kfd_process *p;
	2747	+ struct mm_struct *mm;
	2748	+
	2749	+ svm_bo = container_of(work, struct svm_range_bo, eviction_work);
	2750	+ if (!svm_bo_ref_unless_zero(svm_bo))
	2751	+ return; /* svm_bo was freed while eviction was pending */
	2752	+
	2753	+ /* svm_range_bo_release destroys this worker thread. So during
	2754	+ * the lifetime of this thread, kfd_process and mm will be valid.
	2755	+ */
	2756	+ p = container_of(svm_bo->svms, struct kfd_process, svms);
	2757	+ mm = p->mm;
	2758	+ if (!mm)
	2759	+ return;
	2760	+
	2761	+ mmap_read_lock(mm);
	2762	+ spin_lock(&svm_bo->list_lock);
	2763	+ while (!list_empty(&svm_bo->range_list)) {
	2764	+ struct svm_range *prange =
	2765	+ list_first_entry(&svm_bo->range_list,
	2766	+ struct svm_range, svm_bo_list);
	2767	+ list_del_init(&prange->svm_bo_list);
	2768	+ spin_unlock(&svm_bo->list_lock);
	2769	+
	2770	+ pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms,
	2771	+ prange->start, prange->last);
	2772	+
	2773	+ mutex_lock(&prange->migrate_mutex);
	2774	+ svm_migrate_vram_to_ram(prange, svm_bo->eviction_fence->mm);
	2775	+
	2776	+ mutex_lock(&prange->lock);
	2777	+ prange->svm_bo = NULL;
	2778	+ mutex_unlock(&prange->lock);
	2779	+
	2780	+ mutex_unlock(&prange->migrate_mutex);
	2781	+
	2782	+ spin_lock(&svm_bo->list_lock);
	2783	+ }
	2784	+ spin_unlock(&svm_bo->list_lock);
	2785	+ mmap_read_unlock(mm);
	2786	+
	2787	+ dma_fence_signal(&svm_bo->eviction_fence->base);
	2788	+ /* This is the last reference to svm_bo, after svm_range_vram_node_free
	2789	+ * has been called in svm_migrate_vram_to_ram
	2790	+ */
	2791	+ WARN_ONCE(kref_read(&svm_bo->kref) != 1, "This was not the last reference\n");
	2792	+ svm_range_bo_unref(svm_bo);
	2793	+}
	2794	+
	2795	+static int
	2796	+svm_range_set_attr(struct kfd_process *p, uint64_t start, uint64_t size,
	2797	+ uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs)
	2798	+{
	2799	+ struct amdkfd_process_info *process_info = p->kgd_process_info;
	2800	+ struct mm_struct *mm = current->mm;
	2801	+ struct list_head update_list;
	2802	+ struct list_head insert_list;
	2803	+ struct list_head remove_list;
	2804	+ struct svm_range_list *svms;
	2805	+ struct svm_range *prange;
	2806	+ struct svm_range *next;
	2807	+ int r = 0;
	2808	+
	2809	+ pr_debug("pasid 0x%x svms 0x%p [0x%llx 0x%llx] pages 0x%llx\n",
	2810	+ p->pasid, &p->svms, start, start + size - 1, size);
	2811	+
	2812	+ r = svm_range_check_attr(p, nattr, attrs);
	2813	+ if (r)
	2814	+ return r;
	2815	+
	2816	+ svms = &p->svms;
	2817	+
	2818	+ mutex_lock(&process_info->lock);
	2819	+
	2820	+ svm_range_list_lock_and_flush_work(svms, mm);
	2821	+
	2822	+ if (!svm_range_is_valid(mm, start, size)) {
	2823	+ pr_debug("invalid range\n");
	2824	+ r = -EFAULT;
	2825	+ mmap_write_unlock(mm);
	2826	+ goto out;
	2827	+ }
	2828	+
	2829	+ mutex_lock(&svms->lock);
	2830	+
	2831	+ /* Add new range and split existing ranges as needed */
	2832	+ r = svm_range_add(p, start, size, nattr, attrs, &update_list,
	2833	+ &insert_list, &remove_list);
	2834	+ if (r) {
	2835	+ mutex_unlock(&svms->lock);
	2836	+ mmap_write_unlock(mm);
	2837	+ goto out;
	2838	+ }
	2839	+ /* Apply changes as a transaction */
	2840	+ list_for_each_entry_safe(prange, next, &insert_list, insert_list) {
	2841	+ svm_range_add_to_svms(prange);
	2842	+ svm_range_add_notifier_locked(mm, prange);
	2843	+ }
	2844	+ list_for_each_entry(prange, &update_list, update_list) {
	2845	+ svm_range_apply_attrs(p, prange, nattr, attrs);
	2846	+ /* TODO: unmap ranges from GPU that lost access */
	2847	+ }
	2848	+ list_for_each_entry_safe(prange, next, &remove_list,
	2849	+ remove_list) {
	2850	+ pr_debug("unlink old 0x%p prange 0x%p [0x%lx 0x%lx]\n",
	2851	+ prange->svms, prange, prange->start,
	2852	+ prange->last);
	2853	+ svm_range_unlink(prange);
	2854	+ svm_range_remove_notifier(prange);
	2855	+ svm_range_free(prange);
	2856	+ }
	2857	+
	2858	+ mmap_write_downgrade(mm);
	2859	+ /* Trigger migrations and revalidate and map to GPUs as needed. If
	2860	+ * this fails we may be left with partially completed actions. There
	2861	+ * is no clean way of rolling back to the previous state in such a
	2862	+ * case because the rollback wouldn't be guaranteed to work either.
	2863	+ */
	2864	+ list_for_each_entry(prange, &update_list, update_list) {
	2865	+ bool migrated;
	2866	+
	2867	+ mutex_lock(&prange->migrate_mutex);
	2868	+
	2869	+ r = svm_range_trigger_migration(mm, prange, &migrated);
	2870	+ if (r)
	2871	+ goto out_unlock_range;
	2872	+
	2873	+ if (migrated && !p->xnack_enabled) {
	2874	+ pr_debug("restore_work will update mappings of GPUs\n");
	2875	+ mutex_unlock(&prange->migrate_mutex);
	2876	+ continue;
	2877	+ }
	2878	+
	2879	+ r = svm_range_validate_and_map(mm, prange, MAX_GPU_INSTANCE,
	2880	+ true, true);
	2881	+ if (r)
	2882	+ pr_debug("failed %d to map svm range\n", r);
	2883	+
	2884	+out_unlock_range:
	2885	+ mutex_unlock(&prange->migrate_mutex);
	2886	+ if (r)
	2887	+ break;
	2888	+ }
	2889	+
	2890	+ svm_range_debug_dump(svms);
	2891	+
	2892	+ mutex_unlock(&svms->lock);
	2893	+ mmap_read_unlock(mm);
	2894	+out:
	2895	+ mutex_unlock(&process_info->lock);
	2896	+
	2897	+ pr_debug("pasid 0x%x svms 0x%p [0x%llx 0x%llx] done, r=%d\n", p->pasid,
	2898	+ &p->svms, start, start + size - 1, r);
	2899	+
	2900	+ return r;
	2901	+}
	2902	+
	2903	+static int
	2904	+svm_range_get_attr(struct kfd_process *p, uint64_t start, uint64_t size,
	2905	+ uint32_t nattr, struct kfd_ioctl_svm_attribute *attrs)
	2906	+{
	2907	+ DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE);
	2908	+ DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE);
	2909	+ bool get_preferred_loc = false;
	2910	+ bool get_prefetch_loc = false;
	2911	+ bool get_granularity = false;
	2912	+ bool get_accessible = false;
	2913	+ bool get_flags = false;
	2914	+ uint64_t last = start + size - 1UL;
	2915	+ struct mm_struct *mm = current->mm;
	2916	+ uint8_t granularity = 0xff;
	2917	+ struct interval_tree_node *node;
	2918	+ struct svm_range_list *svms;
	2919	+ struct svm_range *prange;
	2920	+ uint32_t prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
	2921	+ uint32_t location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
	2922	+ uint32_t flags = 0xffffffff;
	2923	+ int gpuidx;
	2924	+ uint32_t i;
	2925	+
	2926	+ pr_debug("svms 0x%p [0x%llx 0x%llx] nattr 0x%x\n", &p->svms, start,
	2927	+ start + size - 1, nattr);
	2928	+
	2929	+ mmap_read_lock(mm);
	2930	+ if (!svm_range_is_valid(mm, start, size)) {
	2931	+ pr_debug("invalid range\n");
	2932	+ mmap_read_unlock(mm);
	2933	+ return -EINVAL;
	2934	+ }
	2935	+ mmap_read_unlock(mm);
	2936	+
	2937	+ for (i = 0; i < nattr; i++) {
	2938	+ switch (attrs[i].type) {
	2939	+ case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
	2940	+ get_preferred_loc = true;
	2941	+ break;
	2942	+ case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
	2943	+ get_prefetch_loc = true;
	2944	+ break;
	2945	+ case KFD_IOCTL_SVM_ATTR_ACCESS:
	2946	+ get_accessible = true;
	2947	+ break;
	2948	+ case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
	2949	+ get_flags = true;
	2950	+ break;
	2951	+ case KFD_IOCTL_SVM_ATTR_GRANULARITY:
	2952	+ get_granularity = true;
	2953	+ break;
	2954	+ case KFD_IOCTL_SVM_ATTR_CLR_FLAGS:
	2955	+ case KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:
	2956	+ case KFD_IOCTL_SVM_ATTR_NO_ACCESS:
	2957	+ fallthrough;
	2958	+ default:
	2959	+ pr_debug("get invalid attr type 0x%x\n", attrs[i].type);
	2960	+ return -EINVAL;
	2961	+ }
	2962	+ }
	2963	+
	2964	+ svms = &p->svms;
	2965	+
	2966	+ mutex_lock(&svms->lock);
	2967	+
	2968	+ node = interval_tree_iter_first(&svms->objects, start, last);
	2969	+ if (!node) {
	2970	+ pr_debug("range attrs not found return default values\n");
	2971	+ svm_range_set_default_attributes(&location, &prefetch_loc,
	2972	+ &granularity, &flags);
	2973	+ if (p->xnack_enabled)
	2974	+ bitmap_fill(bitmap_access, MAX_GPU_INSTANCE);
	2975	+ else
	2976	+ bitmap_zero(bitmap_access, MAX_GPU_INSTANCE);
	2977	+ bitmap_zero(bitmap_aip, MAX_GPU_INSTANCE);
	2978	+ goto fill_values;
	2979	+ }
	2980	+ bitmap_fill(bitmap_access, MAX_GPU_INSTANCE);
	2981	+ bitmap_fill(bitmap_aip, MAX_GPU_INSTANCE);
	2982	+
	2983	+ while (node) {
	2984	+ struct interval_tree_node *next;
	2985	+
	2986	+ prange = container_of(node, struct svm_range, it_node);
	2987	+ next = interval_tree_iter_next(node, start, last);
	2988	+
	2989	+ if (get_preferred_loc) {
	2990	+ if (prange->preferred_loc ==
	2991	+ KFD_IOCTL_SVM_LOCATION_UNDEFINED \|\|
	2992	+ (location != KFD_IOCTL_SVM_LOCATION_UNDEFINED &&
	2993	+ location != prange->preferred_loc)) {
	2994	+ location = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
	2995	+ get_preferred_loc = false;
	2996	+ } else {
	2997	+ location = prange->preferred_loc;
	2998	+ }
	2999	+ }
	3000	+ if (get_prefetch_loc) {
	3001	+ if (prange->prefetch_loc ==
	3002	+ KFD_IOCTL_SVM_LOCATION_UNDEFINED \|\|
	3003	+ (prefetch_loc != KFD_IOCTL_SVM_LOCATION_UNDEFINED &&
	3004	+ prefetch_loc != prange->prefetch_loc)) {
	3005	+ prefetch_loc = KFD_IOCTL_SVM_LOCATION_UNDEFINED;
	3006	+ get_prefetch_loc = false;
	3007	+ } else {
	3008	+ prefetch_loc = prange->prefetch_loc;
	3009	+ }
	3010	+ }
	3011	+ if (get_accessible) {
	3012	+ bitmap_and(bitmap_access, bitmap_access,
	3013	+ prange->bitmap_access, MAX_GPU_INSTANCE);
	3014	+ bitmap_and(bitmap_aip, bitmap_aip,
	3015	+ prange->bitmap_aip, MAX_GPU_INSTANCE);
	3016	+ }
	3017	+ if (get_flags)
	3018	+ flags &= prange->flags;
	3019	+
	3020	+ if (get_granularity && prange->granularity < granularity)
	3021	+ granularity = prange->granularity;
	3022	+
	3023	+ node = next;
	3024	+ }
	3025	+fill_values:
	3026	+ mutex_unlock(&svms->lock);
	3027	+
	3028	+ for (i = 0; i < nattr; i++) {
	3029	+ switch (attrs[i].type) {
	3030	+ case KFD_IOCTL_SVM_ATTR_PREFERRED_LOC:
	3031	+ attrs[i].value = location;
	3032	+ break;
	3033	+ case KFD_IOCTL_SVM_ATTR_PREFETCH_LOC:
	3034	+ attrs[i].value = prefetch_loc;
	3035	+ break;
	3036	+ case KFD_IOCTL_SVM_ATTR_ACCESS:
	3037	+ gpuidx = kfd_process_gpuidx_from_gpuid(p,
	3038	+ attrs[i].value);
	3039	+ if (gpuidx < 0) {
	3040	+ pr_debug("invalid gpuid %x\n", attrs[i].value);
	3041	+ return -EINVAL;
	3042	+ }
	3043	+ if (test_bit(gpuidx, bitmap_access))
	3044	+ attrs[i].type = KFD_IOCTL_SVM_ATTR_ACCESS;
	3045	+ else if (test_bit(gpuidx, bitmap_aip))
	3046	+ attrs[i].type =
	3047	+ KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE;
	3048	+ else
	3049	+ attrs[i].type = KFD_IOCTL_SVM_ATTR_NO_ACCESS;
	3050	+ break;
	3051	+ case KFD_IOCTL_SVM_ATTR_SET_FLAGS:
	3052	+ attrs[i].value = flags;
	3053	+ break;
	3054	+ case KFD_IOCTL_SVM_ATTR_GRANULARITY:
	3055	+ attrs[i].value = (uint32_t)granularity;
	3056	+ break;
	3057	+ }
	3058	+ }
	3059	+
	3060	+ return 0;
	3061	+}
	3062	+
	3063	+int
	3064	+svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,
	3065	+ uint64_t size, uint32_t nattrs, struct kfd_ioctl_svm_attribute *attrs)
	3066	+{
	3067	+ int r;
	3068	+
	3069	+ start >>= PAGE_SHIFT;
	3070	+ size >>= PAGE_SHIFT;
	3071	+
	3072	+ switch (op) {
	3073	+ case KFD_IOCTL_SVM_OP_SET_ATTR:
	3074	+ r = svm_range_set_attr(p, start, size, nattrs, attrs);
	3075	+ break;
	3076	+ case KFD_IOCTL_SVM_OP_GET_ATTR:
	3077	+ r = svm_range_get_attr(p, start, size, nattrs, attrs);
	3078	+ break;
	3079	+ default:
	3080	+ r = EINVAL;
	3081	+ break;
	3082	+ }
	3083	+
	3084	+ return r;
	3085	+}

drivers/gpu/drm/amd/amdkfd/kfd_svm.h+206−0

@@ -0,0 +1,206 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 OR MIT */
	2	+/*
	3	+ * Copyright 2020-2021 Advanced Micro Devices, Inc.
	4	+ *
	5	+ * Permission is hereby granted, free of charge, to any person obtaining a
	6	+ * copy of this software and associated documentation files (the "Software"),
	7	+ * to deal in the Software without restriction, including without limitation
	8	+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
	9	+ * and/or sell copies of the Software, and to permit persons to whom the
	10	+ * Software is furnished to do so, subject to the following conditions:
	11	+ *
	12	+ * The above copyright notice and this permission notice shall be included in
	13	+ * all copies or substantial portions of the Software.
	14	+ *
	15	+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
	16	+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	17	+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
	18	+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
	19	+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
	20	+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
	21	+ * OTHER DEALINGS IN THE SOFTWARE.
	22	+ *
	23	+ */
	24	+
	25	+#ifndef KFD_SVM_H_
	26	+#define KFD_SVM_H_
	27	+
	28	+#if IS_ENABLED(CONFIG_HSA_AMD_SVM)
	29	+
	30	+#include <linux/rwsem.h>
	31	+#include <linux/list.h>
	32	+#include <linux/mutex.h>
	33	+#include <linux/sched/mm.h>
	34	+#include <linux/hmm.h>
	35	+#include "amdgpu.h"
	36	+#include "kfd_priv.h"
	37	+
	38	+struct svm_range_bo {
	39	+ struct amdgpu_bo *bo;
	40	+ struct kref kref;
	41	+ struct list_head range_list; /* all svm ranges shared this bo */
	42	+ spinlock_t list_lock;
	43	+ struct amdgpu_amdkfd_fence *eviction_fence;
	44	+ struct work_struct eviction_work;
	45	+ struct svm_range_list *svms;
	46	+ uint32_t evicting;
	47	+};
	48	+
	49	+enum svm_work_list_ops {
	50	+ SVM_OP_NULL,
	51	+ SVM_OP_UNMAP_RANGE,
	52	+ SVM_OP_UPDATE_RANGE_NOTIFIER,
	53	+ SVM_OP_UPDATE_RANGE_NOTIFIER_AND_MAP,
	54	+ SVM_OP_ADD_RANGE,
	55	+ SVM_OP_ADD_RANGE_AND_MAP
	56	+};
	57	+
	58	+struct svm_work_list_item {
	59	+ enum svm_work_list_ops op;
	60	+ struct mm_struct *mm;
	61	+};
	62	+
	63	+/**
	64	+ * struct svm_range - shared virtual memory range
	65	+ *
	66	+ * @svms: list of svm ranges, structure defined in kfd_process
	67	+ * @migrate_mutex: to serialize range migration, validation and mapping update
	68	+ * @start: range start address in pages
	69	+ * @last: range last address in pages
	70	+ * @it_node: node [start, last] stored in interval tree, start, last are page
	71	+ * aligned, page size is (last - start + 1)
	72	+ * @list: link list node, used to scan all ranges of svms
	73	+ * @update_list:link list node used to add to update_list
	74	+ * @remove_list:link list node used to add to remove list
	75	+ * @insert_list:link list node used to add to insert list
	76	+ * @mapping: bo_va mapping structure to create and update GPU page table
	77	+ * @npages: number of pages
	78	+ * @dma_addr: dma mapping address on each GPU for system memory physical page
	79	+ * @ttm_res: vram ttm resource map
	80	+ * @offset: range start offset within mm_nodes
	81	+ * @svm_bo: struct to manage splited amdgpu_bo
	82	+ * @svm_bo_list:link list node, to scan all ranges which share same svm_bo
	83	+ * @lock: protect prange start, last, child_list, svm_bo_list
	84	+ * @saved_flags:save/restore current PF_MEMALLOC flags
	85	+ * @flags: flags defined as KFD_IOCTL_SVM_FLAG_*
	86	+ * @perferred_loc: perferred location, 0 for CPU, or GPU id
	87	+ * @perfetch_loc: last prefetch location, 0 for CPU, or GPU id
	88	+ * @actual_loc: the actual location, 0 for CPU, or GPU id
	89	+ * @granularity:migration granularity, log2 num pages
	90	+ * @invalid: not 0 means cpu page table is invalidated
	91	+ * @validate_timestamp: system timestamp when range is validated
	92	+ * @notifier: register mmu interval notifier
	93	+ * @work_item: deferred work item information
	94	+ * @deferred_list: list header used to add range to deferred list
	95	+ * @child_list: list header for split ranges which are not added to svms yet
	96	+ * @bitmap_access: index bitmap of GPUs which can access the range
	97	+ * @bitmap_aip: index bitmap of GPUs which can access the range in place
	98	+ *
	99	+ * Data structure for virtual memory range shared by CPU and GPUs, it can be
	100	+ * allocated from system memory ram or device vram, and migrate from ram to vram
	101	+ * or from vram to ram.
	102	+ */
	103	+struct svm_range {
	104	+ struct svm_range_list *svms;
	105	+ struct mutex migrate_mutex;
	106	+ unsigned long start;
	107	+ unsigned long last;
	108	+ struct interval_tree_node it_node;
	109	+ struct list_head list;
	110	+ struct list_head update_list;
	111	+ struct list_head remove_list;
	112	+ struct list_head insert_list;
	113	+ struct amdgpu_bo_va_mapping mapping;
	114	+ uint64_t npages;
	115	+ dma_addr_t *dma_addr[MAX_GPU_INSTANCE];
	116	+ struct ttm_resource *ttm_res;
	117	+ uint64_t offset;
	118	+ struct svm_range_bo *svm_bo;
	119	+ struct list_head svm_bo_list;
	120	+ struct mutex lock;
	121	+ unsigned int saved_flags;
	122	+ uint32_t flags;
	123	+ uint32_t preferred_loc;
	124	+ uint32_t prefetch_loc;
	125	+ uint32_t actual_loc;
	126	+ uint8_t granularity;
	127	+ atomic_t invalid;
	128	+ uint64_t validate_timestamp;
	129	+ struct mmu_interval_notifier notifier;
	130	+ struct svm_work_list_item work_item;
	131	+ struct list_head deferred_list;
	132	+ struct list_head child_list;
	133	+ DECLARE_BITMAP(bitmap_access, MAX_GPU_INSTANCE);
	134	+ DECLARE_BITMAP(bitmap_aip, MAX_GPU_INSTANCE);
	135	+ bool validated_once;
	136	+};
	137	+
	138	+static inline void svm_range_lock(struct svm_range *prange)
	139	+{
	140	+ mutex_lock(&prange->lock);
	141	+ prange->saved_flags = memalloc_noreclaim_save();
	142	+
	143	+}
	144	+static inline void svm_range_unlock(struct svm_range *prange)
	145	+{
	146	+ memalloc_noreclaim_restore(prange->saved_flags);
	147	+ mutex_unlock(&prange->lock);
	148	+}
	149	+
	150	+int svm_range_list_init(struct kfd_process *p);
	151	+void svm_range_list_fini(struct kfd_process *p);
	152	+int svm_ioctl(struct kfd_process *p, enum kfd_ioctl_svm_op op, uint64_t start,
	153	+ uint64_t size, uint32_t nattrs,
	154	+ struct kfd_ioctl_svm_attribute *attrs);
	155	+struct svm_range svm_range_from_addr(struct svm_range_list svms,
	156	+ unsigned long addr,
	157	+ struct svm_range **parent);
	158	+struct amdgpu_device svm_range_get_adev_by_id(struct svm_range prange,
	159	+ uint32_t id);
	160	+int svm_range_vram_node_new(struct amdgpu_device *adev,
	161	+ struct svm_range *prange, bool clear);
	162	+void svm_range_vram_node_free(struct svm_range *prange);
	163	+int svm_range_split_by_granularity(struct kfd_process p, struct mm_struct mm,
	164	+ unsigned long addr, struct svm_range *parent,
	165	+ struct svm_range *prange);
	166	+int svm_range_restore_pages(struct amdgpu_device *adev,
	167	+ unsigned int pasid, uint64_t addr);
	168	+int svm_range_schedule_evict_svm_bo(struct amdgpu_amdkfd_fence *fence);
	169	+void svm_range_add_list_work(struct svm_range_list *svms,
	170	+ struct svm_range prange, struct mm_struct mm,
	171	+ enum svm_work_list_ops op);
	172	+void schedule_deferred_list_work(struct svm_range_list *svms);
	173	+void svm_range_dma_unmap(struct device dev, dma_addr_t dma_addr,
	174	+ unsigned long offset, unsigned long npages);
	175	+void svm_range_free_dma_mappings(struct svm_range *prange);
	176	+void svm_range_prefault(struct svm_range prange, struct mm_struct mm);
	177	+
	178	+#else
	179	+
	180	+struct kfd_process;
	181	+
	182	+static inline int svm_range_list_init(struct kfd_process *p)
	183	+{
	184	+ return 0;
	185	+}
	186	+static inline void svm_range_list_fini(struct kfd_process *p)
	187	+{
	188	+ /* empty */
	189	+}
	190	+
	191	+static inline int svm_range_restore_pages(struct amdgpu_device *adev,
	192	+ unsigned int pasid, uint64_t addr)
	193	+{
	194	+ return -EFAULT;
	195	+}
	196	+
	197	+static inline int svm_range_schedule_evict_svm_bo(
	198	+ struct amdgpu_amdkfd_fence *fence)
	199	+{
	200	+ WARN_ONCE(1, "SVM eviction fence triggered, but SVM is disabled");
	201	+ return -EINVAL;
	202	+}
	203	+
	204	+#endif /* IS_ENABLED(CONFIG_HSA_AMD_SVM) */
	205	+
	206	+#endif /* KFD_SVM_H_ */

drivers/gpu/drm/amd/amdkfd/kfd_topology.c+54−28

@@ -1192,40 +1192,60 @@ static void kfd_fill_mem_clk_max_info(struct kfd_topology_device *dev) · 1191 unmodified lines
1192	1192	mem->mem_clk_max = local_mem_info.mem_clk_max;
1193	1193	}
1194	1194
1195		−static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
	1195	+static void kfd_set_iolink_no_atomics(struct kfd_topology_device *dev,
	1196	+ struct kfd_topology_device *target_gpu_dev,
	1197	+ struct kfd_iolink_properties *link)
1196	1198	{
1197		− struct kfd_iolink_properties link, cpu_link;
1198		− struct kfd_topology_device *cpu_dev;
1199		− uint32_t cap;
1200		− uint32_t cpu_flag = CRAT_IOLINK_FLAGS_ENABLED;
1201		− uint32_t flag = CRAT_IOLINK_FLAGS_ENABLED;
1202		−
1203		− if (!dev \|\| !dev->gpu)
	1199	+ /* xgmi always supports atomics between links. */
	1200	+ if (link->iolink_type == CRAT_IOLINK_TYPE_XGMI)
1204	1201	return;
1205	1202
1206		− pcie_capability_read_dword(dev->gpu->pdev,
1207		− PCI_EXP_DEVCAP2, &cap);
	1203	+ /* check pcie support to set cpu(dev) flags for target_gpu_dev link. */
	1204	+ if (target_gpu_dev) {
	1205	+ uint32_t cap;
1208	1206
1209		− if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 \|
1210		− PCI_EXP_DEVCAP2_ATOMIC_COMP64)))
1211		− cpu_flag \|= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT \|
1212		− CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
	1207	+ pcie_capability_read_dword(target_gpu_dev->gpu->pdev,
	1208	+ PCI_EXP_DEVCAP2, &cap);
	1209	+
	1210	+ if (!(cap & (PCI_EXP_DEVCAP2_ATOMIC_COMP32 \|
	1211	+ PCI_EXP_DEVCAP2_ATOMIC_COMP64)))
	1212	+ link->flags \|= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT \|
	1213	+ CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
	1214	+ /* set gpu (dev) flags. */
	1215	+ } else {
	1216	+ if (!dev->gpu->pci_atomic_requested \|\|
	1217	+ dev->gpu->device_info->asic_family ==
	1218	+ CHIP_HAWAII)
	1219	+ link->flags \|= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT \|
	1220	+ CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
	1221	+ }
	1222	+}
	1223	+
	1224	+static void kfd_fill_iolink_non_crat_info(struct kfd_topology_device *dev)
	1225	+{
	1226	+ struct kfd_iolink_properties link, inbound_link;
	1227	+ struct kfd_topology_device *peer_dev;
1213	1228
1214		− if (!dev->gpu->pci_atomic_requested \|\|
1215		− dev->gpu->device_info->asic_family == CHIP_HAWAII)
1216		− flag \|= CRAT_IOLINK_FLAGS_NO_ATOMICS_32_BIT \|
1217		− CRAT_IOLINK_FLAGS_NO_ATOMICS_64_BIT;
	1229	+ if (!dev \|\| !dev->gpu)
	1230	+ return;
1218	1231
1219	1232	/* GPU only creates direct links so apply flags setting to all */
1220	1233	list_for_each_entry(link, &dev->io_link_props, list) {
1221		− link->flags = flag;
1222		− cpu_dev = kfd_topology_device_by_proximity_domain(
	1234	+ link->flags = CRAT_IOLINK_FLAGS_ENABLED;
	1235	+ kfd_set_iolink_no_atomics(dev, NULL, link);
	1236	+ peer_dev = kfd_topology_device_by_proximity_domain(
1223	1237	link->node_to);
1224		− if (cpu_dev) {
1225		− list_for_each_entry(cpu_link,
1226		− &cpu_dev->io_link_props, list)
1227		− if (cpu_link->node_to == link->node_from)
1228		− cpu_link->flags = cpu_flag;
	1238	+
	1239	+ if (!peer_dev)
	1240	+ continue;
	1241	+
	1242	+ list_for_each_entry(inbound_link, &peer_dev->io_link_props,
	1243	+ list) {
	1244	+ if (inbound_link->node_to != link->node_from)
	1245	+ continue;
	1246	+
	1247	+ inbound_link->flags = CRAT_IOLINK_FLAGS_ENABLED;
	1248	+ kfd_set_iolink_no_atomics(peer_dev, dev, inbound_link);
1229	1249	}
1230	1250	}
1231	1251	}
@@ -1410,15 +1430,21 @@ int kfd_topology_add_device(struct kfd_dev *gpu) · 178 unmodified lines
1410	1430	adev = (struct amdgpu_device *)(dev->gpu->kgd);
1411	1431	/* kfd only concerns sram ecc on GFX and HBM ecc on UMC */
1412	1432	dev->node_props.capability \|=
1413		− ((adev->ras_features & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0) ?
	1433	+ ((adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__GFX)) != 0) ?
1414	1434	HSA_CAP_SRAM_EDCSUPPORTED : 0;
1415		− dev->node_props.capability \|= ((adev->ras_features & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ?
	1435	+ dev->node_props.capability \|= ((adev->ras_enabled & BIT(AMDGPU_RAS_BLOCK__UMC)) != 0) ?
1416	1436	HSA_CAP_MEM_EDCSUPPORTED : 0;
1417	1437
1418	1438	if (adev->asic_type != CHIP_VEGA10)
1419		− dev->node_props.capability \|= (adev->ras_features != 0) ?
	1439	+ dev->node_props.capability \|= (adev->ras_enabled != 0) ?
1420	1440	HSA_CAP_RASEVENTNOTIFY : 0;
1421	1441
	1442	+ /* SVM API and HMM page migration work together, device memory type
	1443	+ * is initialized to not 0 when page migration register device memory.
	1444	+ */
	1445	+ if (adev->kfd.dev->pgmap.type != 0)
	1446	+ dev->node_props.capability \|= HSA_CAP_SVMAPI_SUPPORTED;
	1447	+
1422	1448	kfd_debug_print_topology();
1423	1449
1424	1450	if (!res)

drivers/gpu/drm/amd/amdkfd/kfd_topology.h+6−4

@@ -53,8 +53,9 @@ · 52 unmodified lines
53	53	#define HSA_CAP_ASIC_REVISION_MASK 0x03c00000
54	54	#define HSA_CAP_ASIC_REVISION_SHIFT 22
55	55	#define HSA_CAP_SRAM_EDCSUPPORTED 0x04000000
	56	+#define HSA_CAP_SVMAPI_SUPPORTED 0x08000000
56	57
57		−#define HSA_CAP_RESERVED 0xf80f8000
	58	+#define HSA_CAP_RESERVED 0xf00f8000
58	59
59	60	struct kfd_node_properties {
60	61	uint64_t hive_id;
@@ -98,9 +99,10 @@ struct kfd_node_properties { · 37 unmodified lines
98	99	#define HSA_MEM_HEAP_TYPE_GPU_LDS 4
99	100	#define HSA_MEM_HEAP_TYPE_GPU_SCRATCH 5
100	101
101		−#define HSA_MEM_FLAGS_HOT_PLUGGABLE 0x00000001
102		−#define HSA_MEM_FLAGS_NON_VOLATILE 0x00000002
103		−#define HSA_MEM_FLAGS_RESERVED 0xfffffffc
	102	+#define HSA_MEM_FLAGS_HOT_PLUGGABLE 0x00000001
	103	+#define HSA_MEM_FLAGS_NON_VOLATILE 0x00000002
	104	+#define HSA_MEM_FLAGS_COHERENTHOSTACCESS 0x00000004
	105	+#define HSA_MEM_FLAGS_RESERVED 0xfffffff8
104	106
105	107	struct kfd_mem_properties {
106	108	struct list_head list;

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c+188−71

@@ -35,6 +35,7 @@ · 34 unmodified lines
35	35	#include "dc/inc/hw/abm.h"
36	36	#include "dc/dc_dmub_srv.h"
37	37	#include "dc/dc_edid_parser.h"
	38	+#include "dc/dc_stat.h"
38	39	#include "amdgpu_dm_trace.h"
39	40
40	41	#include "vid.h"
@@ -59,6 +60,7 @@ · 18 unmodified lines
59	60
60	61	#include "ivsrcid/ivsrcid_vislands30.h"
61	62
	63	+#include "i2caux_interface.h"
62	64	#include <linux/module.h>
63	65	#include <linux/moduleparam.h>
64	66	#include <linux/types.h>
@@ -618,6 +620,58 @@ static void dm_dcn_vertical_interrupt0_high_irq(void *interrupt_params) · 553 unmodified lines
618	620	amdgpu_dm_crtc_handle_crc_window_irq(&acrtc->base);
619	621	}
620	622	#endif
	623	+
	624	+/**
	625	+ * dm_dmub_outbox1_low_irq() - Handles Outbox interrupt
	626	+ * @interrupt_params: used for determining the Outbox instance
	627	+ *
	628	+ * Handles the Outbox Interrupt
	629	+ * event handler.
	630	+ */
	631	+#define DMUB_TRACE_MAX_READ 64
	632	+static void dm_dmub_outbox1_low_irq(void *interrupt_params)
	633	+{
	634	+ struct dmub_notification notify;
	635	+ struct common_irq_params *irq_params = interrupt_params;
	636	+ struct amdgpu_device *adev = irq_params->adev;
	637	+ struct amdgpu_display_manager *dm = &adev->dm;
	638	+ struct dmcub_trace_buf_entry entry = { 0 };
	639	+ uint32_t count = 0;
	640	+
	641	+ if (dc_enable_dmub_notifications(adev->dm.dc)) {
	642	+ if (irq_params->irq_src == DC_IRQ_SOURCE_DMCUB_OUTBOX) {
	643	+ do {
	644	+ dc_stat_get_dmub_notification(adev->dm.dc, &notify);
	645	+ } while (notify.pending_notification);
	646	+
	647	+ if (adev->dm.dmub_notify)
	648	+ memcpy(adev->dm.dmub_notify, &notify, sizeof(struct dmub_notification));
	649	+ if (notify.type == DMUB_NOTIFICATION_AUX_REPLY)
	650	+ complete(&adev->dm.dmub_aux_transfer_done);
	651	+ // TODO : HPD Implementation
	652	+
	653	+ } else {
	654	+ DRM_ERROR("DM: Failed to receive correct outbox IRQ !");
	655	+ }
	656	+ }
	657	+
	658	+
	659	+ do {
	660	+ if (dc_dmub_srv_get_dmub_outbox0_msg(dm->dc, &entry)) {
	661	+ trace_amdgpu_dmub_trace_high_irq(entry.trace_code, entry.tick_count,
	662	+ entry.param0, entry.param1);
	663	+
	664	+ DRM_DEBUG_DRIVER("trace_code:%u, tick_count:%u, param0:%u, param1:%u\n",
	665	+ entry.trace_code, entry.tick_count, entry.param0, entry.param1);
	666	+ } else
	667	+ break;
	668	+
	669	+ count++;
	670	+
	671	+ } while (count <= DMUB_TRACE_MAX_READ);
	672	+
	673	+ ASSERT(count <= DMUB_TRACE_MAX_READ);
	674	+}
621	675	#endif
622	676
623	677	static int dm_set_clockgating_state(void *handle,
@@ -938,32 +992,6 @@ static int dm_dmub_hw_init(struct amdgpu_device *adev) · 314 unmodified lines
938	992	}
939	993
940	994	#if defined(CONFIG_DRM_AMD_DC_DCN)
941		−#define DMUB_TRACE_MAX_READ 64
942		−static void dm_dmub_trace_high_irq(void *interrupt_params)
943		−{
944		− struct common_irq_params *irq_params = interrupt_params;
945		− struct amdgpu_device *adev = irq_params->adev;
946		− struct amdgpu_display_manager *dm = &adev->dm;
947		− struct dmcub_trace_buf_entry entry = { 0 };
948		− uint32_t count = 0;
949		−
950		− do {
951		− if (dc_dmub_srv_get_dmub_outbox0_msg(dm->dc, &entry)) {
952		− trace_amdgpu_dmub_trace_high_irq(entry.trace_code, entry.tick_count,
953		− entry.param0, entry.param1);
954		−
955		− DRM_DEBUG_DRIVER("trace_code:%u, tick_count:%u, param0:%u, param1:%u\n",
956		− entry.trace_code, entry.tick_count, entry.param0, entry.param1);
957		− } else
958		− break;
959		−
960		− count++;
961		−
962		− } while (count <= DMUB_TRACE_MAX_READ);
963		−
964		− ASSERT(count <= DMUB_TRACE_MAX_READ);
965		−}
966		−
967	995	static void mmhub_read_system_context(struct amdgpu_device adev, struct dc_phy_addr_space_config pa_config)
968	996	{
969	997	uint64_t pt_base;
@@ -1220,6 +1248,16 @@ static int amdgpu_dm_init(struct amdgpu_device *adev) · 250 unmodified lines
1220	1248	#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY)
1221	1249	adev->dm.crc_rd_wrk = amdgpu_dm_crtc_secure_display_create_work();
1222	1250	#endif
	1251	+ if (dc_enable_dmub_notifications(adev->dm.dc)) {
	1252	+ init_completion(&adev->dm.dmub_aux_transfer_done);
	1253	+ adev->dm.dmub_notify = kzalloc(sizeof(struct dmub_notification), GFP_KERNEL);
	1254	+ if (!adev->dm.dmub_notify) {
	1255	+ DRM_INFO("amdgpu: fail to allocate adev->dm.dmub_notify");
	1256	+ goto error;
	1257	+ }
	1258	+ amdgpu_dm_outbox_init(adev);
	1259	+ }
	1260	+
1223	1261	if (amdgpu_dm_initialize_drm_device(adev)) {
1224	1262	DRM_ERROR(
1225	1263	"amdgpu: failed to initialize sw for display support.\n");
@@ -1293,6 +1331,11 @@ static void amdgpu_dm_fini(struct amdgpu_device *adev) · 67 unmodified lines
1293	1331	adev->dm.dc->ctx->dmub_srv = NULL;
1294	1332	}
1295	1333
	1334	+ if (dc_enable_dmub_notifications(adev->dm.dc)) {
	1335	+ kfree(adev->dm.dmub_notify);
	1336	+ adev->dm.dmub_notify = NULL;
	1337	+ }
	1338	+
1296	1339	if (adev->dm.dmub_bo)
1297	1340	amdgpu_bo_free_kernel(&adev->dm.dmub_bo,
1298	1341	&adev->dm.dmub_bo_gpu_addr,
@@ -2708,8 +2751,7 @@ static void handle_hpd_rx_irq(void *param) · 1409 unmodified lines
2708	2751	* conflict, after implement i2c helper, this mutex should be
2709	2752	* retired.
2710	2753	*/
2711		− if (dc_link->type != dc_connection_mst_branch)
2712		− mutex_lock(&aconnector->hpd_lock);
	2754	+ mutex_lock(&aconnector->hpd_lock);
2713	2755
2714	2756	read_hpd_rx_irq_data(dc_link, &hpd_irq_data);
2715	2757
@@ -2726,13 +2768,15 @@ static void handle_hpd_rx_irq(void *param) · 10 unmodified lines
2726	2768	}
2727	2769	}
2728	2770
2729		− mutex_lock(&adev->dm.dc_lock);
	2771	+ if (!amdgpu_in_reset(adev)) {
	2772	+ mutex_lock(&adev->dm.dc_lock);
2730	2773	#ifdef CONFIG_DRM_AMD_DC_HDCP
2731	2774	result = dc_link_handle_hpd_rx_irq(dc_link, &hpd_irq_data, NULL);
2732	2775	#else
2733	2776	result = dc_link_handle_hpd_rx_irq(dc_link, NULL, NULL);
2734	2777	#endif
2735		− mutex_unlock(&adev->dm.dc_lock);
	2778	+ mutex_unlock(&adev->dm.dc_lock);
	2779	+ }
2736	2780
2737	2781	out:
2738	2782	if (result && !is_mst_root_connector) {
@@ -2776,10 +2820,10 @@ out: · 37 unmodified lines
2776	2820	}
2777	2821	#endif
2778	2822
2779		− if (dc_link->type != dc_connection_mst_branch) {
	2823	+ if (dc_link->type != dc_connection_mst_branch)
2780	2824	drm_dp_cec_irq(&aconnector->dm_dp_aux.aux);
2781		− mutex_unlock(&aconnector->hpd_lock);
2782		− }
	2825	+
	2826	+ mutex_unlock(&aconnector->hpd_lock);
2783	2827	}
2784	2828
2785	2829	static void register_hpd_handlers(struct amdgpu_device *adev)
@@ -3151,38 +3195,51 @@ static int dcn10_register_irq_handlers(struct amdgpu_device *adev) · 365 unmodified lines
3151	3195
3152	3196	}
3153	3197
3154		− if (dc->ctx->dmub_srv) {
3155		− i = DCN_1_0__SRCID__DMCUB_OUTBOX_HIGH_PRIORITY_READY_INT;
3156		− r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, i, &adev->dmub_trace_irq);
	3198	+ /* HPD */
	3199	+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT,
	3200	+ &adev->hpd_irq);
	3201	+ if (r) {
	3202	+ DRM_ERROR("Failed to add hpd irq id!\n");
	3203	+ return r;
	3204	+ }
3157	3205
3158		− if (r) {
3159		− DRM_ERROR("Failed to add dmub trace irq id!\n");
3160		− return r;
3161		− }
	3206	+ register_hpd_handlers(adev);
3162	3207
3163		− int_params.int_context = INTERRUPT_HIGH_IRQ_CONTEXT;
	3208	+ return 0;
	3209	+}
	3210	+/* Register Outbox IRQ sources and initialize IRQ callbacks */
	3211	+static int register_outbox_irq_handlers(struct amdgpu_device *adev)
	3212	+{
	3213	+ struct dc *dc = adev->dm.dc;
	3214	+ struct common_irq_params *c_irq_params;
	3215	+ struct dc_interrupt_params int_params = {0};
	3216	+ int r, i;
	3217	+
	3218	+ int_params.requested_polarity = INTERRUPT_POLARITY_DEFAULT;
	3219	+ int_params.current_polarity = INTERRUPT_POLARITY_DEFAULT;
	3220	+
	3221	+ r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT,
	3222	+ &adev->dmub_outbox_irq);
	3223	+ if (r) {
	3224	+ DRM_ERROR("Failed to add outbox irq id!\n");
	3225	+ return r;
	3226	+ }
	3227	+
	3228	+ if (dc->ctx->dmub_srv) {
	3229	+ i = DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT;
	3230	+ int_params.int_context = INTERRUPT_LOW_IRQ_CONTEXT;
3164	3231	int_params.irq_source =
3165		− dc_interrupt_to_irq_source(dc, i, 0);
	3232	+ dc_interrupt_to_irq_source(dc, i, 0);
3166	3233
3167		− c_irq_params = &adev->dm.dmub_trace_params[0];
	3234	+ c_irq_params = &adev->dm.dmub_outbox_params[0];
3168	3235
3169	3236	c_irq_params->adev = adev;
3170	3237	c_irq_params->irq_src = int_params.irq_source;
3171	3238
3172	3239	amdgpu_dm_irq_register_interrupt(adev, &int_params,
3173		− dm_dmub_trace_high_irq, c_irq_params);
3174		− }
3175		−
3176		− /* HPD */
3177		− r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_DCE, DCN_1_0__SRCID__DC_HPD1_INT,
3178		− &adev->hpd_irq);
3179		− if (r) {
3180		− DRM_ERROR("Failed to add hpd irq id!\n");
3181		− return r;
	3240	+ dm_dmub_outbox1_low_irq, c_irq_params);
3182	3241	}
3183	3242
3184		− register_hpd_handlers(adev);
3185		−
3186	3243	return 0;
3187	3244	}
3188	3245	#endif
@@ -3414,22 +3471,37 @@ static int amdgpu_dm_backlight_update_status(struct backlight_device *bd) · 225 unmodified lines
3414	3471	{
3415	3472	struct amdgpu_display_manager *dm = bl_get_data(bd);
3416	3473	struct amdgpu_dm_backlight_caps caps;
3417		− struct dc_link *link = NULL;
	3474	+ struct dc_link *link[AMDGPU_DM_MAX_NUM_EDP];
3418	3475	u32 brightness;
3419	3476	bool rc;
	3477	+ int i;
3420	3478
3421	3479	amdgpu_dm_update_backlight_caps(dm);
3422	3480	caps = dm->backlight_caps;
3423	3481
3424		− link = (struct dc_link *)dm->backlight_link;
	3482	+ for (i = 0; i < dm->num_of_edps; i++)
	3483	+ link[i] = (struct dc_link *)dm->backlight_link[i];
3425	3484
3426	3485	brightness = convert_brightness_from_user(&caps, bd->props.brightness);
3427	3486	// Change brightness based on AUX property
3428		− if (caps.aux_support)
3429		− rc = dc_link_set_backlight_level_nits(link, true, brightness,
3430		− AUX_BL_DEFAULT_TRANSITION_TIME_MS);
3431		− else
3432		− rc = dc_link_set_backlight_level(dm->backlight_link, brightness, 0);
	3487	+ if (caps.aux_support) {
	3488	+ for (i = 0; i < dm->num_of_edps; i++) {
	3489	+ rc = dc_link_set_backlight_level_nits(link[i], true, brightness,
	3490	+ AUX_BL_DEFAULT_TRANSITION_TIME_MS);
	3491	+ if (!rc) {
	3492	+ DRM_ERROR("DM: Failed to update backlight via AUX on eDP[%d]\n", i);
	3493	+ break;
	3494	+ }
	3495	+ }
	3496	+ } else {
	3497	+ for (i = 0; i < dm->num_of_edps; i++) {
	3498	+ rc = dc_link_set_backlight_level(dm->backlight_link[i], brightness, 0);
	3499	+ if (!rc) {
	3500	+ DRM_ERROR("DM: Failed to update backlight on eDP[%d]\n", i);
	3501	+ break;
	3502	+ }
	3503	+ }
	3504	+ }
3433	3505
3434	3506	return rc ? 0 : 1;
3435	3507	}
@@ -3443,7 +3515,7 @@ static int amdgpu_dm_backlight_get_brightness(struct backlight_device *bd) · 7 unmodified lines
3443	3515	caps = dm->backlight_caps;
3444	3516
3445	3517	if (caps.aux_support) {
3446		− struct dc_link link = (struct dc_link )dm->backlight_link;
	3518	+ struct dc_link link = (struct dc_link )dm->backlight_link[0];
3447	3519	u32 avg, peak;
3448	3520	bool rc;
3449	3521
@@ -3452,7 +3524,7 @@ static int amdgpu_dm_backlight_get_brightness(struct backlight_device *bd) · 2 unmodified lines
3452	3524	return bd->props.brightness;
3453	3525	return convert_brightness_to_user(&caps, avg);
3454	3526	} else {
3455		− int ret = dc_link_get_backlight_level(dm->backlight_link);
	3527	+ int ret = dc_link_get_backlight_level(dm->backlight_link[0]);
3456	3528
3457	3529	if (ret == DC_ERROR_UNEXPECTED)
3458	3530	return bd->props.brightness;
@@ -3549,10 +3621,13 @@ static void register_backlight_device(struct amdgpu_display_manager *dm, · 90 unmodified lines
3549	3621	* DM initialization because not having a backlight control
3550	3622	* is better then a black screen.
3551	3623	*/
3552		− amdgpu_dm_register_backlight_device(dm);
	3624	+ if (!dm->backlight_dev)
	3625	+ amdgpu_dm_register_backlight_device(dm);
3553	3626
3554		− if (dm->backlight_dev)
3555		− dm->backlight_link = link;
	3627	+ if (dm->backlight_dev) {
	3628	+ dm->backlight_link[dm->num_of_edps] = link;
	3629	+ dm->num_of_edps++;
	3630	+ }
3556	3631	}
3557	3632	#endif
3558	3633	}
@@ -3643,6 +3718,22 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev) · 84 unmodified lines
3643	3718	goto fail;
3644	3719	}
3645	3720
	3721	+#if defined(CONFIG_DRM_AMD_DC_DCN)
	3722	+ /* Use Outbox interrupt */
	3723	+ switch (adev->asic_type) {
	3724	+ case CHIP_SIENNA_CICHLID:
	3725	+ case CHIP_NAVY_FLOUNDER:
	3726	+ case CHIP_RENOIR:
	3727	+ if (register_outbox_irq_handlers(dm->adev)) {
	3728	+ DRM_ERROR("DM: Failed to initialize IRQ\n");
	3729	+ goto fail;
	3730	+ }
	3731	+ break;
	3732	+ default:
	3733	+ DRM_DEBUG_KMS("Unsupported ASIC type for outbox: 0x%X\n", adev->asic_type);
	3734	+ }
	3735	+#endif
	3736	+
3646	3737	/* loops over all connectors on the board */
3647	3738	for (i = 0; i < link_cnt; i++) {
3648	3739	struct dc_link *link = NULL;
@@ -6560,13 +6651,13 @@ static int dm_update_mst_vcpi_slots_for_dsc(struct drm_atomic_state *state, · 2911 unmodified lines
6560	6651	{
6561	6652	struct dc_stream_state *stream = NULL;
6562	6653	struct drm_connector *connector;
6563		− struct drm_connector_state new_con_state, old_con_state;
	6654	+ struct drm_connector_state *new_con_state;
6564	6655	struct amdgpu_dm_connector *aconnector;
6565	6656	struct dm_connector_state *dm_conn_state;
6566	6657	int i, j, clock, bpp;
6567	6658	int vcpi, pbn_div, pbn = 0;
6568	6659
6569		− for_each_oldnew_connector_in_state(state, connector, old_con_state, new_con_state, i) {
	6660	+ for_each_new_connector_in_state(state, connector, new_con_state, i) {
6570	6661
6571	6662	aconnector = to_amdgpu_dm_connector(connector);
6572	6663
@@ -8164,15 +8255,14 @@ static void amdgpu_dm_handle_vrr_transition(struct dm_crtc_state *old_state, · 1591 unmodified lines
8164	8255	static void amdgpu_dm_commit_cursors(struct drm_atomic_state *state)
8165	8256	{
8166	8257	struct drm_plane *plane;
8167		− struct drm_plane_state old_plane_state, new_plane_state;
	8258	+ struct drm_plane_state *old_plane_state;
8168	8259	int i;
8169	8260
8170	8261	/*
8171	8262	* TODO: Make this per-stream so we don't issue redundant updates for
8172	8263	* commits with multiple streams.
8173	8264	*/
8174		− for_each_oldnew_plane_in_state(state, plane, old_plane_state,
8175		− new_plane_state, i)
	8265	+ for_each_old_plane_in_state(state, plane, old_plane_state, i)
8176	8266	if (plane->type == DRM_PLANE_TYPE_CURSOR)
8177	8267	handle_cursor_update(plane, old_plane_state);
8178	8268	}
@@ -10668,3 +10758,30 @@ uint32_t dm_read_reg_func(const struct dc_context *ctx, uint32_t address, · 2489 unmodified lines
10668	10758
10669	10759	return value;
10670	10760	}
	10761	+
	10762	+int amdgpu_dm_process_dmub_aux_transfer_sync(struct dc_context *ctx, unsigned int linkIndex,
	10763	+ struct aux_payload payload, enum aux_return_code_type operation_result)
	10764	+{
	10765	+ struct amdgpu_device *adev = ctx->driver_context;
	10766	+ int ret = 0;
	10767	+
	10768	+ dc_process_dmub_aux_transfer_async(ctx->dc, linkIndex, payload);
	10769	+ ret = wait_for_completion_interruptible_timeout(&adev->dm.dmub_aux_transfer_done, 10*HZ);
	10770	+ if (ret == 0) {
	10771	+ *operation_result = AUX_RET_ERROR_TIMEOUT;
	10772	+ return -1;
	10773	+ }
	10774	+ *operation_result = (enum aux_return_code_type)adev->dm.dmub_notify->result;
	10775	+
	10776	+ if (adev->dm.dmub_notify->result == AUX_RET_SUCCESS) {
	10777	+ (*payload->reply) = adev->dm.dmub_notify->aux_reply.command;
	10778	+
	10779	+ // For read case, Copy data to payload
	10780	+ if (!payload->write && adev->dm.dmub_notify->aux_reply.length &&
	10781	+ (*payload->reply == AUX_TRANSACTION_REPLY_AUX_ACK))
	10782	+ memcpy(payload->data, adev->dm.dmub_notify->aux_reply.data,
	10783	+ adev->dm.dmub_notify->aux_reply.length);
	10784	+ }
	10785	+
	10786	+ return adev->dm.dmub_notify->aux_reply.length;
	10787	+}

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.h+21−1

@@ -46,6 +46,7 @@ · 45 unmodified lines
46	46
47	47	#define AMDGPU_DM_MAX_CRTC 6
48	48
	49	+#define AMDGPU_DM_MAX_NUM_EDP 2
49	50	/*
50	51	#include "include/amdgpu_dal_power_if.h"
51	52	#include "amdgpu_dm_irq.h"
@@ -54,6 +55,8 @@ · 2 unmodified lines
54	55	#include "irq_types.h"
55	56	#include "signal_types.h"
56	57	#include "amdgpu_dm_crc.h"
	58	+struct aux_payload;
	59	+enum aux_return_code_type;
57	60
58	61	/* Forward declarations */
59	62	struct amdgpu_device;
@@ -62,6 +65,7 @@ struct dc; · 2 unmodified lines
62	65	struct amdgpu_bo;
63	66	struct dmub_srv;
64	67	struct dc_plane_state;
	68	+struct dmub_notification;
65	69
66	70	struct common_irq_params {
67	71	struct amdgpu_device *adev;
@@ -135,6 +139,10 @@ struct amdgpu_dm_backlight_caps { · 67 unmodified lines
135	139
136	140	/**
137	141	* struct dal_allocation - Tracks mapped FB memory for SMU communication
	142	+ * @list: list of dal allocations
	143	+ * @bo: GPU buffer object
	144	+ * @cpu_ptr: CPU virtual address of the GPU buffer object
	145	+ * @gpu_addr: GPU virtual address of the GPU buffer object
138	146	*/
139	147	struct dal_allocation {
140	148	struct list_head list;
@@ -164,6 +172,7 @@ struct dal_allocation { · 23 unmodified lines
164	172	* @compressor: Frame buffer compression buffer. See &struct dm_compressor_info
165	173	* @force_timing_sync: set via debugfs. When set, indicates that all connected
166	174	* displays will be forced to synchronize.
	175	+ * @dmcub_trace_event_en: enable dmcub trace events
167	176	*/
168	177	struct amdgpu_display_manager {
169	178
@@ -178,6 +187,8 @@ struct amdgpu_display_manager { · 8 unmodified lines
178	187	*/
179	188	struct dmub_srv *dmub_srv;
180	189
	190	+ struct dmub_notification *dmub_notify;
	191	+
181	192	/**
182	193	* @dmub_fb_info:
183	194	*
@@ -349,11 +360,17 @@ struct amdgpu_display_manager { · 165 unmodified lines
349	360	struct common_irq_params
350	361	dmub_trace_params[1];
351	362
	363	+ struct common_irq_params
	364	+ dmub_outbox_params[1];
	365	+
352	366	spinlock_t irq_handler_list_table_lock;
353	367
354	368	struct backlight_device *backlight_dev;
355	369
356		− const struct dc_link *backlight_link;
	370	+ const struct dc_link *backlight_link[AMDGPU_DM_MAX_NUM_EDP];
	371	+
	372	+ uint8_t num_of_edps;
	373	+
357	374	struct amdgpu_dm_backlight_caps backlight_caps;
358	375
359	376	struct mod_freesync *freesync_module;
@@ -418,6 +435,7 @@ struct amdgpu_display_manager { · 58 unmodified lines
418	435	* DAL fb memory allocation list, for communication with SMU.
419	436	*/
420	437	struct list_head da_list;
	438	+ struct completion dmub_aux_transfer_done;
421	439	};
422	440
423	441	enum dsc_clock_force_state {
@@ -600,4 +618,6 @@ void amdgpu_dm_update_connector_after_detect( · 176 unmodified lines
600	618
601	619	extern const struct drm_encoder_helper_funcs amdgpu_dm_encoder_helper_funcs;
602	620
	621	+int amdgpu_dm_process_dmub_aux_transfer_sync(struct dc_context *ctx, unsigned int linkIndex,
	622	+ struct aux_payload payload, enum aux_return_code_type operation_result);
603	623	#endif /* __AMDGPU_DM_H__ */

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_debugfs.c+66−40

@@ -925,6 +925,22 @@ static int hdcp_sink_capability_show(struct seq_file m, void data) · 924 unmodified lines
925	925	return 0;
926	926	}
927	927	#endif
	928	+
	929	+/*
	930	+ * Returns whether the connected display is internal and not hotpluggable.
	931	+ * Example usage: cat /sys/kernel/debug/dri/0/DP-1/internal_display
	932	+ */
	933	+static int internal_display_show(struct seq_file m, void data)
	934	+{
	935	+ struct drm_connector *connector = m->private;
	936	+ struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
	937	+ struct dc_link *link = aconnector->dc_link;
	938	+
	939	+ seq_printf(m, "Internal: %u\n", link->is_internal_display);
	940	+
	941	+ return 0;
	942	+}
	943	+
928	944	/* function description
929	945	*
930	946	* generic SDP message access for testing
@@ -2361,6 +2377,44 @@ unlock: · 1430 unmodified lines
2361	2377	return size;
2362	2378	}
2363	2379
	2380	+/*
	2381	+ * Backlight at this moment. Read only.
	2382	+ * As written to display, taking ABM and backlight lut into account.
	2383	+ * Ranges from 0x0 to 0x10000 (= 100% PWM)
	2384	+ *
	2385	+ * Example usage: cat /sys/kernel/debug/dri/0/eDP-1/current_backlight
	2386	+ */
	2387	+static int current_backlight_show(struct seq_file m, void unused)
	2388	+{
	2389	+ struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(m->private);
	2390	+ struct dc_link *link = aconnector->dc_link;
	2391	+ unsigned int backlight;
	2392	+
	2393	+ backlight = dc_link_get_backlight_level(link);
	2394	+ seq_printf(m, "0x%x\n", backlight);
	2395	+
	2396	+ return 0;
	2397	+}
	2398	+
	2399	+/*
	2400	+ * Backlight value that is being approached. Read only.
	2401	+ * As written to display, taking ABM and backlight lut into account.
	2402	+ * Ranges from 0x0 to 0x10000 (= 100% PWM)
	2403	+ *
	2404	+ * Example usage: cat /sys/kernel/debug/dri/0/eDP-1/target_backlight
	2405	+ */
	2406	+static int target_backlight_show(struct seq_file m, void unused)
	2407	+{
	2408	+ struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(m->private);
	2409	+ struct dc_link *link = aconnector->dc_link;
	2410	+ unsigned int backlight;
	2411	+
	2412	+ backlight = dc_link_get_target_backlight_pwm(link);
	2413	+ seq_printf(m, "0x%x\n", backlight);
	2414	+
	2415	+ return 0;
	2416	+}
	2417	+
2364	2418	DEFINE_SHOW_ATTRIBUTE(dp_dsc_fec_support);
2365	2419	DEFINE_SHOW_ATTRIBUTE(dmub_fw_state);
2366	2420	DEFINE_SHOW_ATTRIBUTE(dmub_tracebuffer);
@@ -2369,6 +2423,7 @@ DEFINE_SHOW_ATTRIBUTE(dp_lttpr_status); · 2 unmodified lines
2369	2423	#ifdef CONFIG_DRM_AMD_DC_HDCP
2370	2424	DEFINE_SHOW_ATTRIBUTE(hdcp_sink_capability);
2371	2425	#endif
	2426	+DEFINE_SHOW_ATTRIBUTE(internal_display);
2372	2427
2373	2428	static const struct file_operations dp_dsc_clock_en_debugfs_fops = {
2374	2429	.owner = THIS_MODULE,
@@ -2594,13 +2649,17 @@ DEFINE_DEBUGFS_ATTRIBUTE(dmcub_trace_event_state_fops, dmcub_trace_event_state_g · 219 unmodified lines
2594	2649
2595	2650	DEFINE_DEBUGFS_ATTRIBUTE(psr_fops, psr_get, NULL, "%llu\n");
2596	2651
	2652	+DEFINE_SHOW_ATTRIBUTE(current_backlight);
	2653	+DEFINE_SHOW_ATTRIBUTE(target_backlight);
	2654	+
2597	2655	static const struct {
2598	2656	char *name;
2599	2657	const struct file_operations *fops;
2600	2658	} connector_debugfs_entries[] = {
2601	2659	{"force_yuv420_output", &force_yuv420_output_fops},
2602	2660	{"output_bpc", &output_bpc_fops},
2603		− {"trigger_hotplug", &trigger_hotplug_debugfs_fops}
	2661	+ {"trigger_hotplug", &trigger_hotplug_debugfs_fops},
	2662	+ {"internal_display", &internal_display_fops}
2604	2663	};
2605	2664
2606	2665	void connector_debugfs_init(struct amdgpu_dm_connector *connector)
@@ -2616,8 +2675,13 @@ void connector_debugfs_init(struct amdgpu_dm_connector *connector) · 9 unmodified lines
2616	2675	dp_debugfs_entries[i].fops);
2617	2676	}
2618	2677	}
2619		− if (connector->base.connector_type == DRM_MODE_CONNECTOR_eDP)
	2678	+ if (connector->base.connector_type == DRM_MODE_CONNECTOR_eDP) {
2620	2679	debugfs_create_file_unsafe("psr_state", 0444, dir, connector, &psr_fops);
	2680	+ debugfs_create_file("amdgpu_current_backlight_pwm", 0444, dir, connector,
	2681	+ &current_backlight_fops);
	2682	+ debugfs_create_file("amdgpu_target_backlight_pwm", 0444, dir, connector,
	2683	+ &target_backlight_fops);
	2684	+ }
2621	2685
2622	2686	for (i = 0; i < ARRAY_SIZE(connector_debugfs_entries); i++) {
2623	2687	debugfs_create_file(connector_debugfs_entries[i].name,
@@ -2920,38 +2984,6 @@ static ssize_t dtn_log_write( · 296 unmodified lines
2920	2984	return size;
2921	2985	}
2922	2986
2923		−/*
2924		− * Backlight at this moment. Read only.
2925		− * As written to display, taking ABM and backlight lut into account.
2926		− * Ranges from 0x0 to 0x10000 (= 100% PWM)
2927		− */
2928		−static int current_backlight_show(struct seq_file m, void unused)
2929		−{
2930		− struct amdgpu_device adev = (struct amdgpu_device )m->private;
2931		− struct amdgpu_display_manager *dm = &adev->dm;
2932		−
2933		− unsigned int backlight = dc_link_get_backlight_level(dm->backlight_link);
2934		−
2935		− seq_printf(m, "0x%x\n", backlight);
2936		− return 0;
2937		−}
2938		−
2939		−/*
2940		− * Backlight value that is being approached. Read only.
2941		− * As written to display, taking ABM and backlight lut into account.
2942		− * Ranges from 0x0 to 0x10000 (= 100% PWM)
2943		− */
2944		−static int target_backlight_show(struct seq_file m, void unused)
2945		−{
2946		− struct amdgpu_device adev = (struct amdgpu_device )m->private;
2947		− struct amdgpu_display_manager *dm = &adev->dm;
2948		−
2949		− unsigned int backlight = dc_link_get_target_backlight_pwm(dm->backlight_link);
2950		−
2951		− seq_printf(m, "0x%x\n", backlight);
2952		− return 0;
2953		−}
2954		−
2955	2987	static int mst_topo_show(struct seq_file m, void unused)
2956	2988	{
2957	2989	struct amdgpu_device adev = (struct amdgpu_device )m->private;
@@ -3134,8 +3166,6 @@ static int visual_confirm_get(void data, u64 val) · 176 unmodified lines
3134	3166	return 0;
3135	3167	}
3136	3168
3137		−DEFINE_SHOW_ATTRIBUTE(current_backlight);
3138		−DEFINE_SHOW_ATTRIBUTE(target_backlight);
3139	3169	DEFINE_SHOW_ATTRIBUTE(mst_topo);
3140	3170	DEFINE_DEBUGFS_ATTRIBUTE(visual_confirm_fops, visual_confirm_get,
3141	3171	visual_confirm_set, "%llu\n");
@@ -3215,10 +3245,6 @@ void dtn_debugfs_init(struct amdgpu_device *adev) · 73 unmodified lines
3215	3245	struct drm_minor *minor = adev_to_drm(adev)->primary;
3216	3246	struct dentry *root = minor->debugfs_root;
3217	3247
3218		− debugfs_create_file("amdgpu_current_backlight_pwm", 0444,
3219		− root, adev, &current_backlight_fops);
3220		− debugfs_create_file("amdgpu_target_backlight_pwm", 0444,
3221		− root, adev, &target_backlight_fops);
3222	3248	debugfs_create_file("amdgpu_mst_topology", 0444, root,
3223	3249	adev, &mst_topo_fops);
3224	3250	debugfs_create_file("amdgpu_dm_dtn_log", 0644, root, adev,

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c+14−5

@@ -544,8 +544,10 @@ bool dm_helpers_dp_write_dsc_enable( · 543 unmodified lines
544	544	ret = drm_dp_dpcd_write(aconnector->dsc_aux, DP_DSC_ENABLE, &enable_dsc, 1);
545	545	}
546	546
547		− if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT)
548		− return dm_helpers_dp_write_dpcd(ctx, stream->link, DP_DSC_ENABLE, &enable_dsc, 1);
	547	+ if (stream->signal == SIGNAL_TYPE_DISPLAY_PORT) {
	548	+ ret = dm_helpers_dp_write_dpcd(ctx, stream->link, DP_DSC_ENABLE, &enable_dsc, 1);
	549	+ DC_LOG_DC("Send DSC %s to sst display\n", enable_dsc ? "enable" : "disable");
	550	+ }
549	551
550	552	return (ret > 0);
551	553	}
@@ -640,7 +642,14 @@ enum dc_edid_status dm_helpers_read_local_edid( · 88 unmodified lines
640	642
641	643	return edid_status;
642	644	}
643		−
	645	+int dm_helper_dmub_aux_transfer_sync(
	646	+ struct dc_context *ctx,
	647	+ const struct dc_link *link,
	648	+ struct aux_payload *payload,
	649	+ enum aux_return_code_type *operation_result)
	650	+{
	651	+ return amdgpu_dm_process_dmub_aux_transfer_sync(ctx, link->link_index, payload, operation_result);
	652	+}
644	653	void dm_set_dcn_clocks(struct dc_context ctx, struct dc_clocks clks)
645	654	{
646	655	/* TODO: something */
@@ -698,12 +707,12 @@ void dm_helpers_free_gpu_mem( · 51 unmodified lines
698	707	}
699	708	}
700	709
701		−bool dm_helpers_dmub_outbox0_interrupt_control(struct dc_context *ctx, bool enable)
	710	+bool dm_helpers_dmub_outbox_interrupt_control(struct dc_context *ctx, bool enable)
702	711	{
703	712	enum dc_irq_source irq_source;
704	713	bool ret;
705	714
706		− irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX0;
	715	+ irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX;
707	716
708	717	ret = dc_interrupt_set(ctx->dc, irq_source, enable);
709	718

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.c+26−1

@@ -769,6 +769,18 @@ static int amdgpu_dm_set_vline0_irq_state(struct amdgpu_device *adev, · 768 unmodified lines
769	769	__func__);
770	770	}
771	771
	772	+static int amdgpu_dm_set_dmub_outbox_irq_state(struct amdgpu_device *adev,
	773	+ struct amdgpu_irq_src *source,
	774	+ unsigned int crtc_id,
	775	+ enum amdgpu_interrupt_state state)
	776	+{
	777	+ enum dc_irq_source irq_source = DC_IRQ_SOURCE_DMCUB_OUTBOX;
	778	+ bool st = (state == AMDGPU_IRQ_STATE_ENABLE);
	779	+
	780	+ dc_interrupt_set(adev->dm.dc, irq_source, st);
	781	+ return 0;
	782	+}
	783	+
772	784	static int amdgpu_dm_set_vupdate_irq_state(struct amdgpu_device *adev,
773	785	struct amdgpu_irq_src *source,
774	786	unsigned int crtc_id,
@@ -805,6 +817,11 @@ static const struct amdgpu_irq_src_funcs dm_vline0_irq_funcs = { · 30 unmodified lines
805	817	.process = amdgpu_dm_irq_handler,
806	818	};
807	819
	820	+static const struct amdgpu_irq_src_funcs dm_dmub_outbox_irq_funcs = {
	821	+ .set = amdgpu_dm_set_dmub_outbox_irq_state,
	822	+ .process = amdgpu_dm_irq_handler,
	823	+};
	824	+
808	825	static const struct amdgpu_irq_src_funcs dm_vupdate_irq_funcs = {
809	826	.set = amdgpu_dm_set_vupdate_irq_state,
810	827	.process = amdgpu_dm_irq_handler,
@@ -827,13 +844,15 @@ static const struct amdgpu_irq_src_funcs dm_hpd_irq_funcs = { · 16 unmodified lines
827	844
828	845	void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev)
829	846	{
830		−
831	847	adev->crtc_irq.num_types = adev->mode_info.num_crtc;
832	848	adev->crtc_irq.funcs = &dm_crtc_irq_funcs;
833	849
834	850	adev->vline0_irq.num_types = adev->mode_info.num_crtc;
835	851	adev->vline0_irq.funcs = &dm_vline0_irq_funcs;
836	852
	853	+ adev->dmub_outbox_irq.num_types = 1;
	854	+ adev->dmub_outbox_irq.funcs = &dm_dmub_outbox_irq_funcs;
	855	+
837	856	adev->vupdate_irq.num_types = adev->mode_info.num_crtc;
838	857	adev->vupdate_irq.funcs = &dm_vupdate_irq_funcs;
839	858
@@ -846,6 +865,12 @@ void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev) · 6 unmodified lines
846	865	adev->hpd_irq.num_types = adev->mode_info.num_hpd;
847	866	adev->hpd_irq.funcs = &dm_hpd_irq_funcs;
848	867	}
	868	+void amdgpu_dm_outbox_init(struct amdgpu_device *adev)
	869	+{
	870	+ dc_interrupt_set(adev->dm.dc,
	871	+ DC_IRQ_SOURCE_DMCUB_OUTBOX,
	872	+ true);
	873	+}
849	874
850	875	/**
851	876	* amdgpu_dm_hpd_init - hpd setup callback.

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_irq.h+1−0

@@ -82,6 +82,7 @@ void amdgpu_dm_irq_unregister_interrupt(struct amdgpu_device *adev, · 81 unmodified lines
82	82
83	83	void amdgpu_dm_set_irq_funcs(struct amdgpu_device *adev);
84	84
	85	+void amdgpu_dm_outbox_init(struct amdgpu_device *adev);
85	86	void amdgpu_dm_hpd_init(struct amdgpu_device *adev);
86	87	void amdgpu_dm_hpd_fini(struct amdgpu_device *adev);
87	88

drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_mst_types.c+3−0

@@ -278,6 +278,9 @@ dm_dp_mst_detect(struct drm_connector *connector, · 277 unmodified lines
278	278	struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector);
279	279	struct amdgpu_dm_connector *master = aconnector->mst_port;
280	280
	281	+ if (drm_connector_is_unregistered(connector))
	282	+ return connector_status_disconnected;
	283	+
281	284	return drm_dp_mst_detect_port(connector, ctx, &master->mst_mgr,
282	285	aconnector->port);
283	286	}

drivers/gpu/drm/amd/display/dc/Makefile+1−1

@@ -54,7 +54,7 @@ AMD_DC = $(addsuffix /Makefile, $(addprefix $(FULL_AMD_DISPLAY_PATH)/dc/,$(DC_LI · 53 unmodified lines
54	54
55	55	include $(AMD_DC)
56	56
57		−DISPLAY_CORE = dc.o dc_link.o dc_resource.o dc_hw_sequencer.o dc_sink.o \
	57	+DISPLAY_CORE = dc.o dc_stat.o dc_link.o dc_resource.o dc_hw_sequencer.o dc_sink.o \
58	58	dc_surface.o dc_link_hwss.o dc_link_dp.o dc_link_ddc.o dc_debug.o dc_stream.o \
59	59	dc_link_enc_cfg.o
60	60

drivers/gpu/drm/amd/display/dc/bios/bios_parser2.c+2−0

@@ -836,8 +836,10 @@ static enum bp_result bios_parser_get_spread_spectrum_info( · 835 unmodified lines
836	836	return get_ss_info_v4_1(bp, signal, index, ss_info);
837	837	case 2:
838	838	case 3:
	839	+ case 4:
839	840	return get_ss_info_v4_2(bp, signal, index, ss_info);
840	841	default:
	842	+ ASSERT(0);
841	843	break;
842	844	}
843	845	break;

drivers/gpu/drm/amd/display/dc/clk_mgr/dcn21/rn_clk_mgr.c+59−85

@@ -106,10 +106,10 @@ static void rn_update_clocks_update_dpp_dto(struct clk_mgr_internal *clk_mgr, · 105 unmodified lines
106	106	for (i = 0; i < clk_mgr->base.ctx->dc->res_pool->pipe_count; i++) {
107	107	int dpp_inst, dppclk_khz, prev_dppclk_khz;
108	108
109		− /* Loop index will match dpp->inst if resource exists,
110		− * and we want to avoid dependency on dpp object
	109	+ /* Loop index may not match dpp->inst if some pipes disabled,
	110	+ * so select correct inst from res_pool
111	111	*/
112		− dpp_inst = i;
	112	+ dpp_inst = clk_mgr->base.ctx->dc->res_pool->dpps[i]->inst;
113	113	dppclk_khz = context->res_ctx.pipe_ctx[i].plane_res.bw.dppclk_khz;
114	114
115	115	prev_dppclk_khz = clk_mgr->dccg->pipe_dppclk_khz[i];
@@ -128,7 +128,7 @@ void rn_update_clocks(struct clk_mgr *clk_mgr_base, · 12 unmodified lines
128	128	struct clk_mgr_internal *clk_mgr = TO_CLK_MGR_INTERNAL(clk_mgr_base);
129	129	struct dc_clocks *new_clocks = &context->bw_ctx.bw.dcn.clk;
130	130	struct dc *dc = clk_mgr_base->ctx->dc;
131		− int display_count, i;
	131	+ int display_count;
132	132	bool update_dppclk = false;
133	133	bool update_dispclk = false;
134	134	bool dpp_clock_lowered = false;
@@ -210,14 +210,6 @@ void rn_update_clocks(struct clk_mgr *clk_mgr_base, · 75 unmodified lines
210	210	clk_mgr_base->clks.dppclk_khz,
211	211	safe_to_lower);
212	212
213		− for (i = 0; i < context->stream_count; i++) {
214		− if (context->streams[i]->signal == SIGNAL_TYPE_EDP &&
215		− context->streams[i]->apply_seamless_boot_optimization) {
216		− dc_wait_for_vblank(dc, context->streams[i]);
217		− break;
218		− }
219		− }
220		−
221	213	clk_mgr_base->clks.actual_dppclk_khz =
222	214	rn_vbios_smu_set_dppclk(clk_mgr, clk_mgr_base->clks.dppclk_khz);
223	215
@@ -769,43 +761,6 @@ static struct wm_table ddr4_wm_table_rn = { · 545 unmodified lines
769	761	}
770	762	};
771	763
772		−static struct wm_table ddr4_1R_wm_table_rn = {
773		− .entries = {
774		− {
775		− .wm_inst = WM_A,
776		− .wm_type = WM_TYPE_PSTATE_CHG,
777		− .pstate_latency_us = 11.72,
778		− .sr_exit_time_us = 13.90,
779		− .sr_enter_plus_exit_time_us = 14.80,
780		− .valid = true,
781		− },
782		− {
783		− .wm_inst = WM_B,
784		− .wm_type = WM_TYPE_PSTATE_CHG,
785		− .pstate_latency_us = 11.72,
786		− .sr_exit_time_us = 13.90,
787		− .sr_enter_plus_exit_time_us = 14.80,
788		− .valid = true,
789		− },
790		− {
791		− .wm_inst = WM_C,
792		− .wm_type = WM_TYPE_PSTATE_CHG,
793		− .pstate_latency_us = 11.72,
794		− .sr_exit_time_us = 13.90,
795		− .sr_enter_plus_exit_time_us = 14.80,
796		− .valid = true,
797		− },
798		− {
799		− .wm_inst = WM_D,
800		− .wm_type = WM_TYPE_PSTATE_CHG,
801		− .pstate_latency_us = 11.72,
802		− .sr_exit_time_us = 13.90,
803		− .sr_enter_plus_exit_time_us = 14.80,
804		− .valid = true,
805		− },
806		− }
807		−};
808		−
809	764	static struct wm_table lpddr4_wm_table_rn = {
810	765	.entries = {
811	766	{
@@ -842,46 +797,67 @@ static struct wm_table lpddr4_wm_table_rn = { · 30 unmodified lines
842	797	},
843	798	}
844	799	};
845		−static unsigned int find_socclk_for_voltage(struct dpm_clocks *clock_table, unsigned int voltage)
	800	+
	801	+static unsigned int find_max_fclk_for_voltage(struct dpm_clocks *clock_table,
	802	+ unsigned int voltage)
846	803	{
847	804	int i;
	805	+ uint32_t max_clk = 0;
848	806
849		− for (i = 0; i < PP_SMU_NUM_SOCCLK_DPM_LEVELS; i++) {
850		− if (clock_table->SocClocks[i].Vol == voltage)
851		− return clock_table->SocClocks[i].Freq;
	807	+ for (i = 0; i < PP_SMU_NUM_FCLK_DPM_LEVELS; i++) {
	808	+ if (clock_table->FClocks[i].Vol <= voltage) {
	809	+ max_clk = clock_table->FClocks[i].Freq > max_clk ?
	810	+ clock_table->FClocks[i].Freq : max_clk;
	811	+ }
	812	+ }
	813	+
	814	+ return max_clk;
	815	+}
	816	+
	817	+static unsigned int find_max_memclk_for_voltage(struct dpm_clocks *clock_table,
	818	+ unsigned int voltage)
	819	+{
	820	+ int i;
	821	+ uint32_t max_clk = 0;
	822	+
	823	+ for (i = 0; i < PP_SMU_NUM_MEMCLK_DPM_LEVELS; i++) {
	824	+ if (clock_table->MemClocks[i].Vol <= voltage) {
	825	+ max_clk = clock_table->MemClocks[i].Freq > max_clk ?
	826	+ clock_table->MemClocks[i].Freq : max_clk;
	827	+ }
852	828	}
853	829
854		− ASSERT(0);
855		− return 0;
	830	+ return max_clk;
856	831	}
857		−static unsigned int find_dcfclk_for_voltage(struct dpm_clocks *clock_table, unsigned int voltage)
	832	+
	833	+static unsigned int find_max_socclk_for_voltage(struct dpm_clocks *clock_table,
	834	+ unsigned int voltage)
858	835	{
859	836	int i;
	837	+ uint32_t max_clk = 0;
860	838
861		− for (i = 0; i < PP_SMU_NUM_DCFCLK_DPM_LEVELS; i++) {
862		− if (clock_table->DcfClocks[i].Vol == voltage)
863		− return clock_table->DcfClocks[i].Freq;
	839	+ for (i = 0; i < PP_SMU_NUM_SOCCLK_DPM_LEVELS; i++) {
	840	+ if (clock_table->SocClocks[i].Vol <= voltage) {
	841	+ max_clk = clock_table->SocClocks[i].Freq > max_clk ?
	842	+ clock_table->SocClocks[i].Freq : max_clk;
	843	+ }
864	844	}
865	845
866		− ASSERT(0);
867		− return 0;
	846	+ return max_clk;
868	847	}
869	848
870	849	static void rn_clk_mgr_helper_populate_bw_params(struct clk_bw_params bw_params, struct dpm_clocks clock_table, struct integrated_info *bios_info)
871	850	{
872	851	int i, j = 0;
	852	+ unsigned int volt;
873	853
874	854	j = -1;
875	855
876		− ASSERT(PP_SMU_NUM_FCLK_DPM_LEVELS <= MAX_NUM_DPM_LVL);
877		−
878		− /* Find lowest DPM, FCLK is filled in reverse order*/
879		−
880		− for (i = PP_SMU_NUM_FCLK_DPM_LEVELS - 1; i >= 0; i--) {
881		− if (clock_table->FClocks[i].Freq != 0 && clock_table->FClocks[i].Vol != 0) {
	856	+ /* Find max DPM */
	857	+ for (i = 0; i < PP_SMU_NUM_DCFCLK_DPM_LEVELS; ++i) {
	858	+ if (clock_table->DcfClocks[i].Freq != 0 &&
	859	+ clock_table->DcfClocks[i].Vol != 0)
882	860	j = i;
883		− break;
884		− }
885	861	}
886	862
887	863	if (j == -1) {
@@ -892,13 +868,18 @@ static void rn_clk_mgr_helper_populate_bw_params(struct clk_bw_params *bw_params · 4 unmodified lines
892	868
893	869	bw_params->clk_table.num_entries = j + 1;
894	870
895		− for (i = 0; i < bw_params->clk_table.num_entries; i++, j--) {
896		− bw_params->clk_table.entries[i].fclk_mhz = clock_table->FClocks[j].Freq;
897		− bw_params->clk_table.entries[i].memclk_mhz = clock_table->MemClocks[j].Freq;
898		− bw_params->clk_table.entries[i].voltage = clock_table->FClocks[j].Vol;
899		− bw_params->clk_table.entries[i].dcfclk_mhz = find_dcfclk_for_voltage(clock_table, clock_table->FClocks[j].Vol);
900		− bw_params->clk_table.entries[i].socclk_mhz = find_socclk_for_voltage(clock_table,
901		− bw_params->clk_table.entries[i].voltage);
	871	+ for (i = 0; i < bw_params->clk_table.num_entries; i++) {
	872	+ volt = clock_table->DcfClocks[i].Vol;
	873	+
	874	+ bw_params->clk_table.entries[i].voltage = volt;
	875	+ bw_params->clk_table.entries[i].dcfclk_mhz =
	876	+ clock_table->DcfClocks[i].Freq;
	877	+ bw_params->clk_table.entries[i].fclk_mhz =
	878	+ find_max_fclk_for_voltage(clock_table, volt);
	879	+ bw_params->clk_table.entries[i].memclk_mhz =
	880	+ find_max_memclk_for_voltage(clock_table, volt);
	881	+ bw_params->clk_table.entries[i].socclk_mhz =
	882	+ find_max_socclk_for_voltage(clock_table, volt);
902	883	}
903	884
904	885	bw_params->vram_type = bios_info->memory_type;
@@ -990,12 +971,8 @@ void rn_clk_mgr_construct( · 85 unmodified lines
990	971	} else {
991	972	if (is_green_sardine)
992	973	rn_bw_params.wm_table = ddr4_wm_table_gs;
993		− else {
994		− if (ctx->dc->config.is_single_rank_dimm)
995		− rn_bw_params.wm_table = ddr4_1R_wm_table_rn;
996		− else
997		− rn_bw_params.wm_table = ddr4_wm_table_rn;
998		− }
	974	+ else
	975	+ rn_bw_params.wm_table = ddr4_wm_table_rn;
999	976	}
1000	977	/* Saved clocks configured at boot for debug purposes */
1001	978	rn_dump_clk_registers(&clk_mgr->base.boot_snapshot, &clk_mgr->base, &log_info);
@@ -1013,9 +990,6 @@ void rn_clk_mgr_construct( · 11 unmodified lines
1013	990	if (status == PP_SMU_RESULT_OK &&
1014	991	ctx->dc_bios && ctx->dc_bios->integrated_info) {
1015	992	rn_clk_mgr_helper_populate_bw_params (clk_mgr->base.bw_params, &clock_table, ctx->dc_bios->integrated_info);
1016		− /* treat memory config as single channel if memory is asymmetrics. */
1017		− if (ctx->dc->config.is_asymmetric_memory)
1018		− clk_mgr->base.bw_params->num_channels = 1;
1019	993	}
1020	994	}
1021	995

drivers/gpu/drm/amd/display/dc/core/dc.c+1−15

@@ -59,7 +59,6 @@ · 58 unmodified lines
59	59	#include "dc_link_ddc.h"
60	60	#include "dm_helpers.h"
61	61	#include "mem_input.h"
62		−#include "hubp.h"
63	62
64	63	#include "dc_link_dp.h"
65	64	#include "dc_dmub_srv.h"
@@ -3219,19 +3218,6 @@ void dc_link_remove_remote_sink(struct dc_link link, struct dc_sink sink) · 3153 unmodified lines
3219	3218	}
3220	3219	}
3221	3220
3222		−void dc_wait_for_vblank(struct dc dc, struct dc_stream_state stream)
3223		−{
3224		− int i;
3225		−
3226		− for (i = 0; i < dc->res_pool->pipe_count; i++)
3227		− if (dc->current_state->res_ctx.pipe_ctx[i].stream == stream) {
3228		− struct timing_generator *tg =
3229		− dc->current_state->res_ctx.pipe_ctx[i].stream_res.tg;
3230		− tg->funcs->wait_for_state(tg, CRTC_STATE_VBLANK);
3231		− break;
3232		− }
3233		−}
3234		−
3235	3221	void get_clock_requirements_for_state(struct dc_state state, struct AsicStateEx info)
3236	3222	{
3237	3223	info->displayClock = (unsigned int)state->bw_ctx.bw.dcn.clk.dispclk_khz;
@@ -3287,7 +3273,7 @@ void dc_allow_idle_optimizations(struct dc *dc, bool allow) · 49 unmodified lines
3287	3273	if (dc->debug.disable_idle_power_optimizations)
3288	3274	return;
3289	3275
3290		− if (dc->clk_mgr->funcs->is_smu_present)
	3276	+ if (dc->clk_mgr != NULL && dc->clk_mgr->funcs->is_smu_present)
3291	3277	if (!dc->clk_mgr->funcs->is_smu_present(dc->clk_mgr))
3292	3278	return;
3293	3279

drivers/gpu/drm/amd/display/dc/core/dc_link.c+71−31

@@ -48,6 +48,7 @@ · 47 unmodified lines
48	48	#include "dce/dmub_psr.h"
49	49	#include "dmub/dmub_srv.h"
50	50	#include "inc/hw/panel_cntl.h"
	51	+#include "inc/link_enc_cfg.h"
51	52
52	53	#define DC_LOGGER_INIT(logger)
53	54
@@ -247,6 +248,16 @@ bool dc_link_detect_sink(struct dc_link link, enum dc_connection_type type) · 193 unmodified lines
247	248	link->dc->hwss.edp_wait_for_hpd_ready(link, true);
248	249	}
249	250
	251	+ /* Link may not have physical HPD pin. */
	252	+ if (link->ep_type != DISPLAY_ENDPOINT_PHY) {
	253	+ if (link->hpd_status)
	254	+ *type = dc_connection_single;
	255	+ else
	256	+ *type = dc_connection_none;
	257	+
	258	+ return true;
	259	+ }
	260	+
250	261	/* todo: may need to lock gpio access */
251	262	hpd_pin = get_hpd_gpio(link->ctx->dc_bios, link->link_id,
252	263	link->ctx->gpio_service);
@@ -432,8 +443,18 @@ bool dc_link_is_dp_sink_present(struct dc_link *link) · 179 unmodified lines
432	443	static enum signal_type link_detect_sink(struct dc_link *link,
433	444	enum dc_detect_reason reason)
434	445	{
435		− enum signal_type result = get_basic_signal_type(link->link_enc->id,
436		− link->link_id);
	446	+ enum signal_type result;
	447	+ struct graphics_object_id enc_id;
	448	+
	449	+ if (link->is_dig_mapping_flexible)
	450	+ enc_id = (struct graphics_object_id){.id = ENCODER_ID_UNKNOWN};
	451	+ else
	452	+ enc_id = link->link_enc->id;
	453	+ result = get_basic_signal_type(enc_id, link->link_id);
	454	+
	455	+ /* Use basic signal type for link without physical connector. */
	456	+ if (link->ep_type != DISPLAY_ENDPOINT_PHY)
	457	+ return result;
437	458
438	459	/* Internal digital encoder will detect only dongles
439	460	* that require digital signal
@@ -762,19 +783,20 @@ static bool detect_dp(struct dc_link *link, · 322 unmodified lines
762	783	}
763	784
764	785	if (link->type != dc_connection_mst_branch &&
765		− is_dp_active_dongle(link)) {
766		− /* DP active dongles */
767		− link->type = dc_connection_active_dongle;
	786	+ is_dp_branch_device(link)) {
	787	+ /* DP SST branch */
	788	+ link->type = dc_connection_sst_branch;
768	789	if (!link->dpcd_caps.sink_count.bits.SINK_COUNT) {
769	790	/*
770		− * active dongle unplug processing for short irq
	791	+ * SST branch unplug processing for short irq
771	792	*/
772	793	link_disconnect_sink(link);
773	794	return true;
774	795	}
775	796
776		− if (link->dpcd_caps.dongle_type !=
777		− DISPLAY_DONGLE_DP_HDMI_CONVERTER)
	797	+ if (is_dp_active_dongle(link) &&
	798	+ (link->dpcd_caps.dongle_type !=
	799	+ DISPLAY_DONGLE_DP_HDMI_CONVERTER))
778	800	*converter_disable_audio = true;
779	801	}
780	802	} else {
@@ -954,7 +976,8 @@ static bool dc_link_detect_helper(struct dc_link *link, · 173 unmodified lines
954	976
955	977	case SIGNAL_TYPE_DISPLAY_PORT: {
956	978	/* wa HPD high coming too early*/
957		− if (link->link_enc->features.flags.bits.DP_IS_USB_C == 1) {
	979	+ if (link->ep_type == DISPLAY_ENDPOINT_PHY &&
	980	+ link->link_enc->features.flags.bits.DP_IS_USB_C == 1) {
958	981	/* if alt mode times out, return false */
959	982	if (!wait_for_entering_dp_alt_mode(link))
960	983	return false;
@@ -974,8 +997,8 @@ static bool dc_link_detect_helper(struct dc_link *link, · 13 unmodified lines
974	997	sizeof(struct dpcd_caps)))
975	998	same_dpcd = false;
976	999	}
977		− /* Active dongle downstream unplug*/
978		− if (link->type == dc_connection_active_dongle &&
	1000	+ /* Active SST downstream branch device unplug*/
	1001	+ if (link->type == dc_connection_sst_branch &&
979	1002	link->dpcd_caps.sink_count.bits.SINK_COUNT == 0) {
980	1003	if (prev_sink)
981	1004	/* Downstream unplug */
@@ -1206,14 +1229,25 @@ bool dc_link_detect(struct dc_link *link, enum dc_detect_reason reason) · 224 unmodified lines
1206	1229	{
1207	1230	const struct dc *dc = link->dc;
1208	1231	bool ret;
	1232	+ bool can_apply_seamless_boot = false;
	1233	+ int i;
	1234	+
	1235	+ for (i = 0; i < dc->current_state->stream_count; i++) {
	1236	+ if (dc->current_state->streams[i]->apply_seamless_boot_optimization) {
	1237	+ can_apply_seamless_boot = true;
	1238	+ break;
	1239	+ }
	1240	+ }
1209	1241
1210	1242	/* get out of low power state */
1211		− clk_mgr_exit_optimized_pwr_state(dc, dc->clk_mgr);
	1243	+ if (!can_apply_seamless_boot && reason != DETECT_REASON_BOOT)
	1244	+ clk_mgr_exit_optimized_pwr_state(dc, dc->clk_mgr);
1212	1245
1213	1246	ret = dc_link_detect_helper(link, reason);
1214	1247
1215	1248	/* Go back to power optimized state */
1216		− clk_mgr_optimize_pwr_state(dc, dc->clk_mgr);
	1249	+ if (!can_apply_seamless_boot && reason != DETECT_REASON_BOOT)
	1250	+ clk_mgr_optimize_pwr_state(dc, dc->clk_mgr);
1217	1251
1218	1252	return ret;
1219	1253	}
@@ -1716,6 +1750,8 @@ static enum dc_status enable_link_dp(struct dc_state *state, · 496 unmodified lines
1716	1750	bool apply_seamless_boot_optimization = false;
1717	1751	uint32_t bl_oled_enable_delay = 50; // in ms
1718	1752	const uint32_t post_oui_delay = 30; // 30ms
	1753	+ /* Reduce link bandwidth between failed link training attempts. */
	1754	+ bool do_fallback = false;
1719	1755
1720	1756	// check for seamless boot
1721	1757	for (i = 0; i < state->stream_count; i++) {
@@ -1754,7 +1790,8 @@ static enum dc_status enable_link_dp(struct dc_state *state, · 32 unmodified lines
1754	1790	skip_video_pattern,
1755	1791	LINK_TRAINING_ATTEMPTS,
1756	1792	pipe_ctx,
1757		− pipe_ctx->stream->signal)) {
	1793	+ pipe_ctx->stream->signal,
	1794	+ do_fallback)) {
1758	1795	link->cur_link_settings = link_settings;
1759	1796	status = DC_OK;
1760	1797	} else {
@@ -3475,9 +3512,11 @@ uint32_t dc_bandwidth_in_kbps_from_timing( · 1714 unmodified lines
3475	3512	uint32_t kbps;
3476	3513
3477	3514	#if defined(CONFIG_DRM_AMD_DC_DCN)
3478		− if (timing->flags.DSC) {
3479		− return dc_dsc_stream_bandwidth_in_kbps(timing->pix_clk_100hz, timing->dsc_cfg.bits_per_pixel);
3480		− }
	3515	+ if (timing->flags.DSC)
	3516	+ return dc_dsc_stream_bandwidth_in_kbps(timing,
	3517	+ timing->dsc_cfg.bits_per_pixel,
	3518	+ timing->dsc_cfg.num_slices_h,
	3519	+ timing->dsc_cfg.is_dp);
3481	3520	#endif
3482	3521
3483	3522	switch (timing->display_color_depth) {
@@ -3539,19 +3578,6 @@ void dc_link_set_drive_settings(struct dc *dc, · 55 unmodified lines
3539	3578	dc_link_dp_set_drive_settings(dc->links[i], lt_settings);
3540	3579	}
3541	3580
3542		−void dc_link_perform_link_training(struct dc *dc,
3543		− struct dc_link_settings *link_setting,
3544		− bool skip_video_pattern)
3545		−{
3546		− int i;
3547		−
3548		− for (i = 0; i < dc->link_count; i++)
3549		− dc_link_dp_perform_link_training(
3550		− dc->links[i],
3551		− link_setting,
3552		− skip_video_pattern);
3553		−}
3554		−
3555	3581	void dc_link_set_preferred_link_settings(struct dc *dc,
3556	3582	struct dc_link_settings *link_setting,
3557	3583	struct dc_link *link)
@@ -3702,8 +3728,22 @@ void dc_link_overwrite_extended_receiver_cap( · 144 unmodified lines
3702	3728
3703	3729	bool dc_link_is_fec_supported(const struct dc_link *link)
3704	3730	{
	3731	+ struct link_encoder *link_enc = NULL;
	3732	+
	3733	+ /* Links supporting dynamically assigned link encoder will be assigned next
	3734	+ * available encoder if one not already assigned.
	3735	+ */
	3736	+ if (link->is_dig_mapping_flexible &&
	3737	+ link->dc->res_pool->funcs->link_encs_assign) {
	3738	+ link_enc = link_enc_cfg_get_link_enc_used_by_link(link->dc->current_state, link);
	3739	+ if (link_enc == NULL)
	3740	+ link_enc = link_enc_cfg_get_next_avail_link_enc(link->dc, link->dc->current_state);
	3741	+ } else
	3742	+ link_enc = link->link_enc;
	3743	+ ASSERT(link_enc);
	3744	+
3705	3745	return (dc_is_dp_signal(link->connector_signal) &&
3706		− link->link_enc->features.fec_supported &&
	3746	+ link_enc->features.fec_supported &&
3707	3747	link->dpcd_caps.fec_cap.bits.FEC_CAPABLE &&
3708	3748	!IS_FPGA_MAXIMUS_DC(link->ctx->dce_environment));
3709	3749	}

drivers/gpu/drm/amd/display/dc/core/dc_link_ddc.c+8−1

@@ -658,7 +658,10 @@ int dc_link_aux_transfer_raw(struct ddc_service *ddc, · 657 unmodified lines
658	658	struct aux_payload *payload,
659	659	enum aux_return_code_type *operation_result)
660	660	{
661		− return dce_aux_transfer_raw(ddc, payload, operation_result);
	661	+ if (dc_enable_dmub_notifications(ddc->ctx->dc))
	662	+ return dce_aux_transfer_dmub_raw(ddc, payload, operation_result);
	663	+ else
	664	+ return dce_aux_transfer_raw(ddc, payload, operation_result);
662	665	}
663	666
664	667	/* dc_link_aux_transfer_with_retries() - Attempt to submit an
@@ -682,6 +685,10 @@ bool dc_link_aux_try_to_configure_timeout(struct ddc_service *ddc, · 17 unmodified lines
682	685	bool result = false;
683	686	struct ddc *ddc_pin = ddc->ddc_pin;
684	687
	688	+ /* Do not try to access nonexistent DDC pin. */
	689	+ if (ddc->link->ep_type != DISPLAY_ENDPOINT_PHY)
	690	+ return true;
	691	+
685	692	if (ddc->ctx->dc->res_pool->engines[ddc_pin->pin_data->en]->funcs->configure_timeout) {
686	693	ddc->ctx->dc->res_pool->engines[ddc_pin->pin_data->en]->funcs->configure_timeout(ddc, timeout);
687	694	result = true;

drivers/gpu/drm/amd/display/dc/core/dc_link_dp.c+157−74

@@ -14,6 +14,7 @@ · 13 unmodified lines
14	14	#include "dpcd_defs.h"
15	15	#include "dc_dmub_srv.h"
16	16	#include "dce/dmub_hw_lock_mgr.h"
	17	+#include "inc/link_enc_cfg.h"
17	18
18	19	/Travis/
19	20	static const uint8_t DP_VGA_LVDS_CONVERTER_ID_2[] = "sivarT";
@@ -107,10 +108,50 @@ static void wait_for_training_aux_rd_interval( · 87 unmodified lines
107	108	wait_in_micro_secs);
108	109	}
109	110
	111	+static enum dpcd_training_patterns
	112	+ dc_dp_training_pattern_to_dpcd_training_pattern(
	113	+ struct dc_link *link,
	114	+ enum dc_dp_training_pattern pattern)
	115	+{
	116	+ enum dpcd_training_patterns dpcd_tr_pattern =
	117	+ DPCD_TRAINING_PATTERN_VIDEOIDLE;
	118	+
	119	+ switch (pattern) {
	120	+ case DP_TRAINING_PATTERN_SEQUENCE_1:
	121	+ dpcd_tr_pattern = DPCD_TRAINING_PATTERN_1;
	122	+ break;
	123	+ case DP_TRAINING_PATTERN_SEQUENCE_2:
	124	+ dpcd_tr_pattern = DPCD_TRAINING_PATTERN_2;
	125	+ break;
	126	+ case DP_TRAINING_PATTERN_SEQUENCE_3:
	127	+ dpcd_tr_pattern = DPCD_TRAINING_PATTERN_3;
	128	+ break;
	129	+ case DP_TRAINING_PATTERN_SEQUENCE_4:
	130	+ dpcd_tr_pattern = DPCD_TRAINING_PATTERN_4;
	131	+ break;
	132	+ case DP_TRAINING_PATTERN_VIDEOIDLE:
	133	+ dpcd_tr_pattern = DPCD_TRAINING_PATTERN_VIDEOIDLE;
	134	+ break;
	135	+ default:
	136	+ ASSERT(0);
	137	+ DC_LOG_HW_LINK_TRAINING("%s: Invalid HW Training pattern: %d\n",
	138	+ __func__, pattern);
	139	+ break;
	140	+ }
	141	+
	142	+ return dpcd_tr_pattern;
	143	+}
	144	+
110	145	static void dpcd_set_training_pattern(
111	146	struct dc_link *link,
112		− union dpcd_training_pattern dpcd_pattern)
	147	+ enum dc_dp_training_pattern training_pattern)
113	148	{
	149	+ union dpcd_training_pattern dpcd_pattern = { {0} };
	150	+
	151	+ dpcd_pattern.v1_4.TRAINING_PATTERN_SET =
	152	+ dc_dp_training_pattern_to_dpcd_training_pattern(
	153	+ link, training_pattern);
	154	+
114	155	core_link_write_dpcd(
115	156	link,
116	157	DP_TRAINING_PATTERN_SET,
@@ -132,10 +173,22 @@ static enum dc_dp_training_pattern decide_cr_training_pattern( · 15 unmodified lines
132	173	static enum dc_dp_training_pattern decide_eq_training_pattern(struct dc_link *link,
133	174	const struct dc_link_settings *link_settings)
134	175	{
	176	+ struct link_encoder *link_enc;
135	177	enum dc_dp_training_pattern highest_tp = DP_TRAINING_PATTERN_SEQUENCE_2;
136		− struct encoder_feature_support *features = &link->link_enc->features;
	178	+ struct encoder_feature_support *features;
137	179	struct dpcd_caps *dpcd_caps = &link->dpcd_caps;
138	180
	181	+ /* Access link encoder capability based on whether it is statically
	182	+ * or dynamically assigned to a link.
	183	+ */
	184	+ if (link->is_dig_mapping_flexible &&
	185	+ link->dc->res_pool->funcs->link_encs_assign)
	186	+ link_enc = link_enc_cfg_get_link_enc_used_by_link(link->dc->current_state, link);
	187	+ else
	188	+ link_enc = link->link_enc;
	189	+ ASSERT(link_enc);
	190	+ features = &link_enc->features;
	191	+
139	192	if (features->flags.bits.IS_TPS3_CAPABLE)
140	193	highest_tp = DP_TRAINING_PATTERN_SEQUENCE_3;
141	194
@@ -227,37 +280,6 @@ static void dpcd_set_link_settings( · 85 unmodified lines
227	280	}
228	281	}
229	282
230		−static enum dpcd_training_patterns
231		− dc_dp_training_pattern_to_dpcd_training_pattern(
232		− struct dc_link *link,
233		− enum dc_dp_training_pattern pattern)
234		−{
235		− enum dpcd_training_patterns dpcd_tr_pattern =
236		− DPCD_TRAINING_PATTERN_VIDEOIDLE;
237		−
238		− switch (pattern) {
239		− case DP_TRAINING_PATTERN_SEQUENCE_1:
240		− dpcd_tr_pattern = DPCD_TRAINING_PATTERN_1;
241		− break;
242		− case DP_TRAINING_PATTERN_SEQUENCE_2:
243		− dpcd_tr_pattern = DPCD_TRAINING_PATTERN_2;
244		− break;
245		− case DP_TRAINING_PATTERN_SEQUENCE_3:
246		− dpcd_tr_pattern = DPCD_TRAINING_PATTERN_3;
247		− break;
248		− case DP_TRAINING_PATTERN_SEQUENCE_4:
249		− dpcd_tr_pattern = DPCD_TRAINING_PATTERN_4;
250		− break;
251		− default:
252		− ASSERT(0);
253		− DC_LOG_HW_LINK_TRAINING("%s: Invalid HW Training pattern: %d\n",
254		− __func__, pattern);
255		− break;
256		− }
257		−
258		− return dpcd_tr_pattern;
259		−}
260		−
261	283	static uint8_t dc_dp_initialize_scrambling_data_symbols(
262	284	struct dc_link *link,
263	285	enum dc_dp_training_pattern pattern)
@@ -420,20 +442,30 @@ static bool is_cr_done(enum dc_lane_count ln_count, · 156 unmodified lines
420	442	}
421	443
422	444	static bool is_ch_eq_done(enum dc_lane_count ln_count,
423		− union lane_status *dpcd_lane_status,
424		− union lane_align_status_updated *lane_status_updated)
	445	+ union lane_status *dpcd_lane_status)
425	446	{
	447	+ bool done = true;
426	448	uint32_t lane;
427		− if (!lane_status_updated->bits.INTERLANE_ALIGN_DONE)
428		− return false;
429		− else {
430		− for (lane = 0; lane < (uint32_t)(ln_count); lane++) {
431		− if (!dpcd_lane_status[lane].bits.SYMBOL_LOCKED_0 \|\|
432		− !dpcd_lane_status[lane].bits.CHANNEL_EQ_DONE_0)
433		− return false;
434		− }
435		− }
436		− return true;
	449	+ for (lane = 0; lane < (uint32_t)(ln_count); lane++)
	450	+ if (!dpcd_lane_status[lane].bits.CHANNEL_EQ_DONE_0)
	451	+ done = false;
	452	+ return done;
	453	+}
	454	+
	455	+static bool is_symbol_locked(enum dc_lane_count ln_count,
	456	+ union lane_status *dpcd_lane_status)
	457	+{
	458	+ bool locked = true;
	459	+ uint32_t lane;
	460	+ for (lane = 0; lane < (uint32_t)(ln_count); lane++)
	461	+ if (!dpcd_lane_status[lane].bits.SYMBOL_LOCKED_0)
	462	+ locked = false;
	463	+ return locked;
	464	+}
	465	+
	466	+static inline bool is_interlane_aligned(union lane_align_status_updated align_status)
	467	+{
	468	+ return align_status.bits.INTERLANE_ALIGN_DONE == 1;
437	469	}
438	470
439	471	static void update_drive_settings(
@@ -835,10 +867,9 @@ static bool perform_post_lt_adj_req_sequence( · 395 unmodified lines
835	867	if (!is_cr_done(lane_count, dpcd_lane_status))
836	868	return false;
837	869
838		− if (!is_ch_eq_done(
839		− lane_count,
840		− dpcd_lane_status,
841		− &dpcd_lane_status_updated))
	870	+ if (!is_ch_eq_done(lane_count, dpcd_lane_status) \|\|
	871	+ !is_symbol_locked(lane_count, dpcd_lane_status) \|\|
	872	+ !is_interlane_aligned(dpcd_lane_status_updated))
842	873	return false;
843	874
844	875	for (lane = 0; lane < (uint32_t)(lane_count); lane++) {
@@ -992,9 +1023,9 @@ static enum link_training_result perform_channel_equalization_sequence( · 147 unmodified lines
992	1023	return LINK_TRAINING_EQ_FAIL_CR;
993	1024
994	1025	/* 6. check CHEQ done*/
995		− if (is_ch_eq_done(lane_count,
996		− dpcd_lane_status,
997		− &dpcd_lane_status_updated))
	1026	+ if (is_ch_eq_done(lane_count, dpcd_lane_status) &&
	1027	+ is_symbol_locked(lane_count, dpcd_lane_status) &&
	1028	+ is_interlane_aligned(dpcd_lane_status_updated))
998	1029	return LINK_TRAINING_SUCCESS;
999	1030
1000	1031	/* 7. update VS/PE/PC2 in lt_settings*/
@@ -1162,7 +1193,7 @@ static inline enum link_training_result perform_link_training_int( · 161 unmodified lines
1162	1193	return status;
1163	1194	}
1164	1195
1165		−static enum link_training_result check_link_loss_status(
	1196	+enum link_training_result dp_check_link_loss_status(
1166	1197	struct dc_link *link,
1167	1198	const struct link_training_settings *link_training_setting)
1168	1199	{
@@ -1296,7 +1327,7 @@ static void initialize_training_settings( · 127 unmodified lines
1296	1327	lt_settings->enhanced_framing = 1;
1297	1328	}
1298	1329
1299		−static uint8_t convert_to_count(uint8_t lttpr_repeater_count)
	1330	+uint8_t dp_convert_to_count(uint8_t lttpr_repeater_count)
1300	1331	{
1301	1332	switch (lttpr_repeater_count) {
1302	1333	case 0x80: // 1 lttpr repeater
@@ -1365,7 +1396,8 @@ static void configure_lttpr_mode_non_transparent(struct dc_link *link) · 62 unmodified lines
1365	1396	link->dpcd_caps.lttpr_caps.mode = repeater_mode;
1366	1397	}
1367	1398
1368		− repeater_cnt = convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt);
	1399	+ repeater_cnt = dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt);
	1400	+
1369	1401	for (repeater_id = repeater_cnt; repeater_id > 0; repeater_id--) {
1370	1402	aux_interval_address = DP_TRAINING_AUX_RD_INTERVAL_PHY_REPEATER1 +
1371	1403	((DP_REPEATER_CONFIGURATION_AND_STATUS_SIZE) * (repeater_id - 1));
@@ -1555,7 +1587,6 @@ enum link_training_result dc_link_dp_perform_link_training( · 183 unmodified lines
1555	1587	{
1556	1588	enum link_training_result status = LINK_TRAINING_SUCCESS;
1557	1589	struct link_training_settings lt_settings;
1558		− union dpcd_training_pattern dpcd_pattern = { { 0 } };
1559	1590
1560	1591	bool fec_enable;
1561	1592	uint8_t repeater_cnt;
@@ -1591,7 +1622,7 @@ enum link_training_result dc_link_dp_perform_link_training( · 29 unmodified lines
1591	1622	/* 2. perform link training (set link training done
1592	1623	* to false is done as well)
1593	1624	*/
1594		− repeater_cnt = convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt);
	1625	+ repeater_cnt = dp_convert_to_count(link->dpcd_caps.lttpr_caps.phy_repeater_cnt);
1595	1626
1596	1627	for (repeater_id = repeater_cnt; (repeater_id > 0 && status == LINK_TRAINING_SUCCESS);
1597	1628	repeater_id--) {
@@ -1621,8 +1652,7 @@ enum link_training_result dc_link_dp_perform_link_training( · 23 unmodified lines
1621	1652	}
1622	1653
1623	1654	/* 3. set training not in progress*/
1624		− dpcd_pattern.v1_4.TRAINING_PATTERN_SET = DPCD_TRAINING_PATTERN_VIDEOIDLE;
1625		− dpcd_set_training_pattern(link, dpcd_pattern);
	1655	+ dpcd_set_training_pattern(link, DP_TRAINING_PATTERN_VIDEOIDLE);
1626	1656	if ((status == LINK_TRAINING_SUCCESS) \|\| !skip_video_pattern) {
1627	1657	status = perform_link_training_int(link,
1628	1658	&lt_settings,
@@ -1634,7 +1664,7 @@ enum link_training_result dc_link_dp_perform_link_training( · 5 unmodified lines
1634	1664	*/
1635	1665	if (link->connector_signal != SIGNAL_TYPE_EDP && status == LINK_TRAINING_SUCCESS) {
1636	1666	msleep(5);
1637		− status = check_link_loss_status(link, &lt_settings);
	1667	+ status = dp_check_link_loss_status(link, &lt_settings);
1638	1668	}
1639	1669
1640	1670	/* 6. print status message*/
@@ -1687,18 +1717,31 @@ bool perform_link_training_with_retries( · 46 unmodified lines
1687	1717	bool skip_video_pattern,
1688	1718	int attempts,
1689	1719	struct pipe_ctx *pipe_ctx,
1690		− enum signal_type signal)
	1720	+ enum signal_type signal,
	1721	+ bool do_fallback)
1691	1722	{
1692	1723	uint8_t j;
1693	1724	uint8_t delay_between_attempts = LINK_TRAINING_RETRY_DELAY;
1694	1725	struct dc_stream_state *stream = pipe_ctx->stream;
1695	1726	struct dc_link *link = stream->link;
1696	1727	enum dp_panel_mode panel_mode;
	1728	+ struct link_encoder *link_enc;
	1729	+ enum link_training_result status = LINK_TRAINING_CR_FAIL_LANE0;
	1730	+ struct dc_link_settings currnet_setting = *link_setting;
	1731	+
	1732	+ /* Dynamically assigned link encoders associated with stream rather than
	1733	+ * link.
	1734	+ */
	1735	+ if (link->dc->res_pool->funcs->link_encs_assign)
	1736	+ link_enc = stream->link_enc;
	1737	+ else
	1738	+ link_enc = link->link_enc;
	1739	+ ASSERT(link_enc);
1697	1740
1698	1741	/* We need to do this before the link training to ensure the idle pattern in SST
1699	1742	* mode will be sent right after the link training
1700	1743	*/
1701		− link->link_enc->funcs->connect_dig_be_to_fe(link->link_enc,
	1744	+ link_enc->funcs->connect_dig_be_to_fe(link_enc,
1702	1745	pipe_ctx->stream_res.stream_enc->id, true);
1703	1746
1704	1747	for (j = 0; j < attempts; ++j) {
@@ -1710,7 +1753,7 @@ bool perform_link_training_with_retries( · 5 unmodified lines
1710	1753	link,
1711	1754	signal,
1712	1755	pipe_ctx->clock_source->id,
1713		− link_setting);
	1756	+ &currnet_setting);
1714	1757
1715	1758	if (stream->sink_patches.dppowerup_delay > 0) {
1716	1759	int delay_dp_power_up_in_ms = stream->sink_patches.dppowerup_delay;
@@ -1725,14 +1768,12 @@ bool perform_link_training_with_retries( · 8 unmodified lines
1725	1768	panel_mode != DP_PANEL_MODE_DEFAULT);
1726	1769
1727	1770	if (link->aux_access_disabled) {
1728		− dc_link_dp_perform_link_training_skip_aux(link, link_setting);
	1771	+ dc_link_dp_perform_link_training_skip_aux(link, &currnet_setting);
1729	1772	return true;
1730	1773	} else {
1731		− enum link_training_result status = LINK_TRAINING_CR_FAIL_LANE0;
1732		−
1733	1774	status = dc_link_dp_perform_link_training(
1734	1775	link,
1735		− link_setting,
	1776	+ &currnet_setting,
1736	1777	skip_video_pattern);
1737	1778	if (status == LINK_TRAINING_SUCCESS)
1738	1779	return true;
@@ -1740,7 +1781,7 @@ bool perform_link_training_with_retries( · 1 unmodified line
1740	1781
1741	1782	/* latest link training still fail, skip delay and keep PHY on
1742	1783	*/
1743		− if (j == (attempts - 1))
	1784	+ if (j == (attempts - 1) && link->ep_type == DISPLAY_ENDPOINT_PHY)
1744	1785	break;
1745	1786
1746	1787	DC_LOG_WARNING("%s: Link training attempt %u of %d failed\n",
@@ -1748,6 +1789,19 @@ bool perform_link_training_with_retries( · 1 unmodified line
1748	1789
1749	1790	dp_disable_link_phy(link, signal);
1750	1791
	1792	+ /* Abort link training if failure due to sink being unplugged. */
	1793	+ if (status == LINK_TRAINING_ABORT)
	1794	+ break;
	1795	+ else if (do_fallback) {
	1796	+ decide_fallback_link_setting(*link_setting, &currnet_setting, status);
	1797	+ /* Fail link training if reduced link bandwidth no longer meets
	1798	+ * stream requirements.
	1799	+ */
	1800	+ if (dc_bandwidth_in_kbps_from_timing(&stream->timing) <
	1801	+ dc_link_bandwidth_kbps(link, &currnet_setting))
	1802	+ break;
	1803	+ }
	1804	+
1751	1805	msleep(delay_between_attempts);
1752	1806
1753	1807	delay_between_attempts += LINK_TRAINING_RETRY_DELAY;
@@ -2429,6 +2483,12 @@ bool dp_validate_mode_timing( · 675 unmodified lines
2429	2483
2430	2484	const struct dc_link_settings *link_setting;
2431	2485
	2486	+ /* According to spec, VSC SDP should be used if pixel format is YCbCr420 */
	2487	+ if (timing->pixel_encoding == PIXEL_ENCODING_YCBCR420 &&
	2488	+ !link->dpcd_caps.dprx_feature.bits.VSC_SDP_COLORIMETRY_SUPPORTED &&
	2489	+ dal_graphics_object_id_get_connector_id(link->link_id) != CONNECTOR_ID_VIRTUAL)
	2490	+ return false;
	2491	+
2432	2492	/always DP fail safe mode/
2433	2493	if ((timing->pix_clk_100hz / 10) == (uint32_t) 25175 &&
2434	2494	timing->h_addressable == (uint32_t) 640 &&
@@ -2611,13 +2671,11 @@ static bool allow_hpd_rx_irq(const struct dc_link *link) · 176 unmodified lines
2611	2671	/*
2612	2672	* Don't handle RX IRQ unless one of following is met:
2613	2673	* 1) The link is established (cur_link_settings != unknown)
2614		− * 2) We kicked off MST detection
2615		− * 3) We know we're dealing with an active dongle
	2674	+ * 2) We know we're dealing with a branch device, SST or MST
2616	2675	*/
2617	2676
2618	2677	if ((link->cur_link_settings.lane_count != LANE_COUNT_UNKNOWN) \|\|
2619		− (link->type == dc_connection_mst_branch) \|\|
2620		− is_dp_active_dongle(link))
	2678	+ is_dp_branch_device(link))
2621	2679	return true;
2622	2680
2623	2681	return false;
@@ -2917,6 +2975,22 @@ static void dp_test_send_link_test_pattern(struct dc_link *link) · 293 unmodified lines
2917	2975	break;
2918	2976	}
2919	2977
	2978	+ switch (dpcd_test_params.bits.CLR_FORMAT) {
	2979	+ case 0:
	2980	+ pipe_ctx->stream->timing.pixel_encoding = PIXEL_ENCODING_RGB;
	2981	+ break;
	2982	+ case 1:
	2983	+ pipe_ctx->stream->timing.pixel_encoding = PIXEL_ENCODING_YCBCR422;
	2984	+ break;
	2985	+ case 2:
	2986	+ pipe_ctx->stream->timing.pixel_encoding = PIXEL_ENCODING_YCBCR444;
	2987	+ break;
	2988	+ default:
	2989	+ pipe_ctx->stream->timing.pixel_encoding = PIXEL_ENCODING_RGB;
	2990	+ break;
	2991	+ }
	2992	+
	2993	+
2920	2994	if (requestColorDepth != COLOR_DEPTH_UNDEFINED
2921	2995	&& pipe_ctx->stream->timing.display_color_depth != requestColorDepth) {
2922	2996	DC_LOG_DEBUG("%s: original bpc %d, changing to %d\n",
@@ -2924,9 +2998,10 @@ static void dp_test_send_link_test_pattern(struct dc_link *link) · 1 unmodified line
2924	2998	pipe_ctx->stream->timing.display_color_depth,
2925	2999	requestColorDepth);
2926	3000	pipe_ctx->stream->timing.display_color_depth = requestColorDepth;
2927		− dp_update_dsc_config(pipe_ctx);
2928	3001	}
2929	3002
	3003	+ dp_update_dsc_config(pipe_ctx);
	3004	+
2930	3005	dc_link_dp_set_test_pattern(
2931	3006	link,
2932	3007	test_pattern,
@@ -3182,7 +3257,7 @@ bool dc_link_handle_hpd_rx_irq(struct dc_link link, union hpd_irq_data out_hpd · 249 unmodified lines
3182	3257	*out_link_loss = true;
3183	3258	}
3184	3259
3185		− if (link->type == dc_connection_active_dongle &&
	3260	+ if (link->type == dc_connection_sst_branch &&
3186	3261	hpd_irq_dpcd_data.bytes.sink_cnt.bits.SINK_COUNT
3187	3262	!= link->dpcd_sink_count)
3188	3263	status = true;
@@ -3232,6 +3307,12 @@ bool is_mst_supported(struct dc_link *link) · 43 unmodified lines
3232	3307	}
3233	3308
3234	3309	bool is_dp_active_dongle(const struct dc_link *link)
	3310	+{
	3311	+ return (link->dpcd_caps.dongle_type >= DISPLAY_DONGLE_DP_VGA_CONVERTER) &&
	3312	+ (link->dpcd_caps.dongle_type <= DISPLAY_DONGLE_DP_HDMI_CONVERTER);
	3313	+}
	3314	+
	3315	+bool is_dp_branch_device(const struct dc_link *link)
3235	3316	{
3236	3317	return link->dpcd_caps.is_branch_dev;
3237	3318	}
@@ -3593,7 +3674,9 @@ static bool retrieve_link_cap(struct dc_link *link) · 355 unmodified lines
3593	3674	lttpr_dpcd_data[DP_PHY_REPEATER_EXTENDED_WAIT_TIMEOUT -
3594	3675	DP_LT_TUNABLE_PHY_REPEATER_FIELD_DATA_STRUCTURE_REV];
3595	3676
	3677	+ /* Attempt to train in LTTPR transparent mode if repeater count exceeds 8. */
3596	3678	is_lttpr_present = (link->dpcd_caps.lttpr_caps.phy_repeater_cnt > 0 &&
	3679	+ link->dpcd_caps.lttpr_caps.phy_repeater_cnt < 0xff &&
3597	3680	link->dpcd_caps.lttpr_caps.max_lane_count > 0 &&
3598	3681	link->dpcd_caps.lttpr_caps.max_lane_count <= 4 &&
3599	3682	link->dpcd_caps.lttpr_caps.revision.raw >= 0x14);

drivers/gpu/drm/amd/display/dc/core/dc_link_enc_cfg.c+17−5

@@ -112,8 +112,8 @@ static void update_link_enc_assignment( · 111 unmodified lines
112	112
113	113	/* Return first available DIG link encoder. */
114	114	static enum engine_id find_first_avail_link_enc(
115		− struct dc_context *ctx,
116		− struct dc_state *state)
	115	+ const struct dc_context *ctx,
	116	+ const struct dc_state *state)
117	117	{
118	118	enum engine_id eng_id = ENGINE_ID_UNKNOWN;
119	119	int i;
@@ -270,7 +270,7 @@ struct dc_link *link_enc_cfg_get_link_using_link_enc( · 150 unmodified lines
270	270
271	271	struct link_encoder *link_enc_cfg_get_link_enc_used_by_link(
272	272	struct dc_state *state,
273		− struct dc_link *link)
	273	+ const struct dc_link *link)
274	274	{
275	275	struct link_encoder *link_enc = NULL;
276	276	struct display_endpoint_id ep_id;
@@ -296,8 +296,20 @@ struct link_encoder *link_enc_cfg_get_link_enc_used_by_link( · 19 unmodified lines
296	296
297	297	if (stream_idx != -1)
298	298	link_enc = state->streams[stream_idx]->link_enc;
299		− else
300		− dm_output_to_console("%s: No link encoder used by link(%d).\n", __func__, link->link_index);
	299	+
	300	+ return link_enc;
	301	+}
	302	+
	303	+struct link_encoder *link_enc_cfg_get_next_avail_link_enc(
	304	+ const struct dc *dc,
	305	+ const struct dc_state *state)
	306	+{
	307	+ struct link_encoder *link_enc = NULL;
	308	+ enum engine_id eng_id = ENGINE_ID_UNKNOWN;
	309	+
	310	+ eng_id = find_first_avail_link_enc(dc->ctx, state);
	311	+ if (eng_id != ENGINE_ID_UNKNOWN)
	312	+ link_enc = dc->res_pool->link_encoders[eng_id - ENGINE_ID_DIGA];
301	313
302	314	return link_enc;
303	315	}

drivers/gpu/drm/amd/display/dc/core/dc_link_hwss.c+2−1

@@ -384,7 +384,8 @@ void dp_retrain_link_dp_test(struct dc_link *link, · 383 unmodified lines
384	384	skip_video_pattern,
385	385	LINK_TRAINING_ATTEMPTS,
386	386	&pipes[i],
387		− SIGNAL_TYPE_DISPLAY_PORT);
	387	+ SIGNAL_TYPE_DISPLAY_PORT,
	388	+ false);
388	389
389	390	link->dc->hwss.enable_stream(&pipes[i]);
390	391

drivers/gpu/drm/amd/display/dc/core/dc_resource.c+1−6

@@ -1706,12 +1706,6 @@ static bool is_timing_changed(struct dc_stream_state *cur_stream, · 1705 unmodified lines
1706	1706	if (cur_stream == NULL)
1707	1707	return true;
1708	1708
1709		− /* If sink pointer changed, it means this is a hotplug, we should do
1710		− * full hw setting.
1711		− */
1712		− if (cur_stream->sink != new_stream->sink)
1713		− return true;
1714		−
1715	1709	/* If output color space is changed, need to reprogram info frames */
1716	1710	if (cur_stream->output_color_space != new_stream->output_color_space)
1717	1711	return true;
@@ -2679,6 +2673,7 @@ void dc_resource_state_destruct(struct dc_state *context) · 961 unmodified lines
2679	2673	dc_stream_release(context->streams[i]);
2680	2674	context->streams[i] = NULL;
2681	2675	}
	2676	+ context->stream_count = 0;
2682	2677	}
2683	2678
2684	2679	void dc_resource_state_copy_construct(

drivers/gpu/drm/amd/display/dc/dc.h+1−5

@@ -45,7 +45,7 @@ · 44 unmodified lines
45	45	/* forward declaration */
46	46	struct aux_payload;
47	47
48		−#define DC_VER "3.2.132"
	48	+#define DC_VER "3.2.135.1"
49	49
50	50	#define MAX_SURFACES 3
51	51	#define MAX_PLANES 6
@@ -308,8 +308,6 @@ struct dc_config { · 256 unmodified lines
308	308	#endif
309	309	uint64_t vblank_alignment_dto_params;
310	310	uint8_t vblank_alignment_max_frame_time_diff;
311		− bool is_asymmetric_memory;
312		− bool is_single_rank_dimm;
313	311	};
314	312
315	313	enum visual_confirm {
@@ -600,7 +598,6 @@ struct dc_bounding_box_overrides { · 284 unmodified lines
600	598	int min_dcfclk_mhz;
601	599	};
602	600
603		−struct dc_state;
604	601	struct resource_pool;
605	602	struct dce_hwseq;
606	603	struct gpu_info_soc_bounding_box_v1_0;
@@ -719,7 +716,6 @@ void dc_init_callbacks(struct dc *dc, · 112 unmodified lines
719	716	void dc_deinit_callbacks(struct dc *dc);
720	717	void dc_destroy(struct dc **dc);
721	718
722		−void dc_wait_for_vblank(struct dc dc, struct dc_stream_state stream);
723	719	/*******************************************************************************
724	720	* Surface Interfaces
725	721	******************************************************************************/

drivers/gpu/drm/amd/display/dc/dc_dmub_srv.c+1−1

@@ -180,5 +180,5 @@ bool dc_dmub_srv_get_dmub_outbox0_msg(const struct dc *dc, struct dmcub_trace_bu · 179 unmodified lines
180	180
181	181	void dc_dmub_trace_event_control(struct dc *dc, bool enable)
182	182	{
183		− dm_helpers_dmub_outbox0_interrupt_control(dc->ctx, enable);
	183	+ dm_helpers_dmub_outbox_interrupt_control(dc->ctx, enable);
184	184	}

drivers/gpu/drm/amd/display/dc/dc_dp_types.h+1−0

@@ -95,6 +95,7 @@ enum dc_dp_training_pattern { · 94 unmodified lines
95	95	DP_TRAINING_PATTERN_SEQUENCE_2,
96	96	DP_TRAINING_PATTERN_SEQUENCE_3,
97	97	DP_TRAINING_PATTERN_SEQUENCE_4,
	98	+ DP_TRAINING_PATTERN_VIDEOIDLE,
98	99	};
99	100
100	101	struct dc_link_settings {

drivers/gpu/drm/amd/display/dc/dc_dsc.h+3−2

@@ -78,7 +78,8 @@ bool dc_dsc_compute_config( · 77 unmodified lines
78	78	const struct dc_crtc_timing *timing,
79	79	struct dc_dsc_config *dsc_cfg);
80	80
81		−uint32_t dc_dsc_stream_bandwidth_in_kbps(uint32_t pix_clk_100hz, uint32_t bpp_x16);
	81	+uint32_t dc_dsc_stream_bandwidth_in_kbps(const struct dc_crtc_timing *timing,
	82	+ uint32_t bpp_x16, uint32_t num_slices_h, bool is_dp);
82	83
83	84	void dc_dsc_get_policy_for_timing(const struct dc_crtc_timing *timing,
84	85	uint32_t max_target_bpp_limit_override_x16,
@@ -88,6 +89,6 @@ void dc_dsc_policy_set_max_target_bpp_limit(uint32_t limit); · 3 unmodified lines
88	89
89	90	void dc_dsc_policy_set_enable_dsc_when_not_needed(bool enable);
90	91
91		−uint32_t dc_dsc_stream_bandwidth_in_kbps(uint32_t pix_clk_100hz, uint32_t bpp_x16);
	92	+void dc_dsc_policy_set_disable_dsc_stream_overhead(bool disable);
92	93
93	94	#endif

drivers/gpu/drm/amd/display/dc/dc_hw_types.h+1−0

@@ -739,6 +739,7 @@ struct dc_dsc_config { · 738 unmodified lines
739	739	uint32_t version_minor; /* DSC minor version. Full version is formed as 1.version_minor. */
740	740	bool ycbcr422_simple; /* Tell DSC engine to convert YCbCr 4:2:2 to 'YCbCr 4:2:2 simple'. */
741	741	int32_t rc_buffer_size; /* DSC RC buffer block size in bytes */
	742	+ bool is_dp; /* indicate if DSC is applied based on DP's capability */
742	743	};
743	744	struct dc_crtc_timing {
744	745	uint32_t h_total;

drivers/gpu/drm/amd/display/dc/dc_link.h+1−9

@@ -36,12 +36,6 @@ enum dc_link_fec_state { · 35 unmodified lines
36	36	dc_link_fec_enabled
37	37	};
38	38
39		−enum lttpr_mode {
40		− LTTPR_MODE_NON_LTTPR,
41		− LTTPR_MODE_TRANSPARENT,
42		− LTTPR_MODE_NON_TRANSPARENT,
43		−};
44		−
45	39	struct dc_link_status {
46	40	bool link_active;
47	41	struct dpcd_caps *dpcd_caps;
@@ -113,6 +107,7 @@ struct dc_link { · 65 unmodified lines
113	107	/* TODO: Rename. Flag an endpoint as having a programmable mapping to a
114	108	* DIG encoder. */
115	109	bool is_dig_mapping_flexible;
	110	+ bool hpd_status; /* HPD status of link without physical HPD pin. */
116	111
117	112	bool edp_sink_present;
118	113
@@ -363,9 +358,6 @@ bool dc_link_is_hdcp22(struct dc_link *link, enum signal_type signal); · 244 unmodified lines
363	358	void dc_link_set_drive_settings(struct dc *dc,
364	359	struct link_training_settings *lt_settings,
365	360	const struct dc_link *link);
366		−void dc_link_perform_link_training(struct dc *dc,
367		− struct dc_link_settings *link_setting,
368		− bool skip_video_pattern);
369	361	void dc_link_set_preferred_link_settings(struct dc *dc,
370	362	struct dc_link_settings *link_setting,
371	363	struct dc_link *link);

drivers/gpu/drm/amd/display/dc/dc_types.h+2−1

@@ -404,7 +404,7 @@ enum dc_connection_type { · 403 unmodified lines
404	404	dc_connection_none,
405	405	dc_connection_single,
406	406	dc_connection_mst_branch,
407		− dc_connection_active_dongle
	407	+ dc_connection_sst_branch
408	408	};
409	409
410	410	struct dc_csc_adjustments {
@@ -909,6 +909,7 @@ struct dsc_dec_dpcd_caps { · 498 unmodified lines
909	909	uint32_t branch_overall_throughput_0_mps; /* In MPs */
910	910	uint32_t branch_overall_throughput_1_mps; /* In MPs */
911	911	uint32_t branch_max_line_width;
	912	+ bool is_dp;
912	913	};
913	914
914	915	struct dc_golden_table {

drivers/gpu/drm/amd/display/dc/dce/dce_aux.c+19−0

@@ -595,6 +595,25 @@ int dce_aux_transfer_raw(struct ddc_service *ddc, · 594 unmodified lines
595	595	return res;
596	596	}
597	597
	598	+int dce_aux_transfer_dmub_raw(struct ddc_service *ddc,
	599	+ struct aux_payload *payload,
	600	+ enum aux_return_code_type *operation_result)
	601	+{
	602	+ struct ddc *ddc_pin = ddc->ddc_pin;
	603	+
	604	+ if (ddc_pin != NULL) {
	605	+ struct dce_aux *aux_engine = ddc->ctx->dc->res_pool->engines[ddc_pin->pin_data->en];
	606	+ /* XXX: Workaround to configure ddc channels for aux transactions */
	607	+ if (!acquire(aux_engine, ddc_pin)) {
	608	+ *operation_result = AUX_RET_ERROR_ENGINE_ACQUIRE;
	609	+ return -1;
	610	+ }
	611	+ release_engine(aux_engine);
	612	+ }
	613	+
	614	+ return dm_helper_dmub_aux_transfer_sync(ddc->ctx, ddc->link, payload, operation_result);
	615	+}
	616	+
598	617	#define AUX_MAX_RETRIES 7
599	618	#define AUX_MAX_DEFER_RETRIES 7
600	619	#define AUX_MAX_I2C_DEFER_RETRIES 7

drivers/gpu/drm/amd/display/dc/dce/dce_aux.h+3−0

@@ -304,6 +304,9 @@ int dce_aux_transfer_raw(struct ddc_service *ddc, · 303 unmodified lines
304	304	struct aux_payload *cmd,
305	305	enum aux_return_code_type *operation_result);
306	306
	307	+int dce_aux_transfer_dmub_raw(struct ddc_service *ddc,
	308	+ struct aux_payload *payload,
	309	+ enum aux_return_code_type *operation_result);
307	310	bool dce_aux_transfer_with_retries(struct ddc_service *ddc,
308	311	struct aux_payload *cmd);
309	312

drivers/gpu/drm/amd/display/dc/dce/dmub_abm.c+30−0

@@ -56,11 +56,19 @@ static void dmub_abm_enable_fractional_pwm(struct dc_context *dc) · 55 unmodified lines
56	56	{
57	57	union dmub_rb_cmd cmd;
58	58	uint32_t fractional_pwm = (dc->dc->config.disable_fractional_pwm == false) ? 1 : 0;
	59	+ uint32_t edp_id_count = dc->dc_edp_id_count;
	60	+ int i;
	61	+ uint8_t panel_mask = 0;
	62	+
	63	+ for (i = 0; i < edp_id_count; i++)
	64	+ panel_mask \|= 0x01 << i;
59	65
60	66	memset(&cmd, 0, sizeof(cmd));
61	67	cmd.abm_set_pwm_frac.header.type = DMUB_CMD__ABM;
62	68	cmd.abm_set_pwm_frac.header.sub_type = DMUB_CMD__ABM_SET_PWM_FRAC;
63	69	cmd.abm_set_pwm_frac.abm_set_pwm_frac_data.fractional_pwm = fractional_pwm;
	70	+ cmd.abm_set_pwm_frac.abm_set_pwm_frac_data.version = DMUB_CMD_ABM_CONTROL_VERSION_1;
	71	+ cmd.abm_set_pwm_frac.abm_set_pwm_frac_data.panel_mask = panel_mask;
64	72	cmd.abm_set_pwm_frac.header.payload_bytes = sizeof(struct dmub_cmd_abm_set_pwm_frac_data);
65	73
66	74	dc_dmub_srv_cmd_queue(dc->dmub_srv, &cmd);
@@ -135,11 +143,24 @@ static bool dmub_abm_set_level(struct abm *abm, uint32_t level) · 68 unmodified lines
135	143	{
136	144	union dmub_rb_cmd cmd;
137	145	struct dc_context *dc = abm->ctx;
	146	+ struct dc_link *edp_links[MAX_NUM_EDP];
	147	+ int i;
	148	+ int edp_num;
	149	+ uint8_t panel_mask = 0;
	150	+
	151	+ get_edp_links(dc->dc, edp_links, &edp_num);
	152	+
	153	+ for (i = 0; i < edp_num; i++) {
	154	+ if (edp_links[i]->link_status.link_active)
	155	+ panel_mask \|= (0x01 << i);
	156	+ }
138	157
139	158	memset(&cmd, 0, sizeof(cmd));
140	159	cmd.abm_set_level.header.type = DMUB_CMD__ABM;
141	160	cmd.abm_set_level.header.sub_type = DMUB_CMD__ABM_SET_LEVEL;
142	161	cmd.abm_set_level.abm_set_level_data.level = level;
	162	+ cmd.abm_set_level.abm_set_level_data.version = DMUB_CMD_ABM_CONTROL_VERSION_1;
	163	+ cmd.abm_set_level.abm_set_level_data.panel_mask = panel_mask;
143	164	cmd.abm_set_level.header.payload_bytes = sizeof(struct dmub_cmd_abm_set_level_data);
144	165
145	166	dc_dmub_srv_cmd_queue(dc->dmub_srv, &cmd);
@@ -155,6 +176,12 @@ static bool dmub_abm_init_config(struct abm *abm, · 9 unmodified lines
155	176	{
156	177	union dmub_rb_cmd cmd;
157	178	struct dc_context *dc = abm->ctx;
	179	+ uint32_t edp_id_count = dc->dc_edp_id_count;
	180	+ int i;
	181	+ uint8_t panel_mask = 0;
	182	+
	183	+ for (i = 0; i < edp_id_count; i++)
	184	+ panel_mask \|= 0x01 << i;
158	185
159	186	// TODO: Optimize by only reading back final 4 bytes
160	187	dmub_flush_buffer_mem(&dc->dmub_srv->dmub->scratch_mem_fb);
@@ -168,6 +195,9 @@ static bool dmub_abm_init_config(struct abm *abm, · 7 unmodified lines
168	195	cmd.abm_init_config.header.sub_type = DMUB_CMD__ABM_INIT_CONFIG;
169	196	cmd.abm_init_config.abm_init_config_data.src.quad_part = dc->dmub_srv->dmub->scratch_mem_fb.gpu_addr;
170	197	cmd.abm_init_config.abm_init_config_data.bytes = bytes;
	198	+ cmd.abm_init_config.abm_init_config_data.version = DMUB_CMD_ABM_CONTROL_VERSION_1;
	199	+ cmd.abm_init_config.abm_init_config_data.panel_mask = panel_mask;
	200	+
171	201	cmd.abm_init_config.header.payload_bytes = sizeof(struct dmub_cmd_abm_init_config_data);
172	202
173	203	dc_dmub_srv_cmd_queue(dc->dmub_srv, &cmd);

drivers/gpu/drm/amd/display/dc/dcn10/dcn10_dpp_dscl.c+6−3

@@ -496,10 +496,13 @@ static enum lb_memory_config dpp1_dscl_find_lb_memory_config(struct dcn10_dpp *d · 495 unmodified lines
496	496	int vtaps_c = scl_data->taps.v_taps_c;
497	497	int ceil_vratio = dc_fixpt_ceil(scl_data->ratios.vert);
498	498	int ceil_vratio_c = dc_fixpt_ceil(scl_data->ratios.vert_c);
499		− enum lb_memory_config mem_cfg = LB_MEMORY_CONFIG_0;
500	499
501		− if (dpp->base.ctx->dc->debug.use_max_lb)
502		− return mem_cfg;
	500	+ if (dpp->base.ctx->dc->debug.use_max_lb) {
	501	+ if (scl_data->format == PIXEL_FORMAT_420BPP8
	502	+ \|\| scl_data->format == PIXEL_FORMAT_420BPP10)
	503	+ return LB_MEMORY_CONFIG_3;
	504	+ return LB_MEMORY_CONFIG_0;
	505	+ }
503	506
504	507	dpp->base.caps->dscl_calc_lb_num_partitions(
505	508	scl_data, LB_MEMORY_CONFIG_1, &num_part_y, &num_part_c);

drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.c+16−139

@@ -2956,35 +2956,6 @@ void dcn10_program_pipe( · 2955 unmodified lines
2956	2956	{
2957	2957	struct dce_hwseq *hws = dc->hwseq;
2958	2958
2959		− if (pipe_ctx->plane_state->update_flags.bits.full_update)
2960		− dcn10_enable_plane(dc, pipe_ctx, context);
2961		−
2962		− dcn10_update_dchubp_dpp(dc, pipe_ctx, context);
2963		−
2964		− hws->funcs.set_hdr_multiplier(pipe_ctx);
2965		−
2966		− if (pipe_ctx->plane_state->update_flags.bits.full_update \|\|
2967		− pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change \|\|
2968		− pipe_ctx->plane_state->update_flags.bits.gamma_change)
2969		− hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
2970		−
2971		− /* dcn10_translate_regamma_to_hw_format takes 750us to finish
2972		− * only do gamma programming for full update.
2973		− * TODO: This can be further optimized/cleaned up
2974		− * Always call this for now since it does memcmp inside before
2975		− * doing heavy calculation and programming
2976		− */
2977		− if (pipe_ctx->plane_state->update_flags.bits.full_update)
2978		− hws->funcs.set_output_transfer_func(dc, pipe_ctx, pipe_ctx->stream);
2979		−}
2980		−
2981		−static void dcn10_program_all_pipe_in_tree(
2982		− struct dc *dc,
2983		− struct pipe_ctx *pipe_ctx,
2984		− struct dc_state *context)
2985		−{
2986		− struct dce_hwseq *hws = dc->hwseq;
2987		−
2988	2959	if (pipe_ctx->top_pipe == NULL) {
2989	2960	bool blank = !is_pipe_tree_visible(pipe_ctx);
2990	2961
@@ -3004,35 +2975,26 @@ static void dcn10_program_all_pipe_in_tree( · 13 unmodified lines
3004	2975	hws->funcs.blank_pixel_data(dc, pipe_ctx, blank);
3005	2976	}
3006	2977
3007		− if (pipe_ctx->plane_state != NULL)
3008		− hws->funcs.program_pipe(dc, pipe_ctx, context);
3009		−
3010		− if (pipe_ctx->bottom_pipe != NULL && pipe_ctx->bottom_pipe != pipe_ctx)
3011		− dcn10_program_all_pipe_in_tree(dc, pipe_ctx->bottom_pipe, context);
3012		−}
3013		−
3014		−static struct pipe_ctx *dcn10_find_top_pipe_for_stream(
3015		− struct dc *dc,
3016		− struct dc_state *context,
3017		− const struct dc_stream_state *stream)
3018		−{
3019		− int i;
	2978	+ if (pipe_ctx->plane_state->update_flags.bits.full_update)
	2979	+ dcn10_enable_plane(dc, pipe_ctx, context);
3020	2980
3021		− for (i = 0; i < dc->res_pool->pipe_count; i++) {
3022		− struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
3023		− struct pipe_ctx *old_pipe_ctx =
3024		− &dc->current_state->res_ctx.pipe_ctx[i];
	2981	+ dcn10_update_dchubp_dpp(dc, pipe_ctx, context);
3025	2982
3026		− if (!pipe_ctx->plane_state && !old_pipe_ctx->plane_state)
3027		− continue;
	2983	+ hws->funcs.set_hdr_multiplier(pipe_ctx);
3028	2984
3029		− if (pipe_ctx->stream != stream)
3030		− continue;
	2985	+ if (pipe_ctx->plane_state->update_flags.bits.full_update \|\|
	2986	+ pipe_ctx->plane_state->update_flags.bits.in_transfer_func_change \|\|
	2987	+ pipe_ctx->plane_state->update_flags.bits.gamma_change)
	2988	+ hws->funcs.set_input_transfer_func(dc, pipe_ctx, pipe_ctx->plane_state);
3031	2989
3032		− if (!pipe_ctx->top_pipe && !pipe_ctx->prev_odm_pipe)
3033		− return pipe_ctx;
3034		− }
3035		− return NULL;
	2990	+ /* dcn10_translate_regamma_to_hw_format takes 750us to finish
	2991	+ * only do gamma programming for full update.
	2992	+ * TODO: This can be further optimized/cleaned up
	2993	+ * Always call this for now since it does memcmp inside before
	2994	+ * doing heavy calculation and programming
	2995	+ */
	2996	+ if (pipe_ctx->plane_state->update_flags.bits.full_update)
	2997	+ hws->funcs.set_output_transfer_func(dc, pipe_ctx, pipe_ctx->stream);
3036	2998	}
3037	2999
3038	3000	void dcn10_wait_for_pending_cleared(struct dc *dc,
@@ -3067,91 +3029,6 @@ void dcn10_wait_for_pending_cleared(struct dc *dc, · 28 unmodified lines
3067	3029	}
3068	3030	}
3069	3031
3070		−void dcn10_apply_ctx_for_surface(
3071		− struct dc *dc,
3072		− const struct dc_stream_state *stream,
3073		− int num_planes,
3074		− struct dc_state *context)
3075		−{
3076		− struct dce_hwseq *hws = dc->hwseq;
3077		− int i;
3078		− struct timing_generator *tg;
3079		− uint32_t underflow_check_delay_us;
3080		− bool interdependent_update = false;
3081		− struct pipe_ctx *top_pipe_to_program =
3082		− dcn10_find_top_pipe_for_stream(dc, context, stream);
3083		− DC_LOGGER_INIT(dc->ctx->logger);
3084		−
3085		− // Clear pipe_ctx flag
3086		− for (i = 0; i < dc->res_pool->pipe_count; i++) {
3087		− struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
3088		− pipe_ctx->update_flags.raw = 0;
3089		− }
3090		−
3091		− if (!top_pipe_to_program)
3092		− return;
3093		−
3094		− tg = top_pipe_to_program->stream_res.tg;
3095		−
3096		− interdependent_update = top_pipe_to_program->plane_state &&
3097		− top_pipe_to_program->plane_state->update_flags.bits.full_update;
3098		−
3099		− underflow_check_delay_us = dc->debug.underflow_assert_delay_us;
3100		−
3101		− if (underflow_check_delay_us != 0xFFFFFFFF && hws->funcs.did_underflow_occur)
3102		− ASSERT(hws->funcs.did_underflow_occur(dc, top_pipe_to_program));
3103		−
3104		− if (underflow_check_delay_us != 0xFFFFFFFF)
3105		− udelay(underflow_check_delay_us);
3106		−
3107		− if (underflow_check_delay_us != 0xFFFFFFFF && hws->funcs.did_underflow_occur)
3108		− ASSERT(hws->funcs.did_underflow_occur(dc, top_pipe_to_program));
3109		−
3110		− if (num_planes == 0) {
3111		− /* OTG blank before remove all front end */
3112		− hws->funcs.blank_pixel_data(dc, top_pipe_to_program, true);
3113		− }
3114		−
3115		− /* Disconnect unused mpcc */
3116		− for (i = 0; i < dc->res_pool->pipe_count; i++) {
3117		− struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
3118		− struct pipe_ctx *old_pipe_ctx =
3119		− &dc->current_state->res_ctx.pipe_ctx[i];
3120		−
3121		− if ((!pipe_ctx->plane_state \|\|
3122		− pipe_ctx->stream_res.tg != old_pipe_ctx->stream_res.tg) &&
3123		− old_pipe_ctx->plane_state &&
3124		− old_pipe_ctx->stream_res.tg == tg) {
3125		−
3126		− hws->funcs.plane_atomic_disconnect(dc, old_pipe_ctx);
3127		− pipe_ctx->update_flags.bits.disable = 1;
3128		−
3129		− DC_LOG_DC("Reset mpcc for pipe %d\n",
3130		− old_pipe_ctx->pipe_idx);
3131		− }
3132		− }
3133		−
3134		− if (num_planes > 0)
3135		− dcn10_program_all_pipe_in_tree(dc, top_pipe_to_program, context);
3136		−
3137		− /* Program secondary blending tree and writeback pipes */
3138		− if ((stream->num_wb_info > 0) && (hws->funcs.program_all_writeback_pipes_in_tree))
3139		− hws->funcs.program_all_writeback_pipes_in_tree(dc, stream, context);
3140		− if (interdependent_update)
3141		− for (i = 0; i < dc->res_pool->pipe_count; i++) {
3142		− struct pipe_ctx *pipe_ctx = &context->res_ctx.pipe_ctx[i];
3143		− /* Skip inactive pipes and ones already updated */
3144		− if (!pipe_ctx->stream \|\| pipe_ctx->stream == stream \|\|
3145		− !pipe_ctx->plane_state \|\| !tg->funcs->is_tg_enabled(tg))
3146		− continue;
3147		−
3148		− pipe_ctx->plane_res.hubp->funcs->hubp_setup_interdependent(
3149		− pipe_ctx->plane_res.hubp,
3150		− &pipe_ctx->dlg_regs,
3151		− &pipe_ctx->ttu_regs);
3152		− }
3153		−}
3154		−
3155	3032	void dcn10_post_unlock_program_front_end(
3156	3033	struct dc *dc,
3157	3034	struct dc_state *context)

drivers/gpu/drm/amd/display/dc/dcn10/dcn10_hw_sequencer.h+0−5

@@ -80,11 +80,6 @@ void dcn10_lock_all_pipes( · 79 unmodified lines
80	80	struct dc *dc,
81	81	struct dc_state *context,
82	82	bool lock);
83		−void dcn10_apply_ctx_for_surface(
84		− struct dc *dc,
85		− const struct dc_stream_state *stream,
86		− int num_planes,
87		− struct dc_state *context);
88	83	void dcn10_post_unlock_program_front_end(
89	84	struct dc *dc,
90	85	struct dc_state *context);

drivers/gpu/drm/amd/display/dc/dcn10/dcn10_init.c+3−1

@@ -26,13 +26,15 @@ · 25 unmodified lines
26	26	#include "hw_sequencer_private.h"
27	27	#include "dce110/dce110_hw_sequencer.h"
28	28	#include "dcn10_hw_sequencer.h"
	29	+#include "dcn20/dcn20_hwseq.h"
29	30
30	31	static const struct hw_sequencer_funcs dcn10_funcs = {
31	32	.program_gamut_remap = dcn10_program_gamut_remap,
32	33	.init_hw = dcn10_init_hw,
33	34	.power_down_on_boot = dcn10_power_down_on_boot,
34	35	.apply_ctx_to_hw = dce110_apply_ctx_to_hw,
35		− .apply_ctx_for_surface = dcn10_apply_ctx_for_surface,
	36	+ .apply_ctx_for_surface = NULL,
	37	+ .program_front_end_for_ctx = dcn20_program_front_end_for_ctx,
36	38	.post_unlock_program_front_end = dcn10_post_unlock_program_front_end,
37	39	.wait_for_pending_cleared = dcn10_wait_for_pending_cleared,
38	40	.update_plane_addr = dcn10_update_plane_addr,

drivers/gpu/drm/amd/display/dc/dcn20/dcn20_dpp_cm.c+2−2

@@ -482,7 +482,7 @@ bool dpp20_program_blnd_lut( · 481 unmodified lines
482	482	next_mode = LUT_RAM_A;
483	483
484	484	dpp20_power_on_blnd_lut(dpp_base, true);
485		− dpp20_configure_blnd_lut(dpp_base, next_mode == LUT_RAM_A ? true:false);
	485	+ dpp20_configure_blnd_lut(dpp_base, next_mode == LUT_RAM_A);
486	486
487	487	if (next_mode == LUT_RAM_A)
488	488	dpp20_program_blnd_luta_settings(dpp_base, params);
@@ -893,7 +893,7 @@ bool dpp20_program_shaper( · 404 unmodified lines
893	893	else
894	894	next_mode = LUT_RAM_A;
895	895
896		− dpp20_configure_shaper_lut(dpp_base, next_mode == LUT_RAM_A ? true:false);
	896	+ dpp20_configure_shaper_lut(dpp_base, next_mode == LUT_RAM_A);
897	897
898	898	if (next_mode == LUT_RAM_A)
899	899	dpp20_program_shaper_luta_settings(dpp_base, params);

drivers/gpu/drm/amd/display/dc/dcn20/dcn20_hwseq.c+5−1

@@ -1700,7 +1700,11 @@ void dcn20_program_front_end_for_ctx( · 1699 unmodified lines
1700	1700
1701	1701	if (pipe->plane_state && !pipe->top_pipe) {
1702	1702	while (pipe) {
1703		− dcn20_program_pipe(dc, pipe, context);
	1703	+ if (hws->funcs.program_pipe)
	1704	+ hws->funcs.program_pipe(dc, pipe, context);
	1705	+ else
	1706	+ dcn20_program_pipe(dc, pipe, context);
	1707	+
1704	1708	pipe = pipe->bottom_pipe;
1705	1709	}
1706	1710	/* Program secondary blending tree and writeback pipes */

drivers/gpu/drm/amd/display/dc/dcn20/dcn20_resource.c+1−1

@@ -3667,7 +3667,7 @@ static bool dcn20_resource_construct( · 3666 unmodified lines
3667	3667	int i;
3668	3668	struct dc_context *ctx = dc->ctx;
3669	3669	struct irq_service_init_data init_data;
3670		− struct ddc_service_init_data ddc_init_data;
	3670	+ struct ddc_service_init_data ddc_init_data = {0};
3671	3671	struct _vcs_dpi_soc_bounding_box_st *loaded_bb =
3672	3672	get_asic_rev_soc_bb(ctx->asic_id.hw_internal_rev);
3673	3673	struct _vcs_dpi_ip_params_st *loaded_ip =

drivers/gpu/drm/amd/display/dc/dcn21/dcn21_hwseq.c+1−1

@@ -218,7 +218,7 @@ bool dcn21_set_backlight_level(struct pipe_ctx *pipe_ctx, · 217 unmodified lines
218	218	cmd.abm_set_backlight.header.sub_type = DMUB_CMD__ABM_SET_BACKLIGHT;
219	219	cmd.abm_set_backlight.abm_set_backlight_data.frame_ramp = frame_ramp;
220	220	cmd.abm_set_backlight.abm_set_backlight_data.backlight_user_level = backlight_pwm_u16_16;
221		− cmd.abm_set_backlight.abm_set_backlight_data.version = DMUB_CMD_ABM_SET_BACKLIGHT_VERSION_1;
	221	+ cmd.abm_set_backlight.abm_set_backlight_data.version = DMUB_CMD_ABM_CONTROL_VERSION_1;
222	222	cmd.abm_set_backlight.abm_set_backlight_data.panel_mask = (0x01 << panel_cntl->inst);
223	223	cmd.abm_set_backlight.header.payload_bytes = sizeof(struct dmub_cmd_abm_set_backlight_data);
224	224

drivers/gpu/drm/amd/display/dc/dcn21/dcn21_resource.c+21−12

@@ -1575,10 +1575,12 @@ static struct _vcs_dpi_voltage_scaling_st construct_low_pstate_lvl(struct clk_li · 1574 unmodified lines
1575	1575	low_pstate_lvl.phyclk_d18_mhz = dcn2_1_soc.clock_limits[high_voltage_lvl].phyclk_d18_mhz;
1576	1576	low_pstate_lvl.phyclk_mhz = dcn2_1_soc.clock_limits[high_voltage_lvl].phyclk_mhz;
1577	1577
1578		− for (i = clk_table->num_entries; i > 1; i--)
1579		− clk_table->entries[i] = clk_table->entries[i-1];
1580		− clk_table->entries[1] = clk_table->entries[0];
1581		− clk_table->num_entries++;
	1578	+ if (clk_table->num_entries < MAX_NUM_DPM_LVL) {
	1579	+ for (i = clk_table->num_entries; i > 1; i--)
	1580	+ clk_table->entries[i] = clk_table->entries[i-1];
	1581	+ clk_table->entries[1] = clk_table->entries[0];
	1582	+ clk_table->num_entries++;
	1583	+ }
1582	1584
1583	1585	return low_pstate_lvl;
1584	1586	}
@@ -1610,10 +1612,6 @@ static void update_bw_bounding_box(struct dc dc, struct clk_bw_params bw_param · 25 unmodified lines
1610	1612	}
1611	1613	}
1612	1614
1613		− /* clk_table[1] is reserved for min DF PState. skip here to fill in later. */
1614		− if (i == 1)
1615		− k++;
1616		−
1617	1615	clock_limits[k].state = k;
1618	1616	clock_limits[k].dcfclk_mhz = clk_table->entries[i].dcfclk_mhz;
1619	1617	clock_limits[k].fabricclk_mhz = clk_table->entries[i].fclk_mhz;
@@ -1630,14 +1628,25 @@ static void update_bw_bounding_box(struct dc dc, struct clk_bw_params bw_param · 10 unmodified lines
1630	1628
1631	1629	k++;
1632	1630	}
1633		− for (i = 0; i < clk_table->num_entries + 1; i++)
1634		− dcn2_1_soc.clock_limits[i] = clock_limits[i];
	1631	+
	1632	+ if (clk_table->num_entries >= MAX_NUM_DPM_LVL) {
	1633	+ for (i = 0; i < clk_table->num_entries + 1; i++)
	1634	+ dcn2_1_soc.clock_limits[i] = clock_limits[i];
	1635	+ } else {
	1636	+ dcn2_1_soc.clock_limits[0] = clock_limits[0];
	1637	+ for (i = 2; i < clk_table->num_entries + 1; i++) {
	1638	+ dcn2_1_soc.clock_limits[i] = clock_limits[i - 1];
	1639	+ dcn2_1_soc.clock_limits[i].state = i;
	1640	+ }
	1641	+ }
	1642	+
1635	1643	if (clk_table->num_entries) {
1636		− dcn2_1_soc.num_states = clk_table->num_entries + 1;
1637	1644	/* fill in min DF PState */
1638	1645	dcn2_1_soc.clock_limits[1] = construct_low_pstate_lvl(clk_table, closest_clk_lvl);
	1646	+ dcn2_1_soc.num_states = clk_table->num_entries;
1639	1647	/* duplicate last level */
1640		− dcn2_1_soc.clock_limits[dcn2_1_soc.num_states] = dcn2_1_soc.clock_limits[dcn2_1_soc.num_states - 1];
	1648	+ dcn2_1_soc.clock_limits[dcn2_1_soc.num_states] =
	1649	+ dcn2_1_soc.clock_limits[dcn2_1_soc.num_states - 1];
1641	1650	dcn2_1_soc.clock_limits[dcn2_1_soc.num_states].state = dcn2_1_soc.num_states;
1642	1651	}
1643	1652

drivers/gpu/drm/amd/display/dc/dcn30/dcn30_dio_stream_encoder.h+4−0

@@ -85,7 +85,9 @@ · 84 unmodified lines
85	85	SRI(DP_MSE_RATE_UPDATE, DP, id), \
86	86	SRI(DP_PIXEL_FORMAT, DP, id), \
87	87	SRI(DP_SEC_CNTL, DP, id), \
	88	+ SRI(DP_SEC_CNTL1, DP, id), \
88	89	SRI(DP_SEC_CNTL2, DP, id), \
	90	+ SRI(DP_SEC_CNTL5, DP, id), \
89	91	SRI(DP_SEC_CNTL6, DP, id), \
90	92	SRI(DP_STEER_FIFO, DP, id), \
91	93	SRI(DP_VID_M, DP, id), \
@@ -169,7 +171,9 @@ · 77 unmodified lines
169	171	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP5_ENABLE, mask_sh),\
170	172	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP6_ENABLE, mask_sh),\
171	173	SE_SF(DP0_DP_SEC_CNTL, DP_SEC_GSP7_ENABLE, mask_sh),\
	174	+ SE_SF(DP0_DP_SEC_CNTL1, DP_SEC_GSP5_LINE_REFERENCE, mask_sh),\
172	175	SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP7_SEND, mask_sh),\
	176	+ SE_SF(DP0_DP_SEC_CNTL5, DP_SEC_GSP5_LINE_NUM, mask_sh),\
173	177	SE_SF(DP0_DP_SEC_CNTL6, DP_SEC_GSP7_LINE_NUM, mask_sh),\
174	178	SE_SF(DP0_DP_SEC_CNTL2, DP_SEC_GSP11_PPS, mask_sh),\
175	179	SE_SF(DP0_DP_GSP11_CNTL, DP_SEC_GSP11_ENABLE, mask_sh),\

drivers/gpu/drm/amd/display/dc/dcn30/dcn30_mpc.c+1−1

@@ -874,7 +874,7 @@ bool mpc3_program_shaper( · 873 unmodified lines
874	874	else
875	875	next_mode = LUT_RAM_A;
876	876
877		− mpc3_configure_shaper_lut(mpc, next_mode == LUT_RAM_A ? true:false, rmu_idx);
	877	+ mpc3_configure_shaper_lut(mpc, next_mode == LUT_RAM_A, rmu_idx);
878	878
879	879	if (next_mode == LUT_RAM_A)
880	880	mpc3_program_shaper_luta_settings(mpc, params, rmu_idx);

drivers/gpu/drm/amd/display/dc/dcn30/dcn30_optc.c+1−1

@@ -97,7 +97,7 @@ void optc3_lock_doublebuffer_disable(struct timing_generator *optc) · 96 unmodified lines
97	97	MASTER_UPDATE_LOCK_DB_END_Y, 0);
98	98
99	99	REG_UPDATE(OTG_GLOBAL_CONTROL2, GLOBAL_UPDATE_LOCK_EN, 0);
100		− REG_UPDATE(OTG_GLOBAL_CONTROL0, MASTER_UPDATE_LOCK_DB_EN, 1);
	100	+ REG_UPDATE(OTG_GLOBAL_CONTROL0, MASTER_UPDATE_LOCK_DB_EN, 0);
101	101	}
102	102
103	103	void optc3_lock(struct timing_generator *optc)

drivers/gpu/drm/amd/display/dc/dcn30/dcn30_resource.c+1−1

@@ -2538,7 +2538,7 @@ static bool dcn30_resource_construct( · 2537 unmodified lines
2538	2538	int i;
2539	2539	struct dc_context *ctx = dc->ctx;
2540	2540	struct irq_service_init_data init_data;
2541		− struct ddc_service_init_data ddc_init_data;
	2541	+ struct ddc_service_init_data ddc_init_data = {0};
2542	2542	uint32_t pipe_fuses = read_pipe_fuses(ctx);
2543	2543	uint32_t num_pipes = 0;
2544	2544

drivers/gpu/drm/amd/display/dc/dm_helpers.h+8−1

@@ -34,6 +34,8 @@ · 33 unmodified lines
34	34	#include "dc.h"
35	35
36	36	struct dp_mst_stream_allocation_table;
	37	+struct aux_payload;
	38	+enum aux_return_code_type;
37	39
38	40	/*
39	41	* Allocate memory accessible by the GPU
@@ -158,6 +160,11 @@ void dm_set_dcn_clocks( · 118 unmodified lines
158	160	struct dc_context *ctx,
159	161	struct dc_clocks *clks);
160	162
161		−bool dm_helpers_dmub_outbox0_interrupt_control(struct dc_context *ctx, bool enable);
	163	+bool dm_helpers_dmub_outbox_interrupt_control(struct dc_context *ctx, bool enable);
162	164
	165	+int dm_helper_dmub_aux_transfer_sync(
	166	+ struct dc_context *ctx,
	167	+ const struct dc_link *link,
	168	+ struct aux_payload *payload,
	169	+ enum aux_return_code_type *operation_result);
163	170	#endif /* __DM_HELPERS__ */

drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20.c+13−13

@@ -2895,7 +2895,7 @@ static void dml20_DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 2894 unmodified lines
2895	2895	RoundedUpMaxSwathSizeBytesC = 0.0;
2896	2896
2897	2897	if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC
2898		− <= mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0) {
	2898	+ <= mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0) {
2899	2899	mode_lib->vba.SwathHeightY[k] = MaximumSwathHeightY;
2900	2900	mode_lib->vba.SwathHeightC[k] = MaximumSwathHeightC;
2901	2901	} else {
@@ -2904,17 +2904,17 @@ static void dml20_DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 2 unmodified lines
2904	2904	}
2905	2905
2906	2906	if (mode_lib->vba.SwathHeightC[k] == 0) {
2907		− mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte * 1024;
	2907	+ mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0] * 1024;
2908	2908	mode_lib->vba.DETBufferSizeC[k] = 0;
2909	2909	} else if (mode_lib->vba.SwathHeightY[k] <= mode_lib->vba.SwathHeightC[k]) {
2910		− mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte
	2910	+ mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2911	2911	* 1024.0 / 2;
2912		− mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte
	2912	+ mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2913	2913	* 1024.0 / 2;
2914	2914	} else {
2915		− mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte
	2915	+ mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2916	2916	* 1024.0 * 2 / 3;
2917		− mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte
	2917	+ mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2918	2918	* 1024.0 / 3;
2919	2919	}
2920	2920	}
@@ -3819,7 +3819,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 898 unmodified lines
3819	3819	mode_lib->vba.MaximumSwathWidthInDETBuffer =
3820	3820	dml_min(
3821	3821	mode_lib->vba.MaximumSwathWidthSupport,
3822		− mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0
	3822	+ mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0
3823	3823	/ (locals->BytePerPixelInDETY[k]
3824	3824	* locals->MinSwathHeightY[k]
3825	3825	+ locals->BytePerPixelInDETC[k]
@@ -4322,7 +4322,7 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 496 unmodified lines
4322	4322	locals->RoundedUpMaxSwathSizeBytesC = 0;
4323	4323	}
4324	4324
4325		− if (locals->RoundedUpMaxSwathSizeBytesY + locals->RoundedUpMaxSwathSizeBytesC <= locals->DETBufferSizeInKByte * 1024 / 2) {
	4325	+ if (locals->RoundedUpMaxSwathSizeBytesY + locals->RoundedUpMaxSwathSizeBytesC <= locals->DETBufferSizeInKByte[0] * 1024 / 2) {
4326	4326	locals->SwathHeightYPerState[i][j][k] = locals->MaxSwathHeightY[k];
4327	4327	locals->SwathHeightCPerState[i][j][k] = locals->MaxSwathHeightC[k];
4328	4328	} else {
@@ -4331,15 +4331,15 @@ void dml20_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 2 unmodified lines
4331	4331	}
4332	4332
4333	4333	if (locals->BytePerPixelInDETC[k] == 0) {
4334		− locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
	4334	+ locals->LinesInDETLuma = locals->DETBufferSizeInKByte[0] * 1024 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
4335	4335	locals->LinesInDETChroma = 0;
4336	4336	} else if (locals->SwathHeightYPerState[i][j][k] <= locals->SwathHeightCPerState[i][j][k]) {
4337		− locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 / 2 / locals->BytePerPixelInDETY[k] /
	4337	+ locals->LinesInDETLuma = locals->DETBufferSizeInKByte[0] * 1024 / 2 / locals->BytePerPixelInDETY[k] /
4338	4338	locals->SwathWidthYPerState[i][j][k];
4339		− locals->LinesInDETChroma = locals->DETBufferSizeInKByte * 1024 / 2 / locals->BytePerPixelInDETC[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
	4339	+ locals->LinesInDETChroma = locals->DETBufferSizeInKByte[0] * 1024 / 2 / locals->BytePerPixelInDETC[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
4340	4340	} else {
4341		− locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 * 2 / 3 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
4342		− locals->LinesInDETChroma = locals->DETBufferSizeInKByte * 1024 / 3 / locals->BytePerPixelInDETY[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
	4341	+ locals->LinesInDETLuma = locals->DETBufferSizeInKByte[0] * 1024 * 2 / 3 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
	4342	+ locals->LinesInDETChroma = locals->DETBufferSizeInKByte[0] * 1024 / 3 / locals->BytePerPixelInDETY[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
4343	4343	}
4344	4344
4345	4345	locals->EffectiveLBLatencyHidingSourceLinesLuma = dml_min(locals->MaxLineBufferLines,

drivers/gpu/drm/amd/display/dc/dml/dcn20/display_mode_vba_20v2.c+13−13

@@ -2968,7 +2968,7 @@ static void dml20v2_DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 2967 unmodified lines
2968	2968	RoundedUpMaxSwathSizeBytesC = 0.0;
2969	2969
2970	2970	if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC
2971		− <= mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0) {
	2971	+ <= mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0) {
2972	2972	mode_lib->vba.SwathHeightY[k] = MaximumSwathHeightY;
2973	2973	mode_lib->vba.SwathHeightC[k] = MaximumSwathHeightC;
2974	2974	} else {
@@ -2977,17 +2977,17 @@ static void dml20v2_DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 2 unmodified lines
2977	2977	}
2978	2978
2979	2979	if (mode_lib->vba.SwathHeightC[k] == 0) {
2980		− mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte * 1024;
	2980	+ mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0] * 1024;
2981	2981	mode_lib->vba.DETBufferSizeC[k] = 0;
2982	2982	} else if (mode_lib->vba.SwathHeightY[k] <= mode_lib->vba.SwathHeightC[k]) {
2983		− mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte
	2983	+ mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2984	2984	* 1024.0 / 2;
2985		− mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte
	2985	+ mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2986	2986	* 1024.0 / 2;
2987	2987	} else {
2988		− mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte
	2988	+ mode_lib->vba.DETBufferSizeY[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2989	2989	* 1024.0 * 2 / 3;
2990		− mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte
	2990	+ mode_lib->vba.DETBufferSizeC[k] = mode_lib->vba.DETBufferSizeInKByte[0]
2991	2991	* 1024.0 / 3;
2992	2992	}
2993	2993	}
@@ -3926,7 +3926,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode · 932 unmodified lines
3926	3926	mode_lib->vba.MaximumSwathWidthInDETBuffer =
3927	3927	dml_min(
3928	3928	mode_lib->vba.MaximumSwathWidthSupport,
3929		− mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0
	3929	+ mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0
3930	3930	/ (locals->BytePerPixelInDETY[k]
3931	3931	* locals->MinSwathHeightY[k]
3932	3932	+ locals->BytePerPixelInDETC[k]
@@ -4443,7 +4443,7 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode · 510 unmodified lines
4443	4443	locals->RoundedUpMaxSwathSizeBytesC = 0;
4444	4444	}
4445	4445
4446		− if (locals->RoundedUpMaxSwathSizeBytesY + locals->RoundedUpMaxSwathSizeBytesC <= locals->DETBufferSizeInKByte * 1024 / 2) {
	4446	+ if (locals->RoundedUpMaxSwathSizeBytesY + locals->RoundedUpMaxSwathSizeBytesC <= locals->DETBufferSizeInKByte[0] * 1024 / 2) {
4447	4447	locals->SwathHeightYPerState[i][j][k] = locals->MaxSwathHeightY[k];
4448	4448	locals->SwathHeightCPerState[i][j][k] = locals->MaxSwathHeightC[k];
4449	4449	} else {
@@ -4452,15 +4452,15 @@ void dml20v2_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode · 2 unmodified lines
4452	4452	}
4453	4453
4454	4454	if (locals->BytePerPixelInDETC[k] == 0) {
4455		− locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
	4455	+ locals->LinesInDETLuma = locals->DETBufferSizeInKByte[0] * 1024 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
4456	4456	locals->LinesInDETChroma = 0;
4457	4457	} else if (locals->SwathHeightYPerState[i][j][k] <= locals->SwathHeightCPerState[i][j][k]) {
4458		− locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 / 2 / locals->BytePerPixelInDETY[k] /
	4458	+ locals->LinesInDETLuma = locals->DETBufferSizeInKByte[0] * 1024 / 2 / locals->BytePerPixelInDETY[k] /
4459	4459	locals->SwathWidthYPerState[i][j][k];
4460		− locals->LinesInDETChroma = locals->DETBufferSizeInKByte * 1024 / 2 / locals->BytePerPixelInDETC[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
	4460	+ locals->LinesInDETChroma = locals->DETBufferSizeInKByte[0] * 1024 / 2 / locals->BytePerPixelInDETC[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
4461	4461	} else {
4462		− locals->LinesInDETLuma = locals->DETBufferSizeInKByte * 1024 * 2 / 3 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
4463		− locals->LinesInDETChroma = locals->DETBufferSizeInKByte * 1024 / 3 / locals->BytePerPixelInDETY[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
	4462	+ locals->LinesInDETLuma = locals->DETBufferSizeInKByte[0] * 1024 * 2 / 3 / locals->BytePerPixelInDETY[k] / locals->SwathWidthYPerState[i][j][k];
	4463	+ locals->LinesInDETChroma = locals->DETBufferSizeInKByte[0] * 1024 / 3 / locals->BytePerPixelInDETY[k] / (locals->SwathWidthYPerState[i][j][k] / 2);
4464	4464	}
4465	4465
4466	4466	locals->EffectiveLBLatencyHidingSourceLinesLuma = dml_min(locals->MaxLineBufferLines,

drivers/gpu/drm/amd/display/dc/dml/dcn21/display_mode_vba_21.c+29−29

@@ -148,7 +148,7 @@ static double CalculateDCCConfiguration( · 147 unmodified lines
148	148	bool DCCProgrammingAssumesScanDirectionUnknown,
149	149	unsigned int ViewportWidth,
150	150	unsigned int ViewportHeight,
151		− double DETBufferSize,
	151	+ unsigned int DETBufferSize,
152	152	unsigned int RequestHeight256Byte,
153	153	unsigned int SwathHeight,
154	154	enum dm_swizzle_mode TilingFormat,
@@ -289,7 +289,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 134 unmodified lines
289	289	unsigned int MaxLineBufferLines,
290	290	unsigned int LineBufferSize,
291	291	unsigned int DPPOutputBufferPixels,
292		− double DETBufferSizeInKByte,
	292	+ unsigned int DETBufferSizeInKByte,
293	293	unsigned int WritebackInterfaceLumaBufferSize,
294	294	unsigned int WritebackInterfaceChromaBufferSize,
295	295	double DCFCLK,
@@ -354,11 +354,11 @@ static void CalculateDCFCLKDeepSleep( · 58 unmodified lines
354	354	double DPPCLK[],
355	355	double *DCFCLKDeepSleep);
356	356	static void CalculateDETBufferSize(
357		− double DETBufferSizeInKByte,
	357	+ unsigned int DETBufferSizeInKByte,
358	358	unsigned int SwathHeightY,
359	359	unsigned int SwathHeightC,
360		− double *DETBufferSizeY,
361		− double *DETBufferSizeC);
	360	+ unsigned int *DETBufferSizeY,
	361	+ unsigned int *DETBufferSizeC);
362	362	static void CalculateUrgentBurstFactor(
363	363	unsigned int DETBufferSizeInKByte,
364	364	unsigned int SwathHeightY,
@@ -1074,7 +1074,7 @@ static double CalculateDCCConfiguration( · 709 unmodified lines
1074	1074	bool DCCProgrammingAssumesScanDirectionUnknown,
1075	1075	unsigned int ViewportWidth,
1076	1076	unsigned int ViewportHeight,
1077		− double DETBufferSize,
	1077	+ unsigned int DETBufferSize,
1078	1078	unsigned int RequestHeight256Byte,
1079	1079	unsigned int SwathHeight,
1080	1080	enum dm_swizzle_mode TilingFormat,
@@ -2246,7 +2246,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 1165 unmodified lines
2246	2246	}
2247	2247
2248	2248	CalculateUrgentBurstFactor(
2249		− mode_lib->vba.DETBufferSizeInKByte,
	2249	+ mode_lib->vba.DETBufferSizeInKByte[0],
2250	2250	mode_lib->vba.SwathHeightY[k],
2251	2251	mode_lib->vba.SwathHeightC[k],
2252	2252	locals->SwathWidthY[k],
@@ -2415,7 +2415,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 162 unmodified lines
2415	2415	mode_lib->vba.MaxLineBufferLines,
2416	2416	mode_lib->vba.LineBufferSize,
2417	2417	mode_lib->vba.DPPOutputBufferPixels,
2418		− mode_lib->vba.DETBufferSizeInKByte,
	2418	+ mode_lib->vba.DETBufferSizeInKByte[0],
2419	2419	mode_lib->vba.WritebackInterfaceLumaBufferSize,
2420	2420	mode_lib->vba.WritebackInterfaceChromaBufferSize,
2421	2421	mode_lib->vba.DCFCLK,
@@ -2588,7 +2588,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 166 unmodified lines
2588	2588	false, // We should always know the direction DCCProgrammingAssumesScanDirectionUnknown,
2589	2589	mode_lib->vba.ViewportWidth[k],
2590	2590	mode_lib->vba.ViewportHeight[k],
2591		− mode_lib->vba.DETBufferSizeInKByte * 1024,
	2591	+ mode_lib->vba.DETBufferSizeInKByte[0] * 1024,
2592	2592	locals->BlockHeight256BytesY[k],
2593	2593	mode_lib->vba.SwathHeightY[k],
2594	2594	mode_lib->vba.SurfaceTiling[k],
@@ -2689,13 +2689,13 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 94 unmodified lines
2689	2689	// Stutter Efficiency
2690	2690	for (k = 0; k < mode_lib->vba.NumberOfActivePlanes; ++k) {
2691	2691	CalculateDETBufferSize(
2692		− mode_lib->vba.DETBufferSizeInKByte,
	2692	+ mode_lib->vba.DETBufferSizeInKByte[0],
2693	2693	mode_lib->vba.SwathHeightY[k],
2694	2694	mode_lib->vba.SwathHeightC[k],
2695	2695	&locals->DETBufferSizeY[k],
2696	2696	&locals->DETBufferSizeC[k]);
2697	2697
2698		− locals->LinesInDETY[k] = locals->DETBufferSizeY[k]
	2698	+ locals->LinesInDETY[k] = (double)locals->DETBufferSizeY[k]
2699	2699	/ locals->BytePerPixelDETY[k] / locals->SwathWidthY[k];
2700	2700	locals->LinesInDETYRoundedDownToSwath[k] = dml_floor(
2701	2701	locals->LinesInDETY[k],
@@ -2984,7 +2984,7 @@ static void DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 282 unmodified lines
2984	2984	RoundedUpMaxSwathSizeBytesC = 0.0;
2985	2985
2986	2986	if (RoundedUpMaxSwathSizeBytesY + RoundedUpMaxSwathSizeBytesC
2987		− <= mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0) {
	2987	+ <= mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0) {
2988	2988	mode_lib->vba.SwathHeightY[k] = MaximumSwathHeightY;
2989	2989	mode_lib->vba.SwathHeightC[k] = MaximumSwathHeightC;
2990	2990	} else {
@@ -2993,7 +2993,7 @@ static void DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 2 unmodified lines
2993	2993	}
2994	2994
2995	2995	CalculateDETBufferSize(
2996		− mode_lib->vba.DETBufferSizeInKByte,
	2996	+ mode_lib->vba.DETBufferSizeInKByte[0],
2997	2997	mode_lib->vba.SwathHeightY[k],
2998	2998	mode_lib->vba.SwathHeightC[k],
2999	2999	&mode_lib->vba.DETBufferSizeY[k],
@@ -3888,7 +3888,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 888 unmodified lines
3888	3888	mode_lib->vba.MaximumSwathWidthInDETBuffer =
3889	3889	dml_min(
3890	3890	mode_lib->vba.MaximumSwathWidthSupport,
3891		− mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0
	3891	+ mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0
3892	3892	/ (locals->BytePerPixelInDETY[k]
3893	3893	* locals->MinSwathHeightY[k]
3894	3894	+ locals->BytePerPixelInDETC[k]
@@ -4437,7 +4437,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 542 unmodified lines
4437	4437	mode_lib->vba.RoundedUpMaxSwathSizeBytesC = 0.0;
4438	4438	}
4439	4439	if (mode_lib->vba.RoundedUpMaxSwathSizeBytesY + mode_lib->vba.RoundedUpMaxSwathSizeBytesC
4440		− <= mode_lib->vba.DETBufferSizeInKByte * 1024.0 / 2.0) {
	4440	+ <= mode_lib->vba.DETBufferSizeInKByte[0] * 1024.0 / 2.0) {
4441	4441	locals->SwathHeightYThisState[k] = locals->MaxSwathHeightY[k];
4442	4442	locals->SwathHeightCThisState[k] = locals->MaxSwathHeightC[k];
4443	4443	} else {
@@ -4801,7 +4801,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 357 unmodified lines
4801	4801	}
4802	4802
4803	4803	CalculateUrgentBurstFactor(
4804		− mode_lib->vba.DETBufferSizeInKByte,
	4804	+ mode_lib->vba.DETBufferSizeInKByte[0],
4805	4805	locals->SwathHeightYThisState[k],
4806	4806	locals->SwathHeightCThisState[k],
4807	4807	locals->SwathWidthYThisState[k],
@@ -4975,7 +4975,7 @@ void dml21_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 167 unmodified lines
4975	4975	mode_lib->vba.MaxLineBufferLines,
4976	4976	mode_lib->vba.LineBufferSize,
4977	4977	mode_lib->vba.DPPOutputBufferPixels,
4978		− mode_lib->vba.DETBufferSizeInKByte,
	4978	+ mode_lib->vba.DETBufferSizeInKByte[0],
4979	4979	mode_lib->vba.WritebackInterfaceLumaBufferSize,
4980	4980	mode_lib->vba.WritebackInterfaceChromaBufferSize,
4981	4981	mode_lib->vba.DCFCLKPerState[i],
@@ -5230,7 +5230,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 248 unmodified lines
5230	5230	unsigned int MaxLineBufferLines,
5231	5231	unsigned int LineBufferSize,
5232	5232	unsigned int DPPOutputBufferPixels,
5233		− double DETBufferSizeInKByte,
	5233	+ unsigned int DETBufferSizeInKByte,
5234	5234	unsigned int WritebackInterfaceLumaBufferSize,
5235	5235	unsigned int WritebackInterfaceChromaBufferSize,
5236	5236	double DCFCLK,
@@ -5285,8 +5285,8 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 48 unmodified lines
5285	5285	double EffectiveLBLatencyHidingC;
5286	5286	double DPPOutputBufferLinesY;
5287	5287	double DPPOutputBufferLinesC;
5288		− double DETBufferSizeY;
5289		− double DETBufferSizeC;
	5288	+ unsigned int DETBufferSizeY;
	5289	+ unsigned int DETBufferSizeC;
5290	5290	double LinesInDETY[DC__NUM_DPP__MAX];
5291	5291	double LinesInDETC;
5292	5292	unsigned int LinesInDETYRoundedDownToSwath[DC__NUM_DPP__MAX];
@@ -5382,12 +5382,12 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 89 unmodified lines
5382	5382	&DETBufferSizeY,
5383	5383	&DETBufferSizeC);
5384	5384
5385		− LinesInDETY[k] = DETBufferSizeY / BytePerPixelDETY[k] / SwathWidthY[k];
	5385	+ LinesInDETY[k] = (double)DETBufferSizeY / BytePerPixelDETY[k] / SwathWidthY[k];
5386	5386	LinesInDETYRoundedDownToSwath[k] = dml_floor(LinesInDETY[k], SwathHeightY[k]);
5387	5387	FullDETBufferingTimeY[k] = LinesInDETYRoundedDownToSwath[k]
5388	5388	* (HTotal[k] / PixelClock[k]) / VRatio[k];
5389	5389	if (BytePerPixelDETC[k] > 0) {
5390		− LinesInDETC = DETBufferSizeC / BytePerPixelDETC[k] / (SwathWidthY[k] / 2.0);
	5390	+ LinesInDETC = (double)DETBufferSizeC / BytePerPixelDETC[k] / (SwathWidthY[k] / 2.0);
5391	5391	LinesInDETCRoundedDownToSwath = dml_floor(LinesInDETC, SwathHeightC[k]);
5392	5392	FullDETBufferingTimeC = LinesInDETCRoundedDownToSwath
5393	5393	* (HTotal[k] / PixelClock[k]) / (VRatio[k] / 2);
@@ -5574,11 +5574,11 @@ static void CalculateDCFCLKDeepSleep( · 180 unmodified lines
5574	5574	}
5575	5575
5576	5576	static void CalculateDETBufferSize(
5577		− double DETBufferSizeInKByte,
	5577	+ unsigned int DETBufferSizeInKByte,
5578	5578	unsigned int SwathHeightY,
5579	5579	unsigned int SwathHeightC,
5580		− double *DETBufferSizeY,
5581		− double *DETBufferSizeC)
	5580	+ unsigned int *DETBufferSizeY,
	5581	+ unsigned int *DETBufferSizeC)
5582	5582	{
5583	5583	if (SwathHeightC == 0) {
5584	5584	DETBufferSizeY = DETBufferSizeInKByte 1024;
@@ -5625,8 +5625,8 @@ static void CalculateUrgentBurstFactor( · 40 unmodified lines
5625	5625	double DETBufferSizeInTimeLumaPre;
5626	5626	double DETBufferSizeInTimeChroma;
5627	5627	double DETBufferSizeInTimeChromaPre;
5628		− double DETBufferSizeY;
5629		− double DETBufferSizeC;
	5628	+ unsigned int DETBufferSizeY;
	5629	+ unsigned int DETBufferSizeC;
5630	5630
5631	5631	*NotEnoughUrgentLatencyHiding = 0;
5632	5632	*NotEnoughUrgentLatencyHidingPre = 0;
@@ -5663,7 +5663,7 @@ static void CalculateUrgentBurstFactor( · 30 unmodified lines
5663	5663	&DETBufferSizeY,
5664	5664	&DETBufferSizeC);
5665	5665
5666		− LinesInDETLuma = DETBufferSizeY / BytePerPixelInDETY / SwathWidthY;
	5666	+ LinesInDETLuma = (double)DETBufferSizeY / BytePerPixelInDETY / SwathWidthY;
5667	5667	DETBufferSizeInTimeLuma = dml_floor(LinesInDETLuma, SwathHeightY) * LineTime / VRatio;
5668	5668	if (DETBufferSizeInTimeLuma - UrgentLatency <= 0) {
5669	5669	*NotEnoughUrgentLatencyHiding = 1;
@@ -5687,7 +5687,7 @@ static void CalculateUrgentBurstFactor( · 17 unmodified lines
5687	5687	}
5688	5688
5689	5689	if (BytePerPixelInDETC > 0) {
5690		− LinesInDETChroma = DETBufferSizeC / BytePerPixelInDETC / (SwathWidthY / 2);
	5690	+ LinesInDETChroma = (double)DETBufferSizeC / BytePerPixelInDETC / (SwathWidthY / 2);
5691	5691	DETBufferSizeInTimeChroma = dml_floor(LinesInDETChroma, SwathHeightC) * LineTime
5692	5692	/ (VRatio / 2);
5693	5693	if (DETBufferSizeInTimeChroma - UrgentLatency <= 0) {

drivers/gpu/drm/amd/display/dc/dml/dcn30/display_mode_vba_30.c+24−24

@@ -299,7 +299,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 298 unmodified lines
299	299	unsigned int MaxLineBufferLines,
300	300	unsigned int LineBufferSize,
301	301	unsigned int DPPOutputBufferPixels,
302		− double DETBufferSizeInKByte,
	302	+ unsigned int DETBufferSizeInKByte,
303	303	unsigned int WritebackInterfaceBufferSize,
304	304	double DCFCLK,
305	305	double ReturnBW,
@@ -318,8 +318,8 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 12 unmodified lines
318	318	unsigned int DPPPerPlane[],
319	319	bool DCCEnable[],
320	320	double DPPCLK[],
321		− double DETBufferSizeY[],
322		− double DETBufferSizeC[],
	321	+ unsigned int DETBufferSizeY[],
	322	+ unsigned int DETBufferSizeC[],
323	323	unsigned int SwathHeightY[],
324	324	unsigned int SwathHeightC[],
325	325	unsigned int LBBitPerPixel[],
@@ -570,7 +570,7 @@ static void CalculateStutterEfficiency( · 244 unmodified lines
570	570	double SRExitTime,
571	571	bool SynchronizedVBlank,
572	572	int DPPPerPlane[],
573		− double DETBufferSizeY[],
	573	+ unsigned int DETBufferSizeY[],
574	574	int BytePerPixelY[],
575	575	double BytePerPixelDETY[],
576	576	double SwathWidthY[],
@@ -603,7 +603,7 @@ static void CalculateStutterEfficiency( · 26 unmodified lines
603	603	static void CalculateSwathAndDETConfiguration(
604	604	bool ForceSingleDPP,
605	605	int NumberOfActivePlanes,
606		− long DETBufferSizeInKByte,
	606	+ unsigned int DETBufferSizeInKByte,
607	607	double MaximumSwathWidthLuma[],
608	608	double MaximumSwathWidthChroma[],
609	609	enum scan_direction_class SourceScan[],
@@ -635,8 +635,8 @@ static void CalculateSwathAndDETConfiguration( · 25 unmodified lines
635	635	double SwathWidthChroma[],
636	636	int SwathHeightY[],
637	637	int SwathHeightC[],
638		− double DETBufferSizeY[],
639		− double DETBufferSizeC[],
	638	+ unsigned int DETBufferSizeY[],
	639	+ unsigned int DETBufferSizeC[],
640	640	bool ViewportSizeSupportPerPlane[],
641	641	bool *ViewportSizeSupport);
642	642	static void CalculateSwathWidth(
@@ -2613,7 +2613,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 1970 unmodified lines
2613	2613	CalculateUrgentBurstFactor(
2614	2614	v->swath_width_luma_ub[k],
2615	2615	v->swath_width_chroma_ub[k],
2616		− v->DETBufferSizeInKByte,
	2616	+ v->DETBufferSizeInKByte[0],
2617	2617	v->SwathHeightY[k],
2618	2618	v->SwathHeightC[k],
2619	2619	v->HTotal[k] / v->PixelClock[k],
@@ -2635,7 +2635,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 15 unmodified lines
2635	2635	CalculateUrgentBurstFactor(
2636	2636	v->swath_width_luma_ub[k],
2637	2637	v->swath_width_chroma_ub[k],
2638		− v->DETBufferSizeInKByte,
	2638	+ v->DETBufferSizeInKByte[0],
2639	2639	v->SwathHeightY[k],
2640	2640	v->SwathHeightC[k],
2641	2641	v->HTotal[k] / v->PixelClock[k],
@@ -2808,7 +2808,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 166 unmodified lines
2808	2808	v->MaxLineBufferLines,
2809	2809	v->LineBufferSize,
2810	2810	v->DPPOutputBufferPixels,
2811		− v->DETBufferSizeInKByte,
	2811	+ v->DETBufferSizeInKByte[0],
2812	2812	v->WritebackInterfaceBufferSize,
2813	2813	v->DCFCLK,
2814	2814	v->ReturnBW,
@@ -3027,7 +3027,7 @@ static void DISPCLKDPPCLKDCFCLKDeepSleepPrefetchParametersWatermarksAndPerforman · 212 unmodified lines
3027	3027	v->SurfaceWidthC[k],
3028	3028	v->SurfaceHeightY[k],
3029	3029	v->SurfaceHeightC[k],
3030		− v->DETBufferSizeInKByte * 1024,
	3030	+ v->DETBufferSizeInKByte[0] * 1024,
3031	3031	v->BlockHeight256BytesY[k],
3032	3032	v->BlockHeight256BytesC[k],
3033	3033	v->SurfaceTiling[k],
@@ -3177,7 +3177,7 @@ static void DisplayPipeConfiguration(struct display_mode_lib *mode_lib) · 143 unmodified lines
3177	3177	CalculateSwathAndDETConfiguration(
3178	3178	false,
3179	3179	mode_lib->vba.NumberOfActivePlanes,
3180		− mode_lib->vba.DETBufferSizeInKByte,
	3180	+ mode_lib->vba.DETBufferSizeInKByte[0],
3181	3181	dummy1,
3182	3182	dummy2,
3183	3183	mode_lib->vba.SourceScan,
@@ -3911,7 +3911,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 727 unmodified lines
3911	3911	CalculateSwathAndDETConfiguration(
3912	3912	true,
3913	3913	v->NumberOfActivePlanes,
3914		− v->DETBufferSizeInKByte,
	3914	+ v->DETBufferSizeInKByte[0],
3915	3915	v->MaximumSwathWidthLuma,
3916	3916	v->MaximumSwathWidthChroma,
3917	3917	v->SourceScan,
@@ -4399,7 +4399,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 481 unmodified lines
4399	4399	CalculateSwathAndDETConfiguration(
4400	4400	false,
4401	4401	v->NumberOfActivePlanes,
4402		− v->DETBufferSizeInKByte,
	4402	+ v->DETBufferSizeInKByte[0],
4403	4403	v->MaximumSwathWidthLuma,
4404	4404	v->MaximumSwathWidthChroma,
4405	4405	v->SourceScan,
@@ -4622,7 +4622,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 216 unmodified lines
4622	4622	CalculateUrgentBurstFactor(
4623	4623	v->swath_width_luma_ub_this_state[k],
4624	4624	v->swath_width_chroma_ub_this_state[k],
4625		− v->DETBufferSizeInKByte,
	4625	+ v->DETBufferSizeInKByte[0],
4626	4626	v->SwathHeightYThisState[k],
4627	4627	v->SwathHeightCThisState[k],
4628	4628	v->HTotal[k] / v->PixelClock[k],
@@ -5025,7 +5025,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 396 unmodified lines
5025	5025	CalculateUrgentBurstFactor(
5026	5026	v->swath_width_luma_ub_this_state[k],
5027	5027	v->swath_width_chroma_ub_this_state[k],
5028		− v->DETBufferSizeInKByte,
	5028	+ v->DETBufferSizeInKByte[0],
5029	5029	v->SwathHeightYThisState[k],
5030	5030	v->SwathHeightCThisState[k],
5031	5031	v->HTotal[k] / v->PixelClock[k],
@@ -5197,7 +5197,7 @@ void dml30_ModeSupportAndSystemConfigurationFull(struct display_mode_lib *mode_l · 165 unmodified lines
5197	5197	v->MaxLineBufferLines,
5198	5198	v->LineBufferSize,
5199	5199	v->DPPOutputBufferPixels,
5200		− v->DETBufferSizeInKByte,
	5200	+ v->DETBufferSizeInKByte[0],
5201	5201	v->WritebackInterfaceBufferSize,
5202	5202	v->DCFCLKState[i][j],
5203	5203	v->ReturnBWPerState[i][j],
@@ -5369,7 +5369,7 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 165 unmodified lines
5369	5369	unsigned int MaxLineBufferLines,
5370	5370	unsigned int LineBufferSize,
5371	5371	unsigned int DPPOutputBufferPixels,
5372		− double DETBufferSizeInKByte,
	5372	+ unsigned int DETBufferSizeInKByte,
5373	5373	unsigned int WritebackInterfaceBufferSize,
5374	5374	double DCFCLK,
5375	5375	double ReturnBW,
@@ -5388,8 +5388,8 @@ static void CalculateWatermarksAndDRAMSpeedChangeSupport( · 12 unmodified lines
5388	5388	unsigned int DPPPerPlane[],
5389	5389	bool DCCEnable[],
5390	5390	double DPPCLK[],
5391		− double DETBufferSizeY[],
5392		− double DETBufferSizeC[],
	5391	+ unsigned int DETBufferSizeY[],
	5392	+ unsigned int DETBufferSizeC[],
5393	5393	unsigned int SwathHeightY[],
5394	5394	unsigned int SwathHeightC[],
5395	5395	unsigned int LBBitPerPixel[],
@@ -6126,7 +6126,7 @@ static void CalculateStutterEfficiency( · 730 unmodified lines
6126	6126	double SRExitTime,
6127	6127	bool SynchronizedVBlank,
6128	6128	int DPPPerPlane[],
6129		− double DETBufferSizeY[],
	6129	+ unsigned int DETBufferSizeY[],
6130	6130	int BytePerPixelY[],
6131	6131	double BytePerPixelDETY[],
6132	6132	double SwathWidthY[],
@@ -6273,7 +6273,7 @@ static void CalculateStutterEfficiency( · 140 unmodified lines
6273	6273	static void CalculateSwathAndDETConfiguration(
6274	6274	bool ForceSingleDPP,
6275	6275	int NumberOfActivePlanes,
6276		− long DETBufferSizeInKByte,
	6276	+ unsigned int DETBufferSizeInKByte,
6277	6277	double MaximumSwathWidthLuma[],
6278	6278	double MaximumSwathWidthChroma[],
6279	6279	enum scan_direction_class SourceScan[],
@@ -6305,8 +6305,8 @@ static void CalculateSwathAndDETConfiguration( · 25 unmodified lines
6305	6305	double SwathWidthChroma[],
6306	6306	int SwathHeightY[],
6307	6307	int SwathHeightC[],
6308		− double DETBufferSizeY[],
6309		− double DETBufferSizeC[],
	6308	+ unsigned int DETBufferSizeY[],
	6309	+ unsigned int DETBufferSizeC[],
6310	6310	bool ViewportSizeSupportPerPlane[],
6311	6311	bool *ViewportSizeSupport)
6312	6312	{

drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.c+1−1

@@ -299,7 +299,7 @@ static void fetch_ip_params(struct display_mode_lib *mode_lib) · 298 unmodified lines
299	299	mode_lib->vba.MaxDCHUBToPSCLThroughput = ip->max_dchub_pscl_bw_pix_per_clk;
300	300	mode_lib->vba.MaxPSCLToLBThroughput = ip->max_pscl_lb_bw_pix_per_clk;
301	301	mode_lib->vba.ROBBufferSizeInKByte = ip->rob_buffer_size_kbytes;
302		− mode_lib->vba.DETBufferSizeInKByte = ip->det_buffer_size_kbytes;
	302	+ mode_lib->vba.DETBufferSizeInKByte[0] = ip->det_buffer_size_kbytes;
303	303
304	304	mode_lib->vba.PixelChunkSizeInKByte = ip->pixel_chunk_size_kbytes;
305	305	mode_lib->vba.MetaChunkSize = ip->meta_chunk_size_kbytes;

drivers/gpu/drm/amd/display/dc/dml/display_mode_vba.h+7−7

@@ -233,7 +233,7 @@ struct vba_vars_st { · 232 unmodified lines
233	233	// IP Parameters
234	234	//
235	235	unsigned int ROBBufferSizeInKByte;
236		− double DETBufferSizeInKByte;
	236	+ unsigned int DETBufferSizeInKByte[DC__NUM_DPP__MAX];
237	237	double DETBufferSizeInTime;
238	238	unsigned int DPPOutputBufferPixels;
239	239	unsigned int OPPOutputBufferLines;
@@ -351,8 +351,8 @@ struct vba_vars_st { · 111 unmodified lines
351	351
352	352	// Intermediates/Informational
353	353	bool ImmediateFlipSupport;
354		− double DETBufferSizeY[DC__NUM_DPP__MAX];
355		− double DETBufferSizeC[DC__NUM_DPP__MAX];
	354	+ unsigned int DETBufferSizeY[DC__NUM_DPP__MAX];
	355	+ unsigned int DETBufferSizeC[DC__NUM_DPP__MAX];
356	356	unsigned int SwathHeightY[DC__NUM_DPP__MAX];
357	357	unsigned int SwathHeightC[DC__NUM_DPP__MAX];
358	358	unsigned int LBBitPerPixel[DC__NUM_DPP__MAX];
@@ -631,8 +631,8 @@ struct vba_vars_st { · 272 unmodified lines
631	631	enum odm_combine_mode odm_combine_dummy[DC__NUM_DPP__MAX];
632	632	double dummy1[DC__NUM_DPP__MAX];
633	633	double dummy2[DC__NUM_DPP__MAX];
634		− double dummy3[DC__NUM_DPP__MAX];
635		− double dummy4[DC__NUM_DPP__MAX];
	634	+ unsigned int dummy3[DC__NUM_DPP__MAX];
	635	+ unsigned int dummy4[DC__NUM_DPP__MAX];
636	636	double dummy5;
637	637	double dummy6;
638	638	double dummy7[DC__NUM_DPP__MAX];
@@ -872,8 +872,8 @@ struct vba_vars_st { · 233 unmodified lines
872	872	int PercentMarginOverMinimumRequiredDCFCLK;
873	873	bool DynamicMetadataSupported[DC__VOLTAGE_STATES][2];
874	874	enum immediate_flip_requirement ImmediateFlipRequirement;
875		− double DETBufferSizeYThisState[DC__NUM_DPP__MAX];
876		− double DETBufferSizeCThisState[DC__NUM_DPP__MAX];
	875	+ unsigned int DETBufferSizeYThisState[DC__NUM_DPP__MAX];
	876	+ unsigned int DETBufferSizeCThisState[DC__NUM_DPP__MAX];
877	877	bool NoUrgentLatencyHiding[DC__NUM_DPP__MAX];
878	878	bool NoUrgentLatencyHidingPre[DC__NUM_DPP__MAX];
879	879	int swath_width_luma_ub_this_state[DC__NUM_DPP__MAX];

drivers/gpu/drm/amd/display/dc/dsc/dc_dsc.c+103−25

@@ -37,6 +37,8 @@ static uint32_t dsc_policy_max_target_bpp_limit = 16; · 36 unmodified lines
37	37	/* default DSC policy enables DSC only when needed */
38	38	static bool dsc_policy_enable_dsc_when_not_needed;
39	39
	40	+static bool dsc_policy_disable_dsc_stream_overhead;
	41	+
40	42	static bool dsc_buff_block_size_from_dpcd(int dpcd_buff_block_size, int *buff_block_size)
41	43	{
42	44
@@ -250,6 +252,7 @@ static bool intersect_dsc_caps( · 207 unmodified lines
250	252	if (pixel_encoding == PIXEL_ENCODING_YCBCR422 \|\| pixel_encoding == PIXEL_ENCODING_YCBCR420)
251	253	dsc_common_caps->bpp_increment_div = min(dsc_common_caps->bpp_increment_div, (uint32_t)8);
252	254
	255	+ dsc_common_caps->is_dp = dsc_sink_caps->is_dp;
253	256	return true;
254	257	}
255	258
@@ -258,12 +261,63 @@ static inline uint32_t dsc_div_by_10_round_up(uint32_t value) · 2 unmodified lines
258	261	return (value + 9) / 10;
259	262	}
260	263
	264	+static struct fixed31_32 compute_dsc_max_bandwidth_overhead(
	265	+ const struct dc_crtc_timing *timing,
	266	+ const int num_slices_h,
	267	+ const bool is_dp)
	268	+{
	269	+ struct fixed31_32 max_dsc_overhead;
	270	+ struct fixed31_32 refresh_rate;
	271	+
	272	+ if (dsc_policy_disable_dsc_stream_overhead \|\| !is_dp)
	273	+ return dc_fixpt_from_int(0);
	274	+
	275	+ /* use target bpp that can take entire target bandwidth */
	276	+ refresh_rate = dc_fixpt_from_int(timing->pix_clk_100hz);
	277	+ refresh_rate = dc_fixpt_div_int(refresh_rate, timing->h_total);
	278	+ refresh_rate = dc_fixpt_div_int(refresh_rate, timing->v_total);
	279	+ refresh_rate = dc_fixpt_mul_int(refresh_rate, 100);
	280	+
	281	+ max_dsc_overhead = dc_fixpt_from_int(num_slices_h);
	282	+ max_dsc_overhead = dc_fixpt_mul_int(max_dsc_overhead, timing->v_total);
	283	+ max_dsc_overhead = dc_fixpt_mul_int(max_dsc_overhead, 256);
	284	+ max_dsc_overhead = dc_fixpt_div_int(max_dsc_overhead, 1000);
	285	+ max_dsc_overhead = dc_fixpt_mul(max_dsc_overhead, refresh_rate);
	286	+
	287	+ return max_dsc_overhead;
	288	+}
	289	+
	290	+static uint32_t compute_bpp_x16_from_target_bandwidth(
	291	+ const uint32_t bandwidth_in_kbps,
	292	+ const struct dc_crtc_timing *timing,
	293	+ const uint32_t num_slices_h,
	294	+ const uint32_t bpp_increment_div,
	295	+ const bool is_dp)
	296	+{
	297	+ struct fixed31_32 overhead_in_kbps;
	298	+ struct fixed31_32 effective_bandwidth_in_kbps;
	299	+ struct fixed31_32 bpp_x16;
	300	+
	301	+ overhead_in_kbps = compute_dsc_max_bandwidth_overhead(
	302	+ timing, num_slices_h, is_dp);
	303	+ effective_bandwidth_in_kbps = dc_fixpt_from_int(bandwidth_in_kbps);
	304	+ effective_bandwidth_in_kbps = dc_fixpt_sub(effective_bandwidth_in_kbps,
	305	+ overhead_in_kbps);
	306	+ bpp_x16 = dc_fixpt_mul_int(effective_bandwidth_in_kbps, 10);
	307	+ bpp_x16 = dc_fixpt_div_int(bpp_x16, timing->pix_clk_100hz);
	308	+ bpp_x16 = dc_fixpt_from_int(dc_fixpt_floor(dc_fixpt_mul_int(bpp_x16, bpp_increment_div)));
	309	+ bpp_x16 = dc_fixpt_div_int(bpp_x16, bpp_increment_div);
	310	+ bpp_x16 = dc_fixpt_mul_int(bpp_x16, 16);
	311	+ return dc_fixpt_floor(bpp_x16);
	312	+}
	313	+
261	314	/* Get DSC bandwidth range based on [min_bpp, max_bpp] target bitrate range, and timing's pixel clock
262	315	* and uncompressed bandwidth.
263	316	*/
264	317	static void get_dsc_bandwidth_range(
265	318	const uint32_t min_bpp_x16,
266	319	const uint32_t max_bpp_x16,
	320	+ const uint32_t num_slices_h,
267	321	const struct dsc_enc_caps *dsc_caps,
268	322	const struct dc_crtc_timing *timing,
269	323	struct dc_dsc_bw_range *range)
@@ -272,16 +326,21 @@ static void get_dsc_bandwidth_range( · 2 unmodified lines
272	326	range->stream_kbps = dc_bandwidth_in_kbps_from_timing(timing);
273	327
274	328	/* max dsc target bpp */
275		− range->max_kbps = dc_dsc_stream_bandwidth_in_kbps(timing->pix_clk_100hz, max_bpp_x16);
	329	+ range->max_kbps = dc_dsc_stream_bandwidth_in_kbps(timing,
	330	+ max_bpp_x16, num_slices_h, dsc_caps->is_dp);
276	331	range->max_target_bpp_x16 = max_bpp_x16;
277	332	if (range->max_kbps > range->stream_kbps) {
278	333	/* max dsc target bpp is capped to native bandwidth */
279	334	range->max_kbps = range->stream_kbps;
280		− range->max_target_bpp_x16 = calc_dsc_bpp_x16(range->stream_kbps, timing->pix_clk_100hz, dsc_caps->bpp_increment_div);
	335	+ range->max_target_bpp_x16 = compute_bpp_x16_from_target_bandwidth(
	336	+ range->max_kbps, timing, num_slices_h,
	337	+ dsc_caps->bpp_increment_div,
	338	+ dsc_caps->is_dp);
281	339	}
282	340
283	341	/* min dsc target bpp */
284		− range->min_kbps = dc_dsc_stream_bandwidth_in_kbps(timing->pix_clk_100hz, min_bpp_x16);
	342	+ range->min_kbps = dc_dsc_stream_bandwidth_in_kbps(timing,
	343	+ min_bpp_x16, num_slices_h, dsc_caps->is_dp);
285	344	range->min_target_bpp_x16 = min_bpp_x16;
286	345	if (range->min_kbps > range->max_kbps) {
287	346	/* min dsc target bpp is capped to max dsc bandwidth*/
@@ -290,7 +349,6 @@ static void get_dsc_bandwidth_range( · 2 unmodified lines
290	349	}
291	350	}
292	351
293		−
294	352	/* Decides if DSC should be used and calculates target bpp if it should, applying DSC policy.
295	353	*
296	354	* Returns:
@@ -303,6 +361,7 @@ static bool decide_dsc_target_bpp_x16( · 6 unmodified lines
303	361	const struct dsc_enc_caps *dsc_common_caps,
304	362	const int target_bandwidth_kbps,
305	363	const struct dc_crtc_timing *timing,
	364	+ const int num_slices_h,
306	365	int *target_bpp_x16)
307	366	{
308	367	bool should_use_dsc = false;
@@ -311,7 +370,7 @@ static bool decide_dsc_target_bpp_x16( · 2 unmodified lines
311	370	memset(&range, 0, sizeof(range));
312	371
313	372	get_dsc_bandwidth_range(policy->min_target_bpp * 16, policy->max_target_bpp * 16,
314		− dsc_common_caps, timing, &range);
	373	+ num_slices_h, dsc_common_caps, timing, &range);
315	374	if (!policy->enable_dsc_when_not_needed && target_bandwidth_kbps >= range.stream_kbps) {
316	375	/* enough bandwidth without dsc */
317	376	*target_bpp_x16 = 0;
@@ -327,7 +386,10 @@ static bool decide_dsc_target_bpp_x16( · 9 unmodified lines
327	386	should_use_dsc = true;
328	387	} else if (target_bandwidth_kbps >= range.min_kbps) {
329	388	/* use target bpp that can take entire target bandwidth */
330		− *target_bpp_x16 = calc_dsc_bpp_x16(target_bandwidth_kbps, timing->pix_clk_100hz, dsc_common_caps->bpp_increment_div);
	389	+ *target_bpp_x16 = compute_bpp_x16_from_target_bandwidth(
	390	+ target_bandwidth_kbps, timing, num_slices_h,
	391	+ dsc_common_caps->bpp_increment_div,
	392	+ dsc_common_caps->is_dp);
331	393	should_use_dsc = true;
332	394	} else {
333	395	/* not enough bandwidth to fulfill minimum requirement */
@@ -531,18 +593,6 @@ static bool setup_dsc_config( · 197 unmodified lines
531	593	if (!is_dsc_possible)
532	594	goto done;
533	595
534		− if (target_bandwidth_kbps > 0) {
535		− is_dsc_possible = decide_dsc_target_bpp_x16(
536		− &policy,
537		− &dsc_common_caps,
538		− target_bandwidth_kbps,
539		− timing,
540		− &target_bpp);
541		− dsc_cfg->bits_per_pixel = target_bpp;
542		− }
543		− if (!is_dsc_possible)
544		− goto done;
545		−
546	596	sink_per_slice_throughput_mps = 0;
547	597
548	598	// Validate available DSC settings against the mode timing
@@ -690,12 +740,26 @@ static bool setup_dsc_config( · 141 unmodified lines
690	740
691	741	dsc_cfg->num_slices_v = pic_height/slice_height;
692	742
	743	+ if (target_bandwidth_kbps > 0) {
	744	+ is_dsc_possible = decide_dsc_target_bpp_x16(
	745	+ &policy,
	746	+ &dsc_common_caps,
	747	+ target_bandwidth_kbps,
	748	+ timing,
	749	+ num_slices_h,
	750	+ &target_bpp);
	751	+ dsc_cfg->bits_per_pixel = target_bpp;
	752	+ }
	753	+ if (!is_dsc_possible)
	754	+ goto done;
	755	+
693	756	// Final decission: can we do DSC or not?
694	757	if (is_dsc_possible) {
695	758	// Fill out the rest of DSC settings
696	759	dsc_cfg->block_pred_enable = dsc_common_caps.is_block_pred_supported;
697	760	dsc_cfg->linebuf_depth = dsc_common_caps.lb_bit_depth;
698	761	dsc_cfg->version_minor = (dsc_common_caps.dsc_version & 0xf0) >> 4;
	762	+ dsc_cfg->is_dp = dsc_sink_caps->is_dp;
699	763	}
700	764
701	765	done:
@@ -806,6 +870,7 @@ bool dc_dsc_parse_dsc_dpcd(const struct dc dc, const uint8_t dpcd_dsc_basic_da · 104 unmodified lines
806	870	dsc_sink_caps->branch_max_line_width = dpcd_dsc_branch_decoder_caps[DP_DSC_BRANCH_MAX_LINE_WIDTH - DP_DSC_BRANCH_OVERALL_THROUGHPUT_0] * 320;
807	871	ASSERT(dsc_sink_caps->branch_max_line_width == 0 \|\| dsc_sink_caps->branch_max_line_width >= 5120);
808	872
	873	+ dsc_sink_caps->is_dp = true;
809	874	return true;
810	875	}
811	876
@@ -838,7 +903,8 @@ bool dc_dsc_compute_bandwidth_range( · 26 unmodified lines
838	903	dsc_min_slice_height_override, max_bpp_x16, &config);
839	904
840	905	if (is_dsc_possible)
841		− get_dsc_bandwidth_range(min_bpp_x16, max_bpp_x16, &dsc_common_caps, timing, range);
	906	+ get_dsc_bandwidth_range(min_bpp_x16, max_bpp_x16,
	907	+ config.num_slices_h, &dsc_common_caps, timing, range);
842	908
843	909	return is_dsc_possible;
844	910	}
@@ -864,13 +930,20 @@ bool dc_dsc_compute_config( · 19 unmodified lines
864	930	return is_dsc_possible;
865	931	}
866	932
867		−uint32_t dc_dsc_stream_bandwidth_in_kbps(uint32_t pix_clk_100hz, uint32_t bpp_x16)
	933	+uint32_t dc_dsc_stream_bandwidth_in_kbps(const struct dc_crtc_timing *timing,
	934	+ uint32_t bpp_x16, uint32_t num_slices_h, bool is_dp)
868	935	{
869		− struct fixed31_32 link_bw_kbps;
870		− link_bw_kbps = dc_fixpt_from_int(pix_clk_100hz);
871		− link_bw_kbps = dc_fixpt_div_int(link_bw_kbps, 160);
872		− link_bw_kbps = dc_fixpt_mul_int(link_bw_kbps, bpp_x16);
873		− return dc_fixpt_ceil(link_bw_kbps);
	936	+ struct fixed31_32 overhead_in_kbps;
	937	+ struct fixed31_32 bpp;
	938	+ struct fixed31_32 actual_bandwidth_in_kbps;
	939	+
	940	+ overhead_in_kbps = compute_dsc_max_bandwidth_overhead(
	941	+ timing, num_slices_h, is_dp);
	942	+ bpp = dc_fixpt_from_fraction(bpp_x16, 16);
	943	+ actual_bandwidth_in_kbps = dc_fixpt_from_fraction(timing->pix_clk_100hz, 10);
	944	+ actual_bandwidth_in_kbps = dc_fixpt_mul(actual_bandwidth_in_kbps, bpp);
	945	+ actual_bandwidth_in_kbps = dc_fixpt_add(actual_bandwidth_in_kbps, overhead_in_kbps);
	946	+ return dc_fixpt_ceil(actual_bandwidth_in_kbps);
874	947	}
875	948
876	949	void dc_dsc_get_policy_for_timing(const struct dc_crtc_timing timing, uint32_t max_target_bpp_limit_override_x16, struct dc_dsc_policy policy)
@@ -954,3 +1027,8 @@ void dc_dsc_policy_set_enable_dsc_when_not_needed(bool enable) · 77 unmodified lines
954	1027	{
955	1028	dsc_policy_enable_dsc_when_not_needed = enable;
956	1029	}
	1030	+
	1031	+void dc_dsc_policy_set_disable_dsc_stream_overhead(bool disable)
	1032	+{
	1033	+ dsc_policy_disable_dsc_stream_overhead = disable;
	1034	+}

drivers/gpu/drm/amd/display/dc/dsc/rc_calc.c+0−43

@@ -284,26 +284,6 @@ static u32 _do_bytes_per_pixel_calc(int slice_width, u16 drm_bpp, · 283 unmodified lines
284	284	return bytes_per_pixel;
285	285	}
286	286
287		−static u32 _do_calc_dsc_bpp_x16(u32 stream_bandwidth_kbps, u32 pix_clk_100hz,
288		− u32 bpp_increment_div)
289		−{
290		− u32 dsc_target_bpp_x16;
291		− float f_dsc_target_bpp;
292		− float f_stream_bandwidth_100bps;
293		− // bpp_increment_div is actually precision
294		− u32 precision = bpp_increment_div;
295		−
296		− f_stream_bandwidth_100bps = stream_bandwidth_kbps * 10.0f;
297		− f_dsc_target_bpp = f_stream_bandwidth_100bps / pix_clk_100hz;
298		−
299		− // Round down to the nearest precision stop to bring it into DSC spec
300		− // range
301		− dsc_target_bpp_x16 = (u32)(f_dsc_target_bpp * precision);
302		− dsc_target_bpp_x16 = (dsc_target_bpp_x16 * 16) / precision;
303		−
304		− return dsc_target_bpp_x16;
305		−}
306		−
307	287	/**
308	288	* calc_rc_params - reads the user's cmdline mode
309	289	* @rc: DC internal DSC parameters
@@ -367,26 +347,3 @@ u32 calc_dsc_bytes_per_pixel(const struct drm_dsc_config *pps) · 57 unmodified lines
367	347	DC_FP_END();
368	348	return ret;
369	349	}
370		−
371		−/**
372		− * calc_dsc_bpp_x16 - retrieve the dsc bits per pixel
373		− * @stream_bandwidth_kbps:
374		− * @pix_clk_100hz:
375		− * @bpp_increment_div:
376		− *
377		− * Calculate the total of bits per pixel for DSC configuration.
378		− *
379		− * @note This calculation requires float point operation, most of it executes
380		− * under kernel_fpu_{begin,end}.
381		− */
382		−u32 calc_dsc_bpp_x16(u32 stream_bandwidth_kbps, u32 pix_clk_100hz,
383		− u32 bpp_increment_div)
384		−{
385		− u32 dsc_bpp;
386		−
387		− DC_FP_START();
388		− dsc_bpp = _do_calc_dsc_bpp_x16(stream_bandwidth_kbps, pix_clk_100hz,
389		− bpp_increment_div);
390		− DC_FP_END();
391		− return dsc_bpp;
392		−}

drivers/gpu/drm/amd/display/dc/dsc/rc_calc.h+0−2

@@ -79,8 +79,6 @@ typedef struct qp_entry qp_table[]; · 78 unmodified lines
79	79
80	80	void calc_rc_params(struct rc_params rc, const struct drm_dsc_config pps);
81	81	u32 calc_dsc_bytes_per_pixel(const struct drm_dsc_config *pps);
82		−u32 calc_dsc_bpp_x16(u32 stream_bandwidth_kbps, u32 pix_clk_100hz,
83		− u32 bpp_increment_div);
84	82
85	83	#endif
86	84

drivers/gpu/drm/amd/display/dc/inc/dc_link_dp.h+11−1

@@ -65,7 +65,8 @@ bool perform_link_training_with_retries( · 64 unmodified lines
65	65	bool skip_video_pattern,
66	66	int attempts,
67	67	struct pipe_ctx *pipe_ctx,
68		− enum signal_type signal);
	68	+ enum signal_type signal,
	69	+ bool do_fallback);
69	70
70	71	bool is_mst_supported(struct dc_link *link);
71	72
@@ -75,6 +76,8 @@ void detect_edp_sink_caps(struct dc_link *link); · 3 unmodified lines
75	76
76	77	bool is_dp_active_dongle(const struct dc_link *link);
77	78
	79	+bool is_dp_branch_device(const struct dc_link *link);
	80	+
78	81	bool is_edp_ilr_optimization_required(struct dc_link link, struct dc_crtc_timing crtc_timing);
79	82
80	83	void dp_enable_mst_on_sink(struct dc_link *link, bool enable);
@@ -94,5 +97,12 @@ void dp_set_dsc_on_stream(struct pipe_ctx *pipe_ctx, bool enable); · 13 unmodified lines
94	97	bool dp_update_dsc_config(struct pipe_ctx *pipe_ctx);
95	98	bool dp_set_dsc_on_rx(struct pipe_ctx *pipe_ctx, bool enable);
96	99
	100	+/* Convert PHY repeater count read from DPCD uint8_t. */
	101	+uint8_t dp_convert_to_count(uint8_t lttpr_repeater_count);
	102	+
	103	+/* Check DPCD training status registers to detect link loss. */
	104	+enum link_training_result dp_check_link_loss_status(
	105	+ struct dc_link *link,
	106	+ const struct link_training_settings *link_training_setting);
97	107
98	108	#endif /* __DC_LINK_DP_H__ */

drivers/gpu/drm/amd/display/dc/inc/hw/dsc.h+1−0

@@ -88,6 +88,7 @@ struct dsc_enc_caps { · 87 unmodified lines
88	88	int32_t max_total_throughput_mps; /* Maximum total throughput with all the slices combined */
89	89	int32_t max_slice_width;
90	90	uint32_t bpp_increment_div; /* bpp increment divisor, e.g. if 16, it's 1/16th of a bit */
	91	+ bool is_dp;
91	92	};
92	93
93	94	struct dsc_funcs {

drivers/gpu/drm/amd/display/dc/inc/link_enc_cfg.h+6−1

@@ -81,6 +81,11 @@ struct dc_link *link_enc_cfg_get_link_using_link_enc( · 80 unmodified lines
81	81	/* Return DIG link encoder used by link. NULL if unused. */
82	82	struct link_encoder *link_enc_cfg_get_link_enc_used_by_link(
83	83	struct dc_state *state,
84		− struct dc_link *link);
	84	+ const struct dc_link *link);
	85	+
	86	+/* Return next available DIG link encoder. NULL if none available. */
	87	+struct link_encoder *link_enc_cfg_get_next_avail_link_enc(
	88	+ const struct dc *dc,
	89	+ const struct dc_state *state);
85	90
86	91	#endif /* DC_INC_LINK_ENC_CFG_H_ */

drivers/gpu/drm/amd/display/dc/irq/dcn21/irq_service_dcn21.c+9−9

@@ -58,8 +58,8 @@ enum dc_irq_source to_dal_irq_source_dcn21( · 57 unmodified lines
58	58	return DC_IRQ_SOURCE_VBLANK5;
59	59	case DCN_1_0__SRCID__DC_D6_OTG_VSTARTUP:
60	60	return DC_IRQ_SOURCE_VBLANK6;
61		− case DCN_1_0__SRCID__DMCUB_OUTBOX_HIGH_PRIORITY_READY_INT:
62		− return DC_IRQ_SOURCE_DMCUB_OUTBOX0;
	61	+ case DCN_1_0__SRCID__DMCUB_OUTBOX_LOW_PRIORITY_READY_INT:
	62	+ return DC_IRQ_SOURCE_DMCUB_OUTBOX;
63	63	case DCN_1_0__SRCID__OTG1_VERTICAL_INTERRUPT0_CONTROL:
64	64	return DC_IRQ_SOURCE_DC1_VLINE0;
65	65	case DCN_1_0__SRCID__OTG2_VERTICAL_INTERRUPT0_CONTROL:
@@ -187,7 +187,7 @@ static const struct irq_source_info_funcs vupdate_no_lock_irq_info_funcs = { · 121 unmodified lines
187	187	.ack = NULL
188	188	};
189	189
190		−static const struct irq_source_info_funcs dmub_trace_irq_info_funcs = {
	190	+static const struct irq_source_info_funcs dmub_outbox_irq_info_funcs = {
191	191	.set = NULL,
192	192	.ack = NULL
193	193	};
@@ -301,11 +301,11 @@ static const struct irq_source_info_funcs vline0_irq_info_funcs = { · 107 unmodified lines
301	301	.funcs = &vline0_irq_info_funcs\
302	302	}
303	303
304		−#define dmub_trace_int_entry()\
305		− [DC_IRQ_SOURCE_DMCUB_OUTBOX0] = {\
306		− IRQ_REG_ENTRY_DMUB(DMCUB_INTERRUPT_ENABLE, DMCUB_OUTBOX0_READY_INT_EN,\
307		− DMCUB_INTERRUPT_ACK, DMCUB_OUTBOX0_READY_INT_ACK),\
308		− .funcs = &dmub_trace_irq_info_funcs\
	304	+#define dmub_outbox_int_entry()\
	305	+ [DC_IRQ_SOURCE_DMCUB_OUTBOX] = {\
	306	+ IRQ_REG_ENTRY_DMUB(DMCUB_INTERRUPT_ENABLE, DMCUB_OUTBOX1_READY_INT_EN,\
	307	+ DMCUB_INTERRUPT_ACK, DMCUB_OUTBOX1_READY_INT_ACK),\
	308	+ .funcs = &dmub_outbox_irq_info_funcs\
309	309	}
310	310
311	311	#define dummy_irq_entry() \
@@ -426,7 +426,7 @@ irq_source_info_dcn21[DAL_IRQ_SOURCES_NUMBER] = { · 114 unmodified lines
426	426	vline0_int_entry(3),
427	427	vline0_int_entry(4),
428	428	vline0_int_entry(5),
429		− dmub_trace_int_entry(),
	429	+ dmub_outbox_int_entry(),
430	430	};
431	431
432	432	static const struct irq_service_funcs irq_service_funcs_dcn21 = {

drivers/gpu/drm/amd/display/dc/irq_types.h+1−1

@@ -150,7 +150,7 @@ enum dc_irq_source { · 149 unmodified lines
150	150	DC_IRQ_SOURCE_DC4_VLINE1,
151	151	DC_IRQ_SOURCE_DC5_VLINE1,
152	152	DC_IRQ_SOURCE_DC6_VLINE1,
153		− DC_IRQ_DMCUB_OUTBOX1,
	153	+ DC_IRQ_SOURCE_DMCUB_OUTBOX,
154	154	DC_IRQ_SOURCE_DMCUB_OUTBOX0,
155	155
156	156	DAL_IRQ_SOURCES_NUMBER

drivers/gpu/drm/amd/display/dmub/dmub_srv.h+19−18

@@ -216,6 +216,23 @@ struct dmub_srv_fb_info { · 215 unmodified lines
216	216	struct dmub_fb fb[DMUB_WINDOW_TOTAL];
217	217	};
218	218
	219	+/*
	220	+ * struct dmub_srv_hw_params - params for dmub hardware initialization
	221	+ * @fb: framebuffer info for each region
	222	+ * @fb_base: base of the framebuffer aperture
	223	+ * @fb_offset: offset of the framebuffer aperture
	224	+ * @psp_version: psp version to pass for DMCU init
	225	+ * @load_inst_const: true if DMUB should load inst const fw
	226	+ */
	227	+struct dmub_srv_hw_params {
	228	+ struct dmub_fb *fb[DMUB_WINDOW_TOTAL];
	229	+ uint64_t fb_base;
	230	+ uint64_t fb_offset;
	231	+ uint32_t psp_version;
	232	+ bool load_inst_const;
	233	+ bool skip_panel_power_sequence;
	234	+};
	235	+
219	236	/**
220	237	* struct dmub_srv_base_funcs - Driver specific base callbacks
221	238	*/
@@ -290,7 +307,8 @@ struct dmub_srv_hw_funcs { · 68 unmodified lines
290	307	bool (is_hw_init)(struct dmub_srv dmub);
291	308
292	309	bool (is_phy_init)(struct dmub_srv dmub);
293		− void (enable_dmub_boot_options)(struct dmub_srv dmub);
	310	+ void (enable_dmub_boot_options)(struct dmub_srv dmub,
	311	+ const struct dmub_srv_hw_params *params);
294	312
295	313	void (skip_dmub_panel_power_sequence)(struct dmub_srv dmub, bool skip);
296	314
@@ -325,23 +343,6 @@ struct dmub_srv_create_params { · 28 unmodified lines
325	343	bool is_virtual;
326	344	};
327	345
328		−/*
329		− * struct dmub_srv_hw_params - params for dmub hardware initialization
330		− * @fb: framebuffer info for each region
331		− * @fb_base: base of the framebuffer aperture
332		− * @fb_offset: offset of the framebuffer aperture
333		− * @psp_version: psp version to pass for DMCU init
334		− * @load_inst_const: true if DMUB should load inst const fw
335		− */
336		−struct dmub_srv_hw_params {
337		− struct dmub_fb *fb[DMUB_WINDOW_TOTAL];
338		− uint64_t fb_base;
339		− uint64_t fb_offset;
340		− uint32_t psp_version;
341		− bool load_inst_const;
342		− bool skip_panel_power_sequence;
343		−};
344		−
345	346	/**
346	347	* struct dmub_srv - software state for dmcub
347	348	* @asic: dmub asic identifier

drivers/gpu/drm/amd/display/dmub/inc/dmub_cmd.h+194−13

@@ -47,10 +47,10 @@ · 46 unmodified lines
47	47
48	48	/* Firmware versioning. */
49	49	#ifdef DMUB_EXPOSE_VERSION
50		−#define DMUB_FW_VERSION_GIT_HASH 0x23db9b126
	50	+#define DMUB_FW_VERSION_GIT_HASH 0x2cab49dfb
51	51	#define DMUB_FW_VERSION_MAJOR 0
52	52	#define DMUB_FW_VERSION_MINOR 0
53		−#define DMUB_FW_VERSION_REVISION 62
	53	+#define DMUB_FW_VERSION_REVISION 65
54	54	#define DMUB_FW_VERSION_TEST 0
55	55	#define DMUB_FW_VERSION_VBIOS 0
56	56	#define DMUB_FW_VERSION_HOTFIX 0
@@ -117,18 +117,20 @@ · 60 unmodified lines
117	117	/* Maximum number of planes on any ASIC. */
118	118	#define DMUB_MAX_PLANES 6
119	119
	120	+#define DMUB_MAX_SUBVP_STREAMS 2
	121	+
120	122	/* Trace buffer offset for entry */
121	123	#define TRACE_BUFFER_ENTRY_OFFSET 16
122	124
123	125	/**
124		− * ABM backlight control version legacy
	126	+ * ABM control version legacy
125	127	*/
126		−#define DMUB_CMD_ABM_SET_BACKLIGHT_VERSION_UNKNOWN 0x0
	128	+#define DMUB_CMD_ABM_CONTROL_VERSION_UNKNOWN 0x0
127	129
128	130	/**
129		− * ABM backlight control version with multi edp support
	131	+ * ABM control version with multi edp support
130	132	*/
131		−#define DMUB_CMD_ABM_SET_BACKLIGHT_VERSION_1 0x1
	133	+#define DMUB_CMD_ABM_CONTROL_VERSION_1 0x1
132	134
133	135	/**
134	136	* Physical framebuffer address location, 64-bit.
@@ -162,6 +164,13 @@ extern "C" { · 27 unmodified lines
162	164	#define dmub_udelay(microseconds) udelay(microseconds)
163	165	#endif
164	166
	167	+/**
	168	+ * Number of nanoseconds per DMUB tick.
	169	+ * DMCUB_TIMER_CURRENT increments in DMUB ticks, which are 10ns by default.
	170	+ * If DMCUB_TIMER_WINDOW is non-zero this will no longer be true.
	171	+ */
	172	+#define NS_PER_DMUB_TICK 10
	173	+
165	174	/**
166	175	* union dmub_addr - DMUB physical/virtual 64-bit address.
167	176	*/
@@ -328,7 +337,8 @@ union dmub_fw_boot_options { · 160 unmodified lines
328	337	uint32_t skip_phy_access : 1; /*< 1 if PHY access should be skipped /
329	338	uint32_t disable_clk_gate: 1; /*< 1 if clock gating should be disabled /
330	339	uint32_t skip_phy_init_panel_sequence: 1; /*< 1 to skip panel init seq /
331		− uint32_t reserved : 26; /*< reserved /
	340	+ uint32_t reserved_unreleased: 1; /*< reserved for an unreleased feature /
	341	+ uint32_t reserved : 25; /*< reserved /
332	342	} bits; /*< boot bits /
333	343	uint32_t all; /*< 32-bit access to bits /
334	344	};
@@ -452,6 +462,61 @@ enum dmub_gpint_command { · 117 unmodified lines
452	462	DMUB_GPINT__PSR_RESIDENCY = 9,
453	463	};
454	464
	465	+/**
	466	+ * INBOX0 generic command definition
	467	+ */
	468	+union dmub_inbox0_cmd_common {
	469	+ struct {
	470	+ uint32_t command_code: 8; /*< INBOX0 command code /
	471	+ uint32_t param: 24; /*< 24-bit parameter /
	472	+ } bits;
	473	+ uint32_t all;
	474	+};
	475	+
	476	+/**
	477	+ * INBOX0 hw_lock command definition
	478	+ */
	479	+union dmub_inbox0_cmd_lock_hw {
	480	+ struct {
	481	+ uint32_t command_code: 8;
	482	+
	483	+ /* NOTE: Must be have enough bits to match: enum hw_lock_client */
	484	+ uint32_t hw_lock_client: 1;
	485	+
	486	+ /* NOTE: Below fields must match with: struct dmub_hw_lock_inst_flags */
	487	+ uint32_t otg_inst: 3;
	488	+ uint32_t opp_inst: 3;
	489	+ uint32_t dig_inst: 3;
	490	+
	491	+ /* NOTE: Below fields must match with: union dmub_hw_lock_flags */
	492	+ uint32_t lock_pipe: 1;
	493	+ uint32_t lock_cursor: 1;
	494	+ uint32_t lock_dig: 1;
	495	+ uint32_t triple_buffer_lock: 1;
	496	+
	497	+ uint32_t lock: 1; /*< Lock /
	498	+ uint32_t should_release: 1; /*< Release /
	499	+ uint32_t reserved: 8; /*< Reserved for extending more clients, HW, etc. /
	500	+ } bits;
	501	+ uint32_t all;
	502	+};
	503	+
	504	+union dmub_inbox0_data_register {
	505	+ union dmub_inbox0_cmd_common inbox0_cmd_common;
	506	+ union dmub_inbox0_cmd_lock_hw inbox0_cmd_lock_hw;
	507	+};
	508	+
	509	+enum dmub_inbox0_command {
	510	+ /**
	511	+ * DESC: Invalid command, ignored.
	512	+ */
	513	+ DMUB_INBOX0_CMD__INVALID_COMMAND = 0,
	514	+ /**
	515	+ * DESC: Notification to acquire/release HW lock
	516	+ * ARGS:
	517	+ */
	518	+ DMUB_INBOX0_CMD__HW_LOCK = 1,
	519	+};
455	520	//==============================================================================
456	521	//</DMUB_GPINT>=================================================================
457	522	//==============================================================================
@@ -570,7 +635,8 @@ struct dmub_cmd_header { · 112 unmodified lines
570	635	unsigned int type : 8; /*< command type /
571	636	unsigned int sub_type : 8; /*< command sub type /
572	637	unsigned int ret_status : 1; /*< 1 if returned data, 0 otherwise /
573		− unsigned int reserved0 : 7; /*< reserved bits /
	638	+ unsigned int multi_cmd_pending : 1; /*< 1 if multiple commands chained together /
	639	+ unsigned int reserved0 : 6; /*< reserved bits /
574	640	unsigned int payload_bytes : 6; /* payload excluding header - up to 60 bytes */
575	641	unsigned int reserved1 : 2; /*< reserved bits /
576	642	};
@@ -1343,6 +1409,9 @@ struct dmub_rb_cmd_psr_force_static { · 766 unmodified lines
1343	1409
1344	1410	/**
1345	1411	* Set of HW components that can be locked.
	1412	+ *
	1413	+ * Note: If updating with more HW components, fields
	1414	+ * in dmub_inbox0_cmd_lock_hw must be updated to match.
1346	1415	*/
1347	1416	union dmub_hw_lock_flags {
1348	1417	/**
@@ -1375,6 +1444,9 @@ union dmub_hw_lock_flags { · 26 unmodified lines
1375	1444
1376	1445	/**
1377	1446	* Instances of HW to be locked.
	1447	+ *
	1448	+ * Note: If updating with more HW components, fields
	1449	+ * in dmub_inbox0_cmd_lock_hw must be updated to match.
1378	1450	*/
1379	1451	struct dmub_hw_lock_inst_flags {
1380	1452	/**
@@ -1398,16 +1470,16 @@ struct dmub_hw_lock_inst_flags { · 17 unmodified lines
1398	1470
1399	1471	/**
1400	1472	* Clients that can acquire the HW Lock Manager.
	1473	+ *
	1474	+ * Note: If updating with more clients, fields in
	1475	+ * dmub_inbox0_cmd_lock_hw must be updated to match.
1401	1476	*/
1402	1477	enum hw_lock_client {
1403	1478	/**
1404	1479	* Driver is the client of HW Lock Manager.
1405	1480	*/
1406	1481	HW_LOCK_CLIENT_DRIVER = 0,
1407		− /**
1408		− * FW is the client of HW Lock Manager.
1409		− */
1410		− HW_LOCK_CLIENT_FW,
	1482	+ HW_LOCK_CLIENT_SUBVP = 3,
1411	1483	/**
1412	1484	* Invalid client.
1413	1485	*/
@@ -1637,7 +1709,7 @@ struct dmub_cmd_abm_set_backlight_data { · 223 unmodified lines
1637	1709	uint32_t backlight_user_level;
1638	1710
1639	1711	/**
1640		− * Backlight data version.
	1712	+ * ABM control version.
1641	1713	*/
1642	1714	uint8_t version;
1643	1715
@@ -1677,6 +1749,23 @@ struct dmub_cmd_abm_set_level_data { · 33 unmodified lines
1677	1749	* Set current ABM operating/aggression level.
1678	1750	*/
1679	1751	uint32_t level;
	1752	+
	1753	+ /**
	1754	+ * ABM control version.
	1755	+ */
	1756	+ uint8_t version;
	1757	+
	1758	+ /**
	1759	+ * Panel Control HW instance mask.
	1760	+ * Bit 0 is Panel Control HW instance 0.
	1761	+ * Bit 1 is Panel Control HW instance 1.
	1762	+ */
	1763	+ uint8_t panel_mask;
	1764	+
	1765	+ /**
	1766	+ * Explicit padding to 4 byte boundary.
	1767	+ */
	1768	+ uint8_t pad[2];
1680	1769	};
1681	1770
1682	1771	/**
@@ -1702,6 +1791,23 @@ struct dmub_cmd_abm_set_ambient_level_data { · 19 unmodified lines
1702	1791	* Ambient light sensor reading from OS.
1703	1792	*/
1704	1793	uint32_t ambient_lux;
	1794	+
	1795	+ /**
	1796	+ * ABM control version.
	1797	+ */
	1798	+ uint8_t version;
	1799	+
	1800	+ /**
	1801	+ * Panel Control HW instance mask.
	1802	+ * Bit 0 is Panel Control HW instance 0.
	1803	+ * Bit 1 is Panel Control HW instance 1.
	1804	+ */
	1805	+ uint8_t panel_mask;
	1806	+
	1807	+ /**
	1808	+ * Explicit padding to 4 byte boundary.
	1809	+ */
	1810	+ uint8_t pad[2];
1705	1811	};
1706	1812
1707	1813	/**
@@ -1728,6 +1834,23 @@ struct dmub_cmd_abm_set_pwm_frac_data { · 20 unmodified lines
1728	1834	* TODO: Convert to uint8_t.
1729	1835	*/
1730	1836	uint32_t fractional_pwm;
	1837	+
	1838	+ /**
	1839	+ * ABM control version.
	1840	+ */
	1841	+ uint8_t version;
	1842	+
	1843	+ /**
	1844	+ * Panel Control HW instance mask.
	1845	+ * Bit 0 is Panel Control HW instance 0.
	1846	+ * Bit 1 is Panel Control HW instance 1.
	1847	+ */
	1848	+ uint8_t panel_mask;
	1849	+
	1850	+ /**
	1851	+ * Explicit padding to 4 byte boundary.
	1852	+ */
	1853	+ uint8_t pad[2];
1731	1854	};
1732	1855
1733	1856	/**
@@ -1758,6 +1881,24 @@ struct dmub_cmd_abm_init_config_data { · 24 unmodified lines
1758	1881	* Indirect buffer length.
1759	1882	*/
1760	1883	uint16_t bytes;
	1884	+
	1885	+
	1886	+ /**
	1887	+ * ABM control version.
	1888	+ */
	1889	+ uint8_t version;
	1890	+
	1891	+ /**
	1892	+ * Panel Control HW instance mask.
	1893	+ * Bit 0 is Panel Control HW instance 0.
	1894	+ * Bit 1 is Panel Control HW instance 1.
	1895	+ */
	1896	+ uint8_t panel_mask;
	1897	+
	1898	+ /**
	1899	+ * Explicit padding to 4 byte boundary.
	1900	+ */
	1901	+ uint8_t pad[2];
1761	1902	};
1762	1903
1763	1904	/**
@@ -2126,6 +2267,46 @@ static inline bool dmub_rb_front(struct dmub_rb *rb, · 362 unmodified lines
2126	2267	return true;
2127	2268	}
2128	2269
	2270	+/**
	2271	+ * @brief Determines the next ringbuffer offset.
	2272	+ *
	2273	+ * @param rb DMUB inbox ringbuffer
	2274	+ * @param num_cmds Number of commands
	2275	+ * @param next_rptr The next offset in the ringbuffer
	2276	+ */
	2277	+static inline void dmub_rb_get_rptr_with_offset(struct dmub_rb *rb,
	2278	+ uint32_t num_cmds,
	2279	+ uint32_t *next_rptr)
	2280	+{
	2281	+ next_rptr = rb->rptr + DMUB_RB_CMD_SIZE num_cmds;
	2282	+
	2283	+ if (*next_rptr >= rb->capacity)
	2284	+ *next_rptr %= rb->capacity;
	2285	+}
	2286	+
	2287	+/**
	2288	+ * @brief Returns a pointer to a command in the inbox.
	2289	+ *
	2290	+ * @param rb DMUB inbox ringbuffer
	2291	+ * @param cmd The inbox command to return
	2292	+ * @param rptr The ringbuffer offset
	2293	+ * @return true if not empty
	2294	+ * @return false otherwise
	2295	+ */
	2296	+static inline bool dmub_rb_peek_offset(struct dmub_rb *rb,
	2297	+ union dmub_rb_cmd **cmd,
	2298	+ uint32_t rptr)
	2299	+{
	2300	+ uint8_t rb_cmd = (uint8_t )(rb->base_address) + rptr;
	2301	+
	2302	+ if (dmub_rb_empty(rb))
	2303	+ return false;
	2304	+
	2305	+ cmd = (union dmub_rb_cmd )rb_cmd;
	2306	+
	2307	+ return true;
	2308	+}
	2309	+
2129	2310	/**
2130	2311	* @brief Returns the next unprocessed command in the outbox.
2131	2312	*

drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.c+1−1

@@ -385,7 +385,7 @@ union dmub_fw_boot_status dmub_dcn20_get_fw_boot_status(struct dmub_srv *dmub) · 384 unmodified lines
385	385	return status;
386	386	}
387	387
388		−void dmub_dcn20_enable_dmub_boot_options(struct dmub_srv *dmub)
	388	+void dmub_dcn20_enable_dmub_boot_options(struct dmub_srv dmub, const struct dmub_srv_hw_params params)
389	389	{
390	390	union dmub_fw_boot_options boot_options = {0};
391	391

drivers/gpu/drm/amd/display/dmub/src/dmub_dcn20.h+1−1

@@ -221,7 +221,7 @@ bool dmub_dcn20_is_gpint_acked(struct dmub_srv *dmub, · 220 unmodified lines
221	221
222	222	uint32_t dmub_dcn20_get_gpint_response(struct dmub_srv *dmub);
223	223
224		−void dmub_dcn20_enable_dmub_boot_options(struct dmub_srv *dmub);
	224	+void dmub_dcn20_enable_dmub_boot_options(struct dmub_srv dmub, const struct dmub_srv_hw_params params);
225	225
226	226	void dmub_dcn20_skip_dmub_panel_power_sequence(struct dmub_srv *dmub, bool skip);
227	227

drivers/gpu/drm/amd/display/dmub/src/dmub_srv.c+4−0

@@ -514,6 +514,10 @@ enum dmub_status dmub_srv_hw_init(struct dmub_srv *dmub, · 513 unmodified lines
514	514	outbox0_rb_params.capacity = tracebuff_fb->size - dmub_align(TRACE_BUFFER_ENTRY_OFFSET, 64);
515	515	dmub_rb_init(&dmub->outbox0_rb, &outbox0_rb_params);
516	516
	517	+ /* Report to DMUB what features are supported by current driver */
	518	+ if (dmub->hw_funcs.enable_dmub_boot_options)
	519	+ dmub->hw_funcs.enable_dmub_boot_options(dmub, params);
	520	+
517	521	if (dmub->hw_funcs.reset_release)
518	522	dmub->hw_funcs.reset_release(dmub);
519	523

drivers/gpu/drm/amd/display/include/link_service_types.h+8−0

@@ -68,6 +68,14 @@ enum link_training_result { · 67 unmodified lines
68	68	LINK_TRAINING_LQA_FAIL,
69	69	/* one of the CR,EQ or symbol lock is dropped */
70	70	LINK_TRAINING_LINK_LOSS,
	71	+ /* Abort link training (because sink unplugged) */
	72	+ LINK_TRAINING_ABORT,
	73	+};
	74	+
	75	+enum lttpr_mode {
	76	+ LTTPR_MODE_NON_LTTPR,
	77	+ LTTPR_MODE_TRANSPARENT,
	78	+ LTTPR_MODE_NON_TRANSPARENT,
71	79	};
72	80
73	81	struct link_training_settings {

drivers/gpu/drm/amd/display/modules/hdcp/hdcp.c+0−1

@@ -260,7 +260,6 @@ enum mod_hdcp_status mod_hdcp_setup(struct mod_hdcp *hdcp, · 259 unmodified lines
260	260	struct mod_hdcp_output output;
261	261	enum mod_hdcp_status status = MOD_HDCP_STATUS_SUCCESS;
262	262
263		− memset(hdcp, 0, sizeof(struct mod_hdcp));
264	263	memset(&output, 0, sizeof(output));
265	264	hdcp->config = *config;
266	265	HDCP_TOP_INTERFACE_TRACE(hdcp);

drivers/gpu/drm/amd/display/modules/hdcp/hdcp1_execution.c+5−0

@@ -128,6 +128,11 @@ static inline uint8_t get_device_count(struct mod_hdcp *hdcp) · 127 unmodified lines
128	128
129	129	static inline enum mod_hdcp_status check_device_count(struct mod_hdcp *hdcp)
130	130	{
	131	+ /* Avoid device count == 0 to do authentication */
	132	+ if (0 == get_device_count(hdcp)) {
	133	+ return MOD_HDCP_STATUS_HDCP1_DEVICE_COUNT_MISMATCH_FAILURE;
	134	+ }
	135	+
131	136	/* Some MST display may choose to report the internal panel as an HDCP RX.
132	137	* To update this condition with 1(because the immediate repeater's internal
133	138	* panel is possibly not included in DEVICE_COUNT) + get_device_count(hdcp).

drivers/gpu/drm/amd/display/modules/hdcp/hdcp2_execution.c+5−0

@@ -207,6 +207,11 @@ static inline uint8_t get_device_count(struct mod_hdcp *hdcp) · 206 unmodified lines
207	207
208	208	static enum mod_hdcp_status check_device_count(struct mod_hdcp *hdcp)
209	209	{
	210	+ /* Avoid device count == 0 to do authentication */
	211	+ if (0 == get_device_count(hdcp)) {
	212	+ return MOD_HDCP_STATUS_HDCP1_DEVICE_COUNT_MISMATCH_FAILURE;
	213	+ }
	214	+
210	215	/* Some MST display may choose to report the internal panel as an HDCP RX. */
211	216	/* To update this condition with 1(because the immediate repeater's internal */
212	217	/* panel is possibly not included in DEVICE_COUNT) + get_device_count(hdcp). */

drivers/gpu/drm/amd/include/asic_reg/df/df_3_6_sh_mask.h+1−0

@@ -50,6 +50,7 @@ · 49 unmodified lines
50	50	#define DF_CS_UMC_AON0_DramBaseAddress0__AddrRngVal_MASK 0x00000001L
51	51	#define DF_CS_UMC_AON0_DramBaseAddress0__LgcyMmioHoleEn_MASK 0x00000002L
52	52	#define DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK 0x0000003CL
	53	+#define ALDEBARAN_DF_CS_UMC_AON0_DramBaseAddress0__IntLvNumChan_MASK 0x0000007CL
53	54	#define DF_CS_UMC_AON0_DramBaseAddress0__IntLvAddrSel_MASK 0x00000E00L
54	55	#define DF_CS_UMC_AON0_DramBaseAddress0__DramBaseAddr_MASK 0xFFFFF000L
55	56

drivers/gpu/drm/amd/include/kgd_pp_interface.h+63−0

@@ -350,6 +350,7 @@ struct amd_pm_funcs { · 349 unmodified lines
350	350	unsigned int *num_states);
351	351	int (get_dpm_clock_table)(void handle,
352	352	struct dpm_clocks *clock_table);
	353	+ int (get_smu_prv_buf_details)(void handle, void *addr, size_t size);
353	354	};
354	355
355	356	struct metrics_table_header {
@@ -473,6 +474,68 @@ struct gpu_metrics_v1_1 { · 117 unmodified lines
473	474	uint16_t temperature_hbm[NUM_HBM_INSTANCES];
474	475	};
475	476
	477	+struct gpu_metrics_v1_2 {
	478	+ struct metrics_table_header common_header;
	479	+
	480	+ /* Temperature */
	481	+ uint16_t temperature_edge;
	482	+ uint16_t temperature_hotspot;
	483	+ uint16_t temperature_mem;
	484	+ uint16_t temperature_vrgfx;
	485	+ uint16_t temperature_vrsoc;
	486	+ uint16_t temperature_vrmem;
	487	+
	488	+ /* Utilization */
	489	+ uint16_t average_gfx_activity;
	490	+ uint16_t average_umc_activity; // memory controller
	491	+ uint16_t average_mm_activity; // UVD or VCN
	492	+
	493	+ /* Power/Energy */
	494	+ uint16_t average_socket_power;
	495	+ uint64_t energy_accumulator;
	496	+
	497	+ /* Driver attached timestamp (in ns) */
	498	+ uint64_t system_clock_counter;
	499	+
	500	+ /* Average clocks */
	501	+ uint16_t average_gfxclk_frequency;
	502	+ uint16_t average_socclk_frequency;
	503	+ uint16_t average_uclk_frequency;
	504	+ uint16_t average_vclk0_frequency;
	505	+ uint16_t average_dclk0_frequency;
	506	+ uint16_t average_vclk1_frequency;
	507	+ uint16_t average_dclk1_frequency;
	508	+
	509	+ /* Current clocks */
	510	+ uint16_t current_gfxclk;
	511	+ uint16_t current_socclk;
	512	+ uint16_t current_uclk;
	513	+ uint16_t current_vclk0;
	514	+ uint16_t current_dclk0;
	515	+ uint16_t current_vclk1;
	516	+ uint16_t current_dclk1;
	517	+
	518	+ /* Throttle status */
	519	+ uint32_t throttle_status;
	520	+
	521	+ /* Fans */
	522	+ uint16_t current_fan_speed;
	523	+
	524	+ /* Link width/speed */
	525	+ uint16_t pcie_link_width;
	526	+ uint16_t pcie_link_speed; // in 0.1 GT/s
	527	+
	528	+ uint16_t padding;
	529	+
	530	+ uint32_t gfx_activity_acc;
	531	+ uint32_t mem_activity_acc;
	532	+
	533	+ uint16_t temperature_hbm[NUM_HBM_INSTANCES];
	534	+
	535	+ /* PMFW attached timestamp (10ns resolution) */
	536	+ uint64_t firmware_timestamp;
	537	+};
	538	+
476	539	/*
477	540	* gpu_metrics_v2_0 is not recommended as it's not naturally aligned.
478	541	* Use gpu_metrics_v2_1 or later instead.

drivers/gpu/drm/amd/pm/amdgpu_pm.c+65−1

@@ -735,6 +735,23 @@ static ssize_t amdgpu_set_pp_table(struct device *dev, · 734 unmodified lines
735	735	* - a list of valid ranges for sclk, mclk, and voltage curve points
736	736	* labeled OD_RANGE
737	737	*
	738	+ * < For APUs >
	739	+ *
	740	+ * Reading the file will display:
	741	+ *
	742	+ * - minimum and maximum engine clock labeled OD_SCLK
	743	+ *
	744	+ * - a list of valid ranges for sclk labeled OD_RANGE
	745	+ *
	746	+ * < For VanGogh >
	747	+ *
	748	+ * Reading the file will display:
	749	+ *
	750	+ * - minimum and maximum engine clock labeled OD_SCLK
	751	+ * - minimum and maximum core clocks labeled OD_CCLK
	752	+ *
	753	+ * - a list of valid ranges for sclk and cclk labeled OD_RANGE
	754	+ *
738	755	* To manually adjust these settings:
739	756	*
740	757	* - First select manual using power_dpm_force_performance_level
@@ -743,7 +760,10 @@ static ssize_t amdgpu_set_pp_table(struct device *dev, · 2 unmodified lines
743	760	* string that contains "s/m index clock" to the file. The index
744	761	* should be 0 if to set minimum clock. And 1 if to set maximum
745	762	* clock. E.g., "s 0 500" will update minimum sclk to be 500 MHz.
746		− * "m 1 800" will update maximum mclk to be 800Mhz.
	763	+ * "m 1 800" will update maximum mclk to be 800Mhz. For core
	764	+ * clocks on VanGogh, the string contains "p core index clock".
	765	+ * E.g., "p 2 0 800" would set the minimum core clock on core
	766	+ * 2 to 800Mhz.
747	767	*
748	768	* For sclk voltage curve, enter the new values by writing a
749	769	* string that contains "vc point clock voltage" to the file. The
@@ -3534,6 +3554,45 @@ out: · 2784 unmodified lines
3534	3554
3535	3555	DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_pm_info);
3536	3556
	3557	+/*
	3558	+ * amdgpu_pm_priv_buffer_read - Read memory region allocated to FW
	3559	+ *
	3560	+ * Reads debug memory region allocated to PMFW
	3561	+ */
	3562	+static ssize_t amdgpu_pm_prv_buffer_read(struct file f, char __user buf,
	3563	+ size_t size, loff_t *pos)
	3564	+{
	3565	+ struct amdgpu_device *adev = file_inode(f)->i_private;
	3566	+ const struct amd_pm_funcs *pp_funcs = adev->powerplay.pp_funcs;
	3567	+ void *pp_handle = adev->powerplay.pp_handle;
	3568	+ size_t smu_prv_buf_size;
	3569	+ void *smu_prv_buf;
	3570	+
	3571	+ if (amdgpu_in_reset(adev))
	3572	+ return -EPERM;
	3573	+ if (adev->in_suspend && !adev->in_runpm)
	3574	+ return -EPERM;
	3575	+
	3576	+ if (pp_funcs && pp_funcs->get_smu_prv_buf_details)
	3577	+ pp_funcs->get_smu_prv_buf_details(pp_handle, &smu_prv_buf,
	3578	+ &smu_prv_buf_size);
	3579	+ else
	3580	+ return -ENOSYS;
	3581	+
	3582	+ if (!smu_prv_buf \|\| !smu_prv_buf_size)
	3583	+ return -EINVAL;
	3584	+
	3585	+ return simple_read_from_buffer(buf, size, pos, smu_prv_buf,
	3586	+ smu_prv_buf_size);
	3587	+}
	3588	+
	3589	+static const struct file_operations amdgpu_debugfs_pm_prv_buffer_fops = {
	3590	+ .owner = THIS_MODULE,
	3591	+ .open = simple_open,
	3592	+ .read = amdgpu_pm_prv_buffer_read,
	3593	+ .llseek = default_llseek,
	3594	+};
	3595	+
3537	3596	#endif
3538	3597
3539	3598	void amdgpu_debugfs_pm_init(struct amdgpu_device *adev)
@@ -3545,5 +3604,10 @@ void amdgpu_debugfs_pm_init(struct amdgpu_device *adev) · 5 unmodified lines
3545	3604	debugfs_create_file("amdgpu_pm_info", 0444, root, adev,
3546	3605	&amdgpu_debugfs_pm_info_fops);
3547	3606
	3607	+ if (adev->pm.smu_prv_buffer_size > 0)
	3608	+ debugfs_create_file_size("amdgpu_pm_prv_buffer", 0444, root,
	3609	+ adev,
	3610	+ &amdgpu_debugfs_pm_prv_buffer_fops,
	3611	+ adev->pm.smu_prv_buffer_size);
3548	3612	#endif
3549	3613	}

drivers/gpu/drm/amd/pm/inc/smu13_driver_if_aldebaran.h+5−2

@@ -64,7 +64,7 @@ · 63 unmodified lines
64	64	#define FEATURE_SMUIO_CG_BIT 28
65	65	#define FEATURE_THM_CG_BIT 29
66	66	#define FEATURE_CLK_CG_BIT 30
67		−#define FEATURE_SPARE_31_BIT 31
	67	+#define FEATURE_EDC_BIT 31
68	68	#define FEATURE_SPARE_32_BIT 32
69	69	#define FEATURE_SPARE_33_BIT 33
70	70	#define FEATURE_SPARE_34_BIT 34
@@ -439,8 +439,11 @@ typedef struct { · 368 unmodified lines
439	439	int8_t XgmiOffset; // in Amps
440	440	uint8_t Padding_TelemetryXgmi;
441	441
	442	+ uint16_t EdcPowerLimit;
	443	+ uint16_t spare6;
	444	+
442	445	//reserved
443		− uint32_t reserved[15];
	446	+ uint32_t reserved[14];
444	447
445	448	} PPTable_t;
446	449

drivers/gpu/drm/amd/pm/inc/smu_v13_0.h+1−1

@@ -26,7 +26,7 @@ · 25 unmodified lines
26	26	#include "amdgpu_smu.h"
27	27
28	28	#define SMU13_DRIVER_IF_VERSION_INV 0xFFFFFFFF
29		−#define SMU13_DRIVER_IF_VERSION_ALDE 0x6
	29	+#define SMU13_DRIVER_IF_VERSION_ALDE 0x07
30	30
31	31	/* MP Apertures */
32	32	#define MP0_Public 0x03800000

drivers/gpu/drm/amd/pm/powerplay/amd_powerplay.c+21−0

@@ -1651,6 +1651,26 @@ static int pp_gfx_state_change_set(void *handle, uint32_t state) · 1650 unmodified lines
1651	1651	return 0;
1652	1652	}
1653	1653
	1654	+static int pp_get_prv_buffer_details(void handle, void addr, size_t size)
	1655	+{
	1656	+ struct pp_hwmgr *hwmgr = handle;
	1657	+ struct amdgpu_device *adev = hwmgr->adev;
	1658	+
	1659	+ if (!addr \|\| !size)
	1660	+ return -EINVAL;
	1661	+
	1662	+ *addr = NULL;
	1663	+ *size = 0;
	1664	+ mutex_lock(&hwmgr->smu_lock);
	1665	+ if (adev->pm.smu_prv_buffer) {
	1666	+ amdgpu_bo_kmap(adev->pm.smu_prv_buffer, addr);
	1667	+ *size = adev->pm.smu_prv_buffer_size;
	1668	+ }
	1669	+ mutex_unlock(&hwmgr->smu_lock);
	1670	+
	1671	+ return 0;
	1672	+}
	1673	+
1654	1674	static const struct amd_pm_funcs pp_dpm_funcs = {
1655	1675	.load_firmware = pp_dpm_load_fw,
1656	1676	.wait_for_fw_loading_complete = pp_dpm_fw_loading_complete,
@@ -1714,4 +1734,5 @@ static const struct amd_pm_funcs pp_dpm_funcs = { · 57 unmodified lines
1714	1734	.set_xgmi_pstate = pp_set_xgmi_pstate,
1715	1735	.get_gpu_metrics = pp_get_gpu_metrics,
1716	1736	.gfx_state_change_set = pp_gfx_state_change_set,
	1737	+ .get_smu_prv_buf_details = pp_get_prv_buffer_details,
1717	1738	};

drivers/gpu/drm/amd/pm/powerplay/hwmgr/ppatomctrl.c+4−4

@@ -275,7 +275,7 @@ static const ATOM_VOLTAGE_OBJECT_V3 *atomctrl_lookup_voltage_type_v3( · 274 unmodified lines
275	275	}
276	276
277	277	/**
278		− * atomctrl_get_memory_pll_dividers_si().
	278	+ * atomctrl_get_memory_pll_dividers_si
279	279	*
280	280	* @hwmgr: input parameter: pointer to HwMgr
281	281	* @clock_value: input parameter: memory clock
@@ -328,7 +328,7 @@ int atomctrl_get_memory_pll_dividers_si( · 46 unmodified lines
328	328	}
329	329
330	330	/**
331		− * atomctrl_get_memory_pll_dividers_vi().
	331	+ * atomctrl_get_memory_pll_dividers_vi
332	332	*
333	333	* @hwmgr: input parameter: pointer to HwMgr
334	334	* @clock_value: input parameter: memory clock
@@ -1104,7 +1104,7 @@ int atomctrl_calculate_voltage_evv_on_sclk( · 769 unmodified lines
1104	1104	}
1105	1105
1106	1106	/**
1107		− * atomctrl_get_voltage_evv_on_sclk gets voltage via call to ATOM COMMAND table.
	1107	+ * atomctrl_get_voltage_evv_on_sclk: gets voltage via call to ATOM COMMAND table.
1108	1108	* @hwmgr: input: pointer to hwManager
1109	1109	* @voltage_type: input: type of EVV voltage VDDC or VDDGFX
1110	1110	* @sclk: input: in 10Khz unit. DPM state SCLK frequency
@@ -1144,7 +1144,7 @@ int atomctrl_get_voltage_evv_on_sclk( · 33 unmodified lines
1144	1144	}
1145	1145
1146	1146	/**
1147		− * atomctrl_get_voltage_evv gets voltage via call to ATOM COMMAND table.
	1147	+ * atomctrl_get_voltage_evv: gets voltage via call to ATOM COMMAND table.
1148	1148	* @hwmgr: input: pointer to hwManager
1149	1149	* @virtual_voltage_id: input: voltage id which match per voltage DPM state: 0xff01, 0xff02.. 0xff08
1150	1150	* @voltage: output: real voltage level in unit of mv

drivers/gpu/drm/amd/pm/powerplay/hwmgr/vega20_baco.c+1−1

@@ -85,7 +85,7 @@ int vega20_baco_set_state(struct pp_hwmgr *hwmgr, enum BACO_STATE state) · 84 unmodified lines
85	85	return 0;
86	86
87	87	if (state == BACO_STATE_IN) {
88		− if (!ras \|\| !ras->supported) {
	88	+ if (!ras \|\| !adev->ras_enabled) {
89	89	data = RREG32_SOC15(THM, 0, mmTHM_BACO_CNTL);
90	90	data \|= 0x80000000;
91	91	WREG32_SOC15(THM, 0, mmTHM_BACO_CNTL, data);

drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c+21−0

@@ -2933,6 +2933,26 @@ int smu_set_light_sbr(struct smu_context *smu, bool enable) · 2932 unmodified lines
2933	2933	return ret;
2934	2934	}
2935	2935
	2936	+static int smu_get_prv_buffer_details(void handle, void addr, size_t size)
	2937	+{
	2938	+ struct smu_context *smu = handle;
	2939	+ struct smu_table_context *smu_table = &smu->smu_table;
	2940	+ struct smu_table *memory_pool = &smu_table->memory_pool;
	2941	+
	2942	+ if (!addr \|\| !size)
	2943	+ return -EINVAL;
	2944	+
	2945	+ *addr = NULL;
	2946	+ *size = 0;
	2947	+ mutex_lock(&smu->mutex);
	2948	+ if (memory_pool->bo) {
	2949	+ *addr = memory_pool->cpu_addr;
	2950	+ *size = memory_pool->size;
	2951	+ }
	2952	+ mutex_unlock(&smu->mutex);
	2953	+
	2954	+ return 0;
	2955	+}
2936	2956
2937	2957	static const struct amd_pm_funcs swsmu_pm_funcs = {
2938	2958	/* export for sysfs */
@@ -2984,6 +3004,7 @@ static const struct amd_pm_funcs swsmu_pm_funcs = { · 45 unmodified lines
2984	3004	.get_max_sustainable_clocks_by_dc = smu_get_max_sustainable_clocks_by_dc,
2985	3005	.load_firmware = smu_load_microcode,
2986	3006	.gfx_state_change_set = smu_gfx_state_change_set,
	3007	+ .get_smu_prv_buf_details = smu_get_prv_buffer_details,
2987	3008	};
2988	3009
2989	3010	int smu_wait_for_event(struct amdgpu_device *adev, enum smu_event_type event,

drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c+1−1

@@ -302,7 +302,7 @@ sienna_cichlid_get_allowed_feature_mask(struct smu_context *smu, · 301 unmodified lines
302	302	if (smu->dc_controlled_by_gpio)
303	303	(uint64_t )feature_mask \|= FEATURE_MASK(FEATURE_ACDC_BIT);
304	304
305		− if (amdgpu_aspm == 1)
	305	+ if (amdgpu_aspm)
306	306	(uint64_t )feature_mask \|= FEATURE_MASK(FEATURE_DS_LCLK_BIT);
307	307
308	308	return 0;

drivers/gpu/drm/amd/pm/swsmu/smu11/smu_v11_0.c+2−1

@@ -1531,7 +1531,8 @@ int smu_v11_0_baco_set_state(struct smu_context *smu, enum smu_baco_state state) · 1530 unmodified lines
1531	1531	NULL);
1532	1532	break;
1533	1533	default:
1534		− if (!ras \|\| !ras->supported \|\| adev->gmc.xgmi.pending_reset) {
	1534	+ if (!ras \|\| !adev->ras_enabled \|\|
	1535	+ adev->gmc.xgmi.pending_reset) {
1535	1536	if (adev->asic_type == CHIP_ARCTURUS) {
1536	1537	data = RREG32_SOC15(THM, 0, mmTHM_BACO_CNTL_ARCT);
1537	1538	data \|= 0x80000000;

drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c+11−6

@@ -209,7 +209,7 @@ static int aldebaran_tables_init(struct smu_context *smu) · 208 unmodified lines
209	209	return -ENOMEM;
210	210	smu_table->metrics_time = 0;
211	211
212		− smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v1_1);
	212	+ smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v1_2);
213	213	smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL);
214	214	if (!smu_table->gpu_metrics_table) {
215	215	kfree(smu_table->metrics_table);
@@ -1632,8 +1632,8 @@ static ssize_t aldebaran_get_gpu_metrics(struct smu_context *smu, · 1416 unmodified lines
1632	1632	void **table)
1633	1633	{
1634	1634	struct smu_table_context *smu_table = &smu->smu_table;
1635		− struct gpu_metrics_v1_1 *gpu_metrics =
1636		− (struct gpu_metrics_v1_1 *)smu_table->gpu_metrics_table;
	1635	+ struct gpu_metrics_v1_2 *gpu_metrics =
	1636	+ (struct gpu_metrics_v1_2 *)smu_table->gpu_metrics_table;
1637	1637	SmuMetrics_t metrics;
1638	1638	int i, ret = 0;
1639	1639
@@ -1643,7 +1643,7 @@ static ssize_t aldebaran_get_gpu_metrics(struct smu_context *smu, · 3 unmodified lines
1643	1643	if (ret)
1644	1644	return ret;
1645	1645
1646		− smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 1);
	1646	+ smu_cmn_init_soft_gpu_metrics(gpu_metrics, 1, 2);
1647	1647
1648	1648	gpu_metrics->temperature_edge = metrics.TemperatureEdge;
1649	1649	gpu_metrics->temperature_hotspot = metrics.TemperatureHotspot;
@@ -1657,7 +1657,9 @@ static ssize_t aldebaran_get_gpu_metrics(struct smu_context *smu, · 7 unmodified lines
1657	1657	gpu_metrics->average_mm_activity = 0;
1658	1658
1659	1659	gpu_metrics->average_socket_power = metrics.AverageSocketPower;
1660		− gpu_metrics->energy_accumulator = 0;
	1660	+ gpu_metrics->energy_accumulator =
	1661	+ (uint64_t)metrics.EnergyAcc64bitHigh << 32 \|
	1662	+ metrics.EnergyAcc64bitLow;
1661	1663
1662	1664	gpu_metrics->average_gfxclk_frequency = metrics.AverageGfxclkFrequency;
1663	1665	gpu_metrics->average_socclk_frequency = metrics.AverageSocclkFrequency;
@@ -1688,9 +1690,12 @@ static ssize_t aldebaran_get_gpu_metrics(struct smu_context *smu, · 24 unmodified lines
1688	1690	for (i = 0; i < NUM_HBM_INSTANCES; i++)
1689	1691	gpu_metrics->temperature_hbm[i] = metrics.TemperatureAllHBM[i];
1690	1692
	1693	+ gpu_metrics->firmware_timestamp = ((uint64_t)metrics.TimeStampHigh << 32) \|
	1694	+ metrics.TimeStampLow;
	1695	+
1691	1696	table = (void )gpu_metrics;
1692	1697
1693		− return sizeof(struct gpu_metrics_v1_1);
	1698	+ return sizeof(struct gpu_metrics_v1_2);
1694	1699	}
1695	1700
1696	1701	static int aldebaran_mode2_reset(struct smu_context *smu)

drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c+5−2

@@ -104,8 +104,8 @@ int smu_cmn_send_msg_without_waiting(struct smu_context *smu, · 103 unmodified lines
104	104
105	105	ret = smu_cmn_wait_for_response(smu);
106	106	if (ret != 0x1) {
107		− dev_err(adev->dev, "Msg issuing pre-check failed and "
108		− "SMU may be not in the right state!\n");
	107	+ dev_err(adev->dev, "Msg issuing pre-check failed(0x%x) and "
	108	+ "SMU may be not in the right state!\n", ret);
109	109	if (ret != -ETIME)
110	110	ret = -EIO;
111	111	return ret;
@@ -761,6 +761,9 @@ void smu_cmn_init_soft_gpu_metrics(void *table, uint8_t frev, uint8_t crev) · 649 unmodified lines
761	761	case METRICS_VERSION(1, 1):
762	762	structure_size = sizeof(struct gpu_metrics_v1_1);
763	763	break;
	764	+ case METRICS_VERSION(1, 2):
	765	+ structure_size = sizeof(struct gpu_metrics_v1_2);
	766	+ break;
764	767	case METRICS_VERSION(2, 0):
765	768	structure_size = sizeof(struct gpu_metrics_v2_0);
766	769	break;

drivers/gpu/drm/radeon/radeon_device.c+2−1

@@ -1098,7 +1098,8 @@ static bool radeon_check_pot_argument(int arg) · 1097 unmodified lines
1098	1098	}
1099	1099
1100	1100	/**
1101		− * Determine a sensible default GART size according to ASIC family.
	1101	+ * radeon_gart_size_auto - Determine a sensible default GART size
	1102	+ * according to ASIC family.
1102	1103	*
1103	1104	* @family: ASIC family name
1104	1105	*/

include/uapi/drm/amdgpu_drm.h+0−2

@@ -116,8 +116,6 @@ extern "C" { · 115 unmodified lines
116	116	#define AMDGPU_GEM_CREATE_CPU_GTT_USWC (1 << 2)
117	117	/* Flag that the memory should be in VRAM and cleared */
118	118	#define AMDGPU_GEM_CREATE_VRAM_CLEARED (1 << 3)
119		−/* Flag that create shadow bo(GTT) while allocating vram bo */
120		−#define AMDGPU_GEM_CREATE_SHADOW (1 << 4)
121	119	/* Flag that allocating the BO should use linear VRAM */
122	120	#define AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS (1 << 5)
123	121	/* Flag that BO is always valid in this VM */

include/uapi/linux/kfd_ioctl.h+169−2

@@ -30,9 +30,10 @@ · 29 unmodified lines
30	30	* - 1.1 - initial version
31	31	* - 1.3 - Add SMI events support
32	32	* - 1.4 - Indicate new SRAM EDC bit in device properties
	33	+ * - 1.5 - Add SVM API
33	34	*/
34	35	#define KFD_IOCTL_MAJOR_VERSION 1
35		−#define KFD_IOCTL_MINOR_VERSION 4
	36	+#define KFD_IOCTL_MINOR_VERSION 5
36	37
37	38	struct kfd_ioctl_get_version_args {
38	39	__u32 major_version; /* from KFD */
@@ -473,6 +474,167 @@ enum kfd_mmio_remap { · 434 unmodified lines
473	474	KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL = 4,
474	475	};
475	476
	477	+/* Guarantee host access to memory */
	478	+#define KFD_IOCTL_SVM_FLAG_HOST_ACCESS 0x00000001
	479	+/* Fine grained coherency between all devices with access */
	480	+#define KFD_IOCTL_SVM_FLAG_COHERENT 0x00000002
	481	+/* Use any GPU in same hive as preferred device */
	482	+#define KFD_IOCTL_SVM_FLAG_HIVE_LOCAL 0x00000004
	483	+/* GPUs only read, allows replication */
	484	+#define KFD_IOCTL_SVM_FLAG_GPU_RO 0x00000008
	485	+/* Allow execution on GPU */
	486	+#define KFD_IOCTL_SVM_FLAG_GPU_EXEC 0x00000010
	487	+/* GPUs mostly read, may allow similar optimizations as RO, but writes fault */
	488	+#define KFD_IOCTL_SVM_FLAG_GPU_READ_MOSTLY 0x00000020
	489	+
	490	+/**
	491	+ * kfd_ioctl_svm_op - SVM ioctl operations
	492	+ *
	493	+ * @KFD_IOCTL_SVM_OP_SET_ATTR: Modify one or more attributes
	494	+ * @KFD_IOCTL_SVM_OP_GET_ATTR: Query one or more attributes
	495	+ */
	496	+enum kfd_ioctl_svm_op {
	497	+ KFD_IOCTL_SVM_OP_SET_ATTR,
	498	+ KFD_IOCTL_SVM_OP_GET_ATTR
	499	+};
	500	+
	501	+/** kfd_ioctl_svm_location - Enum for preferred and prefetch locations
	502	+ *
	503	+ * GPU IDs are used to specify GPUs as preferred and prefetch locations.
	504	+ * Below definitions are used for system memory or for leaving the preferred
	505	+ * location unspecified.
	506	+ */
	507	+enum kfd_ioctl_svm_location {
	508	+ KFD_IOCTL_SVM_LOCATION_SYSMEM = 0,
	509	+ KFD_IOCTL_SVM_LOCATION_UNDEFINED = 0xffffffff
	510	+};
	511	+
	512	+/**
	513	+ * kfd_ioctl_svm_attr_type - SVM attribute types
	514	+ *
	515	+ * @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC: gpuid of the preferred location, 0 for
	516	+ * system memory
	517	+ * @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC: gpuid of the prefetch location, 0 for
	518	+ * system memory. Setting this triggers an
	519	+ * immediate prefetch (migration).
	520	+ * @KFD_IOCTL_SVM_ATTR_ACCESS:
	521	+ * @KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE:
	522	+ * @KFD_IOCTL_SVM_ATTR_NO_ACCESS: specify memory access for the gpuid given
	523	+ * by the attribute value
	524	+ * @KFD_IOCTL_SVM_ATTR_SET_FLAGS: bitmask of flags to set (see
	525	+ * KFD_IOCTL_SVM_FLAG_...)
	526	+ * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS: bitmask of flags to clear
	527	+ * @KFD_IOCTL_SVM_ATTR_GRANULARITY: migration granularity
	528	+ * (log2 num pages)
	529	+ */
	530	+enum kfd_ioctl_svm_attr_type {
	531	+ KFD_IOCTL_SVM_ATTR_PREFERRED_LOC,
	532	+ KFD_IOCTL_SVM_ATTR_PREFETCH_LOC,
	533	+ KFD_IOCTL_SVM_ATTR_ACCESS,
	534	+ KFD_IOCTL_SVM_ATTR_ACCESS_IN_PLACE,
	535	+ KFD_IOCTL_SVM_ATTR_NO_ACCESS,
	536	+ KFD_IOCTL_SVM_ATTR_SET_FLAGS,
	537	+ KFD_IOCTL_SVM_ATTR_CLR_FLAGS,
	538	+ KFD_IOCTL_SVM_ATTR_GRANULARITY
	539	+};
	540	+
	541	+/**
	542	+ * kfd_ioctl_svm_attribute - Attributes as pairs of type and value
	543	+ *
	544	+ * The meaning of the @value depends on the attribute type.
	545	+ *
	546	+ * @type: attribute type (see enum @kfd_ioctl_svm_attr_type)
	547	+ * @value: attribute value
	548	+ */
	549	+struct kfd_ioctl_svm_attribute {
	550	+ __u32 type;
	551	+ __u32 value;
	552	+};
	553	+
	554	+/**
	555	+ * kfd_ioctl_svm_args - Arguments for SVM ioctl
	556	+ *
	557	+ * @op specifies the operation to perform (see enum
	558	+ * @kfd_ioctl_svm_op). @start_addr and @size are common for all
	559	+ * operations.
	560	+ *
	561	+ * A variable number of attributes can be given in @attrs.
	562	+ * @nattr specifies the number of attributes. New attributes can be
	563	+ * added in the future without breaking the ABI. If unknown attributes
	564	+ * are given, the function returns -EINVAL.
	565	+ *
	566	+ * @KFD_IOCTL_SVM_OP_SET_ATTR sets attributes for a virtual address
	567	+ * range. It may overlap existing virtual address ranges. If it does,
	568	+ * the existing ranges will be split such that the attribute changes
	569	+ * only apply to the specified address range.
	570	+ *
	571	+ * @KFD_IOCTL_SVM_OP_GET_ATTR returns the intersection of attributes
	572	+ * over all memory in the given range and returns the result as the
	573	+ * attribute value. If different pages have different preferred or
	574	+ * prefetch locations, 0xffffffff will be returned for
	575	+ * @KFD_IOCTL_SVM_ATTR_PREFERRED_LOC or
	576	+ * @KFD_IOCTL_SVM_ATTR_PREFETCH_LOC resepctively. For
	577	+ * @KFD_IOCTL_SVM_ATTR_SET_FLAGS, flags of all pages will be
	578	+ * aggregated by bitwise AND. The minimum migration granularity
	579	+ * throughout the range will be returned for
	580	+ * @KFD_IOCTL_SVM_ATTR_GRANULARITY.
	581	+ *
	582	+ * Querying of accessibility attributes works by initializing the
	583	+ * attribute type to @KFD_IOCTL_SVM_ATTR_ACCESS and the value to the
	584	+ * GPUID being queried. Multiple attributes can be given to allow
	585	+ * querying multiple GPUIDs. The ioctl function overwrites the
	586	+ * attribute type to indicate the access for the specified GPU.
	587	+ *
	588	+ * @KFD_IOCTL_SVM_ATTR_CLR_FLAGS is invalid for
	589	+ * @KFD_IOCTL_SVM_OP_GET_ATTR.
	590	+ */
	591	+struct kfd_ioctl_svm_args {
	592	+ __u64 start_addr;
	593	+ __u64 size;
	594	+ __u32 op;
	595	+ __u32 nattr;
	596	+ /* Variable length array of attributes */
	597	+ struct kfd_ioctl_svm_attribute attrs[0];
	598	+};
	599	+
	600	+/**
	601	+ * kfd_ioctl_set_xnack_mode_args - Arguments for set_xnack_mode
	602	+ *
	603	+ * @xnack_enabled: [in/out] Whether to enable XNACK mode for this process
	604	+ *
	605	+ * @xnack_enabled indicates whether recoverable page faults should be
	606	+ * enabled for the current process. 0 means disabled, positive means
	607	+ * enabled, negative means leave unchanged. If enabled, virtual address
	608	+ * translations on GFXv9 and later AMD GPUs can return XNACK and retry
	609	+ * the access until a valid PTE is available. This is used to implement
	610	+ * device page faults.
	611	+ *
	612	+ * On output, @xnack_enabled returns the (new) current mode (0 or
	613	+ * positive). Therefore, a negative input value can be used to query
	614	+ * the current mode without changing it.
	615	+ *
	616	+ * The XNACK mode fundamentally changes the way SVM managed memory works
	617	+ * in the driver, with subtle effects on application performance and
	618	+ * functionality.
	619	+ *
	620	+ * Enabling XNACK mode requires shader programs to be compiled
	621	+ * differently. Furthermore, not all GPUs support changing the mode
	622	+ * per-process. Therefore changing the mode is only allowed while no
	623	+ * user mode queues exist in the process. This ensure that no shader
	624	+ * code is running that may be compiled for the wrong mode. And GPUs
	625	+ * that cannot change to the requested mode will prevent the XNACK
	626	+ * mode from occurring. All GPUs used by the process must be in the
	627	+ * same XNACK mode.
	628	+ *
	629	+ * GFXv8 or older GPUs do not support 48 bit virtual addresses or SVM.
	630	+ * Therefore those GPUs are not considered for the XNACK mode switch.
	631	+ *
	632	+ * Return: 0 on success, -errno on failure
	633	+ */
	634	+struct kfd_ioctl_set_xnack_mode_args {
	635	+ __s32 xnack_enabled;
	636	+};
	637	+
476	638	#define AMDKFD_IOCTL_BASE 'K'
477	639	#define AMDKFD_IO(nr) _IO(AMDKFD_IOCTL_BASE, nr)
478	640	#define AMDKFD_IOR(nr, type) _IOR(AMDKFD_IOCTL_BASE, nr, type)
@@ -573,7 +735,12 @@ enum kfd_mmio_remap { · 94 unmodified lines
573	735	#define AMDKFD_IOC_SMI_EVENTS \
574	736	AMDKFD_IOWR(0x1F, struct kfd_ioctl_smi_events_args)
575	737
	738	+#define AMDKFD_IOC_SVM AMDKFD_IOWR(0x20, struct kfd_ioctl_svm_args)
	739	+
	740	+#define AMDKFD_IOC_SET_XNACK_MODE \
	741	+ AMDKFD_IOWR(0x21, struct kfd_ioctl_set_xnack_mode_args)
	742	+
576	743	#define AMDKFD_COMMAND_START 0x01
577		−#define AMDKFD_COMMAND_END 0x20
	744	+#define AMDKFD_COMMAND_END 0x22
578	745
579	746	#endif