[airlied: had to reapply drm/amdgpu: Implement "color format" DRM property by hand to amdgpu_dm_connector.c] amd-drm-next-7.3-2026-07-02: amdgpu: - Queue reset updates - Initial compute pipe reset support - Improved boundary checking for bios parsing - Cleaned up sysfs input parsing - devcoredump fixes - RAS updates and rework - VCN secure submission fixes - 8K panel fix - Add display KUnit tests - Display CRC fixes - UserQ updates - Backlight fixes - Parse panel type info from DisplayID - Align IP discovery to pci device lifetime - IOCTL boundary check fixes - Convert amdgpu_vm_lock_by_pasid() to drm_exec - Ctx fixes and cleanup - SOC15 register macro cleanups - Memory placement fixes for UVD - Disable KQ support for MI3xx - GFX9 mode2 reset fix - BO list cleanup - Soc24 aborted suspend fix - Gfx8 soft reset rework - Enable soft reset on gfx8 - Drop unnecessary BUG() and BUG_ON() in error paths - Fix power reporting unit conversion - Improve vbios command table bounds checking - UVD bounds checking improvements - VCN bounds checking improvements - PSR and replay fixes - DCN 4.2 updates - Colorop updates - DC GPIO rework - ACP fixes - Fix aperture mapping leak - Ignore_damage_clips fix - Fixes for non-4K pages - JPEG idle check fixes - Userptr fixes - GPUVM fixes - GC 11.7 updates - SMU 13 fixes amdkfd: - Initial compute pipe reset support - Allow applications to opt out of sigbus on fatal errors - Fix doorbell/mmio BO cleanup - Improved CRIU boundary checking - MQD handling rework - SMI fixes - Reset event fixes - CRIU fixes - Sysfs teardown fixes - IOCTL boundary check fixes - SVM fixes - Soft IH ring fixes - Move TBA/TMA from system to device memory radeon: - Blit fix for large BOs - r600 dpm cleanup fix drm: - Extract EDID base section header processing into helper - Parse panel type from DisplayID 2.x Display Parameters UAPI: - KFD interface for applications to select sigbus behavior on fatal errors Proposed userspace: https://github.com/ROCm/rocm-systems/pull/6190
Unified · Split
| @@ -70,7 +70,7 @@ amdgpu-y += amdgpu_device.o amdgpu_reg_access.o amdgpu_doorbell_mgr.o amdgpu_kms · 69 unmodified lines | ||
| 70 | 70 | amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o \ |
| 71 | 71 | amdgpu_fw_attestation.o amdgpu_securedisplay.o \ |
| 72 | 72 | amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o amdgpu_lockdep.o \ |
| 73 | − amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_aca.o amdgpu_dev_coredump.o \ | |
| 73 | + amdgpu_ring_mux.o amdgpu_xcp.o amdgpu_seq64.o amdgpu_dev_coredump.o \ | |
| 74 | 74 | amdgpu_cper.o amdgpu_userq_fence.o amdgpu_eviction_fence.o amdgpu_ip.o |
| 75 | 75 | |
| 76 | 76 | amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o |
| @@ -44,6 +44,7 @@ · 43 unmodified lines | ||
| 44 | 44 | #include <linux/hashtable.h> |
| 45 | 45 | #include <linux/dma-fence.h> |
| 46 | 46 | #include <linux/pci.h> |
| 47 | +#include <linux/xarray.h> | |
| 47 | 48 | |
| 48 | 49 | #include <drm/ttm/ttm_bo.h> |
| 49 | 50 | #include <drm/ttm/ttm_placement.h> |
| @@ -103,7 +104,6 @@ · 53 unmodified lines | ||
| 103 | 104 | #include "amdgpu_smuio.h" |
| 104 | 105 | #include "amdgpu_fdinfo.h" |
| 105 | 106 | #include "amdgpu_mca.h" |
| 106 | −#include "amdgpu_aca.h" | |
| 107 | 107 | #include "amdgpu_ras.h" |
| 108 | 108 | #include "amdgpu_lockdep.h" |
| 109 | 109 | #include "amdgpu_cper.h" |
| @@ -113,6 +113,7 @@ · 3 unmodified lines | ||
| 113 | 113 | #include "amdgpu_userq.h" |
| 114 | 114 | #include "amdgpu_eviction_fence.h" |
| 115 | 115 | #include "amdgpu_ip.h" |
| 116 | +#include "amdgpu_sa.h" | |
| 116 | 117 | #if defined(CONFIG_DRM_AMD_ISP) |
| 117 | 118 | #include "amdgpu_isp.h" |
| 118 | 119 | #endif |
| @@ -272,7 +273,6 @@ extern int amdgpu_ptl; · 153 unmodified lines | ||
| 272 | 273 | |
| 273 | 274 | extern uint amdgpu_hdmi_hpd_debounce_delay_ms; |
| 274 | 275 | |
| 275 | −#define AMDGPU_VM_MAX_NUM_CTX 4096 | |
| 276 | 276 | #define AMDGPU_SG_THRESHOLD (256*1024*1024) |
| 277 | 277 | #define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS 3000 |
| 278 | 278 | #define AMDGPU_MAX_USEC_TIMEOUT 100000 /* 100 ms */ |
| @@ -305,9 +305,10 @@ extern uint amdgpu_hdmi_hpd_debounce_delay_ms; · 26 unmodified lines | ||
| 305 | 305 | |
| 306 | 306 | /* reset mask */ |
| 307 | 307 | #define AMDGPU_RESET_TYPE_FULL (1 << 0) /* full adapter reset, mode1/mode2/BACO/etc. */ |
| 308 | −#define AMDGPU_RESET_TYPE_SOFT_RESET (1 << 1) /* IP level soft reset */ | |
| 308 | +#define AMDGPU_RESET_TYPE_SOFT_RECOVERY (1 << 1) /* soft recovery, eg. kill shaders */ | |
| 309 | 309 | #define AMDGPU_RESET_TYPE_PER_QUEUE (1 << 2) /* per queue */ |
| 310 | 310 | #define AMDGPU_RESET_TYPE_PER_PIPE (1 << 3) /* per pipe */ |
| 311 | +#define AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET (1 << 4) /* soft-resets an IP block */ | |
| 311 | 312 | |
| 312 | 313 | /* max cursor sizes (in pixels) */ |
| 313 | 314 | #define CIK_CURSOR_WIDTH 128 |
| @@ -387,37 +388,6 @@ struct amdgpu_clock { · 73 unmodified lines | ||
| 387 | 388 | uint32_t max_pixel_clock; |
| 388 | 389 | }; |
| 389 | 390 | |
| 390 | −/* sub-allocation manager, it has to be protected by another lock. | |
| 391 | − * By conception this is an helper for other part of the driver | |
| 392 | − * like the indirect buffer or semaphore, which both have their | |
| 393 | − * locking. | |
| 394 | − * | |
| 395 | − * Principe is simple, we keep a list of sub allocation in offset | |
| 396 | − * order (first entry has offset == 0, last entry has the highest | |
| 397 | − * offset). | |
| 398 | − * | |
| 399 | − * When allocating new object we first check if there is room at | |
| 400 | − * the end total_size - (last_object_offset + last_object_size) >= | |
| 401 | − * alloc_size. If so we allocate new object there. | |
| 402 | − * | |
| 403 | − * When there is not enough room at the end, we start waiting for | |
| 404 | − * each sub object until we reach object_offset+object_size >= | |
| 405 | − * alloc_size, this object then become the sub object we return. | |
| 406 | − * | |
| 407 | − * Alignment can't be bigger than page size. | |
| 408 | − * | |
| 409 | − * Hole are not considered for allocation to keep things simple. | |
| 410 | − * Assumption is that there won't be hole (all object on same | |
| 411 | − * alignment). | |
| 412 | − */ | |
| 413 | − | |
| 414 | −struct amdgpu_sa_manager { | |
| 415 | − struct drm_suballoc_manager base; | |
| 416 | − struct amdgpu_bo *bo; | |
| 417 | − uint64_t gpu_addr; | |
| 418 | − void *cpu_ptr; | |
| 419 | −}; | |
| 420 | − | |
| 421 | 391 | /* |
| 422 | 392 | * IRQS. |
| 423 | 393 | */ |
| @@ -446,8 +416,7 @@ struct amdgpu_fpriv { · 22 unmodified lines | ||
| 446 | 416 | struct amdgpu_bo_va *prt_va; |
| 447 | 417 | struct amdgpu_bo_va *csa_va; |
| 448 | 418 | struct amdgpu_bo_va *seq64_va; |
| 449 | − struct mutex bo_list_lock; | |
| 450 | − struct idr bo_list_handles; | |
| 419 | + struct xarray bo_list_handles; | |
| 451 | 420 | struct amdgpu_ctx_mgr ctx_mgr; |
| 452 | 421 | struct amdgpu_userq_mgr userq_mgr; |
| 453 | 422 | |
| @@ -587,8 +556,6 @@ struct amdgpu_asic_funcs { · 133 unmodified lines | ||
| 587 | 556 | /* invalidate hdp read cache */ |
| 588 | 557 | void (*invalidate_hdp)(struct amdgpu_device *adev, |
| 589 | 558 | struct amdgpu_ring *ring); |
| 590 | − /* check if the asic needs a full reset of if soft reset will work */ | |
| 591 | − bool (*need_full_reset)(struct amdgpu_device *adev); | |
| 592 | 559 | /* initialize doorbell layout for specific asic*/ |
| 593 | 560 | void (*init_doorbell_index)(struct amdgpu_device *adev); |
| 594 | 561 | /* PCIe bandwidth usage */ |
| @@ -851,6 +818,7 @@ struct amdgpu_device { · 256 unmodified lines | ||
| 851 | 818 | struct dev_pm_domain vga_pm_domain; |
| 852 | 819 | bool have_disp_power_ref; |
| 853 | 820 | bool have_atomics_support; |
| 821 | + bool is_sw_smu; | |
| 854 | 822 | |
| 855 | 823 | /* BIOS */ |
| 856 | 824 | bool is_atom_fw; |
| @@ -1022,9 +990,6 @@ struct amdgpu_device { · 165 unmodified lines | ||
| 1022 | 990 | /* MCA */ |
| 1023 | 991 | struct amdgpu_mca mca; |
| 1024 | 992 | |
| 1025 | − /* ACA */ | |
| 1026 | − struct amdgpu_aca aca; | |
| 1027 | − | |
| 1028 | 993 | /* CPER */ |
| 1029 | 994 | struct amdgpu_cper cper; |
| 1030 | 995 | |
| @@ -1136,6 +1101,8 @@ struct amdgpu_device { · 105 unmodified lines | ||
| 1136 | 1101 | bool debug_vm_userptr; |
| 1137 | 1102 | bool debug_disable_ce_logs; |
| 1138 | 1103 | bool debug_enable_ce_cs; |
| 1104 | + bool debug_hibernation_thaw_resume_gpu; | |
| 1105 | + bool debug_disable_ip_block_soft_reset; | |
| 1139 | 1106 | |
| 1140 | 1107 | /* Protection for the following isolation structure */ |
| 1141 | 1108 | struct mutex enforce_isolation_mutex; |
| @@ -1356,7 +1323,6 @@ int emu_soc_asic_init(struct amdgpu_device *adev); · 214 unmodified lines | ||
| 1356 | 1323 | #define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l)) |
| 1357 | 1324 | #define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v))) |
| 1358 | 1325 | #define amdgpu_asic_get_config_memsize(adev) (adev)->asic_funcs->get_config_memsize((adev)) |
| 1359 | −#define amdgpu_asic_need_full_reset(adev) (adev)->asic_funcs->need_full_reset((adev)) | |
| 1360 | 1326 | #define amdgpu_asic_init_doorbell_index(adev) (adev)->asic_funcs->init_doorbell_index((adev)) |
| 1361 | 1327 | #define amdgpu_asic_get_pcie_usage(adev, cnt0, cnt1) ((adev)->asic_funcs->get_pcie_usage((adev), (cnt0), (cnt1))) |
| 1362 | 1328 | #define amdgpu_asic_need_reset_on_init(adev) (adev)->asic_funcs->need_reset_on_init((adev)) |
| @@ -1468,6 +1434,8 @@ int amdgpu_enable_vblank_kms(struct drm_crtc *crtc); · 105 unmodified lines | ||
| 1468 | 1434 | void amdgpu_disable_vblank_kms(struct drm_crtc *crtc); |
| 1469 | 1435 | int amdgpu_info_ioctl(struct drm_device *dev, void *data, |
| 1470 | 1436 | struct drm_file *filp); |
| 1437 | +int amdgpu_proc_options_ioctl(struct drm_device *dev, void *data, | |
| 1438 | + struct drm_file *filp); | |
| 1471 | 1439 | |
| 1472 | 1440 | /* |
| 1473 | 1441 | * functions used by amdgpu_encoder.c |
| @@ -1,985 +0,0 @@ | ||
| 1 | −/* | |
| 2 | − * Copyright 2023 Advanced Micro Devices, Inc. | |
| 3 | − * | |
| 4 | − * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | − * copy of this software and associated documentation files (the "Software"), | |
| 6 | − * to deal in the Software without restriction, including without limitation | |
| 7 | − * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | − * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | − * Software is furnished to do so, subject to the following conditions: | |
| 10 | − * | |
| 11 | − * The above copyright notice and this permission notice shall be included in | |
| 12 | − * all copies or substantial portions of the Software. | |
| 13 | − * | |
| 14 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | − * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | − * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | − * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | − * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | − * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | − * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | − * | |
| 22 | − */ | |
| 23 | − | |
| 24 | −#include <linux/list.h> | |
| 25 | −#include "amdgpu.h" | |
| 26 | −#include "amdgpu_aca.h" | |
| 27 | −#include "amdgpu_ras.h" | |
| 28 | − | |
| 29 | −#define ACA_BANK_HWID(type, hwid, mcatype) [ACA_HWIP_TYPE_##type] = {hwid, mcatype} | |
| 30 | − | |
| 31 | −typedef int bank_handler_t(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type, void *data); | |
| 32 | − | |
| 33 | −static struct aca_hwip aca_hwid_mcatypes[ACA_HWIP_TYPE_COUNT] = { | |
| 34 | − ACA_BANK_HWID(SMU, 0x01, 0x01), | |
| 35 | − ACA_BANK_HWID(PCS_XGMI, 0x50, 0x00), | |
| 36 | − ACA_BANK_HWID(UMC, 0x96, 0x00), | |
| 37 | −}; | |
| 38 | − | |
| 39 | −static void aca_banks_init(struct aca_banks *banks) | |
| 40 | −{ | |
| 41 | − if (!banks) | |
| 42 | − return; | |
| 43 | − | |
| 44 | − memset(banks, 0, sizeof(*banks)); | |
| 45 | − INIT_LIST_HEAD(&banks->list); | |
| 46 | −} | |
| 47 | − | |
| 48 | −static int aca_banks_add_bank(struct aca_banks *banks, struct aca_bank *bank) | |
| 49 | −{ | |
| 50 | − struct aca_bank_node *node; | |
| 51 | − | |
| 52 | − if (!bank) | |
| 53 | − return -EINVAL; | |
| 54 | − | |
| 55 | − node = kvzalloc_obj(*node); | |
| 56 | − if (!node) | |
| 57 | − return -ENOMEM; | |
| 58 | − | |
| 59 | − memcpy(&node->bank, bank, sizeof(*bank)); | |
| 60 | − | |
| 61 | − INIT_LIST_HEAD(&node->node); | |
| 62 | − list_add_tail(&node->node, &banks->list); | |
| 63 | − | |
| 64 | − banks->nr_banks++; | |
| 65 | − | |
| 66 | − return 0; | |
| 67 | −} | |
| 68 | − | |
| 69 | −static void aca_banks_release(struct aca_banks *banks) | |
| 70 | −{ | |
| 71 | − struct aca_bank_node *node, *tmp; | |
| 72 | − | |
| 73 | − if (list_empty(&banks->list)) | |
| 74 | − return; | |
| 75 | − | |
| 76 | − list_for_each_entry_safe(node, tmp, &banks->list, node) { | |
| 77 | − list_del(&node->node); | |
| 78 | − kvfree(node); | |
| 79 | − banks->nr_banks--; | |
| 80 | − } | |
| 81 | −} | |
| 82 | − | |
| 83 | −static int aca_smu_get_valid_aca_count(struct amdgpu_device *adev, enum aca_smu_type type, u32 *count) | |
| 84 | −{ | |
| 85 | − struct amdgpu_aca *aca = &adev->aca; | |
| 86 | − const struct aca_smu_funcs *smu_funcs = aca->smu_funcs; | |
| 87 | − | |
| 88 | − if (!count) | |
| 89 | − return -EINVAL; | |
| 90 | − | |
| 91 | − if (!smu_funcs || !smu_funcs->get_valid_aca_count) | |
| 92 | − return -EOPNOTSUPP; | |
| 93 | − | |
| 94 | − return smu_funcs->get_valid_aca_count(adev, type, count); | |
| 95 | −} | |
| 96 | − | |
| 97 | −static struct aca_regs_dump { | |
| 98 | − const char *name; | |
| 99 | − int reg_idx; | |
| 100 | −} aca_regs[] = { | |
| 101 | − {"CONTROL", ACA_REG_IDX_CTL}, | |
| 102 | − {"STATUS", ACA_REG_IDX_STATUS}, | |
| 103 | − {"ADDR", ACA_REG_IDX_ADDR}, | |
| 104 | − {"MISC", ACA_REG_IDX_MISC0}, | |
| 105 | − {"CONFIG", ACA_REG_IDX_CONFIG}, | |
| 106 | − {"IPID", ACA_REG_IDX_IPID}, | |
| 107 | − {"SYND", ACA_REG_IDX_SYND}, | |
| 108 | − {"DESTAT", ACA_REG_IDX_DESTAT}, | |
| 109 | − {"DEADDR", ACA_REG_IDX_DEADDR}, | |
| 110 | − {"CONTROL_MASK", ACA_REG_IDX_CTL_MASK}, | |
| 111 | −}; | |
| 112 | − | |
| 113 | −static void aca_smu_bank_dump(struct amdgpu_device *adev, int idx, int total, struct aca_bank *bank, | |
| 114 | − struct ras_query_context *qctx) | |
| 115 | −{ | |
| 116 | − u64 event_id = qctx ? qctx->evid.event_id : RAS_EVENT_INVALID_ID; | |
| 117 | − int i; | |
| 118 | − | |
| 119 | − if (adev->debug_disable_ce_logs && | |
| 120 | − bank->smu_err_type == ACA_SMU_TYPE_CE && | |
| 121 | − !ACA_BANK_ERR_IS_DEFFERED(bank)) | |
| 122 | − return; | |
| 123 | − | |
| 124 | − RAS_EVENT_LOG(adev, event_id, HW_ERR "Accelerator Check Architecture events logged\n"); | |
| 125 | − /* plus 1 for output format, e.g: ACA[08/08]: xxxx */ | |
| 126 | − for (i = 0; i < ARRAY_SIZE(aca_regs); i++) | |
| 127 | − RAS_EVENT_LOG(adev, event_id, HW_ERR "ACA[%02d/%02d].%s=0x%016llx\n", | |
| 128 | − idx + 1, total, aca_regs[i].name, bank->regs[aca_regs[i].reg_idx]); | |
| 129 | − | |
| 130 | − if (ACA_REG__STATUS__SCRUB(bank->regs[ACA_REG_IDX_STATUS])) | |
| 131 | − RAS_EVENT_LOG(adev, event_id, HW_ERR "hardware error logged by the scrubber\n"); | |
| 132 | −} | |
| 133 | − | |
| 134 | −static bool aca_bank_hwip_is_matched(struct aca_bank *bank, enum aca_hwip_type type) | |
| 135 | −{ | |
| 136 | − | |
| 137 | − struct aca_hwip *hwip; | |
| 138 | − int hwid, mcatype; | |
| 139 | − u64 ipid; | |
| 140 | − | |
| 141 | − if (!bank || type == ACA_HWIP_TYPE_UNKNOW) | |
| 142 | − return false; | |
| 143 | − | |
| 144 | − hwip = &aca_hwid_mcatypes[type]; | |
| 145 | − if (!hwip->hwid) | |
| 146 | − return false; | |
| 147 | − | |
| 148 | − ipid = bank->regs[ACA_REG_IDX_IPID]; | |
| 149 | − hwid = ACA_REG__IPID__HARDWAREID(ipid); | |
| 150 | − mcatype = ACA_REG__IPID__MCATYPE(ipid); | |
| 151 | − | |
| 152 | − return hwip->hwid == hwid && hwip->mcatype == mcatype; | |
| 153 | −} | |
| 154 | − | |
| 155 | −static int aca_smu_get_valid_aca_banks(struct amdgpu_device *adev, enum aca_smu_type type, | |
| 156 | − int start, int count, | |
| 157 | − struct aca_banks *banks, struct ras_query_context *qctx) | |
| 158 | −{ | |
| 159 | − struct amdgpu_aca *aca = &adev->aca; | |
| 160 | − const struct aca_smu_funcs *smu_funcs = aca->smu_funcs; | |
| 161 | − struct aca_bank bank; | |
| 162 | − int i, max_count, ret; | |
| 163 | − | |
| 164 | − if (!count) | |
| 165 | − return 0; | |
| 166 | − | |
| 167 | − if (!smu_funcs || !smu_funcs->get_valid_aca_bank) | |
| 168 | − return -EOPNOTSUPP; | |
| 169 | − | |
| 170 | − switch (type) { | |
| 171 | − case ACA_SMU_TYPE_UE: | |
| 172 | − max_count = smu_funcs->max_ue_bank_count; | |
| 173 | − break; | |
| 174 | − case ACA_SMU_TYPE_CE: | |
| 175 | − max_count = smu_funcs->max_ce_bank_count; | |
| 176 | − break; | |
| 177 | − default: | |
| 178 | − return -EINVAL; | |
| 179 | − } | |
| 180 | − | |
| 181 | − if (start + count > max_count) | |
| 182 | − return -EINVAL; | |
| 183 | − | |
| 184 | − count = min_t(int, count, max_count); | |
| 185 | − for (i = 0; i < count; i++) { | |
| 186 | − memset(&bank, 0, sizeof(bank)); | |
| 187 | − ret = smu_funcs->get_valid_aca_bank(adev, type, start + i, &bank); | |
| 188 | − if (ret) | |
| 189 | − return ret; | |
| 190 | − | |
| 191 | − bank.smu_err_type = type; | |
| 192 | − | |
| 193 | − /* | |
| 194 | − * Poison being consumed when injecting a UE while running background workloads, | |
| 195 | − * which are unexpected. | |
| 196 | − */ | |
| 197 | − if (type == ACA_SMU_TYPE_UE && | |
| 198 | − ACA_REG__STATUS__POISON(bank.regs[ACA_REG_IDX_STATUS]) && | |
| 199 | − !aca_bank_hwip_is_matched(&bank, ACA_HWIP_TYPE_UMC)) | |
| 200 | − continue; | |
| 201 | − | |
| 202 | − aca_smu_bank_dump(adev, i, count, &bank, qctx); | |
| 203 | − | |
| 204 | − ret = aca_banks_add_bank(banks, &bank); | |
| 205 | − if (ret) | |
| 206 | − return ret; | |
| 207 | − } | |
| 208 | − | |
| 209 | − return 0; | |
| 210 | −} | |
| 211 | − | |
| 212 | −static bool aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type) | |
| 213 | −{ | |
| 214 | − const struct aca_bank_ops *bank_ops = handle->bank_ops; | |
| 215 | − | |
| 216 | − /* Parse all deferred errors with UMC aca handle */ | |
| 217 | − if (ACA_BANK_ERR_IS_DEFFERED(bank)) | |
| 218 | − return handle->hwip == ACA_HWIP_TYPE_UMC; | |
| 219 | − | |
| 220 | − if (!aca_bank_hwip_is_matched(bank, handle->hwip)) | |
| 221 | − return false; | |
| 222 | − | |
| 223 | − if (!bank_ops->aca_bank_is_valid) | |
| 224 | − return true; | |
| 225 | − | |
| 226 | − return bank_ops->aca_bank_is_valid(handle, bank, type, handle->data); | |
| 227 | −} | |
| 228 | − | |
| 229 | −static struct aca_bank_error *new_bank_error(struct aca_error *aerr, struct aca_bank_info *info) | |
| 230 | −{ | |
| 231 | − struct aca_bank_error *bank_error; | |
| 232 | − | |
| 233 | − bank_error = kvzalloc_obj(*bank_error); | |
| 234 | − if (!bank_error) | |
| 235 | − return NULL; | |
| 236 | − | |
| 237 | − INIT_LIST_HEAD(&bank_error->node); | |
| 238 | − memcpy(&bank_error->info, info, sizeof(*info)); | |
| 239 | − | |
| 240 | − mutex_lock(&aerr->lock); | |
| 241 | − list_add_tail(&bank_error->node, &aerr->list); | |
| 242 | − aerr->nr_errors++; | |
| 243 | − mutex_unlock(&aerr->lock); | |
| 244 | − | |
| 245 | − return bank_error; | |
| 246 | −} | |
| 247 | − | |
| 248 | −static struct aca_bank_error *find_bank_error(struct aca_error *aerr, struct aca_bank_info *info) | |
| 249 | −{ | |
| 250 | − struct aca_bank_error *bank_error = NULL; | |
| 251 | − struct aca_bank_info *tmp_info; | |
| 252 | − bool found = false; | |
| 253 | − | |
| 254 | − mutex_lock(&aerr->lock); | |
| 255 | − list_for_each_entry(bank_error, &aerr->list, node) { | |
| 256 | − tmp_info = &bank_error->info; | |
| 257 | − if (tmp_info->socket_id == info->socket_id && | |
| 258 | − tmp_info->die_id == info->die_id) { | |
| 259 | − found = true; | |
| 260 | − goto out_unlock; | |
| 261 | − } | |
| 262 | − } | |
| 263 | − | |
| 264 | −out_unlock: | |
| 265 | − mutex_unlock(&aerr->lock); | |
| 266 | − | |
| 267 | − return found ? bank_error : NULL; | |
| 268 | −} | |
| 269 | − | |
| 270 | −static void aca_bank_error_remove(struct aca_error *aerr, struct aca_bank_error *bank_error) | |
| 271 | −{ | |
| 272 | − if (!aerr || !bank_error) | |
| 273 | − return; | |
| 274 | − | |
| 275 | − list_del(&bank_error->node); | |
| 276 | − aerr->nr_errors--; | |
| 277 | − | |
| 278 | − kvfree(bank_error); | |
| 279 | −} | |
| 280 | − | |
| 281 | −static struct aca_bank_error *get_bank_error(struct aca_error *aerr, struct aca_bank_info *info) | |
| 282 | −{ | |
| 283 | − struct aca_bank_error *bank_error; | |
| 284 | − | |
| 285 | − if (!aerr || !info) | |
| 286 | − return NULL; | |
| 287 | − | |
| 288 | − bank_error = find_bank_error(aerr, info); | |
| 289 | − if (bank_error) | |
| 290 | − return bank_error; | |
| 291 | − | |
| 292 | − return new_bank_error(aerr, info); | |
| 293 | −} | |
| 294 | − | |
| 295 | −int aca_error_cache_log_bank_error(struct aca_handle *handle, struct aca_bank_info *info, | |
| 296 | − enum aca_error_type type, u64 count) | |
| 297 | −{ | |
| 298 | − struct aca_error_cache *error_cache = &handle->error_cache; | |
| 299 | − struct aca_bank_error *bank_error; | |
| 300 | − struct aca_error *aerr; | |
| 301 | − | |
| 302 | − if (!handle || !info || type >= ACA_ERROR_TYPE_COUNT) | |
| 303 | − return -EINVAL; | |
| 304 | − | |
| 305 | − if (!count) | |
| 306 | − return 0; | |
| 307 | − | |
| 308 | − aerr = &error_cache->errors[type]; | |
| 309 | − bank_error = get_bank_error(aerr, info); | |
| 310 | − if (!bank_error) | |
| 311 | − return -ENOMEM; | |
| 312 | − | |
| 313 | − bank_error->count += count; | |
| 314 | − | |
| 315 | − return 0; | |
| 316 | −} | |
| 317 | − | |
| 318 | −static int aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type) | |
| 319 | −{ | |
| 320 | − const struct aca_bank_ops *bank_ops = handle->bank_ops; | |
| 321 | − | |
| 322 | − if (!bank) | |
| 323 | − return -EINVAL; | |
| 324 | − | |
| 325 | − if (!bank_ops->aca_bank_parser) | |
| 326 | − return -EOPNOTSUPP; | |
| 327 | − | |
| 328 | − return bank_ops->aca_bank_parser(handle, bank, type, | |
| 329 | − handle->data); | |
| 330 | −} | |
| 331 | − | |
| 332 | −static int handler_aca_log_bank_error(struct aca_handle *handle, struct aca_bank *bank, | |
| 333 | − enum aca_smu_type type, void *data) | |
| 334 | −{ | |
| 335 | − int ret; | |
| 336 | − | |
| 337 | − ret = aca_bank_parser(handle, bank, type); | |
| 338 | − if (ret) | |
| 339 | − return ret; | |
| 340 | − | |
| 341 | − return 0; | |
| 342 | −} | |
| 343 | − | |
| 344 | −static int aca_dispatch_bank(struct aca_handle_manager *mgr, struct aca_bank *bank, | |
| 345 | − enum aca_smu_type type, bank_handler_t handler, void *data) | |
| 346 | −{ | |
| 347 | − struct aca_handle *handle; | |
| 348 | − int ret; | |
| 349 | − | |
| 350 | − if (list_empty(&mgr->list)) | |
| 351 | − return 0; | |
| 352 | − | |
| 353 | − list_for_each_entry(handle, &mgr->list, node) { | |
| 354 | − if (!aca_bank_is_valid(handle, bank, type)) | |
| 355 | − continue; | |
| 356 | − | |
| 357 | − ret = handler(handle, bank, type, data); | |
| 358 | − if (ret) | |
| 359 | − return ret; | |
| 360 | − } | |
| 361 | − | |
| 362 | − return 0; | |
| 363 | −} | |
| 364 | − | |
| 365 | −static int aca_dispatch_banks(struct aca_handle_manager *mgr, struct aca_banks *banks, | |
| 366 | − enum aca_smu_type type, bank_handler_t handler, void *data) | |
| 367 | −{ | |
| 368 | − struct aca_bank_node *node; | |
| 369 | − struct aca_bank *bank; | |
| 370 | − int ret; | |
| 371 | − | |
| 372 | − if (!mgr || !banks) | |
| 373 | − return -EINVAL; | |
| 374 | − | |
| 375 | − /* pre check to avoid unnecessary operations */ | |
| 376 | − if (list_empty(&mgr->list) || list_empty(&banks->list)) | |
| 377 | − return 0; | |
| 378 | − | |
| 379 | − list_for_each_entry(node, &banks->list, node) { | |
| 380 | − bank = &node->bank; | |
| 381 | − | |
| 382 | − ret = aca_dispatch_bank(mgr, bank, type, handler, data); | |
| 383 | − if (ret) | |
| 384 | − return ret; | |
| 385 | − } | |
| 386 | − | |
| 387 | − return 0; | |
| 388 | −} | |
| 389 | − | |
| 390 | −static bool aca_bank_should_update(struct amdgpu_device *adev, enum aca_smu_type type) | |
| 391 | −{ | |
| 392 | − struct amdgpu_aca *aca = &adev->aca; | |
| 393 | − bool ret = true; | |
| 394 | − | |
| 395 | − /* | |
| 396 | − * Because the UE Valid MCA count will only be cleared after reset, | |
| 397 | − * in order to avoid repeated counting of the error count, | |
| 398 | − * the aca bank is only updated once during the gpu recovery stage. | |
| 399 | − */ | |
| 400 | − if (type == ACA_SMU_TYPE_UE) { | |
| 401 | − if (amdgpu_ras_intr_triggered()) | |
| 402 | − ret = atomic_cmpxchg(&aca->ue_update_flag, 0, 1) == 0; | |
| 403 | − else | |
| 404 | − atomic_set(&aca->ue_update_flag, 0); | |
| 405 | − } | |
| 406 | − | |
| 407 | − return ret; | |
| 408 | −} | |
| 409 | − | |
| 410 | −static void aca_banks_generate_cper(struct amdgpu_device *adev, | |
| 411 | − enum aca_smu_type type, | |
| 412 | − struct aca_banks *banks, | |
| 413 | − int count) | |
| 414 | −{ | |
| 415 | − struct aca_bank_node *node; | |
| 416 | − struct aca_bank *bank; | |
| 417 | − int r; | |
| 418 | − | |
| 419 | − if (!adev->cper.enabled) | |
| 420 | − return; | |
| 421 | − | |
| 422 | − if (!banks || !count) { | |
| 423 | − dev_warn(adev->dev, "fail to generate cper records\n"); | |
| 424 | − return; | |
| 425 | − } | |
| 426 | − | |
| 427 | − /* UEs must be encoded into separate CPER entries */ | |
| 428 | − if (type == ACA_SMU_TYPE_UE) { | |
| 429 | − struct aca_banks de_banks; | |
| 430 | − | |
| 431 | − aca_banks_init(&de_banks); | |
| 432 | − list_for_each_entry(node, &banks->list, node) { | |
| 433 | − bank = &node->bank; | |
| 434 | − if (bank->aca_err_type == ACA_ERROR_TYPE_DEFERRED) { | |
| 435 | − r = aca_banks_add_bank(&de_banks, bank); | |
| 436 | − if (r) | |
| 437 | − dev_warn(adev->dev, "fail to add de banks, ret = %d\n", r); | |
| 438 | − } else { | |
| 439 | − if (amdgpu_cper_generate_ue_record(adev, bank)) | |
| 440 | − dev_warn(adev->dev, "fail to generate ue cper records\n"); | |
| 441 | − } | |
| 442 | − } | |
| 443 | − | |
| 444 | − if (!list_empty(&de_banks.list)) { | |
| 445 | − if (amdgpu_cper_generate_ce_records(adev, &de_banks, de_banks.nr_banks)) | |
| 446 | − dev_warn(adev->dev, "fail to generate de cper records\n"); | |
| 447 | − } | |
| 448 | − | |
| 449 | − aca_banks_release(&de_banks); | |
| 450 | − } else { | |
| 451 | − /* | |
| 452 | − * SMU_TYPE_CE banks are combined into 1 CPER entries, | |
| 453 | − * they could be CEs or DEs or both | |
| 454 | − */ | |
| 455 | − if (amdgpu_cper_generate_ce_records(adev, banks, count)) | |
| 456 | − dev_warn(adev->dev, "fail to generate ce cper records\n"); | |
| 457 | − } | |
| 458 | −} | |
| 459 | − | |
| 460 | −static int aca_banks_update(struct amdgpu_device *adev, enum aca_smu_type type, | |
| 461 | − bank_handler_t handler, struct ras_query_context *qctx, void *data) | |
| 462 | −{ | |
| 463 | − struct amdgpu_aca *aca = &adev->aca; | |
| 464 | − struct aca_banks banks; | |
| 465 | − u32 count = 0; | |
| 466 | − int ret; | |
| 467 | − | |
| 468 | − if (list_empty(&aca->mgr.list)) | |
| 469 | − return 0; | |
| 470 | − | |
| 471 | − if (!aca_bank_should_update(adev, type)) | |
| 472 | − return 0; | |
| 473 | − | |
| 474 | − ret = aca_smu_get_valid_aca_count(adev, type, &count); | |
| 475 | − if (ret) | |
| 476 | − return ret; | |
| 477 | − | |
| 478 | − if (!count) | |
| 479 | − return 0; | |
| 480 | − | |
| 481 | − aca_banks_init(&banks); | |
| 482 | − | |
| 483 | − ret = aca_smu_get_valid_aca_banks(adev, type, 0, count, &banks, qctx); | |
| 484 | − if (ret) | |
| 485 | − goto err_release_banks; | |
| 486 | − | |
| 487 | − if (list_empty(&banks.list)) { | |
| 488 | − ret = 0; | |
| 489 | − goto err_release_banks; | |
| 490 | − } | |
| 491 | − | |
| 492 | − ret = aca_dispatch_banks(&aca->mgr, &banks, type, | |
| 493 | − handler, data); | |
| 494 | − if (ret) | |
| 495 | − goto err_release_banks; | |
| 496 | − | |
| 497 | − aca_banks_generate_cper(adev, type, &banks, count); | |
| 498 | − | |
| 499 | −err_release_banks: | |
| 500 | − aca_banks_release(&banks); | |
| 501 | − | |
| 502 | − return ret; | |
| 503 | −} | |
| 504 | − | |
| 505 | −static int aca_log_aca_error_data(struct aca_bank_error *bank_error, enum aca_error_type type, struct ras_err_data *err_data) | |
| 506 | −{ | |
| 507 | − struct aca_bank_info *info; | |
| 508 | − struct amdgpu_smuio_mcm_config_info mcm_info; | |
| 509 | − u64 count; | |
| 510 | − | |
| 511 | − if (type >= ACA_ERROR_TYPE_COUNT) | |
| 512 | − return -EINVAL; | |
| 513 | − | |
| 514 | − count = bank_error->count; | |
| 515 | − if (!count) | |
| 516 | − return 0; | |
| 517 | − | |
| 518 | − info = &bank_error->info; | |
| 519 | − mcm_info.die_id = info->die_id; | |
| 520 | − mcm_info.socket_id = info->socket_id; | |
| 521 | − | |
| 522 | − switch (type) { | |
| 523 | − case ACA_ERROR_TYPE_UE: | |
| 524 | − amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, count); | |
| 525 | − break; | |
| 526 | − case ACA_ERROR_TYPE_CE: | |
| 527 | − amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, count); | |
| 528 | − break; | |
| 529 | − case ACA_ERROR_TYPE_DEFERRED: | |
| 530 | − amdgpu_ras_error_statistic_de_count(err_data, &mcm_info, count); | |
| 531 | − break; | |
| 532 | − default: | |
| 533 | − break; | |
| 534 | − } | |
| 535 | − | |
| 536 | − return 0; | |
| 537 | −} | |
| 538 | − | |
| 539 | −static int aca_log_aca_error(struct aca_handle *handle, enum aca_error_type type, struct ras_err_data *err_data) | |
| 540 | −{ | |
| 541 | − struct aca_error_cache *error_cache = &handle->error_cache; | |
| 542 | − struct aca_error *aerr = &error_cache->errors[type]; | |
| 543 | − struct aca_bank_error *bank_error, *tmp; | |
| 544 | − | |
| 545 | − mutex_lock(&aerr->lock); | |
| 546 | − | |
| 547 | − if (list_empty(&aerr->list)) | |
| 548 | − goto out_unlock; | |
| 549 | − | |
| 550 | − list_for_each_entry_safe(bank_error, tmp, &aerr->list, node) { | |
| 551 | − aca_log_aca_error_data(bank_error, type, err_data); | |
| 552 | − aca_bank_error_remove(aerr, bank_error); | |
| 553 | − } | |
| 554 | − | |
| 555 | −out_unlock: | |
| 556 | − mutex_unlock(&aerr->lock); | |
| 557 | − | |
| 558 | − return 0; | |
| 559 | −} | |
| 560 | − | |
| 561 | −static int __aca_get_error_data(struct amdgpu_device *adev, struct aca_handle *handle, enum aca_error_type type, | |
| 562 | − struct ras_err_data *err_data, struct ras_query_context *qctx) | |
| 563 | −{ | |
| 564 | − enum aca_smu_type smu_type; | |
| 565 | − int ret; | |
| 566 | − | |
| 567 | − switch (type) { | |
| 568 | − case ACA_ERROR_TYPE_UE: | |
| 569 | − smu_type = ACA_SMU_TYPE_UE; | |
| 570 | − break; | |
| 571 | − case ACA_ERROR_TYPE_CE: | |
| 572 | − case ACA_ERROR_TYPE_DEFERRED: | |
| 573 | − smu_type = ACA_SMU_TYPE_CE; | |
| 574 | − break; | |
| 575 | − default: | |
| 576 | − return -EINVAL; | |
| 577 | − } | |
| 578 | − | |
| 579 | − /* update aca bank to aca source error_cache first */ | |
| 580 | − ret = aca_banks_update(adev, smu_type, handler_aca_log_bank_error, qctx, NULL); | |
| 581 | − if (ret) | |
| 582 | − return ret; | |
| 583 | − | |
| 584 | − /* DEs may contain in CEs or UEs */ | |
| 585 | − if (type != ACA_ERROR_TYPE_DEFERRED) | |
| 586 | − aca_log_aca_error(handle, ACA_ERROR_TYPE_DEFERRED, err_data); | |
| 587 | − | |
| 588 | − return aca_log_aca_error(handle, type, err_data); | |
| 589 | −} | |
| 590 | − | |
| 591 | −static bool aca_handle_is_valid(struct aca_handle *handle) | |
| 592 | −{ | |
| 593 | − if (!handle->mask || !list_empty(&handle->node)) | |
| 594 | − return false; | |
| 595 | − | |
| 596 | − return true; | |
| 597 | −} | |
| 598 | − | |
| 599 | −int amdgpu_aca_get_error_data(struct amdgpu_device *adev, struct aca_handle *handle, | |
| 600 | − enum aca_error_type type, struct ras_err_data *err_data, | |
| 601 | − struct ras_query_context *qctx) | |
| 602 | −{ | |
| 603 | − if (!handle || !err_data) | |
| 604 | − return -EINVAL; | |
| 605 | − | |
| 606 | − if (aca_handle_is_valid(handle)) | |
| 607 | − return -EOPNOTSUPP; | |
| 608 | − | |
| 609 | − if ((type < 0) || (!(BIT(type) & handle->mask))) | |
| 610 | − return 0; | |
| 611 | − | |
| 612 | − return __aca_get_error_data(adev, handle, type, err_data, qctx); | |
| 613 | −} | |
| 614 | − | |
| 615 | −static void aca_error_init(struct aca_error *aerr, enum aca_error_type type) | |
| 616 | −{ | |
| 617 | − mutex_init(&aerr->lock); | |
| 618 | − INIT_LIST_HEAD(&aerr->list); | |
| 619 | − aerr->type = type; | |
| 620 | − aerr->nr_errors = 0; | |
| 621 | −} | |
| 622 | − | |
| 623 | −static void aca_init_error_cache(struct aca_handle *handle) | |
| 624 | −{ | |
| 625 | − struct aca_error_cache *error_cache = &handle->error_cache; | |
| 626 | − int type; | |
| 627 | − | |
| 628 | − for (type = ACA_ERROR_TYPE_UE; type < ACA_ERROR_TYPE_COUNT; type++) | |
| 629 | − aca_error_init(&error_cache->errors[type], type); | |
| 630 | −} | |
| 631 | − | |
| 632 | −static void aca_error_fini(struct aca_error *aerr) | |
| 633 | −{ | |
| 634 | − struct aca_bank_error *bank_error, *tmp; | |
| 635 | − | |
| 636 | − mutex_lock(&aerr->lock); | |
| 637 | − if (list_empty(&aerr->list)) | |
| 638 | − goto out_unlock; | |
| 639 | − | |
| 640 | − list_for_each_entry_safe(bank_error, tmp, &aerr->list, node) | |
| 641 | − aca_bank_error_remove(aerr, bank_error); | |
| 642 | − | |
| 643 | −out_unlock: | |
| 644 | − mutex_unlock(&aerr->lock); | |
| 645 | − mutex_destroy(&aerr->lock); | |
| 646 | −} | |
| 647 | − | |
| 648 | −static void aca_fini_error_cache(struct aca_handle *handle) | |
| 649 | −{ | |
| 650 | − struct aca_error_cache *error_cache = &handle->error_cache; | |
| 651 | − int type; | |
| 652 | − | |
| 653 | − for (type = ACA_ERROR_TYPE_UE; type < ACA_ERROR_TYPE_COUNT; type++) | |
| 654 | − aca_error_fini(&error_cache->errors[type]); | |
| 655 | −} | |
| 656 | − | |
| 657 | −static int add_aca_handle(struct amdgpu_device *adev, struct aca_handle_manager *mgr, struct aca_handle *handle, | |
| 658 | − const char *name, const struct aca_info *ras_info, void *data) | |
| 659 | −{ | |
| 660 | − memset(handle, 0, sizeof(*handle)); | |
| 661 | − | |
| 662 | − handle->adev = adev; | |
| 663 | − handle->mgr = mgr; | |
| 664 | − handle->name = name; | |
| 665 | − handle->hwip = ras_info->hwip; | |
| 666 | − handle->mask = ras_info->mask; | |
| 667 | − handle->bank_ops = ras_info->bank_ops; | |
| 668 | − handle->data = data; | |
| 669 | − aca_init_error_cache(handle); | |
| 670 | − | |
| 671 | − INIT_LIST_HEAD(&handle->node); | |
| 672 | − list_add_tail(&handle->node, &mgr->list); | |
| 673 | − mgr->nr_handles++; | |
| 674 | − | |
| 675 | − return 0; | |
| 676 | −} | |
| 677 | − | |
| 678 | −static ssize_t aca_sysfs_read(struct device *dev, | |
| 679 | − struct device_attribute *attr, char *buf) | |
| 680 | −{ | |
| 681 | − struct aca_handle *handle = container_of(attr, struct aca_handle, aca_attr); | |
| 682 | − | |
| 683 | − /* NOTE: the aca cache will be auto cleared once read, | |
| 684 | − * So the driver should unify the query entry point, forward request to ras query interface directly */ | |
| 685 | − return amdgpu_ras_aca_sysfs_read(dev, attr, handle, buf, handle->data); | |
| 686 | −} | |
| 687 | − | |
| 688 | −static int add_aca_sysfs(struct amdgpu_device *adev, struct aca_handle *handle) | |
| 689 | −{ | |
| 690 | − struct device_attribute *aca_attr = &handle->aca_attr; | |
| 691 | − | |
| 692 | − snprintf(handle->attr_name, sizeof(handle->attr_name) - 1, "aca_%s", handle->name); | |
| 693 | − aca_attr->show = aca_sysfs_read; | |
| 694 | − aca_attr->attr.name = handle->attr_name; | |
| 695 | − aca_attr->attr.mode = S_IRUGO; | |
| 696 | − sysfs_attr_init(&aca_attr->attr); | |
| 697 | − | |
| 698 | − return sysfs_add_file_to_group(&adev->dev->kobj, | |
| 699 | − &aca_attr->attr, | |
| 700 | − "ras"); | |
| 701 | −} | |
| 702 | − | |
| 703 | −int amdgpu_aca_add_handle(struct amdgpu_device *adev, struct aca_handle *handle, | |
| 704 | − const char *name, const struct aca_info *ras_info, void *data) | |
| 705 | −{ | |
| 706 | − struct amdgpu_aca *aca = &adev->aca; | |
| 707 | − int ret; | |
| 708 | − | |
| 709 | − if (!amdgpu_aca_is_enabled(adev)) | |
| 710 | − return 0; | |
| 711 | − | |
| 712 | − ret = add_aca_handle(adev, &aca->mgr, handle, name, ras_info, data); | |
| 713 | − if (ret) | |
| 714 | − return ret; | |
| 715 | − | |
| 716 | − return add_aca_sysfs(adev, handle); | |
| 717 | −} | |
| 718 | − | |
| 719 | −static void remove_aca_handle(struct aca_handle *handle) | |
| 720 | −{ | |
| 721 | − struct aca_handle_manager *mgr = handle->mgr; | |
| 722 | − | |
| 723 | − aca_fini_error_cache(handle); | |
| 724 | − list_del(&handle->node); | |
| 725 | − mgr->nr_handles--; | |
| 726 | −} | |
| 727 | − | |
| 728 | −static void remove_aca_sysfs(struct aca_handle *handle) | |
| 729 | −{ | |
| 730 | − struct amdgpu_device *adev = handle->adev; | |
| 731 | − struct device_attribute *aca_attr = &handle->aca_attr; | |
| 732 | − | |
| 733 | − if (adev->dev->kobj.sd) | |
| 734 | − sysfs_remove_file_from_group(&adev->dev->kobj, | |
| 735 | − &aca_attr->attr, | |
| 736 | − "ras"); | |
| 737 | −} | |
| 738 | − | |
| 739 | −void amdgpu_aca_remove_handle(struct aca_handle *handle) | |
| 740 | −{ | |
| 741 | − if (!handle || list_empty(&handle->node)) | |
| 742 | − return; | |
| 743 | − | |
| 744 | − remove_aca_sysfs(handle); | |
| 745 | − remove_aca_handle(handle); | |
| 746 | −} | |
| 747 | − | |
| 748 | −static int aca_manager_init(struct aca_handle_manager *mgr) | |
| 749 | −{ | |
| 750 | − INIT_LIST_HEAD(&mgr->list); | |
| 751 | − mgr->nr_handles = 0; | |
| 752 | − | |
| 753 | − return 0; | |
| 754 | −} | |
| 755 | − | |
| 756 | −static void aca_manager_fini(struct aca_handle_manager *mgr) | |
| 757 | −{ | |
| 758 | − struct aca_handle *handle, *tmp; | |
| 759 | − | |
| 760 | − if (list_empty(&mgr->list)) | |
| 761 | − return; | |
| 762 | − | |
| 763 | − list_for_each_entry_safe(handle, tmp, &mgr->list, node) | |
| 764 | − amdgpu_aca_remove_handle(handle); | |
| 765 | −} | |
| 766 | − | |
| 767 | −bool amdgpu_aca_is_enabled(struct amdgpu_device *adev) | |
| 768 | −{ | |
| 769 | − return (adev->aca.is_enabled || | |
| 770 | − adev->debug_enable_ras_aca); | |
| 771 | −} | |
| 772 | − | |
| 773 | −int amdgpu_aca_init(struct amdgpu_device *adev) | |
| 774 | −{ | |
| 775 | − struct amdgpu_aca *aca = &adev->aca; | |
| 776 | − int ret; | |
| 777 | − | |
| 778 | − atomic_set(&aca->ue_update_flag, 0); | |
| 779 | − | |
| 780 | − ret = aca_manager_init(&aca->mgr); | |
| 781 | − if (ret) | |
| 782 | − return ret; | |
| 783 | − | |
| 784 | − return 0; | |
| 785 | −} | |
| 786 | − | |
| 787 | −void amdgpu_aca_fini(struct amdgpu_device *adev) | |
| 788 | −{ | |
| 789 | − struct amdgpu_aca *aca = &adev->aca; | |
| 790 | − | |
| 791 | − aca_manager_fini(&aca->mgr); | |
| 792 | − | |
| 793 | − atomic_set(&aca->ue_update_flag, 0); | |
| 794 | −} | |
| 795 | − | |
| 796 | −int amdgpu_aca_reset(struct amdgpu_device *adev) | |
| 797 | −{ | |
| 798 | − struct amdgpu_aca *aca = &adev->aca; | |
| 799 | − | |
| 800 | − atomic_set(&aca->ue_update_flag, 0); | |
| 801 | − | |
| 802 | − return 0; | |
| 803 | −} | |
| 804 | − | |
| 805 | −void amdgpu_aca_set_smu_funcs(struct amdgpu_device *adev, const struct aca_smu_funcs *smu_funcs) | |
| 806 | −{ | |
| 807 | − struct amdgpu_aca *aca = &adev->aca; | |
| 808 | − | |
| 809 | − WARN_ON(aca->smu_funcs); | |
| 810 | − aca->smu_funcs = smu_funcs; | |
| 811 | −} | |
| 812 | − | |
| 813 | −int aca_bank_info_decode(struct aca_bank *bank, struct aca_bank_info *info) | |
| 814 | −{ | |
| 815 | − u64 ipid; | |
| 816 | − u32 instidhi, instidlo; | |
| 817 | − | |
| 818 | − if (!bank || !info) | |
| 819 | − return -EINVAL; | |
| 820 | − | |
| 821 | − ipid = bank->regs[ACA_REG_IDX_IPID]; | |
| 822 | − info->hwid = ACA_REG__IPID__HARDWAREID(ipid); | |
| 823 | − info->mcatype = ACA_REG__IPID__MCATYPE(ipid); | |
| 824 | − /* | |
| 825 | − * Unfied DieID Format: SAASS. A:AID, S:Socket. | |
| 826 | − * Unfied DieID[4:4] = InstanceId[0:0] | |
| 827 | − * Unfied DieID[0:3] = InstanceIdHi[0:3] | |
| 828 | − */ | |
| 829 | − instidhi = ACA_REG__IPID__INSTANCEIDHI(ipid); | |
| 830 | − instidlo = ACA_REG__IPID__INSTANCEIDLO(ipid); | |
| 831 | − info->die_id = ((instidhi >> 2) & 0x03); | |
| 832 | − info->socket_id = ((instidlo & 0x1) << 2) | (instidhi & 0x03); | |
| 833 | − | |
| 834 | − return 0; | |
| 835 | −} | |
| 836 | − | |
| 837 | −static int aca_bank_get_error_code(struct amdgpu_device *adev, struct aca_bank *bank) | |
| 838 | −{ | |
| 839 | − struct amdgpu_aca *aca = &adev->aca; | |
| 840 | − const struct aca_smu_funcs *smu_funcs = aca->smu_funcs; | |
| 841 | − | |
| 842 | − if (!smu_funcs || !smu_funcs->parse_error_code) | |
| 843 | − return -EOPNOTSUPP; | |
| 844 | − | |
| 845 | − return smu_funcs->parse_error_code(adev, bank); | |
| 846 | −} | |
| 847 | − | |
| 848 | −int aca_bank_check_error_codes(struct amdgpu_device *adev, struct aca_bank *bank, int *err_codes, int size) | |
| 849 | −{ | |
| 850 | − int i, error_code; | |
| 851 | − | |
| 852 | − if (!bank || !err_codes) | |
| 853 | − return -EINVAL; | |
| 854 | − | |
| 855 | − error_code = aca_bank_get_error_code(adev, bank); | |
| 856 | − if (error_code < 0) | |
| 857 | − return error_code; | |
| 858 | − | |
| 859 | − for (i = 0; i < size; i++) { | |
| 860 | − if (err_codes[i] == error_code) | |
| 861 | − return 0; | |
| 862 | − } | |
| 863 | − | |
| 864 | − return -EINVAL; | |
| 865 | −} | |
| 866 | − | |
| 867 | −int amdgpu_aca_smu_set_debug_mode(struct amdgpu_device *adev, bool en) | |
| 868 | −{ | |
| 869 | − struct amdgpu_aca *aca = &adev->aca; | |
| 870 | − const struct aca_smu_funcs *smu_funcs = aca->smu_funcs; | |
| 871 | − | |
| 872 | − if (!smu_funcs || !smu_funcs->set_debug_mode) | |
| 873 | − return -EOPNOTSUPP; | |
| 874 | − | |
| 875 | − return smu_funcs->set_debug_mode(adev, en); | |
| 876 | −} | |
| 877 | − | |
| 878 | −#if defined(CONFIG_DEBUG_FS) | |
| 879 | −static int amdgpu_aca_smu_debug_mode_set(void *data, u64 val) | |
| 880 | −{ | |
| 881 | − struct amdgpu_device *adev = (struct amdgpu_device *)data; | |
| 882 | − int ret; | |
| 883 | − | |
| 884 | − ret = amdgpu_ras_set_aca_debug_mode(adev, val ? true : false); | |
| 885 | − if (ret) | |
| 886 | − return ret; | |
| 887 | − | |
| 888 | − dev_info(adev->dev, "amdgpu set smu aca debug mode %s success\n", val ? "on" : "off"); | |
| 889 | − | |
| 890 | − return 0; | |
| 891 | −} | |
| 892 | − | |
| 893 | −static void aca_dump_entry(struct seq_file *m, struct aca_bank *bank, enum aca_smu_type type, int idx) | |
| 894 | −{ | |
| 895 | − struct aca_bank_info info; | |
| 896 | − int i, ret; | |
| 897 | − | |
| 898 | − ret = aca_bank_info_decode(bank, &info); | |
| 899 | − if (ret) | |
| 900 | − return; | |
| 901 | − | |
| 902 | − seq_printf(m, "aca entry[%d].type: %s\n", idx, type == ACA_SMU_TYPE_UE ? "UE" : "CE"); | |
| 903 | − seq_printf(m, "aca entry[%d].info: socketid:%d aid:%d hwid:0x%03x mcatype:0x%04x\n", | |
| 904 | − idx, info.socket_id, info.die_id, info.hwid, info.mcatype); | |
| 905 | − | |
| 906 | − for (i = 0; i < ARRAY_SIZE(aca_regs); i++) | |
| 907 | − seq_printf(m, "aca entry[%d].regs[%d]: 0x%016llx\n", idx, aca_regs[i].reg_idx, bank->regs[aca_regs[i].reg_idx]); | |
| 908 | −} | |
| 909 | − | |
| 910 | −struct aca_dump_context { | |
| 911 | − struct seq_file *m; | |
| 912 | − int idx; | |
| 913 | −}; | |
| 914 | − | |
| 915 | −static int handler_aca_bank_dump(struct aca_handle *handle, struct aca_bank *bank, | |
| 916 | − enum aca_smu_type type, void *data) | |
| 917 | −{ | |
| 918 | − struct aca_dump_context *ctx = (struct aca_dump_context *)data; | |
| 919 | − | |
| 920 | − aca_dump_entry(ctx->m, bank, type, ctx->idx++); | |
| 921 | − | |
| 922 | − return handler_aca_log_bank_error(handle, bank, type, NULL); | |
| 923 | −} | |
| 924 | − | |
| 925 | −static int aca_dump_show(struct seq_file *m, enum aca_smu_type type) | |
| 926 | −{ | |
| 927 | − struct amdgpu_device *adev = (struct amdgpu_device *)m->private; | |
| 928 | − struct aca_dump_context context = { | |
| 929 | − .m = m, | |
| 930 | − .idx = 0, | |
| 931 | − }; | |
| 932 | − | |
| 933 | − return aca_banks_update(adev, type, handler_aca_bank_dump, NULL, (void *)&context); | |
| 934 | −} | |
| 935 | − | |
| 936 | −static int aca_dump_ce_show(struct seq_file *m, void *unused) | |
| 937 | −{ | |
| 938 | − return aca_dump_show(m, ACA_SMU_TYPE_CE); | |
| 939 | −} | |
| 940 | − | |
| 941 | −static int aca_dump_ce_open(struct inode *inode, struct file *file) | |
| 942 | −{ | |
| 943 | − return single_open(file, aca_dump_ce_show, inode->i_private); | |
| 944 | −} | |
| 945 | − | |
| 946 | −static const struct file_operations aca_ce_dump_debug_fops = { | |
| 947 | − .owner = THIS_MODULE, | |
| 948 | − .open = aca_dump_ce_open, | |
| 949 | − .read = seq_read, | |
| 950 | − .llseek = seq_lseek, | |
| 951 | − .release = single_release, | |
| 952 | −}; | |
| 953 | − | |
| 954 | −static int aca_dump_ue_show(struct seq_file *m, void *unused) | |
| 955 | −{ | |
| 956 | − return aca_dump_show(m, ACA_SMU_TYPE_UE); | |
| 957 | −} | |
| 958 | − | |
| 959 | −static int aca_dump_ue_open(struct inode *inode, struct file *file) | |
| 960 | −{ | |
| 961 | − return single_open(file, aca_dump_ue_show, inode->i_private); | |
| 962 | −} | |
| 963 | − | |
| 964 | −static const struct file_operations aca_ue_dump_debug_fops = { | |
| 965 | − .owner = THIS_MODULE, | |
| 966 | − .open = aca_dump_ue_open, | |
| 967 | − .read = seq_read, | |
| 968 | − .llseek = seq_lseek, | |
| 969 | − .release = single_release, | |
| 970 | −}; | |
| 971 | − | |
| 972 | −DEFINE_DEBUGFS_ATTRIBUTE(aca_debug_mode_fops, NULL, amdgpu_aca_smu_debug_mode_set, "%llu\n"); | |
| 973 | −#endif | |
| 974 | − | |
| 975 | −void amdgpu_aca_smu_debugfs_init(struct amdgpu_device *adev, struct dentry *root) | |
| 976 | −{ | |
| 977 | −#if defined(CONFIG_DEBUG_FS) | |
| 978 | − if (!root) | |
| 979 | − return; | |
| 980 | − | |
| 981 | − debugfs_create_file("aca_debug_mode", 0200, root, adev, &aca_debug_mode_fops); | |
| 982 | − debugfs_create_file("aca_ue_dump", 0400, root, adev, &aca_ue_dump_debug_fops); | |
| 983 | − debugfs_create_file("aca_ce_dump", 0400, root, adev, &aca_ce_dump_debug_fops); | |
| 984 | −#endif | |
| 985 | −} | |
| @@ -1,232 +0,0 @@ | ||
| 1 | −/* | |
| 2 | − * Copyright 2023 Advanced Micro Devices, Inc. | |
| 3 | − * | |
| 4 | − * Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 | − * copy of this software and associated documentation files (the "Software"), | |
| 6 | − * to deal in the Software without restriction, including without limitation | |
| 7 | − * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 8 | − * and/or sell copies of the Software, and to permit persons to whom the | |
| 9 | − * Software is furnished to do so, subject to the following conditions: | |
| 10 | − * | |
| 11 | − * The above copyright notice and this permission notice shall be included in | |
| 12 | − * all copies or substantial portions of the Software. | |
| 13 | − * | |
| 14 | − * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 15 | − * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 16 | − * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 17 | − * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 18 | − * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 19 | − * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 20 | − * OTHER DEALINGS IN THE SOFTWARE. | |
| 21 | − * | |
| 22 | − */ | |
| 23 | − | |
| 24 | −#ifndef __AMDGPU_ACA_H__ | |
| 25 | −#define __AMDGPU_ACA_H__ | |
| 26 | − | |
| 27 | −#include <linux/list.h> | |
| 28 | − | |
| 29 | −struct ras_err_data; | |
| 30 | −struct ras_query_context; | |
| 31 | − | |
| 32 | −#define ACA_MAX_REGS_COUNT (16) | |
| 33 | − | |
| 34 | −#define ACA_REG_FIELD(x, h, l) (((x) & GENMASK_ULL(h, l)) >> l) | |
| 35 | −#define ACA_REG__STATUS__VAL(x) ACA_REG_FIELD(x, 63, 63) | |
| 36 | −#define ACA_REG__STATUS__OVERFLOW(x) ACA_REG_FIELD(x, 62, 62) | |
| 37 | −#define ACA_REG__STATUS__UC(x) ACA_REG_FIELD(x, 61, 61) | |
| 38 | −#define ACA_REG__STATUS__EN(x) ACA_REG_FIELD(x, 60, 60) | |
| 39 | −#define ACA_REG__STATUS__MISCV(x) ACA_REG_FIELD(x, 59, 59) | |
| 40 | −#define ACA_REG__STATUS__ADDRV(x) ACA_REG_FIELD(x, 58, 58) | |
| 41 | −#define ACA_REG__STATUS__PCC(x) ACA_REG_FIELD(x, 57, 57) | |
| 42 | −#define ACA_REG__STATUS__ERRCOREIDVAL(x) ACA_REG_FIELD(x, 56, 56) | |
| 43 | −#define ACA_REG__STATUS__TCC(x) ACA_REG_FIELD(x, 55, 55) | |
| 44 | −#define ACA_REG__STATUS__SYNDV(x) ACA_REG_FIELD(x, 53, 53) | |
| 45 | −#define ACA_REG__STATUS__CECC(x) ACA_REG_FIELD(x, 46, 46) | |
| 46 | −#define ACA_REG__STATUS__UECC(x) ACA_REG_FIELD(x, 45, 45) | |
| 47 | −#define ACA_REG__STATUS__DEFERRED(x) ACA_REG_FIELD(x, 44, 44) | |
| 48 | −#define ACA_REG__STATUS__POISON(x) ACA_REG_FIELD(x, 43, 43) | |
| 49 | −#define ACA_REG__STATUS__SCRUB(x) ACA_REG_FIELD(x, 40, 40) | |
| 50 | −#define ACA_REG__STATUS__ERRCOREID(x) ACA_REG_FIELD(x, 37, 32) | |
| 51 | −#define ACA_REG__STATUS__ADDRLSB(x) ACA_REG_FIELD(x, 29, 24) | |
| 52 | −#define ACA_REG__STATUS__ERRORCODEEXT(x) ACA_REG_FIELD(x, 21, 16) | |
| 53 | −#define ACA_REG__STATUS__ERRORCODE(x) ACA_REG_FIELD(x, 15, 0) | |
| 54 | − | |
| 55 | −#define ACA_REG__IPID__MCATYPE(x) ACA_REG_FIELD(x, 63, 48) | |
| 56 | −#define ACA_REG__IPID__INSTANCEIDHI(x) ACA_REG_FIELD(x, 47, 44) | |
| 57 | −#define ACA_REG__IPID__HARDWAREID(x) ACA_REG_FIELD(x, 43, 32) | |
| 58 | −#define ACA_REG__IPID__INSTANCEIDLO(x) ACA_REG_FIELD(x, 31, 0) | |
| 59 | − | |
| 60 | −#define ACA_REG__MISC0__VALID(x) ACA_REG_FIELD(x, 63, 63) | |
| 61 | −#define ACA_REG__MISC0__OVRFLW(x) ACA_REG_FIELD(x, 48, 48) | |
| 62 | −#define ACA_REG__MISC0__ERRCNT(x) ACA_REG_FIELD(x, 43, 32) | |
| 63 | − | |
| 64 | −#define ACA_REG__SYND__ERRORINFORMATION(x) ACA_REG_FIELD(x, 17, 0) | |
| 65 | − | |
| 66 | −/* NOTE: The following codes refers to the smu header file */ | |
| 67 | −#define ACA_EXTERROR_CODE_CE 0x3a | |
| 68 | −#define ACA_EXTERROR_CODE_FAULT 0x3b | |
| 69 | − | |
| 70 | −#define ACA_ERROR_UE_MASK BIT_MASK(ACA_ERROR_TYPE_UE) | |
| 71 | −#define ACA_ERROR_CE_MASK BIT_MASK(ACA_ERROR_TYPE_CE) | |
| 72 | −#define ACA_ERROR_DEFERRED_MASK BIT_MASK(ACA_ERROR_TYPE_DEFERRED) | |
| 73 | − | |
| 74 | −#define mmSMNAID_AID0_MCA_SMU 0x03b30400 /* SMN AID AID0 */ | |
| 75 | −#define mmSMNAID_XCD0_MCA_SMU 0x36430400 /* SMN AID XCD0 */ | |
| 76 | −#define mmSMNAID_XCD1_MCA_SMU 0x38430400 /* SMN AID XCD1 */ | |
| 77 | −#define mmSMNXCD_XCD0_MCA_SMU 0x40430400 /* SMN XCD XCD0 */ | |
| 78 | − | |
| 79 | −#define ACA_BANK_ERR_IS_DEFFERED(bank) \ | |
| 80 | − (ACA_REG__STATUS__POISON((bank)->regs[ACA_REG_IDX_STATUS]) || \ | |
| 81 | − ACA_REG__STATUS__DEFERRED((bank)->regs[ACA_REG_IDX_STATUS])) | |
| 82 | − | |
| 83 | −enum aca_reg_idx { | |
| 84 | − ACA_REG_IDX_CTL = 0, | |
| 85 | − ACA_REG_IDX_STATUS = 1, | |
| 86 | − ACA_REG_IDX_ADDR = 2, | |
| 87 | − ACA_REG_IDX_MISC0 = 3, | |
| 88 | − ACA_REG_IDX_CONFIG = 4, | |
| 89 | − ACA_REG_IDX_IPID = 5, | |
| 90 | − ACA_REG_IDX_SYND = 6, | |
| 91 | − ACA_REG_IDX_DESTAT = 8, | |
| 92 | − ACA_REG_IDX_DEADDR = 9, | |
| 93 | − ACA_REG_IDX_CTL_MASK = 10, | |
| 94 | − ACA_REG_IDX_COUNT = 16, | |
| 95 | −}; | |
| 96 | − | |
| 97 | −enum aca_hwip_type { | |
| 98 | − ACA_HWIP_TYPE_UNKNOW = -1, | |
| 99 | − ACA_HWIP_TYPE_PSP = 0, | |
| 100 | − ACA_HWIP_TYPE_UMC, | |
| 101 | − ACA_HWIP_TYPE_SMU, | |
| 102 | − ACA_HWIP_TYPE_PCS_XGMI, | |
| 103 | − ACA_HWIP_TYPE_COUNT, | |
| 104 | −}; | |
| 105 | − | |
| 106 | −enum aca_error_type { | |
| 107 | − ACA_ERROR_TYPE_INVALID = -1, | |
| 108 | − ACA_ERROR_TYPE_UE = 0, | |
| 109 | − ACA_ERROR_TYPE_CE, | |
| 110 | − ACA_ERROR_TYPE_DEFERRED, | |
| 111 | − ACA_ERROR_TYPE_COUNT | |
| 112 | −}; | |
| 113 | − | |
| 114 | −enum aca_smu_type { | |
| 115 | − ACA_SMU_TYPE_INVALID = -1, | |
| 116 | − ACA_SMU_TYPE_UE = 0, | |
| 117 | − ACA_SMU_TYPE_CE, | |
| 118 | − ACA_SMU_TYPE_COUNT, | |
| 119 | −}; | |
| 120 | − | |
| 121 | −struct aca_hwip { | |
| 122 | − int hwid; | |
| 123 | − int mcatype; | |
| 124 | −}; | |
| 125 | − | |
| 126 | −struct aca_bank { | |
| 127 | − enum aca_error_type aca_err_type; | |
| 128 | − enum aca_smu_type smu_err_type; | |
| 129 | − u64 regs[ACA_MAX_REGS_COUNT]; | |
| 130 | −}; | |
| 131 | − | |
| 132 | −struct aca_bank_node { | |
| 133 | − struct aca_bank bank; | |
| 134 | − struct list_head node; | |
| 135 | −}; | |
| 136 | − | |
| 137 | −struct aca_banks { | |
| 138 | − int nr_banks; | |
| 139 | − struct list_head list; | |
| 140 | −}; | |
| 141 | − | |
| 142 | −struct aca_bank_info { | |
| 143 | − int die_id; | |
| 144 | − int socket_id; | |
| 145 | − int hwid; | |
| 146 | − int mcatype; | |
| 147 | −}; | |
| 148 | − | |
| 149 | −struct aca_bank_error { | |
| 150 | − struct list_head node; | |
| 151 | − struct aca_bank_info info; | |
| 152 | − u64 count; | |
| 153 | −}; | |
| 154 | − | |
| 155 | −struct aca_error { | |
| 156 | − struct list_head list; | |
| 157 | − struct mutex lock; | |
| 158 | − enum aca_error_type type; | |
| 159 | − int nr_errors; | |
| 160 | −}; | |
| 161 | − | |
| 162 | −struct aca_handle_manager { | |
| 163 | − struct list_head list; | |
| 164 | − int nr_handles; | |
| 165 | −}; | |
| 166 | − | |
| 167 | −struct aca_error_cache { | |
| 168 | − struct aca_error errors[ACA_ERROR_TYPE_COUNT]; | |
| 169 | −}; | |
| 170 | − | |
| 171 | −struct aca_handle { | |
| 172 | − struct list_head node; | |
| 173 | − enum aca_hwip_type hwip; | |
| 174 | − struct amdgpu_device *adev; | |
| 175 | − struct aca_handle_manager *mgr; | |
| 176 | − struct aca_error_cache error_cache; | |
| 177 | − const struct aca_bank_ops *bank_ops; | |
| 178 | − struct device_attribute aca_attr; | |
| 179 | − char attr_name[64]; | |
| 180 | − const char *name; | |
| 181 | − u32 mask; | |
| 182 | − void *data; | |
| 183 | −}; | |
| 184 | − | |
| 185 | −struct aca_bank_ops { | |
| 186 | − int (*aca_bank_parser)(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type, void *data); | |
| 187 | − bool (*aca_bank_is_valid)(struct aca_handle *handle, struct aca_bank *bank, enum aca_smu_type type, | |
| 188 | − void *data); | |
| 189 | −}; | |
| 190 | − | |
| 191 | −struct aca_smu_funcs { | |
| 192 | − int max_ue_bank_count; | |
| 193 | − int max_ce_bank_count; | |
| 194 | − int (*set_debug_mode)(struct amdgpu_device *adev, bool enable); | |
| 195 | − int (*get_valid_aca_count)(struct amdgpu_device *adev, enum aca_smu_type type, u32 *count); | |
| 196 | − int (*get_valid_aca_bank)(struct amdgpu_device *adev, enum aca_smu_type type, int idx, struct aca_bank *bank); | |
| 197 | − int (*parse_error_code)(struct amdgpu_device *adev, struct aca_bank *bank); | |
| 198 | −}; | |
| 199 | − | |
| 200 | −struct amdgpu_aca { | |
| 201 | − struct aca_handle_manager mgr; | |
| 202 | − const struct aca_smu_funcs *smu_funcs; | |
| 203 | − atomic_t ue_update_flag; | |
| 204 | − bool is_enabled; | |
| 205 | −}; | |
| 206 | − | |
| 207 | −struct aca_info { | |
| 208 | − enum aca_hwip_type hwip; | |
| 209 | − const struct aca_bank_ops *bank_ops; | |
| 210 | − u32 mask; | |
| 211 | −}; | |
| 212 | − | |
| 213 | −int amdgpu_aca_init(struct amdgpu_device *adev); | |
| 214 | −void amdgpu_aca_fini(struct amdgpu_device *adev); | |
| 215 | −int amdgpu_aca_reset(struct amdgpu_device *adev); | |
| 216 | −void amdgpu_aca_set_smu_funcs(struct amdgpu_device *adev, const struct aca_smu_funcs *smu_funcs); | |
| 217 | −bool amdgpu_aca_is_enabled(struct amdgpu_device *adev); | |
| 218 | − | |
| 219 | −int aca_bank_info_decode(struct aca_bank *bank, struct aca_bank_info *info); | |
| 220 | −int aca_bank_check_error_codes(struct amdgpu_device *adev, struct aca_bank *bank, int *err_codes, int size); | |
| 221 | − | |
| 222 | −int amdgpu_aca_add_handle(struct amdgpu_device *adev, struct aca_handle *handle, | |
| 223 | − const char *name, const struct aca_info *aca_info, void *data); | |
| 224 | −void amdgpu_aca_remove_handle(struct aca_handle *handle); | |
| 225 | −int amdgpu_aca_get_error_data(struct amdgpu_device *adev, struct aca_handle *handle, | |
| 226 | − enum aca_error_type type, struct ras_err_data *err_data, | |
| 227 | − struct ras_query_context *qctx); | |
| 228 | −int amdgpu_aca_smu_set_debug_mode(struct amdgpu_device *adev, bool en); | |
| 229 | −void amdgpu_aca_smu_debugfs_init(struct amdgpu_device *adev, struct dentry *root); | |
| 230 | −int aca_error_cache_log_bank_error(struct aca_handle *handle, struct aca_bank_info *info, | |
| 231 | − enum aca_error_type type, u64 count); | |
| 232 | −#endif | |
| @@ -140,13 +140,15 @@ static struct amdgpu_acpi_priv { · 139 unmodified lines | ||
| 140 | 140 | * @atif: atif structure |
| 141 | 141 | * @function: the ATIF function to execute |
| 142 | 142 | * @params: ATIF function params |
| 143 | + * @min_size: minimum size of the expected output buffer in bytes | |
| 143 | 144 | * |
| 144 | 145 | * Executes the requested ATIF function (all asics). |
| 145 | 146 | * Returns a pointer to the acpi output buffer. |
| 146 | 147 | */ |
| 147 | 148 | static union acpi_object *amdgpu_atif_call(struct amdgpu_atif *atif, |
| 148 | 149 | int function, |
| 149 | − struct acpi_buffer *params) | |
| 150 | + struct acpi_buffer *params, | |
| 151 | + size_t min_size) | |
| 150 | 152 | { |
| 151 | 153 | acpi_status status; |
| 152 | 154 | union acpi_object *obj; |
| @@ -189,6 +191,28 @@ static union acpi_object *amdgpu_atif_call(struct amdgpu_atif *atif, · 36 unmodified lines | ||
| 189 | 191 | return NULL; |
| 190 | 192 | } |
| 191 | 193 | |
| 194 | + if (obj->buffer.length < sizeof(u16)) { | |
| 195 | + DRM_DEBUG_DRIVER("ATIF buffer too small to hold size field: %u\n", | |
| 196 | + obj->buffer.length); | |
| 197 | + kfree(obj); | |
| 198 | + return NULL; | |
| 199 | + } | |
| 200 | + | |
| 201 | + if (obj->buffer.length < *(u16 *)obj->buffer.pointer) { | |
| 202 | + DRM_DEBUG_DRIVER("ATIF buffer length mismatch: reported %u, actual %u\n", | |
| 203 | + *(u16 *)obj->buffer.pointer, | |
| 204 | + obj->buffer.length); | |
| 205 | + kfree(obj); | |
| 206 | + return NULL; | |
| 207 | + } | |
| 208 | + | |
| 209 | + if (*(u16 *)obj->buffer.pointer < min_size) { | |
| 210 | + DRM_DEBUG_DRIVER("ATIF buffer too small: expected %zu, got %u\n", | |
| 211 | + min_size, *(u16 *)obj->buffer.pointer); | |
| 212 | + kfree(obj); | |
| 213 | + return NULL; | |
| 214 | + } | |
| 215 | + | |
| 192 | 216 | return obj; |
| 193 | 217 | } |
| 194 | 218 | |
| @@ -251,19 +275,14 @@ int amdgpu_atif_verify_interface(struct amdgpu_atif *atif) · 56 unmodified lines | ||
| 251 | 275 | size_t size; |
| 252 | 276 | int err = 0; |
| 253 | 277 | |
| 254 | − info = amdgpu_atif_call(atif, ATIF_FUNCTION_VERIFY_INTERFACE, NULL); | |
| 278 | + info = amdgpu_atif_call(atif, ATIF_FUNCTION_VERIFY_INTERFACE, NULL, | |
| 279 | + sizeof(output)); | |
| 255 | 280 | if (!info) |
| 256 | 281 | return -EIO; |
| 257 | 282 | |
| 258 | 283 | memset(&output, 0, sizeof(output)); |
| 259 | 284 | |
| 260 | − size = *(u16 *) info->buffer.pointer; | |
| 261 | − if (size < 12) { | |
| 262 | − DRM_INFO("ATIF buffer is too small: %zu\n", size); | |
| 263 | − err = -EINVAL; | |
| 264 | − goto out; | |
| 265 | − } | |
| 266 | − size = min(sizeof(output), size); | |
| 285 | + size = min(sizeof(output), (size_t)*(u16 *)info->buffer.pointer); | |
| 267 | 286 | |
| 268 | 287 | memcpy(&output, info->buffer.pointer, size); |
| 269 | 288 | |
| @@ -273,7 +292,6 @@ int amdgpu_atif_verify_interface(struct amdgpu_atif *atif) · 3 unmodified lines | ||
| 273 | 292 | amdgpu_atif_parse_notification(&atif->notifications, output.notification_mask); |
| 274 | 293 | amdgpu_atif_parse_functions(&atif->functions, output.function_bits); |
| 275 | 294 | |
| 276 | −out: | |
| 277 | 295 | kfree(info); |
| 278 | 296 | return err; |
| 279 | 297 | } |
| @@ -299,20 +317,14 @@ int amdgpu_atif_get_notification_params(struct amdgpu_atif *atif) · 19 unmodified lines | ||
| 299 | 317 | int err = 0; |
| 300 | 318 | |
| 301 | 319 | info = amdgpu_atif_call(atif, ATIF_FUNCTION_GET_SYSTEM_PARAMETERS, |
| 302 | − NULL); | |
| 320 | + NULL, offsetof(struct atif_system_params, command_code)); | |
| 303 | 321 | if (!info) { |
| 304 | 322 | err = -EIO; |
| 305 | 323 | goto out; |
| 306 | 324 | } |
| 307 | 325 | |
| 308 | − size = *(u16 *) info->buffer.pointer; | |
| 309 | − if (size < 10) { | |
| 310 | − err = -EINVAL; | |
| 311 | − goto out; | |
| 312 | − } | |
| 313 | − | |
| 314 | 326 | memset(¶ms, 0, sizeof(params)); |
| 315 | − size = min(sizeof(params), size); | |
| 327 | + size = min(sizeof(params), (size_t)*(u16 *)info->buffer.pointer); | |
| 316 | 328 | memcpy(¶ms, info->buffer.pointer, size); |
| 317 | 329 | |
| 318 | 330 | DRM_DEBUG_DRIVER("SYSTEM_PARAMS: mask = %#x, flags = %#x\n", |
| @@ -376,20 +388,14 @@ int amdgpu_atif_query_backlight_caps(struct amdgpu_atif *atif) · 57 unmodified lines | ||
| 376 | 388 | |
| 377 | 389 | info = amdgpu_atif_call(atif, |
| 378 | 390 | ATIF_FUNCTION_QUERY_BRIGHTNESS_TRANSFER_CHARACTERISTICS, |
| 379 | − ¶ms); | |
| 391 | + ¶ms, offsetof(struct atif_qbtc_output, data_points)); | |
| 380 | 392 | if (!info) { |
| 381 | 393 | err = -EIO; |
| 382 | 394 | goto out; |
| 383 | 395 | } |
| 384 | 396 | |
| 385 | − size = *(u16 *) info->buffer.pointer; | |
| 386 | − if (size < 10) { | |
| 387 | − err = -EINVAL; | |
| 388 | − goto out; | |
| 389 | − } | |
| 390 | − | |
| 391 | 397 | memset(&characteristics, 0, sizeof(characteristics)); |
| 392 | − size = min(sizeof(characteristics), size); | |
| 398 | + size = min(sizeof(characteristics), (size_t)*(u16 *)info->buffer.pointer); | |
| 393 | 399 | memcpy(&characteristics, info->buffer.pointer, size); |
| 394 | 400 | |
| 395 | 401 | atif->backlight_caps.caps_valid = true; |
| @@ -427,24 +433,18 @@ static int amdgpu_atif_get_sbios_requests(struct amdgpu_atif *atif, · 31 unmodified lines | ||
| 427 | 433 | int count = 0; |
| 428 | 434 | |
| 429 | 435 | info = amdgpu_atif_call(atif, ATIF_FUNCTION_GET_SYSTEM_BIOS_REQUESTS, |
| 430 | − NULL); | |
| 436 | + NULL, sizeof(*req)); | |
| 431 | 437 | if (!info) |
| 432 | 438 | return -EIO; |
| 433 | 439 | |
| 434 | − size = *(u16 *)info->buffer.pointer; | |
| 435 | − if (size < 0xd) { | |
| 436 | − count = -EINVAL; | |
| 437 | − goto out; | |
| 438 | − } | |
| 439 | 440 | memset(req, 0, sizeof(*req)); |
| 440 | 441 | |
| 441 | − size = min(sizeof(*req), size); | |
| 442 | + size = min(sizeof(*req), (size_t)*(u16 *)info->buffer.pointer); | |
| 442 | 443 | memcpy(req, info->buffer.pointer, size); |
| 443 | 444 | DRM_DEBUG_DRIVER("SBIOS pending requests: %#x\n", req->pending); |
| 444 | 445 | |
| 445 | 446 | count = hweight32(req->pending); |
| 446 | 447 | |
| 447 | −out: | |
| 448 | 448 | kfree(info); |
| 449 | 449 | return count; |
| 450 | 450 | } |
| @@ -957,3 +957,17 @@ int amdgpu_amdkfd_config_sq_perfmon(struct amdgpu_device *adev, uint32_t xcp_id, · 956 unmodified lines | ||
| 957 | 957 | |
| 958 | 958 | return r; |
| 959 | 959 | } |
| 960 | + | |
| 961 | +/* Reset an MES queue */ | |
| 962 | +int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev, | |
| 963 | + uint32_t node_id, | |
| 964 | + int queue_type, | |
| 965 | + int pipe, int queue, | |
| 966 | + unsigned int db) | |
| 967 | +{ | |
| 968 | + if (!adev->kfd.init_complete) | |
| 969 | + return 0; | |
| 970 | + | |
| 971 | + return kgd2kfd_reset_mes_queue(adev->kfd.dev, node_id, queue_type, | |
| 972 | + pipe, queue, db); | |
| 973 | +} | |
| @@ -210,6 +210,7 @@ int amdgpu_amdkfd_evict_userptr(struct mmu_interval_notifier *mni, · 209 unmodified lines | ||
| 210 | 210 | int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo, |
| 211 | 211 | uint32_t domain, |
| 212 | 212 | struct dma_fence *fence); |
| 213 | +int amdgpu_amdkfd_set_sigbus_delay(struct task_struct *task, u32 ms); | |
| 213 | 214 | #else |
| 214 | 215 | static inline |
| 215 | 216 | bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm) |
| @@ -241,6 +242,11 @@ int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo, · 25 unmodified lines | ||
| 241 | 242 | { |
| 242 | 243 | return 0; |
| 243 | 244 | } |
| 245 | +static inline | |
| 246 | +int amdgpu_amdkfd_set_sigbus_delay(struct task_struct *task, u32 ms) | |
| 247 | +{ | |
| 248 | + return -EOPNOTSUPP; | |
| 249 | +} | |
| 244 | 250 | #endif |
| 245 | 251 | /* Shared API */ |
| 246 | 252 | int amdgpu_amdkfd_alloc_kernel_mem(struct amdgpu_device *adev, size_t size, |
| @@ -275,7 +281,11 @@ int amdgpu_amdkfd_stop_sched(struct amdgpu_device *adev, uint32_t node_id); · 28 unmodified lines | ||
| 275 | 281 | int amdgpu_amdkfd_config_sq_perfmon(struct amdgpu_device *adev, uint32_t xcp_id, |
| 276 | 282 | bool core_override_enable, bool reg_override_enable, bool perfmon_override_enable); |
| 277 | 283 | bool amdgpu_amdkfd_compute_active(struct amdgpu_device *adev, uint32_t node_id); |
| 278 | − | |
| 284 | +int amdgpu_amdkfd_reset_mes_queue(struct amdgpu_device *adev, | |
| 285 | + uint32_t node_id, | |
| 286 | + int queue_type, | |
| 287 | + int pipe, int queue, | |
| 288 | + unsigned int db); | |
| 279 | 289 | |
| 280 | 290 | /* Read user wptr from a specified user address space with page fault |
| 281 | 291 | * disabled. The memory must be pinned and mapped to the hardware when |
| @@ -326,9 +336,9 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu( · 44 unmodified lines | ||
| 326 | 336 | int amdgpu_amdkfd_gpuvm_dmaunmap_mem(struct kgd_mem *mem, void *drm_priv); |
| 327 | 337 | int amdgpu_amdkfd_gpuvm_sync_memory( |
| 328 | 338 | struct amdgpu_device *adev, struct kgd_mem *mem, bool intr); |
| 329 | −int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem, | |
| 330 | − void **kptr, uint64_t *size); | |
| 331 | −void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem); | |
| 339 | +int amdgpu_amdkfd_gpuvm_map_bo_to_kernel(struct kgd_mem *mem, void **kptr, | |
| 340 | + u64 *size, u32 domain); | |
| 341 | +void amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(struct kgd_mem *mem); | |
| 332 | 342 | |
| 333 | 343 | int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo_gart); |
| 334 | 344 | |
| @@ -446,6 +456,9 @@ bool kgd2kfd_vmfault_fast_path(struct amdgpu_device *adev, struct amdgpu_iv_entr · 111 unmodified lines | ||
| 446 | 456 | bool retry_fault); |
| 447 | 457 | void kgd2kfd_lock_kfd(void); |
| 448 | 458 | void kgd2kfd_teardown_processes(struct amdgpu_device *adev); |
| 459 | +int kgd2kfd_reset_mes_queue(struct kfd_dev *kfd, uint32_t node_id, | |
| 460 | + int queue_type, int pipe, int queue, | |
| 461 | + unsigned int db); | |
| 449 | 462 | |
| 450 | 463 | #else |
| 451 | 464 | static inline int kgd2kfd_init(void) |
| @@ -576,5 +589,12 @@ static inline void kgd2kfd_teardown_processes(struct amdgpu_device *adev) · 124 unmodified lines | ||
| 576 | 589 | { |
| 577 | 590 | } |
| 578 | 591 | |
| 592 | +static inline int kgd2kfd_reset_mes_queue(struct kfd_dev *kfd, uint32_t node_id, | |
| 593 | + int queue_type, int pipe, int queue, | |
| 594 | + unsigned int db) | |
| 595 | +{ | |
| 596 | + return 0; | |
| 597 | +} | |
| 598 | + | |
| 579 | 599 | #endif |
| 580 | 600 | #endif /* AMDGPU_AMDKFD_H_INCLUDED */ |
| @@ -530,6 +530,66 @@ static uint32_t kgd_v9_4_3_ptl_ctrl(struct amdgpu_device *adev, · 529 unmodified lines | ||
| 530 | 530 | ptl_state, fmt1, fmt2); |
| 531 | 531 | } |
| 532 | 532 | |
| 533 | +static int kgd_gfx_v9_4_3_hqd_sdma_get_counter(struct amdgpu_device *adev, | |
| 534 | + void *mqd, uint32_t num_sdma_queues_per_eng, | |
| 535 | + uint64_t *val) | |
| 536 | +{ | |
| 537 | + struct v9_sdma_mqd *m = get_sdma_mqd(mqd); | |
| 538 | + uint32_t sdma_rlc_reg_offset = 0; | |
| 539 | + uint32_t sdma_rlc_rb_cntl; | |
| 540 | + uint32_t engine_id, queue_id; | |
| 541 | + uint32_t engines = adev->sdma.num_instances; | |
| 542 | + uint32_t sdma_rlcx_rb_base, sdma_rlcx_rb_base_hi; | |
| 543 | + bool found = false; | |
| 544 | + | |
| 545 | + if (!m) | |
| 546 | + return -EINVAL; | |
| 547 | + | |
| 548 | + if (((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) || | |
| 549 | + amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 4)) && | |
| 550 | + adev->gfx.mec_fw_version < 194) || | |
| 551 | + (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0) && | |
| 552 | + adev->gfx.mec_fw_version < 44)) { | |
| 553 | + pr_warn_once("MEC FW doesn't support SDMA counter!\n"); | |
| 554 | + return -EOPNOTSUPP; | |
| 555 | + } | |
| 556 | + | |
| 557 | + /* SDMA doesn't support over-subscription, there must be | |
| 558 | + * a HQD associated with a MQD, so found must be true in | |
| 559 | + * the finding loop. | |
| 560 | + */ | |
| 561 | + for (engine_id = 0; engine_id < engines && !found; engine_id++) { | |
| 562 | + for (queue_id = 0; queue_id < num_sdma_queues_per_eng; queue_id++) { | |
| 563 | + sdma_rlc_reg_offset = get_sdma_rlc_reg_offset(adev, | |
| 564 | + engine_id, queue_id); | |
| 565 | + sdma_rlcx_rb_base = RREG32(sdma_rlc_reg_offset + | |
| 566 | + regSDMA_RLC0_RB_BASE); | |
| 567 | + sdma_rlcx_rb_base_hi = RREG32(sdma_rlc_reg_offset + | |
| 568 | + regSDMA_RLC0_RB_BASE_HI); | |
| 569 | + | |
| 570 | + if (m->sdmax_rlcx_rb_base == sdma_rlcx_rb_base && | |
| 571 | + m->sdmax_rlcx_rb_base_hi == sdma_rlcx_rb_base_hi) { | |
| 572 | + found = true; | |
| 573 | + break; | |
| 574 | + } | |
| 575 | + } | |
| 576 | + } | |
| 577 | + | |
| 578 | + sdma_rlc_rb_cntl = RREG32(sdma_rlc_reg_offset + regSDMA_RLC0_RB_CNTL); | |
| 579 | + | |
| 580 | + /* Read sdma activity counter from utilization register | |
| 581 | + * if hw queue is enabled, otherwise read from MQD. | |
| 582 | + */ | |
| 583 | + if (sdma_rlc_rb_cntl & SDMA_RLC0_RB_CNTL__RB_ENABLE_MASK) | |
| 584 | + *val = (uint64_t)RREG32(sdma_rlc_reg_offset + regSDMA_RLC0_UTILIZATION_HI) << 32 | | |
| 585 | + RREG32(sdma_rlc_reg_offset + regSDMA_RLC0_UTILIZATION_LO); | |
| 586 | + else | |
| 587 | + *val = (uint64_t)m->sdmax_rlcx_utilization_hi << 32 | | |
| 588 | + m->sdmax_rlcx_utilization_lo; | |
| 589 | + | |
| 590 | + return 0; | |
| 591 | +} | |
| 592 | + | |
| 533 | 593 | const struct kfd2kgd_calls gc_9_4_3_kfd2kgd = { |
| 534 | 594 | .program_sh_mem_settings = kgd_gfx_v9_program_sh_mem_settings, |
| 535 | 595 | .set_pasid_vmid_mapping = kgd_gfx_v9_4_3_set_pasid_vmid_mapping, |
| @@ -566,5 +626,6 @@ const struct kfd2kgd_calls gc_9_4_3_kfd2kgd = { · 30 unmodified lines | ||
| 566 | 626 | .hqd_get_pq_addr = kgd_gfx_v9_hqd_get_pq_addr, |
| 567 | 627 | .hqd_reset = kgd_gfx_v9_hqd_reset, |
| 568 | 628 | .hqd_sdma_get_doorbell = kgd_gfx_v9_4_3_hqd_sdma_get_doorbell, |
| 569 | − .ptl_ctrl = kgd_v9_4_3_ptl_ctrl | |
| 629 | + .ptl_ctrl = kgd_v9_4_3_ptl_ctrl, | |
| 630 | + .hqd_sdma_get_counter = kgd_gfx_v9_4_3_hqd_sdma_get_counter | |
| 570 | 631 | }; |
| @@ -2271,11 +2271,14 @@ err_reserve_bo_failed: · 2270 unmodified lines | ||
| 2271 | 2271 | return ret; |
| 2272 | 2272 | } |
| 2273 | 2273 | |
| 2274 | −/** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Map a GTT BO for kernel CPU access | |
| 2274 | +/** amdgpu_amdkfd_gpuvm_map_bo_to_kernel() - Map GTT or VRAM BO for kernel CPU access | |
| 2275 | 2275 | * |
| 2276 | 2276 | * @mem: Buffer object to be mapped for CPU access |
| 2277 | 2277 | * @kptr[out]: pointer in kernel CPU address space |
| 2278 | 2278 | * @size[out]: size of the buffer |
| 2279 | + * @domain[IN]: domain for pinning (AMDGPU_GEM_DOMAIN_GTT, AMDGPU_GEM_DOMAIN_VRAM, | |
| 2280 | + * or their combination to let the driver choose). CPU visibility is | |
| 2281 | + * automatically enforced by amdgpu_bo_pin() | |
| 2279 | 2282 | * |
| 2280 | 2283 | * Pins the BO and maps it for kernel CPU access. The eviction fence is removed |
| 2281 | 2284 | * from the BO, since pinned BOs cannot be evicted. The bo must remain on the |
| @@ -2284,8 +2287,8 @@ err_reserve_bo_failed: · 2 unmodified lines | ||
| 2284 | 2287 | * |
| 2285 | 2288 | * Return: 0 on success, error code on failure |
| 2286 | 2289 | */ |
| 2287 | −int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem, | |
| 2288 | − void **kptr, uint64_t *size) | |
| 2290 | +int amdgpu_amdkfd_gpuvm_map_bo_to_kernel(struct kgd_mem *mem, void **kptr, | |
| 2291 | + u64 *size, u32 domain) | |
| 2289 | 2292 | { |
| 2290 | 2293 | int ret; |
| 2291 | 2294 | struct amdgpu_bo *bo = mem->bo; |
| @@ -2295,6 +2298,11 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem, · 3 unmodified lines | ||
| 2295 | 2298 | return -EINVAL; |
| 2296 | 2299 | } |
| 2297 | 2300 | |
| 2301 | + if (!(domain & (AMDGPU_GEM_DOMAIN_GTT | AMDGPU_GEM_DOMAIN_VRAM))) { | |
| 2302 | + pr_debug("Invalid domain 0x%x for kernel mapping\n", domain); | |
| 2303 | + return -EINVAL; | |
| 2304 | + } | |
| 2305 | + | |
| 2298 | 2306 | mutex_lock(&mem->process_info->lock); |
| 2299 | 2307 | |
| 2300 | 2308 | ret = amdgpu_bo_reserve(bo, true); |
| @@ -2303,7 +2311,7 @@ int amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(struct kgd_mem *mem, · 2 unmodified lines | ||
| 2303 | 2311 | goto bo_reserve_failed; |
| 2304 | 2312 | } |
| 2305 | 2313 | |
| 2306 | − ret = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); | |
| 2314 | + ret = amdgpu_bo_pin(bo, domain); | |
| 2307 | 2315 | if (ret) { |
| 2308 | 2316 | pr_err("Failed to pin bo. ret %d\n", ret); |
| 2309 | 2317 | goto pin_failed; |
| @@ -2336,7 +2344,7 @@ bo_reserve_failed: · 26 unmodified lines | ||
| 2336 | 2344 | return ret; |
| 2337 | 2345 | } |
| 2338 | 2346 | |
| 2339 | −/** amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel() - Unmap a GTT BO for kernel CPU access | |
| 2347 | +/** amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel() - Unmap GTT or VRAM BO for kernel CPU access | |
| 2340 | 2348 | * |
| 2341 | 2349 | * @mem: Buffer object to be unmapped for CPU access |
| 2342 | 2350 | * |
| @@ -2344,7 +2352,7 @@ bo_reserve_failed: · 1 unmodified line | ||
| 2344 | 2352 | * eviction fence, so this function should only be used for cleanup before the |
| 2345 | 2353 | * BO is destroyed. |
| 2346 | 2354 | */ |
| 2347 | −void amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(struct kgd_mem *mem) | |
| 2355 | +void amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(struct kgd_mem *mem) | |
| 2348 | 2356 | { |
| 2349 | 2357 | struct amdgpu_bo *bo = mem->bo; |
| 2350 | 2358 | |
| @@ -1923,7 +1923,7 @@ int amdgpu_atombios_init(struct amdgpu_device *adev) · 1922 unmodified lines | ||
| 1923 | 1923 | atom_card_info->pll_read = cail_pll_read; |
| 1924 | 1924 | atom_card_info->pll_write = cail_pll_write; |
| 1925 | 1925 | |
| 1926 | − adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios); | |
| 1926 | + adev->mode_info.atom_context = amdgpu_atom_parse(atom_card_info, adev->bios, adev->bios_size); | |
| 1927 | 1927 | if (!adev->mode_info.atom_context) { |
| 1928 | 1928 | amdgpu_atombios_fini(adev); |
| 1929 | 1929 | return -ENOMEM; |
| @@ -89,6 +89,15 @@ bool amdgpu_is_atpx_hybrid(void) · 88 unmodified lines | ||
| 89 | 89 | return amdgpu_atpx_priv.atpx.is_hybrid; |
| 90 | 90 | } |
| 91 | 91 | |
| 92 | +static bool amdgpu_atpx_buffer_validate(const union acpi_object *obj, | |
| 93 | + size_t min_size) | |
| 94 | +{ | |
| 95 | + return obj && obj->type == ACPI_TYPE_BUFFER && | |
| 96 | + obj->buffer.length >= sizeof(u16) && | |
| 97 | + obj->buffer.length >= *(u16 *)obj->buffer.pointer && | |
| 98 | + *(u16 *)obj->buffer.pointer >= min_size; | |
| 99 | +} | |
| 100 | + | |
| 92 | 101 | /** |
| 93 | 102 | * amdgpu_atpx_call - call an ATPX method |
| 94 | 103 | * |
| @@ -179,15 +188,15 @@ static int amdgpu_atpx_validate(struct amdgpu_atpx *atpx) · 84 unmodified lines | ||
| 179 | 188 | if (!info) |
| 180 | 189 | return -EIO; |
| 181 | 190 | |
| 182 | − memset(&output, 0, sizeof(output)); | |
| 183 | − | |
| 184 | − size = *(u16 *) info->buffer.pointer; | |
| 185 | − if (size < 10) { | |
| 186 | − pr_err("ATPX buffer is too small: %zu\n", size); | |
| 191 | + if (!amdgpu_atpx_buffer_validate(info, sizeof(output))) { | |
| 192 | + pr_err("Invalid ATPX GET_PX_PARAMETERS response\n"); | |
| 187 | 193 | kfree(info); |
| 188 | 194 | return -EINVAL; |
| 189 | 195 | } |
| 190 | − size = min(sizeof(output), size); | |
| 196 | + | |
| 197 | + memset(&output, 0, sizeof(output)); | |
| 198 | + | |
| 199 | + size = min(sizeof(output), (size_t)*(u16 *)info->buffer.pointer); | |
| 191 | 200 | |
| 192 | 201 | memcpy(&output, info->buffer.pointer, size); |
| 193 | 202 | |
| @@ -258,15 +267,15 @@ static int amdgpu_atpx_verify_interface(struct amdgpu_atpx *atpx) · 64 unmodified lines | ||
| 258 | 267 | if (!info) |
| 259 | 268 | return -EIO; |
| 260 | 269 | |
| 261 | − memset(&output, 0, sizeof(output)); | |
| 262 | − | |
| 263 | − size = *(u16 *) info->buffer.pointer; | |
| 264 | − if (size < 8) { | |
| 265 | − pr_err("ATPX buffer is too small: %zu\n", size); | |
| 270 | + if (!amdgpu_atpx_buffer_validate(info, sizeof(output))) { | |
| 271 | + pr_err("Invalid ATPX VERIFY_INTERFACE response\n"); | |
| 266 | 272 | err = -EINVAL; |
| 267 | 273 | goto out; |
| 268 | 274 | } |
| 269 | − size = min(sizeof(output), size); | |
| 275 | + | |
| 276 | + memset(&output, 0, sizeof(output)); | |
| 277 | + | |
| 278 | + size = min(sizeof(output), (size_t)*(u16 *)info->buffer.pointer); | |
| 270 | 279 | |
| 271 | 280 | memcpy(&output, info->buffer.pointer, size); |
| 272 | 281 | |
| @@ -296,8 +296,14 @@ static int amdgpu_atrm_call(acpi_handle atrm_handle, uint8_t *bios, · 295 unmodified lines | ||
| 296 | 296 | } |
| 297 | 297 | |
| 298 | 298 | obj = (union acpi_object *)buffer.pointer; |
| 299 | − memcpy(bios+offset, obj->buffer.pointer, obj->buffer.length); | |
| 300 | − len = obj->buffer.length; | |
| 299 | + if (!obj || obj->type != ACPI_TYPE_BUFFER) { | |
| 300 | + DRM_ERROR("ATRM returned an invalid object\n"); | |
| 301 | + kfree(buffer.pointer); | |
| 302 | + return -EINVAL; | |
| 303 | + } | |
| 304 | + | |
| 305 | + len = min_t(size_t, obj->buffer.length, len); | |
| 306 | + memcpy(bios+offset, obj->buffer.pointer, len); | |
| 301 | 307 | kfree(buffer.pointer); |
| 302 | 308 | return len; |
| 303 | 309 | } |
| @@ -38,14 +38,6 @@ · 37 unmodified lines | ||
| 38 | 38 | #define AMDGPU_BO_LIST_NUM_BUCKETS (AMDGPU_BO_LIST_MAX_PRIORITY + 1) |
| 39 | 39 | #define AMDGPU_BO_LIST_MAX_ENTRIES (128 * 1024) |
| 40 | 40 | |
| 41 | −static void amdgpu_bo_list_free_rcu(struct rcu_head *rcu) | |
| 42 | −{ | |
| 43 | − struct amdgpu_bo_list *list = container_of(rcu, struct amdgpu_bo_list, | |
| 44 | − rhead); | |
| 45 | − mutex_destroy(&list->bo_list_mutex); | |
| 46 | − kvfree(list); | |
| 47 | −} | |
| 48 | − | |
| 49 | 41 | static void amdgpu_bo_list_free(struct kref *ref) |
| 50 | 42 | { |
| 51 | 43 | struct amdgpu_bo_list *list = container_of(ref, struct amdgpu_bo_list, |
| @@ -54,7 +46,8 @@ static void amdgpu_bo_list_free(struct kref *ref) · 2 unmodified lines | ||
| 54 | 46 | |
| 55 | 47 | amdgpu_bo_list_for_each_entry(e, list) |
| 56 | 48 | amdgpu_bo_unref(&e->bo); |
| 57 | − call_rcu(&list->rhead, amdgpu_bo_list_free_rcu); | |
| 49 | + | |
| 50 | + kvfree(list); | |
| 58 | 51 | } |
| 59 | 52 | |
| 60 | 53 | static int amdgpu_bo_list_entry_cmp(const void *_a, const void *_b) |
| @@ -66,9 +59,9 @@ static int amdgpu_bo_list_entry_cmp(const void *_a, const void *_b) · 5 unmodified lines | ||
| 66 | 59 | return (int)a->priority - (int)b->priority; |
| 67 | 60 | } |
| 68 | 61 | |
| 69 | −int amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp, | |
| 70 | − struct drm_amdgpu_bo_list_entry *info, | |
| 71 | − size_t num_entries, struct amdgpu_bo_list **result) | |
| 62 | +struct amdgpu_bo_list * | |
| 63 | +amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp, | |
| 64 | + struct drm_amdgpu_bo_list_entry *info, size_t num_entries) | |
| 72 | 65 | { |
| 73 | 66 | unsigned last_entry = 0, first_userptr = num_entries; |
| 74 | 67 | struct amdgpu_bo_list_entry *array; |
| @@ -79,7 +72,7 @@ int amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp, · 4 unmodified lines | ||
| 79 | 72 | |
| 80 | 73 | list = kvzalloc_flex(*list, entries, num_entries); |
| 81 | 74 | if (!list) |
| 82 | − return -ENOMEM; | |
| 75 | + return ERR_PTR(-ENOMEM); | |
| 83 | 76 | |
| 84 | 77 | kref_init(&list->refcount); |
| 85 | 78 | |
| @@ -134,9 +127,7 @@ int amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp, · 48 unmodified lines | ||
| 134 | 127 | |
| 135 | 128 | trace_amdgpu_cs_bo_status(list->num_entries, total_size); |
| 136 | 129 | |
| 137 | − mutex_init(&list->bo_list_mutex); | |
| 138 | − *result = list; | |
| 139 | − return 0; | |
| 130 | + return list; | |
| 140 | 131 | |
| 141 | 132 | error_free: |
| 142 | 133 | for (i = 0; i < last_entry; ++i) |
| @@ -144,150 +135,125 @@ error_free: · 1 unmodified line | ||
| 144 | 135 | for (i = first_userptr; i < num_entries; ++i) |
| 145 | 136 | amdgpu_bo_unref(&array[i].bo); |
| 146 | 137 | kvfree(list); |
| 147 | − return r; | |
| 138 | + return ERR_PTR(r); | |
| 148 | 139 | |
| 149 | 140 | } |
| 150 | 141 | |
| 151 | −static void amdgpu_bo_list_destroy(struct amdgpu_fpriv *fpriv, int id) | |
| 142 | +struct amdgpu_bo_list *amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, u32 id) | |
| 152 | 143 | { |
| 153 | 144 | struct amdgpu_bo_list *list; |
| 154 | 145 | |
| 155 | − mutex_lock(&fpriv->bo_list_lock); | |
| 156 | − list = idr_remove(&fpriv->bo_list_handles, id); | |
| 157 | − mutex_unlock(&fpriv->bo_list_lock); | |
| 146 | + xa_lock(&fpriv->bo_list_handles); | |
| 147 | + list = xa_load(&fpriv->bo_list_handles, id); | |
| 158 | 148 | if (list) |
| 159 | − kref_put(&list->refcount, amdgpu_bo_list_free); | |
| 160 | −} | |
| 161 | − | |
| 162 | −int amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id, | |
| 163 | − struct amdgpu_bo_list **result) | |
| 164 | −{ | |
| 165 | − rcu_read_lock(); | |
| 166 | − *result = idr_find(&fpriv->bo_list_handles, id); | |
| 167 | − | |
| 168 | − if (*result && kref_get_unless_zero(&(*result)->refcount)) { | |
| 169 | − rcu_read_unlock(); | |
| 170 | − return 0; | |
| 171 | − } | |
| 149 | + kref_get(&list->refcount); | |
| 150 | + else | |
| 151 | + list = ERR_PTR(-ENOENT); | |
| 152 | + xa_unlock(&fpriv->bo_list_handles); | |
| 172 | 153 | |
| 173 | − rcu_read_unlock(); | |
| 174 | − *result = NULL; | |
| 175 | − return -ENOENT; | |
| 154 | + return list; | |
| 176 | 155 | } |
| 177 | 156 | |
| 178 | 157 | void amdgpu_bo_list_put(struct amdgpu_bo_list *list) |
| 179 | 158 | { |
| 180 | − kref_put(&list->refcount, amdgpu_bo_list_free); | |
| 159 | + if (list) | |
| 160 | + kref_put(&list->refcount, amdgpu_bo_list_free); | |
| 181 | 161 | } |
| 182 | 162 | |
| 183 | −int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in, | |
| 184 | − struct drm_amdgpu_bo_list_entry **info_param) | |
| 163 | +struct drm_amdgpu_bo_list_entry * | |
| 164 | +amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in) | |
| 185 | 165 | { |
| 186 | − const uint32_t info_size = sizeof(struct drm_amdgpu_bo_list_entry); | |
| 187 | 166 | const void __user *uptr = u64_to_user_ptr(in->bo_info_ptr); |
| 188 | − const uint32_t bo_info_size = in->bo_info_size; | |
| 189 | 167 | const uint32_t bo_number = in->bo_number; |
| 190 | − struct drm_amdgpu_bo_list_entry *info; | |
| 191 | 168 | |
| 192 | 169 | if (bo_number > AMDGPU_BO_LIST_MAX_ENTRIES) |
| 193 | − return -EINVAL; | |
| 170 | + return ERR_PTR(-EINVAL); | |
| 194 | 171 | |
| 195 | − /* copy the handle array from userspace to a kernel buffer */ | |
| 196 | − if (likely(info_size == bo_info_size)) { | |
| 197 | − info = vmemdup_array_user(uptr, bo_number, info_size); | |
| 198 | − if (IS_ERR(info)) | |
| 199 | − return PTR_ERR(info); | |
| 200 | − } else { | |
| 201 | − const uint32_t bytes = min(bo_info_size, info_size); | |
| 202 | − unsigned i; | |
| 203 | − | |
| 204 | − info = kvmalloc_array(bo_number, info_size, GFP_KERNEL); | |
| 205 | − if (!info) | |
| 206 | − return -ENOMEM; | |
| 207 | − | |
| 208 | − memset(info, 0, bo_number * info_size); | |
| 209 | − for (i = 0; i < bo_number; ++i, uptr += bo_info_size) { | |
| 210 | − if (copy_from_user(&info[i], uptr, bytes)) { | |
| 211 | − kvfree(info); | |
| 212 | − return -EFAULT; | |
| 213 | − } | |
| 214 | − } | |
| 215 | − } | |
| 172 | + if (in->bo_info_size != sizeof(struct drm_amdgpu_bo_list_entry)) | |
| 173 | + return ERR_PTR(-EINVAL); | |
| 216 | 174 | |
| 217 | − *info_param = info; | |
| 218 | − return 0; | |
| 175 | + return vmemdup_array_user(uptr, bo_number, | |
| 176 | + sizeof(struct drm_amdgpu_bo_list_entry)); | |
| 219 | 177 | } |
| 220 | 178 | |
| 221 | 179 | int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data, |
| 222 | 180 | struct drm_file *filp) |
| 223 | 181 | { |
| 224 | − struct amdgpu_device *adev = drm_to_adev(dev); | |
| 225 | 182 | struct amdgpu_fpriv *fpriv = filp->driver_priv; |
| 183 | + struct amdgpu_device *adev = drm_to_adev(dev); | |
| 184 | + struct amdgpu_bo_list *list, *prev, *curr; | |
| 226 | 185 | union drm_amdgpu_bo_list *args = data; |
| 227 | 186 | uint32_t handle = args->in.list_handle; |
| 228 | − struct drm_amdgpu_bo_list_entry *info = NULL; | |
| 229 | − struct amdgpu_bo_list *list, *old; | |
| 187 | + struct drm_amdgpu_bo_list_entry *info; | |
| 230 | 188 | int r; |
| 231 | 189 | |
| 232 | − r = amdgpu_bo_create_list_entry_array(&args->in, &info); | |
| 233 | − if (r) | |
| 234 | − return r; | |
| 235 | − | |
| 236 | 190 | switch (args->in.operation) { |
| 237 | 191 | case AMDGPU_BO_LIST_OP_CREATE: |
| 238 | − r = amdgpu_bo_list_create(adev, filp, info, args->in.bo_number, | |
| 239 | − &list); | |
| 240 | − if (r) | |
| 241 | − goto error_free; | |
| 192 | + case AMDGPU_BO_LIST_OP_UPDATE: | |
| 193 | + info = amdgpu_bo_create_list_entry_array(&args->in); | |
| 194 | + if (IS_ERR(info)) | |
| 195 | + return PTR_ERR(info); | |
| 242 | 196 | |
| 243 | − mutex_lock(&fpriv->bo_list_lock); | |
| 244 | − r = idr_alloc(&fpriv->bo_list_handles, list, 1, 0, GFP_KERNEL); | |
| 245 | − mutex_unlock(&fpriv->bo_list_lock); | |
| 246 | − if (r < 0) { | |
| 247 | − goto error_put_list; | |
| 248 | − } | |
| 197 | + list = amdgpu_bo_list_create(adev, filp, info, | |
| 198 | + args->in.bo_number); | |
| 199 | + kvfree(info); | |
| 200 | + if (IS_ERR(list)) | |
| 201 | + return PTR_ERR(list); | |
| 249 | 202 | |
| 250 | − handle = r; | |
| 251 | 203 | break; |
| 252 | 204 | |
| 253 | 205 | case AMDGPU_BO_LIST_OP_DESTROY: |
| 254 | − amdgpu_bo_list_destroy(fpriv, handle); | |
| 206 | + list = xa_erase(&fpriv->bo_list_handles, handle); | |
| 207 | + amdgpu_bo_list_put(list); | |
| 255 | 208 | handle = 0; |
| 209 | + | |
| 256 | 210 | break; |
| 257 | 211 | |
| 258 | − case AMDGPU_BO_LIST_OP_UPDATE: | |
| 259 | − r = amdgpu_bo_list_create(adev, filp, info, args->in.bo_number, | |
| 260 | − &list); | |
| 212 | + default: | |
| 213 | + return -EINVAL; | |
| 214 | + }; | |
| 215 | + | |
| 216 | + switch (args->in.operation) { | |
| 217 | + case AMDGPU_BO_LIST_OP_CREATE: | |
| 218 | + r = xa_alloc(&fpriv->bo_list_handles, &handle, list, | |
| 219 | + xa_limit_32b, GFP_KERNEL); | |
| 261 | 220 | if (r) |
| 262 | − goto error_free; | |
| 221 | + goto error_put_list; | |
| 222 | + | |
| 223 | + break; | |
| 263 | 224 | |
| 264 | − mutex_lock(&fpriv->bo_list_lock); | |
| 265 | − old = idr_replace(&fpriv->bo_list_handles, list, handle); | |
| 266 | − mutex_unlock(&fpriv->bo_list_lock); | |
| 225 | + case AMDGPU_BO_LIST_OP_UPDATE: | |
| 226 | + curr = xa_load(&fpriv->bo_list_handles, handle); | |
| 227 | + if (!curr) { | |
| 228 | + r = -ENOENT; | |
| 229 | + goto error_put_list; | |
| 230 | + } | |
| 267 | 231 | |
| 268 | − if (IS_ERR(old)) { | |
| 269 | − r = PTR_ERR(old); | |
| 232 | + prev = xa_cmpxchg(&fpriv->bo_list_handles, handle, curr, list, | |
| 233 | + GFP_KERNEL); | |
| 234 | + if (xa_is_err(prev)) { | |
| 235 | + r = xa_err(prev); | |
| 236 | + goto error_put_list; | |
| 237 | + } else if (prev != curr) { | |
| 238 | + r = -ENOENT; | |
| 270 | 239 | goto error_put_list; |
| 271 | 240 | } |
| 272 | 241 | |
| 273 | − amdgpu_bo_list_put(old); | |
| 242 | + amdgpu_bo_list_put(curr); | |
| 274 | 243 | break; |
| 275 | 244 | |
| 245 | + case AMDGPU_BO_LIST_OP_DESTROY: | |
| 276 | 246 | default: |
| 277 | − r = -EINVAL; | |
| 278 | − goto error_free; | |
| 247 | + /* Handled above. */ | |
| 248 | + break; | |
| 279 | 249 | } |
| 280 | 250 | |
| 281 | 251 | memset(args, 0, sizeof(*args)); |
| 282 | 252 | args->out.list_handle = handle; |
| 283 | − kvfree(info); | |
| 284 | 253 | |
| 285 | 254 | return 0; |
| 286 | 255 | |
| 287 | 256 | error_put_list: |
| 288 | 257 | amdgpu_bo_list_put(list); |
| 289 | − | |
| 290 | −error_free: | |
| 291 | − kvfree(info); | |
| 292 | 258 | return r; |
| 293 | 259 | } |
| @@ -43,7 +43,6 @@ struct amdgpu_bo_list_entry { · 42 unmodified lines | ||
| 43 | 43 | }; |
| 44 | 44 | |
| 45 | 45 | struct amdgpu_bo_list { |
| 46 | − struct rcu_head rhead; | |
| 47 | 46 | struct kref refcount; |
| 48 | 47 | struct amdgpu_bo *gds_obj; |
| 49 | 48 | struct amdgpu_bo *gws_obj; |
| @@ -51,24 +50,19 @@ struct amdgpu_bo_list { · 1 unmodified line | ||
| 51 | 50 | unsigned first_userptr; |
| 52 | 51 | unsigned num_entries; |
| 53 | 52 | |
| 54 | − /* Protect access during command submission. | |
| 55 | − */ | |
| 56 | − struct mutex bo_list_mutex; | |
| 57 | − | |
| 58 | 53 | struct amdgpu_bo_list_entry entries[] __counted_by(num_entries); |
| 59 | 54 | }; |
| 60 | 55 | |
| 61 | −int amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id, | |
| 62 | − struct amdgpu_bo_list **result); | |
| 56 | +struct amdgpu_bo_list *amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, u32 id); | |
| 63 | 57 | void amdgpu_bo_list_put(struct amdgpu_bo_list *list); |
| 64 | −int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in, | |
| 65 | − struct drm_amdgpu_bo_list_entry **info_param); | |
| 58 | +struct drm_amdgpu_bo_list_entry * | |
| 59 | +amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in); | |
| 66 | 60 | |
| 67 | −int amdgpu_bo_list_create(struct amdgpu_device *adev, | |
| 68 | − struct drm_file *filp, | |
| 69 | − struct drm_amdgpu_bo_list_entry *info, | |
| 70 | − size_t num_entries, | |
| 71 | − struct amdgpu_bo_list **list); | |
| 61 | +struct amdgpu_bo_list * | |
| 62 | +amdgpu_bo_list_create(struct amdgpu_device *adev, | |
| 63 | + struct drm_file *filp, | |
| 64 | + struct drm_amdgpu_bo_list_entry *info, | |
| 65 | + size_t num_entries); | |
| 72 | 66 | |
| 73 | 67 | #define amdgpu_bo_list_for_each_entry(e, list) \ |
| 74 | 68 | for (e = list->entries; \ |
| @@ -289,40 +289,6 @@ struct cper_hdr *amdgpu_cper_alloc_entry(struct amdgpu_device *adev, · 288 unmodified lines | ||
| 289 | 289 | return hdr; |
| 290 | 290 | } |
| 291 | 291 | |
| 292 | −int amdgpu_cper_generate_ue_record(struct amdgpu_device *adev, | |
| 293 | − struct aca_bank *bank) | |
| 294 | −{ | |
| 295 | − struct cper_hdr *fatal = NULL; | |
| 296 | − struct cper_sec_crashdump_reg_data reg_data = { 0 }; | |
| 297 | − struct amdgpu_ring *ring = &adev->cper.ring_buf; | |
| 298 | − int ret; | |
| 299 | − | |
| 300 | − fatal = amdgpu_cper_alloc_entry(adev, AMDGPU_CPER_TYPE_FATAL, 1); | |
| 301 | − if (!fatal) { | |
| 302 | − dev_err(adev->dev, "fail to alloc cper entry for ue record\n"); | |
| 303 | − return -ENOMEM; | |
| 304 | − } | |
| 305 | − | |
| 306 | − reg_data.status_lo = lower_32_bits(bank->regs[ACA_REG_IDX_STATUS]); | |
| 307 | − reg_data.status_hi = upper_32_bits(bank->regs[ACA_REG_IDX_STATUS]); | |
| 308 | − reg_data.addr_lo = lower_32_bits(bank->regs[ACA_REG_IDX_ADDR]); | |
| 309 | − reg_data.addr_hi = upper_32_bits(bank->regs[ACA_REG_IDX_ADDR]); | |
| 310 | − reg_data.ipid_lo = lower_32_bits(bank->regs[ACA_REG_IDX_IPID]); | |
| 311 | − reg_data.ipid_hi = upper_32_bits(bank->regs[ACA_REG_IDX_IPID]); | |
| 312 | − reg_data.synd_lo = lower_32_bits(bank->regs[ACA_REG_IDX_SYND]); | |
| 313 | − reg_data.synd_hi = upper_32_bits(bank->regs[ACA_REG_IDX_SYND]); | |
| 314 | − | |
| 315 | − amdgpu_cper_entry_fill_hdr(adev, fatal, AMDGPU_CPER_TYPE_FATAL, CPER_SEV_FATAL_UNCORRECTED); | |
| 316 | − ret = amdgpu_cper_entry_fill_fatal_section(adev, fatal, 0, reg_data); | |
| 317 | − if (ret) | |
| 318 | − return ret; | |
| 319 | − | |
| 320 | − amdgpu_cper_ring_write(ring, fatal, fatal->record_length); | |
| 321 | − kfree(fatal); | |
| 322 | − | |
| 323 | − return 0; | |
| 324 | −} | |
| 325 | − | |
| 326 | 292 | int amdgpu_cper_generate_bp_threshold_record(struct amdgpu_device *adev) |
| 327 | 293 | { |
| 328 | 294 | struct cper_hdr *bp_threshold = NULL; |
| @@ -348,83 +314,6 @@ int amdgpu_cper_generate_bp_threshold_record(struct amdgpu_device *adev) · 19 unmodified lines | ||
| 348 | 314 | return 0; |
| 349 | 315 | } |
| 350 | 316 | |
| 351 | −static enum cper_error_severity amdgpu_aca_err_type_to_cper_sev(struct amdgpu_device *adev, | |
| 352 | − enum aca_error_type aca_err_type) | |
| 353 | −{ | |
| 354 | − switch (aca_err_type) { | |
| 355 | − case ACA_ERROR_TYPE_UE: | |
| 356 | − return CPER_SEV_FATAL_UNCORRECTED; | |
| 357 | − case ACA_ERROR_TYPE_CE: | |
| 358 | − return CPER_SEV_NON_FATAL_CORRECTED; | |
| 359 | − case ACA_ERROR_TYPE_DEFERRED: | |
| 360 | − return CPER_SEV_NON_FATAL_UNCORRECTED; | |
| 361 | − default: | |
| 362 | − dev_err(adev->dev, "Unknown ACA error type!\n"); | |
| 363 | − return CPER_SEV_FATAL_UNCORRECTED; | |
| 364 | − } | |
| 365 | −} | |
| 366 | − | |
| 367 | −int amdgpu_cper_generate_ce_records(struct amdgpu_device *adev, | |
| 368 | − struct aca_banks *banks, | |
| 369 | − uint16_t bank_count) | |
| 370 | −{ | |
| 371 | − struct cper_hdr *corrected = NULL; | |
| 372 | − enum cper_error_severity sev = CPER_SEV_NON_FATAL_CORRECTED; | |
| 373 | − struct amdgpu_ring *ring = &adev->cper.ring_buf; | |
| 374 | − uint32_t reg_data[CPER_ACA_REG_COUNT] = { 0 }; | |
| 375 | − struct aca_bank_node *node; | |
| 376 | − struct aca_bank *bank; | |
| 377 | − uint32_t i = 0; | |
| 378 | − int ret; | |
| 379 | − | |
| 380 | − corrected = amdgpu_cper_alloc_entry(adev, AMDGPU_CPER_TYPE_RUNTIME, bank_count); | |
| 381 | − if (!corrected) { | |
| 382 | − dev_err(adev->dev, "fail to allocate cper entry for ce records\n"); | |
| 383 | − return -ENOMEM; | |
| 384 | − } | |
| 385 | − | |
| 386 | − /* Raise severity if any DE is detected in the ACA bank list */ | |
| 387 | − list_for_each_entry(node, &banks->list, node) { | |
| 388 | − bank = &node->bank; | |
| 389 | − if (bank->aca_err_type == ACA_ERROR_TYPE_DEFERRED) { | |
| 390 | − sev = CPER_SEV_NON_FATAL_UNCORRECTED; | |
| 391 | − break; | |
| 392 | − } | |
| 393 | − } | |
| 394 | − | |
| 395 | − amdgpu_cper_entry_fill_hdr(adev, corrected, AMDGPU_CPER_TYPE_RUNTIME, sev); | |
| 396 | − | |
| 397 | − /* Combine CE and DE in cper record */ | |
| 398 | − list_for_each_entry(node, &banks->list, node) { | |
| 399 | − bank = &node->bank; | |
| 400 | − reg_data[CPER_ACA_REG_CTL_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_CTL]); | |
| 401 | − reg_data[CPER_ACA_REG_CTL_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_CTL]); | |
| 402 | − reg_data[CPER_ACA_REG_STATUS_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_STATUS]); | |
| 403 | − reg_data[CPER_ACA_REG_STATUS_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_STATUS]); | |
| 404 | − reg_data[CPER_ACA_REG_ADDR_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_ADDR]); | |
| 405 | − reg_data[CPER_ACA_REG_ADDR_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_ADDR]); | |
| 406 | − reg_data[CPER_ACA_REG_MISC0_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_MISC0]); | |
| 407 | − reg_data[CPER_ACA_REG_MISC0_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_MISC0]); | |
| 408 | − reg_data[CPER_ACA_REG_CONFIG_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_CONFIG]); | |
| 409 | − reg_data[CPER_ACA_REG_CONFIG_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_CONFIG]); | |
| 410 | − reg_data[CPER_ACA_REG_IPID_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_IPID]); | |
| 411 | − reg_data[CPER_ACA_REG_IPID_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_IPID]); | |
| 412 | − reg_data[CPER_ACA_REG_SYND_LO] = lower_32_bits(bank->regs[ACA_REG_IDX_SYND]); | |
| 413 | − reg_data[CPER_ACA_REG_SYND_HI] = upper_32_bits(bank->regs[ACA_REG_IDX_SYND]); | |
| 414 | − | |
| 415 | − ret = amdgpu_cper_entry_fill_runtime_section(adev, corrected, i++, | |
| 416 | − amdgpu_aca_err_type_to_cper_sev(adev, bank->aca_err_type), | |
| 417 | − reg_data, CPER_ACA_REG_COUNT); | |
| 418 | − if (ret) | |
| 419 | − return ret; | |
| 420 | − } | |
| 421 | − | |
| 422 | − amdgpu_cper_ring_write(ring, corrected, corrected->record_length); | |
| 423 | − kfree(corrected); | |
| 424 | − | |
| 425 | − return 0; | |
| 426 | −} | |
| 427 | − | |
| 428 | 317 | static bool amdgpu_cper_is_hdr(struct amdgpu_ring *ring, u64 pos) |
| 429 | 318 | { |
| 430 | 319 | char signature[CPER_SIGNATURE_SZ]; |
| @@ -592,8 +481,7 @@ int amdgpu_cper_init(struct amdgpu_device *adev) · 161 unmodified lines | ||
| 592 | 481 | |
| 593 | 482 | if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_cper_en(adev)) |
| 594 | 483 | return 0; |
| 595 | − else if (!amdgpu_sriov_vf(adev) && !amdgpu_uniras_enabled(adev) && | |
| 596 | − !amdgpu_aca_is_enabled(adev)) | |
| 484 | + else if (!amdgpu_sriov_vf(adev) && !amdgpu_uniras_enabled(adev)) | |
| 597 | 485 | return 0; |
| 598 | 486 | |
| 599 | 487 | r = amdgpu_cper_ring_init(adev); |
| @@ -612,7 +500,7 @@ int amdgpu_cper_init(struct amdgpu_device *adev) · 12 unmodified lines | ||
| 612 | 500 | |
| 613 | 501 | int amdgpu_cper_fini(struct amdgpu_device *adev) |
| 614 | 502 | { |
| 615 | − if (!amdgpu_aca_is_enabled(adev) && !amdgpu_sriov_ras_cper_en(adev)) | |
| 503 | + if (amdgpu_sriov_vf(adev)) | |
| 616 | 504 | return 0; |
| 617 | 505 | |
| 618 | 506 | adev->cper.enabled = false; |
| @@ -26,7 +26,6 @@ · 25 unmodified lines | ||
| 26 | 26 | #define __AMDGPU_CPER_H__ |
| 27 | 27 | |
| 28 | 28 | #include "amd_cper.h" |
| 29 | −#include "amdgpu_aca.h" | |
| 30 | 29 | |
| 31 | 30 | #define CPER_MAX_ALLOWED_COUNT 0x1000 |
| 32 | 31 | #define CPER_MAX_RING_SIZE 0X100000 |
| @@ -88,13 +87,6 @@ int amdgpu_cper_entry_fill_bad_page_threshold_section(struct amdgpu_device *adev · 55 unmodified lines | ||
| 88 | 87 | struct cper_hdr *amdgpu_cper_alloc_entry(struct amdgpu_device *adev, |
| 89 | 88 | enum amdgpu_cper_type type, |
| 90 | 89 | uint16_t section_count); |
| 91 | −/* UE must be encoded into separated cper entries, 1 UE 1 cper */ | |
| 92 | −int amdgpu_cper_generate_ue_record(struct amdgpu_device *adev, | |
| 93 | − struct aca_bank *bank); | |
| 94 | −/* CEs and DEs are combined into 1 cper entry */ | |
| 95 | −int amdgpu_cper_generate_ce_records(struct amdgpu_device *adev, | |
| 96 | − struct aca_banks *banks, | |
| 97 | − uint16_t bank_count); | |
| 98 | 90 | /* Bad page threshold is encoded into separated cper entry */ |
| 99 | 91 | int amdgpu_cper_generate_bp_threshold_record(struct amdgpu_device *adev); |
| 100 | 92 | void amdgpu_cper_ring_write(struct amdgpu_ring *ring, |
| @@ -140,24 +140,19 @@ static int amdgpu_cs_p1_bo_handles(struct amdgpu_cs_parser *p, · 139 unmodified lines | ||
| 140 | 140 | struct drm_amdgpu_bo_list_in *data) |
| 141 | 141 | { |
| 142 | 142 | struct drm_amdgpu_bo_list_entry *info; |
| 143 | − int r; | |
| 144 | − | |
| 145 | − r = amdgpu_bo_create_list_entry_array(data, &info); | |
| 146 | − if (r) | |
| 147 | − return r; | |
| 148 | − | |
| 149 | − r = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number, | |
| 150 | − &p->bo_list); | |
| 151 | − if (r) | |
| 152 | − goto error_free; | |
| 143 | + struct amdgpu_bo_list *list; | |
| 153 | 144 | |
| 154 | − kvfree(info); | |
| 155 | − return 0; | |
| 145 | + info = amdgpu_bo_create_list_entry_array(data); | |
| 146 | + if (IS_ERR(info)) | |
| 147 | + return PTR_ERR(info); | |
| 156 | 148 | |
| 157 | −error_free: | |
| 149 | + list = amdgpu_bo_list_create(p->adev, p->filp, info, data->bo_number); | |
| 158 | 150 | kvfree(info); |
| 151 | + if (IS_ERR(list)) | |
| 152 | + return PTR_ERR(list); | |
| 159 | 153 | |
| 160 | − return r; | |
| 154 | + p->bo_list = list; | |
| 155 | + return 0; | |
| 161 | 156 | } |
| 162 | 157 | |
| 163 | 158 | /* Copy the data from userspace and go over it the first time */ |
| @@ -846,6 +841,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, · 682 unmodified lines | ||
| 846 | 841 | { |
| 847 | 842 | struct amdgpu_fpriv *fpriv = p->filp->driver_priv; |
| 848 | 843 | struct ttm_operation_ctx ctx = { true, false }; |
| 844 | + struct amdgpu_bo_list *list = NULL; | |
| 849 | 845 | struct amdgpu_vm *vm = &fpriv->vm; |
| 850 | 846 | struct amdgpu_bo_list_entry *e; |
| 851 | 847 | struct drm_gem_object *obj; |
| @@ -857,25 +853,24 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, · 5 unmodified lines | ||
| 857 | 853 | if (p->bo_list) |
| 858 | 854 | return -EINVAL; |
| 859 | 855 | |
| 860 | − r = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle, | |
| 861 | − &p->bo_list); | |
| 862 | − if (r) | |
| 863 | − return r; | |
| 856 | + list = amdgpu_bo_list_get(fpriv, cs->in.bo_list_handle); | |
| 864 | 857 | } else if (!p->bo_list) { |
| 865 | 858 | /* Create a empty bo_list when no handle is provided */ |
| 866 | − r = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0, | |
| 867 | − &p->bo_list); | |
| 868 | − if (r) | |
| 869 | − return r; | |
| 859 | + list = amdgpu_bo_list_create(p->adev, p->filp, NULL, 0); | |
| 870 | 860 | } |
| 871 | 861 | |
| 872 | − mutex_lock(&p->bo_list->bo_list_mutex); | |
| 862 | + if (IS_ERR(list)) | |
| 863 | + return PTR_ERR(list); | |
| 864 | + else if (list) | |
| 865 | + p->bo_list = list; | |
| 866 | + else | |
| 867 | + list = p->bo_list; | |
| 873 | 868 | |
| 874 | 869 | /* Get userptr backing pages. If pages are updated after registered |
| 875 | 870 | * in amdgpu_gem_userptr_ioctl(), amdgpu_cs_list_validate() will do |
| 876 | 871 | * amdgpu_ttm_backend_bind() to flush and invalidate new pages |
| 877 | 872 | */ |
| 878 | − amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { | |
| 873 | + amdgpu_bo_list_for_each_userptr_entry(e, list) { | |
| 879 | 874 | bool userpage_invalidated = false; |
| 880 | 875 | struct amdgpu_bo *bo = e->bo; |
| 881 | 876 | |
| @@ -905,7 +900,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, · 23 unmodified lines | ||
| 905 | 900 | if (unlikely(r)) |
| 906 | 901 | goto out_free_user_pages; |
| 907 | 902 | |
| 908 | − amdgpu_bo_list_for_each_entry(e, p->bo_list) { | |
| 903 | + amdgpu_bo_list_for_each_entry(e, list) { | |
| 909 | 904 | r = drm_exec_prepare_obj(&p->exec, &e->bo->tbo.base, |
| 910 | 905 | TTM_NUM_MOVE_FENCES + p->gang_size); |
| 911 | 906 | drm_exec_retry_on_contention(&p->exec); |
| @@ -924,7 +919,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, · 12 unmodified lines | ||
| 924 | 919 | } |
| 925 | 920 | } |
| 926 | 921 | |
| 927 | − amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { | |
| 922 | + amdgpu_bo_list_for_each_userptr_entry(e, list) { | |
| 928 | 923 | struct mm_struct *usermm; |
| 929 | 924 | |
| 930 | 925 | usermm = amdgpu_ttm_tt_get_usermm(e->bo->tbo.ttm); |
| @@ -977,17 +972,15 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p, · 46 unmodified lines | ||
| 977 | 972 | p->bytes_moved_vis); |
| 978 | 973 | |
| 979 | 974 | for (i = 0; i < p->gang_size; ++i) |
| 980 | − amdgpu_job_set_resources(p->jobs[i], p->bo_list->gds_obj, | |
| 981 | − p->bo_list->gws_obj, | |
| 982 | − p->bo_list->oa_obj); | |
| 975 | + amdgpu_job_set_resources(p->jobs[i], list->gds_obj, | |
| 976 | + list->gws_obj, list->oa_obj); | |
| 983 | 977 | return 0; |
| 984 | 978 | |
| 985 | 979 | out_free_user_pages: |
| 986 | − amdgpu_bo_list_for_each_userptr_entry(e, p->bo_list) { | |
| 980 | + amdgpu_bo_list_for_each_userptr_entry(e, list) { | |
| 987 | 981 | amdgpu_hmm_range_free(e->range); |
| 988 | 982 | e->range = NULL; |
| 989 | 983 | } |
| 990 | − mutex_unlock(&p->bo_list->bo_list_mutex); | |
| 991 | 984 | return r; |
| 992 | 985 | } |
| 993 | 986 | |
| @@ -1371,7 +1364,6 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p, · 377 unmodified lines | ||
| 1371 | 1364 | amdgpu_vm_move_to_lru_tail(p->adev, &fpriv->vm); |
| 1372 | 1365 | |
| 1373 | 1366 | mutex_unlock(&p->adev->notifier_lock); |
| 1374 | − mutex_unlock(&p->bo_list->bo_list_mutex); | |
| 1375 | 1367 | return 0; |
| 1376 | 1368 | } |
| 1377 | 1369 | |
| @@ -1443,28 +1435,25 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) · 65 unmodified lines | ||
| 1443 | 1435 | |
| 1444 | 1436 | r = amdgpu_cs_patch_jobs(&parser); |
| 1445 | 1437 | if (r) |
| 1446 | − goto error_backoff; | |
| 1438 | + goto error_fini; | |
| 1447 | 1439 | |
| 1448 | 1440 | r = amdgpu_cs_vm_handling(&parser); |
| 1449 | 1441 | if (r) |
| 1450 | − goto error_backoff; | |
| 1442 | + goto error_fini; | |
| 1451 | 1443 | |
| 1452 | 1444 | r = amdgpu_cs_sync_rings(&parser); |
| 1453 | 1445 | if (r) |
| 1454 | − goto error_backoff; | |
| 1446 | + goto error_fini; | |
| 1455 | 1447 | |
| 1456 | 1448 | trace_amdgpu_cs_ibs(&parser); |
| 1457 | 1449 | |
| 1458 | 1450 | r = amdgpu_cs_submit(&parser, data); |
| 1459 | 1451 | if (r) |
| 1460 | − goto error_backoff; | |
| 1452 | + goto error_fini; | |
| 1461 | 1453 | |
| 1462 | 1454 | amdgpu_cs_parser_fini(&parser); |
| 1463 | 1455 | return 0; |
| 1464 | 1456 | |
| 1465 | −error_backoff: | |
| 1466 | − mutex_unlock(&parser.bo_list->bo_list_mutex); | |
| 1467 | − | |
| 1468 | 1457 | error_fini: |
| 1469 | 1458 | amdgpu_cs_parser_fini(&parser); |
| 1470 | 1459 | return r; |
| @@ -283,6 +283,8 @@ static ktime_t amdgpu_ctx_fini_entity(struct amdgpu_device *adev, · 282 unmodified lines | ||
| 283 | 283 | if (!entity) |
| 284 | 284 | return res; |
| 285 | 285 | |
| 286 | + drm_sched_entity_destroy(&entity->entity); | |
| 287 | + | |
| 286 | 288 | for (i = 0; i < amdgpu_sched_jobs; ++i) { |
| 287 | 289 | res = ktime_add(res, amdgpu_ctx_fence_time(entity->fences[i])); |
| 288 | 290 | dma_fence_put(entity->fences[i]); |
| @@ -294,32 +296,20 @@ static ktime_t amdgpu_ctx_fini_entity(struct amdgpu_device *adev, · 5 unmodified lines | ||
| 294 | 296 | return res; |
| 295 | 297 | } |
| 296 | 298 | |
| 297 | −static int amdgpu_ctx_get_stable_pstate(struct amdgpu_ctx *ctx, | |
| 298 | − u32 *stable_pstate) | |
| 299 | +static u32 amdgpu_get_stable_pstate(struct amdgpu_device *adev) | |
| 299 | 300 | { |
| 300 | − struct amdgpu_device *adev = ctx->mgr->adev; | |
| 301 | − enum amd_dpm_forced_level current_level; | |
| 302 | − | |
| 303 | − current_level = amdgpu_dpm_get_performance_level(adev); | |
| 304 | − | |
| 305 | − switch (current_level) { | |
| 301 | + switch (amdgpu_dpm_get_performance_level(adev)) { | |
| 306 | 302 | case AMD_DPM_FORCED_LEVEL_PROFILE_STANDARD: |
| 307 | − *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_STANDARD; | |
| 308 | − break; | |
| 303 | + return AMDGPU_CTX_STABLE_PSTATE_STANDARD; | |
| 309 | 304 | case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_SCLK: |
| 310 | − *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK; | |
| 311 | − break; | |
| 305 | + return AMDGPU_CTX_STABLE_PSTATE_MIN_SCLK; | |
| 312 | 306 | case AMD_DPM_FORCED_LEVEL_PROFILE_MIN_MCLK: |
| 313 | − *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK; | |
| 314 | − break; | |
| 307 | + return AMDGPU_CTX_STABLE_PSTATE_MIN_MCLK; | |
| 315 | 308 | case AMD_DPM_FORCED_LEVEL_PROFILE_PEAK: |
| 316 | − *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_PEAK; | |
| 317 | − break; | |
| 309 | + return AMDGPU_CTX_STABLE_PSTATE_PEAK; | |
| 318 | 310 | default: |
| 319 | − *stable_pstate = AMDGPU_CTX_STABLE_PSTATE_NONE; | |
| 320 | − break; | |
| 311 | + return AMDGPU_CTX_STABLE_PSTATE_NONE; | |
| 321 | 312 | } |
| 322 | − return 0; | |
| 323 | 313 | } |
| 324 | 314 | |
| 325 | 315 | static int amdgpu_ctx_init(struct amdgpu_ctx_mgr *mgr, int32_t priority, |
| @@ -383,9 +373,9 @@ static int __amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, · 57 unmodified lines | ||
| 383 | 373 | if (current_ctx && current_ctx != ctx) |
| 384 | 374 | return -EBUSY; |
| 385 | 375 | |
| 386 | − r = amdgpu_ctx_get_stable_pstate(ctx, ¤t_stable_pstate); | |
| 387 | − if (r || current_stable_pstate == stable_pstate) | |
| 388 | − return r; | |
| 376 | + current_stable_pstate = amdgpu_get_stable_pstate(adev); | |
| 377 | + if (current_stable_pstate == stable_pstate) | |
| 378 | + return 0; | |
| 389 | 379 | |
| 390 | 380 | r = amdgpu_dpm_force_performance_level(adev, level); |
| 391 | 381 | if (r) |
| @@ -416,7 +406,7 @@ static int amdgpu_ctx_set_stable_pstate(struct amdgpu_ctx *ctx, · 24 unmodified lines | ||
| 416 | 406 | return r; |
| 417 | 407 | } |
| 418 | 408 | |
| 419 | −static void amdgpu_ctx_fini(struct kref *ref) | |
| 409 | +void amdgpu_ctx_fini(struct kref *ref) | |
| 420 | 410 | { |
| 421 | 411 | struct amdgpu_ctx *ctx = container_of(ref, struct amdgpu_ctx, refcount); |
| 422 | 412 | struct amdgpu_ctx_mgr *mgr = ctx->mgr; |
| @@ -504,53 +494,26 @@ static int amdgpu_ctx_alloc(struct amdgpu_device *adev, · 81 unmodified lines | ||
| 504 | 494 | if (!ctx) |
| 505 | 495 | return -ENOMEM; |
| 506 | 496 | |
| 507 | − mutex_lock(&mgr->lock); | |
| 508 | − r = idr_alloc(&mgr->ctx_handles, ctx, 1, AMDGPU_VM_MAX_NUM_CTX, GFP_KERNEL); | |
| 509 | − if (r < 0) { | |
| 510 | − mutex_unlock(&mgr->lock); | |
| 511 | − kfree(ctx); | |
| 512 | − return r; | |
| 513 | − } | |
| 514 | − | |
| 515 | − *id = (uint32_t)r; | |
| 516 | 497 | r = amdgpu_ctx_init(mgr, priority, filp, ctx); |
| 517 | 498 | if (r) { |
| 518 | − idr_remove(&mgr->ctx_handles, *id); | |
| 519 | − *id = 0; | |
| 520 | 499 | kfree(ctx); |
| 500 | + return r; | |
| 521 | 501 | } |
| 522 | − mutex_unlock(&mgr->lock); | |
| 523 | − return r; | |
| 524 | −} | |
| 525 | − | |
| 526 | −static void amdgpu_ctx_do_release(struct kref *ref) | |
| 527 | −{ | |
| 528 | − struct amdgpu_ctx *ctx; | |
| 529 | − u32 i, j; | |
| 530 | 502 | |
| 531 | − ctx = container_of(ref, struct amdgpu_ctx, refcount); | |
| 532 | − for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { | |
| 533 | − for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { | |
| 534 | − if (!ctx->entities[i][j]) | |
| 535 | − continue; | |
| 536 | − | |
| 537 | − drm_sched_entity_destroy(&ctx->entities[i][j]->entity); | |
| 538 | − } | |
| 539 | − } | |
| 503 | + r = xa_alloc(&mgr->ctx_handles, id, ctx, xa_limit_32b, GFP_KERNEL); | |
| 504 | + if (r) | |
| 505 | + amdgpu_ctx_put(ctx); | |
| 540 | 506 | |
| 541 | − amdgpu_ctx_fini(ref); | |
| 507 | + return r; | |
| 542 | 508 | } |
| 543 | 509 | |
| 544 | 510 | static int amdgpu_ctx_free(struct amdgpu_fpriv *fpriv, uint32_t id) |
| 545 | 511 | { |
| 546 | − struct amdgpu_ctx_mgr *mgr = &fpriv->ctx_mgr; | |
| 547 | 512 | struct amdgpu_ctx *ctx; |
| 548 | 513 | |
| 549 | − mutex_lock(&mgr->lock); | |
| 550 | − ctx = idr_remove(&mgr->ctx_handles, id); | |
| 551 | − if (ctx) | |
| 552 | − kref_put(&ctx->refcount, amdgpu_ctx_do_release); | |
| 553 | − mutex_unlock(&mgr->lock); | |
| 514 | + ctx = xa_erase(&fpriv->ctx_mgr.ctx_handles, id); | |
| 515 | + amdgpu_ctx_put(ctx); | |
| 516 | + | |
| 554 | 517 | return ctx ? 0 : -EINVAL; |
| 555 | 518 | } |
| 556 | 519 | |
| @@ -559,19 +522,11 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev, · 2 unmodified lines | ||
| 559 | 522 | union drm_amdgpu_ctx_out *out) |
| 560 | 523 | { |
| 561 | 524 | struct amdgpu_ctx *ctx; |
| 562 | − struct amdgpu_ctx_mgr *mgr; | |
| 563 | 525 | unsigned reset_counter; |
| 564 | 526 | |
| 565 | − if (!fpriv) | |
| 566 | − return -EINVAL; | |
| 567 | − | |
| 568 | − mgr = &fpriv->ctx_mgr; | |
| 569 | − mutex_lock(&mgr->lock); | |
| 570 | − ctx = idr_find(&mgr->ctx_handles, id); | |
| 571 | − if (!ctx) { | |
| 572 | − mutex_unlock(&mgr->lock); | |
| 527 | + ctx = amdgpu_ctx_get(fpriv, id); | |
| 528 | + if (!ctx) | |
| 573 | 529 | return -EINVAL; |
| 574 | − } | |
| 575 | 530 | |
| 576 | 531 | /* TODO: these two are always zero */ |
| 577 | 532 | out->state.flags = 0x0; |
| @@ -586,7 +541,8 @@ static int amdgpu_ctx_query(struct amdgpu_device *adev, · 8 unmodified lines | ||
| 586 | 541 | out->state.reset_status = AMDGPU_CTX_UNKNOWN_RESET; |
| 587 | 542 | ctx->reset_counter_query = reset_counter; |
| 588 | 543 | |
| 589 | − mutex_unlock(&mgr->lock); | |
| 544 | + amdgpu_ctx_put(ctx); | |
| 545 | + | |
| 590 | 546 | return 0; |
| 591 | 547 | } |
| 592 | 548 | |
| @@ -619,18 +575,10 @@ static int amdgpu_ctx_query2(struct amdgpu_device *adev, · 26 unmodified lines | ||
| 619 | 575 | { |
| 620 | 576 | struct amdgpu_ras *con = amdgpu_ras_get_context(adev); |
| 621 | 577 | struct amdgpu_ctx *ctx; |
| 622 | − struct amdgpu_ctx_mgr *mgr; | |
| 623 | 578 | |
| 624 | − if (!fpriv) | |
| 625 | − return -EINVAL; | |
| 626 | − | |
| 627 | − mgr = &fpriv->ctx_mgr; | |
| 628 | − mutex_lock(&mgr->lock); | |
| 629 | − ctx = idr_find(&mgr->ctx_handles, id); | |
| 630 | − if (!ctx) { | |
| 631 | − mutex_unlock(&mgr->lock); | |
| 579 | + ctx = amdgpu_ctx_get(fpriv, id); | |
| 580 | + if (!ctx) | |
| 632 | 581 | return -EINVAL; |
| 633 | − } | |
| 634 | 582 | |
| 635 | 583 | out->state.flags = 0x0; |
| 636 | 584 | out->state.hangs = 0x0; |
| @@ -671,7 +619,8 @@ static int amdgpu_ctx_query2(struct amdgpu_device *adev, · 34 unmodified lines | ||
| 671 | 619 | msecs_to_jiffies(AMDGPU_RAS_COUNTE_DELAY_MS)); |
| 672 | 620 | } |
| 673 | 621 | |
| 674 | − mutex_unlock(&mgr->lock); | |
| 622 | + amdgpu_ctx_put(ctx); | |
| 623 | + | |
| 675 | 624 | return 0; |
| 676 | 625 | } |
| 677 | 626 | |
| @@ -680,26 +629,26 @@ static int amdgpu_ctx_stable_pstate(struct amdgpu_device *adev, · 2 unmodified lines | ||
| 680 | 629 | bool set, u32 *stable_pstate) |
| 681 | 630 | { |
| 682 | 631 | struct amdgpu_ctx *ctx; |
| 683 | − struct amdgpu_ctx_mgr *mgr; | |
| 684 | − int r; | |
| 632 | + int r = 0; | |
| 685 | 633 | |
| 686 | − if (!fpriv) | |
| 634 | + ctx = amdgpu_ctx_get(fpriv, id); | |
| 635 | + if (!ctx) | |
| 687 | 636 | return -EINVAL; |
| 688 | 637 | |
| 689 | − mgr = &fpriv->ctx_mgr; | |
| 690 | − mutex_lock(&mgr->lock); | |
| 691 | − ctx = idr_find(&mgr->ctx_handles, id); | |
| 692 | − if (!ctx) { | |
| 693 | − mutex_unlock(&mgr->lock); | |
| 694 | − return -EINVAL; | |
| 695 | − } | |
| 638 | + /* | |
| 639 | + * The get path is odd in this uapi - it will check whether the context | |
| 640 | + * id exist, but otherwise does nothing with it. In other words, the | |
| 641 | + * uapi has historically been implemented as being able to query the | |
| 642 | + * global device state, as long as the caller supplies a random valid | |
| 643 | + * context id. | |
| 644 | + */ | |
| 696 | 645 | |
| 697 | 646 | if (set) |
| 698 | 647 | r = amdgpu_ctx_set_stable_pstate(ctx, *stable_pstate); |
| 699 | 648 | else |
| 700 | − r = amdgpu_ctx_get_stable_pstate(ctx, stable_pstate); | |
| 649 | + *stable_pstate = amdgpu_get_stable_pstate(adev); | |
| 701 | 650 | |
| 702 | − mutex_unlock(&mgr->lock); | |
| 651 | + amdgpu_ctx_put(ctx); | |
| 703 | 652 | return r; |
| 704 | 653 | } |
| 705 | 654 | |
| @@ -778,23 +727,14 @@ struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id) · 72 unmodified lines | ||
| 778 | 727 | |
| 779 | 728 | mgr = &fpriv->ctx_mgr; |
| 780 | 729 | |
| 781 | − mutex_lock(&mgr->lock); | |
| 782 | − ctx = idr_find(&mgr->ctx_handles, id); | |
| 730 | + xa_lock(&mgr->ctx_handles); | |
| 731 | + ctx = xa_load(&mgr->ctx_handles, id); | |
| 783 | 732 | if (ctx) |
| 784 | 733 | kref_get(&ctx->refcount); |
| 785 | − mutex_unlock(&mgr->lock); | |
| 734 | + xa_unlock(&mgr->ctx_handles); | |
| 786 | 735 | return ctx; |
| 787 | 736 | } |
| 788 | 737 | |
| 789 | −int amdgpu_ctx_put(struct amdgpu_ctx *ctx) | |
| 790 | −{ | |
| 791 | − if (ctx == NULL) | |
| 792 | − return -EINVAL; | |
| 793 | − | |
| 794 | − kref_put(&ctx->refcount, amdgpu_ctx_do_release); | |
| 795 | − return 0; | |
| 796 | −} | |
| 797 | − | |
| 798 | 738 | uint64_t amdgpu_ctx_add_fence(struct amdgpu_ctx *ctx, |
| 799 | 739 | struct drm_sched_entity *entity, |
| 800 | 740 | struct dma_fence *fence) |
| @@ -928,8 +868,7 @@ void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr, · 127 unmodified lines | ||
| 928 | 868 | unsigned int i; |
| 929 | 869 | |
| 930 | 870 | mgr->adev = adev; |
| 931 | − mutex_init(&mgr->lock); | |
| 932 | − idr_init_base(&mgr->ctx_handles, 1); | |
| 871 | + xa_init_flags(&mgr->ctx_handles, XA_FLAGS_ALLOC1); | |
| 933 | 872 | |
| 934 | 873 | for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) |
| 935 | 874 | atomic64_set(&mgr->time_spend[i], 0); |
| @@ -938,13 +877,13 @@ void amdgpu_ctx_mgr_init(struct amdgpu_ctx_mgr *mgr, · 2 unmodified lines | ||
| 938 | 877 | long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout) |
| 939 | 878 | { |
| 940 | 879 | struct amdgpu_ctx *ctx; |
| 941 | − struct idr *idp; | |
| 942 | − uint32_t id, i, j; | |
| 880 | + unsigned long id; | |
| 881 | + int i, j; | |
| 943 | 882 | |
| 944 | − idp = &mgr->ctx_handles; | |
| 945 | − | |
| 946 | − mutex_lock(&mgr->lock); | |
| 947 | − idr_for_each_entry(idp, ctx, id) { | |
| 883 | + xa_lock(&mgr->ctx_handles); | |
| 884 | + xa_for_each(&mgr->ctx_handles, id, ctx) { | |
| 885 | + kref_get(&ctx->refcount); | |
| 886 | + xa_unlock(&mgr->ctx_handles); | |
| 948 | 887 | for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { |
| 949 | 888 | for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { |
| 950 | 889 | struct drm_sched_entity *entity; |
| @@ -956,45 +895,21 @@ long amdgpu_ctx_mgr_entity_flush(struct amdgpu_ctx_mgr *mgr, long timeout) · 5 unmodified lines | ||
| 956 | 895 | timeout = drm_sched_entity_flush(entity, timeout); |
| 957 | 896 | } |
| 958 | 897 | } |
| 898 | + amdgpu_ctx_put(ctx); | |
| 899 | + xa_lock(&mgr->ctx_handles); | |
| 959 | 900 | } |
| 960 | − mutex_unlock(&mgr->lock); | |
| 901 | + xa_unlock(&mgr->ctx_handles); | |
| 961 | 902 | return timeout; |
| 962 | 903 | } |
| 963 | 904 | |
| 964 | −static void amdgpu_ctx_mgr_entity_fini(struct amdgpu_ctx_mgr *mgr) | |
| 905 | +void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr) | |
| 965 | 906 | { |
| 966 | 907 | struct amdgpu_ctx *ctx; |
| 967 | − struct idr *idp; | |
| 968 | − uint32_t id, i, j; | |
| 969 | − | |
| 970 | − idp = &mgr->ctx_handles; | |
| 971 | − | |
| 972 | − idr_for_each_entry(idp, ctx, id) { | |
| 973 | − if (kref_read(&ctx->refcount) != 1) { | |
| 974 | − drm_err(adev_to_drm(mgr->adev), "ctx %p is still alive\n", ctx); | |
| 975 | − continue; | |
| 976 | − } | |
| 908 | + unsigned long id; | |
| 977 | 909 | |
| 978 | − for (i = 0; i < AMDGPU_HW_IP_NUM; ++i) { | |
| 979 | − for (j = 0; j < amdgpu_ctx_num_entities[i]; ++j) { | |
| 980 | − struct drm_sched_entity *entity; | |
| 981 | − | |
| 982 | − if (!ctx->entities[i][j]) | |
| 983 | − continue; | |
| 984 | − | |
| 985 | − entity = &ctx->entities[i][j]->entity; | |
| 986 | − drm_sched_entity_fini(entity); | |
| 987 | − } | |
| 988 | − } | |
| 989 | − kref_put(&ctx->refcount, amdgpu_ctx_fini); | |
| 990 | − } | |
| 991 | −} | |
| 992 | − | |
| 993 | −void amdgpu_ctx_mgr_fini(struct amdgpu_ctx_mgr *mgr) | |
| 994 | −{ | |
| 995 | − amdgpu_ctx_mgr_entity_fini(mgr); | |
| 996 | − idr_destroy(&mgr->ctx_handles); | |
| 997 | − mutex_destroy(&mgr->lock); | |
| 910 | + xa_for_each(&mgr->ctx_handles, id, ctx) | |
| 911 | + amdgpu_ctx_put(ctx); | |
| 912 | + xa_destroy(&mgr->ctx_handles); | |
| 998 | 913 | } |
| 999 | 914 | |
| 1000 | 915 | void amdgpu_ctx_mgr_usage(struct amdgpu_ctx_mgr *mgr, |
| @@ -1002,21 +917,21 @@ void amdgpu_ctx_mgr_usage(struct amdgpu_ctx_mgr *mgr, · 1 unmodified line | ||
| 1002 | 917 | { |
| 1003 | 918 | struct amdgpu_ctx *ctx; |
| 1004 | 919 | unsigned int hw_ip, i; |
| 1005 | − uint32_t id; | |
| 920 | + unsigned long id; | |
| 1006 | 921 | |
| 1007 | 922 | /* |
| 1008 | 923 | * This is a little bit racy because it can be that a ctx or a fence are |
| 1009 | 924 | * destroyed just in the moment we try to account them. But that is ok |
| 1010 | 925 | * since exactly that case is explicitely allowed by the interface. |
| 1011 | 926 | */ |
| 1012 | − mutex_lock(&mgr->lock); | |
| 1013 | 927 | for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) { |
| 1014 | 928 | uint64_t ns = atomic64_read(&mgr->time_spend[hw_ip]); |
| 1015 | 929 | |
| 1016 | 930 | usage[hw_ip] = ns_to_ktime(ns); |
| 1017 | 931 | } |
| 1018 | 932 | |
| 1019 | − idr_for_each_entry(&mgr->ctx_handles, ctx, id) { | |
| 933 | + xa_lock(&mgr->ctx_handles); | |
| 934 | + xa_for_each(&mgr->ctx_handles, id, ctx) { | |
| 1020 | 935 | for (hw_ip = 0; hw_ip < AMDGPU_HW_IP_NUM; ++hw_ip) { |
| 1021 | 936 | for (i = 0; i < amdgpu_ctx_num_entities[hw_ip]; ++i) { |
| 1022 | 937 | struct amdgpu_ctx_entity *centity; |
| @@ -1030,5 +945,5 @@ void amdgpu_ctx_mgr_usage(struct amdgpu_ctx_mgr *mgr, · 7 unmodified lines | ||
| 1030 | 945 | } |
| 1031 | 946 | } |
| 1032 | 947 | } |
| 1033 | − mutex_unlock(&mgr->lock); | |
| 948 | + xa_unlock(&mgr->ctx_handles); | |
| 1034 | 949 | } |
| @@ -25,6 +25,7 @@ · 24 unmodified lines | ||
| 25 | 25 | |
| 26 | 26 | #include <linux/ktime.h> |
| 27 | 27 | #include <linux/types.h> |
| 28 | +#include <linux/xarray.h> | |
| 28 | 29 | |
| 29 | 30 | #include "amdgpu_ring.h" |
| 30 | 31 | |
| @@ -60,16 +61,21 @@ struct amdgpu_ctx { · 29 unmodified lines | ||
| 60 | 61 | |
| 61 | 62 | struct amdgpu_ctx_mgr { |
| 62 | 63 | struct amdgpu_device *adev; |
| 63 | − struct mutex lock; | |
| 64 | − /* protected by lock */ | |
| 65 | − struct idr ctx_handles; | |
| 64 | + struct xarray ctx_handles; | |
| 66 | 65 | atomic64_t time_spend[AMDGPU_HW_IP_NUM]; |
| 67 | 66 | }; |
| 68 | 67 | |
| 69 | 68 | extern const unsigned int amdgpu_ctx_num_entities[AMDGPU_HW_IP_NUM]; |
| 70 | 69 | |
| 71 | 70 | struct amdgpu_ctx *amdgpu_ctx_get(struct amdgpu_fpriv *fpriv, uint32_t id); |
| 72 | −int amdgpu_ctx_put(struct amdgpu_ctx *ctx); | |
| 71 | + | |
| 72 | +void amdgpu_ctx_fini(struct kref *kref); | |
| 73 | + | |
| 74 | +static inline void amdgpu_ctx_put(struct amdgpu_ctx *ctx) | |
| 75 | +{ | |
| 76 | + if (ctx) | |
| 77 | + kref_put(&ctx->refcount, amdgpu_ctx_fini); | |
| 78 | +} | |
| 73 | 79 | |
| 74 | 80 | int amdgpu_ctx_get_entity(struct amdgpu_ctx *ctx, u32 hw_ip, u32 instance, |
| 75 | 81 | u32 ring, struct drm_sched_entity **entity); |
| @@ -26,6 +26,7 @@ · 25 unmodified lines | ||
| 26 | 26 | #include <linux/kthread.h> |
| 27 | 27 | #include <linux/pci.h> |
| 28 | 28 | #include <linux/uaccess.h> |
| 29 | +#include <linux/security.h> | |
| 29 | 30 | #include <linux/pm_runtime.h> |
| 30 | 31 | |
| 31 | 32 | #include "amdgpu.h" |
| @@ -1739,6 +1740,12 @@ int amdgpu_debugfs_regs_init(struct amdgpu_device *adev) · 1707 unmodified lines | ||
| 1739 | 1740 | struct dentry *ent, *root = minor->debugfs_root; |
| 1740 | 1741 | unsigned int i; |
| 1741 | 1742 | |
| 1743 | + if (security_locked_down(LOCKDOWN_PCI_ACCESS)) { | |
| 1744 | + drm_info(adev_to_drm(adev), | |
| 1745 | + "amdgpu: HW debugfs nodes disabled (kernel lockdown)\n"); | |
| 1746 | + return 0; | |
| 1747 | + } | |
| 1748 | + | |
| 1742 | 1749 | for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) { |
| 1743 | 1750 | ent = debugfs_create_file(debugfs_regs_names[i], |
| 1744 | 1751 | S_IFREG | 0400, root, |
| @@ -64,6 +64,7 @@ const char *hw_ip_names[MAX_HWIP] = { · 63 unmodified lines | ||
| 64 | 64 | [VCN1_HWIP] = "VCN1", |
| 65 | 65 | [VCE_HWIP] = "VCE", |
| 66 | 66 | [VPE_HWIP] = "VPE", |
| 67 | + [UMSCH_HWIP] = "UMSCH", | |
| 67 | 68 | [DF_HWIP] = "DF", |
| 68 | 69 | [DCE_HWIP] = "DCE", |
| 69 | 70 | [OSSSYS_HWIP] = "OSSSYS", |
| @@ -74,6 +74,7 @@ · 73 unmodified lines | ||
| 74 | 74 | #include "amdgpu_ras.h" |
| 75 | 75 | #include "amdgpu_ras_mgr.h" |
| 76 | 76 | #include "amdgpu_pmu.h" |
| 77 | +#include "amdgpu_smu.h" | |
| 77 | 78 | #include "amdgpu_fru_eeprom.h" |
| 78 | 79 | #include "amdgpu_reset.h" |
| 79 | 80 | #include "amdgpu_virt.h" |
| @@ -2130,6 +2131,8 @@ static int amdgpu_device_ip_early_init(struct amdgpu_device *adev) · 2050 unmodified lines | ||
| 2130 | 2131 | adev->cg_flags &= amdgpu_cg_mask; |
| 2131 | 2132 | adev->pg_flags &= amdgpu_pg_mask; |
| 2132 | 2133 | |
| 2134 | + amdgpu_smu_early_init(adev); | |
| 2135 | + | |
| 2133 | 2136 | return 0; |
| 2134 | 2137 | } |
| 2135 | 2138 | |
| @@ -3677,6 +3680,10 @@ static void amdgpu_device_sys_interface_fini(struct amdgpu_device *adev) · 1541 unmodified lines | ||
| 3677 | 3680 | amdgpu_pm_sysfs_fini(adev); |
| 3678 | 3681 | if (adev->ucode_sysfs_en) |
| 3679 | 3682 | amdgpu_ucode_sysfs_fini(adev); |
| 3683 | + | |
| 3684 | + amdgpu_discovery_sysfs_fini(adev); | |
| 3685 | + amdgpu_preempt_mgr_sysfs_fini(adev); | |
| 3686 | + | |
| 3680 | 3687 | amdgpu_device_attr_sysfs_fini(adev); |
| 3681 | 3688 | amdgpu_fru_sysfs_fini(adev); |
| 3682 | 3689 | |
| @@ -3773,6 +3780,7 @@ int amdgpu_device_init(struct amdgpu_device *adev, · 90 unmodified lines | ||
| 3773 | 3780 | |
| 3774 | 3781 | spin_lock_init(&adev->irq.lock); |
| 3775 | 3782 | |
| 3783 | + amdgpu_early_init_rlc_reg_funcs(adev); | |
| 3776 | 3784 | amdgpu_device_init_apu_flags(adev); |
| 3777 | 3785 | |
| 3778 | 3786 | r = amdgpu_device_check_arguments(adev); |
| @@ -4208,6 +4216,7 @@ void amdgpu_device_fini_hw(struct amdgpu_device *adev) · 429 unmodified lines | ||
| 4208 | 4216 | |
| 4209 | 4217 | if (adev->mman.initialized) |
| 4210 | 4218 | drain_workqueue(adev->mman.bdev.wq); |
| 4219 | + | |
| 4211 | 4220 | adev->shutdown = true; |
| 4212 | 4221 | |
| 4213 | 4222 | unregister_pm_notifier(&adev->pm_nb); |
| @@ -4706,161 +4715,6 @@ exit: · 492 unmodified lines | ||
| 4706 | 4715 | return 0; |
| 4707 | 4716 | } |
| 4708 | 4717 | |
| 4709 | −/** | |
| 4710 | − * amdgpu_device_ip_check_soft_reset - did soft reset succeed | |
| 4711 | − * | |
| 4712 | − * @adev: amdgpu_device pointer | |
| 4713 | − * | |
| 4714 | − * The list of all the hardware IPs that make up the asic is walked and | |
| 4715 | − * the check_soft_reset callbacks are run. check_soft_reset determines | |
| 4716 | − * if the asic is still hung or not. | |
| 4717 | − * Returns true if any of the IPs are still in a hung state, false if not. | |
| 4718 | − */ | |
| 4719 | −static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev) | |
| 4720 | −{ | |
| 4721 | − int i; | |
| 4722 | − bool asic_hang = false; | |
| 4723 | − | |
| 4724 | − if (amdgpu_sriov_vf(adev)) | |
| 4725 | − return true; | |
| 4726 | − | |
| 4727 | − if (amdgpu_asic_need_full_reset(adev)) | |
| 4728 | − return true; | |
| 4729 | − | |
| 4730 | − for (i = 0; i < adev->num_ip_blocks; i++) { | |
| 4731 | − if (!adev->ip_blocks[i].status.valid) | |
| 4732 | − continue; | |
| 4733 | − if (adev->ip_blocks[i].version->funcs->check_soft_reset) | |
| 4734 | − adev->ip_blocks[i].status.hang = | |
| 4735 | − adev->ip_blocks[i].version->funcs->check_soft_reset( | |
| 4736 | − &adev->ip_blocks[i]); | |
| 4737 | − if (adev->ip_blocks[i].status.hang) { | |
| 4738 | − dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name); | |
| 4739 | − asic_hang = true; | |
| 4740 | − } | |
| 4741 | − } | |
| 4742 | − return asic_hang; | |
| 4743 | −} | |
| 4744 | − | |
| 4745 | −/** | |
| 4746 | − * amdgpu_device_ip_pre_soft_reset - prepare for soft reset | |
| 4747 | − * | |
| 4748 | − * @adev: amdgpu_device pointer | |
| 4749 | − * | |
| 4750 | − * The list of all the hardware IPs that make up the asic is walked and the | |
| 4751 | − * pre_soft_reset callbacks are run if the block is hung. pre_soft_reset | |
| 4752 | − * handles any IP specific hardware or software state changes that are | |
| 4753 | − * necessary for a soft reset to succeed. | |
| 4754 | − * Returns 0 on success, negative error code on failure. | |
| 4755 | − */ | |
| 4756 | −static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev) | |
| 4757 | −{ | |
| 4758 | − int i, r = 0; | |
| 4759 | − | |
| 4760 | − for (i = 0; i < adev->num_ip_blocks; i++) { | |
| 4761 | − if (!adev->ip_blocks[i].status.valid) | |
| 4762 | − continue; | |
| 4763 | − if (adev->ip_blocks[i].status.hang && | |
| 4764 | − adev->ip_blocks[i].version->funcs->pre_soft_reset) { | |
| 4765 | − r = adev->ip_blocks[i].version->funcs->pre_soft_reset(&adev->ip_blocks[i]); | |
| 4766 | − if (r) | |
| 4767 | − return r; | |
| 4768 | − } | |
| 4769 | − } | |
| 4770 | − | |
| 4771 | − return 0; | |
| 4772 | −} | |
| 4773 | − | |
| 4774 | −/** | |
| 4775 | − * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed | |
| 4776 | − * | |
| 4777 | − * @adev: amdgpu_device pointer | |
| 4778 | − * | |
| 4779 | − * Some hardware IPs cannot be soft reset. If they are hung, a full gpu | |
| 4780 | − * reset is necessary to recover. | |
| 4781 | − * Returns true if a full asic reset is required, false if not. | |
| 4782 | − */ | |
| 4783 | −static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev) | |
| 4784 | −{ | |
| 4785 | − int i; | |
| 4786 | − | |
| 4787 | − if (amdgpu_asic_need_full_reset(adev)) | |
| 4788 | − return true; | |
| 4789 | − | |
| 4790 | − for (i = 0; i < adev->num_ip_blocks; i++) { | |
| 4791 | − if (!adev->ip_blocks[i].status.valid) | |
| 4792 | − continue; | |
| 4793 | − if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) || | |
| 4794 | − (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) || | |
| 4795 | − (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) || | |
| 4796 | − (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) || | |
| 4797 | − adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) { | |
| 4798 | − if (adev->ip_blocks[i].status.hang) { | |
| 4799 | − dev_info(adev->dev, "Some block need full reset!\n"); | |
| 4800 | − return true; | |
| 4801 | − } | |
| 4802 | − } | |
| 4803 | − } | |
| 4804 | − return false; | |
| 4805 | −} | |
| 4806 | − | |
| 4807 | −/** | |
| 4808 | − * amdgpu_device_ip_soft_reset - do a soft reset | |
| 4809 | − * | |
| 4810 | − * @adev: amdgpu_device pointer | |
| 4811 | − * | |
| 4812 | − * The list of all the hardware IPs that make up the asic is walked and the | |
| 4813 | − * soft_reset callbacks are run if the block is hung. soft_reset handles any | |
| 4814 | − * IP specific hardware or software state changes that are necessary to soft | |
| 4815 | − * reset the IP. | |
| 4816 | − * Returns 0 on success, negative error code on failure. | |
| 4817 | − */ | |
| 4818 | −static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev) | |
| 4819 | −{ | |
| 4820 | − int i, r = 0; | |
| 4821 | − | |
| 4822 | − for (i = 0; i < adev->num_ip_blocks; i++) { | |
| 4823 | − if (!adev->ip_blocks[i].status.valid) | |
| 4824 | − continue; | |
| 4825 | − if (adev->ip_blocks[i].status.hang && | |
| 4826 | − adev->ip_blocks[i].version->funcs->soft_reset) { | |
| 4827 | − r = adev->ip_blocks[i].version->funcs->soft_reset(&adev->ip_blocks[i]); | |
| 4828 | − if (r) | |
| 4829 | − return r; | |
| 4830 | − } | |
| 4831 | − } | |
| 4832 | − | |
| 4833 | − return 0; | |
| 4834 | −} | |
| 4835 | − | |
| 4836 | −/** | |
| 4837 | − * amdgpu_device_ip_post_soft_reset - clean up from soft reset | |
| 4838 | − * | |
| 4839 | − * @adev: amdgpu_device pointer | |
| 4840 | − * | |
| 4841 | − * The list of all the hardware IPs that make up the asic is walked and the | |
| 4842 | − * post_soft_reset callbacks are run if the asic was hung. post_soft_reset | |
| 4843 | − * handles any IP specific hardware or software state changes that are | |
| 4844 | − * necessary after the IP has been soft reset. | |
| 4845 | − * Returns 0 on success, negative error code on failure. | |
| 4846 | − */ | |
| 4847 | −static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev) | |
| 4848 | −{ | |
| 4849 | − int i, r = 0; | |
| 4850 | − | |
| 4851 | − for (i = 0; i < adev->num_ip_blocks; i++) { | |
| 4852 | − if (!adev->ip_blocks[i].status.valid) | |
| 4853 | − continue; | |
| 4854 | − if (adev->ip_blocks[i].status.hang && | |
| 4855 | − adev->ip_blocks[i].version->funcs->post_soft_reset) | |
| 4856 | − r = adev->ip_blocks[i].version->funcs->post_soft_reset(&adev->ip_blocks[i]); | |
| 4857 | − if (r) | |
| 4858 | − return r; | |
| 4859 | − } | |
| 4860 | − | |
| 4861 | − return 0; | |
| 4862 | −} | |
| 4863 | − | |
| 4864 | 4718 | /** |
| 4865 | 4719 | * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf |
| 4866 | 4720 | * |
| @@ -5152,20 +5006,7 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, · 285 unmodified lines | ||
| 5152 | 5006 | |
| 5153 | 5007 | /* Don't suspend on bare metal if we are not going to HW reset the ASIC */ |
| 5154 | 5008 | if (!amdgpu_sriov_vf(adev)) { |
| 5155 | − | |
| 5156 | − if (!need_full_reset) | |
| 5157 | − need_full_reset = amdgpu_device_ip_need_full_reset(adev); | |
| 5158 | − | |
| 5159 | − if (!need_full_reset && amdgpu_gpu_recovery && | |
| 5160 | − amdgpu_device_ip_check_soft_reset(adev)) { | |
| 5161 | − amdgpu_device_ip_pre_soft_reset(adev); | |
| 5162 | − r = amdgpu_device_ip_soft_reset(adev); | |
| 5163 | − amdgpu_device_ip_post_soft_reset(adev); | |
| 5164 | − if (r || amdgpu_device_ip_check_soft_reset(adev)) { | |
| 5165 | − dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n"); | |
| 5166 | − need_full_reset = true; | |
| 5167 | − } | |
| 5168 | − } | |
| 5009 | + need_full_reset = true; | |
| 5169 | 5010 | |
| 5170 | 5011 | if (!test_bit(AMDGPU_SKIP_COREDUMP, &reset_context->flags)) { |
| 5171 | 5012 | dev_info(tmp_adev->dev, "Dumping IP State\n"); |
| @@ -5618,8 +5459,7 @@ static void amdgpu_device_halt_activities(struct amdgpu_device *adev, · 446 unmodified lines | ||
| 5618 | 5459 | drm_client_dev_suspend(adev_to_drm(tmp_adev)); |
| 5619 | 5460 | |
| 5620 | 5461 | /* disable ras on ALL IPs */ |
| 5621 | − if (!need_emergency_restart && !amdgpu_reset_in_dpc(adev) && | |
| 5622 | − amdgpu_device_ip_need_full_reset(tmp_adev)) | |
| 5462 | + if (!need_emergency_restart && !amdgpu_reset_in_dpc(adev)) | |
| 5623 | 5463 | amdgpu_ras_suspend(tmp_adev); |
| 5624 | 5464 | |
| 5625 | 5465 | amdgpu_userq_pre_reset(tmp_adev); |
| @@ -6891,7 +6731,7 @@ ssize_t amdgpu_get_soft_full_reset_mask(struct amdgpu_ring *ring) · 1265 unmodified lines | ||
| 6891 | 6731 | |
| 6892 | 6732 | if (unlikely(!ring->adev->debug_disable_soft_recovery) && |
| 6893 | 6733 | !amdgpu_sriov_vf(ring->adev) && ring->funcs->soft_recovery) |
| 6894 | − size |= AMDGPU_RESET_TYPE_SOFT_RESET; | |
| 6734 | + size |= AMDGPU_RESET_TYPE_SOFT_RECOVERY; | |
| 6895 | 6735 | |
| 6896 | 6736 | return size; |
| 6897 | 6737 | } |
| @@ -6907,8 +6747,8 @@ ssize_t amdgpu_show_reset_mask(char *buf, uint32_t supported_reset) · 9 unmodified lines | ||
| 6907 | 6747 | |
| 6908 | 6748 | } |
| 6909 | 6749 | |
| 6910 | − if (supported_reset & AMDGPU_RESET_TYPE_SOFT_RESET) | |
| 6911 | − size += sysfs_emit_at(buf, size, "soft "); | |
| 6750 | + if (supported_reset & AMDGPU_RESET_TYPE_SOFT_RECOVERY) | |
| 6751 | + size += sysfs_emit_at(buf, size, "soft_recovery "); | |
| 6912 | 6752 | |
| 6913 | 6753 | if (supported_reset & AMDGPU_RESET_TYPE_PER_QUEUE) |
| 6914 | 6754 | size += sysfs_emit_at(buf, size, "queue "); |
| @@ -22,6 +22,7 @@ · 21 unmodified lines | ||
| 22 | 22 | */ |
| 23 | 23 | |
| 24 | 24 | #include <linux/firmware.h> |
| 25 | +#include <linux/kernfs.h> | |
| 25 | 26 | |
| 26 | 27 | #include "amdgpu.h" |
| 27 | 28 | #include "amdgpu_discovery.h" |
| @@ -148,6 +149,26 @@ MODULE_FIRMWARE("amdgpu/aldebaran_ip_discovery.bin"); · 120 unmodified lines | ||
| 148 | 149 | #define mmDRIVER_SCRATCH_1 0x95 |
| 149 | 150 | #define mmDRIVER_SCRATCH_2 0x96 |
| 150 | 151 | |
| 152 | +struct ip_discovery_top { | |
| 153 | + struct kobject kobj; | |
| 154 | + struct kset die_kset; | |
| 155 | + struct pci_dev *pdev; | |
| 156 | + struct amdgpu_device *adev; | |
| 157 | + uint8_t *discovery_bin; | |
| 158 | + uint32_t bin_size; | |
| 159 | + bool standalone_mode; | |
| 160 | +}; | |
| 161 | + | |
| 162 | +/* List to track early-initialized ip_discovery_top entries */ | |
| 163 | +struct early_ip_discovery { | |
| 164 | + struct list_head list; | |
| 165 | + struct pci_dev *pdev; | |
| 166 | + struct ip_discovery_top *ip_top; | |
| 167 | +}; | |
| 168 | + | |
| 169 | +static LIST_HEAD(early_ip_discovery_list); | |
| 170 | +static DEFINE_MUTEX(early_ip_discovery_mutex); | |
| 171 | + | |
| 151 | 172 | static const char *hw_id_names[HW_ID_MAX] = { |
| 152 | 173 | [MP1_HWID] = "MP1", |
| 153 | 174 | [MP2_HWID] = "MP2", |
| @@ -226,6 +247,7 @@ static const char *hw_id_names[HW_ID_MAX] = { · 72 unmodified lines | ||
| 226 | 247 | [XGBE_HWID] = "XGBE", |
| 227 | 248 | [MP0_HWID] = "MP0", |
| 228 | 249 | [VPE_HWID] = "VPE", |
| 250 | + [UMSCH_HWID] = "UMSCH", | |
| 229 | 251 | [ATU_HWID] = "ATU", |
| 230 | 252 | [AIGC_HWID] = "AIGC", |
| 231 | 253 | }; |
| @@ -258,6 +280,7 @@ static int hw_id_map[MAX_HWIP] = { · 26 unmodified lines | ||
| 258 | 280 | [DCI_HWIP] = DCI_HWID, |
| 259 | 281 | [PCIE_HWIP] = PCIE_HWID, |
| 260 | 282 | [VPE_HWIP] = VPE_HWID, |
| 283 | + [UMSCH_HWIP] = UMSCH_HWID, | |
| 261 | 284 | [ISP_HWIP] = ISP_HWID, |
| 262 | 285 | [ATU_HWIP] = ATU_HWID, |
| 263 | 286 | }; |
| @@ -542,25 +565,37 @@ static const char *amdgpu_discovery_get_fw_name(struct amdgpu_device *adev) · 278 unmodified lines | ||
| 542 | 565 | } |
| 543 | 566 | } |
| 544 | 567 | |
| 545 | −static int amdgpu_discovery_get_table_info(struct amdgpu_device *adev, | |
| 546 | − struct table_info **info, | |
| 547 | − uint16_t table_id) | |
| 568 | +static struct table_info * | |
| 569 | +amdgpu_discovery_get_table_info_from_bin(uint8_t *discovery_bin, | |
| 570 | + uint16_t table_id) | |
| 548 | 571 | { |
| 549 | − struct binary_header *bhdr = | |
| 550 | − (struct binary_header *)adev->discovery.bin; | |
| 572 | + struct binary_header *bhdr = (struct binary_header *)discovery_bin; | |
| 551 | 573 | struct binary_header_v2 *bhdrv2; |
| 552 | 574 | |
| 553 | 575 | switch (bhdr->version_major) { |
| 554 | 576 | case 2: |
| 555 | − bhdrv2 = (struct binary_header_v2 *)adev->discovery.bin; | |
| 556 | − *info = &bhdrv2->table_list[table_id]; | |
| 557 | − break; | |
| 577 | + bhdrv2 = (struct binary_header_v2 *)discovery_bin; | |
| 578 | + return &bhdrv2->table_list[table_id]; | |
| 558 | 579 | case 1: |
| 559 | 580 | case 0: |
| 560 | − *info = &bhdr->table_list[table_id]; | |
| 561 | − break; | |
| 581 | + return &bhdr->table_list[table_id]; | |
| 562 | 582 | default: |
| 563 | − dev_err(adev->dev, "Invalid ip discovery table version %d\n",bhdr->version_major); | |
| 583 | + return NULL; | |
| 584 | + } | |
| 585 | +} | |
| 586 | + | |
| 587 | +static int amdgpu_discovery_get_table_info(struct amdgpu_device *adev, | |
| 588 | + struct table_info **info, | |
| 589 | + uint16_t table_id) | |
| 590 | +{ | |
| 591 | + struct binary_header *bhdr = | |
| 592 | + (struct binary_header *)adev->discovery.bin; | |
| 593 | + | |
| 594 | + *info = amdgpu_discovery_get_table_info_from_bin(adev->discovery.bin, | |
| 595 | + table_id); | |
| 596 | + if (!*info) { | |
| 597 | + dev_err(adev->dev, "Invalid ip discovery table version %d\n", | |
| 598 | + bhdr->version_major); | |
| 564 | 599 | return -EINVAL; |
| 565 | 600 | } |
| 566 | 601 | |
| @@ -724,11 +759,11 @@ out: · 157 unmodified lines | ||
| 724 | 759 | return r; |
| 725 | 760 | } |
| 726 | 761 | |
| 727 | −static void amdgpu_discovery_sysfs_fini(struct amdgpu_device *adev); | |
| 728 | − | |
| 729 | 762 | void amdgpu_discovery_fini(struct amdgpu_device *adev) |
| 730 | 763 | { |
| 731 | − amdgpu_discovery_sysfs_fini(adev); | |
| 764 | + if (adev->discovery.ip_top && !adev->discovery.ip_top->standalone_mode) | |
| 765 | + amdgpu_discovery_sysfs_fini(adev); | |
| 766 | + | |
| 732 | 767 | kfree(adev->discovery.bin); |
| 733 | 768 | adev->discovery.bin = NULL; |
| 734 | 769 | } |
| @@ -737,15 +772,17 @@ static int amdgpu_discovery_validate_ip(struct amdgpu_device *adev, · 2 unmodified lines | ||
| 737 | 772 | uint8_t instance, uint16_t hw_id) |
| 738 | 773 | { |
| 739 | 774 | if (instance >= HWIP_MAX_INSTANCE) { |
| 740 | − dev_err(adev->dev, | |
| 741 | − "Unexpected instance_number (%d) from ip discovery blob\n", | |
| 742 | − instance); | |
| 775 | + if (adev) | |
| 776 | + dev_err(adev->dev, | |
| 777 | + "Unexpected instance_number (%d) from ip discovery blob\n", | |
| 778 | + instance); | |
| 743 | 779 | return -EINVAL; |
| 744 | 780 | } |
| 745 | 781 | if (hw_id >= HW_ID_MAX) { |
| 746 | − dev_err(adev->dev, | |
| 747 | − "Unexpected hw_id (%d) from ip discovery blob\n", | |
| 748 | − hw_id); | |
| 782 | + if (adev) | |
| 783 | + dev_err(adev->dev, | |
| 784 | + "Unexpected hw_id (%d) from ip discovery blob\n", | |
| 785 | + hw_id); | |
| 749 | 786 | return -EINVAL; |
| 750 | 787 | } |
| 751 | 788 | |
| @@ -1111,12 +1148,6 @@ static const struct kobj_type ip_discovery_ktype = { · 359 unmodified lines | ||
| 1111 | 1148 | .sysfs_ops = &kobj_sysfs_ops, |
| 1112 | 1149 | }; |
| 1113 | 1150 | |
| 1114 | −struct ip_discovery_top { | |
| 1115 | − struct kobject kobj; /* ip_discovery/ */ | |
| 1116 | − struct kset die_kset; /* ip_discovery/die/, contains ip_die_entry */ | |
| 1117 | − struct amdgpu_device *adev; | |
| 1118 | −}; | |
| 1119 | − | |
| 1120 | 1151 | static void die_kobj_release(struct kobject *kobj) |
| 1121 | 1152 | { |
| 1122 | 1153 | struct ip_discovery_top *ip_top = container_of(to_kset(kobj), |
| @@ -1132,8 +1163,14 @@ static void ip_disc_release(struct kobject *kobj) · 9 unmodified lines | ||
| 1132 | 1163 | kobj); |
| 1133 | 1164 | struct amdgpu_device *adev = ip_top->adev; |
| 1134 | 1165 | |
| 1166 | + /* In standalone mode, discovery_bin is managed by devm and will be | |
| 1167 | + * freed automatically when the PCI device is removed. Do not manually | |
| 1168 | + * free it here to avoid double-free. | |
| 1169 | + */ | |
| 1170 | + | |
| 1135 | 1171 | kfree(ip_top); |
| 1136 | − adev->discovery.ip_top = NULL; | |
| 1172 | + if (adev) | |
| 1173 | + adev->discovery.ip_top = NULL; | |
| 1137 | 1174 | } |
| 1138 | 1175 | |
| 1139 | 1176 | static uint8_t amdgpu_discovery_get_harvest_info(struct amdgpu_device *adev, |
| @@ -1141,6 +1178,10 @@ static uint8_t amdgpu_discovery_get_harvest_info(struct amdgpu_device *adev, · 1 unmodified line | ||
| 1141 | 1178 | { |
| 1142 | 1179 | uint8_t harvest = 0; |
| 1143 | 1180 | |
| 1181 | + /* In early init mode (adev == NULL), harvest info is not available */ | |
| 1182 | + if (!adev) | |
| 1183 | + return 0; | |
| 1184 | + | |
| 1144 | 1185 | /* Until a uniform way is figured, get mask based on hwid */ |
| 1145 | 1186 | switch (hw_id) { |
| 1146 | 1187 | case VCN_HWID: |
| @@ -1169,11 +1210,14 @@ static uint8_t amdgpu_discovery_get_harvest_info(struct amdgpu_device *adev, · 22 unmodified lines | ||
| 1169 | 1210 | } |
| 1170 | 1211 | |
| 1171 | 1212 | static int amdgpu_discovery_sysfs_ips(struct amdgpu_device *adev, |
| 1213 | + struct ip_discovery_top *ip_top, | |
| 1172 | 1214 | struct ip_die_entry *ip_die_entry, |
| 1173 | 1215 | const size_t _ip_offset, const int num_ips, |
| 1174 | 1216 | bool reg_base_64) |
| 1175 | 1217 | { |
| 1176 | − uint8_t *discovery_bin = adev->discovery.bin; | |
| 1218 | + uint8_t *discovery_bin = ip_top->standalone_mode ? | |
| 1219 | + ip_top->discovery_bin : | |
| 1220 | + adev->discovery.bin; | |
| 1177 | 1221 | int ii, jj, kk, res; |
| 1178 | 1222 | uint16_t hw_id; |
| 1179 | 1223 | uint8_t inst; |
| @@ -1270,10 +1314,12 @@ next_ip: · 90 unmodified lines | ||
| 1270 | 1314 | return 0; |
| 1271 | 1315 | } |
| 1272 | 1316 | |
| 1273 | −static int amdgpu_discovery_sysfs_recurse(struct amdgpu_device *adev) | |
| 1317 | +static int amdgpu_discovery_sysfs_recurse(struct amdgpu_device *adev, | |
| 1318 | + struct ip_discovery_top *ip_top) | |
| 1274 | 1319 | { |
| 1275 | − struct ip_discovery_top *ip_top = adev->discovery.ip_top; | |
| 1276 | − uint8_t *discovery_bin = adev->discovery.bin; | |
| 1320 | + uint8_t *discovery_bin = ip_top->standalone_mode ? | |
| 1321 | + ip_top->discovery_bin : | |
| 1322 | + adev->discovery.bin; | |
| 1277 | 1323 | struct table_info *info; |
| 1278 | 1324 | struct ip_discovery_header *ihdr; |
| 1279 | 1325 | struct die_header *dhdr; |
| @@ -1282,9 +1328,10 @@ static int amdgpu_discovery_sysfs_recurse(struct amdgpu_device *adev) · 2 unmodified lines | ||
| 1282 | 1328 | size_t ip_offset; |
| 1283 | 1329 | int ii, res; |
| 1284 | 1330 | |
| 1285 | − res = amdgpu_discovery_get_table_info(adev, &info, IP_DISCOVERY); | |
| 1286 | − if (res) | |
| 1287 | − return res; | |
| 1331 | + info = amdgpu_discovery_get_table_info_from_bin(discovery_bin, | |
| 1332 | + IP_DISCOVERY); | |
| 1333 | + if (!info) | |
| 1334 | + return -EINVAL; | |
| 1288 | 1335 | ihdr = (struct ip_discovery_header |
| 1289 | 1336 | *)(discovery_bin + |
| 1290 | 1337 | le16_to_cpu(info->offset)); |
| @@ -1322,7 +1369,8 @@ static int amdgpu_discovery_sysfs_recurse(struct amdgpu_device *adev) · 31 unmodified lines | ||
| 1322 | 1369 | return res; |
| 1323 | 1370 | } |
| 1324 | 1371 | |
| 1325 | − amdgpu_discovery_sysfs_ips(adev, ip_die_entry, ip_offset, num_ips, !!ihdr->base_addr_64_bit); | |
| 1372 | + amdgpu_discovery_sysfs_ips(adev, ip_top, ip_die_entry, ip_offset, | |
| 1373 | + num_ips, !!ihdr->base_addr_64_bit); | |
| 1326 | 1374 | } |
| 1327 | 1375 | |
| 1328 | 1376 | return 0; |
| @@ -1338,12 +1386,30 @@ static int amdgpu_discovery_sysfs_init(struct amdgpu_device *adev) · 9 unmodified lines | ||
| 1338 | 1386 | if (!discovery_bin) |
| 1339 | 1387 | return -EINVAL; |
| 1340 | 1388 | |
| 1389 | + /* If early init already created sysfs in standalone mode, skip normal init */ | |
| 1390 | + if (adev->discovery.ip_top && adev->discovery.ip_top->standalone_mode) | |
| 1391 | + return 0; | |
| 1392 | + | |
| 1341 | 1393 | ip_top = kzalloc_obj(*ip_top); |
| 1342 | 1394 | if (!ip_top) |
| 1343 | 1395 | return -ENOMEM; |
| 1344 | 1396 | |
| 1345 | 1397 | ip_top->adev = adev; |
| 1346 | − adev->discovery.ip_top = ip_top; | |
| 1398 | + | |
| 1399 | + /* Check if ip_discovery already exists before creating. | |
| 1400 | + * This shouldn't normally happen but handle it gracefully. | |
| 1401 | + */ | |
| 1402 | + if (adev->dev->kobj.sd) { | |
| 1403 | + struct kernfs_node *existing; | |
| 1404 | + | |
| 1405 | + existing = kernfs_find_and_get(adev->dev->kobj.sd, "ip_discovery"); | |
| 1406 | + if (existing) { | |
| 1407 | + kernfs_put(existing); | |
| 1408 | + kfree(ip_top); | |
| 1409 | + return 0; | |
| 1410 | + } | |
| 1411 | + } | |
| 1412 | + | |
| 1347 | 1413 | res = kobject_init_and_add(&ip_top->kobj, &ip_discovery_ktype, |
| 1348 | 1414 | &adev->dev->kobj, "ip_discovery"); |
| 1349 | 1415 | if (res) { |
| @@ -1351,6 +1417,8 @@ static int amdgpu_discovery_sysfs_init(struct amdgpu_device *adev) · 1 unmodified line | ||
| 1351 | 1417 | goto Err; |
| 1352 | 1418 | } |
| 1353 | 1419 | |
| 1420 | + adev->discovery.ip_top = ip_top; | |
| 1421 | + | |
| 1354 | 1422 | die_kset = &ip_top->die_kset; |
| 1355 | 1423 | kobject_set_name(&die_kset->kobj, "%s", "die"); |
| 1356 | 1424 | die_kset->kobj.parent = &ip_top->kobj; |
| @@ -1365,7 +1433,7 @@ static int amdgpu_discovery_sysfs_init(struct amdgpu_device *adev) · 8 unmodified lines | ||
| 1365 | 1433 | ip_hw_instance_attrs[ii] = &ip_hw_attr[ii].attr; |
| 1366 | 1434 | ip_hw_instance_attrs[ii] = NULL; |
| 1367 | 1435 | |
| 1368 | − res = amdgpu_discovery_sysfs_recurse(adev); | |
| 1436 | + res = amdgpu_discovery_sysfs_recurse(adev, ip_top); | |
| 1369 | 1437 | |
| 1370 | 1438 | return res; |
| 1371 | 1439 | Err: |
| @@ -1412,7 +1480,7 @@ static void amdgpu_discovery_sysfs_die_free(struct ip_die_entry *ip_die_entry) · 40 unmodified lines | ||
| 1412 | 1480 | kobject_put(&ip_die_entry->ip_kset.kobj); |
| 1413 | 1481 | } |
| 1414 | 1482 | |
| 1415 | −static void amdgpu_discovery_sysfs_fini(struct amdgpu_device *adev) | |
| 1483 | +void amdgpu_discovery_sysfs_fini(struct amdgpu_device *adev) | |
| 1416 | 1484 | { |
| 1417 | 1485 | struct ip_discovery_top *ip_top = adev->discovery.ip_top; |
| 1418 | 1486 | struct list_head *el, *tmp; |
| @@ -1421,6 +1489,16 @@ static void amdgpu_discovery_sysfs_fini(struct amdgpu_device *adev) · 2 unmodified lines | ||
| 1421 | 1489 | if (!ip_top) |
| 1422 | 1490 | return; |
| 1423 | 1491 | |
| 1492 | + /* | |
| 1493 | + * In standalone mode the sysfs hierarchy is tied to the PCI device | |
| 1494 | + * lifetime and is torn down by amdgpu_discovery_sysfs_early_fini(). | |
| 1495 | + * Freeing it here would leave a dangling pointer in the early | |
| 1496 | + * discovery list, causing a use-after-free on driver unbind. | |
| 1497 | + */ | |
| 1498 | + if (ip_top->standalone_mode) | |
| 1499 | + return; | |
| 1500 | + | |
| 1501 | + adev->discovery.ip_top = NULL; | |
| 1424 | 1502 | die_kset = &ip_top->die_kset; |
| 1425 | 1503 | spin_lock(&die_kset->list_lock); |
| 1426 | 1504 | list_for_each_prev_safe(el, tmp, &die_kset->list) { |
| @@ -1479,6 +1557,150 @@ void amdgpu_discovery_dump(struct amdgpu_device *adev, struct drm_printer *p) · 52 unmodified lines | ||
| 1479 | 1557 | spin_unlock(&die_kset->list_lock); |
| 1480 | 1558 | } |
| 1481 | 1559 | |
| 1560 | +int amdgpu_discovery_sysfs_early_init(struct amdgpu_device *adev, struct pci_dev *pdev) | |
| 1561 | +{ | |
| 1562 | + struct ip_discovery_top *ip_top; | |
| 1563 | + struct early_ip_discovery *early_entry, *tmp; | |
| 1564 | + struct kset *die_kset; | |
| 1565 | + uint8_t *discovery_bin; | |
| 1566 | + int res, ii; | |
| 1567 | + | |
| 1568 | + if (!adev || !adev->discovery.bin) | |
| 1569 | + return -EINVAL; | |
| 1570 | + | |
| 1571 | + if (adev->discovery.ip_top) | |
| 1572 | + return 0; | |
| 1573 | + | |
| 1574 | + mutex_lock(&early_ip_discovery_mutex); | |
| 1575 | + list_for_each_entry_safe(early_entry, tmp, &early_ip_discovery_list, list) { | |
| 1576 | + if (early_entry->pdev == pdev) { | |
| 1577 | + adev->discovery.ip_top = early_entry->ip_top; | |
| 1578 | + early_entry->ip_top->adev = adev; | |
| 1579 | + mutex_unlock(&early_ip_discovery_mutex); | |
| 1580 | + return 0; | |
| 1581 | + } | |
| 1582 | + } | |
| 1583 | + mutex_unlock(&early_ip_discovery_mutex); | |
| 1584 | + | |
| 1585 | + discovery_bin = adev->discovery.bin; | |
| 1586 | + | |
| 1587 | + early_entry = kzalloc(sizeof(*early_entry), GFP_KERNEL); | |
| 1588 | + if (!early_entry) | |
| 1589 | + return -ENOMEM; | |
| 1590 | + | |
| 1591 | + ip_top = kzalloc(sizeof(*ip_top), GFP_KERNEL); | |
| 1592 | + if (!ip_top) { | |
| 1593 | + kfree(early_entry); | |
| 1594 | + return -ENOMEM; | |
| 1595 | + } | |
| 1596 | + | |
| 1597 | + ip_top->discovery_bin = devm_kmemdup(&pdev->dev, discovery_bin, | |
| 1598 | + DISCOVERY_TMR_SIZE, GFP_KERNEL); | |
| 1599 | + if (!ip_top->discovery_bin) { | |
| 1600 | + kfree(ip_top); | |
| 1601 | + kfree(early_entry); | |
| 1602 | + return -ENOMEM; | |
| 1603 | + } | |
| 1604 | + | |
| 1605 | + ip_top->bin_size = DISCOVERY_TMR_SIZE; | |
| 1606 | + ip_top->pdev = pdev; | |
| 1607 | + ip_top->adev = adev; | |
| 1608 | + ip_top->standalone_mode = true; | |
| 1609 | + | |
| 1610 | + /* Check if ip_discovery already exists (from previous probe attempt). | |
| 1611 | + * This can happen if the module was unloaded and reloaded but the | |
| 1612 | + * sysfs persisted (tied to PCI device lifetime). | |
| 1613 | + */ | |
| 1614 | + if (pdev->dev.kobj.sd) { | |
| 1615 | + struct kernfs_node *existing; | |
| 1616 | + | |
| 1617 | + existing = kernfs_find_and_get(pdev->dev.kobj.sd, "ip_discovery"); | |
| 1618 | + if (existing) { | |
| 1619 | + kernfs_put(existing); | |
| 1620 | + kfree(ip_top); | |
| 1621 | + kfree(early_entry); | |
| 1622 | + return 0; | |
| 1623 | + } | |
| 1624 | + } | |
| 1625 | + | |
| 1626 | + res = kobject_init_and_add(&ip_top->kobj, &ip_discovery_ktype, | |
| 1627 | + &pdev->dev.kobj, "ip_discovery"); | |
| 1628 | + if (res) | |
| 1629 | + goto err_put_kobj; | |
| 1630 | + | |
| 1631 | + adev->discovery.ip_top = ip_top; | |
| 1632 | + | |
| 1633 | + die_kset = &ip_top->die_kset; | |
| 1634 | + kobject_set_name(&die_kset->kobj, "%s", "die"); | |
| 1635 | + die_kset->kobj.parent = &ip_top->kobj; | |
| 1636 | + die_kset->kobj.ktype = &die_kobj_ktype; | |
| 1637 | + res = kset_register(&ip_top->die_kset); | |
| 1638 | + if (res) | |
| 1639 | + goto err_put_die_kset; | |
| 1640 | + | |
| 1641 | + for (ii = 0; ii < ARRAY_SIZE(ip_hw_attr); ii++) | |
| 1642 | + ip_hw_instance_attrs[ii] = &ip_hw_attr[ii].attr; | |
| 1643 | + ip_hw_instance_attrs[ii] = NULL; | |
| 1644 | + | |
| 1645 | + res = amdgpu_discovery_sysfs_recurse(NULL, ip_top); | |
| 1646 | + if (res) | |
| 1647 | + goto err_put_die_kset; | |
| 1648 | + | |
| 1649 | + early_entry->pdev = pdev; | |
| 1650 | + early_entry->ip_top = ip_top; | |
| 1651 | + mutex_lock(&early_ip_discovery_mutex); | |
| 1652 | + list_add(&early_entry->list, &early_ip_discovery_list); | |
| 1653 | + mutex_unlock(&early_ip_discovery_mutex); | |
| 1654 | + | |
| 1655 | + return 0; | |
| 1656 | + | |
| 1657 | +err_put_die_kset: | |
| 1658 | + kobject_put(&ip_top->die_kset.kobj); | |
| 1659 | +err_put_kobj: | |
| 1660 | + kobject_put(&ip_top->kobj); | |
| 1661 | + kfree(early_entry); | |
| 1662 | + adev->discovery.ip_top = NULL; | |
| 1663 | + return res; | |
| 1664 | +} | |
| 1665 | + | |
| 1666 | +void amdgpu_discovery_sysfs_early_fini(struct pci_dev *pdev) | |
| 1667 | +{ | |
| 1668 | + struct early_ip_discovery *entry, *tmp_entry; | |
| 1669 | + struct ip_discovery_top *ip_top = NULL; | |
| 1670 | + struct list_head *el, *tmp; | |
| 1671 | + struct kset *die_kset; | |
| 1672 | + | |
| 1673 | + /* Find the entry in our tracking list */ | |
| 1674 | + mutex_lock(&early_ip_discovery_mutex); | |
| 1675 | + list_for_each_entry_safe(entry, tmp_entry, &early_ip_discovery_list, list) { | |
| 1676 | + if (entry->pdev == pdev) { | |
| 1677 | + ip_top = entry->ip_top; | |
| 1678 | + list_del(&entry->list); | |
| 1679 | + kfree(entry); | |
| 1680 | + break; | |
| 1681 | + } | |
| 1682 | + } | |
| 1683 | + mutex_unlock(&early_ip_discovery_mutex); | |
| 1684 | + | |
| 1685 | + if (!ip_top) | |
| 1686 | + return; | |
| 1687 | + | |
| 1688 | + /* Clean up sysfs hierarchy */ | |
| 1689 | + die_kset = &ip_top->die_kset; | |
| 1690 | + | |
| 1691 | + spin_lock(&die_kset->list_lock); | |
| 1692 | + list_for_each_prev_safe(el, tmp, &die_kset->list) { | |
| 1693 | + list_del_init(el); | |
| 1694 | + spin_unlock(&die_kset->list_lock); | |
| 1695 | + amdgpu_discovery_sysfs_die_free(to_ip_die_entry(list_to_kobj(el))); | |
| 1696 | + spin_lock(&die_kset->list_lock); | |
| 1697 | + } | |
| 1698 | + spin_unlock(&die_kset->list_lock); | |
| 1699 | + | |
| 1700 | + kobject_put(&ip_top->die_kset.kobj); | |
| 1701 | + kobject_put(&ip_top->kobj); | |
| 1702 | + /* ip_top itself will be freed by kobject_put via ip_disc_release */ | |
| 1703 | +} | |
| 1482 | 1704 | |
| 1483 | 1705 | /* ================================================== */ |
| 1484 | 1706 | |
| @@ -1504,6 +1726,9 @@ static int amdgpu_discovery_reg_base_init(struct amdgpu_device *adev) · 19 unmodified lines | ||
| 1504 | 1726 | r = amdgpu_discovery_init(adev); |
| 1505 | 1727 | if (r) |
| 1506 | 1728 | return r; |
| 1729 | + | |
| 1730 | + amdgpu_discovery_sysfs_early_init(adev, adev->pdev); | |
| 1731 | + | |
| 1507 | 1732 | discovery_bin = adev->discovery.bin; |
| 1508 | 1733 | wafl_ver = 0; |
| 1509 | 1734 | adev->gfx.xcc_mask = 0; |
| @@ -2636,7 +2861,12 @@ static int amdgpu_discovery_set_mm_ip_blocks(struct amdgpu_device *adev) · 1126 unmodified lines | ||
| 2636 | 2861 | return -EINVAL; |
| 2637 | 2862 | } |
| 2638 | 2863 | } else { |
| 2639 | − switch (amdgpu_ip_version(adev, UVD_HWIP, 0)) { | |
| 2864 | + uint32_t vcn_version = amdgpu_ip_version(adev, UVD_HWIP, 0); | |
| 2865 | + | |
| 2866 | + /* no VCN discovered; nothing to add */ | |
| 2867 | + if (!vcn_version) | |
| 2868 | + return 0; | |
| 2869 | + switch (vcn_version) { | |
| 2640 | 2870 | case IP_VERSION(1, 0, 0): |
| 2641 | 2871 | case IP_VERSION(1, 0, 1): |
| 2642 | 2872 | amdgpu_device_ip_block_add(adev, &vcn_v1_0_ip_block); |
| @@ -2704,7 +2934,7 @@ static int amdgpu_discovery_set_mm_ip_blocks(struct amdgpu_device *adev) · 61 unmodified lines | ||
| 2704 | 2934 | default: |
| 2705 | 2935 | dev_err(adev->dev, |
| 2706 | 2936 | "Failed to add vcn/jpeg ip block(UVD_HWIP:0x%x)\n", |
| 2707 | − amdgpu_ip_version(adev, UVD_HWIP, 0)); | |
| 2937 | + vcn_version); | |
| 2708 | 2938 | return -EINVAL; |
| 2709 | 2939 | } |
| 2710 | 2940 | } |
| @@ -41,6 +41,7 @@ struct amdgpu_discovery_info { · 40 unmodified lines | ||
| 41 | 41 | bool reserve_tmr; |
| 42 | 42 | }; |
| 43 | 43 | |
| 44 | +void amdgpu_discovery_sysfs_fini(struct amdgpu_device *adev); | |
| 44 | 45 | void amdgpu_discovery_fini(struct amdgpu_device *adev); |
| 45 | 46 | int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev); |
| 46 | 47 | |
| @@ -53,4 +54,9 @@ int amdgpu_discovery_get_gc_major_minor_version(struct amdgpu_device *adev, · 6 unmodified lines | ||
| 53 | 54 | |
| 54 | 55 | void amdgpu_discovery_dump(struct amdgpu_device *adev, struct drm_printer *p); |
| 55 | 56 | |
| 57 | +/* Early sysfs functions for persistent ip_discovery export */ | |
| 58 | +int amdgpu_discovery_sysfs_early_init(struct amdgpu_device *adev, | |
| 59 | + struct pci_dev *pdev); | |
| 60 | +void amdgpu_discovery_sysfs_early_fini(struct pci_dev *pdev); | |
| 61 | + | |
| 56 | 62 | #endif /* __AMDGPU_DISCOVERY__ */ |
| @@ -33,7 +33,6 @@ · 32 unmodified lines | ||
| 33 | 33 | #include <drm/drm_vblank.h> |
| 34 | 34 | |
| 35 | 35 | #include <linux/cc_platform.h> |
| 36 | −#include <linux/console.h> | |
| 37 | 36 | #include <linux/dynamic_debug.h> |
| 38 | 37 | #include <linux/module.h> |
| 39 | 38 | #include <linux/mmu_notifier.h> |
| @@ -146,7 +145,9 @@ enum AMDGPU_DEBUG_MASK { · 106 unmodified lines | ||
| 146 | 145 | AMDGPU_DEBUG_SMU_POOL = BIT(7), |
| 147 | 146 | AMDGPU_DEBUG_VM_USERPTR = BIT(8), |
| 148 | 147 | AMDGPU_DEBUG_DISABLE_RAS_CE_LOG = BIT(9), |
| 149 | − AMDGPU_DEBUG_ENABLE_CE_CS = BIT(10) | |
| 148 | + AMDGPU_DEBUG_ENABLE_CE_CS = BIT(10), | |
| 149 | + AMDGPU_DEBUG_HIBERNATION_THAW_RESUME_GPU = BIT(11), | |
| 150 | + AMDGPU_DEBUG_DISABLE_IP_BLOCK_SOFT_RESET = BIT(12), | |
| 150 | 151 | }; |
| 151 | 152 | |
| 152 | 153 | unsigned int amdgpu_vram_limit = UINT_MAX; |
| @@ -1939,6 +1940,7 @@ static const struct pci_device_id pciidlist[] = { · 1786 unmodified lines | ||
| 1939 | 1940 | {0x1002, 0x6646, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMD_IS_MOBILITY}, |
| 1940 | 1941 | {0x1002, 0x6647, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE|AMD_IS_MOBILITY}, |
| 1941 | 1942 | {0x1002, 0x6649, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE}, |
| 1943 | + {0x1002, 0x664D, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE}, | |
| 1942 | 1944 | {0x1002, 0x6650, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE}, |
| 1943 | 1945 | {0x1002, 0x6651, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE}, |
| 1944 | 1946 | {0x1002, 0x6658, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_BONAIRE}, |
| @@ -2008,6 +2010,7 @@ static const struct pci_device_id pciidlist[] = { · 63 unmodified lines | ||
| 2008 | 2010 | {0x1002, 0x6930, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA}, |
| 2009 | 2011 | {0x1002, 0x6938, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA}, |
| 2010 | 2012 | {0x1002, 0x6939, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA}, |
| 2013 | + {0x1002, 0x693B, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_TONGA}, | |
| 2011 | 2014 | /* fiji */ |
| 2012 | 2015 | {0x1002, 0x7300, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_FIJI}, |
| 2013 | 2016 | {0x1002, 0x730F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_FIJI}, |
| @@ -2036,6 +2039,7 @@ static const struct pci_device_id pciidlist[] = { · 22 unmodified lines | ||
| 2036 | 2039 | {0x1002, 0x67C4, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10}, |
| 2037 | 2040 | {0x1002, 0x67C7, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10}, |
| 2038 | 2041 | {0x1002, 0x67D0, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10}, |
| 2042 | + {0x1002, 0x67D4, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10}, | |
| 2039 | 2043 | {0x1002, 0x67DF, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10}, |
| 2040 | 2044 | {0x1002, 0x67C8, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10}, |
| 2041 | 2045 | {0x1002, 0x67C9, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS10}, |
| @@ -2049,6 +2053,7 @@ static const struct pci_device_id pciidlist[] = { · 7 unmodified lines | ||
| 2049 | 2053 | {0x1002, 0x6985, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12}, |
| 2050 | 2054 | {0x1002, 0x6986, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12}, |
| 2051 | 2055 | {0x1002, 0x6987, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12}, |
| 2056 | + {0x1002, 0x698F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12}, | |
| 2052 | 2057 | {0x1002, 0x6995, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12}, |
| 2053 | 2058 | {0x1002, 0x6997, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12}, |
| 2054 | 2059 | {0x1002, 0x699F, PCI_ANY_ID, PCI_ANY_ID, 0, 0, CHIP_POLARIS12}, |
| @@ -2250,7 +2255,7 @@ static void amdgpu_init_debug_options(struct amdgpu_device *adev) · 195 unmodified lines | ||
| 2250 | 2255 | } |
| 2251 | 2256 | |
| 2252 | 2257 | if (amdgpu_debug_mask & AMDGPU_DEBUG_DISABLE_GPU_SOFT_RECOVERY) { |
| 2253 | − pr_info("debug: soft reset for GPU recovery disabled\n"); | |
| 2258 | + pr_info("debug: soft recovery disabled\n"); | |
| 2254 | 2259 | adev->debug_disable_soft_recovery = true; |
| 2255 | 2260 | } |
| 2256 | 2261 | |
| @@ -2291,6 +2296,16 @@ static void amdgpu_init_debug_options(struct amdgpu_device *adev) · 34 unmodified lines | ||
| 2291 | 2296 | pr_info("debug: allowing command submission to CE engine\n"); |
| 2292 | 2297 | adev->debug_enable_ce_cs = true; |
| 2293 | 2298 | } |
| 2299 | + | |
| 2300 | + if (amdgpu_debug_mask & AMDGPU_DEBUG_HIBERNATION_THAW_RESUME_GPU) { | |
| 2301 | + pr_info("debug: resume gpu in thaw() of hibernation\n"); | |
| 2302 | + adev->debug_hibernation_thaw_resume_gpu = true; | |
| 2303 | + } | |
| 2304 | + | |
| 2305 | + if (amdgpu_debug_mask & AMDGPU_DEBUG_DISABLE_IP_BLOCK_SOFT_RESET) { | |
| 2306 | + pr_info("debug: IP block soft reset disabled\n"); | |
| 2307 | + adev->debug_disable_ip_block_soft_reset = true; | |
| 2308 | + } | |
| 2294 | 2309 | } |
| 2295 | 2310 | |
| 2296 | 2311 | static unsigned long amdgpu_fix_asic_type(struct pci_dev *pdev, unsigned long flags) |
| @@ -2552,6 +2567,8 @@ amdgpu_pci_remove(struct pci_dev *pdev) · 255 unmodified lines | ||
| 2552 | 2567 | |
| 2553 | 2568 | amdgpu_driver_unload_kms(dev); |
| 2554 | 2569 | |
| 2570 | + amdgpu_discovery_sysfs_early_fini(pdev); | |
| 2571 | + | |
| 2555 | 2572 | /* |
| 2556 | 2573 | * Flush any in flight DMA operations from device. |
| 2557 | 2574 | * Clear the Bus Master Enable bit and then wait on the PCIe Device |
| @@ -2705,9 +2722,10 @@ static int amdgpu_pmops_freeze(struct device *dev) · 147 unmodified lines | ||
| 2705 | 2722 | static int amdgpu_pmops_thaw(struct device *dev) |
| 2706 | 2723 | { |
| 2707 | 2724 | struct drm_device *drm_dev = dev_get_drvdata(dev); |
| 2725 | + struct amdgpu_device *adev = drm_to_adev(drm_dev); | |
| 2708 | 2726 | |
| 2709 | 2727 | /* do not resume device if it's normal hibernation */ |
| 2710 | − if (console_suspend_enabled && | |
| 2728 | + if (!adev->debug_hibernation_thaw_resume_gpu && | |
| 2711 | 2729 | !pm_hibernate_is_recovering() && |
| 2712 | 2730 | !pm_hibernation_mode_is_suspend()) |
| 2713 | 2731 | return 0; |
| @@ -3076,6 +3094,7 @@ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = { · 362 unmodified lines | ||
| 3076 | 3094 | DRM_IOCTL_DEF_DRV(AMDGPU_USERQ_SIGNAL, amdgpu_userq_signal_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), |
| 3077 | 3095 | DRM_IOCTL_DEF_DRV(AMDGPU_USERQ_WAIT, amdgpu_userq_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), |
| 3078 | 3096 | DRM_IOCTL_DEF_DRV(AMDGPU_GEM_LIST_HANDLES, amdgpu_gem_list_handles_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), |
| 3097 | + DRM_IOCTL_DEF_DRV(AMDGPU_PROC_OPTIONS, amdgpu_proc_options_ioctl, DRM_AUTH|DRM_RENDER_ALLOW), | |
| 3079 | 3098 | }; |
| 3080 | 3099 | |
| 3081 | 3100 | static const struct drm_driver amdgpu_kms_driver = { |
| @@ -727,6 +727,15 @@ void amdgpu_ring_set_fence_errors_and_reemit(struct amdgpu_ring *ring, · 726 unmodified lines | ||
| 727 | 727 | last_seq = amdgpu_fence_read(ring) & ring->fence_drv.num_fences_mask; |
| 728 | 728 | seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask; |
| 729 | 729 | |
| 730 | + /* If there is nothing to reemit, return early and set an error on the fence | |
| 731 | + * if applicable. If all of the fences are siganlled, this will be a nop. | |
| 732 | + * if there are still fences and ring_backup_entries_to_copy is 0, then | |
| 733 | + * we are skipping it on purpose. | |
| 734 | + */ | |
| 735 | + if (!ring->ring_backup_entries_to_copy) { | |
| 736 | + amdgpu_fence_driver_force_completion(ring, &guilty_fence->base); | |
| 737 | + return; | |
| 738 | + } | |
| 730 | 739 | ring->reemit = true; |
| 731 | 740 | amdgpu_ring_alloc(ring, ring->ring_backup_entries_to_copy); |
| 732 | 741 | spin_lock_irqsave(&ring->fence_drv.lock, flags); |
| @@ -741,7 +750,8 @@ void amdgpu_ring_set_fence_errors_and_reemit(struct amdgpu_ring *ring, · 8 unmodified lines | ||
| 741 | 750 | if (unprocessed && !dma_fence_is_signaled_locked(unprocessed)) { |
| 742 | 751 | fence = container_of(unprocessed, struct amdgpu_fence, base); |
| 743 | 752 | is_guilty_fence = fence == guilty_fence; |
| 744 | − is_guilty_context = fence->context == guilty_fence->context; | |
| 753 | + is_guilty_context = guilty_fence ? | |
| 754 | + (fence->context == guilty_fence->context) : false; | |
| 745 | 755 | |
| 746 | 756 | /* mark all fences from the guilty context with an error */ |
| 747 | 757 | if (is_guilty_fence) |
| @@ -794,6 +804,17 @@ void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring, · 46 unmodified lines | ||
| 794 | 804 | seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask; |
| 795 | 805 | ring->ring_backup_entries_to_copy = 0; |
| 796 | 806 | |
| 807 | + /* if we've already seen this fence, return early. | |
| 808 | + * ring->ring_backup_entries_to_copy is set to 0 so | |
| 809 | + * the reemit helper will return early as well to | |
| 810 | + * avoid getting stuck in a reemit loop. | |
| 811 | + */ | |
| 812 | + if (ring->guilty_fence == guilty_fence) { | |
| 813 | + ring->guilty_fence = NULL; | |
| 814 | + return; | |
| 815 | + } | |
| 816 | + ring->guilty_fence = guilty_fence; | |
| 817 | + | |
| 797 | 818 | do { |
| 798 | 819 | last_seq++; |
| 799 | 820 | last_seq &= ring->fence_drv.num_fences_mask; |
| @@ -811,6 +832,36 @@ void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring, · 11 unmodified lines | ||
| 811 | 832 | } while (last_seq != seq); |
| 812 | 833 | } |
| 813 | 834 | |
| 835 | +struct amdgpu_fence * | |
| 836 | +amdgpu_ring_find_guilty_fence(struct amdgpu_ring *ring) | |
| 837 | +{ | |
| 838 | + struct dma_fence *unprocessed; | |
| 839 | + struct dma_fence __rcu **ptr; | |
| 840 | + struct amdgpu_fence *fence; | |
| 841 | + u32 seq, last_seq; | |
| 842 | + | |
| 843 | + last_seq = amdgpu_fence_read(ring) & ring->fence_drv.num_fences_mask; | |
| 844 | + seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask; | |
| 845 | + | |
| 846 | + do { | |
| 847 | + last_seq++; | |
| 848 | + last_seq &= ring->fence_drv.num_fences_mask; | |
| 849 | + | |
| 850 | + ptr = &ring->fence_drv.fences[last_seq]; | |
| 851 | + rcu_read_lock(); | |
| 852 | + unprocessed = rcu_dereference(*ptr); | |
| 853 | + | |
| 854 | + if (unprocessed && !dma_fence_is_signaled(unprocessed)) { | |
| 855 | + fence = container_of(unprocessed, struct amdgpu_fence, base); | |
| 856 | + rcu_read_unlock(); | |
| 857 | + return fence; | |
| 858 | + } | |
| 859 | + rcu_read_unlock(); | |
| 860 | + } while (last_seq != seq); | |
| 861 | + | |
| 862 | + return NULL; | |
| 863 | +} | |
| 864 | + | |
| 814 | 865 | /* |
| 815 | 866 | * Common fence implementation |
| 816 | 867 | */ |
| @@ -34,6 +34,7 @@ · 33 unmodified lines | ||
| 34 | 34 | #include "amdgpu_xcp.h" |
| 35 | 35 | #include "amdgpu_xgmi.h" |
| 36 | 36 | #include "amdgpu_mes.h" |
| 37 | +#include "mes_userqueue.h" | |
| 37 | 38 | #include "nvd.h" |
| 38 | 39 | |
| 39 | 40 | /* delay 0.1 second to enable gfx off feature */ |
| @@ -377,6 +378,30 @@ int amdgpu_gfx_kiq_init(struct amdgpu_device *adev, · 337 unmodified lines | ||
| 377 | 378 | return 0; |
| 378 | 379 | } |
| 379 | 380 | |
| 381 | +static void amdgpu_gfx_mqd_reset_restore(struct amdgpu_ring *ring) | |
| 382 | +{ | |
| 383 | + struct amdgpu_device *adev = ring->adev; | |
| 384 | + int mqd_idx, mqd_size; | |
| 385 | + | |
| 386 | + /* restore mqd with the backup copy */ | |
| 387 | + if (ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) { | |
| 388 | + mqd_idx = ring - &adev->gfx.compute_ring[0]; | |
| 389 | + mqd_size = adev->mqds[AMDGPU_HW_IP_COMPUTE].mqd_size; | |
| 390 | + if (adev->gfx.mec.mqd_backup[mqd_idx]) | |
| 391 | + memcpy_toio(ring->mqd_ptr, adev->gfx.mec.mqd_backup[mqd_idx], mqd_size); | |
| 392 | + } else if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) { | |
| 393 | + mqd_size = adev->mqds[AMDGPU_HW_IP_GFX].mqd_size; | |
| 394 | + mqd_idx = ring - &adev->gfx.gfx_ring[0]; | |
| 395 | + | |
| 396 | + if (adev->gfx.me.mqd_backup[mqd_idx]) | |
| 397 | + memcpy_toio(ring->mqd_ptr, adev->gfx.me.mqd_backup[mqd_idx], mqd_size); | |
| 398 | + } | |
| 399 | + /* reset the ring */ | |
| 400 | + ring->wptr = 0; | |
| 401 | + atomic64_set((atomic64_t *)ring->wptr_cpu_addr, 0); | |
| 402 | + amdgpu_ring_clear_ring(ring); | |
| 403 | +} | |
| 404 | + | |
| 380 | 405 | /* create MQD for each compute/gfx queue */ |
| 381 | 406 | int amdgpu_gfx_mqd_sw_init(struct amdgpu_device *adev, |
| 382 | 407 | unsigned int mqd_size, int xcc_id) |
| @@ -1964,6 +1989,60 @@ static ssize_t amdgpu_gfx_get_compute_reset_mask(struct device *dev, · 1581 unmodified lines | ||
| 1964 | 1989 | return amdgpu_show_reset_mask(buf, adev->gfx.compute_supported_reset); |
| 1965 | 1990 | } |
| 1966 | 1991 | |
| 1992 | +static int amdgpu_gfx_mes_reset_queue_start(struct amdgpu_ring *ring, | |
| 1993 | + unsigned int vmid, | |
| 1994 | + struct amdgpu_fence *timedout_fence, | |
| 1995 | + bool use_mmio) | |
| 1996 | +{ | |
| 1997 | + struct amdgpu_device *adev = ring->adev; | |
| 1998 | + bool reinit_queue; | |
| 1999 | + int r; | |
| 2000 | + | |
| 2001 | + if ((ring->funcs->type == AMDGPU_RING_TYPE_COMPUTE) && | |
| 2002 | + adev->mes.compute_pipe_reset_enabled) | |
| 2003 | + reinit_queue = true; | |
| 2004 | + else if ((ring->funcs->type == AMDGPU_RING_TYPE_GFX) && | |
| 2005 | + adev->mes.gfx_pipe_reset_enabled) | |
| 2006 | + reinit_queue = true; | |
| 2007 | + else | |
| 2008 | + reinit_queue = use_mmio; | |
| 2009 | + | |
| 2010 | + amdgpu_ring_reset_helper_begin(ring, timedout_fence); | |
| 2011 | + | |
| 2012 | + r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, use_mmio, 0); | |
| 2013 | + if (r) | |
| 2014 | + return r; | |
| 2015 | + | |
| 2016 | + if (reinit_queue) { | |
| 2017 | + r = amdgpu_mes_unmap_legacy_queue(adev, ring, | |
| 2018 | + RESET_QUEUES, 0, 0, 0); | |
| 2019 | + if (r) | |
| 2020 | + return r; | |
| 2021 | + amdgpu_gfx_mqd_reset_restore(ring); | |
| 2022 | + | |
| 2023 | + r = amdgpu_mes_map_legacy_queue(adev, ring, 0); | |
| 2024 | + if (r) { | |
| 2025 | + dev_err(adev->dev, "failed to remap kgq\n"); | |
| 2026 | + return r; | |
| 2027 | + } | |
| 2028 | + } | |
| 2029 | + return 0; | |
| 2030 | +} | |
| 2031 | + | |
| 2032 | +int amdgpu_gfx_mes_reset_queue(struct amdgpu_ring *ring, | |
| 2033 | + unsigned int vmid, | |
| 2034 | + struct amdgpu_fence *timedout_fence, | |
| 2035 | + bool use_mmio) | |
| 2036 | +{ | |
| 2037 | + int r; | |
| 2038 | + | |
| 2039 | + r = amdgpu_gfx_mes_reset_queue_start(ring, vmid, timedout_fence, | |
| 2040 | + use_mmio); | |
| 2041 | + if (r) | |
| 2042 | + return r; | |
| 2043 | + return amdgpu_ring_reset_helper_end(ring, timedout_fence); | |
| 2044 | +} | |
| 2045 | + | |
| 1967 | 2046 | static DEVICE_ATTR(run_cleaner_shader, 0200, |
| 1968 | 2047 | NULL, amdgpu_gfx_set_run_cleaner_shader); |
| 1969 | 2048 | |
| @@ -2122,6 +2201,200 @@ void amdgpu_gfx_sysfs_fini(struct amdgpu_device *adev) · 152 unmodified lines | ||
| 2122 | 2201 | } |
| 2123 | 2202 | } |
| 2124 | 2203 | |
| 2204 | +static void amdgpu_gfx_reset_start_compute_scheds(struct amdgpu_device *adev, | |
| 2205 | + struct amdgpu_ring *guilty_ring) | |
| 2206 | +{ | |
| 2207 | + struct amdgpu_ring *ring; | |
| 2208 | + int i; | |
| 2209 | + | |
| 2210 | + for (i = 0; i < adev->gfx.num_compute_rings; i++) { | |
| 2211 | + ring = &adev->gfx.compute_ring[i]; | |
| 2212 | + if (ring == guilty_ring) | |
| 2213 | + continue; | |
| 2214 | + drm_sched_wqueue_start(&ring->sched); | |
| 2215 | + } | |
| 2216 | +} | |
| 2217 | + | |
| 2218 | +static void amdgpu_gfx_reset_stop_compute_scheds(struct amdgpu_device *adev, | |
| 2219 | + struct amdgpu_ring *guilty_ring) | |
| 2220 | +{ | |
| 2221 | + struct amdgpu_ring *ring; | |
| 2222 | + int i; | |
| 2223 | + | |
| 2224 | + for (i = 0; i < adev->gfx.num_compute_rings; i++) { | |
| 2225 | + ring = &adev->gfx.compute_ring[i]; | |
| 2226 | + if (ring == guilty_ring) | |
| 2227 | + continue; | |
| 2228 | + drm_sched_wqueue_stop(&ring->sched); | |
| 2229 | + } | |
| 2230 | +} | |
| 2231 | + | |
| 2232 | +/* | |
| 2233 | + * Match the MES-reported hung doorbell against a compute ring and run | |
| 2234 | + * the reset. On hit, the matched ring and its guilty fence are returned | |
| 2235 | + * via *out_ring / *out_fence so the caller can defer reset end until | |
| 2236 | + * after MES has resumed all gangs. | |
| 2237 | + */ | |
| 2238 | +static int amdgpu_gfx_reset_mes_kcq(struct amdgpu_device *adev, | |
| 2239 | + struct amdgpu_ring *guilty_ring, | |
| 2240 | + unsigned int db, | |
| 2241 | + struct amdgpu_ring **out_ring, | |
| 2242 | + struct amdgpu_fence **out_fence) | |
| 2243 | +{ | |
| 2244 | + bool use_mmio = adev->gfx.mec.use_mmio_for_reset; | |
| 2245 | + struct amdgpu_fence *fence; | |
| 2246 | + struct amdgpu_ring *ring; | |
| 2247 | + int i, r; | |
| 2248 | + | |
| 2249 | + *out_ring = NULL; | |
| 2250 | + *out_fence = NULL; | |
| 2251 | + for (i = 0; i < adev->gfx.num_compute_rings; i++) { | |
| 2252 | + ring = &adev->gfx.compute_ring[i]; | |
| 2253 | + if (ring == guilty_ring) | |
| 2254 | + continue; | |
| 2255 | + if (ring->doorbell_index == db) { | |
| 2256 | + fence = amdgpu_ring_find_guilty_fence(ring); | |
| 2257 | + r = amdgpu_gfx_mes_reset_queue_start(ring, 0, fence, | |
| 2258 | + use_mmio); | |
| 2259 | + if (r) | |
| 2260 | + return r; | |
| 2261 | + *out_ring = ring; | |
| 2262 | + *out_fence = fence; | |
| 2263 | + break; | |
| 2264 | + } | |
| 2265 | + } | |
| 2266 | + return 0; | |
| 2267 | +} | |
| 2268 | + | |
| 2269 | +int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev, | |
| 2270 | + struct amdgpu_ring *ring, | |
| 2271 | + struct amdgpu_fence *guilty_fence, | |
| 2272 | + struct amdgpu_usermode_queue *uq, | |
| 2273 | + unsigned int *hung_queue_count, | |
| 2274 | + void *faulty_queue_input) | |
| 2275 | +{ | |
| 2276 | + struct amdgpu_mes_hung_queue_hqd_info *hqd_info = | |
| 2277 | + (struct amdgpu_mes_hung_queue_hqd_info *) | |
| 2278 | + &adev->gfx.mec.mes_hung_db_array[adev->mes.hung_queue_hqd_info_offset]; | |
| 2279 | + int i, r, pipe, queue, queue_type; | |
| 2280 | + unsigned int num_hung = 0; | |
| 2281 | + bool use_mmio = adev->gfx.mec.use_mmio_for_reset; | |
| 2282 | + struct mes_remove_queue_input *queue_input = (struct mes_remove_queue_input *)faulty_queue_input; | |
| 2283 | + struct amdgpu_gfx_deferred_entry deferred_end[AMDGPU_MAX_COMPUTE_RINGS + 1]; | |
| 2284 | + int n_deferred = 0; | |
| 2285 | + int ring_err; | |
| 2286 | + | |
| 2287 | + guard(mutex)(&adev->gfx.mec.reset_mutex); | |
| 2288 | + /* stop the drm schedulers for all compute queues */ | |
| 2289 | + amdgpu_gfx_reset_stop_compute_scheds(adev, ring); | |
| 2290 | + /* suspend all will determine which queues are hung. | |
| 2291 | + * reset detect will return the array of bad queue doorbells | |
| 2292 | + */ | |
| 2293 | + r = amdgpu_mes_suspend(adev, 0); | |
| 2294 | + /* if suspend all success, it should no hang queue */ | |
| 2295 | + if (!r) | |
| 2296 | + /* always reset the KCQ/userq since we need to signal the fence | |
| 2297 | + * and we could be stuck in a loop which is preemptable. | |
| 2298 | + */ | |
| 2299 | + goto fence_reset; | |
| 2300 | + r = amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE, | |
| 2301 | + true, &num_hung, adev->gfx.mec.mes_hung_db_array, 0); | |
| 2302 | + if (r) | |
| 2303 | + goto out; | |
| 2304 | + if (hung_queue_count) | |
| 2305 | + *hung_queue_count = num_hung; | |
| 2306 | + | |
| 2307 | +fence_reset: | |
| 2308 | + /* reset the queue this came from if specified */ | |
| 2309 | + if (ring) { | |
| 2310 | + r = amdgpu_gfx_mes_reset_queue_start(ring, 0, guilty_fence, | |
| 2311 | + use_mmio); | |
| 2312 | + if (r) | |
| 2313 | + goto out; | |
| 2314 | + deferred_end[n_deferred].ring = ring; | |
| 2315 | + deferred_end[n_deferred].fence = guilty_fence; | |
| 2316 | + n_deferred++; | |
| 2317 | + } | |
| 2318 | + if (uq) { | |
| 2319 | + r = mes_userq_reset(uq); | |
| 2320 | + if (r) | |
| 2321 | + goto out; | |
| 2322 | + } | |
| 2323 | + for (i = 0; i < num_hung; i++) { | |
| 2324 | + struct amdgpu_ring *hr = NULL; | |
| 2325 | + struct amdgpu_fence *hf = NULL; | |
| 2326 | + | |
| 2327 | + pipe = hqd_info[i].pipe_index; | |
| 2328 | + queue = hqd_info[i].queue_index; | |
| 2329 | + queue_type = hqd_info[i].queue_type; | |
| 2330 | + | |
| 2331 | + /* reset any KCQs */ | |
| 2332 | + r = amdgpu_gfx_reset_mes_kcq(adev, ring, | |
| 2333 | + adev->gfx.mec.mes_hung_db_array[i], | |
| 2334 | + &hr, &hf); | |
| 2335 | + if (r) | |
| 2336 | + goto out; | |
| 2337 | + if (hr) { | |
| 2338 | + deferred_end[n_deferred].ring = hr; | |
| 2339 | + deferred_end[n_deferred].fence = hf; | |
| 2340 | + n_deferred++; | |
| 2341 | + } | |
| 2342 | + /* reset any KFD queues */ | |
| 2343 | + r = amdgpu_amdkfd_reset_mes_queue(adev, 0, queue_type, pipe, queue, | |
| 2344 | + adev->gfx.mec.mes_hung_db_array[i]); | |
| 2345 | + if (r) | |
| 2346 | + goto out; | |
| 2347 | + /* reset KGD user queues */ | |
| 2348 | + r = mes_userq_reset_queue(adev, uq, queue_type, pipe, queue, | |
| 2349 | + adev->gfx.mec.mes_hung_db_array[i]); | |
| 2350 | + if (r) | |
| 2351 | + goto out; | |
| 2352 | + } | |
| 2353 | + | |
| 2354 | + /* MES doesn't detect any hung queue but we have a known bad queue | |
| 2355 | + * and it is not KCQ | |
| 2356 | + */ | |
| 2357 | + if (!num_hung && queue_input && !ring) { | |
| 2358 | + /* MES suspend_all is successful means this bad queue is | |
| 2359 | + * preempted successfuly. Remove it before resume all so it | |
| 2360 | + * doesn't get mapped back | |
| 2361 | + */ | |
| 2362 | + if (!down_read_trylock(&adev->reset_domain->sem)) { | |
| 2363 | + r = -EIO; | |
| 2364 | + goto out; | |
| 2365 | + } | |
| 2366 | + amdgpu_mes_lock(&adev->mes); | |
| 2367 | + r = adev->mes.funcs->remove_hw_queue(&adev->mes, queue_input); | |
| 2368 | + amdgpu_mes_unlock(&adev->mes); | |
| 2369 | + up_read(&adev->reset_domain->sem); | |
| 2370 | + } | |
| 2371 | + | |
| 2372 | +out: | |
| 2373 | + /* resume all will enable the non-hung queues */ | |
| 2374 | + amdgpu_mes_resume(adev, 0); | |
| 2375 | + | |
| 2376 | + /* Now CP is running again — replay backed-up commands and ring | |
| 2377 | + * doorbells on each reset queue. | |
| 2378 | + */ | |
| 2379 | + ring_err = r; | |
| 2380 | + for (i = 0; i < n_deferred; i++) { | |
| 2381 | + int er = amdgpu_ring_reset_helper_end(deferred_end[i].ring, | |
| 2382 | + deferred_end[i].fence); | |
| 2383 | + | |
| 2384 | + if (er && !ring_err) | |
| 2385 | + ring_err = er; | |
| 2386 | + } | |
| 2387 | + | |
| 2388 | + if (!ring_err) | |
| 2389 | + amdgpu_gfx_reset_start_compute_scheds(adev, ring); | |
| 2390 | + | |
| 2391 | + /* If this reset is triggered by non-KCQ, the KCQ result after resume must | |
| 2392 | + * not override the reset result; otherwise a false reset failure is returned | |
| 2393 | + * to the non-KCQ caller | |
| 2394 | + */ | |
| 2395 | + return ring ? ring_err : r; | |
| 2396 | +} | |
| 2397 | + | |
| 2125 | 2398 | int amdgpu_gfx_cleaner_shader_sw_init(struct amdgpu_device *adev, |
| 2126 | 2399 | unsigned int cleaner_shader_size) |
| 2127 | 2400 | { |
| @@ -2460,9 +2733,8 @@ void amdgpu_gfx_profile_ring_begin_use(struct amdgpu_ring *ring) · 332 unmodified lines | ||
| 2460 | 2733 | else |
| 2461 | 2734 | profile = PP_SMC_POWER_PROFILE_COMPUTE; |
| 2462 | 2735 | |
| 2463 | − atomic_inc(&adev->gfx.total_submission_cnt); | |
| 2464 | − | |
| 2465 | − cancel_delayed_work_sync(&adev->gfx.idle_work); | |
| 2736 | + if (!atomic_fetch_inc(&adev->gfx.total_submission_cnt)) | |
| 2737 | + cancel_delayed_work_sync(&adev->gfx.idle_work); | |
| 2466 | 2738 | |
| 2467 | 2739 | /* We can safely return early here because we've cancelled the |
| 2468 | 2740 | * the delayed work so there is no one else to set it to false |
| @@ -2490,9 +2762,9 @@ void amdgpu_gfx_profile_ring_end_use(struct amdgpu_ring *ring) · 21 unmodified lines | ||
| 2490 | 2762 | if (amdgpu_dpm_is_overdrive_enabled(adev)) |
| 2491 | 2763 | return; |
| 2492 | 2764 | |
| 2493 | − atomic_dec(&ring->adev->gfx.total_submission_cnt); | |
| 2494 | − | |
| 2495 | − schedule_delayed_work(&ring->adev->gfx.idle_work, GFX_PROFILE_IDLE_TIMEOUT); | |
| 2765 | + if (atomic_dec_and_test(&ring->adev->gfx.total_submission_cnt)) | |
| 2766 | + schedule_delayed_work(&ring->adev->gfx.idle_work, | |
| 2767 | + GFX_PROFILE_IDLE_TIMEOUT); | |
| 2496 | 2768 | } |
| 2497 | 2769 | |
| 2498 | 2770 | /** |
| @@ -36,6 +36,8 @@ · 35 unmodified lines | ||
| 36 | 36 | #include "amdgpu_ring_mux.h" |
| 37 | 37 | #include "amdgpu_xcp.h" |
| 38 | 38 | |
| 39 | +struct amdgpu_usermode_queue; | |
| 40 | + | |
| 39 | 41 | /* GFX current status */ |
| 40 | 42 | #define AMDGPU_GFX_NORMAL_MODE 0x00000000L |
| 41 | 43 | #define AMDGPU_GFX_SAFE_MODE 0x00000001L |
| @@ -116,6 +118,9 @@ struct amdgpu_mec { · 74 unmodified lines | ||
| 116 | 118 | u32 num_pipe_per_mec; |
| 117 | 119 | u32 num_queue_per_pipe; |
| 118 | 120 | void *mqd_backup[AMDGPU_MAX_COMPUTE_RINGS * AMDGPU_MAX_GC_INSTANCES]; |
| 121 | + bool use_mmio_for_reset; | |
| 122 | + u32 *mes_hung_db_array; | |
| 123 | + struct mutex reset_mutex; | |
| 119 | 124 | }; |
| 120 | 125 | |
| 121 | 126 | struct amdgpu_mec_bitmap { |
| @@ -401,6 +406,7 @@ struct amdgpu_me { · 279 unmodified lines | ||
| 401 | 406 | uint32_t num_pipe_per_me; |
| 402 | 407 | uint32_t num_queue_per_pipe; |
| 403 | 408 | void *mqd_backup[AMDGPU_MAX_GFX_RINGS]; |
| 409 | + bool use_mmio_for_reset; | |
| 404 | 410 | |
| 405 | 411 | /* These are the resources for which amdgpu takes ownership */ |
| 406 | 412 | DECLARE_BITMAP(queue_bitmap, AMDGPU_MAX_GFX_QUEUES); |
| @@ -479,8 +485,6 @@ struct amdgpu_gfx { · 72 unmodified lines | ||
| 479 | 485 | const struct amdgpu_gfx_funcs *funcs; |
| 480 | 486 | |
| 481 | 487 | /* reset mask */ |
| 482 | − uint32_t grbm_soft_reset; | |
| 483 | − uint32_t srbm_soft_reset; | |
| 484 | 488 | uint32_t gfx_supported_reset; |
| 485 | 489 | uint32_t compute_supported_reset; |
| 486 | 490 | |
| @@ -543,6 +547,11 @@ struct amdgpu_gfx { · 56 unmodified lines | ||
| 543 | 547 | bool disable_uq; |
| 544 | 548 | }; |
| 545 | 549 | |
| 550 | +struct amdgpu_gfx_deferred_entry { | |
| 551 | + struct amdgpu_ring *ring; | |
| 552 | + struct amdgpu_fence *fence; | |
| 553 | +}; | |
| 554 | + | |
| 546 | 555 | struct amdgpu_gfx_ras_reg_entry { |
| 547 | 556 | struct amdgpu_ras_err_status_reg_entry reg_entry; |
| 548 | 557 | enum amdgpu_gfx_ras_mem_id_type mem_id_type; |
| @@ -641,6 +650,12 @@ int amdgpu_gfx_poison_consumption_handler(struct amdgpu_device *adev, · 92 unmodified lines | ||
| 641 | 650 | bool amdgpu_gfx_is_master_xcc(struct amdgpu_device *adev, int xcc_id); |
| 642 | 651 | int amdgpu_gfx_sysfs_init(struct amdgpu_device *adev); |
| 643 | 652 | void amdgpu_gfx_sysfs_fini(struct amdgpu_device *adev); |
| 653 | +int amdgpu_gfx_reset_mes_compute(struct amdgpu_device *adev, | |
| 654 | + struct amdgpu_ring *ring, | |
| 655 | + struct amdgpu_fence *guilty_fence, | |
| 656 | + struct amdgpu_usermode_queue *uq, | |
| 657 | + unsigned int *hung_queue_count, | |
| 658 | + void *faulty_queue_input); | |
| 644 | 659 | void amdgpu_gfx_ras_error_func(struct amdgpu_device *adev, |
| 645 | 660 | void *ras_error_status, |
| 646 | 661 | void (*func)(struct amdgpu_device *adev, void *ras_error_status, |
| @@ -667,6 +682,11 @@ void amdgpu_debugfs_compute_sched_mask_init(struct amdgpu_device *adev); · 20 unmodified lines | ||
| 667 | 682 | |
| 668 | 683 | int amdgpu_gfx_ring_preempt_ib(struct amdgpu_ring *ring); |
| 669 | 684 | |
| 685 | +int amdgpu_gfx_mes_reset_queue(struct amdgpu_ring *ring, | |
| 686 | + unsigned int vmid, | |
| 687 | + struct amdgpu_fence *timedout_fence, | |
| 688 | + bool use_mmio); | |
| 689 | + | |
| 670 | 690 | static inline const char *amdgpu_gfx_compute_mode_desc(int mode) |
| 671 | 691 | { |
| 672 | 692 | switch (mode) { |
| @@ -1763,10 +1763,15 @@ int amdgpu_gmc_init_mem_ranges(struct amdgpu_device *adev) · 1762 unmodified lines | ||
| 1763 | 1763 | valid = true; |
| 1764 | 1764 | else |
| 1765 | 1765 | valid = amdgpu_gmc_validate_partition_info(adev); |
| 1766 | − if (!valid) { | |
| 1767 | − /* TODO: handle invalid case */ | |
| 1766 | + if (!valid) | |
| 1768 | 1767 | dev_warn(adev->dev, |
| 1769 | 1768 | "Mem ranges not matching with hardware config\n"); |
| 1769 | + | |
| 1770 | + if (!adev->gmc.num_mem_partitions) { | |
| 1771 | + dev_err(adev->dev, "num_mem_partitions is zero\n"); | |
| 1772 | + kfree(adev->gmc.mem_partitions); | |
| 1773 | + adev->gmc.mem_partitions = NULL; | |
| 1774 | + return -EINVAL; | |
| 1770 | 1775 | } |
| 1771 | 1776 | |
| 1772 | 1777 | return 0; |
| @@ -286,7 +286,6 @@ struct amdgpu_gmc { · 285 unmodified lines | ||
| 286 | 286 | struct amdgpu_irq_src vm_fault; |
| 287 | 287 | uint32_t vram_type; |
| 288 | 288 | uint8_t vram_vendor; |
| 289 | − uint32_t srbm_soft_reset; | |
| 290 | 289 | bool prt_warning; |
| 291 | 290 | uint32_t sdpif_register; |
| 292 | 291 | /* apertures */ |
| @@ -369,43 +369,152 @@ int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, · 368 unmodified lines | ||
| 369 | 369 | } |
| 370 | 370 | |
| 371 | 371 | /** |
| 372 | − * amdgpu_device_ip_is_hw - is the hardware IP enabled | |
| 372 | + * amdgpu_device_ip_is_valid - is the hardware IP valid | |
| 373 | 373 | * |
| 374 | 374 | * @adev: amdgpu_device pointer |
| 375 | 375 | * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) |
| 376 | 376 | * |
| 377 | − * Check if the hardware IP is enable or not. | |
| 378 | − * Returns true if it the IP is enable, false if not. | |
| 377 | + * Check if the hardware IP is valid or not. | |
| 378 | + * Returns true if it the IP is valid, false if not. | |
| 379 | 379 | */ |
| 380 | −bool amdgpu_device_ip_is_hw(struct amdgpu_device *adev, | |
| 381 | − enum amd_ip_block_type block_type) | |
| 380 | +bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev, | |
| 381 | + enum amd_ip_block_type block_type) | |
| 382 | 382 | { |
| 383 | 383 | struct amdgpu_ip_block *ip_block; |
| 384 | 384 | |
| 385 | 385 | ip_block = amdgpu_device_ip_get_ip_block(adev, block_type); |
| 386 | 386 | if (ip_block) |
| 387 | − return ip_block->status.hw; | |
| 387 | + return ip_block->status.valid; | |
| 388 | 388 | |
| 389 | 389 | return false; |
| 390 | 390 | } |
| 391 | 391 | |
| 392 | 392 | /** |
| 393 | − * amdgpu_device_ip_is_valid - is the hardware IP valid | |
| 393 | + * amdgpu_ip_from_ring() - Find IP block type corresponding to ring type. | |
| 394 | 394 | * |
| 395 | − * @adev: amdgpu_device pointer | |
| 396 | − * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.) | |
| 395 | + * @ring_type: The ring type whose IP block you are looking for. | |
| 396 | + */ | |
| 397 | +static enum amd_ip_block_type amdgpu_ip_from_ring(const enum amdgpu_ring_type ring_type) | |
| 398 | +{ | |
| 399 | + switch (ring_type) { | |
| 400 | + case AMDGPU_RING_TYPE_GFX: | |
| 401 | + case AMDGPU_RING_TYPE_COMPUTE: | |
| 402 | + return AMD_IP_BLOCK_TYPE_GFX; | |
| 403 | + | |
| 404 | + case AMDGPU_RING_TYPE_SDMA: | |
| 405 | + return AMD_IP_BLOCK_TYPE_SDMA; | |
| 406 | + | |
| 407 | + case AMDGPU_RING_TYPE_UVD: | |
| 408 | + case AMDGPU_RING_TYPE_UVD_ENC: | |
| 409 | + return AMD_IP_BLOCK_TYPE_UVD; | |
| 410 | + | |
| 411 | + case AMDGPU_RING_TYPE_VCE: | |
| 412 | + return AMD_IP_BLOCK_TYPE_VCE; | |
| 413 | + | |
| 414 | + case AMDGPU_RING_TYPE_VCN_DEC: | |
| 415 | + case AMDGPU_RING_TYPE_VCN_ENC: | |
| 416 | + return AMD_IP_BLOCK_TYPE_VCN; | |
| 417 | + | |
| 418 | + case AMDGPU_RING_TYPE_VCN_JPEG: | |
| 419 | + return AMD_IP_BLOCK_TYPE_JPEG; | |
| 420 | + | |
| 421 | + case AMDGPU_RING_TYPE_VPE: | |
| 422 | + return AMD_IP_BLOCK_TYPE_VPE; | |
| 423 | + | |
| 424 | + default: | |
| 425 | + return AMD_IP_BLOCK_TYPE_NUM; | |
| 426 | + } | |
| 427 | +} | |
| 428 | + | |
| 429 | +/** | |
| 430 | + * amdgpu_ring_mask_from_ip() - Find mask of ring types corresponding to an IP block type. | |
| 397 | 431 | * |
| 398 | − * Check if the hardware IP is valid or not. | |
| 399 | − * Returns true if it the IP is valid, false if not. | |
| 432 | + * @ip_type: The IP block type whose rings you are looking for. | |
| 400 | 433 | */ |
| 401 | −bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev, | |
| 402 | − enum amd_ip_block_type block_type) | |
| 434 | +static u32 amdgpu_ring_mask_from_ip(const enum amd_ip_block_type ip_type) | |
| 435 | +{ | |
| 436 | + switch (ip_type) { | |
| 437 | + case AMD_IP_BLOCK_TYPE_GFX: | |
| 438 | + return BIT(AMDGPU_RING_TYPE_GFX) | BIT(AMDGPU_RING_TYPE_COMPUTE); | |
| 439 | + | |
| 440 | + case AMD_IP_BLOCK_TYPE_SDMA: | |
| 441 | + return BIT(AMDGPU_RING_TYPE_SDMA); | |
| 442 | + | |
| 443 | + case AMD_IP_BLOCK_TYPE_UVD: | |
| 444 | + return BIT(AMDGPU_RING_TYPE_UVD) | BIT(AMDGPU_RING_TYPE_UVD_ENC); | |
| 445 | + | |
| 446 | + case AMD_IP_BLOCK_TYPE_VCE: | |
| 447 | + return BIT(AMD_IP_BLOCK_TYPE_VCE); | |
| 448 | + | |
| 449 | + case AMD_IP_BLOCK_TYPE_VCN: | |
| 450 | + return BIT(AMDGPU_RING_TYPE_VCN_DEC) | BIT(AMDGPU_RING_TYPE_VCN_ENC); | |
| 451 | + | |
| 452 | + case AMD_IP_BLOCK_TYPE_JPEG: | |
| 453 | + return BIT(AMDGPU_RING_TYPE_VCN_JPEG); | |
| 454 | + | |
| 455 | + case AMD_IP_BLOCK_TYPE_VPE: | |
| 456 | + return BIT(AMDGPU_RING_TYPE_VPE); | |
| 457 | + | |
| 458 | + default: | |
| 459 | + return 0; | |
| 460 | + } | |
| 461 | +} | |
| 462 | + | |
| 463 | +/** | |
| 464 | + * amdgpu_device_ip_soft_reset() - Perform a graceful soft reset on an IP block. | |
| 465 | + * | |
| 466 | + * @guilty_ring: The ring which is guilty of causing a reset. | |
| 467 | + * @guilty_fence: The fence which didn't signal. | |
| 468 | + * | |
| 469 | + * IP block soft reset is used when attempting to recover | |
| 470 | + * from a GPU hang in a situation where a more fine grained | |
| 471 | + * reset type isn't available or didn't work. This effectively | |
| 472 | + * resets all rings that belong to the same device IP block | |
| 473 | + * and re-initializes the device IP block. | |
| 474 | + * | |
| 475 | + * The reset is handled gracefully, meaning that we try to | |
| 476 | + * minimize collateral damage (ie. avoid rejecting non-guilty jobs) | |
| 477 | + * as well as back up and restore the contents of all rings | |
| 478 | + * so that the system can move on from the hang. | |
| 479 | + */ | |
| 480 | +int amdgpu_device_ip_soft_reset(struct amdgpu_ring *guilty_ring, | |
| 481 | + struct amdgpu_fence *guilty_fence) | |
| 403 | 482 | { |
| 483 | + struct amdgpu_device *adev = guilty_ring->adev; | |
| 404 | 484 | struct amdgpu_ip_block *ip_block; |
| 485 | + enum amd_ip_block_type ip_type; | |
| 486 | + u32 ring_type_mask; | |
| 487 | + int r; | |
| 405 | 488 | |
| 406 | − ip_block = amdgpu_device_ip_get_ip_block(adev, block_type); | |
| 407 | − if (ip_block) | |
| 408 | − return ip_block->status.valid; | |
| 489 | + ip_type = amdgpu_ip_from_ring(guilty_ring->funcs->type); | |
| 490 | + ip_block = amdgpu_device_ip_get_ip_block(adev, ip_type); | |
| 409 | 491 | |
| 410 | − return false; | |
| 492 | + if (!ip_block || !ip_block->version->funcs->soft_reset) { | |
| 493 | + dev_warn(adev->dev, "IP block soft reset not supported on %s\n", | |
| 494 | + ip_block->version->funcs->name); | |
| 495 | + return -EOPNOTSUPP; | |
| 496 | + } | |
| 497 | + | |
| 498 | + dev_err(adev->dev, "Starting %s IP block soft reset\n", | |
| 499 | + ip_block->version->funcs->name); | |
| 500 | + | |
| 501 | + ring_type_mask = amdgpu_ring_mask_from_ip(ip_type); | |
| 502 | + | |
| 503 | + amdgpu_device_lock_reset_domain(adev->reset_domain); | |
| 504 | + amdgpu_multi_ring_reset_helper_begin(ring_type_mask, guilty_ring, guilty_fence); | |
| 505 | + | |
| 506 | + r = ip_block->version->funcs->soft_reset(ip_block); | |
| 507 | + | |
| 508 | + r = amdgpu_multi_ring_reset_helper_end(ring_type_mask, guilty_ring, r); | |
| 509 | + amdgpu_device_unlock_reset_domain(adev->reset_domain); | |
| 510 | + | |
| 511 | + if (r) { | |
| 512 | + dev_err(adev->dev, "Failed %s IP block soft reset: %d\n", | |
| 513 | + ip_block->version->funcs->name, r); | |
| 514 | + return r; | |
| 515 | + } | |
| 516 | + | |
| 517 | + dev_err(adev->dev, "Successful %s IP block soft reset\n", | |
| 518 | + ip_block->version->funcs->name); | |
| 519 | + return 0; | |
| 411 | 520 | } |
| @@ -68,6 +68,7 @@ enum amd_hw_ip_block_type { · 67 unmodified lines | ||
| 68 | 68 | ISP_HWIP, |
| 69 | 69 | ATU_HWIP, |
| 70 | 70 | AIGC_HWIP, |
| 71 | + UMSCH_HWIP, | |
| 71 | 72 | MAX_HWIP |
| 72 | 73 | }; |
| 73 | 74 | |
| @@ -84,6 +85,9 @@ enum amd_hw_ip_block_type { · 10 unmodified lines | ||
| 84 | 85 | #define IP_VERSION_SUBREV(ver) ((ver) & 0xF) |
| 85 | 86 | #define IP_VERSION_MAJ_MIN_REV(ver) ((ver) >> 8) |
| 86 | 87 | |
| 88 | +struct amdgpu_ring; | |
| 89 | +struct amdgpu_fence; | |
| 90 | + | |
| 87 | 91 | struct amdgpu_ip_map_info { |
| 88 | 92 | /* Map of logical to actual dev instances/mask */ |
| 89 | 93 | uint32_t dev_inst[MAX_HWIP][HWIP_MAX_INSTANCE]; |
| @@ -146,9 +150,9 @@ void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev, · 56 unmodified lines | ||
| 146 | 150 | u64 *flags); |
| 147 | 151 | int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev, |
| 148 | 152 | enum amd_ip_block_type block_type); |
| 149 | −bool amdgpu_device_ip_is_hw(struct amdgpu_device *adev, | |
| 150 | − enum amd_ip_block_type block_type); | |
| 151 | 153 | bool amdgpu_device_ip_is_valid(struct amdgpu_device *adev, |
| 152 | 154 | enum amd_ip_block_type block_type); |
| 155 | +int amdgpu_device_ip_soft_reset(struct amdgpu_ring *guilty_ring, | |
| 156 | + struct amdgpu_fence *guilty_fence); | |
| 153 | 157 | |
| 154 | 158 | #endif /* __AMDGPU_IP_H__ */ |
| @@ -112,7 +112,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) · 111 unmodified lines | ||
| 112 | 112 | amdgpu_job_core_dump(adev, job); |
| 113 | 113 | |
| 114 | 114 | if (amdgpu_gpu_recovery && |
| 115 | − amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_SOFT_RESET) && | |
| 115 | + amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_SOFT_RECOVERY) && | |
| 116 | 116 | amdgpu_ring_soft_recovery(ring, job->vmid, s_job->s_fence->parent)) { |
| 117 | 117 | dev_err(adev->dev, "ring %s timeout, but soft recovered\n", |
| 118 | 118 | s_job->sched->name); |
| @@ -151,6 +151,17 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job) · 32 unmodified lines | ||
| 151 | 151 | dev_err(adev->dev, "Ring %s reset failed\n", ring->sched.name); |
| 152 | 152 | } |
| 153 | 153 | |
| 154 | + /* Attempt an IP block soft reset, if supported. */ | |
| 155 | + if (amdgpu_gpu_recovery && | |
| 156 | + amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET)) { | |
| 157 | + r = amdgpu_device_ip_soft_reset(ring, job->hw_fence); | |
| 158 | + if (!r) { | |
| 159 | + atomic_inc(&ring->adev->gpu_reset_counter); | |
| 160 | + drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, info); | |
| 161 | + goto exit; | |
| 162 | + } | |
| 163 | + } | |
| 164 | + | |
| 154 | 165 | if (dma_fence_get_status(&s_job->s_fence->finished) == 0) |
| 155 | 166 | dma_fence_set_error(&s_job->s_fence->finished, -ETIME); |
| 156 | 167 | |
| @@ -134,8 +134,8 @@ void amdgpu_jpeg_ring_begin_use(struct amdgpu_ring *ring) · 133 unmodified lines | ||
| 134 | 134 | { |
| 135 | 135 | struct amdgpu_device *adev = ring->adev; |
| 136 | 136 | |
| 137 | − atomic_inc(&adev->jpeg.total_submission_cnt); | |
| 138 | − cancel_delayed_work_sync(&adev->jpeg.idle_work); | |
| 137 | + if (!atomic_fetch_inc(&adev->jpeg.total_submission_cnt)) | |
| 138 | + cancel_delayed_work_sync(&adev->jpeg.idle_work); | |
| 139 | 139 | |
| 140 | 140 | mutex_lock(&adev->jpeg.jpeg_pg_lock); |
| 141 | 141 | amdgpu_device_ip_set_powergating_state(adev, AMD_IP_BLOCK_TYPE_JPEG, |
| @@ -145,8 +145,9 @@ void amdgpu_jpeg_ring_begin_use(struct amdgpu_ring *ring) · 3 unmodified lines | ||
| 145 | 145 | |
| 146 | 146 | void amdgpu_jpeg_ring_end_use(struct amdgpu_ring *ring) |
| 147 | 147 | { |
| 148 | − atomic_dec(&ring->adev->jpeg.total_submission_cnt); | |
| 149 | − schedule_delayed_work(&ring->adev->jpeg.idle_work, JPEG_IDLE_TIMEOUT); | |
| 148 | + if (atomic_dec_and_test(&ring->adev->jpeg.total_submission_cnt)) | |
| 149 | + schedule_delayed_work(&ring->adev->jpeg.idle_work, | |
| 150 | + JPEG_IDLE_TIMEOUT); | |
| 150 | 151 | } |
| 151 | 152 | |
| 152 | 153 | int amdgpu_jpeg_dec_ring_test_ring(struct amdgpu_ring *ring) |
| @@ -149,6 +149,9 @@ struct amdgpu_jpeg { · 148 unmodified lines | ||
| 149 | 149 | u32 *ip_dump; |
| 150 | 150 | u32 reg_count; |
| 151 | 151 | const struct amdgpu_hwip_reg_entry *reg_list; |
| 152 | + | |
| 153 | + bool disable_uq; | |
| 154 | + bool disable_kq; | |
| 152 | 155 | }; |
| 153 | 156 | |
| 154 | 157 | int amdgpu_jpeg_sw_init(struct amdgpu_device *adev); |
| @@ -1423,6 +1423,33 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp) · 1422 unmodified lines | ||
| 1423 | 1423 | return 0; |
| 1424 | 1424 | } |
| 1425 | 1425 | |
| 1426 | +/** | |
| 1427 | + * amdgpu_proc_options_ioctl - set per-fd user options | |
| 1428 | + * | |
| 1429 | + * @dev: drm dev pointer | |
| 1430 | + * @data: pointer to struct drm_amdgpu_proc_options | |
| 1431 | + * @filp: drm file | |
| 1432 | + * | |
| 1433 | + * Sets options stored on the per-file amdgpu_fpriv. Currently the only | |
| 1434 | + * supported option is %AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY which | |
| 1435 | + * controls how KFD delivers SIGBUS for poison/RAS events to the calling | |
| 1436 | + * process (immediate, suppressed, or delayed by N milliseconds). | |
| 1437 | + */ | |
| 1438 | +int amdgpu_proc_options_ioctl(struct drm_device *dev, void *data, | |
| 1439 | + struct drm_file *filp) | |
| 1440 | +{ | |
| 1441 | + struct drm_amdgpu_proc_options *args = data; | |
| 1442 | + | |
| 1443 | + switch (args->op) { | |
| 1444 | + case AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY: | |
| 1445 | + return amdgpu_amdkfd_set_sigbus_delay(current, | |
| 1446 | + args->kfd_sigbus_delay.value); | |
| 1447 | + default: | |
| 1448 | + DRM_DEBUG_KMS("Invalid user option op %u\n", args->op); | |
| 1449 | + return -EINVAL; | |
| 1450 | + } | |
| 1451 | +} | |
| 1452 | + | |
| 1426 | 1453 | /** |
| 1427 | 1454 | * amdgpu_driver_open_kms - drm callback for open |
| 1428 | 1455 | * |
| @@ -1504,8 +1531,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv) · 75 unmodified lines | ||
| 1504 | 1531 | if (r) |
| 1505 | 1532 | goto error_vm; |
| 1506 | 1533 | |
| 1507 | − mutex_init(&fpriv->bo_list_lock); | |
| 1508 | − idr_init_base(&fpriv->bo_list_handles, 1); | |
| 1534 | + xa_init_flags(&fpriv->bo_list_handles, XA_FLAGS_ALLOC1); | |
| 1509 | 1535 | |
| 1510 | 1536 | r = amdgpu_userq_mgr_init(&fpriv->userq_mgr, file_priv, adev); |
| 1511 | 1537 | if (r) |
| @@ -1550,8 +1576,8 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev, · 38 unmodified lines | ||
| 1550 | 1576 | struct amdgpu_fpriv *fpriv = file_priv->driver_priv; |
| 1551 | 1577 | struct amdgpu_bo_list *list; |
| 1552 | 1578 | struct amdgpu_bo *pd; |
| 1579 | + unsigned long handle; | |
| 1553 | 1580 | u32 pasid; |
| 1554 | − int handle; | |
| 1555 | 1581 | |
| 1556 | 1582 | if (!fpriv) |
| 1557 | 1583 | return; |
| @@ -1587,11 +1613,9 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev, · 29 unmodified lines | ||
| 1587 | 1613 | amdgpu_pasid_free_delayed(pd->tbo.base.resv, pasid); |
| 1588 | 1614 | amdgpu_bo_unref(&pd); |
| 1589 | 1615 | |
| 1590 | − idr_for_each_entry(&fpriv->bo_list_handles, list, handle) | |
| 1616 | + xa_for_each(&fpriv->bo_list_handles, handle, list) | |
| 1591 | 1617 | amdgpu_bo_list_put(list); |
| 1592 | − | |
| 1593 | − idr_destroy(&fpriv->bo_list_handles); | |
| 1594 | − mutex_destroy(&fpriv->bo_list_lock); | |
| 1618 | + xa_destroy(&fpriv->bo_list_handles); | |
| 1595 | 1619 | |
| 1596 | 1620 | kfree(fpriv); |
| 1597 | 1621 | file_priv->driver_priv = NULL; |
| @@ -27,16 +27,6 @@ · 26 unmodified lines | ||
| 27 | 27 | #include "umc/umc_6_7_0_offset.h" |
| 28 | 28 | #include "umc/umc_6_7_0_sh_mask.h" |
| 29 | 29 | |
| 30 | −static bool amdgpu_mca_is_deferred_error(struct amdgpu_device *adev, | |
| 31 | − uint64_t mc_status) | |
| 32 | −{ | |
| 33 | − if (adev->umc.ras->check_ecc_err_status) | |
| 34 | − return adev->umc.ras->check_ecc_err_status(adev, | |
| 35 | − AMDGPU_MCA_ERROR_TYPE_DE, &mc_status); | |
| 36 | − | |
| 37 | − return false; | |
| 38 | −} | |
| 39 | − | |
| 40 | 30 | void amdgpu_mca_query_correctable_error_count(struct amdgpu_device *adev, |
| 41 | 31 | uint64_t mc_status_addr, |
| 42 | 32 | unsigned long *error_count) |
| @@ -155,479 +145,3 @@ int amdgpu_mca_mpio_ras_sw_init(struct amdgpu_device *adev) · 112 unmodified lines | ||
| 155 | 145 | |
| 156 | 146 | return 0; |
| 157 | 147 | } |
| 158 | − | |
| 159 | −static void amdgpu_mca_bank_set_init(struct mca_bank_set *mca_set) | |
| 160 | −{ | |
| 161 | − if (!mca_set) | |
| 162 | − return; | |
| 163 | − | |
| 164 | − memset(mca_set, 0, sizeof(*mca_set)); | |
| 165 | − INIT_LIST_HEAD(&mca_set->list); | |
| 166 | −} | |
| 167 | − | |
| 168 | −static int amdgpu_mca_bank_set_add_entry(struct mca_bank_set *mca_set, struct mca_bank_entry *entry) | |
| 169 | −{ | |
| 170 | − struct mca_bank_node *node; | |
| 171 | − | |
| 172 | − if (!entry) | |
| 173 | − return -EINVAL; | |
| 174 | − | |
| 175 | − node = kvzalloc_obj(*node); | |
| 176 | − if (!node) | |
| 177 | − return -ENOMEM; | |
| 178 | − | |
| 179 | − memcpy(&node->entry, entry, sizeof(*entry)); | |
| 180 | − | |
| 181 | − INIT_LIST_HEAD(&node->node); | |
| 182 | − list_add_tail(&node->node, &mca_set->list); | |
| 183 | − | |
| 184 | − mca_set->nr_entries++; | |
| 185 | − | |
| 186 | − return 0; | |
| 187 | −} | |
| 188 | − | |
| 189 | −static int amdgpu_mca_bank_set_merge(struct mca_bank_set *mca_set, struct mca_bank_set *new) | |
| 190 | −{ | |
| 191 | − struct mca_bank_node *node; | |
| 192 | − | |
| 193 | − list_for_each_entry(node, &new->list, node) | |
| 194 | − amdgpu_mca_bank_set_add_entry(mca_set, &node->entry); | |
| 195 | − | |
| 196 | − return 0; | |
| 197 | −} | |
| 198 | − | |
| 199 | −static void amdgpu_mca_bank_set_remove_node(struct mca_bank_set *mca_set, struct mca_bank_node *node) | |
| 200 | −{ | |
| 201 | − if (!node) | |
| 202 | − return; | |
| 203 | − | |
| 204 | − list_del(&node->node); | |
| 205 | − kvfree(node); | |
| 206 | − | |
| 207 | − mca_set->nr_entries--; | |
| 208 | −} | |
| 209 | − | |
| 210 | −static void amdgpu_mca_bank_set_release(struct mca_bank_set *mca_set) | |
| 211 | −{ | |
| 212 | − struct mca_bank_node *node, *tmp; | |
| 213 | − | |
| 214 | − if (list_empty(&mca_set->list)) | |
| 215 | − return; | |
| 216 | − | |
| 217 | − list_for_each_entry_safe(node, tmp, &mca_set->list, node) | |
| 218 | − amdgpu_mca_bank_set_remove_node(mca_set, node); | |
| 219 | −} | |
| 220 | − | |
| 221 | −void amdgpu_mca_smu_init_funcs(struct amdgpu_device *adev, const struct amdgpu_mca_smu_funcs *mca_funcs) | |
| 222 | −{ | |
| 223 | − struct amdgpu_mca *mca = &adev->mca; | |
| 224 | − | |
| 225 | − mca->mca_funcs = mca_funcs; | |
| 226 | −} | |
| 227 | − | |
| 228 | −int amdgpu_mca_init(struct amdgpu_device *adev) | |
| 229 | −{ | |
| 230 | − struct amdgpu_mca *mca = &adev->mca; | |
| 231 | − struct mca_bank_cache *mca_cache; | |
| 232 | − int i; | |
| 233 | − | |
| 234 | − atomic_set(&mca->ue_update_flag, 0); | |
| 235 | − | |
| 236 | − for (i = 0; i < ARRAY_SIZE(mca->mca_caches); i++) { | |
| 237 | − mca_cache = &mca->mca_caches[i]; | |
| 238 | − mutex_init(&mca_cache->lock); | |
| 239 | − amdgpu_mca_bank_set_init(&mca_cache->mca_set); | |
| 240 | − } | |
| 241 | − | |
| 242 | − return 0; | |
| 243 | −} | |
| 244 | − | |
| 245 | −void amdgpu_mca_fini(struct amdgpu_device *adev) | |
| 246 | −{ | |
| 247 | − struct amdgpu_mca *mca = &adev->mca; | |
| 248 | − struct mca_bank_cache *mca_cache; | |
| 249 | − int i; | |
| 250 | − | |
| 251 | − atomic_set(&mca->ue_update_flag, 0); | |
| 252 | − | |
| 253 | − for (i = 0; i < ARRAY_SIZE(mca->mca_caches); i++) { | |
| 254 | − mca_cache = &mca->mca_caches[i]; | |
| 255 | − amdgpu_mca_bank_set_release(&mca_cache->mca_set); | |
| 256 | − mutex_destroy(&mca_cache->lock); | |
| 257 | − } | |
| 258 | −} | |
| 259 | − | |
| 260 | −int amdgpu_mca_reset(struct amdgpu_device *adev) | |
| 261 | −{ | |
| 262 | − amdgpu_mca_fini(adev); | |
| 263 | − | |
| 264 | − return amdgpu_mca_init(adev); | |
| 265 | −} | |
| 266 | − | |
| 267 | −int amdgpu_mca_smu_set_debug_mode(struct amdgpu_device *adev, bool enable) | |
| 268 | −{ | |
| 269 | − const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; | |
| 270 | − | |
| 271 | − if (mca_funcs && mca_funcs->mca_set_debug_mode) | |
| 272 | − return mca_funcs->mca_set_debug_mode(adev, enable); | |
| 273 | − | |
| 274 | − return -EOPNOTSUPP; | |
| 275 | −} | |
| 276 | − | |
| 277 | −static void amdgpu_mca_smu_mca_bank_dump(struct amdgpu_device *adev, int idx, struct mca_bank_entry *entry, | |
| 278 | − struct ras_query_context *qctx) | |
| 279 | −{ | |
| 280 | − u64 event_id = qctx ? qctx->evid.event_id : RAS_EVENT_INVALID_ID; | |
| 281 | − | |
| 282 | − RAS_EVENT_LOG(adev, event_id, HW_ERR "Accelerator Check Architecture events logged\n"); | |
| 283 | − RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].STATUS=0x%016llx\n", | |
| 284 | − idx, entry->regs[MCA_REG_IDX_STATUS]); | |
| 285 | − RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].ADDR=0x%016llx\n", | |
| 286 | − idx, entry->regs[MCA_REG_IDX_ADDR]); | |
| 287 | − RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].MISC0=0x%016llx\n", | |
| 288 | − idx, entry->regs[MCA_REG_IDX_MISC0]); | |
| 289 | − RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].IPID=0x%016llx\n", | |
| 290 | − idx, entry->regs[MCA_REG_IDX_IPID]); | |
| 291 | − RAS_EVENT_LOG(adev, event_id, HW_ERR "aca entry[%02d].SYND=0x%016llx\n", | |
| 292 | − idx, entry->regs[MCA_REG_IDX_SYND]); | |
| 293 | −} | |
| 294 | − | |
| 295 | −static int amdgpu_mca_smu_get_valid_mca_count(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, uint32_t *count) | |
| 296 | −{ | |
| 297 | − const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; | |
| 298 | − | |
| 299 | − if (!count) | |
| 300 | − return -EINVAL; | |
| 301 | − | |
| 302 | − if (mca_funcs && mca_funcs->mca_get_valid_mca_count) | |
| 303 | − return mca_funcs->mca_get_valid_mca_count(adev, type, count); | |
| 304 | − | |
| 305 | − return -EOPNOTSUPP; | |
| 306 | −} | |
| 307 | − | |
| 308 | −static int amdgpu_mca_smu_get_mca_entry(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, | |
| 309 | − int idx, struct mca_bank_entry *entry) | |
| 310 | −{ | |
| 311 | − const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; | |
| 312 | − int count; | |
| 313 | − | |
| 314 | − if (!mca_funcs || !mca_funcs->mca_get_mca_entry) | |
| 315 | − return -EOPNOTSUPP; | |
| 316 | − | |
| 317 | − switch (type) { | |
| 318 | − case AMDGPU_MCA_ERROR_TYPE_UE: | |
| 319 | − count = mca_funcs->max_ue_count; | |
| 320 | − break; | |
| 321 | − case AMDGPU_MCA_ERROR_TYPE_CE: | |
| 322 | − count = mca_funcs->max_ce_count; | |
| 323 | − break; | |
| 324 | − default: | |
| 325 | − return -EINVAL; | |
| 326 | − } | |
| 327 | − | |
| 328 | − if (idx >= count) | |
| 329 | − return -EINVAL; | |
| 330 | − | |
| 331 | − return mca_funcs->mca_get_mca_entry(adev, type, idx, entry); | |
| 332 | −} | |
| 333 | − | |
| 334 | −static bool amdgpu_mca_bank_should_update(struct amdgpu_device *adev, enum amdgpu_mca_error_type type) | |
| 335 | −{ | |
| 336 | − struct amdgpu_mca *mca = &adev->mca; | |
| 337 | − bool ret = true; | |
| 338 | − | |
| 339 | − /* | |
| 340 | − * Because the UE Valid MCA count will only be cleared after reset, | |
| 341 | − * in order to avoid repeated counting of the error count, | |
| 342 | − * the aca bank is only updated once during the gpu recovery stage. | |
| 343 | − */ | |
| 344 | − if (type == AMDGPU_MCA_ERROR_TYPE_UE) { | |
| 345 | − if (amdgpu_ras_intr_triggered()) | |
| 346 | − ret = atomic_cmpxchg(&mca->ue_update_flag, 0, 1) == 0; | |
| 347 | − else | |
| 348 | − atomic_set(&mca->ue_update_flag, 0); | |
| 349 | − } | |
| 350 | − | |
| 351 | − return ret; | |
| 352 | −} | |
| 353 | − | |
| 354 | −static bool amdgpu_mca_bank_should_dump(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, | |
| 355 | − struct mca_bank_entry *entry) | |
| 356 | −{ | |
| 357 | − bool ret; | |
| 358 | − | |
| 359 | − switch (type) { | |
| 360 | − case AMDGPU_MCA_ERROR_TYPE_CE: | |
| 361 | − ret = amdgpu_mca_is_deferred_error(adev, entry->regs[MCA_REG_IDX_STATUS]); | |
| 362 | − break; | |
| 363 | − case AMDGPU_MCA_ERROR_TYPE_UE: | |
| 364 | − default: | |
| 365 | − ret = true; | |
| 366 | − break; | |
| 367 | − } | |
| 368 | − | |
| 369 | − return ret; | |
| 370 | −} | |
| 371 | − | |
| 372 | −static int amdgpu_mca_smu_get_mca_set(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, struct mca_bank_set *mca_set, | |
| 373 | − struct ras_query_context *qctx) | |
| 374 | −{ | |
| 375 | − struct mca_bank_entry entry; | |
| 376 | − uint32_t count = 0, i; | |
| 377 | − int ret; | |
| 378 | − | |
| 379 | − if (!mca_set) | |
| 380 | − return -EINVAL; | |
| 381 | − | |
| 382 | − if (!amdgpu_mca_bank_should_update(adev, type)) | |
| 383 | − return 0; | |
| 384 | − | |
| 385 | − ret = amdgpu_mca_smu_get_valid_mca_count(adev, type, &count); | |
| 386 | − if (ret) | |
| 387 | − return ret; | |
| 388 | − | |
| 389 | − for (i = 0; i < count; i++) { | |
| 390 | − memset(&entry, 0, sizeof(entry)); | |
| 391 | − ret = amdgpu_mca_smu_get_mca_entry(adev, type, i, &entry); | |
| 392 | − if (ret) | |
| 393 | − return ret; | |
| 394 | − | |
| 395 | − amdgpu_mca_bank_set_add_entry(mca_set, &entry); | |
| 396 | − | |
| 397 | − if (amdgpu_mca_bank_should_dump(adev, type, &entry)) | |
| 398 | − amdgpu_mca_smu_mca_bank_dump(adev, i, &entry, qctx); | |
| 399 | − } | |
| 400 | − | |
| 401 | − return 0; | |
| 402 | −} | |
| 403 | − | |
| 404 | −static int amdgpu_mca_smu_parse_mca_error_count(struct amdgpu_device *adev, enum amdgpu_ras_block blk, | |
| 405 | − enum amdgpu_mca_error_type type, struct mca_bank_entry *entry, uint32_t *count) | |
| 406 | −{ | |
| 407 | − const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; | |
| 408 | − | |
| 409 | − if (!count || !entry) | |
| 410 | − return -EINVAL; | |
| 411 | − | |
| 412 | − if (!mca_funcs || !mca_funcs->mca_parse_mca_error_count) | |
| 413 | − return -EOPNOTSUPP; | |
| 414 | − | |
| 415 | − return mca_funcs->mca_parse_mca_error_count(adev, blk, type, entry, count); | |
| 416 | −} | |
| 417 | − | |
| 418 | −static int amdgpu_mca_dispatch_mca_set(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type, | |
| 419 | − struct mca_bank_set *mca_set, struct ras_err_data *err_data) | |
| 420 | −{ | |
| 421 | − struct amdgpu_smuio_mcm_config_info mcm_info; | |
| 422 | − struct mca_bank_node *node, *tmp; | |
| 423 | − struct mca_bank_entry *entry; | |
| 424 | − uint32_t count; | |
| 425 | − int ret; | |
| 426 | − | |
| 427 | − if (!mca_set) | |
| 428 | − return -EINVAL; | |
| 429 | − | |
| 430 | − if (!mca_set->nr_entries) | |
| 431 | − return 0; | |
| 432 | − | |
| 433 | − list_for_each_entry_safe(node, tmp, &mca_set->list, node) { | |
| 434 | − entry = &node->entry; | |
| 435 | − | |
| 436 | − count = 0; | |
| 437 | − ret = amdgpu_mca_smu_parse_mca_error_count(adev, blk, type, entry, &count); | |
| 438 | − if (ret && ret != -EOPNOTSUPP) | |
| 439 | − return ret; | |
| 440 | − | |
| 441 | − if (!count) | |
| 442 | − continue; | |
| 443 | − | |
| 444 | − memset(&mcm_info, 0, sizeof(mcm_info)); | |
| 445 | − | |
| 446 | − mcm_info.socket_id = entry->info.socket_id; | |
| 447 | − mcm_info.die_id = entry->info.aid; | |
| 448 | − | |
| 449 | − if (type == AMDGPU_MCA_ERROR_TYPE_UE) { | |
| 450 | − amdgpu_ras_error_statistic_ue_count(err_data, | |
| 451 | − &mcm_info, (uint64_t)count); | |
| 452 | − } else { | |
| 453 | − if (amdgpu_mca_is_deferred_error(adev, entry->regs[MCA_REG_IDX_STATUS])) | |
| 454 | − amdgpu_ras_error_statistic_de_count(err_data, | |
| 455 | − &mcm_info, (uint64_t)count); | |
| 456 | − else | |
| 457 | − amdgpu_ras_error_statistic_ce_count(err_data, | |
| 458 | − &mcm_info, (uint64_t)count); | |
| 459 | − } | |
| 460 | − | |
| 461 | − amdgpu_mca_bank_set_remove_node(mca_set, node); | |
| 462 | − } | |
| 463 | − | |
| 464 | − return 0; | |
| 465 | −} | |
| 466 | − | |
| 467 | −static int amdgpu_mca_add_mca_set_to_cache(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, struct mca_bank_set *new) | |
| 468 | −{ | |
| 469 | − struct mca_bank_cache *mca_cache = &adev->mca.mca_caches[type]; | |
| 470 | − int ret; | |
| 471 | − | |
| 472 | − mutex_lock(&mca_cache->lock); | |
| 473 | − ret = amdgpu_mca_bank_set_merge(&mca_cache->mca_set, new); | |
| 474 | − mutex_unlock(&mca_cache->lock); | |
| 475 | − | |
| 476 | − return ret; | |
| 477 | −} | |
| 478 | − | |
| 479 | −int amdgpu_mca_smu_log_ras_error(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type, | |
| 480 | − struct ras_err_data *err_data, struct ras_query_context *qctx) | |
| 481 | −{ | |
| 482 | − struct mca_bank_set mca_set; | |
| 483 | − struct mca_bank_cache *mca_cache = &adev->mca.mca_caches[type]; | |
| 484 | − int ret; | |
| 485 | − | |
| 486 | − amdgpu_mca_bank_set_init(&mca_set); | |
| 487 | − | |
| 488 | − ret = amdgpu_mca_smu_get_mca_set(adev, type, &mca_set, qctx); | |
| 489 | − if (ret) | |
| 490 | − goto out_mca_release; | |
| 491 | − | |
| 492 | − ret = amdgpu_mca_dispatch_mca_set(adev, blk, type, &mca_set, err_data); | |
| 493 | − if (ret) | |
| 494 | − goto out_mca_release; | |
| 495 | − | |
| 496 | − /* add remain mca bank to mca cache */ | |
| 497 | − if (mca_set.nr_entries) { | |
| 498 | − ret = amdgpu_mca_add_mca_set_to_cache(adev, type, &mca_set); | |
| 499 | − if (ret) | |
| 500 | − goto out_mca_release; | |
| 501 | − } | |
| 502 | − | |
| 503 | − /* dispatch mca set again if mca cache has valid data */ | |
| 504 | − mutex_lock(&mca_cache->lock); | |
| 505 | − if (mca_cache->mca_set.nr_entries) | |
| 506 | − ret = amdgpu_mca_dispatch_mca_set(adev, blk, type, &mca_cache->mca_set, err_data); | |
| 507 | − mutex_unlock(&mca_cache->lock); | |
| 508 | − | |
| 509 | −out_mca_release: | |
| 510 | − amdgpu_mca_bank_set_release(&mca_set); | |
| 511 | − | |
| 512 | − return ret; | |
| 513 | −} | |
| 514 | − | |
| 515 | −#if defined(CONFIG_DEBUG_FS) | |
| 516 | −static int amdgpu_mca_smu_debug_mode_set(void *data, u64 val) | |
| 517 | −{ | |
| 518 | − struct amdgpu_device *adev = (struct amdgpu_device *)data; | |
| 519 | − int ret; | |
| 520 | − | |
| 521 | − ret = amdgpu_ras_set_mca_debug_mode(adev, val ? true : false); | |
| 522 | − if (ret) | |
| 523 | − return ret; | |
| 524 | − | |
| 525 | − dev_info(adev->dev, "amdgpu set smu mca debug mode %s success\n", val ? "on" : "off"); | |
| 526 | − | |
| 527 | − return 0; | |
| 528 | −} | |
| 529 | − | |
| 530 | −static void mca_dump_entry(struct seq_file *m, struct mca_bank_entry *entry) | |
| 531 | −{ | |
| 532 | − int i, idx = entry->idx; | |
| 533 | − int reg_idx_array[] = { | |
| 534 | − MCA_REG_IDX_STATUS, | |
| 535 | − MCA_REG_IDX_ADDR, | |
| 536 | − MCA_REG_IDX_MISC0, | |
| 537 | − MCA_REG_IDX_IPID, | |
| 538 | − MCA_REG_IDX_SYND, | |
| 539 | − }; | |
| 540 | − | |
| 541 | − seq_printf(m, "mca entry[%d].type: %s\n", idx, entry->type == AMDGPU_MCA_ERROR_TYPE_UE ? "UE" : "CE"); | |
| 542 | − seq_printf(m, "mca entry[%d].ip: %d\n", idx, entry->ip); | |
| 543 | − seq_printf(m, "mca entry[%d].info: socketid:%d aid:%d hwid:0x%03x mcatype:0x%04x\n", | |
| 544 | − idx, entry->info.socket_id, entry->info.aid, entry->info.hwid, entry->info.mcatype); | |
| 545 | − | |
| 546 | − for (i = 0; i < ARRAY_SIZE(reg_idx_array); i++) | |
| 547 | − seq_printf(m, "mca entry[%d].regs[%d]: 0x%016llx\n", idx, reg_idx_array[i], entry->regs[reg_idx_array[i]]); | |
| 548 | −} | |
| 549 | − | |
| 550 | −static int mca_dump_show(struct seq_file *m, enum amdgpu_mca_error_type type) | |
| 551 | −{ | |
| 552 | − struct amdgpu_device *adev = (struct amdgpu_device *)m->private; | |
| 553 | − struct mca_bank_node *node; | |
| 554 | − struct mca_bank_set mca_set; | |
| 555 | − struct ras_query_context qctx; | |
| 556 | − int ret; | |
| 557 | − | |
| 558 | − amdgpu_mca_bank_set_init(&mca_set); | |
| 559 | − | |
| 560 | − qctx.evid.event_id = RAS_EVENT_INVALID_ID; | |
| 561 | − ret = amdgpu_mca_smu_get_mca_set(adev, type, &mca_set, &qctx); | |
| 562 | − if (ret) | |
| 563 | − goto err_free_mca_set; | |
| 564 | − | |
| 565 | − seq_printf(m, "amdgpu smu %s valid mca count: %d\n", | |
| 566 | − type == AMDGPU_MCA_ERROR_TYPE_UE ? "UE" : "CE", mca_set.nr_entries); | |
| 567 | − | |
| 568 | − if (!mca_set.nr_entries) | |
| 569 | − goto err_free_mca_set; | |
| 570 | − | |
| 571 | − list_for_each_entry(node, &mca_set.list, node) | |
| 572 | − mca_dump_entry(m, &node->entry); | |
| 573 | − | |
| 574 | − /* add mca bank to mca bank cache */ | |
| 575 | − ret = amdgpu_mca_add_mca_set_to_cache(adev, type, &mca_set); | |
| 576 | − | |
| 577 | −err_free_mca_set: | |
| 578 | − amdgpu_mca_bank_set_release(&mca_set); | |
| 579 | − | |
| 580 | − return ret; | |
| 581 | −} | |
| 582 | − | |
| 583 | −static int mca_dump_ce_show(struct seq_file *m, void *unused) | |
| 584 | −{ | |
| 585 | − return mca_dump_show(m, AMDGPU_MCA_ERROR_TYPE_CE); | |
| 586 | −} | |
| 587 | − | |
| 588 | −static int mca_dump_ce_open(struct inode *inode, struct file *file) | |
| 589 | −{ | |
| 590 | − return single_open(file, mca_dump_ce_show, inode->i_private); | |
| 591 | −} | |
| 592 | − | |
| 593 | −static const struct file_operations mca_ce_dump_debug_fops = { | |
| 594 | − .owner = THIS_MODULE, | |
| 595 | − .open = mca_dump_ce_open, | |
| 596 | − .read = seq_read, | |
| 597 | − .llseek = seq_lseek, | |
| 598 | − .release = single_release, | |
| 599 | −}; | |
| 600 | − | |
| 601 | −static int mca_dump_ue_show(struct seq_file *m, void *unused) | |
| 602 | −{ | |
| 603 | − return mca_dump_show(m, AMDGPU_MCA_ERROR_TYPE_UE); | |
| 604 | −} | |
| 605 | − | |
| 606 | −static int mca_dump_ue_open(struct inode *inode, struct file *file) | |
| 607 | −{ | |
| 608 | − return single_open(file, mca_dump_ue_show, inode->i_private); | |
| 609 | −} | |
| 610 | − | |
| 611 | −static const struct file_operations mca_ue_dump_debug_fops = { | |
| 612 | − .owner = THIS_MODULE, | |
| 613 | − .open = mca_dump_ue_open, | |
| 614 | − .read = seq_read, | |
| 615 | − .llseek = seq_lseek, | |
| 616 | − .release = single_release, | |
| 617 | −}; | |
| 618 | − | |
| 619 | −DEFINE_DEBUGFS_ATTRIBUTE(mca_debug_mode_fops, NULL, amdgpu_mca_smu_debug_mode_set, "%llu\n"); | |
| 620 | −#endif | |
| 621 | − | |
| 622 | −void amdgpu_mca_smu_debugfs_init(struct amdgpu_device *adev, struct dentry *root) | |
| 623 | −{ | |
| 624 | −#if defined(CONFIG_DEBUG_FS) | |
| 625 | − if (!root) | |
| 626 | − return; | |
| 627 | − | |
| 628 | − debugfs_create_file("mca_debug_mode", 0200, root, adev, &mca_debug_mode_fops); | |
| 629 | − debugfs_create_file("mca_ue_dump", 0400, root, adev, &mca_ue_dump_debug_fops); | |
| 630 | − debugfs_create_file("mca_ce_dump", 0400, root, adev, &mca_ce_dump_debug_fops); | |
| 631 | −#endif | |
| 632 | −} | |
| 633 | − | |
| @@ -23,45 +23,6 @@ · 22 unmodified lines | ||
| 23 | 23 | |
| 24 | 24 | #include "amdgpu_ras.h" |
| 25 | 25 | |
| 26 | −#define MCA_MAX_REGS_COUNT (16) | |
| 27 | − | |
| 28 | −#define MCA_REG_FIELD(x, h, l) (((x) & GENMASK_ULL(h, l)) >> l) | |
| 29 | −#define MCA_REG__STATUS__VAL(x) MCA_REG_FIELD(x, 63, 63) | |
| 30 | −#define MCA_REG__STATUS__OVERFLOW(x) MCA_REG_FIELD(x, 62, 62) | |
| 31 | −#define MCA_REG__STATUS__UC(x) MCA_REG_FIELD(x, 61, 61) | |
| 32 | −#define MCA_REG__STATUS__EN(x) MCA_REG_FIELD(x, 60, 60) | |
| 33 | −#define MCA_REG__STATUS__MISCV(x) MCA_REG_FIELD(x, 59, 59) | |
| 34 | −#define MCA_REG__STATUS__ADDRV(x) MCA_REG_FIELD(x, 58, 58) | |
| 35 | −#define MCA_REG__STATUS__PCC(x) MCA_REG_FIELD(x, 57, 57) | |
| 36 | −#define MCA_REG__STATUS__ERRCOREIDVAL(x) MCA_REG_FIELD(x, 56, 56) | |
| 37 | −#define MCA_REG__STATUS__TCC(x) MCA_REG_FIELD(x, 55, 55) | |
| 38 | −#define MCA_REG__STATUS__SYNDV(x) MCA_REG_FIELD(x, 53, 53) | |
| 39 | −#define MCA_REG__STATUS__CECC(x) MCA_REG_FIELD(x, 46, 46) | |
| 40 | −#define MCA_REG__STATUS__UECC(x) MCA_REG_FIELD(x, 45, 45) | |
| 41 | −#define MCA_REG__STATUS__DEFERRED(x) MCA_REG_FIELD(x, 44, 44) | |
| 42 | −#define MCA_REG__STATUS__POISON(x) MCA_REG_FIELD(x, 43, 43) | |
| 43 | −#define MCA_REG__STATUS__SCRUB(x) MCA_REG_FIELD(x, 40, 40) | |
| 44 | −#define MCA_REG__STATUS__ERRCOREID(x) MCA_REG_FIELD(x, 37, 32) | |
| 45 | −#define MCA_REG__STATUS__ADDRLSB(x) MCA_REG_FIELD(x, 29, 24) | |
| 46 | −#define MCA_REG__STATUS__ERRORCODEEXT(x) MCA_REG_FIELD(x, 21, 16) | |
| 47 | −#define MCA_REG__STATUS__ERRORCODE(x) MCA_REG_FIELD(x, 15, 0) | |
| 48 | − | |
| 49 | −#define MCA_REG__MISC0__ERRCNT(x) MCA_REG_FIELD(x, 43, 32) | |
| 50 | − | |
| 51 | −#define MCA_REG__SYND__ERRORINFORMATION(x) MCA_REG_FIELD(x, 17, 0) | |
| 52 | − | |
| 53 | −enum amdgpu_mca_ip { | |
| 54 | − AMDGPU_MCA_IP_UNKNOW = -1, | |
| 55 | − AMDGPU_MCA_IP_PSP = 0, | |
| 56 | − AMDGPU_MCA_IP_SDMA, | |
| 57 | − AMDGPU_MCA_IP_GC, | |
| 58 | − AMDGPU_MCA_IP_SMU, | |
| 59 | − AMDGPU_MCA_IP_MP5, | |
| 60 | − AMDGPU_MCA_IP_UMC, | |
| 61 | − AMDGPU_MCA_IP_PCS_XGMI, | |
| 62 | − AMDGPU_MCA_IP_COUNT, | |
| 63 | −}; | |
| 64 | − | |
| 65 | 26 | enum amdgpu_mca_error_type { |
| 66 | 27 | AMDGPU_MCA_ERROR_TYPE_UE = 0, |
| 67 | 28 | AMDGPU_MCA_ERROR_TYPE_CE, |
| @@ -77,77 +38,20 @@ struct amdgpu_mca_ras { · 9 unmodified lines | ||
| 77 | 38 | struct amdgpu_mca_ras_block *ras; |
| 78 | 39 | }; |
| 79 | 40 | |
| 80 | −struct mca_bank_set { | |
| 81 | − int nr_entries; | |
| 82 | − struct list_head list; | |
| 83 | −}; | |
| 84 | − | |
| 85 | −struct mca_bank_cache { | |
| 86 | − struct mca_bank_set mca_set; | |
| 87 | − struct mutex lock; | |
| 88 | −}; | |
| 89 | − | |
| 90 | 41 | struct amdgpu_mca { |
| 91 | 42 | struct amdgpu_mca_ras mp0; |
| 92 | 43 | struct amdgpu_mca_ras mp1; |
| 93 | 44 | struct amdgpu_mca_ras mpio; |
| 94 | − const struct amdgpu_mca_smu_funcs *mca_funcs; | |
| 95 | − struct mca_bank_cache mca_caches[AMDGPU_MCA_ERROR_TYPE_DE]; | |
| 96 | − atomic_t ue_update_flag; | |
| 97 | −}; | |
| 98 | − | |
| 99 | −enum mca_reg_idx { | |
| 100 | − MCA_REG_IDX_STATUS = 1, | |
| 101 | − MCA_REG_IDX_ADDR = 2, | |
| 102 | − MCA_REG_IDX_MISC0 = 3, | |
| 103 | − MCA_REG_IDX_IPID = 5, | |
| 104 | − MCA_REG_IDX_SYND = 6, | |
| 105 | − MCA_REG_IDX_COUNT = 16, | |
| 106 | −}; | |
| 107 | − | |
| 108 | −struct mca_bank_info { | |
| 109 | − int socket_id; | |
| 110 | − int aid; | |
| 111 | − int hwid; | |
| 112 | − int mcatype; | |
| 113 | −}; | |
| 114 | − | |
| 115 | −struct mca_bank_entry { | |
| 116 | − int idx; | |
| 117 | − enum amdgpu_mca_error_type type; | |
| 118 | − enum amdgpu_mca_ip ip; | |
| 119 | − struct mca_bank_info info; | |
| 120 | − uint64_t regs[MCA_MAX_REGS_COUNT]; | |
| 121 | −}; | |
| 122 | − | |
| 123 | −struct mca_bank_node { | |
| 124 | − struct mca_bank_entry entry; | |
| 125 | − struct list_head node; | |
| 126 | −}; | |
| 127 | − | |
| 128 | −struct amdgpu_mca_smu_funcs { | |
| 129 | − int max_ue_count; | |
| 130 | − int max_ce_count; | |
| 131 | − int (*mca_set_debug_mode)(struct amdgpu_device *adev, bool enable); | |
| 132 | − int (*mca_parse_mca_error_count)(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type, | |
| 133 | − struct mca_bank_entry *entry, uint32_t *count); | |
| 134 | − int (*mca_get_valid_mca_count)(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, | |
| 135 | − uint32_t *count); | |
| 136 | − int (*mca_get_mca_entry)(struct amdgpu_device *adev, enum amdgpu_mca_error_type type, | |
| 137 | − int idx, struct mca_bank_entry *entry); | |
| 138 | 45 | }; |
| 139 | 46 | |
| 140 | 47 | void amdgpu_mca_query_correctable_error_count(struct amdgpu_device *adev, |
| 141 | 48 | uint64_t mc_status_addr, |
| 142 | 49 | unsigned long *error_count); |
| 143 | − | |
| 144 | 50 | void amdgpu_mca_query_uncorrectable_error_count(struct amdgpu_device *adev, |
| 145 | 51 | uint64_t mc_status_addr, |
| 146 | 52 | unsigned long *error_count); |
| 147 | − | |
| 148 | 53 | void amdgpu_mca_reset_error_count(struct amdgpu_device *adev, |
| 149 | 54 | uint64_t mc_status_addr); |
| 150 | − | |
| 151 | 55 | void amdgpu_mca_query_ras_error_count(struct amdgpu_device *adev, |
| 152 | 56 | uint64_t mc_status_addr, |
| 153 | 57 | void *ras_error_status); |
| @@ -155,15 +59,4 @@ int amdgpu_mca_mp0_ras_sw_init(struct amdgpu_device *adev); · 1 unmodified line | ||
| 155 | 59 | int amdgpu_mca_mp1_ras_sw_init(struct amdgpu_device *adev); |
| 156 | 60 | int amdgpu_mca_mpio_ras_sw_init(struct amdgpu_device *adev); |
| 157 | 61 | |
| 158 | −void amdgpu_mca_smu_init_funcs(struct amdgpu_device *adev, const struct amdgpu_mca_smu_funcs *mca_funcs); | |
| 159 | −int amdgpu_mca_init(struct amdgpu_device *adev); | |
| 160 | −void amdgpu_mca_fini(struct amdgpu_device *adev); | |
| 161 | −int amdgpu_mca_reset(struct amdgpu_device *adev); | |
| 162 | −int amdgpu_mca_smu_set_debug_mode(struct amdgpu_device *adev, bool enable); | |
| 163 | −int amdgpu_mca_smu_get_mca_set_error_count(struct amdgpu_device *adev, enum amdgpu_ras_block blk, | |
| 164 | − enum amdgpu_mca_error_type type, uint32_t *total); | |
| 165 | −void amdgpu_mca_smu_debugfs_init(struct amdgpu_device *adev, struct dentry *root); | |
| 166 | −int amdgpu_mca_smu_log_ras_error(struct amdgpu_device *adev, enum amdgpu_ras_block blk, enum amdgpu_mca_error_type type, | |
| 167 | − struct ras_err_data *err_data, struct ras_query_context *qctx); | |
| 168 | − | |
| 169 | 62 | #endif |
| @@ -217,7 +217,7 @@ int amdgpu_mes_init(struct amdgpu_device *adev) · 216 unmodified lines | ||
| 217 | 217 | if (r) |
| 218 | 218 | goto error_doorbell; |
| 219 | 219 | |
| 220 | − if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(12, 1, 0)) { | |
| 220 | + if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) { | |
| 221 | 221 | /* When queue/pipe reset is done in MES instead of in the |
| 222 | 222 | * driver, MES passes hung queues information to the driver in |
| 223 | 223 | * hung_queue_hqd_info. Calculate required space to store this |
| @@ -252,6 +252,10 @@ int amdgpu_mes_init(struct amdgpu_device *adev) · 28 unmodified lines | ||
| 252 | 252 | } |
| 253 | 253 | } |
| 254 | 254 | |
| 255 | + adev->gfx.mec.mes_hung_db_array = | |
| 256 | + kcalloc(amdgpu_mes_get_hung_queue_db_array_size(adev), | |
| 257 | + sizeof(u32), GFP_KERNEL); | |
| 258 | + | |
| 255 | 259 | return 0; |
| 256 | 260 | |
| 257 | 261 | error_doorbell: |
| @@ -279,6 +283,8 @@ void amdgpu_mes_fini(struct amdgpu_device *adev) · 21 unmodified lines | ||
| 279 | 283 | int i; |
| 280 | 284 | int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; |
| 281 | 285 | |
| 286 | + kfree(adev->gfx.mec.mes_hung_db_array); | |
| 287 | + | |
| 282 | 288 | amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj, |
| 283 | 289 | &adev->mes.event_log_gpu_addr, |
| 284 | 290 | &adev->mes.event_log_cpu_addr); |
| @@ -439,6 +445,59 @@ int amdgpu_mes_reset_legacy_queue(struct amdgpu_device *adev, · 154 unmodified lines | ||
| 439 | 445 | return r; |
| 440 | 446 | } |
| 441 | 447 | |
| 448 | +int amdgpu_mes_reset_queue_mmio(struct amdgpu_device *adev, | |
| 449 | + int queue_type, | |
| 450 | + unsigned int vmid, | |
| 451 | + unsigned int me, | |
| 452 | + unsigned int pipe, | |
| 453 | + unsigned int queue, | |
| 454 | + uint32_t xcc_id) | |
| 455 | +{ | |
| 456 | + struct mes_reset_queue_input queue_input; | |
| 457 | + int r; | |
| 458 | + | |
| 459 | + memset(&queue_input, 0, sizeof(queue_input)); | |
| 460 | + | |
| 461 | + queue_input.xcc_id = xcc_id; | |
| 462 | + queue_input.me_id = me; | |
| 463 | + queue_input.pipe_id = pipe; | |
| 464 | + queue_input.queue_id = queue; | |
| 465 | + queue_input.vmid = vmid; | |
| 466 | + queue_input.queue_type = queue_type; | |
| 467 | + queue_input.use_mmio = true; | |
| 468 | + | |
| 469 | + amdgpu_mes_lock(&adev->mes); | |
| 470 | + r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); | |
| 471 | + amdgpu_mes_unlock(&adev->mes); | |
| 472 | + if (r) | |
| 473 | + dev_err(adev->dev, "failed to reset legacy queue\n"); | |
| 474 | + | |
| 475 | + return r; | |
| 476 | +} | |
| 477 | + | |
| 478 | +int amdgpu_mes_reset_user_queue(struct amdgpu_device *adev, | |
| 479 | + int queue_type, | |
| 480 | + unsigned int doorbell_index, | |
| 481 | + unsigned int xcc_id) | |
| 482 | +{ | |
| 483 | + struct mes_reset_queue_input queue_input; | |
| 484 | + int r; | |
| 485 | + | |
| 486 | + memset(&queue_input, 0, sizeof(queue_input)); | |
| 487 | + | |
| 488 | + queue_input.xcc_id = xcc_id; | |
| 489 | + queue_input.queue_type = queue_type; | |
| 490 | + queue_input.doorbell_offset = doorbell_index; | |
| 491 | + | |
| 492 | + amdgpu_mes_lock(&adev->mes); | |
| 493 | + r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); | |
| 494 | + amdgpu_mes_unlock(&adev->mes); | |
| 495 | + if (r) | |
| 496 | + dev_err(adev->dev, "failed to reset user queue\n"); | |
| 497 | + | |
| 498 | + return r; | |
| 499 | +} | |
| 500 | + | |
| 442 | 501 | int amdgpu_mes_get_hung_queue_db_array_size(struct amdgpu_device *adev) |
| 443 | 502 | { |
| 444 | 503 | return adev->mes.hung_queue_db_array_size; |
| @@ -805,8 +864,13 @@ bool amdgpu_mes_suspend_resume_all_supported(struct amdgpu_device *adev) · 360 unmodified lines | ||
| 805 | 864 | |
| 806 | 865 | bool amdgpu_mes_queue_reset_by_mes_supported(struct amdgpu_device *adev) |
| 807 | 866 | { |
| 808 | − return (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0) && | |
| 809 | − (adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x73); | |
| 867 | + u32 ip_maj = IP_VERSION_MAJ(amdgpu_ip_version(adev, GC_HWIP, 0)); | |
| 868 | + u32 ip_min = IP_VERSION_MIN(amdgpu_ip_version(adev, GC_HWIP, 0)); | |
| 869 | + u32 mes_sched = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK; | |
| 870 | + | |
| 871 | + return (ip_maj == 11 && mes_sched >= 0x8c) || | |
| 872 | + ((ip_maj == 12 && ip_min == 0) && mes_sched >= 0x8d) || | |
| 873 | + ((ip_maj == 12 && ip_min == 1) && mes_sched >= 0x73); | |
| 810 | 874 | } |
| 811 | 875 | |
| 812 | 876 | /* Fix me -- node_id is used to identify the correct MES instances in the future */ |
| @@ -168,6 +168,9 @@ struct amdgpu_mes { · 167 unmodified lines | ||
| 168 | 168 | int master_xcc_ids[AMDGPU_MAX_MES_INST_PIPES]; |
| 169 | 169 | struct amdgpu_bo *shared_cmd_buf_obj[AMDGPU_MAX_MES_INST_PIPES]; |
| 170 | 170 | uint64_t shared_cmd_buf_gpu_addr[AMDGPU_MAX_MES_INST_PIPES]; |
| 171 | + | |
| 172 | + bool compute_pipe_reset_enabled; | |
| 173 | + bool gfx_pipe_reset_enabled; | |
| 171 | 174 | }; |
| 172 | 175 | |
| 173 | 176 | struct amdgpu_mes_hung_queue_hqd_info { |
| @@ -271,6 +274,7 @@ struct mes_remove_queue_input { · 97 unmodified lines | ||
| 271 | 274 | uint32_t xcc_id; |
| 272 | 275 | uint32_t doorbell_offset; |
| 273 | 276 | uint64_t gang_context_addr; |
| 277 | + uint32_t queue_type; | |
| 274 | 278 | bool remove_queue_after_reset; |
| 275 | 279 | }; |
| 276 | 280 | |
| @@ -461,6 +465,17 @@ int amdgpu_mes_reset_legacy_queue(struct amdgpu_device *adev, · 184 unmodified lines | ||
| 461 | 465 | unsigned int vmid, |
| 462 | 466 | bool use_mmio, |
| 463 | 467 | uint32_t xcc_id); |
| 468 | +int amdgpu_mes_reset_queue_mmio(struct amdgpu_device *adev, | |
| 469 | + int queue_type, | |
| 470 | + unsigned int vmid, | |
| 471 | + unsigned int me, | |
| 472 | + unsigned int pipe, | |
| 473 | + unsigned int queue, | |
| 474 | + uint32_t xcc_id); | |
| 475 | +int amdgpu_mes_reset_user_queue(struct amdgpu_device *adev, | |
| 476 | + int queue_type, | |
| 477 | + unsigned int doorbell_index, | |
| 478 | + unsigned int xcc_id); | |
| 464 | 479 | |
| 465 | 480 | int amdgpu_mes_get_hung_queue_db_array_size(struct amdgpu_device *adev); |
| 466 | 481 | int amdgpu_mes_detect_and_reset_hung_queues(struct amdgpu_device *adev, |
| @@ -21,29 +21,6 @@ · 20 unmodified lines | ||
| 21 | 21 | #ifndef __AMDGPU_MMHUB_H__ |
| 22 | 22 | #define __AMDGPU_MMHUB_H__ |
| 23 | 23 | |
| 24 | −enum amdgpu_mmhub_ras_memory_id { | |
| 25 | − AMDGPU_MMHUB_WGMI_PAGEMEM = 0, | |
| 26 | − AMDGPU_MMHUB_RGMI_PAGEMEM = 1, | |
| 27 | − AMDGPU_MMHUB_WDRAM_PAGEMEM = 2, | |
| 28 | − AMDGPU_MMHUB_RDRAM_PAGEMEM = 3, | |
| 29 | − AMDGPU_MMHUB_WIO_CMDMEM = 4, | |
| 30 | − AMDGPU_MMHUB_RIO_CMDMEM = 5, | |
| 31 | − AMDGPU_MMHUB_WGMI_CMDMEM = 6, | |
| 32 | − AMDGPU_MMHUB_RGMI_CMDMEM = 7, | |
| 33 | − AMDGPU_MMHUB_WDRAM_CMDMEM = 8, | |
| 34 | − AMDGPU_MMHUB_RDRAM_CMDMEM = 9, | |
| 35 | − AMDGPU_MMHUB_MAM_DMEM0 = 10, | |
| 36 | − AMDGPU_MMHUB_MAM_DMEM1 = 11, | |
| 37 | − AMDGPU_MMHUB_MAM_DMEM2 = 12, | |
| 38 | − AMDGPU_MMHUB_MAM_DMEM3 = 13, | |
| 39 | − AMDGPU_MMHUB_WRET_TAGMEM = 19, | |
| 40 | − AMDGPU_MMHUB_RRET_TAGMEM = 20, | |
| 41 | − AMDGPU_MMHUB_WIO_DATAMEM = 21, | |
| 42 | − AMDGPU_MMHUB_WGMI_DATAMEM = 22, | |
| 43 | − AMDGPU_MMHUB_WDRAM_DATAMEM = 23, | |
| 44 | − AMDGPU_MMHUB_MEMORY_BLOCK_LAST, | |
| 45 | −}; | |
| 46 | − | |
| 47 | 24 | struct amdgpu_mmhub_ras { |
| 48 | 25 | struct amdgpu_ras_block_object ras_block; |
| 49 | 26 | }; |
| @@ -312,46 +312,6 @@ uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo); · 311 unmodified lines | ||
| 312 | 312 | uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev, |
| 313 | 313 | uint32_t domain); |
| 314 | 314 | |
| 315 | −/* | |
| 316 | − * sub allocation | |
| 317 | − */ | |
| 318 | −static inline struct amdgpu_sa_manager * | |
| 319 | −to_amdgpu_sa_manager(struct drm_suballoc_manager *manager) | |
| 320 | −{ | |
| 321 | − return container_of(manager, struct amdgpu_sa_manager, base); | |
| 322 | −} | |
| 323 | − | |
| 324 | −static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo) | |
| 325 | −{ | |
| 326 | − return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr + | |
| 327 | − drm_suballoc_soffset(sa_bo); | |
| 328 | −} | |
| 329 | − | |
| 330 | −static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo) | |
| 331 | −{ | |
| 332 | − return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr + | |
| 333 | − drm_suballoc_soffset(sa_bo); | |
| 334 | −} | |
| 335 | − | |
| 336 | −int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev, | |
| 337 | − struct amdgpu_sa_manager *sa_manager, | |
| 338 | − unsigned size, u32 align, u32 domain); | |
| 339 | −void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev, | |
| 340 | − struct amdgpu_sa_manager *sa_manager); | |
| 341 | −int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev, | |
| 342 | − struct amdgpu_sa_manager *sa_manager); | |
| 343 | −int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager, | |
| 344 | − struct drm_suballoc **sa_bo, | |
| 345 | − unsigned int size); | |
| 346 | −void amdgpu_sa_bo_free(struct drm_suballoc **sa_bo, | |
| 347 | − struct dma_fence *fence); | |
| 348 | −#if defined(CONFIG_DEBUG_FS) | |
| 349 | −void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager, | |
| 350 | − struct seq_file *m); | |
| 351 | −u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m); | |
| 352 | −#endif | |
| 353 | −void amdgpu_debugfs_sa_init(struct amdgpu_device *adev); | |
| 354 | − | |
| 355 | 315 | bool amdgpu_bo_support_uswc(u64 bo_flags); |
| 356 | 316 | |
| 357 | 317 | |
| @@ -46,6 +46,17 @@ static ssize_t mem_info_preempt_used_show(struct device *dev, · 45 unmodified lines | ||
| 46 | 46 | |
| 47 | 47 | static DEVICE_ATTR_RO(mem_info_preempt_used); |
| 48 | 48 | |
| 49 | +/** | |
| 50 | + * amdgpu_preempt_mgr_sysfs_fini - remove PREEMPT manager sysfs attributes | |
| 51 | + * | |
| 52 | + * @adev: amdgpu_device pointer | |
| 53 | + */ | |
| 54 | +void amdgpu_preempt_mgr_sysfs_fini(struct amdgpu_device *adev) | |
| 55 | +{ | |
| 56 | + if (adev->dev->kobj.sd) | |
| 57 | + device_remove_file(adev->dev, &dev_attr_mem_info_preempt_used); | |
| 58 | +} | |
| 59 | + | |
| 49 | 60 | /** |
| 50 | 61 | * amdgpu_preempt_mgr_new - allocate a new node |
| 51 | 62 | * |
| @@ -137,9 +148,6 @@ void amdgpu_preempt_mgr_fini(struct amdgpu_device *adev) · 85 unmodified lines | ||
| 137 | 148 | if (ret) |
| 138 | 149 | return; |
| 139 | 150 | |
| 140 | − if (adev->dev->kobj.sd) | |
| 141 | − device_remove_file(adev->dev, &dev_attr_mem_info_preempt_used); | |
| 142 | − | |
| 143 | 151 | ttm_resource_manager_cleanup(man); |
| 144 | 152 | ttm_set_driver_manager(&adev->mman.bdev, AMDGPU_PL_PREEMPT, NULL); |
| 145 | 153 | } |
| @@ -166,7 +166,8 @@ static ssize_t ta_if_load_debugfs_write(struct file *fp, const char *buf, size_t · 165 unmodified lines | ||
| 166 | 166 | if (ret) |
| 167 | 167 | return -EFAULT; |
| 168 | 168 | |
| 169 | − if (ta_bin_len > PSP_1_MEG) | |
| 169 | + if (ta_bin_len < sizeof(struct common_firmware_header) || | |
| 170 | + ta_bin_len > PSP_1_MEG) | |
| 170 | 171 | return -EINVAL; |
| 171 | 172 | |
| 172 | 173 | copy_pos += sizeof(uint32_t); |
| @@ -321,6 +322,8 @@ static ssize_t ta_if_invoke_debugfs_write(struct file *fp, const char *buf, size · 148 unmodified lines | ||
| 321 | 322 | ret = copy_from_user((void *)&shared_buf_len, &buf[copy_pos], sizeof(uint32_t)); |
| 322 | 323 | if (ret) |
| 323 | 324 | return -EFAULT; |
| 325 | + if (!shared_buf_len || shared_buf_len > PSP_1_MEG) | |
| 326 | + return -EINVAL; | |
| 324 | 327 | copy_pos += sizeof(uint32_t); |
| 325 | 328 | |
| 326 | 329 | shared_buf = memdup_user(&buf[copy_pos], shared_buf_len); |
| @@ -128,12 +128,6 @@ const char *get_ras_block_str(struct ras_common_if *ras_block) · 127 unmodified lines | ||
| 128 | 128 | /* typical ECC bad page rate is 1 bad page per 100MB VRAM */ |
| 129 | 129 | #define RAS_BAD_PAGE_COVER (100 * 1024 * 1024ULL) |
| 130 | 130 | |
| 131 | −#define MAX_UMC_POISON_POLLING_TIME_ASYNC 10 | |
| 132 | − | |
| 133 | −#define AMDGPU_RAS_RETIRE_PAGE_INTERVAL 100 //ms | |
| 134 | − | |
| 135 | −#define MAX_FLUSH_RETIRE_DWORK_TIMES 100 | |
| 136 | − | |
| 137 | 131 | #define BYPASS_ALLOCATED_ADDRESS 0x0 |
| 138 | 132 | #define BYPASS_INITIALIZATION_ADDRESS 0x1 |
| 139 | 133 | |
| @@ -249,16 +243,12 @@ static int amdgpu_check_address_validity(struct amdgpu_device *adev, · 109 unmodified lines | ||
| 249 | 243 | (address >= RAS_UMC_INJECT_ADDR_LIMIT)) |
| 250 | 244 | return -EFAULT; |
| 251 | 245 | |
| 252 | − if (amdgpu_uniras_enabled(adev)) { | |
| 253 | − if (amdgpu_sriov_vf(adev)) | |
| 254 | − count = amdgpu_virt_ras_convert_retired_address(adev, address, | |
| 255 | − page_pfns, ARRAY_SIZE(page_pfns)); | |
| 256 | − else | |
| 257 | − count = amdgpu_ras_mgr_lookup_bad_pages_in_a_row(adev, address, | |
| 258 | − page_pfns, ARRAY_SIZE(page_pfns)); | |
| 259 | − } else | |
| 260 | − count = amdgpu_umc_lookup_bad_pages_in_a_row(adev, | |
| 261 | − address, page_pfns, ARRAY_SIZE(page_pfns)); | |
| 246 | + if (amdgpu_sriov_vf(adev)) | |
| 247 | + count = amdgpu_virt_ras_convert_retired_address(adev, address, | |
| 248 | + page_pfns, ARRAY_SIZE(page_pfns)); | |
| 249 | + else | |
| 250 | + count = amdgpu_ras_mgr_lookup_bad_pages_in_a_row(adev, address, | |
| 251 | + page_pfns, ARRAY_SIZE(page_pfns)); | |
| 262 | 252 | |
| 263 | 253 | if (count <= 0) |
| 264 | 254 | return -EPERM; |
| @@ -1381,76 +1371,6 @@ static void amdgpu_ras_mgr_virt_error_data_statistics_update(struct ras_manager · 1116 unmodified lines | ||
| 1381 | 1371 | obj->err_data.de_count = err_data->de_count; |
| 1382 | 1372 | } |
| 1383 | 1373 | |
| 1384 | −static struct ras_manager *get_ras_manager(struct amdgpu_device *adev, enum amdgpu_ras_block blk) | |
| 1385 | −{ | |
| 1386 | − struct ras_common_if head; | |
| 1387 | − | |
| 1388 | − memset(&head, 0, sizeof(head)); | |
| 1389 | − head.block = blk; | |
| 1390 | − | |
| 1391 | − return amdgpu_ras_find_obj(adev, &head); | |
| 1392 | −} | |
| 1393 | − | |
| 1394 | −int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk, | |
| 1395 | − const struct aca_info *aca_info, void *data) | |
| 1396 | −{ | |
| 1397 | − struct ras_manager *obj; | |
| 1398 | − | |
| 1399 | − /* in resume phase, no need to create aca fs node */ | |
| 1400 | − if (adev->in_suspend || amdgpu_reset_in_recovery(adev)) | |
| 1401 | − return 0; | |
| 1402 | − | |
| 1403 | − obj = get_ras_manager(adev, blk); | |
| 1404 | − if (!obj) | |
| 1405 | − return -EINVAL; | |
| 1406 | − | |
| 1407 | − return amdgpu_aca_add_handle(adev, &obj->aca_handle, ras_block_str(blk), aca_info, data); | |
| 1408 | −} | |
| 1409 | − | |
| 1410 | −int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk) | |
| 1411 | −{ | |
| 1412 | − struct ras_manager *obj; | |
| 1413 | − | |
| 1414 | − obj = get_ras_manager(adev, blk); | |
| 1415 | − if (!obj) | |
| 1416 | − return -EINVAL; | |
| 1417 | − | |
| 1418 | − amdgpu_aca_remove_handle(&obj->aca_handle); | |
| 1419 | − | |
| 1420 | − return 0; | |
| 1421 | −} | |
| 1422 | − | |
| 1423 | −static int amdgpu_aca_log_ras_error_data(struct amdgpu_device *adev, enum amdgpu_ras_block blk, | |
| 1424 | − enum aca_error_type type, struct ras_err_data *err_data, | |
| 1425 | − struct ras_query_context *qctx) | |
| 1426 | −{ | |
| 1427 | − struct ras_manager *obj; | |
| 1428 | − | |
| 1429 | − obj = get_ras_manager(adev, blk); | |
| 1430 | − if (!obj) | |
| 1431 | − return -EINVAL; | |
| 1432 | − | |
| 1433 | − return amdgpu_aca_get_error_data(adev, &obj->aca_handle, type, err_data, qctx); | |
| 1434 | −} | |
| 1435 | − | |
| 1436 | −ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr, | |
| 1437 | − struct aca_handle *handle, char *buf, void *data) | |
| 1438 | −{ | |
| 1439 | − struct ras_manager *obj = container_of(handle, struct ras_manager, aca_handle); | |
| 1440 | − struct ras_query_if info = { | |
| 1441 | − .head = obj->head, | |
| 1442 | − }; | |
| 1443 | − | |
| 1444 | − if (!amdgpu_ras_get_error_query_ready(obj->adev)) | |
| 1445 | − return sysfs_emit(buf, "Query currently inaccessible\n"); | |
| 1446 | − | |
| 1447 | − if (amdgpu_ras_query_error_status(obj->adev, &info)) | |
| 1448 | − return -EINVAL; | |
| 1449 | − | |
| 1450 | − return sysfs_emit(buf, "%s: %lu\n%s: %lu\n%s: %lu\n", "ue", info.ue_count, | |
| 1451 | − "ce", info.ce_count, "de", info.de_count); | |
| 1452 | −} | |
| 1453 | − | |
| 1454 | 1374 | static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, |
| 1455 | 1375 | struct ras_query_if *info, |
| 1456 | 1376 | struct ras_err_data *err_data, |
| @@ -1459,7 +1379,6 @@ static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, · 2 unmodified lines | ||
| 1459 | 1379 | { |
| 1460 | 1380 | enum amdgpu_ras_block blk = info ? info->head.block : AMDGPU_RAS_BLOCK_COUNT; |
| 1461 | 1381 | struct amdgpu_ras_block_object *block_obj = NULL; |
| 1462 | − int ret; | |
| 1463 | 1382 | |
| 1464 | 1383 | if (blk == AMDGPU_RAS_BLOCK_COUNT) |
| 1465 | 1384 | return -EINVAL; |
| @@ -1469,7 +1388,7 @@ static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, · 3 unmodified lines | ||
| 1469 | 1388 | |
| 1470 | 1389 | if (error_query_mode == AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY) { |
| 1471 | 1390 | return amdgpu_virt_req_ras_err_count(adev, blk, err_data); |
| 1472 | − } else if (error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { | |
| 1391 | + } else { | |
| 1473 | 1392 | if (info->head.block == AMDGPU_RAS_BLOCK__UMC) { |
| 1474 | 1393 | amdgpu_ras_get_ecc_info(adev, err_data); |
| 1475 | 1394 | } else { |
| @@ -1490,24 +1409,6 @@ static int amdgpu_ras_query_error_status_helper(struct amdgpu_device *adev, · 14 unmodified lines | ||
| 1490 | 1409 | block_obj->hw_ops->query_ras_error_status(adev); |
| 1491 | 1410 | } |
| 1492 | 1411 | } |
| 1493 | − } else { | |
| 1494 | − if (amdgpu_aca_is_enabled(adev)) { | |
| 1495 | − ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_UE, err_data, qctx); | |
| 1496 | − if (ret) | |
| 1497 | − return ret; | |
| 1498 | − | |
| 1499 | − ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_CE, err_data, qctx); | |
| 1500 | − if (ret) | |
| 1501 | − return ret; | |
| 1502 | − | |
| 1503 | − ret = amdgpu_aca_log_ras_error_data(adev, blk, ACA_ERROR_TYPE_DEFERRED, err_data, qctx); | |
| 1504 | − if (ret) | |
| 1505 | − return ret; | |
| 1506 | − } else { | |
| 1507 | − /* FIXME: add code to check return value later */ | |
| 1508 | − amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_UE, err_data, qctx); | |
| 1509 | − amdgpu_mca_smu_log_ras_error(adev, blk, AMDGPU_MCA_ERROR_TYPE_CE, err_data, qctx); | |
| 1510 | − } | |
| 1511 | 1412 | } |
| 1512 | 1413 | |
| 1513 | 1414 | return 0; |
| @@ -1624,8 +1525,6 @@ int amdgpu_ras_reset_error_count(struct amdgpu_device *adev, · 110 unmodified lines | ||
| 1624 | 1525 | enum amdgpu_ras_block block) |
| 1625 | 1526 | { |
| 1626 | 1527 | struct amdgpu_ras_block_object *block_obj = amdgpu_ras_get_ras_block(adev, block, 0); |
| 1627 | − const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; | |
| 1628 | − const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; | |
| 1629 | 1528 | |
| 1630 | 1529 | if (!block_obj || !block_obj->hw_ops) { |
| 1631 | 1530 | dev_dbg_once(adev->dev, "%s doesn't config RAS function\n", |
| @@ -1633,17 +1532,14 @@ int amdgpu_ras_reset_error_count(struct amdgpu_device *adev, · 1 unmodified line | ||
| 1633 | 1532 | return -EOPNOTSUPP; |
| 1634 | 1533 | } |
| 1635 | 1534 | |
| 1636 | − if (!amdgpu_ras_is_supported(adev, block) || | |
| 1637 | − !amdgpu_ras_get_aca_debug_mode(adev)) | |
| 1535 | + if (!amdgpu_ras_is_supported(adev, block)) | |
| 1638 | 1536 | return -EOPNOTSUPP; |
| 1639 | 1537 | |
| 1640 | 1538 | if (amdgpu_sriov_vf(adev)) |
| 1641 | 1539 | return -EOPNOTSUPP; |
| 1642 | 1540 | |
| 1643 | 1541 | /* skip ras error reset in gpu reset */ |
| 1644 | − if ((amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) && | |
| 1645 | − ((smu_funcs && smu_funcs->set_debug_mode) || | |
| 1646 | − (mca_funcs && mca_funcs->mca_set_debug_mode))) | |
| 1542 | + if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) | |
| 1647 | 1543 | return -EOPNOTSUPP; |
| 1648 | 1544 | |
| 1649 | 1545 | if (block_obj->hw_ops->reset_ras_error_count) |
| @@ -2090,9 +1986,6 @@ int amdgpu_ras_sysfs_create(struct amdgpu_device *adev, · 440 unmodified lines | ||
| 2090 | 1986 | { |
| 2091 | 1987 | struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); |
| 2092 | 1988 | |
| 2093 | − if (amdgpu_aca_is_enabled(adev)) | |
| 2094 | − return 0; | |
| 2095 | − | |
| 2096 | 1989 | if (!obj || obj->attr_inuse) |
| 2097 | 1990 | return -EINVAL; |
| 2098 | 1991 | |
| @@ -2130,9 +2023,6 @@ int amdgpu_ras_sysfs_remove(struct amdgpu_device *adev, · 31 unmodified lines | ||
| 2130 | 2023 | { |
| 2131 | 2024 | struct ras_manager *obj = amdgpu_ras_find_obj(adev, head); |
| 2132 | 2025 | |
| 2133 | − if (amdgpu_aca_is_enabled(adev)) | |
| 2134 | − return 0; | |
| 2135 | − | |
| 2136 | 2026 | if (!obj || !obj->attr_inuse) |
| 2137 | 2027 | return -EINVAL; |
| 2138 | 2028 | |
| @@ -2245,25 +2135,6 @@ static void amdgpu_ras_debugfs_create(struct amdgpu_device *adev, · 106 unmodified lines | ||
| 2245 | 2135 | obj, &amdgpu_ras_debugfs_ops); |
| 2246 | 2136 | } |
| 2247 | 2137 | |
| 2248 | −static bool amdgpu_ras_aca_is_supported(struct amdgpu_device *adev) | |
| 2249 | −{ | |
| 2250 | − bool ret; | |
| 2251 | − | |
| 2252 | − switch (amdgpu_ip_version(adev, MP0_HWIP, 0)) { | |
| 2253 | − case IP_VERSION(13, 0, 6): | |
| 2254 | − case IP_VERSION(13, 0, 12): | |
| 2255 | − case IP_VERSION(13, 0, 14): | |
| 2256 | − case IP_VERSION(13, 0, 15): | |
| 2257 | − ret = true; | |
| 2258 | − break; | |
| 2259 | − default: | |
| 2260 | − ret = false; | |
| 2261 | − break; | |
| 2262 | − } | |
| 2263 | − | |
| 2264 | − return ret; | |
| 2265 | −} | |
| 2266 | − | |
| 2267 | 2138 | void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) |
| 2268 | 2139 | { |
| 2269 | 2140 | struct amdgpu_ras *con = amdgpu_ras_get_context(adev); |
| @@ -2290,13 +2161,6 @@ void amdgpu_ras_debugfs_create_all(struct amdgpu_device *adev) · 20 unmodified lines | ||
| 2290 | 2161 | amdgpu_ras_debugfs_create(adev, &fs_info, dir); |
| 2291 | 2162 | } |
| 2292 | 2163 | } |
| 2293 | − | |
| 2294 | − if (amdgpu_ras_aca_is_supported(adev)) { | |
| 2295 | − if (amdgpu_aca_is_enabled(adev)) | |
| 2296 | − amdgpu_aca_smu_debugfs_init(adev, dir); | |
| 2297 | − else | |
| 2298 | − amdgpu_mca_smu_debugfs_init(adev, dir); | |
| 2299 | − } | |
| 2300 | 2164 | } |
| 2301 | 2165 | |
| 2302 | 2166 | /* debugfs end */ |
| @@ -2489,14 +2353,6 @@ static void amdgpu_ras_interrupt_poison_creation_handler(struct ras_manager *obj · 186 unmodified lines | ||
| 2489 | 2353 | event_id = amdgpu_ras_acquire_event_id(adev, type); |
| 2490 | 2354 | RAS_EVENT_LOG(adev, event_id, "Poison is created\n"); |
| 2491 | 2355 | |
| 2492 | − if (amdgpu_ip_version(obj->adev, UMC_HWIP, 0) >= IP_VERSION(12, 0, 0)) { | |
| 2493 | − struct amdgpu_ras *con = amdgpu_ras_get_context(obj->adev); | |
| 2494 | − | |
| 2495 | − atomic_inc(&con->page_retirement_req_cnt); | |
| 2496 | − atomic_inc(&con->poison_creation_count); | |
| 2497 | − | |
| 2498 | − wake_up(&con->page_retirement_wq); | |
| 2499 | − } | |
| 2500 | 2356 | } |
| 2501 | 2357 | |
| 2502 | 2358 | static void amdgpu_ras_interrupt_umc_handler(struct ras_manager *obj, |
| @@ -3026,77 +2882,6 @@ static int amdgpu_ras_realloc_eh_data_space(struct amdgpu_device *adev, · 523 unmodified lines | ||
| 3026 | 2882 | return 0; |
| 3027 | 2883 | } |
| 3028 | 2884 | |
| 3029 | −static int amdgpu_ras_mca2pa_by_idx(struct amdgpu_device *adev, | |
| 3030 | − struct eeprom_table_record *bps, | |
| 3031 | − struct ras_err_data *err_data) | |
| 3032 | −{ | |
| 3033 | − struct ta_ras_query_address_input addr_in; | |
| 3034 | − uint32_t socket = 0; | |
| 3035 | − int ret = 0; | |
| 3036 | − | |
| 3037 | − if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) | |
| 3038 | − socket = adev->smuio.funcs->get_socket_id(adev); | |
| 3039 | − | |
| 3040 | − /* reinit err_data */ | |
| 3041 | − err_data->err_addr_cnt = 0; | |
| 3042 | − err_data->err_addr_len = adev->umc.retire_unit; | |
| 3043 | − | |
| 3044 | − memset(&addr_in, 0, sizeof(addr_in)); | |
| 3045 | − addr_in.ma.err_addr = bps->address; | |
| 3046 | − addr_in.ma.socket_id = socket; | |
| 3047 | − addr_in.ma.ch_inst = bps->mem_channel; | |
| 3048 | − if (!amdgpu_ras_smu_eeprom_supported(adev)) { | |
| 3049 | − /* tell RAS TA the node instance is not used */ | |
| 3050 | − addr_in.ma.node_inst = TA_RAS_INV_NODE; | |
| 3051 | − } else { | |
| 3052 | − addr_in.ma.umc_inst = bps->mcumc_id; | |
| 3053 | − addr_in.ma.node_inst = bps->cu; | |
| 3054 | − } | |
| 3055 | − | |
| 3056 | − if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) | |
| 3057 | − ret = adev->umc.ras->convert_ras_err_addr(adev, err_data, | |
| 3058 | − &addr_in, NULL, false); | |
| 3059 | − | |
| 3060 | − return ret; | |
| 3061 | −} | |
| 3062 | − | |
| 3063 | −static int amdgpu_ras_mca2pa(struct amdgpu_device *adev, | |
| 3064 | − struct eeprom_table_record *bps, | |
| 3065 | − struct ras_err_data *err_data) | |
| 3066 | −{ | |
| 3067 | − struct ta_ras_query_address_input addr_in; | |
| 3068 | − uint32_t die_id, socket = 0; | |
| 3069 | − | |
| 3070 | − if (adev->smuio.funcs && adev->smuio.funcs->get_socket_id) | |
| 3071 | − socket = adev->smuio.funcs->get_socket_id(adev); | |
| 3072 | − | |
| 3073 | − /* although die id is gotten from PA in nps1 mode, the id is | |
| 3074 | − * fitable for any nps mode | |
| 3075 | − */ | |
| 3076 | − if (adev->umc.ras && adev->umc.ras->get_die_id_from_pa) | |
| 3077 | − die_id = adev->umc.ras->get_die_id_from_pa(adev, bps->address, | |
| 3078 | − bps->retired_page << AMDGPU_GPU_PAGE_SHIFT); | |
| 3079 | − else | |
| 3080 | − return -EINVAL; | |
| 3081 | − | |
| 3082 | − /* reinit err_data */ | |
| 3083 | − err_data->err_addr_cnt = 0; | |
| 3084 | − err_data->err_addr_len = adev->umc.retire_unit; | |
| 3085 | − | |
| 3086 | − memset(&addr_in, 0, sizeof(addr_in)); | |
| 3087 | − addr_in.ma.err_addr = bps->address; | |
| 3088 | − addr_in.ma.ch_inst = bps->mem_channel; | |
| 3089 | − addr_in.ma.umc_inst = bps->mcumc_id; | |
| 3090 | − addr_in.ma.node_inst = die_id; | |
| 3091 | − addr_in.ma.socket_id = socket; | |
| 3092 | − | |
| 3093 | − if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) | |
| 3094 | − return adev->umc.ras->convert_ras_err_addr(adev, err_data, | |
| 3095 | − &addr_in, NULL, false); | |
| 3096 | − else | |
| 3097 | − return -EINVAL; | |
| 3098 | −} | |
| 3099 | − | |
| 3100 | 2885 | static bool __check_record_in_range(struct amdgpu_device *adev, |
| 3101 | 2886 | struct eeprom_table_record *bps, int count) |
| 3102 | 2887 | { |
| @@ -3157,117 +2942,13 @@ static int __amdgpu_ras_convert_rec_array_from_rom(struct amdgpu_device *adev, · 54 unmodified lines | ||
| 3157 | 2942 | struct eeprom_table_record *bps, struct ras_err_data *err_data, |
| 3158 | 2943 | enum amdgpu_memory_partition nps) |
| 3159 | 2944 | { |
| 3160 | − int i = 0; | |
| 3161 | − uint64_t chan_idx_v2; | |
| 3162 | − enum amdgpu_memory_partition save_nps; | |
| 3163 | − | |
| 3164 | − save_nps = (bps[0].retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; | |
| 3165 | − chan_idx_v2 = bps[0].retired_page & UMC_CHANNEL_IDX_V2; | |
| 3166 | − | |
| 3167 | 2945 | /*old asics just have pa in eeprom*/ |
| 3168 | − if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { | |
| 3169 | − memcpy(err_data->err_addr, bps, | |
| 3170 | − sizeof(struct eeprom_table_record) * adev->umc.retire_unit); | |
| 3171 | − goto out; | |
| 3172 | − } | |
| 3173 | − | |
| 3174 | − for (i = 0; i < adev->umc.retire_unit; i++) | |
| 3175 | − bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); | |
| 2946 | + memcpy(err_data->err_addr, bps, | |
| 2947 | + sizeof(struct eeprom_table_record) * adev->umc.retire_unit); | |
| 3176 | 2948 | |
| 3177 | − if (save_nps || chan_idx_v2) { | |
| 3178 | − if (save_nps == nps) { | |
| 3179 | − if (amdgpu_umc_pages_in_a_row(adev, err_data, | |
| 3180 | − bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT)) | |
| 3181 | − return -EINVAL; | |
| 3182 | − for (i = 0; i < adev->umc.retire_unit; i++) { | |
| 3183 | − err_data->err_addr[i].address = bps[0].address; | |
| 3184 | − err_data->err_addr[i].mem_channel = bps[0].mem_channel; | |
| 3185 | − err_data->err_addr[i].bank = bps[0].bank; | |
| 3186 | − err_data->err_addr[i].err_type = bps[0].err_type; | |
| 3187 | − err_data->err_addr[i].mcumc_id = bps[0].mcumc_id; | |
| 3188 | − } | |
| 3189 | − } else { | |
| 3190 | − if (amdgpu_ras_mca2pa_by_idx(adev, &bps[0], err_data)) | |
| 3191 | − return -EINVAL; | |
| 3192 | − } | |
| 3193 | − } else { | |
| 3194 | − if (bps[0].address == 0) { | |
| 3195 | − /* for specific old eeprom data, mca address is not stored, | |
| 3196 | − * calc it from pa | |
| 3197 | − */ | |
| 3198 | − if (amdgpu_umc_pa2mca(adev, bps[0].retired_page << AMDGPU_GPU_PAGE_SHIFT, | |
| 3199 | − &(bps[0].address), AMDGPU_NPS1_PARTITION_MODE)) | |
| 3200 | − return -EINVAL; | |
| 3201 | − } | |
| 3202 | − | |
| 3203 | − if (amdgpu_ras_mca2pa(adev, &bps[0], err_data)) { | |
| 3204 | − if (nps == AMDGPU_NPS1_PARTITION_MODE) | |
| 3205 | − memcpy(err_data->err_addr, bps, | |
| 3206 | − sizeof(struct eeprom_table_record) * adev->umc.retire_unit); | |
| 3207 | − else | |
| 3208 | − return -EOPNOTSUPP; | |
| 3209 | − } | |
| 3210 | − } | |
| 3211 | − | |
| 3212 | −out: | |
| 3213 | 2949 | return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, adev->umc.retire_unit); |
| 3214 | 2950 | } |
| 3215 | 2951 | |
| 3216 | −static int __amdgpu_ras_convert_rec_from_rom(struct amdgpu_device *adev, | |
| 3217 | − struct eeprom_table_record *bps, struct ras_err_data *err_data, | |
| 3218 | − enum amdgpu_memory_partition nps) | |
| 3219 | −{ | |
| 3220 | − int i = 0; | |
| 3221 | − uint64_t chan_idx_v2; | |
| 3222 | − enum amdgpu_memory_partition save_nps; | |
| 3223 | − | |
| 3224 | − if (!amdgpu_ras_smu_eeprom_supported(adev)) { | |
| 3225 | − save_nps = (bps->retired_page >> UMC_NPS_SHIFT) & UMC_NPS_MASK; | |
| 3226 | − chan_idx_v2 = bps->retired_page & UMC_CHANNEL_IDX_V2; | |
| 3227 | − bps->retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); | |
| 3228 | − } else { | |
| 3229 | − /* if pmfw manages eeprom, save_nps is not stored on eeprom, | |
| 3230 | − * we should always convert mca address into physical address, | |
| 3231 | − * make save_nps different from nps | |
| 3232 | − */ | |
| 3233 | − save_nps = nps + 1; | |
| 3234 | − } | |
| 3235 | − | |
| 3236 | − if (save_nps == nps) { | |
| 3237 | − if (amdgpu_umc_pages_in_a_row(adev, err_data, | |
| 3238 | − bps->retired_page << AMDGPU_GPU_PAGE_SHIFT)) | |
| 3239 | − return -EINVAL; | |
| 3240 | − for (i = 0; i < adev->umc.retire_unit; i++) { | |
| 3241 | − err_data->err_addr[i].address = bps->address; | |
| 3242 | − err_data->err_addr[i].mem_channel = bps->mem_channel; | |
| 3243 | − err_data->err_addr[i].bank = bps->bank; | |
| 3244 | − err_data->err_addr[i].err_type = bps->err_type; | |
| 3245 | − err_data->err_addr[i].mcumc_id = bps->mcumc_id; | |
| 3246 | − } | |
| 3247 | − } else { | |
| 3248 | − if (save_nps || chan_idx_v2) { | |
| 3249 | − if (amdgpu_ras_mca2pa_by_idx(adev, bps, err_data)) | |
| 3250 | − return -EINVAL; | |
| 3251 | − } else { | |
| 3252 | − /* for specific old eeprom data, mca address is not stored, | |
| 3253 | − * calc it from pa | |
| 3254 | − */ | |
| 3255 | − if (bps->address == 0) | |
| 3256 | − if (amdgpu_umc_pa2mca(adev, | |
| 3257 | − bps->retired_page << AMDGPU_GPU_PAGE_SHIFT, | |
| 3258 | − &(bps->address), | |
| 3259 | − AMDGPU_NPS1_PARTITION_MODE)) | |
| 3260 | − return -EINVAL; | |
| 3261 | − | |
| 3262 | − if (amdgpu_ras_mca2pa(adev, bps, err_data)) | |
| 3263 | − return -EOPNOTSUPP; | |
| 3264 | − } | |
| 3265 | − } | |
| 3266 | − | |
| 3267 | − return __amdgpu_ras_restore_bad_pages(adev, err_data->err_addr, | |
| 3268 | − adev->umc.retire_unit); | |
| 3269 | −} | |
| 3270 | − | |
| 3271 | 2952 | /* it deal with vram only. */ |
| 3272 | 2953 | int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, |
| 3273 | 2954 | struct eeprom_table_record *bps, int pages, bool from_rom) |
| @@ -3300,8 +2981,7 @@ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, · 26 unmodified lines | ||
| 3300 | 2981 | |
| 3301 | 2982 | if (from_rom) { |
| 3302 | 2983 | /* there is no pa recs in V3, so skip pa recs processing */ |
| 3303 | − if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) && | |
| 3304 | − !amdgpu_ras_smu_eeprom_supported(adev)) { | |
| 2984 | + if (control->tbl_hdr.version < RAS_TABLE_VER_V3) { | |
| 3305 | 2985 | for (i = 0; i < pages; i++) { |
| 3306 | 2986 | if (control->ras_num_recs - i >= adev->umc.retire_unit) { |
| 3307 | 2987 | if ((bps[i].address == bps[i + 1].address) && |
| @@ -3318,10 +2998,8 @@ int amdgpu_ras_add_bad_pages(struct amdgpu_device *adev, · 10 unmodified lines | ||
| 3318 | 2998 | } |
| 3319 | 2999 | } |
| 3320 | 3000 | } |
| 3321 | − for (; i < pages; i++) { | |
| 3322 | − ret = __amdgpu_ras_convert_rec_from_rom(adev, | |
| 3323 | − &bps[i], &err_data, nps); | |
| 3324 | − } | |
| 3001 | + for (; i < pages; i++) | |
| 3002 | + bps[i].retired_page &= ~(UMC_NPS_MASK << UMC_NPS_SHIFT); | |
| 3325 | 3003 | |
| 3326 | 3004 | con->eh_data->count_saved = con->eh_data->count; |
| 3327 | 3005 | } else { |
| @@ -3346,7 +3024,7 @@ int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, · 18 unmodified lines | ||
| 3346 | 3024 | struct amdgpu_ras *con = amdgpu_ras_get_context(adev); |
| 3347 | 3025 | struct ras_err_handler_data *data; |
| 3348 | 3026 | struct amdgpu_ras_eeprom_control *control; |
| 3349 | − int save_count, unit_num, i; | |
| 3027 | + int save_count, unit_num; | |
| 3350 | 3028 | |
| 3351 | 3029 | if (!con || !con->eh_data) { |
| 3352 | 3030 | if (new_cnt) |
| @@ -3367,12 +3045,7 @@ int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, · 14 unmodified lines | ||
| 3367 | 3045 | mutex_lock(&con->recovery_lock); |
| 3368 | 3046 | control = &con->eeprom_control; |
| 3369 | 3047 | data = con->eh_data; |
| 3370 | − if (amdgpu_ras_smu_eeprom_supported(adev)) | |
| 3371 | − unit_num = control->ras_num_recs - | |
| 3372 | − control->ras_num_recs_old; | |
| 3373 | − else | |
| 3374 | − unit_num = data->count / adev->umc.retire_unit - | |
| 3375 | − control->ras_num_recs; | |
| 3048 | + unit_num = data->count / adev->umc.retire_unit - control->ras_num_recs; | |
| 3376 | 3049 | |
| 3377 | 3050 | save_count = con->bad_page_num - control->ras_num_bad_pages; |
| 3378 | 3051 | mutex_unlock(&con->recovery_lock); |
| @@ -3383,21 +3056,10 @@ int amdgpu_ras_save_bad_pages(struct amdgpu_device *adev, · 4 unmodified lines | ||
| 3383 | 3056 | /* only new entries are saved */ |
| 3384 | 3057 | if (unit_num && save_count) { |
| 3385 | 3058 | /*old asics only save pa to eeprom like before*/ |
| 3386 | − if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) { | |
| 3387 | − if (amdgpu_ras_eeprom_append(control, | |
| 3388 | − &data->bps[data->count_saved], unit_num)) { | |
| 3389 | − dev_err(adev->dev, "Failed to save EEPROM table data!"); | |
| 3390 | − return -EIO; | |
| 3391 | − } | |
| 3392 | − } else { | |
| 3393 | − for (i = 0; i < unit_num; i++) { | |
| 3394 | − if (amdgpu_ras_eeprom_append(control, | |
| 3395 | − &data->bps[data->count_saved + | |
| 3396 | − i * adev->umc.retire_unit], 1)) { | |
| 3397 | − dev_err(adev->dev, "Failed to save EEPROM table data!"); | |
| 3398 | − return -EIO; | |
| 3399 | − } | |
| 3400 | − } | |
| 3059 | + if (amdgpu_ras_eeprom_append(control, | |
| 3060 | + &data->bps[data->count_saved], unit_num)) { | |
| 3061 | + dev_err(adev->dev, "Failed to save EEPROM table data!"); | |
| 3062 | + return -EIO; | |
| 3401 | 3063 | } |
| 3402 | 3064 | |
| 3403 | 3065 | dev_info(adev->dev, "Saved %d pages to EEPROM table.\n", save_count); |
| @@ -3416,7 +3078,7 @@ static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) · 12 unmodified lines | ||
| 3416 | 3078 | struct amdgpu_ras_eeprom_control *control = |
| 3417 | 3079 | &adev->psp.ras_context.ras->eeprom_control; |
| 3418 | 3080 | struct eeprom_table_record *bps; |
| 3419 | − int ret, i = 0; | |
| 3081 | + int ret; | |
| 3420 | 3082 | |
| 3421 | 3083 | /* no bad page record, skip eeprom access */ |
| 3422 | 3084 | if (control->ras_num_recs == 0 || amdgpu_bad_page_threshold == 0) |
| @@ -3430,33 +3092,6 @@ static int amdgpu_ras_load_bad_pages(struct amdgpu_device *adev) · 7 unmodified lines | ||
| 3430 | 3092 | if (ret) { |
| 3431 | 3093 | dev_err(adev->dev, "Failed to load EEPROM table records!"); |
| 3432 | 3094 | } else { |
| 3433 | − if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) { | |
| 3434 | − /*In V3, there is no pa recs, and some cases(when address==0) may be parsed | |
| 3435 | − as pa recs, so add verion check to avoid it. | |
| 3436 | − */ | |
| 3437 | − if ((control->tbl_hdr.version < RAS_TABLE_VER_V3) && | |
| 3438 | − !amdgpu_ras_smu_eeprom_supported(adev)) { | |
| 3439 | − for (i = 0; i < control->ras_num_recs; i++) { | |
| 3440 | − if ((control->ras_num_recs - i) >= adev->umc.retire_unit) { | |
| 3441 | − if ((bps[i].address == bps[i + 1].address) && | |
| 3442 | − (bps[i].mem_channel == bps[i + 1].mem_channel)) { | |
| 3443 | − control->ras_num_pa_recs += adev->umc.retire_unit; | |
| 3444 | − i += (adev->umc.retire_unit - 1); | |
| 3445 | − } else { | |
| 3446 | − control->ras_num_mca_recs += | |
| 3447 | − (control->ras_num_recs - i); | |
| 3448 | − break; | |
| 3449 | − } | |
| 3450 | − } else { | |
| 3451 | − control->ras_num_mca_recs += (control->ras_num_recs - i); | |
| 3452 | − break; | |
| 3453 | − } | |
| 3454 | − } | |
| 3455 | − } else { | |
| 3456 | − control->ras_num_mca_recs = control->ras_num_recs; | |
| 3457 | − } | |
| 3458 | − } | |
| 3459 | − | |
| 3460 | 3095 | ret = amdgpu_ras_add_bad_pages(adev, bps, control->ras_num_recs, true); |
| 3461 | 3096 | if (ret) |
| 3462 | 3097 | goto out; |
| @@ -3550,293 +3185,6 @@ static void amdgpu_ras_validate_threshold(struct amdgpu_device *adev, · 87 unmodified lines | ||
| 3550 | 3185 | } |
| 3551 | 3186 | } |
| 3552 | 3187 | |
| 3553 | −int amdgpu_ras_put_poison_req(struct amdgpu_device *adev, | |
| 3554 | − enum amdgpu_ras_block block, uint16_t pasid, | |
| 3555 | − pasid_notify pasid_fn, void *data, uint32_t reset) | |
| 3556 | −{ | |
| 3557 | − int ret = 0; | |
| 3558 | − struct ras_poison_msg poison_msg; | |
| 3559 | − struct amdgpu_ras *con = amdgpu_ras_get_context(adev); | |
| 3560 | − | |
| 3561 | − memset(&poison_msg, 0, sizeof(poison_msg)); | |
| 3562 | − poison_msg.block = block; | |
| 3563 | − poison_msg.pasid = pasid; | |
| 3564 | − poison_msg.reset = reset; | |
| 3565 | − poison_msg.pasid_fn = pasid_fn; | |
| 3566 | − poison_msg.data = data; | |
| 3567 | − | |
| 3568 | − ret = kfifo_put(&con->poison_fifo, poison_msg); | |
| 3569 | − if (!ret) { | |
| 3570 | − dev_err(adev->dev, "Poison message fifo is full!\n"); | |
| 3571 | − return -ENOSPC; | |
| 3572 | − } | |
| 3573 | − | |
| 3574 | − return 0; | |
| 3575 | −} | |
| 3576 | − | |
| 3577 | −static int amdgpu_ras_get_poison_req(struct amdgpu_device *adev, | |
| 3578 | − struct ras_poison_msg *poison_msg) | |
| 3579 | −{ | |
| 3580 | − struct amdgpu_ras *con = amdgpu_ras_get_context(adev); | |
| 3581 | − | |
| 3582 | − return kfifo_get(&con->poison_fifo, poison_msg); | |
| 3583 | −} | |
| 3584 | − | |
| 3585 | −static void amdgpu_ras_ecc_log_init(struct ras_ecc_log_info *ecc_log) | |
| 3586 | −{ | |
| 3587 | − mutex_init(&ecc_log->lock); | |
| 3588 | − | |
| 3589 | − INIT_RADIX_TREE(&ecc_log->de_page_tree, GFP_KERNEL); | |
| 3590 | − ecc_log->de_queried_count = 0; | |
| 3591 | − ecc_log->consumption_q_count = 0; | |
| 3592 | −} | |
| 3593 | − | |
| 3594 | −static void amdgpu_ras_ecc_log_fini(struct ras_ecc_log_info *ecc_log) | |
| 3595 | −{ | |
| 3596 | − struct radix_tree_iter iter; | |
| 3597 | − void __rcu **slot; | |
| 3598 | − struct ras_ecc_err *ecc_err; | |
| 3599 | − | |
| 3600 | − mutex_lock(&ecc_log->lock); | |
| 3601 | − radix_tree_for_each_slot(slot, &ecc_log->de_page_tree, &iter, 0) { | |
| 3602 | − ecc_err = radix_tree_deref_slot(slot); | |
| 3603 | − kfree(ecc_err->err_pages.pfn); | |
| 3604 | − kfree(ecc_err); | |
| 3605 | − radix_tree_iter_delete(&ecc_log->de_page_tree, &iter, slot); | |
| 3606 | − } | |
| 3607 | − mutex_unlock(&ecc_log->lock); | |
| 3608 | − | |
| 3609 | − mutex_destroy(&ecc_log->lock); | |
| 3610 | − ecc_log->de_queried_count = 0; | |
| 3611 | − ecc_log->consumption_q_count = 0; | |
| 3612 | −} | |
| 3613 | − | |
| 3614 | −static bool amdgpu_ras_schedule_retirement_dwork(struct amdgpu_ras *con, | |
| 3615 | − uint32_t delayed_ms) | |
| 3616 | −{ | |
| 3617 | − int ret; | |
| 3618 | − | |
| 3619 | − mutex_lock(&con->umc_ecc_log.lock); | |
| 3620 | − ret = radix_tree_tagged(&con->umc_ecc_log.de_page_tree, | |
| 3621 | − UMC_ECC_NEW_DETECTED_TAG); | |
| 3622 | − mutex_unlock(&con->umc_ecc_log.lock); | |
| 3623 | − | |
| 3624 | − if (ret) | |
| 3625 | − schedule_delayed_work(&con->page_retirement_dwork, | |
| 3626 | − msecs_to_jiffies(delayed_ms)); | |
| 3627 | − | |
| 3628 | − return ret ? true : false; | |
| 3629 | −} | |
| 3630 | − | |
| 3631 | −static void amdgpu_ras_do_page_retirement(struct work_struct *work) | |
| 3632 | −{ | |
| 3633 | − struct amdgpu_ras *con = container_of(work, struct amdgpu_ras, | |
| 3634 | − page_retirement_dwork.work); | |
| 3635 | − struct amdgpu_device *adev = con->adev; | |
| 3636 | − struct ras_err_data err_data; | |
| 3637 | − | |
| 3638 | − /* If gpu reset is ongoing, delay retiring the bad pages */ | |
| 3639 | − if (amdgpu_in_reset(adev) || amdgpu_ras_in_recovery(adev)) { | |
| 3640 | − amdgpu_ras_schedule_retirement_dwork(con, | |
| 3641 | − AMDGPU_RAS_RETIRE_PAGE_INTERVAL * 3); | |
| 3642 | − return; | |
| 3643 | − } | |
| 3644 | − | |
| 3645 | − amdgpu_ras_error_data_init(&err_data); | |
| 3646 | − | |
| 3647 | − amdgpu_umc_handle_bad_pages(adev, &err_data); | |
| 3648 | − | |
| 3649 | − amdgpu_ras_error_data_fini(&err_data); | |
| 3650 | − | |
| 3651 | − amdgpu_ras_schedule_retirement_dwork(con, | |
| 3652 | − AMDGPU_RAS_RETIRE_PAGE_INTERVAL); | |
| 3653 | −} | |
| 3654 | − | |
| 3655 | −static int amdgpu_ras_poison_creation_handler(struct amdgpu_device *adev, | |
| 3656 | − uint32_t poison_creation_count) | |
| 3657 | −{ | |
| 3658 | − int ret = 0; | |
| 3659 | − struct ras_ecc_log_info *ecc_log; | |
| 3660 | − struct ras_query_if info; | |
| 3661 | − u32 timeout = MAX_UMC_POISON_POLLING_TIME_ASYNC; | |
| 3662 | − struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); | |
| 3663 | − u64 de_queried_count; | |
| 3664 | − u64 consumption_q_count; | |
| 3665 | − enum ras_event_type type = RAS_EVENT_TYPE_POISON_CREATION; | |
| 3666 | − | |
| 3667 | − memset(&info, 0, sizeof(info)); | |
| 3668 | − info.head.block = AMDGPU_RAS_BLOCK__UMC; | |
| 3669 | − | |
| 3670 | − ecc_log = &ras->umc_ecc_log; | |
| 3671 | − ecc_log->de_queried_count = 0; | |
| 3672 | − ecc_log->consumption_q_count = 0; | |
| 3673 | − | |
| 3674 | − do { | |
| 3675 | − ret = amdgpu_ras_query_error_status_with_event(adev, &info, type); | |
| 3676 | − if (ret) | |
| 3677 | − return ret; | |
| 3678 | − | |
| 3679 | − de_queried_count = ecc_log->de_queried_count; | |
| 3680 | − consumption_q_count = ecc_log->consumption_q_count; | |
| 3681 | − | |
| 3682 | − if (de_queried_count && consumption_q_count) | |
| 3683 | − break; | |
| 3684 | − | |
| 3685 | − msleep(100); | |
| 3686 | − } while (--timeout); | |
| 3687 | − | |
| 3688 | − if (de_queried_count) | |
| 3689 | − schedule_delayed_work(&ras->page_retirement_dwork, 0); | |
| 3690 | − | |
| 3691 | − if (amdgpu_ras_is_rma(adev) && atomic_cmpxchg(&ras->rma_in_recovery, 0, 1) == 0) | |
| 3692 | − amdgpu_ras_reset_gpu(adev); | |
| 3693 | − | |
| 3694 | − return 0; | |
| 3695 | −} | |
| 3696 | − | |
| 3697 | −static void amdgpu_ras_clear_poison_fifo(struct amdgpu_device *adev) | |
| 3698 | −{ | |
| 3699 | − struct amdgpu_ras *con = amdgpu_ras_get_context(adev); | |
| 3700 | − struct ras_poison_msg msg; | |
| 3701 | − int ret; | |
| 3702 | − | |
| 3703 | − do { | |
| 3704 | − ret = kfifo_get(&con->poison_fifo, &msg); | |
| 3705 | − } while (ret); | |
| 3706 | −} | |
| 3707 | − | |
| 3708 | −static int amdgpu_ras_poison_consumption_handler(struct amdgpu_device *adev, | |
| 3709 | − uint32_t msg_count, uint32_t *gpu_reset) | |
| 3710 | −{ | |
| 3711 | − struct amdgpu_ras *con = amdgpu_ras_get_context(adev); | |
| 3712 | − uint32_t reset_flags = 0, reset = 0; | |
| 3713 | − struct ras_poison_msg msg; | |
| 3714 | − int ret, i; | |
| 3715 | − | |
| 3716 | − kgd2kfd_set_sram_ecc_flag(adev->kfd.dev); | |
| 3717 | − | |
| 3718 | − for (i = 0; i < msg_count; i++) { | |
| 3719 | − ret = amdgpu_ras_get_poison_req(adev, &msg); | |
| 3720 | − if (!ret) | |
| 3721 | − continue; | |
| 3722 | − | |
| 3723 | − if (msg.pasid_fn) | |
| 3724 | − msg.pasid_fn(adev, msg.pasid, msg.data); | |
| 3725 | − | |
| 3726 | − reset_flags |= msg.reset; | |
| 3727 | − } | |
| 3728 | − | |
| 3729 | − /* | |
| 3730 | − * Try to ensure poison creation handler is completed first | |
| 3731 | − * to set rma if bad page exceed threshold. | |
| 3732 | − */ | |
| 3733 | − flush_delayed_work(&con->page_retirement_dwork); | |
| 3734 | − | |
| 3735 | − /* for RMA, amdgpu_ras_poison_creation_handler will trigger gpu reset */ | |
| 3736 | − if (reset_flags && !amdgpu_ras_is_rma(adev)) { | |
| 3737 | − if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE1_RESET) | |
| 3738 | − reset = AMDGPU_RAS_GPU_RESET_MODE1_RESET; | |
| 3739 | − else if (reset_flags & AMDGPU_RAS_GPU_RESET_MODE2_RESET) | |
| 3740 | − reset = AMDGPU_RAS_GPU_RESET_MODE2_RESET; | |
| 3741 | − else | |
| 3742 | − reset = reset_flags; | |
| 3743 | − | |
| 3744 | − con->gpu_reset_flags |= reset; | |
| 3745 | − amdgpu_ras_reset_gpu(adev); | |
| 3746 | − | |
| 3747 | − *gpu_reset = reset; | |
| 3748 | − | |
| 3749 | − /* Wait for gpu recovery to complete */ | |
| 3750 | − flush_work(&con->recovery_work); | |
| 3751 | − } | |
| 3752 | − | |
| 3753 | − return 0; | |
| 3754 | −} | |
| 3755 | − | |
| 3756 | −static int amdgpu_ras_page_retirement_thread(void *param) | |
| 3757 | −{ | |
| 3758 | − struct amdgpu_device *adev = (struct amdgpu_device *)param; | |
| 3759 | − struct amdgpu_ras *con = amdgpu_ras_get_context(adev); | |
| 3760 | − uint32_t poison_creation_count, msg_count; | |
| 3761 | − uint32_t gpu_reset; | |
| 3762 | − int ret; | |
| 3763 | − | |
| 3764 | − while (!kthread_should_stop()) { | |
| 3765 | − | |
| 3766 | − wait_event_interruptible(con->page_retirement_wq, | |
| 3767 | − kthread_should_stop() || | |
| 3768 | − atomic_read(&con->page_retirement_req_cnt)); | |
| 3769 | − | |
| 3770 | − if (kthread_should_stop()) | |
| 3771 | − break; | |
| 3772 | − | |
| 3773 | − mutex_lock(&con->poison_lock); | |
| 3774 | − gpu_reset = 0; | |
| 3775 | − | |
| 3776 | − do { | |
| 3777 | − poison_creation_count = atomic_read(&con->poison_creation_count); | |
| 3778 | − ret = amdgpu_ras_poison_creation_handler(adev, poison_creation_count); | |
| 3779 | − if (ret == -EIO) | |
| 3780 | − break; | |
| 3781 | − | |
| 3782 | − if (poison_creation_count) { | |
| 3783 | − atomic_sub(poison_creation_count, &con->poison_creation_count); | |
| 3784 | − atomic_sub(poison_creation_count, &con->page_retirement_req_cnt); | |
| 3785 | − } | |
| 3786 | − } while (atomic_read(&con->poison_creation_count) && | |
| 3787 | − !atomic_read(&con->poison_consumption_count)); | |
| 3788 | − | |
| 3789 | − if (ret != -EIO) { | |
| 3790 | − msg_count = kfifo_len(&con->poison_fifo); | |
| 3791 | − if (msg_count) { | |
| 3792 | − ret = amdgpu_ras_poison_consumption_handler(adev, | |
| 3793 | − msg_count, &gpu_reset); | |
| 3794 | − if ((ret != -EIO) && | |
| 3795 | − (gpu_reset != AMDGPU_RAS_GPU_RESET_MODE1_RESET)) | |
| 3796 | − atomic_sub(msg_count, &con->page_retirement_req_cnt); | |
| 3797 | − } | |
| 3798 | − } | |
| 3799 | − | |
| 3800 | − if ((ret == -EIO) || (gpu_reset == AMDGPU_RAS_GPU_RESET_MODE1_RESET)) { | |
| 3801 | − /* gpu mode-1 reset is ongoing or just completed ras mode-1 reset */ | |
| 3802 | − /* Clear poison creation request */ | |
| 3803 | − atomic_set(&con->poison_creation_count, 0); | |
| 3804 | − atomic_set(&con->poison_consumption_count, 0); | |
| 3805 | − | |
| 3806 | − /* Clear poison fifo */ | |
| 3807 | − amdgpu_ras_clear_poison_fifo(adev); | |
| 3808 | − | |
| 3809 | − /* Clear all poison requests */ | |
| 3810 | − atomic_set(&con->page_retirement_req_cnt, 0); | |
| 3811 | − | |
| 3812 | − if (ret == -EIO) { | |
| 3813 | − /* Wait for mode-1 reset to complete */ | |
| 3814 | − down_read(&adev->reset_domain->sem); | |
| 3815 | − up_read(&adev->reset_domain->sem); | |
| 3816 | − } | |
| 3817 | − | |
| 3818 | − /* Wake up work to save bad pages to eeprom */ | |
| 3819 | − schedule_delayed_work(&con->page_retirement_dwork, 0); | |
| 3820 | − } else if (gpu_reset) { | |
| 3821 | − /* gpu just completed mode-2 reset or other reset */ | |
| 3822 | − /* Clear poison consumption messages cached in fifo */ | |
| 3823 | − msg_count = kfifo_len(&con->poison_fifo); | |
| 3824 | − if (msg_count) { | |
| 3825 | − amdgpu_ras_clear_poison_fifo(adev); | |
| 3826 | − atomic_sub(msg_count, &con->page_retirement_req_cnt); | |
| 3827 | − } | |
| 3828 | − | |
| 3829 | − atomic_set(&con->poison_consumption_count, 0); | |
| 3830 | − | |
| 3831 | − /* Wake up work to save bad pages to eeprom */ | |
| 3832 | − schedule_delayed_work(&con->page_retirement_dwork, 0); | |
| 3833 | − } | |
| 3834 | − mutex_unlock(&con->poison_lock); | |
| 3835 | − } | |
| 3836 | − | |
| 3837 | − return 0; | |
| 3838 | −} | |
| 3839 | − | |
| 3840 | 3188 | int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) |
| 3841 | 3189 | { |
| 3842 | 3190 | struct amdgpu_ras *con = amdgpu_ras_get_context(adev); |
| @@ -3846,7 +3194,14 @@ int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) · 3 unmodified lines | ||
| 3846 | 3194 | if (!con || amdgpu_sriov_vf(adev)) |
| 3847 | 3195 | return 0; |
| 3848 | 3196 | |
| 3849 | − if (amdgpu_uniras_enabled(adev)) | |
| 3197 | + /* | |
| 3198 | + * For the reset-on-init path (e.g. an NPS memory partition, | |
| 3199 | + * switch) the RAS IP block hw_init has not been enabled and | |
| 3200 | + * the amdgpu_uniras_enabled return false, check amdgpu ras | |
| 3201 | + * context uniras_enabled flag, eeprom init will be called | |
| 3202 | + * during RAS IP block hw_init. | |
| 3203 | + */ | |
| 3204 | + if (amdgpu_uniras_enabled(adev) || con->uniras_enabled) | |
| 3850 | 3205 | return 0; |
| 3851 | 3206 | |
| 3852 | 3207 | control = &con->eeprom_control; |
| @@ -3855,9 +3210,6 @@ int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) · 2 unmodified lines | ||
| 3855 | 3210 | ret = amdgpu_ras_eeprom_init(control); |
| 3856 | 3211 | control->is_eeprom_valid = !ret; |
| 3857 | 3212 | |
| 3858 | − if (!adev->umc.ras || !adev->umc.ras->convert_ras_err_addr) | |
| 3859 | − control->ras_num_pa_recs = control->ras_num_recs; | |
| 3860 | − | |
| 3861 | 3213 | if (adev->umc.ras && |
| 3862 | 3214 | adev->umc.ras->get_retire_flip_bits) |
| 3863 | 3215 | adev->umc.ras->get_retire_flip_bits(adev); |
| @@ -3877,13 +3229,6 @@ int amdgpu_ras_init_badpage_info(struct amdgpu_device *adev) · 13 unmodified lines | ||
| 3877 | 3229 | adev, control->bad_channel_bitmap); |
| 3878 | 3230 | con->update_channel_flag = false; |
| 3879 | 3231 | } |
| 3880 | − | |
| 3881 | − /* The format action is only applied to new ASICs */ | |
| 3882 | − if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) >= 12 && | |
| 3883 | − control->tbl_hdr.version < RAS_TABLE_VER_V3) | |
| 3884 | − if (!amdgpu_ras_eeprom_reset_table(control)) | |
| 3885 | − if (amdgpu_ras_save_bad_pages(adev, NULL)) | |
| 3886 | − dev_warn(adev->dev, "Failed to format RAS EEPROM data in V3 version!\n"); | |
| 3887 | 3232 | } |
| 3888 | 3233 | |
| 3889 | 3234 | return 0; |
| @@ -3917,10 +3262,8 @@ int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info) · 27 unmodified lines | ||
| 3917 | 3262 | } |
| 3918 | 3263 | |
| 3919 | 3264 | mutex_init(&con->recovery_lock); |
| 3920 | − mutex_init(&con->poison_lock); | |
| 3921 | 3265 | INIT_WORK(&con->recovery_work, amdgpu_ras_do_recovery); |
| 3922 | 3266 | atomic_set(&con->in_recovery, 0); |
| 3923 | − atomic_set(&con->rma_in_recovery, 0); | |
| 3924 | 3267 | con->eeprom_control.bad_channel_bitmap = 0; |
| 3925 | 3268 | |
| 3926 | 3269 | max_eeprom_records_count = amdgpu_ras_eeprom_max_record_count(&con->eeprom_control); |
| @@ -3933,21 +3276,8 @@ int amdgpu_ras_recovery_init(struct amdgpu_device *adev, bool init_bp_info) · 6 unmodified lines | ||
| 3933 | 3276 | } |
| 3934 | 3277 | |
| 3935 | 3278 | mutex_init(&con->page_rsv_lock); |
| 3936 | − INIT_KFIFO(con->poison_fifo); | |
| 3937 | 3279 | mutex_init(&con->page_retirement_lock); |
| 3938 | − init_waitqueue_head(&con->page_retirement_wq); | |
| 3939 | − atomic_set(&con->page_retirement_req_cnt, 0); | |
| 3940 | − atomic_set(&con->poison_creation_count, 0); | |
| 3941 | − atomic_set(&con->poison_consumption_count, 0); | |
| 3942 | − con->page_retirement_thread = | |
| 3943 | − kthread_run(amdgpu_ras_page_retirement_thread, adev, "umc_page_retirement"); | |
| 3944 | − if (IS_ERR(con->page_retirement_thread)) { | |
| 3945 | − con->page_retirement_thread = NULL; | |
| 3946 | − dev_warn(adev->dev, "Failed to create umc_page_retirement thread!!!\n"); | |
| 3947 | − } | |
| 3948 | − | |
| 3949 | − INIT_DELAYED_WORK(&con->page_retirement_dwork, amdgpu_ras_do_page_retirement); | |
| 3950 | − amdgpu_ras_ecc_log_init(&con->umc_ecc_log); | |
| 3280 | + | |
| 3951 | 3281 | #ifdef CONFIG_X86_MCE_AMD |
| 3952 | 3282 | if ((adev->asic_type == CHIP_ALDEBARAN) && |
| 3953 | 3283 | (adev->gmc.xgmi.connected_to_cpu)) |
| @@ -3978,33 +3308,15 @@ static int amdgpu_ras_recovery_fini(struct amdgpu_device *adev) · 24 unmodified lines | ||
| 3978 | 3308 | { |
| 3979 | 3309 | struct amdgpu_ras *con = amdgpu_ras_get_context(adev); |
| 3980 | 3310 | struct ras_err_handler_data *data = con->eh_data; |
| 3981 | − int max_flush_timeout = MAX_FLUSH_RETIRE_DWORK_TIMES; | |
| 3982 | − bool ret; | |
| 3983 | 3311 | |
| 3984 | 3312 | /* recovery_init failed to init it, fini is useless */ |
| 3985 | 3313 | if (!data) |
| 3986 | 3314 | return 0; |
| 3987 | 3315 | |
| 3988 | − /* Save all cached bad pages to eeprom */ | |
| 3989 | − do { | |
| 3990 | − flush_delayed_work(&con->page_retirement_dwork); | |
| 3991 | − ret = amdgpu_ras_schedule_retirement_dwork(con, 0); | |
| 3992 | − } while (ret && max_flush_timeout--); | |
| 3993 | − | |
| 3994 | − if (con->page_retirement_thread) | |
| 3995 | − kthread_stop(con->page_retirement_thread); | |
| 3996 | − | |
| 3997 | − atomic_set(&con->page_retirement_req_cnt, 0); | |
| 3998 | − atomic_set(&con->poison_creation_count, 0); | |
| 3999 | − | |
| 4000 | 3316 | mutex_destroy(&con->page_rsv_lock); |
| 4001 | 3317 | |
| 4002 | 3318 | cancel_work_sync(&con->recovery_work); |
| 4003 | 3319 | |
| 4004 | − cancel_delayed_work_sync(&con->page_retirement_dwork); | |
| 4005 | − | |
| 4006 | − amdgpu_ras_ecc_log_fini(&con->umc_ecc_log); | |
| 4007 | − | |
| 4008 | 3320 | mutex_lock(&con->recovery_lock); |
| 4009 | 3321 | con->eh_data = NULL; |
| 4010 | 3322 | kfree(data->bps); |
| @@ -4206,15 +3518,6 @@ init_ras_enabled_flag: · 195 unmodified lines | ||
| 4206 | 3518 | adev->ras_enabled = amdgpu_ras_enable == 0 ? 0 : |
| 4207 | 3519 | adev->ras_hw_enabled & amdgpu_ras_mask; |
| 4208 | 3520 | |
| 4209 | − /* aca is disabled by default except for psp v13_0_6/v13_0_12/v13_0_14 */ | |
| 4210 | − if (!amdgpu_sriov_vf(adev)) { | |
| 4211 | − adev->aca.is_enabled = | |
| 4212 | − (amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 6) || | |
| 4213 | − amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 12) || | |
| 4214 | − amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 14) || | |
| 4215 | − amdgpu_ip_version(adev, MP0_HWIP, 0) == IP_VERSION(13, 0, 15)); | |
| 4216 | − } | |
| 4217 | − | |
| 4218 | 3521 | /* bad page feature is not applicable to specific app platform */ |
| 4219 | 3522 | if (adev->gmc.is_app_apu && |
| 4220 | 3523 | amdgpu_ip_version(adev, UMC_HWIP, 0) == IP_VERSION(12, 0, 0)) |
| @@ -4435,15 +3738,6 @@ int amdgpu_ras_init(struct amdgpu_device *adev) · 214 unmodified lines | ||
| 4435 | 3738 | goto release_con; |
| 4436 | 3739 | } |
| 4437 | 3740 | |
| 4438 | − if (amdgpu_ras_aca_is_supported(adev)) { | |
| 4439 | − if (amdgpu_aca_is_enabled(adev)) | |
| 4440 | − r = amdgpu_aca_init(adev); | |
| 4441 | − else | |
| 4442 | − r = amdgpu_mca_init(adev); | |
| 4443 | − if (r) | |
| 4444 | − goto release_con; | |
| 4445 | − } | |
| 4446 | − | |
| 4447 | 3741 | con->init_task_pid = task_pid_nr(current); |
| 4448 | 3742 | get_task_comm(con->init_task_comm, current); |
| 4449 | 3743 | |
| @@ -4541,9 +3835,9 @@ int amdgpu_ras_block_late_init(struct amdgpu_device *adev, · 91 unmodified lines | ||
| 4541 | 3835 | goto cleanup; |
| 4542 | 3836 | } |
| 4543 | 3837 | |
| 4544 | − if (ras_obj->hw_ops && | |
| 3838 | + if (amdgpu_uniras_enabled(adev) || (ras_obj->hw_ops && | |
| 4545 | 3839 | (ras_obj->hw_ops->query_ras_error_count || |
| 4546 | − ras_obj->hw_ops->query_ras_error_status)) { | |
| 3840 | + ras_obj->hw_ops->query_ras_error_status))) { | |
| 4547 | 3841 | r = amdgpu_ras_sysfs_create(adev, ras_block); |
| 4548 | 3842 | if (r) |
| 4549 | 3843 | goto interrupt; |
| @@ -4671,28 +3965,13 @@ int amdgpu_ras_late_init(struct amdgpu_device *adev) · 121 unmodified lines | ||
| 4671 | 3965 | |
| 4672 | 3966 | amdgpu_ras_event_mgr_init(adev); |
| 4673 | 3967 | |
| 4674 | − if (amdgpu_ras_aca_is_supported(adev)) { | |
| 4675 | − if (amdgpu_reset_in_recovery(adev)) { | |
| 4676 | − if (amdgpu_aca_is_enabled(adev)) | |
| 4677 | − r = amdgpu_aca_reset(adev); | |
| 4678 | − else | |
| 4679 | − r = amdgpu_mca_reset(adev); | |
| 4680 | − if (r) | |
| 4681 | − return r; | |
| 4682 | − } | |
| 4683 | − | |
| 4684 | − if (!amdgpu_sriov_vf(adev)) { | |
| 4685 | − if (amdgpu_aca_is_enabled(adev)) | |
| 4686 | − amdgpu_ras_set_aca_debug_mode(adev, false); | |
| 4687 | − else | |
| 4688 | − amdgpu_ras_set_mca_debug_mode(adev, false); | |
| 4689 | − } | |
| 4690 | − } | |
| 4691 | − | |
| 4692 | 3968 | /* Guest side doesn't need init ras feature */ |
| 4693 | 3969 | if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_ras_telemetry_en(adev)) |
| 4694 | 3970 | return 0; |
| 4695 | 3971 | |
| 3972 | + if (amdgpu_uniras_enabled(adev)) | |
| 3973 | + amdgpu_ras_mgr_set_debug_mode(adev, false); | |
| 3974 | + | |
| 4696 | 3975 | list_for_each_entry_safe(node, tmp, &adev->ras_list, node) { |
| 4697 | 3976 | obj = node->ras_obj; |
| 4698 | 3977 | if (!obj) { |
| @@ -4773,13 +4052,6 @@ int amdgpu_ras_fini(struct amdgpu_device *adev) · 74 unmodified lines | ||
| 4773 | 4052 | amdgpu_ras_fs_fini(adev); |
| 4774 | 4053 | amdgpu_ras_interrupt_remove_all(adev); |
| 4775 | 4054 | |
| 4776 | − if (amdgpu_ras_aca_is_supported(adev)) { | |
| 4777 | − if (amdgpu_aca_is_enabled(adev)) | |
| 4778 | − amdgpu_aca_fini(adev); | |
| 4779 | − else | |
| 4780 | − amdgpu_mca_fini(adev); | |
| 4781 | − } | |
| 4782 | − | |
| 4783 | 4055 | WARN(AMDGPU_RAS_GET_FEATURES(con->features), "Feature mask is not cleared"); |
| 4784 | 4056 | |
| 4785 | 4057 | if (AMDGPU_RAS_GET_FEATURES(con->features)) |
| @@ -5064,6 +4336,13 @@ static void amdgpu_register_bad_pages_mca_notifier(struct amdgpu_device *adev) · 278 unmodified lines | ||
| 5064 | 4336 | * Use this list instead of mgpu_info to find the amdgpu |
| 5065 | 4337 | * device on which the UMC error was reported. |
| 5066 | 4338 | */ |
| 4339 | + if (mce_adev_list.num_gpu >= MAX_GPU_INSTANCE) { | |
| 4340 | + dev_warn_ratelimited(adev->dev, | |
| 4341 | + "mce_adev_list full, skip notifier registration (max=%d)\n", | |
| 4342 | + MAX_GPU_INSTANCE); | |
| 4343 | + return; | |
| 4344 | + } | |
| 4345 | + | |
| 5067 | 4346 | mce_adev_list.devs[mce_adev_list.num_gpu++] = adev; |
| 5068 | 4347 | |
| 5069 | 4348 | /* |
| @@ -5181,59 +4460,10 @@ int amdgpu_ras_reset_gpu(struct amdgpu_device *adev) · 111 unmodified lines | ||
| 5181 | 4460 | return 0; |
| 5182 | 4461 | } |
| 5183 | 4462 | |
| 5184 | −int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable) | |
| 5185 | −{ | |
| 5186 | − struct amdgpu_ras *con = amdgpu_ras_get_context(adev); | |
| 5187 | − int ret = 0; | |
| 5188 | − | |
| 5189 | − if (con) { | |
| 5190 | − ret = amdgpu_mca_smu_set_debug_mode(adev, enable); | |
| 5191 | − if (!ret) | |
| 5192 | − con->is_aca_debug_mode = enable; | |
| 5193 | − } | |
| 5194 | − | |
| 5195 | − return ret; | |
| 5196 | −} | |
| 5197 | − | |
| 5198 | −int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable) | |
| 5199 | −{ | |
| 5200 | − struct amdgpu_ras *con = amdgpu_ras_get_context(adev); | |
| 5201 | − int ret = 0; | |
| 5202 | − | |
| 5203 | − if (con) { | |
| 5204 | − if (amdgpu_aca_is_enabled(adev)) | |
| 5205 | − ret = amdgpu_aca_smu_set_debug_mode(adev, enable); | |
| 5206 | − else | |
| 5207 | − ret = amdgpu_mca_smu_set_debug_mode(adev, enable); | |
| 5208 | − if (!ret) | |
| 5209 | − con->is_aca_debug_mode = enable; | |
| 5210 | − } | |
| 5211 | − | |
| 5212 | − return ret; | |
| 5213 | −} | |
| 5214 | − | |
| 5215 | −bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev) | |
| 5216 | −{ | |
| 5217 | − struct amdgpu_ras *con = amdgpu_ras_get_context(adev); | |
| 5218 | − const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; | |
| 5219 | − const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; | |
| 5220 | − | |
| 5221 | − if (!con) | |
| 5222 | − return false; | |
| 5223 | − | |
| 5224 | − if ((amdgpu_aca_is_enabled(adev) && smu_funcs && smu_funcs->set_debug_mode) || | |
| 5225 | − (!amdgpu_aca_is_enabled(adev) && mca_funcs && mca_funcs->mca_set_debug_mode)) | |
| 5226 | − return con->is_aca_debug_mode; | |
| 5227 | − else | |
| 5228 | − return true; | |
| 5229 | −} | |
| 5230 | − | |
| 5231 | 4463 | bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, |
| 5232 | 4464 | unsigned int *error_query_mode) |
| 5233 | 4465 | { |
| 5234 | 4466 | struct amdgpu_ras *con = amdgpu_ras_get_context(adev); |
| 5235 | − const struct amdgpu_mca_smu_funcs *mca_funcs = adev->mca.mca_funcs; | |
| 5236 | − const struct aca_smu_funcs *smu_funcs = adev->aca.smu_funcs; | |
| 5237 | 4467 | |
| 5238 | 4468 | if (!con) { |
| 5239 | 4469 | *error_query_mode = AMDGPU_RAS_INVALID_ERROR_QUERY; |
| @@ -5242,9 +4472,6 @@ bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, · 2 unmodified lines | ||
| 5242 | 4472 | |
| 5243 | 4473 | if (amdgpu_sriov_vf(adev)) { |
| 5244 | 4474 | *error_query_mode = AMDGPU_RAS_VIRT_ERROR_COUNT_QUERY; |
| 5245 | − } else if ((smu_funcs && smu_funcs->set_debug_mode) || (mca_funcs && mca_funcs->mca_set_debug_mode)) { | |
| 5246 | − *error_query_mode = | |
| 5247 | − (con->is_aca_debug_mode) ? AMDGPU_RAS_DIRECT_ERROR_QUERY : AMDGPU_RAS_FIRMWARE_ERROR_QUERY; | |
| 5248 | 4475 | } else { |
| 5249 | 4476 | *error_query_mode = AMDGPU_RAS_DIRECT_ERROR_QUERY; |
| 5250 | 4477 | } |
| @@ -5834,3 +5061,8 @@ void amdgpu_ras_post_reset(struct amdgpu_device *adev, · 583 unmodified lines | ||
| 5834 | 5061 | amdgpu_ras_mgr_post_reset(tmp_adev); |
| 5835 | 5062 | } |
| 5836 | 5063 | } |
| 5064 | + | |
| 5065 | +void amdgpu_ras_resume_after_reset(struct amdgpu_device *adev) | |
| 5066 | +{ | |
| 5067 | + amdgpu_ras_mgr_resume_after_reset(adev); | |
| 5068 | +} | |
| @@ -31,7 +31,6 @@ · 30 unmodified lines | ||
| 31 | 31 | #include "ta_ras_if.h" |
| 32 | 32 | #include "amdgpu_ras_eeprom.h" |
| 33 | 33 | #include "amdgpu_smuio.h" |
| 34 | −#include "amdgpu_aca.h" | |
| 35 | 34 | |
| 36 | 35 | struct amdgpu_iv_entry; |
| 37 | 36 | |
| @@ -466,14 +465,6 @@ struct ras_query_context { · 428 unmodified lines | ||
| 466 | 465 | typedef int (*pasid_notify)(struct amdgpu_device *adev, |
| 467 | 466 | uint16_t pasid, void *data); |
| 468 | 467 | |
| 469 | −struct ras_poison_msg { | |
| 470 | − enum amdgpu_ras_block block; | |
| 471 | − uint16_t pasid; | |
| 472 | − uint32_t reset; | |
| 473 | − pasid_notify pasid_fn; | |
| 474 | − void *data; | |
| 475 | −}; | |
| 476 | − | |
| 477 | 468 | struct ras_err_pages { |
| 478 | 469 | uint32_t count; |
| 479 | 470 | uint64_t *pfn; |
| @@ -492,8 +483,6 @@ struct ras_ecc_err { · 12 unmodified lines | ||
| 492 | 483 | struct ras_ecc_log_info { |
| 493 | 484 | struct mutex lock; |
| 494 | 485 | struct radix_tree_root de_page_tree; |
| 495 | − uint64_t de_queried_count; | |
| 496 | − uint64_t consumption_q_count; | |
| 497 | 486 | }; |
| 498 | 487 | |
| 499 | 488 | struct ras_critical_region { |
| @@ -549,7 +538,6 @@ struct amdgpu_ras { · 49 unmodified lines | ||
| 549 | 538 | /* gpu recovery */ |
| 550 | 539 | struct work_struct recovery_work; |
| 551 | 540 | atomic_t in_recovery; |
| 552 | − atomic_t rma_in_recovery; | |
| 553 | 541 | struct amdgpu_device *adev; |
| 554 | 542 | /* error handler data */ |
| 555 | 543 | struct ras_err_handler_data *eh_data; |
| @@ -581,22 +569,15 @@ struct amdgpu_ras { · 25 unmodified lines | ||
| 581 | 569 | /* Indicates smu whether need update bad channel info */ |
| 582 | 570 | bool update_channel_flag; |
| 583 | 571 | /* Record status of smu mca debug mode */ |
| 584 | − bool is_aca_debug_mode; | |
| 572 | + bool is_mca_debug_mode; | |
| 585 | 573 | bool is_rma; |
| 586 | 574 | |
| 587 | 575 | /* Record special requirements of gpu reset caller */ |
| 588 | 576 | uint32_t gpu_reset_flags; |
| 589 | 577 | |
| 590 | − struct task_struct *page_retirement_thread; | |
| 591 | − wait_queue_head_t page_retirement_wq; | |
| 592 | 578 | struct mutex page_retirement_lock; |
| 593 | − atomic_t page_retirement_req_cnt; | |
| 594 | − atomic_t poison_creation_count; | |
| 595 | − atomic_t poison_consumption_count; | |
| 596 | 579 | struct mutex page_rsv_lock; |
| 597 | − DECLARE_KFIFO(poison_fifo, struct ras_poison_msg, 128); | |
| 598 | 580 | struct ras_ecc_log_info umc_ecc_log; |
| 599 | − struct delayed_work page_retirement_dwork; | |
| 600 | 581 | |
| 601 | 582 | /* ras errors detected */ |
| 602 | 583 | unsigned long ras_err_state; |
| @@ -615,9 +596,6 @@ struct amdgpu_ras { · 12 unmodified lines | ||
| 615 | 596 | struct list_head critical_region_head; |
| 616 | 597 | struct mutex critical_region_lock; |
| 617 | 598 | |
| 618 | − /* Protect poison injection */ | |
| 619 | − struct mutex poison_lock; | |
| 620 | − | |
| 621 | 599 | /* Disable/Enable uniras switch */ |
| 622 | 600 | bool uniras_enabled; |
| 623 | 601 | const struct ras_smu_drv *ras_smu_drv; |
| @@ -702,8 +680,6 @@ struct ras_manager { · 78 unmodified lines | ||
| 702 | 680 | struct ras_ih_data ih_data; |
| 703 | 681 | |
| 704 | 682 | struct ras_err_data err_data; |
| 705 | − | |
| 706 | − struct aca_handle aca_handle; | |
| 707 | 683 | }; |
| 708 | 684 | |
| 709 | 685 | struct ras_badpage { |
| @@ -964,8 +940,7 @@ struct amdgpu_ras* amdgpu_ras_get_context(struct amdgpu_device *adev); · 254 unmodified lines | ||
| 964 | 940 | int amdgpu_ras_set_context(struct amdgpu_device *adev, struct amdgpu_ras *ras_con); |
| 965 | 941 | |
| 966 | 942 | int amdgpu_ras_set_mca_debug_mode(struct amdgpu_device *adev, bool enable); |
| 967 | −int amdgpu_ras_set_aca_debug_mode(struct amdgpu_device *adev, bool enable); | |
| 968 | −bool amdgpu_ras_get_aca_debug_mode(struct amdgpu_device *adev); | |
| 943 | +bool amdgpu_ras_get_mca_debug_mode(struct amdgpu_device *adev); | |
| 969 | 944 | bool amdgpu_ras_get_error_query_mode(struct amdgpu_device *adev, |
| 970 | 945 | unsigned int *mode); |
| 971 | 946 | |
| @@ -1006,12 +981,6 @@ int amdgpu_ras_error_statistic_de_count(struct ras_err_data *err_data, · 34 unmodified lines | ||
| 1006 | 981 | struct amdgpu_smuio_mcm_config_info *mcm_info, |
| 1007 | 982 | u64 count); |
| 1008 | 983 | void amdgpu_ras_query_boot_status(struct amdgpu_device *adev, u32 num_instances); |
| 1009 | −int amdgpu_ras_bind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk, | |
| 1010 | − const struct aca_info *aca_info, void *data); | |
| 1011 | −int amdgpu_ras_unbind_aca(struct amdgpu_device *adev, enum amdgpu_ras_block blk); | |
| 1012 | − | |
| 1013 | −ssize_t amdgpu_ras_aca_sysfs_read(struct device *dev, struct device_attribute *attr, | |
| 1014 | − struct aca_handle *handle, char *buf, void *data); | |
| 1015 | 984 | |
| 1016 | 985 | void amdgpu_ras_set_fed(struct amdgpu_device *adev, bool status); |
| 1017 | 986 | bool amdgpu_ras_get_fed_status(struct amdgpu_device *adev); |
| @@ -1029,10 +998,6 @@ int amdgpu_ras_reserve_page(struct amdgpu_device *adev, uint64_t pfn); · 11 unmodified lines | ||
| 1029 | 998 | int amdgpu_ras_add_critical_region(struct amdgpu_device *adev, struct amdgpu_bo *bo); |
| 1030 | 999 | bool amdgpu_ras_check_critical_address(struct amdgpu_device *adev, uint64_t addr); |
| 1031 | 1000 | |
| 1032 | −int amdgpu_ras_put_poison_req(struct amdgpu_device *adev, | |
| 1033 | − enum amdgpu_ras_block block, uint16_t pasid, | |
| 1034 | − pasid_notify pasid_fn, void *data, uint32_t reset); | |
| 1035 | − | |
| 1036 | 1001 | bool amdgpu_ras_in_recovery(struct amdgpu_device *adev); |
| 1037 | 1002 | |
| 1038 | 1003 | __printf(3, 4) |
| @@ -1045,4 +1010,5 @@ void amdgpu_ras_pre_reset(struct amdgpu_device *adev, · 6 unmodified lines | ||
| 1045 | 1010 | struct list_head *device_list); |
| 1046 | 1011 | void amdgpu_ras_post_reset(struct amdgpu_device *adev, |
| 1047 | 1012 | struct list_head *device_list); |
| 1013 | +void amdgpu_ras_resume_after_reset(struct amdgpu_device *adev); | |
| 1048 | 1014 | #endif |
| @@ -124,8 +124,6 @@ · 123 unmodified lines | ||
| 124 | 124 | RAS_TABLE_V2_1_INFO_SIZE) \ |
| 125 | 125 | / RAS_TABLE_RECORD_SIZE) |
| 126 | 126 | |
| 127 | −#define RAS_SMU_MESSAGE_TIMEOUT_MS 1000 /* 1s */ | |
| 128 | − | |
| 129 | 127 | /* Given a zero-based index of an EEPROM RAS record, yields the EEPROM |
| 130 | 128 | * offset off of RAS_TABLE_START. That is, this is something you can |
| 131 | 129 | * add to control->i2c_address, and then tell I2C layer to read |
| @@ -159,6 +157,9 @@ · 27 unmodified lines | ||
| 159 | 157 | |
| 160 | 158 | static bool __is_ras_eeprom_supported(struct amdgpu_device *adev) |
| 161 | 159 | { |
| 160 | + if (amdgpu_sriov_vf(adev)) | |
| 161 | + return false; | |
| 162 | + | |
| 162 | 163 | switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) { |
| 163 | 164 | case IP_VERSION(11, 0, 2): /* VEGA20 and ARCTURUS */ |
| 164 | 165 | case IP_VERSION(11, 0, 7): /* Sienna cichlid */ |
| @@ -449,57 +450,46 @@ int amdgpu_ras_eeprom_reset_table(struct amdgpu_ras_eeprom_control *control) · 284 unmodified lines | ||
| 449 | 450 | struct amdgpu_ras_eeprom_table_header *hdr = &control->tbl_hdr; |
| 450 | 451 | struct amdgpu_ras_eeprom_table_ras_info *rai = &control->tbl_rai; |
| 451 | 452 | struct amdgpu_ras *con = amdgpu_ras_get_context(adev); |
| 452 | − u32 erase_res = 0; | |
| 453 | 453 | u8 csum; |
| 454 | 454 | int res; |
| 455 | 455 | |
| 456 | 456 | mutex_lock(&control->ras_tbl_mutex); |
| 457 | 457 | |
| 458 | − if (!amdgpu_ras_smu_eeprom_supported(adev)) { | |
| 459 | − hdr->header = RAS_TABLE_HDR_VAL; | |
| 460 | − amdgpu_ras_set_eeprom_table_version(control); | |
| 461 | − | |
| 462 | − if (hdr->version >= RAS_TABLE_VER_V2_1) { | |
| 463 | − hdr->first_rec_offset = RAS_RECORD_START_V2_1; | |
| 464 | − hdr->tbl_size = RAS_TABLE_HEADER_SIZE + | |
| 465 | − RAS_TABLE_V2_1_INFO_SIZE; | |
| 466 | − rai->rma_status = GPU_HEALTH_USABLE; | |
| 467 | − | |
| 468 | − control->ras_record_offset = RAS_RECORD_START_V2_1; | |
| 469 | − control->ras_max_record_count = RAS_MAX_RECORD_COUNT_V2_1; | |
| 470 | − /** | |
| 471 | − * GPU health represented as a percentage. | |
| 472 | − * 0 means worst health, 100 means fully health. | |
| 473 | − */ | |
| 474 | − rai->health_percent = 100; | |
| 475 | − /* ecc_page_threshold = 0 means disable bad page retirement */ | |
| 476 | − rai->ecc_page_threshold = con->bad_page_cnt_threshold; | |
| 477 | − } else { | |
| 478 | − hdr->first_rec_offset = RAS_RECORD_START; | |
| 479 | − hdr->tbl_size = RAS_TABLE_HEADER_SIZE; | |
| 458 | + hdr->header = RAS_TABLE_HDR_VAL; | |
| 459 | + amdgpu_ras_set_eeprom_table_version(control); | |
| 480 | 460 | |
| 481 | − control->ras_record_offset = RAS_RECORD_START; | |
| 482 | − control->ras_max_record_count = RAS_MAX_RECORD_COUNT; | |
| 483 | − } | |
| 461 | + if (hdr->version >= RAS_TABLE_VER_V2_1) { | |
| 462 | + hdr->first_rec_offset = RAS_RECORD_START_V2_1; | |
| 463 | + hdr->tbl_size = RAS_TABLE_HEADER_SIZE + | |
| 464 | + RAS_TABLE_V2_1_INFO_SIZE; | |
| 465 | + rai->rma_status = GPU_HEALTH_USABLE; | |
| 484 | 466 | |
| 485 | − csum = __calc_hdr_byte_sum(control); | |
| 486 | − if (hdr->version >= RAS_TABLE_VER_V2_1) | |
| 487 | − csum += __calc_ras_info_byte_sum(control); | |
| 488 | − csum = -csum; | |
| 489 | − hdr->checksum = csum; | |
| 490 | − res = __write_table_header(control); | |
| 491 | − if (!res && hdr->version > RAS_TABLE_VER_V1) | |
| 492 | − res = __write_table_ras_info(control); | |
| 467 | + control->ras_record_offset = RAS_RECORD_START_V2_1; | |
| 468 | + control->ras_max_record_count = RAS_MAX_RECORD_COUNT_V2_1; | |
| 469 | + /** | |
| 470 | + * GPU health represented as a percentage. | |
| 471 | + * 0 means worst health, 100 means fully health. | |
| 472 | + */ | |
| 473 | + rai->health_percent = 100; | |
| 474 | + /* ecc_page_threshold = 0 means disable bad page retirement */ | |
| 475 | + rai->ecc_page_threshold = con->bad_page_cnt_threshold; | |
| 493 | 476 | } else { |
| 494 | − res = amdgpu_ras_smu_erase_ras_table(adev, &erase_res); | |
| 495 | − if (res || erase_res) { | |
| 496 | − dev_warn(adev->dev, "RAS EEPROM reset failed, res:%d result:%d", | |
| 497 | − res, erase_res); | |
| 498 | − if (!res) | |
| 499 | − res = -EIO; | |
| 500 | − } | |
| 477 | + hdr->first_rec_offset = RAS_RECORD_START; | |
| 478 | + hdr->tbl_size = RAS_TABLE_HEADER_SIZE; | |
| 479 | + | |
| 480 | + control->ras_record_offset = RAS_RECORD_START; | |
| 481 | + control->ras_max_record_count = RAS_MAX_RECORD_COUNT; | |
| 501 | 482 | } |
| 502 | 483 | |
| 484 | + csum = __calc_hdr_byte_sum(control); | |
| 485 | + if (hdr->version >= RAS_TABLE_VER_V2_1) | |
| 486 | + csum += __calc_ras_info_byte_sum(control); | |
| 487 | + csum = -csum; | |
| 488 | + hdr->checksum = csum; | |
| 489 | + res = __write_table_header(control); | |
| 490 | + if (!res && hdr->version > RAS_TABLE_VER_V1) | |
| 491 | + res = __write_table_ras_info(control); | |
| 492 | + | |
| 503 | 493 | control->ras_num_recs = 0; |
| 504 | 494 | control->ras_num_bad_pages = 0; |
| 505 | 495 | control->ras_num_mca_recs = 0; |
| @@ -662,7 +652,6 @@ amdgpu_ras_eeprom_append_table(struct amdgpu_ras_eeprom_control *control, · 156 unmodified lines | ||
| 662 | 652 | const u32 num) |
| 663 | 653 | { |
| 664 | 654 | struct amdgpu_ras *con = amdgpu_ras_get_context(to_amdgpu_device(control)); |
| 665 | − struct amdgpu_device *adev = to_amdgpu_device(control); | |
| 666 | 655 | u32 a, b, i; |
| 667 | 656 | u8 *buf, *pp; |
| 668 | 657 | int res; |
| @@ -767,10 +756,7 @@ amdgpu_ras_eeprom_append_table(struct amdgpu_ras_eeprom_control *control, · 98 unmodified lines | ||
| 767 | 756 | % control->ras_max_record_count; |
| 768 | 757 | |
| 769 | 758 | /*old asics only save pa to eeprom like before*/ |
| 770 | − if (IP_VERSION_MAJ(amdgpu_ip_version(adev, UMC_HWIP, 0)) < 12) | |
| 771 | − control->ras_num_pa_recs += num; | |
| 772 | − else | |
| 773 | − control->ras_num_mca_recs += num; | |
| 759 | + control->ras_num_pa_recs += num; | |
| 774 | 760 | |
| 775 | 761 | control->ras_num_bad_pages = con->bad_page_num; |
| 776 | 762 | Out: |
| @@ -879,71 +865,6 @@ Out: · 102 unmodified lines | ||
| 879 | 865 | return res; |
| 880 | 866 | } |
| 881 | 867 | |
| 882 | −int amdgpu_ras_eeprom_update_record_num(struct amdgpu_ras_eeprom_control *control) | |
| 883 | −{ | |
| 884 | − struct amdgpu_device *adev = to_amdgpu_device(control); | |
| 885 | − int ret, retry = 20; | |
| 886 | − | |
| 887 | − if (!amdgpu_ras_smu_eeprom_supported(adev)) | |
| 888 | − return 0; | |
| 889 | − | |
| 890 | − control->ras_num_recs_old = control->ras_num_recs; | |
| 891 | − | |
| 892 | − do { | |
| 893 | − /* 1000ms timeout is long enough, smu_get_badpage_count won't | |
| 894 | − * return -EBUSY before timeout. | |
| 895 | − */ | |
| 896 | − ret = amdgpu_ras_smu_get_badpage_count(adev, | |
| 897 | − &(control->ras_num_recs), RAS_SMU_MESSAGE_TIMEOUT_MS); | |
| 898 | − if (!ret && | |
| 899 | − (control->ras_num_recs_old == control->ras_num_recs)) { | |
| 900 | − /* record number update in PMFW needs some time, | |
| 901 | − * smu_get_badpage_count may return immediately without | |
| 902 | − * count update, sleep for a while and retry again. | |
| 903 | − */ | |
| 904 | − msleep(50); | |
| 905 | − retry--; | |
| 906 | − } else { | |
| 907 | − break; | |
| 908 | − } | |
| 909 | − } while (retry); | |
| 910 | − | |
| 911 | − /* no update of record number is not a real failure, | |
| 912 | − * don't print warning here | |
| 913 | − */ | |
| 914 | − if (!ret && (control->ras_num_recs_old == control->ras_num_recs)) | |
| 915 | − ret = -EINVAL; | |
| 916 | − | |
| 917 | − return ret; | |
| 918 | −} | |
| 919 | − | |
| 920 | −static int amdgpu_ras_smu_eeprom_append(struct amdgpu_ras_eeprom_control *control) | |
| 921 | −{ | |
| 922 | − struct amdgpu_device *adev = to_amdgpu_device(control); | |
| 923 | − struct amdgpu_ras *con = amdgpu_ras_get_context(adev); | |
| 924 | − | |
| 925 | − if (!amdgpu_ras_smu_eeprom_supported(adev) || !con) | |
| 926 | − return 0; | |
| 927 | − | |
| 928 | − control->ras_num_bad_pages = con->bad_page_num; | |
| 929 | − | |
| 930 | − if (amdgpu_bad_page_threshold != 0 && | |
| 931 | − control->ras_num_bad_pages > con->bad_page_cnt_threshold) { | |
| 932 | − dev_warn(adev->dev, | |
| 933 | − "Saved bad pages %d reaches threshold value %d\n", | |
| 934 | − control->ras_num_bad_pages, con->bad_page_cnt_threshold); | |
| 935 | − | |
| 936 | − if (adev->cper.enabled && amdgpu_cper_generate_bp_threshold_record(adev)) | |
| 937 | − dev_warn(adev->dev, "fail to generate bad page threshold cper records\n"); | |
| 938 | − | |
| 939 | − if ((amdgpu_bad_page_threshold != -1) && | |
| 940 | − (amdgpu_bad_page_threshold != -2)) | |
| 941 | − con->is_rma = true; | |
| 942 | − } | |
| 943 | − | |
| 944 | − return 0; | |
| 945 | −} | |
| 946 | − | |
| 947 | 868 | /** |
| 948 | 869 | * amdgpu_ras_eeprom_append -- append records to the EEPROM RAS table |
| 949 | 870 | * @control: pointer to control structure |
| @@ -968,9 +889,6 @@ int amdgpu_ras_eeprom_append(struct amdgpu_ras_eeprom_control *control, · 18 unmodified lines | ||
| 968 | 889 | if (!__is_ras_eeprom_supported(adev)) |
| 969 | 890 | return 0; |
| 970 | 891 | |
| 971 | − if (amdgpu_ras_smu_eeprom_supported(adev)) | |
| 972 | − return amdgpu_ras_smu_eeprom_append(control); | |
| 973 | − | |
| 974 | 892 | if (num == 0) { |
| 975 | 893 | dev_err(adev->dev, "will not append 0 records\n"); |
| 976 | 894 | return -EINVAL; |
| @@ -1046,52 +964,6 @@ static int __amdgpu_ras_eeprom_read(struct amdgpu_ras_eeprom_control *control, · 69 unmodified lines | ||
| 1046 | 964 | return res; |
| 1047 | 965 | } |
| 1048 | 966 | |
| 1049 | −int amdgpu_ras_eeprom_read_idx(struct amdgpu_ras_eeprom_control *control, | |
| 1050 | − struct eeprom_table_record *record, u32 rec_idx, | |
| 1051 | − const u32 num) | |
| 1052 | −{ | |
| 1053 | − struct amdgpu_device *adev = to_amdgpu_device(control); | |
| 1054 | − uint64_t ts, end_idx; | |
| 1055 | − int i, ret; | |
| 1056 | − u64 mca, ipid; | |
| 1057 | − u32 cu, mem_channel, mcumc_id; | |
| 1058 | − | |
| 1059 | − if (!amdgpu_ras_smu_eeprom_supported(adev)) | |
| 1060 | − return 0; | |
| 1061 | − | |
| 1062 | − if (!adev->umc.ras || !adev->umc.ras->mca_ipid_parse) | |
| 1063 | − return -EOPNOTSUPP; | |
| 1064 | − | |
| 1065 | − end_idx = rec_idx + num; | |
| 1066 | − for (i = rec_idx; i < end_idx; i++) { | |
| 1067 | − ret = amdgpu_ras_smu_get_badpage_mca_addr(adev, i, &mca); | |
| 1068 | − if (ret) | |
| 1069 | − return ret; | |
| 1070 | − | |
| 1071 | − ret = amdgpu_ras_smu_get_badpage_ipid(adev, i, &ipid); | |
| 1072 | − if (ret) | |
| 1073 | − return ret; | |
| 1074 | − | |
| 1075 | − ret = amdgpu_ras_smu_get_timestamp(adev, i, &ts); | |
| 1076 | − if (ret) | |
| 1077 | − return ret; | |
| 1078 | − | |
| 1079 | − record[i - rec_idx].address = mca; | |
| 1080 | − /* retired_page (pa) is unused now */ | |
| 1081 | − record[i - rec_idx].retired_page = 0x1ULL; | |
| 1082 | − record[i - rec_idx].ts = ts; | |
| 1083 | − record[i - rec_idx].err_type = AMDGPU_RAS_EEPROM_ERR_NON_RECOVERABLE; | |
| 1084 | − | |
| 1085 | − adev->umc.ras->mca_ipid_parse(adev, ipid, | |
| 1086 | − &cu, &mem_channel, &mcumc_id, NULL); | |
| 1087 | − record[i - rec_idx].cu = (u8)cu; | |
| 1088 | − record[i - rec_idx].mem_channel = (u8)mem_channel; | |
| 1089 | − record[i - rec_idx].mcumc_id = (u8)mcumc_id; | |
| 1090 | − } | |
| 1091 | − | |
| 1092 | − return 0; | |
| 1093 | −} | |
| 1094 | − | |
| 1095 | 967 | /** |
| 1096 | 968 | * amdgpu_ras_eeprom_read -- read EEPROM |
| 1097 | 969 | * @control: pointer to control structure |
| @@ -1113,9 +985,6 @@ int amdgpu_ras_eeprom_read(struct amdgpu_ras_eeprom_control *control, · 15 unmodified lines | ||
| 1113 | 985 | u8 *buf, *pp; |
| 1114 | 986 | u32 g0, g1; |
| 1115 | 987 | |
| 1116 | − if (amdgpu_ras_smu_eeprom_supported(adev)) | |
| 1117 | − return amdgpu_ras_eeprom_read_idx(control, record, 0, num); | |
| 1118 | − | |
| 1119 | 988 | if (!__is_ras_eeprom_supported(adev)) |
| 1120 | 989 | return 0; |
| 1121 | 990 | |
| @@ -1395,6 +1264,86 @@ Out: · 273 unmodified lines | ||
| 1395 | 1264 | return res < 0 ? res : orig_size - size; |
| 1396 | 1265 | } |
| 1397 | 1266 | |
| 1267 | +static ssize_t | |
| 1268 | +amdgpu_ras_debugfs_table_read_uniras(struct amdgpu_device *adev, | |
| 1269 | + char __user *buf, | |
| 1270 | + size_t size, loff_t *pos) | |
| 1271 | +{ | |
| 1272 | + struct amdgpu_ras_mgr *ras_mgr = amdgpu_ras_mgr_get_context(adev); | |
| 1273 | + struct ras_core_context *ras_core = ras_mgr ? ras_mgr->ras_core : NULL; | |
| 1274 | + struct eeprom_umc_record *records = NULL; | |
| 1275 | + struct ras_eeprom_control *control; | |
| 1276 | + size_t bufsz, len = 0; | |
| 1277 | + u32 num_recs; | |
| 1278 | + char *kbuf; | |
| 1279 | + ssize_t res; | |
| 1280 | + int i; | |
| 1281 | + | |
| 1282 | + if (!ras_core) | |
| 1283 | + return 0; | |
| 1284 | + | |
| 1285 | + /* pmfw manages eeprom data by itself */ | |
| 1286 | + if (ras_fw_eeprom_supported(ras_core)) | |
| 1287 | + return 0; | |
| 1288 | + | |
| 1289 | + control = &ras_core->ras_eeprom; | |
| 1290 | + num_recs = ras_eeprom_get_record_count(ras_core); | |
| 1291 | + | |
| 1292 | + bufsz = strlen(tbl_hdr_str) + tbl_hdr_fmt_size + | |
| 1293 | + strlen(rec_hdr_str) + (size_t)rec_hdr_fmt_size * num_recs + 1; | |
| 1294 | + | |
| 1295 | + kbuf = kvmalloc(bufsz, GFP_KERNEL); | |
| 1296 | + if (!kbuf) | |
| 1297 | + return -ENOMEM; | |
| 1298 | + | |
| 1299 | + if (num_recs) { | |
| 1300 | + records = kvcalloc(num_recs, sizeof(*records), GFP_KERNEL); | |
| 1301 | + if (!records) { | |
| 1302 | + res = -ENOMEM; | |
| 1303 | + goto out; | |
| 1304 | + } | |
| 1305 | + | |
| 1306 | + res = ras_eeprom_read(ras_core, records, num_recs); | |
| 1307 | + if (res) | |
| 1308 | + goto out; | |
| 1309 | + } | |
| 1310 | + | |
| 1311 | + len += scnprintf(kbuf + len, bufsz - len, "%s", tbl_hdr_str); | |
| 1312 | + len += scnprintf(kbuf + len, bufsz - len, tbl_hdr_fmt, | |
| 1313 | + control->tbl_hdr.header, | |
| 1314 | + control->tbl_hdr.version, | |
| 1315 | + control->tbl_hdr.first_rec_offset, | |
| 1316 | + control->tbl_hdr.tbl_size, | |
| 1317 | + control->tbl_hdr.checksum); | |
| 1318 | + len += scnprintf(kbuf + len, bufsz - len, "%s", rec_hdr_str); | |
| 1319 | + | |
| 1320 | + for (i = 0; i < num_recs; i++) { | |
| 1321 | + u32 ai = RAS_RI_TO_AI(control, i); | |
| 1322 | + int et = records[i].err_type; | |
| 1323 | + const char *ets = (et >= 0 && et < AMDGPU_RAS_EEPROM_ERR_COUNT) ? | |
| 1324 | + record_err_type_str[et] : "na"; | |
| 1325 | + | |
| 1326 | + len += scnprintf(kbuf + len, bufsz - len, rec_hdr_fmt, | |
| 1327 | + i, | |
| 1328 | + RAS_INDEX_TO_OFFSET(control, ai), | |
| 1329 | + ets, | |
| 1330 | + records[i].bank, | |
| 1331 | + records[i].ts, | |
| 1332 | + records[i].offset, | |
| 1333 | + records[i].mem_channel, | |
| 1334 | + records[i].mcumc_id, | |
| 1335 | + records[i].retired_row_pfn); | |
| 1336 | + } | |
| 1337 | + | |
| 1338 | + res = simple_read_from_buffer(buf, size, pos, kbuf, len); | |
| 1339 | + | |
| 1340 | +out: | |
| 1341 | + kvfree(records); | |
| 1342 | + kvfree(kbuf); | |
| 1343 | + | |
| 1344 | + return res; | |
| 1345 | +} | |
| 1346 | + | |
| 1398 | 1347 | static ssize_t |
| 1399 | 1348 | amdgpu_ras_debugfs_eeprom_table_read(struct file *f, char __user *buf, |
| 1400 | 1349 | size_t size, loff_t *pos) |
| @@ -1408,6 +1357,10 @@ amdgpu_ras_debugfs_eeprom_table_read(struct file *f, char __user *buf, · 7 unmodified lines | ||
| 1408 | 1357 | if (!size) |
| 1409 | 1358 | return size; |
| 1410 | 1359 | |
| 1360 | + if (amdgpu_uniras_enabled(adev)) | |
| 1361 | + return amdgpu_ras_debugfs_table_read_uniras(adev, buf, | |
| 1362 | + size, pos); | |
| 1363 | + | |
| 1411 | 1364 | if (!ras || !control) { |
| 1412 | 1365 | res = snprintf(data, sizeof(data), "Not supported\n"); |
| 1413 | 1366 | if (*pos >= res) |
| @@ -1521,42 +1474,6 @@ Out: · 107 unmodified lines | ||
| 1521 | 1474 | return res == RAS_TABLE_V2_1_INFO_SIZE ? 0 : res; |
| 1522 | 1475 | } |
| 1523 | 1476 | |
| 1524 | −static int amdgpu_ras_smu_eeprom_init(struct amdgpu_ras_eeprom_control *control) | |
| 1525 | −{ | |
| 1526 | − struct amdgpu_device *adev = to_amdgpu_device(control); | |
| 1527 | − struct amdgpu_ras_eeprom_table_header *hdr = &control->tbl_hdr; | |
| 1528 | − struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); | |
| 1529 | − uint64_t local_time; | |
| 1530 | − int res; | |
| 1531 | − | |
| 1532 | − ras->is_rma = false; | |
| 1533 | − | |
| 1534 | − if (!__is_ras_eeprom_supported(adev)) | |
| 1535 | − return 0; | |
| 1536 | − mutex_init(&control->ras_tbl_mutex); | |
| 1537 | − | |
| 1538 | − res = amdgpu_ras_smu_get_table_version(adev, &(hdr->version)); | |
| 1539 | − if (res) | |
| 1540 | − return res; | |
| 1541 | − | |
| 1542 | − res = amdgpu_ras_smu_get_badpage_count(adev, | |
| 1543 | − &(control->ras_num_recs), 100); | |
| 1544 | − if (res) | |
| 1545 | − return res; | |
| 1546 | − | |
| 1547 | − local_time = (uint64_t)ktime_get_real_seconds(); | |
| 1548 | − res = amdgpu_ras_smu_set_timestamp(adev, local_time); | |
| 1549 | − if (res) | |
| 1550 | − return res; | |
| 1551 | − | |
| 1552 | − control->ras_max_record_count = 4000; | |
| 1553 | − | |
| 1554 | − control->ras_num_mca_recs = 0; | |
| 1555 | − control->ras_num_pa_recs = 0; | |
| 1556 | − | |
| 1557 | − return 0; | |
| 1558 | −} | |
| 1559 | − | |
| 1560 | 1477 | int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control) |
| 1561 | 1478 | { |
| 1562 | 1479 | struct amdgpu_device *adev = to_amdgpu_device(control); |
| @@ -1567,9 +1484,6 @@ int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control) · 4 unmodified lines | ||
| 1567 | 1484 | uint32_t vram_type = adev->gmc.vram_type; |
| 1568 | 1485 | int res; |
| 1569 | 1486 | |
| 1570 | − if (amdgpu_ras_smu_eeprom_supported(adev)) | |
| 1571 | − return amdgpu_ras_smu_eeprom_init(control); | |
| 1572 | − | |
| 1573 | 1487 | ras->is_rma = false; |
| 1574 | 1488 | |
| 1575 | 1489 | if (!__is_ras_eeprom_supported(adev)) |
| @@ -1663,47 +1577,6 @@ int amdgpu_ras_eeprom_init(struct amdgpu_ras_eeprom_control *control) · 87 unmodified lines | ||
| 1663 | 1577 | return 0; |
| 1664 | 1578 | } |
| 1665 | 1579 | |
| 1666 | −static int amdgpu_ras_smu_eeprom_check(struct amdgpu_ras_eeprom_control *control) | |
| 1667 | −{ | |
| 1668 | − struct amdgpu_device *adev = to_amdgpu_device(control); | |
| 1669 | − struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); | |
| 1670 | − | |
| 1671 | − if (!__is_ras_eeprom_supported(adev)) | |
| 1672 | − return 0; | |
| 1673 | − | |
| 1674 | − control->ras_num_bad_pages = ras->bad_page_num; | |
| 1675 | − | |
| 1676 | − if ((ras->bad_page_cnt_threshold < control->ras_num_bad_pages) && | |
| 1677 | − amdgpu_bad_page_threshold != 0) { | |
| 1678 | − dev_warn(adev->dev, | |
| 1679 | − "RAS records:%d exceed threshold:%d\n", | |
| 1680 | − control->ras_num_bad_pages, ras->bad_page_cnt_threshold); | |
| 1681 | − if ((amdgpu_bad_page_threshold == -1) || | |
| 1682 | − (amdgpu_bad_page_threshold == -2)) { | |
| 1683 | − dev_warn(adev->dev, | |
| 1684 | − "Please consult AMD Service Action Guide (SAG) for appropriate service procedures\n"); | |
| 1685 | − } else { | |
| 1686 | − ras->is_rma = true; | |
| 1687 | − dev_warn(adev->dev, | |
| 1688 | − "User defined threshold is set, runtime service will be halt when threshold is reached\n"); | |
| 1689 | − } | |
| 1690 | − | |
| 1691 | − return 0; | |
| 1692 | − } | |
| 1693 | − | |
| 1694 | − dev_dbg(adev->dev, | |
| 1695 | − "Found existing EEPROM table with %d records", | |
| 1696 | − control->ras_num_bad_pages); | |
| 1697 | − | |
| 1698 | − /* Warn if we are at 90% of the threshold or above | |
| 1699 | − */ | |
| 1700 | − if (10 * control->ras_num_bad_pages >= 9 * ras->bad_page_cnt_threshold) | |
| 1701 | − dev_warn(adev->dev, "RAS records:%u exceeds 90%% of threshold:%d", | |
| 1702 | − control->ras_num_bad_pages, | |
| 1703 | − ras->bad_page_cnt_threshold); | |
| 1704 | − return 0; | |
| 1705 | −} | |
| 1706 | − | |
| 1707 | 1580 | int amdgpu_ras_eeprom_check(struct amdgpu_ras_eeprom_control *control) |
| 1708 | 1581 | { |
| 1709 | 1582 | struct amdgpu_device *adev = to_amdgpu_device(control); |
| @@ -1711,9 +1584,6 @@ int amdgpu_ras_eeprom_check(struct amdgpu_ras_eeprom_control *control) · 1 unmodified line | ||
| 1711 | 1584 | struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); |
| 1712 | 1585 | int res = 0; |
| 1713 | 1586 | |
| 1714 | − if (amdgpu_ras_smu_eeprom_supported(adev)) | |
| 1715 | − return amdgpu_ras_smu_eeprom_check(control); | |
| 1716 | − | |
| 1717 | 1587 | if (!__is_ras_eeprom_supported(adev)) |
| 1718 | 1588 | return 0; |
| 1719 | 1589 | |
| @@ -1973,7 +1843,7 @@ void amdgpu_ras_check_bad_page_status(struct amdgpu_device *adev) · 253 unmodified lines | ||
| 1973 | 1843 | struct amdgpu_ras *ras = amdgpu_ras_get_context(adev); |
| 1974 | 1844 | struct amdgpu_ras_eeprom_control *control = ras ? &ras->eeprom_control : NULL; |
| 1975 | 1845 | |
| 1976 | − if (!control || amdgpu_bad_page_threshold == 0) | |
| 1846 | + if (!__is_ras_eeprom_supported(adev) || !control || amdgpu_bad_page_threshold == 0) | |
| 1977 | 1847 | return; |
| 1978 | 1848 | |
| 1979 | 1849 | if (control->ras_num_bad_pages > ras->bad_page_cnt_threshold) { |
| @@ -82,7 +82,6 @@ struct amdgpu_ras_eeprom_control { · 81 unmodified lines | ||
| 82 | 82 | /* Number of records in the table. |
| 83 | 83 | */ |
| 84 | 84 | u32 ras_num_recs; |
| 85 | − u32 ras_num_recs_old; | |
| 86 | 85 | |
| 87 | 86 | /* the bad page number is ras_num_recs or |
| 88 | 87 | * ras_num_recs * umc.retire_unit |
| @@ -191,8 +190,6 @@ int amdgpu_ras_eeprom_read_idx(struct amdgpu_ras_eeprom_control *control, · 102 unmodified lines | ||
| 191 | 190 | struct eeprom_table_record *record, u32 rec_idx, |
| 192 | 191 | const u32 num); |
| 193 | 192 | |
| 194 | −int amdgpu_ras_eeprom_update_record_num(struct amdgpu_ras_eeprom_control *control); | |
| 195 | − | |
| 196 | 193 | void amdgpu_ras_check_bad_page_status(struct amdgpu_device *adev); |
| 197 | 194 | |
| 198 | 195 | extern const struct file_operations amdgpu_ras_debugfs_eeprom_size_ops; |
| @@ -935,6 +935,194 @@ int amdgpu_ring_reset_helper_end(struct amdgpu_ring *ring, · 934 unmodified lines | ||
| 935 | 935 | return 0; |
| 936 | 936 | } |
| 937 | 937 | |
| 938 | +/** | |
| 939 | + * amdgpu_multi_ring_reset_helper_begin() - Prepare multiple rings for a reset. | |
| 940 | + * | |
| 941 | + * @ring_type_mask: Bitmask of affected ring types | |
| 942 | + * @guilty_ring: The ring which is guilty of causing a reset. | |
| 943 | + * @guilty_fence: The fence which didn't signal on the guilty ring. | |
| 944 | + * | |
| 945 | + * Useful when performing a GPU reset method that affects | |
| 946 | + * multiple rings at the same time, such as an IP block soft | |
| 947 | + * reset. For example, a GFX IP block soft reset will affect | |
| 948 | + * every graphics and compute queue. | |
| 949 | + * | |
| 950 | + * This function should be called before such a reset. | |
| 951 | + * | |
| 952 | + * Prepare the affected rings before the reset, make sure to | |
| 953 | + * minimize collateral damage, and backup the contents of | |
| 954 | + * the rings. Then the caller can call the actual HW specific | |
| 955 | + * reset function. | |
| 956 | + * | |
| 957 | + * After the reset is complete, the caller should then call | |
| 958 | + * amdgpu_multi_ring_reset_helper_end() to restore the rings. | |
| 959 | + */ | |
| 960 | +void amdgpu_multi_ring_reset_helper_begin(const u32 ring_type_mask, | |
| 961 | + struct amdgpu_ring *guilty_ring, | |
| 962 | + struct amdgpu_fence *guilty_fence) | |
| 963 | +{ | |
| 964 | + struct amdgpu_device *adev = guilty_ring->adev; | |
| 965 | + struct amdgpu_fence *ring_guilty_fence; | |
| 966 | + struct amdgpu_ring *ring; | |
| 967 | + bool rings_busy; | |
| 968 | + int i; | |
| 969 | + u32 t; | |
| 970 | + | |
| 971 | + for (i = 0; i < adev->num_rings; ++i) { | |
| 972 | + ring = adev->rings[i]; | |
| 973 | + | |
| 974 | + if (!(BIT(ring->funcs->type) & ring_type_mask)) | |
| 975 | + continue; | |
| 976 | + | |
| 977 | + /* Don't accept new submissions on the ring. */ | |
| 978 | + if (amdgpu_ring_sched_ready(ring) && !drm_sched_is_stopped(&ring->sched)) | |
| 979 | + drm_sched_wqueue_stop(&ring->sched); | |
| 980 | + | |
| 981 | + /* | |
| 982 | + * Clear the preempt condition to stop the ring | |
| 983 | + * from starting its next submission. This ensures | |
| 984 | + * that only the currently executing submission | |
| 985 | + * can be rejected because of the reset and helps | |
| 986 | + * minimize collateral damage. | |
| 987 | + */ | |
| 988 | + if (ring->funcs->init_cond_exec) | |
| 989 | + amdgpu_ring_set_preempt_cond_exec(ring, false); | |
| 990 | + } | |
| 991 | + | |
| 992 | + /* Flush HDP cache so the GPU can see the updated COND_EXEC values */ | |
| 993 | + amdgpu_device_flush_hdp(adev, NULL); | |
| 994 | + | |
| 995 | + /* | |
| 996 | + * Give some time for non-guilty rings to finish their | |
| 997 | + * current submission, to try to minimize collateral damage. | |
| 998 | + * | |
| 999 | + * Note that this is just a best effort, but really there | |
| 1000 | + * is no way to really know which ring is actually responsible | |
| 1001 | + * because different rings may share resources, eg. a compute | |
| 1002 | + * ring may hog shader engines, causing a graphics ring to hang. | |
| 1003 | + */ | |
| 1004 | + for (t = 0; t < adev->usec_timeout; t += 10000) { | |
| 1005 | + rings_busy = false; | |
| 1006 | + | |
| 1007 | + /* Check if any of the non-guilty rings are busy */ | |
| 1008 | + for (i = 0; i < adev->num_rings; ++i) { | |
| 1009 | + ring = adev->rings[i]; | |
| 1010 | + | |
| 1011 | + if (!(BIT(ring->funcs->type) & ring_type_mask)) | |
| 1012 | + continue; | |
| 1013 | + | |
| 1014 | + if (ring == guilty_ring) | |
| 1015 | + continue; | |
| 1016 | + | |
| 1017 | + rings_busy |= | |
| 1018 | + atomic_read(&ring->fence_drv.last_seq) != | |
| 1019 | + READ_ONCE(ring->fence_drv.sync_seq); | |
| 1020 | + } | |
| 1021 | + | |
| 1022 | + if (!rings_busy) | |
| 1023 | + break; | |
| 1024 | + | |
| 1025 | + mdelay(10); | |
| 1026 | + } | |
| 1027 | + | |
| 1028 | + for (i = 0; i < adev->num_rings; ++i) { | |
| 1029 | + ring = adev->rings[i]; | |
| 1030 | + | |
| 1031 | + if (!(BIT(ring->funcs->type) & ring_type_mask)) | |
| 1032 | + continue; | |
| 1033 | + | |
| 1034 | + /* | |
| 1035 | + * Find guilty fences, ie. the fences that didn't signal | |
| 1036 | + * on each ring. At this point there is no way to know | |
| 1037 | + * which one is really responsible for the hang, and no | |
| 1038 | + * way to save any of them, so we treat all of them as guilty. | |
| 1039 | + */ | |
| 1040 | + ring_guilty_fence = | |
| 1041 | + ring == guilty_ring ? guilty_fence : | |
| 1042 | + amdgpu_ring_find_guilty_fence(ring); | |
| 1043 | + | |
| 1044 | + /* | |
| 1045 | + * Backup current contents of the ring. | |
| 1046 | + * The helper takes care to only reemit unsignalled fences | |
| 1047 | + * so we don't have to worry about that here. | |
| 1048 | + */ | |
| 1049 | + amdgpu_ring_reset_helper_begin(ring, ring_guilty_fence); | |
| 1050 | + } | |
| 1051 | +} | |
| 1052 | + | |
| 1053 | +/** | |
| 1054 | + * amdgpu_multi_ring_reset_helper_end() - Prepare multiple rings for a reset. | |
| 1055 | + * | |
| 1056 | + * @ring_type_mask: Bitmask of affected ring types | |
| 1057 | + * @guilty_ring: The ring which is guilty of causing a reset. | |
| 1058 | + * @ret: Return code from the reset function. | |
| 1059 | + * | |
| 1060 | + * After calling amdgpu_multi_ring_reset_helper_begin() | |
| 1061 | + * and executing the actual reset method, call this | |
| 1062 | + * function to restore normal operation. | |
| 1063 | + * | |
| 1064 | + * In case the reset failed, this function should still | |
| 1065 | + * be called to restore preemption state, but it won't attempt to | |
| 1066 | + * fully restore the ring contents. | |
| 1067 | + */ | |
| 1068 | +int amdgpu_multi_ring_reset_helper_end(const u32 ring_type_mask, | |
| 1069 | + struct amdgpu_ring *guilty_ring, int ret) | |
| 1070 | +{ | |
| 1071 | + struct amdgpu_device *adev = guilty_ring->adev; | |
| 1072 | + struct amdgpu_ring *ring; | |
| 1073 | + int i, r; | |
| 1074 | + | |
| 1075 | + /* Set preempt condition, rings are now allowed to execute submissions */ | |
| 1076 | + for (i = 0; i < adev->num_rings; ++i) { | |
| 1077 | + ring = adev->rings[i]; | |
| 1078 | + | |
| 1079 | + if (!(BIT(ring->funcs->type) & ring_type_mask)) | |
| 1080 | + continue; | |
| 1081 | + | |
| 1082 | + if (ring->funcs->init_cond_exec) | |
| 1083 | + amdgpu_ring_set_preempt_cond_exec(ring, true); | |
| 1084 | + } | |
| 1085 | + | |
| 1086 | + /* Flush HDP cache so the GPU can see the updated COND_EXEC values */ | |
| 1087 | + amdgpu_device_flush_hdp(adev, NULL); | |
| 1088 | + | |
| 1089 | + /* If the reset was unsuccessful, return without restoring anything else. */ | |
| 1090 | + if (ret) | |
| 1091 | + return ret; | |
| 1092 | + | |
| 1093 | + /* Restore contents of all rings */ | |
| 1094 | + for (i = 0; i < adev->num_rings; ++i) { | |
| 1095 | + ring = adev->rings[i]; | |
| 1096 | + | |
| 1097 | + if (!(BIT(ring->funcs->type) & ring_type_mask)) | |
| 1098 | + continue; | |
| 1099 | + | |
| 1100 | + /* Restore contents of the ring */ | |
| 1101 | + r = amdgpu_ring_reset_helper_end(ring, ring->guilty_fence); | |
| 1102 | + if (r) { | |
| 1103 | + dev_err(adev->dev, | |
| 1104 | + "Failed to recover ring %s after soft reset\n", | |
| 1105 | + ring->name); | |
| 1106 | + return r; | |
| 1107 | + } | |
| 1108 | + } | |
| 1109 | + | |
| 1110 | + /* Accept submissions on all rings again */ | |
| 1111 | + for (i = 0; i < adev->num_rings; ++i) { | |
| 1112 | + ring = adev->rings[i]; | |
| 1113 | + | |
| 1114 | + if (!(BIT(ring->funcs->type) & ring_type_mask)) | |
| 1115 | + continue; | |
| 1116 | + | |
| 1117 | + if (!amdgpu_ring_sched_ready(ring)) | |
| 1118 | + continue; | |
| 1119 | + | |
| 1120 | + drm_sched_wqueue_start(&ring->sched); | |
| 1121 | + } | |
| 1122 | + | |
| 1123 | + return 0; | |
| 1124 | +} | |
| 1125 | + | |
| 938 | 1126 | bool amdgpu_ring_is_reset_type_supported(struct amdgpu_ring *ring, |
| 939 | 1127 | u32 reset_type) |
| 940 | 1128 | { |
| @@ -314,6 +314,7 @@ struct amdgpu_ring { · 313 unmodified lines | ||
| 314 | 314 | uint32_t *ring_backup; |
| 315 | 315 | unsigned int ring_backup_entries_to_copy; |
| 316 | 316 | bool reemit; |
| 317 | + struct amdgpu_fence *guilty_fence; | |
| 317 | 318 | unsigned rptr_offs; |
| 318 | 319 | u64 rptr_gpu_addr; |
| 319 | 320 | u32 *rptr_cpu_addr; |
| @@ -588,10 +589,17 @@ int amdgpu_ib_ring_tests(struct amdgpu_device *adev); · 268 unmodified lines | ||
| 588 | 589 | bool amdgpu_ring_sched_ready(struct amdgpu_ring *ring); |
| 589 | 590 | void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring, |
| 590 | 591 | struct amdgpu_fence *guilty_fence); |
| 592 | +struct amdgpu_fence * | |
| 593 | +amdgpu_ring_find_guilty_fence(struct amdgpu_ring *ring); | |
| 591 | 594 | void amdgpu_ring_reset_helper_begin(struct amdgpu_ring *ring, |
| 592 | 595 | struct amdgpu_fence *guilty_fence); |
| 593 | 596 | int amdgpu_ring_reset_helper_end(struct amdgpu_ring *ring, |
| 594 | 597 | struct amdgpu_fence *guilty_fence); |
| 598 | +void amdgpu_multi_ring_reset_helper_begin(const u32 ring_type_mask, | |
| 599 | + struct amdgpu_ring *guilty_ring, | |
| 600 | + struct amdgpu_fence *guilty_fence); | |
| 601 | +int amdgpu_multi_ring_reset_helper_end(const u32 ring_type_mask, | |
| 602 | + struct amdgpu_ring *guilty_ring, int ret); | |
| 595 | 603 | bool amdgpu_ring_is_reset_type_supported(struct amdgpu_ring *ring, |
| 596 | 604 | u32 reset_type); |
| 597 | 605 | #endif |
| @@ -583,3 +583,42 @@ int amdgpu_gfx_rlc_init_microcode(struct amdgpu_device *adev, · 582 unmodified lines | ||
| 583 | 583 | amdgpu_gfx_rlc_init_microcode_v2_5(adev); |
| 584 | 584 | return 0; |
| 585 | 585 | } |
| 586 | + | |
| 587 | +static const struct amdgpu_rlc_reg_funcs amdgpu_sriov_rlc_reg_funcs = { | |
| 588 | + .rreg32 = amdgpu_sriov_rreg, | |
| 589 | + .wreg32 = amdgpu_sriov_wreg, | |
| 590 | +}; | |
| 591 | + | |
| 592 | +static u32 | |
| 593 | +amdgpu_rlc_rreg(struct amdgpu_device *adev, u32 reg, u32 acc_flags, u32 hwip, | |
| 594 | + u32 xcc_id) | |
| 595 | +{ | |
| 596 | + return amdgpu_device_rreg(adev, reg, 0); | |
| 597 | +} | |
| 598 | + | |
| 599 | +static void | |
| 600 | +amdgpu_rlc_wreg(struct amdgpu_device *adev, u32 reg, u32 value, u32 acc_flags, | |
| 601 | + u32 hwip, u32 xcc_id) | |
| 602 | +{ | |
| 603 | + amdgpu_device_wreg(adev, reg, value, 0); | |
| 604 | +} | |
| 605 | + | |
| 606 | +static const struct amdgpu_rlc_reg_funcs amdgpu_rlc_reg_funcs = { | |
| 607 | + .rreg32 = amdgpu_rlc_rreg, | |
| 608 | + .wreg32 = amdgpu_rlc_wreg, | |
| 609 | +}; | |
| 610 | + | |
| 611 | +void amdgpu_early_init_rlc_reg_funcs(struct amdgpu_device *adev) | |
| 612 | +{ | |
| 613 | + adev->gfx.rlc.reg_funcs = &amdgpu_rlc_reg_funcs; | |
| 614 | +} | |
| 615 | + | |
| 616 | +void amdgpu_init_rlc_reg_funcs(struct amdgpu_device *adev) | |
| 617 | +{ | |
| 618 | + if (amdgpu_sriov_vf(adev) && | |
| 619 | + adev->gfx.rlc.funcs && | |
| 620 | + adev->gfx.rlc.rlcg_reg_access_supported) | |
| 621 | + adev->gfx.rlc.reg_funcs = &amdgpu_sriov_rlc_reg_funcs; | |
| 622 | + else | |
| 623 | + adev->gfx.rlc.reg_funcs = &amdgpu_rlc_reg_funcs; | |
| 624 | +} | |
| @@ -262,6 +262,11 @@ struct amdgpu_rlc_funcs { · 261 unmodified lines | ||
| 262 | 262 | bool (*is_rlcg_access_range)(struct amdgpu_device *adev, uint32_t reg); |
| 263 | 263 | }; |
| 264 | 264 | |
| 265 | +struct amdgpu_rlc_reg_funcs { | |
| 266 | + u32 (*rreg32)(struct amdgpu_device *adev, u32 reg, u32 acc_flags, u32 hwip, u32 xcc_id); | |
| 267 | + void (*wreg32)(struct amdgpu_device *adev, u32 reg, u32 val, u32 acc_flags, u32 hwip, u32 xcc_id); | |
| 268 | +}; | |
| 269 | + | |
| 265 | 270 | struct amdgpu_rlcg_reg_access_ctrl { |
| 266 | 271 | uint32_t scratch_reg0; |
| 267 | 272 | uint32_t scratch_reg1; |
| @@ -303,6 +308,7 @@ struct amdgpu_rlc { · 35 unmodified lines | ||
| 303 | 308 | /* safe mode for updating CG/PG state */ |
| 304 | 309 | bool in_safe_mode[AMDGPU_MAX_RLC_INSTANCES]; |
| 305 | 310 | const struct amdgpu_rlc_funcs *funcs; |
| 311 | + const struct amdgpu_rlc_reg_funcs *reg_funcs; | |
| 306 | 312 | |
| 307 | 313 | /* for firmware data */ |
| 308 | 314 | u32 save_and_restore_offset; |
| @@ -374,4 +380,8 @@ void amdgpu_gfx_rlc_fini(struct amdgpu_device *adev); · 65 unmodified lines | ||
| 374 | 380 | int amdgpu_gfx_rlc_init_microcode(struct amdgpu_device *adev, |
| 375 | 381 | uint16_t version_major, |
| 376 | 382 | uint16_t version_minor); |
| 383 | + | |
| 384 | +void amdgpu_early_init_rlc_reg_funcs(struct amdgpu_device *adev); | |
| 385 | +void amdgpu_init_rlc_reg_funcs(struct amdgpu_device *adev); | |
| 386 | + | |
| 377 | 387 | #endif |
| @@ -0,0 +1,77 @@ | ||
| 1 | +/* SPDX-License-Identifier: MIT */ | |
| 2 | +/* | |
| 3 | + * Copyright 2026 Advanced Micro Devices, Inc. | |
| 4 | + * | |
| 5 | + * Permission is hereby granted, free of charge, to any person obtaining a | |
| 6 | + * copy of this software and associated documentation files (the "Software"), | |
| 7 | + * to deal in the Software without restriction, including without limitation | |
| 8 | + * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
| 9 | + * and/or sell copies of the Software, and to permit persons to whom the | |
| 10 | + * Software is furnished to do so, subject to the following conditions: | |
| 11 | + * | |
| 12 | + * The above copyright notice and this permission notice shall be included in | |
| 13 | + * all copies or substantial portions of the Software. | |
| 14 | + * | |
| 15 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
| 16 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
| 17 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
| 18 | + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
| 19 | + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
| 20 | + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
| 21 | + * OTHER DEALINGS IN THE SOFTWARE. | |
| 22 | + * | |
| 23 | + */ | |
| 24 | + | |
| 25 | +#ifndef AMDGPU_SA_H_ | |
| 26 | +#define AMDGPU_SA_H_ | |
| 27 | + | |
| 28 | +#include <drm/drm_suballoc.h> | |
| 29 | + | |
| 30 | +struct amdgpu_device; | |
| 31 | +struct amdgpu_bo; | |
| 32 | + | |
| 33 | +struct amdgpu_sa_manager { | |
| 34 | + struct drm_suballoc_manager base; | |
| 35 | + struct amdgpu_bo *bo; | |
| 36 | + uint64_t gpu_addr; | |
| 37 | + void *cpu_ptr; | |
| 38 | +}; | |
| 39 | + | |
| 40 | +static inline struct amdgpu_sa_manager * | |
| 41 | +to_amdgpu_sa_manager(struct drm_suballoc_manager *manager) | |
| 42 | +{ | |
| 43 | + return container_of(manager, struct amdgpu_sa_manager, base); | |
| 44 | +} | |
| 45 | + | |
| 46 | +static inline uint64_t amdgpu_sa_bo_gpu_addr(struct drm_suballoc *sa_bo) | |
| 47 | +{ | |
| 48 | + return to_amdgpu_sa_manager(sa_bo->manager)->gpu_addr + | |
| 49 | + drm_suballoc_soffset(sa_bo); | |
| 50 | +} | |
| 51 | + | |
| 52 | +static inline void *amdgpu_sa_bo_cpu_addr(struct drm_suballoc *sa_bo) | |
| 53 | +{ | |
| 54 | + return to_amdgpu_sa_manager(sa_bo->manager)->cpu_ptr + | |
| 55 | + drm_suballoc_soffset(sa_bo); | |
| 56 | +} | |
| 57 | + | |
| 58 | +int amdgpu_sa_bo_manager_init(struct amdgpu_device *adev, | |
| 59 | + struct amdgpu_sa_manager *sa_manager, | |
| 60 | + unsigned size, u32 align, u32 domain); | |
| 61 | +void amdgpu_sa_bo_manager_fini(struct amdgpu_device *adev, | |
| 62 | + struct amdgpu_sa_manager *sa_manager); | |
| 63 | +int amdgpu_sa_bo_manager_start(struct amdgpu_device *adev, | |
| 64 | + struct amdgpu_sa_manager *sa_manager); | |
| 65 | +int amdgpu_sa_bo_new(struct amdgpu_sa_manager *sa_manager, | |
| 66 | + struct drm_suballoc **sa_bo, | |
| 67 | + unsigned int size); | |
| 68 | +void amdgpu_sa_bo_free(struct drm_suballoc **sa_bo, | |
| 69 | + struct dma_fence *fence); | |
| 70 | +#if defined(CONFIG_DEBUG_FS) | |
| 71 | +void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager, | |
| 72 | + struct seq_file *m); | |
| 73 | +u64 amdgpu_bo_print_info(int id, struct amdgpu_bo *bo, struct seq_file *m); | |
| 74 | +#endif | |
| 75 | +void amdgpu_debugfs_sa_init(struct amdgpu_device *adev); | |
| 76 | + | |
| 77 | +#endif | |
| @@ -39,7 +39,7 @@ static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev, · 38 unmodified lines | ||
| 39 | 39 | struct amdgpu_fpriv *fpriv; |
| 40 | 40 | struct amdgpu_ctx_mgr *mgr; |
| 41 | 41 | struct amdgpu_ctx *ctx; |
| 42 | − uint32_t id; | |
| 42 | + unsigned long id; | |
| 43 | 43 | int r; |
| 44 | 44 | |
| 45 | 45 | if (fd_empty(f)) |
| @@ -50,10 +50,10 @@ static int amdgpu_sched_process_priority_override(struct amdgpu_device *adev, · 4 unmodified lines | ||
| 50 | 50 | return r; |
| 51 | 51 | |
| 52 | 52 | mgr = &fpriv->ctx_mgr; |
| 53 | − mutex_lock(&mgr->lock); | |
| 54 | − idr_for_each_entry(&mgr->ctx_handles, ctx, id) | |
| 53 | + xa_lock(&mgr->ctx_handles); | |
| 54 | + xa_for_each(&mgr->ctx_handles, id, ctx) | |
| 55 | 55 | amdgpu_ctx_priority_override(ctx, priority); |
| 56 | − mutex_unlock(&mgr->lock); | |
| 56 | + xa_unlock(&mgr->ctx_handles); | |
| 57 | 57 | |
| 58 | 58 | return 0; |
| 59 | 59 | } |
| @@ -553,10 +553,11 @@ static int amdgpu_sdma_soft_reset(struct amdgpu_device *adev, u32 instance_id) · 552 unmodified lines | ||
| 553 | 553 | int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id, |
| 554 | 554 | bool caller_handles_kernel_queues) |
| 555 | 555 | { |
| 556 | − int ret = 0; | |
| 557 | 556 | struct amdgpu_sdma_instance *sdma_instance = &adev->sdma.instance[instance_id]; |
| 558 | 557 | struct amdgpu_ring *gfx_ring = &sdma_instance->ring; |
| 559 | 558 | struct amdgpu_ring *page_ring = &sdma_instance->page; |
| 559 | + struct amdgpu_fence *gfx_fence, *page_fence; | |
| 560 | + int ret = 0; | |
| 560 | 561 | |
| 561 | 562 | if (amdgpu_sriov_vf(adev)) |
| 562 | 563 | return -EOPNOTSUPP; |
| @@ -569,9 +570,14 @@ int amdgpu_sdma_reset_engine(struct amdgpu_device *adev, uint32_t instance_id, · 6 unmodified lines | ||
| 569 | 570 | * the reset is in progress. |
| 570 | 571 | */ |
| 571 | 572 | drm_sched_wqueue_stop(&gfx_ring->sched); |
| 573 | + gfx_fence = amdgpu_ring_find_guilty_fence(gfx_ring); | |
| 574 | + amdgpu_ring_reset_helper_begin(gfx_ring, gfx_fence); | |
| 572 | 575 | |
| 573 | − if (adev->sdma.has_page_queue) | |
| 576 | + if (adev->sdma.has_page_queue) { | |
| 574 | 577 | drm_sched_wqueue_stop(&page_ring->sched); |
| 578 | + page_fence = amdgpu_ring_find_guilty_fence(page_ring); | |
| 579 | + amdgpu_ring_reset_helper_begin(page_ring, page_fence); | |
| 580 | + } | |
| 575 | 581 | } |
| 576 | 582 | |
| 577 | 583 | if (sdma_instance->funcs->stop_kernel_queue) { |
| @@ -600,14 +606,19 @@ exit: · 22 unmodified lines | ||
| 600 | 606 | * to be submitted to the queues after the reset is complete. |
| 601 | 607 | */ |
| 602 | 608 | if (!ret) { |
| 603 | − amdgpu_fence_driver_force_completion(gfx_ring, NULL); | |
| 609 | + ret = amdgpu_ring_reset_helper_end(gfx_ring, gfx_fence); | |
| 610 | + if (ret) | |
| 611 | + goto unlock; | |
| 604 | 612 | drm_sched_wqueue_start(&gfx_ring->sched); |
| 605 | 613 | if (adev->sdma.has_page_queue) { |
| 606 | − amdgpu_fence_driver_force_completion(page_ring, NULL); | |
| 614 | + ret = amdgpu_ring_reset_helper_end(page_ring, page_fence); | |
| 615 | + if (ret) | |
| 616 | + goto unlock; | |
| 607 | 617 | drm_sched_wqueue_start(&page_ring->sched); |
| 608 | 618 | } |
| 609 | 619 | } |
| 610 | 620 | } |
| 621 | +unlock: | |
| 611 | 622 | mutex_unlock(&sdma_instance->engine_reset_mutex); |
| 612 | 623 | |
| 613 | 624 | return ret; |
| @@ -85,34 +85,6 @@ struct amdgpu_sdma_instance { · 84 unmodified lines | ||
| 85 | 85 | const struct amdgpu_sdma_funcs *funcs; |
| 86 | 86 | }; |
| 87 | 87 | |
| 88 | −enum amdgpu_sdma_ras_memory_id { | |
| 89 | − AMDGPU_SDMA_MBANK_DATA_BUF0 = 1, | |
| 90 | − AMDGPU_SDMA_MBANK_DATA_BUF1 = 2, | |
| 91 | − AMDGPU_SDMA_MBANK_DATA_BUF2 = 3, | |
| 92 | − AMDGPU_SDMA_MBANK_DATA_BUF3 = 4, | |
| 93 | − AMDGPU_SDMA_MBANK_DATA_BUF4 = 5, | |
| 94 | − AMDGPU_SDMA_MBANK_DATA_BUF5 = 6, | |
| 95 | − AMDGPU_SDMA_MBANK_DATA_BUF6 = 7, | |
| 96 | − AMDGPU_SDMA_MBANK_DATA_BUF7 = 8, | |
| 97 | − AMDGPU_SDMA_MBANK_DATA_BUF8 = 9, | |
| 98 | − AMDGPU_SDMA_MBANK_DATA_BUF9 = 10, | |
| 99 | − AMDGPU_SDMA_MBANK_DATA_BUF10 = 11, | |
| 100 | − AMDGPU_SDMA_MBANK_DATA_BUF11 = 12, | |
| 101 | − AMDGPU_SDMA_MBANK_DATA_BUF12 = 13, | |
| 102 | − AMDGPU_SDMA_MBANK_DATA_BUF13 = 14, | |
| 103 | − AMDGPU_SDMA_MBANK_DATA_BUF14 = 15, | |
| 104 | − AMDGPU_SDMA_MBANK_DATA_BUF15 = 16, | |
| 105 | − AMDGPU_SDMA_UCODE_BUF = 17, | |
| 106 | − AMDGPU_SDMA_RB_CMD_BUF = 18, | |
| 107 | − AMDGPU_SDMA_IB_CMD_BUF = 19, | |
| 108 | − AMDGPU_SDMA_UTCL1_RD_FIFO = 20, | |
| 109 | − AMDGPU_SDMA_UTCL1_RDBST_FIFO = 21, | |
| 110 | − AMDGPU_SDMA_UTCL1_WR_FIFO = 22, | |
| 111 | − AMDGPU_SDMA_DATA_LUT_FIFO = 23, | |
| 112 | − AMDGPU_SDMA_SPLIT_DAT_BUF = 24, | |
| 113 | − AMDGPU_SDMA_MEMORY_BLOCK_LAST, | |
| 114 | −}; | |
| 115 | − | |
| 116 | 88 | struct amdgpu_sdma_ras { |
| 117 | 89 | struct amdgpu_ras_block_object ras_block; |
| 118 | 90 | }; |
| @@ -28,6 +28,8 @@ · 27 unmodified lines | ||
| 28 | 28 | #include <linux/types.h> |
| 29 | 29 | #include <linux/tracepoint.h> |
| 30 | 30 | |
| 31 | +#include "amdgpu_userq_fence.h" | |
| 32 | + | |
| 31 | 33 | #undef TRACE_SYSTEM |
| 32 | 34 | #define TRACE_SYSTEM amdgpu |
| 33 | 35 | #define TRACE_INCLUDE_FILE amdgpu_trace |
| @@ -582,6 +584,154 @@ TRACE_EVENT(amdgpu_reset_reg_dumps, · 548 unmodified lines | ||
| 582 | 584 | __entry->value) |
| 583 | 585 | ); |
| 584 | 586 | |
| 587 | +DECLARE_EVENT_CLASS(amdgpu_userq_queue, | |
| 588 | + TP_PROTO(struct amdgpu_usermode_queue *queue), | |
| 589 | + TP_ARGS(queue), | |
| 590 | + TP_STRUCT__entry( | |
| 591 | + __field(void *, queue) | |
| 592 | + __field(u64, doorbell_index) | |
| 593 | + __field(int, queue_type) | |
| 594 | + __field(int, state) | |
| 595 | + __field(u32, xcp_id) | |
| 596 | + ), | |
| 597 | + TP_fast_assign( | |
| 598 | + __entry->queue = queue; | |
| 599 | + __entry->doorbell_index = queue->doorbell_index; | |
| 600 | + __entry->queue_type = queue->queue_type; | |
| 601 | + __entry->state = queue->state; | |
| 602 | + __entry->xcp_id = queue->xcp_id; | |
| 603 | + ), | |
| 604 | + TP_printk("queue=%p, doorbell=%llu, type=%d, state=%d, xcp_id=%u", | |
| 605 | + __entry->queue, __entry->doorbell_index, | |
| 606 | + __entry->queue_type, __entry->state, __entry->xcp_id) | |
| 607 | +); | |
| 608 | +DEFINE_EVENT(amdgpu_userq_queue, amdgpu_userq_create_start, | |
| 609 | + TP_PROTO(struct amdgpu_usermode_queue *queue), | |
| 610 | + TP_ARGS(queue)); | |
| 611 | +DEFINE_EVENT(amdgpu_userq_queue, amdgpu_userq_destroy_start, | |
| 612 | + TP_PROTO(struct amdgpu_usermode_queue *queue), | |
| 613 | + TP_ARGS(queue)); | |
| 614 | +DECLARE_EVENT_CLASS(amdgpu_userq_queue_result, | |
| 615 | + TP_PROTO(struct amdgpu_usermode_queue *queue, int result), | |
| 616 | + TP_ARGS(queue, result), | |
| 617 | + TP_STRUCT__entry( | |
| 618 | + __field(void *, queue) | |
| 619 | + __field(u64, doorbell_index) | |
| 620 | + __field(int, queue_type) | |
| 621 | + __field(int, state) | |
| 622 | + __field(u32, xcp_id) | |
| 623 | + __field(int, result) | |
| 624 | + ), | |
| 625 | + TP_fast_assign( | |
| 626 | + __entry->queue = queue; | |
| 627 | + __entry->doorbell_index = queue->doorbell_index; | |
| 628 | + __entry->queue_type = queue->queue_type; | |
| 629 | + __entry->state = queue->state; | |
| 630 | + __entry->xcp_id = queue->xcp_id; | |
| 631 | + __entry->result = result; | |
| 632 | + ), | |
| 633 | + TP_printk("queue=%p, doorbell=%llu, type=%d, state=%d, xcp_id=%u, result=%d", | |
| 634 | + __entry->queue, __entry->doorbell_index, | |
| 635 | + __entry->queue_type, __entry->state, | |
| 636 | + __entry->xcp_id, __entry->result) | |
| 637 | +); | |
| 638 | +DEFINE_EVENT(amdgpu_userq_queue_result, amdgpu_userq_create_end, | |
| 639 | + TP_PROTO(struct amdgpu_usermode_queue *queue, int result), | |
| 640 | + TP_ARGS(queue, result)); | |
| 641 | +DEFINE_EVENT(amdgpu_userq_queue_result, amdgpu_userq_destroy_end, | |
| 642 | + TP_PROTO(struct amdgpu_usermode_queue *queue, int result), | |
| 643 | + TP_ARGS(queue, result)); | |
| 644 | + | |
| 645 | +TRACE_EVENT(amdgpu_userq_emit_fence, | |
| 646 | + TP_PROTO(struct device *device, struct amdgpu_usermode_queue *queue, struct amdgpu_userq_fence *fence), | |
| 647 | + TP_ARGS(device, queue, fence), | |
| 648 | + TP_STRUCT__entry( | |
| 649 | + __field(u64, fence_context) | |
| 650 | + __field(u64, fence_seqno) | |
| 651 | + __string(dev, dev_name(device)) | |
| 652 | + __field(u64, doorbell_index) | |
| 653 | + __field(u64, client_id) | |
| 654 | + __field(u32, queue_type) | |
| 655 | + ), | |
| 656 | + TP_fast_assign( | |
| 657 | + __entry->fence_context = fence->base.context; | |
| 658 | + __entry->fence_seqno = fence->base.seqno; | |
| 659 | + __assign_str(dev); | |
| 660 | + __entry->doorbell_index = queue->doorbell_index; | |
| 661 | + __entry->client_id = queue->userq_mgr->file->client_id; | |
| 662 | + __entry->queue_type = queue->queue_type; | |
| 663 | + ), | |
| 664 | + TP_printk("dev=%s, client_id=%llu, type=%u, doorbell=%llu, fence=%llu:%llu", | |
| 665 | + __get_str(dev), __entry->client_id, __entry->queue_type, __entry->doorbell_index, | |
| 666 | + __entry->fence_context, | |
| 667 | + __entry->fence_seqno) | |
| 668 | +); | |
| 669 | + | |
| 670 | +TRACE_EVENT(amdgpu_userq_wait_deps, | |
| 671 | + TP_PROTO(struct device *device, struct amdgpu_usermode_queue *queue, struct amdgpu_userq_fence *dep), | |
| 672 | + TP_ARGS(device, queue, dep), | |
| 673 | + TP_STRUCT__entry( | |
| 674 | + __field(u64, context) | |
| 675 | + __field(u64, dep_context) | |
| 676 | + __field(u64, dep_seqno) | |
| 677 | + __string(dev, dev_name(device)) | |
| 678 | + __field(u64, doorbell_index) | |
| 679 | + __field(u64, client_id) | |
| 680 | + __field(u32, queue_type) | |
| 681 | + ), | |
| 682 | + TP_fast_assign( | |
| 683 | + __assign_str(dev); | |
| 684 | + __entry->doorbell_index = queue->doorbell_index; | |
| 685 | + __entry->queue_type = queue->queue_type; | |
| 686 | + __entry->client_id = queue->userq_mgr->file->client_id; | |
| 687 | + __entry->context = queue->fence_drv->context; | |
| 688 | + __entry->dep_context = dep->base.context; | |
| 689 | + __entry->dep_seqno = dep->base.seqno; | |
| 690 | + ), | |
| 691 | + TP_printk("dev=%s, client_id=%llu, type=%u, doorbell=%llu, context=%llu depends on fence=%llu:%llu", | |
| 692 | + __get_str(dev), __entry->client_id, __entry->queue_type, __entry->doorbell_index, __entry->context, | |
| 693 | + __entry->dep_context, | |
| 694 | + __entry->dep_seqno) | |
| 695 | +); | |
| 696 | + | |
| 697 | +TRACE_EVENT(amdgpu_userq_state_start, | |
| 698 | + TP_PROTO(struct amdgpu_usermode_queue *queue), | |
| 699 | + TP_ARGS(queue), | |
| 700 | + TP_STRUCT__entry( | |
| 701 | + __field(u64, doorbell_index) | |
| 702 | + __field(u64, client_id) | |
| 703 | + __field(u32, queue_type) | |
| 704 | + __field(u32, from) | |
| 705 | + ), | |
| 706 | + TP_fast_assign( | |
| 707 | + __entry->doorbell_index = queue->doorbell_index; | |
| 708 | + __entry->queue_type = queue->queue_type; | |
| 709 | + __entry->client_id = queue->userq_mgr->file->client_id; | |
| 710 | + __entry->from = queue->state; | |
| 711 | + ), | |
| 712 | + TP_printk("client_id=%llu, type=%u, doorbell=%llu, from=%d", | |
| 713 | + __entry->client_id, __entry->queue_type, __entry->doorbell_index, __entry->from) | |
| 714 | +); | |
| 715 | + | |
| 716 | +TRACE_EVENT(amdgpu_userq_state_changed, | |
| 717 | + TP_PROTO(struct amdgpu_usermode_queue *queue, enum amdgpu_userq_state new_state), | |
| 718 | + TP_ARGS(queue, new_state), | |
| 719 | + TP_STRUCT__entry( | |
| 720 | + __field(u64, doorbell_index) | |
| 721 | + __field(u64, client_id) | |
| 722 | + __field(u32, queue_type) | |
| 723 | + __field(u32, to) | |
| 724 | + ), | |
| 725 | + TP_fast_assign( | |
| 726 | + __entry->doorbell_index = queue->doorbell_index; | |
| 727 | + __entry->queue_type = queue->queue_type; | |
| 728 | + __entry->client_id = queue->userq_mgr->file->client_id; | |
| 729 | + __entry->to = new_state; | |
| 730 | + ), | |
| 731 | + TP_printk("client_id=%llu, type=%u, doorbell=%llu, to=%d", | |
| 732 | + __entry->client_id, __entry->queue_type, __entry->doorbell_index, __entry->to) | |
| 733 | +); | |
| 734 | + | |
| 585 | 735 | #undef AMDGPU_JOB_GET_TIMELINE_NAME |
| 586 | 736 | #endif |
| 587 | 737 | |
| @@ -2194,7 +2194,7 @@ int amdgpu_ttm_init(struct amdgpu_device *adev) · 2193 unmodified lines | ||
| 2194 | 2194 | return r; |
| 2195 | 2195 | } |
| 2196 | 2196 | |
| 2197 | − /* Create a boorbell page for kernel usages */ | |
| 2197 | + /* Create a doorbell page for kernel usages */ | |
| 2198 | 2198 | r = amdgpu_doorbell_create_kernel_doorbells(adev); |
| 2199 | 2199 | if (r) { |
| 2200 | 2200 | dev_err(adev->dev, "Failed to initialize kernel doorbells.\n"); |
| @@ -140,6 +140,7 @@ int amdgpu_gtt_mgr_init(struct amdgpu_device *adev, uint64_t gtt_size); · 139 unmodified lines | ||
| 140 | 140 | void amdgpu_gtt_mgr_fini(struct amdgpu_device *adev); |
| 141 | 141 | int amdgpu_preempt_mgr_init(struct amdgpu_device *adev); |
| 142 | 142 | void amdgpu_preempt_mgr_fini(struct amdgpu_device *adev); |
| 143 | +void amdgpu_preempt_mgr_sysfs_fini(struct amdgpu_device *adev); | |
| 143 | 144 | int amdgpu_vram_mgr_init(struct amdgpu_device *adev); |
| 144 | 145 | void amdgpu_vram_mgr_fini(struct amdgpu_device *adev); |
| 145 | 146 | |
| @@ -97,7 +97,6 @@ void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev, · 96 unmodified lines | ||
| 97 | 97 | { |
| 98 | 98 | struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; |
| 99 | 99 | struct amdgpu_ras *con = amdgpu_ras_get_context(adev); |
| 100 | − struct amdgpu_ras_eeprom_control *control = &con->eeprom_control; | |
| 101 | 100 | unsigned int error_query_mode; |
| 102 | 101 | int ret = 0; |
| 103 | 102 | unsigned long err_count; |
| @@ -118,77 +117,66 @@ void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev, · 14 unmodified lines | ||
| 118 | 117 | err_data->err_addr_len = adev->umc.max_ras_err_cnt_per_query; |
| 119 | 118 | |
| 120 | 119 | mutex_lock(&con->page_retirement_lock); |
| 121 | − if (!amdgpu_ras_smu_eeprom_supported(adev)) { | |
| 122 | − ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(con->umc_ecc)); | |
| 123 | − if (ret == -EOPNOTSUPP && | |
| 124 | − error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { | |
| 125 | − if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && | |
| 126 | − adev->umc.ras->ras_block.hw_ops->query_ras_error_count) | |
| 127 | − adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, | |
| 128 | − ras_error_status); | |
| 129 | − | |
| 130 | − if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && | |
| 131 | − adev->umc.ras->ras_block.hw_ops->query_ras_error_address && | |
| 132 | − adev->umc.max_ras_err_cnt_per_query) { | |
| 133 | − kfree(err_data->err_addr); | |
| 134 | − err_data->err_addr = | |
| 135 | − kzalloc_objs(struct eeprom_table_record, | |
| 136 | − adev->umc.max_ras_err_cnt_per_query); | |
| 137 | − | |
| 138 | − /* still call query_ras_error_address to clear error status | |
| 139 | − * even NOMEM error is encountered | |
| 140 | − */ | |
| 141 | − if (!err_data->err_addr) | |
| 142 | − dev_warn(adev->dev, | |
| 143 | − "Failed to alloc memory for umc error address record!\n"); | |
| 144 | − else | |
| 145 | − err_data->err_addr_len = | |
| 146 | − adev->umc.max_ras_err_cnt_per_query; | |
| 147 | − | |
| 148 | − /* umc query_ras_error_address is also responsible for clearing | |
| 149 | − * error status | |
| 150 | − */ | |
| 151 | − adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, | |
| 152 | − ras_error_status); | |
| 153 | − } | |
| 154 | − } else if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY || | |
| 155 | − (!ret && error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY)) { | |
| 156 | − if (adev->umc.ras && | |
| 157 | − adev->umc.ras->ecc_info_query_ras_error_count) | |
| 158 | − adev->umc.ras->ecc_info_query_ras_error_count(adev, | |
| 159 | − ras_error_status); | |
| 160 | − | |
| 161 | − if (adev->umc.ras && | |
| 162 | − adev->umc.ras->ecc_info_query_ras_error_address && | |
| 163 | − adev->umc.max_ras_err_cnt_per_query) { | |
| 164 | − kfree(err_data->err_addr); | |
| 165 | − err_data->err_addr = | |
| 166 | − kzalloc_objs(struct eeprom_table_record, | |
| 167 | − adev->umc.max_ras_err_cnt_per_query); | |
| 168 | − | |
| 169 | − /* still call query_ras_error_address to clear error status | |
| 170 | − * even NOMEM error is encountered | |
| 171 | − */ | |
| 172 | − if (!err_data->err_addr) | |
| 173 | − dev_warn(adev->dev, | |
| 174 | − "Failed to alloc memory for umc error address record!\n"); | |
| 175 | − else | |
| 176 | − err_data->err_addr_len = | |
| 177 | − adev->umc.max_ras_err_cnt_per_query; | |
| 178 | − | |
| 179 | − /* umc query_ras_error_address is also responsible for clearing | |
| 180 | − * error status | |
| 181 | − */ | |
| 182 | − adev->umc.ras->ecc_info_query_ras_error_address(adev, | |
| 183 | − ras_error_status); | |
| 184 | − } | |
| 120 | + ret = amdgpu_dpm_get_ecc_info(adev, (void *)&(con->umc_ecc)); | |
| 121 | + if (ret == -EOPNOTSUPP && | |
| 122 | + error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY) { | |
| 123 | + if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && | |
| 124 | + adev->umc.ras->ras_block.hw_ops->query_ras_error_count) | |
| 125 | + adev->umc.ras->ras_block.hw_ops->query_ras_error_count(adev, | |
| 126 | + ras_error_status); | |
| 127 | + | |
| 128 | + if (adev->umc.ras && adev->umc.ras->ras_block.hw_ops && | |
| 129 | + adev->umc.ras->ras_block.hw_ops->query_ras_error_address && | |
| 130 | + adev->umc.max_ras_err_cnt_per_query) { | |
| 131 | + err_data->err_addr = | |
| 132 | + kzalloc_objs(struct eeprom_table_record, | |
| 133 | + adev->umc.max_ras_err_cnt_per_query); | |
| 134 | + | |
| 135 | + /* still call query_ras_error_address to clear error status | |
| 136 | + * even NOMEM error is encountered | |
| 137 | + */ | |
| 138 | + if (!err_data->err_addr) | |
| 139 | + dev_warn(adev->dev, | |
| 140 | + "Failed to alloc memory for umc error address record!\n"); | |
| 141 | + else | |
| 142 | + err_data->err_addr_len = | |
| 143 | + adev->umc.max_ras_err_cnt_per_query; | |
| 144 | + | |
| 145 | + /* umc query_ras_error_address is also responsible for clearing | |
| 146 | + * error status | |
| 147 | + */ | |
| 148 | + adev->umc.ras->ras_block.hw_ops->query_ras_error_address(adev, | |
| 149 | + ras_error_status); | |
| 185 | 150 | } |
| 186 | − } else { | |
| 187 | − if (!amdgpu_ras_eeprom_update_record_num(control)) { | |
| 188 | − err_data->err_addr_cnt = err_data->de_count = | |
| 189 | − control->ras_num_recs - control->ras_num_recs_old; | |
| 190 | − amdgpu_ras_eeprom_read_idx(control, err_data->err_addr, | |
| 191 | − control->ras_num_recs_old, err_data->de_count); | |
| 151 | + } else if (error_query_mode == AMDGPU_RAS_FIRMWARE_ERROR_QUERY || | |
| 152 | + (!ret && error_query_mode == AMDGPU_RAS_DIRECT_ERROR_QUERY)) { | |
| 153 | + if (adev->umc.ras && | |
| 154 | + adev->umc.ras->ecc_info_query_ras_error_count) | |
| 155 | + adev->umc.ras->ecc_info_query_ras_error_count(adev, | |
| 156 | + ras_error_status); | |
| 157 | + | |
| 158 | + if (adev->umc.ras && | |
| 159 | + adev->umc.ras->ecc_info_query_ras_error_address && | |
| 160 | + adev->umc.max_ras_err_cnt_per_query) { | |
| 161 | + err_data->err_addr = | |
| 162 | + kcalloc(adev->umc.max_ras_err_cnt_per_query, | |
| 163 | + sizeof(struct eeprom_table_record), GFP_KERNEL); | |
| 164 | + | |
| 165 | + /* still call query_ras_error_address to clear error status | |
| 166 | + * even NOMEM error is encountered | |
| 167 | + */ | |
| 168 | + if (!err_data->err_addr) | |
| 169 | + dev_warn(adev->dev, | |
| 170 | + "Failed to alloc memory for umc error address record!\n"); | |
| 171 | + else | |
| 172 | + err_data->err_addr_len = | |
| 173 | + adev->umc.max_ras_err_cnt_per_query; | |
| 174 | + | |
| 175 | + /* umc query_ras_error_address is also responsible for clearing | |
| 176 | + * error status | |
| 177 | + */ | |
| 178 | + adev->umc.ras->ecc_info_query_ras_error_address(adev, | |
| 179 | + ras_error_status); | |
| 192 | 180 | } |
| 193 | 181 | } |
| 194 | 182 | |
| @@ -276,7 +264,7 @@ int amdgpu_umc_pasid_poison_handler(struct amdgpu_device *adev, · 81 unmodified lines | ||
| 276 | 264 | } |
| 277 | 265 | |
| 278 | 266 | amdgpu_ras_error_data_fini(&err_data); |
| 279 | − } else if (amdgpu_uniras_enabled(adev)) { | |
| 267 | + } else { | |
| 280 | 268 | struct ras_ih_info ih_info = {0}; |
| 281 | 269 | |
| 282 | 270 | ih_info.block = block; |
| @@ -285,17 +273,6 @@ int amdgpu_umc_pasid_poison_handler(struct amdgpu_device *adev, · 2 unmodified lines | ||
| 285 | 273 | ih_info.pasid_fn = pasid_fn; |
| 286 | 274 | ih_info.data = data; |
| 287 | 275 | amdgpu_ras_mgr_handle_consumer_interrupt(adev, &ih_info); |
| 288 | − } else { | |
| 289 | − struct amdgpu_ras *con = amdgpu_ras_get_context(adev); | |
| 290 | − int ret; | |
| 291 | − | |
| 292 | − ret = amdgpu_ras_put_poison_req(adev, | |
| 293 | − block, pasid, pasid_fn, data, reset); | |
| 294 | − if (!ret) { | |
| 295 | − atomic_inc(&con->page_retirement_req_cnt); | |
| 296 | − atomic_inc(&con->poison_consumption_count); | |
| 297 | − wake_up(&con->page_retirement_wq); | |
| 298 | − } | |
| 299 | 276 | } |
| 300 | 277 | } else { |
| 301 | 278 | if (adev->virt.ops && adev->virt.ops->ras_poison_handler) |
| @@ -512,129 +489,3 @@ int amdgpu_umc_loop_channels(struct amdgpu_device *adev, · 210 unmodified lines | ||
| 512 | 489 | |
| 513 | 490 | return 0; |
| 514 | 491 | } |
| 515 | − | |
| 516 | −int amdgpu_umc_update_ecc_status(struct amdgpu_device *adev, | |
| 517 | − uint64_t status, uint64_t ipid, uint64_t addr) | |
| 518 | −{ | |
| 519 | − if (adev->umc.ras->update_ecc_status) | |
| 520 | − return adev->umc.ras->update_ecc_status(adev, | |
| 521 | − status, ipid, addr); | |
| 522 | − return 0; | |
| 523 | −} | |
| 524 | − | |
| 525 | −int amdgpu_umc_logs_ecc_err(struct amdgpu_device *adev, | |
| 526 | − struct radix_tree_root *ecc_tree, struct ras_ecc_err *ecc_err) | |
| 527 | −{ | |
| 528 | − struct amdgpu_ras *con = amdgpu_ras_get_context(adev); | |
| 529 | − struct ras_ecc_log_info *ecc_log; | |
| 530 | − int ret; | |
| 531 | − | |
| 532 | − ecc_log = &con->umc_ecc_log; | |
| 533 | − | |
| 534 | − mutex_lock(&ecc_log->lock); | |
| 535 | − ret = radix_tree_insert(ecc_tree, ecc_err->pa_pfn, ecc_err); | |
| 536 | − if (!ret) | |
| 537 | − radix_tree_tag_set(ecc_tree, | |
| 538 | − ecc_err->pa_pfn, UMC_ECC_NEW_DETECTED_TAG); | |
| 539 | − mutex_unlock(&ecc_log->lock); | |
| 540 | − | |
| 541 | − return ret; | |
| 542 | −} | |
| 543 | − | |
| 544 | −int amdgpu_umc_pages_in_a_row(struct amdgpu_device *adev, | |
| 545 | − struct ras_err_data *err_data, uint64_t pa_addr) | |
| 546 | −{ | |
| 547 | − struct ta_ras_query_address_output addr_out; | |
| 548 | − | |
| 549 | − /* reinit err_data */ | |
| 550 | − err_data->err_addr_cnt = 0; | |
| 551 | − err_data->err_addr_len = adev->umc.retire_unit; | |
| 552 | − | |
| 553 | − addr_out.pa.pa = pa_addr; | |
| 554 | − if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) | |
| 555 | − return adev->umc.ras->convert_ras_err_addr(adev, err_data, NULL, | |
| 556 | − &addr_out, false); | |
| 557 | − else | |
| 558 | − return -EINVAL; | |
| 559 | −} | |
| 560 | − | |
| 561 | −int amdgpu_umc_lookup_bad_pages_in_a_row(struct amdgpu_device *adev, | |
| 562 | − uint64_t pa_addr, uint64_t *pfns, int len) | |
| 563 | −{ | |
| 564 | − int i, ret; | |
| 565 | − struct ras_err_data err_data; | |
| 566 | − | |
| 567 | − err_data.err_addr = kzalloc_objs(struct eeprom_table_record, | |
| 568 | − adev->umc.retire_unit); | |
| 569 | − if (!err_data.err_addr) { | |
| 570 | − dev_warn(adev->dev, "Failed to alloc memory in bad page lookup!\n"); | |
| 571 | − return 0; | |
| 572 | − } | |
| 573 | − | |
| 574 | − ret = amdgpu_umc_pages_in_a_row(adev, &err_data, pa_addr); | |
| 575 | − if (ret) | |
| 576 | − goto out; | |
| 577 | − | |
| 578 | − for (i = 0; i < adev->umc.retire_unit; i++) { | |
| 579 | − if (i >= len) | |
| 580 | − goto out; | |
| 581 | − | |
| 582 | − pfns[i] = err_data.err_addr[i].retired_page; | |
| 583 | − } | |
| 584 | − ret = i; | |
| 585 | − adev->umc.err_addr_cnt = err_data.err_addr_cnt; | |
| 586 | − | |
| 587 | −out: | |
| 588 | − kfree(err_data.err_addr); | |
| 589 | − return ret; | |
| 590 | −} | |
| 591 | − | |
| 592 | −int amdgpu_umc_mca_to_addr(struct amdgpu_device *adev, | |
| 593 | − uint64_t err_addr, uint32_t ch, uint32_t umc, | |
| 594 | − uint32_t node, uint32_t socket, | |
| 595 | − struct ta_ras_query_address_output *addr_out, bool dump_addr) | |
| 596 | −{ | |
| 597 | − struct ta_ras_query_address_input addr_in; | |
| 598 | − int ret; | |
| 599 | − | |
| 600 | − memset(&addr_in, 0, sizeof(addr_in)); | |
| 601 | − addr_in.ma.err_addr = err_addr; | |
| 602 | − addr_in.ma.ch_inst = ch; | |
| 603 | − addr_in.ma.umc_inst = umc; | |
| 604 | − addr_in.ma.node_inst = node; | |
| 605 | − addr_in.ma.socket_id = socket; | |
| 606 | − | |
| 607 | − if (adev->umc.ras && adev->umc.ras->convert_ras_err_addr) { | |
| 608 | − ret = adev->umc.ras->convert_ras_err_addr(adev, NULL, &addr_in, | |
| 609 | − addr_out, dump_addr); | |
| 610 | − if (ret) | |
| 611 | − return ret; | |
| 612 | − } else { | |
| 613 | − return 0; | |
| 614 | − } | |
| 615 | − | |
| 616 | − return 0; | |
| 617 | −} | |
| 618 | − | |
| 619 | −int amdgpu_umc_pa2mca(struct amdgpu_device *adev, | |
| 620 | − uint64_t pa, uint64_t *mca, enum amdgpu_memory_partition nps) | |
| 621 | −{ | |
| 622 | − struct ta_ras_query_address_input addr_in; | |
| 623 | − struct ta_ras_query_address_output addr_out; | |
| 624 | − int ret; | |
| 625 | − | |
| 626 | − /* nps: the pa belongs to */ | |
| 627 | − addr_in.pa.pa = pa | ((uint64_t)nps << 58); | |
| 628 | − addr_in.addr_type = TA_RAS_PA_TO_MCA; | |
| 629 | − ret = psp_ras_query_address(&adev->psp, &addr_in, &addr_out); | |
| 630 | − if (ret) { | |
| 631 | − dev_warn(adev->dev, "Failed to query RAS MCA address for 0x%llx", | |
| 632 | − pa); | |
| 633 | − | |
| 634 | − return ret; | |
| 635 | − } | |
| 636 | − | |
| 637 | − *mca = addr_out.ma.err_addr; | |
| 638 | − | |
| 639 | − return 0; | |
| 640 | −} | |
| @@ -103,18 +103,7 @@ struct amdgpu_umc_ras { · 102 unmodified lines | ||
| 103 | 103 | void *ras_error_status); |
| 104 | 104 | bool (*check_ecc_err_status)(struct amdgpu_device *adev, |
| 105 | 105 | enum amdgpu_mca_error_type type, void *ras_error_status); |
| 106 | − int (*update_ecc_status)(struct amdgpu_device *adev, | |
| 107 | − uint64_t status, uint64_t ipid, uint64_t addr); | |
| 108 | − int (*convert_ras_err_addr)(struct amdgpu_device *adev, | |
| 109 | − struct ras_err_data *err_data, | |
| 110 | − struct ta_ras_query_address_input *addr_in, | |
| 111 | − struct ta_ras_query_address_output *addr_out, | |
| 112 | − bool dump_addr); | |
| 113 | − uint32_t (*get_die_id_from_pa)(struct amdgpu_device *adev, | |
| 114 | − uint64_t mca_addr, uint64_t retired_page); | |
| 115 | 106 | void (*get_retire_flip_bits)(struct amdgpu_device *adev); |
| 116 | − void (*mca_ipid_parse)(struct amdgpu_device *adev, uint64_t ipid, | |
| 117 | − uint32_t *did, uint32_t *ch, uint32_t *umc_inst, uint32_t *sid); | |
| 118 | 107 | }; |
| 119 | 108 | |
| 120 | 109 | struct amdgpu_umc_funcs { |
| @@ -179,21 +168,6 @@ int amdgpu_umc_page_retirement_mca(struct amdgpu_device *adev, · 58 unmodified lines | ||
| 179 | 168 | int amdgpu_umc_loop_channels(struct amdgpu_device *adev, |
| 180 | 169 | umc_func func, void *data); |
| 181 | 170 | |
| 182 | −int amdgpu_umc_update_ecc_status(struct amdgpu_device *adev, | |
| 183 | − uint64_t status, uint64_t ipid, uint64_t addr); | |
| 184 | −int amdgpu_umc_logs_ecc_err(struct amdgpu_device *adev, | |
| 185 | − struct radix_tree_root *ecc_tree, struct ras_ecc_err *ecc_err); | |
| 186 | − | |
| 187 | 171 | void amdgpu_umc_handle_bad_pages(struct amdgpu_device *adev, |
| 188 | 172 | void *ras_error_status); |
| 189 | −int amdgpu_umc_pages_in_a_row(struct amdgpu_device *adev, | |
| 190 | − struct ras_err_data *err_data, uint64_t pa_addr); | |
| 191 | −int amdgpu_umc_lookup_bad_pages_in_a_row(struct amdgpu_device *adev, | |
| 192 | − uint64_t pa_addr, uint64_t *pfns, int len); | |
| 193 | −int amdgpu_umc_mca_to_addr(struct amdgpu_device *adev, | |
| 194 | − uint64_t err_addr, uint32_t ch, uint32_t umc, | |
| 195 | − uint32_t node, uint32_t socket, | |
| 196 | − struct ta_ras_query_address_output *addr_out, bool dump_addr); | |
| 197 | −int amdgpu_umc_pa2mca(struct amdgpu_device *adev, | |
| 198 | − uint64_t pa, uint64_t *mca, enum amdgpu_memory_partition nps); | |
| 199 | 173 | #endif |
| @@ -33,6 +33,7 @@ · 32 unmodified lines | ||
| 33 | 33 | #include "amdgpu_userq.h" |
| 34 | 34 | #include "amdgpu_hmm.h" |
| 35 | 35 | #include "amdgpu_userq_fence.h" |
| 36 | +#include "amdgpu_trace.h" | |
| 36 | 37 | |
| 37 | 38 | u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev) |
| 38 | 39 | { |
| @@ -88,14 +89,7 @@ static void amdgpu_userq_mgr_reset_work(struct work_struct *work) · 49 unmodified lines | ||
| 88 | 89 | container_of(work, struct amdgpu_userq_mgr, |
| 89 | 90 | reset_work); |
| 90 | 91 | struct amdgpu_device *adev = uq_mgr->adev; |
| 91 | − const int queue_types[] = { | |
| 92 | − AMDGPU_RING_TYPE_COMPUTE, | |
| 93 | − AMDGPU_RING_TYPE_GFX, | |
| 94 | − AMDGPU_RING_TYPE_SDMA | |
| 95 | − }; | |
| 96 | − const int num_queue_types = ARRAY_SIZE(queue_types); | |
| 97 | − bool gpu_reset = false; | |
| 98 | − int i, r; | |
| 92 | + struct amdgpu_reset_context reset_context; | |
| 99 | 93 | |
| 100 | 94 | if (unlikely(adev->debug_disable_gpu_ring_reset)) { |
| 101 | 95 | dev_err(adev->dev, "userq reset disabled by debug mask\n"); |
| @@ -109,42 +103,15 @@ static void amdgpu_userq_mgr_reset_work(struct work_struct *work) · 7 unmodified lines | ||
| 109 | 103 | if (!amdgpu_gpu_recovery) |
| 110 | 104 | return; |
| 111 | 105 | |
| 112 | − /* | |
| 113 | − * Iterate through all queue types to detect and reset problematic queues | |
| 114 | − * Process each queue type in the defined order | |
| 115 | − */ | |
| 116 | − for (i = 0; i < num_queue_types; i++) { | |
| 117 | − int ring_type = queue_types[i]; | |
| 118 | − const struct amdgpu_userq_funcs *funcs = | |
| 119 | − adev->userq_funcs[ring_type]; | |
| 120 | − | |
| 121 | − if (!amdgpu_userq_is_reset_type_supported(adev, ring_type, | |
| 122 | − AMDGPU_RESET_TYPE_PER_QUEUE)) | |
| 123 | − continue; | |
| 106 | + memset(&reset_context, 0, sizeof(reset_context)); | |
| 124 | 107 | |
| 125 | − if (atomic_read(&uq_mgr->userq_count[ring_type]) > 0 && | |
| 126 | − funcs && funcs->detect_and_reset) { | |
| 127 | − r = funcs->detect_and_reset(adev, ring_type); | |
| 128 | − if (r) { | |
| 129 | − gpu_reset = true; | |
| 130 | − break; | |
| 131 | − } | |
| 132 | − } | |
| 133 | − } | |
| 108 | + reset_context.method = AMD_RESET_METHOD_NONE; | |
| 109 | + reset_context.reset_req_dev = adev; | |
| 110 | + reset_context.src = AMDGPU_RESET_SRC_USERQ; | |
| 111 | + set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); | |
| 112 | + /*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/ | |
| 134 | 113 | |
| 135 | − if (gpu_reset) { | |
| 136 | − struct amdgpu_reset_context reset_context; | |
| 137 | − | |
| 138 | − memset(&reset_context, 0, sizeof(reset_context)); | |
| 139 | − | |
| 140 | − reset_context.method = AMD_RESET_METHOD_NONE; | |
| 141 | − reset_context.reset_req_dev = adev; | |
| 142 | − reset_context.src = AMDGPU_RESET_SRC_USERQ; | |
| 143 | − set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); | |
| 144 | − /*set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);*/ | |
| 145 | − | |
| 146 | − amdgpu_device_gpu_recover(adev, NULL, &reset_context); | |
| 147 | − } | |
| 114 | + amdgpu_device_gpu_recover(adev, NULL, &reset_context); | |
| 148 | 115 | } |
| 149 | 116 | |
| 150 | 117 | static void amdgpu_userq_hang_detect_work(struct work_struct *work) |
| @@ -152,12 +119,45 @@ static void amdgpu_userq_hang_detect_work(struct work_struct *work) · 1 unmodified line | ||
| 152 | 119 | struct amdgpu_usermode_queue *queue = |
| 153 | 120 | container_of(work, struct amdgpu_usermode_queue, |
| 154 | 121 | hang_detect_work.work); |
| 122 | + struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; | |
| 123 | + struct amdgpu_device *adev = uq_mgr->adev; | |
| 124 | + const struct amdgpu_userq_funcs *userq_funcs = | |
| 125 | + adev->userq_funcs[queue->queue_type]; | |
| 126 | + bool gpu_reset = false; | |
| 127 | + | |
| 128 | + if (unlikely(adev->debug_disable_gpu_ring_reset)) { | |
| 129 | + dev_err(adev->dev, "userq reset disabled by debug mask\n"); | |
| 130 | + return; | |
| 131 | + } | |
| 132 | + | |
| 133 | + /* | |
| 134 | + * If GPU recovery feature is disabled system-wide, | |
| 135 | + * skip all reset detection logic | |
| 136 | + */ | |
| 137 | + if (!amdgpu_gpu_recovery) | |
| 138 | + return; | |
| 139 | + | |
| 140 | + if (amdgpu_userq_is_reset_type_supported(adev, queue->queue_type, | |
| 141 | + AMDGPU_RESET_TYPE_PER_QUEUE)) { | |
| 142 | + int r; | |
| 143 | + | |
| 144 | + if (queue->queue_type == AMDGPU_HW_IP_COMPUTE) | |
| 145 | + r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL, | |
| 146 | + queue, NULL, NULL); | |
| 147 | + else | |
| 148 | + r = userq_funcs->reset(queue); | |
| 149 | + if (r) | |
| 150 | + gpu_reset = true; | |
| 151 | + } else { | |
| 152 | + gpu_reset = true; | |
| 153 | + } | |
| 155 | 154 | |
| 156 | 155 | /* |
| 157 | 156 | * Don't schedule the work here! Scheduling or queue work from one reset |
| 158 | 157 | * handler to another is illegal if you don't take extra precautions! |
| 159 | 158 | */ |
| 160 | − amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work); | |
| 159 | + if (gpu_reset) | |
| 160 | + amdgpu_userq_mgr_reset_work(&queue->userq_mgr->reset_work); | |
| 161 | 161 | } |
| 162 | 162 | |
| 163 | 163 | /* |
| @@ -293,11 +293,15 @@ static int amdgpu_userq_preempt_helper(struct amdgpu_usermode_queue *queue) · 129 unmodified lines | ||
| 293 | 293 | int r; |
| 294 | 294 | |
| 295 | 295 | if (queue->state == AMDGPU_USERQ_STATE_MAPPED) { |
| 296 | + trace_amdgpu_userq_state_start(queue); | |
| 297 | + | |
| 296 | 298 | r = userq_funcs->preempt(queue); |
| 297 | 299 | if (r) { |
| 300 | + trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG); | |
| 298 | 301 | queue->state = AMDGPU_USERQ_STATE_HUNG; |
| 299 | 302 | return r; |
| 300 | 303 | } else { |
| 304 | + trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_PREEMPTED); | |
| 301 | 305 | queue->state = AMDGPU_USERQ_STATE_PREEMPTED; |
| 302 | 306 | } |
| 303 | 307 | } |
| @@ -313,10 +317,14 @@ static int amdgpu_userq_restore_helper(struct amdgpu_usermode_queue *queue) · 9 unmodified lines | ||
| 313 | 317 | int r = 0; |
| 314 | 318 | |
| 315 | 319 | if (queue->state == AMDGPU_USERQ_STATE_PREEMPTED) { |
| 320 | + trace_amdgpu_userq_state_start(queue); | |
| 321 | + | |
| 316 | 322 | r = userq_funcs->restore(queue); |
| 317 | 323 | if (r) { |
| 324 | + trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG); | |
| 318 | 325 | queue->state = AMDGPU_USERQ_STATE_HUNG; |
| 319 | 326 | } else { |
| 327 | + trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED); | |
| 320 | 328 | queue->state = AMDGPU_USERQ_STATE_MAPPED; |
| 321 | 329 | } |
| 322 | 330 | } |
| @@ -334,12 +342,15 @@ static int amdgpu_userq_unmap_helper(struct amdgpu_usermode_queue *queue) · 11 unmodified lines | ||
| 334 | 342 | |
| 335 | 343 | if ((queue->state == AMDGPU_USERQ_STATE_MAPPED) || |
| 336 | 344 | (queue->state == AMDGPU_USERQ_STATE_PREEMPTED)) { |
| 345 | + trace_amdgpu_userq_state_start(queue); | |
| 337 | 346 | |
| 338 | 347 | r = userq_funcs->unmap(queue); |
| 339 | 348 | if (r) { |
| 349 | + trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG); | |
| 340 | 350 | queue->state = AMDGPU_USERQ_STATE_HUNG; |
| 341 | 351 | return r; |
| 342 | 352 | } else { |
| 353 | + trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_UNMAPPED); | |
| 343 | 354 | queue->state = AMDGPU_USERQ_STATE_UNMAPPED; |
| 344 | 355 | } |
| 345 | 356 | } |
| @@ -356,11 +367,15 @@ static int amdgpu_userq_map_helper(struct amdgpu_usermode_queue *queue) · 10 unmodified lines | ||
| 356 | 367 | int r; |
| 357 | 368 | |
| 358 | 369 | if (queue->state == AMDGPU_USERQ_STATE_UNMAPPED) { |
| 370 | + trace_amdgpu_userq_state_start(queue); | |
| 371 | + | |
| 359 | 372 | r = userq_funcs->map(queue); |
| 360 | 373 | if (r) { |
| 374 | + trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG); | |
| 361 | 375 | queue->state = AMDGPU_USERQ_STATE_HUNG; |
| 362 | 376 | return r; |
| 363 | 377 | } else { |
| 378 | + trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED); | |
| 364 | 379 | queue->state = AMDGPU_USERQ_STATE_MAPPED; |
| 365 | 380 | } |
| 366 | 381 | } |
| @@ -507,6 +522,8 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que · 140 unmodified lines | ||
| 507 | 522 | const struct amdgpu_userq_funcs *uq_funcs = adev->userq_funcs[queue->queue_type]; |
| 508 | 523 | int r = 0; |
| 509 | 524 | |
| 525 | + trace_amdgpu_userq_destroy_start(queue); | |
| 526 | + | |
| 510 | 527 | cancel_delayed_work_sync(&uq_mgr->resume_work); |
| 511 | 528 | |
| 512 | 529 | /* Cancel any pending hang detection work and cleanup */ |
| @@ -532,6 +549,7 @@ amdgpu_userq_destroy(struct amdgpu_userq_mgr *uq_mgr, struct amdgpu_usermode_que · 19 unmodified lines | ||
| 532 | 549 | amdgpu_bo_unreserve(queue->db_obj.obj); |
| 533 | 550 | amdgpu_bo_unref(&queue->db_obj.obj); |
| 534 | 551 | |
| 552 | + trace_amdgpu_userq_destroy_end(queue, r); | |
| 535 | 553 | kfree(queue); |
| 536 | 554 | |
| 537 | 555 | pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); |
| @@ -629,6 +647,8 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) · 91 unmodified lines | ||
| 629 | 647 | queue->queue_type = args->in.ip_type; |
| 630 | 648 | queue->vm = &fpriv->vm; |
| 631 | 649 | queue->priority = priority; |
| 650 | + queue->xcp_id = (fpriv->xcp_id != AMDGPU_XCP_NO_PARTITION) ? | |
| 651 | + fpriv->xcp_id : 0; | |
| 632 | 652 | queue->userq_mgr = uq_mgr; |
| 633 | 653 | INIT_DELAYED_WORK(&queue->hang_detect_work, |
| 634 | 654 | amdgpu_userq_hang_detect_work); |
| @@ -671,6 +691,8 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) · 36 unmodified lines | ||
| 671 | 691 | } |
| 672 | 692 | |
| 673 | 693 | queue->doorbell_index = index; |
| 694 | + queue->doorbell_offset = (u32)args->in.doorbell_offset; | |
| 695 | + trace_amdgpu_userq_create_start(queue); | |
| 674 | 696 | r = uq_funcs->mqd_create(queue, &args->in); |
| 675 | 697 | if (r) { |
| 676 | 698 | drm_file_err(uq_mgr->file, "Failed to create Queue\n"); |
| @@ -694,6 +716,7 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) · 17 unmodified lines | ||
| 694 | 716 | r = amdgpu_userq_map_helper(queue); |
| 695 | 717 | if (r) { |
| 696 | 718 | drm_file_err(uq_mgr->file, "Failed to map Queue\n"); |
| 719 | + trace_amdgpu_userq_create_end(queue, r); | |
| 697 | 720 | mutex_unlock(&uq_mgr->userq_mutex); |
| 698 | 721 | goto erase_doorbell; |
| 699 | 722 | } |
| @@ -710,11 +733,13 @@ amdgpu_userq_create(struct drm_file *filp, union drm_amdgpu_userq *args) · 10 unmodified lines | ||
| 710 | 733 | * This drops the last reference which should take care of |
| 711 | 734 | * all cleanup. |
| 712 | 735 | */ |
| 736 | + trace_amdgpu_userq_create_end(queue, r); | |
| 713 | 737 | amdgpu_userq_put(queue); |
| 714 | 738 | return r; |
| 715 | 739 | } |
| 716 | 740 | |
| 717 | 741 | amdgpu_debugfs_userq_init(filp, queue, qid); |
| 742 | + trace_amdgpu_userq_create_end(queue, 0); | |
| 718 | 743 | args->out.queue_id = qid; |
| 719 | 744 | return 0; |
| 720 | 745 | |
| @@ -730,6 +755,7 @@ clean_doorbell_bo: · 9 unmodified lines | ||
| 730 | 755 | free_fence_drv: |
| 731 | 756 | amdgpu_userq_fence_driver_free(queue); |
| 732 | 757 | free_queue: |
| 758 | + trace_amdgpu_userq_create_end(queue, r); | |
| 733 | 759 | kfree(queue); |
| 734 | 760 | err_pm_runtime: |
| 735 | 761 | pm_runtime_put_autosuspend(adev_to_drm(adev)->dev); |
| @@ -862,16 +888,10 @@ int amdgpu_userq_ioctl(struct drm_device *dev, void *data, · 126 unmodified lines | ||
| 862 | 888 | static int |
| 863 | 889 | amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr) |
| 864 | 890 | { |
| 865 | − struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); | |
| 866 | − struct amdgpu_vm *vm = &fpriv->vm; | |
| 867 | 891 | struct amdgpu_usermode_queue *queue; |
| 868 | 892 | unsigned long queue_id; |
| 869 | 893 | int ret = 0, r; |
| 870 | 894 | |
| 871 | − | |
| 872 | − if (amdgpu_bo_reserve(vm->root.bo, false)) | |
| 873 | − return false; | |
| 874 | − | |
| 875 | 895 | mutex_lock(&uq_mgr->userq_mutex); |
| 876 | 896 | /* Resume all the queues for this process */ |
| 877 | 897 | xa_for_each(&uq_mgr->userq_xa, queue_id, queue) { |
| @@ -879,6 +899,7 @@ amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr) · 1 unmodified line | ||
| 879 | 899 | if (!amdgpu_userq_buffer_vas_mapped(queue)) { |
| 880 | 900 | drm_file_err(uq_mgr->file, |
| 881 | 901 | "trying restore queue without va mapping\n"); |
| 902 | + trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_INVALID_VA); | |
| 882 | 903 | queue->state = AMDGPU_USERQ_STATE_INVALID_VA; |
| 883 | 904 | continue; |
| 884 | 905 | } |
| @@ -886,10 +907,8 @@ amdgpu_userq_restore_all(struct amdgpu_userq_mgr *uq_mgr) · 1 unmodified line | ||
| 886 | 907 | r = amdgpu_userq_map_helper(queue); |
| 887 | 908 | if (r) |
| 888 | 909 | ret = r; |
| 889 | − | |
| 890 | 910 | } |
| 891 | 911 | mutex_unlock(&uq_mgr->userq_mutex); |
| 892 | − amdgpu_bo_unreserve(vm->root.bo); | |
| 893 | 912 | |
| 894 | 913 | if (ret) |
| 895 | 914 | drm_file_err(uq_mgr->file, |
| @@ -923,7 +942,8 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec, · 27 unmodified lines | ||
| 923 | 942 | spin_unlock(&vm->individual_lock); |
| 924 | 943 | |
| 925 | 944 | bo = bo_va->base.bo; |
| 926 | − ret = drm_exec_prepare_obj(exec, &bo->tbo.base, 2); | |
| 945 | + ret = drm_exec_prepare_obj(exec, &bo->tbo.base, | |
| 946 | + TTM_NUM_MOVE_FENCES + 1); | |
| 927 | 947 | if (unlikely(ret)) |
| 928 | 948 | return ret; |
| 929 | 949 | |
| @@ -946,7 +966,7 @@ amdgpu_userq_bo_validate(struct amdgpu_device *adev, struct drm_exec *exec, · 16 unmodified lines | ||
| 946 | 966 | |
| 947 | 967 | /* Make sure the whole VM is ready to be used */ |
| 948 | 968 | static int |
| 949 | −amdgpu_userq_vm_validate(struct amdgpu_userq_mgr *uq_mgr) | |
| 969 | +amdgpu_userq_vm_validate_and_restore_queue(struct amdgpu_userq_mgr *uq_mgr) | |
| 950 | 970 | { |
| 951 | 971 | struct amdgpu_fpriv *fpriv = uq_mgr_to_fpriv(uq_mgr); |
| 952 | 972 | bool invalidated = false, new_addition = false; |
| @@ -1072,8 +1092,12 @@ retry_lock: · 119 unmodified lines | ||
| 1072 | 1092 | dma_fence_wait(vm->last_update, false); |
| 1073 | 1093 | |
| 1074 | 1094 | ret = amdgpu_evf_mgr_rearm(&fpriv->evf_mgr, &exec); |
| 1075 | − if (ret) | |
| 1095 | + if (ret) { | |
| 1076 | 1096 | drm_file_err(uq_mgr->file, "Failed to replace eviction fence\n"); |
| 1097 | + goto unlock_all; | |
| 1098 | + } | |
| 1099 | + | |
| 1100 | + ret = amdgpu_userq_restore_all(uq_mgr); | |
| 1077 | 1101 | |
| 1078 | 1102 | unlock_all: |
| 1079 | 1103 | drm_exec_fini(&exec); |
| @@ -1099,18 +1123,34 @@ static void amdgpu_userq_restore_worker(struct work_struct *work) · 19 unmodified lines | ||
| 1099 | 1123 | if (!dma_fence_is_signaled(ev_fence)) |
| 1100 | 1124 | goto put_fence; |
| 1101 | 1125 | |
| 1102 | − ret = amdgpu_userq_vm_validate(uq_mgr); | |
| 1126 | + ret = amdgpu_userq_vm_validate_and_restore_queue(uq_mgr); | |
| 1103 | 1127 | if (ret) { |
| 1104 | 1128 | drm_file_err(uq_mgr->file, "Failed to validate BOs to restore ret=%d\n", ret); |
| 1105 | 1129 | goto put_fence; |
| 1106 | 1130 | } |
| 1107 | 1131 | |
| 1108 | − amdgpu_userq_restore_all(uq_mgr); | |
| 1109 | − | |
| 1110 | 1132 | put_fence: |
| 1111 | 1133 | dma_fence_put(ev_fence); |
| 1112 | 1134 | } |
| 1113 | 1135 | |
| 1136 | +void amdgpu_userq_process_reset_irq(struct amdgpu_device *adev, | |
| 1137 | + u32 pasid, u32 doorbell_offset) | |
| 1138 | +{ | |
| 1139 | + struct xarray *xa = &adev->userq_doorbell_xa; | |
| 1140 | + struct amdgpu_usermode_queue *queue; | |
| 1141 | + unsigned long flags, idx; | |
| 1142 | + | |
| 1143 | + xa_lock_irqsave(xa, flags); | |
| 1144 | + xa_for_each(xa, idx, queue) { | |
| 1145 | + if (queue->vm && queue->vm->pasid == pasid && | |
| 1146 | + queue->doorbell_offset == doorbell_offset) { | |
| 1147 | + amdgpu_userq_start_hang_detect_work(queue); | |
| 1148 | + break; | |
| 1149 | + } | |
| 1150 | + } | |
| 1151 | + xa_unlock_irqrestore(xa, flags); | |
| 1152 | +} | |
| 1153 | + | |
| 1114 | 1154 | static int |
| 1115 | 1155 | amdgpu_userq_evict_all(struct amdgpu_userq_mgr *uq_mgr) |
| 1116 | 1156 | { |
| @@ -1166,6 +1206,7 @@ int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *f · 49 unmodified lines | ||
| 1166 | 1206 | xa_init_flags(&userq_mgr->userq_xa, XA_FLAGS_ALLOC); |
| 1167 | 1207 | userq_mgr->adev = adev; |
| 1168 | 1208 | userq_mgr->file = file_priv; |
| 1209 | + mutex_init(&userq_mgr->proc_ctx_lock); | |
| 1169 | 1210 | |
| 1170 | 1211 | INIT_DELAYED_WORK(&userq_mgr->resume_work, amdgpu_userq_restore_worker); |
| 1171 | 1212 | INIT_WORK(&userq_mgr->reset_work, amdgpu_userq_mgr_reset_work); |
| @@ -1219,6 +1260,11 @@ void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr) · 47 unmodified lines | ||
| 1219 | 1260 | */ |
| 1220 | 1261 | cancel_work_sync(&userq_mgr->reset_work); |
| 1221 | 1262 | |
| 1263 | + amdgpu_bo_free_kernel(&userq_mgr->proc_ctx_obj.obj, | |
| 1264 | + &userq_mgr->proc_ctx_obj.gpu_addr, | |
| 1265 | + &userq_mgr->proc_ctx_obj.cpu_ptr); | |
| 1266 | + | |
| 1267 | + mutex_destroy(&userq_mgr->proc_ctx_lock); | |
| 1222 | 1268 | mutex_destroy(&userq_mgr->userq_mutex); |
| 1223 | 1269 | } |
| 1224 | 1270 | |
| @@ -1370,12 +1416,14 @@ void amdgpu_userq_pre_reset(struct amdgpu_device *adev) · 145 unmodified lines | ||
| 1370 | 1416 | if (queue->state != AMDGPU_USERQ_STATE_MAPPED) |
| 1371 | 1417 | continue; |
| 1372 | 1418 | |
| 1419 | + trace_amdgpu_userq_state_start(queue); | |
| 1373 | 1420 | userq_funcs = adev->userq_funcs[queue->queue_type]; |
| 1374 | 1421 | userq_funcs->unmap(queue); |
| 1375 | 1422 | /* just mark all queues as hung at this point. |
| 1376 | 1423 | * if unmap succeeds, we could map again |
| 1377 | 1424 | * in amdgpu_userq_post_reset() if vram is not lost |
| 1378 | 1425 | */ |
| 1426 | + trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_HUNG); | |
| 1379 | 1427 | queue->state = AMDGPU_USERQ_STATE_HUNG; |
| 1380 | 1428 | amdgpu_userq_fence_driver_force_completion(queue); |
| 1381 | 1429 | } |
| @@ -1394,6 +1442,8 @@ int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost) · 12 unmodified lines | ||
| 1394 | 1442 | |
| 1395 | 1443 | xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { |
| 1396 | 1444 | if (queue->state == AMDGPU_USERQ_STATE_HUNG && !vram_lost) { |
| 1445 | + trace_amdgpu_userq_state_start(queue); | |
| 1446 | + | |
| 1397 | 1447 | userq_funcs = adev->userq_funcs[queue->queue_type]; |
| 1398 | 1448 | /* Re-map queue */ |
| 1399 | 1449 | r = userq_funcs->map(queue); |
| @@ -1401,6 +1451,7 @@ int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost) · 1 unmodified line | ||
| 1401 | 1451 | dev_err(adev->dev, "Failed to remap queue %ld\n", queue_id); |
| 1402 | 1452 | continue; |
| 1403 | 1453 | } |
| 1454 | + trace_amdgpu_userq_state_changed(queue, AMDGPU_USERQ_STATE_MAPPED); | |
| 1404 | 1455 | queue->state = AMDGPU_USERQ_STATE_MAPPED; |
| 1405 | 1456 | } |
| 1406 | 1457 | } |
| @@ -53,6 +53,7 @@ struct amdgpu_usermode_queue { · 52 unmodified lines | ||
| 53 | 53 | enum amdgpu_userq_state state; |
| 54 | 54 | uint64_t doorbell_handle; |
| 55 | 55 | uint64_t doorbell_index; |
| 56 | + u32 doorbell_offset; | |
| 56 | 57 | uint64_t flags; |
| 57 | 58 | struct amdgpu_mqd_prop *userq_prop; |
| 58 | 59 | struct amdgpu_userq_mgr *userq_mgr; |
| @@ -111,8 +112,7 @@ struct amdgpu_userq_funcs { · 52 unmodified lines | ||
| 111 | 112 | int (*map)(struct amdgpu_usermode_queue *queue); |
| 112 | 113 | int (*preempt)(struct amdgpu_usermode_queue *queue); |
| 113 | 114 | int (*restore)(struct amdgpu_usermode_queue *queue); |
| 114 | − int (*detect_and_reset)(struct amdgpu_device *adev, | |
| 115 | − int queue_type); | |
| 115 | + int (*reset)(struct amdgpu_usermode_queue *queue); | |
| 116 | 116 | }; |
| 117 | 117 | |
| 118 | 118 | /* Usermode queues for gfx */ |
| @@ -127,6 +127,8 @@ struct amdgpu_userq_mgr { · 8 unmodified lines | ||
| 127 | 127 | struct amdgpu_device *adev; |
| 128 | 128 | struct delayed_work resume_work; |
| 129 | 129 | struct drm_file *file; |
| 130 | + struct mutex proc_ctx_lock; | |
| 131 | + struct amdgpu_userq_obj proc_ctx_obj; | |
| 130 | 132 | |
| 131 | 133 | /** |
| 132 | 134 | * @reset_work: |
| @@ -177,6 +179,16 @@ int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost); · 44 unmodified lines | ||
| 177 | 179 | void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue); |
| 178 | 180 | void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell); |
| 179 | 181 | |
| 182 | +/* | |
| 183 | + * CP packs the per-process doorbell_id of the queue in | |
| 184 | + * CTXID0[9:0] on priv-fault (same encoding KFD uses via | |
| 185 | + * KFD_CTXID0_DOORBELL_ID_MASK) | |
| 186 | + */ | |
| 187 | +#define AMDGPU_CTXID0_DOORBELL_ID_MASK 0x3ff | |
| 188 | + | |
| 189 | +void amdgpu_userq_process_reset_irq(struct amdgpu_device *adev, | |
| 190 | + u32 pasid, u32 doorbell_offset); | |
| 191 | + | |
| 180 | 192 | int amdgpu_userq_input_va_validate(struct amdgpu_device *adev, |
| 181 | 193 | struct amdgpu_usermode_queue *queue, |
| 182 | 194 | u64 addr, u64 expected_size, u64 *va_out); |
| @@ -30,7 +30,7 @@ · 29 unmodified lines | ||
| 30 | 30 | #include <drm/drm_syncobj.h> |
| 31 | 31 | |
| 32 | 32 | #include "amdgpu.h" |
| 33 | −#include "amdgpu_userq_fence.h" | |
| 33 | +#include "amdgpu_trace.h" | |
| 34 | 34 | |
| 35 | 35 | #define AMDGPU_USERQ_MAX_HANDLES (1U << 16) |
| 36 | 36 | |
| @@ -528,6 +528,8 @@ int amdgpu_userq_signal_ioctl(struct drm_device *dev, void *data, · 491 unmodified lines | ||
| 528 | 528 | /* Create the new fence */ |
| 529 | 529 | amdgpu_userq_fence_init(queue, fence, wptr); |
| 530 | 530 | |
| 531 | + trace_amdgpu_userq_emit_fence(dev->dev, queue, fence); | |
| 532 | + | |
| 531 | 533 | mutex_unlock(&userq_mgr->userq_mutex); |
| 532 | 534 | |
| 533 | 535 | /* |
| @@ -701,7 +703,7 @@ amdgpu_userq_wait_add_fence(struct drm_amdgpu_userq_wait *wait_info, · 167 unmodified lines | ||
| 701 | 703 | } |
| 702 | 704 | |
| 703 | 705 | static int |
| 704 | −amdgpu_userq_wait_return_fence_info(struct drm_file *filp, | |
| 706 | +amdgpu_userq_wait_return_fence_info(struct drm_device *dev, struct drm_file *filp, | |
| 705 | 707 | struct drm_amdgpu_userq_wait *wait_info, |
| 706 | 708 | u32 *syncobj_handles, u64 *timeline_points, |
| 707 | 709 | u32 *timeline_handles, |
| @@ -869,6 +871,8 @@ amdgpu_userq_wait_return_fence_info(struct drm_file *filp, · 161 unmodified lines | ||
| 869 | 871 | |
| 870 | 872 | amdgpu_userq_fence_driver_get(fence_drv); |
| 871 | 873 | |
| 874 | + trace_amdgpu_userq_wait_deps(dev->dev, waitq, userq_fence); | |
| 875 | + | |
| 872 | 876 | /* Store drm syncobj's gpu va address and value */ |
| 873 | 877 | fence_info[cnt].va = fence_drv->va; |
| 874 | 878 | fence_info[cnt].value = fences[i]->seqno; |
| @@ -969,7 +973,7 @@ int amdgpu_userq_wait_ioctl(struct drm_device *dev, void *data, · 94 unmodified lines | ||
| 969 | 973 | gobj_write, |
| 970 | 974 | gobj_read); |
| 971 | 975 | } else { |
| 972 | − r = amdgpu_userq_wait_return_fence_info(filp, wait_info, | |
| 976 | + r = amdgpu_userq_wait_return_fence_info(dev, filp, wait_info, | |
| 973 | 977 | syncobj_handles, |
| 974 | 978 | timeline_points, |
| 975 | 979 | timeline_handles, |
| @@ -506,9 +506,8 @@ void amdgpu_vcn_ring_begin_use(struct amdgpu_ring *ring) · 505 unmodified lines | ||
| 506 | 506 | struct amdgpu_device *adev = ring->adev; |
| 507 | 507 | struct amdgpu_vcn_inst *vcn_inst = &adev->vcn.inst[ring->me]; |
| 508 | 508 | |
| 509 | − atomic_inc(&vcn_inst->total_submission_cnt); | |
| 510 | − | |
| 511 | − cancel_delayed_work_sync(&vcn_inst->idle_work); | |
| 509 | + if (!atomic_fetch_inc(&vcn_inst->total_submission_cnt)) | |
| 510 | + cancel_delayed_work_sync(&vcn_inst->idle_work); | |
| 512 | 511 | |
| 513 | 512 | mutex_lock(&vcn_inst->vcn_pg_lock); |
| 514 | 513 | vcn_inst->set_pg_state(vcn_inst, AMD_PG_STATE_UNGATE); |
| @@ -550,10 +549,9 @@ void amdgpu_vcn_ring_end_use(struct amdgpu_ring *ring) · 35 unmodified lines | ||
| 550 | 549 | !adev->vcn.inst[ring->me].using_unified_queue) |
| 551 | 550 | atomic_dec(&ring->adev->vcn.inst[ring->me].dpg_enc_submission_cnt); |
| 552 | 551 | |
| 553 | − atomic_dec(&ring->adev->vcn.inst[ring->me].total_submission_cnt); | |
| 554 | − | |
| 555 | − schedule_delayed_work(&ring->adev->vcn.inst[ring->me].idle_work, | |
| 556 | − VCN_IDLE_TIMEOUT); | |
| 552 | + if (atomic_dec_and_test(&ring->adev->vcn.inst[ring->me].total_submission_cnt)) | |
| 553 | + schedule_delayed_work(&ring->adev->vcn.inst[ring->me].idle_work, | |
| 554 | + VCN_IDLE_TIMEOUT); | |
| 557 | 555 | } |
| 558 | 556 | |
| 559 | 557 | int amdgpu_vcn_dec_ring_test_ring(struct amdgpu_ring *ring) |
| @@ -1485,6 +1483,37 @@ int vcn_set_powergating_state(struct amdgpu_ip_block *ip_block, · 925 unmodified lines | ||
| 1485 | 1483 | return ret; |
| 1486 | 1484 | } |
| 1487 | 1485 | |
| 1486 | +static struct amdgpu_fence * | |
| 1487 | +amdgpu_vcn_ring_reset_begin_helper(struct amdgpu_ring *ring, | |
| 1488 | + struct amdgpu_ring *guilty_ring, | |
| 1489 | + struct amdgpu_fence *timedout_fence) | |
| 1490 | +{ | |
| 1491 | + struct amdgpu_fence *fence; | |
| 1492 | + | |
| 1493 | + drm_sched_wqueue_stop(&ring->sched); | |
| 1494 | + if (ring == guilty_ring) | |
| 1495 | + fence = timedout_fence; | |
| 1496 | + else | |
| 1497 | + fence = amdgpu_ring_find_guilty_fence(ring); | |
| 1498 | + amdgpu_ring_reset_helper_begin(ring, fence); | |
| 1499 | + | |
| 1500 | + return fence; | |
| 1501 | +} | |
| 1502 | + | |
| 1503 | +static int | |
| 1504 | +amdgpu_vcn_ring_reset_end_helper(struct amdgpu_ring *ring, | |
| 1505 | + struct amdgpu_fence *fence) | |
| 1506 | +{ | |
| 1507 | + int r; | |
| 1508 | + | |
| 1509 | + r = amdgpu_ring_reset_helper_end(ring, fence); | |
| 1510 | + if (r) | |
| 1511 | + return r; | |
| 1512 | + | |
| 1513 | + drm_sched_wqueue_start(&ring->sched); | |
| 1514 | + return 0; | |
| 1515 | +} | |
| 1516 | + | |
| 1488 | 1517 | /** |
| 1489 | 1518 | * amdgpu_vcn_ring_reset - Reset a VCN ring |
| 1490 | 1519 | * @ring: ring to reset |
| @@ -1502,48 +1531,33 @@ int amdgpu_vcn_ring_reset(struct amdgpu_ring *ring, · 11 unmodified lines | ||
| 1502 | 1531 | { |
| 1503 | 1532 | struct amdgpu_device *adev = ring->adev; |
| 1504 | 1533 | struct amdgpu_vcn_inst *vinst = &adev->vcn.inst[ring->me]; |
| 1534 | + struct amdgpu_fence *dec_fence; | |
| 1535 | + struct amdgpu_fence *enc_fence[AMDGPU_VCN_MAX_ENC_RINGS]; | |
| 1505 | 1536 | int r, i; |
| 1506 | 1537 | |
| 1507 | 1538 | if (adev->vcn.inst[ring->me].using_unified_queue) |
| 1508 | 1539 | return -EINVAL; |
| 1509 | 1540 | |
| 1510 | 1541 | mutex_lock(&vinst->engine_reset_mutex); |
| 1511 | − /* Stop the scheduler's work queue for the dec and enc rings if they are running. | |
| 1512 | − * This ensures that no new tasks are submitted to the queues while | |
| 1513 | − * the reset is in progress. | |
| 1514 | − */ | |
| 1515 | − drm_sched_wqueue_stop(&vinst->ring_dec.sched); | |
| 1542 | + dec_fence = amdgpu_vcn_ring_reset_begin_helper(&vinst->ring_dec, ring, | |
| 1543 | + timedout_fence); | |
| 1516 | 1544 | for (i = 0; i < vinst->num_enc_rings; i++) |
| 1517 | − drm_sched_wqueue_stop(&vinst->ring_enc[i].sched); | |
| 1545 | + enc_fence[i] = amdgpu_vcn_ring_reset_begin_helper(&vinst->ring_enc[i], ring, | |
| 1546 | + timedout_fence); | |
| 1518 | 1547 | |
| 1519 | 1548 | /* Perform the VCN reset for the specified instance */ |
| 1520 | 1549 | r = vinst->reset(vinst); |
| 1521 | 1550 | if (r) |
| 1522 | 1551 | goto unlock; |
| 1523 | − r = amdgpu_ring_test_ring(&vinst->ring_dec); | |
| 1552 | + | |
| 1553 | + r = amdgpu_vcn_ring_reset_end_helper(&vinst->ring_dec, dec_fence); | |
| 1524 | 1554 | if (r) |
| 1525 | 1555 | goto unlock; |
| 1526 | 1556 | for (i = 0; i < vinst->num_enc_rings; i++) { |
| 1527 | − r = amdgpu_ring_test_ring(&vinst->ring_enc[i]); | |
| 1557 | + r = amdgpu_vcn_ring_reset_end_helper(&vinst->ring_enc[i], enc_fence[i]); | |
| 1528 | 1558 | if (r) |
| 1529 | 1559 | goto unlock; |
| 1530 | 1560 | } |
| 1531 | − amdgpu_fence_driver_force_completion(&vinst->ring_dec, | |
| 1532 | − (&vinst->ring_dec == ring) ? | |
| 1533 | − &timedout_fence->base : NULL); | |
| 1534 | − for (i = 0; i < vinst->num_enc_rings; i++) | |
| 1535 | − amdgpu_fence_driver_force_completion(&vinst->ring_enc[i], | |
| 1536 | − (&vinst->ring_enc[i] == ring) ? | |
| 1537 | − &timedout_fence->base : NULL); | |
| 1538 | − | |
| 1539 | − /* Restart the scheduler's work queue for the dec and enc rings | |
| 1540 | − * if they were stopped by this function. This allows new tasks | |
| 1541 | − * to be submitted to the queues after the reset is complete. | |
| 1542 | − */ | |
| 1543 | − drm_sched_wqueue_start(&vinst->ring_dec.sched); | |
| 1544 | − for (i = 0; i < vinst->num_enc_rings; i++) | |
| 1545 | − drm_sched_wqueue_start(&vinst->ring_enc[i].sched); | |
| 1546 | − | |
| 1547 | 1561 | unlock: |
| 1548 | 1562 | mutex_unlock(&vinst->engine_reset_mutex); |
| 1549 | 1563 | |
| @@ -368,6 +368,9 @@ struct amdgpu_vcn { · 367 unmodified lines | ||
| 368 | 368 | struct mutex workload_profile_mutex; |
| 369 | 369 | u32 reg_count; |
| 370 | 370 | const struct amdgpu_hwip_reg_entry *reg_list; |
| 371 | + | |
| 372 | + bool disable_uq; | |
| 373 | + bool disable_kq; | |
| 371 | 374 | }; |
| 372 | 375 | |
| 373 | 376 | struct amdgpu_fw_shared_rb_ptrs_struct { |
| @@ -381,7 +381,8 @@ int amdgpu_xcp_get_inst_details(struct amdgpu_xcp *xcp, · 380 unmodified lines | ||
| 381 | 381 | enum AMDGPU_XCP_IP_BLOCK ip, |
| 382 | 382 | uint32_t *inst_mask) |
| 383 | 383 | { |
| 384 | − if (!xcp->valid || !inst_mask || !(xcp->ip[ip].valid)) | |
| 384 | + if (!xcp->valid || !inst_mask || ip >= AMDGPU_XCP_MAX_BLOCKS || | |
| 385 | + !(xcp->ip[ip].valid)) | |
| 385 | 386 | return -EINVAL; |
| 386 | 387 | |
| 387 | 388 | *inst_mask = xcp->ip[ip].inst_mask; |
| @@ -468,14 +469,18 @@ void amdgpu_xcp_release_sched(struct amdgpu_device *adev, · 80 unmodified lines | ||
| 468 | 469 | { |
| 469 | 470 | struct drm_gpu_scheduler *sched = |
| 470 | 471 | container_of(entity->entity.rq, typeof(*sched), rq); |
| 472 | + struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; | |
| 471 | 473 | |
| 472 | − if (!adev->xcp_mgr) | |
| 474 | + if (!xcp_mgr) | |
| 473 | 475 | return; |
| 474 | 476 | |
| 475 | 477 | if (drm_sched_wqueue_ready(sched)) { |
| 476 | 478 | struct amdgpu_ring *ring = to_amdgpu_ring(sched); |
| 477 | 479 | |
| 478 | − atomic_dec(&adev->xcp_mgr->xcp[ring->xcp_id].ref_cnt); | |
| 480 | + mutex_lock(&xcp_mgr->xcp_lock); | |
| 481 | + if (ring->xcp_id < xcp_mgr->num_xcps && xcp_mgr->xcp[ring->xcp_id].valid) | |
| 482 | + atomic_dec(&xcp_mgr->xcp[ring->xcp_id].ref_cnt); | |
| 483 | + mutex_unlock(&xcp_mgr->xcp_lock); | |
| 479 | 484 | } |
| 480 | 485 | } |
| 481 | 486 | |
| @@ -488,7 +493,9 @@ int amdgpu_xcp_select_scheds(struct amdgpu_device *adev, · 6 unmodified lines | ||
| 488 | 493 | u32 sel_xcp_id; |
| 489 | 494 | int i; |
| 490 | 495 | struct amdgpu_xcp_mgr *xcp_mgr = adev->xcp_mgr; |
| 496 | + int r = 0; | |
| 491 | 497 | |
| 498 | + mutex_lock(&xcp_mgr->xcp_lock); | |
| 492 | 499 | if (fpriv->xcp_id == AMDGPU_XCP_NO_PARTITION) { |
| 493 | 500 | u32 least_ref_cnt = ~0; |
| 494 | 501 | |
| @@ -505,19 +512,27 @@ int amdgpu_xcp_select_scheds(struct amdgpu_device *adev, · 10 unmodified lines | ||
| 505 | 512 | } |
| 506 | 513 | sel_xcp_id = fpriv->xcp_id; |
| 507 | 514 | |
| 515 | + if (sel_xcp_id >= xcp_mgr->num_xcps || !xcp_mgr->xcp[sel_xcp_id].valid) { | |
| 516 | + dev_err(adev->dev, "Selected partition #%d is not valid.", sel_xcp_id); | |
| 517 | + r = -ENODEV; | |
| 518 | + goto out; | |
| 519 | + } | |
| 520 | + | |
| 508 | 521 | if (xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds) { |
| 509 | 522 | *num_scheds = |
| 510 | − xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds; | |
| 523 | + xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].num_scheds; | |
| 511 | 524 | *scheds = |
| 512 | − xcp_mgr->xcp[fpriv->xcp_id].gpu_sched[hw_ip][hw_prio].sched; | |
| 513 | − atomic_inc(&adev->xcp_mgr->xcp[sel_xcp_id].ref_cnt); | |
| 525 | + xcp_mgr->xcp[sel_xcp_id].gpu_sched[hw_ip][hw_prio].sched; | |
| 526 | + atomic_inc(&xcp_mgr->xcp[sel_xcp_id].ref_cnt); | |
| 514 | 527 | dev_dbg(adev->dev, "Selected partition #%d", sel_xcp_id); |
| 515 | 528 | } else { |
| 516 | 529 | dev_err(adev->dev, "Failed to schedule partition #%d.", sel_xcp_id); |
| 517 | − return -ENOENT; | |
| 530 | + r = -ENOENT; | |
| 518 | 531 | } |
| 519 | 532 | |
| 520 | − return 0; | |
| 533 | +out: | |
| 534 | + mutex_unlock(&xcp_mgr->xcp_lock); | |
| 535 | + return r; | |
| 521 | 536 | } |
| 522 | 537 | |
| 523 | 538 | static void amdgpu_set_xcp_id(struct amdgpu_device *adev, |
| @@ -574,6 +589,9 @@ static void amdgpu_xcp_gpu_sched_update(struct amdgpu_device *adev, · 50 unmodified lines | ||
| 574 | 589 | { |
| 575 | 590 | unsigned int *num_gpu_sched; |
| 576 | 591 | |
| 592 | + if (sel_xcp_id >= MAX_XCP || sel_xcp_id == AMDGPU_XCP_NO_PARTITION) | |
| 593 | + return; | |
| 594 | + | |
| 577 | 595 | num_gpu_sched = &adev->xcp_mgr->xcp[sel_xcp_id] |
| 578 | 596 | .gpu_sched[ring->funcs->type][ring->hw_prio].num_scheds; |
| 579 | 597 | adev->xcp_mgr->xcp[sel_xcp_id].gpu_sched[ring->funcs->type][ring->hw_prio] |
| @@ -903,7 +921,7 @@ static void amdgpu_xcp_cfg_sysfs_init(struct amdgpu_device *adev) · 323 unmodified lines | ||
| 903 | 921 | { |
| 904 | 922 | struct amdgpu_xcp_res_details *xcp_res; |
| 905 | 923 | struct amdgpu_xcp_cfg *xcp_cfg; |
| 906 | − int i, r, j, rid, mode; | |
| 924 | + int i, r, rid, mode; | |
| 907 | 925 | |
| 908 | 926 | if (!adev->xcp_mgr) |
| 909 | 927 | return; |
| @@ -949,14 +967,16 @@ static void amdgpu_xcp_cfg_sysfs_init(struct amdgpu_device *adev) · 39 unmodified lines | ||
| 949 | 967 | &xcp_cfg_res_sysfs_ktype, |
| 950 | 968 | &xcp_cfg->kobj, "%s", |
| 951 | 969 | xcp_res_names[rid]); |
| 952 | − if (r) | |
| 970 | + if (r) { | |
| 971 | + kobject_put(&xcp_res->kobj); | |
| 953 | 972 | goto err; |
| 973 | + } | |
| 954 | 974 | } |
| 955 | 975 | |
| 956 | 976 | adev->xcp_mgr->xcp_cfg = xcp_cfg; |
| 957 | 977 | return; |
| 958 | 978 | err: |
| 959 | − for (j = 0; j < i; j++) { | |
| 979 | + while (i--) { | |
| 960 | 980 | xcp_res = &xcp_cfg->xcp_res[i]; |
| 961 | 981 | kobject_put(&xcp_res->kobj); |
| 962 | 982 | } |
| @@ -106,53 +106,6 @@ static const int walf_pcs_err_noncorrectable_mask_reg_aldebaran[] = { · 105 unmodified lines | ||
| 106 | 106 | smnPCS_GOPX1_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000 |
| 107 | 107 | }; |
| 108 | 108 | |
| 109 | −static const int xgmi3x16_pcs_err_status_reg_v6_4[] = { | |
| 110 | − smnPCS_XGMI3X16_PCS_ERROR_STATUS, | |
| 111 | − smnPCS_XGMI3X16_PCS_ERROR_STATUS + 0x100000 | |
| 112 | −}; | |
| 113 | − | |
| 114 | −static const int xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[] = { | |
| 115 | − smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK, | |
| 116 | − smnPCS_XGMI3X16_PCS_ERROR_NONCORRECTABLE_MASK + 0x100000 | |
| 117 | −}; | |
| 118 | − | |
| 119 | −static const u64 xgmi_v6_4_0_mca_base_array[] = { | |
| 120 | − 0x11a09200, | |
| 121 | − 0x11b09200, | |
| 122 | −}; | |
| 123 | − | |
| 124 | −static const char *xgmi_v6_4_0_ras_error_code_ext[32] = { | |
| 125 | − [0x00] = "XGMI PCS DataLossErr", | |
| 126 | − [0x01] = "XGMI PCS TrainingErr", | |
| 127 | − [0x02] = "XGMI PCS FlowCtrlAckErr", | |
| 128 | − [0x03] = "XGMI PCS RxFifoUnderflowErr", | |
| 129 | − [0x04] = "XGMI PCS RxFifoOverflowErr", | |
| 130 | − [0x05] = "XGMI PCS CRCErr", | |
| 131 | − [0x06] = "XGMI PCS BERExceededErr", | |
| 132 | − [0x07] = "XGMI PCS TxMetaDataErr", | |
| 133 | − [0x08] = "XGMI PCS ReplayBufParityErr", | |
| 134 | − [0x09] = "XGMI PCS DataParityErr", | |
| 135 | − [0x0a] = "XGMI PCS ReplayFifoOverflowErr", | |
| 136 | − [0x0b] = "XGMI PCS ReplayFifoUnderflowErr", | |
| 137 | − [0x0c] = "XGMI PCS ElasticFifoOverflowErr", | |
| 138 | − [0x0d] = "XGMI PCS DeskewErr", | |
| 139 | − [0x0e] = "XGMI PCS FlowCtrlCRCErr", | |
| 140 | − [0x0f] = "XGMI PCS DataStartupLimitErr", | |
| 141 | − [0x10] = "XGMI PCS FCInitTimeoutErr", | |
| 142 | − [0x11] = "XGMI PCS RecoveryTimeoutErr", | |
| 143 | − [0x12] = "XGMI PCS ReadySerialTimeoutErr", | |
| 144 | − [0x13] = "XGMI PCS ReadySerialAttemptErr", | |
| 145 | − [0x14] = "XGMI PCS RecoveryAttemptErr", | |
| 146 | − [0x15] = "XGMI PCS RecoveryRelockAttemptErr", | |
| 147 | − [0x16] = "XGMI PCS ReplayAttemptErr", | |
| 148 | − [0x17] = "XGMI PCS SyncHdrErr", | |
| 149 | − [0x18] = "XGMI PCS TxReplayTimeoutErr", | |
| 150 | − [0x19] = "XGMI PCS RxReplayTimeoutErr", | |
| 151 | − [0x1a] = "XGMI PCS LinkSubTxTimeoutErr", | |
| 152 | − [0x1b] = "XGMI PCS LinkSubRxTimeoutErr", | |
| 153 | − [0x1c] = "XGMI PCS RxCMDPktErr", | |
| 154 | −}; | |
| 155 | − | |
| 156 | 109 | static const struct amdgpu_pcs_ras_field xgmi_pcs_ras_fields[] = { |
| 157 | 110 | {"XGMI PCS DataLossErr", |
| 158 | 111 | SOC15_REG_FIELD(XGMI0_PCS_GOPX16_PCS_ERROR_STATUS, DataLossErr)}, |
| @@ -1152,91 +1105,15 @@ int amdgpu_xgmi_remove_device(struct amdgpu_device *adev) · 993 unmodified lines | ||
| 1152 | 1105 | return 0; |
| 1153 | 1106 | } |
| 1154 | 1107 | |
| 1155 | −static int xgmi_v6_4_0_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, | |
| 1156 | − enum aca_smu_type type, void *data) | |
| 1157 | −{ | |
| 1158 | − struct amdgpu_device *adev = handle->adev; | |
| 1159 | − struct aca_bank_info info; | |
| 1160 | − const char *error_str; | |
| 1161 | − u64 status, count; | |
| 1162 | − int ret, ext_error_code; | |
| 1163 | − | |
| 1164 | − ret = aca_bank_info_decode(bank, &info); | |
| 1165 | − if (ret) | |
| 1166 | − return ret; | |
| 1167 | − | |
| 1168 | − status = bank->regs[ACA_REG_IDX_STATUS]; | |
| 1169 | − ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status); | |
| 1170 | − | |
| 1171 | − error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ? | |
| 1172 | − xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL; | |
| 1173 | − if (error_str) | |
| 1174 | − dev_info(adev->dev, "%s detected\n", error_str); | |
| 1175 | − | |
| 1176 | − count = ACA_REG__MISC0__ERRCNT(bank->regs[ACA_REG_IDX_MISC0]); | |
| 1177 | − | |
| 1178 | − switch (type) { | |
| 1179 | − case ACA_SMU_TYPE_UE: | |
| 1180 | − if (ext_error_code != 0 && ext_error_code != 1 && ext_error_code != 9) | |
| 1181 | − count = 0ULL; | |
| 1182 | − | |
| 1183 | − bank->aca_err_type = ACA_ERROR_TYPE_UE; | |
| 1184 | − ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, count); | |
| 1185 | − break; | |
| 1186 | − case ACA_SMU_TYPE_CE: | |
| 1187 | − count = ext_error_code == 6 ? count : 0ULL; | |
| 1188 | − bank->aca_err_type = ACA_ERROR_TYPE_CE; | |
| 1189 | − ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, count); | |
| 1190 | − break; | |
| 1191 | − default: | |
| 1192 | − return -EINVAL; | |
| 1193 | − } | |
| 1194 | − | |
| 1195 | − return ret; | |
| 1196 | −} | |
| 1197 | − | |
| 1198 | −static const struct aca_bank_ops xgmi_v6_4_0_aca_bank_ops = { | |
| 1199 | − .aca_bank_parser = xgmi_v6_4_0_aca_bank_parser, | |
| 1200 | −}; | |
| 1201 | − | |
| 1202 | −static const struct aca_info xgmi_v6_4_0_aca_info = { | |
| 1203 | − .hwip = ACA_HWIP_TYPE_PCS_XGMI, | |
| 1204 | − .mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK, | |
| 1205 | − .bank_ops = &xgmi_v6_4_0_aca_bank_ops, | |
| 1206 | −}; | |
| 1207 | − | |
| 1208 | 1108 | static int amdgpu_xgmi_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) |
| 1209 | 1109 | { |
| 1210 | − int r; | |
| 1211 | − | |
| 1212 | 1110 | if (!adev->gmc.xgmi.supported || |
| 1213 | 1111 | adev->gmc.xgmi.num_physical_nodes == 0) |
| 1214 | 1112 | return 0; |
| 1215 | 1113 | |
| 1216 | 1114 | amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL); |
| 1217 | 1115 | |
| 1218 | − r = amdgpu_ras_block_late_init(adev, ras_block); | |
| 1219 | − if (r) | |
| 1220 | − return r; | |
| 1221 | − | |
| 1222 | − switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) { | |
| 1223 | − case IP_VERSION(6, 4, 0): | |
| 1224 | − case IP_VERSION(6, 4, 1): | |
| 1225 | − r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__XGMI_WAFL, | |
| 1226 | − &xgmi_v6_4_0_aca_info, NULL); | |
| 1227 | − if (r) | |
| 1228 | − goto late_fini; | |
| 1229 | − break; | |
| 1230 | − default: | |
| 1231 | − break; | |
| 1232 | − } | |
| 1233 | − | |
| 1234 | − return 0; | |
| 1235 | − | |
| 1236 | −late_fini: | |
| 1237 | − amdgpu_ras_block_late_fini(adev, ras_block); | |
| 1238 | − | |
| 1239 | − return r; | |
| 1116 | + return amdgpu_ras_block_late_init(adev, ras_block); | |
| 1240 | 1117 | } |
| 1241 | 1118 | |
| 1242 | 1119 | uint64_t amdgpu_xgmi_get_relative_phy_addr(struct amdgpu_device *adev, |
| @@ -1252,7 +1129,7 @@ static void pcs_clear_status(struct amdgpu_device *adev, uint32_t pcs_status_reg · 9 unmodified lines | ||
| 1252 | 1129 | WREG32_PCIE(pcs_status_reg, 0); |
| 1253 | 1130 | } |
| 1254 | 1131 | |
| 1255 | −static void amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device *adev) | |
| 1132 | +static void amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device *adev) | |
| 1256 | 1133 | { |
| 1257 | 1134 | uint32_t i; |
| 1258 | 1135 | |
| @@ -1278,54 +1155,6 @@ static void amdgpu_xgmi_legacy_reset_ras_error_count(struct amdgpu_device *adev) · 19 unmodified lines | ||
| 1278 | 1155 | default: |
| 1279 | 1156 | break; |
| 1280 | 1157 | } |
| 1281 | − | |
| 1282 | − switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) { | |
| 1283 | − case IP_VERSION(6, 4, 0): | |
| 1284 | − case IP_VERSION(6, 4, 1): | |
| 1285 | − for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++) | |
| 1286 | − pcs_clear_status(adev, | |
| 1287 | − xgmi3x16_pcs_err_status_reg_v6_4[i]); | |
| 1288 | − break; | |
| 1289 | − default: | |
| 1290 | − break; | |
| 1291 | − } | |
| 1292 | −} | |
| 1293 | − | |
| 1294 | −static void __xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst, u64 mca_base) | |
| 1295 | −{ | |
| 1296 | − uint64_t smn_base = | |
| 1297 | − amdgpu_reg_get_smn_base64(adev, XGMI_HWIP, xgmi_inst); | |
| 1298 | − | |
| 1299 | − WREG64_MCA(smn_base, mca_base, ACA_REG_IDX_STATUS, 0ULL); | |
| 1300 | −} | |
| 1301 | − | |
| 1302 | −static void xgmi_v6_4_0_reset_error_count(struct amdgpu_device *adev, int xgmi_inst) | |
| 1303 | −{ | |
| 1304 | − int i; | |
| 1305 | − | |
| 1306 | − for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++) | |
| 1307 | − __xgmi_v6_4_0_reset_error_count(adev, xgmi_inst, xgmi_v6_4_0_mca_base_array[i]); | |
| 1308 | −} | |
| 1309 | − | |
| 1310 | −static void xgmi_v6_4_0_reset_ras_error_count(struct amdgpu_device *adev) | |
| 1311 | −{ | |
| 1312 | − int i; | |
| 1313 | − | |
| 1314 | − for_each_inst(i, adev->aid_mask) | |
| 1315 | − xgmi_v6_4_0_reset_error_count(adev, i); | |
| 1316 | −} | |
| 1317 | − | |
| 1318 | −static void amdgpu_xgmi_reset_ras_error_count(struct amdgpu_device *adev) | |
| 1319 | −{ | |
| 1320 | − switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) { | |
| 1321 | − case IP_VERSION(6, 4, 0): | |
| 1322 | − case IP_VERSION(6, 4, 1): | |
| 1323 | − xgmi_v6_4_0_reset_ras_error_count(adev); | |
| 1324 | − break; | |
| 1325 | − default: | |
| 1326 | − amdgpu_xgmi_legacy_reset_ras_error_count(adev); | |
| 1327 | − break; | |
| 1328 | − } | |
| 1329 | 1158 | } |
| 1330 | 1159 | |
| 1331 | 1160 | static int amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device *adev, |
| @@ -1343,11 +1172,7 @@ static int amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device *adev, · 11 unmodified lines | ||
| 1343 | 1172 | |
| 1344 | 1173 | if (is_xgmi_pcs) { |
| 1345 | 1174 | if (amdgpu_ip_version(adev, XGMI_HWIP, 0) == |
| 1346 | − IP_VERSION(6, 1, 0) || | |
| 1347 | − amdgpu_ip_version(adev, XGMI_HWIP, 0) == | |
| 1348 | − IP_VERSION(6, 4, 0) || | |
| 1349 | − amdgpu_ip_version(adev, XGMI_HWIP, 0) == | |
| 1350 | − IP_VERSION(6, 4, 1)) { | |
| 1175 | + IP_VERSION(6, 1, 0)) { | |
| 1351 | 1176 | pcs_ras_fields = &xgmi3x16_pcs_ras_fields[0]; |
| 1352 | 1177 | field_array_size = ARRAY_SIZE(xgmi3x16_pcs_ras_fields); |
| 1353 | 1178 | } else { |
| @@ -1381,11 +1206,11 @@ static int amdgpu_xgmi_query_pcs_error_status(struct amdgpu_device *adev, · 27 unmodified lines | ||
| 1381 | 1206 | return 0; |
| 1382 | 1207 | } |
| 1383 | 1208 | |
| 1384 | −static void amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device *adev, | |
| 1209 | +static void amdgpu_xgmi_query_ras_error_count(struct amdgpu_device *adev, | |
| 1385 | 1210 | void *ras_error_status) |
| 1386 | 1211 | { |
| 1387 | 1212 | struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; |
| 1388 | − int i, supported = 1; | |
| 1213 | + int i; | |
| 1389 | 1214 | uint32_t data, mask_data = 0; |
| 1390 | 1215 | uint32_t ue_cnt = 0, ce_cnt = 0; |
| 1391 | 1216 | |
| @@ -1449,26 +1274,6 @@ static void amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device *adev, · 57 unmodified lines | ||
| 1449 | 1274 | } |
| 1450 | 1275 | break; |
| 1451 | 1276 | default: |
| 1452 | − supported = 0; | |
| 1453 | − break; | |
| 1454 | − } | |
| 1455 | − | |
| 1456 | − switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) { | |
| 1457 | − case IP_VERSION(6, 4, 0): | |
| 1458 | − case IP_VERSION(6, 4, 1): | |
| 1459 | − /* check xgmi3x16 pcs error */ | |
| 1460 | − for (i = 0; i < ARRAY_SIZE(xgmi3x16_pcs_err_status_reg_v6_4); i++) { | |
| 1461 | − data = RREG32_PCIE(xgmi3x16_pcs_err_status_reg_v6_4[i]); | |
| 1462 | − mask_data = | |
| 1463 | − RREG32_PCIE(xgmi3x16_pcs_err_noncorrectable_mask_reg_v6_4[i]); | |
| 1464 | − if (data) | |
| 1465 | − amdgpu_xgmi_query_pcs_error_status(adev, data, | |
| 1466 | − mask_data, &ue_cnt, &ce_cnt, true, true); | |
| 1467 | − } | |
| 1468 | − break; | |
| 1469 | − default: | |
| 1470 | − if (!supported) | |
| 1471 | − dev_warn(adev->dev, "XGMI RAS error query not supported"); | |
| 1472 | 1277 | break; |
| 1473 | 1278 | } |
| 1474 | 1279 | |
| @@ -1478,90 +1283,6 @@ static void amdgpu_xgmi_legacy_query_ras_error_count(struct amdgpu_device *adev, · 3 unmodified lines | ||
| 1478 | 1283 | err_data->ce_count += ce_cnt; |
| 1479 | 1284 | } |
| 1480 | 1285 | |
| 1481 | −static enum aca_error_type xgmi_v6_4_0_pcs_mca_get_error_type(struct amdgpu_device *adev, u64 status) | |
| 1482 | −{ | |
| 1483 | − const char *error_str; | |
| 1484 | − int ext_error_code; | |
| 1485 | − | |
| 1486 | − ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status); | |
| 1487 | − | |
| 1488 | − error_str = ext_error_code < ARRAY_SIZE(xgmi_v6_4_0_ras_error_code_ext) ? | |
| 1489 | − xgmi_v6_4_0_ras_error_code_ext[ext_error_code] : NULL; | |
| 1490 | − if (error_str) | |
| 1491 | − dev_info(adev->dev, "%s detected\n", error_str); | |
| 1492 | − | |
| 1493 | − switch (ext_error_code) { | |
| 1494 | − case 0: | |
| 1495 | − return ACA_ERROR_TYPE_UE; | |
| 1496 | − case 6: | |
| 1497 | − return ACA_ERROR_TYPE_CE; | |
| 1498 | − default: | |
| 1499 | − return -EINVAL; | |
| 1500 | − } | |
| 1501 | − | |
| 1502 | − return -EINVAL; | |
| 1503 | −} | |
| 1504 | − | |
| 1505 | −static void __xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, struct amdgpu_smuio_mcm_config_info *mcm_info, | |
| 1506 | − u64 mca_base, struct ras_err_data *err_data) | |
| 1507 | −{ | |
| 1508 | − int xgmi_inst = mcm_info->die_id; | |
| 1509 | − uint64_t smn_base; | |
| 1510 | − u64 status = 0; | |
| 1511 | − | |
| 1512 | − status = RREG64_MCA(xgmi_inst, mca_base, ACA_REG_IDX_STATUS); | |
| 1513 | − if (!ACA_REG__STATUS__VAL(status)) | |
| 1514 | − return; | |
| 1515 | − | |
| 1516 | − switch (xgmi_v6_4_0_pcs_mca_get_error_type(adev, status)) { | |
| 1517 | − case ACA_ERROR_TYPE_UE: | |
| 1518 | − amdgpu_ras_error_statistic_ue_count(err_data, mcm_info, 1ULL); | |
| 1519 | − break; | |
| 1520 | − case ACA_ERROR_TYPE_CE: | |
| 1521 | − amdgpu_ras_error_statistic_ce_count(err_data, mcm_info, 1ULL); | |
| 1522 | − break; | |
| 1523 | − default: | |
| 1524 | − break; | |
| 1525 | − } | |
| 1526 | − smn_base = amdgpu_reg_get_smn_base64(adev, XGMI_HWIP, xgmi_inst); | |
| 1527 | − WREG64_MCA(smn_base, mca_base, ACA_REG_IDX_STATUS, 0ULL); | |
| 1528 | −} | |
| 1529 | − | |
| 1530 | −static void xgmi_v6_4_0_query_error_count(struct amdgpu_device *adev, int xgmi_inst, struct ras_err_data *err_data) | |
| 1531 | −{ | |
| 1532 | − struct amdgpu_smuio_mcm_config_info mcm_info = { | |
| 1533 | − .socket_id = adev->smuio.funcs->get_socket_id(adev), | |
| 1534 | − .die_id = xgmi_inst, | |
| 1535 | − }; | |
| 1536 | − int i; | |
| 1537 | − | |
| 1538 | − for (i = 0; i < ARRAY_SIZE(xgmi_v6_4_0_mca_base_array); i++) | |
| 1539 | − __xgmi_v6_4_0_query_error_count(adev, &mcm_info, xgmi_v6_4_0_mca_base_array[i], err_data); | |
| 1540 | −} | |
| 1541 | − | |
| 1542 | −static void xgmi_v6_4_0_query_ras_error_count(struct amdgpu_device *adev, void *ras_error_status) | |
| 1543 | −{ | |
| 1544 | − struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; | |
| 1545 | − int i; | |
| 1546 | − | |
| 1547 | − for_each_inst(i, adev->aid_mask) | |
| 1548 | − xgmi_v6_4_0_query_error_count(adev, i, err_data); | |
| 1549 | −} | |
| 1550 | − | |
| 1551 | −static void amdgpu_xgmi_query_ras_error_count(struct amdgpu_device *adev, | |
| 1552 | − void *ras_error_status) | |
| 1553 | −{ | |
| 1554 | − switch (amdgpu_ip_version(adev, XGMI_HWIP, 0)) { | |
| 1555 | − case IP_VERSION(6, 4, 0): | |
| 1556 | − case IP_VERSION(6, 4, 1): | |
| 1557 | − xgmi_v6_4_0_query_ras_error_count(adev, ras_error_status); | |
| 1558 | − break; | |
| 1559 | − default: | |
| 1560 | − amdgpu_xgmi_legacy_query_ras_error_count(adev, ras_error_status); | |
| 1561 | − break; | |
| 1562 | − } | |
| 1563 | −} | |
| 1564 | − | |
| 1565 | 1286 | /* Trigger XGMI/WAFL error */ |
| 1566 | 1287 | static int amdgpu_ras_error_inject_xgmi(struct amdgpu_device *adev, |
| 1567 | 1288 | void *inject_if, uint32_t instance_mask) |
| @@ -1663,6 +1384,16 @@ static void amdgpu_xgmi_reset_on_init_work(struct work_struct *work) · 95 unmodified lines | ||
| 1663 | 1384 | if (r && r != -EHWPOISON) |
| 1664 | 1385 | dev_err(tmp_adev->dev, |
| 1665 | 1386 | "error during bad page data initialization"); |
| 1387 | + | |
| 1388 | + /* | |
| 1389 | + * For the reset-on-init path (e.g. an NPS memory partition | |
| 1390 | + * switch) the RAS IP block hw_init was skipped under the | |
| 1391 | + * minimal init level, so uniras was never enabled. Bring it | |
| 1392 | + * up now that the reset domain has been unlocked. This is a | |
| 1393 | + * no-op for any other reset path where RAS is already | |
| 1394 | + * initialized, and for non-uniras devices. | |
| 1395 | + */ | |
| 1396 | + amdgpu_ras_resume_after_reset(tmp_adev); | |
| 1666 | 1397 | } |
| 1667 | 1398 | } |
| 1668 | 1399 | |
| @@ -273,8 +273,10 @@ static int aqua_vanjaram_get_xcp_res_info(struct amdgpu_xcp_mgr *xcp_mgr, · 272 unmodified lines | ||
| 273 | 273 | xcp_cfg->num_res = ARRAY_SIZE(max_res); |
| 274 | 274 | |
| 275 | 275 | for (i = 0; i < xcp_cfg->num_res; i++) { |
| 276 | − res_lt_xcp = max_res[i] < num_xcp; | |
| 277 | 276 | xcp_cfg->xcp_res[i].id = i; |
| 277 | + if (!max_res[i]) | |
| 278 | + continue; | |
| 279 | + res_lt_xcp = max_res[i] < num_xcp; | |
| 278 | 280 | xcp_cfg->xcp_res[i].num_inst = |
| 279 | 281 | res_lt_xcp ? 1 : max_res[i] / num_xcp; |
| 280 | 282 | xcp_cfg->xcp_res[i].num_inst = |
| @@ -589,6 +591,29 @@ static struct aqua_reg_list pcie_reg_addrs[] = { · 308 unmodified lines | ||
| 589 | 591 | { smreg_0x1A380088, 6, DW_ADDR_INCR }, |
| 590 | 592 | }; |
| 591 | 593 | |
| 594 | +/* | |
| 595 | + * Return the GPU's internal US switch port, or NULL if it is not visible | |
| 596 | + * (e.g. passthrough) or the EP is parented under an unrelated bridge. | |
| 597 | + */ | |
| 598 | +static struct pci_dev *aqua_vanjaram_get_us_pdev(struct amdgpu_device *adev) | |
| 599 | +{ | |
| 600 | + struct pci_dev *ds_pdev, *us_pdev; | |
| 601 | + | |
| 602 | + ds_pdev = pci_upstream_bridge(adev->pdev); | |
| 603 | + if (!ds_pdev || ds_pdev->vendor != PCI_VENDOR_ID_ATI || | |
| 604 | + pci_pcie_type(ds_pdev) != PCI_EXP_TYPE_DOWNSTREAM) | |
| 605 | + return NULL; | |
| 606 | + | |
| 607 | + us_pdev = pci_upstream_bridge(ds_pdev); | |
| 608 | + if (!us_pdev || | |
| 609 | + (us_pdev->vendor != PCI_VENDOR_ID_ATI && | |
| 610 | + us_pdev->vendor != PCI_VENDOR_ID_AMD) || | |
| 611 | + pci_pcie_type(us_pdev) != PCI_EXP_TYPE_UPSTREAM) | |
| 612 | + return NULL; | |
| 613 | + | |
| 614 | + return us_pdev; | |
| 615 | +} | |
| 616 | + | |
| 592 | 617 | static ssize_t aqua_vanjaram_read_pcie_state(struct amdgpu_device *adev, |
| 593 | 618 | void *buf, size_t max_size) |
| 594 | 619 | { |
| @@ -596,7 +621,7 @@ static ssize_t aqua_vanjaram_read_pcie_state(struct amdgpu_device *adev, · 1 unmodified line | ||
| 596 | 621 | uint32_t start_addr, incrx, num_regs, szbuf; |
| 597 | 622 | struct amdgpu_regs_pcie_v1_0 *pcie_regs; |
| 598 | 623 | struct amdgpu_smn_reg_data *reg_data; |
| 599 | − struct pci_dev *us_pdev, *ds_pdev; | |
| 624 | + struct pci_dev *us_pdev; | |
| 600 | 625 | int aer_cap, r, n; |
| 601 | 626 | |
| 602 | 627 | if (!buf || !max_size) |
| @@ -628,25 +653,27 @@ static ssize_t aqua_vanjaram_read_pcie_state(struct amdgpu_device *adev, · 25 unmodified lines | ||
| 628 | 653 | } |
| 629 | 654 | } |
| 630 | 655 | |
| 631 | − ds_pdev = pci_upstream_bridge(adev->pdev); | |
| 632 | − us_pdev = pci_upstream_bridge(ds_pdev); | |
| 656 | + us_pdev = aqua_vanjaram_get_us_pdev(adev); | |
| 657 | + if (us_pdev) { | |
| 658 | + pcie_capability_read_word(us_pdev, PCI_EXP_DEVSTA, | |
| 659 | + &pcie_regs->device_status); | |
| 660 | + pcie_capability_read_word(us_pdev, PCI_EXP_LNKSTA, | |
| 661 | + &pcie_regs->link_status); | |
| 662 | + | |
| 663 | + aer_cap = pci_find_ext_capability(us_pdev, PCI_EXT_CAP_ID_ERR); | |
| 664 | + if (aer_cap) { | |
| 665 | + pci_read_config_dword(us_pdev, | |
| 666 | + aer_cap + PCI_ERR_COR_STATUS, | |
| 667 | + &pcie_regs->pcie_corr_err_status); | |
| 668 | + pci_read_config_dword(us_pdev, | |
| 669 | + aer_cap + PCI_ERR_UNCOR_STATUS, | |
| 670 | + &pcie_regs->pcie_uncorr_err_status); | |
| 671 | + } | |
| 633 | 672 | |
| 634 | − pcie_capability_read_word(us_pdev, PCI_EXP_DEVSTA, | |
| 635 | − &pcie_regs->device_status); | |
| 636 | − pcie_capability_read_word(us_pdev, PCI_EXP_LNKSTA, | |
| 637 | − &pcie_regs->link_status); | |
| 638 | − | |
| 639 | − aer_cap = pci_find_ext_capability(us_pdev, PCI_EXT_CAP_ID_ERR); | |
| 640 | − if (aer_cap) { | |
| 641 | − pci_read_config_dword(us_pdev, aer_cap + PCI_ERR_COR_STATUS, | |
| 642 | − &pcie_regs->pcie_corr_err_status); | |
| 643 | − pci_read_config_dword(us_pdev, aer_cap + PCI_ERR_UNCOR_STATUS, | |
| 644 | − &pcie_regs->pcie_uncorr_err_status); | |
| 673 | + pci_read_config_dword(us_pdev, PCI_PRIMARY_BUS, | |
| 674 | + &pcie_regs->sub_bus_number_latency); | |
| 645 | 675 | } |
| 646 | 676 | |
| 647 | − pci_read_config_dword(us_pdev, PCI_PRIMARY_BUS, | |
| 648 | − &pcie_regs->sub_bus_number_latency); | |
| 649 | − | |
| 650 | 677 | pcie_reg_state->common_header.structure_size = szbuf; |
| 651 | 678 | pcie_reg_state->common_header.format_revision = 1; |
| 652 | 679 | pcie_reg_state->common_header.content_revision = 0; |
| @@ -114,8 +114,10 @@ static uint32_t atom_iio_execute(struct atom_context *ctx, int base, · 113 unmodified lines | ||
| 114 | 114 | uint32_t index, uint32_t data) |
| 115 | 115 | { |
| 116 | 116 | uint32_t temp = 0xCDCDCDCD; |
| 117 | + int start = base; | |
| 117 | 118 | |
| 118 | − while (1) | |
| 119 | + /* IIO opcodes read up to base+3; keep within the BIOS image */ | |
| 120 | + while (base + 3 < ctx->bios_size) | |
| 119 | 121 | switch (CU8(base)) { |
| 120 | 122 | case ATOM_IIO_NOP: |
| 121 | 123 | base++; |
| @@ -180,6 +182,9 @@ static uint32_t atom_iio_execute(struct atom_context *ctx, int base, · 58 unmodified lines | ||
| 180 | 182 | pr_info("Unknown IIO opcode\n"); |
| 181 | 183 | return 0; |
| 182 | 184 | } |
| 185 | + | |
| 186 | + pr_info("IIO method starting at offset %d runs past BIOS image\n", start); | |
| 187 | + return 0; | |
| 183 | 188 | } |
| 184 | 189 | |
| 185 | 190 | static uint32_t atom_get_src_int(atom_exec_context *ctx, uint8_t attr, |
| @@ -1327,11 +1332,25 @@ static void atom_index_iio(struct atom_context *ctx, int base) · 1141 unmodified lines | ||
| 1327 | 1332 | ctx->iio = kzalloc(2 * 256, GFP_KERNEL); |
| 1328 | 1333 | if (!ctx->iio) |
| 1329 | 1334 | return; |
| 1330 | − while (CU8(base) == ATOM_IIO_START) { | |
| 1331 | − ctx->iio[CU8(base + 1)] = base + 2; | |
| 1335 | + while (base + 1 < ctx->bios_size && CU8(base) == ATOM_IIO_START) { | |
| 1336 | + uint8_t index = CU8(base + 1); | |
| 1337 | + int start = base + 2; | |
| 1332 | 1338 | base += 2; |
| 1333 | − while (CU8(base) != ATOM_IIO_END) | |
| 1334 | − base += atom_iio_len[CU8(base)]; | |
| 1339 | + while (base < ctx->bios_size && CU8(base) != ATOM_IIO_END) { | |
| 1340 | + uint8_t op = CU8(base); | |
| 1341 | + | |
| 1342 | + /* | |
| 1343 | + * Unknown opcode: its length is unknown so the byte | |
| 1344 | + * stream cannot be resynced reliably. | |
| 1345 | + */ | |
| 1346 | + if (op >= ARRAY_SIZE(atom_iio_len)) | |
| 1347 | + return; | |
| 1348 | + base += atom_iio_len[op]; | |
| 1349 | + } | |
| 1350 | + if (base >= ctx->bios_size) | |
| 1351 | + return; | |
| 1352 | + /* Only index well-formed methods, others stay 0 */ | |
| 1353 | + ctx->iio[index] = start; | |
| 1335 | 1354 | base += 3; |
| 1336 | 1355 | } |
| 1337 | 1356 | } |
| @@ -1339,6 +1358,7 @@ static void atom_index_iio(struct atom_context *ctx, int base) · 1 unmodified line | ||
| 1339 | 1358 | static void atom_get_vbios_name(struct atom_context *ctx) |
| 1340 | 1359 | { |
| 1341 | 1360 | unsigned char *p_rom; |
| 1361 | + unsigned char *p_end; | |
| 1342 | 1362 | unsigned char str_num; |
| 1343 | 1363 | unsigned short off_to_vbios_str; |
| 1344 | 1364 | unsigned char *c_ptr; |
| @@ -1349,39 +1369,48 @@ static void atom_get_vbios_name(struct atom_context *ctx) · 4 unmodified lines | ||
| 1349 | 1369 | char *back; |
| 1350 | 1370 | |
| 1351 | 1371 | p_rom = ctx->bios; |
| 1372 | + p_end = p_rom + ctx->bios_size; | |
| 1373 | + | |
| 1374 | + if (p_rom + OFFSET_TO_GET_ATOMBIOS_STRING_START + 1 >= p_end) | |
| 1375 | + goto no_name; | |
| 1352 | 1376 | |
| 1353 | 1377 | str_num = *(p_rom + OFFSET_TO_GET_ATOMBIOS_NUMBER_OF_STRINGS); |
| 1354 | − if (str_num != 0) { | |
| 1355 | − off_to_vbios_str = | |
| 1356 | − *(unsigned short *)(p_rom + OFFSET_TO_GET_ATOMBIOS_STRING_START); | |
| 1378 | + if (!str_num) | |
| 1379 | + goto no_name; | |
| 1357 | 1380 | |
| 1358 | − c_ptr = (unsigned char *)(p_rom + off_to_vbios_str); | |
| 1359 | − } else { | |
| 1360 | − /* do not know where to find name */ | |
| 1361 | − memcpy(ctx->name, na, 7); | |
| 1362 | − ctx->name[7] = 0; | |
| 1363 | − return; | |
| 1364 | − } | |
| 1381 | + off_to_vbios_str = | |
| 1382 | + *(unsigned short *)(p_rom + OFFSET_TO_GET_ATOMBIOS_STRING_START); | |
| 1383 | + | |
| 1384 | + c_ptr = (unsigned char *)(p_rom + off_to_vbios_str); | |
| 1385 | + if (c_ptr >= p_end) | |
| 1386 | + goto no_name; | |
| 1365 | 1387 | |
| 1366 | 1388 | /* |
| 1367 | 1389 | * skip the atombios strings, usually 4 |
| 1368 | 1390 | * 1st is P/N, 2nd is ASIC, 3rd is PCI type, 4th is Memory type |
| 1369 | 1391 | */ |
| 1370 | 1392 | for (i = 0; i < str_num; i++) { |
| 1371 | − while (*c_ptr != 0) | |
| 1393 | + while (c_ptr < p_end && *c_ptr != 0) | |
| 1372 | 1394 | c_ptr++; |
| 1373 | 1395 | c_ptr++; |
| 1374 | 1396 | } |
| 1375 | 1397 | |
| 1376 | 1398 | /* skip the following 2 chars: 0x0D 0x0A */ |
| 1377 | 1399 | c_ptr += 2; |
| 1400 | + if (c_ptr >= p_end) | |
| 1401 | + goto no_name; | |
| 1378 | 1402 | |
| 1379 | − name_size = strnlen(c_ptr, STRLEN_LONG - 1); | |
| 1403 | + name_size = strnlen(c_ptr, min(STRLEN_LONG - 1, (int)(p_end - c_ptr))); | |
| 1380 | 1404 | memcpy(ctx->name, c_ptr, name_size); |
| 1381 | 1405 | back = ctx->name + name_size; |
| 1382 | 1406 | while ((*--back) == ' ') |
| 1383 | 1407 | ; |
| 1384 | 1408 | *(back + 1) = '\0'; |
| 1409 | + return; | |
| 1410 | + | |
| 1411 | +no_name: | |
| 1412 | + /* do not know where to find name */ | |
| 1413 | + strscpy(ctx->name, na, sizeof(ctx->name)); | |
| 1385 | 1414 | } |
| 1386 | 1415 | |
| 1387 | 1416 | static void atom_get_vbios_date(struct atom_context *ctx) |
| @@ -1553,7 +1582,7 @@ static inline void atom_print_vbios_info(struct atom_context *ctx) · 165 unmodified lines | ||
| 1553 | 1582 | drm_info(ctx->card->dev, "ATOM BIOS: %s\n", vbios_info); |
| 1554 | 1583 | } |
| 1555 | 1584 | |
| 1556 | −struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios) | |
| 1585 | +struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios, uint32_t bios_size) | |
| 1557 | 1586 | { |
| 1558 | 1587 | int base; |
| 1559 | 1588 | struct atom_context *ctx = |
| @@ -1567,6 +1596,7 @@ struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios) · 7 unmodified lines | ||
| 1567 | 1596 | |
| 1568 | 1597 | ctx->card = card; |
| 1569 | 1598 | ctx->bios = bios; |
| 1599 | + ctx->bios_size = bios_size; | |
| 1570 | 1600 | |
| 1571 | 1601 | if (CU16(0) != ATOM_BIOS_MAGIC) { |
| 1572 | 1602 | pr_info("Invalid BIOS magic\n"); |
| @@ -133,6 +133,7 @@ struct atom_context { · 132 unmodified lines | ||
| 133 | 133 | struct card_info *card; |
| 134 | 134 | struct mutex mutex; |
| 135 | 135 | void *bios; |
| 136 | + uint32_t bios_size; | |
| 136 | 137 | uint32_t cmd_table, data_table; |
| 137 | 138 | uint16_t *iio; |
| 138 | 139 | |
| @@ -160,7 +161,7 @@ struct atom_context { · 21 unmodified lines | ||
| 160 | 161 | |
| 161 | 162 | extern int amdgpu_atom_debug; |
| 162 | 163 | |
| 163 | −struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios); | |
| 164 | +struct atom_context *amdgpu_atom_parse(struct card_info *card, void *bios, uint32_t bios_size); | |
| 164 | 165 | int amdgpu_atom_execute_table(struct atom_context *ctx, int index, uint32_t *params, int params_size); |
| 165 | 166 | int amdgpu_atom_asic_init(struct atom_context *ctx); |
| 166 | 167 | void amdgpu_atom_destroy(struct atom_context *ctx); |
| @@ -1876,12 +1876,6 @@ static void cik_invalidate_hdp(struct amdgpu_device *adev, · 1875 unmodified lines | ||
| 1876 | 1876 | } |
| 1877 | 1877 | } |
| 1878 | 1878 | |
| 1879 | −static bool cik_need_full_reset(struct amdgpu_device *adev) | |
| 1880 | −{ | |
| 1881 | − /* change this when we support soft reset */ | |
| 1882 | − return true; | |
| 1883 | −} | |
| 1884 | − | |
| 1885 | 1879 | static void cik_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0, |
| 1886 | 1880 | uint64_t *count1) |
| 1887 | 1881 | { |
| @@ -1971,7 +1965,6 @@ static const struct amdgpu_asic_funcs cik_asic_funcs = · 83 unmodified lines | ||
| 1971 | 1965 | .get_config_memsize = &cik_get_config_memsize, |
| 1972 | 1966 | .flush_hdp = &cik_flush_hdp, |
| 1973 | 1967 | .invalidate_hdp = &cik_invalidate_hdp, |
| 1974 | − .need_full_reset = &cik_need_full_reset, | |
| 1975 | 1968 | .init_doorbell_index = &legacy_doorbell_index_init, |
| 1976 | 1969 | .get_pcie_usage = &cik_get_pcie_usage, |
| 1977 | 1970 | .need_reset_on_init = &cik_need_reset_on_init, |
| @@ -410,36 +410,6 @@ static u32 dce_v10_0_hpd_get_gpio_reg(struct amdgpu_device *adev) · 409 unmodified lines | ||
| 410 | 410 | return mmDC_GPIO_HPD_A; |
| 411 | 411 | } |
| 412 | 412 | |
| 413 | −static bool dce_v10_0_is_display_hung(struct amdgpu_device *adev) | |
| 414 | −{ | |
| 415 | − u32 crtc_hung = 0; | |
| 416 | − u32 crtc_status[6]; | |
| 417 | − u32 i, j, tmp; | |
| 418 | − | |
| 419 | − for (i = 0; i < adev->mode_info.num_crtc; i++) { | |
| 420 | − tmp = RREG32(mmCRTC_CONTROL + crtc_offsets[i]); | |
| 421 | − if (REG_GET_FIELD(tmp, CRTC_CONTROL, CRTC_MASTER_EN)) { | |
| 422 | − crtc_status[i] = RREG32(mmCRTC_STATUS_HV_COUNT + crtc_offsets[i]); | |
| 423 | − crtc_hung |= (1 << i); | |
| 424 | − } | |
| 425 | − } | |
| 426 | − | |
| 427 | − for (j = 0; j < 10; j++) { | |
| 428 | − for (i = 0; i < adev->mode_info.num_crtc; i++) { | |
| 429 | − if (crtc_hung & (1 << i)) { | |
| 430 | − tmp = RREG32(mmCRTC_STATUS_HV_COUNT + crtc_offsets[i]); | |
| 431 | − if (tmp != crtc_status[i]) | |
| 432 | − crtc_hung &= ~(1 << i); | |
| 433 | − } | |
| 434 | − } | |
| 435 | − if (crtc_hung == 0) | |
| 436 | − return false; | |
| 437 | − udelay(100); | |
| 438 | − } | |
| 439 | − | |
| 440 | − return true; | |
| 441 | −} | |
| 442 | − | |
| 443 | 413 | static void dce_v10_0_set_vga_render_state(struct amdgpu_device *adev, |
| 444 | 414 | bool render) |
| 445 | 415 | { |
| @@ -2956,40 +2926,6 @@ static bool dce_v10_0_is_idle(struct amdgpu_ip_block *ip_block) · 2510 unmodified lines | ||
| 2956 | 2926 | return true; |
| 2957 | 2927 | } |
| 2958 | 2928 | |
| 2959 | −static bool dce_v10_0_check_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 2960 | −{ | |
| 2961 | − struct amdgpu_device *adev = ip_block->adev; | |
| 2962 | − | |
| 2963 | − return dce_v10_0_is_display_hung(adev); | |
| 2964 | −} | |
| 2965 | − | |
| 2966 | −static int dce_v10_0_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 2967 | −{ | |
| 2968 | − u32 srbm_soft_reset = 0, tmp; | |
| 2969 | − struct amdgpu_device *adev = ip_block->adev; | |
| 2970 | − | |
| 2971 | − if (dce_v10_0_is_display_hung(adev)) | |
| 2972 | − srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_DC_MASK; | |
| 2973 | − | |
| 2974 | − if (srbm_soft_reset) { | |
| 2975 | − tmp = RREG32(mmSRBM_SOFT_RESET); | |
| 2976 | − tmp |= srbm_soft_reset; | |
| 2977 | − dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); | |
| 2978 | − WREG32(mmSRBM_SOFT_RESET, tmp); | |
| 2979 | − tmp = RREG32(mmSRBM_SOFT_RESET); | |
| 2980 | − | |
| 2981 | − udelay(50); | |
| 2982 | − | |
| 2983 | − tmp &= ~srbm_soft_reset; | |
| 2984 | − WREG32(mmSRBM_SOFT_RESET, tmp); | |
| 2985 | − tmp = RREG32(mmSRBM_SOFT_RESET); | |
| 2986 | − | |
| 2987 | − /* Wait a little for things to settle down */ | |
| 2988 | − udelay(50); | |
| 2989 | − } | |
| 2990 | − return 0; | |
| 2991 | −} | |
| 2992 | − | |
| 2993 | 2929 | static void dce_v10_0_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev, |
| 2994 | 2930 | int crtc, |
| 2995 | 2931 | enum amdgpu_interrupt_state state) |
| @@ -3332,8 +3268,6 @@ static const struct amd_ip_funcs dce_v10_0_ip_funcs = { · 336 unmodified lines | ||
| 3332 | 3268 | .suspend = dce_v10_0_suspend, |
| 3333 | 3269 | .resume = dce_v10_0_resume, |
| 3334 | 3270 | .is_idle = dce_v10_0_is_idle, |
| 3335 | − .check_soft_reset = dce_v10_0_check_soft_reset, | |
| 3336 | − .soft_reset = dce_v10_0_soft_reset, | |
| 3337 | 3271 | .set_clockgating_state = dce_v10_0_set_clockgating_state, |
| 3338 | 3272 | .set_powergating_state = dce_v10_0_set_powergating_state, |
| 3339 | 3273 | }; |
| @@ -378,35 +378,6 @@ static u32 dce_v6_0_hpd_get_gpio_reg(struct amdgpu_device *adev) · 377 unmodified lines | ||
| 378 | 378 | return mmDC_GPIO_HPD_A; |
| 379 | 379 | } |
| 380 | 380 | |
| 381 | −static bool dce_v6_0_is_display_hung(struct amdgpu_device *adev) | |
| 382 | −{ | |
| 383 | − u32 crtc_hung = 0; | |
| 384 | − u32 crtc_status[6]; | |
| 385 | − u32 i, j, tmp; | |
| 386 | − | |
| 387 | − for (i = 0; i < adev->mode_info.num_crtc; i++) { | |
| 388 | − if (RREG32(mmCRTC_CONTROL + crtc_offsets[i]) & CRTC_CONTROL__CRTC_MASTER_EN_MASK) { | |
| 389 | − crtc_status[i] = RREG32(mmCRTC_STATUS_HV_COUNT + crtc_offsets[i]); | |
| 390 | − crtc_hung |= (1 << i); | |
| 391 | − } | |
| 392 | − } | |
| 393 | − | |
| 394 | − for (j = 0; j < 10; j++) { | |
| 395 | − for (i = 0; i < adev->mode_info.num_crtc; i++) { | |
| 396 | − if (crtc_hung & (1 << i)) { | |
| 397 | − tmp = RREG32(mmCRTC_STATUS_HV_COUNT + crtc_offsets[i]); | |
| 398 | − if (tmp != crtc_status[i]) | |
| 399 | − crtc_hung &= ~(1 << i); | |
| 400 | − } | |
| 401 | − } | |
| 402 | − if (crtc_hung == 0) | |
| 403 | − return false; | |
| 404 | − udelay(100); | |
| 405 | − } | |
| 406 | − | |
| 407 | − return true; | |
| 408 | −} | |
| 409 | − | |
| 410 | 381 | static void dce_v6_0_set_vga_render_state(struct amdgpu_device *adev, |
| 411 | 382 | bool render) |
| 412 | 383 | { |
| @@ -2901,33 +2872,6 @@ static bool dce_v6_0_is_idle(struct amdgpu_ip_block *ip_block) · 2488 unmodified lines | ||
| 2901 | 2872 | return true; |
| 2902 | 2873 | } |
| 2903 | 2874 | |
| 2904 | −static int dce_v6_0_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 2905 | −{ | |
| 2906 | − u32 srbm_soft_reset = 0, tmp; | |
| 2907 | − struct amdgpu_device *adev = ip_block->adev; | |
| 2908 | − | |
| 2909 | − if (dce_v6_0_is_display_hung(adev)) | |
| 2910 | − srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_DC_MASK; | |
| 2911 | − | |
| 2912 | − if (srbm_soft_reset) { | |
| 2913 | − tmp = RREG32(mmSRBM_SOFT_RESET); | |
| 2914 | − tmp |= srbm_soft_reset; | |
| 2915 | − dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); | |
| 2916 | − WREG32(mmSRBM_SOFT_RESET, tmp); | |
| 2917 | − tmp = RREG32(mmSRBM_SOFT_RESET); | |
| 2918 | − | |
| 2919 | − udelay(50); | |
| 2920 | − | |
| 2921 | − tmp &= ~srbm_soft_reset; | |
| 2922 | − WREG32(mmSRBM_SOFT_RESET, tmp); | |
| 2923 | − tmp = RREG32(mmSRBM_SOFT_RESET); | |
| 2924 | − | |
| 2925 | − /* Wait a little for things to settle down */ | |
| 2926 | − udelay(50); | |
| 2927 | − } | |
| 2928 | − return 0; | |
| 2929 | −} | |
| 2930 | − | |
| 2931 | 2875 | static void dce_v6_0_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev, |
| 2932 | 2876 | int crtc, |
| 2933 | 2877 | enum amdgpu_interrupt_state state) |
| @@ -3224,7 +3168,6 @@ static const struct amd_ip_funcs dce_v6_0_ip_funcs = { · 290 unmodified lines | ||
| 3224 | 3168 | .suspend = dce_v6_0_suspend, |
| 3225 | 3169 | .resume = dce_v6_0_resume, |
| 3226 | 3170 | .is_idle = dce_v6_0_is_idle, |
| 3227 | − .soft_reset = dce_v6_0_soft_reset, | |
| 3228 | 3171 | .set_clockgating_state = dce_v6_0_set_clockgating_state, |
| 3229 | 3172 | .set_powergating_state = dce_v6_0_set_powergating_state, |
| 3230 | 3173 | }; |
| @@ -362,35 +362,6 @@ static u32 dce_v8_0_hpd_get_gpio_reg(struct amdgpu_device *adev) · 361 unmodified lines | ||
| 362 | 362 | return mmDC_GPIO_HPD_A; |
| 363 | 363 | } |
| 364 | 364 | |
| 365 | −static bool dce_v8_0_is_display_hung(struct amdgpu_device *adev) | |
| 366 | −{ | |
| 367 | − u32 crtc_hung = 0; | |
| 368 | − u32 crtc_status[6]; | |
| 369 | − u32 i, j, tmp; | |
| 370 | − | |
| 371 | − for (i = 0; i < adev->mode_info.num_crtc; i++) { | |
| 372 | − if (RREG32(mmCRTC_CONTROL + crtc_offsets[i]) & CRTC_CONTROL__CRTC_MASTER_EN_MASK) { | |
| 373 | − crtc_status[i] = RREG32(mmCRTC_STATUS_HV_COUNT + crtc_offsets[i]); | |
| 374 | − crtc_hung |= (1 << i); | |
| 375 | − } | |
| 376 | − } | |
| 377 | − | |
| 378 | − for (j = 0; j < 10; j++) { | |
| 379 | − for (i = 0; i < adev->mode_info.num_crtc; i++) { | |
| 380 | − if (crtc_hung & (1 << i)) { | |
| 381 | − tmp = RREG32(mmCRTC_STATUS_HV_COUNT + crtc_offsets[i]); | |
| 382 | − if (tmp != crtc_status[i]) | |
| 383 | − crtc_hung &= ~(1 << i); | |
| 384 | − } | |
| 385 | − } | |
| 386 | − if (crtc_hung == 0) | |
| 387 | − return false; | |
| 388 | − udelay(100); | |
| 389 | − } | |
| 390 | − | |
| 391 | − return true; | |
| 392 | −} | |
| 393 | − | |
| 394 | 365 | static void dce_v8_0_set_vga_render_state(struct amdgpu_device *adev, |
| 395 | 366 | bool render) |
| 396 | 367 | { |
| @@ -2873,33 +2844,6 @@ static bool dce_v8_0_is_idle(struct amdgpu_ip_block *ip_block) · 2476 unmodified lines | ||
| 2873 | 2844 | return true; |
| 2874 | 2845 | } |
| 2875 | 2846 | |
| 2876 | −static int dce_v8_0_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 2877 | −{ | |
| 2878 | − u32 srbm_soft_reset = 0, tmp; | |
| 2879 | − struct amdgpu_device *adev = ip_block->adev; | |
| 2880 | − | |
| 2881 | − if (dce_v8_0_is_display_hung(adev)) | |
| 2882 | − srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_DC_MASK; | |
| 2883 | − | |
| 2884 | − if (srbm_soft_reset) { | |
| 2885 | − tmp = RREG32(mmSRBM_SOFT_RESET); | |
| 2886 | − tmp |= srbm_soft_reset; | |
| 2887 | − dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); | |
| 2888 | − WREG32(mmSRBM_SOFT_RESET, tmp); | |
| 2889 | − tmp = RREG32(mmSRBM_SOFT_RESET); | |
| 2890 | − | |
| 2891 | − udelay(50); | |
| 2892 | − | |
| 2893 | − tmp &= ~srbm_soft_reset; | |
| 2894 | − WREG32(mmSRBM_SOFT_RESET, tmp); | |
| 2895 | − tmp = RREG32(mmSRBM_SOFT_RESET); | |
| 2896 | − | |
| 2897 | − /* Wait a little for things to settle down */ | |
| 2898 | − udelay(50); | |
| 2899 | − } | |
| 2900 | − return 0; | |
| 2901 | −} | |
| 2902 | − | |
| 2903 | 2847 | static void dce_v8_0_set_crtc_vblank_interrupt_state(struct amdgpu_device *adev, |
| 2904 | 2848 | int crtc, |
| 2905 | 2849 | enum amdgpu_interrupt_state state) |
| @@ -3241,7 +3185,6 @@ static const struct amd_ip_funcs dce_v8_0_ip_funcs = { · 335 unmodified lines | ||
| 3241 | 3185 | .suspend = dce_v8_0_suspend, |
| 3242 | 3186 | .resume = dce_v8_0_resume, |
| 3243 | 3187 | .is_idle = dce_v8_0_is_idle, |
| 3244 | − .soft_reset = dce_v8_0_soft_reset, | |
| 3245 | 3188 | .set_clockgating_state = dce_v8_0_set_clockgating_state, |
| 3246 | 3189 | .set_powergating_state = dce_v8_0_set_powergating_state, |
| 3247 | 3190 | }; |
| @@ -7852,6 +7852,8 @@ static int gfx_v10_0_early_init(struct amdgpu_ip_block *ip_block) · 7851 unmodified lines | ||
| 7852 | 7852 | /* init rlcg reg access ctrl */ |
| 7853 | 7853 | gfx_v10_0_init_rlcg_reg_access_ctrl(adev); |
| 7854 | 7854 | |
| 7855 | + amdgpu_init_rlc_reg_funcs(adev); | |
| 7856 | + | |
| 7855 | 7857 | return gfx_v10_0_init_microcode(adev); |
| 7856 | 7858 | } |
| 7857 | 7859 | |
| @@ -1923,6 +1923,11 @@ static int gfx_v11_0_sw_init(struct amdgpu_ip_block *ip_block) · 1922 unmodified lines | ||
| 1923 | 1923 | if (r) |
| 1924 | 1924 | return r; |
| 1925 | 1925 | |
| 1926 | + adev->gfx.me.use_mmio_for_reset = false; | |
| 1927 | + adev->gfx.mec.use_mmio_for_reset = true; | |
| 1928 | + | |
| 1929 | + mutex_init(&adev->gfx.mec.reset_mutex); | |
| 1930 | + | |
| 1926 | 1931 | return 0; |
| 1927 | 1932 | } |
| 1928 | 1933 | |
| @@ -4233,13 +4238,13 @@ static int gfx_v11_0_gfx_mqd_init(struct amdgpu_device *adev, void *m, · 2304 unmodified lines | ||
| 4233 | 4238 | return 0; |
| 4234 | 4239 | } |
| 4235 | 4240 | |
| 4236 | −static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset) | |
| 4241 | +static int gfx_v11_0_kgq_init_queue(struct amdgpu_ring *ring) | |
| 4237 | 4242 | { |
| 4238 | 4243 | struct amdgpu_device *adev = ring->adev; |
| 4239 | 4244 | struct v11_gfx_mqd *mqd = ring->mqd_ptr; |
| 4240 | 4245 | int mqd_idx = ring - &adev->gfx.gfx_ring[0]; |
| 4241 | 4246 | |
| 4242 | − if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) { | |
| 4247 | + if (!amdgpu_in_reset(adev) && !adev->in_suspend) { | |
| 4243 | 4248 | memset((void *)mqd, 0, sizeof(*mqd)); |
| 4244 | 4249 | mutex_lock(&adev->srbm_mutex); |
| 4245 | 4250 | soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); |
| @@ -4266,7 +4271,7 @@ static int gfx_v11_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev) · 20 unmodified lines | ||
| 4266 | 4271 | int r, i; |
| 4267 | 4272 | |
| 4268 | 4273 | for (i = 0; i < adev->gfx.num_gfx_rings; i++) { |
| 4269 | − r = gfx_v11_0_kgq_init_queue(&adev->gfx.gfx_ring[i], false); | |
| 4274 | + r = gfx_v11_0_kgq_init_queue(&adev->gfx.gfx_ring[i]); | |
| 4270 | 4275 | if (r) |
| 4271 | 4276 | return r; |
| 4272 | 4277 | } |
| @@ -4603,13 +4608,13 @@ static int gfx_v11_0_kiq_init_queue(struct amdgpu_ring *ring) · 330 unmodified lines | ||
| 4603 | 4608 | return 0; |
| 4604 | 4609 | } |
| 4605 | 4610 | |
| 4606 | −static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring, bool reset) | |
| 4611 | +static int gfx_v11_0_kcq_init_queue(struct amdgpu_ring *ring) | |
| 4607 | 4612 | { |
| 4608 | 4613 | struct amdgpu_device *adev = ring->adev; |
| 4609 | 4614 | struct v11_compute_mqd *mqd = ring->mqd_ptr; |
| 4610 | 4615 | int mqd_idx = ring - &adev->gfx.compute_ring[0]; |
| 4611 | 4616 | |
| 4612 | − if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) { | |
| 4617 | + if (!amdgpu_in_reset(adev) && !adev->in_suspend) { | |
| 4613 | 4618 | memset((void *)mqd, 0, sizeof(*mqd)); |
| 4614 | 4619 | mutex_lock(&adev->srbm_mutex); |
| 4615 | 4620 | soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); |
| @@ -4646,7 +4651,7 @@ static int gfx_v11_0_kcq_resume(struct amdgpu_device *adev) · 30 unmodified lines | ||
| 4646 | 4651 | gfx_v11_0_cp_compute_enable(adev, true); |
| 4647 | 4652 | |
| 4648 | 4653 | for (i = 0; i < adev->gfx.num_compute_rings; i++) { |
| 4649 | − r = gfx_v11_0_kcq_init_queue(&adev->gfx.compute_ring[i], false); | |
| 4654 | + r = gfx_v11_0_kcq_init_queue(&adev->gfx.compute_ring[i]); | |
| 4650 | 4655 | if (r) |
| 4651 | 4656 | return r; |
| 4652 | 4657 | } |
| @@ -5265,38 +5270,12 @@ static int gfx_v11_0_soft_reset(struct amdgpu_ip_block *ip_block) · 612 unmodified lines | ||
| 5265 | 5270 | |
| 5266 | 5271 | amdgpu_gfx_rlc_exit_safe_mode(adev, 0); |
| 5267 | 5272 | |
| 5268 | − return gfx_v11_0_cp_resume(adev); | |
| 5269 | −} | |
| 5270 | − | |
| 5271 | −static bool gfx_v11_0_check_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 5272 | −{ | |
| 5273 | − int i, r; | |
| 5274 | − struct amdgpu_device *adev = ip_block->adev; | |
| 5275 | − struct amdgpu_ring *ring; | |
| 5276 | − long tmo = msecs_to_jiffies(1000); | |
| 5277 | − | |
| 5278 | − for (i = 0; i < adev->gfx.num_gfx_rings; i++) { | |
| 5279 | − ring = &adev->gfx.gfx_ring[i]; | |
| 5280 | − r = amdgpu_ring_test_ib(ring, tmo); | |
| 5281 | − if (r) | |
| 5282 | − return true; | |
| 5283 | − } | |
| 5284 | − | |
| 5285 | − for (i = 0; i < adev->gfx.num_compute_rings; i++) { | |
| 5286 | − ring = &adev->gfx.compute_ring[i]; | |
| 5287 | − r = amdgpu_ring_test_ib(ring, tmo); | |
| 5288 | − if (r) | |
| 5289 | − return true; | |
| 5290 | − } | |
| 5291 | − | |
| 5292 | − return false; | |
| 5293 | −} | |
| 5273 | + r = gfx_v11_0_cp_resume(adev); | |
| 5274 | + if (r) | |
| 5275 | + return r; | |
| 5294 | 5276 | |
| 5295 | −static int gfx_v11_0_post_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 5296 | −{ | |
| 5297 | − struct amdgpu_device *adev = ip_block->adev; | |
| 5298 | 5277 | /** |
| 5299 | − * GFX soft reset will impact MES, need resume MES when do GFX soft reset | |
| 5278 | + * GFX soft reset impacts MES, resume MES after GFX soft reset is finished | |
| 5300 | 5279 | */ |
| 5301 | 5280 | return amdgpu_mes_resume(adev, 0); |
| 5302 | 5281 | } |
| @@ -5420,6 +5399,8 @@ static int gfx_v11_0_early_init(struct amdgpu_ip_block *ip_block) · 117 unmodified lines | ||
| 5420 | 5399 | |
| 5421 | 5400 | gfx_v11_0_init_rlcg_reg_access_ctrl(adev); |
| 5422 | 5401 | |
| 5402 | + amdgpu_init_rlc_reg_funcs(adev); | |
| 5403 | + | |
| 5423 | 5404 | return gfx_v11_0_init_microcode(adev); |
| 5424 | 5405 | } |
| 5425 | 5406 | |
| @@ -6708,22 +6689,29 @@ static int gfx_v11_0_set_priv_inst_fault_state(struct amdgpu_device *adev, · 1282 unmodified lines | ||
| 6708 | 6689 | static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev, |
| 6709 | 6690 | struct amdgpu_iv_entry *entry) |
| 6710 | 6691 | { |
| 6711 | − u8 me_id, pipe_id, queue_id; | |
| 6712 | − struct amdgpu_ring *ring; | |
| 6713 | − int i; | |
| 6714 | − | |
| 6715 | − me_id = (entry->ring_id & 0x0c) >> 2; | |
| 6716 | − pipe_id = (entry->ring_id & 0x03) >> 0; | |
| 6717 | − queue_id = (entry->ring_id & 0x70) >> 4; | |
| 6692 | + u32 doorbell_offset = entry->src_data[0] & AMDGPU_CTXID0_DOORBELL_ID_MASK; | |
| 6718 | 6693 | |
| 6694 | + /* | |
| 6695 | + * Try KQ first by ring_id (HW slot is authoritative). The | |
| 6696 | + * KMD compute_hqd_mask contract guarantees KCQ and user queues | |
| 6697 | + * never share a HW slot. | |
| 6698 | + */ | |
| 6719 | 6699 | if (!adev->gfx.disable_kq) { |
| 6700 | + u8 me_id = (entry->ring_id & 0x0c) >> 2; | |
| 6701 | + u8 pipe_id = (entry->ring_id & 0x03) >> 0; | |
| 6702 | + u8 queue_id = (entry->ring_id & 0x70) >> 4; | |
| 6703 | + struct amdgpu_ring *ring; | |
| 6704 | + int i; | |
| 6705 | + | |
| 6720 | 6706 | switch (me_id) { |
| 6721 | 6707 | case 0: |
| 6722 | 6708 | for (i = 0; i < adev->gfx.num_gfx_rings; i++) { |
| 6723 | 6709 | ring = &adev->gfx.gfx_ring[i]; |
| 6724 | 6710 | if (ring->me == me_id && ring->pipe == pipe_id && |
| 6725 | − ring->queue == queue_id) | |
| 6711 | + ring->queue == queue_id) { | |
| 6726 | 6712 | drm_sched_fault(&ring->sched); |
| 6713 | + return; | |
| 6714 | + } | |
| 6727 | 6715 | } |
| 6728 | 6716 | break; |
| 6729 | 6717 | case 1: |
| @@ -6731,8 +6719,10 @@ static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev, · 1 unmodified line | ||
| 6731 | 6719 | for (i = 0; i < adev->gfx.num_compute_rings; i++) { |
| 6732 | 6720 | ring = &adev->gfx.compute_ring[i]; |
| 6733 | 6721 | if (ring->me == me_id && ring->pipe == pipe_id && |
| 6734 | − ring->queue == queue_id) | |
| 6722 | + ring->queue == queue_id) { | |
| 6735 | 6723 | drm_sched_fault(&ring->sched); |
| 6724 | + return; | |
| 6725 | + } | |
| 6736 | 6726 | } |
| 6737 | 6727 | break; |
| 6738 | 6728 | default: |
| @@ -6740,6 +6730,11 @@ static void gfx_v11_0_handle_priv_fault(struct amdgpu_device *adev, · 1 unmodified line | ||
| 6740 | 6730 | break; |
| 6741 | 6731 | } |
| 6742 | 6732 | } |
| 6733 | + | |
| 6734 | + /* No KQ matched: HW slot is a MES-scheduled user queue. */ | |
| 6735 | + if (adev->enable_mes && doorbell_offset) | |
| 6736 | + amdgpu_userq_process_reset_irq(adev, entry->pasid, | |
| 6737 | + doorbell_offset); | |
| 6743 | 6738 | } |
| 6744 | 6739 | |
| 6745 | 6740 | static int gfx_v11_0_priv_reg_irq(struct amdgpu_device *adev, |
| @@ -6846,233 +6841,14 @@ static void gfx_v11_0_emit_mem_sync(struct amdgpu_ring *ring) · 100 unmodified lines | ||
| 6846 | 6841 | amdgpu_ring_write(ring, gcr_cntl); /* GCR_CNTL */ |
| 6847 | 6842 | } |
| 6848 | 6843 | |
| 6849 | −static bool gfx_v11_pipe_reset_support(struct amdgpu_device *adev) | |
| 6850 | −{ | |
| 6851 | − /* Disable the pipe reset until the CPFW fully support it.*/ | |
| 6852 | − dev_warn_once(adev->dev, "The CPFW hasn't support pipe reset yet.\n"); | |
| 6853 | − return false; | |
| 6854 | −} | |
| 6855 | − | |
| 6856 | − | |
| 6857 | −static int gfx_v11_reset_gfx_pipe(struct amdgpu_ring *ring) | |
| 6858 | −{ | |
| 6859 | − struct amdgpu_device *adev = ring->adev; | |
| 6860 | − uint32_t reset_pipe = 0, clean_pipe = 0; | |
| 6861 | − int r; | |
| 6862 | − | |
| 6863 | − if (!gfx_v11_pipe_reset_support(adev)) | |
| 6864 | − return -EOPNOTSUPP; | |
| 6865 | − | |
| 6866 | − gfx_v11_0_set_safe_mode(adev, 0); | |
| 6867 | − mutex_lock(&adev->srbm_mutex); | |
| 6868 | − soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); | |
| 6869 | − | |
| 6870 | − switch (ring->pipe) { | |
| 6871 | − case 0: | |
| 6872 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 6873 | − PFP_PIPE0_RESET, 1); | |
| 6874 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 6875 | − ME_PIPE0_RESET, 1); | |
| 6876 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 6877 | − PFP_PIPE0_RESET, 0); | |
| 6878 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 6879 | − ME_PIPE0_RESET, 0); | |
| 6880 | − break; | |
| 6881 | − case 1: | |
| 6882 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 6883 | − PFP_PIPE1_RESET, 1); | |
| 6884 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 6885 | − ME_PIPE1_RESET, 1); | |
| 6886 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 6887 | − PFP_PIPE1_RESET, 0); | |
| 6888 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 6889 | − ME_PIPE1_RESET, 0); | |
| 6890 | − break; | |
| 6891 | − default: | |
| 6892 | − break; | |
| 6893 | − } | |
| 6894 | − | |
| 6895 | − WREG32_SOC15(GC, 0, regCP_ME_CNTL, reset_pipe); | |
| 6896 | − WREG32_SOC15(GC, 0, regCP_ME_CNTL, clean_pipe); | |
| 6897 | − | |
| 6898 | − r = (RREG32(SOC15_REG_OFFSET(GC, 0, regCP_GFX_RS64_INSTR_PNTR1)) << 2) - | |
| 6899 | − RS64_FW_UC_START_ADDR_LO; | |
| 6900 | − soc21_grbm_select(adev, 0, 0, 0, 0); | |
| 6901 | − mutex_unlock(&adev->srbm_mutex); | |
| 6902 | − gfx_v11_0_unset_safe_mode(adev, 0); | |
| 6903 | − | |
| 6904 | − dev_info(adev->dev, "The ring %s pipe reset to the ME firmware start PC: %s\n", ring->name, | |
| 6905 | − r == 0 ? "successfully" : "failed"); | |
| 6906 | − /* FIXME: Sometimes driver can't cache the ME firmware start PC correctly, | |
| 6907 | − * so the pipe reset status relies on the later gfx ring test result. | |
| 6908 | − */ | |
| 6909 | − return 0; | |
| 6910 | −} | |
| 6911 | − | |
| 6912 | 6844 | static int gfx_v11_0_reset_kgq(struct amdgpu_ring *ring, |
| 6913 | 6845 | unsigned int vmid, |
| 6914 | 6846 | struct amdgpu_fence *timedout_fence) |
| 6915 | 6847 | { |
| 6916 | 6848 | struct amdgpu_device *adev = ring->adev; |
| 6917 | − bool use_mmio = false; | |
| 6918 | − int r; | |
| 6849 | + bool use_mmio = adev->gfx.me.use_mmio_for_reset; | |
| 6919 | 6850 | |
| 6920 | − amdgpu_ring_reset_helper_begin(ring, timedout_fence); | |
| 6921 | − | |
| 6922 | − r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, use_mmio, 0); | |
| 6923 | − if (r) { | |
| 6924 | − | |
| 6925 | − dev_warn(adev->dev, "reset via MES failed and try pipe reset %d\n", r); | |
| 6926 | − r = gfx_v11_reset_gfx_pipe(ring); | |
| 6927 | − if (r) | |
| 6928 | − return r; | |
| 6929 | − } | |
| 6930 | − | |
| 6931 | − if (use_mmio) { | |
| 6932 | − r = gfx_v11_0_kgq_init_queue(ring, true); | |
| 6933 | − if (r) { | |
| 6934 | − dev_err(adev->dev, "failed to init kgq\n"); | |
| 6935 | − return r; | |
| 6936 | − } | |
| 6937 | − | |
| 6938 | − r = amdgpu_mes_map_legacy_queue(adev, ring, 0); | |
| 6939 | − if (r) { | |
| 6940 | − dev_err(adev->dev, "failed to remap kgq\n"); | |
| 6941 | − return r; | |
| 6942 | − } | |
| 6943 | − } | |
| 6944 | − | |
| 6945 | − return amdgpu_ring_reset_helper_end(ring, timedout_fence); | |
| 6946 | −} | |
| 6947 | − | |
| 6948 | −static int gfx_v11_0_reset_compute_pipe(struct amdgpu_ring *ring) | |
| 6949 | −{ | |
| 6950 | − | |
| 6951 | − struct amdgpu_device *adev = ring->adev; | |
| 6952 | − uint32_t reset_pipe = 0, clean_pipe = 0; | |
| 6953 | − int r; | |
| 6954 | − | |
| 6955 | − if (!gfx_v11_pipe_reset_support(adev)) | |
| 6956 | − return -EOPNOTSUPP; | |
| 6957 | − | |
| 6958 | − gfx_v11_0_set_safe_mode(adev, 0); | |
| 6959 | − mutex_lock(&adev->srbm_mutex); | |
| 6960 | − soc21_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); | |
| 6961 | − | |
| 6962 | − reset_pipe = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); | |
| 6963 | − clean_pipe = reset_pipe; | |
| 6964 | − | |
| 6965 | − if (adev->gfx.rs64_enable) { | |
| 6966 | − | |
| 6967 | − switch (ring->pipe) { | |
| 6968 | − case 0: | |
| 6969 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL, | |
| 6970 | − MEC_PIPE0_RESET, 1); | |
| 6971 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL, | |
| 6972 | − MEC_PIPE0_RESET, 0); | |
| 6973 | − break; | |
| 6974 | − case 1: | |
| 6975 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL, | |
| 6976 | − MEC_PIPE1_RESET, 1); | |
| 6977 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL, | |
| 6978 | − MEC_PIPE1_RESET, 0); | |
| 6979 | − break; | |
| 6980 | − case 2: | |
| 6981 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL, | |
| 6982 | − MEC_PIPE2_RESET, 1); | |
| 6983 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL, | |
| 6984 | − MEC_PIPE2_RESET, 0); | |
| 6985 | − break; | |
| 6986 | − case 3: | |
| 6987 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL, | |
| 6988 | − MEC_PIPE3_RESET, 1); | |
| 6989 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL, | |
| 6990 | − MEC_PIPE3_RESET, 0); | |
| 6991 | − break; | |
| 6992 | − default: | |
| 6993 | − break; | |
| 6994 | − } | |
| 6995 | − WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, reset_pipe); | |
| 6996 | − WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, clean_pipe); | |
| 6997 | − r = (RREG32_SOC15(GC, 0, regCP_MEC_RS64_INSTR_PNTR) << 2) - | |
| 6998 | − RS64_FW_UC_START_ADDR_LO; | |
| 6999 | − } else { | |
| 7000 | − if (ring->me == 1) { | |
| 7001 | − switch (ring->pipe) { | |
| 7002 | − case 0: | |
| 7003 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, | |
| 7004 | − MEC_ME1_PIPE0_RESET, 1); | |
| 7005 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, | |
| 7006 | − MEC_ME1_PIPE0_RESET, 0); | |
| 7007 | − break; | |
| 7008 | − case 1: | |
| 7009 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, | |
| 7010 | − MEC_ME1_PIPE1_RESET, 1); | |
| 7011 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, | |
| 7012 | − MEC_ME1_PIPE1_RESET, 0); | |
| 7013 | − break; | |
| 7014 | − case 2: | |
| 7015 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, | |
| 7016 | − MEC_ME1_PIPE2_RESET, 1); | |
| 7017 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, | |
| 7018 | − MEC_ME1_PIPE2_RESET, 0); | |
| 7019 | − break; | |
| 7020 | − case 3: | |
| 7021 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, | |
| 7022 | − MEC_ME1_PIPE3_RESET, 1); | |
| 7023 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, | |
| 7024 | − MEC_ME1_PIPE3_RESET, 0); | |
| 7025 | − break; | |
| 7026 | − default: | |
| 7027 | − break; | |
| 7028 | − } | |
| 7029 | − /* mec1 fw pc: CP_MEC1_INSTR_PNTR */ | |
| 7030 | − } else { | |
| 7031 | − switch (ring->pipe) { | |
| 7032 | − case 0: | |
| 7033 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, | |
| 7034 | − MEC_ME2_PIPE0_RESET, 1); | |
| 7035 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, | |
| 7036 | − MEC_ME2_PIPE0_RESET, 0); | |
| 7037 | − break; | |
| 7038 | − case 1: | |
| 7039 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, | |
| 7040 | − MEC_ME2_PIPE1_RESET, 1); | |
| 7041 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, | |
| 7042 | − MEC_ME2_PIPE1_RESET, 0); | |
| 7043 | − break; | |
| 7044 | − case 2: | |
| 7045 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, | |
| 7046 | − MEC_ME2_PIPE2_RESET, 1); | |
| 7047 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, | |
| 7048 | − MEC_ME2_PIPE2_RESET, 0); | |
| 7049 | − break; | |
| 7050 | − case 3: | |
| 7051 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, | |
| 7052 | − MEC_ME2_PIPE3_RESET, 1); | |
| 7053 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, | |
| 7054 | − MEC_ME2_PIPE3_RESET, 0); | |
| 7055 | − break; | |
| 7056 | − default: | |
| 7057 | − break; | |
| 7058 | − } | |
| 7059 | − /* mec2 fw pc: CP:CP_MEC2_INSTR_PNTR */ | |
| 7060 | − } | |
| 7061 | − WREG32_SOC15(GC, 0, regCP_MEC_CNTL, reset_pipe); | |
| 7062 | − WREG32_SOC15(GC, 0, regCP_MEC_CNTL, clean_pipe); | |
| 7063 | − r = RREG32(SOC15_REG_OFFSET(GC, 0, regCP_MEC1_INSTR_PNTR)); | |
| 7064 | − } | |
| 7065 | − | |
| 7066 | − soc21_grbm_select(adev, 0, 0, 0, 0); | |
| 7067 | − mutex_unlock(&adev->srbm_mutex); | |
| 7068 | − gfx_v11_0_unset_safe_mode(adev, 0); | |
| 7069 | − | |
| 7070 | − dev_info(adev->dev, "The ring %s pipe resets to MEC FW start PC: %s\n", ring->name, | |
| 7071 | − r == 0 ? "successfully" : "failed"); | |
| 7072 | − /*FIXME:Sometimes driver can't cache the MEC firmware start PC correctly, so the pipe | |
| 7073 | − * reset status relies on the compute ring test result. | |
| 7074 | − */ | |
| 7075 | − return 0; | |
| 6851 | + return amdgpu_gfx_mes_reset_queue(ring, vmid, timedout_fence, use_mmio); | |
| 7076 | 6852 | } |
| 7077 | 6853 | |
| 7078 | 6854 | static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring, |
| @@ -7080,30 +6856,8 @@ static int gfx_v11_0_reset_kcq(struct amdgpu_ring *ring, · 1 unmodified line | ||
| 7080 | 6856 | struct amdgpu_fence *timedout_fence) |
| 7081 | 6857 | { |
| 7082 | 6858 | struct amdgpu_device *adev = ring->adev; |
| 7083 | − int r = 0; | |
| 7084 | − | |
| 7085 | − amdgpu_ring_reset_helper_begin(ring, timedout_fence); | |
| 7086 | − | |
| 7087 | − r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, true, 0); | |
| 7088 | − if (r) { | |
| 7089 | − dev_warn(adev->dev, "fail(%d) to reset kcq and try pipe reset\n", r); | |
| 7090 | − r = gfx_v11_0_reset_compute_pipe(ring); | |
| 7091 | − if (r) | |
| 7092 | − return r; | |
| 7093 | − } | |
| 7094 | − | |
| 7095 | − r = gfx_v11_0_kcq_init_queue(ring, true); | |
| 7096 | − if (r) { | |
| 7097 | − dev_err(adev->dev, "fail to init kcq\n"); | |
| 7098 | − return r; | |
| 7099 | − } | |
| 7100 | − r = amdgpu_mes_map_legacy_queue(adev, ring, 0); | |
| 7101 | − if (r) { | |
| 7102 | − dev_err(adev->dev, "failed to remap kcq\n"); | |
| 7103 | − return r; | |
| 7104 | − } | |
| 7105 | 6859 | |
| 7106 | − return amdgpu_ring_reset_helper_end(ring, timedout_fence); | |
| 6860 | + return amdgpu_gfx_reset_mes_compute(adev, ring, timedout_fence, NULL, NULL, NULL); | |
| 7107 | 6861 | } |
| 7108 | 6862 | |
| 7109 | 6863 | static void gfx_v11_ip_print(struct amdgpu_ip_block *ip_block, struct drm_printer *p) |
| @@ -7281,8 +7035,6 @@ static const struct amd_ip_funcs gfx_v11_0_ip_funcs = { · 171 unmodified lines | ||
| 7281 | 7035 | .is_idle = gfx_v11_0_is_idle, |
| 7282 | 7036 | .wait_for_idle = gfx_v11_0_wait_for_idle, |
| 7283 | 7037 | .soft_reset = gfx_v11_0_soft_reset, |
| 7284 | − .check_soft_reset = gfx_v11_0_check_soft_reset, | |
| 7285 | − .post_soft_reset = gfx_v11_0_post_soft_reset, | |
| 7286 | 7038 | .set_clockgating_state = gfx_v11_0_set_clockgating_state, |
| 7287 | 7039 | .set_powergating_state = gfx_v11_0_set_powergating_state, |
| 7288 | 7040 | .get_clockgating_state = gfx_v11_0_get_clockgating_state, |
| @@ -1603,6 +1603,11 @@ static int gfx_v12_0_sw_init(struct amdgpu_ip_block *ip_block) · 1602 unmodified lines | ||
| 1603 | 1603 | if (r) |
| 1604 | 1604 | return r; |
| 1605 | 1605 | |
| 1606 | + adev->gfx.me.use_mmio_for_reset = false; | |
| 1607 | + adev->gfx.mec.use_mmio_for_reset = true; | |
| 1608 | + | |
| 1609 | + mutex_init(&adev->gfx.mec.reset_mutex); | |
| 1610 | + | |
| 1606 | 1611 | return 0; |
| 1607 | 1612 | } |
| 1608 | 1613 | |
| @@ -3071,13 +3076,13 @@ static int gfx_v12_0_gfx_mqd_init(struct amdgpu_device *adev, void *m, · 1462 unmodified lines | ||
| 3071 | 3076 | return 0; |
| 3072 | 3077 | } |
| 3073 | 3078 | |
| 3074 | −static int gfx_v12_0_kgq_init_queue(struct amdgpu_ring *ring, bool reset) | |
| 3079 | +static int gfx_v12_0_kgq_init_queue(struct amdgpu_ring *ring) | |
| 3075 | 3080 | { |
| 3076 | 3081 | struct amdgpu_device *adev = ring->adev; |
| 3077 | 3082 | struct v12_gfx_mqd *mqd = ring->mqd_ptr; |
| 3078 | 3083 | int mqd_idx = ring - &adev->gfx.gfx_ring[0]; |
| 3079 | 3084 | |
| 3080 | − if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) { | |
| 3085 | + if (!amdgpu_in_reset(adev) && !adev->in_suspend) { | |
| 3081 | 3086 | memset((void *)mqd, 0, sizeof(*mqd)); |
| 3082 | 3087 | mutex_lock(&adev->srbm_mutex); |
| 3083 | 3088 | soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); |
| @@ -3104,7 +3109,7 @@ static int gfx_v12_0_cp_async_gfx_ring_resume(struct amdgpu_device *adev) · 20 unmodified lines | ||
| 3104 | 3109 | int i, r; |
| 3105 | 3110 | |
| 3106 | 3111 | for (i = 0; i < adev->gfx.num_gfx_rings; i++) { |
| 3107 | − r = gfx_v12_0_kgq_init_queue(&adev->gfx.gfx_ring[i], false); | |
| 3112 | + r = gfx_v12_0_kgq_init_queue(&adev->gfx.gfx_ring[i]); | |
| 3108 | 3113 | if (r) |
| 3109 | 3114 | return r; |
| 3110 | 3115 | } |
| @@ -3441,13 +3446,13 @@ static int gfx_v12_0_kiq_init_queue(struct amdgpu_ring *ring) · 330 unmodified lines | ||
| 3441 | 3446 | return 0; |
| 3442 | 3447 | } |
| 3443 | 3448 | |
| 3444 | −static int gfx_v12_0_kcq_init_queue(struct amdgpu_ring *ring, bool reset) | |
| 3449 | +static int gfx_v12_0_kcq_init_queue(struct amdgpu_ring *ring) | |
| 3445 | 3450 | { |
| 3446 | 3451 | struct amdgpu_device *adev = ring->adev; |
| 3447 | 3452 | struct v12_compute_mqd *mqd = ring->mqd_ptr; |
| 3448 | 3453 | int mqd_idx = ring - &adev->gfx.compute_ring[0]; |
| 3449 | 3454 | |
| 3450 | − if (!reset && !amdgpu_in_reset(adev) && !adev->in_suspend) { | |
| 3455 | + if (!amdgpu_in_reset(adev) && !adev->in_suspend) { | |
| 3451 | 3456 | memset((void *)mqd, 0, sizeof(*mqd)); |
| 3452 | 3457 | mutex_lock(&adev->srbm_mutex); |
| 3453 | 3458 | soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); |
| @@ -3485,7 +3490,7 @@ static int gfx_v12_0_kcq_resume(struct amdgpu_device *adev) · 31 unmodified lines | ||
| 3485 | 3490 | gfx_v12_0_cp_compute_enable(adev, true); |
| 3486 | 3491 | |
| 3487 | 3492 | for (i = 0; i < adev->gfx.num_compute_rings; i++) { |
| 3488 | − r = gfx_v12_0_kcq_init_queue(&adev->gfx.compute_ring[i], false); | |
| 3493 | + r = gfx_v12_0_kcq_init_queue(&adev->gfx.compute_ring[i]); | |
| 3489 | 3494 | if (r) |
| 3490 | 3495 | return r; |
| 3491 | 3496 | } |
| @@ -3986,6 +3991,8 @@ static int gfx_v12_0_early_init(struct amdgpu_ip_block *ip_block) · 494 unmodified lines | ||
| 3986 | 3991 | |
| 3987 | 3992 | gfx_v12_0_init_rlcg_reg_access_ctrl(adev); |
| 3988 | 3993 | |
| 3994 | + amdgpu_init_rlc_reg_funcs(adev); | |
| 3995 | + | |
| 3989 | 3996 | return gfx_v12_0_init_microcode(adev); |
| 3990 | 3997 | } |
| 3991 | 3998 | |
| @@ -5025,22 +5032,30 @@ static int gfx_v12_0_set_priv_inst_fault_state(struct amdgpu_device *adev, · 1033 unmodified lines | ||
| 5025 | 5032 | static void gfx_v12_0_handle_priv_fault(struct amdgpu_device *adev, |
| 5026 | 5033 | struct amdgpu_iv_entry *entry) |
| 5027 | 5034 | { |
| 5028 | − u8 me_id, pipe_id, queue_id; | |
| 5029 | − struct amdgpu_ring *ring; | |
| 5030 | − int i; | |
| 5031 | − | |
| 5032 | − me_id = (entry->ring_id & 0x0c) >> 2; | |
| 5033 | − pipe_id = (entry->ring_id & 0x03) >> 0; | |
| 5034 | − queue_id = (entry->ring_id & 0x70) >> 4; | |
| 5035 | + u32 doorbell_offset = entry->src_data[0] & AMDGPU_CTXID0_DOORBELL_ID_MASK; | |
| 5035 | 5036 | |
| 5037 | + /* | |
| 5038 | + * Try KQ first by ring_id; UQ as fallback. KCQ and UQ never share | |
| 5039 | + * a HW slot (compute_hqd_mask contract). | |
| 5040 | + */ | |
| 5036 | 5041 | if (!adev->gfx.disable_kq) { |
| 5042 | + u8 me_id, pipe_id, queue_id; | |
| 5043 | + struct amdgpu_ring *ring; | |
| 5044 | + int i; | |
| 5045 | + | |
| 5046 | + me_id = (entry->ring_id & 0x0c) >> 2; | |
| 5047 | + pipe_id = (entry->ring_id & 0x03) >> 0; | |
| 5048 | + queue_id = (entry->ring_id & 0x70) >> 4; | |
| 5049 | + | |
| 5037 | 5050 | switch (me_id) { |
| 5038 | 5051 | case 0: |
| 5039 | 5052 | for (i = 0; i < adev->gfx.num_gfx_rings; i++) { |
| 5040 | 5053 | ring = &adev->gfx.gfx_ring[i]; |
| 5041 | 5054 | if (ring->me == me_id && ring->pipe == pipe_id && |
| 5042 | − ring->queue == queue_id) | |
| 5055 | + ring->queue == queue_id) { | |
| 5043 | 5056 | drm_sched_fault(&ring->sched); |
| 5057 | + return; | |
| 5058 | + } | |
| 5044 | 5059 | } |
| 5045 | 5060 | break; |
| 5046 | 5061 | case 1: |
| @@ -5048,8 +5063,10 @@ static void gfx_v12_0_handle_priv_fault(struct amdgpu_device *adev, · 1 unmodified line | ||
| 5048 | 5063 | for (i = 0; i < adev->gfx.num_compute_rings; i++) { |
| 5049 | 5064 | ring = &adev->gfx.compute_ring[i]; |
| 5050 | 5065 | if (ring->me == me_id && ring->pipe == pipe_id && |
| 5051 | − ring->queue == queue_id) | |
| 5066 | + ring->queue == queue_id) { | |
| 5052 | 5067 | drm_sched_fault(&ring->sched); |
| 5068 | + return; | |
| 5069 | + } | |
| 5053 | 5070 | } |
| 5054 | 5071 | break; |
| 5055 | 5072 | default: |
| @@ -5057,6 +5074,11 @@ static void gfx_v12_0_handle_priv_fault(struct amdgpu_device *adev, · 1 unmodified line | ||
| 5057 | 5074 | break; |
| 5058 | 5075 | } |
| 5059 | 5076 | } |
| 5077 | + | |
| 5078 | + /* No KQ matched: HW slot is a MES-scheduled user queue. */ | |
| 5079 | + if (adev->enable_mes && doorbell_offset) | |
| 5080 | + amdgpu_userq_process_reset_irq(adev, entry->pasid, | |
| 5081 | + doorbell_offset); | |
| 5060 | 5082 | } |
| 5061 | 5083 | |
| 5062 | 5084 | static int gfx_v12_0_priv_reg_irq(struct amdgpu_device *adev, |
| @@ -5261,185 +5283,14 @@ static void gfx_v12_ip_dump(struct amdgpu_ip_block *ip_block) · 198 unmodified lines | ||
| 5261 | 5283 | amdgpu_gfx_off_ctrl(adev, true); |
| 5262 | 5284 | } |
| 5263 | 5285 | |
| 5264 | −static bool gfx_v12_pipe_reset_support(struct amdgpu_device *adev) | |
| 5265 | −{ | |
| 5266 | − /* Disable the pipe reset until the CPFW fully support it.*/ | |
| 5267 | − dev_warn_once(adev->dev, "The CPFW hasn't support pipe reset yet.\n"); | |
| 5268 | − return false; | |
| 5269 | −} | |
| 5270 | − | |
| 5271 | −static int gfx_v12_reset_gfx_pipe(struct amdgpu_ring *ring) | |
| 5272 | −{ | |
| 5273 | − struct amdgpu_device *adev = ring->adev; | |
| 5274 | − uint32_t reset_pipe = 0, clean_pipe = 0; | |
| 5275 | − int r; | |
| 5276 | − | |
| 5277 | − if (!gfx_v12_pipe_reset_support(adev)) | |
| 5278 | − return -EOPNOTSUPP; | |
| 5279 | − | |
| 5280 | − gfx_v12_0_set_safe_mode(adev, 0); | |
| 5281 | − mutex_lock(&adev->srbm_mutex); | |
| 5282 | − soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); | |
| 5283 | − | |
| 5284 | − switch (ring->pipe) { | |
| 5285 | − case 0: | |
| 5286 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 5287 | − PFP_PIPE0_RESET, 1); | |
| 5288 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 5289 | − ME_PIPE0_RESET, 1); | |
| 5290 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 5291 | − PFP_PIPE0_RESET, 0); | |
| 5292 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 5293 | − ME_PIPE0_RESET, 0); | |
| 5294 | − break; | |
| 5295 | − case 1: | |
| 5296 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 5297 | − PFP_PIPE1_RESET, 1); | |
| 5298 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 5299 | − ME_PIPE1_RESET, 1); | |
| 5300 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 5301 | − PFP_PIPE1_RESET, 0); | |
| 5302 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 5303 | − ME_PIPE1_RESET, 0); | |
| 5304 | − break; | |
| 5305 | − default: | |
| 5306 | − break; | |
| 5307 | − } | |
| 5308 | − | |
| 5309 | − WREG32_SOC15(GC, 0, regCP_ME_CNTL, reset_pipe); | |
| 5310 | − WREG32_SOC15(GC, 0, regCP_ME_CNTL, clean_pipe); | |
| 5311 | − | |
| 5312 | − r = (RREG32(SOC15_REG_OFFSET(GC, 0, regCP_GFX_RS64_INSTR_PNTR1)) << 2) - | |
| 5313 | − RS64_FW_UC_START_ADDR_LO; | |
| 5314 | − soc24_grbm_select(adev, 0, 0, 0, 0); | |
| 5315 | − mutex_unlock(&adev->srbm_mutex); | |
| 5316 | − gfx_v12_0_unset_safe_mode(adev, 0); | |
| 5317 | − | |
| 5318 | − dev_info(adev->dev, "The ring %s pipe reset: %s\n", ring->name, | |
| 5319 | − r == 0 ? "successfully" : "failed"); | |
| 5320 | − /* Sometimes the ME start pc counter can't cache correctly, so the | |
| 5321 | − * PC check only as a reference and pipe reset result rely on the | |
| 5322 | − * later ring test. | |
| 5323 | − */ | |
| 5324 | − return 0; | |
| 5325 | −} | |
| 5326 | − | |
| 5327 | 5286 | static int gfx_v12_0_reset_kgq(struct amdgpu_ring *ring, |
| 5328 | 5287 | unsigned int vmid, |
| 5329 | 5288 | struct amdgpu_fence *timedout_fence) |
| 5330 | 5289 | { |
| 5331 | 5290 | struct amdgpu_device *adev = ring->adev; |
| 5332 | − bool use_mmio = false; | |
| 5333 | − int r; | |
| 5334 | − | |
| 5335 | − amdgpu_ring_reset_helper_begin(ring, timedout_fence); | |
| 5336 | − | |
| 5337 | − r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, use_mmio, 0); | |
| 5338 | − if (r) { | |
| 5339 | − dev_warn(adev->dev, "reset via MES failed and try pipe reset %d\n", r); | |
| 5340 | − r = gfx_v12_reset_gfx_pipe(ring); | |
| 5341 | − if (r) | |
| 5342 | − return r; | |
| 5343 | − } | |
| 5344 | − | |
| 5345 | − if (use_mmio) { | |
| 5346 | − r = gfx_v12_0_kgq_init_queue(ring, true); | |
| 5347 | − if (r) { | |
| 5348 | − dev_err(adev->dev, "failed to init kgq\n"); | |
| 5349 | − return r; | |
| 5350 | − } | |
| 5351 | − | |
| 5352 | − r = amdgpu_mes_map_legacy_queue(adev, ring, 0); | |
| 5353 | − if (r) { | |
| 5354 | − dev_err(adev->dev, "failed to remap kgq\n"); | |
| 5355 | − return r; | |
| 5356 | − } | |
| 5357 | − } | |
| 5358 | − | |
| 5359 | − return amdgpu_ring_reset_helper_end(ring, timedout_fence); | |
| 5360 | −} | |
| 5361 | − | |
| 5362 | −static int gfx_v12_0_reset_compute_pipe(struct amdgpu_ring *ring) | |
| 5363 | −{ | |
| 5364 | − struct amdgpu_device *adev = ring->adev; | |
| 5365 | − uint32_t reset_pipe = 0, clean_pipe = 0; | |
| 5366 | − int r = 0; | |
| 5367 | − | |
| 5368 | − if (!gfx_v12_pipe_reset_support(adev)) | |
| 5369 | − return -EOPNOTSUPP; | |
| 5370 | − | |
| 5371 | − gfx_v12_0_set_safe_mode(adev, 0); | |
| 5372 | − mutex_lock(&adev->srbm_mutex); | |
| 5373 | − soc24_grbm_select(adev, ring->me, ring->pipe, ring->queue, 0); | |
| 5374 | − | |
| 5375 | − reset_pipe = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); | |
| 5376 | − clean_pipe = reset_pipe; | |
| 5291 | + bool use_mmio = adev->gfx.me.use_mmio_for_reset; | |
| 5377 | 5292 | |
| 5378 | − if (adev->gfx.rs64_enable) { | |
| 5379 | − switch (ring->pipe) { | |
| 5380 | − case 0: | |
| 5381 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL, | |
| 5382 | − MEC_PIPE0_RESET, 1); | |
| 5383 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL, | |
| 5384 | − MEC_PIPE0_RESET, 0); | |
| 5385 | − break; | |
| 5386 | − case 1: | |
| 5387 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL, | |
| 5388 | − MEC_PIPE1_RESET, 1); | |
| 5389 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL, | |
| 5390 | − MEC_PIPE1_RESET, 0); | |
| 5391 | − break; | |
| 5392 | − case 2: | |
| 5393 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL, | |
| 5394 | − MEC_PIPE2_RESET, 1); | |
| 5395 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL, | |
| 5396 | − MEC_PIPE2_RESET, 0); | |
| 5397 | − break; | |
| 5398 | − case 3: | |
| 5399 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_RS64_CNTL, | |
| 5400 | − MEC_PIPE3_RESET, 1); | |
| 5401 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_RS64_CNTL, | |
| 5402 | − MEC_PIPE3_RESET, 0); | |
| 5403 | − break; | |
| 5404 | − default: | |
| 5405 | − break; | |
| 5406 | − } | |
| 5407 | − WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, reset_pipe); | |
| 5408 | − WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, clean_pipe); | |
| 5409 | − r = (RREG32_SOC15(GC, 0, regCP_MEC_RS64_INSTR_PNTR) << 2) - | |
| 5410 | − RS64_FW_UC_START_ADDR_LO; | |
| 5411 | − } else { | |
| 5412 | − switch (ring->pipe) { | |
| 5413 | − case 0: | |
| 5414 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, | |
| 5415 | − MEC_ME1_PIPE0_RESET, 1); | |
| 5416 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, | |
| 5417 | − MEC_ME1_PIPE0_RESET, 0); | |
| 5418 | − break; | |
| 5419 | − case 1: | |
| 5420 | − reset_pipe = REG_SET_FIELD(reset_pipe, CP_MEC_CNTL, | |
| 5421 | − MEC_ME1_PIPE1_RESET, 1); | |
| 5422 | − clean_pipe = REG_SET_FIELD(clean_pipe, CP_MEC_CNTL, | |
| 5423 | − MEC_ME1_PIPE1_RESET, 0); | |
| 5424 | − break; | |
| 5425 | − default: | |
| 5426 | − break; | |
| 5427 | − } | |
| 5428 | − WREG32_SOC15(GC, 0, regCP_MEC_CNTL, reset_pipe); | |
| 5429 | − WREG32_SOC15(GC, 0, regCP_MEC_CNTL, clean_pipe); | |
| 5430 | − /* Doesn't find the F32 MEC instruction pointer register, and suppose | |
| 5431 | − * the driver won't run into the F32 mode. | |
| 5432 | − */ | |
| 5433 | − } | |
| 5434 | − | |
| 5435 | − soc24_grbm_select(adev, 0, 0, 0, 0); | |
| 5436 | − mutex_unlock(&adev->srbm_mutex); | |
| 5437 | − gfx_v12_0_unset_safe_mode(adev, 0); | |
| 5438 | − | |
| 5439 | − dev_info(adev->dev, "The ring %s pipe resets: %s\n", ring->name, | |
| 5440 | − r == 0 ? "successfully" : "failed"); | |
| 5441 | − /* Need the ring test to verify the pipe reset result.*/ | |
| 5442 | − return 0; | |
| 5293 | + return amdgpu_gfx_mes_reset_queue(ring, vmid, timedout_fence, use_mmio); | |
| 5443 | 5294 | } |
| 5444 | 5295 | |
| 5445 | 5296 | static int gfx_v12_0_reset_kcq(struct amdgpu_ring *ring, |
| @@ -5447,30 +5298,8 @@ static int gfx_v12_0_reset_kcq(struct amdgpu_ring *ring, · 1 unmodified line | ||
| 5447 | 5298 | struct amdgpu_fence *timedout_fence) |
| 5448 | 5299 | { |
| 5449 | 5300 | struct amdgpu_device *adev = ring->adev; |
| 5450 | − int r; | |
| 5451 | − | |
| 5452 | − amdgpu_ring_reset_helper_begin(ring, timedout_fence); | |
| 5453 | − | |
| 5454 | − r = amdgpu_mes_reset_legacy_queue(ring->adev, ring, vmid, true, 0); | |
| 5455 | − if (r) { | |
| 5456 | − dev_warn(adev->dev, "fail(%d) to reset kcq and try pipe reset\n", r); | |
| 5457 | − r = gfx_v12_0_reset_compute_pipe(ring); | |
| 5458 | − if (r) | |
| 5459 | − return r; | |
| 5460 | − } | |
| 5461 | − | |
| 5462 | − r = gfx_v12_0_kcq_init_queue(ring, true); | |
| 5463 | − if (r) { | |
| 5464 | − dev_err(adev->dev, "failed to init kcq\n"); | |
| 5465 | − return r; | |
| 5466 | − } | |
| 5467 | − r = amdgpu_mes_map_legacy_queue(adev, ring, 0); | |
| 5468 | − if (r) { | |
| 5469 | − dev_err(adev->dev, "failed to remap kcq\n"); | |
| 5470 | − return r; | |
| 5471 | − } | |
| 5472 | 5301 | |
| 5473 | − return amdgpu_ring_reset_helper_end(ring, timedout_fence); | |
| 5302 | + return amdgpu_gfx_reset_mes_compute(adev, ring, timedout_fence, NULL, NULL, NULL); | |
| 5474 | 5303 | } |
| 5475 | 5304 | |
| 5476 | 5305 | static void gfx_v12_0_ring_begin_use(struct amdgpu_ring *ring) |
| @@ -1287,6 +1287,8 @@ static int gfx_v12_1_sw_init(struct amdgpu_ip_block *ip_block) · 1286 unmodified lines | ||
| 1287 | 1287 | if (r) |
| 1288 | 1288 | return r; |
| 1289 | 1289 | |
| 1290 | + mutex_init(&adev->gfx.mec.reset_mutex); | |
| 1291 | + | |
| 1290 | 1292 | return 0; |
| 1291 | 1293 | } |
| 1292 | 1294 | |
| @@ -3004,6 +3006,8 @@ static int gfx_v12_1_early_init(struct amdgpu_ip_block *ip_block) · 1711 unmodified lines | ||
| 3004 | 3006 | |
| 3005 | 3007 | gfx_v12_1_init_rlcg_reg_access_ctrl(adev); |
| 3006 | 3008 | |
| 3009 | + amdgpu_init_rlc_reg_funcs(adev); | |
| 3010 | + | |
| 3007 | 3011 | return gfx_v12_1_init_microcode(adev); |
| 3008 | 3012 | } |
| 3009 | 3013 | |
| @@ -1487,7 +1487,14 @@ static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev) · 1486 unmodified lines | ||
| 1487 | 1487 | |
| 1488 | 1488 | /* bail if the compute ring is not ready */ |
| 1489 | 1489 | if (!ring->sched.ready) |
| 1490 | − return 0; | |
| 1490 | + return -EBUSY; | |
| 1491 | + | |
| 1492 | + if (amdgpu_in_reset(adev)) { | |
| 1493 | + /* Set preempt condition to execute IB */ | |
| 1494 | + amdgpu_ring_set_preempt_cond_exec(ring, true); | |
| 1495 | + /* Flush HDP cache so the GPU can see the updated COND_EXEC value */ | |
| 1496 | + amdgpu_device_flush_hdp(adev, NULL); | |
| 1497 | + } | |
| 1491 | 1498 | |
| 1492 | 1499 | tmp = RREG32(mmGB_EDC_MODE); |
| 1493 | 1500 | WREG32(mmGB_EDC_MODE, 0); |
| @@ -2028,6 +2035,11 @@ static int gfx_v8_0_sw_init(struct amdgpu_ip_block *ip_block) · 534 unmodified lines | ||
| 2028 | 2035 | adev->gfx.compute_supported_reset = |
| 2029 | 2036 | amdgpu_get_soft_full_reset_mask(&adev->gfx.compute_ring[0]); |
| 2030 | 2037 | |
| 2038 | + if (!amdgpu_sriov_vf(adev) && !adev->debug_disable_ip_block_soft_reset) { | |
| 2039 | + adev->gfx.compute_supported_reset |= AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET; | |
| 2040 | + adev->gfx.gfx_supported_reset |= AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET; | |
| 2041 | + } | |
| 2042 | + | |
| 2031 | 2043 | return 0; |
| 2032 | 2044 | } |
| 2033 | 2045 | |
| @@ -4703,12 +4715,14 @@ static int gfx_v8_0_cp_test_all_rings(struct amdgpu_device *adev) · 2669 unmodified lines | ||
| 4703 | 4715 | if (r) |
| 4704 | 4716 | return r; |
| 4705 | 4717 | |
| 4718 | + r = 0; | |
| 4719 | + | |
| 4706 | 4720 | for (i = 0; i < adev->gfx.num_compute_rings; i++) { |
| 4707 | 4721 | ring = &adev->gfx.compute_ring[i]; |
| 4708 | − amdgpu_ring_test_helper(ring); | |
| 4722 | + r |= amdgpu_ring_test_helper(ring); | |
| 4709 | 4723 | } |
| 4710 | 4724 | |
| 4711 | − return 0; | |
| 4725 | + return r; | |
| 4712 | 4726 | } |
| 4713 | 4727 | |
| 4714 | 4728 | static int gfx_v8_0_cp_resume(struct amdgpu_device *adev) |
| @@ -4868,14 +4882,12 @@ static int gfx_v8_0_hw_fini(struct amdgpu_ip_block *ip_block) · 153 unmodified lines | ||
| 4868 | 4882 | } |
| 4869 | 4883 | |
| 4870 | 4884 | amdgpu_gfx_rlc_enter_safe_mode(adev, 0); |
| 4871 | − if (!gfx_v8_0_wait_for_idle(ip_block)) | |
| 4872 | − gfx_v8_0_cp_enable(adev, false); | |
| 4873 | − else | |
| 4885 | + if (!amdgpu_in_reset(adev) && gfx_v8_0_wait_for_idle(ip_block)) | |
| 4874 | 4886 | pr_err("cp is busy, skip halt cp\n"); |
| 4875 | − if (!gfx_v8_0_wait_for_rlc_idle(adev)) | |
| 4876 | − adev->gfx.rlc.funcs->stop(adev); | |
| 4877 | − else | |
| 4878 | − pr_err("rlc is busy, skip halt rlc\n"); | |
| 4887 | + if (!amdgpu_in_reset(adev) && gfx_v8_0_wait_for_rlc_idle(adev)) | |
| 4888 | + pr_err("rlc is busy\n"); | |
| 4889 | + gfx_v8_0_cp_enable(adev, false); | |
| 4890 | + adev->gfx.rlc.funcs->stop(adev); | |
| 4879 | 4891 | amdgpu_gfx_rlc_exit_safe_mode(adev, 0); |
| 4880 | 4892 | |
| 4881 | 4893 | return 0; |
| @@ -4891,128 +4903,49 @@ static int gfx_v8_0_resume(struct amdgpu_ip_block *ip_block) · 9 unmodified lines | ||
| 4891 | 4903 | return gfx_v8_0_hw_init(ip_block); |
| 4892 | 4904 | } |
| 4893 | 4905 | |
| 4894 | −static bool gfx_v8_0_check_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 4906 | +static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 4895 | 4907 | { |
| 4896 | 4908 | struct amdgpu_device *adev = ip_block->adev; |
| 4897 | 4909 | u32 grbm_soft_reset = 0, srbm_soft_reset = 0; |
| 4898 | 4910 | u32 tmp; |
| 4911 | + int i; | |
| 4912 | + int r; | |
| 4899 | 4913 | |
| 4900 | − /* GRBM_STATUS */ | |
| 4901 | − tmp = RREG32(mmGRBM_STATUS); | |
| 4902 | − if (tmp & (GRBM_STATUS__PA_BUSY_MASK | GRBM_STATUS__SC_BUSY_MASK | | |
| 4903 | − GRBM_STATUS__BCI_BUSY_MASK | GRBM_STATUS__SX_BUSY_MASK | | |
| 4904 | − GRBM_STATUS__TA_BUSY_MASK | GRBM_STATUS__VGT_BUSY_MASK | | |
| 4905 | − GRBM_STATUS__DB_BUSY_MASK | GRBM_STATUS__CB_BUSY_MASK | | |
| 4906 | − GRBM_STATUS__GDS_BUSY_MASK | GRBM_STATUS__SPI_BUSY_MASK | | |
| 4907 | − GRBM_STATUS__IA_BUSY_MASK | GRBM_STATUS__IA_BUSY_NO_DMA_MASK | | |
| 4908 | − GRBM_STATUS__CP_BUSY_MASK | GRBM_STATUS__CP_COHERENCY_BUSY_MASK)) { | |
| 4909 | − grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, | |
| 4910 | − GRBM_SOFT_RESET, SOFT_RESET_CP, 1); | |
| 4911 | − grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, | |
| 4912 | − GRBM_SOFT_RESET, SOFT_RESET_GFX, 1); | |
| 4913 | − srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, | |
| 4914 | − SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1); | |
| 4915 | − } | |
| 4916 | − | |
| 4917 | − /* GRBM_STATUS2 */ | |
| 4918 | − tmp = RREG32(mmGRBM_STATUS2); | |
| 4919 | − if (REG_GET_FIELD(tmp, GRBM_STATUS2, RLC_BUSY)) | |
| 4920 | − grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, | |
| 4921 | − GRBM_SOFT_RESET, SOFT_RESET_RLC, 1); | |
| 4922 | − | |
| 4923 | − if (REG_GET_FIELD(tmp, GRBM_STATUS2, CPF_BUSY) || | |
| 4924 | − REG_GET_FIELD(tmp, GRBM_STATUS2, CPC_BUSY) || | |
| 4925 | − REG_GET_FIELD(tmp, GRBM_STATUS2, CPG_BUSY)) { | |
| 4926 | − grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, | |
| 4927 | − SOFT_RESET_CPF, 1); | |
| 4928 | − grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, | |
| 4929 | − SOFT_RESET_CPC, 1); | |
| 4930 | − grbm_soft_reset = REG_SET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, | |
| 4931 | − SOFT_RESET_CPG, 1); | |
| 4932 | − srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, | |
| 4933 | − SOFT_RESET_GRBM, 1); | |
| 4934 | − } | |
| 4935 | − | |
| 4936 | − /* SRBM_STATUS */ | |
| 4937 | − tmp = RREG32(mmSRBM_STATUS); | |
| 4938 | − if (REG_GET_FIELD(tmp, SRBM_STATUS, GRBM_RQ_PENDING)) | |
| 4939 | − srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, | |
| 4940 | − SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1); | |
| 4941 | − if (REG_GET_FIELD(tmp, SRBM_STATUS, SEM_BUSY)) | |
| 4942 | − srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, | |
| 4943 | − SRBM_SOFT_RESET, SOFT_RESET_SEM, 1); | |
| 4944 | − | |
| 4945 | − if (grbm_soft_reset || srbm_soft_reset) { | |
| 4946 | − adev->gfx.grbm_soft_reset = grbm_soft_reset; | |
| 4947 | − adev->gfx.srbm_soft_reset = srbm_soft_reset; | |
| 4948 | − return true; | |
| 4949 | − } else { | |
| 4950 | − adev->gfx.grbm_soft_reset = 0; | |
| 4951 | − adev->gfx.srbm_soft_reset = 0; | |
| 4952 | − return false; | |
| 4953 | − } | |
| 4954 | −} | |
| 4955 | − | |
| 4956 | −static int gfx_v8_0_pre_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 4957 | −{ | |
| 4958 | − struct amdgpu_device *adev = ip_block->adev; | |
| 4959 | − u32 grbm_soft_reset = 0; | |
| 4960 | − | |
| 4961 | − if ((!adev->gfx.grbm_soft_reset) && | |
| 4962 | − (!adev->gfx.srbm_soft_reset)) | |
| 4963 | − return 0; | |
| 4964 | − | |
| 4965 | − grbm_soft_reset = adev->gfx.grbm_soft_reset; | |
| 4966 | − | |
| 4967 | − /* stop the rlc */ | |
| 4968 | − adev->gfx.rlc.funcs->stop(adev); | |
| 4914 | + grbm_soft_reset = | |
| 4915 | + REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_RLC, 1) | | |
| 4916 | + REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_GFX, 1) | | |
| 4917 | + REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CP, 1) | | |
| 4918 | + REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPF, 1) | | |
| 4919 | + REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPC, 1) | | |
| 4920 | + REG_SET_FIELD(0, GRBM_SOFT_RESET, SOFT_RESET_CPG, 1); | |
| 4969 | 4921 | |
| 4970 | − if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) || | |
| 4971 | − REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX)) | |
| 4972 | − /* Disable GFX parsing/prefetching */ | |
| 4973 | − gfx_v8_0_cp_gfx_enable(adev, false); | |
| 4922 | + srbm_soft_reset = | |
| 4923 | + REG_SET_FIELD(0, SRBM_SOFT_RESET, SOFT_RESET_GRBM, 1) | | |
| 4924 | + REG_SET_FIELD(0, SRBM_SOFT_RESET, SOFT_RESET_SEM, 1); | |
| 4974 | 4925 | |
| 4975 | − if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) || | |
| 4976 | − REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) || | |
| 4977 | − REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) || | |
| 4978 | − REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) { | |
| 4979 | − int i; | |
| 4926 | + for (i = 0; i < adev->gfx.num_compute_rings; i++) { | |
| 4927 | + struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; | |
| 4980 | 4928 | |
| 4981 | − for (i = 0; i < adev->gfx.num_compute_rings; i++) { | |
| 4982 | − struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; | |
| 4929 | + mutex_lock(&adev->srbm_mutex); | |
| 4930 | + vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); | |
| 4931 | + gfx_v8_0_deactivate_hqd(adev, 2); | |
| 4932 | + vi_srbm_select(adev, 0, 0, 0, 0); | |
| 4933 | + mutex_unlock(&adev->srbm_mutex); | |
| 4983 | 4934 | |
| 4984 | − mutex_lock(&adev->srbm_mutex); | |
| 4985 | − vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); | |
| 4986 | − gfx_v8_0_deactivate_hqd(adev, 2); | |
| 4987 | − vi_srbm_select(adev, 0, 0, 0, 0); | |
| 4988 | − mutex_unlock(&adev->srbm_mutex); | |
| 4989 | − } | |
| 4990 | − /* Disable MEC parsing/prefetching */ | |
| 4991 | − gfx_v8_0_cp_compute_enable(adev, false); | |
| 4935 | + udelay(50); | |
| 4992 | 4936 | } |
| 4993 | 4937 | |
| 4994 | − return 0; | |
| 4995 | −} | |
| 4996 | − | |
| 4997 | −static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 4998 | −{ | |
| 4999 | − struct amdgpu_device *adev = ip_block->adev; | |
| 5000 | − u32 grbm_soft_reset = 0, srbm_soft_reset = 0; | |
| 5001 | − u32 tmp; | |
| 5002 | − | |
| 5003 | − if ((!adev->gfx.grbm_soft_reset) && | |
| 5004 | − (!adev->gfx.srbm_soft_reset)) | |
| 5005 | − return 0; | |
| 5006 | − | |
| 5007 | − grbm_soft_reset = adev->gfx.grbm_soft_reset; | |
| 5008 | − srbm_soft_reset = adev->gfx.srbm_soft_reset; | |
| 4938 | + ip_block->version->funcs->set_clockgating_state(ip_block, AMD_CG_STATE_UNGATE); | |
| 4939 | + ip_block->version->funcs->set_powergating_state(ip_block, AMD_PG_STATE_UNGATE); | |
| 4940 | + ip_block->version->funcs->suspend(ip_block); | |
| 5009 | 4941 | |
| 5010 | 4942 | if (grbm_soft_reset || srbm_soft_reset) { |
| 5011 | 4943 | tmp = RREG32(mmGMCON_DEBUG); |
| 5012 | 4944 | tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_STALL, 1); |
| 5013 | 4945 | tmp = REG_SET_FIELD(tmp, GMCON_DEBUG, GFX_CLEAR, 1); |
| 5014 | 4946 | WREG32(mmGMCON_DEBUG, tmp); |
| 5015 | − udelay(50); | |
| 4947 | + | |
| 4948 | + udelay(100); | |
| 5016 | 4949 | } |
| 5017 | 4950 | |
| 5018 | 4951 | if (grbm_soft_reset) { |
| @@ -5022,11 +4955,13 @@ static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block) · 3 unmodified lines | ||
| 5022 | 4955 | WREG32(mmGRBM_SOFT_RESET, tmp); |
| 5023 | 4956 | tmp = RREG32(mmGRBM_SOFT_RESET); |
| 5024 | 4957 | |
| 5025 | − udelay(50); | |
| 4958 | + udelay(100); | |
| 5026 | 4959 | |
| 5027 | 4960 | tmp &= ~grbm_soft_reset; |
| 5028 | 4961 | WREG32(mmGRBM_SOFT_RESET, tmp); |
| 5029 | 4962 | tmp = RREG32(mmGRBM_SOFT_RESET); |
| 4963 | + | |
| 4964 | + udelay(100); | |
| 5030 | 4965 | } |
| 5031 | 4966 | |
| 5032 | 4967 | if (srbm_soft_reset) { |
| @@ -5036,11 +4971,13 @@ static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block) · 3 unmodified lines | ||
| 5036 | 4971 | WREG32(mmSRBM_SOFT_RESET, tmp); |
| 5037 | 4972 | tmp = RREG32(mmSRBM_SOFT_RESET); |
| 5038 | 4973 | |
| 5039 | − udelay(50); | |
| 4974 | + udelay(100); | |
| 5040 | 4975 | |
| 5041 | 4976 | tmp &= ~srbm_soft_reset; |
| 5042 | 4977 | WREG32(mmSRBM_SOFT_RESET, tmp); |
| 5043 | 4978 | tmp = RREG32(mmSRBM_SOFT_RESET); |
| 4979 | + | |
| 4980 | + udelay(100); | |
| 5044 | 4981 | } |
| 5045 | 4982 | |
| 5046 | 4983 | if (grbm_soft_reset || srbm_soft_reset) { |
| @@ -5051,48 +4988,15 @@ static int gfx_v8_0_soft_reset(struct amdgpu_ip_block *ip_block) · 4 unmodified lines | ||
| 5051 | 4988 | } |
| 5052 | 4989 | |
| 5053 | 4990 | /* Wait a little for things to settle down */ |
| 5054 | − udelay(50); | |
| 4991 | + udelay(100); | |
| 5055 | 4992 | |
| 5056 | − return 0; | |
| 5057 | −} | |
| 5058 | − | |
| 5059 | −static int gfx_v8_0_post_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 5060 | −{ | |
| 5061 | − struct amdgpu_device *adev = ip_block->adev; | |
| 5062 | − u32 grbm_soft_reset = 0; | |
| 5063 | − | |
| 5064 | − if ((!adev->gfx.grbm_soft_reset) && | |
| 5065 | − (!adev->gfx.srbm_soft_reset)) | |
| 5066 | − return 0; | |
| 5067 | − | |
| 5068 | − grbm_soft_reset = adev->gfx.grbm_soft_reset; | |
| 5069 | − | |
| 5070 | − if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) || | |
| 5071 | − REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPF) || | |
| 5072 | − REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPC) || | |
| 5073 | − REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CPG)) { | |
| 5074 | − int i; | |
| 5075 | − | |
| 5076 | − for (i = 0; i < adev->gfx.num_compute_rings; i++) { | |
| 5077 | − struct amdgpu_ring *ring = &adev->gfx.compute_ring[i]; | |
| 5078 | − | |
| 5079 | − mutex_lock(&adev->srbm_mutex); | |
| 5080 | − vi_srbm_select(adev, ring->me, ring->pipe, ring->queue, 0); | |
| 5081 | − gfx_v8_0_deactivate_hqd(adev, 2); | |
| 5082 | − vi_srbm_select(adev, 0, 0, 0, 0); | |
| 5083 | − mutex_unlock(&adev->srbm_mutex); | |
| 5084 | − } | |
| 5085 | − gfx_v8_0_kiq_resume(adev); | |
| 5086 | − gfx_v8_0_kcq_resume(adev); | |
| 5087 | − } | |
| 5088 | − | |
| 5089 | − if (REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_CP) || | |
| 5090 | − REG_GET_FIELD(grbm_soft_reset, GRBM_SOFT_RESET, SOFT_RESET_GFX)) | |
| 5091 | − gfx_v8_0_cp_gfx_resume(adev); | |
| 4993 | + r = ip_block->version->funcs->resume(ip_block); | |
| 4994 | + r |= ip_block->version->funcs->late_init(ip_block); | |
| 4995 | + if (r) | |
| 4996 | + return r; | |
| 5092 | 4997 | |
| 5093 | − gfx_v8_0_cp_test_all_rings(adev); | |
| 5094 | − | |
| 5095 | − adev->gfx.rlc.funcs->start(adev); | |
| 4998 | + ip_block->version->funcs->set_clockgating_state(ip_block, AMD_CG_STATE_GATE); | |
| 4999 | + ip_block->version->funcs->set_powergating_state(ip_block, AMD_PG_STATE_GATE); | |
| 5096 | 5000 | |
| 5097 | 5001 | return 0; |
| 5098 | 5002 | } |
| @@ -6859,10 +6763,7 @@ static const struct amd_ip_funcs gfx_v8_0_ip_funcs = { · 1760 unmodified lines | ||
| 6859 | 6763 | .resume = gfx_v8_0_resume, |
| 6860 | 6764 | .is_idle = gfx_v8_0_is_idle, |
| 6861 | 6765 | .wait_for_idle = gfx_v8_0_wait_for_idle, |
| 6862 | − .check_soft_reset = gfx_v8_0_check_soft_reset, | |
| 6863 | − .pre_soft_reset = gfx_v8_0_pre_soft_reset, | |
| 6864 | 6766 | .soft_reset = gfx_v8_0_soft_reset, |
| 6865 | − .post_soft_reset = gfx_v8_0_post_soft_reset, | |
| 6866 | 6767 | .set_clockgating_state = gfx_v8_0_set_clockgating_state, |
| 6867 | 6768 | .set_powergating_state = gfx_v8_0_set_powergating_state, |
| 6868 | 6769 | .get_clockgating_state = gfx_v8_0_get_clockgating_state, |
| @@ -6923,10 +6824,12 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { · 54 unmodified lines | ||
| 6923 | 6824 | .get_wptr = gfx_v8_0_ring_get_wptr_compute, |
| 6924 | 6825 | .set_wptr = gfx_v8_0_ring_set_wptr_compute, |
| 6925 | 6826 | .emit_frame_size = |
| 6827 | + 5 + /* gfx_v8_0_ring_emit_init_cond_exec (from amdgpu_ib_schedule) */ | |
| 6926 | 6828 | 20 + /* gfx_v8_0_ring_emit_gds_switch */ |
| 6927 | 6829 | 7 + /* gfx_v8_0_ring_emit_hdp_flush */ |
| 6928 | 6830 | 5 + /* hdp_invalidate */ |
| 6929 | 6831 | 7 + /* gfx_v8_0_ring_emit_pipeline_sync */ |
| 6832 | + 5 + /* gfx_v8_0_ring_emit_init_cond_exec (from amdgpu_vm_flush) */ | |
| 6930 | 6833 | VI_FLUSH_GPU_TLB_NUM_WREG * 5 + 7 + /* gfx_v8_0_ring_emit_vm_flush */ |
| 6931 | 6834 | 7 + 7 + 7 + /* gfx_v8_0_ring_emit_fence_compute x3 for user fence, vm fence */ |
| 6932 | 6835 | 7 + /* gfx_v8_0_emit_mem_sync_compute */ |
| @@ -6947,6 +6850,7 @@ static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_compute = { · 14 unmodified lines | ||
| 6947 | 6850 | .soft_recovery = gfx_v8_0_ring_soft_recovery, |
| 6948 | 6851 | .emit_mem_sync = gfx_v8_0_emit_mem_sync_compute, |
| 6949 | 6852 | .emit_wave_limit = gfx_v8_0_emit_wave_limit, |
| 6853 | + .init_cond_exec = gfx_v8_0_ring_emit_init_cond_exec, | |
| 6950 | 6854 | }; |
| 6951 | 6855 | |
| 6952 | 6856 | static const struct amdgpu_ring_funcs gfx_v8_0_ring_funcs_kiq = { |
| @@ -4875,6 +4875,8 @@ static int gfx_v9_0_early_init(struct amdgpu_ip_block *ip_block) · 4874 unmodified lines | ||
| 4875 | 4875 | /* init rlcg reg access ctrl */ |
| 4876 | 4876 | gfx_v9_0_init_rlcg_reg_access_ctrl(adev); |
| 4877 | 4877 | |
| 4878 | + amdgpu_init_rlc_reg_funcs(adev); | |
| 4879 | + | |
| 4878 | 4880 | return gfx_v9_0_init_microcode(adev); |
| 4879 | 4881 | } |
| 4880 | 4882 | |
| @@ -39,7 +39,6 @@ · 38 unmodified lines | ||
| 39 | 39 | #include "gfx_v9_4_3.h" |
| 40 | 40 | #include "gfx_v9_4_3_cleaner_shader.h" |
| 41 | 41 | #include "amdgpu_xcp.h" |
| 42 | −#include "amdgpu_aca.h" | |
| 43 | 42 | |
| 44 | 43 | MODULE_FIRMWARE("amdgpu/gc_9_4_3_mec.bin"); |
| 45 | 44 | MODULE_FIRMWARE("amdgpu/gc_9_4_4_mec.bin"); |
| @@ -851,73 +850,6 @@ static const struct amdgpu_gfx_funcs gfx_v9_4_3_gfx_funcs = { · 805 unmodified lines | ||
| 851 | 850 | .get_hdp_flush_mask = &amdgpu_gfx_get_hdp_flush_mask, |
| 852 | 851 | }; |
| 853 | 852 | |
| 854 | −static int gfx_v9_4_3_aca_bank_parser(struct aca_handle *handle, | |
| 855 | − struct aca_bank *bank, enum aca_smu_type type, | |
| 856 | − void *data) | |
| 857 | −{ | |
| 858 | − struct aca_bank_info info; | |
| 859 | − u64 misc0; | |
| 860 | − u32 instlo; | |
| 861 | − int ret; | |
| 862 | − | |
| 863 | − ret = aca_bank_info_decode(bank, &info); | |
| 864 | − if (ret) | |
| 865 | − return ret; | |
| 866 | − | |
| 867 | − /* NOTE: overwrite info.die_id with xcd id for gfx */ | |
| 868 | − instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); | |
| 869 | − instlo &= GENMASK(31, 1); | |
| 870 | − info.die_id = instlo == mmSMNAID_XCD0_MCA_SMU ? 0 : 1; | |
| 871 | − | |
| 872 | − misc0 = bank->regs[ACA_REG_IDX_MISC0]; | |
| 873 | − | |
| 874 | − switch (type) { | |
| 875 | − case ACA_SMU_TYPE_UE: | |
| 876 | − bank->aca_err_type = ACA_ERROR_TYPE_UE; | |
| 877 | − ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, 1ULL); | |
| 878 | − break; | |
| 879 | − case ACA_SMU_TYPE_CE: | |
| 880 | − bank->aca_err_type = ACA_ERROR_TYPE_CE; | |
| 881 | − ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, | |
| 882 | − ACA_REG__MISC0__ERRCNT(misc0)); | |
| 883 | − break; | |
| 884 | − default: | |
| 885 | − return -EINVAL; | |
| 886 | − } | |
| 887 | − | |
| 888 | − return ret; | |
| 889 | −} | |
| 890 | − | |
| 891 | −static bool gfx_v9_4_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, | |
| 892 | − enum aca_smu_type type, void *data) | |
| 893 | −{ | |
| 894 | − u32 instlo; | |
| 895 | − | |
| 896 | − instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); | |
| 897 | − instlo &= GENMASK(31, 1); | |
| 898 | − switch (instlo) { | |
| 899 | − case mmSMNAID_XCD0_MCA_SMU: | |
| 900 | − case mmSMNAID_XCD1_MCA_SMU: | |
| 901 | − case mmSMNXCD_XCD0_MCA_SMU: | |
| 902 | − return true; | |
| 903 | − default: | |
| 904 | − break; | |
| 905 | − } | |
| 906 | − | |
| 907 | − return false; | |
| 908 | −} | |
| 909 | − | |
| 910 | −static const struct aca_bank_ops gfx_v9_4_3_aca_bank_ops = { | |
| 911 | − .aca_bank_parser = gfx_v9_4_3_aca_bank_parser, | |
| 912 | − .aca_bank_is_valid = gfx_v9_4_3_aca_bank_is_valid, | |
| 913 | −}; | |
| 914 | − | |
| 915 | −static const struct aca_info gfx_v9_4_3_aca_info = { | |
| 916 | − .hwip = ACA_HWIP_TYPE_SMU, | |
| 917 | − .mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK, | |
| 918 | − .bank_ops = &gfx_v9_4_3_aca_bank_ops, | |
| 919 | −}; | |
| 920 | − | |
| 921 | 853 | static int gfx_v9_4_3_gpu_early_init(struct amdgpu_device *adev) |
| 922 | 854 | { |
| 923 | 855 | adev->gfx.funcs = &gfx_v9_4_3_gfx_funcs; |
| @@ -1107,22 +1039,24 @@ static int gfx_v9_4_3_sw_init(struct amdgpu_ip_block *ip_block) · 183 unmodified lines | ||
| 1107 | 1039 | /* set up the compute queues - allocate horizontally across pipes */ |
| 1108 | 1040 | for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) { |
| 1109 | 1041 | ring_id = 0; |
| 1110 | − for (i = 0; i < adev->gfx.mec.num_mec; ++i) { | |
| 1111 | − for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { | |
| 1112 | − for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; | |
| 1113 | − k++) { | |
| 1114 | − if (!amdgpu_gfx_is_mec_queue_enabled( | |
| 1115 | − adev, xcc_id, i, k, j)) | |
| 1116 | − continue; | |
| 1117 | − | |
| 1118 | − r = gfx_v9_4_3_compute_ring_init(adev, | |
| 1119 | − ring_id, | |
| 1120 | − xcc_id, | |
| 1121 | − i, k, j); | |
| 1122 | − if (r) | |
| 1123 | − return r; | |
| 1124 | − | |
| 1125 | − ring_id++; | |
| 1042 | + if (!adev->gfx.disable_kq) { | |
| 1043 | + for (i = 0; i < adev->gfx.mec.num_mec; ++i) { | |
| 1044 | + for (j = 0; j < adev->gfx.mec.num_queue_per_pipe; j++) { | |
| 1045 | + for (k = 0; k < adev->gfx.mec.num_pipe_per_mec; | |
| 1046 | + k++) { | |
| 1047 | + if (!amdgpu_gfx_is_mec_queue_enabled( | |
| 1048 | + adev, xcc_id, i, k, j)) | |
| 1049 | + continue; | |
| 1050 | + | |
| 1051 | + r = gfx_v9_4_3_compute_ring_init(adev, | |
| 1052 | + ring_id, | |
| 1053 | + xcc_id, | |
| 1054 | + i, k, j); | |
| 1055 | + if (r) | |
| 1056 | + return r; | |
| 1057 | + | |
| 1058 | + ring_id++; | |
| 1059 | + } | |
| 1126 | 1060 | } |
| 1127 | 1061 | } |
| 1128 | 1062 | } |
| @@ -2350,6 +2284,65 @@ static void gfx_v9_4_3_xcc_fini(struct amdgpu_device *adev, int xcc_id) · 1221 unmodified lines | ||
| 2350 | 2284 | gfx_v9_4_3_xcc_cp_compute_enable(adev, false, xcc_id); |
| 2351 | 2285 | } |
| 2352 | 2286 | |
| 2287 | +static int gfx_v9_4_3_set_userq_eop_interrupts(struct amdgpu_device *adev, | |
| 2288 | + bool enable) | |
| 2289 | +{ | |
| 2290 | + int num_xcc = NUM_XCC(adev->gfx.xcc_mask); | |
| 2291 | + unsigned int irq_type; | |
| 2292 | + int m, p, xcc_id, r; | |
| 2293 | + | |
| 2294 | + if (adev->gfx.disable_kq) { | |
| 2295 | + for (xcc_id = 0; xcc_id < num_xcc; xcc_id++) { | |
| 2296 | + for (m = 0; m < adev->gfx.mec.num_mec; ++m) { | |
| 2297 | + for (p = 0; p < adev->gfx.mec.num_pipe_per_mec; p++) { | |
| 2298 | + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP | |
| 2299 | + + (m * adev->gfx.mec.num_pipe_per_mec) | |
| 2300 | + + p; | |
| 2301 | + | |
| 2302 | + if (enable) | |
| 2303 | + r = amdgpu_irq_get(adev, &adev->gfx.eop_irq, | |
| 2304 | + irq_type); | |
| 2305 | + else | |
| 2306 | + r = amdgpu_irq_put(adev, &adev->gfx.eop_irq, | |
| 2307 | + irq_type); | |
| 2308 | + if (r) { | |
| 2309 | + if (!enable) | |
| 2310 | + return r; | |
| 2311 | + goto err_compute; | |
| 2312 | + } | |
| 2313 | + } | |
| 2314 | + } | |
| 2315 | + } | |
| 2316 | + } | |
| 2317 | + | |
| 2318 | + return 0; | |
| 2319 | + | |
| 2320 | +err_compute: | |
| 2321 | + for (p--; p >= 0; p--) { | |
| 2322 | + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP | |
| 2323 | + + (m * adev->gfx.mec.num_pipe_per_mec) + p; | |
| 2324 | + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); | |
| 2325 | + } | |
| 2326 | + for (m--; m >= 0; m--) { | |
| 2327 | + for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) { | |
| 2328 | + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP | |
| 2329 | + + (m * adev->gfx.mec.num_pipe_per_mec) + p; | |
| 2330 | + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); | |
| 2331 | + } | |
| 2332 | + } | |
| 2333 | + for (xcc_id--; xcc_id >= 0; xcc_id--) { | |
| 2334 | + for (m = adev->gfx.mec.num_mec - 1; m <= 0; m--) { | |
| 2335 | + for (p = adev->gfx.mec.num_pipe_per_mec - 1; p >= 0; p--) { | |
| 2336 | + irq_type = AMDGPU_CP_IRQ_COMPUTE_MEC1_PIPE0_EOP | |
| 2337 | + + (m * adev->gfx.mec.num_pipe_per_mec) + p; | |
| 2338 | + amdgpu_irq_put(adev, &adev->gfx.eop_irq, irq_type); | |
| 2339 | + } | |
| 2340 | + } | |
| 2341 | + } | |
| 2342 | + | |
| 2343 | + return r; | |
| 2344 | +} | |
| 2345 | + | |
| 2353 | 2346 | static int gfx_v9_4_3_hw_init(struct amdgpu_ip_block *ip_block) |
| 2354 | 2347 | { |
| 2355 | 2348 | int r; |
| @@ -2382,9 +2375,14 @@ static int gfx_v9_4_3_hw_init(struct amdgpu_ip_block *ip_block) · 26 unmodified lines | ||
| 2382 | 2375 | r = amdgpu_irq_get(adev, &adev->gfx.bad_op_irq, 0); |
| 2383 | 2376 | if (r) |
| 2384 | 2377 | goto err_bad_op; |
| 2378 | + r = gfx_v9_4_3_set_userq_eop_interrupts(adev, true); | |
| 2379 | + if (r) | |
| 2380 | + goto err_bad_eop; | |
| 2385 | 2381 | |
| 2386 | 2382 | return 0; |
| 2387 | 2383 | |
| 2384 | +err_bad_eop: | |
| 2385 | + amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); | |
| 2388 | 2386 | err_bad_op: |
| 2389 | 2387 | amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); |
| 2390 | 2388 | err_priv_inst: |
| @@ -2467,6 +2465,7 @@ static int gfx_v9_4_3_hw_fini(struct amdgpu_ip_block *ip_block) · 76 unmodified lines | ||
| 2467 | 2465 | amdgpu_irq_put(adev, &adev->gfx.bad_op_irq, 0); |
| 2468 | 2466 | amdgpu_irq_put(adev, &adev->gfx.priv_inst_irq, 0); |
| 2469 | 2467 | amdgpu_irq_put(adev, &adev->gfx.priv_reg_irq, 0); |
| 2468 | + gfx_v9_4_3_set_userq_eop_interrupts(adev, false); | |
| 2470 | 2469 | |
| 2471 | 2470 | num_xcc = NUM_XCC(adev->gfx.xcc_mask); |
| 2472 | 2471 | for (i = 0; i < num_xcc; i++) { |
| @@ -2612,8 +2611,24 @@ static int gfx_v9_4_3_early_init(struct amdgpu_ip_block *ip_block) · 139 unmodified lines | ||
| 2612 | 2611 | { |
| 2613 | 2612 | struct amdgpu_device *adev = ip_block->adev; |
| 2614 | 2613 | |
| 2615 | − adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), | |
| 2616 | − AMDGPU_MAX_COMPUTE_RINGS); | |
| 2614 | + switch (amdgpu_user_queue) { | |
| 2615 | + case -1: | |
| 2616 | + case 0: | |
| 2617 | + default: | |
| 2618 | + adev->gfx.disable_kq = false; | |
| 2619 | + adev->gfx.disable_uq = true; | |
| 2620 | + break; | |
| 2621 | + case 2: | |
| 2622 | + adev->gfx.disable_kq = true; | |
| 2623 | + adev->gfx.disable_uq = true; | |
| 2624 | + break; | |
| 2625 | + } | |
| 2626 | + | |
| 2627 | + if (adev->gfx.disable_kq) | |
| 2628 | + adev->gfx.num_compute_rings = 0; | |
| 2629 | + else | |
| 2630 | + adev->gfx.num_compute_rings = min(amdgpu_gfx_get_num_kcq(adev), | |
| 2631 | + AMDGPU_MAX_COMPUTE_RINGS); | |
| 2617 | 2632 | gfx_v9_4_3_set_kiq_pm4_funcs(adev); |
| 2618 | 2633 | gfx_v9_4_3_set_ring_funcs(adev); |
| 2619 | 2634 | gfx_v9_4_3_set_irq_funcs(adev); |
| @@ -2623,6 +2638,8 @@ static int gfx_v9_4_3_early_init(struct amdgpu_ip_block *ip_block) · 3 unmodified lines | ||
| 2623 | 2638 | /* init rlcg reg access ctrl */ |
| 2624 | 2639 | gfx_v9_4_3_init_rlcg_reg_access_ctrl(adev); |
| 2625 | 2640 | |
| 2641 | + amdgpu_init_rlc_reg_funcs(adev); | |
| 2642 | + | |
| 2626 | 2643 | return gfx_v9_4_3_init_microcode(adev); |
| 2627 | 2644 | } |
| 2628 | 2645 | |
| @@ -3709,872 +3726,6 @@ pipe_reset: · 1080 unmodified lines | ||
| 3709 | 3726 | return amdgpu_ring_reset_helper_end(ring, timedout_fence); |
| 3710 | 3727 | } |
| 3711 | 3728 | |
| 3712 | −enum amdgpu_gfx_cp_ras_mem_id { | |
| 3713 | − AMDGPU_GFX_CP_MEM1 = 1, | |
| 3714 | − AMDGPU_GFX_CP_MEM2, | |
| 3715 | − AMDGPU_GFX_CP_MEM3, | |
| 3716 | − AMDGPU_GFX_CP_MEM4, | |
| 3717 | − AMDGPU_GFX_CP_MEM5, | |
| 3718 | −}; | |
| 3719 | − | |
| 3720 | −enum amdgpu_gfx_gcea_ras_mem_id { | |
| 3721 | − AMDGPU_GFX_GCEA_IOWR_CMDMEM = 4, | |
| 3722 | − AMDGPU_GFX_GCEA_IORD_CMDMEM, | |
| 3723 | − AMDGPU_GFX_GCEA_GMIWR_CMDMEM, | |
| 3724 | − AMDGPU_GFX_GCEA_GMIRD_CMDMEM, | |
| 3725 | − AMDGPU_GFX_GCEA_DRAMWR_CMDMEM, | |
| 3726 | − AMDGPU_GFX_GCEA_DRAMRD_CMDMEM, | |
| 3727 | − AMDGPU_GFX_GCEA_MAM_DMEM0, | |
| 3728 | − AMDGPU_GFX_GCEA_MAM_DMEM1, | |
| 3729 | − AMDGPU_GFX_GCEA_MAM_DMEM2, | |
| 3730 | − AMDGPU_GFX_GCEA_MAM_DMEM3, | |
| 3731 | − AMDGPU_GFX_GCEA_MAM_AMEM0, | |
| 3732 | − AMDGPU_GFX_GCEA_MAM_AMEM1, | |
| 3733 | − AMDGPU_GFX_GCEA_MAM_AMEM2, | |
| 3734 | − AMDGPU_GFX_GCEA_MAM_AMEM3, | |
| 3735 | − AMDGPU_GFX_GCEA_MAM_AFLUSH_BUFFER, | |
| 3736 | − AMDGPU_GFX_GCEA_WRET_TAGMEM, | |
| 3737 | − AMDGPU_GFX_GCEA_RRET_TAGMEM, | |
| 3738 | − AMDGPU_GFX_GCEA_IOWR_DATAMEM, | |
| 3739 | − AMDGPU_GFX_GCEA_GMIWR_DATAMEM, | |
| 3740 | − AMDGPU_GFX_GCEA_DRAM_DATAMEM, | |
| 3741 | −}; | |
| 3742 | − | |
| 3743 | −enum amdgpu_gfx_gc_cane_ras_mem_id { | |
| 3744 | − AMDGPU_GFX_GC_CANE_MEM0 = 0, | |
| 3745 | −}; | |
| 3746 | − | |
| 3747 | −enum amdgpu_gfx_gcutcl2_ras_mem_id { | |
| 3748 | − AMDGPU_GFX_GCUTCL2_MEM2P512X95 = 160, | |
| 3749 | −}; | |
| 3750 | − | |
| 3751 | −enum amdgpu_gfx_gds_ras_mem_id { | |
| 3752 | − AMDGPU_GFX_GDS_MEM0 = 0, | |
| 3753 | −}; | |
| 3754 | − | |
| 3755 | −enum amdgpu_gfx_lds_ras_mem_id { | |
| 3756 | − AMDGPU_GFX_LDS_BANK0 = 0, | |
| 3757 | − AMDGPU_GFX_LDS_BANK1, | |
| 3758 | − AMDGPU_GFX_LDS_BANK2, | |
| 3759 | − AMDGPU_GFX_LDS_BANK3, | |
| 3760 | − AMDGPU_GFX_LDS_BANK4, | |
| 3761 | − AMDGPU_GFX_LDS_BANK5, | |
| 3762 | − AMDGPU_GFX_LDS_BANK6, | |
| 3763 | − AMDGPU_GFX_LDS_BANK7, | |
| 3764 | − AMDGPU_GFX_LDS_BANK8, | |
| 3765 | − AMDGPU_GFX_LDS_BANK9, | |
| 3766 | − AMDGPU_GFX_LDS_BANK10, | |
| 3767 | − AMDGPU_GFX_LDS_BANK11, | |
| 3768 | − AMDGPU_GFX_LDS_BANK12, | |
| 3769 | − AMDGPU_GFX_LDS_BANK13, | |
| 3770 | − AMDGPU_GFX_LDS_BANK14, | |
| 3771 | − AMDGPU_GFX_LDS_BANK15, | |
| 3772 | − AMDGPU_GFX_LDS_BANK16, | |
| 3773 | − AMDGPU_GFX_LDS_BANK17, | |
| 3774 | − AMDGPU_GFX_LDS_BANK18, | |
| 3775 | − AMDGPU_GFX_LDS_BANK19, | |
| 3776 | − AMDGPU_GFX_LDS_BANK20, | |
| 3777 | − AMDGPU_GFX_LDS_BANK21, | |
| 3778 | − AMDGPU_GFX_LDS_BANK22, | |
| 3779 | − AMDGPU_GFX_LDS_BANK23, | |
| 3780 | − AMDGPU_GFX_LDS_BANK24, | |
| 3781 | − AMDGPU_GFX_LDS_BANK25, | |
| 3782 | − AMDGPU_GFX_LDS_BANK26, | |
| 3783 | − AMDGPU_GFX_LDS_BANK27, | |
| 3784 | − AMDGPU_GFX_LDS_BANK28, | |
| 3785 | − AMDGPU_GFX_LDS_BANK29, | |
| 3786 | − AMDGPU_GFX_LDS_BANK30, | |
| 3787 | − AMDGPU_GFX_LDS_BANK31, | |
| 3788 | − AMDGPU_GFX_LDS_SP_BUFFER_A, | |
| 3789 | − AMDGPU_GFX_LDS_SP_BUFFER_B, | |
| 3790 | −}; | |
| 3791 | − | |
| 3792 | −enum amdgpu_gfx_rlc_ras_mem_id { | |
| 3793 | − AMDGPU_GFX_RLC_GPMF32 = 1, | |
| 3794 | − AMDGPU_GFX_RLC_RLCVF32, | |
| 3795 | − AMDGPU_GFX_RLC_SCRATCH, | |
| 3796 | − AMDGPU_GFX_RLC_SRM_ARAM, | |
| 3797 | − AMDGPU_GFX_RLC_SRM_DRAM, | |
| 3798 | − AMDGPU_GFX_RLC_TCTAG, | |
| 3799 | − AMDGPU_GFX_RLC_SPM_SE, | |
| 3800 | − AMDGPU_GFX_RLC_SPM_GRBMT, | |
| 3801 | −}; | |
| 3802 | − | |
| 3803 | −enum amdgpu_gfx_sp_ras_mem_id { | |
| 3804 | − AMDGPU_GFX_SP_SIMDID0 = 0, | |
| 3805 | −}; | |
| 3806 | − | |
| 3807 | −enum amdgpu_gfx_spi_ras_mem_id { | |
| 3808 | − AMDGPU_GFX_SPI_MEM0 = 0, | |
| 3809 | − AMDGPU_GFX_SPI_MEM1, | |
| 3810 | − AMDGPU_GFX_SPI_MEM2, | |
| 3811 | − AMDGPU_GFX_SPI_MEM3, | |
| 3812 | −}; | |
| 3813 | − | |
| 3814 | −enum amdgpu_gfx_sqc_ras_mem_id { | |
| 3815 | − AMDGPU_GFX_SQC_INST_CACHE_A = 100, | |
| 3816 | − AMDGPU_GFX_SQC_INST_CACHE_B = 101, | |
| 3817 | − AMDGPU_GFX_SQC_INST_CACHE_TAG_A = 102, | |
| 3818 | − AMDGPU_GFX_SQC_INST_CACHE_TAG_B = 103, | |
| 3819 | − AMDGPU_GFX_SQC_INST_CACHE_MISS_FIFO_A = 104, | |
| 3820 | − AMDGPU_GFX_SQC_INST_CACHE_MISS_FIFO_B = 105, | |
| 3821 | − AMDGPU_GFX_SQC_INST_CACHE_GATCL1_MISS_FIFO_A = 106, | |
| 3822 | − AMDGPU_GFX_SQC_INST_CACHE_GATCL1_MISS_FIFO_B = 107, | |
| 3823 | − AMDGPU_GFX_SQC_DATA_CACHE_A = 200, | |
| 3824 | − AMDGPU_GFX_SQC_DATA_CACHE_B = 201, | |
| 3825 | − AMDGPU_GFX_SQC_DATA_CACHE_TAG_A = 202, | |
| 3826 | − AMDGPU_GFX_SQC_DATA_CACHE_TAG_B = 203, | |
| 3827 | − AMDGPU_GFX_SQC_DATA_CACHE_MISS_FIFO_A = 204, | |
| 3828 | − AMDGPU_GFX_SQC_DATA_CACHE_MISS_FIFO_B = 205, | |
| 3829 | − AMDGPU_GFX_SQC_DATA_CACHE_HIT_FIFO_A = 206, | |
| 3830 | − AMDGPU_GFX_SQC_DATA_CACHE_HIT_FIFO_B = 207, | |
| 3831 | − AMDGPU_GFX_SQC_DIRTY_BIT_A = 208, | |
| 3832 | − AMDGPU_GFX_SQC_DIRTY_BIT_B = 209, | |
| 3833 | − AMDGPU_GFX_SQC_WRITE_DATA_BUFFER_CU0 = 210, | |
| 3834 | − AMDGPU_GFX_SQC_WRITE_DATA_BUFFER_CU1 = 211, | |
| 3835 | − AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_DATA_CACHE_A = 212, | |
| 3836 | − AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_DATA_CACHE_B = 213, | |
| 3837 | − AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_INST_CACHE = 108, | |
| 3838 | −}; | |
| 3839 | − | |
| 3840 | −enum amdgpu_gfx_sq_ras_mem_id { | |
| 3841 | − AMDGPU_GFX_SQ_SGPR_MEM0 = 0, | |
| 3842 | − AMDGPU_GFX_SQ_SGPR_MEM1, | |
| 3843 | − AMDGPU_GFX_SQ_SGPR_MEM2, | |
| 3844 | − AMDGPU_GFX_SQ_SGPR_MEM3, | |
| 3845 | −}; | |
| 3846 | − | |
| 3847 | −enum amdgpu_gfx_ta_ras_mem_id { | |
| 3848 | − AMDGPU_GFX_TA_FS_AFIFO_RAM_LO = 1, | |
| 3849 | − AMDGPU_GFX_TA_FS_AFIFO_RAM_HI, | |
| 3850 | − AMDGPU_GFX_TA_FS_CFIFO_RAM, | |
| 3851 | − AMDGPU_GFX_TA_FSX_LFIFO, | |
| 3852 | − AMDGPU_GFX_TA_FS_DFIFO_RAM, | |
| 3853 | −}; | |
| 3854 | − | |
| 3855 | −enum amdgpu_gfx_tcc_ras_mem_id { | |
| 3856 | − AMDGPU_GFX_TCC_MEM1 = 1, | |
| 3857 | −}; | |
| 3858 | − | |
| 3859 | −enum amdgpu_gfx_tca_ras_mem_id { | |
| 3860 | − AMDGPU_GFX_TCA_MEM1 = 1, | |
| 3861 | −}; | |
| 3862 | − | |
| 3863 | −enum amdgpu_gfx_tci_ras_mem_id { | |
| 3864 | − AMDGPU_GFX_TCIW_MEM = 1, | |
| 3865 | −}; | |
| 3866 | − | |
| 3867 | −enum amdgpu_gfx_tcp_ras_mem_id { | |
| 3868 | − AMDGPU_GFX_TCP_LFIFO0 = 1, | |
| 3869 | − AMDGPU_GFX_TCP_SET0BANK0_RAM, | |
| 3870 | − AMDGPU_GFX_TCP_SET0BANK1_RAM, | |
| 3871 | − AMDGPU_GFX_TCP_SET0BANK2_RAM, | |
| 3872 | − AMDGPU_GFX_TCP_SET0BANK3_RAM, | |
| 3873 | − AMDGPU_GFX_TCP_SET1BANK0_RAM, | |
| 3874 | − AMDGPU_GFX_TCP_SET1BANK1_RAM, | |
| 3875 | − AMDGPU_GFX_TCP_SET1BANK2_RAM, | |
| 3876 | − AMDGPU_GFX_TCP_SET1BANK3_RAM, | |
| 3877 | − AMDGPU_GFX_TCP_SET2BANK0_RAM, | |
| 3878 | − AMDGPU_GFX_TCP_SET2BANK1_RAM, | |
| 3879 | − AMDGPU_GFX_TCP_SET2BANK2_RAM, | |
| 3880 | − AMDGPU_GFX_TCP_SET2BANK3_RAM, | |
| 3881 | − AMDGPU_GFX_TCP_SET3BANK0_RAM, | |
| 3882 | − AMDGPU_GFX_TCP_SET3BANK1_RAM, | |
| 3883 | − AMDGPU_GFX_TCP_SET3BANK2_RAM, | |
| 3884 | − AMDGPU_GFX_TCP_SET3BANK3_RAM, | |
| 3885 | − AMDGPU_GFX_TCP_VM_FIFO, | |
| 3886 | − AMDGPU_GFX_TCP_DB_TAGRAM0, | |
| 3887 | − AMDGPU_GFX_TCP_DB_TAGRAM1, | |
| 3888 | − AMDGPU_GFX_TCP_DB_TAGRAM2, | |
| 3889 | − AMDGPU_GFX_TCP_DB_TAGRAM3, | |
| 3890 | − AMDGPU_GFX_TCP_UTCL1_LFIFO_PROBE0, | |
| 3891 | − AMDGPU_GFX_TCP_UTCL1_LFIFO_PROBE1, | |
| 3892 | − AMDGPU_GFX_TCP_CMD_FIFO, | |
| 3893 | −}; | |
| 3894 | − | |
| 3895 | −enum amdgpu_gfx_td_ras_mem_id { | |
| 3896 | − AMDGPU_GFX_TD_UTD_CS_FIFO_MEM = 1, | |
| 3897 | − AMDGPU_GFX_TD_UTD_SS_FIFO_LO_MEM, | |
| 3898 | − AMDGPU_GFX_TD_UTD_SS_FIFO_HI_MEM, | |
| 3899 | −}; | |
| 3900 | − | |
| 3901 | −enum amdgpu_gfx_tcx_ras_mem_id { | |
| 3902 | − AMDGPU_GFX_TCX_FIFOD0 = 0, | |
| 3903 | − AMDGPU_GFX_TCX_FIFOD1, | |
| 3904 | − AMDGPU_GFX_TCX_FIFOD2, | |
| 3905 | − AMDGPU_GFX_TCX_FIFOD3, | |
| 3906 | − AMDGPU_GFX_TCX_FIFOD4, | |
| 3907 | − AMDGPU_GFX_TCX_FIFOD5, | |
| 3908 | − AMDGPU_GFX_TCX_FIFOD6, | |
| 3909 | − AMDGPU_GFX_TCX_FIFOD7, | |
| 3910 | − AMDGPU_GFX_TCX_FIFOB0, | |
| 3911 | − AMDGPU_GFX_TCX_FIFOB1, | |
| 3912 | − AMDGPU_GFX_TCX_FIFOB2, | |
| 3913 | − AMDGPU_GFX_TCX_FIFOB3, | |
| 3914 | − AMDGPU_GFX_TCX_FIFOB4, | |
| 3915 | − AMDGPU_GFX_TCX_FIFOB5, | |
| 3916 | − AMDGPU_GFX_TCX_FIFOB6, | |
| 3917 | − AMDGPU_GFX_TCX_FIFOB7, | |
| 3918 | − AMDGPU_GFX_TCX_FIFOA0, | |
| 3919 | − AMDGPU_GFX_TCX_FIFOA1, | |
| 3920 | − AMDGPU_GFX_TCX_FIFOA2, | |
| 3921 | − AMDGPU_GFX_TCX_FIFOA3, | |
| 3922 | − AMDGPU_GFX_TCX_FIFOA4, | |
| 3923 | − AMDGPU_GFX_TCX_FIFOA5, | |
| 3924 | − AMDGPU_GFX_TCX_FIFOA6, | |
| 3925 | − AMDGPU_GFX_TCX_FIFOA7, | |
| 3926 | − AMDGPU_GFX_TCX_CFIFO0, | |
| 3927 | − AMDGPU_GFX_TCX_CFIFO1, | |
| 3928 | − AMDGPU_GFX_TCX_CFIFO2, | |
| 3929 | − AMDGPU_GFX_TCX_CFIFO3, | |
| 3930 | − AMDGPU_GFX_TCX_CFIFO4, | |
| 3931 | − AMDGPU_GFX_TCX_CFIFO5, | |
| 3932 | − AMDGPU_GFX_TCX_CFIFO6, | |
| 3933 | − AMDGPU_GFX_TCX_CFIFO7, | |
| 3934 | − AMDGPU_GFX_TCX_FIFO_ACKB0, | |
| 3935 | − AMDGPU_GFX_TCX_FIFO_ACKB1, | |
| 3936 | − AMDGPU_GFX_TCX_FIFO_ACKB2, | |
| 3937 | − AMDGPU_GFX_TCX_FIFO_ACKB3, | |
| 3938 | − AMDGPU_GFX_TCX_FIFO_ACKB4, | |
| 3939 | − AMDGPU_GFX_TCX_FIFO_ACKB5, | |
| 3940 | − AMDGPU_GFX_TCX_FIFO_ACKB6, | |
| 3941 | − AMDGPU_GFX_TCX_FIFO_ACKB7, | |
| 3942 | − AMDGPU_GFX_TCX_FIFO_ACKD0, | |
| 3943 | − AMDGPU_GFX_TCX_FIFO_ACKD1, | |
| 3944 | − AMDGPU_GFX_TCX_FIFO_ACKD2, | |
| 3945 | − AMDGPU_GFX_TCX_FIFO_ACKD3, | |
| 3946 | − AMDGPU_GFX_TCX_FIFO_ACKD4, | |
| 3947 | − AMDGPU_GFX_TCX_FIFO_ACKD5, | |
| 3948 | − AMDGPU_GFX_TCX_FIFO_ACKD6, | |
| 3949 | − AMDGPU_GFX_TCX_FIFO_ACKD7, | |
| 3950 | − AMDGPU_GFX_TCX_DST_FIFOA0, | |
| 3951 | − AMDGPU_GFX_TCX_DST_FIFOA1, | |
| 3952 | − AMDGPU_GFX_TCX_DST_FIFOA2, | |
| 3953 | − AMDGPU_GFX_TCX_DST_FIFOA3, | |
| 3954 | − AMDGPU_GFX_TCX_DST_FIFOA4, | |
| 3955 | − AMDGPU_GFX_TCX_DST_FIFOA5, | |
| 3956 | − AMDGPU_GFX_TCX_DST_FIFOA6, | |
| 3957 | − AMDGPU_GFX_TCX_DST_FIFOA7, | |
| 3958 | − AMDGPU_GFX_TCX_DST_FIFOB0, | |
| 3959 | − AMDGPU_GFX_TCX_DST_FIFOB1, | |
| 3960 | − AMDGPU_GFX_TCX_DST_FIFOB2, | |
| 3961 | − AMDGPU_GFX_TCX_DST_FIFOB3, | |
| 3962 | − AMDGPU_GFX_TCX_DST_FIFOB4, | |
| 3963 | − AMDGPU_GFX_TCX_DST_FIFOB5, | |
| 3964 | − AMDGPU_GFX_TCX_DST_FIFOB6, | |
| 3965 | − AMDGPU_GFX_TCX_DST_FIFOB7, | |
| 3966 | − AMDGPU_GFX_TCX_DST_FIFOD0, | |
| 3967 | − AMDGPU_GFX_TCX_DST_FIFOD1, | |
| 3968 | − AMDGPU_GFX_TCX_DST_FIFOD2, | |
| 3969 | − AMDGPU_GFX_TCX_DST_FIFOD3, | |
| 3970 | − AMDGPU_GFX_TCX_DST_FIFOD4, | |
| 3971 | − AMDGPU_GFX_TCX_DST_FIFOD5, | |
| 3972 | − AMDGPU_GFX_TCX_DST_FIFOD6, | |
| 3973 | − AMDGPU_GFX_TCX_DST_FIFOD7, | |
| 3974 | − AMDGPU_GFX_TCX_DST_FIFO_ACKB0, | |
| 3975 | − AMDGPU_GFX_TCX_DST_FIFO_ACKB1, | |
| 3976 | − AMDGPU_GFX_TCX_DST_FIFO_ACKB2, | |
| 3977 | − AMDGPU_GFX_TCX_DST_FIFO_ACKB3, | |
| 3978 | − AMDGPU_GFX_TCX_DST_FIFO_ACKB4, | |
| 3979 | − AMDGPU_GFX_TCX_DST_FIFO_ACKB5, | |
| 3980 | − AMDGPU_GFX_TCX_DST_FIFO_ACKB6, | |
| 3981 | − AMDGPU_GFX_TCX_DST_FIFO_ACKB7, | |
| 3982 | − AMDGPU_GFX_TCX_DST_FIFO_ACKD0, | |
| 3983 | − AMDGPU_GFX_TCX_DST_FIFO_ACKD1, | |
| 3984 | − AMDGPU_GFX_TCX_DST_FIFO_ACKD2, | |
| 3985 | − AMDGPU_GFX_TCX_DST_FIFO_ACKD3, | |
| 3986 | − AMDGPU_GFX_TCX_DST_FIFO_ACKD4, | |
| 3987 | − AMDGPU_GFX_TCX_DST_FIFO_ACKD5, | |
| 3988 | − AMDGPU_GFX_TCX_DST_FIFO_ACKD6, | |
| 3989 | − AMDGPU_GFX_TCX_DST_FIFO_ACKD7, | |
| 3990 | −}; | |
| 3991 | − | |
| 3992 | −enum amdgpu_gfx_atc_l2_ras_mem_id { | |
| 3993 | − AMDGPU_GFX_ATC_L2_MEM0 = 0, | |
| 3994 | −}; | |
| 3995 | − | |
| 3996 | −enum amdgpu_gfx_utcl2_ras_mem_id { | |
| 3997 | − AMDGPU_GFX_UTCL2_MEM0 = 0, | |
| 3998 | −}; | |
| 3999 | − | |
| 4000 | −enum amdgpu_gfx_vml2_ras_mem_id { | |
| 4001 | − AMDGPU_GFX_VML2_MEM0 = 0, | |
| 4002 | −}; | |
| 4003 | − | |
| 4004 | −enum amdgpu_gfx_vml2_walker_ras_mem_id { | |
| 4005 | − AMDGPU_GFX_VML2_WALKER_MEM0 = 0, | |
| 4006 | −}; | |
| 4007 | − | |
| 4008 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_cp_mem_list[] = { | |
| 4009 | − {AMDGPU_GFX_CP_MEM1, "CP_MEM1"}, | |
| 4010 | − {AMDGPU_GFX_CP_MEM2, "CP_MEM2"}, | |
| 4011 | − {AMDGPU_GFX_CP_MEM3, "CP_MEM3"}, | |
| 4012 | − {AMDGPU_GFX_CP_MEM4, "CP_MEM4"}, | |
| 4013 | − {AMDGPU_GFX_CP_MEM5, "CP_MEM5"}, | |
| 4014 | −}; | |
| 4015 | − | |
| 4016 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_gcea_mem_list[] = { | |
| 4017 | − {AMDGPU_GFX_GCEA_IOWR_CMDMEM, "GCEA_IOWR_CMDMEM"}, | |
| 4018 | − {AMDGPU_GFX_GCEA_IORD_CMDMEM, "GCEA_IORD_CMDMEM"}, | |
| 4019 | − {AMDGPU_GFX_GCEA_GMIWR_CMDMEM, "GCEA_GMIWR_CMDMEM"}, | |
| 4020 | − {AMDGPU_GFX_GCEA_GMIRD_CMDMEM, "GCEA_GMIRD_CMDMEM"}, | |
| 4021 | − {AMDGPU_GFX_GCEA_DRAMWR_CMDMEM, "GCEA_DRAMWR_CMDMEM"}, | |
| 4022 | − {AMDGPU_GFX_GCEA_DRAMRD_CMDMEM, "GCEA_DRAMRD_CMDMEM"}, | |
| 4023 | − {AMDGPU_GFX_GCEA_MAM_DMEM0, "GCEA_MAM_DMEM0"}, | |
| 4024 | − {AMDGPU_GFX_GCEA_MAM_DMEM1, "GCEA_MAM_DMEM1"}, | |
| 4025 | − {AMDGPU_GFX_GCEA_MAM_DMEM2, "GCEA_MAM_DMEM2"}, | |
| 4026 | − {AMDGPU_GFX_GCEA_MAM_DMEM3, "GCEA_MAM_DMEM3"}, | |
| 4027 | − {AMDGPU_GFX_GCEA_MAM_AMEM0, "GCEA_MAM_AMEM0"}, | |
| 4028 | − {AMDGPU_GFX_GCEA_MAM_AMEM1, "GCEA_MAM_AMEM1"}, | |
| 4029 | − {AMDGPU_GFX_GCEA_MAM_AMEM2, "GCEA_MAM_AMEM2"}, | |
| 4030 | − {AMDGPU_GFX_GCEA_MAM_AMEM3, "GCEA_MAM_AMEM3"}, | |
| 4031 | − {AMDGPU_GFX_GCEA_MAM_AFLUSH_BUFFER, "GCEA_MAM_AFLUSH_BUFFER"}, | |
| 4032 | − {AMDGPU_GFX_GCEA_WRET_TAGMEM, "GCEA_WRET_TAGMEM"}, | |
| 4033 | − {AMDGPU_GFX_GCEA_RRET_TAGMEM, "GCEA_RRET_TAGMEM"}, | |
| 4034 | − {AMDGPU_GFX_GCEA_IOWR_DATAMEM, "GCEA_IOWR_DATAMEM"}, | |
| 4035 | − {AMDGPU_GFX_GCEA_GMIWR_DATAMEM, "GCEA_GMIWR_DATAMEM"}, | |
| 4036 | − {AMDGPU_GFX_GCEA_DRAM_DATAMEM, "GCEA_DRAM_DATAMEM"}, | |
| 4037 | −}; | |
| 4038 | − | |
| 4039 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_gc_cane_mem_list[] = { | |
| 4040 | − {AMDGPU_GFX_GC_CANE_MEM0, "GC_CANE_MEM0"}, | |
| 4041 | −}; | |
| 4042 | − | |
| 4043 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_gcutcl2_mem_list[] = { | |
| 4044 | − {AMDGPU_GFX_GCUTCL2_MEM2P512X95, "GCUTCL2_MEM2P512X95"}, | |
| 4045 | −}; | |
| 4046 | − | |
| 4047 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_gds_mem_list[] = { | |
| 4048 | − {AMDGPU_GFX_GDS_MEM0, "GDS_MEM"}, | |
| 4049 | −}; | |
| 4050 | − | |
| 4051 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_lds_mem_list[] = { | |
| 4052 | − {AMDGPU_GFX_LDS_BANK0, "LDS_BANK0"}, | |
| 4053 | − {AMDGPU_GFX_LDS_BANK1, "LDS_BANK1"}, | |
| 4054 | − {AMDGPU_GFX_LDS_BANK2, "LDS_BANK2"}, | |
| 4055 | − {AMDGPU_GFX_LDS_BANK3, "LDS_BANK3"}, | |
| 4056 | − {AMDGPU_GFX_LDS_BANK4, "LDS_BANK4"}, | |
| 4057 | − {AMDGPU_GFX_LDS_BANK5, "LDS_BANK5"}, | |
| 4058 | − {AMDGPU_GFX_LDS_BANK6, "LDS_BANK6"}, | |
| 4059 | − {AMDGPU_GFX_LDS_BANK7, "LDS_BANK7"}, | |
| 4060 | − {AMDGPU_GFX_LDS_BANK8, "LDS_BANK8"}, | |
| 4061 | − {AMDGPU_GFX_LDS_BANK9, "LDS_BANK9"}, | |
| 4062 | − {AMDGPU_GFX_LDS_BANK10, "LDS_BANK10"}, | |
| 4063 | − {AMDGPU_GFX_LDS_BANK11, "LDS_BANK11"}, | |
| 4064 | − {AMDGPU_GFX_LDS_BANK12, "LDS_BANK12"}, | |
| 4065 | − {AMDGPU_GFX_LDS_BANK13, "LDS_BANK13"}, | |
| 4066 | − {AMDGPU_GFX_LDS_BANK14, "LDS_BANK14"}, | |
| 4067 | − {AMDGPU_GFX_LDS_BANK15, "LDS_BANK15"}, | |
| 4068 | − {AMDGPU_GFX_LDS_BANK16, "LDS_BANK16"}, | |
| 4069 | − {AMDGPU_GFX_LDS_BANK17, "LDS_BANK17"}, | |
| 4070 | − {AMDGPU_GFX_LDS_BANK18, "LDS_BANK18"}, | |
| 4071 | − {AMDGPU_GFX_LDS_BANK19, "LDS_BANK19"}, | |
| 4072 | − {AMDGPU_GFX_LDS_BANK20, "LDS_BANK20"}, | |
| 4073 | − {AMDGPU_GFX_LDS_BANK21, "LDS_BANK21"}, | |
| 4074 | − {AMDGPU_GFX_LDS_BANK22, "LDS_BANK22"}, | |
| 4075 | − {AMDGPU_GFX_LDS_BANK23, "LDS_BANK23"}, | |
| 4076 | − {AMDGPU_GFX_LDS_BANK24, "LDS_BANK24"}, | |
| 4077 | − {AMDGPU_GFX_LDS_BANK25, "LDS_BANK25"}, | |
| 4078 | − {AMDGPU_GFX_LDS_BANK26, "LDS_BANK26"}, | |
| 4079 | − {AMDGPU_GFX_LDS_BANK27, "LDS_BANK27"}, | |
| 4080 | − {AMDGPU_GFX_LDS_BANK28, "LDS_BANK28"}, | |
| 4081 | − {AMDGPU_GFX_LDS_BANK29, "LDS_BANK29"}, | |
| 4082 | − {AMDGPU_GFX_LDS_BANK30, "LDS_BANK30"}, | |
| 4083 | − {AMDGPU_GFX_LDS_BANK31, "LDS_BANK31"}, | |
| 4084 | − {AMDGPU_GFX_LDS_SP_BUFFER_A, "LDS_SP_BUFFER_A"}, | |
| 4085 | − {AMDGPU_GFX_LDS_SP_BUFFER_B, "LDS_SP_BUFFER_B"}, | |
| 4086 | −}; | |
| 4087 | − | |
| 4088 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_rlc_mem_list[] = { | |
| 4089 | − {AMDGPU_GFX_RLC_GPMF32, "RLC_GPMF32"}, | |
| 4090 | − {AMDGPU_GFX_RLC_RLCVF32, "RLC_RLCVF32"}, | |
| 4091 | − {AMDGPU_GFX_RLC_SCRATCH, "RLC_SCRATCH"}, | |
| 4092 | − {AMDGPU_GFX_RLC_SRM_ARAM, "RLC_SRM_ARAM"}, | |
| 4093 | − {AMDGPU_GFX_RLC_SRM_DRAM, "RLC_SRM_DRAM"}, | |
| 4094 | − {AMDGPU_GFX_RLC_TCTAG, "RLC_TCTAG"}, | |
| 4095 | − {AMDGPU_GFX_RLC_SPM_SE, "RLC_SPM_SE"}, | |
| 4096 | − {AMDGPU_GFX_RLC_SPM_GRBMT, "RLC_SPM_GRBMT"}, | |
| 4097 | −}; | |
| 4098 | − | |
| 4099 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_sp_mem_list[] = { | |
| 4100 | − {AMDGPU_GFX_SP_SIMDID0, "SP_SIMDID0"}, | |
| 4101 | −}; | |
| 4102 | − | |
| 4103 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_spi_mem_list[] = { | |
| 4104 | − {AMDGPU_GFX_SPI_MEM0, "SPI_MEM0"}, | |
| 4105 | − {AMDGPU_GFX_SPI_MEM1, "SPI_MEM1"}, | |
| 4106 | − {AMDGPU_GFX_SPI_MEM2, "SPI_MEM2"}, | |
| 4107 | − {AMDGPU_GFX_SPI_MEM3, "SPI_MEM3"}, | |
| 4108 | −}; | |
| 4109 | − | |
| 4110 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_sqc_mem_list[] = { | |
| 4111 | − {AMDGPU_GFX_SQC_INST_CACHE_A, "SQC_INST_CACHE_A"}, | |
| 4112 | − {AMDGPU_GFX_SQC_INST_CACHE_B, "SQC_INST_CACHE_B"}, | |
| 4113 | − {AMDGPU_GFX_SQC_INST_CACHE_TAG_A, "SQC_INST_CACHE_TAG_A"}, | |
| 4114 | − {AMDGPU_GFX_SQC_INST_CACHE_TAG_B, "SQC_INST_CACHE_TAG_B"}, | |
| 4115 | − {AMDGPU_GFX_SQC_INST_CACHE_MISS_FIFO_A, "SQC_INST_CACHE_MISS_FIFO_A"}, | |
| 4116 | − {AMDGPU_GFX_SQC_INST_CACHE_MISS_FIFO_B, "SQC_INST_CACHE_MISS_FIFO_B"}, | |
| 4117 | − {AMDGPU_GFX_SQC_INST_CACHE_GATCL1_MISS_FIFO_A, "SQC_INST_CACHE_GATCL1_MISS_FIFO_A"}, | |
| 4118 | − {AMDGPU_GFX_SQC_INST_CACHE_GATCL1_MISS_FIFO_B, "SQC_INST_CACHE_GATCL1_MISS_FIFO_B"}, | |
| 4119 | − {AMDGPU_GFX_SQC_DATA_CACHE_A, "SQC_DATA_CACHE_A"}, | |
| 4120 | − {AMDGPU_GFX_SQC_DATA_CACHE_B, "SQC_DATA_CACHE_B"}, | |
| 4121 | − {AMDGPU_GFX_SQC_DATA_CACHE_TAG_A, "SQC_DATA_CACHE_TAG_A"}, | |
| 4122 | − {AMDGPU_GFX_SQC_DATA_CACHE_TAG_B, "SQC_DATA_CACHE_TAG_B"}, | |
| 4123 | − {AMDGPU_GFX_SQC_DATA_CACHE_MISS_FIFO_A, "SQC_DATA_CACHE_MISS_FIFO_A"}, | |
| 4124 | − {AMDGPU_GFX_SQC_DATA_CACHE_MISS_FIFO_B, "SQC_DATA_CACHE_MISS_FIFO_B"}, | |
| 4125 | − {AMDGPU_GFX_SQC_DATA_CACHE_HIT_FIFO_A, "SQC_DATA_CACHE_HIT_FIFO_A"}, | |
| 4126 | − {AMDGPU_GFX_SQC_DATA_CACHE_HIT_FIFO_B, "SQC_DATA_CACHE_HIT_FIFO_B"}, | |
| 4127 | − {AMDGPU_GFX_SQC_DIRTY_BIT_A, "SQC_DIRTY_BIT_A"}, | |
| 4128 | − {AMDGPU_GFX_SQC_DIRTY_BIT_B, "SQC_DIRTY_BIT_B"}, | |
| 4129 | − {AMDGPU_GFX_SQC_WRITE_DATA_BUFFER_CU0, "SQC_WRITE_DATA_BUFFER_CU0"}, | |
| 4130 | − {AMDGPU_GFX_SQC_WRITE_DATA_BUFFER_CU1, "SQC_WRITE_DATA_BUFFER_CU1"}, | |
| 4131 | − {AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_DATA_CACHE_A, "SQC_UTCL1_MISS_LFIFO_DATA_CACHE_A"}, | |
| 4132 | − {AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_DATA_CACHE_B, "SQC_UTCL1_MISS_LFIFO_DATA_CACHE_B"}, | |
| 4133 | − {AMDGPU_GFX_SQC_UTCL1_MISS_LFIFO_INST_CACHE, "SQC_UTCL1_MISS_LFIFO_INST_CACHE"}, | |
| 4134 | −}; | |
| 4135 | − | |
| 4136 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_sq_mem_list[] = { | |
| 4137 | − {AMDGPU_GFX_SQ_SGPR_MEM0, "SQ_SGPR_MEM0"}, | |
| 4138 | − {AMDGPU_GFX_SQ_SGPR_MEM1, "SQ_SGPR_MEM1"}, | |
| 4139 | − {AMDGPU_GFX_SQ_SGPR_MEM2, "SQ_SGPR_MEM2"}, | |
| 4140 | − {AMDGPU_GFX_SQ_SGPR_MEM3, "SQ_SGPR_MEM3"}, | |
| 4141 | −}; | |
| 4142 | − | |
| 4143 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_ta_mem_list[] = { | |
| 4144 | − {AMDGPU_GFX_TA_FS_AFIFO_RAM_LO, "TA_FS_AFIFO_RAM_LO"}, | |
| 4145 | − {AMDGPU_GFX_TA_FS_AFIFO_RAM_HI, "TA_FS_AFIFO_RAM_HI"}, | |
| 4146 | − {AMDGPU_GFX_TA_FS_CFIFO_RAM, "TA_FS_CFIFO_RAM"}, | |
| 4147 | − {AMDGPU_GFX_TA_FSX_LFIFO, "TA_FSX_LFIFO"}, | |
| 4148 | − {AMDGPU_GFX_TA_FS_DFIFO_RAM, "TA_FS_DFIFO_RAM"}, | |
| 4149 | −}; | |
| 4150 | − | |
| 4151 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tcc_mem_list[] = { | |
| 4152 | − {AMDGPU_GFX_TCC_MEM1, "TCC_MEM1"}, | |
| 4153 | −}; | |
| 4154 | − | |
| 4155 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tca_mem_list[] = { | |
| 4156 | − {AMDGPU_GFX_TCA_MEM1, "TCA_MEM1"}, | |
| 4157 | −}; | |
| 4158 | − | |
| 4159 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tci_mem_list[] = { | |
| 4160 | − {AMDGPU_GFX_TCIW_MEM, "TCIW_MEM"}, | |
| 4161 | −}; | |
| 4162 | − | |
| 4163 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tcp_mem_list[] = { | |
| 4164 | − {AMDGPU_GFX_TCP_LFIFO0, "TCP_LFIFO0"}, | |
| 4165 | − {AMDGPU_GFX_TCP_SET0BANK0_RAM, "TCP_SET0BANK0_RAM"}, | |
| 4166 | − {AMDGPU_GFX_TCP_SET0BANK1_RAM, "TCP_SET0BANK1_RAM"}, | |
| 4167 | − {AMDGPU_GFX_TCP_SET0BANK2_RAM, "TCP_SET0BANK2_RAM"}, | |
| 4168 | − {AMDGPU_GFX_TCP_SET0BANK3_RAM, "TCP_SET0BANK3_RAM"}, | |
| 4169 | − {AMDGPU_GFX_TCP_SET1BANK0_RAM, "TCP_SET1BANK0_RAM"}, | |
| 4170 | − {AMDGPU_GFX_TCP_SET1BANK1_RAM, "TCP_SET1BANK1_RAM"}, | |
| 4171 | − {AMDGPU_GFX_TCP_SET1BANK2_RAM, "TCP_SET1BANK2_RAM"}, | |
| 4172 | − {AMDGPU_GFX_TCP_SET1BANK3_RAM, "TCP_SET1BANK3_RAM"}, | |
| 4173 | − {AMDGPU_GFX_TCP_SET2BANK0_RAM, "TCP_SET2BANK0_RAM"}, | |
| 4174 | − {AMDGPU_GFX_TCP_SET2BANK1_RAM, "TCP_SET2BANK1_RAM"}, | |
| 4175 | − {AMDGPU_GFX_TCP_SET2BANK2_RAM, "TCP_SET2BANK2_RAM"}, | |
| 4176 | − {AMDGPU_GFX_TCP_SET2BANK3_RAM, "TCP_SET2BANK3_RAM"}, | |
| 4177 | − {AMDGPU_GFX_TCP_SET3BANK0_RAM, "TCP_SET3BANK0_RAM"}, | |
| 4178 | − {AMDGPU_GFX_TCP_SET3BANK1_RAM, "TCP_SET3BANK1_RAM"}, | |
| 4179 | − {AMDGPU_GFX_TCP_SET3BANK2_RAM, "TCP_SET3BANK2_RAM"}, | |
| 4180 | − {AMDGPU_GFX_TCP_SET3BANK3_RAM, "TCP_SET3BANK3_RAM"}, | |
| 4181 | − {AMDGPU_GFX_TCP_VM_FIFO, "TCP_VM_FIFO"}, | |
| 4182 | − {AMDGPU_GFX_TCP_DB_TAGRAM0, "TCP_DB_TAGRAM0"}, | |
| 4183 | − {AMDGPU_GFX_TCP_DB_TAGRAM1, "TCP_DB_TAGRAM1"}, | |
| 4184 | − {AMDGPU_GFX_TCP_DB_TAGRAM2, "TCP_DB_TAGRAM2"}, | |
| 4185 | − {AMDGPU_GFX_TCP_DB_TAGRAM3, "TCP_DB_TAGRAM3"}, | |
| 4186 | − {AMDGPU_GFX_TCP_UTCL1_LFIFO_PROBE0, "TCP_UTCL1_LFIFO_PROBE0"}, | |
| 4187 | − {AMDGPU_GFX_TCP_UTCL1_LFIFO_PROBE1, "TCP_UTCL1_LFIFO_PROBE1"}, | |
| 4188 | − {AMDGPU_GFX_TCP_CMD_FIFO, "TCP_CMD_FIFO"}, | |
| 4189 | −}; | |
| 4190 | − | |
| 4191 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_td_mem_list[] = { | |
| 4192 | − {AMDGPU_GFX_TD_UTD_CS_FIFO_MEM, "TD_UTD_CS_FIFO_MEM"}, | |
| 4193 | − {AMDGPU_GFX_TD_UTD_SS_FIFO_LO_MEM, "TD_UTD_SS_FIFO_LO_MEM"}, | |
| 4194 | − {AMDGPU_GFX_TD_UTD_SS_FIFO_HI_MEM, "TD_UTD_SS_FIFO_HI_MEM"}, | |
| 4195 | −}; | |
| 4196 | − | |
| 4197 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_tcx_mem_list[] = { | |
| 4198 | − {AMDGPU_GFX_TCX_FIFOD0, "TCX_FIFOD0"}, | |
| 4199 | − {AMDGPU_GFX_TCX_FIFOD1, "TCX_FIFOD1"}, | |
| 4200 | − {AMDGPU_GFX_TCX_FIFOD2, "TCX_FIFOD2"}, | |
| 4201 | − {AMDGPU_GFX_TCX_FIFOD3, "TCX_FIFOD3"}, | |
| 4202 | − {AMDGPU_GFX_TCX_FIFOD4, "TCX_FIFOD4"}, | |
| 4203 | − {AMDGPU_GFX_TCX_FIFOD5, "TCX_FIFOD5"}, | |
| 4204 | − {AMDGPU_GFX_TCX_FIFOD6, "TCX_FIFOD6"}, | |
| 4205 | − {AMDGPU_GFX_TCX_FIFOD7, "TCX_FIFOD7"}, | |
| 4206 | − {AMDGPU_GFX_TCX_FIFOB0, "TCX_FIFOB0"}, | |
| 4207 | − {AMDGPU_GFX_TCX_FIFOB1, "TCX_FIFOB1"}, | |
| 4208 | − {AMDGPU_GFX_TCX_FIFOB2, "TCX_FIFOB2"}, | |
| 4209 | − {AMDGPU_GFX_TCX_FIFOB3, "TCX_FIFOB3"}, | |
| 4210 | − {AMDGPU_GFX_TCX_FIFOB4, "TCX_FIFOB4"}, | |
| 4211 | − {AMDGPU_GFX_TCX_FIFOB5, "TCX_FIFOB5"}, | |
| 4212 | − {AMDGPU_GFX_TCX_FIFOB6, "TCX_FIFOB6"}, | |
| 4213 | − {AMDGPU_GFX_TCX_FIFOB7, "TCX_FIFOB7"}, | |
| 4214 | − {AMDGPU_GFX_TCX_FIFOA0, "TCX_FIFOA0"}, | |
| 4215 | − {AMDGPU_GFX_TCX_FIFOA1, "TCX_FIFOA1"}, | |
| 4216 | − {AMDGPU_GFX_TCX_FIFOA2, "TCX_FIFOA2"}, | |
| 4217 | − {AMDGPU_GFX_TCX_FIFOA3, "TCX_FIFOA3"}, | |
| 4218 | − {AMDGPU_GFX_TCX_FIFOA4, "TCX_FIFOA4"}, | |
| 4219 | − {AMDGPU_GFX_TCX_FIFOA5, "TCX_FIFOA5"}, | |
| 4220 | − {AMDGPU_GFX_TCX_FIFOA6, "TCX_FIFOA6"}, | |
| 4221 | − {AMDGPU_GFX_TCX_FIFOA7, "TCX_FIFOA7"}, | |
| 4222 | − {AMDGPU_GFX_TCX_CFIFO0, "TCX_CFIFO0"}, | |
| 4223 | − {AMDGPU_GFX_TCX_CFIFO1, "TCX_CFIFO1"}, | |
| 4224 | − {AMDGPU_GFX_TCX_CFIFO2, "TCX_CFIFO2"}, | |
| 4225 | − {AMDGPU_GFX_TCX_CFIFO3, "TCX_CFIFO3"}, | |
| 4226 | − {AMDGPU_GFX_TCX_CFIFO4, "TCX_CFIFO4"}, | |
| 4227 | − {AMDGPU_GFX_TCX_CFIFO5, "TCX_CFIFO5"}, | |
| 4228 | − {AMDGPU_GFX_TCX_CFIFO6, "TCX_CFIFO6"}, | |
| 4229 | − {AMDGPU_GFX_TCX_CFIFO7, "TCX_CFIFO7"}, | |
| 4230 | − {AMDGPU_GFX_TCX_FIFO_ACKB0, "TCX_FIFO_ACKB0"}, | |
| 4231 | − {AMDGPU_GFX_TCX_FIFO_ACKB1, "TCX_FIFO_ACKB1"}, | |
| 4232 | − {AMDGPU_GFX_TCX_FIFO_ACKB2, "TCX_FIFO_ACKB2"}, | |
| 4233 | − {AMDGPU_GFX_TCX_FIFO_ACKB3, "TCX_FIFO_ACKB3"}, | |
| 4234 | − {AMDGPU_GFX_TCX_FIFO_ACKB4, "TCX_FIFO_ACKB4"}, | |
| 4235 | − {AMDGPU_GFX_TCX_FIFO_ACKB5, "TCX_FIFO_ACKB5"}, | |
| 4236 | − {AMDGPU_GFX_TCX_FIFO_ACKB6, "TCX_FIFO_ACKB6"}, | |
| 4237 | − {AMDGPU_GFX_TCX_FIFO_ACKB7, "TCX_FIFO_ACKB7"}, | |
| 4238 | − {AMDGPU_GFX_TCX_FIFO_ACKD0, "TCX_FIFO_ACKD0"}, | |
| 4239 | − {AMDGPU_GFX_TCX_FIFO_ACKD1, "TCX_FIFO_ACKD1"}, | |
| 4240 | − {AMDGPU_GFX_TCX_FIFO_ACKD2, "TCX_FIFO_ACKD2"}, | |
| 4241 | − {AMDGPU_GFX_TCX_FIFO_ACKD3, "TCX_FIFO_ACKD3"}, | |
| 4242 | − {AMDGPU_GFX_TCX_FIFO_ACKD4, "TCX_FIFO_ACKD4"}, | |
| 4243 | − {AMDGPU_GFX_TCX_FIFO_ACKD5, "TCX_FIFO_ACKD5"}, | |
| 4244 | − {AMDGPU_GFX_TCX_FIFO_ACKD6, "TCX_FIFO_ACKD6"}, | |
| 4245 | − {AMDGPU_GFX_TCX_FIFO_ACKD7, "TCX_FIFO_ACKD7"}, | |
| 4246 | − {AMDGPU_GFX_TCX_DST_FIFOA0, "TCX_DST_FIFOA0"}, | |
| 4247 | − {AMDGPU_GFX_TCX_DST_FIFOA1, "TCX_DST_FIFOA1"}, | |
| 4248 | − {AMDGPU_GFX_TCX_DST_FIFOA2, "TCX_DST_FIFOA2"}, | |
| 4249 | − {AMDGPU_GFX_TCX_DST_FIFOA3, "TCX_DST_FIFOA3"}, | |
| 4250 | − {AMDGPU_GFX_TCX_DST_FIFOA4, "TCX_DST_FIFOA4"}, | |
| 4251 | − {AMDGPU_GFX_TCX_DST_FIFOA5, "TCX_DST_FIFOA5"}, | |
| 4252 | − {AMDGPU_GFX_TCX_DST_FIFOA6, "TCX_DST_FIFOA6"}, | |
| 4253 | − {AMDGPU_GFX_TCX_DST_FIFOA7, "TCX_DST_FIFOA7"}, | |
| 4254 | − {AMDGPU_GFX_TCX_DST_FIFOB0, "TCX_DST_FIFOB0"}, | |
| 4255 | − {AMDGPU_GFX_TCX_DST_FIFOB1, "TCX_DST_FIFOB1"}, | |
| 4256 | − {AMDGPU_GFX_TCX_DST_FIFOB2, "TCX_DST_FIFOB2"}, | |
| 4257 | − {AMDGPU_GFX_TCX_DST_FIFOB3, "TCX_DST_FIFOB3"}, | |
| 4258 | − {AMDGPU_GFX_TCX_DST_FIFOB4, "TCX_DST_FIFOB4"}, | |
| 4259 | − {AMDGPU_GFX_TCX_DST_FIFOB5, "TCX_DST_FIFOB5"}, | |
| 4260 | − {AMDGPU_GFX_TCX_DST_FIFOB6, "TCX_DST_FIFOB6"}, | |
| 4261 | − {AMDGPU_GFX_TCX_DST_FIFOB7, "TCX_DST_FIFOB7"}, | |
| 4262 | − {AMDGPU_GFX_TCX_DST_FIFOD0, "TCX_DST_FIFOD0"}, | |
| 4263 | − {AMDGPU_GFX_TCX_DST_FIFOD1, "TCX_DST_FIFOD1"}, | |
| 4264 | − {AMDGPU_GFX_TCX_DST_FIFOD2, "TCX_DST_FIFOD2"}, | |
| 4265 | − {AMDGPU_GFX_TCX_DST_FIFOD3, "TCX_DST_FIFOD3"}, | |
| 4266 | − {AMDGPU_GFX_TCX_DST_FIFOD4, "TCX_DST_FIFOD4"}, | |
| 4267 | − {AMDGPU_GFX_TCX_DST_FIFOD5, "TCX_DST_FIFOD5"}, | |
| 4268 | − {AMDGPU_GFX_TCX_DST_FIFOD6, "TCX_DST_FIFOD6"}, | |
| 4269 | − {AMDGPU_GFX_TCX_DST_FIFOD7, "TCX_DST_FIFOD7"}, | |
| 4270 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKB0, "TCX_DST_FIFO_ACKB0"}, | |
| 4271 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKB1, "TCX_DST_FIFO_ACKB1"}, | |
| 4272 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKB2, "TCX_DST_FIFO_ACKB2"}, | |
| 4273 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKB3, "TCX_DST_FIFO_ACKB3"}, | |
| 4274 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKB4, "TCX_DST_FIFO_ACKB4"}, | |
| 4275 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKB5, "TCX_DST_FIFO_ACKB5"}, | |
| 4276 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKB6, "TCX_DST_FIFO_ACKB6"}, | |
| 4277 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKB7, "TCX_DST_FIFO_ACKB7"}, | |
| 4278 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKD0, "TCX_DST_FIFO_ACKD0"}, | |
| 4279 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKD1, "TCX_DST_FIFO_ACKD1"}, | |
| 4280 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKD2, "TCX_DST_FIFO_ACKD2"}, | |
| 4281 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKD3, "TCX_DST_FIFO_ACKD3"}, | |
| 4282 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKD4, "TCX_DST_FIFO_ACKD4"}, | |
| 4283 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKD5, "TCX_DST_FIFO_ACKD5"}, | |
| 4284 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKD6, "TCX_DST_FIFO_ACKD6"}, | |
| 4285 | − {AMDGPU_GFX_TCX_DST_FIFO_ACKD7, "TCX_DST_FIFO_ACKD7"}, | |
| 4286 | −}; | |
| 4287 | − | |
| 4288 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_atc_l2_mem_list[] = { | |
| 4289 | − {AMDGPU_GFX_ATC_L2_MEM, "ATC_L2_MEM"}, | |
| 4290 | −}; | |
| 4291 | − | |
| 4292 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_utcl2_mem_list[] = { | |
| 4293 | − {AMDGPU_GFX_UTCL2_MEM, "UTCL2_MEM"}, | |
| 4294 | −}; | |
| 4295 | − | |
| 4296 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_vml2_mem_list[] = { | |
| 4297 | − {AMDGPU_GFX_VML2_MEM, "VML2_MEM"}, | |
| 4298 | −}; | |
| 4299 | − | |
| 4300 | −static const struct amdgpu_ras_memory_id_entry gfx_v9_4_3_ras_vml2_walker_mem_list[] = { | |
| 4301 | − {AMDGPU_GFX_VML2_WALKER_MEM, "VML2_WALKER_MEM"}, | |
| 4302 | −}; | |
| 4303 | − | |
| 4304 | −static const struct amdgpu_gfx_ras_mem_id_entry gfx_v9_4_3_ras_mem_list_array[AMDGPU_GFX_MEM_TYPE_NUM] = { | |
| 4305 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_cp_mem_list) | |
| 4306 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_gcea_mem_list) | |
| 4307 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_gc_cane_mem_list) | |
| 4308 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_gcutcl2_mem_list) | |
| 4309 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_gds_mem_list) | |
| 4310 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_lds_mem_list) | |
| 4311 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_rlc_mem_list) | |
| 4312 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_sp_mem_list) | |
| 4313 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_spi_mem_list) | |
| 4314 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_sqc_mem_list) | |
| 4315 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_sq_mem_list) | |
| 4316 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_ta_mem_list) | |
| 4317 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tcc_mem_list) | |
| 4318 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tca_mem_list) | |
| 4319 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tci_mem_list) | |
| 4320 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tcp_mem_list) | |
| 4321 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_td_mem_list) | |
| 4322 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_tcx_mem_list) | |
| 4323 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_atc_l2_mem_list) | |
| 4324 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_utcl2_mem_list) | |
| 4325 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_vml2_mem_list) | |
| 4326 | − AMDGPU_GFX_MEMID_ENT(gfx_v9_4_3_ras_vml2_walker_mem_list) | |
| 4327 | −}; | |
| 4328 | − | |
| 4329 | −static const struct amdgpu_gfx_ras_reg_entry gfx_v9_4_3_ce_reg_list[] = { | |
| 4330 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regRLC_CE_ERR_STATUS_LOW, regRLC_CE_ERR_STATUS_HIGH), | |
| 4331 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "RLC"}, | |
| 4332 | − AMDGPU_GFX_RLC_MEM, 1}, | |
| 4333 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPC_CE_ERR_STATUS_LO, regCPC_CE_ERR_STATUS_HI), | |
| 4334 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPC"}, | |
| 4335 | − AMDGPU_GFX_CP_MEM, 1}, | |
| 4336 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPF_CE_ERR_STATUS_LO, regCPF_CE_ERR_STATUS_HI), | |
| 4337 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPF"}, | |
| 4338 | − AMDGPU_GFX_CP_MEM, 1}, | |
| 4339 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPG_CE_ERR_STATUS_LO, regCPG_CE_ERR_STATUS_HI), | |
| 4340 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPG"}, | |
| 4341 | − AMDGPU_GFX_CP_MEM, 1}, | |
| 4342 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regGDS_CE_ERR_STATUS_LO, regGDS_CE_ERR_STATUS_HI), | |
| 4343 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GDS"}, | |
| 4344 | − AMDGPU_GFX_GDS_MEM, 1}, | |
| 4345 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regGC_CANE_CE_ERR_STATUS_LO, regGC_CANE_CE_ERR_STATUS_HI), | |
| 4346 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CANE"}, | |
| 4347 | − AMDGPU_GFX_GC_CANE_MEM, 1}, | |
| 4348 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSPI_CE_ERR_STATUS_LO, regSPI_CE_ERR_STATUS_HI), | |
| 4349 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SPI"}, | |
| 4350 | − AMDGPU_GFX_SPI_MEM, 1}, | |
| 4351 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP0_CE_ERR_STATUS_LO, regSP0_CE_ERR_STATUS_HI), | |
| 4352 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP0"}, | |
| 4353 | − AMDGPU_GFX_SP_MEM, 4}, | |
| 4354 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP1_CE_ERR_STATUS_LO, regSP1_CE_ERR_STATUS_HI), | |
| 4355 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP1"}, | |
| 4356 | − AMDGPU_GFX_SP_MEM, 4}, | |
| 4357 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQ_CE_ERR_STATUS_LO, regSQ_CE_ERR_STATUS_HI), | |
| 4358 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQ"}, | |
| 4359 | − AMDGPU_GFX_SQ_MEM, 4}, | |
| 4360 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQC_CE_EDC_LO, regSQC_CE_EDC_HI), | |
| 4361 | − 5, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQC"}, | |
| 4362 | − AMDGPU_GFX_SQC_MEM, 4}, | |
| 4363 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCX_CE_ERR_STATUS_LO, regTCX_CE_ERR_STATUS_HI), | |
| 4364 | − 2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCX"}, | |
| 4365 | − AMDGPU_GFX_TCX_MEM, 1}, | |
| 4366 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCC_CE_ERR_STATUS_LO, regTCC_CE_ERR_STATUS_HI), | |
| 4367 | − 16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCC"}, | |
| 4368 | − AMDGPU_GFX_TCC_MEM, 1}, | |
| 4369 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTA_CE_EDC_LO, regTA_CE_EDC_HI), | |
| 4370 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TA"}, | |
| 4371 | − AMDGPU_GFX_TA_MEM, 4}, | |
| 4372 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCI_CE_EDC_LO_REG, regTCI_CE_EDC_HI_REG), | |
| 4373 | − 27, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCI"}, | |
| 4374 | − AMDGPU_GFX_TCI_MEM, 1}, | |
| 4375 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCP_CE_EDC_LO_REG, regTCP_CE_EDC_HI_REG), | |
| 4376 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCP"}, | |
| 4377 | − AMDGPU_GFX_TCP_MEM, 4}, | |
| 4378 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTD_CE_EDC_LO, regTD_CE_EDC_HI), | |
| 4379 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TD"}, | |
| 4380 | − AMDGPU_GFX_TD_MEM, 4}, | |
| 4381 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regGCEA_CE_ERR_STATUS_LO, regGCEA_CE_ERR_STATUS_HI), | |
| 4382 | − 16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GCEA"}, | |
| 4383 | − AMDGPU_GFX_GCEA_MEM, 1}, | |
| 4384 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regLDS_CE_ERR_STATUS_LO, regLDS_CE_ERR_STATUS_HI), | |
| 4385 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "LDS"}, | |
| 4386 | − AMDGPU_GFX_LDS_MEM, 4}, | |
| 4387 | −}; | |
| 4388 | − | |
| 4389 | −static const struct amdgpu_gfx_ras_reg_entry gfx_v9_4_3_ue_reg_list[] = { | |
| 4390 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regRLC_UE_ERR_STATUS_LOW, regRLC_UE_ERR_STATUS_HIGH), | |
| 4391 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "RLC"}, | |
| 4392 | − AMDGPU_GFX_RLC_MEM, 1}, | |
| 4393 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPC_UE_ERR_STATUS_LO, regCPC_UE_ERR_STATUS_HI), | |
| 4394 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPC"}, | |
| 4395 | − AMDGPU_GFX_CP_MEM, 1}, | |
| 4396 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPF_UE_ERR_STATUS_LO, regCPF_UE_ERR_STATUS_HI), | |
| 4397 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPF"}, | |
| 4398 | − AMDGPU_GFX_CP_MEM, 1}, | |
| 4399 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regCPG_UE_ERR_STATUS_LO, regCPG_UE_ERR_STATUS_HI), | |
| 4400 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CPG"}, | |
| 4401 | − AMDGPU_GFX_CP_MEM, 1}, | |
| 4402 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regGDS_UE_ERR_STATUS_LO, regGDS_UE_ERR_STATUS_HI), | |
| 4403 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GDS"}, | |
| 4404 | − AMDGPU_GFX_GDS_MEM, 1}, | |
| 4405 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regGC_CANE_UE_ERR_STATUS_LO, regGC_CANE_UE_ERR_STATUS_HI), | |
| 4406 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "CANE"}, | |
| 4407 | − AMDGPU_GFX_GC_CANE_MEM, 1}, | |
| 4408 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSPI_UE_ERR_STATUS_LO, regSPI_UE_ERR_STATUS_HI), | |
| 4409 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SPI"}, | |
| 4410 | − AMDGPU_GFX_SPI_MEM, 1}, | |
| 4411 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP0_UE_ERR_STATUS_LO, regSP0_UE_ERR_STATUS_HI), | |
| 4412 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP0"}, | |
| 4413 | − AMDGPU_GFX_SP_MEM, 4}, | |
| 4414 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSP1_UE_ERR_STATUS_LO, regSP1_UE_ERR_STATUS_HI), | |
| 4415 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SP1"}, | |
| 4416 | − AMDGPU_GFX_SP_MEM, 4}, | |
| 4417 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQ_UE_ERR_STATUS_LO, regSQ_UE_ERR_STATUS_HI), | |
| 4418 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQ"}, | |
| 4419 | − AMDGPU_GFX_SQ_MEM, 4}, | |
| 4420 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regSQC_UE_EDC_LO, regSQC_UE_EDC_HI), | |
| 4421 | − 5, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SQC"}, | |
| 4422 | − AMDGPU_GFX_SQC_MEM, 4}, | |
| 4423 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCX_UE_ERR_STATUS_LO, regTCX_UE_ERR_STATUS_HI), | |
| 4424 | − 2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCX"}, | |
| 4425 | − AMDGPU_GFX_TCX_MEM, 1}, | |
| 4426 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCC_UE_ERR_STATUS_LO, regTCC_UE_ERR_STATUS_HI), | |
| 4427 | − 16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCC"}, | |
| 4428 | − AMDGPU_GFX_TCC_MEM, 1}, | |
| 4429 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTA_UE_EDC_LO, regTA_UE_EDC_HI), | |
| 4430 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TA"}, | |
| 4431 | − AMDGPU_GFX_TA_MEM, 4}, | |
| 4432 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCI_UE_EDC_LO_REG, regTCI_UE_EDC_HI_REG), | |
| 4433 | − 27, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCI"}, | |
| 4434 | − AMDGPU_GFX_TCI_MEM, 1}, | |
| 4435 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCP_UE_EDC_LO_REG, regTCP_UE_EDC_HI_REG), | |
| 4436 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCP"}, | |
| 4437 | − AMDGPU_GFX_TCP_MEM, 4}, | |
| 4438 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTD_UE_EDC_LO, regTD_UE_EDC_HI), | |
| 4439 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TD"}, | |
| 4440 | − AMDGPU_GFX_TD_MEM, 4}, | |
| 4441 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regTCA_UE_ERR_STATUS_LO, regTCA_UE_ERR_STATUS_HI), | |
| 4442 | − 2, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "TCA"}, | |
| 4443 | − AMDGPU_GFX_TCA_MEM, 1}, | |
| 4444 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regGCEA_UE_ERR_STATUS_LO, regGCEA_UE_ERR_STATUS_HI), | |
| 4445 | − 16, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "GCEA"}, | |
| 4446 | − AMDGPU_GFX_GCEA_MEM, 1}, | |
| 4447 | − {{AMDGPU_RAS_REG_ENTRY(GC, 0, regLDS_UE_ERR_STATUS_LO, regLDS_UE_ERR_STATUS_HI), | |
| 4448 | − 10, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "LDS"}, | |
| 4449 | − AMDGPU_GFX_LDS_MEM, 4}, | |
| 4450 | −}; | |
| 4451 | − | |
| 4452 | −static void gfx_v9_4_3_inst_query_ras_err_count(struct amdgpu_device *adev, | |
| 4453 | − void *ras_error_status, int xcc_id) | |
| 4454 | −{ | |
| 4455 | − struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; | |
| 4456 | − unsigned long ce_count = 0, ue_count = 0; | |
| 4457 | − uint32_t i, j, k; | |
| 4458 | − | |
| 4459 | − /* NOTE: convert xcc_id to physical XCD ID (XCD0 or XCD1) */ | |
| 4460 | − struct amdgpu_smuio_mcm_config_info mcm_info = { | |
| 4461 | − .socket_id = adev->smuio.funcs->get_socket_id(adev), | |
| 4462 | − .die_id = xcc_id & 0x01 ? 1 : 0, | |
| 4463 | − }; | |
| 4464 | − | |
| 4465 | − mutex_lock(&adev->grbm_idx_mutex); | |
| 4466 | − | |
| 4467 | − for (i = 0; i < ARRAY_SIZE(gfx_v9_4_3_ce_reg_list); i++) { | |
| 4468 | − for (j = 0; j < gfx_v9_4_3_ce_reg_list[i].se_num; j++) { | |
| 4469 | − for (k = 0; k < gfx_v9_4_3_ce_reg_list[i].reg_entry.reg_inst; k++) { | |
| 4470 | − /* no need to select if instance number is 1 */ | |
| 4471 | − if (gfx_v9_4_3_ce_reg_list[i].se_num > 1 || | |
| 4472 | − gfx_v9_4_3_ce_reg_list[i].reg_entry.reg_inst > 1) | |
| 4473 | − gfx_v9_4_3_xcc_select_se_sh(adev, j, 0, k, xcc_id); | |
| 4474 | − | |
| 4475 | − amdgpu_ras_inst_query_ras_error_count(adev, | |
| 4476 | − &(gfx_v9_4_3_ce_reg_list[i].reg_entry), | |
| 4477 | − 1, | |
| 4478 | − gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ce_reg_list[i].mem_id_type].mem_id_ent, | |
| 4479 | − gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ce_reg_list[i].mem_id_type].size, | |
| 4480 | − GET_INST(GC, xcc_id), | |
| 4481 | − AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE, | |
| 4482 | − &ce_count); | |
| 4483 | − | |
| 4484 | − amdgpu_ras_inst_query_ras_error_count(adev, | |
| 4485 | − &(gfx_v9_4_3_ue_reg_list[i].reg_entry), | |
| 4486 | − 1, | |
| 4487 | − gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].mem_id_ent, | |
| 4488 | − gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].size, | |
| 4489 | − GET_INST(GC, xcc_id), | |
| 4490 | − AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, | |
| 4491 | − &ue_count); | |
| 4492 | − } | |
| 4493 | − } | |
| 4494 | − } | |
| 4495 | − | |
| 4496 | − /* handle extra register entries of UE */ | |
| 4497 | − for (; i < ARRAY_SIZE(gfx_v9_4_3_ue_reg_list); i++) { | |
| 4498 | − for (j = 0; j < gfx_v9_4_3_ue_reg_list[i].se_num; j++) { | |
| 4499 | − for (k = 0; k < gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst; k++) { | |
| 4500 | − /* no need to select if instance number is 1 */ | |
| 4501 | − if (gfx_v9_4_3_ue_reg_list[i].se_num > 1 || | |
| 4502 | − gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst > 1) | |
| 4503 | − gfx_v9_4_3_xcc_select_se_sh(adev, j, 0, k, xcc_id); | |
| 4504 | − | |
| 4505 | − amdgpu_ras_inst_query_ras_error_count(adev, | |
| 4506 | − &(gfx_v9_4_3_ue_reg_list[i].reg_entry), | |
| 4507 | − 1, | |
| 4508 | − gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].mem_id_ent, | |
| 4509 | − gfx_v9_4_3_ras_mem_list_array[gfx_v9_4_3_ue_reg_list[i].mem_id_type].size, | |
| 4510 | − GET_INST(GC, xcc_id), | |
| 4511 | − AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, | |
| 4512 | − &ue_count); | |
| 4513 | − } | |
| 4514 | − } | |
| 4515 | − } | |
| 4516 | − | |
| 4517 | − gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, | |
| 4518 | − xcc_id); | |
| 4519 | − mutex_unlock(&adev->grbm_idx_mutex); | |
| 4520 | − | |
| 4521 | − /* the caller should make sure initialize value of | |
| 4522 | − * err_data->ue_count and err_data->ce_count | |
| 4523 | − */ | |
| 4524 | − amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count); | |
| 4525 | − amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, ce_count); | |
| 4526 | −} | |
| 4527 | − | |
| 4528 | −static void gfx_v9_4_3_inst_reset_ras_err_count(struct amdgpu_device *adev, | |
| 4529 | − void *ras_error_status, int xcc_id) | |
| 4530 | −{ | |
| 4531 | − uint32_t i, j, k; | |
| 4532 | − | |
| 4533 | − mutex_lock(&adev->grbm_idx_mutex); | |
| 4534 | − | |
| 4535 | − for (i = 0; i < ARRAY_SIZE(gfx_v9_4_3_ce_reg_list); i++) { | |
| 4536 | − for (j = 0; j < gfx_v9_4_3_ce_reg_list[i].se_num; j++) { | |
| 4537 | − for (k = 0; k < gfx_v9_4_3_ce_reg_list[i].reg_entry.reg_inst; k++) { | |
| 4538 | − /* no need to select if instance number is 1 */ | |
| 4539 | − if (gfx_v9_4_3_ce_reg_list[i].se_num > 1 || | |
| 4540 | − gfx_v9_4_3_ce_reg_list[i].reg_entry.reg_inst > 1) | |
| 4541 | − gfx_v9_4_3_xcc_select_se_sh(adev, j, 0, k, xcc_id); | |
| 4542 | − | |
| 4543 | − amdgpu_ras_inst_reset_ras_error_count(adev, | |
| 4544 | − &(gfx_v9_4_3_ce_reg_list[i].reg_entry), | |
| 4545 | − 1, | |
| 4546 | − GET_INST(GC, xcc_id)); | |
| 4547 | − | |
| 4548 | − amdgpu_ras_inst_reset_ras_error_count(adev, | |
| 4549 | − &(gfx_v9_4_3_ue_reg_list[i].reg_entry), | |
| 4550 | − 1, | |
| 4551 | − GET_INST(GC, xcc_id)); | |
| 4552 | − } | |
| 4553 | − } | |
| 4554 | − } | |
| 4555 | − | |
| 4556 | − /* handle extra register entries of UE */ | |
| 4557 | − for (; i < ARRAY_SIZE(gfx_v9_4_3_ue_reg_list); i++) { | |
| 4558 | − for (j = 0; j < gfx_v9_4_3_ue_reg_list[i].se_num; j++) { | |
| 4559 | − for (k = 0; k < gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst; k++) { | |
| 4560 | − /* no need to select if instance number is 1 */ | |
| 4561 | − if (gfx_v9_4_3_ue_reg_list[i].se_num > 1 || | |
| 4562 | − gfx_v9_4_3_ue_reg_list[i].reg_entry.reg_inst > 1) | |
| 4563 | − gfx_v9_4_3_xcc_select_se_sh(adev, j, 0, k, xcc_id); | |
| 4564 | − | |
| 4565 | − amdgpu_ras_inst_reset_ras_error_count(adev, | |
| 4566 | − &(gfx_v9_4_3_ue_reg_list[i].reg_entry), | |
| 4567 | − 1, | |
| 4568 | − GET_INST(GC, xcc_id)); | |
| 4569 | − } | |
| 4570 | − } | |
| 4571 | − } | |
| 4572 | − | |
| 4573 | − gfx_v9_4_3_xcc_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff, | |
| 4574 | − xcc_id); | |
| 4575 | − mutex_unlock(&adev->grbm_idx_mutex); | |
| 4576 | −} | |
| 4577 | − | |
| 4578 | 3729 | static void gfx_v9_4_3_inst_enable_watchdog_timer(struct amdgpu_device *adev, |
| 4579 | 3730 | void *ras_error_status, int xcc_id) |
| 4580 | 3731 | { |
| @@ -4607,18 +3758,6 @@ static void gfx_v9_4_3_inst_enable_watchdog_timer(struct amdgpu_device *adev, · 26 unmodified lines | ||
| 4607 | 3758 | mutex_unlock(&adev->grbm_idx_mutex); |
| 4608 | 3759 | } |
| 4609 | 3760 | |
| 4610 | −static void gfx_v9_4_3_query_ras_error_count(struct amdgpu_device *adev, | |
| 4611 | − void *ras_error_status) | |
| 4612 | −{ | |
| 4613 | − amdgpu_gfx_ras_error_func(adev, ras_error_status, | |
| 4614 | − gfx_v9_4_3_inst_query_ras_err_count); | |
| 4615 | −} | |
| 4616 | − | |
| 4617 | −static void gfx_v9_4_3_reset_ras_error_count(struct amdgpu_device *adev) | |
| 4618 | −{ | |
| 4619 | − amdgpu_gfx_ras_error_func(adev, NULL, gfx_v9_4_3_inst_reset_ras_err_count); | |
| 4620 | −} | |
| 4621 | − | |
| 4622 | 3761 | static void gfx_v9_4_3_enable_watchdog_timer(struct amdgpu_device *adev) |
| 4623 | 3762 | { |
| 4624 | 3763 | amdgpu_gfx_ras_error_func(adev, NULL, gfx_v9_4_3_inst_enable_watchdog_timer); |
| @@ -5099,37 +4238,9 @@ struct amdgpu_xcp_ip_funcs gfx_v9_4_3_xcp_funcs = { · 474 unmodified lines | ||
| 5099 | 4238 | .resume = &gfx_v9_4_3_xcp_resume |
| 5100 | 4239 | }; |
| 5101 | 4240 | |
| 5102 | −struct amdgpu_ras_block_hw_ops gfx_v9_4_3_ras_ops = { | |
| 5103 | − .query_ras_error_count = &gfx_v9_4_3_query_ras_error_count, | |
| 5104 | − .reset_ras_error_count = &gfx_v9_4_3_reset_ras_error_count, | |
| 5105 | −}; | |
| 5106 | − | |
| 5107 | −static int gfx_v9_4_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) | |
| 5108 | −{ | |
| 5109 | − int r; | |
| 5110 | − | |
| 5111 | − r = amdgpu_ras_block_late_init(adev, ras_block); | |
| 5112 | − if (r) | |
| 5113 | − return r; | |
| 5114 | − | |
| 5115 | − r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__GFX, | |
| 5116 | − &gfx_v9_4_3_aca_info, | |
| 5117 | − NULL); | |
| 5118 | − if (r) | |
| 5119 | − goto late_fini; | |
| 5120 | − | |
| 5121 | − return 0; | |
| 5122 | − | |
| 5123 | −late_fini: | |
| 5124 | − amdgpu_ras_block_late_fini(adev, ras_block); | |
| 5125 | − | |
| 5126 | − return r; | |
| 5127 | −} | |
| 5128 | − | |
| 5129 | 4241 | struct amdgpu_gfx_ras gfx_v9_4_3_ras = { |
| 5130 | 4242 | .ras_block = { |
| 5131 | − .hw_ops = &gfx_v9_4_3_ras_ops, | |
| 5132 | − .ras_late_init = &gfx_v9_4_3_ras_late_init, | |
| 4243 | + .hw_ops = NULL, | |
| 5133 | 4244 | }, |
| 5134 | 4245 | .enable_watchdog_timer = &gfx_v9_4_3_enable_watchdog_timer, |
| 5135 | 4246 | }; |
| @@ -167,44 +167,6 @@ static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev) · 166 unmodified lines | ||
| 167 | 167 | } |
| 168 | 168 | } |
| 169 | 169 | |
| 170 | −static void gmc_v8_0_mc_stop(struct amdgpu_device *adev) | |
| 171 | −{ | |
| 172 | − u32 blackout; | |
| 173 | − struct amdgpu_ip_block *ip_block; | |
| 174 | − | |
| 175 | − ip_block = amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GMC); | |
| 176 | − if (!ip_block) | |
| 177 | − return; | |
| 178 | − | |
| 179 | − gmc_v8_0_wait_for_idle(ip_block); | |
| 180 | − | |
| 181 | − blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL); | |
| 182 | − if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) { | |
| 183 | − /* Block CPU access */ | |
| 184 | − WREG32(mmBIF_FB_EN, 0); | |
| 185 | − /* blackout the MC */ | |
| 186 | − blackout = REG_SET_FIELD(blackout, | |
| 187 | − MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 1); | |
| 188 | − WREG32(mmMC_SHARED_BLACKOUT_CNTL, blackout); | |
| 189 | − } | |
| 190 | − /* wait for the MC to settle */ | |
| 191 | − udelay(100); | |
| 192 | −} | |
| 193 | − | |
| 194 | −static void gmc_v8_0_mc_resume(struct amdgpu_device *adev) | |
| 195 | −{ | |
| 196 | − u32 tmp; | |
| 197 | − | |
| 198 | − /* unblackout the MC */ | |
| 199 | − tmp = RREG32(mmMC_SHARED_BLACKOUT_CNTL); | |
| 200 | − tmp = REG_SET_FIELD(tmp, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE, 0); | |
| 201 | − WREG32(mmMC_SHARED_BLACKOUT_CNTL, tmp); | |
| 202 | − /* allow CPU access */ | |
| 203 | − tmp = REG_SET_FIELD(0, BIF_FB_EN, FB_READ_EN, 1); | |
| 204 | − tmp = REG_SET_FIELD(tmp, BIF_FB_EN, FB_WRITE_EN, 1); | |
| 205 | − WREG32(mmBIF_FB_EN, tmp); | |
| 206 | −} | |
| 207 | − | |
| 208 | 170 | /** |
| 209 | 171 | * gmc_v8_0_init_microcode - load ucode images from disk |
| 210 | 172 | * |
| @@ -1293,89 +1255,6 @@ static int gmc_v8_0_wait_for_idle(struct amdgpu_ip_block *ip_block) · 1082 unmodified lines | ||
| 1293 | 1255 | |
| 1294 | 1256 | } |
| 1295 | 1257 | |
| 1296 | −static bool gmc_v8_0_check_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 1297 | −{ | |
| 1298 | − u32 srbm_soft_reset = 0; | |
| 1299 | − struct amdgpu_device *adev = ip_block->adev; | |
| 1300 | − u32 tmp = RREG32(mmSRBM_STATUS); | |
| 1301 | − | |
| 1302 | − if (tmp & SRBM_STATUS__VMC_BUSY_MASK) | |
| 1303 | − srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, | |
| 1304 | − SRBM_SOFT_RESET, SOFT_RESET_VMC, 1); | |
| 1305 | − | |
| 1306 | − if (tmp & (SRBM_STATUS__MCB_BUSY_MASK | SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK | | |
| 1307 | − SRBM_STATUS__MCC_BUSY_MASK | SRBM_STATUS__MCD_BUSY_MASK)) { | |
| 1308 | − if (!(adev->flags & AMD_IS_APU)) | |
| 1309 | − srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, | |
| 1310 | − SRBM_SOFT_RESET, SOFT_RESET_MC, 1); | |
| 1311 | − } | |
| 1312 | − | |
| 1313 | − if (srbm_soft_reset) { | |
| 1314 | − adev->gmc.srbm_soft_reset = srbm_soft_reset; | |
| 1315 | − return true; | |
| 1316 | − } | |
| 1317 | − | |
| 1318 | − adev->gmc.srbm_soft_reset = 0; | |
| 1319 | − | |
| 1320 | − return false; | |
| 1321 | −} | |
| 1322 | − | |
| 1323 | −static int gmc_v8_0_pre_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 1324 | −{ | |
| 1325 | − struct amdgpu_device *adev = ip_block->adev; | |
| 1326 | − | |
| 1327 | − if (!adev->gmc.srbm_soft_reset) | |
| 1328 | − return 0; | |
| 1329 | − | |
| 1330 | − gmc_v8_0_mc_stop(adev); | |
| 1331 | − if (gmc_v8_0_wait_for_idle(ip_block)) | |
| 1332 | − dev_warn(adev->dev, "Wait for GMC idle timed out !\n"); | |
| 1333 | − | |
| 1334 | − return 0; | |
| 1335 | −} | |
| 1336 | − | |
| 1337 | −static int gmc_v8_0_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 1338 | −{ | |
| 1339 | − struct amdgpu_device *adev = ip_block->adev; | |
| 1340 | − u32 srbm_soft_reset; | |
| 1341 | − | |
| 1342 | − if (!adev->gmc.srbm_soft_reset) | |
| 1343 | − return 0; | |
| 1344 | − srbm_soft_reset = adev->gmc.srbm_soft_reset; | |
| 1345 | − | |
| 1346 | − if (srbm_soft_reset) { | |
| 1347 | − u32 tmp; | |
| 1348 | − | |
| 1349 | − tmp = RREG32(mmSRBM_SOFT_RESET); | |
| 1350 | − tmp |= srbm_soft_reset; | |
| 1351 | − dev_info(adev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp); | |
| 1352 | − WREG32(mmSRBM_SOFT_RESET, tmp); | |
| 1353 | − tmp = RREG32(mmSRBM_SOFT_RESET); | |
| 1354 | − | |
| 1355 | − udelay(50); | |
| 1356 | − | |
| 1357 | − tmp &= ~srbm_soft_reset; | |
| 1358 | − WREG32(mmSRBM_SOFT_RESET, tmp); | |
| 1359 | − tmp = RREG32(mmSRBM_SOFT_RESET); | |
| 1360 | − | |
| 1361 | − /* Wait a little for things to settle down */ | |
| 1362 | − udelay(50); | |
| 1363 | − } | |
| 1364 | − | |
| 1365 | − return 0; | |
| 1366 | −} | |
| 1367 | − | |
| 1368 | −static int gmc_v8_0_post_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 1369 | −{ | |
| 1370 | − struct amdgpu_device *adev = ip_block->adev; | |
| 1371 | − | |
| 1372 | − if (!adev->gmc.srbm_soft_reset) | |
| 1373 | − return 0; | |
| 1374 | − | |
| 1375 | − gmc_v8_0_mc_resume(adev); | |
| 1376 | − return 0; | |
| 1377 | −} | |
| 1378 | − | |
| 1379 | 1258 | static int gmc_v8_0_vm_fault_interrupt_state(struct amdgpu_device *adev, |
| 1380 | 1259 | struct amdgpu_irq_src *src, |
| 1381 | 1260 | unsigned int type, |
| @@ -1715,10 +1594,6 @@ static const struct amd_ip_funcs gmc_v8_0_ip_funcs = { · 333 unmodified lines | ||
| 1715 | 1594 | .resume = gmc_v8_0_resume, |
| 1716 | 1595 | .is_idle = gmc_v8_0_is_idle, |
| 1717 | 1596 | .wait_for_idle = gmc_v8_0_wait_for_idle, |
| 1718 | − .check_soft_reset = gmc_v8_0_check_soft_reset, | |
| 1719 | − .pre_soft_reset = gmc_v8_0_pre_soft_reset, | |
| 1720 | − .soft_reset = gmc_v8_0_soft_reset, | |
| 1721 | − .post_soft_reset = gmc_v8_0_post_soft_reset, | |
| 1722 | 1597 | .set_clockgating_state = gmc_v8_0_set_clockgating_state, |
| 1723 | 1598 | .set_powergating_state = gmc_v8_0_set_powergating_state, |
| 1724 | 1599 | .get_clockgating_state = gmc_v8_0_get_clockgating_state, |
| @@ -57,6 +57,7 @@ · 56 unmodified lines | ||
| 57 | 57 | #include "umc_v6_0.h" |
| 58 | 58 | #include "umc_v6_7.h" |
| 59 | 59 | #include "umc_v12_0.h" |
| 60 | +#include "ras_umc_v12_0.h" | |
| 60 | 61 | #include "hdp_v4_0.h" |
| 61 | 62 | #include "mca_v3_0.h" |
| 62 | 63 | |
| @@ -1382,7 +1383,7 @@ static void gmc_v9_0_set_umc_funcs(struct amdgpu_device *adev) · 1319 unmodified lines | ||
| 1382 | 1383 | case IP_VERSION(12, 0, 0): |
| 1383 | 1384 | case IP_VERSION(12, 5, 0): |
| 1384 | 1385 | adev->umc.max_ras_err_cnt_per_query = |
| 1385 | − UMC_V12_0_TOTAL_CHANNEL_NUM(adev) * UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL; | |
| 1386 | + UMC_V12_0_TOTAL_CHANNEL_NUM * UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL; | |
| 1386 | 1387 | adev->umc.channel_inst_num = UMC_V12_0_CHANNEL_INSTANCE_NUM; |
| 1387 | 1388 | adev->umc.umc_inst_num = UMC_V12_0_UMC_INSTANCE_NUM; |
| 1388 | 1389 | adev->umc.node_inst_num /= UMC_V12_0_UMC_INSTANCE_NUM; |
| @@ -2025,11 +2026,19 @@ static int gmc_v9_0_sw_init(struct amdgpu_ip_block *ip_block) · 636 unmodified lines | ||
| 2025 | 2026 | * The first KFD VMID is 8 for GPUs with graphics, 3 for |
| 2026 | 2027 | * compute-only GPUs. On compute-only GPUs that leaves 2 VMIDs |
| 2027 | 2028 | * for video processing. |
| 2029 | + * | |
| 2030 | + * If kernel queues are disabled, allow KFD to use all vmids. | |
| 2028 | 2031 | */ |
| 2029 | − adev->vm_manager.first_kfd_vmid = | |
| 2030 | − (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) || | |
| 2031 | − amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) || | |
| 2032 | − amdgpu_is_multi_aid(adev)) ? | |
| 2032 | + if (adev->gfx.disable_kq && | |
| 2033 | + adev->jpeg.disable_kq && | |
| 2034 | + adev->vcn.disable_kq && | |
| 2035 | + adev->sdma.no_user_submission) | |
| 2036 | + adev->vm_manager.first_kfd_vmid = 1; | |
| 2037 | + else | |
| 2038 | + adev->vm_manager.first_kfd_vmid = | |
| 2039 | + (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 1) || | |
| 2040 | + amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 2) || | |
| 2041 | + amdgpu_is_multi_aid(adev)) ? | |
| 2033 | 2042 | 3 : |
| 2034 | 2043 | 8; |
| 2035 | 2044 | |
| @@ -119,6 +119,19 @@ static int jpeg_v4_0_3_early_init(struct amdgpu_ip_block *ip_block) · 118 unmodified lines | ||
| 119 | 119 | { |
| 120 | 120 | struct amdgpu_device *adev = ip_block->adev; |
| 121 | 121 | |
| 122 | + switch (amdgpu_user_queue) { | |
| 123 | + case -1: | |
| 124 | + case 0: | |
| 125 | + default: | |
| 126 | + adev->jpeg.disable_kq = false; | |
| 127 | + adev->jpeg.disable_uq = true; | |
| 128 | + break; | |
| 129 | + case 2: | |
| 130 | + adev->jpeg.disable_kq = true; | |
| 131 | + adev->jpeg.disable_uq = true; | |
| 132 | + break; | |
| 133 | + } | |
| 134 | + | |
| 122 | 135 | adev->jpeg.num_jpeg_rings = AMDGPU_MAX_JPEG_RINGS_4_0_3; |
| 123 | 136 | |
| 124 | 137 | jpeg_v4_0_3_set_dec_ring_funcs(adev); |
| @@ -175,6 +188,10 @@ static int jpeg_v4_0_3_sw_init(struct amdgpu_ip_block *ip_block) · 50 unmodified lines | ||
| 175 | 188 | for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) { |
| 176 | 189 | ring = &adev->jpeg.inst[i].ring_dec[j]; |
| 177 | 190 | ring->use_doorbell = true; |
| 191 | + if (adev->jpeg.disable_kq) { | |
| 192 | + ring->no_scheduler = true; | |
| 193 | + ring->no_user_submission = true; | |
| 194 | + } | |
| 178 | 195 | ring->vm_hub = AMDGPU_MMHUB0(adev->jpeg.inst[i].aid_id); |
| 179 | 196 | if (!amdgpu_sriov_vf(adev)) { |
| 180 | 197 | ring->doorbell_index = |
| @@ -1425,72 +1442,6 @@ static const struct amdgpu_ras_block_hw_ops jpeg_v4_0_3_ras_hw_ops = { · 1244 unmodified lines | ||
| 1425 | 1442 | .query_poison_status = jpeg_v4_0_3_query_ras_poison_status, |
| 1426 | 1443 | }; |
| 1427 | 1444 | |
| 1428 | −static int jpeg_v4_0_3_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, | |
| 1429 | − enum aca_smu_type type, void *data) | |
| 1430 | −{ | |
| 1431 | − struct aca_bank_info info; | |
| 1432 | − u64 misc0; | |
| 1433 | − int ret; | |
| 1434 | − | |
| 1435 | − ret = aca_bank_info_decode(bank, &info); | |
| 1436 | − if (ret) | |
| 1437 | − return ret; | |
| 1438 | − | |
| 1439 | − misc0 = bank->regs[ACA_REG_IDX_MISC0]; | |
| 1440 | − switch (type) { | |
| 1441 | − case ACA_SMU_TYPE_UE: | |
| 1442 | − bank->aca_err_type = ACA_ERROR_TYPE_UE; | |
| 1443 | − ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, | |
| 1444 | − 1ULL); | |
| 1445 | − break; | |
| 1446 | − case ACA_SMU_TYPE_CE: | |
| 1447 | − bank->aca_err_type = ACA_ERROR_TYPE_CE; | |
| 1448 | − ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, | |
| 1449 | − ACA_REG__MISC0__ERRCNT(misc0)); | |
| 1450 | − break; | |
| 1451 | − default: | |
| 1452 | − return -EINVAL; | |
| 1453 | − } | |
| 1454 | − | |
| 1455 | − return ret; | |
| 1456 | −} | |
| 1457 | − | |
| 1458 | −/* reference to smu driver if header file */ | |
| 1459 | −static int jpeg_v4_0_3_err_codes[] = { | |
| 1460 | − 16, 17, 18, 19, 20, 21, 22, 23, /* JPEG[0-7][S|D] */ | |
| 1461 | − 24, 25, 26, 27, 28, 29, 30, 31 | |
| 1462 | −}; | |
| 1463 | − | |
| 1464 | −static bool jpeg_v4_0_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, | |
| 1465 | − enum aca_smu_type type, void *data) | |
| 1466 | −{ | |
| 1467 | − u32 instlo; | |
| 1468 | − | |
| 1469 | − instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); | |
| 1470 | − instlo &= GENMASK(31, 1); | |
| 1471 | − | |
| 1472 | − if (instlo != mmSMNAID_AID0_MCA_SMU) | |
| 1473 | − return false; | |
| 1474 | − | |
| 1475 | − if (aca_bank_check_error_codes(handle->adev, bank, | |
| 1476 | − jpeg_v4_0_3_err_codes, | |
| 1477 | − ARRAY_SIZE(jpeg_v4_0_3_err_codes))) | |
| 1478 | − return false; | |
| 1479 | − | |
| 1480 | − return true; | |
| 1481 | −} | |
| 1482 | − | |
| 1483 | −static const struct aca_bank_ops jpeg_v4_0_3_aca_bank_ops = { | |
| 1484 | − .aca_bank_parser = jpeg_v4_0_3_aca_bank_parser, | |
| 1485 | − .aca_bank_is_valid = jpeg_v4_0_3_aca_bank_is_valid, | |
| 1486 | −}; | |
| 1487 | − | |
| 1488 | −static const struct aca_info jpeg_v4_0_3_aca_info = { | |
| 1489 | − .hwip = ACA_HWIP_TYPE_SMU, | |
| 1490 | − .mask = ACA_ERROR_UE_MASK, | |
| 1491 | − .bank_ops = &jpeg_v4_0_3_aca_bank_ops, | |
| 1492 | −}; | |
| 1493 | − | |
| 1494 | 1445 | static int jpeg_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) |
| 1495 | 1446 | { |
| 1496 | 1447 | int r; |
| @@ -1506,11 +1457,6 @@ static int jpeg_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_comm · 9 unmodified lines | ||
| 1506 | 1457 | goto late_fini; |
| 1507 | 1458 | } |
| 1508 | 1459 | |
| 1509 | − r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__JPEG, | |
| 1510 | − &jpeg_v4_0_3_aca_info, NULL); | |
| 1511 | − if (r) | |
| 1512 | − goto late_fini; | |
| 1513 | − | |
| 1514 | 1460 | return 0; |
| 1515 | 1461 | |
| 1516 | 1462 | late_fini: |
| @@ -118,6 +118,19 @@ static int jpeg_v5_0_1_early_init(struct amdgpu_ip_block *ip_block) · 117 unmodified lines | ||
| 118 | 118 | if (!adev->jpeg.num_jpeg_inst || adev->jpeg.num_jpeg_inst > AMDGPU_MAX_JPEG_INSTANCES) |
| 119 | 119 | return -ENOENT; |
| 120 | 120 | |
| 121 | + switch (amdgpu_user_queue) { | |
| 122 | + case -1: | |
| 123 | + case 0: | |
| 124 | + default: | |
| 125 | + adev->jpeg.disable_kq = false; | |
| 126 | + adev->jpeg.disable_uq = true; | |
| 127 | + break; | |
| 128 | + case 2: | |
| 129 | + adev->jpeg.disable_kq = true; | |
| 130 | + adev->jpeg.disable_uq = true; | |
| 131 | + break; | |
| 132 | + } | |
| 133 | + | |
| 121 | 134 | adev->jpeg.num_jpeg_rings = AMDGPU_MAX_JPEG_RINGS; |
| 122 | 135 | jpeg_v5_0_1_set_dec_ring_funcs(adev); |
| 123 | 136 | jpeg_v5_0_1_set_irq_funcs(adev); |
| @@ -172,6 +185,10 @@ static int jpeg_v5_0_1_sw_init(struct amdgpu_ip_block *ip_block) · 48 unmodified lines | ||
| 172 | 185 | for (j = 0; j < adev->jpeg.num_jpeg_rings; ++j) { |
| 173 | 186 | ring = &adev->jpeg.inst[i].ring_dec[j]; |
| 174 | 187 | ring->use_doorbell = true; |
| 188 | + if (adev->jpeg.disable_kq) { | |
| 189 | + ring->no_scheduler = true; | |
| 190 | + ring->no_user_submission = true; | |
| 191 | + } | |
| 175 | 192 | ring->vm_hub = AMDGPU_MMHUB0(adev->jpeg.inst[i].aid_id); |
| 176 | 193 | if (!amdgpu_sriov_vf(adev)) { |
| 177 | 194 | ring->doorbell_index = |
| @@ -871,10 +888,7 @@ static const struct amd_ip_funcs jpeg_v5_0_1_ip_funcs = { · 693 unmodified lines | ||
| 871 | 888 | .resume = jpeg_v5_0_1_resume, |
| 872 | 889 | .is_idle = jpeg_v5_0_1_is_idle, |
| 873 | 890 | .wait_for_idle = jpeg_v5_0_1_wait_for_idle, |
| 874 | − .check_soft_reset = NULL, | |
| 875 | − .pre_soft_reset = NULL, | |
| 876 | 891 | .soft_reset = NULL, |
| 877 | − .post_soft_reset = NULL, | |
| 878 | 892 | .set_clockgating_state = jpeg_v5_0_1_set_clockgating_state, |
| 879 | 893 | .set_powergating_state = jpeg_v5_0_1_set_powergating_state, |
| 880 | 894 | .dump_ip_state = amdgpu_jpeg_dump_ip_state, |
| @@ -1003,73 +1017,6 @@ static const struct amdgpu_ras_block_hw_ops jpeg_v5_0_1_ras_hw_ops = { · 122 unmodified lines | ||
| 1003 | 1017 | .query_poison_status = jpeg_v5_0_1_query_ras_poison_status, |
| 1004 | 1018 | }; |
| 1005 | 1019 | |
| 1006 | −static int jpeg_v5_0_1_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, | |
| 1007 | − enum aca_smu_type type, void *data) | |
| 1008 | −{ | |
| 1009 | − struct aca_bank_info info; | |
| 1010 | − u64 misc0; | |
| 1011 | − int ret; | |
| 1012 | − | |
| 1013 | − ret = aca_bank_info_decode(bank, &info); | |
| 1014 | − if (ret) | |
| 1015 | − return ret; | |
| 1016 | − | |
| 1017 | − misc0 = bank->regs[ACA_REG_IDX_MISC0]; | |
| 1018 | − switch (type) { | |
| 1019 | − case ACA_SMU_TYPE_UE: | |
| 1020 | − bank->aca_err_type = ACA_ERROR_TYPE_UE; | |
| 1021 | − ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, | |
| 1022 | − 1ULL); | |
| 1023 | − break; | |
| 1024 | − case ACA_SMU_TYPE_CE: | |
| 1025 | − bank->aca_err_type = ACA_ERROR_TYPE_CE; | |
| 1026 | − ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, | |
| 1027 | − ACA_REG__MISC0__ERRCNT(misc0)); | |
| 1028 | − break; | |
| 1029 | − default: | |
| 1030 | − return -EINVAL; | |
| 1031 | − } | |
| 1032 | − | |
| 1033 | − return ret; | |
| 1034 | −} | |
| 1035 | − | |
| 1036 | −/* reference to smu driver if header file */ | |
| 1037 | −static int jpeg_v5_0_1_err_codes[] = { | |
| 1038 | − 16, 17, 18, 19, 20, 21, 22, 23, /* JPEG[0-9][S|D] */ | |
| 1039 | − 24, 25, 26, 27, 28, 29, 30, 31, | |
| 1040 | − 48, 49, 50, 51, | |
| 1041 | −}; | |
| 1042 | − | |
| 1043 | −static bool jpeg_v5_0_1_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, | |
| 1044 | − enum aca_smu_type type, void *data) | |
| 1045 | −{ | |
| 1046 | − u32 instlo; | |
| 1047 | − | |
| 1048 | − instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); | |
| 1049 | − instlo &= GENMASK(31, 1); | |
| 1050 | − | |
| 1051 | − if (instlo != mmSMNAID_AID0_MCA_SMU) | |
| 1052 | − return false; | |
| 1053 | − | |
| 1054 | − if (aca_bank_check_error_codes(handle->adev, bank, | |
| 1055 | − jpeg_v5_0_1_err_codes, | |
| 1056 | − ARRAY_SIZE(jpeg_v5_0_1_err_codes))) | |
| 1057 | − return false; | |
| 1058 | − | |
| 1059 | − return true; | |
| 1060 | −} | |
| 1061 | − | |
| 1062 | −static const struct aca_bank_ops jpeg_v5_0_1_aca_bank_ops = { | |
| 1063 | − .aca_bank_parser = jpeg_v5_0_1_aca_bank_parser, | |
| 1064 | − .aca_bank_is_valid = jpeg_v5_0_1_aca_bank_is_valid, | |
| 1065 | −}; | |
| 1066 | − | |
| 1067 | −static const struct aca_info jpeg_v5_0_1_aca_info = { | |
| 1068 | − .hwip = ACA_HWIP_TYPE_SMU, | |
| 1069 | − .mask = ACA_ERROR_UE_MASK, | |
| 1070 | − .bank_ops = &jpeg_v5_0_1_aca_bank_ops, | |
| 1071 | −}; | |
| 1072 | − | |
| 1073 | 1020 | static int jpeg_v5_0_1_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) |
| 1074 | 1021 | { |
| 1075 | 1022 | int r; |
| @@ -1078,11 +1025,6 @@ static int jpeg_v5_0_1_ras_late_init(struct amdgpu_device *adev, struct ras_comm · 2 unmodified lines | ||
| 1078 | 1025 | if (r) |
| 1079 | 1026 | return r; |
| 1080 | 1027 | |
| 1081 | − r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__JPEG, | |
| 1082 | − &jpeg_v5_0_1_aca_info, NULL); | |
| 1083 | − if (r) | |
| 1084 | − goto late_fini; | |
| 1085 | − | |
| 1086 | 1028 | if (amdgpu_ras_is_supported(adev, ras_block->block) && |
| 1087 | 1029 | adev->jpeg.inst->ras_poison_irq.funcs) { |
| 1088 | 1030 | r = amdgpu_irq_get(adev, &adev->jpeg.inst->ras_poison_irq, 0); |
| @@ -690,10 +690,7 @@ static const struct amd_ip_funcs jpeg_v5_0_2_ip_funcs = { · 689 unmodified lines | ||
| 690 | 690 | .resume = jpeg_v5_0_2_resume, |
| 691 | 691 | .is_idle = jpeg_v5_0_2_is_idle, |
| 692 | 692 | .wait_for_idle = jpeg_v5_0_2_wait_for_idle, |
| 693 | − .check_soft_reset = NULL, | |
| 694 | − .pre_soft_reset = NULL, | |
| 695 | 693 | .soft_reset = NULL, |
| 696 | − .post_soft_reset = NULL, | |
| 697 | 694 | .set_clockgating_state = jpeg_v5_0_2_set_clockgating_state, |
| 698 | 695 | .set_powergating_state = jpeg_v5_0_2_set_powergating_state, |
| 699 | 696 | .dump_ip_state = amdgpu_jpeg_dump_ip_state, |
| @@ -133,8 +133,8 @@ static int mes_userq_map(struct amdgpu_usermode_queue *queue) · 132 unmodified lines | ||
| 133 | 133 | queue_input.gang_quantum = 10000; |
| 134 | 134 | queue_input.paging = false; |
| 135 | 135 | |
| 136 | − queue_input.process_context_addr = ctx->gpu_addr; | |
| 137 | − queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ; | |
| 136 | + queue_input.process_context_addr = uq_mgr->proc_ctx_obj.gpu_addr; | |
| 137 | + queue_input.gang_context_addr = ctx->gpu_addr; | |
| 138 | 138 | queue_input.inprocess_gang_priority = AMDGPU_MES_PRIORITY_LEVEL_NORMAL; |
| 139 | 139 | queue_input.gang_global_priority_level = convert_to_mes_priority(queue->priority); |
| 140 | 140 | |
| @@ -169,7 +169,8 @@ static int mes_userq_unmap(struct amdgpu_usermode_queue *queue) · 28 unmodified lines | ||
| 169 | 169 | |
| 170 | 170 | memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); |
| 171 | 171 | queue_input.doorbell_offset = queue->doorbell_index; |
| 172 | − queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ; | |
| 172 | + queue_input.gang_context_addr = ctx->gpu_addr; | |
| 173 | + queue_input.queue_type = queue->queue_type; | |
| 173 | 174 | |
| 174 | 175 | amdgpu_mes_lock(&adev->mes); |
| 175 | 176 | r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); |
| @@ -179,6 +180,63 @@ static int mes_userq_unmap(struct amdgpu_usermode_queue *queue) · 3 unmodified lines | ||
| 179 | 180 | return r; |
| 180 | 181 | } |
| 181 | 182 | |
| 183 | +int mes_userq_reset(struct amdgpu_usermode_queue *queue) | |
| 184 | +{ | |
| 185 | + struct amdgpu_userq_mgr *uq_mgr = queue->userq_mgr; | |
| 186 | + struct amdgpu_device *adev = uq_mgr->adev; | |
| 187 | + struct mes_reset_queue_input queue_input; | |
| 188 | + int r; | |
| 189 | + | |
| 190 | + /* XXX: add a FW version check for SDMA per queue reset */ | |
| 191 | + memset(&queue_input, 0x0, sizeof(struct mes_reset_queue_input)); | |
| 192 | + queue_input.doorbell_offset = queue->doorbell_index; | |
| 193 | + queue_input.queue_type = queue->queue_type; | |
| 194 | + | |
| 195 | + amdgpu_mes_lock(&adev->mes); | |
| 196 | + r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); | |
| 197 | + amdgpu_mes_unlock(&adev->mes); | |
| 198 | + if (r) | |
| 199 | + return r; | |
| 200 | + return mes_userq_unmap(queue); | |
| 201 | +} | |
| 202 | + | |
| 203 | +int mes_userq_reset_queue(struct amdgpu_device *adev, | |
| 204 | + struct amdgpu_usermode_queue *guilty_uq, | |
| 205 | + int queue_type, | |
| 206 | + unsigned int pipe, | |
| 207 | + unsigned int queue, | |
| 208 | + unsigned int db) | |
| 209 | +{ | |
| 210 | + struct amdgpu_usermode_queue *uq; | |
| 211 | + bool use_mmio = adev->gfx.mec.use_mmio_for_reset; | |
| 212 | + unsigned long uq_id; | |
| 213 | + int r; | |
| 214 | + | |
| 215 | + xa_for_each(&adev->userq_doorbell_xa, uq_id, uq) { | |
| 216 | + if (uq->queue_type == queue_type) { | |
| 217 | + if (uq == guilty_uq) | |
| 218 | + continue; | |
| 219 | + if (uq->doorbell_index == db) { | |
| 220 | + uq->state = AMDGPU_USERQ_STATE_HUNG; | |
| 221 | + if (use_mmio) | |
| 222 | + r = amdgpu_mes_reset_queue_mmio(adev, queue_type, 0, 1, pipe, queue, 0); | |
| 223 | + else | |
| 224 | + r = amdgpu_mes_reset_user_queue(adev, queue_type, db, 0); | |
| 225 | + if (r) | |
| 226 | + return r; | |
| 227 | + r = mes_userq_unmap(uq); | |
| 228 | + if (r) | |
| 229 | + return r; | |
| 230 | + atomic_inc(&adev->gpu_reset_counter); | |
| 231 | + amdgpu_userq_fence_driver_force_completion(uq); | |
| 232 | + drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, NULL); | |
| 233 | + break; | |
| 234 | + } | |
| 235 | + } | |
| 236 | + } | |
| 237 | + return 0; | |
| 238 | +} | |
| 239 | + | |
| 182 | 240 | static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr, |
| 183 | 241 | struct amdgpu_usermode_queue *queue, |
| 184 | 242 | struct drm_amdgpu_userq_in *mqd_user) |
| @@ -186,12 +244,8 @@ static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr, · 1 unmodified line | ||
| 186 | 244 | struct amdgpu_userq_obj *ctx = &queue->fw_obj; |
| 187 | 245 | int r, size; |
| 188 | 246 | |
| 189 | − /* | |
| 190 | − * The FW expects at least one page space allocated for | |
| 191 | − * process ctx and gang ctx each. Create an object | |
| 192 | − * for the same. | |
| 193 | − */ | |
| 194 | − size = AMDGPU_USERQ_PROC_CTX_SZ + AMDGPU_USERQ_GANG_CTX_SZ; | |
| 247 | + /* The FW expects at least one page space allocated for gang ctx. */ | |
| 248 | + size = AMDGPU_USERQ_GANG_CTX_SZ; | |
| 195 | 249 | r = amdgpu_bo_create_kernel(uq_mgr->adev, size, 0, |
| 196 | 250 | AMDGPU_GEM_DOMAIN_GTT, |
| 197 | 251 | &ctx->obj, &ctx->gpu_addr, |
| @@ -205,54 +259,26 @@ static int mes_userq_create_ctx_space(struct amdgpu_userq_mgr *uq_mgr, · 7 unmodified lines | ||
| 205 | 259 | return 0; |
| 206 | 260 | } |
| 207 | 261 | |
| 208 | −static int mes_userq_detect_and_reset(struct amdgpu_device *adev, | |
| 209 | − int queue_type) | |
| 262 | +static int mes_userq_create_proc_ctx_space(struct amdgpu_userq_mgr *uq_mgr) | |
| 210 | 263 | { |
| 211 | − int db_array_size = amdgpu_mes_get_hung_queue_db_array_size(adev); | |
| 212 | − struct mes_detect_and_reset_queue_input input; | |
| 213 | − struct amdgpu_usermode_queue *queue; | |
| 214 | − unsigned int hung_db_num = 0; | |
| 215 | − unsigned long queue_id; | |
| 216 | − u32 db_array[8]; | |
| 217 | − bool found_hung_queue = false; | |
| 218 | − int r, i; | |
| 219 | − | |
| 220 | − if (db_array_size > 8) { | |
| 221 | − dev_err(adev->dev, "DB array size (%d vs 8) too small\n", | |
| 222 | − db_array_size); | |
| 223 | − return -EINVAL; | |
| 224 | − } | |
| 225 | − | |
| 226 | − memset(&input, 0x0, sizeof(struct mes_detect_and_reset_queue_input)); | |
| 264 | + int r = 0; | |
| 227 | 265 | |
| 228 | − input.queue_type = queue_type; | |
| 229 | − | |
| 230 | − amdgpu_mes_lock(&adev->mes); | |
| 231 | − r = amdgpu_mes_detect_and_reset_hung_queues(adev, queue_type, false, | |
| 232 | − &hung_db_num, db_array, 0); | |
| 233 | − amdgpu_mes_unlock(&adev->mes); | |
| 234 | − if (r) { | |
| 235 | − dev_err(adev->dev, "Failed to detect and reset queues, err (%d)\n", r); | |
| 236 | − } else if (hung_db_num) { | |
| 237 | − xa_for_each(&adev->userq_doorbell_xa, queue_id, queue) { | |
| 238 | − if (queue->queue_type == queue_type) { | |
| 239 | − for (i = 0; i < hung_db_num; i++) { | |
| 240 | − if (queue->doorbell_index == db_array[i]) { | |
| 241 | − queue->state = AMDGPU_USERQ_STATE_HUNG; | |
| 242 | − found_hung_queue = true; | |
| 243 | − atomic_inc(&adev->gpu_reset_counter); | |
| 244 | − amdgpu_userq_fence_driver_force_completion(queue); | |
| 245 | − drm_dev_wedged_event(adev_to_drm(adev), DRM_WEDGE_RECOVERY_NONE, NULL); | |
| 246 | − } | |
| 247 | − } | |
| 248 | − } | |
| 249 | − } | |
| 266 | + mutex_lock(&uq_mgr->proc_ctx_lock); | |
| 267 | + /* This check is a necessary because amdgpu_bo_create_kernel() | |
| 268 | + * calls helpers like amdgpu_bo_pin() and memset() unconditionally | |
| 269 | + */ | |
| 270 | + if (!uq_mgr->proc_ctx_obj.obj) { | |
| 271 | + r = amdgpu_bo_create_kernel(uq_mgr->adev, AMDGPU_USERQ_PROC_CTX_SZ, | |
| 272 | + 0, AMDGPU_GEM_DOMAIN_GTT, | |
| 273 | + &uq_mgr->proc_ctx_obj.obj, | |
| 274 | + &uq_mgr->proc_ctx_obj.gpu_addr, | |
| 275 | + &uq_mgr->proc_ctx_obj.cpu_ptr); | |
| 276 | + | |
| 277 | + if (!r) | |
| 278 | + memset(uq_mgr->proc_ctx_obj.cpu_ptr, 0, AMDGPU_USERQ_PROC_CTX_SZ); | |
| 250 | 279 | } |
| 251 | 280 | |
| 252 | − if (found_hung_queue) { | |
| 253 | − /* Resume scheduling after hang recovery */ | |
| 254 | − r = amdgpu_mes_resume(adev, input.xcc_id); | |
| 255 | − } | |
| 281 | + mutex_unlock(&uq_mgr->proc_ctx_lock); | |
| 256 | 282 | |
| 257 | 283 | return r; |
| 258 | 284 | } |
| @@ -429,7 +455,14 @@ static int mes_userq_mqd_create(struct amdgpu_usermode_queue *queue, · 170 unmodified lines | ||
| 429 | 455 | goto free_mqd; |
| 430 | 456 | } |
| 431 | 457 | |
| 432 | − /* Create BO for FW operations */ | |
| 458 | + /* Create per-process MES process context BO */ | |
| 459 | + r = mes_userq_create_proc_ctx_space(uq_mgr); | |
| 460 | + if (r) { | |
| 461 | + DRM_ERROR("Failed to allocate MES process context space bo, error: %d\n", r); | |
| 462 | + goto free_mqd; | |
| 463 | + } | |
| 464 | + | |
| 465 | + /* Create BO of a gang for FW operations */ | |
| 433 | 466 | r = mes_userq_create_ctx_space(uq_mgr, queue, mqd_user); |
| 434 | 467 | if (r) { |
| 435 | 468 | DRM_ERROR("Failed to allocate BO for userqueue (%d)", r); |
| @@ -497,7 +530,7 @@ static int mes_userq_preempt(struct amdgpu_usermode_queue *queue) · 61 unmodified lines | ||
| 497 | 530 | *fence_ptr = 0; |
| 498 | 531 | |
| 499 | 532 | memset(&queue_input, 0x0, sizeof(struct mes_suspend_gang_input)); |
| 500 | − queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ; | |
| 533 | + queue_input.gang_context_addr = ctx->gpu_addr; | |
| 501 | 534 | queue_input.suspend_fence_addr = fence_gpu_addr; |
| 502 | 535 | queue_input.suspend_fence_value = 1; |
| 503 | 536 | amdgpu_mes_lock(&adev->mes); |
| @@ -534,7 +567,7 @@ static int mes_userq_restore(struct amdgpu_usermode_queue *queue) · 30 unmodified lines | ||
| 534 | 567 | return 0; |
| 535 | 568 | |
| 536 | 569 | memset(&queue_input, 0x0, sizeof(struct mes_resume_gang_input)); |
| 537 | − queue_input.gang_context_addr = ctx->gpu_addr + AMDGPU_USERQ_PROC_CTX_SZ; | |
| 570 | + queue_input.gang_context_addr = ctx->gpu_addr; | |
| 538 | 571 | |
| 539 | 572 | amdgpu_mes_lock(&adev->mes); |
| 540 | 573 | r = adev->mes.funcs->resume_gang(&adev->mes, &queue_input); |
| @@ -549,7 +582,7 @@ const struct amdgpu_userq_funcs userq_mes_funcs = { · 8 unmodified lines | ||
| 549 | 582 | .mqd_destroy = mes_userq_mqd_destroy, |
| 550 | 583 | .unmap = mes_userq_unmap, |
| 551 | 584 | .map = mes_userq_map, |
| 552 | − .detect_and_reset = mes_userq_detect_and_reset, | |
| 553 | 585 | .preempt = mes_userq_preempt, |
| 554 | 586 | .restore = mes_userq_restore, |
| 587 | + .reset = mes_userq_reset, | |
| 555 | 588 | }; |
| @@ -27,4 +27,13 @@ · 26 unmodified lines | ||
| 27 | 27 | #include "amdgpu_userq.h" |
| 28 | 28 | |
| 29 | 29 | extern const struct amdgpu_userq_funcs userq_mes_funcs; |
| 30 | + | |
| 31 | +int mes_userq_reset(struct amdgpu_usermode_queue *queue); | |
| 32 | +int mes_userq_reset_queue(struct amdgpu_device *adev, | |
| 33 | + struct amdgpu_usermode_queue *guilty_uq, | |
| 34 | + int queue_type, | |
| 35 | + unsigned int pipe, | |
| 36 | + unsigned int queue, | |
| 37 | + unsigned int db); | |
| 38 | + | |
| 30 | 39 | #endif |
| @@ -387,6 +387,8 @@ static int mes_v11_0_remove_hw_queue(struct amdgpu_mes *mes, · 386 unmodified lines | ||
| 387 | 387 | |
| 388 | 388 | mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset; |
| 389 | 389 | mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr; |
| 390 | + mes_remove_queue_pkt.queue_type = | |
| 391 | + convert_to_mes_queue_type(input->queue_type); | |
| 390 | 392 | |
| 391 | 393 | if (mes_rev >= 0x60) |
| 392 | 394 | mes_remove_queue_pkt.remove_queue_after_reset = input->remove_queue_after_reset; |
| @@ -396,6 +398,230 @@ static int mes_v11_0_remove_hw_queue(struct amdgpu_mes *mes, · 3 unmodified lines | ||
| 396 | 398 | offsetof(union MESAPI__REMOVE_QUEUE, api_status)); |
| 397 | 399 | } |
| 398 | 400 | |
| 401 | +static bool mes_v11_0_pipe_reset_support(struct amdgpu_device *adev) | |
| 402 | +{ | |
| 403 | + /* Disable the pipe reset until the CPFW fully support it.*/ | |
| 404 | + dev_warn_once(adev->dev, "The CPFW hasn't support pipe reset yet.\n"); | |
| 405 | + return false; | |
| 406 | +} | |
| 407 | +static int mes_v11_0_reset_gfx_pipe_mmio(struct amdgpu_device *adev, | |
| 408 | + u32 me, u32 pipe, u32 queue) | |
| 409 | +{ | |
| 410 | + uint32_t reset_pipe = 0, clean_pipe = 0; | |
| 411 | + int r; | |
| 412 | + | |
| 413 | + if (!mes_v11_0_pipe_reset_support(adev)) | |
| 414 | + return -EOPNOTSUPP; | |
| 415 | + | |
| 416 | + amdgpu_gfx_rlc_enter_safe_mode(adev, 0); | |
| 417 | + mutex_lock(&adev->srbm_mutex); | |
| 418 | + soc21_grbm_select(adev, me, pipe, queue, 0); | |
| 419 | + | |
| 420 | + switch (pipe) { | |
| 421 | + case 0: | |
| 422 | + reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 423 | + PFP_PIPE0_RESET, 1); | |
| 424 | + reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 425 | + ME_PIPE0_RESET, 1); | |
| 426 | + clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 427 | + PFP_PIPE0_RESET, 0); | |
| 428 | + clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 429 | + ME_PIPE0_RESET, 0); | |
| 430 | + break; | |
| 431 | + case 1: | |
| 432 | + reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 433 | + PFP_PIPE1_RESET, 1); | |
| 434 | + reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 435 | + ME_PIPE1_RESET, 1); | |
| 436 | + clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 437 | + PFP_PIPE1_RESET, 0); | |
| 438 | + clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 439 | + ME_PIPE1_RESET, 0); | |
| 440 | + break; | |
| 441 | + default: | |
| 442 | + break; | |
| 443 | + } | |
| 444 | + | |
| 445 | + WREG32_SOC15(GC, 0, regCP_ME_CNTL, reset_pipe); | |
| 446 | + WREG32_SOC15(GC, 0, regCP_ME_CNTL, clean_pipe); | |
| 447 | + | |
| 448 | + r = (RREG32(SOC15_REG_OFFSET(GC, 0, regCP_GFX_RS64_INSTR_PNTR1)) << 2) - | |
| 449 | + RS64_FW_UC_START_ADDR_LO; | |
| 450 | + soc21_grbm_select(adev, 0, 0, 0, 0); | |
| 451 | + mutex_unlock(&adev->srbm_mutex); | |
| 452 | + amdgpu_gfx_rlc_exit_safe_mode(adev, 0); | |
| 453 | + | |
| 454 | + dev_info(adev->dev, "The gfx pipe reset to the ME firmware start PC: %s\n", | |
| 455 | + r == 0 ? "successfully" : "failed"); | |
| 456 | + /* FIXME: Sometimes driver can't cache the ME firmware start PC correctly, | |
| 457 | + * so the pipe reset status relies on the later gfx ring test result. | |
| 458 | + */ | |
| 459 | + return 0; | |
| 460 | +} | |
| 461 | + | |
| 462 | +/* | |
| 463 | + * With MEC pipe reset asserted, clear CP_HQD_ACTIVE / CP_HQD_DEQUEUE_REQUEST for | |
| 464 | + * every queue on (me, pipe). HQDs must be torn down while pipe reset stays | |
| 465 | + * asserted; only then clear the pipe reset bit. | |
| 466 | + * Caller must hold adev->srbm_mutex. | |
| 467 | + */ | |
| 468 | +static void mes_v11_0_clear_hqds_on_mec_pipe(struct amdgpu_device *adev, u32 me, | |
| 469 | + u32 pipe) | |
| 470 | +{ | |
| 471 | + unsigned int q; | |
| 472 | + | |
| 473 | + for (q = 0; q < adev->gfx.mec.num_queue_per_pipe; q++) { | |
| 474 | + soc21_grbm_select(adev, me, pipe, q, 0); | |
| 475 | + /* Start from a clean HQD dequeue state before forcing HQD inactive. */ | |
| 476 | + WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0); | |
| 477 | + WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0); | |
| 478 | + } | |
| 479 | +} | |
| 480 | + | |
| 481 | +static int mes_v11_0_reset_compute_pipe_mmio(struct amdgpu_device *adev, | |
| 482 | + u32 me, u32 pipe, u32 queue) | |
| 483 | +{ | |
| 484 | + uint32_t reset_val, clean_val; | |
| 485 | + int r; | |
| 486 | + | |
| 487 | + amdgpu_gfx_rlc_enter_safe_mode(adev, 0); | |
| 488 | + mutex_lock(&adev->srbm_mutex); | |
| 489 | + soc21_grbm_select(adev, me, pipe, queue, 0); | |
| 490 | + | |
| 491 | + if (adev->gfx.rs64_enable) { | |
| 492 | + reset_val = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); | |
| 493 | + clean_val = reset_val; | |
| 494 | + | |
| 495 | + switch (pipe) { | |
| 496 | + case 0: | |
| 497 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL, | |
| 498 | + MEC_PIPE0_RESET, 1); | |
| 499 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL, | |
| 500 | + MEC_PIPE0_RESET, 0); | |
| 501 | + break; | |
| 502 | + case 1: | |
| 503 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL, | |
| 504 | + MEC_PIPE1_RESET, 1); | |
| 505 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL, | |
| 506 | + MEC_PIPE1_RESET, 0); | |
| 507 | + break; | |
| 508 | + case 2: | |
| 509 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL, | |
| 510 | + MEC_PIPE2_RESET, 1); | |
| 511 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL, | |
| 512 | + MEC_PIPE2_RESET, 0); | |
| 513 | + break; | |
| 514 | + case 3: | |
| 515 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL, | |
| 516 | + MEC_PIPE3_RESET, 1); | |
| 517 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL, | |
| 518 | + MEC_PIPE3_RESET, 0); | |
| 519 | + break; | |
| 520 | + default: | |
| 521 | + break; | |
| 522 | + } | |
| 523 | + WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, reset_val); | |
| 524 | + mes_v11_0_clear_hqds_on_mec_pipe(adev, me, pipe); | |
| 525 | + WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, clean_val); | |
| 526 | + r = (RREG32_SOC15(GC, 0, regCP_MEC_RS64_INSTR_PNTR) << 2) - | |
| 527 | + RS64_FW_UC_START_ADDR_LO; | |
| 528 | + } else { | |
| 529 | + reset_val = RREG32_SOC15(GC, 0, regCP_MEC_CNTL); | |
| 530 | + clean_val = reset_val; | |
| 531 | + | |
| 532 | + if (me == 1) { | |
| 533 | + switch (pipe) { | |
| 534 | + case 0: | |
| 535 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL, | |
| 536 | + MEC_ME1_PIPE0_RESET, 1); | |
| 537 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL, | |
| 538 | + MEC_ME1_PIPE0_RESET, 0); | |
| 539 | + break; | |
| 540 | + case 1: | |
| 541 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL, | |
| 542 | + MEC_ME1_PIPE1_RESET, 1); | |
| 543 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL, | |
| 544 | + MEC_ME1_PIPE1_RESET, 0); | |
| 545 | + break; | |
| 546 | + case 2: | |
| 547 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL, | |
| 548 | + MEC_ME1_PIPE2_RESET, 1); | |
| 549 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL, | |
| 550 | + MEC_ME1_PIPE2_RESET, 0); | |
| 551 | + break; | |
| 552 | + case 3: | |
| 553 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL, | |
| 554 | + MEC_ME1_PIPE3_RESET, 1); | |
| 555 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL, | |
| 556 | + MEC_ME1_PIPE3_RESET, 0); | |
| 557 | + break; | |
| 558 | + default: | |
| 559 | + break; | |
| 560 | + } | |
| 561 | + /* mec1 fw pc: CP_MEC1_INSTR_PNTR */ | |
| 562 | + } else { | |
| 563 | + switch (pipe) { | |
| 564 | + case 0: | |
| 565 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL, | |
| 566 | + MEC_ME2_PIPE0_RESET, 1); | |
| 567 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL, | |
| 568 | + MEC_ME2_PIPE0_RESET, 0); | |
| 569 | + break; | |
| 570 | + case 1: | |
| 571 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL, | |
| 572 | + MEC_ME2_PIPE1_RESET, 1); | |
| 573 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL, | |
| 574 | + MEC_ME2_PIPE1_RESET, 0); | |
| 575 | + break; | |
| 576 | + case 2: | |
| 577 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL, | |
| 578 | + MEC_ME2_PIPE2_RESET, 1); | |
| 579 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL, | |
| 580 | + MEC_ME2_PIPE2_RESET, 0); | |
| 581 | + break; | |
| 582 | + case 3: | |
| 583 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL, | |
| 584 | + MEC_ME2_PIPE3_RESET, 1); | |
| 585 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL, | |
| 586 | + MEC_ME2_PIPE3_RESET, 0); | |
| 587 | + break; | |
| 588 | + default: | |
| 589 | + break; | |
| 590 | + } | |
| 591 | + /* mec2 fw pc: CP:CP_MEC2_INSTR_PNTR */ | |
| 592 | + } | |
| 593 | + WREG32_SOC15(GC, 0, regCP_MEC_CNTL, reset_val); | |
| 594 | + mes_v11_0_clear_hqds_on_mec_pipe(adev, me, pipe); | |
| 595 | + WREG32_SOC15(GC, 0, regCP_MEC_CNTL, clean_val); | |
| 596 | + r = RREG32(SOC15_REG_OFFSET(GC, 0, regCP_MEC1_INSTR_PNTR)); | |
| 597 | + } | |
| 598 | + | |
| 599 | + soc21_grbm_select(adev, 0, 0, 0, 0); | |
| 600 | + mutex_unlock(&adev->srbm_mutex); | |
| 601 | + amdgpu_gfx_rlc_exit_safe_mode(adev, 0); | |
| 602 | + | |
| 603 | + dev_dbg(adev->dev, "MEC pipe me%u pipe%u queue%u resets to MEC FW start PC: %s\n", | |
| 604 | + me, pipe, queue, r == 0 ? "successfully" : "failed"); | |
| 605 | + /*FIXME:Sometimes driver can't cache the MEC firmware start PC correctly, so the pipe | |
| 606 | + * reset status relies on the compute ring test result. | |
| 607 | + */ | |
| 608 | + return 0; | |
| 609 | +} | |
| 610 | + | |
| 611 | +static int mes_v11_0_reset_pipe_mmio(struct amdgpu_mes *mes, uint32_t queue_type, | |
| 612 | + uint32_t me_id, uint32_t pipe_id, | |
| 613 | + uint32_t queue_id, uint32_t vmid) | |
| 614 | +{ | |
| 615 | + struct amdgpu_device *adev = mes->adev; | |
| 616 | + | |
| 617 | + if (queue_type == AMDGPU_RING_TYPE_GFX) | |
| 618 | + return mes_v11_0_reset_gfx_pipe_mmio(adev, me_id, pipe_id, queue_id); | |
| 619 | + else if (queue_type == AMDGPU_RING_TYPE_COMPUTE) | |
| 620 | + return mes_v11_0_reset_compute_pipe_mmio(adev, me_id, pipe_id, queue_id); | |
| 621 | + else | |
| 622 | + return -EOPNOTSUPP; | |
| 623 | +} | |
| 624 | + | |
| 399 | 625 | static int mes_v11_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_type, |
| 400 | 626 | uint32_t me_id, uint32_t pipe_id, |
| 401 | 627 | uint32_t queue_id, uint32_t vmid) |
| @@ -770,10 +996,16 @@ static int mes_v11_0_reset_hw_queue(struct amdgpu_mes *mes, · 368 unmodified lines | ||
| 770 | 996 | { |
| 771 | 997 | union MESAPI__RESET mes_reset_queue_pkt; |
| 772 | 998 | |
| 773 | − if (input->use_mmio) | |
| 774 | − return mes_v11_0_reset_queue_mmio(mes, input->queue_type, | |
| 775 | − input->me_id, input->pipe_id, | |
| 776 | − input->queue_id, input->vmid); | |
| 999 | + if (input->use_mmio) { | |
| 1000 | + int r = mes_v11_0_reset_queue_mmio(mes, input->queue_type, | |
| 1001 | + input->me_id, input->pipe_id, | |
| 1002 | + input->queue_id, input->vmid); | |
| 1003 | + if (r) | |
| 1004 | + return mes_v11_0_reset_pipe_mmio(mes, input->queue_type, | |
| 1005 | + input->me_id, input->pipe_id, | |
| 1006 | + input->queue_id, input->vmid); | |
| 1007 | + return 0; | |
| 1008 | + } | |
| 777 | 1009 | |
| 778 | 1010 | memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt)); |
| 779 | 1011 | |
| @@ -26,7 +26,7 @@ · 25 unmodified lines | ||
| 26 | 26 | #include "amdgpu.h" |
| 27 | 27 | #include "gfx_v12_0.h" |
| 28 | 28 | #include "soc15_common.h" |
| 29 | −#include "soc21.h" | |
| 29 | +#include "soc24.h" | |
| 30 | 30 | #include "gc/gc_12_0_0_offset.h" |
| 31 | 31 | #include "gc/gc_12_0_0_sh_mask.h" |
| 32 | 32 | #include "gc/gc_11_0_0_default.h" |
| @@ -371,6 +371,8 @@ static int mes_v12_0_remove_hw_queue(struct amdgpu_mes *mes, · 338 unmodified lines | ||
| 371 | 371 | |
| 372 | 372 | mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset; |
| 373 | 373 | mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr; |
| 374 | + mes_remove_queue_pkt.queue_type = | |
| 375 | + convert_to_mes_queue_type(input->queue_type); | |
| 374 | 376 | |
| 375 | 377 | if (mes_rev >= 0x5a) |
| 376 | 378 | mes_remove_queue_pkt.remove_queue_after_reset = input->remove_queue_after_reset; |
| @@ -413,6 +415,171 @@ int gfx_v12_0_request_gfx_index_mutex(struct amdgpu_device *adev, · 36 unmodified lines | ||
| 413 | 415 | return 0; |
| 414 | 416 | } |
| 415 | 417 | |
| 418 | +static bool mes_v12_0_pipe_reset_support(struct amdgpu_device *adev) | |
| 419 | +{ | |
| 420 | + /* Disable the pipe reset until the CPFW fully support it.*/ | |
| 421 | + dev_warn_once(adev->dev, "The CPFW hasn't support pipe reset yet.\n"); | |
| 422 | + return false; | |
| 423 | +} | |
| 424 | + | |
| 425 | +static int mes_v12_0_reset_gfx_pipe_mmio(struct amdgpu_device *adev, | |
| 426 | + u32 me, u32 pipe, u32 queue) | |
| 427 | +{ | |
| 428 | + uint32_t reset_pipe = 0, clean_pipe = 0; | |
| 429 | + int r; | |
| 430 | + | |
| 431 | + if (!mes_v12_0_pipe_reset_support(adev)) | |
| 432 | + return -EOPNOTSUPP; | |
| 433 | + | |
| 434 | + amdgpu_gfx_rlc_enter_safe_mode(adev, 0); | |
| 435 | + mutex_lock(&adev->srbm_mutex); | |
| 436 | + soc24_grbm_select(adev, me, pipe, queue, 0); | |
| 437 | + | |
| 438 | + switch (pipe) { | |
| 439 | + case 0: | |
| 440 | + reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 441 | + PFP_PIPE0_RESET, 1); | |
| 442 | + reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 443 | + ME_PIPE0_RESET, 1); | |
| 444 | + clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 445 | + PFP_PIPE0_RESET, 0); | |
| 446 | + clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 447 | + ME_PIPE0_RESET, 0); | |
| 448 | + break; | |
| 449 | + case 1: | |
| 450 | + reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 451 | + PFP_PIPE1_RESET, 1); | |
| 452 | + reset_pipe = REG_SET_FIELD(reset_pipe, CP_ME_CNTL, | |
| 453 | + ME_PIPE1_RESET, 1); | |
| 454 | + clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 455 | + PFP_PIPE1_RESET, 0); | |
| 456 | + clean_pipe = REG_SET_FIELD(clean_pipe, CP_ME_CNTL, | |
| 457 | + ME_PIPE1_RESET, 0); | |
| 458 | + break; | |
| 459 | + default: | |
| 460 | + break; | |
| 461 | + } | |
| 462 | + | |
| 463 | + WREG32_SOC15(GC, 0, regCP_ME_CNTL, reset_pipe); | |
| 464 | + WREG32_SOC15(GC, 0, regCP_ME_CNTL, clean_pipe); | |
| 465 | + | |
| 466 | + r = (RREG32(SOC15_REG_OFFSET(GC, 0, regCP_GFX_RS64_INSTR_PNTR1)) << 2) - | |
| 467 | + RS64_FW_UC_START_ADDR_LO; | |
| 468 | + soc24_grbm_select(adev, 0, 0, 0, 0); | |
| 469 | + mutex_unlock(&adev->srbm_mutex); | |
| 470 | + amdgpu_gfx_rlc_exit_safe_mode(adev, 0); | |
| 471 | + | |
| 472 | + dev_info(adev->dev, "The gfx pipe reset: %s\n", | |
| 473 | + r == 0 ? "successfully" : "failed"); | |
| 474 | + /* Sometimes the ME start pc counter can't cache correctly, so the | |
| 475 | + * PC check only as a reference and pipe reset result rely on the | |
| 476 | + * later ring test. | |
| 477 | + */ | |
| 478 | + return 0; | |
| 479 | +} | |
| 480 | + | |
| 481 | +/* | |
| 482 | + * With MEC pipe reset asserted, clear CP_HQD_ACTIVE / CP_HQD_DEQUEUE_REQUEST for | |
| 483 | + * every queue on (me, pipe). HQDs must be torn down while pipe reset stays | |
| 484 | + * asserted; only then clear the pipe reset bit. | |
| 485 | + * Caller must hold adev->srbm_mutex. | |
| 486 | + */ | |
| 487 | +static void mes_v12_0_clear_hqds_on_mec_pipe(struct amdgpu_device *adev, u32 me, | |
| 488 | + u32 pipe) | |
| 489 | +{ | |
| 490 | + unsigned int q; | |
| 491 | + | |
| 492 | + for (q = 0; q < adev->gfx.mec.num_queue_per_pipe; q++) { | |
| 493 | + soc24_grbm_select(adev, me, pipe, q, 0); | |
| 494 | + /* Start from a clean HQD dequeue state before forcing HQD inactive. */ | |
| 495 | + WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, 0); | |
| 496 | + WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0); | |
| 497 | + } | |
| 498 | +} | |
| 499 | + | |
| 500 | +static int mes_v12_0_reset_compute_pipe_mmio(struct amdgpu_device *adev, | |
| 501 | + u32 me, u32 pipe, u32 queue) | |
| 502 | +{ | |
| 503 | + uint32_t reset_val, clean_val; | |
| 504 | + int r = 0; | |
| 505 | + | |
| 506 | + amdgpu_gfx_rlc_enter_safe_mode(adev, 0); | |
| 507 | + mutex_lock(&adev->srbm_mutex); | |
| 508 | + soc24_grbm_select(adev, me, pipe, queue, 0); | |
| 509 | + if (adev->gfx.rs64_enable) { | |
| 510 | + reset_val = RREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL); | |
| 511 | + clean_val = reset_val; | |
| 512 | + | |
| 513 | + switch (pipe) { | |
| 514 | + case 0: | |
| 515 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL, | |
| 516 | + MEC_PIPE0_RESET, 1); | |
| 517 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL, | |
| 518 | + MEC_PIPE0_RESET, 0); | |
| 519 | + break; | |
| 520 | + case 1: | |
| 521 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL, | |
| 522 | + MEC_PIPE1_RESET, 1); | |
| 523 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL, | |
| 524 | + MEC_PIPE1_RESET, 0); | |
| 525 | + break; | |
| 526 | + case 2: | |
| 527 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL, | |
| 528 | + MEC_PIPE2_RESET, 1); | |
| 529 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL, | |
| 530 | + MEC_PIPE2_RESET, 0); | |
| 531 | + break; | |
| 532 | + case 3: | |
| 533 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_RS64_CNTL, | |
| 534 | + MEC_PIPE3_RESET, 1); | |
| 535 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_RS64_CNTL, | |
| 536 | + MEC_PIPE3_RESET, 0); | |
| 537 | + break; | |
| 538 | + default: | |
| 539 | + break; | |
| 540 | + } | |
| 541 | + WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, reset_val); | |
| 542 | + mes_v12_0_clear_hqds_on_mec_pipe(adev, me, pipe); | |
| 543 | + soc24_grbm_select(adev, me, pipe, queue, 0); | |
| 544 | + WREG32_SOC15(GC, 0, regCP_MEC_RS64_CNTL, clean_val); | |
| 545 | + r = (RREG32_SOC15(GC, 0, regCP_MEC_RS64_INSTR_PNTR) << 2) - | |
| 546 | + RS64_FW_UC_START_ADDR_LO; | |
| 547 | + } else { | |
| 548 | + reset_val = RREG32_SOC15(GC, 0, regCP_MEC_CNTL); | |
| 549 | + clean_val = reset_val; | |
| 550 | + | |
| 551 | + switch (pipe) { | |
| 552 | + case 0: | |
| 553 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL, | |
| 554 | + MEC_ME1_PIPE0_RESET, 1); | |
| 555 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL, | |
| 556 | + MEC_ME1_PIPE0_RESET, 0); | |
| 557 | + break; | |
| 558 | + case 1: | |
| 559 | + reset_val = REG_SET_FIELD(reset_val, CP_MEC_CNTL, | |
| 560 | + MEC_ME1_PIPE1_RESET, 1); | |
| 561 | + clean_val = REG_SET_FIELD(clean_val, CP_MEC_CNTL, | |
| 562 | + MEC_ME1_PIPE1_RESET, 0); | |
| 563 | + break; | |
| 564 | + default: | |
| 565 | + break; | |
| 566 | + } | |
| 567 | + | |
| 568 | + WREG32_SOC15(GC, 0, regCP_MEC_CNTL, reset_val); | |
| 569 | + mes_v12_0_clear_hqds_on_mec_pipe(adev, me, pipe); | |
| 570 | + soc24_grbm_select(adev, me, pipe, queue, 0); | |
| 571 | + WREG32_SOC15(GC, 0, regCP_MEC_CNTL, clean_val); | |
| 572 | + } | |
| 573 | + | |
| 574 | + soc24_grbm_select(adev, 0, 0, 0, 0); | |
| 575 | + mutex_unlock(&adev->srbm_mutex); | |
| 576 | + amdgpu_gfx_rlc_exit_safe_mode(adev, 0); | |
| 577 | + | |
| 578 | + dev_dbg(adev->dev, "MEC pipe me%u pipe%u queue%u resets to MEC FW start PC: %s\n", | |
| 579 | + me, pipe, queue, r == 0 ? "successfully" : "failed"); | |
| 580 | + return 0; | |
| 581 | +} | |
| 582 | + | |
| 416 | 583 | static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_type, |
| 417 | 584 | uint32_t me_id, uint32_t pipe_id, |
| 418 | 585 | uint32_t queue_id, uint32_t vmid) |
| @@ -442,7 +609,7 @@ static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_typ · 23 unmodified lines | ||
| 442 | 609 | mutex_unlock(&adev->gfx.reset_sem_mutex); |
| 443 | 610 | |
| 444 | 611 | mutex_lock(&adev->srbm_mutex); |
| 445 | − soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0); | |
| 612 | + soc24_grbm_select(adev, me_id, pipe_id, queue_id, 0); | |
| 446 | 613 | /* wait till dequeue take effects */ |
| 447 | 614 | for (i = 0; i < adev->usec_timeout; i++) { |
| 448 | 615 | if (!(RREG32_SOC15(GC, 0, regCP_GFX_HQD_ACTIVE) & 1)) |
| @@ -454,13 +621,13 @@ static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_typ · 5 unmodified lines | ||
| 454 | 621 | r = -ETIMEDOUT; |
| 455 | 622 | } |
| 456 | 623 | |
| 457 | − soc21_grbm_select(adev, 0, 0, 0, 0); | |
| 624 | + soc24_grbm_select(adev, 0, 0, 0, 0); | |
| 458 | 625 | mutex_unlock(&adev->srbm_mutex); |
| 459 | 626 | } else if (queue_type == AMDGPU_RING_TYPE_COMPUTE) { |
| 460 | 627 | dev_info(adev->dev, "reset compute queue (%d:%d:%d)\n", |
| 461 | 628 | me_id, pipe_id, queue_id); |
| 462 | 629 | mutex_lock(&adev->srbm_mutex); |
| 463 | − soc21_grbm_select(adev, me_id, pipe_id, queue_id, 0); | |
| 630 | + soc24_grbm_select(adev, me_id, pipe_id, queue_id, 0); | |
| 464 | 631 | WREG32_SOC15(GC, 0, regCP_HQD_DEQUEUE_REQUEST, 0x2); |
| 465 | 632 | WREG32_SOC15(GC, 0, regSPI_COMPUTE_QUEUE_RESET, 0x1); |
| 466 | 633 | |
| @@ -474,7 +641,7 @@ static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_typ · 7 unmodified lines | ||
| 474 | 641 | dev_err(adev->dev, "failed to wait on hqd deactivate\n"); |
| 475 | 642 | r = -ETIMEDOUT; |
| 476 | 643 | } |
| 477 | − soc21_grbm_select(adev, 0, 0, 0, 0); | |
| 644 | + soc24_grbm_select(adev, 0, 0, 0, 0); | |
| 478 | 645 | mutex_unlock(&adev->srbm_mutex); |
| 479 | 646 | } else if (queue_type == AMDGPU_RING_TYPE_SDMA) { |
| 480 | 647 | dev_info(adev->dev, "reset sdma queue (%d:%d:%d)\n", |
| @@ -507,6 +674,20 @@ static int mes_v12_0_reset_queue_mmio(struct amdgpu_mes *mes, uint32_t queue_typ · 26 unmodified lines | ||
| 507 | 674 | return r; |
| 508 | 675 | } |
| 509 | 676 | |
| 677 | +static int mes_v12_0_reset_pipe_mmio(struct amdgpu_mes *mes, uint32_t queue_type, | |
| 678 | + uint32_t me_id, uint32_t pipe_id, | |
| 679 | + uint32_t queue_id, uint32_t vmid) | |
| 680 | +{ | |
| 681 | + struct amdgpu_device *adev = mes->adev; | |
| 682 | + | |
| 683 | + if (queue_type == AMDGPU_RING_TYPE_GFX) | |
| 684 | + return mes_v12_0_reset_gfx_pipe_mmio(adev, me_id, pipe_id, queue_id); | |
| 685 | + else if (queue_type == AMDGPU_RING_TYPE_COMPUTE) | |
| 686 | + return mes_v12_0_reset_compute_pipe_mmio(adev, me_id, pipe_id, queue_id); | |
| 687 | + else | |
| 688 | + return -EOPNOTSUPP; | |
| 689 | +} | |
| 690 | + | |
| 510 | 691 | static int mes_v12_0_map_legacy_queue(struct amdgpu_mes *mes, |
| 511 | 692 | struct mes_map_legacy_queue_input *input) |
| 512 | 693 | { |
| @@ -528,10 +709,15 @@ static int mes_v12_0_map_legacy_queue(struct amdgpu_mes *mes, · 15 unmodified lines | ||
| 528 | 709 | convert_to_mes_queue_type(input->queue_type); |
| 529 | 710 | mes_add_queue_pkt.map_legacy_kq = 1; |
| 530 | 711 | |
| 531 | − if (mes->adev->enable_uni_mes) | |
| 532 | − pipe = AMDGPU_MES_KIQ_PIPE; | |
| 533 | − else | |
| 712 | + if (mes->adev->enable_uni_mes) { | |
| 713 | + /* Keep scheduler queue on KIQ pipe; map all other kernel queues on sched pipe. */ | |
| 714 | + if (input->queue_type == AMDGPU_RING_TYPE_MES) | |
| 715 | + pipe = AMDGPU_MES_KIQ_PIPE; | |
| 716 | + else | |
| 717 | + pipe = AMDGPU_MES_SCHED_PIPE; | |
| 718 | + } else { | |
| 534 | 719 | pipe = AMDGPU_MES_SCHED_PIPE; |
| 720 | + } | |
| 535 | 721 | |
| 536 | 722 | return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe, |
| 537 | 723 | &mes_add_queue_pkt, sizeof(mes_add_queue_pkt), |
| @@ -565,12 +751,28 @@ static int mes_v12_0_unmap_legacy_queue(struct amdgpu_mes *mes, · 27 unmodified lines | ||
| 565 | 751 | mes_remove_queue_pkt.unmap_legacy_queue = 1; |
| 566 | 752 | mes_remove_queue_pkt.queue_type = |
| 567 | 753 | convert_to_mes_queue_type(input->queue_type); |
| 754 | + /* | |
| 755 | + * A reset-time unmap: the queue was already reset via MMIO while | |
| 756 | + * gangs are suspended and it is on the MES hung/fail list. Tell | |
| 757 | + * MES to just drop its internal state for it. Without this flag | |
| 758 | + * MES asks CP to unmap the already-reset (still wedged) queue | |
| 759 | + * again, which times out and forces a GPU reset. | |
| 760 | + */ | |
| 761 | + if (input->action == RESET_QUEUES && | |
| 762 | + (mes->sched_version & AMDGPU_MES_VERSION_MASK) >= 0x5a) | |
| 763 | + mes_remove_queue_pkt.remove_queue_after_reset = 1; | |
| 764 | + | |
| 568 | 765 | } |
| 569 | 766 | |
| 570 | − if (mes->adev->enable_uni_mes) | |
| 571 | − pipe = AMDGPU_MES_KIQ_PIPE; | |
| 572 | − else | |
| 767 | + if (mes->adev->enable_uni_mes) { | |
| 768 | + /* Keep scheduler queue on KIQ pipe; unmap all other kernel queues on sched pipe. */ | |
| 769 | + if (input->queue_type == AMDGPU_RING_TYPE_MES) | |
| 770 | + pipe = AMDGPU_MES_KIQ_PIPE; | |
| 771 | + else | |
| 772 | + pipe = AMDGPU_MES_SCHED_PIPE; | |
| 773 | + } else { | |
| 573 | 774 | pipe = AMDGPU_MES_SCHED_PIPE; |
| 775 | + } | |
| 574 | 776 | |
| 575 | 777 | return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe, |
| 576 | 778 | &mes_remove_queue_pkt, sizeof(mes_remove_queue_pkt), |
| @@ -888,10 +1090,16 @@ static int mes_v12_0_reset_hw_queue(struct amdgpu_mes *mes, · 311 unmodified lines | ||
| 888 | 1090 | union MESAPI__RESET mes_reset_queue_pkt; |
| 889 | 1091 | int pipe; |
| 890 | 1092 | |
| 891 | − if (input->use_mmio) | |
| 892 | − return mes_v12_0_reset_queue_mmio(mes, input->queue_type, | |
| 893 | − input->me_id, input->pipe_id, | |
| 894 | − input->queue_id, input->vmid); | |
| 1093 | + if (input->use_mmio) { | |
| 1094 | + int r = mes_v12_0_reset_queue_mmio(mes, input->queue_type, | |
| 1095 | + input->me_id, input->pipe_id, | |
| 1096 | + input->queue_id, input->vmid); | |
| 1097 | + if (r) | |
| 1098 | + return mes_v12_0_reset_pipe_mmio(mes, input->queue_type, | |
| 1099 | + input->me_id, input->pipe_id, | |
| 1100 | + input->queue_id, input->vmid); | |
| 1101 | + return 0; | |
| 1102 | + } | |
| 895 | 1103 | |
| 896 | 1104 | memset(&mes_reset_queue_pkt, 0, sizeof(mes_reset_queue_pkt)); |
| 897 | 1105 | |
| @@ -915,10 +1123,7 @@ static int mes_v12_0_reset_hw_queue(struct amdgpu_mes *mes, · 17 unmodified lines | ||
| 915 | 1123 | mes_reset_queue_pkt.doorbell_offset = input->doorbell_offset; |
| 916 | 1124 | } |
| 917 | 1125 | |
| 918 | − if (input->is_kq) | |
| 919 | − pipe = AMDGPU_MES_KIQ_PIPE; | |
| 920 | − else | |
| 921 | − pipe = AMDGPU_MES_SCHED_PIPE; | |
| 1126 | + pipe = AMDGPU_MES_SCHED_PIPE; | |
| 922 | 1127 | |
| 923 | 1128 | return mes_v12_0_submit_pkt_and_poll_completion(mes, pipe, |
| 924 | 1129 | &mes_reset_queue_pkt, sizeof(mes_reset_queue_pkt), |
| @@ -1094,7 +1299,7 @@ static void mes_v12_0_enable(struct amdgpu_device *adev, bool enable) · 169 unmodified lines | ||
| 1094 | 1299 | if (enable) { |
| 1095 | 1300 | mutex_lock(&adev->srbm_mutex); |
| 1096 | 1301 | for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) { |
| 1097 | − soc21_grbm_select(adev, 3, pipe, 0, 0); | |
| 1302 | + soc24_grbm_select(adev, 3, pipe, 0, 0); | |
| 1098 | 1303 | if (amdgpu_mes_log_enable) { |
| 1099 | 1304 | u32 log_size = AMDGPU_MES_LOG_BUFFER_SIZE + AMDGPU_MES_MSCRATCH_SIZE; |
| 1100 | 1305 | /* In case uni mes is not enabled, only program for pipe 0 */ |
| @@ -1133,7 +1338,7 @@ static void mes_v12_0_enable(struct amdgpu_device *adev, bool enable) · 32 unmodified lines | ||
| 1133 | 1338 | WREG32_SOC15(GC, 0, regCP_MES_CNTL, data); |
| 1134 | 1339 | |
| 1135 | 1340 | } |
| 1136 | − soc21_grbm_select(adev, 0, 0, 0, 0); | |
| 1341 | + soc24_grbm_select(adev, 0, 0, 0, 0); | |
| 1137 | 1342 | mutex_unlock(&adev->srbm_mutex); |
| 1138 | 1343 | |
| 1139 | 1344 | if (amdgpu_emu_mode) |
| @@ -1165,7 +1370,7 @@ static void mes_v12_0_set_ucode_start_addr(struct amdgpu_device *adev) · 25 unmodified lines | ||
| 1165 | 1370 | mutex_lock(&adev->srbm_mutex); |
| 1166 | 1371 | for (pipe = 0; pipe < AMDGPU_MAX_MES_PIPES; pipe++) { |
| 1167 | 1372 | /* me=3, queue=0 */ |
| 1168 | − soc21_grbm_select(adev, 3, pipe, 0, 0); | |
| 1373 | + soc24_grbm_select(adev, 3, pipe, 0, 0); | |
| 1169 | 1374 | |
| 1170 | 1375 | /* set ucode start address */ |
| 1171 | 1376 | ucode_addr = adev->mes.uc_start_addr[pipe] >> 2; |
| @@ -1174,7 +1379,7 @@ static void mes_v12_0_set_ucode_start_addr(struct amdgpu_device *adev) · 2 unmodified lines | ||
| 1174 | 1379 | WREG32_SOC15(GC, 0, regCP_MES_PRGRM_CNTR_START_HI, |
| 1175 | 1380 | upper_32_bits(ucode_addr)); |
| 1176 | 1381 | |
| 1177 | − soc21_grbm_select(adev, 0, 0, 0, 0); | |
| 1382 | + soc24_grbm_select(adev, 0, 0, 0, 0); | |
| 1178 | 1383 | } |
| 1179 | 1384 | mutex_unlock(&adev->srbm_mutex); |
| 1180 | 1385 | } |
| @@ -1203,7 +1408,7 @@ static int mes_v12_0_load_microcode(struct amdgpu_device *adev, · 22 unmodified lines | ||
| 1203 | 1408 | |
| 1204 | 1409 | mutex_lock(&adev->srbm_mutex); |
| 1205 | 1410 | /* me=3, pipe=0, queue=0 */ |
| 1206 | − soc21_grbm_select(adev, 3, pipe, 0, 0); | |
| 1411 | + soc24_grbm_select(adev, 3, pipe, 0, 0); | |
| 1207 | 1412 | |
| 1208 | 1413 | WREG32_SOC15(GC, 0, regCP_MES_IC_BASE_CNTL, 0); |
| 1209 | 1414 | |
| @@ -1238,7 +1443,7 @@ static int mes_v12_0_load_microcode(struct amdgpu_device *adev, · 28 unmodified lines | ||
| 1238 | 1443 | WREG32_SOC15(GC, 0, regCP_MES_IC_OP_CNTL, data); |
| 1239 | 1444 | } |
| 1240 | 1445 | |
| 1241 | − soc21_grbm_select(adev, 0, 0, 0, 0); | |
| 1446 | + soc24_grbm_select(adev, 0, 0, 0, 0); | |
| 1242 | 1447 | mutex_unlock(&adev->srbm_mutex); |
| 1243 | 1448 | |
| 1244 | 1449 | return 0; |
| @@ -1385,7 +1590,7 @@ static void mes_v12_0_queue_init_register(struct amdgpu_ring *ring) · 140 unmodified lines | ||
| 1385 | 1590 | uint32_t data = 0; |
| 1386 | 1591 | |
| 1387 | 1592 | mutex_lock(&adev->srbm_mutex); |
| 1388 | − soc21_grbm_select(adev, 3, ring->pipe, 0, 0); | |
| 1593 | + soc24_grbm_select(adev, 3, ring->pipe, 0, 0); | |
| 1389 | 1594 | |
| 1390 | 1595 | /* set CP_HQD_VMID.VMID = 0. */ |
| 1391 | 1596 | data = RREG32_SOC15(GC, 0, regCP_HQD_VMID); |
| @@ -1436,7 +1641,7 @@ static void mes_v12_0_queue_init_register(struct amdgpu_ring *ring) · 44 unmodified lines | ||
| 1436 | 1641 | /* set CP_HQD_ACTIVE.ACTIVE=1 */ |
| 1437 | 1642 | WREG32_SOC15(GC, 0, regCP_HQD_ACTIVE, mqd->cp_hqd_active); |
| 1438 | 1643 | |
| 1439 | − soc21_grbm_select(adev, 0, 0, 0, 0); | |
| 1644 | + soc24_grbm_select(adev, 0, 0, 0, 0); | |
| 1440 | 1645 | mutex_unlock(&adev->srbm_mutex); |
| 1441 | 1646 | } |
| 1442 | 1647 | |
| @@ -1502,14 +1707,14 @@ static int mes_v12_0_queue_init(struct amdgpu_device *adev, · 59 unmodified lines | ||
| 1502 | 1707 | ((pipe == AMDGPU_MES_KIQ_PIPE) && !adev->mes.kiq_version)) { |
| 1503 | 1708 | /* get MES scheduler/KIQ versions */ |
| 1504 | 1709 | mutex_lock(&adev->srbm_mutex); |
| 1505 | − soc21_grbm_select(adev, 3, pipe, 0, 0); | |
| 1710 | + soc24_grbm_select(adev, 3, pipe, 0, 0); | |
| 1506 | 1711 | |
| 1507 | 1712 | if (pipe == AMDGPU_MES_SCHED_PIPE) |
| 1508 | 1713 | adev->mes.sched_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO); |
| 1509 | 1714 | else if (pipe == AMDGPU_MES_KIQ_PIPE && adev->enable_mes_kiq) |
| 1510 | 1715 | adev->mes.kiq_version = RREG32_SOC15(GC, 0, regCP_MES_GP3_LO); |
| 1511 | 1716 | |
| 1512 | − soc21_grbm_select(adev, 0, 0, 0, 0); | |
| 1717 | + soc24_grbm_select(adev, 0, 0, 0, 0); | |
| 1513 | 1718 | mutex_unlock(&adev->srbm_mutex); |
| 1514 | 1719 | } |
| 1515 | 1720 | |
| @@ -1697,7 +1902,7 @@ static void mes_v12_0_kiq_dequeue_sched(struct amdgpu_device *adev) · 181 unmodified lines | ||
| 1697 | 1902 | int i; |
| 1698 | 1903 | |
| 1699 | 1904 | mutex_lock(&adev->srbm_mutex); |
| 1700 | − soc21_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0); | |
| 1905 | + soc24_grbm_select(adev, 3, AMDGPU_MES_SCHED_PIPE, 0, 0); | |
| 1701 | 1906 | |
| 1702 | 1907 | /* disable the queue if it's active */ |
| 1703 | 1908 | if (RREG32_SOC15(GC, 0, regCP_HQD_ACTIVE) & 1) { |
| @@ -1721,7 +1926,7 @@ static void mes_v12_0_kiq_dequeue_sched(struct amdgpu_device *adev) · 17 unmodified lines | ||
| 1721 | 1926 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_WPTR_HI, 0); |
| 1722 | 1927 | WREG32_SOC15(GC, 0, regCP_HQD_PQ_RPTR, 0); |
| 1723 | 1928 | |
| 1724 | − soc21_grbm_select(adev, 0, 0, 0, 0); | |
| 1929 | + soc24_grbm_select(adev, 0, 0, 0, 0); | |
| 1725 | 1930 | mutex_unlock(&adev->srbm_mutex); |
| 1726 | 1931 | |
| 1727 | 1932 | adev->mes.ring[0].sched.ready = false; |
| @@ -362,6 +362,8 @@ static int mes_v12_1_remove_hw_queue(struct amdgpu_mes *mes, · 361 unmodified lines | ||
| 362 | 362 | |
| 363 | 363 | mes_remove_queue_pkt.doorbell_offset = input->doorbell_offset; |
| 364 | 364 | mes_remove_queue_pkt.gang_context_addr = input->gang_context_addr; |
| 365 | + mes_remove_queue_pkt.queue_type = | |
| 366 | + convert_to_mes_queue_type(input->queue_type); | |
| 365 | 367 | |
| 366 | 368 | return mes_v12_1_submit_pkt_and_poll_completion(mes, |
| 367 | 369 | xcc_id, AMDGPU_MES_SCHED_PIPE, |
| @@ -417,10 +419,15 @@ static int mes_v12_1_map_legacy_queue(struct amdgpu_mes *mes, · 49 unmodified lines | ||
| 417 | 419 | convert_to_mes_queue_type(input->queue_type); |
| 418 | 420 | mes_add_queue_pkt.map_legacy_kq = 1; |
| 419 | 421 | |
| 420 | − if (mes->adev->enable_uni_mes) | |
| 421 | − pipe = AMDGPU_MES_KIQ_PIPE; | |
| 422 | − else | |
| 422 | + if (mes->adev->enable_uni_mes) { | |
| 423 | + /* Keep scheduler queue on KIQ pipe; map all other kernel queues on sched pipe. */ | |
| 424 | + if (input->queue_type == AMDGPU_RING_TYPE_MES) | |
| 425 | + pipe = AMDGPU_MES_KIQ_PIPE; | |
| 426 | + else | |
| 427 | + pipe = AMDGPU_MES_SCHED_PIPE; | |
| 428 | + } else { | |
| 423 | 429 | pipe = AMDGPU_MES_SCHED_PIPE; |
| 430 | + } | |
| 424 | 431 | |
| 425 | 432 | return mes_v12_1_submit_pkt_and_poll_completion(mes, |
| 426 | 433 | input->xcc_id, pipe, |
| @@ -457,10 +464,15 @@ static int mes_v12_1_unmap_legacy_queue(struct amdgpu_mes *mes, · 30 unmodified lines | ||
| 457 | 464 | convert_to_mes_queue_type(input->queue_type); |
| 458 | 465 | } |
| 459 | 466 | |
| 460 | − if (mes->adev->enable_uni_mes) | |
| 461 | − pipe = AMDGPU_MES_KIQ_PIPE; | |
| 462 | − else | |
| 467 | + if (mes->adev->enable_uni_mes) { | |
| 468 | + /* Keep scheduler queue on KIQ pipe; map all other kernel queues on sched pipe. */ | |
| 469 | + if (input->queue_type == AMDGPU_RING_TYPE_MES) | |
| 470 | + pipe = AMDGPU_MES_KIQ_PIPE; | |
| 471 | + else | |
| 472 | + pipe = AMDGPU_MES_SCHED_PIPE; | |
| 473 | + } else { | |
| 463 | 474 | pipe = AMDGPU_MES_SCHED_PIPE; |
| 475 | + } | |
| 464 | 476 | |
| 465 | 477 | return mes_v12_1_submit_pkt_and_poll_completion(mes, |
| 466 | 478 | input->xcc_id, pipe, |
| @@ -2262,6 +2274,7 @@ static int mes_v12_1_test_queue(struct amdgpu_device *adev, int xcc_id, · 1795 unmodified lines | ||
| 2262 | 2274 | remove_queue.xcc_id = xcc_id; |
| 2263 | 2275 | remove_queue.doorbell_offset = doorbell_idx; |
| 2264 | 2276 | remove_queue.gang_context_addr = add_queue.gang_context_addr; |
| 2277 | + remove_queue.queue_type = queue_type; | |
| 2265 | 2278 | r = mes_v12_1_remove_hw_queue(&adev->mes, &remove_queue); |
| 2266 | 2279 | |
| 2267 | 2280 | error: |
| @@ -29,7 +29,6 @@ · 28 unmodified lines | ||
| 29 | 29 | |
| 30 | 30 | #include "soc15_common.h" |
| 31 | 31 | #include "soc15.h" |
| 32 | −#include "amdgpu_ras.h" | |
| 33 | 32 | #include "amdgpu_psp.h" |
| 34 | 33 | |
| 35 | 34 | #define regVM_L2_CNTL3_DEFAULT 0x80100007 |
| @@ -636,236 +635,8 @@ const struct amdgpu_mmhub_funcs mmhub_v1_8_funcs = { · 600 unmodified lines | ||
| 636 | 635 | .get_clockgating = mmhub_v1_8_get_clockgating, |
| 637 | 636 | }; |
| 638 | 637 | |
| 639 | −static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ce_reg_list[] = { | |
| 640 | − {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA0_CE_ERR_STATUS_LO, regMMEA0_CE_ERR_STATUS_HI), | |
| 641 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA0"}, | |
| 642 | − {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA1_CE_ERR_STATUS_LO, regMMEA1_CE_ERR_STATUS_HI), | |
| 643 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA1"}, | |
| 644 | − {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA2_CE_ERR_STATUS_LO, regMMEA2_CE_ERR_STATUS_HI), | |
| 645 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA2"}, | |
| 646 | − {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA3_CE_ERR_STATUS_LO, regMMEA3_CE_ERR_STATUS_HI), | |
| 647 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA3"}, | |
| 648 | − {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA4_CE_ERR_STATUS_LO, regMMEA4_CE_ERR_STATUS_HI), | |
| 649 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA4"}, | |
| 650 | − {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMM_CANE_CE_ERR_STATUS_LO, regMM_CANE_CE_ERR_STATUS_HI), | |
| 651 | − 1, 0, "MM_CANE"}, | |
| 652 | −}; | |
| 653 | − | |
| 654 | −static const struct amdgpu_ras_err_status_reg_entry mmhub_v1_8_ue_reg_list[] = { | |
| 655 | − {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA0_UE_ERR_STATUS_LO, regMMEA0_UE_ERR_STATUS_HI), | |
| 656 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA0"}, | |
| 657 | − {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA1_UE_ERR_STATUS_LO, regMMEA1_UE_ERR_STATUS_HI), | |
| 658 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA1"}, | |
| 659 | − {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA2_UE_ERR_STATUS_LO, regMMEA2_UE_ERR_STATUS_HI), | |
| 660 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA2"}, | |
| 661 | − {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA3_UE_ERR_STATUS_LO, regMMEA3_UE_ERR_STATUS_HI), | |
| 662 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA3"}, | |
| 663 | − {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMMEA4_UE_ERR_STATUS_LO, regMMEA4_UE_ERR_STATUS_HI), | |
| 664 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "MMEA4"}, | |
| 665 | − {AMDGPU_RAS_REG_ENTRY(MMHUB, 0, regMM_CANE_UE_ERR_STATUS_LO, regMM_CANE_UE_ERR_STATUS_HI), | |
| 666 | − 1, 0, "MM_CANE"}, | |
| 667 | −}; | |
| 668 | − | |
| 669 | −static const struct amdgpu_ras_memory_id_entry mmhub_v1_8_ras_memory_list[] = { | |
| 670 | − {AMDGPU_MMHUB_WGMI_PAGEMEM, "MMEA_WGMI_PAGEMEM"}, | |
| 671 | − {AMDGPU_MMHUB_RGMI_PAGEMEM, "MMEA_RGMI_PAGEMEM"}, | |
| 672 | − {AMDGPU_MMHUB_WDRAM_PAGEMEM, "MMEA_WDRAM_PAGEMEM"}, | |
| 673 | − {AMDGPU_MMHUB_RDRAM_PAGEMEM, "MMEA_RDRAM_PAGEMEM"}, | |
| 674 | − {AMDGPU_MMHUB_WIO_CMDMEM, "MMEA_WIO_CMDMEM"}, | |
| 675 | − {AMDGPU_MMHUB_RIO_CMDMEM, "MMEA_RIO_CMDMEM"}, | |
| 676 | − {AMDGPU_MMHUB_WGMI_CMDMEM, "MMEA_WGMI_CMDMEM"}, | |
| 677 | − {AMDGPU_MMHUB_RGMI_CMDMEM, "MMEA_RGMI_CMDMEM"}, | |
| 678 | − {AMDGPU_MMHUB_WDRAM_CMDMEM, "MMEA_WDRAM_CMDMEM"}, | |
| 679 | − {AMDGPU_MMHUB_RDRAM_CMDMEM, "MMEA_RDRAM_CMDMEM"}, | |
| 680 | − {AMDGPU_MMHUB_MAM_DMEM0, "MMEA_MAM_DMEM0"}, | |
| 681 | − {AMDGPU_MMHUB_MAM_DMEM1, "MMEA_MAM_DMEM1"}, | |
| 682 | − {AMDGPU_MMHUB_MAM_DMEM2, "MMEA_MAM_DMEM2"}, | |
| 683 | − {AMDGPU_MMHUB_MAM_DMEM3, "MMEA_MAM_DMEM3"}, | |
| 684 | − {AMDGPU_MMHUB_WRET_TAGMEM, "MMEA_WRET_TAGMEM"}, | |
| 685 | − {AMDGPU_MMHUB_RRET_TAGMEM, "MMEA_RRET_TAGMEM"}, | |
| 686 | − {AMDGPU_MMHUB_WIO_DATAMEM, "MMEA_WIO_DATAMEM"}, | |
| 687 | − {AMDGPU_MMHUB_WGMI_DATAMEM, "MMEA_WGMI_DATAMEM"}, | |
| 688 | − {AMDGPU_MMHUB_WDRAM_DATAMEM, "MMEA_WDRAM_DATAMEM"}, | |
| 689 | −}; | |
| 690 | − | |
| 691 | −static void mmhub_v1_8_inst_query_ras_error_count(struct amdgpu_device *adev, | |
| 692 | − uint32_t mmhub_inst, | |
| 693 | − void *ras_err_status) | |
| 694 | −{ | |
| 695 | − struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; | |
| 696 | − unsigned long ue_count = 0, ce_count = 0; | |
| 697 | − | |
| 698 | − /* NOTE: mmhub is converted by aid_mask and the range is 0-3, | |
| 699 | − * which can be used as die ID directly */ | |
| 700 | − struct amdgpu_smuio_mcm_config_info mcm_info = { | |
| 701 | − .socket_id = adev->smuio.funcs->get_socket_id(adev), | |
| 702 | − .die_id = mmhub_inst, | |
| 703 | − }; | |
| 704 | − | |
| 705 | − amdgpu_ras_inst_query_ras_error_count(adev, | |
| 706 | − mmhub_v1_8_ce_reg_list, | |
| 707 | − ARRAY_SIZE(mmhub_v1_8_ce_reg_list), | |
| 708 | − mmhub_v1_8_ras_memory_list, | |
| 709 | − ARRAY_SIZE(mmhub_v1_8_ras_memory_list), | |
| 710 | − mmhub_inst, | |
| 711 | − AMDGPU_RAS_ERROR__SINGLE_CORRECTABLE, | |
| 712 | − &ce_count); | |
| 713 | − amdgpu_ras_inst_query_ras_error_count(adev, | |
| 714 | − mmhub_v1_8_ue_reg_list, | |
| 715 | − ARRAY_SIZE(mmhub_v1_8_ue_reg_list), | |
| 716 | − mmhub_v1_8_ras_memory_list, | |
| 717 | − ARRAY_SIZE(mmhub_v1_8_ras_memory_list), | |
| 718 | − mmhub_inst, | |
| 719 | − AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, | |
| 720 | − &ue_count); | |
| 721 | − | |
| 722 | − amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, ce_count); | |
| 723 | − amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count); | |
| 724 | −} | |
| 725 | − | |
| 726 | −static void mmhub_v1_8_query_ras_error_count(struct amdgpu_device *adev, | |
| 727 | − void *ras_err_status) | |
| 728 | −{ | |
| 729 | − uint32_t inst_mask; | |
| 730 | − uint32_t i; | |
| 731 | − | |
| 732 | − if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB)) { | |
| 733 | − dev_warn(adev->dev, "MMHUB RAS is not supported\n"); | |
| 734 | − return; | |
| 735 | − } | |
| 736 | − | |
| 737 | − inst_mask = adev->aid_mask; | |
| 738 | − for_each_inst(i, inst_mask) | |
| 739 | − mmhub_v1_8_inst_query_ras_error_count(adev, i, ras_err_status); | |
| 740 | −} | |
| 741 | − | |
| 742 | −static void mmhub_v1_8_inst_reset_ras_error_count(struct amdgpu_device *adev, | |
| 743 | − uint32_t mmhub_inst) | |
| 744 | −{ | |
| 745 | − amdgpu_ras_inst_reset_ras_error_count(adev, | |
| 746 | − mmhub_v1_8_ce_reg_list, | |
| 747 | − ARRAY_SIZE(mmhub_v1_8_ce_reg_list), | |
| 748 | − mmhub_inst); | |
| 749 | − amdgpu_ras_inst_reset_ras_error_count(adev, | |
| 750 | − mmhub_v1_8_ue_reg_list, | |
| 751 | − ARRAY_SIZE(mmhub_v1_8_ue_reg_list), | |
| 752 | − mmhub_inst); | |
| 753 | −} | |
| 754 | − | |
| 755 | −static void mmhub_v1_8_reset_ras_error_count(struct amdgpu_device *adev) | |
| 756 | −{ | |
| 757 | − uint32_t inst_mask; | |
| 758 | − uint32_t i; | |
| 759 | − | |
| 760 | − if (!amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__MMHUB)) { | |
| 761 | − dev_warn(adev->dev, "MMHUB RAS is not supported\n"); | |
| 762 | − return; | |
| 763 | − } | |
| 764 | − | |
| 765 | − inst_mask = adev->aid_mask; | |
| 766 | − for_each_inst(i, inst_mask) | |
| 767 | − mmhub_v1_8_inst_reset_ras_error_count(adev, i); | |
| 768 | −} | |
| 769 | − | |
| 770 | −static const struct amdgpu_ras_block_hw_ops mmhub_v1_8_ras_hw_ops = { | |
| 771 | − .query_ras_error_count = mmhub_v1_8_query_ras_error_count, | |
| 772 | − .reset_ras_error_count = mmhub_v1_8_reset_ras_error_count, | |
| 773 | −}; | |
| 774 | − | |
| 775 | −static int mmhub_v1_8_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, | |
| 776 | − enum aca_smu_type type, void *data) | |
| 777 | −{ | |
| 778 | − struct aca_bank_info info; | |
| 779 | − u64 misc0; | |
| 780 | − int ret; | |
| 781 | − | |
| 782 | − ret = aca_bank_info_decode(bank, &info); | |
| 783 | − if (ret) | |
| 784 | − return ret; | |
| 785 | − | |
| 786 | − misc0 = bank->regs[ACA_REG_IDX_MISC0]; | |
| 787 | − switch (type) { | |
| 788 | − case ACA_SMU_TYPE_UE: | |
| 789 | − bank->aca_err_type = ACA_ERROR_TYPE_UE; | |
| 790 | − ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, | |
| 791 | − 1ULL); | |
| 792 | − break; | |
| 793 | − case ACA_SMU_TYPE_CE: | |
| 794 | − bank->aca_err_type = ACA_ERROR_TYPE_CE; | |
| 795 | − ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, | |
| 796 | − ACA_REG__MISC0__ERRCNT(misc0)); | |
| 797 | − break; | |
| 798 | − default: | |
| 799 | − return -EINVAL; | |
| 800 | − } | |
| 801 | − | |
| 802 | − return ret; | |
| 803 | −} | |
| 804 | − | |
| 805 | −/* reference to smu driver if header file */ | |
| 806 | −static int mmhub_v1_8_err_codes[] = { | |
| 807 | − 0, 1, 2, 3, 4, /* CODE_DAGB0 - 4 */ | |
| 808 | − 5, 6, 7, 8, 9, /* CODE_EA0 - 4 */ | |
| 809 | − 10, /* CODE_UTCL2_ROUTER */ | |
| 810 | − 11, /* CODE_VML2 */ | |
| 811 | − 12, /* CODE_VML2_WALKER */ | |
| 812 | − 13, /* CODE_MMCANE */ | |
| 813 | −}; | |
| 814 | − | |
| 815 | −static bool mmhub_v1_8_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, | |
| 816 | − enum aca_smu_type type, void *data) | |
| 817 | −{ | |
| 818 | − u32 instlo; | |
| 819 | − | |
| 820 | − instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); | |
| 821 | − instlo &= GENMASK(31, 1); | |
| 822 | − | |
| 823 | − if (instlo != mmSMNAID_AID0_MCA_SMU) | |
| 824 | − return false; | |
| 825 | − | |
| 826 | − if (aca_bank_check_error_codes(handle->adev, bank, | |
| 827 | − mmhub_v1_8_err_codes, | |
| 828 | − ARRAY_SIZE(mmhub_v1_8_err_codes))) | |
| 829 | − return false; | |
| 830 | − | |
| 831 | − return true; | |
| 832 | −} | |
| 833 | − | |
| 834 | −static const struct aca_bank_ops mmhub_v1_8_aca_bank_ops = { | |
| 835 | − .aca_bank_parser = mmhub_v1_8_aca_bank_parser, | |
| 836 | − .aca_bank_is_valid = mmhub_v1_8_aca_bank_is_valid, | |
| 837 | −}; | |
| 838 | − | |
| 839 | −static const struct aca_info mmhub_v1_8_aca_info = { | |
| 840 | − .hwip = ACA_HWIP_TYPE_SMU, | |
| 841 | − .mask = ACA_ERROR_UE_MASK, | |
| 842 | − .bank_ops = &mmhub_v1_8_aca_bank_ops, | |
| 843 | −}; | |
| 844 | − | |
| 845 | −static int mmhub_v1_8_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) | |
| 846 | −{ | |
| 847 | − int r; | |
| 848 | − | |
| 849 | − r = amdgpu_ras_block_late_init(adev, ras_block); | |
| 850 | − if (r) | |
| 851 | − return r; | |
| 852 | − | |
| 853 | − r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__MMHUB, | |
| 854 | − &mmhub_v1_8_aca_info, NULL); | |
| 855 | − if (r) | |
| 856 | − goto late_fini; | |
| 857 | − | |
| 858 | − return 0; | |
| 859 | − | |
| 860 | −late_fini: | |
| 861 | − amdgpu_ras_block_late_fini(adev, ras_block); | |
| 862 | − | |
| 863 | − return r; | |
| 864 | −} | |
| 865 | − | |
| 866 | 638 | struct amdgpu_mmhub_ras mmhub_v1_8_ras = { |
| 867 | 639 | .ras_block = { |
| 868 | − .hw_ops = &mmhub_v1_8_ras_hw_ops, | |
| 869 | − .ras_late_init = mmhub_v1_8_ras_late_init, | |
| 640 | + .hw_ops = NULL, | |
| 870 | 641 | }, |
| 871 | 642 | }; |
| @@ -507,11 +507,6 @@ void nv_set_virt_ops(struct amdgpu_device *adev) · 506 unmodified lines | ||
| 507 | 507 | adev->virt.ops = &xgpu_nv_virt_ops; |
| 508 | 508 | } |
| 509 | 509 | |
| 510 | −static bool nv_need_full_reset(struct amdgpu_device *adev) | |
| 511 | −{ | |
| 512 | − return true; | |
| 513 | −} | |
| 514 | − | |
| 515 | 510 | static bool nv_need_reset_on_init(struct amdgpu_device *adev) |
| 516 | 511 | { |
| 517 | 512 | u32 sol_reg; |
| @@ -595,7 +590,6 @@ static const struct amdgpu_asic_funcs nv_asic_funcs = { · 77 unmodified lines | ||
| 595 | 590 | .set_vce_clocks = &nv_set_vce_clocks, |
| 596 | 591 | .get_config_memsize = &nv_get_config_memsize, |
| 597 | 592 | .init_doorbell_index = &nv_init_doorbell_index, |
| 598 | − .need_full_reset = &nv_need_full_reset, | |
| 599 | 593 | .need_reset_on_init = &nv_need_reset_on_init, |
| 600 | 594 | .get_pcie_replay_count = &amdgpu_nbio_get_pcie_replay_count, |
| 601 | 595 | .supports_baco = &amdgpu_dpm_is_baco_supported, |
| @@ -1237,65 +1237,6 @@ static int sdma_v3_0_wait_for_idle(struct amdgpu_ip_block *ip_block) · 1236 unmodified lines | ||
| 1237 | 1237 | return -ETIMEDOUT; |
| 1238 | 1238 | } |
| 1239 | 1239 | |
| 1240 | −static bool sdma_v3_0_check_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 1241 | −{ | |
| 1242 | − struct amdgpu_device *adev = ip_block->adev; | |
| 1243 | − u32 srbm_soft_reset = 0; | |
| 1244 | − u32 tmp = RREG32(mmSRBM_STATUS2); | |
| 1245 | − | |
| 1246 | − if ((tmp & SRBM_STATUS2__SDMA_BUSY_MASK) || | |
| 1247 | − (tmp & SRBM_STATUS2__SDMA1_BUSY_MASK)) { | |
| 1248 | − srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA_MASK; | |
| 1249 | − srbm_soft_reset |= SRBM_SOFT_RESET__SOFT_RESET_SDMA1_MASK; | |
| 1250 | − } | |
| 1251 | − | |
| 1252 | − if (srbm_soft_reset) { | |
| 1253 | − adev->sdma.srbm_soft_reset = srbm_soft_reset; | |
| 1254 | − return true; | |
| 1255 | − } else { | |
| 1256 | − adev->sdma.srbm_soft_reset = 0; | |
| 1257 | − return false; | |
| 1258 | − } | |
| 1259 | −} | |
| 1260 | − | |
| 1261 | −static int sdma_v3_0_pre_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 1262 | −{ | |
| 1263 | − struct amdgpu_device *adev = ip_block->adev; | |
| 1264 | − u32 srbm_soft_reset = 0; | |
| 1265 | − | |
| 1266 | − if (!adev->sdma.srbm_soft_reset) | |
| 1267 | − return 0; | |
| 1268 | − | |
| 1269 | − srbm_soft_reset = adev->sdma.srbm_soft_reset; | |
| 1270 | − | |
| 1271 | − if (REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA) || | |
| 1272 | − REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA1)) { | |
| 1273 | − sdma_v3_0_ctx_switch_enable(adev, false); | |
| 1274 | − sdma_v3_0_enable(adev, false); | |
| 1275 | − } | |
| 1276 | − | |
| 1277 | − return 0; | |
| 1278 | −} | |
| 1279 | − | |
| 1280 | −static int sdma_v3_0_post_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 1281 | −{ | |
| 1282 | − struct amdgpu_device *adev = ip_block->adev; | |
| 1283 | − u32 srbm_soft_reset = 0; | |
| 1284 | − | |
| 1285 | − if (!adev->sdma.srbm_soft_reset) | |
| 1286 | − return 0; | |
| 1287 | − | |
| 1288 | − srbm_soft_reset = adev->sdma.srbm_soft_reset; | |
| 1289 | − | |
| 1290 | − if (REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA) || | |
| 1291 | − REG_GET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_SDMA1)) { | |
| 1292 | − sdma_v3_0_gfx_resume(adev); | |
| 1293 | − sdma_v3_0_rlc_resume(adev); | |
| 1294 | − } | |
| 1295 | − | |
| 1296 | − return 0; | |
| 1297 | −} | |
| 1298 | − | |
| 1299 | 1240 | static int sdma_v3_0_soft_reset(struct amdgpu_ip_block *ip_block) |
| 1300 | 1241 | { |
| 1301 | 1242 | struct amdgpu_device *adev = ip_block->adev; |
| @@ -1552,9 +1493,6 @@ static const struct amd_ip_funcs sdma_v3_0_ip_funcs = { · 250 unmodified lines | ||
| 1552 | 1493 | .resume = sdma_v3_0_resume, |
| 1553 | 1494 | .is_idle = sdma_v3_0_is_idle, |
| 1554 | 1495 | .wait_for_idle = sdma_v3_0_wait_for_idle, |
| 1555 | − .check_soft_reset = sdma_v3_0_check_soft_reset, | |
| 1556 | − .pre_soft_reset = sdma_v3_0_pre_soft_reset, | |
| 1557 | − .post_soft_reset = sdma_v3_0_post_soft_reset, | |
| 1558 | 1496 | .soft_reset = sdma_v3_0_soft_reset, |
| 1559 | 1497 | .set_clockgating_state = sdma_v3_0_set_clockgating_state, |
| 1560 | 1498 | .set_powergating_state = sdma_v3_0_set_powergating_state, |
| @@ -95,8 +95,6 @@ static const struct amdgpu_hwip_reg_entry sdma_reg_list_4_4_2[] = { · 94 unmodified lines | ||
| 95 | 95 | SOC15_REG_ENTRY_STR(GC, 0, regSDMA_VM_CNTL) |
| 96 | 96 | }; |
| 97 | 97 | |
| 98 | −#define mmSMNAID_AID0_MCA_SMU 0x03b30400 | |
| 99 | − | |
| 100 | 98 | #define WREG32_SDMA(instance, offset, value) \ |
| 101 | 99 | WREG32(sdma_v4_4_2_get_reg_offset(adev, (instance), (offset)), value) |
| 102 | 100 | #define RREG32_SDMA(instance, offset) \ |
| @@ -1359,6 +1357,19 @@ static int sdma_v4_4_2_early_init(struct amdgpu_ip_block *ip_block) · 1256 unmodified lines | ||
| 1359 | 1357 | struct amdgpu_device *adev = ip_block->adev; |
| 1360 | 1358 | int r; |
| 1361 | 1359 | |
| 1360 | + switch (amdgpu_user_queue) { | |
| 1361 | + case -1: | |
| 1362 | + case 0: | |
| 1363 | + default: | |
| 1364 | + adev->sdma.no_user_submission = false; | |
| 1365 | + adev->sdma.disable_uq = true; | |
| 1366 | + break; | |
| 1367 | + case 2: | |
| 1368 | + adev->sdma.no_user_submission = true; | |
| 1369 | + adev->sdma.disable_uq = true; | |
| 1370 | + break; | |
| 1371 | + } | |
| 1372 | + | |
| 1362 | 1373 | r = sdma_v4_4_2_init_microcode(adev); |
| 1363 | 1374 | if (r) |
| 1364 | 1375 | return r; |
| @@ -1478,6 +1489,7 @@ static int sdma_v4_4_2_sw_init(struct amdgpu_ip_block *ip_block) · 113 unmodified lines | ||
| 1478 | 1489 | /* doorbell size is 2 dwords, get DWORD offset */ |
| 1479 | 1490 | ring->doorbell_index = adev->doorbell_index.sdma_engine[i] << 1; |
| 1480 | 1491 | ring->vm_hub = AMDGPU_MMHUB0(aid_id); |
| 1492 | + ring->no_user_submission = adev->sdma.no_user_submission; | |
| 1481 | 1493 | |
| 1482 | 1494 | sprintf(ring->name, "sdma%d.%d", aid_id, |
| 1483 | 1495 | i % adev->sdma.num_inst_per_aid); |
| @@ -2404,187 +2416,9 @@ struct amdgpu_xcp_ip_funcs sdma_v4_4_2_xcp_funcs = { · 920 unmodified lines | ||
| 2404 | 2416 | .resume = &sdma_v4_4_2_xcp_resume |
| 2405 | 2417 | }; |
| 2406 | 2418 | |
| 2407 | −static const struct amdgpu_ras_err_status_reg_entry sdma_v4_2_2_ue_reg_list[] = { | |
| 2408 | − {AMDGPU_RAS_REG_ENTRY(SDMA0, 0, regSDMA_UE_ERR_STATUS_LO, regSDMA_UE_ERR_STATUS_HI), | |
| 2409 | − 1, (AMDGPU_RAS_ERR_INFO_VALID | AMDGPU_RAS_ERR_STATUS_VALID), "SDMA"}, | |
| 2410 | −}; | |
| 2411 | − | |
| 2412 | −static const struct amdgpu_ras_memory_id_entry sdma_v4_4_2_ras_memory_list[] = { | |
| 2413 | − {AMDGPU_SDMA_MBANK_DATA_BUF0, "SDMA_MBANK_DATA_BUF0"}, | |
| 2414 | − {AMDGPU_SDMA_MBANK_DATA_BUF1, "SDMA_MBANK_DATA_BUF1"}, | |
| 2415 | − {AMDGPU_SDMA_MBANK_DATA_BUF2, "SDMA_MBANK_DATA_BUF2"}, | |
| 2416 | − {AMDGPU_SDMA_MBANK_DATA_BUF3, "SDMA_MBANK_DATA_BUF3"}, | |
| 2417 | − {AMDGPU_SDMA_MBANK_DATA_BUF4, "SDMA_MBANK_DATA_BUF4"}, | |
| 2418 | − {AMDGPU_SDMA_MBANK_DATA_BUF5, "SDMA_MBANK_DATA_BUF5"}, | |
| 2419 | − {AMDGPU_SDMA_MBANK_DATA_BUF6, "SDMA_MBANK_DATA_BUF6"}, | |
| 2420 | − {AMDGPU_SDMA_MBANK_DATA_BUF7, "SDMA_MBANK_DATA_BUF7"}, | |
| 2421 | − {AMDGPU_SDMA_MBANK_DATA_BUF8, "SDMA_MBANK_DATA_BUF8"}, | |
| 2422 | − {AMDGPU_SDMA_MBANK_DATA_BUF9, "SDMA_MBANK_DATA_BUF9"}, | |
| 2423 | − {AMDGPU_SDMA_MBANK_DATA_BUF10, "SDMA_MBANK_DATA_BUF10"}, | |
| 2424 | − {AMDGPU_SDMA_MBANK_DATA_BUF11, "SDMA_MBANK_DATA_BUF11"}, | |
| 2425 | − {AMDGPU_SDMA_MBANK_DATA_BUF12, "SDMA_MBANK_DATA_BUF12"}, | |
| 2426 | − {AMDGPU_SDMA_MBANK_DATA_BUF13, "SDMA_MBANK_DATA_BUF13"}, | |
| 2427 | − {AMDGPU_SDMA_MBANK_DATA_BUF14, "SDMA_MBANK_DATA_BUF14"}, | |
| 2428 | − {AMDGPU_SDMA_MBANK_DATA_BUF15, "SDMA_MBANK_DATA_BUF15"}, | |
| 2429 | − {AMDGPU_SDMA_UCODE_BUF, "SDMA_UCODE_BUF"}, | |
| 2430 | − {AMDGPU_SDMA_RB_CMD_BUF, "SDMA_RB_CMD_BUF"}, | |
| 2431 | − {AMDGPU_SDMA_IB_CMD_BUF, "SDMA_IB_CMD_BUF"}, | |
| 2432 | − {AMDGPU_SDMA_UTCL1_RD_FIFO, "SDMA_UTCL1_RD_FIFO"}, | |
| 2433 | − {AMDGPU_SDMA_UTCL1_RDBST_FIFO, "SDMA_UTCL1_RDBST_FIFO"}, | |
| 2434 | − {AMDGPU_SDMA_UTCL1_WR_FIFO, "SDMA_UTCL1_WR_FIFO"}, | |
| 2435 | − {AMDGPU_SDMA_DATA_LUT_FIFO, "SDMA_DATA_LUT_FIFO"}, | |
| 2436 | − {AMDGPU_SDMA_SPLIT_DAT_BUF, "SDMA_SPLIT_DAT_BUF"}, | |
| 2437 | −}; | |
| 2438 | − | |
| 2439 | −static void sdma_v4_4_2_inst_query_ras_error_count(struct amdgpu_device *adev, | |
| 2440 | − uint32_t sdma_inst, | |
| 2441 | − void *ras_err_status) | |
| 2442 | −{ | |
| 2443 | − struct ras_err_data *err_data = (struct ras_err_data *)ras_err_status; | |
| 2444 | − uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst); | |
| 2445 | − unsigned long ue_count = 0; | |
| 2446 | − struct amdgpu_smuio_mcm_config_info mcm_info = { | |
| 2447 | − .socket_id = adev->smuio.funcs->get_socket_id(adev), | |
| 2448 | − .die_id = adev->sdma.instance[sdma_inst].aid_id, | |
| 2449 | − }; | |
| 2450 | − | |
| 2451 | − /* sdma v4_4_2 doesn't support query ce counts */ | |
| 2452 | − amdgpu_ras_inst_query_ras_error_count(adev, | |
| 2453 | − sdma_v4_2_2_ue_reg_list, | |
| 2454 | − ARRAY_SIZE(sdma_v4_2_2_ue_reg_list), | |
| 2455 | − sdma_v4_4_2_ras_memory_list, | |
| 2456 | − ARRAY_SIZE(sdma_v4_4_2_ras_memory_list), | |
| 2457 | − sdma_dev_inst, | |
| 2458 | − AMDGPU_RAS_ERROR__MULTI_UNCORRECTABLE, | |
| 2459 | − &ue_count); | |
| 2460 | − | |
| 2461 | − amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count); | |
| 2462 | −} | |
| 2463 | − | |
| 2464 | −static void sdma_v4_4_2_query_ras_error_count(struct amdgpu_device *adev, | |
| 2465 | − void *ras_err_status) | |
| 2466 | −{ | |
| 2467 | − uint32_t inst_mask; | |
| 2468 | − int i = 0; | |
| 2469 | − | |
| 2470 | − inst_mask = GENMASK(adev->sdma.num_instances - 1, 0); | |
| 2471 | − if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) { | |
| 2472 | − for_each_inst(i, inst_mask) | |
| 2473 | − sdma_v4_4_2_inst_query_ras_error_count(adev, i, ras_err_status); | |
| 2474 | − } else { | |
| 2475 | − dev_warn(adev->dev, "SDMA RAS is not supported\n"); | |
| 2476 | − } | |
| 2477 | −} | |
| 2478 | − | |
| 2479 | −static void sdma_v4_4_2_inst_reset_ras_error_count(struct amdgpu_device *adev, | |
| 2480 | − uint32_t sdma_inst) | |
| 2481 | −{ | |
| 2482 | − uint32_t sdma_dev_inst = GET_INST(SDMA0, sdma_inst); | |
| 2483 | − | |
| 2484 | − amdgpu_ras_inst_reset_ras_error_count(adev, | |
| 2485 | − sdma_v4_2_2_ue_reg_list, | |
| 2486 | − ARRAY_SIZE(sdma_v4_2_2_ue_reg_list), | |
| 2487 | − sdma_dev_inst); | |
| 2488 | −} | |
| 2489 | − | |
| 2490 | −static void sdma_v4_4_2_reset_ras_error_count(struct amdgpu_device *adev) | |
| 2491 | −{ | |
| 2492 | − uint32_t inst_mask; | |
| 2493 | − int i = 0; | |
| 2494 | − | |
| 2495 | − inst_mask = GENMASK(adev->sdma.num_instances - 1, 0); | |
| 2496 | − if (amdgpu_ras_is_supported(adev, AMDGPU_RAS_BLOCK__SDMA)) { | |
| 2497 | − for_each_inst(i, inst_mask) | |
| 2498 | − sdma_v4_4_2_inst_reset_ras_error_count(adev, i); | |
| 2499 | − } else { | |
| 2500 | − dev_warn(adev->dev, "SDMA RAS is not supported\n"); | |
| 2501 | − } | |
| 2502 | −} | |
| 2503 | − | |
| 2504 | −static const struct amdgpu_ras_block_hw_ops sdma_v4_4_2_ras_hw_ops = { | |
| 2505 | − .query_ras_error_count = sdma_v4_4_2_query_ras_error_count, | |
| 2506 | − .reset_ras_error_count = sdma_v4_4_2_reset_ras_error_count, | |
| 2507 | −}; | |
| 2508 | − | |
| 2509 | −static int sdma_v4_4_2_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, | |
| 2510 | − enum aca_smu_type type, void *data) | |
| 2511 | −{ | |
| 2512 | − struct aca_bank_info info; | |
| 2513 | − u64 misc0; | |
| 2514 | − int ret; | |
| 2515 | − | |
| 2516 | − ret = aca_bank_info_decode(bank, &info); | |
| 2517 | − if (ret) | |
| 2518 | − return ret; | |
| 2519 | − | |
| 2520 | − misc0 = bank->regs[ACA_REG_IDX_MISC0]; | |
| 2521 | − switch (type) { | |
| 2522 | − case ACA_SMU_TYPE_UE: | |
| 2523 | − bank->aca_err_type = ACA_ERROR_TYPE_UE; | |
| 2524 | − ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, | |
| 2525 | − 1ULL); | |
| 2526 | − break; | |
| 2527 | − case ACA_SMU_TYPE_CE: | |
| 2528 | − bank->aca_err_type = ACA_ERROR_TYPE_CE; | |
| 2529 | − ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, | |
| 2530 | − ACA_REG__MISC0__ERRCNT(misc0)); | |
| 2531 | − break; | |
| 2532 | − default: | |
| 2533 | − return -EINVAL; | |
| 2534 | − } | |
| 2535 | − | |
| 2536 | − return ret; | |
| 2537 | −} | |
| 2538 | − | |
| 2539 | −/* CODE_SDMA0 - CODE_SDMA4, reference to smu driver if header file */ | |
| 2540 | −static int sdma_v4_4_2_err_codes[] = { 33, 34, 35, 36 }; | |
| 2541 | − | |
| 2542 | −static bool sdma_v4_4_2_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, | |
| 2543 | − enum aca_smu_type type, void *data) | |
| 2544 | −{ | |
| 2545 | − u32 instlo; | |
| 2546 | − | |
| 2547 | − instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); | |
| 2548 | − instlo &= GENMASK(31, 1); | |
| 2549 | − | |
| 2550 | − if (instlo != mmSMNAID_AID0_MCA_SMU) | |
| 2551 | − return false; | |
| 2552 | − | |
| 2553 | − if (aca_bank_check_error_codes(handle->adev, bank, | |
| 2554 | − sdma_v4_4_2_err_codes, | |
| 2555 | − ARRAY_SIZE(sdma_v4_4_2_err_codes))) | |
| 2556 | − return false; | |
| 2557 | − | |
| 2558 | − return true; | |
| 2559 | −} | |
| 2560 | − | |
| 2561 | −static const struct aca_bank_ops sdma_v4_4_2_aca_bank_ops = { | |
| 2562 | − .aca_bank_parser = sdma_v4_4_2_aca_bank_parser, | |
| 2563 | − .aca_bank_is_valid = sdma_v4_4_2_aca_bank_is_valid, | |
| 2564 | −}; | |
| 2565 | − | |
| 2566 | −static const struct aca_info sdma_v4_4_2_aca_info = { | |
| 2567 | − .hwip = ACA_HWIP_TYPE_SMU, | |
| 2568 | − .mask = ACA_ERROR_UE_MASK, | |
| 2569 | − .bank_ops = &sdma_v4_4_2_aca_bank_ops, | |
| 2570 | −}; | |
| 2571 | − | |
| 2572 | −static int sdma_v4_4_2_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) | |
| 2573 | −{ | |
| 2574 | − int r; | |
| 2575 | − | |
| 2576 | − r = amdgpu_sdma_ras_late_init(adev, ras_block); | |
| 2577 | − if (r) | |
| 2578 | − return r; | |
| 2579 | − | |
| 2580 | − return amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__SDMA, | |
| 2581 | − &sdma_v4_4_2_aca_info, NULL); | |
| 2582 | −} | |
| 2583 | − | |
| 2584 | 2419 | static struct amdgpu_sdma_ras sdma_v4_4_2_ras = { |
| 2585 | 2420 | .ras_block = { |
| 2586 | − .hw_ops = &sdma_v4_4_2_ras_hw_ops, | |
| 2587 | − .ras_late_init = sdma_v4_4_2_ras_late_init, | |
| 2421 | + .hw_ops = NULL, | |
| 2588 | 2422 | }, |
| 2589 | 2423 | }; |
| 2590 | 2424 | |
| @@ -793,23 +793,6 @@ static int sdma_v6_0_soft_reset(struct amdgpu_ip_block *ip_block) · 792 unmodified lines | ||
| 793 | 793 | return sdma_v6_0_start(adev); |
| 794 | 794 | } |
| 795 | 795 | |
| 796 | −static bool sdma_v6_0_check_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 797 | −{ | |
| 798 | − struct amdgpu_device *adev = ip_block->adev; | |
| 799 | − struct amdgpu_ring *ring; | |
| 800 | − int i, r; | |
| 801 | − long tmo = msecs_to_jiffies(1000); | |
| 802 | − | |
| 803 | − for (i = 0; i < adev->sdma.num_instances; i++) { | |
| 804 | − ring = &adev->sdma.instance[i].ring; | |
| 805 | − r = amdgpu_ring_test_ib(ring, tmo); | |
| 806 | − if (r) | |
| 807 | − return true; | |
| 808 | − } | |
| 809 | − | |
| 810 | − return false; | |
| 811 | −} | |
| 812 | − | |
| 813 | 796 | /** |
| 814 | 797 | * sdma_v6_0_start - setup and start the async dma engines |
| 815 | 798 | * |
| @@ -1747,7 +1730,6 @@ const struct amd_ip_funcs sdma_v6_0_ip_funcs = { · 931 unmodified lines | ||
| 1747 | 1730 | .is_idle = sdma_v6_0_is_idle, |
| 1748 | 1731 | .wait_for_idle = sdma_v6_0_wait_for_idle, |
| 1749 | 1732 | .soft_reset = sdma_v6_0_soft_reset, |
| 1750 | − .check_soft_reset = sdma_v6_0_check_soft_reset, | |
| 1751 | 1733 | .set_clockgating_state = sdma_v6_0_set_clockgating_state, |
| 1752 | 1734 | .set_powergating_state = sdma_v6_0_set_powergating_state, |
| 1753 | 1735 | .get_clockgating_state = sdma_v6_0_get_clockgating_state, |
| @@ -784,23 +784,6 @@ static int sdma_v7_0_soft_reset(struct amdgpu_ip_block *ip_block) · 783 unmodified lines | ||
| 784 | 784 | return sdma_v7_0_start(adev); |
| 785 | 785 | } |
| 786 | 786 | |
| 787 | −static bool sdma_v7_0_check_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 788 | −{ | |
| 789 | − struct amdgpu_device *adev = ip_block->adev; | |
| 790 | − struct amdgpu_ring *ring; | |
| 791 | − int i, r; | |
| 792 | − long tmo = msecs_to_jiffies(1000); | |
| 793 | − | |
| 794 | − for (i = 0; i < adev->sdma.num_instances; i++) { | |
| 795 | − ring = &adev->sdma.instance[i].ring; | |
| 796 | − r = amdgpu_ring_test_ib(ring, tmo); | |
| 797 | − if (r) | |
| 798 | − return true; | |
| 799 | − } | |
| 800 | − | |
| 801 | − return false; | |
| 802 | −} | |
| 803 | − | |
| 804 | 787 | static int sdma_v7_0_reset_queue(struct amdgpu_ring *ring, |
| 805 | 788 | unsigned int vmid, |
| 806 | 789 | struct amdgpu_fence *timedout_fence) |
| @@ -1679,7 +1662,6 @@ const struct amd_ip_funcs sdma_v7_0_ip_funcs = { · 872 unmodified lines | ||
| 1679 | 1662 | .is_idle = sdma_v7_0_is_idle, |
| 1680 | 1663 | .wait_for_idle = sdma_v7_0_wait_for_idle, |
| 1681 | 1664 | .soft_reset = sdma_v7_0_soft_reset, |
| 1682 | − .check_soft_reset = sdma_v7_0_check_soft_reset, | |
| 1683 | 1665 | .set_clockgating_state = sdma_v7_0_set_clockgating_state, |
| 1684 | 1666 | .set_powergating_state = sdma_v7_0_set_powergating_state, |
| 1685 | 1667 | .get_clockgating_state = sdma_v7_0_get_clockgating_state, |
| @@ -775,23 +775,6 @@ static int sdma_v7_1_soft_reset(struct amdgpu_ip_block *ip_block) · 774 unmodified lines | ||
| 775 | 775 | return sdma_v7_1_inst_start(adev, inst_mask); |
| 776 | 776 | } |
| 777 | 777 | |
| 778 | −static bool sdma_v7_1_check_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 779 | −{ | |
| 780 | − struct amdgpu_device *adev = ip_block->adev; | |
| 781 | − struct amdgpu_ring *ring; | |
| 782 | − int i, r; | |
| 783 | − long tmo = msecs_to_jiffies(1000); | |
| 784 | − | |
| 785 | − for (i = 0; i < adev->sdma.num_instances; i++) { | |
| 786 | − ring = &adev->sdma.instance[i].ring; | |
| 787 | − r = amdgpu_ring_test_ib(ring, tmo); | |
| 788 | − if (r) | |
| 789 | − return true; | |
| 790 | − } | |
| 791 | − | |
| 792 | − return false; | |
| 793 | −} | |
| 794 | − | |
| 795 | 778 | static int sdma_v7_1_reset_queue(struct amdgpu_ring *ring, |
| 796 | 779 | unsigned int vmid, |
| 797 | 780 | struct amdgpu_fence *timedout_fence) |
| @@ -1644,7 +1627,6 @@ const struct amd_ip_funcs sdma_v7_1_ip_funcs = { · 846 unmodified lines | ||
| 1644 | 1627 | .is_idle = sdma_v7_1_is_idle, |
| 1645 | 1628 | .wait_for_idle = sdma_v7_1_wait_for_idle, |
| 1646 | 1629 | .soft_reset = sdma_v7_1_soft_reset, |
| 1647 | − .check_soft_reset = sdma_v7_1_check_soft_reset, | |
| 1648 | 1630 | .set_clockgating_state = sdma_v7_1_set_clockgating_state, |
| 1649 | 1631 | .set_powergating_state = sdma_v7_1_set_powergating_state, |
| 1650 | 1632 | .get_clockgating_state = sdma_v7_1_get_clockgating_state, |
| @@ -1509,12 +1509,6 @@ static void si_invalidate_hdp(struct amdgpu_device *adev, · 1508 unmodified lines | ||
| 1509 | 1509 | } |
| 1510 | 1510 | } |
| 1511 | 1511 | |
| 1512 | −static bool si_need_full_reset(struct amdgpu_device *adev) | |
| 1513 | −{ | |
| 1514 | − /* change this when we support soft reset */ | |
| 1515 | − return true; | |
| 1516 | −} | |
| 1517 | − | |
| 1518 | 1512 | static bool si_need_reset_on_init(struct amdgpu_device *adev) |
| 1519 | 1513 | { |
| 1520 | 1514 | return false; |
| @@ -2019,7 +2013,6 @@ static const struct amdgpu_asic_funcs si_asic_funcs = · 498 unmodified lines | ||
| 2019 | 2013 | .get_config_memsize = &si_get_config_memsize, |
| 2020 | 2014 | .flush_hdp = &si_flush_hdp, |
| 2021 | 2015 | .invalidate_hdp = &si_invalidate_hdp, |
| 2022 | − .need_full_reset = &si_need_full_reset, | |
| 2023 | 2016 | .get_pcie_usage = &si_get_pcie_usage, |
| 2024 | 2017 | .need_reset_on_init = &si_need_reset_on_init, |
| 2025 | 2018 | .get_pcie_replay_count = &si_get_pcie_replay_count, |
| @@ -721,12 +721,6 @@ void soc15_set_virt_ops(struct amdgpu_device *adev) · 720 unmodified lines | ||
| 721 | 721 | soc15_reg_base_init(adev); |
| 722 | 722 | } |
| 723 | 723 | |
| 724 | −static bool soc15_need_full_reset(struct amdgpu_device *adev) | |
| 725 | −{ | |
| 726 | − /* change this when we implement soft reset */ | |
| 727 | − return true; | |
| 728 | −} | |
| 729 | − | |
| 730 | 724 | static void soc15_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0, |
| 731 | 725 | uint64_t *count1) |
| 732 | 726 | { |
| @@ -878,7 +872,6 @@ static const struct amdgpu_asic_funcs soc15_asic_funcs = · 145 unmodified lines | ||
| 878 | 872 | .set_uvd_clocks = &soc15_set_uvd_clocks, |
| 879 | 873 | .set_vce_clocks = &soc15_set_vce_clocks, |
| 880 | 874 | .get_config_memsize = &soc15_get_config_memsize, |
| 881 | − .need_full_reset = &soc15_need_full_reset, | |
| 882 | 875 | .init_doorbell_index = &vega10_doorbell_index_init, |
| 883 | 876 | .get_pcie_usage = &soc15_get_pcie_usage, |
| 884 | 877 | .need_reset_on_init = &soc15_need_reset_on_init, |
| @@ -899,7 +892,6 @@ static const struct amdgpu_asic_funcs vega20_asic_funcs = · 14 unmodified lines | ||
| 899 | 892 | .set_uvd_clocks = &soc15_set_uvd_clocks, |
| 900 | 893 | .set_vce_clocks = &soc15_set_vce_clocks, |
| 901 | 894 | .get_config_memsize = &soc15_get_config_memsize, |
| 902 | − .need_full_reset = &soc15_need_full_reset, | |
| 903 | 895 | .init_doorbell_index = &vega20_doorbell_index_init, |
| 904 | 896 | .get_pcie_usage = &vega20_get_pcie_usage, |
| 905 | 897 | .need_reset_on_init = &soc15_need_reset_on_init, |
| @@ -920,7 +912,6 @@ static const struct amdgpu_asic_funcs aqua_vanjaram_asic_funcs = · 14 unmodified lines | ||
| 920 | 912 | .set_uvd_clocks = &soc15_set_uvd_clocks, |
| 921 | 913 | .set_vce_clocks = &soc15_set_vce_clocks, |
| 922 | 914 | .get_config_memsize = &soc15_get_config_memsize, |
| 923 | − .need_full_reset = &soc15_need_full_reset, | |
| 924 | 915 | .init_doorbell_index = &aqua_vanjaram_doorbell_index_init, |
| 925 | 916 | .need_reset_on_init = &soc15_need_reset_on_init, |
| 926 | 917 | .get_pcie_replay_count = &amdgpu_nbio_get_pcie_replay_count, |
| @@ -38,30 +38,30 @@ · 37 unmodified lines | ||
| 38 | 38 | (adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + (reg)+(offset)) |
| 39 | 39 | |
| 40 | 40 | #define __WREG32_SOC15_RLC__(reg, value, flag, hwip, inst) \ |
| 41 | − ((amdgpu_sriov_vf(adev) && adev->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported) ? \ | |
| 42 | − amdgpu_sriov_wreg(adev, reg, value, flag, hwip, inst) : \ | |
| 43 | − WREG32(reg, value)) | |
| 41 | + adev->gfx.rlc.reg_funcs->wreg32(adev, reg, value, flag, hwip, inst) | |
| 44 | 42 | |
| 45 | 43 | #define __RREG32_SOC15_RLC__(reg, flag, hwip, inst) \ |
| 46 | − ((amdgpu_sriov_vf(adev) && adev->gfx.rlc.funcs && adev->gfx.rlc.rlcg_reg_access_supported) ? \ | |
| 47 | − amdgpu_sriov_rreg(adev, reg, flag, hwip, inst) : \ | |
| 48 | − RREG32(reg)) | |
| 49 | − | |
| 50 | −#define WREG32_FIELD15(ip, idx, reg, field, val) \ | |
| 51 | − __WREG32_SOC15_RLC__(adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg, \ | |
| 52 | − (__RREG32_SOC15_RLC__( \ | |
| 53 | − adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg, \ | |
| 54 | − 0, ip##_HWIP, idx) & \ | |
| 55 | − ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field), \ | |
| 56 | − 0, ip##_HWIP, idx) | |
| 57 | − | |
| 58 | −#define WREG32_FIELD15_PREREG(ip, idx, reg_name, field, val) \ | |
| 59 | − __WREG32_SOC15_RLC__(adev->reg_offset[ip##_HWIP][idx][reg##reg_name##_BASE_IDX] + reg##reg_name, \ | |
| 60 | − (__RREG32_SOC15_RLC__( \ | |
| 61 | − adev->reg_offset[ip##_HWIP][idx][reg##reg_name##_BASE_IDX] + reg##reg_name, \ | |
| 62 | − 0, ip##_HWIP, idx) & \ | |
| 63 | − ~REG_FIELD_MASK(reg_name, field)) | (val) << REG_FIELD_SHIFT(reg_name, field), \ | |
| 64 | − 0, ip##_HWIP, idx) | |
| 44 | + adev->gfx.rlc.reg_funcs->rreg32(adev, reg, flag, hwip, inst) | |
| 45 | + | |
| 46 | +#define WREG32_FIELD15(ip, idx, reg_name, field, val) \ | |
| 47 | +do { \ | |
| 48 | + u32 reg__ = adev->reg_offset[ip##_HWIP][idx][mm##reg_name##_BASE_IDX] + mm##reg_name; \ | |
| 49 | + u32 val__ = __RREG32_SOC15_RLC__(reg__, 0, ip##_HWIP, idx); \ | |
| 50 | +\ | |
| 51 | + val__ &= ~REG_FIELD_MASK(reg_name, field); \ | |
| 52 | + val__ |= (val) << REG_FIELD_SHIFT(reg_name, field); \ | |
| 53 | + __WREG32_SOC15_RLC__(reg__, val__, 0, ip##_HWIP, idx); \ | |
| 54 | +} while (0) | |
| 55 | + | |
| 56 | +#define WREG32_FIELD15_PREREG(ip, idx, reg_name, field, val) \ | |
| 57 | +do { \ | |
| 58 | + u32 reg__ = adev->reg_offset[ip##_HWIP][idx][reg##reg_name##_BASE_IDX] + reg##reg_name; \ | |
| 59 | + u32 val__ = __RREG32_SOC15_RLC__(reg__, 0, ip##_HWIP, idx); \ | |
| 60 | +\ | |
| 61 | + val__ &= ~REG_FIELD_MASK(reg_name, field); \ | |
| 62 | + val__ |= (val) << REG_FIELD_SHIFT(reg_name, field); \ | |
| 63 | + __WREG32_SOC15_RLC__(reg__, val__, 0, ip##_HWIP, idx); \ | |
| 64 | +} while (0) | |
| 65 | 65 | |
| 66 | 66 | #define RREG32_SOC15(ip, inst, reg) \ |
| 67 | 67 | __RREG32_SOC15_RLC__(adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg, \ |
| @@ -181,12 +181,15 @@ · 113 unmodified lines | ||
| 181 | 181 | WREG32_RLC_EX(prefix, target_reg, value, inst); \ |
| 182 | 182 | } while (0) |
| 183 | 183 | |
| 184 | −#define WREG32_FIELD15_RLC(ip, idx, reg, field, val) \ | |
| 185 | − __WREG32_SOC15_RLC__((adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg), \ | |
| 186 | − (__RREG32_SOC15_RLC__(adev->reg_offset[ip##_HWIP][idx][mm##reg##_BASE_IDX] + mm##reg, \ | |
| 187 | − AMDGPU_REGS_RLC, ip##_HWIP, idx) & \ | |
| 188 | − ~REG_FIELD_MASK(reg, field)) | (val) << REG_FIELD_SHIFT(reg, field), \ | |
| 189 | − AMDGPU_REGS_RLC, ip##_HWIP, idx) | |
| 184 | +#define WREG32_FIELD15_RLC(ip, idx, reg_name, field, val) \ | |
| 185 | +do { \ | |
| 186 | + u32 reg__ = adev->reg_offset[ip##_HWIP][idx][mm##reg_name##_BASE_IDX] + mm##reg_name; \ | |
| 187 | + u32 val__ = __RREG32_SOC15_RLC__(reg__, AMDGPU_REGS_RLC, ip##_HWIP, idx); \ | |
| 188 | +\ | |
| 189 | + val__ &= ~REG_FIELD_MASK(reg_name, field); \ | |
| 190 | + val__ |= (val) << REG_FIELD_SHIFT(reg_name, field); \ | |
| 191 | + __WREG32_SOC15_RLC__(reg__, val__, AMDGPU_REGS_RLC, ip##_HWIP, idx); \ | |
| 192 | +} while (0) | |
| 190 | 193 | |
| 191 | 194 | #define WREG32_SOC15_OFFSET_RLC(ip, inst, reg, offset, value) \ |
| 192 | 195 | __WREG32_SOC15_RLC__((adev->reg_offset[ip##_HWIP][inst][reg##_BASE_IDX] + reg) + offset, value, AMDGPU_REGS_RLC, ip##_HWIP, inst) |
| @@ -207,10 +210,4 @@ · 14 unmodified lines | ||
| 207 | 210 | amdgpu_reg_get_smn_base64(adev, ip##_HWIP, inst), \ |
| 208 | 211 | value) |
| 209 | 212 | |
| 210 | −#define RREG64_MCA(smn_base, mca_base, idx) \ | |
| 211 | − RREG64_PCIE_EXT(smn_base + mca_base + (idx * 8)) | |
| 212 | − | |
| 213 | −#define WREG64_MCA(smn_base, mca_base, idx, val) \ | |
| 214 | − WREG64_PCIE_EXT(smn_base + mca_base + (idx * 8), val) | |
| 215 | − | |
| 216 | 213 | #endif |
| @@ -461,17 +461,6 @@ const struct amdgpu_ip_block_version soc21_common_ip_block = { · 460 unmodified lines | ||
| 461 | 461 | .funcs = &soc21_common_ip_funcs, |
| 462 | 462 | }; |
| 463 | 463 | |
| 464 | −static bool soc21_need_full_reset(struct amdgpu_device *adev) | |
| 465 | −{ | |
| 466 | − switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { | |
| 467 | − case IP_VERSION(11, 0, 0): | |
| 468 | − case IP_VERSION(11, 0, 2): | |
| 469 | − case IP_VERSION(11, 0, 3): | |
| 470 | − default: | |
| 471 | − return true; | |
| 472 | − } | |
| 473 | −} | |
| 474 | − | |
| 475 | 464 | static bool soc21_need_reset_on_init(struct amdgpu_device *adev) |
| 476 | 465 | { |
| 477 | 466 | u32 sol_reg; |
| @@ -550,7 +539,6 @@ static const struct amdgpu_asic_funcs soc21_asic_funcs = { · 72 unmodified lines | ||
| 550 | 539 | .set_vce_clocks = &soc21_set_vce_clocks, |
| 551 | 540 | .get_config_memsize = &soc21_get_config_memsize, |
| 552 | 541 | .init_doorbell_index = &soc21_init_doorbell_index, |
| 553 | − .need_full_reset = &soc21_need_full_reset, | |
| 554 | 542 | .need_reset_on_init = &soc21_need_reset_on_init, |
| 555 | 543 | .get_pcie_replay_count = &amdgpu_nbio_get_pcie_replay_count, |
| 556 | 544 | .supports_baco = &amdgpu_dpm_is_baco_supported, |
| @@ -238,16 +238,6 @@ const struct amdgpu_ip_block_version soc24_common_ip_block = { · 237 unmodified lines | ||
| 238 | 238 | .funcs = &soc24_common_ip_funcs, |
| 239 | 239 | }; |
| 240 | 240 | |
| 241 | −static bool soc24_need_full_reset(struct amdgpu_device *adev) | |
| 242 | −{ | |
| 243 | − switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { | |
| 244 | − case IP_VERSION(12, 0, 0): | |
| 245 | − case IP_VERSION(12, 0, 1): | |
| 246 | − default: | |
| 247 | − return true; | |
| 248 | − } | |
| 249 | −} | |
| 250 | − | |
| 251 | 241 | static bool soc24_need_reset_on_init(struct amdgpu_device *adev) |
| 252 | 242 | { |
| 253 | 243 | u32 sol_reg; |
| @@ -330,7 +320,6 @@ static const struct amdgpu_asic_funcs soc24_asic_funcs = { · 76 unmodified lines | ||
| 330 | 320 | .get_xclk = &soc24_get_xclk, |
| 331 | 321 | .get_config_memsize = &soc24_get_config_memsize, |
| 332 | 322 | .init_doorbell_index = &soc24_init_doorbell_index, |
| 333 | − .need_full_reset = &soc24_need_full_reset, | |
| 334 | 323 | .need_reset_on_init = &soc24_need_reset_on_init, |
| 335 | 324 | .get_pcie_replay_count = &soc24_get_pcie_replay_count, |
| 336 | 325 | .supports_baco = &amdgpu_dpm_is_baco_supported, |
| @@ -223,15 +223,6 @@ static int soc_v1_0_read_register(struct amdgpu_device *adev, · 222 unmodified lines | ||
| 223 | 223 | return -EINVAL; |
| 224 | 224 | } |
| 225 | 225 | |
| 226 | −static bool soc_v1_0_need_full_reset(struct amdgpu_device *adev) | |
| 227 | −{ | |
| 228 | − switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { | |
| 229 | − case IP_VERSION(12, 1, 0): | |
| 230 | − default: | |
| 231 | − return true; | |
| 232 | − } | |
| 233 | −} | |
| 234 | − | |
| 235 | 226 | static bool soc_v1_0_need_reset_on_init(struct amdgpu_device *adev) |
| 236 | 227 | { |
| 237 | 228 | |
| @@ -271,7 +262,6 @@ static const struct amdgpu_asic_funcs soc_v1_0_asic_funcs = { · 33 unmodified lines | ||
| 271 | 262 | .read_register = &soc_v1_0_read_register, |
| 272 | 263 | .get_config_memsize = &soc_v1_0_get_config_memsize, |
| 273 | 264 | .get_xclk = &soc_v1_0_get_xclk, |
| 274 | − .need_full_reset = &soc_v1_0_need_full_reset, | |
| 275 | 265 | .init_doorbell_index = &soc_v1_0_doorbell_index_init, |
| 276 | 266 | .need_reset_on_init = &soc_v1_0_need_reset_on_init, |
| 277 | 267 | .encode_ext_smn_addressing = &soc_v1_0_encode_ext_smn_addressing, |
| @@ -600,8 +590,10 @@ static int soc_v1_0_get_xcp_res_info(struct amdgpu_xcp_mgr *xcp_mgr, · 322 unmodified lines | ||
| 600 | 590 | xcp_cfg->num_res = ARRAY_SIZE(max_res); |
| 601 | 591 | |
| 602 | 592 | for (i = 0; i < xcp_cfg->num_res; i++) { |
| 603 | − res_lt_xcp = max_res[i] < num_xcp; | |
| 604 | 593 | xcp_cfg->xcp_res[i].id = i; |
| 594 | + if (!max_res[i]) | |
| 595 | + continue; | |
| 596 | + res_lt_xcp = max_res[i] < num_xcp; | |
| 605 | 597 | xcp_cfg->xcp_res[i].num_inst = |
| 606 | 598 | res_lt_xcp ? 1 : max_res[i] / num_xcp; |
| 607 | 599 | xcp_cfg->xcp_res[i].num_inst = |
| @@ -390,43 +390,6 @@ static int tonga_ih_wait_for_idle(struct amdgpu_ip_block *ip_block) · 389 unmodified lines | ||
| 390 | 390 | return -ETIMEDOUT; |
| 391 | 391 | } |
| 392 | 392 | |
| 393 | −static bool tonga_ih_check_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 394 | −{ | |
| 395 | − struct amdgpu_device *adev = ip_block->adev; | |
| 396 | − u32 srbm_soft_reset = 0; | |
| 397 | − u32 tmp = RREG32(mmSRBM_STATUS); | |
| 398 | − | |
| 399 | − if (tmp & SRBM_STATUS__IH_BUSY_MASK) | |
| 400 | − srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, | |
| 401 | − SOFT_RESET_IH, 1); | |
| 402 | − | |
| 403 | − if (srbm_soft_reset) { | |
| 404 | − adev->irq.srbm_soft_reset = srbm_soft_reset; | |
| 405 | − return true; | |
| 406 | − } else { | |
| 407 | − adev->irq.srbm_soft_reset = 0; | |
| 408 | − return false; | |
| 409 | − } | |
| 410 | −} | |
| 411 | − | |
| 412 | −static int tonga_ih_pre_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 413 | −{ | |
| 414 | − if (!ip_block->adev->irq.srbm_soft_reset) | |
| 415 | − return 0; | |
| 416 | − | |
| 417 | − return tonga_ih_hw_fini(ip_block); | |
| 418 | −} | |
| 419 | − | |
| 420 | −static int tonga_ih_post_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 421 | −{ | |
| 422 | − struct amdgpu_device *adev = ip_block->adev; | |
| 423 | − | |
| 424 | − if (!adev->irq.srbm_soft_reset) | |
| 425 | − return 0; | |
| 426 | − | |
| 427 | − return tonga_ih_hw_init(ip_block); | |
| 428 | −} | |
| 429 | − | |
| 430 | 393 | static int tonga_ih_soft_reset(struct amdgpu_ip_block *ip_block) |
| 431 | 394 | { |
| 432 | 395 | struct amdgpu_device *adev = ip_block->adev; |
| @@ -481,10 +444,7 @@ static const struct amd_ip_funcs tonga_ih_ip_funcs = { · 48 unmodified lines | ||
| 481 | 444 | .resume = tonga_ih_resume, |
| 482 | 445 | .is_idle = tonga_ih_is_idle, |
| 483 | 446 | .wait_for_idle = tonga_ih_wait_for_idle, |
| 484 | − .check_soft_reset = tonga_ih_check_soft_reset, | |
| 485 | − .pre_soft_reset = tonga_ih_pre_soft_reset, | |
| 486 | 447 | .soft_reset = tonga_ih_soft_reset, |
| 487 | − .post_soft_reset = tonga_ih_post_soft_reset, | |
| 488 | 448 | .set_clockgating_state = tonga_ih_set_clockgating_state, |
| 489 | 449 | .set_powergating_state = tonga_ih_set_powergating_state, |
| 490 | 450 | }; |
| @@ -28,48 +28,6 @@ · 27 unmodified lines | ||
| 28 | 28 | #include "umc/umc_12_0_0_sh_mask.h" |
| 29 | 29 | #include "mp/mp_13_0_6_sh_mask.h" |
| 30 | 30 | |
| 31 | −#define MAX_ECC_NUM_PER_RETIREMENT 32 | |
| 32 | −#define DELAYED_TIME_FOR_GPU_RESET 1000 //ms | |
| 33 | − | |
| 34 | −static inline uint64_t get_umc_v12_0_reg_offset(struct amdgpu_device *adev, | |
| 35 | − uint32_t node_inst, | |
| 36 | − uint32_t umc_inst, | |
| 37 | − uint32_t ch_inst) | |
| 38 | −{ | |
| 39 | − uint32_t index = umc_inst * adev->umc.channel_inst_num + ch_inst; | |
| 40 | − uint64_t cross_node_offset = (node_inst == 0) ? 0 : UMC_V12_0_CROSS_NODE_OFFSET; | |
| 41 | − | |
| 42 | − umc_inst = index / 4; | |
| 43 | − ch_inst = index % 4; | |
| 44 | − | |
| 45 | − return adev->umc.channel_offs * ch_inst + UMC_V12_0_INST_DIST * umc_inst + | |
| 46 | − UMC_V12_0_NODE_DIST * node_inst + cross_node_offset; | |
| 47 | −} | |
| 48 | − | |
| 49 | −static int umc_v12_0_reset_error_count_per_channel(struct amdgpu_device *adev, | |
| 50 | − uint32_t node_inst, uint32_t umc_inst, | |
| 51 | − uint32_t ch_inst, void *data) | |
| 52 | −{ | |
| 53 | − uint64_t odecc_err_cnt_addr; | |
| 54 | − uint64_t umc_reg_offset = | |
| 55 | − get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst); | |
| 56 | − | |
| 57 | − odecc_err_cnt_addr = | |
| 58 | − SOC15_REG_OFFSET(UMC, 0, regUMCCH0_OdEccErrCnt); | |
| 59 | − | |
| 60 | − /* clear error count */ | |
| 61 | − WREG32_PCIE_EXT((odecc_err_cnt_addr + umc_reg_offset) * 4, | |
| 62 | − UMC_V12_0_CE_CNT_INIT); | |
| 63 | − | |
| 64 | − return 0; | |
| 65 | −} | |
| 66 | − | |
| 67 | −static void umc_v12_0_reset_error_count(struct amdgpu_device *adev) | |
| 68 | −{ | |
| 69 | − amdgpu_umc_loop_channels(adev, | |
| 70 | − umc_v12_0_reset_error_count_per_channel, NULL); | |
| 71 | −} | |
| 72 | − | |
| 73 | 31 | bool umc_v12_0_is_deferred_error(struct amdgpu_device *adev, uint64_t mc_umc_status) |
| 74 | 32 | { |
| 75 | 33 | dev_dbg(adev->dev, |
| @@ -115,65 +73,6 @@ bool umc_v12_0_is_correctable_error(struct amdgpu_device *adev, uint64_t mc_umc_ · 39 unmodified lines | ||
| 115 | 73 | !(umc_v12_0_is_uncorrectable_error(adev, mc_umc_status))))); |
| 116 | 74 | } |
| 117 | 75 | |
| 118 | −static void umc_v12_0_query_error_count_per_type(struct amdgpu_device *adev, | |
| 119 | − uint64_t umc_reg_offset, | |
| 120 | − unsigned long *error_count, | |
| 121 | − check_error_type_func error_type_func) | |
| 122 | −{ | |
| 123 | − uint64_t mc_umc_status; | |
| 124 | − uint64_t mc_umc_status_addr; | |
| 125 | − | |
| 126 | − mc_umc_status_addr = | |
| 127 | − SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0); | |
| 128 | − | |
| 129 | − /* Check MCUMC_STATUS */ | |
| 130 | − mc_umc_status = | |
| 131 | − RREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4); | |
| 132 | − | |
| 133 | − if (error_type_func(adev, mc_umc_status)) | |
| 134 | − *error_count += 1; | |
| 135 | −} | |
| 136 | − | |
| 137 | −static int umc_v12_0_query_error_count(struct amdgpu_device *adev, | |
| 138 | − uint32_t node_inst, uint32_t umc_inst, | |
| 139 | − uint32_t ch_inst, void *data) | |
| 140 | −{ | |
| 141 | − struct ras_err_data *err_data = (struct ras_err_data *)data; | |
| 142 | − unsigned long ue_count = 0, ce_count = 0, de_count = 0; | |
| 143 | − | |
| 144 | − /* NOTE: node_inst is converted by adev->umc.active_mask and the range is [0-3], | |
| 145 | − * which can be used as die ID directly */ | |
| 146 | − struct amdgpu_smuio_mcm_config_info mcm_info = { | |
| 147 | − .socket_id = adev->smuio.funcs->get_socket_id(adev), | |
| 148 | − .die_id = node_inst, | |
| 149 | − }; | |
| 150 | − | |
| 151 | − uint64_t umc_reg_offset = | |
| 152 | − get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst); | |
| 153 | − | |
| 154 | − umc_v12_0_query_error_count_per_type(adev, umc_reg_offset, | |
| 155 | − &ce_count, umc_v12_0_is_correctable_error); | |
| 156 | − umc_v12_0_query_error_count_per_type(adev, umc_reg_offset, | |
| 157 | − &ue_count, umc_v12_0_is_uncorrectable_error); | |
| 158 | − umc_v12_0_query_error_count_per_type(adev, umc_reg_offset, | |
| 159 | − &de_count, umc_v12_0_is_deferred_error); | |
| 160 | − | |
| 161 | − amdgpu_ras_error_statistic_ue_count(err_data, &mcm_info, ue_count); | |
| 162 | − amdgpu_ras_error_statistic_ce_count(err_data, &mcm_info, ce_count); | |
| 163 | − amdgpu_ras_error_statistic_de_count(err_data, &mcm_info, de_count); | |
| 164 | − | |
| 165 | − return 0; | |
| 166 | −} | |
| 167 | − | |
| 168 | −static void umc_v12_0_query_ras_error_count(struct amdgpu_device *adev, | |
| 169 | − void *ras_error_status) | |
| 170 | −{ | |
| 171 | − amdgpu_umc_loop_channels(adev, | |
| 172 | − umc_v12_0_query_error_count, ras_error_status); | |
| 173 | − | |
| 174 | − umc_v12_0_reset_error_count(adev); | |
| 175 | −} | |
| 176 | − | |
| 177 | 76 | static void umc_v12_0_get_retire_flip_bits(struct amdgpu_device *adev) |
| 178 | 77 | { |
| 179 | 78 | enum amdgpu_memory_partition nps = AMDGPU_NPS1_PARTITION_MODE; |
| @@ -279,190 +178,6 @@ static void umc_v12_0_get_retire_flip_bits(struct amdgpu_device *adev) · 99 unmodified lines | ||
| 279 | 178 | adev->umc.retire_unit = 0x1 << flip_bits->bit_num; |
| 280 | 179 | } |
| 281 | 180 | |
| 282 | −static int umc_v12_0_convert_error_address(struct amdgpu_device *adev, | |
| 283 | − struct ras_err_data *err_data, | |
| 284 | − struct ta_ras_query_address_input *addr_in, | |
| 285 | − struct ta_ras_query_address_output *addr_out, | |
| 286 | − bool dump_addr) | |
| 287 | −{ | |
| 288 | − uint32_t row = 0, row_lower = 0, row_high = 0; | |
| 289 | − uint32_t col = 0, col_lower = 0, bank = 0; | |
| 290 | − uint32_t channel_index = 0, umc_inst = 0; | |
| 291 | − uint32_t i, bit_num, retire_unit, *flip_bits; | |
| 292 | − uint64_t soc_pa, column, err_addr; | |
| 293 | − struct ta_ras_query_address_output addr_out_tmp; | |
| 294 | − struct ta_ras_query_address_output *paddr_out; | |
| 295 | − int ret = 0; | |
| 296 | − | |
| 297 | − if (!addr_out) | |
| 298 | − paddr_out = &addr_out_tmp; | |
| 299 | − else | |
| 300 | − paddr_out = addr_out; | |
| 301 | − | |
| 302 | − err_addr = bank = 0; | |
| 303 | − if (addr_in) { | |
| 304 | − err_addr = addr_in->ma.err_addr; | |
| 305 | − addr_in->addr_type = TA_RAS_MCA_TO_PA; | |
| 306 | − ret = psp_ras_query_address(&adev->psp, addr_in, paddr_out); | |
| 307 | − if (ret) { | |
| 308 | − dev_warn(adev->dev, "Failed to query RAS physical address for 0x%llx", | |
| 309 | − err_addr); | |
| 310 | − | |
| 311 | − goto out; | |
| 312 | − } | |
| 313 | − | |
| 314 | − bank = paddr_out->pa.bank; | |
| 315 | − /* no need to care about umc inst if addr_in is NULL */ | |
| 316 | − umc_inst = addr_in->ma.umc_inst; | |
| 317 | − } | |
| 318 | − | |
| 319 | − flip_bits = adev->umc.flip_bits.flip_bits_in_pa; | |
| 320 | − bit_num = adev->umc.flip_bits.bit_num; | |
| 321 | − retire_unit = adev->umc.retire_unit; | |
| 322 | − | |
| 323 | − soc_pa = paddr_out->pa.pa; | |
| 324 | − channel_index = paddr_out->pa.channel_idx; | |
| 325 | − /* clear loop bits in soc physical address */ | |
| 326 | − for (i = 0; i < bit_num; i++) | |
| 327 | − soc_pa &= ~BIT_ULL(flip_bits[i]); | |
| 328 | − | |
| 329 | − paddr_out->pa.pa = soc_pa; | |
| 330 | − /* get column bit 0 and 1 in mca address */ | |
| 331 | − col_lower = (err_addr >> 1) & 0x3ULL; | |
| 332 | − /* extra row bit will be handled later */ | |
| 333 | − row_lower = (err_addr >> UMC_V12_0_MA_R0_BIT) & 0x1fffULL; | |
| 334 | − row_lower &= ~BIT_ULL(adev->umc.flip_bits.flip_row_bit); | |
| 335 | − | |
| 336 | − if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(9, 5, 0)) { | |
| 337 | − row_high = (soc_pa >> adev->umc.flip_bits.r13_in_pa) & 0x3ULL; | |
| 338 | − /* it's 2.25GB in each channel, from MCA address to PA | |
| 339 | − * [R14 R13] is converted if the two bits value are 0x3, | |
| 340 | − * get them from PA instead of MCA address. | |
| 341 | − */ | |
| 342 | − row_lower |= (row_high << 13); | |
| 343 | − } | |
| 344 | − | |
| 345 | − if (!err_data && !dump_addr) | |
| 346 | − goto out; | |
| 347 | − | |
| 348 | − /* loop for all possibilities of retired bits */ | |
| 349 | − for (column = 0; column < retire_unit; column++) { | |
| 350 | − soc_pa = paddr_out->pa.pa; | |
| 351 | − for (i = 0; i < bit_num; i++) | |
| 352 | − soc_pa |= (((column >> i) & 0x1ULL) << flip_bits[i]); | |
| 353 | − | |
| 354 | − col = ((column & 0x7) << 2) | col_lower; | |
| 355 | − /* handle extra row bit */ | |
| 356 | − if (bit_num == RETIRE_FLIP_BITS_NUM) | |
| 357 | − row = ((column >> 3) << adev->umc.flip_bits.flip_row_bit) | | |
| 358 | − row_lower; | |
| 359 | − | |
| 360 | − if (dump_addr) | |
| 361 | − dev_info(adev->dev, | |
| 362 | − "Error Address(PA):0x%-10llx Row:0x%-4x Col:0x%-2x Bank:0x%x Channel:0x%x\n", | |
| 363 | − soc_pa, row, col, bank, channel_index); | |
| 364 | − | |
| 365 | − if (err_data) | |
| 366 | − amdgpu_umc_fill_error_record(err_data, err_addr, | |
| 367 | − soc_pa, channel_index, umc_inst); | |
| 368 | − } | |
| 369 | − | |
| 370 | −out: | |
| 371 | − return ret; | |
| 372 | −} | |
| 373 | − | |
| 374 | −static int umc_v12_0_query_error_address(struct amdgpu_device *adev, | |
| 375 | − uint32_t node_inst, uint32_t umc_inst, | |
| 376 | − uint32_t ch_inst, void *data) | |
| 377 | −{ | |
| 378 | − struct ras_err_data *err_data = (struct ras_err_data *)data; | |
| 379 | − struct ta_ras_query_address_input addr_in; | |
| 380 | − uint64_t mc_umc_status_addr; | |
| 381 | − uint64_t mc_umc_status, err_addr; | |
| 382 | − uint64_t mc_umc_addrt0; | |
| 383 | − uint64_t umc_reg_offset = | |
| 384 | − get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst); | |
| 385 | − | |
| 386 | − mc_umc_status_addr = | |
| 387 | − SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_STATUST0); | |
| 388 | − | |
| 389 | − mc_umc_status = RREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4); | |
| 390 | − | |
| 391 | − if (mc_umc_status == 0) | |
| 392 | − return 0; | |
| 393 | − | |
| 394 | − if (!err_data->err_addr) { | |
| 395 | − /* clear umc status */ | |
| 396 | − WREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL); | |
| 397 | − | |
| 398 | − return 0; | |
| 399 | − } | |
| 400 | − | |
| 401 | − /* calculate error address if ue error is detected */ | |
| 402 | − if (umc_v12_0_is_uncorrectable_error(adev, mc_umc_status) || | |
| 403 | − umc_v12_0_is_deferred_error(adev, mc_umc_status)) { | |
| 404 | − mc_umc_addrt0 = | |
| 405 | − SOC15_REG_OFFSET(UMC, 0, regMCA_UMC_UMC0_MCUMC_ADDRT0); | |
| 406 | − | |
| 407 | − err_addr = RREG64_PCIE_EXT((mc_umc_addrt0 + umc_reg_offset) * 4); | |
| 408 | − | |
| 409 | − err_addr = REG_GET_FIELD(err_addr, MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr); | |
| 410 | − | |
| 411 | − if (!adev->aid_mask && | |
| 412 | − adev->smuio.funcs && | |
| 413 | − adev->smuio.funcs->get_socket_id) | |
| 414 | − addr_in.ma.socket_id = adev->smuio.funcs->get_socket_id(adev); | |
| 415 | − else | |
| 416 | − addr_in.ma.socket_id = 0; | |
| 417 | − | |
| 418 | − addr_in.ma.err_addr = err_addr; | |
| 419 | − addr_in.ma.ch_inst = ch_inst; | |
| 420 | − addr_in.ma.umc_inst = umc_inst; | |
| 421 | − addr_in.ma.node_inst = node_inst; | |
| 422 | − | |
| 423 | − umc_v12_0_convert_error_address(adev, err_data, &addr_in, NULL, true); | |
| 424 | − } | |
| 425 | − | |
| 426 | − /* clear umc status */ | |
| 427 | − WREG64_PCIE_EXT((mc_umc_status_addr + umc_reg_offset) * 4, 0x0ULL); | |
| 428 | − | |
| 429 | − return 0; | |
| 430 | −} | |
| 431 | − | |
| 432 | −static void umc_v12_0_query_ras_error_address(struct amdgpu_device *adev, | |
| 433 | − void *ras_error_status) | |
| 434 | −{ | |
| 435 | − amdgpu_umc_loop_channels(adev, | |
| 436 | − umc_v12_0_query_error_address, ras_error_status); | |
| 437 | −} | |
| 438 | − | |
| 439 | −static int umc_v12_0_err_cnt_init_per_channel(struct amdgpu_device *adev, | |
| 440 | − uint32_t node_inst, uint32_t umc_inst, | |
| 441 | − uint32_t ch_inst, void *data) | |
| 442 | −{ | |
| 443 | − uint32_t odecc_cnt_sel; | |
| 444 | − uint64_t odecc_cnt_sel_addr, odecc_err_cnt_addr; | |
| 445 | − uint64_t umc_reg_offset = | |
| 446 | − get_umc_v12_0_reg_offset(adev, node_inst, umc_inst, ch_inst); | |
| 447 | − | |
| 448 | − odecc_cnt_sel_addr = | |
| 449 | − SOC15_REG_OFFSET(UMC, 0, regUMCCH0_OdEccCntSel); | |
| 450 | − odecc_err_cnt_addr = | |
| 451 | − SOC15_REG_OFFSET(UMC, 0, regUMCCH0_OdEccErrCnt); | |
| 452 | − | |
| 453 | − odecc_cnt_sel = RREG32_PCIE_EXT((odecc_cnt_sel_addr + umc_reg_offset) * 4); | |
| 454 | − | |
| 455 | − /* set ce error interrupt type to APIC based interrupt */ | |
| 456 | − odecc_cnt_sel = REG_SET_FIELD(odecc_cnt_sel, UMCCH0_OdEccCntSel, | |
| 457 | − OdEccErrInt, 0x1); | |
| 458 | − WREG32_PCIE_EXT((odecc_cnt_sel_addr + umc_reg_offset) * 4, odecc_cnt_sel); | |
| 459 | − | |
| 460 | − /* set error count to initial value */ | |
| 461 | − WREG32_PCIE_EXT((odecc_err_cnt_addr + umc_reg_offset) * 4, UMC_V12_0_CE_CNT_INIT); | |
| 462 | − | |
| 463 | − return 0; | |
| 464 | −} | |
| 465 | − | |
| 466 | 181 | static bool umc_v12_0_check_ecc_err_status(struct amdgpu_device *adev, |
| 467 | 182 | enum amdgpu_mca_error_type type, void *ras_error_status) |
| 468 | 183 | { |
| @@ -482,309 +197,11 @@ static bool umc_v12_0_check_ecc_err_status(struct amdgpu_device *adev, · 13 unmodified lines | ||
| 482 | 197 | return false; |
| 483 | 198 | } |
| 484 | 199 | |
| 485 | −static void umc_v12_0_err_cnt_init(struct amdgpu_device *adev) | |
| 486 | −{ | |
| 487 | − amdgpu_umc_loop_channels(adev, | |
| 488 | − umc_v12_0_err_cnt_init_per_channel, NULL); | |
| 489 | −} | |
| 490 | − | |
| 491 | −static bool umc_v12_0_query_ras_poison_mode(struct amdgpu_device *adev) | |
| 492 | −{ | |
| 493 | − /* | |
| 494 | − * Force return true, because regUMCCH0_EccCtrl | |
| 495 | − * is not accessible from host side | |
| 496 | − */ | |
| 497 | − return true; | |
| 498 | −} | |
| 499 | − | |
| 500 | −const struct amdgpu_ras_block_hw_ops umc_v12_0_ras_hw_ops = { | |
| 501 | − .query_ras_error_count = umc_v12_0_query_ras_error_count, | |
| 502 | − .query_ras_error_address = umc_v12_0_query_ras_error_address, | |
| 503 | −}; | |
| 504 | − | |
| 505 | −static int umc_v12_0_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, | |
| 506 | − enum aca_smu_type type, void *data) | |
| 507 | −{ | |
| 508 | − struct amdgpu_device *adev = handle->adev; | |
| 509 | − struct aca_bank_info info; | |
| 510 | − enum aca_error_type err_type; | |
| 511 | − u64 status, count; | |
| 512 | − u32 ext_error_code; | |
| 513 | − int ret; | |
| 514 | − | |
| 515 | − status = bank->regs[ACA_REG_IDX_STATUS]; | |
| 516 | − if (umc_v12_0_is_deferred_error(adev, status)) | |
| 517 | − err_type = ACA_ERROR_TYPE_DEFERRED; | |
| 518 | − else if (umc_v12_0_is_uncorrectable_error(adev, status)) | |
| 519 | − err_type = ACA_ERROR_TYPE_UE; | |
| 520 | − else if (umc_v12_0_is_correctable_error(adev, status)) | |
| 521 | − err_type = ACA_ERROR_TYPE_CE; | |
| 522 | − else | |
| 523 | − return 0; | |
| 524 | − bank->aca_err_type = err_type; | |
| 525 | − | |
| 526 | − ret = aca_bank_info_decode(bank, &info); | |
| 527 | − if (ret) | |
| 528 | − return ret; | |
| 529 | − | |
| 530 | − amdgpu_umc_update_ecc_status(adev, | |
| 531 | − bank->regs[ACA_REG_IDX_STATUS], | |
| 532 | − bank->regs[ACA_REG_IDX_IPID], | |
| 533 | − bank->regs[ACA_REG_IDX_ADDR]); | |
| 534 | − | |
| 535 | − ext_error_code = ACA_REG__STATUS__ERRORCODEEXT(status); | |
| 536 | − if (umc_v12_0_is_deferred_error(adev, status)) | |
| 537 | − count = ext_error_code == 0 ? | |
| 538 | − adev->umc.err_addr_cnt / adev->umc.retire_unit : 1ULL; | |
| 539 | − else | |
| 540 | − count = ext_error_code == 0 ? | |
| 541 | − ACA_REG__MISC0__ERRCNT(bank->regs[ACA_REG_IDX_MISC0]) : 1ULL; | |
| 542 | − | |
| 543 | − return aca_error_cache_log_bank_error(handle, &info, err_type, count); | |
| 544 | −} | |
| 545 | − | |
| 546 | −static const struct aca_bank_ops umc_v12_0_aca_bank_ops = { | |
| 547 | − .aca_bank_parser = umc_v12_0_aca_bank_parser, | |
| 548 | −}; | |
| 549 | − | |
| 550 | −const struct aca_info umc_v12_0_aca_info = { | |
| 551 | − .hwip = ACA_HWIP_TYPE_UMC, | |
| 552 | − .mask = ACA_ERROR_UE_MASK | ACA_ERROR_CE_MASK | ACA_ERROR_DEFERRED_MASK, | |
| 553 | − .bank_ops = &umc_v12_0_aca_bank_ops, | |
| 554 | −}; | |
| 555 | − | |
| 556 | −static int umc_v12_0_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) | |
| 557 | −{ | |
| 558 | − int ret; | |
| 559 | − | |
| 560 | − ret = amdgpu_umc_ras_late_init(adev, ras_block); | |
| 561 | − if (ret) | |
| 562 | − return ret; | |
| 563 | − | |
| 564 | − ret = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__UMC, | |
| 565 | − &umc_v12_0_aca_info, NULL); | |
| 566 | − if (ret) | |
| 567 | − return ret; | |
| 568 | − | |
| 569 | − return 0; | |
| 570 | −} | |
| 571 | − | |
| 572 | −static int umc_v12_0_update_ecc_status(struct amdgpu_device *adev, | |
| 573 | − uint64_t status, uint64_t ipid, uint64_t addr) | |
| 574 | −{ | |
| 575 | − struct amdgpu_ras *con = amdgpu_ras_get_context(adev); | |
| 576 | − uint16_t hwid, mcatype; | |
| 577 | − uint64_t page_pfn[UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL]; | |
| 578 | − uint64_t err_addr, pa_addr = 0; | |
| 579 | − struct ras_ecc_err *ecc_err; | |
| 580 | − struct ta_ras_query_address_output addr_out; | |
| 581 | − uint32_t shift_bit = adev->umc.flip_bits.flip_bits_in_pa[2]; | |
| 582 | − int count, ret, i; | |
| 583 | − | |
| 584 | − hwid = REG_GET_FIELD(ipid, MCMP1_IPIDT0, HardwareID); | |
| 585 | − mcatype = REG_GET_FIELD(ipid, MCMP1_IPIDT0, McaType); | |
| 586 | − | |
| 587 | − /* The IP block decode of consumption is SMU */ | |
| 588 | − if (hwid != MCA_UMC_HWID_V12_0 || mcatype != MCA_UMC_MCATYPE_V12_0) { | |
| 589 | − con->umc_ecc_log.consumption_q_count++; | |
| 590 | − return 0; | |
| 591 | − } | |
| 592 | − | |
| 593 | − if (!status) | |
| 594 | − return 0; | |
| 595 | − | |
| 596 | − if (!umc_v12_0_is_deferred_error(adev, status)) | |
| 597 | − return 0; | |
| 598 | − | |
| 599 | − err_addr = REG_GET_FIELD(addr, | |
| 600 | − MCA_UMC_UMC0_MCUMC_ADDRT0, ErrorAddr); | |
| 601 | − | |
| 602 | − dev_dbg(adev->dev, | |
| 603 | − "UMC:IPID:0x%llx, socket:%llu, aid:%llu, inst:%llu, ch:%llu, err_addr:0x%llx\n", | |
| 604 | − ipid, | |
| 605 | − MCA_IPID_2_SOCKET_ID(ipid), | |
| 606 | − MCA_IPID_2_DIE_ID(ipid), | |
| 607 | − MCA_IPID_2_UMC_INST(ipid), | |
| 608 | − MCA_IPID_2_UMC_CH(ipid), | |
| 609 | − err_addr); | |
| 610 | − | |
| 611 | − ret = amdgpu_umc_mca_to_addr(adev, | |
| 612 | − err_addr, MCA_IPID_2_UMC_CH(ipid), | |
| 613 | − MCA_IPID_2_UMC_INST(ipid), MCA_IPID_2_DIE_ID(ipid), | |
| 614 | − MCA_IPID_2_SOCKET_ID(ipid), &addr_out, true); | |
| 615 | − if (ret) | |
| 616 | − return ret; | |
| 617 | − | |
| 618 | − ecc_err = kzalloc_obj(*ecc_err); | |
| 619 | − if (!ecc_err) | |
| 620 | − return -ENOMEM; | |
| 621 | − | |
| 622 | − pa_addr = addr_out.pa.pa; | |
| 623 | − ecc_err->status = status; | |
| 624 | − ecc_err->ipid = ipid; | |
| 625 | − ecc_err->addr = addr; | |
| 626 | − ecc_err->pa_pfn = pa_addr >> AMDGPU_GPU_PAGE_SHIFT; | |
| 627 | − ecc_err->channel_idx = addr_out.pa.channel_idx; | |
| 628 | − | |
| 629 | − /* If converted pa_pfn is 0, use pa C4 pfn. */ | |
| 630 | − if (!ecc_err->pa_pfn) | |
| 631 | − ecc_err->pa_pfn = BIT_ULL(shift_bit) >> AMDGPU_GPU_PAGE_SHIFT; | |
| 632 | − | |
| 633 | − ret = amdgpu_umc_logs_ecc_err(adev, &con->umc_ecc_log.de_page_tree, ecc_err); | |
| 634 | − if (ret) { | |
| 635 | − if (ret == -EEXIST) | |
| 636 | − con->umc_ecc_log.de_queried_count++; | |
| 637 | − else | |
| 638 | − dev_err(adev->dev, "Fail to log ecc error! ret:%d\n", ret); | |
| 639 | − | |
| 640 | − kfree(ecc_err); | |
| 641 | − return ret; | |
| 642 | − } | |
| 643 | − | |
| 644 | − con->umc_ecc_log.de_queried_count++; | |
| 645 | − | |
| 646 | − memset(page_pfn, 0, sizeof(page_pfn)); | |
| 647 | − count = amdgpu_umc_lookup_bad_pages_in_a_row(adev, | |
| 648 | − pa_addr, | |
| 649 | − page_pfn, ARRAY_SIZE(page_pfn)); | |
| 650 | − if (count <= 0) { | |
| 651 | − dev_warn(adev->dev, "Fail to convert error address! count:%d\n", count); | |
| 652 | − return 0; | |
| 653 | − } | |
| 654 | − | |
| 655 | − /* Reserve memory */ | |
| 656 | − for (i = 0; i < count; i++) | |
| 657 | − amdgpu_ras_reserve_page(adev, page_pfn[i]); | |
| 658 | − | |
| 659 | − /* The problem case is as follows: | |
| 660 | − * 1. GPU A triggers a gpu ras reset, and GPU A drives | |
| 661 | − * GPU B to also perform a gpu ras reset. | |
| 662 | − * 2. After gpu B ras reset started, gpu B queried a DE | |
| 663 | − * data. Since the DE data was queried in the ras reset | |
| 664 | − * thread instead of the page retirement thread, bad | |
| 665 | − * page retirement work would not be triggered. Then | |
| 666 | − * even if all gpu resets are completed, the bad pages | |
| 667 | − * will be cached in RAM until GPU B's bad page retirement | |
| 668 | − * work is triggered again and then saved to eeprom. | |
| 669 | − * Trigger delayed work to save the bad pages to eeprom in time | |
| 670 | − * after gpu ras reset is completed. | |
| 671 | − */ | |
| 672 | − if (amdgpu_ras_in_recovery(adev)) | |
| 673 | − schedule_delayed_work(&con->page_retirement_dwork, | |
| 674 | − msecs_to_jiffies(DELAYED_TIME_FOR_GPU_RESET)); | |
| 675 | − | |
| 676 | − return 0; | |
| 677 | −} | |
| 678 | − | |
| 679 | −static int umc_v12_0_fill_error_record(struct amdgpu_device *adev, | |
| 680 | − struct ras_ecc_err *ecc_err, void *ras_error_status) | |
| 681 | −{ | |
| 682 | − struct ras_err_data *err_data = (struct ras_err_data *)ras_error_status; | |
| 683 | − uint64_t page_pfn[UMC_V12_0_BAD_PAGE_NUM_PER_CHANNEL]; | |
| 684 | − int ret, i, count; | |
| 685 | − | |
| 686 | − if (!err_data || !ecc_err) | |
| 687 | − return -EINVAL; | |
| 688 | − | |
| 689 | − memset(page_pfn, 0, sizeof(page_pfn)); | |
| 690 | − count = amdgpu_umc_lookup_bad_pages_in_a_row(adev, | |
| 691 | − ecc_err->pa_pfn << AMDGPU_GPU_PAGE_SHIFT, | |
| 692 | − page_pfn, ARRAY_SIZE(page_pfn)); | |
| 693 | − | |
| 694 | − for (i = 0; i < count; i++) { | |
| 695 | − ret = amdgpu_umc_fill_error_record(err_data, | |
| 696 | − ecc_err->addr, | |
| 697 | − page_pfn[i] << AMDGPU_GPU_PAGE_SHIFT, | |
| 698 | − ecc_err->channel_idx, | |
| 699 | − MCA_IPID_2_UMC_INST(ecc_err->ipid)); | |
| 700 | − if (ret) | |
| 701 | − break; | |
| 702 | − } | |
| 703 | − | |
| 704 | − err_data->de_count++; | |
| 705 | − | |
| 706 | − return ret; | |
| 707 | −} | |
| 708 | − | |
| 709 | −static void umc_v12_0_query_ras_ecc_err_addr(struct amdgpu_device *adev, | |
| 710 | − void *ras_error_status) | |
| 711 | −{ | |
| 712 | − struct amdgpu_ras *con = amdgpu_ras_get_context(adev); | |
| 713 | − struct ras_ecc_err *entries[MAX_ECC_NUM_PER_RETIREMENT]; | |
| 714 | − struct radix_tree_root *ecc_tree; | |
| 715 | − int new_detected, ret, i; | |
| 716 | − | |
| 717 | − ecc_tree = &con->umc_ecc_log.de_page_tree; | |
| 718 | − | |
| 719 | − mutex_lock(&con->umc_ecc_log.lock); | |
| 720 | − new_detected = radix_tree_gang_lookup_tag(ecc_tree, (void **)entries, | |
| 721 | − 0, ARRAY_SIZE(entries), UMC_ECC_NEW_DETECTED_TAG); | |
| 722 | − for (i = 0; i < new_detected; i++) { | |
| 723 | − if (!entries[i]) | |
| 724 | − continue; | |
| 725 | − | |
| 726 | − ret = umc_v12_0_fill_error_record(adev, entries[i], ras_error_status); | |
| 727 | − if (ret) { | |
| 728 | − dev_err(adev->dev, "Fail to fill umc error record, ret:%d\n", ret); | |
| 729 | − break; | |
| 730 | − } | |
| 731 | − radix_tree_tag_clear(ecc_tree, | |
| 732 | − entries[i]->pa_pfn, UMC_ECC_NEW_DETECTED_TAG); | |
| 733 | − } | |
| 734 | − mutex_unlock(&con->umc_ecc_log.lock); | |
| 735 | −} | |
| 736 | − | |
| 737 | −static uint32_t umc_v12_0_get_die_id(struct amdgpu_device *adev, | |
| 738 | − uint64_t mca_addr, uint64_t retired_page) | |
| 739 | −{ | |
| 740 | − uint32_t die = 0; | |
| 741 | − | |
| 742 | − /* we only calculate die id for nps1 mode right now */ | |
| 743 | − die += ((((retired_page >> 12) & 0x1ULL)^ | |
| 744 | − ((retired_page >> 20) & 0x1ULL) ^ | |
| 745 | − ((retired_page >> 27) & 0x1ULL) ^ | |
| 746 | − ((retired_page >> 34) & 0x1ULL) ^ | |
| 747 | − ((retired_page >> 41) & 0x1ULL)) << 0); | |
| 748 | − | |
| 749 | − /* the original PA_C4 and PA_R13 may be cleared in retired_page, so | |
| 750 | − * get them from mca_addr. | |
| 751 | − */ | |
| 752 | − die += ((((retired_page >> 13) & 0x1ULL) ^ | |
| 753 | − ((mca_addr >> 5) & 0x1ULL) ^ | |
| 754 | − ((retired_page >> 28) & 0x1ULL) ^ | |
| 755 | − ((mca_addr >> 23) & 0x1ULL) ^ | |
| 756 | − ((retired_page >> 42) & 0x1ULL)) << 1); | |
| 757 | − die &= 3; | |
| 758 | − | |
| 759 | − return die; | |
| 760 | −} | |
| 761 | − | |
| 762 | −static void umc_v12_0_mca_ipid_parse(struct amdgpu_device *adev, uint64_t ipid, | |
| 763 | − uint32_t *did, uint32_t *ch, uint32_t *umc_inst, uint32_t *sid) | |
| 764 | −{ | |
| 765 | − if (did) | |
| 766 | − *did = MCA_IPID_2_DIE_ID(ipid); | |
| 767 | − if (ch) | |
| 768 | − *ch = MCA_IPID_2_UMC_CH(ipid); | |
| 769 | − if (umc_inst) | |
| 770 | − *umc_inst = MCA_IPID_2_UMC_INST(ipid); | |
| 771 | − if (sid) | |
| 772 | − *sid = MCA_IPID_2_SOCKET_ID(ipid); | |
| 773 | −} | |
| 774 | − | |
| 775 | 200 | struct amdgpu_umc_ras umc_v12_0_ras = { |
| 776 | 201 | .ras_block = { |
| 777 | − .hw_ops = &umc_v12_0_ras_hw_ops, | |
| 778 | − .ras_late_init = umc_v12_0_ras_late_init, | |
| 202 | + .hw_ops = NULL, | |
| 779 | 203 | }, |
| 780 | − .err_cnt_init = umc_v12_0_err_cnt_init, | |
| 781 | − .query_ras_poison_mode = umc_v12_0_query_ras_poison_mode, | |
| 782 | − .ecc_info_query_ras_error_address = umc_v12_0_query_ras_ecc_err_addr, | |
| 783 | 204 | .check_ecc_err_status = umc_v12_0_check_ecc_err_status, |
| 784 | − .update_ecc_status = umc_v12_0_update_ecc_status, | |
| 785 | − .convert_ras_err_addr = umc_v12_0_convert_error_address, | |
| 786 | − .get_die_id_from_pa = umc_v12_0_get_die_id, | |
| 787 | 205 | .get_retire_flip_bits = umc_v12_0_get_retire_flip_bits, |
| 788 | − .mca_ipid_parse = umc_v12_0_mca_ipid_parse, | |
| 789 | 206 | }; |
| 790 | 207 | |
| @@ -26,31 +26,6 @@ · 25 unmodified lines | ||
| 26 | 26 | #include "soc15_common.h" |
| 27 | 27 | #include "amdgpu.h" |
| 28 | 28 | |
| 29 | −#define UMC_V12_0_NODE_DIST 0x40000000 | |
| 30 | −#define UMC_V12_0_INST_DIST 0x40000 | |
| 31 | − | |
| 32 | −/* UMC register per channel offset */ | |
| 33 | −#define UMC_V12_0_PER_CHANNEL_OFFSET 0x400 | |
| 34 | − | |
| 35 | −/* UMC cross node offset */ | |
| 36 | −#define UMC_V12_0_CROSS_NODE_OFFSET 0x100000000 | |
| 37 | − | |
| 38 | −/* OdEccErrCnt max value */ | |
| 39 | −#define UMC_V12_0_CE_CNT_MAX 0xffff | |
| 40 | −/* umc ce interrupt threshold */ | |
| 41 | −#define UMC_V12_0_CE_INT_THRESHOLD 0xffff | |
| 42 | −/* umc ce count initial value */ | |
| 43 | −#define UMC_V12_0_CE_CNT_INIT (UMC_V12_0_CE_CNT_MAX - UMC_V12_0_CE_INT_THRESHOLD) | |
| 44 | − | |
| 45 | −/* number of umc channel instance with memory map register access */ | |
| 46 | −#define UMC_V12_0_CHANNEL_INSTANCE_NUM 8 | |
| 47 | −/* number of umc instance with memory map register access */ | |
| 48 | −#define UMC_V12_0_UMC_INSTANCE_NUM 4 | |
| 49 | − | |
| 50 | −/* Total channel instances for all available umc nodes */ | |
| 51 | −#define UMC_V12_0_TOTAL_CHANNEL_NUM(adev) \ | |
| 52 | − (UMC_V12_0_CHANNEL_INSTANCE_NUM * (adev)->gmc.num_umc) | |
| 53 | − | |
| 54 | 29 | /* one piece of normalized address is mapped to 8 pieces of physical address */ |
| 55 | 30 | #define UMC_V12_0_NA_MAP_PA_NUM 8 |
| 56 | 31 | /* R13 bit shift should be considered, double the number */ |
| @@ -75,9 +50,6 @@ · 18 unmodified lines | ||
| 75 | 50 | /* row bits in MCA address */ |
| 76 | 51 | #define UMC_V12_0_MA_R0_BIT 10 |
| 77 | 52 | |
| 78 | −#define MCA_UMC_HWID_V12_0 0x96 | |
| 79 | −#define MCA_UMC_MCATYPE_V12_0 0x0 | |
| 80 | − | |
| 81 | 53 | #define MCA_IPID_LO_2_UMC_CH(_ipid_lo) (((((_ipid_lo) >> 20) & 0x1) * 4) + \ |
| 82 | 54 | (((_ipid_lo) >> 12) & 0xF)) |
| 83 | 55 | #define MCA_IPID_LO_2_UMC_INST(_ipid_lo) (((_ipid_lo) >> 21) & 0x7) |
| @@ -1165,36 +1165,6 @@ static int uvd_v6_0_wait_for_idle(struct amdgpu_ip_block *ip_block) · 1164 unmodified lines | ||
| 1165 | 1165 | } |
| 1166 | 1166 | |
| 1167 | 1167 | #define AMDGPU_UVD_STATUS_BUSY_MASK 0xfd |
| 1168 | −static bool uvd_v6_0_check_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 1169 | −{ | |
| 1170 | − struct amdgpu_device *adev = ip_block->adev; | |
| 1171 | − u32 srbm_soft_reset = 0; | |
| 1172 | − u32 tmp = RREG32(mmSRBM_STATUS); | |
| 1173 | − | |
| 1174 | − if (REG_GET_FIELD(tmp, SRBM_STATUS, UVD_RQ_PENDING) || | |
| 1175 | − REG_GET_FIELD(tmp, SRBM_STATUS, UVD_BUSY) || | |
| 1176 | − (RREG32(mmUVD_STATUS) & AMDGPU_UVD_STATUS_BUSY_MASK)) | |
| 1177 | − srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_UVD, 1); | |
| 1178 | − | |
| 1179 | − if (srbm_soft_reset) { | |
| 1180 | − adev->uvd.inst->srbm_soft_reset = srbm_soft_reset; | |
| 1181 | − return true; | |
| 1182 | − } else { | |
| 1183 | − adev->uvd.inst->srbm_soft_reset = 0; | |
| 1184 | − return false; | |
| 1185 | − } | |
| 1186 | −} | |
| 1187 | − | |
| 1188 | −static int uvd_v6_0_pre_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 1189 | −{ | |
| 1190 | − struct amdgpu_device *adev = ip_block->adev; | |
| 1191 | − | |
| 1192 | − if (!adev->uvd.inst->srbm_soft_reset) | |
| 1193 | − return 0; | |
| 1194 | − | |
| 1195 | − uvd_v6_0_stop(adev); | |
| 1196 | − return 0; | |
| 1197 | −} | |
| 1198 | 1168 | |
| 1199 | 1169 | static int uvd_v6_0_soft_reset(struct amdgpu_ip_block *ip_block) |
| 1200 | 1170 | { |
| @@ -1227,18 +1197,6 @@ static int uvd_v6_0_soft_reset(struct amdgpu_ip_block *ip_block) · 26 unmodified lines | ||
| 1227 | 1197 | return 0; |
| 1228 | 1198 | } |
| 1229 | 1199 | |
| 1230 | −static int uvd_v6_0_post_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 1231 | −{ | |
| 1232 | − struct amdgpu_device *adev = ip_block->adev; | |
| 1233 | − | |
| 1234 | − if (!adev->uvd.inst->srbm_soft_reset) | |
| 1235 | − return 0; | |
| 1236 | − | |
| 1237 | − mdelay(5); | |
| 1238 | − | |
| 1239 | − return uvd_v6_0_start(adev); | |
| 1240 | −} | |
| 1241 | − | |
| 1242 | 1200 | static int uvd_v6_0_set_interrupt_state(struct amdgpu_device *adev, |
| 1243 | 1201 | struct amdgpu_irq_src *source, |
| 1244 | 1202 | unsigned type, |
| @@ -1538,10 +1496,7 @@ static const struct amd_ip_funcs uvd_v6_0_ip_funcs = { · 293 unmodified lines | ||
| 1538 | 1496 | .resume = uvd_v6_0_resume, |
| 1539 | 1497 | .is_idle = uvd_v6_0_is_idle, |
| 1540 | 1498 | .wait_for_idle = uvd_v6_0_wait_for_idle, |
| 1541 | − .check_soft_reset = uvd_v6_0_check_soft_reset, | |
| 1542 | − .pre_soft_reset = uvd_v6_0_pre_soft_reset, | |
| 1543 | 1499 | .soft_reset = uvd_v6_0_soft_reset, |
| 1544 | − .post_soft_reset = uvd_v6_0_post_soft_reset, | |
| 1545 | 1500 | .set_clockgating_state = uvd_v6_0_set_clockgating_state, |
| 1546 | 1501 | .set_powergating_state = uvd_v6_0_set_powergating_state, |
| 1547 | 1502 | .get_clockgating_state = uvd_v6_0_get_clockgating_state, |
| @@ -631,47 +631,6 @@ static int vce_v3_0_wait_for_idle(struct amdgpu_ip_block *ip_block) · 630 unmodified lines | ||
| 631 | 631 | #define AMDGPU_VCE_STATUS_BUSY_MASK (VCE_STATUS_VCPU_REPORT_AUTO_BUSY_MASK | \ |
| 632 | 632 | VCE_STATUS_VCPU_REPORT_RB0_BUSY_MASK) |
| 633 | 633 | |
| 634 | −static bool vce_v3_0_check_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 635 | −{ | |
| 636 | − struct amdgpu_device *adev = ip_block->adev; | |
| 637 | − u32 srbm_soft_reset = 0; | |
| 638 | − | |
| 639 | − /* According to VCE team , we should use VCE_STATUS instead | |
| 640 | − * SRBM_STATUS.VCE_BUSY bit for busy status checking. | |
| 641 | − * GRBM_GFX_INDEX.INSTANCE_INDEX is used to specify which VCE | |
| 642 | − * instance's registers are accessed | |
| 643 | − * (0 for 1st instance, 10 for 2nd instance). | |
| 644 | − * | |
| 645 | − *VCE_STATUS | |
| 646 | − *|UENC|ACPI|AUTO ACTIVE|RB1 |RB0 |RB2 | |FW_LOADED|JOB | | |
| 647 | − *|----+----+-----------+----+----+----+----------+---------+----| | |
| 648 | − *|bit8|bit7| bit6 |bit5|bit4|bit3| bit2 | bit1 |bit0| | |
| 649 | − * | |
| 650 | − * VCE team suggest use bit 3--bit 6 for busy status check | |
| 651 | − */ | |
| 652 | − mutex_lock(&adev->grbm_idx_mutex); | |
| 653 | − WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(0)); | |
| 654 | − if (RREG32(mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) { | |
| 655 | − srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1); | |
| 656 | − srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1); | |
| 657 | − } | |
| 658 | − WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(1)); | |
| 659 | − if (RREG32(mmVCE_STATUS) & AMDGPU_VCE_STATUS_BUSY_MASK) { | |
| 660 | − srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE0, 1); | |
| 661 | − srbm_soft_reset = REG_SET_FIELD(srbm_soft_reset, SRBM_SOFT_RESET, SOFT_RESET_VCE1, 1); | |
| 662 | − } | |
| 663 | − WREG32(mmGRBM_GFX_INDEX, GET_VCE_INSTANCE(0)); | |
| 664 | − mutex_unlock(&adev->grbm_idx_mutex); | |
| 665 | − | |
| 666 | − if (srbm_soft_reset) { | |
| 667 | − adev->vce.srbm_soft_reset = srbm_soft_reset; | |
| 668 | − return true; | |
| 669 | − } else { | |
| 670 | − adev->vce.srbm_soft_reset = 0; | |
| 671 | − return false; | |
| 672 | − } | |
| 673 | −} | |
| 674 | − | |
| 675 | 634 | static int vce_v3_0_soft_reset(struct amdgpu_ip_block *ip_block) |
| 676 | 635 | { |
| 677 | 636 | struct amdgpu_device *adev = ip_block->adev; |
| @@ -703,31 +662,6 @@ static int vce_v3_0_soft_reset(struct amdgpu_ip_block *ip_block) · 25 unmodified lines | ||
| 703 | 662 | return 0; |
| 704 | 663 | } |
| 705 | 664 | |
| 706 | −static int vce_v3_0_pre_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 707 | −{ | |
| 708 | − struct amdgpu_device *adev = ip_block->adev; | |
| 709 | − | |
| 710 | − if (!adev->vce.srbm_soft_reset) | |
| 711 | − return 0; | |
| 712 | − | |
| 713 | − mdelay(5); | |
| 714 | − | |
| 715 | − return vce_v3_0_suspend(ip_block); | |
| 716 | −} | |
| 717 | − | |
| 718 | − | |
| 719 | −static int vce_v3_0_post_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 720 | −{ | |
| 721 | − struct amdgpu_device *adev = ip_block->adev; | |
| 722 | − | |
| 723 | − if (!adev->vce.srbm_soft_reset) | |
| 724 | − return 0; | |
| 725 | − | |
| 726 | − mdelay(5); | |
| 727 | − | |
| 728 | − return vce_v3_0_resume(ip_block); | |
| 729 | −} | |
| 730 | − | |
| 731 | 665 | static int vce_v3_0_set_interrupt_state(struct amdgpu_device *adev, |
| 732 | 666 | struct amdgpu_irq_src *source, |
| 733 | 667 | unsigned type, |
| @@ -909,10 +843,7 @@ static const struct amd_ip_funcs vce_v3_0_ip_funcs = { · 175 unmodified lines | ||
| 909 | 843 | .resume = vce_v3_0_resume, |
| 910 | 844 | .is_idle = vce_v3_0_is_idle, |
| 911 | 845 | .wait_for_idle = vce_v3_0_wait_for_idle, |
| 912 | − .check_soft_reset = vce_v3_0_check_soft_reset, | |
| 913 | − .pre_soft_reset = vce_v3_0_pre_soft_reset, | |
| 914 | 846 | .soft_reset = vce_v3_0_soft_reset, |
| 915 | − .post_soft_reset = vce_v3_0_post_soft_reset, | |
| 916 | 847 | .set_clockgating_state = vce_v3_0_set_clockgating_state, |
| 917 | 848 | .set_powergating_state = vce_v3_0_set_powergating_state, |
| 918 | 849 | .get_clockgating_state = vce_v3_0_get_clockgating_state, |
| @@ -159,9 +159,8 @@ static void vcn_v2_5_ring_begin_use(struct amdgpu_ring *ring) · 158 unmodified lines | ||
| 159 | 159 | struct amdgpu_device *adev = ring->adev; |
| 160 | 160 | struct amdgpu_vcn_inst *v = &adev->vcn.inst[ring->me]; |
| 161 | 161 | |
| 162 | − atomic_inc(&adev->vcn.inst[0].total_submission_cnt); | |
| 163 | − | |
| 164 | − cancel_delayed_work_sync(&adev->vcn.inst[0].idle_work); | |
| 162 | + if (!atomic_fetch_inc(&adev->vcn.inst[0].total_submission_cnt)) | |
| 163 | + cancel_delayed_work_sync(&adev->vcn.inst[0].idle_work); | |
| 165 | 164 | |
| 166 | 165 | /* We can safely return early here because we've cancelled the |
| 167 | 166 | * the delayed work so there is no one else to set it to false |
| @@ -207,10 +206,9 @@ static void vcn_v2_5_ring_end_use(struct amdgpu_ring *ring) · 39 unmodified lines | ||
| 207 | 206 | !adev->vcn.inst[ring->me].using_unified_queue) |
| 208 | 207 | atomic_dec(&adev->vcn.inst[ring->me].dpg_enc_submission_cnt); |
| 209 | 208 | |
| 210 | − atomic_dec(&adev->vcn.inst[0].total_submission_cnt); | |
| 211 | − | |
| 212 | − schedule_delayed_work(&adev->vcn.inst[0].idle_work, | |
| 213 | − VCN_IDLE_TIMEOUT); | |
| 209 | + if (atomic_dec_and_test(&adev->vcn.inst[0].total_submission_cnt)) | |
| 210 | + schedule_delayed_work(&adev->vcn.inst[0].idle_work, | |
| 211 | + VCN_IDLE_TIMEOUT); | |
| 214 | 212 | } |
| 215 | 213 | |
| 216 | 214 | /** |
| @@ -1995,7 +1995,7 @@ static int vcn_v4_0_ring_reset(struct amdgpu_ring *ring, · 1994 unmodified lines | ||
| 1995 | 1995 | return amdgpu_ring_reset_helper_end(ring, timedout_fence); |
| 1996 | 1996 | } |
| 1997 | 1997 | |
| 1998 | −static struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = { | |
| 1998 | +static const struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = { | |
| 1999 | 1999 | .type = AMDGPU_RING_TYPE_VCN_ENC, |
| 2000 | 2000 | .align_mask = 0x3f, |
| 2001 | 2001 | .nop = VCN_ENC_CMD_NO_OP, |
| @@ -2028,6 +2028,40 @@ static struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs = { · 26 unmodified lines | ||
| 2028 | 2028 | .reset = vcn_v4_0_ring_reset, |
| 2029 | 2029 | }; |
| 2030 | 2030 | |
| 2031 | +static const struct amdgpu_ring_funcs vcn_v4_0_unified_ring_vm_funcs_secure = { | |
| 2032 | + .type = AMDGPU_RING_TYPE_VCN_ENC, | |
| 2033 | + .align_mask = 0x3f, | |
| 2034 | + .nop = VCN_ENC_CMD_NO_OP, | |
| 2035 | + .secure_submission_supported = true, | |
| 2036 | + .no_user_fence = true, | |
| 2037 | + .extra_bytes = sizeof(struct amdgpu_vcn_rb_metadata), | |
| 2038 | + .get_rptr = vcn_v4_0_unified_ring_get_rptr, | |
| 2039 | + .get_wptr = vcn_v4_0_unified_ring_get_wptr, | |
| 2040 | + .set_wptr = vcn_v4_0_unified_ring_set_wptr, | |
| 2041 | + .patch_cs_in_place = vcn_v4_0_ring_patch_cs_in_place, | |
| 2042 | + .emit_frame_size = | |
| 2043 | + SOC15_FLUSH_GPU_TLB_NUM_WREG * 3 + | |
| 2044 | + SOC15_FLUSH_GPU_TLB_NUM_REG_WAIT * 4 + | |
| 2045 | + 4 + /* vcn_v2_0_enc_ring_emit_vm_flush */ | |
| 2046 | + 5 + 5 + /* vcn_v2_0_enc_ring_emit_fence x2 vm fence */ | |
| 2047 | + 1, /* vcn_v2_0_enc_ring_insert_end */ | |
| 2048 | + .emit_ib_size = 5, /* vcn_v2_0_enc_ring_emit_ib */ | |
| 2049 | + .emit_ib = vcn_v2_0_enc_ring_emit_ib, | |
| 2050 | + .emit_fence = vcn_v2_0_enc_ring_emit_fence, | |
| 2051 | + .emit_vm_flush = vcn_v2_0_enc_ring_emit_vm_flush, | |
| 2052 | + .test_ring = amdgpu_vcn_enc_ring_test_ring, | |
| 2053 | + .test_ib = amdgpu_vcn_unified_ring_test_ib, | |
| 2054 | + .insert_nop = amdgpu_ring_insert_nop, | |
| 2055 | + .insert_end = vcn_v2_0_enc_ring_insert_end, | |
| 2056 | + .pad_ib = amdgpu_ring_generic_pad_ib, | |
| 2057 | + .begin_use = amdgpu_vcn_ring_begin_use, | |
| 2058 | + .end_use = amdgpu_vcn_ring_end_use, | |
| 2059 | + .emit_wreg = vcn_v2_0_enc_ring_emit_wreg, | |
| 2060 | + .emit_reg_wait = vcn_v2_0_enc_ring_emit_reg_wait, | |
| 2061 | + .emit_reg_write_reg_wait = amdgpu_ring_emit_reg_write_reg_wait_helper, | |
| 2062 | + .reset = vcn_v4_0_ring_reset, | |
| 2063 | +}; | |
| 2064 | + | |
| 2031 | 2065 | /** |
| 2032 | 2066 | * vcn_v4_0_set_unified_ring_funcs - set unified ring functions |
| 2033 | 2067 | * |
| @@ -2044,10 +2078,11 @@ static void vcn_v4_0_set_unified_ring_funcs(struct amdgpu_device *adev) · 10 unmodified lines | ||
| 2044 | 2078 | continue; |
| 2045 | 2079 | |
| 2046 | 2080 | if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 2)) |
| 2047 | − vcn_v4_0_unified_ring_vm_funcs.secure_submission_supported = true; | |
| 2048 | − | |
| 2049 | − adev->vcn.inst[i].ring_enc[0].funcs = | |
| 2050 | − (const struct amdgpu_ring_funcs *)&vcn_v4_0_unified_ring_vm_funcs; | |
| 2081 | + adev->vcn.inst[i].ring_enc[0].funcs = | |
| 2082 | + &vcn_v4_0_unified_ring_vm_funcs_secure; | |
| 2083 | + else | |
| 2084 | + adev->vcn.inst[i].ring_enc[0].funcs = | |
| 2085 | + &vcn_v4_0_unified_ring_vm_funcs; | |
| 2051 | 2086 | adev->vcn.inst[i].ring_enc[0].me = i; |
| 2052 | 2087 | } |
| 2053 | 2088 | } |
| @@ -115,6 +115,19 @@ static int vcn_v4_0_3_early_init(struct amdgpu_ip_block *ip_block) · 114 unmodified lines | ||
| 115 | 115 | struct amdgpu_device *adev = ip_block->adev; |
| 116 | 116 | int i, r; |
| 117 | 117 | |
| 118 | + switch (amdgpu_user_queue) { | |
| 119 | + case -1: | |
| 120 | + case 0: | |
| 121 | + default: | |
| 122 | + adev->vcn.disable_kq = false; | |
| 123 | + adev->vcn.disable_uq = true; | |
| 124 | + break; | |
| 125 | + case 2: | |
| 126 | + adev->vcn.disable_kq = true; | |
| 127 | + adev->vcn.disable_uq = true; | |
| 128 | + break; | |
| 129 | + } | |
| 130 | + | |
| 118 | 131 | for (i = 0; i < adev->vcn.num_vcn_inst; ++i) |
| 119 | 132 | /* re-use enc ring as unified ring */ |
| 120 | 133 | adev->vcn.inst[i].num_enc_rings = 1; |
| @@ -217,6 +230,10 @@ static int vcn_v4_0_3_sw_init(struct amdgpu_ip_block *ip_block) · 96 unmodified lines | ||
| 217 | 230 | |
| 218 | 231 | ring = &adev->vcn.inst[i].ring_enc[0]; |
| 219 | 232 | ring->use_doorbell = true; |
| 233 | + if (adev->vcn.disable_kq) { | |
| 234 | + ring->no_scheduler = true; | |
| 235 | + ring->no_user_submission = true; | |
| 236 | + } | |
| 220 | 237 | |
| 221 | 238 | if (!amdgpu_sriov_vf(adev)) |
| 222 | 239 | ring->doorbell_index = |
| @@ -2146,71 +2163,6 @@ static const struct amdgpu_ras_block_hw_ops vcn_v4_0_3_ras_hw_ops = { · 1923 unmodified lines | ||
| 2146 | 2163 | .query_poison_status = vcn_v4_0_3_query_poison_status, |
| 2147 | 2164 | }; |
| 2148 | 2165 | |
| 2149 | −static int vcn_v4_0_3_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, | |
| 2150 | − enum aca_smu_type type, void *data) | |
| 2151 | −{ | |
| 2152 | − struct aca_bank_info info; | |
| 2153 | − u64 misc0; | |
| 2154 | − int ret; | |
| 2155 | − | |
| 2156 | − ret = aca_bank_info_decode(bank, &info); | |
| 2157 | − if (ret) | |
| 2158 | − return ret; | |
| 2159 | − | |
| 2160 | − misc0 = bank->regs[ACA_REG_IDX_MISC0]; | |
| 2161 | − switch (type) { | |
| 2162 | − case ACA_SMU_TYPE_UE: | |
| 2163 | − bank->aca_err_type = ACA_ERROR_TYPE_UE; | |
| 2164 | − ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, | |
| 2165 | − 1ULL); | |
| 2166 | − break; | |
| 2167 | − case ACA_SMU_TYPE_CE: | |
| 2168 | − bank->aca_err_type = ACA_ERROR_TYPE_CE; | |
| 2169 | − ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, | |
| 2170 | − ACA_REG__MISC0__ERRCNT(misc0)); | |
| 2171 | − break; | |
| 2172 | − default: | |
| 2173 | − return -EINVAL; | |
| 2174 | − } | |
| 2175 | − | |
| 2176 | − return ret; | |
| 2177 | −} | |
| 2178 | − | |
| 2179 | −/* reference to smu driver if header file */ | |
| 2180 | −static int vcn_v4_0_3_err_codes[] = { | |
| 2181 | − 14, 15, /* VCN */ | |
| 2182 | −}; | |
| 2183 | − | |
| 2184 | −static bool vcn_v4_0_3_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, | |
| 2185 | − enum aca_smu_type type, void *data) | |
| 2186 | −{ | |
| 2187 | − u32 instlo; | |
| 2188 | − | |
| 2189 | − instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); | |
| 2190 | − instlo &= GENMASK(31, 1); | |
| 2191 | − | |
| 2192 | − if (instlo != mmSMNAID_AID0_MCA_SMU) | |
| 2193 | − return false; | |
| 2194 | − | |
| 2195 | − if (aca_bank_check_error_codes(handle->adev, bank, | |
| 2196 | − vcn_v4_0_3_err_codes, | |
| 2197 | − ARRAY_SIZE(vcn_v4_0_3_err_codes))) | |
| 2198 | − return false; | |
| 2199 | − | |
| 2200 | − return true; | |
| 2201 | −} | |
| 2202 | − | |
| 2203 | −static const struct aca_bank_ops vcn_v4_0_3_aca_bank_ops = { | |
| 2204 | − .aca_bank_parser = vcn_v4_0_3_aca_bank_parser, | |
| 2205 | − .aca_bank_is_valid = vcn_v4_0_3_aca_bank_is_valid, | |
| 2206 | −}; | |
| 2207 | − | |
| 2208 | −static const struct aca_info vcn_v4_0_3_aca_info = { | |
| 2209 | − .hwip = ACA_HWIP_TYPE_SMU, | |
| 2210 | − .mask = ACA_ERROR_UE_MASK, | |
| 2211 | − .bank_ops = &vcn_v4_0_3_aca_bank_ops, | |
| 2212 | −}; | |
| 2213 | − | |
| 2214 | 2166 | static int vcn_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) |
| 2215 | 2167 | { |
| 2216 | 2168 | int r; |
| @@ -2226,11 +2178,6 @@ static int vcn_v4_0_3_ras_late_init(struct amdgpu_device *adev, struct ras_commo · 9 unmodified lines | ||
| 2226 | 2178 | goto late_fini; |
| 2227 | 2179 | } |
| 2228 | 2180 | |
| 2229 | − r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__VCN, | |
| 2230 | − &vcn_v4_0_3_aca_info, NULL); | |
| 2231 | − if (r) | |
| 2232 | − goto late_fini; | |
| 2233 | − | |
| 2234 | 2181 | return 0; |
| 2235 | 2182 | |
| 2236 | 2183 | late_fini: |
| @@ -1479,10 +1479,11 @@ static int vcn_v4_0_5_ring_reset(struct amdgpu_ring *ring, · 1478 unmodified lines | ||
| 1479 | 1479 | return amdgpu_ring_reset_helper_end(ring, timedout_fence); |
| 1480 | 1480 | } |
| 1481 | 1481 | |
| 1482 | −static struct amdgpu_ring_funcs vcn_v4_0_5_unified_ring_vm_funcs = { | |
| 1482 | +static const struct amdgpu_ring_funcs vcn_v4_0_5_unified_ring_vm_funcs = { | |
| 1483 | 1483 | .type = AMDGPU_RING_TYPE_VCN_ENC, |
| 1484 | 1484 | .align_mask = 0x3f, |
| 1485 | 1485 | .nop = VCN_ENC_CMD_NO_OP, |
| 1486 | + .secure_submission_supported = true, | |
| 1486 | 1487 | .no_user_fence = true, |
| 1487 | 1488 | .get_rptr = vcn_v4_0_5_unified_ring_get_rptr, |
| 1488 | 1489 | .get_wptr = vcn_v4_0_5_unified_ring_get_wptr, |
| @@ -1525,9 +1526,6 @@ static void vcn_v4_0_5_set_unified_ring_funcs(struct amdgpu_device *adev) · 36 unmodified lines | ||
| 1525 | 1526 | if (adev->vcn.harvest_config & (1 << i)) |
| 1526 | 1527 | continue; |
| 1527 | 1528 | |
| 1528 | − if (amdgpu_ip_version(adev, VCN_HWIP, 0) == IP_VERSION(4, 0, 5)) | |
| 1529 | − vcn_v4_0_5_unified_ring_vm_funcs.secure_submission_supported = true; | |
| 1530 | − | |
| 1531 | 1529 | adev->vcn.inst[i].ring_enc[0].funcs = &vcn_v4_0_5_unified_ring_vm_funcs; |
| 1532 | 1530 | adev->vcn.inst[i].ring_enc[0].me = i; |
| 1533 | 1531 | } |
| @@ -94,6 +94,19 @@ static int vcn_v5_0_1_early_init(struct amdgpu_ip_block *ip_block) · 93 unmodified lines | ||
| 94 | 94 | struct amdgpu_device *adev = ip_block->adev; |
| 95 | 95 | int i, r; |
| 96 | 96 | |
| 97 | + switch (amdgpu_user_queue) { | |
| 98 | + case -1: | |
| 99 | + case 0: | |
| 100 | + default: | |
| 101 | + adev->vcn.disable_kq = false; | |
| 102 | + adev->vcn.disable_uq = true; | |
| 103 | + break; | |
| 104 | + case 2: | |
| 105 | + adev->vcn.disable_kq = true; | |
| 106 | + adev->vcn.disable_uq = true; | |
| 107 | + break; | |
| 108 | + } | |
| 109 | + | |
| 97 | 110 | for (i = 0; i < adev->vcn.num_vcn_inst; ++i) |
| 98 | 111 | /* re-use enc ring as unified ring */ |
| 99 | 112 | adev->vcn.inst[i].num_enc_rings = 1; |
| @@ -188,6 +201,10 @@ static int vcn_v5_0_1_sw_init(struct amdgpu_ip_block *ip_block) · 88 unmodified lines | ||
| 188 | 201 | |
| 189 | 202 | ring = &adev->vcn.inst[i].ring_enc[0]; |
| 190 | 203 | ring->use_doorbell = true; |
| 204 | + if (adev->vcn.disable_kq) { | |
| 205 | + ring->no_scheduler = true; | |
| 206 | + ring->no_user_submission = true; | |
| 207 | + } | |
| 191 | 208 | if (!amdgpu_sriov_vf(adev)) |
| 192 | 209 | ring->doorbell_index = |
| 193 | 210 | (adev->doorbell_index.vcn.vcn_ring0_1 << 1) + |
| @@ -1657,10 +1674,7 @@ static const struct amd_ip_funcs vcn_v5_0_1_ip_funcs = { · 1463 unmodified lines | ||
| 1657 | 1674 | .resume = vcn_v5_0_1_resume, |
| 1658 | 1675 | .is_idle = vcn_v5_0_1_is_idle, |
| 1659 | 1676 | .wait_for_idle = vcn_v5_0_1_wait_for_idle, |
| 1660 | − .check_soft_reset = NULL, | |
| 1661 | − .pre_soft_reset = NULL, | |
| 1662 | 1677 | .soft_reset = NULL, |
| 1663 | − .post_soft_reset = NULL, | |
| 1664 | 1678 | .set_clockgating_state = vcn_v5_0_1_set_clockgating_state, |
| 1665 | 1679 | .set_powergating_state = vcn_set_powergating_state, |
| 1666 | 1680 | .dump_ip_state = amdgpu_vcn_dump_ip_state, |
| @@ -1713,71 +1727,6 @@ static const struct amdgpu_ras_block_hw_ops vcn_v5_0_1_ras_hw_ops = { · 46 unmodified lines | ||
| 1713 | 1727 | .query_poison_status = vcn_v5_0_1_query_poison_status, |
| 1714 | 1728 | }; |
| 1715 | 1729 | |
| 1716 | −static int vcn_v5_0_1_aca_bank_parser(struct aca_handle *handle, struct aca_bank *bank, | |
| 1717 | − enum aca_smu_type type, void *data) | |
| 1718 | −{ | |
| 1719 | − struct aca_bank_info info; | |
| 1720 | − u64 misc0; | |
| 1721 | − int ret; | |
| 1722 | − | |
| 1723 | − ret = aca_bank_info_decode(bank, &info); | |
| 1724 | − if (ret) | |
| 1725 | − return ret; | |
| 1726 | − | |
| 1727 | − misc0 = bank->regs[ACA_REG_IDX_MISC0]; | |
| 1728 | − switch (type) { | |
| 1729 | − case ACA_SMU_TYPE_UE: | |
| 1730 | − bank->aca_err_type = ACA_ERROR_TYPE_UE; | |
| 1731 | − ret = aca_error_cache_log_bank_error(handle, &info, ACA_ERROR_TYPE_UE, | |
| 1732 | − 1ULL); | |
| 1733 | − break; | |
| 1734 | − case ACA_SMU_TYPE_CE: | |
| 1735 | − bank->aca_err_type = ACA_ERROR_TYPE_CE; | |
| 1736 | − ret = aca_error_cache_log_bank_error(handle, &info, bank->aca_err_type, | |
| 1737 | − ACA_REG__MISC0__ERRCNT(misc0)); | |
| 1738 | − break; | |
| 1739 | − default: | |
| 1740 | − return -EINVAL; | |
| 1741 | − } | |
| 1742 | − | |
| 1743 | − return ret; | |
| 1744 | −} | |
| 1745 | − | |
| 1746 | −/* reference to smu driver if header file */ | |
| 1747 | −static int vcn_v5_0_1_err_codes[] = { | |
| 1748 | − 14, 15, 47, /* VCN [D|V|S] */ | |
| 1749 | −}; | |
| 1750 | − | |
| 1751 | −static bool vcn_v5_0_1_aca_bank_is_valid(struct aca_handle *handle, struct aca_bank *bank, | |
| 1752 | − enum aca_smu_type type, void *data) | |
| 1753 | −{ | |
| 1754 | − u32 instlo; | |
| 1755 | − | |
| 1756 | − instlo = ACA_REG__IPID__INSTANCEIDLO(bank->regs[ACA_REG_IDX_IPID]); | |
| 1757 | − instlo &= GENMASK(31, 1); | |
| 1758 | − | |
| 1759 | − if (instlo != mmSMNAID_AID0_MCA_SMU) | |
| 1760 | − return false; | |
| 1761 | − | |
| 1762 | − if (aca_bank_check_error_codes(handle->adev, bank, | |
| 1763 | − vcn_v5_0_1_err_codes, | |
| 1764 | − ARRAY_SIZE(vcn_v5_0_1_err_codes))) | |
| 1765 | − return false; | |
| 1766 | − | |
| 1767 | − return true; | |
| 1768 | −} | |
| 1769 | − | |
| 1770 | −static const struct aca_bank_ops vcn_v5_0_1_aca_bank_ops = { | |
| 1771 | − .aca_bank_parser = vcn_v5_0_1_aca_bank_parser, | |
| 1772 | − .aca_bank_is_valid = vcn_v5_0_1_aca_bank_is_valid, | |
| 1773 | −}; | |
| 1774 | − | |
| 1775 | −static const struct aca_info vcn_v5_0_1_aca_info = { | |
| 1776 | − .hwip = ACA_HWIP_TYPE_SMU, | |
| 1777 | − .mask = ACA_ERROR_UE_MASK, | |
| 1778 | − .bank_ops = &vcn_v5_0_1_aca_bank_ops, | |
| 1779 | −}; | |
| 1780 | − | |
| 1781 | 1730 | static int vcn_v5_0_1_ras_late_init(struct amdgpu_device *adev, struct ras_common_if *ras_block) |
| 1782 | 1731 | { |
| 1783 | 1732 | int r; |
| @@ -1786,11 +1735,6 @@ static int vcn_v5_0_1_ras_late_init(struct amdgpu_device *adev, struct ras_commo · 2 unmodified lines | ||
| 1786 | 1735 | if (r) |
| 1787 | 1736 | return r; |
| 1788 | 1737 | |
| 1789 | − r = amdgpu_ras_bind_aca(adev, AMDGPU_RAS_BLOCK__VCN, | |
| 1790 | − &vcn_v5_0_1_aca_info, NULL); | |
| 1791 | − if (r) | |
| 1792 | − goto late_fini; | |
| 1793 | − | |
| 1794 | 1738 | if (amdgpu_ras_is_supported(adev, ras_block->block) && |
| 1795 | 1739 | adev->vcn.inst->ras_poison_irq.funcs) { |
| 1796 | 1740 | r = amdgpu_irq_get(adev, &adev->vcn.inst->ras_poison_irq, 0); |
| @@ -1203,10 +1203,7 @@ static const struct amd_ip_funcs vcn_v5_0_2_ip_funcs = { · 1202 unmodified lines | ||
| 1203 | 1203 | .resume = vcn_v5_0_2_resume, |
| 1204 | 1204 | .is_idle = vcn_v5_0_2_is_idle, |
| 1205 | 1205 | .wait_for_idle = vcn_v5_0_2_wait_for_idle, |
| 1206 | − .check_soft_reset = NULL, | |
| 1207 | − .pre_soft_reset = NULL, | |
| 1208 | 1206 | .soft_reset = NULL, |
| 1209 | − .post_soft_reset = NULL, | |
| 1210 | 1207 | .set_clockgating_state = vcn_v5_0_2_set_clockgating_state, |
| 1211 | 1208 | .set_powergating_state = vcn_set_powergating_state, |
| 1212 | 1209 | }; |
| @@ -1328,27 +1328,6 @@ static void vi_invalidate_hdp(struct amdgpu_device *adev, · 1327 unmodified lines | ||
| 1328 | 1328 | } |
| 1329 | 1329 | } |
| 1330 | 1330 | |
| 1331 | −static bool vi_need_full_reset(struct amdgpu_device *adev) | |
| 1332 | −{ | |
| 1333 | − switch (adev->asic_type) { | |
| 1334 | − case CHIP_CARRIZO: | |
| 1335 | − case CHIP_STONEY: | |
| 1336 | − /* CZ has hang issues with full reset at the moment */ | |
| 1337 | − return false; | |
| 1338 | − case CHIP_FIJI: | |
| 1339 | − case CHIP_TONGA: | |
| 1340 | − /* XXX: soft reset should work on fiji and tonga */ | |
| 1341 | − return true; | |
| 1342 | − case CHIP_POLARIS10: | |
| 1343 | − case CHIP_POLARIS11: | |
| 1344 | − case CHIP_POLARIS12: | |
| 1345 | − case CHIP_TOPAZ: | |
| 1346 | − default: | |
| 1347 | − /* change this when we support soft reset */ | |
| 1348 | − return true; | |
| 1349 | − } | |
| 1350 | −} | |
| 1351 | − | |
| 1352 | 1331 | static void vi_get_pcie_usage(struct amdgpu_device *adev, uint64_t *count0, |
| 1353 | 1332 | uint64_t *count1) |
| 1354 | 1333 | { |
| @@ -1437,7 +1416,6 @@ static const struct amdgpu_asic_funcs vi_asic_funcs = · 82 unmodified lines | ||
| 1437 | 1416 | .get_config_memsize = &vi_get_config_memsize, |
| 1438 | 1417 | .flush_hdp = &vi_flush_hdp, |
| 1439 | 1418 | .invalidate_hdp = &vi_invalidate_hdp, |
| 1440 | − .need_full_reset = &vi_need_full_reset, | |
| 1441 | 1419 | .init_doorbell_index = &legacy_doorbell_index_init, |
| 1442 | 1420 | .get_pcie_usage = &vi_get_pcie_usage, |
| 1443 | 1421 | .need_reset_on_init = &vi_need_reset_on_init, |
| @@ -1783,9 +1783,6 @@ static int kfd_ptl_control(struct kfd_process_device *pdd, bool enable) · 1782 unmodified lines | ||
| 1783 | 1783 | uint32_t ptl_state = enable ? 1 : 0; |
| 1784 | 1784 | int ret; |
| 1785 | 1785 | |
| 1786 | − if (!ptl->hw_supported) | |
| 1787 | − return -EOPNOTSUPP; | |
| 1788 | − | |
| 1789 | 1786 | if (!pdd->dev->kfd2kgd || !pdd->dev->kfd2kgd->ptl_ctrl) |
| 1790 | 1787 | return -EOPNOTSUPP; |
| 1791 | 1788 | |
| @@ -1804,6 +1801,9 @@ int kfd_ptl_disable_request(struct kfd_process_device *pdd, · 12 unmodified lines | ||
| 1804 | 1801 | struct amdgpu_ptl *ptl = &adev->psp.ptl; |
| 1805 | 1802 | int ret = 0; |
| 1806 | 1803 | |
| 1804 | + if (!ptl->hw_supported) | |
| 1805 | + return -EOPNOTSUPP; | |
| 1806 | + | |
| 1807 | 1807 | mutex_lock(&ptl->mutex); |
| 1808 | 1808 | |
| 1809 | 1809 | if (pdd->ptl_disable_req) |
| @@ -1833,6 +1833,9 @@ int kfd_ptl_disable_release(struct kfd_process_device *pdd, · 23 unmodified lines | ||
| 1833 | 1833 | struct amdgpu_ptl *ptl = &adev->psp.ptl; |
| 1834 | 1834 | int ret = 0; |
| 1835 | 1835 | |
| 1836 | + if (!ptl->hw_supported) | |
| 1837 | + return -EOPNOTSUPP; | |
| 1838 | + | |
| 1836 | 1839 | mutex_lock(&ptl->mutex); |
| 1837 | 1840 | |
| 1838 | 1841 | if (!pdd->ptl_disable_req) |
| @@ -3734,7 +3737,8 @@ static int kfd_mmap(struct file *filep, struct vm_area_struct *vma) · 1895 unmodified lines | ||
| 3734 | 3737 | return kfd_doorbell_mmap(dev, process, vma); |
| 3735 | 3738 | |
| 3736 | 3739 | case KFD_MMAP_TYPE_EVENTS: |
| 3737 | − return kfd_event_mmap(process, vma); | |
| 3740 | + pr_warn("KFD_MMAP_TYPE_EVENTS is no longer supported\n"); | |
| 3741 | + return -EINVAL; | |
| 3738 | 3742 | |
| 3739 | 3743 | case KFD_MMAP_TYPE_RESERVED_MEM: |
| 3740 | 3744 | pr_warn("KFD_MMAP_TYPE_RESERVED_MEM is no longer supported\n"); |
| @@ -1806,6 +1806,30 @@ void kgd2kfd_teardown_processes(struct amdgpu_device *adev) · 1805 unmodified lines | ||
| 1806 | 1806 | cond_resched(); |
| 1807 | 1807 | } |
| 1808 | 1808 | |
| 1809 | +int kgd2kfd_reset_mes_queue(struct kfd_dev *kfd, uint32_t node_id, | |
| 1810 | + int queue_type, int pipe, int queue, | |
| 1811 | + unsigned int db) | |
| 1812 | +{ | |
| 1813 | + struct kfd_node *node; | |
| 1814 | + int ret; | |
| 1815 | + | |
| 1816 | + if (!kfd->init_complete) | |
| 1817 | + return 0; | |
| 1818 | + | |
| 1819 | + if (node_id >= kfd->num_nodes) { | |
| 1820 | + dev_warn(kfd->adev->dev, "Invalid node ID: %u exceeds %u\n", | |
| 1821 | + node_id, kfd->num_nodes - 1); | |
| 1822 | + return -EINVAL; | |
| 1823 | + } | |
| 1824 | + node = kfd->nodes[node_id]; | |
| 1825 | + | |
| 1826 | + ret = kfd_reset_queue_mes(node->dqm, queue_type, pipe, queue, db); | |
| 1827 | + if (ret) | |
| 1828 | + dev_err(kfd_device, "Error resetting queue\n"); | |
| 1829 | + | |
| 1830 | + return ret; | |
| 1831 | +} | |
| 1832 | + | |
| 1809 | 1833 | #if defined(CONFIG_DEBUG_FS) |
| 1810 | 1834 | |
| 1811 | 1835 | /* This function will send a package to HIQ to hang the HWS |
| @@ -37,7 +37,8 @@ · 36 unmodified lines | ||
| 37 | 37 | #include "amdgpu_amdkfd.h" |
| 38 | 38 | #include "amdgpu_reset.h" |
| 39 | 39 | #include "amdgpu_sdma.h" |
| 40 | −#include "mes_v11_api_def.h" | |
| 40 | +#include "amdgpu_ring.h" | |
| 41 | +#include "amdgpu_mes.h" | |
| 41 | 42 | #include "kfd_debug.h" |
| 42 | 43 | |
| 43 | 44 | /* Size of the per-pipe EOP queue */ |
| @@ -71,12 +72,11 @@ static int allocate_sdma_queue(struct device_queue_manager *dqm, · 27 unmodified lines | ||
| 71 | 72 | struct queue *q, const uint32_t *restore_sdma_id); |
| 72 | 73 | |
| 73 | 74 | static int reset_queues_on_hws_hang(struct device_queue_manager *dqm, bool is_sdma); |
| 74 | −static int resume_all_queues_mes(struct device_queue_manager *dqm); | |
| 75 | −static int suspend_all_queues_mes(struct device_queue_manager *dqm); | |
| 76 | 75 | static struct queue *find_queue_by_doorbell_offset(struct device_queue_manager *dqm, |
| 77 | 76 | u32 doorbell_offset); |
| 78 | 77 | static void set_queue_as_reset(struct device_queue_manager *dqm, struct queue *q, |
| 79 | 78 | struct qcm_process_device *qpd); |
| 79 | +static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q); | |
| 80 | 80 | |
| 81 | 81 | static inline |
| 82 | 82 | enum KFD_MQD_TYPE get_mqd_type_from_queue_type(enum kfd_queue_type type) |
| @@ -184,24 +184,24 @@ static void kfd_hws_hang(struct device_queue_manager *dqm) · 101 unmodified lines | ||
| 184 | 184 | amdgpu_amdkfd_gpu_reset(dqm->dev->adev); |
| 185 | 185 | } |
| 186 | 186 | |
| 187 | −static int convert_to_mes_queue_type(int queue_type) | |
| 187 | +static int convert_to_amdgpu_ring_type(int queue_type) | |
| 188 | 188 | { |
| 189 | − int mes_queue_type; | |
| 189 | + int amdgpu_ring_type; | |
| 190 | 190 | |
| 191 | 191 | switch (queue_type) { |
| 192 | 192 | case KFD_QUEUE_TYPE_COMPUTE: |
| 193 | − mes_queue_type = MES_QUEUE_TYPE_COMPUTE; | |
| 193 | + amdgpu_ring_type = AMDGPU_RING_TYPE_COMPUTE; | |
| 194 | 194 | break; |
| 195 | 195 | case KFD_QUEUE_TYPE_SDMA: |
| 196 | − mes_queue_type = MES_QUEUE_TYPE_SDMA; | |
| 196 | + amdgpu_ring_type = AMDGPU_RING_TYPE_SDMA; | |
| 197 | 197 | break; |
| 198 | 198 | default: |
| 199 | 199 | WARN(1, "Invalid queue type %d", queue_type); |
| 200 | − mes_queue_type = -EINVAL; | |
| 200 | + amdgpu_ring_type = -EINVAL; | |
| 201 | 201 | break; |
| 202 | 202 | } |
| 203 | 203 | |
| 204 | − return mes_queue_type; | |
| 204 | + return amdgpu_ring_type; | |
| 205 | 205 | } |
| 206 | 206 | |
| 207 | 207 | static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, |
| @@ -251,7 +251,7 @@ static int add_queue_mes(struct device_queue_manager *dqm, struct queue *q, · 43 unmodified lines | ||
| 251 | 251 | (qpd->pqm->process->debug_trap_enabled || |
| 252 | 252 | kfd_dbg_has_ttmps_always_setup(q->device)); |
| 253 | 253 | |
| 254 | − queue_type = convert_to_mes_queue_type(q->properties.type); | |
| 254 | + queue_type = convert_to_amdgpu_ring_type(q->properties.type); | |
| 255 | 255 | if (queue_type < 0) { |
| 256 | 256 | dev_err(adev->dev, "Queue type not supported with MES, queue:%d\n", |
| 257 | 257 | q->properties.type); |
| @@ -300,6 +300,7 @@ static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, st · 42 unmodified lines | ||
| 300 | 300 | memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); |
| 301 | 301 | queue_input.doorbell_offset = q->properties.doorbell_off; |
| 302 | 302 | queue_input.gang_context_addr = q->gang_ctx_gpu_addr; |
| 303 | + queue_input.queue_type = convert_to_amdgpu_ring_type(q->properties.type); | |
| 303 | 304 | queue_input.remove_queue_after_reset = flush_mes_queue; |
| 304 | 305 | queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1; |
| 305 | 306 | |
| @@ -308,14 +309,14 @@ static int remove_queue_mes_on_reset_option(struct device_queue_manager *dqm, st · 2 unmodified lines | ||
| 308 | 309 | amdgpu_mes_unlock(&adev->mes); |
| 309 | 310 | up_read(&adev->reset_domain->sem); |
| 310 | 311 | |
| 311 | − if (is_for_reset) | |
| 312 | + /* If is_for_reset set, it is a mes internal cleanup */ | |
| 313 | + if (!r || is_for_reset) | |
| 312 | 314 | return r; |
| 313 | 315 | |
| 314 | − if (r) { | |
| 315 | − if (!suspend_all_queues_mes(dqm)) | |
| 316 | − return resume_all_queues_mes(dqm); | |
| 317 | − | |
| 318 | − dev_err(adev->dev, "failed to remove hardware queue from MES, doorbell=0x%x\n", | |
| 316 | + /* remove_hw_queue failure indicates a queue hang. reset the queue */ | |
| 317 | + r = reset_queues_mes(dqm, q); | |
| 318 | + if (r && amdgpu_gpu_recovery) { | |
| 319 | + dev_err(adev->dev, "failed to remove queue from MES, doorbell=0x%x\n", | |
| 319 | 320 | q->properties.doorbell_off); |
| 320 | 321 | dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); |
| 321 | 322 | kfd_hws_hang(dqm); |
| @@ -407,16 +408,50 @@ static int add_all_kfd_queues_mes(struct device_queue_manager *dqm) · 85 unmodified lines | ||
| 407 | 408 | return retval; |
| 408 | 409 | } |
| 409 | 410 | |
| 410 | −static int reset_queues_mes(struct device_queue_manager *dqm) | |
| 411 | +static int reset_queue_mes(struct device_queue_manager *dqm, struct queue *q, | |
| 412 | + int queue_type, int pipe, int queue, unsigned int db) | |
| 411 | 413 | { |
| 412 | 414 | struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; |
| 413 | − int hqd_info_size = adev->mes.hung_queue_hqd_info_offset; | |
| 414 | − int num_hung = 0, r = 0, i, pipe, queue, queue_type; | |
| 415 | − u32 *hung_array = dqm->hung_db_array; | |
| 416 | − struct amdgpu_mes_hung_queue_hqd_info *hqd_info = dqm->hqd_info; | |
| 417 | 415 | struct kfd_process_device *pdd; |
| 416 | + bool use_mmio = adev->gfx.mec.use_mmio_for_reset; | |
| 417 | + int r; | |
| 418 | + | |
| 419 | + pdd = kfd_get_process_device_data(q->device, q->process); | |
| 420 | + if (!pdd) | |
| 421 | + return -ENODEV; | |
| 422 | + | |
| 423 | + if (use_mmio) | |
| 424 | + r = amdgpu_mes_reset_queue_mmio(adev, queue_type, 0, 1, pipe, queue, | |
| 425 | + ffs(dqm->dev->xcc_mask) - 1); | |
| 426 | + else | |
| 427 | + r = amdgpu_mes_reset_user_queue(adev, queue_type, db, | |
| 428 | + ffs(dqm->dev->xcc_mask) - 1); | |
| 429 | + if (r) | |
| 430 | + return r; | |
| 431 | + /* Proceed remove_queue with reset=true */ | |
| 432 | + remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, true); | |
| 433 | + set_queue_as_reset(dqm, q, &pdd->qpd); | |
| 434 | + return 0; | |
| 435 | +} | |
| 436 | + | |
| 437 | +int kfd_reset_queue_mes(struct device_queue_manager *dqm, int queue_type, | |
| 438 | + int pipe, int queue, unsigned int db) | |
| 439 | +{ | |
| 418 | 440 | struct queue *q; |
| 419 | 441 | |
| 442 | + q = find_queue_by_doorbell_offset(dqm, db); | |
| 443 | + if (!q) | |
| 444 | + return 0; | |
| 445 | + return reset_queue_mes(dqm, q, queue_type, pipe, queue, db); | |
| 446 | +} | |
| 447 | + | |
| 448 | +static int reset_queues_mes(struct device_queue_manager *dqm, struct queue *q) | |
| 449 | +{ | |
| 450 | + struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; | |
| 451 | + unsigned int num_hung = 0; | |
| 452 | + int r = 0; | |
| 453 | + struct mes_remove_queue_input queue_input; | |
| 454 | + | |
| 420 | 455 | if (!amdgpu_mes_queue_reset_by_mes_supported(adev)) { |
| 421 | 456 | r = -ENOTRECOVERABLE; |
| 422 | 457 | goto fail; |
| @@ -431,111 +466,29 @@ static int reset_queues_mes(struct device_queue_manager *dqm) · 8 unmodified lines | ||
| 431 | 466 | goto fail; |
| 432 | 467 | } |
| 433 | 468 | |
| 434 | − if (!hung_array || !hqd_info) { | |
| 435 | − r = -ENOMEM; | |
| 436 | − goto fail; | |
| 437 | − } | |
| 438 | − | |
| 439 | − memset(hqd_info, 0, hqd_info_size * sizeof(struct amdgpu_mes_hung_queue_hqd_info)); | |
| 440 | − | |
| 441 | − /* | |
| 442 | − * AMDGPU_RING_TYPE_COMPUTE parameter does not matter if called | |
| 443 | − * post suspend_all as reset & detect will return all hung queue types. | |
| 444 | − * | |
| 445 | − * Passed parameter is for targeting queues not scheduled by MES add_queue. | |
| 446 | − */ | |
| 447 | − r = amdgpu_mes_detect_and_reset_hung_queues(adev, AMDGPU_RING_TYPE_COMPUTE, | |
| 448 | − false, &num_hung, hung_array, ffs(dqm->dev->xcc_mask) - 1); | |
| 449 | − | |
| 450 | − if (!num_hung || r) { | |
| 451 | − r = -ENOTRECOVERABLE; | |
| 469 | + memset(&queue_input, 0x0, sizeof(struct mes_remove_queue_input)); | |
| 470 | + queue_input.doorbell_offset = q->properties.doorbell_off; | |
| 471 | + queue_input.gang_context_addr = q->gang_ctx_gpu_addr; | |
| 472 | + queue_input.queue_type = convert_to_amdgpu_ring_type(q->properties.type); | |
| 473 | + queue_input.remove_queue_after_reset = false; | |
| 474 | + queue_input.xcc_id = ffs(dqm->dev->xcc_mask) - 1; | |
| 475 | + /* pass the known bad queue info to the reset function */ | |
| 476 | + r = amdgpu_gfx_reset_mes_compute(adev, NULL, NULL, NULL, &num_hung, &queue_input); | |
| 477 | + if (r) | |
| 452 | 478 | goto fail; |
| 453 | − } | |
| 454 | − | |
| 455 | − /* MES resets queue/pipe and cleans up internally */ | |
| 456 | − for (i = 0; i < num_hung; i++) { | |
| 457 | − hqd_info[i].bit0_31 = hung_array[i + hqd_info_size]; | |
| 458 | − pipe = hqd_info[i].pipe_index; | |
| 459 | − queue = hqd_info[i].queue_index; | |
| 460 | − queue_type = hqd_info[i].queue_type; | |
| 461 | − | |
| 462 | − if (queue_type != MES_QUEUE_TYPE_COMPUTE && | |
| 463 | − queue_type != MES_QUEUE_TYPE_SDMA) { | |
| 464 | − pr_warn("Unsupported hung queue reset type: %d\n", queue_type); | |
| 465 | − hung_array[i] = AMDGPU_MES_INVALID_DB_OFFSET; | |
| 466 | − continue; | |
| 467 | − } | |
| 468 | − | |
| 469 | − q = find_queue_by_doorbell_offset(dqm, hung_array[i]); | |
| 470 | − if (!q) { | |
| 471 | − r = -ENOTRECOVERABLE; | |
| 472 | − goto fail; | |
| 473 | − } | |
| 474 | − | |
| 475 | − pdd = kfd_get_process_device_data(q->device, q->process); | |
| 476 | − if (!pdd) { | |
| 477 | − r = -ENODEV; | |
| 478 | − goto fail; | |
| 479 | − } | |
| 480 | − | |
| 481 | − pr_warn("Hang detected doorbell %x pipe %d queue %d type %d\n", | |
| 482 | − hung_array[i], pipe, queue, queue_type); | |
| 483 | − /* Proceed remove_queue with reset=true */ | |
| 484 | − remove_queue_mes_on_reset_option(dqm, q, &pdd->qpd, true, false); | |
| 485 | − set_queue_as_reset(dqm, q, &pdd->qpd); | |
| 486 | − } | |
| 487 | 479 | |
| 488 | 480 | dqm->detect_hang_count = num_hung; |
| 489 | − kfd_signal_reset_event(dqm->dev); | |
| 481 | + /* When MES doesn't detect any queue hang, no reset happens. Don't signal reset | |
| 482 | + * event. | |
| 483 | + */ | |
| 484 | + if (dqm->detect_hang_count) | |
| 485 | + kfd_signal_reset_event(dqm->dev); | |
| 490 | 486 | |
| 491 | 487 | fail: |
| 492 | 488 | dqm->detect_hang_count = 0; |
| 493 | 489 | return r; |
| 494 | 490 | } |
| 495 | 491 | |
| 496 | −static int suspend_all_queues_mes(struct device_queue_manager *dqm) | |
| 497 | −{ | |
| 498 | − struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; | |
| 499 | − int r = 0; | |
| 500 | − | |
| 501 | − if (!down_read_trylock(&adev->reset_domain->sem)) | |
| 502 | − return -EIO; | |
| 503 | − | |
| 504 | − r = amdgpu_mes_suspend(adev, ffs(dqm->dev->xcc_mask) - 1); | |
| 505 | − up_read(&adev->reset_domain->sem); | |
| 506 | − | |
| 507 | − if (r) { | |
| 508 | − if (!reset_queues_mes(dqm)) | |
| 509 | − return 0; | |
| 510 | − | |
| 511 | − dev_err(adev->dev, "failed to suspend gangs from MES\n"); | |
| 512 | − dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); | |
| 513 | − kfd_hws_hang(dqm); | |
| 514 | − } | |
| 515 | − | |
| 516 | − return r; | |
| 517 | −} | |
| 518 | − | |
| 519 | −static int resume_all_queues_mes(struct device_queue_manager *dqm) | |
| 520 | −{ | |
| 521 | − struct amdgpu_device *adev = (struct amdgpu_device *)dqm->dev->adev; | |
| 522 | − int r = 0; | |
| 523 | − | |
| 524 | − if (!down_read_trylock(&adev->reset_domain->sem)) | |
| 525 | − return -EIO; | |
| 526 | − | |
| 527 | − r = amdgpu_mes_resume(adev, ffs(dqm->dev->xcc_mask) - 1); | |
| 528 | − up_read(&adev->reset_domain->sem); | |
| 529 | − | |
| 530 | − if (r) { | |
| 531 | − dev_err(adev->dev, "failed to resume gangs from MES\n"); | |
| 532 | − dev_err(adev->dev, "MES might be in unrecoverable state, issue a GPU reset\n"); | |
| 533 | − kfd_hws_hang(dqm); | |
| 534 | − } | |
| 535 | − | |
| 536 | − return r; | |
| 537 | −} | |
| 538 | − | |
| 539 | 492 | static void increment_queue_count(struct device_queue_manager *dqm, |
| 540 | 493 | struct qcm_process_device *qpd, |
| 541 | 494 | struct queue *q) |
| @@ -1064,8 +1017,15 @@ static int destroy_queue_nocpsch(struct device_queue_manager *dqm, · 522 unmodified lines | ||
| 1064 | 1017 | /* Get the SDMA queue stats */ |
| 1065 | 1018 | if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || |
| 1066 | 1019 | (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { |
| 1067 | − retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, | |
| 1068 | − &sdma_val); | |
| 1020 | + if (dqm->dev->kfd2kgd->hqd_sdma_get_counter) | |
| 1021 | + retval = dqm->dev->kfd2kgd->hqd_sdma_get_counter( | |
| 1022 | + dqm->dev->adev, q->mqd, | |
| 1023 | + dqm->dev->kfd->device_info.num_sdma_queues_per_engine, | |
| 1024 | + &sdma_val); | |
| 1025 | + else | |
| 1026 | + retval = read_sdma_queue_counter( | |
| 1027 | + (uint64_t __user *)q->properties.read_ptr, | |
| 1028 | + &sdma_val); | |
| 1069 | 1029 | if (retval) |
| 1070 | 1030 | dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", |
| 1071 | 1031 | q->properties.queue_id); |
| @@ -2703,8 +2663,16 @@ static int destroy_queue_cpsch(struct device_queue_manager *dqm, · 1631 unmodified lines | ||
| 2703 | 2663 | /* Get the SDMA queue stats */ |
| 2704 | 2664 | if ((q->properties.type == KFD_QUEUE_TYPE_SDMA) || |
| 2705 | 2665 | (q->properties.type == KFD_QUEUE_TYPE_SDMA_XGMI)) { |
| 2706 | − retval = read_sdma_queue_counter((uint64_t __user *)q->properties.read_ptr, | |
| 2707 | − &sdma_val); | |
| 2666 | + if (dqm->dev->kfd2kgd->hqd_sdma_get_counter) | |
| 2667 | + retval = dqm->dev->kfd2kgd->hqd_sdma_get_counter( | |
| 2668 | + dqm->dev->adev, q->mqd, | |
| 2669 | + dqm->dev->kfd->device_info.num_sdma_queues_per_engine, | |
| 2670 | + &sdma_val); | |
| 2671 | + else | |
| 2672 | + retval = read_sdma_queue_counter( | |
| 2673 | + (uint64_t __user *)q->properties.read_ptr, | |
| 2674 | + &sdma_val); | |
| 2675 | + | |
| 2708 | 2676 | if (retval) |
| 2709 | 2677 | dev_err(dev, "Failed to read SDMA queue counter for queue: %d\n", |
| 2710 | 2678 | q->properties.queue_id); |
| @@ -3244,12 +3212,12 @@ void device_queue_manager_uninit(struct device_queue_manager *dqm) · 533 unmodified lines | ||
| 3244 | 3212 | kfree(dqm); |
| 3245 | 3213 | } |
| 3246 | 3214 | |
| 3215 | +/* bad queue notified by interrupt from CP */ | |
| 3247 | 3216 | int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbell_id) |
| 3248 | 3217 | { |
| 3249 | 3218 | struct kfd_process_device *pdd = NULL; |
| 3250 | 3219 | struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, &pdd); |
| 3251 | 3220 | struct device_queue_manager *dqm = knode->dqm; |
| 3252 | − struct device *dev = dqm->dev->adev->dev; | |
| 3253 | 3221 | struct qcm_process_device *qpd; |
| 3254 | 3222 | struct queue *q = NULL; |
| 3255 | 3223 | int ret = 0; |
| @@ -3264,19 +3232,10 @@ int kfd_dqm_suspend_bad_queue_mes(struct kfd_node *knode, u32 pasid, u32 doorbel · 8 unmodified lines | ||
| 3264 | 3232 | |
| 3265 | 3233 | list_for_each_entry(q, &qpd->queues_list, list) { |
| 3266 | 3234 | if (q->doorbell_id == doorbell_id && q->properties.is_active) { |
| 3267 | − /* suspend all queues will save any good queues and mark the rest as bad */ | |
| 3268 | − suspend_all_queues_mes(dqm); | |
| 3269 | − | |
| 3235 | + reset_queues_mes(dqm, q); | |
| 3270 | 3236 | q->properties.is_evicted = true; |
| 3271 | 3237 | q->properties.is_active = false; |
| 3272 | 3238 | decrement_queue_count(dqm, qpd, q); |
| 3273 | − | |
| 3274 | − /* this will remove the bad queue and sched a GPU reset if needed */ | |
| 3275 | − ret = remove_queue_mes(dqm, q, qpd); | |
| 3276 | − if (ret) | |
| 3277 | − dev_err(dev, "Removing bad queue failed"); | |
| 3278 | − /* resume the good queues */ | |
| 3279 | − resume_all_queues_mes(dqm); | |
| 3280 | 3239 | break; |
| 3281 | 3240 | } |
| 3282 | 3241 | } |
| @@ -333,6 +333,8 @@ int debug_refresh_runlist(struct device_queue_manager *dqm); · 332 unmodified lines | ||
| 333 | 333 | bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm, |
| 334 | 334 | struct qcm_process_device *qpd, |
| 335 | 335 | int doorbell_off, u32 *queue_format); |
| 336 | +int kfd_reset_queue_mes(struct device_queue_manager *dqm, int queue_type, | |
| 337 | + int pipe, int queue, unsigned int db); | |
| 336 | 338 | |
| 337 | 339 | static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) |
| 338 | 340 | { |
| @@ -29,10 +29,12 @@ · 28 unmodified lines | ||
| 29 | 29 | #include <linux/uaccess.h> |
| 30 | 30 | #include <linux/mman.h> |
| 31 | 31 | #include <linux/memory.h> |
| 32 | +#include <linux/workqueue.h> | |
| 32 | 33 | #include "kfd_priv.h" |
| 33 | 34 | #include "kfd_events.h" |
| 34 | 35 | #include "kfd_device_queue_manager.h" |
| 35 | 36 | #include <linux/device.h> |
| 37 | +#include <drm/amdgpu_drm.h> | |
| 36 | 38 | |
| 37 | 39 | /* |
| 38 | 40 | * Wrapper around wait_queue_entry_t |
| @@ -44,67 +46,19 @@ struct kfd_event_waiter { · 5 unmodified lines | ||
| 44 | 46 | bool event_age_enabled; /* set to true when last_event_age is non-zero */ |
| 45 | 47 | }; |
| 46 | 48 | |
| 47 | −/* | |
| 48 | − * Each signal event needs a 64-bit signal slot where the signaler will write | |
| 49 | − * a 1 before sending an interrupt. (This is needed because some interrupts | |
| 50 | − * do not contain enough spare data bits to identify an event.) | |
| 51 | − * We get whole pages and map them to the process VA. | |
| 52 | − * Individual signal events use their event_id as slot index. | |
| 53 | − */ | |
| 54 | −struct kfd_signal_page { | |
| 55 | − uint64_t *kernel_address; | |
| 56 | − uint64_t __user *user_address; | |
| 57 | − bool need_to_free_pages; | |
| 58 | −}; | |
| 59 | − | |
| 60 | −static uint64_t *page_slots(struct kfd_signal_page *page) | |
| 61 | −{ | |
| 62 | − return page->kernel_address; | |
| 63 | −} | |
| 64 | − | |
| 65 | −static struct kfd_signal_page *allocate_signal_page(struct kfd_process *p) | |
| 66 | −{ | |
| 67 | − void *backing_store; | |
| 68 | − struct kfd_signal_page *page; | |
| 69 | − | |
| 70 | − page = kzalloc_obj(*page); | |
| 71 | − if (!page) | |
| 72 | − return NULL; | |
| 73 | − | |
| 74 | − backing_store = (void *) __get_free_pages(GFP_KERNEL, | |
| 75 | − get_order(KFD_SIGNAL_EVENT_LIMIT * 8)); | |
| 76 | − if (!backing_store) | |
| 77 | − goto fail_alloc_signal_store; | |
| 78 | − | |
| 79 | − /* Initialize all events to unsignaled */ | |
| 80 | − memset(backing_store, (uint8_t) UNSIGNALED_EVENT_SLOT, | |
| 81 | − KFD_SIGNAL_EVENT_LIMIT * 8); | |
| 82 | − | |
| 83 | − page->kernel_address = backing_store; | |
| 84 | − page->need_to_free_pages = true; | |
| 85 | − pr_debug("Allocated new event signal page at %p, for process %p\n", | |
| 86 | − page, p); | |
| 87 | − | |
| 88 | − return page; | |
| 89 | − | |
| 90 | −fail_alloc_signal_store: | |
| 91 | − kfree(page); | |
| 92 | − return NULL; | |
| 93 | −} | |
| 94 | − | |
| 95 | 49 | static int allocate_event_notification_slot(struct kfd_process *p, |
| 96 | 50 | struct kfd_event *ev, |
| 97 | 51 | const int *restore_id) |
| 98 | 52 | { |
| 99 | 53 | int id; |
| 100 | 54 | |
| 101 | − if (!p->signal_page) { | |
| 102 | − p->signal_page = allocate_signal_page(p); | |
| 103 | − if (!p->signal_page) | |
| 104 | − return -ENOMEM; | |
| 105 | − /* Oldest user mode expects 256 event slots */ | |
| 106 | − p->signal_mapped_size = 256*8; | |
| 107 | − } | |
| 55 | + /* | |
| 56 | + * The signal page is allocated in user mode and mapped to the kernel | |
| 57 | + * via the event_page_offset of the create event IOCTL. Without it no | |
| 58 | + * signal events can be created. | |
| 59 | + */ | |
| 60 | + if (!p->signal_page) | |
| 61 | + return -ENOMEM; | |
| 108 | 62 | |
| 109 | 63 | if (restore_id) { |
| 110 | 64 | id = idr_alloc(&p->event_idr, ev, *restore_id, *restore_id + 1, |
| @@ -123,7 +77,7 @@ static int allocate_event_notification_slot(struct kfd_process *p, · 12 unmodified lines | ||
| 123 | 77 | return id; |
| 124 | 78 | |
| 125 | 79 | ev->event_id = id; |
| 126 | − page_slots(p->signal_page)[id] = UNSIGNALED_EVENT_SLOT; | |
| 80 | + p->signal_page[id] = UNSIGNALED_EVENT_SLOT; | |
| 127 | 81 | |
| 128 | 82 | return 0; |
| 129 | 83 | } |
| @@ -169,7 +123,7 @@ static struct kfd_event *lookup_signaled_event_by_partial_id( · 39 unmodified lines | ||
| 169 | 123 | */ |
| 170 | 124 | if (bits > 31 || (1U << bits) >= KFD_SIGNAL_EVENT_LIMIT) { |
| 171 | 125 | if (signal_mailbox_updated && |
| 172 | − page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT) | |
| 126 | + p->signal_page[id] == UNSIGNALED_EVENT_SLOT) | |
| 173 | 127 | return NULL; |
| 174 | 128 | |
| 175 | 129 | return idr_find(&p->event_idr, id); |
| @@ -179,7 +133,7 @@ static struct kfd_event *lookup_signaled_event_by_partial_id( · 3 unmodified lines | ||
| 179 | 133 | * and find the first one that has signaled. |
| 180 | 134 | */ |
| 181 | 135 | for (ev = NULL; id < KFD_SIGNAL_EVENT_LIMIT && !ev; id += 1U << bits) { |
| 182 | − if (page_slots(p->signal_page)[id] == UNSIGNALED_EVENT_SLOT) | |
| 136 | + if (p->signal_page[id] == UNSIGNALED_EVENT_SLOT) | |
| 183 | 137 | continue; |
| 184 | 138 | |
| 185 | 139 | ev = idr_find(&p->event_idr, id); |
| @@ -210,10 +164,8 @@ static int create_signal_event(struct file *devkfd, struct kfd_process *p, · 24 unmodified lines | ||
| 210 | 164 | |
| 211 | 165 | p->signal_event_count++; |
| 212 | 166 | |
| 213 | − ev->user_signal_address = &p->signal_page->user_address[ev->event_id]; | |
| 214 | − pr_debug("Signal event number %zu created with id %d, address %p\n", | |
| 215 | − p->signal_event_count, ev->event_id, | |
| 216 | − ev->user_signal_address); | |
| 167 | + pr_debug("Signal event number %zu created with id %d\n", | |
| 168 | + p->signal_event_count, ev->event_id); | |
| 217 | 169 | |
| 218 | 170 | return 0; |
| 219 | 171 | } |
| @@ -293,26 +245,9 @@ static void destroy_events(struct kfd_process *p) · 73 unmodified lines | ||
| 293 | 245 | mutex_destroy(&p->event_mutex); |
| 294 | 246 | } |
| 295 | 247 | |
| 296 | −/* | |
| 297 | − * We assume that the process is being destroyed and there is no need to | |
| 298 | − * unmap the pages or keep bookkeeping data in order. | |
| 299 | − */ | |
| 300 | −static void shutdown_signal_page(struct kfd_process *p) | |
| 301 | −{ | |
| 302 | − struct kfd_signal_page *page = p->signal_page; | |
| 303 | − | |
| 304 | − if (page) { | |
| 305 | − if (page->need_to_free_pages) | |
| 306 | − free_pages((unsigned long)page->kernel_address, | |
| 307 | − get_order(KFD_SIGNAL_EVENT_LIMIT * 8)); | |
| 308 | − kfree(page); | |
| 309 | − } | |
| 310 | −} | |
| 311 | − | |
| 312 | 248 | void kfd_event_free_process(struct kfd_process *p) |
| 313 | 249 | { |
| 314 | 250 | destroy_events(p); |
| 315 | − shutdown_signal_page(p); | |
| 316 | 251 | } |
| 317 | 252 | |
| 318 | 253 | static bool event_can_be_gpu_signaled(const struct kfd_event *ev) |
| @@ -329,8 +264,6 @@ static bool event_can_be_cpu_signaled(const struct kfd_event *ev) · 10 unmodified lines | ||
| 329 | 264 | static int kfd_event_page_set(struct kfd_process *p, void *kernel_address, |
| 330 | 265 | uint64_t size, uint64_t user_handle) |
| 331 | 266 | { |
| 332 | − struct kfd_signal_page *page; | |
| 333 | − | |
| 334 | 267 | if (p->signal_page) |
| 335 | 268 | return -EBUSY; |
| 336 | 269 | |
| @@ -340,17 +273,11 @@ static int kfd_event_page_set(struct kfd_process *p, void *kernel_address, · 3 unmodified lines | ||
| 340 | 273 | return -EINVAL; |
| 341 | 274 | } |
| 342 | 275 | |
| 343 | − page = kzalloc_obj(*page); | |
| 344 | − if (!page) | |
| 345 | − return -ENOMEM; | |
| 346 | − | |
| 347 | 276 | /* Initialize all events to unsignaled */ |
| 348 | 277 | memset(kernel_address, (uint8_t) UNSIGNALED_EVENT_SLOT, |
| 349 | 278 | KFD_SIGNAL_EVENT_LIMIT * 8); |
| 350 | 279 | |
| 351 | − page->kernel_address = kernel_address; | |
| 352 | − | |
| 353 | − p->signal_page = page; | |
| 280 | + p->signal_page = kernel_address; | |
| 354 | 281 | p->signal_mapped_size = size; |
| 355 | 282 | p->signal_handle = user_handle; |
| 356 | 283 | return 0; |
| @@ -387,7 +314,8 @@ int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset) · 30 unmodified lines | ||
| 387 | 314 | return -EINVAL; |
| 388 | 315 | } |
| 389 | 316 | |
| 390 | − err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(mem, &kern_addr, &size); | |
| 317 | + err = amdgpu_amdkfd_gpuvm_map_bo_to_kernel(mem, &kern_addr, &size, | |
| 318 | + AMDGPU_GEM_DOMAIN_GTT); | |
| 391 | 319 | if (err) { |
| 392 | 320 | pr_err("Failed to map event page to kernel\n"); |
| 393 | 321 | return err; |
| @@ -396,7 +324,7 @@ int kfd_kmap_event_page(struct kfd_process *p, uint64_t event_page_offset) · 2 unmodified lines | ||
| 396 | 324 | err = kfd_event_page_set(p, kern_addr, size, event_page_offset); |
| 397 | 325 | if (err) { |
| 398 | 326 | pr_err("Failed to set event page\n"); |
| 399 | − amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem); | |
| 327 | + amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem); | |
| 400 | 328 | return err; |
| 401 | 329 | } |
| 402 | 330 | return err; |
| @@ -717,7 +645,7 @@ unlock_rcu: · 314 unmodified lines | ||
| 717 | 645 | |
| 718 | 646 | static void acknowledge_signal(struct kfd_process *p, struct kfd_event *ev) |
| 719 | 647 | { |
| 720 | − WRITE_ONCE(page_slots(p->signal_page)[ev->event_id], UNSIGNALED_EVENT_SLOT); | |
| 648 | + WRITE_ONCE(p->signal_page[ev->event_id], UNSIGNALED_EVENT_SLOT); | |
| 721 | 649 | } |
| 722 | 650 | |
| 723 | 651 | static void set_event_from_interrupt(struct kfd_process *p, |
| @@ -760,7 +688,7 @@ void kfd_signal_event_interrupt(u32 pasid, uint32_t partial_id, · 36 unmodified lines | ||
| 760 | 688 | * in the interrupt payload was invalid and do an |
| 761 | 689 | * exhaustive search of signaled events. |
| 762 | 690 | */ |
| 763 | − uint64_t *slots = page_slots(p->signal_page); | |
| 691 | + uint64_t *slots = p->signal_page; | |
| 764 | 692 | uint32_t id; |
| 765 | 693 | |
| 766 | 694 | if (valid_id_bits) |
| @@ -1068,51 +996,6 @@ out: · 301 unmodified lines | ||
| 1068 | 996 | return ret; |
| 1069 | 997 | } |
| 1070 | 998 | |
| 1071 | −int kfd_event_mmap(struct kfd_process *p, struct vm_area_struct *vma) | |
| 1072 | −{ | |
| 1073 | − unsigned long pfn; | |
| 1074 | − struct kfd_signal_page *page; | |
| 1075 | − int ret; | |
| 1076 | − | |
| 1077 | − /* check required size doesn't exceed the allocated size */ | |
| 1078 | − if (get_order(KFD_SIGNAL_EVENT_LIMIT * 8) < | |
| 1079 | − get_order(vma->vm_end - vma->vm_start)) { | |
| 1080 | − pr_err("Event page mmap requested illegal size\n"); | |
| 1081 | − return -EINVAL; | |
| 1082 | − } | |
| 1083 | − | |
| 1084 | − page = p->signal_page; | |
| 1085 | − if (!page) { | |
| 1086 | − /* Probably KFD bug, but mmap is user-accessible. */ | |
| 1087 | − pr_debug("Signal page could not be found\n"); | |
| 1088 | − return -EINVAL; | |
| 1089 | − } | |
| 1090 | − | |
| 1091 | − pfn = __pa(page->kernel_address); | |
| 1092 | − pfn >>= PAGE_SHIFT; | |
| 1093 | − | |
| 1094 | − vm_flags_set(vma, VM_IO | VM_DONTCOPY | VM_DONTEXPAND | VM_NORESERVE | |
| 1095 | − | VM_DONTDUMP | VM_PFNMAP); | |
| 1096 | − | |
| 1097 | − pr_debug("Mapping signal page\n"); | |
| 1098 | − pr_debug(" start user address == 0x%08lx\n", vma->vm_start); | |
| 1099 | − pr_debug(" end user address == 0x%08lx\n", vma->vm_end); | |
| 1100 | − pr_debug(" pfn == 0x%016lX\n", pfn); | |
| 1101 | − pr_debug(" vm_flags == 0x%08lX\n", vma->vm_flags); | |
| 1102 | − pr_debug(" size == 0x%08lX\n", | |
| 1103 | − vma->vm_end - vma->vm_start); | |
| 1104 | − | |
| 1105 | − page->user_address = (uint64_t __user *)vma->vm_start; | |
| 1106 | − | |
| 1107 | − /* mapping the page to user process */ | |
| 1108 | − ret = remap_pfn_range(vma, vma->vm_start, pfn, | |
| 1109 | − vma->vm_end - vma->vm_start, vma->vm_page_prot); | |
| 1110 | − if (!ret) | |
| 1111 | − p->signal_mapped_size = vma->vm_end - vma->vm_start; | |
| 1112 | − | |
| 1113 | − return ret; | |
| 1114 | −} | |
| 1115 | − | |
| 1116 | 999 | /* |
| 1117 | 1000 | * Assumes that p is not going away. |
| 1118 | 1001 | */ |
| @@ -1338,6 +1221,71 @@ void kfd_signal_reset_event(struct kfd_node *dev) · 219 unmodified lines | ||
| 1338 | 1221 | srcu_read_unlock(&kfd_processes_srcu, idx); |
| 1339 | 1222 | } |
| 1340 | 1223 | |
| 1224 | +/* | |
| 1225 | + * Per-process opt-in for poison-consumption SIGBUS handling. | |
| 1226 | + * | |
| 1227 | + * Default: kernel sends SIGBUS to the process immediately when poison is | |
| 1228 | + * consumed, in addition to delivering the KFD HW/MEMORY exception events. | |
| 1229 | + * | |
| 1230 | + * Userspace (ROCr) can opt-in per-process via the | |
| 1231 | + * DRM_IOCTL_AMDGPU_PROC_OPTIONS / AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY | |
| 1232 | + * option. This lets the app's registered system-event callback handle the | |
| 1233 | + * RAS error first, instead of being killed by SIGBUS. | |
| 1234 | + * | |
| 1235 | + * Encoded value (stored on the kfd_process): | |
| 1236 | + * 0 - default: SIGBUS immediately (no opt-in) | |
| 1237 | + * 0xFFFFFFFF - opt-in, never escalate to SIGBUS | |
| 1238 | + * N (other) - opt-in, escalate to SIGBUS after N ms if app does not | |
| 1239 | + * handle the error in time (safety timeout) | |
| 1240 | + */ | |
| 1241 | + | |
| 1242 | +void kfd_signal_sigbus_delayed_fn(struct work_struct *work) | |
| 1243 | +{ | |
| 1244 | + struct kfd_process *p = container_of(to_delayed_work(work), | |
| 1245 | + struct kfd_process, signal_work); | |
| 1246 | + | |
| 1247 | + if (p->lead_thread) | |
| 1248 | + send_sig(SIGBUS, p->lead_thread, 0); | |
| 1249 | + | |
| 1250 | + kfd_unref_process(p); | |
| 1251 | +} | |
| 1252 | + | |
| 1253 | +static void kfd_signal_sigbus_with_delay(struct kfd_node *dev, | |
| 1254 | + struct kfd_process *p) | |
| 1255 | +{ | |
| 1256 | + u32 delay_ms = atomic_read(&p->kfd_sigbus_delay_ms); | |
| 1257 | + | |
| 1258 | + if (delay_ms == AMDGPU_PROC_OPTIONS_KFD_SIGBUS_DELAY_DISABLED) { | |
| 1259 | + dev_info(dev->adev->dev, | |
| 1260 | + "SIGBUS suppressed for process %s(pid:%d): app opted in to handle RAS error\n", | |
| 1261 | + p->lead_thread->comm, p->lead_thread->pid); | |
| 1262 | + return; | |
| 1263 | + } | |
| 1264 | + | |
| 1265 | + if (delay_ms == 0) | |
| 1266 | + goto send_now; | |
| 1267 | + | |
| 1268 | + /* | |
| 1269 | + * Take an extra reference for the delayed worker. If the work is | |
| 1270 | + * already pending (e.g. another device of this process consumed poison | |
| 1271 | + * just before), drop the reference and skip rescheduling - the process | |
| 1272 | + * only needs to be notified once. | |
| 1273 | + */ | |
| 1274 | + kref_get(&p->ref); | |
| 1275 | + if (!schedule_delayed_work(&p->signal_work, msecs_to_jiffies(delay_ms))) { | |
| 1276 | + kfd_unref_process(p); | |
| 1277 | + return; | |
| 1278 | + } | |
| 1279 | + | |
| 1280 | + dev_info(dev->adev->dev, | |
| 1281 | + "Deferring SIGBUS to process %s(pid:%d) by %u ms (RAS error opt-in safety timeout)\n", | |
| 1282 | + p->lead_thread->comm, p->lead_thread->pid, delay_ms); | |
| 1283 | + return; | |
| 1284 | + | |
| 1285 | +send_now: | |
| 1286 | + send_sig(SIGBUS, p->lead_thread, 0); | |
| 1287 | +} | |
| 1288 | + | |
| 1341 | 1289 | void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid) |
| 1342 | 1290 | { |
| 1343 | 1291 | struct kfd_process *p = kfd_lookup_process_by_pasid(pasid, NULL); |
| @@ -1392,7 +1340,7 @@ void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid) · 48 unmodified lines | ||
| 1392 | 1340 | rcu_read_unlock(); |
| 1393 | 1341 | |
| 1394 | 1342 | /* user application will handle SIGBUS signal */ |
| 1395 | − send_sig(SIGBUS, p->lead_thread, 0); | |
| 1343 | + kfd_signal_sigbus_with_delay(dev, p); | |
| 1396 | 1344 | |
| 1397 | 1345 | kfd_unref_process(p); |
| 1398 | 1346 | } |
| @@ -49,7 +49,6 @@ · 48 unmodified lines | ||
| 49 | 49 | #define UNSIGNALED_EVENT_SLOT ((uint64_t)-1) |
| 50 | 50 | |
| 51 | 51 | struct kfd_event_waiter; |
| 52 | −struct signal_page; | |
| 53 | 52 | |
| 54 | 53 | struct kfd_event { |
| 55 | 54 | u32 event_id; |
| @@ -63,9 +62,6 @@ struct kfd_event { · 7 unmodified lines | ||
| 63 | 62 | spinlock_t lock; |
| 64 | 63 | wait_queue_head_t wq; /* List of event waiters. */ |
| 65 | 64 | |
| 66 | − /* Only for signal events. */ | |
| 67 | − uint64_t __user *user_signal_address; | |
| 68 | − | |
| 69 | 65 | /* type specific data */ |
| 70 | 66 | union { |
| 71 | 67 | struct kfd_hsa_memory_exception_data memory_exception_data; |
| @@ -315,3 +315,19 @@ bool kfd_check_hiq_mqd_doorbell_id(struct kfd_node *node, uint32_t doorbell_id, · 314 unmodified lines | ||
| 315 | 315 | |
| 316 | 316 | return false; |
| 317 | 317 | } |
| 318 | + | |
| 319 | +bool mqd_on_vram(struct amdgpu_device *adev) | |
| 320 | +{ | |
| 321 | + if (adev->apu_prefer_gtt) | |
| 322 | + return false; | |
| 323 | + | |
| 324 | + switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { | |
| 325 | + case IP_VERSION(9, 4, 2): | |
| 326 | + case IP_VERSION(9, 4, 3): | |
| 327 | + case IP_VERSION(9, 4, 4): | |
| 328 | + case IP_VERSION(9, 5, 0): | |
| 329 | + return true; | |
| 330 | + default: | |
| 331 | + return false; | |
| 332 | + } | |
| 333 | +} | |
| @@ -202,4 +202,6 @@ uint64_t kfd_mqd_stride(struct mqd_manager *mm, · 201 unmodified lines | ||
| 202 | 202 | struct queue_properties *q); |
| 203 | 203 | bool kfd_check_hiq_mqd_doorbell_id(struct kfd_node *node, uint32_t doorbell_id, |
| 204 | 204 | uint32_t inst); |
| 205 | +bool mqd_on_vram(struct amdgpu_device *adev); | |
| 206 | + | |
| 205 | 207 | #endif /* KFD_MQD_MANAGER_H_ */ |
| @@ -116,20 +116,6 @@ static void set_priority(struct v9_mqd *m, struct queue_properties *q) · 115 unmodified lines | ||
| 116 | 116 | m->cp_hqd_pipe_priority = pipe_priority_map[q->priority]; |
| 117 | 117 | } |
| 118 | 118 | |
| 119 | −static bool mqd_on_vram(struct amdgpu_device *adev) | |
| 120 | −{ | |
| 121 | − if (adev->apu_prefer_gtt) | |
| 122 | − return false; | |
| 123 | − | |
| 124 | − switch (amdgpu_ip_version(adev, GC_HWIP, 0)) { | |
| 125 | − case IP_VERSION(9, 4, 3): | |
| 126 | − case IP_VERSION(9, 5, 0): | |
| 127 | − return true; | |
| 128 | − default: | |
| 129 | − return false; | |
| 130 | − } | |
| 131 | −} | |
| 132 | − | |
| 133 | 119 | static struct kfd_mem_obj *allocate_mqd(struct mqd_manager *mm, |
| 134 | 120 | struct queue_properties *q) |
| 135 | 121 | { |
| @@ -952,11 +952,32 @@ struct kfd_process { · 951 unmodified lines | ||
| 952 | 952 | struct idr event_idr; |
| 953 | 953 | /* Event page */ |
| 954 | 954 | u64 signal_handle; |
| 955 | − struct kfd_signal_page *signal_page; | |
| 955 | + /* | |
| 956 | + * Each signal event needs a 64-bit signal slot where the signaler will | |
| 957 | + * write a 1 before sending an interrupt. (This is needed because some | |
| 958 | + * interrupts do not contain enough spare data bits to identify an | |
| 959 | + * event.) The signal page is allocated in user mode and mapped to the | |
| 960 | + * kernel; individual signal events use their event_id as slot index. | |
| 961 | + */ | |
| 962 | + uint64_t *signal_page; | |
| 956 | 963 | size_t signal_mapped_size; |
| 957 | 964 | size_t signal_event_count; |
| 958 | 965 | bool signal_event_limit_reached; |
| 959 | 966 | |
| 967 | + /** | |
| 968 | + * @kfd_sigbus_delay_ms: Per-process KFD SIGBUS delivery option for | |
| 969 | + * poison/RAS events (set via DRM_IOCTL_AMDGPU_PROC_OPTIONS / | |
| 970 | + * AMDGPU_PROC_OPTIONS_OP_KFD_SIGBUS_DELAY). | |
| 971 | + * | |
| 972 | + * 0 - send SIGBUS immediately (default) | |
| 973 | + * 0xFFFFFFFF - suppress SIGBUS delivery | |
| 974 | + * other - delay SIGBUS delivery by this many milliseconds | |
| 975 | + */ | |
| 976 | + atomic_t kfd_sigbus_delay_ms; | |
| 977 | + | |
| 978 | + /* Delayed signal delivery to user */ | |
| 979 | + struct delayed_work signal_work; | |
| 980 | + | |
| 960 | 981 | /* Information used for memory eviction */ |
| 961 | 982 | void *kgd_process_info; |
| 962 | 983 | /* Eviction fence that is attached to all the BOs of this process. The |
| @@ -1525,7 +1546,6 @@ extern const struct kfd_device_global_init_class device_global_init_class_cik; · 562 unmodified lines | ||
| 1525 | 1546 | |
| 1526 | 1547 | int kfd_event_init_process(struct kfd_process *p); |
| 1527 | 1548 | void kfd_event_free_process(struct kfd_process *p); |
| 1528 | −int kfd_event_mmap(struct kfd_process *process, struct vm_area_struct *vma); | |
| 1529 | 1549 | int kfd_wait_on_events(struct kfd_process *p, |
| 1530 | 1550 | uint32_t num_events, void __user *data, |
| 1531 | 1551 | bool all, uint32_t *user_timeout_ms, |
| @@ -1554,6 +1574,7 @@ void kfd_signal_vm_fault_event(struct kfd_process_device *pdd, · 22 unmodified lines | ||
| 1554 | 1574 | void kfd_signal_reset_event(struct kfd_node *dev); |
| 1555 | 1575 | |
| 1556 | 1576 | void kfd_signal_poison_consumed_event(struct kfd_node *dev, u32 pasid); |
| 1577 | +void kfd_signal_sigbus_delayed_fn(struct work_struct *work); | |
| 1557 | 1578 | void kfd_signal_process_terminate_event(struct kfd_process *p); |
| 1558 | 1579 | |
| 1559 | 1580 | static inline void kfd_flush_tlb(struct kfd_process_device *pdd) |
| @@ -153,6 +153,21 @@ static void kfd_sdma_activity_worker(struct work_struct *work) · 152 unmodified lines | ||
| 153 | 153 | (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI)) |
| 154 | 154 | continue; |
| 155 | 155 | |
| 156 | + if (dqm->dev->kfd2kgd->hqd_sdma_get_counter) { | |
| 157 | + val = 0; | |
| 158 | + ret = dqm->dev->kfd2kgd->hqd_sdma_get_counter( | |
| 159 | + dqm->dev->adev, q->mqd, | |
| 160 | + dqm->dev->kfd->device_info.num_sdma_queues_per_engine, | |
| 161 | + &val); | |
| 162 | + | |
| 163 | + if (ret) | |
| 164 | + pr_debug("Failed to read SDMA queue active counter %i\n", ret); | |
| 165 | + else | |
| 166 | + workarea->sdma_activity_counter += val; | |
| 167 | + | |
| 168 | + continue; | |
| 169 | + } | |
| 170 | + | |
| 156 | 171 | sdma_q = kzalloc_obj(struct temp_sdma_queue_list); |
| 157 | 172 | if (!sdma_q) { |
| 158 | 173 | dqm_unlock(dqm); |
| @@ -171,7 +186,7 @@ static void kfd_sdma_activity_worker(struct work_struct *work) · 12 unmodified lines | ||
| 171 | 186 | * count |
| 172 | 187 | */ |
| 173 | 188 | if (list_empty(&sdma_q_list.list)) { |
| 174 | − workarea->sdma_activity_counter = pdd->sdma_past_activity_counter; | |
| 189 | + workarea->sdma_activity_counter += pdd->sdma_past_activity_counter; | |
| 175 | 190 | dqm_unlock(dqm); |
| 176 | 191 | return; |
| 177 | 192 | } |
| @@ -721,7 +736,7 @@ static void kfd_process_free_gpuvm(struct kgd_mem *mem, · 543 unmodified lines | ||
| 721 | 736 | struct kfd_node *dev = pdd->dev; |
| 722 | 737 | |
| 723 | 738 | if (kptr && *kptr) { |
| 724 | − amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem); | |
| 739 | + amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem); | |
| 725 | 740 | *kptr = NULL; |
| 726 | 741 | } |
| 727 | 742 | |
| @@ -761,10 +776,16 @@ static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd, · 33 unmodified lines | ||
| 761 | 776 | } |
| 762 | 777 | |
| 763 | 778 | if (kptr) { |
| 764 | − err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel( | |
| 765 | − (struct kgd_mem *)*mem, kptr, NULL); | |
| 779 | + u32 domain; | |
| 780 | + | |
| 781 | + if (flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) | |
| 782 | + domain = AMDGPU_GEM_DOMAIN_VRAM; | |
| 783 | + else | |
| 784 | + domain = AMDGPU_GEM_DOMAIN_GTT; | |
| 785 | + err = amdgpu_amdkfd_gpuvm_map_bo_to_kernel((struct kgd_mem *)*mem, | |
| 786 | + kptr, NULL, domain); | |
| 766 | 787 | if (err) { |
| 767 | − pr_debug("Map GTT BO to kernel failed\n"); | |
| 788 | + pr_debug("Map BO to kernel failed err %d\n", err); | |
| 768 | 789 | goto sync_memory_failed; |
| 769 | 790 | } |
| 770 | 791 | } |
| @@ -986,6 +1007,33 @@ out: · 215 unmodified lines | ||
| 986 | 1007 | return process; |
| 987 | 1008 | } |
| 988 | 1009 | |
| 1010 | +/** | |
| 1011 | + * amdgpu_amdkfd_set_sigbus_delay - Set per-process KFD SIGBUS delay | |
| 1012 | + * @task: task in the target process | |
| 1013 | + * @ms: encoded delay value (0 = immediate, 0xFFFFFFFF = suppress, | |
| 1014 | + * otherwise delay in milliseconds) | |
| 1015 | + * | |
| 1016 | + * Stores the SIGBUS delivery option on the kfd_process associated with | |
| 1017 | + * @task. If the calling process has not opened /dev/kfd yet (no | |
| 1018 | + * kfd_process exists), this is a no-op - the option only applies to | |
| 1019 | + * processes that actually use KFD. | |
| 1020 | + */ | |
| 1021 | +int amdgpu_amdkfd_set_sigbus_delay(struct task_struct *task, u32 ms) | |
| 1022 | +{ | |
| 1023 | + struct kfd_process *p; | |
| 1024 | + | |
| 1025 | + if (!task->mm) | |
| 1026 | + return -EINVAL; | |
| 1027 | + | |
| 1028 | + p = kfd_lookup_process_by_mm(task->mm); | |
| 1029 | + if (!p) | |
| 1030 | + return 0; | |
| 1031 | + | |
| 1032 | + atomic_set(&p->kfd_sigbus_delay_ms, ms); | |
| 1033 | + kfd_unref_process(p); | |
| 1034 | + return 0; | |
| 1035 | +} | |
| 1036 | + | |
| 989 | 1037 | static struct kfd_process *find_process_by_mm(const struct mm_struct *mm) |
| 990 | 1038 | { |
| 991 | 1039 | struct kfd_process *process; |
| @@ -1092,7 +1140,7 @@ static void kfd_process_kunmap_signal_bo(struct kfd_process *p) · 100 unmodified lines | ||
| 1092 | 1140 | if (!mem) |
| 1093 | 1141 | goto out; |
| 1094 | 1142 | |
| 1095 | − amdgpu_amdkfd_gpuvm_unmap_gtt_bo_from_kernel(mem); | |
| 1143 | + amdgpu_amdkfd_gpuvm_unmap_bo_from_kernel(mem); | |
| 1096 | 1144 | |
| 1097 | 1145 | out: |
| 1098 | 1146 | mutex_unlock(&p->mutex); |
| @@ -1330,6 +1378,11 @@ void kfd_process_notifier_release_internal(struct kfd_process *p) · 231 unmodified lines | ||
| 1330 | 1378 | kfd_process_table_remove(p); |
| 1331 | 1379 | cancel_delayed_work_sync(&p->eviction_work); |
| 1332 | 1380 | cancel_delayed_work_sync(&p->restore_work); |
| 1381 | + /* | |
| 1382 | + * If work pending, cancel it and drop the extra ref | |
| 1383 | + */ | |
| 1384 | + if (cancel_delayed_work_sync(&p->signal_work)) | |
| 1385 | + kfd_unref_process(p); | |
| 1333 | 1386 | |
| 1334 | 1387 | /* |
| 1335 | 1388 | * Dequeue and destroy user queues, it is not safe for GPU to access |
| @@ -1436,8 +1489,7 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd) · 100 unmodified lines | ||
| 1436 | 1489 | { |
| 1437 | 1490 | struct kfd_node *dev = pdd->dev; |
| 1438 | 1491 | struct qcm_process_device *qpd = &pdd->qpd; |
| 1439 | − uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT | |
| 1440 | − | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE | |
| 1492 | + u32 flags = KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE | |
| 1441 | 1493 | | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE; |
| 1442 | 1494 | struct kgd_mem *mem; |
| 1443 | 1495 | void *kaddr; |
| @@ -1446,7 +1498,12 @@ static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd) · 2 unmodified lines | ||
| 1446 | 1498 | if (!dev->kfd->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base) |
| 1447 | 1499 | return 0; |
| 1448 | 1500 | |
| 1449 | − /* cwsr_base is only set for dGPU */ | |
| 1501 | + if (KFD_GC_VERSION(dev) >= IP_VERSION(9, 4, 2) && !dev->adev->apu_prefer_gtt) | |
| 1502 | + flags |= KFD_IOC_ALLOC_MEM_FLAGS_VRAM; | |
| 1503 | + else | |
| 1504 | + flags |= KFD_IOC_ALLOC_MEM_FLAGS_GTT; | |
| 1505 | + | |
| 1506 | + /* Allocate CWSR TBA/TMA buffers */ | |
| 1450 | 1507 | ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base, |
| 1451 | 1508 | KFD_CWSR_TBA_TMA_SIZE, flags, &mem, &kaddr); |
| 1452 | 1509 | if (ret) |
| @@ -1586,6 +1643,7 @@ struct kfd_process *create_process(const struct task_struct *thread, bool primar · 133 unmodified lines | ||
| 1586 | 1643 | |
| 1587 | 1644 | INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker); |
| 1588 | 1645 | INIT_DELAYED_WORK(&process->restore_work, restore_process_worker); |
| 1646 | + INIT_DELAYED_WORK(&process->signal_work, kfd_signal_sigbus_delayed_fn); | |
| 1589 | 1647 | process->last_restore_timestamp = get_jiffies_64(); |
| 1590 | 1648 | err = kfd_event_init_process(process); |
| 1591 | 1649 | if (err) |
| @@ -1040,7 +1040,13 @@ int kfd_criu_restore_queue(struct kfd_process *p, · 1039 unmodified lines | ||
| 1040 | 1040 | ctl_stack = mqd + q_data->mqd_size; |
| 1041 | 1041 | |
| 1042 | 1042 | memset(&qp, 0, sizeof(qp)); |
| 1043 | − set_queue_properties_from_criu(&qp, q_data, NUM_XCC(pdd->dev->adev->gfx.xcc_mask)); | |
| 1043 | + set_queue_properties_from_criu(&qp, q_data, NUM_XCC(pdd->dev->xcc_mask)); | |
| 1044 | + | |
| 1045 | + ret = kfd_queue_acquire_buffers(pdd, &qp); | |
| 1046 | + if (ret) { | |
| 1047 | + pr_debug("failed to acquire user queue buffers for CRIU\n"); | |
| 1048 | + goto exit; | |
| 1049 | + } | |
| 1044 | 1050 | |
| 1045 | 1051 | ret = kfd_queue_acquire_buffers(pdd, &qp); |
| 1046 | 1052 | if (ret) { |
| @@ -224,10 +224,10 @@ static void kfd_smi_event_add(struct task_struct *task, struct kfd_node *dev, · 223 unmodified lines | ||
| 224 | 224 | |
| 225 | 225 | pid = kfd_smi_task_to_pid(task); |
| 226 | 226 | |
| 227 | − len = snprintf(fifo_in, sizeof(fifo_in), "%x ", event); | |
| 227 | + len = scnprintf(fifo_in, sizeof(fifo_in), "%x ", event); | |
| 228 | 228 | |
| 229 | 229 | va_start(args, fmt); |
| 230 | − len += vsnprintf(fifo_in + len, sizeof(fifo_in) - len, fmt, args); | |
| 230 | + len += vscnprintf(fifo_in + len, sizeof(fifo_in) - len, fmt, args); | |
| 231 | 231 | va_end(args); |
| 232 | 232 | |
| 233 | 233 | add_event_to_kfifo(pid, dev, event, fifo_in, len); |
| @@ -3473,7 +3473,13 @@ svm_range_is_valid(struct kfd_process *p, uint64_t start, uint64_t size) · 3472 unmodified lines | ||
| 3473 | 3473 | unsigned long start_unchg = start; |
| 3474 | 3474 | |
| 3475 | 3475 | start <<= PAGE_SHIFT; |
| 3476 | − end = start + (size << PAGE_SHIFT); | |
| 3476 | + | |
| 3477 | + if (size == 0) | |
| 3478 | + return -EINVAL; | |
| 3479 | + | |
| 3480 | + if (check_add_overflow(start, size << PAGE_SHIFT, &end)) | |
| 3481 | + return -EOVERFLOW; | |
| 3482 | + | |
| 3477 | 3483 | do { |
| 3478 | 3484 | vma = vma_lookup(p->mm, start); |
| 3479 | 3485 | if (!vma || (vma->vm_flags & device_vma)) |
| @@ -2013,12 +2013,21 @@ static void kfd_topology_set_capabilities(struct kfd_topology_device *dev) · 2012 unmodified lines | ||
| 2013 | 2013 | dev->node_props.capability |= |
| 2014 | 2014 | HSA_CAP_TRAP_DEBUG_PRECISE_MEMORY_OPERATIONS_SUPPORTED; |
| 2015 | 2015 | |
| 2016 | − if (!amdgpu_sriov_vf(dev->gpu->adev)) | |
| 2016 | + if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(9, 4, 3) && | |
| 2017 | + !amdgpu_sriov_vf(dev->gpu->adev)) | |
| 2017 | 2018 | dev->node_props.capability |= HSA_CAP_PER_QUEUE_RESET_SUPPORTED; |
| 2018 | 2019 | |
| 2019 | 2020 | } else { |
| 2020 | 2021 | dev->node_props.debug_prop |= HSA_DBG_WATCH_ADDR_MASK_LO_BIT_GFX10 | |
| 2021 | 2022 | HSA_DBG_WATCH_ADDR_MASK_HI_BIT; |
| 2023 | + /* gfx11 dGPU and gfx12.0 */ | |
| 2024 | + if ((KFD_GC_VERSION(dev->gpu) == IP_VERSION(11, 0, 0) || | |
| 2025 | + KFD_GC_VERSION(dev->gpu) == IP_VERSION(11, 0, 2) || | |
| 2026 | + KFD_GC_VERSION(dev->gpu) == IP_VERSION(11, 0, 3) || | |
| 2027 | + KFD_GC_VERSION(dev->gpu) == IP_VERSION(12, 0, 0) || | |
| 2028 | + KFD_GC_VERSION(dev->gpu) == IP_VERSION(12, 0, 1)) && | |
| 2029 | + !amdgpu_sriov_vf(dev->gpu->adev)) | |
| 2030 | + dev->node_props.capability |= HSA_CAP_PER_QUEUE_RESET_SUPPORTED; | |
| 2022 | 2031 | |
| 2023 | 2032 | if (KFD_GC_VERSION(dev->gpu) >= IP_VERSION(12, 0, 0)) |
| 2024 | 2033 | dev->node_props.capability |= |
| @@ -45,7 +45,7 @@ static const struct drm_driver amdgpu_xcp_driver = { · 44 unmodified lines | ||
| 45 | 45 | .minor = 0, |
| 46 | 46 | }; |
| 47 | 47 | |
| 48 | −static int8_t pdev_num; | |
| 48 | +static u8 pdev_num; | |
| 49 | 49 | static struct xcp_device *xcp_dev[MAX_XCP_PLATFORM_DEVICE]; |
| 50 | 50 | static DEFINE_MUTEX(xcp_mutex); |
| 51 | 51 | |
| @@ -56,6 +56,11 @@ int amdgpu_xcp_drm_dev_alloc(struct drm_device **ddev) · 4 unmodified lines | ||
| 56 | 56 | char *dev_name; |
| 57 | 57 | int ret, i; |
| 58 | 58 | |
| 59 | + if (!ddev) | |
| 60 | + return -EINVAL; | |
| 61 | + | |
| 62 | + BUILD_BUG_ON(MAX_XCP_PLATFORM_DEVICE >= U8_MAX); | |
| 63 | + | |
| 59 | 64 | guard(mutex)(&xcp_mutex); |
| 60 | 65 | |
| 61 | 66 | if (pdev_num >= MAX_XCP_PLATFORM_DEVICE) |
| @@ -105,7 +110,7 @@ out_unregister: · 43 unmodified lines | ||
| 105 | 110 | } |
| 106 | 111 | EXPORT_SYMBOL(amdgpu_xcp_drm_dev_alloc); |
| 107 | 112 | |
| 108 | −static void free_xcp_dev(int8_t index) | |
| 113 | +static void free_xcp_dev(uint8_t index) | |
| 109 | 114 | { |
| 110 | 115 | if ((index < MAX_XCP_PLATFORM_DEVICE) && (xcp_dev[index])) { |
| 111 | 116 | struct platform_device *pdev = xcp_dev[index]->pdev; |
| @@ -114,17 +119,18 @@ static void free_xcp_dev(int8_t index) · 2 unmodified lines | ||
| 114 | 119 | platform_device_unregister(pdev); |
| 115 | 120 | |
| 116 | 121 | xcp_dev[index] = NULL; |
| 117 | − pdev_num--; | |
| 122 | + if (pdev_num > 0) | |
| 123 | + pdev_num--; | |
| 118 | 124 | } |
| 119 | 125 | } |
| 120 | 126 | |
| 121 | 127 | void amdgpu_xcp_drm_dev_free(struct drm_device *ddev) |
| 122 | 128 | { |
| 123 | − int8_t i; | |
| 129 | + uint8_t i; | |
| 124 | 130 | |
| 125 | 131 | guard(mutex)(&xcp_mutex); |
| 126 | 132 | |
| 127 | − for (i = 0; i < MAX_XCP_PLATFORM_DEVICE; i++) { | |
| 133 | + for (i = 0; pdev_num && i < MAX_XCP_PLATFORM_DEVICE; i++) { | |
| 128 | 134 | if ((xcp_dev[i]) && (&xcp_dev[i]->drm == ddev)) { |
| 129 | 135 | free_xcp_dev(i); |
| 130 | 136 | break; |
| @@ -135,7 +141,7 @@ EXPORT_SYMBOL(amdgpu_xcp_drm_dev_free); · 4 unmodified lines | ||
| 135 | 141 | |
| 136 | 142 | void amdgpu_xcp_drv_release(void) |
| 137 | 143 | { |
| 138 | − int8_t i; | |
| 144 | + uint8_t i; | |
| 139 | 145 | |
| 140 | 146 | guard(mutex)(&xcp_mutex); |
| 141 | 147 | |
| @@ -41,7 +41,11 @@ AMDGPUDM = \ · 40 unmodified lines | ||
| 41 | 41 | amdgpu_dm_quirks.o \ |
| 42 | 42 | amdgpu_dm_wb.o \ |
| 43 | 43 | amdgpu_dm_colorop.o \ |
| 44 | − amdgpu_dm_ism.o | |
| 44 | + amdgpu_dm_ism.o \ | |
| 45 | + amdgpu_dm_backlight.o \ | |
| 46 | + amdgpu_dm_audio.o \ | |
| 47 | + amdgpu_dm_dmub.o \ | |
| 48 | + amdgpu_dm_connector.o | |
| 45 | 49 | |
| 46 | 50 | ifdef CONFIG_DRM_AMD_DC_FP |
| 47 | 51 | AMDGPUDM += dc_fpu.o |
| @@ -66,6 +66,11 @@ · 65 unmodified lines | ||
| 66 | 66 | #endif |
| 67 | 67 | #include "amdgpu_dm_psr.h" |
| 68 | 68 | #include "amdgpu_dm_replay.h" |
| 69 | +#include "amdgpu_dm_backlight.h" | |
| 70 | +#include "amdgpu_dm_audio.h" | |
| 71 | +#include "amdgpu_dm_dmub.h" | |
| 72 | +#include "amdgpu_dm_connector.h" | |
| 73 | +#include "amdgpu_dm_kunit_helpers.h" | |
| 69 | 74 | |
| 70 | 75 | #include "ivsrcid/ivsrcid_vislands30.h" |
| 71 | 76 | |
| @@ -94,7 +99,6 @@ · 22 unmodified lines | ||
| 94 | 99 | #include <drm/drm_mode.h> |
| 95 | 100 | #include <drm/drm_utils.h> |
| 96 | 101 | #include <drm/drm_vblank.h> |
| 97 | −#include <drm/drm_audio_component.h> | |
| 98 | 102 | #include <drm/drm_colorop.h> |
| 99 | 103 | #include <drm/drm_gem_atomic_helper.h> |
| 100 | 104 | |
| @@ -107,60 +111,9 @@ · 6 unmodified lines | ||
| 107 | 111 | #include "modules/inc/mod_power.h" |
| 108 | 112 | #include "modules/power/power_helpers.h" |
| 109 | 113 | |
| 110 | −static_assert(AMDGPU_DMUB_NOTIFICATION_MAX == DMUB_NOTIFICATION_MAX, "AMDGPU_DMUB_NOTIFICATION_MAX mismatch"); | |
| 111 | − | |
| 112 | −#define FIRMWARE_RENOIR_DMUB "amdgpu/renoir_dmcub.bin" | |
| 113 | −MODULE_FIRMWARE(FIRMWARE_RENOIR_DMUB); | |
| 114 | −#define FIRMWARE_SIENNA_CICHLID_DMUB "amdgpu/sienna_cichlid_dmcub.bin" | |
| 115 | −MODULE_FIRMWARE(FIRMWARE_SIENNA_CICHLID_DMUB); | |
| 116 | −#define FIRMWARE_NAVY_FLOUNDER_DMUB "amdgpu/navy_flounder_dmcub.bin" | |
| 117 | −MODULE_FIRMWARE(FIRMWARE_NAVY_FLOUNDER_DMUB); | |
| 118 | −#define FIRMWARE_GREEN_SARDINE_DMUB "amdgpu/green_sardine_dmcub.bin" | |
| 119 | −MODULE_FIRMWARE(FIRMWARE_GREEN_SARDINE_DMUB); | |
| 120 | −#define FIRMWARE_VANGOGH_DMUB "amdgpu/vangogh_dmcub.bin" | |
| 121 | −MODULE_FIRMWARE(FIRMWARE_VANGOGH_DMUB); | |
| 122 | −#define FIRMWARE_DIMGREY_CAVEFISH_DMUB "amdgpu/dimgrey_cavefish_dmcub.bin" | |
| 123 | −MODULE_FIRMWARE(FIRMWARE_DIMGREY_CAVEFISH_DMUB); | |
| 124 | −#define FIRMWARE_BEIGE_GOBY_DMUB "amdgpu/beige_goby_dmcub.bin" | |
| 125 | −MODULE_FIRMWARE(FIRMWARE_BEIGE_GOBY_DMUB); | |
| 126 | −#define FIRMWARE_YELLOW_CARP_DMUB "amdgpu/yellow_carp_dmcub.bin" | |
| 127 | −MODULE_FIRMWARE(FIRMWARE_YELLOW_CARP_DMUB); | |
| 128 | −#define FIRMWARE_DCN_314_DMUB "amdgpu/dcn_3_1_4_dmcub.bin" | |
| 129 | −MODULE_FIRMWARE(FIRMWARE_DCN_314_DMUB); | |
| 130 | −#define FIRMWARE_DCN_315_DMUB "amdgpu/dcn_3_1_5_dmcub.bin" | |
| 131 | −MODULE_FIRMWARE(FIRMWARE_DCN_315_DMUB); | |
| 132 | −#define FIRMWARE_DCN316_DMUB "amdgpu/dcn_3_1_6_dmcub.bin" | |
| 133 | −MODULE_FIRMWARE(FIRMWARE_DCN316_DMUB); | |
| 134 | − | |
| 135 | −#define FIRMWARE_DCN_V3_2_0_DMCUB "amdgpu/dcn_3_2_0_dmcub.bin" | |
| 136 | −MODULE_FIRMWARE(FIRMWARE_DCN_V3_2_0_DMCUB); | |
| 137 | −#define FIRMWARE_DCN_V3_2_1_DMCUB "amdgpu/dcn_3_2_1_dmcub.bin" | |
| 138 | −MODULE_FIRMWARE(FIRMWARE_DCN_V3_2_1_DMCUB); | |
| 139 | − | |
| 140 | −#define FIRMWARE_RAVEN_DMCU "amdgpu/raven_dmcu.bin" | |
| 141 | 114 | MODULE_FIRMWARE(FIRMWARE_RAVEN_DMCU); |
| 142 | − | |
| 143 | −#define FIRMWARE_NAVI12_DMCU "amdgpu/navi12_dmcu.bin" | |
| 144 | 115 | MODULE_FIRMWARE(FIRMWARE_NAVI12_DMCU); |
| 145 | 116 | |
| 146 | −#define FIRMWARE_DCN_35_DMUB "amdgpu/dcn_3_5_dmcub.bin" | |
| 147 | −MODULE_FIRMWARE(FIRMWARE_DCN_35_DMUB); | |
| 148 | − | |
| 149 | −#define FIRMWARE_DCN_351_DMUB "amdgpu/dcn_3_5_1_dmcub.bin" | |
| 150 | −MODULE_FIRMWARE(FIRMWARE_DCN_351_DMUB); | |
| 151 | − | |
| 152 | −#define FIRMWARE_DCN_36_DMUB "amdgpu/dcn_3_6_dmcub.bin" | |
| 153 | −MODULE_FIRMWARE(FIRMWARE_DCN_36_DMUB); | |
| 154 | − | |
| 155 | −#define FIRMWARE_DCN_401_DMUB "amdgpu/dcn_4_0_1_dmcub.bin" | |
| 156 | −MODULE_FIRMWARE(FIRMWARE_DCN_401_DMUB); | |
| 157 | − | |
| 158 | −#define FIRMWARE_DCN_42_DMUB "amdgpu/dcn_4_2_dmcub.bin" | |
| 159 | −MODULE_FIRMWARE(FIRMWARE_DCN_42_DMUB); | |
| 160 | − | |
| 161 | −#define FIRMWARE_DCN_42B_DMUB "amdgpu/dcn_4_2_1_dmcub.bin" | |
| 162 | −MODULE_FIRMWARE(FIRMWARE_DCN_42B_DMUB); | |
| 163 | − | |
| 164 | 117 | /** |
| 165 | 118 | * DOC: overview |
| 166 | 119 | * |
| @@ -174,46 +127,7 @@ MODULE_FIRMWARE(FIRMWARE_DCN_42B_DMUB); · 7 unmodified lines | ||
| 174 | 127 | /* basic init/fini API */ |
| 175 | 128 | static int amdgpu_dm_init(struct amdgpu_device *adev); |
| 176 | 129 | static void amdgpu_dm_fini(struct amdgpu_device *adev); |
| 177 | −static bool is_freesync_video_mode(const struct drm_display_mode *mode, struct amdgpu_dm_connector *aconnector); | |
| 178 | 130 | static void reset_freesync_config_for_crtc(struct dm_crtc_state *new_crtc_state); |
| 179 | −static struct amdgpu_i2c_adapter * | |
| 180 | −create_i2c(struct ddc_service *ddc_service, bool oem); | |
| 181 | − | |
| 182 | −static enum drm_mode_subconnector get_subconnector_type(struct dc_link *link) | |
| 183 | −{ | |
| 184 | − switch (link->dpcd_caps.dongle_type) { | |
| 185 | − case DISPLAY_DONGLE_NONE: | |
| 186 | − return DRM_MODE_SUBCONNECTOR_Native; | |
| 187 | − case DISPLAY_DONGLE_DP_VGA_CONVERTER: | |
| 188 | − return DRM_MODE_SUBCONNECTOR_VGA; | |
| 189 | − case DISPLAY_DONGLE_DP_DVI_CONVERTER: | |
| 190 | − case DISPLAY_DONGLE_DP_DVI_DONGLE: | |
| 191 | − return DRM_MODE_SUBCONNECTOR_DVID; | |
| 192 | − case DISPLAY_DONGLE_DP_HDMI_CONVERTER: | |
| 193 | − case DISPLAY_DONGLE_DP_HDMI_DONGLE: | |
| 194 | − return DRM_MODE_SUBCONNECTOR_HDMIA; | |
| 195 | − case DISPLAY_DONGLE_DP_HDMI_MISMATCHED_DONGLE: | |
| 196 | − default: | |
| 197 | − return DRM_MODE_SUBCONNECTOR_Unknown; | |
| 198 | − } | |
| 199 | −} | |
| 200 | − | |
| 201 | −static void update_subconnector_property(struct amdgpu_dm_connector *aconnector) | |
| 202 | −{ | |
| 203 | − struct dc_link *link = aconnector->dc_link; | |
| 204 | − struct drm_connector *connector = &aconnector->base; | |
| 205 | − enum drm_mode_subconnector subconnector = DRM_MODE_SUBCONNECTOR_Unknown; | |
| 206 | − | |
| 207 | − if (connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort) | |
| 208 | − return; | |
| 209 | − | |
| 210 | − if (aconnector->dc_sink) | |
| 211 | − subconnector = get_subconnector_type(link); | |
| 212 | − | |
| 213 | − drm_object_property_set_value(&connector->base, | |
| 214 | − connector->dev->mode_config.dp_subconnector_property, | |
| 215 | − subconnector); | |
| 216 | −} | |
| 217 | 131 | |
| 218 | 132 | /* |
| 219 | 133 | * initializes drm_device display related structures, based on the information |
| @@ -226,32 +140,23 @@ static int amdgpu_dm_initialize_drm_device(struct amdgpu_device *adev); · 6 unmodified lines | ||
| 226 | 140 | /* removes and deallocates the drm structures, created by the above function */ |
| 227 | 141 | static void amdgpu_dm_destroy_drm_device(struct amdgpu_display_manager *dm); |
| 228 | 142 | |
| 229 | −static int amdgpu_dm_connector_init(struct amdgpu_display_manager *dm, | |
| 230 | − struct amdgpu_dm_connector *amdgpu_dm_connector, | |
| 231 | − u32 link_index, | |
| 232 | − struct amdgpu_encoder *amdgpu_encoder); | |
| 233 | −static int amdgpu_dm_encoder_init(struct drm_device *dev, | |
| 234 | − struct amdgpu_encoder *aencoder, | |
| 235 | − uint32_t link_index); | |
| 236 | − | |
| 237 | −static int amdgpu_dm_connector_get_modes(struct drm_connector *connector); | |
| 238 | − | |
| 239 | 143 | static int amdgpu_dm_atomic_setup_commit(struct drm_atomic_commit *state); |
| 240 | 144 | static void amdgpu_dm_atomic_commit_tail(struct drm_atomic_commit *state); |
| 145 | +static void dm_enable_per_frame_crtc_master_sync(struct dc_state *context); | |
| 241 | 146 | |
| 242 | 147 | static int amdgpu_dm_atomic_check(struct drm_device *dev, |
| 243 | 148 | struct drm_atomic_commit *state); |
| 244 | 149 | |
| 245 | −static void handle_hpd_irq_helper(struct amdgpu_dm_connector *aconnector); | |
| 246 | −static void handle_hpd_rx_irq(void *param); | |
| 247 | − | |
| 248 | −static void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm, | |
| 249 | − int bl_idx, | |
| 250 | − u32 user_brightness); | |
| 251 | − | |
| 252 | −static bool | |
| 150 | +STATIC_IFN_KUNIT bool | |
| 253 | 151 | is_timing_unchanged_for_freesync(struct drm_crtc_state *old_crtc_state, |
| 254 | 152 | struct drm_crtc_state *new_crtc_state); |
| 153 | + | |
| 154 | +static inline void amdgpu_dm_exit_ips_for_hw_access(struct dc *dc) | |
| 155 | +{ | |
| 156 | + if (dc->ctx->dmub_srv && !dc->ctx->dmub_srv->idle_exit_counter) | |
| 157 | + dc_exit_ips_for_hw_access(dc); | |
| 158 | +} | |
| 159 | + | |
| 255 | 160 | /* |
| 256 | 161 | * dm_vblank_get_counter |
| 257 | 162 | * |
| @@ -332,40 +237,14 @@ static int dm_wait_for_idle(struct amdgpu_ip_block *ip_block) · 74 unmodified lines | ||
| 332 | 237 | return 0; |
| 333 | 238 | } |
| 334 | 239 | |
| 335 | −static bool dm_check_soft_reset(struct amdgpu_ip_block *ip_block) | |
| 336 | −{ | |
| 337 | − return false; | |
| 338 | −} | |
| 339 | − | |
| 340 | 240 | static int dm_soft_reset(struct amdgpu_ip_block *ip_block) |
| 341 | 241 | { |
| 342 | 242 | /* XXX todo */ |
| 343 | 243 | return 0; |
| 344 | 244 | } |
| 345 | 245 | |
| 346 | −static struct amdgpu_crtc * | |
| 347 | −get_crtc_by_otg_inst(struct amdgpu_device *adev, | |
| 348 | − int otg_inst) | |
| 349 | −{ | |
| 350 | − struct drm_device *dev = adev_to_drm(adev); | |
| 351 | − struct drm_crtc *crtc; | |
| 352 | − struct amdgpu_crtc *amdgpu_crtc; | |
| 353 | − | |
| 354 | − if (WARN_ON(otg_inst == -1)) | |
| 355 | − return adev->mode_info.crtcs[0]; | |
| 356 | − | |
| 357 | − list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) { | |
| 358 | − amdgpu_crtc = to_amdgpu_crtc(crtc); | |
| 359 | − | |
| 360 | − if (amdgpu_crtc->otg_inst == otg_inst) | |
| 361 | − return amdgpu_crtc; | |
| 362 | − } | |
| 363 | − | |
| 364 | − return NULL; | |
| 365 | −} | |
| 366 | − | |
| 367 | −static inline bool is_dc_timing_adjust_needed(struct dm_crtc_state *old_state, | |
| 368 | − struct dm_crtc_state *new_state) | |
| 246 | +STATIC_IFN_KUNIT bool is_dc_timing_adjust_needed(struct dm_crtc_state *old_state, | |
| 247 | + struct dm_crtc_state *new_state) | |
| 369 | 248 | { |
| 370 | 249 | if (new_state->stream->adjust.timing_adjust_pending) |
| 371 | 250 | return true; |
| @@ -376,13 +255,14 @@ static inline bool is_dc_timing_adjust_needed(struct dm_crtc_state *old_state, · 4 unmodified lines | ||
| 376 | 255 | else |
| 377 | 256 | return false; |
| 378 | 257 | } |
| 258 | +EXPORT_IF_KUNIT(is_dc_timing_adjust_needed); | |
| 379 | 259 | |
| 380 | 260 | /* |
| 381 | 261 | * DC will program planes with their z-order determined by their ordering |
| 382 | 262 | * in the dc_surface_updates array. This comparator is used to sort them |
| 383 | 263 | * by descending zpos. |
| 384 | 264 | */ |
| 385 | −static int dm_plane_layer_index_cmp(const void *a, const void *b) | |
| 265 | +STATIC_IFN_KUNIT int dm_plane_layer_index_cmp(const void *a, const void *b) | |
| 386 | 266 | { |
| 387 | 267 | const struct dc_surface_update *sa = (struct dc_surface_update *)a; |
| 388 | 268 | const struct dc_surface_update *sb = (struct dc_surface_update *)b; |
| @@ -390,6 +270,7 @@ static int dm_plane_layer_index_cmp(const void *a, const void *b) · 1 unmodified line | ||
| 390 | 270 | /* Sort by descending dc_plane layer_index (i.e. normalized_zpos) */ |
| 391 | 271 | return sb->surface->layer_index - sa->surface->layer_index; |
| 392 | 272 | } |
| 273 | +EXPORT_IF_KUNIT(dm_plane_layer_index_cmp); | |
| 393 | 274 | |
| 394 | 275 | /** |
| 395 | 276 | * update_planes_and_stream_adapter() - Send planes to be updated in DC |
| @@ -430,9291 +311,3195 @@ static inline bool update_planes_and_stream_adapter(struct dc *dc, · 34 unmodified lines | ||
| 430 | 311 | stream_update); |
| 431 | 312 | } |
| 432 | 313 | |
| 433 | −/** | |
| 434 | − * dm_pflip_high_irq() - Handle pageflip interrupt | |
| 435 | − * @interrupt_params: ignored | |
| 436 | − * | |
| 437 | − * Handles the pageflip interrupt by notifying all interested parties | |
| 438 | − * that the pageflip has been completed. | |
| 439 | − */ | |
| 440 | −static void dm_pflip_high_irq(void *interrupt_params) | |
| 314 | +static int dm_set_clockgating_state(struct amdgpu_ip_block *ip_block, | |
| 315 | + enum amd_clockgating_state state) | |
| 441 | 316 | { |
| 442 | − struct amdgpu_crtc *amdgpu_crtc; | |
| 443 | − struct common_irq_params *irq_params = interrupt_params; | |
| 444 | − struct amdgpu_device *adev = irq_params->adev; | |
| 445 | − struct drm_device *dev = adev_to_drm(adev); | |
| 446 | − unsigned long flags; | |
| 447 | − struct drm_pending_vblank_event *e; | |
| 448 | − u32 vpos, hpos, v_blank_start, v_blank_end; | |
| 449 | − bool vrr_active; | |
| 317 | + return 0; | |
| 318 | +} | |
| 450 | 319 | |
| 451 | − amdgpu_crtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_PFLIP); | |
| 320 | +static int dm_set_powergating_state(struct amdgpu_ip_block *ip_block, | |
| 321 | + enum amd_powergating_state state) | |
| 322 | +{ | |
| 323 | + return 0; | |
| 324 | +} | |
| 452 | 325 | |
| 453 | − /* IRQ could occur when in initial stage */ | |
| 454 | − /* TODO work and BO cleanup */ | |
| 455 | − if (amdgpu_crtc == NULL) { | |
| 456 | − drm_dbg_state(dev, "CRTC is null, returning.\n"); | |
| 457 | − return; | |
| 458 | − } | |
| 326 | +/* Prototypes of private functions */ | |
| 327 | +static int dm_early_init(struct amdgpu_ip_block *ip_block); | |
| 459 | 328 | |
| 460 | − spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); | |
| 329 | +/* Allocate memory for FBC compressed data */ | |
| 330 | +static void mmhub_read_system_context(struct amdgpu_device *adev, struct dc_phy_addr_space_config *pa_config) | |
| 331 | +{ | |
| 332 | + u64 pt_base; | |
| 333 | + u32 logical_addr_low; | |
| 334 | + u32 logical_addr_high; | |
| 335 | + u32 agp_base, agp_bot, agp_top; | |
| 336 | + PHYSICAL_ADDRESS_LOC page_table_start, page_table_end, page_table_base; | |
| 461 | 337 | |
| 462 | − if (amdgpu_crtc->pflip_status != AMDGPU_FLIP_SUBMITTED) { | |
| 463 | − drm_dbg_state(dev, | |
| 464 | − "amdgpu_crtc->pflip_status = %d != AMDGPU_FLIP_SUBMITTED(%d) on crtc:%d[%p]\n", | |
| 465 | − amdgpu_crtc->pflip_status, AMDGPU_FLIP_SUBMITTED, | |
| 466 | − amdgpu_crtc->crtc_id, amdgpu_crtc); | |
| 467 | − spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); | |
| 468 | − return; | |
| 469 | − } | |
| 338 | + memset(pa_config, 0, sizeof(*pa_config)); | |
| 470 | 339 | |
| 471 | − /* page flip completed. */ | |
| 472 | − e = amdgpu_crtc->event; | |
| 473 | − amdgpu_crtc->event = NULL; | |
| 340 | + agp_base = 0; | |
| 341 | + agp_bot = adev->gmc.agp_start >> 24; | |
| 342 | + agp_top = adev->gmc.agp_end >> 24; | |
| 474 | 343 | |
| 475 | − WARN_ON(!e); | |
| 344 | + /* AGP aperture is disabled */ | |
| 345 | + if (agp_bot > agp_top) { | |
| 346 | + logical_addr_low = adev->gmc.fb_start >> 18; | |
| 347 | + if (adev->apu_flags & (AMD_APU_IS_RAVEN2 | | |
| 348 | + AMD_APU_IS_RENOIR | | |
| 349 | + AMD_APU_IS_GREEN_SARDINE)) | |
| 350 | + /* | |
| 351 | + * Raven2 has a HW issue that it is unable to use the vram which | |
| 352 | + * is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR. So here is the | |
| 353 | + * workaround that increase system aperture high address (add 1) | |
| 354 | + * to get rid of the VM fault and hardware hang. | |
| 355 | + */ | |
| 356 | + logical_addr_high = (adev->gmc.fb_end >> 18) + 0x1; | |
| 357 | + else | |
| 358 | + logical_addr_high = adev->gmc.fb_end >> 18; | |
| 359 | + } else { | |
| 360 | + logical_addr_low = min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18; | |
| 361 | + if (adev->apu_flags & (AMD_APU_IS_RAVEN2 | | |
| 362 | + AMD_APU_IS_RENOIR | | |
| 363 | + AMD_APU_IS_GREEN_SARDINE)) | |
| 364 | + /* | |
| 365 | + * Raven2 has a HW issue that it is unable to use the vram which | |
| 366 | + * is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR. So here is the | |
| 367 | + * workaround that increase system aperture high address (add 1) | |
| 368 | + * to get rid of the VM fault and hardware hang. | |
| 369 | + */ | |
| 370 | + logical_addr_high = max((adev->gmc.fb_end >> 18) + 0x1, adev->gmc.agp_end >> 18); | |
| 371 | + else | |
| 372 | + logical_addr_high = max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18; | |
| 373 | + } | |
| 476 | 374 | |
| 477 | − vrr_active = amdgpu_dm_crtc_vrr_active_irq(amdgpu_crtc); | |
| 375 | + pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); | |
| 478 | 376 | |
| 479 | − /* Fixed refresh rate, or VRR scanout position outside front-porch? */ | |
| 480 | − if (!vrr_active || | |
| 481 | − !dc_stream_get_scanoutpos(amdgpu_crtc->dm_irq_params.stream, &v_blank_start, | |
| 482 | − &v_blank_end, &hpos, &vpos) || | |
| 483 | − (vpos < v_blank_start)) { | |
| 484 | − /* Update to correct count and vblank timestamp if racing with | |
| 485 | − * vblank irq. This also updates to the correct vblank timestamp | |
| 486 | − * even in VRR mode, as scanout is past the front-porch atm. | |
| 487 | − */ | |
| 488 | − drm_crtc_accurate_vblank_count(&amdgpu_crtc->base); | |
| 377 | + page_table_start.high_part = upper_32_bits(adev->gmc.gart_start >> | |
| 378 | + AMDGPU_GPU_PAGE_SHIFT); | |
| 379 | + page_table_start.low_part = lower_32_bits(adev->gmc.gart_start >> | |
| 380 | + AMDGPU_GPU_PAGE_SHIFT); | |
| 381 | + page_table_end.high_part = upper_32_bits(adev->gmc.gart_end >> | |
| 382 | + AMDGPU_GPU_PAGE_SHIFT); | |
| 383 | + page_table_end.low_part = lower_32_bits(adev->gmc.gart_end >> | |
| 384 | + AMDGPU_GPU_PAGE_SHIFT); | |
| 385 | + page_table_base.high_part = upper_32_bits(pt_base); | |
| 386 | + page_table_base.low_part = lower_32_bits(pt_base); | |
| 489 | 387 | |
| 490 | − /* Wake up userspace by sending the pageflip event with proper | |
| 491 | − * count and timestamp of vblank of flip completion. | |
| 492 | − */ | |
| 493 | − if (e) { | |
| 494 | − drm_crtc_send_vblank_event(&amdgpu_crtc->base, e); | |
| 495 | − | |
| 496 | − /* Event sent, so done with vblank for this flip */ | |
| 497 | − drm_crtc_vblank_put(&amdgpu_crtc->base); | |
| 498 | − } | |
| 499 | − } else if (e) { | |
| 500 | − /* VRR active and inside front-porch: vblank count and | |
| 501 | − * timestamp for pageflip event will only be up to date after | |
| 502 | − * drm_crtc_handle_vblank() has been executed from late vblank | |
| 503 | − * irq handler after start of back-porch (vline 0). We queue the | |
| 504 | − * pageflip event for send-out by drm_crtc_handle_vblank() with | |
| 505 | − * updated timestamp and count, once it runs after us. | |
| 506 | − * | |
| 507 | − * We need to open-code this instead of using the helper | |
| 508 | − * drm_crtc_arm_vblank_event(), as that helper would | |
| 509 | − * call drm_crtc_accurate_vblank_count(), which we must | |
| 510 | − * not call in VRR mode while we are in front-porch! | |
| 511 | − */ | |
| 388 | + pa_config->system_aperture.start_addr = (uint64_t)logical_addr_low << 18; | |
| 389 | + pa_config->system_aperture.end_addr = (uint64_t)logical_addr_high << 18; | |
| 512 | 390 | |
| 513 | − /* sequence will be replaced by real count during send-out. */ | |
| 514 | − e->sequence = drm_crtc_vblank_count(&amdgpu_crtc->base); | |
| 515 | − e->pipe = amdgpu_crtc->crtc_id; | |
| 391 | + pa_config->system_aperture.agp_base = (uint64_t)agp_base << 24; | |
| 392 | + pa_config->system_aperture.agp_bot = (uint64_t)agp_bot << 24; | |
| 393 | + pa_config->system_aperture.agp_top = (uint64_t)agp_top << 24; | |
| 516 | 394 | |
| 517 | − list_add_tail(&e->base.link, &adev_to_drm(adev)->vblank_event_list); | |
| 518 | − e = NULL; | |
| 519 | − } | |
| 395 | + pa_config->system_aperture.fb_base = adev->gmc.fb_start; | |
| 396 | + pa_config->system_aperture.fb_offset = adev->vm_manager.vram_base_offset; | |
| 397 | + pa_config->system_aperture.fb_top = adev->gmc.fb_end; | |
| 520 | 398 | |
| 521 | − /* Keep track of vblank of this flip for flip throttling. We use the | |
| 522 | − * cooked hw counter, as that one incremented at start of this vblank | |
| 523 | − * of pageflip completion, so last_flip_vblank is the forbidden count | |
| 524 | − * for queueing new pageflips if vsync + VRR is enabled. | |
| 525 | − */ | |
| 526 | − amdgpu_crtc->dm_irq_params.last_flip_vblank = | |
| 527 | − amdgpu_get_vblank_counter_kms(&amdgpu_crtc->base); | |
| 399 | + pa_config->gart_config.page_table_start_addr = page_table_start.quad_part << 12; | |
| 400 | + pa_config->gart_config.page_table_end_addr = page_table_end.quad_part << 12; | |
| 401 | + pa_config->gart_config.page_table_base_addr = page_table_base.quad_part; | |
| 528 | 402 | |
| 529 | − amdgpu_crtc->pflip_status = AMDGPU_FLIP_NONE; | |
| 530 | − spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); | |
| 403 | + pa_config->is_hvm_enabled = adev->mode_info.gpu_vm_support; | |
| 531 | 404 | |
| 532 | − drm_dbg_state(dev, | |
| 533 | − "crtc:%d[%p], pflip_stat:AMDGPU_FLIP_NONE, vrr[%d]-fp %d\n", | |
| 534 | − amdgpu_crtc->crtc_id, amdgpu_crtc, vrr_active, (int)!e); | |
| 535 | 405 | } |
| 536 | 406 | |
| 537 | −static void dm_handle_vmin_vmax_update(struct work_struct *offload_work) | |
| 538 | −{ | |
| 539 | − struct vupdate_offload_work *work = container_of(offload_work, struct vupdate_offload_work, work); | |
| 540 | − struct amdgpu_device *adev = work->adev; | |
| 541 | − struct dc_stream_state *stream = work->stream; | |
| 542 | − struct dc_crtc_timing_adjust *adjust = work->adjust; | |
| 543 | − | |
| 544 | − mutex_lock(&adev->dm.dc_lock); | |
| 545 | − dc_stream_adjust_vmin_vmax(adev->dm.dc, stream, adjust); | |
| 546 | − mutex_unlock(&adev->dm.dc_lock); | |
| 407 | +struct amdgpu_stutter_quirk { | |
| 408 | + u16 chip_vendor; | |
| 409 | + u16 chip_device; | |
| 410 | + u16 subsys_vendor; | |
| 411 | + u16 subsys_device; | |
| 412 | + u8 revision; | |
| 413 | +}; | |
| 547 | 414 | |
| 548 | − dc_stream_release(stream); | |
| 549 | − kfree(work->adjust); | |
| 550 | − kfree(work); | |
| 551 | −} | |
| 415 | +static const struct amdgpu_stutter_quirk amdgpu_stutter_quirk_list[] = { | |
| 416 | + /* https://bugzilla.kernel.org/show_bug.cgi?id=214417 */ | |
| 417 | + { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, | |
| 418 | + { 0, 0, 0, 0, 0 }, | |
| 419 | +}; | |
| 552 | 420 | |
| 553 | −static void schedule_dc_vmin_vmax(struct amdgpu_device *adev, | |
| 554 | − struct dc_stream_state *stream, | |
| 555 | − struct dc_crtc_timing_adjust *adjust) | |
| 421 | +static bool dm_should_disable_stutter(struct pci_dev *pdev) | |
| 556 | 422 | { |
| 557 | − struct vupdate_offload_work *offload_work = kzalloc_obj(*offload_work, | |
| 558 | − GFP_NOWAIT); | |
| 559 | − if (!offload_work) { | |
| 560 | − drm_dbg_driver(adev_to_drm(adev), "Failed to allocate vupdate_offload_work\n"); | |
| 561 | − return; | |
| 562 | − } | |
| 423 | + const struct amdgpu_stutter_quirk *p = amdgpu_stutter_quirk_list; | |
| 563 | 424 | |
| 564 | − struct dc_crtc_timing_adjust *adjust_copy = kzalloc_obj(*adjust_copy, | |
| 565 | − GFP_NOWAIT); | |
| 566 | − if (!adjust_copy) { | |
| 567 | − drm_dbg_driver(adev_to_drm(adev), "Failed to allocate adjust_copy\n"); | |
| 568 | − kfree(offload_work); | |
| 569 | − return; | |
| 425 | + while (p && p->chip_device != 0) { | |
| 426 | + if (pdev->vendor == p->chip_vendor && | |
| 427 | + pdev->device == p->chip_device && | |
| 428 | + pdev->subsystem_vendor == p->subsys_vendor && | |
| 429 | + pdev->subsystem_device == p->subsys_device && | |
| 430 | + pdev->revision == p->revision) { | |
| 431 | + return true; | |
| 432 | + } | |
| 433 | + ++p; | |
| 570 | 434 | } |
| 571 | − | |
| 572 | − dc_stream_retain(stream); | |
| 573 | − memcpy(adjust_copy, adjust, sizeof(*adjust_copy)); | |
| 574 | − | |
| 575 | − INIT_WORK(&offload_work->work, dm_handle_vmin_vmax_update); | |
| 576 | − offload_work->adev = adev; | |
| 577 | − offload_work->stream = stream; | |
| 578 | − offload_work->adjust = adjust_copy; | |
| 579 | − | |
| 580 | − queue_work(system_percpu_wq, &offload_work->work); | |
| 435 | + return false; | |
| 581 | 436 | } |
| 582 | 437 | |
| 583 | −static void dm_vupdate_high_irq(void *interrupt_params) | |
| 584 | −{ | |
| 585 | − struct common_irq_params *irq_params = interrupt_params; | |
| 586 | − struct amdgpu_device *adev = irq_params->adev; | |
| 587 | − struct amdgpu_crtc *acrtc; | |
| 588 | − struct drm_device *drm_dev; | |
| 589 | − struct drm_vblank_crtc *vblank; | |
| 590 | − ktime_t frame_duration_ns, previous_timestamp; | |
| 591 | − unsigned long flags; | |
| 592 | − int vrr_active; | |
| 593 | 438 | |
| 594 | − acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VUPDATE); | |
| 439 | +void* | |
| 440 | +dm_allocate_gpu_mem( | |
| 441 | + struct amdgpu_device *adev, | |
| 442 | + enum dc_gpu_mem_alloc_type type, | |
| 443 | + size_t size, | |
| 444 | + long long *addr) | |
| 445 | +{ | |
| 446 | + struct dal_allocation *da; | |
| 447 | + u32 domain = (type == DC_MEM_ALLOC_TYPE_GART) ? | |
| 448 | + AMDGPU_GEM_DOMAIN_GTT : AMDGPU_GEM_DOMAIN_VRAM; | |
| 449 | + int ret; | |
| 595 | 450 | |
| 596 | − if (acrtc) { | |
| 597 | − vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); | |
| 598 | − drm_dev = acrtc->base.dev; | |
| 599 | − vblank = drm_crtc_vblank_crtc(&acrtc->base); | |
| 600 | − previous_timestamp = atomic64_read(&irq_params->previous_timestamp); | |
| 601 | − frame_duration_ns = vblank->time - previous_timestamp; | |
| 451 | + da = kzalloc_obj(struct dal_allocation); | |
| 452 | + if (!da) | |
| 453 | + return NULL; | |
| 602 | 454 | |
| 603 | − if (frame_duration_ns > 0) { | |
| 604 | − trace_amdgpu_refresh_rate_track(acrtc->base.index, | |
| 605 | − frame_duration_ns, | |
| 606 | − ktime_divns(NSEC_PER_SEC, frame_duration_ns)); | |
| 607 | − atomic64_set(&irq_params->previous_timestamp, vblank->time); | |
| 608 | − } | |
| 455 | + ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, | |
| 456 | + domain, &da->bo, | |
| 457 | + &da->gpu_addr, &da->cpu_ptr); | |
| 609 | 458 | |
| 610 | − drm_dbg_vbl(drm_dev, | |
| 611 | − "crtc:%d, vupdate-vrr:%d\n", acrtc->crtc_id, | |
| 612 | − vrr_active); | |
| 459 | + *addr = da->gpu_addr; | |
| 613 | 460 | |
| 614 | − /* Core vblank handling is done here after end of front-porch in | |
| 615 | − * vrr mode, as vblank timestamping will give valid results | |
| 616 | − * while now done after front-porch. This will also deliver | |
| 617 | − * page-flip completion events that have been queued to us | |
| 618 | − * if a pageflip happened inside front-porch. | |
| 619 | − */ | |
| 620 | − if (vrr_active && acrtc->dm_irq_params.stream) { | |
| 621 | − bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled; | |
| 622 | − bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled; | |
| 623 | − bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state | |
| 624 | − == VRR_STATE_ACTIVE_VARIABLE; | |
| 625 | − | |
| 626 | − amdgpu_dm_crtc_handle_vblank(acrtc); | |
| 627 | − | |
| 628 | − /* BTR processing for pre-DCE12 ASICs */ | |
| 629 | − if (adev->family < AMDGPU_FAMILY_AI) { | |
| 630 | − spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); | |
| 631 | − mod_freesync_handle_v_update( | |
| 632 | − adev->dm.freesync_module, | |
| 633 | − acrtc->dm_irq_params.stream, | |
| 634 | − &acrtc->dm_irq_params.vrr_params); | |
| 635 | − | |
| 636 | − if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) { | |
| 637 | − schedule_dc_vmin_vmax(adev, | |
| 638 | − acrtc->dm_irq_params.stream, | |
| 639 | − &acrtc->dm_irq_params.vrr_params.adjust); | |
| 640 | − } | |
| 641 | − spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); | |
| 642 | − } | |
| 643 | − } | |
| 461 | + if (ret) { | |
| 462 | + kfree(da); | |
| 463 | + return NULL; | |
| 644 | 464 | } |
| 465 | + | |
| 466 | + /* add da to list in dm */ | |
| 467 | + list_add(&da->list, &adev->dm.da_list); | |
| 468 | + | |
| 469 | + return da->cpu_ptr; | |
| 645 | 470 | } |
| 646 | 471 | |
| 647 | −/** | |
| 648 | − * dm_crtc_high_irq() - Handles CRTC interrupt | |
| 649 | − * @interrupt_params: used for determining the CRTC instance | |
| 650 | − * | |
| 651 | − * Handles the CRTC/VSYNC interrupt by notfying DRM's VBLANK | |
| 652 | − * event handler. | |
| 653 | − */ | |
| 654 | −static void dm_crtc_high_irq(void *interrupt_params) | |
| 472 | +void | |
| 473 | +dm_free_gpu_mem( | |
| 474 | + struct amdgpu_device *adev, | |
| 475 | + enum dc_gpu_mem_alloc_type type, | |
| 476 | + void *pvMem) | |
| 655 | 477 | { |
| 656 | − struct common_irq_params *irq_params = interrupt_params; | |
| 657 | − struct amdgpu_device *adev = irq_params->adev; | |
| 658 | − struct drm_writeback_job *job; | |
| 659 | − struct amdgpu_crtc *acrtc; | |
| 660 | − unsigned long flags; | |
| 661 | − int vrr_active; | |
| 662 | − | |
| 663 | − acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VBLANK); | |
| 664 | − if (!acrtc) | |
| 665 | − return; | |
| 478 | + struct dal_allocation *da; | |
| 666 | 479 | |
| 667 | − if (acrtc->wb_conn) { | |
| 668 | − spin_lock_irqsave(&acrtc->wb_conn->job_lock, flags); | |
| 669 | − | |
| 670 | − if (acrtc->wb_pending) { | |
| 671 | − job = list_first_entry_or_null(&acrtc->wb_conn->job_queue, | |
| 672 | − struct drm_writeback_job, | |
| 673 | − list_entry); | |
| 674 | − acrtc->wb_pending = false; | |
| 675 | − spin_unlock_irqrestore(&acrtc->wb_conn->job_lock, flags); | |
| 676 | − | |
| 677 | − if (job) { | |
| 678 | − unsigned int v_total, refresh_hz; | |
| 679 | − struct dc_stream_state *stream = acrtc->dm_irq_params.stream; | |
| 680 | − | |
| 681 | − v_total = stream->adjust.v_total_max ? | |
| 682 | − stream->adjust.v_total_max : stream->timing.v_total; | |
| 683 | − refresh_hz = div_u64((uint64_t) stream->timing.pix_clk_100hz * | |
| 684 | − 100LL, (v_total * stream->timing.h_total)); | |
| 685 | − mdelay(1000 / refresh_hz); | |
| 686 | − | |
| 687 | − drm_writeback_signal_completion(acrtc->wb_conn, 0); | |
| 688 | − dc_stream_fc_disable_writeback(adev->dm.dc, | |
| 689 | − acrtc->dm_irq_params.stream, 0); | |
| 690 | − } | |
| 691 | − } else | |
| 692 | − spin_unlock_irqrestore(&acrtc->wb_conn->job_lock, flags); | |
| 480 | + /* walk the da list in DM */ | |
| 481 | + list_for_each_entry(da, &adev->dm.da_list, list) { | |
| 482 | + if (pvMem == da->cpu_ptr) { | |
| 483 | + amdgpu_bo_free_kernel(&da->bo, &da->gpu_addr, &da->cpu_ptr); | |
| 484 | + list_del(&da->list); | |
| 485 | + kfree(da); | |
| 486 | + break; | |
| 487 | + } | |
| 693 | 488 | } |
| 694 | 489 | |
| 695 | − vrr_active = amdgpu_dm_crtc_vrr_active_irq(acrtc); | |
| 696 | − | |
| 697 | − drm_dbg_vbl(adev_to_drm(adev), | |
| 698 | − "crtc:%d, vupdate-vrr:%d, planes:%d\n", acrtc->crtc_id, | |
| 699 | − vrr_active, acrtc->dm_irq_params.active_planes); | |
| 700 | − | |
| 701 | − /** | |
| 702 | − * Core vblank handling at start of front-porch is only possible | |
| 703 | − * in non-vrr mode, as only there vblank timestamping will give | |
| 704 | − * valid results while done in front-porch. Otherwise defer it | |
| 705 | − * to dm_vupdate_high_irq after end of front-porch. | |
| 706 | − */ | |
| 707 | − if (!vrr_active) | |
| 708 | − amdgpu_dm_crtc_handle_vblank(acrtc); | |
| 709 | − | |
| 710 | − /** | |
| 711 | − * Following stuff must happen at start of vblank, for crc | |
| 712 | − * computation and below-the-range btr support in vrr mode. | |
| 713 | − */ | |
| 714 | − amdgpu_dm_crtc_handle_crc_irq(&acrtc->base); | |
| 715 | − | |
| 716 | − /* BTR updates need to happen before VUPDATE on Vega and above. */ | |
| 717 | − if (adev->family < AMDGPU_FAMILY_AI) | |
| 718 | − return; | |
| 719 | − | |
| 720 | − spin_lock_irqsave(&adev_to_drm(adev)->event_lock, flags); | |
| 721 | − | |
| 722 | − if (acrtc->dm_irq_params.stream && | |
| 723 | − acrtc->dm_irq_params.vrr_params.supported) { | |
| 724 | − bool replay_en = acrtc->dm_irq_params.stream->link->replay_settings.replay_feature_enabled; | |
| 725 | − bool psr_en = acrtc->dm_irq_params.stream->link->psr_settings.psr_feature_enabled; | |
| 726 | − bool fs_active_var_en = acrtc->dm_irq_params.freesync_config.state == VRR_STATE_ACTIVE_VARIABLE; | |
| 490 | +} | |
| 727 | 491 | |
| 728 | − mod_freesync_handle_v_update(adev->dm.freesync_module, | |
| 729 | − acrtc->dm_irq_params.stream, | |
| 730 | − &acrtc->dm_irq_params.vrr_params); | |
| 492 | +static int amdgpu_dm_init_power_module(struct amdgpu_display_manager *dm) | |
| 493 | +{ | |
| 494 | + struct mod_power_init_params init_data[MAX_NUM_EDP]; | |
| 731 | 495 | |
| 732 | − /* update vmin_vmax only if freesync is enabled, or only if PSR and REPLAY are disabled */ | |
| 733 | − if (fs_active_var_en || (!fs_active_var_en && !replay_en && !psr_en)) { | |
| 734 | − schedule_dc_vmin_vmax(adev, acrtc->dm_irq_params.stream, | |
| 735 | − &acrtc->dm_irq_params.vrr_params.adjust); | |
| 736 | − } | |
| 496 | + if (dm->num_of_edps == 0) { | |
| 497 | + drm_dbg_driver( | |
| 498 | + dm->ddev, | |
| 499 | + "amdgpu: No eDP detected, skip initializing power module\n"); | |
| 500 | + return 0; | |
| 737 | 501 | } |
| 738 | 502 | |
| 739 | − /* | |
| 740 | − * If there aren't any active_planes then DCH HUBP may be clock-gated. | |
| 741 | − * In that case, pageflip completion interrupts won't fire and pageflip | |
| 742 | − * completion events won't get delivered. Prevent this by sending | |
| 743 | − * pending pageflip events from here if a flip is still pending. | |
| 744 | − * | |
| 745 | − * If any planes are enabled, use dm_pflip_high_irq() instead, to | |
| 746 | − * avoid race conditions between flip programming and completion, | |
| 747 | − * which could cause too early flip completion events. | |
| 748 | − */ | |
| 749 | − if (adev->family >= AMDGPU_FAMILY_RV && | |
| 750 | − acrtc->pflip_status == AMDGPU_FLIP_SUBMITTED && | |
| 751 | − acrtc->dm_irq_params.active_planes == 0) { | |
| 752 | − if (acrtc->event) { | |
| 753 | − drm_crtc_send_vblank_event(&acrtc->base, acrtc->event); | |
| 754 | − acrtc->event = NULL; | |
| 755 | − drm_crtc_vblank_put(&acrtc->base); | |
| 756 | − } | |
| 757 | − acrtc->pflip_status = AMDGPU_FLIP_NONE; | |
| 758 | − } | |
| 759 | − | |
| 760 | − spin_unlock_irqrestore(&adev_to_drm(adev)->event_lock, flags); | |
| 761 | −} | |
| 762 | − | |
| 763 | −#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) | |
| 764 | −/** | |
| 765 | − * dm_dcn_vertical_interrupt0_high_irq() - Handles OTG Vertical interrupt0 for | |
| 766 | − * DCN generation ASICs | |
| 767 | − * @interrupt_params: interrupt parameters | |
| 768 | − * | |
| 769 | − * Used to set crc window/read out crc value at vertical line 0 position | |
| 770 | − */ | |
| 771 | −static void dm_dcn_vertical_interrupt0_high_irq(void *interrupt_params) | |
| 772 | −{ | |
| 773 | − struct common_irq_params *irq_params = interrupt_params; | |
| 774 | − struct amdgpu_device *adev = irq_params->adev; | |
| 775 | − struct amdgpu_crtc *acrtc; | |
| 503 | + /* Initialize all the power module parameters */ | |
| 504 | + for (int i = 0; i < dm->num_of_edps; i++) { | |
| 505 | + init_data[i].allow_psr_smu_optimizations = | |
| 506 | + !!(amdgpu_dc_feature_mask & DC_PSR_ALLOW_SMU_OPT); | |
| 507 | + init_data[i].allow_psr_multi_disp_optimizations = | |
| 508 | + !!(amdgpu_dc_feature_mask & DC_PSR_ALLOW_MULTI_DISP_OPT); | |
| 509 | + /* See dm_late_init */ | |
| 510 | + init_data[i].backlight_ramping_override = false; | |
| 511 | + init_data[i].backlight_ramping_start = 0xCCCC; | |
| 512 | + init_data[i].backlight_ramping_reduction = 0xCCCCCCCC; | |
| 513 | + init_data[i].def_varibright_level = 0; | |
| 514 | + init_data[i].abm_config_setting = 0; | |
| 515 | + init_data[i].num_backlight_levels = 101; | |
| 516 | + init_data[i].use_nits_based_brightness = false; | |
| 517 | + init_data[i].panel_max_millinits = 0; | |
| 518 | + init_data[i].panel_min_millinits = 0; | |
| 519 | + init_data[i].disable_fractional_pwm = | |
| 520 | + !(amdgpu_dc_feature_mask & DC_DISABLE_FRACTIONAL_PWM_MASK); | |
| 521 | + init_data[i].use_custom_backlight_caps = false; | |
| 522 | + init_data[i].custom_backlight_caps_config_no = 0; | |
| 523 | + init_data[i].use_linear_backlight_curve = false; | |
| 524 | + init_data[i].def_varibright_enable = 0; | |
| 525 | + init_data[i].varibright_level = 0; | |
| 526 | + /* | |
| 527 | + * Power module uses 16-bit backlight levels (0xFFFF max) rather | |
| 528 | + * than 8-bit(0XFF max) | |
| 529 | + */ | |
| 530 | + init_data[i].min_backlight_pwm = | |
| 531 | + dm->backlight_caps[i].min_input_signal * 0x101; | |
| 532 | + init_data[i].max_backlight_pwm = | |
| 533 | + dm->backlight_caps[i].max_input_signal * 0x101; | |
| 534 | + init_data[i].min_abm_backlight = | |
| 535 | + dm->backlight_caps[i].min_input_signal * 0x101; | |
| 776 | 536 | |
| 777 | − acrtc = get_crtc_by_otg_inst(adev, irq_params->irq_src - IRQ_TYPE_VLINE0); | |
| 537 | + /* Min backlight level after ABM reduction, Don't allow below 1% | |
| 538 | + * 0xFFFF x 0.01 = 0x28F | |
| 539 | + */ | |
| 540 | + init_data[i].min_abm_backlight = (init_data[i].min_abm_backlight < 0x28F) ? | |
| 541 | + 0x28F : init_data[i].min_abm_backlight; | |
| 542 | + } | |
| 778 | 543 | |
| 779 | − if (!acrtc) | |
| 780 | − return; | |
| 544 | + dm->power_module = mod_power_create(dm->dc, init_data, dm->num_of_edps); | |
| 545 | + if (!dm->power_module) { | |
| 546 | + drm_err(dm->ddev, "amdgpu: Error allocating memory for power module\n"); | |
| 547 | + return -ENOMEM; | |
| 548 | + } | |
| 781 | 549 | |
| 782 | − amdgpu_dm_crtc_handle_crc_window_irq(&acrtc->base); | |
| 783 | −} | |
| 784 | −#endif /* CONFIG_DRM_AMD_SECURE_DISPLAY */ | |
| 550 | + mod_power_hw_init(dm->power_module); | |
| 551 | + drm_dbg_driver(dm->ddev, "amdgpu: Power module init done\n"); | |
| 785 | 552 | |
| 786 | −/** | |
| 787 | − * dmub_aux_setconfig_callback - Callback for AUX or SET_CONFIG command. | |
| 788 | − * @adev: amdgpu_device pointer | |
| 789 | − * @notify: dmub notification structure | |
| 790 | − * | |
| 791 | − * Dmub AUX or SET_CONFIG command completion processing callback | |
| 792 | − * Copies dmub notification to DM which is to be read by AUX command. | |
| 793 | − * issuing thread and also signals the event to wake up the thread. | |
| 794 | − */ | |
| 795 | −static void dmub_aux_setconfig_callback(struct amdgpu_device *adev, | |
| 796 | − struct dmub_notification *notify) | |
| 797 | −{ | |
| 798 | − if (adev->dm.dmub_notify) | |
| 799 | − memcpy(adev->dm.dmub_notify, notify, sizeof(struct dmub_notification)); | |
| 800 | − if (notify->type == DMUB_NOTIFICATION_AUX_REPLY) | |
| 801 | − complete(&adev->dm.dmub_aux_transfer_done); | |
| 553 | + return 0; | |
| 802 | 554 | } |
| 803 | 555 | |
| 804 | −static void dmub_aux_fused_io_callback(struct amdgpu_device *adev, | |
| 805 | − struct dmub_notification *notify) | |
| 556 | +static int amdgpu_dm_init(struct amdgpu_device *adev) | |
| 806 | 557 | { |
| 807 | − if (!adev || !notify) { | |
| 808 | − ASSERT(false); | |
| 809 | − return; | |
| 810 | − } | |
| 811 | − | |
| 812 | − const struct dmub_cmd_fused_request *req = ¬ify->fused_request; | |
| 813 | − const uint8_t ddc_line = req->u.aux.ddc_line; | |
| 814 | − | |
| 815 | − if (ddc_line >= ARRAY_SIZE(adev->dm.fused_io)) { | |
| 816 | − ASSERT(false); | |
| 817 | − return; | |
| 818 | − } | |
| 558 | + struct dc_init_data init_data; | |
| 559 | + struct dc_callback_init init_params; | |
| 560 | + int r; | |
| 819 | 561 | |
| 820 | − struct fused_io_sync *sync = &adev->dm.fused_io[ddc_line]; | |
| 562 | + adev->dm.ddev = adev_to_drm(adev); | |
| 563 | + adev->dm.adev = adev; | |
| 821 | 564 | |
| 822 | − static_assert(sizeof(*req) <= sizeof(sync->reply_data), "Size mismatch"); | |
| 823 | − memcpy(sync->reply_data, req, sizeof(*req)); | |
| 824 | − complete(&sync->replied); | |
| 825 | −} | |
| 565 | + /* Zero all the fields */ | |
| 566 | + memset(&init_data, 0, sizeof(init_data)); | |
| 567 | + memset(&init_params, 0, sizeof(init_params)); | |
| 826 | 568 | |
| 827 | −/** | |
| 828 | − * dmub_hpd_callback - DMUB HPD interrupt processing callback. | |
| 829 | − * @adev: amdgpu_device pointer | |
| 830 | − * @notify: dmub notification structure | |
| 831 | − * | |
| 832 | − * Dmub Hpd interrupt processing callback. Gets displayindex through the | |
| 833 | − * ink index and calls helper to do the processing. | |
| 834 | − */ | |
| 835 | −static void dmub_hpd_callback(struct amdgpu_device *adev, | |
| 836 | − struct dmub_notification *notify) | |
| 837 | −{ | |
| 838 | − struct amdgpu_dm_connector *aconnector; | |
| 839 | − struct amdgpu_dm_connector *hpd_aconnector = NULL; | |
| 840 | − struct drm_connector *connector; | |
| 841 | − struct drm_connector_list_iter iter; | |
| 842 | − struct dc_link *link; | |
| 843 | − u8 link_index = 0; | |
| 844 | − struct drm_device *dev; | |
| 569 | + mutex_init(&adev->dm.dpia_aux_lock); | |
| 570 | + mutex_init(&adev->dm.dc_lock); | |
| 571 | + mutex_init(&adev->dm.audio_lock); | |
| 845 | 572 | |
| 846 | − if (adev == NULL) | |
| 847 | − return; | |
| 573 | + spin_lock_init(&adev->dm.dmub_lock); | |
| 848 | 574 | |
| 849 | − if (notify == NULL) { | |
| 850 | − drm_err(adev_to_drm(adev), "DMUB HPD callback notification was NULL"); | |
| 851 | − return; | |
| 575 | + if (amdgpu_dm_irq_init(adev)) { | |
| 576 | + drm_err(adev_to_drm(adev), "failed to initialize DM IRQ support.\n"); | |
| 577 | + goto error; | |
| 852 | 578 | } |
| 853 | 579 | |
| 854 | − if (notify->link_index > adev->dm.dc->link_count) { | |
| 855 | − drm_err(adev_to_drm(adev), "DMUB HPD index (%u)is abnormal", notify->link_index); | |
| 856 | − return; | |
| 857 | − } | |
| 580 | + /* special handling for early revisions of GC 11.5.4 */ | |
| 581 | + if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4)) | |
| 582 | + init_data.asic_id.chip_family = AMDGPU_FAMILY_GC_11_5_4; | |
| 583 | + else | |
| 584 | + init_data.asic_id.chip_family = adev->family; | |
| 858 | 585 | |
| 859 | − /* Skip DMUB HPD IRQ in suspend/resume. We will probe them later. */ | |
| 860 | − if (notify->type == DMUB_NOTIFICATION_HPD && adev->in_suspend) { | |
| 861 | − drm_info(adev_to_drm(adev), "Skip DMUB HPD IRQ callback in suspend/resume\n"); | |
| 862 | − return; | |
| 863 | − } | |
| 586 | + init_data.asic_id.pci_revision_id = adev->pdev->revision; | |
| 587 | + init_data.asic_id.hw_internal_rev = adev->external_rev_id; | |
| 588 | + init_data.asic_id.chip_id = adev->pdev->device; | |
| 864 | 589 | |
| 865 | − link_index = notify->link_index; | |
| 866 | − link = adev->dm.dc->links[link_index]; | |
| 867 | − dev = adev->dm.ddev; | |
| 590 | + init_data.asic_id.vram_width = adev->gmc.vram_width; | |
| 591 | + /* TODO: initialize init_data.asic_id.vram_type here!!!! */ | |
| 592 | + init_data.asic_id.atombios_base_address = | |
| 593 | + adev->mode_info.atom_context->bios; | |
| 868 | 594 | |
| 869 | − drm_connector_list_iter_begin(dev, &iter); | |
| 870 | − drm_for_each_connector_iter(connector, &iter) { | |
| 595 | + init_data.driver = adev; | |
| 871 | 596 | |
| 872 | − if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) | |
| 873 | − continue; | |
| 597 | + /* cgs_device was created in dm_sw_init() */ | |
| 598 | + init_data.cgs_device = adev->dm.cgs_device; | |
| 874 | 599 | |
| 875 | − aconnector = to_amdgpu_dm_connector(connector); | |
| 876 | − if (link && aconnector->dc_link == link) { | |
| 877 | − if (notify->type == DMUB_NOTIFICATION_HPD) | |
| 878 | − drm_info(adev_to_drm(adev), "DMUB HPD IRQ callback: link_index=%u\n", link_index); | |
| 879 | − else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) | |
| 880 | − drm_info(adev_to_drm(adev), "DMUB HPD RX IRQ callback: link_index=%u\n", link_index); | |
| 881 | − else | |
| 882 | − drm_warn(adev_to_drm(adev), "DMUB Unknown HPD callback type %d, link_index=%u\n", | |
| 883 | − notify->type, link_index); | |
| 600 | + init_data.dce_environment = DCE_ENV_PRODUCTION_DRV; | |
| 884 | 601 | |
| 885 | − hpd_aconnector = aconnector; | |
| 602 | + switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { | |
| 603 | + case IP_VERSION(2, 1, 0): | |
| 604 | + switch (adev->dm.dmcub_fw_version) { | |
| 605 | + case 0: /* development */ | |
| 606 | + case 0x1: /* linux-firmware.git hash 6d9f399 */ | |
| 607 | + case 0x01000000: /* linux-firmware.git hash 9a0b0f4 */ | |
| 608 | + init_data.flags.disable_dmcu = false; | |
| 886 | 609 | break; |
| 610 | + default: | |
| 611 | + init_data.flags.disable_dmcu = true; | |
| 887 | 612 | } |
| 613 | + break; | |
| 614 | + case IP_VERSION(2, 0, 3): | |
| 615 | + init_data.flags.disable_dmcu = true; | |
| 616 | + break; | |
| 617 | + default: | |
| 618 | + break; | |
| 888 | 619 | } |
| 889 | − drm_connector_list_iter_end(&iter); | |
| 890 | 620 | |
| 891 | − if (hpd_aconnector) { | |
| 892 | − if (notify->type == DMUB_NOTIFICATION_HPD) { | |
| 893 | − if (hpd_aconnector->dc_link->hpd_status == (notify->hpd_status == DP_HPD_PLUG)) | |
| 894 | − drm_warn(adev_to_drm(adev), "DMUB reported hpd status unchanged. link_index=%u\n", link_index); | |
| 895 | − handle_hpd_irq_helper(hpd_aconnector); | |
| 896 | − } else if (notify->type == DMUB_NOTIFICATION_HPD_IRQ) { | |
| 897 | − handle_hpd_rx_irq(hpd_aconnector); | |
| 898 | − } | |
| 899 | − } | |
| 900 | −} | |
| 621 | + /* APU support S/G display by default except: | |
| 622 | + * ASICs before Carrizo, | |
| 623 | + * RAVEN1 (Users reported stability issue) | |
| 624 | + */ | |
| 901 | 625 | |
| 902 | −/** | |
| 903 | − * dmub_hpd_sense_callback - DMUB HPD sense processing callback. | |
| 904 | − * @adev: amdgpu_device pointer | |
| 905 | − * @notify: dmub notification structure | |
| 906 | − * | |
| 907 | − * HPD sense changes can occur during low power states and need to be | |
| 908 | − * notified from firmware to driver. | |
| 909 | − */ | |
| 910 | −static void dmub_hpd_sense_callback(struct amdgpu_device *adev, | |
| 911 | − struct dmub_notification *notify) | |
| 912 | −{ | |
| 913 | − drm_dbg_driver(adev_to_drm(adev), "DMUB HPD SENSE callback.\n"); | |
| 914 | −} | |
| 626 | + if (adev->asic_type < CHIP_CARRIZO) { | |
| 627 | + init_data.flags.gpu_vm_support = false; | |
| 628 | + } else if (adev->asic_type == CHIP_RAVEN) { | |
| 629 | + if (adev->apu_flags & AMD_APU_IS_RAVEN) | |
| 630 | + init_data.flags.gpu_vm_support = false; | |
| 631 | + else | |
| 632 | + init_data.flags.gpu_vm_support = (amdgpu_sg_display != 0); | |
| 633 | + } else { | |
| 634 | + if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(2, 0, 3)) | |
| 635 | + init_data.flags.gpu_vm_support = (amdgpu_sg_display == 1); | |
| 636 | + else | |
| 637 | + init_data.flags.gpu_vm_support = | |
| 638 | + (amdgpu_sg_display != 0) && (adev->flags & AMD_IS_APU); | |
| 639 | + } | |
| 915 | 640 | |
| 916 | −/** | |
| 917 | − * register_dmub_notify_callback - Sets callback for DMUB notify | |
| 918 | − * @adev: amdgpu_device pointer | |
| 919 | − * @type: Type of dmub notification | |
| 920 | − * @callback: Dmub interrupt callback function | |
| 921 | − * @dmub_int_thread_offload: offload indicator | |
| 922 | − * | |
| 923 | − * API to register a dmub callback handler for a dmub notification | |
| 924 | − * Also sets indicator whether callback processing to be offloaded. | |
| 925 | − * to dmub interrupt handling thread | |
| 926 | − * Return: true if successfully registered, false if there is existing registration | |
| 927 | − */ | |
| 928 | −static bool register_dmub_notify_callback(struct amdgpu_device *adev, | |
| 929 | − enum dmub_notification_type type, | |
| 930 | − dmub_notify_interrupt_callback_t callback, | |
| 931 | − bool dmub_int_thread_offload) | |
| 932 | −{ | |
| 933 | − if (callback != NULL && type < ARRAY_SIZE(adev->dm.dmub_thread_offload)) { | |
| 934 | − adev->dm.dmub_callback[type] = callback; | |
| 935 | − adev->dm.dmub_thread_offload[type] = dmub_int_thread_offload; | |
| 936 | − } else | |
| 937 | − return false; | |
| 641 | + adev->mode_info.gpu_vm_support = init_data.flags.gpu_vm_support; | |
| 938 | 642 | |
| 939 | − return true; | |
| 940 | −} | |
| 643 | + if (amdgpu_dc_feature_mask & DC_FBC_MASK) | |
| 644 | + init_data.flags.fbc_support = true; | |
| 941 | 645 | |
| 942 | −static void dm_handle_hpd_work(struct work_struct *work) | |
| 943 | −{ | |
| 944 | − struct dmub_hpd_work *dmub_hpd_wrk; | |
| 646 | + if (amdgpu_dc_feature_mask & DC_MULTI_MON_PP_MCLK_SWITCH_MASK) | |
| 647 | + init_data.flags.multi_mon_pp_mclk_switch = true; | |
| 945 | 648 | |
| 946 | − dmub_hpd_wrk = container_of(work, struct dmub_hpd_work, handle_hpd_work); | |
| 649 | + if (amdgpu_dc_feature_mask & DC_DISABLE_FRACTIONAL_PWM_MASK) | |
| 650 | + init_data.flags.disable_fractional_pwm = true; | |
| 947 | 651 | |
| 948 | − if (!dmub_hpd_wrk->dmub_notify) { | |
| 949 | − drm_err(adev_to_drm(dmub_hpd_wrk->adev), "dmub_hpd_wrk dmub_notify is NULL"); | |
| 950 | − return; | |
| 951 | − } | |
| 652 | + if (amdgpu_dc_feature_mask & DC_EDP_NO_POWER_SEQUENCING) | |
| 653 | + init_data.flags.edp_no_power_sequencing = true; | |
| 952 | 654 | |
| 953 | − if (dmub_hpd_wrk->dmub_notify->type < ARRAY_SIZE(dmub_hpd_wrk->adev->dm.dmub_callback)) { | |
| 954 | − dmub_hpd_wrk->adev->dm.dmub_callback[dmub_hpd_wrk->dmub_notify->type](dmub_hpd_wrk->adev, | |
| 955 | − dmub_hpd_wrk->dmub_notify); | |
| 956 | − } | |
| 655 | + if (amdgpu_dc_feature_mask & DC_DISABLE_LTTPR_DP1_4A) | |
| 656 | + init_data.flags.allow_lttpr_non_transparent_mode.bits.DP1_4A = true; | |
| 657 | + if (amdgpu_dc_feature_mask & DC_DISABLE_LTTPR_DP2_0) | |
| 658 | + init_data.flags.allow_lttpr_non_transparent_mode.bits.DP2_0 = true; | |
| 957 | 659 | |
| 958 | − kfree(dmub_hpd_wrk->dmub_notify); | |
| 959 | − kfree(dmub_hpd_wrk); | |
| 660 | + if (amdgpu_dc_feature_mask & DC_FRL_MASK) | |
| 661 | + init_data.flags.enable_frl = true; | |
| 960 | 662 | |
| 961 | −} | |
| 663 | + init_data.flags.seamless_boot_edp_requested = false; | |
| 962 | 664 | |
| 963 | −static const char *dmub_notification_type_str(enum dmub_notification_type e) | |
| 964 | −{ | |
| 965 | − switch (e) { | |
| 966 | − case DMUB_NOTIFICATION_NO_DATA: | |
| 967 | − return "NO_DATA"; | |
| 968 | − case DMUB_NOTIFICATION_AUX_REPLY: | |
| 969 | − return "AUX_REPLY"; | |
| 970 | − case DMUB_NOTIFICATION_HPD: | |
| 971 | − return "HPD"; | |
| 972 | − case DMUB_NOTIFICATION_HPD_IRQ: | |
| 973 | − return "HPD_IRQ"; | |
| 974 | − case DMUB_NOTIFICATION_SET_CONFIG_REPLY: | |
| 975 | − return "SET_CONFIG_REPLY"; | |
| 976 | − case DMUB_NOTIFICATION_DPIA_NOTIFICATION: | |
| 977 | − return "DPIA_NOTIFICATION"; | |
| 978 | − case DMUB_NOTIFICATION_HPD_SENSE_NOTIFY: | |
| 979 | − return "HPD_SENSE_NOTIFY"; | |
| 980 | − case DMUB_NOTIFICATION_FUSED_IO: | |
| 981 | − return "FUSED_IO"; | |
| 982 | − default: | |
| 983 | − return "<unknown>"; | |
| 665 | + if (amdgpu_device_seamless_boot_supported(adev)) { | |
| 666 | + init_data.flags.seamless_boot_edp_requested = true; | |
| 667 | + init_data.flags.allow_seamless_boot_optimization = true; | |
| 668 | + drm_dbg(adev->dm.ddev, "Seamless boot requested\n"); | |
| 984 | 669 | } |
| 985 | −} | |
| 986 | − | |
| 987 | −#define DMUB_TRACE_MAX_READ 64 | |
| 988 | −/** | |
| 989 | − * dm_dmub_outbox1_low_irq() - Handles Outbox interrupt | |
| 990 | − * @interrupt_params: used for determining the Outbox instance | |
| 991 | − * | |
| 992 | − * Handles the Outbox Interrupt | |
| 993 | − * event handler. | |
| 994 | − */ | |
| 995 | −static void dm_dmub_outbox1_low_irq(void *interrupt_params) | |
| 996 | −{ | |
| 997 | − struct dmub_notification notify = {0}; | |
| 998 | − struct common_irq_params *irq_params = interrupt_params; | |
| 999 | − struct amdgpu_device *adev = irq_params->adev; | |
| 1000 | − struct amdgpu_display_manager *dm = &adev->dm; | |
| 1001 | − struct dmcub_trace_buf_entry entry = { 0 }; | |
| 1002 | − u32 count = 0; | |
| 1003 | − struct dmub_hpd_work *dmub_hpd_wrk; | |
| 1004 | 670 | |
| 1005 | − do { | |
| 1006 | − if (dc_dmub_srv_get_dmub_outbox0_msg(dm->dc, &entry)) { | |
| 1007 | − trace_amdgpu_dmub_trace_high_irq(entry.trace_code, entry.tick_count, | |
| 1008 | − entry.param0, entry.param1); | |
| 671 | + init_data.flags.enable_mipi_converter_optimization = true; | |
| 1009 | 672 | |
| 1010 | − drm_dbg_driver(adev_to_drm(adev), "trace_code:%u, tick_count:%u, param0:%u, param1:%u\n", | |
| 1011 | − entry.trace_code, entry.tick_count, entry.param0, entry.param1); | |
| 1012 | − } else | |
| 1013 | − break; | |
| 673 | + init_data.dcn_reg_offsets = adev->reg_offset[DCE_HWIP][0]; | |
| 674 | + init_data.nbio_reg_offsets = adev->reg_offset[NBIO_HWIP][0]; | |
| 675 | + init_data.clk_reg_offsets = adev->reg_offset[CLK_HWIP][0]; | |
| 1014 | 676 | |
| 1015 | − count++; | |
| 677 | + if (amdgpu_dc_debug_mask & DC_DISABLE_IPS) | |
| 678 | + init_data.flags.disable_ips = DMUB_IPS_DISABLE_ALL; | |
| 679 | + else if (amdgpu_dc_debug_mask & DC_DISABLE_IPS_DYNAMIC) | |
| 680 | + init_data.flags.disable_ips = DMUB_IPS_DISABLE_DYNAMIC; | |
| 681 | + else if (amdgpu_dc_debug_mask & DC_DISABLE_IPS2_DYNAMIC) | |
| 682 | + init_data.flags.disable_ips = DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF; | |
| 683 | + else if (amdgpu_dc_debug_mask & DC_FORCE_IPS_ENABLE) | |
| 684 | + init_data.flags.disable_ips = DMUB_IPS_ENABLE; | |
| 685 | + else | |
| 686 | + init_data.flags.disable_ips = dm_get_default_ips_mode(adev); | |
| 1016 | 687 | |
| 1017 | − } while (count <= DMUB_TRACE_MAX_READ); | |
| 688 | + init_data.flags.disable_ips_in_vpb = 0; | |
| 1018 | 689 | |
| 1019 | − if (count > DMUB_TRACE_MAX_READ) | |
| 1020 | − drm_dbg_driver(adev_to_drm(adev), "Warning : count > DMUB_TRACE_MAX_READ"); | |
| 690 | + /* DCN35 and above supports dynamic DTBCLK switch */ | |
| 691 | + if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 5, 0)) | |
| 692 | + init_data.flags.allow_0_dtb_clk = true; | |
| 1021 | 693 | |
| 1022 | − if (dc_enable_dmub_notifications(adev->dm.dc) && | |
| 1023 | − irq_params->irq_src == DC_IRQ_SOURCE_DMCUB_OUTBOX) { | |
| 694 | + /* Enable DWB for tested platforms only */ | |
| 695 | + if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(3, 0, 0)) | |
| 696 | + init_data.num_virtual_links = 1; | |
| 1024 | 697 | |
| 1025 | − do { | |
| 1026 | − dc_stat_get_dmub_notification(adev->dm.dc, ¬ify); | |
| 1027 | − if (notify.type >= ARRAY_SIZE(dm->dmub_thread_offload)) { | |
| 1028 | − drm_err(adev_to_drm(adev), "DM: notify type %d invalid!", notify.type); | |
| 1029 | − continue; | |
| 1030 | − } | |
| 1031 | − if (!dm->dmub_callback[notify.type]) { | |
| 1032 | − drm_warn(adev_to_drm(adev), "DMUB notification skipped due to no handler: type=%s\n", | |
| 1033 | − dmub_notification_type_str(notify.type)); | |
| 1034 | − continue; | |
| 1035 | − } | |
| 1036 | − if (dm->dmub_thread_offload[notify.type] == true) { | |
| 1037 | − dmub_hpd_wrk = kzalloc_obj(*dmub_hpd_wrk, | |
| 1038 | − GFP_ATOMIC); | |
| 1039 | − if (!dmub_hpd_wrk) { | |
| 1040 | − drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk"); | |
| 1041 | − return; | |
| 1042 | − } | |
| 1043 | − dmub_hpd_wrk->dmub_notify = kmemdup(¬ify, sizeof(struct dmub_notification), | |
| 1044 | − GFP_ATOMIC); | |
| 1045 | − if (!dmub_hpd_wrk->dmub_notify) { | |
| 1046 | − kfree(dmub_hpd_wrk); | |
| 1047 | − drm_err(adev_to_drm(adev), "Failed to allocate dmub_hpd_wrk->dmub_notify"); | |
| 1048 | − return; | |
| 1049 | − } | |
| 1050 | − INIT_WORK(&dmub_hpd_wrk->handle_hpd_work, dm_handle_hpd_work); | |
| 1051 | − dmub_hpd_wrk->adev = adev; | |
| 1052 | − queue_work(adev->dm.delayed_hpd_wq, &dmub_hpd_wrk->handle_hpd_work); | |
| 1053 | − } else { | |
| 1054 | − dm->dmub_callback[notify.type](adev, ¬ify); | |
| 1055 | − } | |
| 1056 | − } while (notify.pending_notification); | |
| 698 | + /* DCN42 and above dpia switch to unified link training path */ | |
| 699 | + if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(4, 2, 0)) { | |
| 700 | + init_data.flags.consolidated_dpia_dp_lt = true; | |
| 701 | + init_data.flags.enable_dpia_pre_training = true; | |
| 702 | + init_data.flags.unify_link_enc_assignment = true; | |
| 703 | + init_data.flags.usb4_bw_alloc_support = true; | |
| 1057 | 704 | } |
| 1058 | −} | |
| 705 | + retrieve_dmi_info(&adev->dm); | |
| 706 | + if (adev->dm.edp0_on_dp1_quirk) | |
| 707 | + init_data.flags.support_edp0_on_dp1 = true; | |
| 1059 | 708 | |
| 1060 | −static int dm_set_clockgating_state(struct amdgpu_ip_block *ip_block, | |
| 1061 | − enum amd_clockgating_state state) | |
| 1062 | −{ | |
| 1063 | − return 0; | |
| 1064 | −} | |
| 1065 | − | |
| 1066 | −static int dm_set_powergating_state(struct amdgpu_ip_block *ip_block, | |
| 1067 | − enum amd_powergating_state state) | |
| 1068 | −{ | |
| 1069 | − return 0; | |
| 1070 | −} | |
| 1071 | − | |
| 1072 | −/* Prototypes of private functions */ | |
| 1073 | −static int dm_early_init(struct amdgpu_ip_block *ip_block); | |
| 1074 | − | |
| 1075 | −/* Allocate memory for FBC compressed data */ | |
| 1076 | −static void amdgpu_dm_fbc_init(struct drm_connector *connector) | |
| 1077 | −{ | |
| 1078 | − struct amdgpu_device *adev = drm_to_adev(connector->dev); | |
| 1079 | − struct dm_compressor_info *compressor = &adev->dm.compressor; | |
| 1080 | − struct amdgpu_dm_connector *aconn = to_amdgpu_dm_connector(connector); | |
| 1081 | − struct drm_display_mode *mode; | |
| 1082 | − unsigned long max_size = 0; | |
| 1083 | − | |
| 1084 | − if (adev->dm.dc->fbc_compressor == NULL) | |
| 1085 | − return; | |
| 1086 | − | |
| 1087 | − if (aconn->dc_link->connector_signal != SIGNAL_TYPE_EDP) | |
| 1088 | − return; | |
| 1089 | − | |
| 1090 | − if (compressor->bo_ptr) | |
| 1091 | − return; | |
| 709 | + if (adev->dm.bb_from_dmub) | |
| 710 | + init_data.bb_from_dmub = adev->dm.bb_from_dmub; | |
| 711 | + else | |
| 712 | + init_data.bb_from_dmub = NULL; | |
| 1092 | 713 | |
| 714 | + /* Display Core create. */ | |
| 715 | + adev->dm.dc = dc_create(&init_data); | |
| 1093 | 716 | |
| 1094 | − list_for_each_entry(mode, &connector->modes, head) { | |
| 1095 | − if (max_size < (unsigned long) mode->htotal * mode->vtotal) | |
| 1096 | − max_size = (unsigned long) mode->htotal * mode->vtotal; | |
| 717 | + if (adev->dm.dc) { | |
| 718 | + drm_info(adev_to_drm(adev), "Display Core v%s initialized on %s\n", DC_VER, | |
| 719 | + dce_version_to_string(adev->dm.dc->ctx->dce_version)); | |
| 720 | + } else { | |
| 721 | + drm_info(adev_to_drm(adev), "Display Core failed to initialize with v%s!\n", DC_VER); | |
| 722 | + goto error; | |
| 1097 | 723 | } |
| 1098 | 724 | |
| 1099 | − if (max_size) { | |
| 1100 | − int r = amdgpu_bo_create_kernel(adev, max_size * 4, PAGE_SIZE, | |
| 1101 | − AMDGPU_GEM_DOMAIN_GTT, &compressor->bo_ptr, | |
| 1102 | − &compressor->gpu_addr, &compressor->cpu_addr); | |
| 1103 | − | |
| 1104 | − if (r) | |
| 1105 | − drm_err(adev_to_drm(adev), "DM: Failed to initialize FBC\n"); | |
| 1106 | − else { | |
| 1107 | − adev->dm.dc->ctx->fbc_gpu_addr = compressor->gpu_addr; | |
| 1108 | − drm_info(adev_to_drm(adev), "DM: FBC alloc %lu\n", max_size*4); | |
| 1109 | − } | |
| 1110 | − | |
| 725 | + if (amdgpu_dc_debug_mask & DC_DISABLE_PIPE_SPLIT) { | |
| 726 | + adev->dm.dc->debug.force_single_disp_pipe_split = false; | |
| 727 | + adev->dm.dc->debug.pipe_split_policy = MPC_SPLIT_AVOID; | |
| 1111 | 728 | } |
| 1112 | 729 | |
| 1113 | −} | |
| 1114 | − | |
| 1115 | −static int amdgpu_dm_audio_component_get_eld(struct device *kdev, int port, | |
| 1116 | − int pipe, bool *enabled, | |
| 1117 | − unsigned char *buf, int max_bytes) | |
| 1118 | −{ | |
| 1119 | − struct drm_device *dev = dev_get_drvdata(kdev); | |
| 1120 | − struct amdgpu_device *adev = drm_to_adev(dev); | |
| 1121 | − struct drm_connector *connector; | |
| 1122 | − struct drm_connector_list_iter conn_iter; | |
| 1123 | − struct amdgpu_dm_connector *aconnector; | |
| 1124 | − int ret = 0; | |
| 1125 | − | |
| 1126 | − *enabled = false; | |
| 1127 | − | |
| 1128 | − mutex_lock(&adev->dm.audio_lock); | |
| 730 | + if (adev->asic_type != CHIP_CARRIZO && adev->asic_type != CHIP_STONEY) | |
| 731 | + adev->dm.dc->debug.disable_stutter = amdgpu_pp_feature_mask & PP_STUTTER_MODE ? false : true; | |
| 732 | + if (dm_should_disable_stutter(adev->pdev)) | |
| 733 | + adev->dm.dc->debug.disable_stutter = true; | |
| 1129 | 734 | |
| 1130 | − drm_connector_list_iter_begin(dev, &conn_iter); | |
| 1131 | − drm_for_each_connector_iter(connector, &conn_iter) { | |
| 735 | + if (amdgpu_dc_debug_mask & DC_DISABLE_STUTTER) | |
| 736 | + adev->dm.dc->debug.disable_stutter = true; | |
| 1132 | 737 | |
| 1133 | − if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) | |
| 1134 | − continue; | |
| 738 | + if (amdgpu_dc_debug_mask & DC_DISABLE_DSC) | |
| 739 | + adev->dm.dc->debug.disable_dsc = true; | |
| 1135 | 740 | |
| 1136 | − aconnector = to_amdgpu_dm_connector(connector); | |
| 1137 | − if (aconnector->audio_inst != port) | |
| 1138 | − continue; | |
| 741 | + if (amdgpu_dc_debug_mask & DC_DISABLE_CLOCK_GATING) | |
| 742 | + adev->dm.dc->debug.disable_clock_gate = true; | |
| 1139 | 743 | |
| 1140 | − *enabled = true; | |
| 1141 | − mutex_lock(&connector->eld_mutex); | |
| 1142 | − ret = drm_eld_size(connector->eld); | |
| 1143 | − memcpy(buf, connector->eld, min(max_bytes, ret)); | |
| 1144 | − mutex_unlock(&connector->eld_mutex); | |
| 744 | + if (amdgpu_dc_debug_mask & DC_FORCE_SUBVP_MCLK_SWITCH) | |
| 745 | + adev->dm.dc->debug.force_subvp_mclk_switch = true; | |
| 1145 | 746 | |
| 1146 | − break; | |
| 747 | + if (amdgpu_dc_debug_mask & DC_DISABLE_SUBVP_FAMS) { | |
| 748 | + adev->dm.dc->debug.force_disable_subvp = true; | |
| 749 | + adev->dm.dc->debug.fams2_config.bits.enable = false; | |
| 1147 | 750 | } |
| 1148 | − drm_connector_list_iter_end(&conn_iter); | |
| 1149 | − | |
| 1150 | − mutex_unlock(&adev->dm.audio_lock); | |
| 1151 | − | |
| 1152 | − drm_dbg_kms(adev_to_drm(adev), "Get ELD : idx=%d ret=%d en=%d\n", port, ret, *enabled); | |
| 1153 | − | |
| 1154 | − return ret; | |
| 1155 | −} | |
| 1156 | − | |
| 1157 | −static const struct drm_audio_component_ops amdgpu_dm_audio_component_ops = { | |
| 1158 | − .get_eld = amdgpu_dm_audio_component_get_eld, | |
| 1159 | −}; | |
| 1160 | − | |
| 1161 | −static int amdgpu_dm_audio_component_bind(struct device *kdev, | |
| 1162 | − struct device *hda_kdev, void *data) | |
| 1163 | −{ | |
| 1164 | − struct drm_device *dev = dev_get_drvdata(kdev); | |
| 1165 | − struct amdgpu_device *adev = drm_to_adev(dev); | |
| 1166 | − struct drm_audio_component *acomp = data; | |
| 1167 | 751 | |
| 1168 | − acomp->ops = &amdgpu_dm_audio_component_ops; | |
| 1169 | − acomp->dev = kdev; | |
| 1170 | − adev->dm.audio_component = acomp; | |
| 752 | + if (amdgpu_dc_debug_mask & DC_ENABLE_DML2) { | |
| 753 | + adev->dm.dc->debug.using_dml2 = true; | |
| 754 | + adev->dm.dc->debug.using_dml21 = true; | |
| 755 | + } | |
| 1171 | 756 | |
| 1172 | − return 0; | |
| 1173 | −} | |
| 757 | + if (amdgpu_dc_debug_mask & DC_HDCP_LC_FORCE_FW_ENABLE) | |
| 758 | + adev->dm.dc->debug.hdcp_lc_force_fw_enable = true; | |
| 1174 | 759 | |
| 1175 | −static void amdgpu_dm_audio_component_unbind(struct device *kdev, | |
| 1176 | − struct device *hda_kdev, void *data) | |
| 1177 | −{ | |
| 1178 | − struct amdgpu_device *adev = drm_to_adev(dev_get_drvdata(kdev)); | |
| 1179 | − struct drm_audio_component *acomp = data; | |
| 760 | + if (amdgpu_dc_debug_mask & DC_HDCP_LC_ENABLE_SW_FALLBACK) | |
| 761 | + adev->dm.dc->debug.hdcp_lc_enable_sw_fallback = true; | |
| 1180 | 762 | |
| 1181 | − acomp->ops = NULL; | |
| 1182 | − acomp->dev = NULL; | |
| 1183 | − adev->dm.audio_component = NULL; | |
| 1184 | −} | |
| 763 | + if (amdgpu_dc_debug_mask & DC_SKIP_DETECTION_LT) | |
| 764 | + adev->dm.dc->debug.skip_detection_link_training = true; | |
| 1185 | 765 | |
| 1186 | −static const struct component_ops amdgpu_dm_audio_component_bind_ops = { | |
| 1187 | − .bind = amdgpu_dm_audio_component_bind, | |
| 1188 | − .unbind = amdgpu_dm_audio_component_unbind, | |
| 1189 | −}; | |
| 766 | + adev->dm.dc->debug.visual_confirm = amdgpu_dc_visual_confirm; | |
| 1190 | 767 | |
| 1191 | −static int amdgpu_dm_audio_init(struct amdgpu_device *adev) | |
| 1192 | −{ | |
| 1193 | − int i, ret; | |
| 768 | + /* TODO: Remove after DP2 receiver gets proper support of Cable ID feature */ | |
| 769 | + adev->dm.dc->debug.ignore_cable_id = true; | |
| 1194 | 770 | |
| 1195 | − if (!amdgpu_audio) | |
| 1196 | − return 0; | |
| 771 | + if (adev->dm.dc->caps.dp_hdmi21_pcon_support) | |
| 772 | + drm_info(adev_to_drm(adev), "DP-HDMI FRL PCON supported\n"); | |
| 1197 | 773 | |
| 1198 | − adev->mode_info.audio.enabled = true; | |
| 774 | + r = dm_dmub_hw_init(adev); | |
| 775 | + if (r) { | |
| 776 | + drm_err(adev_to_drm(adev), "DMUB interface failed to initialize: status=%d\n", r); | |
| 777 | + goto error; | |
| 778 | + } | |
| 1199 | 779 | |
| 1200 | − adev->mode_info.audio.num_pins = adev->dm.dc->res_pool->audio_count; | |
| 780 | + dc_hardware_init(adev->dm.dc); | |
| 1201 | 781 | |
| 1202 | − for (i = 0; i < adev->mode_info.audio.num_pins; i++) { | |
| 1203 | − adev->mode_info.audio.pin[i].channels = -1; | |
| 1204 | − adev->mode_info.audio.pin[i].rate = -1; | |
| 1205 | − adev->mode_info.audio.pin[i].bits_per_sample = -1; | |
| 1206 | − adev->mode_info.audio.pin[i].status_bits = 0; | |
| 1207 | − adev->mode_info.audio.pin[i].category_code = 0; | |
| 1208 | − adev->mode_info.audio.pin[i].connected = false; | |
| 1209 | − adev->mode_info.audio.pin[i].id = | |
| 1210 | − adev->dm.dc->res_pool->audios[i]->inst; | |
| 1211 | − adev->mode_info.audio.pin[i].offset = 0; | |
| 782 | + adev->dm.hpd_rx_offload_wq = amdgpu_dm_hpd_rx_irq_create_workqueue(adev); | |
| 783 | + if (!adev->dm.hpd_rx_offload_wq) { | |
| 784 | + drm_err(adev_to_drm(adev), "failed to create hpd rx offload workqueue.\n"); | |
| 785 | + goto error; | |
| 1212 | 786 | } |
| 1213 | 787 | |
| 1214 | − ret = component_add(adev->dev, &amdgpu_dm_audio_component_bind_ops); | |
| 1215 | − if (ret < 0) | |
| 1216 | − return ret; | |
| 788 | + if ((adev->flags & AMD_IS_APU) && (adev->asic_type >= CHIP_CARRIZO)) { | |
| 789 | + struct dc_phy_addr_space_config pa_config; | |
| 1217 | 790 | |
| 1218 | − adev->dm.audio_registered = true; | |
| 791 | + mmhub_read_system_context(adev, &pa_config); | |
| 1219 | 792 | |
| 1220 | − return 0; | |
| 1221 | −} | |
| 793 | + // Call the DC init_memory func | |
| 794 | + dc_setup_system_context(adev->dm.dc, &pa_config); | |
| 795 | + } | |
| 1222 | 796 | |
| 1223 | −static void amdgpu_dm_audio_fini(struct amdgpu_device *adev) | |
| 1224 | −{ | |
| 1225 | − if (!amdgpu_audio) | |
| 1226 | − return; | |
| 797 | + adev->dm.freesync_module = mod_freesync_create(adev->dm.dc); | |
| 798 | + if (!adev->dm.freesync_module) { | |
| 799 | + drm_err(adev_to_drm(adev), | |
| 800 | + "failed to initialize freesync_module.\n"); | |
| 801 | + } else | |
| 802 | + drm_dbg_driver(adev_to_drm(adev), "freesync_module init done %p.\n", | |
| 803 | + adev->dm.freesync_module); | |
| 1227 | 804 | |
| 1228 | − if (!adev->mode_info.audio.enabled) | |
| 1229 | − return; | |
| 805 | + amdgpu_dm_init_color_mod(); | |
| 1230 | 806 | |
| 1231 | − if (adev->dm.audio_registered) { | |
| 1232 | − component_del(adev->dev, &amdgpu_dm_audio_component_bind_ops); | |
| 1233 | − adev->dm.audio_registered = false; | |
| 807 | + if (adev->dm.dc->caps.max_links > 0) { | |
| 808 | + adev->dm.vblank_control_workqueue = | |
| 809 | + create_singlethread_workqueue("dm_vblank_control_workqueue"); | |
| 810 | + if (!adev->dm.vblank_control_workqueue) | |
| 811 | + drm_err(adev_to_drm(adev), "failed to initialize vblank_workqueue.\n"); | |
| 1234 | 812 | } |
| 1235 | 813 | |
| 1236 | − /* TODO: Disable audio? */ | |
| 1237 | − | |
| 1238 | − adev->mode_info.audio.enabled = false; | |
| 1239 | −} | |
| 814 | + if (adev->dm.dc->caps.ips_support && | |
| 815 | + adev->dm.dc->config.disable_ips != DMUB_IPS_DISABLE_ALL) | |
| 816 | + adev->dm.idle_workqueue = idle_create_workqueue(adev); | |
| 1240 | 817 | |
| 1241 | −static void amdgpu_dm_audio_eld_notify(struct amdgpu_device *adev, int pin) | |
| 1242 | −{ | |
| 1243 | − struct drm_audio_component *acomp = adev->dm.audio_component; | |
| 818 | + if (adev->dm.dc->caps.max_links > 0 && adev->family >= AMDGPU_FAMILY_RV) { | |
| 819 | + adev->dm.hdcp_workqueue = hdcp_create_workqueue(adev, &init_params.cp_psp, adev->dm.dc); | |
| 1244 | 820 | |
| 1245 | − if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) { | |
| 1246 | − drm_dbg_kms(adev_to_drm(adev), "Notify ELD: %d\n", pin); | |
| 821 | + if (!adev->dm.hdcp_workqueue) | |
| 822 | + drm_err(adev_to_drm(adev), "failed to initialize hdcp_workqueue.\n"); | |
| 823 | + else | |
| 824 | + drm_dbg_driver(adev_to_drm(adev), | |
| 825 | + "hdcp_workqueue init done %p.\n", | |
| 826 | + adev->dm.hdcp_workqueue); | |
| 1247 | 827 | |
| 1248 | − acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr, | |
| 1249 | − pin, -1); | |
| 828 | + dc_init_callbacks(adev->dm.dc, &init_params); | |
| 1250 | 829 | } |
| 1251 | −} | |
| 830 | + if (adev->dm.dc->caps.max_links > 0) { | |
| 831 | + adev->dm.hdmi_frl_status_polling_wq = | |
| 832 | + create_singlethread_workqueue("hdmi_frl_status_polling_workqueue"); | |
| 833 | + if (!adev->dm.hdmi_frl_status_polling_wq) | |
| 834 | + drm_err(adev_to_drm(adev), "failed to initialize hdmi_frl_status_polling_workqueue\n"); | |
| 835 | + } | |
| 836 | + if (dc_is_dmub_outbox_supported(adev->dm.dc)) { | |
| 837 | + init_completion(&adev->dm.dmub_aux_transfer_done); | |
| 838 | + adev->dm.dmub_notify = kzalloc_obj(struct dmub_notification); | |
| 839 | + if (!adev->dm.dmub_notify) { | |
| 840 | + drm_info(adev_to_drm(adev), "fail to allocate adev->dm.dmub_notify"); | |
| 841 | + goto error; | |
| 842 | + } | |
| 1252 | 843 | |
| 1253 | −static int dm_dmub_hw_init(struct amdgpu_device *adev) | |
| 1254 | −{ | |
| 1255 | − const struct dmcub_firmware_header_v1_0 *hdr; | |
| 1256 | − struct dmub_srv *dmub_srv = adev->dm.dmub_srv; | |
| 1257 | − struct dmub_srv_fb_info *fb_info = adev->dm.dmub_fb_info; | |
| 1258 | − const struct firmware *dmub_fw = adev->dm.dmub_fw; | |
| 1259 | − struct dc *dc = adev->dm.dc; | |
| 1260 | − struct dmcu *dmcu = adev->dm.dc->res_pool->dmcu; | |
| 1261 | − struct abm *abm = adev->dm.dc->res_pool->abm; | |
| 1262 | − struct dc_context *ctx = adev->dm.dc->ctx; | |
| 1263 | − struct dmub_srv_hw_params hw_params; | |
| 1264 | − enum dmub_status status; | |
| 1265 | − const unsigned char *fw_inst_const, *fw_bss_data; | |
| 1266 | − u32 i, fw_inst_const_size, fw_bss_data_size; | |
| 1267 | − bool has_hw_support; | |
| 1268 | − | |
| 1269 | − if (!dmub_srv) | |
| 1270 | − /* DMUB isn't supported on the ASIC. */ | |
| 1271 | − return 0; | |
| 844 | + adev->dm.delayed_hpd_wq = create_singlethread_workqueue("amdgpu_dm_hpd_wq"); | |
| 845 | + if (!adev->dm.delayed_hpd_wq) { | |
| 846 | + drm_err(adev_to_drm(adev), "failed to create hpd offload workqueue.\n"); | |
| 847 | + goto error; | |
| 848 | + } | |
| 1272 | 849 | |
| 1273 | − if (!fb_info) { | |
| 1274 | − drm_err(adev_to_drm(adev), "No framebuffer info for DMUB service.\n"); | |
| 1275 | − return -EINVAL; | |
| 1276 | − } | |
| 850 | + amdgpu_dm_outbox_init(adev); | |
| 851 | + if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_AUX_REPLY, | |
| 852 | + dm_dmub_aux_setconfig_callback, false)) { | |
| 853 | + drm_err(adev_to_drm(adev), "fail to register dmub aux callback"); | |
| 854 | + goto error; | |
| 855 | + } | |
| 1277 | 856 | |
| 1278 | − if (!dmub_fw) { | |
| 1279 | − /* Firmware required for DMUB support. */ | |
| 1280 | − drm_err(adev_to_drm(adev), "No firmware provided for DMUB.\n"); | |
| 1281 | − return -EINVAL; | |
| 1282 | − } | |
| 857 | + for (size_t i = 0; i < ARRAY_SIZE(adev->dm.fused_io); i++) | |
| 858 | + init_completion(&adev->dm.fused_io[i].replied); | |
| 1283 | 859 | |
| 1284 | − /* initialize register offsets for ASICs with runtime initialization available */ | |
| 1285 | − if (dmub_srv->hw_funcs.init_reg_offsets) | |
| 1286 | − dmub_srv->hw_funcs.init_reg_offsets(dmub_srv, ctx); | |
| 860 | + if (!dm_register_dmub_notify_callback(adev, DMUB_NOTIFICATION_FUSED_IO, | |
| 861 | + dm_dmub_aux_fused_io_callback, false)) { | |
| 862 | + drm_err(adev_to_drm(adev), "fail to register dmub fused io callback"); | |
| 863 | + goto error; | |
| 864 | + } | |
| 865 | + /* Enable outbox notification only after IRQ handlers are registered and DMUB is alive. | |
| 866 | + * It is expected that DMUB will resend any pending notifications at this point. Note | |
| 867 | + * that hpd and hpd_irq handler registration are deferred to | |
| 868 | + * amdgpu_dm_register_hpd_handlers() to align legacy interface initialization | |
| 869 | + * sequence. Connection status will be proactivly detected once in the | |
| 870 | + * amdgpu_dm_initialize_drm_device. | |
| 871 | + */ | |
| 872 | + dc_enable_dmub_outbox(adev->dm.dc); | |
| 1287 | 873 | |
| 1288 | − status = dmub_srv_has_hw_support(dmub_srv, &has_hw_support); | |
| 1289 | − if (status != DMUB_STATUS_OK) { | |
| 1290 | − drm_err(adev_to_drm(adev), "Error checking HW support for DMUB: %d\n", status); | |
| 1291 | − return -EINVAL; | |
| 874 | + /* DPIA trace goes to dmesg logs only if outbox is enabled */ | |
| 875 | + if (amdgpu_dc_debug_mask & DC_ENABLE_DPIA_TRACE) | |
| 876 | + dc_dmub_srv_enable_dpia_trace(adev->dm.dc); | |
| 1292 | 877 | } |
| 1293 | 878 | |
| 1294 | − if (!has_hw_support) { | |
| 1295 | − drm_info(adev_to_drm(adev), "DMUB unsupported on ASIC\n"); | |
| 1296 | − return 0; | |
| 879 | + if (amdgpu_dm_initialize_drm_device(adev)) { | |
| 880 | + drm_err(adev_to_drm(adev), | |
| 881 | + "failed to initialize sw for display support.\n"); | |
| 882 | + goto error; | |
| 1297 | 883 | } |
| 1298 | 884 | |
| 1299 | − /* Reset DMCUB if it was previously running - before we overwrite its memory. */ | |
| 1300 | − status = dmub_srv_hw_reset(dmub_srv); | |
| 1301 | − if (status != DMUB_STATUS_OK) | |
| 1302 | − drm_warn(adev_to_drm(adev), "Error resetting DMUB HW: %d\n", status); | |
| 1303 | − | |
| 1304 | − hdr = (const struct dmcub_firmware_header_v1_0 *)dmub_fw->data; | |
| 1305 | − | |
| 1306 | − fw_inst_const = dmub_fw->data + | |
| 1307 | − le32_to_cpu(hdr->header.ucode_array_offset_bytes) + | |
| 1308 | − PSP_HEADER_BYTES_256; | |
| 885 | + if (amdgpu_dm_init_power_module(&adev->dm)) | |
| 886 | + goto error; | |
| 1309 | 887 | |
| 1310 | − fw_bss_data = dmub_fw->data + | |
| 1311 | − le32_to_cpu(hdr->header.ucode_array_offset_bytes) + | |
| 1312 | − le32_to_cpu(hdr->inst_const_bytes); | |
| 888 | + /* create fake encoders for MST */ | |
| 889 | + dm_dp_create_fake_mst_encoders(adev); | |
| 1313 | 890 | |
| 1314 | − /* Copy firmware and bios info into FB memory. */ | |
| 1315 | − fw_inst_const_size = adev->dm.fw_inst_size; | |
| 891 | + /* TODO: Add_display_info? */ | |
| 1316 | 892 | |
| 1317 | − fw_bss_data_size = le32_to_cpu(hdr->bss_data_bytes); | |
| 893 | + /* TODO use dynamic cursor width */ | |
| 894 | + adev_to_drm(adev)->mode_config.cursor_width = adev->dm.dc->caps.max_cursor_size; | |
| 895 | + adev_to_drm(adev)->mode_config.cursor_height = adev->dm.dc->caps.max_cursor_size; | |
| 1318 | 896 | |
| 1319 | − /* if adev->firmware.load_type == AMDGPU_FW_LOAD_PSP, | |
| 1320 | − * amdgpu_ucode_init_single_fw will load dmub firmware | |
| 1321 | − * fw_inst_const part to cw0; otherwise, the firmware back door load | |
| 1322 | − * will be done by dm_dmub_hw_init | |
| 1323 | − */ | |
| 1324 | − if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { | |
| 1325 | − memcpy(fb_info->fb[DMUB_WINDOW_0_INST_CONST].cpu_addr, fw_inst_const, | |
| 1326 | − fw_inst_const_size); | |
| 897 | + if (drm_vblank_init(adev_to_drm(adev), adev->dm.display_indexes_num)) { | |
| 898 | + drm_err(adev_to_drm(adev), | |
| 899 | + "failed to initialize vblank for display support.\n"); | |
| 900 | + goto error; | |
| 1327 | 901 | } |
| 1328 | 902 | |
| 1329 | − if (fw_bss_data_size) | |
| 1330 | − memcpy(fb_info->fb[DMUB_WINDOW_2_BSS_DATA].cpu_addr, | |
| 1331 | − fw_bss_data, fw_bss_data_size); | |
| 903 | +#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) | |
| 904 | + amdgpu_dm_crtc_secure_display_create_contexts(adev); | |
| 905 | + if (!adev->dm.secure_display_ctx.crtc_ctx) | |
| 906 | + drm_err(adev_to_drm(adev), "failed to initialize secure display contexts.\n"); | |
| 1332 | 907 | |
| 1333 | − /* Copy firmware bios info into FB memory. */ | |
| 1334 | − memcpy(fb_info->fb[DMUB_WINDOW_3_VBIOS].cpu_addr, adev->bios, | |
| 1335 | − adev->bios_size); | |
| 908 | + if (amdgpu_ip_version(adev, DCE_HWIP, 0) >= IP_VERSION(4, 0, 1)) | |
| 909 | + adev->dm.secure_display_ctx.support_mul_roi = true; | |
| 1336 | 910 | |
| 1337 | − /* Reset regions that need to be reset. */ | |
| 1338 | − memset(fb_info->fb[DMUB_WINDOW_4_MAILBOX].cpu_addr, 0, | |
| 1339 | − fb_info->fb[DMUB_WINDOW_4_MAILBOX].size); | |
| 911 | +#endif | |
| 1340 | 912 | |
| 1341 | − memset(fb_info->fb[DMUB_WINDOW_5_TRACEBUFF].cpu_addr, 0, | |
| 1342 | − fb_info->fb[DMUB_WINDOW_5_TRACEBUFF].size); | |
| 913 | + drm_dbg_driver(adev_to_drm(adev), "KMS initialized.\n"); | |
| 1343 | 914 | |
| 1344 | − memset(fb_info->fb[DMUB_WINDOW_6_FW_STATE].cpu_addr, 0, | |
| 1345 | − fb_info->fb[DMUB_WINDOW_6_FW_STATE].size); | |
| 915 | + return 0; | |
| 916 | +error: | |
| 917 | + amdgpu_dm_fini(adev); | |
| 1346 | 918 | |
| 1347 | − memset(fb_info->fb[DMUB_WINDOW_SHARED_STATE].cpu_addr, 0, | |
| 1348 | − fb_info->fb[DMUB_WINDOW_SHARED_STATE].size); | |
| 919 | + return -EINVAL; | |
| 920 | +} | |
| 1349 | 921 | |
| 1350 | − /* Initialize hardware. */ | |
| 1351 | − memset(&hw_params, 0, sizeof(hw_params)); | |
| 1352 | − hw_params.soc_fb_info.fb_base = adev->gmc.fb_start; | |
| 1353 | − hw_params.soc_fb_info.fb_offset = adev->vm_manager.vram_base_offset; | |
| 922 | +static int amdgpu_dm_early_fini(struct amdgpu_ip_block *ip_block) | |
| 923 | +{ | |
| 924 | + struct amdgpu_device *adev = ip_block->adev; | |
| 1354 | 925 | |
| 1355 | − /* backdoor load firmware and trigger dmub running */ | |
| 1356 | − if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) | |
| 1357 | − hw_params.load_inst_const = true; | |
| 926 | + amdgpu_dm_audio_fini(adev); | |
| 1358 | 927 | |
| 1359 | − if (dmcu) | |
| 1360 | − hw_params.psp_version = dmcu->psp_version; | |
| 928 | + return 0; | |
| 929 | +} | |
| 1361 | 930 | |
| 1362 | − for (i = 0; i < fb_info->num_fb; ++i) | |
| 1363 | − hw_params.fb[i] = &fb_info->fb[i]; | |
| 931 | +static void amdgpu_dm_fini(struct amdgpu_device *adev) | |
| 932 | +{ | |
| 933 | + int i; | |
| 1364 | 934 | |
| 1365 | − /* Enable usb4 dpia in the FW APU */ | |
| 1366 | − if (dc->caps.is_apu && | |
| 1367 | − dc->res_pool->usb4_dpia_count != 0 && | |
| 1368 | − !dc->debug.dpia_debug.bits.disable_dpia) { | |
| 1369 | − hw_params.dpia_supported = true; | |
| 1370 | − hw_params.disable_dpia = dc->debug.dpia_debug.bits.disable_dpia; | |
| 1371 | − hw_params.dpia_hpd_int_enable_supported = false; | |
| 1372 | − hw_params.enable_non_transparent_setconfig = dc->config.consolidated_dpia_dp_lt; | |
| 1373 | − hw_params.disable_dpia_bw_allocation = !dc->config.usb4_bw_alloc_support; | |
| 935 | + if (adev->dm.vblank_control_workqueue) { | |
| 936 | + destroy_workqueue(adev->dm.vblank_control_workqueue); | |
| 937 | + adev->dm.vblank_control_workqueue = NULL; | |
| 1374 | 938 | } |
| 1375 | 939 | |
| 1376 | − switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { | |
| 1377 | − case IP_VERSION(3, 5, 0): | |
| 1378 | − case IP_VERSION(3, 5, 1): | |
| 1379 | − case IP_VERSION(3, 6, 0): | |
| 1380 | − case IP_VERSION(4, 2, 0): | |
| 1381 | − case IP_VERSION(4, 2, 1): | |
| 1382 | − hw_params.ips_sequential_ono = adev->external_rev_id > 0x10; | |
| 1383 | − hw_params.lower_hbr3_phy_ssc = true; | |
| 1384 | − break; | |
| 1385 | − default: | |
| 1386 | − break; | |
| 940 | + if (adev->dm.idle_workqueue) { | |
| 941 | + if (adev->dm.idle_workqueue->running) { | |
| 942 | + adev->dm.idle_workqueue->enable = false; | |
| 943 | + flush_work(&adev->dm.idle_workqueue->work); | |
| 944 | + } | |
| 945 | + | |
| 946 | + kfree(adev->dm.idle_workqueue); | |
| 947 | + adev->dm.idle_workqueue = NULL; | |
| 1387 | 948 | } |
| 1388 | 949 | |
| 1389 | − status = dmub_srv_hw_init(dmub_srv, &hw_params); | |
| 1390 | − if (status != DMUB_STATUS_OK) { | |
| 1391 | − drm_err(adev_to_drm(adev), "Error initializing DMUB HW: %d\n", status); | |
| 1392 | − return -EINVAL; | |
| 1393 | − } | |
| 950 | + /* | |
| 951 | + * Disable ISM before dc_destroy() invalidates dm->dc. | |
| 952 | + * | |
| 953 | + * Quiesce workers first without dc_lock (they take dc_lock | |
| 954 | + * themselves, so syncing under it would deadlock), then drive the | |
| 955 | + * FSM back to FULL_POWER_RUNNING under dc_lock. | |
| 956 | + */ | |
| 957 | + amdgpu_dm_ism_disable(&adev->dm); | |
| 958 | + scoped_guard(mutex, &adev->dm.dc_lock) | |
| 959 | + amdgpu_dm_ism_force_full_power(&adev->dm); | |
| 1394 | 960 | |
| 1395 | − /* Wait for firmware load to finish. */ | |
| 1396 | − status = dmub_srv_wait_for_auto_load(dmub_srv, 100000); | |
| 1397 | − if (status != DMUB_STATUS_OK) | |
| 1398 | − drm_warn(adev_to_drm(adev), "Wait for DMUB auto-load failed: %d\n", status); | |
| 961 | + amdgpu_dm_destroy_drm_device(&adev->dm); | |
| 1399 | 962 | |
| 1400 | − /* Init DMCU and ABM if available. */ | |
| 1401 | − if (dmcu && abm) { | |
| 1402 | − dmcu->funcs->dmcu_init(dmcu); | |
| 1403 | − abm->dmcu_is_running = dmcu->funcs->is_dmcu_initialized(dmcu); | |
| 963 | +#if defined(CONFIG_DRM_AMD_SECURE_DISPLAY) | |
| 964 | + if (adev->dm.secure_display_ctx.crtc_ctx) { | |
| 965 | + for (i = 0; i < adev->mode_info.num_crtc; i++) { | |
| 966 | + if (adev->dm.secure_display_ctx.crtc_ctx[i].crtc) { | |
| 967 | + flush_work(&adev->dm.secure_display_ctx.crtc_ctx[i].notify_ta_work); | |
| 968 | + flush_work(&adev->dm.secure_display_ctx.crtc_ctx[i].forward_roi_work); | |
| 969 | + } | |
| 970 | + } | |
| 971 | + kfree(adev->dm.secure_display_ctx.crtc_ctx); | |
| 972 | + adev->dm.secure_display_ctx.crtc_ctx = NULL; | |
| 973 | + } | |
| 974 | +#endif | |
| 975 | + if (adev->dm.hdcp_workqueue) { | |
| 976 | + hdcp_destroy(&adev->dev->kobj, adev->dm.hdcp_workqueue); | |
| 977 | + adev->dm.hdcp_workqueue = NULL; | |
| 1404 | 978 | } |
| 1405 | 979 | |
| 1406 | − if (!adev->dm.dc->ctx->dmub_srv) | |
| 1407 | − adev->dm.dc->ctx->dmub_srv = dc_dmub_srv_create(adev->dm.dc, dmub_srv); | |
| 1408 | − if (!adev->dm.dc->ctx->dmub_srv) { | |
| 1409 | − drm_err(adev_to_drm(adev), "Couldn't allocate DC DMUB server!\n"); | |
| 1410 | − return -ENOMEM; | |
| 980 | + if (adev->dm.dc) { | |
| 981 | + dc_deinit_callbacks(adev->dm.dc); | |
| 982 | + dc_dmub_srv_destroy(&adev->dm.dc->ctx->dmub_srv); | |
| 983 | + if (dc_enable_dmub_notifications(adev->dm.dc)) { | |
| 984 | + kfree(adev->dm.dmub_notify); | |
| 985 | + adev->dm.dmub_notify = NULL; | |
| 986 | + destroy_workqueue(adev->dm.delayed_hpd_wq); | |
| 987 | + adev->dm.delayed_hpd_wq = NULL; | |
| 988 | + } | |
| 1411 | 989 | } |
| 1412 | 990 | |
| 1413 | − drm_info(adev_to_drm(adev), "DMUB hardware initialized: version=0x%08X\n", | |
| 1414 | − adev->dm.dmcub_fw_version); | |
| 991 | + if (adev->dm.dmub_bo) | |
| 992 | + amdgpu_bo_free_kernel(&adev->dm.dmub_bo, | |
| 993 | + &adev->dm.dmub_bo_gpu_addr, | |
| 994 | + &adev->dm.dmub_bo_cpu_addr); | |
| 995 | + | |
| 996 | + if (adev->dm.boot_time_crc_info.bo_ptr) | |
| 997 | + amdgpu_bo_free_kernel(&adev->dm.boot_time_crc_info.bo_ptr, | |
| 998 | + &adev->dm.boot_time_crc_info.gpu_addr, | |
| 999 | + &adev->dm.boot_time_crc_info.cpu_addr); | |
| 1000 | + | |
| 1001 | + if (adev->dm.hpd_rx_offload_wq && adev->dm.dc) { | |
| 1002 | + for (i = 0; i < adev->dm.dc->caps.max_links; i++) { | |
| 1003 | + if (adev->dm.hpd_rx_offload_wq[i].wq) { | |
| 1004 | + destroy_workqueue(adev->dm.hpd_rx_offload_wq[i].wq); | |
| 1005 | + adev->dm.hpd_rx_offload_wq[i].wq = NULL; | |
| 1006 | + } | |
| 1007 | + } | |
| 1008 | + | |
| 1009 | + kfree(adev->dm.hpd_rx_offload_wq); | |
| 1010 | + adev->dm.hpd_rx_offload_wq = NULL; | |
| 1011 | + } | |
| 1415 | 1012 | |
| 1416 | − /* Keeping sanity checks off if | |
| 1417 | − * DCN31 >= 4.0.59.0 | |
| 1418 | − * DCN314 >= 8.0.16.0 | |
| 1419 | − * Otherwise, turn on sanity checks | |
| 1013 | + /* DC Destroy TODO: Replace destroy DAL */ | |
| 1014 | + if (adev->dm.dc) | |
| 1015 | + dc_destroy(&adev->dm.dc); | |
| 1016 | + /* | |
| 1017 | + * TODO: pageflip, vlank interrupt | |
| 1018 | + * | |
| 1019 | + * amdgpu_dm_irq_fini(adev); | |
| 1420 | 1020 | */ |
| 1421 | − switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { | |
| 1422 | − case IP_VERSION(3, 1, 2): | |
| 1423 | − case IP_VERSION(3, 1, 3): | |
| 1424 | − if (adev->dm.dmcub_fw_version && | |
| 1425 | − adev->dm.dmcub_fw_version >= DMUB_FW_VERSION(4, 0, 0) && | |
| 1426 | − adev->dm.dmcub_fw_version < DMUB_FW_VERSION(4, 0, 59)) | |
| 1427 | − adev->dm.dc->debug.sanity_checks = true; | |
| 1021 | + | |
| 1022 | + if (adev->dm.cgs_device) { | |
| 1023 | + amdgpu_cgs_destroy_device(adev->dm.cgs_device); | |
| 1024 | + adev->dm.cgs_device = NULL; | |
| 1025 | + } | |
| 1026 | + if (adev->dm.freesync_module) { | |
| 1027 | + mod_freesync_destroy(adev->dm.freesync_module); | |
| 1028 | + adev->dm.freesync_module = NULL; | |
| 1029 | + } | |
| 1030 | + | |
| 1031 | + if (adev->dm.power_module) { | |
| 1032 | + mod_power_destroy(adev->dm.power_module); | |
| 1033 | + adev->dm.power_module = NULL; | |
| 1034 | + } | |
| 1035 | + mutex_destroy(&adev->dm.audio_lock); | |
| 1036 | + mutex_destroy(&adev->dm.dc_lock); | |
| 1037 | + mutex_destroy(&adev->dm.dpia_aux_lock); | |
| 1038 | +} | |
| 1039 | + | |
| 1040 | +static int load_dmcu_fw(struct amdgpu_device *adev) | |
| 1041 | +{ | |
| 1042 | + const char *fw_name_dmcu = NULL; | |
| 1043 | + int r; | |
| 1044 | + const struct dmcu_firmware_header_v1_0 *hdr; | |
| 1045 | + | |
| 1046 | + switch (adev->asic_type) { | |
| 1047 | +#if defined(CONFIG_DRM_AMD_DC_SI) | |
| 1048 | + case CHIP_TAHITI: | |
| 1049 | + case CHIP_PITCAIRN: | |
| 1050 | + case CHIP_VERDE: | |
| 1051 | + case CHIP_OLAND: | |
| 1052 | +#endif | |
| 1053 | + case CHIP_BONAIRE: | |
| 1054 | + case CHIP_HAWAII: | |
| 1055 | + case CHIP_KAVERI: | |
| 1056 | + case CHIP_KABINI: | |
| 1057 | + case CHIP_MULLINS: | |
| 1058 | + case CHIP_TONGA: | |
| 1059 | + case CHIP_FIJI: | |
| 1060 | + case CHIP_CARRIZO: | |
| 1061 | + case CHIP_STONEY: | |
| 1062 | + case CHIP_POLARIS11: | |
| 1063 | + case CHIP_POLARIS10: | |
| 1064 | + case CHIP_POLARIS12: | |
| 1065 | + case CHIP_VEGAM: | |
| 1066 | + case CHIP_VEGA10: | |
| 1067 | + case CHIP_VEGA12: | |
| 1068 | + case CHIP_VEGA20: | |
| 1069 | + return 0; | |
| 1070 | + case CHIP_NAVI12: | |
| 1071 | + fw_name_dmcu = FIRMWARE_NAVI12_DMCU; | |
| 1428 | 1072 | break; |
| 1429 | − case IP_VERSION(3, 1, 4): | |
| 1430 | − if (adev->dm.dmcub_fw_version && | |
| 1431 | − adev->dm.dmcub_fw_version >= DMUB_FW_VERSION(4, 0, 0) && | |
| 1432 | − adev->dm.dmcub_fw_version < DMUB_FW_VERSION(8, 0, 16)) | |
| 1433 | − adev->dm.dc->debug.sanity_checks = true; | |
| 1073 | + case CHIP_RAVEN: | |
| 1074 | + if (ASICREV_IS_PICASSO(adev->external_rev_id)) | |
| 1075 | + fw_name_dmcu = FIRMWARE_RAVEN_DMCU; | |
| 1076 | + else if (ASICREV_IS_RAVEN2(adev->external_rev_id)) | |
| 1077 | + fw_name_dmcu = FIRMWARE_RAVEN_DMCU; | |
| 1078 | + else | |
| 1079 | + return 0; | |
| 1434 | 1080 | break; |
| 1435 | 1081 | default: |
| 1436 | − break; | |
| 1082 | + switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { | |
| 1083 | + case IP_VERSION(2, 0, 2): | |
| 1084 | + case IP_VERSION(2, 0, 3): | |
| 1085 | + case IP_VERSION(2, 0, 0): | |
| 1086 | + case IP_VERSION(2, 1, 0): | |
| 1087 | + case IP_VERSION(3, 0, 0): | |
| 1088 | + case IP_VERSION(3, 0, 2): | |
| 1089 | + case IP_VERSION(3, 0, 3): | |
| 1090 | + case IP_VERSION(3, 0, 1): | |
| 1091 | + case IP_VERSION(3, 1, 2): | |
| 1092 | + case IP_VERSION(3, 1, 3): | |
| 1093 | + case IP_VERSION(3, 1, 4): | |
| 1094 | + case IP_VERSION(3, 1, 5): | |
| 1095 | + case IP_VERSION(3, 1, 6): | |
| 1096 | + case IP_VERSION(3, 2, 0): | |
| 1097 | + case IP_VERSION(3, 2, 1): | |
| 1098 | + case IP_VERSION(3, 5, 0): | |
| 1099 | + case IP_VERSION(3, 5, 1): | |
| 1100 | + case IP_VERSION(3, 6, 0): | |
| 1101 | + case IP_VERSION(4, 0, 1): | |
| 1102 | + case IP_VERSION(4, 2, 0): | |
| 1103 | + case IP_VERSION(4, 2, 1): | |
| 1104 | + return 0; | |
| 1105 | + default: | |
| 1106 | + break; | |
| 1107 | + } | |
| 1108 | + drm_err(adev_to_drm(adev), "Unsupported ASIC type: 0x%X\n", adev->asic_type); | |
| 1109 | + return -EINVAL; | |
| 1110 | + } | |
| 1111 | + | |
| 1112 | + if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { | |
| 1113 | + drm_dbg_kms(adev_to_drm(adev), "dm: DMCU firmware not supported on direct or SMU loading\n"); | |
| 1114 | + return 0; | |
| 1115 | + } | |
| 1116 | + | |
| 1117 | + r = amdgpu_ucode_request(adev, &adev->dm.fw_dmcu, AMDGPU_UCODE_REQUIRED, | |
| 1118 | + "%s", fw_name_dmcu); | |
| 1119 | + if (r == -ENODEV) { | |
| 1120 | + /* DMCU firmware is not necessary, so don't raise a fuss if it's missing */ | |
| 1121 | + drm_dbg_kms(adev_to_drm(adev), "dm: DMCU firmware not found\n"); | |
| 1122 | + adev->dm.fw_dmcu = NULL; | |
| 1123 | + return 0; | |
| 1124 | + } | |
| 1125 | + if (r) { | |
| 1126 | + drm_err(adev_to_drm(adev), "amdgpu_dm: Can't validate firmware \"%s\"\n", | |
| 1127 | + fw_name_dmcu); | |
| 1128 | + amdgpu_ucode_release(&adev->dm.fw_dmcu); | |
| 1129 | + return r; | |
| 1437 | 1130 | } |
| 1438 | 1131 | |
| 1132 | + hdr = (const struct dmcu_firmware_header_v1_0 *)adev->dm.fw_dmcu->data; | |
| 1133 | + adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_ERAM].ucode_id = AMDGPU_UCODE_ID_DMCU_ERAM; | |
| 1134 | + adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_ERAM].fw = adev->dm.fw_dmcu; | |
| 1135 | + adev->firmware.fw_size += | |
| 1136 | + ALIGN(le32_to_cpu(hdr->header.ucode_size_bytes) - le32_to_cpu(hdr->intv_size_bytes), PAGE_SIZE); | |
| 1137 | + | |
| 1138 | + adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_INTV].ucode_id = AMDGPU_UCODE_ID_DMCU_INTV; | |
| 1139 | + adev->firmware.ucode[AMDGPU_UCODE_ID_DMCU_INTV].fw = adev->dm.fw_dmcu; | |
| 1140 | + adev->firmware.fw_size += | |
| 1141 | + ALIGN(le32_to_cpu(hdr->intv_size_bytes), PAGE_SIZE); | |
| 1142 | + | |
| 1143 | + adev->dm.dmcu_fw_version = le32_to_cpu(hdr->header.ucode_version); | |
| 1144 | + | |
| 1145 | + drm_dbg_kms(adev_to_drm(adev), "PSP loading DMCU firmware\n"); | |
| 1146 | + | |
| 1439 | 1147 | return 0; |
| 1440 | 1148 | } |
| 1441 | 1149 | |
| 1442 | −static void dm_dmub_hw_resume(struct amdgpu_device *adev) | |
| 1150 | +static int dm_sw_init(struct amdgpu_ip_block *ip_block) | |
| 1443 | 1151 | { |
| 1444 | − struct dmub_srv *dmub_srv = adev->dm.dmub_srv; | |
| 1445 | − enum dmub_status status; | |
| 1446 | − bool init; | |
| 1152 | + struct amdgpu_device *adev = ip_block->adev; | |
| 1447 | 1153 | int r; |
| 1448 | 1154 | |
| 1449 | − if (!dmub_srv) { | |
| 1450 | − /* DMUB isn't supported on the ASIC. */ | |
| 1451 | − return; | |
| 1155 | + adev->dm.cgs_device = amdgpu_cgs_create_device(adev); | |
| 1156 | + | |
| 1157 | + if (!adev->dm.cgs_device) { | |
| 1158 | + drm_err(adev_to_drm(adev), "failed to create cgs device.\n"); | |
| 1159 | + return -EINVAL; | |
| 1452 | 1160 | } |
| 1453 | 1161 | |
| 1454 | − status = dmub_srv_is_hw_init(dmub_srv, &init); | |
| 1455 | − if (status != DMUB_STATUS_OK) | |
| 1456 | − drm_warn(adev_to_drm(adev), "DMUB hardware init check failed: %d\n", status); | |
| 1162 | + /* Moved from dm init since we need to use allocations for storing bounding box data */ | |
| 1163 | + INIT_LIST_HEAD(&adev->dm.da_list); | |
| 1457 | 1164 | |
| 1458 | − if (status == DMUB_STATUS_OK && init) { | |
| 1459 | − /* Wait for firmware load to finish. */ | |
| 1460 | − status = dmub_srv_wait_for_auto_load(dmub_srv, 100000); | |
| 1461 | − if (status != DMUB_STATUS_OK) | |
| 1462 | − drm_warn(adev_to_drm(adev), "Wait for DMUB auto-load failed: %d\n", status); | |
| 1463 | − } else { | |
| 1464 | − /* Perform the full hardware initialization. */ | |
| 1465 | − r = dm_dmub_hw_init(adev); | |
| 1466 | − if (r) | |
| 1467 | − drm_err(adev_to_drm(adev), "DMUB interface failed to initialize: status=%d\n", r); | |
| 1468 | − } | |
| 1165 | + r = dm_dmub_sw_init(adev); | |
| 1166 | + if (r) | |
| 1167 | + return r; | |
| 1168 | + | |
| 1169 | + return load_dmcu_fw(adev); | |
| 1469 | 1170 | } |
| 1470 | 1171 | |
| 1471 | −static void mmhub_read_system_context(struct amdgpu_device *adev, struct dc_phy_addr_space_config *pa_config) | |
| 1172 | +static int dm_sw_fini(struct amdgpu_ip_block *ip_block) | |
| 1472 | 1173 | { |
| 1473 | − u64 pt_base; | |
| 1474 | − u32 logical_addr_low; | |
| 1475 | − u32 logical_addr_high; | |
| 1476 | − u32 agp_base, agp_bot, agp_top; | |
| 1477 | − PHYSICAL_ADDRESS_LOC page_table_start, page_table_end, page_table_base; | |
| 1174 | + struct amdgpu_device *adev = ip_block->adev; | |
| 1175 | + struct dal_allocation *da; | |
| 1478 | 1176 | |
| 1479 | − memset(pa_config, 0, sizeof(*pa_config)); | |
| 1177 | + list_for_each_entry(da, &adev->dm.da_list, list) { | |
| 1178 | + if (adev->dm.bb_from_dmub == (void *) da->cpu_ptr) { | |
| 1179 | + amdgpu_bo_free_kernel(&da->bo, &da->gpu_addr, &da->cpu_ptr); | |
| 1180 | + list_del(&da->list); | |
| 1181 | + kfree(da); | |
| 1182 | + adev->dm.bb_from_dmub = NULL; | |
| 1183 | + break; | |
| 1184 | + } | |
| 1185 | + } | |
| 1480 | 1186 | |
| 1481 | − agp_base = 0; | |
| 1482 | − agp_bot = adev->gmc.agp_start >> 24; | |
| 1483 | − agp_top = adev->gmc.agp_end >> 24; | |
| 1484 | 1187 | |
| 1485 | − /* AGP aperture is disabled */ | |
| 1486 | − if (agp_bot > agp_top) { | |
| 1487 | − logical_addr_low = adev->gmc.fb_start >> 18; | |
| 1488 | − if (adev->apu_flags & (AMD_APU_IS_RAVEN2 | | |
| 1489 | − AMD_APU_IS_RENOIR | | |
| 1490 | − AMD_APU_IS_GREEN_SARDINE)) | |
| 1491 | − /* | |
| 1492 | − * Raven2 has a HW issue that it is unable to use the vram which | |
| 1493 | − * is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR. So here is the | |
| 1494 | − * workaround that increase system aperture high address (add 1) | |
| 1495 | − * to get rid of the VM fault and hardware hang. | |
| 1496 | − */ | |
| 1497 | − logical_addr_high = (adev->gmc.fb_end >> 18) + 0x1; | |
| 1498 | − else | |
| 1499 | − logical_addr_high = adev->gmc.fb_end >> 18; | |
| 1500 | − } else { | |
| 1501 | − logical_addr_low = min(adev->gmc.fb_start, adev->gmc.agp_start) >> 18; | |
| 1502 | − if (adev->apu_flags & (AMD_APU_IS_RAVEN2 | | |
| 1503 | − AMD_APU_IS_RENOIR | | |
| 1504 | − AMD_APU_IS_GREEN_SARDINE)) | |
| 1505 | − /* | |
| 1506 | − * Raven2 has a HW issue that it is unable to use the vram which | |
| 1507 | − * is out of MC_VM_SYSTEM_APERTURE_HIGH_ADDR. So here is the | |
| 1508 | − * workaround that increase system aperture high address (add 1) | |
| 1509 | − * to get rid of the VM fault and hardware hang. | |
| 1510 | − */ | |
| 1511 | − logical_addr_high = max((adev->gmc.fb_end >> 18) + 0x1, adev->gmc.agp_end >> 18); | |
| 1512 | − else | |
| 1513 | − logical_addr_high = max(adev->gmc.fb_end, adev->gmc.agp_end) >> 18; | |
| 1514 | − } | |
| 1515 | − | |
| 1516 | − pt_base = amdgpu_gmc_pd_addr(adev->gart.bo); | |
| 1517 | − | |
| 1518 | − page_table_start.high_part = upper_32_bits(adev->gmc.gart_start >> | |
| 1519 | − AMDGPU_GPU_PAGE_SHIFT); | |
| 1520 | − page_table_start.low_part = lower_32_bits(adev->gmc.gart_start >> | |
| 1521 | − AMDGPU_GPU_PAGE_SHIFT); | |
| 1522 | − page_table_end.high_part = upper_32_bits(adev->gmc.gart_end >> | |
| 1523 | − AMDGPU_GPU_PAGE_SHIFT); | |
| 1524 | − page_table_end.low_part = lower_32_bits(adev->gmc.gart_end >> | |
| 1525 | − AMDGPU_GPU_PAGE_SHIFT); | |
| 1526 | − page_table_base.high_part = upper_32_bits(pt_base); | |
| 1527 | − page_table_base.low_part = lower_32_bits(pt_base); | |
| 1528 | − | |
| 1529 | − pa_config->system_aperture.start_addr = (uint64_t)logical_addr_low << 18; | |
| 1530 | − pa_config->system_aperture.end_addr = (uint64_t)logical_addr_high << 18; | |
| 1531 | − | |
| 1532 | − pa_config->system_aperture.agp_base = (uint64_t)agp_base << 24; | |
| 1533 | − pa_config->system_aperture.agp_bot = (uint64_t)agp_bot << 24; | |
| 1534 | − pa_config->system_aperture.agp_top = (uint64_t)agp_top << 24; | |
| 1535 | − | |
| 1536 | − pa_config->system_aperture.fb_base = adev->gmc.fb_start; | |
| 1537 | − pa_config->system_aperture.fb_offset = adev->vm_manager.vram_base_offset; | |
| 1538 | − pa_config->system_aperture.fb_top = adev->gmc.fb_end; | |
| 1188 | + kfree(adev->dm.dmub_fb_info); | |
| 1189 | + adev->dm.dmub_fb_info = NULL; | |
| 1539 | 1190 | |
| 1540 | − pa_config->gart_config.page_table_start_addr = page_table_start.quad_part << 12; | |
| 1541 | − pa_config->gart_config.page_table_end_addr = page_table_end.quad_part << 12; | |
| 1542 | − pa_config->gart_config.page_table_base_addr = page_table_base.quad_part; | |
| 1191 | + if (adev->dm.dmub_srv) { | |
| 1192 | + dmub_srv_destroy(adev->dm.dmub_srv); | |
| 1193 | + kfree(adev->dm.dmub_srv); | |
| 1194 | + adev->dm.dmub_srv = NULL; | |
| 1195 | + } | |
| 1543 | 1196 | |
| 1544 | − pa_config->is_hvm_enabled = adev->mode_info.gpu_vm_support; | |
| 1197 | + amdgpu_ucode_release(&adev->dm.dmub_fw); | |
| 1198 | + amdgpu_ucode_release(&adev->dm.fw_dmcu); | |
| 1545 | 1199 | |
| 1200 | + return 0; | |
| 1546 | 1201 | } |
| 1547 | 1202 | |
| 1548 | −static void force_connector_state( | |
| 1549 | − struct amdgpu_dm_connector *aconnector, | |
| 1550 | − enum drm_connector_force force_state) | |
| 1551 | −{ | |
| 1552 | − struct drm_connector *connector = &aconnector->base; | |
| 1553 | − | |
| 1554 | − mutex_lock(&connector->dev->mode_config.mutex); | |
| 1555 | − aconnector->base.force = force_state; | |
| 1556 | − mutex_unlock(&connector->dev->mode_config.mutex); | |
| 1557 | 1203 | |
| 1558 | − mutex_lock(&aconnector->hpd_lock); | |
| 1559 | − drm_kms_helper_connector_hotplug_event(connector); | |
| 1560 | − mutex_unlock(&aconnector->hpd_lock); | |
| 1561 | −} | |
| 1562 | − | |
| 1563 | −static void dm_handle_hpd_rx_offload_work(struct work_struct *work) | |
| 1204 | +static void amdgpu_dm_boot_time_crc_init(struct amdgpu_device *adev) | |
| 1564 | 1205 | { |
| 1565 | − struct hpd_rx_irq_offload_work *offload_work; | |
| 1566 | − struct amdgpu_dm_connector *aconnector; | |
| 1567 | − struct dc_link *dc_link; | |
| 1568 | − struct amdgpu_device *adev; | |
| 1569 | − enum dc_connection_type new_connection_type = dc_connection_none; | |
| 1570 | − unsigned long flags; | |
| 1571 | − union test_response test_response; | |
| 1572 | − | |
| 1573 | − memset(&test_response, 0, sizeof(test_response)); | |
| 1574 | − | |
| 1575 | − offload_work = container_of(work, struct hpd_rx_irq_offload_work, work); | |
| 1576 | − aconnector = offload_work->offload_wq->aconnector; | |
| 1577 | − adev = offload_work->adev; | |
| 1206 | + struct dm_boot_time_crc_info *bootcrc_info = NULL; | |
| 1207 | + struct dmub_srv *dmub = NULL; | |
| 1208 | + union dmub_fw_boot_options option = {0}; | |
| 1209 | + int ret = 0; | |
| 1210 | + const uint32_t fb_size = 3 * 1024 * 1024; /* 3MB for DCC pattern */ | |
| 1578 | 1211 | |
| 1579 | − if (!aconnector) { | |
| 1580 | − drm_err(adev_to_drm(adev), "Can't retrieve aconnector in hpd_rx_irq_offload_work"); | |
| 1581 | − goto skip; | |
| 1212 | + if (!adev || !adev->dm.dc || !adev->dm.dc->ctx || | |
| 1213 | + !adev->dm.dc->ctx->dmub_srv) { | |
| 1214 | + return; | |
| 1582 | 1215 | } |
| 1583 | 1216 | |
| 1584 | − dc_link = aconnector->dc_link; | |
| 1585 | − | |
| 1586 | − mutex_lock(&aconnector->hpd_lock); | |
| 1587 | − if (!dc_link_detect_connection_type(dc_link, &new_connection_type)) | |
| 1588 | − drm_err(adev_to_drm(adev), "KMS: Failed to detect connector\n"); | |
| 1589 | − mutex_unlock(&aconnector->hpd_lock); | |
| 1590 | − | |
| 1591 | − if (new_connection_type == dc_connection_none) | |
| 1592 | − goto skip; | |
| 1593 | − | |
| 1594 | − if (amdgpu_in_reset(adev)) | |
| 1595 | − goto skip; | |
| 1217 | + dmub = adev->dm.dc->ctx->dmub_srv->dmub; | |
| 1218 | + bootcrc_info = &adev->dm.boot_time_crc_info; | |
| 1596 | 1219 | |
| 1597 | − if (offload_work->data.bytes.device_service_irq.bits.UP_REQ_MSG_RDY || | |
| 1598 | − offload_work->data.bytes.device_service_irq.bits.DOWN_REP_MSG_RDY) { | |
| 1599 | − dm_handle_mst_sideband_msg_ready_event(&aconnector->mst_mgr, DOWN_OR_UP_MSG_RDY_EVENT); | |
| 1600 | − spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags); | |
| 1601 | − offload_work->offload_wq->is_handling_mst_msg_rdy_event = false; | |
| 1602 | − spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags); | |
| 1603 | − goto skip; | |
| 1220 | + if (!dmub || !dmub->hw_funcs.get_fw_boot_option) { | |
| 1221 | + drm_dbg(adev_to_drm(adev), "failed to init boot time crc buffer\n"); | |
| 1222 | + return; | |
| 1604 | 1223 | } |
| 1605 | 1224 | |
| 1606 | − mutex_lock(&adev->dm.dc_lock); | |
| 1607 | − if (offload_work->data.bytes.device_service_irq.bits.AUTOMATED_TEST) { | |
| 1608 | − dc_link_dp_handle_automated_test(dc_link); | |
| 1609 | − | |
| 1610 | − if (aconnector->timing_changed) { | |
| 1611 | − /* force connector disconnect and reconnect */ | |
| 1612 | − force_connector_state(aconnector, DRM_FORCE_OFF); | |
| 1613 | − msleep(100); | |
| 1614 | − force_connector_state(aconnector, DRM_FORCE_UNSPECIFIED); | |
| 1615 | − } | |
| 1616 | − | |
| 1617 | − test_response.bits.ACK = 1; | |
| 1618 | − | |
| 1619 | − core_link_write_dpcd( | |
| 1620 | − dc_link, | |
| 1621 | − DP_TEST_RESPONSE, | |
| 1622 | − &test_response.raw, | |
| 1623 | − sizeof(test_response)); | |
| 1624 | − } else if ((dc_link->connector_signal != SIGNAL_TYPE_EDP) && | |
| 1625 | − dc_link_check_link_loss_status(dc_link, &offload_work->data) && | |
| 1626 | − dc_link_dp_allow_hpd_rx_irq(dc_link)) { | |
| 1627 | − /* offload_work->data is from handle_hpd_rx_irq-> | |
| 1628 | − * schedule_hpd_rx_offload_work.this is defer handle | |
| 1629 | − * for hpd short pulse. upon here, link status may be | |
| 1630 | − * changed, need get latest link status from dpcd | |
| 1631 | − * registers. if link status is good, skip run link | |
| 1632 | − * training again. | |
| 1633 | − */ | |
| 1634 | − union hpd_irq_data irq_data; | |
| 1225 | + option = dmub->hw_funcs.get_fw_boot_option(dmub); | |
| 1635 | 1226 | |
| 1636 | − memset(&irq_data, 0, sizeof(irq_data)); | |
| 1227 | + /* Return if boot time CRC is not enabled */ | |
| 1228 | + if (option.bits.bootcrc_en_at_S0i3 == 0) | |
| 1229 | + return; | |
| 1637 | 1230 | |
| 1638 | − /* before dc_link_dp_handle_link_loss, allow new link lost handle | |
| 1639 | − * request be added to work queue if link lost at end of dc_link_ | |
| 1640 | − * dp_handle_link_loss | |
| 1641 | − */ | |
| 1642 | − spin_lock_irqsave(&offload_work->offload_wq->offload_lock, flags); | |
| 1643 | − offload_work->offload_wq->is_handling_link_loss = false; | |
| 1644 | − spin_unlock_irqrestore(&offload_work->offload_wq->offload_lock, flags); | |
| 1231 | + /* Create a buffer for boot time CRC */ | |
| 1232 | + ret = amdgpu_bo_create_kernel(adev, fb_size, PAGE_SIZE, | |
| 1233 | + AMDGPU_GEM_DOMAIN_VRAM | AMDGPU_GEM_DOMAIN_GTT, | |
| 1234 | + &bootcrc_info->bo_ptr, | |
| 1235 | + &bootcrc_info->gpu_addr, | |
| 1236 | + &bootcrc_info->cpu_addr); | |
| 1645 | 1237 | |
| 1646 | − if ((dc_link_dp_read_hpd_rx_irq_data(dc_link, &irq_data) == DC_OK) && | |
| 1647 | − dc_link_check_link_loss_status(dc_link, &irq_data)) | |
| 1648 | − dc_link_dp_handle_link_loss(dc_link); | |
| 1649 | − } | |
| 1650 | − mutex_unlock(&adev->dm.dc_lock); | |
| 1238 | + if (ret) { | |
| 1239 | + drm_dbg(adev_to_drm(adev), "failed to create boot time crc buffer\n"); | |
| 1240 | + } else { | |
| 1241 | + bootcrc_info->size = fb_size; | |
| 1651 | 1242 | |
| 1652 | −skip: | |
| 1653 | − kfree(offload_work); | |
| 1243 | + drm_dbg(adev_to_drm(adev), "boot time crc buffer created addr 0x%llx, size %u\n", | |
| 1244 | + bootcrc_info->gpu_addr, bootcrc_info->size); | |
| 1654 | 1245 | |
| 1246 | + /* Send the buffer info to DMUB */ | |
| 1247 | + dc_dmub_srv_boot_time_crc_init(adev->dm.dc, | |
| 1248 | + bootcrc_info->gpu_addr, bootcrc_info->size); | |
| 1249 | + } | |
| 1655 | 1250 | } |
| 1656 | 1251 | |
| 1657 | −static struct hpd_rx_irq_offload_work_queue *hpd_rx_irq_create_workqueue(struct amdgpu_device *adev) | |
| 1252 | +static int dm_late_init(struct amdgpu_ip_block *ip_block) | |
| 1658 | 1253 | { |
| 1659 | − struct dc *dc = adev->dm.dc; | |
| 1660 | − int max_caps = dc->caps.max_links; | |
| 1661 | − int i = 0; | |
| 1662 | − struct hpd_rx_irq_offload_work_queue *hpd_rx_offload_wq = NULL; | |
| 1663 | − | |
| 1664 | − hpd_rx_offload_wq = kzalloc_objs(*hpd_rx_offload_wq, max_caps); | |
| 1665 | − | |
| 1666 | − if (!hpd_rx_offload_wq) | |
| 1667 | − return NULL; | |
| 1668 | − | |
| 1669 | − | |
| 1670 | − for (i = 0; i < max_caps; i++) { | |
| 1671 | − hpd_rx_offload_wq[i].wq = | |
| 1672 | − create_singlethread_workqueue("amdgpu_dm_hpd_rx_offload_wq"); | |
| 1673 | − | |
| 1674 | − if (hpd_rx_offload_wq[i].wq == NULL) { | |
| 1675 | − drm_err(adev_to_drm(adev), "create amdgpu_dm_hpd_rx_offload_wq fail!"); | |
| 1676 | − goto out_err; | |
| 1677 | − } | |
| 1254 | + struct amdgpu_device *adev = ip_block->adev; | |
| 1678 | 1255 | |
| 1679 | − spin_lock_init(&hpd_rx_offload_wq[i].offload_lock); | |
| 1680 | − } | |
| 1256 | + struct dmcu_iram_parameters params; | |
| 1257 | + unsigned int linear_lut[16]; | |
| 1258 | + int i; | |
| 1259 | + struct dmcu *dmcu = NULL; | |
| 1681 | 1260 | |
| 1682 | − return hpd_rx_offload_wq; | |
| 1261 | + dmcu = adev->dm.dc->res_pool->dmcu; | |
| 1683 | 1262 | |
| 1684 | −out_err: | |
| 1685 | − for (i = 0; i < max_caps; i++) { | |
| 1686 | − if (hpd_rx_offload_wq[i].wq) | |
| 1687 | − destroy_workqueue(hpd_rx_offload_wq[i].wq); | |
| 1688 | − } | |
| 1689 | − kfree(hpd_rx_offload_wq); | |
| 1690 | − return NULL; | |
| 1691 | −} | |
| 1263 | + /* Init the boot time CRC (skip in resume) */ | |
| 1264 | + if ((adev->in_suspend == 0) && | |
| 1265 | + (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(3, 6, 0))) | |
| 1266 | + amdgpu_dm_boot_time_crc_init(adev); | |
| 1692 | 1267 | |
| 1693 | −struct amdgpu_stutter_quirk { | |
| 1694 | − u16 chip_vendor; | |
| 1695 | − u16 chip_device; | |
| 1696 | − u16 subsys_vendor; | |
| 1697 | − u16 subsys_device; | |
| 1698 | − u8 revision; | |
| 1699 | −}; | |
| 1268 | + for (i = 0; i < 16; i++) | |
| 1269 | + linear_lut[i] = 0xFFFF * i / 15; | |
| 1700 | 1270 | |
| 1701 | −static const struct amdgpu_stutter_quirk amdgpu_stutter_quirk_list[] = { | |
| 1702 | − /* https://bugzilla.kernel.org/show_bug.cgi?id=214417 */ | |
| 1703 | − { 0x1002, 0x15dd, 0x1002, 0x15dd, 0xc8 }, | |
| 1704 | − { 0, 0, 0, 0, 0 }, | |
| 1705 | −}; | |
| 1271 | + params.set = 0; | |
| 1272 | + params.backlight_ramping_override = false; | |
| 1273 | + params.backlight_ramping_start = 0xCCCC; | |
| 1274 | + params.backlight_ramping_reduction = 0xCCCCCCCC; | |
| 1275 | + params.backlight_lut_array_size = 16; | |
| 1276 | + params.backlight_lut_array = linear_lut; | |
| 1706 | 1277 | |
| 1707 | −static bool dm_should_disable_stutter(struct pci_dev *pdev) | |
| 1708 | −{ | |
| 1709 | − const struct amdgpu_stutter_quirk *p = amdgpu_stutter_quirk_list; | |
| 1278 | + /* Min backlight level after ABM reduction, Don't allow below 1% | |
| 1279 | + * 0xFFFF x 0.01 = 0x28F | |
| 1280 | + */ | |
| 1281 | + params.min_abm_backlight = 0x28F; | |
| 1282 | + /* In the case where abm is implemented on dmcub, | |
| 1283 | + * dmcu object will be null. | |
| 1284 | + * ABM 2.4 and up are implemented on dmcub. | |
| 1285 | + */ | |
| 1286 | + if (dmcu) { | |
| 1287 | + if (!dmcu_load_iram(dmcu, params)) | |
| 1288 | + return -EINVAL; | |
| 1289 | + } else if (adev->dm.dc->ctx->dmub_srv) { | |
| 1290 | + struct dc_link *edp_links[MAX_NUM_EDP]; | |
| 1291 | + int edp_num; | |
| 1710 | 1292 | |
| 1711 | − while (p && p->chip_device != 0) { | |
| 1712 | − if (pdev->vendor == p->chip_vendor && | |
| 1713 | − pdev->device == p->chip_device && | |
| 1714 | − pdev->subsystem_vendor == p->subsys_vendor && | |
| 1715 | − pdev->subsystem_device == p->subsys_device && | |
| 1716 | − pdev->revision == p->revision) { | |
| 1717 | − return true; | |
| 1293 | + dc_get_edp_links(adev->dm.dc, edp_links, &edp_num); | |
| 1294 | + for (i = 0; i < edp_num; i++) { | |
| 1295 | + if (!dmub_init_abm_config(adev->dm.dc->res_pool, params, i)) | |
| 1296 | + return -EINVAL; | |
| 1718 | 1297 | } |
| 1719 | − ++p; | |
| 1720 | 1298 | } |
| 1721 | − return false; | |
| 1722 | −} | |
| 1723 | 1299 | |
| 1300 | + return amdgpu_dm_detect_mst_link_for_all_connectors(adev_to_drm(adev)); | |
| 1301 | +} | |
| 1724 | 1302 | |
| 1725 | −void* | |
| 1726 | −dm_allocate_gpu_mem( | |
| 1727 | − struct amdgpu_device *adev, | |
| 1728 | − enum dc_gpu_mem_alloc_type type, | |
| 1729 | − size_t size, | |
| 1730 | − long long *addr) | |
| 1303 | +static void resume_mst_branch_status(struct drm_dp_mst_topology_mgr *mgr) | |
| 1731 | 1304 | { |
| 1732 | − struct dal_allocation *da; | |
| 1733 | − u32 domain = (type == DC_MEM_ALLOC_TYPE_GART) ? | |
| 1734 | − AMDGPU_GEM_DOMAIN_GTT : AMDGPU_GEM_DOMAIN_VRAM; | |
| 1305 | + u8 buf[UUID_SIZE]; | |
| 1306 | + guid_t guid; | |
| 1735 | 1307 | int ret; |
| 1736 | 1308 | |
| 1737 | − da = kzalloc_obj(struct dal_allocation); | |
| 1738 | − if (!da) | |
| 1739 | − return NULL; | |
| 1309 | + mutex_lock(&mgr->lock); | |
| 1310 | + if (!mgr->mst_primary) | |
| 1311 | + goto out_fail; | |
| 1740 | 1312 | |
| 1741 | − ret = amdgpu_bo_create_kernel(adev, size, PAGE_SIZE, | |
| 1742 | − domain, &da->bo, | |
| 1743 | − &da->gpu_addr, &da->cpu_ptr); | |
| 1313 | + if (drm_dp_read_dpcd_caps(mgr->aux, mgr->dpcd) < 0) { | |
| 1314 | + drm_dbg_kms(mgr->dev, "dpcd read failed - undocked during suspend?\n"); | |
| 1315 | + goto out_fail; | |
| 1316 | + } | |
| 1744 | 1317 | |
| 1745 | − *addr = da->gpu_addr; | |
| 1318 | + ret = drm_dp_dpcd_writeb(mgr->aux, DP_MSTM_CTRL, | |
| 1319 | + DP_MST_EN | | |
| 1320 | + DP_UP_REQ_EN | | |
| 1321 | + DP_UPSTREAM_IS_SRC); | |
| 1322 | + if (ret < 0) { | |
| 1323 | + drm_dbg_kms(mgr->dev, "mst write failed - undocked during suspend?\n"); | |
| 1324 | + goto out_fail; | |
| 1325 | + } | |
| 1746 | 1326 | |
| 1747 | − if (ret) { | |
| 1748 | − kfree(da); | |
| 1749 | − return NULL; | |
| 1327 | + /* Some hubs forget their guids after they resume */ | |
| 1328 | + ret = drm_dp_dpcd_read(mgr->aux, DP_GUID, buf, sizeof(buf)); | |
| 1329 | + if (ret != sizeof(buf)) { | |
| 1330 | + drm_dbg_kms(mgr->dev, "dpcd read failed - undocked during suspend?\n"); | |
| 1331 | + goto out_fail; | |
| 1750 | 1332 | } |
| 1751 | 1333 | |
| 1752 | − /* add da to list in dm */ | |
| 1753 | − list_add(&da->list, &adev->dm.da_list); | |
| 1334 | + import_guid(&guid, buf); | |
| 1754 | 1335 | |
| 1755 | − return da->cpu_ptr; | |
| 1756 | −} | |
| 1336 | + if (guid_is_null(&guid)) { | |
| 1337 | + guid_gen(&guid); | |
| 1338 | + export_guid(buf, &guid); | |
| 1757 | 1339 | |
| 1758 | −void | |
| 1759 | −dm_free_gpu_mem( | |
| 1760 | − struct amdgpu_device *adev, | |
| 1761 | − enum dc_gpu_mem_alloc_type type, | |
| 1762 | − void *pvMem) | |
| 1763 | −{ | |
| 1764 | − struct dal_allocation *da; | |
| 1340 | + ret = drm_dp_dpcd_write(mgr->aux, DP_GUID, buf, sizeof(buf)); | |
| 1765 | 1341 | |
| 1766 | − /* walk the da list in DM */ | |
| 1767 | − list_for_each_entry(da, &adev->dm.da_list, list) { | |
| 1768 | − if (pvMem == da->cpu_ptr) { | |
| 1769 | − amdgpu_bo_free_kernel(&da->bo, &da->gpu_addr, &da->cpu_ptr); | |
| 1770 | − list_del(&da->list); | |
| 1771 | − kfree(da); | |
| 1772 | − break; | |
| 1342 | + if (ret != sizeof(buf)) { | |
| 1343 | + drm_dbg_kms(mgr->dev, "check mstb guid failed - undocked during suspend?\n"); | |
| 1344 | + goto out_fail; | |
| 1773 | 1345 | } |
| 1774 | 1346 | } |
| 1775 | 1347 | |
| 1348 | + guid_copy(&mgr->mst_primary->guid, &guid); | |
| 1349 | + | |
| 1350 | +out_fail: | |
| 1351 | + mutex_unlock(&mgr->lock); | |
| 1776 | 1352 | } |
| 1777 | 1353 | |
| 1778 | −static enum dmub_status | |
| 1779 | −dm_dmub_send_vbios_gpint_command(struct amdgpu_device *adev, | |
| 1780 | − enum dmub_gpint_command command_code, | |
| 1781 | − uint16_t param, | |
| 1782 | − uint32_t timeout_us) | |
| 1354 | +static void s3_handle_mst(struct drm_device *dev, bool suspend) | |
| 1783 | 1355 | { |
| 1784 | − union dmub_gpint_data_register reg, test; | |
| 1785 | − uint32_t i; | |
| 1786 | − | |
| 1787 | − /* Assume that VBIOS DMUB is ready to take commands */ | |
| 1788 | − | |
| 1789 | − reg.bits.status = 1; | |
| 1790 | − reg.bits.command_code = command_code; | |
| 1791 | − reg.bits.param = param; | |
| 1356 | + struct amdgpu_dm_connector *aconnector; | |
| 1357 | + struct drm_connector *connector; | |
| 1358 | + struct drm_connector_list_iter iter; | |
| 1359 | + struct drm_dp_mst_topology_mgr *mgr; | |
| 1792 | 1360 | |
| 1793 | − cgs_write_register(adev->dm.cgs_device, 0x34c0 + 0x01f8, reg.all); | |
| 1361 | + drm_connector_list_iter_begin(dev, &iter); | |
| 1362 | + drm_for_each_connector_iter(connector, &iter) { | |
| 1794 | 1363 | |
| 1795 | − for (i = 0; i < timeout_us; ++i) { | |
| 1796 | − udelay(1); | |
| 1364 | + if (connector->connector_type == DRM_MODE_CONNECTOR_WRITEBACK) | |
| 1365 | + continue; | |
| 1797 | 1366 | |
| 1798 | − /* Check if our GPINT got acked */ | |
| 1799 | − reg.bits.status = 0; | |
| 1800 | − test = (union dmub_gpint_data_register) | |
| 1801 | − cgs_read_register(adev->dm.cgs_device, 0x34c0 + 0x01f8); | |
| 1367 | + aconnector = to_amdgpu_dm_connector(connector); | |
| 1368 | + if (aconnector->dc_link->type != dc_connection_mst_branch || | |
| 1369 | + aconnector->mst_root) | |
| 1370 | + continue; | |
| 1802 | 1371 | |
| 1803 | − if (test.all == reg.all) | |
| 1804 | − return DMUB_STATUS_OK; | |
| 1805 | − } | |
| 1372 | + mgr = &aconnector->mst_mgr; | |
| 1373 | + | |
| 1374 | + if (suspend) { | |
| 1375 | + drm_dp_mst_topology_mgr_suspend(mgr); | |
| 1376 | + } else { | |
| 1377 | + /* if extended timeout is supported in hardware, | |
| 1378 | + * default to LTTPR timeout (3.2ms) first as a W/A for DP link layer | |
| 1379 | + * CTS 4.2.1.1 regression introduced by CTS specs requirement update. | |
| 1380 | + */ | |
| 1381 | + try_to_configure_aux_timeout(aconnector->dc_link->ddc, LINK_AUX_DEFAULT_LTTPR_TIMEOUT_PERIOD); | |
| 1382 | + if (!dp_is_lttpr_present(aconnector->dc_link)) | |
| 1383 | + try_to_configure_aux_timeout(aconnector->dc_link->ddc, LINK_AUX_DEFAULT_TIMEOUT_PERIOD); | |
| 1806 | 1384 | |
| 1807 | − return DMUB_STATUS_TIMEOUT; | |
| 1385 | + /* TODO: move resume_mst_branch_status() into drm mst resume again | |
| 1386 | + * once topology probing work is pulled out from mst resume into mst | |
| 1387 | + * resume 2nd step. mst resume 2nd step should be called after old | |
| 1388 | + * state getting restored (i.e. drm_atomic_helper_resume()). | |
| 1389 | + */ | |
| 1390 | + resume_mst_branch_status(mgr); | |
| 1391 | + } | |
| 1392 | + } | |
| 1393 | + drm_connector_list_iter_end(&iter); | |
| 1808 | 1394 | } |
| 1809 | 1395 | |
| 1810 | −static void *dm_dmub_get_vbios_bounding_box(struct amdgpu_device *adev) | |
| 1396 | +static int amdgpu_dm_smu_write_watermarks_table(struct amdgpu_device *adev) | |
| 1811 | 1397 | { |
| 1812 | − void *bb; | |
| 1813 | − long long addr; | |
| 1814 | − unsigned int bb_size; | |
| 1815 | − int i = 0; | |
| 1816 | − uint16_t chunk; | |
| 1817 | − enum dmub_gpint_command send_addrs[] = { | |
| 1818 | − DMUB_GPINT__SET_BB_ADDR_WORD0, | |
| 1819 | − DMUB_GPINT__SET_BB_ADDR_WORD1, | |
| 1820 | − DMUB_GPINT__SET_BB_ADDR_WORD2, | |
| 1821 | − DMUB_GPINT__SET_BB_ADDR_WORD3, | |
| 1822 | − }; | |
| 1823 | − enum dmub_status ret; | |
| 1398 | + int ret = 0; | |
| 1824 | 1399 | |
| 1400 | + /* This interface is for dGPU Navi1x.Linux dc-pplib interface depends | |
| 1401 | + * on window driver dc implementation. | |
| 1402 | + * For Navi1x, clock settings of dcn watermarks are fixed. the settings | |
| 1403 | + * should be passed to smu during boot up and resume from s3. | |
| 1404 | + * boot up: dc calculate dcn watermark clock settings within dc_create, | |
| 1405 | + * dcn20_resource_construct | |
| 1406 | + * then call pplib functions below to pass the settings to smu: | |
| 1407 | + * smu_set_watermarks_for_clock_ranges | |
| 1408 | + * smu_set_watermarks_table | |
| 1409 | + * navi10_set_watermarks_table | |
| 1410 | + * smu_write_watermarks_table | |
| 1411 | + * | |
| 1412 | + * For Renoir, clock settings of dcn watermark are also fixed values. | |
| 1413 | + * dc has implemented different flow for window driver: | |
| 1414 | + * dc_hardware_init / dc_set_power_state | |
| 1415 | + * dcn10_init_hw | |
| 1416 | + * notify_wm_ranges | |
| 1417 | + * set_wm_ranges | |
| 1418 | + * -- Linux | |
| 1419 | + * smu_set_watermarks_for_clock_ranges | |
| 1420 | + * renoir_set_watermarks_table | |
| 1421 | + * smu_write_watermarks_table | |
| 1422 | + * | |
| 1423 | + * For Linux, | |
| 1424 | + * dc_hardware_init -> amdgpu_dm_init | |
| 1425 | + * dc_set_power_state --> dm_resume | |
| 1426 | + * | |
| 1427 | + * therefore, this function apply to navi10/12/14 but not Renoir | |
| 1428 | + * * | |
| 1429 | + */ | |
| 1825 | 1430 | switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { |
| 1826 | − case IP_VERSION(4, 0, 1): | |
| 1827 | − bb_size = sizeof(struct dml2_soc_bb); | |
| 1828 | − break; | |
| 1829 | − case IP_VERSION(4, 2, 0): | |
| 1830 | − case IP_VERSION(4, 2, 1): | |
| 1831 | − bb_size = sizeof(struct dml2_soc_bb); | |
| 1431 | + case IP_VERSION(2, 0, 2): | |
| 1432 | + case IP_VERSION(2, 0, 0): | |
| 1832 | 1433 | break; |
| 1833 | 1434 | default: |
| 1834 | − return NULL; | |
| 1435 | + return 0; | |
| 1835 | 1436 | } |
| 1836 | 1437 | |
| 1837 | − bb = dm_allocate_gpu_mem(adev, | |
| 1838 | − DC_MEM_ALLOC_TYPE_GART, | |
| 1839 | − bb_size, | |
| 1840 | − &addr); | |
| 1841 | − if (!bb) | |
| 1842 | − return NULL; | |
| 1843 | − | |
| 1844 | − for (i = 0; i < 4; i++) { | |
| 1845 | − /* Extract 16-bit chunk */ | |
| 1846 | − chunk = ((uint64_t) addr >> (i * 16)) & 0xFFFF; | |
| 1847 | − /* Send the chunk */ | |
| 1848 | − ret = dm_dmub_send_vbios_gpint_command(adev, send_addrs[i], chunk, 30000); | |
| 1849 | − if (ret != DMUB_STATUS_OK) | |
| 1850 | − goto free_bb; | |
| 1438 | + ret = amdgpu_dpm_write_watermarks_table(adev); | |
| 1439 | + if (ret) { | |
| 1440 | + drm_err(adev_to_drm(adev), "Failed to update WMTABLE!\n"); | |
| 1441 | + return ret; | |
| 1851 | 1442 | } |
| 1852 | 1443 | |
| 1853 | − /* Now ask DMUB to copy the bb */ | |
| 1854 | − ret = dm_dmub_send_vbios_gpint_command(adev, DMUB_GPINT__BB_COPY, 1, 200000); | |
| 1855 | − if (ret != DMUB_STATUS_OK) | |
| 1856 | − goto free_bb; | |
| 1857 | − | |
| 1858 | − return bb; | |
| 1859 | − | |
| 1860 | −free_bb: | |
| 1861 | − dm_free_gpu_mem(adev, DC_MEM_ALLOC_TYPE_GART, (void *) bb); | |
| 1862 | − return NULL; | |
| 1863 | − | |
| 1444 | + return 0; | |
| 1864 | 1445 | } |
| 1865 | 1446 | |
| 1866 | −static enum dmub_ips_disable_type dm_get_default_ips_mode( | |
| 1867 | − struct amdgpu_device *adev) | |
| 1447 | +static int dm_oem_i2c_hw_init(struct amdgpu_device *adev) | |
| 1868 | 1448 | { |
| 1869 | − enum dmub_ips_disable_type ret = DMUB_IPS_ENABLE; | |
| 1449 | + struct amdgpu_display_manager *dm = &adev->dm; | |
| 1450 | + struct amdgpu_i2c_adapter *oem_i2c; | |
| 1451 | + struct ddc_service *oem_ddc_service; | |
| 1452 | + int r; | |
| 1870 | 1453 | |
| 1871 | − switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { | |
| 1872 | − case IP_VERSION(3, 5, 0): | |
| 1873 | − case IP_VERSION(3, 6, 0): | |
| 1874 | − case IP_VERSION(3, 5, 1): | |
| 1875 | − ret = DMUB_IPS_RCG_IN_ACTIVE_IPS2_IN_OFF; | |
| 1876 | − break; | |
| 1877 | − case IP_VERSION(4, 2, 0): | |
| 1878 | − case IP_VERSION(4, 2, 1): | |
| 1879 | − ret = DMUB_IPS_ENABLE; | |
| 1880 | − break; | |
| 1881 | − default: | |
| 1882 | − /* ASICs older than DCN35 do not have IPSs */ | |
| 1883 | − if (amdgpu_ip_version(adev, DCE_HWIP, 0) < IP_VERSION(3, 5, 0)) | |
| 1884 | − ret = DMUB_IPS_DISABLE_ALL; | |
| 1885 | − break; | |
| 1454 | + oem_ddc_service = dc_get_oem_i2c_device(adev->dm.dc); | |
| 1455 | + if (oem_ddc_service) { | |
| 1456 | + oem_i2c = amdgpu_dm_create_i2c(oem_ddc_service, true); | |
| 1457 | + if (!oem_i2c) { | |
| 1458 | + drm_info(adev_to_drm(adev), "Failed to create oem i2c adapter data\n"); | |
| 1459 | + return -ENOMEM; | |
| 1460 | + } | |
| 1461 | + | |
| 1462 | + r = devm_i2c_add_adapter(adev->dev, &oem_i2c->base); | |
| 1463 | + if (r) { | |
| 1464 | + drm_info(adev_to_drm(adev), "Failed to register oem i2c\n"); | |
| 1465 | + kfree(oem_i2c); | |
| 1466 | + return r; | |
| 1467 | + } | |
| 1468 | + dm->oem_i2c = oem_i2c; | |
| 1886 | 1469 | } |
| 1887 | 1470 | |
| 1888 | − return ret; | |
| 1471 | + return 0; | |
| 1889 | 1472 | } |
| 1890 | 1473 | |
| 1891 | −static int amdgpu_dm_init_power_module(struct amdgpu_display_manager *dm) | |
| 1474 | +/** | |
| 1475 | + * dm_hw_init() - Initialize DC device | |
| 1476 | + * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. | |
| 1477 | + * | |
| 1478 | + * Initialize the &struct amdgpu_display_manager device. This involves calling | |
| 1479 | + * the initializers of each DM component, then populating the struct with them. | |
| 1480 | + * | |
| 1481 | + * Although the function implies hardware initialization, both hardware and | |
| 1482 | + * software are initialized here. Splitting them out to their relevant init | |
| 1483 | + * hooks is a future TODO item. | |
| 1484 | + * | |
| 1485 | + * Some notable things that are initialized here: | |
| 1486 | + * | |
| 1487 | + * - Display Core, both software and hardware | |
| 1488 | + * - DC modules that we need (freesync and color management) | |
| 1489 | + * - DRM software states | |
| 1490 | + * - Interrupt sources and handlers | |
| 1491 | + * - Vblank support | |
| 1492 | + * - Debug FS entries, if enabled | |
| 1493 | + */ | |
| 1494 | +static int dm_hw_init(struct amdgpu_ip_block *ip_block) | |
| 1892 | 1495 | { |
| 1893 | − struct mod_power_init_params init_data[MAX_NUM_EDP]; | |
| 1496 | + struct amdgpu_device *adev = ip_block->adev; | |
| 1497 | + int r; | |
| 1894 | 1498 | |
| 1895 | − if (dm->num_of_edps == 0) { | |
| 1896 | − drm_dbg_driver( | |
| 1897 | − dm->ddev, | |
| 1898 | − "amdgpu: No eDP detected, skip initializing power module\n"); | |
| 1899 | − return 0; | |
| 1900 | − } | |
| 1499 | + /* Create DAL display manager */ | |
| 1500 | + r = amdgpu_dm_init(adev); | |
| 1501 | + if (r) | |
| 1502 | + return r; | |
| 1503 | + amdgpu_dm_hpd_init(adev); | |
| 1901 | 1504 | |
| 1902 | − /* Initialize all the power module parameters */ | |
| 1903 | − for (int i = 0; i < dm->num_of_edps; i++) { | |
| 1904 | − init_data[i].allow_psr_smu_optimizations = | |
| 1905 | − !!(amdgpu_dc_feature_mask & DC_PSR_ALLOW_SMU_OPT); | |
| 1906 | − init_data[i].allow_psr_multi_disp_optimizations = | |
| 1907 | − !!(amdgpu_dc_feature_mask & DC_PSR_ALLOW_MULTI_DISP_OPT); | |
| 1908 | − /* See dm_late_init */ | |
| 1909 | − init_data[i].backlight_ramping_override = false; | |
| 1910 | − init_data[i].backlight_ramping_start = 0xCCCC; | |
| 1911 | − init_data[i].backlight_ramping_reduction = 0xCCCCCCCC; | |
| 1912 | − init_data[i].def_varibright_level = 0; | |
| 1913 | − init_data[i].abm_config_setting = 0; | |
| 1914 | − init_data[i].num_backlight_levels = 101; | |
| 1915 | − init_data[i].use_nits_based_brightness = false; | |
| 1916 | − init_data[i].panel_max_millinits = 0; | |
| 1917 | − init_data[i].panel_min_millinits = 0; | |
| 1918 | − init_data[i].disable_fractional_pwm = | |
| 1919 | − !(amdgpu_dc_feature_mask & DC_DISABLE_FRACTIONAL_PWM_MASK); | |
| 1920 | − init_data[i].use_custom_backlight_caps = false; | |
| 1921 | − init_data[i].custom_backlight_caps_config_no = 0; | |
| 1922 | − init_data[i].use_linear_backlight_curve = false; | |
| 1923 | − init_data[i].def_varibright_enable = 0; | |
| 1924 | − init_data[i].varibright_level = 0; | |
| 1925 | − /* | |
| 1926 | − * Power module uses 16-bit backlight levels (0xFFFF max) rather | |
| 1927 | − * than 8-bit(0XFF max) | |
| 1928 | − */ | |
| 1929 | − init_data[i].min_backlight_pwm = | |
| 1930 | − dm->backlight_caps[i].min_input_signal * 0x101; | |
| 1931 | − init_data[i].max_backlight_pwm = | |
| 1932 | − dm->backlight_caps[i].max_input_signal * 0x101; | |
| 1933 | − init_data[i].min_abm_backlight = | |
| 1934 | − dm->backlight_caps[i].min_input_signal * 0x101; | |
| 1505 | + r = dm_oem_i2c_hw_init(adev); | |
| 1506 | + if (r) | |
| 1507 | + drm_info(adev_to_drm(adev), "Failed to add OEM i2c bus\n"); | |
| 1935 | 1508 | |
| 1936 | − /* Min backlight level after ABM reduction, Don't allow below 1% | |
| 1937 | − * 0xFFFF x 0.01 = 0x28F | |
| 1938 | − */ | |
| 1939 | − init_data[i].min_abm_backlight = (init_data[i].min_abm_backlight < 0x28F) ? | |
| 1940 | − 0x28F : init_data[i].min_abm_backlight; | |
| 1941 | − } | |
| 1509 | + return 0; | |
| 1510 | +} | |
| 1942 | 1511 | |
| 1943 | − dm->power_module = mod_power_create(dm->dc, init_data, dm->num_of_edps); | |
| 1944 | − if (!dm->power_module) { | |
| 1945 | − drm_err(dm->ddev, "amdgpu: Error allocating memory for power module\n"); | |
| 1946 | − return -ENOMEM; | |
| 1947 | − } | |
| 1512 | +/** | |
| 1513 | + * dm_hw_fini() - Teardown DC device | |
| 1514 | + * @ip_block: Pointer to the amdgpu_ip_block for this hw instance. | |
| 1515 | + * | |
| 1516 | + * Teardown components within &struct amdgpu_display_manager that require | |
| 1517 | + * cleanup. This involves cleaning up the DRM device, DC, and any modules that | |
| 1518 | + * were loaded. Also flush IRQ workqueues and disable them. | |
| 1519 | + */ | |
| 1520 | +static int dm_hw_fini(struct amdgpu_ip_block *ip_block) | |
| 1521 | +{ | |
| 1522 | + struct amdgpu_device *adev = ip_block->adev; | |
| 1948 | 1523 | |
| 1949 | − mod_power_hw_init(dm->power_module); | |
| 1950 | − drm_dbg_driver(dm->ddev, "amdgpu: Power module init done\n"); | |
| 1524 | + amdgpu_dm_hpd_fini(adev); | |
| 1951 | 1525 | |
| 1526 | + amdgpu_dm_irq_fini(adev); | |
| 1527 | + amdgpu_dm_fini(adev); | |
| 1952 | 1528 | return 0; |
| 1953 | 1529 | } |
| 1954 | 1530 | |
| 1955 | −static void hdmi_frl_status_polling_work(struct work_struct *work) | |
| 1531 | + | |
| 1532 | +static void dm_gpureset_toggle_interrupts(struct amdgpu_device *adev, | |
| 1533 | + struct dc_state *state, bool enable) | |
| 1956 | 1534 | { |
| 1957 | − struct amdgpu_display_manager *dm = | |
| 1958 | − container_of(to_delayed_work(work), struct amdgpu_display_manager, | |
| 1959 | − hdmi_frl_status_polling_work); | |
| 1960 | − struct dc *dc = dm->dc; | |
| 1961 | − struct dc_link *dc_link; | |
| 1962 | − bool link_update = false; | |
| 1535 | + enum dc_irq_source irq_source; | |
| 1536 | + struct amdgpu_crtc *acrtc; | |
| 1537 | + int rc = -EBUSY; | |
| 1538 | + int i = 0; | |
| 1963 | 1539 | |
| 1964 | − for (int i = 0; i < MAX_LINKS; i++) { | |
| 1965 | − dc_link = dc->links[i]; | |
| 1540 | + for (i = 0; i < state->stream_count; i++) { | |
| 1541 | + acrtc = amdgpu_dm_get_crtc_by_otg_inst( | |
| 1542 | + adev, state->stream_status[i].primary_otg_inst); | |
| 1966 | 1543 | |
| 1967 | − if (!dc_link || !dc_link->local_sink) | |
| 1968 | − continue; | |
| 1544 | + if (acrtc && state->stream_status[i].plane_count != 0) { | |
| 1545 | + irq_source = IRQ_TYPE_PFLIP + acrtc->otg_inst; | |
| 1546 | + rc = dc_interrupt_set(adev->dm.dc, irq_source, enable) ? 0 : -EBUSY; | |
| 1547 | + if (rc) | |
| 1548 | + drm_warn(adev_to_drm(adev), "Failed to %s pflip interrupts\n", | |
| 1549 | + enable ? "enable" : "disable"); | |
| 1969 | 1550 | |
| 1970 | − if (!dc_is_hdmi_signal(dc_link->connector_signal)) | |
| 1971 | − continue; | |
| 1551 | + if (dc_supports_vrr(adev->dm.dc->ctx->dce_version)) { | |
| 1552 | + if (enable) { | |
| 1553 | + if (amdgpu_dm_crtc_vrr_active( | |
| 1554 | + to_dm_crtc_state(acrtc->base.state))) | |
| 1555 | + rc = amdgpu_dm_crtc_set_vupdate_irq( | |
| 1556 | + &acrtc->base, true); | |
| 1557 | + } else | |
| 1558 | + rc = amdgpu_dm_crtc_set_vupdate_irq( | |
| 1559 | + &acrtc->base, false); | |
| 1972 | 1560 | |
| 1973 | − if (dc_link->connector_signal != SIGNAL_TYPE_HDMI_FRL) | |
| 1974 | − continue; | |
| 1561 | + if (rc) | |
| 1562 | + drm_warn(adev_to_drm(adev), "Failed to %sable vupdate interrupt\n", | |
| 1563 | + enable ? "en" : "dis"); | |
| 1564 | + } | |
| 1975 | 1565 | |
| 1976 | − link_update = dc_link_frl_poll_status_flag(dc_link); | |
| 1977 | − if (link_update) { | |
| 1978 | − mutex_lock(&dm->dc_lock); | |
| 1979 | − dc_link_detect(dc_link, DETECT_REASON_RETRAIN); | |
| 1980 | − mutex_unlock(&dm->dc_lock); | |
| 1566 | + irq_source = IRQ_TYPE_VBLANK + acrtc->otg_inst; | |
| 1567 | + /* During gpu-reset we disable and then enable vblank irq, so | |
| 1568 | + * don't use amdgpu_irq_get/put() to avoid refcount change. | |
| 1569 | + */ | |
| 1570 | + if (!dc_interrupt_set(adev->dm.dc, irq_source, enable)) | |
| 1571 | + drm_warn(adev_to_drm(adev), "Failed to %sable vblank interrupt\n", enable ? "en" : "dis"); | |
| 1981 | 1572 | } |
| 1982 | 1573 | } |
| 1983 | 1574 | |
| 1984 | − queue_delayed_work(dm->hdmi_frl_status_polling_wq, | |
| 1985 | − &dm->hdmi_frl_status_polling_work, | |
| 1986 | − msecs_to_jiffies(dm->hdmi_frl_status_polling_delay_ms)); | |
| 1987 | 1575 | } |
| 1988 | 1576 | |
| 1989 | −static int amdgpu_dm_init(struct amdgpu_device *adev) | |
| 1990 | −{ | |
| 1991 | − struct dc_init_data init_data; | |
| 1992 | − struct dc_callback_init init_params; | |
| 1993 | − int r; | |
| 1577 | +DEFINE_FREE(state_release, struct dc_state *, if (_T) dc_state_release(_T)) | |
| 1994 | 1578 | |
| 1995 | − adev->dm.ddev = adev_to_drm(adev); | |
| 1996 | − adev->dm.adev = adev; | |
| 1579 | +static enum dc_status amdgpu_dm_commit_zero_streams(struct dc *dc) | |
| 1580 | +{ | |
| 1581 | + struct dc_state *context __free(state_release) = NULL; | |
| 1582 | + int i; | |
| 1583 | + struct dc_stream_state *del_streams[MAX_PIPES]; | |
| 1584 | + int del_streams_count = 0; | |
| 1585 | + struct dc_commit_streams_params params = {}; | |
| 1997 | 1586 | |
| 1998 | − /* Zero all the fields */ | |
| 1999 | − memset(&init_data, 0, sizeof(init_data)); | |
| 2000 | − memset(&init_params, 0, sizeof(init_params)); | |
| 1587 | + memset(del_streams, 0, sizeof(del_streams)); | |
| 2001 | 1588 | |
| 2002 | − mutex_init(&adev->dm.dpia_aux_lock); | |
| 2003 | − mutex_init(&adev->dm.dc_lock); | |
| 2004 | − mutex_init(&adev->dm.audio_lock); | |
| 1589 | + context = dc_state_create_current_copy(dc); | |
| 1590 | + if (context == NULL) | |
| 1591 | + return DC_ERROR_UNEXPECTED; | |
| 2005 | 1592 | |
| 2006 | − spin_lock_init(&adev->dm.dmub_lock); | |
| 1593 | + /* First remove from context all streams */ | |
| 1594 | + for (i = 0; i < context->stream_count; i++) { | |
| 1595 | + struct dc_stream_state *stream = context->streams[i]; | |
| 2007 | 1596 | |
| 2008 | − if (amdgpu_dm_irq_init(adev)) { | |
| 2009 | − drm_err(adev_to_drm(adev), "failed to initialize DM IRQ support.\n"); | |
| 2010 | − goto error; | |
| 1597 | + del_streams[del_streams_count++] = stream; | |
| 2011 | 1598 | } |
| 2012 | 1599 | |
| 2013 | − /* special handling for early revisions of GC 11.5.4 */ | |
| 2014 | − if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 5, 4)) | |
| 2015 | − init_data.asic_id.chip_family = AMDGPU_FAMILY_GC_11_5_4; | |
| 2016 | − else | |
| 2017 | − init_data.asic_id.chip_family = adev->family; | |
| 1600 | + /* Remove all planes for removed streams and then remove the streams */ | |
| 1601 | + for (i = 0; i < del_streams_count; i++) { | |
| 1602 | + enum dc_status res; | |
| 2018 | 1603 | |
| 2019 | − init_data.asic_id.pci_revision_id = adev->pdev->revision; | |
| 2020 | − init_data.asic_id.hw_internal_rev = adev->external_rev_id; | |
| 2021 | − init_data.asic_id.chip_id = adev->pdev->device; | |
| 1604 | + if (!dc_state_rem_all_planes_for_stream(dc, del_streams[i], context)) | |
| 1605 | + return DC_FAIL_DETACH_SURFACES; | |
| 2022 | 1606 | |
| 2023 | − init_data.asic_id.vram_width = adev->gmc.vram_width; | |
| 2024 | − /* TODO: initialize init_data.asic_id.vram_type here!!!! */ | |
| 2025 | − init_data.asic_id.atombios_base_address = | |
| 2026 | − adev->mode_info.atom_context->bios; | |
| 1607 | + res = dc_state_remove_stream(dc, context, del_streams[i]); | |
| 1608 | + if (res != DC_OK) | |
| 1609 | + return res; | |
| 1610 | + } | |
| 2027 | 1611 | |
| 2028 | − init_data.driver = adev; | |
| 1612 | + params.streams = context->streams; | |
| 1613 | + params.stream_count = context->stream_count; | |
| 2029 | 1614 | |
| 2030 | − /* cgs_device was created in dm_sw_init() */ | |
| 2031 | − init_data.cgs_device = adev->dm.cgs_device; | |
| 1615 | + return dc_commit_streams(dc, ¶ms); | |
| 1616 | +} | |
| 2032 | 1617 | |
| 2033 | − init_data.dce_environment = DCE_ENV_PRODUCTION_DRV; | |
| 1618 | +static int dm_cache_state(struct amdgpu_device *adev) | |
| 1619 | +{ | |
| 1620 | + int r; | |
| 2034 | 1621 | |
| 2035 | − switch (amdgpu_ip_version(adev, DCE_HWIP, 0)) { | |
| 2036 | − case IP_VERSION(2, 1, 0): | |
| 2037 | − switch (adev->dm.dmcub_fw_version) { | |
| 2038 | − case 0: /* development */ | |
| 2039 | − case 0x1: /* linux-firmware.git hash 6d9f399 */ | |
| 2040 | − case 0x01000000: /* linux-firmware.git hash 9a0b0f4 */ | |
| 2041 | − init_data.flags.disable_dmcu = false; | |
| 2042 | − break; | |
| 2043 | − default: | |
| 2044 | − init_data.flags.disable_dmcu = true; | |
| 2045 | − } | |
| 2046 | − break; | |
| 2047 | − case IP_VERSION(2, 0, 3): | |
| 2048 | − init_data.flags.disable_dmcu = true; | |
| 2049 | − break; | |
| 2050 | − default: | |
| 2051 | − break; | |
| 1622 | + adev->dm.cached_state = drm_atomic_helper_suspend(adev_to_drm(adev)); | |
| 1623 | + if (IS_ERR(adev->dm.cached_state)) { | |
| 1624 | + r = PTR_ERR(adev->dm.cached_state); | |
| 1625 | + adev->dm.cached_state = NULL; | |
| 2052 | 1626 | } |
| 2053 | 1627 | |
| 2054 | − /* APU support S/G display by default except: | |
| 2055 | − * ASICs before Carrizo, | |
| 2056 | − * RAVEN1 (Users reported stability issue) | |
| 2057 | − */ | |
| 1628 | + return adev->dm.cached_state ? 0 : r; | |
| 1629 | +} | |
| 2058 | 1630 | |
| 2059 | − if (adev->asic_type < CHIP_CARRIZO) { | |
| 2060 | − init_data.flags.gpu_vm_support = false; | |
| 2061 | − } else if (adev->asic_type == CHIP_RAVEN) { | |
| 2062 | − if (adev->apu_flags & AMD_APU_IS_RAVEN) | |
| 2063 | − init_data.flags.gpu_vm_support = false; | |
| 2064 | − else | |
| 2065 | − init_data.flags.gpu_vm_support = (amdgpu_sg_display != 0); | |
| 2066 | − } else { | |
| 2067 | − if (amdgpu_ip_version(adev, DCE_HWIP, 0) == IP_VERSION(2, 0, 3)) | |
| 2068 | − init_data.flags.gpu_vm_support = (amdgpu_sg_display == 1); | |
| 2069 | − else | |
| 2070 | − init_data.flags.gpu_vm_support = | |
| 2071 | − (amdgpu_sg_display != 0) && (adev->flags & AMD_IS_APU); | |
| 2072 | − } | |
| 1631 | +static void dm_destroy_cached_state(struct amdgpu_device *adev) | |
| 1632 | +{ | |
| 1633 | + struct amdgpu_display_manager *dm = &adev->dm; | |
| 1634 | + struct drm_device *ddev = adev_to_drm(adev); | |
| 1635 | + struct dm_plane_state *dm_new_plane_state; | |
| 1636 | + struct drm_plane_state *new_plane_state; | |
| 1637 | + struct dm_crtc_state *dm_new_crtc_state; | |
| 1638 | + struct drm_crtc_state *new_crtc_state; | |
| 1639 | + struct drm_plane *plane; | |
10262 more lines aren’t shown here.
This diff is large: its 121 lines aren’t shown here.
This diff is large: its 324 lines aren’t shown here.
This diff is large: its 59 lines aren’t shown here.
This diff is large: its 727 lines aren’t shown here.
This diff is large: its 76 lines aren’t shown here.
This diff is large: its 374 lines aren’t shown here.
This diff is large: its 28 lines aren’t shown here.
This diff is large: its 5 lines aren’t shown here.
This diff is large: its 3664 lines aren’t shown here.
This diff is large: its 163 lines aren’t shown here.
This diff is large: its 155 lines aren’t shown here.
This diff is large: its 13 lines aren’t shown here.
This diff is large: its 90 lines aren’t shown here.
This diff is large: its 13 lines aren’t shown here.
This diff is large: its 137 lines aren’t shown here.
This diff is large: its 944 lines aren’t shown here.
This diff is large: its 69 lines aren’t shown here.
This diff is large: its 154 lines aren’t shown here.
This diff is large: its 19 lines aren’t shown here.
This diff is large: its 196 lines aren’t shown here.
This diff is large: its 21 lines aren’t shown here.
This diff is large: its 1633 lines aren’t shown here.
This diff is large: its 39 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 386 lines aren’t shown here.
This diff is large: its 39 lines aren’t shown here.
This diff is large: its 324 lines aren’t shown here.
This diff is large: its 63 lines aren’t shown here.
This diff is large: its 585 lines aren’t shown here.
This diff is large: its 62 lines aren’t shown here.
This diff is large: its 83 lines aren’t shown here.
This diff is large: its 13 lines aren’t shown here.
This diff is large: its 13 lines aren’t shown here.
This diff is large: its 49 lines aren’t shown here.
This diff is large: its 43 lines aren’t shown here.
This diff is large: its 62 lines aren’t shown here.
This diff is large: its 22 lines aren’t shown here.
This diff is large: its 13 lines aren’t shown here.
This diff is large: its 30 lines aren’t shown here.
This diff is large: its 491 lines aren’t shown here.
This diff is large: its 1243 lines aren’t shown here.
This diff is large: its 145 lines aren’t shown here.
This diff is large: its 177 lines aren’t shown here.
This diff is large: its 2159 lines aren’t shown here.
This diff is large: its 140 lines aren’t shown here.
This diff is large: its 524 lines aren’t shown here.
This diff is large: its 583 lines aren’t shown here.
This diff is large: its 356 lines aren’t shown here.
This diff is large: its 635 lines aren’t shown here.
This diff is large: its 910 lines aren’t shown here.
This diff is large: its 130 lines aren’t shown here.
This diff is large: its 143 lines aren’t shown here.
This diff is large: its 33 lines aren’t shown here.
This diff is large: its 1093 lines aren’t shown here.
This diff is large: its 1229 lines aren’t shown here.
This diff is large: its 2455 lines aren’t shown here.
This diff is large: its 642 lines aren’t shown here.
This diff is large: its 104 lines aren’t shown here.
This diff is large: its 484 lines aren’t shown here.
This diff is large: its 314 lines aren’t shown here.
This diff is large: its 950 lines aren’t shown here.
This diff is large: its 393 lines aren’t shown here.
This diff is large: its 106 lines aren’t shown here.
This diff is large: its 21 lines aren’t shown here.
This diff is large: its 665 lines aren’t shown here.
This diff is large: its 289 lines aren’t shown here.
This diff is large: its 20 lines aren’t shown here.
This diff is large: its 60 lines aren’t shown here.
This diff is large: its 345 lines aren’t shown here.
This diff is large: its 90 lines aren’t shown here.
This diff is large: its 68 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 275 lines aren’t shown here.
This diff is large: its 36 lines aren’t shown here.
This diff is large: its 115 lines aren’t shown here.
This diff is large: its 97 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 44 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 15 lines aren’t shown here.
This diff is large: its 191 lines aren’t shown here.
This diff is large: its 18 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 11 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 21 lines aren’t shown here.
This diff is large: its 52 lines aren’t shown here.
This diff is large: its 22 lines aren’t shown here.
This diff is large: its 107 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 16 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 19 lines aren’t shown here.
This diff is large: its 11 lines aren’t shown here.
This diff is large: its 12 lines aren’t shown here.
This diff is large: its 507 lines aren’t shown here.
This diff is large: its 456 lines aren’t shown here.
This diff is large: its 442 lines aren’t shown here.
This diff is large: its 456 lines aren’t shown here.
This diff is large: its 442 lines aren’t shown here.
This diff is large: its 409 lines aren’t shown here.
This diff is large: its 416 lines aren’t shown here.
This diff is large: its 215 lines aren’t shown here.
This diff is large: its 90 lines aren’t shown here.
This diff is large: its 26 lines aren’t shown here.
This diff is large: its 15 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 13 lines aren’t shown here.
This diff is large: its 17 lines aren’t shown here.
This diff is large: its 17 lines aren’t shown here.
This diff is large: its 131 lines aren’t shown here.
This diff is large: its 173 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 18 lines aren’t shown here.
This diff is large: its 79 lines aren’t shown here.
This diff is large: its 10 lines aren’t shown here.
This diff is large: its 639 lines aren’t shown here.
This diff is large: its 30 lines aren’t shown here.
This diff is large: its 10 lines aren’t shown here.
This diff is large: its 635 lines aren’t shown here.
This diff is large: its 17 lines aren’t shown here.
This diff is large: its 117 lines aren’t shown here.
This diff is large: its 17 lines aren’t shown here.
This diff is large: its 47 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 24 lines aren’t shown here.
This diff is large: its 27 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 35 lines aren’t shown here.
This diff is large: its 41 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 38 lines aren’t shown here.
This diff is large: its 17 lines aren’t shown here.
This diff is large: its 29 lines aren’t shown here.
This diff is large: its 18 lines aren’t shown here.
This diff is large: its 90 lines aren’t shown here.
This diff is large: its 17 lines aren’t shown here.
This diff is large: its 17 lines aren’t shown here.
This diff is large: its 17 lines aren’t shown here.
This diff is large: its 17 lines aren’t shown here.
This diff is large: its 17 lines aren’t shown here.
This diff is large: its 17 lines aren’t shown here.
This diff is large: its 17 lines aren’t shown here.
This diff is large: its 387 lines aren’t shown here.
This diff is large: its 71 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 26 lines aren’t shown here.
This diff is large: its 18 lines aren’t shown here.
This diff is large: its 26 lines aren’t shown here.
This diff is large: its 26 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 34 lines aren’t shown here.
This diff is large: its 34 lines aren’t shown here.
This diff is large: its 34 lines aren’t shown here.
This diff is large: its 26 lines aren’t shown here.
This diff is large: its 27 lines aren’t shown here.
This diff is large: its 17 lines aren’t shown here.
This diff is large: its 47 lines aren’t shown here.
This diff is large: its 7 lines aren’t shown here.
This diff is large: its 43 lines aren’t shown here.
This diff is large: its 18 lines aren’t shown here.
This diff is large: its 21 lines aren’t shown here.
This diff is large: its 18 lines aren’t shown here.
This diff is large: its 388 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 62 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 23 lines aren’t shown here.
This diff is large: its 28 lines aren’t shown here.
This diff is large: its 16 lines aren’t shown here.
This diff is large: its 11 lines aren’t shown here.
This diff is large: its 11 lines aren’t shown here.
This diff is large: its 10 lines aren’t shown here.
This diff is large: its 12 lines aren’t shown here.
This diff is large: its 12 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 11 lines aren’t shown here.
This diff is large: its 178 lines aren’t shown here.
This diff is large: its 34 lines aren’t shown here.
This diff is large: its 126 lines aren’t shown here.
This diff is large: its 25 lines aren’t shown here.
This diff is large: its 1004 lines aren’t shown here.
This diff is large: its 660 lines aren’t shown here.
This diff is large: its 94 lines aren’t shown here.
This diff is large: its 149 lines aren’t shown here.
This diff is large: its 1032 lines aren’t shown here.
This diff is large: its 14 lines aren’t shown here.
This diff is large: its 14 lines aren’t shown here.
This diff is large: its 22 lines aren’t shown here.
This diff is large: its 68 lines aren’t shown here.
This diff is large: its 72 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 20 lines aren’t shown here.
This diff is large: its 11 lines aren’t shown here.
This diff is large: its 11 lines aren’t shown here.
This diff is large: its 22 lines aren’t shown here.
This diff is large: its 22 lines aren’t shown here.
This diff is large: its 22 lines aren’t shown here.
This diff is large: its 104 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 8 lines aren’t shown here.
This diff is large: its 129 lines aren’t shown here.
This diff is large: its 14 lines aren’t shown here.
This diff is large: its 26 lines aren’t shown here.
This diff is large: its 694 lines aren’t shown here.
This diff is large: its 14 lines aren’t shown here.
This diff is large: its 113 lines aren’t shown here.
This diff is large: its 23 lines aren’t shown here.
This diff is large: its 113 lines aren’t shown here.
This diff is large: its 144 lines aren’t shown here.
This diff is large: its 27 lines aren’t shown here.
This diff is large: its 10 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 120 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 32 lines aren’t shown here.
This diff is large: its 31 lines aren’t shown here.
This diff is large: its 10 lines aren’t shown here.
This diff is large: its 24 lines aren’t shown here.
This diff is large: its 38 lines aren’t shown here.
This diff is large: its 26 lines aren’t shown here.
This diff is large: its 10 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 37 lines aren’t shown here.
This diff is large: its 15 lines aren’t shown here.
This diff is large: its 21 lines aren’t shown here.
This diff is large: its 13 lines aren’t shown here.
This diff is large: its 85 lines aren’t shown here.
This diff is large: its 13 lines aren’t shown here.
This diff is large: its 9 lines aren’t shown here.
This diff is large: its 60 lines aren’t shown here.
This diff is large: its 42 lines aren’t shown here.