/* * Copyright 2019 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */ #include <linux/firmware.h>#include <drm/drm_exec.h> #include "amdgpu_mes.h"#include "amdgpu.h"#include "soc15_common.h"#include "amdgpu_mes_ctx.h" #define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024#define AMDGPU_ONE_DOORBELL_SIZE 8 int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev){ return roundup(AMDGPU_ONE_DOORBELL_SIZE * AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS, PAGE_SIZE);} static int amdgpu_mes_doorbell_init(struct amdgpu_device *adev){ int i; struct amdgpu_mes *mes = &adev->mes; /* Bitmap for dynamic allocation of kernel doorbells */ mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL); if (!mes->doorbell_bitmap) { dev_err(adev->dev, "Failed to allocate MES doorbell bitmap\n"); return -ENOMEM; } mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE; for (i = 0; i < AMDGPU_MES_PRIORITY_NUM_LEVELS; i++) { adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2; set_bit(i, mes->doorbell_bitmap); } return 0;} static int amdgpu_mes_event_log_init(struct amdgpu_device *adev){ int r; if (!amdgpu_mes_log_enable) return 0; r = amdgpu_bo_create_kernel(adev, adev->mes.event_log_size, PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, &adev->mes.event_log_gpu_obj, &adev->mes.event_log_gpu_addr, &adev->mes.event_log_cpu_addr); if (r) { dev_warn(adev->dev, "failed to create MES event log buffer (%d)", r); return r; } memset(adev->mes.event_log_cpu_addr, 0, adev->mes.event_log_size); return 0; } static void amdgpu_mes_doorbell_free(struct amdgpu_device *adev){ bitmap_free(adev->mes.doorbell_bitmap);} static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe, u32 num_hqd_per_pipe, u32 num_reserved_hqd){ if (num_pipe == 0) return 0; u32 total_hqd_mask = (u32)((1ULL << num_hqd_per_pipe) - 1); u32 reserved_hqd_mask = (u32)((1ULL << DIV_ROUND_UP(num_reserved_hqd, num_pipe)) - 1); return (total_hqd_mask & ~reserved_hqd_mask);} int amdgpu_mes_init(struct amdgpu_device *adev){ int i, r, num_pipes, num_queues = 0; u32 total_vmid_mask, reserved_vmid_mask; int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; u32 gfx_hqd_mask = amdgpu_mes_get_hqd_mask(adev->gfx.me.num_pipe_per_me, adev->gfx.me.num_queue_per_pipe, adev->gfx.disable_kq ? 0 : adev->gfx.num_gfx_rings); u32 compute_hqd_mask = amdgpu_mes_get_hqd_mask(adev->gfx.mec.num_pipe_per_mec, adev->gfx.mec.num_queue_per_pipe, adev->gfx.disable_kq ? 0 : adev->gfx.num_compute_rings); adev->mes.adev = adev; ida_init(&adev->mes.doorbell_ida); spin_lock_init(&adev->mes.queue_id_lock); mutex_init(&adev->mes.mutex_hidden); for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) spin_lock_init(&adev->mes.ring_lock[i]); adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK; total_vmid_mask = (u32)((1UL << 16) - 1); reserved_vmid_mask = (u32)((1UL << adev->vm_manager.first_kfd_vmid) - 1); adev->mes.vmid_mask_mmhub = 0xFF00; adev->mes.vmid_mask_gfxhub = total_vmid_mask & ~reserved_vmid_mask; num_pipes = adev->gfx.me.num_pipe_per_me * adev->gfx.me.num_me; if (num_pipes > AMDGPU_MES_MAX_GFX_PIPES) dev_warn(adev->dev, "more gfx pipes than supported by MES! (%d vs %d)\n", num_pipes, AMDGPU_MES_MAX_GFX_PIPES); for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) { if (i >= num_pipes) break; adev->mes.gfx_hqd_mask[i] = gfx_hqd_mask; } num_pipes = adev->gfx.mec.num_pipe_per_mec * adev->gfx.mec.num_mec; if (num_pipes > AMDGPU_MES_MAX_COMPUTE_PIPES) dev_warn(adev->dev, "more compute pipes than supported by MES! (%d vs %d)\n", num_pipes, AMDGPU_MES_MAX_COMPUTE_PIPES); for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) { /* * Currently, only MEC1 is used for both kernel and user compute queue. * To enable other MEC, we need to redistribute queues per pipe and * adjust queue resource shared with kfd that needs a separate patch. * Skip other MEC for now to avoid potential issues. */ if (i >= adev->gfx.mec.num_pipe_per_mec) break; adev->mes.compute_hqd_mask[i] = compute_hqd_mask; } num_pipes = adev->sdma.num_inst_per_xcc ? adev->sdma.num_inst_per_xcc : adev->sdma.num_instances; if (num_pipes > AMDGPU_MES_MAX_SDMA_PIPES) dev_warn(adev->dev, "more SDMA pipes than supported by MES! (%d vs %d)\n", num_pipes, AMDGPU_MES_MAX_SDMA_PIPES); for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) { if (i >= num_pipes) break; adev->mes.sdma_hqd_mask[i] = 0xfc; } dev_info(adev->dev, "MES: vmid_mask_mmhub 0x%08x, vmid_mask_gfxhub 0x%08x\n", adev->mes.vmid_mask_mmhub, adev->mes.vmid_mask_gfxhub); dev_info(adev->dev, "MES: gfx_hqd_mask 0x%08x, compute_hqd_mask 0x%08x, sdma_hqd_mask 0x%08x\n", adev->mes.gfx_hqd_mask[0], adev->mes.compute_hqd_mask[0], adev->mes.sdma_hqd_mask[0]); for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { r = amdgpu_wb_get(adev, &adev->mes.sch_ctx_offs[i]); if (r) { dev_err(adev->dev, "(%d) ring trail_fence_offs wb alloc failed\n", r); goto error; } adev->mes.sch_ctx_gpu_addr[i] = adev->wb.gpu_addr + (adev->mes.sch_ctx_offs[i] * 4); adev->mes.sch_ctx_ptr[i] = (uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs[i]]; r = amdgpu_wb_get(adev, &adev->mes.query_status_fence_offs[i]); if (r) { dev_err(adev->dev, "(%d) query_status_fence_offs wb alloc failed\n", r); goto error; } adev->mes.query_status_fence_gpu_addr[i] = adev->wb.gpu_addr + (adev->mes.query_status_fence_offs[i] * 4); adev->mes.query_status_fence_ptr[i] = (uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs[i]]; } r = amdgpu_mes_doorbell_init(adev); if (r) goto error; r = amdgpu_mes_event_log_init(adev); if (r) goto error_doorbell; if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) { /* When queue/pipe reset is done in MES instead of in the * driver, MES passes hung queues information to the driver in * hung_queue_hqd_info. Calculate required space to store this * information. */ for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) num_queues += hweight32(adev->mes.gfx_hqd_mask[i]); for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) num_queues += hweight32(adev->mes.compute_hqd_mask[i]); for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) * num_xcc; adev->mes.hung_queue_hqd_info_offset = num_queues; adev->mes.hung_queue_db_array_size = num_queues * 2; } if (adev->mes.hung_queue_db_array_size) { for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { r = amdgpu_bo_create_kernel(adev, adev->mes.hung_queue_db_array_size * sizeof(u32), PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, &adev->mes.hung_queue_db_array_gpu_obj[i], &adev->mes.hung_queue_db_array_gpu_addr[i], &adev->mes.hung_queue_db_array_cpu_addr[i]); if (r) { dev_warn(adev->dev, "failed to create MES hung db array buffer (%d)", r); goto error_doorbell; } } adev->gfx.mec.mes_hung_db_array = kcalloc(amdgpu_mes_get_hung_queue_db_array_size(adev), sizeof(u32), GFP_KERNEL); if (!adev->gfx.mec.mes_hung_db_array) { r = -ENOMEM; goto error_doorbell; } } return 0; error_doorbell: amdgpu_mes_doorbell_free(adev);error: for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { if (adev->mes.sch_ctx_ptr[i]) amdgpu_wb_free(adev, adev->mes.sch_ctx_offs[i]); if (adev->mes.query_status_fence_ptr[i]) amdgpu_wb_free(adev, adev->mes.query_status_fence_offs[i]); if (adev->mes.hung_queue_db_array_gpu_obj[i]) amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i], &adev->mes.hung_queue_db_array_gpu_addr[i], &adev->mes.hung_queue_db_array_cpu_addr[i]); } ida_destroy(&adev->mes.doorbell_ida); mutex_destroy(&adev->mes.mutex_hidden); return r;} void amdgpu_mes_fini(struct amdgpu_device *adev){ int i; int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; kfree(adev->gfx.mec.mes_hung_db_array); amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj, &adev->mes.event_log_gpu_addr, &adev->mes.event_log_cpu_addr); for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { if (adev->mes.hung_queue_db_array_gpu_obj[i]) amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i], &adev->mes.hung_queue_db_array_gpu_addr[i], &adev->mes.hung_queue_db_array_cpu_addr[i]); if (adev->mes.sch_ctx_ptr[i]) amdgpu_wb_free(adev, adev->mes.sch_ctx_offs[i]); if (adev->mes.query_status_fence_ptr[i]) amdgpu_wb_free(adev, adev->mes.query_status_fence_offs[i]); } amdgpu_mes_doorbell_free(adev); if (adev->mes.use_rs64mem) amdgpu_mes_rs64mem_fini(&adev->mes); ida_destroy(&adev->mes.doorbell_ida); mutex_destroy(&adev->mes.mutex_hidden); } int amdgpu_mes_suspend(struct amdgpu_device *adev, u32 xcc_id){ struct mes_suspend_gang_input input; int r; if (!amdgpu_mes_suspend_resume_all_supported(adev)) return 0; memset(&input, 0x0, sizeof(struct mes_suspend_gang_input)); input.suspend_all_gangs = 1; input.xcc_id = xcc_id; if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) && ((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x71)) input.suspend_all_sdma_gangs = 1; /* * Avoid taking any other locks under MES lock to avoid circular * lock dependencies. */ amdgpu_mes_lock(&adev->mes); r = adev->mes.funcs->suspend_gang(&adev->mes, &input); amdgpu_mes_unlock(&adev->mes); if (r) dev_err(adev->dev, "failed to suspend all gangs"); return r;} int amdgpu_mes_resume(struct amdgpu_device *adev, u32 xcc_id){ struct mes_resume_gang_input input; int r; if (!amdgpu_mes_suspend_resume_all_supported(adev)) return 0; memset(&input, 0x0, sizeof(struct mes_resume_gang_input)); input.resume_all_gangs = 1; input.xcc_id = xcc_id; /* * Avoid taking any other locks under MES lock to avoid circular * lock dependencies. */ amdgpu_mes_lock(&adev->mes); r = adev->mes.funcs->resume_gang(&adev->mes, &input); amdgpu_mes_unlock(&adev->mes); if (r) dev_err(adev->dev, "failed to resume all gangs"); return r;} int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev, struct amdgpu_ring *ring, uint32_t xcc_id){ struct mes_map_legacy_queue_input queue_input; int r; memset(&queue_input, 0, sizeof(queue_input)); queue_input.xcc_id = xcc_id; queue_input.queue_type = ring->funcs->type; queue_input.doorbell_offset = ring->doorbell_index; queue_input.pipe_id = ring->pipe; queue_input.queue_id = ring->queue; queue_input.mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); queue_input.wptr_addr = ring->wptr_gpu_addr; amdgpu_mes_lock(&adev->mes); r = adev->mes.funcs->map_legacy_queue(&adev->mes, &queue_input); amdgpu_mes_unlock(&adev->mes); if (r) dev_err(adev->dev, "failed to map legacy queue\n"); return r;} int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev, struct amdgpu_ring *ring, enum amdgpu_unmap_queues_action action, u64 gpu_addr, u64 seq, uint32_t xcc_id){ struct mes_unmap_legacy_queue_input queue_input; int r; queue_input.xcc_id = xcc_id; queue_input.action = action; queue_input.queue_type = ring->funcs->type; queue_input.doorbell_offset = ring->doorbell_index; queue_input.pipe_id = ring->pipe; queue_input.queue_id = ring->queue; queue_input.trail_fence_addr = gpu_addr; queue_input.trail_fence_data = seq; amdgpu_mes_lock(&adev->mes); r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input); amdgpu_mes_unlock(&adev->mes); if (r) dev_err(adev->dev, "failed to unmap legacy queue\n"); return r;} int amdgpu_mes_reset_legacy_queue(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int vmid, bool use_mmio, uint32_t xcc_id){ struct mes_reset_queue_input queue_input; int r; memset(&queue_input, 0, sizeof(queue_input)); queue_input.xcc_id = xcc_id; queue_input.queue_type = ring->funcs->type; queue_input.doorbell_offset = ring->doorbell_index; queue_input.me_id = ring->me; queue_input.pipe_id = ring->pipe; queue_input.queue_id = ring->queue; queue_input.mqd_addr = ring->mqd_obj ? amdgpu_bo_gpu_offset(ring->mqd_obj) : 0; queue_input.wptr_addr = ring->wptr_gpu_addr; queue_input.vmid = vmid; queue_input.use_mmio = use_mmio; queue_input.is_kq = true; if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) queue_input.legacy_gfx = true; amdgpu_mes_lock(&adev->mes); r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); amdgpu_mes_unlock(&adev->mes); if (r) dev_err(adev->dev, "failed to reset legacy queue\n"); return r;} int amdgpu_mes_reset_queue_mmio(struct amdgpu_device *adev, int queue_type, unsigned int vmid, unsigned int me, unsigned int pipe, unsigned int queue, uint32_t xcc_id){ struct mes_reset_queue_input queue_input; int r; memset(&queue_input, 0, sizeof(queue_input)); queue_input.xcc_id = xcc_id; queue_input.me_id = me; queue_input.pipe_id = pipe; queue_input.queue_id = queue; queue_input.vmid = vmid; queue_input.queue_type = queue_type; queue_input.use_mmio = true; amdgpu_mes_lock(&adev->mes); r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); amdgpu_mes_unlock(&adev->mes); if (r) dev_err(adev->dev, "failed to reset legacy queue\n"); return r;} int amdgpu_mes_reset_user_queue(struct amdgpu_device *adev, int queue_type, unsigned int doorbell_index, unsigned int xcc_id){ struct mes_reset_queue_input queue_input; int r; memset(&queue_input, 0, sizeof(queue_input)); queue_input.xcc_id = xcc_id; queue_input.queue_type = queue_type; queue_input.doorbell_offset = doorbell_index; Lines 1–500 of 1171. Load more
/* * Copyright 2019 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. * */ #include <linux/firmware.h> #include <drm/drm_exec.h> #include "amdgpu_mes.h" #include "amdgpu.h" #include "soc15_common.h" #include "amdgpu_mes_ctx.h" #define AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS 1024 #define AMDGPU_ONE_DOORBELL_SIZE 8 int amdgpu_mes_doorbell_process_slice(struct amdgpu_device *adev) { return roundup(AMDGPU_ONE_DOORBELL_SIZE * AMDGPU_MES_MAX_NUM_OF_QUEUES_PER_PROCESS, PAGE_SIZE); } static int amdgpu_mes_doorbell_init(struct amdgpu_device *adev) { int i; struct amdgpu_mes *mes = &adev->mes; /* Bitmap for dynamic allocation of kernel doorbells */ mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL); if (!mes->doorbell_bitmap) { dev_err(adev->dev, "Failed to allocate MES doorbell bitmap\n"); return -ENOMEM; } mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE; for (i = 0; i < AMDGPU_MES_PRIORITY_NUM_LEVELS; i++) { adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2; set_bit(i, mes->doorbell_bitmap); } return 0; } static int amdgpu_mes_event_log_init(struct amdgpu_device *adev) { int r; if (!amdgpu_mes_log_enable) return 0; r = amdgpu_bo_create_kernel(adev, adev->mes.event_log_size, PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, &adev->mes.event_log_gpu_obj, &adev->mes.event_log_gpu_addr, &adev->mes.event_log_cpu_addr); if (r) { dev_warn(adev->dev, "failed to create MES event log buffer (%d)", r); return r; } memset(adev->mes.event_log_cpu_addr, 0, adev->mes.event_log_size); return 0; } static void amdgpu_mes_doorbell_free(struct amdgpu_device *adev) { bitmap_free(adev->mes.doorbell_bitmap); } static inline u32 amdgpu_mes_get_hqd_mask(u32 num_pipe, u32 num_hqd_per_pipe, u32 num_reserved_hqd) { if (num_pipe == 0) return 0; u32 total_hqd_mask = (u32)((1ULL << num_hqd_per_pipe) - 1); u32 reserved_hqd_mask = (u32)((1ULL << DIV_ROUND_UP(num_reserved_hqd, num_pipe)) - 1); return (total_hqd_mask & ~reserved_hqd_mask); } int amdgpu_mes_init(struct amdgpu_device *adev) { int i, r, num_pipes, num_queues = 0; u32 total_vmid_mask, reserved_vmid_mask; int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; u32 gfx_hqd_mask = amdgpu_mes_get_hqd_mask(adev->gfx.me.num_pipe_per_me, adev->gfx.me.num_queue_per_pipe, adev->gfx.disable_kq ? 0 : adev->gfx.num_gfx_rings); u32 compute_hqd_mask = amdgpu_mes_get_hqd_mask(adev->gfx.mec.num_pipe_per_mec, adev->gfx.mec.num_queue_per_pipe, adev->gfx.disable_kq ? 0 : adev->gfx.num_compute_rings); adev->mes.adev = adev; ida_init(&adev->mes.doorbell_ida); spin_lock_init(&adev->mes.queue_id_lock); mutex_init(&adev->mes.mutex_hidden); for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) spin_lock_init(&adev->mes.ring_lock[i]); adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK; total_vmid_mask = (u32)((1UL << 16) - 1); reserved_vmid_mask = (u32)((1UL << adev->vm_manager.first_kfd_vmid) - 1); adev->mes.vmid_mask_mmhub = 0xFF00; adev->mes.vmid_mask_gfxhub = total_vmid_mask & ~reserved_vmid_mask; num_pipes = adev->gfx.me.num_pipe_per_me * adev->gfx.me.num_me; if (num_pipes > AMDGPU_MES_MAX_GFX_PIPES) dev_warn(adev->dev, "more gfx pipes than supported by MES! (%d vs %d)\n", num_pipes, AMDGPU_MES_MAX_GFX_PIPES); for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) { if (i >= num_pipes) break; adev->mes.gfx_hqd_mask[i] = gfx_hqd_mask; } num_pipes = adev->gfx.mec.num_pipe_per_mec * adev->gfx.mec.num_mec; if (num_pipes > AMDGPU_MES_MAX_COMPUTE_PIPES) dev_warn(adev->dev, "more compute pipes than supported by MES! (%d vs %d)\n", num_pipes, AMDGPU_MES_MAX_COMPUTE_PIPES); for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) { /* * Currently, only MEC1 is used for both kernel and user compute queue. * To enable other MEC, we need to redistribute queues per pipe and * adjust queue resource shared with kfd that needs a separate patch. * Skip other MEC for now to avoid potential issues. */ if (i >= adev->gfx.mec.num_pipe_per_mec) break; adev->mes.compute_hqd_mask[i] = compute_hqd_mask; } num_pipes = adev->sdma.num_inst_per_xcc ? adev->sdma.num_inst_per_xcc : adev->sdma.num_instances; if (num_pipes > AMDGPU_MES_MAX_SDMA_PIPES) dev_warn(adev->dev, "more SDMA pipes than supported by MES! (%d vs %d)\n", num_pipes, AMDGPU_MES_MAX_SDMA_PIPES); for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) { if (i >= num_pipes) break; adev->mes.sdma_hqd_mask[i] = 0xfc; } dev_info(adev->dev, "MES: vmid_mask_mmhub 0x%08x, vmid_mask_gfxhub 0x%08x\n", adev->mes.vmid_mask_mmhub, adev->mes.vmid_mask_gfxhub); dev_info(adev->dev, "MES: gfx_hqd_mask 0x%08x, compute_hqd_mask 0x%08x, sdma_hqd_mask 0x%08x\n", adev->mes.gfx_hqd_mask[0], adev->mes.compute_hqd_mask[0], adev->mes.sdma_hqd_mask[0]); for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { r = amdgpu_wb_get(adev, &adev->mes.sch_ctx_offs[i]); if (r) { dev_err(adev->dev, "(%d) ring trail_fence_offs wb alloc failed\n", r); goto error; } adev->mes.sch_ctx_gpu_addr[i] = adev->wb.gpu_addr + (adev->mes.sch_ctx_offs[i] * 4); adev->mes.sch_ctx_ptr[i] = (uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs[i]]; r = amdgpu_wb_get(adev, &adev->mes.query_status_fence_offs[i]); if (r) { dev_err(adev->dev, "(%d) query_status_fence_offs wb alloc failed\n", r); goto error; } adev->mes.query_status_fence_gpu_addr[i] = adev->wb.gpu_addr + (adev->mes.query_status_fence_offs[i] * 4); adev->mes.query_status_fence_ptr[i] = (uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs[i]]; } r = amdgpu_mes_doorbell_init(adev); if (r) goto error; r = amdgpu_mes_event_log_init(adev); if (r) goto error_doorbell; if (amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) { /* When queue/pipe reset is done in MES instead of in the * driver, MES passes hung queues information to the driver in * hung_queue_hqd_info. Calculate required space to store this * information. */ for (i = 0; i < AMDGPU_MES_MAX_GFX_PIPES; i++) num_queues += hweight32(adev->mes.gfx_hqd_mask[i]); for (i = 0; i < AMDGPU_MES_MAX_COMPUTE_PIPES; i++) num_queues += hweight32(adev->mes.compute_hqd_mask[i]); for (i = 0; i < AMDGPU_MES_MAX_SDMA_PIPES; i++) num_queues += hweight32(adev->mes.sdma_hqd_mask[i]) * num_xcc; adev->mes.hung_queue_hqd_info_offset = num_queues; adev->mes.hung_queue_db_array_size = num_queues * 2; } if (adev->mes.hung_queue_db_array_size) { for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { r = amdgpu_bo_create_kernel(adev, adev->mes.hung_queue_db_array_size * sizeof(u32), PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT, &adev->mes.hung_queue_db_array_gpu_obj[i], &adev->mes.hung_queue_db_array_gpu_addr[i], &adev->mes.hung_queue_db_array_cpu_addr[i]); if (r) { dev_warn(adev->dev, "failed to create MES hung db array buffer (%d)", r); goto error_doorbell; } } adev->gfx.mec.mes_hung_db_array = kcalloc(amdgpu_mes_get_hung_queue_db_array_size(adev), sizeof(u32), GFP_KERNEL); if (!adev->gfx.mec.mes_hung_db_array) { r = -ENOMEM; goto error_doorbell; } } return 0; error_doorbell: amdgpu_mes_doorbell_free(adev); error: for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { if (adev->mes.sch_ctx_ptr[i]) amdgpu_wb_free(adev, adev->mes.sch_ctx_offs[i]); if (adev->mes.query_status_fence_ptr[i]) amdgpu_wb_free(adev, adev->mes.query_status_fence_offs[i]); if (adev->mes.hung_queue_db_array_gpu_obj[i]) amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i], &adev->mes.hung_queue_db_array_gpu_addr[i], &adev->mes.hung_queue_db_array_cpu_addr[i]); } ida_destroy(&adev->mes.doorbell_ida); mutex_destroy(&adev->mes.mutex_hidden); return r; } void amdgpu_mes_fini(struct amdgpu_device *adev) { int i; int num_xcc = adev->gfx.xcc_mask ? NUM_XCC(adev->gfx.xcc_mask) : 1; kfree(adev->gfx.mec.mes_hung_db_array); amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj, &adev->mes.event_log_gpu_addr, &adev->mes.event_log_cpu_addr); for (i = 0; i < AMDGPU_MAX_MES_PIPES * num_xcc; i++) { if (adev->mes.hung_queue_db_array_gpu_obj[i]) amdgpu_bo_free_kernel(&adev->mes.hung_queue_db_array_gpu_obj[i], &adev->mes.hung_queue_db_array_gpu_addr[i], &adev->mes.hung_queue_db_array_cpu_addr[i]); if (adev->mes.sch_ctx_ptr[i]) amdgpu_wb_free(adev, adev->mes.sch_ctx_offs[i]); if (adev->mes.query_status_fence_ptr[i]) amdgpu_wb_free(adev, adev->mes.query_status_fence_offs[i]); } amdgpu_mes_doorbell_free(adev); if (adev->mes.use_rs64mem) amdgpu_mes_rs64mem_fini(&adev->mes); ida_destroy(&adev->mes.doorbell_ida); mutex_destroy(&adev->mes.mutex_hidden); } int amdgpu_mes_suspend(struct amdgpu_device *adev, u32 xcc_id) { struct mes_suspend_gang_input input; int r; if (!amdgpu_mes_suspend_resume_all_supported(adev)) return 0; memset(&input, 0x0, sizeof(struct mes_suspend_gang_input)); input.suspend_all_gangs = 1; input.xcc_id = xcc_id; if ((amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(12, 1, 0)) && ((adev->mes.sched_version & AMDGPU_MES_VERSION_MASK) >= 0x71)) input.suspend_all_sdma_gangs = 1; /* * Avoid taking any other locks under MES lock to avoid circular * lock dependencies. */ amdgpu_mes_lock(&adev->mes); r = adev->mes.funcs->suspend_gang(&adev->mes, &input); amdgpu_mes_unlock(&adev->mes); if (r) dev_err(adev->dev, "failed to suspend all gangs"); return r; } int amdgpu_mes_resume(struct amdgpu_device *adev, u32 xcc_id) { struct mes_resume_gang_input input; int r; if (!amdgpu_mes_suspend_resume_all_supported(adev)) return 0; memset(&input, 0x0, sizeof(struct mes_resume_gang_input)); input.resume_all_gangs = 1; input.xcc_id = xcc_id; /* * Avoid taking any other locks under MES lock to avoid circular * lock dependencies. */ amdgpu_mes_lock(&adev->mes); r = adev->mes.funcs->resume_gang(&adev->mes, &input); amdgpu_mes_unlock(&adev->mes); if (r) dev_err(adev->dev, "failed to resume all gangs"); return r; } int amdgpu_mes_map_legacy_queue(struct amdgpu_device *adev, struct amdgpu_ring *ring, uint32_t xcc_id) { struct mes_map_legacy_queue_input queue_input; int r; memset(&queue_input, 0, sizeof(queue_input)); queue_input.xcc_id = xcc_id; queue_input.queue_type = ring->funcs->type; queue_input.doorbell_offset = ring->doorbell_index; queue_input.pipe_id = ring->pipe; queue_input.queue_id = ring->queue; queue_input.mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); queue_input.wptr_addr = ring->wptr_gpu_addr; amdgpu_mes_lock(&adev->mes); r = adev->mes.funcs->map_legacy_queue(&adev->mes, &queue_input); amdgpu_mes_unlock(&adev->mes); if (r) dev_err(adev->dev, "failed to map legacy queue\n"); return r; } int amdgpu_mes_unmap_legacy_queue(struct amdgpu_device *adev, struct amdgpu_ring *ring, enum amdgpu_unmap_queues_action action, u64 gpu_addr, u64 seq, uint32_t xcc_id) { struct mes_unmap_legacy_queue_input queue_input; int r; queue_input.xcc_id = xcc_id; queue_input.action = action; queue_input.queue_type = ring->funcs->type; queue_input.doorbell_offset = ring->doorbell_index; queue_input.pipe_id = ring->pipe; queue_input.queue_id = ring->queue; queue_input.trail_fence_addr = gpu_addr; queue_input.trail_fence_data = seq; amdgpu_mes_lock(&adev->mes); r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input); amdgpu_mes_unlock(&adev->mes); if (r) dev_err(adev->dev, "failed to unmap legacy queue\n"); return r; } int amdgpu_mes_reset_legacy_queue(struct amdgpu_device *adev, struct amdgpu_ring *ring, unsigned int vmid, bool use_mmio, uint32_t xcc_id) { struct mes_reset_queue_input queue_input; int r; memset(&queue_input, 0, sizeof(queue_input)); queue_input.xcc_id = xcc_id; queue_input.queue_type = ring->funcs->type; queue_input.doorbell_offset = ring->doorbell_index; queue_input.me_id = ring->me; queue_input.pipe_id = ring->pipe; queue_input.queue_id = ring->queue; queue_input.mqd_addr = ring->mqd_obj ? amdgpu_bo_gpu_offset(ring->mqd_obj) : 0; queue_input.wptr_addr = ring->wptr_gpu_addr; queue_input.vmid = vmid; queue_input.use_mmio = use_mmio; queue_input.is_kq = true; if (ring->funcs->type == AMDGPU_RING_TYPE_GFX) queue_input.legacy_gfx = true; amdgpu_mes_lock(&adev->mes); r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); amdgpu_mes_unlock(&adev->mes); if (r) dev_err(adev->dev, "failed to reset legacy queue\n"); return r; } int amdgpu_mes_reset_queue_mmio(struct amdgpu_device *adev, int queue_type, unsigned int vmid, unsigned int me, unsigned int pipe, unsigned int queue, uint32_t xcc_id) { struct mes_reset_queue_input queue_input; int r; memset(&queue_input, 0, sizeof(queue_input)); queue_input.xcc_id = xcc_id; queue_input.me_id = me; queue_input.pipe_id = pipe; queue_input.queue_id = queue; queue_input.vmid = vmid; queue_input.queue_type = queue_type; queue_input.use_mmio = true; amdgpu_mes_lock(&adev->mes); r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); amdgpu_mes_unlock(&adev->mes); if (r) dev_err(adev->dev, "failed to reset legacy queue\n"); return r; } int amdgpu_mes_reset_user_queue(struct amdgpu_device *adev, int queue_type, unsigned int doorbell_index, unsigned int xcc_id) { struct mes_reset_queue_input queue_input; int r; memset(&queue_input, 0, sizeof(queue_input)); queue_input.xcc_id = xcc_id; queue_input.queue_type = queue_type; queue_input.doorbell_offset = doorbell_index;