// SPDX-License-Identifier: MIT/* * Copyright 2014 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include "amdgpu_amdkfd.h"#include "amd_pcie.h"#include "amd_shared.h" #include "amdgpu.h"#include "amdgpu_gfx.h"#include "amdgpu_dma_buf.h"#include <drm/ttm/ttm_tt.h>#include <linux/module.h>#include <linux/dma-buf.h>#include "amdgpu_xgmi.h"#include <uapi/linux/kfd_ioctl.h>#include "amdgpu_ras.h"#include "amdgpu_umc.h"#include "amdgpu_reset.h"#if IS_ENABLED(CONFIG_HSA_AMD)#include "kfd_priv.h"#endif#include "kfd_svm.h" /* Total memory size in system memory and all GPU VRAM. Used to * estimate worst case amount of memory to reserve for page tables */uint64_t amdgpu_amdkfd_total_mem_size; static bool kfd_initialized; int amdgpu_amdkfd_init(void){ struct sysinfo si; int ret; si_meminfo(&si); amdgpu_amdkfd_total_mem_size = si.freeram - si.freehigh; amdgpu_amdkfd_total_mem_size *= si.mem_unit; ret = kgd2kfd_init(); kfd_initialized = !ret; return ret;} void amdgpu_amdkfd_fini(void){ if (kfd_initialized) { kgd2kfd_exit(); kfd_initialized = false; }} void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev){ bool vf = amdgpu_sriov_vf(adev); if (!kfd_initialized) return; adev->kfd.dev = kgd2kfd_probe(adev, vf);} /** * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to * setup amdkfd * * @adev: amdgpu_device pointer * @aperture_base: output returning doorbell aperture base physical address * @aperture_size: output returning doorbell aperture size in bytes * @start_offset: output returning # of doorbell bytes reserved for amdgpu. * * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up, * takes doorbells required for its own rings and reports the setup to amdkfd. * amdgpu reserved doorbells are at the start of the doorbell aperture. */static void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev, phys_addr_t *aperture_base, size_t *aperture_size, size_t *start_offset){ /* * The first num_kernel_doorbells are used by amdgpu. * amdkfd takes whatever's left in the aperture. */ if (adev->enable_mes) { /* * With MES enabled, we only need to initialize * the base address. The size and offset are * not initialized as AMDGPU manages the whole * doorbell space. */ *aperture_base = adev->doorbell.base; *aperture_size = 0; *start_offset = 0; } else if (adev->doorbell.size > adev->doorbell.num_kernel_doorbells * sizeof(u32)) { *aperture_base = adev->doorbell.base; *aperture_size = adev->doorbell.size; *start_offset = adev->doorbell.num_kernel_doorbells * sizeof(u32); } else { *aperture_base = 0; *aperture_size = 0; *start_offset = 0; }} static void amdgpu_amdkfd_reset_work(struct work_struct *work){ struct amdgpu_device *adev = container_of(work, struct amdgpu_device, kfd.reset_work); struct amdgpu_reset_context reset_context; memset(&reset_context, 0, sizeof(reset_context)); reset_context.method = AMD_RESET_METHOD_NONE; reset_context.reset_req_dev = adev; reset_context.src = adev->enable_mes ? AMDGPU_RESET_SRC_MES : AMDGPU_RESET_SRC_HWS; clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); amdgpu_device_gpu_recover(adev, NULL, &reset_context);} static const struct drm_client_funcs kfd_client_funcs = { .unregister = drm_client_release,}; int amdgpu_amdkfd_drm_client_create(struct amdgpu_device *adev){ int ret; if (!adev->kfd.init_complete || adev->kfd.client.dev) return 0; ret = drm_client_init(&adev->ddev, &adev->kfd.client, "kfd", &kfd_client_funcs); if (ret) { dev_err(adev->dev, "Failed to init DRM client: %d\n", ret); return ret; } drm_client_register(&adev->kfd.client); return 0;} void amdgpu_amdkfd_device_init(struct amdgpu_device *adev){ int i; int last_valid_bit; amdgpu_amdkfd_gpuvm_init_mem_limits(); if (adev->kfd.dev) { struct kgd2kfd_shared_resources gpu_resources = { .compute_vmid_bitmap = ((1 << AMDGPU_NUM_VMID) - 1) - ((1 << adev->vm_manager.first_kfd_vmid) - 1), .num_pipe_per_mec = adev->gfx.mec.num_pipe_per_mec, .num_queue_per_pipe = adev->gfx.mec.num_queue_per_pipe, .gpuvm_size = min(adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT, AMDGPU_GMC_HOLE_START), .drm_render_minor = adev_to_drm(adev)->render->index, .sdma_doorbell_idx = adev->doorbell_index.sdma_engine, .enable_mes = adev->enable_mes, }; /* this is going to have a few of the MSBs set that we need to * clear */ bitmap_complement(gpu_resources.cp_queue_bitmap, adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_QUEUES); /* According to linux/bitmap.h we shouldn't use bitmap_clear if * nbits is not compile time constant */ last_valid_bit = 1 /* only first MEC can have compute queues */ * adev->gfx.mec.num_pipe_per_mec * adev->gfx.mec.num_queue_per_pipe; for (i = last_valid_bit; i < AMDGPU_MAX_QUEUES; ++i) clear_bit(i, gpu_resources.cp_queue_bitmap); amdgpu_doorbell_get_kfd_info(adev, &gpu_resources.doorbell_physical_address, &gpu_resources.doorbell_aperture_size, &gpu_resources.doorbell_start_offset); /* Since SOC15, BIF starts to statically use the * lower 12 bits of doorbell addresses for routing * based on settings in registers like * SDMA0_DOORBELL_RANGE etc.. * In order to route a doorbell to CP engine, the lower * 12 bits of its address has to be outside the range * set for SDMA, VCN, and IH blocks. */ if (adev->asic_type >= CHIP_VEGA10) { gpu_resources.non_cp_doorbells_start = adev->doorbell_index.first_non_cp; gpu_resources.non_cp_doorbells_end = adev->doorbell_index.last_non_cp; } adev->kfd.init_complete = kgd2kfd_device_init(adev->kfd.dev, &gpu_resources); amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size; INIT_WORK(&adev->kfd.reset_work, amdgpu_amdkfd_reset_work); }} void amdgpu_amdkfd_device_fini_sw(struct amdgpu_device *adev){ if (adev->kfd.dev) { kgd2kfd_device_exit(adev->kfd.dev); adev->kfd.dev = NULL; amdgpu_amdkfd_total_mem_size -= adev->gmc.real_vram_size; }} void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev, const void *ih_ring_entry){ if (adev->kfd.dev) kgd2kfd_interrupt(adev->kfd.dev, ih_ring_entry);} void amdgpu_amdkfd_teardown_processes(struct amdgpu_device *adev){ kgd2kfd_teardown_processes(adev);} void amdgpu_amdkfd_suspend(struct amdgpu_device *adev, bool suspend_proc){ if (adev->kfd.dev) { if (adev->in_s0ix) kgd2kfd_stop_sched_all_nodes(adev->kfd.dev); else kgd2kfd_suspend(adev->kfd.dev, suspend_proc); }} int amdgpu_amdkfd_resume(struct amdgpu_device *adev, bool resume_proc){ int r = 0; if (adev->kfd.dev) { if (adev->in_s0ix) r = kgd2kfd_start_sched_all_nodes(adev->kfd.dev); else r = kgd2kfd_resume(adev->kfd.dev, resume_proc); } return r;} void amdgpu_amdkfd_suspend_process(struct amdgpu_device *adev){ if (adev->kfd.dev) kgd2kfd_suspend_process(adev->kfd.dev);} int amdgpu_amdkfd_resume_process(struct amdgpu_device *adev){ int r = 0; if (adev->kfd.dev) r = kgd2kfd_resume_process(adev->kfd.dev); return r;} int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev, struct amdgpu_reset_context *reset_context){ int r = 0; if (adev->kfd.dev) r = kgd2kfd_pre_reset(adev->kfd.dev, reset_context); return r;} int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev){ int r = 0; if (adev->kfd.dev) r = kgd2kfd_post_reset(adev->kfd.dev); return r;} void amdgpu_amdkfd_gpu_reset(struct amdgpu_device *adev){ if (amdgpu_device_should_recover_gpu(adev)) (void)amdgpu_reset_domain_schedule(adev->reset_domain, &adev->kfd.reset_work);} void amdgpu_amdkfd_clear_kfd_mapping(struct amdgpu_device *adev){#if IS_ENABLED(CONFIG_HSA_AMD) struct kfd_dev *kfd = adev->kfd.dev; unsigned int i; if (!kfd) return; for (i = 0; i < kfd->num_nodes; i++) { struct kfd_node *node = kfd->nodes[i]; kfd_dev_unmap_mapping_range(KFD_MMAP_TYPE_DOORBELL | KFD_MMAP_GPU_ID(node->id), kfd_doorbell_process_slice(kfd)); kfd_dev_unmap_mapping_range(KFD_MMAP_TYPE_MMIO | KFD_MMAP_GPU_ID(node->id), PAGE_SIZE); }#endif} int amdgpu_amdkfd_alloc_kernel_mem(struct amdgpu_device *adev, size_t size, u32 domain, void **mem_obj, uint64_t *gpu_addr, void **cpu_ptr, bool cp_mqd_gfx9){ struct amdgpu_bo *bo = NULL; struct amdgpu_bo_param bp; int r; void *cpu_ptr_tmp = NULL; memset(&bp, 0, sizeof(bp)); bp.size = size; bp.byte_align = PAGE_SIZE; bp.domain = domain; bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | AMDGPU_GEM_CREATE_CPU_GTT_USWC; bp.type = ttm_bo_type_kernel; bp.resv = NULL; bp.bo_ptr_size = sizeof(struct amdgpu_bo); if (cp_mqd_gfx9) bp.flags |= AMDGPU_GEM_CREATE_CP_MQD_GFX9; r = amdgpu_bo_create(adev, &bp, &bo); if (r) { dev_err(adev->dev, "failed to allocate BO for amdkfd (%d)\n", r); return r; } /* map the buffer */ r = amdgpu_bo_reserve(bo, true); if (r) { dev_err(adev->dev, "(%d) failed to reserve bo for amdkfd\n", r); goto allocate_mem_reserve_bo_failed; } r = amdgpu_bo_pin(bo, domain); if (r) { dev_err(adev->dev, "(%d) failed to pin bo for amdkfd\n", r); goto allocate_mem_pin_bo_failed; } r = amdgpu_ttm_alloc_gart(&bo->tbo); if (r) { dev_err(adev->dev, "%p bind failed\n", bo); goto allocate_mem_kmap_bo_failed; } r = amdgpu_bo_kmap(bo, &cpu_ptr_tmp); if (r) { dev_err(adev->dev, "(%d) failed to map bo to kernel for amdkfd\n", r); goto allocate_mem_kmap_bo_failed; } *mem_obj = bo; *gpu_addr = amdgpu_bo_gpu_offset(bo); *cpu_ptr = cpu_ptr_tmp; amdgpu_bo_unreserve(bo); return 0; allocate_mem_kmap_bo_failed: amdgpu_bo_unpin(bo);allocate_mem_pin_bo_failed: amdgpu_bo_unreserve(bo);allocate_mem_reserve_bo_failed: amdgpu_bo_unref(&bo); return r;} void amdgpu_amdkfd_free_kernel_mem(struct amdgpu_device *adev, void **mem_obj){ struct amdgpu_bo **bo = (struct amdgpu_bo **) mem_obj; if (!bo || !*bo) return; (void)amdgpu_bo_reserve(*bo, true); amdgpu_bo_kunmap(*bo); amdgpu_bo_unpin(*bo); amdgpu_bo_unreserve(*bo); amdgpu_bo_unref(bo);} int amdgpu_amdkfd_alloc_gws(struct amdgpu_device *adev, size_t size, void **mem_obj){ struct amdgpu_bo *bo = NULL; struct amdgpu_bo_user *ubo; struct amdgpu_bo_param bp; int r; memset(&bp, 0, sizeof(bp)); bp.size = size; bp.byte_align = 1; bp.domain = AMDGPU_GEM_DOMAIN_GWS; bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS; bp.type = ttm_bo_type_device; bp.resv = NULL; bp.bo_ptr_size = sizeof(struct amdgpu_bo); r = amdgpu_bo_create_user(adev, &bp, &ubo); if (r) { dev_err(adev->dev, "failed to allocate gws BO for amdkfd (%d)\n", r); return r; } bo = &ubo->bo; *mem_obj = bo; return 0;} void amdgpu_amdkfd_free_gws(struct amdgpu_device *adev, void *mem_obj){ struct amdgpu_bo *bo = (struct amdgpu_bo *)mem_obj; amdgpu_bo_unref(&bo);} uint32_t amdgpu_amdkfd_get_fw_version(struct amdgpu_device *adev, enum kgd_engine_type type){ switch (type) { case KGD_ENGINE_PFP: return adev->gfx.pfp_fw_version; case KGD_ENGINE_ME: return adev->gfx.me_fw_version; case KGD_ENGINE_CE: return adev->gfx.ce_fw_version; case KGD_ENGINE_MEC1: return adev->gfx.mec_fw_version; case KGD_ENGINE_MEC2: return adev->gfx.mec2_fw_version; case KGD_ENGINE_RLC: return adev->gfx.rlc_fw_version; case KGD_ENGINE_SDMA1: return adev->sdma.instance[0].fw_version; case KGD_ENGINE_SDMA2: return adev->sdma.instance[1].fw_version; default:Lines 1–500 of 975. Load more
// SPDX-License-Identifier: MIT /* * Copyright 2014 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include "amdgpu_amdkfd.h" #include "amd_pcie.h" #include "amd_shared.h" #include "amdgpu.h" #include "amdgpu_gfx.h" #include "amdgpu_dma_buf.h" #include <drm/ttm/ttm_tt.h> #include <linux/module.h> #include <linux/dma-buf.h> #include "amdgpu_xgmi.h" #include <uapi/linux/kfd_ioctl.h> #include "amdgpu_ras.h" #include "amdgpu_umc.h" #include "amdgpu_reset.h" #if IS_ENABLED(CONFIG_HSA_AMD) #include "kfd_priv.h" #endif #include "kfd_svm.h" /* Total memory size in system memory and all GPU VRAM. Used to * estimate worst case amount of memory to reserve for page tables */ uint64_t amdgpu_amdkfd_total_mem_size; static bool kfd_initialized; int amdgpu_amdkfd_init(void) { struct sysinfo si; int ret; si_meminfo(&si); amdgpu_amdkfd_total_mem_size = si.freeram - si.freehigh; amdgpu_amdkfd_total_mem_size *= si.mem_unit; ret = kgd2kfd_init(); kfd_initialized = !ret; return ret; } void amdgpu_amdkfd_fini(void) { if (kfd_initialized) { kgd2kfd_exit(); kfd_initialized = false; } } void amdgpu_amdkfd_device_probe(struct amdgpu_device *adev) { bool vf = amdgpu_sriov_vf(adev); if (!kfd_initialized) return; adev->kfd.dev = kgd2kfd_probe(adev, vf); } /** * amdgpu_doorbell_get_kfd_info - Report doorbell configuration required to * setup amdkfd * * @adev: amdgpu_device pointer * @aperture_base: output returning doorbell aperture base physical address * @aperture_size: output returning doorbell aperture size in bytes * @start_offset: output returning # of doorbell bytes reserved for amdgpu. * * amdgpu and amdkfd share the doorbell aperture. amdgpu sets it up, * takes doorbells required for its own rings and reports the setup to amdkfd. * amdgpu reserved doorbells are at the start of the doorbell aperture. */ static void amdgpu_doorbell_get_kfd_info(struct amdgpu_device *adev, phys_addr_t *aperture_base, size_t *aperture_size, size_t *start_offset) { /* * The first num_kernel_doorbells are used by amdgpu. * amdkfd takes whatever's left in the aperture. */ if (adev->enable_mes) { /* * With MES enabled, we only need to initialize * the base address. The size and offset are * not initialized as AMDGPU manages the whole * doorbell space. */ *aperture_base = adev->doorbell.base; *aperture_size = 0; *start_offset = 0; } else if (adev->doorbell.size > adev->doorbell.num_kernel_doorbells * sizeof(u32)) { *aperture_base = adev->doorbell.base; *aperture_size = adev->doorbell.size; *start_offset = adev->doorbell.num_kernel_doorbells * sizeof(u32); } else { *aperture_base = 0; *aperture_size = 0; *start_offset = 0; } } static void amdgpu_amdkfd_reset_work(struct work_struct *work) { struct amdgpu_device *adev = container_of(work, struct amdgpu_device, kfd.reset_work); struct amdgpu_reset_context reset_context; memset(&reset_context, 0, sizeof(reset_context)); reset_context.method = AMD_RESET_METHOD_NONE; reset_context.reset_req_dev = adev; reset_context.src = adev->enable_mes ? AMDGPU_RESET_SRC_MES : AMDGPU_RESET_SRC_HWS; clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags); amdgpu_device_gpu_recover(adev, NULL, &reset_context); } static const struct drm_client_funcs kfd_client_funcs = { .unregister = drm_client_release, }; int amdgpu_amdkfd_drm_client_create(struct amdgpu_device *adev) { int ret; if (!adev->kfd.init_complete || adev->kfd.client.dev) return 0; ret = drm_client_init(&adev->ddev, &adev->kfd.client, "kfd", &kfd_client_funcs); if (ret) { dev_err(adev->dev, "Failed to init DRM client: %d\n", ret); return ret; } drm_client_register(&adev->kfd.client); return 0; } void amdgpu_amdkfd_device_init(struct amdgpu_device *adev) { int i; int last_valid_bit; amdgpu_amdkfd_gpuvm_init_mem_limits(); if (adev->kfd.dev) { struct kgd2kfd_shared_resources gpu_resources = { .compute_vmid_bitmap = ((1 << AMDGPU_NUM_VMID) - 1) - ((1 << adev->vm_manager.first_kfd_vmid) - 1), .num_pipe_per_mec = adev->gfx.mec.num_pipe_per_mec, .num_queue_per_pipe = adev->gfx.mec.num_queue_per_pipe, .gpuvm_size = min(adev->vm_manager.max_pfn << AMDGPU_GPU_PAGE_SHIFT, AMDGPU_GMC_HOLE_START), .drm_render_minor = adev_to_drm(adev)->render->index, .sdma_doorbell_idx = adev->doorbell_index.sdma_engine, .enable_mes = adev->enable_mes, }; /* this is going to have a few of the MSBs set that we need to * clear */ bitmap_complement(gpu_resources.cp_queue_bitmap, adev->gfx.mec_bitmap[0].queue_bitmap, AMDGPU_MAX_QUEUES); /* According to linux/bitmap.h we shouldn't use bitmap_clear if * nbits is not compile time constant */ last_valid_bit = 1 /* only first MEC can have compute queues */ * adev->gfx.mec.num_pipe_per_mec * adev->gfx.mec.num_queue_per_pipe; for (i = last_valid_bit; i < AMDGPU_MAX_QUEUES; ++i) clear_bit(i, gpu_resources.cp_queue_bitmap); amdgpu_doorbell_get_kfd_info(adev, &gpu_resources.doorbell_physical_address, &gpu_resources.doorbell_aperture_size, &gpu_resources.doorbell_start_offset); /* Since SOC15, BIF starts to statically use the * lower 12 bits of doorbell addresses for routing * based on settings in registers like * SDMA0_DOORBELL_RANGE etc.. * In order to route a doorbell to CP engine, the lower * 12 bits of its address has to be outside the range * set for SDMA, VCN, and IH blocks. */ if (adev->asic_type >= CHIP_VEGA10) { gpu_resources.non_cp_doorbells_start = adev->doorbell_index.first_non_cp; gpu_resources.non_cp_doorbells_end = adev->doorbell_index.last_non_cp; } adev->kfd.init_complete = kgd2kfd_device_init(adev->kfd.dev, &gpu_resources); amdgpu_amdkfd_total_mem_size += adev->gmc.real_vram_size; INIT_WORK(&adev->kfd.reset_work, amdgpu_amdkfd_reset_work); } } void amdgpu_amdkfd_device_fini_sw(struct amdgpu_device *adev) { if (adev->kfd.dev) { kgd2kfd_device_exit(adev->kfd.dev); adev->kfd.dev = NULL; amdgpu_amdkfd_total_mem_size -= adev->gmc.real_vram_size; } } void amdgpu_amdkfd_interrupt(struct amdgpu_device *adev, const void *ih_ring_entry) { if (adev->kfd.dev) kgd2kfd_interrupt(adev->kfd.dev, ih_ring_entry); } void amdgpu_amdkfd_teardown_processes(struct amdgpu_device *adev) { kgd2kfd_teardown_processes(adev); } void amdgpu_amdkfd_suspend(struct amdgpu_device *adev, bool suspend_proc) { if (adev->kfd.dev) { if (adev->in_s0ix) kgd2kfd_stop_sched_all_nodes(adev->kfd.dev); else kgd2kfd_suspend(adev->kfd.dev, suspend_proc); } } int amdgpu_amdkfd_resume(struct amdgpu_device *adev, bool resume_proc) { int r = 0; if (adev->kfd.dev) { if (adev->in_s0ix) r = kgd2kfd_start_sched_all_nodes(adev->kfd.dev); else r = kgd2kfd_resume(adev->kfd.dev, resume_proc); } return r; } void amdgpu_amdkfd_suspend_process(struct amdgpu_device *adev) { if (adev->kfd.dev) kgd2kfd_suspend_process(adev->kfd.dev); } int amdgpu_amdkfd_resume_process(struct amdgpu_device *adev) { int r = 0; if (adev->kfd.dev) r = kgd2kfd_resume_process(adev->kfd.dev); return r; } int amdgpu_amdkfd_pre_reset(struct amdgpu_device *adev, struct amdgpu_reset_context *reset_context) { int r = 0; if (adev->kfd.dev) r = kgd2kfd_pre_reset(adev->kfd.dev, reset_context); return r; } int amdgpu_amdkfd_post_reset(struct amdgpu_device *adev) { int r = 0; if (adev->kfd.dev) r = kgd2kfd_post_reset(adev->kfd.dev); return r; } void amdgpu_amdkfd_gpu_reset(struct amdgpu_device *adev) { if (amdgpu_device_should_recover_gpu(adev)) (void)amdgpu_reset_domain_schedule(adev->reset_domain, &adev->kfd.reset_work); } void amdgpu_amdkfd_clear_kfd_mapping(struct amdgpu_device *adev) { #if IS_ENABLED(CONFIG_HSA_AMD) struct kfd_dev *kfd = adev->kfd.dev; unsigned int i; if (!kfd) return; for (i = 0; i < kfd->num_nodes; i++) { struct kfd_node *node = kfd->nodes[i]; kfd_dev_unmap_mapping_range(KFD_MMAP_TYPE_DOORBELL | KFD_MMAP_GPU_ID(node->id), kfd_doorbell_process_slice(kfd)); kfd_dev_unmap_mapping_range(KFD_MMAP_TYPE_MMIO | KFD_MMAP_GPU_ID(node->id), PAGE_SIZE); } #endif } int amdgpu_amdkfd_alloc_kernel_mem(struct amdgpu_device *adev, size_t size, u32 domain, void **mem_obj, uint64_t *gpu_addr, void **cpu_ptr, bool cp_mqd_gfx9) { struct amdgpu_bo *bo = NULL; struct amdgpu_bo_param bp; int r; void *cpu_ptr_tmp = NULL; memset(&bp, 0, sizeof(bp)); bp.size = size; bp.byte_align = PAGE_SIZE; bp.domain = domain; bp.flags = AMDGPU_GEM_CREATE_VRAM_CONTIGUOUS | AMDGPU_GEM_CREATE_CPU_GTT_USWC; bp.type = ttm_bo_type_kernel; bp.resv = NULL; bp.bo_ptr_size = sizeof(struct amdgpu_bo); if (cp_mqd_gfx9) bp.flags |= AMDGPU_GEM_CREATE_CP_MQD_GFX9; r = amdgpu_bo_create(adev, &bp, &bo); if (r) { dev_err(adev->dev, "failed to allocate BO for amdkfd (%d)\n", r); return r; } /* map the buffer */ r = amdgpu_bo_reserve(bo, true); if (r) { dev_err(adev->dev, "(%d) failed to reserve bo for amdkfd\n", r); goto allocate_mem_reserve_bo_failed; } r = amdgpu_bo_pin(bo, domain); if (r) { dev_err(adev->dev, "(%d) failed to pin bo for amdkfd\n", r); goto allocate_mem_pin_bo_failed; } r = amdgpu_ttm_alloc_gart(&bo->tbo); if (r) { dev_err(adev->dev, "%p bind failed\n", bo); goto allocate_mem_kmap_bo_failed; } r = amdgpu_bo_kmap(bo, &cpu_ptr_tmp); if (r) { dev_err(adev->dev, "(%d) failed to map bo to kernel for amdkfd\n", r); goto allocate_mem_kmap_bo_failed; } *mem_obj = bo; *gpu_addr = amdgpu_bo_gpu_offset(bo); *cpu_ptr = cpu_ptr_tmp; amdgpu_bo_unreserve(bo); return 0; allocate_mem_kmap_bo_failed: amdgpu_bo_unpin(bo); allocate_mem_pin_bo_failed: amdgpu_bo_unreserve(bo); allocate_mem_reserve_bo_failed: amdgpu_bo_unref(&bo); return r; } void amdgpu_amdkfd_free_kernel_mem(struct amdgpu_device *adev, void **mem_obj) { struct amdgpu_bo **bo = (struct amdgpu_bo **) mem_obj; if (!bo || !*bo) return; (void)amdgpu_bo_reserve(*bo, true); amdgpu_bo_kunmap(*bo); amdgpu_bo_unpin(*bo); amdgpu_bo_unreserve(*bo); amdgpu_bo_unref(bo); } int amdgpu_amdkfd_alloc_gws(struct amdgpu_device *adev, size_t size, void **mem_obj) { struct amdgpu_bo *bo = NULL; struct amdgpu_bo_user *ubo; struct amdgpu_bo_param bp; int r; memset(&bp, 0, sizeof(bp)); bp.size = size; bp.byte_align = 1; bp.domain = AMDGPU_GEM_DOMAIN_GWS; bp.flags = AMDGPU_GEM_CREATE_NO_CPU_ACCESS; bp.type = ttm_bo_type_device; bp.resv = NULL; bp.bo_ptr_size = sizeof(struct amdgpu_bo); r = amdgpu_bo_create_user(adev, &bp, &ubo); if (r) { dev_err(adev->dev, "failed to allocate gws BO for amdkfd (%d)\n", r); return r; } bo = &ubo->bo; *mem_obj = bo; return 0; } void amdgpu_amdkfd_free_gws(struct amdgpu_device *adev, void *mem_obj) { struct amdgpu_bo *bo = (struct amdgpu_bo *)mem_obj; amdgpu_bo_unref(&bo); } uint32_t amdgpu_amdkfd_get_fw_version(struct amdgpu_device *adev, enum kgd_engine_type type) { switch (type) { case KGD_ENGINE_PFP: return adev->gfx.pfp_fw_version; case KGD_ENGINE_ME: return adev->gfx.me_fw_version; case KGD_ENGINE_CE: return adev->gfx.ce_fw_version; case KGD_ENGINE_MEC1: return adev->gfx.mec_fw_version; case KGD_ENGINE_MEC2: return adev->gfx.mec2_fw_version; case KGD_ENGINE_RLC: return adev->gfx.rlc_fw_version; case KGD_ENGINE_SDMA1: return adev->sdma.instance[0].fw_version; case KGD_ENGINE_SDMA2: return adev->sdma.instance[1].fw_version; default: