// SPDX-License-Identifier: MIT/* * Copyright 2014-2018 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */#include <linux/dma-buf.h>#include <linux/list.h>#include <linux/pagemap.h>#include <linux/sched/mm.h>#include <linux/sched/task.h>#include <drm/ttm/ttm_tt.h> #include <drm/drm_exec.h> #include "amdgpu_object.h"#include "amdgpu_gem.h"#include "amdgpu_vm.h"#include "amdgpu_hmm.h"#include "amdgpu_amdkfd.h"#include "amdgpu_dma_buf.h"#include <uapi/linux/kfd_ioctl.h>#include "amdgpu_xgmi.h"#include "kfd_priv.h"#include "kfd_smi_events.h" /* Userptr restore delay, just long enough to allow consecutive VM * changes to accumulate */#define AMDGPU_USERPTR_RESTORE_DELAY_MS 1#define AMDGPU_RESERVE_MEM_LIMIT (3UL << 29) /* * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB * BO chunk */#define VRAM_AVAILABLITY_ALIGN (1 << 21) /* Impose limit on how much memory KFD can use */static struct { uint64_t max_system_mem_limit; uint64_t max_ttm_mem_limit; int64_t system_mem_used; int64_t ttm_mem_used; spinlock_t mem_limit_lock;} kfd_mem_limit; static const char * const domain_bit_to_string[] = { "CPU", "GTT", "VRAM", "GDS", "GWS", "OA"}; #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); static bool kfd_mem_is_attached(struct amdgpu_vm *avm, struct kgd_mem *mem){ struct kfd_mem_attachment *entry; list_for_each_entry(entry, &mem->attachments, list) if (entry->bo_va->base.vm == avm) return true; return false;} /** * reuse_dmamap() - Check whether adev can share the original * userptr BO * * If both adev and bo_adev are in direct mapping or * in the same iommu group, they can share the original BO. * * @adev: Device to which can or cannot share the original BO * @bo_adev: Device to which allocated BO belongs to * * Return: returns true if adev can share original userptr BO, * false otherwise. */static bool reuse_dmamap(struct amdgpu_device *adev, struct amdgpu_device *bo_adev){ return (adev->ram_is_direct_mapped && bo_adev->ram_is_direct_mapped) || (adev->dev->iommu_group == bo_adev->dev->iommu_group);} /* Set memory usage limits. Current, limits are * System (TTM + userptr) memory - 15/16th System RAM * TTM memory - 3/8th System RAM */void amdgpu_amdkfd_gpuvm_init_mem_limits(void){ struct sysinfo si; uint64_t mem; if (kfd_mem_limit.max_system_mem_limit) return; si_meminfo(&si); mem = si.totalram - si.totalhigh; mem *= si.mem_unit; spin_lock_init(&kfd_mem_limit.mem_limit_lock); kfd_mem_limit.max_system_mem_limit = mem - (mem >> 6); if (kfd_mem_limit.max_system_mem_limit < 2 * AMDGPU_RESERVE_MEM_LIMIT) kfd_mem_limit.max_system_mem_limit >>= 1; else kfd_mem_limit.max_system_mem_limit -= AMDGPU_RESERVE_MEM_LIMIT; kfd_mem_limit.max_ttm_mem_limit = ttm_tt_pages_limit() << PAGE_SHIFT; pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", (kfd_mem_limit.max_system_mem_limit >> 20), (kfd_mem_limit.max_ttm_mem_limit >> 20));} void amdgpu_amdkfd_reserve_system_mem(uint64_t size){ kfd_mem_limit.system_mem_used += size;} /* Estimate page table size needed to represent a given memory size * * With 4KB pages, we need one 8 byte PTE for each 4KB of memory * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB * of memory (factor 256K, >> 18). ROCm user mode tries to optimize * for 2MB pages for TLB efficiency. However, small allocations and * fragmented system memory still need some 4KB pages. We choose a * compromise that should work in most cases without reserving too * much memory for page tables unnecessarily (factor 16K, >> 14). */ #define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM) /** * amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size * of buffer. * * @adev: Device to which allocated BO belongs to * @size: Size of buffer, in bytes, encapsulated by B0. This should be * equivalent to amdgpu_bo_size(BO) * @alloc_flag: Flag used in allocating a BO as noted above * @xcp_id: xcp_id is used to get xcp from xcp manager, one xcp is * managed as one compute node in driver for app * * Return: * returns -ENOMEM in case of error, ZERO otherwise */int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 alloc_flag, int8_t xcp_id){ uint64_t reserved_for_pt = ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); struct amdgpu_ras *con = amdgpu_ras_get_context(adev); uint64_t reserved_for_ras = (con ? con->reserved_pages_in_bytes : 0); size_t system_mem_needed, ttm_mem_needed, vram_needed; int ret = 0; uint64_t vram_size = 0; system_mem_needed = 0; ttm_mem_needed = 0; vram_needed = 0; if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { system_mem_needed = size; ttm_mem_needed = size; } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { /* * Conservatively round up the allocation requirement to 2 MB * to avoid fragmentation caused by 4K allocations in the tail * 2M BO chunk. */ vram_needed = size; /* * For GFX 9.4.3, get the VRAM size from XCP structs */ if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id)) return -EINVAL; vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id); if (adev->apu_prefer_gtt) { system_mem_needed = size; ttm_mem_needed = size; } } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { system_mem_needed = size; } else if (!(alloc_flag & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); return -ENOMEM; } spin_lock(&kfd_mem_limit.mem_limit_lock); if (kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit) { pr_debug("Set no_system_mem_limit=1 if using shared memory\n"); if (!no_system_mem_limit) { ret = -ENOMEM; goto release; } } if (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > kfd_mem_limit.max_ttm_mem_limit) { ret = -ENOMEM; goto release; } /*if is_app_apu is false and apu_prefer_gtt is true, it is an APU with * carve out < gtt. In that case, VRAM allocation will go to gtt domain, skip * VRAM check since ttm_mem_limit check already cover this allocation */ if (adev && xcp_id >= 0 && (!adev->apu_prefer_gtt || adev->gmc.is_app_apu)) { uint64_t vram_available = vram_size - reserved_for_pt - reserved_for_ras - atomic64_read(&adev->vram_pin_size); if (adev->kfd.vram_used[xcp_id] + vram_needed > vram_available) { ret = -ENOMEM; goto release; } } /* Update memory accounting by decreasing available system * memory, TTM memory and GPU memory as computed above */ WARN_ONCE(vram_needed && !adev, "adev reference can't be null when vram is used"); if (adev && xcp_id >= 0) { adev->kfd.vram_used[xcp_id] += vram_needed; adev->kfd.vram_used_aligned[xcp_id] += adev->apu_prefer_gtt ? vram_needed : ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN); } kfd_mem_limit.system_mem_used += system_mem_needed; kfd_mem_limit.ttm_mem_used += ttm_mem_needed; release: spin_unlock(&kfd_mem_limit.mem_limit_lock); return ret;} void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 alloc_flag, int8_t xcp_id){ spin_lock(&kfd_mem_limit.mem_limit_lock); if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { kfd_mem_limit.system_mem_used -= size; kfd_mem_limit.ttm_mem_used -= size; } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { WARN_ONCE(!adev, "adev reference can't be null when alloc mem flags vram is set"); if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id)) goto release; if (adev) { adev->kfd.vram_used[xcp_id] -= size; if (adev->apu_prefer_gtt) { adev->kfd.vram_used_aligned[xcp_id] -= size; kfd_mem_limit.system_mem_used -= size; kfd_mem_limit.ttm_mem_used -= size; } else { adev->kfd.vram_used_aligned[xcp_id] -= ALIGN(size, VRAM_AVAILABLITY_ALIGN); } } } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { kfd_mem_limit.system_mem_used -= size; } else if (!(alloc_flag & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); goto release; } WARN_ONCE(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] < 0, "KFD VRAM memory accounting unbalanced for xcp: %d", xcp_id); WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, "KFD TTM memory accounting unbalanced"); WARN_ONCE(kfd_mem_limit.system_mem_used < 0, "KFD system memory accounting unbalanced"); release: spin_unlock(&kfd_mem_limit.mem_limit_lock);} void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo){ struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); u32 alloc_flags = bo->kfd_bo->alloc_flags; u64 size = amdgpu_bo_size(bo); amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags, bo->xcp_id); kfree(bo->kfd_bo);} /** * create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information * about USERPTR or DOOREBELL or MMIO BO. * * @adev: Device for which dmamap BO is being created * @mem: BO of peer device that is being DMA mapped. Provides parameters * in building the dmamap BO * @bo_out: Output parameter updated with handle of dmamap BO */static intcreate_dmamap_sg_bo(struct amdgpu_device *adev, struct kgd_mem *mem, struct amdgpu_bo **bo_out){ struct drm_gem_object *gem_obj; int ret; uint64_t flags = 0; ret = amdgpu_bo_reserve(mem->bo, false); if (ret) return ret; if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT | AMDGPU_GEM_CREATE_UNCACHED); ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, 1, AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags, ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj, 0); amdgpu_bo_unreserve(mem->bo); if (ret) { pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret); return -EINVAL; } *bo_out = gem_to_amdgpu_bo(gem_obj); (*bo_out)->parent = amdgpu_bo_ref(mem->bo); return ret;} /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's * reservation object. * * @bo: [IN] Remove eviction fence(s) from this BO * @ef: [IN] This eviction fence is removed if it * is present in the shared list. * * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. */static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, struct amdgpu_amdkfd_fence *ef){ struct dma_fence *replacement; if (!ef) return -EINVAL; /* TODO: Instead of block before we should use the fence of the page * table update and TLB flush here directly. */ replacement = dma_fence_get_stub(); dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context, replacement, DMA_RESV_USAGE_BOOKKEEP); dma_fence_put(replacement); return 0;} /** * amdgpu_amdkfd_remove_all_eviction_fences - Remove all eviction fences * @bo: the BO where to remove the evictions fences from. * * This functions should only be used on release when all references to the BO * are already dropped. We remove the eviction fence from the private copy of * the dma_resv object here since that is what is used during release to * determine of the BO is idle or not. */void amdgpu_amdkfd_remove_all_eviction_fences(struct amdgpu_bo *bo){ struct dma_resv *resv = &bo->tbo.base._resv; struct dma_fence *fence, *stub; struct dma_resv_iter cursor; dma_resv_assert_held(resv); stub = dma_fence_get_stub(); dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) { if (!to_amdgpu_amdkfd_fence(fence)) continue; dma_resv_replace_fences(resv, fence->context, stub, DMA_RESV_USAGE_BOOKKEEP); } dma_fence_put(stub);} static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, bool wait){ struct ttm_operation_ctx ctx = { false, false }; int ret; if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), "Called with userptr BO")) return -EINVAL; /* bo has been pinned, not need validate it */ if (bo->tbo.pin_count) return 0; amdgpu_bo_placement_from_domain(bo, domain); ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (ret) goto validate_fail; if (wait) amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); validate_fail: return ret;} int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo, uint32_t domain, struct dma_fence *fence){ int ret = amdgpu_bo_reserve(bo, false); if (ret) return ret; ret = amdgpu_amdkfd_bo_validate(bo, domain, true); if (ret) goto unreserve_out; ret = dma_resv_reserve_fences(bo->tbo.base.resv, 1); if (ret) goto unreserve_out; dma_resv_add_fence(bo->tbo.base.resv, fence, DMA_RESV_USAGE_BOOKKEEP); unreserve_out: amdgpu_bo_unreserve(bo); return ret;} static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo){ return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false);} /* vm_validate_pt_pd_bos - Validate page table and directory BOs * * Page directories are not updated here because huge page handling * during page table updates can invalidate page directory entries * again. Page directories are only updated after updating page * tables. */static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket){ struct amdgpu_bo *pd = vm->root.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); int ret; ret = amdgpu_vm_validate(adev, vm, ticket, amdgpu_amdkfd_validate_vm_bo, NULL); if (ret) { pr_err("failed to validate PT BOs\n"); return ret; } vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo); return 0;} static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)Lines 1–500 of 3286. Load more
// SPDX-License-Identifier: MIT /* * Copyright 2014-2018 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <linux/dma-buf.h> #include <linux/list.h> #include <linux/pagemap.h> #include <linux/sched/mm.h> #include <linux/sched/task.h> #include <drm/ttm/ttm_tt.h> #include <drm/drm_exec.h> #include "amdgpu_object.h" #include "amdgpu_gem.h" #include "amdgpu_vm.h" #include "amdgpu_hmm.h" #include "amdgpu_amdkfd.h" #include "amdgpu_dma_buf.h" #include <uapi/linux/kfd_ioctl.h> #include "amdgpu_xgmi.h" #include "kfd_priv.h" #include "kfd_smi_events.h" /* Userptr restore delay, just long enough to allow consecutive VM * changes to accumulate */ #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 #define AMDGPU_RESERVE_MEM_LIMIT (3UL << 29) /* * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB * BO chunk */ #define VRAM_AVAILABLITY_ALIGN (1 << 21) /* Impose limit on how much memory KFD can use */ static struct { uint64_t max_system_mem_limit; uint64_t max_ttm_mem_limit; int64_t system_mem_used; int64_t ttm_mem_used; spinlock_t mem_limit_lock; } kfd_mem_limit; static const char * const domain_bit_to_string[] = { "CPU", "GTT", "VRAM", "GDS", "GWS", "OA" }; #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); static bool kfd_mem_is_attached(struct amdgpu_vm *avm, struct kgd_mem *mem) { struct kfd_mem_attachment *entry; list_for_each_entry(entry, &mem->attachments, list) if (entry->bo_va->base.vm == avm) return true; return false; } /** * reuse_dmamap() - Check whether adev can share the original * userptr BO * * If both adev and bo_adev are in direct mapping or * in the same iommu group, they can share the original BO. * * @adev: Device to which can or cannot share the original BO * @bo_adev: Device to which allocated BO belongs to * * Return: returns true if adev can share original userptr BO, * false otherwise. */ static bool reuse_dmamap(struct amdgpu_device *adev, struct amdgpu_device *bo_adev) { return (adev->ram_is_direct_mapped && bo_adev->ram_is_direct_mapped) || (adev->dev->iommu_group == bo_adev->dev->iommu_group); } /* Set memory usage limits. Current, limits are * System (TTM + userptr) memory - 15/16th System RAM * TTM memory - 3/8th System RAM */ void amdgpu_amdkfd_gpuvm_init_mem_limits(void) { struct sysinfo si; uint64_t mem; if (kfd_mem_limit.max_system_mem_limit) return; si_meminfo(&si); mem = si.totalram - si.totalhigh; mem *= si.mem_unit; spin_lock_init(&kfd_mem_limit.mem_limit_lock); kfd_mem_limit.max_system_mem_limit = mem - (mem >> 6); if (kfd_mem_limit.max_system_mem_limit < 2 * AMDGPU_RESERVE_MEM_LIMIT) kfd_mem_limit.max_system_mem_limit >>= 1; else kfd_mem_limit.max_system_mem_limit -= AMDGPU_RESERVE_MEM_LIMIT; kfd_mem_limit.max_ttm_mem_limit = ttm_tt_pages_limit() << PAGE_SHIFT; pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", (kfd_mem_limit.max_system_mem_limit >> 20), (kfd_mem_limit.max_ttm_mem_limit >> 20)); } void amdgpu_amdkfd_reserve_system_mem(uint64_t size) { kfd_mem_limit.system_mem_used += size; } /* Estimate page table size needed to represent a given memory size * * With 4KB pages, we need one 8 byte PTE for each 4KB of memory * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB * of memory (factor 256K, >> 18). ROCm user mode tries to optimize * for 2MB pages for TLB efficiency. However, small allocations and * fragmented system memory still need some 4KB pages. We choose a * compromise that should work in most cases without reserving too * much memory for page tables unnecessarily (factor 16K, >> 14). */ #define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM) /** * amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size * of buffer. * * @adev: Device to which allocated BO belongs to * @size: Size of buffer, in bytes, encapsulated by B0. This should be * equivalent to amdgpu_bo_size(BO) * @alloc_flag: Flag used in allocating a BO as noted above * @xcp_id: xcp_id is used to get xcp from xcp manager, one xcp is * managed as one compute node in driver for app * * Return: * returns -ENOMEM in case of error, ZERO otherwise */ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 alloc_flag, int8_t xcp_id) { uint64_t reserved_for_pt = ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); struct amdgpu_ras *con = amdgpu_ras_get_context(adev); uint64_t reserved_for_ras = (con ? con->reserved_pages_in_bytes : 0); size_t system_mem_needed, ttm_mem_needed, vram_needed; int ret = 0; uint64_t vram_size = 0; system_mem_needed = 0; ttm_mem_needed = 0; vram_needed = 0; if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { system_mem_needed = size; ttm_mem_needed = size; } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { /* * Conservatively round up the allocation requirement to 2 MB * to avoid fragmentation caused by 4K allocations in the tail * 2M BO chunk. */ vram_needed = size; /* * For GFX 9.4.3, get the VRAM size from XCP structs */ if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id)) return -EINVAL; vram_size = KFD_XCP_MEMORY_SIZE(adev, xcp_id); if (adev->apu_prefer_gtt) { system_mem_needed = size; ttm_mem_needed = size; } } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { system_mem_needed = size; } else if (!(alloc_flag & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); return -ENOMEM; } spin_lock(&kfd_mem_limit.mem_limit_lock); if (kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit) { pr_debug("Set no_system_mem_limit=1 if using shared memory\n"); if (!no_system_mem_limit) { ret = -ENOMEM; goto release; } } if (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > kfd_mem_limit.max_ttm_mem_limit) { ret = -ENOMEM; goto release; } /*if is_app_apu is false and apu_prefer_gtt is true, it is an APU with * carve out < gtt. In that case, VRAM allocation will go to gtt domain, skip * VRAM check since ttm_mem_limit check already cover this allocation */ if (adev && xcp_id >= 0 && (!adev->apu_prefer_gtt || adev->gmc.is_app_apu)) { uint64_t vram_available = vram_size - reserved_for_pt - reserved_for_ras - atomic64_read(&adev->vram_pin_size); if (adev->kfd.vram_used[xcp_id] + vram_needed > vram_available) { ret = -ENOMEM; goto release; } } /* Update memory accounting by decreasing available system * memory, TTM memory and GPU memory as computed above */ WARN_ONCE(vram_needed && !adev, "adev reference can't be null when vram is used"); if (adev && xcp_id >= 0) { adev->kfd.vram_used[xcp_id] += vram_needed; adev->kfd.vram_used_aligned[xcp_id] += adev->apu_prefer_gtt ? vram_needed : ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN); } kfd_mem_limit.system_mem_used += system_mem_needed; kfd_mem_limit.ttm_mem_used += ttm_mem_needed; release: spin_unlock(&kfd_mem_limit.mem_limit_lock); return ret; } void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 alloc_flag, int8_t xcp_id) { spin_lock(&kfd_mem_limit.mem_limit_lock); if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { kfd_mem_limit.system_mem_used -= size; kfd_mem_limit.ttm_mem_used -= size; } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { WARN_ONCE(!adev, "adev reference can't be null when alloc mem flags vram is set"); if (WARN_ONCE(xcp_id < 0, "invalid XCP ID %d", xcp_id)) goto release; if (adev) { adev->kfd.vram_used[xcp_id] -= size; if (adev->apu_prefer_gtt) { adev->kfd.vram_used_aligned[xcp_id] -= size; kfd_mem_limit.system_mem_used -= size; kfd_mem_limit.ttm_mem_used -= size; } else { adev->kfd.vram_used_aligned[xcp_id] -= ALIGN(size, VRAM_AVAILABLITY_ALIGN); } } } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { kfd_mem_limit.system_mem_used -= size; } else if (!(alloc_flag & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); goto release; } WARN_ONCE(adev && xcp_id >= 0 && adev->kfd.vram_used[xcp_id] < 0, "KFD VRAM memory accounting unbalanced for xcp: %d", xcp_id); WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, "KFD TTM memory accounting unbalanced"); WARN_ONCE(kfd_mem_limit.system_mem_used < 0, "KFD system memory accounting unbalanced"); release: spin_unlock(&kfd_mem_limit.mem_limit_lock); } void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); u32 alloc_flags = bo->kfd_bo->alloc_flags; u64 size = amdgpu_bo_size(bo); amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags, bo->xcp_id); kfree(bo->kfd_bo); } /** * create_dmamap_sg_bo() - Creates a amdgpu_bo object to reflect information * about USERPTR or DOOREBELL or MMIO BO. * * @adev: Device for which dmamap BO is being created * @mem: BO of peer device that is being DMA mapped. Provides parameters * in building the dmamap BO * @bo_out: Output parameter updated with handle of dmamap BO */ static int create_dmamap_sg_bo(struct amdgpu_device *adev, struct kgd_mem *mem, struct amdgpu_bo **bo_out) { struct drm_gem_object *gem_obj; int ret; uint64_t flags = 0; ret = amdgpu_bo_reserve(mem->bo, false); if (ret) return ret; if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) flags |= mem->bo->flags & (AMDGPU_GEM_CREATE_COHERENT | AMDGPU_GEM_CREATE_UNCACHED); ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, 1, AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE | flags, ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj, 0); amdgpu_bo_unreserve(mem->bo); if (ret) { pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret); return -EINVAL; } *bo_out = gem_to_amdgpu_bo(gem_obj); (*bo_out)->parent = amdgpu_bo_ref(mem->bo); return ret; } /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's * reservation object. * * @bo: [IN] Remove eviction fence(s) from this BO * @ef: [IN] This eviction fence is removed if it * is present in the shared list. * * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. */ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, struct amdgpu_amdkfd_fence *ef) { struct dma_fence *replacement; if (!ef) return -EINVAL; /* TODO: Instead of block before we should use the fence of the page * table update and TLB flush here directly. */ replacement = dma_fence_get_stub(); dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context, replacement, DMA_RESV_USAGE_BOOKKEEP); dma_fence_put(replacement); return 0; } /** * amdgpu_amdkfd_remove_all_eviction_fences - Remove all eviction fences * @bo: the BO where to remove the evictions fences from. * * This functions should only be used on release when all references to the BO * are already dropped. We remove the eviction fence from the private copy of * the dma_resv object here since that is what is used during release to * determine of the BO is idle or not. */ void amdgpu_amdkfd_remove_all_eviction_fences(struct amdgpu_bo *bo) { struct dma_resv *resv = &bo->tbo.base._resv; struct dma_fence *fence, *stub; struct dma_resv_iter cursor; dma_resv_assert_held(resv); stub = dma_fence_get_stub(); dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, fence) { if (!to_amdgpu_amdkfd_fence(fence)) continue; dma_resv_replace_fences(resv, fence->context, stub, DMA_RESV_USAGE_BOOKKEEP); } dma_fence_put(stub); } static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, bool wait) { struct ttm_operation_ctx ctx = { false, false }; int ret; if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), "Called with userptr BO")) return -EINVAL; /* bo has been pinned, not need validate it */ if (bo->tbo.pin_count) return 0; amdgpu_bo_placement_from_domain(bo, domain); ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (ret) goto validate_fail; if (wait) amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); validate_fail: return ret; } int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo, uint32_t domain, struct dma_fence *fence) { int ret = amdgpu_bo_reserve(bo, false); if (ret) return ret; ret = amdgpu_amdkfd_bo_validate(bo, domain, true); if (ret) goto unreserve_out; ret = dma_resv_reserve_fences(bo->tbo.base.resv, 1); if (ret) goto unreserve_out; dma_resv_add_fence(bo->tbo.base.resv, fence, DMA_RESV_USAGE_BOOKKEEP); unreserve_out: amdgpu_bo_unreserve(bo); return ret; } static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo) { return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false); } /* vm_validate_pt_pd_bos - Validate page table and directory BOs * * Page directories are not updated here because huge page handling * during page table updates can invalidate page directory entries * again. Page directories are only updated after updating page * tables. */ static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm, struct ww_acquire_ctx *ticket) { struct amdgpu_bo *pd = vm->root.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); int ret; ret = amdgpu_vm_validate(adev, vm, ticket, amdgpu_amdkfd_validate_vm_bo, NULL); if (ret) { pr_err("failed to validate PT BOs\n"); return ret; } vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo); return 0; } static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)