// SPDX-License-Identifier: MIT/* * Copyright 2014-2018 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */#include <linux/dma-buf.h>#include <linux/list.h>#include <linux/pagemap.h>#include <linux/sched/mm.h>#include <linux/sched/task.h> #include "amdgpu_object.h"#include "amdgpu_gem.h"#include "amdgpu_vm.h"#include "amdgpu_hmm.h"#include "amdgpu_amdkfd.h"#include "amdgpu_dma_buf.h"#include <uapi/linux/kfd_ioctl.h>#include "amdgpu_xgmi.h"#include "kfd_smi_events.h" /* Userptr restore delay, just long enough to allow consecutive VM * changes to accumulate */#define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 /* * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB * BO chunk */#define VRAM_AVAILABLITY_ALIGN (1 << 21) /* Impose limit on how much memory KFD can use */static struct { uint64_t max_system_mem_limit; uint64_t max_ttm_mem_limit; int64_t system_mem_used; int64_t ttm_mem_used; spinlock_t mem_limit_lock;} kfd_mem_limit; static const char * const domain_bit_to_string[] = { "CPU", "GTT", "VRAM", "GDS", "GWS", "OA"}; #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); static bool kfd_mem_is_attached(struct amdgpu_vm *avm, struct kgd_mem *mem){ struct kfd_mem_attachment *entry; list_for_each_entry(entry, &mem->attachments, list) if (entry->bo_va->base.vm == avm) return true; return false;} /* Set memory usage limits. Current, limits are * System (TTM + userptr) memory - 15/16th System RAM * TTM memory - 3/8th System RAM */void amdgpu_amdkfd_gpuvm_init_mem_limits(void){ struct sysinfo si; uint64_t mem; si_meminfo(&si); mem = si.freeram - si.freehigh; mem *= si.mem_unit; spin_lock_init(&kfd_mem_limit.mem_limit_lock); kfd_mem_limit.max_system_mem_limit = mem - (mem >> 4); kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3); pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", (kfd_mem_limit.max_system_mem_limit >> 20), (kfd_mem_limit.max_ttm_mem_limit >> 20));} void amdgpu_amdkfd_reserve_system_mem(uint64_t size){ kfd_mem_limit.system_mem_used += size;} /* Estimate page table size needed to represent a given memory size * * With 4KB pages, we need one 8 byte PTE for each 4KB of memory * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB * of memory (factor 256K, >> 18). ROCm user mode tries to optimize * for 2MB pages for TLB efficiency. However, small allocations and * fragmented system memory still need some 4KB pages. We choose a * compromise that should work in most cases without reserving too * much memory for page tables unnecessarily (factor 16K, >> 14). */ #define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM) /** * amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size * of buffer. * * @adev: Device to which allocated BO belongs to * @size: Size of buffer, in bytes, encapsulated by B0. This should be * equivalent to amdgpu_bo_size(BO) * @alloc_flag: Flag used in allocating a BO as noted above * * Return: returns -ENOMEM in case of error, ZERO otherwise */int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 alloc_flag){ uint64_t reserved_for_pt = ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); size_t system_mem_needed, ttm_mem_needed, vram_needed; int ret = 0; system_mem_needed = 0; ttm_mem_needed = 0; vram_needed = 0; if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { system_mem_needed = size; ttm_mem_needed = size; } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { /* * Conservatively round up the allocation requirement to 2 MB * to avoid fragmentation caused by 4K allocations in the tail * 2M BO chunk. */ vram_needed = size; } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { system_mem_needed = size; } else if (!(alloc_flag & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); return -ENOMEM; } spin_lock(&kfd_mem_limit.mem_limit_lock); if (kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit) pr_debug("Set no_system_mem_limit=1 if using shared memory\n"); if ((kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) || (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > kfd_mem_limit.max_ttm_mem_limit) || (adev && adev->kfd.vram_used + vram_needed > adev->gmc.real_vram_size - reserved_for_pt)) { ret = -ENOMEM; goto release; } /* Update memory accounting by decreasing available system * memory, TTM memory and GPU memory as computed above */ WARN_ONCE(vram_needed && !adev, "adev reference can't be null when vram is used"); if (adev) { adev->kfd.vram_used += vram_needed; adev->kfd.vram_used_aligned += ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN); } kfd_mem_limit.system_mem_used += system_mem_needed; kfd_mem_limit.ttm_mem_used += ttm_mem_needed; release: spin_unlock(&kfd_mem_limit.mem_limit_lock); return ret;} void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 alloc_flag){ spin_lock(&kfd_mem_limit.mem_limit_lock); if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { kfd_mem_limit.system_mem_used -= size; kfd_mem_limit.ttm_mem_used -= size; } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { WARN_ONCE(!adev, "adev reference can't be null when alloc mem flags vram is set"); if (adev) { adev->kfd.vram_used -= size; adev->kfd.vram_used_aligned -= ALIGN(size, VRAM_AVAILABLITY_ALIGN); } } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { kfd_mem_limit.system_mem_used -= size; } else if (!(alloc_flag & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); goto release; } WARN_ONCE(adev && adev->kfd.vram_used < 0, "KFD VRAM memory accounting unbalanced"); WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, "KFD TTM memory accounting unbalanced"); WARN_ONCE(kfd_mem_limit.system_mem_used < 0, "KFD system memory accounting unbalanced"); release: spin_unlock(&kfd_mem_limit.mem_limit_lock);} void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo){ struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); u32 alloc_flags = bo->kfd_bo->alloc_flags; u64 size = amdgpu_bo_size(bo); amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags); kfree(bo->kfd_bo);} /** * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information * about USERPTR or DOOREBELL or MMIO BO. * @adev: Device for which dmamap BO is being created * @mem: BO of peer device that is being DMA mapped. Provides parameters * in building the dmamap BO * @bo_out: Output parameter updated with handle of dmamap BO */static intcreate_dmamap_sg_bo(struct amdgpu_device *adev, struct kgd_mem *mem, struct amdgpu_bo **bo_out){ struct drm_gem_object *gem_obj; int ret, align; ret = amdgpu_bo_reserve(mem->bo, false); if (ret) return ret; align = 1; ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align, AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE, ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj); amdgpu_bo_unreserve(mem->bo); if (ret) { pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret); return -EINVAL; } *bo_out = gem_to_amdgpu_bo(gem_obj); (*bo_out)->parent = amdgpu_bo_ref(mem->bo); return ret;} /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's * reservation object. * * @bo: [IN] Remove eviction fence(s) from this BO * @ef: [IN] This eviction fence is removed if it * is present in the shared list. * * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. */static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, struct amdgpu_amdkfd_fence *ef){ struct dma_fence *replacement; if (!ef) return -EINVAL; /* TODO: Instead of block before we should use the fence of the page * table update and TLB flush here directly. */ replacement = dma_fence_get_stub(); dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context, replacement, DMA_RESV_USAGE_BOOKKEEP); dma_fence_put(replacement); return 0;} int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo){ struct amdgpu_bo *root = bo; struct amdgpu_vm_bo_base *vm_bo; struct amdgpu_vm *vm; struct amdkfd_process_info *info; struct amdgpu_amdkfd_fence *ef; int ret; /* we can always get vm_bo from root PD bo.*/ while (root->parent) root = root->parent; vm_bo = root->vm_bo; if (!vm_bo) return 0; vm = vm_bo->vm; if (!vm) return 0; info = vm->process_info; if (!info || !info->eviction_fence) return 0; ef = container_of(dma_fence_get(&info->eviction_fence->base), struct amdgpu_amdkfd_fence, base); BUG_ON(!dma_resv_trylock(bo->tbo.base.resv)); ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef); dma_resv_unlock(bo->tbo.base.resv); dma_fence_put(&ef->base); return ret;} static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, bool wait){ struct ttm_operation_ctx ctx = { false, false }; int ret; if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), "Called with userptr BO")) return -EINVAL; amdgpu_bo_placement_from_domain(bo, domain); ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (ret) goto validate_fail; if (wait) amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); validate_fail: return ret;} static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo){ return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false);} /* vm_validate_pt_pd_bos - Validate page table and directory BOs * * Page directories are not updated here because huge page handling * during page table updates can invalidate page directory entries * again. Page directories are only updated after updating page * tables. */static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm){ struct amdgpu_bo *pd = vm->root.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); int ret; ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate_vm_bo, NULL); if (ret) { pr_err("failed to validate PT BOs\n"); return ret; } vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo); return 0;} static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync){ struct amdgpu_bo *pd = vm->root.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); int ret; ret = amdgpu_vm_update_pdes(adev, vm, false); if (ret) return ret; return amdgpu_sync_fence(sync, vm->last_update);} static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem){ uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_MTYPE_DEFAULT; if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE) mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE; if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE) mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; return amdgpu_gem_va_map_flags(adev, mapping_flags);} /** * create_sg_table() - Create an sg_table for a contiguous DMA addr range * @addr: The starting address to point to * @size: Size of memory area in bytes being pointed to * * Allocates an instance of sg_table and initializes it to point to memory * area specified by input parameters. The address used to build is assumed * to be DMA mapped, if needed. * * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table * because they are physically contiguous. * * Return: Initialized instance of SG Table or NULL */static struct sg_table *create_sg_table(uint64_t addr, uint32_t size){ struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL); if (!sg) return NULL; if (sg_alloc_table(sg, 1, GFP_KERNEL)) { kfree(sg); return NULL; } sg_dma_address(sg->sgl) = addr; sg->sgl->length = size;#ifdef CONFIG_NEED_SG_DMA_LENGTH sg->sgl->dma_length = size;#endif return sg;} static intkfd_mem_dmamap_userptr(struct kgd_mem *mem, struct kfd_mem_attachment *attachment){ enum dma_data_direction direction = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; struct ttm_operation_ctx ctx = {.interruptible = true}; struct amdgpu_bo *bo = attachment->bo_va->base.bo; struct amdgpu_device *adev = attachment->adev; struct ttm_tt *src_ttm = mem->bo->tbo.ttm; struct ttm_tt *ttm = bo->tbo.ttm; int ret; if (WARN_ON(ttm->num_pages != src_ttm->num_pages)) return -EINVAL; ttm->sg = kmalloc(sizeof(*ttm->sg), GFP_KERNEL); if (unlikely(!ttm->sg)) return -ENOMEM; /* Same sequence as in amdgpu_ttm_tt_pin_userptr */ ret = sg_alloc_table_from_pages(ttm->sg, src_ttm->pages, ttm->num_pages, 0, (u64)ttm->num_pages << PAGE_SHIFT, GFP_KERNEL); if (unlikely(ret)) goto free_sg; ret = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); if (unlikely(ret)) goto release_sg; drm_prime_sg_to_dma_addr_array(ttm->sg, ttm->dma_address, ttm->num_pages); amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (ret) goto unmap_sg; return 0; unmap_sg: dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0);release_sg: pr_err("DMA map userptr failed: %d\n", ret); sg_free_table(ttm->sg);free_sg: kfree(ttm->sg); ttm->sg = NULL;Lines 1–500 of 2962. Load more
// SPDX-License-Identifier: MIT /* * Copyright 2014-2018 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <linux/dma-buf.h> #include <linux/list.h> #include <linux/pagemap.h> #include <linux/sched/mm.h> #include <linux/sched/task.h> #include "amdgpu_object.h" #include "amdgpu_gem.h" #include "amdgpu_vm.h" #include "amdgpu_hmm.h" #include "amdgpu_amdkfd.h" #include "amdgpu_dma_buf.h" #include <uapi/linux/kfd_ioctl.h> #include "amdgpu_xgmi.h" #include "kfd_smi_events.h" /* Userptr restore delay, just long enough to allow consecutive VM * changes to accumulate */ #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 /* * Align VRAM availability to 2MB to avoid fragmentation caused by 4K allocations in the tail 2MB * BO chunk */ #define VRAM_AVAILABLITY_ALIGN (1 << 21) /* Impose limit on how much memory KFD can use */ static struct { uint64_t max_system_mem_limit; uint64_t max_ttm_mem_limit; int64_t system_mem_used; int64_t ttm_mem_used; spinlock_t mem_limit_lock; } kfd_mem_limit; static const char * const domain_bit_to_string[] = { "CPU", "GTT", "VRAM", "GDS", "GWS", "OA" }; #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); static bool kfd_mem_is_attached(struct amdgpu_vm *avm, struct kgd_mem *mem) { struct kfd_mem_attachment *entry; list_for_each_entry(entry, &mem->attachments, list) if (entry->bo_va->base.vm == avm) return true; return false; } /* Set memory usage limits. Current, limits are * System (TTM + userptr) memory - 15/16th System RAM * TTM memory - 3/8th System RAM */ void amdgpu_amdkfd_gpuvm_init_mem_limits(void) { struct sysinfo si; uint64_t mem; si_meminfo(&si); mem = si.freeram - si.freehigh; mem *= si.mem_unit; spin_lock_init(&kfd_mem_limit.mem_limit_lock); kfd_mem_limit.max_system_mem_limit = mem - (mem >> 4); kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3); pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", (kfd_mem_limit.max_system_mem_limit >> 20), (kfd_mem_limit.max_ttm_mem_limit >> 20)); } void amdgpu_amdkfd_reserve_system_mem(uint64_t size) { kfd_mem_limit.system_mem_used += size; } /* Estimate page table size needed to represent a given memory size * * With 4KB pages, we need one 8 byte PTE for each 4KB of memory * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB * of memory (factor 256K, >> 18). ROCm user mode tries to optimize * for 2MB pages for TLB efficiency. However, small allocations and * fragmented system memory still need some 4KB pages. We choose a * compromise that should work in most cases without reserving too * much memory for page tables unnecessarily (factor 16K, >> 14). */ #define ESTIMATE_PT_SIZE(mem_size) max(((mem_size) >> 14), AMDGPU_VM_RESERVED_VRAM) /** * amdgpu_amdkfd_reserve_mem_limit() - Decrease available memory by size * of buffer. * * @adev: Device to which allocated BO belongs to * @size: Size of buffer, in bytes, encapsulated by B0. This should be * equivalent to amdgpu_bo_size(BO) * @alloc_flag: Flag used in allocating a BO as noted above * * Return: returns -ENOMEM in case of error, ZERO otherwise */ int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 alloc_flag) { uint64_t reserved_for_pt = ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); size_t system_mem_needed, ttm_mem_needed, vram_needed; int ret = 0; system_mem_needed = 0; ttm_mem_needed = 0; vram_needed = 0; if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { system_mem_needed = size; ttm_mem_needed = size; } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { /* * Conservatively round up the allocation requirement to 2 MB * to avoid fragmentation caused by 4K allocations in the tail * 2M BO chunk. */ vram_needed = size; } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { system_mem_needed = size; } else if (!(alloc_flag & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); return -ENOMEM; } spin_lock(&kfd_mem_limit.mem_limit_lock); if (kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit) pr_debug("Set no_system_mem_limit=1 if using shared memory\n"); if ((kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) || (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > kfd_mem_limit.max_ttm_mem_limit) || (adev && adev->kfd.vram_used + vram_needed > adev->gmc.real_vram_size - reserved_for_pt)) { ret = -ENOMEM; goto release; } /* Update memory accounting by decreasing available system * memory, TTM memory and GPU memory as computed above */ WARN_ONCE(vram_needed && !adev, "adev reference can't be null when vram is used"); if (adev) { adev->kfd.vram_used += vram_needed; adev->kfd.vram_used_aligned += ALIGN(vram_needed, VRAM_AVAILABLITY_ALIGN); } kfd_mem_limit.system_mem_used += system_mem_needed; kfd_mem_limit.ttm_mem_used += ttm_mem_needed; release: spin_unlock(&kfd_mem_limit.mem_limit_lock); return ret; } void amdgpu_amdkfd_unreserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 alloc_flag) { spin_lock(&kfd_mem_limit.mem_limit_lock); if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_GTT) { kfd_mem_limit.system_mem_used -= size; kfd_mem_limit.ttm_mem_used -= size; } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { WARN_ONCE(!adev, "adev reference can't be null when alloc mem flags vram is set"); if (adev) { adev->kfd.vram_used -= size; adev->kfd.vram_used_aligned -= ALIGN(size, VRAM_AVAILABLITY_ALIGN); } } else if (alloc_flag & KFD_IOC_ALLOC_MEM_FLAGS_USERPTR) { kfd_mem_limit.system_mem_used -= size; } else if (!(alloc_flag & (KFD_IOC_ALLOC_MEM_FLAGS_DOORBELL | KFD_IOC_ALLOC_MEM_FLAGS_MMIO_REMAP))) { pr_err("%s: Invalid BO type %#x\n", __func__, alloc_flag); goto release; } WARN_ONCE(adev && adev->kfd.vram_used < 0, "KFD VRAM memory accounting unbalanced"); WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, "KFD TTM memory accounting unbalanced"); WARN_ONCE(kfd_mem_limit.system_mem_used < 0, "KFD system memory accounting unbalanced"); release: spin_unlock(&kfd_mem_limit.mem_limit_lock); } void amdgpu_amdkfd_release_notify(struct amdgpu_bo *bo) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); u32 alloc_flags = bo->kfd_bo->alloc_flags; u64 size = amdgpu_bo_size(bo); amdgpu_amdkfd_unreserve_mem_limit(adev, size, alloc_flags); kfree(bo->kfd_bo); } /** * @create_dmamap_sg_bo: Creates a amdgpu_bo object to reflect information * about USERPTR or DOOREBELL or MMIO BO. * @adev: Device for which dmamap BO is being created * @mem: BO of peer device that is being DMA mapped. Provides parameters * in building the dmamap BO * @bo_out: Output parameter updated with handle of dmamap BO */ static int create_dmamap_sg_bo(struct amdgpu_device *adev, struct kgd_mem *mem, struct amdgpu_bo **bo_out) { struct drm_gem_object *gem_obj; int ret, align; ret = amdgpu_bo_reserve(mem->bo, false); if (ret) return ret; align = 1; ret = amdgpu_gem_object_create(adev, mem->bo->tbo.base.size, align, AMDGPU_GEM_DOMAIN_CPU, AMDGPU_GEM_CREATE_PREEMPTIBLE, ttm_bo_type_sg, mem->bo->tbo.base.resv, &gem_obj); amdgpu_bo_unreserve(mem->bo); if (ret) { pr_err("Error in creating DMA mappable SG BO on domain: %d\n", ret); return -EINVAL; } *bo_out = gem_to_amdgpu_bo(gem_obj); (*bo_out)->parent = amdgpu_bo_ref(mem->bo); return ret; } /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's * reservation object. * * @bo: [IN] Remove eviction fence(s) from this BO * @ef: [IN] This eviction fence is removed if it * is present in the shared list. * * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. */ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, struct amdgpu_amdkfd_fence *ef) { struct dma_fence *replacement; if (!ef) return -EINVAL; /* TODO: Instead of block before we should use the fence of the page * table update and TLB flush here directly. */ replacement = dma_fence_get_stub(); dma_resv_replace_fences(bo->tbo.base.resv, ef->base.context, replacement, DMA_RESV_USAGE_BOOKKEEP); dma_fence_put(replacement); return 0; } int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo) { struct amdgpu_bo *root = bo; struct amdgpu_vm_bo_base *vm_bo; struct amdgpu_vm *vm; struct amdkfd_process_info *info; struct amdgpu_amdkfd_fence *ef; int ret; /* we can always get vm_bo from root PD bo.*/ while (root->parent) root = root->parent; vm_bo = root->vm_bo; if (!vm_bo) return 0; vm = vm_bo->vm; if (!vm) return 0; info = vm->process_info; if (!info || !info->eviction_fence) return 0; ef = container_of(dma_fence_get(&info->eviction_fence->base), struct amdgpu_amdkfd_fence, base); BUG_ON(!dma_resv_trylock(bo->tbo.base.resv)); ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef); dma_resv_unlock(bo->tbo.base.resv); dma_fence_put(&ef->base); return ret; } static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, bool wait) { struct ttm_operation_ctx ctx = { false, false }; int ret; if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), "Called with userptr BO")) return -EINVAL; amdgpu_bo_placement_from_domain(bo, domain); ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (ret) goto validate_fail; if (wait) amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); validate_fail: return ret; } static int amdgpu_amdkfd_validate_vm_bo(void *_unused, struct amdgpu_bo *bo) { return amdgpu_amdkfd_bo_validate(bo, bo->allowed_domains, false); } /* vm_validate_pt_pd_bos - Validate page table and directory BOs * * Page directories are not updated here because huge page handling * during page table updates can invalidate page directory entries * again. Page directories are only updated after updating page * tables. */ static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm) { struct amdgpu_bo *pd = vm->root.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); int ret; ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate_vm_bo, NULL); if (ret) { pr_err("failed to validate PT BOs\n"); return ret; } vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.bo); return 0; } static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync) { struct amdgpu_bo *pd = vm->root.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); int ret; ret = amdgpu_vm_update_pdes(adev, vm, false); if (ret) return ret; return amdgpu_sync_fence(sync, vm->last_update); } static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem) { uint32_t mapping_flags = AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_MTYPE_DEFAULT; if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE) mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE; if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE) mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; return amdgpu_gem_va_map_flags(adev, mapping_flags); } /** * create_sg_table() - Create an sg_table for a contiguous DMA addr range * @addr: The starting address to point to * @size: Size of memory area in bytes being pointed to * * Allocates an instance of sg_table and initializes it to point to memory * area specified by input parameters. The address used to build is assumed * to be DMA mapped, if needed. * * DOORBELL or MMIO BOs use only one scatterlist node in their sg_table * because they are physically contiguous. * * Return: Initialized instance of SG Table or NULL */ static struct sg_table *create_sg_table(uint64_t addr, uint32_t size) { struct sg_table *sg = kmalloc(sizeof(*sg), GFP_KERNEL); if (!sg) return NULL; if (sg_alloc_table(sg, 1, GFP_KERNEL)) { kfree(sg); return NULL; } sg_dma_address(sg->sgl) = addr; sg->sgl->length = size; #ifdef CONFIG_NEED_SG_DMA_LENGTH sg->sgl->dma_length = size; #endif return sg; } static int kfd_mem_dmamap_userptr(struct kgd_mem *mem, struct kfd_mem_attachment *attachment) { enum dma_data_direction direction = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; struct ttm_operation_ctx ctx = {.interruptible = true}; struct amdgpu_bo *bo = attachment->bo_va->base.bo; struct amdgpu_device *adev = attachment->adev; struct ttm_tt *src_ttm = mem->bo->tbo.ttm; struct ttm_tt *ttm = bo->tbo.ttm; int ret; if (WARN_ON(ttm->num_pages != src_ttm->num_pages)) return -EINVAL; ttm->sg = kmalloc(sizeof(*ttm->sg), GFP_KERNEL); if (unlikely(!ttm->sg)) return -ENOMEM; /* Same sequence as in amdgpu_ttm_tt_pin_userptr */ ret = sg_alloc_table_from_pages(ttm->sg, src_ttm->pages, ttm->num_pages, 0, (u64)ttm->num_pages << PAGE_SHIFT, GFP_KERNEL); if (unlikely(ret)) goto free_sg; ret = dma_map_sgtable(adev->dev, ttm->sg, direction, 0); if (unlikely(ret)) goto release_sg; drm_prime_sg_to_dma_addr_array(ttm->sg, ttm->dma_address, ttm->num_pages); amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (ret) goto unmap_sg; return 0; unmap_sg: dma_unmap_sgtable(adev->dev, ttm->sg, direction, 0); release_sg: pr_err("DMA map userptr failed: %d\n", ret); sg_free_table(ttm->sg); free_sg: kfree(ttm->sg); ttm->sg = NULL;