/* * Copyright 2014-2018 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */#include <linux/dma-buf.h>#include <linux/list.h>#include <linux/pagemap.h>#include <linux/sched/mm.h>#include <linux/sched/task.h> #include "amdgpu_object.h"#include "amdgpu_gem.h"#include "amdgpu_vm.h"#include "amdgpu_amdkfd.h"#include "amdgpu_dma_buf.h"#include <uapi/linux/kfd_ioctl.h>#include "amdgpu_xgmi.h" /* BO flag to indicate a KFD userptr BO */#define AMDGPU_AMDKFD_USERPTR_BO (1ULL << 63) /* Userptr restore delay, just long enough to allow consecutive VM * changes to accumulate */#define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 /* Impose limit on how much memory KFD can use */static struct { uint64_t max_system_mem_limit; uint64_t max_ttm_mem_limit; int64_t system_mem_used; int64_t ttm_mem_used; spinlock_t mem_limit_lock;} kfd_mem_limit; /* Struct used for amdgpu_amdkfd_bo_validate */struct amdgpu_vm_parser { uint32_t domain; bool wait;}; static const char * const domain_bit_to_string[] = { "CPU", "GTT", "VRAM", "GDS", "GWS", "OA"}; #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd){ return (struct amdgpu_device *)kgd;} static bool check_if_add_bo_to_vm(struct amdgpu_vm *avm, struct kgd_mem *mem){ struct kfd_bo_va_list *entry; list_for_each_entry(entry, &mem->bo_va_list, bo_list) if (entry->bo_va->base.vm == avm) return false; return true;} /* Set memory usage limits. Current, limits are * System (TTM + userptr) memory - 15/16th System RAM * TTM memory - 3/8th System RAM */void amdgpu_amdkfd_gpuvm_init_mem_limits(void){ struct sysinfo si; uint64_t mem; si_meminfo(&si); mem = si.freeram - si.freehigh; mem *= si.mem_unit; spin_lock_init(&kfd_mem_limit.mem_limit_lock); kfd_mem_limit.max_system_mem_limit = mem - (mem >> 4); kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3); pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", (kfd_mem_limit.max_system_mem_limit >> 20), (kfd_mem_limit.max_ttm_mem_limit >> 20));} /* Estimate page table size needed to represent a given memory size * * With 4KB pages, we need one 8 byte PTE for each 4KB of memory * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB * of memory (factor 256K, >> 18). ROCm user mode tries to optimize * for 2MB pages for TLB efficiency. However, small allocations and * fragmented system memory still need some 4KB pages. We choose a * compromise that should work in most cases without reserving too * much memory for page tables unnecessarily (factor 16K, >> 14). */#define ESTIMATE_PT_SIZE(mem_size) ((mem_size) >> 14) static size_t amdgpu_amdkfd_acc_size(uint64_t size){ size >>= PAGE_SHIFT; size *= sizeof(dma_addr_t) + sizeof(void *); return __roundup_pow_of_two(sizeof(struct amdgpu_bo)) + __roundup_pow_of_two(sizeof(struct ttm_tt)) + PAGE_ALIGN(size);} static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 domain, bool sg){ uint64_t reserved_for_pt = ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); size_t acc_size, system_mem_needed, ttm_mem_needed, vram_needed; int ret = 0; acc_size = amdgpu_amdkfd_acc_size(size); vram_needed = 0; if (domain == AMDGPU_GEM_DOMAIN_GTT) { /* TTM GTT memory */ system_mem_needed = acc_size + size; ttm_mem_needed = acc_size + size; } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) { /* Userptr */ system_mem_needed = acc_size + size; ttm_mem_needed = acc_size; } else { /* VRAM and SG */ system_mem_needed = acc_size; ttm_mem_needed = acc_size; if (domain == AMDGPU_GEM_DOMAIN_VRAM) vram_needed = size; } spin_lock(&kfd_mem_limit.mem_limit_lock); if (kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit) pr_debug("Set no_system_mem_limit=1 if using shared memory\n"); if ((kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) || (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > kfd_mem_limit.max_ttm_mem_limit) || (adev->kfd.vram_used + vram_needed > adev->gmc.real_vram_size - reserved_for_pt)) { ret = -ENOMEM; } else { kfd_mem_limit.system_mem_used += system_mem_needed; kfd_mem_limit.ttm_mem_used += ttm_mem_needed; adev->kfd.vram_used += vram_needed; } spin_unlock(&kfd_mem_limit.mem_limit_lock); return ret;} static void unreserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 domain, bool sg){ size_t acc_size; acc_size = amdgpu_amdkfd_acc_size(size); spin_lock(&kfd_mem_limit.mem_limit_lock); if (domain == AMDGPU_GEM_DOMAIN_GTT) { kfd_mem_limit.system_mem_used -= (acc_size + size); kfd_mem_limit.ttm_mem_used -= (acc_size + size); } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) { kfd_mem_limit.system_mem_used -= (acc_size + size); kfd_mem_limit.ttm_mem_used -= acc_size; } else { kfd_mem_limit.system_mem_used -= acc_size; kfd_mem_limit.ttm_mem_used -= acc_size; if (domain == AMDGPU_GEM_DOMAIN_VRAM) { adev->kfd.vram_used -= size; WARN_ONCE(adev->kfd.vram_used < 0, "kfd VRAM memory accounting unbalanced"); } } WARN_ONCE(kfd_mem_limit.system_mem_used < 0, "kfd system memory accounting unbalanced"); WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, "kfd TTM memory accounting unbalanced"); spin_unlock(&kfd_mem_limit.mem_limit_lock);} void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo){ struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); u32 domain = bo->preferred_domains; bool sg = (bo->preferred_domains == AMDGPU_GEM_DOMAIN_CPU); if (bo->flags & AMDGPU_AMDKFD_USERPTR_BO) { domain = AMDGPU_GEM_DOMAIN_CPU; sg = false; } unreserve_mem_limit(adev, amdgpu_bo_size(bo), domain, sg);} /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's * reservation object. * * @bo: [IN] Remove eviction fence(s) from this BO * @ef: [IN] This eviction fence is removed if it * is present in the shared list. * * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. */static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, struct amdgpu_amdkfd_fence *ef){ struct dma_resv *resv = bo->tbo.base.resv; struct dma_resv_list *old, *new; unsigned int i, j, k; if (!ef) return -EINVAL; old = dma_resv_get_list(resv); if (!old) return 0; new = kmalloc(struct_size(new, shared, old->shared_max), GFP_KERNEL); if (!new) return -ENOMEM; /* Go through all the shared fences in the resevation object and sort * the interesting ones to the end of the list. */ for (i = 0, j = old->shared_count, k = 0; i < old->shared_count; ++i) { struct dma_fence *f; f = rcu_dereference_protected(old->shared[i], dma_resv_held(resv)); if (f->context == ef->base.context) RCU_INIT_POINTER(new->shared[--j], f); else RCU_INIT_POINTER(new->shared[k++], f); } new->shared_max = old->shared_max; new->shared_count = k; /* Install the new fence list, seqcount provides the barriers */ write_seqcount_begin(&resv->seq); RCU_INIT_POINTER(resv->fence, new); write_seqcount_end(&resv->seq); /* Drop the references to the removed fences or move them to ef_list */ for (i = j, k = 0; i < old->shared_count; ++i) { struct dma_fence *f; f = rcu_dereference_protected(new->shared[i], dma_resv_held(resv)); dma_fence_put(f); } kfree_rcu(old, rcu); return 0;} int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo){ struct amdgpu_bo *root = bo; struct amdgpu_vm_bo_base *vm_bo; struct amdgpu_vm *vm; struct amdkfd_process_info *info; struct amdgpu_amdkfd_fence *ef; int ret; /* we can always get vm_bo from root PD bo.*/ while (root->parent) root = root->parent; vm_bo = root->vm_bo; if (!vm_bo) return 0; vm = vm_bo->vm; if (!vm) return 0; info = vm->process_info; if (!info || !info->eviction_fence) return 0; ef = container_of(dma_fence_get(&info->eviction_fence->base), struct amdgpu_amdkfd_fence, base); BUG_ON(!dma_resv_trylock(bo->tbo.base.resv)); ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef); dma_resv_unlock(bo->tbo.base.resv); dma_fence_put(&ef->base); return ret;} static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, bool wait){ struct ttm_operation_ctx ctx = { false, false }; int ret; if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), "Called with userptr BO")) return -EINVAL; amdgpu_bo_placement_from_domain(bo, domain); ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (ret) goto validate_fail; if (wait) amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); validate_fail: return ret;} static int amdgpu_amdkfd_validate(void *param, struct amdgpu_bo *bo){ struct amdgpu_vm_parser *p = param; return amdgpu_amdkfd_bo_validate(bo, p->domain, p->wait);} /* vm_validate_pt_pd_bos - Validate page table and directory BOs * * Page directories are not updated here because huge page handling * during page table updates can invalidate page directory entries * again. Page directories are only updated after updating page * tables. */static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm){ struct amdgpu_bo *pd = vm->root.base.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); struct amdgpu_vm_parser param; int ret; param.domain = AMDGPU_GEM_DOMAIN_VRAM; param.wait = false; ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate, ¶m); if (ret) { pr_err("failed to validate PT BOs\n"); return ret; } ret = amdgpu_amdkfd_validate(¶m, pd); if (ret) { pr_err("failed to validate PD\n"); return ret; } vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.base.bo); if (vm->use_cpu_for_update) { ret = amdgpu_bo_kmap(pd, NULL); if (ret) { pr_err("failed to kmap PD, ret=%d\n", ret); return ret; } } return 0;} static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync){ struct amdgpu_bo *pd = vm->root.base.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); int ret; ret = amdgpu_vm_update_pdes(adev, vm, false); if (ret) return ret; return amdgpu_sync_fence(sync, vm->last_update);} static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem){ struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); bool coherent = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; bool uncached = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; uint32_t mapping_flags; uint64_t pte_flags; bool snoop = false; mapping_flags = AMDGPU_VM_PAGE_READABLE; if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE) mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE; if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE) mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; switch (adev->asic_type) { case CHIP_ARCTURUS: if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { if (bo_adev == adev) mapping_flags |= coherent ? AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW; else mapping_flags |= AMDGPU_VM_MTYPE_UC; } else { mapping_flags |= coherent ? AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; } break; case CHIP_ALDEBARAN: if (coherent && uncached) { if (adev->gmc.xgmi.connected_to_cpu || !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) snoop = true; mapping_flags |= AMDGPU_VM_MTYPE_UC; } else if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { if (bo_adev == adev) { mapping_flags |= coherent ? AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW; if (adev->gmc.xgmi.connected_to_cpu) snoop = true; } else { mapping_flags |= AMDGPU_VM_MTYPE_UC; if (amdgpu_xgmi_same_hive(adev, bo_adev)) snoop = true; } } else { snoop = true; mapping_flags |= coherent ? AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; } break; default: mapping_flags |= coherent ? AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; } pte_flags = amdgpu_gem_va_map_flags(adev, mapping_flags); pte_flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; return pte_flags;} /* add_bo_to_vm - Add a BO to a VM * * Everything that needs to bo done only once when a BO is first added * to a VM. It can later be mapped and unmapped many times without * repeating these steps. * * 1. Allocate and initialize BO VA entry data structure * 2. Add BO to the VM * 3. Determine ASIC-specific PTE flags * 4. Alloc page tables and directories if needed * 4a. Validate new page tables and directories */static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, struct amdgpu_vm *vm, bool is_aql, struct kfd_bo_va_list **p_bo_va_entry){ int ret; struct kfd_bo_va_list *bo_va_entry; struct amdgpu_bo *bo = mem->bo; uint64_t va = mem->va; struct list_head *list_bo_va = &mem->bo_va_list; unsigned long bo_size = bo->tbo.base.size; if (!va) { pr_err("Invalid VA when adding BO to VM\n"); return -EINVAL; }Lines 1–500 of 2281. Load more
/* * Copyright 2014-2018 Advanced Micro Devices, Inc. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ #include <linux/dma-buf.h> #include <linux/list.h> #include <linux/pagemap.h> #include <linux/sched/mm.h> #include <linux/sched/task.h> #include "amdgpu_object.h" #include "amdgpu_gem.h" #include "amdgpu_vm.h" #include "amdgpu_amdkfd.h" #include "amdgpu_dma_buf.h" #include <uapi/linux/kfd_ioctl.h> #include "amdgpu_xgmi.h" /* BO flag to indicate a KFD userptr BO */ #define AMDGPU_AMDKFD_USERPTR_BO (1ULL << 63) /* Userptr restore delay, just long enough to allow consecutive VM * changes to accumulate */ #define AMDGPU_USERPTR_RESTORE_DELAY_MS 1 /* Impose limit on how much memory KFD can use */ static struct { uint64_t max_system_mem_limit; uint64_t max_ttm_mem_limit; int64_t system_mem_used; int64_t ttm_mem_used; spinlock_t mem_limit_lock; } kfd_mem_limit; /* Struct used for amdgpu_amdkfd_bo_validate */ struct amdgpu_vm_parser { uint32_t domain; bool wait; }; static const char * const domain_bit_to_string[] = { "CPU", "GTT", "VRAM", "GDS", "GWS", "OA" }; #define domain_string(domain) domain_bit_to_string[ffs(domain)-1] static void amdgpu_amdkfd_restore_userptr_worker(struct work_struct *work); static inline struct amdgpu_device *get_amdgpu_device(struct kgd_dev *kgd) { return (struct amdgpu_device *)kgd; } static bool check_if_add_bo_to_vm(struct amdgpu_vm *avm, struct kgd_mem *mem) { struct kfd_bo_va_list *entry; list_for_each_entry(entry, &mem->bo_va_list, bo_list) if (entry->bo_va->base.vm == avm) return false; return true; } /* Set memory usage limits. Current, limits are * System (TTM + userptr) memory - 15/16th System RAM * TTM memory - 3/8th System RAM */ void amdgpu_amdkfd_gpuvm_init_mem_limits(void) { struct sysinfo si; uint64_t mem; si_meminfo(&si); mem = si.freeram - si.freehigh; mem *= si.mem_unit; spin_lock_init(&kfd_mem_limit.mem_limit_lock); kfd_mem_limit.max_system_mem_limit = mem - (mem >> 4); kfd_mem_limit.max_ttm_mem_limit = (mem >> 1) - (mem >> 3); pr_debug("Kernel memory limit %lluM, TTM limit %lluM\n", (kfd_mem_limit.max_system_mem_limit >> 20), (kfd_mem_limit.max_ttm_mem_limit >> 20)); } /* Estimate page table size needed to represent a given memory size * * With 4KB pages, we need one 8 byte PTE for each 4KB of memory * (factor 512, >> 9). With 2MB pages, we need one 8 byte PTE for 2MB * of memory (factor 256K, >> 18). ROCm user mode tries to optimize * for 2MB pages for TLB efficiency. However, small allocations and * fragmented system memory still need some 4KB pages. We choose a * compromise that should work in most cases without reserving too * much memory for page tables unnecessarily (factor 16K, >> 14). */ #define ESTIMATE_PT_SIZE(mem_size) ((mem_size) >> 14) static size_t amdgpu_amdkfd_acc_size(uint64_t size) { size >>= PAGE_SHIFT; size *= sizeof(dma_addr_t) + sizeof(void *); return __roundup_pow_of_two(sizeof(struct amdgpu_bo)) + __roundup_pow_of_two(sizeof(struct ttm_tt)) + PAGE_ALIGN(size); } static int amdgpu_amdkfd_reserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 domain, bool sg) { uint64_t reserved_for_pt = ESTIMATE_PT_SIZE(amdgpu_amdkfd_total_mem_size); size_t acc_size, system_mem_needed, ttm_mem_needed, vram_needed; int ret = 0; acc_size = amdgpu_amdkfd_acc_size(size); vram_needed = 0; if (domain == AMDGPU_GEM_DOMAIN_GTT) { /* TTM GTT memory */ system_mem_needed = acc_size + size; ttm_mem_needed = acc_size + size; } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) { /* Userptr */ system_mem_needed = acc_size + size; ttm_mem_needed = acc_size; } else { /* VRAM and SG */ system_mem_needed = acc_size; ttm_mem_needed = acc_size; if (domain == AMDGPU_GEM_DOMAIN_VRAM) vram_needed = size; } spin_lock(&kfd_mem_limit.mem_limit_lock); if (kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit) pr_debug("Set no_system_mem_limit=1 if using shared memory\n"); if ((kfd_mem_limit.system_mem_used + system_mem_needed > kfd_mem_limit.max_system_mem_limit && !no_system_mem_limit) || (kfd_mem_limit.ttm_mem_used + ttm_mem_needed > kfd_mem_limit.max_ttm_mem_limit) || (adev->kfd.vram_used + vram_needed > adev->gmc.real_vram_size - reserved_for_pt)) { ret = -ENOMEM; } else { kfd_mem_limit.system_mem_used += system_mem_needed; kfd_mem_limit.ttm_mem_used += ttm_mem_needed; adev->kfd.vram_used += vram_needed; } spin_unlock(&kfd_mem_limit.mem_limit_lock); return ret; } static void unreserve_mem_limit(struct amdgpu_device *adev, uint64_t size, u32 domain, bool sg) { size_t acc_size; acc_size = amdgpu_amdkfd_acc_size(size); spin_lock(&kfd_mem_limit.mem_limit_lock); if (domain == AMDGPU_GEM_DOMAIN_GTT) { kfd_mem_limit.system_mem_used -= (acc_size + size); kfd_mem_limit.ttm_mem_used -= (acc_size + size); } else if (domain == AMDGPU_GEM_DOMAIN_CPU && !sg) { kfd_mem_limit.system_mem_used -= (acc_size + size); kfd_mem_limit.ttm_mem_used -= acc_size; } else { kfd_mem_limit.system_mem_used -= acc_size; kfd_mem_limit.ttm_mem_used -= acc_size; if (domain == AMDGPU_GEM_DOMAIN_VRAM) { adev->kfd.vram_used -= size; WARN_ONCE(adev->kfd.vram_used < 0, "kfd VRAM memory accounting unbalanced"); } } WARN_ONCE(kfd_mem_limit.system_mem_used < 0, "kfd system memory accounting unbalanced"); WARN_ONCE(kfd_mem_limit.ttm_mem_used < 0, "kfd TTM memory accounting unbalanced"); spin_unlock(&kfd_mem_limit.mem_limit_lock); } void amdgpu_amdkfd_unreserve_memory_limit(struct amdgpu_bo *bo) { struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); u32 domain = bo->preferred_domains; bool sg = (bo->preferred_domains == AMDGPU_GEM_DOMAIN_CPU); if (bo->flags & AMDGPU_AMDKFD_USERPTR_BO) { domain = AMDGPU_GEM_DOMAIN_CPU; sg = false; } unreserve_mem_limit(adev, amdgpu_bo_size(bo), domain, sg); } /* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence from BO's * reservation object. * * @bo: [IN] Remove eviction fence(s) from this BO * @ef: [IN] This eviction fence is removed if it * is present in the shared list. * * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held. */ static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo, struct amdgpu_amdkfd_fence *ef) { struct dma_resv *resv = bo->tbo.base.resv; struct dma_resv_list *old, *new; unsigned int i, j, k; if (!ef) return -EINVAL; old = dma_resv_get_list(resv); if (!old) return 0; new = kmalloc(struct_size(new, shared, old->shared_max), GFP_KERNEL); if (!new) return -ENOMEM; /* Go through all the shared fences in the resevation object and sort * the interesting ones to the end of the list. */ for (i = 0, j = old->shared_count, k = 0; i < old->shared_count; ++i) { struct dma_fence *f; f = rcu_dereference_protected(old->shared[i], dma_resv_held(resv)); if (f->context == ef->base.context) RCU_INIT_POINTER(new->shared[--j], f); else RCU_INIT_POINTER(new->shared[k++], f); } new->shared_max = old->shared_max; new->shared_count = k; /* Install the new fence list, seqcount provides the barriers */ write_seqcount_begin(&resv->seq); RCU_INIT_POINTER(resv->fence, new); write_seqcount_end(&resv->seq); /* Drop the references to the removed fences or move them to ef_list */ for (i = j, k = 0; i < old->shared_count; ++i) { struct dma_fence *f; f = rcu_dereference_protected(new->shared[i], dma_resv_held(resv)); dma_fence_put(f); } kfree_rcu(old, rcu); return 0; } int amdgpu_amdkfd_remove_fence_on_pt_pd_bos(struct amdgpu_bo *bo) { struct amdgpu_bo *root = bo; struct amdgpu_vm_bo_base *vm_bo; struct amdgpu_vm *vm; struct amdkfd_process_info *info; struct amdgpu_amdkfd_fence *ef; int ret; /* we can always get vm_bo from root PD bo.*/ while (root->parent) root = root->parent; vm_bo = root->vm_bo; if (!vm_bo) return 0; vm = vm_bo->vm; if (!vm) return 0; info = vm->process_info; if (!info || !info->eviction_fence) return 0; ef = container_of(dma_fence_get(&info->eviction_fence->base), struct amdgpu_amdkfd_fence, base); BUG_ON(!dma_resv_trylock(bo->tbo.base.resv)); ret = amdgpu_amdkfd_remove_eviction_fence(bo, ef); dma_resv_unlock(bo->tbo.base.resv); dma_fence_put(&ef->base); return ret; } static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain, bool wait) { struct ttm_operation_ctx ctx = { false, false }; int ret; if (WARN(amdgpu_ttm_tt_get_usermm(bo->tbo.ttm), "Called with userptr BO")) return -EINVAL; amdgpu_bo_placement_from_domain(bo, domain); ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); if (ret) goto validate_fail; if (wait) amdgpu_bo_sync_wait(bo, AMDGPU_FENCE_OWNER_KFD, false); validate_fail: return ret; } static int amdgpu_amdkfd_validate(void *param, struct amdgpu_bo *bo) { struct amdgpu_vm_parser *p = param; return amdgpu_amdkfd_bo_validate(bo, p->domain, p->wait); } /* vm_validate_pt_pd_bos - Validate page table and directory BOs * * Page directories are not updated here because huge page handling * during page table updates can invalidate page directory entries * again. Page directories are only updated after updating page * tables. */ static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm) { struct amdgpu_bo *pd = vm->root.base.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); struct amdgpu_vm_parser param; int ret; param.domain = AMDGPU_GEM_DOMAIN_VRAM; param.wait = false; ret = amdgpu_vm_validate_pt_bos(adev, vm, amdgpu_amdkfd_validate, ¶m); if (ret) { pr_err("failed to validate PT BOs\n"); return ret; } ret = amdgpu_amdkfd_validate(¶m, pd); if (ret) { pr_err("failed to validate PD\n"); return ret; } vm->pd_phys_addr = amdgpu_gmc_pd_addr(vm->root.base.bo); if (vm->use_cpu_for_update) { ret = amdgpu_bo_kmap(pd, NULL); if (ret) { pr_err("failed to kmap PD, ret=%d\n", ret); return ret; } } return 0; } static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync) { struct amdgpu_bo *pd = vm->root.base.bo; struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev); int ret; ret = amdgpu_vm_update_pdes(adev, vm, false); if (ret) return ret; return amdgpu_sync_fence(sync, vm->last_update); } static uint64_t get_pte_flags(struct amdgpu_device *adev, struct kgd_mem *mem) { struct amdgpu_device *bo_adev = amdgpu_ttm_adev(mem->bo->tbo.bdev); bool coherent = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_COHERENT; bool uncached = mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED; uint32_t mapping_flags; uint64_t pte_flags; bool snoop = false; mapping_flags = AMDGPU_VM_PAGE_READABLE; if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE) mapping_flags |= AMDGPU_VM_PAGE_WRITEABLE; if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE) mapping_flags |= AMDGPU_VM_PAGE_EXECUTABLE; switch (adev->asic_type) { case CHIP_ARCTURUS: if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { if (bo_adev == adev) mapping_flags |= coherent ? AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW; else mapping_flags |= AMDGPU_VM_MTYPE_UC; } else { mapping_flags |= coherent ? AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; } break; case CHIP_ALDEBARAN: if (coherent && uncached) { if (adev->gmc.xgmi.connected_to_cpu || !(mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM)) snoop = true; mapping_flags |= AMDGPU_VM_MTYPE_UC; } else if (mem->alloc_flags & KFD_IOC_ALLOC_MEM_FLAGS_VRAM) { if (bo_adev == adev) { mapping_flags |= coherent ? AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_RW; if (adev->gmc.xgmi.connected_to_cpu) snoop = true; } else { mapping_flags |= AMDGPU_VM_MTYPE_UC; if (amdgpu_xgmi_same_hive(adev, bo_adev)) snoop = true; } } else { snoop = true; mapping_flags |= coherent ? AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; } break; default: mapping_flags |= coherent ? AMDGPU_VM_MTYPE_UC : AMDGPU_VM_MTYPE_NC; } pte_flags = amdgpu_gem_va_map_flags(adev, mapping_flags); pte_flags |= snoop ? AMDGPU_PTE_SNOOPED : 0; return pte_flags; } /* add_bo_to_vm - Add a BO to a VM * * Everything that needs to bo done only once when a BO is first added * to a VM. It can later be mapped and unmapped many times without * repeating these steps. * * 1. Allocate and initialize BO VA entry data structure * 2. Add BO to the VM * 3. Determine ASIC-specific PTE flags * 4. Alloc page tables and directories if needed * 4a. Validate new page tables and directories */ static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem, struct amdgpu_vm *vm, bool is_aql, struct kfd_bo_va_list **p_bo_va_entry) { int ret; struct kfd_bo_va_list *bo_va_entry; struct amdgpu_bo *bo = mem->bo; uint64_t va = mem->va; struct list_head *list_bo_va = &mem->bo_va_list; unsigned long bo_size = bo->tbo.base.size; if (!va) { pr_err("Invalid VA when adding BO to VM\n"); return -EINVAL; }